ArchPOWER/packages
1
0
Fork 0
Code Issues 8 Pull Requests 1 Actions Packages Projects Releases 18 Wiki Activity
packages/js128/823088.diff
kth5 c2db4a51cb * add js128
2024-10-25 10:59:22 +02:00

22302 lines
726 KiB
Diff
Raw Blame History

# HG changeset patch
# User Cameron Kaiser <spectre@floodgap.com>
# Date 1723177432 25200
# Thu Aug 08 21:23:52 2024 -0700
# Node ID 9c245f4665be241b51f2b8b02ba90a054f55252f
# Parent 485b15bb4a20860aaa2d58a028fd09bc7b9638e0
ppc64 Baseline JIT
diff -r 485b15bb4a20 -r 9c245f4665be js/src/jit/ppc64/Architecture-ppc64.cpp
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/js/src/jit/ppc64/Architecture-ppc64.cpp Thu Aug 08 21:23:52 2024 -0700
@@ -0,0 +1,141 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/ppc64/Architecture-ppc64.h"
+
+#include <fcntl.h>
+#include <unistd.h>
+
+#include "jit/RegisterSets.h"
+
+#if defined(XP_LINUX)
+// Use the getauxval() function if available.
+// ARCH_3_00 wasn't defined until glibc 2.23, so include just in case.
+# include <sys/auxv.h>
+# ifndef PPC_FEATURE2_ARCH_3_00
+# define PPC_FEATURE2_ARCH_3_00 0x00800000
+# endif
+#endif
+
+namespace js {
+namespace jit {
+
+// Don't bother checking if the JIT is statically enabled (ISA 3.0+ on LE).
+#if !defined(__POWER9_VECTOR__) || !defined(__LITTLE_ENDIAN__)
+bool
+HasPPCISA3() {
+ // This is duplicated from mozglue/build/ppc.* since mozglue for some
+ // reason can't be used here.
+
+ // The operation could be expensive, so cache the result.
+ static bool checked = false;
+ static bool result = false;
+
+ if (checked) {
+ return result;
+ }
+
+ checked = true;
+#if defined(XP_LINUX)
+ // Try getauxval().
+ unsigned long int cap = getauxval(AT_HWCAP);
+ unsigned long int cap2 = getauxval(AT_HWCAP2);
+
+ if ((cap & PPC_FEATURE_HAS_ALTIVEC) &&
+ (cap & PPC_FEATURE_HAS_VSX) &&
+ (cap2 & PPC_FEATURE2_ARCH_3_00)) {
+ result = true;
+ }
+#else
+ // Non-Linux detection here. Currently, on systems other than Linux,
+ // no CPU features will be detected.
+#endif
+
+ return result;
+}
+#endif
+
+void
+FlushICache(void* code, size_t size) {
+#if defined(__GNUC__)
+ intptr_t end = reinterpret_cast<intptr_t>(code) + size;
+ __builtin___clear_cache(reinterpret_cast<char*>(code),
+ reinterpret_cast<char*>(end));
+#else
+ _flush_cache(reinterpret_cast<char*>(code), size, BCACHE);
+#endif
+}
+
+// We do it on demand because we're service-oriented. Flags as a Service.
+bool CPUFlagsHaveBeenComputed() { return true; }
+
+Registers::Code
+Registers::FromName(const char *name)
+{
+ // Check for some register aliases first.
+ if (strcmp(name, "sp")==0 || strcmp(name, "r1")== 0)
+ return Code(1);
+ if (strcmp(name, "r12") == 0)
+ return Code(12);
+ if (strcmp(name, "r3") == 0)
+ return Code(3); // Dispatch, this is Floodgap, Code 3. Over.
+
+ for (uint32_t i = 0; i < Total; i++) {
+ if (strcmp(GetName(i), name) == 0)
+ return Code(i);
+ }
+
+ return Invalid;
+}
+
+FloatRegisters::Code
+FloatRegisters::FromName(const char *name)
+{
+ for (size_t i = 0; i < TotalPhys; i++) { // no alternate names
+ if (strcmp(GetName(i), name) == 0)
+ return Code(i); // thus double
+ }
+
+ return Invalid;
+}
+
+FloatRegisterSet FloatRegister::ReduceSetForPush(const FloatRegisterSet& s) {
+#ifdef ENABLE_WASM_SIMD
+# error "Needs more careful logic if SIMD is enabled"
+#endif
+
+ LiveFloatRegisterSet mod;
+ for (FloatRegisterIterator iter(s); iter.more(); ++iter) {
+ if ((*iter).isSingle()) {
+ // Even for floats, save a full double.
+ mod.addUnchecked((*iter).asDouble());
+ } else {
+ mod.addUnchecked(*iter);
+ }
+ }
+ return mod.set();
+}
+
+uint32_t FloatRegister::GetPushSizeInBytes(const FloatRegisterSet& s) {
+#ifdef ENABLE_WASM_SIMD
+# error "Needs more careful logic if SIMD is enabled"
+#endif
+
+ FloatRegisterSet ss = s.reduceSetForPush();
+ uint64_t bits = ss.bits();
+ // We only push double registers.
+ MOZ_ASSERT((bits & 0xffffffff00000000) == 0);
+ uint32_t ret = mozilla::CountPopulation32(bits) * sizeof(double);
+ return ret;
+}
+
+uint32_t FloatRegister::getRegisterDumpOffsetInBytes() {
+ return encoding() * sizeof(double);
+}
+
+} // namespace ion
+} // namespace js
+
diff -r 485b15bb4a20 -r 9c245f4665be js/src/jit/ppc64/Architecture-ppc64.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/js/src/jit/ppc64/Architecture-ppc64.h Thu Aug 08 21:23:52 2024 -0700
@@ -0,0 +1,645 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_ppc_Architecture_ppc_h
+#define jit_ppc_Architecture_ppc_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/MathAlgorithms.h"
+
+#include "jit/shared/Architecture-shared.h"
+
+#include "js/Utility.h"
+
+namespace js {
+namespace jit {
+
+// Despite my hopes, this does not help protect Wasm Frames from ABI callouts
+// unknowingly stomping on them expecting a regular linkage area; the "shadow
+// stack space" that this allocates is actually allocated at *higher* addresses
+// than the Frame. The Frame demands to be on top of the stack, but that's
+// exactly where the linkage area is supposed to go, and everything assumes
+// that the Frame will be exactly two pointers in size which defeats my earlier
+// attempt to just add the linkage area to the Frame. (On top of that, Wasm GC
+// won't let you nab more than 32 bytes anyway, the bare minimum space required
+// for simply LR, TOC, CR and SP, and includes no parameter area.) Instead, for
+// now we have to tediously pull down dummy frames on demand when calling out
+// to heavy functions that are ABI-compliant. This also does nothing for the
+// regular JIT, where periodically we need to do the same thing.
+//
+// See also MacroAssembler::call(wasm::SymbolicAddress) in
+// MacroAssembler-ppc64.cpp.
+static const uint32_t ShadowStackSpace = 0;
+
+// The return address is in LR, not in memory/stack.
+static const uint32_t SizeOfReturnAddressAfterCall = 0u;
+
+// Size of each bailout table entry.
+// For Power ISA this is a single bl.
+static const uint32_t BAILOUT_TABLE_ENTRY_SIZE = sizeof(void *);
+
+// Range of an immediate jump (26 bit jumps). Take a fudge out in case.
+static constexpr uint32_t JumpImmediateRange = (32 * 1024 * 1024) - 32;
+
+// GPRs.
+class Registers
+{
+ public:
+ enum RegisterID {
+ r0 = 0,
+ tempRegister = r0,
+ r1,
+ sp = r1,
+ stackPointerRegister = r1,
+ r2,
+ r3,
+ r4,
+ r5,
+ r6,
+ r7,
+ r8,
+ r9,
+ r10,
+ r11,
+ r12,
+ addressTempRegister = r12,
+ r13,
+ r14,
+ r15,
+ r16,
+ r17,
+ r18,
+ r19,
+ r20,
+ r21,
+ r22,
+ r23,
+ r24,
+ r25,
+ r26,
+ r27,
+ r28,
+ r29,
+ r30,
+ r31,
+ invalid_reg
+ };
+
+ typedef uint8_t Code;
+ typedef uint32_t Encoding;
+ typedef uint32_t SetType;
+
+ // Content spilled during bailouts.
+ union RegisterContent {
+ uintptr_t r;
+ };
+
+ static const char *GetName(Code code) {
+ static const char *Names[] = {
+ "r0", "sp", "toc", "r3", "r4", "r5", "r6", "r7",
+ "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15",
+ "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23",
+ "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31"};
+ return Names[code];
+ }
+ static const char *GetName(uint32_t i) {
+ MOZ_ASSERT(i < Total);
+ return GetName(Code(i));
+ }
+
+ static Code FromName(const char *name);
+
+ static const Encoding StackPointer = sp;
+ static const Encoding Invalid = invalid_reg;
+
+ // XXX: Currently Safepoints restricts us to a uint32_t-sized non-FPR
+ // mask, so we can't work SPRs into this yet.
+ static const uint32_t Total = 32;
+ static const uint32_t Allocatable = 23;
+
+ static const SetType AllMask = 0xffffffff;
+ static const SetType ArgRegMask =
+ (1 << Registers::r3) |
+ (1 << Registers::r4) |
+ (1 << Registers::r5) |
+ (1 << Registers::r6) |
+ (1 << Registers::r7) |
+ (1 << Registers::r8) |
+ (1 << Registers::r9) |
+ (1 << Registers::r10);
+
+ // We use this constant to save registers when entering functions.
+ // Don't bother saving r0, r1, r11 or r12.
+ static const SetType VolatileMask = ArgRegMask;
+
+ static const SetType NonVolatileMask = (
+ (1 << Registers::r2) |
+ (1 << Registers::r13) |
+ (1 << Registers::r14) |
+ (1 << Registers::r15) |
+ (1 << Registers::r16) |
+ (1 << Registers::r17) |
+ (1 << Registers::r18) |
+ (1 << Registers::r19) |
+ (1 << Registers::r20) |
+ (1 << Registers::r21) |
+ (1 << Registers::r22) |
+ (1 << Registers::r23) |
+ (1 << Registers::r24) |
+ (1 << Registers::r25) |
+ (1 << Registers::r26) |
+ (1 << Registers::r27) |
+ (1 << Registers::r28) |
+ (1 << Registers::r29) |
+ (1 << Registers::r30) |
+ (1 << Registers::r31)
+ // Watch out for sign extension if this is ever 64-bit!
+ ) & AllMask;
+
+ // Also uses r11.
+ static const SetType WrapperMask = VolatileMask;
+
+ static const SetType NonAllocatableMask =
+ // Used by assembler.
+ (1 << Registers::r0) |
+ (1 << Registers::sp) |
+ (1 << Registers::r2) |
+ // Temp registers.
+ (1 << Registers::r11) |
+ (1 << Registers::r12) |
+ // r13 is the pointer for TLS in ELF v2.
+ (1 << Registers::r13) |
+ // Non-volatile work registers.
+ (1 << Registers::r16) |
+ // r17 is the InterpreterPCReg and must be allocatable.
+ // r18 is the WasmTlsReg and must be allocatable.
+ // Despite its use as a rectifier, r19 must be allocatable (see
+ // ICCallScriptedCompiler::generateStubCode).
+ // r28 used by trampoline for stack recovery.
+ (1 << Registers::r28) |
+ // r31 is the Frame Pointer and is not allocatable.
+ (1 << Registers::r31) |
+ 0;
+
+ // Registers that can be allocated without being saved, generally.
+ static const SetType TempMask = VolatileMask & ~NonAllocatableMask;
+
+ // Registers returned from a JS -> JS call.
+ static const SetType JSCallMask =
+ (1 << Registers::r5);
+
+ // Registers returned from a JS -> C call.
+ static const SetType CallMask =
+ (1 << Registers::r3);
+
+ static const SetType AllocatableMask = (
+ // Be explicit
+ (1 << Registers::r3) |
+ (1 << Registers::r4) |
+ (1 << Registers::r5) |
+ (1 << Registers::r6) |
+ (1 << Registers::r7) |
+ (1 << Registers::r8) |
+ (1 << Registers::r9) |
+ (1 << Registers::r10) |
+ (1 << Registers::r14) |
+ (1 << Registers::r15) |
+ (1 << Registers::r17) |
+ (1 << Registers::r18) |
+ (1 << Registers::r19) |
+ (1 << Registers::r20) |
+ (1 << Registers::r21) |
+ (1 << Registers::r22) |
+ (1 << Registers::r23) |
+ (1 << Registers::r24) |
+ //(1 << Registers::r25) |
+ (1 << Registers::r26) |
+ (1 << Registers::r27) |
+ //(1 << Registers::r28) |
+ (1 << Registers::r29) |
+ (1 << Registers::r30) |
+ //(1 << Registers::r31)
+ // Watch out for sign extension!
+ 0) & AllMask;
+
+ static uint32_t SetSize(SetType x) {
+ // XXX: see above
+ static_assert(sizeof(SetType) == 4, "SetType must be 32 bits");
+ return mozilla::CountPopulation32(x);
+ }
+ static uint32_t FirstBit(SetType x) {
+ return mozilla::CountTrailingZeroes32(x);
+ }
+ static uint32_t LastBit(SetType x) {
+ // XXX: see above (31 would be 63 if we used uint64_t)
+ return 31 - mozilla::CountLeadingZeroes32(x);
+ }
+};
+
+// Smallest integer type that can hold a register bitmask. (GPRs.)
+// Safepoints asserts this is 32-bit or smaller.
+typedef uint32_t PackedRegisterMask;
+
+// FPRs.
+// PowerPC FPRs can be both double and single precision, like MIPS. We tell
+// Ion there are 64 FPRs, but each is an aliased pair.
+class FloatRegisters
+{
+ public:
+ enum FPRegisterID {
+ f0 = 0,
+ f1,
+ f2,
+ f3,
+ f4,
+ f5,
+ f6,
+ f7,
+ f8,
+ f9,
+ f10,
+ f11,
+ f12,
+ f13,
+ f14,
+ f15,
+ f16,
+ f17,
+ f18,
+ f19,
+ f20,
+ f21,
+ f22,
+ f23,
+ f24,
+ f25,
+ f26,
+ f27,
+ f28,
+ f29,
+ f30,
+ f31,
+ invalid_freg
+ };
+ static_assert(f31 == 31);
+ // Eight bits: (invalid << 7) | (kind << 5) | encoding
+ typedef uint8_t Code;
+ typedef FPRegisterID Encoding;
+ typedef uint64_t SetType;
+
+ // Make clear that the base type is Double; Single is a veneer upon it. XXX: VSX
+ enum Kind : uint8_t {
+ Double,
+ Single,
+ NumTypes
+ };
+
+ // Content spilled during bailouts.
+ union RegisterContent {
+ float s;
+ double d;
+ };
+ static const Code Invalid = 0x80;
+
+ static const char *GetName(Encoding code) {
+ static const char * const Names[] = { "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7",
+ "f8", "f9", "f10", "f11", "f12", "f13",
+ "f14", "f15", "f16", "f17", "f18", "f19",
+ "f20", "f21", "f22", "f23", "f24", "f25",
+ "f26", "f27", "f28", "f29", "f30", "f31"};
+ MOZ_ASSERT(code < TotalPhys);
+ return Names[code];
+ }
+ static const char *GetName(Code i) {
+ MOZ_ASSERT(i < Total);
+ MOZ_ASSERT(i != Invalid);
+ return GetName(Encoding(i % TotalPhys));
+ }
+
+ static Code FromName(const char *name);
+
+
+ static const uint32_t TotalPhys = 32;
+ static const uint32_t NumScalarTypes = 2;
+ static const uint32_t Total = TotalPhys * NumScalarTypes;
+ static const uint32_t TotalWithSimd = TotalPhys * NumTypes;
+ static const uint32_t Allocatable = 31; // Without f0, the scratch register.
+
+ static_assert(sizeof(SetType) * 8 >= Total,
+ "SetType should be large enough to enumerate all registers.");
+
+ static const SetType SpreadSingle = SetType(1)
+ << (uint32_t(Single) * TotalPhys);
+ static const SetType SpreadDouble = SetType(1)
+ << (uint32_t(Double) * TotalPhys);
+ static const SetType Spread = SpreadSingle | SpreadDouble;
+
+ static const SetType AllPhysMask = (SetType(1) << TotalPhys) - 1;
+ static const SetType AllMask = AllPhysMask * Spread;
+ static const SetType AllDoubleMask = AllPhysMask * SpreadDouble;
+ static const SetType AllSingleMask = AllPhysMask * SpreadSingle;
+ static const SetType NoneMask = SetType(0);
+
+ // f0 is the ScratchFloatReg.
+ static const SetType VolatileMask =
+ SetType((1 << FloatRegisters::f1) | (1 << FloatRegisters::f2) |
+ (1 << FloatRegisters::f3) | (1 << FloatRegisters::f4) |
+ (1 << FloatRegisters::f5) | (1 << FloatRegisters::f6) |
+ (1 << FloatRegisters::f7) | (1 << FloatRegisters::f8) |
+ (1 << FloatRegisters::f9) | (1 << FloatRegisters::f10) |
+ (1 << FloatRegisters::f11) | (1 << FloatRegisters::f12) |
+ (1 << FloatRegisters::f13)) *
+ Spread;
+
+ static const SetType NonVolatileMask = AllMask & ~VolatileMask;
+
+ static const SetType WrapperMask = VolatileMask;
+
+ // d31 is the ScratchFloatReg.
+ static const SetType NonAllocatableMask =
+ (SetType(1) << FloatRegisters::f0) * Spread;
+
+ static const SetType AllocatableMask = AllMask & ~NonAllocatableMask;
+
+ static constexpr Encoding encoding(Code c) {
+ // assert() not available in constexpr function.
+ // assert(c < TotalWithSimd);
+ return Encoding(c & 31);
+ }
+
+ static constexpr Kind kind(Code c) {
+ // assert() not available in constexpr function.
+ // assert(c < TotalWithSimd && ((c >> 5) & 3) < NumTypes);
+ return Kind((c >> 5) & 3);
+ }
+
+ static constexpr Code fromParts(uint32_t encoding, uint32_t kind,
+ uint32_t invalid) {
+ return Code((invalid << 7) | (kind << 5) | encoding);
+ }
+};
+
+template <typename T>
+class TypedRegisterSet;
+
+// Shamelessly rip off aargh, um, aarch64
+struct FloatRegister {
+ typedef FloatRegisters Codes;
+ typedef Codes::Code Code;
+ typedef Codes::Encoding Encoding;
+ typedef Codes::SetType SetType;
+ typedef Codes::Kind Kind;
+
+ static uint32_t SetSize(SetType x) {
+ static_assert(sizeof(SetType) == 8, "SetType must be 64 bits");
+ x |= x >> FloatRegisters::TotalPhys;
+ x &= FloatRegisters::AllPhysMask;
+ return mozilla::CountPopulation32(x);
+ }
+
+ static uint32_t FirstBit(SetType x) {
+ static_assert(sizeof(SetType) == 8, "SetType");
+ return mozilla::CountTrailingZeroes64(x);
+ }
+ static uint32_t LastBit(SetType x) {
+ static_assert(sizeof(SetType) == 8, "SetType");
+ return 63 - mozilla::CountLeadingZeroes64(x);
+ }
+
+ static constexpr size_t SizeOfSimd128 = 16;
+
+ private:
+ // These fields only hold valid values: an invalid register is always
+ // represented as a valid encoding and kind with the invalid_ bit set.
+ uint8_t encoding_; // 32 encodings
+ uint8_t kind_; // Double, Single, Simd128
+ bool invalid_;
+
+ public:
+ constexpr FloatRegister(Encoding encoding, Kind kind = FloatRegisters::Double)
+ : encoding_(encoding), kind_(kind), invalid_(false) {
+ // assert(uint32_t(encoding) < Codes::TotalPhys);
+ }
+ constexpr FloatRegister(uint32_t r, Kind kind = FloatRegisters::Double)
+ : encoding_(Codes::Encoding(r)), kind_(kind), invalid_(false) {}
+ constexpr FloatRegister()
+ : encoding_(0), kind_(FloatRegisters::Double), invalid_(true) {}
+
+ static FloatRegister FromCode(uint32_t i) {
+ MOZ_ASSERT(i < Codes::TotalWithSimd);
+ return FloatRegister(FloatRegisters::encoding(i), FloatRegisters::kind(i));
+ }
+
+ bool isSingle() const {
+ MOZ_ASSERT(!invalid_);
+ return kind_ == FloatRegisters::Single;
+ }
+ bool isDouble() const {
+ MOZ_ASSERT(!invalid_);
+ return kind_ == FloatRegisters::Double;
+ }
+ bool isSimd128() const {
+ MOZ_ASSERT(!invalid_);
+ return false;
+ }
+ bool isInvalid() const { return invalid_; }
+
+ FloatRegister asSingle() const {
+ MOZ_ASSERT(!invalid_);
+ return FloatRegister(Encoding(encoding_), FloatRegisters::Single);
+ }
+ FloatRegister asDouble() const {
+ MOZ_ASSERT(!invalid_);
+ return FloatRegister(Encoding(encoding_), FloatRegisters::Double);
+ }
+ FloatRegister asSimd128() const {
+ MOZ_CRASH("No SIMD support");
+ }
+
+ constexpr uint32_t size() const {
+ MOZ_ASSERT(!invalid_);
+ if (kind_ == FloatRegisters::Double) {
+ return sizeof(double);
+ }
+ if (kind_ == FloatRegisters::Single) {
+ return sizeof(float); // not used in stack calculations, apparently
+ }
+ MOZ_CRASH("No SIMD support");
+ }
+
+ constexpr Code code() const {
+ // assert(!invalid_);
+ return Codes::fromParts(encoding_, kind_, invalid_);
+ }
+
+ constexpr Encoding encoding() const {
+ MOZ_ASSERT(!invalid_);
+ return Encoding(encoding_);
+ }
+
+ const char* name() const { return FloatRegisters::GetName(code()); }
+ bool volatile_() const {
+ MOZ_ASSERT(!invalid_);
+ return !!((SetType(1) << code()) & FloatRegisters::VolatileMask);
+ }
+ constexpr bool operator!=(FloatRegister other) const {
+ return code() != other.code();
+ }
+ constexpr bool operator==(FloatRegister other) const {
+ return code() == other.code();
+ }
+
+ bool aliases(FloatRegister other) const {
+ return other.encoding_ == encoding_;
+ }
+ // All spades are groovy. -- Firesign Theatre
+ bool equiv(FloatRegister other) const {
+ MOZ_ASSERT(!invalid_);
+ return kind_ == other.kind_;
+ }
+
+ // numAliased is used only by Ion's register allocator, ergo we ignore SIMD
+ // registers here as Ion will not be exposed to SIMD on this platform.
+ uint32_t numAliased() const { return Codes::NumScalarTypes; }
+ uint32_t numAlignedAliased() { return numAliased(); }
+
+ FloatRegister aliased(uint32_t aliasIdx) {
+ MOZ_ASSERT(!invalid_);
+ MOZ_ASSERT(aliasIdx < numAliased());
+ return FloatRegister(Encoding(encoding_),
+ Kind((aliasIdx + kind_) % numAliased()));
+ }
+ FloatRegister alignedAliased(uint32_t aliasIdx) {
+ MOZ_ASSERT(aliasIdx < numAliased());
+ return aliased(aliasIdx);
+ }
+ SetType alignedOrDominatedAliasedSet() const {
+ return Codes::Spread << encoding_;
+ }
+
+ static constexpr RegTypeName DefaultType = RegTypeName::Float64;
+
+ template <RegTypeName Name = DefaultType>
+ static SetType LiveAsIndexableSet(SetType s) {
+ return SetType(0);
+ }
+
+ template <RegTypeName Name = DefaultType>
+ static SetType AllocatableAsIndexableSet(SetType s) {
+ static_assert(Name != RegTypeName::Any, "Allocatable set are not iterable");
+ return LiveAsIndexableSet<Name>(s);
+ }
+
+ static TypedRegisterSet<FloatRegister> ReduceSetForPush(
+ const TypedRegisterSet<FloatRegister>& s);
+ static uint32_t GetPushSizeInBytes(const TypedRegisterSet<FloatRegister>& s);
+ uint32_t getRegisterDumpOffsetInBytes();
+};
+
+template <>
+inline FloatRegister::SetType
+FloatRegister::LiveAsIndexableSet<RegTypeName::Float32>(SetType set) {
+ return set & FloatRegisters::AllSingleMask;
+}
+
+template <>
+inline FloatRegister::SetType
+FloatRegister::LiveAsIndexableSet<RegTypeName::Float64>(SetType set) {
+ return set & FloatRegisters::AllDoubleMask;
+}
+
+template <>
+inline FloatRegister::SetType
+FloatRegister::LiveAsIndexableSet<RegTypeName::Any>(SetType set) {
+ return set;
+}
+
+inline bool hasUnaliasedDouble() { return false; }
+inline bool hasMultiAlias() { return false; }
+
+// XXX: This needs to be rolled into somewhere else
+
+// SPRs (PPC backend specific).
+// These have no peer in lesser chips. That is because PPC has no peer in
+// lesser chips. These don't count against the register cap because the
+// allocator is unaware of them. In fact, we don't treat these as regular
+// registers at all (hey, they're Special Purpose anyway).
+#if(0)
+class SPRs // Class definition not yet supported.
+{
+ public:
+#endif
+ enum SPRegisterID {
+ xer = 1,
+ lr_spr = 8, /* I'm in the REAL witness protection program. */
+ ctr = 9,
+ vrsave = 256, /* for future SIMD JS */
+ invalid_spreg
+ };
+#if(0)
+ static const char *getSPRName(SPRegisterID code) {
+#define XXX "INVALID"
+ static const char *N_vrsave = "vrsave";
+ static const char *N_bogus = XXX;
+ static const char *Names[] = {
+ XXX, "xer", XXX, XXX, XXX, XXX, XXX, XXX,
+ "lr","ctr"
+ };
+#undef XXX
+ return
+ (code == vrsave) ? N_vrsave :
+ (code > ctr) ? N_bogus :
+ Names[code];
+ }
+};
+#endif
+
+// CRs (PPC backend specific).
+// We have eight condition registers, each for how unconditionally wonderful
+// PowerPC is, and sometimes for storing condition results.
+// Assume CR0 as default.
+#if(0)
+class CRs // Class definition not yet supported.
+{
+ public:
+#endif
+ enum CRegisterID {
+ cr0 = 0,
+ cr1,
+/*
+Suppress CR2-CR4 because these are non-volatile.
+ cr2,
+ cr3,
+ cr4,
+*/
+ cr5 = 5,
+ cr6,
+ cr7,
+ invalid_creg
+ };
+#if(0)
+ static const char *getCRName(CRegisterID code) {
+ static const char *Names[] = {
+ "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7"
+ };
+ return Names[code];
+ }
+}
+#endif
+
+inline uint32_t GetPPC64Flags() { return 0; /* XXX? */ }
+
+// Statically true if --mcpu=power9.
+#if defined(__POWER9_VECTOR__) && defined(__LITTLE_ENDIAN__)
+inline bool HasPPCISA3() { return true; }
+#else
+bool HasPPCISA3();
+#endif
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_ppc_Architecture_ppc_h */
diff -r 485b15bb4a20 -r 9c245f4665be js/src/jit/ppc64/Assembler-ppc64.cpp
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/js/src/jit/ppc64/Assembler-ppc64.cpp Thu Aug 08 21:23:52 2024 -0700
@@ -0,0 +1,2048 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/ppc64/Assembler-ppc64.h"
+#include "mozilla/DebugOnly.h"
+#include "jit/AutoWritableJitCode.h"
+#include "jit/FlushICache.h"
+
+
+#if DEBUG
+#define spew(...) JitSpew(JitSpew_Codegen, __VA_ARGS__)
+#else
+#define spew(...)
+#endif
+
+using mozilla::DebugOnly;
+
+using namespace js;
+using namespace js::jit;
+
+ABIArgGenerator::ABIArgGenerator()
+ : stackOffset_(0),
+ usedGPRs_(0),
+ usedFPRs_(0),
+ current_()
+{}
+
+// This is used for inter-Wasm procedure calls as well as regular ABI calls,
+// so when we're compiling Wasm we can "expand" the ABI.
+// However, we must not do anything that assumes the presence of an argument
+// area, since our variant Frame doesn't have one.
+
+ABIArg
+ABIArgGenerator::next(MIRType type)
+{
+ switch (type) {
+ case MIRType::Int32:
+ case MIRType::Int64:
+ case MIRType::Pointer:
+ case MIRType::WasmAnyRef:
+ case MIRType::StackResults: {
+ if (usedGPRs_ > 7) {
+ // We only support spilling arguments to the stack with Wasm calls,
+ // but we could be generating Wasm code from the interpreter, so
+ // we can't assume there is a JIT context available.
+ MOZ_ASSERT(!MaybeGetJitContext() || IsCompilingWasm(), "no stack corruption from GPR overflow kthxbye");
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += sizeof(uintptr_t);
+ break;
+ }
+ // Note: we could be passing a full 64-bit quantity as an argument to,
+ // say, uint32_t. We have to compensate for that in other ways when
+ // it makes a difference (see notes in wasm).
+ current_ = ABIArg(Register::FromCode((Register::Code)(usedGPRs_ + 3)));
+ usedGPRs_++;
+ break;
+ }
+ case MIRType::Float32:
+ case MIRType::Double: {
+ if (usedFPRs_ == 12) {
+ MOZ_ASSERT(!MaybeGetJitContext() || IsCompilingWasm(), "no stack corruption from FPR overflow kthxbye");
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += sizeof(double); // keep stack aligned to double
+ break;
+ }
+ current_ = ABIArg(FloatRegister(FloatRegisters::Encoding(usedFPRs_ + 1),
+ type == MIRType::Double ? FloatRegisters::Double : FloatRegisters::Single));
+ usedGPRs_++;
+ usedFPRs_++;
+ break;
+ }
+ default:
+ MOZ_CRASH("Unexpected argument type");
+ }
+ return current_;
+}
+
+uintptr_t
+Assembler::GetPointer(uint8_t* instPtr)
+{
+ Instruction* inst = (Instruction*)instPtr;
+ return Assembler::ExtractLoad64Value(inst);
+}
+
+static JitCode *
+CodeFromJump(Instruction* jump)
+{
+ uint8_t* target = (uint8_t*)Assembler::ExtractLoad64Value(jump);
+ return JitCode::FromExecutable(target);
+}
+
+void
+Assembler::TraceJumpRelocations(JSTracer* trc, JitCode* code, CompactBufferReader& reader)
+{
+ while (reader.more()) {
+ JitCode* child = CodeFromJump((Instruction*)(code->raw() + reader.readUnsigned()));
+ TraceManuallyBarrieredEdge(trc, &child, "rel32");
+ }
+}
+
+#if(0) // old code
+static void
+TraceOneDataRelocation(JSTracer* trc, Instruction* inst)
+{
+ void* ptr = (void*)Assembler::ExtractLoad64Value(inst);
+ void* prior = ptr;
+
+ // All pointers will have the top bits clear. If those bits
+ // are not cleared, this must be a Value.
+ uintptr_t word = reinterpret_cast<uintptr_t>(ptr);
+ if (word >> JSVAL_TAG_SHIFT) {
+ Value v = Value::fromRawBits(word);
+ TraceManuallyBarrieredEdge(trc, &v, "ion-masm-value");
+ ptr = (void*)v.bitsAsPunboxPointer();
+ } else {
+ // No barrier needed since these are constants.
+ TraceManuallyBarrieredGenericPointerEdge(trc, reinterpret_cast<gc::Cell**>(&ptr),
+ "ion-masm-ptr");
+ }
+
+ if (ptr != prior) {
+ Assembler::UpdateLoad64Value(inst, uint64_t(ptr));
+ FlushICache(inst, 5 * sizeof(uint32_t));
+ }
+}
+
+/* static */ void
+Assembler::TraceDataRelocations(JSTracer* trc, JitCode* code, CompactBufferReader& reader)
+{
+ mozilla::Maybe<AutoWritableJitCode> awjc;
+
+ while (reader.more()) {
+ size_t offset = reader.readUnsigned();
+ Instruction* inst = (Instruction*)(code->raw() + offset);
+ if (awjc.isNothing()) {
+ awjc.emplace(code);
+ }
+ TraceOneDataRelocation(trc, inst);
+ }
+}
+#else
+static void TraceOneDataRelocation(JSTracer* trc,
+ mozilla::Maybe<AutoWritableJitCode>& awjc,
+ JitCode* code, Instruction* inst) {
+ void* ptr = (void*)Assembler::ExtractLoad64Value(inst);
+ void* prior = ptr;
+
+ // Data relocations can be for Values or for raw pointers. If a Value is
+ // zero-tagged, we can trace it as if it were a raw pointer. If a Value
+ // is not zero-tagged, we have to interpret it as a Value to ensure that the
+ // tag bits are masked off to recover the actual pointer.
+ uintptr_t word = reinterpret_cast<uintptr_t>(ptr);
+ if (word >> JSVAL_TAG_SHIFT) {
+ // This relocation is a Value with a non-zero tag.
+ Value v = Value::fromRawBits(word);
+ TraceManuallyBarrieredEdge(trc, &v, "jit-masm-value");
+ ptr = (void*)v.bitsAsPunboxPointer();
+ } else {
+ // This relocation is a raw pointer or a Value with a zero tag.
+ // No barrier needed since these are constants.
+ TraceManuallyBarrieredGenericPointerEdge(
+ trc, reinterpret_cast<gc::Cell**>(&ptr), "jit-masm-ptr");
+ }
+
+ if (ptr != prior) {
+ if (awjc.isNothing()) {
+ awjc.emplace(code);
+ }
+ Assembler::UpdateLoad64Value(inst, uint64_t(ptr));
+ FlushICache(inst, 5 * sizeof(uint32_t));
+ }
+}
+
+/* static */
+void Assembler::TraceDataRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader) {
+ mozilla::Maybe<AutoWritableJitCode> awjc;
+ while (reader.more()) {
+ size_t offset = reader.readUnsigned();
+ Instruction* inst = (Instruction*)(code->raw() + offset);
+ TraceOneDataRelocation(trc, awjc, code, inst);
+ }
+}
+#endif
+
+void
+Assembler::Bind(uint8_t* rawCode, const CodeLabel& label)
+{
+ if (label.patchAt().bound()) {
+ auto mode = label.linkMode();
+ intptr_t offset = label.patchAt().offset();
+ intptr_t target = label.target().offset();
+ spew("# Bind mode=%d rawCode=%p offset=%lx target=%lx", (int)mode, rawCode, offset, target);
+
+ if (mode == CodeLabel::RawPointer) {
+ *reinterpret_cast<const void**>(rawCode + offset) = rawCode + target;
+ } else {
+ MOZ_ASSERT(mode == CodeLabel::MoveImmediate || mode == CodeLabel::JumpImmediate);
+ Instruction* inst = (Instruction*) (rawCode + offset);
+ Assembler::UpdateLoad64Value(inst, (uint64_t)(rawCode + target));
+ FlushICache(inst, 5 * sizeof(uint32_t));
+ }
+ }
+}
+
+void
+Assembler::bind(InstImm* inst, uintptr_t b, uintptr_t target, bool bound)
+{
+#if 0 // TODO: Assembler::bind()
+ int64_t offset = target - branch;
+ InstImm inst_bgezal = InstImm(op_regimm, r0, rt_bgezal, BOffImm16(0));
+ InstImm inst_beq = InstImm(PPC_bc, r0, r0, BOffImm16(0));
+
+ // If encoded offset is 4, then the jump must be short
+ if (BOffImm16(inst[0]).decode() == 4) {
+ MOZ_ASSERT(BOffImm16::IsInRange(offset));
+ inst[0].setBOffImm16(BOffImm16(offset));
+ inst[1].makeOp_nop();
+ return;
+ }
+
+ // Generate the long jump for calls because return address has to be the
+ // address after the reserved block.
+ if (inst[0].encode() == inst_bgezal.encode()) {
+ addLongJump(BufferOffset(branch));
+ Assembler::WriteLoad64Instructions(inst, ScratchRegister, LabelBase::INVALID_OFFSET);
+ inst[4] = InstReg(PPC_b | LinkB, ScratchRegister, r0).encode();
+ // There is 1 nop after this.
+ return;
+ }
+
+ if (BOffImm16::IsInRange(offset)) {
+ // Don't skip trailing nops can improve performance
+ // on Loongson3 platform.
+ bool skipNops = (inst[0].encode() != inst_bgezal.encode() &&
+ inst[0].encode() != inst_beq.encode());
+
+ inst[0].setBOffImm16(BOffImm16(offset));
+ inst[1].makeOp_nop();
+
+ if (skipNops) {
+ inst[2] = InstImm(op_regimm, r0, rt_bgez, BOffImm16(5 * sizeof(uint32_t))).encode();
+ // There are 4 nops after this
+ }
+ return;
+ }
+
+ if (inst[0].encode() == inst_beq.encode()) {
+ // Handle long unconditional jump.
+ addLongJump(BufferOffset(branch));
+ Assembler::WriteLoad64Instructions(inst, ScratchRegister, LabelBase::INVALID_OFFSET);
+ inst[4] = InstReg(op_special, ScratchRegister, r0, r0, ff_jr).encode();
+ // There is 1 nop after this.
+ } else {
+ // Handle long conditional jump.
+ inst[0] = invertBranch(inst[0], BOffImm16(7 * sizeof(uint32_t)));
+ // No need for a "nop" here because we can clobber scratch.
+ addLongJump(BufferOffset(branch + sizeof(uint32_t)));
+ Assembler::WriteLoad64Instructions(&inst[1], ScratchRegister, LabelBase::INVALID_OFFSET);
+ inst[5] = InstReg(op_special, ScratchRegister, r0, r0, ff_jr).encode();
+ // There is 1 nop after this.
+ }
+#endif
+ int64_t offset = target - b;
+ spew("# bind %lx %lx (instruction: %08x)", b, target, inst[0].encode());
+ MOZ_ASSERT(!(offset & 3));
+
+ if (inst[0].isOpcode(PPC_addis)) { // pre-existing long stanza
+ spew("# pending long jump");
+ addLongJump(BufferOffset(b), BufferOffset(target));
+ } else if (inst[0].isOpcode(PPC_tw)) { // tagged trap
+ TrapTag t = inst[0].traptag();
+
+ if (t == BCTag) {
+ // Reverse-sense long stanza:
+ // bc inverted, fail
+ // tagged trap << inst[0] patch to lis
+ // .long next_in_chain patch to ori
+ // nop patch to rldicr
+ // nop patch to oris
+ // nop patch to ori
+ // nop patch to mtctr
+ // nop patch to bctr
+
+ // The negative index is required because of an invariant in ::retarget() that
+ // always expects the next-in-chain word in slot 1 (any value there could be
+ // potentially valid, including a trap word, so no sentinel value can
+ // disambiguate).
+ MOZ_ASSERT(inst[-1].isOpcode(PPC_bc));
+ // I always like a clean stanza.
+ MOZ_ASSERT(inst[2].encode() == PPC_nop);
+ MOZ_ASSERT(inst[3].encode() == PPC_nop);
+ MOZ_ASSERT(inst[4].encode() == PPC_nop);
+ MOZ_ASSERT(inst[5].encode() == PPC_nop);
+ MOZ_ASSERT(inst[6].encode() == PPC_nop);
+
+ // If this was actually assigned, see if it's a short jump after all.
+ if (bound && BOffImm16::IsInSignedRange(offset + sizeof(uint32_t))) { // see below
+ // It's a short jump after all.
+ // It's a short jump after all.
+ // It's a short jump after all.
+ // It's a short, short jump.
+ // Patch bc directly by inverting the sense, adding an instruction to
+ // offset the negative index.
+
+ spew("# writing in long jump as short bc");
+ // Make sure we're not going to patch in the wrong place.
+ MOZ_ASSERT(inst[7].encode() != PPC_bctr);
+ // Weirdo instructions like bdnz shouldn't come through here, just any
+ // bc with a BO of 0x04 or 0x0c (i.e., CR bit set or CR bit not set).
+ MOZ_ASSERT((inst[-1].encode() & 0x03e00000) == 0x00800000 ||
+ (inst[-1].encode() & 0x03e00000) == 0x01800000);
+ // XXX: should invert likely bits, too.
+ inst[-1].setData(((inst[-1].encode() ^ 0x01000000) & (0xffff0003)) | BOffImm16(offset+sizeof(uint32_t)).encode());
+ inst[0].setData(PPC_nop); // obliterate tagged trap
+ inst[1].setData(PPC_nop); // obliterate next in chain
+ } else if (bound && JOffImm26::IsInRange(offset)) {
+ // It's a short(er) jump after all.
+ // It's a short(er) jump after all.
+ // It's ... why did you pick up that chainsaw?
+ // Why is it running?
+
+ spew("# writing in long jump as short bc/b");
+ // Make sure we're not going to patch in the wrong place.
+ MOZ_ASSERT(inst[7].encode() != PPC_bctr);
+ inst[0].setData(PPC_b | JOffImm26(offset).encode());
+ inst[1].setData(PPC_nop); // obliterate next in chain
+ } else {
+ // Dang, no Disney songs for this.
+ // Although this should be to Ion code, use r12 to keep calls "as expected."
+
+ spew("# writing in and pending long bc");
+ addLongJump(BufferOffset(b), BufferOffset(target));
+ Assembler::WriteLoad64Instructions(inst, SecondScratchReg, LabelBase::INVALID_OFFSET);
+ inst[5].makeOp_mtctr(SecondScratchReg);
+ inst[6].makeOp_bctr(DontLinkB);
+ }
+ } else if (t == CallTag) {
+ // I wanna taste pizzazz, all the taste clean Stanza has!
+ // I wanna clean, I wanna ... Stan-za.
+ MOZ_ASSERT(inst[2].encode() == PPC_nop);
+ MOZ_ASSERT(inst[3].encode() == PPC_nop);
+ MOZ_ASSERT(inst[4].encode() == PPC_nop);
+ MOZ_ASSERT(inst[5].encode() == PPC_nop);
+ MOZ_ASSERT(inst[6].encode() == PPC_nop);
+
+ // And I get to sing Disney songs again!
+ // See if it's a short jump after all!
+ if (bound && JOffImm26::IsInRange(offset - 24)) {
+ // It's a short jump after all!
+ // It's a short #${{@~NO CARRIER
+ spew("# writing in short call");
+ // Make sure we're not going to patch in the wrong place.
+ MOZ_ASSERT(inst[7].encode() != PPC_bctr);
+ inst[0].setData(PPC_nop); // obliterate trap
+ inst[1].setData(PPC_nop); // obliterate next-in-chain
+ // So that the return is after the stanza, the link call
+ // must be the last instruction in the stanza, not the
+ // first. This means we also need to adjust the offset,
+ // which is where the 24 comes from (stanza length minus
+ // the bl instruction).
+ offset -= 24;
+ inst[6].setData(PPC_b | JOffImm26(offset).encode() | LinkB);
+ } else {
+ // Why doesn't anyone like my singing?
+ spew("# writing in and pending long call");
+ addLongJump(BufferOffset(b), BufferOffset(target));
+ Assembler::WriteLoad64Instructions(inst, SecondScratchReg, LabelBase::INVALID_OFFSET);
+ inst[5].makeOp_mtctr(SecondScratchReg);
+ inst[6].makeOp_bctr(LinkB);
+ }
+ } else if (t == BTag) {
+ // More or less a degenerate case of BCTag. But do they let me
+ // sing Disney songs about that? Noooooooooo. They said it was
+ // an HR violation and would summon lawyers from their undead
+ // crypts to lay waste upon the earth. Wimps.
+ MOZ_ASSERT(inst[2].encode() == PPC_nop);
+ MOZ_ASSERT(inst[3].encode() == PPC_nop);
+ MOZ_ASSERT(inst[4].encode() == PPC_nop);
+ MOZ_ASSERT(inst[5].encode() == PPC_nop);
+ MOZ_ASSERT(inst[6].encode() == PPC_nop);
+
+ if (bound && JOffImm26::IsInRange(offset)) {
+ spew("# writing in short b");
+ // Make sure we're not going to patch in the wrong place.
+ MOZ_ASSERT(inst[7].encode() != PPC_bctr);
+ // The branch, in this case, really is in slot 0.
+ inst[0].setData(PPC_b | JOffImm26(offset).encode());
+ inst[1].setData(PPC_nop); // obliterate next in chain
+ } else {
+ spew("# writing in and pending long b");
+ addLongJump(BufferOffset(b), BufferOffset(target));
+ Assembler::WriteLoad64Instructions(inst, SecondScratchReg, LabelBase::INVALID_OFFSET);
+ inst[5].makeOp_mtctr(SecondScratchReg);
+ inst[6].makeOp_bctr(DontLinkB);
+ }
+ } else {
+ MOZ_CRASH("unhandled trap in slot 0");
+ }
+ } else if (inst[0].isOpcode(PPC_b)) {
+ // Short jump emitted by ma_b. Set the b target and nop out the next-in-chain.
+ spew("# setting short b");
+ MOZ_ASSERT(JOffImm26::IsInRange(offset));
+ inst[0].setData(PPC_b | JOffImm26(offset).encode());
+ inst[1].setData(PPC_nop); // obliterate next-in-chain
+ } else if (inst[0].isOpcode(PPC_bc)) {
+ // Short jump emitted by ma_bc. Set the bc target and nop out the next-in-chain.
+ spew("# setting short bc");
+ MOZ_ASSERT(BOffImm16::IsInSignedRange(offset));
+ inst[0].setData((inst[0].encode() & 0xffff0000) | BOffImm16(offset).encode());
+ inst[1].setData(PPC_nop); // obliterate next-in-chain
+ } else {
+ MOZ_CRASH("Unhandled bind()");
+ }
+
+#if 0
+ InstImm inst_beq = InstImm(op_beq, r0, r0, BOffImm16(0));
+ uint64_t offset = dest.getOffset() - label->offset();
+
+ // If first instruction is lui, then this is a long jump.
+ // If second instruction is lui, then this is a loop backedge.
+ if (inst[0].extractOpcode() == (uint32_t(op_lui) >> OpcodeShift)) {
+ // For unconditional long branches generated by ma_liPatchable,
+ // such as under:
+ // jumpWithpatch
+ addLongJump(BufferOffset(label->offset()));
+ } else if (inst[1].extractOpcode() == (uint32_t(op_lui) >> OpcodeShift) ||
+ BOffImm16::IsInRange(offset))
+ {
+ // Handle code produced by:
+ // backedgeJump
+ MOZ_ASSERT(BOffImm16::IsInRange(offset));
+ MOZ_ASSERT(inst[0].extractOpcode() == (uint32_t(op_beq) >> OpcodeShift) ||
+ inst[0].extractOpcode() == (uint32_t(op_bne) >> OpcodeShift) ||
+ inst[0].extractOpcode() == (uint32_t(op_blez) >> OpcodeShift) ||
+ inst[0].extractOpcode() == (uint32_t(op_bgtz) >> OpcodeShift) ||
+ (inst[0].extractOpcode() == (uint32_t(op_regimm) >> OpcodeShift) &&
+ inst[0].extractRT() == (uint32_t(rt_bltz) >> RTShift)));
+ inst[0].setBOffImm16(BOffImm16(offset));
+ } else if (inst[0].encode() == inst_beq.encode()) {
+ // Handle open long unconditional jumps created by
+ // MacroAssemblerMIPSShared::ma_bc(..., wasm::Trap, ...).
+ // We need to add it to long jumps array here.
+ MOZ_ASSERT(inst[1].encode() == NopInst);
+ MOZ_ASSERT(inst[2].encode() == NopInst);
+ MOZ_ASSERT(inst[3].encode() == NopInst);
+ MOZ_ASSERT(inst[4].encode() == NopInst);
+ MOZ_ASSERT(inst[5].encode() == NopInst);
+ addLongJump(BufferOffset(label->offset()));
+ Assembler::WriteLoad64Instructions(inst, ScratchRegister, LabelBase::INVALID_OFFSET);
+ inst[4] = InstReg(op_special, ScratchRegister, r0, r0, ff_jr).encode();
+ } else {
+ // Handle open long conditional jumps created by
+ // MacroAssemblerMIPSShared::ma_bc(..., wasm::Trap, ...).
+ inst[0] = invertBranch(inst[0], BOffImm16(7 * sizeof(uint32_t)));
+ // No need for a "nop" here because we can clobber scratch.
+ // We need to add it to long jumps array here.
+ // See MacroAssemblerMIPS64::branchWithCode().
+ MOZ_ASSERT(inst[1].encode() == NopInst);
+ MOZ_ASSERT(inst[2].encode() == NopInst);
+ MOZ_ASSERT(inst[3].encode() == NopInst);
+ MOZ_ASSERT(inst[4].encode() == NopInst);
+ MOZ_ASSERT(inst[5].encode() == NopInst);
+ MOZ_ASSERT(inst[6].encode() == NopInst);
+ addLongJump(BufferOffset(label->offset() + sizeof(uint32_t)));
+ Assembler::WriteLoad64Instructions(&inst[1], ScratchRegister, LabelBase::INVALID_OFFSET);
+ inst[5] = InstReg(op_special, ScratchRegister, r0, r0, ff_jr).encode();
+ }
+#endif
+}
+
+void
+Assembler::bind(Label* label, BufferOffset boff)
+{
+ spew(".set Llabel %p", label);
+ // If our caller didn't give us an explicit target to bind to
+ // then we want to bind to the location of the next instruction
+ BufferOffset dest = boff.assigned() ? boff : nextOffset();
+ if (label->used()) {
+ int32_t next;
+
+ // A used label holds a link to branch that uses it.
+ BufferOffset b(label);
+ do {
+ // Even a 0 offset may be invalid if we're out of memory.
+ if (oom()) {
+ return;
+ }
+
+ Instruction* inst = editSrc(b);
+
+ // Second word holds a pointer to the next branch in label's chain.
+ next = inst[1].encode();
+ bind(reinterpret_cast<InstImm*>(inst), b.getOffset(), dest.getOffset(), boff.assigned());
+
+ b = BufferOffset(next);
+ } while (next != LabelBase::INVALID_OFFSET);
+ }
+ label->bind(dest.getOffset());
+}
+
+void
+Assembler::retarget(Label* label, Label* target)
+{
+ spew("retarget %p -> %p", label, target);
+ if (label->used() && !oom()) {
+ if (target->bound()) {
+ bind(label, BufferOffset(target));
+ } else if (target->used()) {
+ // The target is not bound but used. Prepend label's branch list
+ // onto target's.
+ int32_t next;
+ BufferOffset labelBranchOffset(label);
+
+ // Find the head of the use chain for label.
+ do {
+ Instruction* inst = editSrc(labelBranchOffset);
+
+ // Second word holds a pointer to the next branch in chain.
+ next = inst[1].encode();
+ labelBranchOffset = BufferOffset(next);
+ } while (next != LabelBase::INVALID_OFFSET);
+
+ // Then patch the head of label's use chain to the tail of
+ // target's use chain, prepending the entire use chain of target.
+ Instruction* inst = editSrc(labelBranchOffset);
+ int32_t prev = target->offset();
+ target->use(label->offset());
+ inst[1].setData(prev);
+ } else {
+ // The target is unbound and unused. We can just take the head of
+ // the list hanging off of label, and dump that into target.
+ target->use(label->offset());
+ }
+ }
+ label->reset();
+}
+
+void
+Assembler::processCodeLabels(uint8_t* rawCode)
+{
+ for (const CodeLabel& label : codeLabels_) {
+ Bind(rawCode, label);
+ }
+}
+
+uint32_t
+Assembler::PatchWrite_NearCallSize()
+{
+ // Load an address needs 5 instructions, mtctr, bctrl
+ return (5 + 2) * sizeof(uint32_t);
+}
+
+void
+Assembler::PatchWrite_NearCall(CodeLocationLabel start, CodeLocationLabel toCall)
+{
+ // Overwrite whatever instruction used to be here with a call.
+ MOZ_ASSERT(PatchWrite_NearCallSize() == 7 * sizeof(uint32_t));
+ Instruction* inst = (Instruction*) start.raw();
+ uint8_t* dest = toCall.raw();
+
+ // We use a long stanza, but if we can, put nops/bl instead of a full
+ // lis/ori/rldicr/lis/ori/mtctr/bctr because we can speculate better.
+ // If we short it, the short branch is at the END of the stanza because
+ // the return address must follow it.
+ int64_t offset = ((uint64_t)dest - (uint64_t)inst) - 24;
+
+ if (JOffImm26::IsInRange(offset)) {
+ // "$5 says he shorts it."
+ // Since this is a near call, we expect we won't need to repatch this.
+ inst[0] = Instruction(PPC_nop);
+ inst[1] = Instruction(PPC_nop);
+ inst[2] = Instruction(PPC_nop);
+ inst[3] = Instruction(PPC_nop);
+ inst[4] = Instruction(PPC_nop);
+ inst[5] = Instruction(PPC_nop);
+ inst[6].setData(PPC_b | JOffImm26(offset).encode() | LinkB);
+ } else {
+ // Long jump required ...
+ Assembler::WriteLoad64Instructions(inst, ScratchRegister, (uint64_t)dest);
+ inst[5].makeOp_mtctr(ScratchRegister);
+ inst[6].makeOp_bctr(LinkB);
+ }
+
+ // Ensure everyone sees the code that was just written into memory.
+ FlushICache(inst, PatchWrite_NearCallSize());
+}
+
+void
+Assembler::PatchWrite_Imm32(CodeLocationLabel label, Imm32 imm)
+{
+ uint32_t *l = (uint32_t *)label.raw();
+
+ *(l - 1) = imm.value;
+}
+
+void
+Assembler::UpdateLisOriValue(Instruction *inst0, Instruction *inst1,
+ uint32_t value)
+{
+ MOZ_ASSERT(inst0->extractOpcode() == (uint32_t)PPC_addis);
+ MOZ_ASSERT(inst1->extractOpcode() == (uint32_t)PPC_ori);
+
+ reinterpret_cast<InstImm*>(inst0)->setImm16(value >> 16);
+ reinterpret_cast<InstImm*>(inst1)->setImm16(value & 0xffff);
+}
+
+uint64_t
+Assembler::ExtractLoad64Value(Instruction* inst0)
+{
+ InstImm* i0 = (InstImm*) inst0; // lis
+ InstImm* i1 = (InstImm*) i0->next(); // ori
+ Instruction* i2 = (Instruction*) i1->next(); // rldicr
+ InstImm* i3 = (InstImm*) i2->next(); // oris
+ InstImm* i4 = (InstImm*) i3->next(); // ori
+
+ MOZ_ASSERT(i0->extractOpcode() == (uint32_t)PPC_addis);
+ MOZ_ASSERT(i1->extractOpcode() == (uint32_t)PPC_ori);
+ MOZ_ASSERT(i2->extractOpcode() == (uint32_t)PPC_rldicl); // XXX: 0x78000000
+ MOZ_ASSERT(i3->extractOpcode() == (uint32_t)PPC_oris);
+ MOZ_ASSERT(i4->extractOpcode() == (uint32_t)PPC_ori);
+
+ uint64_t value = (uint64_t(i0->extractImm16Value()) << 48) |
+ (uint64_t(i1->extractImm16Value()) << 32) |
+ (uint64_t(i3->extractImm16Value()) << 16) |
+ uint64_t(i4->extractImm16Value());
+ return value;
+}
+
+void
+Assembler::UpdateLoad64Value(Instruction* inst0, uint64_t value)
+{
+ InstImm* i0 = (InstImm*) inst0;
+ InstImm* i1 = (InstImm*) i0->next();
+ Instruction* i2 = (Instruction*) i1->next();
+ InstImm* i3 = (InstImm*) i2->next();
+ InstImm* i4 = (InstImm*) i3->next();
+
+ MOZ_ASSERT(i0->extractOpcode() == (uint32_t)PPC_addis);
+ MOZ_ASSERT(i1->extractOpcode() == (uint32_t)PPC_ori);
+ MOZ_ASSERT(i2->extractOpcode() == (uint32_t)PPC_rldicl); // XXX
+ MOZ_ASSERT(i3->extractOpcode() == (uint32_t)PPC_oris);
+ MOZ_ASSERT(i4->extractOpcode() == (uint32_t)PPC_ori);
+
+ i0->setImm16(Imm16::Upper(Imm32(value >> 32)));
+ i1->setImm16(Imm16::Lower(Imm32(value >> 32)));
+ i3->setImm16(Imm16::Upper(Imm32(value)));
+ i4->setImm16(Imm16::Lower(Imm32(value)));
+}
+
+void
+Assembler::WriteLoad64Instructions(Instruction* inst0, Register reg, uint64_t value)
+{
+ Instruction* inst1 = inst0->next();
+ Instruction* inst2 = inst1->next();
+ Instruction* inst3 = inst2->next();
+ Instruction* inst4 = inst3->next();
+
+ *inst0 = InstImm(PPC_addis, reg, r0, Imm16::Upper(Imm32(value >> 32)).encode()); // mscdfr0
+ *inst1 = InstImm(PPC_ori, reg, reg, Imm16::Lower(Imm32(value >> 32)).encode());
+ // rldicr reg, reg, 32, 31
+ *inst2 = InstImm(PPC_rldicr, reg, reg, ((31 << 6) | (32 >> 4)));
+ *inst3 = InstImm(PPC_oris, reg, reg, Imm16::Upper(Imm32(value)).encode());
+ *inst4 = InstImm(PPC_ori, reg, reg, Imm16::Lower(Imm32(value)).encode());
+}
+
+void
+Assembler::PatchDataWithValueCheck(CodeLocationLabel label, ImmPtr newValue,
+ ImmPtr expectedValue)
+{
+ PatchDataWithValueCheck(label, PatchedImmPtr(newValue.value),
+ PatchedImmPtr(expectedValue.value));
+}
+
+void
+Assembler::PatchDataWithValueCheck(CodeLocationLabel label, PatchedImmPtr newValue,
+ PatchedImmPtr expectedValue)
+{
+ spew("# PatchDataWithValueCheck 0x%p", label.raw());
+ Instruction* inst = (Instruction*) label.raw();
+
+ // Extract old Value
+ DebugOnly<uint64_t> value = Assembler::ExtractLoad64Value(inst);
+ MOZ_ASSERT(value == uint64_t(expectedValue.value));
+
+ // Replace with new value
+ Assembler::UpdateLoad64Value(inst, uint64_t(newValue.value));
+ FlushICache(inst, 5 * sizeof(uint32_t));
+}
+
+// See MacroAssemblerPPC64Compat::toggledJump and
+// MacroAssemblerPPC64Compat::toggledCall.
+// These patch our suspicious oris r0,r0,0 to either skip the next
+// instruction or not.
+
+void
+Assembler::ToggleCall(CodeLocationLabel inst_, bool enabled)
+{
+ Instruction* inst = (Instruction*)inst_.raw();
+ MOZ_ASSERT((inst->encode() == PPC_oris) || (inst->encode() == (PPC_b | 0x20)));
+ if (enabled) {
+ inst->setData(PPC_oris); // oris 0,0,0
+ } else {
+ inst->setData(PPC_b | 0x20); // b .+32
+ }
+
+ FlushICache(inst, sizeof(uint32_t));
+}
+
+// JMP and CMP are from the icky x86 perspective. Since we have a jump
+// stanza, making it a "JMP" means setting the gate instruction to oris
+// so the stanza is run; making it a "CMP" means setting it to b+.32 so it
+// isn't.
+
+void
+Assembler::ToggleToJmp(CodeLocationLabel inst_)
+{
+ Instruction* inst = (Instruction*)inst_.raw();
+ MOZ_ASSERT(inst->encode() == (PPC_b | 0x20));
+ inst->setData(PPC_oris);
+
+ FlushICache(inst, sizeof(uint32_t));
+}
+
+void
+Assembler::ToggleToCmp(CodeLocationLabel inst_)
+{
+ Instruction* inst = (Instruction*)inst_.raw();
+ MOZ_ASSERT(inst->encode() == PPC_oris);
+ inst->setData(PPC_b | 0x20);
+
+ FlushICache(inst, sizeof(uint32_t));
+}
+
+Assembler::Condition
+Assembler::InvertCondition( Condition cond)
+{
+ switch (cond) {
+ case Equal:
+ return NotEqual;
+ case NotEqual:
+ return Equal;
+ case LessThan:
+ return GreaterThanOrEqual;
+ case LessThanOrEqual:
+ return GreaterThan;
+ case GreaterThan:
+ return LessThanOrEqual;
+ case GreaterThanOrEqual:
+ return LessThan;
+ case Above:
+ return BelowOrEqual;
+ case AboveOrEqual:
+ return Below;
+ case Below:
+ return AboveOrEqual;
+ case BelowOrEqual:
+ return Above;
+ case BitEqual:
+ return BitNotEqual;
+ case BitNotEqual:
+ return BitEqual;
+ case Zero:
+ return NonZero;
+ case NonZero:
+ return Zero;
+ case Signed:
+ return NotSigned;
+ case NotSigned:
+ return Signed;
+ case SOBit:
+ return NSOBit;
+ case NSOBit:
+ return SOBit;
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+Assembler::DoubleCondition
+Assembler::InvertCondition( DoubleCondition cond)
+{
+ switch (cond) {
+ case DoubleOrdered:
+ return DoubleUnordered;
+ case DoubleEqual:
+ return DoubleNotEqualOrUnordered;
+ case DoubleNotEqual:
+ return DoubleEqualOrUnordered;
+ case DoubleGreaterThan:
+ return DoubleLessThanOrEqualOrUnordered;
+ case DoubleGreaterThanOrEqual:
+ return DoubleLessThanOrUnordered;
+ case DoubleLessThan:
+ return DoubleGreaterThanOrEqualOrUnordered;
+ case DoubleLessThanOrEqual:
+ return DoubleGreaterThanOrUnordered;
+ case DoubleUnordered:
+ return DoubleOrdered;
+ case DoubleEqualOrUnordered:
+ return DoubleNotEqual;
+ case DoubleNotEqualOrUnordered:
+ return DoubleEqual;
+ case DoubleGreaterThanOrUnordered:
+ return DoubleLessThanOrEqual;
+ case DoubleGreaterThanOrEqualOrUnordered:
+ return DoubleLessThan;
+ case DoubleLessThanOrUnordered:
+ return DoubleGreaterThanOrEqual;
+ case DoubleLessThanOrEqualOrUnordered:
+ return DoubleGreaterThan;
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+bool
+Assembler::swapBuffer(wasm::Bytes& bytes) {
+ // For now, specialize to the one use case. As long as wasm::Bytes is a
+ // Vector, not a linked-list of chunks, there's not much we can do other
+ // than copy.
+ MOZ_ASSERT(bytes.empty());
+ if (!bytes.resize(bytesNeeded())) {
+ return false;
+ }
+ m_buffer.executableCopy(bytes.begin());
+ return true;
+}
+
+void
+Assembler::copyJumpRelocationTable(uint8_t* dest)
+{
+ if (jumpRelocations_.length()) {
+ memcpy(dest, jumpRelocations_.buffer(), jumpRelocations_.length());
+ }
+}
+
+void
+Assembler::copyDataRelocationTable(uint8_t* dest)
+{
+ if (dataRelocations_.length()) {
+ memcpy(dest, dataRelocations_.buffer(), dataRelocations_.length());
+ }
+}
+
+void
+Assembler::finish()
+{
+ MOZ_ASSERT(!isFinished);
+ isFinished = true;
+}
+
+void
+Assembler::executableCopy(uint8_t* buffer)
+{
+ spew("# EXECUTABLE COPY TO %p\n", buffer);
+
+ MOZ_ASSERT(isFinished);
+ m_buffer.executableCopy(buffer);
+}
+
+bool
+Assembler::appendRawCode(unsigned char const *bytes, unsigned long length)
+{
+ return m_buffer.appendRawCode(bytes, length);
+}
+
+bool Assembler::oom() const {
+ return AssemblerShared::oom() || m_buffer.oom() || jumpRelocations_.oom() ||
+ dataRelocations_.oom();
+}
+
+BufferOffset Assembler::haltingAlign(int align)
+{
+ BufferOffset ret;
+ MOZ_ASSERT(m_buffer.isAligned(4));
+ if (align == 8) {
+ if (!m_buffer.isAligned(align)) {
+ BufferOffset tmp = xs_trap();
+ if (!ret.assigned()) {
+ ret = tmp;
+ }
+ }
+ } else {
+ MOZ_ASSERT((align& (align- 1)) == 0);
+ while (size() & (align- 1)) {
+ BufferOffset tmp = xs_trap();
+ if (!ret.assigned()) {
+ ret = tmp;
+ }
+ }
+ }
+ return ret;
+}
+
+BufferOffset Assembler::nopAlign(int align)
+{
+ BufferOffset ret;
+ MOZ_ASSERT(m_buffer.isAligned(4));
+ if (align == 8) {
+ if (!m_buffer.isAligned(align)) {
+ BufferOffset tmp = as_nop();
+ if (!ret.assigned()) {
+ ret = tmp;
+ }
+ }
+ } else {
+ MOZ_ASSERT((align& (align- 1)) == 0);
+ while (size() & (align- 1)) {
+ BufferOffset tmp = as_nop();
+ if (!ret.assigned()) {
+ ret = tmp;
+ }
+ }
+ }
+ return ret;
+}
+
+size_t
+Assembler::size() const
+{
+ return m_buffer.size();
+}
+
+size_t
+Assembler::dataRelocationTableBytes() const
+{
+ return dataRelocations_.length();
+}
+
+size_t
+Assembler::jumpRelocationTableBytes() const
+{
+ return jumpRelocations_.length();
+}
+
+size_t
+Assembler::bytesNeeded() const
+{
+ return m_buffer.size() +
+ jumpRelocations_.length() +
+ dataRelocations_.length() +
+ preBarriers_.length();
+}
+
+BufferOffset
+Assembler::writeInst(uint32_t x, uint32_t *dest)
+{
+ if (dest == nullptr)
+ return m_buffer.putInt(x);
+
+ *dest = x;
+ return BufferOffset();
+}
+
+BufferOffset Assembler::as_nop()
+{
+ spew("nop");
+ return writeInst(PPC_nop);
+}
+
+BufferOffset Assembler::as_eieio()
+{
+ // Old McDonald had to order memory access ...
+ spew("eieio");
+ return writeInst(PPC_eieio);
+}
+
+BufferOffset Assembler::as_isync()
+{
+ spew("isync");
+ return writeInst(PPC_isync);
+}
+
+BufferOffset Assembler::xs_lwsync()
+{
+ spew("lwsync");
+ return writeInst(PPC_lwsync);
+}
+
+BufferOffset Assembler::as_sync()
+{
+ spew("sync");
+ return writeInst(PPC_sync);
+}
+
+// Branch and jump instructions.
+BufferOffset Assembler::as_b(JOffImm26 off, BranchAddressType bat, LinkBit lb)
+{
+ return as_b(off.encode(), bat, lb);
+}
+
+BufferOffset Assembler::as_b(int32_t off, BranchAddressType bat, LinkBit lb)
+{
+ spew("b%s%s\t%x", bat == AbsoluteBranch ? "a" : "", lb ? "l" : "", off);
+ MOZ_ASSERT(!(off & 0x03));
+ return writeInst(PPC_b | ((uint32_t)off & 0x3fffffc) | bat | lb);
+}
+
+BufferOffset Assembler::as_blr(LinkBit lb)
+{
+ spew("blr%s", lb ? "l" : "");
+ return writeInst(PPC_blr | lb);
+}
+
+BufferOffset Assembler::as_bctr(LinkBit lb)
+{
+ spew("bctr%s", lb ? "l" : "");
+ return writeInst(PPC_bctr | lb);
+}
+
+// Conditional branches.
+//
+// These utility functions turn a condition (possibly synthetic) and CR
+// field number into BO _|_ BI. With DoubleConditions we may issue CR bit
+// twiddles and change the op.
+uint16_t Assembler::computeConditionCode(DoubleCondition op, CRegisterID cr)
+{
+ // Use condition register logic to combine the FU (FUUUU-! I mean, unordered)
+ // bit with the actual condition bit.
+ const uint8_t condBit = crBit(cr, op);
+ const uint8_t fuBit = crBit(cr, DoubleUnordered);
+ uint32_t newop = (uint32_t)op & 255;
+
+ if (op & DoubleConditionUnordered) {
+ // branch if condition true OR Unordered
+ if ((op & BranchOptionMask) == BranchOnClear) {
+ // invert the condBit, or it with fuBit, and branch on Set
+ as_crorc(condBit, fuBit, condBit);
+ newop |= BranchOnSet;
+ } else {
+ // or the condBit with fuBit, and then branch on Set
+ if (condBit != fuBit)
+ as_cror(condBit, fuBit, condBit);
+ }
+ } else {
+ // branch if condition true AND ordered
+ if ((op & BranchOptionMask) == BranchOnClear) {
+ // or the condBit with fuBit, and branch on Clear
+ if (condBit != fuBit)
+ as_cror(condBit, fuBit, condBit);
+ } else {
+ // and the condBit with (!fuBit), and branch on Set, but
+ // don't clear SO if this is actually DoubleUnordered
+ // (fuBit == condBit), which is NOT a synthetic condition.
+ if (condBit != fuBit)
+ as_crandc(condBit, condBit, fuBit);
+ }
+ }
+
+ // Set BIF to the proper CR. In cr0, the normal state, this just returns newop.
+ return (newop + ((uint8_t)cr << 6));
+}
+
+// Do the same for GPR compares and XER-based conditions.
+uint16_t Assembler::computeConditionCode(Condition op, CRegisterID cr)
+{
+ // Mask off the synthetic bits, if present. Hopefully we handled them already!
+ uint32_t newop = (uint32_t)op & 255;
+
+ // If this is an XER-mediated condition, then extract it.
+ if (op & ConditionOnlyXER) {
+ MOZ_ASSERT(op == Overflow);
+ // Get XER into CR.
+ as_mcrxrx(cr);
+ // Convert op to GT (using the OV32 bit).
+ newop = (uint32_t)GreaterThan;
+ }
+
+ // Set BIF to the proper CR. In cr0, the normal state, this just returns newop.
+ return (newop + ((uint8_t)cr << 6));
+}
+
+// Given BO _|_ BI in a 16-bit quantity, emit the two halves suitable for bit masking.
+static uint32_t makeOpMask(uint16_t op)
+{
+ MOZ_ASSERT(!(op & 0xfc00)); // must fit in 10 bits
+ return ((op & 0x0f) << 21) | ((op & 0xfff0) << 12);
+}
+
+BufferOffset Assembler::as_bcctr(Condition cond, CRegisterID cr, LikelyBit lkb,
+ LinkBit lb)
+{
+ return as_bcctr(computeConditionCode(cond, cr), lkb, lb);
+}
+
+BufferOffset Assembler::as_bcctr(DoubleCondition cond, CRegisterID cr,
+ LikelyBit lkb, LinkBit lb)
+{
+ return as_bcctr(computeConditionCode(cond, cr), lkb, lb);
+}
+
+BufferOffset Assembler::as_bc(BOffImm16 off, Condition cond, CRegisterID cr,
+ LikelyBit lkb, LinkBit lb)
+{
+ // fall through to the next one
+ return as_bc(off.encode(), cond, cr, lkb, lb);
+}
+
+BufferOffset Assembler::as_bc(int16_t off, Condition cond, CRegisterID cr,
+ LikelyBit lkb, LinkBit lb)
+{
+ // No current need to adjust for the |mcrxr|; see ma_bc() in the
+ // MacroAssembler for why (no branch we generate that uses the result
+ // needs to be retargeted).
+ return as_bc(off, computeConditionCode(cond, cr), lkb, lb);
+}
+
+BufferOffset Assembler::as_bc(BOffImm16 off, DoubleCondition cond,
+ CRegisterID cr, LikelyBit lkb, LinkBit lb)
+{
+ // fall through to the next one
+ return as_bc(off.encode(), cond, cr, lkb, lb);
+}
+
+BufferOffset Assembler::as_bc(int16_t off, DoubleCondition cond, CRegisterID cr,
+ LikelyBit lkb, LinkBit lb)
+{
+ // Adjust for issued CR twiddles, if any.
+ uint32_t offs = currentOffset();
+ uint16_t op = computeConditionCode(cond, cr);
+ offs = currentOffset() - offs;
+ MOZ_ASSERT(offs == 0 || offs == 4); // currently zero or one instruction
+ return as_bc((off-offs), op, lkb, lb);
+}
+
+// These are the actual instruction emitters after conditions are converted
+// and any offsets recalculated.
+BufferOffset Assembler::as_bc(int16_t off, uint16_t op, LikelyBit lkb, LinkBit lb)
+{
+ spew("bc%s%s BO_BI=0x%04x,%d", (lb) ? "l" : "", (lkb) ? "+" : "", op, off);
+ MOZ_ASSERT(!(off & 0x03));
+ return writeInst(Instruction(PPC_bc | makeOpMask(op) | lkb << 21 | ((uint16_t)off & 0xfffc) | lb).encode());
+}
+BufferOffset Assembler::as_bcctr(uint16_t op, LikelyBit lkb, LinkBit lb)
+{
+ spew("bcctr%s%s", (lb) ? "l" : "", (lkb) ? "+" : "");
+ return writeInst(PPC_bcctr | makeOpMask(op) | lkb << 21 | lb);
+}
+
+// SPR operations.
+BufferOffset Assembler::as_mtspr(SPRegisterID spr, Register ra)
+{
+ spew("mtspr\t%d,%3s", spr, ra.name());
+ return writeInst(PPC_mtspr | ra.code() << 21 | PPC_SPR(spr) << 11);
+}
+BufferOffset Assembler::as_mfspr(Register rd, SPRegisterID spr)
+{
+ spew("mfspr\t%3s,%d", rd.name(), spr);
+ return writeInst(PPC_mfspr | rd.code() << 21 | PPC_SPR(spr) << 11);
+}
+
+// CR operations.
+#define DEF_CRCR(op) \
+ BufferOffset Assembler::as_##op(uint8_t t, uint8_t a, uint8_t b) { \
+ spew(#op"\t%d,%d,%d", t, a, b); \
+ return writeInst(PPC_##op | t << 21 | a << 16 | b << 11); }
+DEF_CRCR(crand)
+DEF_CRCR(crandc)
+DEF_CRCR(cror)
+DEF_CRCR(crorc)
+DEF_CRCR(crxor)
+#undef DEF_CRCR
+
+BufferOffset Assembler::as_mtcrf(uint32_t mask, Register rs)
+{
+ spew("mtcrf %d,%3s", mask, rs.name());
+ return writeInst(PPC_mtcrf | rs.code() << 21 | mask << 12);
+}
+
+BufferOffset Assembler::as_mfcr(Register rd)
+{
+ spew("mfcr %3s", rd.name());
+ return writeInst(PPC_mfcr | rd.code() << 21);
+}
+
+BufferOffset Assembler::as_mfocrf(Register rd, CRegisterID crfs)
+{
+ spew("mfocrf %3s,cr%d", rd.name(), crfs);
+ return writeInst(PPC_mfocrf | rd.code() << 21 | crfs << 12);
+}
+
+BufferOffset Assembler::as_mcrxrx(CRegisterID cr)
+{
+ spew("mcrxrx\tcr%d", cr);
+ return writeInst(PPC_mcrxrx | cr << 23);
+}
+
+// GPR operations and load-stores.
+BufferOffset Assembler::as_neg(Register rd, Register rs)
+{
+ spew("neg %3s,%3s", rd.name(), rs.name());
+ return writeInst(InstReg(PPC_neg, rd, rs, r0).encode());
+}
+
+BufferOffset Assembler::as_nego(Register rd, Register rs)
+{
+ spew("nego %3s,%3s", rd.name(), rs.name());
+ return writeInst(InstReg(PPC_nego, rd, rs, r0).encode());
+}
+
+BufferOffset Assembler::as_cmpd(CRegisterID cr, Register ra, Register rb)
+{
+ spew("cmpd\tcr%d,%3s,%3s", cr, ra.name(), rb.name());
+ return writeInst(PPC_cmpd | cr << 23 | ra.code() << 16 | rb.code() << 11);
+}
+
+BufferOffset Assembler::as_cmpdi(CRegisterID cr, Register ra, int16_t im)
+{
+ spew("cmpdi\tcr%d,%3s,%d", cr, ra.name(), im);
+ return writeInst(PPC_cmpdi | cr << 23 | ra.code() << 16 | ((uint16_t)im & 0xffff));
+}
+
+BufferOffset Assembler::as_cmpld(CRegisterID cr, Register ra, Register rb)
+{
+ spew("cmpld\tcr%d,%3s,%3s", cr, ra.name(), rb.name());
+ return writeInst(PPC_cmpld | cr << 23 | ra.code() << 16 | rb.code() << 11);
+}
+BufferOffset Assembler::as_cmpldi(CRegisterID cr, Register ra, int16_t im)
+{
+ spew("cmpldi\tcr%d,%3s,%d", cr, ra.name(), im);
+ return writeInst(PPC_cmpldi | cr << 23 | ra.code() << 16 | ((uint16_t)im & 0xffff));
+}
+BufferOffset Assembler::as_cmpw(CRegisterID cr, Register ra, Register rb)
+{
+ spew("cmpw\t%3s,%3s", ra.name(), rb.name());
+ return writeInst(PPC_cmpw | cr << 23 | ra.code() << 16 | rb.code() << 11);
+}
+BufferOffset Assembler::as_cmpwi(CRegisterID cr, Register ra, int16_t im)
+{
+ spew("cmpwi\tcr%d,%3s,%d", cr, ra.name(), im);
+ return writeInst(PPC_cmpwi | cr << 23 | ra.code() << 16 | ((uint16_t)im & 0xffff));
+}
+BufferOffset Assembler::as_cmplw(CRegisterID cr, Register ra, Register rb)
+{
+ spew("cmplw\tcr%d,%3s,%3s", cr, ra.name(), rb.name());
+ return writeInst(PPC_cmplw | cr << 23 | ra.code() << 16 | rb.code() << 11);
+}
+BufferOffset Assembler::as_cmplwi(CRegisterID cr, Register ra, int16_t im)
+{
+ spew("cmplwi\tcr%d,%3s,%d", cr, ra.name(), im);
+ return writeInst(PPC_cmplwi | cr << 23 | ra.code() << 16 | ((uint16_t)im & 0xffff));
+}
+BufferOffset Assembler::as_cmpd(Register ra, Register rb)
+{
+ spew("cmpd\t%3s,%3s", ra.name(), rb.name());
+ return writeInst(PPC_cmpd | ra.code() << 16 | rb.code() << 11);
+}
+BufferOffset Assembler::as_cmpdi(Register ra, int16_t im)
+{
+ spew("cmpdi\t%3s,%d", ra.name(), im);
+ return writeInst(PPC_cmpdi | ra.code() << 16 | ((uint16_t)im & 0xffff));
+}
+BufferOffset Assembler::as_cmpld(Register ra, Register rb)
+{
+ spew("cmpld\t%3s,%3s", ra.name(), rb.name());
+ return writeInst(PPC_cmpld | ra.code() << 16 | rb.code() << 11);
+}
+BufferOffset Assembler::as_cmpldi(Register ra, int16_t im)
+{
+ spew("cmpldi\t%3s,%d", ra.name(), im);
+ return writeInst(PPC_cmpldi | ra.code() << 16 | ((uint16_t)im & 0xffff));
+}
+BufferOffset Assembler::as_cmpw(Register ra, Register rb)
+{
+ spew("cmpw\t%3s,%3s", ra.name(), rb.name());
+ return writeInst(PPC_cmpw | ra.code() << 16 | rb.code() << 11);
+}
+BufferOffset Assembler::as_cmpwi(Register ra, int16_t im)
+{
+ spew("cmpwi\t%3s,%d", ra.name(), im);
+ return writeInst(PPC_cmpwi | ra.code() << 16 | ((uint16_t)im & 0xffff));
+}
+
+BufferOffset Assembler::as_cmplw(Register ra, Register rb)
+{
+ spew("cmplw\t%3s,%3s", ra.name(), rb.name());
+ return writeInst(PPC_cmplw | ra.code() << 16 | rb.code() << 11);
+}
+BufferOffset Assembler::as_cmplwi(Register ra, int16_t im)
+{
+ spew("cmplwi\t%3s,%d", ra.name(), im);
+ return writeInst(PPC_cmplwi | ra.code() << 16 | ((uint16_t)im & 0xffff));
+}
+
+static uint32_t
+AForm(uint32_t op, FloatRegister frt, FloatRegister fra, FloatRegister frb,
+ FloatRegister frc, bool rc)
+{
+ return (op | (frt.encoding() << 21) | (fra.encoding() << 16) | (frb.encoding() << 11) |
+ (frc.encoding() << 6) | rc);
+}
+
+static uint32_t
+XForm(uint32_t op, FloatRegister frt, FloatRegister fra, FloatRegister frb, bool rc)
+{
+ return (op | (frt.encoding() << 21) | (fra.encoding() << 16) | (frb.encoding() << 11) | rc);
+}
+
+static uint32_t
+XForm(uint32_t op, FloatRegister frt, Register ra, Register rb, bool rc)
+{
+ return (op | (frt.encoding() << 21) | (ra.code() << 16) | (rb.code() << 11) | rc);
+}
+
+static uint32_t
+DForm(uint32_t op, FloatRegister frt, Register ra, int16_t imm)
+{
+ return (op | (frt.encoding() << 21) | (ra.code() << 16) | ((uint16_t)imm & 0xffff));
+}
+
+#define DEF_XFORM(op) \
+ BufferOffset Assembler::as_##op(Register rd, Register ra, Register rb) { \
+ spew(#op "\t%3s,%3s,%3s", rd.name(), ra.name(), rb.name()); \
+ return writeInst(InstReg(PPC_##op, rd, ra, rb).encode()); }
+
+#define DEF_XFORM_RC(op) \
+ BufferOffset Assembler::as_##op##_rc(Register rd, Register ra, Register rb) {\
+ spew(#op ".\t%3s,%3s,%3s", rd.name(), ra.name(), rb.name()); \
+ return writeInst(InstReg(PPC_##op, rd, ra, rb).encode() | 0x1); }
+
+#define DEF_XFORMS(op) \
+ BufferOffset Assembler::as_##op(Register rd, Register ra, Register rb) { \
+ spew(#op "\t%3s,%3s,%3s", rd.name(), ra.name(), rb.name()); \
+ return writeInst(InstReg(PPC_##op, ra, rd, rb).encode()); }
+
+#define DEF_XFORMS_RC(op) \
+ BufferOffset Assembler::as_##op##_rc(Register rd, Register ra, Register rb) {\
+ spew(#op ".\t%3s,%3s,%3s", rd.name(), ra.name(), rb.name()); \
+ return writeInst(InstReg(PPC_##op, ra, rd, rb).encode() | 0x1); }
+
+#define DEF_AFORM_C(op) \
+ BufferOffset Assembler::as_##op(FloatRegister rd, FloatRegister ra, FloatRegister rc) { \
+ spew(#op "\t%3s,%3s,%3s", rd.name(), ra.name(), rc.name()); \
+ return writeInst(AForm(PPC_##op, rd, ra, f0, rc, false)); }
+
+#define DEF_AFORM_C_RC(op) \
+ BufferOffset Assembler::as_##op##_rc(FloatRegister rd, FloatRegister ra, FloatRegister rc) {\
+ spew(#op ".\t%3s,%3s,%3s", rd.name(), ra.name(), rc.name()); \
+ return writeInst(AForm(PPC_##op, rd, ra, f0, rc, true)); }
+
+#define DEF_AFORM_B(op) \
+ BufferOffset Assembler::as_##op(FloatRegister rd, FloatRegister ra, FloatRegister rb) { \
+ spew(#op "\t%3s,%3s,%3s", rd.name(), ra.name(), rb.name()); \
+ return writeInst(AForm(PPC_##op, rd, ra, rb, f0, false)); }
+
+#define DEF_AFORM_B_RC(op) \
+ BufferOffset Assembler::as_##op##_rc(FloatRegister rd, FloatRegister ra, FloatRegister rb) {\
+ spew(#op ".\t%3s,%3s,%3s", rd.name(), ra.name(), rb.name()); \
+ return writeInst(AForm(PPC_##op, rd, ra, rb, f0, true)); }
+
+#define DEF_AFORM(op) \
+ BufferOffset Assembler::as_##op(FloatRegister rd, FloatRegister ra, FloatRegister rc, FloatRegister rb) { \
+ spew(#op "\t%3s,%3s,%3s,%3s", rd.name(), ra.name(), rc.name(), rb.name()); \
+ return writeInst(AForm(PPC_##op, rd, ra, rb, rc, false)); }
+
+#define DEF_AFORM_RC(op) \
+ BufferOffset Assembler::as_##op##_rc(FloatRegister rd, FloatRegister ra, FloatRegister rc, FloatRegister rb) {\
+ spew(#op ".\t%3s,%3s,%3s,%3s", rd.name(), ra.name(), rc.name(), rb.name()); \
+ return writeInst(AForm(PPC_##op, rd, ra, rb, rc, true)); }
+
+#define DEF_XFORMS_I(op) \
+ BufferOffset Assembler::as_##op(Register rd, Register ra, uint8_t sh) { \
+ spew(#op "\t%3s,%3s,%d", rd.name(), ra.name(), sh); \
+ return writeInst(PPC_##op | ra.code() << 21 | rd.code() << 16 | sh << 11); }
+
+#define DEF_XFORMS_I_RC(op) \
+ BufferOffset Assembler::as_##op##_rc(Register rd, Register ra, uint8_t sh) {\
+ spew(#op ".\t%3s,%3s,%d", rd.name(), ra.name(), sh); \
+ return writeInst(PPC_##op | ra.code() << 21 | rd.code() << 16 | sh << 11 | 0x1); }
+
+#define DEF_XFORM2(op) \
+ BufferOffset Assembler::as_##op(Register rd, Register ra) { \
+ spew(#op "\t%3s,%3s", rd.name(), ra.name()); \
+ return writeInst(InstReg(PPC_##op, rd, ra, r0).encode()); }
+
+#define DEF_XFORM2_RC(op) \
+ BufferOffset Assembler::as_##op##_rc(Register rd, Register ra) {\
+ spew(#op ".\t%3s,%3s", rd.name(), ra.name()); \
+ return writeInst(InstReg(PPC_##op, rd, ra, r0).encode() | 0x1); }
+
+#define DEF_XFORM2_F(op) \
+ BufferOffset Assembler::as_##op(FloatRegister rd, FloatRegister ra) { \
+ spew(#op "\t%3s,%3s", rd.name(), ra.name()); \
+ return writeInst(XForm(PPC_##op, rd, f0, ra, false)); }
+
+#define DEF_XFORM2_F_RC(op) \
+ BufferOffset Assembler::as_##op##_rc(FloatRegister rd, FloatRegister ra) {\
+ spew(#op ".\t%3s,%3s", rd.name(), ra.name()); \
+ return writeInst(XForm(PPC_##op, rd, f0, ra, true)); }
+
+#define DEF_XFORM2S(op) \
+ BufferOffset Assembler::as_##op(Register rd, Register ra) { \
+ spew(#op "\t%3s,%3s", rd.name(), ra.name()); \
+ return writeInst(InstReg(PPC_##op, ra, rd, r0).encode()); }
+
+#define DEF_XFORM2S_RC(op) \
+ BufferOffset Assembler::as_##op##_rc(Register rd, Register ra) {\
+ spew(#op ".\t%3s,%3s", rd.name(), ra.name()); \
+ return writeInst(InstReg(PPC_##op, ra, rd, r0).encode() | 0x1); }
+
+#define DEF_DFORM(op) \
+ BufferOffset Assembler::as_##op(Register ra, Register rs, int16_t im) { \
+ spew(#op "\t%3s,%d(%3s)", ra.name(), im, rs.name()); \
+ MOZ_ASSERT(rs != r0); \
+ return writeInst(InstImm(PPC_##op, ra, rs, im).encode()); }
+
+#define DEF_DFORMS(op) \
+ BufferOffset Assembler::as_##op(Register ra, Register rs, uint16_t im) { \
+ spew(#op "\t%3s,%d(%3s)", ra.name(), im, rs.name()); \
+ return writeInst(InstImm(PPC_##op, rs, ra, im).encode()); }
+
+#define DEF_DFORM_F(op) \
+ BufferOffset Assembler::as_##op(FloatRegister rt, Register ra, int16_t im) { \
+ spew(#op "\t%3s,%d(%3s)", rt.name(), im, ra.name()); \
+ MOZ_ASSERT(ra != r0); \
+ return writeInst(DForm(PPC_##op, rt, ra, im)); }
+
+#define DEF_MFORM(op) \
+ BufferOffset Assembler::as_##op(Register ra, Register rs, Register rb, uint8_t mb, uint8_t me) {\
+ spew(#op "\t%3s,%3s,%3s,%d,%d", ra.name(), rs.name(), rb.name(), mb, me); \
+ return writeInst(PPC_##op | rs.code() << 21 | ra.code() << 16 | rb.code() << 11 | mb << 6 | me << 1); }
+
+#define DEF_MFORM_RC(op) \
+ BufferOffset Assembler::as_##op##_rc(Register ra, Register rs, Register rb, uint8_t mb, uint8_t me) {\
+ spew(#op ".\t%3s,%3s,%3s,%d,%d", ra.name(), rs.name(), rb.name(), mb, me); \
+ return writeInst(PPC_##op | rs.code() << 21 | ra.code() << 16 | rb.code() << 11 | mb << 6 | me << 1 | 1); }
+
+#define DEF_MFORM_I(op) \
+ BufferOffset Assembler::as_##op(Register ra, Register rs, uint8_t sh, uint8_t mb, uint8_t me) {\
+ spew(#op "\t%3s,%3s,%d,%d,%d", ra.name(), rs.name(), sh, mb, me); \
+ MOZ_ASSERT(sh < 32); \
+ MOZ_ASSERT(mb < 32); \
+ return writeInst(PPC_##op | rs.code() << 21 | ra.code() << 16 | sh << 11 | mb << 6 | me << 1); }
+
+#define DEF_MFORM_I_RC(op) \
+ BufferOffset Assembler::as_##op##_rc(Register ra, Register rs, uint8_t sh, uint8_t mb, uint8_t me) {\
+ spew(#op ".\t%3s,%3s,%d,%d,%d", ra.name(), rs.name(), sh, mb, me); \
+ MOZ_ASSERT(sh < 32); \
+ MOZ_ASSERT(mb < 32); \
+ return writeInst(PPC_##op | rs.code() << 21 | ra.code() << 16 | sh << 11 | mb << 6 | me << 1 | 1); }
+
+#define DEF_MDSFORM(op) \
+ BufferOffset Assembler::as_##op(Register ra, Register rs, Register rb, uint8_t mb) {\
+ spew(#op "\t%3s,%3s,%3s,%d", ra.name(), rs.name(), rb.name(), mb); \
+ return writeInst(PPC_##op | rs.code() << 21 | ra.code() << 16 | rb.code() << 11 | mb << 6); }
+
+#define DEF_MDSFORM_RC(op) \
+ BufferOffset Assembler::as_##op##_rc(Register ra, Register rs, Register rb, uint8_t mb) {\
+ spew(#op ".\t%3s,%3s,%3s,%d", ra.name(), rs.name(), rb.name(), mb); \
+ return writeInst(PPC_##op | rs.code() << 21 | ra.code() << 16 | rb.code() << 11 | mb << 6); }
+
+/* NOTHING documents these encodings well, not OPPCC, not even the 3.1 ISA book. */
+#define DEF_MDFORM(op) \
+ BufferOffset Assembler::as_##op(Register ra, Register rs, uint8_t sh, uint8_t mb) {\
+ spew(#op "\t%3s,%3s,%d,%d", ra.name(), rs.name(), sh, mb); \
+ MOZ_ASSERT(sh < 64); MOZ_ASSERT(mb < 64); \
+ return writeInst(PPC_##op | rs.code() << 21 | ra.code() << 16 | ((sh & 0x1f) << 11) | ((mb & 0x1f) << 6) | (mb & 0x20) | ((sh & 0x20) >> 4)); }
+
+#define DEF_MDFORM_RC(op) \
+ BufferOffset Assembler::as_##op##_rc(Register ra, Register rs, uint8_t sh, uint8_t mb) {\
+ spew(#op ".\t%3s,%3s,%d,%d", ra.name(), rs.name(), sh, mb); \
+ MOZ_ASSERT(sh < 64); MOZ_ASSERT(mb < 64); \
+ return writeInst(PPC_##op | rs.code() << 21 | ra.code() << 16 | ((sh & 0x1f) << 11) | ((mb & 0x1f) << 6) | (mb & 0x20) | ((sh & 0x20) >> 4) | 0x01); }
+
+DEF_MFORM(rlwnm)
+DEF_MFORM_I(rlwinm)
+DEF_MFORM_I_RC(rlwinm)
+DEF_MFORM_I(rlwimi)
+DEF_XFORMS_I(srawi)
+
+DEF_MDSFORM(rldcl)
+//DEF_MDSFORM_RC(rldcl)
+//DEF_MDSFORM(rldcr)
+DEF_MDFORM(rldicl)
+DEF_MDFORM_RC(rldicl)
+DEF_MDFORM(rldicr)
+DEF_MDFORM_RC(rldicr)
+DEF_MDFORM(rldimi)
+//DEF_MDFORM_RC(rldimi)
+BufferOffset Assembler::as_sradi(Register rd, Register rs, int sh)
+{
+ spew("sradi\t%3s,%3s,%d", rd.name(), rs.name(), sh);
+ return writeInst(PPC_sradi | rd.code() << 16 | rs.code() << 21 |
+ (sh & 0x1f) << 11 | (sh & 0x20) >> 4);
+}
+
+#define DEF_ALU2(op) DEF_XFORM(op) DEF_XFORM_RC(op)
+
+DEF_ALU2(add)
+DEF_ALU2(addc)
+DEF_ALU2(adde)
+DEF_ALU2(addo)
+DEF_ALU2(subf)
+DEF_ALU2(subfc)
+DEF_ALU2(subfe)
+DEF_ALU2(subfo)
+DEF_ALU2(divd)
+DEF_ALU2(divdo)
+DEF_ALU2(divdu)
+DEF_ALU2(divduo)
+DEF_ALU2(divw)
+DEF_ALU2(divwo)
+DEF_ALU2(divwu)
+DEF_ALU2(divwuo)
+DEF_ALU2(mulld)
+DEF_ALU2(mulhd)
+DEF_ALU2(mulhdu)
+DEF_ALU2(mulldo)
+DEF_ALU2(mullw)
+DEF_ALU2(mulhw)
+DEF_ALU2(mulhwu)
+DEF_ALU2(mullwo)
+DEF_ALU2(eqv) // NB: Implemented differently.
+#undef DEF_ALU2
+
+#define DEF_ALU2_NORC(op) DEF_XFORM(op)
+DEF_ALU2_NORC(modsd)
+DEF_ALU2_NORC(modud)
+DEF_ALU2_NORC(modsw)
+DEF_ALU2_NORC(moduw)
+#undef DEF_ALU2_NORC
+
+#define DEF_ALUI(op) \
+ BufferOffset Assembler::as_##op(Register rd, Register ra, int16_t im) { \
+ spew(#op "\t%3s,%3s,%d", rd.name(), ra.name(), im); \
+ return writeInst(InstImm(PPC_##op, rd, ra, im).encode()); }\
+ BufferOffset Assembler::as_##op##_rc(Register rd, Register ra, int16_t im) { \
+ spew(#op ".\t%3s,%3s,%d", rd.name(), ra.name(), im); \
+ return writeInst(InstImm(PPC_##op, rd, ra, im).encode() | 0x1); }
+// mscdfr0
+BufferOffset Assembler::as_addi(Register rd, Register ra, int16_t im, bool actually_li) {
+#if DEBUG
+ if (actually_li) {
+ spew("li\t%3s,%d", rd.name(), im);
+ } else {
+ MOZ_ASSERT(ra != r0); // Because that would be li
+ spew("addi\t%3s,%3s,%d", rd.name(), ra.name(), im);
+ }
+#endif
+ return writeInst(InstImm(PPC_addi, rd, ra, im).encode());
+}
+BufferOffset Assembler::as_addis(Register rd, Register ra, int16_t im, bool actually_lis) {
+#if DEBUG
+ if (actually_lis) {
+ spew("lis\t%3s,%d", rd.name(), im);
+ } else {
+ MOZ_ASSERT(ra != r0); // Because that would be lis
+ spew("addis\t%3s,%3s,%d", rd.name(), ra.name(), im);
+ }
+#endif
+ return writeInst(InstImm(PPC_addis, rd, ra, im).encode());
+}
+
+DEF_ALUI(addic)
+// NB: mulli is usually strength-reduced, since it can take up to five
+// cycles in the worst case. See xs_sr_mulli.
+DEF_ALUI(mulli)
+DEF_ALUI(subfic)
+#undef DEF_ALUI
+
+#define DEF_ALUE(op) DEF_XFORM2(op) DEF_XFORM2_RC(op)
+DEF_ALUE(addme)
+DEF_ALUE(addze)
+DEF_ALUE(subfze)
+#undef DEF_ALUE
+
+#define DEF_ALUE(op) DEF_XFORM2S(op) DEF_XFORM2S_RC(op)
+DEF_ALUE(cntlzw) // NB: In this case, rd = ra and ra = rs, but no biggie here.
+DEF_ALUE(cntlzd) // NB: In this case, rd = ra and ra = rs, but no biggie here.
+DEF_ALUE(cnttzd) // NB: In this case, rd = ra and ra = rs, but no biggie here.
+DEF_ALUE(cnttzw) // NB: In this case, rd = ra and ra = rs, but no biggie here.
+#undef DEF_ALUE
+
+DEF_XFORM2S(popcntd)
+DEF_XFORM2S(popcntw)
+
+#define DEF_BITALU2(op) DEF_XFORMS(op) DEF_XFORMS_RC(op)
+DEF_BITALU2(andc)
+DEF_BITALU2(nand)
+DEF_BITALU2(nor)
+DEF_BITALU2(slw)
+DEF_BITALU2(srw)
+DEF_BITALU2(sraw)
+DEF_BITALU2(sld)
+DEF_BITALU2(srd)
+DEF_BITALU2(srad)
+DEF_BITALU2(and)
+//DEF_BITALU2(or)
+ BufferOffset Assembler::as_or(Register rd, Register ra, Register rb) {
+ spew("or\t%3s,%3s,%3s", rd.name(), ra.name(), rb.name());
+ MOZ_ASSERT(!(rd == ra && ra == rb));
+ return writeInst(InstReg(PPC_or, ra, rd, rb).encode()); }
+
+ BufferOffset Assembler::as_or_rc(Register rd, Register ra, Register rb) {
+ spew("or.\t%3s,%3s,%3s", rd.name(), ra.name(), rb.name());
+ return writeInst(InstReg(PPC_or, ra, rd, rb).encode() | 0x1); }
+
+DEF_BITALU2(xor)
+#undef DEF_BITALU2
+
+// No Rc bit for these.
+#define DEF_BITALUI(op) DEF_DFORMS(op)
+DEF_BITALUI(ori)
+DEF_BITALUI(oris)
+DEF_BITALUI(xori)
+DEF_BITALUI(xoris)
+#undef DEF_BITALUI
+// Implied Rc bit for these.
+ BufferOffset Assembler::as_andi_rc(Register ra, Register rs, uint16_t im) {
+ spew("andi.\t%3s,%3s,%d", ra.name(), rs.name(), im);
+ return writeInst(InstImm(PPC_andi, rs, ra, im).encode()); }
+
+ BufferOffset Assembler::as_andis_rc(Register ra, Register rs, uint16_t im) {
+ spew("andis.\t%3s,%3s,%d", ra.name(), rs.name(), im);
+ return writeInst(InstImm(PPC_andis, rs, ra, im).encode()); }
+
+#define DEF_ALUEXT(op) DEF_XFORM2S(op) DEF_XFORM2S_RC(op)
+DEF_ALUEXT(extsb)
+DEF_ALUEXT(extsh)
+DEF_ALUEXT(extsw)
+#undef DEF_ALUEXT
+
+#define DEF_MEMd(op) DEF_DFORM(op)
+DEF_MEMd(lbz)
+DEF_MEMd(lha)
+DEF_MEMd(lhz)
+ BufferOffset Assembler::as_lwa(Register ra, Register rs, int16_t im) {
+ spew("lwa\t%3s,%d(%3s)", ra.name(), im, rs.name());
+ MOZ_ASSERT(rs != r0);
+ MOZ_ASSERT(!(im & 0x03));
+ return writeInst(InstImm(PPC_lwa, ra, rs, im).encode()); }
+
+DEF_MEMd(lwz)
+//DEF_MEMd(ld)
+// Assert if the two LSBs of ld's immediate are set, since this assembles
+// to a different instruction. (E.g., if we want ldu, we should ask for it.)
+ BufferOffset Assembler::as_ld(Register ra, Register rs, int16_t im) {
+ spew("ld\t%3s,%d(%3s)", ra.name(), im, rs.name());
+ MOZ_ASSERT(rs != r0);
+ MOZ_ASSERT(!(im & 0x03));
+ return writeInst(InstImm(PPC_ld, ra, rs, im).encode()); }
+
+DEF_MEMd(stb)
+DEF_MEMd(stw)
+DEF_MEMd(stwu)
+DEF_MEMd(sth)
+//DEF_MEMd(std)
+ BufferOffset Assembler::as_std(Register ra, Register rs, int16_t im) {
+ spew("std\t%3s,%d(%3s)", ra.name(), im, rs.name());
+ MOZ_ASSERT(rs != r0);
+ MOZ_ASSERT(!(im & 0x03));
+ return writeInst(InstImm(PPC_std, ra, rs, im).encode()); }
+DEF_MEMd(stdu)
+#undef DEF_MEMd
+
+#define DEF_MEMx(op) DEF_XFORM(op)
+DEF_MEMx(lbzx)
+DEF_MEMx(lhax)
+DEF_MEMx(lhzx)
+DEF_MEMx(lhbrx)
+DEF_MEMx(lwzx)
+DEF_MEMx(lwbrx)
+DEF_MEMx(lwax)
+DEF_MEMx(lwarx)
+DEF_MEMx(ldx)
+DEF_MEMx(ldarx)
+
+DEF_MEMx(stbx)
+DEF_MEMx(stwx)
+DEF_MEMx(stwux)
+DEF_MEMx(stwbrx)
+DEF_MEMx(sthx)
+DEF_MEMx(sthbrx)
+DEF_MEMx(stdx)
+DEF_MEMx(stdcx)
+DEF_MEMx(stdux)
+DEF_MEMx(stwcx)
+#undef DEF_MEMx
+
+BufferOffset Assembler::as_isel(Register rt, Register ra, Register rb, uint16_t bc, CRegisterID cr)
+{
+ MOZ_ASSERT(ra != r0); // mscdfr0, but see below, because sometimes we want this
+ return as_isel0(rt, ra, rb, bc, cr);
+}
+
+// This variant allows ra to be r0, because sometimes we feel like a zero.
+// Sometimes you don't. Almond Joy's got nuts, Mounds don't.
+BufferOffset Assembler::as_isel0(Register rt, Register ra, Register rb, uint16_t bc, CRegisterID cr)
+{
+ spew("isel\t%3s,%3s,%3s,cr%d:0x%02x", rt.name(), ra.name(), rb.name(), cr, bc);
+ // Only bits that can be directly tested for in the CR are valid.
+ // The upper nybble of the condition contains the CR bit.
+ MOZ_ASSERT((bc < 0x40) && ((bc & 0x0f) == 0x0c));
+ uint16_t nbc = (bc >> 4) + (cr << 2);
+ return writeInst(PPC_isel | rt.code() << 21 | ra.code() << 16 | rb.code() << 11 | nbc << 6);
+}
+
+// FPR operations and load-stores.
+BufferOffset Assembler::as_fcmpo(CRegisterID cr, FloatRegister ra, FloatRegister rb)
+{
+ spew("fcmpo\t%d,%3s,%3s", cr, ra.name(), rb.name());
+ return writeInst(PPC_fcmpo | cr << 23 | ra.encoding() << 16 | rb.encoding() << 11);
+}
+
+BufferOffset Assembler::as_fcmpo(FloatRegister ra, FloatRegister rb)
+{
+ return as_fcmpo(cr0, ra, rb);
+}
+
+BufferOffset Assembler::as_fcmpu(CRegisterID cr, FloatRegister ra, FloatRegister rb)
+{
+ spew("fcmpu\t%d,%3s,%3s", cr, ra.name(), rb.name());
+ return writeInst(PPC_fcmpu | cr << 23 | ra.encoding() << 16 | rb.encoding() << 11);
+}
+
+BufferOffset Assembler::as_fcmpu(FloatRegister ra, FloatRegister rb)
+{
+ return as_fcmpu(cr0, ra, rb);
+}
+
+#define DEF_FPUAC(op) DEF_AFORM_C(op) DEF_AFORM_C_RC(op)
+DEF_FPUAC(fmul)
+DEF_FPUAC(fmuls)
+#undef DEF_FPUAC
+
+#define DEF_FPUAB(op) DEF_AFORM_B(op) DEF_AFORM_B_RC(op)
+DEF_FPUAB(fadd)
+DEF_FPUAB(fdiv)
+DEF_FPUAB(fsub)
+DEF_FPUAB(fadds)
+DEF_FPUAB(fdivs)
+DEF_FPUAB(fsubs)
+DEF_FPUAB(fcpsgn)
+DEF_FPUAB(fmrgew) /* rc form invalid */
+#undef DEF_FPUAB
+
+#define DEF_FPUDS(op) DEF_XFORM2_F(op) DEF_XFORM2_F_RC(op)
+DEF_FPUDS(fabs)
+DEF_FPUDS(fneg)
+DEF_FPUDS(fmr)
+DEF_FPUDS(fcfid)
+DEF_FPUDS(fcfids)
+DEF_FPUDS(fcfidu)
+DEF_FPUDS(fcfidus)
+DEF_FPUDS(fctid)
+DEF_FPUDS(fctidz)
+DEF_FPUDS(fctidu)
+DEF_FPUDS(fctiduz)
+DEF_FPUDS(fctiw)
+DEF_FPUDS(fctiwz)
+DEF_FPUDS(fctiwu)
+DEF_FPUDS(fctiwuz)
+DEF_FPUDS(frim)
+DEF_FPUDS(frin)
+DEF_FPUDS(frip)
+DEF_FPUDS(friz)
+DEF_FPUDS(frsp)
+DEF_FPUDS(frsqrte)
+DEF_FPUDS(fsqrt)
+DEF_FPUDS(fsqrts)
+#undef DEF_FPUDS
+
+// In Ion, the semantics for this macro are now corrected compared to JM/PPCBC.
+// (See OPPCC p.432, etc.)
+#define DEF_FPUACB(op) DEF_AFORM(op) DEF_AFORM_RC(op)
+DEF_FPUACB(fmadd)
+DEF_FPUACB(fnmsub)
+DEF_FPUACB(fsel)
+#undef DEF_FPUACB
+
+#define DEF_FMEMd(op) DEF_DFORM_F(op)
+DEF_FMEMd(lfd)
+DEF_FMEMd(lfs)
+DEF_FMEMd(stfd)
+DEF_FMEMd(stfs)
+DEF_FMEMd(stfdu)
+DEF_FMEMd(stfsu)
+#undef DEF_FMEMd
+
+#define DEF_FMEMx(op) \
+ BufferOffset Assembler::as_##op(FloatRegister rd, Register ra, Register rb) { \
+ spew(#op "\t%3s,%3s,%3s", rd.name(), ra.name(), rb.name()); \
+ return writeInst(XForm(PPC_##op, rd, ra, rb, false)); }
+DEF_FMEMx(lfdx)
+DEF_FMEMx(lfsx)
+DEF_FMEMx(lfiwax)
+DEF_FMEMx(stfiwx)
+DEF_FMEMx(stfdx)
+DEF_FMEMx(stfsx)
+#undef DEF_FMEMx
+
+BufferOffset Assembler::as_mtfsb0(uint8_t bt)
+{
+ spew("mtfsb0\t%d", bt);
+ return writeInst(PPC_mtfsb0 | (uint32_t)bt << 21);
+}
+
+BufferOffset Assembler::as_mtfsb1(uint8_t bt)
+{
+ spew("mtfsb1\t%d", bt);
+ return writeInst(PPC_mtfsb1 | (uint32_t)bt << 21);
+}
+
+BufferOffset Assembler::as_mtfsfi(uint8_t fi, uint8_t imm)
+{
+ spew("mtfsfi\t%d,%d", fi, imm);
+ return writeInst(PPC_mtfsfi | fi << 23 | imm << 12);
+}
+
+BufferOffset Assembler::as_mcrf(CRegisterID bt, CRegisterID bs)
+{
+ spew("mcrf\t%d,%d", bt, bs);
+ return writeInst(PPC_mcrf | (uint32_t)bt << 23 | (uint32_t)bs << 18);
+}
+
+BufferOffset Assembler::as_mcrfs(CRegisterID bf, uint8_t bfa)
+{
+ spew("mcrfs\t%d,%d", bf, bfa);
+ return writeInst(PPC_mcrfs | (uint32_t)bf << 23 | (uint32_t)bfa << 18);
+}
+
+// VSX
+// Currently only supported for FPRs.
+// No RC forms for these (least significant bit sets vector or FPR).
+ BufferOffset Assembler::as_mfvsrd(Register ra, FloatRegister xs) {
+ spew("mfvsrd\t%3s,%3s", ra.name(), xs.name());
+ return writeInst(XForm(PPC_mfvsrd, xs, ra, r0, false));
+ }
+ BufferOffset Assembler::as_mtvsrd(FloatRegister xt, Register ra) {
+ spew("mtvsrd\t%3s,%3s", xt.name(), ra.name());
+ // Yes, same operand order (see PowerISA v3.1 page 121)
+ return writeInst(XForm(PPC_mtvsrd, xt, ra, r0, false));
+ }
+ BufferOffset Assembler::as_mtvsrws(FloatRegister xt, Register ra) {
+ spew("mtvsrws\t%3s,%3s", xt.name(), ra.name());
+ return writeInst(XForm(PPC_mtvsrws, xt, ra, r0, false));
+ }
+ BufferOffset Assembler::as_mtvsrwz(FloatRegister xt, Register ra) {
+ spew("mtvsrwz\t%3s,%3s", xt.name(), ra.name());
+ return writeInst(XForm(PPC_mtvsrwz, xt, ra, r0, false));
+ }
+ BufferOffset Assembler::as_xxbrd(FloatRegister xt, FloatRegister xb) {
+ spew("xxbrd\t%3s,%3s", xt.name(), xb.name());
+ return writeInst(XForm(PPC_xxbrd, xt, f0, xb, false));
+ }
+ BufferOffset Assembler::as_xscvdpsp(FloatRegister xt, FloatRegister xb) {
+ spew("xscvdpsp\t%3s,%3s", xt.name(), xb.name());
+ return writeInst(XForm(PPC_xscvdpsp, xt, f0, xb, false));
+ }
+ BufferOffset Assembler::as_xscvspdp(FloatRegister xt, FloatRegister xb) {
+ spew("xscvspdp\t%3s,%3s", xt.name(), xb.name());
+ return writeInst(XForm(PPC_xscvspdp, xt, f0, xb, false));
+ }
+ BufferOffset Assembler::as_xscvdpspn(FloatRegister xt, FloatRegister xb) {
+ spew("xscvdpspn\t%3s,%3s", xt.name(), xb.name());
+ return writeInst(XForm(PPC_xscvdpspn, xt, f0, xb, false));
+ }
+ BufferOffset Assembler::as_xscvspdpn(FloatRegister xt, FloatRegister xb) {
+ spew("xscvspdpn\t%3s,%3s", xt.name(), xb.name());
+ return writeInst(XForm(PPC_xscvspdpn, xt, f0, xb, false));
+ }
+ BufferOffset Assembler::as_xxlxor(FloatRegister xt, FloatRegister xa, FloatRegister xb) {
+ spew("xxlxor\t%3s,%3s,%3s", xt.name(), xa.name(), xb.name());
+ return writeInst(XForm(PPC_xxlxor, xt, xa, xb, false));
+ }
+
+ BufferOffset Assembler::as_addpcis(Register rt, uint16_t im) {
+ spew("addpcis\t%s,%d", rt.name(), im);
+ MOZ_ASSERT(im == 0); // not implemented for anything other than lnia
+
+ return writeInst(PPC_addpcis | (rt.code() << 21));
+ }
+
+// Conveniences and generally accepted alternate mnemonics.
+// XXX: change these to xs_
+BufferOffset Assembler::xs_trap()
+{
+ spew("trap @ %08x", currentOffset());
+ return writeInst(PPC_trap);
+}
+// trap with metadata encoded as register
+BufferOffset Assembler::xs_trap_tagged(TrapTag tag) {
+ uint32_t tv = PPC_trap | ((uint8_t)tag << 16) | ((uint8_t)tag << 11);
+ spew("trap @ %08x ; MARK %d %08x", currentOffset(), (uint8_t)tag, tv);
+ return writeInst(tv);
+}
+Assembler::TrapTag InstImm::traptag() {
+ // Extract a tag from a tagged trap instruction.
+ uint8_t r = ((data & 0x001f0000) >> 16);
+ MOZ_ASSERT(isOpcode(PPC_tw)); // not a trap
+ MOZ_ASSERT(r == ((data & 0x0000f800) >> 11)); // probably not a tagged trap
+ return (Assembler::TrapTag)(r & 0xfe); // mask bit 0
+}
+
+BufferOffset Assembler::xs_mr(Register rd, Register ra)
+{
+ return as_or(rd, ra, ra);
+}
+
+BufferOffset Assembler::x_beq(CRegisterID cr, int16_t off, LikelyBit lkb, LinkBit lb)
+{
+ return as_bc(off, Equal, cr, lkb, lb);
+}
+
+BufferOffset Assembler::x_bne(CRegisterID cr, int16_t off, LikelyBit lkb, LinkBit lb)
+{
+ return as_bc(off, NotEqual, cr, lkb, lb);
+}
+
+BufferOffset Assembler::xs_bdnz(int16_t off, LikelyBit lkb, LinkBit lb)
+{
+ spew("bdnz .+%d", off);
+ MOZ_ASSERT(!(off & 0x03));
+ return writeInst(PPC_bc | (0x10 << 21) | (off & 0xfffc) | lkb << 21 | lb);
+}
+
+// Emit specialized bcl form to avoid tainting branch history.
+// Link bit is implied; this is meaningless without it.
+BufferOffset Assembler::xs_bcl_always(int16_t off, LikelyBit lkb)
+{
+ spew("bcl 20,4*cr7+so,.+%d", off);
+ MOZ_ASSERT(!(off & 0x03));
+ return writeInst(PPC_bc | (20 << 21) | (31 << 16) | (off & 0xfffc) | 0x01);
+}
+
+BufferOffset Assembler::xs_mtctr(Register ra)
+{
+ return as_mtspr(ctr, ra);
+}
+
+BufferOffset Assembler::xs_mtlr(Register ra)
+{
+ return as_mtspr(lr_spr, ra);
+}
+
+BufferOffset Assembler::xs_mflr(Register rd)
+{
+ return as_mfspr(rd, lr_spr);
+}
+
+BufferOffset Assembler::xs_mtcr(Register rs)
+{
+ return as_mtcrf(0xff, rs);
+}
+
+BufferOffset Assembler::xs_mfxer(Register ra)
+{
+ return as_mfspr(ra, xer);
+}
+
+BufferOffset Assembler::xs_mtxer(Register ra)
+{
+ return as_mtspr(xer, ra);
+}
+
+BufferOffset Assembler::x_bit_value(Register rd, Register rs, unsigned bit)
+{
+ return as_rlwinm(rd, rs, bit + 1, 31, 31);
+}
+
+BufferOffset Assembler::x_slwi(Register rd, Register rs, int n)
+{
+ return as_rlwinm(rd, rs, n, 0, 31 - n);
+}
+
+BufferOffset Assembler::x_sldi(Register rd, Register rs, int n)
+{
+ return as_rldicr(rd, rs, n, 63 - n);
+}
+
+BufferOffset Assembler::x_srwi(Register rd, Register rs, int n)
+{
+ return as_rlwinm(rd, rs, 32 - n, n, 31);
+}
+
+BufferOffset Assembler::x_srdi(Register rd, Register rs, int n)
+{
+ return as_rldicl(rd, rs, 64 - n, n);
+}
+
+BufferOffset Assembler::x_subi(Register rd, Register ra, int16_t im)
+{
+ return as_addi(rd, ra, -im);
+}
+
+BufferOffset Assembler::xs_li(Register rd, int16_t im)
+{
+ return as_addi(rd, r0, im, true /* actually_li */);
+}
+
+BufferOffset Assembler::xs_lis(Register rd, int16_t im)
+{
+ return as_addis(rd, r0, im, true /* actually_lis */);
+}
+
+BufferOffset Assembler::x_not(Register rd, Register ra)
+{
+ return as_nor(rd, ra, ra);
+}
+
+BufferOffset Assembler::xs_sr_mulli(Register rd, Register ra, int16_t im)
+{
+ // XXX: expand this
+ if (im == -1) {
+ return as_neg(rd, ra);
+ }
+ if (im == 0) {
+ return xs_li(rd, 0);
+ }
+ if (im == 1) {
+ if (ra != rd) {
+ return xs_mr(rd, ra);
+ }
+ return BufferOffset(currentOffset());
+ }
+ if (im == 2) {
+ return as_add(rd, ra, ra);
+ }
+ if (im == 3 && rd != ra) {
+ as_add(rd, ra, ra);
+ return as_add(rd, rd, ra);
+ }
+ // XXX: convert 2^x to bit shift
+
+ return as_mulli(rd, ra, im);
+}
+
+// Traps
+BufferOffset Assembler::as_tw(uint8_t to, Register ra, Register rb)
+{
+ return writeInst(PPC_tw | (uint32_t)to << 21 | ra.code() << 16 |
+ rb.code() << 11);
+}
+
+BufferOffset Assembler::as_twi(uint8_t to, Register ra, int16_t si)
+{
+ return writeInst(PPC_twi | (uint32_t)to << 21 | ra.code() << 16 | si);
+}
+
+BufferOffset Assembler::as_stop()
+{
+ spew("stop!!!\n");
+ return writeInst(PPC_stop);
+}
diff -r 485b15bb4a20 -r 9c245f4665be js/src/jit/ppc64/Assembler-ppc64.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/js/src/jit/ppc64/Assembler-ppc64.h Thu Aug 08 21:23:52 2024 -0700
@@ -0,0 +1,1847 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_ppc_Assembler_ppc_h
+#define jit_ppc_Assembler_ppc_h
+
+/* Mostly derived from TenFourFox IonPower (r.i.p.). */
+
+#include "mozilla/ArrayUtils.h"
+#include "mozilla/Attributes.h"
+#include "mozilla/MathAlgorithms.h"
+
+#include "jit/CompactBuffer.h"
+#include "jit/JitCode.h"
+#include "jit/JitSpewer.h"
+#include "jit/ppc64/Architecture-ppc64.h"
+#include "jit/shared/Assembler-shared.h"
+#include "jit/shared/IonAssemblerBuffer.h"
+
+#if DEBUG
+#define ispew(x) JitSpew(JitSpew_Codegen, "== " x " ==")
+#else
+#define ispew(x) ;
+#endif
+
+namespace js {
+namespace jit {
+
+static constexpr Register r0{ Registers::r0 };
+static constexpr Register r1{ Registers::r1 };
+static constexpr Register sp{ Registers::r1 };
+static constexpr Register r2{ Registers::r2 };
+static constexpr Register r3{ Registers::r3 };
+static constexpr Register r4{ Registers::r4 };
+static constexpr Register r5{ Registers::r5 };
+static constexpr Register r6{ Registers::r6 };
+static constexpr Register r7{ Registers::r7 };
+static constexpr Register r8{ Registers::r8 };
+static constexpr Register r9{ Registers::r9 };
+static constexpr Register r10{ Registers::r10 };
+static constexpr Register r11{ Registers::r11 };
+static constexpr Register r12{ Registers::r12 };
+static constexpr Register r13{ Registers::r13 };
+static constexpr Register r14{ Registers::r14 };
+static constexpr Register r15{ Registers::r15 };
+static constexpr Register r16{ Registers::r16 };
+static constexpr Register r17{ Registers::r17 };
+static constexpr Register r18{ Registers::r18 };
+static constexpr Register r19{ Registers::r19 };
+static constexpr Register r20{ Registers::r20 };
+static constexpr Register r21{ Registers::r21 };
+static constexpr Register r22{ Registers::r22 };
+static constexpr Register r23{ Registers::r23 };
+static constexpr Register r24{ Registers::r24 };
+static constexpr Register r25{ Registers::r25 };
+static constexpr Register r26{ Registers::r26 };
+static constexpr Register r27{ Registers::r27 };
+static constexpr Register r28{ Registers::r28 };
+static constexpr Register r29{ Registers::r29 };
+static constexpr Register r30{ Registers::r30 };
+static constexpr Register r31{ Registers::r31 };
+
+static constexpr FloatRegister f0{ FloatRegisters::f0, FloatRegisters::Double };
+static constexpr FloatRegister f1{ FloatRegisters::f1, FloatRegisters::Double };
+static constexpr FloatRegister f2{ FloatRegisters::f2, FloatRegisters::Double };
+static constexpr FloatRegister f3{ FloatRegisters::f3, FloatRegisters::Double };
+static constexpr FloatRegister f4{ FloatRegisters::f4, FloatRegisters::Double };
+static constexpr FloatRegister f5{ FloatRegisters::f5, FloatRegisters::Double };
+static constexpr FloatRegister f6{ FloatRegisters::f6, FloatRegisters::Double };
+static constexpr FloatRegister f7{ FloatRegisters::f7, FloatRegisters::Double };
+static constexpr FloatRegister f8{ FloatRegisters::f8, FloatRegisters::Double };
+static constexpr FloatRegister f9{ FloatRegisters::f9, FloatRegisters::Double };
+static constexpr FloatRegister f10{ FloatRegisters::f10, FloatRegisters::Double };
+static constexpr FloatRegister f11{ FloatRegisters::f11, FloatRegisters::Double };
+static constexpr FloatRegister f12{ FloatRegisters::f12, FloatRegisters::Double };
+static constexpr FloatRegister f13{ FloatRegisters::f13, FloatRegisters::Double };
+static constexpr FloatRegister f14{ FloatRegisters::f14, FloatRegisters::Double };
+static constexpr FloatRegister f15{ FloatRegisters::f15, FloatRegisters::Double };
+static constexpr FloatRegister f16{ FloatRegisters::f16, FloatRegisters::Double };
+static constexpr FloatRegister f17{ FloatRegisters::f17, FloatRegisters::Double };
+static constexpr FloatRegister f18{ FloatRegisters::f18, FloatRegisters::Double };
+static constexpr FloatRegister f19{ FloatRegisters::f19, FloatRegisters::Double };
+static constexpr FloatRegister f20{ FloatRegisters::f20, FloatRegisters::Double };
+static constexpr FloatRegister f21{ FloatRegisters::f21, FloatRegisters::Double };
+static constexpr FloatRegister f22{ FloatRegisters::f22, FloatRegisters::Double };
+static constexpr FloatRegister f23{ FloatRegisters::f23, FloatRegisters::Double };
+static constexpr FloatRegister f24{ FloatRegisters::f24, FloatRegisters::Double };
+static constexpr FloatRegister f25{ FloatRegisters::f25, FloatRegisters::Double };
+static constexpr FloatRegister f26{ FloatRegisters::f26, FloatRegisters::Double };
+static constexpr FloatRegister f27{ FloatRegisters::f27, FloatRegisters::Double };
+static constexpr FloatRegister f28{ FloatRegisters::f28, FloatRegisters::Double };
+static constexpr FloatRegister f29{ FloatRegisters::f29, FloatRegisters::Double };
+static constexpr FloatRegister f30{ FloatRegisters::f30, FloatRegisters::Double };
+static constexpr FloatRegister f31{ FloatRegisters::f31, FloatRegisters::Double };
+// The rest of the FPRs are the business of the allocator, not the assembler.
+// SPRs and CRs are defined in their respective enums (see Architecture-ppc.h).
+
+static constexpr Register OsrFrameReg = r6;
+static constexpr Register ArgumentsRectifierReg = r19;
+static constexpr Register CallTempReg0 = r8;
+static constexpr Register CallTempReg1 = r9;
+static constexpr Register CallTempReg2 = r10;
+static constexpr Register CallTempReg3 = r7;
+static constexpr Register CallTempReg4 = r5; // Bad things! Try not to use these!
+static constexpr Register CallTempReg5 = r6;
+
+static constexpr Register InterpreterPCReg = r17;
+
+// irregexp
+static constexpr Register IntArgReg0 = r3;
+static constexpr Register IntArgReg1 = r4;
+static constexpr Register IntArgReg2 = r5;
+static constexpr Register IntArgReg3 = r6;
+static constexpr Register IntArgReg4 = r7;
+static constexpr Register IntArgReg5 = r8;
+static constexpr Register IntArgReg6 = r9;
+static constexpr Register IntArgReg7 = r10;
+
+static constexpr Register GlobalReg = r23; // used by AsmJS. Allocatable, but non-volatile. Must not clash with wasm.
+static constexpr Register HeapReg = r24; // Ditto.
+
+// These are defined, but not actually used, at least by us (see GetTempRegForIntArg).
+static constexpr Register CallTempNonArgRegs[] = { r10, r9, r8, r7 };
+static const uint32_t NumCallTempNonArgRegs = mozilla::ArrayLength(CallTempNonArgRegs);
+
+class ABIArgGenerator
+{
+ uint32_t stackOffset_;
+ uint32_t usedGPRs_;
+ uint32_t usedFPRs_;
+ ABIArg current_;
+
+ public:
+ ABIArgGenerator();
+ ABIArg next(MIRType argType);
+ ABIArg &current() { return current_; }
+
+ uint32_t stackBytesConsumedSoFar() const { return stackOffset_; }
+ void increaseStackOffset(uint32_t bytes) { stackOffset_ += bytes; }
+};
+
+static constexpr Register ABINonArgReg0 = r19;
+static constexpr Register ABINonArgReg1 = r20;
+static constexpr Register ABINonArgReg2 = r21;
+static constexpr Register ABINonArgReg3 = r22;
+// These can be non-volatile; they are only used by Wasm, and only after
+// all registers have been spilled. These must not be argregs as they may
+// be used after all argregs are exhausted.
+static constexpr Register ABINonArgReturnReg0 = r29;
+static constexpr Register ABINonArgReturnReg1 = r30;
+static constexpr Register ABINonArgReturnVolatileReg = r11;
+static constexpr Register ABINonVolatileReg = r14;
+
+static constexpr Register PreBarrierReg = r4;
+
+static constexpr Register InvalidReg{ Registers::invalid_reg };
+static constexpr FloatRegister InvalidFloatReg;
+
+static constexpr Register StackPointer = sp;
+static constexpr Register FramePointer = r31;
+
+static constexpr Register ScratchRegister = r0;
+static constexpr Register SecondScratchReg = r12;
+static constexpr Register ThirdScratchReg = r11; // EMERGENCY! RESCUE r11!
+
+// All return registers must be allocatable.
+static constexpr Register JSReturnReg_Type = r6;
+static constexpr Register JSReturnReg_Data = r5;
+static constexpr Register JSReturnReg = r4;
+static constexpr Register ReturnReg = r3;
+static constexpr Register64 ReturnReg64{ReturnReg};
+static constexpr FloatRegister ReturnFloat32Reg = {FloatRegisters::f1,
+ FloatRegisters::Single};
+static constexpr FloatRegister ReturnDoubleReg = {FloatRegisters::f1,
+ FloatRegisters::Double};
+static constexpr FloatRegister ABINonArgDoubleReg = {FloatRegisters::f14,
+ FloatRegisters::Double};
+static constexpr ValueOperand JSReturnOperand = ValueOperand(JSReturnReg);
+
+// Registers used by RegExpMatcher and RegExpExecMatch stubs (do not use
+// JSReturnOperand).
+static constexpr Register RegExpMatcherRegExpReg = CallTempReg0;
+static constexpr Register RegExpMatcherStringReg = CallTempReg1;
+static constexpr Register RegExpMatcherLastIndexReg = CallTempReg2;
+
+// Registers used by RegExpExecTest stub (do not use ReturnReg).
+static constexpr Register RegExpExecTestRegExpReg = CallTempReg0;
+static constexpr Register RegExpExecTestStringReg = CallTempReg1;
+
+// Registers used by RegExpSearcher stub (do not use ReturnReg).
+static constexpr Register RegExpSearcherRegExpReg = CallTempReg0;
+static constexpr Register RegExpSearcherStringReg = CallTempReg1;
+static constexpr Register RegExpSearcherLastIndexReg = CallTempReg2;
+
+// TLS pointer argument register for WebAssembly functions. This must not alias
+// any other register used for passing function arguments or return values.
+// Preserved by WebAssembly functions.
+static constexpr Register InstanceReg = r18;
+
+// Registers used for wasm table calls. These registers must be disjoint
+// from the ABI argument registers, WasmTlsReg and each other.
+static constexpr Register WasmTableCallScratchReg0 = ABINonArgReg0;
+static constexpr Register WasmTableCallScratchReg1 = ABINonArgReg1;
+static constexpr Register WasmTableCallSigReg = ABINonArgReg2;
+static constexpr Register WasmTableCallIndexReg = ABINonArgReg3;
+
+// Registers used for ref calls.
+static constexpr Register WasmCallRefCallScratchReg0 = ABINonArgReg0;
+static constexpr Register WasmCallRefCallScratchReg1 = ABINonArgReg1;
+static constexpr Register WasmCallRefReg = ABINonArgReg3;
+
+// Register used as a scratch along the return path in the fast js -> wasm stub
+// code. This must not overlap ReturnReg, JSReturnOperand, or WasmTlsReg. It
+// must be a volatile register.
+static constexpr Register WasmJitEntryReturnScratch = r10;
+
+static constexpr uint32_t WasmCheckedCallEntryOffset = 0u;
+static constexpr uint32_t WasmCheckedTailEntryOffset = 32u; // damn mtspr
+
+// Good grief. Must FPRs be vector registers on every architecture?
+// I guess we could only support SIMD on processors with VSX.
+static constexpr FloatRegister ReturnSimdReg = InvalidFloatReg;
+static constexpr FloatRegister ReturnSimd128Reg = InvalidFloatReg;
+static constexpr FloatRegister ReturnInt32x4Reg = InvalidFloatReg;
+static constexpr FloatRegister ReturnFloat32x4Reg = InvalidFloatReg;
+static constexpr FloatRegister ScratchSimdReg = InvalidFloatReg;
+static constexpr FloatRegister ScratchSimd128Reg = InvalidFloatReg;
+
+static constexpr FloatRegister ScratchFloat32Reg = {FloatRegisters::f0,
+ FloatRegisters::Single};
+static constexpr FloatRegister ScratchDoubleReg = {FloatRegisters::f0,
+ FloatRegisters::Double};
+
+struct ScratchFloat32Scope : public AutoFloatRegisterScope {
+ explicit ScratchFloat32Scope(MacroAssembler& masm)
+ : AutoFloatRegisterScope(masm, ScratchFloat32Reg) {}
+};
+struct ScratchDoubleScope : public AutoFloatRegisterScope {
+ explicit ScratchDoubleScope(MacroAssembler& masm)
+ : AutoFloatRegisterScope(masm, ScratchDoubleReg) {}
+};
+
+// A bias applied to the GlobalReg to allow the use of instructions with small
+// negative immediate offsets which doubles the range of global data that can be
+// accessed with a single instruction. (XXX)
+static const int32_t AsmJSGlobalRegBias = 32768;
+
+// Registers used in the GenerateFFIIonExit Enable Activation block. (Mirror MIPS.)
+static constexpr Register AsmJSIonExitRegCallee = r7;
+static constexpr Register AsmJSIonExitRegE0 = r3;
+static constexpr Register AsmJSIonExitRegE1 = r4;
+static constexpr Register AsmJSIonExitRegE2 = r5;
+static constexpr Register AsmJSIonExitRegE3 = r6;
+
+// Registers used in the GenerateFFIIonExit Disable Activation block.
+static constexpr Register AsmJSIonExitRegReturnData = JSReturnReg_Data;
+static constexpr Register AsmJSIonExitRegReturnType = JSReturnReg_Type;
+static constexpr Register AsmJSIonExitRegD0 = r3;
+static constexpr Register AsmJSIonExitRegD1 = r4;
+static constexpr Register AsmJSIonExitRegD2 = r7;
+
+static const uint32_t ABIStackAlignment = 16;
+static const uint32_t CodeAlignment = 16;
+// Ion code only. The 8-alignment is necessary because frames always have the return address
+// at the top of the stack and are not padded (see the common frame layout in JitFrames.h).
+static const uint32_t StackAlignment = 8;
+static const uint32_t JitStackAlignment = 16;
+static const uint32_t JitStackValueAlignment = 2;
+
+// Helper classes for ScratchRegister usage. Asserts that only one piece
+// of code thinks it has exclusive ownership of each scratch register.
+struct ScratchRegisterScope : public AutoRegisterScope {
+ explicit ScratchRegisterScope(MacroAssembler& masm)
+ : AutoRegisterScope(masm, ScratchRegister) {}
+};
+struct SecondScratchRegisterScope : public AutoRegisterScope {
+ explicit SecondScratchRegisterScope(MacroAssembler& masm)
+ : AutoRegisterScope(masm, SecondScratchReg) {}
+};
+
+// Future.
+static constexpr bool SupportsSimd = false;
+static constexpr uint32_t SimdStackAlignment = 16;
+static constexpr uint32_t SimdMemoryAlignment = 16;
+static constexpr uint32_t AsmJSStackAlignment = 16;
+
+static constexpr uint32_t WasmStackAlignment = SimdStackAlignment;
+
+static const uint32_t WasmTrapInstructionLength = 4;
+
+static const Scale ScalePointer = TimesEight;
+
+enum PPCOpcodes {
+ // Some we don't use yet (but we will).
+ PPC_add = 0x7C000214, // add
+ PPC_addc = 0x7C000014, // add carrying
+ PPC_adde = 0x7C000114, // add extended
+ PPC_addi = 0x38000000, // add immediate
+ PPC_addic = 0x30000000, // add immediate carrying
+ PPC_addis = 0x3C000000, // add immediate shifted
+ PPC_addme = 0x7C0001D4, // add -1 extended
+ PPC_addo = 0x7C000614, // add & OE=1 (can set OV)
+ PPC_addpcis = 0x4C000004, // load next instruction address into register
+ PPC_addze = 0x7C000194, // add zero extended
+ PPC_and = 0x7C000038, // and
+ PPC_andc = 0x7C000078, // and with compliment
+ PPC_andi = 0x70000000, // and immediate
+ PPC_andis = 0x74000000, // and immediate shifted
+ PPC_b = 0x48000000, // branch
+ PPC_bc = 0x40000000, // branch conditional
+ PPC_bctr = 0x4E800420, // branch to CTR (+/- LR)
+ PPC_bcctr = 0x4C000420, // branch conditional to count register
+ PPC_blr = 0x4E800020, // branch to link register
+ PPC_cmpd = 0x7C200000, // compare
+ PPC_cmpdi = 0x2C200000, // compare immediate
+ PPC_cmpld = 0x7C200040, // compare logical
+ PPC_cmpldi = 0x28200000, // compare logical immediate
+ PPC_cmpw = 0x7C000000, // compare
+ PPC_cmpwi = 0x2C000000, // compare immediate
+ PPC_cmplw = 0x7C000040, // compare logical
+ PPC_cmplwi = 0x28000000, // compare logical immediate
+ PPC_cntlzd = 0x7C000074, // count leading zeroes
+ PPC_cntlzw = 0x7C000034, // count leading zeroes
+ PPC_cnttzd = 0x7C000474, // count leading zeroes
+ PPC_cnttzw = 0x7C000434, // count leading zeroes
+ PPC_crand = 0x4C000202, // condition register and
+ PPC_crandc = 0x4C000102, // condition register and-with-complement
+ PPC_cror = 0x4C000382, // condition register or
+ PPC_crorc = 0x4C000342, // condition register or-with-complement
+ PPC_crxor = 0x4C000182, // condition register xor
+ PPC_divd = 0x7C0003D2, // integer divide
+ PPC_divdo = 0x7C0007D2, // integer divide & OE=1 (can set OV)
+ PPC_divdu = 0x7C000392, // integer divide unsigned
+ PPC_divduo = 0x7C000792, // integer divide unsigned & OE=1 (can set OV)
+ PPC_divw = 0x7C0003D6, // integer divide
+ PPC_divwo = 0x7C0007D6, // integer divide & OE=1 (can set OV)
+ PPC_divwu = 0x7C000396, // integer divide unsigned
+ PPC_divwuo = 0x7C000796, // integer divide unsigned & OE=1 (can set OV)
+ PPC_eieio = 0x7C0006AC, // enforce in-order execution of I/O
+ PPC_eqv = 0x7C000238, // equivalence operator
+ PPC_extsb = 0x7C000774, // extend sign byte
+ PPC_extsh = 0x7C000734, // extend sign halfword
+ PPC_extsw = 0x7C0007B4, // extend sign word
+ PPC_fabs = 0xFC000210, // floating absolute value (double precision)
+ PPC_fadd = 0xFC00002A, // floating add (double precision)
+ PPC_fadds = 0xEC00002A, // floating add (single precision)
+ PPC_fcpsgn = 0xFC000010, // floating copy sign
+ PPC_fcfid = 0xFC00069C, // floating convert from integer doubleword
+ PPC_fcfids = 0xEC00069C, // floating convert from integer doubleword SP
+ PPC_fcfidu = 0xFC00079C, // floating convert from integer doubleword US
+ PPC_fcfidus = 0xEC00079C, // floating convert from integer DW (SP+US)
+ PPC_fcmpo = 0xFC000040, // floating compare unordered
+ PPC_fcmpu = 0xFC000000, // floating compare unordered
+ PPC_fctid = 0xFC00065C, // floating convert to integer (to -Inf)
+ PPC_fctidu = 0xFC00075C, // floating convert to integer doubleword unsigned
+ PPC_fctidz = 0xFC00065E, // floating convert to integer DW signed (to zero)
+ PPC_fctiduz = 0xFC00075E, // floating convert to integer USDW (to zero)
+ PPC_fctiw = 0xFC00001C, // floating convert to integer (to -Inf)
+ PPC_fctiwu = 0xFC00011C, // floating convert to integer (to -Inf)
+ PPC_fctiwuz = 0xFC00011E, // floating convert to integer (to zero)
+ PPC_fctiwz = 0xFC00001E, // floating convert to integer (to zero)
+ PPC_fdiv = 0xFC000024, // floating divide (double precision)
+ PPC_fdivs = 0xEC000024, // floating divide (single precision)
+ PPC_fmr = 0xFC000090, // floating move register
+ PPC_fmrgew = 0xFC00078C, // floating merge even word
+ PPC_fmul = 0xFC000032, // floating multiply (double precision)
+ PPC_fmuls = 0xEC000032, // floating multiply (single precision)
+ PPC_fneg = 0xFC000050, // floating negate
+ PPC_frim = 0xFC0003d0, // floating round to integer minus
+ PPC_frin = 0xFC000310, // floating round to integer nearest
+ PPC_frip = 0xFC000390, // floating round to integer plus
+ PPC_friz = 0xFC000350, // floating round to integer toward zero
+ PPC_frsp = 0xFC000018, // convert to single precision
+ PPC_fsel = 0xFC00002E, // floating point select
+ PPC_fsub = 0xFC000028, // floating subtract (double precision)
+ PPC_fsubs = 0xEC000028, // floating subtract (single precision)
+ PPC_fsqrt = 0xFC00002C, // floating square root (double)
+ PPC_fsqrts = 0xEC00002C, // floating square root (double)
+ PPC_frsqrte = 0xFC000034, // floating reciprocal square root estimate
+ PPC_fnmsub = 0xFC00003C, // floating fused negative multiply-subtract
+ PPC_fmadd = 0xFC00003A, // floating fused multiply-add
+ PPC_isel = 0x7C00001E, // integer select
+ PPC_isync = 0x4C00012C, // instruction synchronize
+ PPC_lbz = 0x88000000, // load byte and zero
+ PPC_lbzx = 0x7C0000AE, // load byte and zero indexed
+ PPC_ld = 0xE8000000, // load doubleword
+ PPC_ldarx = 0x7C0000A8, // load doubleword indexed
+ PPC_ldx = 0x7C00002A, // load doubleword indexed
+ PPC_lfd = 0xC8000000, // load floating point double
+ PPC_lfdx = 0x7C0004AE, // load floating-point double indexed
+ PPC_lfiwax = 0x7C0006AE, // load floating-point as integer word algebraic indexed
+ PPC_lfiwzx = 0x7C0006EE, // load floating-point as integer word algebraic indexed
+ PPC_lfs = 0xC0000000, // load single precision float
+ PPC_lfsx = 0x7C00042E, // load single precision float indexed
+ PPC_lha = 0xA8000000, // load halfword algebraic
+ PPC_lhax = 0x7C0002AE, // load halfword algebraic indexed
+ PPC_lhz = 0xA0000000, // load halfword and zero
+ PPC_lhzx = 0x7C00022E, // load halfword and zero indexed
+ PPC_lhbrx = 0x7C00062C, // load hw and zero indexed (byte swapped)
+ PPC_lwa = 0xE8000002, // load word algebraic
+ PPC_lwax = 0x7C0002AA, // load word algebraic indexed
+ PPC_lwarx = 0x7c000028, // load word and reserve indexed
+ PPC_lwz = 0x80000000, // load word and zero
+ PPC_lwzx = 0x7C00002E, // load word and zero indexed
+ PPC_lwbrx = 0x7C00042C, // load word and zero indexed (byte swapped)
+ PPC_mcrxrx = 0x7C000480, // move XER[OV, OV32, CA, CA32] to CR[0-3]
+ PPC_mcrf = 0x4C000000, // move CR[0-3] to CR[0-3]
+ PPC_mcrfs = 0xFC000080, // move FPSCR fields to CR
+ PPC_mfcr = 0x7C000026, // move from condition register
+ PPC_mfocrf = 0x7C100120, // move from one condition register field
+ PPC_mffs = 0xFC00048E, // move from fpscr to fpr
+ PPC_mfspr = 0x7C0002A6, // move from spr (special purpose register)
+ PPC_mfvsrd = 0x7C000066, // move from VSR doubleword (used for FPR)
+ PPC_modsd = 0x7C000612, // integer remainder 64-bit (signed)
+ PPC_modud = 0x7c000212, // integer remainder 64-bit (unsigned)
+ PPC_modsw = 0x7C000616, // integer remainder (signed)
+ PPC_moduw = 0x7c000216, // integer remainder (unsigned)
+ PPC_mtcrf = 0x7C000120, // move to condition register field
+ PPC_mtfsb0 = 0xFC00008C, // move zero bit into FPSCR
+ PPC_mtfsb1 = 0xFC00004C, // move one bit into FPSCR
+ PPC_mtfsfi = 0xFC00010C, // move 4-bit immediate into FPSCR field
+ PPC_mtvsrd = 0x7C000166, // move to VSR doubleword (used for FPR)
+ PPC_mtvsrws = 0x7C000326, // move to VSR word and splat (used for FPR)
+ PPC_mtvsrwz = 0x7C0001E6, // move to VSR word and zero (used for FPR)
+ PPC_mtspr = 0x7C0003A6, // move to spr
+ PPC_mulhd = 0x7C000092, // multiply high signed doubleword
+ PPC_mulhdu = 0x7C000012, // multiply high signed doubleword
+ PPC_mulhw = 0x7C000096, // multiply high signed
+ PPC_mulhwu = 0x7C000016, // multiply high unsigned
+ PPC_mulli = 0x1C000000, // multiply low immediate
+ PPC_mulld = 0x7C0001D2, // multiply low doubleword
+ PPC_mulldo = 0x7C0005D2, // multiply low doubleword
+ PPC_mullw = 0x7C0001D6, // multiply low word
+ PPC_mullwo = 0x7C0005D6, // multiply low word with overflow
+ PPC_nand = 0x7C0003B8, // nand
+ PPC_neg = 0x7C0000D0, // negate
+ PPC_nego = 0x7C0004D0, // negate & OE=1 (can set OV)
+ PPC_nor = 0x7C0000F8, // nor
+ PPC_or = 0x7C000378, // or
+ PPC_ori = 0x60000000, // or immediate
+ PPC_oris = 0x64000000, // or immediate shifted
+ PPC_popcntb = 0x7C0000F4, // population count doubleword
+ PPC_popcntd = 0x7C0003F4, // population count doubleword
+ PPC_popcntw = 0x7C0002F4, // population count doubleword
+ PPC_rldcl = 0x78000010, // rotate left doubleword then clear left
+ PPC_rldicl = 0x78000000, // rotate left doubleword immediate then clear left
+ PPC_rldcr = 0x78000012, // rotate left doubleword then clear right
+ PPC_rldicr = 0x78000004, // rotate left doubleword immediate then clear right
+ PPC_rldimi = 0x7800000C, // rotate left doubleword immediate then mask insert
+ PPC_rlwimi = 0x50000000, // rotate left word imm then mask insert
+ PPC_rlwinm = 0x54000000, // rotate left word imm then and with mask
+ PPC_rlwnm = 0x5C000000, // rotate left word then AND with mask
+ PPC_sld = 0x7C000036, // shift left doubleword
+ PPC_slw = 0x7C000030, // shift left word
+ PPC_srad = 0x7C000634, // shift right algebraic doubleword (sign ext)
+ PPC_sradi = 0x7C000674, // shift right algebraic doubleword immediate
+ PPC_sraw = 0x7C000630, // shift right algebraic word (sign ext)
+ PPC_srawi = 0x7C000670, // shift right algebraic word immediate
+ PPC_srd = 0x7C000436, // shift right doubleword (zero ext)
+ PPC_srw = 0x7C000430, // shift right word (zero ext)
+ PPC_stb = 0x98000000, // store byte
+ PPC_stbx = 0x7C0001AE, // store byte indexed
+ PPC_std = 0xF8000000, // store doubleword
+ PPC_stdcx = 0x7C0001AD, // store doubleword conditional indexed
+ PPC_stdu = 0xF8000001, // store doubleword with update
+ PPC_stdux = 0x7C00016A, // store doubleword with update indexed
+ PPC_stdx = 0x7C00012A, // store doubleword indexed
+ PPC_stfd = 0xD8000000, // store floating-point double
+ PPC_stfdu = 0xDC000000, // store floating-point double with update
+ PPC_stfdx = 0x7C0005AE, // store floating-point double indexed
+ PPC_stfiwx = 0x7C0007AE, // Store floating-point as integer word indexed
+ PPC_stfs = 0xD0000000, // store floating-point single
+ PPC_stfsu = 0xD4000000, // store floating-point single
+ PPC_stfsx = 0x7C00052E, // store floating-point single indexed
+ PPC_sth = 0xB0000000, // store halfword
+ PPC_sthx = 0x7C00032E, // store halfword indexed
+ PPC_sthbrx = 0x7C00072C, // store halfword indexed (byte swapped)
+ PPC_stop = 0x4C0002E4, // wasm-specific trap word (see note)
+ PPC_stw = 0x90000000, // store word
+ PPC_stwu = 0x94000000, // store word with update
+ PPC_stwux = 0x7C00016E, // store word with update indexed
+ PPC_stwx = 0x7C00012E, // store word indexed
+ PPC_stwbrx = 0x7C00052C, // store word indexed (byte swapped)
+ PPC_stwcx = 0x7C00012D, // store word indexed
+ PPC_subf = 0x7C000050, // subtract from
+ PPC_subfc = 0x7C000010, // subtract from with carry
+ PPC_subfe = 0x7C000110, // subtract from extended
+ PPC_subfic = 0x20000000, // subtract from immediate
+ PPC_subfze = 0x7C000190, // subtract from zero extended
+ PPC_subfo = 0x7C000450, // subtract from with overflow
+ PPC_sync = 0x7C0004AC, // sync
+#if defined(__APPLE__) || defined(__linux__)
+ PPC_trap = 0x7FE00008, // trap word (extended from tw 31,r0,r0)
+#elif defined(__FreeBSD__)
+ PPC_trap = 0x7C810808, // trap word (tweq r1, r1)
+#else
+#error Specify the trap word for your PPC operating system
+#endif
+ PPC_tw = 0x7C000008, // trap word immediate
+ PPC_twi = 0x0C000000, // trap word immediate
+ PPC_xor = 0x7C000278, // xor
+ PPC_xori = 0x68000000, // xor immediate
+ PPC_xoris = 0x6C000000, // xor immediate shifted
+ PPC_xscvdpsp= 0xF0000424, // VSX scalar convert double to single (for FPR)
+ PPC_xscvdpspn=0xF000042C, // VSX scalar convert double to single sNaN-pres
+ PPC_xscvspdp= 0xF0000524, // VSX scalar convert single to double (for FPR)
+ PPC_xscvspdpn=0xF000052C, // VSX scalar convert single to double sNaN-pres
+ PPC_xxbrd = 0xF017076C, // VSX byte-reverse doubleword
+ PPC_xxlxor = 0xF00004D0, // VSX logical XOR (I love this mnemonic)
+
+ // simplified mnemonics
+ PPC_mr = PPC_or,
+ PPC_not = PPC_nor,
+ PPC_nop = PPC_ori,
+ PPC_lwsync = PPC_sync | (1 << 21),
+
+ PPC_MAJOR_OPCODE_MASK = 0xFC000000 // AND with this to get some idea of the opcode
+};
+
+class Instruction;
+class InstImm;
+class MacroAssemblerPPC;
+class Operand;
+
+// A BOffImm16 is a 16 bit (signed or unsigned) immediate that is used for branches.
+class BOffImm16
+{
+ int32_t data;
+
+ public:
+ uint32_t encode() {
+ MOZ_ASSERT(!isInvalid());
+ return (uint32_t)data & 0xFFFC;
+ }
+ int32_t decode() {
+ MOZ_ASSERT(!isInvalid());
+ return data;
+ }
+
+ explicit BOffImm16(int offset)
+ {
+ MOZ_ASSERT((offset & 0x3) == 0);
+ MOZ_ASSERT(IsInRange(offset) || IsInSignedRange(offset));
+ data = offset;
+ }
+ static bool IsInRange(int offset) {
+ if (offset > 65535)
+ return false;
+ if (offset < 0)
+ return false;
+ return true;
+ }
+ static bool IsInSignedRange(int offset) {
+ if (offset > 32767)
+ return false;
+ if (offset < -32767)
+ return false;
+ return true;
+ }
+ static const int32_t INVALID = 0x00020000;
+ BOffImm16()
+ : data(INVALID)
+ { }
+
+ bool isInvalid() {
+ return data == INVALID;
+ }
+ Instruction *getDest(Instruction *src);
+
+ BOffImm16(InstImm inst);
+};
+
+// A JOffImm26 is a 26 bit signed immediate that is used for unconditional jumps.
+class JOffImm26
+{
+ int32_t data;
+
+ public:
+ uint32_t encode() {
+ MOZ_ASSERT(!isInvalid());
+ return (uint32_t)data & 0x03FFFFFC;
+ }
+ int32_t decode() {
+ MOZ_ASSERT(!isInvalid());
+ return data;
+ }
+
+ explicit JOffImm26(int offset)
+ {
+ MOZ_ASSERT((offset & 0x3) == 0);
+ MOZ_ASSERT(IsInRange(offset));
+ data = offset;
+ }
+ static bool IsInRange(int offset) {
+ if (offset < -33554431)
+ return false;
+ if (offset > 33554431)
+ return false;
+ return true;
+ }
+ static const int32_t INVALID = 0x20000000;
+ JOffImm26()
+ : data(INVALID)
+ { }
+
+ bool isInvalid() {
+ return data == INVALID;
+ }
+ Instruction *getDest(Instruction *src);
+
+};
+
+class Imm16
+{
+ int32_t value;
+
+ public:
+ Imm16();
+ Imm16(uint32_t imm)
+ : value(imm)
+ {
+ }
+ uint32_t encode() {
+ return (uint32_t)value & 0xffff;
+ }
+ int32_t decodeSigned() {
+ return value;
+ }
+ uint32_t decodeUnsigned() {
+ return value;
+ }
+ static bool IsInSignedRange(int32_t imm) {
+ return imm >= INT16_MIN && imm <= INT16_MAX;
+ }
+ static bool IsInUnsignedRange(uint32_t imm) {
+ return imm <= UINT16_MAX ;
+ }
+ static Imm16 Lower (Imm32 imm) {
+ return Imm16(imm.value & 0xffff);
+ }
+ static Imm16 Upper (Imm32 imm) {
+ return Imm16((imm.value >> 16) & 0xffff);
+ }
+};
+
+class Imm8
+{
+ uint8_t value;
+
+ public:
+ Imm8();
+ Imm8(uint32_t imm) : value(imm) {}
+ uint32_t encode(uint32_t shift) { return value << shift; }
+ int32_t decodeSigned() { return value; }
+ uint32_t decodeUnsigned() { return value; }
+ static bool IsInSignedRange(int32_t imm) {
+ return imm >= INT8_MIN && imm <= INT8_MAX;
+ }
+ static bool IsInUnsignedRange(uint32_t imm) { return imm <= UINT8_MAX; }
+ static Imm8 Lower(Imm16 imm) { return Imm8(imm.decodeSigned() & 0xff); }
+ static Imm8 Upper(Imm16 imm) {
+ return Imm8((imm.decodeSigned() >> 8) & 0xff);
+ }
+};
+
+class Operand
+{
+ public:
+ enum Tag {
+ REG,
+ FREG,
+ MEM
+ };
+
+ private:
+ Tag tag : 3;
+ uint32_t reg : 5; // XXX. This really should be Register::Code, but then float regs ...
+ int32_t offset;
+
+ public:
+ Operand (Register reg_)
+ : tag(REG), reg(reg_.code())
+ { }
+
+ Operand (FloatRegister freg)
+ : tag(FREG), reg(freg.code())
+ { }
+
+ Operand (Register base, Imm32 off)
+ : tag(MEM), reg(base.code()), offset(off.value)
+ { }
+
+ Operand (Register base, int32_t off)
+ : tag(MEM), reg(base.code()), offset(off)
+ { }
+
+ Operand (const Address &addr)
+ : tag(MEM), reg(addr.base.code()), offset(addr.offset)
+ { }
+
+ Tag getTag() const {
+ return tag;
+ }
+
+ Register toReg() const {
+ MOZ_ASSERT(tag == REG);
+ return Register::FromCode((Register::Code)reg);
+ }
+
+ FloatRegister toFReg() const {
+ MOZ_ASSERT(tag == FREG);
+ return FloatRegister::FromCode((FloatRegister::Code)reg);
+ }
+
+ void toAddr(Register *r, Imm32 *dest) const {
+ MOZ_ASSERT(tag == MEM);
+ *r = Register::FromCode((Register::Code)reg);
+ *dest = Imm32(offset);
+ }
+ Address toAddress() const {
+ MOZ_ASSERT(tag == MEM);
+ return Address(Register::FromCode((Register::Code)reg), offset);
+ }
+ int32_t disp() const {
+ MOZ_ASSERT(tag == MEM);
+ return offset;
+ }
+
+ int32_t base() const {
+ MOZ_ASSERT(tag == MEM);
+ return reg;
+ }
+ Register baseReg() const {
+ MOZ_ASSERT(tag == MEM);
+ return Register::FromCode((Register::Code)reg);
+ }
+};
+
+inline Imm32
+Imm64::firstHalf() const
+{
+ return hi(); // ENDIAN!
+}
+
+inline Imm32
+Imm64::secondHalf() const
+{
+ return low(); // ENDIAN!
+}
+
+class Assembler;
+typedef js::jit::AssemblerBuffer<1024, Instruction> PPCBuffer;
+
+class PPCBufferWithExecutableCopy : public PPCBuffer
+{
+ static const int SliceSize = 1024;
+ public:
+ void executableCopy(uint8_t* buffer) {
+ if (this->oom())
+ return;
+
+ for (Slice* cur = head; cur != nullptr; cur = cur->getNext()) {
+ memcpy(buffer, &cur->instructions, cur->length());
+ buffer += cur->length();
+ }
+ }
+
+ bool appendRawCode(const uint8_t* code, size_t numBytes) {
+ if (this->oom()) {
+ return false;
+ }
+ while (numBytes > SliceSize) {
+ this->putBytes(SliceSize, code);
+ numBytes -= SliceSize;
+ code += SliceSize;
+ }
+ this->putBytes(numBytes, code);
+ return !this->oom();
+ }
+};
+
+class Assembler : public AssemblerShared
+{
+ public:
+ enum TrapTag { // FreeBSD and others may use r1 in their trap word, so don't allow bit 0 or > 15.
+ BTag = 2,
+ BCTag = 4,
+ CallTag = 6,
+ DebugTag0 = 10,
+ DebugTag1 = 12,
+ DebugTag2 = 14
+ };
+
+ enum BranchBits {
+ BranchOnClear = 0x04,
+ BranchOnSet = 0x0c,
+ BranchOptionMask = 0x0f,
+ BranchOptionInvert = 0x08 // XOR with this to invert the sense of a Condition
+ };
+
+ enum Condition {
+ // Bit flag for unsigned comparisons (remember that you have to
+ // choose the type of comparison at the compare step, not the
+ // branch). We just mask this bit off, but the MacroAsssembler
+ // may use it as a flag. This is a synthetic code.
+ ConditionUnsigned = 0x100, // Computation only
+ ConditionUnsignedHandled = 0x2ff, // Mask off bit 8 but not 9 or 0-7
+
+ // Bit flag for zero-relative Conditions. These are treated as
+ // equivalent conditions relative to 0, but the MacroAssembler and
+ // CodeGenerator may use this bit to reason about the intent of
+ // generated instructions. This is a synthetic code.
+ ConditionZero = 0x400, // Computation only
+
+ // Bit flag for XER-only codes. We need to have XER in the CR using
+ // mcrxrx or an equivalent first, but we don't need to check any CR
+ // bits otherwise. This is a synthetic code. We use the 32-bit flags.
+ ConditionOnlyXER = 0x200, // Computation only
+ ConditionXERCA = 0x23c, // CA32 same as SO bit
+ ConditionXERNCA = 0x234,
+ ConditionXEROV = 0x21c, // OV32 same as GT bit
+
+ // These are off pp370-1 in OPPCC. The top nybble is the offset
+ // to the CR field (the x in BIF*4+x), and the bottom is the BO.
+ // Synthetic condition flags sit in the MSB.
+ Equal = 0x2c,
+ NotEqual = 0x24,
+ GreaterThan = 0x1c,
+ GreaterThanOrEqual = 0x04,
+ LessThan = 0x0c,
+ LessThanOrEqual = 0x14,
+
+ Above = GreaterThan | ConditionUnsigned,
+ AboveOrEqual = GreaterThanOrEqual | ConditionUnsigned,
+ Below = LessThan | ConditionUnsigned,
+ BelowOrEqual = LessThanOrEqual | ConditionUnsigned,
+
+ Signed = LessThan | ConditionZero,
+ // Don't mention negative zero to me. Don't wanna hear it. Nope.
+ NotSigned = GreaterThanOrEqual | ConditionZero,
+ Zero = Equal | ConditionZero,
+ NonZero = NotEqual | ConditionZero,
+
+ // Hard hints that we need an unsigned compare.
+ BitEqual = Equal | ConditionUnsigned,
+ BitNotEqual = NotEqual | ConditionUnsigned,
+
+ Overflow = ConditionXEROV,
+ CarrySet = ConditionXERCA,
+ CarryClear = ConditionXERNCA,
+
+ Always = 0x1f,
+
+ // This is specific to the SO bits in the CR, not the general overflow
+ // condition in the way Ion conceives of it.
+ SOBit = 0x3c,
+ NSOBit = 0x34
+ };
+
+ enum DoubleCondition {
+ DoubleConditionUnordered = 0x100, // Computation only. This is also synthetic.
+
+ // These conditions will only evaluate to true if the comparison is ordered - i.e. neither operand is NaN.
+ DoubleOrdered = 0x34,
+ DoubleEqual = 0x2c,
+ DoubleNotEqual = 0x24,
+ DoubleGreaterThan = 0x1c,
+ DoubleGreaterThanOrEqual = 0x04,
+ DoubleLessThan = 0x0c,
+ DoubleLessThanOrEqual = 0x14,
+ // If either operand is NaN, these conditions always evaluate to true.
+ // Except for DoubleUnordered, synthetic condition flags sit in the MSB
+ // and are masked off by us but may be used by the MacroAssembler.
+ DoubleUnordered = 0x3c,
+ DoubleEqualOrUnordered = DoubleEqual | DoubleConditionUnordered,
+ DoubleNotEqualOrUnordered = DoubleNotEqual | DoubleConditionUnordered,
+ DoubleGreaterThanOrUnordered = DoubleGreaterThan | DoubleConditionUnordered,
+ DoubleGreaterThanOrEqualOrUnordered = DoubleGreaterThanOrEqual | DoubleConditionUnordered,
+ DoubleLessThanOrUnordered = DoubleLessThan | DoubleConditionUnordered,
+ DoubleLessThanOrEqualOrUnordered = DoubleLessThanOrEqual | DoubleConditionUnordered,
+ };
+
+ enum JumpOrCall { BranchIsJump, BranchIsCall };
+
+ enum LinkBit {
+ DontLinkB = 0,
+ LinkB = 1,
+ };
+
+ enum LikelyBit {
+ NotLikelyB = 0,
+ LikelyB = 1,
+ };
+
+ enum BranchAddressType {
+ RelativeBranch = 0,
+ AbsoluteBranch = 2,
+ };
+
+ BufferOffset nextOffset() {
+ return m_buffer.nextOffset();
+ }
+
+ enum FloatFormat { SingleFloat, DoubleFloat };
+ enum FloatTestKind { TestForTrue, TestForFalse };
+
+ protected:
+ Instruction * editSrc (BufferOffset bo) {
+ return m_buffer.getInst(bo);
+ }
+ public:
+ uint32_t actualOffset(uint32_t) const;
+ uint32_t actualIndex(uint32_t) const;
+ static uint8_t *PatchableJumpAddress(JitCode *code, uint32_t index);
+ static uint64_t ExtractLoad64Value(Instruction *inst);
+ static void UpdateLoad64Value(Instruction *inst0, uint64_t value);
+ static void WriteLoad64Instructions(Instruction* inst0, Register reg,
+ uint64_t value);
+ protected:
+
+ // structure for fixing up pc-relative loads/jumps when a the machine code
+ // gets moved (executable copy, gc, etc.)
+ struct RelativePatch
+ {
+ // the offset within the code buffer where the value is loaded that
+ // we want to fix-up
+ BufferOffset offset;
+ void *target;
+ RelocationKind kind;
+
+ RelativePatch(BufferOffset offset, void *target, RelocationKind kind)
+ : offset(offset),
+ target(target),
+ kind(kind)
+ { }
+ };
+
+ //js::Vector<CodeLabel, 0, SystemAllocPolicy> codeLabels_;
+ js::Vector<RelativePatch, 8, SystemAllocPolicy> jumps_;
+ //js::Vector<uint32_t, 8, SystemAllocPolicy> longJumps_;
+
+ CompactBufferWriter jumpRelocations_;
+ CompactBufferWriter dataRelocations_;
+ CompactBufferWriter relocations_;
+ CompactBufferWriter preBarriers_;
+
+ PPCBufferWithExecutableCopy m_buffer;
+
+ private:
+ char const * nGPR(Register rreg)
+ {
+ uint32_t reg = rreg.code();
+ MOZ_ASSERT(reg <= 31);
+ //MOZ_ASSERT(reg >= 0);
+ static char const *names[] = {
+ "r0", "sp", "r2", "r3", "r4", "r5", "r6", "r7",
+ "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15",
+ "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23",
+ "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31"
+ };
+ return names[reg];
+ }
+
+ char const * nFPR(FloatRegister freg)
+ {
+ uint32_t reg = freg.code();
+ MOZ_ASSERT(reg <= 31);
+ //MOZ_ASSERT(reg >= 0);
+ static char const *names[] = {
+ "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7",
+ "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15",
+ "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",
+ "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31"
+ };
+ return names[reg];
+ }
+
+ char const * nCR(CRegisterID reg)
+ {
+ MOZ_ASSERT(reg <= 7);
+ MOZ_ASSERT(reg >= 0);
+ static char const *names[] = {
+ "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7"
+ };
+ return names[reg];
+ }
+
+ char const * nSPR(SPRegisterID reg)
+ {
+ // XXX: we don't handle VRSAVE with this, but we don't use it yet.
+ MOZ_ASSERT(reg >= 1);
+ MOZ_ASSERT(reg <= 9);
+ static char const *names[] = {
+ "", "xer", "", "", "", "", "", "", "lr", "ctr"
+ };
+ return names[reg];
+ }
+
+ public: // used by MacroAssembler
+ // Which absolute bit number does a condition register + Condition pair
+ // refer to?
+ static uint8_t crBit(CRegisterID cr, Condition cond)
+ {
+ return (cr << 2) + ((cond & 0xf0) >> 4);
+ }
+
+ static uint8_t crBit(CRegisterID cr, DoubleCondition cond)
+ {
+ return (cr << 2) + ((cond & 0xf0) >> 4);
+ }
+
+ public:
+ Assembler()
+ : m_buffer(),
+ isFinished(false)
+ { }
+
+ void setUnlimitedBuffer() { m_buffer.setUnlimited(); }
+ static Condition InvertCondition(Condition cond);
+ static DoubleCondition InvertCondition(DoubleCondition cond);
+
+ // MacroAssemblers hold onto gcthings, so they are traced by the GC.
+ void trace(JSTracer *trc);
+ void writeRelocation(BufferOffset src) {
+ jumpRelocations_.writeUnsigned(src.getOffset());
+ }
+
+#if(1)
+ // As opposed to the x86/x64 version, the data relocation must be executed
+ // beforehand to recover the pointer, not after.
+ void writeDataRelocation(ImmGCPtr ptr) {
+ if (ptr.value) {
+ if (gc::IsInsideNursery(ptr.value))
+ embedsNurseryPointers_ = true;
+ dataRelocations_.writeUnsigned(nextOffset().getOffset());
+ }
+ }
+#endif
+
+ void writePrebarrierOffset(CodeOffset label) {
+ preBarriers_.writeUnsigned(label.offset());
+ }
+
+ public:
+ static uintptr_t GetPointer(uint8_t *);
+
+ bool oom() const;
+
+ void setPrinter(Sprinter *sp) {
+ }
+
+ static const Register getStackPointer() {
+ // This is stupid.
+ return StackPointer;
+ }
+
+ private:
+ bool isFinished;
+ public:
+#if defined(DEBUG)
+ void spew_with_address(const char *fmt, uint32_t ins, ...);
+#endif
+ void finish();
+ bool appendRawCode(const uint8_t* code, size_t numBytes);
+ void executableCopy(void *buffer);
+ void copyJumpRelocationTable(uint8_t *dest);
+ void copyDataRelocationTable(uint8_t *dest);
+ void copyPreBarrierTable(uint8_t *dest);
+
+/*
+ size_t numCodeLabels() const {
+ return codeLabels_.length();
+ }
+ CodeLabel codeLabel(size_t i) {
+ return codeLabels_[i];
+ }
+*/
+
+ // Size of the instruction stream, in bytes.
+ size_t size() const;
+ // Size of the jump relocation table, in bytes.
+ size_t jumpRelocationTableBytes() const;
+ size_t dataRelocationTableBytes() const;
+ size_t preBarrierTableBytes() const;
+
+ // Size of the data table, in bytes.
+ size_t bytesNeeded() const;
+
+ // Write a blob of binary into the instruction stream *or*
+ // into a destination address. If dest is nullptr (the default), then the
+ // instruction gets written into the instruction stream. If dest is not null
+ // it is interpreted as a pointer to the location that we want the
+ // instruction to be written.
+ BufferOffset writeInst(uint32_t x, uint32_t *dest = nullptr);
+ // A static variant for the cases where we don't want to have an assembler
+ // object at all. Normally, you would use the dummy (nullptr) object.
+ static void WriteInstStatic(uint32_t x, uint32_t *dest);
+
+ public:
+ BufferOffset align(int alignment, bool useTrap = false);
+
+ BufferOffset as_nop();
+ BufferOffset as_eieio();
+ BufferOffset as_isync();
+ BufferOffset xs_lwsync();
+ BufferOffset as_sync();
+
+ // Branch and jump instructions.
+ uint16_t computeConditionCode(Condition op, CRegisterID cr = cr0);
+ uint16_t computeConditionCode(DoubleCondition cond, CRegisterID cr = cr0);
+ BufferOffset as_b(JOffImm26 off, BranchAddressType bat = RelativeBranch, LinkBit lb = DontLinkB);
+ BufferOffset as_b(int32_t off, BranchAddressType bat = RelativeBranch, LinkBit lb = DontLinkB); // stubs into the above
+ BufferOffset as_blr(LinkBit lb = DontLinkB);
+ BufferOffset as_bctr(LinkBit lb = DontLinkB);
+
+ // Conditional branches.
+ BufferOffset as_bc(BOffImm16 off, Condition cond, CRegisterID cr = cr0, LikelyBit lkb = NotLikelyB, LinkBit lb = DontLinkB);
+ BufferOffset as_bc(int16_t off, Condition cond, CRegisterID cr = cr0, LikelyBit lkb = NotLikelyB, LinkBit lb = DontLinkB);
+ BufferOffset as_bc(BOffImm16 off, DoubleCondition cond, CRegisterID = cr0, LikelyBit lkb = NotLikelyB, LinkBit lb = DontLinkB);
+ BufferOffset as_bc(int16_t off, DoubleCondition cond, CRegisterID = cr0, LikelyBit lkb = NotLikelyB, LinkBit lb = DontLinkB);
+ BufferOffset as_bcctr(Condition cond, CRegisterID cr = cr0, LikelyBit lkb = NotLikelyB, LinkBit lb = DontLinkB);
+ BufferOffset as_bcctr(DoubleCondition cond, CRegisterID cr = cr0, LikelyBit lkb = NotLikelyB, LinkBit lb = DontLinkB);
+
+ BufferOffset as_bc(int16_t off, uint16_t op, LikelyBit lkb = NotLikelyB, LinkBit lb = DontLinkB);
+ BufferOffset as_bcctr(uint16_t op, LikelyBit lkb = NotLikelyB, LinkBit lb = DontLinkB);
+
+
+ // SPR operations.
+ BufferOffset as_mtspr(SPRegisterID spr, Register ra);
+ BufferOffset as_mfspr(Register rd, SPRegisterID spr);
+
+ // CR operations.
+#define DEF_CRCR(op) BufferOffset as_##op(uint8_t t, uint8_t a, uint8_t b);
+
+ DEF_CRCR(crand)
+ DEF_CRCR(crandc)
+ DEF_CRCR(cror)
+ DEF_CRCR(crorc)
+ DEF_CRCR(crxor)
+#undef DEF_CRCR
+ BufferOffset as_mtcrf(uint32_t mask, Register rs);
+ BufferOffset as_mfcr(Register rd);
+ BufferOffset as_mfocrf(Register rd, CRegisterID crfs);
+ BufferOffset as_mcrxrx(CRegisterID crt);
+
+ // GPR operations and load-stores.
+ BufferOffset as_neg(Register rd, Register rs);
+ BufferOffset as_nego(Register rd, Register rs);
+
+ BufferOffset as_cmpd(CRegisterID cr, Register ra, Register rb);
+ BufferOffset as_cmpdi(CRegisterID cr, Register ra, int16_t im);
+ BufferOffset as_cmpld(CRegisterID cr, Register ra, Register rb);
+ BufferOffset as_cmpldi(CRegisterID cr, Register ra, int16_t im);
+ BufferOffset as_cmpw(CRegisterID cr, Register ra, Register rb);
+ BufferOffset as_cmpwi(CRegisterID cr, Register ra, int16_t im);
+ BufferOffset as_cmplw(CRegisterID cr, Register ra, Register rb);
+ BufferOffset as_cmplwi(CRegisterID cr, Register ra, int16_t im);
+ BufferOffset as_cmpd(Register ra, Register rb); // all implied cr0
+ BufferOffset as_cmpdi(Register ra, int16_t im);
+ BufferOffset as_cmpld(Register ra, Register rb);
+ BufferOffset as_cmpldi(Register ra, int16_t im);
+ BufferOffset as_cmpw(Register ra, Register rb);
+ BufferOffset as_cmpwi(Register ra, int16_t im);
+ BufferOffset as_cmplw(Register ra, Register rb);
+ BufferOffset as_cmplwi(Register ra, int16_t im);
+
+ BufferOffset as_srawi(Register id, Register rs, uint8_t n);
+
+ BufferOffset as_rldcl(Register ra, Register rs, Register rb, uint8_t mb);
+ BufferOffset as_rldcl_rc(Register ra, Register rs, Register rb, uint8_t mb);
+ BufferOffset as_rldicl(Register ra, Register rs, uint8_t sh, uint8_t mb);
+ BufferOffset as_rldicl_rc(Register ra, Register rs, uint8_t sh, uint8_t mb);
+ BufferOffset as_rldicr(Register ra, Register rs, uint8_t sh, uint8_t mb);
+ BufferOffset as_rldicr_rc(Register ra, Register rs, uint8_t sh, uint8_t mb);
+ BufferOffset as_rlwinm(Register rd, Register rs, uint8_t sh, uint8_t mb, uint8_t me);
+ BufferOffset as_rlwinm_rc(Register rd, Register rs, uint8_t sh, uint8_t mb, uint8_t me);
+ BufferOffset as_rlwimi(Register rd, Register rs, uint8_t sh, uint8_t mb, uint8_t me); // cracked on G5
+ BufferOffset as_rldimi(Register rd, Register rs, uint8_t sh, uint8_t mb);
+ BufferOffset as_rlwnm(Register rd, Register rs, Register rb, uint8_t mb, uint8_t me);
+ BufferOffset as_sradi(Register rd, Register rs, int n);
+
+#define DEF_ALU2(op) BufferOffset as_##op(Register rd, Register ra, Register rb); \
+ BufferOffset as_##op##_rc(Register rd, Register ra, Register rb);
+ DEF_ALU2(add)
+ DEF_ALU2(addc)
+ DEF_ALU2(adde)
+ DEF_ALU2(addo)
+ DEF_ALU2(subf)
+ DEF_ALU2(subfc)
+ DEF_ALU2(subfe)
+ DEF_ALU2(subfo)
+ DEF_ALU2(divd)
+ DEF_ALU2(divdo)
+ DEF_ALU2(divdu)
+ DEF_ALU2(divduo)
+ DEF_ALU2(divw)
+ DEF_ALU2(divwo)
+ DEF_ALU2(divwu)
+ DEF_ALU2(divwuo)
+ DEF_ALU2(mulld)
+ DEF_ALU2(mulhd)
+ DEF_ALU2(mulhdu)
+ DEF_ALU2(mulldo)
+ DEF_ALU2(mullw)
+ DEF_ALU2(mulhw)
+ DEF_ALU2(mulhwu)
+ DEF_ALU2(mullwo)
+ DEF_ALU2(eqv) // NB: Implemented differently.
+#undef DEF_ALU2
+
+#define DEF_ALU2_NORC(op) BufferOffset as_##op(Register rt, Register ra, Register rb);
+ DEF_ALU2_NORC(modsd)
+ DEF_ALU2_NORC(modud)
+ DEF_ALU2_NORC(modsw)
+ DEF_ALU2_NORC(moduw)
+#undef DEF_ALU2_NORC
+
+// Special handling due to mscdfr0 (no _rc)
+BufferOffset as_addi(Register rd, Register ra, int16_t im, bool actually_li = false);
+BufferOffset as_addis(Register rd, Register ra, int16_t im, bool actually_lis = false);
+
+#define DEF_ALUI(op) BufferOffset as_##op(Register rd, Register ra, int16_t im); \
+ BufferOffset as_##op##_rc(Register rd, Register ra, int16_t im);
+ DEF_ALUI(addic)
+ // NB: mulli is usually strength-reduced, since it can take up to five
+ // cycles in the worst case. See xs_sr_mulli.
+ DEF_ALUI(mulli)
+ DEF_ALUI(subfic)
+#undef DEF_ALUI
+
+#define DEF_ALUE(op) BufferOffset as_##op(Register rd, Register ra); \
+ BufferOffset as_##op##_rc(Register rd, Register ra);
+ DEF_ALUE(addme)
+ DEF_ALUE(addze)
+ DEF_ALUE(subfze)
+ DEF_ALUE(cntlzw) // NB: In this case, rd = ra and ra = rs, but no biggie here.
+ DEF_ALUE(cntlzd) // NB: In this case, rd = ra and ra = rs, but no biggie here.
+ DEF_ALUE(cnttzd) // NB: In this case, rd = ra and ra = rs, but no biggie here.
+ DEF_ALUE(cnttzw) // NB: In this case, rd = ra and ra = rs, but no biggie here.
+
+ BufferOffset as_popcntd(Register ra, Register rs);
+ BufferOffset as_popcntw(Register ra, Register rs);
+#undef DEF_ALUE
+
+#define DEF_BITALU2(op) BufferOffset as_##op(Register rd, Register rs, Register rb); \
+ BufferOffset as_##op##_rc(Register rd, Register rs, Register rb);
+ DEF_BITALU2(andc)
+ DEF_BITALU2(nand)
+ DEF_BITALU2(nor)
+ DEF_BITALU2(slw)
+ DEF_BITALU2(srw)
+ DEF_BITALU2(sraw)
+ DEF_BITALU2(sld)
+ DEF_BITALU2(srd)
+ DEF_BITALU2(srad)
+ DEF_BITALU2(and) // NB: See terminal _ constants above. This will have and_ and and__rc.
+ DEF_BITALU2(or)
+ DEF_BITALU2(xor)
+#undef DEF_BITALU2
+
+#define DEF_BITALUI(op) BufferOffset as_##op(Register rd, Register ra, uint16_t im);
+ // There is no Rc form for these instructions.
+ DEF_BITALUI(ori)
+ DEF_BITALUI(oris)
+ DEF_BITALUI(xori)
+ DEF_BITALUI(xoris)
+ // There is no Rc-less version of andi/andis.
+ DEF_BITALUI(andi_rc)
+ DEF_BITALUI(andis_rc)
+#undef DEF_BITALUI
+
+#define DEF_ALUEXT(op) BufferOffset as_##op(Register rd, Register rs); \
+ BufferOffset as_##op##_rc(Register rd, Register rs);
+ DEF_ALUEXT(extsb)
+ DEF_ALUEXT(extsh)
+ DEF_ALUEXT(extsw)
+#undef DEF_ALUEXT
+
+#define DEF_MEMd(op) BufferOffset as_##op(Register rd, Register rb, int16_t off);
+ DEF_MEMd(lbz)
+ DEF_MEMd(lha)
+ DEF_MEMd(lhz)
+ DEF_MEMd(lwa)
+ DEF_MEMd(lwz)
+ DEF_MEMd(ld)
+
+ DEF_MEMd(stb)
+ DEF_MEMd(stw)
+ DEF_MEMd(stwu)
+ DEF_MEMd(sth)
+ DEF_MEMd(std)
+ DEF_MEMd(stdu)
+#undef DEF_MEMd
+
+#define DEF_MEMx(op) BufferOffset as_##op(Register rd, Register ra, Register rb);
+ DEF_MEMx(lbzx)
+ DEF_MEMx(lhax)
+ DEF_MEMx(lhzx)
+ DEF_MEMx(lhbrx)
+ DEF_MEMx(lwzx)
+ DEF_MEMx(lwbrx)
+ DEF_MEMx(lwax)
+ DEF_MEMx(lwarx)
+ DEF_MEMx(ldx)
+ DEF_MEMx(ldarx)
+
+ DEF_MEMx(stbx)
+ DEF_MEMx(stwx)
+ DEF_MEMx(stwux)
+ DEF_MEMx(stwbrx)
+ DEF_MEMx(sthx)
+ DEF_MEMx(sthbrx)
+ DEF_MEMx(stdx)
+ DEF_MEMx(stdcx)
+ DEF_MEMx(stdux)
+ DEF_MEMx(stwcx)
+#undef DEF_MEMx
+
+ BufferOffset as_isel(Register rt, Register ra, Register rb, uint16_t rc, CRegisterID cr = cr0);
+ BufferOffset as_isel0(Register rt, Register ra, Register rb, uint16_t rc, CRegisterID cr = cr0);
+
+ // FPR operations and load-stores.
+ BufferOffset as_fcmpo(CRegisterID cr, FloatRegister ra, FloatRegister rb);
+ BufferOffset as_fcmpo(FloatRegister ra, FloatRegister rb); // implied cr0
+ BufferOffset as_fcmpu(CRegisterID cr, FloatRegister ra, FloatRegister rb);
+ BufferOffset as_fcmpu(FloatRegister ra, FloatRegister rb); // implied cr0
+#define DEF_FPUAC(op) BufferOffset as_##op(FloatRegister rd, FloatRegister ra, FloatRegister rc); \
+ BufferOffset as_##op##_rc(FloatRegister rd, FloatRegister ra, FloatRegister rc);
+ DEF_FPUAC(fmul)
+ DEF_FPUAC(fmuls)
+#undef DEF_FPUAC
+
+#define DEF_FPUAB(op) BufferOffset as_##op(FloatRegister rd, FloatRegister ra, FloatRegister rc); \
+ BufferOffset as_##op##_rc(FloatRegister rd, FloatRegister ra, FloatRegister rc);
+ DEF_FPUAB(fadd)
+ DEF_FPUAB(fdiv)
+ DEF_FPUAB(fsub)
+ DEF_FPUAB(fadds)
+ DEF_FPUAB(fdivs)
+ DEF_FPUAB(fsubs)
+ DEF_FPUAB(fcpsgn)
+ DEF_FPUAB(fmrgew)
+#undef DEF_FPUAB
+
+#define DEF_FPUDS(op) BufferOffset as_##op(FloatRegister rd, FloatRegister rs); \
+ BufferOffset as_##op##_rc(FloatRegister rd, FloatRegister rs);
+ DEF_FPUDS(fabs)
+ DEF_FPUDS(fneg)
+ DEF_FPUDS(fmr)
+ DEF_FPUDS(fcfid)
+ DEF_FPUDS(fcfids)
+ DEF_FPUDS(fcfidu)
+ DEF_FPUDS(fcfidus)
+ DEF_FPUDS(fctid)
+ DEF_FPUDS(fctidz)
+ DEF_FPUDS(fctidu)
+ DEF_FPUDS(fctiduz)
+ DEF_FPUDS(fctiw)
+ DEF_FPUDS(fctiwz)
+ DEF_FPUDS(fctiwu)
+ DEF_FPUDS(fctiwuz)
+ DEF_FPUDS(frim)
+ DEF_FPUDS(frin)
+ DEF_FPUDS(frip)
+ DEF_FPUDS(friz)
+ DEF_FPUDS(frsp)
+ DEF_FPUDS(frsqrte)
+ DEF_FPUDS(fsqrt)
+ DEF_FPUDS(fsqrts)
+#undef DEF_FPUDS
+
+// In Ion, the semantics for this macro are now corrected compared to JM/PPCBC.
+// (See OPPCC p.432, etc.)
+#define DEF_FPUABC(op) BufferOffset as_##op(FloatRegister rd, FloatRegister ra, FloatRegister rc, FloatRegister rb); \
+ BufferOffset as_##op##_rc(FloatRegister rd, FloatRegister ra, FloatRegister rc, FloatRegister rb);
+ DEF_FPUABC(fmadd)
+ DEF_FPUABC(fnmsub)
+ DEF_FPUABC(fsel)
+#undef DEF_FPUABC
+
+#define DEF_FMEMd(op) BufferOffset as_##op(FloatRegister rd, Register rb, int16_t off);
+ DEF_FMEMd(lfd)
+ DEF_FMEMd(lfs)
+ DEF_FMEMd(stfd)
+ DEF_FMEMd(stfs)
+ DEF_FMEMd(stfdu)
+ DEF_FMEMd(stfsu)
+#undef DEF_FMEMd
+
+#define DEF_FMEMx(op) BufferOffset as_##op(FloatRegister rd, Register ra, Register rb);
+ DEF_FMEMx(lfdx)
+ DEF_FMEMx(lfsx)
+ DEF_FMEMx(lfiwax)
+ DEF_FMEMx(stfiwx)
+ DEF_FMEMx(stfdx)
+ DEF_FMEMx(stfsx)
+#undef DEF_FMEMx
+
+// convert SPRid to 10-bit split encoding (OPPCC appendix A, p.514)
+#define PPC_SPR(x) (((int)x>>5) | ((int)x & 31)<<5)
+
+ BufferOffset as_mtfsb0(uint8_t bt);
+ BufferOffset as_mtfsb1(uint8_t bt);
+ BufferOffset as_mtfsfi(uint8_t fi, uint8_t imm);
+ BufferOffset as_mcrf(CRegisterID bt, CRegisterID bs);
+ BufferOffset as_mcrfs(CRegisterID bf, uint8_t bfa);
+
+// VSX
+// Currently supported only for FPRs.
+ BufferOffset as_mfvsrd(Register ra, FloatRegister xs);
+ BufferOffset as_mtvsrd(FloatRegister xs, Register ra);
+ BufferOffset as_mtvsrwz(FloatRegister xs, Register ra);
+ BufferOffset as_mtvsrws(FloatRegister xs, Register ra);
+ BufferOffset as_xxbrd(FloatRegister xt, FloatRegister xb);
+ BufferOffset as_xscvdpsp(FloatRegister xt, FloatRegister xb);
+ BufferOffset as_xscvspdp(FloatRegister xt, FloatRegister xb);
+ BufferOffset as_xscvdpspn(FloatRegister xt, FloatRegister xb);
+ BufferOffset as_xscvspdpn(FloatRegister xt, FloatRegister xb);
+ BufferOffset as_xxlxor(FloatRegister xt, FloatRegister xa, FloatRegister xb);
+
+ BufferOffset as_addpcis(Register rt, uint16_t im = 0);
+
+ // Conveniences and generally accepted alternate mnemonics.
+// XXX: change these to xs_ and remove ones we don't actually use
+ BufferOffset xs_trap();
+ BufferOffset xs_trap_tagged(TrapTag tag); // Codegen for marking traps in output.
+ BufferOffset xs_mr(Register rd, Register ra);
+ BufferOffset xs_bcl_always(int16_t off, LikelyBit lkb = NotLikelyB);
+ BufferOffset x_beq(CRegisterID cr, int16_t off, LikelyBit lkb = NotLikelyB, LinkBit lb = DontLinkB);
+ BufferOffset x_bne(CRegisterID cr, int16_t off, LikelyBit lkb = NotLikelyB, LinkBit lb = DontLinkB);
+ BufferOffset xs_bdnz(int16_t off, LikelyBit lkb = NotLikelyB, LinkBit lb = DontLinkB);
+ BufferOffset xs_mtctr(Register ra);
+ BufferOffset xs_mtlr(Register ra);
+ BufferOffset xs_mflr(Register rd);
+ BufferOffset xs_mtcr(Register rs);
+ BufferOffset xs_mfxer(Register ra);
+ BufferOffset xs_mtxer(Register ra);
+ BufferOffset x_insertbits0_15(Register rd, Register rs);
+ BufferOffset x_bit_value(Register rd, Register rs, unsigned bit);
+ BufferOffset x_slwi(Register rd, Register rs, int n);
+ BufferOffset x_sldi(Register rd, Register rs, int n);
+ BufferOffset x_srwi(Register rd, Register rs, int n);
+ BufferOffset x_srdi(Register rd, Register rs, int n);
+ BufferOffset x_subi(Register rd, Register ra, int16_t im);
+ BufferOffset xs_sr_mulli(Register rd, Register ra, int16_t im);
+
+ BufferOffset x_not(Register rd, Register ra);
+
+ // Large loads.
+ BufferOffset xs_li(Register rd, int16_t im);
+ BufferOffset xs_lis(Register rd, int16_t im);
+
+ // Traps
+ BufferOffset as_tw(uint8_t to, Register ra, Register rb);
+ BufferOffset as_twi(uint8_t to, Register ra, int16_t si);
+ BufferOffset as_stop();
+
+ // Label operations.
+ void bind(InstImm* inst, uintptr_t branch, uintptr_t target, bool bound = false);
+ void bind(Label* label) { bind(label, nextOffset()); }
+ void bind(Label* label, BufferOffset boff);
+ void bind(CodeLabel *label) { label->target()->bind(currentOffset()); }
+ uint32_t currentOffset() {
+ return nextOffset().getOffset();
+ }
+ void retarget(Label *label, Label *target);
+ static void Bind(uint8_t *rawCode, const CodeLabel& label);
+
+/* XXX: Remove bindS*, those are TenFourFox-specific and we aren't going to implement them */
+ // Fast fixed branches.
+#define SHORT_LABEL(w) BufferOffset w = nextOffset()
+#define SHORT_LABEL_MASM(w) BufferOffset w = masm.nextOffset()
+ // Binds instruction i to offset s as a fixed short branch.
+ void bindS(BufferOffset s, BufferOffset i);
+ void bindSS(BufferOffset i); // s is implied as the current offset.
+
+ // See Bind.
+ size_t labelOffsetToPatchOffset(size_t offset) {
+ return actualOffset(offset);
+ }
+
+ void call(Label *label);
+ void call(void *target);
+
+ static void TraceJumpRelocations(JSTracer *trc, JitCode *code, CompactBufferReader &reader);
+ static void TraceDataRelocations(JSTracer *trc, JitCode *code, CompactBufferReader &reader);
+ /*
+ static void FixupNurseryObjects(JSContext* cx, JitCode* code, CompactBufferReader& reader,
+ const ObjectVector& nurseryObjects);
+ */
+ void assertNoGCThings() const {
+#ifdef DEBUG
+ MOZ_ASSERT(dataRelocations_.length() == 0);
+/* XXX: This can only be uint32.
+ for (auto& j : longJumps_) {
+ MOZ_ASSERT(j.kind == RelocationKind::HARDCODED);
+ }
+*/
+ for (auto& j : jumps_) {
+ MOZ_ASSERT(j.kind == RelocationKind::HARDCODED);
+ }
+#endif
+ }
+
+ static bool SupportsFloatingPoint() {
+ // BaselineInterpreter is not enabled unless this is true, and nothing
+ // else is enabled if BaselineInterpreter isn't enabled, which yields
+ // a convenient way to disable the JIT at runtime if needed. (We also
+ // added code to disable irregexp JIT gated here as well.) Currently
+ // the JIT is only supported on ISA 3.0 and up (POWER9 and up) in
+ // little-endian mode. You can decide the manner in which you want the
+ // JIT handled:
+#if(0)
+ // Manually: unless you turn the JIT off, it's on. If you're writing
+ // support for a presently unsupported Power ISA CPU or big-endian
+ // mode, then change if(0) to if(1) and grab onto your ankles.
+ return true;
+#elif defined(__POWER9_VECTOR__) && defined(__LITTLE_ENDIAN__)
+ // Statically at compile time: if gcc is given --mcpu=power9, and it
+ // was compiled on a little-endian host, then the resulting binary will
+ // only work properly on ISA 3.0+ and thus the JIT must work too. All
+ // ISA checks become static as well, so it's also slightly faster.
+ return true;
+#elif defined(__LITTLE_ENDIAN__)
+ // Dynamically at runtime: ask the operating system. If AltiVec, VSX
+ // and VSX3 are all supported, then this must be a supported CPU. When
+ // additional CPU support is added, adjust the check here.
+ return HasPPCISA3();
+#else
+ // Otherwise, statically disable on all unsupported or big-endian CPUs.
+ return false;
+#endif
+ }
+ static bool SupportsSimd() { return false; } // todo
+
+ // Technically unaligned integer loads are supported in hardware, but
+ // there is a non-zero penalty (though small on many implementations),
+ // and certain unlikely edge cases can potentially fault to the operating
+ // system.
+ static bool SupportsUnalignedAccesses() { return true; }
+ static bool SupportsFastUnalignedAccesses() { return false; }
+
+ // However, 64-bit FP loads invariably fault to the operating system
+ // when unaligned, so we really want to avoid that.
+ static bool SupportsFastUnalignedFPAccesses() { return false; }
+
+ static bool HasRoundInstruction(RoundingMode mode) {
+ switch (mode) {
+ case RoundingMode::Up:
+ case RoundingMode::Down:
+ case RoundingMode::TowardsZero:
+ return true;
+
+ // See note in MacroAssembler::nearbyIntDouble.
+ case RoundingMode::NearestTiesToEven:
+ return false;
+
+ // fall through
+ }
+ MOZ_CRASH("unexpected mode");
+ }
+
+ protected:
+ void bind(Instruction *inst, uint32_t branch, uint32_t target);
+ void addPendingJump(BufferOffset src, ImmPtr target, RelocationKind kind) {
+ enoughMemory_ &= jumps_.append(RelativePatch(src, target.value, kind));
+ if (kind == RelocationKind::JITCODE)
+ writeRelocation(src);
+ }
+
+ void addLongJump(BufferOffset src, BufferOffset dst) {
+ //enoughMemory_ &= longJumps_.append(src.getOffset());
+ CodeLabel cl;
+ cl.patchAt()->bind(src.getOffset());
+ cl.target()->bind(dst.getOffset());
+ cl.setLinkMode(CodeLabel::JumpImmediate);
+ addCodeLabel(std::move(cl));
+ }
+
+ public:
+ /*
+ size_t numLongJumps() const {
+ return longJumps_.length();
+ }
+ uint32_t longJump(size_t i) {
+ return longJumps_[i];
+ }
+ */
+
+ void comment(const char *msg) {
+ }
+ // Copy the assembly code to the given buffer, and perform any pending
+ // relocations relying on the target address.
+ void executableCopy(uint8_t *buffer);
+
+ void flushBuffer() {
+ }
+
+ BufferOffset haltingAlign(int alignment);
+ BufferOffset nopAlign(int alignment);
+
+ static uint32_t PatchWrite_NearCallSize();
+ static uint32_t NopSize() { return 4; }
+
+ static uint32_t ExtractLisOriValue(Instruction *inst0, Instruction *inst1);
+ static void UpdateLisOriValue(Instruction *inst0, Instruction *inst1, uint32_t value);
+ static void WriteLisOriInstructions(Instruction *inst, Instruction *inst1,
+ Register reg, uint32_t value);
+
+ static void PatchWrite_NearCall(CodeLocationLabel start, CodeLocationLabel toCall);
+ static void PatchDataWithValueCheck(CodeLocationLabel label, PatchedImmPtr newValue,
+ PatchedImmPtr expectedValue);
+ static void PatchDataWithValueCheck(CodeLocationLabel label, ImmPtr newValue,
+ ImmPtr expectedValue);
+ static void PatchWrite_Imm32(CodeLocationLabel label, Imm32 imm);
+
+ static void PatchInstructionImmediate(uint8_t *code, PatchedImmPtr imm);
+
+ static uint8_t *NextInstruction(uint8_t *instruction, uint32_t *count = nullptr);
+
+ static void ToggleToJmp(CodeLocationLabel inst_);
+ static void ToggleToCmp(CodeLocationLabel inst_);
+
+ static void ToggleCall(CodeLocationLabel inst_, bool enabled);
+
+ static void UpdateBoundsCheck(uint32_t logHeapSize, Instruction *inst);
+ void processCodeLabels(uint8_t *rawCode);
+ static int32_t ExtractCodeLabelOffset(uint8_t *code);
+
+ void verifyHeapAccessDisassembly(uint32_t begin, uint32_t end,
+ const Disassembler::HeapAccess& heapAccess)
+ {
+ // Implement this if we implement a disassembler.
+ }
+
+ bool swapBuffer(wasm::Bytes& bytes);
+ bool reserve(size_t size) { return !oom(); }
+}; // Assembler
+
+static const uint32_t OpcodeShift = 26;
+static const uint32_t OpcodeBits = 6;
+
+class Instruction
+{
+ protected:
+ uint32_t data;
+
+ public:
+ Instruction (uint32_t data_) : data(data_) { }
+ Instruction (PPCOpcodes op_) : data((uint32_t)op_) { }
+
+ uint32_t encode() const {
+ return data;
+ }
+
+ // Quickies
+ void makeOp_nop() {
+ data = PPC_nop;
+ }
+ void makeOp_mtctr(Register r) {
+ data = PPC_mtspr | ((uint32_t)r.code())<<21 | PPC_SPR(ctr)<<11 ;
+ }
+ void makeOp_bctr(Assembler::LinkBit l = Assembler::DontLinkB) {
+ data = PPC_bctr | l;
+ }
+
+ void setData(uint32_t data) {
+ this->data = data;
+ }
+
+ const Instruction & operator=(const Instruction &src) {
+ data = src.data;
+ return *this;
+ }
+
+ uint32_t extractOpcode() {
+ return (data & PPC_MAJOR_OPCODE_MASK); // ">> 26"
+ }
+ bool isOpcode(uint32_t op) {
+ return (extractOpcode() == (op & PPC_MAJOR_OPCODE_MASK));
+ }
+ void assertOpcode(uint32_t op) {
+ MOZ_ASSERT(isOpcode(op));
+ }
+
+ // Get the next instruction in the instruction stream.
+ // This will do neat things like ignore constant pools and their guards,
+ // if we ever implement those again.
+ Instruction *next() { return this + 1; };
+
+ // Sometimes the backend wants a uint32_t (or a pointer to it) rather than
+ // an instruction. raw() just coerces this into a pointer to a uint32_t.
+ const uint32_t *raw() const { return &data; }
+ uint32_t size() const { return 4; }
+}; // Instruction
+
+class InstReg : public Instruction
+{
+ // Valid for reg/reg/reg instructions only.
+ // XXX: Assert that at some point.
+ public:
+ InstReg (PPCOpcodes op) : Instruction(op) { }
+ InstReg (PPCOpcodes op, Register rd, Register ra, Register rb)
+ : Instruction(op | ((uint32_t)rd.code() << 21) |
+ ((uint32_t)ra.code() << 16) | (uint32_t)rb.code() << 11) {}
+
+
+ void setBOffImm16(BOffImm16 off) {
+ data = (data & 0xFFFF0000) | off.encode();
+ }
+ void setImm16(Imm16 off) {
+ data = (data & 0xFFFF0000) | off.encode();
+ }
+ uint32_t extractImm16Value() {
+ return (data & 0x0000FFFF);
+ }
+ void setTargetReg(Register rt) {
+ data = (data & 0xFC1FFFFF) | ((uint32_t)rt.code() << 21);
+ }
+ void setUpperReg(Register ru) {
+ // Mask out upper register field and put in the new one.
+ // For addi/addis, this is the DESTINATION.
+ // For ori/oris, this is the SOURCE.
+ // For bc, this is BO.
+ data = (data & 0xFFE0FFFF) | ((uint32_t)ru.code() << 16);
+ }
+ void setLowerReg(Register rl) {
+ // Mask out lower register field and put in the new one.
+ // For addi/addis, this is the SOURCE. (For li/lis, this should be ZERO.)
+ // For ori/oris, this is the DESTINATION.
+ // For bc, this is BI.
+ data = (data & 0xFFFF07FF) | ((uint32_t)rl.code() << 11);
+ }
+}; // InstReg
+
+class InstImm : public Instruction
+{
+ // Valid for reg/reg/imm instructions only and bc.
+ // XXX: Assert that at some point.
+ public:
+ InstImm (PPCOpcodes op) : Instruction(op) { }
+ InstImm (PPCOpcodes op, Register ra, Register rs, Imm16 im)
+ : Instruction(op | ((uint32_t)ra.code() << 21) |
+ ((uint32_t)rs.code() << 16) | im.encode()) {}
+
+ void setBOffImm16(BOffImm16 off) {
+ data = (data & 0xFFFF0000) | off.encode();
+ }
+ void setImm16(Imm16 off) {
+ data = (data & 0xFFFF0000) | off.encode();
+ }
+ uint32_t extractImm16Value() {
+ return (data & 0x0000FFFF);
+ }
+ void setUpperReg(Register ru) {
+ // Mask out upper register field and put in the new one.
+ // For addi/addis, this is the DESTINATION.
+ // For ori/oris, this is the SOURCE.
+ // For bc, this is BO.
+ data = (data & 0xFC1FFFFF) | ((uint32_t)ru.code() << 21);
+ }
+ void setLowerReg(Register rl) {
+ // Mask out lower register field and put in the new one.
+ // For addi/addis, this is the SOURCE. (For li/lis, this should be ZERO.)
+ // For ori/oris, this is the DESTINATION.
+ // For bc, this is BI.
+ data = (data & 0xFFE0FFFF) | ((uint32_t)rl.code() << 16);
+ }
+ Assembler::TrapTag traptag();
+}; // InstImm
+
+// If this assert is not satisfied, we can't use Instruction to patch in-place.
+static_assert(sizeof(Instruction) == 4, "sizeof(Instruction) must be 32-bit word.");
+
+static const uint32_t NumIntArgRegs = 8;
+
+static inline bool
+GetIntArgReg(uint32_t usedArgSlots, Register *out)
+{
+ if (usedArgSlots < NumIntArgRegs) {
+ // Argregs start at r3!
+ *out = Register::FromCode((Register::Code)(3 + usedArgSlots));
+ return true;
+ }
+ return false;
+}
+
+// Get a register in which we plan to put a quantity that will be used as an
+// integer argument. Because we have so many argument registers on PowerPC, this
+// essentially stubs into GetIntArgReg because failure is incredibly improbable.
+static inline bool
+GetTempRegForIntArg(uint32_t usedIntArgs, uint32_t usedFloatArgs, Register *out)
+{
+ // NB: this implementation can't properly determine yet which regs are used if there are
+ // float arguments.
+ MOZ_ASSERT(usedFloatArgs == 0);
+
+ if (GetIntArgReg(usedIntArgs, out))
+ return true;
+ return false;
+}
+
+static inline uint32_t
+GetArgStackDisp(uint32_t usedArgSlots)
+{
+#if(0)
+ // NYI. This situation should never occur.
+ MOZ_ASSERT(usedArgSlots >= NumIntArgRegs);
+ return usedArgSlots * sizeof(intptr_t);
+#else
+ MOZ_CRASH("unexpected spill to stack");
+ return 0;
+#endif
+}
+
+inline bool IsUnaligned(const wasm::MemoryAccessDesc& access) {
+ if (!access.align()) {
+ return false;
+ }
+
+ return access.align() < access.byteSize();
+}
+
+} // namespace jit
+} // namespace js
+
+// Convenience macros from JM/PPCBC.
+
+// whether a (Trusted)Imm32 fits in an unsigned immediate value
+#define PPC_IMM_OK_U(x) (MOZ_LIKELY(((x).m_value & 0xffff0000) == 0))
+
+// whether a (Trusted)Imm32 fits in a signed immediate value
+#define PPC_IMM_OK_S(x) (MOZ_LIKELY(((x).m_value & 0xffff8000) == 0 || \
+ ((x).m_value & 0xffff8000) == 0xffff8000))
+
+// whether the offset part of an Address fits in a (signed) immediate value
+#define PPC_OFFS_OK(x) (MOZ_LIKELY(((x).offset & 0xffff8000) == 0 || \
+ ((x).offset & 0xffff8000) == 0xffff8000))
+
+// same test, but checking a bit ahead (for multiple loads)
+#define PPC_OFFS_INCR_OK(x, incr) (MOZ_LIKELY((((x).offset + incr) & 0xffff8000) == 0 || \
+ (((x).offset + incr) & 0xffff8000) == 0xffff8000))
+
+#endif /* jit_ppc_Assembler_ppc_h */
diff -r 485b15bb4a20 -r 9c245f4665be js/src/jit/ppc64/AtomicOperations-ppc64.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/js/src/jit/ppc64/AtomicOperations-ppc64.h Thu Aug 08 21:23:52 2024 -0700
@@ -0,0 +1,200 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* For documentation, see jit/AtomicOperations.h */
+
+#ifndef jit_ppc64le_AtomicOperations_ppc64le_h
+#define jit_ppc64le_AtomicOperations_ppc64le_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/Types.h"
+
+#include "builtin/AtomicsObject.h"
+#include "vm/ArrayBufferObject.h"
+
+#if !defined(__clang__) && !defined(__GNUC__)
+# error "This file only for gcc-compatible compilers"
+#endif
+
+inline bool
+js::jit::AtomicOperations::hasAtomic8()
+{
+ return true; // XXX?
+}
+
+inline bool
+js::jit::AtomicOperations::isLockfree8()
+{
+ MOZ_ASSERT(__atomic_always_lock_free(sizeof(int8_t), 0));
+ MOZ_ASSERT(__atomic_always_lock_free(sizeof(int16_t), 0));
+ MOZ_ASSERT(__atomic_always_lock_free(sizeof(int32_t), 0));
+ MOZ_ASSERT(__atomic_always_lock_free(sizeof(int64_t), 0));
+ return true;
+}
+
+inline void
+js::jit::AtomicOperations::fenceSeqCst()
+{
+ __atomic_thread_fence(__ATOMIC_SEQ_CST);
+}
+
+template<typename T>
+inline T
+js::jit::AtomicOperations::loadSeqCst(T* addr)
+{
+ static_assert(sizeof(T) <= sizeof(void*), "atomics supported up to pointer size only");
+ T v;
+ __atomic_load(addr, &v, __ATOMIC_SEQ_CST);
+ return v;
+}
+
+template<typename T>
+inline void
+js::jit::AtomicOperations::storeSeqCst(T* addr, T val)
+{
+ static_assert(sizeof(T) <= sizeof(void*), "atomics supported up to pointer size only");
+ __atomic_store(addr, &val, __ATOMIC_SEQ_CST);
+}
+
+template<typename T>
+inline T
+js::jit::AtomicOperations::compareExchangeSeqCst(T* addr, T oldval, T newval)
+{
+ static_assert(sizeof(T) <= sizeof(void*), "atomics supported up to pointer size only");
+ __atomic_compare_exchange(addr, &oldval, &newval, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
+ return oldval;
+}
+
+template<typename T>
+inline T
+js::jit::AtomicOperations::fetchAddSeqCst(T* addr, T val)
+{
+ static_assert(sizeof(T) <= sizeof(void*), "atomics supported up to pointer size only");
+ return __atomic_fetch_add(addr, val, __ATOMIC_SEQ_CST);
+}
+
+template<typename T>
+inline T
+js::jit::AtomicOperations::fetchSubSeqCst(T* addr, T val)
+{
+ static_assert(sizeof(T) <= sizeof(void*), "atomics supported up to pointer size only");
+ return __atomic_fetch_sub(addr, val, __ATOMIC_SEQ_CST);
+}
+
+template<typename T>
+inline T
+js::jit::AtomicOperations::fetchAndSeqCst(T* addr, T val)
+{
+ static_assert(sizeof(T) <= sizeof(void*), "atomics supported up to pointer size only");
+ return __atomic_fetch_and(addr, val, __ATOMIC_SEQ_CST);
+}
+
+template<typename T>
+inline T
+js::jit::AtomicOperations::fetchOrSeqCst(T* addr, T val)
+{
+ static_assert(sizeof(T) <= sizeof(void*), "atomics supported up to pointer size only");
+ return __atomic_fetch_or(addr, val, __ATOMIC_SEQ_CST);
+}
+
+template<typename T>
+inline T
+js::jit::AtomicOperations::fetchXorSeqCst(T* addr, T val)
+{
+ static_assert(sizeof(T) <= sizeof(void*), "atomics supported up to pointer size only");
+ return __atomic_fetch_xor(addr, val, __ATOMIC_SEQ_CST);
+
+}
+
+template<typename T>
+inline T
+js::jit::AtomicOperations::loadSafeWhenRacy(T* addr)
+{
+ static_assert(sizeof(T) <= sizeof(void*), "atomics supported up to pointer size only");
+ T v;
+ __atomic_load(addr, &v, __ATOMIC_RELAXED);
+ return v;
+}
+
+template<>
+inline uint8_clamped
+js::jit::AtomicOperations::loadSafeWhenRacy(uint8_clamped* addr)
+{
+ uint8_t v;
+ __atomic_load(&addr->val, &v, __ATOMIC_RELAXED);
+ return uint8_clamped(v);
+}
+
+template<>
+inline float
+js::jit::AtomicOperations::loadSafeWhenRacy(float* addr)
+{
+ return *addr;
+}
+
+template<>
+inline double
+js::jit::AtomicOperations::loadSafeWhenRacy(double* addr)
+{
+ return *addr;
+}
+
+} }
+
+template<typename T>
+inline void
+js::jit::AtomicOperations::storeSafeWhenRacy(T* addr, T val)
+{
+ static_assert(sizeof(T) <= sizeof(void*), "atomics supported up to pointer size only");
+ __atomic_store(addr, &val, __ATOMIC_RELAXED);
+}
+
+template<>
+inline void
+js::jit::AtomicOperations::storeSafeWhenRacy(uint8_clamped* addr, uint8_clamped val)
+{
+ __atomic_store(&addr->val, &val.val, __ATOMIC_RELAXED);
+}
+
+template<>
+inline void
+js::jit::AtomicOperations::storeSafeWhenRacy(float* addr, float val)
+{
+ *addr = val;
+}
+
+template<>
+inline void
+js::jit::AtomicOperations::storeSafeWhenRacy(double* addr, double val)
+{
+ *addr = val;
+}
+
+inline void
+js::jit::AtomicOperations::memcpySafeWhenRacy(void* dest, const void* src, size_t nbytes)
+{
+ MOZ_ASSERT(!((char*)dest <= (char*)src && (char*)src < (char*)dest+nbytes));
+ MOZ_ASSERT(!((char*)src <= (char*)dest && (char*)dest < (char*)src+nbytes));
+ ::memcpy(dest, src, nbytes);
+}
+
+inline void
+js::jit::AtomicOperations::memmoveSafeWhenRacy(void* dest, const void* src, size_t nbytes)
+{
+ ::memmove(dest, src, nbytes);
+}
+
+template<typename T>
+inline T
+js::jit::AtomicOperations::exchangeSeqCst(T* addr, T val)
+{
+ static_assert(sizeof(T) <= sizeof(void*), "atomics supported up to pointer size only");
+ T v;
+ __atomic_exchange(addr, &val, &v, __ATOMIC_SEQ_CST);
+ return v;
+}
+
+#endif // jit_ppc64le_AtomicOperations_ppc64le_h
diff -r 485b15bb4a20 -r 9c245f4665be js/src/jit/ppc64/CodeGenerator-ppc64.cpp
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/js/src/jit/ppc64/CodeGenerator-ppc64.cpp Thu Aug 08 21:23:52 2024 -0700
@@ -0,0 +1,3596 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* Code generator for little-endian 64-bit PowerPC compliant to Power ISA v3.0B. */
+
+#include "jit/ppc64/CodeGenerator-ppc64.h"
+
+#include "mozilla/DebugOnly.h"
+#include "mozilla/MathAlgorithms.h"
+
+#include "jsnum.h"
+
+#include "jit/CodeGenerator.h"
+#include "jit/JitFrames.h"
+//#include "jit/JitRealm.h"
+#include "jit/JitRuntime.h"
+#include "jit/MIR.h"
+#include "jit/MIRGraph.h"
+#include "js/Conversions.h"
+#include "vm/JSContext.h"
+#include "vm/Realm.h"
+#include "vm/Shape.h"
+
+#include "jit/MacroAssembler-inl.h"
+#include "jit/shared/CodeGenerator-shared-inl.h"
+#include "vm/JSScript-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::DebugOnly;
+using mozilla::FloorLog2;
+using mozilla::NegativeInfinity;
+using JS::GenericNaN;
+using JS::ToInt32;
+
+#if DEBUG
+
+/* Useful class to print visual guard blocks. */
+class AutoDeBlock
+{
+ private:
+ const char *blockname;
+
+ public:
+ AutoDeBlock(const char *name) {
+ blockname = name;
+ JitSpew(JitSpew_Codegen, "[[ CGPPC64: %s", blockname);
+ }
+
+ ~AutoDeBlock() {
+ JitSpew(JitSpew_Codegen, " CGPPC64: %s ]]", blockname);
+ }
+};
+#undef ADBlock
+#define ADBlock() AutoDeBlock _adbx(__PRETTY_FUNCTION__)
+
+#else
+
+/* Useful macro to completely elide visual guard blocks. */
+#define ADBlock() ;
+
+#endif
+
+void OutOfLineBailout::accept(CodeGeneratorPPC64* codegen) {
+ codegen->visitOutOfLineBailout(this);
+}
+
+CodeGeneratorPPC64::CodeGeneratorPPC64(MIRGenerator* gen, LIRGraph* graph, MacroAssembler* masm)
+ : CodeGeneratorShared(gen, graph, masm)
+{
+}
+
+ValueOperand
+CodeGeneratorPPC64::ToValue(LInstruction* ins, size_t pos)
+{
+ return ValueOperand(ToRegister(ins->getOperand(pos)));
+}
+
+ValueOperand
+CodeGeneratorPPC64::ToTempValue(LInstruction* ins, size_t pos)
+{
+ return ValueOperand(ToRegister(ins->getTemp(pos)));
+}
+
+void
+CodeGenerator::visitBox(LBox* box)
+{
+ ADBlock();
+ const LAllocation* in = box->getOperand(0);
+ const LDefinition* result = box->getDef(0);
+
+ if (IsFloatingPointType(box->type())) {
+ FloatRegister reg = ToFloatRegister(in);
+ if (box->type() == MIRType::Float32) {
+ masm.convertFloat32ToDouble(reg, ScratchDoubleReg);
+ reg = ScratchDoubleReg;
+ }
+ masm.moveFromDouble(reg, ToRegister(result));
+ } else {
+ masm.boxValue(ValueTypeFromMIRType(box->type()), ToRegister(in), ToRegister(result));
+ }
+}
+
+void
+CodeGenerator::visitUnbox(LUnbox* unbox)
+{
+ ADBlock();
+ MUnbox* mir = unbox->mir();
+ Register result = ToRegister(unbox->output());
+
+ if (mir->fallible()) {
+ const ValueOperand value = ToValue(unbox, LUnbox::Input);
+ Label bail;
+ switch (mir->type()) {
+ case MIRType::Int32:
+ masm.fallibleUnboxInt32(value, result, &bail);
+ break;
+ case MIRType::Boolean:
+ masm.fallibleUnboxBoolean(value, result, &bail);
+ break;
+ case MIRType::Object:
+ masm.fallibleUnboxObject(value, result, &bail);
+ break;
+ case MIRType::String:
+ masm.fallibleUnboxString(value, result, &bail);
+ break;
+ case MIRType::Symbol:
+ masm.fallibleUnboxSymbol(value, result, &bail);
+ break;
+ case MIRType::BigInt:
+ masm.fallibleUnboxBigInt(value, result, &bail);
+ break;
+ default:
+ MOZ_CRASH("Given MIRType cannot be unboxed.");
+ }
+ bailoutFrom(&bail, unbox->snapshot());
+ return;
+ }
+
+ LAllocation* input = unbox->getOperand(LUnbox::Input);
+ if (input->isRegister()) {
+ Register inputReg = ToRegister(input);
+ switch (mir->type()) {
+ case MIRType::Int32:
+ masm.unboxInt32(inputReg, result);
+ break;
+ case MIRType::Boolean:
+ masm.unboxBoolean(inputReg, result);
+ break;
+ case MIRType::Object:
+ masm.unboxObject(inputReg, result);
+ break;
+ case MIRType::String:
+ masm.unboxString(inputReg, result);
+ break;
+ case MIRType::Symbol:
+ masm.unboxSymbol(inputReg, result);
+ break;
+ case MIRType::BigInt:
+ masm.unboxBigInt(inputReg, result);
+ break;
+ default:
+ MOZ_CRASH("Given MIRType cannot be unboxed.");
+ }
+ return;
+ }
+
+ Address inputAddr = ToAddress(input);
+ switch (mir->type()) {
+ case MIRType::Int32:
+ masm.unboxInt32(inputAddr, result);
+ break;
+ case MIRType::Boolean:
+ masm.unboxBoolean(inputAddr, result);
+ break;
+ case MIRType::Object:
+ masm.unboxObject(inputAddr, result);
+ break;
+ case MIRType::String:
+ masm.unboxString(inputAddr, result);
+ break;
+ case MIRType::Symbol:
+ masm.unboxSymbol(inputAddr, result);
+ break;
+ case MIRType::BigInt:
+ masm.unboxBigInt(inputAddr, result);
+ break;
+ default:
+ MOZ_CRASH("Given MIRType cannot be unboxed.");
+ }
+}
+
+void
+CodeGeneratorPPC64::splitTagForTest(const ValueOperand& value, ScratchTagScope& tag)
+{
+ masm.splitTag(value.valueReg(), tag);
+}
+
+// XXX: needs move or rewrite
+#if(0)
+void
+CodeGenerator::visitCompareB(LCompareB* lir)
+{
+ ADBlock();
+ MCompare* mir = lir->mir();
+
+ const ValueOperand lhs = ToValue(lir, LCompareB::Lhs);
+ const LAllocation* rhs = lir->rhs();
+ const Register output = ToRegister(lir->output());
+
+ MOZ_ASSERT(mir->jsop() == JSOp::StrictEq || mir->jsop() == JSOp::StrictNe);
+ Assembler::Condition cond = JSOpToCondition(mir->compareType(), mir->jsop());
+
+ // Load boxed boolean in ScratchRegister.
+ if (rhs->isConstant())
+ masm.moveValue(rhs->toConstant()->toJSValue(), ValueOperand(ScratchRegister));
+ else
+ masm.boxValue(JSVAL_TYPE_BOOLEAN, ToRegister(rhs), ScratchRegister);
+
+ // Perform the comparison.
+ masm.cmpPtrSet(cond, lhs.valueReg(), ScratchRegister, output);
+}
+
+void
+CodeGenerator::visitCompareBAndBranch(LCompareBAndBranch* lir)
+{
+ ADBlock();
+ MCompare* mir = lir->cmpMir();
+ const ValueOperand lhs = ToValue(lir, LCompareBAndBranch::Lhs);
+ const LAllocation* rhs = lir->rhs();
+
+ MOZ_ASSERT(mir->jsop() == JSOp::StrictEq || mir->jsop() == JSOp::StrictNe);
+
+ // Load boxed boolean in ScratchRegister.
+ if (rhs->isConstant())
+ masm.moveValue(rhs->toConstant()->toJSValue(), ValueOperand(ScratchRegister));
+ else
+ masm.boxValue(JSVAL_TYPE_BOOLEAN, ToRegister(rhs), ScratchRegister);
+
+ // Perform the comparison.
+ Assembler::Condition cond = JSOpToCondition(mir->compareType(), mir->jsop());
+ emitBranch(lhs.valueReg(), ScratchRegister, cond, lir->ifTrue(), lir->ifFalse());
+}
+#endif
+
+void
+CodeGenerator::visitCompareI64(LCompareI64* lir)
+{
+ ADBlock();
+ MCompare* mir = lir->mir();
+ MOZ_ASSERT(mir->compareType() == MCompare::Compare_Int64 ||
+ mir->compareType() == MCompare::Compare_UInt64);
+
+ const LInt64Allocation lhs = lir->getInt64Operand(LCompareI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LCompareI64::Rhs);
+ Register lhsReg = ToRegister64(lhs).reg;
+ Register output = ToRegister(lir->output());
+ Register rhsReg;
+
+ if (IsConstant(rhs)) {
+ rhsReg = ScratchRegister;
+ masm.ma_li(rhsReg, ImmWord(ToInt64(rhs)));
+ } else {
+ rhsReg = ToRegister64(rhs).reg;
+ }
+
+ bool isSigned = mir->compareType() == MCompare::Compare_Int64;
+ masm.cmpPtrSet(JSOpToCondition(lir->jsop(), isSigned), lhsReg, rhsReg, output);
+}
+
+void CodeGenerator::visitWasmExtendU32Index(LWasmExtendU32Index* lir) {
+ // Generates no code on this platform because the input is assumed to have
+ // canonical form.
+ Register output = ToRegister(lir->output());
+ MOZ_ASSERT(ToRegister(lir->input()) == output);
+ masm.debugAssertCanonicalInt32(output);
+}
+
+void CodeGenerator::visitWasmWrapU32Index(LWasmWrapU32Index* lir) {
+ // Generates no code on this platform because the input is assumed to have
+ // canonical form.
+ Register output = ToRegister(lir->output());
+ MOZ_ASSERT(ToRegister(lir->input()) == output);
+ masm.debugAssertCanonicalInt32(output);
+}
+
+void
+CodeGenerator::visitCompareI64AndBranch(LCompareI64AndBranch* lir)
+{
+ ADBlock();
+ MCompare* mir = lir->cmpMir();
+ MOZ_ASSERT(mir->compareType() == MCompare::Compare_Int64 ||
+ mir->compareType() == MCompare::Compare_UInt64);
+
+ const LInt64Allocation lhs = lir->getInt64Operand(LCompareI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LCompareI64::Rhs);
+ Register lhsReg = ToRegister64(lhs).reg;
+ Register rhsReg;
+
+ if (IsConstant(rhs)) {
+ rhsReg = ScratchRegister;
+ masm.ma_li(rhsReg, ImmWord(ToInt64(rhs)));
+ } else {
+ rhsReg = ToRegister64(rhs).reg;
+ }
+
+ bool isSigned = mir->compareType() == MCompare::Compare_Int64;
+ Assembler::Condition cond = JSOpToCondition(lir->jsop(), isSigned);
+ emitBranch(lhsReg, rhsReg, cond, lir->ifTrue(), lir->ifFalse());
+}
+
+void
+CodeGenerator::visitDivOrModI64(LDivOrModI64* lir)
+{
+ ADBlock();
+ // divdo will automatically set the Overflow bit if INT64_MIN/-1 is
+ // performed, or if we divide by zero, so we just bailout based on that.
+
+ Label noOverflow, done;
+ Register lhs = ToRegister(lir->lhs());
+ Register rhs = ToRegister(lir->rhs());
+ Register output = ToRegister(lir->output());
+
+ // DIVIDE! AND! CONQUER!
+ if (lir->snapshot()) {
+ // If Overflow is set, we must bail out. But only do this if
+ // there is a snapshot, because if there isn't, Ion already knows
+ // this operation is infallible (and we can avoid checking
+ // for overflow, which is slightly faster).
+ masm.ma_li(output, 0L);
+ masm.xs_mtxer(output); // whack XER[SO]
+ masm.as_divdo_rc(output, lhs, rhs); // T = A / B
+
+ // Did we trap?
+ masm.ma_bc(Assembler::NSOBit, &noOverflow, ShortJump);
+ // Yes. Determine how.
+ MOZ_ASSERT(lir->canBeNegativeOverflow() || lir->canBeDivideByZero());
+ if (lir->canBeNegativeOverflow()) {
+ if (lir->canBeDivideByZero()) {
+ Label notNegOne;
+
+ masm.branchPtr(Assembler::NotEqual, rhs, ImmWord(-1), &notNegOne);
+ // If rhs was -1, then the offending operation must have been
+ // INT64_MIN/-1.
+ if (lir->mir()->isMod())
+ masm.move32(Imm32(0), output);
+ else
+ masm.wasmTrap(wasm::Trap::IntegerOverflow, lir->bytecodeOffset());
+ masm.ma_b(&done, ShortJump);
+ masm.bind(&notNegOne);
+ } else {
+ // Just trap, it's the only possible triggering condition.
+ masm.wasmTrap(wasm::Trap::IntegerOverflow, lir->bytecodeOffset());
+ }
+ }
+ if (lir->canBeDivideByZero()) {
+ // Must have been divide by zero.
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->bytecodeOffset());
+ }
+ } else {
+ masm.as_divd(output, lhs, rhs);
+ }
+
+ masm.bind(&noOverflow);
+ if (lir->mir()->isMod()) {
+// XXX: convert to modsd/modud
+ // Recover the remainder.
+ // We don't need to check overflow; we know it can't.
+ masm.as_mulld(ScratchRegister, output, rhs);
+ masm.as_subf(output, ScratchRegister, lhs); // T = B - A
+ }
+
+ masm.bind(&done);
+}
+
+void
+CodeGenerator::visitUDivOrModI64(LUDivOrModI64* lir)
+{
+ // Simpler, because no -1 check.
+ ADBlock();
+ Register lhs = ToRegister(lir->lhs());
+ Register rhs = ToRegister(lir->rhs());
+ Register output = ToRegister(lir->output());
+ LSnapshot* snap = lir->snapshot();
+ Label noOverflow;
+
+ if (snap) {
+ masm.ma_li(output, 0L);
+ masm.xs_mtxer(output); // whack XER[SO]
+ masm.as_divduo_rc(output, lhs, rhs); // T = A / B
+
+ // Did we trap?
+ masm.ma_bc(Assembler::NSOBit, &noOverflow, ShortJump);
+ // Yes. Only one way that can happen here.
+ MOZ_ASSERT(lir->canBeDivideByZero());
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->bytecodeOffset());
+ }
+ masm.bind(&noOverflow);
+
+ if (lir->mir()->isMod()) {
+ if (HasPPCISA3()) {
+ masm.as_modud(output, lhs, rhs);
+ } else {
+ // Recover the remainder manually.
+ // We don't need to check overflow; we know it can't.
+ // Reuse the overflow check result if available.
+ if (!snap) {
+ masm.as_divdu(output, lhs, rhs);
+ }
+ masm.as_mulld(ScratchRegister, output, rhs);
+ masm.as_subf(output, ScratchRegister, lhs); // T = B - A
+ }
+ } else {
+ // Reuse the overflow check result if available.
+ if (!snap) {
+ masm.as_divdu(output, lhs, rhs);
+ }
+ }
+}
+
+void CodeGeneratorPPC64::emitBigIntDiv(LBigIntDiv* ins, Register dividend,
+ Register divisor, Register output,
+ Label* fail)
+{
+ // Callers handle division by zero and integer overflow.
+
+ masm.as_divd(/* result= */ dividend, dividend, divisor);
+
+ // Create and return the result.
+ masm.newGCBigInt(output, divisor, initialBigIntHeap(), fail);
+ masm.initializeBigInt(output, dividend);
+}
+
+void CodeGeneratorPPC64::emitBigIntMod(LBigIntMod* ins, Register dividend,
+ Register divisor, Register output,
+ Label* fail)
+{
+ // Callers handle division by zero and integer overflow.
+
+// XXX: use modsd, etc. on POWER9
+ // Signed division.
+ masm.as_divd(output, dividend, divisor);
+
+ // Compute the remainder: output = dividend - (output * divisor).
+ masm.as_mulld(output, output, divisor);
+ masm.as_subf(dividend, output, dividend);
+
+ // Create and return the result.
+ masm.newGCBigInt(output, divisor, initialBigIntHeap(), fail);
+ masm.initializeBigInt(output, dividend);
+}
+
+template <typename T>
+void
+CodeGeneratorPPC64::emitWasmLoadI64(T* lir)
+{
+ ADBlock();
+ const MWasmLoad* mir = lir->mir();
+
+ Register ptrScratch = InvalidReg;
+ if(!lir->ptrCopy()->isBogusTemp()){
+ ptrScratch = ToRegister(lir->ptrCopy());
+ }
+
+ if (IsUnaligned(mir->access())) {
+ masm.wasmUnalignedLoadI64(mir->access(), HeapReg, ToRegister(lir->ptr()),
+ ptrScratch, ToOutRegister64(lir), ToRegister(lir->getTemp(1)));
+ } else {
+ masm.wasmLoadI64(mir->access(), HeapReg, ToRegister(lir->ptr()), ptrScratch,
+ ToOutRegister64(lir));
+ }
+}
+
+void
+CodeGenerator::visitWasmLoadI64(LWasmLoadI64* lir)
+{
+ emitWasmLoadI64(lir);
+}
+
+void
+CodeGenerator::visitWasmUnalignedLoadI64(LWasmUnalignedLoadI64* lir)
+{
+ emitWasmLoadI64(lir);
+}
+
+template <typename T>
+void
+CodeGeneratorPPC64::emitWasmStoreI64(T* lir)
+{
+ ADBlock();
+ const MWasmStore* mir = lir->mir();
+
+ Register ptrScratch = InvalidReg;
+ if(!lir->ptrCopy()->isBogusTemp()){
+ ptrScratch = ToRegister(lir->ptrCopy());
+ }
+
+ if (IsUnaligned(mir->access())) {
+ masm.wasmUnalignedStoreI64(mir->access(), ToRegister64(lir->value()), HeapReg,
+ ToRegister(lir->ptr()), ptrScratch, ToRegister(lir->getTemp(1)));
+ } else {
+ masm.wasmStoreI64(mir->access(), ToRegister64(lir->value()), HeapReg,
+ ToRegister(lir->ptr()), ptrScratch);
+ }
+}
+
+void
+CodeGenerator::visitWasmStoreI64(LWasmStoreI64* lir)
+{
+ emitWasmStoreI64(lir);
+}
+
+void
+CodeGenerator::visitWasmUnalignedStoreI64(LWasmUnalignedStoreI64* lir)
+{
+ emitWasmStoreI64(lir);
+}
+
+void
+CodeGenerator::visitWasmSelectI64(LWasmSelectI64* lir)
+{
+ ADBlock();
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Int64);
+
+ Register cond = ToRegister(lir->condExpr());
+ const LInt64Allocation falseExpr = lir->falseExpr();
+
+ Register64 out = ToOutRegister64(lir);
+ MOZ_ASSERT(ToRegister64(lir->trueExpr()) == out, "true expr is reused for input");
+
+ if (falseExpr.value().isRegister()) {
+ masm.as_cmpdi(cond, 0);
+ __asm__("trap\n"); // XXX: fix condition below
+ masm.as_isel(out.reg, out.reg, ToRegister(falseExpr.value()), 2); // CR0[EQ]
+ } else {
+ Label done;
+ masm.ma_bc(cond, cond, &done, Assembler::NonZero, ShortJump);
+ masm.loadPtr(ToAddress(falseExpr.value()), out.reg);
+ masm.bind(&done);
+ }
+}
+
+void
+CodeGenerator::visitWasmReinterpretFromI64(LWasmReinterpretFromI64* lir)
+{
+ ADBlock();
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Double);
+ MOZ_ASSERT(lir->mir()->input()->type() == MIRType::Int64);
+
+ if (HasPPCISA3()) {
+ masm.as_mtvsrd(ToFloatRegister(lir->output()), ToRegister(lir->input()));
+ } else {
+ // Alternative, if we're not assuming POWER8.
+ masm.as_stdu(ToRegister(lir->input()), StackPointer, -8);
+ masm.as_lfd(ToFloatRegister(lir->output()), StackPointer, 0);
+ masm.as_addi(StackPointer, StackPointer, 8);
+ }
+}
+
+void
+CodeGenerator::visitWasmReinterpretToI64(LWasmReinterpretToI64* lir)
+{
+ ADBlock();
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Int64);
+ MOZ_ASSERT(lir->mir()->input()->type() == MIRType::Double);
+
+ if (HasPPCISA3()) {
+ masm.as_mfvsrd(ToRegister(lir->output()), ToFloatRegister(lir->input()));
+ } else { // Sigh.
+ masm.as_stfdu(ToFloatRegister(lir->input()), StackPointer, -8);
+ masm.as_ld(ToRegister(lir->output()), StackPointer, 0);
+ masm.as_addi(StackPointer, StackPointer, 8);
+ }
+}
+
+void
+CodeGenerator::visitExtendInt32ToInt64(LExtendInt32ToInt64* lir)
+{
+ ADBlock();
+ const LAllocation* input = lir->getOperand(0);
+ Register output = ToRegister(lir->output());
+ Register inpreg = ToRegister(input);
+
+ // If the value is unsigned, then we just copy the input
+ // register; it automatically is promoted to 64 bits.
+ // If the value is signed, then sign extend it.
+ if (lir->mir()->isUnsigned()) {
+ if (inpreg != output)
+ masm.as_or(output, inpreg, inpreg);
+ } else
+ masm.as_extsw(output, inpreg);
+}
+
+void
+CodeGenerator::visitWrapInt64ToInt32(LWrapInt64ToInt32* lir)
+{
+ ADBlock();
+ const LAllocation* input = lir->getOperand(0);
+ Register output = ToRegister(lir->output());
+
+ if (lir->mir()->bottomHalf()) {
+ if (input->isMemory())
+ masm.load32(ToAddress(input), output); // endian!
+ else
+ masm.as_srawi(output, ToRegister(input), 0); // sign-extends
+ } else {
+ MOZ_CRASH("Not implemented.");
+ // but it would be rldicl rA,rS,32,32 (don't tell anyone, k?)
+ }
+}
+
+void
+CodeGenerator::visitSignExtendInt64(LSignExtendInt64* lir)
+{
+ ADBlock();
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register64 output = ToOutRegister64(lir);
+ switch (lir->mode()) {
+ case MSignExtendInt64::Byte:
+ masm.move32To64SignExtend(input.reg, output);
+ masm.move8SignExtend(output.reg, output.reg);
+ break;
+ case MSignExtendInt64::Half:
+ masm.move32To64SignExtend(input.reg, output);
+ masm.move16SignExtend(output.reg, output.reg);
+ break;
+ case MSignExtendInt64::Word:
+ masm.move32To64SignExtend(input.reg, output);
+ break;
+ }
+}
+
+void
+CodeGenerator::visitClzI64(LClzI64* lir)
+{
+ ADBlock();
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register64 output = ToOutRegister64(lir);
+ masm.as_cntlzd(output.reg, input.reg);
+}
+
+void
+CodeGenerator::visitCtzI64(LCtzI64* lir)
+{
+ ADBlock();
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register64 output = ToOutRegister64(lir);
+ // Damn, it's good to be a POWER9 gangsta. When I was a boy,
+ // we had to count our trailing zeroes on my G5 walking
+ // uphill in the snow BOTH WAYS. We even have that fancy
+ // population count instruction now. You kids have it so good.
+// XXX: POWER9
+ masm.as_cnttzd(output.reg, input.reg);
+}
+
+void
+CodeGenerator::visitNotI64(LNotI64* lir)
+{
+ ADBlock();
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register output = ToRegister(lir->output());
+
+// XXX: isel would be better
+ masm.ma_cmp_set(output, input.reg, Imm64(0), Assembler::Equal);
+}
+
+void CodeGenerator::visitBitNotI64(LBitNotI64* ins) {
+ ADBlock();
+ const LAllocation* input = ins->getOperand(0);
+ MOZ_ASSERT(!input->isConstant());
+ Register inputReg = ToRegister(input);
+ MOZ_ASSERT(inputReg == ToRegister(ins->output())); // MIPS. Do we truly care?
+ masm.ma_not(inputReg, inputReg);
+}
+
+void
+CodeGenerator::visitWasmTruncateToInt64(LWasmTruncateToInt64* lir)
+{
+ ADBlock();
+ FloatRegister input = ToFloatRegister(lir->input());
+ Register64 output = ToOutRegister64(lir);
+
+ MWasmTruncateToInt64* mir = lir->mir();
+ MIRType fromType = mir->input()->type();
+
+ MOZ_ASSERT(fromType == MIRType::Double || fromType == MIRType::Float32);
+
+ auto* ool = new (alloc()) OutOfLineWasmTruncateCheck(mir, input, output);
+ addOutOfLineCode(ool, mir);
+
+ Label* oolEntry = ool->entry();
+ Label* oolRejoin = ool->rejoin();
+ bool isSaturating = mir->isSaturating();
+
+ if (fromType == MIRType::Double) {
+ if (mir->isUnsigned())
+ masm.wasmTruncateDoubleToUInt64(input, output, isSaturating, oolEntry, oolRejoin,
+ InvalidFloatReg);
+ else
+ masm.wasmTruncateDoubleToInt64(input, output, isSaturating, oolEntry, oolRejoin,
+ InvalidFloatReg);
+ } else {
+ if (mir->isUnsigned())
+ masm.wasmTruncateFloat32ToUInt64(input, output, isSaturating, oolEntry, oolRejoin,
+ InvalidFloatReg);
+ else
+ masm.wasmTruncateFloat32ToInt64(input, output, isSaturating, oolEntry, oolRejoin,
+ InvalidFloatReg);
+ }
+}
+
+void
+CodeGenerator::visitInt64ToFloatingPoint(LInt64ToFloatingPoint* lir)
+{
+ ADBlock();
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ FloatRegister output = ToFloatRegister(lir->output());
+
+ MIRType outputType = lir->mir()->type();
+ MOZ_ASSERT(outputType == MIRType::Double || outputType == MIRType::Float32);
+
+ if (outputType == MIRType::Double) {
+ if (lir->mir()->isUnsigned())
+ masm.convertUInt64ToDouble(input, output, Register::Invalid());
+ else
+ masm.convertInt64ToDouble(input, output);
+ } else {
+ if (lir->mir()->isUnsigned())
+ masm.convertUInt64ToFloat32(input, output, Register::Invalid());
+ else
+ masm.convertInt64ToFloat32(input, output);
+ }
+}
+
+void
+CodeGenerator::visitTestI64AndBranch(LTestI64AndBranch* lir)
+{
+ ADBlock();
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ MBasicBlock* ifTrue = lir->ifTrue();
+ MBasicBlock* ifFalse = lir->ifFalse();
+
+ emitBranch(input.reg, Imm32(0), Assembler::NonZero, ifTrue, ifFalse);
+}
+
+Operand
+CodeGeneratorPPC64::ToOperand(const LAllocation& a)
+{
+ if (a.isGeneralReg()) {
+ return Operand(a.toGeneralReg()->reg());
+ }
+ if (a.isFloatReg()) {
+ return Operand(a.toFloatReg()->reg());
+ }
+ return Operand(ToAddress(a));
+}
+
+Operand
+CodeGeneratorPPC64::ToOperand(const LAllocation* a)
+{
+ return ToOperand(*a);
+}
+
+Operand
+CodeGeneratorPPC64::ToOperand(const LDefinition* def)
+{
+ return ToOperand(def->output());
+}
+
+#ifdef JS_PUNBOX64
+Operand
+CodeGeneratorPPC64::ToOperandOrRegister64(const LInt64Allocation input)
+{
+ return ToOperand(input.value());
+}
+#else
+Register64
+CodeGeneratorPPC64::ToOperandOrRegister64(const LInt64Allocation input)
+{
+ return ToRegister64(input);
+}
+#endif
+
+void
+CodeGeneratorPPC64::branchToBlock(Assembler::FloatFormat fmt, FloatRegister lhs, FloatRegister rhs,
+ MBasicBlock* mir, Assembler::DoubleCondition cond)
+{
+ // Skip past trivial blocks.
+ Label* label = skipTrivialBlocks(mir)->lir()->label();
+ if (fmt == Assembler::DoubleFloat)
+ masm.branchDouble(cond, lhs, rhs, label);
+ else
+ masm.branchFloat(cond, lhs, rhs, label);
+}
+
+void
+CodeGenerator::visitTestIAndBranch(LTestIAndBranch* test)
+{
+ ADBlock();
+ const LAllocation* opd = test->getOperand(0);
+ MBasicBlock* ifTrue = test->ifTrue();
+ MBasicBlock* ifFalse = test->ifFalse();
+
+ emitBranch(ToRegister(opd), Imm32(0), Assembler::NonZero, ifTrue, ifFalse);
+}
+
+void
+CodeGenerator::visitCompare(LCompare* comp)
+{
+ ADBlock();
+ MCompare* mir = comp->mir();
+ Assembler::Condition cond = JSOpToCondition(mir->compareType(), comp->jsop());
+ const LAllocation* left = comp->getOperand(0);
+ const LAllocation* right = comp->getOperand(1);
+ const LDefinition* def = comp->getDef(0);
+
+ if (mir->compareType() == MCompare::Compare_Object ||
+ mir->compareType() == MCompare::Compare_Symbol ||
+ mir->compareType() == MCompare::Compare_UIntPtr) {
+ if (right->isConstant()) {
+ MOZ_ASSERT(mir->compareType() == MCompare::Compare_UIntPtr);
+ // Force 64-bit compare.
+ masm.ma_cmp_set(ToRegister(def), ToRegister(left), Imm64(ToInt32(right)),
+ cond, /* useCmpw */ false);
+ } else if (right->isGeneralReg()) {
+ masm.cmpPtrSet(cond, ToRegister(left), ToRegister(right),
+ ToRegister(def));
+ } else {
+ masm.cmpPtrSet(cond, ToRegister(left), ToAddress(right), ToRegister(def));
+ }
+ return;
+ }
+
+ if (right->isConstant()) {
+ masm.cmp32Set(cond, ToRegister(left), Imm32(ToInt32(right)),
+ ToRegister(def));
+ } else if (right->isGeneralReg()) {
+ masm.cmp32Set(cond, ToRegister(left), ToRegister(right), ToRegister(def));
+ } else {
+ masm.cmp32Set(cond, ToRegister(left), ToAddress(right), ToRegister(def));
+ }
+}
+
+void
+CodeGenerator::visitCompareAndBranch(LCompareAndBranch* comp)
+{
+ ADBlock();
+ MCompare* mir = comp->cmpMir();
+ Assembler::Condition cond = JSOpToCondition(mir->compareType(), comp->jsop());
+
+ if (mir->compareType() == MCompare::Compare_Object ||
+ mir->compareType() == MCompare::Compare_Symbol ||
+ mir->compareType() == MCompare::Compare_UIntPtr) {
+ if (comp->right()->isConstant()) {
+ MOZ_ASSERT(mir->compareType() == MCompare::Compare_UIntPtr);
+ emitBranch(ToRegister(comp->left()), ImmWord(ToInt32(comp->right())), cond,
+ comp->ifTrue(), comp->ifFalse());
+ } else if (comp->right()->isGeneralReg()) {
+ emitBranch(ToRegister(comp->left()), ToRegister(comp->right()), cond,
+ comp->ifTrue(), comp->ifFalse());
+ } else {
+ masm.loadPtr(ToAddress(comp->right()), ScratchRegister);
+ emitBranch(ToRegister(comp->left()), ScratchRegister, cond,
+ comp->ifTrue(), comp->ifFalse());
+ }
+ return;
+ }
+
+ if (comp->right()->isConstant()) {
+ emitBranch(ToRegister(comp->left()), Imm32(ToInt32(comp->right())), cond,
+ comp->ifTrue(), comp->ifFalse());
+ } else if (comp->right()->isGeneralReg()) {
+ emitBranch(ToRegister(comp->left()), ToRegister(comp->right()), cond,
+ comp->ifTrue(), comp->ifFalse());
+ } else {
+ masm.load32(ToAddress(comp->right()), ScratchRegister);
+ emitBranch(ToRegister(comp->left()), ScratchRegister, cond, comp->ifTrue(),
+ comp->ifFalse());
+ }
+
+}
+
+bool
+CodeGeneratorPPC64::generateOutOfLineCode()
+{
+ ADBlock();
+ if (!CodeGeneratorShared::generateOutOfLineCode())
+ return false;
+
+ if (deoptLabel_.used()) {
+ // All non-table-based bailouts will go here.
+ masm.bind(&deoptLabel_);
+
+ // Push the frame size, so the handler can recover the IonScript.
+ // Frame size is stored in LR and pushed by GenerateBailoutThunk.
+ // We have to use LR because generateBailoutTable will implicitly do
+ // the same.
+ masm.ma_li(ScratchRegister, frameSize());
+ masm.xs_mtlr(ScratchRegister);
+
+ TrampolinePtr handler = gen->jitRuntime()->getGenericBailoutHandler();
+ masm.jump(handler);
+ }
+
+ return !masm.oom();
+}
+
+void
+CodeGeneratorPPC64::bailoutFrom(Label* label, LSnapshot* snapshot)
+{
+ ADBlock();
+
+ MOZ_ASSERT_IF(!masm.oom(), label->used());
+ MOZ_ASSERT_IF(!masm.oom(), !label->bound());
+
+ encode(snapshot);
+
+ InlineScriptTree* tree = snapshot->mir()->block()->trackedTree();
+ OutOfLineBailout* ool = new(alloc()) OutOfLineBailout(snapshot, masm.framePushed());
+ addOutOfLineCode(ool, new(alloc()) BytecodeSite(tree, tree->script()->code()));
+
+ masm.retarget(label, ool->entry());
+}
+
+void
+CodeGeneratorPPC64::bailout(LSnapshot* snapshot)
+{
+ Label label;
+ masm.jump(&label);
+ bailoutFrom(&label, snapshot);
+}
+
+void
+CodeGenerator::visitMinMaxD(LMinMaxD* ins)
+{
+ ADBlock();
+ FloatRegister first = ToFloatRegister(ins->first());
+ FloatRegister second = ToFloatRegister(ins->second());
+
+ MOZ_ASSERT(first == ToFloatRegister(ins->output()));
+
+ if (ins->mir()->isMax())
+ masm.maxDouble(second, first, true);
+ else
+ masm.minDouble(second, first, true);
+}
+
+void
+CodeGenerator::visitMinMaxF(LMinMaxF* ins)
+{
+ ADBlock();
+ FloatRegister first = ToFloatRegister(ins->first());
+ FloatRegister second = ToFloatRegister(ins->second());
+
+ MOZ_ASSERT(first == ToFloatRegister(ins->output()));
+
+ if (ins->mir()->isMax())
+ masm.maxFloat32(second, first, true);
+ else
+ masm.minFloat32(second, first, true);
+}
+
+void
+CodeGenerator::visitAddI(LAddI* ins)
+{
+ ADBlock();
+ const LAllocation* lhs = ins->getOperand(0);
+ const LAllocation* rhs = ins->getOperand(1);
+ const LDefinition* dest = ins->getDef(0);
+
+ MOZ_ASSERT(rhs->isConstant() || rhs->isGeneralReg());
+
+ // If there is no snapshot, we don't need to check for overflow.
+ if (!ins->snapshot()) {
+ if (rhs->isConstant())
+ masm.ma_add(ToRegister(dest), ToRegister(lhs), Imm32(ToInt32(rhs)));
+ else
+ masm.as_add(ToRegister(dest), ToRegister(lhs), ToRegister(rhs));
+ return;
+ }
+
+ Label overflow;
+ if (rhs->isConstant())
+ masm.ma_addTestOverflow(ToRegister(dest), ToRegister(lhs), Imm32(ToInt32(rhs)), &overflow);
+ else
+ masm.ma_addTestOverflow(ToRegister(dest), ToRegister(lhs), ToRegister(rhs), &overflow);
+
+ bailoutFrom(&overflow, ins->snapshot());
+}
+
+void
+CodeGenerator::visitAddI64(LAddI64* lir)
+{
+ ADBlock();
+ const LInt64Allocation lhs = lir->getInt64Operand(LAddI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LAddI64::Rhs);
+
+ MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
+
+ if (IsConstant(rhs)) {
+ masm.add64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
+ return;
+ }
+
+ masm.add64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
+}
+
+void
+CodeGenerator::visitSubI(LSubI* ins)
+{
+ ADBlock();
+ const LAllocation* lhs = ins->getOperand(0);
+ const LAllocation* rhs = ins->getOperand(1);
+ const LDefinition* dest = ins->getDef(0);
+
+ MOZ_ASSERT(rhs->isConstant() || rhs->isGeneralReg());
+
+ // If there is no snapshot, we don't need to check for overflow.
+ if (!ins->snapshot()) {
+ if (rhs->isConstant())
+ masm.ma_subu(ToRegister(dest), ToRegister(lhs), Imm32(ToInt32(rhs)));
+ else
+ masm.as_subf(ToRegister(dest), ToRegister(rhs), ToRegister(lhs)); // T = B - A
+ return;
+ }
+
+ Label overflow;
+ if (rhs->isConstant())
+ masm.ma_subTestOverflow(ToRegister(dest), ToRegister(lhs), Imm32(ToInt32(rhs)), &overflow);
+ else
+ masm.ma_subTestOverflow(ToRegister(dest), ToRegister(lhs), ToRegister(rhs), &overflow);
+
+ bailoutFrom(&overflow, ins->snapshot());
+}
+
+void
+CodeGenerator::visitSubI64(LSubI64* lir)
+{
+ ADBlock();
+ const LInt64Allocation lhs = lir->getInt64Operand(LSubI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LSubI64::Rhs);
+
+ MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
+
+ if (IsConstant(rhs)) {
+ masm.sub64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
+ return;
+ }
+
+ masm.sub64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
+}
+
+void
+CodeGenerator::visitMulI(LMulI* ins)
+{
+ ADBlock();
+ const LAllocation* lhs = ins->lhs();
+ const LAllocation* rhs = ins->rhs();
+ Register dest = ToRegister(ins->output());
+ MMul* mul = ins->mir();
+
+ MOZ_ASSERT_IF(mul->mode() == MMul::Integer, !mul->canBeNegativeZero() && !mul->canOverflow());
+
+ if (rhs->isConstant()) {
+ int32_t constant = ToInt32(rhs);
+ Register src = ToRegister(lhs);
+
+ // Bailout on -0.0
+ if (mul->canBeNegativeZero() && constant <= 0) {
+ Assembler::Condition cond = (constant == 0) ? Assembler::LessThan : Assembler::Equal;
+ bailoutCmp32(cond, src, Imm32(0), ins->snapshot());
+ }
+
+ switch (constant) {
+ case -1:
+ if (mul->canOverflow())
+ bailoutCmp32(Assembler::Equal, src, Imm32(INT32_MIN), ins->snapshot());
+
+ masm.as_neg(dest, src);
+ break;
+ case 0:
+ masm.move32(Imm32(0), dest);
+ break;
+ case 1:
+ masm.move32(src, dest);
+ break;
+ case 2:
+ if (mul->canOverflow()) {
+ Label mulTwoOverflow;
+ masm.ma_addTestOverflow(dest, src, src, &mulTwoOverflow);
+
+ bailoutFrom(&mulTwoOverflow, ins->snapshot());
+ } else {
+ masm.as_add(dest, src, src);
+ }
+ break;
+ default:
+ uint32_t shift = FloorLog2(constant);
+
+ if (!mul->canOverflow() && (constant > 0)) {
+ // If it cannot overflow, we can do lots of optimizations.
+ uint32_t rest = constant - (1 << shift);
+
+ // See if the constant has one bit set, meaning it can be
+ // encoded as a bitshift.
+ if ((1 << shift) == constant) {
+ MOZ_ASSERT(shift < 32);
+ masm.x_slwi(dest, src, shift);
+ return;
+ }
+
+ // If the constant cannot be encoded as (1<<C1), see if it can
+ // be encoded as (1<<C1) | (1<<C2), which can be computed
+ // using an add and a shift.
+ uint32_t shift_rest = FloorLog2(rest);
+ if (src != dest && (1u << shift_rest) == rest) {
+ masm.x_slwi(dest, src, shift - shift_rest);
+ masm.as_add(dest, dest, src);
+ if (shift_rest != 0)
+ masm.x_slwi(dest, dest, shift_rest);
+ return;
+ }
+ }
+
+ if (mul->canOverflow() && (constant > 0) && (src != dest)) {
+ // To stay on the safe side, only optimize things that are a
+ // power of 2.
+
+ if ((1 << shift) == constant) {
+ // dest = lhs * pow(2, shift)
+ MOZ_ASSERT(shift < 32);
+ masm.x_slwi(dest, src, shift);
+ // At runtime, check (lhs == dest >> shift). If this does
+ // not hold, some bits were lost due to overflow, and the
+ // computation should be resumed as a double.
+ masm.as_srawi(ScratchRegister, dest, shift);
+ bailoutCmp32(Assembler::NotEqual, src, ScratchRegister, ins->snapshot());
+ return;
+ }
+ }
+
+ if (mul->canOverflow()) {
+ Label mulConstOverflow;
+ masm.ma_mul_branch_overflow(dest, ToRegister(lhs), Imm32(ToInt32(rhs)),
+ &mulConstOverflow);
+
+ bailoutFrom(&mulConstOverflow, ins->snapshot());
+ } else {
+ masm.ma_mul(dest, src, Imm32(ToInt32(rhs)));
+ }
+ break;
+ }
+ } else {
+ Label multRegOverflow;
+
+ if (mul->canOverflow()) {
+ masm.ma_mul_branch_overflow(dest, ToRegister(lhs), ToRegister(rhs), &multRegOverflow);
+ bailoutFrom(&multRegOverflow, ins->snapshot());
+ } else {
+ masm.as_mullw(dest, ToRegister(lhs), ToRegister(rhs));
+ }
+
+ if (mul->canBeNegativeZero()) {
+ Label done;
+ masm.ma_bc(dest, dest, &done, Assembler::NonZero, ShortJump);
+
+ // Result is -0 if lhs or rhs is negative.
+ // In that case result must be double value so bailout
+ Register scratch = SecondScratchReg;
+ masm.as_or(scratch, ToRegister(lhs), ToRegister(rhs));
+ bailoutCmp32(Assembler::Signed, scratch, scratch, ins->snapshot());
+
+ masm.bind(&done);
+ }
+ }
+}
+
+void
+CodeGenerator::visitMulI64(LMulI64* lir)
+{
+ ADBlock();
+ const LInt64Allocation lhs = lir->getInt64Operand(LMulI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LMulI64::Rhs);
+ const Register64 output = ToOutRegister64(lir);
+
+ if (IsConstant(rhs)) {
+ int64_t constant = ToInt64(rhs);
+ switch (constant) {
+ case -1:
+ masm.neg64(ToRegister64(lhs));
+ return;
+ case 0:
+ masm.xor64(ToRegister64(lhs), ToRegister64(lhs));
+ return;
+ case 1:
+ // nop
+ return;
+ default:
+ if (constant > 0) {
+ if (mozilla::IsPowerOfTwo(static_cast<uint32_t>(constant + 1))) {
+ masm.move64(ToRegister64(lhs), output);
+ masm.lshift64(Imm32(FloorLog2(constant + 1)), output);
+ masm.sub64(ToRegister64(lhs), output);
+ return;
+ } else if (mozilla::IsPowerOfTwo(static_cast<uint32_t>(constant - 1))) {
+ masm.move64(ToRegister64(lhs), output);
+ masm.lshift64(Imm32(FloorLog2(constant - 1u)), output);
+ masm.add64(ToRegister64(lhs), output);
+ return;
+ }
+ // Use shift if constant is power of 2.
+ int32_t shift = mozilla::FloorLog2(constant);
+ if (int64_t(1) << shift == constant) {
+ masm.lshift64(Imm32(shift), ToRegister64(lhs));
+ return;
+ }
+ }
+ Register temp = ToTempRegisterOrInvalid(lir->temp());
+ masm.mul64(Imm64(constant), ToRegister64(lhs), temp);
+ }
+ } else {
+ Register temp = ToTempRegisterOrInvalid(lir->temp());
+ masm.mul64(ToOperandOrRegister64(rhs), ToRegister64(lhs), temp);
+ }
+}
+
+void
+CodeGenerator::visitDivI(LDivI* ins)
+{
+ ADBlock();
+ // divwo will automatically set the Overflow bit if INT32_MIN/-1 is
+ // performed, or if we divide by zero, so we just bailout based on that.
+ // However, it does not trigger an exception for a negative zero result,
+ // so we must check that first.
+
+ // Extract the registers from this instruction. We don't use a temp.
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register dest = ToRegister(ins->output());
+ MDiv* mir = ins->mir();
+
+ Label done, noOverflow;
+
+ // Handle negative 0. (0/-Y)
+ if (!mir->canTruncateNegativeZero() && mir->canBeNegativeZero()) {
+ Label nonzero;
+
+ MOZ_ASSERT(ins->snapshot());
+ masm.ma_bc(lhs, lhs, &nonzero, Assembler::NonZero, ShortJump);
+ bailoutCmp32(Assembler::LessThan, rhs, Imm32(0), ins->snapshot());
+ masm.bind(&nonzero);
+ }
+
+ // Not negative zero. So: DIVIDE! AND! CONQUER!
+ if (mir->canBeNegativeOverflow() || mir->canBeDivideByZero()) {
+ // If Overflow is set, we must bail out. But only check this if Ion
+ // doesn't already know this operation is infallible (we can emit
+ // a lot fewer instructions).
+
+ masm.ma_li(dest, 0L);
+ masm.xs_mtxer(dest); // whack XER[SO]
+ masm.as_divwo_rc(dest, lhs, rhs); // T = A / B
+
+ // Did we trap?
+ masm.ma_bc(Assembler::NSOBit, &noOverflow, ShortJump);
+ // Yes. Determine how.
+ if (mir->canBeNegativeOverflow()) {
+ Label notNegOne;
+
+ if (mir->canBeDivideByZero()) {
+ // If rhs is -1, then we overflowed by dividing INT32_MIN;
+ // otherwise, it must be divide by zero.
+ masm.as_cmpdi(rhs, -1);
+ masm.ma_bc(Assembler::NotEqual, &notNegOne, ShortJump);
+ }
+
+ if (mir->trapOnError()) {
+ masm.wasmTrap(wasm::Trap::IntegerOverflow, mir->bytecodeOffset());
+ } else if (mir->canTruncateOverflow()) {
+ // (-INT32_MIN)|0 == INT32_MIN
+ masm.move32(Imm32(INT32_MIN), dest);
+ masm.ma_b(&done, ShortJump);
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ bailout(ins->snapshot());
+ }
+
+ masm.bind(&notNegOne);
+ }
+ if (mir->canBeDivideByZero()) {
+ // Must have been divide by zero.
+ if (mir->trapOnError()) {
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->bytecodeOffset());
+ } else if (mir->canTruncateInfinities()) {
+ // If we can truncate division, then a truncated division by
+ // zero is always zero, and not actually an exception.
+ masm.move32(Imm32(0), dest);
+ masm.ma_b(&done, ShortJump);
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ bailout(ins->snapshot());
+ }
+ }
+ } else {
+ masm.as_divw(dest, lhs, rhs);
+ }
+ masm.bind(&noOverflow);
+
+ if (!mir->canTruncateRemainder()) {
+ MOZ_ASSERT(mir->fallible());
+
+ // Recover the remainder to see if we need to bailout.
+ if (HasPPCISA3()) {
+ masm.as_modsw(SecondScratchReg, lhs, rhs);
+ } else {
+ // Do it the old way.
+ // We don't need to check overflow with mullw; we know it can't.
+ masm.as_mullw(ScratchRegister, dest, rhs);
+ masm.as_subf(SecondScratchReg, ScratchRegister, lhs); // T = B - A
+ }
+ bailoutCmp32(Assembler::NotEqual, SecondScratchReg, Imm32(0), ins->snapshot());
+ }
+
+ masm.bind(&done);
+}
+
+void
+CodeGenerator::visitDivPowTwoI(LDivPowTwoI* ins)
+{
+ ADBlock();
+ Register lhs = ToRegister(ins->numerator());
+ Register dest = ToRegister(ins->output());
+ Register tmp = ToRegister(ins->getTemp(0));
+ int32_t shift = ins->shift();
+
+ if (shift != 0) {
+ MOZ_ASSERT(shift < 32);
+ MDiv* mir = ins->mir();
+ if (!mir->isTruncated()) {
+ // If the remainder is going to be != 0, bailout since this must
+ // be a double.
+ masm.x_slwi(tmp, lhs, 32 - shift);
+ bailoutCmp32(Assembler::NonZero, tmp, tmp, ins->snapshot());
+ }
+
+ if (!mir->canBeNegativeDividend()) {
+ // Numerator is unsigned, so needs no adjusting. Do the shift.
+ masm.as_srawi(dest, lhs, shift);
+ return;
+ }
+
+ // Adjust the value so that shifting produces a correctly rounded result
+ // when the numerator is negative. See 10-1 "Signed Division by a Known
+ // Power of 2" in Henry S. Warren, Jr.'s Hacker's Delight.
+ if (shift > 1) {
+ masm.as_srawi(tmp, lhs, 31);
+ masm.x_srwi(tmp, tmp, 32 - shift);
+ masm.add32(lhs, tmp);
+ } else {
+ MOZ_ASSERT(shift == 1);
+ masm.x_srwi(tmp, lhs, 31);
+ masm.add32(lhs, tmp);
+ }
+
+ // Do the shift.
+ masm.as_srawi(dest, tmp, shift);
+ } else {
+ masm.move32(lhs, dest);
+ }
+}
+
+void
+CodeGenerator::visitModI(LModI* ins)
+{
+ ADBlock();
+
+ // Extract the registers from this instruction (callTemp not currently
+ // used for POWER9, but may be a useful temp register for POWER8-).
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register dest = ToRegister(ins->output());
+ MMod* mir = ins->mir();
+ Label done, prevent;
+
+ // If computing 0/x where x < 0, modsw won't raise an exception but
+ // we have to return -0.0 (which requires a bailout). Since we have to
+ // probe the y/0 case necessarily, let's just check for all of them.
+ if (mir->canBeDivideByZero()) {
+ if (mir->isTruncated()) {
+ if (mir->trapOnError()) {
+ Label nonZero;
+ masm.ma_bc(rhs, rhs, &nonZero, Assembler::NonZero, ShortJump);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->bytecodeOffset());
+ masm.bind(&nonZero);
+ } else {
+ Label skip;
+ masm.ma_bc(rhs, Imm32(0), &skip, Assembler::NotEqual, ShortJump);
+ masm.move32(Imm32(0), dest);
+ masm.ma_b(&done, ShortJump);
+ masm.bind(&skip);
+ }
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ bailoutCmp32(Assembler::Equal, rhs, Imm32(0), ins->snapshot());
+ }
+ }
+
+#if(0)
+ if (mir->canBeNegativeDividend()) {
+ Label dividendOk;
+ masm.ma_bc(rhs, Imm32(0), &dividendOk, Assembler::GreaterThan, ShortJump);
+
+ if (mir->isTruncated()) {
+ // NaN|0 == 0 and (0 % -X)|0 == 0
+ Label skip;
+ masm.ma_bc(lhs, Imm32(0), &skip, Assembler::NotEqual, ShortJump);
+ masm.move32(Imm32(0), dest);
+ masm.ma_b(&done, ShortJump);
+ masm.bind(&skip);
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ bailoutCmp32(Assembler::Equal, lhs, Imm32(0), ins->snapshot());
+ }
+
+ // Now we need to do a test division. If divwo. flags overflow,
+ // then we must be trying to do INT_MIN % -1. modsw can't detect
+ // this either.
+ masm.ma_li(dest, 0L);
+ masm.xs_mtxer(dest); // whack XER[SO]
+ masm.as_divwo_rc(dest, lhs, rhs);
+ masm.ma_bc(Assembler::NSOBit, &dividendOk, ShortJump);
+
+ // Overflowed.
+ if (mir->isTruncated()) {
+ // (INT_MIN % -1)|0 == 0
+ Label skip;
+ masm.ma_bc(rhs, Imm32(-1), &skip, Assembler::NotEqual, ShortJump);
+ masm.move32(Imm32(0), dest);
+ masm.ma_b(&done, ShortJump);
+ masm.bind(&skip);
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ bailoutCmp32(Assembler::Equal, rhs, Imm32(-1), ins->snapshot());
+ }
+
+ // Dividend wasn't negative, or not INT_MIN % -1.
+ masm.bind(&dividendOk);
+ }
+#endif
+
+ // Test division functioned (or statically known not to fail), so now
+ // we can safely compute the modulo. The definition is consistent with
+ // modsw on POWER9.
+ if (HasPPCISA3()) {
+ masm.as_modsw(dest, lhs, rhs);
+ } else {
+ // Compute remainder manually. We can use the already computed
+ // quotient in |dest| if we know we computed it, or do a divw here
+ // since we have already tested for all the needed failure cases.
+ MOZ_CRASH("implement for POWER8 and earlier");
+ }
+
+ // If X%Y == 0 and X < 0, then we *actually* wanted to return -0.0
+ if (mir->canBeNegativeDividend()) {
+ if (mir->isTruncated()) {
+ // -0.0|0 == 0
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ // See if X < 0
+ masm.ma_bc(dest, Imm32(0), &done, Assembler::NotEqual, ShortJump);
+ bailoutCmp32(Assembler::LessThan, lhs, Imm32(0), ins->snapshot());
+ }
+ }
+ masm.bind(&done);
+}
+
+void
+CodeGenerator::visitModPowTwoI(LModPowTwoI* ins)
+{
+ ADBlock();
+
+ Register in = ToRegister(ins->getOperand(0));
+ Register out = ToRegister(ins->getDef(0));
+ MMod* mir = ins->mir();
+ Label negative, done;
+
+ masm.move32(in, out);
+ masm.ma_bc(in, in, &done, Assembler::Zero, ShortJump);
+ // Switch based on sign of the lhs.
+ // Positive numbers are just a bitmask
+ masm.ma_bc(in, in, &negative, Assembler::Signed, ShortJump);
+ {
+ masm.and32(Imm32((1 << ins->shift()) - 1), out);
+ masm.ma_b(&done, ShortJump);
+ }
+
+ // Negative numbers need a negate, bitmask, negate
+ {
+ masm.bind(&negative);
+ masm.neg32(out);
+ masm.and32(Imm32((1 << ins->shift()) - 1), out);
+ masm.neg32(out);
+ }
+ if (mir->canBeNegativeDividend()) {
+ if (!mir->isTruncated()) {
+ MOZ_ASSERT(mir->fallible());
+ bailoutCmp32(Assembler::Equal, out, Imm32(0L), ins->snapshot());
+ } else {
+ // -0|0 == 0
+ }
+ }
+ masm.bind(&done);
+}
+
+// This isn't used on ISA 3.0+ because we have hardware modulo, but it's
+// here for future expansion on older chips.
+void
+CodeGenerator::visitModMaskI(LModMaskI* ins)
+{
+ ADBlock();
+ MOZ_ASSERT(!HasPPCISA3());
+ // We wish to compute x % (1<<y) - 1 for a known constant, y.
+
+ //Register src = ToRegister(ins->getOperand(0));
+ //Register dest = ToRegister(ins->getDef(0));
+ //Register tmp0 = ToRegister(ins->getTemp(0));
+ //Register tmp1 = ToRegister(ins->getTemp(1));
+ MMod* mir = ins->mir();
+
+ if (!mir->isTruncated() && mir->canBeNegativeDividend()) {
+ // Sorry, POWER8, you'll need to do like MIPS.
+ MOZ_CRASH("implement visitModMaskI for POWER8-");
+ } else {
+ MOZ_CRASH("implement visitModMaskI for POWER8-");
+ }
+}
+
+void
+CodeGenerator::visitBitNotI(LBitNotI* ins)
+{
+ ADBlock();
+
+ const LAllocation* input = ins->getOperand(0);
+ const LDefinition* dest = ins->getDef(0);
+ MOZ_ASSERT(!input->isConstant());
+
+ masm.x_not(ToRegister(dest), ToRegister(input));
+}
+
+void
+CodeGenerator::visitBitOpI(LBitOpI* ins)
+{
+ ADBlock();
+
+ const LAllocation* lhs = ins->getOperand(0);
+ const LAllocation* rhs = ins->getOperand(1);
+ const LDefinition* dest = ins->getDef(0);
+ // all of these bitops should be either imm32's, or integer registers.
+ switch (ins->bitop()) {
+ case JSOp::BitOr:
+ if (rhs->isConstant())
+ masm.ma_or(ToRegister(dest), ToRegister(lhs), Imm32(ToInt32(rhs)));
+ else
+ masm.as_or(ToRegister(dest), ToRegister(lhs), ToRegister(rhs));
+ break;
+ case JSOp::BitXor:
+ if (rhs->isConstant())
+ masm.ma_xor(ToRegister(dest), ToRegister(lhs), Imm32(ToInt32(rhs)));
+ else
+ masm.as_xor(ToRegister(dest), ToRegister(lhs), ToRegister(rhs));
+ break;
+ case JSOp::BitAnd:
+ if (rhs->isConstant())
+ masm.ma_and(ToRegister(dest), ToRegister(lhs), Imm32(ToInt32(rhs)));
+ else
+ masm.as_and(ToRegister(dest), ToRegister(lhs), ToRegister(rhs));
+ break;
+ default:
+ MOZ_CRASH("unexpected binary opcode");
+ }
+}
+
+void
+CodeGenerator::visitBitOpI64(LBitOpI64* lir)
+{
+ ADBlock();
+ const LInt64Allocation lhs = lir->getInt64Operand(LBitOpI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LBitOpI64::Rhs);
+
+ MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
+
+ switch (lir->bitop()) {
+ case JSOp::BitOr:
+ if (IsConstant(rhs))
+ masm.or64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
+ else
+ masm.or64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
+ break;
+ case JSOp::BitXor:
+ if (IsConstant(rhs))
+ masm.xor64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
+ else
+ masm.xor64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
+ break;
+ case JSOp::BitAnd:
+ if (IsConstant(rhs))
+ masm.and64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
+ else
+ masm.and64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
+ break;
+ default:
+ MOZ_CRASH("unexpected binary opcode");
+ }
+}
+
+void
+CodeGenerator::visitShiftI(LShiftI* ins)
+{
+ ADBlock();
+ Register lhs = ToRegister(ins->lhs());
+ const LAllocation* rhs = ins->rhs();
+ Register dest = ToRegister(ins->output());
+
+ if (rhs->isConstant()) {
+ int32_t shift = ToInt32(rhs) & 0x1F;
+ switch (ins->bitop()) {
+ case JSOp::Lsh:
+ if (shift)
+ masm.x_slwi(dest, lhs, shift);
+ else
+ masm.move32(lhs, dest);
+ break;
+ case JSOp::Rsh:
+ if (shift)
+ masm.as_srawi(dest, lhs, shift);
+ else
+ masm.move32(lhs, dest);
+ break;
+ case JSOp::Ursh:
+ if (shift) {
+ masm.x_srwi(dest, lhs, shift);
+ } else {
+ // x >>> 0 can overflow.
+ if (ins->mir()->toUrsh()->fallible()) {
+ bailoutCmp32(Assembler::LessThan, lhs, Imm32(0), ins->snapshot());
+ }
+ masm.move32(lhs, dest);
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift op");
+ }
+ } else {
+ // The shift amounts should be AND'ed into the 0-31 range.
+ masm.as_andi_rc(dest, ToRegister(rhs), 0x1f);
+
+ switch (ins->bitop()) {
+ case JSOp::Lsh:
+ masm.as_slw(dest, lhs, dest);
+ break;
+ case JSOp::Rsh:
+ masm.as_sraw(dest, lhs, dest);
+ break;
+ case JSOp::Ursh:
+ masm.as_srw(dest, lhs, dest);
+ if (ins->mir()->toUrsh()->fallible()) {
+ // x >>> 0 can overflow.
+ bailoutCmp32(Assembler::LessThan, dest, Imm32(0), ins->snapshot());
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift op");
+ }
+ }
+}
+
+void
+CodeGenerator::visitShiftI64(LShiftI64* lir)
+{
+ ADBlock();
+ const LInt64Allocation lhs = lir->getInt64Operand(LShiftI64::Lhs);
+ LAllocation* rhs = lir->getOperand(LShiftI64::Rhs);
+
+ MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
+
+ if (rhs->isConstant()) {
+ int32_t shift = int32_t(rhs->toConstant()->toInt64() & 0x3F);
+ switch (lir->bitop()) {
+ case JSOp::Lsh:
+ if (shift)
+ masm.lshift64(Imm32(shift), ToRegister64(lhs));
+ break;
+ case JSOp::Rsh:
+ if (shift)
+ masm.rshift64Arithmetic(Imm32(shift), ToRegister64(lhs));
+ break;
+ case JSOp::Ursh:
+ if (shift)
+ masm.rshift64(Imm32(shift), ToRegister64(lhs));
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift op");
+ }
+ return;
+ }
+
+ switch (lir->bitop()) {
+ case JSOp::Lsh:
+ masm.lshift64(ToRegister(rhs), ToRegister64(lhs));
+ break;
+ case JSOp::Rsh:
+ masm.rshift64Arithmetic(ToRegister(rhs), ToRegister64(lhs));
+ break;
+ case JSOp::Ursh:
+ masm.rshift64(ToRegister(rhs), ToRegister64(lhs));
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift op");
+ }
+}
+
+void
+CodeGenerator::visitRotateI64(LRotateI64* lir)
+{
+ ADBlock();
+ MRotate* mir = lir->mir();
+ LAllocation* count = lir->count();
+
+ Register64 input = ToRegister64(lir->input());
+ Register64 output = ToOutRegister64(lir);
+ Register temp = ToTempRegisterOrInvalid(lir->temp());
+
+ MOZ_ASSERT(input == output);
+
+ if (count->isConstant()) {
+ int32_t c = int32_t(count->toConstant()->toInt64() & 0x3F);
+ if (!c) {
+ masm.move64(input, output);
+ return;
+ }
+ if (mir->isLeftRotate())
+ masm.rotateLeft64(Imm32(c), input, output, temp);
+ else
+ masm.rotateRight64(Imm32(c), input, output, temp);
+ } else {
+ if (mir->isLeftRotate())
+ masm.rotateLeft64(ToRegister(count), input, output, temp);
+ else
+ masm.rotateRight64(ToRegister(count), input, output, temp);
+ }
+}
+
+void
+CodeGenerator::visitUrshD(LUrshD* ins)
+{
+ ADBlock();
+ Register lhs = ToRegister(ins->lhs());
+ Register temp = ToRegister(ins->temp());
+
+ const LAllocation* rhs = ins->rhs();
+ FloatRegister out = ToFloatRegister(ins->output());
+
+ if (rhs->isConstant()) {
+ // See MacroAssembler::rshift32 for constraints.
+ int32_t rrhs = ToInt32(rhs);
+ MOZ_ASSERT(rrhs >= 0);
+
+ if (!(rrhs % 32)) {
+ // Effective no-op. Drop the mic.
+ masm.convertUInt32ToDouble(lhs, out);
+ return;
+ } else {
+ masm.x_srwi(temp, lhs, (rrhs & 31));
+ }
+ } else {
+ masm.as_srw(temp, lhs, ToRegister(rhs));
+ }
+
+ masm.convertUInt32ToDouble(temp, out);
+}
+
+void
+CodeGenerator::visitClzI(LClzI* ins)
+{
+ ADBlock();
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+
+ masm.as_cntlzw(output, input);
+}
+
+void
+CodeGenerator::visitCtzI(LCtzI* ins)
+{
+ ADBlock();
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+
+ if (HasPPCISA3()) {
+ masm.as_cnttzw(output, input);
+ } else {
+ MOZ_CRASH("cnttzw");
+ }
+}
+
+void
+CodeGenerator::visitPopcntI(LPopcntI* ins)
+{
+ ADBlock();
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+
+ masm.as_popcntw(output, input);
+}
+
+void
+CodeGenerator::visitPopcntI64(LPopcntI64* ins)
+{
+ ADBlock();
+ Register64 input = ToRegister64(ins->getInt64Operand(0));
+ Register64 output = ToOutRegister64(ins);
+
+ masm.as_popcntd(output.reg, input.reg);
+}
+
+void
+CodeGenerator::visitPowHalfD(LPowHalfD* ins)
+{
+ ADBlock();
+ FloatRegister input = ToFloatRegister(ins->input());
+ FloatRegister output = ToFloatRegister(ins->output());
+
+ Label done, skip;
+
+ // Masm.pow(-Infinity, 0.5) == Infinity.
+ masm.loadConstantDouble(NegativeInfinity<double>(), ScratchDoubleReg);
+ masm.as_fcmpu(input, ScratchDoubleReg);
+ masm.ma_bc(Assembler::DoubleNotEqualOrUnordered, &skip, ShortJump);
+ masm.as_fneg(output, ScratchDoubleReg);
+ masm.ma_b(&done, ShortJump);
+
+ masm.bind(&skip);
+ // Math.pow(-0, 0.5) == 0 == Math.pow(0, 0.5).
+ // Adding 0 converts any -0 to 0.
+ masm.zeroDouble(ScratchDoubleReg);
+ masm.as_fadd(output, input, ScratchDoubleReg);
+ masm.as_fsqrt(output, output);
+
+ masm.bind(&done);
+}
+
+MoveOperand
+CodeGeneratorPPC64::toMoveOperand(LAllocation a) const
+{
+ if (a.isGeneralReg())
+ return MoveOperand(ToRegister(a));
+ if (a.isFloatReg()) {
+ return MoveOperand(ToFloatRegister(a));
+ }
+ MoveOperand::Kind kind =
+ a.isStackArea() ? MoveOperand::Kind::EffectiveAddress : MoveOperand::Kind::Memory;
+ Address address = ToAddress(a);
+ MOZ_ASSERT((address.offset & 3) == 0);
+ return MoveOperand(address, kind);
+}
+
+void
+CodeGenerator::visitMathD(LMathD* math)
+{
+ ADBlock();
+ FloatRegister src1 = ToFloatRegister(math->getOperand(0));
+ FloatRegister src2 = ToFloatRegister(math->getOperand(1));
+ FloatRegister output = ToFloatRegister(math->getDef(0));
+
+ switch (math->jsop()) {
+ case JSOp::Add:
+ masm.as_fadd(output, src1, src2);
+ break;
+ case JSOp::Sub:
+ masm.as_fsub(output, src1, src2); // T = A - B
+ break;
+ case JSOp::Mul:
+ masm.as_fmul(output, src1, src2);
+ break;
+ case JSOp::Div:
+ masm.as_fdiv(output, src1, src2);
+ break;
+ default:
+ MOZ_CRASH("unexpected opcode");
+ }
+}
+
+void
+CodeGenerator::visitMathF(LMathF* math)
+{
+ ADBlock();
+ FloatRegister src1 = ToFloatRegister(math->getOperand(0));
+ FloatRegister src2 = ToFloatRegister(math->getOperand(1));
+ FloatRegister output = ToFloatRegister(math->getDef(0));
+
+ switch (math->jsop()) {
+ case JSOp::Add:
+ masm.as_fadds(output, src1, src2);
+ break;
+ case JSOp::Sub:
+ masm.as_fsubs(output, src1, src2);
+ break;
+ case JSOp::Mul:
+ masm.as_fmuls(output, src1, src2);
+ break;
+ case JSOp::Div:
+ masm.as_fdivs(output, src1, src2);
+ break;
+ default:
+ MOZ_CRASH("unexpected opcode");
+ }
+}
+
+void
+CodeGenerator::visitTruncateDToInt32(LTruncateDToInt32* ins)
+{
+ emitTruncateDouble(ToFloatRegister(ins->input()), ToRegister(ins->output()),
+ ins->mir());
+}
+
+void
+CodeGenerator::visitTruncateFToInt32(LTruncateFToInt32* ins)
+{
+ emitTruncateFloat32(ToFloatRegister(ins->input()), ToRegister(ins->output()),
+ ins->mir());
+}
+
+void
+CodeGenerator::visitWasmTruncateToInt32(LWasmTruncateToInt32* lir)
+{
+ ADBlock();
+ auto input = ToFloatRegister(lir->input());
+ auto output = ToRegister(lir->output());
+
+ MWasmTruncateToInt32* mir = lir->mir();
+ MIRType fromType = mir->input()->type();
+
+ MOZ_ASSERT(fromType == MIRType::Double || fromType == MIRType::Float32);
+
+ auto* ool = new (alloc()) OutOfLineWasmTruncateCheck(mir, input, output);
+ addOutOfLineCode(ool, mir);
+
+ Label* oolEntry = ool->entry();
+ if (mir->isUnsigned()) {
+ if (fromType == MIRType::Double)
+ masm.wasmTruncateDoubleToUInt32(input, output, mir->isSaturating(), oolEntry);
+ else if (fromType == MIRType::Float32)
+ masm.wasmTruncateFloat32ToUInt32(input, output, mir->isSaturating(), oolEntry);
+ else
+ MOZ_CRASH("unexpected type");
+
+ masm.bind(ool->rejoin());
+ return;
+ }
+
+ if (fromType == MIRType::Double)
+ masm.wasmTruncateDoubleToInt32(input, output, mir->isSaturating(), oolEntry);
+ else if (fromType == MIRType::Float32)
+ masm.wasmTruncateFloat32ToInt32(input, output, mir->isSaturating(), oolEntry);
+ else
+ MOZ_CRASH("unexpected type");
+
+ masm.bind(ool->rejoin());
+}
+
+
+void
+CodeGeneratorPPC64::visitOutOfLineBailout(OutOfLineBailout* ool)
+{
+ ADBlock();
+
+ // Push snapshotOffset and make sure stack is aligned.
+// XXX: this should just be an stdu
+ masm.subPtr(Imm32(sizeof(Value)), StackPointer);
+ masm.storePtr(ImmWord(ool->snapshot()->snapshotOffset()), Address(StackPointer, 0));
+
+ masm.jump(&deoptLabel_);
+}
+
+void
+CodeGeneratorPPC64::visitOutOfLineWasmTruncateCheck(OutOfLineWasmTruncateCheck* ool)
+{
+ if(ool->toType() == MIRType::Int32)
+ {
+ masm.outOfLineWasmTruncateToInt32Check(ool->input(), ool->output(), ool->fromType(),
+ ool->flags(), ool->rejoin(), ool->bytecodeOffset());
+ } else {
+ MOZ_ASSERT(ool->toType() == MIRType::Int64);
+ masm.outOfLineWasmTruncateToInt64Check(ool->input(), ool->output64(), ool->fromType(),
+ ool->flags(), ool->rejoin(), ool->bytecodeOffset());
+ }
+}
+
+void
+CodeGenerator::visitCopySignF(LCopySignF* ins)
+{
+ ADBlock();
+ FloatRegister lhs = ToFloatRegister(ins->getOperand(0));
+ FloatRegister rhs = ToFloatRegister(ins->getOperand(1));
+ FloatRegister output = ToFloatRegister(ins->getDef(0));
+
+ // ISA 2.05+
+ masm.as_fcpsgn(output, lhs, rhs);
+}
+
+void
+CodeGenerator::visitCopySignD(LCopySignD* ins)
+{
+ ADBlock();
+ FloatRegister lhs = ToFloatRegister(ins->getOperand(0));
+ FloatRegister rhs = ToFloatRegister(ins->getOperand(1));
+ FloatRegister output = ToFloatRegister(ins->getDef(0));
+
+ // ISA 2.05+
+ masm.as_fcpsgn(output, lhs, rhs);
+}
+
+void
+CodeGenerator::visitValue(LValue* value)
+{
+ ADBlock();
+ const ValueOperand out = ToOutValue(value);
+
+ masm.moveValue(value->value(), out);
+}
+
+void
+CodeGenerator::visitDouble(LDouble* ins)
+{
+ ADBlock();
+ const LDefinition* out = ins->getDef(0);
+
+ masm.loadConstantDouble(ins->value(), ToFloatRegister(out));
+}
+
+void
+CodeGenerator::visitFloat32(LFloat32* ins)
+{
+ ADBlock();
+ const LDefinition* out = ins->getDef(0);
+ masm.loadConstantFloat32(ins->value(), ToFloatRegister(out));
+}
+
+void
+CodeGenerator::visitTestDAndBranch(LTestDAndBranch* test)
+{
+ ADBlock();
+ FloatRegister input = ToFloatRegister(test->input());
+
+ MBasicBlock* ifTrue = test->ifTrue();
+ MBasicBlock* ifFalse = test->ifFalse();
+
+// XXX: zeroDouble
+ masm.loadConstantDouble(0.0, ScratchDoubleReg);
+ // If 0, or NaN, the result is false.
+
+ if (isNextBlock(ifFalse->lir())) {
+ branchToBlock(Assembler::DoubleFloat, input, ScratchDoubleReg, ifTrue,
+ Assembler::DoubleNotEqual);
+ } else {
+ branchToBlock(Assembler::DoubleFloat, input, ScratchDoubleReg, ifFalse,
+ Assembler::DoubleEqualOrUnordered);
+ jumpToBlock(ifTrue);
+ }
+}
+
+// XXX: duplicate code
+void
+CodeGenerator::visitTestFAndBranch(LTestFAndBranch* test)
+{
+ ADBlock();
+ FloatRegister input = ToFloatRegister(test->input());
+
+ MBasicBlock* ifTrue = test->ifTrue();
+ MBasicBlock* ifFalse = test->ifFalse();
+
+// XXX: zeroDouble
+ masm.loadConstantFloat32(0.0f, ScratchFloat32Reg);
+ // If 0, or NaN, the result is false.
+
+ if (isNextBlock(ifFalse->lir())) {
+ branchToBlock(Assembler::SingleFloat, input, ScratchFloat32Reg, ifTrue,
+ Assembler::DoubleNotEqual);
+ } else {
+ branchToBlock(Assembler::SingleFloat, input, ScratchFloat32Reg, ifFalse,
+ Assembler::DoubleEqualOrUnordered);
+ jumpToBlock(ifTrue);
+ }
+}
+
+void
+CodeGenerator::visitCompareD(LCompareD* comp)
+{
+ ADBlock();
+ FloatRegister lhs = ToFloatRegister(comp->left());
+ FloatRegister rhs = ToFloatRegister(comp->right());
+ Register dest = ToRegister(comp->output());
+
+ Assembler::DoubleCondition cond = JSOpToDoubleCondition(comp->mir()->jsop());
+ masm.ma_cmp_set_double(dest, lhs, rhs, cond);
+}
+
+// XXX: duplicate code
+void
+CodeGenerator::visitCompareF(LCompareF* comp)
+{
+ ADBlock();
+ FloatRegister lhs = ToFloatRegister(comp->left());
+ FloatRegister rhs = ToFloatRegister(comp->right());
+ Register dest = ToRegister(comp->output());
+
+ Assembler::DoubleCondition cond = JSOpToDoubleCondition(comp->mir()->jsop());
+ masm.ma_cmp_set_double(dest, lhs, rhs, cond);
+}
+
+void
+CodeGenerator::visitCompareDAndBranch(LCompareDAndBranch* comp)
+{
+ ADBlock();
+ FloatRegister lhs = ToFloatRegister(comp->left());
+ FloatRegister rhs = ToFloatRegister(comp->right());
+
+ Assembler::DoubleCondition cond = JSOpToDoubleCondition(comp->cmpMir()->jsop());
+ MBasicBlock* ifTrue = comp->ifTrue();
+ MBasicBlock* ifFalse = comp->ifFalse();
+
+ if (isNextBlock(ifFalse->lir())) {
+ branchToBlock(Assembler::DoubleFloat, lhs, rhs, ifTrue, cond);
+ } else {
+ branchToBlock(Assembler::DoubleFloat, lhs, rhs, ifFalse,
+ Assembler::InvertCondition(cond));
+ jumpToBlock(ifTrue);
+ }
+}
+
+// XXX; duplicate code
+void
+CodeGenerator::visitCompareFAndBranch(LCompareFAndBranch* comp)
+{
+ ADBlock();
+ FloatRegister lhs = ToFloatRegister(comp->left());
+ FloatRegister rhs = ToFloatRegister(comp->right());
+
+ Assembler::DoubleCondition cond = JSOpToDoubleCondition(comp->cmpMir()->jsop());
+ MBasicBlock* ifTrue = comp->ifTrue();
+ MBasicBlock* ifFalse = comp->ifFalse();
+
+ if (isNextBlock(ifFalse->lir())) {
+ branchToBlock(Assembler::SingleFloat, lhs, rhs, ifTrue, cond);
+ } else {
+ branchToBlock(Assembler::SingleFloat, lhs, rhs, ifFalse,
+ Assembler::InvertCondition(cond));
+ jumpToBlock(ifTrue);
+ }
+}
+
+void
+CodeGenerator::visitBitAndAndBranch(LBitAndAndBranch* lir)
+{
+ ADBlock();
+ if (lir->right()->isConstant())
+ masm.ma_and(ScratchRegister, ToRegister(lir->left()), Imm32(ToInt32(lir->right())));
+ else
+ masm.as_and(ScratchRegister, ToRegister(lir->left()), ToRegister(lir->right()));
+// XXX: use direct result instead of doing a second compare here
+ emitBranch(ScratchRegister, ScratchRegister, lir->cond(), lir->ifTrue(),
+ lir->ifFalse());
+}
+
+void
+CodeGenerator::visitWasmUint32ToDouble(LWasmUint32ToDouble* lir)
+{
+ masm.convertUInt32ToDouble(ToRegister(lir->input()), ToFloatRegister(lir->output()));
+}
+
+void
+CodeGenerator::visitWasmUint32ToFloat32(LWasmUint32ToFloat32* lir)
+{
+ masm.convertUInt32ToFloat32(ToRegister(lir->input()), ToFloatRegister(lir->output()));
+}
+
+void
+CodeGenerator::visitNotI(LNotI* ins)
+{
+ ADBlock();
+ masm.cmp32Set(Assembler::Equal, ToRegister(ins->input()), Imm32(0),
+ ToRegister(ins->output()));
+}
+
+void
+CodeGenerator::visitNotD(LNotD* ins)
+{
+ ADBlock();
+ // Since this operation is |not|, we want to set a bit if
+ // the double is falsey, which means 0.0, -0.0 or NaN.
+ FloatRegister in = ToFloatRegister(ins->input());
+ Register dest = ToRegister(ins->output());
+
+ masm.zeroDouble(ScratchDoubleReg);
+ masm.ma_cmp_set_double(dest, in, ScratchDoubleReg, Assembler::DoubleEqualOrUnordered);
+}
+
+void
+CodeGenerator::visitNotF(LNotF* ins)
+{
+ ADBlock();
+ // Since this operation is |not|, we want to set a bit if
+ // the float32 is falsey, which means 0.0, -0.0 or NaN.
+ FloatRegister in = ToFloatRegister(ins->input());
+ Register dest = ToRegister(ins->output());
+
+ masm.zeroDouble(ScratchFloat32Reg);
+ masm.ma_cmp_set_double(dest, in, ScratchFloat32Reg, Assembler::DoubleEqualOrUnordered);
+}
+
+#if(0)
+void
+CodeGenerator::visitMemoryBarrier(LMemoryBarrier* ins)
+{
+ ADBlock();
+ masm.memoryBarrier(ins->type());
+}
+#endif
+
+void
+CodeGeneratorPPC64::generateInvalidateEpilogue()
+{
+ ADBlock();
+
+ // Ensure that there is enough space in the buffer for the OsiPoint
+ // patching to occur. Otherwise, we could overwrite the invalidation
+ // epilogue.
+ for (size_t i = 0; i < sizeof(void*); i += Assembler::NopSize())
+ masm.nop();
+
+ masm.bind(&invalidate_);
+
+ // Push the return address of the point that we bailed out at to the stack
+ masm.pushReturnAddress(); // LR
+
+ // Push the Ion script onto the stack (when we determine what that
+ // pointer is).
+ invalidateEpilogueData_ = masm.pushWithPatch(ImmWord(uintptr_t(-1)));
+ TrampolinePtr thunk = gen->jitRuntime()->getInvalidationThunk();
+ masm.jump(thunk); // A thunk sat on a thump.
+}
+
+class js::jit::OutOfLineTableSwitch : public OutOfLineCodeBase<CodeGeneratorPPC64>
+{
+ MTableSwitch* mir_;
+ CodeLabel jumpLabel_;
+
+ void accept(CodeGeneratorPPC64* codegen) {
+ codegen->visitOutOfLineTableSwitch(this);
+ }
+
+ public:
+ OutOfLineTableSwitch(MTableSwitch* mir)
+ : mir_(mir)
+ {}
+
+ MTableSwitch* mir() const {
+ return mir_;
+ }
+
+ CodeLabel* jumpLabel() {
+ return &jumpLabel_;
+ }
+};
+
+void
+CodeGeneratorPPC64::visitOutOfLineTableSwitch(OutOfLineTableSwitch* ool)
+{
+ ADBlock();
+ MTableSwitch* mir = ool->mir();
+
+ masm.haltingAlign(sizeof(void*));
+ masm.bind(ool->jumpLabel());
+ masm.addCodeLabel(*ool->jumpLabel());
+
+ for (size_t i = 0; i < mir->numCases(); i++) {
+ LBlock* caseblock = skipTrivialBlocks(mir->getCase(i))->lir();
+ Label* caseheader = caseblock->label();
+ uint32_t caseoffset = caseheader->offset();
+
+ // The entries of the jump table need to be absolute addresses and thus
+ // must be patched after codegen is finished.
+ CodeLabel cl;
+ masm.writeCodePointer(&cl);
+ cl.target()->bind(caseoffset);
+ masm.addCodeLabel(cl);
+ }
+}
+
+void
+CodeGeneratorPPC64::emitTableSwitchDispatch(MTableSwitch* mir, Register index,
+ Register base)
+{
+ ADBlock();
+ Label* defaultcase = skipTrivialBlocks(mir->getDefault())->lir()->label();
+
+ // Lower value with low value
+ if (mir->low() != 0)
+ masm.subPtr(Imm32(mir->low()), index);
+
+ // Jump to default case if input is out of range
+ int32_t cases = mir->numCases();
+ masm.branchPtr(Assembler::AboveOrEqual, index, ImmWord(cases), defaultcase);
+
+ // To fill in the CodeLabels for the case entries, we need to first
+ // generate the case entries (we don't yet know their offsets in the
+ // instruction stream).
+ OutOfLineTableSwitch* ool = new(alloc()) OutOfLineTableSwitch(mir);
+ addOutOfLineCode(ool, mir);
+
+ // Compute the position where a pointer to the right case stands.
+ masm.ma_li(base, ool->jumpLabel());
+
+ BaseIndex pointer(base, index, ScalePointer);
+
+ // Jump to the right case
+ masm.branchToComputedAddress(pointer);
+}
+
+template <typename T>
+void
+CodeGeneratorPPC64::emitWasmLoad(T* lir)
+{
+ ADBlock();
+ const MWasmLoad* mir = lir->mir();
+
+ Register ptrScratch = InvalidReg;
+ if(!lir->ptrCopy()->isBogusTemp()){
+ ptrScratch = ToRegister(lir->ptrCopy());
+ }
+
+ if (IsUnaligned(mir->access())) {
+ if (IsFloatingPointType(mir->type())) {
+ masm.wasmUnalignedLoadFP(mir->access(), HeapReg, ToRegister(lir->ptr()), ptrScratch,
+ ToFloatRegister(lir->output()), ToRegister(lir->getTemp(1)));
+ } else {
+ masm.wasmUnalignedLoad(mir->access(), HeapReg, ToRegister(lir->ptr()),
+ ptrScratch, ToRegister(lir->output()), ToRegister(lir->getTemp(1)));
+ }
+ } else {
+ masm.wasmLoad(mir->access(), HeapReg, ToRegister(lir->ptr()), ptrScratch,
+ ToAnyRegister(lir->output()));
+ }
+}
+
+void
+CodeGenerator::visitWasmLoad(LWasmLoad* lir)
+{
+ emitWasmLoad(lir);
+}
+
+void
+CodeGenerator::visitWasmUnalignedLoad(LWasmUnalignedLoad* lir)
+{
+ emitWasmLoad(lir);
+}
+
+template <typename T>
+void
+CodeGeneratorPPC64::emitWasmStore(T* lir)
+{
+ const MWasmStore* mir = lir->mir();
+
+ Register ptrScratch = InvalidReg;
+ if(!lir->ptrCopy()->isBogusTemp()){
+ ptrScratch = ToRegister(lir->ptrCopy());
+ }
+
+ if (IsUnaligned(mir->access())) {
+ if (mir->access().type() == Scalar::Float32 ||
+ mir->access().type() == Scalar::Float64) {
+ masm.wasmUnalignedStoreFP(mir->access(), ToFloatRegister(lir->value()),
+ HeapReg, ToRegister(lir->ptr()), ptrScratch,
+ ToRegister(lir->getTemp(1)));
+ } else {
+ masm.wasmUnalignedStore(mir->access(), ToRegister(lir->value()), HeapReg,
+ ToRegister(lir->ptr()), ptrScratch,
+ ToRegister(lir->getTemp(1)));
+ }
+ } else {
+ masm.wasmStore(mir->access(), ToAnyRegister(lir->value()), HeapReg,
+ ToRegister(lir->ptr()), ptrScratch);
+ }
+}
+
+void
+CodeGenerator::visitWasmStore(LWasmStore* lir)
+{
+ emitWasmStore(lir);
+}
+
+void
+CodeGenerator::visitWasmUnalignedStore(LWasmUnalignedStore* lir)
+{
+ emitWasmStore(lir);
+}
+
+void
+CodeGenerator::visitAsmJSLoadHeap(LAsmJSLoadHeap* ins)
+{
+ ADBlock();
+ const MAsmJSLoadHeap* mir = ins->mir();
+ const LAllocation* ptr = ins->ptr();
+ const LDefinition* out = ins->output();
+ const LAllocation* boundsCheckLimit = ins->boundsCheckLimit();
+
+ bool isSigned;
+ int size;
+ bool isFloat = false;
+ switch (mir->access().type()) {
+ case Scalar::Int8: isSigned = true; size = 8; break;
+ case Scalar::Uint8: isSigned = false; size = 8; break;
+ case Scalar::Int16: isSigned = true; size = 16; break;
+ case Scalar::Uint16: isSigned = false; size = 16; break;
+ case Scalar::Int32: isSigned = true; size = 32; break;
+ case Scalar::Uint32: isSigned = false; size = 32; break;
+ case Scalar::Float64: isFloat = true; size = 64; break;
+ case Scalar::Float32: isFloat = true; size = 32; break;
+ default: MOZ_CRASH("unexpected array type");
+ }
+
+ if (ptr->isConstant()) {
+ MOZ_ASSERT(!mir->needsBoundsCheck());
+ int32_t ptrImm = ptr->toConstant()->toInt32();
+ MOZ_ASSERT(ptrImm >= 0);
+ if (isFloat) {
+ if (size == 32) {
+ masm.loadFloat32(Address(HeapReg, ptrImm), ToFloatRegister(out));
+ } else {
+ masm.loadDouble(Address(HeapReg, ptrImm), ToFloatRegister(out));
+ }
+ } else {
+ masm.ma_load(ToRegister(out), Address(HeapReg, ptrImm),
+ static_cast<LoadStoreSize>(size), isSigned ? SignExtend : ZeroExtend);
+ }
+ return;
+ }
+
+ Register ptrReg = ToRegister(ptr);
+
+ if (!mir->needsBoundsCheck()) {
+ if (isFloat) {
+ if (size == 32) {
+ masm.loadFloat32(BaseIndex(HeapReg, ptrReg, TimesOne), ToFloatRegister(out));
+ } else {
+ masm.loadDouble(BaseIndex(HeapReg, ptrReg, TimesOne), ToFloatRegister(out));
+ }
+ } else {
+ masm.ma_load(ToRegister(out), BaseIndex(HeapReg, ptrReg, TimesOne),
+ static_cast<LoadStoreSize>(size), isSigned ? SignExtend : ZeroExtend);
+ }
+ return;
+ }
+
+ Label done, outOfRange;
+ masm.wasmBoundsCheck32(Assembler::AboveOrEqual, ptrReg, ToRegister(boundsCheckLimit),
+ &outOfRange);
+ // Offset is ok, let's load value.
+ if (isFloat) {
+ if (size == 32)
+ masm.loadFloat32(BaseIndex(HeapReg, ptrReg, TimesOne), ToFloatRegister(out));
+ else
+ masm.loadDouble(BaseIndex(HeapReg, ptrReg, TimesOne), ToFloatRegister(out));
+ } else {
+ masm.ma_load(ToRegister(out), BaseIndex(HeapReg, ptrReg, TimesOne),
+ static_cast<LoadStoreSize>(size), isSigned ? SignExtend : ZeroExtend);
+ }
+ masm.ma_b(&done, ShortJump);
+ masm.bind(&outOfRange);
+ // Offset is out of range. Load default values.
+ if (isFloat) {
+ if (size == 32)
+ masm.loadConstantFloat32(float(GenericNaN()), ToFloatRegister(out));
+ else
+ masm.loadConstantDouble(GenericNaN(), ToFloatRegister(out));
+ } else {
+ masm.move32(Imm32(0), ToRegister(out));
+ }
+ masm.bind(&done);
+}
+
+void
+CodeGenerator::visitAsmJSStoreHeap(LAsmJSStoreHeap* ins)
+{
+ ADBlock();
+ const MAsmJSStoreHeap* mir = ins->mir();
+ const LAllocation* value = ins->value();
+ const LAllocation* ptr = ins->ptr();
+ const LAllocation* boundsCheckLimit = ins->boundsCheckLimit();
+
+ bool isSigned; // Not currently used, but we might later.
+ int size;
+ bool isFloat = false;
+ switch (mir->access().type()) {
+ case Scalar::Int8: isSigned = true; size = 8; break;
+ case Scalar::Uint8: isSigned = false; size = 8; break;
+ case Scalar::Int16: isSigned = true; size = 16; break;
+ case Scalar::Uint16: isSigned = false; size = 16; break;
+ case Scalar::Int32: isSigned = true; size = 32; break;
+ case Scalar::Uint32: isSigned = false; size = 32; break;
+ case Scalar::Float64: isFloat = true; size = 64; break;
+ case Scalar::Float32: isFloat = true; size = 32; break;
+ default: MOZ_CRASH("unexpected array type");
+ }
+
+ if (ptr->isConstant()) {
+ MOZ_ASSERT(!mir->needsBoundsCheck());
+ int32_t ptrImm = ptr->toConstant()->toInt32();
+ MOZ_ASSERT(ptrImm >= 0);
+
+ if (isFloat) {
+ FloatRegister freg = ToFloatRegister(value);
+ Address addr(HeapReg, ptrImm);
+ if (size == 32)
+ masm.storeFloat32(freg, addr);
+ else
+ masm.storeDouble(freg, addr);
+ } else {
+ masm.ma_store(ToRegister(value), Address(HeapReg, ptrImm),
+ static_cast<LoadStoreSize>(size));
+ }
+ return;
+ }
+
+ Register ptrReg = ToRegister(ptr);
+ Address dstAddr(ptrReg, 0);
+
+ if (!mir->needsBoundsCheck()) {
+ if (isFloat) {
+ FloatRegister freg = ToFloatRegister(value);
+ BaseIndex bi(HeapReg, ptrReg, TimesOne);
+ if (size == 32)
+ masm.storeFloat32(freg, bi);
+ else
+ masm.storeDouble(freg, bi);
+ } else {
+ masm.ma_store(ToRegister(value), BaseIndex(HeapReg, ptrReg, TimesOne),
+ static_cast<LoadStoreSize>(size));
+ }
+ return;
+ }
+
+ Label outOfRange;
+ masm.wasmBoundsCheck32(Assembler::AboveOrEqual, ptrReg, ToRegister(boundsCheckLimit),
+ &outOfRange);
+
+ // Offset is ok, let's store value.
+ if (isFloat) {
+ if (size == 32) {
+ masm.storeFloat32(ToFloatRegister(value), BaseIndex(HeapReg, ptrReg, TimesOne));
+ } else
+ masm.storeDouble(ToFloatRegister(value), BaseIndex(HeapReg, ptrReg, TimesOne));
+ } else {
+ masm.ma_store(ToRegister(value), BaseIndex(HeapReg, ptrReg, TimesOne),
+ static_cast<LoadStoreSize>(size));
+ }
+
+ masm.bind(&outOfRange);
+}
+
+void
+CodeGenerator::visitWasmCompareExchangeHeap(LWasmCompareExchangeHeap* ins)
+{
+ ADBlock();
+ MWasmCompareExchangeHeap* mir = ins->mir();
+ Scalar::Type vt = mir->access().type();
+ Register ptrReg = ToRegister(ins->ptr());
+ BaseIndex srcAddr(HeapReg, ptrReg, TimesOne, mir->access().offset());
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ Register oldval = ToRegister(ins->oldValue());
+ Register newval = ToRegister(ins->newValue());
+ Register valueTemp = ToTempRegisterOrInvalid(ins->valueTemp());
+ Register offsetTemp = ToTempRegisterOrInvalid(ins->offsetTemp());
+ Register maskTemp = ToTempRegisterOrInvalid(ins->maskTemp());
+
+ masm.compareExchange(vt, Synchronization::Full(), srcAddr, oldval, newval, valueTemp,
+ offsetTemp, maskTemp, ToRegister(ins->output()));
+}
+
+void
+CodeGenerator::visitWasmAtomicExchangeHeap(LWasmAtomicExchangeHeap* ins)
+{
+ ADBlock();
+ MWasmAtomicExchangeHeap* mir = ins->mir();
+ Scalar::Type vt = mir->access().type();
+ Register ptrReg = ToRegister(ins->ptr());
+ Register value = ToRegister(ins->value());
+ BaseIndex srcAddr(HeapReg, ptrReg, TimesOne, mir->access().offset());
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ Register valueTemp = ToTempRegisterOrInvalid(ins->valueTemp());
+ Register offsetTemp = ToTempRegisterOrInvalid(ins->offsetTemp());
+ Register maskTemp = ToTempRegisterOrInvalid(ins->maskTemp());
+
+ masm.atomicExchange(vt, Synchronization::Full(), srcAddr, value, valueTemp, offsetTemp,
+ maskTemp, ToRegister(ins->output()));
+}
+
+void
+CodeGenerator::visitWasmAtomicBinopHeap(LWasmAtomicBinopHeap* ins)
+{
+ ADBlock();
+ MOZ_ASSERT(ins->mir()->hasUses());
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ MWasmAtomicBinopHeap* mir = ins->mir();
+ Scalar::Type vt = mir->access().type();
+ Register ptrReg = ToRegister(ins->ptr());
+ Register valueTemp = ToTempRegisterOrInvalid(ins->valueTemp());
+ Register offsetTemp = ToTempRegisterOrInvalid(ins->offsetTemp());
+ Register maskTemp = ToTempRegisterOrInvalid(ins->maskTemp());
+
+ BaseIndex srcAddr(HeapReg, ptrReg, TimesOne, mir->access().offset());
+
+ masm.atomicFetchOp(vt, Synchronization::Full(), mir->operation(), ToRegister(ins->value()),
+ srcAddr, valueTemp, offsetTemp, maskTemp, ToRegister(ins->output()));
+}
+
+void
+CodeGenerator::visitWasmAtomicBinopHeapForEffect(LWasmAtomicBinopHeapForEffect* ins)
+{
+ ADBlock();
+ MOZ_ASSERT(!ins->mir()->hasUses());
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ MWasmAtomicBinopHeap* mir = ins->mir();
+ Scalar::Type vt = mir->access().type();
+ Register ptrReg = ToRegister(ins->ptr());
+ Register valueTemp = ToTempRegisterOrInvalid(ins->valueTemp());
+ Register offsetTemp = ToTempRegisterOrInvalid(ins->offsetTemp());
+ Register maskTemp = ToTempRegisterOrInvalid(ins->maskTemp());
+
+ BaseIndex srcAddr(HeapReg, ptrReg, TimesOne, mir->access().offset());
+ masm.atomicEffectOpJS(vt, Synchronization::Full(), mir->operation(), ToRegister(ins->value()),
+ srcAddr, valueTemp, offsetTemp, maskTemp);
+}
+
+void
+CodeGenerator::visitWasmStackArg(LWasmStackArg* ins)
+{
+ ADBlock();
+ const MWasmStackArg* mir = ins->mir();
+ if (ins->arg()->isConstant()) {
+ masm.storePtr(ImmWord(ToInt32(ins->arg())), Address(StackPointer, mir->spOffset()));
+ } else {
+ if (ins->arg()->isGeneralReg()) {
+ masm.storePtr(ToRegister(ins->arg()), Address(StackPointer, mir->spOffset()));
+ } else if (mir->input()->type() == MIRType::Double) {
+ masm.storeDouble(ToFloatRegister(ins->arg()),
+ Address(StackPointer, mir->spOffset()));
+ } else {
+ masm.storeFloat32(ToFloatRegister(ins->arg()),
+ Address(StackPointer, mir->spOffset()));
+ }
+ }
+}
+
+void
+CodeGenerator::visitWasmStackArgI64(LWasmStackArgI64* ins)
+{
+ ADBlock();
+ const MWasmStackArg* mir = ins->mir();
+ Address dst(StackPointer, mir->spOffset());
+ if (IsConstant(ins->arg()))
+ masm.store64(Imm64(ToInt64(ins->arg())), dst);
+ else
+ masm.store64(ToRegister64(ins->arg()), dst);
+}
+
+void
+CodeGenerator::visitWasmSelect(LWasmSelect* ins)
+{
+ ADBlock();
+ MIRType mirType = ins->mir()->type();
+
+ Register cond = ToRegister(ins->condExpr());
+ const LAllocation* falseExpr = ins->falseExpr();
+
+ if (mirType == MIRType::Int32) {
+ Register out = ToRegister(ins->output());
+ MOZ_ASSERT(ToRegister(ins->trueExpr()) == out, "true expr input is reused for output");
+ masm.as_cmpdi(cond, 0);
+MOZ_CRASH("visitWasmSelect NYI for isel"); // XXX: also check if 64-bit int
+ masm.as_isel(out, out, ToRegister(falseExpr), 2); // CR0[EQ]
+ return;
+ }
+
+ FloatRegister out = ToFloatRegister(ins->output());
+ MOZ_ASSERT(ToFloatRegister(ins->trueExpr()) == out, "true expr input is reused for output");
+
+ // We can't use fsel because cond isn't a float register.
+ Label done;
+ masm.ma_bc(cond, cond, &done, Assembler::NonZero, ShortJump);
+
+ if (mirType == MIRType::Float32)
+ masm.loadFloat32(ToAddress(falseExpr), out);
+ else if (mirType == MIRType::Double)
+ masm.loadDouble(ToAddress(falseExpr), out);
+ else
+ MOZ_CRASH("unhandled type in visitWasmSelect!");
+
+ masm.bind(&done);
+}
+
+// We expect to handle only the case where compare is {U,}Int32 and select is
+// {U,}Int32, and the "true" input is reused for the output.
+void CodeGenerator::visitWasmCompareAndSelect(LWasmCompareAndSelect* ins) {
+ bool cmpIs32bit = ins->compareType() == MCompare::Compare_Int32 ||
+ ins->compareType() == MCompare::Compare_UInt32;
+ bool selIs32bit = ins->mir()->type() == MIRType::Int32;
+
+ MOZ_RELEASE_ASSERT(
+ cmpIs32bit && selIs32bit,
+ "CodeGenerator::visitWasmCompareAndSelect: unexpected types");
+
+ Register trueExprAndDest = ToRegister(ins->output());
+ MOZ_ASSERT(ToRegister(ins->ifTrueExpr()) == trueExprAndDest,
+ "true expr input is reused for output");
+
+ Assembler::Condition cond = Assembler::InvertCondition(
+ JSOpToCondition(ins->compareType(), ins->jsop()));
+ const LAllocation* rhs = ins->rightExpr();
+ const LAllocation* falseExpr = ins->ifFalseExpr();
+ Register lhs = ToRegister(ins->leftExpr());
+
+ // This turns into an isel.
+ masm.cmp32Move32(cond, lhs, ToRegister(rhs), ToRegister(falseExpr),
+ trueExprAndDest);
+}
+
+void
+CodeGenerator::visitWasmReinterpret(LWasmReinterpret* lir)
+{
+ ADBlock();
+ MOZ_ASSERT(gen->compilingWasm());
+ MWasmReinterpret* ins = lir->mir();
+
+ MIRType to = ins->type();
+ DebugOnly<MIRType> from = ins->input()->type();
+
+ switch (to) {
+// XXX: use VSX float move instructions (moveToDouble, etc)
+ // We don't have much choice other than to spill to memory for these.
+ case MIRType::Int32:
+ MOZ_ASSERT(from == MIRType::Float32);
+ masm.as_stfsu(ToFloatRegister(lir->input()), StackPointer, -4);
+ // Keep in separate dispatch groups.
+ masm.nop();
+ masm.nop();
+ masm.nop();
+ masm.as_lwz(ToRegister(lir->output()), StackPointer, 0);
+ masm.as_addi(StackPointer, StackPointer, 4);
+ break;
+ case MIRType::Float32:
+ MOZ_ASSERT(from == MIRType::Int32);
+ masm.as_stwu(ToRegister(lir->input()), StackPointer, -4);
+ // Keep in separate dispatch groups.
+ masm.nop();
+ masm.nop();
+ masm.nop();
+ masm.as_lfs(ToFloatRegister(lir->output()), StackPointer, 0);
+ masm.as_addi(StackPointer, StackPointer, 4);
+ break;
+ case MIRType::Double:
+ case MIRType::Int64:
+ MOZ_CRASH("not handled by this LIR opcode");
+ default:
+ MOZ_CRASH("unexpected WasmReinterpret");
+ }
+}
+
+void
+CodeGenerator::visitUDivOrMod(LUDivOrMod* ins)
+{
+ ADBlock();
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register output = ToRegister(ins->output());
+ Label done;
+
+ // Although divwuo can flag overflow for divide by zero, we end up
+ // checking anyway to deal with the Infinity|0 situation, so we just don't
+ // bother and use regular (cheaper) divwu.
+ if (ins->canBeDivideByZero()) {
+ if (ins->mir()->isTruncated()) {
+ if (ins->trapOnError()) {
+ Label nonZero;
+ masm.ma_bc(rhs, rhs, &nonZero, Assembler::NonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, ins->bytecodeOffset());
+ masm.bind(&nonZero);
+ } else {
+ // Infinity|0 == 0
+ Label notzero;
+ masm.ma_bc(rhs, rhs, &notzero, Assembler::NonZero, ShortJump);
+ masm.move32(Imm32(0), output);
+ masm.ma_b(&done, ShortJump);
+ masm.bind(&notzero);
+ }
+ } else {
+ bailoutCmp32(Assembler::Equal, rhs, Imm32(0), ins->snapshot());
+ }
+ }
+
+ // If the remainder is > 0, bailout since this must be a double.
+ if (ins->mir()->isDiv()) {
+ // Compute quotient to output register; separately recover remainder.
+ masm.as_divwu(output, lhs, rhs);
+ masm.as_mullw(SecondScratchReg, output, rhs);
+ masm.as_subf(ScratchRegister, SecondScratchReg, lhs); // T = B - A
+ if (!ins->mir()->toDiv()->canTruncateRemainder())
+ bailoutCmp32(Assembler::NonZero, ScratchRegister, ScratchRegister, ins->snapshot());
+ } else {
+ // Compute remainder to output register.
+ masm.as_divwu(ScratchRegister, lhs, rhs);
+ masm.as_mullw(SecondScratchReg, ScratchRegister, rhs);
+ masm.as_subf(output, SecondScratchReg, lhs);
+ }
+
+ if (!ins->mir()->isTruncated())
+ bailoutCmp32(Assembler::LessThan, output, Imm32(0), ins->snapshot());
+
+ masm.bind(&done);
+}
+
+void
+CodeGenerator::visitEffectiveAddress(LEffectiveAddress* ins)
+{
+ ADBlock();
+ const MEffectiveAddress* mir = ins->mir();
+ Register base = ToRegister(ins->base());
+ Register index = ToRegister(ins->index());
+ Register output = ToRegister(ins->output());
+
+ BaseIndex address(base, index, mir->scale(), mir->displacement());
+ masm.computeEffectiveAddress(address, output);
+}
+
+void
+CodeGenerator::visitNegI(LNegI* ins)
+{
+ ADBlock();
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+
+ masm.as_neg(output, input);
+}
+
+void
+CodeGenerator::visitNegI64(LNegI64* ins)
+{
+ ADBlock();
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+
+ masm.as_neg(output, input);
+}
+
+void
+CodeGenerator::visitNegD(LNegD* ins)
+{
+ ADBlock();
+ FloatRegister input = ToFloatRegister(ins->input());
+ FloatRegister output = ToFloatRegister(ins->output());
+
+ masm.as_fneg(output, input);
+}
+
+void
+CodeGenerator::visitNegF(LNegF* ins)
+{
+ ADBlock();
+ FloatRegister input = ToFloatRegister(ins->input());
+ FloatRegister output = ToFloatRegister(ins->output());
+
+ masm.as_fneg(output, input);
+}
+
+void
+CodeGenerator::visitWasmAddOffset(LWasmAddOffset* lir)
+{
+ ADBlock();
+ MWasmAddOffset* mir = lir->mir();
+ Register base = ToRegister(lir->base());
+ Register out = ToRegister(lir->output());
+
+ Label ok;
+// XXX: add short parameter since usually short
+ masm.ma_addTestCarry(Assembler::CarryClear, out, base, Imm32(mir->offset()), &ok, /* is32 */ true);
+ masm.wasmTrap(wasm::Trap::OutOfBounds, mir->bytecodeOffset());
+ masm.bind(&ok);
+}
+
+void CodeGenerator::visitWasmAddOffset64(LWasmAddOffset64* lir) {
+ ADBlock();
+ MWasmAddOffset* mir = lir->mir();
+ Register64 base = ToRegister64(lir->base());
+ Register64 out = ToOutRegister64(lir);
+
+ Label ok;
+// XXX: add short parameter since usually short
+ masm.ma_addTestCarry(Assembler::CarryClear, out.reg, base.reg,
+ ImmWord(mir->offset()), &ok, /* is32 */ false);
+ masm.wasmTrap(wasm::Trap::OutOfBounds, mir->bytecodeOffset());
+ masm.bind(&ok);
+}
+
+void CodeGenerator::visitNearbyInt(LNearbyInt* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ FloatRegister output = ToFloatRegister(lir->output());
+
+ RoundingMode roundingMode = lir->mir()->roundingMode();
+ masm.nearbyIntDouble(roundingMode, input, output);
+}
+
+void CodeGenerator::visitNearbyIntF(LNearbyIntF* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ FloatRegister output = ToFloatRegister(lir->output());
+
+ RoundingMode roundingMode = lir->mir()->roundingMode();
+ masm.nearbyIntFloat32(roundingMode, input, output);
+}
+
+void CodeGenerator::visitWasmBuiltinTruncateDToInt32(
+ LWasmBuiltinTruncateDToInt32* lir) {
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitWasmBuiltinTruncateFToInt32(LWasmBuiltinTruncateFToInt32 *lir)
+{
+ MOZ_CRASH("NYI");
+}
+
+/*
+void
+CodeGenerator::visitWasmHeapBase(LWasmHeapBase *lir)
+{
+ MOZ_ASSERT(lir->instance()->isBogus());
+ masm.movePtr(HeapReg, ToRegister(lir->output()));
+}
+*/
+
+void
+CodeGenerator::visitAtomicTypedArrayElementBinop(LAtomicTypedArrayElementBinop* lir)
+{
+ ADBlock();
+ MOZ_ASSERT(lir->mir()->hasUses());
+
+ AnyRegister output = ToAnyRegister(lir->output());
+ Register elements = ToRegister(lir->elements());
+ Register outTemp = ToTempRegisterOrInvalid(lir->temp2());
+ Register valueTemp = ToTempRegisterOrInvalid(lir->valueTemp());
+ Register offsetTemp = ToTempRegisterOrInvalid(lir->offsetTemp());
+ Register maskTemp = ToTempRegisterOrInvalid(lir->maskTemp());
+ Register value = ToRegister(lir->value());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+ int width = Scalar::byteSize(arrayType);
+
+ if (lir->index()->isConstant()) {
+ Address mem(elements, ToInt32(lir->index()) * width);
+ masm.atomicFetchOpJS(arrayType, Synchronization::Full(), lir->mir()->operation(), value,
+ mem, valueTemp, offsetTemp, maskTemp, outTemp, output);
+ } else {
+ BaseIndex mem(elements, ToRegister(lir->index()), ScaleFromElemWidth(width));
+ masm.atomicFetchOpJS(arrayType, Synchronization::Full(), lir->mir()->operation(), value,
+ mem, valueTemp, offsetTemp, maskTemp, outTemp, output);
+ }
+}
+
+void
+CodeGenerator::visitAtomicTypedArrayElementBinopForEffect(LAtomicTypedArrayElementBinopForEffect* lir)
+{
+ ADBlock();
+ MOZ_ASSERT(!lir->mir()->hasUses());
+
+ Register elements = ToRegister(lir->elements());
+ Register valueTemp = ToTempRegisterOrInvalid(lir->valueTemp());
+ Register offsetTemp = ToTempRegisterOrInvalid(lir->offsetTemp());
+ Register maskTemp = ToTempRegisterOrInvalid(lir->maskTemp());
+ Register value = ToRegister(lir->value());
+ Scalar::Type arrayType = lir->mir()->arrayType();
+ int width = Scalar::byteSize(arrayType);
+
+ if (lir->index()->isConstant()) {
+ Address mem(elements, ToInt32(lir->index()) * width);
+ masm.atomicEffectOpJS(arrayType, Synchronization::Full(), lir->mir()->operation(), value,
+ mem, valueTemp, offsetTemp, maskTemp);
+ } else {
+ BaseIndex mem(elements, ToRegister(lir->index()), ScaleFromElemWidth(width));
+ masm.atomicEffectOpJS(arrayType, Synchronization::Full(), lir->mir()->operation(), value,
+ mem, valueTemp, offsetTemp, maskTemp);
+ }
+}
+
+void
+CodeGenerator::visitCompareExchangeTypedArrayElement(LCompareExchangeTypedArrayElement* lir)
+{
+ ADBlock();
+ Register elements = ToRegister(lir->elements());
+ AnyRegister output = ToAnyRegister(lir->output());
+ Register outTemp = ToTempRegisterOrInvalid(lir->temp());
+
+ Register oldval = ToRegister(lir->oldval());
+ Register newval = ToRegister(lir->newval());
+ Register valueTemp = ToTempRegisterOrInvalid(lir->valueTemp());
+ Register offsetTemp = ToTempRegisterOrInvalid(lir->offsetTemp());
+ Register maskTemp = ToTempRegisterOrInvalid(lir->maskTemp());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+ int width = Scalar::byteSize(arrayType);
+
+ if (lir->index()->isConstant()) {
+ Address dest(elements, ToInt32(lir->index()) * width);
+ masm.compareExchangeJS(arrayType, Synchronization::Full(), dest, oldval, newval,
+ valueTemp, offsetTemp, maskTemp, outTemp, output);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()), ScaleFromElemWidth(width));
+ masm.compareExchangeJS(arrayType, Synchronization::Full(), dest, oldval, newval,
+ valueTemp, offsetTemp, maskTemp, outTemp, output);
+ }
+}
+
+void
+CodeGenerator::visitAtomicExchangeTypedArrayElement(LAtomicExchangeTypedArrayElement* lir)
+{
+ ADBlock();
+ Register elements = ToRegister(lir->elements());
+ AnyRegister output = ToAnyRegister(lir->output());
+ Register outTemp = ToTempRegisterOrInvalid(lir->temp());
+
+ Register value = ToRegister(lir->value());
+ Register valueTemp = ToTempRegisterOrInvalid(lir->valueTemp());
+ Register offsetTemp = ToTempRegisterOrInvalid(lir->offsetTemp());
+ Register maskTemp = ToTempRegisterOrInvalid(lir->maskTemp());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+ int width = Scalar::byteSize(arrayType);
+
+ if (lir->index()->isConstant()) {
+ Address dest(elements, ToInt32(lir->index()) * width);
+ masm.atomicExchangeJS(arrayType, Synchronization::Full(), dest, value, valueTemp,
+ offsetTemp, maskTemp, outTemp, output);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()), ScaleFromElemWidth(width));
+ masm.atomicExchangeJS(arrayType, Synchronization::Full(), dest, value, valueTemp,
+ offsetTemp, maskTemp, outTemp, output);
+ }
+}
+
+void CodeGenerator::visitAtomicLoad64(LAtomicLoad64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register temp = ToRegister(lir->temp());
+ Register64 temp64 = ToRegister64(lir->temp64());
+ Register out = ToRegister(lir->output());
+
+ const MLoadUnboxedScalar* mir = lir->mir();
+
+ Scalar::Type storageType = mir->storageType();
+
+ auto sync = Synchronization::Load();
+
+ masm.memoryBarrierBefore(sync);
+ if (lir->index()->isConstant()) {
+ Address source =
+ ToAddress(elements, lir->index(), storageType, mir->offsetAdjustment());
+ masm.load64(source, temp64);
+ } else {
+ BaseIndex source(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(storageType), mir->offsetAdjustment());
+ masm.load64(source, temp64);
+ }
+ masm.memoryBarrierAfter(sync);
+
+ emitCreateBigInt(lir, storageType, temp64, out, temp);
+}
+
+void CodeGenerator::visitAtomicStore64(LAtomicStore64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+
+ Scalar::Type writeType = lir->mir()->writeType();
+
+ masm.loadBigInt64(value, temp1);
+
+ auto sync = Synchronization::Store();
+
+ masm.memoryBarrierBefore(sync);
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), writeType);
+ masm.store64(temp1, dest);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(writeType));
+ masm.store64(temp1, dest);
+ }
+ masm.memoryBarrierAfter(sync);
+}
+
+void CodeGenerator::visitCompareExchangeTypedArrayElement64(
+ LCompareExchangeTypedArrayElement64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register oldval = ToRegister(lir->oldval());
+ Register newval = ToRegister(lir->newval());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register64 temp2 = ToRegister64(lir->temp2());
+ Register out = ToRegister(lir->output());
+ Register64 tempOut(out);
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ masm.loadBigInt64(oldval, temp1);
+ masm.loadBigInt64(newval, tempOut);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.compareExchange64(Synchronization::Full(), dest, temp1, tempOut,
+ temp2);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.compareExchange64(Synchronization::Full(), dest, temp1, tempOut,
+ temp2);
+ }
+
+ emitCreateBigInt(lir, arrayType, temp2, out, temp1.scratchReg());
+}
+
+void CodeGenerator::visitAtomicExchangeTypedArrayElement64(
+ LAtomicExchangeTypedArrayElement64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register64 temp2 = Register64(ToRegister(lir->temp2()));
+ Register out = ToRegister(lir->output());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ masm.loadBigInt64(value, temp1);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicExchange64(Synchronization::Full(), dest, temp1, temp2);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicExchange64(Synchronization::Full(), dest, temp1, temp2);
+ }
+
+ emitCreateBigInt(lir, arrayType, temp2, out, temp1.scratchReg());
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinop64(
+ LAtomicTypedArrayElementBinop64* lir) {
+ MOZ_ASSERT(lir->mir()->hasUses());
+
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register64 temp2 = ToRegister64(lir->temp2());
+ Register out = ToRegister(lir->output());
+ Register64 tempOut = Register64(out);
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+ AtomicOp atomicOp = lir->mir()->operation();
+
+ masm.loadBigInt64(value, temp1);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicFetchOp64(Synchronization::Full(), atomicOp, temp1, dest,
+ tempOut, temp2);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicFetchOp64(Synchronization::Full(), atomicOp, temp1, dest,
+ tempOut, temp2);
+ }
+
+ emitCreateBigInt(lir, arrayType, temp2, out, temp1.scratchReg());
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinopForEffect64(
+ LAtomicTypedArrayElementBinopForEffect64* lir) {
+ MOZ_ASSERT(!lir->mir()->hasUses());
+
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register64 temp2 = ToRegister64(lir->temp2());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+ AtomicOp atomicOp = lir->mir()->operation();
+
+ masm.loadBigInt64(value, temp1);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicEffectOp64(Synchronization::Full(), atomicOp, temp1, dest,
+ temp2);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicEffectOp64(Synchronization::Full(), atomicOp, temp1, dest,
+ temp2);
+ }
+}
+
+void
+CodeGenerator::visitWasmCompareExchangeI64(LWasmCompareExchangeI64* lir)
+{
+ ADBlock();
+ Register ptr = ToRegister(lir->ptr());
+ Register64 oldValue = ToRegister64(lir->oldValue());
+ Register64 newValue = ToRegister64(lir->newValue());
+ Register64 output = ToOutRegister64(lir);
+ uint32_t offset = lir->mir()->access().offset();
+
+ BaseIndex addr(HeapReg, ptr, TimesOne, offset);
+ masm.wasmCompareExchange64(lir->mir()->access(), addr, oldValue, newValue,
+ output);
+}
+
+void
+CodeGenerator::visitWasmAtomicExchangeI64(LWasmAtomicExchangeI64* lir)
+{
+ ADBlock();
+ Register ptr = ToRegister(lir->ptr());
+ Register64 value = ToRegister64(lir->value());
+ Register64 output = ToOutRegister64(lir);
+ uint32_t offset = lir->mir()->access().offset();
+
+ BaseIndex addr(HeapReg, ptr, TimesOne, offset);
+ masm.wasmAtomicExchange64(lir->mir()->access(), addr, value, output);
+}
+
+void
+CodeGenerator::visitWasmAtomicBinopI64(LWasmAtomicBinopI64* lir)
+{
+ ADBlock();
+ Register ptr = ToRegister(lir->ptr());
+ Register64 value = ToRegister64(lir->value());
+ Register64 output = ToOutRegister64(lir);
+ Register64 temp(ToRegister(lir->getTemp(0)));
+ uint32_t offset = lir->mir()->access().offset();
+
+ BaseIndex addr(HeapReg, ptr, TimesOne, offset);
+
+ masm.wasmAtomicFetchOp64(lir->mir()->access(), lir->mir()->operation(),
+ value, addr, temp, output);
+}
+
+void CodeGenerator::visitSimd128(LSimd128* ins) { MOZ_CRASH("No SIMD"); }
+
+void CodeGenerator::visitWasmTernarySimd128(LWasmTernarySimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmBinarySimd128(LWasmBinarySimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmBinarySimd128WithConstant(
+ LWasmBinarySimd128WithConstant* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmVariableShiftSimd128(
+ LWasmVariableShiftSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmConstantShiftSimd128(
+ LWasmConstantShiftSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmSignReplicationSimd128(
+ LWasmSignReplicationSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmShuffleSimd128(LWasmShuffleSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmPermuteSimd128(LWasmPermuteSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmReplaceLaneSimd128(LWasmReplaceLaneSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmReplaceInt64LaneSimd128(
+ LWasmReplaceInt64LaneSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmScalarToSimd128(LWasmScalarToSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmInt64ToSimd128(LWasmInt64ToSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmUnarySimd128(LWasmUnarySimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmReduceSimd128(LWasmReduceSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmReduceAndBranchSimd128(
+ LWasmReduceAndBranchSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmReduceSimd128ToInt64(
+ LWasmReduceSimd128ToInt64* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmLoadLaneSimd128(LWasmLoadLaneSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmStoreLaneSimd128(LWasmStoreLaneSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+#if 0
+void
+CodeGenerator::visitSimdSplatX4(LSimdSplatX4* lir)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimd128Int(LSimd128Int* ins)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimd128Float(LSimd128Float* ins)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdExtractElementI(LSimdExtractElementI* ins)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdExtractElementF(LSimdExtractElementF* ins)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdBinaryCompIx4(LSimdBinaryCompIx4* lir)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdBinaryCompFx4(LSimdBinaryCompFx4* lir)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdBinaryArithIx4(LSimdBinaryArithIx4* lir)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdBinaryArithFx4(LSimdBinaryArithFx4* lir)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdBinaryBitwise(LSimdBinaryBitwise* lir)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitNearbyInt(LNearbyInt*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdShift(LSimdShift*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitNearbyIntF(LNearbyIntF*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdSelect(LSimdSelect*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdAllTrue(LSimdAllTrue*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdAnyTrue(LSimdAnyTrue*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdShuffle(LSimdShuffle*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdSplatX8(LSimdSplatX8*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdSplatX16(LSimdSplatX16*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdSwizzleF(LSimdSwizzleF*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdSwizzleI(LSimdSwizzleI*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdShuffleX4(LSimdShuffleX4*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdBinaryCompIx8(LSimdBinaryCompIx8*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdUnaryArithFx4(LSimdUnaryArithFx4*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdUnaryArithIx4(LSimdUnaryArithIx4*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdUnaryArithIx8(LSimdUnaryArithIx8*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitFloat32x4ToInt32x4(LFloat32x4ToInt32x4*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitInt32x4ToFloat32x4(LInt32x4ToFloat32x4*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdBinaryArithIx8(LSimdBinaryArithIx8*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdBinaryCompIx16(LSimdBinaryCompIx16*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdInsertElementF(LSimdInsertElementF*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdInsertElementI(LSimdInsertElementI*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdUnaryArithIx16(LSimdUnaryArithIx16*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitFloat32x4ToUint32x4(LFloat32x4ToUint32x4*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdBinaryArithIx16(LSimdBinaryArithIx16*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdExtractElementB(LSimdExtractElementB*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdGeneralShuffleF(LSimdGeneralShuffleF*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdGeneralShuffleI(LSimdGeneralShuffleI*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdReinterpretCast(LSimdReinterpretCast*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdBinarySaturating(LSimdBinarySaturating*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+CodeGenerator::visitSimdExtractElementU2D(LSimdExtractElementU2D*)
+{
+ MOZ_CRASH("NYI");
+}
+#endif
diff -r 485b15bb4a20 -r 9c245f4665be js/src/jit/ppc64/CodeGenerator-ppc64.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/js/src/jit/ppc64/CodeGenerator-ppc64.h Thu Aug 08 21:23:52 2024 -0700
@@ -0,0 +1,191 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_PPC64_CodeGenerator_PPC64_h
+#define jit_PPC64_CodeGenerator_PPC64_h
+
+#include "jit/ppc64/Assembler-ppc64.h"
+#include "jit/shared/CodeGenerator-shared.h"
+
+namespace js {
+namespace jit {
+
+class OutOfLineBailout;
+class OutOfLineTableSwitch;
+class CodeGeneratorPPC64;
+
+using OutOfLineWasmTruncateCheck = OutOfLineWasmTruncateCheckBase<CodeGeneratorPPC64>;
+
+class CodeGeneratorPPC64 : public CodeGeneratorShared
+{
+ friend class MoveResolverPPC64;
+
+ protected:
+ CodeGeneratorPPC64(MIRGenerator* gen, LIRGraph* graph, MacroAssembler* masm);
+
+ NonAssertingLabel deoptLabel_;
+
+ Operand ToOperand(const LAllocation& a);
+ Operand ToOperand(const LAllocation* a);
+ Operand ToOperand(const LDefinition* def);
+
+ Operand ToOperandOrRegister64(const LInt64Allocation input);
+
+ MoveOperand toMoveOperand(LAllocation a) const;
+
+ template <typename T1, typename T2>
+ void bailoutCmp32(Assembler::Condition c, T1 lhs, T2 rhs, LSnapshot* snapshot) {
+ Label bail;
+ masm.branch32(c, lhs, rhs, &bail);
+ bailoutFrom(&bail, snapshot);
+ }
+ template<typename T>
+ void bailoutTest32(Assembler::Condition c, Register lhs, T rhs, LSnapshot* snapshot) {
+ Label bail;
+ masm.branchTest32(c, lhs, rhs, &bail);
+ bailoutFrom(&bail, snapshot);
+ }
+ template <typename T1, typename T2>
+ void bailoutCmpPtr(Assembler::Condition c, T1 lhs, T2 rhs, LSnapshot* snapshot) {
+ Label bail;
+ masm.branchPtr(c, lhs, rhs, &bail);
+ bailoutFrom(&bail, snapshot);
+ }
+ void bailoutTestPtr(Assembler::Condition c, Register lhs, Register rhs, LSnapshot* snapshot) {
+ Label bail;
+ masm.branchTestPtr(c, lhs, rhs, &bail);
+ bailoutFrom(&bail, snapshot);
+ }
+ void bailoutIfFalseBool(Register reg, LSnapshot* snapshot) {
+ Label bail;
+ masm.branchTest32(Assembler::Zero, reg, Imm32(0xFF), &bail);
+ bailoutFrom(&bail, snapshot);
+ }
+
+ void bailoutFrom(Label* label, LSnapshot* snapshot);
+ void bailout(LSnapshot* snapshot);
+
+ bool generateOutOfLineCode();
+
+ template <typename T>
+ void branchToBlock(Register lhs, T rhs, MBasicBlock* mir, Assembler::Condition cond)
+ {
+ masm.ma_bc(lhs, rhs, skipTrivialBlocks(mir)->lir()->label(), cond);
+ }
+ void branchToBlock(Assembler::FloatFormat fmt, FloatRegister lhs, FloatRegister rhs,
+ MBasicBlock* mir, Assembler::DoubleCondition cond);
+
+ // Emits a branch that directs control flow to the true block if |cond| is
+ // true, and the false block if |cond| is false.
+ template <typename T>
+ void emitBranch(Register lhs, T rhs, Assembler::Condition cond,
+ MBasicBlock* mirTrue, MBasicBlock* mirFalse)
+ {
+ if (isNextBlock(mirFalse->lir())) {
+ branchToBlock(lhs, rhs, mirTrue, cond);
+ } else {
+ branchToBlock(lhs, rhs, mirFalse, Assembler::InvertCondition(cond));
+ jumpToBlock(mirTrue);
+ }
+ }
+ void testZeroEmitBranch(Assembler::Condition cond, Register reg,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse)
+ {
+ emitBranch(reg, Imm32(0), cond, ifTrue, ifFalse);
+ }
+
+ void emitBigIntDiv(LBigIntDiv*, Register, Register, Register, Label*);
+ void emitBigIntMod(LBigIntMod*, Register, Register, Register, Label*);
+
+ void emitTableSwitchDispatch(MTableSwitch* mir, Register index, Register base);
+
+ template <typename T>
+ void emitWasmLoad(T* ins);
+ template <typename T>
+ void emitWasmStore(T* ins);
+
+ void generateInvalidateEpilogue();
+
+ // Generating a result.
+ template<typename S, typename T>
+ void atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, const S& value,
+ const T& mem, Register flagTemp, Register outTemp,
+ Register valueTemp, Register offsetTemp, Register maskTemp,
+ AnyRegister output);
+
+ // Generating no result.
+ template<typename S, typename T>
+ void atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, const S& value,
+ const T& mem, Register flagTemp, Register valueTemp,
+ Register offsetTemp, Register maskTemp);
+
+ public:
+ // Out of line visitors.
+ void visitOutOfLineBailout(OutOfLineBailout* ool);
+ void visitOutOfLineTableSwitch(OutOfLineTableSwitch* ool);
+ void visitOutOfLineWasmTruncateCheck(OutOfLineWasmTruncateCheck* ool);
+
+ protected:
+ void testNullEmitBranch(Assembler::Condition cond, const ValueOperand& value,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse)
+ {
+ MOZ_ASSERT(value.valueReg() != SecondScratchReg);
+ masm.splitTag(value.valueReg(), SecondScratchReg);
+ emitBranch(SecondScratchReg, ImmTag(JSVAL_TAG_NULL), cond, ifTrue, ifFalse);
+ }
+ void testUndefinedEmitBranch(Assembler::Condition cond, const ValueOperand& value,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse)
+ {
+ MOZ_ASSERT(value.valueReg() != SecondScratchReg);
+ masm.splitTag(value.valueReg(), SecondScratchReg);
+ emitBranch(SecondScratchReg, ImmTag(JSVAL_TAG_UNDEFINED), cond, ifTrue, ifFalse);
+ }
+ void testObjectEmitBranch(Assembler::Condition cond, const ValueOperand& value,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse)
+ {
+ MOZ_ASSERT(value.valueReg() != SecondScratchReg);
+ masm.splitTag(value.valueReg(), SecondScratchReg);
+ emitBranch(SecondScratchReg, ImmTag(JSVAL_TAG_OBJECT), cond, ifTrue, ifFalse);
+ }
+
+
+ template <typename T>
+ void emitWasmLoadI64(T* ins);
+ template <typename T>
+ void emitWasmStoreI64(T* ins);
+
+ ValueOperand ToValue(LInstruction* ins, size_t pos);
+ ValueOperand ToTempValue(LInstruction* ins, size_t pos);
+
+ // Functions for LTestVAndBranch.
+ void splitTagForTest(const ValueOperand& value, ScratchTagScope& tag);
+};
+
+typedef CodeGeneratorPPC64 CodeGeneratorSpecific;
+
+// An out-of-line bailout thunk.
+class OutOfLineBailout : public OutOfLineCodeBase<CodeGeneratorPPC64>
+{
+ LSnapshot* snapshot_;
+ uint32_t frameSize_;
+
+ public:
+ OutOfLineBailout(LSnapshot* snapshot, uint32_t frameSize)
+ : snapshot_(snapshot),
+ frameSize_(frameSize)
+ { }
+
+ void accept(CodeGeneratorPPC64* codegen) override;
+
+ LSnapshot* snapshot() const {
+ return snapshot_;
+ }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_PPC64_CodeGenerator_PPC64_h */
diff -r 485b15bb4a20 -r 9c245f4665be js/src/jit/ppc64/LIR-ppc64.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/js/src/jit/ppc64/LIR-ppc64.h Thu Aug 08 21:23:52 2024 -0700
@@ -0,0 +1,611 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_ppc64le_LIR_ppc64le_h
+#define jit_ppc64le_LIR_ppc64le_h
+
+namespace js {
+namespace jit {
+
+class LUnbox : public LInstructionHelper<1, 1, 0>
+{
+ protected:
+ LUnbox(LNode::Opcode opcode, const LAllocation& input)
+ : LInstructionHelper(opcode)
+ {
+ setOperand(0, input);
+ }
+
+ public:
+ LIR_HEADER(Unbox);
+
+ explicit LUnbox(const LAllocation& input)
+ : LInstructionHelper(classOpcode)
+ {
+ setOperand(0, input);
+ }
+
+ static const size_t Input = 0;
+
+ MUnbox* mir() const {
+ return mir_->toUnbox();
+ }
+ const char* extraName() const {
+ return StringFromMIRType(mir()->type());
+ }
+};
+
+class LUnboxFloatingPoint : public LUnbox
+{
+ MIRType type_;
+
+ public:
+ LIR_HEADER(UnboxFloatingPoint);
+
+ LUnboxFloatingPoint(const LAllocation& input, MIRType type)
+ : LUnbox(classOpcode, input),
+ type_(type)
+ { }
+
+ MIRType type() const {
+ return type_;
+ }
+};
+
+class LDivOrModI64 : public LBinaryMath<1>
+{
+ public:
+ LIR_HEADER(DivOrModI64)
+
+ LDivOrModI64(const LAllocation& lhs, const LAllocation& rhs, const LDefinition& temp)
+ : LBinaryMath(classOpcode)
+ {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp);
+ }
+
+ const LDefinition* remainder() {
+ return getTemp(0);
+ }
+ MBinaryArithInstruction* mir() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ return static_cast<MBinaryArithInstruction*>(mir_);
+ }
+
+ bool canBeDivideByZero() const {
+ if (mir_->isMod())
+ return mir_->toMod()->canBeDivideByZero();
+ return mir_->toDiv()->canBeDivideByZero();
+ }
+ bool canBeNegativeOverflow() const {
+ if (mir_->isMod())
+ return mir_->toMod()->canBeNegativeDividend();
+ return mir_->toDiv()->canBeNegativeOverflow();
+ }
+ wasm::BytecodeOffset bytecodeOffset() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ if (mir_->isMod())
+ return mir_->toMod()->bytecodeOffset();
+ return mir_->toDiv()->bytecodeOffset();
+ }
+};
+
+class LUDivOrModI64 : public LBinaryMath<1>
+{
+ public:
+ LIR_HEADER(UDivOrModI64);
+
+ LUDivOrModI64(const LAllocation& lhs, const LAllocation& rhs, const LDefinition& temp)
+ : LBinaryMath(classOpcode)
+ {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp);
+ }
+
+ const LDefinition* remainder() {
+ return getTemp(0);
+ }
+ const char* extraName() const {
+ return mir()->isTruncated() ? "Truncated" : nullptr;
+ }
+
+ MBinaryArithInstruction* mir() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ return static_cast<MBinaryArithInstruction*>(mir_);
+ }
+ bool canBeDivideByZero() const {
+ if (mir_->isMod())
+ return mir_->toMod()->canBeDivideByZero();
+ return mir_->toDiv()->canBeDivideByZero();
+ }
+ wasm::BytecodeOffset bytecodeOffset() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ if (mir_->isMod())
+ return mir_->toMod()->bytecodeOffset();
+ return mir_->toDiv()->bytecodeOffset();
+ }
+};
+
+class LWasmTruncateToInt64 : public LInstructionHelper<1, 1, 0>
+{
+ public:
+ LIR_HEADER(WasmTruncateToInt64);
+
+ explicit LWasmTruncateToInt64(const LAllocation& in)
+ : LInstructionHelper(classOpcode)
+ {
+ setOperand(0, in);
+ }
+
+ MWasmTruncateToInt64* mir() const {
+ return mir_->toWasmTruncateToInt64();
+ }
+};
+
+class LInt64ToFloatingPoint : public LInstructionHelper<1, 1, 0>
+{
+ public:
+ LIR_HEADER(Int64ToFloatingPoint);
+
+ explicit LInt64ToFloatingPoint(const LInt64Allocation& in)
+ : LInstructionHelper(classOpcode)
+ {
+ setInt64Operand(0, in);
+ }
+
+ MInt64ToFloatingPoint* mir() const {
+ return mir_->toInt64ToFloatingPoint();
+ }
+};
+
+// Convert a 32-bit unsigned integer to a double.
+class LWasmUint32ToDouble : public LInstructionHelper<1, 1, 0>
+{
+ public:
+ LIR_HEADER(WasmUint32ToDouble)
+
+ LWasmUint32ToDouble(const LAllocation& input)
+ : LInstructionHelper(classOpcode)
+ {
+ setOperand(0, input);
+ }
+};
+
+// Convert a 32-bit unsigned integer to a float32.
+class LWasmUint32ToFloat32 : public LInstructionHelper<1, 1, 0>
+{
+ public:
+ LIR_HEADER(WasmUint32ToFloat32)
+
+ LWasmUint32ToFloat32(const LAllocation& input)
+ : LInstructionHelper(classOpcode)
+ {
+ setOperand(0, input);
+ }
+};
+
+
+class LDivI : public LBinaryMath<1>
+{
+ public:
+ LIR_HEADER(DivI);
+
+ LDivI(const LAllocation& lhs, const LAllocation& rhs, const LDefinition& temp)
+ : LBinaryMath(classOpcode)
+ {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp);
+ }
+
+ MDiv* mir() const {
+ return mir_->toDiv();
+ }
+};
+
+class LDivPowTwoI : public LInstructionHelper<1, 1, 1>
+{
+ const int32_t shift_;
+
+ public:
+ LIR_HEADER(DivPowTwoI)
+
+ LDivPowTwoI(const LAllocation& lhs, int32_t shift, const LDefinition& temp)
+ : LInstructionHelper(classOpcode),
+ shift_(shift)
+ {
+ setOperand(0, lhs);
+ setTemp(0, temp);
+ }
+
+ const LAllocation* numerator() {
+ return getOperand(0);
+ }
+ int32_t shift() const {
+ return shift_;
+ }
+ MDiv* mir() const {
+ return mir_->toDiv();
+ }
+};
+
+class LModI : public LBinaryMath<1>
+{
+ public:
+ LIR_HEADER(ModI);
+
+ LModI(const LAllocation& lhs, const LAllocation& rhs, const LDefinition& callTemp)
+ : LBinaryMath(classOpcode)
+ {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, callTemp);
+ }
+
+ const LDefinition* callTemp() {
+ return getTemp(0);
+ }
+ MMod* mir() const {
+ return mir_->toMod();
+ }
+};
+
+class LModPowTwoI : public LInstructionHelper<1, 1, 0>
+{
+ const int32_t shift_;
+
+ public:
+ LIR_HEADER(ModPowTwoI);
+
+ LModPowTwoI(const LAllocation& lhs, int32_t shift)
+ : LInstructionHelper(classOpcode),
+ shift_(shift)
+ {
+ setOperand(0, lhs);
+ }
+
+ int32_t shift() const {
+ return shift_;
+ }
+ MMod* mir() const {
+ return mir_->toMod();
+ }
+};
+
+class LModMaskI : public LInstructionHelper<1, 1, 2>
+{
+ const int32_t shift_;
+
+ public:
+ LIR_HEADER(ModMaskI);
+
+ LModMaskI(const LAllocation& lhs, const LDefinition& temp0, const LDefinition& temp1,
+ int32_t shift)
+ : LInstructionHelper(classOpcode),
+ shift_(shift)
+ {
+ setOperand(0, lhs);
+ setTemp(0, temp0);
+ setTemp(1, temp1);
+ }
+
+ int32_t shift() const {
+ return shift_;
+ }
+ MMod* mir() const {
+ return mir_->toMod();
+ }
+};
+
+// Takes a tableswitch with an integer to decide
+class LTableSwitch : public LInstructionHelper<0, 1, 2>
+{
+ public:
+ LIR_HEADER(TableSwitch);
+
+ LTableSwitch(const LAllocation& in, const LDefinition& inputCopy,
+ const LDefinition& jumpTablePointer, MTableSwitch* ins)
+ : LInstructionHelper(classOpcode)
+ {
+ setOperand(0, in);
+ setTemp(0, inputCopy);
+ setTemp(1, jumpTablePointer);
+ setMir(ins);
+ }
+
+ MTableSwitch* mir() const {
+ return mir_->toTableSwitch();
+ }
+ const LAllocation* index() {
+ return getOperand(0);
+ }
+ const LDefinition* tempInt() {
+ return getTemp(0);
+ }
+ // This is added to share the same CodeGenerator prefixes.
+ const LDefinition* tempPointer() {
+ return getTemp(1);
+ }
+};
+
+// Takes a tableswitch with an integer to decide
+class LTableSwitchV : public LInstructionHelper<0, BOX_PIECES, 3>
+{
+ public:
+ LIR_HEADER(TableSwitchV);
+
+ LTableSwitchV(const LBoxAllocation& input, const LDefinition& inputCopy,
+ const LDefinition& floatCopy, const LDefinition& jumpTablePointer,
+ MTableSwitch* ins)
+ : LInstructionHelper(classOpcode)
+ {
+ setBoxOperand(InputValue, input);
+ setTemp(0, inputCopy);
+ setTemp(1, floatCopy);
+ setTemp(2, jumpTablePointer);
+ setMir(ins);
+ }
+
+ MTableSwitch* mir() const {
+ return mir_->toTableSwitch();
+ }
+
+ static const size_t InputValue = 0;
+
+ const LDefinition* tempInt() {
+ return getTemp(0);
+ }
+ const LDefinition* tempFloat() {
+ return getTemp(1);
+ }
+ const LDefinition* tempPointer() {
+ return getTemp(2);
+ }
+};
+
+class LMulI : public LBinaryMath<0>
+{
+ public:
+ LIR_HEADER(MulI);
+
+ LMulI()
+ : LBinaryMath(classOpcode)
+ {}
+
+ MMul* mir() {
+ return mir_->toMul();
+ }
+};
+
+class LUDivOrMod : public LBinaryMath<0>
+{
+ public:
+ LIR_HEADER(UDivOrMod);
+
+ LUDivOrMod()
+ : LBinaryMath(classOpcode)
+ {}
+
+ MBinaryArithInstruction* mir() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ return static_cast<MBinaryArithInstruction*>(mir_);
+ }
+
+ bool canBeDivideByZero() const {
+ if (mir_->isMod())
+ return mir_->toMod()->canBeDivideByZero();
+ return mir_->toDiv()->canBeDivideByZero();
+ }
+
+ bool trapOnError() const {
+ if (mir_->isMod())
+ return mir_->toMod()->trapOnError();
+ return mir_->toDiv()->trapOnError();
+ }
+
+ wasm::BytecodeOffset bytecodeOffset() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ if (mir_->isMod())
+ return mir_->toMod()->bytecodeOffset();
+ return mir_->toDiv()->bytecodeOffset();
+ }
+};
+
+namespace details {
+
+// Base class for the int64 and non-int64 variants.
+template<size_t NumDefs>
+class LWasmUnalignedLoadBase : public details::LWasmLoadBase<NumDefs, 2>
+{
+ public:
+ typedef LWasmLoadBase<NumDefs, 2> Base;
+
+ explicit LWasmUnalignedLoadBase(LNode::Opcode opcode, const LAllocation& ptr,
+ const LDefinition& valueHelper)
+ : Base(opcode, ptr, LAllocation())
+ {
+ Base::setTemp(0, LDefinition::BogusTemp());
+ Base::setTemp(1, valueHelper);
+ }
+
+ const LAllocation* ptr() {
+ return Base::getOperand(0);
+ }
+ const LDefinition* ptrCopy() {
+ return Base::getTemp(0);
+ }
+};
+
+} // namespace details
+
+class LWasmUnalignedLoad : public details::LWasmUnalignedLoadBase<1>
+{
+ public:
+ LIR_HEADER(WasmUnalignedLoad);
+
+ explicit LWasmUnalignedLoad(const LAllocation& ptr, const LDefinition& valueHelper)
+ : LWasmUnalignedLoadBase(classOpcode, ptr, valueHelper)
+ {}
+};
+
+class LWasmUnalignedLoadI64 : public details::LWasmUnalignedLoadBase<INT64_PIECES>
+{
+ public:
+ LIR_HEADER(WasmUnalignedLoadI64);
+
+ explicit LWasmUnalignedLoadI64(const LAllocation& ptr, const LDefinition& valueHelper)
+ : LWasmUnalignedLoadBase(classOpcode, ptr, valueHelper)
+ {}
+};
+
+namespace details {
+
+// Base class for the int64 and non-int64 variants.
+template<size_t NumOps>
+class LWasmUnalignedStoreBase : public LInstructionHelper<0, NumOps, 2>
+{
+ public:
+ typedef LInstructionHelper<0, NumOps, 2> Base;
+
+ static const size_t PtrIndex = 0;
+ static const size_t ValueIndex = 1;
+
+ LWasmUnalignedStoreBase(LNode::Opcode opcode, const LAllocation& ptr,
+ const LDefinition& valueHelper)
+ : Base(opcode)
+ {
+ Base::setOperand(0, ptr);
+ Base::setTemp(0, LDefinition::BogusTemp());
+ Base::setTemp(1, valueHelper);
+ }
+
+ MWasmStore* mir() const {
+ return Base::mir_->toWasmStore();
+ }
+ const LAllocation* ptr() {
+ return Base::getOperand(PtrIndex);
+ }
+ const LDefinition* ptrCopy() {
+ return Base::getTemp(0);
+ }
+};
+
+} // namespace details
+
+class LWasmUnalignedStore : public details::LWasmUnalignedStoreBase<2>
+{
+ public:
+ LIR_HEADER(WasmUnalignedStore);
+
+ LWasmUnalignedStore(const LAllocation& ptr, const LAllocation& value,
+ const LDefinition& valueHelper)
+ : LWasmUnalignedStoreBase(classOpcode, ptr, valueHelper)
+ {
+ setOperand(1, value);
+ }
+
+ const LAllocation* value() {
+ return Base::getOperand(ValueIndex);
+ }
+};
+
+class LWasmUnalignedStoreI64 : public details::LWasmUnalignedStoreBase<1 + INT64_PIECES>
+{
+ public:
+ LIR_HEADER(WasmUnalignedStoreI64);
+ LWasmUnalignedStoreI64(const LAllocation& ptr, const LInt64Allocation& value,
+ const LDefinition& valueHelper)
+ : LWasmUnalignedStoreBase(classOpcode, ptr, valueHelper)
+ {
+ setInt64Operand(1, value);
+ }
+
+ const LInt64Allocation value() {
+ return getInt64Operand(ValueIndex);
+ }
+};
+
+class LWasmCompareExchangeI64 : public LInstructionHelper<INT64_PIECES, 1 + INT64_PIECES + INT64_PIECES, 0>
+{
+ public:
+ LIR_HEADER(WasmCompareExchangeI64);
+
+ LWasmCompareExchangeI64(const LAllocation& ptr, const LInt64Allocation& oldValue,
+ const LInt64Allocation& newValue)
+ : LInstructionHelper(classOpcode)
+ {
+ setOperand(0, ptr);
+ setInt64Operand(1, oldValue);
+ setInt64Operand(1 + INT64_PIECES, newValue);
+ }
+
+ const LAllocation* ptr() {
+ return getOperand(0);
+ }
+ const LInt64Allocation oldValue() {
+ return getInt64Operand(1);
+ }
+ const LInt64Allocation newValue() {
+ return getInt64Operand(1 + INT64_PIECES);
+ }
+ const MWasmCompareExchangeHeap* mir() const {
+ return mir_->toWasmCompareExchangeHeap();
+ }
+};
+
+class LWasmAtomicExchangeI64 : public LInstructionHelper<INT64_PIECES, 1 + INT64_PIECES, 0>
+{
+ public:
+ LIR_HEADER(WasmAtomicExchangeI64);
+
+ LWasmAtomicExchangeI64(const LAllocation& ptr, const LInt64Allocation& value)
+ : LInstructionHelper(classOpcode)
+ {
+ setOperand(0, ptr);
+ setInt64Operand(1, value);
+ }
+
+ const LAllocation* ptr() {
+ return getOperand(0);
+ }
+ const LInt64Allocation value() {
+ return getInt64Operand(1);
+ }
+ const MWasmAtomicExchangeHeap* mir() const {
+ return mir_->toWasmAtomicExchangeHeap();
+ }
+};
+
+class LWasmAtomicBinopI64 : public LInstructionHelper<INT64_PIECES, 1 + INT64_PIECES, 2>
+{
+ public:
+ LIR_HEADER(WasmAtomicBinopI64);
+
+ LWasmAtomicBinopI64(const LAllocation& ptr, const LInt64Allocation& value)
+ : LInstructionHelper(classOpcode)
+ {
+ setOperand(0, ptr);
+ setInt64Operand(1, value);
+ }
+
+ const LAllocation* ptr() {
+ return getOperand(0);
+ }
+ const LInt64Allocation value() {
+ return getInt64Operand(1);
+ }
+ const MWasmAtomicBinopHeap* mir() const {
+ return mir_->toWasmAtomicBinopHeap();
+ }
+};
+
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_ppc64le_LIR_ppc64le_h */
diff -r 485b15bb4a20 -r 9c245f4665be js/src/jit/ppc64/Lowering-ppc64.cpp
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/js/src/jit/ppc64/Lowering-ppc64.cpp Thu Aug 08 21:23:52 2024 -0700
@@ -0,0 +1,1289 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/MathAlgorithms.h"
+
+#include "jit/Lowering.h"
+#include "jit/MIR.h"
+#include "jit/ppc64/Assembler-ppc64.h"
+
+#include "jit/shared/Lowering-shared-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::FloorLog2;
+
+LAllocation
+LIRGeneratorPPC64::useByteOpRegister(MDefinition* mir)
+{
+ return useRegister(mir);
+}
+
+LAllocation
+LIRGeneratorPPC64::useByteOpRegisterAtStart(MDefinition* mir)
+{
+ return useRegisterAtStart(mir);
+}
+
+LAllocation
+LIRGeneratorPPC64::useByteOpRegisterOrNonDoubleConstant(MDefinition* mir)
+{
+ return useRegisterOrNonDoubleConstant(mir);
+}
+
+LDefinition
+LIRGeneratorPPC64::tempByteOpRegister()
+{
+ return temp();
+}
+
+// x = !y
+void
+LIRGeneratorPPC64::lowerForALU(LInstructionHelper<1, 1, 0>* ins,
+ MDefinition* mir, MDefinition* input)
+{
+ ins->setOperand(0, useRegister(input));
+ define(ins, mir, LDefinition(LDefinition::TypeFrom(mir->type()), LDefinition::REGISTER));
+}
+
+// z = x+y
+void
+LIRGeneratorPPC64::lowerForALU(LInstructionHelper<1, 2, 0>* ins, MDefinition* mir,
+ MDefinition* lhs, MDefinition* rhs)
+{
+ ins->setOperand(0, useRegister(lhs));
+ ins->setOperand(1, useRegisterOrConstant(rhs));
+ define(ins, mir, LDefinition(LDefinition::TypeFrom(mir->type()), LDefinition::REGISTER));
+}
+
+void LIRGeneratorPPC64::lowerForALUInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES, 0>* ins, MDefinition* mir,
+ MDefinition* input) {
+ ins->setInt64Operand(0, useInt64RegisterAtStart(input));
+ defineInt64ReuseInput(ins, mir, 0);
+}
+
+void LIRGeneratorPPC64::lowerForALUInt64(
+ LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs) {
+ ins->setInt64Operand(0, useInt64RegisterAtStart(lhs));
+ ins->setInt64Operand(INT64_PIECES, willHaveDifferentLIRNodes(lhs, rhs)
+ ? useInt64OrConstant(rhs)
+ : useInt64OrConstantAtStart(rhs));
+ defineInt64ReuseInput(ins, mir, 0);
+}
+
+void
+LIRGeneratorPPC64::lowerForMulInt64(LMulI64* ins, MMul* mir, MDefinition* lhs, MDefinition* rhs)
+{
+// XXX: simplify this because these are all static conditions
+ bool needsTemp = false;
+ bool cannotAliasRhs = false;
+ bool reuseInput = true;
+
+/*
+ ins->setInt64Operand(0, useInt64RegisterAtStart(lhs));
+ ins->setInt64Operand(INT64_PIECES,
+ (lhs != rhs || cannotAliasRhs) ? useInt64OrConstant(rhs) : useInt64OrConstantAtStart(rhs));
+*/
+ ins->setInt64Operand(0, useInt64RegisterAtStart(lhs));
+ ins->setInt64Operand(INT64_PIECES,
+ (willHaveDifferentLIRNodes(lhs, rhs) || cannotAliasRhs)
+ ? useInt64OrConstant(rhs)
+ : useInt64OrConstantAtStart(rhs));
+
+ if (needsTemp)
+ ins->setTemp(0, temp());
+ if(reuseInput)
+ defineInt64ReuseInput(ins, mir, 0);
+ else
+ defineInt64(ins, mir);
+}
+
+template<size_t Temps>
+void
+LIRGeneratorPPC64::lowerForShiftInt64(LInstructionHelper<INT64_PIECES, INT64_PIECES + 1, Temps>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs)
+{
+ ins->setInt64Operand(0, useInt64RegisterAtStart(lhs));
+
+ static_assert(LShiftI64::Rhs == INT64_PIECES, "Assume Rhs is located at INT64_PIECES.");
+ static_assert(LRotateI64::Count == INT64_PIECES, "Assume Count is located at INT64_PIECES.");
+
+ ins->setOperand(INT64_PIECES, useRegisterOrConstant(rhs));
+
+ defineInt64ReuseInput(ins, mir, 0);
+}
+
+template void LIRGeneratorPPC64::lowerForShiftInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES+1, 0>* ins, MDefinition* mir,
+ MDefinition* lhs, MDefinition* rhs);
+template void LIRGeneratorPPC64::lowerForShiftInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES+1, 1>* ins, MDefinition* mir,
+ MDefinition* lhs, MDefinition* rhs);
+
+void LIRGeneratorPPC64::lowerForCompareI64AndBranch(MTest* mir, MCompare* comp,
+ JSOp op, MDefinition* left,
+ MDefinition* right,
+ MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse) {
+ auto* lir = new (alloc())
+ LCompareI64AndBranch(comp, op, useInt64Register(left),
+ useInt64RegisterOrConstant(right), ifTrue, ifFalse);
+ add(lir, mir);
+}
+
+void
+LIRGeneratorPPC64::lowerForFPU(LInstructionHelper<1, 1, 0>* ins, MDefinition* mir,
+ MDefinition* input)
+{
+ ins->setOperand(0, useRegister(input));
+ define(ins, mir, LDefinition(LDefinition::TypeFrom(mir->type()), LDefinition::REGISTER));
+}
+
+template<size_t Temps>
+void
+LIRGeneratorPPC64::lowerForFPU(LInstructionHelper<1, 2, Temps>* ins, MDefinition* mir,
+ MDefinition* lhs, MDefinition* rhs)
+{
+ ins->setOperand(0, useRegister(lhs));
+ ins->setOperand(1, useRegister(rhs));
+ define(ins, mir, LDefinition(LDefinition::TypeFrom(mir->type()), LDefinition::REGISTER));
+}
+
+template void LIRGeneratorPPC64::lowerForFPU(LInstructionHelper<1, 2, 0>* ins, MDefinition* mir,
+ MDefinition* lhs, MDefinition* rhs);
+template void LIRGeneratorPPC64::lowerForFPU(LInstructionHelper<1, 2, 1>* ins, MDefinition* mir,
+ MDefinition* lhs, MDefinition* rhs);
+
+void
+LIRGeneratorPPC64::lowerForBitAndAndBranch(LBitAndAndBranch* baab, MInstruction* mir,
+ MDefinition* lhs, MDefinition* rhs)
+{
+ baab->setOperand(0, useRegisterAtStart(lhs));
+ baab->setOperand(1, useRegisterOrConstantAtStart(rhs));
+ add(baab, mir);
+}
+
+void LIRGeneratorPPC64::lowerWasmBuiltinTruncateToInt32(
+ MWasmBuiltinTruncateToInt32* ins) {
+/*
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Double || opd->type() == MIRType::Float32);
+
+ if (opd->type() == MIRType::Double) {
+ define(new (alloc()) LWasmBuiltinTruncateDToInt32(
+ useRegister(opd), useFixed(ins->tls(), WasmTlsReg),
+ LDefinition::BogusTemp()),
+ ins);
+ return;
+ }
+
+ define(new (alloc()) LWasmBuiltinTruncateFToInt32(
+ useRegister(opd), useFixed(ins->tls(), WasmTlsReg),
+ LDefinition::BogusTemp()),
+ ins);
+*/
+ MOZ_CRASH("NYI");
+}
+
+void
+LIRGeneratorPPC64::lowerForShift(LInstructionHelper<1, 2, 0>* ins, MDefinition* mir,
+ MDefinition* lhs, MDefinition* rhs)
+{
+ ins->setOperand(0, useRegister(lhs));
+ ins->setOperand(1, useRegisterOrConstant(rhs));
+ define(ins, mir);
+}
+
+void
+LIRGeneratorPPC64::lowerDivI(MDiv* div)
+{
+ if (div->isUnsigned()) {
+ lowerUDiv(div);
+ return;
+ }
+
+ // Division instructions are slow. Division by constant denominators can be
+ // rewritten to use other instructions.
+ if (div->rhs()->isConstant()) {
+ int32_t rhs = div->rhs()->toConstant()->toInt32();
+ // Check for division by a positive power of two, which is an easy and
+ // important case to optimize. Note that other optimizations are also
+ // possible; division by negative powers of two can be optimized in a
+ // similar manner as positive powers of two, and division by other
+ // constants can be optimized by a reciprocal multiplication technique.
+ int32_t shift = FloorLog2(rhs);
+ if (rhs > 0 && 1 << shift == rhs) {
+ LDivPowTwoI* lir = new(alloc()) LDivPowTwoI(useRegister(div->lhs()), shift, temp());
+ if (div->fallible())
+ assignSnapshot(lir, div->bailoutKind());
+ define(lir, div);
+ return;
+ }
+ }
+
+ LDivI* lir = new(alloc()) LDivI(useRegister(div->lhs()), useRegister(div->rhs()), temp());
+ if (div->fallible())
+ assignSnapshot(lir, div->bailoutKind());
+ define(lir, div);
+}
+
+void LIRGeneratorPPC64::lowerNegI(MInstruction* ins, MDefinition* input) {
+ define(new (alloc()) LNegI(useRegisterAtStart(input)), ins);
+}
+
+void LIRGeneratorPPC64::lowerNegI64(MInstruction* ins, MDefinition* input) {
+ defineInt64(new (alloc()) LNegI64(useInt64RegisterAtStart(input)), ins);
+}
+
+void LIRGenerator::visitAbs(MAbs* ins) {
+ define(allocateAbs(ins, useRegisterAtStart(ins->input())), ins);
+}
+
+void
+LIRGeneratorPPC64::lowerMulI(MMul* mul, MDefinition* lhs, MDefinition* rhs)
+{
+ // Automatically reduced by xs_sr_mulli.
+ LMulI* lir = new(alloc()) LMulI;
+ if (mul->fallible())
+ assignSnapshot(lir, mul->bailoutKind());
+
+ lowerForALU(lir, mul, lhs, rhs);
+}
+
+void
+LIRGeneratorPPC64::lowerModI(MMod* mod)
+{
+ if (mod->isUnsigned()) {
+ lowerUMod(mod);
+ return;
+ }
+
+ // POWER9 has hardware modulo, so we don't need to lower it there.
+ if (!HasPPCISA3() && mod->rhs()->isConstant()) {
+ int32_t rhs = mod->rhs()->toConstant()->toInt32();
+ int32_t shift = FloorLog2(rhs);
+ if (rhs > 0 && 1 << shift == rhs) {
+ LModPowTwoI* lir = new(alloc()) LModPowTwoI(useRegister(mod->lhs()), shift);
+ if (mod->fallible())
+ assignSnapshot(lir, mod->bailoutKind());
+ define(lir, mod);
+ return;
+ } else if (shift < 31 && (1 << (shift + 1)) - 1 == rhs) {
+ LModMaskI* lir = new(alloc()) LModMaskI(useRegister(mod->lhs()),
+ temp(LDefinition::GENERAL),
+ temp(LDefinition::GENERAL),
+ shift + 1);
+ if (mod->fallible())
+ assignSnapshot(lir, mod->bailoutKind());
+ define(lir, mod);
+ return;
+ }
+ }
+ LModI* lir = new(alloc()) LModI(useRegister(mod->lhs()), useRegister(mod->rhs()),
+ temp(LDefinition::GENERAL));
+
+ if (mod->fallible())
+ assignSnapshot(lir, mod->bailoutKind());
+ define(lir, mod);
+}
+
+void
+LIRGenerator::visitPowHalf(MPowHalf* ins)
+{
+ MDefinition* input = ins->input();
+ MOZ_ASSERT(input->type() == MIRType::Double);
+ LPowHalfD* lir = new(alloc()) LPowHalfD(useRegisterAtStart(input));
+ defineReuseInput(lir, ins, 0);
+}
+
+void LIRGeneratorPPC64::lowerWasmSelectI(MWasmSelect* select) {
+/*
+ auto* lir = new (alloc())
+ LWasmSelect(useRegisterAtStart(select->trueExpr()),
+ useAny(select->falseExpr()), useRegister(select->condExpr()));
+ defineReuseInput(lir, select, LWasmSelect::TrueExprIndex);
+*/
+ MOZ_CRASH("NYI");
+}
+
+void LIRGeneratorPPC64::lowerWasmSelectI64(MWasmSelect* select) {
+/*
+ auto* lir = new (alloc()) LWasmSelectI64(
+ useInt64RegisterAtStart(select->trueExpr()),
+ useInt64(select->falseExpr()), useRegister(select->condExpr()));
+ defineInt64ReuseInput(lir, select, LWasmSelectI64::TrueExprIndex);
+*/
+ MOZ_CRASH("NYI");
+}
+
+LTableSwitch*
+LIRGeneratorPPC64::newLTableSwitch(const LAllocation& in, const LDefinition& inputCopy,
+ MTableSwitch* tableswitch)
+{
+ return new(alloc()) LTableSwitch(in, inputCopy, temp(), tableswitch);
+}
+
+LTableSwitchV*
+LIRGeneratorPPC64::newLTableSwitchV(MTableSwitch* tableswitch)
+{
+ return new(alloc()) LTableSwitchV(useBox(tableswitch->getOperand(0)),
+ temp(), tempDouble(), temp(), tableswitch);
+}
+
+void
+LIRGeneratorPPC64::lowerUrshD(MUrsh* mir)
+{
+ MDefinition* lhs = mir->lhs();
+ MDefinition* rhs = mir->rhs();
+
+ MOZ_ASSERT(lhs->type() == MIRType::Int32);
+ MOZ_ASSERT(rhs->type() == MIRType::Int32);
+
+ LUrshD* lir = new(alloc()) LUrshD(useRegister(lhs), useRegisterOrConstant(rhs), temp());
+ define(lir, mir);
+}
+
+void LIRGeneratorPPC64::lowerPowOfTwoI(MPow* mir) {
+ int32_t base = mir->input()->toConstant()->toInt32();
+ MDefinition* power = mir->power();
+
+ auto* lir = new (alloc()) LPowOfTwoI(useRegister(power), base);
+ assignSnapshot(lir, mir->bailoutKind());
+ define(lir, mir);
+}
+
+void LIRGeneratorPPC64::lowerBigIntLsh(MBigIntLsh* ins) {
+ auto* lir = new (alloc()) LBigIntLsh(
+ useRegister(ins->lhs()), useRegister(ins->rhs()), temp(), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorPPC64::lowerBigIntRsh(MBigIntRsh* ins) {
+ auto* lir = new (alloc()) LBigIntRsh(
+ useRegister(ins->lhs()), useRegister(ins->rhs()), temp(), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void
+LIRGenerator::visitWasmNeg(MWasmNeg* ins)
+{
+ if (ins->type() == MIRType::Int32) {
+ define(new(alloc()) LNegI(useRegisterAtStart(ins->input())), ins);
+ } else if (ins->type() == MIRType::Float32) {
+ define(new(alloc()) LNegF(useRegisterAtStart(ins->input())), ins);
+ } else {
+ MOZ_ASSERT(ins->type() == MIRType::Double);
+ define(new(alloc()) LNegD(useRegisterAtStart(ins->input())), ins);
+ }
+}
+
+/*
+void LIRGenerator::visitWasmHeapBase(MWasmHeapBase* ins) {
+ auto* lir = new (alloc()) LWasmHeapBase(LAllocation());
+ define(lir, ins);
+ MOZ_CRASH("NYI");
+}
+*/
+
+void
+LIRGenerator::visitWasmLoad(MWasmLoad* ins)
+{
+ MDefinition* base = ins->base();
+ // |base| is a GPR but may be of either type. If it is 32-bit, it is
+ // sign-extended on 64-bit Power and we should explicitly promote it to
+ // 64-bit when used as an index register in memory accesses.
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ LAllocation ptr = useRegisterAtStart(base);
+
+ if (IsUnaligned(ins->access())) {
+ if (ins->type() == MIRType::Int64) {
+ auto* lir = new(alloc()) LWasmUnalignedLoadI64(ptr, temp());
+ if (ins->access().offset())
+ lir->setTemp(0, tempCopy(base, 0));
+
+ defineInt64(lir, ins);
+ return;
+ }
+
+ auto* lir = new(alloc()) LWasmUnalignedLoad(ptr, temp());
+ if (ins->access().offset())
+ lir->setTemp(0, tempCopy(base, 0));
+
+ define(lir, ins);
+ return;
+ }
+
+ if (ins->type() == MIRType::Int64) {
+ auto* lir = new(alloc()) LWasmLoadI64(ptr);
+ if (ins->access().offset())
+ lir->setTemp(0, tempCopy(base, 0));
+
+ defineInt64(lir, ins);
+ return;
+ }
+
+ auto* lir = new(alloc()) LWasmLoad(ptr);
+ if (ins->access().offset())
+ lir->setTemp(0, tempCopy(base, 0));
+
+ define(lir, ins);
+}
+
+void
+LIRGenerator::visitWasmStore(MWasmStore* ins)
+{
+ MDefinition* base = ins->base();
+ // All about the two kinds of bases, no treble. Or trouble, maybe.
+ // How's that line go again?
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ MDefinition* value = ins->value();
+
+ if (IsUnaligned(ins->access())) {
+ LAllocation baseAlloc = useRegisterAtStart(base);
+ if (ins->access().type() == Scalar::Int64) {
+ LInt64Allocation valueAlloc = useInt64RegisterAtStart(value);
+ auto* lir = new(alloc()) LWasmUnalignedStoreI64(baseAlloc, valueAlloc, temp());
+ if (ins->access().offset())
+ lir->setTemp(0, tempCopy(base, 0));
+
+ add(lir, ins);
+ return;
+ }
+
+ LAllocation valueAlloc = useRegisterAtStart(value);
+ auto* lir = new(alloc()) LWasmUnalignedStore(baseAlloc, valueAlloc, temp());
+ if (ins->access().offset())
+ lir->setTemp(0, tempCopy(base, 0));
+
+ add(lir, ins);
+ return;
+ }
+
+ if (ins->access().type() == Scalar::Int64) {
+ LAllocation baseAlloc = useRegisterAtStart(base);
+ LInt64Allocation valueAlloc = useInt64RegisterAtStart(value);
+ auto* lir = new(alloc()) LWasmStoreI64(baseAlloc, valueAlloc);
+ if (ins->access().offset())
+ lir->setTemp(0, tempCopy(base, 0));
+
+ add(lir, ins);
+ return;
+ }
+
+ LAllocation baseAlloc = useRegisterAtStart(base);
+ LAllocation valueAlloc = useRegisterAtStart(value);
+ auto* lir = new(alloc()) LWasmStore(baseAlloc, valueAlloc);
+ if (ins->access().offset())
+ lir->setTemp(0, tempCopy(base, 0));
+
+ add(lir, ins);
+}
+
+void
+LIRGeneratorPPC64::lowerUDiv(MDiv* div)
+{
+ MDefinition* lhs = div->getOperand(0);
+ MDefinition* rhs = div->getOperand(1);
+
+ LUDivOrMod* lir = new(alloc()) LUDivOrMod;
+ lir->setOperand(0, useRegister(lhs));
+ lir->setOperand(1, useRegister(rhs));
+ if (div->fallible())
+ assignSnapshot(lir, div->bailoutKind());
+
+ define(lir, div);
+}
+
+void
+LIRGeneratorPPC64::lowerUMod(MMod* mod)
+{
+ MDefinition* lhs = mod->getOperand(0);
+ MDefinition* rhs = mod->getOperand(1);
+
+ LUDivOrMod* lir = new(alloc()) LUDivOrMod;
+ lir->setOperand(0, useRegister(lhs));
+ lir->setOperand(1, useRegister(rhs));
+ if (mod->fallible())
+ assignSnapshot(lir, mod->bailoutKind());
+
+ define(lir, mod);
+}
+
+void
+LIRGenerator::visitWasmUnsignedToDouble(MWasmUnsignedToDouble* ins)
+{
+ MOZ_ASSERT(ins->input()->type() == MIRType::Int32);
+ LWasmUint32ToDouble* lir = new(alloc()) LWasmUint32ToDouble(useRegisterAtStart(ins->input()));
+ define(lir, ins);
+}
+
+void
+LIRGenerator::visitWasmUnsignedToFloat32(MWasmUnsignedToFloat32* ins)
+{
+ MOZ_ASSERT(ins->input()->type() == MIRType::Int32);
+ LWasmUint32ToFloat32* lir = new(alloc()) LWasmUint32ToFloat32(useRegisterAtStart(ins->input()));
+ define(lir, ins);
+}
+
+void
+LIRGenerator::visitAsmJSLoadHeap(MAsmJSLoadHeap* ins)
+{
+ MOZ_ASSERT(ins->access().offset() == 0);
+
+ MDefinition* base = ins->base();
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+ LAllocation baseAlloc;
+ LAllocation limitAlloc;
+ // We'd prefer base in a register or this will be slow.
+ if (base->isConstant() && !ins->needsBoundsCheck()) {
+ // A bounds check is only skipped for a positive index.
+ MOZ_ASSERT(base->toConstant()->toInt32() >= 0);
+ baseAlloc = LAllocation(base->toConstant());
+ } else {
+ baseAlloc = useRegisterAtStart(base);
+ if (ins->needsBoundsCheck()) {
+ MDefinition* boundsCheckLimit = ins->boundsCheckLimit();
+ MOZ_ASSERT(boundsCheckLimit->type() == MIRType::Int32);
+ limitAlloc = useRegisterAtStart(boundsCheckLimit);
+ }
+ }
+
+ define(new(alloc()) LAsmJSLoadHeap(baseAlloc, limitAlloc, LAllocation()), ins);
+}
+
+void
+LIRGenerator::visitAsmJSStoreHeap(MAsmJSStoreHeap* ins)
+{
+ MOZ_ASSERT(ins->access().offset() == 0);
+
+ MDefinition* base = ins->base();
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+ LAllocation baseAlloc;
+ LAllocation limitAlloc;
+ if (base->isConstant() && !ins->needsBoundsCheck()) {
+ MOZ_ASSERT(base->toConstant()->toInt32() >= 0);
+ baseAlloc = LAllocation(base->toConstant());
+ } else{
+ baseAlloc = useRegisterAtStart(base);
+ if (ins->needsBoundsCheck()) {
+ MDefinition* boundsCheckLimit = ins->boundsCheckLimit();
+ MOZ_ASSERT(boundsCheckLimit->type() == MIRType::Int32);
+ limitAlloc = useRegisterAtStart(boundsCheckLimit);
+ }
+ }
+
+ add(new(alloc()) LAsmJSStoreHeap(baseAlloc, useRegisterAtStart(ins->value()), limitAlloc, LAllocation()), ins);
+}
+
+void
+LIRGenerator::visitSubstr(MSubstr* ins)
+{
+ LSubstr* lir = new (alloc()) LSubstr(useRegister(ins->string()),
+ useRegister(ins->begin()),
+ useRegister(ins->length()),
+ temp(),
+ temp(),
+ tempByteOpRegister());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitCompareExchangeTypedArrayElement(
+ MCompareExchangeTypedArrayElement* ins) {
+ MOZ_ASSERT(ins->arrayType() != Scalar::Float32);
+ MOZ_ASSERT(ins->arrayType() != Scalar::Float64);
+
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+
+ const LAllocation newval = useRegister(ins->newval());
+ const LAllocation oldval = useRegister(ins->oldval());
+
+ if (Scalar::isBigIntType(ins->arrayType())) {
+ LInt64Definition temp1 = tempInt64();
+ LInt64Definition temp2 = tempInt64();
+
+ auto* lir = new (alloc()) LCompareExchangeTypedArrayElement64(
+ elements, index, oldval, newval, temp1, temp2);
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ return;
+ }
+
+ // If the target is an FPR then we need a temp at the
+ // CodeGenerator level for creating the result. XXX: we do?
+
+ LDefinition outTemp = LDefinition::BogusTemp();
+ LDefinition valueTemp = LDefinition::BogusTemp();
+ LDefinition offsetTemp = LDefinition::BogusTemp();
+ LDefinition maskTemp = LDefinition::BogusTemp();
+
+ if (ins->arrayType() == Scalar::Uint32 && IsFloatingPointType(ins->type())) {
+ outTemp = temp();
+ }
+
+ if (Scalar::byteSize(ins->arrayType()) < 4) {
+ valueTemp = temp();
+ offsetTemp = temp();
+ maskTemp = temp();
+ }
+
+ LCompareExchangeTypedArrayElement* lir = new (alloc())
+ LCompareExchangeTypedArrayElement(elements, index, oldval, newval,
+ outTemp, valueTemp, offsetTemp,
+ maskTemp);
+
+ define(lir, ins);
+}
+
+void LIRGenerator::visitAtomicExchangeTypedArrayElement(
+ MAtomicExchangeTypedArrayElement* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+
+ const LAllocation value = useRegister(ins->value());
+
+ if (Scalar::isBigIntType(ins->arrayType())) {
+ LInt64Definition temp1 = tempInt64();
+ LDefinition temp2 = temp();
+
+ auto* lir = new (alloc()) LAtomicExchangeTypedArrayElement64(
+ elements, index, value, temp1, temp2);
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ return;
+ }
+
+ // Same thing.
+ // XXX: same thing?
+
+ MOZ_ASSERT(ins->arrayType() <= Scalar::Uint32);
+
+ LDefinition outTemp = LDefinition::BogusTemp();
+ LDefinition valueTemp = LDefinition::BogusTemp();
+ LDefinition offsetTemp = LDefinition::BogusTemp();
+ LDefinition maskTemp = LDefinition::BogusTemp();
+
+ if (ins->arrayType() == Scalar::Uint32) {
+ MOZ_ASSERT(ins->type() == MIRType::Double);
+ outTemp = temp();
+ }
+
+ if (Scalar::byteSize(ins->arrayType()) < 4) {
+ valueTemp = temp();
+ offsetTemp = temp();
+ maskTemp = temp();
+ }
+
+ LAtomicExchangeTypedArrayElement* lir =
+ new (alloc()) LAtomicExchangeTypedArrayElement(
+ elements, index, value, outTemp, valueTemp, offsetTemp, maskTemp);
+
+ define(lir, ins);
+}
+
+void
+LIRGenerator::visitWasmCompareExchangeHeap(MWasmCompareExchangeHeap* ins)
+{
+ MDefinition* base = ins->base();
+ // Again with the bases?
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ if (ins->access().type() == Scalar::Int64) {
+ auto* lir = new (alloc()) LWasmCompareExchangeI64(
+ useRegister(base), useInt64Register(ins->oldValue()),
+ useInt64Register(ins->newValue()));
+ defineInt64(lir, ins);
+ return;
+ }
+
+ LDefinition valueTemp = LDefinition::BogusTemp();
+ LDefinition offsetTemp = LDefinition::BogusTemp();
+ LDefinition maskTemp = LDefinition::BogusTemp();
+
+ if (ins->access().byteSize() < 4) {
+ valueTemp = temp();
+ offsetTemp = temp();
+ maskTemp = temp();
+ }
+
+ LWasmCompareExchangeHeap* lir = new (alloc()) LWasmCompareExchangeHeap(
+ useRegister(base), useRegister(ins->oldValue()),
+ useRegister(ins->newValue()), valueTemp, offsetTemp, maskTemp);
+
+ define(lir, ins);
+}
+
+void
+LIRGenerator::visitWasmAtomicExchangeHeap(MWasmAtomicExchangeHeap* ins)
+{
+ MDefinition* base = ins->base();
+ // See comment in visitWasmLoad re the type of 'base'.
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ if (ins->access().type() == Scalar::Int64) {
+ auto* lir = new (alloc()) LWasmAtomicExchangeI64(
+ useRegister(base), useInt64Register(ins->value()));
+ defineInt64(lir, ins);
+ return;
+ }
+
+ LDefinition valueTemp = LDefinition::BogusTemp();
+ LDefinition offsetTemp = LDefinition::BogusTemp();
+ LDefinition maskTemp = LDefinition::BogusTemp();
+
+ if (ins->access().byteSize() < 4) {
+ valueTemp = temp();
+ offsetTemp = temp();
+ maskTemp = temp();
+ }
+
+ LWasmAtomicExchangeHeap* lir = new (alloc())
+ LWasmAtomicExchangeHeap(useRegister(base), useRegister(ins->value()),
+ valueTemp, offsetTemp, maskTemp);
+ define(lir, ins);
+}
+
+void
+LIRGenerator::visitWasmAtomicBinopHeap(MWasmAtomicBinopHeap* ins)
+{
+ MDefinition* base = ins->base();
+ // See comment in visitWasmLoad re the type of 'base'.
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ if (ins->access().type() == Scalar::Int64) {
+ auto* lir = new (alloc())
+ LWasmAtomicBinopI64(useRegister(base), useInt64Register(ins->value()));
+ lir->setTemp(0, temp());
+ defineInt64(lir, ins);
+ return;
+ }
+
+ LDefinition valueTemp = LDefinition::BogusTemp();
+ LDefinition offsetTemp = LDefinition::BogusTemp();
+ LDefinition maskTemp = LDefinition::BogusTemp();
+
+ if (ins->access().byteSize() < 4) {
+ valueTemp = temp();
+ offsetTemp = temp();
+ maskTemp = temp();
+ }
+
+ if (!ins->hasUses()) {
+ LWasmAtomicBinopHeapForEffect* lir = new (alloc())
+ LWasmAtomicBinopHeapForEffect(useRegister(base),
+ useRegister(ins->value()), valueTemp,
+ offsetTemp, maskTemp);
+ add(lir, ins);
+ return;
+ }
+
+ LWasmAtomicBinopHeap* lir = new (alloc())
+ LWasmAtomicBinopHeap(useRegister(base), useRegister(ins->value()),
+ valueTemp, offsetTemp, maskTemp);
+
+ define(lir, ins);
+}
+
+void LIRGenerator::visitAtomicTypedArrayElementBinop(
+ MAtomicTypedArrayElementBinop* ins) {
+ MOZ_ASSERT(ins->arrayType() != Scalar::Uint8Clamped);
+ MOZ_ASSERT(ins->arrayType() != Scalar::Float32);
+ MOZ_ASSERT(ins->arrayType() != Scalar::Float64);
+
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+ const LAllocation value = useRegister(ins->value());
+
+ if (Scalar::isBigIntType(ins->arrayType())) {
+ LInt64Definition temp1 = tempInt64();
+ LInt64Definition temp2 = tempInt64();
+
+ // Case 1: the result of the operation is not used.
+ //
+ // We can omit allocating the result BigInt.
+
+ if (ins->isForEffect()) {
+ auto* lir = new (alloc()) LAtomicTypedArrayElementBinopForEffect64(
+ elements, index, value, temp1, temp2);
+ add(lir, ins);
+ return;
+ }
+
+ // Case 2: the result of the operation is used.
+
+ auto* lir = new (alloc())
+ LAtomicTypedArrayElementBinop64(elements, index, value, temp1, temp2);
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ return;
+ }
+
+ LDefinition valueTemp = LDefinition::BogusTemp();
+ LDefinition offsetTemp = LDefinition::BogusTemp();
+ LDefinition maskTemp = LDefinition::BogusTemp();
+
+ if (Scalar::byteSize(ins->arrayType()) < 4) {
+ valueTemp = temp();
+ offsetTemp = temp();
+ maskTemp = temp();
+ }
+
+ if (ins->isForEffect()) {
+ LAtomicTypedArrayElementBinopForEffect* lir =
+ new (alloc()) LAtomicTypedArrayElementBinopForEffect(
+ elements, index, value, valueTemp, offsetTemp, maskTemp);
+ add(lir, ins);
+ return;
+ }
+
+ // For a Uint32Array with a known double result we need a temp for
+ // the intermediate output.
+
+ LDefinition outTemp = LDefinition::BogusTemp();
+
+ if (ins->arrayType() == Scalar::Uint32 && IsFloatingPointType(ins->type())) {
+ outTemp = temp();
+ }
+
+ LAtomicTypedArrayElementBinop* lir =
+ new (alloc()) LAtomicTypedArrayElementBinop(
+ elements, index, value, outTemp, valueTemp, offsetTemp, maskTemp);
+ define(lir, ins);
+}
+
+void
+LIRGenerator::visitCopySign(MCopySign* ins)
+{
+ MDefinition* lhs = ins->lhs();
+ MDefinition* rhs = ins->rhs();
+
+ MOZ_ASSERT(IsFloatingPointType(lhs->type()));
+ MOZ_ASSERT(lhs->type() == rhs->type());
+ MOZ_ASSERT(lhs->type() == ins->type());
+
+ LInstructionHelper<1, 2, 2>* lir;
+ if (lhs->type() == MIRType::Double)
+ lir = new(alloc()) LCopySignD();
+ else
+ lir = new(alloc()) LCopySignF();
+
+ lir->setTemp(0, temp());
+ lir->setTemp(1, temp());
+
+ lir->setOperand(0, useRegisterAtStart(lhs));
+ lir->setOperand(1, useRegister(rhs));
+ defineReuseInput(lir, ins, 0);
+}
+
+void
+LIRGenerator::visitExtendInt32ToInt64(MExtendInt32ToInt64* ins)
+{
+ defineInt64(new(alloc()) LExtendInt32ToInt64(useRegisterAtStart(ins->input())), ins);
+}
+
+void
+LIRGenerator::visitSignExtendInt64(MSignExtendInt64* ins)
+{
+ defineInt64(new(alloc()) LSignExtendInt64(useInt64RegisterAtStart(ins->input())), ins);
+}
+
+bool LIRGeneratorShared::canSpecializeWasmCompareAndSelect(
+ MCompare::CompareType compTy, MIRType insTy) {
+ // XXX: for now
+ // Should be able to implement this for (u)int32 compare/select
+ // with isel, and probably floats/doubles with fsel.
+/*
+ return insTy == MIRType::Int32 && (compTy == MCompare::Compare_Int32 ||
+ compTy == MCompare::Compare_UInt32);
+*/
+ return false;
+}
+
+void LIRGeneratorShared::lowerWasmCompareAndSelect(MWasmSelect* ins,
+ MDefinition* lhs,
+ MDefinition* rhs,
+ MCompare::CompareType compTy,
+ JSOp jsop) {
+ MOZ_ASSERT(canSpecializeWasmCompareAndSelect(compTy, ins->type()));
+/*
+ auto* lir = new (alloc()) LWasmCompareAndSelect(
+ useRegister(lhs), useRegister(rhs), compTy, jsop,
+ useRegisterAtStart(ins->trueExpr()), useRegister(ins->falseExpr()));
+ defineReuseInput(lir, ins, LWasmCompareAndSelect::IfTrueExprIndex);
+*/
+ MOZ_CRASH("that was unexpected");
+}
+
+void LIRGenerator::visitWasmTernarySimd128(MWasmTernarySimd128* ins) {
+ MOZ_CRASH("ternary SIMD NYI");
+}
+
+void LIRGenerator::visitWasmBinarySimd128(MWasmBinarySimd128* ins) {
+ MOZ_CRASH("binary SIMD NYI");
+}
+
+bool MWasmBinarySimd128::specializeForConstantRhs() {
+ // Probably many we want to do here
+ return false;
+}
+
+void LIRGenerator::visitWasmBinarySimd128WithConstant(
+ MWasmBinarySimd128WithConstant* ins) {
+ MOZ_CRASH("binary SIMD with constant NYI");
+}
+
+void LIRGenerator::visitWasmShiftSimd128(MWasmShiftSimd128* ins) {
+ MOZ_CRASH("shift SIMD NYI");
+}
+
+void LIRGenerator::visitWasmShuffleSimd128(MWasmShuffleSimd128* ins) {
+ MOZ_CRASH("shuffle SIMD NYI");
+}
+
+void LIRGenerator::visitWasmReplaceLaneSimd128(MWasmReplaceLaneSimd128* ins) {
+ MOZ_CRASH("replace-lane SIMD NYI");
+}
+
+void LIRGenerator::visitWasmScalarToSimd128(MWasmScalarToSimd128* ins) {
+ MOZ_CRASH("scalar-to-SIMD NYI");
+}
+
+void LIRGenerator::visitWasmUnarySimd128(MWasmUnarySimd128* ins) {
+ MOZ_CRASH("unary SIMD NYI");
+}
+
+void LIRGenerator::visitWasmReduceSimd128(MWasmReduceSimd128* ins) {
+ MOZ_CRASH("reduce-SIMD NYI");
+}
+
+void LIRGenerator::visitWasmLoadLaneSimd128(MWasmLoadLaneSimd128* ins) {
+ MOZ_CRASH("load-lane SIMD NYI");
+}
+
+void LIRGenerator::visitWasmStoreLaneSimd128(MWasmStoreLaneSimd128* ins) {
+ MOZ_CRASH("store-lane SIMD NYI");
+}
+
+// XXX: end mips-shared
+
+void
+LIRGeneratorPPC64::defineInt64Phi(MPhi* phi, size_t lirIndex)
+{
+ defineTypedPhi(phi, lirIndex);
+}
+
+void
+LIRGeneratorPPC64::lowerInt64PhiInput(MPhi* phi, uint32_t inputPosition,
+ LBlock* block, size_t lirIndex)
+{
+ lowerTypedPhiInput(phi, inputPosition, block, lirIndex);
+}
+
+LBoxAllocation
+LIRGeneratorPPC64::useBoxFixed(MDefinition* mir, Register reg1, Register reg2, bool useAtStart)
+{
+ MOZ_ASSERT(mir->type() == MIRType::Value);
+
+ ensureDefined(mir);
+ return LBoxAllocation(LUse(reg1, mir->virtualRegister(), useAtStart));
+}
+
+void
+LIRGeneratorPPC64::lowerDivI64(MDiv* div)
+{
+ if (div->isUnsigned()) {
+ lowerUDivI64(div);
+ return;
+ }
+
+ LDivOrModI64* lir = new(alloc()) LDivOrModI64(useRegister(div->lhs()), useRegister(div->rhs()),
+ temp());
+ defineInt64(lir, div);
+}
+
+void LIRGeneratorPPC64::lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div) {
+// XXX: we don't?
+ MOZ_CRASH("We don't use runtime div for this architecture");
+}
+
+void
+LIRGeneratorPPC64::lowerModI64(MMod* mod)
+{
+ if (mod->isUnsigned()) {
+ lowerUModI64(mod);
+ return;
+ }
+
+ LDivOrModI64* lir = new(alloc()) LDivOrModI64(useRegister(mod->lhs()), useRegister(mod->rhs()),
+ temp());
+ defineInt64(lir, mod);
+}
+
+void LIRGeneratorPPC64::lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod) {
+// XXX: we don't?
+ MOZ_CRASH("We don't use runtime mod for this architecture");
+}
+
+void
+LIRGeneratorPPC64::lowerUDivI64(MDiv* div)
+{
+ LUDivOrModI64* lir = new(alloc()) LUDivOrModI64(useRegister(div->lhs()),
+ useRegister(div->rhs()),
+ temp());
+ defineInt64(lir, div);
+}
+
+void
+LIRGeneratorPPC64::lowerUModI64(MMod* mod)
+{
+ LUDivOrModI64* lir = new(alloc()) LUDivOrModI64(useRegister(mod->lhs()),
+ useRegister(mod->rhs()),
+ temp());
+ defineInt64(lir, mod);
+}
+
+void LIRGeneratorPPC64::lowerBigIntDiv(MBigIntDiv* ins) {
+ auto* lir = new (alloc()) LBigIntDiv(useRegister(ins->lhs()),
+ useRegister(ins->rhs()), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorPPC64::lowerBigIntMod(MBigIntMod* ins) {
+ auto* lir = new (alloc()) LBigIntMod(useRegister(ins->lhs()),
+ useRegister(ins->rhs()), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorPPC64::lowerAtomicLoad64(MLoadUnboxedScalar* ins) {
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->storageType());
+
+ auto* lir = new (alloc()) LAtomicLoad64(elements, index, temp(), tempInt64());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorPPC64::lowerAtomicStore64(MStoreUnboxedScalar* ins) {
+ LUse elements = useRegister(ins->elements());
+ LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->writeType());
+ LAllocation value = useRegister(ins->value());
+
+ add(new (alloc()) LAtomicStore64(elements, index, value, tempInt64()), ins);
+}
+
+void
+LIRGenerator::visitBox(MBox* box)
+{
+ MDefinition* opd = box->getOperand(0);
+
+ // If the operand is a constant, emit near its uses.
+ if (opd->isConstant() && box->canEmitAtUses()) {
+ emitAtUses(box);
+ return;
+ }
+
+ if (opd->isConstant()) {
+ define(new(alloc()) LValue(opd->toConstant()->toJSValue()), box, LDefinition(LDefinition::BOX));
+ } else {
+ LBox* ins = new(alloc()) LBox(useRegister(opd), opd->type());
+ define(ins, box, LDefinition(LDefinition::BOX));
+ }
+}
+
+void
+LIRGenerator::visitUnbox(MUnbox* unbox)
+{
+ MDefinition* box = unbox->getOperand(0);
+
+/*
+ if (box->type() == MIRType::ObjectOrNull) {
+ LUnboxObjectOrNull* lir = new(alloc()) LUnboxObjectOrNull(useRegisterAtStart(box));
+ if (unbox->fallible())
+ assignSnapshot(lir, unbox->bailoutKind());
+ defineReuseInput(lir, unbox, 0);
+ return;
+ }
+*/
+
+ MOZ_ASSERT(box->type() == MIRType::Value);
+
+ LUnbox* lir;
+ if (IsFloatingPointType(unbox->type())) {
+ lir = new(alloc()) LUnboxFloatingPoint(useRegisterAtStart(box), unbox->type());
+ } else if (unbox->fallible()) {
+ // If the unbox is fallible, load the Value in a register first to
+ // avoid multiple loads.
+ lir = new(alloc()) LUnbox(useRegisterAtStart(box));
+ } else {
+ lir = new(alloc()) LUnbox(useAtStart(box));
+ }
+
+ if (unbox->fallible())
+ assignSnapshot(lir, unbox->bailoutKind());
+
+ define(lir, unbox);
+}
+
+void
+LIRGenerator::visitReturnImpl(MDefinition* opd, bool isGenerator)
+{
+ MOZ_ASSERT(opd->type() == MIRType::Value);
+
+ LReturn* ins = new(alloc()) LReturn(isGenerator);
+ ins->setOperand(0, useFixed(opd, JSReturnReg));
+ add(ins);
+}
+
+void
+LIRGeneratorPPC64::defineUntypedPhi(MPhi* phi, size_t lirIndex)
+{
+ defineTypedPhi(phi, lirIndex);
+}
+
+void
+LIRGeneratorPPC64::lowerUntypedPhiInput(MPhi* phi, uint32_t inputPosition,
+ LBlock* block, size_t lirIndex)
+{
+ lowerTypedPhiInput(phi, inputPosition, block, lirIndex);
+}
+
+void
+LIRGeneratorPPC64::lowerTruncateDToInt32(MTruncateToInt32* ins)
+{
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Double);
+
+ define(new(alloc())
+ LTruncateDToInt32(useRegister(opd), tempDouble()), ins);
+}
+
+void
+LIRGeneratorPPC64::lowerTruncateFToInt32(MTruncateToInt32* ins)
+{
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Float32);
+
+ define(new(alloc())
+ LTruncateFToInt32(useRegister(opd), tempFloat32()), ins);
+}
+
+void
+LIRGenerator::visitWasmTruncateToInt64(MWasmTruncateToInt64* ins)
+{
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Double || opd->type() == MIRType::Float32);
+
+ defineInt64(new(alloc()) LWasmTruncateToInt64(useRegister(opd)), ins);
+}
+
+void LIRGeneratorPPC64::lowerWasmBuiltinTruncateToInt64(
+ MWasmBuiltinTruncateToInt64* ins) {
+ MOZ_CRASH("We don't use it for this architecture");
+}
+
+void
+LIRGenerator::visitInt64ToFloatingPoint(MInt64ToFloatingPoint* ins)
+{
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Int64);
+ MOZ_ASSERT(IsFloatingPointType(ins->type()));
+
+ define(new(alloc()) LInt64ToFloatingPoint(useInt64Register(opd)), ins);
+}
+
+void
+LIRGeneratorPPC64::lowerBuiltinInt64ToFloatingPoint(MBuiltinInt64ToFloatingPoint *)
+{
+ MOZ_CRASH("NYI");
+}
+
+#if 0
+// A beautiful AltiVec and VSX world awaits me if I could only stop procrastinating.
+void
+LIRGenerator::visitSimdInsertElement(MSimdInsertElement*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+LIRGenerator::visitSimdExtractElement(MSimdExtractElement*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+LIRGenerator::visitSimdBinaryArith(MSimdBinaryArith*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+LIRGenerator::visitSimdSelect(MSimdSelect*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+LIRGenerator::visitSimdSplat(MSimdSplat*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+LIRGenerator::visitSimdValueX4(MSimdValueX4*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+LIRGenerator::visitSimdBinarySaturating(MSimdBinarySaturating*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+LIRGenerator::visitSimdSwizzle(MSimdSwizzle*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+LIRGenerator::visitSimdShuffle(MSimdShuffle*)
+{
+ MOZ_CRASH("NYI");
+}
+
+void
+LIRGenerator::visitSimdGeneralShuffle(MSimdGeneralShuffle*)
+{
+ MOZ_CRASH("NYI");
+}
+#endif
+
diff -r 485b15bb4a20 -r 9c245f4665be js/src/jit/ppc64/Lowering-ppc64.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/js/src/jit/ppc64/Lowering-ppc64.h Thu Aug 08 21:23:52 2024 -0700
@@ -0,0 +1,136 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_ppc64le_Lowering_ppc64le_h
+#define jit_ppc64le_Lowering_ppc64le_h
+
+#include "jit/shared/Lowering-shared.h"
+
+namespace js {
+namespace jit {
+
+class LIRGeneratorPPC64 : public LIRGeneratorShared
+{
+ protected:
+ LIRGeneratorPPC64(MIRGenerator* gen, MIRGraph& graph, LIRGraph& lirGraph)
+ : LIRGeneratorShared(gen, graph, lirGraph)
+ { }
+
+ // x86 has constraints on what registers can be formatted for 1-byte
+ // stores and loads, but on Power all GPRs are okay.
+ LAllocation useByteOpRegister(MDefinition* mir);
+ LAllocation useByteOpRegisterAtStart(MDefinition* mir);
+ LAllocation useByteOpRegisterOrNonDoubleConstant(MDefinition* mir);
+ LDefinition tempByteOpRegister();
+
+ bool needTempForPostBarrier() { return false; }
+
+ void lowerForShift(LInstructionHelper<1, 2, 0>* ins, MDefinition* mir, MDefinition* lhs,
+ MDefinition* rhs);
+ void lowerUrshD(MUrsh* mir);
+
+ void lowerForALU(LInstructionHelper<1, 1, 0>* ins, MDefinition* mir,
+ MDefinition* input);
+ void lowerForALU(LInstructionHelper<1, 2, 0>* ins, MDefinition* mir,
+ MDefinition* lhs, MDefinition* rhs);
+
+ void lowerForALUInt64(LInstructionHelper<INT64_PIECES, INT64_PIECES, 0>* ins,
+ MDefinition* mir, MDefinition* input);
+ void lowerForALUInt64(
+ LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs);
+
+ void lowerForMulInt64(LMulI64* ins, MMul* mir, MDefinition* lhs, MDefinition* rhs);
+ template<size_t Temps>
+ void lowerForShiftInt64(LInstructionHelper<INT64_PIECES, INT64_PIECES + 1, Temps>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs);
+ void lowerForCompareI64AndBranch(MTest* mir, MCompare* comp, JSOp op,
+ MDefinition* left, MDefinition* right,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse);
+
+ void lowerForFPU(LInstructionHelper<1, 1, 0>* ins, MDefinition* mir,
+ MDefinition* src);
+ template<size_t Temps>
+ void lowerForFPU(LInstructionHelper<1, 2, Temps>* ins, MDefinition* mir,
+ MDefinition* lhs, MDefinition* rhs);
+
+#if 0
+ void lowerForCompIx4(LSimdBinaryCompIx4* ins, MSimdBinaryComp* mir,
+ MDefinition* lhs, MDefinition* rhs)
+ {
+ return lowerForFPU(ins, mir, lhs, rhs);
+ }
+ void lowerForCompFx4(LSimdBinaryCompFx4* ins, MSimdBinaryComp* mir,
+ MDefinition* lhs, MDefinition* rhs)
+ {
+ return lowerForFPU(ins, mir, lhs, rhs);
+ }
+#endif
+
+ void lowerForBitAndAndBranch(LBitAndAndBranch* baab, MInstruction* mir,
+ MDefinition* lhs, MDefinition* rhs);
+ void lowerDivI(MDiv* div);
+ void lowerModI(MMod* mod);
+ void lowerMulI(MMul* mul, MDefinition* lhs, MDefinition* rhs);
+ void lowerPowOfTwoI(MPow* mir);
+ void lowerUDiv(MDiv* div);
+ void lowerUMod(MMod* mod);
+
+ LTableSwitch* newLTableSwitch(const LAllocation& in, const LDefinition& inputCopy,
+ MTableSwitch* ins);
+ LTableSwitchV* newLTableSwitchV(MTableSwitch* ins);
+
+ void lowerPhi(MPhi* phi);
+ void lowerInt64PhiInput(MPhi*, uint32_t, LBlock*, size_t);
+ void defineInt64Phi(MPhi*, size_t);
+
+ // Returns a box allocation. reg2 is ignored since we're 64-bit.
+ LBoxAllocation useBoxFixed(MDefinition* mir, Register reg1, Register reg2,
+ bool useAtStart = false);
+
+ inline LDefinition tempToUnbox() {
+ return temp();
+ }
+
+ void lowerUntypedPhiInput(MPhi* phi, uint32_t inputPosition, LBlock* block, size_t lirIndex);
+ void defineUntypedPhi(MPhi* phi, size_t lirIndex);
+
+ void lowerTruncateDToInt32(MTruncateToInt32* ins);
+ void lowerTruncateFToInt32(MTruncateToInt32* ins);
+
+ void lowerDivI64(MDiv* div);
+ void lowerModI64(MMod* mod);
+ void lowerUDivI64(MDiv* div);
+ void lowerUModI64(MMod* mod);
+ void lowerNegI(MInstruction* ins, MDefinition* input);
+ void lowerNegI64(MInstruction* ins, MDefinition* input);
+
+ void lowerAtomicLoad64(MLoadUnboxedScalar* ins);
+ void lowerAtomicStore64(MStoreUnboxedScalar* ins);
+
+ // BigInt
+ void lowerBigIntDiv(MBigIntDiv *ins);
+ void lowerBigIntMod(MBigIntMod *ins);
+ void lowerBigIntLsh(MBigIntLsh *ins);
+ void lowerBigIntRsh(MBigIntRsh *ins);
+
+ // WASM bits
+ void lowerWasmBuiltinTruncateToInt32(MWasmBuiltinTruncateToInt32 *ins);
+ void lowerWasmBuiltinTruncateToInt64(MWasmBuiltinTruncateToInt64 *ins);
+ void lowerWasmBuiltinDivI64(MWasmBuiltinDivI64 *ins);
+ void lowerWasmBuiltinModI64(MWasmBuiltinModI64 *ins);
+ void lowerWasmSelectI(MWasmSelect* select);
+ void lowerWasmSelectI64(MWasmSelect* select);
+
+ void lowerBuiltinInt64ToFloatingPoint(MBuiltinInt64ToFloatingPoint *ins);
+};
+
+typedef LIRGeneratorPPC64 LIRGeneratorSpecific;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_ppc64le_Lowering_ppc64le_h */
diff -r 485b15bb4a20 -r 9c245f4665be js/src/jit/ppc64/MacroAssembler-ppc64-inl.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/js/src/jit/ppc64/MacroAssembler-ppc64-inl.h Thu Aug 08 21:23:52 2024 -0700
@@ -0,0 +1,2769 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_ppc64le_MacroAssembler_ppc64le_inl_h
+#define jit_ppc64le_MacroAssembler_ppc64le_inl_h
+
+#include "jit/ppc64/MacroAssembler-ppc64.h"
+#include "jit/FlushICache.h"
+
+namespace js {
+namespace jit {
+
+//{{{ check_macroassembler_style
+
+void
+MacroAssembler::move64(Register64 src, Register64 dest)
+{
+ movePtr(src.reg, dest.reg);
+}
+
+void
+MacroAssembler::move64(Imm64 imm, Register64 dest)
+{
+ movePtr(ImmWord(imm.value), dest.reg);
+}
+
+void
+MacroAssembler::moveDoubleToGPR64(FloatRegister src, Register64 dest)
+{
+ moveFromDouble(src, dest.reg);
+}
+
+void
+MacroAssembler::moveGPR64ToDouble(Register64 src, FloatRegister dest)
+{
+ moveToDouble(src.reg, dest);
+}
+
+void
+MacroAssembler::move64To32(Register64 src, Register dest)
+{
+ // Registers are registers, so why should it be
+ // 32 bits are treated differently?
+ // (with apologies to Depeche Mode)
+ as_srawi(dest, src.reg, 0); // clear upper word and sign extend
+}
+
+void
+MacroAssembler::move32To64ZeroExtend(Register src, Register64 dest)
+{
+ as_rldicl(dest.reg, src, 0, 32); // "clrldi"
+}
+
+void MacroAssembler::move8ZeroExtend(Register src, Register dest) {
+ // probably andi. dest, src, 0xff is simplest
+ // used in CodeGenerator
+ MOZ_CRASH("NYI");
+}
+
+void
+MacroAssembler::move8To64SignExtend(Register src, Register64 dest)
+{
+ move32To64SignExtend(src, dest);
+ move8SignExtend(dest.reg, dest.reg);
+}
+
+void
+MacroAssembler::move16To64SignExtend(Register src, Register64 dest)
+{
+ move32To64SignExtend(src, dest);
+ move16SignExtend(dest.reg, dest.reg);
+}
+
+void
+MacroAssembler::move32To64SignExtend(Register src, Register64 dest)
+{
+ as_srawi(dest.reg, src, 0);
+}
+
+void
+MacroAssembler::move32SignExtendToPtr(Register src, Register dest)
+{
+ as_srawi(dest, src, 0);
+}
+
+void
+MacroAssembler::move32ZeroExtendToPtr(Register src, Register dest)
+{
+ as_rldicl(dest, src, 0, 32); // "clrldi"
+}
+
+
+// ===============================================================
+// Logical instructions
+
+void
+MacroAssembler::andPtr(Register src, Register dest)
+{
+ ma_and(dest, src);
+}
+
+void
+MacroAssembler::andPtr(Imm32 imm, Register dest)
+{
+ ma_and(dest, imm);
+}
+
+void
+MacroAssembler::and64(Imm64 imm, Register64 dest)
+{
+ MOZ_ASSERT(dest.reg != ScratchRegister);
+ ma_li(ScratchRegister, ImmWord(imm.value));
+ ma_and(dest.reg, ScratchRegister);
+}
+
+void
+MacroAssembler::and64(Register64 src, Register64 dest)
+{
+ ma_and(dest.reg, src.reg);
+}
+
+void
+MacroAssembler::and64(const Operand& src, Register64 dest)
+{
+ if (src.getTag() == Operand::MEM) {
+ MOZ_ASSERT(dest.reg != ScratchRegister);
+ Register64 scratch(ScratchRegister);
+
+ load64(src.toAddress(), scratch);
+ and64(scratch, dest);
+ } else {
+ and64(Register64(src.toReg()), dest);
+ }
+}
+
+void
+MacroAssembler::or64(Imm64 imm, Register64 dest)
+{
+ MOZ_ASSERT(dest.reg != ScratchRegister);
+ ma_li(ScratchRegister, ImmWord(imm.value));
+ ma_or(dest.reg, ScratchRegister);
+}
+
+void
+MacroAssembler::xor64(Imm64 imm, Register64 dest)
+{
+ MOZ_ASSERT(dest.reg != ScratchRegister);
+ ma_li(ScratchRegister, ImmWord(imm.value));
+ ma_xor(dest.reg, ScratchRegister);
+}
+
+void
+MacroAssembler::orPtr(Register src, Register dest)
+{
+ ma_or(dest, src);
+}
+
+void
+MacroAssembler::orPtr(Imm32 imm, Register dest)
+{
+ ma_or(dest, imm);
+}
+
+void
+MacroAssembler::or64(Register64 src, Register64 dest)
+{
+ ma_or(dest.reg, src.reg);
+}
+
+void
+MacroAssembler::or64(const Operand& src, Register64 dest)
+{
+ if (src.getTag() == Operand::MEM) {
+ Register64 scratch(ScratchRegister);
+
+ load64(src.toAddress(), scratch);
+ or64(scratch, dest);
+ } else {
+ or64(Register64(src.toReg()), dest);
+ }
+}
+
+void
+MacroAssembler::xor64(Register64 src, Register64 dest)
+{
+ ma_xor(dest.reg, src.reg);
+}
+
+void
+MacroAssembler::xor64(const Operand& src, Register64 dest)
+{
+ if (src.getTag() == Operand::MEM) {
+ MOZ_ASSERT(dest.reg != ScratchRegister);
+ Register64 scratch(ScratchRegister);
+
+ load64(src.toAddress(), scratch);
+ xor64(scratch, dest);
+ } else {
+ xor64(Register64(src.toReg()), dest);
+ }
+}
+
+void
+MacroAssembler::xorPtr(Register src, Register dest)
+{
+ ma_xor(dest, src);
+}
+
+void
+MacroAssembler::xorPtr(Imm32 imm, Register dest)
+{
+ ma_xor(dest, imm);
+}
+
+// ===============================================================
+// Arithmetic functions
+
+void
+MacroAssembler::addPtr(Register src, Register dest)
+{
+ ma_add(dest, src);
+}
+
+void
+MacroAssembler::addPtr(Imm32 imm, Register dest)
+{
+ ma_add(dest, imm);
+}
+
+void
+MacroAssembler::addPtr(ImmWord imm, Register dest)
+{
+ movePtr(imm, ScratchRegister);
+ addPtr(ScratchRegister, dest);
+}
+
+void
+MacroAssembler::add64(Register64 src, Register64 dest)
+{
+ addPtr(src.reg, dest.reg);
+}
+
+void
+MacroAssembler::add64(const Operand& src, Register64 dest)
+{
+ if (src.getTag() == Operand::MEM) {
+ Register64 scratch(ScratchRegister);
+
+ load64(src.toAddress(), scratch);
+ add64(scratch, dest);
+ } else {
+ add64(Register64(src.toReg()), dest);
+ }
+}
+
+void
+MacroAssembler::add64(Imm32 imm, Register64 dest)
+{
+ ma_add(dest.reg, imm);
+}
+
+void
+MacroAssembler::add64(Imm64 imm, Register64 dest)
+{
+ MOZ_ASSERT(dest.reg != ScratchRegister);
+ mov(ImmWord(imm.value), ScratchRegister);
+ ma_add(dest.reg, ScratchRegister);
+}
+
+void
+MacroAssembler::notPtr(Register reg)
+{
+ as_nor(reg, reg, reg);
+}
+
+CodeOffset
+MacroAssembler::sub32FromStackPtrWithPatch(Register dest)
+{
+ CodeOffset offset = CodeOffset(currentOffset());
+ MacroAssemblerPPC64::ma_liPatchable(dest, Imm32(0));
+ as_subf(dest, dest, StackPointer); // T = B - A
+ return offset;
+}
+
+void
+MacroAssembler::patchSub32FromStackPtr(CodeOffset offset, Imm32 imm)
+{
+ Instruction* lis = (Instruction*) m_buffer.getInst(BufferOffset(offset.offset()));
+ MacroAssemblerPPC64::UpdateLisOriValue(lis, lis->next(), imm.value);
+ FlushICache(lis, 2 * sizeof(uint32_t));
+}
+
+void
+MacroAssembler::subPtr(Register src, Register dest)
+{
+ as_subf(dest, src, dest); // T = B - A
+}
+
+void
+MacroAssembler::subPtr(Imm32 imm, Register dest)
+{
+ ma_dsubu(dest, dest, imm); // inverted at MacroAssembler level
+}
+
+void
+MacroAssembler::sub64(Register64 src, Register64 dest)
+{
+ as_subf(dest.reg, src.reg, dest.reg);
+}
+
+void
+MacroAssembler::sub64(const Operand& src, Register64 dest)
+{
+ if (src.getTag() == Operand::MEM) {
+ Register64 scratch(ScratchRegister);
+
+ load64(src.toAddress(), scratch);
+ sub64(scratch, dest);
+ } else {
+ sub64(Register64(src.toReg()), dest);
+ }
+}
+
+void
+MacroAssembler::sub64(Imm64 imm, Register64 dest)
+{
+ MOZ_ASSERT(dest.reg != ScratchRegister);
+ mov(ImmWord(imm.value), ScratchRegister);
+ as_subf(dest.reg, ScratchRegister, dest.reg); // T = B - A
+}
+
+void
+MacroAssembler::mul64(Imm64 imm, const Register64& dest)
+{
+ MOZ_ASSERT(dest.reg != ScratchRegister);
+ mov(ImmWord(imm.value), ScratchRegister);
+ as_mulld(dest.reg, ScratchRegister, dest.reg); // low order word
+}
+
+void
+MacroAssembler::mul64(Imm64 imm, const Register64& dest, const Register temp)
+{
+ MOZ_ASSERT(temp == InvalidReg);
+ mul64(imm, dest);
+}
+
+void
+MacroAssembler::mul64(const Register64& src, const Register64& dest, const Register temp)
+{
+ MOZ_ASSERT(temp == InvalidReg);
+ as_mulld(dest.reg, src.reg, dest.reg); // low order word
+}
+
+void
+MacroAssembler::mul64(const Operand& src, const Register64& dest, const Register temp)
+{
+ if (src.getTag() == Operand::MEM) {
+ Register64 scratch(ScratchRegister);
+
+ load64(src.toAddress(), scratch);
+ mul64(scratch, dest, temp);
+ } else {
+ mul64(Register64(src.toReg()), dest, temp);
+ }
+}
+
+void MacroAssembler::mulPtr(Register rhs, Register srcDest) {
+ as_mulld(srcDest, srcDest, rhs); // low order word
+}
+
+void
+MacroAssembler::mulBy3(Register src, Register dest)
+{
+ // I guess this *is* better than mulli.
+ MOZ_ASSERT(src != ScratchRegister);
+ as_add(ScratchRegister, src, src);
+ as_add(dest, ScratchRegister, src);
+}
+
+// This is used in MacroAssembler::loadInt32ToStringWithBase. Instead of
+// letting us use our superior arithmetic instructions, the JIT has reduced
+// us to faffing around with magic constants because that's what x86* does.
+// This leads to sign extension hazards.
+void MacroAssembler::mulHighUnsigned32(Imm32 imm, Register src, Register dest) {
+ MOZ_ASSERT(src != ScratchRegister);
+ // Compensate for (likely) sign extension by always clearing upper bits.
+ move32(imm, ScratchRegister);
+ as_rldicl(ScratchRegister, ScratchRegister, 0, 32); // "clrldi"
+ // loadInt32ToStringWithBase expects what is effectively unsigned multiply.
+ as_mulhwu(dest, ScratchRegister, src);
+ // Clear upper bits again, as they are undefined by the spec.
+ as_rldicl(dest, dest, 0, 32); // "clrldi"
+}
+
+void
+MacroAssembler::inc64(AbsoluteAddress dest)
+{
+ ma_li(SecondScratchReg, ImmWord(uintptr_t(dest.addr)));
+ as_ld(ThirdScratchReg, SecondScratchReg, 0);
+ as_addi(ScratchRegister, ThirdScratchReg, 1);
+ as_std(ScratchRegister, SecondScratchReg, 0);
+}
+
+void
+MacroAssembler::negPtr(Register reg)
+{
+ as_neg(reg, reg);
+}
+
+void
+MacroAssembler::neg64(Register64 reg)
+{
+ negPtr(reg.reg);
+}
+
+void
+MacroAssembler::quotient32(Register rhs, Register srcDest, bool isUnsigned)
+{
+ if (isUnsigned) {
+ as_divwu(srcDest, srcDest, rhs);
+ } else {
+ as_divw(srcDest, srcDest, rhs);
+ }
+}
+
+// byte swaps
+void
+MacroAssembler::byteSwap16SignExtend(Register reg)
+{
+ byteSwap16ZeroExtend(reg);
+ as_extsh(reg, reg);
+}
+
+void
+MacroAssembler::byteSwap16ZeroExtend(Register reg)
+{
+ MOZ_ASSERT(reg != ScratchRegister);
+ xs_mr(ScratchRegister, reg);
+
+ as_rlwinm(reg, ScratchRegister, 8, 16, 23);
+ as_rlwimi(reg, ScratchRegister, 24, 24, 31);
+}
+
+void
+MacroAssembler::byteSwap32(Register reg)
+{
+ // cribbed off gcc __builtin_swap32
+ // POWER10 has/will have brw, but POWER10 is naughty and has non-free
+ // RAM firmware. Naughty OMI! Spank spank!
+ MOZ_ASSERT(reg != ScratchRegister);
+ as_rlwinm(ScratchRegister, reg, 8, 0, 31); // "rotlwi"
+ as_rlwimi(ScratchRegister, reg, 24, 0, 7);
+ as_rlwimi(ScratchRegister, reg, 24, 16, 23);
+ xs_mr(reg, ScratchRegister);
+}
+
+void
+MacroAssembler::byteSwap64(Register64 reg)
+{
+ Register r = reg.reg;
+ // Use VSX for this because the 64-bit swap with rld* is just hideous.
+ // POWER10 has/will have brd, but, more electric spanking of war babies!
+ if (HasPPCISA3()) {
+ as_mtvsrd(ScratchDoubleReg, r);
+ as_xxbrd(ScratchDoubleReg, ScratchDoubleReg);
+ as_mfvsrd(r, ScratchDoubleReg);
+ } else {
+ MOZ_CRASH("NYI for pre-POWER9");
+ }
+}
+
+// ===============================================================
+// Shift functions
+
+void
+MacroAssembler::lshiftPtr(Imm32 imm, Register dest)
+{
+ MOZ_ASSERT(imm.value >= 0);
+
+ if (imm.value == 0) {
+ // No-op
+ } else {
+ as_rldicr(dest, dest, imm.value % 64, 63 - (imm.value % 64)); // "sldi"
+ }
+}
+
+void
+MacroAssembler::lshiftPtr(Register shift, Register dest)
+{
+ // sld will zero out any shift amount greater than 64, but JavaScript
+ // expects this to act like a modulo, so ...
+ MOZ_ASSERT(shift != ScratchRegister);
+ MOZ_ASSERT(dest != ScratchRegister);
+
+ as_andi_rc(ScratchRegister, shift, 63);
+ as_sld(dest, dest, ScratchRegister);
+}
+
+void
+MacroAssembler::lshift64(Imm32 imm, Register64 dest)
+{
+ lshiftPtr(imm, dest.reg);
+}
+
+void
+MacroAssembler::lshift64(Register shift, Register64 dest)
+{
+ lshiftPtr(shift, dest.reg);
+}
+
+void
+MacroAssembler::rshiftPtr(Imm32 imm, Register dest)
+{
+ MOZ_ASSERT(imm.value >= 0);
+
+ // Same deal as rshift32, same twist.
+ if (!(imm.value % 64)) {
+ if (imm.value == 0) {
+ // No-op
+ } else {
+ // 64 bits right-shifted 64 bits is zero.
+ xs_li(dest, 0);
+ }
+ } else {
+ x_srdi(dest, dest, imm.value % 64);
+ }
+}
+
+void
+MacroAssembler::rshiftPtr(Register shift, Register dest)
+{
+ // JavaScript expects the shift to act like a modulo.
+ MOZ_ASSERT(shift != ScratchRegister);
+ as_andi_rc(ScratchRegister, shift, 63);
+ as_srd(dest, dest, ScratchRegister);
+}
+
+void
+MacroAssembler::rshift64(Imm32 imm, Register64 dest)
+{
+ rshiftPtr(imm, dest.reg);
+}
+
+void
+MacroAssembler::rshift64(Register shift, Register64 dest)
+{
+ rshiftPtr(shift, dest.reg);
+}
+
+void
+MacroAssembler::rshiftPtrArithmetic(Imm32 imm, Register dest)
+{
+ MOZ_ASSERT(0 <= imm.value);
+ as_sradi(dest, dest, imm.value % 64);
+}
+
+void
+MacroAssembler::rshift64Arithmetic(Imm32 imm, Register64 dest)
+{
+ rshiftPtrArithmetic(imm, dest.reg);
+}
+
+void
+MacroAssembler::rshift64Arithmetic(Register shift, Register64 dest)
+{
+ // JavaScript expects the shift to act like a modulo.
+ MOZ_ASSERT(shift != ScratchRegister);
+ as_andi_rc(ScratchRegister, shift, 63);
+ as_srad(dest.reg, dest.reg, ScratchRegister);
+}
+
+// ===============================================================
+// Rotation functions
+
+void
+MacroAssembler::rotateLeft64(Imm32 count, Register64 src, Register64 dest, Register temp)
+{
+ MOZ_ASSERT(temp == InvalidReg);
+ MOZ_ASSERT(count.value >= 0);
+
+ if (!(count.value % 64)) {
+ if (src.reg != dest.reg)
+ xs_mr(dest.reg, src.reg);
+ } else {
+ as_rldicl(dest.reg, src.reg, (count.value % 64), 0); // "rotldi"
+ }
+}
+
+void
+MacroAssembler::rotateLeft64(Register count, Register64 src, Register64 dest, Register temp)
+{
+ MOZ_ASSERT(temp == InvalidReg);
+ as_rldcl(dest.reg, src.reg, count, 0); // "rotld"
+}
+
+void
+MacroAssembler::rotateRight64(Imm32 count, Register64 src, Register64 dest, Register temp)
+{
+ MOZ_ASSERT(temp == InvalidReg);
+ MOZ_ASSERT(count.value >= 0);
+
+ if (!(count.value % 64)) {
+ if (src.reg != dest.reg)
+ xs_mr(dest.reg, src.reg);
+ } else {
+ as_rldicl(dest.reg, src.reg, 64 - (count.value % 64), 0); // "rotrdi"
+ }
+}
+
+void
+MacroAssembler::rotateRight64(Register count, Register64 src, Register64 dest, Register temp)
+{
+ MOZ_ASSERT(temp == InvalidReg);
+ MOZ_ASSERT(count != ScratchRegister);
+ MOZ_ASSERT(count != SecondScratchReg);
+ MOZ_ASSERT(src.reg != ScratchRegister);
+ MOZ_ASSERT(src.reg != SecondScratchReg);
+
+ xs_li(ScratchRegister, 64);
+ as_andi_rc(SecondScratchReg, count, 63);
+ as_subf(ScratchRegister, SecondScratchReg, ScratchRegister); // T = B - A
+ as_rldcl(dest.reg, src.reg, ScratchRegister, 0); // "rotrd"
+}
+
+// ===============================================================
+// Condition functions
+
+template <typename T1, typename T2>
+void
+MacroAssembler::cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest)
+{
+ ma_cmp_set(dest, lhs, rhs, cond, /* useCmpw */ false);
+}
+
+// Also see below for specializations of cmpPtrSet.
+
+template <typename T1, typename T2>
+void
+MacroAssembler::cmp32Set(Condition cond, T1 lhs, T2 rhs, Register dest)
+{
+ ma_cmp_set(dest, lhs, rhs, cond, /* useCmpw */ true);
+}
+
+void
+MacroAssembler::cmp64Set(Condition cond, Address lhs, Imm64 rhs, Register dest)
+{
+ ma_cmp_set(dest, lhs, rhs, cond, /* useCmpw */ false);
+}
+
+template <typename T>
+void
+MacroAssembler::testStringSet(Condition cond, const T& value, Register dest)
+{
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ splitTag(value, SecondScratchReg);
+ ma_cmp_set(dest, SecondScratchReg, ImmTag(JSVAL_TAG_STRING), cond);
+}
+
+template <typename T>
+void
+MacroAssembler::testBooleanSet(Condition cond, const T& value, Register dest)
+{
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ splitTag(value, SecondScratchReg);
+ ma_cmp_set(dest, SecondScratchReg, ImmTag(JSVAL_TAG_BOOLEAN), cond);
+}
+
+template <typename T>
+void
+MacroAssembler::testSymbolSet(Condition cond, const T& value, Register dest)
+{
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ splitTag(value, SecondScratchReg);
+ ma_cmp_set(dest, SecondScratchReg, ImmTag(JSVAL_TAG_SYMBOL), cond);
+}
+
+template <typename T>
+void
+MacroAssembler::testBigIntSet(Condition cond, const T& value, Register dest)
+{
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ splitTag(value, SecondScratchReg);
+ ma_cmp_set(dest, SecondScratchReg, ImmTag(JSVAL_TAG_BIGINT), cond);
+}
+
+template <typename T>
+void
+MacroAssembler::testNumberSet(Condition cond, const T& value, Register dest)
+{
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ splitTag(value, SecondScratchReg);
+ ma_cmp_set(dest, SecondScratchReg,
+ ImmTag(JS::detail::ValueUpperInclNumberTag),
+ cond == Equal ? BelowOrEqual : Above);
+}
+
+
+// ===============================================================
+// Bit counting functions
+
+void
+MacroAssembler::clz64(Register64 src, Register dest)
+{
+ as_cntlzd(dest, src.reg);
+}
+
+void
+MacroAssembler::ctz64(Register64 src, Register dest)
+{
+ if (HasPPCISA3()) {
+ as_cnttzd(dest, src.reg);
+ } else {
+ MOZ_CRASH("NYI for pre-POWER9");
+ }
+}
+
+void
+MacroAssembler::popcnt64(Register64 input, Register64 output, Register tmp)
+{
+#if (1)
+ as_popcntd(output.reg, input.reg);
+#else
+ // Prior to ISA 2.06. Not tested.
+ ma_move(output.reg, input.reg);
+ as_sradi(tmp, input.reg, Imm32(1));
+ ma_li(ScratchRegister, ImmWord(0x5555555555555555UL));
+ ma_and(tmp, ScratchRegister);
+ ma_dsubu(output.reg, tmp);
+ as_sradi(tmp, output.reg, Imm32(2));
+ ma_li(ScratchRegister, ImmWord(0x3333333333333333UL));
+ ma_and(output.reg, ScratchRegister);
+ ma_and(tmp, ScratchRegister);
+ ma_add(output.reg, tmp);
+ ma_dsrl(tmp, output.reg, Imm32(4));
+ ma_add(output.reg, tmp);
+ ma_li(ScratchRegister, ImmWord(0xF0F0F0F0F0F0F0FUL));
+ ma_and(output.reg, ScratchRegister);
+ ma_dsll(tmp, output.reg, Imm32(8));
+ ma_add(output.reg, tmp);
+ ma_dsll(tmp, output.reg, Imm32(16));
+ ma_add(output.reg, tmp);
+ ma_dsll(tmp, output.reg, Imm32(32));
+ ma_add(output.reg, tmp);
+ as_sradi(output.reg, output.reg, Imm32(56));
+#endif
+}
+
+// ===============================================================
+// Branch functions
+
+void
+MacroAssembler::branch64(Condition cond, Register64 lhs, Imm64 val, Label* success, Label* fail)
+{
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal ||
+ cond == Assembler::LessThan || cond == Assembler::LessThanOrEqual ||
+ cond == Assembler::GreaterThan || cond == Assembler::GreaterThanOrEqual ||
+ cond == Assembler::Below || cond == Assembler::BelowOrEqual ||
+ cond == Assembler::Above || cond == Assembler::AboveOrEqual,
+ "other condition codes not supported");
+
+ branchPtr(cond, lhs.reg, ImmWord(val.value), success);
+ if (fail)
+ jump(fail);
+}
+
+void
+MacroAssembler::branch64(Condition cond, Register64 lhs, Register64 rhs, Label* success, Label* fail)
+{
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal ||
+ cond == Assembler::LessThan || cond == Assembler::LessThanOrEqual ||
+ cond == Assembler::GreaterThan || cond == Assembler::GreaterThanOrEqual ||
+ cond == Assembler::Below || cond == Assembler::BelowOrEqual ||
+ cond == Assembler::Above || cond == Assembler::AboveOrEqual,
+ "other condition codes not supported");
+
+ branchPtr(cond, lhs.reg, rhs.reg, success);
+ if (fail)
+ jump(fail);
+}
+
+void
+MacroAssembler::branch64(Condition cond, const Address& lhs, Imm64 val, Label* label)
+{
+ MOZ_ASSERT(cond == Assembler::NotEqual,
+ "other condition codes not supported");
+
+ branchPtr(cond, lhs, ImmWord(val.value), label);
+}
+
+void
+MacroAssembler::branch64(Condition cond, const Address& lhs, Register64 reg, Label* label)
+{
+ MOZ_ASSERT(reg.reg != SecondScratchReg);
+ MOZ_ASSERT(reg.reg != ScratchRegister);
+
+ loadPtr(lhs, ScratchRegister);
+ branchPtr(cond, ScratchRegister, reg.reg, label);
+}
+
+void
+MacroAssembler::branch64(Condition cond, const Address& lhs, const Address& rhs, Register scratch,
+ Label* label)
+{
+ MOZ_ASSERT(cond == Assembler::NotEqual,
+ "other condition codes not supported");
+ MOZ_ASSERT(lhs.base != scratch);
+ MOZ_ASSERT(rhs.base != scratch);
+
+ loadPtr(rhs, scratch);
+ branchPtr(cond, lhs, scratch, label);
+}
+
+void MacroAssembler::branchNeg32(Condition cond, Register reg, Label* label) {
+ MOZ_ASSERT(cond == Overflow);
+ ma_negTestOverflow(reg, label);
+}
+
+void
+MacroAssembler::branchPrivatePtr(Condition cond, const Address& lhs, Register rhs, Label* label)
+{
+ if (rhs != ScratchRegister)
+ movePtr(rhs, ScratchRegister);
+ // Instead of unboxing lhs, box rhs and do direct comparison with lhs.
+ rshiftPtr(Imm32(1), ScratchRegister);
+ branchPtr(cond, lhs, ScratchRegister, label);
+}
+
+template <class L>
+void
+MacroAssembler::branchTest64(Condition cond, Register64 lhs, Register64 rhs, Register temp,
+ L label)
+{
+ branchTestPtr(cond, lhs.reg, rhs.reg, label);
+}
+
+void
+MacroAssembler::branchTestUndefined(Condition cond, const ValueOperand& value, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestUndefined(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestInt32(Condition cond, const ValueOperand& value, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestInt32(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestInt32Truthy(bool b, const ValueOperand& value, Label* label)
+{
+ ScratchRegisterScope scratch(*this);
+ unboxBoolean(value, scratch);
+ ma_bc(scratch, scratch, label, b ? NonZero : Zero);
+}
+
+void
+MacroAssembler::branchTestDouble(Condition cond, Register tag, Label* label)
+{
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ Condition actual = (cond == Equal) ? BelowOrEqual : Above;
+ ma_bc(tag, ImmTag(JSVAL_TAG_MAX_DOUBLE), label, actual);
+}
+
+void
+MacroAssembler::branchTestDouble(Condition cond, const ValueOperand& value, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestDouble(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestNumber(Condition cond, const ValueOperand& value, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestNumber(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestBoolean(Condition cond, const ValueOperand& value, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestBoolean(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestBooleanTruthy(bool b, const ValueOperand& value, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ unboxBoolean(value, scratch2);
+ ma_bc(scratch2, scratch2, label, b ? NonZero : Zero);
+}
+
+void
+MacroAssembler::branchTestString(Condition cond, const ValueOperand& value, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestString(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestStringTruthy(bool b, const ValueOperand& value, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ unboxString(value, scratch2);
+ ma_load(scratch2, Address(scratch2, JSString::offsetOfLength()), SizeWord, ZeroExtend);
+ ma_bc(scratch2, Imm32(0), label, b ? NotEqual : Equal);
+}
+
+void
+MacroAssembler::branchTestSymbol(Condition cond, const ValueOperand& value, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestSymbol(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestNull(Condition cond, const ValueOperand& value, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestNull(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestObject(Condition cond, const ValueOperand& value, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestObject(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestPrimitive(Condition cond, const ValueOperand& value, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestPrimitive(cond, scratch2, label);
+}
+
+template <class L>
+void
+MacroAssembler::branchTestMagic(Condition cond, const ValueOperand& value, L label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ ma_bc(scratch2, ImmTag(JSVAL_TAG_MAGIC), label, cond);
+}
+
+void
+MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr, JSWhyMagic why, Label* label)
+{
+ uint64_t magic = MagicValue(why).asRawBits();
+ SecondScratchRegisterScope scratch(*this);
+ loadPtr(valaddr, scratch);
+ ma_bc(scratch, ImmWord(magic), label, cond);
+}
+
+void
+MacroAssembler::branchTestBigInt(Condition cond, const BaseIndex& address,
+ Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ computeEffectiveAddress(address, scratch2);
+ splitTag(scratch2, scratch2);
+ branchTestBigInt(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestBigInt(Condition cond, const ValueOperand& value,
+ Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestBigInt(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestBigInt(Condition cond, Register tag,
+ Label* label)
+{
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_bc(tag, ImmTag(JSVAL_TAG_BIGINT), label, cond);
+}
+
+void
+MacroAssembler::branchTestBigIntTruthy(bool b, const ValueOperand& value,
+ Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ unboxBigInt(value, scratch2);
+ ma_load(scratch2, Address(scratch2, BigInt::offsetOfDigitLength()), SizeWord, ZeroExtend);
+ ma_bc(scratch2, Imm32(0), label, b ? NotEqual : Equal);
+}
+
+void
+MacroAssembler::branchTruncateDoubleMaybeModUint32(FloatRegister src, Register dest, Label* fail)
+{
+ MOZ_ASSERT(src != ScratchDoubleReg);
+
+ // If the conversion is invalid (see Power ISA v3.1, page 173), fail.
+ as_mtfsb0(23); // whack VXCVI
+ as_fctiwz(ScratchDoubleReg, src);
+ as_mcrfs(cr0, 5); // reserved - VXSOFT - VXSQRT - VXCVI
+ moveFromDouble(ScratchDoubleReg, dest);
+ as_srawi(dest, dest, 0); // clear upper word and sign extend
+ ma_bc(Assembler::SOBit, fail);
+}
+
+void
+MacroAssembler::branchTruncateFloat32MaybeModUint32(FloatRegister src, Register dest, Label* fail)
+{
+ branchTruncateDoubleMaybeModUint32(src, dest, fail);
+}
+
+//}}} check_macroassembler_style
+// ===============================================================
+
+// The specializations for cmpPtrSet are outside the braces because check_macroassembler_style can't yet
+// deal with specializations.
+
+template<>
+inline void
+MacroAssembler::cmpPtrSet(Assembler::Condition cond, Address lhs, ImmPtr rhs,
+ Register dest)
+{
+ loadPtr(lhs, SecondScratchReg);
+ cmpPtrSet(cond, SecondScratchReg, rhs, dest);
+}
+
+template<>
+inline void
+MacroAssembler::cmpPtrSet(Assembler::Condition cond, Register lhs, Address rhs,
+ Register dest)
+{
+ loadPtr(rhs, ScratchRegister);
+ cmpPtrSet(cond, lhs, ScratchRegister, dest);
+}
+
+template<>
+inline void
+MacroAssembler::cmp32Set(Assembler::Condition cond, Register lhs, Address rhs,
+ Register dest)
+{
+ load32(rhs, ScratchRegister);
+ cmp32Set(cond, lhs, ScratchRegister, dest);
+}
+
+void
+MacroAssemblerPPC64Compat::incrementInt32Value(const Address& addr)
+{
+ asMasm().add32(Imm32(1), addr);
+}
+
+/*
+void
+MacroAssemblerPPC64Compat::computeEffectiveAddress(const BaseIndex& address, Register dest)
+{
+ computeScaledAddress(address, dest);
+ if (address.offset)
+ asMasm().addPtr(Imm32(address.offset), dest);
+}
+*/
+
+void
+MacroAssemblerPPC64Compat::retn(Imm32 n)
+{
+ // pc <- [sp]; sp += n
+ loadPtr(Address(StackPointer, 0), ScratchRegister);
+ asMasm().addPtr(n, StackPointer);
+ xs_mtlr(ScratchRegister);
+ as_blr();
+}
+
+void MacroAssembler::load32SignExtendToPtr(const Address& src, Register dest)
+{
+ load32(src, dest);
+}
+
+void
+MacroAssembler::moveFloat32ToGPR(FloatRegister src, Register dest)
+{
+ moveFromFloat32(src, dest);
+}
+
+void
+MacroAssembler::moveGPRToFloat32(Register src, FloatRegister dest)
+{
+ moveToFloat32(src, dest);
+}
+
+void
+MacroAssembler::move8SignExtend(Register src, Register dest)
+{
+ as_extsb(dest, src);
+}
+
+void
+MacroAssembler::move16SignExtend(Register src, Register dest)
+{
+ as_extsh(dest, src);
+}
+
+void
+MacroAssembler::loadAbiReturnAddress(Register dest)
+{
+ xs_mflr(dest);
+}
+
+// ===============================================================
+// Logical instructions
+
+void
+MacroAssembler::not32(Register reg)
+{
+ as_nor(reg, reg, reg);
+}
+
+void
+MacroAssembler::and32(Register src, Register dest)
+{
+ as_and(dest, dest, src);
+}
+
+void
+MacroAssembler::and32(Imm32 imm, Register dest)
+{
+ ma_and(dest, imm);
+}
+
+void
+MacroAssembler::and32(Imm32 imm, const Address& dest)
+{
+ ma_load(SecondScratchReg, dest, SizeWord, ZeroExtend);
+ ma_and(SecondScratchReg, imm);
+ store32(SecondScratchReg, dest);
+}
+
+void
+MacroAssembler::and32(const Address& src, Register dest)
+{
+ ma_load(SecondScratchReg, src, SizeWord, ZeroExtend);
+ ma_and(dest, SecondScratchReg);
+}
+
+void
+MacroAssembler::or32(Register src, Register dest)
+{
+ ma_or(dest, src);
+}
+
+void
+MacroAssembler::or32(Imm32 imm, Register dest)
+{
+ ma_or(dest, imm);
+}
+
+void
+MacroAssembler::or32(Imm32 imm, const Address& dest)
+{
+ ma_load(SecondScratchReg, dest, SizeWord, ZeroExtend);
+ ma_or(SecondScratchReg, imm);
+ store32(SecondScratchReg, dest);
+}
+
+void
+MacroAssembler::xor32(Register src, Register dest)
+{
+ ma_xor(dest, src);
+}
+
+void
+MacroAssembler::xor32(Imm32 imm, Register dest)
+{
+ ma_xor(dest, imm);
+}
+
+void
+MacroAssembler::xor32(Imm32 imm, const Address &dest)
+{
+ ma_load(SecondScratchReg, dest, SizeWord, ZeroExtend);
+ ma_xor(SecondScratchReg, imm);
+ store32(SecondScratchReg, dest);
+}
+
+void
+MacroAssembler::xor32(const Address& src, Register dest)
+{
+ ma_load(SecondScratchReg, src, SizeWord, ZeroExtend);
+ as_xor(dest, dest, SecondScratchReg);
+}
+
+// ===============================================================
+// Arithmetic instructions
+
+void
+MacroAssembler::add32(Register src, Register dest)
+{
+ as_add(dest, dest, src);
+}
+
+void
+MacroAssembler::add32(Imm32 imm, Register dest)
+{
+ ma_add(dest, dest, imm);
+}
+
+void
+MacroAssembler::add32(Imm32 imm, const Address& dest)
+{
+ load32(dest, SecondScratchReg);
+ ma_add(SecondScratchReg, imm);
+ store32(SecondScratchReg, dest);
+}
+
+void MacroAssembler::add32(Imm32 imm, Register src, Register dest) {
+ ma_add(dest, src, imm);
+}
+
+void
+MacroAssembler::addPtr(Imm32 imm, const Address& dest)
+{
+ MOZ_ASSERT(dest.base != SecondScratchReg);
+ loadPtr(dest, SecondScratchReg);
+ addPtr(imm, SecondScratchReg);
+ storePtr(SecondScratchReg, dest);
+}
+
+void
+MacroAssembler::addPtr(const Address& src, Register dest)
+{
+ loadPtr(src, ScratchRegister);
+ addPtr(ScratchRegister, dest);
+}
+
+void
+MacroAssembler::addDouble(FloatRegister src, FloatRegister dest)
+{
+ as_fadd(dest, dest, src);
+}
+
+void
+MacroAssembler::addFloat32(FloatRegister src, FloatRegister dest)
+{
+ as_fadds(dest, dest, src);
+}
+
+void
+MacroAssembler::sub32(Register src, Register dest)
+{
+ as_subf(dest, src, dest); // T = B - A
+}
+
+void
+MacroAssembler::sub32(Imm32 imm, Register dest)
+{
+ ma_subu(dest, dest, imm); // switched at MA level
+}
+
+void
+MacroAssembler::sub32(const Address& src, Register dest)
+{
+ load32(src, SecondScratchReg);
+ as_subf(dest, SecondScratchReg, dest); // T = B - A
+}
+
+void
+MacroAssembler::subPtr(Register src, const Address& dest)
+{
+ loadPtr(dest, SecondScratchReg);
+ subPtr(src, SecondScratchReg);
+ storePtr(SecondScratchReg, dest);
+}
+
+void
+MacroAssembler::subPtr(const Address& addr, Register dest)
+{
+ loadPtr(addr, SecondScratchReg);
+ subPtr(SecondScratchReg, dest);
+}
+
+void
+MacroAssembler::subDouble(FloatRegister src, FloatRegister dest)
+{
+ as_fsub(dest, dest, src); // T = A - B
+}
+
+void
+MacroAssembler::subFloat32(FloatRegister src, FloatRegister dest)
+{
+ as_fsubs(dest, dest, src); // T = A - B
+}
+
+void
+MacroAssembler::mul32(Imm32 rhs, Register srcDest)
+{
+ if (Imm16::IsInSignedRange(rhs.value)) {
+ // See also ::mulBy3
+ xs_sr_mulli(srcDest, srcDest, (int16_t)rhs.value);
+ } else {
+ // Wouldn't be any of our strength-reduced shortcuts anyway.
+ MOZ_ASSERT(srcDest != ScratchRegister);
+ ma_li(ScratchRegister, rhs);
+ as_mullw(srcDest, srcDest, ScratchRegister);
+ }
+ // Clear the upper 32 bits. We wouldn't use this for an intermediate
+ // multiply anyway. Do not sign extend.
+ as_rldicl(srcDest, srcDest, 0, 32); // "clrldi"
+}
+
+void
+MacroAssembler::mul32(Register rhs, Register srcDest)
+{
+ as_mullw(srcDest, srcDest, rhs);
+ // Clear the upper 32 bits. Do not sign extend.
+ as_rldicl(srcDest, srcDest, 0, 32); // "clrldi"
+}
+
+void
+MacroAssembler::mulFloat32(FloatRegister src, FloatRegister dest)
+{
+ as_fmuls(dest, dest, src);
+}
+
+void
+MacroAssembler::mulDouble(FloatRegister src, FloatRegister dest)
+{
+ as_fmul(dest, dest, src);
+}
+
+void
+MacroAssembler::mulDoublePtr(ImmPtr imm, Register temp, FloatRegister dest)
+{
+ movePtr(imm, SecondScratchReg);
+ loadDouble(Address(SecondScratchReg, 0), ScratchDoubleReg);
+ mulDouble(ScratchDoubleReg, dest);
+}
+
+void
+MacroAssembler::remainder32(Register rhs, Register srcDest, bool isUnsigned)
+{
+ if (HasPPCISA3()) {
+ if (isUnsigned) {
+ as_moduw(srcDest, srcDest, rhs);
+ } else {
+ as_modsw(srcDest, srcDest, rhs);
+ }
+ return;
+ }
+
+ if (isUnsigned)
+ as_divwu(ScratchRegister, srcDest, rhs);
+ else
+ as_divw(ScratchRegister, srcDest, rhs);
+ // Recover remainder.
+ as_mullw(SecondScratchReg, ScratchRegister, rhs);
+ as_subf(srcDest, SecondScratchReg, srcDest); // T = B - A
+}
+
+void
+MacroAssembler::divFloat32(FloatRegister src, FloatRegister dest)
+{
+ as_fdivs(dest, dest, src);
+}
+
+void
+MacroAssembler::divDouble(FloatRegister src, FloatRegister dest)
+{
+ as_fdiv(dest, dest, src);
+}
+
+void
+MacroAssembler::neg32(Register reg)
+{
+ as_neg(reg, reg);
+}
+
+void
+MacroAssembler::negateDouble(FloatRegister reg)
+{
+ as_fneg(reg, reg);
+}
+
+void
+MacroAssembler::negateFloat(FloatRegister reg)
+{
+ as_fneg(reg, reg);
+}
+
+void
+MacroAssembler::abs32(Register src, Register dest)
+{
+ // PowerPC Compiler Writer's Guide page 50
+ as_srawi(ScratchRegister, src, 31);
+ as_xor(SecondScratchReg, ScratchRegister, src);
+ as_subf(dest, ScratchRegister, SecondScratchReg);
+}
+
+void
+MacroAssembler::absFloat32(FloatRegister src, FloatRegister dest)
+{
+ as_fabs(dest, src);
+}
+
+void
+MacroAssembler::absDouble(FloatRegister src, FloatRegister dest)
+{
+ as_fabs(dest, src);
+}
+
+void
+MacroAssembler::sqrtFloat32(FloatRegister src, FloatRegister dest)
+{
+ as_fsqrts(dest, src);
+}
+
+void
+MacroAssembler::sqrtDouble(FloatRegister src, FloatRegister dest)
+{
+ as_fsqrt(dest, src);
+}
+
+void
+MacroAssembler::minFloat32(FloatRegister other, FloatRegister srcDest, bool handleNaN)
+{
+ minMaxDouble(srcDest, other, handleNaN, false);
+}
+
+void
+MacroAssembler::minDouble(FloatRegister other, FloatRegister srcDest, bool handleNaN)
+{
+ minMaxDouble(srcDest, other, handleNaN, false);
+}
+
+void
+MacroAssembler::maxFloat32(FloatRegister other, FloatRegister srcDest, bool handleNaN)
+{
+ minMaxDouble(srcDest, other, handleNaN, true);
+}
+
+void
+MacroAssembler::maxDouble(FloatRegister other, FloatRegister srcDest, bool handleNaN)
+{
+ minMaxDouble(srcDest, other, handleNaN, true);
+}
+
+// ===============================================================
+// Shift functions
+
+void
+MacroAssembler::lshift32(Register src, Register dest)
+{
+ // slw will zero out any shift amount greater than 32, but JavaScript
+ // expects this to act like a modulo, so ...
+ MOZ_ASSERT(src != ScratchRegister);
+ MOZ_ASSERT(dest != ScratchRegister);
+
+ as_andi_rc(ScratchRegister, src, 31);
+ as_slw(dest, dest, ScratchRegister);
+}
+
+void
+MacroAssembler::lshift32(Imm32 imm, Register dest)
+{
+ // Mod the constant directly, et voila.
+ x_slwi(dest, dest, imm.value % 32);
+}
+
+void
+MacroAssembler::flexibleLshift32(Register src, Register dest)
+{
+ lshift32(src, dest);
+}
+
+void
+MacroAssembler::rshift32(Register src, Register dest)
+{
+ // Same deal.
+ MOZ_ASSERT(src != ScratchRegister);
+ MOZ_ASSERT(dest != ScratchRegister);
+
+ as_andi_rc(ScratchRegister, src, 31);
+ as_srw(dest, dest, ScratchRegister);
+}
+
+void
+MacroAssembler::rshift32(Imm32 imm, Register dest)
+{
+ // Same deal with a twist:
+ // If imm.value is a multiple of 32, then n = 0, and we assert because
+ // the underlying rlwinm has to encode 32 in a 5-bit field. So, if the
+ // modulo is zero, do nothing or load zero instead (based on what srawi
+ // does on 64-bit systems, minus the sign extension).
+ if (!(imm.value % 32)) {
+ if (imm.value == 0) {
+ // No-op // as_rldicl(dest, dest, 0, 32); // "clrldi"
+ } else {
+ // Right-shifting a 32-bit quantity 32 bits is zero.
+ xs_li(dest, 0);
+ }
+ } else {
+ x_srwi(dest, dest, imm.value % 32);
+ }
+}
+
+void
+MacroAssembler::flexibleRshift32(Register src, Register dest)
+{
+ rshift32(src, dest);
+}
+
+void
+MacroAssembler::rshift32Arithmetic(Register src, Register dest)
+{
+ // Same deal.
+ MOZ_ASSERT(src != ScratchRegister);
+ as_andi_rc(ScratchRegister, src, 31);
+ as_sraw(dest, dest, ScratchRegister);
+}
+
+void
+MacroAssembler::rshift32Arithmetic(Imm32 imm, Register dest)
+{
+ as_srawi(dest, dest, imm.value % 32);
+}
+
+void
+MacroAssembler::flexibleRshift32Arithmetic(Register src, Register dest)
+{
+ rshift32Arithmetic(src, dest);
+}
+
+// ===============================================================
+// Rotation functions (32-bit unless specified otherwise)
+void
+MacroAssembler::rotateLeft(Imm32 count, Register input, Register dest)
+{
+ if (count.value) {
+ as_rlwinm(dest, input, count.value % 32, 0, 31); // "rotlwi"
+ } else {
+ if (input != dest)
+ ma_move(dest, input);
+ }
+}
+void
+MacroAssembler::rotateLeft(Register count, Register input, Register dest)
+{
+ as_rlwnm(dest, input, count, 0, 31); // "rotlw"
+}
+void
+MacroAssembler::rotateRight(Imm32 count, Register input, Register dest)
+{
+ if (count.value) {
+ as_rlwinm(dest, input, 32 - (count.value % 32), 0, 31); // "rotrwi"
+ } else {
+ if (input != dest)
+ ma_move(dest, input);
+ }
+}
+void
+MacroAssembler::rotateRight(Register count, Register input, Register dest)
+{
+ MOZ_ASSERT(input != ScratchRegister);
+ MOZ_ASSERT(count != ScratchRegister);
+ MOZ_ASSERT(input != SecondScratchReg);
+ MOZ_ASSERT(count != SecondScratchReg);
+
+ xs_li(ScratchRegister, 32);
+ as_andi_rc(SecondScratchReg, count, 31);
+ as_subf(ScratchRegister, SecondScratchReg, ScratchRegister); // T = B - A
+ as_rlwnm(dest, input, ScratchRegister, 0, 31); // "rotrw"
+}
+
+// ===============================================================
+// Bit counting functions
+
+void
+MacroAssembler::clz32(Register src, Register dest, bool knownNotZero)
+{
+ as_cntlzw(dest, src);
+}
+
+void
+MacroAssembler::ctz32(Register src, Register dest, bool knownNotZero)
+{
+ as_cnttzw(dest, src);
+}
+
+void
+MacroAssembler::popcnt32(Register input, Register output, Register tmp)
+{
+ // Sing to the tune of Revolution No. 9:
+ // POWER9
+ // POWER9
+ // POWER9
+ // POWER9 etc.
+ as_popcntw(output, input);
+}
+
+// ===============================================================
+// Condition functions
+
+void MacroAssembler::cmp8Set(Condition cond, Address lhs, Imm32 rhs,
+ Register dest) {
+ MOZ_ASSERT(lhs.base != SecondScratchReg);
+
+ // Automatically generates a 32-bit compare.
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ load8ZeroExtend(lhs, SecondScratchReg);
+ ma_cmp_set(dest, SecondScratchReg, Imm32(uint8_t(rhs.value)), cond);
+ break;
+
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ load8SignExtend(lhs, SecondScratchReg);
+ ma_cmp_set(dest, SecondScratchReg, Imm32(int8_t(rhs.value)), cond);
+ break;
+
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+void MacroAssembler::cmp16Set(Condition cond, Address lhs, Imm32 rhs,
+ Register dest) {
+ MOZ_ASSERT(lhs.base != SecondScratchReg);
+
+ // Automatically generates a 32-bit compare.
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ load16ZeroExtend(lhs, SecondScratchReg);
+ ma_cmp_set(dest, SecondScratchReg, Imm32(uint16_t(rhs.value)), cond);
+ break;
+
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ load16SignExtend(lhs, SecondScratchReg);
+ ma_cmp_set(dest, SecondScratchReg, Imm32(int16_t(rhs.value)), cond);
+ break;
+
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+
+// ===============================================================
+// Branch functions
+
+void MacroAssembler::branch8(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) {
+ MOZ_ASSERT(lhs.base != SecondScratchReg);
+
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ load8ZeroExtend(lhs, SecondScratchReg);
+ branch32(cond, SecondScratchReg, Imm32(uint8_t(rhs.value)), label);
+ break;
+
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ load8SignExtend(lhs, SecondScratchReg);
+ branch32(cond, SecondScratchReg, Imm32(int8_t(rhs.value)), label);
+ break;
+
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+void MacroAssembler::branch8(Condition cond, const BaseIndex& lhs, Register rhs,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ MOZ_ASSERT(scratch2 != lhs.base);
+
+ computeScaledAddress(lhs, scratch2);
+
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ load8ZeroExtend(Address(scratch2, lhs.offset), scratch2);
+ branch32(cond, scratch2, rhs, label);
+ break;
+
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ load8SignExtend(Address(scratch2, lhs.offset), scratch2);
+ branch32(cond, scratch2, rhs, label);
+ break;
+
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+void MacroAssembler::branch16(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) {
+ MOZ_ASSERT(lhs.base != SecondScratchReg);
+
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ load16ZeroExtend(lhs, SecondScratchReg);
+ branch32(cond, SecondScratchReg, Imm32(uint16_t(rhs.value)), label);
+ break;
+
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ load16SignExtend(lhs, SecondScratchReg);
+ branch32(cond, SecondScratchReg, Imm32(int16_t(rhs.value)), label);
+ break;
+
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+// XXX: need 32-bit hints here
+template <class L>
+void
+MacroAssembler::branch32(Condition cond, Register lhs, Register rhs, L label)
+{
+ ma_bc(lhs, rhs, label, cond);
+}
+
+template <class L>
+void
+MacroAssembler::branch32(Condition cond, Register lhs, Imm32 imm, L label)
+{
+ ma_bc(lhs, imm, label, cond);
+}
+
+void
+MacroAssembler::branch32(Condition cond, const Address& lhs, Register rhs, Label* label)
+{
+ load32(lhs, SecondScratchReg);
+ ma_bc(SecondScratchReg, rhs, label, cond);
+}
+
+void
+MacroAssembler::branch32(Condition cond, const Address& lhs, Imm32 rhs, Label* label)
+{
+ load32(lhs, SecondScratchReg);
+ ma_bc(SecondScratchReg, rhs, label, cond);
+}
+
+void
+MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs, Register rhs, Label* label)
+{
+ load32(lhs, SecondScratchReg);
+ ma_bc(SecondScratchReg, rhs, label, cond);
+}
+
+void
+MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs, Imm32 rhs, Label* label)
+{
+ load32(lhs, SecondScratchReg);
+ ma_bc(SecondScratchReg, rhs, label, cond);
+}
+
+void
+MacroAssembler::branch32(Condition cond, const BaseIndex& lhs, Imm32 rhs, Label* label)
+{
+ load32(lhs, SecondScratchReg);
+ ma_bc(SecondScratchReg, rhs, label, cond);
+}
+
+void
+MacroAssembler::branch32(Condition cond, wasm::SymbolicAddress addr, Imm32 imm, Label* label)
+{
+ load32(addr, SecondScratchReg);
+ ma_bc(SecondScratchReg, imm, label, cond);
+}
+
+template <class L>
+void
+MacroAssembler::branchPtr(Condition cond, Register lhs, Register rhs, L label)
+{
+ ma_bc(lhs, rhs, label, cond);
+}
+
+void
+MacroAssembler::branchPtr(Condition cond, Register lhs, Imm32 rhs, Label* label)
+{
+ // Need to hint the MacroAssembler that this needs a 64-bit compare.
+ ma_bc64(lhs, rhs, label, cond);
+}
+
+void
+MacroAssembler::branchPtr(Condition cond, Register lhs, ImmPtr rhs, Label* label)
+{
+ ma_bc(lhs, rhs, label, cond);
+}
+
+void
+MacroAssembler::branchPtr(Condition cond, Register lhs, ImmGCPtr rhs, Label* label)
+{
+ ma_bc(lhs, rhs, label, cond);
+}
+
+void
+MacroAssembler::branchPtr(Condition cond, Register lhs, ImmWord rhs, Label* label)
+{
+ ma_bc(lhs, rhs, label, cond);
+}
+
+template <class L>
+void
+MacroAssembler::branchPtr(Condition cond, const Address& lhs, Register rhs, L label)
+{
+ MOZ_ASSERT(rhs != SecondScratchReg);
+ loadPtr(lhs, SecondScratchReg);
+ branchPtr(cond, SecondScratchReg, rhs, label);
+}
+
+void
+MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmPtr rhs, Label* label)
+{
+ loadPtr(lhs, SecondScratchReg);
+ branchPtr(cond, SecondScratchReg, rhs, label);
+}
+
+void
+MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmGCPtr rhs, Label* label)
+{
+ loadPtr(lhs, SecondScratchReg);
+ branchPtr(cond, SecondScratchReg, rhs, label);
+}
+
+void
+MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmWord rhs, Label* label)
+{
+ loadPtr(lhs, SecondScratchReg);
+ branchPtr(cond, SecondScratchReg, rhs, label);
+}
+
+void
+MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs, Register rhs, Label* label)
+{
+ MOZ_ASSERT(rhs != SecondScratchReg);
+ loadPtr(lhs, SecondScratchReg);
+ branchPtr(cond, SecondScratchReg, rhs, label);
+}
+
+void
+MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs, ImmWord rhs, Label* label)
+{
+ loadPtr(lhs, SecondScratchReg);
+ branchPtr(cond, SecondScratchReg, rhs, label);
+}
+
+void
+MacroAssembler::branchPtr(Condition cond, wasm::SymbolicAddress lhs, Register rhs, Label* label)
+{
+ MOZ_ASSERT(rhs != SecondScratchReg);
+ loadPtr(lhs, SecondScratchReg);
+ branchPtr(cond, SecondScratchReg, rhs, label);
+}
+
+void
+MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs, ImmWord rhs, Label* label)
+{
+ loadPtr(lhs, SecondScratchReg);
+ branchPtr(cond, SecondScratchReg, rhs, label);
+}
+
+void
+MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs, Register rhs, Label* label)
+{
+ MOZ_ASSERT(rhs != SecondScratchReg);
+ loadPtr(lhs, SecondScratchReg);
+ branchPtr(cond, SecondScratchReg, rhs, label);
+}
+
+void MacroAssembler::branchTestValue(Condition cond, const BaseIndex& lhs,
+ const ValueOperand& rhs, Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ branchPtr(cond, lhs, rhs.valueReg(), label);
+}
+
+void
+MacroAssembler::branchFloat(DoubleCondition cond, FloatRegister lhs, FloatRegister rhs,
+ Label* label)
+{
+ ma_bc(cond, lhs, rhs, label);
+}
+
+void
+MacroAssembler::branchTruncateFloat32ToInt32(FloatRegister src, Register dest, Label* fail)
+{
+ truncDoubleToInt32(src, dest, fail);
+}
+
+void
+MacroAssembler::branchDouble(DoubleCondition cond, FloatRegister lhs, FloatRegister rhs,
+ Label* label)
+{
+ ma_bc(cond, lhs, rhs, label);
+}
+
+void
+MacroAssembler::branchTruncateDoubleToInt32(FloatRegister src, Register dest, Label* fail)
+{
+ truncDoubleToInt32(src, dest, fail);
+}
+
+void
+MacroAssembler::branchMulPtr(Condition cond, Register src, Register dest, Label *overflow)
+{
+ as_mulldo_rc(dest, src, dest);
+ ma_bc(cond, overflow);
+}
+
+// The src type will autodetermine 32-bit mode.
+template <typename T>
+void
+MacroAssembler::branchAdd32(Condition cond, T src, Register dest, Label* overflow)
+{
+ switch (cond) {
+ case Overflow:
+ ma_addTestOverflow(dest, dest, src, overflow);
+ break;
+ case CarryClear:
+ case CarrySet:
+ ma_addTestCarry(cond, dest, dest, src, overflow);
+ break;
+ default:
+ MOZ_CRASH("NYI");
+ }
+}
+
+template <typename T>
+void
+MacroAssembler::branchSub32(Condition cond, T src, Register dest, Label* overflow)
+{
+ switch (cond) {
+ case Overflow:
+ ma_subTestOverflow(dest, dest, src, overflow);
+ break;
+ case NonZero:
+ case Zero:
+ case NotSigned:
+ case Signed:
+ ma_subu(dest, src);
+ ma_bc(dest, dest, overflow, cond);
+ break;
+ default:
+ MOZ_CRASH("NYI");
+ }
+}
+
+template <typename T>
+void
+MacroAssembler::branchAddPtr(Condition cond, T src, Register dest, Label* overflow)
+{
+ switch (cond) {
+ case Overflow:
+ ma_addTestOverflow(dest, dest, src, overflow, /* is32 */ false);
+ break;
+ case CarryClear:
+ case CarrySet:
+ ma_addTestCarry(cond, dest, dest, src, overflow, /* is32 */ false);
+ break;
+ default:
+ MOZ_CRASH("NYI");
+ }
+}
+
+template <typename T>
+void
+MacroAssembler::branchSubPtr(Condition cond, T src, Register dest, Label* overflow)
+{
+ switch (cond) {
+ case Overflow:
+ ma_subTestOverflow(dest, dest, src, overflow, /* is32 */ false);
+ break;
+ case NonZero:
+ case Zero:
+ case NotSigned:
+ case Signed:
+ ma_subu(dest, src);
+ ma_bc(dest, dest, overflow, cond);
+ break;
+ default:
+ MOZ_CRASH("NYI");
+ }
+}
+
+template <>
+inline void
+MacroAssembler::branchMul32(Condition cond, Register src, Register dest, Label* label)
+{
+ as_mullwo_rc(dest, dest, src);
+ ma_bc(cond, label);
+}
+
+template <typename T>
+void
+MacroAssembler::branchMul32(Condition cond, T src, Register dest, Label* label)
+{
+ MOZ_CRASH("NYI");
+}
+
+template <>
+inline void
+MacroAssembler::branchRshift32(Condition cond, Imm32 shift, Register srcDest,
+ Label *label)
+{
+ as_rlwinm_rc(srcDest, srcDest, 32 - shift.value, shift.value, 31);
+ ma_bc(cond, label);
+}
+
+template<typename T>
+void
+MacroAssembler::branchRshift32(Condition cond, T shift, Register srcDest,
+ Label *label)
+{
+ MOZ_CRASH("No default implementation");
+}
+
+void
+MacroAssembler::decBranchPtr(Condition cond, Register lhs, Imm32 rhs, Label* label)
+{
+ subPtr(rhs, lhs);
+ branchPtr(cond, lhs, Imm32(0), label);
+}
+
+template <class L>
+void
+MacroAssembler::branchTest32(Condition cond, Register lhs, Register rhs, L label)
+{
+ MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed || cond == NotSigned);
+ if (cond == Signed || cond == NotSigned) {
+ MOZ_ASSERT(lhs == rhs);
+ // Sign extend first.
+ as_extsw(lhs, lhs);
+ }
+ if (lhs == rhs) {
+ ma_bc(lhs, rhs, label, cond);
+ } else {
+ // XXX: trim to 32 bits?
+ as_and(ScratchRegister, lhs, rhs);
+ ma_bc(ScratchRegister, ScratchRegister, label, cond);
+ }
+}
+
+template <class L>
+void
+MacroAssembler::branchTest32(Condition cond, Register lhs, Imm32 rhs, L label)
+{
+ MOZ_ASSERT(cond == Zero || cond == NonZero);
+ // XXX: trim to 32 bits?
+ ma_and(ScratchRegister, lhs, rhs);
+ ma_bc(ScratchRegister, ScratchRegister, label, cond);
+}
+
+void
+MacroAssembler::branchTest32(Condition cond, const Address& lhs, Imm32 rhs, Label* label)
+{
+ load32(lhs, SecondScratchReg);
+ branchTest32(cond, SecondScratchReg, rhs, label);
+}
+
+void
+MacroAssembler::branchTest32(Condition cond, const AbsoluteAddress& lhs, Imm32 rhs, Label* label)
+{
+ load32(lhs, SecondScratchReg);
+ branchTest32(cond, SecondScratchReg, rhs, label);
+}
+
+// XXX: Improve branchTest* to use |and.| and test results directly, as noted:
+template <class L>
+void
+MacroAssembler::branchTestPtr(Condition cond, Register lhs, Register rhs, L label)
+{
+ MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed || cond == NotSigned);
+ if (lhs == rhs) {
+ ma_bc(lhs, rhs, label, cond);
+ } else {
+ as_and(ScratchRegister, lhs, rhs);
+ ma_bc(ScratchRegister, ScratchRegister, label, cond);
+ }
+}
+
+void
+MacroAssembler::branchTestPtr(Condition cond, Register lhs, Imm32 rhs, Label* label)
+{
+ MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed || cond == NotSigned);
+ ma_and(ScratchRegister, lhs, rhs);
+ ma_bc(ScratchRegister, ScratchRegister, label, cond);
+}
+
+void
+MacroAssembler::branchTestPtr(Condition cond, const Address& lhs, Imm32 rhs, Label* label)
+{
+ loadPtr(lhs, SecondScratchReg);
+ branchTestPtr(cond, SecondScratchReg, rhs, label);
+}
+
+void
+MacroAssembler::branchTestUndefined(Condition cond, Register tag, Label* label)
+{
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_bc(tag, ImmTag(JSVAL_TAG_UNDEFINED), label, cond);
+}
+
+void
+MacroAssembler::branchTestUndefined(Condition cond, const Address& address, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ extractTag(address, scratch2);
+ branchTestUndefined(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestUndefined(Condition cond, const BaseIndex& address, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ extractTag(address, scratch2);
+ branchTestUndefined(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestInt32(Condition cond, Register tag, Label* label)
+{
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_bc(tag, ImmTag(JSVAL_TAG_INT32), label, cond);
+}
+
+void
+MacroAssembler::branchTestInt32(Condition cond, const Address& address, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ extractTag(address, scratch2);
+ branchTestInt32(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestInt32(Condition cond, const BaseIndex& address, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ extractTag(address, scratch2);
+ branchTestInt32(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestDouble(Condition cond, const Address& address, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ extractTag(address, scratch2);
+ branchTestDouble(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestDouble(Condition cond, const BaseIndex& address, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ extractTag(address, scratch2);
+ branchTestDouble(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestDoubleTruthy(bool b, FloatRegister value, Label* label)
+{
+ ma_lid(ScratchDoubleReg, 0.0);
+ DoubleCondition cond = b ? DoubleNotEqual : DoubleEqualOrUnordered;
+ ma_bc(cond, value, ScratchDoubleReg, label);
+}
+
+void
+MacroAssembler::branchTestNumber(Condition cond, Register tag, Label* label)
+{
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ Condition actual = cond == Equal ? BelowOrEqual : Above;
+ ma_bc(tag, ImmTag(JS::detail::ValueUpperInclNumberTag), label, actual);
+}
+
+void
+MacroAssembler::branchTestBoolean(Condition cond, Register tag, Label* label)
+{
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_bc(tag, ImmTag(JSVAL_TAG_BOOLEAN), label, cond);
+}
+
+void
+MacroAssembler::branchTestBoolean(Condition cond, const Address& address, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ extractTag(address, scratch2);
+ branchTestBoolean(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestBoolean(Condition cond, const BaseIndex& address, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ extractTag(address, scratch2);
+ branchTestBoolean(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestString(Condition cond, Register tag, Label* label)
+{
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_bc(tag, ImmTag(JSVAL_TAG_STRING), label, cond);
+}
+
+void
+MacroAssembler::branchTestString(Condition cond, const Address& address, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ extractTag(address, scratch2);
+ branchTestString(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestString(Condition cond, const BaseIndex& address, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ extractTag(address, scratch2);
+ branchTestString(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestSymbol(Condition cond, Register tag, Label* label)
+{
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_bc(tag, ImmTag(JSVAL_TAG_SYMBOL), label, cond);
+}
+
+void
+MacroAssembler::branchTestSymbol(Condition cond, const BaseIndex& address, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ extractTag(address, scratch2);
+ branchTestSymbol(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestNull(Condition cond, Register tag, Label* label)
+{
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_bc(tag, ImmTag(JSVAL_TAG_NULL), label, cond);
+}
+
+void
+MacroAssembler::branchTestNull(Condition cond, const Address& address, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ extractTag(address, scratch2);
+ branchTestNull(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestNull(Condition cond, const BaseIndex& address, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ extractTag(address, scratch2);
+ branchTestNull(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestObject(Condition cond, Register tag, Label* label)
+{
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_bc(tag, ImmTag(JSVAL_TAG_OBJECT), label, cond);
+}
+
+void
+MacroAssembler::branchTestObject(Condition cond, const Address& address, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ extractTag(address, scratch2);
+ branchTestObject(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestObject(Condition cond, const BaseIndex& address, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ extractTag(address, scratch2);
+ branchTestObject(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestGCThing(Condition cond, const Address& address, Label* label)
+{
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ SecondScratchRegisterScope scratch2(*this);
+ extractTag(address, scratch2);
+ ma_bc(scratch2, ImmTag(JS::detail::ValueLowerInclGCThingTag), label,
+ (cond == Equal) ? AboveOrEqual : Below);
+}
+void
+MacroAssembler::branchTestGCThing(Condition cond, const BaseIndex& address, Label* label)
+{
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ SecondScratchRegisterScope scratch2(*this);
+ extractTag(address, scratch2);
+ ma_bc(scratch2, ImmTag(JS::detail::ValueLowerInclGCThingTag), label,
+ (cond == Equal) ? AboveOrEqual : Below);
+}
+void
+MacroAssembler::branchTestGCThing(Condition cond, const ValueOperand& address, Label* label)
+{
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ SecondScratchRegisterScope scratch2(*this);
+ extractTag(address, scratch2);
+ ma_bc(scratch2, ImmTag(JS::detail::ValueLowerInclGCThingTag), label,
+ (cond == Equal) ? AboveOrEqual : Below);
+}
+
+void
+MacroAssembler::branchTestPrimitive(Condition cond, Register tag, Label* label)
+{
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_bc(tag, ImmTag(JS::detail::ValueUpperExclPrimitiveTag), label,
+ (cond == Equal) ? Below : AboveOrEqual);
+}
+
+void
+MacroAssembler::branchTestMagic(Condition cond, Register tag, Label* label)
+{
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_bc(tag, ImmTag(JSVAL_TAG_MAGIC), label, cond);
+}
+
+void
+MacroAssembler::branchTestMagic(Condition cond, const Address& address, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ extractTag(address, scratch2);
+ branchTestMagic(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchTestMagic(Condition cond, const BaseIndex& address, Label* label)
+{
+ SecondScratchRegisterScope scratch2(*this);
+ extractTag(address, scratch2);
+ branchTestMagic(cond, scratch2, label);
+}
+
+void
+MacroAssembler::branchToComputedAddress(const BaseIndex& addr)
+{
+ // TODO: We should just be able to call loadPtr here, but we get values
+ // with the high word still set.
+ //loadPtr(addr, ScratchRegister);
+
+ int32_t shift = Imm32::ShiftOf(addr.scale).value;
+ MOZ_ASSERT(addr.base != ScratchRegister);
+ MOZ_ASSERT(addr.index != ScratchRegister);
+ MOZ_ASSERT(addr.base != SecondScratchReg);
+ MOZ_ASSERT(addr.index != SecondScratchReg);
+
+ // Ass-U-Me that addr.index is always positive.
+ if (shift) {
+ ma_dsll(ScratchRegister, addr.index, Imm32(shift));
+ as_rldicl(ScratchRegister, ScratchRegister, 0, 32); // "clrldi"
+ as_add(SecondScratchReg, addr.base, ScratchRegister);
+ } else {
+ // XXX: merge with following ld into ldx
+ as_add(SecondScratchReg, addr.base, addr.index);
+ }
+ as_ld(SecondScratchReg, SecondScratchReg, 0);
+ xs_mtctr(SecondScratchReg);
+ as_bctr(DontLinkB);
+}
+
+void
+MacroAssembler::cmp32Move32(Condition cond, Register lhs, Register rhs, Register src,
+ Register dest)
+{
+ ma_cmp32(lhs, rhs, cond);
+ // Ass-U-Me that ma_cmp32 selected the correct compare, and mask off any
+ // synthetic bits. isel will assert on any conditions it can't encode.
+ as_isel(dest, src, dest, (cond & 0xff));
+}
+
+void MacroAssembler::cmp32Move32(Condition cond, Register lhs, Imm32 rhs,
+ Register src, Register dest) {
+ MOZ_ASSERT(lhs != SecondScratchReg && src != SecondScratchReg &&
+ dest != SecondScratchReg);
+ ma_li(SecondScratchReg, rhs); // XXX: probably could be r0
+ cmp32Move32(cond, lhs, SecondScratchReg, src, dest);
+}
+
+// better version? below
+#if(0)
+void MacroAssembler::cmp32Move32(Condition cond, Register lhs,
+ const Address& rhs, Register src,
+ Register dest) {
+ MOZ_ASSERT(lhs != ScratchRegister && src != ScratchRegister &&
+ dest != ScratchRegister);
+ load32(rhs, ScratchRegister); // XXX: is r12 live?
+ cmp32Move32(cond, lhs, ScratchRegister, src, dest);
+}
+#endif
+
+void
+MacroAssembler::cmp32MovePtr(Condition cond, Register lhs, Imm32 rhs, Register src,
+ Register dest)
+{
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_cmp32(lhs, rhs, cond);
+ // isel cannot test for the absence of a bit.
+ if (cond == Equal) {
+ as_isel(dest, src, dest, Equal);
+ } else {
+ // Flip the order.
+ as_isel(dest, dest, src, Equal);
+ }
+}
+
+void
+MacroAssembler::cmp32Move32(Condition cond, Register lhs, const Address& rhs,
+ Register src,
+ Register dest)
+{
+ if (cond == Below) {
+ ma_cmp32(lhs, rhs, cond);
+ as_isel(dest, src, dest, LessThan);
+ return;
+ }
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_cmp32(lhs, rhs, cond);
+ if (cond == Equal) {
+ as_isel(dest, src, dest, Equal);
+ } else {
+ as_isel(dest, dest, src, Equal);
+ }
+}
+
+void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs, Register rhs,
+ Register src, Register dest) {
+ if (cond == Below) {
+ as_cmpld(lhs, rhs);
+ as_isel(dest, src, dest, LessThan);
+ return;
+ }
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ as_cmpd(lhs, rhs);
+ if (cond == Equal) {
+ as_isel(dest, src, dest, Equal);
+ } else {
+ as_isel(dest, dest, src, Equal);
+ }
+}
+
+void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs,
+ const Address& rhs, Register src,
+ Register dest) {
+ MOZ_ASSERT(lhs != ScratchRegister);
+ MOZ_ASSERT(src != ScratchRegister);
+ MOZ_ASSERT(lhs != SecondScratchReg); // may be used by ma_load
+ MOZ_ASSERT(src != SecondScratchReg);
+ MOZ_ASSERT(dest != ScratchRegister);
+ MOZ_ASSERT(dest != SecondScratchReg);
+
+ ma_load(ScratchRegister, rhs, SizeDouble);
+ cmpPtrMovePtr(cond, lhs, ScratchRegister, src, dest);
+}
+
+void MacroAssembler::cmp32Load32(Condition cond, Register lhs,
+ const Address& rhs, const Address& src,
+ Register dest) {
+ MOZ_ASSERT(lhs != ScratchRegister && dest != ScratchRegister);
+ load32(rhs, ScratchRegister);
+ cmp32Load32(cond, lhs, ScratchRegister, src, dest);
+}
+
+void MacroAssembler::cmp32Load32(Condition cond, Register lhs, Register rhs,
+ const Address& src, Register dest) {
+ Label skip;
+ // XXX: turn into a load and isel
+ branch32(Assembler::InvertCondition(cond), lhs, rhs, &skip);
+ load32(src, dest);
+ bind(&skip);
+}
+
+void MacroAssembler::cmp32Load32(Condition cond, Register lhs, Imm32 rhs,
+ const Address& src, Register dest) {
+ Label skip;
+ // XXX: turn into a load and isel
+ branch32(Assembler::InvertCondition(cond), lhs, rhs, &skip);
+ load32(src, dest);
+ bind(&skip);
+}
+
+void
+MacroAssembler::cmp32LoadPtr(Condition cond, const Address& lhs, Imm32 rhs,
+ const Address& src, Register dest)
+{
+ Label skip;
+ // XXX: turn into a load and isel
+ branch32(Assembler::InvertCondition(cond), lhs, rhs, &skip);
+ loadPtr(src, dest);
+ bind(&skip);
+}
+
+// Constant-time conditional moves (basically isel all the things, because
+// it is not subject to branch prediction).
+void
+MacroAssembler::test32LoadPtr(Condition cond, const Address& addr, Imm32 mask,
+ const Address& src, Register dest)
+{
+ MOZ_ASSERT(cond == Zero || cond == NonZero);
+ const uint32_t mm = mask.value;
+
+ ma_load(ScratchRegister, addr, SizeWord, ZeroExtend);
+ if (Imm16::IsInUnsignedRange(mm)) {
+ as_andi_rc(ScratchRegister, ScratchRegister, mm); // -> CR0[EQ]
+ } else {
+ ma_li(SecondScratchReg, mm);
+ as_and_rc(ScratchRegister, ScratchRegister, SecondScratchReg);
+ }
+ ma_load(SecondScratchReg, src, SizeDouble); // pointer-sized
+ // If the condition is true, set dest to src. However, isel cannot
+ // test for the absence of a bit, and it cannot test for multiple bits, so
+ // footwork is required.
+ if (cond == Zero) {
+ MOZ_ASSERT(cond == (Equal | ConditionZero));
+ as_isel(dest, SecondScratchReg, dest, Assembler::Equal);
+ } else {
+ // Flip the order.
+ MOZ_ASSERT(cond == (NotEqual | ConditionZero));
+ as_isel(dest, dest, SecondScratchReg, Assembler::Equal);
+ }
+}
+
+void
+MacroAssembler::test32MovePtr(Condition cond, const Address& addr, Imm32 mask,
+ Register src, Register dest)
+{
+ MOZ_ASSERT(cond == Zero || cond == NonZero);
+ MOZ_ASSERT(src != ScratchRegister);
+ MOZ_ASSERT(src != SecondScratchReg);
+ const uint32_t mm = mask.value;
+
+ ma_load(ScratchRegister, addr, SizeWord, ZeroExtend);
+ if (Imm16::IsInUnsignedRange(mm)) {
+ as_andi_rc(ScratchRegister, ScratchRegister, mm); // -> CR0[EQ]
+ } else {
+ ma_li(SecondScratchReg, mm);
+ as_and_rc(ScratchRegister, ScratchRegister, SecondScratchReg);
+ }
+ if (cond == Zero) {
+ MOZ_ASSERT(cond == (Equal | ConditionZero));
+ as_isel(dest, src, dest, Assembler::Equal);
+ } else {
+ // Flip the order.
+ MOZ_ASSERT(cond == (NotEqual | ConditionZero));
+ as_isel(dest, dest, src, Assembler::Equal);
+ }
+}
+
+void
+MacroAssembler::spectreBoundsCheck32(Register index, Register length,
+ Register maybeScratch, Label* failure)
+{
+ branch32(Assembler::BelowOrEqual, length, index, failure);
+ if (JitOptions.spectreIndexMasking) {
+ // The result of the compare is still in cr0, and the compare was
+ // already done unsigned, so we just generate an iselgt. The second
+ // register is unimportant, because we know this will always be true.
+ as_isel(index, index, length, Assembler::GreaterThan);
+ }
+}
+
+void
+MacroAssembler::spectreBoundsCheck32(Register index, const Address& length,
+ Register maybeScratch, Label* failure)
+{
+ branch32(Assembler::BelowOrEqual, length, index, failure);
+ if (JitOptions.spectreIndexMasking) {
+ // r12 will likely have |length| in it anyway from the above
+ // operation, but it doesn't matter anyhow; see above.
+ as_isel(index, index, SecondScratchReg, Assembler::GreaterThan);
+ }
+}
+
+// Same as above
+void
+MacroAssembler::spectreBoundsCheckPtr(Register index, Register length,
+ Register maybeScratch, Label* failure)
+{
+ branchPtr(Assembler::BelowOrEqual, length, index, failure);
+ if (JitOptions.spectreIndexMasking) {
+ as_isel(index, index, length, Assembler::GreaterThan);
+ }
+}
+
+void
+MacroAssembler::spectreBoundsCheckPtr(Register index, const Address& length,
+ Register maybeScratch, Label* failure)
+{
+ branchPtr(Assembler::BelowOrEqual, length, index, failure);
+ if (JitOptions.spectreIndexMasking) {
+ as_isel(index, index, SecondScratchReg, Assembler::GreaterThan);
+ }
+}
+
+void
+MacroAssembler::spectreMovePtr(Condition cond, Register src, Register dest)
+{
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+
+ // isel cannot test for the non-existence of a bit.
+ if (cond == Equal) {
+ as_isel(dest, src, dest, Assembler::Equal);
+ } else {
+ // Flip the order.
+ as_isel(dest, dest, src, Assembler::Equal);
+ }
+}
+
+void
+MacroAssembler::spectreZeroRegister(Condition cond, Register scratch, Register dest)
+{
+ // Zero the register if *true*. Hold my beer.
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+
+ if (cond == NotEqual) {
+ xs_li(ScratchRegister, 0);
+ as_isel(dest, dest, ScratchRegister, Assembler::Equal);
+ } else {
+ as_isel0(dest, ScratchRegister, dest, Assembler::Equal); // mscdfr0
+ }
+}
+
+void
+MacroAssembler::fallibleUnboxPtr(const ValueOperand& src, Register dest,
+ JSValueType type, Label* fail)
+{
+ MOZ_ASSERT(type == JSVAL_TYPE_OBJECT || type == JSVAL_TYPE_STRING ||
+ type == JSVAL_TYPE_SYMBOL || type == JSVAL_TYPE_BIGINT);
+ // dest := src XOR mask
+ // scratch := dest >> JSVAL_TAG_SHIFT
+ // fail if scratch != 0
+ //
+ // Note: src and dest can be the same register
+ ScratchRegisterScope scratch(asMasm());
+ mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), scratch);
+ ma_xor(scratch, src.valueReg());
+ ma_move(dest, scratch);
+ x_srdi(scratch, scratch, JSVAL_TAG_SHIFT);
+ ma_bc(scratch, Imm32(0), fail, Assembler::NotEqual);
+}
+
+void
+MacroAssembler::fallibleUnboxPtr(const Address& src, Register dest,
+ JSValueType type, Label* fail)
+{
+ loadValue(src, ValueOperand(dest));
+ fallibleUnboxPtr(ValueOperand(dest), dest, type, fail);
+}
+
+void MacroAssembler::fallibleUnboxPtr(const BaseIndex& src, Register dest,
+ JSValueType type, Label* fail)
+{
+ loadValue(src, ValueOperand(dest));
+ fallibleUnboxPtr(ValueOperand(dest), dest, type, fail);
+}
+
+// ========================================================================
+// Memory access primitives.
+
+FaultingCodeOffset
+MacroAssembler::storeUncanonicalizedDouble(FloatRegister src, const Address& addr)
+{
+ return ma_sd(src, addr);
+}
+FaultingCodeOffset
+MacroAssembler::storeUncanonicalizedDouble(FloatRegister src, const BaseIndex& addr)
+{
+ return ma_sd(src, addr);
+}
+
+FaultingCodeOffset
+MacroAssembler::storeUncanonicalizedFloat32(FloatRegister src, const Address& addr)
+{
+ return ma_ss(src, addr);
+}
+FaultingCodeOffset
+MacroAssembler::storeUncanonicalizedFloat32(FloatRegister src, const BaseIndex& addr)
+{
+ return ma_ss(src, addr);
+}
+
+void
+MacroAssembler::memoryBarrier(MemoryBarrierBits barrier)
+{
+ xs_lwsync();
+}
+
+// ===============================================================
+// Clamping functions.
+
+void
+MacroAssembler::clampIntToUint8(Register reg)
+{
+ // If reg is < 0, then we want to clamp to 0.
+ // If reg is >= 255, then we want to clamp to 255.
+ // Essentially, compute max(reg,0), then min(reg,255).
+ // This is pretty much what isel was designed for.
+ ma_li(ScratchRegister, (int64_t)0); // make gcc happy just make it happy
+ ma_li(SecondScratchReg, 255);
+ as_cmpd(reg, ScratchRegister); // emit to CR0
+ as_cmpd(cr1, reg, SecondScratchReg); // emit to CR1
+ // Naughtiness: since ScratchRegister is r0, the load is
+ // zero anyway (this is a "mscdfr0" instruction). I just
+ // wanted to point out to you how clever I am.
+ as_isel0(reg, ScratchRegister, reg, (uint16_t)Assembler::LessThan); // CR0[LT]
+ as_isel(reg, SecondScratchReg, reg, (uint16_t)Assembler::GreaterThan, cr1); // CR1[GT]
+}
+
+//}}} check_macroassembler_style
+// ===============================================================
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_ppc64le_MacroAssembler_ppc64le_inl_h */
diff -r 485b15bb4a20 -r 9c245f4665be js/src/jit/ppc64/MacroAssembler-ppc64.cpp
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/js/src/jit/ppc64/MacroAssembler-ppc64.cpp Thu Aug 08 21:23:52 2024 -0700
@@ -0,0 +1,5904 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/ppc64/MacroAssembler-ppc64.h"
+
+#include "mozilla/CheckedInt.h"
+#include "mozilla/DebugOnly.h"
+#include "mozilla/MathAlgorithms.h"
+
+#include "jsmath.h"
+#include "jit/Bailouts.h"
+#include "jit/BaselineFrame.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/MacroAssembler.h"
+#include "jit/MoveEmitter.h"
+#include "jit/SharedICRegisters.h"
+#include "util/Memory.h"
+
+#include "vm/JitActivation.h"
+#include "vm/JSContext.h"
+#include "wasm/WasmStubs.h"
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace jit;
+
+using mozilla::Abs;
+using mozilla::CheckedInt;
+
+#if DEBUG
+#define spew(...) JitSpew(JitSpew_Codegen, __VA_ARGS__)
+#else
+#define spew(...)
+#endif
+
+#if DEBUG
+
+/* Useful class to print visual guard blocks. */
+class MASMAutoDeBlock
+{
+ private:
+ const char *blockname;
+
+ public:
+ MASMAutoDeBlock(const char *name, int line) {
+ blockname = name;
+ JitSpew(JitSpew_Codegen, "[[ CGPPC line %d: %s", line, blockname);
+ }
+
+ ~MASMAutoDeBlock() {
+ JitSpew(JitSpew_Codegen, " CGPPC: %s ]]", blockname);
+ }
+};
+#undef ADBlock
+#define ADBlock() MASMAutoDeBlock _adbx(__PRETTY_FUNCTION__, __LINE__)
+#else
+
+/* Useful macro to completely elide visual guard blocks. */
+#define ADBlock() ;
+
+#endif
+
+
+static_assert(sizeof(intptr_t) == 8, "Not 64-bit clean.");
+
+void
+MacroAssemblerPPC64Compat::convertBoolToInt32(Register src, Register dest)
+{
+ ADBlock();
+ // Note that C++ bool is only 1 byte, so zero extend it to clear the
+ // higher-order bits.
+ ma_and(dest, src, Imm32(0xff));
+}
+
+void
+MacroAssemblerPPC64Compat::convertInt32ToDouble(Register src, FloatRegister dest)
+{
+ ADBlock();
+
+ moveToDouble(src, dest);
+ as_fcfid(dest, dest); // easy!
+}
+
+void
+MacroAssemblerPPC64Compat::convertUInt64ToDouble(Register src, FloatRegister dest)
+{
+ // Approximately the same as above, but using fcfidu.
+ ADBlock();
+
+ moveToDouble(src, dest);
+ as_fcfidu(dest, dest);
+}
+
+void
+MacroAssemblerPPC64Compat::convertInt32ToDouble(const Address& src, FloatRegister dest)
+{
+ ADBlock();
+ MOZ_ASSERT(src.base != SecondScratchReg);
+
+ load32(src, SecondScratchReg);
+ convertInt32ToDouble(SecondScratchReg, dest);
+}
+
+void
+MacroAssemblerPPC64Compat::convertInt32ToDouble(const BaseIndex& src, FloatRegister dest)
+{
+ ADBlock();
+ MOZ_ASSERT(src.base != SecondScratchReg);
+
+ computeScaledAddress(src, SecondScratchReg);
+ load32(Address(SecondScratchReg, src.offset), ScratchRegister);
+ convertInt32ToDouble(ScratchRegister, dest);
+}
+
+void
+MacroAssemblerPPC64Compat::convertUInt32ToDouble(Register src, FloatRegister dest)
+{
+ ADBlock();
+ // Clear any tag and sign.
+ as_rldicl(ScratchRegister, src, 0, 32); // "clrldi"
+ asMasm().convertUInt64ToDouble(Register64(ScratchRegister), dest, InvalidReg);
+}
+
+
+void
+MacroAssemblerPPC64Compat::convertUInt32ToFloat32(Register src, FloatRegister dest)
+{
+ ADBlock();
+ as_rldicl(ScratchRegister, src, 0, 32); // "clrldi"
+ asMasm().convertUInt64ToFloat32(Register64(ScratchRegister), dest, InvalidReg);
+}
+
+void
+MacroAssemblerPPC64Compat::convertDoubleToFloat32(FloatRegister src, FloatRegister dest)
+{
+ ADBlock();
+ as_frsp(dest, src);
+}
+
+// Checks whether a double is representable as a 32-bit integer. If so, the
+// integer is written to the output register. Otherwise, a bailout is taken to
+// the given snapshot. This function overwrites the scratch float register.
+void
+MacroAssemblerPPC64Compat::convertDoubleToInt32(FloatRegister src, Register dest,
+ Label* fail, bool negativeZeroCheck)
+{
+ ADBlock();
+ MOZ_ASSERT(src != ScratchDoubleReg);
+
+ // Throw an failure if the integer conversion is invalid (VXCVI) or
+ // inexact (FI).
+ as_mtfsb0(23); // whack VXCVI; FI is not sticky
+ as_fctiwz(ScratchDoubleReg, src);
+ as_mcrfs(cr0, 3); // VXVC - FR - FI - FPRF[C]
+ as_mcrfs(cr1, 5); // resv'd - VXSOFT - VXSQRT - VXCVI
+ moveFromDouble(ScratchDoubleReg, dest);
+ as_cror(0, 2, 7); // CR0[LT] = FI->CR0[EQ] | VXCVI->CR1[SO]
+ as_srawi(dest, dest, 0); // clear upper word and sign extend
+ ma_bc(Assembler::LessThan, fail);
+
+ if (negativeZeroCheck) {
+ // If we need to check negative 0, then grab the original FPR
+ // and look at the sign bit.
+ // The MIPS version happily clobbers dest from the beginning, so
+ // no worries doing this check here to save some work.
+ Label done;
+ MOZ_ASSERT(dest != ScratchRegister);
+
+ // Don't bother if the result was not zero.
+ as_cmpldi(dest, 0);
+ ma_bc(Assembler::NotEqual, &done, ShortJump);
+
+ // Damn, the result was zero. Treat as a 64-bit int and check sign.
+ moveFromDouble(src, ScratchRegister);
+ as_cmpdi(ScratchRegister, 0);
+ ma_bc(Assembler::LessThan, fail);
+
+ bind(&done);
+ }
+}
+
+// Checks whether a float32 is representable as a 32-bit integer.
+void
+MacroAssemblerPPC64Compat::convertFloat32ToInt32(FloatRegister src, Register dest,
+ Label* fail, bool negativeZeroCheck)
+{
+ // Since 32-bit and 64-bit FPRs are the same registers, use the same
+ // routine above.
+ ADBlock();
+ convertDoubleToInt32(src, dest, fail, negativeZeroCheck);
+}
+
+// Same, but 64-bit. XXX: consolidate with 32-bit version, minimal difference
+void
+MacroAssemblerPPC64Compat::convertDoubleToPtr(FloatRegister src, Register dest,
+ Label* fail, bool negativeZeroCheck)
+{
+ ADBlock();
+ MOZ_ASSERT(src != ScratchDoubleReg);
+
+ // Throw an failure if the integer conversion is invalid (VXCVI) or
+ // inexact (FI).
+ as_mtfsb0(23); // whack VXCVI; FI is not sticky
+ as_fctidz(ScratchDoubleReg, src);
+ as_mcrfs(cr0, 3); // VXVC - FR - FI - FPRF[C]
+ as_mcrfs(cr1, 5); // resv'd - VXSOFT - VXSQRT - VXCVI
+ // Don't clear the upper word or sign extend as we do for 32-bit.
+ moveFromDouble(ScratchDoubleReg, dest);
+ as_cror(0, 2, 7); // CR0[LT] = FI->CR0[EQ] | VXCVI->CR1[SO]
+ ma_bc(Assembler::LessThan, fail);
+
+ if (negativeZeroCheck) {
+ // If we need to check negative 0, then grab the original FPR
+ // and look at the sign bit.
+ // The MIPS version happily clobbers dest from the beginning, so
+ // no worries doing this check here to save some work.
+ Label done;
+ MOZ_ASSERT(dest != ScratchRegister);
+
+ // Don't bother if the result was not zero.
+ as_cmpldi(dest, 0);
+ ma_bc(Assembler::NotEqual, &done, ShortJump);
+
+ // Damn, the result was zero. Treat as a 64-bit int and check sign.
+ moveFromDouble(src, ScratchRegister);
+ as_cmpdi(ScratchRegister, 0);
+ ma_bc(Assembler::LessThan, fail);
+
+ bind(&done);
+ }
+}
+
+void
+MacroAssemblerPPC64Compat::convertFloat32ToDouble(FloatRegister src, FloatRegister dest)
+{
+ ADBlock();
+ // Although the ISA allows us to treat single and double FPRs mostly
+ // interchangeably, the Wasm spec requires that NaNs be canonicalized,
+ // essentially making them qNaNs. frsp does this in the other direction
+ // but VSX conversion does no such cleanup in this direction. The most
+ // straightforward workaround is to just frsp again: the value is already
+ // supposed to be 32-bit, and the next 64-bit operation will upconvert.
+ // Unfortunately in opt builds we don't know if we're in Wasm code or not,
+ // so we have to do this everywhere (grumpf); it's not necessary in
+ // non-Wasm code (i.e., can't check IsCompilingWasm()). This may also not
+ // be needed on non-VSX CPUs, which is some consolation for their heavier
+ // float-GPR conversions.
+ as_frsp(dest, src); // even if dest == src
+}
+
+void
+MacroAssemblerPPC64Compat::convertInt32ToFloat32(Register src, FloatRegister dest)
+{
+ ADBlock();
+ moveToDouble(src, dest);
+ // Enforce rounding mode 0b00 (round-to-nearest ties-to-even).
+// XXX: make this into a round followed by fcfids
+ as_mtfsfi(7, 0);
+ as_fcfids(dest, dest);
+}
+
+void
+MacroAssemblerPPC64Compat::convertInt32ToFloat32(const Address& src, FloatRegister dest)
+{
+ ADBlock();
+MOZ_CRASH("NYI convertInt32ToFloat32");
+xs_trap();
+ ma_li(ScratchRegister, ImmWord(src.offset));
+ as_lfiwax(dest, src.base, ScratchRegister);
+ as_fcfid(dest, dest);
+}
+
+void
+MacroAssemblerPPC64Compat::movq(Register rs, Register rd)
+{
+ ma_move(rd, rs);
+}
+
+void
+MacroAssemblerPPC64::ma_li(Register dest, CodeLabel* label)
+{
+ BufferOffset bo = m_buffer.nextOffset();
+ ma_liPatchable(dest, ImmWord(/* placeholder */ 0));
+ label->patchAt()->bind(bo.getOffset());
+ label->setLinkMode(CodeLabel::MoveImmediate);
+}
+
+// Generate an optimized sequence to load a 64-bit immediate.
+void
+MacroAssemblerPPC64::ma_li(Register dest, int64_t value)
+{
+#if(1)
+ // Shamelessly cribbed from MIPS64 because I like its style.
+
+ // Handle low-short only values first.
+ if (-1 == (value >> 15) || 0 == (value >> 15)) {
+ // Sign extension OK!
+ xs_li(dest, value);
+ return;
+ }
+ if (0 == (value >> 16)) {
+ // Sign extension NOT OK!
+ xs_li(dest, 0);
+ as_ori(dest, dest, value);
+ return;
+ }
+
+ if (-1 == (value >> 31) || 0 == (value >> 31)) {
+ // Sign extension OK!
+ xs_lis(dest, uint16_t(value >> 16));
+ } else if (0 == (value >> 32)) {
+ // Sign extension NOT OK!
+ xs_lis(dest, uint16_t(value >> 16));
+ as_rldicl(dest, dest, 0, 32); // "clrldi" ~== "dinsu" for mips $zero
+ } else if (-1 == (value >> 47) || 0 == (value >> 47)) {
+ // Sign extension OK!
+ xs_lis(dest, uint16_t(value >> 32));
+ if (uint16_t(value >> 16)) {
+ as_ori(dest, dest, uint16_t(value >> 16));
+ }
+ as_rldicr(dest, dest, 16, 47); // "sld" == "dsll"
+ } else if (0 == (value >> 48)) {
+ // Sign extension NOT OK!
+ xs_lis(dest, uint16_t(value >> 32));
+ as_rldicl(dest, dest, 0, 32); // "clrldi" ~== "dinsu" for mips $zero
+ if (uint16_t(value >> 16)) {
+ as_ori(dest, dest, uint16_t(value >> 16));
+ }
+ as_rldicr(dest, dest, 16, 47); // "sld" == "dsll"
+ } else {
+ // Sign extension ... IRRELEVANT!
+ xs_lis(dest, uint16_t(value >> 48));
+ if (uint16_t(value >> 32)) {
+ as_ori(dest, dest, uint16_t(value >> 32));
+ }
+ as_rldicr(dest, dest, 32, 31);
+ if (uint16_t(value >> 16)) {
+ as_oris(dest, dest, uint16_t(value >> 16));
+ }
+ }
+
+ // Lowest short handled separately.
+ if (uint16_t(value)) {
+ as_ori(dest, dest, uint16_t(value));
+ }
+#else
+ uint64_t bits = (uint64_t)value;
+ bool loweronly = true;
+
+ // Handle trivial 16-bit quantities.
+ if (value > -32769 && value < 32768) {
+ // fits in 16 low bits
+ xs_li(dest, value); // mscdfr0 asserts
+ return;
+ }
+ if ((bits & 0xffffffff8000ffff) == 0 || // sign extension!
+ (bits & 0xffffffff8000ffff) == 0xffffffff80000000) {
+ // fits in 16 high bits
+ xs_lis(dest, value >> 16); // mscdfr0 asserts
+ return;
+ }
+
+ // Emit optimized sequence based on occupied bits.
+ if (bits & 0xffff000000000000) {
+ // Need to set upper word and shift.
+ xs_lis(dest, bits >> 48);
+ if (bits & 0x0000ffff00000000) {
+ as_ori(dest, dest, (bits >> 32) & 0xffff);
+ }
+ as_rldicr(dest, dest, 32, 31);
+ loweronly = false;
+ } else if (bits & 0x0000ffff00000000) {
+ if (bits & 0x0000800000000000) { // sign extension!
+ xs_li(dest, 0);
+ as_ori(dest, dest, (bits >> 32) & 0xffff);
+ } else {
+ xs_li(dest, (bits >> 32) & 0xffff);
+ }
+ as_rldicr(dest, dest, 32, 31);
+ loweronly = false;
+ } else if ((bits & 0x80000000) || (bits & 0x00008000)) {
+ // No upper bits were set, so we can't use addi(s) for the lower word
+ // or it will improperly sign-extend.
+ xs_li(dest, 0);
+ loweronly = false;
+ }
+
+ // Now the lower word. Don't clobber the upper word!
+ bits &= 0x00000000ffffffff;
+ if (bits & 0xffff0000) {
+ if (loweronly) {
+ xs_lis(dest, bits >> 16);
+ } else {
+ as_oris(dest, dest, bits >> 16);
+ }
+ if (bits & 0x0000ffff) {
+ as_ori(dest, dest, bits & 0xffff);
+ }
+ } else if (bits & 0x0000ffff) {
+ if (loweronly) {
+ xs_li(dest, bits & 0xffff);
+ } else {
+ as_ori(dest, dest, bits & 0xffff);
+ }
+ }
+#endif
+}
+void
+MacroAssemblerPPC64::ma_li(Register dest, ImmWord imm)
+{
+ ADBlock();
+ ma_li(dest, (int64_t)imm.value);
+}
+
+void
+MacroAssemblerPPC64::ma_li(Register dest, Imm64 imm)
+{
+ ADBlock();
+ ma_li(dest, (int64_t)imm.value);
+}
+
+// This generates immediate loads as well, but always in the
+// long form so that they can be patched.
+void
+MacroAssemblerPPC64::ma_liPatchable(Register dest, ImmPtr imm)
+{
+ ma_liPatchable(dest, ImmWord(uintptr_t(imm.value)));
+}
+
+void
+MacroAssemblerPPC64::ma_liPatchable(Register dest, ImmWord imm)
+{
+ // 64-bit load.
+ m_buffer.ensureSpace(5 * sizeof(uint32_t));
+ xs_lis(dest, Imm16::Upper(Imm32(imm.value >> 32)).encode());
+ as_ori(dest, dest, Imm16::Lower(Imm32(imm.value >> 32)).encode());
+ as_rldicr(dest, dest, 32, 31);
+ as_oris(dest, dest, Imm16::Upper(Imm32(imm.value)).encode());
+ as_ori(dest, dest, Imm16::Lower(Imm32(imm.value)).encode());
+}
+
+void
+MacroAssemblerPPC64::ma_dnegu(Register rd, Register rs)
+{
+ as_neg(rd, rs);
+}
+
+// Shifts
+void
+MacroAssemblerPPC64::ma_dsll(Register rd, Register rt, Imm32 shift)
+{
+ MOZ_ASSERT(shift.value < 64);
+ as_rldicr(rd, rt, shift.value, 63-(shift.value)); // "sldi"
+}
+
+void
+MacroAssemblerPPC64::ma_dsrl(Register rd, Register rt, Imm32 shift)
+{
+ MOZ_ASSERT(shift.value < 64);
+ as_rldicl(rd, rt, 64-(shift.value), shift.value); // "srdi"
+}
+
+void
+MacroAssemblerPPC64::ma_dsll(Register rd, Register rt, Register shift)
+{
+ as_sld(rd, rt, shift);
+}
+
+void
+MacroAssemblerPPC64::ma_dsrl(Register rd, Register rt, Register shift)
+{
+ as_srd(rd, rt, shift);
+}
+
+void
+MacroAssemblerPPC64::ma_dins(Register rt, Register rs, Imm32 pos, Imm32 size)
+{
+ as_rldimi(rt, rs, 64-(pos.value + size.value), pos.value); // "insrdi"
+}
+
+void
+MacroAssemblerPPC64::ma_dext(Register rt, Register rs, Imm32 pos, Imm32 size)
+{
+ // MIPS dext is right-justified, so use rldicl to simulate.
+MOZ_CRASH("ma_dext whyyyy");
+xs_trap(); // not sure if trap
+ as_rldicl(rt, rs, (pos.value + size.value), 64 - (size.value)); // "extrdi"
+}
+
+void
+MacroAssemblerPPC64::ma_dctz(Register rd, Register rs)
+{
+ as_cnttzd(rd, rs);
+}
+
+// Arithmetic-based ops.
+
+// Add.
+void
+MacroAssemblerPPC64::ma_add(Register rd, Register rs, Imm32 imm)
+{
+ MOZ_ASSERT(rs != ScratchRegister);
+ if (Imm16::IsInSignedRange(imm.value)) {
+ as_addi(rd, rs, imm.value);
+ } else {
+ ma_li(ScratchRegister, imm);
+ as_add(rd, rs, ScratchRegister);
+ }
+}
+
+void
+MacroAssemblerPPC64::ma_add(Register rd, Register rs)
+{
+ as_add(rd, rd, rs);
+}
+
+void
+MacroAssemblerPPC64::ma_add(Register rd, Imm32 imm)
+{
+ ma_add(rd, rd, imm);
+}
+
+void
+MacroAssemblerPPC64::ma_addTestOverflow(Register rd, Register rs, Register rt, Label* overflow, bool is32)
+{
+ // MIPS clobbers rd, so we can too.
+ ADBlock();
+ MOZ_ASSERT(rs != ScratchRegister);
+ MOZ_ASSERT(rt != ScratchRegister);
+ // This is testing a 32-bit overflow, so we need to whack and test
+ // XER[OV32].
+ xs_li(ScratchRegister, 0);
+ xs_mtxer(ScratchRegister);
+ as_addo(rd, rs, rt);
+ if (is32) {
+ // Handled in assembler.
+ ma_bc(Assembler::Overflow, overflow);
+ } else {
+ // Do it manually. (XXX: Hoist into assembler too?)
+ as_mcrxrx(cr0);
+ ma_bc(Assembler::LessThan, overflow); // OV bit
+ }
+}
+
+void
+MacroAssemblerPPC64::ma_addTestOverflow(Register rd, Register rs, Imm32 imm, Label* overflow, bool is32)
+{
+ // There is no addio, daddy-o, so use the regular overflow, yo.
+ ADBlock();
+ MOZ_ASSERT(rs != SecondScratchReg);
+
+ ma_li(SecondScratchReg, imm);
+ ma_addTestOverflow(rd, rs, SecondScratchReg, overflow, is32);
+}
+
+void
+MacroAssemblerPPC64::ma_addTestOverflow(Register rd, Register rs, ImmWord imm, Label* overflow, bool is32)
+{
+ ADBlock();
+ MOZ_ASSERT(rs != SecondScratchReg);
+ // This can only come in from a 64-bit routine, so assert that.
+ MOZ_ASSERT(!is32);
+
+ ma_li(SecondScratchReg, imm);
+ ma_addTestOverflow(rd, rs, SecondScratchReg, overflow, is32);
+}
+
+void
+MacroAssemblerPPC64::ma_negTestOverflow(Register rd, Label* overflow)
+{
+ ADBlock();
+ MOZ_ASSERT(rd != ScratchRegister);
+
+ xs_li(ScratchRegister, 0);
+ xs_mtxer(ScratchRegister);
+ as_nego(rd, rd);
+ ma_bc(Assembler::Overflow, overflow);
+}
+
+void
+MacroAssemblerPPC64::ma_not(Register rd, Register ra)
+{
+ ADBlock();
+ as_nor(rd, ra, ra); // OPPCC p.540
+}
+
+// Subtract.
+// ma_* subtraction functions invert operand order for as_subf.
+void
+MacroAssemblerPPC64::ma_dsubu(Register rd, Register rs, Imm32 imm)
+{
+ MOZ_ASSERT(rs != ScratchRegister);
+ if (Imm16::IsInSignedRange(-imm.value)) {
+ as_addi(rd, rs, -imm.value);
+ } else {
+ ma_li(ScratchRegister, imm);
+ as_subf(rd, ScratchRegister, rs); // T = B - A
+ }
+}
+
+void
+MacroAssemblerPPC64::ma_dsubu(Register rd, Register rs)
+{
+ as_subf(rd, rs, rd); // T = B - A
+}
+
+void
+MacroAssemblerPPC64::ma_dsubu(Register rd, Imm32 imm)
+{
+ ma_dsubu(rd, rd, imm);
+}
+
+void
+MacroAssemblerPPC64::ma_subTestOverflow(Register rd, Register rs, Register rt, Label* overflow, bool is32)
+{
+ // MIPS clobbers rd, so we can too.
+ ADBlock();
+ MOZ_ASSERT(rs != ScratchRegister);
+ MOZ_ASSERT(rt != ScratchRegister);
+ xs_li(ScratchRegister, 0);
+ xs_mtxer(ScratchRegister);
+ as_subfo(rd, rt, rs); // T = B - A
+ if (is32) {
+ // This is a 32-bit operation, so we need to whack and test XER[OV32].
+ // Handled in assembler.
+ ma_bc(Assembler::Overflow, overflow);
+ } else {
+ // Do it manually. (XXX: Hoist into assembler too?)
+ as_mcrxrx(cr0);
+ ma_bc(Assembler::LessThan, overflow); // OV bit
+ }
+}
+
+void
+MacroAssemblerPPC64::ma_subTestOverflow(Register rd, Register rs, Imm32 imm, Label* overflow, bool is32)
+{
+ ADBlock();
+ if (imm.value != INT32_MIN) {
+ asMasm().ma_addTestOverflow(rd, rs, Imm32(-imm.value), overflow, is32);
+ } else {
+ ma_li(ScratchRegister, Imm32(imm.value));
+ asMasm().ma_subTestOverflow(rd, rs, ScratchRegister, overflow, is32);
+ }
+}
+
+// Memory.
+// In these master GPR load/store routines, return a FaultingCodeOffset for
+// future proofing, since our wasmLoad* and wasmStore* routines consume them
+// too.
+
+FaultingCodeOffset
+MacroAssemblerPPC64::ma_load(Register dest, Address address,
+ LoadStoreSize size, LoadStoreExtension extension)
+{
+ // ADBlock(); // spammy
+ FaultingCodeOffset fco;
+ int16_t encodedOffset;
+ //uint32_t loadInst;
+ Register base;
+ MOZ_ASSERT(extension == ZeroExtend || extension == SignExtend);
+ MOZ_ASSERT(address.base != ScratchRegister);
+ Register t = (dest == ScratchRegister && base == SecondScratchReg) ? ThirdScratchReg : (dest == ScratchRegister) ? SecondScratchReg : ScratchRegister;
+
+ // TODO: Some of this logic is repeated.
+ if (!Imm16::IsInSignedRange(address.offset)) {
+ MOZ_ASSERT(address.base != SecondScratchReg);
+ ma_li(SecondScratchReg, Imm32(address.offset));
+ as_add(SecondScratchReg, address.base, SecondScratchReg);
+ base = SecondScratchReg;
+ encodedOffset = 0;
+ } else {
+ encodedOffset = Imm16(address.offset).encode();
+ base = address.base;
+ }
+
+ // It is entirely possible for the encodedOffset to trigger an
+ // unaligned load (for example, this regularly occurs with the Baseline
+ // Interpreter). We should see if there is potential for optimization,
+ // but the operation is deliberate, and we will fall back on the hardware
+ // to do the unaligned load as necessary.
+ switch (size) {
+ case SizeByte:
+ fco = FaultingCodeOffset(currentOffset());
+ as_lbz(dest, base, encodedOffset);
+ if (SignExtend == extension)
+ as_extsb(dest, dest);
+ break;
+ case SizeHalfWord:
+ fco = FaultingCodeOffset(currentOffset());
+ if (SignExtend == extension)
+ as_lha(dest, base, encodedOffset);
+ else
+ as_lhz(dest, base, encodedOffset);
+ break;
+ case SizeWord:
+ if (ZeroExtend == extension) {
+ fco = FaultingCodeOffset(currentOffset());
+ as_lwz(dest, base, encodedOffset);
+ } else {
+ // lwa only valid if the offset is word-aligned.
+ if (encodedOffset & 0x03) {
+ MOZ_ASSERT(base != t);
+ ma_li(t, encodedOffset);
+ fco = FaultingCodeOffset(currentOffset());
+ as_lwax(dest, base, t);
+ } else {
+ fco = FaultingCodeOffset(currentOffset());
+ as_lwa(dest, base, encodedOffset);
+ }
+ }
+ break;
+ case SizeDouble:
+ // ld only valid if the offset is word-aligned.
+ if (encodedOffset & 0x03) {
+ MOZ_ASSERT(base != t);
+ ma_li(t, encodedOffset);
+ fco = FaultingCodeOffset(currentOffset());
+ as_ldx(dest, base, t);
+ } else {
+ fco = FaultingCodeOffset(currentOffset());
+ as_ld(dest, base, encodedOffset);
+ }
+ break;
+ default:
+ MOZ_CRASH("Invalid argument for ma_load");
+ }
+
+ return fco;
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64::ma_store(Register data, Address address, LoadStoreSize size)
+{
+ //ADBlock(); // spammy
+ int16_t encodedOffset;
+ //uint32_t loadInst = 0;
+ FaultingCodeOffset fco;
+ Register base;
+ MOZ_ASSERT(address.base != ScratchRegister);
+
+ if (!Imm16::IsInSignedRange(address.offset)) {
+ MOZ_ASSERT(address.base != SecondScratchReg);
+ ma_li(SecondScratchReg, Imm32(address.offset));
+ as_add(SecondScratchReg, address.base, SecondScratchReg);
+ base = SecondScratchReg;
+ encodedOffset = 0;
+ } else {
+ encodedOffset = Imm16(address.offset).encode();
+ base = address.base;
+ }
+
+ switch (size) {
+ case SizeByte:
+ fco = FaultingCodeOffset(currentOffset());
+ as_stb(data, base, encodedOffset);
+ break;
+ case SizeHalfWord:
+ fco = FaultingCodeOffset(currentOffset());
+ as_sth(data, base, encodedOffset);
+ break;
+ case SizeWord:
+ fco = FaultingCodeOffset(currentOffset());
+ as_stw(data, base, encodedOffset);
+ break;
+ case SizeDouble:
+ // std only valid if the offset is word-aligned.
+ if (encodedOffset & 0x03) {
+ Register t = (data == ScratchRegister && base == SecondScratchReg) ? ThirdScratchReg : (data == ScratchRegister) ? SecondScratchReg : ScratchRegister;
+ MOZ_ASSERT(base != t);
+ MOZ_ASSERT(data != t);
+
+ ma_li(t, encodedOffset);
+ fco = FaultingCodeOffset(currentOffset());
+ as_stdx(data, base, t);
+ } else {
+ fco = FaultingCodeOffset(currentOffset());
+ as_std(data, base, encodedOffset);
+ }
+ break;
+ default:
+ MOZ_CRASH("Invalid argument for ma_store");
+ }
+
+ return fco;
+}
+
+void
+MacroAssemblerPPC64Compat::computeScaledAddress(const BaseIndex& address, Register dest)
+{
+ // TODO: The non-shift case can be essentially an ldx or stdx (or whatever
+ // size), so we should figure out a way to signal the caller that this
+ // optimization is possible.
+ int32_t shift = Imm32::ShiftOf(address.scale).value;
+ if (shift) {
+ MOZ_ASSERT(address.base != ScratchRegister);
+ ma_dsll(ScratchRegister, address.index, Imm32(shift)); // 64-bit needed
+ as_add(dest, address.base, ScratchRegister);
+ } else {
+ as_add(dest, address.base, address.index);
+ }
+}
+
+void
+MacroAssemblerPPC64::ma_pop(Register r)
+{
+ ADBlock();
+ MOZ_ASSERT(sizeof(uintptr_t) == 8);
+ as_ld(r, StackPointer, 0);
+ as_addi(StackPointer, StackPointer, sizeof(uintptr_t));
+}
+
+void
+MacroAssemblerPPC64::ma_push(Register r)
+{
+ ADBlock();
+ MOZ_ASSERT(sizeof(uintptr_t) == 8);
+ as_stdu(r, StackPointer, (int32_t)-sizeof(intptr_t));
+}
+
+// Branches when done from within PPC-specific code.
+void
+MacroAssemblerPPC64::ma_bc(Condition c, Label* l, JumpKind jumpKind)
+{
+ // Shorthand for a branch based on CR0.
+ ma_bc(cr0, c, l, jumpKind);
+}
+
+void
+MacroAssemblerPPC64::ma_bc(DoubleCondition c, Label *l, JumpKind jumpKind)
+{
+ ma_bc(cr0, c, l, jumpKind);
+}
+
+void
+MacroAssemblerPPC64::ma_bc(DoubleCondition c, FloatRegister lhs,
+ FloatRegister rhs, Label *label, JumpKind jumpKind)
+{
+ ADBlock();
+
+ compareFloatingPoint(lhs, rhs, c);
+ ma_bc(c, label, jumpKind);
+}
+
+template <typename T>
+void
+MacroAssemblerPPC64::ma_bc(CRegisterID cr, T c, Label* label, JumpKind jumpKind)
+{
+ ADBlock();
+ // Branch on the condition bit in the specified condition register.
+ // TODO: Support likely bits.
+ spew("bc .Llabel %p @ %08x", label, currentOffset());
+ if (label->bound()) {
+ int64_t offset = label->offset() - m_buffer.nextOffset().getOffset();
+ spew("# target offset: %08x (diff: %ld)\n", label->offset(), offset);
+
+ if (BOffImm16::IsInSignedRange(offset))
+ jumpKind = ShortJump;
+
+ if (jumpKind == ShortJump) {
+ MOZ_ASSERT(BOffImm16::IsInSignedRange(offset));
+ as_bc(BOffImm16(offset).encode(), c, cr, NotLikelyB, DontLinkB);
+ return;
+ }
+
+ // Generate a long branch stanza, but invert the sense so that we
+ // can run a short branch, assuming the "real" branch is not taken.
+ // However, overflow doesn't do reversed sense, so we do "footwork."
+ m_buffer.ensureSpace(12 * sizeof(uint32_t)); // Worst case + CR ops
+ if (c & ConditionOnlyXER) {
+ // This is why we don't need to account for the mcrxrx in
+ // Assembler-ppc64, because we generate this:
+ //
+ // bc cond .+8 (DO NOT ADJUST)
+ // b .+32
+ // long jump
+ as_bc(2 * sizeof(uint32_t), c, cr, NotLikelyB, DontLinkB);
+ as_b(8 * sizeof(uint32_t));
+ } else {
+ as_bc(8 * sizeof(uint32_t), InvertCondition(c), cr, NotLikelyB, DontLinkB);
+ }
+ addLongJump(nextOffset(), BufferOffset(label->offset()));
+ ma_liPatchable(SecondScratchReg, ImmWord(LabelBase::INVALID_OFFSET)); // 5
+ xs_mtctr(SecondScratchReg); // 6
+ as_bctr(); // 7
+ return;
+ }
+
+ // Generate open jump and link it to a label.
+ // Second word holds a pointer to the next branch in label's chain.
+ uint32_t nextInChain = label->used() ? label->offset() : LabelBase::INVALID_OFFSET;
+
+ if (jumpKind == ShortJump) {
+ // Store the condition with a dummy branch, plus the next in chain.
+ // Unfortunately there is no way to make this take up less than two
+ // instructions, so we end up burning a nop at link time. Make the
+ // whole branch continuous in the buffer.
+ m_buffer.ensureSpace(4 * sizeof(uint32_t));
+
+ // Use a dummy short jump. This includes all the branch encoding, so
+ // we just have to change the offset at link time.
+ BufferOffset bo = as_bc(4, c, cr, NotLikelyB, DontLinkB);
+ spew(".long %08x ; next in chain", nextInChain);
+ writeInst(nextInChain);
+ if (!oom())
+ label->use(bo.getOffset());
+ return;
+ }
+
+ // As above with a reverse-sense long stanza.
+ BufferOffset bo;
+ m_buffer.ensureSpace(12 * sizeof(uint32_t)); // Worst case, with CR ops
+ if (c & ConditionOnlyXER) {
+ // bc cond .+8 (DO NOT ADJUST: see above)
+ // b .+32
+ // long jump
+ as_bc(2 * sizeof(uint32_t), c, cr, NotLikelyB, DontLinkB);
+ as_b(8 * sizeof(uint32_t));
+ bo = xs_trap_tagged(BTag); // don't try to flip sense when optimizing
+ } else {
+ as_bc(8 * sizeof(uint32_t), InvertCondition(c), cr, NotLikelyB, DontLinkB);
+ bo = xs_trap_tagged(BCTag);
+ }
+ spew(".long %08x ; next in chain", nextInChain);
+ // The tagged trap must be the offset, not the leading bc. See Assembler::bind and
+ // Assembler::retarget for why.
+ writeInst(nextInChain);
+ if (!oom())
+ label->use(bo.getOffset());
+ // Leave space for potential long jump.
+ as_nop(); // rldicr
+ as_nop(); // oris
+ as_nop(); // ori
+ as_nop(); // mtctr
+ as_nop(); // bctr
+}
+
+void
+MacroAssemblerPPC64::ma_bc(Register lhs, ImmWord imm, Label* label, Condition c, JumpKind jumpKind)
+{
+ if (imm.value <= INT32_MAX) {
+ ma_bc64(lhs, Imm32(uint32_t(imm.value)), label, c, jumpKind);
+ } else {
+ MOZ_ASSERT(lhs != ScratchRegister);
+ ma_li(ScratchRegister, imm);
+ ma_bc(lhs, ScratchRegister, label, c, jumpKind);
+ }
+}
+
+void
+MacroAssemblerPPC64::ma_bc(Register lhs, Address addr, Label* label, Condition c, JumpKind jumpKind)
+{
+ MOZ_ASSERT(lhs != ScratchRegister);
+ ma_load(ScratchRegister, addr, SizeDouble);
+ ma_bc(lhs, ScratchRegister, label, c, jumpKind);
+}
+
+void
+MacroAssemblerPPC64::ma_bc(Address addr, Imm32 imm, Label* label, Condition c, JumpKind jumpKind)
+{
+ ma_load(SecondScratchReg, addr, SizeDouble);
+ ma_bc(SecondScratchReg, imm, label, c, jumpKind);
+}
+
+void
+MacroAssemblerPPC64::ma_bc(Address addr, ImmGCPtr imm, Label* label, Condition c, JumpKind jumpKind)
+{
+ ma_load(SecondScratchReg, addr, SizeDouble);
+ ma_bc(SecondScratchReg, imm, label, c, jumpKind);
+}
+
+void
+MacroAssemblerPPC64::ma_bal(Label* label) // The whole world has gone MIPS, I tell ya.
+{
+ ADBlock();
+
+ // Branch to a subroutine.
+ spew("bl .Llabel %p", label);
+ if (label->bound()) {
+ // An entire 7-instruction stanza must be generated so that no matter how this
+ // is patched, the return address is the same (i.e., the instruction after the
+ // stanza). If this is a short branch, then it's 6 nops with the bl at the end.
+ BufferOffset b(label->offset());
+ m_buffer.ensureSpace(7 * sizeof(uint32_t));
+ BufferOffset dest = nextOffset();
+ int64_t offset = label->offset() - (dest.getOffset() + 6*sizeof(uint32_t));
+ if (JOffImm26::IsInRange(offset)) {
+ JOffImm26 j(offset);
+
+ as_nop();
+ as_nop();
+ as_nop();
+ as_nop(); // Yawn.
+ as_nop();
+ as_nop(); // Sigh.
+ as_b(j, RelativeBranch, LinkB);
+ return;
+ }
+
+ // Although this is to Ion code, use r12 to keep calls "as expected."
+ addLongJump(dest, b);
+ ma_liPatchable(SecondScratchReg, ImmWord(LabelBase::INVALID_OFFSET));
+ xs_mtctr(SecondScratchReg);
+ as_bctr(LinkB); // bctrl
+ return;
+ }
+
+ // Second word holds a pointer to the next branch in label's chain.
+ uint32_t nextInChain = label->used() ? label->offset() : LabelBase::INVALID_OFFSET;
+ // Keep the whole branch stanza continuous in the buffer.
+ m_buffer.ensureSpace(7 * sizeof(uint32_t));
+ // Insert a tagged trap so the patcher knows what this is supposed to be.
+ BufferOffset bo = xs_trap_tagged(CallTag);
+ writeInst(nextInChain);
+ if (!oom())
+ label->use(bo.getOffset());
+ // Leave space for long jump.
+ as_nop(); // rldicr
+ as_nop(); // oris
+ as_nop(); // ori
+ as_nop(); // mtctr
+ as_nop(); // bctrl
+}
+
+void
+MacroAssemblerPPC64::ma_cmp_set(Register rd, Register rs, ImmWord imm, Condition c, bool useCmpw)
+{
+ ADBlock();
+ if (imm.value <= INT16_MAX) { // unsigned
+ ma_cmp_set(rd, rs, Imm16(imm.value), c, useCmpw);
+ } else if (imm.value <= INT32_MAX) { // unsigned
+ ma_cmp_set(rd, rs, Imm32(imm.value), c, useCmpw);
+ } else {
+ MOZ_ASSERT(rd != ScratchRegister);
+ MOZ_ASSERT(rs != ScratchRegister);
+ ma_li(ScratchRegister, imm);
+ ma_cmp_set(rd, rs, ScratchRegister, c, useCmpw);
+ }
+}
+
+void
+MacroAssemblerPPC64::ma_cmp_set(Register rd, Register rs, ImmPtr imm, Condition c, bool useCmpw)
+{
+ ma_cmp_set(rd, rs, ImmWord(uintptr_t(imm.value)), c, useCmpw);
+}
+
+void
+MacroAssemblerPPC64::ma_cmp_set(Register rd, Address addr, Register rs, Condition c, bool useCmpw)
+{
+ ADBlock();
+ MOZ_ASSERT(rd != ScratchRegister);
+ MOZ_ASSERT(rs != ScratchRegister);
+
+ asMasm().loadPtr(addr, ScratchRegister);
+ ma_cmp_set(rd, ScratchRegister, rs, c, useCmpw);
+}
+
+void
+MacroAssemblerPPC64::ma_cmp_set(Register rd, Address addr, Imm32 imm, Condition c, bool useCmpw)
+{
+ ADBlock();
+ MOZ_ASSERT(rd != ScratchRegister);
+ MOZ_ASSERT(rd != SecondScratchReg);
+
+ asMasm().loadPtr(addr, ScratchRegister);
+ ma_li(SecondScratchReg, imm);
+ ma_cmp_set(rd, ScratchRegister, SecondScratchReg, c, useCmpw);
+}
+
+void
+MacroAssemblerPPC64::ma_cmp_set(Register rd, Address addr, Imm64 imm, Condition c, bool useCmpw)
+{
+ ADBlock();
+ MOZ_ASSERT(rd != ScratchRegister);
+ MOZ_ASSERT(rd != SecondScratchReg);
+ MOZ_ASSERT(!useCmpw);
+
+ asMasm().loadPtr(addr, ScratchRegister);
+ ma_li(SecondScratchReg, imm);
+ ma_cmp_set(rd, ScratchRegister, SecondScratchReg, c, useCmpw);
+}
+
+// fp instructions
+void
+MacroAssemblerPPC64::ma_lid(FloatRegister dest, double value)
+{
+ ADBlock();
+ ImmWord imm(mozilla::BitwiseCast<uint64_t>(value));
+
+ ma_li(ScratchRegister, imm);
+ asMasm().moveToDouble(ScratchRegister, dest);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64::ma_ls(FloatRegister ft, Address address)
+{
+ FaultingCodeOffset fco;
+ if (Imm16::IsInSignedRange(address.offset)) {
+ fco = FaultingCodeOffset(currentOffset());
+ as_lfs(ft, address.base, address.offset);
+ } else {
+ MOZ_ASSERT(address.base != ScratchRegister);
+ ma_li(ScratchRegister, Imm32(address.offset));
+ fco = FaultingCodeOffset(currentOffset());
+ as_lfsx(ft, address.base, ScratchRegister);
+ }
+ return fco;
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64::ma_ld(FloatRegister ft, Address address)
+{
+ FaultingCodeOffset fco;
+ if (Imm16::IsInSignedRange(address.offset)) {
+ fco = FaultingCodeOffset(currentOffset());
+ as_lfd(ft, address.base, address.offset);
+ } else {
+ MOZ_ASSERT(address.base != ScratchRegister);
+ ma_li(ScratchRegister, Imm32(address.offset));
+ fco = FaultingCodeOffset(currentOffset());
+ as_lfdx(ft, address.base, ScratchRegister);
+ }
+ return fco;
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64::ma_sd(FloatRegister ft, Address address)
+{
+ FaultingCodeOffset fco;
+ if (Imm16::IsInSignedRange(address.offset)) {
+ fco = FaultingCodeOffset(currentOffset());
+ as_stfd(ft, address.base, address.offset);
+ } else {
+ MOZ_ASSERT(address.base != ScratchRegister);
+ ma_li(ScratchRegister, Imm32(address.offset));
+ fco = FaultingCodeOffset(currentOffset());
+ as_stfdx(ft, address.base, ScratchRegister);
+ }
+ return fco;
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64::ma_ss(FloatRegister ft, Address address)
+{
+ FaultingCodeOffset fco;
+ if (Imm16::IsInSignedRange(address.offset)) {
+ fco = FaultingCodeOffset(currentOffset());
+ as_stfs(ft, address.base, address.offset);
+ } else {
+ MOZ_ASSERT(address.base != ScratchRegister);
+ ma_li(ScratchRegister, Imm32(address.offset));
+ fco = FaultingCodeOffset(currentOffset());
+ as_stfsx(ft, address.base, ScratchRegister);
+ }
+ return fco;
+}
+
+// Keep pushes and pops of floats and doubles aligned to float.
+void
+MacroAssemblerPPC64::ma_pop(FloatRegister f)
+{
+ if (f.isSingle()) {
+ as_lfs(f, StackPointer, 0);
+ } else {
+ as_lfd(f, StackPointer, 0);
+ }
+ as_addi(StackPointer, StackPointer, sizeof(double));
+}
+
+void
+MacroAssemblerPPC64::ma_push(FloatRegister f)
+{
+ if (f.isSingle())
+ as_stfsu(f, StackPointer, (int32_t)-sizeof(double));
+ else
+ as_stfdu(f, StackPointer, (int32_t)-sizeof(double));
+}
+
+bool
+MacroAssemblerPPC64Compat::buildOOLFakeExitFrame(void* fakeReturnAddr)
+{
+ asMasm().PushFrameDescriptor(FrameType::IonJS); // descriptor_
+ asMasm().Push(ImmPtr(fakeReturnAddr));
+ asMasm().Push(FramePointer);
+ return true;
+}
+
+void
+MacroAssemblerPPC64Compat::move32(Imm32 imm, Register dest)
+{
+ ADBlock();
+ //uint64_t bits = (uint64_t)((int64_t)imm.value & 0x00000000ffffffff);
+ //ma_li(dest, bits);
+// XXX: why not just cast to int64_t now that we're fixed?
+ ma_li(dest, imm);
+}
+
+void
+MacroAssemblerPPC64Compat::move32(Register src, Register dest)
+{
+ ADBlock();
+ ma_move(dest, src);
+ // Ass-U-Me we explicitly want the upper 32-bits cleared.
+ //as_rldicl(dest, src, 0, 32); // "clrldi"
+}
+
+void
+MacroAssemblerPPC64Compat::movePtr(Register src, Register dest)
+{
+ ma_move(dest, src);
+}
+void
+MacroAssemblerPPC64Compat::movePtr(ImmWord imm, Register dest)
+{
+ ma_li(dest, imm);
+}
+
+void
+MacroAssemblerPPC64Compat::movePtr(ImmGCPtr imm, Register dest)
+{
+ ma_li(dest, imm); // tags the pointer for us
+}
+
+void
+MacroAssemblerPPC64Compat::movePtr(ImmPtr imm, Register dest)
+{
+ movePtr(ImmWord(uintptr_t(imm.value)), dest);
+}
+void
+MacroAssemblerPPC64Compat::movePtr(wasm::SymbolicAddress imm, Register dest)
+{
+ append(wasm::SymbolicAccess(CodeOffset(nextOffset().getOffset()), imm));
+ ma_liPatchable(dest, ImmWord(-1));
+}
+
+CodeOffset MacroAssembler::moveNearAddressWithPatch(Register dest) {
+ return movWithPatch(ImmPtr(nullptr), dest);
+}
+
+void
+MacroAssembler::patchNearAddressMove(CodeLocationLabel loc,
+ CodeLocationLabel target)
+{
+ PatchDataWithValueCheck(loc, ImmPtr(target.raw()), ImmPtr(nullptr));
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::load8ZeroExtend(const Address& address, Register dest)
+{
+ return ma_load(dest, address, SizeByte, ZeroExtend);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::load8ZeroExtend(const BaseIndex& src, Register dest)
+{
+ return ma_load(dest, src, SizeByte, ZeroExtend);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::load8SignExtend(const Address& address, Register dest)
+{
+ return ma_load(dest, address, SizeByte, SignExtend);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::load8SignExtend(const BaseIndex& src, Register dest)
+{
+ return ma_load(dest, src, SizeByte, SignExtend);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::load16ZeroExtend(const Address& address, Register dest)
+{
+ return ma_load(dest, address, SizeHalfWord, ZeroExtend);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::load16ZeroExtend(const BaseIndex& src, Register dest)
+{
+ return ma_load(dest, src, SizeHalfWord, ZeroExtend);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::load16SignExtend(const Address& address, Register dest)
+{
+ return ma_load(dest, address, SizeHalfWord, SignExtend);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::load16SignExtend(const BaseIndex& src, Register dest)
+{
+ return ma_load(dest, src, SizeHalfWord, SignExtend);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::load32(const Address& address, Register dest)
+{
+ // This must sign-extend for arithmetic to function correctly.
+ return ma_load(dest, address, SizeWord);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::load32(const BaseIndex& address, Register dest)
+{
+ // This must sign-extend for arithmetic to function correctly.
+ return ma_load(dest, address, SizeWord);
+}
+
+// Zero-extend versions, mostly for wasm.
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::load32ZeroExtend(const Address& address, Register dest)
+{
+ // This must sign-extend for arithmetic to function correctly.
+ return ma_load(dest, address, SizeWord, ZeroExtend);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::load32ZeroExtend(const BaseIndex& address, Register dest)
+{
+ // This must sign-extend for arithmetic to function correctly.
+ return ma_load(dest, address, SizeWord, ZeroExtend);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::load32(AbsoluteAddress address, Register dest)
+{
+ movePtr(ImmPtr(address.addr), SecondScratchReg);
+ return load32(Address(SecondScratchReg, 0), dest);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::load32(wasm::SymbolicAddress address, Register dest)
+{
+ movePtr(address, SecondScratchReg);
+ return load32(Address(SecondScratchReg, 0), dest);
+}
+
+void
+MacroAssemblerPPC64Compat::loadPtr(Register src, Register dest)
+{
+ if (src != dest)
+ xs_mr(dest, src);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::loadPtr(const Address& address, Register dest)
+{
+ return ma_load(dest, address, SizeDouble);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::loadPtr(const BaseIndex& src, Register dest)
+{
+ return ma_load(dest, src, SizeDouble);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::loadPtr(AbsoluteAddress address, Register dest)
+{
+ movePtr(ImmPtr(address.addr), SecondScratchReg);
+ return loadPtr(Address(SecondScratchReg, 0), dest);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::loadPtr(wasm::SymbolicAddress address, Register dest)
+{
+ movePtr(address, SecondScratchReg);
+ return loadPtr(Address(SecondScratchReg, 0), dest);
+}
+
+void
+MacroAssemblerPPC64Compat::loadPrivate(const Address& address, Register dest)
+{
+ loadPtr(address, dest);
+}
+
+/*
+void
+MacroAssemblerPPC64Compat::loadUnalignedDouble(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& src, Register temp, FloatRegister dest)
+{
+ loadDouble(src, dest);
+ asMasm().append(access, asMasm().size() - 4); // lfd terminates
+}
+
+void
+MacroAssemblerPPC64Compat::loadUnalignedFloat32(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& src, Register temp, FloatRegister dest)
+{
+ loadFloat32(src, dest);
+ asMasm().append(access, asMasm().size() - 4); // lfs terminates
+}
+*/
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::store8(Imm32 imm, const Address& address)
+{
+ MOZ_ASSERT(address.base != ScratchRegister);
+ ma_li(ScratchRegister, imm);
+ return ma_store(ScratchRegister, address, SizeByte);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::store8(Register src, const Address& address)
+{
+ return ma_store(src, address, SizeByte);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::store8(Imm32 imm, const BaseIndex& dest)
+{
+ return ma_store(imm, dest, SizeByte);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::store8(Register src, const BaseIndex& dest)
+{
+ return ma_store(src, dest, SizeByte);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::store16(Imm32 imm, const Address& address)
+{
+ MOZ_ASSERT(address.base != ScratchRegister);
+ ma_li(ScratchRegister, imm);
+ return ma_store(ScratchRegister, address, SizeHalfWord);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::store16(Register src, const Address& address)
+{
+ return ma_store(src, address, SizeHalfWord);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::store16(Imm32 imm, const BaseIndex& dest)
+{
+ return ma_store(imm, dest, SizeHalfWord);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::store16(Register src, const BaseIndex& address)
+{
+ return ma_store(src, address, SizeHalfWord);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::store32(Register src, AbsoluteAddress address)
+{
+ MOZ_ASSERT(src != SecondScratchReg);
+ movePtr(ImmPtr(address.addr), SecondScratchReg);
+ return store32(src, Address(SecondScratchReg, 0));
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::store32(Register src, const Address& address)
+{
+ return ma_store(src, address, SizeWord);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::store32(Imm32 src, const Address& address)
+{
+ MOZ_ASSERT(address.base != ScratchRegister);
+ move32(src, ScratchRegister);
+ return ma_store(ScratchRegister, address, SizeWord);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::store32(Imm32 imm, const BaseIndex& dest)
+{
+ return ma_store(imm, dest, SizeWord);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::store32(Register src, const BaseIndex& dest)
+{
+ return ma_store(src, dest, SizeWord);
+}
+
+template <typename T>
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::storePtr(ImmWord imm, T address)
+{
+ ma_li(ScratchRegister, imm);
+ return ma_store(ScratchRegister, address, SizeDouble);
+}
+
+template FaultingCodeOffset MacroAssemblerPPC64Compat::storePtr<Address>(ImmWord imm, Address address);
+template FaultingCodeOffset MacroAssemblerPPC64Compat::storePtr<BaseIndex>(ImmWord imm, BaseIndex address);
+
+template <typename T>
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::storePtr(ImmPtr imm, T address)
+{
+ return storePtr(ImmWord(uintptr_t(imm.value)), address);
+}
+
+template FaultingCodeOffset MacroAssemblerPPC64Compat::storePtr<Address>(ImmPtr imm, Address address);
+template FaultingCodeOffset MacroAssemblerPPC64Compat::storePtr<BaseIndex>(ImmPtr imm, BaseIndex address);
+
+template <typename T>
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::storePtr(ImmGCPtr imm, T address)
+{
+ movePtr(imm, ScratchRegister);
+ return storePtr(ScratchRegister, address);
+}
+
+template FaultingCodeOffset MacroAssemblerPPC64Compat::storePtr<Address>(ImmGCPtr imm, Address address);
+template FaultingCodeOffset MacroAssemblerPPC64Compat::storePtr<BaseIndex>(ImmGCPtr imm, BaseIndex address);
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::storePtr(Register src, const Address& address)
+{
+ return ma_store(src, address, SizeDouble);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::storePtr(Register src, const BaseIndex& address)
+{
+ return ma_store(src, address, SizeDouble);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64Compat::storePtr(Register src, AbsoluteAddress dest)
+{
+ MOZ_ASSERT(src != SecondScratchReg);
+ movePtr(ImmPtr(dest.addr), SecondScratchReg);
+ return storePtr(src, Address(SecondScratchReg, 0));
+}
+
+/*
+void
+MacroAssemblerPPC64Compat::storeUnalignedFloat32(const wasm::MemoryAccessDesc& access,
+ FloatRegister src, Register temp, const BaseIndex& dest)
+{
+ computeScaledAddress(dest, SecondScratchReg);
+
+ as_stfs(src, SecondScratchReg, 0);
+ append(access, asMasm().size() - 4);
+}
+
+void
+MacroAssemblerPPC64Compat::storeUnalignedDouble(const wasm::MemoryAccessDesc& access,
+ FloatRegister src, Register temp, const BaseIndex& dest)
+{
+ computeScaledAddress(dest, SecondScratchReg);
+
+ as_stfd(src, SecondScratchReg, 0);
+ append(access, asMasm().size() - 4);
+}
+*/
+
+void
+MacroAssembler::clampDoubleToUint8(FloatRegister input, Register output)
+{
+ ADBlock();
+ MOZ_ASSERT(input != ScratchDoubleReg);
+
+ // Default rounding mode 0b00 (round nearest)
+ as_fctid(ScratchDoubleReg, input);
+ moveFromDouble(ScratchDoubleReg, output);
+ clampIntToUint8(output);
+}
+
+void
+MacroAssemblerPPC64Compat::testNullSet(Condition cond, const ValueOperand& value, Register dest)
+{
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ splitTag(value, SecondScratchReg);
+ ma_cmp_set(dest, SecondScratchReg, ImmTag(JSVAL_TAG_NULL), cond);
+}
+
+void
+MacroAssemblerPPC64Compat::testObjectSet(Condition cond, const ValueOperand& value, Register dest)
+{
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ splitTag(value, SecondScratchReg);
+ ma_cmp_set(dest, SecondScratchReg, ImmTag(JSVAL_TAG_OBJECT), cond);
+}
+
+void
+MacroAssemblerPPC64Compat::testUndefinedSet(Condition cond, const ValueOperand& value, Register dest)
+{
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ splitTag(value, SecondScratchReg);
+ ma_cmp_set(dest, SecondScratchReg, ImmTag(JSVAL_TAG_UNDEFINED), cond);
+}
+
+void
+MacroAssemblerPPC64Compat::unboxInt32(const ValueOperand& operand, Register dest)
+{
+ unboxInt32(operand.valueReg(), dest);
+}
+
+void
+MacroAssemblerPPC64Compat::unboxInt32(Register src, Register dest)
+{
+ // Ensure there is no tag. This erases bits 32-63 and sign extends
+ // in one single operation.
+ as_srawi(dest, src, 0);
+}
+
+void
+MacroAssemblerPPC64Compat::unboxInt32(const Address& src, Register dest)
+{
+ load32(Address(src.base, src.offset), dest);
+}
+
+void
+MacroAssemblerPPC64Compat::unboxInt32(const BaseIndex& src, Register dest)
+{
+ computeScaledAddress(src, SecondScratchReg);
+ load32(Address(SecondScratchReg, src.offset), dest);
+}
+
+void
+MacroAssemblerPPC64Compat::unboxBoolean(const ValueOperand& operand, Register dest)
+{
+ unboxBoolean(operand.valueReg(), dest);
+}
+
+void
+MacroAssemblerPPC64Compat::unboxBoolean(Register src, Register dest)
+{
+ // Clear upper 32 bits. srawi would also work.
+ as_rldicl(dest, src, 0, 32); // "clrldi"
+}
+
+void
+MacroAssemblerPPC64Compat::unboxBoolean(const Address& src, Register dest)
+{
+ ma_load(dest, Address(src.base, src.offset), SizeWord, ZeroExtend);
+}
+
+void
+MacroAssemblerPPC64Compat::unboxBoolean(const BaseIndex& src, Register dest)
+{
+ computeScaledAddress(src, SecondScratchReg);
+ ma_load(dest, Address(SecondScratchReg, src.offset), SizeWord, ZeroExtend);
+}
+
+void
+MacroAssemblerPPC64Compat::unboxDouble(const ValueOperand& operand, FloatRegister dest)
+{
+ moveToDouble(operand.valueReg(), dest);
+}
+
+void
+MacroAssemblerPPC64Compat::unboxDouble(const Address& src, FloatRegister dest)
+{
+ ma_ld(dest, src);
+}
+
+void
+MacroAssemblerPPC64Compat::unboxDouble(const BaseIndex& src, FloatRegister dest)
+{
+ ADBlock();
+ MOZ_ASSERT(src.base != SecondScratchReg);
+
+ computeScaledAddress(src, SecondScratchReg);
+ ma_ld(dest, Address(SecondScratchReg, src.offset));
+}
+
+void
+MacroAssemblerPPC64Compat::unboxString(const ValueOperand& operand, Register dest)
+{
+ unboxNonDouble(operand, dest, JSVAL_TYPE_STRING);
+}
+
+void
+MacroAssemblerPPC64Compat::unboxString(Register src, Register dest)
+{
+ unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
+}
+
+void
+MacroAssemblerPPC64Compat::unboxString(const Address& src, Register dest)
+{
+ unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
+}
+
+void
+MacroAssemblerPPC64Compat::unboxSymbol(const ValueOperand& operand, Register dest)
+{
+ unboxNonDouble(operand, dest, JSVAL_TYPE_SYMBOL);
+}
+
+void
+MacroAssemblerPPC64Compat::unboxSymbol(Register src, Register dest)
+{
+ unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL);
+}
+
+void
+MacroAssemblerPPC64Compat::unboxSymbol(const Address& src, Register dest)
+{
+ unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL);
+}
+
+void
+MacroAssemblerPPC64Compat::unboxObject(const ValueOperand& src, Register dest)
+{
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+}
+
+void
+MacroAssemblerPPC64Compat::unboxObject(Register src, Register dest)
+{
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+}
+
+void
+MacroAssemblerPPC64Compat::unboxObject(const Address& src, Register dest)
+{
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+}
+
+void
+MacroAssemblerPPC64Compat::unboxValue(const ValueOperand& src, AnyRegister dest, JSValueType type)
+{
+ if (dest.isFloat()) {
+ Label notInt32, end;
+ asMasm().branchTestInt32(Assembler::NotEqual, src, &notInt32);
+ convertInt32ToDouble(src.valueReg(), dest.fpu());
+ ma_b(&end, ShortJump);
+ bind(&notInt32);
+ unboxDouble(src, dest.fpu());
+ bind(&end);
+ } else {
+ unboxNonDouble(src, dest.gpr(), type);
+ }
+}
+
+void
+MacroAssemblerPPC64Compat::unboxPrivate(const ValueOperand& src, Register dest)
+{
+ ma_dsll(dest, src.valueReg(), Imm32(1));
+}
+
+void
+MacroAssemblerPPC64Compat::boxDouble(FloatRegister src, const ValueOperand& dest, FloatRegister)
+{
+ moveFromDouble(src, dest.valueReg());
+}
+
+void MacroAssemblerPPC64Compat::unboxBigInt(const ValueOperand& operand,
+ Register dest) {
+ unboxNonDouble(operand, dest, JSVAL_TYPE_BIGINT);
+}
+
+void MacroAssemblerPPC64Compat::unboxBigInt(Register src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
+}
+
+void MacroAssemblerPPC64Compat::unboxBigInt(const Address& src,
+ Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
+}
+
+void
+MacroAssemblerPPC64Compat::boxNonDouble(JSValueType type, Register src,
+ const ValueOperand& dest)
+{
+ boxValue(type, src, dest.valueReg());
+}
+
+void
+MacroAssemblerPPC64Compat::boolValueToDouble(const ValueOperand& operand, FloatRegister dest)
+{
+ ADBlock();
+ convertBoolToInt32(operand.valueReg(), ScratchRegister);
+ convertInt32ToDouble(ScratchRegister, dest);
+}
+
+void
+MacroAssemblerPPC64Compat::int32ValueToDouble(const ValueOperand& operand,
+ FloatRegister dest)
+{
+ ADBlock();
+ // Use the lower bits of the operand's value register.
+ MOZ_ASSERT(operand.valueReg() != ScratchRegister);
+ as_srawi(ScratchRegister, operand.valueReg(), 0);
+ convertInt32ToDouble(ScratchRegister, dest);
+}
+
+void
+MacroAssemblerPPC64Compat::boolValueToFloat32(const ValueOperand& operand,
+ FloatRegister dest)
+{
+ ADBlock();
+ convertBoolToInt32(operand.valueReg(), ScratchRegister);
+ convertInt32ToFloat32(ScratchRegister, dest);
+}
+
+void
+MacroAssemblerPPC64Compat::int32ValueToFloat32(const ValueOperand& operand,
+ FloatRegister dest)
+{
+ ADBlock();
+ // Use the lower bits of the operand's value register.
+ MOZ_ASSERT(operand.valueReg() != ScratchRegister);
+ as_srawi(ScratchRegister, operand.valueReg(), 0);
+ convertInt32ToFloat32(ScratchRegister, dest);
+}
+
+void
+MacroAssemblerPPC64Compat::loadConstantFloat32(float f, FloatRegister dest)
+{
+ if (f == 0.0 && !signbit(f)) zeroDouble(dest);
+ else ma_lis(dest, f);
+}
+
+void
+MacroAssemblerPPC64Compat::loadInt32OrDouble(const Address& src, FloatRegister dest)
+{
+ ADBlock();
+ Label notInt32, end;
+
+MOZ_CRASH("NYI loadInt32OrDouble");
+xs_trap();
+ // If it's an int, convert it to double.
+ loadPtr(Address(src.base, src.offset), ScratchRegister);
+ ma_dsrl(SecondScratchReg, ScratchRegister, Imm32(JSVAL_TAG_SHIFT));
+ asMasm().branchTestInt32(Assembler::NotEqual, SecondScratchReg, &notInt32);
+ loadPtr(Address(src.base, src.offset), SecondScratchReg);
+ convertInt32ToDouble(SecondScratchReg, dest);
+ ma_b(&end, ShortJump);
+
+ // Not an int, just load as double.
+ bind(&notInt32);
+ ma_ld(dest, src);
+ bind(&end);
+}
+
+void
+MacroAssemblerPPC64Compat::loadInt32OrDouble(const BaseIndex& addr, FloatRegister dest)
+{
+ ADBlock();
+ Label notInt32, end;
+
+MOZ_CRASH("NYI loadInt32OrDouble BI");
+xs_trap();
+ // If it's an int, convert it to double.
+ computeScaledAddress(addr, SecondScratchReg);
+ // Since we only have one scratch, we need to stomp over it with the tag.
+ loadPtr(Address(SecondScratchReg, 0), ScratchRegister);
+ ma_dsrl(SecondScratchReg, ScratchRegister, Imm32(JSVAL_TAG_SHIFT));
+ asMasm().branchTestInt32(Assembler::NotEqual, SecondScratchReg, &notInt32);
+
+ computeScaledAddress(addr, SecondScratchReg);
+ loadPtr(Address(SecondScratchReg, 0), SecondScratchReg);
+ convertInt32ToDouble(SecondScratchReg, dest);
+ ma_b(&end, ShortJump);
+
+ // Not an int, just load as double.
+ bind(&notInt32);
+ // First, recompute the offset that had been stored in the scratch register
+ // since the scratch register was overwritten loading in the type.
+ computeScaledAddress(addr, SecondScratchReg);
+ loadDouble(Address(SecondScratchReg, 0), dest);
+ bind(&end);
+}
+
+void
+MacroAssemblerPPC64Compat::loadConstantDouble(double dp, FloatRegister dest)
+{
+ if (dp == 0.0 && !signbit(dp)) zeroDouble(dest);
+ else ma_lid(dest, dp);
+}
+
+Register
+MacroAssemblerPPC64Compat::extractObject(const Address& address, Register scratch)
+{
+ loadPtr(Address(address.base, address.offset), scratch);
+ as_rldicl(scratch, scratch, 0, 64-JSVAL_TAG_SHIFT); // clrldi the tag
+ return scratch;
+}
+
+Register
+MacroAssemblerPPC64Compat::extractTag(const Address& address, Register scratch)
+{
+ loadPtr(Address(address.base, address.offset), scratch);
+ as_rldicl(scratch, scratch, 64-JSVAL_TAG_SHIFT, JSVAL_TAG_SHIFT); // "srdi"
+ return scratch;
+}
+
+Register
+MacroAssemblerPPC64Compat::extractTag(const BaseIndex& address, Register scratch)
+{
+ computeScaledAddress(address, scratch);
+ return extractTag(Address(scratch, address.offset), scratch);
+}
+
+/////////////////////////////////////////////////////////////////
+// X86/X64-common/ARM/MIPS interface.
+/////////////////////////////////////////////////////////////////
+void
+MacroAssemblerPPC64Compat::storeValue(ValueOperand val, Operand dst)
+{
+ storeValue(val, Address(Register::FromCode(dst.base()), dst.disp()));
+}
+
+void
+MacroAssemblerPPC64Compat::storeValue(ValueOperand val, const BaseIndex& dest)
+{
+ ADBlock();
+ computeScaledAddress(dest, SecondScratchReg);
+
+ int32_t offset = dest.offset;
+ if (!Imm16::IsInSignedRange(offset)) {
+ ma_li(ScratchRegister, Imm32(offset));
+ as_add(SecondScratchReg, ScratchRegister, SecondScratchReg);
+ offset = 0;
+ }
+
+ storeValue(val, Address(SecondScratchReg, offset));
+}
+
+void
+MacroAssemblerPPC64Compat::storeValue(JSValueType type, Register reg, BaseIndex dest)
+{
+ ADBlock();
+ computeScaledAddress(dest, SecondScratchReg);
+
+ int32_t offset = dest.offset;
+ if (!Imm16::IsInSignedRange(offset)) {
+ ma_li(ScratchRegister, Imm32(offset));
+ as_add(SecondScratchReg, ScratchRegister, SecondScratchReg);
+ offset = 0;
+ }
+
+ storeValue(type, reg, Address(SecondScratchReg, offset));
+}
+
+void
+MacroAssemblerPPC64Compat::storeValue(ValueOperand val, const Address& dest)
+{
+ storePtr(val.valueReg(), dest);
+}
+
+static_assert(JSVAL_TAG_SHIFT == 47); // a lot would break if this changed ...
+void
+MacroAssemblerPPC64Compat::storeValue(JSValueType type, Register reg, Address dest)
+{
+ ADBlock();
+
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ // This isn't elegant, but saves us some scratch register contention.
+ MOZ_ASSERT(reg != ScratchRegister);
+ MOZ_ASSERT(dest.base != ScratchRegister);
+ MOZ_ASSERT(Imm16::IsInSignedRange(dest.offset + 4));
+
+ store32(reg, dest);
+ uint32_t utag = (uint32_t)JSVAL_TYPE_TO_TAG(type) << (JSVAL_TAG_SHIFT - 32);
+ // Hardcode this as ma_li doesn't generate good code here. It will
+ // never fit into a 16-bit immediate.
+ xs_lis(ScratchRegister, (utag >> 16));
+ as_ori(ScratchRegister, ScratchRegister, utag & 0x0000ffff);
+ // ENDIAN!!!
+ store32(ScratchRegister, Address(dest.base, dest.offset + 4));
+ } else {
+ MOZ_ASSERT(reg != SecondScratchReg);
+ if (dest.base == SecondScratchReg) {
+ MOZ_ASSERT(reg != ScratchRegister);
+
+ // The destination register for boxValue can't be r0, so we need
+ // the third scratch register.
+ boxValue(type, reg, ThirdScratchReg);
+ storePtr(ThirdScratchReg, dest);
+ } else {
+ boxValue(type, reg, SecondScratchReg);
+ storePtr(SecondScratchReg, dest);
+ }
+ }
+}
+
+void
+MacroAssemblerPPC64Compat::storeValue(const Value& val, Address dest)
+{
+ ADBlock();
+ if (val.isGCThing()) {
+ writeDataRelocation(val);
+ movWithPatch(ImmWord(val.asRawBits()), ScratchRegister);
+ } else {
+ ma_li(ScratchRegister, ImmWord(val.asRawBits()));
+ }
+ storePtr(ScratchRegister, dest);
+}
+
+void
+MacroAssemblerPPC64Compat::storeValue(const Value& val, BaseIndex dest)
+{
+ ADBlock();
+ computeScaledAddress(dest, SecondScratchReg);
+
+ int32_t offset = dest.offset;
+ if (!Imm16::IsInSignedRange(offset)) {
+ ma_li(ScratchRegister, Imm32(offset));
+ as_add(SecondScratchReg, ScratchRegister, SecondScratchReg);
+ offset = 0;
+ }
+ storeValue(val, Address(SecondScratchReg, offset));
+}
+
+void
+MacroAssemblerPPC64Compat::loadValue(const BaseIndex& addr, ValueOperand val)
+{
+ computeScaledAddress(addr, SecondScratchReg);
+ loadValue(Address(SecondScratchReg, addr.offset), val);
+}
+
+void
+MacroAssemblerPPC64Compat::loadValue(Address src, ValueOperand val)
+{
+ loadPtr(src, val.valueReg());
+}
+
+void
+MacroAssemblerPPC64Compat::boxValue(JSValueType type, Register src, Register dest) {
+ MOZ_ASSERT(dest != ScratchRegister);
+ if (dest != src)
+ ma_move(dest, src);
+ ma_li(ScratchRegister, ImmTag(JSVAL_TYPE_TO_TAG(type)));
+ // It is possible we sign extended too far, and this matters to code
+ // working directly with values, so clear the bits for int32 and bool.
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ as_rldicl(dest, dest, 0, 32); // "clrldi"
+ }
+ // Shift the tag left and mask in the value, which is pretty much
+ // what rldimi/rlwimi were created for.
+ as_rldimi(dest, ScratchRegister, JSVAL_TAG_SHIFT, 0);
+}
+
+void
+MacroAssemblerPPC64Compat::tagValue(JSValueType type, Register payload, ValueOperand dest)
+{
+ ADBlock();
+ boxValue(type, payload, dest.valueReg());
+}
+
+void
+MacroAssemblerPPC64Compat::pushValue(ValueOperand val)
+{
+ as_stdu(val.valueReg(), StackPointer, -8);
+}
+
+void
+MacroAssemblerPPC64Compat::pushValue(const Address& addr)
+{
+ // Load value before allocating stack: addr.base may be SP.
+ loadPtr(Address(addr.base, addr.offset), ScratchRegister);
+ ma_dsubu(StackPointer, StackPointer, Imm32(sizeof(Value)));
+ storePtr(ScratchRegister, Address(StackPointer, 0));
+}
+
+void
+MacroAssemblerPPC64Compat::popValue(ValueOperand val)
+{
+ as_ld(val.valueReg(), StackPointer, 0);
+ as_addi(StackPointer, StackPointer, sizeof(Value));
+}
+
+void
+MacroAssemblerPPC64Compat::breakpoint()
+{
+ xs_trap();
+}
+
+void
+MacroAssemblerPPC64Compat::ensureDouble(const ValueOperand& source, FloatRegister dest,
+ Label* failure)
+{
+ Label isDouble, done;
+ {
+ ScratchTagScope tag(asMasm(), source);
+ splitTagForTest(source, tag);
+ asMasm().branchTestDouble(Assembler::Equal, tag, &isDouble);
+ asMasm().branchTestInt32(Assembler::NotEqual, tag, failure);
+ }
+
+ unboxInt32(source, ScratchRegister);
+ convertInt32ToDouble(ScratchRegister, dest);
+ jump(&done);
+
+ bind(&isDouble);
+ unboxDouble(source, dest);
+
+ bind(&done);
+}
+
+void
+MacroAssemblerPPC64Compat::checkStackAlignment()
+{
+#ifdef DEBUG
+ Label aligned;
+ as_andi_rc(ScratchRegister, sp, StackAlignment - 1);
+ ma_bc(ScratchRegister, ScratchRegister, &aligned, Zero, ShortJump);
+ xs_trap(); /* untagged so we know it's a bug */
+ bind(&aligned);
+#endif
+}
+
+void
+MacroAssemblerPPC64Compat::handleFailureWithHandlerTail(Label* profilerExitTail, Label* bailoutTail)
+{
+ // Reserve space for exception information.
+ int size = (sizeof(ResumeFromException) + ABIStackAlignment) & ~(ABIStackAlignment - 1);
+ asMasm().subPtr(Imm32(size), StackPointer);
+ ma_move(r3, StackPointer); // Use r3 since it is a first function argument
+
+ // Call the handler.
+ using Fn = void (*)(ResumeFromException * rfe);
+ asMasm().setupUnalignedABICall(r4);
+ asMasm().passABIArg(r3);
+ asMasm().callWithABI<Fn, HandleException>(ABIType::General,
+ CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ Label entryFrame;
+ Label catch_;
+ Label finally;
+ Label returnBaseline;
+ Label returnIon;
+ Label bailout;
+ Label wasm;
+ Label wasmCatch;
+
+ // Already clobbered r3, so use it...
+ load32(Address(StackPointer, ResumeFromException::offsetOfKind()), r3);
+ asMasm().branch32(Assembler::Equal, r3,
+ Imm32(ExceptionResumeKind::EntryFrame), &entryFrame);
+ asMasm().branch32(Assembler::Equal, r3, Imm32(ExceptionResumeKind::Catch),
+ &catch_);
+ asMasm().branch32(Assembler::Equal, r3, Imm32(ExceptionResumeKind::Finally),
+ &finally);
+ asMasm().branch32(Assembler::Equal, r3,
+ Imm32(ExceptionResumeKind::ForcedReturnBaseline),
+ &returnBaseline);
+ asMasm().branch32(Assembler::Equal, r3,
+ Imm32(ExceptionResumeKind::ForcedReturnIon), &returnIon);
+ asMasm().branch32(Assembler::Equal, r3, Imm32(ExceptionResumeKind::Bailout),
+ &bailout);
+ asMasm().branch32(Assembler::Equal, r3, Imm32(ExceptionResumeKind::Wasm),
+ &wasm);
+ asMasm().branch32(Assembler::Equal, r3, Imm32(ExceptionResumeKind::WasmCatch),
+ &wasmCatch);
+
+ xs_trap(); // Invalid kind.
+
+ // No exception handler. Load the error value, restore state
+ // and return from the entry frame.
+ bind(&entryFrame);
+ asMasm().moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
+ StackPointer);
+
+ // We're going to be returning by the ion calling convention
+ ma_pop(ScratchRegister);
+ xs_mtlr(ScratchRegister);
+ as_blr();
+
+ // If we found a catch handler, this must be a baseline frame. Restore
+ // state and jump to the catch block.
+ bind(&catch_);
+ // Use r12 here to save a register swap later in case we jump to ABI code.
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfTarget()), r12);
+ xs_mtctr(r12); // mtspr immediately, could be clobbered by subsequent loads
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
+ StackPointer);
+ as_bctr();
+
+ // If we found a finally block, this must be a baseline frame. Push
+ // two values expected by the finally block: the exception and
+ // BooleanValue(true).
+ bind(&finally);
+ ValueOperand exception = ValueOperand(r4);
+ loadValue(Address(sp, ResumeFromException::offsetOfException()), exception);
+
+ ValueOperand exceptionStack = ValueOperand(r5);
+ loadValue(Address(sp, ResumeFromException::offsetOfExceptionStack()),
+ exceptionStack);
+
+ loadPtr(Address(sp, ResumeFromException::offsetOfTarget()), r12);
+ xs_mtctr(r12);
+ loadPtr(Address(sp, ResumeFromException::offsetOfFramePointer()), FramePointer);
+ loadPtr(Address(sp, ResumeFromException::offsetOfStackPointer()), sp);
+
+ pushValue(exception);
+ pushValue(exceptionStack);
+ pushValue(BooleanValue(true));
+ as_bctr();
+
+ // Return BaselineFrame->returnValue() to the caller.
+ // Used in debug mode and for GeneratorReturn.
+ Label profilingInstrumentation;
+ bind(&returnBaseline);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
+ StackPointer);
+ loadValue(Address(FramePointer, BaselineFrame::reverseOffsetOfReturnValue()),
+ JSReturnOperand);
+ jump(&profilingInstrumentation);
+
+ // Return the given value to the caller.
+ bind(&returnIon);
+ loadValue(Address(StackPointer, ResumeFromException::offsetOfException()),
+ JSReturnOperand);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
+ StackPointer);
+
+ // If profiling is enabled, then update the lastProfilingFrame to refer to
+ // caller frame before returning. This code is shared by ForcedReturnIon
+ // and ForcedReturnBaseline.
+ bind(&profilingInstrumentation);
+ {
+ Label skipProfilingInstrumentation;
+ // Test if profiler enabled.
+ AbsoluteAddress addressOfEnabled(
+ asMasm().runtime()->geckoProfiler().addressOfEnabled());
+ asMasm().branch32(Assembler::Equal, addressOfEnabled, Imm32(0),
+ &skipProfilingInstrumentation);
+ jump(profilerExitTail);
+ bind(&skipProfilingInstrumentation);
+ }
+
+ ma_move(StackPointer, FramePointer);
+ pop(FramePointer);
+ ret();
+
+ // If we are bailing out to baseline to handle an exception, jump to
+ // the bailout tail stub. Load 1 (true) in ReturnReg to indicate
+ // success.
+ bind(&bailout);
+ loadPtr(Address(sp, ResumeFromException::offsetOfBailoutInfo()), r5);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
+ StackPointer);
+ ma_li(ReturnReg, Imm32(1));
+ jump(bailoutTail);
+
+ // If we are throwing and the innermost frame was a wasm frame, reset SP and
+ // FP; SP is pointing to the unwound return address to the wasm entry, so
+ // we can just ret().
+ bind(&wasm);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()), FramePointer);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()), StackPointer);
+ ma_li(InstanceReg, ImmWord(wasm::FailInstanceReg));
+ ret();
+
+ // Found a wasm catch handler, restore state and jump to it.
+ bind(&wasmCatch);
+xs_trap(); // XXX I'm suspicious of this code
+ wasm::GenerateJumpToCatchHandler(asMasm(), sp, r4, r5);
+#if(0)
+ loadPtr(Address(sp, ResumeFromException::offsetOfTarget()), r12);
+ xs_mtctr(r12);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
+ StackPointer);
+ as_bctr();
+#endif
+}
+
+// Toggled jumps and calls consist of oris r0,r0,0 followed by a full 7-instruction stanza.
+// This distinguishes it from other kinds of nops and is unusual enough to be noticed.
+// The leading oris 0,0,0 gets patched to a b .+32 when disabled.
+CodeOffset
+MacroAssemblerPPC64Compat::toggledJump(Label* label)
+{
+ ADBlock();
+ CodeOffset ret(nextOffset().getOffset());
+ as_oris(r0, r0, 0);
+ ma_b(label);
+ // The b() may emit a varying number of instructions, so add enough nops to pad.
+ while((nextOffset().getOffset() - ret.offset()) < ToggledCallSize(nullptr)) {
+ as_nop();
+ }
+ return ret;
+}
+
+CodeOffset
+MacroAssemblerPPC64Compat::toggledCall(JitCode* target, bool enabled)
+{
+ ADBlock();
+ BufferOffset bo = nextOffset();
+ CodeOffset offset(bo.getOffset());
+ if (enabled) {
+ as_oris(r0, r0, 0);
+ } else {
+ as_b(32, RelativeBranch, DontLinkB);
+ }
+ addPendingJump(nextOffset(), ImmPtr(target->raw()), RelocationKind::JITCODE);
+ ma_liPatchable(SecondScratchReg, ImmPtr(target->raw()));
+ xs_mtctr(SecondScratchReg);
+ as_bctr(LinkB);
+ MOZ_ASSERT_IF(!oom(), nextOffset().getOffset() - offset.offset() == ToggledCallSize(nullptr));
+ return offset;
+}
+
+void
+MacroAssemblerPPC64Compat::profilerEnterFrame(Register framePtr, Register scratch)
+{
+ asMasm().loadJSContext(scratch);
+ loadPtr(Address(scratch, offsetof(JSContext, profilingActivation_)), scratch);
+ storePtr(framePtr, Address(scratch, JitActivation::offsetOfLastProfilingFrame()));
+ storePtr(ImmPtr(nullptr), Address(scratch, JitActivation::offsetOfLastProfilingCallSite()));
+}
+
+void
+MacroAssemblerPPC64Compat::profilerExitFrame()
+{
+ jump(GetJitContext()->runtime->jitRuntime()->getProfilerExitFrameTail());
+}
+
+void
+MacroAssembler::subFromStackPtr(Imm32 imm32)
+{
+ if (imm32.value)
+ asMasm().subPtr(imm32, StackPointer);
+}
+
+//{{{ check_macroassembler_style
+// ===============================================================
+// Stack manipulation functions.
+
+// XXX: Check usage of this routine in Ion and see what assumes LR is a GPR. If so, then
+// maybe we need to find a way to abstract away SPRs vs GPRs after all.
+void
+MacroAssembler::PushRegsInMask(LiveRegisterSet set)
+{
+ int32_t diff = PushRegsInMaskSizeInBytes(set);
+ const int32_t reserved = diff;
+
+ reserveStack(reserved);
+ for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) {
+ diff -= sizeof(intptr_t);
+ storePtr(*iter, Address(StackPointer, diff));
+ }
+ for (FloatRegisterBackwardIterator iter(set.fpus().reduceSetForPush()); iter.more(); ++iter) {
+ diff -= sizeof(double);
+ storeDouble(*iter, Address(StackPointer, diff));
+ }
+ MOZ_ASSERT(diff == 0);
+}
+size_t
+MacroAssembler::PushRegsInMaskSizeInBytes(LiveRegisterSet set)
+{
+ return (set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes());
+}
+
+void
+MacroAssembler::PopRegsInMaskIgnore(LiveRegisterSet set, LiveRegisterSet ignore)
+{
+ int32_t diff = set.gprs().size() * sizeof(intptr_t) +
+ set.fpus().getPushSizeInBytes();
+ const int32_t reserved = diff;
+
+ for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) {
+ diff -= sizeof(intptr_t);
+ if (!ignore.has(*iter))
+ loadPtr(Address(StackPointer, diff), *iter);
+ }
+ for (FloatRegisterBackwardIterator iter(set.fpus().reduceSetForPush()); iter.more(); ++iter) {
+ diff -= sizeof(double);
+ if (!ignore.has(*iter))
+ loadDouble(Address(StackPointer, diff), *iter);
+ }
+ MOZ_ASSERT(diff == 0);
+ freeStack(reserved);
+}
+
+void
+MacroAssembler::storeRegsInMask(LiveRegisterSet set, Address dest, Register)
+{
+ FloatRegisterSet fpuSet(set.fpus().reduceSetForPush());
+ unsigned numFpu = fpuSet.size();
+ int32_t diffF = fpuSet.getPushSizeInBytes();
+ int32_t diffG = set.gprs().size() * sizeof(intptr_t);
+
+ MOZ_ASSERT(dest.offset >= diffG + diffF);
+
+ for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) {
+ diffG -= sizeof(intptr_t);
+ dest.offset -= sizeof(intptr_t);
+ storePtr(*iter, dest);
+ }
+ MOZ_ASSERT(diffG == 0);
+
+ for (FloatRegisterBackwardIterator iter(fpuSet); iter.more(); ++iter) {
+ FloatRegister reg = *iter;
+ diffF -= reg.size();
+ numFpu -= 1;
+ dest.offset -= reg.size();
+ if (reg.isDouble())
+ storeDouble(reg, dest);
+ else if (reg.isSingle())
+ storeFloat32(reg, dest);
+ else
+ MOZ_CRASH("Unknown register type.");
+ }
+ MOZ_ASSERT(numFpu == 0);
+ diffF -= diffF % sizeof(uintptr_t);
+ MOZ_ASSERT(diffF == 0);
+}
+// ===============================================================
+// ABI function calls.
+
+void
+MacroAssembler::setupUnalignedABICall(Register scratch)
+{
+ ADBlock();
+ MOZ_ASSERT(!IsCompilingWasm(), "wasm should only use aligned ABI calls"); // XXX?? arm doesn't do this
+ MOZ_ASSERT(scratch != ScratchRegister);
+ MOZ_ASSERT(scratch != SecondScratchReg);
+
+ setupNativeABICall();
+ dynamicAlignment_ = true;
+
+ // Even though this is ostensibly an ABI-compliant call, save both LR
+ // and SP; Baseline ICs assume that LR isn't modified.
+ xs_mflr(SecondScratchReg); // ma_and may clobber r0.
+ ma_move(scratch, StackPointer);
+ asMasm().subPtr(Imm32(sizeof(uintptr_t)*2), StackPointer);
+ ma_and(StackPointer, StackPointer, Imm32(~(ABIStackAlignment - 1)));
+ as_std(SecondScratchReg, StackPointer, sizeof(uintptr_t));
+ storePtr(scratch, Address(StackPointer, 0));
+}
+
+void
+MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm)
+{
+ ADBlock();
+ MOZ_ASSERT(inCall_);
+ uint32_t stackForCall = abiArgs_.stackBytesConsumedSoFar();
+
+ if (dynamicAlignment_) {
+ stackForCall += ComputeByteAlignment(stackForCall, ABIStackAlignment);
+ } else {
+ uint32_t alignmentAtPrologue = callFromWasm ? sizeof(wasm::Frame) : 0;
+ stackForCall += ComputeByteAlignment(stackForCall + framePushed() + alignmentAtPrologue,
+ ABIStackAlignment);
+ }
+
+ // The callee save area must minimally include room for the SP back chain
+ // pointer plus CR and LR. For 16-byte alignment we'll just ask for 32
+ // bytes. This guarantees nothing we're trying to keep on the stack will
+ // get overwritten.
+ stackForCall += 32;
+
+ *stackAdjust = stackForCall;
+ reserveStack(stackForCall);
+
+ // Position all arguments.
+ {
+ enoughMemory_ &= moveResolver_.resolve();
+ if (!enoughMemory_)
+ return;
+
+ MoveEmitter emitter(*this);
+ emitter.emit(moveResolver_);
+ emitter.finish();
+ }
+
+ assertStackAlignment(ABIStackAlignment);
+}
+
+void
+MacroAssembler::callWithABIPost(uint32_t stackAdjust, ABIType result, bool callFromWasm)
+{
+ ADBlock();
+
+ if (dynamicAlignment_) {
+ // Restore LR and SP (as stored in setupUnalignedABICall).
+ as_ld(ScratchRegister, StackPointer, stackAdjust+sizeof(uintptr_t));
+ xs_mtlr(ScratchRegister);
+ loadPtr(Address(StackPointer, stackAdjust), StackPointer);
+ // Use adjustFrame instead of freeStack because we already restored SP.
+ adjustFrame(-stackAdjust);
+ } else {
+ // LR isn't stored in this instance.
+ freeStack(stackAdjust);
+ }
+
+#ifdef DEBUG
+ MOZ_ASSERT(inCall_);
+ inCall_ = false;
+#endif
+}
+
+void
+MacroAssembler::callWithABINoProfiler(Register fun, ABIType result)
+{
+ ADBlock();
+
+ uint32_t stackAdjust;
+ callWithABIPre(&stackAdjust);
+ call(fun);
+ callWithABIPost(stackAdjust, result);
+}
+
+void
+MacroAssembler::callWithABINoProfiler(const Address& fun, ABIType result)
+{
+ uint32_t stackAdjust;
+ ADBlock();
+ MOZ_ASSERT(fun.base != SecondScratchReg);
+
+ // This requires a bit of fancy dancing: the address base could be one
+ // of the argregs and the MoveEmitter might clobber it positioning the
+ // arguments. To avoid this problem we'll load CTR early, a great
+ // example of turning necessity into virtue since it's faster too.
+ MOZ_ASSERT(Imm16::IsInSignedRange(fun.offset));
+ loadPtr(Address(fun.base, fun.offset), SecondScratchReg); // must use r12
+ xs_mtctr(SecondScratchReg);
+
+ // It's now safe to call the MoveEmitter.
+ callWithABIPre(&stackAdjust);
+ as_bctr(LinkB);
+ callWithABIPost(stackAdjust, result);
+}
+
+// ===============================================================
+// Move
+
+void
+MacroAssembler::moveValue(const TypedOrValueRegister& src, const ValueOperand& dest)
+{
+ if (src.hasValue()) {
+ moveValue(src.valueReg(), dest);
+ return;
+ }
+
+ MIRType type = src.type();
+ AnyRegister reg = src.typedReg();
+
+ if (!IsFloatingPointType(type)) {
+ boxNonDouble(ValueTypeFromMIRType(type), reg.gpr(), dest);
+ return;
+ }
+
+ FloatRegister scratch = ScratchDoubleReg;
+ FloatRegister freg = reg.fpu();
+ if (type == MIRType::Float32) {
+ convertFloat32ToDouble(freg, scratch);
+ freg = scratch;
+ }
+ boxDouble(freg, dest, scratch);
+}
+
+void
+MacroAssembler::moveValue(const ValueOperand& src, const ValueOperand& dest)
+{
+ if (src == dest)
+ return;
+ movePtr(src.valueReg(), dest.valueReg());
+}
+
+void
+MacroAssembler::moveValue(const Value& src, const ValueOperand& dest)
+{
+ if(!src.isGCThing()) {
+ ma_li(dest.valueReg(), ImmWord(src.asRawBits()));
+ return;
+ }
+
+ writeDataRelocation(src);
+ movWithPatch(ImmWord(src.asRawBits()), dest.valueReg());
+}
+
+// ===============================================================
+// Branch functions
+
+// assumed by unboxGCThingForGCBarrier
+static_assert(JS::detail::ValueGCThingPayloadMask == 0x0000'7FFF'FFFF'FFFF);
+
+void
+MacroAssembler::branchValueIsNurseryCell(Condition cond, const Address& address, Register temp,
+ Label* label)
+{
+ ADBlock();
+ MOZ_ASSERT(temp != InvalidReg);
+ loadValue(address, ValueOperand(temp));
+ branchValueIsNurseryCell(cond, ValueOperand(temp), InvalidReg, label);
+}
+void
+MacroAssembler::branchValueIsNurseryCell(Condition cond, ValueOperand value, Register temp,
+ Label* label)
+{
+ ADBlock();
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ Label done;
+ branchTestGCThing(Assembler::NotEqual, value,
+ cond == Assembler::Equal ? &done : label);
+
+ getGCThingValueChunk(value, SecondScratchReg);
+ loadPtr(Address(SecondScratchReg, gc::ChunkStoreBufferOffset), ScratchRegister);
+ branchPtr(InvertCondition(cond), ScratchRegister, ImmWord(0), label);
+
+ bind(&done);
+}
+
+void
+MacroAssembler::branchTestValue(Condition cond, const ValueOperand& lhs,
+ const Value& rhs, Label* label)
+{
+ ADBlock();
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(lhs.valueReg() != scratch);
+ moveValue(rhs, ValueOperand(scratch));
+ ma_bc(lhs.valueReg(), scratch, label, cond);
+}
+
+// ========================================================================
+// Memory access primitives.
+template <typename T>
+void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
+ MIRType valueType, const T& dest) {
+ MOZ_ASSERT(valueType < MIRType::Value);
+
+ if (valueType == MIRType::Double) {
+ boxDouble(value.reg().typedReg().fpu(), dest);
+ return;
+ }
+
+ if (value.constant()) {
+ storeValue(value.value(), dest);
+ } else {
+ storeValue(ValueTypeFromMIRType(valueType), value.reg().typedReg().gpr(),
+ dest);
+ }
+}
+
+template void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
+ MIRType valueType,
+ const Address& dest);
+template void MacroAssembler::storeUnboxedValue(
+ const ConstantOrRegister& value, MIRType valueType,
+ const BaseObjectElementIndex& dest);
+
+void
+MacroAssembler::PushBoxed(FloatRegister reg)
+{
+ MOZ_ASSERT(reg.isDouble());
+ subFromStackPtr(Imm32(sizeof(double)));
+ boxDouble(reg, Address(getStackPointer(), 0));
+ adjustFrame(sizeof(double));
+}
+
+void
+MacroAssemblerPPC64::ma_spectre_isel(Condition cond, Register lhs, Register rhs) {
+ if (JitOptions.spectreIndexMasking) {
+ if (cond == NotEqual) {
+ as_isel(lhs, lhs, rhs, Assembler::Equal);
+ } else if (cond == Equal) {
+ as_isel(lhs, rhs, lhs, Assembler::Equal);
+ } else if (cond == LessThan || cond == Below) {
+ as_isel(lhs, rhs, lhs, Assembler::LessThan);
+ } else if (cond == GreaterThanOrEqual || cond == AboveOrEqual) {
+ as_isel(lhs, lhs, rhs, Assembler::LessThan);
+ } else if (cond == GreaterThan || cond == Above) {
+ as_isel(lhs, rhs, lhs, Assembler::GreaterThan);
+ } else if (cond == LessThanOrEqual || cond == BelowOrEqual) {
+ as_isel(lhs, lhs, rhs, Assembler::GreaterThan);
+ } else {
+ MOZ_CRASH("unhandled condition");
+ }
+ }
+}
+
+void
+MacroAssembler::wasmBoundsCheck32(Condition cond, Register index,
+ Register boundsCheckLimit, Label* label)
+{
+ ADBlock();
+ // We can safely clean the upper word out because this must be 32-bit.
+ as_rldicl(index, index, 0, 32); // "clrldi"
+ ma_bc32(index, boundsCheckLimit, label, cond);
+ ma_spectre_isel(cond, index, boundsCheckLimit);
+}
+
+void
+MacroAssembler::wasmBoundsCheck32(Condition cond, Register index,
+ Address boundsCheckLimit, Label* label)
+{
+ ADBlock();
+ SecondScratchRegisterScope scratch2(*this);
+ // We can safely clean the upper word out because this must be 32-bit.
+ as_rldicl(index, index, 0, 32); // "clrldi"
+ load32ZeroExtend(boundsCheckLimit, SecondScratchReg);
+ ma_bc32(index, SecondScratchReg, label, cond);
+ ma_spectre_isel(cond, index, SecondScratchReg);
+}
+
+void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
+ Register64 boundsCheckLimit,
+ Label* label) {
+ ADBlock();
+ branchPtr(cond, index.reg, boundsCheckLimit.reg, label);
+ ma_spectre_isel(cond, index.reg, boundsCheckLimit.reg);
+}
+
+void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
+ Address boundsCheckLimit, Label* label) {
+ ADBlock();
+ MOZ_ASSERT(index.reg != SecondScratchReg);
+
+ Register64 limit(SecondScratchReg);
+ loadPtr(boundsCheckLimit, SecondScratchReg);
+ wasmBoundsCheck64(cond, index, limit, label);
+}
+
+void MacroAssembler::widenInt32(Register r) {
+ ADBlock();
+
+ move32To64SignExtend(r, Register64(r));
+}
+
+void
+MacroAssembler::wasmTruncateDoubleToUInt32(FloatRegister input, Register output, bool isSaturating,
+ Label* oolEntry)
+{
+ ADBlock();
+
+ // We only care if the conversion is invalid, not if it's inexact.
+ // Negative zero is treated like positive zero.
+ // Whack VXCVI.
+ as_mtfsb0(23);
+ as_fctiwuz(ScratchDoubleReg, input);
+
+ if (isSaturating) {
+ // There is no need to call the out-of-line routine: fctiwuz saturates
+ // NaN to 0 and all other values to their extents, so we're done. We
+ // ignore any bits that are set in the FPSCR.
+ } else {
+ // VXCVI is a failure (over/underflow, NaN, etc.)
+ as_mcrfs(cr0, 5); // reserved - VXSOFT - VXSQRT - VXCVI -> CR0[...SO]
+ // OutOfLineTruncateCheckF32/F64ToU32 -> outOfLineWasmTruncateToInt32Check
+ ma_bc(Assembler::SOBit, oolEntry);
+ }
+
+ moveFromDouble(ScratchDoubleReg, ScratchRegister);
+ // Don't sign extend.
+ as_rldicl(output, ScratchRegister, 0, 32); // "clrldi"
+}
+
+void
+MacroAssembler::wasmTruncateFloat32ToUInt32(FloatRegister input, Register output, bool isSaturating,
+ Label* oolEntry)
+{
+ wasmTruncateDoubleToUInt32(input, output, isSaturating, oolEntry);
+}
+
+void
+MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access, Register memoryBase, Register ptr,
+ Register ptrScratch, Register64 output)
+{
+ wasmLoadI64Impl(access, memoryBase, ptr, ptrScratch, output, InvalidReg);
+}
+
+void
+MacroAssembler::wasmUnalignedLoadI64(const wasm::MemoryAccessDesc& access, Register memoryBase,
+ Register ptr, Register ptrScratch, Register64 output,
+ Register tmp)
+{
+ wasmLoadI64Impl(access, memoryBase, ptr, ptrScratch, output, tmp);
+}
+
+void
+MacroAssembler::wasmStoreI64(const wasm::MemoryAccessDesc& access, Register64 value,
+ Register memoryBase, Register ptr, Register ptrScratch)
+{
+ wasmStoreI64Impl(access, value, memoryBase, ptr, ptrScratch, InvalidReg);
+}
+
+void
+MacroAssembler::wasmUnalignedStoreI64(const wasm::MemoryAccessDesc& access, Register64 value,
+ Register memoryBase, Register ptr, Register ptrScratch,
+ Register tmp)
+{
+ wasmStoreI64Impl(access, value, memoryBase, ptr, ptrScratch, tmp);
+}
+
+void
+MacroAssembler::wasmTruncateDoubleToInt64(FloatRegister input, Register64 output_,
+ bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempDouble)
+{
+ ADBlock();
+ MOZ_ASSERT(tempDouble.isInvalid());
+ Register output = output_.reg;
+
+ // We only care if the conversion is invalid, not if it's inexact.
+ // Negative zero is treated like positive zero.
+ // Whack VXCVI.
+ as_mtfsb0(23);
+ as_fctidz(ScratchDoubleReg, input);
+ // VXCVI is a failure (over/underflow, NaN, etc.)
+ as_mcrfs(cr0, 5); // reserved - VXSOFT - VXSQRT - VXCVI -> CR0[...SO]
+ // OutOfLineTruncateCheckF32OrF64ToI64 -> outOfLineWasmTruncateToInt64Check
+ ma_bc(Assembler::SOBit, oolEntry);
+
+ moveFromDouble(ScratchDoubleReg, output);
+ bind(oolRejoin);
+}
+
+void
+MacroAssembler::wasmTruncateDoubleToUInt64(FloatRegister input, Register64 output_,
+ bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempDouble)
+{
+ ADBlock();
+ MOZ_ASSERT(tempDouble.isInvalid());
+ Register output = output_.reg;
+
+ // We only care if the conversion is invalid, not if it's inexact.
+ // Negative zero is treated like positive zero.
+ // Whack VXCVI.
+ as_mtfsb0(23);
+ as_fctiduz(ScratchDoubleReg, input);
+ if (isSaturating) {
+ // There is no need to call the out-of-line routine: fctiduz saturates
+ // NaN to 0 and all other values to their extents, so we're done. We
+ // ignore any bits that are set in the FPSCR.
+ } else {
+ // VXCVI is a failure (over/underflow, NaN, etc.)
+ as_mcrfs(cr0, 5); // reserved - VXSOFT - VXSQRT - VXCVI -> CR0[...SO]
+ // OutOfLineTruncateCheckF32OrF64ToI64 -> outOfLineWasmTruncateToInt64Check
+ ma_bc(Assembler::SOBit, oolEntry);
+ }
+
+ moveFromDouble(ScratchDoubleReg, output);
+ bind(oolRejoin);
+}
+
+void
+MacroAssembler::wasmTruncateFloat32ToInt64(FloatRegister input, Register64 output,
+ bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempFloat)
+{
+ wasmTruncateDoubleToInt64(input, output, isSaturating, oolEntry,
+ oolRejoin, tempFloat);
+}
+
+void
+MacroAssembler::wasmTruncateFloat32ToUInt64(FloatRegister input, Register64 output,
+ bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempFloat)
+{
+ wasmTruncateDoubleToUInt64(input, output, isSaturating, oolEntry,
+ oolRejoin, tempFloat);
+}
+
+void
+MacroAssemblerPPC64Compat::wasmLoadI64Impl(const wasm::MemoryAccessDesc& access,
+ Register memoryBase, Register ptr, Register ptrScratch,
+ Register64 output, Register tmp)
+{
+ access.assertOffsetInGuardPages();
+ uint32_t offset = access.offset();
+ MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg);
+
+ // Maybe add the offset.
+ if (offset) {
+ asMasm().addPtr(Imm32(offset), ptrScratch);
+ ptr = ptrScratch;
+ }
+
+ unsigned byteSize = access.byteSize();
+ bool isSigned;
+
+ switch (access.type()) {
+ case Scalar::Int8: isSigned = true; break;
+ case Scalar::Uint8: isSigned = false; break;
+ case Scalar::Int16: isSigned = true; break;
+ case Scalar::Uint16: isSigned = false; break;
+ case Scalar::Int32: isSigned = true; break;
+ case Scalar::Uint32: isSigned = false; break;
+ case Scalar::Int64: isSigned = true; break;
+ default: MOZ_CRASH("unexpected array type");
+ }
+
+ // threadsafe
+ BaseIndex address(memoryBase, ptr, TimesOne);
+ asMasm().memoryBarrierBefore(access.sync());
+ asMasm().append(access, js::wasm::TrapMachineInsnForLoad(access.byteSize()),
+asMasm().ma_load(output.reg, address, static_cast<LoadStoreSize>(8 * byteSize),
+ isSigned ? SignExtend : ZeroExtend));
+ asMasm().memoryBarrierAfter(access.sync());
+}
+
+void
+MacroAssemblerPPC64Compat::wasmStoreI64Impl(const wasm::MemoryAccessDesc& access, Register64 value,
+ Register memoryBase, Register ptr, Register ptrScratch,
+ Register tmp)
+{
+ access.assertOffsetInGuardPages();
+ uint32_t offset = access.offset();
+ MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg);
+
+ // Maybe add the offset.
+ if (offset) {
+ asMasm().addPtr(Imm32(offset), ptrScratch);
+ ptr = ptrScratch;
+ }
+
+ unsigned byteSize = access.byteSize();
+ bool isSigned; // not used yet
+ switch (access.type()) {
+ case Scalar::Int8: isSigned = true; break;
+ case Scalar::Uint8: isSigned = false; break;
+ case Scalar::Int16: isSigned = true; break;
+ case Scalar::Uint16: isSigned = false; break;
+ case Scalar::Int32: isSigned = true; break;
+ case Scalar::Uint32: isSigned = false; break;
+ case Scalar::Int64: isSigned = true; break;
+ default: MOZ_CRASH("unexpected array type");
+ }
+
+ BaseIndex address(memoryBase, ptr, TimesOne);
+
+ // threadsafe
+ asMasm().memoryBarrierBefore(access.sync());
+ asMasm().append(access, js::wasm::TrapMachineInsnForLoad(access.byteSize()),
+asMasm().ma_store(value.reg, address, static_cast<LoadStoreSize>(8 * byteSize)));
+ asMasm().memoryBarrierAfter(access.sync());
+}
+
+template <typename T>
+static void
+CompareExchange64(MacroAssembler& masm, const wasm::MemoryAccessDesc* access, Synchronization sync, const T& mem,
+ Register64 expect, Register64 replace, Register64 output)
+{
+ // use Acquired Lock semantics, ISA v3.0B reference p915
+ masm.computeEffectiveAddress(mem, SecondScratchReg);
+
+ Label tryAgain;
+ Label exit;
+
+ masm.memoryBarrierBefore(sync);
+ masm.bind(&tryAgain);
+
+ if (access) masm.append(*access, wasm::TrapMachineInsn::Atomic,
+ FaultingCodeOffset(masm.currentOffset()));
+ // 'r0' for 'ra' indicates hard 0, not GPR r0
+ masm.as_ldarx(output.reg, r0, SecondScratchReg);
+ masm.ma_bc(output.reg, expect.reg, &exit, Assembler::NotEqual, ShortJump);
+ masm.movePtr(replace.reg, ScratchRegister);
+ /* if (access) masm.append(*access, wasm::TrapMachineInsn::Atomic,
+ FaultingCodeOffset(masm.currentOffset())); */
+ masm.as_stdcx(ScratchRegister, r0, SecondScratchReg);
+ masm.ma_bc(Assembler::NotEqual, &tryAgain, ShortJump);
+
+ masm.memoryBarrierAfter(sync);
+ //masm.as_isync();
+ masm.bind(&exit);
+}
+
+void
+MacroAssembler::compareExchange64(Synchronization sync, const Address& mem,
+ Register64 expect, Register64 replace, Register64 output)
+{
+ CompareExchange64(*this, nullptr, sync, mem, expect, replace, output);
+}
+
+void MacroAssembler::compareExchange64(Synchronization sync,
+ const BaseIndex& mem, Register64 expect,
+ Register64 replace, Register64 output) {
+ CompareExchange64(*this, nullptr, sync, mem, expect, replace, output);
+}
+
+template <typename T>
+static void
+AtomicExchange64(MacroAssembler& masm, const wasm::MemoryAccessDesc* access, Synchronization sync, const T& mem,
+ Register64 src, Register64 output)
+{
+ masm.computeEffectiveAddress(mem, SecondScratchReg);
+
+ Label tryAgain;
+
+ masm.memoryBarrierBefore(sync);
+
+ masm.bind(&tryAgain);
+
+ if (access) masm.append(*access, js::wasm::TrapMachineInsn::Load64,
+ FaultingCodeOffset(masm.currentOffset()));
+ // 'r0' for 'ra' indicates hard 0, not GPR r0
+ masm.as_ldarx(output.reg, r0, SecondScratchReg);
+ /* if (access) masm.append(*access, js::wasm::TrapMachineInsn::Load64,
+ FaultingCodeOffset(masm.currentOffset())); */
+ masm.as_stdcx(src.reg, r0, SecondScratchReg);
+ masm.ma_bc(cr0, Assembler::NotEqual, &tryAgain, ShortJump);
+
+ //masm.as_isync();
+ masm.memoryBarrierAfter(sync);
+}
+
+void
+MacroAssembler::atomicExchange64(Synchronization sync, const Address& mem, Register64 src,
+ Register64 output)
+{
+ AtomicExchange64(*this, nullptr, sync, mem, src, output);
+}
+
+void
+MacroAssembler::atomicExchange64(Synchronization sync, const BaseIndex& mem, Register64 src,
+ Register64 output)
+{
+ AtomicExchange64(*this, nullptr, sync, mem, src, output);
+}
+
+template<typename T>
+static void
+AtomicFetchOp64(MacroAssembler& masm, const wasm::MemoryAccessDesc* access, Synchronization sync, AtomicOp op, Register64 value,
+ const T& mem, Register64 temp, Register64 output)
+{
+ masm.computeEffectiveAddress(mem, SecondScratchReg);
+
+ Label tryAgain;
+
+ masm.memoryBarrierBefore(sync);
+
+ masm.bind(&tryAgain);
+
+ if (access) masm.append(*access, js::wasm::TrapMachineInsn::Load64,
+ FaultingCodeOffset(masm.currentOffset()));
+ // 'r0' for 'ra' indicates hard 0, not GPR r0
+ masm.as_ldarx(output.reg, r0, SecondScratchReg);
+
+ switch(op) {
+ case AtomicOp::Add:
+ masm.as_add(temp.reg, output.reg, value.reg);
+ break;
+ case AtomicOp::Sub:
+ masm.as_subf(temp.reg, value.reg, output.reg);
+ break;
+ case AtomicOp::And:
+ masm.as_and(temp.reg, output.reg, value.reg);
+ break;
+ case AtomicOp::Or:
+ masm.as_or(temp.reg, output.reg, value.reg);
+ break;
+ case AtomicOp::Xor:
+ masm.as_xor(temp.reg, output.reg, value.reg);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ /* if (access) masm.append(*access, masm.size()); */
+ masm.as_stdcx(temp.reg, r0, SecondScratchReg);
+ masm.ma_bc(Assembler::NotEqual, &tryAgain, ShortJump);
+
+ //masm.as_isync();
+ masm.memoryBarrierAfter(sync);
+}
+
+void
+MacroAssembler::atomicFetchOp64(Synchronization sync, AtomicOp op, Register64 value,
+ const Address& mem, Register64 temp, Register64 output)
+{
+ AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, output);
+}
+
+void MacroAssembler::atomicFetchOp64(Synchronization sync, AtomicOp op,
+ Register64 value, const BaseIndex& mem,
+ Register64 temp, Register64 output) {
+ AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, output);
+}
+
+void MacroAssembler::atomicEffectOp64(Synchronization sync, AtomicOp op,
+ Register64 value, const Address& mem,
+ Register64 temp) {
+ AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, temp);
+}
+
+void MacroAssembler::atomicEffectOp64(Synchronization sync, AtomicOp op,
+ Register64 value, const BaseIndex& mem,
+ Register64 temp) {
+ AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, temp);
+}
+
+void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
+ const Address& mem,
+ Register64 expect,
+ Register64 replace,
+ Register64 output) {
+ CompareExchange64(*this, &access, access.sync(), mem, expect, replace,
+ output);
+}
+
+void
+MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem,
+ Register64 expect,
+ Register64 replace,
+ Register64 output) {
+ CompareExchange64(*this, &access, access.sync(), mem, expect, replace, output);
+}
+
+void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register64 src,
+ Register64 output) {
+ //WasmAtomicExchange64(*this, access, mem, src, output);
+ AtomicExchange64(*this, &access, access.sync(), mem, src, output);
+}
+
+void
+MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem,
+ Register64 value, Register64 output)
+{
+ AtomicExchange64(*this, &access, access.sync(), mem, value, output);
+}
+
+void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register64 value,
+ const Address& mem, Register64 temp,
+ Register64 output) {
+ AtomicFetchOp64(*this, &access, access.sync(), op, value, mem, temp, output);
+}
+
+void
+MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register64 value,
+ const BaseIndex& mem, Register64 temp,
+ Register64 output)
+{
+ AtomicFetchOp64(*this, &access, access.sync(), op, value, mem, temp, output);
+}
+
+
+// ========================================================================
+// Convert floating point.
+
+void
+MacroAssembler::convertInt64ToDouble(Register64 src, FloatRegister dest)
+{
+ ADBlock();
+
+ moveToDouble(src.reg, dest);
+ as_fcfid(dest, dest);
+}
+
+void
+MacroAssembler::convertIntPtrToDouble(Register src, FloatRegister dest)
+{
+ ADBlock();
+
+ moveToDouble(src, dest);
+ as_fcfid(dest, dest);
+}
+
+void
+MacroAssembler::convertInt64ToFloat32(Register64 src, FloatRegister dest)
+{
+ ADBlock();
+
+ moveToDouble(src.reg, dest);
+ // Enforce rounding mode 0b00 (round-to-nearest ties-to-even).
+ as_mtfsfi(7, 0);
+ as_fcfids(dest, dest);
+}
+
+bool
+MacroAssembler::convertUInt64ToDoubleNeedsTemp()
+{
+ // We're not like those other inferior architectures.
+ return false;
+}
+
+void
+MacroAssembler::convertUInt64ToDouble(Register64 src, FloatRegister dest, Register temp)
+{
+ ADBlock();
+ MOZ_ASSERT(temp == Register::Invalid());
+
+ MacroAssemblerSpecific::convertUInt64ToDouble(src.reg, dest);
+}
+
+void
+MacroAssembler::convertUInt64ToFloat32(Register64 src, FloatRegister dest, Register temp)
+{
+ ADBlock();
+ MOZ_ASSERT(temp == Register::Invalid());
+
+ moveToDouble(src.reg, dest);
+ // Enforce rounding mode 0b00 (round-to-nearest ties-to-even).
+ as_mtfsfi(7, 0);
+ as_fcfidus(dest, dest);
+}
+
+void
+MacroAssembler::copySignDouble(FloatRegister lhs, FloatRegister rhs, FloatRegister dest)
+{
+ // From inspection, 'rhs' is the sign and 'lhs' is the value. Opposite of
+ // what the instruction takes.
+ as_fcpsgn(dest, rhs, lhs);
+}
+
+void MacroAssembler::shiftIndex32AndAdd(Register indexTemp32, int shift,
+ Register pointer) {
+ if (IsShiftInScaleRange(shift)) {
+ computeEffectiveAddress(
+ BaseIndex(pointer, indexTemp32, ShiftToScale(shift)), pointer);
+ return;
+ }
+ lshift32(Imm32(shift), indexTemp32);
+ addPtr(indexTemp32, pointer);
+}
+
+void
+MacroAssembler::truncFloat32ToInt32(FloatRegister src, Register dest, Label* fail)
+{
+ return truncDoubleToInt32(src, dest, fail);
+}
+
+void
+MacroAssembler::truncDoubleToInt32(FloatRegister src, Register dest, Label* fail)
+{
+ ADBlock();
+ MOZ_ASSERT(dest != ScratchRegister);
+ MOZ_ASSERT(src != ScratchDoubleReg);
+
+ // We only care if the conversion is invalid, not if it's inexact.
+ // However, JavaScript defines Math.trunc(-0) == -0, so we need a check.
+ // Whack VXCVI.
+ as_mtfsb0(23);
+ as_fctiwz(ScratchDoubleReg, src);
+ // VXCVI is a failure (over/underflow, NaN, etc.)
+ as_mcrfs(cr1, 5); // reserved - VXSOFT - VXSQRT - VXCVI -> CR1[...SO]
+ moveFromDouble(ScratchDoubleReg, dest);
+ as_srawi(dest, dest, 0); // clear upper word and sign extend
+ as_cmpdi(cr7, dest, 0); // check for zero
+ moveFromDouble(src, ScratchRegister);
+ as_cmpdi(ScratchRegister, 0); // check sign bit of original float
+ as_crand(0, 0, 30); // CR0[LT] &= CR7[EQ] (only bail if - and 0)
+ as_cror(0, 0, 7); // Bond, James Bond: CR0[LT] |= CR1[SO]
+ ma_bc(Assembler::LessThan, fail);
+}
+
+void
+MacroAssembler::nearbyIntDouble(RoundingMode mode, FloatRegister src,
+ FloatRegister dest)
+{
+ ADBlock();
+
+ switch (mode) {
+ case RoundingMode::Up:
+ as_frip(dest, src);
+ break;
+ case RoundingMode::Down:
+ as_frim(dest, src);
+ break;
+ case RoundingMode::NearestTiesToEven:
+ // This is actually IEEE nearest ties-to-even. |frin| doesn't
+ // do this behaviour as specified by JavaScript, so we don't
+ // support this, and it should never be called ... right?
+ MOZ_CRASH("on a scale of one to even I just can't");
+ break;
+ case RoundingMode::TowardsZero:
+ as_friz(dest, src);
+ break;
+ default:
+ MOZ_CRASH("unsupported mode");
+ break;
+ }
+}
+
+void
+MacroAssembler::nearbyIntFloat32(RoundingMode mode, FloatRegister src,
+ FloatRegister dest)
+{
+ return nearbyIntDouble(mode, src, dest);
+}
+
+void
+MacroAssembler::ceilFloat32ToInt32(FloatRegister src, Register dest,
+ Label* fail)
+{
+ return ceilDoubleToInt32(src, dest, fail);
+}
+
+void
+MacroAssembler::ceilDoubleToInt32(FloatRegister src, Register dest, Label* fail)
+{
+ ADBlock();
+ MOZ_ASSERT(dest != ScratchRegister);
+ MOZ_ASSERT(src != ScratchDoubleReg);
+
+ // We only care if the conversion is invalid, not if it's inexact.
+ // However, JavaScript defines Math.ceil(-0) == -0, so we need a check.
+ // Whack VXCVI.
+ as_mtfsb0(23);
+ // "Pre-round" to +inf. Any NaN will get passed to fctiw.
+ // (For pre-v2.02, set rounding to 0b10.)
+ as_frip(ScratchDoubleReg, src);
+ as_fctiw(ScratchDoubleReg, ScratchDoubleReg);
+ // VXCVI is a failure (over/underflow, NaN, etc.)
+ as_mcrfs(cr1, 5); // reserved - VXSOFT - VXSQRT - VXCVI -> CR1[...SO]
+ moveFromDouble(ScratchDoubleReg, dest);
+ as_srawi(dest, dest, 0); // clear upper word and sign extend
+ as_cmpdi(cr7, dest, 0); // check for zero
+ moveFromDouble(src, ScratchRegister);
+ as_cmpdi(ScratchRegister, 0); // check sign bit of original float
+ as_crand(0, 0, 30); // CR0[LT] &= CR7[EQ] (only bail if - and 0)
+ as_cror(0, 0, 7); // Licenced to kill: CR0[LT] |= CR1[SO]
+ ma_bc(Assembler::LessThan, fail);
+}
+
+void
+MacroAssembler::floorFloat32ToInt32(FloatRegister src, Register dest,
+ Label* fail)
+{
+ return floorDoubleToInt32(src, dest, fail);
+}
+
+void
+MacroAssembler::floorDoubleToInt32(FloatRegister src, Register dest, Label* fail)
+{
+ ADBlock();
+ MOZ_ASSERT(dest != ScratchRegister);
+ MOZ_ASSERT(src != ScratchDoubleReg);
+
+ // We only care if the conversion is invalid, not if it's inexact.
+ // However, we have to check -0 here too for the same stupid reason.
+ // Whack VXCVI.
+ as_mtfsb0(23);
+ // "Pre-round" to -inf. Any NaN will get passed to fctiw.
+ // (For pre-v2.02, set rounding to 0b11.)
+ as_frim(ScratchDoubleReg, src);
+ as_fctiw(ScratchDoubleReg, ScratchDoubleReg);
+ // VXCVI is a failure (over/underflow, NaN, etc.)
+ as_mcrfs(cr1, 5); // reserved - VXSOFT - VXSQRT - VXCVI -> CR1[...SO]
+ moveFromDouble(ScratchDoubleReg, dest);
+ as_srawi(dest, dest, 0); // clear upper word and sign extend
+ as_cmpdi(cr7, dest, 0); // check for zero
+ moveFromDouble(src, ScratchRegister);
+ as_cmpdi(ScratchRegister, 0); // check sign bit of original float
+ as_crand(0, 0, 30); // CR0[LT] &= CR7[EQ] (only bail if - and 0)
+ as_cror(0, 0, 7); // Nobody does it better: CR0[LT] |= CR1[SO]
+ ma_bc(Assembler::LessThan, fail);
+}
+
+void
+MacroAssembler::roundFloat32ToInt32(FloatRegister src, Register dest,
+ FloatRegister temp, Label* fail)
+{
+ return floorDoubleToInt32(src, dest, fail);
+}
+
+void
+MacroAssembler::roundDoubleToInt32(FloatRegister src, Register dest,
+ FloatRegister temp, Label* fail)
+{
+ ADBlock();
+ MOZ_ASSERT(dest != ScratchRegister);
+ MOZ_ASSERT(src != ScratchDoubleReg);
+
+ // We only care if the conversion is invalid, not if it's inexact.
+ // And, you know, negative zero. BECAUSE THAT HAPPENS SOOO MUCH.
+ // Whack VXCVI.
+ as_mtfsb0(23);
+ // The default b00 rounding mode is implemented as IEEE round-to-nearest
+ // and ties-to-even. This means round(0.5) == 0. However, JavaScript
+ // expects round(0.5) == 1. |frin| is not an exact duplicate for this
+ // behaviour either. The best option is to fudge it by adding 0.5 and
+ // round towards -Inf, which works in both the positive and negative cases.
+ xs_lis(ScratchRegister, 0x3f00); // 0x3f000000 = 0.5
+ moveToFloat32(ScratchRegister, ScratchDoubleReg);
+ as_fadd(ScratchDoubleReg, ScratchDoubleReg, src); // preserves NaN
+ as_mtfsfi(7, 3); // RN to -Inf
+ as_fctiw(ScratchDoubleReg, ScratchDoubleReg); // truncate according to RN
+ // VXCVI is a failure (over/underflow, NaN, etc.)
+ as_mcrfs(cr1, 5); // reserved - VXSOFT - VXSQRT - VXCVI -> CR1[...SO]
+ as_mtfsfi(7, 0); // RN to default
+ moveFromDouble(ScratchDoubleReg, dest);
+ as_srawi(dest, dest, 0); // clear upper word and sign extend
+ as_cmpdi(cr7, dest, 0); // check for zero
+ moveFromDouble(src, ScratchRegister);
+ as_cmpdi(ScratchRegister, 0); // check sign bit of original float
+ as_crand(0, 0, 30); // CR0[LT] &= CR7[EQ] (only bail if - and 0)
+ as_cror(0, 0, 7); // Makes me feel sad for the rest: CR0[LT] |= CR1[SO]
+ ma_bc(Assembler::LessThan, fail);
+}
+
+void
+MacroAssembler::flexibleRemainder32(Register rhs, Register srcDest,
+ bool isUnsigned, const LiveRegisterSet&)
+{
+ ADBlock();
+ remainder32(rhs, srcDest, isUnsigned);
+}
+
+void
+MacroAssembler::flexibleQuotient32(Register rhs, Register srcDest,
+ bool isUnsigned,
+ const LiveRegisterSet&)
+{
+ ADBlock();
+ quotient32(rhs, srcDest, isUnsigned);
+}
+
+void
+MacroAssembler::flexibleDivMod32(Register rhs, Register srcDest,
+ Register remOutput, bool isUnsigned,
+ const LiveRegisterSet&)
+{
+ ADBlock();
+
+ if (HasPPCISA3()) {
+ if (isUnsigned) {
+ as_moduw(remOutput, srcDest, rhs);
+ as_divwu(srcDest, srcDest, rhs);
+ } else {
+ as_modsw(remOutput, srcDest, rhs);
+ as_divw(srcDest, srcDest, rhs);
+ }
+ return;
+ }
+
+ Register scratch = ScratchRegister;
+ if (isUnsigned) {
+ as_divwu(scratch, srcDest, rhs);
+ } else {
+ as_divw(scratch, srcDest, rhs);
+ }
+ // Compute remainder
+ as_mullw(remOutput, srcDest, rhs);
+ as_subf(remOutput, scratch, srcDest);
+ xs_mr(srcDest, scratch);
+}
+
+//}}} check_macroassembler_style
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/EndianUtils.h"
+
+#include "jit/MacroAssembler.h"
+
+using namespace js;
+using namespace jit;
+
+void
+MacroAssemblerPPC64::ma_move(Register rd, Register rs)
+{
+ if (rd != rs)
+ as_or(rd, rs, rs);
+}
+
+void
+MacroAssemblerPPC64::ma_li(Register dest, ImmGCPtr ptr)
+{
+ writeDataRelocation(ptr);
+ asMasm().ma_liPatchable(dest, ImmPtr(ptr.value));
+}
+
+void
+MacroAssemblerPPC64::ma_li(Register dest, Imm32 imm)
+{
+ ADBlock();
+ // signed
+// XXX: why??? why not just call ma_li(Register dest, int64_t value)?
+ if (Imm16::IsInSignedRange(imm.value)) {
+ xs_li(dest, imm.value);
+ } else if (Imm16::IsInUnsignedRange(imm.value)) {
+ xs_li(dest, 0);
+ as_ori(dest, dest, Imm16::Lower(imm).encode());
+ } else if (Imm16::Lower(imm).encode() == 0) {
+ xs_lis(dest, Imm16::Upper(imm).encode());
+ } else {
+ xs_lis(dest, Imm16::Upper(imm).encode());
+ as_ori(dest, dest, Imm16::Lower(imm).encode());
+ }
+}
+
+// This method generates lis and ori instruction pair that can be modified by
+// UpdateLisOriValue, either during compilation (eg. Assembler::bind), or
+// during execution (eg. jit::PatchJump).
+void
+MacroAssemblerPPC64::ma_liPatchable(Register dest, Imm32 imm)
+{
+ m_buffer.ensureSpace(2 * sizeof(uint32_t));
+ xs_lis(dest, Imm16::Upper(imm).encode());
+ as_ori(dest, dest, Imm16::Lower(imm).encode());
+}
+
+// Shifts
+
+// Bit extract/insert
+void
+MacroAssemblerPPC64::ma_ext(Register rt, Register rs, uint16_t pos, uint16_t size) {
+ MOZ_ASSERT(pos < 32);
+ MOZ_ASSERT(pos + size < 33);
+
+ as_rlwinm(rt, rs, 0, pos, size);
+}
+
+void
+MacroAssemblerPPC64::ma_ins(Register rt, Register rs, uint16_t pos, uint16_t size) {
+ MOZ_ASSERT(pos < 32);
+ MOZ_ASSERT(pos + size <= 32);
+ MOZ_ASSERT(size != 0);
+
+ as_rlwimi(rt, rs, 0, pos, size);
+}
+
+
+// And.
+void
+MacroAssemblerPPC64::ma_and(Register rd, Register rs)
+{
+ as_and(rd, rd, rs);
+}
+
+void
+MacroAssemblerPPC64::ma_and(Register rd, Imm32 imm)
+{
+ ma_and(rd, rd, imm);
+}
+
+void
+MacroAssemblerPPC64::ma_and(Register rd, Register rs, Imm32 imm)
+{
+ if (Imm16::IsInUnsignedRange(imm.value)) {
+ as_andi_rc(rd, rs, imm.value);
+ } else {
+ MOZ_ASSERT(rs != ScratchRegister);
+
+ ma_li(ScratchRegister, imm);
+ as_and(rd, rs, ScratchRegister);
+ }
+}
+
+// Or.
+void
+MacroAssemblerPPC64::ma_or(Register rd, Register rs)
+{
+ as_or(rd, rd, rs);
+}
+
+void
+MacroAssemblerPPC64::ma_or(Register rd, Imm32 imm)
+{
+ ma_or(rd, rd, imm);
+}
+
+void
+MacroAssemblerPPC64::ma_or(Register rd, Register rs, Imm32 imm)
+{
+ if (Imm16::IsInUnsignedRange(imm.value)) {
+ as_ori(rd, rs, imm.value);
+ } else {
+ MOZ_ASSERT(rs != ScratchRegister);
+
+ ma_li(ScratchRegister, imm);
+ as_or(rd, rs, ScratchRegister);
+ }
+}
+
+// xor
+void
+MacroAssemblerPPC64::ma_xor(Register rd, Register rs)
+{
+ as_xor(rd, rd, rs);
+}
+
+void
+MacroAssemblerPPC64::ma_xor(Register rd, Imm32 imm)
+{
+ ma_xor(rd, rd, imm);
+}
+
+void
+MacroAssemblerPPC64::ma_xor(Register rd, Register rs, Imm32 imm)
+{
+ if (Imm16::IsInUnsignedRange(imm.value)) {
+ as_xori(rd, rs, imm.value);
+ } else {
+ MOZ_ASSERT(rs != ScratchRegister);
+
+ ma_li(ScratchRegister, imm);
+ as_xor(rd, rs, ScratchRegister);
+ }
+}
+
+
+
+// Arithmetic-based ops.
+
+// Add.
+
+void
+MacroAssemblerPPC64::ma_addTestCarry(Condition cond, Register rd, Register rs, Register rt,
+ Label* overflow, bool is32)
+{
+ ADBlock();
+ MOZ_ASSERT(cond == Assembler::CarrySet || cond == Assembler::CarryClear);
+ MOZ_ASSERT_IF(rd == rs, rt != rd);
+ as_addc(rd, rs, rt);
+ as_mcrxrx(cr0);
+ if (is32) {
+ // CA32 (Power ISA 3.0B spec page 120)
+ ma_bc(cond == Assembler::CarrySet ? Assembler::SOBit : Assembler::NSOBit, overflow);
+ } else {
+ // regular CA
+ ma_bc(cond == Assembler::CarrySet ? Assembler::Equal : Assembler::NotEqual, overflow);
+ }
+}
+
+// XXX: add short option
+void
+MacroAssemblerPPC64::ma_addTestCarry(Condition cond, Register rd, Register rs, Imm32 imm,
+ Label* overflow, bool is32)
+{
+ ADBlock();
+ MOZ_ASSERT(cond == Assembler::CarrySet || cond == Assembler::CarryClear);
+ if (!Imm16::IsInSignedRange(imm.value)) {
+ MOZ_ASSERT(rs != ScratchRegister);
+ ma_li(ScratchRegister, imm);
+ ma_addTestCarry(cond, rd, rs, ScratchRegister, overflow, is32);
+ return;
+ }
+ as_addic(rd, rs, imm.value);
+ as_mcrxrx(cr0);
+ if (is32) {
+ // CA32
+ ma_bc(cond == Assembler::CarrySet ? Assembler::SOBit : Assembler::NSOBit, overflow);
+ } else {
+ // regular CA
+ ma_bc(cond == Assembler::CarrySet ? Assembler::Equal : Assembler::NotEqual, overflow);
+ }
+}
+
+// XXX: add short option
+void
+MacroAssemblerPPC64::ma_addTestCarry(Condition cond, Register rd, Register rs, ImmWord imm,
+ Label* overflow, bool is32)
+{
+ ADBlock();
+ MOZ_ASSERT(cond == Assembler::CarrySet || cond == Assembler::CarryClear);
+ // This can only enter through a 64-bit path.
+ MOZ_ASSERT(!is32);
+;
+ if (!Imm16::IsInSignedRange(imm.value)) {
+ MOZ_ASSERT(rs != ScratchRegister);
+ ma_li(ScratchRegister, imm);
+ ma_addTestCarry(cond, rd, rs, ScratchRegister, overflow, is32);
+ return;
+ }
+ as_addic(rd, rs, imm.value);
+ as_mcrxrx(cr0);
+ ma_bc(cond == Assembler::CarrySet ? Assembler::Equal : Assembler::NotEqual, overflow);
+}
+
+// Subtract.
+void
+MacroAssemblerPPC64::ma_subu(Register rd, Register rs, Imm32 imm)
+{
+ if (Imm16::IsInSignedRange(-imm.value)) {
+ as_addi(rd, rs, -imm.value);
+ } else {
+ ma_li(ScratchRegister, imm);
+ as_subf(rd, ScratchRegister, rs);
+ }
+}
+
+void
+MacroAssemblerPPC64::ma_subu(Register rd, Imm32 imm)
+{
+ ma_subu(rd, rd, imm);
+}
+
+void
+MacroAssemblerPPC64::ma_subu(Register rd, Register rs)
+{
+ as_subf(rd, rs, rd);
+}
+
+void
+MacroAssemblerPPC64::ma_mul(Register rd, Register rs, Imm32 imm)
+{
+ ADBlock();
+ MOZ_ASSERT(rs != ScratchRegister);
+ ma_li(ScratchRegister, imm);
+ as_mulld(rd, rs, ScratchRegister);
+}
+
+void
+MacroAssemblerPPC64::ma_mul_branch_overflow(Register rd, Register rs, Register rt, Label* overflow)
+{
+ ADBlock();
+ MOZ_ASSERT(rs != ScratchRegister);
+ MOZ_ASSERT(rt != ScratchRegister);
+
+ // This is a 32-bit operation, so we need to whack and test XER[OV32].
+ xs_li(ScratchRegister, 0);
+ xs_mtxer(ScratchRegister);
+ as_mullwo(rd, rs, rt);
+ ma_bc(Assembler::Overflow, overflow);
+}
+
+void
+MacroAssemblerPPC64::ma_mul_branch_overflow(Register rd, Register rs, Imm32 imm, Label* overflow)
+{
+ ADBlock();
+ ma_li(SecondScratchReg, imm);
+ ma_mul_branch_overflow(rd, rs, SecondScratchReg, overflow);
+}
+
+// Memory.
+
+FaultingCodeOffset
+MacroAssemblerPPC64::ma_load(Register dest, const BaseIndex& src,
+ LoadStoreSize size, LoadStoreExtension extension)
+{
+ // TODO: See note in ::computeScaledAddress. Can we turn this into
+ // a smaller instruction sequence?
+ asMasm().computeScaledAddress(src, SecondScratchReg);
+
+ // If src.offset is out of 16-bit signed range, we will hit an assert
+ // doing the next ma_load() because the second scratch register is needed
+ // again. In that case, hoist the add up since we can freely clobber it.
+ if (!Imm16::IsInSignedRange(src.offset)) {
+ ma_add(SecondScratchReg, SecondScratchReg, Imm32(src.offset));
+ return ma_load(dest, Address(SecondScratchReg, 0), size, extension);
+ } else {
+ return asMasm().ma_load(dest, Address(SecondScratchReg, src.offset), size, extension);
+ }
+}
+
+#if(0)
+// XXX: remove
+void
+MacroAssemblerPPC64::ma_load_unaligned(const wasm::MemoryAccessDesc& access, Register dest, const BaseIndex& src, Register temp,
+ LoadStoreSize size, LoadStoreExtension extension)
+{
+ MOZ_ASSERT(MOZ_LITTLE_ENDIAN(), "Wasm-only; wasm is disabled on big-endian.");
+ MOZ_CRASH("only needed for 64-bit loads");
+#if 0
+ int16_t lowOffset, hiOffset;
+ Register base;
+
+ asMasm().computeScaledAddress(src, SecondScratchReg);
+
+ if (Imm16::IsInSignedRange(src.offset) && Imm16::IsInSignedRange(src.offset + size / 8 - 1)) {
+ base = SecondScratchReg;
+ lowOffset = Imm16(src.offset).encode();
+ hiOffset = Imm16(src.offset + size / 8 - 1).encode();
+ } else {
+ ma_li(ScratchRegister, Imm32(src.offset));
+ asMasm().addPtr(SecondScratchReg, ScratchRegister);
+ base = ScratchRegister;
+ lowOffset = Imm16(0).encode();
+ hiOffset = Imm16(size / 8 - 1).encode();
+ }
+
+ BufferOffset load;
+ switch (size) {
+ case SizeHalfWord:
+ if (extension != ZeroExtend)
+ load = as_lbu(temp, base, hiOffset);
+ else
+ load = as_lb(temp, base, hiOffset);
+ as_lbu(dest, base, lowOffset);
+ ma_ins(dest, temp, 8, 24);
+ break;
+ case SizeWord:
+ load = as_lwl(dest, base, hiOffset);
+ as_lwr(dest, base, lowOffset);
+#ifdef JS_CODEGEN_PPC64
+ if (extension != ZeroExtend)
+ as_dext(dest, dest, 0, 32);
+#endif
+ break;
+#ifdef JS_CODEGEN_PPC64
+ case SizeDouble:
+ load = as_ldl(dest, base, hiOffset);
+ as_ldr(dest, base, lowOffset);
+ break;
+#endif
+ default:
+ MOZ_CRASH("Invalid argument for ma_load");
+ }
+
+ append(access, load.getOffset());
+#endif
+}
+#endif
+
+FaultingCodeOffset
+MacroAssemblerPPC64::ma_store(Register data, const BaseIndex& dest,
+ LoadStoreSize size)
+{
+ // TODO: See note in ::computeScaledAddress. Can we turn this into
+ // a smaller instruction sequence?
+ MOZ_ASSERT(data != SecondScratchReg);
+ asMasm().computeScaledAddress(dest, SecondScratchReg);
+
+ // If dest.offset is out of 16-bit signed range, we will hit an assert
+ // doing the next ma_store() because the second scratch register is needed
+ // again. In that case, hoist the add up since we can freely clobber it.
+ if (!Imm16::IsInSignedRange(dest.offset)) {
+ ma_add(SecondScratchReg, SecondScratchReg, Imm32(dest.offset));
+ return ma_store(data, Address(SecondScratchReg, 0), size);
+ } else {
+ return asMasm().ma_store(data, Address(SecondScratchReg, dest.offset), size);
+ }
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64::ma_store(Imm32 imm, const BaseIndex& dest,
+ LoadStoreSize size)
+{
+ // Make sure that SecondScratchReg contains the absolute address so that
+ // offset is 0.
+ asMasm().computeEffectiveAddress(dest, SecondScratchReg);
+
+ // With offset of 0 ScratchRegister will not be used in ma_store(),
+ // so we can use it as a parameter here.
+ ma_li(ScratchRegister, imm);
+ return asMasm().ma_store(ScratchRegister, Address(SecondScratchReg, 0), size);
+}
+
+#if(0)
+// XXX: remove
+void
+MacroAssemblerPPC64::ma_store_unaligned(Register src, const BaseIndex& dest,
+ LoadStoreSize size)
+{
+ MOZ_CRASH("NYI");
+}
+
+// XXX: remove
+void
+MacroAssemblerPPC64::ma_store_unaligned(const wasm::MemoryAccessDesc& access, Register data,
+ const BaseIndex& dest, Register temp,
+ LoadStoreSize size, LoadStoreExtension extension)
+{
+ MOZ_ASSERT(MOZ_LITTLE_ENDIAN(), "Wasm-only; wasm is disabled on big-endian.");
+ MOZ_CRASH("only needed for 64-bit loads");
+#if 0
+ int16_t lowOffset, hiOffset;
+ Register base;
+
+ asMasm().computeScaledAddress(dest, SecondScratchReg);
+
+ if (Imm16::IsInSignedRange(dest.offset) && Imm16::IsInSignedRange(dest.offset + size / 8 - 1)) {
+ base = SecondScratchReg;
+ lowOffset = Imm16(dest.offset).encode();
+ hiOffset = Imm16(dest.offset + size / 8 - 1).encode();
+ } else {
+ ma_li(ScratchRegister, Imm32(dest.offset));
+ asMasm().addPtr(SecondScratchReg, ScratchRegister);
+ base = ScratchRegister;
+ lowOffset = Imm16(0).encode();
+ hiOffset = Imm16(size / 8 - 1).encode();
+ }
+
+ BufferOffset store;
+ switch (size) {
+ case SizeHalfWord:
+ ma_ext(temp, data, 8, 8);
+ store = as_sb(temp, base, hiOffset);
+ as_sb(data, base, lowOffset);
+ break;
+ case SizeWord:
+ store = as_swl(data, base, hiOffset);
+ as_swr(data, base, lowOffset);
+ break;
+#ifdef JS_CODEGEN_PPC64
+ case SizeDouble:
+ store = as_sdl(data, base, hiOffset);
+ as_sdr(data, base, lowOffset);
+ break;
+#endif
+ default:
+ MOZ_CRASH("Invalid argument for ma_store");
+ }
+ append(access, store.getOffset());
+#endif
+}
+#endif
+
+// Branches when done from within ppc64-specific code.
+void
+MacroAssemblerPPC64::ma_bc(Register lhs, Register rhs, Label* label, Condition c, JumpKind jumpKind)
+{
+ ADBlock();
+ MOZ_ASSERT(!(c & ConditionOnlyXER));
+ if (c == Always) {
+ ma_b(label, jumpKind);
+ } else if (c & ConditionZero) {
+ MOZ_ASSERT(lhs == rhs);
+ as_cmpdi(lhs, 0);
+ ma_bc(c, label, jumpKind);
+ } else if (c & ConditionUnsigned) {
+ as_cmpld(lhs, rhs);
+ ma_bc(c, label, jumpKind);
+ } else {
+ MOZ_ASSERT(c < 0x100); // paranoia
+ as_cmpd(lhs, rhs);
+ ma_bc(c, label, jumpKind);
+ }
+}
+
+// For an explicit 32-bit compare (mostly wasm).
+void
+MacroAssemblerPPC64::ma_bc32(Register lhs, Register rhs, Label* label, Condition c, JumpKind jumpKind)
+{
+ ADBlock();
+ MOZ_ASSERT(!(c & ConditionOnlyXER));
+ if (c == Always) {
+ ma_b(label, jumpKind);
+ } else if (c & ConditionZero) {
+ MOZ_ASSERT(lhs == rhs);
+ as_cmpwi(lhs, 0);
+ ma_bc(c, label, jumpKind);
+ } else if (c & ConditionUnsigned) {
+ as_cmplw(lhs, rhs);
+ ma_bc(c, label, jumpKind);
+ } else {
+ MOZ_ASSERT(c < 0x100); // paranoia
+ as_cmpw(lhs, rhs);
+ ma_bc(c, label, jumpKind);
+ }
+}
+
+// For an explicit 64-bit compare.
+void
+MacroAssemblerPPC64::ma_bc64(Register lhs, Imm32 imm, Label* label, Condition c, JumpKind jumpKind)
+{
+ ADBlock();
+ MOZ_ASSERT(!(c & ConditionOnlyXER));
+ if (c == Always) {
+ ma_b(label, jumpKind);
+ return;
+ }
+ if (c & ConditionZero) {
+ MOZ_ASSERT(imm.value == 0);
+ as_cmpdi(lhs, 0);
+ ma_bc(c, label, jumpKind);
+ return;
+ }
+ if (c & ConditionUnsigned) {
+ if (Imm16::IsInUnsignedRange(imm.value)) {
+ as_cmpldi(lhs, imm.value);
+ } else {
+ MOZ_ASSERT(lhs != ScratchRegister);
+ ma_li(ScratchRegister, imm);
+ as_cmpld(lhs, ScratchRegister);
+ }
+ } else {
+ MOZ_ASSERT(c < 0x100); // just in case
+ if (Imm16::IsInSignedRange(imm.value)) {
+ as_cmpdi(lhs, imm.value);
+ } else {
+ MOZ_ASSERT(lhs != ScratchRegister);
+ ma_li(ScratchRegister, imm);
+ as_cmpd(lhs, ScratchRegister);
+ }
+ }
+ ma_bc(c, label, jumpKind);
+}
+
+// For everyone else, there's MasterCard.
+void
+MacroAssemblerPPC64::ma_bc(Register lhs, Imm32 imm, Label* label, Condition c, JumpKind jumpKind)
+{
+ ADBlock();
+ MOZ_ASSERT(!(c & ConditionOnlyXER));
+ if (c == Always) {
+ ma_b(label, jumpKind);
+ return;
+ }
+ if (c & ConditionZero) {
+ MOZ_ASSERT(imm.value == 0);
+ as_cmpdi(lhs, 0);
+ ma_bc(c, label, jumpKind);
+ return;
+ }
+ if (c & ConditionUnsigned) {
+ if (Imm16::IsInUnsignedRange(imm.value)) {
+ as_cmplwi(lhs, imm.value);
+ } else {
+ MOZ_ASSERT(lhs != ScratchRegister);
+ ma_li(ScratchRegister, imm);
+ as_cmplw(lhs, ScratchRegister);
+ }
+ } else {
+ MOZ_ASSERT(c < 0x100); // just in case
+ if (Imm16::IsInSignedRange(imm.value)) {
+ as_cmpwi(lhs, imm.value);
+ } else {
+ MOZ_ASSERT(lhs != ScratchRegister);
+ ma_li(ScratchRegister, imm);
+ as_cmpw(lhs, ScratchRegister);
+ }
+ }
+ ma_bc(c, label, jumpKind);
+}
+
+void
+MacroAssemblerPPC64::ma_bc(Register lhs, ImmPtr imm, Label* l, Condition c, JumpKind jumpKind)
+{
+ asMasm().ma_bc(lhs, ImmWord(uintptr_t(imm.value)), l, c, jumpKind);
+}
+
+void
+MacroAssemblerPPC64::ma_b(Label* label, JumpKind jumpKind)
+{
+ ADBlock();
+ if (!label->bound()) {
+ BufferOffset bo;
+
+ // Emit an unbound branch to be bound later by |Assembler::bind|.
+ // This and ma_bc() are largely the same in this respect.
+ spew(".Llabel %p", label);
+ uint32_t nextInChain = label->used() ? label->offset() : LabelBase::INVALID_OFFSET;
+ if (jumpKind == ShortJump) {
+ // We know this branch must be short. Unfortunately, because we
+ // have to also store the next-in-chain, we can't make this less
+ // than two instructions.
+ m_buffer.ensureSpace(2 * sizeof(uint32_t));
+ bo = as_b(4, RelativeBranch, DontLinkB);
+ spew(".long %08x ; next in chain", nextInChain);
+ writeInst(nextInChain);
+ if (!oom())
+ label->use(bo.getOffset());
+ } else {
+ m_buffer.ensureSpace(7 * sizeof(uint32_t));
+ bo = xs_trap_tagged(BTag);
+ spew(".long %08x ; next in chain", nextInChain);
+ writeInst(nextInChain);
+ if (!oom())
+ label->use(bo.getOffset());
+ // Leave space for potential long jump.
+ as_nop(); // rldicr
+ as_nop(); // oris
+ as_nop(); // ori
+ as_nop(); // mtctr
+ as_nop(); // bctr
+ }
+ return;
+ }
+
+ // Label is bound, emit final code.
+ int64_t offset = label->offset() - currentOffset();
+ if (jumpKind == ShortJump || JOffImm26::IsInRange(offset)) {
+ spew("# static short jump %08x to label %p @ %08x (offset %ld)",
+ currentOffset(), label, label->offset(), offset);
+ MOZ_ASSERT(JOffImm26::IsInRange(offset));
+ as_b(offset);
+ } else {
+ // Use r12 "as expected" even though this is probably not to ABI-compliant code.
+ m_buffer.ensureSpace(7 * sizeof(uint32_t));
+ addLongJump(nextOffset(), BufferOffset(label->offset()));
+ ma_liPatchable(SecondScratchReg, ImmWord(LabelBase::INVALID_OFFSET));
+ xs_mtctr(SecondScratchReg);
+ as_bctr();
+ }
+}
+
+void
+MacroAssemblerPPC64::ma_cmp32(Register lhs, Register rhs, Condition c)
+{
+ ADBlock();
+ MOZ_ASSERT(!(c & ConditionOnlyXER));
+ MOZ_ASSERT(!(c & ConditionZero));
+
+ if (c & ConditionUnsigned) {
+ as_cmplw(lhs, rhs);
+ } else {
+ as_cmpw(lhs, rhs);
+ }
+}
+
+void
+MacroAssemblerPPC64::ma_cmp32(Register lhs, Imm32 rhs, Condition c)
+{
+ ADBlock();
+ MOZ_ASSERT(!(c & ConditionOnlyXER));
+ MOZ_ASSERT_IF((c & ConditionZero), (rhs.value == 0));
+
+ if (c & ConditionZero) {
+ as_cmpwi(lhs, 0);
+ } else {
+ if (c & ConditionUnsigned) {
+ if (Imm16::IsInUnsignedRange(rhs.value)) {
+ as_cmplwi(lhs, rhs.value);
+ } else {
+ MOZ_ASSERT(lhs != ScratchRegister);
+ ma_li(ScratchRegister, rhs);
+ as_cmplw(lhs, ScratchRegister);
+ }
+ } else {
+ if (Imm16::IsInSignedRange(rhs.value)) {
+ as_cmpwi(lhs, rhs.value);
+ } else {
+ MOZ_ASSERT(lhs != ScratchRegister);
+ ma_li(ScratchRegister, rhs);
+ as_cmpw(lhs, ScratchRegister);
+ }
+ }
+ }
+}
+
+void
+MacroAssemblerPPC64::ma_cmp64(Register lhs, Imm64 rhs, Condition c)
+{
+ ADBlock();
+ MOZ_ASSERT(!(c & ConditionOnlyXER));
+ MOZ_ASSERT_IF((c & ConditionZero), (rhs.value == 0));
+
+ if (c & ConditionZero) {
+ as_cmpdi(lhs, 0);
+ } else {
+ if (c & ConditionUnsigned) {
+ if (Imm16::IsInUnsignedRange(rhs.value)) {
+ as_cmpldi(lhs, rhs.value);
+ } else {
+ MOZ_ASSERT(lhs != ScratchRegister);
+ ma_li(ScratchRegister, rhs);
+ as_cmpld(lhs, ScratchRegister);
+ }
+ } else {
+ if (Imm16::IsInSignedRange(rhs.value)) {
+ as_cmpdi(lhs, rhs.value);
+ } else {
+ MOZ_ASSERT(lhs != ScratchRegister);
+ ma_li(ScratchRegister, rhs);
+ as_cmpd(lhs, ScratchRegister);
+ }
+ }
+ }
+}
+
+void
+MacroAssemblerPPC64::ma_cmp32(Register lhs, const Address& rhs, Condition c)
+{
+ MOZ_ASSERT(lhs != ScratchRegister);
+ ma_load(ScratchRegister, rhs, SizeWord);
+ ma_cmp32(lhs, ScratchRegister, c);
+}
+
+void
+MacroAssemblerPPC64::compareFloatingPoint(FloatRegister lhs, FloatRegister rhs,
+ DoubleCondition c)
+{
+ if ((c & DoubleConditionUnordered) || (c == DoubleUnordered)) {
+ as_fcmpu(lhs, rhs);
+ } else {
+ as_fcmpo(lhs, rhs);
+ }
+}
+
+void
+MacroAssemblerPPC64::ma_cmp_set_double(Register dest, FloatRegister lhs, FloatRegister rhs,
+ DoubleCondition c)
+{
+ Label skip;
+ compareFloatingPoint(lhs, rhs, c);
+
+ ma_li(dest, 1L);
+
+// XXX: use CR
+ ma_bc(c, &skip);
+ ma_li(dest, 0L);
+ bind(&skip);
+}
+
+void
+MacroAssemblerPPC64::ma_cmp_set(Register rd, Register rs, Imm16 imm, Condition c, bool useCmpw)
+{
+ ADBlock();
+// XXX: remove duplicate code, just call ma_cmp32 or ma_cmp64 as appropriate
+
+ // Handle any synthetic codes.
+ MOZ_ASSERT_IF((c & ConditionZero), (imm.encode() == 0));
+ MOZ_ASSERT(!(c & ConditionOnlyXER));
+ if (c & ConditionUnsigned) {
+ MOZ_ASSERT(Imm16::IsInUnsignedRange(imm.encode())); // paranoia
+ if (useCmpw) {
+ as_cmplwi(rs, imm.encode());
+ } else {
+ as_cmpldi(rs, imm.encode());
+ }
+ } else {
+ // Just because it's an Imm16 doesn't mean it always fits.
+ if (!Imm16::IsInSignedRange(imm.decodeSigned())) {
+ MOZ_ASSERT(rs != ScratchRegister);
+ ma_li(ScratchRegister, imm.decodeSigned());
+ if (useCmpw) {
+ as_cmpw(rs, ScratchRegister);
+ } else {
+ as_cmpd(rs, ScratchRegister);
+ }
+ } else {
+ if (useCmpw) {
+ as_cmpwi(rs, imm.decodeSigned());
+ } else {
+ as_cmpdi(rs, imm.decodeSigned());
+ }
+ }
+ }
+ // Common routine to extract or flip the appropriate CR bit.
+ ma_cmp_set_coda(rd, c);
+}
+
+void
+MacroAssemblerPPC64::ma_cmp_set(Register rd, Register rs, Register rt, Condition c, bool useCmpw)
+{
+ ADBlock();
+
+ // Handle any synthetic codes.
+ MOZ_ASSERT(!(c & ConditionOnlyXER));
+ MOZ_ASSERT(!(c & ConditionZero));
+ if (c & ConditionUnsigned) {
+ // Some compares should only pay attention to the lower word.
+ if (useCmpw) {
+ as_cmplw(rs, rt);
+ } else {
+ as_cmpld(rs, rt);
+ }
+ } else {
+ if (useCmpw) {
+ as_cmpw(rs, rt);
+ } else {
+ as_cmpd(rs, rt);
+ }
+ }
+ ma_cmp_set_coda(rd, c);
+}
+
+static_assert(((Assembler::LessThanOrEqual & Assembler::BranchOptionMask) == Assembler::BranchOnClear),
+ "Assembler conditions don't match CR bits");
+void
+MacroAssemblerPPC64::ma_cmp_set_coda(Register rd, Condition c) {
+ MOZ_ASSERT(!(c & ConditionOnlyXER));
+
+ // Extract the underlying CR field bit.
+ as_mfcr(rd);
+ switch(c & 0xff) {
+ case Equal:
+ case NotEqual:
+ as_rlwinm(rd, rd, 3, 31, 31); // PowerPC CWG page 38
+ break;
+ case GreaterThan:
+ case LessThanOrEqual:
+ as_rlwinm(rd, rd, 2, 31, 31);
+ break;
+ case LessThan:
+ case GreaterThanOrEqual:
+ as_rlwinm(rd, rd, 1, 31, 31); // p40
+ break;
+ default:
+ MOZ_CRASH("Unhandled condition");
+ break;
+ }
+
+ // Negate the boolean if necessary.
+ if ((c & BranchOptionMask) == BranchOnClear) {
+ as_xori(rd, rd, 1);
+ }
+}
+
+// fp instructions
+void
+MacroAssemblerPPC64::ma_lis(FloatRegister dest, float value)
+{
+ Imm32 imm(mozilla::BitwiseCast<uint32_t>(value));
+
+ ma_li(ScratchRegister, imm);
+ asMasm().moveToFloat32(ScratchRegister, dest);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64::ma_sd(FloatRegister ft, BaseIndex address)
+{
+/*
+ if (Imm16::IsInSignedRange(address.offset) && address.scale == TimesOne) {
+ as_stfd(ft, address.base, address.offset);
+ return;
+ }
+*/
+
+ asMasm().computeScaledAddress(address, SecondScratchReg);
+ return asMasm().ma_sd(ft, Address(SecondScratchReg, address.offset));
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64::ma_ss(FloatRegister ft, BaseIndex address)
+{
+/*
+ if (Imm16::IsInSignedRange(address.offset) && address.scale == TimesOne) {
+ as_stfs(ft, address.base, address.offset);
+ return;
+ }
+*/
+
+ asMasm().computeScaledAddress(address, SecondScratchReg);
+ return asMasm().ma_ss(ft, Address(SecondScratchReg, address.offset));
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64::ma_ld(FloatRegister ft, const BaseIndex& src)
+{
+ asMasm().computeScaledAddress(src, SecondScratchReg);
+ return asMasm().ma_ld(ft, Address(SecondScratchReg, src.offset));
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64::ma_ls(FloatRegister ft, const BaseIndex& src)
+{
+ asMasm().computeScaledAddress(src, SecondScratchReg);
+ return asMasm().ma_ls(ft, Address(SecondScratchReg, src.offset));
+}
+
+void
+MacroAssemblerPPC64::minMaxDouble(FloatRegister srcDest, FloatRegister second,
+ bool handleNaN, bool isMax)
+{
+ ADBlock();
+ Label zero, done, nan;
+
+ if (handleNaN) {
+ // First or second is NaN, result is NaN.
+ compareFloatingPoint(srcDest, second, Assembler::DoubleUnordered);
+ ma_bc(Assembler::DoubleUnordered, &nan, ShortJump);
+ }
+ // Check for zero and equal.
+ asMasm().loadConstantDouble(0.0, ScratchDoubleReg);
+ as_fcmpo(srcDest, ScratchDoubleReg);
+ as_fcmpo(cr1, second, ScratchDoubleReg);
+ // Hoist the sub here because we've already done the compare and the
+ // crand will serialize.
+ as_fsub(ScratchDoubleReg, srcDest, second);
+ as_crand(2, 6, 2); // CR0[EQ] &= CR1[EQ]
+ // We can use ::Equal, because the unordered check is done, and save
+ // emitting a crandc we don't actually need.
+ ma_bc(Assembler::Equal, &zero, ShortJump);
+
+ // Neither can be zero. Use fsel.
+ if (isMax) {
+ as_fsel(srcDest, ScratchDoubleReg, srcDest, second);
+ } else {
+ as_fsel(srcDest, ScratchDoubleReg, second, srcDest);
+ }
+ ma_b(&done, ShortJump);
+
+ if (handleNaN) {
+ bind(&nan);
+ asMasm().loadConstantDouble(JS::GenericNaN(), srcDest);
+ ma_b(&done, ShortJump);
+ }
+
+ // Make sure we handle -0 and 0 right. Dump A into a GPR and check
+ // the sign bit.
+ bind(&zero);
+ asMasm().moveFromDouble(srcDest, ScratchRegister);
+ as_cmpdi(ScratchRegister, 0);
+ if (isMax) {
+ // If a == -0, then b is either -0 or 0. In this case, return b.
+ ma_bc(Assembler::GreaterThanOrEqual, &done, ShortJump); // a == 0
+ } else {
+ // If a == -0, then b is either -0 or 0. In this case, return a.
+ ma_bc(Assembler::LessThan, &done, ShortJump); // a == -0
+ }
+ as_fmr(srcDest, second);
+ bind(&done);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64::loadDouble(const Address& address, FloatRegister dest)
+{
+ return ma_ld(dest, address);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64::loadDouble(const BaseIndex& src, FloatRegister dest)
+{
+ return ma_ld(dest, src);
+}
+
+void
+MacroAssemblerPPC64::loadFloatAsDouble(const Address& address, FloatRegister dest)
+{
+ ma_ls(dest, address);
+}
+
+void
+MacroAssemblerPPC64::loadFloatAsDouble(const BaseIndex& src, FloatRegister dest)
+{
+ ma_ls(dest, src);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64::loadFloat32(const Address& address, FloatRegister dest)
+{
+ return ma_ls(dest, address);
+}
+
+FaultingCodeOffset
+MacroAssemblerPPC64::loadFloat32(const BaseIndex& src, FloatRegister dest)
+{
+ return ma_ls(dest, src);
+}
+
+void
+MacroAssemblerPPC64::ma_call(ImmPtr dest)
+{
+ asMasm().ma_liPatchable(CallReg, dest);
+ xs_mtctr(CallReg);
+ as_bctr(LinkB);
+}
+
+void
+MacroAssemblerPPC64::ma_jump(ImmPtr dest)
+{
+ asMasm().ma_liPatchable(SecondScratchReg, dest);
+ xs_mtctr(SecondScratchReg);
+ as_bctr();
+}
+
+MacroAssembler&
+MacroAssemblerPPC64::asMasm()
+{
+ return *static_cast<MacroAssembler*>(this);
+}
+
+const MacroAssembler&
+MacroAssemblerPPC64::asMasm() const
+{
+ return *static_cast<const MacroAssembler*>(this);
+}
+
+//{{{ check_macroassembler_style
+// ===============================================================
+// MacroAssembler high-level usage.
+
+void
+MacroAssembler::flush()
+{
+}
+
+// ===============================================================
+// Stack manipulation functions.
+
+void
+MacroAssembler::Push(Register reg)
+{
+ ma_push(reg);
+ adjustFrame(int32_t(sizeof(intptr_t)));
+}
+
+void
+MacroAssembler::Push(const Imm32 imm)
+{
+ ma_li(ScratchRegister, imm);
+ ma_push(ScratchRegister);
+ adjustFrame(int32_t(sizeof(intptr_t)));
+}
+
+void
+MacroAssembler::Push(const ImmWord imm)
+{
+ ma_li(ScratchRegister, imm);
+ ma_push(ScratchRegister);
+ adjustFrame(int32_t(sizeof(intptr_t)));
+}
+
+void
+MacroAssembler::Push(const ImmPtr imm)
+{
+ Push(ImmWord(uintptr_t(imm.value)));
+}
+
+void
+MacroAssembler::Push(const ImmGCPtr ptr)
+{
+ ma_li(ScratchRegister, ptr);
+ ma_push(ScratchRegister);
+ adjustFrame(int32_t(sizeof(intptr_t)));
+}
+
+void
+MacroAssembler::Push(FloatRegister f)
+{
+ ma_push(f);
+ adjustFrame(int16_t(sizeof(double))); // Keep stack aligned to double, even if it's a float.
+}
+
+void
+MacroAssembler::Pop(Register reg)
+{
+ ma_pop(reg);
+ adjustFrame(-int16_t(sizeof(intptr_t)));
+}
+
+void
+MacroAssembler::Pop(FloatRegister f)
+{
+ ma_pop(f);
+ adjustFrame(-int16_t(sizeof(double)));
+}
+
+void
+MacroAssembler::Pop(const ValueOperand& val)
+{
+ popValue(val);
+ adjustFrame(-int16_t(sizeof(Value)));
+}
+
+void
+MacroAssembler::PopStackPtr()
+{
+ loadPtr(Address(StackPointer, 0), StackPointer);
+ adjustFrame(-int16_t(sizeof(intptr_t)));
+}
+
+void MacroAssembler::freeStackTo(uint32_t framePushed) {
+ MOZ_ASSERT(framePushed <= framePushed_);
+ MOZ_ASSERT(framePushed <= 32767);
+ as_addi(StackPointer, FramePointer, -(int16_t(framePushed)));
+ framePushed_ = framePushed;
+}
+
+
+// ===============================================================
+// Simple call functions.
+
+CodeOffset
+MacroAssembler::call(Register reg)
+{
+ xs_mtctr(reg);
+ as_bctr(LinkB);
+ return CodeOffset(currentOffset());
+}
+
+CodeOffset
+MacroAssembler::call(Label* label)
+{
+ ma_bal(label);
+ return CodeOffset(currentOffset());
+}
+
+CodeOffset
+MacroAssembler::callWithPatch()
+{
+ ADBlock();
+
+ // Patched with |patchCall|. Assuming we set JumpImmediateRange correctly
+ // in Architecture-ppc64.h, we will never get a long branch given to this
+ // routine, so a naked bl is sufficient.
+ m_buffer.ensureSpace(sizeof(uint32_t));
+ as_b(0, RelativeBranch, LinkB); // infinite loop if unpatched
+ return CodeOffset(currentOffset());
+}
+
+void
+MacroAssembler::patchCall(uint32_t callerOffset, uint32_t calleeOffset)
+{
+ // Account for the bl (the code offset points to the return address).
+ int32_t offset = calleeOffset - callerOffset + 4;
+
+ // Get a handle to the instruction.
+ Instruction* inst = editSrc(BufferOffset(callerOffset - 4));
+ MOZ_ASSERT(inst->extractOpcode() == PPC_b);
+ MOZ_ASSERT(inst[0].encode() & LinkB); // only patch calls!
+ MOZ_ASSERT(JOffImm26::IsInRange(offset)); // damn well better be
+
+ inst->setData(PPC_b | JOffImm26(offset).encode() | LinkB);
+ FlushICache(inst, sizeof(uint32_t));
+}
+
+CodeOffset
+MacroAssembler::farJumpWithPatch()
+{
+ ADBlock();
+ CodeOffset farJump;
+
+ // Patched with |farJumpWithPatch|. It is guaranteed to be a full stanza.
+ if (HasPPCISA3()) {
+ // We can use addpcis to access the PC directly. Use r12 "as expected"
+ // even though this is not necessarily to ABI-compliant code.
+ m_buffer.ensureSpace(10 * sizeof(uint32_t));
+ as_addpcis(SecondScratchReg, 0); // r12 = address of jump
+
+ farJump.bind(currentOffset());
+ // NIP is here (one word after the non-POWER9 version).
+ ma_liPatchable(ScratchRegister, ImmWord(LabelBase::INVALID_OFFSET));
+ as_add(SecondScratchReg, SecondScratchReg, ScratchRegister);
+ xs_mtctr(SecondScratchReg);
+ as_bctr();
+ } else {
+ // We need to do some footwork to get LR since this is PC-relative.
+ m_buffer.ensureSpace(13 * sizeof(uint32_t));
+
+ // Because this whacks LR, we have to save it first. Unfortunately
+ // we'll need a third scratch register here.
+ xs_mflr(ThirdScratchReg); // 4 // 48
+ // Use the special bcl-Always (20,31) form to avoid tainting the BHRB.
+ xs_bcl_always(4); // 8 // 44
+ xs_mflr(SecondScratchReg); // 12; LR points here // 40
+
+ farJump.bind(currentOffset());
+ ma_liPatchable(ScratchRegister, ImmWord(LabelBase::INVALID_OFFSET)); // 32
+ as_add(SecondScratchReg, SecondScratchReg, ScratchRegister); // 36 // 16
+ xs_mtctr(SecondScratchReg); // 40 // 12
+ xs_mtlr(ThirdScratchReg); // 44; restore LR // 8
+ as_bctr(); // 48 // 4
+ }
+
+ return farJump;
+}
+
+void
+MacroAssembler::patchFarJump(CodeOffset farJump, uint32_t targetOffset)
+{
+ int32_t offset;
+
+ if (HasPPCISA3()) {
+ // NIP is pointing at the stanza.
+ offset = targetOffset - farJump.offset();
+ } else {
+ // The offset is the patchable stanza, but LR is pointing at the mflr
+ // right before it.
+ offset = sizeof(uint32_t) + targetOffset - farJump.offset();
+ }
+
+ // Get a handle to the branch stanza.
+ Instruction* inst = editSrc(BufferOffset(farJump.offset()));
+ MOZ_ASSERT(inst->extractOpcode() == PPC_addis);
+
+ Assembler::WriteLoad64Instructions(inst, ScratchRegister, (uint64_t)offset);
+ FlushICache(inst, sizeof(uint32_t) * 5);
+}
+
+CodeOffset
+MacroAssembler::call(wasm::SymbolicAddress target)
+{
+ ADBlock();
+
+ // This call is very likely to ABI compliant code. Since this is coming
+ // from Wasm and Wasm Frames sit on the top of the stack where the linkage
+ // area goes, we need to pull down a dummy ABI stack frame to prevent the
+ // callee from unwittingly stomping on the Wasm Frame. ShadowStackSpace
+ // does not fix this; see Architecture-ppc64le.h for a more intemperate
+ // explanation. We can get away with this in the general case because the
+ // argument registers have already been calculated relative to the prior
+ // (unsafe) value of the stack pointer. If it's not to ABI compliant code,
+ // then we just bloat the stack temporarily and life goes on.
+ //
+ // 512 bytes ought to be enough for anybody ...
+ as_addi(StackPointer, StackPointer, -512);
+ movePtr(target, CallReg);
+ // XXX: No current consumer seems to care about the return value.
+ // Should it be after the call, or after the stack pointer adjustment?
+ CodeOffset c = call(CallReg);
+ as_addi(StackPointer, StackPointer, 512);
+ return c;
+}
+
+void
+MacroAssembler::call(const Address& addr)
+{
+ loadPtr(addr, CallReg);
+ call(CallReg);
+}
+
+void
+MacroAssembler::call(ImmWord target)
+{
+ call(ImmPtr((void*)target.value));
+}
+
+void
+MacroAssembler::call(ImmPtr target)
+{
+ BufferOffset bo = m_buffer.nextOffset();
+ addPendingJump(bo, target, RelocationKind::HARDCODED);
+ ma_call(target);
+}
+
+void
+MacroAssembler::call(JitCode* c)
+{
+ BufferOffset bo = m_buffer.nextOffset();
+ addPendingJump(bo, ImmPtr(c->raw()), RelocationKind::JITCODE);
+ ma_liPatchable(SecondScratchReg, ImmPtr(c->raw()));
+ callJitNoProfiler(SecondScratchReg);
+}
+
+CodeOffset
+MacroAssembler::nopPatchableToCall()
+{
+ ADBlock();
+ as_nop(); // lis
+ as_nop(); // ori
+ as_nop(); // rldicr
+ as_nop(); // oris
+ as_nop(); // ori
+ as_nop(); // mtctr
+ as_nop(); // bctrl
+ // Even though this isn't a call (yet), treat it like one, and have
+ // the returned offset match the future return address.
+ return CodeOffset(currentOffset());
+}
+
+void
+MacroAssembler::patchNopToCall(uint8_t* call, uint8_t* target)
+{
+ // This will always be a full call stanza.
+ Instruction* inst = (Instruction*) call;
+ inst -= 7; // rewind to the first nop
+#if DEBUG
+ if (inst[0].encode() == PPC_nop) {
+ MOZ_ASSERT(inst[1].encode() == PPC_nop);
+ MOZ_ASSERT(inst[2].encode() == PPC_nop);
+ MOZ_ASSERT(inst[3].encode() == PPC_nop);
+ MOZ_ASSERT(inst[4].encode() == PPC_nop);
+ MOZ_ASSERT(inst[5].encode() == PPC_nop);
+ MOZ_ASSERT(inst[6].encode() == PPC_nop);
+ } else {
+ // It is permitted to patch a pre-existing call.
+ MOZ_ASSERT(inst->extractOpcode() == PPC_addis); // lis
+ MOZ_ASSERT(inst[6].encode() == (PPC_bctr | LinkB)); // bctrl
+ }
+#endif
+
+ Assembler::WriteLoad64Instructions(inst, SecondScratchReg, (uint64_t) target);
+ inst[5].makeOp_mtctr(SecondScratchReg);
+ inst[6].makeOp_bctr(LinkB);
+
+ FlushICache(inst, sizeof(uint32_t) * 7);
+}
+
+void
+MacroAssembler::patchCallToNop(uint8_t* call)
+{
+ // everything be nops now yo
+ Instruction* inst = (Instruction*) call;
+ inst -= 7; // rewind to the stanza entry
+ MOZ_ASSERT(inst->extractOpcode() == PPC_addis); // lis
+ MOZ_ASSERT(inst[6].encode() == (PPC_bctr | LinkB)); // bctrl
+
+ inst[0].makeOp_nop();
+ inst[1].makeOp_nop();
+ inst[2].makeOp_nop();
+ inst[3].makeOp_nop();
+ inst[4].makeOp_nop();
+ inst[5].makeOp_nop();
+ inst[6].makeOp_nop();
+
+ FlushICache(inst, sizeof(uint32_t) * 7);
+}
+
+void
+MacroAssembler::pushReturnAddress()
+{
+ ADBlock();
+ xs_mflr(ScratchRegister);
+ as_stdu(ScratchRegister, StackPointer, -8);
+}
+
+void
+MacroAssembler::popReturnAddress()
+{
+ ADBlock();
+ as_ld(ScratchRegister, StackPointer, 0);
+ xs_mtlr(ScratchRegister);
+ as_addi(StackPointer, StackPointer, 8);
+}
+
+// ===============================================================
+// Jit Frames.
+
+uint32_t
+MacroAssembler::pushFakeReturnAddress(Register scratch)
+{
+ ADBlock();
+ CodeLabel cl;
+
+ ma_li(scratch, &cl);
+ Push(scratch);
+ bind(&cl);
+ uint32_t retAddr = currentOffset();
+
+ addCodeLabel(cl);
+ return retAddr;
+}
+
+void
+MacroAssembler::loadStoreBuffer(Register ptr, Register buffer)
+{
+ ma_and(buffer, ptr, Imm32(int32_t(~gc::ChunkMask)));
+ loadPtr(Address(buffer, gc::ChunkStoreBufferOffset), buffer);
+}
+
+void
+MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr, Register temp,
+ Label* label)
+{
+ ADBlock();
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ MOZ_ASSERT(ptr != temp);
+ MOZ_ASSERT(ptr != ScratchRegister); // Both may be used internally.
+ MOZ_ASSERT(ptr != SecondScratchReg);
+ MOZ_ASSERT(temp != ScratchRegister); // probably unpossible
+ MOZ_ASSERT(temp != SecondScratchReg);
+ MOZ_ASSERT(gc::ChunkStoreBufferOffset < 32767);
+
+ ma_and(SecondScratchReg, ptr, Imm32(int32_t(~gc::ChunkMask)));
+ branchPtr(InvertCondition(cond), Address(SecondScratchReg, gc::ChunkStoreBufferOffset),
+ ImmWord(0), label);
+}
+
+void
+MacroAssembler::comment(const char* msg)
+{
+ Assembler::comment(msg);
+}
+
+// ===============================================================
+// WebAssembly
+
+FaultingCodeOffset
+MacroAssembler::wasmTrapInstruction()
+{
+ FaultingCodeOffset offset(currentOffset());
+ // Only Wasm uses this, and it uses it to force a signal that the Wasm
+ // handler will then intercept. We don't want to use a trap instruction
+ // because hooking SIGTRAP will interfere with debugging. So we use a
+ // stop instruction: it was illegal prior to ISA 3.0, and is privileged
+ // on 3.0+, so it will cause a SIGILL no matter what CPU it's run on.
+ // Helpfully, SIGILL is exactly what the Wasm signal handler is watching
+ // for.
+ as_stop();
+ return offset;
+}
+
+void
+MacroAssembler::wasmTruncateDoubleToInt32(FloatRegister input, Register output, bool isSaturating,
+ Label* oolEntry)
+{
+ ADBlock();
+
+ // We only care if the conversion is invalid, not if it's inexact.
+ // Negative zero is treated like positive zero.
+ // Whack VXCVI.
+ as_mtfsb0(23);
+ as_fctiwz(ScratchDoubleReg, input);
+ // VXCVI is a failure (over/underflow, NaN, etc.)
+ as_mcrfs(cr0, 5); // reserved - VXSOFT - VXSQRT - VXCVI -> CR0[...SO]
+ ma_bc(Assembler::SOBit, oolEntry);
+
+ // OutOfLineTruncateCheckF32/F64ToI32 -> outOfLineWasmTruncateToInt32Check
+ moveFromDouble(ScratchDoubleReg, output);
+ // Clear and sign extend.
+ as_srawi(output, output, 0);
+}
+
+
+void
+MacroAssembler::wasmTruncateFloat32ToInt32(FloatRegister input, Register output, bool isSaturating,
+ Label* oolEntry)
+{
+ wasmTruncateDoubleToInt32(input, output, isSaturating, oolEntry);
+}
+
+void
+MacroAssembler::oolWasmTruncateCheckF32ToI32(FloatRegister input, Register output,
+ TruncFlags flags, wasm::BytecodeOffset off,
+ Label* rejoin)
+{
+ outOfLineWasmTruncateToInt32Check(input, output, MIRType::Float32, flags, rejoin, off);
+}
+
+void
+MacroAssembler::oolWasmTruncateCheckF64ToI32(FloatRegister input, Register output,
+ TruncFlags flags, wasm::BytecodeOffset off,
+ Label* rejoin)
+{
+ outOfLineWasmTruncateToInt32Check(input, output, MIRType::Double, flags, rejoin, off);
+}
+
+void
+MacroAssembler::oolWasmTruncateCheckF32ToI64(FloatRegister input, Register64 output,
+ TruncFlags flags, wasm::BytecodeOffset off,
+ Label* rejoin)
+{
+ outOfLineWasmTruncateToInt64Check(input, output, MIRType::Float32, flags, rejoin, off);
+}
+
+void
+MacroAssembler::oolWasmTruncateCheckF64ToI64(FloatRegister input, Register64 output,
+ TruncFlags flags, wasm::BytecodeOffset off,
+ Label* rejoin)
+{
+ outOfLineWasmTruncateToInt64Check(input, output, MIRType::Double, flags, rejoin, off);
+}
+
+void
+MacroAssemblerPPC64::outOfLineWasmTruncateToInt32Check(FloatRegister input, Register output,
+ MIRType fromType, TruncFlags flags,
+ Label* rejoin,
+ wasm::BytecodeOffset trapOffset)
+{
+ ADBlock();
+ // We must be using a signed truncation or a truncation that is not
+ // saturating, and the FPSCR VXCVI bit got set indicating an inexact
+ // conversion or a NaN. Unsigned saturating truncates already "do the
+ // right thing" with their conversion, so they never end up here.
+
+ Label inputIsNaN;
+ bool isSaturating = flags & TRUNC_SATURATING;
+
+ // Test for NaN.
+ compareFloatingPoint(input, input, Assembler::DoubleUnordered);
+ ma_bc(Assembler::DoubleUnordered, &inputIsNaN, ShortJump);
+
+ if (isSaturating) {
+ // fctiwz and fctiwuz both saturate to their respective extents, so
+ // we can rejoin; the output value is correct. Do the work we would
+ // have done inline if there had been no exception. The scratch
+ // register still contains the result.
+ asMasm().moveFromDouble(ScratchDoubleReg, ScratchRegister);
+ // Don't sign extend.
+ as_rldicl(output, ScratchRegister, 0, 32); // "clrldi"
+ // Rejoin. This follows the truncation stanza.
+ ma_b(rejoin);
+ } else {
+ // We must have overflowed.
+ asMasm().wasmTrap(wasm::Trap::IntegerOverflow, trapOffset);
+ }
+
+ asMasm().bind(&inputIsNaN);
+ if (isSaturating) {
+ // A saturated NaN is zero. (fctiwuz does this for us.)
+ xs_li(output, 0);
+ ma_b(rejoin);
+ } else {
+ asMasm().wasmTrap(wasm::Trap::InvalidConversionToInteger, trapOffset);
+ }
+}
+
+void
+MacroAssemblerPPC64::outOfLineWasmTruncateToInt64Check(FloatRegister input, Register64 output,
+ MIRType fromType, TruncFlags flags,
+ Label* rejoin,
+ wasm::BytecodeOffset trapOffset)
+{
+ ADBlock();
+ // We must be using a signed truncation or a truncation that is not
+ // saturating, and the FPSCR VXCVI bit got set indicating an inexact
+ // conversion or a NaN. Unsigned saturating truncates already "do the
+ // right thing" with their conversion, so they never end up here.
+
+ Label inputIsNaN;
+ bool isSaturating = flags & TRUNC_SATURATING;
+
+ // Test for NaN.
+ compareFloatingPoint(input, input, Assembler::DoubleUnordered);
+ ma_bc(Assembler::DoubleUnordered, &inputIsNaN, ShortJump);
+
+ if (isSaturating) {
+ // fctidz and fctiduz both saturate to their respective extents, so
+ // we can rejoin; the output value is correct. Do the work we would
+ // have done inline if there had been no exception. The scratch
+ // register still contains the result.
+ asMasm().moveFromDouble(ScratchDoubleReg, output.reg);
+ ma_b(rejoin);
+ } else {
+ // We must have overflowed.
+ asMasm().wasmTrap(wasm::Trap::IntegerOverflow, trapOffset);
+ }
+
+ asMasm().bind(&inputIsNaN);
+ if (isSaturating) {
+ // A saturated NaN is zero. (fctiduz does this for us.)
+ xs_li(output.reg, 0);
+ ma_b(rejoin);
+ } else {
+ asMasm().wasmTrap(wasm::Trap::InvalidConversionToInteger, trapOffset);
+ }
+}
+
+void
+MacroAssembler::wasmLoad(const wasm::MemoryAccessDesc& access, Register memoryBase, Register ptr,
+ Register ptrScratch, AnyRegister output)
+{
+ ADBlock();
+ wasmLoadImpl(access, memoryBase, ptr, ptrScratch, output, InvalidReg);
+}
+
+void
+MacroAssembler::wasmUnalignedLoad(const wasm::MemoryAccessDesc& access, Register memoryBase,
+ Register ptr, Register ptrScratch, Register output, Register tmp)
+{
+ ADBlock();
+ wasmLoadImpl(access, memoryBase, ptr, ptrScratch, AnyRegister(output), tmp);
+}
+
+void
+MacroAssembler::wasmUnalignedLoadFP(const wasm::MemoryAccessDesc& access, Register memoryBase,
+ Register ptr, Register ptrScratch, FloatRegister output,
+ Register tmp1)
+{
+ ADBlock();
+ wasmLoadImpl(access, memoryBase, ptr, ptrScratch, AnyRegister(output), tmp1);
+}
+
+void
+MacroAssembler::wasmStore(const wasm::MemoryAccessDesc& access, AnyRegister value,
+ Register memoryBase, Register ptr, Register ptrScratch)
+{
+ ADBlock();
+ wasmStoreImpl(access, value, memoryBase, ptr, ptrScratch, InvalidReg);
+}
+
+void
+MacroAssembler::wasmUnalignedStore(const wasm::MemoryAccessDesc& access, Register value,
+ Register memoryBase, Register ptr, Register ptrScratch,
+ Register tmp)
+{
+ ADBlock();
+ wasmStoreImpl(access, AnyRegister(value), memoryBase, ptr, ptrScratch, tmp);
+}
+
+void
+MacroAssembler::wasmUnalignedStoreFP(const wasm::MemoryAccessDesc& access, FloatRegister floatValue,
+ Register memoryBase, Register ptr, Register ptrScratch,
+ Register tmp)
+{
+ ADBlock();
+ wasmStoreImpl(access, AnyRegister(floatValue), memoryBase, ptr, ptrScratch, tmp);
+}
+
+void
+MacroAssemblerPPC64::wasmLoadImpl(const wasm::MemoryAccessDesc& access, Register memoryBase,
+ Register ptr, Register ptrScratch, AnyRegister output,
+ Register tmp)
+{
+ ADBlock();
+ access.assertOffsetInGuardPages();
+ uint32_t offset = access.offset();
+ FaultingCodeOffset fco;
+ MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg);
+
+ // Maybe add the offset.
+ if (offset) {
+ asMasm().addPtr(Imm32(offset), ptrScratch);
+ ptr = ptrScratch;
+ }
+
+ unsigned byteSize = access.byteSize();
+ bool isSigned;
+ bool isFloat = false;
+
+ switch (access.type()) {
+ case Scalar::Int8: isSigned = true; break;
+ case Scalar::Uint8: isSigned = false; break;
+ case Scalar::Int16: isSigned = true; break;
+ case Scalar::Uint16: isSigned = false; break;
+ case Scalar::Int32: isSigned = true; break;
+ case Scalar::Uint32: isSigned = false; break;
+ case Scalar::Float64: isFloat = true; break;
+ case Scalar::Float32: isFloat = true; break;
+ default: MOZ_CRASH("unexpected array type");
+ }
+
+ BaseIndex address(memoryBase, ptr, TimesOne);
+/*
+ if (IsUnaligned(access)) {
+ if (isFloat) {
+ if (byteSize == 4)
+ asMasm().loadUnalignedFloat32(access, address, tmp, output.fpu());
+ else
+ asMasm().loadUnalignedDouble(access, address, tmp, output.fpu());
+ } else {
+ asMasm().ma_load(output.gpr(), address, static_cast<LoadStoreSize>(8 * byteSize), isSigned ? SignExtend : ZeroExtend);
+ }
+ return;
+ }
+*/
+
+ asMasm().memoryBarrierBefore(access.sync());
+ if (isFloat) {
+ if (byteSize == 4)
+ fco = asMasm().ma_ls(output.fpu(), address);
+ else
+ fco = asMasm().ma_ld(output.fpu(), address);
+ } else {
+ // This function doesn't handle 64-bit ints, so we should never
+ // end up in a situation where we have to break a load apart.
+ MOZ_ASSERT(byteSize < 8);
+ fco = asMasm().ma_load(output.gpr(), address, static_cast<LoadStoreSize>(8 * byteSize),
+ isSigned ? SignExtend : ZeroExtend);
+ }
+ asMasm().append(access, js::wasm::TrapMachineInsnForLoad(access.byteSize()),
+ fco);
+ asMasm().memoryBarrierAfter(access.sync());
+}
+
+void
+MacroAssemblerPPC64::wasmStoreImpl(const wasm::MemoryAccessDesc& access, AnyRegister value,
+ Register memoryBase, Register ptr, Register ptrScratch,
+ Register tmp)
+{
+ ADBlock();
+ access.assertOffsetInGuardPages();
+ uint32_t offset = access.offset();
+ MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg);
+
+ // Maybe add the offset.
+ if (offset) {
+ asMasm().addPtr(Imm32(offset), ptrScratch);
+ ptr = ptrScratch;
+ }
+
+ unsigned byteSize = access.byteSize();
+ bool isSigned;
+ bool isFloat = false;
+
+ switch (access.type()) {
+ case Scalar::Int8: isSigned = true; break;
+ case Scalar::Uint8: isSigned = false; break;
+ case Scalar::Int16: isSigned = true; break;
+ case Scalar::Uint16: isSigned = false; break;
+ case Scalar::Int32: isSigned = true; break;
+ case Scalar::Uint32: isSigned = false; break;
+ case Scalar::Int64: isSigned = true; break;
+ case Scalar::Float64: isFloat = true; break;
+ case Scalar::Float32: isFloat = true; break;
+ default: MOZ_CRASH("unexpected array type");
+ }
+
+ BaseIndex address(memoryBase, ptr, TimesOne);
+/*
+ if (IsUnaligned(access)) {
+ if (isFloat) {
+ if (byteSize == 4)
+ asMasm().storeUnalignedFloat32(access, value.fpu(), tmp, address);
+ else
+ asMasm().storeUnalignedDouble(access, value.fpu(), tmp, address);
+ } else {
+ asMasm().ma_store(value.gpr(), address,
+ static_cast<LoadStoreSize>(8 * byteSize),
+ isSigned ? SignExtend : ZeroExtend);
+ }
+ return;
+ }
+*/
+
+ asMasm().memoryBarrierBefore(access.sync());
+ FaultingCodeOffset fco;
+ if (isFloat) {
+ // The store instruction will always be last.
+ if (byteSize == 4)
+ fco = asMasm().ma_ss(value.fpu(), address);
+ else
+ fco = asMasm().ma_sd(value.fpu(), address);
+ } else {
+ fco = asMasm().ma_store(value.gpr(), address,
+ static_cast<LoadStoreSize>(8 * byteSize));
+/*
+ if (loadSize & 0x01) {
+ // Split store emitted.
+ asMasm().append(access, loadSize - 1);
+ // The second store is always the last instruction.
+ asMasm().append(access, asMasm().size() - 4);
+ } else {
+ asMasm().append(access, loadSize);
+ }
+*/
+ }
+ asMasm().append(access, js::wasm::TrapMachineInsnForStore(access.byteSize()), fco);
+ asMasm().memoryBarrierAfter(access.sync());
+}
+
+void
+MacroAssembler::enterFakeExitFrameForWasm(Register cxreg, Register scratch, ExitFrameType type)
+{
+ enterFakeExitFrame(cxreg, scratch, type);
+}
+
+// ========================================================================
+// Primitive atomic operations.
+
+template<typename T>
+static void
+CompareExchange(MacroAssembler& masm, const wasm::MemoryAccessDesc* access,
+Scalar::Type type, Synchronization sync, const T& mem,
+ Register oldval, Register newval, Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register output)
+{
+ bool signExtend = Scalar::isSignedIntType(type);
+ unsigned nbytes = Scalar::byteSize(type);
+
+ switch (nbytes) {
+ case 1:
+ case 2:
+ break;
+ case 4:
+ MOZ_ASSERT(valueTemp == InvalidReg);
+ MOZ_ASSERT(offsetTemp == InvalidReg);
+ MOZ_ASSERT(maskTemp == InvalidReg);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ Label again, end;
+ masm.computeEffectiveAddress(mem, SecondScratchReg);
+
+ if (nbytes == 4) {
+ masm.memoryBarrierBefore(sync);
+ masm.bind(&again);
+
+ if (access) masm.append(*access, wasm::TrapMachineInsn::Atomic,
+ FaultingCodeOffset(masm.currentOffset()));
+ masm.as_lwarx(output, r0, SecondScratchReg);
+ masm.ma_bc(output, oldval, &end, Assembler::NotEqual, ShortJump);
+ /* if (access) masm.append(*access, masm.size()); */
+ masm.as_stwcx(newval, r0, SecondScratchReg);
+ masm.ma_bc(Assembler::NotEqual, &again, ShortJump);
+
+ masm.memoryBarrierAfter(sync);
+ masm.bind(&end);
+
+ return;
+ }
+
+ masm.as_andi_rc(offsetTemp, SecondScratchReg, 3);
+ masm.subPtr(offsetTemp, SecondScratchReg);
+#if !MOZ_LITTLE_ENDIAN()
+ masm.as_xori(offsetTemp, offsetTemp, 3);
+#endif
+ masm.x_slwi(offsetTemp, offsetTemp, 3);
+ masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8)));
+ masm.as_slw(maskTemp, maskTemp, offsetTemp);
+ masm.as_nor(maskTemp, maskTemp, maskTemp);
+
+ masm.memoryBarrierBefore(sync);
+
+ masm.bind(&again);
+
+ if (access) masm.append(*access, wasm::TrapMachineInsn::Atomic,
+ FaultingCodeOffset(masm.currentOffset()));
+ masm.as_lwarx(ScratchRegister, r0, SecondScratchReg);
+
+ masm.as_srw(output, ScratchRegister, offsetTemp);
+
+ switch (nbytes) {
+ case 1:
+ masm.as_andi_rc(valueTemp, oldval, 0xff);
+ masm.as_andi_rc(output, output, 0xff);
+ if (signExtend) {
+ masm.as_extsb(valueTemp, oldval);
+ masm.as_extsb(output, output);
+ }
+ break;
+ case 2:
+ masm.as_andi_rc(valueTemp, oldval, 0xffff);
+ masm.as_andi_rc(output, output, 0xffff);
+ if (signExtend) {
+ masm.as_extsh(valueTemp, oldval);
+ masm.as_extsh(output, output);
+ }
+ break;
+ }
+
+ masm.ma_bc(output, valueTemp, &end, Assembler::NotEqual, ShortJump);
+
+ masm.as_slw(valueTemp, newval, offsetTemp);
+ masm.as_and(ScratchRegister, ScratchRegister, maskTemp);
+ masm.as_or(ScratchRegister, ScratchRegister, valueTemp);
+
+ /* if (access) masm.append(*access, masm.size()); */
+ masm.as_stwcx(ScratchRegister, r0, SecondScratchReg);
+ masm.ma_bc(Assembler::NotEqual, &again, ShortJump);
+
+ masm.memoryBarrierAfter(sync);
+ masm.bind(&end);
+}
+
+
+void
+MacroAssembler::compareExchange(Scalar::Type type, Synchronization sync, const Address& mem,
+ Register oldval, Register newval, Register valueTemp,
+ Register offsetTemp, Register maskTemp, Register output)
+{
+ CompareExchange(*this, nullptr, type, sync, mem, oldval, newval, valueTemp, offsetTemp, maskTemp,
+ output);
+}
+
+void
+MacroAssembler::compareExchange(Scalar::Type type, Synchronization sync, const BaseIndex& mem,
+ Register oldval, Register newval, Register valueTemp,
+ Register offsetTemp, Register maskTemp, Register output)
+{
+ CompareExchange(*this, nullptr, type, sync, mem, oldval, newval, valueTemp, offsetTemp, maskTemp,
+ output);
+}
+void
+MacroAssembler::wasmCompareExchange(const wasm::MemoryAccessDesc& access, const Address& mem,
+ Register oldval, Register newval, Register valueTemp,
+ Register offsetTemp, Register maskTemp, Register output)
+{
+ CompareExchange(*this, &access, access.type(), access.sync(), mem, oldval, newval, valueTemp, offsetTemp, maskTemp,
+ output);
+}
+
+void
+MacroAssembler::wasmCompareExchange(const wasm::MemoryAccessDesc& access, const BaseIndex& mem,
+ Register oldval, Register newval, Register valueTemp,
+ Register offsetTemp, Register maskTemp, Register output)
+{
+ CompareExchange(*this, &access, access.type(), access.sync(), mem, oldval, newval, valueTemp, offsetTemp, maskTemp,
+ output);
+}
+
+
+template<typename T>
+static void
+AtomicExchange(MacroAssembler& masm, const wasm::MemoryAccessDesc* access,
+Scalar::Type type, Synchronization sync, const T& mem,
+ Register value, Register valueTemp, Register offsetTemp, Register maskTemp,
+ Register output)
+{
+ bool signExtend = Scalar::isSignedIntType(type);
+ unsigned nbytes = Scalar::byteSize(type);
+
+ switch (nbytes) {
+ case 1:
+ case 2:
+ break;
+ case 4:
+ MOZ_ASSERT(valueTemp == InvalidReg);
+ MOZ_ASSERT(offsetTemp == InvalidReg);
+ MOZ_ASSERT(maskTemp == InvalidReg);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ Label again;
+
+ masm.computeEffectiveAddress(mem, SecondScratchReg);
+
+ if (nbytes == 4) {
+
+ masm.memoryBarrierBefore(sync);
+ masm.bind(&again);
+
+ if (access) masm.append(*access, wasm::TrapMachineInsn::Atomic,
+ FaultingCodeOffset(masm.currentOffset()));
+ masm.as_lwarx(output, r0, SecondScratchReg);
+ /* if (access) masm.append(*access, masm.size()); */
+ masm.as_stwcx(value, r0, SecondScratchReg);
+ masm.ma_bc(Assembler::NotEqual, &again, ShortJump);
+
+ masm.memoryBarrierAfter(sync);
+
+ return;
+ }
+
+ masm.as_andi_rc(offsetTemp, SecondScratchReg, 3);
+ masm.subPtr(offsetTemp, SecondScratchReg);
+#if !MOZ_LITTLE_ENDIAN()
+ masm.as_xori(offsetTemp, offsetTemp, 3);
+#endif
+ masm.x_sldi(offsetTemp, offsetTemp, 3);
+ masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8)));
+ masm.as_sld(maskTemp, maskTemp, offsetTemp);
+ masm.as_nor(maskTemp, maskTemp, maskTemp);
+ switch (nbytes) {
+ case 1:
+ masm.as_andi_rc(valueTemp, value, 0xff);
+ break;
+ case 2:
+ masm.as_andi_rc(valueTemp, value, 0xffff);
+ break;
+ }
+ masm.as_sld(valueTemp, valueTemp, offsetTemp);
+ masm.memoryBarrierBefore(sync);
+ masm.bind(&again);
+
+ if (access) masm.append(*access, wasm::TrapMachineInsn::Atomic,
+ FaultingCodeOffset(masm.currentOffset()));
+ masm.as_lwarx(output, r0, SecondScratchReg);
+ masm.as_and(ScratchRegister, output, maskTemp);
+ masm.as_or(ScratchRegister, ScratchRegister, valueTemp);
+
+ /* if (access) masm.append(*access, masm.size()); */
+ masm.as_stwcx(ScratchRegister, r0, SecondScratchReg);
+ masm.ma_bc(Assembler::NotEqual, &again, ShortJump);
+
+ masm.as_srd(output, output, offsetTemp);
+
+ switch (nbytes) {
+ case 1:
+ masm.as_andi_rc(output, output, 0xff);
+ if (signExtend) {
+ masm.as_extsb(output, output);
+ }
+ break;
+ case 2:
+ masm.as_andi_rc(output, output, 0xffff);
+ if (signExtend) {
+ masm.as_extsh(output, output);
+ }
+ break;
+ }
+
+ masm.memoryBarrierAfter(sync);
+}
+
+void
+MacroAssembler::atomicExchange(Scalar::Type type, Synchronization sync, const Address& mem,
+ Register value, Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register output)
+{
+ AtomicExchange(*this, nullptr, type, sync, mem, value, valueTemp, offsetTemp, maskTemp, output);
+}
+
+void
+MacroAssembler::atomicExchange(Scalar::Type type, Synchronization sync, const BaseIndex& mem,
+ Register value, Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register output)
+{
+ AtomicExchange(*this, nullptr, type, sync, mem, value, valueTemp, offsetTemp, maskTemp, output);
+}
+
+void
+MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access, const Address& mem,
+ Register value, Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register output)
+{
+ AtomicExchange(*this, &access, access.type(), access.sync(), mem, value, valueTemp, offsetTemp, maskTemp, output);
+}
+
+void
+MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access, const BaseIndex& mem,
+ Register value, Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register output)
+{
+ AtomicExchange(*this, &access, access.type(), access.sync(), mem, value, valueTemp, offsetTemp, maskTemp, output);
+}
+
+template<typename T>
+static void
+AtomicFetchOp(MacroAssembler& masm, const wasm::MemoryAccessDesc* access,
+Scalar::Type type, Synchronization sync,
+ AtomicOp op, const T& mem, Register value, Register valueTemp,
+ Register offsetTemp, Register maskTemp, Register output)
+{
+ bool signExtend = Scalar::isSignedIntType(type);
+ unsigned nbytes = Scalar::byteSize(type);
+
+ switch (nbytes) {
+ case 1:
+ case 2:
+ break;
+ case 4:
+ MOZ_ASSERT(valueTemp == InvalidReg);
+ MOZ_ASSERT(offsetTemp == InvalidReg);
+ MOZ_ASSERT(maskTemp == InvalidReg);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ Label again;
+
+ masm.computeEffectiveAddress(mem, SecondScratchReg);
+
+ if (nbytes == 4) {
+
+ masm.memoryBarrierBefore(sync);
+ masm.bind(&again);
+
+ if (access) masm.append(*access, wasm::TrapMachineInsn::Atomic,
+ FaultingCodeOffset(masm.currentOffset()));
+ masm.as_lwarx(output, r0, SecondScratchReg);
+
+ switch (op) {
+ case AtomicOp::Add:
+ masm.as_add(ScratchRegister, output, value);
+ break;
+ case AtomicOp::Sub:
+ masm.as_subf(ScratchRegister, value, output);
+ break;
+ case AtomicOp::And:
+ masm.as_and(ScratchRegister, output, value);
+ break;
+ case AtomicOp::Or:
+ masm.as_or(ScratchRegister, output, value);
+ break;
+ case AtomicOp::Xor:
+ masm.as_xor(ScratchRegister, output, value);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ /* if (access) masm.append(*access, masm.size()); */
+ masm.as_stwcx(ScratchRegister, r0, SecondScratchReg);
+ masm.ma_bc(Assembler::NotEqual, &again, ShortJump);
+
+ masm.memoryBarrierAfter(sync);
+
+ return;
+ }
+
+
+ masm.as_andi_rc(offsetTemp, SecondScratchReg, 3);
+ masm.subPtr(offsetTemp, SecondScratchReg);
+#if !MOZ_LITTLE_ENDIAN()
+ masm.as_xori(offsetTemp, offsetTemp, 3);
+#endif
+ masm.x_sldi(offsetTemp, offsetTemp, 3);
+ masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8)));
+ masm.as_sld(maskTemp, maskTemp, offsetTemp);
+ masm.as_nor(maskTemp, maskTemp, maskTemp);
+
+ masm.memoryBarrierBefore(sync);
+
+ masm.bind(&again);
+
+ if (access) masm.append(*access, wasm::TrapMachineInsn::Atomic,
+ FaultingCodeOffset(masm.currentOffset()));
+ masm.as_lwarx(ScratchRegister, r0, SecondScratchReg);
+ masm.as_srd(output, ScratchRegister, offsetTemp);
+
+ switch (op) {
+ case AtomicOp::Add:
+ masm.as_add(valueTemp, output, value);
+ break;
+ case AtomicOp::Sub:
+ masm.as_subf(valueTemp, value, output);
+ break;
+ case AtomicOp::And:
+ masm.as_and(valueTemp, output, value);
+ break;
+ case AtomicOp::Or:
+ masm.as_or(valueTemp, output, value);
+ break;
+ case AtomicOp::Xor:
+ masm.as_xor(valueTemp, output, value);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ switch (nbytes) {
+ case 1:
+ masm.as_andi_rc(valueTemp, valueTemp, 0xff);
+ break;
+ case 2:
+ masm.as_andi_rc(valueTemp, valueTemp, 0xffff);
+ break;
+ }
+
+ masm.as_sld(valueTemp, valueTemp, offsetTemp);
+
+ masm.as_and(ScratchRegister, ScratchRegister, maskTemp);
+ masm.as_or(ScratchRegister, ScratchRegister, valueTemp);
+
+ /* if (access) masm.append(*access, masm.size()); */
+ masm.as_stwcx(ScratchRegister, r0, SecondScratchReg);
+ masm.ma_bc(Assembler::NotEqual, &again, ShortJump);
+
+ switch (nbytes) {
+ case 1:
+ masm.as_andi_rc(output, output, 0xff);
+ if (signExtend) {
+ masm.as_extsb(output, output);
+ }
+ break;
+ case 2:
+ masm.as_andi_rc(output, output, 0xffff);
+ if (signExtend) {
+ masm.as_extsh(output, output);
+ }
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ masm.memoryBarrierAfter(sync);
+}
+
+void
+MacroAssembler::atomicFetchOp(Scalar::Type type, Synchronization sync, AtomicOp op,
+ Register value, const Address& mem, Register valueTemp,
+ Register offsetTemp, Register maskTemp, Register output)
+{
+ AtomicFetchOp(*this, nullptr, type, sync, op, mem, value, valueTemp, offsetTemp, maskTemp, output);
+}
+
+void
+MacroAssembler::atomicFetchOp(Scalar::Type type, Synchronization sync, AtomicOp op,
+ Register value, const BaseIndex& mem, Register valueTemp,
+ Register offsetTemp, Register maskTemp, Register output)
+{
+ AtomicFetchOp(*this, nullptr, type, sync, op, mem, value, valueTemp, offsetTemp, maskTemp, output);
+}
+
+void
+MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
+ Register value, const Address& mem, Register valueTemp,
+ Register offsetTemp, Register maskTemp, Register output)
+{
+ AtomicFetchOp(*this, &access, access.type(), access.sync(), op, mem, value, valueTemp, offsetTemp, maskTemp, output);
+}
+
+void
+MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
+ Register value, const BaseIndex& mem, Register valueTemp,
+ Register offsetTemp, Register maskTemp, Register output)
+{
+ AtomicFetchOp(*this, &access, access.type(), access.sync(), op, mem, value, valueTemp, offsetTemp, maskTemp, output);
+}
+
+template<typename T>
+static void
+AtomicEffectOp(MacroAssembler& masm, const wasm::MemoryAccessDesc* access, Scalar::Type type, Synchronization sync, AtomicOp op,
+ const T& mem, Register value, Register valueTemp, Register offsetTemp, Register maskTemp)
+{
+ unsigned nbytes = Scalar::byteSize(type);
+
+ switch (nbytes) {
+ case 1:
+ case 2:
+ break;
+ case 4:
+ MOZ_ASSERT(valueTemp == InvalidReg);
+ MOZ_ASSERT(offsetTemp == InvalidReg);
+ MOZ_ASSERT(maskTemp == InvalidReg);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ Label again;
+
+ masm.computeEffectiveAddress(mem, SecondScratchReg);
+
+ if (nbytes == 4) {
+
+ masm.memoryBarrierBefore(sync);
+ masm.bind(&again);
+
+ if (access) masm.append(*access, wasm::TrapMachineInsn::Atomic,
+ FaultingCodeOffset(masm.currentOffset()));
+ masm.as_lwarx(ScratchRegister, r0, SecondScratchReg);
+
+ switch (op) {
+ case AtomicOp::Add:
+ masm.as_add(ScratchRegister, ScratchRegister, value);
+ break;
+ case AtomicOp::Sub:
+ masm.as_subf(ScratchRegister, value, ScratchRegister);
+ break;
+ case AtomicOp::And:
+ masm.as_and(ScratchRegister, ScratchRegister, value);
+ break;
+ case AtomicOp::Or:
+ masm.as_or(ScratchRegister, ScratchRegister, value);
+ break;
+ case AtomicOp::Xor:
+ masm.as_xor(ScratchRegister, ScratchRegister, value);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ /* if (access) masm.append(*access, masm.size()); */
+ masm.as_stwcx(ScratchRegister, r0, SecondScratchReg);
+ masm.ma_bc(Assembler::NotEqual, &again, ShortJump);
+
+ masm.memoryBarrierAfter(sync);
+
+ return;
+ }
+
+ masm.as_andi_rc(offsetTemp, SecondScratchReg, 3);
+ masm.subPtr(offsetTemp, SecondScratchReg);
+#if !MOZ_LITTLE_ENDIAN()
+ masm.as_xori(offsetTemp, offsetTemp, 3);
+#endif
+ masm.x_sldi(offsetTemp, offsetTemp, 3);
+ masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8)));
+ masm.as_sld(maskTemp, maskTemp, offsetTemp);
+ masm.as_nor(maskTemp, maskTemp, maskTemp);
+
+ masm.memoryBarrierBefore(sync);
+
+ masm.bind(&again);
+
+ if (access) masm.append(*access, wasm::TrapMachineInsn::Atomic,
+ FaultingCodeOffset(masm.currentOffset()));
+ masm.as_lwarx(ScratchRegister, r0, SecondScratchReg);
+ masm.as_srd(valueTemp, ScratchRegister, offsetTemp);
+
+ switch (op) {
+ case AtomicOp::Add:
+ masm.as_add(valueTemp, valueTemp, value);
+ break;
+ case AtomicOp::Sub:
+ masm.as_subf(valueTemp, value, valueTemp);
+ break;
+ case AtomicOp::And:
+ masm.as_and(valueTemp, valueTemp, value);
+ break;
+ case AtomicOp::Or:
+ masm.as_or(valueTemp, valueTemp, value);
+ break;
+ case AtomicOp::Xor:
+ masm.as_xor(valueTemp, valueTemp, value);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ switch (nbytes) {
+ case 1:
+ masm.as_andi_rc(valueTemp, valueTemp, 0xff);
+ break;
+ case 2:
+ masm.as_andi_rc(valueTemp, valueTemp, 0xffff);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ masm.as_sld(valueTemp, valueTemp, offsetTemp);
+
+ masm.as_and(ScratchRegister, ScratchRegister, maskTemp);
+ masm.as_or(ScratchRegister, ScratchRegister, valueTemp);
+
+ /* if (access) masm.append(*access, masm.size()); */
+ masm.as_stwcx(ScratchRegister, r0, SecondScratchReg);
+ masm.ma_bc(Assembler::NotEqual, &again, ShortJump);
+
+ masm.memoryBarrierAfter(sync);
+}
+
+
+void
+MacroAssembler::atomicEffectOpJS(Scalar::Type type, Synchronization sync, AtomicOp op,
+ Register value, const Address& mem, Register valueTemp,
+ Register offsetTemp, Register maskTemp)
+{
+ AtomicEffectOp(*this, nullptr, type, sync, op, mem, value, valueTemp, offsetTemp, maskTemp);
+}
+
+void
+MacroAssembler::atomicEffectOpJS(Scalar::Type type, Synchronization sync, AtomicOp op,
+ Register value, const BaseIndex& mem, Register valueTemp,
+ Register offsetTemp, Register maskTemp)
+{
+ AtomicEffectOp(*this, nullptr, type, sync, op, mem, value, valueTemp, offsetTemp, maskTemp);
+}
+
+// ========================================================================
+// JS atomic operations.
+
+template<typename T>
+static void
+CompareExchangeJS(MacroAssembler& masm, Scalar::Type arrayType, Synchronization sync,
+ const T& mem, Register oldval, Register newval, Register valueTemp,
+ Register offsetTemp, Register maskTemp, Register temp, AnyRegister output)
+{
+ if (arrayType == Scalar::Uint32) {
+ masm.compareExchange(arrayType, sync, mem, oldval, newval, valueTemp, offsetTemp, maskTemp,
+ temp);
+ masm.convertUInt32ToDouble(temp, output.fpu());
+ } else {
+ masm.compareExchange(arrayType, sync, mem, oldval, newval, valueTemp, offsetTemp, maskTemp,
+ output.gpr());
+ }
+}
+
+void
+MacroAssembler::compareExchangeJS(Scalar::Type arrayType, Synchronization sync,
+ const Address& mem, Register oldval, Register newval,
+ Register valueTemp, Register offsetTemp, Register maskTemp,
+ Register temp, AnyRegister output)
+{
+ CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, valueTemp, offsetTemp, maskTemp,
+ temp, output);
+}
+
+void
+MacroAssembler::compareExchangeJS(Scalar::Type arrayType, Synchronization sync,
+ const BaseIndex& mem, Register oldval, Register newval,
+ Register valueTemp, Register offsetTemp, Register maskTemp,
+ Register temp, AnyRegister output)
+{
+ CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval,valueTemp, offsetTemp, maskTemp,
+ temp, output);
+}
+
+template<typename T>
+static void
+AtomicExchangeJS(MacroAssembler& masm, Scalar::Type arrayType, Synchronization sync,
+ const T& mem, Register value, Register valueTemp,
+ Register offsetTemp, Register maskTemp, Register temp, AnyRegister output)
+{
+ if (arrayType == Scalar::Uint32) {
+ masm.atomicExchange(arrayType, sync, mem, value, valueTemp, offsetTemp, maskTemp, temp);
+ masm.convertUInt32ToDouble(temp, output.fpu());
+ } else {
+ masm.atomicExchange(arrayType, sync, mem, value, valueTemp, offsetTemp, maskTemp,
+ output.gpr());
+ }
+}
+
+void
+MacroAssembler::atomicExchangeJS(Scalar::Type arrayType, Synchronization sync,
+ const Address& mem, Register value, Register valueTemp,
+ Register offsetTemp, Register maskTemp, Register temp,
+ AnyRegister output)
+{
+ AtomicExchangeJS(*this, arrayType, sync, mem, value, valueTemp, offsetTemp, maskTemp, temp,
+ output);
+}
+
+void
+MacroAssembler::atomicExchangeJS(Scalar::Type arrayType, Synchronization sync,
+ const BaseIndex& mem, Register value, Register valueTemp,
+ Register offsetTemp, Register maskTemp, Register temp,
+ AnyRegister output)
+{
+ AtomicExchangeJS(*this, arrayType, sync, mem, value, valueTemp, offsetTemp, maskTemp, temp, output);
+}
+
+template<typename T>
+static void
+AtomicFetchOpJS(MacroAssembler& masm, Scalar::Type arrayType, Synchronization sync,
+ AtomicOp op, Register value, const T& mem, Register valueTemp,
+ Register offsetTemp, Register maskTemp, Register temp,
+ AnyRegister output)
+{
+ if (arrayType == Scalar::Uint32) {
+ masm.atomicFetchOp(arrayType, sync, op, value, mem, valueTemp, offsetTemp, maskTemp, temp);
+ masm.convertUInt32ToDouble(temp, output.fpu());
+ } else {
+ masm.atomicFetchOp(arrayType, sync, op, value, mem, valueTemp, offsetTemp, maskTemp,
+ output.gpr());
+ }
+}
+
+void
+MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType, Synchronization sync, AtomicOp op,
+ Register value, const Address& mem, Register valueTemp,
+ Register offsetTemp, Register maskTemp, Register temp,
+ AnyRegister output)
+{
+ AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, valueTemp, offsetTemp, maskTemp, temp,
+ output);
+}
+
+void
+MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType, Synchronization sync, AtomicOp op,
+ Register value, const BaseIndex& mem, Register valueTemp,
+ Register offsetTemp, Register maskTemp, Register temp,
+ AnyRegister output)
+{
+ AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, valueTemp, offsetTemp, maskTemp, temp,
+ output);
+}
+
+// ========================================================================
+// Spectre Mitigations.
+
+void
+MacroAssembler::speculationBarrier()
+{
+ // eieio appears to be the fastest way of defeating Spectre, since its
+ // memory ordering is sufficient to defeat gadgets and it's less heavy
+ // than even so-called lwsync. Doo doo doo doo doo, a Spectre gadget ...
+ // See a real world demonstration in Shen et al, Restricting Control
+ // Flow During Speculative Execution with Venkman, pp6-7.
+ as_eieio();
+ // Go, Gadget, Go!
+}
diff -r 485b15bb4a20 -r 9c245f4665be js/src/jit/ppc64/MacroAssembler-ppc64.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/js/src/jit/ppc64/MacroAssembler-ppc64.h Thu Aug 08 21:23:52 2024 -0700
@@ -0,0 +1,1178 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_ppc64_shared_MacroAssembler_ppc64_shared_h
+#define jit_ppc64_shared_MacroAssembler_ppc64_shared_h
+
+#include "jit/AtomicOp.h"
+#include "jit/JitFrames.h"
+#include "jit/MoveResolver.h"
+#include "jit/ppc64/Assembler-ppc64.h"
+#include "vm/BytecodeUtil.h"
+
+
+namespace js {
+namespace jit {
+
+enum LoadStoreSize
+{
+ SizeByte = 8,
+ SizeHalfWord = 16,
+ SizeWord = 32,
+ SizeDouble = 64
+};
+
+enum LoadStoreExtension
+{
+ ZeroExtend = 0,
+ SignExtend = 1
+};
+
+enum JumpKind
+{
+ LongJump = 0,
+ ShortJump = 1
+};
+
+enum LiFlags
+{
+ Li64 = 0,
+ Li48 = 1,
+};
+
+struct ImmShiftedTag : public ImmWord
+{
+ explicit ImmShiftedTag(JSValueShiftedTag shtag)
+ : ImmWord((uintptr_t)shtag)
+ { }
+
+ explicit ImmShiftedTag(JSValueType type)
+ : ImmWord(uintptr_t(JSValueShiftedTag(JSVAL_TYPE_TO_SHIFTED_TAG(type))))
+ { }
+};
+
+struct ImmTag : public Imm32
+{
+ ImmTag(JSValueTag mask)
+ : Imm32(int32_t(mask))
+ { }
+};
+
+static const int defaultShift = 3;
+static_assert(1 << defaultShift == sizeof(JS::Value), "The defaultShift is wrong");
+
+// See documentation for ScratchTagScope and ScratchTagScopeRelease in
+// MacroAssembler-x64.h.
+class ScratchTagScope : public SecondScratchRegisterScope
+{
+ public:
+ ScratchTagScope(MacroAssembler& masm, const ValueOperand&)
+ : SecondScratchRegisterScope(masm)
+ {}
+};
+
+class ScratchTagScopeRelease
+{
+ ScratchTagScope* ts_;
+ public:
+ explicit ScratchTagScopeRelease(ScratchTagScope* ts) : ts_(ts) {
+ ts_->release();
+ }
+ ~ScratchTagScopeRelease() {
+ ts_->reacquire();
+ }
+};
+
+static Register CallReg = r12; // XXX
+
+class MacroAssemblerPPC64 : public Assembler
+{
+ protected:
+ // Perform a downcast.
+ MacroAssembler& asMasm();
+ const MacroAssembler& asMasm() const;
+
+/*
+ Condition ma_cmp(Register rd, Register lhs, Register rhs, Condition c);
+ Condition ma_cmp(Register rd, Register lhs, Imm32 imm, Condition c);
+*/
+ void compareFloatingPoint(FloatRegister lhs, FloatRegister rhs,
+ DoubleCondition c);
+
+/* XXX: TO DO: ma_d* variants are probably superfluous */
+ public:
+ void ma_move(Register rd, Register rs);
+
+ void ma_li(Register dest, ImmGCPtr ptr);
+ void ma_li(Register dest, Imm32 imm);
+ void ma_li(Register dest, Imm64 imm);
+ void ma_liPatchable(Register dest, Imm32 imm);
+
+ void ma_li(Register dest, CodeLabel* label);
+ void ma_li(Register dest, ImmWord imm);
+ void ma_li(Register dest, int64_t imm);
+ void ma_liPatchable(Register dest, ImmPtr imm);
+ void ma_liPatchable(Register dest, ImmWord imm);
+
+ // Shift operations
+ void ma_sll(Register rd, Register rt, Imm32 shift);
+ void ma_srl(Register rd, Register rt, Imm32 shift);
+ void ma_sra(Register rd, Register rt, Imm32 shift);
+
+ void ma_sll(Register rd, Register rt, Register shift);
+
+ void ma_dsll(Register rd, Register rt, Imm32 shift);
+ void ma_dsrl(Register rd, Register rt, Imm32 shift);
+ void ma_dsra(Register rd, Register rt, Imm32 shift);
+ void ma_dror(Register rd, Register rt, Imm32 shift);
+ void ma_drol(Register rd, Register rt, Imm32 shift);
+
+ void ma_dsll(Register rd, Register rt, Register shift);
+ void ma_dsrl(Register rd, Register rt, Register shift);
+ void ma_dsra(Register rd, Register rt, Register shift);
+
+ // Negate
+ void ma_negu(Register rd, Register rs);
+ void ma_dnegu(Register rd, Register rs);
+
+ void ma_not(Register rd, Register rs);
+
+ // Bit extract/insert
+ void ma_ext(Register rt, Register rs, uint16_t pos, uint16_t size);
+ void ma_ins(Register rt, Register rs, uint16_t pos, uint16_t size);
+
+ // and
+ void ma_and(Register rd, Register rs);
+ void ma_and(Register rd, Imm32 imm);
+ void ma_and(Register rd, Register rs, Imm32 imm);
+
+ // or
+ void ma_or(Register rd, Register rs);
+ void ma_or(Register rd, Imm32 imm);
+ void ma_or(Register rd, Register rs, Imm32 imm);
+
+ // xor
+ void ma_xor(Register rd, Register rs);
+ void ma_xor(Register rd, Imm32 imm);
+ void ma_xor(Register rd, Register rs, Imm32 imm);
+
+ void ma_ctz(Register rd, Register rs);
+
+ // load
+ // Power ISA handles unaligned scalar LSU operations in hardware.
+ FaultingCodeOffset ma_load(Register dest, const BaseIndex& src, LoadStoreSize size = SizeWord,
+ LoadStoreExtension extension = SignExtend);
+
+ // store
+ FaultingCodeOffset ma_store(Register data, const BaseIndex& dest, LoadStoreSize size = SizeWord);
+ FaultingCodeOffset ma_store(Imm32 imm, const BaseIndex& dest, LoadStoreSize size = SizeWord);
+
+ // arithmetic based ops
+ // add
+ void ma_add(Register rd, Register rs, Imm32 imm);
+ void ma_add(Register rd, Register rs);
+ void ma_add(Register rd, Imm32 imm);
+ void ma_addTestCarry(Condition cond, Register rd, Register rs, Register rt, Label* overflow, bool is32 = true);
+ void ma_addTestCarry(Condition cond, Register rd, Register rs, ImmWord imm, Label* overflow, bool is32 = false);
+ void ma_addTestCarry(Condition cond, Register rd, Register rs, Imm32 imm, Label* overflow, bool is32 = true);
+
+ // subtract
+ void ma_subu(Register rd, Register rs, Imm32 imm);
+ void ma_subu(Register rd, Register rs);
+ void ma_subu(Register rd, Imm32 imm);
+ void ma_subTestOverflow(Register rd, Register rs, Imm32 imm, Label* overflow, bool is32 = true);
+
+ // multiplies. For now, there are only few that we care about.
+ void ma_mul(Register rd, Register rs, Imm32 imm);
+ void ma_mul_branch_overflow(Register rd, Register rs, Register rt, Label* overflow);
+ void ma_mul_branch_overflow(Register rd, Register rs, Imm32 imm, Label* overflow);
+
+ /*
+ // divisions
+ void ma_div_branch_overflow(Register rd, Register rs, Register rt, Label* overflow);
+ void ma_div_branch_overflow(Register rd, Register rs, Imm32 imm, Label* overflow);
+
+ // fast mod, uses scratch registers, and thus needs to be in the assembler
+ // implicitly assumes that we can overwrite dest at the beginning of the sequence
+ void ma_mod_mask(Register src, Register dest, Register hold, Register remain,
+ int32_t shift, Label* negZero = nullptr);
+ */
+
+ // branches when done from within platform-specific code
+ void ma_bc(Condition c, Label* l, JumpKind jumpKind = LongJump);
+ template <typename T>
+ void ma_bc(CRegisterID cr, T c, Label* l, JumpKind jumpKind = LongJump);
+
+ void ma_bc(Register lhs, Register rhs, Label* l, Condition c, JumpKind jumpKind = LongJump);
+ void ma_bc(Register lhs, Imm32 imm, Label* l, Condition c, JumpKind jumpKind = LongJump);
+ void ma_bc(Register lhs, ImmPtr imm, Label* l, Condition c, JumpKind jumpKind = LongJump);
+ void ma_bc(Register lhs, ImmGCPtr imm, Label* l, Condition c, JumpKind jumpKind = LongJump) {
+ MOZ_ASSERT(lhs != ScratchRegister);
+ ma_li(ScratchRegister, imm);
+ ma_bc(lhs, ScratchRegister, l, c, jumpKind);
+ }
+ // Explicit compare width
+ void ma_bc32(Register lhs, Register rhs, Label* l, Condition c, JumpKind jumpKind = LongJump);
+ void ma_bc64(Register lhs, Imm32 imm, Label* l, Condition c, JumpKind jumpKind = LongJump);
+
+ void ma_b(Label* l, JumpKind jumpKind = LongJump);
+
+ // FP instructions
+ void ma_lis(FloatRegister dest, float value);
+
+ FaultingCodeOffset ma_sd(FloatRegister ft, Address address);
+ FaultingCodeOffset ma_ss(FloatRegister ft, Address address);
+ FaultingCodeOffset ma_sd(FloatRegister src, BaseIndex address);
+ FaultingCodeOffset ma_ss(FloatRegister src, BaseIndex address);
+
+ FaultingCodeOffset ma_ld(FloatRegister ft, Address address);
+ FaultingCodeOffset ma_ls(FloatRegister ft, Address address);
+ FaultingCodeOffset ma_ld(FloatRegister dest, const BaseIndex& src);
+ FaultingCodeOffset ma_ls(FloatRegister dest, const BaseIndex& src);
+
+ // FP branches
+ void ma_bc(DoubleCondition c, FloatRegister lhs, FloatRegister rhs, Label* label,
+ JumpKind jumpKind = LongJump);
+ void ma_bc(DoubleCondition c, Label* label, JumpKind jumpKind = LongJump);
+ //void ma_bc(FloatRegister lhs, FloatRegister rhs, Label* label,
+ // ConditionRegister cr,
+ // DoubleCondition c, JumpKind jumpKind = LongJump);
+
+ void ma_call(ImmPtr dest);
+
+ void ma_spectre_isel(Condition cond, Register lhs, Register rhs);
+
+ void ma_jump(ImmPtr dest);
+
+ void ma_cmp64(Register lhs, Imm64 rhs, Condition c);
+ void ma_cmp32(Register lhs, Imm32 rhs, Condition c);
+ void ma_cmp32(Register lhs, Register rhs, Condition c);
+ void ma_cmp32(Register lhs, const Address& rhs, Condition c);
+ void ma_cmp_set(Register dest, Address lhs, Register rhs, Condition c, bool useCmpw = false);
+ void ma_cmp_set(Register dest, Address lhs, Imm32 rhs, Condition c, bool useCmpw = false);
+ void ma_cmp_set(Register dest, Address lhs, Imm64 rhs, Condition c, bool useCmpw = true);
+ void ma_cmp_set(Register dst, Register lhs, Register rhs, Condition c, bool useCmpw = false);
+ void ma_cmp_set(Register dst, Register lhs, Imm16 imm, Condition c, bool useCmpw = false);
+ void ma_cmp_set(Register dst, Register lhs, Imm32 imm, Condition c, bool useCmpw = true) {
+ MOZ_ASSERT(useCmpw);
+ ma_cmp32(lhs, imm, c);
+ ma_cmp_set_coda(dst, c);
+ }
+ void ma_cmp_set(Register dst, Register lhs, Imm64 imm, Condition c, bool useCmpw = false) {
+ MOZ_ASSERT(!useCmpw);
+ ma_cmp64(lhs, imm, c);
+ ma_cmp_set_coda(dst, c);
+ }
+ void ma_cmp_set_coda(Register rd, Condition c);
+ void ma_cmp_set_double(Register dst, FloatRegister lhs, FloatRegister rhs, DoubleCondition c);
+
+ // Evaluate srcDest = minmax<isMax>{Float32,Double}(srcDest, other).
+ // Handle NaN specially if handleNaN is true.
+ void minMaxDouble(FloatRegister srcDest, FloatRegister other, bool handleNaN, bool isMax);
+ void minMaxFloat32(FloatRegister srcDest, FloatRegister other, bool handleNaN, bool isMax);
+
+ FaultingCodeOffset loadDouble(const Address& addr, FloatRegister dest);
+ FaultingCodeOffset loadDouble(const BaseIndex& src, FloatRegister dest);
+
+ // Load a float value into a register, then expand it to a double.
+ void loadFloatAsDouble(const Address& addr, FloatRegister dest);
+ void loadFloatAsDouble(const BaseIndex& src, FloatRegister dest);
+
+ FaultingCodeOffset loadFloat32(const Address& addr, FloatRegister dest);
+ FaultingCodeOffset loadFloat32(const BaseIndex& src, FloatRegister dest);
+
+ void outOfLineWasmTruncateToInt32Check(FloatRegister input, Register output, MIRType fromType,
+ TruncFlags flags, Label* rejoin,
+ wasm::BytecodeOffset trapOffset);
+ void outOfLineWasmTruncateToInt64Check(FloatRegister input, Register64 output, MIRType fromType,
+ TruncFlags flags, Label* rejoin,
+ wasm::BytecodeOffset trapOffset);
+
+ protected:
+ void wasmLoadImpl(const wasm::MemoryAccessDesc& access, Register memoryBase, Register ptr,
+ Register ptrScratch, AnyRegister output, Register tmp);
+ void wasmStoreImpl(const wasm::MemoryAccessDesc& access, AnyRegister value, Register memoryBase,
+ Register ptr, Register ptrScratch, Register tmp);
+
+ public:
+
+ // Negate
+
+ void ma_dins(Register rt, Register rs, Imm32 pos, Imm32 size);
+ void ma_dext(Register rt, Register rs, Imm32 pos, Imm32 size);
+
+ void ma_dctz(Register rd, Register rs);
+
+ // load
+ FaultingCodeOffset ma_load(Register dest, Address address, LoadStoreSize size = SizeWord,
+ LoadStoreExtension extension = SignExtend);
+
+ // store
+ FaultingCodeOffset ma_store(Register data, Address address, LoadStoreSize size = SizeWord);
+
+ // arithmetic based ops
+ // add
+ void ma_addTestOverflow(Register rd, Register rs, Register rt, Label* overflow, bool is32 = true);
+ void ma_addTestOverflow(Register rd, Register rs, ImmWord imm, Label* overflow, bool is32 = false);
+ void ma_addTestOverflow(Register rd, Register rs, Imm32 imm, Label* overflow, bool is32 = true);
+
+ // neg
+ void ma_negTestOverflow(Register rd, Label* overflow);
+
+ // subtract
+ void ma_dsubu(Register rd, Register rs, Imm32 imm);
+ void ma_dsubu(Register rd, Register rs);
+ void ma_dsubu(Register rd, Imm32 imm);
+ void ma_subTestOverflow(Register rd, Register rs, Register rt, Label* overflow, bool is32 = true);
+
+ // multiplies. For now, there are only few that we care about.
+ void ma_dmult(Register rs, Imm32 imm);
+
+ // stack
+ void ma_pop(Register r);
+ void ma_push(Register r);
+
+ // branches when done from within PPC64-specific code
+ void ma_bc(Register lhs, ImmWord imm, Label* l, Condition c, JumpKind jumpKind = LongJump);
+ void ma_bc(Register lhs, Address addr, Label* l, Condition c, JumpKind jumpKind = LongJump);
+ void ma_bc(Address addr, Imm32 imm, Label* l, Condition c, JumpKind jumpKind = LongJump);
+ void ma_bc(Address addr, ImmGCPtr imm, Label* l, Condition c, JumpKind jumpKind = LongJump);
+ void ma_bc(Address addr, Register rhs, Label* l, Condition c, JumpKind jumpKind = LongJump) {
+ MOZ_ASSERT(rhs != ScratchRegister);
+ ma_load(ScratchRegister, addr, SizeDouble);
+ ma_bc(ScratchRegister, rhs, l, c, jumpKind);
+ }
+
+ void ma_bal(Label* l);
+
+ // fp instructions
+ void ma_lid(FloatRegister dest, double value);
+
+ void ma_mv(FloatRegister src, ValueOperand dest);
+ void ma_mv(ValueOperand src, FloatRegister dest);
+
+ void ma_pop(FloatRegister f);
+ void ma_push(FloatRegister f);
+
+ void ma_cmp_set(Register dst, Register lhs, ImmWord imm, Condition c, bool useCmpw = false);
+ void ma_cmp_set(Register dst, Register lhs, ImmPtr imm, Condition c, bool useCmpw = false);
+};
+
+class MacroAssembler;
+
+class MacroAssemblerPPC64Compat : public MacroAssemblerPPC64
+{
+ public:
+ using MacroAssemblerPPC64::call;
+
+ MacroAssemblerPPC64Compat()
+ { }
+
+ void convertBoolToInt32(Register source, Register dest);
+ void convertInt32ToDouble(Register src, FloatRegister dest);
+ void convertInt32ToDouble(const Address& src, FloatRegister dest);
+ void convertInt32ToDouble(const BaseIndex& src, FloatRegister dest);
+ void convertUInt32ToDouble(Register src, FloatRegister dest);
+ void convertUInt32ToFloat32(Register src, FloatRegister dest);
+ void convertDoubleToFloat32(FloatRegister src, FloatRegister dest);
+ void convertDoubleToInt32(FloatRegister src, Register dest, Label* fail,
+ bool negativeZeroCheck = true);
+ void convertDoubleToPtr(FloatRegister src, Register dest, Label* fail,
+ bool negativeZeroCheck = true);
+ void convertFloat32ToInt32(FloatRegister src, Register dest, Label* fail,
+ bool negativeZeroCheck = true);
+
+ void convertFloat32ToDouble(FloatRegister src, FloatRegister dest);
+ void convertInt32ToFloat32(Register src, FloatRegister dest);
+ void convertInt32ToFloat32(const Address& src, FloatRegister dest);
+
+ void movq(Register rs, Register rd);
+
+ void computeScaledAddress(const BaseIndex& address, Register dest);
+
+ inline void computeEffectiveAddress(const Address& address, Register dest) {
+ ma_add(dest, address.base, Imm32(address.offset));
+ }
+
+ inline void computeEffectiveAddress(const BaseIndex &address, Register dest) {
+ computeScaledAddress(address, dest);
+ if (address.offset) {
+ ma_add(dest, dest, Imm32(address.offset));
+ }
+ }
+
+
+ void j(Condition cond, Label* dest) {
+ ma_b(dest);
+ }
+
+ void mov(Register src, Register dest) {
+ if (dest != src)
+ as_or(dest, src, src);
+ }
+ void mov(ImmWord imm, Register dest) {
+ ma_li(dest, imm);
+ }
+ void mov(ImmPtr imm, Register dest) {
+ mov(ImmWord(uintptr_t(imm.value)), dest);
+ }
+ void mov(CodeLabel* label, Register dest) {
+ ma_li(dest, label);
+ }
+ void mov(Register src, Address dest) {
+ MOZ_CRASH("NYI-IC");
+ }
+ void mov(Address src, Register dest) {
+ MOZ_CRASH("NYI-IC");
+ }
+
+#if(0)
+ // load: offset to the load instruction obtained by movePatchablePtr().
+ void writeDataRelocation(ImmGCPtr ptr, BufferOffset load) {
+ // Raw GC pointer relocations and Value relocations both end up in
+ // Assembler::TraceDataRelocations.
+ if (ptr.value) {
+ if (gc::IsInsideNursery(ptr.value)) {
+ embedsNurseryPointers_ = true;
+ }
+ dataRelocations_.writeUnsigned(load.getOffset());
+ }
+ }
+ void writeDataRelocation(const Value& val, BufferOffset load) {
+ // Raw GC pointer relocations and Value relocations both end up in
+ // Assembler::TraceDataRelocations.
+ if (val.isGCThing()) {
+ gc::Cell* cell = val.toGCThing();
+ if (cell && gc::IsInsideNursery(cell)) {
+ embedsNurseryPointers_ = true;
+ }
+ dataRelocations_.writeUnsigned(load.getOffset());
+ }
+ }
+#else
+ void writeDataRelocation(const Value& val) {
+ if (val.isGCThing()) {
+ gc::Cell* cell = val.toGCThing();
+ if (cell && gc::IsInsideNursery(cell))
+ embedsNurseryPointers_ = true;
+ dataRelocations_.writeUnsigned(currentOffset());
+ }
+ }
+#endif
+
+ void hop_skip_nop_jump() {
+ // Common stanza at the end of these CTR branches.
+ as_nop();
+ as_nop();
+ as_nop(); // branch slot
+ as_bctr();
+ }
+
+ void jump(Label* label) {
+ ma_b(label);
+ }
+ void jump(ImmPtr ptr) {
+ m_buffer.ensureSpace(7 * sizeof(uint32_t));
+ BufferOffset bo = m_buffer.nextOffset();
+ addPendingJump(bo, ptr, RelocationKind::HARDCODED);
+ ma_jump(ptr);
+ }
+ void jump(Register reg) {
+ // This could be to any code, so definitely use r12.
+ if (reg != SecondScratchReg)
+ as_or(SecondScratchReg, reg, reg); // make r12 == CTR
+ xs_mtctr(SecondScratchReg); // new dispatch group
+ hop_skip_nop_jump();
+ }
+ void jump(const Address& address) {
+ loadPtr(address, SecondScratchReg);
+ xs_mtctr(SecondScratchReg); // new dispatch group
+ as_nop();
+ hop_skip_nop_jump();
+ }
+ void jump(TrampolinePtr code) {
+ jump(ImmPtr(code.value));
+ }
+ void branch(JitCode* c) {
+ // This is to Ion code, but we still use r12 anyway.
+ BufferOffset bo = m_buffer.nextOffset();
+ addPendingJump(bo, ImmPtr(c->raw()), RelocationKind::JITCODE);
+ ma_liPatchable(SecondScratchReg, ImmPtr(c->raw()));
+ xs_mtctr(SecondScratchReg); // new dispatch group
+ // Keep the branch out of the same dispatch group.
+ as_nop();
+ hop_skip_nop_jump();
+ }
+ void jump(JitCode* code) {
+ branch(code);
+ }
+ void branch(const Register reg) {
+ jump(reg);
+ }
+
+ void nop() {
+ as_nop();
+ }
+ void ret() {
+ // Pop LR and return.
+ ma_pop(ScratchRegister);
+ xs_mtlr(ScratchRegister);
+ as_blr();
+ }
+ inline void retn(Imm32 n);
+ void push(Imm32 imm) {
+ ma_li(ScratchRegister, imm);
+ ma_push(ScratchRegister);
+ }
+ void push(ImmWord imm) {
+ ma_li(ScratchRegister, imm);
+ ma_push(ScratchRegister);
+ }
+ void push(ImmGCPtr imm) {
+ ma_li(ScratchRegister, imm);
+ ma_push(ScratchRegister);
+ }
+ void push(const Address& address) {
+ loadPtr(address, ScratchRegister);
+ ma_push(ScratchRegister);
+ }
+ void push(Register reg) {
+ ma_push(reg);
+ }
+ void push(FloatRegister reg) {
+ ma_push(reg);
+ }
+ void pop(Register reg) {
+ ma_pop(reg);
+ }
+ void pop(FloatRegister reg) {
+ ma_pop(reg);
+ }
+
+ CodeOffset toggledJump(Label* label);
+ CodeOffset toggledCall(JitCode* target, bool enabled);
+ static size_t ToggledCallSize(uint8_t* code) {
+ // A full branch stanza plus the oris/b gate.
+ return 8 * sizeof(uint32_t);
+ }
+
+ CodeOffset pushWithPatch(ImmWord imm) {
+ CodeOffset offset = movWithPatch(imm, ScratchRegister);
+ ma_push(ScratchRegister);
+ return offset;
+ }
+
+ CodeOffset movWithPatch(ImmWord imm, Register dest) {
+ CodeOffset offset = CodeOffset(currentOffset());
+ ma_liPatchable(dest, imm);
+ return offset;
+ }
+ CodeOffset movWithPatch(ImmPtr imm, Register dest) {
+ CodeOffset offset = CodeOffset(currentOffset());
+ ma_liPatchable(dest, imm);
+ return offset;
+ }
+
+ void writeCodePointer(CodeLabel* label) {
+ label->patchAt()->bind(currentOffset());
+ label->setLinkMode(CodeLabel::RawPointer);
+ m_buffer.ensureSpace(sizeof(void*));
+ writeInst(-1);
+ writeInst(-1);
+ }
+
+ void splitTag(Register src, Register dest) {
+ x_srdi(dest, src, JSVAL_TAG_SHIFT);
+ }
+
+ void splitTag(const ValueOperand& operand, Register dest) {
+ splitTag(operand.valueReg(), dest);
+ }
+
+ void splitTagForTest(const ValueOperand& value, ScratchTagScope& tag) {
+ splitTag(value, tag);
+ }
+
+ // unboxing code
+ void unboxNonDouble(const ValueOperand& operand, Register dest, JSValueType type) {
+ unboxNonDouble(operand.valueReg(), dest, type);
+ }
+
+ template <typename T>
+ void unboxNonDouble(T src, Register dest, JSValueType type) {
+ MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE);
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ load32(src, dest);
+ return;
+ }
+ loadPtr(src, dest);
+ unboxNonDouble(dest, dest, type);
+ }
+
+ void unboxNonDouble(Register src, Register dest, JSValueType type) {
+ MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE);
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ // This has the effect of clearing bits 32-63 and the tag
+ // with it, plus extending the sign, all in one operation.
+ as_srawi(dest, src, 0);
+ return;
+ }
+ // Blank out the tag.
+ as_rldicl(dest, src, 0, 64-JSVAL_TAG_SHIFT); // "clrldi"
+ }
+
+ template <typename T>
+ void unboxObjectOrNull(const T& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ // Assuming this assert is true, we've already cleared that bit
+ // by clearing the tag. We could also do what ARM does.
+ static_assert(JS::detail::ValueObjectOrNullBit ==
+ (uint64_t(0x8) << JSVAL_TAG_SHIFT));
+ }
+
+ void unboxGCThingForGCBarrier(const Address& src, Register dest) {
+ loadPtr(src, dest);
+ as_rldicl(dest, dest, 0, 64-JSVAL_TAG_SHIFT); // "clrldi"
+ }
+ void unboxGCThingForGCBarrier(const ValueOperand& src, Register dest) {
+ as_rldicl(dest, src.valueReg(), 0, 64-JSVAL_TAG_SHIFT); // "clrldi"
+ }
+
+ void unboxWasmAnyRefGCThingForGCBarrier(const Address& src, Register dest) {
+ MOZ_ASSERT(ScratchRegister != dest);
+ movePtr(ImmWord(wasm::AnyRef::GCThingMask), ScratchRegister);
+ loadPtr(src, dest);
+ as_and(dest, dest, ScratchRegister);
+ }
+
+ void getWasmAnyRefGCThingChunk(Register src, Register dest) {
+ MOZ_ASSERT(src != dest);
+ movePtr(ImmWord(wasm::AnyRef::GCThingChunkMask), dest);
+ as_and(dest, dest, src);
+ }
+
+ // Like unboxGCThingForGCBarrier, but loads the GC thing's chunk base.
+ void getGCThingValueChunk(const Address& src, Register dest) {
+// ScratchRegisterScope scratch(asMasm());
+// MOZ_ASSERT(scratch != dest);
+ MOZ_ASSERT(dest != ScratchRegister);
+ loadPtr(src, dest);
+ movePtr(ImmWord(JS::detail::ValueGCThingPayloadChunkMask), ScratchRegister);
+ as_and(dest, dest, ScratchRegister);
+ }
+ void getGCThingValueChunk(const ValueOperand& src, Register dest) {
+ MOZ_ASSERT(src.valueReg() != dest);
+ movePtr(ImmWord(JS::detail::ValueGCThingPayloadChunkMask), dest);
+ as_and(dest, dest, src.valueReg());
+ }
+
+ void unboxInt32(const ValueOperand& operand, Register dest);
+ void unboxInt32(Register src, Register dest);
+ void unboxInt32(const Address& src, Register dest);
+ void unboxInt32(const BaseIndex& src, Register dest);
+ void unboxBoolean(const ValueOperand& operand, Register dest);
+ void unboxBoolean(Register src, Register dest);
+ void unboxBoolean(const Address& src, Register dest);
+ void unboxBoolean(const BaseIndex& src, Register dest);
+ void unboxDouble(const ValueOperand& operand, FloatRegister dest);
+ void unboxDouble(Register src, Register dest);
+ void unboxDouble(const Address& src, FloatRegister dest);
+ void unboxDouble(const BaseIndex& src, FloatRegister dest);
+ void unboxString(const ValueOperand& operand, Register dest);
+ void unboxString(Register src, Register dest);
+ void unboxString(const Address& src, Register dest);
+ void unboxSymbol(const ValueOperand& src, Register dest);
+ void unboxSymbol(Register src, Register dest);
+ void unboxSymbol(const Address& src, Register dest);
+ void unboxBigInt(const ValueOperand& operand, Register dest);
+ void unboxBigInt(Register src, Register dest);
+ void unboxBigInt(const Address& src, Register dest);
+ void unboxObject(const ValueOperand& src, Register dest);
+ void unboxObject(Register src, Register dest);
+ void unboxObject(const Address& src, Register dest);
+ void unboxObject(const BaseIndex& src, Register dest) { unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT); }
+ void unboxValue(const ValueOperand& src, AnyRegister dest, JSValueType type);
+ void unboxPrivate(const ValueOperand& src, Register dest);
+
+ void notBoolean(const ValueOperand& val) {
+ as_xori(val.valueReg(), val.valueReg(), 1);
+ }
+
+ // boxing code
+ void boxDouble(FloatRegister src, const ValueOperand& dest, FloatRegister);
+ void boxNonDouble(JSValueType type, Register src, const ValueOperand& dest);
+
+ // Extended unboxing API. If the payload is already in a register, returns
+ // that register. Otherwise, provides a move to the given scratch register,
+ // and returns that.
+ Register extractObject(const Address& address, Register scratch);
+ [[nodiscard]] Register extractObject(const ValueOperand& value, Register scratch) {
+ unboxObject(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractString(const ValueOperand& value, Register scratch) {
+ unboxString(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractSymbol(const ValueOperand& value, Register scratch) {
+ unboxSymbol(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractInt32(const ValueOperand& value, Register scratch) {
+ unboxInt32(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractBoolean(const ValueOperand& value, Register scratch) {
+ unboxBoolean(value, scratch);
+ return scratch;
+ }
+ Register extractTag(const Address& address, Register scratch);
+ Register extractTag(const BaseIndex& address, Register scratch);
+ Register extractTag(const ValueOperand& value, Register scratch) {
+ MOZ_ASSERT(scratch != ScratchRegister);
+ splitTag(value, scratch);
+ return scratch;
+ }
+
+ void boolValueToDouble(const ValueOperand& operand, FloatRegister dest);
+ void int32ValueToDouble(const ValueOperand& operand, FloatRegister dest);
+ void loadInt32OrDouble(const Address& src, FloatRegister dest);
+ void loadInt32OrDouble(const BaseIndex& addr, FloatRegister dest);
+ void loadConstantDouble(double dp, FloatRegister dest);
+
+ void boolValueToFloat32(const ValueOperand& operand, FloatRegister dest);
+ void int32ValueToFloat32(const ValueOperand& operand, FloatRegister dest);
+ void loadConstantFloat32(float f, FloatRegister dest);
+
+ void testNullSet(Condition cond, const ValueOperand& value, Register dest);
+ void testObjectSet(Condition cond, const ValueOperand& value, Register dest);
+ void testBigIntSet(Condition cond, const ValueOperand& value, Register dest);
+ void testNumberSet(Condition cond, const ValueOperand& value, Register dest);
+ void testStringSet(Condition cond, const ValueOperand& value, Register dest);
+ void testSymbolSet(Condition cond, const ValueOperand& value, Register dest);
+ void testBooleanSet(Condition cond, const ValueOperand& value, Register dest);
+ void testUndefinedSet(Condition cond, const ValueOperand& value, Register dest);
+
+ // higher level tag testing code
+ Address ToPayload(Address value) {
+ return value;
+ }
+
+ template <typename T>
+ void loadUnboxedValue(const T& address, MIRType type, AnyRegister dest) {
+ if (dest.isFloat())
+ loadInt32OrDouble(address, dest.fpu());
+ else
+ unboxNonDouble(address, dest.gpr(), ValueTypeFromMIRType(type));
+ }
+
+ void storeUnboxedPayload(ValueOperand value, BaseIndex address, size_t nbytes, JSValueType type) {
+ switch (nbytes) {
+ case 8:
+ if (type == JSVAL_TYPE_OBJECT)
+ unboxObjectOrNull(value, SecondScratchReg);
+ else
+ unboxNonDouble(value, SecondScratchReg, type);
+ computeEffectiveAddress(address, ScratchRegister);
+ as_std(SecondScratchReg, ScratchRegister, 0);
+ return;
+ case 4:
+ store32(value.valueReg(), address);
+ return;
+ case 1:
+ store8(value.valueReg(), address);
+ return;
+ default: MOZ_CRASH("Bad payload width");
+ }
+ }
+
+ void storeUnboxedPayload(ValueOperand value, Address address, size_t nbytes, JSValueType type) {
+ switch (nbytes) {
+ case 8:
+ if (type == JSVAL_TYPE_OBJECT)
+ unboxObjectOrNull(value, SecondScratchReg);
+ else
+ unboxNonDouble(value, SecondScratchReg, type);
+ storePtr(SecondScratchReg, address);
+ return;
+ case 4:
+ store32(value.valueReg(), address);
+ return;
+ case 1:
+ store8(value.valueReg(), address);
+ return;
+ default: MOZ_CRASH("Bad payload width");
+ }
+ }
+
+ void boxValue(JSValueType type, Register src, Register dest);
+ void storeValue(ValueOperand val, Operand dst);
+ void storeValue(ValueOperand val, const BaseIndex& dest);
+ void storeValue(JSValueType type, Register reg, BaseIndex dest);
+ void storeValue(ValueOperand val, const Address& dest);
+ void storeValue(JSValueType type, Register reg, Address dest);
+ void storeValue(const Value& val, Address dest);
+ void storeValue(const Value& val, BaseIndex dest);
+ void storeValue(const Address& src, const Address& dest, Register temp) {
+ loadPtr(src, temp);
+ storePtr(temp, dest);
+ }
+ void storePrivateValue(Register src, const Address& dest) {
+ storePtr(src, dest);
+ }
+ void storePrivateValue(ImmGCPtr imm, const Address& dest) {
+ storePtr(imm, dest);
+ }
+
+ void loadValue(Address src, ValueOperand val);
+ void loadValue(Operand dest, ValueOperand val) {
+ loadValue(dest.toAddress(), val);
+ }
+ void loadValue(const BaseIndex& addr, ValueOperand val);
+
+ void loadUnalignedValue(const Address& src, ValueOperand dest) {
+ loadValue(src, dest);
+ }
+
+ void tagValue(JSValueType type, Register payload, ValueOperand dest);
+
+ void pushValue(ValueOperand val);
+ void popValue(ValueOperand val);
+ void pushValue(const Value& val) {
+ if (val.isGCThing()) {
+ writeDataRelocation(val);
+ movWithPatch(ImmWord(val.asRawBits()), ScratchRegister);
+ push(ScratchRegister);
+ } else {
+ push(ImmWord(val.asRawBits()));
+ }
+ }
+ void pushValue(JSValueType type, Register reg) {
+ // Use SecondScratchReg as the temp since boxValue uses ScratchRegister
+ // for the tag.
+ boxValue(type, reg, SecondScratchReg);
+ push(SecondScratchReg);
+ }
+ void pushValue(const Address& addr);
+ void pushValue(const BaseIndex& addr, Register scratch) {
+ loadValue(addr, ValueOperand(scratch));
+ pushValue(ValueOperand(scratch));
+ }
+
+ void handleFailureWithHandlerTail(Label* profilerExitTail, Label* bailoutTail);
+
+ /////////////////////////////////////////////////////////////////
+ // Common interface.
+ /////////////////////////////////////////////////////////////////
+ public:
+ // The following functions are exposed for use in platform-shared code.
+
+ inline void incrementInt32Value(const Address& addr);
+
+ void move32(Imm32 imm, Register dest);
+ void move32(Register src, Register dest);
+
+ void movePtr(Register src, Register dest);
+ void movePtr(ImmWord imm, Register dest);
+ void movePtr(ImmPtr imm, Register dest);
+ void movePtr(wasm::SymbolicAddress imm, Register dest);
+ void movePtr(ImmGCPtr imm, Register dest);
+
+ FaultingCodeOffset load8SignExtend(const Address& address, Register dest);
+ FaultingCodeOffset load8SignExtend(const BaseIndex& src, Register dest);
+
+ FaultingCodeOffset load8ZeroExtend(const Address& address, Register dest);
+ FaultingCodeOffset load8ZeroExtend(const BaseIndex& src, Register dest);
+
+ FaultingCodeOffset load16SignExtend(const Address& address, Register dest);
+ FaultingCodeOffset load16SignExtend(const BaseIndex& src, Register dest);
+
+ template <typename S>
+ FaultingCodeOffset load16UnalignedSignExtend(const S& src, Register dest) {
+ return ma_load(dest, src, SizeHalfWord, SignExtend);
+ }
+
+ FaultingCodeOffset load16ZeroExtend(const Address& address, Register dest);
+ FaultingCodeOffset load16ZeroExtend(const BaseIndex& src, Register dest);
+
+ template <typename S>
+ FaultingCodeOffset load16UnalignedZeroExtend(const S& src, Register dest) {
+ return ma_load(dest, src, SizeHalfWord, ZeroExtend);
+ }
+
+ FaultingCodeOffset load32(const Address& address, Register dest);
+ FaultingCodeOffset load32(const BaseIndex& address, Register dest);
+ FaultingCodeOffset load32ZeroExtend(const Address& address, Register dest);
+ FaultingCodeOffset load32ZeroExtend(const BaseIndex& address, Register dest);
+ FaultingCodeOffset load32(AbsoluteAddress address, Register dest);
+ FaultingCodeOffset load32(wasm::SymbolicAddress address, Register dest);
+
+ FaultingCodeOffset load64(const Address& address, Register64 dest) {
+ return loadPtr(address, dest.reg);
+ }
+
+ template <typename S>
+ FaultingCodeOffset load32Unaligned(const S& src, Register dest) {
+ return ma_load(dest, src, SizeWord, SignExtend);
+ }
+
+ FaultingCodeOffset load64(const BaseIndex& address, Register64 dest) {
+ return loadPtr(address, dest.reg);
+ }
+
+ template <typename S>
+ FaultingCodeOffset load64Unaligned(const S& src, Register64 dest) {
+ return ma_load(dest.reg, src, SizeDouble, ZeroExtend);
+ }
+
+ void loadPtr(Register src, Register dest);
+ FaultingCodeOffset loadPtr(const Address& address, Register dest);
+ FaultingCodeOffset loadPtr(const BaseIndex& src, Register dest);
+ FaultingCodeOffset loadPtr(AbsoluteAddress address, Register dest);
+ FaultingCodeOffset loadPtr(wasm::SymbolicAddress address, Register dest);
+
+ void loadPrivate(const Address& address, Register dest);
+
+ void loadInt32x1(const Address& addr, FloatRegister dest) { MOZ_CRASH("NYI"); }
+ void loadInt32x1(const BaseIndex& addr, FloatRegister dest) { MOZ_CRASH("NYI"); }
+ void loadInt32x2(const Address& addr, FloatRegister dest) { MOZ_CRASH("NYI"); }
+ void loadInt32x2(const BaseIndex& addr, FloatRegister dest) { MOZ_CRASH("NYI"); }
+ void loadInt32x3(const Address& src, FloatRegister dest) { MOZ_CRASH("NYI"); }
+ void loadInt32x3(const BaseIndex& src, FloatRegister dest) { MOZ_CRASH("NYI"); }
+ void loadInt32x4(const Address& src, FloatRegister dest) { MOZ_CRASH("NYI"); }
+ void storeInt32x1(FloatRegister src, const Address& dest) { MOZ_CRASH("NYI"); }
+ void storeInt32x1(FloatRegister src, const BaseIndex& dest) { MOZ_CRASH("NYI"); }
+ void storeInt32x2(FloatRegister src, const Address& dest) { MOZ_CRASH("NYI"); }
+ void storeInt32x2(FloatRegister src, const BaseIndex& dest) { MOZ_CRASH("NYI"); }
+ void storeInt32x3(FloatRegister src, const Address& dest) { MOZ_CRASH("NYI"); }
+ void storeInt32x3(FloatRegister src, const BaseIndex& dest) { MOZ_CRASH("NYI"); }
+ void storeInt32x4(FloatRegister src, const Address& dest) { MOZ_CRASH("NYI"); }
+ void loadAlignedSimd128Int(const Address& addr, FloatRegister dest) { MOZ_CRASH("NYI"); }
+ void storeAlignedSimd128Int(FloatRegister src, Address addr) { MOZ_CRASH("NYI"); }
+ void loadUnalignedSimd128Int(const Address& addr, FloatRegister dest) { MOZ_CRASH("NYI"); }
+ void loadUnalignedSimd128Int(const BaseIndex& addr, FloatRegister dest) { MOZ_CRASH("NYI"); }
+ void storeUnalignedSimd128Int(FloatRegister src, Address addr) { MOZ_CRASH("NYI"); }
+ void storeUnalignedSimd128Int(FloatRegister src, BaseIndex addr) { MOZ_CRASH("NYI"); }
+
+ void loadFloat32x3(const Address& src, FloatRegister dest) { MOZ_CRASH("NYI"); }
+ void loadFloat32x3(const BaseIndex& src, FloatRegister dest) { MOZ_CRASH("NYI"); }
+ void loadFloat32x4(const Address& src, FloatRegister dest) { MOZ_CRASH("NYI"); }
+ void storeFloat32x4(FloatRegister src, const Address& addr) { MOZ_CRASH("NYI"); }
+
+ void loadAlignedSimd128Float(const Address& addr, FloatRegister dest) { MOZ_CRASH("NYI"); }
+ void storeAlignedSimd128Float(FloatRegister src, Address addr) { MOZ_CRASH("NYI"); }
+ void loadUnalignedSimd128Float(const Address& addr, FloatRegister dest) { MOZ_CRASH("NYI"); }
+ void loadUnalignedSimd128Float(const BaseIndex& addr, FloatRegister dest) { MOZ_CRASH("NYI"); }
+ void storeUnalignedSimd128Float(FloatRegister src, Address addr) { MOZ_CRASH("NYI"); }
+ void storeUnalignedSimd128Float(FloatRegister src, BaseIndex addr) { MOZ_CRASH("NYI"); }
+
+/*
+ void loadUnalignedDouble(const wasm::MemoryAccessDesc& access, const BaseIndex& src,
+ Register temp, FloatRegister dest);
+ void loadUnalignedFloat32(const wasm::MemoryAccessDesc& access, const BaseIndex& src,
+ Register temp, FloatRegister dest);
+*/
+
+ FaultingCodeOffset store8(Register src, const Address& address);
+ FaultingCodeOffset store8(Imm32 imm, const Address& address);
+ FaultingCodeOffset store8(Register src, const BaseIndex& address);
+ FaultingCodeOffset store8(Imm32 imm, const BaseIndex& address);
+
+ FaultingCodeOffset store16(Register src, const Address& address);
+ FaultingCodeOffset store16(Imm32 imm, const Address& address);
+ FaultingCodeOffset store16(Register src, const BaseIndex& address);
+ FaultingCodeOffset store16(Imm32 imm, const BaseIndex& address);
+
+ template <typename T>
+ FaultingCodeOffset store16Unaligned(Register src, const T& dest) {
+ //ma_store_unaligned(src, dest, SizeHalfWord);
+ return store16(src, dest);
+ }
+
+ FaultingCodeOffset store32(Register src, AbsoluteAddress address);
+ FaultingCodeOffset store32(Register src, const Address& address);
+ FaultingCodeOffset store32(Register src, const BaseIndex& address);
+ FaultingCodeOffset store32(Imm32 src, const Address& address);
+ FaultingCodeOffset store32(Imm32 src, const BaseIndex& address);
+
+ // NOTE: This will use second scratch on PPC64. Only ARM needs the
+ // implementation without second scratch.
+ FaultingCodeOffset store32_NoSecondScratch(Imm32 src, const Address& address) {
+ return store32(src, address);
+ }
+
+ template <typename T>
+ FaultingCodeOffset store32Unaligned(Register src, const T& dest) {
+ //ma_store_unaligned(src, dest, SizeWord);
+ return store32(src, dest);
+ }
+
+ FaultingCodeOffset store64(Imm64 imm, Address address) {
+ return storePtr(ImmWord(imm.value), address);
+ }
+
+ FaultingCodeOffset store64(Register64 src, Address address) {
+ return storePtr(src.reg, address);
+ }
+
+ FaultingCodeOffset store64(Register64 src, const BaseIndex& address) {
+ return storePtr(src.reg, address);
+ }
+
+ template <typename T>
+ FaultingCodeOffset store64Unaligned(Register64 src, const T& dest) {
+ //ma_store_unaligned(src.reg, dest, SizeDouble);
+ return store64(src, dest);
+ }
+
+ template <typename T> FaultingCodeOffset storePtr(ImmWord imm, T address);
+ template <typename T> FaultingCodeOffset storePtr(ImmPtr imm, T address);
+ template <typename T> FaultingCodeOffset storePtr(ImmGCPtr imm, T address);
+ FaultingCodeOffset storePtr(Register src, const Address& address);
+ FaultingCodeOffset storePtr(Register src, const BaseIndex& address);
+ FaultingCodeOffset storePtr(Register src, AbsoluteAddress dest);
+
+/*
+ void storeUnalignedFloat32(const wasm::MemoryAccessDesc& access, FloatRegister src,
+ Register temp, const BaseIndex& dest);
+ void storeUnalignedDouble(const wasm::MemoryAccessDesc& access, FloatRegister src,
+ Register temp, const BaseIndex& dest);
+*/
+
+ void moveDouble(FloatRegister src, FloatRegister dest) {
+ if (src != dest) as_fmr(dest, src);
+ }
+
+ void zeroDouble(FloatRegister reg) {
+ if (HasPPCISA3()) {
+ // This should also work on POWER8.
+ // It's my favourite mnemonic in the whole instruction set.
+ // Just say it six times fast. It's positively Asgardian.
+ as_xxlxor(reg, reg, reg);
+ } else {
+ xs_li(ScratchRegister, 0);
+ moveToDouble(ScratchRegister, reg);
+ }
+ }
+
+ void moveFromDouble(FloatRegister src, Register dest) {
+ if (HasPPCISA3()) {
+ // This should also work on POWER8.
+ as_mfvsrd(dest, src);
+ } else {
+ // Sigh.
+ as_stfdu(src, StackPointer, -8);
+ as_ld(dest, StackPointer, 0);
+ as_addi(StackPointer, StackPointer, 8);
+ }
+ }
+
+ void moveToDouble(Register src, FloatRegister dest) {
+ if (HasPPCISA3()) {
+ // This should also work on POWER8.
+ as_mtvsrd(dest, src);
+ } else {
+ // Sigh.
+ as_stdu(src, StackPointer, -8);
+ as_lfd(dest, StackPointer, 0);
+ as_addi(StackPointer, StackPointer, 8);
+ }
+ }
+
+ void moveFromFloat32(FloatRegister src, Register dest) {
+ if (HasPPCISA3()) {
+ MOZ_ASSERT(src != ScratchDoubleReg);
+ // Enforce rounding mode 0b00 (round-to-nearest tie-to-even).
+ as_mtfsfi(7, 0);
+ // Downconvert prior to processing and splat it into 32-bit singles.
+ as_xscvdpspn(ScratchDoubleReg, src); // preserve sNaN bit, all equal
+ as_mfvsrd(dest, ScratchDoubleReg);
+ // Take off the top word, leaving the float.
+ as_rldicl(dest, dest, 0, 32); // "clrldi"
+ } else {
+ // Sigh.
+ as_stfsu(src, StackPointer, -4);
+ as_lwz(dest, StackPointer, 0);
+ as_addi(StackPointer, StackPointer, 4);
+ }
+ }
+
+ void moveToFloat32(Register src, FloatRegister dest) {
+ if (HasPPCISA3()) {
+ // Splat the 32-bit word as singles throughout the VSR, the
+ // upconvert to double.
+ as_mtvsrws(dest, src); // loads into both words of DW 0
+ as_xscvspdpn(dest, dest); // preserve sNaN bit, both doubles equal
+// XXX: what if we put an frsp here to get rid of it in the 32->64 upconvert?
+ } else {
+ // Sigh.
+ as_stwu(src, StackPointer, -4);
+ as_lfs(dest, StackPointer, 0);
+ as_addi(StackPointer, StackPointer, 4);
+ }
+ }
+ void convertUInt64ToDouble(Register src, FloatRegister dest);
+
+ void breakpoint();
+
+ void checkStackAlignment();
+
+ static void calculateAlignedStackPointer(void** stackPointer);
+
+ // If source is a double, load it into dest. If source is int32,
+ // convert it to double. Else, branch to failure.
+ void ensureDouble(const ValueOperand& source, FloatRegister dest, Label* failure);
+
+ void cmpPtrSet(Assembler::Condition cond, Address lhs, ImmPtr rhs, Register dest);
+ void cmpPtrSet(Assembler::Condition cond, Register lhs, Address rhs, Register dest);
+
+ void cmp32Set(Assembler::Condition cond, Register lhs, Address rhs, Register dest);
+
+ protected:
+ bool buildOOLFakeExitFrame(void* fakeReturnAddr);
+
+ void wasmLoadI64Impl(const wasm::MemoryAccessDesc& access, Register memoryBase, Register ptr,
+ Register ptrScratch, Register64 output, Register tmp);
+ void wasmStoreI64Impl(const wasm::MemoryAccessDesc& access, Register64 value, Register memoryBase,
+ Register ptr, Register ptrScratch, Register tmp);
+
+ public:
+ CodeOffset labelForPatch() {
+ return CodeOffset(nextOffset().getOffset());
+ }
+
+ void lea(Operand addr, Register dest) {
+ ma_add(dest, addr.baseReg(), Imm32(addr.disp()));
+ }
+
+ void abiret() {
+ as_blr();
+ }
+
+ void moveFloat32(FloatRegister src, FloatRegister dest) {
+ as_fmr(dest, src);
+ }
+
+/*
+ void loadWasmGlobalPtr(uint32_t globalDataOffset, Register dest) {
+ loadPtr(Address(WasmTlsReg, offsetof(wasm::TlsData, globalArea) + globalDataOffset), dest);
+ }
+ void loadWasmPinnedRegsFromTls() {
+ loadPtr(Address(WasmTlsReg, offsetof(wasm::TlsData, memoryBase)), HeapReg);
+ }
+*/
+
+ // Instrumentation for entering and leaving the profiler.
+ void profilerEnterFrame(Register framePtr, Register scratch);
+ void profilerExitFrame();
+};
+
+typedef MacroAssemblerPPC64Compat MacroAssemblerSpecific;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_ppc64_MacroAssembler_ppc64_h */
diff -r 485b15bb4a20 -r 9c245f4665be js/src/jit/ppc64/MoveEmitter-ppc64.cpp
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/js/src/jit/ppc64/MoveEmitter-ppc64.cpp Thu Aug 08 21:23:52 2024 -0700
@@ -0,0 +1,366 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/ppc64/MoveEmitter-ppc64.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+void
+MoveEmitterPPC64::emit(const MoveResolver& moves)
+{
+ if (moves.numCycles()) {
+ // Reserve stack for cycle resolution
+ masm.reserveStack(moves.numCycles() * sizeof(double));
+ pushedAtCycle_ = masm.framePushed();
+ }
+
+ for (size_t i = 0; i < moves.numMoves(); i++)
+ emit(moves.getMove(i));
+}
+
+Address
+MoveEmitterPPC64::cycleSlot(uint32_t slot, uint32_t subslot) const
+{
+ int32_t offset = masm.framePushed() - pushedAtCycle_;
+ MOZ_ASSERT(Imm16::IsInSignedRange(offset));
+ return Address(StackPointer, offset + slot * sizeof(double) + subslot);
+}
+
+int32_t
+MoveEmitterPPC64::getAdjustedOffset(const MoveOperand& operand)
+{
+ MOZ_ASSERT(operand.isMemoryOrEffectiveAddress());
+ if (operand.base() != StackPointer)
+ return operand.disp();
+
+ // Adjust offset if stack pointer has been moved.
+ return operand.disp() + masm.framePushed() - pushedAtStart_;
+}
+
+Address
+MoveEmitterPPC64::getAdjustedAddress(const MoveOperand& operand)
+{
+ return Address(operand.base(), getAdjustedOffset(operand));
+}
+
+
+Register
+MoveEmitterPPC64::tempReg()
+{
+ spilledReg_ = SecondScratchReg;
+ return SecondScratchReg;
+}
+
+void
+MoveEmitterPPC64::emitMove(const MoveOperand& from, const MoveOperand& to)
+{
+ if (from.isGeneralReg()) {
+ // Second scratch register should not be moved by MoveEmitter.
+ MOZ_ASSERT(from.reg() != spilledReg_);
+
+ if (to.isGeneralReg())
+ masm.movePtr(from.reg(), to.reg());
+ else if (to.isMemory())
+ masm.storePtr(from.reg(), getAdjustedAddress(to));
+ else
+ MOZ_CRASH("Invalid emitMove arguments.");
+ } else if (from.isMemory()) {
+ if (to.isGeneralReg()) {
+ masm.loadPtr(getAdjustedAddress(from), to.reg());
+ } else if (to.isMemory()) {
+ masm.loadPtr(getAdjustedAddress(from), tempReg());
+ masm.storePtr(tempReg(), getAdjustedAddress(to));
+ } else {
+ MOZ_CRASH("Invalid emitMove arguments.");
+ }
+ } else if (from.isEffectiveAddress()) {
+ if (to.isGeneralReg()) {
+ masm.computeEffectiveAddress(getAdjustedAddress(from), to.reg());
+ } else if (to.isMemory()) {
+ masm.computeEffectiveAddress(getAdjustedAddress(from), tempReg());
+ masm.storePtr(tempReg(), getAdjustedAddress(to));
+ } else {
+ MOZ_CRASH("Invalid emitMove arguments.");
+ }
+ } else {
+ MOZ_CRASH("Invalid emitMove arguments.");
+ }
+}
+
+void
+MoveEmitterPPC64::emitInt32Move(const MoveOperand &from, const MoveOperand &to)
+{
+ if (from.isGeneralReg()) {
+ // Second scratch register should not be moved by MoveEmitter.
+ MOZ_ASSERT(from.reg() != spilledReg_);
+
+ if (to.isGeneralReg())
+ masm.move32(from.reg(), to.reg());
+ else if (to.isMemory())
+ masm.store32(from.reg(), getAdjustedAddress(to));
+ else
+ MOZ_CRASH("Invalid emitInt32Move arguments.");
+ } else if (from.isMemory()) {
+ if (to.isGeneralReg()) {
+ masm.load32(getAdjustedAddress(from), to.reg());
+ } else if (to.isMemory()) {
+ masm.load32(getAdjustedAddress(from), tempReg());
+ masm.store32(tempReg(), getAdjustedAddress(to));
+ } else {
+ MOZ_CRASH("Invalid emitInt32Move arguments.");
+ }
+ } else if (from.isEffectiveAddress()) {
+ if (to.isGeneralReg()) {
+ masm.computeEffectiveAddress(getAdjustedAddress(from), to.reg());
+ } else if (to.isMemory()) {
+ masm.computeEffectiveAddress(getAdjustedAddress(from), tempReg());
+ masm.store32(tempReg(), getAdjustedAddress(to));
+ } else {
+ MOZ_CRASH("Invalid emitInt32Move arguments.");
+ }
+ } else {
+ MOZ_CRASH("Invalid emitInt32Move arguments.");
+ }
+}
+
+void
+MoveEmitterPPC64::emitFloat32Move(const MoveOperand& from, const MoveOperand& to)
+{
+ // Don't clobber the temp register if it's the source.
+ MOZ_ASSERT_IF(from.isFloatReg(), from.floatReg() != ScratchFloat32Reg);
+ MOZ_ASSERT_IF(to.isFloatReg(), to.floatReg() != ScratchFloat32Reg);
+
+ if (from.isFloatReg()) {
+ if (to.isFloatReg()) {
+ masm.moveFloat32(from.floatReg(), to.floatReg());
+ } else if (to.isGeneralReg()) {
+ // Don't bother handling this case.
+ MOZ_CRASH("emitFloat32Move -> GPR NYI");
+ } else {
+ MOZ_ASSERT(to.isMemory());
+ masm.storeFloat32(from.floatReg(), getAdjustedAddress(to));
+ }
+ } else if (to.isFloatReg()) {
+ MOZ_ASSERT(from.isMemory());
+ masm.loadFloat32(getAdjustedAddress(from), to.floatReg());
+ } else if (to.isGeneralReg()) {
+ MOZ_ASSERT(from.isMemory());
+ masm.loadPtr(getAdjustedAddress(from), to.reg());
+ } else {
+ MOZ_ASSERT(from.isMemory());
+ MOZ_ASSERT(to.isMemory());
+ masm.loadFloat32(getAdjustedAddress(from), ScratchFloat32Reg);
+ masm.storeFloat32(ScratchFloat32Reg, getAdjustedAddress(to));
+ }
+}
+
+// This is almost the same.
+void
+MoveEmitterPPC64::emitDoubleMove(const MoveOperand& from, const MoveOperand& to)
+{
+ // Don't clobber the temp register if it's the source.
+ MOZ_ASSERT_IF(from.isFloatReg(), from.floatReg() != ScratchFloat32Reg);
+ MOZ_ASSERT_IF(to.isFloatReg(), to.floatReg() != ScratchFloat32Reg);
+
+ if (from.isFloatReg()) {
+ if (to.isFloatReg()) {
+ masm.moveDouble(from.floatReg(), to.floatReg());
+ } else if (to.isGeneralReg()) {
+ // Maximum bogosity again, dude.
+ MOZ_CRASH("emitDoubleMove FPR -> GPR NYI");
+ } else {
+ MOZ_ASSERT(to.isMemory());
+ masm.storeDouble(from.floatReg(), getAdjustedAddress(to));
+ }
+ } else if (to.isFloatReg()) {
+ MOZ_ASSERT(from.isMemory());
+ masm.loadDouble(getAdjustedAddress(from), to.floatReg());
+ } else if (to.isGeneralReg()) {
+ // Not handled (yet?)
+ MOZ_CRASH("emitDoubleMove mem -> GPR NYI");
+ } else {
+ MOZ_ASSERT(from.isMemory());
+ MOZ_ASSERT(to.isMemory());
+ masm.loadDouble(getAdjustedAddress(from), ScratchDoubleReg);
+ masm.storeDouble(ScratchDoubleReg, getAdjustedAddress(to));
+ }
+}
+
+void
+MoveEmitterPPC64::breakCycle(const MoveOperand& from, const MoveOperand& to,
+ MoveOp::Type type, uint32_t slotId)
+{
+ // There is some pattern:
+ // (A -> B)
+ // (B -> A)
+ //
+ // This case handles (A -> B), which we reach first. We save B, then allow
+ // the original move to continue.
+ switch (type) {
+ case MoveOp::FLOAT32:
+ if (to.isMemory()) {
+ FloatRegister temp = ScratchFloat32Reg;
+ masm.loadFloat32(getAdjustedAddress(to), temp);
+ masm.storeFloat32(temp, cycleSlot(slotId));
+ } else {
+ masm.storeFloat32(to.floatReg(), cycleSlot(slotId));
+ }
+ break;
+ case MoveOp::DOUBLE:
+ if (to.isMemory()) {
+ FloatRegister temp = ScratchDoubleReg;
+ masm.loadDouble(getAdjustedAddress(to), temp);
+ masm.storeDouble(temp, cycleSlot(slotId));
+ } else {
+ masm.storeDouble(to.floatReg(), cycleSlot(slotId));
+ }
+ break;
+ case MoveOp::INT32:
+ if (to.isMemory()) {
+ Register temp = tempReg();
+ masm.load32(getAdjustedAddress(to), temp);
+ masm.store32(temp, cycleSlot(0));
+ } else {
+ // Second scratch register should not be moved by MoveEmitter.
+ MOZ_ASSERT(to.reg() != spilledReg_);
+ masm.store32(to.reg(), cycleSlot(0));
+ }
+ break;
+ case MoveOp::GENERAL:
+ if (to.isMemory()) {
+ Register temp = tempReg();
+ masm.loadPtr(getAdjustedAddress(to), temp);
+ masm.storePtr(temp, cycleSlot(0));
+ } else {
+ // Second scratch register should not be moved by MoveEmitter.
+ MOZ_ASSERT(to.reg() != spilledReg_);
+ masm.storePtr(to.reg(), cycleSlot(0));
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected move type");
+ }
+}
+
+void
+MoveEmitterPPC64::completeCycle(const MoveOperand& from, const MoveOperand& to,
+ MoveOp::Type type, uint32_t slotId)
+{
+ // There is some pattern:
+ // (A -> B)
+ // (B -> A)
+ //
+ // This case handles (B -> A), which we reach last. We emit a move from the
+ // saved value of B, to A.
+ switch (type) {
+ case MoveOp::FLOAT32:
+ if (to.isMemory()) {
+ FloatRegister temp = ScratchFloat32Reg;
+ masm.loadFloat32(cycleSlot(slotId), temp);
+ masm.storeFloat32(temp, getAdjustedAddress(to));
+ } else {
+ masm.loadFloat32(cycleSlot(slotId), to.floatReg());
+ }
+ break;
+ case MoveOp::DOUBLE:
+ if (to.isMemory()) {
+ FloatRegister temp = ScratchDoubleReg;
+ masm.loadDouble(cycleSlot(slotId), temp);
+ masm.storeDouble(temp, getAdjustedAddress(to));
+ } else {
+ masm.loadDouble(cycleSlot(slotId), to.floatReg());
+ }
+ break;
+ case MoveOp::INT32:
+ MOZ_ASSERT(slotId == 0);
+ if (to.isMemory()) {
+ Register temp = tempReg();
+ masm.load32(cycleSlot(0), temp);
+ masm.store32(temp, getAdjustedAddress(to));
+ } else {
+ // Second scratch register should not be moved by MoveEmitter.
+ MOZ_ASSERT(to.reg() != spilledReg_);
+ masm.load32(cycleSlot(0), to.reg());
+ }
+ break;
+ case MoveOp::GENERAL:
+ MOZ_ASSERT(slotId == 0);
+ if (to.isMemory()) {
+ Register temp = tempReg();
+ masm.loadPtr(cycleSlot(0), temp);
+ masm.storePtr(temp, getAdjustedAddress(to));
+ } else {
+ // Second scratch register should not be moved by MoveEmitter.
+ MOZ_ASSERT(to.reg() != spilledReg_);
+ masm.loadPtr(cycleSlot(0), to.reg());
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected move type");
+ }
+}
+
+void
+MoveEmitterPPC64::emit(const MoveOp& move)
+{
+ const MoveOperand& from = move.from();
+ const MoveOperand& to = move.to();
+
+ if (move.isCycleEnd() && move.isCycleBegin()) {
+ // A fun consequence of aliased registers is you can have multiple
+ // cycles at once, and one can end exactly where another begins.
+ breakCycle(from, to, move.endCycleType(), move.cycleBeginSlot());
+ completeCycle(from, to, move.type(), move.cycleEndSlot());
+ return;
+ }
+
+ if (move.isCycleEnd()) {
+ MOZ_ASSERT(inCycle_);
+ completeCycle(from, to, move.type(), move.cycleEndSlot());
+ MOZ_ASSERT(inCycle_ > 0);
+ inCycle_--;
+ return;
+ }
+
+ if (move.isCycleBegin()) {
+ breakCycle(from, to, move.endCycleType(), move.cycleBeginSlot());
+ inCycle_++;
+ }
+
+ switch (move.type()) {
+ case MoveOp::FLOAT32:
+ emitFloat32Move(from, to);
+ break;
+ case MoveOp::DOUBLE:
+ emitDoubleMove(from, to);
+ break;
+ case MoveOp::INT32:
+ emitInt32Move(from, to);
+ break;
+ case MoveOp::GENERAL:
+ emitMove(from, to);
+ break;
+ default:
+ MOZ_CRASH("Unexpected move type");
+ }
+}
+
+void
+MoveEmitterPPC64::assertDone()
+{
+ MOZ_ASSERT(inCycle_ == 0);
+}
+
+void
+MoveEmitterPPC64::finish()
+{
+ assertDone();
+
+ masm.freeStack(masm.framePushed() - pushedAtStart_);
+}
diff -r 485b15bb4a20 -r 9c245f4665be js/src/jit/ppc64/MoveEmitter-ppc64.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/js/src/jit/ppc64/MoveEmitter-ppc64.h Thu Aug 08 21:23:52 2024 -0700
@@ -0,0 +1,78 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_ppc64le_MoveEmitter_ppc64le_h
+#define jit_ppc64le_MoveEmitter_ppc64le_h
+
+#include "jit/MacroAssembler.h"
+#include "jit/MoveResolver.h"
+
+namespace js {
+namespace jit {
+
+class MoveEmitterPPC64
+{
+ protected:
+ uint32_t inCycle_;
+ MacroAssembler& masm;
+
+ // Original stack push value.
+ uint32_t pushedAtStart_;
+
+ // These store stack offsets to spill locations, snapshotting
+ // codegen->framePushed_ at the time they were allocated. They are -1 if no
+ // stack space has been allocated for that particular spill.
+ int32_t pushedAtCycle_;
+ int32_t pushedAtSpill_;
+
+ // These are registers that are available for temporary use. They may be
+ // assigned InvalidReg. If no corresponding spill space has been assigned,
+ // then these registers do not need to be spilled.
+ Register spilledReg_;
+ FloatRegister spilledFloatReg_;
+
+ void assertDone();
+ Register tempReg();
+ FloatRegister tempFloatReg();
+ Address cycleSlot(uint32_t slot, uint32_t subslot = 0) const;
+ int32_t getAdjustedOffset(const MoveOperand& operand);
+ Address getAdjustedAddress(const MoveOperand& operand);
+
+ void emitMove(const MoveOperand& from, const MoveOperand& to);
+ void emitInt32Move(const MoveOperand& from, const MoveOperand& to);
+ void emitFloat32Move(const MoveOperand& from, const MoveOperand& to);
+ void emitDoubleMove(const MoveOperand& from, const MoveOperand& to);
+ void breakCycle(const MoveOperand& from, const MoveOperand& to,
+ MoveOp::Type type, uint32_t slot);
+ void completeCycle(const MoveOperand& from, const MoveOperand& to,
+ MoveOp::Type type, uint32_t slot);
+ void emit(const MoveOp& move);
+
+ public:
+ MoveEmitterPPC64(MacroAssembler& masm)
+ : inCycle_(0),
+ masm(masm),
+ pushedAtStart_(masm.framePushed()),
+ pushedAtCycle_(-1),
+ pushedAtSpill_(-1),
+ spilledReg_(InvalidReg),
+ spilledFloatReg_(InvalidFloatReg)
+ { }
+ ~MoveEmitterPPC64() {
+ assertDone();
+ }
+ void emit(const MoveResolver& moves);
+ void finish();
+
+ void setScratchRegister(Register reg) {}
+};
+
+typedef MoveEmitterPPC64 MoveEmitter;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_ppc64le_MoveEmitter_ppc64le_h */
diff -r 485b15bb4a20 -r 9c245f4665be js/src/jit/ppc64/SharedICHelpers-ppc64-inl.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/js/src/jit/ppc64/SharedICHelpers-ppc64-inl.h Thu Aug 08 21:23:52 2024 -0700
@@ -0,0 +1,80 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_ppc64le_SharedICHelpers_ppc64le_inl_h
+#define jit_ppc64le_SharedICHelpers_ppc64le_inl_h
+
+#include "jit/SharedICHelpers.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+namespace js {
+namespace jit {
+
+inline void EmitBaselineTailCallVM(TrampolinePtr target, MacroAssembler& masm,
+ uint32_t argSize) {
+#ifdef DEBUG
+ Register scratch = R2.scratchReg();
+
+ // Compute frame size.
+ masm.movePtr(FramePointer, scratch);
+ masm.subPtr(StackPointer, scratch);
+
+ // Store frame size without VMFunction arguments for debug assertions.
+ masm.subPtr(Imm32(argSize), scratch);
+ Address frameSizeAddr(FramePointer,
+ BaselineFrame::reverseOffsetOfDebugFrameSize());
+ masm.store32(scratch, frameSizeAddr);
+ masm.addPtr(Imm32(argSize), scratch);
+#endif
+
+ // Push frame descriptor and perform the tail call.
+ // The return address will be pushed by the VM wrapper for compatibility
+ // with direct calls, but since LR is not a GPR, keep ICTailCallReg current.
+ masm.xs_mflr(ICTailCallReg);
+ masm.pushFrameDescriptor(FrameType::BaselineJS);
+ masm.jump(target);
+}
+
+inline void EmitBaselineCallVM(TrampolinePtr target, MacroAssembler& masm) {
+ masm.pushFrameDescriptor(FrameType::BaselineStub);
+ masm.call(target);
+}
+
+inline void EmitBaselineEnterStubFrame(MacroAssembler& masm, Register scratch) {
+ MOZ_ASSERT(scratch != ICTailCallReg);
+
+#ifdef DEBUG
+ // Compute frame size.
+ masm.movePtr(FramePointer, scratch);
+ masm.subPtr(StackPointer, scratch);
+
+ Address frameSizeAddr(FramePointer,
+ BaselineFrame::reverseOffsetOfDebugFrameSize());
+ masm.store32(scratch, frameSizeAddr);
+#endif
+
+ // Note: when making changes here, don't forget to update
+ // BaselineStubFrame if needed.
+
+ // Push frame descriptor and return address.
+ masm.xs_mflr(ICTailCallReg); // keep current
+ masm.PushFrameDescriptor(FrameType::BaselineJS);
+ masm.Push(ICTailCallReg);
+
+ // Save old frame pointer, stack pointer and stub reg.
+ masm.Push(FramePointer);
+ masm.movePtr(StackPointer, FramePointer);
+ masm.Push(ICStubReg);
+
+ // Stack should remain aligned.
+ masm.assertStackAlignment(sizeof(Value), 0);
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_ppc64le_SharedICHelpers_ppc64le_inl_h */
diff -r 485b15bb4a20 -r 9c245f4665be js/src/jit/ppc64/SharedICHelpers-ppc64.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/js/src/jit/ppc64/SharedICHelpers-ppc64.h Thu Aug 08 21:23:52 2024 -0700
@@ -0,0 +1,106 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_ppc64le_SharedICHelpers_ppc64le_h
+#define jit_ppc64le_SharedICHelpers_ppc64le_h
+
+#include "jit/BaselineFrame.h"
+#include "jit/BaselineIC.h"
+#include "jit/MacroAssembler.h"
+#include "jit/SharedICRegisters.h"
+
+namespace js {
+namespace jit {
+
+// Distance from sp to the top Value inside an IC stub (no return address on
+// the stack on Power ISA).
+static const size_t ICStackValueOffset = 0;
+
+struct BaselineStubFrame {
+ uintptr_t savedFrame;
+ uintptr_t savedStub;
+ uintptr_t returnAddress;
+ uintptr_t descriptor;
+};
+
+inline void
+EmitRestoreTailCallReg(MacroAssembler& masm)
+{
+ // No-op; LR is always the return address.
+}
+
+inline void
+EmitRepushTailCallReg(MacroAssembler& masm)
+{
+ // No-op the second; LR is always the return address.
+}
+
+inline void
+EmitCallIC(MacroAssembler& masm, CodeOffset* callOffset)
+{
+ // The stub pointer must already be in ICStubReg.
+ // Load stubcode pointer from the ICStub.
+ // R2 won't be active when we call ICs, so we can use it as scratch.
+ masm.loadPtr(Address(ICStubReg, ICStub::offsetOfStubCode()), R2.scratchReg());
+
+ // Call the stubcode via a direct jump-and-link
+ masm.call(R2.scratchReg());
+ *callOffset = CodeOffset(masm.currentOffset());
+}
+
+inline void
+EmitReturnFromIC(MacroAssembler& masm)
+{
+ masm.as_blr();
+}
+
+inline void
+EmitBaselineLeaveStubFrame(MacroAssembler& masm)
+{
+ masm.loadPtr(
+ Address(FramePointer, BaselineStubFrameLayout::ICStubOffsetFromFP),
+ ICStubReg);
+ masm.movePtr(FramePointer, StackPointer);
+ masm.Pop(FramePointer);
+
+ // Load the return address into our GPR "mirror."
+ masm.Pop(ICTailCallReg);
+ // Move to LR for branching.
+ masm.xs_mtlr(ICTailCallReg);
+
+ // Discard the frame descriptor.
+ {
+ SecondScratchRegisterScope scratch2(masm);
+ masm.Pop(scratch2);
+ }
+}
+
+template <typename AddrType>
+inline void
+EmitPreBarrier(MacroAssembler& masm, const AddrType& addr, MIRType type)
+{
+ // Calls made in the prebarrier may clobber LR, so save it first.
+ masm.xs_mflr(ScratchRegister);
+ masm.push(ScratchRegister);
+ masm.guardedCallPreBarrier(addr, type);
+ masm.pop(ScratchRegister);
+ masm.xs_mtlr(ScratchRegister);
+}
+
+inline void
+EmitStubGuardFailure(MacroAssembler& masm)
+{
+ // Load next stub into ICStubReg
+ masm.loadPtr(Address(ICStubReg, ICCacheIRStub::offsetOfNext()), ICStubReg);
+
+ // Return address is already loaded, just jump to the next stubcode.
+ masm.jump(Address(ICStubReg, ICStub::offsetOfStubCode()));
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_ppc64le_SharedICHelpers_ppc64le_h */
diff -r 485b15bb4a20 -r 9c245f4665be js/src/jit/ppc64/SharedICRegisters-ppc64.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/js/src/jit/ppc64/SharedICRegisters-ppc64.h Thu Aug 08 21:23:52 2024 -0700
@@ -0,0 +1,44 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_ppc64le_SharedICRegisters_ppc64le_h
+#define jit_ppc64le_SharedICRegisters_ppc64le_h
+
+#include "jit/MacroAssembler.h"
+
+namespace js {
+namespace jit {
+
+// This is just an alias for the stack pointer currently.
+static constexpr Register BaselineStackReg = r1;
+
+// ValueOperands R0, R1, and R2.
+// R0 == JSReturnReg, and R2 uses registers not preserved across calls. R1 value
+// should be preserved across calls.
+static constexpr ValueOperand R0(r4);
+static constexpr ValueOperand R1(r15); // non-volatile
+static constexpr ValueOperand R2(r5);
+
+// ICTailCallReg and ICStubReg
+static constexpr Register ICTailCallReg = r25;
+static constexpr Register ICStubReg = r7;
+
+static constexpr Register ExtractTemp0 = InvalidReg;
+static constexpr Register ExtractTemp1 = InvalidReg;
+
+// Register used internally by the Power Macro Assembler.
+static constexpr Register BaselineSecondScratchReg = SecondScratchReg;
+
+// FloatReg0 must be equal to ReturnFloatReg.
+static constexpr FloatRegister FloatReg0 = f1;
+static constexpr FloatRegister FloatReg1 = f2;
+static constexpr FloatRegister FloatReg2 = f3;
+static constexpr FloatRegister FloatReg3 = f4;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_ppc64le_SharedICRegisters_ppc64le_h */
diff -r 485b15bb4a20 -r 9c245f4665be js/src/jit/ppc64/Trampoline-ppc64.cpp
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/js/src/jit/ppc64/Trampoline-ppc64.cpp Thu Aug 08 21:23:52 2024 -0700
@@ -0,0 +1,988 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/Bailouts.h"
+#include "jit/BaselineFrame.h"
+#include "jit/CalleeToken.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#ifdef JS_ION_PERF
+# include "jit/PerfSpewer.h"
+#endif
+#include "jit/ppc64/SharedICRegisters-ppc64.h"
+#include "jit/VMFunctions.h"
+#include "vm/JitActivation.h" // js::jit::JitActivation
+#include "vm/JSContext.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+#if DEBUG
+
+/* Useful class to print visual guard blocks. */
+class TrampolineAutoDeBlock
+{
+ private:
+ const char *blockname;
+
+ public:
+ TrampolineAutoDeBlock(const char *name) {
+ blockname = name;
+ JitSpew(JitSpew_Codegen, "[[[[[[[[ Trampoline: %s", blockname);
+ }
+
+ ~TrampolineAutoDeBlock() {
+ JitSpew(JitSpew_Codegen, " Trampoline: %s ]]]]]]]]", blockname);
+ }
+};
+
+#undef ADBlock
+#define ADBlock(x) TrampolineAutoDeBlock _adbx(x)
+
+#else
+
+/* Useful preprocessor macros to completely elide visual guard blocks. */
+
+#undef ADBlock
+#define ADBlock(x) ;
+
+#endif
+
+using namespace js;
+using namespace js::jit;
+
+// All registers to save and restore. This includes the stack pointer, since we
+// use the ability to reference register values on the stack by index.
+static const LiveRegisterSet AllRegs =
+ LiveRegisterSet(GeneralRegisterSet(Registers::AllMask),
+ FloatRegisterSet(FloatRegisters::AllMask));
+
+// This trampoline adheres to the PowerPC ELF64 ABI.
+static_assert(sizeof(uintptr_t) == sizeof(uint64_t), "Not 32-bit clean.");
+
+/*
+
+From http://refspecs.linuxfoundation.org/ELF/ppc64/PPC-elf64abi.html#FUNC-CALL :
+
+"Registers r1 [SP], r14 through r31, and f14 through f31 are nonvolatile."
+"The condition code register fields CR0, CR1, CR5, CR6, and CR7 are volatile."
+"Registers r0, r3 through r12, f0 through f13, and the special purpose registers LR, CTR,
+XER, and FPSCR are volatile."
+"Furthermore, registers r0, r2, r11, and r12 may be modified by cross-module calls."
+
+Figure 3-17. Stack Frame Organization
+(marked up with our values)
+
+High Address
+
+ +-> Back chain (SP + 416)
+ | Floating point register save area (SP + 272) (18dw here for f14-f31)
+ | General register save area (SP + 128) (18dw here for r14-r31)
+ | VRSAVE save word (32 bits) (SP + 124)
+ | Alignment padding (4 or 12 bytes) (SP + 112) (12b here)
+ | Vector register save area (quadword aligned) (SP + 112) (not used)
+ | Local variable space (SP + 112) (not used)
+ | Parameter save area (SP + 48) (fixed size minimum 8dw here)
+ | TOC save area (SP + 40)
+ | link editor doubleword (SP + 32)
+ | compiler doubleword (SP + 24)
+ | LR save area (SP + 16)
+ | CR save area (SP + 8)
+SP ---> +-- Back chain (SP + 0)
+
+Low Address
+
+"The stack pointer shall maintain quadword alignment."
+
+*/
+
+struct EnterJITRegs
+{
+// SP + 0
+ uint64_t sp;
+ uint64_t cr;
+ uint64_t lr;
+
+ uint64_t comp;
+ uint64_t le;
+ uint64_t toc;
+
+ uint64_t parameters[7];
+
+// SP + 112
+// We cheat here: this is a safe place to store our old VP.
+// In the ABI definition this is just padding.
+ uint64_t savedvp;
+ uint32_t padding;
+ uint32_t vrsave;
+
+// SP + 128
+ uint64_t r14;
+ uint64_t r15;
+ uint64_t r16;
+ uint64_t r17;
+ uint64_t r18;
+ uint64_t r19;
+ uint64_t r20;
+ uint64_t r21;
+ uint64_t r22;
+ uint64_t r23;
+ uint64_t r24;
+ uint64_t r25;
+ uint64_t r26;
+ uint64_t r27;
+ uint64_t r28;
+ uint64_t r29;
+ uint64_t r30;
+ uint64_t r31;
+
+ double f14;
+ double f15;
+ double f16;
+ double f17;
+ double f18;
+ double f19;
+ double f20;
+ double f21;
+ double f22;
+ double f23;
+ double f24;
+ double f25;
+ double f26;
+ double f27;
+ double f28;
+ double f29;
+ double f30;
+ double f31;
+
+ uint64_t mo_paddin; // make size 16-byte
+};
+
+static_assert(sizeof(EnterJITRegs) == 416, "Unexpected size of register save frame");
+static_assert(offsetof(EnterJITRegs, sp) == 0, "Register save frame is incorrectly oriented");
+
+// Generates a trampoline for calling JIT code from a C++ function.
+// The signature is
+// EnterJitCode(void* code, unsigned argc, Value* argv, InterpreterFrame* fp,
+// CalleeToken calleeToken, JSObject* envChain,
+// size_t numStackValues, Value* vp);
+// Happily, this all fits into registers.
+void
+JitRuntime::generateEnterJIT(JSContext* cx, MacroAssembler& masm)
+{
+ AutoCreatedBy acb(masm, "JitRuntime::generateEnterJIT");
+ ADBlock("generateEnterJIT");
+
+ enterJITOffset_ = startTrampolineCode(masm);
+
+ const Register reg_code = IntArgReg0; // r3
+ const Register reg_argc = IntArgReg1; // r4
+ const Register reg_argv = IntArgReg2; // r5
+ const mozilla::DebugOnly<Register> reg_frame = IntArgReg3; // r6
+ const Register reg_token = IntArgReg4; // r7
+ const Register reg_chain = IntArgReg5; // r8
+ const Register reg_values = IntArgReg6; // r9
+ const Register reg_vp = IntArgReg7; // r10
+
+ MOZ_ASSERT(OsrFrameReg == reg_frame);
+
+ // Standard Power prologue, more or less.
+ // First save LR and CR to the caller linkage area.
+ masm.xs_mflr(ScratchRegister);
+ masm.as_std(ScratchRegister, StackPointer, offsetof(EnterJITRegs, lr)); // caller
+ masm.as_mfcr(ScratchRegister);
+ masm.as_std(ScratchRegister, StackPointer, offsetof(EnterJITRegs, cr)); // caller
+ // Save SP to the caller linkage area and pull down the new frame.
+ masm.as_stdu(StackPointer, StackPointer, -((uint16_t)sizeof(EnterJITRegs)));
+
+ // Save non-volatile registers.
+#define SAVE(x) masm.as_std(x, StackPointer, offsetof(EnterJITRegs, x));
+ SAVE(r14)
+ SAVE(r15)
+ SAVE(r16)
+ SAVE(r17)
+ SAVE(r18)
+ SAVE(r19)
+ SAVE(r20)
+ SAVE(r21)
+ SAVE(r22)
+ SAVE(r23)
+ SAVE(r24)
+ SAVE(r25)
+ SAVE(r26)
+ SAVE(r27)
+ SAVE(r28)
+ SAVE(r29)
+ SAVE(r30)
+ SAVE(r31)
+#undef SAVE
+#define SAVE(x) masm.as_stfd(x, StackPointer, offsetof(EnterJITRegs, x));
+ SAVE(f14)
+ SAVE(f15)
+ SAVE(f16)
+ SAVE(f17)
+ SAVE(f18)
+ SAVE(f19)
+ SAVE(f20)
+ SAVE(f21)
+ SAVE(f22)
+ SAVE(f23)
+ SAVE(f24)
+ SAVE(f25)
+ SAVE(f26)
+ SAVE(f27)
+ SAVE(f28)
+ SAVE(f29)
+ SAVE(f30)
+ SAVE(f31)
+#undef SAVE
+
+ // Save VP for the end.
+ // We would also save VRSAVE here, if we were likely to use VMX/VSX.
+ // We load nargs a little later.
+ masm.as_std(reg_vp, StackPointer, offsetof(EnterJITRegs, savedvp));
+
+ // Save stack pointer as baseline frame.
+ masm.movePtr(StackPointer, FramePointer);
+
+ /***************************************************************
+ Loop over argv vector, push arguments onto stack in reverse order
+ ***************************************************************/
+
+ // If we are constructing, the count also needs to include newTarget.
+ MOZ_ASSERT(CalleeToken_FunctionConstructing == 0x01);
+ masm.as_andi_rc(ScratchRegister, reg_token, CalleeToken_FunctionConstructing);
+ masm.as_add(reg_argc, reg_argc, ScratchRegister);
+
+ // |Value| is 8-byte aligned, but we want to maintain 16-byte alignment,
+ // so tack on an extra Value if the number of arguments is odd.
+ // Set the address to copy from to *vp + (argc * 8).
+ // WARNING: ABI compliant stack frames are now no longer guaranteed.
+ MOZ_ASSERT(sizeof(Value) == 8);
+ masm.as_andi_rc(ScratchRegister, reg_argc, 1);
+ masm.x_slwi(SecondScratchReg, reg_argc, 3); // times 8
+ masm.x_slwi(ScratchRegister, ScratchRegister, 3); // times 8
+ masm.addPtr(reg_argv, SecondScratchReg);
+ masm.as_add(StackPointer, StackPointer, ScratchRegister);
+
+ // Loop over arguments, copying them from the JS buffer onto the Ion
+ // stack so they can be accessed from JIT'ed code.
+ Label header, footer;
+ // If there aren't any arguments, don't do anything.
+ masm.ma_bc(SecondScratchReg, reg_argv, &footer, Assembler::BelowOrEqual, ShortJump);
+ {
+ masm.bind(&header);
+
+ masm.subPtr(Imm32(sizeof(Value)), SecondScratchReg);
+ masm.subPtr(Imm32(sizeof(Value)), StackPointer);
+
+ masm.as_ld(ScratchRegister, SecondScratchReg, 0);
+ // XXX: Is this usually on stack? Would inserting nops here help?
+ masm.as_std(ScratchRegister, StackPointer, 0);
+
+// XXX: bdnz
+ masm.ma_bc(SecondScratchReg, reg_argv, &header, Assembler::Above, ShortJump);
+ }
+ masm.bind(&footer);
+
+ // Load the number of actual arguments (a 32-bit word), then push the
+ // callee token and actual arguments as part of the new frame.
+ masm.push(reg_token);
+ masm.as_lwz(ScratchRegister, reg_vp, 0);
+ masm.pushFrameDescriptorForJitCall(FrameType::CppToJSJit,
+ ScratchRegister, ScratchRegister);
+
+ CodeLabel returnLabel;
+ Label oomReturnLabel;
+ {
+ // Handle Interpreter -> Baseline OSR.
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ MOZ_ASSERT(!regs.has(FramePointer));
+ regs.take(OsrFrameReg);
+ regs.take(reg_code);
+ MOZ_ASSERT(reg_code == ReturnReg); // regs.take(ReturnReg);
+ MOZ_ASSERT(!regs.has(ReturnReg), "ReturnReg matches reg_code");
+#if(0)
+ // On Power reg_code and the ReturnReg are always aliased because of
+ // ABI requirements. The first argument passed, the code pointer,
+ // comes in r3, and the ABI requires that r3 be the return register.
+ // Therefore, we don't implement the changes in bug 1770922.
+ regs.take(JSReturnOperand); // ???
+#endif
+
+ Label notOsr;
+ masm.ma_bc(OsrFrameReg, OsrFrameReg, &notOsr, Assembler::Zero, ShortJump);
+
+ Register numStackValues = reg_values;
+ regs.take(numStackValues);
+ Register scratch = regs.takeAny();
+
+ // Frame prologue.
+ // Push return address.
+ masm.subPtr(Imm32(sizeof(uintptr_t) * 2), StackPointer);
+ masm.ma_li(scratch, &returnLabel);
+ masm.storePtr(scratch, Address(StackPointer, sizeof(uintptr_t)));
+ // Push previous frame pointer. Recovered at frame epilogue.
+ masm.storePtr(FramePointer, Address(StackPointer, 0));
+
+ // Reserve frame.
+ masm.movePtr(StackPointer, FramePointer);
+ masm.subPtr(Imm32(BaselineFrame::Size()), StackPointer);
+
+ Register framePtrScratch = regs.takeAny();
+ masm.movePtr(StackPointer, framePtrScratch);
+
+ // Reserve space for locals and stack values.
+ masm.x_sldi(scratch, numStackValues, 3);
+ masm.subPtr(scratch, StackPointer);
+
+ // Enter exit frame.
+ masm.reserveStack(3 * sizeof(uintptr_t));
+ masm.storePtr(
+ ImmWord(MakeFrameDescriptor(FrameType::BaselineJS)),
+ Address(StackPointer, 2 * sizeof(uintptr_t))); // Frame descriptor
+ masm.storePtr(scratch,
+ Address(StackPointer, sizeof(uintptr_t))); // fake return address
+ masm.storePtr(FramePointer, Address(StackPointer, 0));
+
+ // No GC things to mark, so push a bare token.
+ masm.loadJSContext(scratch);
+ masm.enterFakeExitFrame(scratch, scratch, ExitFrameType::Bare);
+
+ masm.reserveStack(2 * sizeof(uintptr_t));
+ masm.storePtr(FramePointer,
+ Address(StackPointer, sizeof(uintptr_t))); // BaselineFrame
+ masm.storePtr(reg_code, Address(StackPointer, 0)); // jitcode
+
+ using Fn = bool (*)(BaselineFrame * frame, InterpreterFrame * interpFrame,
+ uint32_t numStackValues);
+ masm.setupUnalignedABICall(scratch);
+ masm.passABIArg(framePtrScratch); // BaselineFrame
+ masm.passABIArg(OsrFrameReg); // InterpreterFrame
+ masm.passABIArg(numStackValues);
+ masm.callWithABI<Fn, jit::InitBaselineFrameForOsr>(
+ ABIType::General, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ regs.add(OsrFrameReg);
+ Register jitcode = regs.takeAny();
+ masm.loadPtr(Address(StackPointer, 0), jitcode);
+ masm.loadPtr(Address(StackPointer, sizeof(uintptr_t)), FramePointer);
+ masm.freeStack(2 * sizeof(uintptr_t));
+
+ Label error;
+ masm.freeStack(ExitFrameLayout::SizeWithFooter());
+ masm.branchIfFalseBool(ReturnReg, &error);
+
+ // If OSR-ing, then emit instrumentation for setting lastProfilerFrame
+ // if profiler instrumentation is enabled.
+ {
+ Label skipProfilingInstrumentation;
+ AbsoluteAddress addressOfEnabled(cx->runtime()->geckoProfiler().addressOfEnabled());
+ masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0),
+ &skipProfilingInstrumentation);
+ masm.profilerEnterFrame(FramePointer, scratch);
+ masm.bind(&skipProfilingInstrumentation);
+ }
+
+//masm.xs_trap_tagged(Assembler::DebugTag0);
+ masm.jump(jitcode);
+
+ // OOM: frame epilogue, load error value, discard return address
+ // and return.
+ masm.bind(&error);
+ masm.movePtr(FramePointer, StackPointer); // don't need to reload FP
+ masm.addPtr(Imm32(2 * sizeof(uintptr_t)), StackPointer);
+ masm.moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
+ masm.jump(&oomReturnLabel);
+
+ masm.bind(&notOsr);
+ // Load the scope chain in R1.
+ MOZ_ASSERT(R1.scratchReg() != reg_code);
+ masm.ma_move(R1.scratchReg(), reg_chain);
+ }
+
+ // The call will push the return address on the stack, thus we check that
+ // the stack would be aligned once the call is complete.
+// XXX: I don't think this is appropriate for us. We don't claim to be ABI
+// compliant at this point, and we pass all the stack invariants.
+// masm.assertStackAlignment(JitStackAlignment, 16);
+
+ // Call the function with pushing return address to stack.
+//masm.xs_trap_tagged(Assembler::DebugTag0);
+ masm.callJitNoProfiler(reg_code);
+
+ {
+ // Interpreter -> Baseline OSR will return here.
+ masm.bind(&returnLabel);
+ masm.addCodeLabel(returnLabel);
+ masm.bind(&oomReturnLabel);
+ }
+
+ // Discard arguments and padding. Set sp to the address of the EnterJITRegs
+ // on the stack.
+ masm.mov(FramePointer, StackPointer);
+
+ // Store the returned value into the vp.
+ masm.as_ld(reg_vp, StackPointer, offsetof(EnterJITRegs, savedvp));
+ masm.storeValue(JSReturnOperand, Address(reg_vp, 0));
+
+ // Restore non-volatile registers and return.
+ // Standard PowerPC epilogue, more or less.
+ // Load registers.
+#define LOAD(x) masm.as_ld(x, StackPointer, offsetof(EnterJITRegs, x));
+ LOAD(r14)
+ LOAD(r15)
+ LOAD(r16)
+ LOAD(r17)
+ LOAD(r18)
+ LOAD(r19)
+ LOAD(r20)
+ LOAD(r21)
+ LOAD(r22)
+ LOAD(r23)
+ LOAD(r24)
+ LOAD(r25)
+ LOAD(r26)
+ LOAD(r27)
+ LOAD(r28)
+ LOAD(r29)
+ LOAD(r30)
+ LOAD(r31)
+#undef LOAD
+#define LOAD(x) masm.as_lfd(x, StackPointer, offsetof(EnterJITRegs, x));
+ LOAD(f14)
+ LOAD(f15)
+ LOAD(f16)
+ LOAD(f17)
+ LOAD(f18)
+ LOAD(f19)
+ LOAD(f20)
+ LOAD(f21)
+ LOAD(f22)
+ LOAD(f23)
+ LOAD(f24)
+ LOAD(f25)
+ LOAD(f26)
+ LOAD(f27)
+ LOAD(f28)
+ LOAD(f29)
+ LOAD(f30)
+ LOAD(f31)
+#undef LOAD
+
+ // Tear down frame and retrieve the saved LR and CR from the caller's linkage area.
+ masm.as_addi(StackPointer, StackPointer, (uint16_t)sizeof(EnterJITRegs));
+ masm.as_ld(ScratchRegister, StackPointer, offsetof(EnterJITRegs, lr)); // caller
+ masm.xs_mtlr(ScratchRegister);
+ masm.as_ld(ScratchRegister, StackPointer, offsetof(EnterJITRegs, cr)); // caller
+ masm.xs_mtcr(ScratchRegister);
+
+ // Bye!
+ masm.as_blr();
+}
+
+// static
+mozilla::Maybe<::JS::ProfilingFrameIterator::RegisterState>
+JitRuntime::getCppEntryRegisters(JitFrameLayout* frameStackAddress) {
+ // Not supported, or not implemented yet.
+ // TODO: Implement along with the corresponding stack-walker changes, in
+ // coordination with the Gecko Profiler, see bug 1635987 and follow-ups.
+ return mozilla::Nothing{};
+}
+
+void
+JitRuntime::generateInvalidator(MacroAssembler& masm, Label* bailoutTail)
+{
+ AutoCreatedBy acb(masm, "JitRuntime::generateInvalidator");
+ ADBlock("generateInvalidator");
+
+ invalidatorOffset_ = startTrampolineCode(masm);
+
+ // The InvalidationBailoutStack r3 points to must have:
+ // - osiPointReturnAddress_
+ // - ionScript_ (pushed by CodeGeneratorPPC64::generateInvalidateEpilogue())
+ // - regs_ (pushed here)
+ // - fpregs_ (pushed here) => r3
+
+ // Stack has to be aligned here. If not, we will have to fix it.
+ masm.checkStackAlignment();
+
+ // Push registers such that we can access them from [base + code].
+ masm.PushRegsInMask(AllRegs);
+
+ // Pass pointer to InvalidationBailoutStack structure.
+ masm.movePtr(StackPointer, r3);
+
+ // Reserve place for BailoutInfo pointer. Two words to ensure alignment for
+ // setupAlignedABICall.
+ masm.subPtr(Imm32(2 * sizeof(uintptr_t)), StackPointer);
+ // Pass pointer to BailoutInfo
+ masm.movePtr(StackPointer, r4);
+
+ using Fn = bool (*)(InvalidationBailoutStack * sp,
+ BaselineBailoutInfo * *info);
+ masm.setupAlignedABICall();
+ masm.passABIArg(r3);
+ masm.passABIArg(r4);
+ masm.callWithABI<Fn, InvalidationBailout>(
+ ABIType::General, CheckUnsafeCallWithABI::DontCheckOther);
+
+ masm.pop(r5);
+
+ // Pop the machine state and the dead frame.
+ masm.moveToStackPtr(FramePointer);
+
+ // Jump to shared bailout tail.
+ // The return code is left unchanged by this routine in r3.
+ masm.jump(bailoutTail);
+}
+
+// XXX: completely rewritten, check in tests
+void
+JitRuntime::generateArgumentsRectifier(MacroAssembler& masm,
+ ArgumentsRectifierKind kind)
+{
+ // Do not erase the frame pointer in this function.
+
+ AutoCreatedBy acb(masm, "JitRuntime::generateArgumentsRectifier");
+ ADBlock("generateArgumentsRectifier");
+
+masm.xs_trap();
+
+ // MIPS uses a5-a7, t0-t3 and s3, with s3 being the only callee-save reg.
+ // We will do something similar for Power and use r4-r6, r7-r10 and r15.
+ const Register nvRectReg = r15;
+
+ const Register numArgsReg = r4;
+ const Register numActArgsReg = r5;
+ const Register calleeTokenReg = r6;
+ const Register tempValue = r7;
+ const Register numToPush = r8;
+ const Register tempCalleeTokenReg = r9;
+ const Register tempNumArgsReg = r10;
+
+ switch (kind) {
+ case ArgumentsRectifierKind::Normal:
+ argumentsRectifierOffset_ = startTrampolineCode(masm);
+ break;
+ case ArgumentsRectifierKind::TrialInlining:
+ trialInliningArgumentsRectifierOffset_ = startTrampolineCode(masm);
+ break;
+ }
+ masm.pushReturnAddress();
+
+ // Frame prologue.
+ //
+ // NOTE: if this changes, fix the Baseline bailout code too!
+ // See BaselineStackBuilder::calculatePrevFramePtr and
+ // BaselineStackBuilder::buildRectifierFrame (in BaselineBailouts.cpp).
+ masm.push(FramePointer);
+ masm.mov(StackPointer, FramePointer);
+
+ // Load argc.
+ masm.loadNumActualArgs(FramePointer, nvRectReg);
+ // Load |nformals| into numArgsReg.
+ masm.loadPtr(Address(FramePointer,
+ RectifierFrameLayout::offsetOfCalleeToken()),
+ calleeTokenReg);
+ masm.mov(calleeTokenReg, numArgsReg);
+ masm.andPtr(Imm32(uint32_t(CalleeTokenMask)), numArgsReg);
+ masm.loadFunctionArgCount(numArgsReg, numArgsReg);
+
+ // Stash another copy since we're going to clobber numArgsReg.
+ masm.as_or(tempNumArgsReg, numArgsReg, numArgsReg);
+
+ static_assert(CalleeToken_FunctionConstructing == 1,
+ "Ensure that we can use the constructing bit to count the value");
+ masm.mov(calleeTokenReg, tempCalleeTokenReg); // t2
+ masm.ma_and(tempCalleeTokenReg, Imm32(uint32_t(CalleeToken_FunctionConstructing)));
+
+ // Including |this|, and |new.target|, there are (|nformals| + 1 +
+ // isConstructing) arguments to push to the stack. Then we push a
+ // JitFrameLayout. We compute the padding expressed in the number of extra
+ // |undefined| values to push on the stack.
+ static_assert(
+ sizeof(JitFrameLayout) % JitStackAlignment == 0,
+ "No need to consider the JitFrameLayout for aligning the stack");
+ static_assert(
+ JitStackAlignment % sizeof(Value) == 0,
+ "Ensure that we can pad the stack by pushing extra UndefinedValue");
+
+ MOZ_ASSERT(mozilla::IsPowerOfTwo(JitStackValueAlignment));
+ masm.add32(
+ Imm32(JitStackValueAlignment - 1 /* for padding */ + 1 /* for |this| */),
+ numArgsReg);
+ masm.add32(tempCalleeTokenReg, numArgsReg);
+ masm.and32(Imm32(~(JitStackValueAlignment - 1)), numArgsReg);
+
+// s3 is nvRectReg and has argc
+
+ // Load the number of |undefined|s to push (nargs - nvRectReg).
+ masm.as_subf(numToPush, nvRectReg, numArgsReg); // T = B - A
+ // ... and remove one for |this|
+ masm.as_addi(numToPush, numToPush, -1);
+
+ // Caller:
+ // [arg2] [arg1] [this] [ [argc] [callee] [descr] [raddr] ] <- sp
+ // '-nvRectReg-'
+ //
+ // Rectifier frame:
+ // [fp'][undef] [undef] [undef] [arg2] [arg1] [this] [ [argc] [callee]
+ // [descr] [raddr] ]
+ // '-------- r8 ---------' '-nvRectReg-'
+
+ // Copy number of actual arguments into numActArgsReg.
+ masm.mov(nvRectReg, numActArgsReg);
+
+ masm.moveValue(UndefinedValue(), ValueOperand(tempValue));
+
+ // Push undefined (including the padding).
+ {
+#if(0)
+ Label undefLoopTop;
+
+ masm.bind(&undefLoopTop);
+ masm.sub32(Imm32(1), numToPush);
+ masm.subPtr(Imm32(sizeof(Value)), StackPointer);
+ masm.storeValue(ValueOperand(tempValue), Address(StackPointer, 0));
+
+ masm.ma_bc(numToPush, numToPush, &undefLoopTop, Assembler::NonZero, ShortJump);
+#else
+ masm.xs_mtctr(numToPush);
+ masm.as_stdu(tempValue, StackPointer, -sizeof(Value)); // -4
+ masm.xs_bdnz(-4);
+#endif
+ }
+
+ // Get the topmost argument.
+ static_assert(sizeof(Value) == 8, "TimesEight is used to skip arguments");
+ MOZ_ASSERT(tempValue == r7); // can clobber
+ MOZ_ASSERT(numToPush == r8);
+ MOZ_ASSERT(tempCalleeTokenReg == r9); // can clobber
+ masm.x_slwi(r7, nvRectReg, 3); // r7 <- nargs * 8
+ masm.as_add(numToPush, FramePointer, r7); // r8 <- fp + nargs * 8
+ masm.addPtr(Imm32(sizeof(RectifierFrameLayout)), numToPush);
+
+ // Push arguments, |nargs| + 1 times (to include |this|).
+ masm.as_addi(nvRectReg, nvRectReg, 1);
+ {
+#if(0)
+ Label copyLoopTop;
+
+ masm.bind(&copyLoopTop);
+ masm.sub32(Imm32(1), nvRectReg);
+ masm.subPtr(Imm32(sizeof(Value)), StackPointer);
+ masm.loadValue(Address(numToPush, 0), ValueOperand(tempValue));
+ masm.storeValue(ValueOperand(tempValue), Address(StackPointer, 0));
+ masm.subPtr(Imm32(sizeof(Value)), numToPush);
+
+ masm.ma_bc(nvRectReg, nvRectReg, &copyLoopTop, Assembler::NonZero, ShortJump);
+#else
+ masm.xs_mtctr(nvRectReg);
+ masm.as_ld(tempValue, numToPush, 0); // -12
+ masm.as_addi(numToPush, numToPush, -8);
+ masm.as_stdu(tempValue, StackPointer, -8);
+ masm.xs_bdnz(-12);
+#endif
+ }
+
+ // If constructing, copy newTarget also.
+ {
+ Label notConstructing;
+
+ masm.branchTest32(Assembler::Zero, calleeTokenReg, Imm32(CalleeToken_FunctionConstructing),
+ &notConstructing);
+
+ // thisFrame[numFormals] = prevFrame[argc]
+ ValueOperand newTarget(tempValue);
+
+ // Load vp[argc]. Add sizeof(Value) for |this|.
+ BaseIndex newTargetSrc(FramePointer, numActArgsReg, TimesEight,
+ sizeof(RectifierFrameLayout) + sizeof(Value));
+ masm.loadValue(newTargetSrc, newTarget);
+
+ // Again, 1 for |this|. We bring back our saved register from above.
+ BaseIndex newTargetDest(StackPointer, tempNumArgsReg, TimesEight, sizeof(Value));
+ masm.storeValue(newTarget, newTargetDest);
+
+ masm.bind(&notConstructing);
+ }
+
+ // Caller:
+ // [arg2] [arg1] [this] [ [argc] [callee] [descr] [raddr] ]
+ //
+ //
+ // Rectifier frame:
+ // [fp'] <- fp [undef] [undef] [undef] [arg2] [arg1] [this] <- sp [ [argc]
+ // [callee] [descr] [raddr] ]
+
+ // Construct JitFrameLayout.
+ masm.push(calleeTokenReg);
+ masm.pushFrameDescriptorForJitCall(FrameType::Rectifier, numActArgsReg,
+ numActArgsReg);
+
+ // Call the target function.
+ masm.andPtr(Imm32(uint32_t(CalleeTokenMask)), calleeTokenReg);
+ switch (kind) {
+ case ArgumentsRectifierKind::Normal:
+ masm.loadJitCodeRaw(calleeTokenReg, r8); // can clobber r8
+ argumentsRectifierReturnOffset_ = masm.callJitNoProfiler(r8);
+ break;
+ case ArgumentsRectifierKind::TrialInlining:
+ Label noBaselineScript, done;
+ masm.loadBaselineJitCodeRaw(calleeTokenReg, r8, &noBaselineScript);
+ masm.callJitNoProfiler(r8);
+ masm.ma_b(&done, ShortJump);
+
+ // See BaselineCacheIRCompiler::emitCallInlinedFunction.
+ masm.bind(&noBaselineScript);
+ masm.loadJitCodeRaw(calleeTokenReg, r8);
+ masm.callJitNoProfiler(r8);
+ masm.bind(&done);
+ break;
+ }
+
+ masm.mov(FramePointer, StackPointer);
+ masm.pop(FramePointer);
+ masm.ret();
+}
+
+/* - When bailout is done via out of line code (lazy bailout).
+ * Frame size is stored in LR (look at
+ * CodeGeneratorPPC64::generateOutOfLineCode()) and thunk code should save it
+ * on stack. In addition, snapshotOffset_ and padding_ are
+ * pushed to the stack by CodeGeneratorPPC64::visitOutOfLineBailout().
+ */
+static void
+PushBailoutFrame(MacroAssembler& masm, Register spArg)
+{
+ // Push the frameSize_ stored in LR.
+ masm.xs_mflr(ScratchRegister);
+ masm.push(ScratchRegister);
+
+ // Push registers such that we can access them from [base + code].
+ masm.PushRegsInMask(AllRegs);
+
+ // Put pointer to BailoutStack as first argument to the Bailout().
+ masm.movePtr(StackPointer, spArg);
+}
+
+static void
+GenerateBailoutThunk(MacroAssembler& masm, Label* bailoutTail)
+{
+ PushBailoutFrame(masm, r3);
+
+ // Put pointer to BailoutInfo.
+ static const uint32_t sizeOfBailoutInfo = sizeof(uintptr_t); // * 2;
+ masm.subPtr(Imm32(sizeOfBailoutInfo), StackPointer);
+ masm.movePtr(StackPointer, r4);
+
+ using Fn = bool (*)(BailoutStack * sp, BaselineBailoutInfo * *info);
+ masm.setupUnalignedABICall(r5);
+ masm.passABIArg(r3);
+ masm.passABIArg(r4);
+ masm.callWithABI<Fn, Bailout>(ABIType::General,
+ CheckUnsafeCallWithABI::DontCheckOther);
+
+ // Get BailoutInfo pointer.
+ masm.loadPtr(Address(StackPointer, 0), r5);
+
+ // Remove both the bailout frame and the topmost Ion frame's stack.
+ masm.moveToStackPtr(FramePointer);
+
+ // Jump to shared bailout tail. The BailoutInfo pointer is still in r5 and
+ // the return code is already in r3, so we can just branch.
+ masm.jump(bailoutTail);
+}
+
+void
+JitRuntime::generateBailoutHandler(MacroAssembler& masm, Label* bailoutTail)
+{
+ AutoCreatedBy acb(masm, "JitRuntime::generateBailoutHandler");
+ ADBlock("generateBailoutHandler");
+
+ bailoutHandlerOffset_ = startTrampolineCode(masm);
+
+ GenerateBailoutThunk(masm, bailoutTail);
+}
+
+bool
+JitRuntime::generateVMWrapper(JSContext* cx, MacroAssembler& masm,
+ VMFunctionId id,
+ const VMFunctionData& f, DynFn nativeFun,
+ uint32_t* wrapperOffset)
+{
+ AutoCreatedBy acb(masm, "JitRuntime::generateVMWrapper");
+ ADBlock("generateVMWrapper");
+
+ *wrapperOffset = startTrampolineCode(masm);
+
+ // Avoid conflicts with argument registers while discarding the result
+ // after the function call.
+ AllocatableGeneralRegisterSet regs(Register::Codes::WrapperMask);
+
+ static_assert((Register::Codes::VolatileMask & ~Register::Codes::WrapperMask) == 0,
+ "Wrapper register set should be a superset of Volatile register set.");
+
+ // The context is the first argument; r3 is the first argument register.
+ Register cxreg = r3;
+ regs.take(cxreg);
+
+ // Even if this is a tail call, still push (see SharedICHelpers-ppc64-inl.h).
+ // On link-register platforms, it is the responsibility of the VM *callee* to
+ // push the return address, while the caller must ensure that the address
+ // is stored in LR on entry. This allows the VM wrapper to work with both
+ // direct calls and tail calls.
+ masm.pushReturnAddress();
+
+
+ // Push the frame pointer to finish the exit frame, then link it up.
+ masm.Push(FramePointer);
+ masm.moveStackPtrTo(FramePointer);
+ masm.loadJSContext(cxreg);
+ masm.enterExitFrame(cxreg, regs.getAny(), id);
+
+ // Reserve space for the outparameter.
+ masm.reserveVMFunctionOutParamSpace(f);
+
+ masm.setupUnalignedABICallDontSaveRestoreSP();
+ masm.passABIArg(cxreg);
+
+ size_t argDisp = ExitFrameLayout::Size();
+
+ // Copy any arguments.
+ for (uint32_t explicitArg = 0; explicitArg < f.explicitArgs; explicitArg++) {
+ switch (f.argProperties(explicitArg)) {
+ case VMFunctionData::WordByValue:
+ if (f.argPassedInFloatReg(explicitArg)) {
+ masm.passABIArg(MoveOperand(FramePointer, argDisp), ABIType::Float64);
+ } else {
+ masm.passABIArg(MoveOperand(FramePointer, argDisp), ABIType::General);
+ }
+ argDisp += sizeof(void*);
+ break;
+ case VMFunctionData::WordByRef:
+ masm.passABIArg(MoveOperand(FramePointer, argDisp,
+ MoveOperand::Kind::EffectiveAddress),
+ ABIType::General);
+ argDisp += sizeof(void*);
+ break;
+ case VMFunctionData::DoubleByValue:
+ case VMFunctionData::DoubleByRef:
+ MOZ_CRASH("NYI: riscv callVM should not be used with 128bits values.");
+ break;
+ }
+ }
+
+ // Copy the implicit outparam, if any.
+ //if (InvalidReg != outReg)
+ // masm.passABIArg(outReg);
+ // It is not a C++-abi outparam, which would get passed in the
+ // outparam register, but a real parameter to the function, which
+ // was stack-allocated above.
+ const int32_t outParamOffset =
+ -int32_t(ExitFooterFrame::Size()) - f.sizeOfOutParamStackSlot();
+ if (f.outParam != Type_Void) {
+ masm.passABIArg(MoveOperand(FramePointer, outParamOffset,
+ MoveOperand::Kind::EffectiveAddress),
+ ABIType::General);
+ }
+
+ masm.callWithABI(nativeFun, ABIType::General,
+ CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ // Test for failure.
+ switch (f.failType()) {
+ case Type_Cell:
+ masm.branchTestPtr(Assembler::Zero, r3, r3, masm.failureLabel());
+ break;
+ case Type_Bool:
+ // Called functions return bools, which are 0/false and non-zero/true.
+ masm.branchIfFalseBool(r3, masm.failureLabel());
+ break;
+ case Type_Void:
+ break;
+ default:
+ MOZ_CRASH("unknown failure kind");
+ }
+
+ // Load the outparam. Leave the cannoli.
+ masm.loadVMFunctionOutParam(f, Address(FramePointer, outParamOffset));
+
+ // Pop frame and restore frame pointer.
+ masm.moveToStackPtr(FramePointer);
+ masm.pop(FramePointer);
+
+ // Return. Subtract sizeof(void*) for the frame pointer.
+ masm.retn(Imm32(sizeof(ExitFrameLayout) - sizeof(void*) +
+ f.explicitStackSlots() * sizeof(void*) +
+ f.extraValuesToPop * sizeof(Value)));
+
+ return true;
+}
+
+uint32_t
+JitRuntime::generatePreBarrier(JSContext* cx, MacroAssembler& masm, MIRType type)
+{
+ AutoCreatedBy acb(masm, "JitRuntime::generatePreBarrier");
+ ADBlock("generatePreBarrier");
+
+ uint32_t offset = startTrampolineCode(masm);
+
+ MOZ_ASSERT(PreBarrierReg == r4);
+ Register temp1 = r3;
+ Register temp2 = r5;
+ Register temp3 = r6;
+ // TODO: could be more efficient with multipush/pop
+ masm.push(temp1);
+ masm.push(temp2);
+ masm.push(temp3);
+
+ Label noBarrier;
+ masm.emitPreBarrierFastPath(cx->runtime(), type, temp1, temp2, temp3, &noBarrier);
+
+ // Call into C++ to mark this GC thing.
+ masm.pop(temp3);
+ masm.pop(temp2);
+ masm.pop(temp1);
+
+ // Explicitly save LR, since we can't use it in PushRegsInMask.
+ masm.xs_mflr(ScratchRegister);
+ masm.push(ScratchRegister);
+ LiveRegisterSet save;
+ save.set() = RegisterSet(GeneralRegisterSet(Registers::VolatileMask),
+ FloatRegisterSet(FloatRegisters::VolatileMask));
+ masm.PushRegsInMask(save);
+
+ masm.movePtr(ImmPtr(cx->runtime()), r3);
+
+ masm.setupUnalignedABICall(r5);
+ masm.passABIArg(r3);
+ masm.passABIArg(r4);
+ masm.callWithABI(JitPreWriteBarrier(type));
+
+ masm.PopRegsInMask(save);
+ // ret() pops LR for us.
+ masm.ret();
+
+ masm.bind(&noBarrier);
+ masm.pop(temp3);
+ masm.pop(temp2);
+ masm.pop(temp1);
+ masm.abiret();
+
+ return offset;
+}
+
+void
+JitRuntime::generateBailoutTailStub(MacroAssembler& masm, Label* bailoutTail)
+{
+ AutoCreatedBy acb(masm, "JitRuntime::generateBailoutTailStub");
+ ADBlock("generateBailoutTailStub");
+
+ masm.bind(bailoutTail);
+ masm.generateBailoutTail(r4, r5);
+}
+
Reference in New Issue View Git Blame Copy Permalink