Upgrade V8 to 8.8.278.14
Bug: 162604069
Bug: 167389063
Test: gts-tradefed run gts-dev --module GtsGmscoreHostTestCases
--test com.google.android.gts.devicepolicy.DeviceOwnerTest#testProxyPacProxyTest
Test: m -j proxy_resolver_v8_unittest && adb sync && adb shell \
/data/nativetest/proxy_resolver_v8_unittest/proxy_resolver_v8_unittest
Merged-In: Ifb09923b9d7f6d8990fb062d7dc0294edf2c098e
Change-Id: Ifb09923b9d7f6d8990fb062d7dc0294edf2c098e
(cherry picked from commit 9580a23bc5b8874a0979001d3595d027cbb68128)
diff --git a/src/codegen/DEPS b/src/codegen/DEPS
new file mode 100644
index 0000000..67e29bc
--- /dev/null
+++ b/src/codegen/DEPS
@@ -0,0 +1,13 @@
+# Copyright 2019 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+specific_include_rules = {
+ "external-reference.cc": [
+ # Required to call into IrregexpInterpreter and RegexpExperimental from
+ # builtin.
+ "+src/regexp/regexp-interpreter.h",
+ "+src/regexp/experimental/experimental.h",
+ "+src/regexp/regexp-macro-assembler-arch.h",
+ ],
+}
diff --git a/src/codegen/DIR_METADATA b/src/codegen/DIR_METADATA
new file mode 100644
index 0000000..fc01866
--- /dev/null
+++ b/src/codegen/DIR_METADATA
@@ -0,0 +1,11 @@
+# Metadata information for this directory.
+#
+# For more information on DIR_METADATA files, see:
+# https://source.chromium.org/chromium/infra/infra/+/master:go/src/infra/tools/dirmd/README.md
+#
+# For the schema of this file, see Metadata message:
+# https://source.chromium.org/chromium/infra/infra/+/master:go/src/infra/tools/dirmd/proto/dir_metadata.proto
+
+monorail {
+ component: "Blink>JavaScript>Compiler"
+}
\ No newline at end of file
diff --git a/src/codegen/OWNERS b/src/codegen/OWNERS
new file mode 100644
index 0000000..332c170
--- /dev/null
+++ b/src/codegen/OWNERS
@@ -0,0 +1,19 @@
+bbudge@chromium.org
+bmeurer@chromium.org
+clemensb@chromium.org
+delphick@chromium.org
+gdeepti@chromium.org
+ishell@chromium.org
+jarin@chromium.org
+jgruber@chromium.org
+jkummerow@chromium.org
+leszeks@chromium.org
+mslekova@chromium.org
+mvstanton@chromium.org
+neis@chromium.org
+rmcilroy@chromium.org
+sigurds@chromium.org
+solanes@chromium.org
+tebbi@chromium.org
+titzer@chromium.org
+mythria@chromium.org
diff --git a/src/codegen/arm/assembler-arm-inl.h b/src/codegen/arm/assembler-arm-inl.h
new file mode 100644
index 0000000..b5f8b96
--- /dev/null
+++ b/src/codegen/arm/assembler-arm-inl.h
@@ -0,0 +1,362 @@
+// Copyright (c) 1994-2006 Sun Microsystems Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//
+// - Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// - Redistribution in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the
+// distribution.
+//
+// - Neither the name of Sun Microsystems or the names of contributors may
+// be used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+// COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+// OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// The original source code covered by the above license above has been modified
+// significantly by Google Inc.
+// Copyright 2012 the V8 project authors. All rights reserved.
+
+#ifndef V8_CODEGEN_ARM_ASSEMBLER_ARM_INL_H_
+#define V8_CODEGEN_ARM_ASSEMBLER_ARM_INL_H_
+
+#include "src/codegen/arm/assembler-arm.h"
+
+#include "src/codegen/assembler.h"
+#include "src/debug/debug.h"
+#include "src/objects/objects-inl.h"
+#include "src/objects/smi.h"
+
+namespace v8 {
+namespace internal {
+
+bool CpuFeatures::SupportsOptimizer() { return true; }
+
+bool CpuFeatures::SupportsWasmSimd128() { return IsSupported(NEON); }
+
+int DoubleRegister::SupportedRegisterCount() {
+ return CpuFeatures::IsSupported(VFP32DREGS) ? 32 : 16;
+}
+
+void RelocInfo::apply(intptr_t delta) {
+ if (RelocInfo::IsInternalReference(rmode_)) {
+ // absolute code pointer inside code object moves with the code object.
+ int32_t* p = reinterpret_cast<int32_t*>(pc_);
+ *p += delta; // relocate entry
+ } else if (RelocInfo::IsRelativeCodeTarget(rmode_)) {
+ Instruction* branch = Instruction::At(pc_);
+ int32_t branch_offset = branch->GetBranchOffset() - delta;
+ branch->SetBranchOffset(branch_offset);
+ }
+}
+
+Address RelocInfo::target_address() {
+ DCHECK(IsCodeTargetMode(rmode_) || IsRuntimeEntry(rmode_) ||
+ IsWasmCall(rmode_));
+ return Assembler::target_address_at(pc_, constant_pool_);
+}
+
+Address RelocInfo::target_address_address() {
+ DCHECK(HasTargetAddressAddress());
+ if (Assembler::IsMovW(Memory<int32_t>(pc_))) {
+ return pc_;
+ } else if (Assembler::IsLdrPcImmediateOffset(Memory<int32_t>(pc_))) {
+ return constant_pool_entry_address();
+ } else {
+ DCHECK(Assembler::IsBOrBlPcImmediateOffset(Memory<int32_t>(pc_)));
+ DCHECK(IsRelativeCodeTarget(rmode_));
+ return pc_;
+ }
+}
+
+Address RelocInfo::constant_pool_entry_address() {
+ DCHECK(IsInConstantPool());
+ return Assembler::constant_pool_entry_address(pc_, constant_pool_);
+}
+
+int RelocInfo::target_address_size() { return kPointerSize; }
+
+HeapObject RelocInfo::target_object() {
+ DCHECK(IsCodeTarget(rmode_) || rmode_ == FULL_EMBEDDED_OBJECT);
+ return HeapObject::cast(
+ Object(Assembler::target_address_at(pc_, constant_pool_)));
+}
+
+HeapObject RelocInfo::target_object_no_host(Isolate* isolate) {
+ return target_object();
+}
+
+Handle<HeapObject> RelocInfo::target_object_handle(Assembler* origin) {
+ if (IsCodeTarget(rmode_) || rmode_ == FULL_EMBEDDED_OBJECT) {
+ return Handle<HeapObject>(reinterpret_cast<Address*>(
+ Assembler::target_address_at(pc_, constant_pool_)));
+ }
+ DCHECK(IsRelativeCodeTarget(rmode_));
+ return origin->relative_code_target_object_handle_at(pc_);
+}
+
+void RelocInfo::set_target_object(Heap* heap, HeapObject target,
+ WriteBarrierMode write_barrier_mode,
+ ICacheFlushMode icache_flush_mode) {
+ DCHECK(IsCodeTarget(rmode_) || rmode_ == FULL_EMBEDDED_OBJECT);
+ Assembler::set_target_address_at(pc_, constant_pool_, target.ptr(),
+ icache_flush_mode);
+ if (write_barrier_mode == UPDATE_WRITE_BARRIER && !host().is_null() &&
+ !FLAG_disable_write_barriers) {
+ WriteBarrierForCode(host(), this, target);
+ }
+}
+
+Address RelocInfo::target_external_reference() {
+ DCHECK(rmode_ == EXTERNAL_REFERENCE);
+ return Assembler::target_address_at(pc_, constant_pool_);
+}
+
+void RelocInfo::set_target_external_reference(
+ Address target, ICacheFlushMode icache_flush_mode) {
+ DCHECK(rmode_ == RelocInfo::EXTERNAL_REFERENCE);
+ Assembler::set_target_address_at(pc_, constant_pool_, target,
+ icache_flush_mode);
+}
+
+Address RelocInfo::target_internal_reference() {
+ DCHECK(rmode_ == INTERNAL_REFERENCE);
+ return Memory<Address>(pc_);
+}
+
+Address RelocInfo::target_internal_reference_address() {
+ DCHECK(rmode_ == INTERNAL_REFERENCE);
+ return pc_;
+}
+
+Address RelocInfo::target_runtime_entry(Assembler* origin) {
+ DCHECK(IsRuntimeEntry(rmode_));
+ return target_address();
+}
+
+void RelocInfo::set_target_runtime_entry(Address target,
+ WriteBarrierMode write_barrier_mode,
+ ICacheFlushMode icache_flush_mode) {
+ DCHECK(IsRuntimeEntry(rmode_));
+ if (target_address() != target)
+ set_target_address(target, write_barrier_mode, icache_flush_mode);
+}
+
+Address RelocInfo::target_off_heap_target() {
+ DCHECK(IsOffHeapTarget(rmode_));
+ return Assembler::target_address_at(pc_, constant_pool_);
+}
+
+void RelocInfo::WipeOut() {
+ DCHECK(IsFullEmbeddedObject(rmode_) || IsCodeTarget(rmode_) ||
+ IsRuntimeEntry(rmode_) || IsExternalReference(rmode_) ||
+ IsInternalReference(rmode_) || IsOffHeapTarget(rmode_));
+ if (IsInternalReference(rmode_)) {
+ Memory<Address>(pc_) = kNullAddress;
+ } else {
+ Assembler::set_target_address_at(pc_, constant_pool_, kNullAddress);
+ }
+}
+
+Handle<Code> Assembler::relative_code_target_object_handle_at(
+ Address pc) const {
+ Instruction* branch = Instruction::At(pc);
+ int code_target_index = branch->GetBranchOffset() / kInstrSize;
+ return GetCodeTarget(code_target_index);
+}
+
+Operand Operand::Zero() { return Operand(static_cast<int32_t>(0)); }
+
+Operand::Operand(const ExternalReference& f)
+ : rmode_(RelocInfo::EXTERNAL_REFERENCE) {
+ value_.immediate = static_cast<int32_t>(f.address());
+}
+
+Operand::Operand(Smi value) : rmode_(RelocInfo::NONE) {
+ value_.immediate = static_cast<intptr_t>(value.ptr());
+}
+
+Operand::Operand(Register rm) : rm_(rm), shift_op_(LSL), shift_imm_(0) {}
+
+void Assembler::CheckBuffer() {
+ if (buffer_space() <= kGap) {
+ GrowBuffer();
+ }
+ MaybeCheckConstPool();
+}
+
+void Assembler::emit(Instr x) {
+ CheckBuffer();
+ *reinterpret_cast<Instr*>(pc_) = x;
+ pc_ += kInstrSize;
+}
+
+void Assembler::deserialization_set_special_target_at(
+ Address constant_pool_entry, Code code, Address target) {
+ DCHECK(!Builtins::IsIsolateIndependentBuiltin(code));
+ Memory<Address>(constant_pool_entry) = target;
+}
+
+int Assembler::deserialization_special_target_size(Address location) {
+ return kSpecialTargetSize;
+}
+
+void Assembler::deserialization_set_target_internal_reference_at(
+ Address pc, Address target, RelocInfo::Mode mode) {
+ Memory<Address>(pc) = target;
+}
+
+bool Assembler::is_constant_pool_load(Address pc) {
+ return IsLdrPcImmediateOffset(Memory<int32_t>(pc));
+}
+
+Address Assembler::constant_pool_entry_address(Address pc,
+ Address constant_pool) {
+ DCHECK(Assembler::IsLdrPcImmediateOffset(Memory<int32_t>(pc)));
+ Instr instr = Memory<int32_t>(pc);
+ return pc + GetLdrRegisterImmediateOffset(instr) + Instruction::kPcLoadDelta;
+}
+
+Address Assembler::target_address_at(Address pc, Address constant_pool) {
+ if (is_constant_pool_load(pc)) {
+ // This is a constant pool lookup. Return the value in the constant pool.
+ return Memory<Address>(constant_pool_entry_address(pc, constant_pool));
+ } else if (CpuFeatures::IsSupported(ARMv7) && IsMovW(Memory<int32_t>(pc))) {
+ // This is an movw / movt immediate load. Return the immediate.
+ DCHECK(IsMovW(Memory<int32_t>(pc)) &&
+ IsMovT(Memory<int32_t>(pc + kInstrSize)));
+ Instruction* movw_instr = Instruction::At(pc);
+ Instruction* movt_instr = Instruction::At(pc + kInstrSize);
+ return static_cast<Address>((movt_instr->ImmedMovwMovtValue() << 16) |
+ movw_instr->ImmedMovwMovtValue());
+ } else if (IsMovImmed(Memory<int32_t>(pc))) {
+ // This is an mov / orr immediate load. Return the immediate.
+ DCHECK(IsMovImmed(Memory<int32_t>(pc)) &&
+ IsOrrImmed(Memory<int32_t>(pc + kInstrSize)) &&
+ IsOrrImmed(Memory<int32_t>(pc + 2 * kInstrSize)) &&
+ IsOrrImmed(Memory<int32_t>(pc + 3 * kInstrSize)));
+ Instr mov_instr = instr_at(pc);
+ Instr orr_instr_1 = instr_at(pc + kInstrSize);
+ Instr orr_instr_2 = instr_at(pc + 2 * kInstrSize);
+ Instr orr_instr_3 = instr_at(pc + 3 * kInstrSize);
+ Address ret = static_cast<Address>(
+ DecodeShiftImm(mov_instr) | DecodeShiftImm(orr_instr_1) |
+ DecodeShiftImm(orr_instr_2) | DecodeShiftImm(orr_instr_3));
+ return ret;
+ } else {
+ Instruction* branch = Instruction::At(pc);
+ int32_t delta = branch->GetBranchOffset();
+ return pc + delta + Instruction::kPcLoadDelta;
+ }
+}
+
+void Assembler::set_target_address_at(Address pc, Address constant_pool,
+ Address target,
+ ICacheFlushMode icache_flush_mode) {
+ if (is_constant_pool_load(pc)) {
+ // This is a constant pool lookup. Update the entry in the constant pool.
+ Memory<Address>(constant_pool_entry_address(pc, constant_pool)) = target;
+ // Intuitively, we would think it is necessary to always flush the
+ // instruction cache after patching a target address in the code as follows:
+ // FlushInstructionCache(pc, sizeof(target));
+ // However, on ARM, no instruction is actually patched in the case
+ // of embedded constants of the form:
+ // ldr ip, [pp, #...]
+ // since the instruction accessing this address in the constant pool remains
+ // unchanged.
+ } else if (CpuFeatures::IsSupported(ARMv7) && IsMovW(Memory<int32_t>(pc))) {
+ // This is an movw / movt immediate load. Patch the immediate embedded in
+ // the instructions.
+ DCHECK(IsMovW(Memory<int32_t>(pc)));
+ DCHECK(IsMovT(Memory<int32_t>(pc + kInstrSize)));
+ uint32_t* instr_ptr = reinterpret_cast<uint32_t*>(pc);
+ uint32_t immediate = static_cast<uint32_t>(target);
+ instr_ptr[0] = PatchMovwImmediate(instr_ptr[0], immediate & 0xFFFF);
+ instr_ptr[1] = PatchMovwImmediate(instr_ptr[1], immediate >> 16);
+ DCHECK(IsMovW(Memory<int32_t>(pc)));
+ DCHECK(IsMovT(Memory<int32_t>(pc + kInstrSize)));
+ if (icache_flush_mode != SKIP_ICACHE_FLUSH) {
+ FlushInstructionCache(pc, 2 * kInstrSize);
+ }
+ } else if (IsMovImmed(Memory<int32_t>(pc))) {
+ // This is an mov / orr immediate load. Patch the immediate embedded in
+ // the instructions.
+ DCHECK(IsMovImmed(Memory<int32_t>(pc)) &&
+ IsOrrImmed(Memory<int32_t>(pc + kInstrSize)) &&
+ IsOrrImmed(Memory<int32_t>(pc + 2 * kInstrSize)) &&
+ IsOrrImmed(Memory<int32_t>(pc + 3 * kInstrSize)));
+ uint32_t* instr_ptr = reinterpret_cast<uint32_t*>(pc);
+ uint32_t immediate = static_cast<uint32_t>(target);
+ instr_ptr[0] = PatchShiftImm(instr_ptr[0], immediate & kImm8Mask);
+ instr_ptr[1] = PatchShiftImm(instr_ptr[1], immediate & (kImm8Mask << 8));
+ instr_ptr[2] = PatchShiftImm(instr_ptr[2], immediate & (kImm8Mask << 16));
+ instr_ptr[3] = PatchShiftImm(instr_ptr[3], immediate & (kImm8Mask << 24));
+ DCHECK(IsMovImmed(Memory<int32_t>(pc)) &&
+ IsOrrImmed(Memory<int32_t>(pc + kInstrSize)) &&
+ IsOrrImmed(Memory<int32_t>(pc + 2 * kInstrSize)) &&
+ IsOrrImmed(Memory<int32_t>(pc + 3 * kInstrSize)));
+ if (icache_flush_mode != SKIP_ICACHE_FLUSH) {
+ FlushInstructionCache(pc, 4 * kInstrSize);
+ }
+ } else {
+ intptr_t branch_offset = target - pc - Instruction::kPcLoadDelta;
+ Instruction* branch = Instruction::At(pc);
+ branch->SetBranchOffset(branch_offset);
+ if (icache_flush_mode != SKIP_ICACHE_FLUSH) {
+ FlushInstructionCache(pc, kInstrSize);
+ }
+ }
+}
+
+EnsureSpace::EnsureSpace(Assembler* assembler) { assembler->CheckBuffer(); }
+
+template <typename T>
+bool UseScratchRegisterScope::CanAcquireVfp() const {
+ VfpRegList* available = assembler_->GetScratchVfpRegisterList();
+ DCHECK_NOT_NULL(available);
+ for (int index = 0; index < T::kNumRegisters; index++) {
+ T reg = T::from_code(index);
+ uint64_t mask = reg.ToVfpRegList();
+ if ((*available & mask) == mask) {
+ return true;
+ }
+ }
+ return false;
+}
+
+template <typename T>
+T UseScratchRegisterScope::AcquireVfp() {
+ VfpRegList* available = assembler_->GetScratchVfpRegisterList();
+ DCHECK_NOT_NULL(available);
+ for (int index = 0; index < T::kNumRegisters; index++) {
+ T reg = T::from_code(index);
+ uint64_t mask = reg.ToVfpRegList();
+ if ((*available & mask) == mask) {
+ *available &= ~mask;
+ return reg;
+ }
+ }
+ UNREACHABLE();
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_ARM_ASSEMBLER_ARM_INL_H_
diff --git a/src/codegen/arm/assembler-arm.cc b/src/codegen/arm/assembler-arm.cc
new file mode 100644
index 0000000..cc5d629
--- /dev/null
+++ b/src/codegen/arm/assembler-arm.cc
@@ -0,0 +1,5314 @@
+// Copyright (c) 1994-2006 Sun Microsystems Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//
+// - Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// - Redistribution in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the
+// distribution.
+//
+// - Neither the name of Sun Microsystems or the names of contributors may
+// be used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+// COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+// OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// The original source code covered by the above license above has been
+// modified significantly by Google Inc.
+// Copyright 2012 the V8 project authors. All rights reserved.
+
+#include "src/codegen/arm/assembler-arm.h"
+
+#if V8_TARGET_ARCH_ARM
+
+#include "src/base/bits.h"
+#include "src/base/cpu.h"
+#include "src/base/overflowing-math.h"
+#include "src/codegen/arm/assembler-arm-inl.h"
+#include "src/codegen/assembler-inl.h"
+#include "src/codegen/macro-assembler.h"
+#include "src/codegen/string-constants.h"
+#include "src/deoptimizer/deoptimizer.h"
+#include "src/objects/objects-inl.h"
+
+namespace v8 {
+namespace internal {
+
+static const unsigned kArmv6 = 0u;
+static const unsigned kArmv7 = kArmv6 | (1u << ARMv7);
+static const unsigned kArmv7WithSudiv = kArmv7 | (1u << ARMv7_SUDIV);
+static const unsigned kArmv8 = kArmv7WithSudiv | (1u << ARMv8);
+
+static unsigned CpuFeaturesFromCommandLine() {
+ unsigned result;
+ if (strcmp(FLAG_arm_arch, "armv8") == 0) {
+ result = kArmv8;
+ } else if (strcmp(FLAG_arm_arch, "armv7+sudiv") == 0) {
+ result = kArmv7WithSudiv;
+ } else if (strcmp(FLAG_arm_arch, "armv7") == 0) {
+ result = kArmv7;
+ } else if (strcmp(FLAG_arm_arch, "armv6") == 0) {
+ result = kArmv6;
+ } else {
+ fprintf(stderr, "Error: unrecognised value for --arm-arch ('%s').\n",
+ FLAG_arm_arch);
+ fprintf(stderr,
+ "Supported values are: armv8\n"
+ " armv7+sudiv\n"
+ " armv7\n"
+ " armv6\n");
+ FATAL("arm-arch");
+ }
+
+ // If any of the old (deprecated) flags are specified, print a warning, but
+ // otherwise try to respect them for now.
+ // TODO(jbramley): When all the old bots have been updated, remove this.
+ if (FLAG_enable_armv7.has_value || FLAG_enable_vfp3.has_value ||
+ FLAG_enable_32dregs.has_value || FLAG_enable_neon.has_value ||
+ FLAG_enable_sudiv.has_value || FLAG_enable_armv8.has_value) {
+ // As an approximation of the old behaviour, set the default values from the
+ // arm_arch setting, then apply the flags over the top.
+ bool enable_armv7 = (result & (1u << ARMv7)) != 0;
+ bool enable_vfp3 = (result & (1u << ARMv7)) != 0;
+ bool enable_32dregs = (result & (1u << ARMv7)) != 0;
+ bool enable_neon = (result & (1u << ARMv7)) != 0;
+ bool enable_sudiv = (result & (1u << ARMv7_SUDIV)) != 0;
+ bool enable_armv8 = (result & (1u << ARMv8)) != 0;
+ if (FLAG_enable_armv7.has_value) {
+ fprintf(stderr,
+ "Warning: --enable_armv7 is deprecated. "
+ "Use --arm_arch instead.\n");
+ enable_armv7 = FLAG_enable_armv7.value;
+ }
+ if (FLAG_enable_vfp3.has_value) {
+ fprintf(stderr,
+ "Warning: --enable_vfp3 is deprecated. "
+ "Use --arm_arch instead.\n");
+ enable_vfp3 = FLAG_enable_vfp3.value;
+ }
+ if (FLAG_enable_32dregs.has_value) {
+ fprintf(stderr,
+ "Warning: --enable_32dregs is deprecated. "
+ "Use --arm_arch instead.\n");
+ enable_32dregs = FLAG_enable_32dregs.value;
+ }
+ if (FLAG_enable_neon.has_value) {
+ fprintf(stderr,
+ "Warning: --enable_neon is deprecated. "
+ "Use --arm_arch instead.\n");
+ enable_neon = FLAG_enable_neon.value;
+ }
+ if (FLAG_enable_sudiv.has_value) {
+ fprintf(stderr,
+ "Warning: --enable_sudiv is deprecated. "
+ "Use --arm_arch instead.\n");
+ enable_sudiv = FLAG_enable_sudiv.value;
+ }
+ if (FLAG_enable_armv8.has_value) {
+ fprintf(stderr,
+ "Warning: --enable_armv8 is deprecated. "
+ "Use --arm_arch instead.\n");
+ enable_armv8 = FLAG_enable_armv8.value;
+ }
+ // Emulate the old implications.
+ if (enable_armv8) {
+ enable_vfp3 = true;
+ enable_neon = true;
+ enable_32dregs = true;
+ enable_sudiv = true;
+ }
+ // Select the best available configuration.
+ if (enable_armv7 && enable_vfp3 && enable_32dregs && enable_neon) {
+ if (enable_sudiv) {
+ if (enable_armv8) {
+ result = kArmv8;
+ } else {
+ result = kArmv7WithSudiv;
+ }
+ } else {
+ result = kArmv7;
+ }
+ } else {
+ result = kArmv6;
+ }
+ }
+ return result;
+}
+
+// Get the CPU features enabled by the build.
+// For cross compilation the preprocessor symbols such as
+// CAN_USE_ARMV7_INSTRUCTIONS and CAN_USE_VFP3_INSTRUCTIONS can be used to
+// enable ARMv7 and VFPv3 instructions when building the snapshot. However,
+// these flags should be consistent with a supported ARM configuration:
+// "armv6": ARMv6 + VFPv2
+// "armv7": ARMv7 + VFPv3-D32 + NEON
+// "armv7+sudiv": ARMv7 + VFPv4-D32 + NEON + SUDIV
+// "armv8": ARMv8 (+ all of the above)
+static constexpr unsigned CpuFeaturesFromCompiler() {
+// TODO(jbramley): Once the build flags are simplified, these tests should
+// also be simplified.
+
+// Check *architectural* implications.
+#if defined(CAN_USE_ARMV8_INSTRUCTIONS) && !defined(CAN_USE_ARMV7_INSTRUCTIONS)
+#error "CAN_USE_ARMV8_INSTRUCTIONS should imply CAN_USE_ARMV7_INSTRUCTIONS"
+#endif
+#if defined(CAN_USE_ARMV8_INSTRUCTIONS) && !defined(CAN_USE_SUDIV)
+#error "CAN_USE_ARMV8_INSTRUCTIONS should imply CAN_USE_SUDIV"
+#endif
+#if defined(CAN_USE_ARMV7_INSTRUCTIONS) != defined(CAN_USE_VFP3_INSTRUCTIONS)
+// V8 requires VFP, and all ARMv7 devices with VFP have VFPv3. Similarly,
+// VFPv3 isn't available before ARMv7.
+#error "CAN_USE_ARMV7_INSTRUCTIONS should match CAN_USE_VFP3_INSTRUCTIONS"
+#endif
+#if defined(CAN_USE_NEON) && !defined(CAN_USE_ARMV7_INSTRUCTIONS)
+#error "CAN_USE_NEON should imply CAN_USE_ARMV7_INSTRUCTIONS"
+#endif
+
+// Find compiler-implied features.
+#if defined(CAN_USE_ARMV8_INSTRUCTIONS) && \
+ defined(CAN_USE_ARMV7_INSTRUCTIONS) && defined(CAN_USE_SUDIV) && \
+ defined(CAN_USE_NEON) && defined(CAN_USE_VFP3_INSTRUCTIONS)
+ return kArmv8;
+#elif defined(CAN_USE_ARMV7_INSTRUCTIONS) && defined(CAN_USE_SUDIV) && \
+ defined(CAN_USE_NEON) && defined(CAN_USE_VFP3_INSTRUCTIONS)
+ return kArmv7WithSudiv;
+#elif defined(CAN_USE_ARMV7_INSTRUCTIONS) && defined(CAN_USE_NEON) && \
+ defined(CAN_USE_VFP3_INSTRUCTIONS)
+ return kArmv7;
+#else
+ return kArmv6;
+#endif
+}
+
+void CpuFeatures::ProbeImpl(bool cross_compile) {
+ dcache_line_size_ = 64;
+
+ unsigned command_line = CpuFeaturesFromCommandLine();
+ // Only use statically determined features for cross compile (snapshot).
+ if (cross_compile) {
+ supported_ |= command_line & CpuFeaturesFromCompiler();
+ return;
+ }
+
+#ifndef __arm__
+ // For the simulator build, use whatever the flags specify.
+ supported_ |= command_line;
+
+#else // __arm__
+ // Probe for additional features at runtime.
+ base::CPU cpu;
+ // Runtime detection is slightly fuzzy, and some inferences are necessary.
+ unsigned runtime = kArmv6;
+ // NEON and VFPv3 imply at least ARMv7-A.
+ if (cpu.has_neon() && cpu.has_vfp3_d32()) {
+ DCHECK(cpu.has_vfp3());
+ runtime |= kArmv7;
+ if (cpu.has_idiva()) {
+ runtime |= kArmv7WithSudiv;
+ if (cpu.architecture() >= 8) {
+ runtime |= kArmv8;
+ }
+ }
+ }
+
+ // Use the best of the features found by CPU detection and those inferred from
+ // the build system. In both cases, restrict available features using the
+ // command-line. Note that the command-line flags are very permissive (kArmv8)
+ // by default.
+ supported_ |= command_line & CpuFeaturesFromCompiler();
+ supported_ |= command_line & runtime;
+
+ // Additional tuning options.
+
+ // ARM Cortex-A9 and Cortex-A5 have 32 byte cachelines.
+ if (cpu.implementer() == base::CPU::ARM &&
+ (cpu.part() == base::CPU::ARM_CORTEX_A5 ||
+ cpu.part() == base::CPU::ARM_CORTEX_A9)) {
+ dcache_line_size_ = 32;
+ }
+#endif
+
+ DCHECK_IMPLIES(IsSupported(ARMv7_SUDIV), IsSupported(ARMv7));
+ DCHECK_IMPLIES(IsSupported(ARMv8), IsSupported(ARMv7_SUDIV));
+}
+
+void CpuFeatures::PrintTarget() {
+ const char* arm_arch = nullptr;
+ const char* arm_target_type = "";
+ const char* arm_no_probe = "";
+ const char* arm_fpu = "";
+ const char* arm_thumb = "";
+ const char* arm_float_abi = nullptr;
+
+#if !defined __arm__
+ arm_target_type = " simulator";
+#endif
+
+#if defined ARM_TEST_NO_FEATURE_PROBE
+ arm_no_probe = " noprobe";
+#endif
+
+#if defined CAN_USE_ARMV8_INSTRUCTIONS
+ arm_arch = "arm v8";
+#elif defined CAN_USE_ARMV7_INSTRUCTIONS
+ arm_arch = "arm v7";
+#else
+ arm_arch = "arm v6";
+#endif
+
+#if defined CAN_USE_NEON
+ arm_fpu = " neon";
+#elif defined CAN_USE_VFP3_INSTRUCTIONS
+#if defined CAN_USE_VFP32DREGS
+ arm_fpu = " vfp3";
+#else
+ arm_fpu = " vfp3-d16";
+#endif
+#else
+ arm_fpu = " vfp2";
+#endif
+
+#ifdef __arm__
+ arm_float_abi = base::OS::ArmUsingHardFloat() ? "hard" : "softfp";
+#elif USE_EABI_HARDFLOAT
+ arm_float_abi = "hard";
+#else
+ arm_float_abi = "softfp";
+#endif
+
+#if defined __arm__ && (defined __thumb__) || (defined __thumb2__)
+ arm_thumb = " thumb";
+#endif
+
+ printf("target%s%s %s%s%s %s\n", arm_target_type, arm_no_probe, arm_arch,
+ arm_fpu, arm_thumb, arm_float_abi);
+}
+
+void CpuFeatures::PrintFeatures() {
+ printf("ARMv8=%d ARMv7=%d VFPv3=%d VFP32DREGS=%d NEON=%d SUDIV=%d",
+ CpuFeatures::IsSupported(ARMv8), CpuFeatures::IsSupported(ARMv7),
+ CpuFeatures::IsSupported(VFPv3), CpuFeatures::IsSupported(VFP32DREGS),
+ CpuFeatures::IsSupported(NEON), CpuFeatures::IsSupported(SUDIV));
+#ifdef __arm__
+ bool eabi_hardfloat = base::OS::ArmUsingHardFloat();
+#elif USE_EABI_HARDFLOAT
+ bool eabi_hardfloat = true;
+#else
+ bool eabi_hardfloat = false;
+#endif
+ printf(" USE_EABI_HARDFLOAT=%d\n", eabi_hardfloat);
+}
+
+// -----------------------------------------------------------------------------
+// Implementation of RelocInfo
+
+// static
+const int RelocInfo::kApplyMask =
+ RelocInfo::ModeMask(RelocInfo::RELATIVE_CODE_TARGET);
+
+bool RelocInfo::IsCodedSpecially() {
+ // The deserializer needs to know whether a pointer is specially coded. Being
+ // specially coded on ARM means that it is a movw/movt instruction. We don't
+ // generate those for relocatable pointers.
+ return false;
+}
+
+bool RelocInfo::IsInConstantPool() {
+ return Assembler::is_constant_pool_load(pc_);
+}
+
+uint32_t RelocInfo::wasm_call_tag() const {
+ DCHECK(rmode_ == WASM_CALL || rmode_ == WASM_STUB_CALL);
+ return static_cast<uint32_t>(
+ Assembler::target_address_at(pc_, constant_pool_));
+}
+
+// -----------------------------------------------------------------------------
+// Implementation of Operand and MemOperand
+// See assembler-arm-inl.h for inlined constructors
+
+Operand::Operand(Handle<HeapObject> handle) {
+ rm_ = no_reg;
+ value_.immediate = static_cast<intptr_t>(handle.address());
+ rmode_ = RelocInfo::FULL_EMBEDDED_OBJECT;
+}
+
+Operand::Operand(Register rm, ShiftOp shift_op, int shift_imm) {
+ DCHECK(is_uint5(shift_imm));
+
+ rm_ = rm;
+ rs_ = no_reg;
+ shift_op_ = shift_op;
+ shift_imm_ = shift_imm & 31;
+
+ if ((shift_op == ROR) && (shift_imm == 0)) {
+ // ROR #0 is functionally equivalent to LSL #0 and this allow us to encode
+ // RRX as ROR #0 (See below).
+ shift_op = LSL;
+ } else if (shift_op == RRX) {
+ // encoded as ROR with shift_imm == 0
+ DCHECK_EQ(shift_imm, 0);
+ shift_op_ = ROR;
+ shift_imm_ = 0;
+ }
+}
+
+Operand::Operand(Register rm, ShiftOp shift_op, Register rs) {
+ DCHECK(shift_op != RRX);
+ rm_ = rm;
+ rs_ = no_reg;
+ shift_op_ = shift_op;
+ rs_ = rs;
+}
+
+Operand Operand::EmbeddedNumber(double value) {
+ int32_t smi;
+ if (DoubleToSmiInteger(value, &smi)) return Operand(Smi::FromInt(smi));
+ Operand result(0, RelocInfo::FULL_EMBEDDED_OBJECT);
+ result.is_heap_object_request_ = true;
+ result.value_.heap_object_request = HeapObjectRequest(value);
+ return result;
+}
+
+Operand Operand::EmbeddedStringConstant(const StringConstantBase* str) {
+ Operand result(0, RelocInfo::FULL_EMBEDDED_OBJECT);
+ result.is_heap_object_request_ = true;
+ result.value_.heap_object_request = HeapObjectRequest(str);
+ return result;
+}
+
+MemOperand::MemOperand(Register rn, int32_t offset, AddrMode am)
+ : rn_(rn), rm_(no_reg), offset_(offset), am_(am) {
+ // Accesses below the stack pointer are not safe, and are prohibited by the
+ // ABI. We can check obvious violations here.
+ if (rn == sp) {
+ if (am == Offset) DCHECK_LE(0, offset);
+ if (am == NegOffset) DCHECK_GE(0, offset);
+ }
+}
+
+MemOperand::MemOperand(Register rn, Register rm, AddrMode am)
+ : rn_(rn), rm_(rm), shift_op_(LSL), shift_imm_(0), am_(am) {}
+
+MemOperand::MemOperand(Register rn, Register rm, ShiftOp shift_op,
+ int shift_imm, AddrMode am)
+ : rn_(rn),
+ rm_(rm),
+ shift_op_(shift_op),
+ shift_imm_(shift_imm & 31),
+ am_(am) {
+ DCHECK(is_uint5(shift_imm));
+}
+
+NeonMemOperand::NeonMemOperand(Register rn, AddrMode am, int align)
+ : rn_(rn), rm_(am == Offset ? pc : sp) {
+ DCHECK((am == Offset) || (am == PostIndex));
+ SetAlignment(align);
+}
+
+NeonMemOperand::NeonMemOperand(Register rn, Register rm, int align)
+ : rn_(rn), rm_(rm) {
+ SetAlignment(align);
+}
+
+void NeonMemOperand::SetAlignment(int align) {
+ switch (align) {
+ case 0:
+ align_ = 0;
+ break;
+ case 64:
+ align_ = 1;
+ break;
+ case 128:
+ align_ = 2;
+ break;
+ case 256:
+ align_ = 3;
+ break;
+ default:
+ UNREACHABLE();
+ }
+}
+
+void Assembler::AllocateAndInstallRequestedHeapObjects(Isolate* isolate) {
+ DCHECK_IMPLIES(isolate == nullptr, heap_object_requests_.empty());
+ for (auto& request : heap_object_requests_) {
+ Handle<HeapObject> object;
+ switch (request.kind()) {
+ case HeapObjectRequest::kHeapNumber:
+ object = isolate->factory()->NewHeapNumber<AllocationType::kOld>(
+ request.heap_number());
+ break;
+ case HeapObjectRequest::kStringConstant: {
+ const StringConstantBase* str = request.string();
+ CHECK_NOT_NULL(str);
+ object = str->AllocateStringConstant(isolate);
+ break;
+ }
+ }
+ Address pc = reinterpret_cast<Address>(buffer_start_) + request.offset();
+ Memory<Address>(constant_pool_entry_address(pc, 0 /* unused */)) =
+ object.address();
+ }
+}
+
+// -----------------------------------------------------------------------------
+// Specific instructions, constants, and masks.
+
+// str(r, MemOperand(sp, 4, NegPreIndex), al) instruction (aka push(r))
+// register r is not encoded.
+const Instr kPushRegPattern = al | B26 | 4 | NegPreIndex | sp.code() * B16;
+// ldr(r, MemOperand(sp, 4, PostIndex), al) instruction (aka pop(r))
+// register r is not encoded.
+const Instr kPopRegPattern = al | B26 | L | 4 | PostIndex | sp.code() * B16;
+// ldr rd, [pc, #offset]
+const Instr kLdrPCImmedMask = 15 * B24 | 7 * B20 | 15 * B16;
+const Instr kLdrPCImmedPattern = 5 * B24 | L | pc.code() * B16;
+// Pc-relative call or jump to a signed imm24 offset.
+// bl pc + #offset
+// b pc + #offset
+const Instr kBOrBlPCImmedMask = 0xE * B24;
+const Instr kBOrBlPCImmedPattern = 0xA * B24;
+// vldr dd, [pc, #offset]
+const Instr kVldrDPCMask = 15 * B24 | 3 * B20 | 15 * B16 | 15 * B8;
+const Instr kVldrDPCPattern = 13 * B24 | L | pc.code() * B16 | 11 * B8;
+// blxcc rm
+const Instr kBlxRegMask =
+ 15 * B24 | 15 * B20 | 15 * B16 | 15 * B12 | 15 * B8 | 15 * B4;
+const Instr kBlxRegPattern = B24 | B21 | 15 * B16 | 15 * B12 | 15 * B8 | BLX;
+const Instr kBlxIp = al | kBlxRegPattern | ip.code();
+const Instr kMovMvnMask = 0x6D * B21 | 0xF * B16;
+const Instr kMovMvnPattern = 0xD * B21;
+const Instr kMovMvnFlip = B22;
+const Instr kMovLeaveCCMask = 0xDFF * B16;
+const Instr kMovLeaveCCPattern = 0x1A0 * B16;
+const Instr kMovwPattern = 0x30 * B20;
+const Instr kMovtPattern = 0x34 * B20;
+const Instr kMovwLeaveCCFlip = 0x5 * B21;
+const Instr kMovImmedMask = 0x7F * B21;
+const Instr kMovImmedPattern = 0x1D * B21;
+const Instr kOrrImmedMask = 0x7F * B21;
+const Instr kOrrImmedPattern = 0x1C * B21;
+const Instr kCmpCmnMask = 0xDD * B20 | 0xF * B12;
+const Instr kCmpCmnPattern = 0x15 * B20;
+const Instr kCmpCmnFlip = B21;
+const Instr kAddSubFlip = 0x6 * B21;
+const Instr kAndBicFlip = 0xE * B21;
+
+// A mask for the Rd register for push, pop, ldr, str instructions.
+const Instr kLdrRegFpOffsetPattern = al | B26 | L | Offset | fp.code() * B16;
+const Instr kStrRegFpOffsetPattern = al | B26 | Offset | fp.code() * B16;
+const Instr kLdrRegFpNegOffsetPattern =
+ al | B26 | L | NegOffset | fp.code() * B16;
+const Instr kStrRegFpNegOffsetPattern = al | B26 | NegOffset | fp.code() * B16;
+const Instr kLdrStrInstrTypeMask = 0xFFFF0000;
+
+Assembler::Assembler(const AssemblerOptions& options,
+ std::unique_ptr<AssemblerBuffer> buffer)
+ : AssemblerBase(options, std::move(buffer)),
+ pending_32_bit_constants_(),
+ scratch_register_list_(ip.bit()) {
+ pending_32_bit_constants_.reserve(kMinNumPendingConstants);
+ reloc_info_writer.Reposition(buffer_start_ + buffer_->size(), pc_);
+ next_buffer_check_ = 0;
+ const_pool_blocked_nesting_ = 0;
+ no_const_pool_before_ = 0;
+ first_const_pool_32_use_ = -1;
+ last_bound_pos_ = 0;
+ if (CpuFeatures::IsSupported(VFP32DREGS)) {
+ // Register objects tend to be abstracted and survive between scopes, so
+ // it's awkward to use CpuFeatures::VFP32DREGS with CpuFeatureScope. To make
+ // its use consistent with other features, we always enable it if we can.
+ EnableCpuFeature(VFP32DREGS);
+ // Make sure we pick two D registers which alias a Q register. This way, we
+ // can use a Q as a scratch if NEON is supported.
+ scratch_vfp_register_list_ = d14.ToVfpRegList() | d15.ToVfpRegList();
+ } else {
+ // When VFP32DREGS is not supported, d15 become allocatable. Therefore we
+ // cannot use it as a scratch.
+ scratch_vfp_register_list_ = d14.ToVfpRegList();
+ }
+}
+
+Assembler::~Assembler() { DCHECK_EQ(const_pool_blocked_nesting_, 0); }
+
+void Assembler::GetCode(Isolate* isolate, CodeDesc* desc,
+ SafepointTableBuilder* safepoint_table_builder,
+ int handler_table_offset) {
+ // As a crutch to avoid having to add manual Align calls wherever we use a
+ // raw workflow to create Code objects (mostly in tests), add another Align
+ // call here. It does no harm - the end of the Code object is aligned to the
+ // (larger) kCodeAlignment anyways.
+ // TODO(jgruber): Consider moving responsibility for proper alignment to
+ // metadata table builders (safepoint, handler, constant pool, code
+ // comments).
+ DataAlign(Code::kMetadataAlignment);
+
+ // Emit constant pool if necessary.
+ CheckConstPool(true, false);
+ DCHECK(pending_32_bit_constants_.empty());
+
+ int code_comments_size = WriteCodeComments();
+
+ AllocateAndInstallRequestedHeapObjects(isolate);
+
+ // Set up code descriptor.
+ // TODO(jgruber): Reconsider how these offsets and sizes are maintained up to
+ // this point to make CodeDesc initialization less fiddly.
+
+ static constexpr int kConstantPoolSize = 0;
+ const int instruction_size = pc_offset();
+ const int code_comments_offset = instruction_size - code_comments_size;
+ const int constant_pool_offset = code_comments_offset - kConstantPoolSize;
+ const int handler_table_offset2 = (handler_table_offset == kNoHandlerTable)
+ ? constant_pool_offset
+ : handler_table_offset;
+ const int safepoint_table_offset =
+ (safepoint_table_builder == kNoSafepointTable)
+ ? handler_table_offset2
+ : safepoint_table_builder->GetCodeOffset();
+ const int reloc_info_offset =
+ static_cast<int>(reloc_info_writer.pos() - buffer_->start());
+ CodeDesc::Initialize(desc, this, safepoint_table_offset,
+ handler_table_offset2, constant_pool_offset,
+ code_comments_offset, reloc_info_offset);
+}
+
+void Assembler::Align(int m) {
+ DCHECK(m >= 4 && base::bits::IsPowerOfTwo(m));
+ DCHECK_EQ(pc_offset() & (kInstrSize - 1), 0);
+ while ((pc_offset() & (m - 1)) != 0) {
+ nop();
+ }
+}
+
+void Assembler::CodeTargetAlign() {
+ // Preferred alignment of jump targets on some ARM chips.
+ Align(8);
+}
+
+Condition Assembler::GetCondition(Instr instr) {
+ return Instruction::ConditionField(instr);
+}
+
+bool Assembler::IsLdrRegisterImmediate(Instr instr) {
+ return (instr & (B27 | B26 | B25 | B22 | B20)) == (B26 | B20);
+}
+
+bool Assembler::IsVldrDRegisterImmediate(Instr instr) {
+ return (instr & (15 * B24 | 3 * B20 | 15 * B8)) == (13 * B24 | B20 | 11 * B8);
+}
+
+int Assembler::GetLdrRegisterImmediateOffset(Instr instr) {
+ DCHECK(IsLdrRegisterImmediate(instr));
+ bool positive = (instr & B23) == B23;
+ int offset = instr & kOff12Mask; // Zero extended offset.
+ return positive ? offset : -offset;
+}
+
+int Assembler::GetVldrDRegisterImmediateOffset(Instr instr) {
+ DCHECK(IsVldrDRegisterImmediate(instr));
+ bool positive = (instr & B23) == B23;
+ int offset = instr & kOff8Mask; // Zero extended offset.
+ offset <<= 2;
+ return positive ? offset : -offset;
+}
+
+Instr Assembler::SetLdrRegisterImmediateOffset(Instr instr, int offset) {
+ DCHECK(IsLdrRegisterImmediate(instr));
+ bool positive = offset >= 0;
+ if (!positive) offset = -offset;
+ DCHECK(is_uint12(offset));
+ // Set bit indicating whether the offset should be added.
+ instr = (instr & ~B23) | (positive ? B23 : 0);
+ // Set the actual offset.
+ return (instr & ~kOff12Mask) | offset;
+}
+
+Instr Assembler::SetVldrDRegisterImmediateOffset(Instr instr, int offset) {
+ DCHECK(IsVldrDRegisterImmediate(instr));
+ DCHECK((offset & ~3) == offset); // Must be 64-bit aligned.
+ bool positive = offset >= 0;
+ if (!positive) offset = -offset;
+ DCHECK(is_uint10(offset));
+ // Set bit indicating whether the offset should be added.
+ instr = (instr & ~B23) | (positive ? B23 : 0);
+ // Set the actual offset. Its bottom 2 bits are zero.
+ return (instr & ~kOff8Mask) | (offset >> 2);
+}
+
+bool Assembler::IsStrRegisterImmediate(Instr instr) {
+ return (instr & (B27 | B26 | B25 | B22 | B20)) == B26;
+}
+
+Instr Assembler::SetStrRegisterImmediateOffset(Instr instr, int offset) {
+ DCHECK(IsStrRegisterImmediate(instr));
+ bool positive = offset >= 0;
+ if (!positive) offset = -offset;
+ DCHECK(is_uint12(offset));
+ // Set bit indicating whether the offset should be added.
+ instr = (instr & ~B23) | (positive ? B23 : 0);
+ // Set the actual offset.
+ return (instr & ~kOff12Mask) | offset;
+}
+
+bool Assembler::IsAddRegisterImmediate(Instr instr) {
+ return (instr & (B27 | B26 | B25 | B24 | B23 | B22 | B21)) == (B25 | B23);
+}
+
+Instr Assembler::SetAddRegisterImmediateOffset(Instr instr, int offset) {
+ DCHECK(IsAddRegisterImmediate(instr));
+ DCHECK_GE(offset, 0);
+ DCHECK(is_uint12(offset));
+ // Set the offset.
+ return (instr & ~kOff12Mask) | offset;
+}
+
+Register Assembler::GetRd(Instr instr) {
+ return Register::from_code(Instruction::RdValue(instr));
+}
+
+Register Assembler::GetRn(Instr instr) {
+ return Register::from_code(Instruction::RnValue(instr));
+}
+
+Register Assembler::GetRm(Instr instr) {
+ return Register::from_code(Instruction::RmValue(instr));
+}
+
+bool Assembler::IsPush(Instr instr) {
+ return ((instr & ~kRdMask) == kPushRegPattern);
+}
+
+bool Assembler::IsPop(Instr instr) {
+ return ((instr & ~kRdMask) == kPopRegPattern);
+}
+
+bool Assembler::IsStrRegFpOffset(Instr instr) {
+ return ((instr & kLdrStrInstrTypeMask) == kStrRegFpOffsetPattern);
+}
+
+bool Assembler::IsLdrRegFpOffset(Instr instr) {
+ return ((instr & kLdrStrInstrTypeMask) == kLdrRegFpOffsetPattern);
+}
+
+bool Assembler::IsStrRegFpNegOffset(Instr instr) {
+ return ((instr & kLdrStrInstrTypeMask) == kStrRegFpNegOffsetPattern);
+}
+
+bool Assembler::IsLdrRegFpNegOffset(Instr instr) {
+ return ((instr & kLdrStrInstrTypeMask) == kLdrRegFpNegOffsetPattern);
+}
+
+bool Assembler::IsLdrPcImmediateOffset(Instr instr) {
+ // Check the instruction is indeed a
+ // ldr<cond> <Rd>, [pc +/- offset_12].
+ return (instr & kLdrPCImmedMask) == kLdrPCImmedPattern;
+}
+
+bool Assembler::IsBOrBlPcImmediateOffset(Instr instr) {
+ return (instr & kBOrBlPCImmedMask) == kBOrBlPCImmedPattern;
+}
+
+bool Assembler::IsVldrDPcImmediateOffset(Instr instr) {
+ // Check the instruction is indeed a
+ // vldr<cond> <Dd>, [pc +/- offset_10].
+ return (instr & kVldrDPCMask) == kVldrDPCPattern;
+}
+
+bool Assembler::IsBlxReg(Instr instr) {
+ // Check the instruction is indeed a
+ // blxcc <Rm>
+ return (instr & kBlxRegMask) == kBlxRegPattern;
+}
+
+bool Assembler::IsBlxIp(Instr instr) {
+ // Check the instruction is indeed a
+ // blx ip
+ return instr == kBlxIp;
+}
+
+bool Assembler::IsTstImmediate(Instr instr) {
+ return (instr & (B27 | B26 | I | kOpCodeMask | S | kRdMask)) == (I | TST | S);
+}
+
+bool Assembler::IsCmpRegister(Instr instr) {
+ return (instr & (B27 | B26 | I | kOpCodeMask | S | kRdMask | B4)) ==
+ (CMP | S);
+}
+
+bool Assembler::IsCmpImmediate(Instr instr) {
+ return (instr & (B27 | B26 | I | kOpCodeMask | S | kRdMask)) == (I | CMP | S);
+}
+
+Register Assembler::GetCmpImmediateRegister(Instr instr) {
+ DCHECK(IsCmpImmediate(instr));
+ return GetRn(instr);
+}
+
+int Assembler::GetCmpImmediateRawImmediate(Instr instr) {
+ DCHECK(IsCmpImmediate(instr));
+ return instr & kOff12Mask;
+}
+
+// Labels refer to positions in the (to be) generated code.
+// There are bound, linked, and unused labels.
+//
+// Bound labels refer to known positions in the already
+// generated code. pos() is the position the label refers to.
+//
+// Linked labels refer to unknown positions in the code
+// to be generated; pos() is the position of the last
+// instruction using the label.
+//
+// The linked labels form a link chain by making the branch offset
+// in the instruction steam to point to the previous branch
+// instruction using the same label.
+//
+// The link chain is terminated by a branch offset pointing to the
+// same position.
+
+int Assembler::target_at(int pos) {
+ Instr instr = instr_at(pos);
+ if (is_uint24(instr)) {
+ // Emitted link to a label, not part of a branch.
+ return instr;
+ }
+ DCHECK_EQ(5 * B25, instr & 7 * B25); // b, bl, or blx imm24
+ int imm26 = ((instr & kImm24Mask) << 8) >> 6;
+ if ((Instruction::ConditionField(instr) == kSpecialCondition) &&
+ ((instr & B24) != 0)) {
+ // blx uses bit 24 to encode bit 2 of imm26
+ imm26 += 2;
+ }
+ return pos + Instruction::kPcLoadDelta + imm26;
+}
+
+void Assembler::target_at_put(int pos, int target_pos) {
+ Instr instr = instr_at(pos);
+ if (is_uint24(instr)) {
+ DCHECK(target_pos == pos || target_pos >= 0);
+ // Emitted link to a label, not part of a branch.
+ // Load the position of the label relative to the generated code object
+ // pointer in a register.
+
+ // The existing code must be a single 24-bit label chain link, followed by
+ // nops encoding the destination register. See mov_label_offset.
+
+ // Extract the destination register from the first nop instructions.
+ Register dst =
+ Register::from_code(Instruction::RmValue(instr_at(pos + kInstrSize)));
+ // In addition to the 24-bit label chain link, we expect to find one nop for
+ // ARMv7 and above, or two nops for ARMv6. See mov_label_offset.
+ DCHECK(IsNop(instr_at(pos + kInstrSize), dst.code()));
+ if (!CpuFeatures::IsSupported(ARMv7)) {
+ DCHECK(IsNop(instr_at(pos + 2 * kInstrSize), dst.code()));
+ }
+
+ // Here are the instructions we need to emit:
+ // For ARMv7: target24 => target16_1:target16_0
+ // movw dst, #target16_0
+ // movt dst, #target16_1
+ // For ARMv6: target24 => target8_2:target8_1:target8_0
+ // mov dst, #target8_0
+ // orr dst, dst, #target8_1 << 8
+ // orr dst, dst, #target8_2 << 16
+
+ uint32_t target24 = target_pos + (Code::kHeaderSize - kHeapObjectTag);
+ DCHECK(is_uint24(target24));
+ if (is_uint8(target24)) {
+ // If the target fits in a byte then only patch with a mov
+ // instruction.
+ PatchingAssembler patcher(
+ options(), reinterpret_cast<byte*>(buffer_start_ + pos), 1);
+ patcher.mov(dst, Operand(target24));
+ } else {
+ uint16_t target16_0 = target24 & kImm16Mask;
+ uint16_t target16_1 = target24 >> 16;
+ if (CpuFeatures::IsSupported(ARMv7)) {
+ // Patch with movw/movt.
+ if (target16_1 == 0) {
+ PatchingAssembler patcher(
+ options(), reinterpret_cast<byte*>(buffer_start_ + pos), 1);
+ CpuFeatureScope scope(&patcher, ARMv7);
+ patcher.movw(dst, target16_0);
+ } else {
+ PatchingAssembler patcher(
+ options(), reinterpret_cast<byte*>(buffer_start_ + pos), 2);
+ CpuFeatureScope scope(&patcher, ARMv7);
+ patcher.movw(dst, target16_0);
+ patcher.movt(dst, target16_1);
+ }
+ } else {
+ // Patch with a sequence of mov/orr/orr instructions.
+ uint8_t target8_0 = target16_0 & kImm8Mask;
+ uint8_t target8_1 = target16_0 >> 8;
+ uint8_t target8_2 = target16_1 & kImm8Mask;
+ if (target8_2 == 0) {
+ PatchingAssembler patcher(
+ options(), reinterpret_cast<byte*>(buffer_start_ + pos), 2);
+ patcher.mov(dst, Operand(target8_0));
+ patcher.orr(dst, dst, Operand(target8_1 << 8));
+ } else {
+ PatchingAssembler patcher(
+ options(), reinterpret_cast<byte*>(buffer_start_ + pos), 3);
+ patcher.mov(dst, Operand(target8_0));
+ patcher.orr(dst, dst, Operand(target8_1 << 8));
+ patcher.orr(dst, dst, Operand(target8_2 << 16));
+ }
+ }
+ }
+ return;
+ }
+ int imm26 = target_pos - (pos + Instruction::kPcLoadDelta);
+ DCHECK_EQ(5 * B25, instr & 7 * B25); // b, bl, or blx imm24
+ if (Instruction::ConditionField(instr) == kSpecialCondition) {
+ // blx uses bit 24 to encode bit 2 of imm26
+ DCHECK_EQ(0, imm26 & 1);
+ instr = (instr & ~(B24 | kImm24Mask)) | ((imm26 & 2) >> 1) * B24;
+ } else {
+ DCHECK_EQ(0, imm26 & 3);
+ instr &= ~kImm24Mask;
+ }
+ int imm24 = imm26 >> 2;
+ DCHECK(is_int24(imm24));
+ instr_at_put(pos, instr | (imm24 & kImm24Mask));
+}
+
+void Assembler::print(const Label* L) {
+ if (L->is_unused()) {
+ PrintF("unused label\n");
+ } else if (L->is_bound()) {
+ PrintF("bound label to %d\n", L->pos());
+ } else if (L->is_linked()) {
+ Label l;
+ l.link_to(L->pos());
+ PrintF("unbound label");
+ while (l.is_linked()) {
+ PrintF("@ %d ", l.pos());
+ Instr instr = instr_at(l.pos());
+ if ((instr & ~kImm24Mask) == 0) {
+ PrintF("value\n");
+ } else {
+ DCHECK_EQ(instr & 7 * B25, 5 * B25); // b, bl, or blx
+ Condition cond = Instruction::ConditionField(instr);
+ const char* b;
+ const char* c;
+ if (cond == kSpecialCondition) {
+ b = "blx";
+ c = "";
+ } else {
+ if ((instr & B24) != 0)
+ b = "bl";
+ else
+ b = "b";
+
+ switch (cond) {
+ case eq:
+ c = "eq";
+ break;
+ case ne:
+ c = "ne";
+ break;
+ case hs:
+ c = "hs";
+ break;
+ case lo:
+ c = "lo";
+ break;
+ case mi:
+ c = "mi";
+ break;
+ case pl:
+ c = "pl";
+ break;
+ case vs:
+ c = "vs";
+ break;
+ case vc:
+ c = "vc";
+ break;
+ case hi:
+ c = "hi";
+ break;
+ case ls:
+ c = "ls";
+ break;
+ case ge:
+ c = "ge";
+ break;
+ case lt:
+ c = "lt";
+ break;
+ case gt:
+ c = "gt";
+ break;
+ case le:
+ c = "le";
+ break;
+ case al:
+ c = "";
+ break;
+ default:
+ c = "";
+ UNREACHABLE();
+ }
+ }
+ PrintF("%s%s\n", b, c);
+ }
+ next(&l);
+ }
+ } else {
+ PrintF("label in inconsistent state (pos = %d)\n", L->pos_);
+ }
+}
+
+void Assembler::bind_to(Label* L, int pos) {
+ DCHECK(0 <= pos && pos <= pc_offset()); // must have a valid binding position
+ while (L->is_linked()) {
+ int fixup_pos = L->pos();
+ next(L); // call next before overwriting link with target at fixup_pos
+ target_at_put(fixup_pos, pos);
+ }
+ L->bind_to(pos);
+
+ // Keep track of the last bound label so we don't eliminate any instructions
+ // before a bound label.
+ if (pos > last_bound_pos_) last_bound_pos_ = pos;
+}
+
+void Assembler::bind(Label* L) {
+ DCHECK(!L->is_bound()); // label can only be bound once
+ bind_to(L, pc_offset());
+}
+
+void Assembler::next(Label* L) {
+ DCHECK(L->is_linked());
+ int link = target_at(L->pos());
+ if (link == L->pos()) {
+ // Branch target points to the same instruction. This is the end of the link
+ // chain.
+ L->Unuse();
+ } else {
+ DCHECK_GE(link, 0);
+ L->link_to(link);
+ }
+}
+
+namespace {
+
+// Low-level code emission routines depending on the addressing mode.
+// If this returns true then you have to use the rotate_imm and immed_8
+// that it returns, because it may have already changed the instruction
+// to match them!
+bool FitsShifter(uint32_t imm32, uint32_t* rotate_imm, uint32_t* immed_8,
+ Instr* instr) {
+ // imm32 must be unsigned.
+ for (int rot = 0; rot < 16; rot++) {
+ uint32_t imm8 = base::bits::RotateLeft32(imm32, 2 * rot);
+ if ((imm8 <= 0xFF)) {
+ *rotate_imm = rot;
+ *immed_8 = imm8;
+ return true;
+ }
+ }
+ // If the opcode is one with a complementary version and the complementary
+ // immediate fits, change the opcode.
+ if (instr != nullptr) {
+ if ((*instr & kMovMvnMask) == kMovMvnPattern) {
+ if (FitsShifter(~imm32, rotate_imm, immed_8, nullptr)) {
+ *instr ^= kMovMvnFlip;
+ return true;
+ } else if ((*instr & kMovLeaveCCMask) == kMovLeaveCCPattern) {
+ if (CpuFeatures::IsSupported(ARMv7)) {
+ if (imm32 < 0x10000) {
+ *instr ^= kMovwLeaveCCFlip;
+ *instr |= Assembler::EncodeMovwImmediate(imm32);
+ *rotate_imm = *immed_8 = 0; // Not used for movw.
+ return true;
+ }
+ }
+ }
+ } else if ((*instr & kCmpCmnMask) == kCmpCmnPattern) {
+ if (FitsShifter(-static_cast<int>(imm32), rotate_imm, immed_8, nullptr)) {
+ *instr ^= kCmpCmnFlip;
+ return true;
+ }
+ } else {
+ Instr alu_insn = (*instr & kALUMask);
+ if (alu_insn == ADD || alu_insn == SUB) {
+ if (FitsShifter(-static_cast<int>(imm32), rotate_imm, immed_8,
+ nullptr)) {
+ *instr ^= kAddSubFlip;
+ return true;
+ }
+ } else if (alu_insn == AND || alu_insn == BIC) {
+ if (FitsShifter(~imm32, rotate_imm, immed_8, nullptr)) {
+ *instr ^= kAndBicFlip;
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+}
+
+// We have to use the temporary register for things that can be relocated even
+// if they can be encoded in the ARM's 12 bits of immediate-offset instruction
+// space. There is no guarantee that the relocated location can be similarly
+// encoded.
+bool MustOutputRelocInfo(RelocInfo::Mode rmode, const Assembler* assembler) {
+ if (RelocInfo::IsOnlyForSerializer(rmode)) {
+ if (assembler->predictable_code_size()) return true;
+ return assembler->options().record_reloc_info_for_serialization;
+ } else if (RelocInfo::IsNone(rmode)) {
+ return false;
+ }
+ return true;
+}
+
+bool UseMovImmediateLoad(const Operand& x, const Assembler* assembler) {
+ DCHECK_NOT_NULL(assembler);
+ if (x.MustOutputRelocInfo(assembler)) {
+ // Prefer constant pool if data is likely to be patched.
+ return false;
+ } else {
+ // Otherwise, use immediate load if movw / movt is available.
+ return CpuFeatures::IsSupported(ARMv7);
+ }
+}
+
+} // namespace
+
+bool Operand::MustOutputRelocInfo(const Assembler* assembler) const {
+ return v8::internal::MustOutputRelocInfo(rmode_, assembler);
+}
+
+int Operand::InstructionsRequired(const Assembler* assembler,
+ Instr instr) const {
+ DCHECK_NOT_NULL(assembler);
+ if (rm_.is_valid()) return 1;
+ uint32_t dummy1, dummy2;
+ if (MustOutputRelocInfo(assembler) ||
+ !FitsShifter(immediate(), &dummy1, &dummy2, &instr)) {
+ // The immediate operand cannot be encoded as a shifter operand, or use of
+ // constant pool is required. First account for the instructions required
+ // for the constant pool or immediate load
+ int instructions;
+ if (UseMovImmediateLoad(*this, assembler)) {
+ DCHECK(CpuFeatures::IsSupported(ARMv7));
+ // A movw / movt immediate load.
+ instructions = 2;
+ } else {
+ // A small constant pool load.
+ instructions = 1;
+ }
+ if ((instr & ~kCondMask) != 13 * B21) { // mov, S not set
+ // For a mov or mvn instruction which doesn't set the condition
+ // code, the constant pool or immediate load is enough, otherwise we need
+ // to account for the actual instruction being requested.
+ instructions += 1;
+ }
+ return instructions;
+ } else {
+ // No use of constant pool and the immediate operand can be encoded as a
+ // shifter operand.
+ return 1;
+ }
+}
+
+void Assembler::Move32BitImmediate(Register rd, const Operand& x,
+ Condition cond) {
+ if (UseMovImmediateLoad(x, this)) {
+ CpuFeatureScope scope(this, ARMv7);
+ // UseMovImmediateLoad should return false when we need to output
+ // relocation info, since we prefer the constant pool for values that
+ // can be patched.
+ DCHECK(!x.MustOutputRelocInfo(this));
+ UseScratchRegisterScope temps(this);
+ // Re-use the destination register as a scratch if possible.
+ Register target = rd != pc && rd != sp ? rd : temps.Acquire();
+ uint32_t imm32 = static_cast<uint32_t>(x.immediate());
+ movw(target, imm32 & 0xFFFF, cond);
+ movt(target, imm32 >> 16, cond);
+ if (target.code() != rd.code()) {
+ mov(rd, target, LeaveCC, cond);
+ }
+ } else {
+ int32_t immediate;
+ if (x.IsHeapObjectRequest()) {
+ RequestHeapObject(x.heap_object_request());
+ immediate = 0;
+ } else {
+ immediate = x.immediate();
+ }
+ ConstantPoolAddEntry(pc_offset(), x.rmode_, immediate);
+ ldr_pcrel(rd, 0, cond);
+ }
+}
+
+void Assembler::AddrMode1(Instr instr, Register rd, Register rn,
+ const Operand& x) {
+ CheckBuffer();
+ uint32_t opcode = instr & kOpCodeMask;
+ bool set_flags = (instr & S) != 0;
+ DCHECK((opcode == ADC) || (opcode == ADD) || (opcode == AND) ||
+ (opcode == BIC) || (opcode == EOR) || (opcode == ORR) ||
+ (opcode == RSB) || (opcode == RSC) || (opcode == SBC) ||
+ (opcode == SUB) || (opcode == CMN) || (opcode == CMP) ||
+ (opcode == TEQ) || (opcode == TST) || (opcode == MOV) ||
+ (opcode == MVN));
+ // For comparison instructions, rd is not defined.
+ DCHECK(rd.is_valid() || (opcode == CMN) || (opcode == CMP) ||
+ (opcode == TEQ) || (opcode == TST));
+ // For move instructions, rn is not defined.
+ DCHECK(rn.is_valid() || (opcode == MOV) || (opcode == MVN));
+ DCHECK(rd.is_valid() || rn.is_valid());
+ DCHECK_EQ(instr & ~(kCondMask | kOpCodeMask | S), 0);
+ if (!AddrMode1TryEncodeOperand(&instr, x)) {
+ DCHECK(x.IsImmediate());
+ // Upon failure to encode, the opcode should not have changed.
+ DCHECK(opcode == (instr & kOpCodeMask));
+ UseScratchRegisterScope temps(this);
+ Condition cond = Instruction::ConditionField(instr);
+ if ((opcode == MOV) && !set_flags) {
+ // Generate a sequence of mov instructions or a load from the constant
+ // pool only for a MOV instruction which does not set the flags.
+ DCHECK(!rn.is_valid());
+ Move32BitImmediate(rd, x, cond);
+ } else if ((opcode == ADD) && !set_flags && (rd == rn) &&
+ !temps.CanAcquire()) {
+ // Split the operation into a sequence of additions if we cannot use a
+ // scratch register. In this case, we cannot re-use rn and the assembler
+ // does not have any scratch registers to spare.
+ uint32_t imm = x.immediate();
+ do {
+ // The immediate encoding format is composed of 8 bits of data and 4
+ // bits encoding a rotation. Each of the 16 possible rotations accounts
+ // for a rotation by an even number.
+ // 4 bits -> 16 rotations possible
+ // -> 16 rotations of 2 bits each fits in a 32-bit value.
+ // This means that finding the even number of trailing zeroes of the
+ // immediate allows us to more efficiently split it:
+ int trailing_zeroes = base::bits::CountTrailingZeros(imm) & ~1u;
+ uint32_t mask = (0xFF << trailing_zeroes);
+ add(rd, rd, Operand(imm & mask), LeaveCC, cond);
+ imm = imm & ~mask;
+ } while (!ImmediateFitsAddrMode1Instruction(imm));
+ add(rd, rd, Operand(imm), LeaveCC, cond);
+ } else {
+ // The immediate operand cannot be encoded as a shifter operand, so load
+ // it first to a scratch register and change the original instruction to
+ // use it.
+ // Re-use the destination register if possible.
+ Register scratch = (rd.is_valid() && rd != rn && rd != pc && rd != sp)
+ ? rd
+ : temps.Acquire();
+ mov(scratch, x, LeaveCC, cond);
+ AddrMode1(instr, rd, rn, Operand(scratch));
+ }
+ return;
+ }
+ if (!rd.is_valid()) {
+ // Emit a comparison instruction.
+ emit(instr | rn.code() * B16);
+ } else if (!rn.is_valid()) {
+ // Emit a move instruction. If the operand is a register-shifted register,
+ // then prevent the destination from being PC as this is unpredictable.
+ DCHECK(!x.IsRegisterShiftedRegister() || rd != pc);
+ emit(instr | rd.code() * B12);
+ } else {
+ emit(instr | rn.code() * B16 | rd.code() * B12);
+ }
+ if (rn == pc || x.rm_ == pc) {
+ // Block constant pool emission for one instruction after reading pc.
+ BlockConstPoolFor(1);
+ }
+}
+
+bool Assembler::AddrMode1TryEncodeOperand(Instr* instr, const Operand& x) {
+ if (x.IsImmediate()) {
+ // Immediate.
+ uint32_t rotate_imm;
+ uint32_t immed_8;
+ if (x.MustOutputRelocInfo(this) ||
+ !FitsShifter(x.immediate(), &rotate_imm, &immed_8, instr)) {
+ // Let the caller handle generating multiple instructions.
+ return false;
+ }
+ *instr |= I | rotate_imm * B8 | immed_8;
+ } else if (x.IsImmediateShiftedRegister()) {
+ *instr |= x.shift_imm_ * B7 | x.shift_op_ | x.rm_.code();
+ } else {
+ DCHECK(x.IsRegisterShiftedRegister());
+ // It is unpredictable to use the PC in this case.
+ DCHECK(x.rm_ != pc && x.rs_ != pc);
+ *instr |= x.rs_.code() * B8 | x.shift_op_ | B4 | x.rm_.code();
+ }
+
+ return true;
+}
+
+void Assembler::AddrMode2(Instr instr, Register rd, const MemOperand& x) {
+ DCHECK((instr & ~(kCondMask | B | L)) == B26);
+ // This method does not handle pc-relative addresses. ldr_pcrel() should be
+ // used instead.
+ DCHECK(x.rn_ != pc);
+ int am = x.am_;
+ if (!x.rm_.is_valid()) {
+ // Immediate offset.
+ int offset_12 = x.offset_;
+ if (offset_12 < 0) {
+ offset_12 = -offset_12;
+ am ^= U;
+ }
+ if (!is_uint12(offset_12)) {
+ // Immediate offset cannot be encoded, load it first to a scratch
+ // register.
+ UseScratchRegisterScope temps(this);
+ // Allow re-using rd for load instructions if possible.
+ bool is_load = (instr & L) == L;
+ Register scratch = (is_load && rd != x.rn_ && rd != pc && rd != sp)
+ ? rd
+ : temps.Acquire();
+ mov(scratch, Operand(x.offset_), LeaveCC,
+ Instruction::ConditionField(instr));
+ AddrMode2(instr, rd, MemOperand(x.rn_, scratch, x.am_));
+ return;
+ }
+ DCHECK_GE(offset_12, 0); // no masking needed
+ instr |= offset_12;
+ } else {
+ // Register offset (shift_imm_ and shift_op_ are 0) or scaled
+ // register offset the constructors make sure than both shift_imm_
+ // and shift_op_ are initialized.
+ DCHECK(x.rm_ != pc);
+ instr |= B25 | x.shift_imm_ * B7 | x.shift_op_ | x.rm_.code();
+ }
+ DCHECK((am & (P | W)) == P || x.rn_ != pc); // no pc base with writeback
+ emit(instr | am | x.rn_.code() * B16 | rd.code() * B12);
+}
+
+void Assembler::AddrMode3(Instr instr, Register rd, const MemOperand& x) {
+ DCHECK((instr & ~(kCondMask | L | S6 | H)) == (B4 | B7));
+ DCHECK(x.rn_.is_valid());
+ // This method does not handle pc-relative addresses. ldr_pcrel() should be
+ // used instead.
+ DCHECK(x.rn_ != pc);
+ int am = x.am_;
+ bool is_load = (instr & L) == L;
+ if (!x.rm_.is_valid()) {
+ // Immediate offset.
+ int offset_8 = x.offset_;
+ if (offset_8 < 0) {
+ offset_8 = -offset_8;
+ am ^= U;
+ }
+ if (!is_uint8(offset_8)) {
+ // Immediate offset cannot be encoded, load it first to a scratch
+ // register.
+ UseScratchRegisterScope temps(this);
+ // Allow re-using rd for load instructions if possible.
+ Register scratch = (is_load && rd != x.rn_ && rd != pc && rd != sp)
+ ? rd
+ : temps.Acquire();
+ mov(scratch, Operand(x.offset_), LeaveCC,
+ Instruction::ConditionField(instr));
+ AddrMode3(instr, rd, MemOperand(x.rn_, scratch, x.am_));
+ return;
+ }
+ DCHECK_GE(offset_8, 0); // no masking needed
+ instr |= B | (offset_8 >> 4) * B8 | (offset_8 & 0xF);
+ } else if (x.shift_imm_ != 0) {
+ // Scaled register offsets are not supported, compute the offset separately
+ // to a scratch register.
+ UseScratchRegisterScope temps(this);
+ // Allow re-using rd for load instructions if possible.
+ Register scratch =
+ (is_load && rd != x.rn_ && rd != pc && rd != sp) ? rd : temps.Acquire();
+ mov(scratch, Operand(x.rm_, x.shift_op_, x.shift_imm_), LeaveCC,
+ Instruction::ConditionField(instr));
+ AddrMode3(instr, rd, MemOperand(x.rn_, scratch, x.am_));
+ return;
+ } else {
+ // Register offset.
+ DCHECK((am & (P | W)) == P || x.rm_ != pc); // no pc index with writeback
+ instr |= x.rm_.code();
+ }
+ DCHECK((am & (P | W)) == P || x.rn_ != pc); // no pc base with writeback
+ emit(instr | am | x.rn_.code() * B16 | rd.code() * B12);
+}
+
+void Assembler::AddrMode4(Instr instr, Register rn, RegList rl) {
+ DCHECK((instr & ~(kCondMask | P | U | W | L)) == B27);
+ DCHECK_NE(rl, 0);
+ DCHECK(rn != pc);
+ emit(instr | rn.code() * B16 | rl);
+}
+
+void Assembler::AddrMode5(Instr instr, CRegister crd, const MemOperand& x) {
+ // Unindexed addressing is not encoded by this function.
+ DCHECK_EQ((B27 | B26),
+ (instr & ~(kCondMask | kCoprocessorMask | P | U | N | W | L)));
+ DCHECK(x.rn_.is_valid() && !x.rm_.is_valid());
+ int am = x.am_;
+ int offset_8 = x.offset_;
+ DCHECK_EQ(offset_8 & 3, 0); // offset must be an aligned word offset
+ offset_8 >>= 2;
+ if (offset_8 < 0) {
+ offset_8 = -offset_8;
+ am ^= U;
+ }
+ DCHECK(is_uint8(offset_8)); // unsigned word offset must fit in a byte
+ DCHECK((am & (P | W)) == P || x.rn_ != pc); // no pc base with writeback
+
+ // Post-indexed addressing requires W == 1; different than in AddrMode2/3.
+ if ((am & P) == 0) am |= W;
+
+ DCHECK_GE(offset_8, 0); // no masking needed
+ emit(instr | am | x.rn_.code() * B16 | crd.code() * B12 | offset_8);
+}
+
+int Assembler::branch_offset(Label* L) {
+ int target_pos;
+ if (L->is_bound()) {
+ target_pos = L->pos();
+ } else {
+ if (L->is_linked()) {
+ // Point to previous instruction that uses the link.
+ target_pos = L->pos();
+ } else {
+ // First entry of the link chain points to itself.
+ target_pos = pc_offset();
+ }
+ L->link_to(pc_offset());
+ }
+
+ // Block the emission of the constant pool, since the branch instruction must
+ // be emitted at the pc offset recorded by the label.
+ if (!is_const_pool_blocked()) BlockConstPoolFor(1);
+
+ return target_pos - (pc_offset() + Instruction::kPcLoadDelta);
+}
+
+// Branch instructions.
+void Assembler::b(int branch_offset, Condition cond, RelocInfo::Mode rmode) {
+ if (!RelocInfo::IsNone(rmode)) RecordRelocInfo(rmode);
+ DCHECK_EQ(branch_offset & 3, 0);
+ int imm24 = branch_offset >> 2;
+ const bool b_imm_check = is_int24(imm24);
+ CHECK(b_imm_check);
+ emit(cond | B27 | B25 | (imm24 & kImm24Mask));
+
+ if (cond == al) {
+ // Dead code is a good location to emit the constant pool.
+ CheckConstPool(false, false);
+ }
+}
+
+void Assembler::bl(int branch_offset, Condition cond, RelocInfo::Mode rmode) {
+ if (!RelocInfo::IsNone(rmode)) RecordRelocInfo(rmode);
+ DCHECK_EQ(branch_offset & 3, 0);
+ int imm24 = branch_offset >> 2;
+ const bool bl_imm_check = is_int24(imm24);
+ CHECK(bl_imm_check);
+ emit(cond | B27 | B25 | B24 | (imm24 & kImm24Mask));
+}
+
+void Assembler::blx(int branch_offset) {
+ DCHECK_EQ(branch_offset & 1, 0);
+ int h = ((branch_offset & 2) >> 1) * B24;
+ int imm24 = branch_offset >> 2;
+ const bool blx_imm_check = is_int24(imm24);
+ CHECK(blx_imm_check);
+ emit(kSpecialCondition | B27 | B25 | h | (imm24 & kImm24Mask));
+}
+
+void Assembler::blx(Register target, Condition cond) {
+ DCHECK(target != pc);
+ emit(cond | B24 | B21 | 15 * B16 | 15 * B12 | 15 * B8 | BLX | target.code());
+}
+
+void Assembler::bx(Register target, Condition cond) {
+ DCHECK(target != pc); // use of pc is actually allowed, but discouraged
+ emit(cond | B24 | B21 | 15 * B16 | 15 * B12 | 15 * B8 | BX | target.code());
+}
+
+void Assembler::b(Label* L, Condition cond) {
+ CheckBuffer();
+ b(branch_offset(L), cond);
+}
+
+void Assembler::bl(Label* L, Condition cond) {
+ CheckBuffer();
+ bl(branch_offset(L), cond);
+}
+
+void Assembler::blx(Label* L) {
+ CheckBuffer();
+ blx(branch_offset(L));
+}
+
+// Data-processing instructions.
+
+void Assembler::and_(Register dst, Register src1, const Operand& src2, SBit s,
+ Condition cond) {
+ AddrMode1(cond | AND | s, dst, src1, src2);
+}
+
+void Assembler::and_(Register dst, Register src1, Register src2, SBit s,
+ Condition cond) {
+ and_(dst, src1, Operand(src2), s, cond);
+}
+
+void Assembler::eor(Register dst, Register src1, const Operand& src2, SBit s,
+ Condition cond) {
+ AddrMode1(cond | EOR | s, dst, src1, src2);
+}
+
+void Assembler::eor(Register dst, Register src1, Register src2, SBit s,
+ Condition cond) {
+ AddrMode1(cond | EOR | s, dst, src1, Operand(src2));
+}
+
+void Assembler::sub(Register dst, Register src1, const Operand& src2, SBit s,
+ Condition cond) {
+ AddrMode1(cond | SUB | s, dst, src1, src2);
+}
+
+void Assembler::sub(Register dst, Register src1, Register src2, SBit s,
+ Condition cond) {
+ sub(dst, src1, Operand(src2), s, cond);
+}
+
+void Assembler::rsb(Register dst, Register src1, const Operand& src2, SBit s,
+ Condition cond) {
+ AddrMode1(cond | RSB | s, dst, src1, src2);
+}
+
+void Assembler::add(Register dst, Register src1, const Operand& src2, SBit s,
+ Condition cond) {
+ AddrMode1(cond | ADD | s, dst, src1, src2);
+}
+
+void Assembler::add(Register dst, Register src1, Register src2, SBit s,
+ Condition cond) {
+ add(dst, src1, Operand(src2), s, cond);
+}
+
+void Assembler::adc(Register dst, Register src1, const Operand& src2, SBit s,
+ Condition cond) {
+ AddrMode1(cond | ADC | s, dst, src1, src2);
+}
+
+void Assembler::sbc(Register dst, Register src1, const Operand& src2, SBit s,
+ Condition cond) {
+ AddrMode1(cond | SBC | s, dst, src1, src2);
+}
+
+void Assembler::rsc(Register dst, Register src1, const Operand& src2, SBit s,
+ Condition cond) {
+ AddrMode1(cond | RSC | s, dst, src1, src2);
+}
+
+void Assembler::tst(Register src1, const Operand& src2, Condition cond) {
+ AddrMode1(cond | TST | S, no_reg, src1, src2);
+}
+
+void Assembler::tst(Register src1, Register src2, Condition cond) {
+ tst(src1, Operand(src2), cond);
+}
+
+void Assembler::teq(Register src1, const Operand& src2, Condition cond) {
+ AddrMode1(cond | TEQ | S, no_reg, src1, src2);
+}
+
+void Assembler::cmp(Register src1, const Operand& src2, Condition cond) {
+ AddrMode1(cond | CMP | S, no_reg, src1, src2);
+}
+
+void Assembler::cmp(Register src1, Register src2, Condition cond) {
+ cmp(src1, Operand(src2), cond);
+}
+
+void Assembler::cmp_raw_immediate(Register src, int raw_immediate,
+ Condition cond) {
+ DCHECK(is_uint12(raw_immediate));
+ emit(cond | I | CMP | S | src.code() << 16 | raw_immediate);
+}
+
+void Assembler::cmn(Register src1, const Operand& src2, Condition cond) {
+ AddrMode1(cond | CMN | S, no_reg, src1, src2);
+}
+
+void Assembler::orr(Register dst, Register src1, const Operand& src2, SBit s,
+ Condition cond) {
+ AddrMode1(cond | ORR | s, dst, src1, src2);
+}
+
+void Assembler::orr(Register dst, Register src1, Register src2, SBit s,
+ Condition cond) {
+ orr(dst, src1, Operand(src2), s, cond);
+}
+
+void Assembler::mov(Register dst, const Operand& src, SBit s, Condition cond) {
+ // Don't allow nop instructions in the form mov rn, rn to be generated using
+ // the mov instruction. They must be generated using nop(int/NopMarkerTypes).
+ DCHECK(!(src.IsRegister() && src.rm() == dst && s == LeaveCC && cond == al));
+ AddrMode1(cond | MOV | s, dst, no_reg, src);
+}
+
+void Assembler::mov(Register dst, Register src, SBit s, Condition cond) {
+ mov(dst, Operand(src), s, cond);
+}
+
+void Assembler::mov_label_offset(Register dst, Label* label) {
+ if (label->is_bound()) {
+ mov(dst, Operand(label->pos() + (Code::kHeaderSize - kHeapObjectTag)));
+ } else {
+ // Emit the link to the label in the code stream followed by extra nop
+ // instructions.
+ // If the label is not linked, then start a new link chain by linking it to
+ // itself, emitting pc_offset().
+ int link = label->is_linked() ? label->pos() : pc_offset();
+ label->link_to(pc_offset());
+
+ // When the label is bound, these instructions will be patched with a
+ // sequence of movw/movt or mov/orr/orr instructions. They will load the
+ // destination register with the position of the label from the beginning
+ // of the code.
+ //
+ // The link will be extracted from the first instruction and the destination
+ // register from the second.
+ // For ARMv7:
+ // link
+ // mov dst, dst
+ // For ARMv6:
+ // link
+ // mov dst, dst
+ // mov dst, dst
+ //
+ // When the label gets bound: target_at extracts the link and target_at_put
+ // patches the instructions.
+ CHECK(is_uint24(link));
+ BlockConstPoolScope block_const_pool(this);
+ emit(link);
+ nop(dst.code());
+ if (!CpuFeatures::IsSupported(ARMv7)) {
+ nop(dst.code());
+ }
+ }
+}
+
+void Assembler::movw(Register reg, uint32_t immediate, Condition cond) {
+ DCHECK(IsEnabled(ARMv7));
+ emit(cond | 0x30 * B20 | reg.code() * B12 | EncodeMovwImmediate(immediate));
+}
+
+void Assembler::movt(Register reg, uint32_t immediate, Condition cond) {
+ DCHECK(IsEnabled(ARMv7));
+ emit(cond | 0x34 * B20 | reg.code() * B12 | EncodeMovwImmediate(immediate));
+}
+
+void Assembler::bic(Register dst, Register src1, const Operand& src2, SBit s,
+ Condition cond) {
+ AddrMode1(cond | BIC | s, dst, src1, src2);
+}
+
+void Assembler::mvn(Register dst, const Operand& src, SBit s, Condition cond) {
+ AddrMode1(cond | MVN | s, dst, no_reg, src);
+}
+
+void Assembler::asr(Register dst, Register src1, const Operand& src2, SBit s,
+ Condition cond) {
+ if (src2.IsRegister()) {
+ mov(dst, Operand(src1, ASR, src2.rm()), s, cond);
+ } else {
+ mov(dst, Operand(src1, ASR, src2.immediate()), s, cond);
+ }
+}
+
+void Assembler::lsl(Register dst, Register src1, const Operand& src2, SBit s,
+ Condition cond) {
+ if (src2.IsRegister()) {
+ mov(dst, Operand(src1, LSL, src2.rm()), s, cond);
+ } else {
+ mov(dst, Operand(src1, LSL, src2.immediate()), s, cond);
+ }
+}
+
+void Assembler::lsr(Register dst, Register src1, const Operand& src2, SBit s,
+ Condition cond) {
+ if (src2.IsRegister()) {
+ mov(dst, Operand(src1, LSR, src2.rm()), s, cond);
+ } else {
+ mov(dst, Operand(src1, LSR, src2.immediate()), s, cond);
+ }
+}
+
+// Multiply instructions.
+void Assembler::mla(Register dst, Register src1, Register src2, Register srcA,
+ SBit s, Condition cond) {
+ DCHECK(dst != pc && src1 != pc && src2 != pc && srcA != pc);
+ emit(cond | A | s | dst.code() * B16 | srcA.code() * B12 | src2.code() * B8 |
+ B7 | B4 | src1.code());
+}
+
+void Assembler::mls(Register dst, Register src1, Register src2, Register srcA,
+ Condition cond) {
+ DCHECK(dst != pc && src1 != pc && src2 != pc && srcA != pc);
+ DCHECK(IsEnabled(ARMv7));
+ emit(cond | B22 | B21 | dst.code() * B16 | srcA.code() * B12 |
+ src2.code() * B8 | B7 | B4 | src1.code());
+}
+
+void Assembler::sdiv(Register dst, Register src1, Register src2,
+ Condition cond) {
+ DCHECK(dst != pc && src1 != pc && src2 != pc);
+ DCHECK(IsEnabled(SUDIV));
+ emit(cond | B26 | B25 | B24 | B20 | dst.code() * B16 | 0xF * B12 |
+ src2.code() * B8 | B4 | src1.code());
+}
+
+void Assembler::udiv(Register dst, Register src1, Register src2,
+ Condition cond) {
+ DCHECK(dst != pc && src1 != pc && src2 != pc);
+ DCHECK(IsEnabled(SUDIV));
+ emit(cond | B26 | B25 | B24 | B21 | B20 | dst.code() * B16 | 0xF * B12 |
+ src2.code() * B8 | B4 | src1.code());
+}
+
+void Assembler::mul(Register dst, Register src1, Register src2, SBit s,
+ Condition cond) {
+ DCHECK(dst != pc && src1 != pc && src2 != pc);
+ // dst goes in bits 16-19 for this instruction!
+ emit(cond | s | dst.code() * B16 | src2.code() * B8 | B7 | B4 | src1.code());
+}
+
+void Assembler::smmla(Register dst, Register src1, Register src2, Register srcA,
+ Condition cond) {
+ DCHECK(dst != pc && src1 != pc && src2 != pc && srcA != pc);
+ emit(cond | B26 | B25 | B24 | B22 | B20 | dst.code() * B16 |
+ srcA.code() * B12 | src2.code() * B8 | B4 | src1.code());
+}
+
+void Assembler::smmul(Register dst, Register src1, Register src2,
+ Condition cond) {
+ DCHECK(dst != pc && src1 != pc && src2 != pc);
+ emit(cond | B26 | B25 | B24 | B22 | B20 | dst.code() * B16 | 0xF * B12 |
+ src2.code() * B8 | B4 | src1.code());
+}
+
+void Assembler::smlal(Register dstL, Register dstH, Register src1,
+ Register src2, SBit s, Condition cond) {
+ DCHECK(dstL != pc && dstH != pc && src1 != pc && src2 != pc);
+ DCHECK(dstL != dstH);
+ emit(cond | B23 | B22 | A | s | dstH.code() * B16 | dstL.code() * B12 |
+ src2.code() * B8 | B7 | B4 | src1.code());
+}
+
+void Assembler::smull(Register dstL, Register dstH, Register src1,
+ Register src2, SBit s, Condition cond) {
+ DCHECK(dstL != pc && dstH != pc && src1 != pc && src2 != pc);
+ DCHECK(dstL != dstH);
+ emit(cond | B23 | B22 | s | dstH.code() * B16 | dstL.code() * B12 |
+ src2.code() * B8 | B7 | B4 | src1.code());
+}
+
+void Assembler::umlal(Register dstL, Register dstH, Register src1,
+ Register src2, SBit s, Condition cond) {
+ DCHECK(dstL != pc && dstH != pc && src1 != pc && src2 != pc);
+ DCHECK(dstL != dstH);
+ emit(cond | B23 | A | s | dstH.code() * B16 | dstL.code() * B12 |
+ src2.code() * B8 | B7 | B4 | src1.code());
+}
+
+void Assembler::umull(Register dstL, Register dstH, Register src1,
+ Register src2, SBit s, Condition cond) {
+ DCHECK(dstL != pc && dstH != pc && src1 != pc && src2 != pc);
+ DCHECK(dstL != dstH);
+ emit(cond | B23 | s | dstH.code() * B16 | dstL.code() * B12 |
+ src2.code() * B8 | B7 | B4 | src1.code());
+}
+
+// Miscellaneous arithmetic instructions.
+void Assembler::clz(Register dst, Register src, Condition cond) {
+ DCHECK(dst != pc && src != pc);
+ emit(cond | B24 | B22 | B21 | 15 * B16 | dst.code() * B12 | 15 * B8 | CLZ |
+ src.code());
+}
+
+// Saturating instructions.
+
+// Unsigned saturate.
+void Assembler::usat(Register dst, int satpos, const Operand& src,
+ Condition cond) {
+ DCHECK(dst != pc && src.rm_ != pc);
+ DCHECK((satpos >= 0) && (satpos <= 31));
+ DCHECK(src.IsImmediateShiftedRegister());
+ DCHECK((src.shift_op_ == ASR) || (src.shift_op_ == LSL));
+
+ int sh = 0;
+ if (src.shift_op_ == ASR) {
+ sh = 1;
+ }
+
+ emit(cond | 0x6 * B24 | 0xE * B20 | satpos * B16 | dst.code() * B12 |
+ src.shift_imm_ * B7 | sh * B6 | 0x1 * B4 | src.rm_.code());
+}
+
+// Bitfield manipulation instructions.
+
+// Unsigned bit field extract.
+// Extracts #width adjacent bits from position #lsb in a register, and
+// writes them to the low bits of a destination register.
+// ubfx dst, src, #lsb, #width
+void Assembler::ubfx(Register dst, Register src, int lsb, int width,
+ Condition cond) {
+ DCHECK(IsEnabled(ARMv7));
+ DCHECK(dst != pc && src != pc);
+ DCHECK((lsb >= 0) && (lsb <= 31));
+ DCHECK((width >= 1) && (width <= (32 - lsb)));
+ emit(cond | 0xF * B23 | B22 | B21 | (width - 1) * B16 | dst.code() * B12 |
+ lsb * B7 | B6 | B4 | src.code());
+}
+
+// Signed bit field extract.
+// Extracts #width adjacent bits from position #lsb in a register, and
+// writes them to the low bits of a destination register. The extracted
+// value is sign extended to fill the destination register.
+// sbfx dst, src, #lsb, #width
+void Assembler::sbfx(Register dst, Register src, int lsb, int width,
+ Condition cond) {
+ DCHECK(IsEnabled(ARMv7));
+ DCHECK(dst != pc && src != pc);
+ DCHECK((lsb >= 0) && (lsb <= 31));
+ DCHECK((width >= 1) && (width <= (32 - lsb)));
+ emit(cond | 0xF * B23 | B21 | (width - 1) * B16 | dst.code() * B12 |
+ lsb * B7 | B6 | B4 | src.code());
+}
+
+// Bit field clear.
+// Sets #width adjacent bits at position #lsb in the destination register
+// to zero, preserving the value of the other bits.
+// bfc dst, #lsb, #width
+void Assembler::bfc(Register dst, int lsb, int width, Condition cond) {
+ DCHECK(IsEnabled(ARMv7));
+ DCHECK(dst != pc);
+ DCHECK((lsb >= 0) && (lsb <= 31));
+ DCHECK((width >= 1) && (width <= (32 - lsb)));
+ int msb = lsb + width - 1;
+ emit(cond | 0x1F * B22 | msb * B16 | dst.code() * B12 | lsb * B7 | B4 | 0xF);
+}
+
+// Bit field insert.
+// Inserts #width adjacent bits from the low bits of the source register
+// into position #lsb of the destination register.
+// bfi dst, src, #lsb, #width
+void Assembler::bfi(Register dst, Register src, int lsb, int width,
+ Condition cond) {
+ DCHECK(IsEnabled(ARMv7));
+ DCHECK(dst != pc && src != pc);
+ DCHECK((lsb >= 0) && (lsb <= 31));
+ DCHECK((width >= 1) && (width <= (32 - lsb)));
+ int msb = lsb + width - 1;
+ emit(cond | 0x1F * B22 | msb * B16 | dst.code() * B12 | lsb * B7 | B4 |
+ src.code());
+}
+
+void Assembler::pkhbt(Register dst, Register src1, const Operand& src2,
+ Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.125.
+ // cond(31-28) | 01101000(27-20) | Rn(19-16) |
+ // Rd(15-12) | imm5(11-7) | 0(6) | 01(5-4) | Rm(3-0)
+ DCHECK(dst != pc);
+ DCHECK(src1 != pc);
+ DCHECK(src2.IsImmediateShiftedRegister());
+ DCHECK(src2.rm() != pc);
+ DCHECK((src2.shift_imm_ >= 0) && (src2.shift_imm_ <= 31));
+ DCHECK(src2.shift_op() == LSL);
+ emit(cond | 0x68 * B20 | src1.code() * B16 | dst.code() * B12 |
+ src2.shift_imm_ * B7 | B4 | src2.rm().code());
+}
+
+void Assembler::pkhtb(Register dst, Register src1, const Operand& src2,
+ Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.125.
+ // cond(31-28) | 01101000(27-20) | Rn(19-16) |
+ // Rd(15-12) | imm5(11-7) | 1(6) | 01(5-4) | Rm(3-0)
+ DCHECK(dst != pc);
+ DCHECK(src1 != pc);
+ DCHECK(src2.IsImmediateShiftedRegister());
+ DCHECK(src2.rm() != pc);
+ DCHECK((src2.shift_imm_ >= 1) && (src2.shift_imm_ <= 32));
+ DCHECK(src2.shift_op() == ASR);
+ int asr = (src2.shift_imm_ == 32) ? 0 : src2.shift_imm_;
+ emit(cond | 0x68 * B20 | src1.code() * B16 | dst.code() * B12 | asr * B7 |
+ B6 | B4 | src2.rm().code());
+}
+
+void Assembler::sxtb(Register dst, Register src, int rotate, Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.233.
+ // cond(31-28) | 01101010(27-20) | 1111(19-16) |
+ // Rd(15-12) | rotate(11-10) | 00(9-8)| 0111(7-4) | Rm(3-0)
+ DCHECK(dst != pc);
+ DCHECK(src != pc);
+ DCHECK(rotate == 0 || rotate == 8 || rotate == 16 || rotate == 24);
+ emit(cond | 0x6A * B20 | 0xF * B16 | dst.code() * B12 |
+ ((rotate >> 1) & 0xC) * B8 | 7 * B4 | src.code());
+}
+
+void Assembler::sxtab(Register dst, Register src1, Register src2, int rotate,
+ Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.233.
+ // cond(31-28) | 01101010(27-20) | Rn(19-16) |
+ // Rd(15-12) | rotate(11-10) | 00(9-8)| 0111(7-4) | Rm(3-0)
+ DCHECK(dst != pc);
+ DCHECK(src1 != pc);
+ DCHECK(src2 != pc);
+ DCHECK(rotate == 0 || rotate == 8 || rotate == 16 || rotate == 24);
+ emit(cond | 0x6A * B20 | src1.code() * B16 | dst.code() * B12 |
+ ((rotate >> 1) & 0xC) * B8 | 7 * B4 | src2.code());
+}
+
+void Assembler::sxth(Register dst, Register src, int rotate, Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.235.
+ // cond(31-28) | 01101011(27-20) | 1111(19-16) |
+ // Rd(15-12) | rotate(11-10) | 00(9-8)| 0111(7-4) | Rm(3-0)
+ DCHECK(dst != pc);
+ DCHECK(src != pc);
+ DCHECK(rotate == 0 || rotate == 8 || rotate == 16 || rotate == 24);
+ emit(cond | 0x6B * B20 | 0xF * B16 | dst.code() * B12 |
+ ((rotate >> 1) & 0xC) * B8 | 7 * B4 | src.code());
+}
+
+void Assembler::sxtah(Register dst, Register src1, Register src2, int rotate,
+ Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.235.
+ // cond(31-28) | 01101011(27-20) | Rn(19-16) |
+ // Rd(15-12) | rotate(11-10) | 00(9-8)| 0111(7-4) | Rm(3-0)
+ DCHECK(dst != pc);
+ DCHECK(src1 != pc);
+ DCHECK(src2 != pc);
+ DCHECK(rotate == 0 || rotate == 8 || rotate == 16 || rotate == 24);
+ emit(cond | 0x6B * B20 | src1.code() * B16 | dst.code() * B12 |
+ ((rotate >> 1) & 0xC) * B8 | 7 * B4 | src2.code());
+}
+
+void Assembler::uxtb(Register dst, Register src, int rotate, Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.274.
+ // cond(31-28) | 01101110(27-20) | 1111(19-16) |
+ // Rd(15-12) | rotate(11-10) | 00(9-8)| 0111(7-4) | Rm(3-0)
+ DCHECK(dst != pc);
+ DCHECK(src != pc);
+ DCHECK(rotate == 0 || rotate == 8 || rotate == 16 || rotate == 24);
+ emit(cond | 0x6E * B20 | 0xF * B16 | dst.code() * B12 |
+ ((rotate >> 1) & 0xC) * B8 | 7 * B4 | src.code());
+}
+
+void Assembler::uxtab(Register dst, Register src1, Register src2, int rotate,
+ Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.271.
+ // cond(31-28) | 01101110(27-20) | Rn(19-16) |
+ // Rd(15-12) | rotate(11-10) | 00(9-8)| 0111(7-4) | Rm(3-0)
+ DCHECK(dst != pc);
+ DCHECK(src1 != pc);
+ DCHECK(src2 != pc);
+ DCHECK(rotate == 0 || rotate == 8 || rotate == 16 || rotate == 24);
+ emit(cond | 0x6E * B20 | src1.code() * B16 | dst.code() * B12 |
+ ((rotate >> 1) & 0xC) * B8 | 7 * B4 | src2.code());
+}
+
+void Assembler::uxtb16(Register dst, Register src, int rotate, Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.275.
+ // cond(31-28) | 01101100(27-20) | 1111(19-16) |
+ // Rd(15-12) | rotate(11-10) | 00(9-8)| 0111(7-4) | Rm(3-0)
+ DCHECK(dst != pc);
+ DCHECK(src != pc);
+ DCHECK(rotate == 0 || rotate == 8 || rotate == 16 || rotate == 24);
+ emit(cond | 0x6C * B20 | 0xF * B16 | dst.code() * B12 |
+ ((rotate >> 1) & 0xC) * B8 | 7 * B4 | src.code());
+}
+
+void Assembler::uxth(Register dst, Register src, int rotate, Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.276.
+ // cond(31-28) | 01101111(27-20) | 1111(19-16) |
+ // Rd(15-12) | rotate(11-10) | 00(9-8)| 0111(7-4) | Rm(3-0)
+ DCHECK(dst != pc);
+ DCHECK(src != pc);
+ DCHECK(rotate == 0 || rotate == 8 || rotate == 16 || rotate == 24);
+ emit(cond | 0x6F * B20 | 0xF * B16 | dst.code() * B12 |
+ ((rotate >> 1) & 0xC) * B8 | 7 * B4 | src.code());
+}
+
+void Assembler::uxtah(Register dst, Register src1, Register src2, int rotate,
+ Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.273.
+ // cond(31-28) | 01101111(27-20) | Rn(19-16) |
+ // Rd(15-12) | rotate(11-10) | 00(9-8)| 0111(7-4) | Rm(3-0)
+ DCHECK(dst != pc);
+ DCHECK(src1 != pc);
+ DCHECK(src2 != pc);
+ DCHECK(rotate == 0 || rotate == 8 || rotate == 16 || rotate == 24);
+ emit(cond | 0x6F * B20 | src1.code() * B16 | dst.code() * B12 |
+ ((rotate >> 1) & 0xC) * B8 | 7 * B4 | src2.code());
+}
+
+void Assembler::rbit(Register dst, Register src, Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.144.
+ // cond(31-28) | 011011111111(27-16) | Rd(15-12) | 11110011(11-4) | Rm(3-0)
+ DCHECK(IsEnabled(ARMv7));
+ DCHECK(dst != pc);
+ DCHECK(src != pc);
+ emit(cond | 0x6FF * B16 | dst.code() * B12 | 0xF3 * B4 | src.code());
+}
+
+void Assembler::rev(Register dst, Register src, Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.144.
+ // cond(31-28) | 011010111111(27-16) | Rd(15-12) | 11110011(11-4) | Rm(3-0)
+ DCHECK(dst != pc);
+ DCHECK(src != pc);
+ emit(cond | 0x6BF * B16 | dst.code() * B12 | 0xF3 * B4 | src.code());
+}
+
+// Status register access instructions.
+void Assembler::mrs(Register dst, SRegister s, Condition cond) {
+ DCHECK(dst != pc);
+ emit(cond | B24 | s | 15 * B16 | dst.code() * B12);
+}
+
+void Assembler::msr(SRegisterFieldMask fields, const Operand& src,
+ Condition cond) {
+ DCHECK_NE(fields & 0x000F0000, 0); // At least one field must be set.
+ DCHECK(((fields & 0xFFF0FFFF) == CPSR) || ((fields & 0xFFF0FFFF) == SPSR));
+ Instr instr;
+ if (src.IsImmediate()) {
+ // Immediate.
+ uint32_t rotate_imm;
+ uint32_t immed_8;
+ if (src.MustOutputRelocInfo(this) ||
+ !FitsShifter(src.immediate(), &rotate_imm, &immed_8, nullptr)) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ // Immediate operand cannot be encoded, load it first to a scratch
+ // register.
+ Move32BitImmediate(scratch, src);
+ msr(fields, Operand(scratch), cond);
+ return;
+ }
+ instr = I | rotate_imm * B8 | immed_8;
+ } else {
+ DCHECK(src.IsRegister()); // Only rm is allowed.
+ instr = src.rm_.code();
+ }
+ emit(cond | instr | B24 | B21 | fields | 15 * B12);
+}
+
+// Load/Store instructions.
+void Assembler::ldr(Register dst, const MemOperand& src, Condition cond) {
+ AddrMode2(cond | B26 | L, dst, src);
+}
+
+void Assembler::str(Register src, const MemOperand& dst, Condition cond) {
+ AddrMode2(cond | B26, src, dst);
+}
+
+void Assembler::ldrb(Register dst, const MemOperand& src, Condition cond) {
+ AddrMode2(cond | B26 | B | L, dst, src);
+}
+
+void Assembler::strb(Register src, const MemOperand& dst, Condition cond) {
+ AddrMode2(cond | B26 | B, src, dst);
+}
+
+void Assembler::ldrh(Register dst, const MemOperand& src, Condition cond) {
+ AddrMode3(cond | L | B7 | H | B4, dst, src);
+}
+
+void Assembler::strh(Register src, const MemOperand& dst, Condition cond) {
+ AddrMode3(cond | B7 | H | B4, src, dst);
+}
+
+void Assembler::ldrsb(Register dst, const MemOperand& src, Condition cond) {
+ AddrMode3(cond | L | B7 | S6 | B4, dst, src);
+}
+
+void Assembler::ldrsh(Register dst, const MemOperand& src, Condition cond) {
+ AddrMode3(cond | L | B7 | S6 | H | B4, dst, src);
+}
+
+void Assembler::ldrd(Register dst1, Register dst2, const MemOperand& src,
+ Condition cond) {
+ DCHECK(src.rm() == no_reg);
+ DCHECK(dst1 != lr); // r14.
+ DCHECK_EQ(0, dst1.code() % 2);
+ DCHECK_EQ(dst1.code() + 1, dst2.code());
+ AddrMode3(cond | B7 | B6 | B4, dst1, src);
+}
+
+void Assembler::strd(Register src1, Register src2, const MemOperand& dst,
+ Condition cond) {
+ DCHECK(dst.rm() == no_reg);
+ DCHECK(src1 != lr); // r14.
+ DCHECK_EQ(0, src1.code() % 2);
+ DCHECK_EQ(src1.code() + 1, src2.code());
+ AddrMode3(cond | B7 | B6 | B5 | B4, src1, dst);
+}
+
+void Assembler::ldr_pcrel(Register dst, int imm12, Condition cond) {
+ AddrMode am = Offset;
+ if (imm12 < 0) {
+ imm12 = -imm12;
+ am = NegOffset;
+ }
+ DCHECK(is_uint12(imm12));
+ emit(cond | B26 | am | L | pc.code() * B16 | dst.code() * B12 | imm12);
+}
+
+// Load/Store exclusive instructions.
+void Assembler::ldrex(Register dst, Register src, Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.75.
+ // cond(31-28) | 00011001(27-20) | Rn(19-16) | Rt(15-12) | 111110011111(11-0)
+ DCHECK(dst != pc);
+ DCHECK(src != pc);
+ emit(cond | B24 | B23 | B20 | src.code() * B16 | dst.code() * B12 | 0xF9F);
+}
+
+void Assembler::strex(Register src1, Register src2, Register dst,
+ Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.212.
+ // cond(31-28) | 00011000(27-20) | Rn(19-16) | Rd(15-12) | 11111001(11-4) |
+ // Rt(3-0)
+ DCHECK(dst != pc);
+ DCHECK(src1 != pc);
+ DCHECK(src2 != pc);
+ DCHECK(src1 != dst);
+ DCHECK(src1 != src2);
+ emit(cond | B24 | B23 | dst.code() * B16 | src1.code() * B12 | 0xF9 * B4 |
+ src2.code());
+}
+
+void Assembler::ldrexb(Register dst, Register src, Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.76.
+ // cond(31-28) | 00011101(27-20) | Rn(19-16) | Rt(15-12) | 111110011111(11-0)
+ DCHECK(dst != pc);
+ DCHECK(src != pc);
+ emit(cond | B24 | B23 | B22 | B20 | src.code() * B16 | dst.code() * B12 |
+ 0xF9F);
+}
+
+void Assembler::strexb(Register src1, Register src2, Register dst,
+ Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.213.
+ // cond(31-28) | 00011100(27-20) | Rn(19-16) | Rd(15-12) | 11111001(11-4) |
+ // Rt(3-0)
+ DCHECK(dst != pc);
+ DCHECK(src1 != pc);
+ DCHECK(src2 != pc);
+ DCHECK(src1 != dst);
+ DCHECK(src1 != src2);
+ emit(cond | B24 | B23 | B22 | dst.code() * B16 | src1.code() * B12 |
+ 0xF9 * B4 | src2.code());
+}
+
+void Assembler::ldrexh(Register dst, Register src, Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.78.
+ // cond(31-28) | 00011111(27-20) | Rn(19-16) | Rt(15-12) | 111110011111(11-0)
+ DCHECK(dst != pc);
+ DCHECK(src != pc);
+ emit(cond | B24 | B23 | B22 | B21 | B20 | src.code() * B16 |
+ dst.code() * B12 | 0xF9F);
+}
+
+void Assembler::strexh(Register src1, Register src2, Register dst,
+ Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.215.
+ // cond(31-28) | 00011110(27-20) | Rn(19-16) | Rd(15-12) | 11111001(11-4) |
+ // Rt(3-0)
+ DCHECK(dst != pc);
+ DCHECK(src1 != pc);
+ DCHECK(src2 != pc);
+ DCHECK(src1 != dst);
+ DCHECK(src1 != src2);
+ emit(cond | B24 | B23 | B22 | B21 | dst.code() * B16 | src1.code() * B12 |
+ 0xF9 * B4 | src2.code());
+}
+
+void Assembler::ldrexd(Register dst1, Register dst2, Register src,
+ Condition cond) {
+ // cond(31-28) | 00011011(27-20) | Rn(19-16) | Rt(15-12) | 111110011111(11-0)
+ DCHECK(dst1 != lr); // r14.
+ // The pair of destination registers is restricted to being an even-numbered
+ // register and the odd-numbered register that immediately follows it.
+ DCHECK_EQ(0, dst1.code() % 2);
+ DCHECK_EQ(dst1.code() + 1, dst2.code());
+ emit(cond | B24 | B23 | B21 | B20 | src.code() * B16 | dst1.code() * B12 |
+ 0xF9F);
+}
+
+void Assembler::strexd(Register res, Register src1, Register src2, Register dst,
+ Condition cond) {
+ // cond(31-28) | 00011010(27-20) | Rn(19-16) | Rt(15-12) | 111110011111(11-0)
+ DCHECK(src1 != lr); // r14.
+ // The pair of source registers is restricted to being an even-numbered
+ // register and the odd-numbered register that immediately follows it.
+ DCHECK_EQ(0, src1.code() % 2);
+ DCHECK_EQ(src1.code() + 1, src2.code());
+ emit(cond | B24 | B23 | B21 | dst.code() * B16 | res.code() * B12 |
+ 0xF9 * B4 | src1.code());
+}
+
+// Preload instructions.
+void Assembler::pld(const MemOperand& address) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.128.
+ // 1111(31-28) | 0111(27-24) | U(23) | R(22) | 01(21-20) | Rn(19-16) |
+ // 1111(15-12) | imm5(11-07) | type(6-5) | 0(4)| Rm(3-0) |
+ DCHECK(address.rm() == no_reg);
+ DCHECK(address.am() == Offset);
+ int U = B23;
+ int offset = address.offset();
+ if (offset < 0) {
+ offset = -offset;
+ U = 0;
+ }
+ DCHECK_LT(offset, 4096);
+ emit(kSpecialCondition | B26 | B24 | U | B22 | B20 |
+ address.rn().code() * B16 | 0xF * B12 | offset);
+}
+
+// Load/Store multiple instructions.
+void Assembler::ldm(BlockAddrMode am, Register base, RegList dst,
+ Condition cond) {
+ // ABI stack constraint: ldmxx base, {..sp..} base != sp is not restartable.
+ DCHECK(base == sp || (dst & sp.bit()) == 0);
+
+ AddrMode4(cond | B27 | am | L, base, dst);
+
+ // Emit the constant pool after a function return implemented by ldm ..{..pc}.
+ if (cond == al && (dst & pc.bit()) != 0) {
+ // There is a slight chance that the ldm instruction was actually a call,
+ // in which case it would be wrong to return into the constant pool; we
+ // recognize this case by checking if the emission of the pool was blocked
+ // at the pc of the ldm instruction by a mov lr, pc instruction; if this is
+ // the case, we emit a jump over the pool.
+ CheckConstPool(true, no_const_pool_before_ == pc_offset() - kInstrSize);
+ }
+}
+
+void Assembler::stm(BlockAddrMode am, Register base, RegList src,
+ Condition cond) {
+ AddrMode4(cond | B27 | am, base, src);
+}
+
+// Exception-generating instructions and debugging support.
+// Stops with a non-negative code less than kNumOfWatchedStops support
+// enabling/disabling and a counter feature. See simulator-arm.h .
+void Assembler::stop(Condition cond, int32_t code) {
+#ifndef __arm__
+ DCHECK_GE(code, kDefaultStopCode);
+ {
+ BlockConstPoolScope block_const_pool(this);
+ if (code >= 0) {
+ svc(kStopCode + code, cond);
+ } else {
+ svc(kStopCode + kMaxStopCode, cond);
+ }
+ }
+#else // def __arm__
+ if (cond != al) {
+ Label skip;
+ b(&skip, NegateCondition(cond));
+ bkpt(0);
+ bind(&skip);
+ } else {
+ bkpt(0);
+ }
+#endif // def __arm__
+}
+
+void Assembler::bkpt(uint32_t imm16) {
+ DCHECK(is_uint16(imm16));
+ emit(al | B24 | B21 | (imm16 >> 4) * B8 | BKPT | (imm16 & 0xF));
+}
+
+void Assembler::svc(uint32_t imm24, Condition cond) {
+ DCHECK(is_uint24(imm24));
+ emit(cond | 15 * B24 | imm24);
+}
+
+void Assembler::dmb(BarrierOption option) {
+ if (CpuFeatures::IsSupported(ARMv7)) {
+ // Details available in ARM DDI 0406C.b, A8-378.
+ emit(kSpecialCondition | 0x57FF * B12 | 5 * B4 | option);
+ } else {
+ // Details available in ARM DDI 0406C.b, B3-1750.
+ // CP15DMB: CRn=c7, opc1=0, CRm=c10, opc2=5, Rt is ignored.
+ mcr(p15, 0, r0, cr7, cr10, 5);
+ }
+}
+
+void Assembler::dsb(BarrierOption option) {
+ if (CpuFeatures::IsSupported(ARMv7)) {
+ // Details available in ARM DDI 0406C.b, A8-380.
+ emit(kSpecialCondition | 0x57FF * B12 | 4 * B4 | option);
+ } else {
+ // Details available in ARM DDI 0406C.b, B3-1750.
+ // CP15DSB: CRn=c7, opc1=0, CRm=c10, opc2=4, Rt is ignored.
+ mcr(p15, 0, r0, cr7, cr10, 4);
+ }
+}
+
+void Assembler::isb(BarrierOption option) {
+ if (CpuFeatures::IsSupported(ARMv7)) {
+ // Details available in ARM DDI 0406C.b, A8-389.
+ emit(kSpecialCondition | 0x57FF * B12 | 6 * B4 | option);
+ } else {
+ // Details available in ARM DDI 0406C.b, B3-1750.
+ // CP15ISB: CRn=c7, opc1=0, CRm=c5, opc2=4, Rt is ignored.
+ mcr(p15, 0, r0, cr7, cr5, 4);
+ }
+}
+
+void Assembler::csdb() {
+ // Details available in Arm Cache Speculation Side-channels white paper,
+ // version 1.1, page 4.
+ emit(0xE320F014);
+}
+
+// Coprocessor instructions.
+void Assembler::cdp(Coprocessor coproc, int opcode_1, CRegister crd,
+ CRegister crn, CRegister crm, int opcode_2,
+ Condition cond) {
+ DCHECK(is_uint4(opcode_1) && is_uint3(opcode_2));
+ emit(cond | B27 | B26 | B25 | (opcode_1 & 15) * B20 | crn.code() * B16 |
+ crd.code() * B12 | coproc * B8 | (opcode_2 & 7) * B5 | crm.code());
+}
+
+void Assembler::cdp2(Coprocessor coproc, int opcode_1, CRegister crd,
+ CRegister crn, CRegister crm, int opcode_2) {
+ cdp(coproc, opcode_1, crd, crn, crm, opcode_2, kSpecialCondition);
+}
+
+void Assembler::mcr(Coprocessor coproc, int opcode_1, Register rd,
+ CRegister crn, CRegister crm, int opcode_2,
+ Condition cond) {
+ DCHECK(is_uint3(opcode_1) && is_uint3(opcode_2));
+ emit(cond | B27 | B26 | B25 | (opcode_1 & 7) * B21 | crn.code() * B16 |
+ rd.code() * B12 | coproc * B8 | (opcode_2 & 7) * B5 | B4 | crm.code());
+}
+
+void Assembler::mcr2(Coprocessor coproc, int opcode_1, Register rd,
+ CRegister crn, CRegister crm, int opcode_2) {
+ mcr(coproc, opcode_1, rd, crn, crm, opcode_2, kSpecialCondition);
+}
+
+void Assembler::mrc(Coprocessor coproc, int opcode_1, Register rd,
+ CRegister crn, CRegister crm, int opcode_2,
+ Condition cond) {
+ DCHECK(is_uint3(opcode_1) && is_uint3(opcode_2));
+ emit(cond | B27 | B26 | B25 | (opcode_1 & 7) * B21 | L | crn.code() * B16 |
+ rd.code() * B12 | coproc * B8 | (opcode_2 & 7) * B5 | B4 | crm.code());
+}
+
+void Assembler::mrc2(Coprocessor coproc, int opcode_1, Register rd,
+ CRegister crn, CRegister crm, int opcode_2) {
+ mrc(coproc, opcode_1, rd, crn, crm, opcode_2, kSpecialCondition);
+}
+
+void Assembler::ldc(Coprocessor coproc, CRegister crd, const MemOperand& src,
+ LFlag l, Condition cond) {
+ AddrMode5(cond | B27 | B26 | l | L | coproc * B8, crd, src);
+}
+
+void Assembler::ldc(Coprocessor coproc, CRegister crd, Register rn, int option,
+ LFlag l, Condition cond) {
+ // Unindexed addressing.
+ DCHECK(is_uint8(option));
+ emit(cond | B27 | B26 | U | l | L | rn.code() * B16 | crd.code() * B12 |
+ coproc * B8 | (option & 255));
+}
+
+void Assembler::ldc2(Coprocessor coproc, CRegister crd, const MemOperand& src,
+ LFlag l) {
+ ldc(coproc, crd, src, l, kSpecialCondition);
+}
+
+void Assembler::ldc2(Coprocessor coproc, CRegister crd, Register rn, int option,
+ LFlag l) {
+ ldc(coproc, crd, rn, option, l, kSpecialCondition);
+}
+
+// Support for VFP.
+
+void Assembler::vldr(const DwVfpRegister dst, const Register base, int offset,
+ const Condition cond) {
+ // Ddst = MEM(Rbase + offset).
+ // Instruction details available in ARM DDI 0406C.b, A8-924.
+ // cond(31-28) | 1101(27-24)| U(23) | D(22) | 01(21-20) | Rbase(19-16) |
+ // Vd(15-12) | 1011(11-8) | offset
+ DCHECK(VfpRegisterIsAvailable(dst));
+ int u = 1;
+ if (offset < 0) {
+ CHECK_NE(offset, kMinInt);
+ offset = -offset;
+ u = 0;
+ }
+ int vd, d;
+ dst.split_code(&vd, &d);
+
+ DCHECK_GE(offset, 0);
+ if ((offset % 4) == 0 && (offset / 4) < 256) {
+ emit(cond | 0xD * B24 | u * B23 | d * B22 | B20 | base.code() * B16 |
+ vd * B12 | 0xB * B8 | ((offset / 4) & 255));
+ } else {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ // Larger offsets must be handled by computing the correct address in a
+ // scratch register.
+ DCHECK(base != scratch);
+ if (u == 1) {
+ add(scratch, base, Operand(offset));
+ } else {
+ sub(scratch, base, Operand(offset));
+ }
+ emit(cond | 0xD * B24 | d * B22 | B20 | scratch.code() * B16 | vd * B12 |
+ 0xB * B8);
+ }
+}
+
+void Assembler::vldr(const DwVfpRegister dst, const MemOperand& operand,
+ const Condition cond) {
+ DCHECK(VfpRegisterIsAvailable(dst));
+ DCHECK(operand.am_ == Offset);
+ if (operand.rm().is_valid()) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ add(scratch, operand.rn(),
+ Operand(operand.rm(), operand.shift_op_, operand.shift_imm_));
+ vldr(dst, scratch, 0, cond);
+ } else {
+ vldr(dst, operand.rn(), operand.offset(), cond);
+ }
+}
+
+void Assembler::vldr(const SwVfpRegister dst, const Register base, int offset,
+ const Condition cond) {
+ // Sdst = MEM(Rbase + offset).
+ // Instruction details available in ARM DDI 0406A, A8-628.
+ // cond(31-28) | 1101(27-24)| U001(23-20) | Rbase(19-16) |
+ // Vdst(15-12) | 1010(11-8) | offset
+ int u = 1;
+ if (offset < 0) {
+ offset = -offset;
+ u = 0;
+ }
+ int sd, d;
+ dst.split_code(&sd, &d);
+ DCHECK_GE(offset, 0);
+
+ if ((offset % 4) == 0 && (offset / 4) < 256) {
+ emit(cond | u * B23 | d * B22 | 0xD1 * B20 | base.code() * B16 | sd * B12 |
+ 0xA * B8 | ((offset / 4) & 255));
+ } else {
+ // Larger offsets must be handled by computing the correct address in a
+ // scratch register.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(base != scratch);
+ if (u == 1) {
+ add(scratch, base, Operand(offset));
+ } else {
+ sub(scratch, base, Operand(offset));
+ }
+ emit(cond | d * B22 | 0xD1 * B20 | scratch.code() * B16 | sd * B12 |
+ 0xA * B8);
+ }
+}
+
+void Assembler::vldr(const SwVfpRegister dst, const MemOperand& operand,
+ const Condition cond) {
+ DCHECK(operand.am_ == Offset);
+ if (operand.rm().is_valid()) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ add(scratch, operand.rn(),
+ Operand(operand.rm(), operand.shift_op_, operand.shift_imm_));
+ vldr(dst, scratch, 0, cond);
+ } else {
+ vldr(dst, operand.rn(), operand.offset(), cond);
+ }
+}
+
+void Assembler::vstr(const DwVfpRegister src, const Register base, int offset,
+ const Condition cond) {
+ // MEM(Rbase + offset) = Dsrc.
+ // Instruction details available in ARM DDI 0406C.b, A8-1082.
+ // cond(31-28) | 1101(27-24)| U(23) | D(22) | 00(21-20) | Rbase(19-16) |
+ // Vd(15-12) | 1011(11-8) | (offset/4)
+ DCHECK(VfpRegisterIsAvailable(src));
+ int u = 1;
+ if (offset < 0) {
+ CHECK_NE(offset, kMinInt);
+ offset = -offset;
+ u = 0;
+ }
+ DCHECK_GE(offset, 0);
+ int vd, d;
+ src.split_code(&vd, &d);
+
+ if ((offset % 4) == 0 && (offset / 4) < 256) {
+ emit(cond | 0xD * B24 | u * B23 | d * B22 | base.code() * B16 | vd * B12 |
+ 0xB * B8 | ((offset / 4) & 255));
+ } else {
+ // Larger offsets must be handled by computing the correct address in the a
+ // scratch register.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(base != scratch);
+ if (u == 1) {
+ add(scratch, base, Operand(offset));
+ } else {
+ sub(scratch, base, Operand(offset));
+ }
+ emit(cond | 0xD * B24 | d * B22 | scratch.code() * B16 | vd * B12 |
+ 0xB * B8);
+ }
+}
+
+void Assembler::vstr(const DwVfpRegister src, const MemOperand& operand,
+ const Condition cond) {
+ DCHECK(VfpRegisterIsAvailable(src));
+ DCHECK(operand.am_ == Offset);
+ if (operand.rm().is_valid()) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ add(scratch, operand.rn(),
+ Operand(operand.rm(), operand.shift_op_, operand.shift_imm_));
+ vstr(src, scratch, 0, cond);
+ } else {
+ vstr(src, operand.rn(), operand.offset(), cond);
+ }
+}
+
+void Assembler::vstr(const SwVfpRegister src, const Register base, int offset,
+ const Condition cond) {
+ // MEM(Rbase + offset) = SSrc.
+ // Instruction details available in ARM DDI 0406A, A8-786.
+ // cond(31-28) | 1101(27-24)| U000(23-20) | Rbase(19-16) |
+ // Vdst(15-12) | 1010(11-8) | (offset/4)
+ int u = 1;
+ if (offset < 0) {
+ CHECK_NE(offset, kMinInt);
+ offset = -offset;
+ u = 0;
+ }
+ int sd, d;
+ src.split_code(&sd, &d);
+ DCHECK_GE(offset, 0);
+ if ((offset % 4) == 0 && (offset / 4) < 256) {
+ emit(cond | u * B23 | d * B22 | 0xD0 * B20 | base.code() * B16 | sd * B12 |
+ 0xA * B8 | ((offset / 4) & 255));
+ } else {
+ // Larger offsets must be handled by computing the correct address in a
+ // scratch register.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(base != scratch);
+ if (u == 1) {
+ add(scratch, base, Operand(offset));
+ } else {
+ sub(scratch, base, Operand(offset));
+ }
+ emit(cond | d * B22 | 0xD0 * B20 | scratch.code() * B16 | sd * B12 |
+ 0xA * B8);
+ }
+}
+
+void Assembler::vstr(const SwVfpRegister src, const MemOperand& operand,
+ const Condition cond) {
+ DCHECK(operand.am_ == Offset);
+ if (operand.rm().is_valid()) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ add(scratch, operand.rn(),
+ Operand(operand.rm(), operand.shift_op_, operand.shift_imm_));
+ vstr(src, scratch, 0, cond);
+ } else {
+ vstr(src, operand.rn(), operand.offset(), cond);
+ }
+}
+
+void Assembler::vldm(BlockAddrMode am, Register base, DwVfpRegister first,
+ DwVfpRegister last, Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8-922.
+ // cond(31-28) | 110(27-25)| PUDW1(24-20) | Rbase(19-16) |
+ // first(15-12) | 1011(11-8) | (count * 2)
+ DCHECK_LE(first.code(), last.code());
+ DCHECK(VfpRegisterIsAvailable(last));
+ DCHECK(am == ia || am == ia_w || am == db_w);
+ DCHECK(base != pc);
+
+ int sd, d;
+ first.split_code(&sd, &d);
+ int count = last.code() - first.code() + 1;
+ DCHECK_LE(count, 16);
+ emit(cond | B27 | B26 | am | d * B22 | B20 | base.code() * B16 | sd * B12 |
+ 0xB * B8 | count * 2);
+}
+
+void Assembler::vstm(BlockAddrMode am, Register base, DwVfpRegister first,
+ DwVfpRegister last, Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8-1080.
+ // cond(31-28) | 110(27-25)| PUDW0(24-20) | Rbase(19-16) |
+ // first(15-12) | 1011(11-8) | (count * 2)
+ DCHECK_LE(first.code(), last.code());
+ DCHECK(VfpRegisterIsAvailable(last));
+ DCHECK(am == ia || am == ia_w || am == db_w);
+ DCHECK(base != pc);
+
+ int sd, d;
+ first.split_code(&sd, &d);
+ int count = last.code() - first.code() + 1;
+ DCHECK_LE(count, 16);
+ emit(cond | B27 | B26 | am | d * B22 | base.code() * B16 | sd * B12 |
+ 0xB * B8 | count * 2);
+}
+
+void Assembler::vldm(BlockAddrMode am, Register base, SwVfpRegister first,
+ SwVfpRegister last, Condition cond) {
+ // Instruction details available in ARM DDI 0406A, A8-626.
+ // cond(31-28) | 110(27-25)| PUDW1(24-20) | Rbase(19-16) |
+ // first(15-12) | 1010(11-8) | (count/2)
+ DCHECK_LE(first.code(), last.code());
+ DCHECK(am == ia || am == ia_w || am == db_w);
+ DCHECK(base != pc);
+
+ int sd, d;
+ first.split_code(&sd, &d);
+ int count = last.code() - first.code() + 1;
+ emit(cond | B27 | B26 | am | d * B22 | B20 | base.code() * B16 | sd * B12 |
+ 0xA * B8 | count);
+}
+
+void Assembler::vstm(BlockAddrMode am, Register base, SwVfpRegister first,
+ SwVfpRegister last, Condition cond) {
+ // Instruction details available in ARM DDI 0406A, A8-784.
+ // cond(31-28) | 110(27-25)| PUDW0(24-20) | Rbase(19-16) |
+ // first(15-12) | 1011(11-8) | (count/2)
+ DCHECK_LE(first.code(), last.code());
+ DCHECK(am == ia || am == ia_w || am == db_w);
+ DCHECK(base != pc);
+
+ int sd, d;
+ first.split_code(&sd, &d);
+ int count = last.code() - first.code() + 1;
+ emit(cond | B27 | B26 | am | d * B22 | base.code() * B16 | sd * B12 |
+ 0xA * B8 | count);
+}
+
+static void DoubleAsTwoUInt32(Double d, uint32_t* lo, uint32_t* hi) {
+ uint64_t i = d.AsUint64();
+
+ *lo = i & 0xFFFFFFFF;
+ *hi = i >> 32;
+}
+
+static void WriteVmovIntImmEncoding(uint8_t imm, uint32_t* encoding) {
+ // Integer promotion from uint8_t to int makes these all okay.
+ *encoding = ((imm & 0x80) << (24 - 7)); // a
+ *encoding |= ((imm & 0x70) << (16 - 4)); // bcd
+ *encoding |= (imm & 0x0f); // efgh
+}
+
+// This checks if imm can be encoded into an immediate for vmov.
+// See Table A7-15 in ARM DDI 0406C.d.
+// Currently only supports the first row and op=0 && cmode=1110.
+static bool FitsVmovIntImm(uint64_t imm, uint32_t* encoding, uint8_t* cmode) {
+ uint32_t lo = imm & 0xFFFFFFFF;
+ uint32_t hi = imm >> 32;
+ if ((lo == hi && ((lo & 0xffffff00) == 0))) {
+ WriteVmovIntImmEncoding(imm & 0xff, encoding);
+ *cmode = 0;
+ return true;
+ } else if ((lo == hi) && ((lo & 0xffff) == (lo >> 16)) &&
+ ((lo & 0xff) == (lo >> 24))) {
+ // Check that all bytes in imm are the same.
+ WriteVmovIntImmEncoding(imm & 0xff, encoding);
+ *cmode = 0xe;
+ return true;
+ }
+
+ return false;
+}
+
+void Assembler::vmov(const DwVfpRegister dst, uint64_t imm) {
+ uint32_t enc;
+ uint8_t cmode;
+ uint8_t op = 0;
+ if (CpuFeatures::IsSupported(NEON) && FitsVmovIntImm(imm, &enc, &cmode)) {
+ CpuFeatureScope scope(this, NEON);
+ // Instruction details available in ARM DDI 0406C.b, A8-937.
+ // 001i1(27-23) | D(22) | 000(21-19) | imm3(18-16) | Vd(15-12) | cmode(11-8)
+ // | 0(7) | 0(6) | op(5) | 4(1) | imm4(3-0)
+ int vd, d;
+ dst.split_code(&vd, &d);
+ emit(kSpecialCondition | 0x05 * B23 | d * B22 | vd * B12 | cmode * B8 |
+ op * B5 | 0x1 * B4 | enc);
+ } else {
+ UNIMPLEMENTED();
+ }
+}
+
+void Assembler::vmov(const QwNeonRegister dst, uint64_t imm) {
+ uint32_t enc;
+ uint8_t cmode;
+ uint8_t op = 0;
+ if (CpuFeatures::IsSupported(NEON) && FitsVmovIntImm(imm, &enc, &cmode)) {
+ CpuFeatureScope scope(this, NEON);
+ // Instruction details available in ARM DDI 0406C.b, A8-937.
+ // 001i1(27-23) | D(22) | 000(21-19) | imm3(18-16) | Vd(15-12) | cmode(11-8)
+ // | 0(7) | Q(6) | op(5) | 4(1) | imm4(3-0)
+ int vd, d;
+ dst.split_code(&vd, &d);
+ emit(kSpecialCondition | 0x05 * B23 | d * B22 | vd * B12 | cmode * B8 |
+ 0x1 * B6 | op * B5 | 0x1 * B4 | enc);
+ } else {
+ UNIMPLEMENTED();
+ }
+}
+
+// Only works for little endian floating point formats.
+// We don't support VFP on the mixed endian floating point platform.
+static bool FitsVmovFPImmediate(Double d, uint32_t* encoding) {
+ // VMOV can accept an immediate of the form:
+ //
+ // +/- m * 2^(-n) where 16 <= m <= 31 and 0 <= n <= 7
+ //
+ // The immediate is encoded using an 8-bit quantity, comprised of two
+ // 4-bit fields. For an 8-bit immediate of the form:
+ //
+ // [abcdefgh]
+ //
+ // where a is the MSB and h is the LSB, an immediate 64-bit double can be
+ // created of the form:
+ //
+ // [aBbbbbbb,bbcdefgh,00000000,00000000,
+ // 00000000,00000000,00000000,00000000]
+ //
+ // where B = ~b.
+ //
+
+ uint32_t lo, hi;
+ DoubleAsTwoUInt32(d, &lo, &hi);
+
+ // The most obvious constraint is the long block of zeroes.
+ if ((lo != 0) || ((hi & 0xFFFF) != 0)) {
+ return false;
+ }
+
+ // Bits 61:54 must be all clear or all set.
+ if (((hi & 0x3FC00000) != 0) && ((hi & 0x3FC00000) != 0x3FC00000)) {
+ return false;
+ }
+
+ // Bit 62 must be NOT bit 61.
+ if (((hi ^ (hi << 1)) & (0x40000000)) == 0) {
+ return false;
+ }
+
+ // Create the encoded immediate in the form:
+ // [00000000,0000abcd,00000000,0000efgh]
+ *encoding = (hi >> 16) & 0xF; // Low nybble.
+ *encoding |= (hi >> 4) & 0x70000; // Low three bits of the high nybble.
+ *encoding |= (hi >> 12) & 0x80000; // Top bit of the high nybble.
+
+ return true;
+}
+
+void Assembler::vmov(const SwVfpRegister dst, Float32 imm) {
+ uint32_t enc;
+ if (CpuFeatures::IsSupported(VFPv3) &&
+ FitsVmovFPImmediate(Double(imm.get_scalar()), &enc)) {
+ CpuFeatureScope scope(this, VFPv3);
+ // The float can be encoded in the instruction.
+ //
+ // Sd = immediate
+ // Instruction details available in ARM DDI 0406C.b, A8-936.
+ // cond(31-28) | 11101(27-23) | D(22) | 11(21-20) | imm4H(19-16) |
+ // Vd(15-12) | 101(11-9) | sz=0(8) | imm4L(3-0)
+ int vd, d;
+ dst.split_code(&vd, &d);
+ emit(al | 0x1D * B23 | d * B22 | 0x3 * B20 | vd * B12 | 0x5 * B9 | enc);
+ } else {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ mov(scratch, Operand(imm.get_bits()));
+ vmov(dst, scratch);
+ }
+}
+
+void Assembler::vmov(const DwVfpRegister dst, Double imm,
+ const Register extra_scratch) {
+ DCHECK(VfpRegisterIsAvailable(dst));
+ uint32_t enc;
+ if (CpuFeatures::IsSupported(VFPv3) && FitsVmovFPImmediate(imm, &enc)) {
+ CpuFeatureScope scope(this, VFPv3);
+ // The double can be encoded in the instruction.
+ //
+ // Dd = immediate
+ // Instruction details available in ARM DDI 0406C.b, A8-936.
+ // cond(31-28) | 11101(27-23) | D(22) | 11(21-20) | imm4H(19-16) |
+ // Vd(15-12) | 101(11-9) | sz=1(8) | imm4L(3-0)
+ int vd, d;
+ dst.split_code(&vd, &d);
+ emit(al | 0x1D * B23 | d * B22 | 0x3 * B20 | vd * B12 | 0x5 * B9 | B8 |
+ enc);
+ } else {
+ // Synthesise the double from ARM immediates.
+ uint32_t lo, hi;
+ DoubleAsTwoUInt32(imm, &lo, &hi);
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+
+ if (lo == hi) {
+ // Move the low and high parts of the double to a D register in one
+ // instruction.
+ mov(scratch, Operand(lo));
+ vmov(dst, scratch, scratch);
+ } else if (extra_scratch == no_reg) {
+ // We only have one spare scratch register.
+ mov(scratch, Operand(lo));
+ vmov(NeonS32, dst, 0, scratch);
+ if (((lo & 0xFFFF) == (hi & 0xFFFF)) && CpuFeatures::IsSupported(ARMv7)) {
+ CpuFeatureScope scope(this, ARMv7);
+ movt(scratch, hi >> 16);
+ } else {
+ mov(scratch, Operand(hi));
+ }
+ vmov(NeonS32, dst, 1, scratch);
+ } else {
+ // Move the low and high parts of the double to a D register in one
+ // instruction.
+ mov(scratch, Operand(lo));
+ mov(extra_scratch, Operand(hi));
+ vmov(dst, scratch, extra_scratch);
+ }
+ }
+}
+
+void Assembler::vmov(const SwVfpRegister dst, const SwVfpRegister src,
+ const Condition cond) {
+ // Sd = Sm
+ // Instruction details available in ARM DDI 0406B, A8-642.
+ int sd, d, sm, m;
+ dst.split_code(&sd, &d);
+ src.split_code(&sm, &m);
+ emit(cond | 0xE * B24 | d * B22 | 0xB * B20 | sd * B12 | 0xA * B8 | B6 |
+ m * B5 | sm);
+}
+
+void Assembler::vmov(const DwVfpRegister dst, const DwVfpRegister src,
+ const Condition cond) {
+ // Dd = Dm
+ // Instruction details available in ARM DDI 0406C.b, A8-938.
+ // cond(31-28) | 11101(27-23) | D(22) | 11(21-20) | 0000(19-16) | Vd(15-12) |
+ // 101(11-9) | sz=1(8) | 0(7) | 1(6) | M(5) | 0(4) | Vm(3-0)
+ DCHECK(VfpRegisterIsAvailable(dst));
+ DCHECK(VfpRegisterIsAvailable(src));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vm, m;
+ src.split_code(&vm, &m);
+ emit(cond | 0x1D * B23 | d * B22 | 0x3 * B20 | vd * B12 | 0x5 * B9 | B8 | B6 |
+ m * B5 | vm);
+}
+
+void Assembler::vmov(const DwVfpRegister dst, const Register src1,
+ const Register src2, const Condition cond) {
+ // Dm = <Rt,Rt2>.
+ // Instruction details available in ARM DDI 0406C.b, A8-948.
+ // cond(31-28) | 1100(27-24)| 010(23-21) | op=0(20) | Rt2(19-16) |
+ // Rt(15-12) | 1011(11-8) | 00(7-6) | M(5) | 1(4) | Vm
+ DCHECK(VfpRegisterIsAvailable(dst));
+ DCHECK(src1 != pc && src2 != pc);
+ int vm, m;
+ dst.split_code(&vm, &m);
+ emit(cond | 0xC * B24 | B22 | src2.code() * B16 | src1.code() * B12 |
+ 0xB * B8 | m * B5 | B4 | vm);
+}
+
+void Assembler::vmov(const Register dst1, const Register dst2,
+ const DwVfpRegister src, const Condition cond) {
+ // <Rt,Rt2> = Dm.
+ // Instruction details available in ARM DDI 0406C.b, A8-948.
+ // cond(31-28) | 1100(27-24)| 010(23-21) | op=1(20) | Rt2(19-16) |
+ // Rt(15-12) | 1011(11-8) | 00(7-6) | M(5) | 1(4) | Vm
+ DCHECK(VfpRegisterIsAvailable(src));
+ DCHECK(dst1 != pc && dst2 != pc);
+ int vm, m;
+ src.split_code(&vm, &m);
+ emit(cond | 0xC * B24 | B22 | B20 | dst2.code() * B16 | dst1.code() * B12 |
+ 0xB * B8 | m * B5 | B4 | vm);
+}
+
+void Assembler::vmov(const SwVfpRegister dst, const Register src,
+ const Condition cond) {
+ // Sn = Rt.
+ // Instruction details available in ARM DDI 0406A, A8-642.
+ // cond(31-28) | 1110(27-24)| 000(23-21) | op=0(20) | Vn(19-16) |
+ // Rt(15-12) | 1010(11-8) | N(7)=0 | 00(6-5) | 1(4) | 0000(3-0)
+ DCHECK(src != pc);
+ int sn, n;
+ dst.split_code(&sn, &n);
+ emit(cond | 0xE * B24 | sn * B16 | src.code() * B12 | 0xA * B8 | n * B7 | B4);
+}
+
+void Assembler::vmov(const Register dst, const SwVfpRegister src,
+ const Condition cond) {
+ // Rt = Sn.
+ // Instruction details available in ARM DDI 0406A, A8-642.
+ // cond(31-28) | 1110(27-24)| 000(23-21) | op=1(20) | Vn(19-16) |
+ // Rt(15-12) | 1010(11-8) | N(7)=0 | 00(6-5) | 1(4) | 0000(3-0)
+ DCHECK(dst != pc);
+ int sn, n;
+ src.split_code(&sn, &n);
+ emit(cond | 0xE * B24 | B20 | sn * B16 | dst.code() * B12 | 0xA * B8 |
+ n * B7 | B4);
+}
+
+// Type of data to read from or write to VFP register.
+// Used as specifier in generic vcvt instruction.
+enum VFPType { S32, U32, F32, F64 };
+
+static bool IsSignedVFPType(VFPType type) {
+ switch (type) {
+ case S32:
+ return true;
+ case U32:
+ return false;
+ default:
+ UNREACHABLE();
+ }
+}
+
+static bool IsIntegerVFPType(VFPType type) {
+ switch (type) {
+ case S32:
+ case U32:
+ return true;
+ case F32:
+ case F64:
+ return false;
+ default:
+ UNREACHABLE();
+ }
+}
+
+static bool IsDoubleVFPType(VFPType type) {
+ switch (type) {
+ case F32:
+ return false;
+ case F64:
+ return true;
+ default:
+ UNREACHABLE();
+ }
+}
+
+// Split five bit reg_code based on size of reg_type.
+// 32-bit register codes are Vm:M
+// 64-bit register codes are M:Vm
+// where Vm is four bits, and M is a single bit.
+static void SplitRegCode(VFPType reg_type, int reg_code, int* vm, int* m) {
+ DCHECK((reg_code >= 0) && (reg_code <= 31));
+ if (IsIntegerVFPType(reg_type) || !IsDoubleVFPType(reg_type)) {
+ SwVfpRegister::split_code(reg_code, vm, m);
+ } else {
+ DwVfpRegister::split_code(reg_code, vm, m);
+ }
+}
+
+// Encode vcvt.src_type.dst_type instruction.
+static Instr EncodeVCVT(const VFPType dst_type, const int dst_code,
+ const VFPType src_type, const int src_code,
+ VFPConversionMode mode, const Condition cond) {
+ DCHECK(src_type != dst_type);
+ int D, Vd, M, Vm;
+ SplitRegCode(src_type, src_code, &Vm, &M);
+ SplitRegCode(dst_type, dst_code, &Vd, &D);
+
+ if (IsIntegerVFPType(dst_type) || IsIntegerVFPType(src_type)) {
+ // Conversion between IEEE floating point and 32-bit integer.
+ // Instruction details available in ARM DDI 0406B, A8.6.295.
+ // cond(31-28) | 11101(27-23)| D(22) | 11(21-20) | 1(19) | opc2(18-16) |
+ // Vd(15-12) | 101(11-9) | sz(8) | op(7) | 1(6) | M(5) | 0(4) | Vm(3-0)
+ DCHECK(!IsIntegerVFPType(dst_type) || !IsIntegerVFPType(src_type));
+
+ int sz, opc2, op;
+
+ if (IsIntegerVFPType(dst_type)) {
+ opc2 = IsSignedVFPType(dst_type) ? 0x5 : 0x4;
+ sz = IsDoubleVFPType(src_type) ? 0x1 : 0x0;
+ op = mode;
+ } else {
+ DCHECK(IsIntegerVFPType(src_type));
+ opc2 = 0x0;
+ sz = IsDoubleVFPType(dst_type) ? 0x1 : 0x0;
+ op = IsSignedVFPType(src_type) ? 0x1 : 0x0;
+ }
+
+ return (cond | 0xE * B24 | B23 | D * B22 | 0x3 * B20 | B19 | opc2 * B16 |
+ Vd * B12 | 0x5 * B9 | sz * B8 | op * B7 | B6 | M * B5 | Vm);
+ } else {
+ // Conversion between IEEE double and single precision.
+ // Instruction details available in ARM DDI 0406B, A8.6.298.
+ // cond(31-28) | 11101(27-23)| D(22) | 11(21-20) | 0111(19-16) |
+ // Vd(15-12) | 101(11-9) | sz(8) | 1(7) | 1(6) | M(5) | 0(4) | Vm(3-0)
+ int sz = IsDoubleVFPType(src_type) ? 0x1 : 0x0;
+ return (cond | 0xE * B24 | B23 | D * B22 | 0x3 * B20 | 0x7 * B16 |
+ Vd * B12 | 0x5 * B9 | sz * B8 | B7 | B6 | M * B5 | Vm);
+ }
+}
+
+void Assembler::vcvt_f64_s32(const DwVfpRegister dst, const SwVfpRegister src,
+ VFPConversionMode mode, const Condition cond) {
+ DCHECK(VfpRegisterIsAvailable(dst));
+ emit(EncodeVCVT(F64, dst.code(), S32, src.code(), mode, cond));
+}
+
+void Assembler::vcvt_f32_s32(const SwVfpRegister dst, const SwVfpRegister src,
+ VFPConversionMode mode, const Condition cond) {
+ emit(EncodeVCVT(F32, dst.code(), S32, src.code(), mode, cond));
+}
+
+void Assembler::vcvt_f64_u32(const DwVfpRegister dst, const SwVfpRegister src,
+ VFPConversionMode mode, const Condition cond) {
+ DCHECK(VfpRegisterIsAvailable(dst));
+ emit(EncodeVCVT(F64, dst.code(), U32, src.code(), mode, cond));
+}
+
+void Assembler::vcvt_f32_u32(const SwVfpRegister dst, const SwVfpRegister src,
+ VFPConversionMode mode, const Condition cond) {
+ emit(EncodeVCVT(F32, dst.code(), U32, src.code(), mode, cond));
+}
+
+void Assembler::vcvt_s32_f32(const SwVfpRegister dst, const SwVfpRegister src,
+ VFPConversionMode mode, const Condition cond) {
+ emit(EncodeVCVT(S32, dst.code(), F32, src.code(), mode, cond));
+}
+
+void Assembler::vcvt_u32_f32(const SwVfpRegister dst, const SwVfpRegister src,
+ VFPConversionMode mode, const Condition cond) {
+ emit(EncodeVCVT(U32, dst.code(), F32, src.code(), mode, cond));
+}
+
+void Assembler::vcvt_s32_f64(const SwVfpRegister dst, const DwVfpRegister src,
+ VFPConversionMode mode, const Condition cond) {
+ DCHECK(VfpRegisterIsAvailable(src));
+ emit(EncodeVCVT(S32, dst.code(), F64, src.code(), mode, cond));
+}
+
+void Assembler::vcvt_u32_f64(const SwVfpRegister dst, const DwVfpRegister src,
+ VFPConversionMode mode, const Condition cond) {
+ DCHECK(VfpRegisterIsAvailable(src));
+ emit(EncodeVCVT(U32, dst.code(), F64, src.code(), mode, cond));
+}
+
+void Assembler::vcvt_f64_f32(const DwVfpRegister dst, const SwVfpRegister src,
+ VFPConversionMode mode, const Condition cond) {
+ DCHECK(VfpRegisterIsAvailable(dst));
+ emit(EncodeVCVT(F64, dst.code(), F32, src.code(), mode, cond));
+}
+
+void Assembler::vcvt_f32_f64(const SwVfpRegister dst, const DwVfpRegister src,
+ VFPConversionMode mode, const Condition cond) {
+ DCHECK(VfpRegisterIsAvailable(src));
+ emit(EncodeVCVT(F32, dst.code(), F64, src.code(), mode, cond));
+}
+
+void Assembler::vcvt_f64_s32(const DwVfpRegister dst, int fraction_bits,
+ const Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8-874.
+ // cond(31-28) | 11101(27-23) | D(22) | 11(21-20) | 1010(19-16) | Vd(15-12) |
+ // 101(11-9) | sf=1(8) | sx=1(7) | 1(6) | i(5) | 0(4) | imm4(3-0)
+ DCHECK(IsEnabled(VFPv3));
+ DCHECK(VfpRegisterIsAvailable(dst));
+ DCHECK(fraction_bits > 0 && fraction_bits <= 32);
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int imm5 = 32 - fraction_bits;
+ int i = imm5 & 1;
+ int imm4 = (imm5 >> 1) & 0xF;
+ emit(cond | 0xE * B24 | B23 | d * B22 | 0x3 * B20 | B19 | 0x2 * B16 |
+ vd * B12 | 0x5 * B9 | B8 | B7 | B6 | i * B5 | imm4);
+}
+
+void Assembler::vneg(const DwVfpRegister dst, const DwVfpRegister src,
+ const Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8-968.
+ // cond(31-28) | 11101(27-23) | D(22) | 11(21-20) | 0001(19-16) | Vd(15-12) |
+ // 101(11-9) | sz=1(8) | 0(7) | 1(6) | M(5) | 0(4) | Vm(3-0)
+ DCHECK(VfpRegisterIsAvailable(dst));
+ DCHECK(VfpRegisterIsAvailable(src));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vm, m;
+ src.split_code(&vm, &m);
+
+ emit(cond | 0x1D * B23 | d * B22 | 0x3 * B20 | B16 | vd * B12 | 0x5 * B9 |
+ B8 | B6 | m * B5 | vm);
+}
+
+void Assembler::vneg(const SwVfpRegister dst, const SwVfpRegister src,
+ const Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8-968.
+ // cond(31-28) | 11101(27-23) | D(22) | 11(21-20) | 0001(19-16) | Vd(15-12) |
+ // 101(11-9) | sz=0(8) | 0(7) | 1(6) | M(5) | 0(4) | Vm(3-0)
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vm, m;
+ src.split_code(&vm, &m);
+
+ emit(cond | 0x1D * B23 | d * B22 | 0x3 * B20 | B16 | vd * B12 | 0x5 * B9 |
+ B6 | m * B5 | vm);
+}
+
+void Assembler::vabs(const DwVfpRegister dst, const DwVfpRegister src,
+ const Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8-524.
+ // cond(31-28) | 11101(27-23) | D(22) | 11(21-20) | 0000(19-16) | Vd(15-12) |
+ // 101(11-9) | sz=1(8) | 1(7) | 1(6) | M(5) | 0(4) | Vm(3-0)
+ DCHECK(VfpRegisterIsAvailable(dst));
+ DCHECK(VfpRegisterIsAvailable(src));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vm, m;
+ src.split_code(&vm, &m);
+ emit(cond | 0x1D * B23 | d * B22 | 0x3 * B20 | vd * B12 | 0x5 * B9 | B8 | B7 |
+ B6 | m * B5 | vm);
+}
+
+void Assembler::vabs(const SwVfpRegister dst, const SwVfpRegister src,
+ const Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8-524.
+ // cond(31-28) | 11101(27-23) | D(22) | 11(21-20) | 0000(19-16) | Vd(15-12) |
+ // 101(11-9) | sz=0(8) | 1(7) | 1(6) | M(5) | 0(4) | Vm(3-0)
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vm, m;
+ src.split_code(&vm, &m);
+ emit(cond | 0x1D * B23 | d * B22 | 0x3 * B20 | vd * B12 | 0x5 * B9 | B7 | B6 |
+ m * B5 | vm);
+}
+
+void Assembler::vadd(const DwVfpRegister dst, const DwVfpRegister src1,
+ const DwVfpRegister src2, const Condition cond) {
+ // Dd = vadd(Dn, Dm) double precision floating point addition.
+ // Dd = D:Vd; Dm=M:Vm; Dn=N:Vm.
+ // Instruction details available in ARM DDI 0406C.b, A8-830.
+ // cond(31-28) | 11100(27-23)| D(22) | 11(21-20) | Vn(19-16) |
+ // Vd(15-12) | 101(11-9) | sz=1(8) | N(7) | 0(6) | M(5) | 0(4) | Vm(3-0)
+ DCHECK(VfpRegisterIsAvailable(dst));
+ DCHECK(VfpRegisterIsAvailable(src1));
+ DCHECK(VfpRegisterIsAvailable(src2));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vn, n;
+ src1.split_code(&vn, &n);
+ int vm, m;
+ src2.split_code(&vm, &m);
+ emit(cond | 0x1C * B23 | d * B22 | 0x3 * B20 | vn * B16 | vd * B12 |
+ 0x5 * B9 | B8 | n * B7 | m * B5 | vm);
+}
+
+void Assembler::vadd(const SwVfpRegister dst, const SwVfpRegister src1,
+ const SwVfpRegister src2, const Condition cond) {
+ // Sd = vadd(Sn, Sm) single precision floating point addition.
+ // Sd = D:Vd; Sm=M:Vm; Sn=N:Vm.
+ // Instruction details available in ARM DDI 0406C.b, A8-830.
+ // cond(31-28) | 11100(27-23)| D(22) | 11(21-20) | Vn(19-16) |
+ // Vd(15-12) | 101(11-9) | sz=0(8) | N(7) | 0(6) | M(5) | 0(4) | Vm(3-0)
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vn, n;
+ src1.split_code(&vn, &n);
+ int vm, m;
+ src2.split_code(&vm, &m);
+ emit(cond | 0x1C * B23 | d * B22 | 0x3 * B20 | vn * B16 | vd * B12 |
+ 0x5 * B9 | n * B7 | m * B5 | vm);
+}
+
+void Assembler::vsub(const DwVfpRegister dst, const DwVfpRegister src1,
+ const DwVfpRegister src2, const Condition cond) {
+ // Dd = vsub(Dn, Dm) double precision floating point subtraction.
+ // Dd = D:Vd; Dm=M:Vm; Dn=N:Vm.
+ // Instruction details available in ARM DDI 0406C.b, A8-1086.
+ // cond(31-28) | 11100(27-23)| D(22) | 11(21-20) | Vn(19-16) |
+ // Vd(15-12) | 101(11-9) | sz=1(8) | N(7) | 1(6) | M(5) | 0(4) | Vm(3-0)
+ DCHECK(VfpRegisterIsAvailable(dst));
+ DCHECK(VfpRegisterIsAvailable(src1));
+ DCHECK(VfpRegisterIsAvailable(src2));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vn, n;
+ src1.split_code(&vn, &n);
+ int vm, m;
+ src2.split_code(&vm, &m);
+ emit(cond | 0x1C * B23 | d * B22 | 0x3 * B20 | vn * B16 | vd * B12 |
+ 0x5 * B9 | B8 | n * B7 | B6 | m * B5 | vm);
+}
+
+void Assembler::vsub(const SwVfpRegister dst, const SwVfpRegister src1,
+ const SwVfpRegister src2, const Condition cond) {
+ // Sd = vsub(Sn, Sm) single precision floating point subtraction.
+ // Sd = D:Vd; Sm=M:Vm; Sn=N:Vm.
+ // Instruction details available in ARM DDI 0406C.b, A8-1086.
+ // cond(31-28) | 11100(27-23)| D(22) | 11(21-20) | Vn(19-16) |
+ // Vd(15-12) | 101(11-9) | sz=0(8) | N(7) | 1(6) | M(5) | 0(4) | Vm(3-0)
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vn, n;
+ src1.split_code(&vn, &n);
+ int vm, m;
+ src2.split_code(&vm, &m);
+ emit(cond | 0x1C * B23 | d * B22 | 0x3 * B20 | vn * B16 | vd * B12 |
+ 0x5 * B9 | n * B7 | B6 | m * B5 | vm);
+}
+
+void Assembler::vmul(const DwVfpRegister dst, const DwVfpRegister src1,
+ const DwVfpRegister src2, const Condition cond) {
+ // Dd = vmul(Dn, Dm) double precision floating point multiplication.
+ // Dd = D:Vd; Dm=M:Vm; Dn=N:Vm.
+ // Instruction details available in ARM DDI 0406C.b, A8-960.
+ // cond(31-28) | 11100(27-23)| D(22) | 10(21-20) | Vn(19-16) |
+ // Vd(15-12) | 101(11-9) | sz=1(8) | N(7) | 0(6) | M(5) | 0(4) | Vm(3-0)
+ DCHECK(VfpRegisterIsAvailable(dst));
+ DCHECK(VfpRegisterIsAvailable(src1));
+ DCHECK(VfpRegisterIsAvailable(src2));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vn, n;
+ src1.split_code(&vn, &n);
+ int vm, m;
+ src2.split_code(&vm, &m);
+ emit(cond | 0x1C * B23 | d * B22 | 0x2 * B20 | vn * B16 | vd * B12 |
+ 0x5 * B9 | B8 | n * B7 | m * B5 | vm);
+}
+
+void Assembler::vmul(const SwVfpRegister dst, const SwVfpRegister src1,
+ const SwVfpRegister src2, const Condition cond) {
+ // Sd = vmul(Sn, Sm) single precision floating point multiplication.
+ // Sd = D:Vd; Sm=M:Vm; Sn=N:Vm.
+ // Instruction details available in ARM DDI 0406C.b, A8-960.
+ // cond(31-28) | 11100(27-23)| D(22) | 10(21-20) | Vn(19-16) |
+ // Vd(15-12) | 101(11-9) | sz=0(8) | N(7) | 0(6) | M(5) | 0(4) | Vm(3-0)
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vn, n;
+ src1.split_code(&vn, &n);
+ int vm, m;
+ src2.split_code(&vm, &m);
+ emit(cond | 0x1C * B23 | d * B22 | 0x2 * B20 | vn * B16 | vd * B12 |
+ 0x5 * B9 | n * B7 | m * B5 | vm);
+}
+
+void Assembler::vmla(const DwVfpRegister dst, const DwVfpRegister src1,
+ const DwVfpRegister src2, const Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8-932.
+ // cond(31-28) | 11100(27-23) | D(22) | 00(21-20) | Vn(19-16) |
+ // Vd(15-12) | 101(11-9) | sz=1(8) | N(7) | op=0(6) | M(5) | 0(4) | Vm(3-0)
+ DCHECK(VfpRegisterIsAvailable(dst));
+ DCHECK(VfpRegisterIsAvailable(src1));
+ DCHECK(VfpRegisterIsAvailable(src2));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vn, n;
+ src1.split_code(&vn, &n);
+ int vm, m;
+ src2.split_code(&vm, &m);
+ emit(cond | 0x1C * B23 | d * B22 | vn * B16 | vd * B12 | 0x5 * B9 | B8 |
+ n * B7 | m * B5 | vm);
+}
+
+void Assembler::vmla(const SwVfpRegister dst, const SwVfpRegister src1,
+ const SwVfpRegister src2, const Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8-932.
+ // cond(31-28) | 11100(27-23) | D(22) | 00(21-20) | Vn(19-16) |
+ // Vd(15-12) | 101(11-9) | sz=0(8) | N(7) | op=0(6) | M(5) | 0(4) | Vm(3-0)
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vn, n;
+ src1.split_code(&vn, &n);
+ int vm, m;
+ src2.split_code(&vm, &m);
+ emit(cond | 0x1C * B23 | d * B22 | vn * B16 | vd * B12 | 0x5 * B9 | n * B7 |
+ m * B5 | vm);
+}
+
+void Assembler::vmls(const DwVfpRegister dst, const DwVfpRegister src1,
+ const DwVfpRegister src2, const Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8-932.
+ // cond(31-28) | 11100(27-23) | D(22) | 00(21-20) | Vn(19-16) |
+ // Vd(15-12) | 101(11-9) | sz=1(8) | N(7) | op=1(6) | M(5) | 0(4) | Vm(3-0)
+ DCHECK(VfpRegisterIsAvailable(dst));
+ DCHECK(VfpRegisterIsAvailable(src1));
+ DCHECK(VfpRegisterIsAvailable(src2));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vn, n;
+ src1.split_code(&vn, &n);
+ int vm, m;
+ src2.split_code(&vm, &m);
+ emit(cond | 0x1C * B23 | d * B22 | vn * B16 | vd * B12 | 0x5 * B9 | B8 |
+ n * B7 | B6 | m * B5 | vm);
+}
+
+void Assembler::vmls(const SwVfpRegister dst, const SwVfpRegister src1,
+ const SwVfpRegister src2, const Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8-932.
+ // cond(31-28) | 11100(27-23) | D(22) | 00(21-20) | Vn(19-16) |
+ // Vd(15-12) | 101(11-9) | sz=0(8) | N(7) | op=1(6) | M(5) | 0(4) | Vm(3-0)
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vn, n;
+ src1.split_code(&vn, &n);
+ int vm, m;
+ src2.split_code(&vm, &m);
+ emit(cond | 0x1C * B23 | d * B22 | vn * B16 | vd * B12 | 0x5 * B9 | n * B7 |
+ B6 | m * B5 | vm);
+}
+
+void Assembler::vdiv(const DwVfpRegister dst, const DwVfpRegister src1,
+ const DwVfpRegister src2, const Condition cond) {
+ // Dd = vdiv(Dn, Dm) double precision floating point division.
+ // Dd = D:Vd; Dm=M:Vm; Dn=N:Vm.
+ // Instruction details available in ARM DDI 0406C.b, A8-882.
+ // cond(31-28) | 11101(27-23)| D(22) | 00(21-20) | Vn(19-16) |
+ // Vd(15-12) | 101(11-9) | sz=1(8) | N(7) | 0(6) | M(5) | 0(4) | Vm(3-0)
+ DCHECK(VfpRegisterIsAvailable(dst));
+ DCHECK(VfpRegisterIsAvailable(src1));
+ DCHECK(VfpRegisterIsAvailable(src2));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vn, n;
+ src1.split_code(&vn, &n);
+ int vm, m;
+ src2.split_code(&vm, &m);
+ emit(cond | 0x1D * B23 | d * B22 | vn * B16 | vd * B12 | 0x5 * B9 | B8 |
+ n * B7 | m * B5 | vm);
+}
+
+void Assembler::vdiv(const SwVfpRegister dst, const SwVfpRegister src1,
+ const SwVfpRegister src2, const Condition cond) {
+ // Sd = vdiv(Sn, Sm) single precision floating point division.
+ // Sd = D:Vd; Sm=M:Vm; Sn=N:Vm.
+ // Instruction details available in ARM DDI 0406C.b, A8-882.
+ // cond(31-28) | 11101(27-23)| D(22) | 00(21-20) | Vn(19-16) |
+ // Vd(15-12) | 101(11-9) | sz=0(8) | N(7) | 0(6) | M(5) | 0(4) | Vm(3-0)
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vn, n;
+ src1.split_code(&vn, &n);
+ int vm, m;
+ src2.split_code(&vm, &m);
+ emit(cond | 0x1D * B23 | d * B22 | vn * B16 | vd * B12 | 0x5 * B9 | n * B7 |
+ m * B5 | vm);
+}
+
+void Assembler::vcmp(const DwVfpRegister src1, const DwVfpRegister src2,
+ const Condition cond) {
+ // vcmp(Dd, Dm) double precision floating point comparison.
+ // Instruction details available in ARM DDI 0406C.b, A8-864.
+ // cond(31-28) | 11101(27-23)| D(22) | 11(21-20) | 0100(19-16) |
+ // Vd(15-12) | 101(11-9) | sz=1(8) | E=0(7) | 1(6) | M(5) | 0(4) | Vm(3-0)
+ DCHECK(VfpRegisterIsAvailable(src1));
+ DCHECK(VfpRegisterIsAvailable(src2));
+ int vd, d;
+ src1.split_code(&vd, &d);
+ int vm, m;
+ src2.split_code(&vm, &m);
+ emit(cond | 0x1D * B23 | d * B22 | 0x3 * B20 | 0x4 * B16 | vd * B12 |
+ 0x5 * B9 | B8 | B6 | m * B5 | vm);
+}
+
+void Assembler::vcmp(const SwVfpRegister src1, const SwVfpRegister src2,
+ const Condition cond) {
+ // vcmp(Sd, Sm) single precision floating point comparison.
+ // Instruction details available in ARM DDI 0406C.b, A8-864.
+ // cond(31-28) | 11101(27-23)| D(22) | 11(21-20) | 0100(19-16) |
+ // Vd(15-12) | 101(11-9) | sz=0(8) | E=0(7) | 1(6) | M(5) | 0(4) | Vm(3-0)
+ int vd, d;
+ src1.split_code(&vd, &d);
+ int vm, m;
+ src2.split_code(&vm, &m);
+ emit(cond | 0x1D * B23 | d * B22 | 0x3 * B20 | 0x4 * B16 | vd * B12 |
+ 0x5 * B9 | B6 | m * B5 | vm);
+}
+
+void Assembler::vcmp(const DwVfpRegister src1, const double src2,
+ const Condition cond) {
+ // vcmp(Dd, #0.0) double precision floating point comparison.
+ // Instruction details available in ARM DDI 0406C.b, A8-864.
+ // cond(31-28) | 11101(27-23)| D(22) | 11(21-20) | 0101(19-16) |
+ // Vd(15-12) | 101(11-9) | sz=1(8) | E=0(7) | 1(6) | 0(5) | 0(4) | 0000(3-0)
+ DCHECK(VfpRegisterIsAvailable(src1));
+ DCHECK_EQ(src2, 0.0);
+ int vd, d;
+ src1.split_code(&vd, &d);
+ emit(cond | 0x1D * B23 | d * B22 | 0x3 * B20 | 0x5 * B16 | vd * B12 |
+ 0x5 * B9 | B8 | B6);
+}
+
+void Assembler::vcmp(const SwVfpRegister src1, const float src2,
+ const Condition cond) {
+ // vcmp(Sd, #0.0) single precision floating point comparison.
+ // Instruction details available in ARM DDI 0406C.b, A8-864.
+ // cond(31-28) | 11101(27-23)| D(22) | 11(21-20) | 0101(19-16) |
+ // Vd(15-12) | 101(11-9) | sz=0(8) | E=0(7) | 1(6) | 0(5) | 0(4) | 0000(3-0)
+ DCHECK_EQ(src2, 0.0);
+ int vd, d;
+ src1.split_code(&vd, &d);
+ emit(cond | 0x1D * B23 | d * B22 | 0x3 * B20 | 0x5 * B16 | vd * B12 |
+ 0x5 * B9 | B6);
+}
+
+void Assembler::vmaxnm(const DwVfpRegister dst, const DwVfpRegister src1,
+ const DwVfpRegister src2) {
+ // kSpecialCondition(31-28) | 11101(27-23) | D(22) | 00(21-20) | Vn(19-16) |
+ // Vd(15-12) | 101(11-9) | sz=1(8) | N(7) | 0(6) | M(5) | 0(4) | Vm(3-0)
+ DCHECK(IsEnabled(ARMv8));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vn, n;
+ src1.split_code(&vn, &n);
+ int vm, m;
+ src2.split_code(&vm, &m);
+
+ emit(kSpecialCondition | 0x1D * B23 | d * B22 | vn * B16 | vd * B12 |
+ 0x5 * B9 | B8 | n * B7 | m * B5 | vm);
+}
+
+void Assembler::vmaxnm(const SwVfpRegister dst, const SwVfpRegister src1,
+ const SwVfpRegister src2) {
+ // kSpecialCondition(31-28) | 11101(27-23) | D(22) | 00(21-20) | Vn(19-16) |
+ // Vd(15-12) | 101(11-9) | sz=0(8) | N(7) | 0(6) | M(5) | 0(4) | Vm(3-0)
+ DCHECK(IsEnabled(ARMv8));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vn, n;
+ src1.split_code(&vn, &n);
+ int vm, m;
+ src2.split_code(&vm, &m);
+
+ emit(kSpecialCondition | 0x1D * B23 | d * B22 | vn * B16 | vd * B12 |
+ 0x5 * B9 | n * B7 | m * B5 | vm);
+}
+
+void Assembler::vminnm(const DwVfpRegister dst, const DwVfpRegister src1,
+ const DwVfpRegister src2) {
+ // kSpecialCondition(31-28) | 11101(27-23) | D(22) | 00(21-20) | Vn(19-16) |
+ // Vd(15-12) | 101(11-9) | sz=1(8) | N(7) | 1(6) | M(5) | 0(4) | Vm(3-0)
+ DCHECK(IsEnabled(ARMv8));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vn, n;
+ src1.split_code(&vn, &n);
+ int vm, m;
+ src2.split_code(&vm, &m);
+
+ emit(kSpecialCondition | 0x1D * B23 | d * B22 | vn * B16 | vd * B12 |
+ 0x5 * B9 | B8 | n * B7 | B6 | m * B5 | vm);
+}
+
+void Assembler::vminnm(const SwVfpRegister dst, const SwVfpRegister src1,
+ const SwVfpRegister src2) {
+ // kSpecialCondition(31-28) | 11101(27-23) | D(22) | 00(21-20) | Vn(19-16) |
+ // Vd(15-12) | 101(11-9) | sz=0(8) | N(7) | 1(6) | M(5) | 0(4) | Vm(3-0)
+ DCHECK(IsEnabled(ARMv8));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vn, n;
+ src1.split_code(&vn, &n);
+ int vm, m;
+ src2.split_code(&vm, &m);
+
+ emit(kSpecialCondition | 0x1D * B23 | d * B22 | vn * B16 | vd * B12 |
+ 0x5 * B9 | n * B7 | B6 | m * B5 | vm);
+}
+
+void Assembler::vsel(Condition cond, const DwVfpRegister dst,
+ const DwVfpRegister src1, const DwVfpRegister src2) {
+ // cond=kSpecialCondition(31-28) | 11100(27-23) | D(22) |
+ // vsel_cond=XX(21-20) | Vn(19-16) | Vd(15-12) | 101(11-9) | sz=1(8) | N(7) |
+ // 0(6) | M(5) | 0(4) | Vm(3-0)
+ DCHECK(IsEnabled(ARMv8));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vn, n;
+ src1.split_code(&vn, &n);
+ int vm, m;
+ src2.split_code(&vm, &m);
+ int sz = 1;
+
+ // VSEL has a special (restricted) condition encoding.
+ // eq(0b0000)... -> 0b00
+ // ge(0b1010)... -> 0b10
+ // gt(0b1100)... -> 0b11
+ // vs(0b0110)... -> 0b01
+ // No other conditions are supported.
+ int vsel_cond = (cond >> 30) & 0x3;
+ if ((cond != eq) && (cond != ge) && (cond != gt) && (cond != vs)) {
+ // We can implement some other conditions by swapping the inputs.
+ DCHECK((cond == ne) | (cond == lt) | (cond == le) | (cond == vc));
+ std::swap(vn, vm);
+ std::swap(n, m);
+ }
+
+ emit(kSpecialCondition | 0x1C * B23 | d * B22 | vsel_cond * B20 | vn * B16 |
+ vd * B12 | 0x5 * B9 | sz * B8 | n * B7 | m * B5 | vm);
+}
+
+void Assembler::vsel(Condition cond, const SwVfpRegister dst,
+ const SwVfpRegister src1, const SwVfpRegister src2) {
+ // cond=kSpecialCondition(31-28) | 11100(27-23) | D(22) |
+ // vsel_cond=XX(21-20) | Vn(19-16) | Vd(15-12) | 101(11-9) | sz=0(8) | N(7) |
+ // 0(6) | M(5) | 0(4) | Vm(3-0)
+ DCHECK(IsEnabled(ARMv8));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vn, n;
+ src1.split_code(&vn, &n);
+ int vm, m;
+ src2.split_code(&vm, &m);
+ int sz = 0;
+
+ // VSEL has a special (restricted) condition encoding.
+ // eq(0b0000)... -> 0b00
+ // ge(0b1010)... -> 0b10
+ // gt(0b1100)... -> 0b11
+ // vs(0b0110)... -> 0b01
+ // No other conditions are supported.
+ int vsel_cond = (cond >> 30) & 0x3;
+ if ((cond != eq) && (cond != ge) && (cond != gt) && (cond != vs)) {
+ // We can implement some other conditions by swapping the inputs.
+ DCHECK((cond == ne) | (cond == lt) | (cond == le) | (cond == vc));
+ std::swap(vn, vm);
+ std::swap(n, m);
+ }
+
+ emit(kSpecialCondition | 0x1C * B23 | d * B22 | vsel_cond * B20 | vn * B16 |
+ vd * B12 | 0x5 * B9 | sz * B8 | n * B7 | m * B5 | vm);
+}
+
+void Assembler::vsqrt(const DwVfpRegister dst, const DwVfpRegister src,
+ const Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8-1058.
+ // cond(31-28) | 11101(27-23)| D(22) | 11(21-20) | 0001(19-16) |
+ // Vd(15-12) | 101(11-9) | sz=1(8) | 11(7-6) | M(5) | 0(4) | Vm(3-0)
+ DCHECK(VfpRegisterIsAvailable(dst));
+ DCHECK(VfpRegisterIsAvailable(src));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vm, m;
+ src.split_code(&vm, &m);
+ emit(cond | 0x1D * B23 | d * B22 | 0x3 * B20 | B16 | vd * B12 | 0x5 * B9 |
+ B8 | 0x3 * B6 | m * B5 | vm);
+}
+
+void Assembler::vsqrt(const SwVfpRegister dst, const SwVfpRegister src,
+ const Condition cond) {
+ // Instruction details available in ARM DDI 0406C.b, A8-1058.
+ // cond(31-28) | 11101(27-23)| D(22) | 11(21-20) | 0001(19-16) |
+ // Vd(15-12) | 101(11-9) | sz=0(8) | 11(7-6) | M(5) | 0(4) | Vm(3-0)
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vm, m;
+ src.split_code(&vm, &m);
+ emit(cond | 0x1D * B23 | d * B22 | 0x3 * B20 | B16 | vd * B12 | 0x5 * B9 |
+ 0x3 * B6 | m * B5 | vm);
+}
+
+void Assembler::vmsr(Register dst, Condition cond) {
+ // Instruction details available in ARM DDI 0406A, A8-652.
+ // cond(31-28) | 1110 (27-24) | 1110(23-20)| 0001 (19-16) |
+ // Rt(15-12) | 1010 (11-8) | 0(7) | 00 (6-5) | 1(4) | 0000(3-0)
+ emit(cond | 0xE * B24 | 0xE * B20 | B16 | dst.code() * B12 | 0xA * B8 | B4);
+}
+
+void Assembler::vmrs(Register dst, Condition cond) {
+ // Instruction details available in ARM DDI 0406A, A8-652.
+ // cond(31-28) | 1110 (27-24) | 1111(23-20)| 0001 (19-16) |
+ // Rt(15-12) | 1010 (11-8) | 0(7) | 00 (6-5) | 1(4) | 0000(3-0)
+ emit(cond | 0xE * B24 | 0xF * B20 | B16 | dst.code() * B12 | 0xA * B8 | B4);
+}
+
+void Assembler::vrinta(const SwVfpRegister dst, const SwVfpRegister src) {
+ // cond=kSpecialCondition(31-28) | 11101(27-23)| D(22) | 11(21-20) |
+ // 10(19-18) | RM=00(17-16) | Vd(15-12) | 101(11-9) | sz=0(8) | 01(7-6) |
+ // M(5) | 0(4) | Vm(3-0)
+ DCHECK(IsEnabled(ARMv8));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vm, m;
+ src.split_code(&vm, &m);
+ emit(kSpecialCondition | 0x1D * B23 | d * B22 | 0x3 * B20 | B19 | vd * B12 |
+ 0x5 * B9 | B6 | m * B5 | vm);
+}
+
+void Assembler::vrinta(const DwVfpRegister dst, const DwVfpRegister src) {
+ // cond=kSpecialCondition(31-28) | 11101(27-23)| D(22) | 11(21-20) |
+ // 10(19-18) | RM=00(17-16) | Vd(15-12) | 101(11-9) | sz=1(8) | 01(7-6) |
+ // M(5) | 0(4) | Vm(3-0)
+ DCHECK(IsEnabled(ARMv8));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vm, m;
+ src.split_code(&vm, &m);
+ emit(kSpecialCondition | 0x1D * B23 | d * B22 | 0x3 * B20 | B19 | vd * B12 |
+ 0x5 * B9 | B8 | B6 | m * B5 | vm);
+}
+
+void Assembler::vrintn(const SwVfpRegister dst, const SwVfpRegister src) {
+ // cond=kSpecialCondition(31-28) | 11101(27-23)| D(22) | 11(21-20) |
+ // 10(19-18) | RM=01(17-16) | Vd(15-12) | 101(11-9) | sz=0(8) | 01(7-6) |
+ // M(5) | 0(4) | Vm(3-0)
+ DCHECK(IsEnabled(ARMv8));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vm, m;
+ src.split_code(&vm, &m);
+ emit(kSpecialCondition | 0x1D * B23 | d * B22 | 0x3 * B20 | B19 | 0x1 * B16 |
+ vd * B12 | 0x5 * B9 | B6 | m * B5 | vm);
+}
+
+void Assembler::vrintn(const DwVfpRegister dst, const DwVfpRegister src) {
+ // cond=kSpecialCondition(31-28) | 11101(27-23)| D(22) | 11(21-20) |
+ // 10(19-18) | RM=01(17-16) | Vd(15-12) | 101(11-9) | sz=1(8) | 01(7-6) |
+ // M(5) | 0(4) | Vm(3-0)
+ DCHECK(IsEnabled(ARMv8));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vm, m;
+ src.split_code(&vm, &m);
+ emit(kSpecialCondition | 0x1D * B23 | d * B22 | 0x3 * B20 | B19 | 0x1 * B16 |
+ vd * B12 | 0x5 * B9 | B8 | B6 | m * B5 | vm);
+}
+
+void Assembler::vrintp(const SwVfpRegister dst, const SwVfpRegister src) {
+ // cond=kSpecialCondition(31-28) | 11101(27-23)| D(22) | 11(21-20) |
+ // 10(19-18) | RM=10(17-16) | Vd(15-12) | 101(11-9) | sz=0(8) | 01(7-6) |
+ // M(5) | 0(4) | Vm(3-0)
+ DCHECK(IsEnabled(ARMv8));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vm, m;
+ src.split_code(&vm, &m);
+ emit(kSpecialCondition | 0x1D * B23 | d * B22 | 0x3 * B20 | B19 | 0x2 * B16 |
+ vd * B12 | 0x5 * B9 | B6 | m * B5 | vm);
+}
+
+void Assembler::vrintp(const DwVfpRegister dst, const DwVfpRegister src) {
+ // cond=kSpecialCondition(31-28) | 11101(27-23)| D(22) | 11(21-20) |
+ // 10(19-18) | RM=10(17-16) | Vd(15-12) | 101(11-9) | sz=1(8) | 01(7-6) |
+ // M(5) | 0(4) | Vm(3-0)
+ DCHECK(IsEnabled(ARMv8));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vm, m;
+ src.split_code(&vm, &m);
+ emit(kSpecialCondition | 0x1D * B23 | d * B22 | 0x3 * B20 | B19 | 0x2 * B16 |
+ vd * B12 | 0x5 * B9 | B8 | B6 | m * B5 | vm);
+}
+
+void Assembler::vrintm(const SwVfpRegister dst, const SwVfpRegister src) {
+ // cond=kSpecialCondition(31-28) | 11101(27-23)| D(22) | 11(21-20) |
+ // 10(19-18) | RM=11(17-16) | Vd(15-12) | 101(11-9) | sz=0(8) | 01(7-6) |
+ // M(5) | 0(4) | Vm(3-0)
+ DCHECK(IsEnabled(ARMv8));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vm, m;
+ src.split_code(&vm, &m);
+ emit(kSpecialCondition | 0x1D * B23 | d * B22 | 0x3 * B20 | B19 | 0x3 * B16 |
+ vd * B12 | 0x5 * B9 | B6 | m * B5 | vm);
+}
+
+void Assembler::vrintm(const DwVfpRegister dst, const DwVfpRegister src) {
+ // cond=kSpecialCondition(31-28) | 11101(27-23)| D(22) | 11(21-20) |
+ // 10(19-18) | RM=11(17-16) | Vd(15-12) | 101(11-9) | sz=1(8) | 01(7-6) |
+ // M(5) | 0(4) | Vm(3-0)
+ DCHECK(IsEnabled(ARMv8));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vm, m;
+ src.split_code(&vm, &m);
+ emit(kSpecialCondition | 0x1D * B23 | d * B22 | 0x3 * B20 | B19 | 0x3 * B16 |
+ vd * B12 | 0x5 * B9 | B8 | B6 | m * B5 | vm);
+}
+
+void Assembler::vrintz(const SwVfpRegister dst, const SwVfpRegister src,
+ const Condition cond) {
+ // cond(31-28) | 11101(27-23)| D(22) | 11(21-20) | 011(19-17) | 0(16) |
+ // Vd(15-12) | 101(11-9) | sz=0(8) | op=1(7) | 1(6) | M(5) | 0(4) | Vm(3-0)
+ DCHECK(IsEnabled(ARMv8));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vm, m;
+ src.split_code(&vm, &m);
+ emit(cond | 0x1D * B23 | d * B22 | 0x3 * B20 | 0x3 * B17 | vd * B12 |
+ 0x5 * B9 | B7 | B6 | m * B5 | vm);
+}
+
+void Assembler::vrintz(const DwVfpRegister dst, const DwVfpRegister src,
+ const Condition cond) {
+ // cond(31-28) | 11101(27-23)| D(22) | 11(21-20) | 011(19-17) | 0(16) |
+ // Vd(15-12) | 101(11-9) | sz=1(8) | op=1(7) | 1(6) | M(5) | 0(4) | Vm(3-0)
+ DCHECK(IsEnabled(ARMv8));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vm, m;
+ src.split_code(&vm, &m);
+ emit(cond | 0x1D * B23 | d * B22 | 0x3 * B20 | 0x3 * B17 | vd * B12 |
+ 0x5 * B9 | B8 | B7 | B6 | m * B5 | vm);
+}
+
+// Support for NEON.
+
+void Assembler::vld1(NeonSize size, const NeonListOperand& dst,
+ const NeonMemOperand& src) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.320.
+ // 1111(31-28) | 01000(27-23) | D(22) | 10(21-20) | Rn(19-16) |
+ // Vd(15-12) | type(11-8) | size(7-6) | align(5-4) | Rm(3-0)
+ DCHECK(IsEnabled(NEON));
+ int vd, d;
+ dst.base().split_code(&vd, &d);
+ emit(0xFU * B28 | 4 * B24 | d * B22 | 2 * B20 | src.rn().code() * B16 |
+ vd * B12 | dst.type() * B8 | size * B6 | src.align() * B4 |
+ src.rm().code());
+}
+
+// vld1s(ingle element to one lane).
+void Assembler::vld1s(NeonSize size, const NeonListOperand& dst, uint8_t index,
+ const NeonMemOperand& src) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.322.
+ // 1111(31-28) | 01001(27-23) | D(22) | 10(21-20) | Rn(19-16) |
+ // Vd(15-12) | size(11-10) | index_align(7-4) | Rm(3-0)
+ // See vld1 (single element to all lanes) if size == 0x3, implemented as
+ // vld1r(eplicate).
+ DCHECK_NE(size, 0x3);
+ // Check for valid lane indices.
+ DCHECK_GT(1 << (3 - size), index);
+ // Specifying alignment not supported, use standard alignment.
+ uint8_t index_align = index << (size + 1);
+
+ DCHECK(IsEnabled(NEON));
+ int vd, d;
+ dst.base().split_code(&vd, &d);
+ emit(0xFU * B28 | 4 * B24 | 1 * B23 | d * B22 | 2 * B20 |
+ src.rn().code() * B16 | vd * B12 | size * B10 | index_align * B4 |
+ src.rm().code());
+}
+
+// vld1r(eplicate)
+void Assembler::vld1r(NeonSize size, const NeonListOperand& dst,
+ const NeonMemOperand& src) {
+ DCHECK(IsEnabled(NEON));
+ int vd, d;
+ dst.base().split_code(&vd, &d);
+ emit(0xFU * B28 | 4 * B24 | 1 * B23 | d * B22 | 2 * B20 |
+ src.rn().code() * B16 | vd * B12 | 0xC * B8 | size * B6 |
+ dst.length() * B5 | src.rm().code());
+}
+
+void Assembler::vst1(NeonSize size, const NeonListOperand& src,
+ const NeonMemOperand& dst) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.404.
+ // 1111(31-28) | 01000(27-23) | D(22) | 00(21-20) | Rn(19-16) |
+ // Vd(15-12) | type(11-8) | size(7-6) | align(5-4) | Rm(3-0)
+ DCHECK(IsEnabled(NEON));
+ int vd, d;
+ src.base().split_code(&vd, &d);
+ emit(0xFU * B28 | 4 * B24 | d * B22 | dst.rn().code() * B16 | vd * B12 |
+ src.type() * B8 | size * B6 | dst.align() * B4 | dst.rm().code());
+}
+
+void Assembler::vmovl(NeonDataType dt, QwNeonRegister dst, DwVfpRegister src) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.346.
+ // 1111(31-28) | 001(27-25) | U(24) | 1(23) | D(22) | imm3(21-19) |
+ // 000(18-16) | Vd(15-12) | 101000(11-6) | M(5) | 1(4) | Vm(3-0)
+ DCHECK(IsEnabled(NEON));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vm, m;
+ src.split_code(&vm, &m);
+ int U = NeonU(dt);
+ int imm3 = 1 << NeonSz(dt);
+ emit(0xFU * B28 | B25 | U * B24 | B23 | d * B22 | imm3 * B19 | vd * B12 |
+ 0xA * B8 | m * B5 | B4 | vm);
+}
+
+void Assembler::vqmovn(NeonDataType dst_dt, NeonDataType src_dt,
+ DwVfpRegister dst, QwNeonRegister src) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.1004.
+ // vqmovn.<type><size> Dd, Qm. ARM vector narrowing move with saturation.
+ // vqmovun.<type><size> Dd, Qm. Same as above, but produces unsigned results.
+ DCHECK(IsEnabled(NEON));
+ DCHECK_IMPLIES(NeonU(src_dt), NeonU(dst_dt));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vm, m;
+ src.split_code(&vm, &m);
+ int size = NeonSz(dst_dt);
+ int op = NeonU(src_dt) ? 0b11 : NeonU(dst_dt) ? 0b01 : 0b10;
+ emit(0x1E7U * B23 | d * B22 | 0x3 * B20 | size * B18 | 0x2 * B16 | vd * B12 |
+ 0x2 * B8 | op * B6 | m * B5 | vm);
+}
+
+static int EncodeScalar(NeonDataType dt, int index) {
+ int opc1_opc2 = 0;
+ DCHECK_LE(0, index);
+ switch (dt) {
+ case NeonS8:
+ case NeonU8:
+ DCHECK_GT(8, index);
+ opc1_opc2 = 0x8 | index;
+ break;
+ case NeonS16:
+ case NeonU16:
+ DCHECK_GT(4, index);
+ opc1_opc2 = 0x1 | (index << 1);
+ break;
+ case NeonS32:
+ case NeonU32:
+ DCHECK_GT(2, index);
+ opc1_opc2 = index << 2;
+ break;
+ default:
+ UNREACHABLE();
+ }
+ return (opc1_opc2 >> 2) * B21 | (opc1_opc2 & 0x3) * B5;
+}
+
+void Assembler::vmov(NeonDataType dt, DwVfpRegister dst, int index,
+ Register src) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.940.
+ // vmov ARM core register to scalar.
+ DCHECK(dt == NeonS32 || dt == NeonU32 || IsEnabled(NEON));
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int opc1_opc2 = EncodeScalar(dt, index);
+ emit(0xEEu * B24 | vd * B16 | src.code() * B12 | 0xB * B8 | d * B7 | B4 |
+ opc1_opc2);
+}
+
+void Assembler::vmov(NeonDataType dt, Register dst, DwVfpRegister src,
+ int index) {
+ // Instruction details available in ARM DDI 0406C.b, A8.8.942.
+ // vmov Arm scalar to core register.
+ DCHECK(dt == NeonS32 || dt == NeonU32 || IsEnabled(NEON));
+ int vn, n;
+ src.split_code(&vn, &n);
+ int opc1_opc2 = EncodeScalar(dt, index);
+ int u = NeonU(dt);
+ emit(0xEEu * B24 | u * B23 | B20 | vn * B16 | dst.code() * B12 | 0xB * B8 |
+ n * B7 | B4 | opc1_opc2);
+}
+
+void Assembler::vmov(QwNeonRegister dst, QwNeonRegister src) {
+ // Instruction details available in ARM DDI 0406C.b, A8-938.
+ // vmov is encoded as vorr.
+ vorr(dst, src, src);
+}
+
+void Assembler::vdup(NeonSize size, QwNeonRegister dst, Register src) {
+ DCHECK(IsEnabled(NEON));
+ // Instruction details available in ARM DDI 0406C.b, A8-886.
+ int B = 0, E = 0;
+ switch (size) {
+ case Neon8:
+ B = 1;
+ break;
+ case Neon16:
+ E = 1;
+ break;
+ case Neon32:
+ break;
+ default:
+ UNREACHABLE();
+ }
+ int vd, d;
+ dst.split_code(&vd, &d);
+
+ emit(al | 0x1D * B23 | B * B22 | B21 | vd * B16 | src.code() * B12 |
+ 0xB * B8 | d * B7 | E * B5 | B4);
+}
+
+enum NeonRegType { NEON_D, NEON_Q };
+
+void NeonSplitCode(NeonRegType type, int code, int* vm, int* m, int* encoding) {
+ if (type == NEON_D) {
+ DwVfpRegister::split_code(code, vm, m);
+ } else {
+ DCHECK_EQ(type, NEON_Q);
+ QwNeonRegister::split_code(code, vm, m);
+ *encoding |= B6;
+ }
+}
+
+static Instr EncodeNeonDupOp(NeonSize size, NeonRegType reg_type, int dst_code,
+ DwVfpRegister src, int index) {
+ DCHECK_NE(Neon64, size);
+ int sz = static_cast<int>(size);
+ DCHECK_LE(0, index);
+ DCHECK_GT(kSimd128Size / (1 << sz), index);
+ int imm4 = (1 << sz) | ((index << (sz + 1)) & 0xF);
+ int qbit = 0;
+ int vd, d;
+ NeonSplitCode(reg_type, dst_code, &vd, &d, &qbit);
+ int vm, m;
+ src.split_code(&vm, &m);
+
+ return 0x1E7U * B23 | d * B22 | 0x3 * B20 | imm4 * B16 | vd * B12 |
+ 0x18 * B7 | qbit | m * B5 | vm;
+}
+
+void Assembler::vdup(NeonSize size, DwVfpRegister dst, DwVfpRegister src,
+ int index) {
+ DCHECK(IsEnabled(NEON));
+ // Instruction details available in ARM DDI 0406C.b, A8-884.
+ emit(EncodeNeonDupOp(size, NEON_D, dst.code(), src, index));
+}
+
+void Assembler::vdup(NeonSize size, QwNeonRegister dst, DwVfpRegister src,
+ int index) {
+ // Instruction details available in ARM DDI 0406C.b, A8-884.
+ DCHECK(IsEnabled(NEON));
+ emit(EncodeNeonDupOp(size, NEON_Q, dst.code(), src, index));
+}
+
+// Encode NEON vcvt.src_type.dst_type instruction.
+static Instr EncodeNeonVCVT(VFPType dst_type, QwNeonRegister dst,
+ VFPType src_type, QwNeonRegister src) {
+ DCHECK(src_type != dst_type);
+ DCHECK(src_type == F32 || dst_type == F32);
+ // Instruction details available in ARM DDI 0406C.b, A8.8.868.
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vm, m;
+ src.split_code(&vm, &m);
+
+ int op = 0;
+ if (src_type == F32) {
+ DCHECK(dst_type == S32 || dst_type == U32);
+ op = dst_type == U32 ? 3 : 2;
+ } else {
+ DCHECK(src_type == S32 || src_type == U32);
+ op = src_type == U32 ? 1 : 0;
+ }
+
+ return 0x1E7U * B23 | d * B22 | 0x3B * B16 | vd * B12 | 0x3 * B9 | op * B7 |
+ B6 | m * B5 | vm;
+}
+
+void Assembler::vcvt_f32_s32(QwNeonRegister dst, QwNeonRegister src) {
+ DCHECK(IsEnabled(NEON));
+ DCHECK(VfpRegisterIsAvailable(dst));
+ DCHECK(VfpRegisterIsAvailable(src));
+ emit(EncodeNeonVCVT(F32, dst, S32, src));
+}
+
+void Assembler::vcvt_f32_u32(QwNeonRegister dst, QwNeonRegister src) {
+ DCHECK(IsEnabled(NEON));
+ DCHECK(VfpRegisterIsAvailable(dst));
+ DCHECK(VfpRegisterIsAvailable(src));
+ emit(EncodeNeonVCVT(F32, dst, U32, src));
+}
+
+void Assembler::vcvt_s32_f32(QwNeonRegister dst, QwNeonRegister src) {
+ DCHECK(IsEnabled(NEON));
+ DCHECK(VfpRegisterIsAvailable(dst));
+ DCHECK(VfpRegisterIsAvailable(src));
+ emit(EncodeNeonVCVT(S32, dst, F32, src));
+}
+
+void Assembler::vcvt_u32_f32(QwNeonRegister dst, QwNeonRegister src) {
+ DCHECK(IsEnabled(NEON));
+ DCHECK(VfpRegisterIsAvailable(dst));
+ DCHECK(VfpRegisterIsAvailable(src));
+ emit(EncodeNeonVCVT(U32, dst, F32, src));
+}
+
+enum UnaryOp {
+ VMVN,
+ VSWP,
+ VABS,
+ VABSF,
+ VNEG,
+ VNEGF,
+ VRINTM,
+ VRINTN,
+ VRINTP,
+ VRINTZ
+};
+
+static Instr EncodeNeonUnaryOp(UnaryOp op, NeonRegType reg_type, NeonSize size,
+ int dst_code, int src_code) {
+ int op_encoding = 0;
+ switch (op) {
+ case VMVN:
+ DCHECK_EQ(Neon8, size); // size == 0 for vmvn
+ op_encoding = B10 | 0x3 * B7;
+ break;
+ case VSWP:
+ DCHECK_EQ(Neon8, size); // size == 0 for vswp
+ op_encoding = B17;
+ break;
+ case VABS:
+ op_encoding = B16 | 0x6 * B7;
+ break;
+ case VABSF:
+ DCHECK_EQ(Neon32, size);
+ op_encoding = B16 | B10 | 0x6 * B7;
+ break;
+ case VNEG:
+ op_encoding = B16 | 0x7 * B7;
+ break;
+ case VNEGF:
+ DCHECK_EQ(Neon32, size);
+ op_encoding = B16 | B10 | 0x7 * B7;
+ break;
+ case VRINTM:
+ op_encoding = B17 | 0xD * B7;
+ break;
+ case VRINTN:
+ op_encoding = B17 | 0x8 * B7;
+ break;
+ case VRINTP:
+ op_encoding = B17 | 0xF * B7;
+ break;
+ case VRINTZ:
+ op_encoding = B17 | 0xB * B7;
+ break;
+ default:
+ UNREACHABLE();
+ }
+ int vd, d;
+ NeonSplitCode(reg_type, dst_code, &vd, &d, &op_encoding);
+ int vm, m;
+ NeonSplitCode(reg_type, src_code, &vm, &m, &op_encoding);
+
+ return 0x1E7U * B23 | d * B22 | 0x3 * B20 | size * B18 | vd * B12 | m * B5 |
+ vm | op_encoding;
+}
+
+void Assembler::vmvn(QwNeonRegister dst, QwNeonRegister src) {
+ // Qd = vmvn(Qn, Qm) SIMD bitwise negate.
+ // Instruction details available in ARM DDI 0406C.b, A8-966.
+ DCHECK(IsEnabled(NEON));
+ emit(EncodeNeonUnaryOp(VMVN, NEON_Q, Neon8, dst.code(), src.code()));
+}
+
+void Assembler::vswp(DwVfpRegister dst, DwVfpRegister src) {
+ DCHECK(IsEnabled(NEON));
+ // Dd = vswp(Dn, Dm) SIMD d-register swap.
+ // Instruction details available in ARM DDI 0406C.b, A8.8.418.
+ DCHECK(IsEnabled(NEON));
+ emit(EncodeNeonUnaryOp(VSWP, NEON_D, Neon8, dst.code(), src.code()));
+}
+
+void Assembler::vswp(QwNeonRegister dst, QwNeonRegister src) {
+ // Qd = vswp(Qn, Qm) SIMD q-register swap.
+ // Instruction details available in ARM DDI 0406C.b, A8.8.418.
+ DCHECK(IsEnabled(NEON));
+ emit(EncodeNeonUnaryOp(VSWP, NEON_Q, Neon8, dst.code(), src.code()));
+}
+
+void Assembler::vabs(QwNeonRegister dst, QwNeonRegister src) {
+ // Qd = vabs.f<size>(Qn, Qm) SIMD floating point absolute value.
+ // Instruction details available in ARM DDI 0406C.b, A8.8.824.
+ DCHECK(IsEnabled(NEON));
+ emit(EncodeNeonUnaryOp(VABSF, NEON_Q, Neon32, dst.code(), src.code()));
+}
+
+void Assembler::vabs(NeonSize size, QwNeonRegister dst, QwNeonRegister src) {
+ // Qd = vabs.s<size>(Qn, Qm) SIMD integer absolute value.
+ // Instruction details available in ARM DDI 0406C.b, A8.8.824.
+ DCHECK(IsEnabled(NEON));
+ emit(EncodeNeonUnaryOp(VABS, NEON_Q, size, dst.code(), src.code()));
+}
+
+void Assembler::vneg(QwNeonRegister dst, QwNeonRegister src) {
+ // Qd = vabs.f<size>(Qn, Qm) SIMD floating point negate.
+ // Instruction details available in ARM DDI 0406C.b, A8.8.968.
+ DCHECK(IsEnabled(NEON));
+ emit(EncodeNeonUnaryOp(VNEGF, NEON_Q, Neon32, dst.code(), src.code()));
+}
+
+void Assembler::vneg(NeonSize size, QwNeonRegister dst, QwNeonRegister src) {
+ // Qd = vabs.s<size>(Qn, Qm) SIMD integer negate.
+ // Instruction details available in ARM DDI 0406C.b, A8.8.968.
+ DCHECK(IsEnabled(NEON));
+ emit(EncodeNeonUnaryOp(VNEG, NEON_Q, size, dst.code(), src.code()));
+}
+
+enum BinaryBitwiseOp { VAND, VBIC, VBIF, VBIT, VBSL, VEOR, VORR, VORN };
+
+static Instr EncodeNeonBinaryBitwiseOp(BinaryBitwiseOp op, NeonRegType reg_type,
+ int dst_code, int src_code1,
+ int src_code2) {
+ int op_encoding = 0;
+ switch (op) {
+ case VBIC:
+ op_encoding = 0x1 * B20;
+ break;
+ case VBIF:
+ op_encoding = B24 | 0x3 * B20;
+ break;
+ case VBIT:
+ op_encoding = B24 | 0x2 * B20;
+ break;
+ case VBSL:
+ op_encoding = B24 | 0x1 * B20;
+ break;
+ case VEOR:
+ op_encoding = B24;
+ break;
+ case VORR:
+ op_encoding = 0x2 * B20;
+ break;
+ case VORN:
+ op_encoding = 0x3 * B20;
+ break;
+ case VAND:
+ // op_encoding is 0.
+ break;
+ default:
+ UNREACHABLE();
+ }
+ int vd, d;
+ NeonSplitCode(reg_type, dst_code, &vd, &d, &op_encoding);
+ int vn, n;
+ NeonSplitCode(reg_type, src_code1, &vn, &n, &op_encoding);
+ int vm, m;
+ NeonSplitCode(reg_type, src_code2, &vm, &m, &op_encoding);
+
+ return 0x1E4U * B23 | op_encoding | d * B22 | vn * B16 | vd * B12 | B8 |
+ n * B7 | m * B5 | B4 | vm;
+}
+
+void Assembler::vand(QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ // Qd = vand(Qn, Qm) SIMD AND.
+ // Instruction details available in ARM DDI 0406C.b, A8.8.836.
+ DCHECK(IsEnabled(NEON));
+ emit(EncodeNeonBinaryBitwiseOp(VAND, NEON_Q, dst.code(), src1.code(),
+ src2.code()));
+}
+
+void Assembler::vbic(QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ // Qd = vbic(Qn, Qm) SIMD AND.
+ // Instruction details available in ARM DDI 0406C.b, A8-840.
+ DCHECK(IsEnabled(NEON));
+ emit(EncodeNeonBinaryBitwiseOp(VBIC, NEON_Q, dst.code(), src1.code(),
+ src2.code()));
+}
+
+void Assembler::vbsl(QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ // Qd = vbsl(Qn, Qm) SIMD bitwise select.
+ // Instruction details available in ARM DDI 0406C.b, A8-844.
+ DCHECK(IsEnabled(NEON));
+ emit(EncodeNeonBinaryBitwiseOp(VBSL, NEON_Q, dst.code(), src1.code(),
+ src2.code()));
+}
+
+void Assembler::veor(DwVfpRegister dst, DwVfpRegister src1,
+ DwVfpRegister src2) {
+ // Dd = veor(Dn, Dm) SIMD exclusive OR.
+ // Instruction details available in ARM DDI 0406C.b, A8.8.888.
+ DCHECK(IsEnabled(NEON));
+ emit(EncodeNeonBinaryBitwiseOp(VEOR, NEON_D, dst.code(), src1.code(),
+ src2.code()));
+}
+
+void Assembler::veor(QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ // Qd = veor(Qn, Qm) SIMD exclusive OR.
+ // Instruction details available in ARM DDI 0406C.b, A8.8.888.
+ DCHECK(IsEnabled(NEON));
+ emit(EncodeNeonBinaryBitwiseOp(VEOR, NEON_Q, dst.code(), src1.code(),
+ src2.code()));
+}
+
+void Assembler::vorr(QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ // Qd = vorr(Qn, Qm) SIMD OR.
+ // Instruction details available in ARM DDI 0406C.b, A8.8.976.
+ DCHECK(IsEnabled(NEON));
+ emit(EncodeNeonBinaryBitwiseOp(VORR, NEON_Q, dst.code(), src1.code(),
+ src2.code()));
+}
+
+enum FPBinOp {
+ VADDF,
+ VSUBF,
+ VMULF,
+ VMINF,
+ VMAXF,
+ VRECPS,
+ VRSQRTS,
+ VCEQF,
+ VCGEF,
+ VCGTF
+};
+
+static Instr EncodeNeonBinOp(FPBinOp op, QwNeonRegister dst,
+ QwNeonRegister src1, QwNeonRegister src2) {
+ int op_encoding = 0;
+ switch (op) {
+ case VADDF:
+ op_encoding = 0xD * B8;
+ break;
+ case VSUBF:
+ op_encoding = B21 | 0xD * B8;
+ break;
+ case VMULF:
+ op_encoding = B24 | 0xD * B8 | B4;
+ break;
+ case VMINF:
+ op_encoding = B21 | 0xF * B8;
+ break;
+ case VMAXF:
+ op_encoding = 0xF * B8;
+ break;
+ case VRECPS:
+ op_encoding = 0xF * B8 | B4;
+ break;
+ case VRSQRTS:
+ op_encoding = B21 | 0xF * B8 | B4;
+ break;
+ case VCEQF:
+ op_encoding = 0xE * B8;
+ break;
+ case VCGEF:
+ op_encoding = B24 | 0xE * B8;
+ break;
+ case VCGTF:
+ op_encoding = B24 | B21 | 0xE * B8;
+ break;
+ default:
+ UNREACHABLE();
+ }
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vn, n;
+ src1.split_code(&vn, &n);
+ int vm, m;
+ src2.split_code(&vm, &m);
+ return 0x1E4U * B23 | d * B22 | vn * B16 | vd * B12 | n * B7 | B6 | m * B5 |
+ vm | op_encoding;
+}
+
+enum IntegerBinOp {
+ VADD,
+ VQADD,
+ VSUB,
+ VQSUB,
+ VMUL,
+ VMIN,
+ VMAX,
+ VTST,
+ VCEQ,
+ VCGE,
+ VCGT,
+ VRHADD
+};
+
+static Instr EncodeNeonBinOp(IntegerBinOp op, NeonDataType dt,
+ QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ int op_encoding = 0;
+ switch (op) {
+ case VADD:
+ op_encoding = 0x8 * B8;
+ break;
+ case VQADD:
+ op_encoding = B4;
+ break;
+ case VSUB:
+ op_encoding = B24 | 0x8 * B8;
+ break;
+ case VQSUB:
+ op_encoding = 0x2 * B8 | B4;
+ break;
+ case VMUL:
+ op_encoding = 0x9 * B8 | B4;
+ break;
+ case VMIN:
+ op_encoding = 0x6 * B8 | B4;
+ break;
+ case VMAX:
+ op_encoding = 0x6 * B8;
+ break;
+ case VTST:
+ op_encoding = 0x8 * B8 | B4;
+ break;
+ case VCEQ:
+ op_encoding = B24 | 0x8 * B8 | B4;
+ break;
+ case VCGE:
+ op_encoding = 0x3 * B8 | B4;
+ break;
+ case VCGT:
+ op_encoding = 0x3 * B8;
+ break;
+ case VRHADD:
+ op_encoding = B8;
+ break;
+ default:
+ UNREACHABLE();
+ }
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vn, n;
+ src1.split_code(&vn, &n);
+ int vm, m;
+ src2.split_code(&vm, &m);
+ int size = NeonSz(dt);
+ int u = NeonU(dt);
+ return 0x1E4U * B23 | u * B24 | d * B22 | size * B20 | vn * B16 | vd * B12 |
+ n * B7 | B6 | m * B5 | vm | op_encoding;
+}
+
+static Instr EncodeNeonBinOp(IntegerBinOp op, NeonSize size, QwNeonRegister dst,
+ QwNeonRegister src1, QwNeonRegister src2) {
+ // Map NeonSize values to the signed values in NeonDataType, so the U bit
+ // will be 0.
+ return EncodeNeonBinOp(op, static_cast<NeonDataType>(size), dst, src1, src2);
+}
+
+void Assembler::vadd(QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vadd(Qn, Qm) SIMD floating point addition.
+ // Instruction details available in ARM DDI 0406C.b, A8-830.
+ emit(EncodeNeonBinOp(VADDF, dst, src1, src2));
+}
+
+void Assembler::vadd(NeonSize size, QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vadd(Qn, Qm) SIMD integer addition.
+ // Instruction details available in ARM DDI 0406C.b, A8-828.
+ emit(EncodeNeonBinOp(VADD, size, dst, src1, src2));
+}
+
+void Assembler::vqadd(NeonDataType dt, QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vqadd(Qn, Qm) SIMD integer saturating addition.
+ // Instruction details available in ARM DDI 0406C.b, A8-996.
+ emit(EncodeNeonBinOp(VQADD, dt, dst, src1, src2));
+}
+
+void Assembler::vsub(QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vsub(Qn, Qm) SIMD floating point subtraction.
+ // Instruction details available in ARM DDI 0406C.b, A8-1086.
+ emit(EncodeNeonBinOp(VSUBF, dst, src1, src2));
+}
+
+void Assembler::vsub(NeonSize size, QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vsub(Qn, Qm) SIMD integer subtraction.
+ // Instruction details available in ARM DDI 0406C.b, A8-1084.
+ emit(EncodeNeonBinOp(VSUB, size, dst, src1, src2));
+}
+
+void Assembler::vqsub(NeonDataType dt, QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vqsub(Qn, Qm) SIMD integer saturating subtraction.
+ // Instruction details available in ARM DDI 0406C.b, A8-1020.
+ emit(EncodeNeonBinOp(VQSUB, dt, dst, src1, src2));
+}
+
+void Assembler::vmlal(NeonDataType dt, QwNeonRegister dst, DwVfpRegister src1,
+ DwVfpRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vmlal(Dn, Dm) Vector Multiply Accumulate Long (integer)
+ // Instruction details available in ARM DDI 0406C.b, A8-931.
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vn, n;
+ src1.split_code(&vn, &n);
+ int vm, m;
+ src2.split_code(&vm, &m);
+ int size = NeonSz(dt);
+ int u = NeonU(dt);
+ if (!u) UNIMPLEMENTED();
+ DCHECK_NE(size, 3); // SEE "Related encodings"
+ emit(0xFU * B28 | B25 | u * B24 | B23 | d * B22 | size * B20 | vn * B16 |
+ vd * B12 | 0x8 * B8 | n * B7 | m * B5 | vm);
+}
+
+void Assembler::vmul(QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vadd(Qn, Qm) SIMD floating point multiply.
+ // Instruction details available in ARM DDI 0406C.b, A8-958.
+ emit(EncodeNeonBinOp(VMULF, dst, src1, src2));
+}
+
+void Assembler::vmul(NeonSize size, QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vadd(Qn, Qm) SIMD integer multiply.
+ // Instruction details available in ARM DDI 0406C.b, A8-960.
+ emit(EncodeNeonBinOp(VMUL, size, dst, src1, src2));
+}
+
+void Assembler::vmull(NeonDataType dt, QwNeonRegister dst, DwVfpRegister src1,
+ DwVfpRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vmull(Dn, Dm) Vector Multiply Long (integer).
+ // Instruction details available in ARM DDI 0406C.b, A8-960.
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vn, n;
+ src1.split_code(&vn, &n);
+ int vm, m;
+ src2.split_code(&vm, &m);
+ int size = NeonSz(dt);
+ int u = NeonU(dt);
+ emit(0xFU * B28 | B25 | u * B24 | B23 | d * B22 | size * B20 | vn * B16 |
+ vd * B12 | 0xC * B8 | n * B7 | m * B5 | vm);
+}
+
+void Assembler::vmin(QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vmin(Qn, Qm) SIMD floating point MIN.
+ // Instruction details available in ARM DDI 0406C.b, A8-928.
+ emit(EncodeNeonBinOp(VMINF, dst, src1, src2));
+}
+
+void Assembler::vmin(NeonDataType dt, QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vmin(Qn, Qm) SIMD integer MIN.
+ // Instruction details available in ARM DDI 0406C.b, A8-926.
+ emit(EncodeNeonBinOp(VMIN, dt, dst, src1, src2));
+}
+
+void Assembler::vmax(QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vmax(Qn, Qm) SIMD floating point MAX.
+ // Instruction details available in ARM DDI 0406C.b, A8-928.
+ emit(EncodeNeonBinOp(VMAXF, dst, src1, src2));
+}
+
+void Assembler::vmax(NeonDataType dt, QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vmax(Qn, Qm) SIMD integer MAX.
+ // Instruction details available in ARM DDI 0406C.b, A8-926.
+ emit(EncodeNeonBinOp(VMAX, dt, dst, src1, src2));
+}
+
+enum NeonShiftOp { VSHL, VSHR, VSLI, VSRI };
+
+static Instr EncodeNeonShiftRegisterOp(NeonShiftOp op, NeonDataType dt,
+ NeonRegType reg_type, int dst_code,
+ int src_code, int shift_code) {
+ DCHECK_EQ(op, VSHL);
+ int op_encoding = 0;
+ int vd, d;
+ NeonSplitCode(reg_type, dst_code, &vd, &d, &op_encoding);
+ int vm, m;
+ NeonSplitCode(reg_type, src_code, &vm, &m, &op_encoding);
+ int vn, n;
+ NeonSplitCode(reg_type, shift_code, &vn, &n, &op_encoding);
+ int size = NeonSz(dt);
+ int u = NeonU(dt);
+
+ return 0x1E4U * B23 | u * B24 | d * B22 | size * B20 | vn * B16 | vd * B12 |
+ 0x4 * B8 | n * B7 | m * B5 | vm | op_encoding;
+}
+
+static Instr EncodeNeonShiftOp(NeonShiftOp op, NeonSize size, bool is_unsigned,
+ NeonRegType reg_type, int dst_code, int src_code,
+ int shift) {
+ int size_in_bits = kBitsPerByte << static_cast<int>(size);
+ int op_encoding = 0, imm6 = 0, L = 0;
+ switch (op) {
+ case VSHL: {
+ DCHECK(shift >= 0 && size_in_bits > shift);
+ imm6 = size_in_bits + shift;
+ op_encoding = 0x5 * B8;
+ break;
+ }
+ case VSHR: {
+ DCHECK(shift > 0 && size_in_bits >= shift);
+ imm6 = 2 * size_in_bits - shift;
+ if (is_unsigned) op_encoding |= B24;
+ break;
+ }
+ case VSLI: {
+ DCHECK(shift >= 0 && size_in_bits > shift);
+ imm6 = size_in_bits + shift;
+ op_encoding = B24 | 0x5 * B8;
+ break;
+ }
+ case VSRI: {
+ DCHECK(shift > 0 && size_in_bits >= shift);
+ imm6 = 2 * size_in_bits - shift;
+ op_encoding = B24 | 0x4 * B8;
+ break;
+ }
+ default:
+ UNREACHABLE();
+ }
+
+ L = imm6 >> 6;
+ imm6 &= 0x3F;
+
+ int vd, d;
+ NeonSplitCode(reg_type, dst_code, &vd, &d, &op_encoding);
+ int vm, m;
+ NeonSplitCode(reg_type, src_code, &vm, &m, &op_encoding);
+
+ return 0x1E5U * B23 | d * B22 | imm6 * B16 | vd * B12 | L * B7 | m * B5 | B4 |
+ vm | op_encoding;
+}
+
+void Assembler::vshl(NeonDataType dt, QwNeonRegister dst, QwNeonRegister src,
+ int shift) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vshl(Qm, bits) SIMD shift left immediate.
+ // Instruction details available in ARM DDI 0406C.b, A8-1046.
+ emit(EncodeNeonShiftOp(VSHL, NeonDataTypeToSize(dt), false, NEON_Q,
+ dst.code(), src.code(), shift));
+}
+
+void Assembler::vshl(NeonDataType dt, QwNeonRegister dst, QwNeonRegister src,
+ QwNeonRegister shift) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vshl(Qm, Qn) SIMD shift left Register.
+ // Instruction details available in ARM DDI 0487A.a, F8-3340..
+ emit(EncodeNeonShiftRegisterOp(VSHL, dt, NEON_Q, dst.code(), src.code(),
+ shift.code()));
+}
+
+void Assembler::vshr(NeonDataType dt, QwNeonRegister dst, QwNeonRegister src,
+ int shift) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vshl(Qm, bits) SIMD shift right immediate.
+ // Instruction details available in ARM DDI 0406C.b, A8-1052.
+ emit(EncodeNeonShiftOp(VSHR, NeonDataTypeToSize(dt), NeonU(dt), NEON_Q,
+ dst.code(), src.code(), shift));
+}
+
+void Assembler::vsli(NeonSize size, DwVfpRegister dst, DwVfpRegister src,
+ int shift) {
+ DCHECK(IsEnabled(NEON));
+ // Dd = vsli(Dm, bits) SIMD shift left and insert.
+ // Instruction details available in ARM DDI 0406C.b, A8-1056.
+ emit(EncodeNeonShiftOp(VSLI, size, false, NEON_D, dst.code(), src.code(),
+ shift));
+}
+
+void Assembler::vsri(NeonSize size, DwVfpRegister dst, DwVfpRegister src,
+ int shift) {
+ DCHECK(IsEnabled(NEON));
+ // Dd = vsri(Dm, bits) SIMD shift right and insert.
+ // Instruction details available in ARM DDI 0406C.b, A8-1062.
+ emit(EncodeNeonShiftOp(VSRI, size, false, NEON_D, dst.code(), src.code(),
+ shift));
+}
+
+static Instr EncodeNeonEstimateOp(bool is_rsqrt, QwNeonRegister dst,
+ QwNeonRegister src) {
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vm, m;
+ src.split_code(&vm, &m);
+ int rsqrt = is_rsqrt ? 1 : 0;
+ return 0x1E7U * B23 | d * B22 | 0x3B * B16 | vd * B12 | 0x5 * B8 |
+ rsqrt * B7 | B6 | m * B5 | vm;
+}
+
+void Assembler::vrecpe(QwNeonRegister dst, QwNeonRegister src) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vrecpe(Qm) SIMD reciprocal estimate.
+ // Instruction details available in ARM DDI 0406C.b, A8-1024.
+ emit(EncodeNeonEstimateOp(false, dst, src));
+}
+
+void Assembler::vrsqrte(QwNeonRegister dst, QwNeonRegister src) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vrsqrte(Qm) SIMD reciprocal square root estimate.
+ // Instruction details available in ARM DDI 0406C.b, A8-1038.
+ emit(EncodeNeonEstimateOp(true, dst, src));
+}
+
+void Assembler::vrecps(QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vrecps(Qn, Qm) SIMD reciprocal refinement step.
+ // Instruction details available in ARM DDI 0406C.b, A8-1026.
+ emit(EncodeNeonBinOp(VRECPS, dst, src1, src2));
+}
+
+void Assembler::vrsqrts(QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vrsqrts(Qn, Qm) SIMD reciprocal square root refinement step.
+ // Instruction details available in ARM DDI 0406C.b, A8-1040.
+ emit(EncodeNeonBinOp(VRSQRTS, dst, src1, src2));
+}
+
+enum NeonPairwiseOp { VPADD, VPMIN, VPMAX };
+
+static Instr EncodeNeonPairwiseOp(NeonPairwiseOp op, NeonDataType dt,
+ DwVfpRegister dst, DwVfpRegister src1,
+ DwVfpRegister src2) {
+ int op_encoding = 0;
+ switch (op) {
+ case VPADD:
+ op_encoding = 0xB * B8 | B4;
+ break;
+ case VPMIN:
+ op_encoding = 0xA * B8 | B4;
+ break;
+ case VPMAX:
+ op_encoding = 0xA * B8;
+ break;
+ default:
+ UNREACHABLE();
+ }
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vn, n;
+ src1.split_code(&vn, &n);
+ int vm, m;
+ src2.split_code(&vm, &m);
+ int size = NeonSz(dt);
+ int u = NeonU(dt);
+ return 0x1E4U * B23 | u * B24 | d * B22 | size * B20 | vn * B16 | vd * B12 |
+ n * B7 | m * B5 | vm | op_encoding;
+}
+
+void Assembler::vpadd(DwVfpRegister dst, DwVfpRegister src1,
+ DwVfpRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Dd = vpadd(Dn, Dm) SIMD integer pairwise ADD.
+ // Instruction details available in ARM DDI 0406C.b, A8-982.
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vn, n;
+ src1.split_code(&vn, &n);
+ int vm, m;
+ src2.split_code(&vm, &m);
+
+ emit(0x1E6U * B23 | d * B22 | vn * B16 | vd * B12 | 0xD * B8 | n * B7 |
+ m * B5 | vm);
+}
+
+void Assembler::vpadd(NeonSize size, DwVfpRegister dst, DwVfpRegister src1,
+ DwVfpRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Dd = vpadd(Dn, Dm) SIMD integer pairwise ADD.
+ // Instruction details available in ARM DDI 0406C.b, A8-980.
+ emit(EncodeNeonPairwiseOp(VPADD, NeonSizeToDataType(size), dst, src1, src2));
+}
+
+void Assembler::vpmin(NeonDataType dt, DwVfpRegister dst, DwVfpRegister src1,
+ DwVfpRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Dd = vpmin(Dn, Dm) SIMD integer pairwise MIN.
+ // Instruction details available in ARM DDI 0406C.b, A8-986.
+ emit(EncodeNeonPairwiseOp(VPMIN, dt, dst, src1, src2));
+}
+
+void Assembler::vpmax(NeonDataType dt, DwVfpRegister dst, DwVfpRegister src1,
+ DwVfpRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Dd = vpmax(Dn, Dm) SIMD integer pairwise MAX.
+ // Instruction details available in ARM DDI 0406C.b, A8-986.
+ emit(EncodeNeonPairwiseOp(VPMAX, dt, dst, src1, src2));
+}
+
+void Assembler::vrintm(NeonDataType dt, const QwNeonRegister dst,
+ const QwNeonRegister src) {
+ // SIMD vector round floating-point to integer towards -Infinity.
+ // See ARM DDI 0487F.b, F6-5493.
+ DCHECK(IsEnabled(ARMv8));
+ emit(EncodeNeonUnaryOp(VRINTM, NEON_Q, NeonSize(dt), dst.code(), src.code()));
+}
+
+void Assembler::vrintn(NeonDataType dt, const QwNeonRegister dst,
+ const QwNeonRegister src) {
+ // SIMD vector round floating-point to integer to Nearest.
+ // See ARM DDI 0487F.b, F6-5497.
+ DCHECK(IsEnabled(ARMv8));
+ emit(EncodeNeonUnaryOp(VRINTN, NEON_Q, NeonSize(dt), dst.code(), src.code()));
+}
+
+void Assembler::vrintp(NeonDataType dt, const QwNeonRegister dst,
+ const QwNeonRegister src) {
+ // SIMD vector round floating-point to integer towards +Infinity.
+ // See ARM DDI 0487F.b, F6-5501.
+ DCHECK(IsEnabled(ARMv8));
+ emit(EncodeNeonUnaryOp(VRINTP, NEON_Q, NeonSize(dt), dst.code(), src.code()));
+}
+
+void Assembler::vrintz(NeonDataType dt, const QwNeonRegister dst,
+ const QwNeonRegister src) {
+ // SIMD vector round floating-point to integer towards Zero.
+ // See ARM DDI 0487F.b, F6-5511.
+ DCHECK(IsEnabled(ARMv8));
+ emit(EncodeNeonUnaryOp(VRINTZ, NEON_Q, NeonSize(dt), dst.code(), src.code()));
+}
+
+void Assembler::vtst(NeonSize size, QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vtst(Qn, Qm) SIMD test integer operands.
+ // Instruction details available in ARM DDI 0406C.b, A8-1098.
+ emit(EncodeNeonBinOp(VTST, size, dst, src1, src2));
+}
+
+void Assembler::vceq(QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vceq(Qn, Qm) SIMD floating point compare equal.
+ // Instruction details available in ARM DDI 0406C.b, A8-844.
+ emit(EncodeNeonBinOp(VCEQF, dst, src1, src2));
+}
+
+void Assembler::vceq(NeonSize size, QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vceq(Qn, Qm) SIMD integer compare equal.
+ // Instruction details available in ARM DDI 0406C.b, A8-844.
+ emit(EncodeNeonBinOp(VCEQ, size, dst, src1, src2));
+}
+
+void Assembler::vcge(QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vcge(Qn, Qm) SIMD floating point compare greater or equal.
+ // Instruction details available in ARM DDI 0406C.b, A8-848.
+ emit(EncodeNeonBinOp(VCGEF, dst, src1, src2));
+}
+
+void Assembler::vcge(NeonDataType dt, QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vcge(Qn, Qm) SIMD integer compare greater or equal.
+ // Instruction details available in ARM DDI 0406C.b, A8-848.
+ emit(EncodeNeonBinOp(VCGE, dt, dst, src1, src2));
+}
+
+void Assembler::vcgt(QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vcgt(Qn, Qm) SIMD floating point compare greater than.
+ // Instruction details available in ARM DDI 0406C.b, A8-852.
+ emit(EncodeNeonBinOp(VCGTF, dst, src1, src2));
+}
+
+void Assembler::vcgt(NeonDataType dt, QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vcgt(Qn, Qm) SIMD integer compare greater than.
+ // Instruction details available in ARM DDI 0406C.b, A8-852.
+ emit(EncodeNeonBinOp(VCGT, dt, dst, src1, src2));
+}
+
+void Assembler::vrhadd(NeonDataType dt, QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vrhadd(Qn, Qm) SIMD integer rounding halving add.
+ // Instruction details available in ARM DDI 0406C.b, A8-1030.
+ emit(EncodeNeonBinOp(VRHADD, dt, dst, src1, src2));
+}
+
+void Assembler::vext(QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2, int bytes) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vext(Qn, Qm) SIMD byte extract.
+ // Instruction details available in ARM DDI 0406C.b, A8-890.
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vn, n;
+ src1.split_code(&vn, &n);
+ int vm, m;
+ src2.split_code(&vm, &m);
+ DCHECK_GT(16, bytes);
+ emit(0x1E5U * B23 | d * B22 | 0x3 * B20 | vn * B16 | vd * B12 | bytes * B8 |
+ n * B7 | B6 | m * B5 | vm);
+}
+
+enum NeonSizedOp { VZIP, VUZP, VREV16, VREV32, VREV64, VTRN };
+
+static Instr EncodeNeonSizedOp(NeonSizedOp op, NeonRegType reg_type,
+ NeonSize size, int dst_code, int src_code) {
+ int op_encoding = 0;
+ switch (op) {
+ case VZIP:
+ op_encoding = 0x2 * B16 | 0x3 * B7;
+ break;
+ case VUZP:
+ op_encoding = 0x2 * B16 | 0x2 * B7;
+ break;
+ case VREV16:
+ op_encoding = 0x2 * B7;
+ break;
+ case VREV32:
+ op_encoding = 0x1 * B7;
+ break;
+ case VREV64:
+ // op_encoding is 0;
+ break;
+ case VTRN:
+ op_encoding = 0x2 * B16 | B7;
+ break;
+ default:
+ UNREACHABLE();
+ }
+ int vd, d;
+ NeonSplitCode(reg_type, dst_code, &vd, &d, &op_encoding);
+ int vm, m;
+ NeonSplitCode(reg_type, src_code, &vm, &m, &op_encoding);
+
+ int sz = static_cast<int>(size);
+ return 0x1E7U * B23 | d * B22 | 0x3 * B20 | sz * B18 | vd * B12 | m * B5 |
+ vm | op_encoding;
+}
+
+void Assembler::vzip(NeonSize size, DwVfpRegister src1, DwVfpRegister src2) {
+ if (size == Neon32) { // vzip.32 Dd, Dm is a pseudo-op for vtrn.32 Dd, Dm.
+ vtrn(size, src1, src2);
+ } else {
+ DCHECK(IsEnabled(NEON));
+ // vzip.<size>(Dn, Dm) SIMD zip (interleave).
+ // Instruction details available in ARM DDI 0406C.b, A8-1102.
+ emit(EncodeNeonSizedOp(VZIP, NEON_D, size, src1.code(), src2.code()));
+ }
+}
+
+void Assembler::vzip(NeonSize size, QwNeonRegister src1, QwNeonRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // vzip.<size>(Qn, Qm) SIMD zip (interleave).
+ // Instruction details available in ARM DDI 0406C.b, A8-1102.
+ emit(EncodeNeonSizedOp(VZIP, NEON_Q, size, src1.code(), src2.code()));
+}
+
+void Assembler::vuzp(NeonSize size, DwVfpRegister src1, DwVfpRegister src2) {
+ if (size == Neon32) { // vuzp.32 Dd, Dm is a pseudo-op for vtrn.32 Dd, Dm.
+ vtrn(size, src1, src2);
+ } else {
+ DCHECK(IsEnabled(NEON));
+ // vuzp.<size>(Dn, Dm) SIMD un-zip (de-interleave).
+ // Instruction details available in ARM DDI 0406C.b, A8-1100.
+ emit(EncodeNeonSizedOp(VUZP, NEON_D, size, src1.code(), src2.code()));
+ }
+}
+
+void Assembler::vuzp(NeonSize size, QwNeonRegister src1, QwNeonRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // vuzp.<size>(Qn, Qm) SIMD un-zip (de-interleave).
+ // Instruction details available in ARM DDI 0406C.b, A8-1100.
+ emit(EncodeNeonSizedOp(VUZP, NEON_Q, size, src1.code(), src2.code()));
+}
+
+void Assembler::vrev16(NeonSize size, QwNeonRegister dst, QwNeonRegister src) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vrev16.<size>(Qm) SIMD element reverse.
+ // Instruction details available in ARM DDI 0406C.b, A8-1028.
+ emit(EncodeNeonSizedOp(VREV16, NEON_Q, size, dst.code(), src.code()));
+}
+
+void Assembler::vrev32(NeonSize size, QwNeonRegister dst, QwNeonRegister src) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vrev32.<size>(Qm) SIMD element reverse.
+ // Instruction details available in ARM DDI 0406C.b, A8-1028.
+ emit(EncodeNeonSizedOp(VREV32, NEON_Q, size, dst.code(), src.code()));
+}
+
+void Assembler::vrev64(NeonSize size, QwNeonRegister dst, QwNeonRegister src) {
+ DCHECK(IsEnabled(NEON));
+ // Qd = vrev64.<size>(Qm) SIMD element reverse.
+ // Instruction details available in ARM DDI 0406C.b, A8-1028.
+ emit(EncodeNeonSizedOp(VREV64, NEON_Q, size, dst.code(), src.code()));
+}
+
+void Assembler::vtrn(NeonSize size, DwVfpRegister src1, DwVfpRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // vtrn.<size>(Dn, Dm) SIMD element transpose.
+ // Instruction details available in ARM DDI 0406C.b, A8-1096.
+ emit(EncodeNeonSizedOp(VTRN, NEON_D, size, src1.code(), src2.code()));
+}
+
+void Assembler::vtrn(NeonSize size, QwNeonRegister src1, QwNeonRegister src2) {
+ DCHECK(IsEnabled(NEON));
+ // vtrn.<size>(Qn, Qm) SIMD element transpose.
+ // Instruction details available in ARM DDI 0406C.b, A8-1096.
+ emit(EncodeNeonSizedOp(VTRN, NEON_Q, size, src1.code(), src2.code()));
+}
+
+// Encode NEON vtbl / vtbx instruction.
+static Instr EncodeNeonVTB(DwVfpRegister dst, const NeonListOperand& list,
+ DwVfpRegister index, bool vtbx) {
+ // Dd = vtbl(table, Dm) SIMD vector permute, zero at out of range indices.
+ // Instruction details available in ARM DDI 0406C.b, A8-1094.
+ // Dd = vtbx(table, Dm) SIMD vector permute, skip out of range indices.
+ // Instruction details available in ARM DDI 0406C.b, A8-1094.
+ int vd, d;
+ dst.split_code(&vd, &d);
+ int vn, n;
+ list.base().split_code(&vn, &n);
+ int vm, m;
+ index.split_code(&vm, &m);
+ int op = vtbx ? 1 : 0; // vtbl = 0, vtbx = 1.
+ return 0x1E7U * B23 | d * B22 | 0x3 * B20 | vn * B16 | vd * B12 | 0x2 * B10 |
+ list.length() * B8 | n * B7 | op * B6 | m * B5 | vm;
+}
+
+void Assembler::vtbl(DwVfpRegister dst, const NeonListOperand& list,
+ DwVfpRegister index) {
+ DCHECK(IsEnabled(NEON));
+ emit(EncodeNeonVTB(dst, list, index, false));
+}
+
+void Assembler::vtbx(DwVfpRegister dst, const NeonListOperand& list,
+ DwVfpRegister index) {
+ DCHECK(IsEnabled(NEON));
+ emit(EncodeNeonVTB(dst, list, index, true));
+}
+
+// Pseudo instructions.
+void Assembler::nop(int type) {
+ // ARMv6{K/T2} and v7 have an actual NOP instruction but it serializes
+ // some of the CPU's pipeline and has to issue. Older ARM chips simply used
+ // MOV Rx, Rx as NOP and it performs better even in newer CPUs.
+ // We therefore use MOV Rx, Rx, even on newer CPUs, and use Rx to encode
+ // a type.
+ DCHECK(0 <= type && type <= 14); // mov pc, pc isn't a nop.
+ emit(al | 13 * B21 | type * B12 | type);
+}
+
+void Assembler::pop() { add(sp, sp, Operand(kPointerSize)); }
+
+bool Assembler::IsMovT(Instr instr) {
+ instr &= ~(((kNumberOfConditions - 1) << 28) | // Mask off conditions
+ ((kNumRegisters - 1) * B12) | // mask out register
+ EncodeMovwImmediate(0xFFFF)); // mask out immediate value
+ return instr == kMovtPattern;
+}
+
+bool Assembler::IsMovW(Instr instr) {
+ instr &= ~(((kNumberOfConditions - 1) << 28) | // Mask off conditions
+ ((kNumRegisters - 1) * B12) | // mask out destination
+ EncodeMovwImmediate(0xFFFF)); // mask out immediate value
+ return instr == kMovwPattern;
+}
+
+Instr Assembler::GetMovTPattern() { return kMovtPattern; }
+
+Instr Assembler::GetMovWPattern() { return kMovwPattern; }
+
+Instr Assembler::EncodeMovwImmediate(uint32_t immediate) {
+ DCHECK_LT(immediate, 0x10000);
+ return ((immediate & 0xF000) << 4) | (immediate & 0xFFF);
+}
+
+Instr Assembler::PatchMovwImmediate(Instr instruction, uint32_t immediate) {
+ instruction &= ~EncodeMovwImmediate(0xFFFF);
+ return instruction | EncodeMovwImmediate(immediate);
+}
+
+int Assembler::DecodeShiftImm(Instr instr) {
+ int rotate = Instruction::RotateValue(instr) * 2;
+ int immed8 = Instruction::Immed8Value(instr);
+ return base::bits::RotateRight32(immed8, rotate);
+}
+
+Instr Assembler::PatchShiftImm(Instr instr, int immed) {
+ uint32_t rotate_imm = 0;
+ uint32_t immed_8 = 0;
+ bool immed_fits = FitsShifter(immed, &rotate_imm, &immed_8, nullptr);
+ DCHECK(immed_fits);
+ USE(immed_fits);
+ return (instr & ~kOff12Mask) | (rotate_imm << 8) | immed_8;
+}
+
+bool Assembler::IsNop(Instr instr, int type) {
+ DCHECK(0 <= type && type <= 14); // mov pc, pc isn't a nop.
+ // Check for mov rx, rx where x = type.
+ return instr == (al | 13 * B21 | type * B12 | type);
+}
+
+bool Assembler::IsMovImmed(Instr instr) {
+ return (instr & kMovImmedMask) == kMovImmedPattern;
+}
+
+bool Assembler::IsOrrImmed(Instr instr) {
+ return (instr & kOrrImmedMask) == kOrrImmedPattern;
+}
+
+// static
+bool Assembler::ImmediateFitsAddrMode1Instruction(int32_t imm32) {
+ uint32_t dummy1;
+ uint32_t dummy2;
+ return FitsShifter(imm32, &dummy1, &dummy2, nullptr);
+}
+
+bool Assembler::ImmediateFitsAddrMode2Instruction(int32_t imm32) {
+ return is_uint12(abs(imm32));
+}
+
+// Debugging.
+void Assembler::RecordConstPool(int size) {
+ // We only need this for debugger support, to correctly compute offsets in the
+ // code.
+ RecordRelocInfo(RelocInfo::CONST_POOL, static_cast<intptr_t>(size));
+}
+
+void Assembler::GrowBuffer() {
+ DCHECK_EQ(buffer_start_, buffer_->start());
+
+ // Compute new buffer size.
+ int old_size = buffer_->size();
+ int new_size = std::min(2 * old_size, old_size + 1 * MB);
+
+ // Some internal data structures overflow for very large buffers,
+ // they must ensure that kMaximalBufferSize is not too large.
+ if (new_size > kMaximalBufferSize) {
+ V8::FatalProcessOutOfMemory(nullptr, "Assembler::GrowBuffer");
+ }
+
+ // Set up new buffer.
+ std::unique_ptr<AssemblerBuffer> new_buffer = buffer_->Grow(new_size);
+ DCHECK_EQ(new_size, new_buffer->size());
+ byte* new_start = new_buffer->start();
+
+ // Copy the data.
+ int pc_delta = new_start - buffer_start_;
+ int rc_delta = (new_start + new_size) - (buffer_start_ + old_size);
+ size_t reloc_size = (buffer_start_ + old_size) - reloc_info_writer.pos();
+ MemMove(new_start, buffer_start_, pc_offset());
+ byte* new_reloc_start = reinterpret_cast<byte*>(
+ reinterpret_cast<Address>(reloc_info_writer.pos()) + rc_delta);
+ MemMove(new_reloc_start, reloc_info_writer.pos(), reloc_size);
+
+ // Switch buffers.
+ buffer_ = std::move(new_buffer);
+ buffer_start_ = new_start;
+ pc_ = reinterpret_cast<byte*>(reinterpret_cast<Address>(pc_) + pc_delta);
+ byte* new_last_pc = reinterpret_cast<byte*>(
+ reinterpret_cast<Address>(reloc_info_writer.last_pc()) + pc_delta);
+ reloc_info_writer.Reposition(new_reloc_start, new_last_pc);
+
+ // None of our relocation types are pc relative pointing outside the code
+ // buffer nor pc absolute pointing inside the code buffer, so there is no need
+ // to relocate any emitted relocation entries.
+}
+
+void Assembler::db(uint8_t data) {
+ // db is used to write raw data. The constant pool should be emitted or
+ // blocked before using db.
+ DCHECK(is_const_pool_blocked() || pending_32_bit_constants_.empty());
+ CheckBuffer();
+ *reinterpret_cast<uint8_t*>(pc_) = data;
+ pc_ += sizeof(uint8_t);
+}
+
+void Assembler::dd(uint32_t data) {
+ // dd is used to write raw data. The constant pool should be emitted or
+ // blocked before using dd.
+ DCHECK(is_const_pool_blocked() || pending_32_bit_constants_.empty());
+ CheckBuffer();
+ base::WriteUnalignedValue(reinterpret_cast<Address>(pc_), data);
+ pc_ += sizeof(uint32_t);
+}
+
+void Assembler::dq(uint64_t value) {
+ // dq is used to write raw data. The constant pool should be emitted or
+ // blocked before using dq.
+ DCHECK(is_const_pool_blocked() || pending_32_bit_constants_.empty());
+ CheckBuffer();
+ base::WriteUnalignedValue(reinterpret_cast<Address>(pc_), value);
+ pc_ += sizeof(uint64_t);
+}
+
+void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) {
+ if (!ShouldRecordRelocInfo(rmode)) return;
+ DCHECK_GE(buffer_space(), kMaxRelocSize); // too late to grow buffer here
+ RelocInfo rinfo(reinterpret_cast<Address>(pc_), rmode, data, Code());
+ reloc_info_writer.Write(&rinfo);
+}
+
+void Assembler::ConstantPoolAddEntry(int position, RelocInfo::Mode rmode,
+ intptr_t value) {
+ DCHECK(rmode != RelocInfo::CONST_POOL);
+ // We can share CODE_TARGETs and embedded objects, but we must make sure we
+ // only emit one reloc info for them (thus delta patching will apply the delta
+ // only once). At the moment, we do not deduplicate heap object request which
+ // are indicated by value == 0.
+ bool sharing_ok = RelocInfo::IsShareableRelocMode(rmode) ||
+ (rmode == RelocInfo::CODE_TARGET && value != 0) ||
+ (RelocInfo::IsEmbeddedObjectMode(rmode) && value != 0);
+ DCHECK_LT(pending_32_bit_constants_.size(), kMaxNumPending32Constants);
+ if (pending_32_bit_constants_.empty()) {
+ first_const_pool_32_use_ = position;
+ }
+ ConstantPoolEntry entry(position, value, sharing_ok, rmode);
+
+ bool shared = false;
+ if (sharing_ok) {
+ // Merge the constant, if possible.
+ for (size_t i = 0; i < pending_32_bit_constants_.size(); i++) {
+ ConstantPoolEntry& current_entry = pending_32_bit_constants_[i];
+ if (!current_entry.sharing_ok()) continue;
+ if (entry.value() == current_entry.value() &&
+ entry.rmode() == current_entry.rmode()) {
+ entry.set_merged_index(i);
+ shared = true;
+ break;
+ }
+ }
+ }
+
+ pending_32_bit_constants_.push_back(entry);
+
+ // Make sure the constant pool is not emitted in place of the next
+ // instruction for which we just recorded relocation info.
+ BlockConstPoolFor(1);
+
+ // Emit relocation info.
+ if (MustOutputRelocInfo(rmode, this) && !shared) {
+ RecordRelocInfo(rmode);
+ }
+}
+
+void Assembler::BlockConstPoolFor(int instructions) {
+ int pc_limit = pc_offset() + instructions * kInstrSize;
+ if (no_const_pool_before_ < pc_limit) {
+ // Max pool start (if we need a jump and an alignment).
+#ifdef DEBUG
+ int start = pc_limit + kInstrSize + 2 * kPointerSize;
+ DCHECK(pending_32_bit_constants_.empty() ||
+ (start < first_const_pool_32_use_ + kMaxDistToIntPool));
+#endif
+ no_const_pool_before_ = pc_limit;
+ }
+
+ if (next_buffer_check_ < no_const_pool_before_) {
+ next_buffer_check_ = no_const_pool_before_;
+ }
+}
+
+void Assembler::CheckConstPool(bool force_emit, bool require_jump) {
+ // Some short sequence of instruction mustn't be broken up by constant pool
+ // emission, such sequences are protected by calls to BlockConstPoolFor and
+ // BlockConstPoolScope.
+ if (is_const_pool_blocked()) {
+ // Something is wrong if emission is forced and blocked at the same time.
+ DCHECK(!force_emit);
+ return;
+ }
+
+ // There is nothing to do if there are no pending constant pool entries.
+ if (pending_32_bit_constants_.empty()) {
+ // Calculate the offset of the next check.
+ next_buffer_check_ = pc_offset() + kCheckPoolInterval;
+ return;
+ }
+
+ // Check that the code buffer is large enough before emitting the constant
+ // pool (include the jump over the pool and the constant pool marker and
+ // the gap to the relocation information).
+ int jump_instr = require_jump ? kInstrSize : 0;
+ int size_up_to_marker = jump_instr + kInstrSize;
+ int estimated_size_after_marker =
+ pending_32_bit_constants_.size() * kPointerSize;
+ int estimated_size = size_up_to_marker + estimated_size_after_marker;
+
+ // We emit a constant pool when:
+ // * requested to do so by parameter force_emit (e.g. after each function).
+ // * the distance from the first instruction accessing the constant pool to
+ // any of the constant pool entries will exceed its limit the next
+ // time the pool is checked. This is overly restrictive, but we don't emit
+ // constant pool entries in-order so it's conservatively correct.
+ // * the instruction doesn't require a jump after itself to jump over the
+ // constant pool, and we're getting close to running out of range.
+ if (!force_emit) {
+ DCHECK(!pending_32_bit_constants_.empty());
+ bool need_emit = false;
+ int dist32 = pc_offset() + estimated_size - first_const_pool_32_use_;
+ if ((dist32 >= kMaxDistToIntPool - kCheckPoolInterval) ||
+ (!require_jump && (dist32 >= kMaxDistToIntPool / 2))) {
+ need_emit = true;
+ }
+ if (!need_emit) return;
+ }
+
+ // Deduplicate constants.
+ int size_after_marker = estimated_size_after_marker;
+
+ for (size_t i = 0; i < pending_32_bit_constants_.size(); i++) {
+ ConstantPoolEntry& entry = pending_32_bit_constants_[i];
+ if (entry.is_merged()) size_after_marker -= kPointerSize;
+ }
+
+ int size = size_up_to_marker + size_after_marker;
+
+ int needed_space = size + kGap;
+ while (buffer_space() <= needed_space) GrowBuffer();
+
+ {
+ // Block recursive calls to CheckConstPool.
+ BlockConstPoolScope block_const_pool(this);
+ RecordComment("[ Constant Pool");
+ RecordConstPool(size);
+
+ Label size_check;
+ bind(&size_check);
+
+ // Emit jump over constant pool if necessary.
+ Label after_pool;
+ if (require_jump) {
+ b(&after_pool);
+ }
+
+ // Put down constant pool marker "Undefined instruction".
+ // The data size helps disassembly know what to print.
+ emit(kConstantPoolMarker |
+ EncodeConstantPoolLength(size_after_marker / kPointerSize));
+
+ // Emit 32-bit constant pool entries.
+ for (size_t i = 0; i < pending_32_bit_constants_.size(); i++) {
+ ConstantPoolEntry& entry = pending_32_bit_constants_[i];
+ Instr instr = instr_at(entry.position());
+
+ // 64-bit loads shouldn't get here.
+ DCHECK(!IsVldrDPcImmediateOffset(instr));
+ DCHECK(!IsMovW(instr));
+ DCHECK(IsLdrPcImmediateOffset(instr) &&
+ GetLdrRegisterImmediateOffset(instr) == 0);
+
+ int delta = pc_offset() - entry.position() - Instruction::kPcLoadDelta;
+ DCHECK(is_uint12(delta));
+ // 0 is the smallest delta:
+ // ldr rd, [pc, #0]
+ // constant pool marker
+ // data
+
+ if (entry.is_merged()) {
+ DCHECK(entry.sharing_ok());
+ ConstantPoolEntry& merged =
+ pending_32_bit_constants_[entry.merged_index()];
+ DCHECK(entry.value() == merged.value());
+ Instr merged_instr = instr_at(merged.position());
+ DCHECK(IsLdrPcImmediateOffset(merged_instr));
+ delta = GetLdrRegisterImmediateOffset(merged_instr);
+ delta += merged.position() - entry.position();
+ }
+ instr_at_put(entry.position(),
+ SetLdrRegisterImmediateOffset(instr, delta));
+ if (!entry.is_merged()) {
+ emit(entry.value());
+ }
+ }
+
+ pending_32_bit_constants_.clear();
+
+ first_const_pool_32_use_ = -1;
+
+ RecordComment("]");
+
+ DCHECK_EQ(size, SizeOfCodeGeneratedSince(&size_check));
+
+ if (after_pool.is_linked()) {
+ bind(&after_pool);
+ }
+ }
+
+ // Since a constant pool was just emitted, move the check offset forward by
+ // the standard interval.
+ next_buffer_check_ = pc_offset() + kCheckPoolInterval;
+}
+
+PatchingAssembler::PatchingAssembler(const AssemblerOptions& options,
+ byte* address, int instructions)
+ : Assembler(options, ExternalAssemblerBuffer(
+ address, instructions * kInstrSize + kGap)) {
+ DCHECK_EQ(reloc_info_writer.pos(), buffer_start_ + buffer_->size());
+}
+
+PatchingAssembler::~PatchingAssembler() {
+ // Check that we don't have any pending constant pools.
+ DCHECK(pending_32_bit_constants_.empty());
+
+ // Check that the code was patched as expected.
+ DCHECK_EQ(pc_, buffer_start_ + buffer_->size() - kGap);
+ DCHECK_EQ(reloc_info_writer.pos(), buffer_start_ + buffer_->size());
+}
+
+void PatchingAssembler::Emit(Address addr) { emit(static_cast<Instr>(addr)); }
+
+void PatchingAssembler::PadWithNops() {
+ DCHECK_LE(pc_, buffer_start_ + buffer_->size() - kGap);
+ while (pc_ < buffer_start_ + buffer_->size() - kGap) {
+ nop();
+ }
+}
+
+UseScratchRegisterScope::UseScratchRegisterScope(Assembler* assembler)
+ : assembler_(assembler),
+ old_available_(*assembler->GetScratchRegisterList()),
+ old_available_vfp_(*assembler->GetScratchVfpRegisterList()) {}
+
+UseScratchRegisterScope::~UseScratchRegisterScope() {
+ *assembler_->GetScratchRegisterList() = old_available_;
+ *assembler_->GetScratchVfpRegisterList() = old_available_vfp_;
+}
+
+Register UseScratchRegisterScope::Acquire() {
+ RegList* available = assembler_->GetScratchRegisterList();
+ DCHECK_NOT_NULL(available);
+ DCHECK_NE(*available, 0);
+ int index = static_cast<int>(base::bits::CountTrailingZeros32(*available));
+ Register reg = Register::from_code(index);
+ *available &= ~reg.bit();
+ return reg;
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_ARM
diff --git a/src/codegen/arm/assembler-arm.h b/src/codegen/arm/assembler-arm.h
new file mode 100644
index 0000000..cb8b762
--- /dev/null
+++ b/src/codegen/arm/assembler-arm.h
@@ -0,0 +1,1385 @@
+// Copyright (c) 1994-2006 Sun Microsystems Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//
+// - Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// - Redistribution in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the
+// distribution.
+//
+// - Neither the name of Sun Microsystems or the names of contributors may
+// be used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+// COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+// OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// The original source code covered by the above license above has been
+// modified significantly by Google Inc.
+// Copyright 2012 the V8 project authors. All rights reserved.
+
+// A light-weight ARM Assembler
+// Generates user mode instructions for the ARM architecture up to version 5
+
+#ifndef V8_CODEGEN_ARM_ASSEMBLER_ARM_H_
+#define V8_CODEGEN_ARM_ASSEMBLER_ARM_H_
+
+#include <stdio.h>
+#include <memory>
+#include <vector>
+
+#include "src/codegen/arm/constants-arm.h"
+#include "src/codegen/arm/register-arm.h"
+#include "src/codegen/assembler.h"
+#include "src/codegen/constant-pool.h"
+#include "src/numbers/double.h"
+#include "src/utils/boxed-float.h"
+
+namespace v8 {
+namespace internal {
+
+class SafepointTableBuilder;
+
+// Coprocessor number
+enum Coprocessor {
+ p0 = 0,
+ p1 = 1,
+ p2 = 2,
+ p3 = 3,
+ p4 = 4,
+ p5 = 5,
+ p6 = 6,
+ p7 = 7,
+ p8 = 8,
+ p9 = 9,
+ p10 = 10,
+ p11 = 11,
+ p12 = 12,
+ p13 = 13,
+ p14 = 14,
+ p15 = 15
+};
+
+// -----------------------------------------------------------------------------
+// Machine instruction Operands
+
+// Class Operand represents a shifter operand in data processing instructions
+class V8_EXPORT_PRIVATE Operand {
+ public:
+ // immediate
+ V8_INLINE explicit Operand(int32_t immediate,
+ RelocInfo::Mode rmode = RelocInfo::NONE)
+ : rmode_(rmode) {
+ value_.immediate = immediate;
+ }
+ V8_INLINE static Operand Zero();
+ V8_INLINE explicit Operand(const ExternalReference& f);
+ explicit Operand(Handle<HeapObject> handle);
+ V8_INLINE explicit Operand(Smi value);
+
+ // rm
+ V8_INLINE explicit Operand(Register rm);
+
+ // rm <shift_op> shift_imm
+ explicit Operand(Register rm, ShiftOp shift_op, int shift_imm);
+ V8_INLINE static Operand SmiUntag(Register rm) {
+ return Operand(rm, ASR, kSmiTagSize);
+ }
+ V8_INLINE static Operand PointerOffsetFromSmiKey(Register key) {
+ STATIC_ASSERT(kSmiTag == 0 && kSmiTagSize < kPointerSizeLog2);
+ return Operand(key, LSL, kPointerSizeLog2 - kSmiTagSize);
+ }
+ V8_INLINE static Operand DoubleOffsetFromSmiKey(Register key) {
+ STATIC_ASSERT(kSmiTag == 0 && kSmiTagSize < kDoubleSizeLog2);
+ return Operand(key, LSL, kDoubleSizeLog2 - kSmiTagSize);
+ }
+
+ // rm <shift_op> rs
+ explicit Operand(Register rm, ShiftOp shift_op, Register rs);
+
+ static Operand EmbeddedNumber(double number); // Smi or HeapNumber.
+ static Operand EmbeddedStringConstant(const StringConstantBase* str);
+
+ // Return true if this is a register operand.
+ bool IsRegister() const {
+ return rm_.is_valid() && rs_ == no_reg && shift_op_ == LSL &&
+ shift_imm_ == 0;
+ }
+ // Return true if this is a register operand shifted with an immediate.
+ bool IsImmediateShiftedRegister() const {
+ return rm_.is_valid() && !rs_.is_valid();
+ }
+ // Return true if this is a register operand shifted with a register.
+ bool IsRegisterShiftedRegister() const {
+ return rm_.is_valid() && rs_.is_valid();
+ }
+
+ // Return the number of actual instructions required to implement the given
+ // instruction for this particular operand. This can be a single instruction,
+ // if no load into a scratch register is necessary, or anything between 2 and
+ // 4 instructions when we need to load from the constant pool (depending upon
+ // whether the constant pool entry is in the small or extended section). If
+ // the instruction this operand is used for is a MOV or MVN instruction the
+ // actual instruction to use is required for this calculation. For other
+ // instructions instr is ignored.
+ //
+ // The value returned is only valid as long as no entries are added to the
+ // constant pool between this call and the actual instruction being emitted.
+ int InstructionsRequired(const Assembler* assembler, Instr instr = 0) const;
+ bool MustOutputRelocInfo(const Assembler* assembler) const;
+
+ inline int32_t immediate() const {
+ DCHECK(IsImmediate());
+ DCHECK(!IsHeapObjectRequest());
+ return value_.immediate;
+ }
+ bool IsImmediate() const { return !rm_.is_valid(); }
+
+ HeapObjectRequest heap_object_request() const {
+ DCHECK(IsHeapObjectRequest());
+ return value_.heap_object_request;
+ }
+ bool IsHeapObjectRequest() const {
+ DCHECK_IMPLIES(is_heap_object_request_, IsImmediate());
+ DCHECK_IMPLIES(is_heap_object_request_,
+ rmode_ == RelocInfo::FULL_EMBEDDED_OBJECT ||
+ rmode_ == RelocInfo::CODE_TARGET);
+ return is_heap_object_request_;
+ }
+
+ Register rm() const { return rm_; }
+ Register rs() const { return rs_; }
+ ShiftOp shift_op() const { return shift_op_; }
+
+ private:
+ Register rm_ = no_reg;
+ Register rs_ = no_reg;
+ ShiftOp shift_op_;
+ int shift_imm_; // valid if rm_ != no_reg && rs_ == no_reg
+ union Value {
+ Value() {}
+ HeapObjectRequest heap_object_request; // if is_heap_object_request_
+ int32_t immediate; // otherwise
+ } value_; // valid if rm_ == no_reg
+ bool is_heap_object_request_ = false;
+ RelocInfo::Mode rmode_;
+
+ friend class Assembler;
+};
+
+// Class MemOperand represents a memory operand in load and store instructions
+class V8_EXPORT_PRIVATE MemOperand {
+ public:
+ // [rn +/- offset] Offset/NegOffset
+ // [rn +/- offset]! PreIndex/NegPreIndex
+ // [rn], +/- offset PostIndex/NegPostIndex
+ // offset is any signed 32-bit value; offset is first loaded to a scratch
+ // register if it does not fit the addressing mode (12-bit unsigned and sign
+ // bit)
+ explicit MemOperand(Register rn, int32_t offset = 0, AddrMode am = Offset);
+
+ // [rn +/- rm] Offset/NegOffset
+ // [rn +/- rm]! PreIndex/NegPreIndex
+ // [rn], +/- rm PostIndex/NegPostIndex
+ explicit MemOperand(Register rn, Register rm, AddrMode am = Offset);
+
+ // [rn +/- rm <shift_op> shift_imm] Offset/NegOffset
+ // [rn +/- rm <shift_op> shift_imm]! PreIndex/NegPreIndex
+ // [rn], +/- rm <shift_op> shift_imm PostIndex/NegPostIndex
+ explicit MemOperand(Register rn, Register rm, ShiftOp shift_op, int shift_imm,
+ AddrMode am = Offset);
+ V8_INLINE static MemOperand PointerAddressFromSmiKey(Register array,
+ Register key,
+ AddrMode am = Offset) {
+ STATIC_ASSERT(kSmiTag == 0 && kSmiTagSize < kPointerSizeLog2);
+ return MemOperand(array, key, LSL, kPointerSizeLog2 - kSmiTagSize, am);
+ }
+
+ void set_offset(int32_t offset) {
+ DCHECK(rm_ == no_reg);
+ offset_ = offset;
+ }
+
+ uint32_t offset() const {
+ DCHECK(rm_ == no_reg);
+ return offset_;
+ }
+
+ Register rn() const { return rn_; }
+ Register rm() const { return rm_; }
+ AddrMode am() const { return am_; }
+
+ bool OffsetIsUint12Encodable() const {
+ return offset_ >= 0 ? is_uint12(offset_) : is_uint12(-offset_);
+ }
+
+ private:
+ Register rn_; // base
+ Register rm_; // register offset
+ int32_t offset_; // valid if rm_ == no_reg
+ ShiftOp shift_op_;
+ int shift_imm_; // valid if rm_ != no_reg && rs_ == no_reg
+ AddrMode am_; // bits P, U, and W
+
+ friend class Assembler;
+};
+
+// Class NeonMemOperand represents a memory operand in load and
+// store NEON instructions
+class V8_EXPORT_PRIVATE NeonMemOperand {
+ public:
+ // [rn {:align}] Offset
+ // [rn {:align}]! PostIndex
+ explicit NeonMemOperand(Register rn, AddrMode am = Offset, int align = 0);
+
+ // [rn {:align}], rm PostIndex
+ explicit NeonMemOperand(Register rn, Register rm, int align = 0);
+
+ Register rn() const { return rn_; }
+ Register rm() const { return rm_; }
+ int align() const { return align_; }
+
+ private:
+ void SetAlignment(int align);
+
+ Register rn_; // base
+ Register rm_; // register increment
+ int align_;
+};
+
+// Class NeonListOperand represents a list of NEON registers
+class NeonListOperand {
+ public:
+ explicit NeonListOperand(DoubleRegister base, int register_count = 1)
+ : base_(base), register_count_(register_count) {}
+ explicit NeonListOperand(QwNeonRegister q_reg)
+ : base_(q_reg.low()), register_count_(2) {}
+ DoubleRegister base() const { return base_; }
+ int register_count() { return register_count_; }
+ int length() const { return register_count_ - 1; }
+ NeonListType type() const {
+ switch (register_count_) {
+ default:
+ UNREACHABLE();
+ // Fall through.
+ case 1:
+ return nlt_1;
+ case 2:
+ return nlt_2;
+ case 3:
+ return nlt_3;
+ case 4:
+ return nlt_4;
+ }
+ }
+
+ private:
+ DoubleRegister base_;
+ int register_count_;
+};
+
+class V8_EXPORT_PRIVATE Assembler : public AssemblerBase {
+ public:
+ // Create an assembler. Instructions and relocation information are emitted
+ // into a buffer, with the instructions starting from the beginning and the
+ // relocation information starting from the end of the buffer. See CodeDesc
+ // for a detailed comment on the layout (globals.h).
+ //
+ // If the provided buffer is nullptr, the assembler allocates and grows its
+ // own buffer. Otherwise it takes ownership of the provided buffer.
+ explicit Assembler(const AssemblerOptions&,
+ std::unique_ptr<AssemblerBuffer> = {});
+
+ ~Assembler() override;
+
+ void AbortedCodeGeneration() override { pending_32_bit_constants_.clear(); }
+
+ // GetCode emits any pending (non-emitted) code and fills the descriptor desc.
+ static constexpr int kNoHandlerTable = 0;
+ static constexpr SafepointTableBuilder* kNoSafepointTable = nullptr;
+ void GetCode(Isolate* isolate, CodeDesc* desc,
+ SafepointTableBuilder* safepoint_table_builder,
+ int handler_table_offset);
+
+ // Convenience wrapper for code without safepoint or handler tables.
+ void GetCode(Isolate* isolate, CodeDesc* desc) {
+ GetCode(isolate, desc, kNoSafepointTable, kNoHandlerTable);
+ }
+
+ // Label operations & relative jumps (PPUM Appendix D)
+ //
+ // Takes a branch opcode (cc) and a label (L) and generates
+ // either a backward branch or a forward branch and links it
+ // to the label fixup chain. Usage:
+ //
+ // Label L; // unbound label
+ // j(cc, &L); // forward branch to unbound label
+ // bind(&L); // bind label to the current pc
+ // j(cc, &L); // backward branch to bound label
+ // bind(&L); // illegal: a label may be bound only once
+ //
+ // Note: The same Label can be used for forward and backward branches
+ // but it may be bound only once.
+
+ void bind(Label* L); // binds an unbound label L to the current code position
+
+ // Returns the branch offset to the given label from the current code position
+ // Links the label to the current position if it is still unbound
+ // Manages the jump elimination optimization if the second parameter is true.
+ int branch_offset(Label* L);
+
+ // Returns true if the given pc address is the start of a constant pool load
+ // instruction sequence.
+ V8_INLINE static bool is_constant_pool_load(Address pc);
+
+ // Return the address in the constant pool of the code target address used by
+ // the branch/call instruction at pc, or the object in a mov.
+ V8_INLINE static Address constant_pool_entry_address(Address pc,
+ Address constant_pool);
+
+ // Read/Modify the code target address in the branch/call instruction at pc.
+ // The isolate argument is unused (and may be nullptr) when skipping flushing.
+ V8_INLINE static Address target_address_at(Address pc, Address constant_pool);
+ V8_INLINE static void set_target_address_at(
+ Address pc, Address constant_pool, Address target,
+ ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED);
+
+ // This sets the branch destination (which is in the constant pool on ARM).
+ // This is for calls and branches within generated code.
+ inline static void deserialization_set_special_target_at(
+ Address constant_pool_entry, Code code, Address target);
+
+ // Get the size of the special target encoded at 'location'.
+ inline static int deserialization_special_target_size(Address location);
+
+ // This sets the internal reference at the pc.
+ inline static void deserialization_set_target_internal_reference_at(
+ Address pc, Address target,
+ RelocInfo::Mode mode = RelocInfo::INTERNAL_REFERENCE);
+
+ // Here we are patching the address in the constant pool, not the actual call
+ // instruction. The address in the constant pool is the same size as a
+ // pointer.
+ static constexpr int kSpecialTargetSize = kPointerSize;
+
+ RegList* GetScratchRegisterList() { return &scratch_register_list_; }
+ VfpRegList* GetScratchVfpRegisterList() {
+ return &scratch_vfp_register_list_;
+ }
+
+ // ---------------------------------------------------------------------------
+ // Code generation
+
+ // Insert the smallest number of nop instructions
+ // possible to align the pc offset to a multiple
+ // of m. m must be a power of 2 (>= 4).
+ void Align(int m);
+ // Insert the smallest number of zero bytes possible to align the pc offset
+ // to a mulitple of m. m must be a power of 2 (>= 2).
+ void DataAlign(int m);
+ // Aligns code to something that's optimal for a jump target for the platform.
+ void CodeTargetAlign();
+
+ // Branch instructions
+ void b(int branch_offset, Condition cond = al,
+ RelocInfo::Mode rmode = RelocInfo::NONE);
+ void bl(int branch_offset, Condition cond = al,
+ RelocInfo::Mode rmode = RelocInfo::NONE);
+ void blx(int branch_offset); // v5 and above
+ void blx(Register target, Condition cond = al); // v5 and above
+ void bx(Register target, Condition cond = al); // v5 and above, plus v4t
+
+ // Convenience branch instructions using labels
+ void b(Label* L, Condition cond = al);
+ void b(Condition cond, Label* L) { b(L, cond); }
+ void bl(Label* L, Condition cond = al);
+ void bl(Condition cond, Label* L) { bl(L, cond); }
+ void blx(Label* L); // v5 and above
+
+ // Data-processing instructions
+
+ void and_(Register dst, Register src1, const Operand& src2, SBit s = LeaveCC,
+ Condition cond = al);
+ void and_(Register dst, Register src1, Register src2, SBit s = LeaveCC,
+ Condition cond = al);
+
+ void eor(Register dst, Register src1, const Operand& src2, SBit s = LeaveCC,
+ Condition cond = al);
+ void eor(Register dst, Register src1, Register src2, SBit s = LeaveCC,
+ Condition cond = al);
+
+ void sub(Register dst, Register src1, const Operand& src2, SBit s = LeaveCC,
+ Condition cond = al);
+ void sub(Register dst, Register src1, Register src2, SBit s = LeaveCC,
+ Condition cond = al);
+
+ void rsb(Register dst, Register src1, const Operand& src2, SBit s = LeaveCC,
+ Condition cond = al);
+
+ void add(Register dst, Register src1, const Operand& src2, SBit s = LeaveCC,
+ Condition cond = al);
+ void add(Register dst, Register src1, Register src2, SBit s = LeaveCC,
+ Condition cond = al);
+
+ void adc(Register dst, Register src1, const Operand& src2, SBit s = LeaveCC,
+ Condition cond = al);
+
+ void sbc(Register dst, Register src1, const Operand& src2, SBit s = LeaveCC,
+ Condition cond = al);
+
+ void rsc(Register dst, Register src1, const Operand& src2, SBit s = LeaveCC,
+ Condition cond = al);
+
+ void tst(Register src1, const Operand& src2, Condition cond = al);
+ void tst(Register src1, Register src2, Condition cond = al);
+
+ void teq(Register src1, const Operand& src2, Condition cond = al);
+
+ void cmp(Register src1, const Operand& src2, Condition cond = al);
+ void cmp(Register src1, Register src2, Condition cond = al);
+
+ void cmp_raw_immediate(Register src1, int raw_immediate, Condition cond = al);
+
+ void cmn(Register src1, const Operand& src2, Condition cond = al);
+
+ void orr(Register dst, Register src1, const Operand& src2, SBit s = LeaveCC,
+ Condition cond = al);
+ void orr(Register dst, Register src1, Register src2, SBit s = LeaveCC,
+ Condition cond = al);
+
+ void mov(Register dst, const Operand& src, SBit s = LeaveCC,
+ Condition cond = al);
+ void mov(Register dst, Register src, SBit s = LeaveCC, Condition cond = al);
+
+ // Load the position of the label relative to the generated code object
+ // pointer in a register.
+ void mov_label_offset(Register dst, Label* label);
+
+ // ARMv7 instructions for loading a 32 bit immediate in two instructions.
+ // The constant for movw and movt should be in the range 0-0xffff.
+ void movw(Register reg, uint32_t immediate, Condition cond = al);
+ void movt(Register reg, uint32_t immediate, Condition cond = al);
+
+ void bic(Register dst, Register src1, const Operand& src2, SBit s = LeaveCC,
+ Condition cond = al);
+
+ void mvn(Register dst, const Operand& src, SBit s = LeaveCC,
+ Condition cond = al);
+
+ // Shift instructions
+
+ void asr(Register dst, Register src1, const Operand& src2, SBit s = LeaveCC,
+ Condition cond = al);
+
+ void lsl(Register dst, Register src1, const Operand& src2, SBit s = LeaveCC,
+ Condition cond = al);
+
+ void lsr(Register dst, Register src1, const Operand& src2, SBit s = LeaveCC,
+ Condition cond = al);
+
+ // Multiply instructions
+
+ void mla(Register dst, Register src1, Register src2, Register srcA,
+ SBit s = LeaveCC, Condition cond = al);
+
+ void mls(Register dst, Register src1, Register src2, Register srcA,
+ Condition cond = al);
+
+ void sdiv(Register dst, Register src1, Register src2, Condition cond = al);
+
+ void udiv(Register dst, Register src1, Register src2, Condition cond = al);
+
+ void mul(Register dst, Register src1, Register src2, SBit s = LeaveCC,
+ Condition cond = al);
+
+ void smmla(Register dst, Register src1, Register src2, Register srcA,
+ Condition cond = al);
+
+ void smmul(Register dst, Register src1, Register src2, Condition cond = al);
+
+ void smlal(Register dstL, Register dstH, Register src1, Register src2,
+ SBit s = LeaveCC, Condition cond = al);
+
+ void smull(Register dstL, Register dstH, Register src1, Register src2,
+ SBit s = LeaveCC, Condition cond = al);
+
+ void umlal(Register dstL, Register dstH, Register src1, Register src2,
+ SBit s = LeaveCC, Condition cond = al);
+
+ void umull(Register dstL, Register dstH, Register src1, Register src2,
+ SBit s = LeaveCC, Condition cond = al);
+
+ // Miscellaneous arithmetic instructions
+
+ void clz(Register dst, Register src, Condition cond = al); // v5 and above
+
+ // Saturating instructions. v6 and above.
+
+ // Unsigned saturate.
+ //
+ // Saturate an optionally shifted signed value to an unsigned range.
+ //
+ // usat dst, #satpos, src
+ // usat dst, #satpos, src, lsl #sh
+ // usat dst, #satpos, src, asr #sh
+ //
+ // Register dst will contain:
+ //
+ // 0, if s < 0
+ // (1 << satpos) - 1, if s > ((1 << satpos) - 1)
+ // s, otherwise
+ //
+ // where s is the contents of src after shifting (if used.)
+ void usat(Register dst, int satpos, const Operand& src, Condition cond = al);
+
+ // Bitfield manipulation instructions. v7 and above.
+
+ void ubfx(Register dst, Register src, int lsb, int width,
+ Condition cond = al);
+
+ void sbfx(Register dst, Register src, int lsb, int width,
+ Condition cond = al);
+
+ void bfc(Register dst, int lsb, int width, Condition cond = al);
+
+ void bfi(Register dst, Register src, int lsb, int width, Condition cond = al);
+
+ void pkhbt(Register dst, Register src1, const Operand& src2,
+ Condition cond = al);
+
+ void pkhtb(Register dst, Register src1, const Operand& src2,
+ Condition cond = al);
+
+ void sxtb(Register dst, Register src, int rotate = 0, Condition cond = al);
+ void sxtab(Register dst, Register src1, Register src2, int rotate = 0,
+ Condition cond = al);
+ void sxth(Register dst, Register src, int rotate = 0, Condition cond = al);
+ void sxtah(Register dst, Register src1, Register src2, int rotate = 0,
+ Condition cond = al);
+
+ void uxtb(Register dst, Register src, int rotate = 0, Condition cond = al);
+ void uxtab(Register dst, Register src1, Register src2, int rotate = 0,
+ Condition cond = al);
+ void uxtb16(Register dst, Register src, int rotate = 0, Condition cond = al);
+ void uxth(Register dst, Register src, int rotate = 0, Condition cond = al);
+ void uxtah(Register dst, Register src1, Register src2, int rotate = 0,
+ Condition cond = al);
+
+ // Reverse the bits in a register.
+ void rbit(Register dst, Register src, Condition cond = al);
+ void rev(Register dst, Register src, Condition cond = al);
+
+ // Status register access instructions
+
+ void mrs(Register dst, SRegister s, Condition cond = al);
+ void msr(SRegisterFieldMask fields, const Operand& src, Condition cond = al);
+
+ // Load/Store instructions
+ void ldr(Register dst, const MemOperand& src, Condition cond = al);
+ void str(Register src, const MemOperand& dst, Condition cond = al);
+ void ldrb(Register dst, const MemOperand& src, Condition cond = al);
+ void strb(Register src, const MemOperand& dst, Condition cond = al);
+ void ldrh(Register dst, const MemOperand& src, Condition cond = al);
+ void strh(Register src, const MemOperand& dst, Condition cond = al);
+ void ldrsb(Register dst, const MemOperand& src, Condition cond = al);
+ void ldrsh(Register dst, const MemOperand& src, Condition cond = al);
+ void ldrd(Register dst1, Register dst2, const MemOperand& src,
+ Condition cond = al);
+ void strd(Register src1, Register src2, const MemOperand& dst,
+ Condition cond = al);
+
+ // Load literal from a pc relative address.
+ void ldr_pcrel(Register dst, int imm12, Condition cond = al);
+
+ // Load/Store exclusive instructions
+ void ldrex(Register dst, Register src, Condition cond = al);
+ void strex(Register src1, Register src2, Register dst, Condition cond = al);
+ void ldrexb(Register dst, Register src, Condition cond = al);
+ void strexb(Register src1, Register src2, Register dst, Condition cond = al);
+ void ldrexh(Register dst, Register src, Condition cond = al);
+ void strexh(Register src1, Register src2, Register dst, Condition cond = al);
+ void ldrexd(Register dst1, Register dst2, Register src, Condition cond = al);
+ void strexd(Register res, Register src1, Register src2, Register dst,
+ Condition cond = al);
+
+ // Preload instructions
+ void pld(const MemOperand& address);
+
+ // Load/Store multiple instructions
+ void ldm(BlockAddrMode am, Register base, RegList dst, Condition cond = al);
+ void stm(BlockAddrMode am, Register base, RegList src, Condition cond = al);
+
+ // Exception-generating instructions and debugging support
+ void stop(Condition cond = al, int32_t code = kDefaultStopCode);
+
+ void bkpt(uint32_t imm16); // v5 and above
+ void svc(uint32_t imm24, Condition cond = al);
+
+ // Synchronization instructions.
+ // On ARMv6, an equivalent CP15 operation will be used.
+ void dmb(BarrierOption option);
+ void dsb(BarrierOption option);
+ void isb(BarrierOption option);
+
+ // Conditional speculation barrier.
+ void csdb();
+
+ // Coprocessor instructions
+
+ void cdp(Coprocessor coproc, int opcode_1, CRegister crd, CRegister crn,
+ CRegister crm, int opcode_2, Condition cond = al);
+
+ void cdp2(Coprocessor coproc, int opcode_1, CRegister crd, CRegister crn,
+ CRegister crm,
+ int opcode_2); // v5 and above
+
+ void mcr(Coprocessor coproc, int opcode_1, Register rd, CRegister crn,
+ CRegister crm, int opcode_2 = 0, Condition cond = al);
+
+ void mcr2(Coprocessor coproc, int opcode_1, Register rd, CRegister crn,
+ CRegister crm,
+ int opcode_2 = 0); // v5 and above
+
+ void mrc(Coprocessor coproc, int opcode_1, Register rd, CRegister crn,
+ CRegister crm, int opcode_2 = 0, Condition cond = al);
+
+ void mrc2(Coprocessor coproc, int opcode_1, Register rd, CRegister crn,
+ CRegister crm,
+ int opcode_2 = 0); // v5 and above
+
+ void ldc(Coprocessor coproc, CRegister crd, const MemOperand& src,
+ LFlag l = Short, Condition cond = al);
+ void ldc(Coprocessor coproc, CRegister crd, Register base, int option,
+ LFlag l = Short, Condition cond = al);
+
+ void ldc2(Coprocessor coproc, CRegister crd, const MemOperand& src,
+ LFlag l = Short); // v5 and above
+ void ldc2(Coprocessor coproc, CRegister crd, Register base, int option,
+ LFlag l = Short); // v5 and above
+
+ // Support for VFP.
+ // All these APIs support S0 to S31 and D0 to D31.
+
+ void vldr(const DwVfpRegister dst, const Register base, int offset,
+ const Condition cond = al);
+ void vldr(const DwVfpRegister dst, const MemOperand& src,
+ const Condition cond = al);
+
+ void vldr(const SwVfpRegister dst, const Register base, int offset,
+ const Condition cond = al);
+ void vldr(const SwVfpRegister dst, const MemOperand& src,
+ const Condition cond = al);
+
+ void vstr(const DwVfpRegister src, const Register base, int offset,
+ const Condition cond = al);
+ void vstr(const DwVfpRegister src, const MemOperand& dst,
+ const Condition cond = al);
+
+ void vstr(const SwVfpRegister src, const Register base, int offset,
+ const Condition cond = al);
+ void vstr(const SwVfpRegister src, const MemOperand& dst,
+ const Condition cond = al);
+
+ void vldm(BlockAddrMode am, Register base, DwVfpRegister first,
+ DwVfpRegister last, Condition cond = al);
+
+ void vstm(BlockAddrMode am, Register base, DwVfpRegister first,
+ DwVfpRegister last, Condition cond = al);
+
+ void vldm(BlockAddrMode am, Register base, SwVfpRegister first,
+ SwVfpRegister last, Condition cond = al);
+
+ void vstm(BlockAddrMode am, Register base, SwVfpRegister first,
+ SwVfpRegister last, Condition cond = al);
+
+ void vmov(const SwVfpRegister dst, Float32 imm);
+ void vmov(const DwVfpRegister dst, Double imm,
+ const Register extra_scratch = no_reg);
+ void vmov(const SwVfpRegister dst, const SwVfpRegister src,
+ const Condition cond = al);
+ void vmov(const DwVfpRegister dst, const DwVfpRegister src,
+ const Condition cond = al);
+ void vmov(const DwVfpRegister dst, const Register src1, const Register src2,
+ const Condition cond = al);
+ void vmov(const Register dst1, const Register dst2, const DwVfpRegister src,
+ const Condition cond = al);
+ void vmov(const SwVfpRegister dst, const Register src,
+ const Condition cond = al);
+ void vmov(const Register dst, const SwVfpRegister src,
+ const Condition cond = al);
+ void vcvt_f64_s32(const DwVfpRegister dst, const SwVfpRegister src,
+ VFPConversionMode mode = kDefaultRoundToZero,
+ const Condition cond = al);
+ void vcvt_f32_s32(const SwVfpRegister dst, const SwVfpRegister src,
+ VFPConversionMode mode = kDefaultRoundToZero,
+ const Condition cond = al);
+ void vcvt_f64_u32(const DwVfpRegister dst, const SwVfpRegister src,
+ VFPConversionMode mode = kDefaultRoundToZero,
+ const Condition cond = al);
+ void vcvt_f32_u32(const SwVfpRegister dst, const SwVfpRegister src,
+ VFPConversionMode mode = kDefaultRoundToZero,
+ const Condition cond = al);
+ void vcvt_s32_f32(const SwVfpRegister dst, const SwVfpRegister src,
+ VFPConversionMode mode = kDefaultRoundToZero,
+ const Condition cond = al);
+ void vcvt_u32_f32(const SwVfpRegister dst, const SwVfpRegister src,
+ VFPConversionMode mode = kDefaultRoundToZero,
+ const Condition cond = al);
+ void vcvt_s32_f64(const SwVfpRegister dst, const DwVfpRegister src,
+ VFPConversionMode mode = kDefaultRoundToZero,
+ const Condition cond = al);
+ void vcvt_u32_f64(const SwVfpRegister dst, const DwVfpRegister src,
+ VFPConversionMode mode = kDefaultRoundToZero,
+ const Condition cond = al);
+ void vcvt_f64_f32(const DwVfpRegister dst, const SwVfpRegister src,
+ VFPConversionMode mode = kDefaultRoundToZero,
+ const Condition cond = al);
+ void vcvt_f32_f64(const SwVfpRegister dst, const DwVfpRegister src,
+ VFPConversionMode mode = kDefaultRoundToZero,
+ const Condition cond = al);
+ void vcvt_f64_s32(const DwVfpRegister dst, int fraction_bits,
+ const Condition cond = al);
+
+ void vmrs(const Register dst, const Condition cond = al);
+ void vmsr(const Register dst, const Condition cond = al);
+
+ void vneg(const DwVfpRegister dst, const DwVfpRegister src,
+ const Condition cond = al);
+ void vneg(const SwVfpRegister dst, const SwVfpRegister src,
+ const Condition cond = al);
+ void vabs(const DwVfpRegister dst, const DwVfpRegister src,
+ const Condition cond = al);
+ void vabs(const SwVfpRegister dst, const SwVfpRegister src,
+ const Condition cond = al);
+ void vadd(const DwVfpRegister dst, const DwVfpRegister src1,
+ const DwVfpRegister src2, const Condition cond = al);
+ void vadd(const SwVfpRegister dst, const SwVfpRegister src1,
+ const SwVfpRegister src2, const Condition cond = al);
+ void vsub(const DwVfpRegister dst, const DwVfpRegister src1,
+ const DwVfpRegister src2, const Condition cond = al);
+ void vsub(const SwVfpRegister dst, const SwVfpRegister src1,
+ const SwVfpRegister src2, const Condition cond = al);
+ void vmul(const DwVfpRegister dst, const DwVfpRegister src1,
+ const DwVfpRegister src2, const Condition cond = al);
+ void vmul(const SwVfpRegister dst, const SwVfpRegister src1,
+ const SwVfpRegister src2, const Condition cond = al);
+ void vmla(const DwVfpRegister dst, const DwVfpRegister src1,
+ const DwVfpRegister src2, const Condition cond = al);
+ void vmla(const SwVfpRegister dst, const SwVfpRegister src1,
+ const SwVfpRegister src2, const Condition cond = al);
+ void vmls(const DwVfpRegister dst, const DwVfpRegister src1,
+ const DwVfpRegister src2, const Condition cond = al);
+ void vmls(const SwVfpRegister dst, const SwVfpRegister src1,
+ const SwVfpRegister src2, const Condition cond = al);
+ void vdiv(const DwVfpRegister dst, const DwVfpRegister src1,
+ const DwVfpRegister src2, const Condition cond = al);
+ void vdiv(const SwVfpRegister dst, const SwVfpRegister src1,
+ const SwVfpRegister src2, const Condition cond = al);
+ void vcmp(const DwVfpRegister src1, const DwVfpRegister src2,
+ const Condition cond = al);
+ void vcmp(const SwVfpRegister src1, const SwVfpRegister src2,
+ const Condition cond = al);
+ void vcmp(const DwVfpRegister src1, const double src2,
+ const Condition cond = al);
+ void vcmp(const SwVfpRegister src1, const float src2,
+ const Condition cond = al);
+
+ void vmaxnm(const DwVfpRegister dst, const DwVfpRegister src1,
+ const DwVfpRegister src2);
+ void vmaxnm(const SwVfpRegister dst, const SwVfpRegister src1,
+ const SwVfpRegister src2);
+ void vminnm(const DwVfpRegister dst, const DwVfpRegister src1,
+ const DwVfpRegister src2);
+ void vminnm(const SwVfpRegister dst, const SwVfpRegister src1,
+ const SwVfpRegister src2);
+
+ // VSEL supports cond in {eq, ne, ge, lt, gt, le, vs, vc}.
+ void vsel(const Condition cond, const DwVfpRegister dst,
+ const DwVfpRegister src1, const DwVfpRegister src2);
+ void vsel(const Condition cond, const SwVfpRegister dst,
+ const SwVfpRegister src1, const SwVfpRegister src2);
+
+ void vsqrt(const DwVfpRegister dst, const DwVfpRegister src,
+ const Condition cond = al);
+ void vsqrt(const SwVfpRegister dst, const SwVfpRegister src,
+ const Condition cond = al);
+
+ // ARMv8 rounding instructions (Scalar).
+ void vrinta(const SwVfpRegister dst, const SwVfpRegister src);
+ void vrinta(const DwVfpRegister dst, const DwVfpRegister src);
+ void vrintn(const SwVfpRegister dst, const SwVfpRegister src);
+ void vrintn(const DwVfpRegister dst, const DwVfpRegister src);
+ void vrintm(const SwVfpRegister dst, const SwVfpRegister src);
+ void vrintm(const DwVfpRegister dst, const DwVfpRegister src);
+ void vrintp(const SwVfpRegister dst, const SwVfpRegister src);
+ void vrintp(const DwVfpRegister dst, const DwVfpRegister src);
+ void vrintz(const SwVfpRegister dst, const SwVfpRegister src,
+ const Condition cond = al);
+ void vrintz(const DwVfpRegister dst, const DwVfpRegister src,
+ const Condition cond = al);
+
+ // Support for NEON.
+
+ // All these APIs support D0 to D31 and Q0 to Q15.
+ void vld1(NeonSize size, const NeonListOperand& dst,
+ const NeonMemOperand& src);
+ // vld1s(ingle element to one lane).
+ void vld1s(NeonSize size, const NeonListOperand& dst, uint8_t index,
+ const NeonMemOperand& src);
+ void vld1r(NeonSize size, const NeonListOperand& dst,
+ const NeonMemOperand& src);
+ void vst1(NeonSize size, const NeonListOperand& src,
+ const NeonMemOperand& dst);
+ // dt represents the narrower type
+ void vmovl(NeonDataType dt, QwNeonRegister dst, DwVfpRegister src);
+ // dst_dt represents the narrower type, src_dt represents the src type.
+ void vqmovn(NeonDataType dst_dt, NeonDataType src_dt, DwVfpRegister dst,
+ QwNeonRegister src);
+
+ // Only unconditional core <-> scalar moves are currently supported.
+ void vmov(NeonDataType dt, DwVfpRegister dst, int index, Register src);
+ void vmov(NeonDataType dt, Register dst, DwVfpRegister src, int index);
+
+ void vmov(DwVfpRegister dst, uint64_t imm);
+ void vmov(QwNeonRegister dst, uint64_t imm);
+ void vmov(QwNeonRegister dst, QwNeonRegister src);
+ void vdup(NeonSize size, QwNeonRegister dst, Register src);
+ void vdup(NeonSize size, QwNeonRegister dst, DwVfpRegister src, int index);
+ void vdup(NeonSize size, DwVfpRegister dst, DwVfpRegister src, int index);
+
+ void vcvt_f32_s32(QwNeonRegister dst, QwNeonRegister src);
+ void vcvt_f32_u32(QwNeonRegister dst, QwNeonRegister src);
+ void vcvt_s32_f32(QwNeonRegister dst, QwNeonRegister src);
+ void vcvt_u32_f32(QwNeonRegister dst, QwNeonRegister src);
+
+ void vmvn(QwNeonRegister dst, QwNeonRegister src);
+ void vswp(DwVfpRegister dst, DwVfpRegister src);
+ void vswp(QwNeonRegister dst, QwNeonRegister src);
+ void vabs(QwNeonRegister dst, QwNeonRegister src);
+ void vabs(NeonSize size, QwNeonRegister dst, QwNeonRegister src);
+ void vneg(QwNeonRegister dst, QwNeonRegister src);
+ void vneg(NeonSize size, QwNeonRegister dst, QwNeonRegister src);
+
+ void vand(QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2);
+ void vbic(QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2);
+ void veor(DwVfpRegister dst, DwVfpRegister src1, DwVfpRegister src2);
+ void veor(QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2);
+ void vbsl(QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2);
+ void vorr(QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2);
+ void vadd(QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2);
+ void vadd(NeonSize size, QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2);
+ void vqadd(NeonDataType dt, QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2);
+ void vsub(QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2);
+ void vsub(NeonSize size, QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2);
+ void vqsub(NeonDataType dt, QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2);
+ void vmlal(NeonDataType size, QwNeonRegister dst, DwVfpRegister src1,
+ DwVfpRegister src2);
+ void vmul(QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2);
+ void vmul(NeonSize size, QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2);
+ void vmull(NeonDataType size, QwNeonRegister dst, DwVfpRegister src1,
+ DwVfpRegister src2);
+ void vmin(QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2);
+ void vmin(NeonDataType dt, QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2);
+ void vmax(QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2);
+ void vmax(NeonDataType dt, QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2);
+ void vpadd(DwVfpRegister dst, DwVfpRegister src1, DwVfpRegister src2);
+ void vpadd(NeonSize size, DwVfpRegister dst, DwVfpRegister src1,
+ DwVfpRegister src2);
+ void vpmin(NeonDataType dt, DwVfpRegister dst, DwVfpRegister src1,
+ DwVfpRegister src2);
+ void vpmax(NeonDataType dt, DwVfpRegister dst, DwVfpRegister src1,
+ DwVfpRegister src2);
+
+ // ARMv8 rounding instructions (NEON).
+ void vrintm(NeonDataType dt, const QwNeonRegister dst,
+ const QwNeonRegister src);
+ void vrintn(NeonDataType dt, const QwNeonRegister dst,
+ const QwNeonRegister src);
+ void vrintp(NeonDataType dt, const QwNeonRegister dst,
+ const QwNeonRegister src);
+ void vrintz(NeonDataType dt, const QwNeonRegister dst,
+ const QwNeonRegister src);
+
+ void vshl(NeonDataType dt, QwNeonRegister dst, QwNeonRegister src, int shift);
+ void vshl(NeonDataType dt, QwNeonRegister dst, QwNeonRegister src,
+ QwNeonRegister shift);
+ void vshr(NeonDataType dt, QwNeonRegister dst, QwNeonRegister src, int shift);
+ void vsli(NeonSize size, DwVfpRegister dst, DwVfpRegister src, int shift);
+ void vsri(NeonSize size, DwVfpRegister dst, DwVfpRegister src, int shift);
+ // vrecpe and vrsqrte only support floating point lanes.
+ void vrecpe(QwNeonRegister dst, QwNeonRegister src);
+ void vrsqrte(QwNeonRegister dst, QwNeonRegister src);
+ void vrecps(QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2);
+ void vrsqrts(QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2);
+ void vtst(NeonSize size, QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2);
+ void vceq(QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2);
+ void vceq(NeonSize size, QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2);
+ void vcge(QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2);
+ void vcge(NeonDataType dt, QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2);
+ void vcgt(QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2);
+ void vcgt(NeonDataType dt, QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2);
+ void vrhadd(NeonDataType dt, QwNeonRegister dst, QwNeonRegister src1,
+ QwNeonRegister src2);
+ void vext(QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2,
+ int bytes);
+ void vzip(NeonSize size, DwVfpRegister src1, DwVfpRegister src2);
+ void vzip(NeonSize size, QwNeonRegister src1, QwNeonRegister src2);
+ void vuzp(NeonSize size, DwVfpRegister src1, DwVfpRegister src2);
+ void vuzp(NeonSize size, QwNeonRegister src1, QwNeonRegister src2);
+ void vrev16(NeonSize size, QwNeonRegister dst, QwNeonRegister src);
+ void vrev32(NeonSize size, QwNeonRegister dst, QwNeonRegister src);
+ void vrev64(NeonSize size, QwNeonRegister dst, QwNeonRegister src);
+ void vtrn(NeonSize size, DwVfpRegister src1, DwVfpRegister src2);
+ void vtrn(NeonSize size, QwNeonRegister src1, QwNeonRegister src2);
+ void vtbl(DwVfpRegister dst, const NeonListOperand& list,
+ DwVfpRegister index);
+ void vtbx(DwVfpRegister dst, const NeonListOperand& list,
+ DwVfpRegister index);
+
+ // Pseudo instructions
+
+ // Different nop operations are used by the code generator to detect certain
+ // states of the generated code.
+ enum NopMarkerTypes {
+ NON_MARKING_NOP = 0,
+ DEBUG_BREAK_NOP,
+ // IC markers.
+ PROPERTY_ACCESS_INLINED,
+ PROPERTY_ACCESS_INLINED_CONTEXT,
+ PROPERTY_ACCESS_INLINED_CONTEXT_DONT_DELETE,
+ // Helper values.
+ LAST_CODE_MARKER,
+ FIRST_IC_MARKER = PROPERTY_ACCESS_INLINED
+ };
+
+ void nop(int type = 0); // 0 is the default non-marking type.
+
+ void push(Register src, Condition cond = al) {
+ str(src, MemOperand(sp, 4, NegPreIndex), cond);
+ }
+
+ void pop(Register dst, Condition cond = al) {
+ ldr(dst, MemOperand(sp, 4, PostIndex), cond);
+ }
+
+ void pop();
+
+ void vpush(QwNeonRegister src, Condition cond = al) {
+ vstm(db_w, sp, src.low(), src.high(), cond);
+ }
+
+ void vpush(DwVfpRegister src, Condition cond = al) {
+ vstm(db_w, sp, src, src, cond);
+ }
+
+ void vpush(SwVfpRegister src, Condition cond = al) {
+ vstm(db_w, sp, src, src, cond);
+ }
+
+ void vpop(DwVfpRegister dst, Condition cond = al) {
+ vldm(ia_w, sp, dst, dst, cond);
+ }
+
+ // Jump unconditionally to given label.
+ void jmp(Label* L) { b(L, al); }
+
+ // Check the code size generated from label to here.
+ int SizeOfCodeGeneratedSince(Label* label) {
+ return pc_offset() - label->pos();
+ }
+
+ // Check the number of instructions generated from label to here.
+ int InstructionsGeneratedSince(Label* label) {
+ return SizeOfCodeGeneratedSince(label) / kInstrSize;
+ }
+
+ // Check whether an immediate fits an addressing mode 1 instruction.
+ static bool ImmediateFitsAddrMode1Instruction(int32_t imm32);
+
+ // Check whether an immediate fits an addressing mode 2 instruction.
+ bool ImmediateFitsAddrMode2Instruction(int32_t imm32);
+
+ // Class for scoping postponing the constant pool generation.
+ class BlockConstPoolScope {
+ public:
+ explicit BlockConstPoolScope(Assembler* assem) : assem_(assem) {
+ assem_->StartBlockConstPool();
+ }
+ ~BlockConstPoolScope() { assem_->EndBlockConstPool(); }
+
+ private:
+ Assembler* assem_;
+
+ DISALLOW_IMPLICIT_CONSTRUCTORS(BlockConstPoolScope);
+ };
+
+ // Unused on this architecture.
+ void MaybeEmitOutOfLineConstantPool() {}
+
+ // Record a deoptimization reason that can be used by a log or cpu profiler.
+ // Use --trace-deopt to enable.
+ void RecordDeoptReason(DeoptimizeReason reason, SourcePosition position,
+ int id);
+
+ // Record the emission of a constant pool.
+ //
+ // The emission of constant pool depends on the size of the code generated and
+ // the number of RelocInfo recorded.
+ // The Debug mechanism needs to map code offsets between two versions of a
+ // function, compiled with and without debugger support (see for example
+ // Debug::PrepareForBreakPoints()).
+ // Compiling functions with debugger support generates additional code
+ // (DebugCodegen::GenerateSlot()). This may affect the emission of the
+ // constant pools and cause the version of the code with debugger support to
+ // have constant pools generated in different places.
+ // Recording the position and size of emitted constant pools allows to
+ // correctly compute the offset mappings between the different versions of a
+ // function in all situations.
+ //
+ // The parameter indicates the size of the constant pool (in bytes), including
+ // the marker and branch over the data.
+ void RecordConstPool(int size);
+
+ // Writes a single byte or word of data in the code stream. Used
+ // for inline tables, e.g., jump-tables. CheckConstantPool() should be
+ // called before any use of db/dd/dq/dp to ensure that constant pools
+ // are not emitted as part of the tables generated.
+ void db(uint8_t data);
+ void dd(uint32_t data);
+ void dq(uint64_t data);
+ void dp(uintptr_t data) { dd(data); }
+
+ // Read/patch instructions
+ Instr instr_at(int pos) {
+ return *reinterpret_cast<Instr*>(buffer_start_ + pos);
+ }
+ void instr_at_put(int pos, Instr instr) {
+ *reinterpret_cast<Instr*>(buffer_start_ + pos) = instr;
+ }
+ static Instr instr_at(Address pc) { return *reinterpret_cast<Instr*>(pc); }
+ static void instr_at_put(Address pc, Instr instr) {
+ *reinterpret_cast<Instr*>(pc) = instr;
+ }
+ static Condition GetCondition(Instr instr);
+ static bool IsLdrRegisterImmediate(Instr instr);
+ static bool IsVldrDRegisterImmediate(Instr instr);
+ static int GetLdrRegisterImmediateOffset(Instr instr);
+ static int GetVldrDRegisterImmediateOffset(Instr instr);
+ static Instr SetLdrRegisterImmediateOffset(Instr instr, int offset);
+ static Instr SetVldrDRegisterImmediateOffset(Instr instr, int offset);
+ static bool IsStrRegisterImmediate(Instr instr);
+ static Instr SetStrRegisterImmediateOffset(Instr instr, int offset);
+ static bool IsAddRegisterImmediate(Instr instr);
+ static Instr SetAddRegisterImmediateOffset(Instr instr, int offset);
+ static Register GetRd(Instr instr);
+ static Register GetRn(Instr instr);
+ static Register GetRm(Instr instr);
+ static bool IsPush(Instr instr);
+ static bool IsPop(Instr instr);
+ static bool IsStrRegFpOffset(Instr instr);
+ static bool IsLdrRegFpOffset(Instr instr);
+ static bool IsStrRegFpNegOffset(Instr instr);
+ static bool IsLdrRegFpNegOffset(Instr instr);
+ static bool IsLdrPcImmediateOffset(Instr instr);
+ static bool IsBOrBlPcImmediateOffset(Instr instr);
+ static bool IsVldrDPcImmediateOffset(Instr instr);
+ static bool IsBlxReg(Instr instr);
+ static bool IsBlxIp(Instr instr);
+ static bool IsTstImmediate(Instr instr);
+ static bool IsCmpRegister(Instr instr);
+ static bool IsCmpImmediate(Instr instr);
+ static Register GetCmpImmediateRegister(Instr instr);
+ static int GetCmpImmediateRawImmediate(Instr instr);
+ static bool IsNop(Instr instr, int type = NON_MARKING_NOP);
+ static bool IsMovImmed(Instr instr);
+ static bool IsOrrImmed(Instr instr);
+ static bool IsMovT(Instr instr);
+ static Instr GetMovTPattern();
+ static bool IsMovW(Instr instr);
+ static Instr GetMovWPattern();
+ static Instr EncodeMovwImmediate(uint32_t immediate);
+ static Instr PatchMovwImmediate(Instr instruction, uint32_t immediate);
+ static int DecodeShiftImm(Instr instr);
+ static Instr PatchShiftImm(Instr instr, int immed);
+
+ // Constants in pools are accessed via pc relative addressing, which can
+ // reach +/-4KB for integer PC-relative loads and +/-1KB for floating-point
+ // PC-relative loads, thereby defining a maximum distance between the
+ // instruction and the accessed constant.
+ static constexpr int kMaxDistToIntPool = 4 * KB;
+ // All relocations could be integer, it therefore acts as the limit.
+ static constexpr int kMinNumPendingConstants = 4;
+ static constexpr int kMaxNumPending32Constants =
+ kMaxDistToIntPool / kInstrSize;
+
+ // Postpone the generation of the constant pool for the specified number of
+ // instructions.
+ void BlockConstPoolFor(int instructions);
+
+ // Check if is time to emit a constant pool.
+ void CheckConstPool(bool force_emit, bool require_jump);
+
+ void MaybeCheckConstPool() {
+ if (pc_offset() >= next_buffer_check_) {
+ CheckConstPool(false, true);
+ }
+ }
+
+ // Move a 32-bit immediate into a register, potentially via the constant pool.
+ void Move32BitImmediate(Register rd, const Operand& x, Condition cond = al);
+
+ // Get the code target object for a pc-relative call or jump.
+ V8_INLINE Handle<Code> relative_code_target_object_handle_at(
+ Address pc_) const;
+
+ protected:
+ int buffer_space() const { return reloc_info_writer.pos() - pc_; }
+
+ // Decode branch instruction at pos and return branch target pos
+ int target_at(int pos);
+
+ // Patch branch instruction at pos to branch to given branch target pos
+ void target_at_put(int pos, int target_pos);
+
+ // Prevent contant pool emission until EndBlockConstPool is called.
+ // Calls to this function can be nested but must be followed by an equal
+ // number of call to EndBlockConstpool.
+ void StartBlockConstPool() {
+ if (const_pool_blocked_nesting_++ == 0) {
+ // Prevent constant pool checks happening by setting the next check to
+ // the biggest possible offset.
+ next_buffer_check_ = kMaxInt;
+ }
+ }
+
+ // Resume constant pool emission. Needs to be called as many times as
+ // StartBlockConstPool to have an effect.
+ void EndBlockConstPool() {
+ if (--const_pool_blocked_nesting_ == 0) {
+#ifdef DEBUG
+ // Max pool start (if we need a jump and an alignment).
+ int start = pc_offset() + kInstrSize + 2 * kPointerSize;
+ // Check the constant pool hasn't been blocked for too long.
+ DCHECK(pending_32_bit_constants_.empty() ||
+ (start < first_const_pool_32_use_ + kMaxDistToIntPool));
+#endif
+ // Two cases:
+ // * no_const_pool_before_ >= next_buffer_check_ and the emission is
+ // still blocked
+ // * no_const_pool_before_ < next_buffer_check_ and the next emit will
+ // trigger a check.
+ next_buffer_check_ = no_const_pool_before_;
+ }
+ }
+
+ bool is_const_pool_blocked() const {
+ return (const_pool_blocked_nesting_ > 0) ||
+ (pc_offset() < no_const_pool_before_);
+ }
+
+ bool VfpRegisterIsAvailable(DwVfpRegister reg) {
+ DCHECK(reg.is_valid());
+ return IsEnabled(VFP32DREGS) ||
+ (reg.code() < LowDwVfpRegister::kNumRegisters);
+ }
+
+ bool VfpRegisterIsAvailable(QwNeonRegister reg) {
+ DCHECK(reg.is_valid());
+ return IsEnabled(VFP32DREGS) ||
+ (reg.code() < LowDwVfpRegister::kNumRegisters / 2);
+ }
+
+ inline void emit(Instr x);
+
+ // Code generation
+ // The relocation writer's position is at least kGap bytes below the end of
+ // the generated instructions. This is so that multi-instruction sequences do
+ // not have to check for overflow. The same is true for writes of large
+ // relocation info entries.
+ static constexpr int kGap = 32;
+ STATIC_ASSERT(AssemblerBase::kMinimalBufferSize >= 2 * kGap);
+
+ // Relocation info generation
+ // Each relocation is encoded as a variable size value
+ static constexpr int kMaxRelocSize = RelocInfoWriter::kMaxSize;
+ RelocInfoWriter reloc_info_writer;
+
+ // ConstantPoolEntry records are used during code generation as temporary
+ // containers for constants and code target addresses until they are emitted
+ // to the constant pool. These records are temporarily stored in a separate
+ // buffer until a constant pool is emitted.
+ // If every instruction in a long sequence is accessing the pool, we need one
+ // pending relocation entry per instruction.
+
+ // The buffers of pending constant pool entries.
+ std::vector<ConstantPoolEntry> pending_32_bit_constants_;
+
+ // Scratch registers available for use by the Assembler.
+ RegList scratch_register_list_;
+ VfpRegList scratch_vfp_register_list_;
+
+ private:
+ // Avoid overflows for displacements etc.
+ static const int kMaximalBufferSize = 512 * MB;
+
+ int next_buffer_check_; // pc offset of next buffer check
+
+ // Constant pool generation
+ // Pools are emitted in the instruction stream, preferably after unconditional
+ // jumps or after returns from functions (in dead code locations).
+ // If a long code sequence does not contain unconditional jumps, it is
+ // necessary to emit the constant pool before the pool gets too far from the
+ // location it is accessed from. In this case, we emit a jump over the emitted
+ // constant pool.
+ // Constants in the pool may be addresses of functions that gets relocated;
+ // if so, a relocation info entry is associated to the constant pool entry.
+
+ // Repeated checking whether the constant pool should be emitted is rather
+ // expensive. By default we only check again once a number of instructions
+ // has been generated. That also means that the sizing of the buffers is not
+ // an exact science, and that we rely on some slop to not overrun buffers.
+ static constexpr int kCheckPoolIntervalInst = 32;
+ static constexpr int kCheckPoolInterval = kCheckPoolIntervalInst * kInstrSize;
+
+ // Emission of the constant pool may be blocked in some code sequences.
+ int const_pool_blocked_nesting_; // Block emission if this is not zero.
+ int no_const_pool_before_; // Block emission before this pc offset.
+
+ // Keep track of the first instruction requiring a constant pool entry
+ // since the previous constant pool was emitted.
+ int first_const_pool_32_use_;
+
+ // The bound position, before this we cannot do instruction elimination.
+ int last_bound_pos_;
+
+ inline void CheckBuffer();
+ void GrowBuffer();
+
+ // Instruction generation
+ void AddrMode1(Instr instr, Register rd, Register rn, const Operand& x);
+ // Attempt to encode operand |x| for instruction |instr| and return true on
+ // success. The result will be encoded in |instr| directly. This method may
+ // change the opcode if deemed beneficial, for instance, MOV may be turned
+ // into MVN, ADD into SUB, AND into BIC, ...etc. The only reason this method
+ // may fail is that the operand is an immediate that cannot be encoded.
+ bool AddrMode1TryEncodeOperand(Instr* instr, const Operand& x);
+
+ void AddrMode2(Instr instr, Register rd, const MemOperand& x);
+ void AddrMode3(Instr instr, Register rd, const MemOperand& x);
+ void AddrMode4(Instr instr, Register rn, RegList rl);
+ void AddrMode5(Instr instr, CRegister crd, const MemOperand& x);
+
+ // Labels
+ void print(const Label* L);
+ void bind_to(Label* L, int pos);
+ void next(Label* L);
+
+ // Record reloc info for current pc_
+ void RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data = 0);
+ void ConstantPoolAddEntry(int position, RelocInfo::Mode rmode,
+ intptr_t value);
+ void AllocateAndInstallRequestedHeapObjects(Isolate* isolate);
+
+ int WriteCodeComments();
+
+ friend class RelocInfo;
+ friend class BlockConstPoolScope;
+ friend class EnsureSpace;
+ friend class UseScratchRegisterScope;
+};
+
+class EnsureSpace {
+ public:
+ V8_INLINE explicit EnsureSpace(Assembler* assembler);
+};
+
+class PatchingAssembler : public Assembler {
+ public:
+ PatchingAssembler(const AssemblerOptions& options, byte* address,
+ int instructions);
+ ~PatchingAssembler();
+
+ void Emit(Address addr);
+ void PadWithNops();
+};
+
+// This scope utility allows scratch registers to be managed safely. The
+// Assembler's GetScratchRegisterList() is used as a pool of scratch
+// registers. These registers can be allocated on demand, and will be returned
+// at the end of the scope.
+//
+// When the scope ends, the Assembler's list will be restored to its original
+// state, even if the list is modified by some other means. Note that this scope
+// can be nested but the destructors need to run in the opposite order as the
+// constructors. We do not have assertions for this.
+class V8_EXPORT_PRIVATE UseScratchRegisterScope {
+ public:
+ explicit UseScratchRegisterScope(Assembler* assembler);
+ ~UseScratchRegisterScope();
+
+ // Take a register from the list and return it.
+ Register Acquire();
+ SwVfpRegister AcquireS() { return AcquireVfp<SwVfpRegister>(); }
+ LowDwVfpRegister AcquireLowD() { return AcquireVfp<LowDwVfpRegister>(); }
+ DwVfpRegister AcquireD() {
+ DwVfpRegister reg = AcquireVfp<DwVfpRegister>();
+ DCHECK(assembler_->VfpRegisterIsAvailable(reg));
+ return reg;
+ }
+ QwNeonRegister AcquireQ() {
+ QwNeonRegister reg = AcquireVfp<QwNeonRegister>();
+ DCHECK(assembler_->VfpRegisterIsAvailable(reg));
+ return reg;
+ }
+
+ // Check if we have registers available to acquire.
+ bool CanAcquire() const { return *assembler_->GetScratchRegisterList() != 0; }
+ bool CanAcquireD() const { return CanAcquireVfp<DwVfpRegister>(); }
+
+ private:
+ friend class Assembler;
+ friend class TurboAssembler;
+
+ template <typename T>
+ bool CanAcquireVfp() const;
+
+ template <typename T>
+ T AcquireVfp();
+
+ Assembler* assembler_;
+ // Available scratch registers at the start of this scope.
+ RegList old_available_;
+ VfpRegList old_available_vfp_;
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_ARM_ASSEMBLER_ARM_H_
diff --git a/src/codegen/arm/constants-arm.cc b/src/codegen/arm/constants-arm.cc
new file mode 100644
index 0000000..ecde19b
--- /dev/null
+++ b/src/codegen/arm/constants-arm.cc
@@ -0,0 +1,100 @@
+// Copyright 2009 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#if V8_TARGET_ARCH_ARM
+
+#include "src/codegen/arm/constants-arm.h"
+
+namespace v8 {
+namespace internal {
+
+Float64 Instruction::DoubleImmedVmov() const {
+ // Reconstruct a double from the immediate encoded in the vmov instruction.
+ //
+ // instruction: [xxxxxxxx,xxxxabcd,xxxxxxxx,xxxxefgh]
+ // double: [aBbbbbbb,bbcdefgh,00000000,00000000,
+ // 00000000,00000000,00000000,00000000]
+ //
+ // where B = ~b. Only the high 16 bits are affected.
+ uint64_t high16;
+ high16 = (Bits(17, 16) << 4) | Bits(3, 0); // xxxxxxxx,xxcdefgh.
+ high16 |= (0xFF * Bit(18)) << 6; // xxbbbbbb,bbxxxxxx.
+ high16 |= (Bit(18) ^ 1) << 14; // xBxxxxxx,xxxxxxxx.
+ high16 |= Bit(19) << 15; // axxxxxxx,xxxxxxxx.
+
+ uint64_t imm = high16 << 48;
+ return Float64::FromBits(imm);
+}
+
+// These register names are defined in a way to match the native disassembler
+// formatting. See for example the command "objdump -d <binary file>".
+const char* Registers::names_[kNumRegisters] = {
+ "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7",
+ "r8", "r9", "r10", "fp", "ip", "sp", "lr", "pc",
+};
+
+// List of alias names which can be used when referring to ARM registers.
+const Registers::RegisterAlias Registers::aliases_[] = {
+ {10, "sl"}, {11, "r11"}, {12, "r12"}, {13, "r13"},
+ {14, "r14"}, {15, "r15"}, {kNoRegister, nullptr}};
+
+// Support for VFP registers s0 to s31 (d0 to d15) and d16-d31.
+// Note that "sN:sM" is the same as "dN/2" up to d15.
+// These register names are defined in a way to match the native disassembler
+// formatting. See for example the command "objdump -d <binary file>".
+const char* VFPRegisters::names_[kNumVFPRegisters] = {
+ "s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10",
+ "s11", "s12", "s13", "s14", "s15", "s16", "s17", "s18", "s19", "s20", "s21",
+ "s22", "s23", "s24", "s25", "s26", "s27", "s28", "s29", "s30", "s31", "d0",
+ "d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9", "d10", "d11",
+ "d12", "d13", "d14", "d15", "d16", "d17", "d18", "d19", "d20", "d21", "d22",
+ "d23", "d24", "d25", "d26", "d27", "d28", "d29", "d30", "d31"};
+
+const char* VFPRegisters::Name(int reg, bool is_double) {
+ DCHECK((0 <= reg) && (reg < kNumVFPRegisters));
+ return names_[reg + (is_double ? kNumVFPSingleRegisters : 0)];
+}
+
+int VFPRegisters::Number(const char* name, bool* is_double) {
+ for (int i = 0; i < kNumVFPRegisters; i++) {
+ if (strcmp(names_[i], name) == 0) {
+ if (i < kNumVFPSingleRegisters) {
+ *is_double = false;
+ return i;
+ } else {
+ *is_double = true;
+ return i - kNumVFPSingleRegisters;
+ }
+ }
+ }
+
+ // No register with the requested name found.
+ return kNoRegister;
+}
+
+int Registers::Number(const char* name) {
+ // Look through the canonical names.
+ for (int i = 0; i < kNumRegisters; i++) {
+ if (strcmp(names_[i], name) == 0) {
+ return i;
+ }
+ }
+
+ // Look through the alias names.
+ int i = 0;
+ while (aliases_[i].reg != kNoRegister) {
+ if (strcmp(aliases_[i].name, name) == 0) {
+ return aliases_[i].reg;
+ }
+ i++;
+ }
+
+ // No register with the requested name found.
+ return kNoRegister;
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_ARM
diff --git a/src/codegen/arm/constants-arm.h b/src/codegen/arm/constants-arm.h
new file mode 100644
index 0000000..a7726a8
--- /dev/null
+++ b/src/codegen/arm/constants-arm.h
@@ -0,0 +1,693 @@
+// Copyright 2011 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_ARM_CONSTANTS_ARM_H_
+#define V8_CODEGEN_ARM_CONSTANTS_ARM_H_
+
+#include <stdint.h>
+
+#include "src/base/logging.h"
+#include "src/base/macros.h"
+#include "src/common/globals.h"
+#include "src/utils/boxed-float.h"
+#include "src/utils/utils.h"
+
+// ARM EABI is required.
+#if defined(__arm__) && !defined(__ARM_EABI__)
+#error ARM EABI support is required.
+#endif
+
+namespace v8 {
+namespace internal {
+
+// Constant pool marker.
+// Use UDF, the permanently undefined instruction.
+const int kConstantPoolMarkerMask = 0xfff000f0;
+const int kConstantPoolMarker = 0xe7f000f0;
+const int kConstantPoolLengthMaxMask = 0xffff;
+inline int EncodeConstantPoolLength(int length) {
+ DCHECK((length & kConstantPoolLengthMaxMask) == length);
+ return ((length & 0xfff0) << 4) | (length & 0xf);
+}
+inline int DecodeConstantPoolLength(int instr) {
+ DCHECK_EQ(instr & kConstantPoolMarkerMask, kConstantPoolMarker);
+ return ((instr >> 4) & 0xfff0) | (instr & 0xf);
+}
+
+// Number of registers in normal ARM mode.
+constexpr int kNumRegisters = 16;
+constexpr int kRegSizeInBitsLog2 = 5;
+
+// VFP support.
+constexpr int kNumVFPSingleRegisters = 32;
+constexpr int kNumVFPDoubleRegisters = 32;
+constexpr int kNumVFPRegisters =
+ kNumVFPSingleRegisters + kNumVFPDoubleRegisters;
+
+// PC is register 15.
+constexpr int kPCRegister = 15;
+constexpr int kNoRegister = -1;
+
+// Used in embedded constant pool builder - max reach in bits for
+// various load instructions (unsigned)
+constexpr int kLdrMaxReachBits = 12;
+constexpr int kVldrMaxReachBits = 10;
+
+// Actual value of root register is offset from the root array's start
+// to take advantage of negative displacement values. Loads allow a uint12
+// value with a separate sign bit (range [-4095, +4095]), so the first root
+// is still addressable with a single load instruction.
+constexpr int kRootRegisterBias = 4095;
+
+// -----------------------------------------------------------------------------
+// Conditions.
+
+// Defines constants and accessor classes to assemble, disassemble and
+// simulate ARM instructions.
+//
+// Section references in the code refer to the "ARM Architecture Reference
+// Manual" from July 2005 (available at http://www.arm.com/miscPDFs/14128.pdf)
+//
+// Constants for specific fields are defined in their respective named enums.
+// General constants are in an anonymous enum in class Instr.
+
+// Values for the condition field as defined in section A3.2
+enum Condition {
+ kNoCondition = -1,
+
+ eq = 0 << 28, // Z set Equal.
+ ne = 1 << 28, // Z clear Not equal.
+ cs = 2 << 28, // C set Unsigned higher or same.
+ cc = 3 << 28, // C clear Unsigned lower.
+ mi = 4 << 28, // N set Negative.
+ pl = 5 << 28, // N clear Positive or zero.
+ vs = 6 << 28, // V set Overflow.
+ vc = 7 << 28, // V clear No overflow.
+ hi = 8 << 28, // C set, Z clear Unsigned higher.
+ ls = 9 << 28, // C clear or Z set Unsigned lower or same.
+ ge = 10 << 28, // N == V Greater or equal.
+ lt = 11 << 28, // N != V Less than.
+ gt = 12 << 28, // Z clear, N == V Greater than.
+ le = 13 << 28, // Z set or N != V Less then or equal
+ al = 14 << 28, // Always.
+
+ kSpecialCondition = 15 << 28, // Special condition (refer to section A3.2.1).
+ kNumberOfConditions = 16,
+
+ // Aliases.
+ hs = cs, // C set Unsigned higher or same.
+ lo = cc // C clear Unsigned lower.
+};
+
+inline Condition NegateCondition(Condition cond) {
+ DCHECK(cond != al);
+ return static_cast<Condition>(cond ^ ne);
+}
+
+// -----------------------------------------------------------------------------
+// Instructions encoding.
+
+// Instr is merely used by the Assembler to distinguish 32bit integers
+// representing instructions from usual 32 bit values.
+// Instruction objects are pointers to 32bit values, and provide methods to
+// access the various ISA fields.
+using Instr = int32_t;
+
+// Opcodes for Data-processing instructions (instructions with a type 0 and 1)
+// as defined in section A3.4
+enum Opcode {
+ AND = 0 << 21, // Logical AND.
+ EOR = 1 << 21, // Logical Exclusive OR.
+ SUB = 2 << 21, // Subtract.
+ RSB = 3 << 21, // Reverse Subtract.
+ ADD = 4 << 21, // Add.
+ ADC = 5 << 21, // Add with Carry.
+ SBC = 6 << 21, // Subtract with Carry.
+ RSC = 7 << 21, // Reverse Subtract with Carry.
+ TST = 8 << 21, // Test.
+ TEQ = 9 << 21, // Test Equivalence.
+ CMP = 10 << 21, // Compare.
+ CMN = 11 << 21, // Compare Negated.
+ ORR = 12 << 21, // Logical (inclusive) OR.
+ MOV = 13 << 21, // Move.
+ BIC = 14 << 21, // Bit Clear.
+ MVN = 15 << 21 // Move Not.
+};
+
+// The bits for bit 7-4 for some type 0 miscellaneous instructions.
+enum MiscInstructionsBits74 {
+ // With bits 22-21 01.
+ BX = 1 << 4,
+ BXJ = 2 << 4,
+ BLX = 3 << 4,
+ BKPT = 7 << 4,
+
+ // With bits 22-21 11.
+ CLZ = 1 << 4
+};
+
+// Instruction encoding bits and masks.
+enum {
+ H = 1 << 5, // Halfword (or byte).
+ S6 = 1 << 6, // Signed (or unsigned).
+ L = 1 << 20, // Load (or store).
+ S = 1 << 20, // Set condition code (or leave unchanged).
+ W = 1 << 21, // Writeback base register (or leave unchanged).
+ A = 1 << 21, // Accumulate in multiply instruction (or not).
+ B = 1 << 22, // Unsigned byte (or word).
+ N = 1 << 22, // Long (or short).
+ U = 1 << 23, // Positive (or negative) offset/index.
+ P = 1 << 24, // Offset/pre-indexed addressing (or post-indexed addressing).
+ I = 1 << 25, // Immediate shifter operand (or not).
+ B0 = 1 << 0,
+ B4 = 1 << 4,
+ B5 = 1 << 5,
+ B6 = 1 << 6,
+ B7 = 1 << 7,
+ B8 = 1 << 8,
+ B9 = 1 << 9,
+ B10 = 1 << 10,
+ B12 = 1 << 12,
+ B16 = 1 << 16,
+ B17 = 1 << 17,
+ B18 = 1 << 18,
+ B19 = 1 << 19,
+ B20 = 1 << 20,
+ B21 = 1 << 21,
+ B22 = 1 << 22,
+ B23 = 1 << 23,
+ B24 = 1 << 24,
+ B25 = 1 << 25,
+ B26 = 1 << 26,
+ B27 = 1 << 27,
+ B28 = 1 << 28,
+
+ // Instruction bit masks.
+ kCondMask = 15 << 28,
+ kALUMask = 0x6f << 21,
+ kRdMask = 15 << 12, // In str instruction.
+ kCoprocessorMask = 15 << 8,
+ kOpCodeMask = 15 << 21, // In data-processing instructions.
+ kImm24Mask = (1 << 24) - 1,
+ kImm16Mask = (1 << 16) - 1,
+ kImm8Mask = (1 << 8) - 1,
+ kOff12Mask = (1 << 12) - 1,
+ kOff8Mask = (1 << 8) - 1
+};
+
+enum BarrierOption {
+ OSHLD = 0x1,
+ OSHST = 0x2,
+ OSH = 0x3,
+ NSHLD = 0x5,
+ NSHST = 0x6,
+ NSH = 0x7,
+ ISHLD = 0x9,
+ ISHST = 0xa,
+ ISH = 0xb,
+ LD = 0xd,
+ ST = 0xe,
+ SY = 0xf,
+};
+
+// -----------------------------------------------------------------------------
+// Addressing modes and instruction variants.
+
+// Condition code updating mode.
+enum SBit {
+ SetCC = 1 << 20, // Set condition code.
+ LeaveCC = 0 << 20 // Leave condition code unchanged.
+};
+
+// Status register selection.
+enum SRegister { CPSR = 0 << 22, SPSR = 1 << 22 };
+
+// Shifter types for Data-processing operands as defined in section A5.1.2.
+enum ShiftOp {
+ LSL = 0 << 5, // Logical shift left.
+ LSR = 1 << 5, // Logical shift right.
+ ASR = 2 << 5, // Arithmetic shift right.
+ ROR = 3 << 5, // Rotate right.
+
+ // RRX is encoded as ROR with shift_imm == 0.
+ // Use a special code to make the distinction. The RRX ShiftOp is only used
+ // as an argument, and will never actually be encoded. The Assembler will
+ // detect it and emit the correct ROR shift operand with shift_imm == 0.
+ RRX = -1,
+ kNumberOfShifts = 4
+};
+
+// Status register fields.
+enum SRegisterField {
+ CPSR_c = CPSR | 1 << 16,
+ CPSR_x = CPSR | 1 << 17,
+ CPSR_s = CPSR | 1 << 18,
+ CPSR_f = CPSR | 1 << 19,
+ SPSR_c = SPSR | 1 << 16,
+ SPSR_x = SPSR | 1 << 17,
+ SPSR_s = SPSR | 1 << 18,
+ SPSR_f = SPSR | 1 << 19
+};
+
+// Status register field mask (or'ed SRegisterField enum values).
+using SRegisterFieldMask = uint32_t;
+
+// Memory operand addressing mode.
+enum AddrMode {
+ // Bit encoding P U W.
+ Offset = (8 | 4 | 0) << 21, // Offset (without writeback to base).
+ PreIndex = (8 | 4 | 1) << 21, // Pre-indexed addressing with writeback.
+ PostIndex = (0 | 4 | 0) << 21, // Post-indexed addressing with writeback.
+ NegOffset =
+ (8 | 0 | 0) << 21, // Negative offset (without writeback to base).
+ NegPreIndex = (8 | 0 | 1) << 21, // Negative pre-indexed with writeback.
+ NegPostIndex = (0 | 0 | 0) << 21 // Negative post-indexed with writeback.
+};
+
+// Load/store multiple addressing mode.
+enum BlockAddrMode {
+ // Bit encoding P U W .
+ da = (0 | 0 | 0) << 21, // Decrement after.
+ ia = (0 | 4 | 0) << 21, // Increment after.
+ db = (8 | 0 | 0) << 21, // Decrement before.
+ ib = (8 | 4 | 0) << 21, // Increment before.
+ da_w = (0 | 0 | 1) << 21, // Decrement after with writeback to base.
+ ia_w = (0 | 4 | 1) << 21, // Increment after with writeback to base.
+ db_w = (8 | 0 | 1) << 21, // Decrement before with writeback to base.
+ ib_w = (8 | 4 | 1) << 21, // Increment before with writeback to base.
+
+ // Alias modes for comparison when writeback does not matter.
+ da_x = (0 | 0 | 0) << 21, // Decrement after.
+ ia_x = (0 | 4 | 0) << 21, // Increment after.
+ db_x = (8 | 0 | 0) << 21, // Decrement before.
+ ib_x = (8 | 4 | 0) << 21, // Increment before.
+
+ kBlockAddrModeMask = (8 | 4 | 1) << 21
+};
+
+// Coprocessor load/store operand size.
+enum LFlag {
+ Long = 1 << 22, // Long load/store coprocessor.
+ Short = 0 << 22 // Short load/store coprocessor.
+};
+
+// Neon sizes.
+enum NeonSize { Neon8 = 0x0, Neon16 = 0x1, Neon32 = 0x2, Neon64 = 0x3 };
+
+// NEON data type, top bit set for unsigned data types.
+enum NeonDataType {
+ NeonS8 = 0,
+ NeonS16 = 1,
+ NeonS32 = 2,
+ NeonS64 = 3,
+ NeonU8 = 4,
+ NeonU16 = 5,
+ NeonU32 = 6,
+ NeonU64 = 7
+};
+
+inline int NeonU(NeonDataType dt) { return static_cast<int>(dt) >> 2; }
+inline int NeonSz(NeonDataType dt) { return static_cast<int>(dt) & 0x3; }
+
+// Convert sizes to data types (U bit is clear).
+inline NeonDataType NeonSizeToDataType(NeonSize size) {
+ DCHECK_NE(Neon64, size);
+ return static_cast<NeonDataType>(size);
+}
+
+inline NeonSize NeonDataTypeToSize(NeonDataType dt) {
+ return static_cast<NeonSize>(NeonSz(dt));
+}
+
+enum NeonListType { nlt_1 = 0x7, nlt_2 = 0xA, nlt_3 = 0x6, nlt_4 = 0x2 };
+
+// -----------------------------------------------------------------------------
+// Supervisor Call (svc) specific support.
+
+// Special Software Interrupt codes when used in the presence of the ARM
+// simulator.
+// svc (formerly swi) provides a 24bit immediate value. Use bits 22:0 for
+// standard SoftwareInterrupCode. Bit 23 is reserved for the stop feature.
+enum SoftwareInterruptCodes {
+ // transition to C code
+ kCallRtRedirected = 0x10,
+ // break point
+ kBreakpoint = 0x20,
+ // stop
+ kStopCode = 1 << 23
+};
+const uint32_t kStopCodeMask = kStopCode - 1;
+const uint32_t kMaxStopCode = kStopCode - 1;
+const int32_t kDefaultStopCode = -1;
+
+// Type of VFP register. Determines register encoding.
+enum VFPRegPrecision {
+ kSinglePrecision = 0,
+ kDoublePrecision = 1,
+ kSimd128Precision = 2
+};
+
+// VFP FPSCR constants.
+enum VFPConversionMode { kFPSCRRounding = 0, kDefaultRoundToZero = 1 };
+
+// This mask does not include the "inexact" or "input denormal" cumulative
+// exceptions flags, because we usually don't want to check for it.
+const uint32_t kVFPExceptionMask = 0xf;
+const uint32_t kVFPInvalidOpExceptionBit = 1 << 0;
+const uint32_t kVFPOverflowExceptionBit = 1 << 2;
+const uint32_t kVFPUnderflowExceptionBit = 1 << 3;
+const uint32_t kVFPInexactExceptionBit = 1 << 4;
+const uint32_t kVFPFlushToZeroMask = 1 << 24;
+const uint32_t kVFPDefaultNaNModeControlBit = 1 << 25;
+
+const uint32_t kVFPNConditionFlagBit = 1 << 31;
+const uint32_t kVFPZConditionFlagBit = 1 << 30;
+const uint32_t kVFPCConditionFlagBit = 1 << 29;
+const uint32_t kVFPVConditionFlagBit = 1 << 28;
+
+// VFP rounding modes. See ARM DDI 0406B Page A2-29.
+enum VFPRoundingMode {
+ RN = 0 << 22, // Round to Nearest.
+ RP = 1 << 22, // Round towards Plus Infinity.
+ RM = 2 << 22, // Round towards Minus Infinity.
+ RZ = 3 << 22, // Round towards zero.
+
+ // Aliases.
+ kRoundToNearest = RN,
+ kRoundToPlusInf = RP,
+ kRoundToMinusInf = RM,
+ kRoundToZero = RZ
+};
+
+const uint32_t kVFPRoundingModeMask = 3 << 22;
+
+enum CheckForInexactConversion {
+ kCheckForInexactConversion,
+ kDontCheckForInexactConversion
+};
+
+// -----------------------------------------------------------------------------
+// Hints.
+
+// Branch hints are not used on the ARM. They are defined so that they can
+// appear in shared function signatures, but will be ignored in ARM
+// implementations.
+enum Hint { no_hint };
+
+// Hints are not used on the arm. Negating is trivial.
+inline Hint NegateHint(Hint ignored) { return no_hint; }
+
+// -----------------------------------------------------------------------------
+// Instruction abstraction.
+
+// The class Instruction enables access to individual fields defined in the ARM
+// architecture instruction set encoding as described in figure A3-1.
+// Note that the Assembler uses typedef int32_t Instr.
+//
+// Example: Test whether the instruction at ptr does set the condition code
+// bits.
+//
+// bool InstructionSetsConditionCodes(byte* ptr) {
+// Instruction* instr = Instruction::At(ptr);
+// int type = instr->TypeValue();
+// return ((type == 0) || (type == 1)) && instr->HasS();
+// }
+//
+
+constexpr uint8_t kInstrSize = 4;
+constexpr uint8_t kInstrSizeLog2 = 2;
+
+class Instruction {
+ public:
+ // Difference between address of current opcode and value read from pc
+ // register.
+ static constexpr int kPcLoadDelta = 8;
+
+// Helper macro to define static accessors.
+// We use the cast to char* trick to bypass the strict anti-aliasing rules.
+#define DECLARE_STATIC_TYPED_ACCESSOR(return_type, Name) \
+ static inline return_type Name(Instr instr) { \
+ char* temp = reinterpret_cast<char*>(&instr); \
+ return reinterpret_cast<Instruction*>(temp)->Name(); \
+ }
+
+#define DECLARE_STATIC_ACCESSOR(Name) DECLARE_STATIC_TYPED_ACCESSOR(int, Name)
+
+ // Get the raw instruction bits.
+ inline Instr InstructionBits() const {
+ return *reinterpret_cast<const Instr*>(this);
+ }
+
+ // Set the raw instruction bits to value.
+ inline void SetInstructionBits(Instr value) {
+ *reinterpret_cast<Instr*>(this) = value;
+ }
+
+ // Extract a single bit from the instruction bits and return it as bit 0 in
+ // the result.
+ inline int Bit(int nr) const { return (InstructionBits() >> nr) & 1; }
+
+ // Extract a bit field <hi:lo> from the instruction bits and return it in the
+ // least-significant bits of the result.
+ inline int Bits(int hi, int lo) const {
+ return (InstructionBits() >> lo) & ((2 << (hi - lo)) - 1);
+ }
+
+ // Read a bit field <hi:lo>, leaving its position unchanged in the result.
+ inline int BitField(int hi, int lo) const {
+ return InstructionBits() & (((2 << (hi - lo)) - 1) << lo);
+ }
+
+ // Accessors for the different named fields used in the ARM encoding.
+ // The naming of these accessor corresponds to figure A3-1.
+ //
+ // Two kind of accessors are declared:
+ // - <Name>Field() will return the raw field, i.e. the field's bits at their
+ // original place in the instruction encoding.
+ // e.g. if instr is the 'addgt r0, r1, r2' instruction, encoded as
+ // 0xC0810002 ConditionField(instr) will return 0xC0000000.
+ // - <Name>Value() will return the field value, shifted back to bit 0.
+ // e.g. if instr is the 'addgt r0, r1, r2' instruction, encoded as
+ // 0xC0810002 ConditionField(instr) will return 0xC.
+
+ // Generally applicable fields
+ inline int ConditionValue() const { return Bits(31, 28); }
+ inline Condition ConditionField() const {
+ return static_cast<Condition>(BitField(31, 28));
+ }
+ DECLARE_STATIC_TYPED_ACCESSOR(int, ConditionValue)
+ DECLARE_STATIC_TYPED_ACCESSOR(Condition, ConditionField)
+
+ inline int TypeValue() const { return Bits(27, 25); }
+ inline int SpecialValue() const { return Bits(27, 23); }
+
+ inline int RnValue() const { return Bits(19, 16); }
+ DECLARE_STATIC_ACCESSOR(RnValue)
+ inline int RdValue() const { return Bits(15, 12); }
+ DECLARE_STATIC_ACCESSOR(RdValue)
+
+ inline int CoprocessorValue() const { return Bits(11, 8); }
+ // Support for VFP.
+ // Vn(19-16) | Vd(15-12) | Vm(3-0)
+ inline int VnValue() const { return Bits(19, 16); }
+ inline int VmValue() const { return Bits(3, 0); }
+ inline int VdValue() const { return Bits(15, 12); }
+ inline int NValue() const { return Bit(7); }
+ inline int MValue() const { return Bit(5); }
+ inline int DValue() const { return Bit(22); }
+ inline int RtValue() const { return Bits(15, 12); }
+ inline int PValue() const { return Bit(24); }
+ inline int UValue() const { return Bit(23); }
+ inline int Opc1Value() const { return (Bit(23) << 2) | Bits(21, 20); }
+ inline int Opc2Value() const { return Bits(19, 16); }
+ inline int Opc3Value() const { return Bits(7, 6); }
+ inline int SzValue() const { return Bit(8); }
+ inline int VLValue() const { return Bit(20); }
+ inline int VCValue() const { return Bit(8); }
+ inline int VAValue() const { return Bits(23, 21); }
+ inline int VBValue() const { return Bits(6, 5); }
+ inline int VFPNRegValue(VFPRegPrecision pre) {
+ return VFPGlueRegValue(pre, 16, 7);
+ }
+ inline int VFPMRegValue(VFPRegPrecision pre) {
+ return VFPGlueRegValue(pre, 0, 5);
+ }
+ inline int VFPDRegValue(VFPRegPrecision pre) {
+ return VFPGlueRegValue(pre, 12, 22);
+ }
+
+ // Fields used in Data processing instructions
+ inline int OpcodeValue() const { return static_cast<Opcode>(Bits(24, 21)); }
+ inline Opcode OpcodeField() const {
+ return static_cast<Opcode>(BitField(24, 21));
+ }
+ inline int SValue() const { return Bit(20); }
+ // with register
+ inline int RmValue() const { return Bits(3, 0); }
+ DECLARE_STATIC_ACCESSOR(RmValue)
+ inline int ShiftValue() const { return static_cast<ShiftOp>(Bits(6, 5)); }
+ inline ShiftOp ShiftField() const {
+ return static_cast<ShiftOp>(BitField(6, 5));
+ }
+ inline int RegShiftValue() const { return Bit(4); }
+ inline int RsValue() const { return Bits(11, 8); }
+ inline int ShiftAmountValue() const { return Bits(11, 7); }
+ // with immediate
+ inline int RotateValue() const { return Bits(11, 8); }
+ DECLARE_STATIC_ACCESSOR(RotateValue)
+ inline int Immed8Value() const { return Bits(7, 0); }
+ DECLARE_STATIC_ACCESSOR(Immed8Value)
+ inline int Immed4Value() const { return Bits(19, 16); }
+ inline int ImmedMovwMovtValue() const {
+ return Immed4Value() << 12 | Offset12Value();
+ }
+ DECLARE_STATIC_ACCESSOR(ImmedMovwMovtValue)
+
+ // Fields used in Load/Store instructions
+ inline int PUValue() const { return Bits(24, 23); }
+ inline int PUField() const { return BitField(24, 23); }
+ inline int BValue() const { return Bit(22); }
+ inline int WValue() const { return Bit(21); }
+ inline int LValue() const { return Bit(20); }
+ // with register uses same fields as Data processing instructions above
+ // with immediate
+ inline int Offset12Value() const { return Bits(11, 0); }
+ // multiple
+ inline int RlistValue() const { return Bits(15, 0); }
+ // extra loads and stores
+ inline int SignValue() const { return Bit(6); }
+ inline int HValue() const { return Bit(5); }
+ inline int ImmedHValue() const { return Bits(11, 8); }
+ inline int ImmedLValue() const { return Bits(3, 0); }
+
+ // Fields used in Branch instructions
+ inline int LinkValue() const { return Bit(24); }
+ inline int SImmed24Value() const {
+ return signed_bitextract_32(23, 0, InstructionBits());
+ }
+
+ bool IsBranch() { return Bit(27) == 1 && Bit(25) == 1; }
+
+ int GetBranchOffset() {
+ DCHECK(IsBranch());
+ return SImmed24Value() * kInstrSize;
+ }
+
+ void SetBranchOffset(int32_t branch_offset) {
+ DCHECK(IsBranch());
+ DCHECK_EQ(branch_offset % kInstrSize, 0);
+ int32_t new_imm24 = branch_offset / kInstrSize;
+ CHECK(is_int24(new_imm24));
+ SetInstructionBits((InstructionBits() & ~(kImm24Mask)) |
+ (new_imm24 & kImm24Mask));
+ }
+
+ // Fields used in Software interrupt instructions
+ inline SoftwareInterruptCodes SvcValue() const {
+ return static_cast<SoftwareInterruptCodes>(Bits(23, 0));
+ }
+
+ // Test for special encodings of type 0 instructions (extra loads and stores,
+ // as well as multiplications).
+ inline bool IsSpecialType0() const { return (Bit(7) == 1) && (Bit(4) == 1); }
+
+ // Test for miscellaneous instructions encodings of type 0 instructions.
+ inline bool IsMiscType0() const {
+ return (Bit(24) == 1) && (Bit(23) == 0) && (Bit(20) == 0) &&
+ ((Bit(7) == 0));
+ }
+
+ // Test for nop-like instructions which fall under type 1.
+ inline bool IsNopLikeType1() const { return Bits(24, 8) == 0x120F0; }
+
+ // Test for a stop instruction.
+ inline bool IsStop() const {
+ return (TypeValue() == 7) && (Bit(24) == 1) && (SvcValue() >= kStopCode);
+ }
+
+ // Special accessors that test for existence of a value.
+ inline bool HasS() const { return SValue() == 1; }
+ inline bool HasB() const { return BValue() == 1; }
+ inline bool HasW() const { return WValue() == 1; }
+ inline bool HasL() const { return LValue() == 1; }
+ inline bool HasU() const { return UValue() == 1; }
+ inline bool HasSign() const { return SignValue() == 1; }
+ inline bool HasH() const { return HValue() == 1; }
+ inline bool HasLink() const { return LinkValue() == 1; }
+
+ // Decode the double immediate from a vmov instruction.
+ Float64 DoubleImmedVmov() const;
+
+ // Instructions are read of out a code stream. The only way to get a
+ // reference to an instruction is to convert a pointer. There is no way
+ // to allocate or create instances of class Instruction.
+ // Use the At(pc) function to create references to Instruction.
+ static Instruction* At(Address pc) {
+ return reinterpret_cast<Instruction*>(pc);
+ }
+
+ private:
+ // Join split register codes, depending on register precision.
+ // four_bit is the position of the least-significant bit of the four
+ // bit specifier. one_bit is the position of the additional single bit
+ // specifier.
+ inline int VFPGlueRegValue(VFPRegPrecision pre, int four_bit, int one_bit) {
+ if (pre == kSinglePrecision) {
+ return (Bits(four_bit + 3, four_bit) << 1) | Bit(one_bit);
+ } else {
+ int reg_num = (Bit(one_bit) << 4) | Bits(four_bit + 3, four_bit);
+ if (pre == kDoublePrecision) {
+ return reg_num;
+ }
+ DCHECK_EQ(kSimd128Precision, pre);
+ DCHECK_EQ(reg_num & 1, 0);
+ return reg_num / 2;
+ }
+ }
+
+ // We need to prevent the creation of instances of class Instruction.
+ DISALLOW_IMPLICIT_CONSTRUCTORS(Instruction);
+};
+
+// Helper functions for converting between register numbers and names.
+class Registers {
+ public:
+ // Return the name of the register.
+ static const char* Name(int reg);
+
+ // Lookup the register number for the name provided.
+ static int Number(const char* name);
+
+ struct RegisterAlias {
+ int reg;
+ const char* name;
+ };
+
+ private:
+ static const char* names_[kNumRegisters];
+ static const RegisterAlias aliases_[];
+};
+
+// Helper functions for converting between VFP register numbers and names.
+class VFPRegisters {
+ public:
+ // Return the name of the register.
+ static const char* Name(int reg, bool is_double);
+
+ // Lookup the register number for the name provided.
+ // Set flag pointed by is_double to true if register
+ // is double-precision.
+ static int Number(const char* name, bool* is_double);
+
+ private:
+ static const char* names_[kNumVFPRegisters];
+};
+
+// Relative jumps on ARM can address ±32 MB.
+constexpr size_t kMaxPCRelativeCodeRangeInMB = 32;
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_ARM_CONSTANTS_ARM_H_
diff --git a/src/codegen/arm/cpu-arm.cc b/src/codegen/arm/cpu-arm.cc
new file mode 100644
index 0000000..47fe4bd
--- /dev/null
+++ b/src/codegen/arm/cpu-arm.cc
@@ -0,0 +1,63 @@
+// Copyright 2006-2009 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// CPU specific code for arm independent of OS goes here.
+#ifdef __arm__
+#ifdef __QNXNTO__
+#include <sys/mman.h> // for cache flushing.
+#undef MAP_TYPE // NOLINT
+#elif V8_OS_FREEBSD
+#include <machine/sysarch.h> // for cache flushing
+#include <sys/types.h>
+#else
+#include <sys/syscall.h> // for cache flushing.
+#endif
+#endif
+
+#if V8_TARGET_ARCH_ARM
+
+#include "src/codegen/cpu-features.h"
+
+namespace v8 {
+namespace internal {
+
+// The inlining of this seems to trigger an LTO bug that clobbers a register,
+// see https://crbug.com/952759 and https://bugs.llvm.org/show_bug.cgi?id=41575.
+V8_NOINLINE void CpuFeatures::FlushICache(void* start, size_t size) {
+#if !defined(USE_SIMULATOR)
+#if V8_OS_QNX
+ msync(start, size, MS_SYNC | MS_INVALIDATE_ICACHE);
+#elif V8_OS_FREEBSD
+ struct arm_sync_icache_args args = {
+ .addr = reinterpret_cast<uintptr_t>(start), .len = size};
+ sysarch(ARM_SYNC_ICACHE, reinterpret_cast<void*>(&args));
+#else
+ register uint32_t beg asm("r0") = reinterpret_cast<uint32_t>(start);
+ register uint32_t end asm("r1") = beg + size;
+ register uint32_t flg asm("r2") = 0;
+
+ asm volatile(
+ // This assembly works for both ARM and Thumb targets.
+
+ // Preserve r7; it is callee-saved, and GCC uses it as a frame pointer for
+ // Thumb targets.
+ " push {r7}\n"
+ // r0 = beg
+ // r1 = end
+ // r2 = flags (0)
+ " ldr r7, =%c[scno]\n" // r7 = syscall number
+ " svc 0\n"
+
+ " pop {r7}\n"
+ :
+ : "r"(beg), "r"(end), "r"(flg), [scno] "i"(__ARM_NR_cacheflush)
+ : "memory");
+#endif
+#endif // !USE_SIMULATOR
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_ARM
diff --git a/src/codegen/arm/interface-descriptors-arm.cc b/src/codegen/arm/interface-descriptors-arm.cc
new file mode 100644
index 0000000..96bf2ae
--- /dev/null
+++ b/src/codegen/arm/interface-descriptors-arm.cc
@@ -0,0 +1,284 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#if V8_TARGET_ARCH_ARM
+
+#include "src/codegen/interface-descriptors.h"
+
+#include "src/execution/frames.h"
+
+namespace v8 {
+namespace internal {
+
+const Register CallInterfaceDescriptor::ContextRegister() { return cp; }
+
+void CallInterfaceDescriptor::DefaultInitializePlatformSpecific(
+ CallInterfaceDescriptorData* data, int register_parameter_count) {
+ const Register default_stub_registers[] = {r0, r1, r2, r3, r4};
+ CHECK_LE(static_cast<size_t>(register_parameter_count),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(register_parameter_count,
+ default_stub_registers);
+}
+
+void RecordWriteDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ const Register default_stub_registers[] = {r0, r1, r2, r3, r4};
+
+ data->RestrictAllocatableRegisters(default_stub_registers,
+ arraysize(default_stub_registers));
+
+ CHECK_LE(static_cast<size_t>(kParameterCount),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(kParameterCount, default_stub_registers);
+}
+
+void EphemeronKeyBarrierDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ const Register default_stub_registers[] = {r0, r1, r2, r3, r4};
+
+ data->RestrictAllocatableRegisters(default_stub_registers,
+ arraysize(default_stub_registers));
+
+ CHECK_LE(static_cast<size_t>(kParameterCount),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(kParameterCount, default_stub_registers);
+}
+
+const Register LoadDescriptor::ReceiverRegister() { return r1; }
+const Register LoadDescriptor::NameRegister() { return r2; }
+const Register LoadDescriptor::SlotRegister() { return r0; }
+
+const Register LoadWithVectorDescriptor::VectorRegister() { return r3; }
+
+const Register
+LoadWithReceiverAndVectorDescriptor::LookupStartObjectRegister() {
+ return r4;
+}
+
+const Register StoreDescriptor::ReceiverRegister() { return r1; }
+const Register StoreDescriptor::NameRegister() { return r2; }
+const Register StoreDescriptor::ValueRegister() { return r0; }
+const Register StoreDescriptor::SlotRegister() { return r4; }
+
+const Register StoreWithVectorDescriptor::VectorRegister() { return r3; }
+
+const Register StoreTransitionDescriptor::SlotRegister() { return r4; }
+const Register StoreTransitionDescriptor::VectorRegister() { return r3; }
+const Register StoreTransitionDescriptor::MapRegister() { return r5; }
+
+const Register ApiGetterDescriptor::HolderRegister() { return r0; }
+const Register ApiGetterDescriptor::CallbackRegister() { return r3; }
+
+const Register GrowArrayElementsDescriptor::ObjectRegister() { return r0; }
+const Register GrowArrayElementsDescriptor::KeyRegister() { return r3; }
+
+// static
+const Register TypeConversionDescriptor::ArgumentRegister() { return r0; }
+
+void TypeofDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {r3};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallTrampolineDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r0 : number of arguments
+ // r1 : the target to call
+ Register registers[] = {r1, r0};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r0 : number of arguments (on the stack, not including receiver)
+ // r1 : the target to call
+ // r4 : arguments list length (untagged)
+ // r2 : arguments list (FixedArray)
+ Register registers[] = {r1, r0, r4, r2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallForwardVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r0 : number of arguments
+ // r2 : start index (to support rest parameters)
+ // r1 : the target to call
+ Register registers[] = {r1, r0, r2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallFunctionTemplateDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r1 : function template info
+ // r2 : number of arguments (on the stack, not including receiver)
+ Register registers[] = {r1, r2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallWithSpreadDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r0 : number of arguments (on the stack, not including receiver)
+ // r1 : the target to call
+ // r2 : the object to spread
+ Register registers[] = {r1, r0, r2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallWithArrayLikeDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r1 : the target to call
+ // r2 : the arguments list
+ Register registers[] = {r1, r2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r0 : number of arguments (on the stack, not including receiver)
+ // r1 : the target to call
+ // r3 : the new target
+ // r4 : arguments list length (untagged)
+ // r2 : arguments list (FixedArray)
+ Register registers[] = {r1, r3, r0, r4, r2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructForwardVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r0 : number of arguments
+ // r3 : the new target
+ // r2 : start index (to support rest parameters)
+ // r1 : the target to call
+ Register registers[] = {r1, r3, r0, r2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructWithSpreadDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r0 : number of arguments (on the stack, not including receiver)
+ // r1 : the target to call
+ // r3 : the new target
+ // r2 : the object to spread
+ Register registers[] = {r1, r3, r0, r2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructWithArrayLikeDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r1 : the target to call
+ // r3 : the new target
+ // r2 : the arguments list
+ Register registers[] = {r1, r3, r2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructStubDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r0 : number of arguments
+ // r1 : the target to call
+ // r3 : the new target
+ // r2 : allocation site or undefined
+ Register registers[] = {r1, r3, r0, r2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void AbortDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {r1};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CompareDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {r1, r0};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void BinaryOpDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {r1, r0};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ArgumentsAdaptorDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ r1, // JSFunction
+ r3, // the new target
+ r0, // actual number of arguments
+ r2, // expected number of arguments
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ApiCallbackDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ r1, // kApiFunctionAddress
+ r2, // kArgc
+ r3, // kCallData
+ r0, // kHolder
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void InterpreterDispatchDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ kInterpreterAccumulatorRegister, kInterpreterBytecodeOffsetRegister,
+ kInterpreterBytecodeArrayRegister, kInterpreterDispatchTableRegister};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void InterpreterPushArgsThenCallDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ r0, // argument count (not including receiver)
+ r2, // address of first argument
+ r1 // the target callable to be call
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void InterpreterPushArgsThenConstructDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ r0, // argument count (not including receiver)
+ r4, // address of the first argument
+ r1, // constructor to call
+ r3, // new target
+ r2, // allocation site feedback if available, undefined otherwise
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ResumeGeneratorDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ r0, // the value to pass to the generator
+ r1 // the JSGeneratorObject to resume
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void FrameDropperTrampolineDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ r1, // loaded new FP
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void RunMicrotasksEntryDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {r0, r1};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_ARM
diff --git a/src/codegen/arm/macro-assembler-arm.cc b/src/codegen/arm/macro-assembler-arm.cc
new file mode 100644
index 0000000..b72e385
--- /dev/null
+++ b/src/codegen/arm/macro-assembler-arm.cc
@@ -0,0 +1,2608 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <limits.h> // For LONG_MIN, LONG_MAX.
+
+#if V8_TARGET_ARCH_ARM
+
+#include "src/base/bits.h"
+#include "src/base/division-by-constant.h"
+#include "src/base/utils/random-number-generator.h"
+#include "src/codegen/assembler-inl.h"
+#include "src/codegen/callable.h"
+#include "src/codegen/code-factory.h"
+#include "src/codegen/external-reference-table.h"
+#include "src/codegen/macro-assembler.h"
+#include "src/codegen/register-configuration.h"
+#include "src/debug/debug.h"
+#include "src/execution/frames-inl.h"
+#include "src/heap/memory-chunk.h"
+#include "src/init/bootstrapper.h"
+#include "src/logging/counters.h"
+#include "src/numbers/double.h"
+#include "src/objects/objects-inl.h"
+#include "src/runtime/runtime.h"
+#include "src/snapshot/embedded/embedded-data.h"
+#include "src/snapshot/snapshot.h"
+#include "src/wasm/wasm-code-manager.h"
+
+// Satisfy cpplint check, but don't include platform-specific header. It is
+// included recursively via macro-assembler.h.
+#if 0
+#include "src/codegen/arm/macro-assembler-arm.h"
+#endif
+
+namespace v8 {
+namespace internal {
+
+int TurboAssembler::RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
+ Register exclusion1,
+ Register exclusion2,
+ Register exclusion3) const {
+ int bytes = 0;
+ RegList exclusions = 0;
+ if (exclusion1 != no_reg) {
+ exclusions |= exclusion1.bit();
+ if (exclusion2 != no_reg) {
+ exclusions |= exclusion2.bit();
+ if (exclusion3 != no_reg) {
+ exclusions |= exclusion3.bit();
+ }
+ }
+ }
+
+ RegList list = (kCallerSaved | lr.bit()) & ~exclusions;
+
+ bytes += NumRegs(list) * kPointerSize;
+
+ if (fp_mode == kSaveFPRegs) {
+ bytes += DwVfpRegister::kNumRegisters * DwVfpRegister::kSizeInBytes;
+ }
+
+ return bytes;
+}
+
+int TurboAssembler::PushCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1,
+ Register exclusion2, Register exclusion3) {
+ int bytes = 0;
+ RegList exclusions = 0;
+ if (exclusion1 != no_reg) {
+ exclusions |= exclusion1.bit();
+ if (exclusion2 != no_reg) {
+ exclusions |= exclusion2.bit();
+ if (exclusion3 != no_reg) {
+ exclusions |= exclusion3.bit();
+ }
+ }
+ }
+
+ RegList list = (kCallerSaved | lr.bit()) & ~exclusions;
+ stm(db_w, sp, list);
+
+ bytes += NumRegs(list) * kPointerSize;
+
+ if (fp_mode == kSaveFPRegs) {
+ SaveFPRegs(sp, lr);
+ bytes += DwVfpRegister::kNumRegisters * DwVfpRegister::kSizeInBytes;
+ }
+
+ return bytes;
+}
+
+int TurboAssembler::PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1,
+ Register exclusion2, Register exclusion3) {
+ int bytes = 0;
+ if (fp_mode == kSaveFPRegs) {
+ RestoreFPRegs(sp, lr);
+ bytes += DwVfpRegister::kNumRegisters * DwVfpRegister::kSizeInBytes;
+ }
+
+ RegList exclusions = 0;
+ if (exclusion1 != no_reg) {
+ exclusions |= exclusion1.bit();
+ if (exclusion2 != no_reg) {
+ exclusions |= exclusion2.bit();
+ if (exclusion3 != no_reg) {
+ exclusions |= exclusion3.bit();
+ }
+ }
+ }
+
+ RegList list = (kCallerSaved | lr.bit()) & ~exclusions;
+ ldm(ia_w, sp, list);
+
+ bytes += NumRegs(list) * kPointerSize;
+
+ return bytes;
+}
+
+void TurboAssembler::LoadFromConstantsTable(Register destination,
+ int constant_index) {
+ DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kBuiltinsConstantsTable));
+
+ const uint32_t offset =
+ FixedArray::kHeaderSize + constant_index * kPointerSize - kHeapObjectTag;
+
+ LoadRoot(destination, RootIndex::kBuiltinsConstantsTable);
+ ldr(destination, MemOperand(destination, offset));
+}
+
+void TurboAssembler::LoadRootRelative(Register destination, int32_t offset) {
+ ldr(destination, MemOperand(kRootRegister, offset));
+}
+
+void TurboAssembler::LoadRootRegisterOffset(Register destination,
+ intptr_t offset) {
+ if (offset == 0) {
+ Move(destination, kRootRegister);
+ } else {
+ add(destination, kRootRegister, Operand(offset));
+ }
+}
+
+void TurboAssembler::Jump(Register target, Condition cond) { bx(target, cond); }
+
+void TurboAssembler::Jump(intptr_t target, RelocInfo::Mode rmode,
+ Condition cond) {
+ mov(pc, Operand(target, rmode), LeaveCC, cond);
+}
+
+void TurboAssembler::Jump(Address target, RelocInfo::Mode rmode,
+ Condition cond) {
+ DCHECK(!RelocInfo::IsCodeTarget(rmode));
+ Jump(static_cast<intptr_t>(target), rmode, cond);
+}
+
+void TurboAssembler::Jump(Handle<Code> code, RelocInfo::Mode rmode,
+ Condition cond) {
+ DCHECK(RelocInfo::IsCodeTarget(rmode));
+ DCHECK_IMPLIES(options().isolate_independent_code,
+ Builtins::IsIsolateIndependentBuiltin(*code));
+ DCHECK_IMPLIES(options().use_pc_relative_calls_and_jumps,
+ Builtins::IsIsolateIndependentBuiltin(*code));
+
+ int builtin_index = Builtins::kNoBuiltinId;
+ bool target_is_builtin =
+ isolate()->builtins()->IsBuiltinHandle(code, &builtin_index);
+
+ if (options().use_pc_relative_calls_and_jumps && target_is_builtin) {
+ int32_t code_target_index = AddCodeTarget(code);
+ b(code_target_index * kInstrSize, cond, RelocInfo::RELATIVE_CODE_TARGET);
+ return;
+ } else if (root_array_available_ && options().isolate_independent_code) {
+ // This branch is taken only for specific cctests, where we force isolate
+ // creation at runtime. At this point, Code space isn't restricted to a
+ // size s.t. pc-relative calls may be used.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ int offset = IsolateData::builtin_entry_slot_offset(
+ static_cast<Builtins::Name>(code->builtin_index()));
+ ldr(scratch, MemOperand(kRootRegister, offset));
+ Jump(scratch, cond);
+ return;
+ } else if (options().inline_offheap_trampolines && target_is_builtin) {
+ // Inline the trampoline.
+ RecordCommentForOffHeapTrampoline(builtin_index);
+ EmbeddedData d = EmbeddedData::FromBlob();
+ Address entry = d.InstructionStartOfBuiltin(builtin_index);
+ // Use ip directly instead of using UseScratchRegisterScope, as we do not
+ // preserve scratch registers across calls.
+ mov(ip, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+ Jump(ip, cond);
+ return;
+ }
+
+ // 'code' is always generated ARM code, never THUMB code
+ Jump(static_cast<intptr_t>(code.address()), rmode, cond);
+}
+
+void TurboAssembler::Jump(const ExternalReference& reference) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ Move(scratch, reference);
+ Jump(scratch);
+}
+
+void TurboAssembler::Call(Register target, Condition cond) {
+ // Block constant pool for the call instruction sequence.
+ BlockConstPoolScope block_const_pool(this);
+ blx(target, cond);
+}
+
+void TurboAssembler::Call(Address target, RelocInfo::Mode rmode, Condition cond,
+ TargetAddressStorageMode mode,
+ bool check_constant_pool) {
+ // Check if we have to emit the constant pool before we block it.
+ if (check_constant_pool) MaybeCheckConstPool();
+ // Block constant pool for the call instruction sequence.
+ BlockConstPoolScope block_const_pool(this);
+
+ bool old_predictable_code_size = predictable_code_size();
+ if (mode == NEVER_INLINE_TARGET_ADDRESS) {
+ set_predictable_code_size(true);
+ }
+
+ // Use ip directly instead of using UseScratchRegisterScope, as we do not
+ // preserve scratch registers across calls.
+
+ // Call sequence on V7 or later may be :
+ // movw ip, #... @ call address low 16
+ // movt ip, #... @ call address high 16
+ // blx ip
+ // @ return address
+ // Or for pre-V7 or values that may be back-patched
+ // to avoid ICache flushes:
+ // ldr ip, [pc, #...] @ call address
+ // blx ip
+ // @ return address
+
+ mov(ip, Operand(target, rmode));
+ blx(ip, cond);
+
+ if (mode == NEVER_INLINE_TARGET_ADDRESS) {
+ set_predictable_code_size(old_predictable_code_size);
+ }
+}
+
+void TurboAssembler::Call(Handle<Code> code, RelocInfo::Mode rmode,
+ Condition cond, TargetAddressStorageMode mode,
+ bool check_constant_pool) {
+ DCHECK(RelocInfo::IsCodeTarget(rmode));
+ DCHECK_IMPLIES(options().isolate_independent_code,
+ Builtins::IsIsolateIndependentBuiltin(*code));
+ DCHECK_IMPLIES(options().use_pc_relative_calls_and_jumps,
+ Builtins::IsIsolateIndependentBuiltin(*code));
+
+ int builtin_index = Builtins::kNoBuiltinId;
+ bool target_is_builtin =
+ isolate()->builtins()->IsBuiltinHandle(code, &builtin_index);
+
+ if (target_is_builtin && options().use_pc_relative_calls_and_jumps) {
+ int32_t code_target_index = AddCodeTarget(code);
+ bl(code_target_index * kInstrSize, cond, RelocInfo::RELATIVE_CODE_TARGET);
+ return;
+ } else if (root_array_available_ && options().isolate_independent_code) {
+ // This branch is taken only for specific cctests, where we force isolate
+ // creation at runtime. At this point, Code space isn't restricted to a
+ // size s.t. pc-relative calls may be used.
+ int offset = IsolateData::builtin_entry_slot_offset(
+ static_cast<Builtins::Name>(code->builtin_index()));
+ ldr(ip, MemOperand(kRootRegister, offset));
+ Call(ip, cond);
+ return;
+ } else if (target_is_builtin && options().inline_offheap_trampolines) {
+ // Inline the trampoline.
+ CallBuiltin(builtin_index);
+ return;
+ }
+
+ // 'code' is always generated ARM code, never THUMB code
+ DCHECK(code->IsExecutable());
+ Call(code.address(), rmode, cond, mode);
+}
+
+void TurboAssembler::LoadEntryFromBuiltinIndex(Register builtin_index) {
+ STATIC_ASSERT(kSystemPointerSize == 4);
+ STATIC_ASSERT(kSmiShiftSize == 0);
+ STATIC_ASSERT(kSmiTagSize == 1);
+ STATIC_ASSERT(kSmiTag == 0);
+
+ // The builtin_index register contains the builtin index as a Smi.
+ // Untagging is folded into the indexing operand below.
+ mov(builtin_index,
+ Operand(builtin_index, LSL, kSystemPointerSizeLog2 - kSmiTagSize));
+ add(builtin_index, builtin_index,
+ Operand(IsolateData::builtin_entry_table_offset()));
+ ldr(builtin_index, MemOperand(kRootRegister, builtin_index));
+}
+
+void TurboAssembler::CallBuiltinByIndex(Register builtin_index) {
+ LoadEntryFromBuiltinIndex(builtin_index);
+ Call(builtin_index);
+}
+
+void TurboAssembler::CallBuiltin(int builtin_index, Condition cond) {
+ DCHECK(Builtins::IsBuiltinId(builtin_index));
+ RecordCommentForOffHeapTrampoline(builtin_index);
+ EmbeddedData d = EmbeddedData::FromBlob();
+ Address entry = d.InstructionStartOfBuiltin(builtin_index);
+ // Use ip directly instead of using UseScratchRegisterScope, as we do not
+ // preserve scratch registers across calls.
+ mov(ip, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+ Call(ip, cond);
+}
+
+void TurboAssembler::LoadCodeObjectEntry(Register destination,
+ Register code_object) {
+ // Code objects are called differently depending on whether we are generating
+ // builtin code (which will later be embedded into the binary) or compiling
+ // user JS code at runtime.
+ // * Builtin code runs in --jitless mode and thus must not call into on-heap
+ // Code targets. Instead, we dispatch through the builtins entry table.
+ // * Codegen at runtime does not have this restriction and we can use the
+ // shorter, branchless instruction sequence. The assumption here is that
+ // targets are usually generated code and not builtin Code objects.
+
+ if (options().isolate_independent_code) {
+ DCHECK(root_array_available());
+ Label if_code_is_off_heap, out;
+
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+
+ DCHECK(!AreAliased(destination, scratch));
+ DCHECK(!AreAliased(code_object, scratch));
+
+ // Check whether the Code object is an off-heap trampoline. If so, call its
+ // (off-heap) entry point directly without going through the (on-heap)
+ // trampoline. Otherwise, just call the Code object as always.
+ ldr(scratch, FieldMemOperand(code_object, Code::kFlagsOffset));
+ tst(scratch, Operand(Code::IsOffHeapTrampoline::kMask));
+ b(ne, &if_code_is_off_heap);
+
+ // Not an off-heap trampoline, the entry point is at
+ // Code::raw_instruction_start().
+ add(destination, code_object, Operand(Code::kHeaderSize - kHeapObjectTag));
+ jmp(&out);
+
+ // An off-heap trampoline, the entry point is loaded from the builtin entry
+ // table.
+ bind(&if_code_is_off_heap);
+ ldr(scratch, FieldMemOperand(code_object, Code::kBuiltinIndexOffset));
+ lsl(destination, scratch, Operand(kSystemPointerSizeLog2));
+ add(destination, destination, kRootRegister);
+ ldr(destination,
+ MemOperand(destination, IsolateData::builtin_entry_table_offset()));
+
+ bind(&out);
+ } else {
+ add(destination, code_object, Operand(Code::kHeaderSize - kHeapObjectTag));
+ }
+}
+
+void TurboAssembler::CallCodeObject(Register code_object) {
+ LoadCodeObjectEntry(code_object, code_object);
+ Call(code_object);
+}
+
+void TurboAssembler::JumpCodeObject(Register code_object) {
+ LoadCodeObjectEntry(code_object, code_object);
+ Jump(code_object);
+}
+
+void TurboAssembler::StoreReturnAddressAndCall(Register target) {
+ // This generates the final instruction sequence for calls to C functions
+ // once an exit frame has been constructed.
+ //
+ // Note that this assumes the caller code (i.e. the Code object currently
+ // being generated) is immovable or that the callee function cannot trigger
+ // GC, since the callee function will return to it.
+
+ // Compute the return address in lr to return to after the jump below. The pc
+ // is already at '+ 8' from the current instruction; but return is after three
+ // instructions, so add another 4 to pc to get the return address.
+ Assembler::BlockConstPoolScope block_const_pool(this);
+ add(lr, pc, Operand(4));
+ str(lr, MemOperand(sp));
+ Call(target);
+}
+
+void TurboAssembler::Ret(Condition cond) { bx(lr, cond); }
+
+void TurboAssembler::Drop(int count, Condition cond) {
+ if (count > 0) {
+ add(sp, sp, Operand(count * kPointerSize), LeaveCC, cond);
+ }
+}
+
+void TurboAssembler::Drop(Register count, Condition cond) {
+ add(sp, sp, Operand(count, LSL, kPointerSizeLog2), LeaveCC, cond);
+}
+
+void TurboAssembler::Ret(int drop, Condition cond) {
+ Drop(drop, cond);
+ Ret(cond);
+}
+
+void TurboAssembler::Call(Label* target) { bl(target); }
+
+void TurboAssembler::Push(Handle<HeapObject> handle) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ mov(scratch, Operand(handle));
+ push(scratch);
+}
+
+void TurboAssembler::Push(Smi smi) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ mov(scratch, Operand(smi));
+ push(scratch);
+}
+
+void TurboAssembler::PushArray(Register array, Register size, Register scratch,
+ PushArrayOrder order) {
+ UseScratchRegisterScope temps(this);
+ Register counter = scratch;
+ Register tmp = temps.Acquire();
+ DCHECK(!AreAliased(array, size, counter, tmp));
+ Label loop, entry;
+ if (order == PushArrayOrder::kReverse) {
+ mov(counter, Operand(0));
+ b(&entry);
+ bind(&loop);
+ ldr(tmp, MemOperand(array, counter, LSL, kSystemPointerSizeLog2));
+ push(tmp);
+ add(counter, counter, Operand(1));
+ bind(&entry);
+ cmp(counter, size);
+ b(lt, &loop);
+ } else {
+ mov(counter, size);
+ b(&entry);
+ bind(&loop);
+ ldr(tmp, MemOperand(array, counter, LSL, kSystemPointerSizeLog2));
+ push(tmp);
+ bind(&entry);
+ sub(counter, counter, Operand(1), SetCC);
+ b(ge, &loop);
+ }
+}
+
+void TurboAssembler::Move(Register dst, Smi smi) { mov(dst, Operand(smi)); }
+
+void TurboAssembler::Move(Register dst, Handle<HeapObject> value) {
+ // TODO(jgruber,v8:8887): Also consider a root-relative load when generating
+ // non-isolate-independent code. In many cases it might be cheaper than
+ // embedding the relocatable value.
+ if (root_array_available_ && options().isolate_independent_code) {
+ IndirectLoadConstant(dst, value);
+ return;
+ }
+ mov(dst, Operand(value));
+}
+
+void TurboAssembler::Move(Register dst, ExternalReference reference) {
+ // TODO(jgruber,v8:8887): Also consider a root-relative load when generating
+ // non-isolate-independent code. In many cases it might be cheaper than
+ // embedding the relocatable value.
+ if (root_array_available_ && options().isolate_independent_code) {
+ IndirectLoadExternalReference(dst, reference);
+ return;
+ }
+ mov(dst, Operand(reference));
+}
+
+void TurboAssembler::Move(Register dst, Register src, Condition cond) {
+ if (dst != src) {
+ mov(dst, src, LeaveCC, cond);
+ }
+}
+
+void TurboAssembler::Move(SwVfpRegister dst, SwVfpRegister src,
+ Condition cond) {
+ if (dst != src) {
+ vmov(dst, src, cond);
+ }
+}
+
+void TurboAssembler::Move(DwVfpRegister dst, DwVfpRegister src,
+ Condition cond) {
+ if (dst != src) {
+ vmov(dst, src, cond);
+ }
+}
+
+void TurboAssembler::Move(QwNeonRegister dst, QwNeonRegister src) {
+ if (dst != src) {
+ vmov(dst, src);
+ }
+}
+
+void TurboAssembler::MovePair(Register dst0, Register src0, Register dst1,
+ Register src1) {
+ DCHECK_NE(dst0, dst1);
+ if (dst0 != src1) {
+ Move(dst0, src0);
+ Move(dst1, src1);
+ } else if (dst1 != src0) {
+ // Swap the order of the moves to resolve the overlap.
+ Move(dst1, src1);
+ Move(dst0, src0);
+ } else {
+ // Worse case scenario, this is a swap.
+ Swap(dst0, src0);
+ }
+}
+
+void TurboAssembler::Swap(Register srcdst0, Register srcdst1) {
+ DCHECK(srcdst0 != srcdst1);
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ mov(scratch, srcdst0);
+ mov(srcdst0, srcdst1);
+ mov(srcdst1, scratch);
+}
+
+void TurboAssembler::Swap(DwVfpRegister srcdst0, DwVfpRegister srcdst1) {
+ DCHECK(srcdst0 != srcdst1);
+ DCHECK(VfpRegisterIsAvailable(srcdst0));
+ DCHECK(VfpRegisterIsAvailable(srcdst1));
+
+ if (CpuFeatures::IsSupported(NEON)) {
+ vswp(srcdst0, srcdst1);
+ } else {
+ UseScratchRegisterScope temps(this);
+ DwVfpRegister scratch = temps.AcquireD();
+ vmov(scratch, srcdst0);
+ vmov(srcdst0, srcdst1);
+ vmov(srcdst1, scratch);
+ }
+}
+
+void TurboAssembler::Swap(QwNeonRegister srcdst0, QwNeonRegister srcdst1) {
+ DCHECK(srcdst0 != srcdst1);
+ vswp(srcdst0, srcdst1);
+}
+
+void MacroAssembler::Mls(Register dst, Register src1, Register src2,
+ Register srcA, Condition cond) {
+ if (CpuFeatures::IsSupported(ARMv7)) {
+ CpuFeatureScope scope(this, ARMv7);
+ mls(dst, src1, src2, srcA, cond);
+ } else {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(srcA != scratch);
+ mul(scratch, src1, src2, LeaveCC, cond);
+ sub(dst, srcA, scratch, LeaveCC, cond);
+ }
+}
+
+void MacroAssembler::And(Register dst, Register src1, const Operand& src2,
+ Condition cond) {
+ if (!src2.IsRegister() && !src2.MustOutputRelocInfo(this) &&
+ src2.immediate() == 0) {
+ mov(dst, Operand::Zero(), LeaveCC, cond);
+ } else if (!(src2.InstructionsRequired(this) == 1) &&
+ !src2.MustOutputRelocInfo(this) &&
+ CpuFeatures::IsSupported(ARMv7) &&
+ base::bits::IsPowerOfTwo(src2.immediate() + 1)) {
+ CpuFeatureScope scope(this, ARMv7);
+ ubfx(dst, src1, 0,
+ base::bits::WhichPowerOfTwo(static_cast<uint32_t>(src2.immediate()) +
+ 1),
+ cond);
+ } else {
+ and_(dst, src1, src2, LeaveCC, cond);
+ }
+}
+
+void MacroAssembler::Ubfx(Register dst, Register src1, int lsb, int width,
+ Condition cond) {
+ DCHECK_LT(lsb, 32);
+ if (!CpuFeatures::IsSupported(ARMv7) || predictable_code_size()) {
+ int mask = (1u << (width + lsb)) - 1u - ((1u << lsb) - 1u);
+ and_(dst, src1, Operand(mask), LeaveCC, cond);
+ if (lsb != 0) {
+ mov(dst, Operand(dst, LSR, lsb), LeaveCC, cond);
+ }
+ } else {
+ CpuFeatureScope scope(this, ARMv7);
+ ubfx(dst, src1, lsb, width, cond);
+ }
+}
+
+void MacroAssembler::Sbfx(Register dst, Register src1, int lsb, int width,
+ Condition cond) {
+ DCHECK_LT(lsb, 32);
+ if (!CpuFeatures::IsSupported(ARMv7) || predictable_code_size()) {
+ int mask = (1 << (width + lsb)) - 1 - ((1 << lsb) - 1);
+ and_(dst, src1, Operand(mask), LeaveCC, cond);
+ int shift_up = 32 - lsb - width;
+ int shift_down = lsb + shift_up;
+ if (shift_up != 0) {
+ mov(dst, Operand(dst, LSL, shift_up), LeaveCC, cond);
+ }
+ if (shift_down != 0) {
+ mov(dst, Operand(dst, ASR, shift_down), LeaveCC, cond);
+ }
+ } else {
+ CpuFeatureScope scope(this, ARMv7);
+ sbfx(dst, src1, lsb, width, cond);
+ }
+}
+
+void TurboAssembler::Bfc(Register dst, Register src, int lsb, int width,
+ Condition cond) {
+ DCHECK_LT(lsb, 32);
+ if (!CpuFeatures::IsSupported(ARMv7) || predictable_code_size()) {
+ int mask = (1 << (width + lsb)) - 1 - ((1 << lsb) - 1);
+ bic(dst, src, Operand(mask));
+ } else {
+ CpuFeatureScope scope(this, ARMv7);
+ Move(dst, src, cond);
+ bfc(dst, lsb, width, cond);
+ }
+}
+
+void TurboAssembler::LoadRoot(Register destination, RootIndex index,
+ Condition cond) {
+ ldr(destination,
+ MemOperand(kRootRegister, RootRegisterOffsetForRootIndex(index)), cond);
+}
+
+void MacroAssembler::RecordWriteField(Register object, int offset,
+ Register value,
+ LinkRegisterStatus lr_status,
+ SaveFPRegsMode save_fp,
+ RememberedSetAction remembered_set_action,
+ SmiCheck smi_check) {
+ // First, check if a write barrier is even needed. The tests below
+ // catch stores of Smis.
+ Label done;
+
+ // Skip barrier if writing a smi.
+ if (smi_check == INLINE_SMI_CHECK) {
+ JumpIfSmi(value, &done);
+ }
+
+ // Although the object register is tagged, the offset is relative to the start
+ // of the object, so so offset must be a multiple of kPointerSize.
+ DCHECK(IsAligned(offset, kPointerSize));
+
+ if (emit_debug_code()) {
+ Label ok;
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ add(scratch, object, Operand(offset - kHeapObjectTag));
+ tst(scratch, Operand(kPointerSize - 1));
+ b(eq, &ok);
+ stop();
+ bind(&ok);
+ }
+
+ RecordWrite(object, Operand(offset - kHeapObjectTag), value, lr_status,
+ save_fp, remembered_set_action, OMIT_SMI_CHECK);
+
+ bind(&done);
+}
+
+void TurboAssembler::SaveRegisters(RegList registers) {
+ DCHECK_GT(NumRegs(registers), 0);
+ RegList regs = 0;
+ for (int i = 0; i < Register::kNumRegisters; ++i) {
+ if ((registers >> i) & 1u) {
+ regs |= Register::from_code(i).bit();
+ }
+ }
+
+ stm(db_w, sp, regs);
+}
+
+void TurboAssembler::RestoreRegisters(RegList registers) {
+ DCHECK_GT(NumRegs(registers), 0);
+ RegList regs = 0;
+ for (int i = 0; i < Register::kNumRegisters; ++i) {
+ if ((registers >> i) & 1u) {
+ regs |= Register::from_code(i).bit();
+ }
+ }
+ ldm(ia_w, sp, regs);
+}
+
+void TurboAssembler::CallEphemeronKeyBarrier(Register object, Operand offset,
+ SaveFPRegsMode fp_mode) {
+ EphemeronKeyBarrierDescriptor descriptor;
+ RegList registers = descriptor.allocatable_registers();
+
+ SaveRegisters(registers);
+
+ Register object_parameter(
+ descriptor.GetRegisterParameter(EphemeronKeyBarrierDescriptor::kObject));
+ Register slot_parameter(descriptor.GetRegisterParameter(
+ EphemeronKeyBarrierDescriptor::kSlotAddress));
+ Register fp_mode_parameter(
+ descriptor.GetRegisterParameter(EphemeronKeyBarrierDescriptor::kFPMode));
+
+ MoveObjectAndSlot(object_parameter, slot_parameter, object, offset);
+ Move(fp_mode_parameter, Smi::FromEnum(fp_mode));
+ Call(isolate()->builtins()->builtin_handle(Builtins::kEphemeronKeyBarrier),
+ RelocInfo::CODE_TARGET);
+ RestoreRegisters(registers);
+}
+
+void TurboAssembler::CallRecordWriteStub(
+ Register object, Operand offset, RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode) {
+ CallRecordWriteStub(
+ object, offset, remembered_set_action, fp_mode,
+ isolate()->builtins()->builtin_handle(Builtins::kRecordWrite),
+ kNullAddress);
+}
+
+void TurboAssembler::CallRecordWriteStub(
+ Register object, Operand offset, RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode, Address wasm_target) {
+ CallRecordWriteStub(object, offset, remembered_set_action, fp_mode,
+ Handle<Code>::null(), wasm_target);
+}
+
+void TurboAssembler::CallRecordWriteStub(
+ Register object, Operand offset, RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode, Handle<Code> code_target, Address wasm_target) {
+ DCHECK_NE(code_target.is_null(), wasm_target == kNullAddress);
+ // TODO(albertnetymk): For now we ignore remembered_set_action and fp_mode,
+ // i.e. always emit remember set and save FP registers in RecordWriteStub. If
+ // large performance regression is observed, we should use these values to
+ // avoid unnecessary work.
+
+ RecordWriteDescriptor descriptor;
+ RegList registers = descriptor.allocatable_registers();
+
+ SaveRegisters(registers);
+
+ Register object_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kObject));
+ Register slot_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kSlot));
+ Register remembered_set_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kRememberedSet));
+ Register fp_mode_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kFPMode));
+
+ MoveObjectAndSlot(object_parameter, slot_parameter, object, offset);
+
+ Move(remembered_set_parameter, Smi::FromEnum(remembered_set_action));
+ Move(fp_mode_parameter, Smi::FromEnum(fp_mode));
+ if (code_target.is_null()) {
+ Call(wasm_target, RelocInfo::WASM_STUB_CALL);
+ } else {
+ Call(code_target, RelocInfo::CODE_TARGET);
+ }
+
+ RestoreRegisters(registers);
+}
+
+void TurboAssembler::MoveObjectAndSlot(Register dst_object, Register dst_slot,
+ Register object, Operand offset) {
+ DCHECK_NE(dst_object, dst_slot);
+ DCHECK(offset.IsRegister() || offset.IsImmediate());
+ // If `offset` is a register, it cannot overlap with `object`.
+ DCHECK_IMPLIES(offset.IsRegister(), offset.rm() != object);
+
+ // If the slot register does not overlap with the object register, we can
+ // overwrite it.
+ if (dst_slot != object) {
+ add(dst_slot, object, offset);
+ Move(dst_object, object);
+ return;
+ }
+
+ DCHECK_EQ(dst_slot, object);
+
+ // If the destination object register does not overlap with the offset
+ // register, we can overwrite it.
+ if (!offset.IsRegister() || (offset.rm() != dst_object)) {
+ Move(dst_object, dst_slot);
+ add(dst_slot, dst_slot, offset);
+ return;
+ }
+
+ DCHECK_EQ(dst_object, offset.rm());
+
+ // We only have `dst_slot` and `dst_object` left as distinct registers so we
+ // have to swap them. We write this as a add+sub sequence to avoid using a
+ // scratch register.
+ add(dst_slot, dst_slot, dst_object);
+ sub(dst_object, dst_slot, dst_object);
+}
+
+// The register 'object' contains a heap object pointer. The heap object tag is
+// shifted away. A scratch register also needs to be available.
+void MacroAssembler::RecordWrite(Register object, Operand offset,
+ Register value, LinkRegisterStatus lr_status,
+ SaveFPRegsMode fp_mode,
+ RememberedSetAction remembered_set_action,
+ SmiCheck smi_check) {
+ DCHECK_NE(object, value);
+ if (emit_debug_code()) {
+ {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ add(scratch, object, offset);
+ ldr(scratch, MemOperand(scratch));
+ cmp(scratch, value);
+ }
+ Check(eq, AbortReason::kWrongAddressOrValuePassedToRecordWrite);
+ }
+
+ if ((remembered_set_action == OMIT_REMEMBERED_SET &&
+ !FLAG_incremental_marking) ||
+ FLAG_disable_write_barriers) {
+ return;
+ }
+
+ // First, check if a write barrier is even needed. The tests below
+ // catch stores of smis and stores into the young generation.
+ Label done;
+
+ if (smi_check == INLINE_SMI_CHECK) {
+ JumpIfSmi(value, &done);
+ }
+
+ CheckPageFlag(value, MemoryChunk::kPointersToHereAreInterestingMask, eq,
+ &done);
+ CheckPageFlag(object, MemoryChunk::kPointersFromHereAreInterestingMask, eq,
+ &done);
+
+ // Record the actual write.
+ if (lr_status == kLRHasNotBeenSaved) {
+ push(lr);
+ }
+ CallRecordWriteStub(object, offset, remembered_set_action, fp_mode);
+ if (lr_status == kLRHasNotBeenSaved) {
+ pop(lr);
+ }
+
+ bind(&done);
+}
+
+void TurboAssembler::PushCommonFrame(Register marker_reg) {
+ if (marker_reg.is_valid()) {
+ if (marker_reg.code() > fp.code()) {
+ stm(db_w, sp, fp.bit() | lr.bit());
+ mov(fp, Operand(sp));
+ Push(marker_reg);
+ } else {
+ stm(db_w, sp, marker_reg.bit() | fp.bit() | lr.bit());
+ add(fp, sp, Operand(kPointerSize));
+ }
+ } else {
+ stm(db_w, sp, fp.bit() | lr.bit());
+ mov(fp, sp);
+ }
+}
+
+void TurboAssembler::PushStandardFrame(Register function_reg) {
+ DCHECK(!function_reg.is_valid() || function_reg.code() < cp.code());
+ stm(db_w, sp,
+ (function_reg.is_valid() ? function_reg.bit() : 0) | cp.bit() | fp.bit() |
+ lr.bit());
+ int offset = -StandardFrameConstants::kContextOffset;
+ offset += function_reg.is_valid() ? kPointerSize : 0;
+ add(fp, sp, Operand(offset));
+ Push(kJavaScriptCallArgCountRegister);
+}
+
+void TurboAssembler::VFPCanonicalizeNaN(const DwVfpRegister dst,
+ const DwVfpRegister src,
+ const Condition cond) {
+ // Subtracting 0.0 preserves all inputs except for signalling NaNs, which
+ // become quiet NaNs. We use vsub rather than vadd because vsub preserves -0.0
+ // inputs: -0.0 + 0.0 = 0.0, but -0.0 - 0.0 = -0.0.
+ vsub(dst, src, kDoubleRegZero, cond);
+}
+
+void TurboAssembler::VFPCompareAndSetFlags(const SwVfpRegister src1,
+ const SwVfpRegister src2,
+ const Condition cond) {
+ // Compare and move FPSCR flags to the normal condition flags.
+ VFPCompareAndLoadFlags(src1, src2, pc, cond);
+}
+
+void TurboAssembler::VFPCompareAndSetFlags(const SwVfpRegister src1,
+ const float src2,
+ const Condition cond) {
+ // Compare and move FPSCR flags to the normal condition flags.
+ VFPCompareAndLoadFlags(src1, src2, pc, cond);
+}
+
+void TurboAssembler::VFPCompareAndSetFlags(const DwVfpRegister src1,
+ const DwVfpRegister src2,
+ const Condition cond) {
+ // Compare and move FPSCR flags to the normal condition flags.
+ VFPCompareAndLoadFlags(src1, src2, pc, cond);
+}
+
+void TurboAssembler::VFPCompareAndSetFlags(const DwVfpRegister src1,
+ const double src2,
+ const Condition cond) {
+ // Compare and move FPSCR flags to the normal condition flags.
+ VFPCompareAndLoadFlags(src1, src2, pc, cond);
+}
+
+void TurboAssembler::VFPCompareAndLoadFlags(const SwVfpRegister src1,
+ const SwVfpRegister src2,
+ const Register fpscr_flags,
+ const Condition cond) {
+ // Compare and load FPSCR.
+ vcmp(src1, src2, cond);
+ vmrs(fpscr_flags, cond);
+}
+
+void TurboAssembler::VFPCompareAndLoadFlags(const SwVfpRegister src1,
+ const float src2,
+ const Register fpscr_flags,
+ const Condition cond) {
+ // Compare and load FPSCR.
+ vcmp(src1, src2, cond);
+ vmrs(fpscr_flags, cond);
+}
+
+void TurboAssembler::VFPCompareAndLoadFlags(const DwVfpRegister src1,
+ const DwVfpRegister src2,
+ const Register fpscr_flags,
+ const Condition cond) {
+ // Compare and load FPSCR.
+ vcmp(src1, src2, cond);
+ vmrs(fpscr_flags, cond);
+}
+
+void TurboAssembler::VFPCompareAndLoadFlags(const DwVfpRegister src1,
+ const double src2,
+ const Register fpscr_flags,
+ const Condition cond) {
+ // Compare and load FPSCR.
+ vcmp(src1, src2, cond);
+ vmrs(fpscr_flags, cond);
+}
+
+void TurboAssembler::VmovHigh(Register dst, DwVfpRegister src) {
+ if (src.code() < 16) {
+ const LowDwVfpRegister loc = LowDwVfpRegister::from_code(src.code());
+ vmov(dst, loc.high());
+ } else {
+ vmov(NeonS32, dst, src, 1);
+ }
+}
+
+void TurboAssembler::VmovHigh(DwVfpRegister dst, Register src) {
+ if (dst.code() < 16) {
+ const LowDwVfpRegister loc = LowDwVfpRegister::from_code(dst.code());
+ vmov(loc.high(), src);
+ } else {
+ vmov(NeonS32, dst, 1, src);
+ }
+}
+
+void TurboAssembler::VmovLow(Register dst, DwVfpRegister src) {
+ if (src.code() < 16) {
+ const LowDwVfpRegister loc = LowDwVfpRegister::from_code(src.code());
+ vmov(dst, loc.low());
+ } else {
+ vmov(NeonS32, dst, src, 0);
+ }
+}
+
+void TurboAssembler::VmovLow(DwVfpRegister dst, Register src) {
+ if (dst.code() < 16) {
+ const LowDwVfpRegister loc = LowDwVfpRegister::from_code(dst.code());
+ vmov(loc.low(), src);
+ } else {
+ vmov(NeonS32, dst, 0, src);
+ }
+}
+
+void TurboAssembler::VmovExtended(Register dst, int src_code) {
+ DCHECK_LE(SwVfpRegister::kNumRegisters, src_code);
+ DCHECK_GT(SwVfpRegister::kNumRegisters * 2, src_code);
+ if (src_code & 0x1) {
+ VmovHigh(dst, DwVfpRegister::from_code(src_code / 2));
+ } else {
+ VmovLow(dst, DwVfpRegister::from_code(src_code / 2));
+ }
+}
+
+void TurboAssembler::VmovExtended(int dst_code, Register src) {
+ DCHECK_LE(SwVfpRegister::kNumRegisters, dst_code);
+ DCHECK_GT(SwVfpRegister::kNumRegisters * 2, dst_code);
+ if (dst_code & 0x1) {
+ VmovHigh(DwVfpRegister::from_code(dst_code / 2), src);
+ } else {
+ VmovLow(DwVfpRegister::from_code(dst_code / 2), src);
+ }
+}
+
+void TurboAssembler::VmovExtended(int dst_code, int src_code) {
+ if (src_code == dst_code) return;
+
+ if (src_code < SwVfpRegister::kNumRegisters &&
+ dst_code < SwVfpRegister::kNumRegisters) {
+ // src and dst are both s-registers.
+ vmov(SwVfpRegister::from_code(dst_code),
+ SwVfpRegister::from_code(src_code));
+ return;
+ }
+ DwVfpRegister dst_d_reg = DwVfpRegister::from_code(dst_code / 2);
+ DwVfpRegister src_d_reg = DwVfpRegister::from_code(src_code / 2);
+ int dst_offset = dst_code & 1;
+ int src_offset = src_code & 1;
+ if (CpuFeatures::IsSupported(NEON)) {
+ UseScratchRegisterScope temps(this);
+ DwVfpRegister scratch = temps.AcquireD();
+ // On Neon we can shift and insert from d-registers.
+ if (src_offset == dst_offset) {
+ // Offsets are the same, use vdup to copy the source to the opposite lane.
+ vdup(Neon32, scratch, src_d_reg, src_offset);
+ // Here we are extending the lifetime of scratch.
+ src_d_reg = scratch;
+ src_offset = dst_offset ^ 1;
+ }
+ if (dst_offset) {
+ if (dst_d_reg == src_d_reg) {
+ vdup(Neon32, dst_d_reg, src_d_reg, 0);
+ } else {
+ vsli(Neon64, dst_d_reg, src_d_reg, 32);
+ }
+ } else {
+ if (dst_d_reg == src_d_reg) {
+ vdup(Neon32, dst_d_reg, src_d_reg, 1);
+ } else {
+ vsri(Neon64, dst_d_reg, src_d_reg, 32);
+ }
+ }
+ return;
+ }
+
+ // Without Neon, use the scratch registers to move src and/or dst into
+ // s-registers.
+ UseScratchRegisterScope temps(this);
+ LowDwVfpRegister d_scratch = temps.AcquireLowD();
+ LowDwVfpRegister d_scratch2 = temps.AcquireLowD();
+ int s_scratch_code = d_scratch.low().code();
+ int s_scratch_code2 = d_scratch2.low().code();
+ if (src_code < SwVfpRegister::kNumRegisters) {
+ // src is an s-register, dst is not.
+ vmov(d_scratch, dst_d_reg);
+ vmov(SwVfpRegister::from_code(s_scratch_code + dst_offset),
+ SwVfpRegister::from_code(src_code));
+ vmov(dst_d_reg, d_scratch);
+ } else if (dst_code < SwVfpRegister::kNumRegisters) {
+ // dst is an s-register, src is not.
+ vmov(d_scratch, src_d_reg);
+ vmov(SwVfpRegister::from_code(dst_code),
+ SwVfpRegister::from_code(s_scratch_code + src_offset));
+ } else {
+ // Neither src or dst are s-registers. Both scratch double registers are
+ // available when there are 32 VFP registers.
+ vmov(d_scratch, src_d_reg);
+ vmov(d_scratch2, dst_d_reg);
+ vmov(SwVfpRegister::from_code(s_scratch_code + dst_offset),
+ SwVfpRegister::from_code(s_scratch_code2 + src_offset));
+ vmov(dst_d_reg, d_scratch2);
+ }
+}
+
+void TurboAssembler::VmovExtended(int dst_code, const MemOperand& src) {
+ if (dst_code < SwVfpRegister::kNumRegisters) {
+ vldr(SwVfpRegister::from_code(dst_code), src);
+ } else {
+ UseScratchRegisterScope temps(this);
+ LowDwVfpRegister scratch = temps.AcquireLowD();
+ // TODO(bbudge) If Neon supported, use load single lane form of vld1.
+ int dst_s_code = scratch.low().code() + (dst_code & 1);
+ vmov(scratch, DwVfpRegister::from_code(dst_code / 2));
+ vldr(SwVfpRegister::from_code(dst_s_code), src);
+ vmov(DwVfpRegister::from_code(dst_code / 2), scratch);
+ }
+}
+
+void TurboAssembler::VmovExtended(const MemOperand& dst, int src_code) {
+ if (src_code < SwVfpRegister::kNumRegisters) {
+ vstr(SwVfpRegister::from_code(src_code), dst);
+ } else {
+ // TODO(bbudge) If Neon supported, use store single lane form of vst1.
+ UseScratchRegisterScope temps(this);
+ LowDwVfpRegister scratch = temps.AcquireLowD();
+ int src_s_code = scratch.low().code() + (src_code & 1);
+ vmov(scratch, DwVfpRegister::from_code(src_code / 2));
+ vstr(SwVfpRegister::from_code(src_s_code), dst);
+ }
+}
+
+void TurboAssembler::ExtractLane(Register dst, QwNeonRegister src,
+ NeonDataType dt, int lane) {
+ int size = NeonSz(dt); // 0, 1, 2
+ int byte = lane << size;
+ int double_word = byte >> kDoubleSizeLog2;
+ int double_byte = byte & (kDoubleSize - 1);
+ int double_lane = double_byte >> size;
+ DwVfpRegister double_source =
+ DwVfpRegister::from_code(src.code() * 2 + double_word);
+ vmov(dt, dst, double_source, double_lane);
+}
+
+void TurboAssembler::ExtractLane(Register dst, DwVfpRegister src,
+ NeonDataType dt, int lane) {
+ int size = NeonSz(dt); // 0, 1, 2
+ int byte = lane << size;
+ int double_byte = byte & (kDoubleSize - 1);
+ int double_lane = double_byte >> size;
+ vmov(dt, dst, src, double_lane);
+}
+
+void TurboAssembler::ExtractLane(SwVfpRegister dst, QwNeonRegister src,
+ int lane) {
+ int s_code = src.code() * 4 + lane;
+ VmovExtended(dst.code(), s_code);
+}
+
+void TurboAssembler::ExtractLane(DwVfpRegister dst, QwNeonRegister src,
+ int lane) {
+ DwVfpRegister double_dst = DwVfpRegister::from_code(src.code() * 2 + lane);
+ vmov(dst, double_dst);
+}
+
+void TurboAssembler::ReplaceLane(QwNeonRegister dst, QwNeonRegister src,
+ Register src_lane, NeonDataType dt, int lane) {
+ Move(dst, src);
+ int size = NeonSz(dt); // 0, 1, 2
+ int byte = lane << size;
+ int double_word = byte >> kDoubleSizeLog2;
+ int double_byte = byte & (kDoubleSize - 1);
+ int double_lane = double_byte >> size;
+ DwVfpRegister double_dst =
+ DwVfpRegister::from_code(dst.code() * 2 + double_word);
+ vmov(dt, double_dst, double_lane, src_lane);
+}
+
+void TurboAssembler::ReplaceLane(QwNeonRegister dst, QwNeonRegister src,
+ SwVfpRegister src_lane, int lane) {
+ Move(dst, src);
+ int s_code = dst.code() * 4 + lane;
+ VmovExtended(s_code, src_lane.code());
+}
+
+void TurboAssembler::ReplaceLane(QwNeonRegister dst, QwNeonRegister src,
+ DwVfpRegister src_lane, int lane) {
+ Move(dst, src);
+ DwVfpRegister double_dst = DwVfpRegister::from_code(dst.code() * 2 + lane);
+ vmov(double_dst, src_lane);
+}
+
+void TurboAssembler::LslPair(Register dst_low, Register dst_high,
+ Register src_low, Register src_high,
+ Register shift) {
+ DCHECK(!AreAliased(dst_high, src_low));
+ DCHECK(!AreAliased(dst_high, shift));
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+
+ Label less_than_32;
+ Label done;
+ rsb(scratch, shift, Operand(32), SetCC);
+ b(gt, &less_than_32);
+ // If shift >= 32
+ and_(scratch, shift, Operand(0x1F));
+ lsl(dst_high, src_low, Operand(scratch));
+ mov(dst_low, Operand(0));
+ jmp(&done);
+ bind(&less_than_32);
+ // If shift < 32
+ lsl(dst_high, src_high, Operand(shift));
+ orr(dst_high, dst_high, Operand(src_low, LSR, scratch));
+ lsl(dst_low, src_low, Operand(shift));
+ bind(&done);
+}
+
+void TurboAssembler::LslPair(Register dst_low, Register dst_high,
+ Register src_low, Register src_high,
+ uint32_t shift) {
+ DCHECK_GE(63, shift);
+ DCHECK(!AreAliased(dst_high, src_low));
+
+ if (shift == 0) {
+ Move(dst_high, src_high);
+ Move(dst_low, src_low);
+ } else if (shift == 32) {
+ Move(dst_high, src_low);
+ Move(dst_low, Operand(0));
+ } else if (shift >= 32) {
+ shift &= 0x1F;
+ lsl(dst_high, src_low, Operand(shift));
+ mov(dst_low, Operand(0));
+ } else {
+ lsl(dst_high, src_high, Operand(shift));
+ orr(dst_high, dst_high, Operand(src_low, LSR, 32 - shift));
+ lsl(dst_low, src_low, Operand(shift));
+ }
+}
+
+void TurboAssembler::LsrPair(Register dst_low, Register dst_high,
+ Register src_low, Register src_high,
+ Register shift) {
+ DCHECK(!AreAliased(dst_low, src_high));
+ DCHECK(!AreAliased(dst_low, shift));
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+
+ Label less_than_32;
+ Label done;
+ rsb(scratch, shift, Operand(32), SetCC);
+ b(gt, &less_than_32);
+ // If shift >= 32
+ and_(scratch, shift, Operand(0x1F));
+ lsr(dst_low, src_high, Operand(scratch));
+ mov(dst_high, Operand(0));
+ jmp(&done);
+ bind(&less_than_32);
+ // If shift < 32
+
+ lsr(dst_low, src_low, Operand(shift));
+ orr(dst_low, dst_low, Operand(src_high, LSL, scratch));
+ lsr(dst_high, src_high, Operand(shift));
+ bind(&done);
+}
+
+void TurboAssembler::LsrPair(Register dst_low, Register dst_high,
+ Register src_low, Register src_high,
+ uint32_t shift) {
+ DCHECK_GE(63, shift);
+ DCHECK(!AreAliased(dst_low, src_high));
+
+ if (shift == 32) {
+ mov(dst_low, src_high);
+ mov(dst_high, Operand(0));
+ } else if (shift > 32) {
+ shift &= 0x1F;
+ lsr(dst_low, src_high, Operand(shift));
+ mov(dst_high, Operand(0));
+ } else if (shift == 0) {
+ Move(dst_low, src_low);
+ Move(dst_high, src_high);
+ } else {
+ lsr(dst_low, src_low, Operand(shift));
+ orr(dst_low, dst_low, Operand(src_high, LSL, 32 - shift));
+ lsr(dst_high, src_high, Operand(shift));
+ }
+}
+
+void TurboAssembler::AsrPair(Register dst_low, Register dst_high,
+ Register src_low, Register src_high,
+ Register shift) {
+ DCHECK(!AreAliased(dst_low, src_high));
+ DCHECK(!AreAliased(dst_low, shift));
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+
+ Label less_than_32;
+ Label done;
+ rsb(scratch, shift, Operand(32), SetCC);
+ b(gt, &less_than_32);
+ // If shift >= 32
+ and_(scratch, shift, Operand(0x1F));
+ asr(dst_low, src_high, Operand(scratch));
+ asr(dst_high, src_high, Operand(31));
+ jmp(&done);
+ bind(&less_than_32);
+ // If shift < 32
+ lsr(dst_low, src_low, Operand(shift));
+ orr(dst_low, dst_low, Operand(src_high, LSL, scratch));
+ asr(dst_high, src_high, Operand(shift));
+ bind(&done);
+}
+
+void TurboAssembler::AsrPair(Register dst_low, Register dst_high,
+ Register src_low, Register src_high,
+ uint32_t shift) {
+ DCHECK_GE(63, shift);
+ DCHECK(!AreAliased(dst_low, src_high));
+
+ if (shift == 32) {
+ mov(dst_low, src_high);
+ asr(dst_high, src_high, Operand(31));
+ } else if (shift > 32) {
+ shift &= 0x1F;
+ asr(dst_low, src_high, Operand(shift));
+ asr(dst_high, src_high, Operand(31));
+ } else if (shift == 0) {
+ Move(dst_low, src_low);
+ Move(dst_high, src_high);
+ } else {
+ lsr(dst_low, src_low, Operand(shift));
+ orr(dst_low, dst_low, Operand(src_high, LSL, 32 - shift));
+ asr(dst_high, src_high, Operand(shift));
+ }
+}
+
+void TurboAssembler::StubPrologue(StackFrame::Type type) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ mov(scratch, Operand(StackFrame::TypeToMarker(type)));
+ PushCommonFrame(scratch);
+}
+
+void TurboAssembler::Prologue() { PushStandardFrame(r1); }
+
+void TurboAssembler::EnterFrame(StackFrame::Type type,
+ bool load_constant_pool_pointer_reg) {
+ // r0-r3: preserved
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ mov(scratch, Operand(StackFrame::TypeToMarker(type)));
+ PushCommonFrame(scratch);
+}
+
+int TurboAssembler::LeaveFrame(StackFrame::Type type) {
+ // r0: preserved
+ // r1: preserved
+ // r2: preserved
+
+ // Drop the execution stack down to the frame pointer and restore
+ // the caller frame pointer and return address.
+ mov(sp, fp);
+ int frame_ends = pc_offset();
+ ldm(ia_w, sp, fp.bit() | lr.bit());
+ return frame_ends;
+}
+
+#ifdef V8_OS_WIN
+void TurboAssembler::AllocateStackSpace(Register bytes_scratch) {
+ // "Functions that allocate 4 KB or more on the stack must ensure that each
+ // page prior to the final page is touched in order." Source:
+ // https://docs.microsoft.com/en-us/cpp/build/overview-of-arm-abi-conventions?view=vs-2019#stack
+ UseScratchRegisterScope temps(this);
+ DwVfpRegister scratch = temps.AcquireD();
+ Label check_offset;
+ Label touch_next_page;
+ jmp(&check_offset);
+ bind(&touch_next_page);
+ sub(sp, sp, Operand(kStackPageSize));
+ // Just to touch the page, before we increment further.
+ vldr(scratch, MemOperand(sp));
+ sub(bytes_scratch, bytes_scratch, Operand(kStackPageSize));
+
+ bind(&check_offset);
+ cmp(bytes_scratch, Operand(kStackPageSize));
+ b(gt, &touch_next_page);
+
+ sub(sp, sp, bytes_scratch);
+}
+
+void TurboAssembler::AllocateStackSpace(int bytes) {
+ UseScratchRegisterScope temps(this);
+ DwVfpRegister scratch = no_dreg;
+ while (bytes > kStackPageSize) {
+ if (scratch == no_dreg) {
+ scratch = temps.AcquireD();
+ }
+ sub(sp, sp, Operand(kStackPageSize));
+ vldr(scratch, MemOperand(sp));
+ bytes -= kStackPageSize;
+ }
+ sub(sp, sp, Operand(bytes));
+}
+#endif
+
+void MacroAssembler::EnterExitFrame(bool save_doubles, int stack_space,
+ StackFrame::Type frame_type) {
+ DCHECK(frame_type == StackFrame::EXIT ||
+ frame_type == StackFrame::BUILTIN_EXIT);
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+
+ // Set up the frame structure on the stack.
+ DCHECK_EQ(2 * kPointerSize, ExitFrameConstants::kCallerSPDisplacement);
+ DCHECK_EQ(1 * kPointerSize, ExitFrameConstants::kCallerPCOffset);
+ DCHECK_EQ(0 * kPointerSize, ExitFrameConstants::kCallerFPOffset);
+ mov(scratch, Operand(StackFrame::TypeToMarker(frame_type)));
+ PushCommonFrame(scratch);
+ // Reserve room for saved entry sp.
+ sub(sp, fp, Operand(ExitFrameConstants::kFixedFrameSizeFromFp));
+ if (emit_debug_code()) {
+ mov(scratch, Operand::Zero());
+ str(scratch, MemOperand(fp, ExitFrameConstants::kSPOffset));
+ }
+
+ // Save the frame pointer and the context in top.
+ Move(scratch, ExternalReference::Create(IsolateAddressId::kCEntryFPAddress,
+ isolate()));
+ str(fp, MemOperand(scratch));
+ Move(scratch,
+ ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
+ str(cp, MemOperand(scratch));
+
+ // Optionally save all double registers.
+ if (save_doubles) {
+ SaveFPRegs(sp, scratch);
+ // Note that d0 will be accessible at
+ // fp - ExitFrameConstants::kFrameSize -
+ // DwVfpRegister::kNumRegisters * kDoubleSize,
+ // since the sp slot and code slot were pushed after the fp.
+ }
+
+ // Reserve place for the return address and stack space and align the frame
+ // preparing for calling the runtime function.
+ const int frame_alignment = MacroAssembler::ActivationFrameAlignment();
+ AllocateStackSpace((stack_space + 1) * kPointerSize);
+ if (frame_alignment > 0) {
+ DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
+ and_(sp, sp, Operand(-frame_alignment));
+ }
+
+ // Set the exit frame sp value to point just before the return address
+ // location.
+ add(scratch, sp, Operand(kPointerSize));
+ str(scratch, MemOperand(fp, ExitFrameConstants::kSPOffset));
+}
+
+int TurboAssembler::ActivationFrameAlignment() {
+#if V8_HOST_ARCH_ARM
+ // Running on the real platform. Use the alignment as mandated by the local
+ // environment.
+ // Note: This will break if we ever start generating snapshots on one ARM
+ // platform for another ARM platform with a different alignment.
+ return base::OS::ActivationFrameAlignment();
+#else // V8_HOST_ARCH_ARM
+ // If we are using the simulator then we should always align to the expected
+ // alignment. As the simulator is used to generate snapshots we do not know
+ // if the target platform will need alignment, so this is controlled from a
+ // flag.
+ return FLAG_sim_stack_alignment;
+#endif // V8_HOST_ARCH_ARM
+}
+
+void MacroAssembler::LeaveExitFrame(bool save_doubles, Register argument_count,
+ bool argument_count_is_length) {
+ ConstantPoolUnavailableScope constant_pool_unavailable(this);
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+
+ // Optionally restore all double registers.
+ if (save_doubles) {
+ // Calculate the stack location of the saved doubles and restore them.
+ const int offset = ExitFrameConstants::kFixedFrameSizeFromFp;
+ sub(r3, fp, Operand(offset + DwVfpRegister::kNumRegisters * kDoubleSize));
+ RestoreFPRegs(r3, scratch);
+ }
+
+ // Clear top frame.
+ mov(r3, Operand::Zero());
+ Move(scratch, ExternalReference::Create(IsolateAddressId::kCEntryFPAddress,
+ isolate()));
+ str(r3, MemOperand(scratch));
+
+ // Restore current context from top and clear it in debug mode.
+ Move(scratch,
+ ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
+ ldr(cp, MemOperand(scratch));
+#ifdef DEBUG
+ mov(r3, Operand(Context::kInvalidContext));
+ Move(scratch,
+ ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
+ str(r3, MemOperand(scratch));
+#endif
+
+ // Tear down the exit frame, pop the arguments, and return.
+ mov(sp, Operand(fp));
+ ldm(ia_w, sp, fp.bit() | lr.bit());
+ if (argument_count.is_valid()) {
+ if (argument_count_is_length) {
+ add(sp, sp, argument_count);
+ } else {
+ add(sp, sp, Operand(argument_count, LSL, kPointerSizeLog2));
+ }
+ }
+}
+
+void TurboAssembler::MovFromFloatResult(const DwVfpRegister dst) {
+ if (use_eabi_hardfloat()) {
+ Move(dst, d0);
+ } else {
+ vmov(dst, r0, r1);
+ }
+}
+
+// On ARM this is just a synonym to make the purpose clear.
+void TurboAssembler::MovFromFloatParameter(DwVfpRegister dst) {
+ MovFromFloatResult(dst);
+}
+
+void TurboAssembler::PrepareForTailCall(Register callee_args_count,
+ Register caller_args_count,
+ Register scratch0, Register scratch1) {
+ DCHECK(!AreAliased(callee_args_count, caller_args_count, scratch0, scratch1));
+
+ // Calculate the end of destination area where we will put the arguments
+ // after we drop current frame. We add kPointerSize to count the receiver
+ // argument which is not included into formal parameters count.
+ Register dst_reg = scratch0;
+ add(dst_reg, fp, Operand(caller_args_count, LSL, kPointerSizeLog2));
+ add(dst_reg, dst_reg,
+ Operand(StandardFrameConstants::kCallerSPOffset + kPointerSize));
+
+ Register src_reg = caller_args_count;
+ // Calculate the end of source area. +kPointerSize is for the receiver.
+ add(src_reg, sp, Operand(callee_args_count, LSL, kPointerSizeLog2));
+ add(src_reg, src_reg, Operand(kPointerSize));
+
+ if (FLAG_debug_code) {
+ cmp(src_reg, dst_reg);
+ Check(lo, AbortReason::kStackAccessBelowStackPointer);
+ }
+
+ // Restore caller's frame pointer and return address now as they will be
+ // overwritten by the copying loop.
+ ldr(lr, MemOperand(fp, StandardFrameConstants::kCallerPCOffset));
+ ldr(fp, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
+
+ // Now copy callee arguments to the caller frame going backwards to avoid
+ // callee arguments corruption (source and destination areas could overlap).
+
+ // Both src_reg and dst_reg are pointing to the word after the one to copy,
+ // so they must be pre-decremented in the loop.
+ Register tmp_reg = scratch1;
+ Label loop, entry;
+ b(&entry);
+ bind(&loop);
+ ldr(tmp_reg, MemOperand(src_reg, -kPointerSize, PreIndex));
+ str(tmp_reg, MemOperand(dst_reg, -kPointerSize, PreIndex));
+ bind(&entry);
+ cmp(sp, src_reg);
+ b(ne, &loop);
+
+ // Leave current frame.
+ mov(sp, dst_reg);
+}
+
+void MacroAssembler::LoadStackLimit(Register destination, StackLimitKind kind) {
+ DCHECK(root_array_available());
+ Isolate* isolate = this->isolate();
+ ExternalReference limit =
+ kind == StackLimitKind::kRealStackLimit
+ ? ExternalReference::address_of_real_jslimit(isolate)
+ : ExternalReference::address_of_jslimit(isolate);
+ DCHECK(TurboAssembler::IsAddressableThroughRootRegister(isolate, limit));
+
+ intptr_t offset =
+ TurboAssembler::RootRegisterOffsetForExternalReference(isolate, limit);
+ CHECK(is_int32(offset));
+ ldr(destination, MemOperand(kRootRegister, offset));
+}
+
+void MacroAssembler::StackOverflowCheck(Register num_args, Register scratch,
+ Label* stack_overflow) {
+ // Check the stack for overflow. We are not trying to catch
+ // interruptions (e.g. debug break and preemption) here, so the "real stack
+ // limit" is checked.
+ LoadStackLimit(scratch, StackLimitKind::kRealStackLimit);
+ // Make scratch the space we have left. The stack might already be overflowed
+ // here which will cause scratch to become negative.
+ sub(scratch, sp, scratch);
+ // Check if the arguments will overflow the stack.
+ cmp(scratch, Operand(num_args, LSL, kPointerSizeLog2));
+ b(le, stack_overflow); // Signed comparison.
+}
+
+void MacroAssembler::InvokePrologue(Register expected_parameter_count,
+ Register actual_parameter_count,
+ Label* done, InvokeFlag flag) {
+ Label regular_invoke;
+ // r0: actual arguments count
+ // r1: function (passed through to callee)
+ // r2: expected arguments count
+ DCHECK_EQ(actual_parameter_count, r0);
+ DCHECK_EQ(expected_parameter_count, r2);
+
+#ifdef V8_NO_ARGUMENTS_ADAPTOR
+ // If the expected parameter count is equal to the adaptor sentinel, no need
+ // to push undefined value as arguments.
+ cmp(expected_parameter_count, Operand(kDontAdaptArgumentsSentinel));
+ b(eq, ®ular_invoke);
+
+ // If overapplication or if the actual argument count is equal to the
+ // formal parameter count, no need to push extra undefined values.
+ sub(expected_parameter_count, expected_parameter_count,
+ actual_parameter_count, SetCC);
+ b(le, ®ular_invoke);
+
+ Label stack_overflow;
+ Register scratch = r4;
+ StackOverflowCheck(expected_parameter_count, scratch, &stack_overflow);
+
+ // Underapplication. Move the arguments already in the stack, including the
+ // receiver and the return address.
+ {
+ Label copy, check;
+ Register num = r5, src = r6, dest = r9; // r7 and r8 are context and root.
+ mov(src, sp);
+ // Update stack pointer.
+ lsl(scratch, expected_parameter_count, Operand(kSystemPointerSizeLog2));
+ AllocateStackSpace(scratch);
+ mov(dest, sp);
+ mov(num, actual_parameter_count);
+ b(&check);
+ bind(©);
+ ldr(scratch, MemOperand(src, kSystemPointerSize, PostIndex));
+ str(scratch, MemOperand(dest, kSystemPointerSize, PostIndex));
+ sub(num, num, Operand(1), SetCC);
+ bind(&check);
+ b(ge, ©);
+ }
+
+ // Fill remaining expected arguments with undefined values.
+ LoadRoot(scratch, RootIndex::kUndefinedValue);
+ {
+ Label loop;
+ bind(&loop);
+ str(scratch, MemOperand(r9, kSystemPointerSize, PostIndex));
+ sub(expected_parameter_count, expected_parameter_count, Operand(1), SetCC);
+ b(gt, &loop);
+ }
+ b(®ular_invoke);
+
+ bind(&stack_overflow);
+ {
+ FrameScope frame(this,
+ has_frame() ? StackFrame::NONE : StackFrame::INTERNAL);
+ CallRuntime(Runtime::kThrowStackOverflow);
+ bkpt(0);
+ }
+#else
+ // Check whether the expected and actual arguments count match. If not,
+ // setup registers according to contract with ArgumentsAdaptorTrampoline.
+ cmp(expected_parameter_count, actual_parameter_count);
+ b(eq, ®ular_invoke);
+
+ Handle<Code> adaptor = BUILTIN_CODE(isolate(), ArgumentsAdaptorTrampoline);
+ if (flag == CALL_FUNCTION) {
+ Call(adaptor);
+ b(done);
+ } else {
+ Jump(adaptor, RelocInfo::CODE_TARGET);
+ }
+#endif
+ bind(®ular_invoke);
+}
+
+void MacroAssembler::CallDebugOnFunctionCall(Register fun, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count) {
+ // Load receiver to pass it later to DebugOnFunctionCall hook.
+ ldr(r4, ReceiverOperand(actual_parameter_count));
+ FrameScope frame(this, has_frame() ? StackFrame::NONE : StackFrame::INTERNAL);
+
+ SmiTag(expected_parameter_count);
+ Push(expected_parameter_count);
+
+ SmiTag(actual_parameter_count);
+ Push(actual_parameter_count);
+
+ if (new_target.is_valid()) {
+ Push(new_target);
+ }
+ Push(fun);
+ Push(fun);
+ Push(r4);
+ CallRuntime(Runtime::kDebugOnFunctionCall);
+ Pop(fun);
+ if (new_target.is_valid()) {
+ Pop(new_target);
+ }
+
+ Pop(actual_parameter_count);
+ SmiUntag(actual_parameter_count);
+
+ Pop(expected_parameter_count);
+ SmiUntag(expected_parameter_count);
+}
+
+void MacroAssembler::InvokeFunctionCode(Register function, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count,
+ InvokeFlag flag) {
+ // You can't call a function without a valid frame.
+ DCHECK_IMPLIES(flag == CALL_FUNCTION, has_frame());
+ DCHECK_EQ(function, r1);
+ DCHECK_IMPLIES(new_target.is_valid(), new_target == r3);
+
+ // On function call, call into the debugger if necessary.
+ Label debug_hook, continue_after_hook;
+ {
+ ExternalReference debug_hook_active =
+ ExternalReference::debug_hook_on_function_call_address(isolate());
+ Move(r4, debug_hook_active);
+ ldrsb(r4, MemOperand(r4));
+ cmp(r4, Operand(0));
+ b(ne, &debug_hook);
+ }
+ bind(&continue_after_hook);
+
+ // Clear the new.target register if not given.
+ if (!new_target.is_valid()) {
+ LoadRoot(r3, RootIndex::kUndefinedValue);
+ }
+
+ Label done;
+ InvokePrologue(expected_parameter_count, actual_parameter_count, &done, flag);
+ // We call indirectly through the code field in the function to
+ // allow recompilation to take effect without changing any of the
+ // call sites.
+ Register code = kJavaScriptCallCodeStartRegister;
+ ldr(code, FieldMemOperand(function, JSFunction::kCodeOffset));
+ if (flag == CALL_FUNCTION) {
+ CallCodeObject(code);
+ } else {
+ DCHECK(flag == JUMP_FUNCTION);
+ JumpCodeObject(code);
+ }
+ b(&done);
+
+ // Deferred debug hook.
+ bind(&debug_hook);
+ CallDebugOnFunctionCall(function, new_target, expected_parameter_count,
+ actual_parameter_count);
+ b(&continue_after_hook);
+
+ // Continue here if InvokePrologue does handle the invocation due to
+ // mismatched parameter counts.
+ bind(&done);
+}
+
+void MacroAssembler::InvokeFunctionWithNewTarget(
+ Register fun, Register new_target, Register actual_parameter_count,
+ InvokeFlag flag) {
+ // You can't call a function without a valid frame.
+ DCHECK_IMPLIES(flag == CALL_FUNCTION, has_frame());
+
+ // Contract with called JS functions requires that function is passed in r1.
+ DCHECK_EQ(fun, r1);
+
+ Register expected_reg = r2;
+ Register temp_reg = r4;
+
+ ldr(temp_reg, FieldMemOperand(r1, JSFunction::kSharedFunctionInfoOffset));
+ ldr(cp, FieldMemOperand(r1, JSFunction::kContextOffset));
+ ldrh(expected_reg,
+ FieldMemOperand(temp_reg,
+ SharedFunctionInfo::kFormalParameterCountOffset));
+
+ InvokeFunctionCode(fun, new_target, expected_reg, actual_parameter_count,
+ flag);
+}
+
+void MacroAssembler::InvokeFunction(Register function,
+ Register expected_parameter_count,
+ Register actual_parameter_count,
+ InvokeFlag flag) {
+ // You can't call a function without a valid frame.
+ DCHECK_IMPLIES(flag == CALL_FUNCTION, has_frame());
+
+ // Contract with called JS functions requires that function is passed in r1.
+ DCHECK_EQ(function, r1);
+
+ // Get the function and setup the context.
+ ldr(cp, FieldMemOperand(r1, JSFunction::kContextOffset));
+
+ InvokeFunctionCode(r1, no_reg, expected_parameter_count,
+ actual_parameter_count, flag);
+}
+
+void MacroAssembler::MaybeDropFrames() {
+ // Check whether we need to drop frames to restart a function on the stack.
+ ExternalReference restart_fp =
+ ExternalReference::debug_restart_fp_address(isolate());
+ Move(r1, restart_fp);
+ ldr(r1, MemOperand(r1));
+ tst(r1, r1);
+ Jump(BUILTIN_CODE(isolate(), FrameDropperTrampoline), RelocInfo::CODE_TARGET,
+ ne);
+}
+
+void MacroAssembler::PushStackHandler() {
+ // Adjust this code if not the case.
+ STATIC_ASSERT(StackHandlerConstants::kSize == 2 * kPointerSize);
+ STATIC_ASSERT(StackHandlerConstants::kNextOffset == 0 * kPointerSize);
+
+ Push(Smi::zero()); // Padding.
+ // Link the current handler as the next handler.
+ Move(r6,
+ ExternalReference::Create(IsolateAddressId::kHandlerAddress, isolate()));
+ ldr(r5, MemOperand(r6));
+ push(r5);
+ // Set this new handler as the current one.
+ str(sp, MemOperand(r6));
+}
+
+void MacroAssembler::PopStackHandler() {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ STATIC_ASSERT(StackHandlerConstants::kNextOffset == 0);
+ pop(r1);
+ Move(scratch,
+ ExternalReference::Create(IsolateAddressId::kHandlerAddress, isolate()));
+ str(r1, MemOperand(scratch));
+ add(sp, sp, Operand(StackHandlerConstants::kSize - kPointerSize));
+}
+
+void MacroAssembler::CompareObjectType(Register object, Register map,
+ Register type_reg, InstanceType type) {
+ UseScratchRegisterScope temps(this);
+ const Register temp = type_reg == no_reg ? temps.Acquire() : type_reg;
+
+ LoadMap(map, object);
+ CompareInstanceType(map, temp, type);
+}
+
+void MacroAssembler::CompareInstanceType(Register map, Register type_reg,
+ InstanceType type) {
+ ldrh(type_reg, FieldMemOperand(map, Map::kInstanceTypeOffset));
+ cmp(type_reg, Operand(type));
+}
+
+void MacroAssembler::CompareRoot(Register obj, RootIndex index) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(obj != scratch);
+ LoadRoot(scratch, index);
+ cmp(obj, scratch);
+}
+
+void MacroAssembler::JumpIfIsInRange(Register value, unsigned lower_limit,
+ unsigned higher_limit,
+ Label* on_in_range) {
+ if (lower_limit != 0) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ sub(scratch, value, Operand(lower_limit));
+ cmp(scratch, Operand(higher_limit - lower_limit));
+ } else {
+ cmp(value, Operand(higher_limit));
+ }
+ b(ls, on_in_range);
+}
+
+void TurboAssembler::TryInlineTruncateDoubleToI(Register result,
+ DwVfpRegister double_input,
+ Label* done) {
+ UseScratchRegisterScope temps(this);
+ SwVfpRegister single_scratch = SwVfpRegister::no_reg();
+ if (temps.CanAcquireVfp<SwVfpRegister>()) {
+ single_scratch = temps.AcquireS();
+ } else {
+ // Re-use the input as a scratch register. However, we can only do this if
+ // the input register is d0-d15 as there are no s32+ registers.
+ DCHECK_LT(double_input.code(), LowDwVfpRegister::kNumRegisters);
+ LowDwVfpRegister double_scratch =
+ LowDwVfpRegister::from_code(double_input.code());
+ single_scratch = double_scratch.low();
+ }
+ vcvt_s32_f64(single_scratch, double_input);
+ vmov(result, single_scratch);
+
+ Register scratch = temps.Acquire();
+ // If result is not saturated (0x7FFFFFFF or 0x80000000), we are done.
+ sub(scratch, result, Operand(1));
+ cmp(scratch, Operand(0x7FFFFFFE));
+ b(lt, done);
+}
+
+void TurboAssembler::TruncateDoubleToI(Isolate* isolate, Zone* zone,
+ Register result,
+ DwVfpRegister double_input,
+ StubCallMode stub_mode) {
+ Label done;
+
+ TryInlineTruncateDoubleToI(result, double_input, &done);
+
+ // If we fell through then inline version didn't succeed - call stub instead.
+ push(lr);
+ AllocateStackSpace(kDoubleSize); // Put input on stack.
+ vstr(double_input, MemOperand(sp, 0));
+
+ if (stub_mode == StubCallMode::kCallWasmRuntimeStub) {
+ Call(wasm::WasmCode::kDoubleToI, RelocInfo::WASM_STUB_CALL);
+ } else if (options().inline_offheap_trampolines) {
+ CallBuiltin(Builtins::kDoubleToI);
+ } else {
+ Call(BUILTIN_CODE(isolate, DoubleToI), RelocInfo::CODE_TARGET);
+ }
+ ldr(result, MemOperand(sp, 0));
+
+ add(sp, sp, Operand(kDoubleSize));
+ pop(lr);
+
+ bind(&done);
+}
+
+void MacroAssembler::CallRuntime(const Runtime::Function* f, int num_arguments,
+ SaveFPRegsMode save_doubles) {
+ // All parameters are on the stack. r0 has the return value after call.
+
+ // If the expected number of arguments of the runtime function is
+ // constant, we check that the actual number of arguments match the
+ // expectation.
+ CHECK(f->nargs < 0 || f->nargs == num_arguments);
+
+ // TODO(1236192): Most runtime routines don't need the number of
+ // arguments passed in because it is constant. At some point we
+ // should remove this need and make the runtime routine entry code
+ // smarter.
+ mov(r0, Operand(num_arguments));
+ Move(r1, ExternalReference::Create(f));
+ Handle<Code> code =
+ CodeFactory::CEntry(isolate(), f->result_size, save_doubles);
+ Call(code, RelocInfo::CODE_TARGET);
+}
+
+void MacroAssembler::TailCallRuntime(Runtime::FunctionId fid) {
+ const Runtime::Function* function = Runtime::FunctionForId(fid);
+ DCHECK_EQ(1, function->result_size);
+ if (function->nargs >= 0) {
+ // TODO(1236192): Most runtime routines don't need the number of
+ // arguments passed in because it is constant. At some point we
+ // should remove this need and make the runtime routine entry code
+ // smarter.
+ mov(r0, Operand(function->nargs));
+ }
+ JumpToExternalReference(ExternalReference::Create(fid));
+}
+
+void MacroAssembler::JumpToExternalReference(const ExternalReference& builtin,
+ bool builtin_exit_frame) {
+#if defined(__thumb__)
+ // Thumb mode builtin.
+ DCHECK_EQ(builtin.address() & 1, 1);
+#endif
+ Move(r1, builtin);
+ Handle<Code> code = CodeFactory::CEntry(isolate(), 1, kDontSaveFPRegs,
+ kArgvOnStack, builtin_exit_frame);
+ Jump(code, RelocInfo::CODE_TARGET);
+}
+
+void MacroAssembler::JumpToInstructionStream(Address entry) {
+ mov(kOffHeapTrampolineRegister, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+ Jump(kOffHeapTrampolineRegister);
+}
+
+void MacroAssembler::LoadWeakValue(Register out, Register in,
+ Label* target_if_cleared) {
+ cmp(in, Operand(kClearedWeakHeapObjectLower32));
+ b(eq, target_if_cleared);
+
+ and_(out, in, Operand(~kWeakHeapObjectMask));
+}
+
+void MacroAssembler::IncrementCounter(StatsCounter* counter, int value,
+ Register scratch1, Register scratch2) {
+ DCHECK_GT(value, 0);
+ if (FLAG_native_code_counters && counter->Enabled()) {
+ Move(scratch2, ExternalReference::Create(counter));
+ ldr(scratch1, MemOperand(scratch2));
+ add(scratch1, scratch1, Operand(value));
+ str(scratch1, MemOperand(scratch2));
+ }
+}
+
+void MacroAssembler::DecrementCounter(StatsCounter* counter, int value,
+ Register scratch1, Register scratch2) {
+ DCHECK_GT(value, 0);
+ if (FLAG_native_code_counters && counter->Enabled()) {
+ Move(scratch2, ExternalReference::Create(counter));
+ ldr(scratch1, MemOperand(scratch2));
+ sub(scratch1, scratch1, Operand(value));
+ str(scratch1, MemOperand(scratch2));
+ }
+}
+
+void TurboAssembler::Assert(Condition cond, AbortReason reason) {
+ if (emit_debug_code()) Check(cond, reason);
+}
+
+void TurboAssembler::AssertUnreachable(AbortReason reason) {
+ if (emit_debug_code()) Abort(reason);
+}
+
+void TurboAssembler::Check(Condition cond, AbortReason reason) {
+ Label L;
+ b(cond, &L);
+ Abort(reason);
+ // will not return here
+ bind(&L);
+}
+
+void TurboAssembler::Abort(AbortReason reason) {
+ Label abort_start;
+ bind(&abort_start);
+#ifdef DEBUG
+ const char* msg = GetAbortReason(reason);
+ RecordComment("Abort message: ");
+ RecordComment(msg);
+#endif
+
+ // Avoid emitting call to builtin if requested.
+ if (trap_on_abort()) {
+ stop();
+ return;
+ }
+
+ if (should_abort_hard()) {
+ // We don't care if we constructed a frame. Just pretend we did.
+ FrameScope assume_frame(this, StackFrame::NONE);
+ Move32BitImmediate(r0, Operand(static_cast<int>(reason)));
+ PrepareCallCFunction(1, 0, r1);
+ Move(r1, ExternalReference::abort_with_reason());
+ // Use Call directly to avoid any unneeded overhead. The function won't
+ // return anyway.
+ Call(r1);
+ return;
+ }
+
+ Move(r1, Smi::FromInt(static_cast<int>(reason)));
+
+ // Disable stub call restrictions to always allow calls to abort.
+ if (!has_frame()) {
+ // We don't actually want to generate a pile of code for this, so just
+ // claim there is a stack frame, without generating one.
+ FrameScope scope(this, StackFrame::NONE);
+ Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
+ } else {
+ Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
+ }
+ // will not return here
+}
+
+void MacroAssembler::LoadMap(Register destination, Register object) {
+ ldr(destination, FieldMemOperand(object, HeapObject::kMapOffset));
+}
+
+void MacroAssembler::LoadGlobalProxy(Register dst) {
+ LoadNativeContextSlot(Context::GLOBAL_PROXY_INDEX, dst);
+}
+
+void MacroAssembler::LoadNativeContextSlot(int index, Register dst) {
+ LoadMap(dst, cp);
+ ldr(dst, FieldMemOperand(
+ dst, Map::kConstructorOrBackPointerOrNativeContextOffset));
+ ldr(dst, MemOperand(dst, Context::SlotOffset(index)));
+}
+
+void TurboAssembler::InitializeRootRegister() {
+ ExternalReference isolate_root = ExternalReference::isolate_root(isolate());
+ mov(kRootRegister, Operand(isolate_root));
+}
+
+void MacroAssembler::SmiTag(Register reg, SBit s) {
+ add(reg, reg, Operand(reg), s);
+}
+
+void MacroAssembler::SmiTag(Register dst, Register src, SBit s) {
+ add(dst, src, Operand(src), s);
+}
+
+void MacroAssembler::SmiTst(Register value) {
+ tst(value, Operand(kSmiTagMask));
+}
+
+void TurboAssembler::JumpIfSmi(Register value, Label* smi_label) {
+ tst(value, Operand(kSmiTagMask));
+ b(eq, smi_label);
+}
+
+void TurboAssembler::JumpIfEqual(Register x, int32_t y, Label* dest) {
+ cmp(x, Operand(y));
+ b(eq, dest);
+}
+
+void TurboAssembler::JumpIfLessThan(Register x, int32_t y, Label* dest) {
+ cmp(x, Operand(y));
+ b(lt, dest);
+}
+
+void MacroAssembler::JumpIfNotSmi(Register value, Label* not_smi_label) {
+ tst(value, Operand(kSmiTagMask));
+ b(ne, not_smi_label);
+}
+
+void MacroAssembler::AssertNotSmi(Register object) {
+ if (emit_debug_code()) {
+ STATIC_ASSERT(kSmiTag == 0);
+ tst(object, Operand(kSmiTagMask));
+ Check(ne, AbortReason::kOperandIsASmi);
+ }
+}
+
+void MacroAssembler::AssertSmi(Register object) {
+ if (emit_debug_code()) {
+ STATIC_ASSERT(kSmiTag == 0);
+ tst(object, Operand(kSmiTagMask));
+ Check(eq, AbortReason::kOperandIsNotASmi);
+ }
+}
+
+void MacroAssembler::AssertConstructor(Register object) {
+ if (emit_debug_code()) {
+ STATIC_ASSERT(kSmiTag == 0);
+ tst(object, Operand(kSmiTagMask));
+ Check(ne, AbortReason::kOperandIsASmiAndNotAConstructor);
+ push(object);
+ LoadMap(object, object);
+ ldrb(object, FieldMemOperand(object, Map::kBitFieldOffset));
+ tst(object, Operand(Map::Bits1::IsConstructorBit::kMask));
+ pop(object);
+ Check(ne, AbortReason::kOperandIsNotAConstructor);
+ }
+}
+
+void MacroAssembler::AssertFunction(Register object) {
+ if (emit_debug_code()) {
+ STATIC_ASSERT(kSmiTag == 0);
+ tst(object, Operand(kSmiTagMask));
+ Check(ne, AbortReason::kOperandIsASmiAndNotAFunction);
+ push(object);
+ CompareObjectType(object, object, object, JS_FUNCTION_TYPE);
+ pop(object);
+ Check(eq, AbortReason::kOperandIsNotAFunction);
+ }
+}
+
+void MacroAssembler::AssertBoundFunction(Register object) {
+ if (emit_debug_code()) {
+ STATIC_ASSERT(kSmiTag == 0);
+ tst(object, Operand(kSmiTagMask));
+ Check(ne, AbortReason::kOperandIsASmiAndNotABoundFunction);
+ push(object);
+ CompareObjectType(object, object, object, JS_BOUND_FUNCTION_TYPE);
+ pop(object);
+ Check(eq, AbortReason::kOperandIsNotABoundFunction);
+ }
+}
+
+void MacroAssembler::AssertGeneratorObject(Register object) {
+ if (!emit_debug_code()) return;
+ tst(object, Operand(kSmiTagMask));
+ Check(ne, AbortReason::kOperandIsASmiAndNotAGeneratorObject);
+
+ // Load map
+ Register map = object;
+ push(object);
+ LoadMap(map, object);
+
+ // Check if JSGeneratorObject
+ Label do_check;
+ Register instance_type = object;
+ CompareInstanceType(map, instance_type, JS_GENERATOR_OBJECT_TYPE);
+ b(eq, &do_check);
+
+ // Check if JSAsyncFunctionObject (See MacroAssembler::CompareInstanceType)
+ cmp(instance_type, Operand(JS_ASYNC_FUNCTION_OBJECT_TYPE));
+ b(eq, &do_check);
+
+ // Check if JSAsyncGeneratorObject (See MacroAssembler::CompareInstanceType)
+ cmp(instance_type, Operand(JS_ASYNC_GENERATOR_OBJECT_TYPE));
+
+ bind(&do_check);
+ // Restore generator object to register and perform assertion
+ pop(object);
+ Check(eq, AbortReason::kOperandIsNotAGeneratorObject);
+}
+
+void MacroAssembler::AssertUndefinedOrAllocationSite(Register object,
+ Register scratch) {
+ if (emit_debug_code()) {
+ Label done_checking;
+ AssertNotSmi(object);
+ CompareRoot(object, RootIndex::kUndefinedValue);
+ b(eq, &done_checking);
+ LoadMap(scratch, object);
+ CompareInstanceType(scratch, scratch, ALLOCATION_SITE_TYPE);
+ Assert(eq, AbortReason::kExpectedUndefinedOrCell);
+ bind(&done_checking);
+ }
+}
+
+void TurboAssembler::CheckFor32DRegs(Register scratch) {
+ Move(scratch, ExternalReference::cpu_features());
+ ldr(scratch, MemOperand(scratch));
+ tst(scratch, Operand(1u << VFP32DREGS));
+}
+
+void TurboAssembler::SaveFPRegs(Register location, Register scratch) {
+ CpuFeatureScope scope(this, VFP32DREGS, CpuFeatureScope::kDontCheckSupported);
+ CheckFor32DRegs(scratch);
+ vstm(db_w, location, d16, d31, ne);
+ sub(location, location, Operand(16 * kDoubleSize), LeaveCC, eq);
+ vstm(db_w, location, d0, d15);
+}
+
+void TurboAssembler::RestoreFPRegs(Register location, Register scratch) {
+ CpuFeatureScope scope(this, VFP32DREGS, CpuFeatureScope::kDontCheckSupported);
+ CheckFor32DRegs(scratch);
+ vldm(ia_w, location, d0, d15);
+ vldm(ia_w, location, d16, d31, ne);
+ add(location, location, Operand(16 * kDoubleSize), LeaveCC, eq);
+}
+
+void TurboAssembler::SaveFPRegsToHeap(Register location, Register scratch) {
+ CpuFeatureScope scope(this, VFP32DREGS, CpuFeatureScope::kDontCheckSupported);
+ CheckFor32DRegs(scratch);
+ vstm(ia_w, location, d0, d15);
+ vstm(ia_w, location, d16, d31, ne);
+ add(location, location, Operand(16 * kDoubleSize), LeaveCC, eq);
+}
+
+void TurboAssembler::RestoreFPRegsFromHeap(Register location,
+ Register scratch) {
+ CpuFeatureScope scope(this, VFP32DREGS, CpuFeatureScope::kDontCheckSupported);
+ CheckFor32DRegs(scratch);
+ vldm(ia_w, location, d0, d15);
+ vldm(ia_w, location, d16, d31, ne);
+ add(location, location, Operand(16 * kDoubleSize), LeaveCC, eq);
+}
+
+template <typename T>
+void TurboAssembler::FloatMaxHelper(T result, T left, T right,
+ Label* out_of_line) {
+ // This trivial case is caught sooner, so that the out-of-line code can be
+ // completely avoided.
+ DCHECK(left != right);
+
+ if (CpuFeatures::IsSupported(ARMv8)) {
+ CpuFeatureScope scope(this, ARMv8);
+ VFPCompareAndSetFlags(left, right);
+ b(vs, out_of_line);
+ vmaxnm(result, left, right);
+ } else {
+ Label done;
+ VFPCompareAndSetFlags(left, right);
+ b(vs, out_of_line);
+ // Avoid a conditional instruction if the result register is unique.
+ bool aliased_result_reg = result == left || result == right;
+ Move(result, right, aliased_result_reg ? mi : al);
+ Move(result, left, gt);
+ b(ne, &done);
+ // Left and right are equal, but check for +/-0.
+ VFPCompareAndSetFlags(left, 0.0);
+ b(eq, out_of_line);
+ // The arguments are equal and not zero, so it doesn't matter which input we
+ // pick. We have already moved one input into the result (if it didn't
+ // already alias) so there's nothing more to do.
+ bind(&done);
+ }
+}
+
+template <typename T>
+void TurboAssembler::FloatMaxOutOfLineHelper(T result, T left, T right) {
+ DCHECK(left != right);
+
+ // ARMv8: At least one of left and right is a NaN.
+ // Anything else: At least one of left and right is a NaN, or both left and
+ // right are zeroes with unknown sign.
+
+ // If left and right are +/-0, select the one with the most positive sign.
+ // If left or right are NaN, vadd propagates the appropriate one.
+ vadd(result, left, right);
+}
+
+template <typename T>
+void TurboAssembler::FloatMinHelper(T result, T left, T right,
+ Label* out_of_line) {
+ // This trivial case is caught sooner, so that the out-of-line code can be
+ // completely avoided.
+ DCHECK(left != right);
+
+ if (CpuFeatures::IsSupported(ARMv8)) {
+ CpuFeatureScope scope(this, ARMv8);
+ VFPCompareAndSetFlags(left, right);
+ b(vs, out_of_line);
+ vminnm(result, left, right);
+ } else {
+ Label done;
+ VFPCompareAndSetFlags(left, right);
+ b(vs, out_of_line);
+ // Avoid a conditional instruction if the result register is unique.
+ bool aliased_result_reg = result == left || result == right;
+ Move(result, left, aliased_result_reg ? mi : al);
+ Move(result, right, gt);
+ b(ne, &done);
+ // Left and right are equal, but check for +/-0.
+ VFPCompareAndSetFlags(left, 0.0);
+ // If the arguments are equal and not zero, it doesn't matter which input we
+ // pick. We have already moved one input into the result (if it didn't
+ // already alias) so there's nothing more to do.
+ b(ne, &done);
+ // At this point, both left and right are either 0 or -0.
+ // We could use a single 'vorr' instruction here if we had NEON support.
+ // The algorithm used is -((-L) + (-R)), which is most efficiently expressed
+ // as -((-L) - R).
+ if (left == result) {
+ DCHECK(right != result);
+ vneg(result, left);
+ vsub(result, result, right);
+ vneg(result, result);
+ } else {
+ DCHECK(left != result);
+ vneg(result, right);
+ vsub(result, result, left);
+ vneg(result, result);
+ }
+ bind(&done);
+ }
+}
+
+template <typename T>
+void TurboAssembler::FloatMinOutOfLineHelper(T result, T left, T right) {
+ DCHECK(left != right);
+
+ // At least one of left and right is a NaN. Use vadd to propagate the NaN
+ // appropriately. +/-0 is handled inline.
+ vadd(result, left, right);
+}
+
+void TurboAssembler::FloatMax(SwVfpRegister result, SwVfpRegister left,
+ SwVfpRegister right, Label* out_of_line) {
+ FloatMaxHelper(result, left, right, out_of_line);
+}
+
+void TurboAssembler::FloatMin(SwVfpRegister result, SwVfpRegister left,
+ SwVfpRegister right, Label* out_of_line) {
+ FloatMinHelper(result, left, right, out_of_line);
+}
+
+void TurboAssembler::FloatMax(DwVfpRegister result, DwVfpRegister left,
+ DwVfpRegister right, Label* out_of_line) {
+ FloatMaxHelper(result, left, right, out_of_line);
+}
+
+void TurboAssembler::FloatMin(DwVfpRegister result, DwVfpRegister left,
+ DwVfpRegister right, Label* out_of_line) {
+ FloatMinHelper(result, left, right, out_of_line);
+}
+
+void TurboAssembler::FloatMaxOutOfLine(SwVfpRegister result, SwVfpRegister left,
+ SwVfpRegister right) {
+ FloatMaxOutOfLineHelper(result, left, right);
+}
+
+void TurboAssembler::FloatMinOutOfLine(SwVfpRegister result, SwVfpRegister left,
+ SwVfpRegister right) {
+ FloatMinOutOfLineHelper(result, left, right);
+}
+
+void TurboAssembler::FloatMaxOutOfLine(DwVfpRegister result, DwVfpRegister left,
+ DwVfpRegister right) {
+ FloatMaxOutOfLineHelper(result, left, right);
+}
+
+void TurboAssembler::FloatMinOutOfLine(DwVfpRegister result, DwVfpRegister left,
+ DwVfpRegister right) {
+ FloatMinOutOfLineHelper(result, left, right);
+}
+
+static const int kRegisterPassedArguments = 4;
+// The hardfloat calling convention passes double arguments in registers d0-d7.
+static const int kDoubleRegisterPassedArguments = 8;
+
+int TurboAssembler::CalculateStackPassedWords(int num_reg_arguments,
+ int num_double_arguments) {
+ int stack_passed_words = 0;
+ if (use_eabi_hardfloat()) {
+ // In the hard floating point calling convention, we can use the first 8
+ // registers to pass doubles.
+ if (num_double_arguments > kDoubleRegisterPassedArguments) {
+ stack_passed_words +=
+ 2 * (num_double_arguments - kDoubleRegisterPassedArguments);
+ }
+ } else {
+ // In the soft floating point calling convention, every double
+ // argument is passed using two registers.
+ num_reg_arguments += 2 * num_double_arguments;
+ }
+ // Up to four simple arguments are passed in registers r0..r3.
+ if (num_reg_arguments > kRegisterPassedArguments) {
+ stack_passed_words += num_reg_arguments - kRegisterPassedArguments;
+ }
+ return stack_passed_words;
+}
+
+void TurboAssembler::PrepareCallCFunction(int num_reg_arguments,
+ int num_double_arguments,
+ Register scratch) {
+ int frame_alignment = ActivationFrameAlignment();
+ int stack_passed_arguments =
+ CalculateStackPassedWords(num_reg_arguments, num_double_arguments);
+ if (frame_alignment > kPointerSize) {
+ UseScratchRegisterScope temps(this);
+ if (!scratch.is_valid()) scratch = temps.Acquire();
+ // Make stack end at alignment and make room for num_arguments - 4 words
+ // and the original value of sp.
+ mov(scratch, sp);
+ AllocateStackSpace((stack_passed_arguments + 1) * kPointerSize);
+ DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
+ and_(sp, sp, Operand(-frame_alignment));
+ str(scratch, MemOperand(sp, stack_passed_arguments * kPointerSize));
+ } else if (stack_passed_arguments > 0) {
+ AllocateStackSpace(stack_passed_arguments * kPointerSize);
+ }
+}
+
+void TurboAssembler::MovToFloatParameter(DwVfpRegister src) {
+ DCHECK(src == d0);
+ if (!use_eabi_hardfloat()) {
+ vmov(r0, r1, src);
+ }
+}
+
+// On ARM this is just a synonym to make the purpose clear.
+void TurboAssembler::MovToFloatResult(DwVfpRegister src) {
+ MovToFloatParameter(src);
+}
+
+void TurboAssembler::MovToFloatParameters(DwVfpRegister src1,
+ DwVfpRegister src2) {
+ DCHECK(src1 == d0);
+ DCHECK(src2 == d1);
+ if (!use_eabi_hardfloat()) {
+ vmov(r0, r1, src1);
+ vmov(r2, r3, src2);
+ }
+}
+
+void TurboAssembler::CallCFunction(ExternalReference function,
+ int num_reg_arguments,
+ int num_double_arguments) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ Move(scratch, function);
+ CallCFunctionHelper(scratch, num_reg_arguments, num_double_arguments);
+}
+
+void TurboAssembler::CallCFunction(Register function, int num_reg_arguments,
+ int num_double_arguments) {
+ CallCFunctionHelper(function, num_reg_arguments, num_double_arguments);
+}
+
+void TurboAssembler::CallCFunction(ExternalReference function,
+ int num_arguments) {
+ CallCFunction(function, num_arguments, 0);
+}
+
+void TurboAssembler::CallCFunction(Register function, int num_arguments) {
+ CallCFunction(function, num_arguments, 0);
+}
+
+void TurboAssembler::CallCFunctionHelper(Register function,
+ int num_reg_arguments,
+ int num_double_arguments) {
+ DCHECK_LE(num_reg_arguments + num_double_arguments, kMaxCParameters);
+ DCHECK(has_frame());
+ // Make sure that the stack is aligned before calling a C function unless
+ // running in the simulator. The simulator has its own alignment check which
+ // provides more information.
+#if V8_HOST_ARCH_ARM
+ if (emit_debug_code()) {
+ int frame_alignment = base::OS::ActivationFrameAlignment();
+ int frame_alignment_mask = frame_alignment - 1;
+ if (frame_alignment > kPointerSize) {
+ DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
+ Label alignment_as_expected;
+ tst(sp, Operand(frame_alignment_mask));
+ b(eq, &alignment_as_expected);
+ // Don't use Check here, as it will call Runtime_Abort possibly
+ // re-entering here.
+ stop();
+ bind(&alignment_as_expected);
+ }
+ }
+#endif
+
+ // Save the frame pointer and PC so that the stack layout remains iterable,
+ // even without an ExitFrame which normally exists between JS and C frames.
+ Register addr_scratch = r4;
+ // See x64 code for reasoning about how to address the isolate data fields.
+ if (root_array_available()) {
+ str(pc,
+ MemOperand(kRootRegister, IsolateData::fast_c_call_caller_pc_offset()));
+ str(fp,
+ MemOperand(kRootRegister, IsolateData::fast_c_call_caller_fp_offset()));
+ } else {
+ DCHECK_NOT_NULL(isolate());
+ Push(addr_scratch);
+
+ Move(addr_scratch,
+ ExternalReference::fast_c_call_caller_pc_address(isolate()));
+ str(pc, MemOperand(addr_scratch));
+ Move(addr_scratch,
+ ExternalReference::fast_c_call_caller_fp_address(isolate()));
+ str(fp, MemOperand(addr_scratch));
+
+ Pop(addr_scratch);
+ }
+
+ // Just call directly. The function called cannot cause a GC, or
+ // allow preemption, so the return address in the link register
+ // stays correct.
+ Call(function);
+
+ // We don't unset the PC; the FP is the source of truth.
+ Register zero_scratch = r5;
+ Push(zero_scratch);
+ mov(zero_scratch, Operand::Zero());
+
+ if (root_array_available()) {
+ str(zero_scratch,
+ MemOperand(kRootRegister, IsolateData::fast_c_call_caller_fp_offset()));
+ } else {
+ DCHECK_NOT_NULL(isolate());
+ Push(addr_scratch);
+ Move(addr_scratch,
+ ExternalReference::fast_c_call_caller_fp_address(isolate()));
+ str(zero_scratch, MemOperand(addr_scratch));
+ Pop(addr_scratch);
+ }
+
+ Pop(zero_scratch);
+
+ int stack_passed_arguments =
+ CalculateStackPassedWords(num_reg_arguments, num_double_arguments);
+ if (ActivationFrameAlignment() > kPointerSize) {
+ ldr(sp, MemOperand(sp, stack_passed_arguments * kPointerSize));
+ } else {
+ add(sp, sp, Operand(stack_passed_arguments * kPointerSize));
+ }
+}
+
+void TurboAssembler::CheckPageFlag(Register object, int mask, Condition cc,
+ Label* condition_met) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(cc == eq || cc == ne);
+ Bfc(scratch, object, 0, kPageSizeBits);
+ ldr(scratch, MemOperand(scratch, BasicMemoryChunk::kFlagsOffset));
+ tst(scratch, Operand(mask));
+ b(cc, condition_met);
+}
+
+Register GetRegisterThatIsNotOneOf(Register reg1, Register reg2, Register reg3,
+ Register reg4, Register reg5,
+ Register reg6) {
+ RegList regs = 0;
+ if (reg1.is_valid()) regs |= reg1.bit();
+ if (reg2.is_valid()) regs |= reg2.bit();
+ if (reg3.is_valid()) regs |= reg3.bit();
+ if (reg4.is_valid()) regs |= reg4.bit();
+ if (reg5.is_valid()) regs |= reg5.bit();
+ if (reg6.is_valid()) regs |= reg6.bit();
+
+ const RegisterConfiguration* config = RegisterConfiguration::Default();
+ for (int i = 0; i < config->num_allocatable_general_registers(); ++i) {
+ int code = config->GetAllocatableGeneralCode(i);
+ Register candidate = Register::from_code(code);
+ if (regs & candidate.bit()) continue;
+ return candidate;
+ }
+ UNREACHABLE();
+}
+
+void TurboAssembler::ComputeCodeStartAddress(Register dst) {
+ // We can use the register pc - 8 for the address of the current instruction.
+ sub(dst, pc, Operand(pc_offset() + Instruction::kPcLoadDelta));
+}
+
+void TurboAssembler::ResetSpeculationPoisonRegister() {
+ mov(kSpeculationPoisonRegister, Operand(-1));
+}
+
+void TurboAssembler::CallForDeoptimization(Builtins::Name target, int,
+ Label* exit, DeoptimizeKind kind,
+ Label*) {
+ BlockConstPoolScope block_const_pool(this);
+ ldr(ip, MemOperand(kRootRegister,
+ IsolateData::builtin_entry_slot_offset(target)));
+ Call(ip);
+ DCHECK_EQ(SizeOfCodeGeneratedSince(exit),
+ (kind == DeoptimizeKind::kLazy)
+ ? Deoptimizer::kLazyDeoptExitSize
+ : Deoptimizer::kNonLazyDeoptExitSize);
+ USE(exit, kind);
+}
+
+void TurboAssembler::Trap() { stop(); }
+void TurboAssembler::DebugBreak() { stop(); }
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_ARM
diff --git a/src/codegen/arm/macro-assembler-arm.h b/src/codegen/arm/macro-assembler-arm.h
new file mode 100644
index 0000000..a4d6632
--- /dev/null
+++ b/src/codegen/arm/macro-assembler-arm.h
@@ -0,0 +1,851 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef INCLUDED_FROM_MACRO_ASSEMBLER_H
+#error This header must be included via macro-assembler.h
+#endif
+
+#ifndef V8_CODEGEN_ARM_MACRO_ASSEMBLER_ARM_H_
+#define V8_CODEGEN_ARM_MACRO_ASSEMBLER_ARM_H_
+
+#include "src/codegen/arm/assembler-arm.h"
+#include "src/codegen/bailout-reason.h"
+#include "src/common/globals.h"
+#include "src/objects/contexts.h"
+
+namespace v8 {
+namespace internal {
+
+// TODO(victorgomes): Move definition to macro-assembler.h, once all other
+// platforms are updated.
+enum class StackLimitKind { kInterruptStackLimit, kRealStackLimit };
+
+// ----------------------------------------------------------------------------
+// Static helper functions
+
+// Generate a MemOperand for loading a field from an object.
+inline MemOperand FieldMemOperand(Register object, int offset) {
+ return MemOperand(object, offset - kHeapObjectTag);
+}
+
+enum RememberedSetAction { EMIT_REMEMBERED_SET, OMIT_REMEMBERED_SET };
+enum SmiCheck { INLINE_SMI_CHECK, OMIT_SMI_CHECK };
+enum LinkRegisterStatus { kLRHasNotBeenSaved, kLRHasBeenSaved };
+
+Register GetRegisterThatIsNotOneOf(Register reg1, Register reg2 = no_reg,
+ Register reg3 = no_reg,
+ Register reg4 = no_reg,
+ Register reg5 = no_reg,
+ Register reg6 = no_reg);
+
+enum TargetAddressStorageMode {
+ CAN_INLINE_TARGET_ADDRESS,
+ NEVER_INLINE_TARGET_ADDRESS
+};
+
+class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase {
+ public:
+ using TurboAssemblerBase::TurboAssemblerBase;
+
+ // Activation support.
+ void EnterFrame(StackFrame::Type type,
+ bool load_constant_pool_pointer_reg = false);
+ // Returns the pc offset at which the frame ends.
+ int LeaveFrame(StackFrame::Type type);
+
+// Allocate stack space of given size (i.e. decrement {sp} by the value
+// stored in the given register, or by a constant). If you need to perform a
+// stack check, do it before calling this function because this function may
+// write into the newly allocated space. It may also overwrite the given
+// register's value, in the version that takes a register.
+#ifdef V8_OS_WIN
+ void AllocateStackSpace(Register bytes_scratch);
+ void AllocateStackSpace(int bytes);
+#else
+ void AllocateStackSpace(Register bytes) { sub(sp, sp, bytes); }
+ void AllocateStackSpace(int bytes) { sub(sp, sp, Operand(bytes)); }
+#endif
+
+ // Push a fixed frame, consisting of lr, fp
+ void PushCommonFrame(Register marker_reg = no_reg);
+
+ // Generates function and stub prologue code.
+ void StubPrologue(StackFrame::Type type);
+ void Prologue();
+
+ // Push a standard frame, consisting of lr, fp, context and JS function
+ void PushStandardFrame(Register function_reg);
+
+ void InitializeRootRegister();
+
+ void Push(Register src) { push(src); }
+
+ void Push(Handle<HeapObject> handle);
+ void Push(Smi smi);
+
+ // Push two registers. Pushes leftmost register first (to highest address).
+ void Push(Register src1, Register src2, Condition cond = al) {
+ if (src1.code() > src2.code()) {
+ stm(db_w, sp, src1.bit() | src2.bit(), cond);
+ } else {
+ str(src1, MemOperand(sp, 4, NegPreIndex), cond);
+ str(src2, MemOperand(sp, 4, NegPreIndex), cond);
+ }
+ }
+
+ // Push three registers. Pushes leftmost register first (to highest address).
+ void Push(Register src1, Register src2, Register src3, Condition cond = al) {
+ if (src1.code() > src2.code()) {
+ if (src2.code() > src3.code()) {
+ stm(db_w, sp, src1.bit() | src2.bit() | src3.bit(), cond);
+ } else {
+ stm(db_w, sp, src1.bit() | src2.bit(), cond);
+ str(src3, MemOperand(sp, 4, NegPreIndex), cond);
+ }
+ } else {
+ str(src1, MemOperand(sp, 4, NegPreIndex), cond);
+ Push(src2, src3, cond);
+ }
+ }
+
+ // Push four registers. Pushes leftmost register first (to highest address).
+ void Push(Register src1, Register src2, Register src3, Register src4,
+ Condition cond = al) {
+ if (src1.code() > src2.code()) {
+ if (src2.code() > src3.code()) {
+ if (src3.code() > src4.code()) {
+ stm(db_w, sp, src1.bit() | src2.bit() | src3.bit() | src4.bit(),
+ cond);
+ } else {
+ stm(db_w, sp, src1.bit() | src2.bit() | src3.bit(), cond);
+ str(src4, MemOperand(sp, 4, NegPreIndex), cond);
+ }
+ } else {
+ stm(db_w, sp, src1.bit() | src2.bit(), cond);
+ Push(src3, src4, cond);
+ }
+ } else {
+ str(src1, MemOperand(sp, 4, NegPreIndex), cond);
+ Push(src2, src3, src4, cond);
+ }
+ }
+
+ // Push five registers. Pushes leftmost register first (to highest address).
+ void Push(Register src1, Register src2, Register src3, Register src4,
+ Register src5, Condition cond = al) {
+ if (src1.code() > src2.code()) {
+ if (src2.code() > src3.code()) {
+ if (src3.code() > src4.code()) {
+ if (src4.code() > src5.code()) {
+ stm(db_w, sp,
+ src1.bit() | src2.bit() | src3.bit() | src4.bit() | src5.bit(),
+ cond);
+ } else {
+ stm(db_w, sp, src1.bit() | src2.bit() | src3.bit() | src4.bit(),
+ cond);
+ str(src5, MemOperand(sp, 4, NegPreIndex), cond);
+ }
+ } else {
+ stm(db_w, sp, src1.bit() | src2.bit() | src3.bit(), cond);
+ Push(src4, src5, cond);
+ }
+ } else {
+ stm(db_w, sp, src1.bit() | src2.bit(), cond);
+ Push(src3, src4, src5, cond);
+ }
+ } else {
+ str(src1, MemOperand(sp, 4, NegPreIndex), cond);
+ Push(src2, src3, src4, src5, cond);
+ }
+ }
+
+ enum class PushArrayOrder { kNormal, kReverse };
+ // `array` points to the first element (the lowest address).
+ // `array` and `size` are not modified.
+ void PushArray(Register array, Register size, Register scratch,
+ PushArrayOrder order = PushArrayOrder::kNormal);
+
+ void Pop(Register dst) { pop(dst); }
+
+ // Pop two registers. Pops rightmost register first (from lower address).
+ void Pop(Register src1, Register src2, Condition cond = al) {
+ DCHECK(src1 != src2);
+ if (src1.code() > src2.code()) {
+ ldm(ia_w, sp, src1.bit() | src2.bit(), cond);
+ } else {
+ ldr(src2, MemOperand(sp, 4, PostIndex), cond);
+ ldr(src1, MemOperand(sp, 4, PostIndex), cond);
+ }
+ }
+
+ // Pop three registers. Pops rightmost register first (from lower address).
+ void Pop(Register src1, Register src2, Register src3, Condition cond = al) {
+ DCHECK(!AreAliased(src1, src2, src3));
+ if (src1.code() > src2.code()) {
+ if (src2.code() > src3.code()) {
+ ldm(ia_w, sp, src1.bit() | src2.bit() | src3.bit(), cond);
+ } else {
+ ldr(src3, MemOperand(sp, 4, PostIndex), cond);
+ ldm(ia_w, sp, src1.bit() | src2.bit(), cond);
+ }
+ } else {
+ Pop(src2, src3, cond);
+ ldr(src1, MemOperand(sp, 4, PostIndex), cond);
+ }
+ }
+
+ // Pop four registers. Pops rightmost register first (from lower address).
+ void Pop(Register src1, Register src2, Register src3, Register src4,
+ Condition cond = al) {
+ DCHECK(!AreAliased(src1, src2, src3, src4));
+ if (src1.code() > src2.code()) {
+ if (src2.code() > src3.code()) {
+ if (src3.code() > src4.code()) {
+ ldm(ia_w, sp, src1.bit() | src2.bit() | src3.bit() | src4.bit(),
+ cond);
+ } else {
+ ldr(src4, MemOperand(sp, 4, PostIndex), cond);
+ ldm(ia_w, sp, src1.bit() | src2.bit() | src3.bit(), cond);
+ }
+ } else {
+ Pop(src3, src4, cond);
+ ldm(ia_w, sp, src1.bit() | src2.bit(), cond);
+ }
+ } else {
+ Pop(src2, src3, src4, cond);
+ ldr(src1, MemOperand(sp, 4, PostIndex), cond);
+ }
+ }
+
+ // Before calling a C-function from generated code, align arguments on stack.
+ // After aligning the frame, non-register arguments must be stored in
+ // sp[0], sp[4], etc., not pushed. The argument count assumes all arguments
+ // are word sized. If double arguments are used, this function assumes that
+ // all double arguments are stored before core registers; otherwise the
+ // correct alignment of the double values is not guaranteed.
+ // Some compilers/platforms require the stack to be aligned when calling
+ // C++ code.
+ // Needs a scratch register to do some arithmetic. This register will be
+ // trashed.
+ void PrepareCallCFunction(int num_reg_arguments, int num_double_registers = 0,
+ Register scratch = no_reg);
+
+ // Removes current frame and its arguments from the stack preserving
+ // the arguments and a return address pushed to the stack for the next call.
+ // Both |callee_args_count| and |caller_args_count| do not include
+ // receiver. |callee_args_count| is not modified. |caller_args_count|
+ // is trashed.
+ void PrepareForTailCall(Register callee_args_count,
+ Register caller_args_count, Register scratch0,
+ Register scratch1);
+
+ // There are two ways of passing double arguments on ARM, depending on
+ // whether soft or hard floating point ABI is used. These functions
+ // abstract parameter passing for the three different ways we call
+ // C functions from generated code.
+ void MovToFloatParameter(DwVfpRegister src);
+ void MovToFloatParameters(DwVfpRegister src1, DwVfpRegister src2);
+ void MovToFloatResult(DwVfpRegister src);
+
+ // Calls a C function and cleans up the space for arguments allocated
+ // by PrepareCallCFunction. The called function is not allowed to trigger a
+ // garbage collection, since that might move the code and invalidate the
+ // return address (unless this is somehow accounted for by the called
+ // function).
+ void CallCFunction(ExternalReference function, int num_arguments);
+ void CallCFunction(Register function, int num_arguments);
+ void CallCFunction(ExternalReference function, int num_reg_arguments,
+ int num_double_arguments);
+ void CallCFunction(Register function, int num_reg_arguments,
+ int num_double_arguments);
+
+ void MovFromFloatParameter(DwVfpRegister dst);
+ void MovFromFloatResult(DwVfpRegister dst);
+
+ void Trap() override;
+ void DebugBreak() override;
+
+ // Calls Abort(msg) if the condition cond is not satisfied.
+ // Use --debug-code to enable.
+ void Assert(Condition cond, AbortReason reason);
+
+ // Like Assert(), but without condition.
+ // Use --debug-code to enable.
+ void AssertUnreachable(AbortReason reason);
+
+ // Like Assert(), but always enabled.
+ void Check(Condition cond, AbortReason reason);
+
+ // Print a message to stdout and abort execution.
+ void Abort(AbortReason msg);
+
+ void LslPair(Register dst_low, Register dst_high, Register src_low,
+ Register src_high, Register shift);
+ void LslPair(Register dst_low, Register dst_high, Register src_low,
+ Register src_high, uint32_t shift);
+ void LsrPair(Register dst_low, Register dst_high, Register src_low,
+ Register src_high, Register shift);
+ void LsrPair(Register dst_low, Register dst_high, Register src_low,
+ Register src_high, uint32_t shift);
+ void AsrPair(Register dst_low, Register dst_high, Register src_low,
+ Register src_high, Register shift);
+ void AsrPair(Register dst_low, Register dst_high, Register src_low,
+ Register src_high, uint32_t shift);
+
+ void LoadFromConstantsTable(Register destination,
+ int constant_index) override;
+ void LoadRootRegisterOffset(Register destination, intptr_t offset) override;
+ void LoadRootRelative(Register destination, int32_t offset) override;
+
+ // Jump, Call, and Ret pseudo instructions implementing inter-working.
+ void Call(Register target, Condition cond = al);
+ void Call(Address target, RelocInfo::Mode rmode, Condition cond = al,
+ TargetAddressStorageMode mode = CAN_INLINE_TARGET_ADDRESS,
+ bool check_constant_pool = true);
+ void Call(Handle<Code> code, RelocInfo::Mode rmode = RelocInfo::CODE_TARGET,
+ Condition cond = al,
+ TargetAddressStorageMode mode = CAN_INLINE_TARGET_ADDRESS,
+ bool check_constant_pool = true);
+ void Call(Label* target);
+
+ // Load the builtin given by the Smi in |builtin_index| into the same
+ // register.
+ void LoadEntryFromBuiltinIndex(Register builtin_index);
+ void CallBuiltinByIndex(Register builtin_index) override;
+ void CallBuiltin(int builtin_index, Condition cond = al);
+
+ void LoadCodeObjectEntry(Register destination, Register code_object) override;
+ void CallCodeObject(Register code_object) override;
+ void JumpCodeObject(Register code_object) override;
+
+ // Generates an instruction sequence s.t. the return address points to the
+ // instruction following the call.
+ // The return address on the stack is used by frame iteration.
+ void StoreReturnAddressAndCall(Register target);
+
+ void CallForDeoptimization(Builtins::Name target, int deopt_id, Label* exit,
+ DeoptimizeKind kind,
+ Label* jump_deoptimization_entry_label);
+
+ // Emit code to discard a non-negative number of pointer-sized elements
+ // from the stack, clobbering only the sp register.
+ void Drop(int count, Condition cond = al);
+ void Drop(Register count, Condition cond = al);
+
+ void Ret(Condition cond = al);
+ void Ret(int drop, Condition cond = al);
+
+ // Compare single values and move the result to the normal condition flags.
+ void VFPCompareAndSetFlags(const SwVfpRegister src1, const SwVfpRegister src2,
+ const Condition cond = al);
+ void VFPCompareAndSetFlags(const SwVfpRegister src1, const float src2,
+ const Condition cond = al);
+
+ // Compare double values and move the result to the normal condition flags.
+ void VFPCompareAndSetFlags(const DwVfpRegister src1, const DwVfpRegister src2,
+ const Condition cond = al);
+ void VFPCompareAndSetFlags(const DwVfpRegister src1, const double src2,
+ const Condition cond = al);
+
+ // If the value is a NaN, canonicalize the value else, do nothing.
+ void VFPCanonicalizeNaN(const DwVfpRegister dst, const DwVfpRegister src,
+ const Condition cond = al);
+ void VFPCanonicalizeNaN(const DwVfpRegister value,
+ const Condition cond = al) {
+ VFPCanonicalizeNaN(value, value, cond);
+ }
+
+ void VmovHigh(Register dst, DwVfpRegister src);
+ void VmovHigh(DwVfpRegister dst, Register src);
+ void VmovLow(Register dst, DwVfpRegister src);
+ void VmovLow(DwVfpRegister dst, Register src);
+
+ void CheckPageFlag(Register object, int mask, Condition cc,
+ Label* condition_met);
+
+ // Check whether d16-d31 are available on the CPU. The result is given by the
+ // Z condition flag: Z==0 if d16-d31 available, Z==1 otherwise.
+ void CheckFor32DRegs(Register scratch);
+
+ void SaveRegisters(RegList registers);
+ void RestoreRegisters(RegList registers);
+
+ void CallRecordWriteStub(Register object, Operand offset,
+ RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode);
+ void CallRecordWriteStub(Register object, Operand offset,
+ RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode, Address wasm_target);
+ void CallEphemeronKeyBarrier(Register object, Operand offset,
+ SaveFPRegsMode fp_mode);
+
+ // For a given |object| and |offset|:
+ // - Move |object| to |dst_object|.
+ // - Compute the address of the slot pointed to by |offset| in |object| and
+ // write it to |dst_slot|. |offset| can be either an immediate or a
+ // register.
+ // This method makes sure |object| and |offset| are allowed to overlap with
+ // the destination registers.
+ void MoveObjectAndSlot(Register dst_object, Register dst_slot,
+ Register object, Operand offset);
+
+ // Does a runtime check for 16/32 FP registers. Either way, pushes 32 double
+ // values to location, saving [d0..(d15|d31)].
+ void SaveFPRegs(Register location, Register scratch);
+
+ // Does a runtime check for 16/32 FP registers. Either way, pops 32 double
+ // values to location, restoring [d0..(d15|d31)].
+ void RestoreFPRegs(Register location, Register scratch);
+
+ // As above, but with heap semantics instead of stack semantics, i.e.: the
+ // location starts at the lowest address and grows towards higher addresses,
+ // for both saves and restores.
+ void SaveFPRegsToHeap(Register location, Register scratch);
+ void RestoreFPRegsFromHeap(Register location, Register scratch);
+
+ // Calculate how much stack space (in bytes) are required to store caller
+ // registers excluding those specified in the arguments.
+ int RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
+ Register exclusion1 = no_reg,
+ Register exclusion2 = no_reg,
+ Register exclusion3 = no_reg) const;
+
+ // Push caller saved registers on the stack, and return the number of bytes
+ // stack pointer is adjusted.
+ int PushCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1 = no_reg,
+ Register exclusion2 = no_reg,
+ Register exclusion3 = no_reg);
+ // Restore caller saved registers from the stack, and return the number of
+ // bytes stack pointer is adjusted.
+ int PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1 = no_reg,
+ Register exclusion2 = no_reg,
+ Register exclusion3 = no_reg);
+ void Jump(Register target, Condition cond = al);
+ void Jump(Address target, RelocInfo::Mode rmode, Condition cond = al);
+ void Jump(Handle<Code> code, RelocInfo::Mode rmode, Condition cond = al);
+ void Jump(const ExternalReference& reference) override;
+
+ // Perform a floating-point min or max operation with the
+ // (IEEE-754-compatible) semantics of ARM64's fmin/fmax. Some cases, typically
+ // NaNs or +/-0.0, are expected to be rare and are handled in out-of-line
+ // code. The specific behaviour depends on supported instructions.
+ //
+ // These functions assume (and assert) that left!=right. It is permitted
+ // for the result to alias either input register.
+ void FloatMax(SwVfpRegister result, SwVfpRegister left, SwVfpRegister right,
+ Label* out_of_line);
+ void FloatMin(SwVfpRegister result, SwVfpRegister left, SwVfpRegister right,
+ Label* out_of_line);
+ void FloatMax(DwVfpRegister result, DwVfpRegister left, DwVfpRegister right,
+ Label* out_of_line);
+ void FloatMin(DwVfpRegister result, DwVfpRegister left, DwVfpRegister right,
+ Label* out_of_line);
+
+ // Generate out-of-line cases for the macros above.
+ void FloatMaxOutOfLine(SwVfpRegister result, SwVfpRegister left,
+ SwVfpRegister right);
+ void FloatMinOutOfLine(SwVfpRegister result, SwVfpRegister left,
+ SwVfpRegister right);
+ void FloatMaxOutOfLine(DwVfpRegister result, DwVfpRegister left,
+ DwVfpRegister right);
+ void FloatMinOutOfLine(DwVfpRegister result, DwVfpRegister left,
+ DwVfpRegister right);
+
+ void ExtractLane(Register dst, QwNeonRegister src, NeonDataType dt, int lane);
+ void ExtractLane(Register dst, DwVfpRegister src, NeonDataType dt, int lane);
+ void ExtractLane(SwVfpRegister dst, QwNeonRegister src, int lane);
+ void ExtractLane(DwVfpRegister dst, QwNeonRegister src, int lane);
+ void ReplaceLane(QwNeonRegister dst, QwNeonRegister src, Register src_lane,
+ NeonDataType dt, int lane);
+ void ReplaceLane(QwNeonRegister dst, QwNeonRegister src,
+ SwVfpRegister src_lane, int lane);
+ void ReplaceLane(QwNeonRegister dst, QwNeonRegister src,
+ DwVfpRegister src_lane, int lane);
+
+ // Register move. May do nothing if the registers are identical.
+ void Move(Register dst, Smi smi);
+ void Move(Register dst, Handle<HeapObject> value);
+ void Move(Register dst, ExternalReference reference);
+ void Move(Register dst, Register src, Condition cond = al);
+ void Move(Register dst, const Operand& src, SBit sbit = LeaveCC,
+ Condition cond = al) {
+ if (!src.IsRegister() || src.rm() != dst || sbit != LeaveCC) {
+ mov(dst, src, sbit, cond);
+ }
+ }
+ // Move src0 to dst0 and src1 to dst1, handling possible overlaps.
+ void MovePair(Register dst0, Register src0, Register dst1, Register src1);
+
+ void Move(SwVfpRegister dst, SwVfpRegister src, Condition cond = al);
+ void Move(DwVfpRegister dst, DwVfpRegister src, Condition cond = al);
+ void Move(QwNeonRegister dst, QwNeonRegister src);
+
+ // Simulate s-register moves for imaginary s32 - s63 registers.
+ void VmovExtended(Register dst, int src_code);
+ void VmovExtended(int dst_code, Register src);
+ // Move between s-registers and imaginary s-registers.
+ void VmovExtended(int dst_code, int src_code);
+ void VmovExtended(int dst_code, const MemOperand& src);
+ void VmovExtended(const MemOperand& dst, int src_code);
+
+ // Register swap. Note that the register operands should be distinct.
+ void Swap(Register srcdst0, Register srcdst1);
+ void Swap(DwVfpRegister srcdst0, DwVfpRegister srcdst1);
+ void Swap(QwNeonRegister srcdst0, QwNeonRegister srcdst1);
+
+ // Get the actual activation frame alignment for target environment.
+ static int ActivationFrameAlignment();
+
+ void Bfc(Register dst, Register src, int lsb, int width, Condition cond = al);
+
+ void SmiUntag(Register reg, SBit s = LeaveCC) {
+ mov(reg, Operand::SmiUntag(reg), s);
+ }
+ void SmiUntag(Register dst, Register src, SBit s = LeaveCC) {
+ mov(dst, Operand::SmiUntag(src), s);
+ }
+
+ // Load an object from the root table.
+ void LoadRoot(Register destination, RootIndex index) override {
+ LoadRoot(destination, index, al);
+ }
+ void LoadRoot(Register destination, RootIndex index, Condition cond);
+
+ // Jump if the register contains a smi.
+ void JumpIfSmi(Register value, Label* smi_label);
+
+ void JumpIfEqual(Register x, int32_t y, Label* dest);
+ void JumpIfLessThan(Register x, int32_t y, Label* dest);
+
+ // Performs a truncating conversion of a floating point number as used by
+ // the JS bitwise operations. See ECMA-262 9.5: ToInt32. Goes to 'done' if it
+ // succeeds, otherwise falls through if result is saturated. On return
+ // 'result' either holds answer, or is clobbered on fall through.
+ void TryInlineTruncateDoubleToI(Register result, DwVfpRegister input,
+ Label* done);
+
+ // Performs a truncating conversion of a floating point number as used by
+ // the JS bitwise operations. See ECMA-262 9.5: ToInt32.
+ // Exits with 'result' holding the answer.
+ void TruncateDoubleToI(Isolate* isolate, Zone* zone, Register result,
+ DwVfpRegister double_input, StubCallMode stub_mode);
+
+ // EABI variant for double arguments in use.
+ bool use_eabi_hardfloat() {
+#ifdef __arm__
+ return base::OS::ArmUsingHardFloat();
+#elif USE_EABI_HARDFLOAT
+ return true;
+#else
+ return false;
+#endif
+ }
+
+ // Compute the start of the generated instruction stream from the current PC.
+ // This is an alternative to embedding the {CodeObject} handle as a reference.
+ void ComputeCodeStartAddress(Register dst);
+
+ void ResetSpeculationPoisonRegister();
+
+ // Control-flow integrity:
+
+ // Define a function entrypoint. This doesn't emit any code for this
+ // architecture, as control-flow integrity is not supported for it.
+ void CodeEntry() {}
+ // Define an exception handler.
+ void ExceptionHandler() {}
+ // Define an exception handler and bind a label.
+ void BindExceptionHandler(Label* label) { bind(label); }
+
+ private:
+ // Compare single values and then load the fpscr flags to a register.
+ void VFPCompareAndLoadFlags(const SwVfpRegister src1,
+ const SwVfpRegister src2,
+ const Register fpscr_flags,
+ const Condition cond = al);
+ void VFPCompareAndLoadFlags(const SwVfpRegister src1, const float src2,
+ const Register fpscr_flags,
+ const Condition cond = al);
+
+ // Compare double values and then load the fpscr flags to a register.
+ void VFPCompareAndLoadFlags(const DwVfpRegister src1,
+ const DwVfpRegister src2,
+ const Register fpscr_flags,
+ const Condition cond = al);
+ void VFPCompareAndLoadFlags(const DwVfpRegister src1, const double src2,
+ const Register fpscr_flags,
+ const Condition cond = al);
+
+ void Jump(intptr_t target, RelocInfo::Mode rmode, Condition cond = al);
+
+ // Implementation helpers for FloatMin and FloatMax.
+ template <typename T>
+ void FloatMaxHelper(T result, T left, T right, Label* out_of_line);
+ template <typename T>
+ void FloatMinHelper(T result, T left, T right, Label* out_of_line);
+ template <typename T>
+ void FloatMaxOutOfLineHelper(T result, T left, T right);
+ template <typename T>
+ void FloatMinOutOfLineHelper(T result, T left, T right);
+
+ int CalculateStackPassedWords(int num_reg_arguments,
+ int num_double_arguments);
+
+ void CallCFunctionHelper(Register function, int num_reg_arguments,
+ int num_double_arguments);
+
+ void CallRecordWriteStub(Register object, Operand offset,
+ RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode, Handle<Code> code_target,
+ Address wasm_target);
+};
+
+// MacroAssembler implements a collection of frequently used macros.
+class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler {
+ public:
+ using TurboAssembler::TurboAssembler;
+
+ void Mls(Register dst, Register src1, Register src2, Register srcA,
+ Condition cond = al);
+ void And(Register dst, Register src1, const Operand& src2,
+ Condition cond = al);
+ void Ubfx(Register dst, Register src, int lsb, int width,
+ Condition cond = al);
+ void Sbfx(Register dst, Register src, int lsb, int width,
+ Condition cond = al);
+
+ // ---------------------------------------------------------------------------
+ // GC Support
+
+ // Notify the garbage collector that we wrote a pointer into an object.
+ // |object| is the object being stored into, |value| is the object being
+ // stored.
+ // The offset is the offset from the start of the object, not the offset from
+ // the tagged HeapObject pointer. For use with FieldMemOperand(reg, off).
+ void RecordWriteField(
+ Register object, int offset, Register value, LinkRegisterStatus lr_status,
+ SaveFPRegsMode save_fp,
+ RememberedSetAction remembered_set_action = EMIT_REMEMBERED_SET,
+ SmiCheck smi_check = INLINE_SMI_CHECK);
+
+ // For a given |object| notify the garbage collector that the slot at |offset|
+ // has been written. |value| is the object being stored.
+ void RecordWrite(
+ Register object, Operand offset, Register value,
+ LinkRegisterStatus lr_status, SaveFPRegsMode save_fp,
+ RememberedSetAction remembered_set_action = EMIT_REMEMBERED_SET,
+ SmiCheck smi_check = INLINE_SMI_CHECK);
+
+ // Enter exit frame.
+ // stack_space - extra stack space, used for alignment before call to C.
+ void EnterExitFrame(bool save_doubles, int stack_space = 0,
+ StackFrame::Type frame_type = StackFrame::EXIT);
+
+ // Leave the current exit frame. Expects the return value in r0.
+ // Expect the number of values, pushed prior to the exit frame, to
+ // remove in a register (or no_reg, if there is nothing to remove).
+ void LeaveExitFrame(bool save_doubles, Register argument_count,
+ bool argument_count_is_length = false);
+
+ void LoadMap(Register destination, Register object);
+
+ // Load the global proxy from the current context.
+ void LoadGlobalProxy(Register dst);
+
+ void LoadNativeContextSlot(int index, Register dst);
+
+ // ---------------------------------------------------------------------------
+ // JavaScript invokes
+
+ // Invoke the JavaScript function code by either calling or jumping.
+ void InvokeFunctionCode(Register function, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count, InvokeFlag flag);
+
+ // On function call, call into the debugger.
+ void CallDebugOnFunctionCall(Register fun, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count);
+
+ // Invoke the JavaScript function in the given register. Changes the
+ // current context to the context in the function before invoking.
+ void InvokeFunctionWithNewTarget(Register function, Register new_target,
+ Register actual_parameter_count,
+ InvokeFlag flag);
+
+ void InvokeFunction(Register function, Register expected_parameter_count,
+ Register actual_parameter_count, InvokeFlag flag);
+
+ // Frame restart support
+ void MaybeDropFrames();
+
+ // Exception handling
+
+ // Push a new stack handler and link into stack handler chain.
+ void PushStackHandler();
+
+ // Unlink the stack handler on top of the stack from the stack handler chain.
+ // Must preserve the result register.
+ void PopStackHandler();
+
+ // ---------------------------------------------------------------------------
+ // Support functions.
+
+ // Compare object type for heap object. heap_object contains a non-Smi
+ // whose object type should be compared with the given type. This both
+ // sets the flags and leaves the object type in the type_reg register.
+ // It leaves the map in the map register (unless the type_reg and map register
+ // are the same register). It leaves the heap object in the heap_object
+ // register unless the heap_object register is the same register as one of the
+ // other registers.
+ // Type_reg can be no_reg. In that case a scratch register is used.
+ void CompareObjectType(Register heap_object, Register map, Register type_reg,
+ InstanceType type);
+
+ // Compare instance type in a map. map contains a valid map object whose
+ // object type should be compared with the given type. This both
+ // sets the flags and leaves the object type in the type_reg register.
+ void CompareInstanceType(Register map, Register type_reg, InstanceType type);
+
+ // Compare the object in a register to a value from the root list.
+ // Acquires a scratch register.
+ void CompareRoot(Register obj, RootIndex index);
+ void PushRoot(RootIndex index) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ LoadRoot(scratch, index);
+ Push(scratch);
+ }
+
+ // Compare the object in a register to a value and jump if they are equal.
+ void JumpIfRoot(Register with, RootIndex index, Label* if_equal) {
+ CompareRoot(with, index);
+ b(eq, if_equal);
+ }
+
+ // Compare the object in a register to a value and jump if they are not equal.
+ void JumpIfNotRoot(Register with, RootIndex index, Label* if_not_equal) {
+ CompareRoot(with, index);
+ b(ne, if_not_equal);
+ }
+
+ // Checks if value is in range [lower_limit, higher_limit] using a single
+ // comparison.
+ void JumpIfIsInRange(Register value, unsigned lower_limit,
+ unsigned higher_limit, Label* on_in_range);
+
+ // It assumes that the arguments are located below the stack pointer.
+ // argc is the number of arguments not including the receiver.
+ // TODO(victorgomes): Remove this function once we stick with the reversed
+ // arguments order.
+ MemOperand ReceiverOperand(Register argc) {
+ return MemOperand(sp, 0);
+ }
+
+ // ---------------------------------------------------------------------------
+ // Runtime calls
+
+ // Call a runtime routine.
+ void CallRuntime(const Runtime::Function* f, int num_arguments,
+ SaveFPRegsMode save_doubles = kDontSaveFPRegs);
+
+ // Convenience function: Same as above, but takes the fid instead.
+ void CallRuntime(Runtime::FunctionId fid,
+ SaveFPRegsMode save_doubles = kDontSaveFPRegs) {
+ const Runtime::Function* function = Runtime::FunctionForId(fid);
+ CallRuntime(function, function->nargs, save_doubles);
+ }
+
+ // Convenience function: Same as above, but takes the fid instead.
+ void CallRuntime(Runtime::FunctionId fid, int num_arguments,
+ SaveFPRegsMode save_doubles = kDontSaveFPRegs) {
+ CallRuntime(Runtime::FunctionForId(fid), num_arguments, save_doubles);
+ }
+
+ // Convenience function: tail call a runtime routine (jump).
+ void TailCallRuntime(Runtime::FunctionId fid);
+
+ // Jump to a runtime routine.
+ void JumpToExternalReference(const ExternalReference& builtin,
+ bool builtin_exit_frame = false);
+
+ // Generates a trampoline to jump to the off-heap instruction stream.
+ void JumpToInstructionStream(Address entry);
+
+ // ---------------------------------------------------------------------------
+ // In-place weak references.
+ void LoadWeakValue(Register out, Register in, Label* target_if_cleared);
+
+ // ---------------------------------------------------------------------------
+ // StatsCounter support
+
+ void IncrementCounter(StatsCounter* counter, int value, Register scratch1,
+ Register scratch2);
+ void DecrementCounter(StatsCounter* counter, int value, Register scratch1,
+ Register scratch2);
+
+ // ---------------------------------------------------------------------------
+ // Stack limit utilities
+ void LoadStackLimit(Register destination, StackLimitKind kind);
+ void StackOverflowCheck(Register num_args, Register scratch,
+ Label* stack_overflow);
+
+ // ---------------------------------------------------------------------------
+ // Smi utilities
+
+ void SmiTag(Register reg, SBit s = LeaveCC);
+ void SmiTag(Register dst, Register src, SBit s = LeaveCC);
+
+ // Test if the register contains a smi (Z == 0 (eq) if true).
+ void SmiTst(Register value);
+ // Jump if either of the registers contain a non-smi.
+ void JumpIfNotSmi(Register value, Label* not_smi_label);
+
+ // Abort execution if argument is a smi, enabled via --debug-code.
+ void AssertNotSmi(Register object);
+ void AssertSmi(Register object);
+
+ // Abort execution if argument is not a Constructor, enabled via --debug-code.
+ void AssertConstructor(Register object);
+
+ // Abort execution if argument is not a JSFunction, enabled via --debug-code.
+ void AssertFunction(Register object);
+
+ // Abort execution if argument is not a JSBoundFunction,
+ // enabled via --debug-code.
+ void AssertBoundFunction(Register object);
+
+ // Abort execution if argument is not a JSGeneratorObject (or subclass),
+ // enabled via --debug-code.
+ void AssertGeneratorObject(Register object);
+
+ // Abort execution if argument is not undefined or an AllocationSite, enabled
+ // via --debug-code.
+ void AssertUndefinedOrAllocationSite(Register object, Register scratch);
+
+ template <typename Field>
+ void DecodeField(Register dst, Register src) {
+ Ubfx(dst, src, Field::kShift, Field::kSize);
+ }
+
+ template <typename Field>
+ void DecodeField(Register reg) {
+ DecodeField<Field>(reg, reg);
+ }
+
+ private:
+ // Helper functions for generating invokes.
+ void InvokePrologue(Register expected_parameter_count,
+ Register actual_parameter_count, Label* done,
+ InvokeFlag flag);
+
+ DISALLOW_IMPLICIT_CONSTRUCTORS(MacroAssembler);
+};
+
+#define ACCESS_MASM(masm) masm->
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_ARM_MACRO_ASSEMBLER_ARM_H_
diff --git a/src/codegen/arm/register-arm.h b/src/codegen/arm/register-arm.h
new file mode 100644
index 0000000..6cb6c60
--- /dev/null
+++ b/src/codegen/arm/register-arm.h
@@ -0,0 +1,362 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_ARM_REGISTER_ARM_H_
+#define V8_CODEGEN_ARM_REGISTER_ARM_H_
+
+#include "src/codegen/register.h"
+#include "src/codegen/reglist.h"
+
+namespace v8 {
+namespace internal {
+
+// clang-format off
+#define GENERAL_REGISTERS(V) \
+ V(r0) V(r1) V(r2) V(r3) V(r4) V(r5) V(r6) V(r7) \
+ V(r8) V(r9) V(r10) V(fp) V(ip) V(sp) V(lr) V(pc)
+
+#define ALLOCATABLE_GENERAL_REGISTERS(V) \
+ V(r0) V(r1) V(r2) V(r3) V(r4) V(r5) V(r6) V(r7) \
+ V(r8) V(r9)
+
+#define FLOAT_REGISTERS(V) \
+ V(s0) V(s1) V(s2) V(s3) V(s4) V(s5) V(s6) V(s7) \
+ V(s8) V(s9) V(s10) V(s11) V(s12) V(s13) V(s14) V(s15) \
+ V(s16) V(s17) V(s18) V(s19) V(s20) V(s21) V(s22) V(s23) \
+ V(s24) V(s25) V(s26) V(s27) V(s28) V(s29) V(s30) V(s31)
+
+#define LOW_DOUBLE_REGISTERS(V) \
+ V(d0) V(d1) V(d2) V(d3) V(d4) V(d5) V(d6) V(d7) \
+ V(d8) V(d9) V(d10) V(d11) V(d12) V(d13) V(d14) V(d15)
+
+#define NON_LOW_DOUBLE_REGISTERS(V) \
+ V(d16) V(d17) V(d18) V(d19) V(d20) V(d21) V(d22) V(d23) \
+ V(d24) V(d25) V(d26) V(d27) V(d28) V(d29) V(d30) V(d31)
+
+#define DOUBLE_REGISTERS(V) \
+ LOW_DOUBLE_REGISTERS(V) NON_LOW_DOUBLE_REGISTERS(V)
+
+#define SIMD128_REGISTERS(V) \
+ V(q0) V(q1) V(q2) V(q3) V(q4) V(q5) V(q6) V(q7) \
+ V(q8) V(q9) V(q10) V(q11) V(q12) V(q13) V(q14) V(q15)
+
+#define ALLOCATABLE_DOUBLE_REGISTERS(V) \
+ V(d0) V(d1) V(d2) V(d3) V(d4) V(d5) V(d6) V(d7) \
+ V(d8) V(d9) V(d10) V(d11) V(d12) \
+ V(d16) V(d17) V(d18) V(d19) V(d20) V(d21) V(d22) V(d23) \
+ V(d24) V(d25) V(d26) V(d27) V(d28) V(d29) V(d30) V(d31)
+
+#define ALLOCATABLE_NO_VFP32_DOUBLE_REGISTERS(V) \
+ V(d0) V(d1) V(d2) V(d3) V(d4) V(d5) V(d6) V(d7) \
+ V(d8) V(d9) V(d10) V(d11) V(d12) V(d15)
+
+#define C_REGISTERS(V) \
+ V(cr0) V(cr1) V(cr2) V(cr3) V(cr4) V(cr5) V(cr6) V(cr7) \
+ V(cr8) V(cr9) V(cr10) V(cr11) V(cr12) V(cr15)
+// clang-format on
+
+// The ARM ABI does not specify the usage of register r9, which may be reserved
+// as the static base or thread register on some platforms, in which case we
+// leave it alone. Adjust the value of kR9Available accordingly:
+const int kR9Available = 1; // 1 if available to us, 0 if reserved
+
+// Register list in load/store instructions
+// Note that the bit values must match those used in actual instruction encoding
+
+// Caller-saved/arguments registers
+const RegList kJSCallerSaved = 1 << 0 | // r0 a1
+ 1 << 1 | // r1 a2
+ 1 << 2 | // r2 a3
+ 1 << 3; // r3 a4
+
+const int kNumJSCallerSaved = 4;
+
+// Callee-saved registers preserved when switching from C to JavaScript
+const RegList kCalleeSaved = 1 << 4 | // r4 v1
+ 1 << 5 | // r5 v2
+ 1 << 6 | // r6 v3
+ 1 << 7 | // r7 v4 (cp in JavaScript code)
+ 1 << 8 | // r8 v5 (pp in JavaScript code)
+ kR9Available << 9 | // r9 v6
+ 1 << 10 | // r10 v7
+ 1 << 11; // r11 v8 (fp in JavaScript code)
+
+// When calling into C++ (only for C++ calls that can't cause a GC).
+// The call code will take care of lr, fp, etc.
+const RegList kCallerSaved = 1 << 0 | // r0
+ 1 << 1 | // r1
+ 1 << 2 | // r2
+ 1 << 3 | // r3
+ 1 << 9; // r9
+
+const int kNumCalleeSaved = 7 + kR9Available;
+
+// Double registers d8 to d15 are callee-saved.
+const int kNumDoubleCalleeSaved = 8;
+
+enum RegisterCode {
+#define REGISTER_CODE(R) kRegCode_##R,
+ GENERAL_REGISTERS(REGISTER_CODE)
+#undef REGISTER_CODE
+ kRegAfterLast
+};
+
+class Register : public RegisterBase<Register, kRegAfterLast> {
+ friend class RegisterBase;
+
+ explicit constexpr Register(int code) : RegisterBase(code) {}
+};
+
+ASSERT_TRIVIALLY_COPYABLE(Register);
+static_assert(sizeof(Register) == sizeof(int),
+ "Register can efficiently be passed by value");
+
+// r7: context register
+#define DECLARE_REGISTER(R) \
+ constexpr Register R = Register::from_code(kRegCode_##R);
+GENERAL_REGISTERS(DECLARE_REGISTER)
+#undef DECLARE_REGISTER
+constexpr Register no_reg = Register::no_reg();
+
+constexpr bool kPadArguments = false;
+constexpr bool kSimpleFPAliasing = false;
+constexpr bool kSimdMaskRegisters = false;
+
+enum SwVfpRegisterCode {
+#define REGISTER_CODE(R) kSwVfpCode_##R,
+ FLOAT_REGISTERS(REGISTER_CODE)
+#undef REGISTER_CODE
+ kSwVfpAfterLast
+};
+
+// Representation of a list of non-overlapping VFP registers. This list
+// represents the data layout of VFP registers as a bitfield:
+// S registers cover 1 bit
+// D registers cover 2 bits
+// Q registers cover 4 bits
+//
+// This way, we make sure no registers in the list ever overlap. However, a list
+// may represent multiple different sets of registers,
+// e.g. [d0 s2 s3] <=> [s0 s1 d1].
+using VfpRegList = uint64_t;
+
+// Single word VFP register.
+class SwVfpRegister : public RegisterBase<SwVfpRegister, kSwVfpAfterLast> {
+ public:
+ static constexpr int kSizeInBytes = 4;
+
+ static void split_code(int reg_code, int* vm, int* m) {
+ DCHECK(from_code(reg_code).is_valid());
+ *m = reg_code & 0x1;
+ *vm = reg_code >> 1;
+ }
+ void split_code(int* vm, int* m) const { split_code(code(), vm, m); }
+ VfpRegList ToVfpRegList() const {
+ DCHECK(is_valid());
+ // Each bit in the list corresponds to a S register.
+ return uint64_t{0x1} << code();
+ }
+
+ private:
+ friend class RegisterBase;
+ explicit constexpr SwVfpRegister(int code) : RegisterBase(code) {}
+};
+
+ASSERT_TRIVIALLY_COPYABLE(SwVfpRegister);
+static_assert(sizeof(SwVfpRegister) == sizeof(int),
+ "SwVfpRegister can efficiently be passed by value");
+
+using FloatRegister = SwVfpRegister;
+
+enum DoubleRegisterCode {
+#define REGISTER_CODE(R) kDoubleCode_##R,
+ DOUBLE_REGISTERS(REGISTER_CODE)
+#undef REGISTER_CODE
+ kDoubleAfterLast
+};
+
+// Double word VFP register.
+class DwVfpRegister : public RegisterBase<DwVfpRegister, kDoubleAfterLast> {
+ public:
+ static constexpr int kSizeInBytes = 8;
+
+ // This function differs from kNumRegisters by returning the number of double
+ // registers supported by the current CPU, while kNumRegisters always returns
+ // 32.
+ inline static int SupportedRegisterCount();
+
+ static void split_code(int reg_code, int* vm, int* m) {
+ DCHECK(from_code(reg_code).is_valid());
+ *m = (reg_code & 0x10) >> 4;
+ *vm = reg_code & 0x0F;
+ }
+ void split_code(int* vm, int* m) const { split_code(code(), vm, m); }
+ VfpRegList ToVfpRegList() const {
+ DCHECK(is_valid());
+ // A D register overlaps two S registers.
+ return uint64_t{0x3} << (code() * 2);
+ }
+
+ private:
+ friend class RegisterBase;
+ friend class LowDwVfpRegister;
+ explicit constexpr DwVfpRegister(int code) : RegisterBase(code) {}
+};
+
+ASSERT_TRIVIALLY_COPYABLE(DwVfpRegister);
+static_assert(sizeof(DwVfpRegister) == sizeof(int),
+ "DwVfpRegister can efficiently be passed by value");
+
+using DoubleRegister = DwVfpRegister;
+
+// Double word VFP register d0-15.
+class LowDwVfpRegister
+ : public RegisterBase<LowDwVfpRegister, kDoubleCode_d16> {
+ public:
+ constexpr operator DwVfpRegister() const { return DwVfpRegister(code()); }
+
+ SwVfpRegister low() const { return SwVfpRegister::from_code(code() * 2); }
+ SwVfpRegister high() const {
+ return SwVfpRegister::from_code(code() * 2 + 1);
+ }
+ VfpRegList ToVfpRegList() const {
+ DCHECK(is_valid());
+ // A D register overlaps two S registers.
+ return uint64_t{0x3} << (code() * 2);
+ }
+
+ private:
+ friend class RegisterBase;
+ explicit constexpr LowDwVfpRegister(int code) : RegisterBase(code) {}
+};
+
+enum Simd128RegisterCode {
+#define REGISTER_CODE(R) kSimd128Code_##R,
+ SIMD128_REGISTERS(REGISTER_CODE)
+#undef REGISTER_CODE
+ kSimd128AfterLast
+};
+
+// Quad word NEON register.
+class QwNeonRegister : public RegisterBase<QwNeonRegister, kSimd128AfterLast> {
+ public:
+ static void split_code(int reg_code, int* vm, int* m) {
+ DCHECK(from_code(reg_code).is_valid());
+ int encoded_code = reg_code << 1;
+ *m = (encoded_code & 0x10) >> 4;
+ *vm = encoded_code & 0x0F;
+ }
+ void split_code(int* vm, int* m) const { split_code(code(), vm, m); }
+ DwVfpRegister low() const { return DwVfpRegister::from_code(code() * 2); }
+ DwVfpRegister high() const {
+ return DwVfpRegister::from_code(code() * 2 + 1);
+ }
+ VfpRegList ToVfpRegList() const {
+ DCHECK(is_valid());
+ // A Q register overlaps four S registers.
+ return uint64_t{0xf} << (code() * 4);
+ }
+
+ private:
+ friend class RegisterBase;
+ explicit constexpr QwNeonRegister(int code) : RegisterBase(code) {}
+};
+
+using QuadRegister = QwNeonRegister;
+
+using Simd128Register = QwNeonRegister;
+
+enum CRegisterCode {
+#define REGISTER_CODE(R) kCCode_##R,
+ C_REGISTERS(REGISTER_CODE)
+#undef REGISTER_CODE
+ kCAfterLast
+};
+
+// Coprocessor register
+class CRegister : public RegisterBase<CRegister, kCAfterLast> {
+ friend class RegisterBase;
+ explicit constexpr CRegister(int code) : RegisterBase(code) {}
+};
+
+// Support for the VFP registers s0 to s31 (d0 to d15).
+// Note that "s(N):s(N+1)" is the same as "d(N/2)".
+#define DECLARE_FLOAT_REGISTER(R) \
+ constexpr SwVfpRegister R = SwVfpRegister::from_code(kSwVfpCode_##R);
+FLOAT_REGISTERS(DECLARE_FLOAT_REGISTER)
+#undef DECLARE_FLOAT_REGISTER
+
+#define DECLARE_LOW_DOUBLE_REGISTER(R) \
+ constexpr LowDwVfpRegister R = LowDwVfpRegister::from_code(kDoubleCode_##R);
+LOW_DOUBLE_REGISTERS(DECLARE_LOW_DOUBLE_REGISTER)
+#undef DECLARE_LOW_DOUBLE_REGISTER
+
+#define DECLARE_DOUBLE_REGISTER(R) \
+ constexpr DwVfpRegister R = DwVfpRegister::from_code(kDoubleCode_##R);
+NON_LOW_DOUBLE_REGISTERS(DECLARE_DOUBLE_REGISTER)
+#undef DECLARE_DOUBLE_REGISTER
+
+constexpr DwVfpRegister no_dreg = DwVfpRegister::no_reg();
+
+#define DECLARE_SIMD128_REGISTER(R) \
+ constexpr Simd128Register R = Simd128Register::from_code(kSimd128Code_##R);
+SIMD128_REGISTERS(DECLARE_SIMD128_REGISTER)
+#undef DECLARE_SIMD128_REGISTER
+
+// Aliases for double registers.
+constexpr LowDwVfpRegister kFirstCalleeSavedDoubleReg = d8;
+constexpr LowDwVfpRegister kLastCalleeSavedDoubleReg = d15;
+constexpr LowDwVfpRegister kDoubleRegZero = d13;
+
+constexpr CRegister no_creg = CRegister::no_reg();
+
+#define DECLARE_C_REGISTER(R) \
+ constexpr CRegister R = CRegister::from_code(kCCode_##R);
+C_REGISTERS(DECLARE_C_REGISTER)
+#undef DECLARE_C_REGISTER
+
+// Define {RegisterName} methods for the register types.
+DEFINE_REGISTER_NAMES(Register, GENERAL_REGISTERS)
+DEFINE_REGISTER_NAMES(SwVfpRegister, FLOAT_REGISTERS)
+DEFINE_REGISTER_NAMES(DwVfpRegister, DOUBLE_REGISTERS)
+DEFINE_REGISTER_NAMES(LowDwVfpRegister, LOW_DOUBLE_REGISTERS)
+DEFINE_REGISTER_NAMES(QwNeonRegister, SIMD128_REGISTERS)
+DEFINE_REGISTER_NAMES(CRegister, C_REGISTERS)
+
+// Give alias names to registers for calling conventions.
+constexpr Register kReturnRegister0 = r0;
+constexpr Register kReturnRegister1 = r1;
+constexpr Register kReturnRegister2 = r2;
+constexpr Register kJSFunctionRegister = r1;
+constexpr Register kContextRegister = r7;
+constexpr Register kAllocateSizeRegister = r1;
+constexpr Register kSpeculationPoisonRegister = r9;
+constexpr Register kInterpreterAccumulatorRegister = r0;
+constexpr Register kInterpreterBytecodeOffsetRegister = r5;
+constexpr Register kInterpreterBytecodeArrayRegister = r6;
+constexpr Register kInterpreterDispatchTableRegister = r8;
+
+constexpr Register kJavaScriptCallArgCountRegister = r0;
+constexpr Register kJavaScriptCallCodeStartRegister = r2;
+constexpr Register kJavaScriptCallTargetRegister = kJSFunctionRegister;
+constexpr Register kJavaScriptCallNewTargetRegister = r3;
+constexpr Register kJavaScriptCallExtraArg1Register = r2;
+
+constexpr Register kOffHeapTrampolineRegister = ip;
+constexpr Register kRuntimeCallFunctionRegister = r1;
+constexpr Register kRuntimeCallArgCountRegister = r0;
+constexpr Register kRuntimeCallArgvRegister = r2;
+constexpr Register kWasmInstanceRegister = r3;
+constexpr Register kWasmCompileLazyFuncIndexRegister = r4;
+
+// Give alias names to registers
+constexpr Register cp = r7; // JavaScript context pointer.
+constexpr Register kRootRegister = r10; // Roots array pointer.
+
+constexpr DoubleRegister kFPReturnRegister0 = d0;
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_ARM_REGISTER_ARM_H_
diff --git a/src/codegen/arm64/assembler-arm64-inl.h b/src/codegen/arm64/assembler-arm64-inl.h
new file mode 100644
index 0000000..c47f8f1
--- /dev/null
+++ b/src/codegen/arm64/assembler-arm64-inl.h
@@ -0,0 +1,1086 @@
+// Copyright 2013 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_ARM64_ASSEMBLER_ARM64_INL_H_
+#define V8_CODEGEN_ARM64_ASSEMBLER_ARM64_INL_H_
+
+#include <type_traits>
+
+#include "src/base/memory.h"
+#include "src/codegen/arm64/assembler-arm64.h"
+#include "src/codegen/assembler.h"
+#include "src/debug/debug.h"
+#include "src/objects/objects-inl.h"
+#include "src/objects/smi.h"
+
+namespace v8 {
+namespace internal {
+
+bool CpuFeatures::SupportsOptimizer() { return true; }
+
+bool CpuFeatures::SupportsWasmSimd128() { return true; }
+
+void RelocInfo::apply(intptr_t delta) {
+ // On arm64 only internal references and immediate branches need extra work.
+ if (RelocInfo::IsInternalReference(rmode_)) {
+ // Absolute code pointer inside code object moves with the code object.
+ intptr_t internal_ref = ReadUnalignedValue<intptr_t>(pc_);
+ internal_ref += delta; // Relocate entry.
+ WriteUnalignedValue<intptr_t>(pc_, internal_ref);
+ } else {
+ Instruction* instr = reinterpret_cast<Instruction*>(pc_);
+ if (instr->IsBranchAndLink() || instr->IsUnconditionalBranch()) {
+ Address old_target =
+ reinterpret_cast<Address>(instr->ImmPCOffsetTarget());
+ Address new_target = old_target - delta;
+ instr->SetBranchImmTarget(reinterpret_cast<Instruction*>(new_target));
+ }
+ }
+}
+
+inline bool CPURegister::IsSameSizeAndType(const CPURegister& other) const {
+ return (reg_size_ == other.reg_size_) && (reg_type_ == other.reg_type_);
+}
+
+inline bool CPURegister::IsZero() const {
+ DCHECK(is_valid());
+ return IsRegister() && (code() == kZeroRegCode);
+}
+
+inline bool CPURegister::IsSP() const {
+ DCHECK(is_valid());
+ return IsRegister() && (code() == kSPRegInternalCode);
+}
+
+inline void CPURegList::Combine(const CPURegList& other) {
+ DCHECK(other.type() == type_);
+ DCHECK(other.RegisterSizeInBits() == size_);
+ list_ |= other.list();
+}
+
+inline void CPURegList::Remove(const CPURegList& other) {
+ if (other.type() == type_) {
+ list_ &= ~other.list();
+ }
+}
+
+inline void CPURegList::Combine(const CPURegister& other) {
+ DCHECK(other.type() == type_);
+ DCHECK(other.SizeInBits() == size_);
+ Combine(other.code());
+}
+
+inline void CPURegList::Remove(const CPURegister& other1,
+ const CPURegister& other2,
+ const CPURegister& other3,
+ const CPURegister& other4) {
+ if (!other1.IsNone() && (other1.type() == type_)) Remove(other1.code());
+ if (!other2.IsNone() && (other2.type() == type_)) Remove(other2.code());
+ if (!other3.IsNone() && (other3.type() == type_)) Remove(other3.code());
+ if (!other4.IsNone() && (other4.type() == type_)) Remove(other4.code());
+}
+
+inline void CPURegList::Combine(int code) {
+ DCHECK(CPURegister::Create(code, size_, type_).is_valid());
+ list_ |= (1ULL << code);
+ DCHECK(is_valid());
+}
+
+inline void CPURegList::Remove(int code) {
+ DCHECK(CPURegister::Create(code, size_, type_).is_valid());
+ list_ &= ~(1ULL << code);
+}
+
+inline Register Register::XRegFromCode(unsigned code) {
+ if (code == kSPRegInternalCode) {
+ return sp;
+ } else {
+ DCHECK_LT(code, static_cast<unsigned>(kNumberOfRegisters));
+ return Register::Create(code, kXRegSizeInBits);
+ }
+}
+
+inline Register Register::WRegFromCode(unsigned code) {
+ if (code == kSPRegInternalCode) {
+ return wsp;
+ } else {
+ DCHECK_LT(code, static_cast<unsigned>(kNumberOfRegisters));
+ return Register::Create(code, kWRegSizeInBits);
+ }
+}
+
+inline VRegister VRegister::BRegFromCode(unsigned code) {
+ DCHECK_LT(code, static_cast<unsigned>(kNumberOfVRegisters));
+ return VRegister::Create(code, kBRegSizeInBits);
+}
+
+inline VRegister VRegister::HRegFromCode(unsigned code) {
+ DCHECK_LT(code, static_cast<unsigned>(kNumberOfVRegisters));
+ return VRegister::Create(code, kHRegSizeInBits);
+}
+
+inline VRegister VRegister::SRegFromCode(unsigned code) {
+ DCHECK_LT(code, static_cast<unsigned>(kNumberOfVRegisters));
+ return VRegister::Create(code, kSRegSizeInBits);
+}
+
+inline VRegister VRegister::DRegFromCode(unsigned code) {
+ DCHECK_LT(code, static_cast<unsigned>(kNumberOfVRegisters));
+ return VRegister::Create(code, kDRegSizeInBits);
+}
+
+inline VRegister VRegister::QRegFromCode(unsigned code) {
+ DCHECK_LT(code, static_cast<unsigned>(kNumberOfVRegisters));
+ return VRegister::Create(code, kQRegSizeInBits);
+}
+
+inline VRegister VRegister::VRegFromCode(unsigned code) {
+ DCHECK_LT(code, static_cast<unsigned>(kNumberOfVRegisters));
+ return VRegister::Create(code, kVRegSizeInBits);
+}
+
+inline Register CPURegister::W() const {
+ DCHECK(IsRegister());
+ return Register::WRegFromCode(code());
+}
+
+inline Register CPURegister::Reg() const {
+ DCHECK(IsRegister());
+ return Register::Create(code(), reg_size_);
+}
+
+inline VRegister CPURegister::VReg() const {
+ DCHECK(IsVRegister());
+ return VRegister::Create(code(), reg_size_);
+}
+
+inline Register CPURegister::X() const {
+ DCHECK(IsRegister());
+ return Register::XRegFromCode(code());
+}
+
+inline VRegister CPURegister::V() const {
+ DCHECK(IsVRegister());
+ return VRegister::VRegFromCode(code());
+}
+
+inline VRegister CPURegister::B() const {
+ DCHECK(IsVRegister());
+ return VRegister::BRegFromCode(code());
+}
+
+inline VRegister CPURegister::H() const {
+ DCHECK(IsVRegister());
+ return VRegister::HRegFromCode(code());
+}
+
+inline VRegister CPURegister::S() const {
+ DCHECK(IsVRegister());
+ return VRegister::SRegFromCode(code());
+}
+
+inline VRegister CPURegister::D() const {
+ DCHECK(IsVRegister());
+ return VRegister::DRegFromCode(code());
+}
+
+inline VRegister CPURegister::Q() const {
+ DCHECK(IsVRegister());
+ return VRegister::QRegFromCode(code());
+}
+
+// Immediate.
+// Default initializer is for int types
+template <typename T>
+struct ImmediateInitializer {
+ static inline RelocInfo::Mode rmode_for(T) { return RelocInfo::NONE; }
+ static inline int64_t immediate_for(T t) {
+ STATIC_ASSERT(sizeof(T) <= 8);
+ STATIC_ASSERT(std::is_integral<T>::value || std::is_enum<T>::value);
+ return t;
+ }
+};
+
+template <>
+struct ImmediateInitializer<Smi> {
+ static inline RelocInfo::Mode rmode_for(Smi t) { return RelocInfo::NONE; }
+ static inline int64_t immediate_for(Smi t) {
+ return static_cast<int64_t>(t.ptr());
+ }
+};
+
+template <>
+struct ImmediateInitializer<ExternalReference> {
+ static inline RelocInfo::Mode rmode_for(ExternalReference t) {
+ return RelocInfo::EXTERNAL_REFERENCE;
+ }
+ static inline int64_t immediate_for(ExternalReference t) {
+ return static_cast<int64_t>(t.address());
+ }
+};
+
+template <typename T>
+Immediate::Immediate(Handle<T> handle, RelocInfo::Mode mode)
+ : value_(static_cast<intptr_t>(handle.address())), rmode_(mode) {
+ DCHECK(RelocInfo::IsEmbeddedObjectMode(mode));
+}
+
+template <typename T>
+Immediate::Immediate(T t)
+ : value_(ImmediateInitializer<T>::immediate_for(t)),
+ rmode_(ImmediateInitializer<T>::rmode_for(t)) {}
+
+template <typename T>
+Immediate::Immediate(T t, RelocInfo::Mode rmode)
+ : value_(ImmediateInitializer<T>::immediate_for(t)), rmode_(rmode) {
+ STATIC_ASSERT(std::is_integral<T>::value);
+}
+
+template <typename T>
+Operand::Operand(T t) : immediate_(t), reg_(NoReg) {}
+
+template <typename T>
+Operand::Operand(T t, RelocInfo::Mode rmode)
+ : immediate_(t, rmode), reg_(NoReg) {}
+
+Operand::Operand(Register reg, Shift shift, unsigned shift_amount)
+ : immediate_(0),
+ reg_(reg),
+ shift_(shift),
+ extend_(NO_EXTEND),
+ shift_amount_(shift_amount) {
+ DCHECK(reg.Is64Bits() || (shift_amount < kWRegSizeInBits));
+ DCHECK(reg.Is32Bits() || (shift_amount < kXRegSizeInBits));
+ DCHECK_IMPLIES(reg.IsSP(), shift_amount == 0);
+}
+
+Operand::Operand(Register reg, Extend extend, unsigned shift_amount)
+ : immediate_(0),
+ reg_(reg),
+ shift_(NO_SHIFT),
+ extend_(extend),
+ shift_amount_(shift_amount) {
+ DCHECK(reg.is_valid());
+ DCHECK_LE(shift_amount, 4);
+ DCHECK(!reg.IsSP());
+
+ // Extend modes SXTX and UXTX require a 64-bit register.
+ DCHECK(reg.Is64Bits() || ((extend != SXTX) && (extend != UXTX)));
+}
+
+bool Operand::IsHeapObjectRequest() const {
+ DCHECK_IMPLIES(heap_object_request_.has_value(), reg_ == NoReg);
+ DCHECK_IMPLIES(heap_object_request_.has_value(),
+ immediate_.rmode() == RelocInfo::FULL_EMBEDDED_OBJECT ||
+ immediate_.rmode() == RelocInfo::CODE_TARGET);
+ return heap_object_request_.has_value();
+}
+
+HeapObjectRequest Operand::heap_object_request() const {
+ DCHECK(IsHeapObjectRequest());
+ return *heap_object_request_;
+}
+
+bool Operand::IsImmediate() const {
+ return reg_ == NoReg && !IsHeapObjectRequest();
+}
+
+bool Operand::IsShiftedRegister() const {
+ return reg_.is_valid() && (shift_ != NO_SHIFT);
+}
+
+bool Operand::IsExtendedRegister() const {
+ return reg_.is_valid() && (extend_ != NO_EXTEND);
+}
+
+bool Operand::IsZero() const {
+ if (IsImmediate()) {
+ return ImmediateValue() == 0;
+ } else {
+ return reg().IsZero();
+ }
+}
+
+Operand Operand::ToExtendedRegister() const {
+ DCHECK(IsShiftedRegister());
+ DCHECK((shift_ == LSL) && (shift_amount_ <= 4));
+ return Operand(reg_, reg_.Is64Bits() ? UXTX : UXTW, shift_amount_);
+}
+
+Operand Operand::ToW() const {
+ if (IsShiftedRegister()) {
+ DCHECK(reg_.Is64Bits());
+ return Operand(reg_.W(), shift(), shift_amount());
+ } else if (IsExtendedRegister()) {
+ DCHECK(reg_.Is64Bits());
+ return Operand(reg_.W(), extend(), shift_amount());
+ }
+ DCHECK(IsImmediate());
+ return *this;
+}
+
+Immediate Operand::immediate_for_heap_object_request() const {
+ DCHECK((heap_object_request().kind() == HeapObjectRequest::kHeapNumber &&
+ immediate_.rmode() == RelocInfo::FULL_EMBEDDED_OBJECT) ||
+ (heap_object_request().kind() == HeapObjectRequest::kStringConstant &&
+ immediate_.rmode() == RelocInfo::FULL_EMBEDDED_OBJECT));
+ return immediate_;
+}
+
+Immediate Operand::immediate() const {
+ DCHECK(IsImmediate());
+ return immediate_;
+}
+
+int64_t Operand::ImmediateValue() const {
+ DCHECK(IsImmediate());
+ return immediate_.value();
+}
+
+RelocInfo::Mode Operand::ImmediateRMode() const {
+ DCHECK(IsImmediate() || IsHeapObjectRequest());
+ return immediate_.rmode();
+}
+
+Register Operand::reg() const {
+ DCHECK(IsShiftedRegister() || IsExtendedRegister());
+ return reg_;
+}
+
+Shift Operand::shift() const {
+ DCHECK(IsShiftedRegister());
+ return shift_;
+}
+
+Extend Operand::extend() const {
+ DCHECK(IsExtendedRegister());
+ return extend_;
+}
+
+unsigned Operand::shift_amount() const {
+ DCHECK(IsShiftedRegister() || IsExtendedRegister());
+ return shift_amount_;
+}
+
+MemOperand::MemOperand()
+ : base_(NoReg),
+ regoffset_(NoReg),
+ offset_(0),
+ addrmode_(Offset),
+ shift_(NO_SHIFT),
+ extend_(NO_EXTEND),
+ shift_amount_(0) {}
+
+MemOperand::MemOperand(Register base, int64_t offset, AddrMode addrmode)
+ : base_(base),
+ regoffset_(NoReg),
+ offset_(offset),
+ addrmode_(addrmode),
+ shift_(NO_SHIFT),
+ extend_(NO_EXTEND),
+ shift_amount_(0) {
+ DCHECK(base.Is64Bits() && !base.IsZero());
+}
+
+MemOperand::MemOperand(Register base, Register regoffset, Extend extend,
+ unsigned shift_amount)
+ : base_(base),
+ regoffset_(regoffset),
+ offset_(0),
+ addrmode_(Offset),
+ shift_(NO_SHIFT),
+ extend_(extend),
+ shift_amount_(shift_amount) {
+ DCHECK(base.Is64Bits() && !base.IsZero());
+ DCHECK(!regoffset.IsSP());
+ DCHECK((extend == UXTW) || (extend == SXTW) || (extend == SXTX));
+
+ // SXTX extend mode requires a 64-bit offset register.
+ DCHECK(regoffset.Is64Bits() || (extend != SXTX));
+}
+
+MemOperand::MemOperand(Register base, Register regoffset, Shift shift,
+ unsigned shift_amount)
+ : base_(base),
+ regoffset_(regoffset),
+ offset_(0),
+ addrmode_(Offset),
+ shift_(shift),
+ extend_(NO_EXTEND),
+ shift_amount_(shift_amount) {
+ DCHECK(base.Is64Bits() && !base.IsZero());
+ DCHECK(regoffset.Is64Bits() && !regoffset.IsSP());
+ DCHECK(shift == LSL);
+}
+
+MemOperand::MemOperand(Register base, const Operand& offset, AddrMode addrmode)
+ : base_(base), regoffset_(NoReg), addrmode_(addrmode) {
+ DCHECK(base.Is64Bits() && !base.IsZero());
+
+ if (offset.IsImmediate()) {
+ offset_ = offset.ImmediateValue();
+ } else if (offset.IsShiftedRegister()) {
+ DCHECK((addrmode == Offset) || (addrmode == PostIndex));
+
+ regoffset_ = offset.reg();
+ shift_ = offset.shift();
+ shift_amount_ = offset.shift_amount();
+
+ extend_ = NO_EXTEND;
+ offset_ = 0;
+
+ // These assertions match those in the shifted-register constructor.
+ DCHECK(regoffset_.Is64Bits() && !regoffset_.IsSP());
+ DCHECK(shift_ == LSL);
+ } else {
+ DCHECK(offset.IsExtendedRegister());
+ DCHECK(addrmode == Offset);
+
+ regoffset_ = offset.reg();
+ extend_ = offset.extend();
+ shift_amount_ = offset.shift_amount();
+
+ shift_ = NO_SHIFT;
+ offset_ = 0;
+
+ // These assertions match those in the extended-register constructor.
+ DCHECK(!regoffset_.IsSP());
+ DCHECK((extend_ == UXTW) || (extend_ == SXTW) || (extend_ == SXTX));
+ DCHECK((regoffset_.Is64Bits() || (extend_ != SXTX)));
+ }
+}
+
+bool MemOperand::IsImmediateOffset() const {
+ return (addrmode_ == Offset) && regoffset_ == NoReg;
+}
+
+bool MemOperand::IsRegisterOffset() const {
+ return (addrmode_ == Offset) && regoffset_ != NoReg;
+}
+
+bool MemOperand::IsPreIndex() const { return addrmode_ == PreIndex; }
+
+bool MemOperand::IsPostIndex() const { return addrmode_ == PostIndex; }
+
+void Assembler::Unreachable() { debug("UNREACHABLE", __LINE__, BREAK); }
+
+Address Assembler::target_pointer_address_at(Address pc) {
+ Instruction* instr = reinterpret_cast<Instruction*>(pc);
+ DCHECK(instr->IsLdrLiteralX() || instr->IsLdrLiteralW());
+ return reinterpret_cast<Address>(instr->ImmPCOffsetTarget());
+}
+
+// Read/Modify the code target address in the branch/call instruction at pc.
+Address Assembler::target_address_at(Address pc, Address constant_pool) {
+ Instruction* instr = reinterpret_cast<Instruction*>(pc);
+ if (instr->IsLdrLiteralX()) {
+ return Memory<Address>(target_pointer_address_at(pc));
+ } else {
+ DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch());
+ return reinterpret_cast<Address>(instr->ImmPCOffsetTarget());
+ }
+}
+
+Tagged_t Assembler::target_compressed_address_at(Address pc,
+ Address constant_pool) {
+ Instruction* instr = reinterpret_cast<Instruction*>(pc);
+ CHECK(instr->IsLdrLiteralW());
+ return Memory<Tagged_t>(target_pointer_address_at(pc));
+}
+
+Handle<Code> Assembler::code_target_object_handle_at(Address pc) {
+ Instruction* instr = reinterpret_cast<Instruction*>(pc);
+ if (instr->IsLdrLiteralX()) {
+ return Handle<Code>(reinterpret_cast<Address*>(
+ Assembler::target_address_at(pc, 0 /* unused */)));
+ } else {
+ DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch());
+ DCHECK_EQ(instr->ImmPCOffset() % kInstrSize, 0);
+ return Handle<Code>::cast(
+ GetEmbeddedObject(instr->ImmPCOffset() >> kInstrSizeLog2));
+ }
+}
+
+AssemblerBase::EmbeddedObjectIndex
+Assembler::embedded_object_index_referenced_from(Address pc) {
+ Instruction* instr = reinterpret_cast<Instruction*>(pc);
+ if (instr->IsLdrLiteralX()) {
+ STATIC_ASSERT(sizeof(EmbeddedObjectIndex) == sizeof(intptr_t));
+ return Memory<EmbeddedObjectIndex>(target_pointer_address_at(pc));
+ } else {
+ DCHECK(instr->IsLdrLiteralW());
+ return Memory<uint32_t>(target_pointer_address_at(pc));
+ }
+}
+
+void Assembler::set_embedded_object_index_referenced_from(
+ Address pc, EmbeddedObjectIndex data) {
+ Instruction* instr = reinterpret_cast<Instruction*>(pc);
+ if (instr->IsLdrLiteralX()) {
+ Memory<EmbeddedObjectIndex>(target_pointer_address_at(pc)) = data;
+ } else {
+ DCHECK(instr->IsLdrLiteralW());
+ DCHECK(is_uint32(data));
+ WriteUnalignedValue<uint32_t>(target_pointer_address_at(pc),
+ static_cast<uint32_t>(data));
+ }
+}
+
+Handle<HeapObject> Assembler::target_object_handle_at(Address pc) {
+ return GetEmbeddedObject(
+ Assembler::embedded_object_index_referenced_from(pc));
+}
+
+Address Assembler::runtime_entry_at(Address pc) {
+ Instruction* instr = reinterpret_cast<Instruction*>(pc);
+ if (instr->IsLdrLiteralX()) {
+ return Assembler::target_address_at(pc, 0 /* unused */);
+ } else {
+ DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch());
+ return instr->ImmPCOffset() + options().code_range_start;
+ }
+}
+
+int Assembler::deserialization_special_target_size(Address location) {
+ Instruction* instr = reinterpret_cast<Instruction*>(location);
+ if (instr->IsBranchAndLink() || instr->IsUnconditionalBranch()) {
+ return kSpecialTargetSize;
+ } else {
+ DCHECK_EQ(instr->InstructionBits(), 0);
+ return kSystemPointerSize;
+ }
+}
+
+void Assembler::deserialization_set_special_target_at(Address location,
+ Code code,
+ Address target) {
+ Instruction* instr = reinterpret_cast<Instruction*>(location);
+ if (instr->IsBranchAndLink() || instr->IsUnconditionalBranch()) {
+ if (target == 0) {
+ // We are simply wiping the target out for serialization. Set the offset
+ // to zero instead.
+ target = location;
+ }
+ instr->SetBranchImmTarget(reinterpret_cast<Instruction*>(target));
+ FlushInstructionCache(location, kInstrSize);
+ } else {
+ DCHECK_EQ(instr->InstructionBits(), 0);
+ Memory<Address>(location) = target;
+ // Intuitively, we would think it is necessary to always flush the
+ // instruction cache after patching a target address in the code. However,
+ // in this case, only the constant pool contents change. The instruction
+ // accessing the constant pool remains unchanged, so a flush is not
+ // required.
+ }
+}
+
+void Assembler::deserialization_set_target_internal_reference_at(
+ Address pc, Address target, RelocInfo::Mode mode) {
+ WriteUnalignedValue<Address>(pc, target);
+}
+
+void Assembler::set_target_address_at(Address pc, Address constant_pool,
+ Address target,
+ ICacheFlushMode icache_flush_mode) {
+ Instruction* instr = reinterpret_cast<Instruction*>(pc);
+ if (instr->IsLdrLiteralX()) {
+ Memory<Address>(target_pointer_address_at(pc)) = target;
+ // Intuitively, we would think it is necessary to always flush the
+ // instruction cache after patching a target address in the code. However,
+ // in this case, only the constant pool contents change. The instruction
+ // accessing the constant pool remains unchanged, so a flush is not
+ // required.
+ } else {
+ DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch());
+ if (target == 0) {
+ // We are simply wiping the target out for serialization. Set the offset
+ // to zero instead.
+ target = pc;
+ }
+ instr->SetBranchImmTarget(reinterpret_cast<Instruction*>(target));
+ if (icache_flush_mode != SKIP_ICACHE_FLUSH) {
+ FlushInstructionCache(pc, kInstrSize);
+ }
+ }
+}
+
+void Assembler::set_target_compressed_address_at(
+ Address pc, Address constant_pool, Tagged_t target,
+ ICacheFlushMode icache_flush_mode) {
+ Instruction* instr = reinterpret_cast<Instruction*>(pc);
+ CHECK(instr->IsLdrLiteralW());
+ Memory<Tagged_t>(target_pointer_address_at(pc)) = target;
+}
+
+int RelocInfo::target_address_size() {
+ if (IsCodedSpecially()) {
+ return Assembler::kSpecialTargetSize;
+ } else {
+ Instruction* instr = reinterpret_cast<Instruction*>(pc_);
+ DCHECK(instr->IsLdrLiteralX() || instr->IsLdrLiteralW());
+ return instr->IsLdrLiteralW() ? kTaggedSize : kSystemPointerSize;
+ }
+}
+
+Address RelocInfo::target_address() {
+ DCHECK(IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_) || IsWasmCall(rmode_));
+ return Assembler::target_address_at(pc_, constant_pool_);
+}
+
+Address RelocInfo::target_address_address() {
+ DCHECK(HasTargetAddressAddress());
+ Instruction* instr = reinterpret_cast<Instruction*>(pc_);
+ // Read the address of the word containing the target_address in an
+ // instruction stream.
+ // The only architecture-independent user of this function is the serializer.
+ // The serializer uses it to find out how many raw bytes of instruction to
+ // output before the next target.
+ // For an instruction like B/BL, where the target bits are mixed into the
+ // instruction bits, the size of the target will be zero, indicating that the
+ // serializer should not step forward in memory after a target is resolved
+ // and written.
+ // For LDR literal instructions, we can skip up to the constant pool entry
+ // address. We make sure that RelocInfo is ordered by the
+ // target_address_address so that we do not skip over any relocatable
+ // instruction sequences.
+ if (instr->IsLdrLiteralX()) {
+ return constant_pool_entry_address();
+ } else {
+ DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch());
+ return pc_;
+ }
+}
+
+Address RelocInfo::constant_pool_entry_address() {
+ DCHECK(IsInConstantPool());
+ return Assembler::target_pointer_address_at(pc_);
+}
+
+HeapObject RelocInfo::target_object() {
+ DCHECK(IsCodeTarget(rmode_) || IsEmbeddedObjectMode(rmode_));
+ if (IsCompressedEmbeddedObject(rmode_)) {
+ CHECK(!host_.is_null());
+ return HeapObject::cast(Object(DecompressTaggedAny(
+ host_.address(),
+ Assembler::target_compressed_address_at(pc_, constant_pool_))));
+ } else {
+ return HeapObject::cast(
+ Object(Assembler::target_address_at(pc_, constant_pool_)));
+ }
+}
+
+HeapObject RelocInfo::target_object_no_host(Isolate* isolate) {
+ if (IsCompressedEmbeddedObject(rmode_)) {
+ return HeapObject::cast(Object(DecompressTaggedAny(
+ isolate,
+ Assembler::target_compressed_address_at(pc_, constant_pool_))));
+ } else {
+ return target_object();
+ }
+}
+
+Handle<HeapObject> RelocInfo::target_object_handle(Assembler* origin) {
+ if (IsEmbeddedObjectMode(rmode_)) {
+ return origin->target_object_handle_at(pc_);
+ } else {
+ DCHECK(IsCodeTarget(rmode_));
+ return origin->code_target_object_handle_at(pc_);
+ }
+}
+
+void RelocInfo::set_target_object(Heap* heap, HeapObject target,
+ WriteBarrierMode write_barrier_mode,
+ ICacheFlushMode icache_flush_mode) {
+ DCHECK(IsCodeTarget(rmode_) || IsEmbeddedObjectMode(rmode_));
+ if (IsCompressedEmbeddedObject(rmode_)) {
+ Assembler::set_target_compressed_address_at(
+ pc_, constant_pool_, CompressTagged(target.ptr()), icache_flush_mode);
+ } else {
+ DCHECK(IsFullEmbeddedObject(rmode_));
+ Assembler::set_target_address_at(pc_, constant_pool_, target.ptr(),
+ icache_flush_mode);
+ }
+ if (write_barrier_mode == UPDATE_WRITE_BARRIER && !host().is_null() &&
+ !FLAG_disable_write_barriers) {
+ WriteBarrierForCode(host(), this, target);
+ }
+}
+
+Address RelocInfo::target_external_reference() {
+ DCHECK(rmode_ == EXTERNAL_REFERENCE);
+ return Assembler::target_address_at(pc_, constant_pool_);
+}
+
+void RelocInfo::set_target_external_reference(
+ Address target, ICacheFlushMode icache_flush_mode) {
+ DCHECK(rmode_ == RelocInfo::EXTERNAL_REFERENCE);
+ Assembler::set_target_address_at(pc_, constant_pool_, target,
+ icache_flush_mode);
+}
+
+Address RelocInfo::target_internal_reference() {
+ DCHECK(rmode_ == INTERNAL_REFERENCE);
+ return ReadUnalignedValue<Address>(pc_);
+}
+
+Address RelocInfo::target_internal_reference_address() {
+ DCHECK(rmode_ == INTERNAL_REFERENCE);
+ return pc_;
+}
+
+Address RelocInfo::target_runtime_entry(Assembler* origin) {
+ DCHECK(IsRuntimeEntry(rmode_));
+ return origin->runtime_entry_at(pc_);
+}
+
+void RelocInfo::set_target_runtime_entry(Address target,
+ WriteBarrierMode write_barrier_mode,
+ ICacheFlushMode icache_flush_mode) {
+ DCHECK(IsRuntimeEntry(rmode_));
+ if (target_address() != target) {
+ set_target_address(target, write_barrier_mode, icache_flush_mode);
+ }
+}
+
+Address RelocInfo::target_off_heap_target() {
+ DCHECK(IsOffHeapTarget(rmode_));
+ return Assembler::target_address_at(pc_, constant_pool_);
+}
+
+void RelocInfo::WipeOut() {
+ DCHECK(IsEmbeddedObjectMode(rmode_) || IsCodeTarget(rmode_) ||
+ IsRuntimeEntry(rmode_) || IsExternalReference(rmode_) ||
+ IsInternalReference(rmode_) || IsOffHeapTarget(rmode_));
+ if (IsInternalReference(rmode_)) {
+ WriteUnalignedValue<Address>(pc_, kNullAddress);
+ } else if (IsCompressedEmbeddedObject(rmode_)) {
+ Assembler::set_target_compressed_address_at(pc_, constant_pool_,
+ kNullAddress);
+ } else {
+ Assembler::set_target_address_at(pc_, constant_pool_, kNullAddress);
+ }
+}
+
+LoadStoreOp Assembler::LoadOpFor(const CPURegister& rt) {
+ DCHECK(rt.is_valid());
+ if (rt.IsRegister()) {
+ return rt.Is64Bits() ? LDR_x : LDR_w;
+ } else {
+ DCHECK(rt.IsVRegister());
+ switch (rt.SizeInBits()) {
+ case kBRegSizeInBits:
+ return LDR_b;
+ case kHRegSizeInBits:
+ return LDR_h;
+ case kSRegSizeInBits:
+ return LDR_s;
+ case kDRegSizeInBits:
+ return LDR_d;
+ default:
+ DCHECK(rt.IsQ());
+ return LDR_q;
+ }
+ }
+}
+
+LoadStoreOp Assembler::StoreOpFor(const CPURegister& rt) {
+ DCHECK(rt.is_valid());
+ if (rt.IsRegister()) {
+ return rt.Is64Bits() ? STR_x : STR_w;
+ } else {
+ DCHECK(rt.IsVRegister());
+ switch (rt.SizeInBits()) {
+ case kBRegSizeInBits:
+ return STR_b;
+ case kHRegSizeInBits:
+ return STR_h;
+ case kSRegSizeInBits:
+ return STR_s;
+ case kDRegSizeInBits:
+ return STR_d;
+ default:
+ DCHECK(rt.IsQ());
+ return STR_q;
+ }
+ }
+}
+
+LoadStorePairOp Assembler::LoadPairOpFor(const CPURegister& rt,
+ const CPURegister& rt2) {
+ DCHECK_EQ(STP_w | LoadStorePairLBit, LDP_w);
+ return static_cast<LoadStorePairOp>(StorePairOpFor(rt, rt2) |
+ LoadStorePairLBit);
+}
+
+LoadStorePairOp Assembler::StorePairOpFor(const CPURegister& rt,
+ const CPURegister& rt2) {
+ DCHECK(AreSameSizeAndType(rt, rt2));
+ USE(rt2);
+ if (rt.IsRegister()) {
+ return rt.Is64Bits() ? STP_x : STP_w;
+ } else {
+ DCHECK(rt.IsVRegister());
+ switch (rt.SizeInBits()) {
+ case kSRegSizeInBits:
+ return STP_s;
+ case kDRegSizeInBits:
+ return STP_d;
+ default:
+ DCHECK(rt.IsQ());
+ return STP_q;
+ }
+ }
+}
+
+LoadLiteralOp Assembler::LoadLiteralOpFor(const CPURegister& rt) {
+ if (rt.IsRegister()) {
+ return rt.Is64Bits() ? LDR_x_lit : LDR_w_lit;
+ } else {
+ DCHECK(rt.IsVRegister());
+ return rt.Is64Bits() ? LDR_d_lit : LDR_s_lit;
+ }
+}
+
+int Assembler::LinkAndGetInstructionOffsetTo(Label* label) {
+ DCHECK_EQ(kStartOfLabelLinkChain, 0);
+ int offset = LinkAndGetByteOffsetTo(label);
+ DCHECK(IsAligned(offset, kInstrSize));
+ return offset >> kInstrSizeLog2;
+}
+
+Instr Assembler::Flags(FlagsUpdate S) {
+ if (S == SetFlags) {
+ return 1 << FlagsUpdate_offset;
+ } else if (S == LeaveFlags) {
+ return 0 << FlagsUpdate_offset;
+ }
+ UNREACHABLE();
+}
+
+Instr Assembler::Cond(Condition cond) { return cond << Condition_offset; }
+
+Instr Assembler::ImmPCRelAddress(int imm21) {
+ CHECK(is_int21(imm21));
+ Instr imm = static_cast<Instr>(truncate_to_int21(imm21));
+ Instr immhi = (imm >> ImmPCRelLo_width) << ImmPCRelHi_offset;
+ Instr immlo = imm << ImmPCRelLo_offset;
+ return (immhi & ImmPCRelHi_mask) | (immlo & ImmPCRelLo_mask);
+}
+
+Instr Assembler::ImmUncondBranch(int imm26) {
+ CHECK(is_int26(imm26));
+ return truncate_to_int26(imm26) << ImmUncondBranch_offset;
+}
+
+Instr Assembler::ImmCondBranch(int imm19) {
+ CHECK(is_int19(imm19));
+ return truncate_to_int19(imm19) << ImmCondBranch_offset;
+}
+
+Instr Assembler::ImmCmpBranch(int imm19) {
+ CHECK(is_int19(imm19));
+ return truncate_to_int19(imm19) << ImmCmpBranch_offset;
+}
+
+Instr Assembler::ImmTestBranch(int imm14) {
+ CHECK(is_int14(imm14));
+ return truncate_to_int14(imm14) << ImmTestBranch_offset;
+}
+
+Instr Assembler::ImmTestBranchBit(unsigned bit_pos) {
+ DCHECK(is_uint6(bit_pos));
+ // Subtract five from the shift offset, as we need bit 5 from bit_pos.
+ unsigned b5 = bit_pos << (ImmTestBranchBit5_offset - 5);
+ unsigned b40 = bit_pos << ImmTestBranchBit40_offset;
+ b5 &= ImmTestBranchBit5_mask;
+ b40 &= ImmTestBranchBit40_mask;
+ return b5 | b40;
+}
+
+Instr Assembler::SF(Register rd) {
+ return rd.Is64Bits() ? SixtyFourBits : ThirtyTwoBits;
+}
+
+Instr Assembler::ImmAddSub(int imm) {
+ DCHECK(IsImmAddSub(imm));
+ if (is_uint12(imm)) { // No shift required.
+ imm <<= ImmAddSub_offset;
+ } else {
+ imm = ((imm >> 12) << ImmAddSub_offset) | (1 << ShiftAddSub_offset);
+ }
+ return imm;
+}
+
+Instr Assembler::ImmS(unsigned imms, unsigned reg_size) {
+ DCHECK(((reg_size == kXRegSizeInBits) && is_uint6(imms)) ||
+ ((reg_size == kWRegSizeInBits) && is_uint5(imms)));
+ USE(reg_size);
+ return imms << ImmS_offset;
+}
+
+Instr Assembler::ImmR(unsigned immr, unsigned reg_size) {
+ DCHECK(((reg_size == kXRegSizeInBits) && is_uint6(immr)) ||
+ ((reg_size == kWRegSizeInBits) && is_uint5(immr)));
+ USE(reg_size);
+ DCHECK(is_uint6(immr));
+ return immr << ImmR_offset;
+}
+
+Instr Assembler::ImmSetBits(unsigned imms, unsigned reg_size) {
+ DCHECK((reg_size == kWRegSizeInBits) || (reg_size == kXRegSizeInBits));
+ DCHECK(is_uint6(imms));
+ DCHECK((reg_size == kXRegSizeInBits) || is_uint6(imms + 3));
+ USE(reg_size);
+ return imms << ImmSetBits_offset;
+}
+
+Instr Assembler::ImmRotate(unsigned immr, unsigned reg_size) {
+ DCHECK((reg_size == kWRegSizeInBits) || (reg_size == kXRegSizeInBits));
+ DCHECK(((reg_size == kXRegSizeInBits) && is_uint6(immr)) ||
+ ((reg_size == kWRegSizeInBits) && is_uint5(immr)));
+ USE(reg_size);
+ return immr << ImmRotate_offset;
+}
+
+Instr Assembler::ImmLLiteral(int imm19) {
+ CHECK(is_int19(imm19));
+ return truncate_to_int19(imm19) << ImmLLiteral_offset;
+}
+
+Instr Assembler::BitN(unsigned bitn, unsigned reg_size) {
+ DCHECK((reg_size == kWRegSizeInBits) || (reg_size == kXRegSizeInBits));
+ DCHECK((reg_size == kXRegSizeInBits) || (bitn == 0));
+ USE(reg_size);
+ return bitn << BitN_offset;
+}
+
+Instr Assembler::ShiftDP(Shift shift) {
+ DCHECK(shift == LSL || shift == LSR || shift == ASR || shift == ROR);
+ return shift << ShiftDP_offset;
+}
+
+Instr Assembler::ImmDPShift(unsigned amount) {
+ DCHECK(is_uint6(amount));
+ return amount << ImmDPShift_offset;
+}
+
+Instr Assembler::ExtendMode(Extend extend) {
+ return extend << ExtendMode_offset;
+}
+
+Instr Assembler::ImmExtendShift(unsigned left_shift) {
+ DCHECK_LE(left_shift, 4);
+ return left_shift << ImmExtendShift_offset;
+}
+
+Instr Assembler::ImmCondCmp(unsigned imm) {
+ DCHECK(is_uint5(imm));
+ return imm << ImmCondCmp_offset;
+}
+
+Instr Assembler::Nzcv(StatusFlags nzcv) {
+ return ((nzcv >> Flags_offset) & 0xf) << Nzcv_offset;
+}
+
+Instr Assembler::ImmLSUnsigned(int imm12) {
+ DCHECK(is_uint12(imm12));
+ return imm12 << ImmLSUnsigned_offset;
+}
+
+Instr Assembler::ImmLS(int imm9) {
+ DCHECK(is_int9(imm9));
+ return truncate_to_int9(imm9) << ImmLS_offset;
+}
+
+Instr Assembler::ImmLSPair(int imm7, unsigned size) {
+ DCHECK_EQ(imm7,
+ static_cast<int>(static_cast<uint32_t>(imm7 >> size) << size));
+ int scaled_imm7 = imm7 >> size;
+ DCHECK(is_int7(scaled_imm7));
+ return truncate_to_int7(scaled_imm7) << ImmLSPair_offset;
+}
+
+Instr Assembler::ImmShiftLS(unsigned shift_amount) {
+ DCHECK(is_uint1(shift_amount));
+ return shift_amount << ImmShiftLS_offset;
+}
+
+Instr Assembler::ImmException(int imm16) {
+ DCHECK(is_uint16(imm16));
+ return imm16 << ImmException_offset;
+}
+
+Instr Assembler::ImmSystemRegister(int imm15) {
+ DCHECK(is_uint15(imm15));
+ return imm15 << ImmSystemRegister_offset;
+}
+
+Instr Assembler::ImmHint(int imm7) {
+ DCHECK(is_uint7(imm7));
+ return imm7 << ImmHint_offset;
+}
+
+Instr Assembler::ImmBarrierDomain(int imm2) {
+ DCHECK(is_uint2(imm2));
+ return imm2 << ImmBarrierDomain_offset;
+}
+
+Instr Assembler::ImmBarrierType(int imm2) {
+ DCHECK(is_uint2(imm2));
+ return imm2 << ImmBarrierType_offset;
+}
+
+unsigned Assembler::CalcLSDataSize(LoadStoreOp op) {
+ DCHECK((LSSize_offset + LSSize_width) == (kInstrSize * 8));
+ unsigned size = static_cast<Instr>(op >> LSSize_offset);
+ if ((op & LSVector_mask) != 0) {
+ // Vector register memory operations encode the access size in the "size"
+ // and "opc" fields.
+ if ((size == 0) && ((op & LSOpc_mask) >> LSOpc_offset) >= 2) {
+ size = kQRegSizeLog2;
+ }
+ }
+ return size;
+}
+
+Instr Assembler::ImmMoveWide(int imm) {
+ DCHECK(is_uint16(imm));
+ return imm << ImmMoveWide_offset;
+}
+
+Instr Assembler::ShiftMoveWide(int shift) {
+ DCHECK(is_uint2(shift));
+ return shift << ShiftMoveWide_offset;
+}
+
+Instr Assembler::FPType(VRegister fd) { return fd.Is64Bits() ? FP64 : FP32; }
+
+Instr Assembler::FPScale(unsigned scale) {
+ DCHECK(is_uint6(scale));
+ return scale << FPScale_offset;
+}
+
+const Register& Assembler::AppropriateZeroRegFor(const CPURegister& reg) const {
+ return reg.Is64Bits() ? xzr : wzr;
+}
+
+inline void Assembler::CheckBufferSpace() {
+ DCHECK_LT(pc_, buffer_start_ + buffer_->size());
+ if (buffer_space() < kGap) {
+ GrowBuffer();
+ }
+}
+
+inline void Assembler::CheckBuffer() {
+ CheckBufferSpace();
+ if (pc_offset() >= next_veneer_pool_check_) {
+ CheckVeneerPool(false, true);
+ }
+ constpool_.MaybeCheck();
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_ARM64_ASSEMBLER_ARM64_INL_H_
diff --git a/src/codegen/arm64/assembler-arm64.cc b/src/codegen/arm64/assembler-arm64.cc
new file mode 100644
index 0000000..4aaa413
--- /dev/null
+++ b/src/codegen/arm64/assembler-arm64.cc
@@ -0,0 +1,4640 @@
+// Copyright 2013 the V8 project authors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following
+// disclaimer in the documentation and/or other materials provided
+// with the distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived
+// from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#if V8_TARGET_ARCH_ARM64
+
+#include "src/codegen/arm64/assembler-arm64.h"
+
+#include "src/base/bits.h"
+#include "src/base/cpu.h"
+#include "src/codegen/arm64/assembler-arm64-inl.h"
+#include "src/codegen/register-configuration.h"
+#include "src/codegen/safepoint-table.h"
+#include "src/codegen/string-constants.h"
+#include "src/execution/frame-constants.h"
+
+namespace v8 {
+namespace internal {
+
+namespace {
+
+#ifdef USE_SIMULATOR
+static unsigned SimulatorFeaturesFromCommandLine() {
+ if (strcmp(FLAG_sim_arm64_optional_features, "none") == 0) {
+ return 0;
+ }
+ if (strcmp(FLAG_sim_arm64_optional_features, "all") == 0) {
+ return (1u << NUMBER_OF_CPU_FEATURES) - 1;
+ }
+ fprintf(
+ stderr,
+ "Error: unrecognised value for --sim-arm64-optional-features ('%s').\n",
+ FLAG_sim_arm64_optional_features);
+ fprintf(stderr,
+ "Supported values are: none\n"
+ " all\n");
+ FATAL("sim-arm64-optional-features");
+}
+#endif // USE_SIMULATOR
+
+static constexpr unsigned CpuFeaturesFromCompiler() {
+ unsigned features = 0;
+#if defined(__ARM_FEATURE_JCVT)
+ features |= 1u << JSCVT;
+#endif
+ return features;
+}
+
+} // namespace
+
+// -----------------------------------------------------------------------------
+// CpuFeatures implementation.
+
+void CpuFeatures::ProbeImpl(bool cross_compile) {
+ // Only use statically determined features for cross compile (snapshot).
+ if (cross_compile) {
+ supported_ |= CpuFeaturesFromCompiler();
+ return;
+ }
+
+ // We used to probe for coherent cache support, but on older CPUs it
+ // causes crashes (crbug.com/524337), and newer CPUs don't even have
+ // the feature any more.
+
+#ifdef USE_SIMULATOR
+ supported_ |= SimulatorFeaturesFromCommandLine();
+#else
+ // Probe for additional features at runtime.
+ base::CPU cpu;
+ unsigned runtime = 0;
+ if (cpu.has_jscvt()) {
+ runtime |= 1u << JSCVT;
+ }
+
+ // Use the best of the features found by CPU detection and those inferred from
+ // the build system.
+ supported_ |= CpuFeaturesFromCompiler();
+ supported_ |= runtime;
+#endif // USE_SIMULATOR
+}
+
+void CpuFeatures::PrintTarget() {}
+void CpuFeatures::PrintFeatures() {}
+
+// -----------------------------------------------------------------------------
+// CPURegList utilities.
+
+CPURegister CPURegList::PopLowestIndex() {
+ if (IsEmpty()) {
+ return NoCPUReg;
+ }
+ int index = base::bits::CountTrailingZeros(list_);
+ DCHECK((1LL << index) & list_);
+ Remove(index);
+ return CPURegister::Create(index, size_, type_);
+}
+
+CPURegister CPURegList::PopHighestIndex() {
+ if (IsEmpty()) {
+ return NoCPUReg;
+ }
+ int index = CountLeadingZeros(list_, kRegListSizeInBits);
+ index = kRegListSizeInBits - 1 - index;
+ DCHECK((1LL << index) & list_);
+ Remove(index);
+ return CPURegister::Create(index, size_, type_);
+}
+
+void CPURegList::Align() {
+ // Use padreg, if necessary, to maintain stack alignment.
+ if (Count() % 2 != 0) {
+ if (IncludesAliasOf(padreg)) {
+ Remove(padreg);
+ } else {
+ Combine(padreg);
+ }
+ }
+
+ DCHECK_EQ(Count() % 2, 0);
+}
+
+CPURegList CPURegList::GetCalleeSaved(int size) {
+ return CPURegList(CPURegister::kRegister, size, 19, 28);
+}
+
+CPURegList CPURegList::GetCalleeSavedV(int size) {
+ return CPURegList(CPURegister::kVRegister, size, 8, 15);
+}
+
+CPURegList CPURegList::GetCallerSaved(int size) {
+ // x18 is the platform register and is reserved for the use of platform ABIs.
+ // Registers x0-x17 are caller-saved.
+ CPURegList list = CPURegList(CPURegister::kRegister, size, 0, 17);
+ return list;
+}
+
+CPURegList CPURegList::GetCallerSavedV(int size) {
+ // Registers d0-d7 and d16-d31 are caller-saved.
+ CPURegList list = CPURegList(CPURegister::kVRegister, size, 0, 7);
+ list.Combine(CPURegList(CPURegister::kVRegister, size, 16, 31));
+ return list;
+}
+
+// -----------------------------------------------------------------------------
+// Implementation of RelocInfo
+
+const int RelocInfo::kApplyMask =
+ RelocInfo::ModeMask(RelocInfo::CODE_TARGET) |
+ RelocInfo::ModeMask(RelocInfo::RUNTIME_ENTRY) |
+ RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE);
+
+bool RelocInfo::IsCodedSpecially() {
+ // The deserializer needs to know whether a pointer is specially coded. Being
+ // specially coded on ARM64 means that it is an immediate branch.
+ Instruction* instr = reinterpret_cast<Instruction*>(pc_);
+ if (instr->IsLdrLiteralX()) {
+ return false;
+ } else {
+ DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch());
+ return true;
+ }
+}
+
+bool RelocInfo::IsInConstantPool() {
+ Instruction* instr = reinterpret_cast<Instruction*>(pc_);
+ DCHECK_IMPLIES(instr->IsLdrLiteralW(), COMPRESS_POINTERS_BOOL);
+ return instr->IsLdrLiteralX() ||
+ (COMPRESS_POINTERS_BOOL && instr->IsLdrLiteralW());
+}
+
+uint32_t RelocInfo::wasm_call_tag() const {
+ DCHECK(rmode_ == WASM_CALL || rmode_ == WASM_STUB_CALL);
+ Instruction* instr = reinterpret_cast<Instruction*>(pc_);
+ if (instr->IsLdrLiteralX()) {
+ return static_cast<uint32_t>(
+ Memory<Address>(Assembler::target_pointer_address_at(pc_)));
+ } else {
+ DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch());
+ return static_cast<uint32_t>(instr->ImmPCOffset() / kInstrSize);
+ }
+}
+
+bool AreAliased(const CPURegister& reg1, const CPURegister& reg2,
+ const CPURegister& reg3, const CPURegister& reg4,
+ const CPURegister& reg5, const CPURegister& reg6,
+ const CPURegister& reg7, const CPURegister& reg8) {
+ int number_of_valid_regs = 0;
+ int number_of_valid_fpregs = 0;
+
+ RegList unique_regs = 0;
+ RegList unique_fpregs = 0;
+
+ const CPURegister regs[] = {reg1, reg2, reg3, reg4, reg5, reg6, reg7, reg8};
+
+ for (unsigned i = 0; i < arraysize(regs); i++) {
+ if (regs[i].IsRegister()) {
+ number_of_valid_regs++;
+ unique_regs |= regs[i].bit();
+ } else if (regs[i].IsVRegister()) {
+ number_of_valid_fpregs++;
+ unique_fpregs |= regs[i].bit();
+ } else {
+ DCHECK(!regs[i].is_valid());
+ }
+ }
+
+ int number_of_unique_regs =
+ CountSetBits(unique_regs, sizeof(unique_regs) * kBitsPerByte);
+ int number_of_unique_fpregs =
+ CountSetBits(unique_fpregs, sizeof(unique_fpregs) * kBitsPerByte);
+
+ DCHECK(number_of_valid_regs >= number_of_unique_regs);
+ DCHECK(number_of_valid_fpregs >= number_of_unique_fpregs);
+
+ return (number_of_valid_regs != number_of_unique_regs) ||
+ (number_of_valid_fpregs != number_of_unique_fpregs);
+}
+
+bool AreSameSizeAndType(const CPURegister& reg1, const CPURegister& reg2,
+ const CPURegister& reg3, const CPURegister& reg4,
+ const CPURegister& reg5, const CPURegister& reg6,
+ const CPURegister& reg7, const CPURegister& reg8) {
+ DCHECK(reg1.is_valid());
+ bool match = true;
+ match &= !reg2.is_valid() || reg2.IsSameSizeAndType(reg1);
+ match &= !reg3.is_valid() || reg3.IsSameSizeAndType(reg1);
+ match &= !reg4.is_valid() || reg4.IsSameSizeAndType(reg1);
+ match &= !reg5.is_valid() || reg5.IsSameSizeAndType(reg1);
+ match &= !reg6.is_valid() || reg6.IsSameSizeAndType(reg1);
+ match &= !reg7.is_valid() || reg7.IsSameSizeAndType(reg1);
+ match &= !reg8.is_valid() || reg8.IsSameSizeAndType(reg1);
+ return match;
+}
+
+bool AreSameFormat(const VRegister& reg1, const VRegister& reg2,
+ const VRegister& reg3, const VRegister& reg4) {
+ DCHECK(reg1.is_valid());
+ return (!reg2.is_valid() || reg2.IsSameFormat(reg1)) &&
+ (!reg3.is_valid() || reg3.IsSameFormat(reg1)) &&
+ (!reg4.is_valid() || reg4.IsSameFormat(reg1));
+}
+
+bool AreConsecutive(const VRegister& reg1, const VRegister& reg2,
+ const VRegister& reg3, const VRegister& reg4) {
+ DCHECK(reg1.is_valid());
+ if (!reg2.is_valid()) {
+ DCHECK(!reg3.is_valid() && !reg4.is_valid());
+ return true;
+ } else if (reg2.code() != ((reg1.code() + 1) % kNumberOfVRegisters)) {
+ return false;
+ }
+
+ if (!reg3.is_valid()) {
+ DCHECK(!reg4.is_valid());
+ return true;
+ } else if (reg3.code() != ((reg2.code() + 1) % kNumberOfVRegisters)) {
+ return false;
+ }
+
+ if (!reg4.is_valid()) {
+ return true;
+ } else if (reg4.code() != ((reg3.code() + 1) % kNumberOfVRegisters)) {
+ return false;
+ }
+
+ return true;
+}
+
+bool Operand::NeedsRelocation(const Assembler* assembler) const {
+ RelocInfo::Mode rmode = immediate_.rmode();
+
+ if (RelocInfo::IsOnlyForSerializer(rmode)) {
+ return assembler->options().record_reloc_info_for_serialization;
+ }
+
+ return !RelocInfo::IsNone(rmode);
+}
+
+// Assembler
+Assembler::Assembler(const AssemblerOptions& options,
+ std::unique_ptr<AssemblerBuffer> buffer)
+ : AssemblerBase(options, std::move(buffer)),
+ unresolved_branches_(),
+ constpool_(this) {
+ veneer_pool_blocked_nesting_ = 0;
+ Reset();
+
+#if defined(V8_OS_WIN)
+ if (options.collect_win64_unwind_info) {
+ xdata_encoder_ = std::make_unique<win64_unwindinfo::XdataEncoder>(*this);
+ }
+#endif
+}
+
+Assembler::~Assembler() {
+ DCHECK(constpool_.IsEmpty());
+ DCHECK_EQ(veneer_pool_blocked_nesting_, 0);
+}
+
+void Assembler::AbortedCodeGeneration() { constpool_.Clear(); }
+
+void Assembler::Reset() {
+#ifdef DEBUG
+ DCHECK((pc_ >= buffer_start_) && (pc_ < buffer_start_ + buffer_->size()));
+ DCHECK_EQ(veneer_pool_blocked_nesting_, 0);
+ DCHECK(unresolved_branches_.empty());
+ memset(buffer_start_, 0, pc_ - buffer_start_);
+#endif
+ pc_ = buffer_start_;
+ reloc_info_writer.Reposition(buffer_start_ + buffer_->size(), pc_);
+ constpool_.Clear();
+ next_veneer_pool_check_ = kMaxInt;
+}
+
+#if defined(V8_OS_WIN)
+win64_unwindinfo::BuiltinUnwindInfo Assembler::GetUnwindInfo() const {
+ DCHECK(options().collect_win64_unwind_info);
+ DCHECK_NOT_NULL(xdata_encoder_);
+ return xdata_encoder_->unwinding_info();
+}
+#endif
+
+void Assembler::AllocateAndInstallRequestedHeapObjects(Isolate* isolate) {
+ DCHECK_IMPLIES(isolate == nullptr, heap_object_requests_.empty());
+ for (auto& request : heap_object_requests_) {
+ Address pc = reinterpret_cast<Address>(buffer_start_) + request.offset();
+ switch (request.kind()) {
+ case HeapObjectRequest::kHeapNumber: {
+ Handle<HeapObject> object =
+ isolate->factory()->NewHeapNumber<AllocationType::kOld>(
+ request.heap_number());
+ EmbeddedObjectIndex index = AddEmbeddedObject(object);
+ set_embedded_object_index_referenced_from(pc, index);
+ break;
+ }
+ case HeapObjectRequest::kStringConstant: {
+ const StringConstantBase* str = request.string();
+ CHECK_NOT_NULL(str);
+ EmbeddedObjectIndex index =
+ AddEmbeddedObject(str->AllocateStringConstant(isolate));
+ set_embedded_object_index_referenced_from(pc, index);
+ break;
+ }
+ }
+ }
+}
+
+void Assembler::GetCode(Isolate* isolate, CodeDesc* desc,
+ SafepointTableBuilder* safepoint_table_builder,
+ int handler_table_offset) {
+ // As a crutch to avoid having to add manual Align calls wherever we use a
+ // raw workflow to create Code objects (mostly in tests), add another Align
+ // call here. It does no harm - the end of the Code object is aligned to the
+ // (larger) kCodeAlignment anyways.
+ // TODO(jgruber): Consider moving responsibility for proper alignment to
+ // metadata table builders (safepoint, handler, constant pool, code
+ // comments).
+ DataAlign(Code::kMetadataAlignment);
+
+ // Emit constant pool if necessary.
+ ForceConstantPoolEmissionWithoutJump();
+ DCHECK(constpool_.IsEmpty());
+
+ int code_comments_size = WriteCodeComments();
+
+ AllocateAndInstallRequestedHeapObjects(isolate);
+
+ // Set up code descriptor.
+ // TODO(jgruber): Reconsider how these offsets and sizes are maintained up to
+ // this point to make CodeDesc initialization less fiddly.
+
+ static constexpr int kConstantPoolSize = 0;
+ const int instruction_size = pc_offset();
+ const int code_comments_offset = instruction_size - code_comments_size;
+ const int constant_pool_offset = code_comments_offset - kConstantPoolSize;
+ const int handler_table_offset2 = (handler_table_offset == kNoHandlerTable)
+ ? constant_pool_offset
+ : handler_table_offset;
+ const int safepoint_table_offset =
+ (safepoint_table_builder == kNoSafepointTable)
+ ? handler_table_offset2
+ : safepoint_table_builder->GetCodeOffset();
+ const int reloc_info_offset =
+ static_cast<int>(reloc_info_writer.pos() - buffer_->start());
+ CodeDesc::Initialize(desc, this, safepoint_table_offset,
+ handler_table_offset2, constant_pool_offset,
+ code_comments_offset, reloc_info_offset);
+}
+
+void Assembler::Align(int m) {
+ // If not, the loop below won't terminate.
+ DCHECK(IsAligned(pc_offset(), kInstrSize));
+ DCHECK(m >= kInstrSize && base::bits::IsPowerOfTwo(m));
+ while ((pc_offset() & (m - 1)) != 0) {
+ nop();
+ }
+}
+
+void Assembler::CodeTargetAlign() {
+ // Preferred alignment of jump targets on some ARM chips.
+ Align(8);
+}
+
+void Assembler::CheckLabelLinkChain(Label const* label) {
+#ifdef DEBUG
+ if (label->is_linked()) {
+ static const int kMaxLinksToCheck = 64; // Avoid O(n2) behaviour.
+ int links_checked = 0;
+ int64_t linkoffset = label->pos();
+ bool end_of_chain = false;
+ while (!end_of_chain) {
+ if (++links_checked > kMaxLinksToCheck) break;
+ Instruction* link = InstructionAt(linkoffset);
+ int64_t linkpcoffset = link->ImmPCOffset();
+ int64_t prevlinkoffset = linkoffset + linkpcoffset;
+
+ end_of_chain = (linkoffset == prevlinkoffset);
+ linkoffset = linkoffset + linkpcoffset;
+ }
+ }
+#endif
+}
+
+void Assembler::RemoveBranchFromLabelLinkChain(Instruction* branch,
+ Label* label,
+ Instruction* label_veneer) {
+ DCHECK(label->is_linked());
+
+ CheckLabelLinkChain(label);
+
+ Instruction* link = InstructionAt(label->pos());
+ Instruction* prev_link = link;
+ Instruction* next_link;
+ bool end_of_chain = false;
+
+ while (link != branch && !end_of_chain) {
+ next_link = link->ImmPCOffsetTarget();
+ end_of_chain = (link == next_link);
+ prev_link = link;
+ link = next_link;
+ }
+
+ DCHECK(branch == link);
+ next_link = branch->ImmPCOffsetTarget();
+
+ if (branch == prev_link) {
+ // The branch is the first instruction in the chain.
+ if (branch == next_link) {
+ // It is also the last instruction in the chain, so it is the only branch
+ // currently referring to this label.
+ label->Unuse();
+ } else {
+ label->link_to(
+ static_cast<int>(reinterpret_cast<byte*>(next_link) - buffer_start_));
+ }
+
+ } else if (branch == next_link) {
+ // The branch is the last (but not also the first) instruction in the chain.
+ prev_link->SetImmPCOffsetTarget(options(), prev_link);
+
+ } else {
+ // The branch is in the middle of the chain.
+ if (prev_link->IsTargetInImmPCOffsetRange(next_link)) {
+ prev_link->SetImmPCOffsetTarget(options(), next_link);
+ } else if (label_veneer != nullptr) {
+ // Use the veneer for all previous links in the chain.
+ prev_link->SetImmPCOffsetTarget(options(), prev_link);
+
+ end_of_chain = false;
+ link = next_link;
+ while (!end_of_chain) {
+ next_link = link->ImmPCOffsetTarget();
+ end_of_chain = (link == next_link);
+ link->SetImmPCOffsetTarget(options(), label_veneer);
+ link = next_link;
+ }
+ } else {
+ // The assert below will fire.
+ // Some other work could be attempted to fix up the chain, but it would be
+ // rather complicated. If we crash here, we may want to consider using an
+ // other mechanism than a chain of branches.
+ //
+ // Note that this situation currently should not happen, as we always call
+ // this function with a veneer to the target label.
+ // However this could happen with a MacroAssembler in the following state:
+ // [previous code]
+ // B(label);
+ // [20KB code]
+ // Tbz(label); // First tbz. Pointing to unconditional branch.
+ // [20KB code]
+ // Tbz(label); // Second tbz. Pointing to the first tbz.
+ // [more code]
+ // and this function is called to remove the first tbz from the label link
+ // chain. Since tbz has a range of +-32KB, the second tbz cannot point to
+ // the unconditional branch.
+ CHECK(prev_link->IsTargetInImmPCOffsetRange(next_link));
+ UNREACHABLE();
+ }
+ }
+
+ CheckLabelLinkChain(label);
+}
+
+void Assembler::bind(Label* label) {
+ // Bind label to the address at pc_. All instructions (most likely branches)
+ // that are linked to this label will be updated to point to the newly-bound
+ // label.
+
+ DCHECK(!label->is_near_linked());
+ DCHECK(!label->is_bound());
+
+ DeleteUnresolvedBranchInfoForLabel(label);
+
+ // If the label is linked, the link chain looks something like this:
+ //
+ // |--I----I-------I-------L
+ // |---------------------->| pc_offset
+ // |-------------->| linkoffset = label->pos()
+ // |<------| link->ImmPCOffset()
+ // |------>| prevlinkoffset = linkoffset + link->ImmPCOffset()
+ //
+ // On each iteration, the last link is updated and then removed from the
+ // chain until only one remains. At that point, the label is bound.
+ //
+ // If the label is not linked, no preparation is required before binding.
+ while (label->is_linked()) {
+ int linkoffset = label->pos();
+ Instruction* link = InstructionAt(linkoffset);
+ int prevlinkoffset = linkoffset + static_cast<int>(link->ImmPCOffset());
+
+ CheckLabelLinkChain(label);
+
+ DCHECK_GE(linkoffset, 0);
+ DCHECK(linkoffset < pc_offset());
+ DCHECK((linkoffset > prevlinkoffset) ||
+ (linkoffset - prevlinkoffset == kStartOfLabelLinkChain));
+ DCHECK_GE(prevlinkoffset, 0);
+
+ // Update the link to point to the label.
+ if (link->IsUnresolvedInternalReference()) {
+ // Internal references do not get patched to an instruction but directly
+ // to an address.
+ internal_reference_positions_.push_back(linkoffset);
+ PatchingAssembler patcher(options(), reinterpret_cast<byte*>(link), 2);
+ patcher.dc64(reinterpret_cast<uintptr_t>(pc_));
+ } else {
+ link->SetImmPCOffsetTarget(options(),
+ reinterpret_cast<Instruction*>(pc_));
+ }
+
+ // Link the label to the previous link in the chain.
+ if (linkoffset - prevlinkoffset == kStartOfLabelLinkChain) {
+ // We hit kStartOfLabelLinkChain, so the chain is fully processed.
+ label->Unuse();
+ } else {
+ // Update the label for the next iteration.
+ label->link_to(prevlinkoffset);
+ }
+ }
+ label->bind_to(pc_offset());
+
+ DCHECK(label->is_bound());
+ DCHECK(!label->is_linked());
+}
+
+int Assembler::LinkAndGetByteOffsetTo(Label* label) {
+ DCHECK_EQ(sizeof(*pc_), 1);
+ CheckLabelLinkChain(label);
+
+ int offset;
+ if (label->is_bound()) {
+ // The label is bound, so it does not need to be updated. Referring
+ // instructions must link directly to the label as they will not be
+ // updated.
+ //
+ // In this case, label->pos() returns the offset of the label from the
+ // start of the buffer.
+ //
+ // Note that offset can be zero for self-referential instructions. (This
+ // could be useful for ADR, for example.)
+ offset = label->pos() - pc_offset();
+ DCHECK_LE(offset, 0);
+ } else {
+ if (label->is_linked()) {
+ // The label is linked, so the referring instruction should be added onto
+ // the end of the label's link chain.
+ //
+ // In this case, label->pos() returns the offset of the last linked
+ // instruction from the start of the buffer.
+ offset = label->pos() - pc_offset();
+ DCHECK_NE(offset, kStartOfLabelLinkChain);
+ // Note that the offset here needs to be PC-relative only so that the
+ // first instruction in a buffer can link to an unbound label. Otherwise,
+ // the offset would be 0 for this case, and 0 is reserved for
+ // kStartOfLabelLinkChain.
+ } else {
+ // The label is unused, so it now becomes linked and the referring
+ // instruction is at the start of the new link chain.
+ offset = kStartOfLabelLinkChain;
+ }
+ // The instruction at pc is now the last link in the label's chain.
+ label->link_to(pc_offset());
+ }
+
+ return offset;
+}
+
+void Assembler::DeleteUnresolvedBranchInfoForLabelTraverse(Label* label) {
+ DCHECK(label->is_linked());
+ CheckLabelLinkChain(label);
+
+ int link_offset = label->pos();
+ int link_pcoffset;
+ bool end_of_chain = false;
+
+ while (!end_of_chain) {
+ Instruction* link = InstructionAt(link_offset);
+ link_pcoffset = static_cast<int>(link->ImmPCOffset());
+
+ // ADR instructions are not handled by veneers.
+ if (link->IsImmBranch()) {
+ int max_reachable_pc =
+ static_cast<int>(InstructionOffset(link) +
+ Instruction::ImmBranchRange(link->BranchType()));
+ using unresolved_info_it = std::multimap<int, FarBranchInfo>::iterator;
+ std::pair<unresolved_info_it, unresolved_info_it> range;
+ range = unresolved_branches_.equal_range(max_reachable_pc);
+ unresolved_info_it it;
+ for (it = range.first; it != range.second; ++it) {
+ if (it->second.pc_offset_ == link_offset) {
+ unresolved_branches_.erase(it);
+ break;
+ }
+ }
+ }
+
+ end_of_chain = (link_pcoffset == 0);
+ link_offset = link_offset + link_pcoffset;
+ }
+}
+
+void Assembler::DeleteUnresolvedBranchInfoForLabel(Label* label) {
+ if (unresolved_branches_.empty()) {
+ DCHECK_EQ(next_veneer_pool_check_, kMaxInt);
+ return;
+ }
+
+ if (label->is_linked()) {
+ // Branches to this label will be resolved when the label is bound, normally
+ // just after all the associated info has been deleted.
+ DeleteUnresolvedBranchInfoForLabelTraverse(label);
+ }
+ if (unresolved_branches_.empty()) {
+ next_veneer_pool_check_ = kMaxInt;
+ } else {
+ next_veneer_pool_check_ =
+ unresolved_branches_first_limit() - kVeneerDistanceCheckMargin;
+ }
+}
+
+bool Assembler::IsConstantPoolAt(Instruction* instr) {
+ // The constant pool marker is made of two instructions. These instructions
+ // will never be emitted by the JIT, so checking for the first one is enough:
+ // 0: ldr xzr, #<size of pool>
+ bool result = instr->IsLdrLiteralX() && (instr->Rt() == kZeroRegCode);
+
+ // It is still worth asserting the marker is complete.
+ // 4: blr xzr
+ DCHECK(!result || (instr->following()->IsBranchAndLinkToRegister() &&
+ instr->following()->Rn() == kZeroRegCode));
+
+ return result;
+}
+
+int Assembler::ConstantPoolSizeAt(Instruction* instr) {
+#ifdef USE_SIMULATOR
+ // Assembler::debug() embeds constants directly into the instruction stream.
+ // Although this is not a genuine constant pool, treat it like one to avoid
+ // disassembling the constants.
+ if ((instr->Mask(ExceptionMask) == HLT) &&
+ (instr->ImmException() == kImmExceptionIsDebug)) {
+ const char* message = reinterpret_cast<const char*>(
+ instr->InstructionAtOffset(kDebugMessageOffset));
+ int size = static_cast<int>(kDebugMessageOffset + strlen(message) + 1);
+ return RoundUp(size, kInstrSize) / kInstrSize;
+ }
+ // Same for printf support, see MacroAssembler::CallPrintf().
+ if ((instr->Mask(ExceptionMask) == HLT) &&
+ (instr->ImmException() == kImmExceptionIsPrintf)) {
+ return kPrintfLength / kInstrSize;
+ }
+#endif
+ if (IsConstantPoolAt(instr)) {
+ return instr->ImmLLiteral();
+ } else {
+ return -1;
+ }
+}
+
+void Assembler::EmitPoolGuard() {
+ // We must generate only one instruction as this is used in scopes that
+ // control the size of the code generated.
+ Emit(BLR | Rn(xzr));
+}
+
+void Assembler::StartBlockVeneerPool() { ++veneer_pool_blocked_nesting_; }
+
+void Assembler::EndBlockVeneerPool() {
+ if (--veneer_pool_blocked_nesting_ == 0) {
+ // Check the veneer pool hasn't been blocked for too long.
+ DCHECK(unresolved_branches_.empty() ||
+ (pc_offset() < unresolved_branches_first_limit()));
+ }
+}
+
+void Assembler::br(const Register& xn) {
+ DCHECK(xn.Is64Bits());
+ Emit(BR | Rn(xn));
+}
+
+void Assembler::blr(const Register& xn) {
+ DCHECK(xn.Is64Bits());
+ // The pattern 'blr xzr' is used as a guard to detect when execution falls
+ // through the constant pool. It should not be emitted.
+ DCHECK_NE(xn, xzr);
+ Emit(BLR | Rn(xn));
+}
+
+void Assembler::ret(const Register& xn) {
+ DCHECK(xn.Is64Bits());
+ Emit(RET | Rn(xn));
+}
+
+void Assembler::b(int imm26) { Emit(B | ImmUncondBranch(imm26)); }
+
+void Assembler::b(Label* label) { b(LinkAndGetInstructionOffsetTo(label)); }
+
+void Assembler::b(int imm19, Condition cond) {
+ Emit(B_cond | ImmCondBranch(imm19) | cond);
+}
+
+void Assembler::b(Label* label, Condition cond) {
+ b(LinkAndGetInstructionOffsetTo(label), cond);
+}
+
+void Assembler::bl(int imm26) { Emit(BL | ImmUncondBranch(imm26)); }
+
+void Assembler::bl(Label* label) { bl(LinkAndGetInstructionOffsetTo(label)); }
+
+void Assembler::cbz(const Register& rt, int imm19) {
+ Emit(SF(rt) | CBZ | ImmCmpBranch(imm19) | Rt(rt));
+}
+
+void Assembler::cbz(const Register& rt, Label* label) {
+ cbz(rt, LinkAndGetInstructionOffsetTo(label));
+}
+
+void Assembler::cbnz(const Register& rt, int imm19) {
+ Emit(SF(rt) | CBNZ | ImmCmpBranch(imm19) | Rt(rt));
+}
+
+void Assembler::cbnz(const Register& rt, Label* label) {
+ cbnz(rt, LinkAndGetInstructionOffsetTo(label));
+}
+
+void Assembler::tbz(const Register& rt, unsigned bit_pos, int imm14) {
+ DCHECK(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSizeInBits)));
+ Emit(TBZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt));
+}
+
+void Assembler::tbz(const Register& rt, unsigned bit_pos, Label* label) {
+ tbz(rt, bit_pos, LinkAndGetInstructionOffsetTo(label));
+}
+
+void Assembler::tbnz(const Register& rt, unsigned bit_pos, int imm14) {
+ DCHECK(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSizeInBits)));
+ Emit(TBNZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt));
+}
+
+void Assembler::tbnz(const Register& rt, unsigned bit_pos, Label* label) {
+ tbnz(rt, bit_pos, LinkAndGetInstructionOffsetTo(label));
+}
+
+void Assembler::adr(const Register& rd, int imm21) {
+ DCHECK(rd.Is64Bits());
+ Emit(ADR | ImmPCRelAddress(imm21) | Rd(rd));
+}
+
+void Assembler::adr(const Register& rd, Label* label) {
+ adr(rd, LinkAndGetByteOffsetTo(label));
+}
+
+void Assembler::nop(NopMarkerTypes n) {
+ DCHECK((FIRST_NOP_MARKER <= n) && (n <= LAST_NOP_MARKER));
+ mov(Register::XRegFromCode(n), Register::XRegFromCode(n));
+}
+
+void Assembler::add(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ AddSub(rd, rn, operand, LeaveFlags, ADD);
+}
+
+void Assembler::adds(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ AddSub(rd, rn, operand, SetFlags, ADD);
+}
+
+void Assembler::cmn(const Register& rn, const Operand& operand) {
+ Register zr = AppropriateZeroRegFor(rn);
+ adds(zr, rn, operand);
+}
+
+void Assembler::sub(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ AddSub(rd, rn, operand, LeaveFlags, SUB);
+}
+
+void Assembler::subs(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ AddSub(rd, rn, operand, SetFlags, SUB);
+}
+
+void Assembler::cmp(const Register& rn, const Operand& operand) {
+ Register zr = AppropriateZeroRegFor(rn);
+ subs(zr, rn, operand);
+}
+
+void Assembler::neg(const Register& rd, const Operand& operand) {
+ Register zr = AppropriateZeroRegFor(rd);
+ sub(rd, zr, operand);
+}
+
+void Assembler::negs(const Register& rd, const Operand& operand) {
+ Register zr = AppropriateZeroRegFor(rd);
+ subs(rd, zr, operand);
+}
+
+void Assembler::adc(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ AddSubWithCarry(rd, rn, operand, LeaveFlags, ADC);
+}
+
+void Assembler::adcs(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ AddSubWithCarry(rd, rn, operand, SetFlags, ADC);
+}
+
+void Assembler::sbc(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ AddSubWithCarry(rd, rn, operand, LeaveFlags, SBC);
+}
+
+void Assembler::sbcs(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ AddSubWithCarry(rd, rn, operand, SetFlags, SBC);
+}
+
+void Assembler::ngc(const Register& rd, const Operand& operand) {
+ Register zr = AppropriateZeroRegFor(rd);
+ sbc(rd, zr, operand);
+}
+
+void Assembler::ngcs(const Register& rd, const Operand& operand) {
+ Register zr = AppropriateZeroRegFor(rd);
+ sbcs(rd, zr, operand);
+}
+
+// Logical instructions.
+void Assembler::and_(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ Logical(rd, rn, operand, AND);
+}
+
+void Assembler::ands(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ Logical(rd, rn, operand, ANDS);
+}
+
+void Assembler::tst(const Register& rn, const Operand& operand) {
+ ands(AppropriateZeroRegFor(rn), rn, operand);
+}
+
+void Assembler::bic(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ Logical(rd, rn, operand, BIC);
+}
+
+void Assembler::bics(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ Logical(rd, rn, operand, BICS);
+}
+
+void Assembler::orr(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ Logical(rd, rn, operand, ORR);
+}
+
+void Assembler::orn(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ Logical(rd, rn, operand, ORN);
+}
+
+void Assembler::eor(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ Logical(rd, rn, operand, EOR);
+}
+
+void Assembler::eon(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ Logical(rd, rn, operand, EON);
+}
+
+void Assembler::lslv(const Register& rd, const Register& rn,
+ const Register& rm) {
+ DCHECK(rd.SizeInBits() == rn.SizeInBits());
+ DCHECK(rd.SizeInBits() == rm.SizeInBits());
+ Emit(SF(rd) | LSLV | Rm(rm) | Rn(rn) | Rd(rd));
+}
+
+void Assembler::lsrv(const Register& rd, const Register& rn,
+ const Register& rm) {
+ DCHECK(rd.SizeInBits() == rn.SizeInBits());
+ DCHECK(rd.SizeInBits() == rm.SizeInBits());
+ Emit(SF(rd) | LSRV | Rm(rm) | Rn(rn) | Rd(rd));
+}
+
+void Assembler::asrv(const Register& rd, const Register& rn,
+ const Register& rm) {
+ DCHECK(rd.SizeInBits() == rn.SizeInBits());
+ DCHECK(rd.SizeInBits() == rm.SizeInBits());
+ Emit(SF(rd) | ASRV | Rm(rm) | Rn(rn) | Rd(rd));
+}
+
+void Assembler::rorv(const Register& rd, const Register& rn,
+ const Register& rm) {
+ DCHECK(rd.SizeInBits() == rn.SizeInBits());
+ DCHECK(rd.SizeInBits() == rm.SizeInBits());
+ Emit(SF(rd) | RORV | Rm(rm) | Rn(rn) | Rd(rd));
+}
+
+// Bitfield operations.
+void Assembler::bfm(const Register& rd, const Register& rn, int immr,
+ int imms) {
+ DCHECK(rd.SizeInBits() == rn.SizeInBits());
+ Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
+ Emit(SF(rd) | BFM | N | ImmR(immr, rd.SizeInBits()) |
+ ImmS(imms, rn.SizeInBits()) | Rn(rn) | Rd(rd));
+}
+
+void Assembler::sbfm(const Register& rd, const Register& rn, int immr,
+ int imms) {
+ DCHECK(rd.Is64Bits() || rn.Is32Bits());
+ Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
+ Emit(SF(rd) | SBFM | N | ImmR(immr, rd.SizeInBits()) |
+ ImmS(imms, rn.SizeInBits()) | Rn(rn) | Rd(rd));
+}
+
+void Assembler::ubfm(const Register& rd, const Register& rn, int immr,
+ int imms) {
+ DCHECK(rd.SizeInBits() == rn.SizeInBits());
+ Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
+ Emit(SF(rd) | UBFM | N | ImmR(immr, rd.SizeInBits()) |
+ ImmS(imms, rn.SizeInBits()) | Rn(rn) | Rd(rd));
+}
+
+void Assembler::extr(const Register& rd, const Register& rn, const Register& rm,
+ int lsb) {
+ DCHECK(rd.SizeInBits() == rn.SizeInBits());
+ DCHECK(rd.SizeInBits() == rm.SizeInBits());
+ Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
+ Emit(SF(rd) | EXTR | N | Rm(rm) | ImmS(lsb, rn.SizeInBits()) | Rn(rn) |
+ Rd(rd));
+}
+
+void Assembler::csel(const Register& rd, const Register& rn, const Register& rm,
+ Condition cond) {
+ ConditionalSelect(rd, rn, rm, cond, CSEL);
+}
+
+void Assembler::csinc(const Register& rd, const Register& rn,
+ const Register& rm, Condition cond) {
+ ConditionalSelect(rd, rn, rm, cond, CSINC);
+}
+
+void Assembler::csinv(const Register& rd, const Register& rn,
+ const Register& rm, Condition cond) {
+ ConditionalSelect(rd, rn, rm, cond, CSINV);
+}
+
+void Assembler::csneg(const Register& rd, const Register& rn,
+ const Register& rm, Condition cond) {
+ ConditionalSelect(rd, rn, rm, cond, CSNEG);
+}
+
+void Assembler::cset(const Register& rd, Condition cond) {
+ DCHECK((cond != al) && (cond != nv));
+ Register zr = AppropriateZeroRegFor(rd);
+ csinc(rd, zr, zr, NegateCondition(cond));
+}
+
+void Assembler::csetm(const Register& rd, Condition cond) {
+ DCHECK((cond != al) && (cond != nv));
+ Register zr = AppropriateZeroRegFor(rd);
+ csinv(rd, zr, zr, NegateCondition(cond));
+}
+
+void Assembler::cinc(const Register& rd, const Register& rn, Condition cond) {
+ DCHECK((cond != al) && (cond != nv));
+ csinc(rd, rn, rn, NegateCondition(cond));
+}
+
+void Assembler::cinv(const Register& rd, const Register& rn, Condition cond) {
+ DCHECK((cond != al) && (cond != nv));
+ csinv(rd, rn, rn, NegateCondition(cond));
+}
+
+void Assembler::cneg(const Register& rd, const Register& rn, Condition cond) {
+ DCHECK((cond != al) && (cond != nv));
+ csneg(rd, rn, rn, NegateCondition(cond));
+}
+
+void Assembler::ConditionalSelect(const Register& rd, const Register& rn,
+ const Register& rm, Condition cond,
+ ConditionalSelectOp op) {
+ DCHECK(rd.SizeInBits() == rn.SizeInBits());
+ DCHECK(rd.SizeInBits() == rm.SizeInBits());
+ Emit(SF(rd) | op | Rm(rm) | Cond(cond) | Rn(rn) | Rd(rd));
+}
+
+void Assembler::ccmn(const Register& rn, const Operand& operand,
+ StatusFlags nzcv, Condition cond) {
+ ConditionalCompare(rn, operand, nzcv, cond, CCMN);
+}
+
+void Assembler::ccmp(const Register& rn, const Operand& operand,
+ StatusFlags nzcv, Condition cond) {
+ ConditionalCompare(rn, operand, nzcv, cond, CCMP);
+}
+
+void Assembler::DataProcessing3Source(const Register& rd, const Register& rn,
+ const Register& rm, const Register& ra,
+ DataProcessing3SourceOp op) {
+ Emit(SF(rd) | op | Rm(rm) | Ra(ra) | Rn(rn) | Rd(rd));
+}
+
+void Assembler::mul(const Register& rd, const Register& rn,
+ const Register& rm) {
+ DCHECK(AreSameSizeAndType(rd, rn, rm));
+ Register zr = AppropriateZeroRegFor(rn);
+ DataProcessing3Source(rd, rn, rm, zr, MADD);
+}
+
+void Assembler::madd(const Register& rd, const Register& rn, const Register& rm,
+ const Register& ra) {
+ DCHECK(AreSameSizeAndType(rd, rn, rm, ra));
+ DataProcessing3Source(rd, rn, rm, ra, MADD);
+}
+
+void Assembler::mneg(const Register& rd, const Register& rn,
+ const Register& rm) {
+ DCHECK(AreSameSizeAndType(rd, rn, rm));
+ Register zr = AppropriateZeroRegFor(rn);
+ DataProcessing3Source(rd, rn, rm, zr, MSUB);
+}
+
+void Assembler::msub(const Register& rd, const Register& rn, const Register& rm,
+ const Register& ra) {
+ DCHECK(AreSameSizeAndType(rd, rn, rm, ra));
+ DataProcessing3Source(rd, rn, rm, ra, MSUB);
+}
+
+void Assembler::smaddl(const Register& rd, const Register& rn,
+ const Register& rm, const Register& ra) {
+ DCHECK(rd.Is64Bits() && ra.Is64Bits());
+ DCHECK(rn.Is32Bits() && rm.Is32Bits());
+ DataProcessing3Source(rd, rn, rm, ra, SMADDL_x);
+}
+
+void Assembler::smsubl(const Register& rd, const Register& rn,
+ const Register& rm, const Register& ra) {
+ DCHECK(rd.Is64Bits() && ra.Is64Bits());
+ DCHECK(rn.Is32Bits() && rm.Is32Bits());
+ DataProcessing3Source(rd, rn, rm, ra, SMSUBL_x);
+}
+
+void Assembler::umaddl(const Register& rd, const Register& rn,
+ const Register& rm, const Register& ra) {
+ DCHECK(rd.Is64Bits() && ra.Is64Bits());
+ DCHECK(rn.Is32Bits() && rm.Is32Bits());
+ DataProcessing3Source(rd, rn, rm, ra, UMADDL_x);
+}
+
+void Assembler::umsubl(const Register& rd, const Register& rn,
+ const Register& rm, const Register& ra) {
+ DCHECK(rd.Is64Bits() && ra.Is64Bits());
+ DCHECK(rn.Is32Bits() && rm.Is32Bits());
+ DataProcessing3Source(rd, rn, rm, ra, UMSUBL_x);
+}
+
+void Assembler::smull(const Register& rd, const Register& rn,
+ const Register& rm) {
+ DCHECK(rd.Is64Bits());
+ DCHECK(rn.Is32Bits() && rm.Is32Bits());
+ DataProcessing3Source(rd, rn, rm, xzr, SMADDL_x);
+}
+
+void Assembler::smulh(const Register& rd, const Register& rn,
+ const Register& rm) {
+ DCHECK(AreSameSizeAndType(rd, rn, rm));
+ DataProcessing3Source(rd, rn, rm, xzr, SMULH_x);
+}
+
+void Assembler::sdiv(const Register& rd, const Register& rn,
+ const Register& rm) {
+ DCHECK(rd.SizeInBits() == rn.SizeInBits());
+ DCHECK(rd.SizeInBits() == rm.SizeInBits());
+ Emit(SF(rd) | SDIV | Rm(rm) | Rn(rn) | Rd(rd));
+}
+
+void Assembler::udiv(const Register& rd, const Register& rn,
+ const Register& rm) {
+ DCHECK(rd.SizeInBits() == rn.SizeInBits());
+ DCHECK(rd.SizeInBits() == rm.SizeInBits());
+ Emit(SF(rd) | UDIV | Rm(rm) | Rn(rn) | Rd(rd));
+}
+
+void Assembler::rbit(const Register& rd, const Register& rn) {
+ DataProcessing1Source(rd, rn, RBIT);
+}
+
+void Assembler::rev16(const Register& rd, const Register& rn) {
+ DataProcessing1Source(rd, rn, REV16);
+}
+
+void Assembler::rev32(const Register& rd, const Register& rn) {
+ DCHECK(rd.Is64Bits());
+ DataProcessing1Source(rd, rn, REV);
+}
+
+void Assembler::rev(const Register& rd, const Register& rn) {
+ DataProcessing1Source(rd, rn, rd.Is64Bits() ? REV_x : REV_w);
+}
+
+void Assembler::clz(const Register& rd, const Register& rn) {
+ DataProcessing1Source(rd, rn, CLZ);
+}
+
+void Assembler::cls(const Register& rd, const Register& rn) {
+ DataProcessing1Source(rd, rn, CLS);
+}
+
+void Assembler::pacib1716() { Emit(PACIB1716); }
+void Assembler::autib1716() { Emit(AUTIB1716); }
+void Assembler::pacibsp() { Emit(PACIBSP); }
+void Assembler::autibsp() { Emit(AUTIBSP); }
+
+void Assembler::bti(BranchTargetIdentifier id) {
+ SystemHint op;
+ switch (id) {
+ case BranchTargetIdentifier::kBti:
+ op = BTI;
+ break;
+ case BranchTargetIdentifier::kBtiCall:
+ op = BTI_c;
+ break;
+ case BranchTargetIdentifier::kBtiJump:
+ op = BTI_j;
+ break;
+ case BranchTargetIdentifier::kBtiJumpCall:
+ op = BTI_jc;
+ break;
+ case BranchTargetIdentifier::kNone:
+ case BranchTargetIdentifier::kPacibsp:
+ // We always want to generate a BTI instruction here, so disallow
+ // skipping its generation or generating a PACIBSP instead.
+ UNREACHABLE();
+ }
+ hint(op);
+}
+
+void Assembler::ldp(const CPURegister& rt, const CPURegister& rt2,
+ const MemOperand& src) {
+ LoadStorePair(rt, rt2, src, LoadPairOpFor(rt, rt2));
+}
+
+void Assembler::stp(const CPURegister& rt, const CPURegister& rt2,
+ const MemOperand& dst) {
+ LoadStorePair(rt, rt2, dst, StorePairOpFor(rt, rt2));
+
+#if defined(V8_OS_WIN)
+ if (xdata_encoder_ && rt == x29 && rt2 == lr && dst.base().IsSP()) {
+ xdata_encoder_->onSaveFpLr();
+ }
+#endif
+}
+
+void Assembler::ldpsw(const Register& rt, const Register& rt2,
+ const MemOperand& src) {
+ DCHECK(rt.Is64Bits());
+ LoadStorePair(rt, rt2, src, LDPSW_x);
+}
+
+void Assembler::LoadStorePair(const CPURegister& rt, const CPURegister& rt2,
+ const MemOperand& addr, LoadStorePairOp op) {
+ // 'rt' and 'rt2' can only be aliased for stores.
+ DCHECK(((op & LoadStorePairLBit) == 0) || rt != rt2);
+ DCHECK(AreSameSizeAndType(rt, rt2));
+ DCHECK(IsImmLSPair(addr.offset(), CalcLSPairDataSize(op)));
+ int offset = static_cast<int>(addr.offset());
+
+ Instr memop = op | Rt(rt) | Rt2(rt2) | RnSP(addr.base()) |
+ ImmLSPair(offset, CalcLSPairDataSize(op));
+
+ Instr addrmodeop;
+ if (addr.IsImmediateOffset()) {
+ addrmodeop = LoadStorePairOffsetFixed;
+ } else {
+ // Pre-index and post-index modes.
+ DCHECK_NE(rt, addr.base());
+ DCHECK_NE(rt2, addr.base());
+ DCHECK_NE(addr.offset(), 0);
+ if (addr.IsPreIndex()) {
+ addrmodeop = LoadStorePairPreIndexFixed;
+ } else {
+ DCHECK(addr.IsPostIndex());
+ addrmodeop = LoadStorePairPostIndexFixed;
+ }
+ }
+ Emit(addrmodeop | memop);
+}
+
+// Memory instructions.
+void Assembler::ldrb(const Register& rt, const MemOperand& src) {
+ LoadStore(rt, src, LDRB_w);
+}
+
+void Assembler::strb(const Register& rt, const MemOperand& dst) {
+ LoadStore(rt, dst, STRB_w);
+}
+
+void Assembler::ldrsb(const Register& rt, const MemOperand& src) {
+ LoadStore(rt, src, rt.Is64Bits() ? LDRSB_x : LDRSB_w);
+}
+
+void Assembler::ldrh(const Register& rt, const MemOperand& src) {
+ LoadStore(rt, src, LDRH_w);
+}
+
+void Assembler::strh(const Register& rt, const MemOperand& dst) {
+ LoadStore(rt, dst, STRH_w);
+}
+
+void Assembler::ldrsh(const Register& rt, const MemOperand& src) {
+ LoadStore(rt, src, rt.Is64Bits() ? LDRSH_x : LDRSH_w);
+}
+
+void Assembler::ldr(const CPURegister& rt, const MemOperand& src) {
+ LoadStore(rt, src, LoadOpFor(rt));
+}
+
+void Assembler::str(const CPURegister& rt, const MemOperand& src) {
+ LoadStore(rt, src, StoreOpFor(rt));
+}
+
+void Assembler::ldrsw(const Register& rt, const MemOperand& src) {
+ DCHECK(rt.Is64Bits());
+ LoadStore(rt, src, LDRSW_x);
+}
+
+void Assembler::ldr_pcrel(const CPURegister& rt, int imm19) {
+ // The pattern 'ldr xzr, #offset' is used to indicate the beginning of a
+ // constant pool. It should not be emitted.
+ DCHECK(!rt.IsZero());
+ Emit(LoadLiteralOpFor(rt) | ImmLLiteral(imm19) | Rt(rt));
+}
+
+Operand Operand::EmbeddedNumber(double number) {
+ int32_t smi;
+ if (DoubleToSmiInteger(number, &smi)) {
+ return Operand(Immediate(Smi::FromInt(smi)));
+ }
+ Operand result(0, RelocInfo::FULL_EMBEDDED_OBJECT);
+ result.heap_object_request_.emplace(number);
+ DCHECK(result.IsHeapObjectRequest());
+ return result;
+}
+
+Operand Operand::EmbeddedStringConstant(const StringConstantBase* str) {
+ Operand result(0, RelocInfo::FULL_EMBEDDED_OBJECT);
+ result.heap_object_request_.emplace(str);
+ DCHECK(result.IsHeapObjectRequest());
+ return result;
+}
+
+void Assembler::ldr(const CPURegister& rt, const Operand& operand) {
+ if (operand.IsHeapObjectRequest()) {
+ BlockPoolsScope no_pool_before_ldr_of_heap_object_request(this);
+ RequestHeapObject(operand.heap_object_request());
+ ldr(rt, operand.immediate_for_heap_object_request());
+ } else {
+ ldr(rt, operand.immediate());
+ }
+}
+
+void Assembler::ldr(const CPURegister& rt, const Immediate& imm) {
+ BlockPoolsScope no_pool_before_ldr_pcrel_instr(this);
+ RecordRelocInfo(imm.rmode(), imm.value());
+ // The load will be patched when the constpool is emitted, patching code
+ // expect a load literal with offset 0.
+ ldr_pcrel(rt, 0);
+}
+
+void Assembler::ldar(const Register& rt, const Register& rn) {
+ DCHECK(rn.Is64Bits());
+ LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? LDAR_w : LDAR_x;
+ Emit(op | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
+}
+
+void Assembler::ldaxr(const Register& rt, const Register& rn) {
+ DCHECK(rn.Is64Bits());
+ LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? LDAXR_w : LDAXR_x;
+ Emit(op | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
+}
+
+void Assembler::stlr(const Register& rt, const Register& rn) {
+ DCHECK(rn.Is64Bits());
+ LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? STLR_w : STLR_x;
+ Emit(op | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
+}
+
+void Assembler::stlxr(const Register& rs, const Register& rt,
+ const Register& rn) {
+ DCHECK(rn.Is64Bits());
+ DCHECK(rs != rt && rs != rn);
+ LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? STLXR_w : STLXR_x;
+ Emit(op | Rs(rs) | Rt2(x31) | RnSP(rn) | Rt(rt));
+}
+
+void Assembler::ldarb(const Register& rt, const Register& rn) {
+ DCHECK(rt.Is32Bits());
+ DCHECK(rn.Is64Bits());
+ Emit(LDAR_b | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
+}
+
+void Assembler::ldaxrb(const Register& rt, const Register& rn) {
+ DCHECK(rt.Is32Bits());
+ DCHECK(rn.Is64Bits());
+ Emit(LDAXR_b | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
+}
+
+void Assembler::stlrb(const Register& rt, const Register& rn) {
+ DCHECK(rt.Is32Bits());
+ DCHECK(rn.Is64Bits());
+ Emit(STLR_b | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
+}
+
+void Assembler::stlxrb(const Register& rs, const Register& rt,
+ const Register& rn) {
+ DCHECK(rs.Is32Bits());
+ DCHECK(rt.Is32Bits());
+ DCHECK(rn.Is64Bits());
+ DCHECK(rs != rt && rs != rn);
+ Emit(STLXR_b | Rs(rs) | Rt2(x31) | RnSP(rn) | Rt(rt));
+}
+
+void Assembler::ldarh(const Register& rt, const Register& rn) {
+ DCHECK(rt.Is32Bits());
+ DCHECK(rn.Is64Bits());
+ Emit(LDAR_h | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
+}
+
+void Assembler::ldaxrh(const Register& rt, const Register& rn) {
+ DCHECK(rt.Is32Bits());
+ DCHECK(rn.Is64Bits());
+ Emit(LDAXR_h | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
+}
+
+void Assembler::stlrh(const Register& rt, const Register& rn) {
+ DCHECK(rt.Is32Bits());
+ DCHECK(rn.Is64Bits());
+ Emit(STLR_h | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
+}
+
+void Assembler::stlxrh(const Register& rs, const Register& rt,
+ const Register& rn) {
+ DCHECK(rs.Is32Bits());
+ DCHECK(rt.Is32Bits());
+ DCHECK(rn.Is64Bits());
+ DCHECK(rs != rt && rs != rn);
+ Emit(STLXR_h | Rs(rs) | Rt2(x31) | RnSP(rn) | Rt(rt));
+}
+
+void Assembler::NEON3DifferentL(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm, NEON3DifferentOp vop) {
+ DCHECK(AreSameFormat(vn, vm));
+ DCHECK((vn.Is1H() && vd.Is1S()) || (vn.Is1S() && vd.Is1D()) ||
+ (vn.Is8B() && vd.Is8H()) || (vn.Is4H() && vd.Is4S()) ||
+ (vn.Is2S() && vd.Is2D()) || (vn.Is16B() && vd.Is8H()) ||
+ (vn.Is8H() && vd.Is4S()) || (vn.Is4S() && vd.Is2D()));
+ Instr format, op = vop;
+ if (vd.IsScalar()) {
+ op |= NEON_Q | NEONScalar;
+ format = SFormat(vn);
+ } else {
+ format = VFormat(vn);
+ }
+ Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
+}
+
+void Assembler::NEON3DifferentW(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm, NEON3DifferentOp vop) {
+ DCHECK(AreSameFormat(vd, vn));
+ DCHECK((vm.Is8B() && vd.Is8H()) || (vm.Is4H() && vd.Is4S()) ||
+ (vm.Is2S() && vd.Is2D()) || (vm.Is16B() && vd.Is8H()) ||
+ (vm.Is8H() && vd.Is4S()) || (vm.Is4S() && vd.Is2D()));
+ Emit(VFormat(vm) | vop | Rm(vm) | Rn(vn) | Rd(vd));
+}
+
+void Assembler::NEON3DifferentHN(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm, NEON3DifferentOp vop) {
+ DCHECK(AreSameFormat(vm, vn));
+ DCHECK((vd.Is8B() && vn.Is8H()) || (vd.Is4H() && vn.Is4S()) ||
+ (vd.Is2S() && vn.Is2D()) || (vd.Is16B() && vn.Is8H()) ||
+ (vd.Is8H() && vn.Is4S()) || (vd.Is4S() && vn.Is2D()));
+ Emit(VFormat(vd) | vop | Rm(vm) | Rn(vn) | Rd(vd));
+}
+
+#define NEON_3DIFF_LONG_LIST(V) \
+ V(pmull, NEON_PMULL, vn.IsVector() && vn.Is8B()) \
+ V(pmull2, NEON_PMULL2, vn.IsVector() && vn.Is16B()) \
+ V(saddl, NEON_SADDL, vn.IsVector() && vn.IsD()) \
+ V(saddl2, NEON_SADDL2, vn.IsVector() && vn.IsQ()) \
+ V(sabal, NEON_SABAL, vn.IsVector() && vn.IsD()) \
+ V(sabal2, NEON_SABAL2, vn.IsVector() && vn.IsQ()) \
+ V(uabal, NEON_UABAL, vn.IsVector() && vn.IsD()) \
+ V(uabal2, NEON_UABAL2, vn.IsVector() && vn.IsQ()) \
+ V(sabdl, NEON_SABDL, vn.IsVector() && vn.IsD()) \
+ V(sabdl2, NEON_SABDL2, vn.IsVector() && vn.IsQ()) \
+ V(uabdl, NEON_UABDL, vn.IsVector() && vn.IsD()) \
+ V(uabdl2, NEON_UABDL2, vn.IsVector() && vn.IsQ()) \
+ V(smlal, NEON_SMLAL, vn.IsVector() && vn.IsD()) \
+ V(smlal2, NEON_SMLAL2, vn.IsVector() && vn.IsQ()) \
+ V(umlal, NEON_UMLAL, vn.IsVector() && vn.IsD()) \
+ V(umlal2, NEON_UMLAL2, vn.IsVector() && vn.IsQ()) \
+ V(smlsl, NEON_SMLSL, vn.IsVector() && vn.IsD()) \
+ V(smlsl2, NEON_SMLSL2, vn.IsVector() && vn.IsQ()) \
+ V(umlsl, NEON_UMLSL, vn.IsVector() && vn.IsD()) \
+ V(umlsl2, NEON_UMLSL2, vn.IsVector() && vn.IsQ()) \
+ V(smull, NEON_SMULL, vn.IsVector() && vn.IsD()) \
+ V(smull2, NEON_SMULL2, vn.IsVector() && vn.IsQ()) \
+ V(umull, NEON_UMULL, vn.IsVector() && vn.IsD()) \
+ V(umull2, NEON_UMULL2, vn.IsVector() && vn.IsQ()) \
+ V(ssubl, NEON_SSUBL, vn.IsVector() && vn.IsD()) \
+ V(ssubl2, NEON_SSUBL2, vn.IsVector() && vn.IsQ()) \
+ V(uaddl, NEON_UADDL, vn.IsVector() && vn.IsD()) \
+ V(uaddl2, NEON_UADDL2, vn.IsVector() && vn.IsQ()) \
+ V(usubl, NEON_USUBL, vn.IsVector() && vn.IsD()) \
+ V(usubl2, NEON_USUBL2, vn.IsVector() && vn.IsQ()) \
+ V(sqdmlal, NEON_SQDMLAL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
+ V(sqdmlal2, NEON_SQDMLAL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
+ V(sqdmlsl, NEON_SQDMLSL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
+ V(sqdmlsl2, NEON_SQDMLSL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
+ V(sqdmull, NEON_SQDMULL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
+ V(sqdmull2, NEON_SQDMULL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S())
+
+#define DEFINE_ASM_FUNC(FN, OP, AS) \
+ void Assembler::FN(const VRegister& vd, const VRegister& vn, \
+ const VRegister& vm) { \
+ DCHECK(AS); \
+ NEON3DifferentL(vd, vn, vm, OP); \
+ }
+NEON_3DIFF_LONG_LIST(DEFINE_ASM_FUNC)
+#undef DEFINE_ASM_FUNC
+
+#define NEON_3DIFF_HN_LIST(V) \
+ V(addhn, NEON_ADDHN, vd.IsD()) \
+ V(addhn2, NEON_ADDHN2, vd.IsQ()) \
+ V(raddhn, NEON_RADDHN, vd.IsD()) \
+ V(raddhn2, NEON_RADDHN2, vd.IsQ()) \
+ V(subhn, NEON_SUBHN, vd.IsD()) \
+ V(subhn2, NEON_SUBHN2, vd.IsQ()) \
+ V(rsubhn, NEON_RSUBHN, vd.IsD()) \
+ V(rsubhn2, NEON_RSUBHN2, vd.IsQ())
+
+#define DEFINE_ASM_FUNC(FN, OP, AS) \
+ void Assembler::FN(const VRegister& vd, const VRegister& vn, \
+ const VRegister& vm) { \
+ DCHECK(AS); \
+ NEON3DifferentHN(vd, vn, vm, OP); \
+ }
+NEON_3DIFF_HN_LIST(DEFINE_ASM_FUNC)
+#undef DEFINE_ASM_FUNC
+
+void Assembler::NEONPerm(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm, NEONPermOp op) {
+ DCHECK(AreSameFormat(vd, vn, vm));
+ DCHECK(!vd.Is1D());
+ Emit(VFormat(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
+}
+
+void Assembler::trn1(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm) {
+ NEONPerm(vd, vn, vm, NEON_TRN1);
+}
+
+void Assembler::trn2(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm) {
+ NEONPerm(vd, vn, vm, NEON_TRN2);
+}
+
+void Assembler::uzp1(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm) {
+ NEONPerm(vd, vn, vm, NEON_UZP1);
+}
+
+void Assembler::uzp2(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm) {
+ NEONPerm(vd, vn, vm, NEON_UZP2);
+}
+
+void Assembler::zip1(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm) {
+ NEONPerm(vd, vn, vm, NEON_ZIP1);
+}
+
+void Assembler::zip2(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm) {
+ NEONPerm(vd, vn, vm, NEON_ZIP2);
+}
+
+void Assembler::NEONShiftImmediate(const VRegister& vd, const VRegister& vn,
+ NEONShiftImmediateOp op, int immh_immb) {
+ DCHECK(AreSameFormat(vd, vn));
+ Instr q, scalar;
+ if (vn.IsScalar()) {
+ q = NEON_Q;
+ scalar = NEONScalar;
+ } else {
+ q = vd.IsD() ? 0 : NEON_Q;
+ scalar = 0;
+ }
+ Emit(q | op | scalar | immh_immb | Rn(vn) | Rd(vd));
+}
+
+void Assembler::NEONShiftLeftImmediate(const VRegister& vd, const VRegister& vn,
+ int shift, NEONShiftImmediateOp op) {
+ int laneSizeInBits = vn.LaneSizeInBits();
+ DCHECK((shift >= 0) && (shift < laneSizeInBits));
+ NEONShiftImmediate(vd, vn, op, (laneSizeInBits + shift) << 16);
+}
+
+void Assembler::NEONShiftRightImmediate(const VRegister& vd,
+ const VRegister& vn, int shift,
+ NEONShiftImmediateOp op) {
+ int laneSizeInBits = vn.LaneSizeInBits();
+ DCHECK((shift >= 1) && (shift <= laneSizeInBits));
+ NEONShiftImmediate(vd, vn, op, ((2 * laneSizeInBits) - shift) << 16);
+}
+
+void Assembler::NEONShiftImmediateL(const VRegister& vd, const VRegister& vn,
+ int shift, NEONShiftImmediateOp op) {
+ int laneSizeInBits = vn.LaneSizeInBits();
+ DCHECK((shift >= 0) && (shift < laneSizeInBits));
+ int immh_immb = (laneSizeInBits + shift) << 16;
+
+ DCHECK((vn.Is8B() && vd.Is8H()) || (vn.Is4H() && vd.Is4S()) ||
+ (vn.Is2S() && vd.Is2D()) || (vn.Is16B() && vd.Is8H()) ||
+ (vn.Is8H() && vd.Is4S()) || (vn.Is4S() && vd.Is2D()));
+ Instr q;
+ q = vn.IsD() ? 0 : NEON_Q;
+ Emit(q | op | immh_immb | Rn(vn) | Rd(vd));
+}
+
+void Assembler::NEONShiftImmediateN(const VRegister& vd, const VRegister& vn,
+ int shift, NEONShiftImmediateOp op) {
+ Instr q, scalar;
+ int laneSizeInBits = vd.LaneSizeInBits();
+ DCHECK((shift >= 1) && (shift <= laneSizeInBits));
+ int immh_immb = (2 * laneSizeInBits - shift) << 16;
+
+ if (vn.IsScalar()) {
+ DCHECK((vd.Is1B() && vn.Is1H()) || (vd.Is1H() && vn.Is1S()) ||
+ (vd.Is1S() && vn.Is1D()));
+ q = NEON_Q;
+ scalar = NEONScalar;
+ } else {
+ DCHECK((vd.Is8B() && vn.Is8H()) || (vd.Is4H() && vn.Is4S()) ||
+ (vd.Is2S() && vn.Is2D()) || (vd.Is16B() && vn.Is8H()) ||
+ (vd.Is8H() && vn.Is4S()) || (vd.Is4S() && vn.Is2D()));
+ scalar = 0;
+ q = vd.IsD() ? 0 : NEON_Q;
+ }
+ Emit(q | op | scalar | immh_immb | Rn(vn) | Rd(vd));
+}
+
+void Assembler::shl(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vd.IsVector() || vd.Is1D());
+ NEONShiftLeftImmediate(vd, vn, shift, NEON_SHL);
+}
+
+void Assembler::sli(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vd.IsVector() || vd.Is1D());
+ NEONShiftLeftImmediate(vd, vn, shift, NEON_SLI);
+}
+
+void Assembler::sqshl(const VRegister& vd, const VRegister& vn, int shift) {
+ NEONShiftLeftImmediate(vd, vn, shift, NEON_SQSHL_imm);
+}
+
+void Assembler::sqshlu(const VRegister& vd, const VRegister& vn, int shift) {
+ NEONShiftLeftImmediate(vd, vn, shift, NEON_SQSHLU);
+}
+
+void Assembler::uqshl(const VRegister& vd, const VRegister& vn, int shift) {
+ NEONShiftLeftImmediate(vd, vn, shift, NEON_UQSHL_imm);
+}
+
+void Assembler::sshll(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vn.IsD());
+ NEONShiftImmediateL(vd, vn, shift, NEON_SSHLL);
+}
+
+void Assembler::sshll2(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vn.IsQ());
+ NEONShiftImmediateL(vd, vn, shift, NEON_SSHLL);
+}
+
+void Assembler::sxtl(const VRegister& vd, const VRegister& vn) {
+ sshll(vd, vn, 0);
+}
+
+void Assembler::sxtl2(const VRegister& vd, const VRegister& vn) {
+ sshll2(vd, vn, 0);
+}
+
+void Assembler::ushll(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vn.IsD());
+ NEONShiftImmediateL(vd, vn, shift, NEON_USHLL);
+}
+
+void Assembler::ushll2(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vn.IsQ());
+ NEONShiftImmediateL(vd, vn, shift, NEON_USHLL);
+}
+
+void Assembler::uxtl(const VRegister& vd, const VRegister& vn) {
+ ushll(vd, vn, 0);
+}
+
+void Assembler::uxtl2(const VRegister& vd, const VRegister& vn) {
+ ushll2(vd, vn, 0);
+}
+
+void Assembler::sri(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vd.IsVector() || vd.Is1D());
+ NEONShiftRightImmediate(vd, vn, shift, NEON_SRI);
+}
+
+void Assembler::sshr(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vd.IsVector() || vd.Is1D());
+ NEONShiftRightImmediate(vd, vn, shift, NEON_SSHR);
+}
+
+void Assembler::ushr(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vd.IsVector() || vd.Is1D());
+ NEONShiftRightImmediate(vd, vn, shift, NEON_USHR);
+}
+
+void Assembler::srshr(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vd.IsVector() || vd.Is1D());
+ NEONShiftRightImmediate(vd, vn, shift, NEON_SRSHR);
+}
+
+void Assembler::urshr(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vd.IsVector() || vd.Is1D());
+ NEONShiftRightImmediate(vd, vn, shift, NEON_URSHR);
+}
+
+void Assembler::ssra(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vd.IsVector() || vd.Is1D());
+ NEONShiftRightImmediate(vd, vn, shift, NEON_SSRA);
+}
+
+void Assembler::usra(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vd.IsVector() || vd.Is1D());
+ NEONShiftRightImmediate(vd, vn, shift, NEON_USRA);
+}
+
+void Assembler::srsra(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vd.IsVector() || vd.Is1D());
+ NEONShiftRightImmediate(vd, vn, shift, NEON_SRSRA);
+}
+
+void Assembler::ursra(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vd.IsVector() || vd.Is1D());
+ NEONShiftRightImmediate(vd, vn, shift, NEON_URSRA);
+}
+
+void Assembler::shrn(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vn.IsVector() && vd.IsD());
+ NEONShiftImmediateN(vd, vn, shift, NEON_SHRN);
+}
+
+void Assembler::shrn2(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vn.IsVector() && vd.IsQ());
+ NEONShiftImmediateN(vd, vn, shift, NEON_SHRN);
+}
+
+void Assembler::rshrn(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vn.IsVector() && vd.IsD());
+ NEONShiftImmediateN(vd, vn, shift, NEON_RSHRN);
+}
+
+void Assembler::rshrn2(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vn.IsVector() && vd.IsQ());
+ NEONShiftImmediateN(vd, vn, shift, NEON_RSHRN);
+}
+
+void Assembler::sqshrn(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
+ NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRN);
+}
+
+void Assembler::sqshrn2(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vn.IsVector() && vd.IsQ());
+ NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRN);
+}
+
+void Assembler::sqrshrn(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
+ NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRN);
+}
+
+void Assembler::sqrshrn2(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vn.IsVector() && vd.IsQ());
+ NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRN);
+}
+
+void Assembler::sqshrun(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
+ NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRUN);
+}
+
+void Assembler::sqshrun2(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vn.IsVector() && vd.IsQ());
+ NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRUN);
+}
+
+void Assembler::sqrshrun(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
+ NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRUN);
+}
+
+void Assembler::sqrshrun2(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vn.IsVector() && vd.IsQ());
+ NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRUN);
+}
+
+void Assembler::uqshrn(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
+ NEONShiftImmediateN(vd, vn, shift, NEON_UQSHRN);
+}
+
+void Assembler::uqshrn2(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vn.IsVector() && vd.IsQ());
+ NEONShiftImmediateN(vd, vn, shift, NEON_UQSHRN);
+}
+
+void Assembler::uqrshrn(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
+ NEONShiftImmediateN(vd, vn, shift, NEON_UQRSHRN);
+}
+
+void Assembler::uqrshrn2(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK(vn.IsVector() && vd.IsQ());
+ NEONShiftImmediateN(vd, vn, shift, NEON_UQRSHRN);
+}
+
+void Assembler::uaddw(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm) {
+ DCHECK(vm.IsD());
+ NEON3DifferentW(vd, vn, vm, NEON_UADDW);
+}
+
+void Assembler::uaddw2(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm) {
+ DCHECK(vm.IsQ());
+ NEON3DifferentW(vd, vn, vm, NEON_UADDW2);
+}
+
+void Assembler::saddw(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm) {
+ DCHECK(vm.IsD());
+ NEON3DifferentW(vd, vn, vm, NEON_SADDW);
+}
+
+void Assembler::saddw2(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm) {
+ DCHECK(vm.IsQ());
+ NEON3DifferentW(vd, vn, vm, NEON_SADDW2);
+}
+
+void Assembler::usubw(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm) {
+ DCHECK(vm.IsD());
+ NEON3DifferentW(vd, vn, vm, NEON_USUBW);
+}
+
+void Assembler::usubw2(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm) {
+ DCHECK(vm.IsQ());
+ NEON3DifferentW(vd, vn, vm, NEON_USUBW2);
+}
+
+void Assembler::ssubw(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm) {
+ DCHECK(vm.IsD());
+ NEON3DifferentW(vd, vn, vm, NEON_SSUBW);
+}
+
+void Assembler::ssubw2(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm) {
+ DCHECK(vm.IsQ());
+ NEON3DifferentW(vd, vn, vm, NEON_SSUBW2);
+}
+
+void Assembler::mov(const Register& rd, const Register& rm) {
+ // Moves involving the stack pointer are encoded as add immediate with
+ // second operand of zero. Otherwise, orr with first operand zr is
+ // used.
+ if (rd.IsSP() || rm.IsSP()) {
+ add(rd, rm, 0);
+ } else {
+ orr(rd, AppropriateZeroRegFor(rd), rm);
+ }
+}
+
+void Assembler::ins(const VRegister& vd, int vd_index, const Register& rn) {
+ // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
+ // number of lanes, and T is b, h, s or d.
+ int lane_size = vd.LaneSizeInBytes();
+ NEONFormatField format;
+ switch (lane_size) {
+ case 1:
+ format = NEON_16B;
+ DCHECK(rn.IsW());
+ break;
+ case 2:
+ format = NEON_8H;
+ DCHECK(rn.IsW());
+ break;
+ case 4:
+ format = NEON_4S;
+ DCHECK(rn.IsW());
+ break;
+ default:
+ DCHECK_EQ(lane_size, 8);
+ DCHECK(rn.IsX());
+ format = NEON_2D;
+ break;
+ }
+
+ DCHECK((0 <= vd_index) &&
+ (vd_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
+ Emit(NEON_INS_GENERAL | ImmNEON5(format, vd_index) | Rn(rn) | Rd(vd));
+}
+
+void Assembler::mov(const Register& rd, const VRegister& vn, int vn_index) {
+ DCHECK_GE(vn.SizeInBytes(), 4);
+ umov(rd, vn, vn_index);
+}
+
+void Assembler::smov(const Register& rd, const VRegister& vn, int vn_index) {
+ // We support vn arguments of the form vn.VxT() or vn.T(), where x is the
+ // number of lanes, and T is b, h, s.
+ int lane_size = vn.LaneSizeInBytes();
+ NEONFormatField format;
+ Instr q = 0;
+ switch (lane_size) {
+ case 1:
+ format = NEON_16B;
+ break;
+ case 2:
+ format = NEON_8H;
+ break;
+ default:
+ DCHECK_EQ(lane_size, 4);
+ DCHECK(rd.IsX());
+ format = NEON_4S;
+ break;
+ }
+ q = rd.IsW() ? 0 : NEON_Q;
+ DCHECK((0 <= vn_index) &&
+ (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
+ Emit(q | NEON_SMOV | ImmNEON5(format, vn_index) | Rn(vn) | Rd(rd));
+}
+
+void Assembler::cls(const VRegister& vd, const VRegister& vn) {
+ DCHECK(AreSameFormat(vd, vn));
+ DCHECK(!vd.Is1D() && !vd.Is2D());
+ Emit(VFormat(vn) | NEON_CLS | Rn(vn) | Rd(vd));
+}
+
+void Assembler::clz(const VRegister& vd, const VRegister& vn) {
+ DCHECK(AreSameFormat(vd, vn));
+ DCHECK(!vd.Is1D() && !vd.Is2D());
+ Emit(VFormat(vn) | NEON_CLZ | Rn(vn) | Rd(vd));
+}
+
+void Assembler::cnt(const VRegister& vd, const VRegister& vn) {
+ DCHECK(AreSameFormat(vd, vn));
+ DCHECK(vd.Is8B() || vd.Is16B());
+ Emit(VFormat(vn) | NEON_CNT | Rn(vn) | Rd(vd));
+}
+
+void Assembler::rev16(const VRegister& vd, const VRegister& vn) {
+ DCHECK(AreSameFormat(vd, vn));
+ DCHECK(vd.Is8B() || vd.Is16B());
+ Emit(VFormat(vn) | NEON_REV16 | Rn(vn) | Rd(vd));
+}
+
+void Assembler::rev32(const VRegister& vd, const VRegister& vn) {
+ DCHECK(AreSameFormat(vd, vn));
+ DCHECK(vd.Is8B() || vd.Is16B() || vd.Is4H() || vd.Is8H());
+ Emit(VFormat(vn) | NEON_REV32 | Rn(vn) | Rd(vd));
+}
+
+void Assembler::rev64(const VRegister& vd, const VRegister& vn) {
+ DCHECK(AreSameFormat(vd, vn));
+ DCHECK(!vd.Is1D() && !vd.Is2D());
+ Emit(VFormat(vn) | NEON_REV64 | Rn(vn) | Rd(vd));
+}
+
+void Assembler::ursqrte(const VRegister& vd, const VRegister& vn) {
+ DCHECK(AreSameFormat(vd, vn));
+ DCHECK(vd.Is2S() || vd.Is4S());
+ Emit(VFormat(vn) | NEON_URSQRTE | Rn(vn) | Rd(vd));
+}
+
+void Assembler::urecpe(const VRegister& vd, const VRegister& vn) {
+ DCHECK(AreSameFormat(vd, vn));
+ DCHECK(vd.Is2S() || vd.Is4S());
+ Emit(VFormat(vn) | NEON_URECPE | Rn(vn) | Rd(vd));
+}
+
+void Assembler::NEONAddlp(const VRegister& vd, const VRegister& vn,
+ NEON2RegMiscOp op) {
+ DCHECK((op == NEON_SADDLP) || (op == NEON_UADDLP) || (op == NEON_SADALP) ||
+ (op == NEON_UADALP));
+
+ DCHECK((vn.Is8B() && vd.Is4H()) || (vn.Is4H() && vd.Is2S()) ||
+ (vn.Is2S() && vd.Is1D()) || (vn.Is16B() && vd.Is8H()) ||
+ (vn.Is8H() && vd.Is4S()) || (vn.Is4S() && vd.Is2D()));
+ Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
+}
+
+void Assembler::saddlp(const VRegister& vd, const VRegister& vn) {
+ NEONAddlp(vd, vn, NEON_SADDLP);
+}
+
+void Assembler::uaddlp(const VRegister& vd, const VRegister& vn) {
+ NEONAddlp(vd, vn, NEON_UADDLP);
+}
+
+void Assembler::sadalp(const VRegister& vd, const VRegister& vn) {
+ NEONAddlp(vd, vn, NEON_SADALP);
+}
+
+void Assembler::uadalp(const VRegister& vd, const VRegister& vn) {
+ NEONAddlp(vd, vn, NEON_UADALP);
+}
+
+void Assembler::NEONAcrossLanesL(const VRegister& vd, const VRegister& vn,
+ NEONAcrossLanesOp op) {
+ DCHECK((vn.Is8B() && vd.Is1H()) || (vn.Is16B() && vd.Is1H()) ||
+ (vn.Is4H() && vd.Is1S()) || (vn.Is8H() && vd.Is1S()) ||
+ (vn.Is4S() && vd.Is1D()));
+ Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
+}
+
+void Assembler::saddlv(const VRegister& vd, const VRegister& vn) {
+ NEONAcrossLanesL(vd, vn, NEON_SADDLV);
+}
+
+void Assembler::uaddlv(const VRegister& vd, const VRegister& vn) {
+ NEONAcrossLanesL(vd, vn, NEON_UADDLV);
+}
+
+void Assembler::NEONAcrossLanes(const VRegister& vd, const VRegister& vn,
+ NEONAcrossLanesOp op) {
+ DCHECK((vn.Is8B() && vd.Is1B()) || (vn.Is16B() && vd.Is1B()) ||
+ (vn.Is4H() && vd.Is1H()) || (vn.Is8H() && vd.Is1H()) ||
+ (vn.Is4S() && vd.Is1S()));
+ if ((op & NEONAcrossLanesFPFMask) == NEONAcrossLanesFPFixed) {
+ Emit(FPFormat(vn) | op | Rn(vn) | Rd(vd));
+ } else {
+ Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
+ }
+}
+
+#define NEON_ACROSSLANES_LIST(V) \
+ V(fmaxv, NEON_FMAXV, vd.Is1S()) \
+ V(fminv, NEON_FMINV, vd.Is1S()) \
+ V(fmaxnmv, NEON_FMAXNMV, vd.Is1S()) \
+ V(fminnmv, NEON_FMINNMV, vd.Is1S()) \
+ V(addv, NEON_ADDV, true) \
+ V(smaxv, NEON_SMAXV, true) \
+ V(sminv, NEON_SMINV, true) \
+ V(umaxv, NEON_UMAXV, true) \
+ V(uminv, NEON_UMINV, true)
+
+#define DEFINE_ASM_FUNC(FN, OP, AS) \
+ void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
+ DCHECK(AS); \
+ NEONAcrossLanes(vd, vn, OP); \
+ }
+NEON_ACROSSLANES_LIST(DEFINE_ASM_FUNC)
+#undef DEFINE_ASM_FUNC
+
+void Assembler::mov(const VRegister& vd, int vd_index, const Register& rn) {
+ ins(vd, vd_index, rn);
+}
+
+void Assembler::umov(const Register& rd, const VRegister& vn, int vn_index) {
+ // We support vn arguments of the form vn.VxT() or vn.T(), where x is the
+ // number of lanes, and T is b, h, s or d.
+ int lane_size = vn.LaneSizeInBytes();
+ NEONFormatField format;
+ Instr q = 0;
+ switch (lane_size) {
+ case 1:
+ format = NEON_16B;
+ DCHECK(rd.IsW());
+ break;
+ case 2:
+ format = NEON_8H;
+ DCHECK(rd.IsW());
+ break;
+ case 4:
+ format = NEON_4S;
+ DCHECK(rd.IsW());
+ break;
+ default:
+ DCHECK_EQ(lane_size, 8);
+ DCHECK(rd.IsX());
+ format = NEON_2D;
+ q = NEON_Q;
+ break;
+ }
+
+ DCHECK((0 <= vn_index) &&
+ (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
+ Emit(q | NEON_UMOV | ImmNEON5(format, vn_index) | Rn(vn) | Rd(rd));
+}
+
+void Assembler::mov(const VRegister& vd, const VRegister& vn, int vn_index) {
+ DCHECK(vd.IsScalar());
+ dup(vd, vn, vn_index);
+}
+
+void Assembler::dup(const VRegister& vd, const Register& rn) {
+ DCHECK(!vd.Is1D());
+ DCHECK_EQ(vd.Is2D(), rn.IsX());
+ Instr q = vd.IsD() ? 0 : NEON_Q;
+ Emit(q | NEON_DUP_GENERAL | ImmNEON5(VFormat(vd), 0) | Rn(rn) | Rd(vd));
+}
+
+void Assembler::ins(const VRegister& vd, int vd_index, const VRegister& vn,
+ int vn_index) {
+ DCHECK(AreSameFormat(vd, vn));
+ // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
+ // number of lanes, and T is b, h, s or d.
+ int lane_size = vd.LaneSizeInBytes();
+ NEONFormatField format;
+ switch (lane_size) {
+ case 1:
+ format = NEON_16B;
+ break;
+ case 2:
+ format = NEON_8H;
+ break;
+ case 4:
+ format = NEON_4S;
+ break;
+ default:
+ DCHECK_EQ(lane_size, 8);
+ format = NEON_2D;
+ break;
+ }
+
+ DCHECK((0 <= vd_index) &&
+ (vd_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
+ DCHECK((0 <= vn_index) &&
+ (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
+ Emit(NEON_INS_ELEMENT | ImmNEON5(format, vd_index) |
+ ImmNEON4(format, vn_index) | Rn(vn) | Rd(vd));
+}
+
+void Assembler::NEONTable(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm, NEONTableOp op) {
+ DCHECK(vd.Is16B() || vd.Is8B());
+ DCHECK(vn.Is16B());
+ DCHECK(AreSameFormat(vd, vm));
+ Emit(op | (vd.IsQ() ? NEON_Q : 0) | Rm(vm) | Rn(vn) | Rd(vd));
+}
+
+void Assembler::tbl(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm) {
+ NEONTable(vd, vn, vm, NEON_TBL_1v);
+}
+
+void Assembler::tbl(const VRegister& vd, const VRegister& vn,
+ const VRegister& vn2, const VRegister& vm) {
+ USE(vn2);
+ DCHECK(AreSameFormat(vn, vn2));
+ DCHECK(AreConsecutive(vn, vn2));
+ NEONTable(vd, vn, vm, NEON_TBL_2v);
+}
+
+void Assembler::tbl(const VRegister& vd, const VRegister& vn,
+ const VRegister& vn2, const VRegister& vn3,
+ const VRegister& vm) {
+ USE(vn2);
+ USE(vn3);
+ DCHECK(AreSameFormat(vn, vn2, vn3));
+ DCHECK(AreConsecutive(vn, vn2, vn3));
+ NEONTable(vd, vn, vm, NEON_TBL_3v);
+}
+
+void Assembler::tbl(const VRegister& vd, const VRegister& vn,
+ const VRegister& vn2, const VRegister& vn3,
+ const VRegister& vn4, const VRegister& vm) {
+ USE(vn2);
+ USE(vn3);
+ USE(vn4);
+ DCHECK(AreSameFormat(vn, vn2, vn3, vn4));
+ DCHECK(AreConsecutive(vn, vn2, vn3, vn4));
+ NEONTable(vd, vn, vm, NEON_TBL_4v);
+}
+
+void Assembler::tbx(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm) {
+ NEONTable(vd, vn, vm, NEON_TBX_1v);
+}
+
+void Assembler::tbx(const VRegister& vd, const VRegister& vn,
+ const VRegister& vn2, const VRegister& vm) {
+ USE(vn2);
+ DCHECK(AreSameFormat(vn, vn2));
+ DCHECK(AreConsecutive(vn, vn2));
+ NEONTable(vd, vn, vm, NEON_TBX_2v);
+}
+
+void Assembler::tbx(const VRegister& vd, const VRegister& vn,
+ const VRegister& vn2, const VRegister& vn3,
+ const VRegister& vm) {
+ USE(vn2);
+ USE(vn3);
+ DCHECK(AreSameFormat(vn, vn2, vn3));
+ DCHECK(AreConsecutive(vn, vn2, vn3));
+ NEONTable(vd, vn, vm, NEON_TBX_3v);
+}
+
+void Assembler::tbx(const VRegister& vd, const VRegister& vn,
+ const VRegister& vn2, const VRegister& vn3,
+ const VRegister& vn4, const VRegister& vm) {
+ USE(vn2);
+ USE(vn3);
+ USE(vn4);
+ DCHECK(AreSameFormat(vn, vn2, vn3, vn4));
+ DCHECK(AreConsecutive(vn, vn2, vn3, vn4));
+ NEONTable(vd, vn, vm, NEON_TBX_4v);
+}
+
+void Assembler::mov(const VRegister& vd, int vd_index, const VRegister& vn,
+ int vn_index) {
+ ins(vd, vd_index, vn, vn_index);
+}
+
+void Assembler::mvn(const Register& rd, const Operand& operand) {
+ orn(rd, AppropriateZeroRegFor(rd), operand);
+}
+
+void Assembler::mrs(const Register& rt, SystemRegister sysreg) {
+ DCHECK(rt.Is64Bits());
+ Emit(MRS | ImmSystemRegister(sysreg) | Rt(rt));
+}
+
+void Assembler::msr(SystemRegister sysreg, const Register& rt) {
+ DCHECK(rt.Is64Bits());
+ Emit(MSR | Rt(rt) | ImmSystemRegister(sysreg));
+}
+
+void Assembler::hint(SystemHint code) { Emit(HINT | ImmHint(code) | Rt(xzr)); }
+
+// NEON structure loads and stores.
+Instr Assembler::LoadStoreStructAddrModeField(const MemOperand& addr) {
+ Instr addr_field = RnSP(addr.base());
+
+ if (addr.IsPostIndex()) {
+ static_assert(NEONLoadStoreMultiStructPostIndex ==
+ static_cast<NEONLoadStoreMultiStructPostIndexOp>(
+ NEONLoadStoreSingleStructPostIndex),
+ "Opcodes must match for NEON post index memop.");
+
+ addr_field |= NEONLoadStoreMultiStructPostIndex;
+ if (addr.offset() == 0) {
+ addr_field |= RmNot31(addr.regoffset());
+ } else {
+ // The immediate post index addressing mode is indicated by rm = 31.
+ // The immediate is implied by the number of vector registers used.
+ addr_field |= (0x1F << Rm_offset);
+ }
+ } else {
+ DCHECK(addr.IsImmediateOffset() && (addr.offset() == 0));
+ }
+ return addr_field;
+}
+
+void Assembler::LoadStoreStructVerify(const VRegister& vt,
+ const MemOperand& addr, Instr op) {
+#ifdef DEBUG
+ // Assert that addressing mode is either offset (with immediate 0), post
+ // index by immediate of the size of the register list, or post index by a
+ // value in a core register.
+ if (addr.IsImmediateOffset()) {
+ DCHECK_EQ(addr.offset(), 0);
+ } else {
+ int offset = vt.SizeInBytes();
+ switch (op) {
+ case NEON_LD1_1v:
+ case NEON_ST1_1v:
+ offset *= 1;
+ break;
+ case NEONLoadStoreSingleStructLoad1:
+ case NEONLoadStoreSingleStructStore1:
+ case NEON_LD1R:
+ offset = (offset / vt.LaneCount()) * 1;
+ break;
+
+ case NEON_LD1_2v:
+ case NEON_ST1_2v:
+ case NEON_LD2:
+ case NEON_ST2:
+ offset *= 2;
+ break;
+ case NEONLoadStoreSingleStructLoad2:
+ case NEONLoadStoreSingleStructStore2:
+ case NEON_LD2R:
+ offset = (offset / vt.LaneCount()) * 2;
+ break;
+
+ case NEON_LD1_3v:
+ case NEON_ST1_3v:
+ case NEON_LD3:
+ case NEON_ST3:
+ offset *= 3;
+ break;
+ case NEONLoadStoreSingleStructLoad3:
+ case NEONLoadStoreSingleStructStore3:
+ case NEON_LD3R:
+ offset = (offset / vt.LaneCount()) * 3;
+ break;
+
+ case NEON_LD1_4v:
+ case NEON_ST1_4v:
+ case NEON_LD4:
+ case NEON_ST4:
+ offset *= 4;
+ break;
+ case NEONLoadStoreSingleStructLoad4:
+ case NEONLoadStoreSingleStructStore4:
+ case NEON_LD4R:
+ offset = (offset / vt.LaneCount()) * 4;
+ break;
+ default:
+ UNREACHABLE();
+ }
+ DCHECK(addr.regoffset() != NoReg || addr.offset() == offset);
+ }
+#else
+ USE(vt);
+ USE(addr);
+ USE(op);
+#endif
+}
+
+void Assembler::LoadStoreStruct(const VRegister& vt, const MemOperand& addr,
+ NEONLoadStoreMultiStructOp op) {
+ LoadStoreStructVerify(vt, addr, op);
+ DCHECK(vt.IsVector() || vt.Is1D());
+ Emit(op | LoadStoreStructAddrModeField(addr) | LSVFormat(vt) | Rt(vt));
+}
+
+void Assembler::LoadStoreStructSingleAllLanes(const VRegister& vt,
+ const MemOperand& addr,
+ NEONLoadStoreSingleStructOp op) {
+ LoadStoreStructVerify(vt, addr, op);
+ Emit(op | LoadStoreStructAddrModeField(addr) | LSVFormat(vt) | Rt(vt));
+}
+
+void Assembler::ld1(const VRegister& vt, const MemOperand& src) {
+ LoadStoreStruct(vt, src, NEON_LD1_1v);
+}
+
+void Assembler::ld1(const VRegister& vt, const VRegister& vt2,
+ const MemOperand& src) {
+ USE(vt2);
+ DCHECK(AreSameFormat(vt, vt2));
+ DCHECK(AreConsecutive(vt, vt2));
+ LoadStoreStruct(vt, src, NEON_LD1_2v);
+}
+
+void Assembler::ld1(const VRegister& vt, const VRegister& vt2,
+ const VRegister& vt3, const MemOperand& src) {
+ USE(vt2);
+ USE(vt3);
+ DCHECK(AreSameFormat(vt, vt2, vt3));
+ DCHECK(AreConsecutive(vt, vt2, vt3));
+ LoadStoreStruct(vt, src, NEON_LD1_3v);
+}
+
+void Assembler::ld1(const VRegister& vt, const VRegister& vt2,
+ const VRegister& vt3, const VRegister& vt4,
+ const MemOperand& src) {
+ USE(vt2);
+ USE(vt3);
+ USE(vt4);
+ DCHECK(AreSameFormat(vt, vt2, vt3, vt4));
+ DCHECK(AreConsecutive(vt, vt2, vt3, vt4));
+ LoadStoreStruct(vt, src, NEON_LD1_4v);
+}
+
+void Assembler::ld2(const VRegister& vt, const VRegister& vt2,
+ const MemOperand& src) {
+ USE(vt2);
+ DCHECK(AreSameFormat(vt, vt2));
+ DCHECK(AreConsecutive(vt, vt2));
+ LoadStoreStruct(vt, src, NEON_LD2);
+}
+
+void Assembler::ld2(const VRegister& vt, const VRegister& vt2, int lane,
+ const MemOperand& src) {
+ USE(vt2);
+ DCHECK(AreSameFormat(vt, vt2));
+ DCHECK(AreConsecutive(vt, vt2));
+ LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad2);
+}
+
+void Assembler::ld2r(const VRegister& vt, const VRegister& vt2,
+ const MemOperand& src) {
+ USE(vt2);
+ DCHECK(AreSameFormat(vt, vt2));
+ DCHECK(AreConsecutive(vt, vt2));
+ LoadStoreStructSingleAllLanes(vt, src, NEON_LD2R);
+}
+
+void Assembler::ld3(const VRegister& vt, const VRegister& vt2,
+ const VRegister& vt3, const MemOperand& src) {
+ USE(vt2);
+ USE(vt3);
+ DCHECK(AreSameFormat(vt, vt2, vt3));
+ DCHECK(AreConsecutive(vt, vt2, vt3));
+ LoadStoreStruct(vt, src, NEON_LD3);
+}
+
+void Assembler::ld3(const VRegister& vt, const VRegister& vt2,
+ const VRegister& vt3, int lane, const MemOperand& src) {
+ USE(vt2);
+ USE(vt3);
+ DCHECK(AreSameFormat(vt, vt2, vt3));
+ DCHECK(AreConsecutive(vt, vt2, vt3));
+ LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad3);
+}
+
+void Assembler::ld3r(const VRegister& vt, const VRegister& vt2,
+ const VRegister& vt3, const MemOperand& src) {
+ USE(vt2);
+ USE(vt3);
+ DCHECK(AreSameFormat(vt, vt2, vt3));
+ DCHECK(AreConsecutive(vt, vt2, vt3));
+ LoadStoreStructSingleAllLanes(vt, src, NEON_LD3R);
+}
+
+void Assembler::ld4(const VRegister& vt, const VRegister& vt2,
+ const VRegister& vt3, const VRegister& vt4,
+ const MemOperand& src) {
+ USE(vt2);
+ USE(vt3);
+ USE(vt4);
+ DCHECK(AreSameFormat(vt, vt2, vt3, vt4));
+ DCHECK(AreConsecutive(vt, vt2, vt3, vt4));
+ LoadStoreStruct(vt, src, NEON_LD4);
+}
+
+void Assembler::ld4(const VRegister& vt, const VRegister& vt2,
+ const VRegister& vt3, const VRegister& vt4, int lane,
+ const MemOperand& src) {
+ USE(vt2);
+ USE(vt3);
+ USE(vt4);
+ DCHECK(AreSameFormat(vt, vt2, vt3, vt4));
+ DCHECK(AreConsecutive(vt, vt2, vt3, vt4));
+ LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad4);
+}
+
+void Assembler::ld4r(const VRegister& vt, const VRegister& vt2,
+ const VRegister& vt3, const VRegister& vt4,
+ const MemOperand& src) {
+ USE(vt2);
+ USE(vt3);
+ USE(vt4);
+ DCHECK(AreSameFormat(vt, vt2, vt3, vt4));
+ DCHECK(AreConsecutive(vt, vt2, vt3, vt4));
+ LoadStoreStructSingleAllLanes(vt, src, NEON_LD4R);
+}
+
+void Assembler::st1(const VRegister& vt, const MemOperand& src) {
+ LoadStoreStruct(vt, src, NEON_ST1_1v);
+}
+
+void Assembler::st1(const VRegister& vt, const VRegister& vt2,
+ const MemOperand& src) {
+ USE(vt2);
+ DCHECK(AreSameFormat(vt, vt2));
+ DCHECK(AreConsecutive(vt, vt2));
+ LoadStoreStruct(vt, src, NEON_ST1_2v);
+}
+
+void Assembler::st1(const VRegister& vt, const VRegister& vt2,
+ const VRegister& vt3, const MemOperand& src) {
+ USE(vt2);
+ USE(vt3);
+ DCHECK(AreSameFormat(vt, vt2, vt3));
+ DCHECK(AreConsecutive(vt, vt2, vt3));
+ LoadStoreStruct(vt, src, NEON_ST1_3v);
+}
+
+void Assembler::st1(const VRegister& vt, const VRegister& vt2,
+ const VRegister& vt3, const VRegister& vt4,
+ const MemOperand& src) {
+ USE(vt2);
+ USE(vt3);
+ USE(vt4);
+ DCHECK(AreSameFormat(vt, vt2, vt3, vt4));
+ DCHECK(AreConsecutive(vt, vt2, vt3, vt4));
+ LoadStoreStruct(vt, src, NEON_ST1_4v);
+}
+
+void Assembler::st2(const VRegister& vt, const VRegister& vt2,
+ const MemOperand& dst) {
+ USE(vt2);
+ DCHECK(AreSameFormat(vt, vt2));
+ DCHECK(AreConsecutive(vt, vt2));
+ LoadStoreStruct(vt, dst, NEON_ST2);
+}
+
+void Assembler::st2(const VRegister& vt, const VRegister& vt2, int lane,
+ const MemOperand& dst) {
+ USE(vt2);
+ DCHECK(AreSameFormat(vt, vt2));
+ DCHECK(AreConsecutive(vt, vt2));
+ LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore2);
+}
+
+void Assembler::st3(const VRegister& vt, const VRegister& vt2,
+ const VRegister& vt3, const MemOperand& dst) {
+ USE(vt2);
+ USE(vt3);
+ DCHECK(AreSameFormat(vt, vt2, vt3));
+ DCHECK(AreConsecutive(vt, vt2, vt3));
+ LoadStoreStruct(vt, dst, NEON_ST3);
+}
+
+void Assembler::st3(const VRegister& vt, const VRegister& vt2,
+ const VRegister& vt3, int lane, const MemOperand& dst) {
+ USE(vt2);
+ USE(vt3);
+ DCHECK(AreSameFormat(vt, vt2, vt3));
+ DCHECK(AreConsecutive(vt, vt2, vt3));
+ LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore3);
+}
+
+void Assembler::st4(const VRegister& vt, const VRegister& vt2,
+ const VRegister& vt3, const VRegister& vt4,
+ const MemOperand& dst) {
+ USE(vt2);
+ USE(vt3);
+ USE(vt4);
+ DCHECK(AreSameFormat(vt, vt2, vt3, vt4));
+ DCHECK(AreConsecutive(vt, vt2, vt3, vt4));
+ LoadStoreStruct(vt, dst, NEON_ST4);
+}
+
+void Assembler::st4(const VRegister& vt, const VRegister& vt2,
+ const VRegister& vt3, const VRegister& vt4, int lane,
+ const MemOperand& dst) {
+ USE(vt2);
+ USE(vt3);
+ USE(vt4);
+ DCHECK(AreSameFormat(vt, vt2, vt3, vt4));
+ DCHECK(AreConsecutive(vt, vt2, vt3, vt4));
+ LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore4);
+}
+
+void Assembler::LoadStoreStructSingle(const VRegister& vt, uint32_t lane,
+ const MemOperand& addr,
+ NEONLoadStoreSingleStructOp op) {
+ LoadStoreStructVerify(vt, addr, op);
+
+ // We support vt arguments of the form vt.VxT() or vt.T(), where x is the
+ // number of lanes, and T is b, h, s or d.
+ unsigned lane_size = vt.LaneSizeInBytes();
+ DCHECK_LT(lane, kQRegSize / lane_size);
+
+ // Lane size is encoded in the opcode field. Lane index is encoded in the Q,
+ // S and size fields.
+ lane *= lane_size;
+
+ // Encodings for S[0]/D[0] and S[2]/D[1] are distinguished using the least-
+ // significant bit of the size field, so we increment lane here to account for
+ // that.
+ if (lane_size == 8) lane++;
+
+ Instr size = (lane << NEONLSSize_offset) & NEONLSSize_mask;
+ Instr s = (lane << (NEONS_offset - 2)) & NEONS_mask;
+ Instr q = (lane << (NEONQ_offset - 3)) & NEONQ_mask;
+
+ Instr instr = op;
+ switch (lane_size) {
+ case 1:
+ instr |= NEONLoadStoreSingle_b;
+ break;
+ case 2:
+ instr |= NEONLoadStoreSingle_h;
+ break;
+ case 4:
+ instr |= NEONLoadStoreSingle_s;
+ break;
+ default:
+ DCHECK_EQ(lane_size, 8U);
+ instr |= NEONLoadStoreSingle_d;
+ }
+
+ Emit(instr | LoadStoreStructAddrModeField(addr) | q | size | s | Rt(vt));
+}
+
+void Assembler::ld1(const VRegister& vt, int lane, const MemOperand& src) {
+ LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad1);
+}
+
+void Assembler::ld1r(const VRegister& vt, const MemOperand& src) {
+ LoadStoreStructSingleAllLanes(vt, src, NEON_LD1R);
+}
+
+void Assembler::st1(const VRegister& vt, int lane, const MemOperand& dst) {
+ LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore1);
+}
+
+void Assembler::dmb(BarrierDomain domain, BarrierType type) {
+ Emit(DMB | ImmBarrierDomain(domain) | ImmBarrierType(type));
+}
+
+void Assembler::dsb(BarrierDomain domain, BarrierType type) {
+ Emit(DSB | ImmBarrierDomain(domain) | ImmBarrierType(type));
+}
+
+void Assembler::isb() {
+ Emit(ISB | ImmBarrierDomain(FullSystem) | ImmBarrierType(BarrierAll));
+}
+
+void Assembler::csdb() { hint(CSDB); }
+
+void Assembler::fmov(const VRegister& vd, double imm) {
+ if (vd.IsScalar()) {
+ DCHECK(vd.Is1D());
+ Emit(FMOV_d_imm | Rd(vd) | ImmFP(imm));
+ } else {
+ DCHECK(vd.Is2D());
+ Instr op = NEONModifiedImmediate_MOVI | NEONModifiedImmediateOpBit;
+ Emit(NEON_Q | op | ImmNEONFP(imm) | NEONCmode(0xF) | Rd(vd));
+ }
+}
+
+void Assembler::fmov(const VRegister& vd, float imm) {
+ if (vd.IsScalar()) {
+ DCHECK(vd.Is1S());
+ Emit(FMOV_s_imm | Rd(vd) | ImmFP(imm));
+ } else {
+ DCHECK(vd.Is2S() | vd.Is4S());
+ Instr op = NEONModifiedImmediate_MOVI;
+ Instr q = vd.Is4S() ? NEON_Q : 0;
+ Emit(q | op | ImmNEONFP(imm) | NEONCmode(0xF) | Rd(vd));
+ }
+}
+
+void Assembler::fmov(const Register& rd, const VRegister& fn) {
+ DCHECK_EQ(rd.SizeInBits(), fn.SizeInBits());
+ FPIntegerConvertOp op = rd.Is32Bits() ? FMOV_ws : FMOV_xd;
+ Emit(op | Rd(rd) | Rn(fn));
+}
+
+void Assembler::fmov(const VRegister& vd, const Register& rn) {
+ DCHECK_EQ(vd.SizeInBits(), rn.SizeInBits());
+ FPIntegerConvertOp op = vd.Is32Bits() ? FMOV_sw : FMOV_dx;
+ Emit(op | Rd(vd) | Rn(rn));
+}
+
+void Assembler::fmov(const VRegister& vd, const VRegister& vn) {
+ DCHECK_EQ(vd.SizeInBits(), vn.SizeInBits());
+ Emit(FPType(vd) | FMOV | Rd(vd) | Rn(vn));
+}
+
+void Assembler::fmov(const VRegister& vd, int index, const Register& rn) {
+ DCHECK((index == 1) && vd.Is1D() && rn.IsX());
+ USE(index);
+ Emit(FMOV_d1_x | Rd(vd) | Rn(rn));
+}
+
+void Assembler::fmov(const Register& rd, const VRegister& vn, int index) {
+ DCHECK((index == 1) && vn.Is1D() && rd.IsX());
+ USE(index);
+ Emit(FMOV_x_d1 | Rd(rd) | Rn(vn));
+}
+
+void Assembler::fmadd(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm, const VRegister& fa) {
+ FPDataProcessing3Source(fd, fn, fm, fa, fd.Is32Bits() ? FMADD_s : FMADD_d);
+}
+
+void Assembler::fmsub(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm, const VRegister& fa) {
+ FPDataProcessing3Source(fd, fn, fm, fa, fd.Is32Bits() ? FMSUB_s : FMSUB_d);
+}
+
+void Assembler::fnmadd(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm, const VRegister& fa) {
+ FPDataProcessing3Source(fd, fn, fm, fa, fd.Is32Bits() ? FNMADD_s : FNMADD_d);
+}
+
+void Assembler::fnmsub(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm, const VRegister& fa) {
+ FPDataProcessing3Source(fd, fn, fm, fa, fd.Is32Bits() ? FNMSUB_s : FNMSUB_d);
+}
+
+void Assembler::fnmul(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm) {
+ DCHECK(AreSameSizeAndType(vd, vn, vm));
+ Instr op = vd.Is1S() ? FNMUL_s : FNMUL_d;
+ Emit(FPType(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
+}
+
+void Assembler::fcmp(const VRegister& fn, const VRegister& fm) {
+ DCHECK_EQ(fn.SizeInBits(), fm.SizeInBits());
+ Emit(FPType(fn) | FCMP | Rm(fm) | Rn(fn));
+}
+
+void Assembler::fcmp(const VRegister& fn, double value) {
+ USE(value);
+ // Although the fcmp instruction can strictly only take an immediate value of
+ // +0.0, we don't need to check for -0.0 because the sign of 0.0 doesn't
+ // affect the result of the comparison.
+ DCHECK_EQ(value, 0.0);
+ Emit(FPType(fn) | FCMP_zero | Rn(fn));
+}
+
+void Assembler::fccmp(const VRegister& fn, const VRegister& fm,
+ StatusFlags nzcv, Condition cond) {
+ DCHECK_EQ(fn.SizeInBits(), fm.SizeInBits());
+ Emit(FPType(fn) | FCCMP | Rm(fm) | Cond(cond) | Rn(fn) | Nzcv(nzcv));
+}
+
+void Assembler::fcsel(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm, Condition cond) {
+ DCHECK_EQ(fd.SizeInBits(), fn.SizeInBits());
+ DCHECK_EQ(fd.SizeInBits(), fm.SizeInBits());
+ Emit(FPType(fd) | FCSEL | Rm(fm) | Cond(cond) | Rn(fn) | Rd(fd));
+}
+
+void Assembler::NEONFPConvertToInt(const Register& rd, const VRegister& vn,
+ Instr op) {
+ Emit(SF(rd) | FPType(vn) | op | Rn(vn) | Rd(rd));
+}
+
+void Assembler::NEONFPConvertToInt(const VRegister& vd, const VRegister& vn,
+ Instr op) {
+ if (vn.IsScalar()) {
+ DCHECK((vd.Is1S() && vn.Is1S()) || (vd.Is1D() && vn.Is1D()));
+ op |= NEON_Q | NEONScalar;
+ }
+ Emit(FPFormat(vn) | op | Rn(vn) | Rd(vd));
+}
+
+void Assembler::fcvt(const VRegister& vd, const VRegister& vn) {
+ FPDataProcessing1SourceOp op;
+ if (vd.Is1D()) {
+ DCHECK(vn.Is1S() || vn.Is1H());
+ op = vn.Is1S() ? FCVT_ds : FCVT_dh;
+ } else if (vd.Is1S()) {
+ DCHECK(vn.Is1D() || vn.Is1H());
+ op = vn.Is1D() ? FCVT_sd : FCVT_sh;
+ } else {
+ DCHECK(vd.Is1H());
+ DCHECK(vn.Is1D() || vn.Is1S());
+ op = vn.Is1D() ? FCVT_hd : FCVT_hs;
+ }
+ FPDataProcessing1Source(vd, vn, op);
+}
+
+void Assembler::fcvtl(const VRegister& vd, const VRegister& vn) {
+ DCHECK((vd.Is4S() && vn.Is4H()) || (vd.Is2D() && vn.Is2S()));
+ Instr format = vd.Is2D() ? (1 << NEONSize_offset) : 0;
+ Emit(format | NEON_FCVTL | Rn(vn) | Rd(vd));
+}
+
+void Assembler::fcvtl2(const VRegister& vd, const VRegister& vn) {
+ DCHECK((vd.Is4S() && vn.Is8H()) || (vd.Is2D() && vn.Is4S()));
+ Instr format = vd.Is2D() ? (1 << NEONSize_offset) : 0;
+ Emit(NEON_Q | format | NEON_FCVTL | Rn(vn) | Rd(vd));
+}
+
+void Assembler::fcvtn(const VRegister& vd, const VRegister& vn) {
+ DCHECK((vn.Is4S() && vd.Is4H()) || (vn.Is2D() && vd.Is2S()));
+ Instr format = vn.Is2D() ? (1 << NEONSize_offset) : 0;
+ Emit(format | NEON_FCVTN | Rn(vn) | Rd(vd));
+}
+
+void Assembler::fcvtn2(const VRegister& vd, const VRegister& vn) {
+ DCHECK((vn.Is4S() && vd.Is8H()) || (vn.Is2D() && vd.Is4S()));
+ Instr format = vn.Is2D() ? (1 << NEONSize_offset) : 0;
+ Emit(NEON_Q | format | NEON_FCVTN | Rn(vn) | Rd(vd));
+}
+
+void Assembler::fcvtxn(const VRegister& vd, const VRegister& vn) {
+ Instr format = 1 << NEONSize_offset;
+ if (vd.IsScalar()) {
+ DCHECK(vd.Is1S() && vn.Is1D());
+ Emit(format | NEON_FCVTXN_scalar | Rn(vn) | Rd(vd));
+ } else {
+ DCHECK(vd.Is2S() && vn.Is2D());
+ Emit(format | NEON_FCVTXN | Rn(vn) | Rd(vd));
+ }
+}
+
+void Assembler::fcvtxn2(const VRegister& vd, const VRegister& vn) {
+ DCHECK(vd.Is4S() && vn.Is2D());
+ Instr format = 1 << NEONSize_offset;
+ Emit(NEON_Q | format | NEON_FCVTXN | Rn(vn) | Rd(vd));
+}
+
+void Assembler::fjcvtzs(const Register& rd, const VRegister& vn) {
+ DCHECK(rd.IsW() && vn.Is1D());
+ Emit(FJCVTZS | Rn(vn) | Rd(rd));
+}
+
+#define NEON_FP2REGMISC_FCVT_LIST(V) \
+ V(fcvtnu, NEON_FCVTNU, FCVTNU) \
+ V(fcvtns, NEON_FCVTNS, FCVTNS) \
+ V(fcvtpu, NEON_FCVTPU, FCVTPU) \
+ V(fcvtps, NEON_FCVTPS, FCVTPS) \
+ V(fcvtmu, NEON_FCVTMU, FCVTMU) \
+ V(fcvtms, NEON_FCVTMS, FCVTMS) \
+ V(fcvtau, NEON_FCVTAU, FCVTAU) \
+ V(fcvtas, NEON_FCVTAS, FCVTAS)
+
+#define DEFINE_ASM_FUNCS(FN, VEC_OP, SCA_OP) \
+ void Assembler::FN(const Register& rd, const VRegister& vn) { \
+ NEONFPConvertToInt(rd, vn, SCA_OP); \
+ } \
+ void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
+ NEONFPConvertToInt(vd, vn, VEC_OP); \
+ }
+NEON_FP2REGMISC_FCVT_LIST(DEFINE_ASM_FUNCS)
+#undef DEFINE_ASM_FUNCS
+
+void Assembler::scvtf(const VRegister& vd, const VRegister& vn, int fbits) {
+ DCHECK_GE(fbits, 0);
+ if (fbits == 0) {
+ NEONFP2RegMisc(vd, vn, NEON_SCVTF);
+ } else {
+ DCHECK(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
+ NEONShiftRightImmediate(vd, vn, fbits, NEON_SCVTF_imm);
+ }
+}
+
+void Assembler::ucvtf(const VRegister& vd, const VRegister& vn, int fbits) {
+ DCHECK_GE(fbits, 0);
+ if (fbits == 0) {
+ NEONFP2RegMisc(vd, vn, NEON_UCVTF);
+ } else {
+ DCHECK(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
+ NEONShiftRightImmediate(vd, vn, fbits, NEON_UCVTF_imm);
+ }
+}
+
+void Assembler::scvtf(const VRegister& vd, const Register& rn, int fbits) {
+ DCHECK_GE(fbits, 0);
+ if (fbits == 0) {
+ Emit(SF(rn) | FPType(vd) | SCVTF | Rn(rn) | Rd(vd));
+ } else {
+ Emit(SF(rn) | FPType(vd) | SCVTF_fixed | FPScale(64 - fbits) | Rn(rn) |
+ Rd(vd));
+ }
+}
+
+void Assembler::ucvtf(const VRegister& fd, const Register& rn, int fbits) {
+ DCHECK_GE(fbits, 0);
+ if (fbits == 0) {
+ Emit(SF(rn) | FPType(fd) | UCVTF | Rn(rn) | Rd(fd));
+ } else {
+ Emit(SF(rn) | FPType(fd) | UCVTF_fixed | FPScale(64 - fbits) | Rn(rn) |
+ Rd(fd));
+ }
+}
+
+void Assembler::NEON3Same(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm, NEON3SameOp vop) {
+ DCHECK(AreSameFormat(vd, vn, vm));
+ DCHECK(vd.IsVector() || !vd.IsQ());
+
+ Instr format, op = vop;
+ if (vd.IsScalar()) {
+ op |= NEON_Q | NEONScalar;
+ format = SFormat(vd);
+ } else {
+ format = VFormat(vd);
+ }
+
+ Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
+}
+
+void Assembler::NEONFP3Same(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm, Instr op) {
+ DCHECK(AreSameFormat(vd, vn, vm));
+ Emit(FPFormat(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
+}
+
+#define NEON_FP2REGMISC_LIST(V) \
+ V(fabs, NEON_FABS, FABS) \
+ V(fneg, NEON_FNEG, FNEG) \
+ V(fsqrt, NEON_FSQRT, FSQRT) \
+ V(frintn, NEON_FRINTN, FRINTN) \
+ V(frinta, NEON_FRINTA, FRINTA) \
+ V(frintp, NEON_FRINTP, FRINTP) \
+ V(frintm, NEON_FRINTM, FRINTM) \
+ V(frintx, NEON_FRINTX, FRINTX) \
+ V(frintz, NEON_FRINTZ, FRINTZ) \
+ V(frinti, NEON_FRINTI, FRINTI) \
+ V(frsqrte, NEON_FRSQRTE, NEON_FRSQRTE_scalar) \
+ V(frecpe, NEON_FRECPE, NEON_FRECPE_scalar)
+
+#define DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP) \
+ void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
+ Instr op; \
+ if (vd.IsScalar()) { \
+ DCHECK(vd.Is1S() || vd.Is1D()); \
+ op = SCA_OP; \
+ } else { \
+ DCHECK(vd.Is2S() || vd.Is2D() || vd.Is4S()); \
+ op = VEC_OP; \
+ } \
+ NEONFP2RegMisc(vd, vn, op); \
+ }
+NEON_FP2REGMISC_LIST(DEFINE_ASM_FUNC)
+#undef DEFINE_ASM_FUNC
+
+void Assembler::shll(const VRegister& vd, const VRegister& vn, int shift) {
+ DCHECK((vd.Is8H() && vn.Is8B() && shift == 8) ||
+ (vd.Is4S() && vn.Is4H() && shift == 16) ||
+ (vd.Is2D() && vn.Is2S() && shift == 32));
+ USE(shift);
+ Emit(VFormat(vn) | NEON_SHLL | Rn(vn) | Rd(vd));
+}
+
+void Assembler::shll2(const VRegister& vd, const VRegister& vn, int shift) {
+ USE(shift);
+ DCHECK((vd.Is8H() && vn.Is16B() && shift == 8) ||
+ (vd.Is4S() && vn.Is8H() && shift == 16) ||
+ (vd.Is2D() && vn.Is4S() && shift == 32));
+ Emit(VFormat(vn) | NEON_SHLL | Rn(vn) | Rd(vd));
+}
+
+void Assembler::NEONFP2RegMisc(const VRegister& vd, const VRegister& vn,
+ NEON2RegMiscOp vop, double value) {
+ DCHECK(AreSameFormat(vd, vn));
+ DCHECK_EQ(value, 0.0);
+ USE(value);
+
+ Instr op = vop;
+ if (vd.IsScalar()) {
+ DCHECK(vd.Is1S() || vd.Is1D());
+ op |= NEON_Q | NEONScalar;
+ } else {
+ DCHECK(vd.Is2S() || vd.Is2D() || vd.Is4S());
+ }
+
+ Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd));
+}
+
+void Assembler::fcmeq(const VRegister& vd, const VRegister& vn, double value) {
+ NEONFP2RegMisc(vd, vn, NEON_FCMEQ_zero, value);
+}
+
+void Assembler::fcmge(const VRegister& vd, const VRegister& vn, double value) {
+ NEONFP2RegMisc(vd, vn, NEON_FCMGE_zero, value);
+}
+
+void Assembler::fcmgt(const VRegister& vd, const VRegister& vn, double value) {
+ NEONFP2RegMisc(vd, vn, NEON_FCMGT_zero, value);
+}
+
+void Assembler::fcmle(const VRegister& vd, const VRegister& vn, double value) {
+ NEONFP2RegMisc(vd, vn, NEON_FCMLE_zero, value);
+}
+
+void Assembler::fcmlt(const VRegister& vd, const VRegister& vn, double value) {
+ NEONFP2RegMisc(vd, vn, NEON_FCMLT_zero, value);
+}
+
+void Assembler::frecpx(const VRegister& vd, const VRegister& vn) {
+ DCHECK(vd.IsScalar());
+ DCHECK(AreSameFormat(vd, vn));
+ DCHECK(vd.Is1S() || vd.Is1D());
+ Emit(FPFormat(vd) | NEON_FRECPX_scalar | Rn(vn) | Rd(vd));
+}
+
+void Assembler::fcvtzs(const Register& rd, const VRegister& vn, int fbits) {
+ DCHECK(vn.Is1S() || vn.Is1D());
+ DCHECK((fbits >= 0) && (fbits <= rd.SizeInBits()));
+ if (fbits == 0) {
+ Emit(SF(rd) | FPType(vn) | FCVTZS | Rn(vn) | Rd(rd));
+ } else {
+ Emit(SF(rd) | FPType(vn) | FCVTZS_fixed | FPScale(64 - fbits) | Rn(vn) |
+ Rd(rd));
+ }
+}
+
+void Assembler::fcvtzs(const VRegister& vd, const VRegister& vn, int fbits) {
+ DCHECK_GE(fbits, 0);
+ if (fbits == 0) {
+ NEONFP2RegMisc(vd, vn, NEON_FCVTZS);
+ } else {
+ DCHECK(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
+ NEONShiftRightImmediate(vd, vn, fbits, NEON_FCVTZS_imm);
+ }
+}
+
+void Assembler::fcvtzu(const Register& rd, const VRegister& vn, int fbits) {
+ DCHECK(vn.Is1S() || vn.Is1D());
+ DCHECK((fbits >= 0) && (fbits <= rd.SizeInBits()));
+ if (fbits == 0) {
+ Emit(SF(rd) | FPType(vn) | FCVTZU | Rn(vn) | Rd(rd));
+ } else {
+ Emit(SF(rd) | FPType(vn) | FCVTZU_fixed | FPScale(64 - fbits) | Rn(vn) |
+ Rd(rd));
+ }
+}
+
+void Assembler::fcvtzu(const VRegister& vd, const VRegister& vn, int fbits) {
+ DCHECK_GE(fbits, 0);
+ if (fbits == 0) {
+ NEONFP2RegMisc(vd, vn, NEON_FCVTZU);
+ } else {
+ DCHECK(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
+ NEONShiftRightImmediate(vd, vn, fbits, NEON_FCVTZU_imm);
+ }
+}
+
+void Assembler::NEONFP2RegMisc(const VRegister& vd, const VRegister& vn,
+ Instr op) {
+ DCHECK(AreSameFormat(vd, vn));
+ Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd));
+}
+
+void Assembler::NEON2RegMisc(const VRegister& vd, const VRegister& vn,
+ NEON2RegMiscOp vop, int value) {
+ DCHECK(AreSameFormat(vd, vn));
+ DCHECK_EQ(value, 0);
+ USE(value);
+
+ Instr format, op = vop;
+ if (vd.IsScalar()) {
+ op |= NEON_Q | NEONScalar;
+ format = SFormat(vd);
+ } else {
+ format = VFormat(vd);
+ }
+
+ Emit(format | op | Rn(vn) | Rd(vd));
+}
+
+void Assembler::cmeq(const VRegister& vd, const VRegister& vn, int value) {
+ DCHECK(vd.IsVector() || vd.Is1D());
+ NEON2RegMisc(vd, vn, NEON_CMEQ_zero, value);
+}
+
+void Assembler::cmge(const VRegister& vd, const VRegister& vn, int value) {
+ DCHECK(vd.IsVector() || vd.Is1D());
+ NEON2RegMisc(vd, vn, NEON_CMGE_zero, value);
+}
+
+void Assembler::cmgt(const VRegister& vd, const VRegister& vn, int value) {
+ DCHECK(vd.IsVector() || vd.Is1D());
+ NEON2RegMisc(vd, vn, NEON_CMGT_zero, value);
+}
+
+void Assembler::cmle(const VRegister& vd, const VRegister& vn, int value) {
+ DCHECK(vd.IsVector() || vd.Is1D());
+ NEON2RegMisc(vd, vn, NEON_CMLE_zero, value);
+}
+
+void Assembler::cmlt(const VRegister& vd, const VRegister& vn, int value) {
+ DCHECK(vd.IsVector() || vd.Is1D());
+ NEON2RegMisc(vd, vn, NEON_CMLT_zero, value);
+}
+
+#define NEON_3SAME_LIST(V) \
+ V(add, NEON_ADD, vd.IsVector() || vd.Is1D()) \
+ V(addp, NEON_ADDP, vd.IsVector() || vd.Is1D()) \
+ V(sub, NEON_SUB, vd.IsVector() || vd.Is1D()) \
+ V(cmeq, NEON_CMEQ, vd.IsVector() || vd.Is1D()) \
+ V(cmge, NEON_CMGE, vd.IsVector() || vd.Is1D()) \
+ V(cmgt, NEON_CMGT, vd.IsVector() || vd.Is1D()) \
+ V(cmhi, NEON_CMHI, vd.IsVector() || vd.Is1D()) \
+ V(cmhs, NEON_CMHS, vd.IsVector() || vd.Is1D()) \
+ V(cmtst, NEON_CMTST, vd.IsVector() || vd.Is1D()) \
+ V(sshl, NEON_SSHL, vd.IsVector() || vd.Is1D()) \
+ V(ushl, NEON_USHL, vd.IsVector() || vd.Is1D()) \
+ V(srshl, NEON_SRSHL, vd.IsVector() || vd.Is1D()) \
+ V(urshl, NEON_URSHL, vd.IsVector() || vd.Is1D()) \
+ V(sqdmulh, NEON_SQDMULH, vd.IsLaneSizeH() || vd.IsLaneSizeS()) \
+ V(sqrdmulh, NEON_SQRDMULH, vd.IsLaneSizeH() || vd.IsLaneSizeS()) \
+ V(shadd, NEON_SHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(uhadd, NEON_UHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(srhadd, NEON_SRHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(urhadd, NEON_URHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(shsub, NEON_SHSUB, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(uhsub, NEON_UHSUB, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(smax, NEON_SMAX, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(smaxp, NEON_SMAXP, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(smin, NEON_SMIN, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(sminp, NEON_SMINP, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(umax, NEON_UMAX, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(umaxp, NEON_UMAXP, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(umin, NEON_UMIN, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(uminp, NEON_UMINP, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(saba, NEON_SABA, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(sabd, NEON_SABD, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(uaba, NEON_UABA, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(uabd, NEON_UABD, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(mla, NEON_MLA, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(mls, NEON_MLS, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(mul, NEON_MUL, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(and_, NEON_AND, vd.Is8B() || vd.Is16B()) \
+ V(orr, NEON_ORR, vd.Is8B() || vd.Is16B()) \
+ V(orn, NEON_ORN, vd.Is8B() || vd.Is16B()) \
+ V(eor, NEON_EOR, vd.Is8B() || vd.Is16B()) \
+ V(bic, NEON_BIC, vd.Is8B() || vd.Is16B()) \
+ V(bit, NEON_BIT, vd.Is8B() || vd.Is16B()) \
+ V(bif, NEON_BIF, vd.Is8B() || vd.Is16B()) \
+ V(bsl, NEON_BSL, vd.Is8B() || vd.Is16B()) \
+ V(pmul, NEON_PMUL, vd.Is8B() || vd.Is16B()) \
+ V(uqadd, NEON_UQADD, true) \
+ V(sqadd, NEON_SQADD, true) \
+ V(uqsub, NEON_UQSUB, true) \
+ V(sqsub, NEON_SQSUB, true) \
+ V(sqshl, NEON_SQSHL, true) \
+ V(uqshl, NEON_UQSHL, true) \
+ V(sqrshl, NEON_SQRSHL, true) \
+ V(uqrshl, NEON_UQRSHL, true)
+
+#define DEFINE_ASM_FUNC(FN, OP, AS) \
+ void Assembler::FN(const VRegister& vd, const VRegister& vn, \
+ const VRegister& vm) { \
+ DCHECK(AS); \
+ NEON3Same(vd, vn, vm, OP); \
+ }
+NEON_3SAME_LIST(DEFINE_ASM_FUNC)
+#undef DEFINE_ASM_FUNC
+
+#define NEON_FP3SAME_LIST_V2(V) \
+ V(fadd, NEON_FADD, FADD) \
+ V(fsub, NEON_FSUB, FSUB) \
+ V(fmul, NEON_FMUL, FMUL) \
+ V(fdiv, NEON_FDIV, FDIV) \
+ V(fmax, NEON_FMAX, FMAX) \
+ V(fmaxnm, NEON_FMAXNM, FMAXNM) \
+ V(fmin, NEON_FMIN, FMIN) \
+ V(fminnm, NEON_FMINNM, FMINNM) \
+ V(fmulx, NEON_FMULX, NEON_FMULX_scalar) \
+ V(frecps, NEON_FRECPS, NEON_FRECPS_scalar) \
+ V(frsqrts, NEON_FRSQRTS, NEON_FRSQRTS_scalar) \
+ V(fabd, NEON_FABD, NEON_FABD_scalar) \
+ V(fmla, NEON_FMLA, 0) \
+ V(fmls, NEON_FMLS, 0) \
+ V(facge, NEON_FACGE, NEON_FACGE_scalar) \
+ V(facgt, NEON_FACGT, NEON_FACGT_scalar) \
+ V(fcmeq, NEON_FCMEQ, NEON_FCMEQ_scalar) \
+ V(fcmge, NEON_FCMGE, NEON_FCMGE_scalar) \
+ V(fcmgt, NEON_FCMGT, NEON_FCMGT_scalar) \
+ V(faddp, NEON_FADDP, 0) \
+ V(fmaxp, NEON_FMAXP, 0) \
+ V(fminp, NEON_FMINP, 0) \
+ V(fmaxnmp, NEON_FMAXNMP, 0) \
+ V(fminnmp, NEON_FMINNMP, 0)
+
+#define DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP) \
+ void Assembler::FN(const VRegister& vd, const VRegister& vn, \
+ const VRegister& vm) { \
+ Instr op; \
+ if ((SCA_OP != 0) && vd.IsScalar()) { \
+ DCHECK(vd.Is1S() || vd.Is1D()); \
+ op = SCA_OP; \
+ } else { \
+ DCHECK(vd.IsVector()); \
+ DCHECK(vd.Is2S() || vd.Is2D() || vd.Is4S()); \
+ op = VEC_OP; \
+ } \
+ NEONFP3Same(vd, vn, vm, op); \
+ }
+NEON_FP3SAME_LIST_V2(DEFINE_ASM_FUNC)
+#undef DEFINE_ASM_FUNC
+
+void Assembler::addp(const VRegister& vd, const VRegister& vn) {
+ DCHECK((vd.Is1D() && vn.Is2D()));
+ Emit(SFormat(vd) | NEON_ADDP_scalar | Rn(vn) | Rd(vd));
+}
+
+void Assembler::faddp(const VRegister& vd, const VRegister& vn) {
+ DCHECK((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()));
+ Emit(FPFormat(vd) | NEON_FADDP_scalar | Rn(vn) | Rd(vd));
+}
+
+void Assembler::fmaxp(const VRegister& vd, const VRegister& vn) {
+ DCHECK((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()));
+ Emit(FPFormat(vd) | NEON_FMAXP_scalar | Rn(vn) | Rd(vd));
+}
+
+void Assembler::fminp(const VRegister& vd, const VRegister& vn) {
+ DCHECK((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()));
+ Emit(FPFormat(vd) | NEON_FMINP_scalar | Rn(vn) | Rd(vd));
+}
+
+void Assembler::fmaxnmp(const VRegister& vd, const VRegister& vn) {
+ DCHECK((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()));
+ Emit(FPFormat(vd) | NEON_FMAXNMP_scalar | Rn(vn) | Rd(vd));
+}
+
+void Assembler::fminnmp(const VRegister& vd, const VRegister& vn) {
+ DCHECK((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()));
+ Emit(FPFormat(vd) | NEON_FMINNMP_scalar | Rn(vn) | Rd(vd));
+}
+
+void Assembler::orr(const VRegister& vd, const int imm8, const int left_shift) {
+ NEONModifiedImmShiftLsl(vd, imm8, left_shift, NEONModifiedImmediate_ORR);
+}
+
+void Assembler::mov(const VRegister& vd, const VRegister& vn) {
+ DCHECK(AreSameFormat(vd, vn));
+ if (vd.IsD()) {
+ orr(vd.V8B(), vn.V8B(), vn.V8B());
+ } else {
+ DCHECK(vd.IsQ());
+ orr(vd.V16B(), vn.V16B(), vn.V16B());
+ }
+}
+
+void Assembler::bic(const VRegister& vd, const int imm8, const int left_shift) {
+ NEONModifiedImmShiftLsl(vd, imm8, left_shift, NEONModifiedImmediate_BIC);
+}
+
+void Assembler::movi(const VRegister& vd, const uint64_t imm, Shift shift,
+ const int shift_amount) {
+ DCHECK((shift == LSL) || (shift == MSL));
+ if (vd.Is2D() || vd.Is1D()) {
+ DCHECK_EQ(shift_amount, 0);
+ int imm8 = 0;
+ for (int i = 0; i < 8; ++i) {
+ int byte = (imm >> (i * 8)) & 0xFF;
+ DCHECK((byte == 0) || (byte == 0xFF));
+ if (byte == 0xFF) {
+ imm8 |= (1 << i);
+ }
+ }
+ Instr q = vd.Is2D() ? NEON_Q : 0;
+ Emit(q | NEONModImmOp(1) | NEONModifiedImmediate_MOVI |
+ ImmNEONabcdefgh(imm8) | NEONCmode(0xE) | Rd(vd));
+ } else if (shift == LSL) {
+ DCHECK(is_uint8(imm));
+ NEONModifiedImmShiftLsl(vd, static_cast<int>(imm), shift_amount,
+ NEONModifiedImmediate_MOVI);
+ } else {
+ DCHECK(is_uint8(imm));
+ NEONModifiedImmShiftMsl(vd, static_cast<int>(imm), shift_amount,
+ NEONModifiedImmediate_MOVI);
+ }
+}
+
+void Assembler::mvn(const VRegister& vd, const VRegister& vn) {
+ DCHECK(AreSameFormat(vd, vn));
+ if (vd.IsD()) {
+ not_(vd.V8B(), vn.V8B());
+ } else {
+ DCHECK(vd.IsQ());
+ not_(vd.V16B(), vn.V16B());
+ }
+}
+
+void Assembler::mvni(const VRegister& vd, const int imm8, Shift shift,
+ const int shift_amount) {
+ DCHECK((shift == LSL) || (shift == MSL));
+ if (shift == LSL) {
+ NEONModifiedImmShiftLsl(vd, imm8, shift_amount, NEONModifiedImmediate_MVNI);
+ } else {
+ NEONModifiedImmShiftMsl(vd, imm8, shift_amount, NEONModifiedImmediate_MVNI);
+ }
+}
+
+void Assembler::NEONFPByElement(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm, int vm_index,
+ NEONByIndexedElementOp vop) {
+ DCHECK(AreSameFormat(vd, vn));
+ DCHECK((vd.Is2S() && vm.Is1S()) || (vd.Is4S() && vm.Is1S()) ||
+ (vd.Is1S() && vm.Is1S()) || (vd.Is2D() && vm.Is1D()) ||
+ (vd.Is1D() && vm.Is1D()));
+ DCHECK((vm.Is1S() && (vm_index < 4)) || (vm.Is1D() && (vm_index < 2)));
+
+ Instr op = vop;
+ int index_num_bits = vm.Is1S() ? 2 : 1;
+ if (vd.IsScalar()) {
+ op |= NEON_Q | NEONScalar;
+ }
+
+ Emit(FPFormat(vd) | op | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) |
+ Rn(vn) | Rd(vd));
+}
+
+void Assembler::NEONByElement(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm, int vm_index,
+ NEONByIndexedElementOp vop) {
+ DCHECK(AreSameFormat(vd, vn));
+ DCHECK((vd.Is4H() && vm.Is1H()) || (vd.Is8H() && vm.Is1H()) ||
+ (vd.Is1H() && vm.Is1H()) || (vd.Is2S() && vm.Is1S()) ||
+ (vd.Is4S() && vm.Is1S()) || (vd.Is1S() && vm.Is1S()));
+ DCHECK((vm.Is1H() && (vm.code() < 16) && (vm_index < 8)) ||
+ (vm.Is1S() && (vm_index < 4)));
+
+ Instr format, op = vop;
+ int index_num_bits = vm.Is1H() ? 3 : 2;
+ if (vd.IsScalar()) {
+ op |= NEONScalar | NEON_Q;
+ format = SFormat(vn);
+ } else {
+ format = VFormat(vn);
+ }
+ Emit(format | op | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) |
+ Rd(vd));
+}
+
+void Assembler::NEONByElementL(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm, int vm_index,
+ NEONByIndexedElementOp vop) {
+ DCHECK((vd.Is4S() && vn.Is4H() && vm.Is1H()) ||
+ (vd.Is4S() && vn.Is8H() && vm.Is1H()) ||
+ (vd.Is1S() && vn.Is1H() && vm.Is1H()) ||
+ (vd.Is2D() && vn.Is2S() && vm.Is1S()) ||
+ (vd.Is2D() && vn.Is4S() && vm.Is1S()) ||
+ (vd.Is1D() && vn.Is1S() && vm.Is1S()));
+
+ DCHECK((vm.Is1H() && (vm.code() < 16) && (vm_index < 8)) ||
+ (vm.Is1S() && (vm_index < 4)));
+
+ Instr format, op = vop;
+ int index_num_bits = vm.Is1H() ? 3 : 2;
+ if (vd.IsScalar()) {
+ op |= NEONScalar | NEON_Q;
+ format = SFormat(vn);
+ } else {
+ format = VFormat(vn);
+ }
+ Emit(format | op | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) |
+ Rd(vd));
+}
+
+#define NEON_BYELEMENT_LIST(V) \
+ V(mul, NEON_MUL_byelement, vn.IsVector()) \
+ V(mla, NEON_MLA_byelement, vn.IsVector()) \
+ V(mls, NEON_MLS_byelement, vn.IsVector()) \
+ V(sqdmulh, NEON_SQDMULH_byelement, true) \
+ V(sqrdmulh, NEON_SQRDMULH_byelement, true)
+
+#define DEFINE_ASM_FUNC(FN, OP, AS) \
+ void Assembler::FN(const VRegister& vd, const VRegister& vn, \
+ const VRegister& vm, int vm_index) { \
+ DCHECK(AS); \
+ NEONByElement(vd, vn, vm, vm_index, OP); \
+ }
+NEON_BYELEMENT_LIST(DEFINE_ASM_FUNC)
+#undef DEFINE_ASM_FUNC
+
+#define NEON_FPBYELEMENT_LIST(V) \
+ V(fmul, NEON_FMUL_byelement) \
+ V(fmla, NEON_FMLA_byelement) \
+ V(fmls, NEON_FMLS_byelement) \
+ V(fmulx, NEON_FMULX_byelement)
+
+#define DEFINE_ASM_FUNC(FN, OP) \
+ void Assembler::FN(const VRegister& vd, const VRegister& vn, \
+ const VRegister& vm, int vm_index) { \
+ NEONFPByElement(vd, vn, vm, vm_index, OP); \
+ }
+NEON_FPBYELEMENT_LIST(DEFINE_ASM_FUNC)
+#undef DEFINE_ASM_FUNC
+
+#define NEON_BYELEMENT_LONG_LIST(V) \
+ V(sqdmull, NEON_SQDMULL_byelement, vn.IsScalar() || vn.IsD()) \
+ V(sqdmull2, NEON_SQDMULL_byelement, vn.IsVector() && vn.IsQ()) \
+ V(sqdmlal, NEON_SQDMLAL_byelement, vn.IsScalar() || vn.IsD()) \
+ V(sqdmlal2, NEON_SQDMLAL_byelement, vn.IsVector() && vn.IsQ()) \
+ V(sqdmlsl, NEON_SQDMLSL_byelement, vn.IsScalar() || vn.IsD()) \
+ V(sqdmlsl2, NEON_SQDMLSL_byelement, vn.IsVector() && vn.IsQ()) \
+ V(smull, NEON_SMULL_byelement, vn.IsVector() && vn.IsD()) \
+ V(smull2, NEON_SMULL_byelement, vn.IsVector() && vn.IsQ()) \
+ V(umull, NEON_UMULL_byelement, vn.IsVector() && vn.IsD()) \
+ V(umull2, NEON_UMULL_byelement, vn.IsVector() && vn.IsQ()) \
+ V(smlal, NEON_SMLAL_byelement, vn.IsVector() && vn.IsD()) \
+ V(smlal2, NEON_SMLAL_byelement, vn.IsVector() && vn.IsQ()) \
+ V(umlal, NEON_UMLAL_byelement, vn.IsVector() && vn.IsD()) \
+ V(umlal2, NEON_UMLAL_byelement, vn.IsVector() && vn.IsQ()) \
+ V(smlsl, NEON_SMLSL_byelement, vn.IsVector() && vn.IsD()) \
+ V(smlsl2, NEON_SMLSL_byelement, vn.IsVector() && vn.IsQ()) \
+ V(umlsl, NEON_UMLSL_byelement, vn.IsVector() && vn.IsD()) \
+ V(umlsl2, NEON_UMLSL_byelement, vn.IsVector() && vn.IsQ())
+
+#define DEFINE_ASM_FUNC(FN, OP, AS) \
+ void Assembler::FN(const VRegister& vd, const VRegister& vn, \
+ const VRegister& vm, int vm_index) { \
+ DCHECK(AS); \
+ NEONByElementL(vd, vn, vm, vm_index, OP); \
+ }
+NEON_BYELEMENT_LONG_LIST(DEFINE_ASM_FUNC)
+#undef DEFINE_ASM_FUNC
+
+void Assembler::suqadd(const VRegister& vd, const VRegister& vn) {
+ NEON2RegMisc(vd, vn, NEON_SUQADD);
+}
+
+void Assembler::usqadd(const VRegister& vd, const VRegister& vn) {
+ NEON2RegMisc(vd, vn, NEON_USQADD);
+}
+
+void Assembler::abs(const VRegister& vd, const VRegister& vn) {
+ DCHECK(vd.IsVector() || vd.Is1D());
+ NEON2RegMisc(vd, vn, NEON_ABS);
+}
+
+void Assembler::sqabs(const VRegister& vd, const VRegister& vn) {
+ NEON2RegMisc(vd, vn, NEON_SQABS);
+}
+
+void Assembler::neg(const VRegister& vd, const VRegister& vn) {
+ DCHECK(vd.IsVector() || vd.Is1D());
+ NEON2RegMisc(vd, vn, NEON_NEG);
+}
+
+void Assembler::sqneg(const VRegister& vd, const VRegister& vn) {
+ NEON2RegMisc(vd, vn, NEON_SQNEG);
+}
+
+void Assembler::NEONXtn(const VRegister& vd, const VRegister& vn,
+ NEON2RegMiscOp vop) {
+ Instr format, op = vop;
+ if (vd.IsScalar()) {
+ DCHECK((vd.Is1B() && vn.Is1H()) || (vd.Is1H() && vn.Is1S()) ||
+ (vd.Is1S() && vn.Is1D()));
+ op |= NEON_Q | NEONScalar;
+ format = SFormat(vd);
+ } else {
+ DCHECK((vd.Is8B() && vn.Is8H()) || (vd.Is4H() && vn.Is4S()) ||
+ (vd.Is2S() && vn.Is2D()) || (vd.Is16B() && vn.Is8H()) ||
+ (vd.Is8H() && vn.Is4S()) || (vd.Is4S() && vn.Is2D()));
+ format = VFormat(vd);
+ }
+ Emit(format | op | Rn(vn) | Rd(vd));
+}
+
+void Assembler::xtn(const VRegister& vd, const VRegister& vn) {
+ DCHECK(vd.IsVector() && vd.IsD());
+ NEONXtn(vd, vn, NEON_XTN);
+}
+
+void Assembler::xtn2(const VRegister& vd, const VRegister& vn) {
+ DCHECK(vd.IsVector() && vd.IsQ());
+ NEONXtn(vd, vn, NEON_XTN);
+}
+
+void Assembler::sqxtn(const VRegister& vd, const VRegister& vn) {
+ DCHECK(vd.IsScalar() || vd.IsD());
+ NEONXtn(vd, vn, NEON_SQXTN);
+}
+
+void Assembler::sqxtn2(const VRegister& vd, const VRegister& vn) {
+ DCHECK(vd.IsVector() && vd.IsQ());
+ NEONXtn(vd, vn, NEON_SQXTN);
+}
+
+void Assembler::sqxtun(const VRegister& vd, const VRegister& vn) {
+ DCHECK(vd.IsScalar() || vd.IsD());
+ NEONXtn(vd, vn, NEON_SQXTUN);
+}
+
+void Assembler::sqxtun2(const VRegister& vd, const VRegister& vn) {
+ DCHECK(vd.IsVector() && vd.IsQ());
+ NEONXtn(vd, vn, NEON_SQXTUN);
+}
+
+void Assembler::uqxtn(const VRegister& vd, const VRegister& vn) {
+ DCHECK(vd.IsScalar() || vd.IsD());
+ NEONXtn(vd, vn, NEON_UQXTN);
+}
+
+void Assembler::uqxtn2(const VRegister& vd, const VRegister& vn) {
+ DCHECK(vd.IsVector() && vd.IsQ());
+ NEONXtn(vd, vn, NEON_UQXTN);
+}
+
+// NEON NOT and RBIT are distinguised by bit 22, the bottom bit of "size".
+void Assembler::not_(const VRegister& vd, const VRegister& vn) {
+ DCHECK(AreSameFormat(vd, vn));
+ DCHECK(vd.Is8B() || vd.Is16B());
+ Emit(VFormat(vd) | NEON_RBIT_NOT | Rn(vn) | Rd(vd));
+}
+
+void Assembler::rbit(const VRegister& vd, const VRegister& vn) {
+ DCHECK(AreSameFormat(vd, vn));
+ DCHECK(vd.Is8B() || vd.Is16B());
+ Emit(VFormat(vn) | (1 << NEONSize_offset) | NEON_RBIT_NOT | Rn(vn) | Rd(vd));
+}
+
+void Assembler::ext(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm, int index) {
+ DCHECK(AreSameFormat(vd, vn, vm));
+ DCHECK(vd.Is8B() || vd.Is16B());
+ DCHECK((0 <= index) && (index < vd.LaneCount()));
+ Emit(VFormat(vd) | NEON_EXT | Rm(vm) | ImmNEONExt(index) | Rn(vn) | Rd(vd));
+}
+
+void Assembler::dup(const VRegister& vd, const VRegister& vn, int vn_index) {
+ Instr q, scalar;
+
+ // We support vn arguments of the form vn.VxT() or vn.T(), where x is the
+ // number of lanes, and T is b, h, s or d.
+ int lane_size = vn.LaneSizeInBytes();
+ NEONFormatField format;
+ switch (lane_size) {
+ case 1:
+ format = NEON_16B;
+ break;
+ case 2:
+ format = NEON_8H;
+ break;
+ case 4:
+ format = NEON_4S;
+ break;
+ default:
+ DCHECK_EQ(lane_size, 8);
+ format = NEON_2D;
+ break;
+ }
+
+ if (vd.IsScalar()) {
+ q = NEON_Q;
+ scalar = NEONScalar;
+ } else {
+ DCHECK(!vd.Is1D());
+ q = vd.IsD() ? 0 : NEON_Q;
+ scalar = 0;
+ }
+ Emit(q | scalar | NEON_DUP_ELEMENT | ImmNEON5(format, vn_index) | Rn(vn) |
+ Rd(vd));
+}
+
+void Assembler::dcptr(Label* label) {
+ BlockPoolsScope no_pool_inbetween(this);
+ RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE);
+ if (label->is_bound()) {
+ // The label is bound, so it does not need to be updated and the internal
+ // reference should be emitted.
+ //
+ // In this case, label->pos() returns the offset of the label from the
+ // start of the buffer.
+ internal_reference_positions_.push_back(pc_offset());
+ dc64(reinterpret_cast<uintptr_t>(buffer_start_ + label->pos()));
+ } else {
+ int32_t offset;
+ if (label->is_linked()) {
+ // The label is linked, so the internal reference should be added
+ // onto the end of the label's link chain.
+ //
+ // In this case, label->pos() returns the offset of the last linked
+ // instruction from the start of the buffer.
+ offset = label->pos() - pc_offset();
+ DCHECK_NE(offset, kStartOfLabelLinkChain);
+ } else {
+ // The label is unused, so it now becomes linked and the internal
+ // reference is at the start of the new link chain.
+ offset = kStartOfLabelLinkChain;
+ }
+ // The instruction at pc is now the last link in the label's chain.
+ label->link_to(pc_offset());
+
+ // Traditionally the offset to the previous instruction in the chain is
+ // encoded in the instruction payload (e.g. branch range) but internal
+ // references are not instructions so while unbound they are encoded as
+ // two consecutive brk instructions. The two 16-bit immediates are used
+ // to encode the offset.
+ offset >>= kInstrSizeLog2;
+ DCHECK(is_int32(offset));
+ uint32_t high16 = unsigned_bitextract_32(31, 16, offset);
+ uint32_t low16 = unsigned_bitextract_32(15, 0, offset);
+
+ brk(high16);
+ brk(low16);
+ }
+}
+
+// Below, a difference in case for the same letter indicates a
+// negated bit. If b is 1, then B is 0.
+uint32_t Assembler::FPToImm8(double imm) {
+ DCHECK(IsImmFP64(imm));
+ // bits: aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
+ // 0000.0000.0000.0000.0000.0000.0000.0000
+ uint64_t bits = bit_cast<uint64_t>(imm);
+ // bit7: a000.0000
+ uint64_t bit7 = ((bits >> 63) & 0x1) << 7;
+ // bit6: 0b00.0000
+ uint64_t bit6 = ((bits >> 61) & 0x1) << 6;
+ // bit5_to_0: 00cd.efgh
+ uint64_t bit5_to_0 = (bits >> 48) & 0x3F;
+
+ return static_cast<uint32_t>(bit7 | bit6 | bit5_to_0);
+}
+
+Instr Assembler::ImmFP(double imm) { return FPToImm8(imm) << ImmFP_offset; }
+Instr Assembler::ImmNEONFP(double imm) {
+ return ImmNEONabcdefgh(FPToImm8(imm));
+}
+
+// Code generation helpers.
+void Assembler::MoveWide(const Register& rd, uint64_t imm, int shift,
+ MoveWideImmediateOp mov_op) {
+ // Ignore the top 32 bits of an immediate if we're moving to a W register.
+ if (rd.Is32Bits()) {
+ // Check that the top 32 bits are zero (a positive 32-bit number) or top
+ // 33 bits are one (a negative 32-bit number, sign extended to 64 bits).
+ DCHECK(((imm >> kWRegSizeInBits) == 0) ||
+ ((imm >> (kWRegSizeInBits - 1)) == 0x1FFFFFFFF));
+ imm &= kWRegMask;
+ }
+
+ if (shift >= 0) {
+ // Explicit shift specified.
+ DCHECK((shift == 0) || (shift == 16) || (shift == 32) || (shift == 48));
+ DCHECK(rd.Is64Bits() || (shift == 0) || (shift == 16));
+ shift /= 16;
+ } else {
+ // Calculate a new immediate and shift combination to encode the immediate
+ // argument.
+ shift = 0;
+ if ((imm & ~0xFFFFULL) == 0) {
+ // Nothing to do.
+ } else if ((imm & ~(0xFFFFULL << 16)) == 0) {
+ imm >>= 16;
+ shift = 1;
+ } else if ((imm & ~(0xFFFFULL << 32)) == 0) {
+ DCHECK(rd.Is64Bits());
+ imm >>= 32;
+ shift = 2;
+ } else if ((imm & ~(0xFFFFULL << 48)) == 0) {
+ DCHECK(rd.Is64Bits());
+ imm >>= 48;
+ shift = 3;
+ }
+ }
+
+ DCHECK(is_uint16(imm));
+
+ Emit(SF(rd) | MoveWideImmediateFixed | mov_op | Rd(rd) |
+ ImmMoveWide(static_cast<int>(imm)) | ShiftMoveWide(shift));
+}
+
+void Assembler::AddSub(const Register& rd, const Register& rn,
+ const Operand& operand, FlagsUpdate S, AddSubOp op) {
+ DCHECK_EQ(rd.SizeInBits(), rn.SizeInBits());
+ DCHECK(!operand.NeedsRelocation(this));
+ if (operand.IsImmediate()) {
+ int64_t immediate = operand.ImmediateValue();
+ DCHECK(IsImmAddSub(immediate));
+ Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd);
+ Emit(SF(rd) | AddSubImmediateFixed | op | Flags(S) |
+ ImmAddSub(static_cast<int>(immediate)) | dest_reg | RnSP(rn));
+ } else if (operand.IsShiftedRegister()) {
+ DCHECK_EQ(operand.reg().SizeInBits(), rd.SizeInBits());
+ DCHECK_NE(operand.shift(), ROR);
+
+ // For instructions of the form:
+ // add/sub wsp, <Wn>, <Wm> [, LSL #0-3 ]
+ // add/sub <Wd>, wsp, <Wm> [, LSL #0-3 ]
+ // add/sub wsp, wsp, <Wm> [, LSL #0-3 ]
+ // adds/subs <Wd>, wsp, <Wm> [, LSL #0-3 ]
+ // or their 64-bit register equivalents, convert the operand from shifted to
+ // extended register mode, and emit an add/sub extended instruction.
+ if (rn.IsSP() || rd.IsSP()) {
+ DCHECK(!(rd.IsSP() && (S == SetFlags)));
+ DataProcExtendedRegister(rd, rn, operand.ToExtendedRegister(), S,
+ AddSubExtendedFixed | op);
+ } else {
+ DataProcShiftedRegister(rd, rn, operand, S, AddSubShiftedFixed | op);
+ }
+ } else {
+ DCHECK(operand.IsExtendedRegister());
+ DataProcExtendedRegister(rd, rn, operand, S, AddSubExtendedFixed | op);
+ }
+}
+
+void Assembler::AddSubWithCarry(const Register& rd, const Register& rn,
+ const Operand& operand, FlagsUpdate S,
+ AddSubWithCarryOp op) {
+ DCHECK_EQ(rd.SizeInBits(), rn.SizeInBits());
+ DCHECK_EQ(rd.SizeInBits(), operand.reg().SizeInBits());
+ DCHECK(operand.IsShiftedRegister() && (operand.shift_amount() == 0));
+ DCHECK(!operand.NeedsRelocation(this));
+ Emit(SF(rd) | op | Flags(S) | Rm(operand.reg()) | Rn(rn) | Rd(rd));
+}
+
+void Assembler::hlt(int code) {
+ DCHECK(is_uint16(code));
+ Emit(HLT | ImmException(code));
+}
+
+void Assembler::brk(int code) {
+ DCHECK(is_uint16(code));
+ Emit(BRK | ImmException(code));
+}
+
+void Assembler::EmitStringData(const char* string) {
+ size_t len = strlen(string) + 1;
+ DCHECK_LE(RoundUp(len, kInstrSize), static_cast<size_t>(kGap));
+ EmitData(string, static_cast<int>(len));
+ // Pad with nullptr characters until pc_ is aligned.
+ const char pad[] = {'\0', '\0', '\0', '\0'};
+ static_assert(sizeof(pad) == kInstrSize,
+ "Size of padding must match instruction size.");
+ EmitData(pad, RoundUp(pc_offset(), kInstrSize) - pc_offset());
+}
+
+void Assembler::debug(const char* message, uint32_t code, Instr params) {
+ if (options().enable_simulator_code) {
+ // The arguments to the debug marker need to be contiguous in memory, so
+ // make sure we don't try to emit pools.
+ BlockPoolsScope scope(this);
+
+ Label start;
+ bind(&start);
+
+ // Refer to instructions-arm64.h for a description of the marker and its
+ // arguments.
+ hlt(kImmExceptionIsDebug);
+ DCHECK_EQ(SizeOfCodeGeneratedSince(&start), kDebugCodeOffset);
+ dc32(code);
+ DCHECK_EQ(SizeOfCodeGeneratedSince(&start), kDebugParamsOffset);
+ dc32(params);
+ DCHECK_EQ(SizeOfCodeGeneratedSince(&start), kDebugMessageOffset);
+ EmitStringData(message);
+ hlt(kImmExceptionIsUnreachable);
+
+ return;
+ }
+
+ if (params & BREAK) {
+ brk(0);
+ }
+}
+
+void Assembler::Logical(const Register& rd, const Register& rn,
+ const Operand& operand, LogicalOp op) {
+ DCHECK(rd.SizeInBits() == rn.SizeInBits());
+ DCHECK(!operand.NeedsRelocation(this));
+ if (operand.IsImmediate()) {
+ int64_t immediate = operand.ImmediateValue();
+ unsigned reg_size = rd.SizeInBits();
+
+ DCHECK_NE(immediate, 0);
+ DCHECK_NE(immediate, -1);
+ DCHECK(rd.Is64Bits() || is_uint32(immediate));
+
+ // If the operation is NOT, invert the operation and immediate.
+ if ((op & NOT) == NOT) {
+ op = static_cast<LogicalOp>(op & ~NOT);
+ immediate = rd.Is64Bits() ? ~immediate : (~immediate & kWRegMask);
+ }
+
+ unsigned n, imm_s, imm_r;
+ if (IsImmLogical(immediate, reg_size, &n, &imm_s, &imm_r)) {
+ // Immediate can be encoded in the instruction.
+ LogicalImmediate(rd, rn, n, imm_s, imm_r, op);
+ } else {
+ // This case is handled in the macro assembler.
+ UNREACHABLE();
+ }
+ } else {
+ DCHECK(operand.IsShiftedRegister());
+ DCHECK(operand.reg().SizeInBits() == rd.SizeInBits());
+ Instr dp_op = static_cast<Instr>(op | LogicalShiftedFixed);
+ DataProcShiftedRegister(rd, rn, operand, LeaveFlags, dp_op);
+ }
+}
+
+void Assembler::LogicalImmediate(const Register& rd, const Register& rn,
+ unsigned n, unsigned imm_s, unsigned imm_r,
+ LogicalOp op) {
+ unsigned reg_size = rd.SizeInBits();
+ Instr dest_reg = (op == ANDS) ? Rd(rd) : RdSP(rd);
+ Emit(SF(rd) | LogicalImmediateFixed | op | BitN(n, reg_size) |
+ ImmSetBits(imm_s, reg_size) | ImmRotate(imm_r, reg_size) | dest_reg |
+ Rn(rn));
+}
+
+void Assembler::ConditionalCompare(const Register& rn, const Operand& operand,
+ StatusFlags nzcv, Condition cond,
+ ConditionalCompareOp op) {
+ Instr ccmpop;
+ DCHECK(!operand.NeedsRelocation(this));
+ if (operand.IsImmediate()) {
+ int64_t immediate = operand.ImmediateValue();
+ DCHECK(IsImmConditionalCompare(immediate));
+ ccmpop = ConditionalCompareImmediateFixed | op |
+ ImmCondCmp(static_cast<unsigned>(immediate));
+ } else {
+ DCHECK(operand.IsShiftedRegister() && (operand.shift_amount() == 0));
+ ccmpop = ConditionalCompareRegisterFixed | op | Rm(operand.reg());
+ }
+ Emit(SF(rn) | ccmpop | Cond(cond) | Rn(rn) | Nzcv(nzcv));
+}
+
+void Assembler::DataProcessing1Source(const Register& rd, const Register& rn,
+ DataProcessing1SourceOp op) {
+ DCHECK(rd.SizeInBits() == rn.SizeInBits());
+ Emit(SF(rn) | op | Rn(rn) | Rd(rd));
+}
+
+void Assembler::FPDataProcessing1Source(const VRegister& vd,
+ const VRegister& vn,
+ FPDataProcessing1SourceOp op) {
+ Emit(FPType(vn) | op | Rn(vn) | Rd(vd));
+}
+
+void Assembler::FPDataProcessing2Source(const VRegister& fd,
+ const VRegister& fn,
+ const VRegister& fm,
+ FPDataProcessing2SourceOp op) {
+ DCHECK(fd.SizeInBits() == fn.SizeInBits());
+ DCHECK(fd.SizeInBits() == fm.SizeInBits());
+ Emit(FPType(fd) | op | Rm(fm) | Rn(fn) | Rd(fd));
+}
+
+void Assembler::FPDataProcessing3Source(const VRegister& fd,
+ const VRegister& fn,
+ const VRegister& fm,
+ const VRegister& fa,
+ FPDataProcessing3SourceOp op) {
+ DCHECK(AreSameSizeAndType(fd, fn, fm, fa));
+ Emit(FPType(fd) | op | Rm(fm) | Rn(fn) | Rd(fd) | Ra(fa));
+}
+
+void Assembler::NEONModifiedImmShiftLsl(const VRegister& vd, const int imm8,
+ const int left_shift,
+ NEONModifiedImmediateOp op) {
+ DCHECK(vd.Is8B() || vd.Is16B() || vd.Is4H() || vd.Is8H() || vd.Is2S() ||
+ vd.Is4S());
+ DCHECK((left_shift == 0) || (left_shift == 8) || (left_shift == 16) ||
+ (left_shift == 24));
+ DCHECK(is_uint8(imm8));
+
+ int cmode_1, cmode_2, cmode_3;
+ if (vd.Is8B() || vd.Is16B()) {
+ DCHECK_EQ(op, NEONModifiedImmediate_MOVI);
+ cmode_1 = 1;
+ cmode_2 = 1;
+ cmode_3 = 1;
+ } else {
+ cmode_1 = (left_shift >> 3) & 1;
+ cmode_2 = left_shift >> 4;
+ cmode_3 = 0;
+ if (vd.Is4H() || vd.Is8H()) {
+ DCHECK((left_shift == 0) || (left_shift == 8));
+ cmode_3 = 1;
+ }
+ }
+ int cmode = (cmode_3 << 3) | (cmode_2 << 2) | (cmode_1 << 1);
+
+ Instr q = vd.IsQ() ? NEON_Q : 0;
+
+ Emit(q | op | ImmNEONabcdefgh(imm8) | NEONCmode(cmode) | Rd(vd));
+}
+
+void Assembler::NEONModifiedImmShiftMsl(const VRegister& vd, const int imm8,
+ const int shift_amount,
+ NEONModifiedImmediateOp op) {
+ DCHECK(vd.Is2S() || vd.Is4S());
+ DCHECK((shift_amount == 8) || (shift_amount == 16));
+ DCHECK(is_uint8(imm8));
+
+ int cmode_0 = (shift_amount >> 4) & 1;
+ int cmode = 0xC | cmode_0;
+
+ Instr q = vd.IsQ() ? NEON_Q : 0;
+
+ Emit(q | op | ImmNEONabcdefgh(imm8) | NEONCmode(cmode) | Rd(vd));
+}
+
+void Assembler::EmitShift(const Register& rd, const Register& rn, Shift shift,
+ unsigned shift_amount) {
+ switch (shift) {
+ case LSL:
+ lsl(rd, rn, shift_amount);
+ break;
+ case LSR:
+ lsr(rd, rn, shift_amount);
+ break;
+ case ASR:
+ asr(rd, rn, shift_amount);
+ break;
+ case ROR:
+ ror(rd, rn, shift_amount);
+ break;
+ default:
+ UNREACHABLE();
+ }
+}
+
+void Assembler::EmitExtendShift(const Register& rd, const Register& rn,
+ Extend extend, unsigned left_shift) {
+ DCHECK(rd.SizeInBits() >= rn.SizeInBits());
+ unsigned reg_size = rd.SizeInBits();
+ // Use the correct size of register.
+ Register rn_ = Register::Create(rn.code(), rd.SizeInBits());
+ // Bits extracted are high_bit:0.
+ unsigned high_bit = (8 << (extend & 0x3)) - 1;
+ // Number of bits left in the result that are not introduced by the shift.
+ unsigned non_shift_bits = (reg_size - left_shift) & (reg_size - 1);
+
+ if ((non_shift_bits > high_bit) || (non_shift_bits == 0)) {
+ switch (extend) {
+ case UXTB:
+ case UXTH:
+ case UXTW:
+ ubfm(rd, rn_, non_shift_bits, high_bit);
+ break;
+ case SXTB:
+ case SXTH:
+ case SXTW:
+ sbfm(rd, rn_, non_shift_bits, high_bit);
+ break;
+ case UXTX:
+ case SXTX: {
+ DCHECK_EQ(rn.SizeInBits(), kXRegSizeInBits);
+ // Nothing to extend. Just shift.
+ lsl(rd, rn_, left_shift);
+ break;
+ }
+ default:
+ UNREACHABLE();
+ }
+ } else {
+ // No need to extend as the extended bits would be shifted away.
+ lsl(rd, rn_, left_shift);
+ }
+}
+
+void Assembler::DataProcShiftedRegister(const Register& rd, const Register& rn,
+ const Operand& operand, FlagsUpdate S,
+ Instr op) {
+ DCHECK(operand.IsShiftedRegister());
+ DCHECK(rn.Is64Bits() || (rn.Is32Bits() && is_uint5(operand.shift_amount())));
+ DCHECK(!operand.NeedsRelocation(this));
+ Emit(SF(rd) | op | Flags(S) | ShiftDP(operand.shift()) |
+ ImmDPShift(operand.shift_amount()) | Rm(operand.reg()) | Rn(rn) |
+ Rd(rd));
+}
+
+void Assembler::DataProcExtendedRegister(const Register& rd, const Register& rn,
+ const Operand& operand, FlagsUpdate S,
+ Instr op) {
+ DCHECK(!operand.NeedsRelocation(this));
+ Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd);
+ Emit(SF(rd) | op | Flags(S) | Rm(operand.reg()) |
+ ExtendMode(operand.extend()) | ImmExtendShift(operand.shift_amount()) |
+ dest_reg | RnSP(rn));
+}
+
+bool Assembler::IsImmAddSub(int64_t immediate) {
+ return is_uint12(immediate) ||
+ (is_uint12(immediate >> 12) && ((immediate & 0xFFF) == 0));
+}
+
+void Assembler::LoadStore(const CPURegister& rt, const MemOperand& addr,
+ LoadStoreOp op) {
+ Instr memop = op | Rt(rt) | RnSP(addr.base());
+
+ if (addr.IsImmediateOffset()) {
+ unsigned size = CalcLSDataSize(op);
+ if (IsImmLSScaled(addr.offset(), size)) {
+ int offset = static_cast<int>(addr.offset());
+ // Use the scaled addressing mode.
+ Emit(LoadStoreUnsignedOffsetFixed | memop |
+ ImmLSUnsigned(offset >> size));
+ } else if (IsImmLSUnscaled(addr.offset())) {
+ int offset = static_cast<int>(addr.offset());
+ // Use the unscaled addressing mode.
+ Emit(LoadStoreUnscaledOffsetFixed | memop | ImmLS(offset));
+ } else {
+ // This case is handled in the macro assembler.
+ UNREACHABLE();
+ }
+ } else if (addr.IsRegisterOffset()) {
+ Extend ext = addr.extend();
+ Shift shift = addr.shift();
+ unsigned shift_amount = addr.shift_amount();
+
+ // LSL is encoded in the option field as UXTX.
+ if (shift == LSL) {
+ ext = UXTX;
+ }
+
+ // Shifts are encoded in one bit, indicating a left shift by the memory
+ // access size.
+ DCHECK((shift_amount == 0) ||
+ (shift_amount == static_cast<unsigned>(CalcLSDataSize(op))));
+ Emit(LoadStoreRegisterOffsetFixed | memop | Rm(addr.regoffset()) |
+ ExtendMode(ext) | ImmShiftLS((shift_amount > 0) ? 1 : 0));
+ } else {
+ // Pre-index and post-index modes.
+ DCHECK_NE(rt, addr.base());
+ if (IsImmLSUnscaled(addr.offset())) {
+ int offset = static_cast<int>(addr.offset());
+ if (addr.IsPreIndex()) {
+ Emit(LoadStorePreIndexFixed | memop | ImmLS(offset));
+ } else {
+ DCHECK(addr.IsPostIndex());
+ Emit(LoadStorePostIndexFixed | memop | ImmLS(offset));
+ }
+ } else {
+ // This case is handled in the macro assembler.
+ UNREACHABLE();
+ }
+ }
+}
+
+bool Assembler::IsImmLSUnscaled(int64_t offset) { return is_int9(offset); }
+
+bool Assembler::IsImmLSScaled(int64_t offset, unsigned size) {
+ bool offset_is_size_multiple =
+ (static_cast<int64_t>(static_cast<uint64_t>(offset >> size) << size) ==
+ offset);
+ return offset_is_size_multiple && is_uint12(offset >> size);
+}
+
+bool Assembler::IsImmLSPair(int64_t offset, unsigned size) {
+ bool offset_is_size_multiple =
+ (static_cast<int64_t>(static_cast<uint64_t>(offset >> size) << size) ==
+ offset);
+ return offset_is_size_multiple && is_int7(offset >> size);
+}
+
+bool Assembler::IsImmLLiteral(int64_t offset) {
+ int inst_size = static_cast<int>(kInstrSizeLog2);
+ bool offset_is_inst_multiple =
+ (static_cast<int64_t>(static_cast<uint64_t>(offset >> inst_size)
+ << inst_size) == offset);
+ DCHECK_GT(offset, 0);
+ offset >>= kLoadLiteralScaleLog2;
+ return offset_is_inst_multiple && is_intn(offset, ImmLLiteral_width);
+}
+
+// Test if a given value can be encoded in the immediate field of a logical
+// instruction.
+// If it can be encoded, the function returns true, and values pointed to by n,
+// imm_s and imm_r are updated with immediates encoded in the format required
+// by the corresponding fields in the logical instruction.
+// If it can not be encoded, the function returns false, and the values pointed
+// to by n, imm_s and imm_r are undefined.
+bool Assembler::IsImmLogical(uint64_t value, unsigned width, unsigned* n,
+ unsigned* imm_s, unsigned* imm_r) {
+ DCHECK((n != nullptr) && (imm_s != nullptr) && (imm_r != nullptr));
+ DCHECK((width == kWRegSizeInBits) || (width == kXRegSizeInBits));
+
+ bool negate = false;
+
+ // Logical immediates are encoded using parameters n, imm_s and imm_r using
+ // the following table:
+ //
+ // N imms immr size S R
+ // 1 ssssss rrrrrr 64 UInt(ssssss) UInt(rrrrrr)
+ // 0 0sssss xrrrrr 32 UInt(sssss) UInt(rrrrr)
+ // 0 10ssss xxrrrr 16 UInt(ssss) UInt(rrrr)
+ // 0 110sss xxxrrr 8 UInt(sss) UInt(rrr)
+ // 0 1110ss xxxxrr 4 UInt(ss) UInt(rr)
+ // 0 11110s xxxxxr 2 UInt(s) UInt(r)
+ // (s bits must not be all set)
+ //
+ // A pattern is constructed of size bits, where the least significant S+1 bits
+ // are set. The pattern is rotated right by R, and repeated across a 32 or
+ // 64-bit value, depending on destination register width.
+ //
+ // Put another way: the basic format of a logical immediate is a single
+ // contiguous stretch of 1 bits, repeated across the whole word at intervals
+ // given by a power of 2. To identify them quickly, we first locate the
+ // lowest stretch of 1 bits, then the next 1 bit above that; that combination
+ // is different for every logical immediate, so it gives us all the
+ // information we need to identify the only logical immediate that our input
+ // could be, and then we simply check if that's the value we actually have.
+ //
+ // (The rotation parameter does give the possibility of the stretch of 1 bits
+ // going 'round the end' of the word. To deal with that, we observe that in
+ // any situation where that happens the bitwise NOT of the value is also a
+ // valid logical immediate. So we simply invert the input whenever its low bit
+ // is set, and then we know that the rotated case can't arise.)
+
+ if (value & 1) {
+ // If the low bit is 1, negate the value, and set a flag to remember that we
+ // did (so that we can adjust the return values appropriately).
+ negate = true;
+ value = ~value;
+ }
+
+ if (width == kWRegSizeInBits) {
+ // To handle 32-bit logical immediates, the very easiest thing is to repeat
+ // the input value twice to make a 64-bit word. The correct encoding of that
+ // as a logical immediate will also be the correct encoding of the 32-bit
+ // value.
+
+ // The most-significant 32 bits may not be zero (ie. negate is true) so
+ // shift the value left before duplicating it.
+ value <<= kWRegSizeInBits;
+ value |= value >> kWRegSizeInBits;
+ }
+
+ // The basic analysis idea: imagine our input word looks like this.
+ //
+ // 0011111000111110001111100011111000111110001111100011111000111110
+ // c b a
+ // |<--d-->|
+ //
+ // We find the lowest set bit (as an actual power-of-2 value, not its index)
+ // and call it a. Then we add a to our original number, which wipes out the
+ // bottommost stretch of set bits and replaces it with a 1 carried into the
+ // next zero bit. Then we look for the new lowest set bit, which is in
+ // position b, and subtract it, so now our number is just like the original
+ // but with the lowest stretch of set bits completely gone. Now we find the
+ // lowest set bit again, which is position c in the diagram above. Then we'll
+ // measure the distance d between bit positions a and c (using CLZ), and that
+ // tells us that the only valid logical immediate that could possibly be equal
+ // to this number is the one in which a stretch of bits running from a to just
+ // below b is replicated every d bits.
+ uint64_t a = LargestPowerOf2Divisor(value);
+ uint64_t value_plus_a = value + a;
+ uint64_t b = LargestPowerOf2Divisor(value_plus_a);
+ uint64_t value_plus_a_minus_b = value_plus_a - b;
+ uint64_t c = LargestPowerOf2Divisor(value_plus_a_minus_b);
+
+ int d, clz_a, out_n;
+ uint64_t mask;
+
+ if (c != 0) {
+ // The general case, in which there is more than one stretch of set bits.
+ // Compute the repeat distance d, and set up a bitmask covering the basic
+ // unit of repetition (i.e. a word with the bottom d bits set). Also, in all
+ // of these cases the N bit of the output will be zero.
+ clz_a = CountLeadingZeros(a, kXRegSizeInBits);
+ int clz_c = CountLeadingZeros(c, kXRegSizeInBits);
+ d = clz_a - clz_c;
+ mask = ((uint64_t{1} << d) - 1);
+ out_n = 0;
+ } else {
+ // Handle degenerate cases.
+ //
+ // If any of those 'find lowest set bit' operations didn't find a set bit at
+ // all, then the word will have been zero thereafter, so in particular the
+ // last lowest_set_bit operation will have returned zero. So we can test for
+ // all the special case conditions in one go by seeing if c is zero.
+ if (a == 0) {
+ // The input was zero (or all 1 bits, which will come to here too after we
+ // inverted it at the start of the function), for which we just return
+ // false.
+ return false;
+ } else {
+ // Otherwise, if c was zero but a was not, then there's just one stretch
+ // of set bits in our word, meaning that we have the trivial case of
+ // d == 64 and only one 'repetition'. Set up all the same variables as in
+ // the general case above, and set the N bit in the output.
+ clz_a = CountLeadingZeros(a, kXRegSizeInBits);
+ d = 64;
+ mask = ~uint64_t{0};
+ out_n = 1;
+ }
+ }
+
+ // If the repeat period d is not a power of two, it can't be encoded.
+ if (!base::bits::IsPowerOfTwo(d)) {
+ return false;
+ }
+
+ if (((b - a) & ~mask) != 0) {
+ // If the bit stretch (b - a) does not fit within the mask derived from the
+ // repeat period, then fail.
+ return false;
+ }
+
+ // The only possible option is b - a repeated every d bits. Now we're going to
+ // actually construct the valid logical immediate derived from that
+ // specification, and see if it equals our original input.
+ //
+ // To repeat a value every d bits, we multiply it by a number of the form
+ // (1 + 2^d + 2^(2d) + ...), i.e. 0x0001000100010001 or similar. These can
+ // be derived using a table lookup on CLZ(d).
+ static const uint64_t multipliers[] = {
+ 0x0000000000000001UL, 0x0000000100000001UL, 0x0001000100010001UL,
+ 0x0101010101010101UL, 0x1111111111111111UL, 0x5555555555555555UL,
+ };
+ int multiplier_idx = CountLeadingZeros(d, kXRegSizeInBits) - 57;
+ // Ensure that the index to the multipliers array is within bounds.
+ DCHECK((multiplier_idx >= 0) &&
+ (static_cast<size_t>(multiplier_idx) < arraysize(multipliers)));
+ uint64_t multiplier = multipliers[multiplier_idx];
+ uint64_t candidate = (b - a) * multiplier;
+
+ if (value != candidate) {
+ // The candidate pattern doesn't match our input value, so fail.
+ return false;
+ }
+
+ // We have a match! This is a valid logical immediate, so now we have to
+ // construct the bits and pieces of the instruction encoding that generates
+ // it.
+
+ // Count the set bits in our basic stretch. The special case of clz(0) == -1
+ // makes the answer come out right for stretches that reach the very top of
+ // the word (e.g. numbers like 0xFFFFC00000000000).
+ int clz_b = (b == 0) ? -1 : CountLeadingZeros(b, kXRegSizeInBits);
+ int s = clz_a - clz_b;
+
+ // Decide how many bits to rotate right by, to put the low bit of that basic
+ // stretch in position a.
+ int r;
+ if (negate) {
+ // If we inverted the input right at the start of this function, here's
+ // where we compensate: the number of set bits becomes the number of clear
+ // bits, and the rotation count is based on position b rather than position
+ // a (since b is the location of the 'lowest' 1 bit after inversion).
+ s = d - s;
+ r = (clz_b + 1) & (d - 1);
+ } else {
+ r = (clz_a + 1) & (d - 1);
+ }
+
+ // Now we're done, except for having to encode the S output in such a way that
+ // it gives both the number of set bits and the length of the repeated
+ // segment. The s field is encoded like this:
+ //
+ // imms size S
+ // ssssss 64 UInt(ssssss)
+ // 0sssss 32 UInt(sssss)
+ // 10ssss 16 UInt(ssss)
+ // 110sss 8 UInt(sss)
+ // 1110ss 4 UInt(ss)
+ // 11110s 2 UInt(s)
+ //
+ // So we 'or' (-d * 2) with our computed s to form imms.
+ *n = out_n;
+ *imm_s = ((-d * 2) | (s - 1)) & 0x3F;
+ *imm_r = r;
+
+ return true;
+}
+
+bool Assembler::IsImmConditionalCompare(int64_t immediate) {
+ return is_uint5(immediate);
+}
+
+bool Assembler::IsImmFP32(float imm) {
+ // Valid values will have the form:
+ // aBbb.bbbc.defg.h000.0000.0000.0000.0000
+ uint32_t bits = bit_cast<uint32_t>(imm);
+ // bits[19..0] are cleared.
+ if ((bits & 0x7FFFF) != 0) {
+ return false;
+ }
+
+ // bits[29..25] are all set or all cleared.
+ uint32_t b_pattern = (bits >> 16) & 0x3E00;
+ if (b_pattern != 0 && b_pattern != 0x3E00) {
+ return false;
+ }
+
+ // bit[30] and bit[29] are opposite.
+ if (((bits ^ (bits << 1)) & 0x40000000) == 0) {
+ return false;
+ }
+
+ return true;
+}
+
+bool Assembler::IsImmFP64(double imm) {
+ // Valid values will have the form:
+ // aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
+ // 0000.0000.0000.0000.0000.0000.0000.0000
+ uint64_t bits = bit_cast<uint64_t>(imm);
+ // bits[47..0] are cleared.
+ if ((bits & 0xFFFFFFFFFFFFL) != 0) {
+ return false;
+ }
+
+ // bits[61..54] are all set or all cleared.
+ uint32_t b_pattern = (bits >> 48) & 0x3FC0;
+ if (b_pattern != 0 && b_pattern != 0x3FC0) {
+ return false;
+ }
+
+ // bit[62] and bit[61] are opposite.
+ if (((bits ^ (bits << 1)) & 0x4000000000000000L) == 0) {
+ return false;
+ }
+
+ return true;
+}
+
+void Assembler::GrowBuffer() {
+ // Compute new buffer size.
+ int old_size = buffer_->size();
+ int new_size = std::min(2 * old_size, old_size + 1 * MB);
+
+ // Some internal data structures overflow for very large buffers,
+ // they must ensure that kMaximalBufferSize is not too large.
+ if (new_size > kMaximalBufferSize) {
+ V8::FatalProcessOutOfMemory(nullptr, "Assembler::GrowBuffer");
+ }
+
+ // Set up new buffer.
+ std::unique_ptr<AssemblerBuffer> new_buffer = buffer_->Grow(new_size);
+ DCHECK_EQ(new_size, new_buffer->size());
+ byte* new_start = new_buffer->start();
+
+ // Copy the data.
+ intptr_t pc_delta = new_start - buffer_start_;
+ intptr_t rc_delta = (new_start + new_size) - (buffer_start_ + old_size);
+ size_t reloc_size = (buffer_start_ + old_size) - reloc_info_writer.pos();
+ memmove(new_start, buffer_start_, pc_offset());
+ memmove(reloc_info_writer.pos() + rc_delta, reloc_info_writer.pos(),
+ reloc_size);
+
+ // Switch buffers.
+ buffer_ = std::move(new_buffer);
+ buffer_start_ = new_start;
+ pc_ += pc_delta;
+ reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta,
+ reloc_info_writer.last_pc() + pc_delta);
+
+ // None of our relocation types are pc relative pointing outside the code
+ // buffer nor pc absolute pointing inside the code buffer, so there is no need
+ // to relocate any emitted relocation entries.
+
+ // Relocate internal references.
+ for (auto pos : internal_reference_positions_) {
+ Address address = reinterpret_cast<intptr_t>(buffer_start_) + pos;
+ intptr_t internal_ref = ReadUnalignedValue<intptr_t>(address);
+ internal_ref += pc_delta;
+ WriteUnalignedValue<intptr_t>(address, internal_ref);
+ }
+
+ // Pending relocation entries are also relative, no need to relocate.
+}
+
+void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data,
+ ConstantPoolMode constant_pool_mode) {
+ if ((rmode == RelocInfo::INTERNAL_REFERENCE) ||
+ (rmode == RelocInfo::CONST_POOL) || (rmode == RelocInfo::VENEER_POOL) ||
+ (rmode == RelocInfo::DEOPT_SCRIPT_OFFSET) ||
+ (rmode == RelocInfo::DEOPT_INLINING_ID) ||
+ (rmode == RelocInfo::DEOPT_REASON) || (rmode == RelocInfo::DEOPT_ID)) {
+ // Adjust code for new modes.
+ DCHECK(RelocInfo::IsDeoptReason(rmode) || RelocInfo::IsDeoptId(rmode) ||
+ RelocInfo::IsDeoptPosition(rmode) ||
+ RelocInfo::IsInternalReference(rmode) ||
+ RelocInfo::IsConstPool(rmode) || RelocInfo::IsVeneerPool(rmode));
+ // These modes do not need an entry in the constant pool.
+ } else if (constant_pool_mode == NEEDS_POOL_ENTRY) {
+ if (RelocInfo::IsEmbeddedObjectMode(rmode)) {
+ Handle<HeapObject> handle(reinterpret_cast<Address*>(data));
+ data = AddEmbeddedObject(handle);
+ }
+ if (rmode == RelocInfo::COMPRESSED_EMBEDDED_OBJECT) {
+ if (constpool_.RecordEntry(static_cast<uint32_t>(data), rmode) ==
+ RelocInfoStatus::kMustOmitForDuplicate) {
+ return;
+ }
+ } else {
+ if (constpool_.RecordEntry(static_cast<uint64_t>(data), rmode) ==
+ RelocInfoStatus::kMustOmitForDuplicate) {
+ return;
+ }
+ }
+ }
+ // For modes that cannot use the constant pool, a different sequence of
+ // instructions will be emitted by this function's caller.
+
+ if (!ShouldRecordRelocInfo(rmode)) return;
+
+ // Callers should ensure that constant pool emission is blocked until the
+ // instruction the reloc info is associated with has been emitted.
+ DCHECK(constpool_.IsBlocked());
+
+ // We do not try to reuse pool constants.
+ RelocInfo rinfo(reinterpret_cast<Address>(pc_), rmode, data, Code());
+
+ DCHECK_GE(buffer_space(), kMaxRelocSize); // too late to grow buffer here
+ reloc_info_writer.Write(&rinfo);
+}
+
+void Assembler::near_jump(int offset, RelocInfo::Mode rmode) {
+ BlockPoolsScope no_pool_before_b_instr(this);
+ if (!RelocInfo::IsNone(rmode)) RecordRelocInfo(rmode, offset, NO_POOL_ENTRY);
+ b(offset);
+}
+
+void Assembler::near_call(int offset, RelocInfo::Mode rmode) {
+ BlockPoolsScope no_pool_before_bl_instr(this);
+ if (!RelocInfo::IsNone(rmode)) RecordRelocInfo(rmode, offset, NO_POOL_ENTRY);
+ bl(offset);
+}
+
+void Assembler::near_call(HeapObjectRequest request) {
+ BlockPoolsScope no_pool_before_bl_instr(this);
+ RequestHeapObject(request);
+ EmbeddedObjectIndex index = AddEmbeddedObject(Handle<Code>());
+ RecordRelocInfo(RelocInfo::CODE_TARGET, index, NO_POOL_ENTRY);
+ DCHECK(is_int32(index));
+ bl(static_cast<int>(index));
+}
+
+// Constant Pool
+
+void ConstantPool::EmitPrologue(Alignment require_alignment) {
+ // Recorded constant pool size is expressed in number of 32-bits words,
+ // and includes prologue and alignment, but not the jump around the pool
+ // and the size of the marker itself.
+ const int marker_size = 1;
+ int word_count =
+ ComputeSize(Jump::kOmitted, require_alignment) / kInt32Size - marker_size;
+ assm_->Emit(LDR_x_lit | Assembler::ImmLLiteral(word_count) |
+ Assembler::Rt(xzr));
+ assm_->EmitPoolGuard();
+}
+
+int ConstantPool::PrologueSize(Jump require_jump) const {
+ // Prologue is:
+ // b over ;; if require_jump
+ // ldr xzr, #pool_size
+ // blr xzr
+ int prologue_size = require_jump == Jump::kRequired ? kInstrSize : 0;
+ prologue_size += 2 * kInstrSize;
+ return prologue_size;
+}
+
+void ConstantPool::SetLoadOffsetToConstPoolEntry(int load_offset,
+ Instruction* entry_offset,
+ const ConstantPoolKey& key) {
+ Instruction* instr = assm_->InstructionAt(load_offset);
+ // Instruction to patch must be 'ldr rd, [pc, #offset]' with offset == 0.
+ DCHECK(instr->IsLdrLiteral() && instr->ImmLLiteral() == 0);
+ instr->SetImmPCOffsetTarget(assm_->options(), entry_offset);
+}
+
+void ConstantPool::Check(Emission force_emit, Jump require_jump,
+ size_t margin) {
+ // Some short sequence of instruction must not be broken up by constant pool
+ // emission, such sequences are protected by a ConstPool::BlockScope.
+ if (IsBlocked()) {
+ // Something is wrong if emission is forced and blocked at the same time.
+ DCHECK_EQ(force_emit, Emission::kIfNeeded);
+ return;
+ }
+
+ // We emit a constant pool only if :
+ // * it is not empty
+ // * emission is forced by parameter force_emit (e.g. at function end).
+ // * emission is mandatory or opportune according to {ShouldEmitNow}.
+ if (!IsEmpty() && (force_emit == Emission::kForced ||
+ ShouldEmitNow(require_jump, margin))) {
+ // Emit veneers for branches that would go out of range during emission of
+ // the constant pool.
+ int worst_case_size = ComputeSize(Jump::kRequired, Alignment::kRequired);
+ assm_->CheckVeneerPool(false, require_jump == Jump::kRequired,
+ assm_->kVeneerDistanceMargin + worst_case_size +
+ static_cast<int>(margin));
+
+ // Check that the code buffer is large enough before emitting the constant
+ // pool (this includes the gap to the relocation information).
+ int needed_space = worst_case_size + assm_->kGap;
+ while (assm_->buffer_space() <= needed_space) {
+ assm_->GrowBuffer();
+ }
+
+ EmitAndClear(require_jump);
+ }
+ // Since a constant pool is (now) empty, move the check offset forward by
+ // the standard interval.
+ SetNextCheckIn(ConstantPool::kCheckInterval);
+}
+
+// Pool entries are accessed with pc relative load therefore this cannot be more
+// than 1 * MB. Since constant pool emission checks are interval based, and we
+// want to keep entries close to the code, we try to emit every 64KB.
+const size_t ConstantPool::kMaxDistToPool32 = 1 * MB;
+const size_t ConstantPool::kMaxDistToPool64 = 1 * MB;
+const size_t ConstantPool::kCheckInterval = 128 * kInstrSize;
+const size_t ConstantPool::kApproxDistToPool32 = 64 * KB;
+const size_t ConstantPool::kApproxDistToPool64 = kApproxDistToPool32;
+
+const size_t ConstantPool::kOpportunityDistToPool32 = 64 * KB;
+const size_t ConstantPool::kOpportunityDistToPool64 = 64 * KB;
+const size_t ConstantPool::kApproxMaxEntryCount = 512;
+
+bool Assembler::ShouldEmitVeneer(int max_reachable_pc, size_t margin) {
+ // Account for the branch around the veneers and the guard.
+ int protection_offset = 2 * kInstrSize;
+ return static_cast<intptr_t>(pc_offset() + margin + protection_offset +
+ unresolved_branches_.size() *
+ kMaxVeneerCodeSize) >= max_reachable_pc;
+}
+
+void Assembler::RecordVeneerPool(int location_offset, int size) {
+ Assembler::BlockPoolsScope block_pools(this, PoolEmissionCheck::kSkip);
+ RelocInfo rinfo(reinterpret_cast<Address>(buffer_start_) + location_offset,
+ RelocInfo::VENEER_POOL, static_cast<intptr_t>(size), Code());
+ reloc_info_writer.Write(&rinfo);
+}
+
+void Assembler::EmitVeneers(bool force_emit, bool need_protection,
+ size_t margin) {
+ BlockPoolsScope scope(this, PoolEmissionCheck::kSkip);
+ RecordComment("[ Veneers");
+
+ // The exact size of the veneer pool must be recorded (see the comment at the
+ // declaration site of RecordConstPool()), but computing the number of
+ // veneers that will be generated is not obvious. So instead we remember the
+ // current position and will record the size after the pool has been
+ // generated.
+ Label size_check;
+ bind(&size_check);
+ int veneer_pool_relocinfo_loc = pc_offset();
+
+ Label end;
+ if (need_protection) {
+ b(&end);
+ }
+
+ EmitVeneersGuard();
+
+#ifdef DEBUG
+ Label veneer_size_check;
+#endif
+
+ std::multimap<int, FarBranchInfo>::iterator it, it_to_delete;
+
+ it = unresolved_branches_.begin();
+ while (it != unresolved_branches_.end()) {
+ if (force_emit || ShouldEmitVeneer(it->first, margin)) {
+ Instruction* branch = InstructionAt(it->second.pc_offset_);
+ Label* label = it->second.label_;
+
+#ifdef DEBUG
+ bind(&veneer_size_check);
+#endif
+ // Patch the branch to point to the current position, and emit a branch
+ // to the label.
+ Instruction* veneer = reinterpret_cast<Instruction*>(pc_);
+ RemoveBranchFromLabelLinkChain(branch, label, veneer);
+ branch->SetImmPCOffsetTarget(options(), veneer);
+ b(label);
+#ifdef DEBUG
+ DCHECK(SizeOfCodeGeneratedSince(&veneer_size_check) <=
+ static_cast<uint64_t>(kMaxVeneerCodeSize));
+ veneer_size_check.Unuse();
+#endif
+
+ it_to_delete = it++;
+ unresolved_branches_.erase(it_to_delete);
+ } else {
+ ++it;
+ }
+ }
+
+ // Record the veneer pool size.
+ int pool_size = static_cast<int>(SizeOfCodeGeneratedSince(&size_check));
+ RecordVeneerPool(veneer_pool_relocinfo_loc, pool_size);
+
+ if (unresolved_branches_.empty()) {
+ next_veneer_pool_check_ = kMaxInt;
+ } else {
+ next_veneer_pool_check_ =
+ unresolved_branches_first_limit() - kVeneerDistanceCheckMargin;
+ }
+
+ bind(&end);
+
+ RecordComment("]");
+}
+
+void Assembler::CheckVeneerPool(bool force_emit, bool require_jump,
+ size_t margin) {
+ // There is nothing to do if there are no pending veneer pool entries.
+ if (unresolved_branches_.empty()) {
+ DCHECK_EQ(next_veneer_pool_check_, kMaxInt);
+ return;
+ }
+
+ DCHECK(pc_offset() < unresolved_branches_first_limit());
+
+ // Some short sequence of instruction mustn't be broken up by veneer pool
+ // emission, such sequences are protected by calls to BlockVeneerPoolFor and
+ // BlockVeneerPoolScope.
+ if (is_veneer_pool_blocked()) {
+ DCHECK(!force_emit);
+ return;
+ }
+
+ if (!require_jump) {
+ // Prefer emitting veneers protected by an existing instruction.
+ margin *= kVeneerNoProtectionFactor;
+ }
+ if (force_emit || ShouldEmitVeneers(margin)) {
+ EmitVeneers(force_emit, require_jump, margin);
+ } else {
+ next_veneer_pool_check_ =
+ unresolved_branches_first_limit() - kVeneerDistanceCheckMargin;
+ }
+}
+
+int Assembler::buffer_space() const {
+ return static_cast<int>(reloc_info_writer.pos() - pc_);
+}
+
+void Assembler::RecordConstPool(int size) {
+ // We only need this for debugger support, to correctly compute offsets in the
+ // code.
+ Assembler::BlockPoolsScope block_pools(this);
+ RecordRelocInfo(RelocInfo::CONST_POOL, static_cast<intptr_t>(size));
+}
+
+void PatchingAssembler::PatchAdrFar(int64_t target_offset) {
+ // The code at the current instruction should be:
+ // adr rd, 0
+ // nop (adr_far)
+ // nop (adr_far)
+ // movz scratch, 0
+
+ // Verify the expected code.
+ Instruction* expected_adr = InstructionAt(0);
+ CHECK(expected_adr->IsAdr() && (expected_adr->ImmPCRel() == 0));
+ int rd_code = expected_adr->Rd();
+ for (int i = 0; i < kAdrFarPatchableNNops; ++i) {
+ CHECK(InstructionAt((i + 1) * kInstrSize)->IsNop(ADR_FAR_NOP));
+ }
+ Instruction* expected_movz =
+ InstructionAt((kAdrFarPatchableNInstrs - 1) * kInstrSize);
+ CHECK(expected_movz->IsMovz() && (expected_movz->ImmMoveWide() == 0) &&
+ (expected_movz->ShiftMoveWide() == 0));
+ int scratch_code = expected_movz->Rd();
+
+ // Patch to load the correct address.
+ Register rd = Register::XRegFromCode(rd_code);
+ Register scratch = Register::XRegFromCode(scratch_code);
+ // Addresses are only 48 bits.
+ adr(rd, target_offset & 0xFFFF);
+ movz(scratch, (target_offset >> 16) & 0xFFFF, 16);
+ movk(scratch, (target_offset >> 32) & 0xFFFF, 32);
+ DCHECK_EQ(target_offset >> 48, 0);
+ add(rd, rd, scratch);
+}
+
+void PatchingAssembler::PatchSubSp(uint32_t immediate) {
+ // The code at the current instruction should be:
+ // sub sp, sp, #0
+
+ // Verify the expected code.
+ Instruction* expected_adr = InstructionAt(0);
+ CHECK(expected_adr->IsAddSubImmediate());
+ sub(sp, sp, immediate);
+}
+
+#undef NEON_3DIFF_LONG_LIST
+#undef NEON_3DIFF_HN_LIST
+#undef NEON_ACROSSLANES_LIST
+#undef NEON_FP2REGMISC_FCVT_LIST
+#undef NEON_FP2REGMISC_LIST
+#undef NEON_3SAME_LIST
+#undef NEON_FP3SAME_LIST_V2
+#undef NEON_BYELEMENT_LIST
+#undef NEON_FPBYELEMENT_LIST
+#undef NEON_BYELEMENT_LONG_LIST
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_ARM64
diff --git a/src/codegen/arm64/assembler-arm64.h b/src/codegen/arm64/assembler-arm64.h
new file mode 100644
index 0000000..f787bad
--- /dev/null
+++ b/src/codegen/arm64/assembler-arm64.h
@@ -0,0 +1,2773 @@
+// Copyright 2013 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_ARM64_ASSEMBLER_ARM64_H_
+#define V8_CODEGEN_ARM64_ASSEMBLER_ARM64_H_
+
+#include <deque>
+#include <list>
+#include <map>
+#include <memory>
+#include <vector>
+
+#include "src/base/optional.h"
+#include "src/codegen/arm64/constants-arm64.h"
+#include "src/codegen/arm64/instructions-arm64.h"
+#include "src/codegen/arm64/register-arm64.h"
+#include "src/codegen/assembler.h"
+#include "src/codegen/constant-pool.h"
+#include "src/common/globals.h"
+#include "src/utils/utils.h"
+
+// Windows arm64 SDK defines mvn to NEON intrinsic neon_not which will not
+// be used here.
+#if defined(V8_OS_WIN) && defined(mvn)
+#undef mvn
+#endif
+
+#if defined(V8_OS_WIN)
+#include "src/diagnostics/unwinding-info-win64.h"
+#endif // V8_OS_WIN
+
+namespace v8 {
+namespace internal {
+
+class SafepointTableBuilder;
+
+// -----------------------------------------------------------------------------
+// Immediates.
+class Immediate {
+ public:
+ template <typename T>
+ inline explicit Immediate(
+ Handle<T> handle, RelocInfo::Mode mode = RelocInfo::FULL_EMBEDDED_OBJECT);
+
+ // This is allowed to be an implicit constructor because Immediate is
+ // a wrapper class that doesn't normally perform any type conversion.
+ template <typename T>
+ inline Immediate(T value); // NOLINT(runtime/explicit)
+
+ template <typename T>
+ inline Immediate(T value, RelocInfo::Mode rmode);
+
+ int64_t value() const { return value_; }
+ RelocInfo::Mode rmode() const { return rmode_; }
+
+ private:
+ int64_t value_;
+ RelocInfo::Mode rmode_;
+};
+
+// -----------------------------------------------------------------------------
+// Operands.
+constexpr int kSmiShift = kSmiTagSize + kSmiShiftSize;
+constexpr uint64_t kSmiShiftMask = (1ULL << kSmiShift) - 1;
+
+// Represents an operand in a machine instruction.
+class Operand {
+ // TODO(all): If necessary, study more in details which methods
+ // TODO(all): should be inlined or not.
+ public:
+ // rm, {<shift> {#<shift_amount>}}
+ // where <shift> is one of {LSL, LSR, ASR, ROR}.
+ // <shift_amount> is uint6_t.
+ // This is allowed to be an implicit constructor because Operand is
+ // a wrapper class that doesn't normally perform any type conversion.
+ inline Operand(Register reg, Shift shift = LSL,
+ unsigned shift_amount = 0); // NOLINT(runtime/explicit)
+
+ // rm, <extend> {#<shift_amount>}
+ // where <extend> is one of {UXTB, UXTH, UXTW, UXTX, SXTB, SXTH, SXTW, SXTX}.
+ // <shift_amount> is uint2_t.
+ inline Operand(Register reg, Extend extend, unsigned shift_amount = 0);
+
+ static Operand EmbeddedNumber(double number); // Smi or HeapNumber.
+ static Operand EmbeddedStringConstant(const StringConstantBase* str);
+
+ inline bool IsHeapObjectRequest() const;
+ inline HeapObjectRequest heap_object_request() const;
+ inline Immediate immediate_for_heap_object_request() const;
+
+ // Implicit constructor for all int types, ExternalReference, and Smi.
+ template <typename T>
+ inline Operand(T t); // NOLINT(runtime/explicit)
+
+ // Implicit constructor for int types.
+ template <typename T>
+ inline Operand(T t, RelocInfo::Mode rmode);
+
+ inline bool IsImmediate() const;
+ inline bool IsShiftedRegister() const;
+ inline bool IsExtendedRegister() const;
+ inline bool IsZero() const;
+
+ // This returns an LSL shift (<= 4) operand as an equivalent extend operand,
+ // which helps in the encoding of instructions that use the stack pointer.
+ inline Operand ToExtendedRegister() const;
+
+ // Returns new Operand adapted for using with W registers.
+ inline Operand ToW() const;
+
+ inline Immediate immediate() const;
+ inline int64_t ImmediateValue() const;
+ inline RelocInfo::Mode ImmediateRMode() const;
+ inline Register reg() const;
+ inline Shift shift() const;
+ inline Extend extend() const;
+ inline unsigned shift_amount() const;
+
+ // Relocation information.
+ bool NeedsRelocation(const Assembler* assembler) const;
+
+ private:
+ base::Optional<HeapObjectRequest> heap_object_request_;
+ Immediate immediate_;
+ Register reg_;
+ Shift shift_;
+ Extend extend_;
+ unsigned shift_amount_;
+};
+
+// MemOperand represents a memory operand in a load or store instruction.
+class MemOperand {
+ public:
+ inline MemOperand();
+ inline explicit MemOperand(Register base, int64_t offset = 0,
+ AddrMode addrmode = Offset);
+ inline explicit MemOperand(Register base, Register regoffset,
+ Shift shift = LSL, unsigned shift_amount = 0);
+ inline explicit MemOperand(Register base, Register regoffset, Extend extend,
+ unsigned shift_amount = 0);
+ inline explicit MemOperand(Register base, const Operand& offset,
+ AddrMode addrmode = Offset);
+
+ const Register& base() const { return base_; }
+ const Register& regoffset() const { return regoffset_; }
+ int64_t offset() const { return offset_; }
+ AddrMode addrmode() const { return addrmode_; }
+ Shift shift() const { return shift_; }
+ Extend extend() const { return extend_; }
+ unsigned shift_amount() const { return shift_amount_; }
+ inline bool IsImmediateOffset() const;
+ inline bool IsRegisterOffset() const;
+ inline bool IsPreIndex() const;
+ inline bool IsPostIndex() const;
+
+ private:
+ Register base_;
+ Register regoffset_;
+ int64_t offset_;
+ AddrMode addrmode_;
+ Shift shift_;
+ Extend extend_;
+ unsigned shift_amount_;
+};
+
+// -----------------------------------------------------------------------------
+// Assembler.
+
+class V8_EXPORT_PRIVATE Assembler : public AssemblerBase {
+ public:
+ // Create an assembler. Instructions and relocation information are emitted
+ // into a buffer, with the instructions starting from the beginning and the
+ // relocation information starting from the end of the buffer. See CodeDesc
+ // for a detailed comment on the layout (globals.h).
+ //
+ // If the provided buffer is nullptr, the assembler allocates and grows its
+ // own buffer. Otherwise it takes ownership of the provided buffer.
+ explicit Assembler(const AssemblerOptions&,
+ std::unique_ptr<AssemblerBuffer> = {});
+
+ ~Assembler() override;
+
+ void AbortedCodeGeneration() override;
+
+ // System functions ---------------------------------------------------------
+ // Start generating code from the beginning of the buffer, discarding any code
+ // and data that has already been emitted into the buffer.
+ //
+ // In order to avoid any accidental transfer of state, Reset DCHECKs that the
+ // constant pool is not blocked.
+ void Reset();
+
+ // GetCode emits any pending (non-emitted) code and fills the descriptor desc.
+ static constexpr int kNoHandlerTable = 0;
+ static constexpr SafepointTableBuilder* kNoSafepointTable = nullptr;
+ void GetCode(Isolate* isolate, CodeDesc* desc,
+ SafepointTableBuilder* safepoint_table_builder,
+ int handler_table_offset);
+
+ // Convenience wrapper for code without safepoint or handler tables.
+ void GetCode(Isolate* isolate, CodeDesc* desc) {
+ GetCode(isolate, desc, kNoSafepointTable, kNoHandlerTable);
+ }
+
+ // Insert the smallest number of nop instructions
+ // possible to align the pc offset to a multiple
+ // of m. m must be a power of 2 (>= 4).
+ void Align(int m);
+ // Insert the smallest number of zero bytes possible to align the pc offset
+ // to a mulitple of m. m must be a power of 2 (>= 2).
+ void DataAlign(int m);
+ // Aligns code to something that's optimal for a jump target for the platform.
+ void CodeTargetAlign();
+
+ inline void Unreachable();
+
+ // Label --------------------------------------------------------------------
+ // Bind a label to the current pc. Note that labels can only be bound once,
+ // and if labels are linked to other instructions, they _must_ be bound
+ // before they go out of scope.
+ void bind(Label* label);
+
+ // RelocInfo and pools ------------------------------------------------------
+
+ // Record relocation information for current pc_.
+ enum ConstantPoolMode { NEEDS_POOL_ENTRY, NO_POOL_ENTRY };
+ void RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data = 0,
+ ConstantPoolMode constant_pool_mode = NEEDS_POOL_ENTRY);
+
+ // Generate a B immediate instruction with the corresponding relocation info.
+ // 'offset' is the immediate to encode in the B instruction (so it is the
+ // difference between the target and the PC of the instruction, divided by
+ // the instruction size).
+ void near_jump(int offset, RelocInfo::Mode rmode);
+ // Generate a BL immediate instruction with the corresponding relocation info.
+ // As for near_jump, 'offset' is the immediate to encode in the BL
+ // instruction.
+ void near_call(int offset, RelocInfo::Mode rmode);
+ // Generate a BL immediate instruction with the corresponding relocation info
+ // for the input HeapObjectRequest.
+ void near_call(HeapObjectRequest request);
+
+ // Return the address in the constant pool of the code target address used by
+ // the branch/call instruction at pc.
+ inline static Address target_pointer_address_at(Address pc);
+
+ // Read/Modify the code target address in the branch/call instruction at pc.
+ // The isolate argument is unused (and may be nullptr) when skipping flushing.
+ inline static Address target_address_at(Address pc, Address constant_pool);
+
+ // Read/Modify the code target address in the branch/call instruction at pc.
+ inline static Tagged_t target_compressed_address_at(Address pc,
+ Address constant_pool);
+ inline static void set_target_address_at(
+ Address pc, Address constant_pool, Address target,
+ ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED);
+
+ inline static void set_target_compressed_address_at(
+ Address pc, Address constant_pool, Tagged_t target,
+ ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED);
+
+ // Returns the handle for the code object called at 'pc'.
+ // This might need to be temporarily encoded as an offset into code_targets_.
+ inline Handle<Code> code_target_object_handle_at(Address pc);
+ inline EmbeddedObjectIndex embedded_object_index_referenced_from(Address pc);
+ inline void set_embedded_object_index_referenced_from(
+ Address p, EmbeddedObjectIndex index);
+ // Returns the handle for the heap object referenced at 'pc'.
+ inline Handle<HeapObject> target_object_handle_at(Address pc);
+
+ // Returns the target address for a runtime function for the call encoded
+ // at 'pc'.
+ // Runtime entries can be temporarily encoded as the offset between the
+ // runtime function entrypoint and the code range start (stored in the
+ // code_range_start field), in order to be encodable as we generate the code,
+ // before it is moved into the code space.
+ inline Address runtime_entry_at(Address pc);
+
+ // This sets the branch destination. 'location' here can be either the pc of
+ // an immediate branch or the address of an entry in the constant pool.
+ // This is for calls and branches within generated code.
+ inline static void deserialization_set_special_target_at(Address location,
+ Code code,
+ Address target);
+
+ // Get the size of the special target encoded at 'location'.
+ inline static int deserialization_special_target_size(Address location);
+
+ // This sets the internal reference at the pc.
+ inline static void deserialization_set_target_internal_reference_at(
+ Address pc, Address target,
+ RelocInfo::Mode mode = RelocInfo::INTERNAL_REFERENCE);
+
+ // This value is used in the serialization process and must be zero for
+ // ARM64, as the code target is split across multiple instructions and does
+ // not exist separately in the code, so the serializer should not step
+ // forwards in memory after a target is resolved and written.
+ static constexpr int kSpecialTargetSize = 0;
+
+ // Size of the generated code in bytes
+ uint64_t SizeOfGeneratedCode() const {
+ DCHECK((pc_ >= buffer_start_) && (pc_ < (buffer_start_ + buffer_->size())));
+ return pc_ - buffer_start_;
+ }
+
+ // Return the code size generated from label to the current position.
+ uint64_t SizeOfCodeGeneratedSince(const Label* label) {
+ DCHECK(label->is_bound());
+ DCHECK_GE(pc_offset(), label->pos());
+ DCHECK_LT(pc_offset(), buffer_->size());
+ return pc_offset() - label->pos();
+ }
+
+ // Return the number of instructions generated from label to the
+ // current position.
+ uint64_t InstructionsGeneratedSince(const Label* label) {
+ return SizeOfCodeGeneratedSince(label) / kInstrSize;
+ }
+
+ static bool IsConstantPoolAt(Instruction* instr);
+ static int ConstantPoolSizeAt(Instruction* instr);
+ // See Assembler::CheckConstPool for more info.
+ void EmitPoolGuard();
+
+ // Prevent veneer pool emission until EndBlockVeneerPool is called.
+ // Call to this function can be nested but must be followed by an equal
+ // number of calls to EndBlockConstpool.
+ void StartBlockVeneerPool();
+
+ // Resume constant pool emission. Need to be called as many time as
+ // StartBlockVeneerPool to have an effect.
+ void EndBlockVeneerPool();
+
+ bool is_veneer_pool_blocked() const {
+ return veneer_pool_blocked_nesting_ > 0;
+ }
+
+ // Record a deoptimization reason that can be used by a log or cpu profiler.
+ // Use --trace-deopt to enable.
+ void RecordDeoptReason(DeoptimizeReason reason, SourcePosition position,
+ int id);
+
+ int buffer_space() const;
+
+ // Record the emission of a constant pool.
+ //
+ // The emission of constant and veneer pools depends on the size of the code
+ // generated and the number of RelocInfo recorded.
+ // The Debug mechanism needs to map code offsets between two versions of a
+ // function, compiled with and without debugger support (see for example
+ // Debug::PrepareForBreakPoints()).
+ // Compiling functions with debugger support generates additional code
+ // (DebugCodegen::GenerateSlot()). This may affect the emission of the pools
+ // and cause the version of the code with debugger support to have pools
+ // generated in different places.
+ // Recording the position and size of emitted pools allows to correctly
+ // compute the offset mappings between the different versions of a function in
+ // all situations.
+ //
+ // The parameter indicates the size of the pool (in bytes), including
+ // the marker and branch over the data.
+ void RecordConstPool(int size);
+
+ // Instruction set functions ------------------------------------------------
+
+ // Branch / Jump instructions.
+ // For branches offsets are scaled, i.e. in instructions not in bytes.
+ // Branch to register.
+ void br(const Register& xn);
+
+ // Branch-link to register.
+ void blr(const Register& xn);
+
+ // Branch to register with return hint.
+ void ret(const Register& xn = lr);
+
+ // Unconditional branch to label.
+ void b(Label* label);
+
+ // Conditional branch to label.
+ void b(Label* label, Condition cond);
+
+ // Unconditional branch to PC offset.
+ void b(int imm26);
+
+ // Conditional branch to PC offset.
+ void b(int imm19, Condition cond);
+
+ // Branch-link to label / pc offset.
+ void bl(Label* label);
+ void bl(int imm26);
+
+ // Compare and branch to label / pc offset if zero.
+ void cbz(const Register& rt, Label* label);
+ void cbz(const Register& rt, int imm19);
+
+ // Compare and branch to label / pc offset if not zero.
+ void cbnz(const Register& rt, Label* label);
+ void cbnz(const Register& rt, int imm19);
+
+ // Test bit and branch to label / pc offset if zero.
+ void tbz(const Register& rt, unsigned bit_pos, Label* label);
+ void tbz(const Register& rt, unsigned bit_pos, int imm14);
+
+ // Test bit and branch to label / pc offset if not zero.
+ void tbnz(const Register& rt, unsigned bit_pos, Label* label);
+ void tbnz(const Register& rt, unsigned bit_pos, int imm14);
+
+ // Address calculation instructions.
+ // Calculate a PC-relative address. Unlike for branches the offset in adr is
+ // unscaled (i.e. the result can be unaligned).
+ void adr(const Register& rd, Label* label);
+ void adr(const Register& rd, int imm21);
+
+ // Data Processing instructions.
+ // Add.
+ void add(const Register& rd, const Register& rn, const Operand& operand);
+
+ // Add and update status flags.
+ void adds(const Register& rd, const Register& rn, const Operand& operand);
+
+ // Compare negative.
+ void cmn(const Register& rn, const Operand& operand);
+
+ // Subtract.
+ void sub(const Register& rd, const Register& rn, const Operand& operand);
+
+ // Subtract and update status flags.
+ void subs(const Register& rd, const Register& rn, const Operand& operand);
+
+ // Compare.
+ void cmp(const Register& rn, const Operand& operand);
+
+ // Negate.
+ void neg(const Register& rd, const Operand& operand);
+
+ // Negate and update status flags.
+ void negs(const Register& rd, const Operand& operand);
+
+ // Add with carry bit.
+ void adc(const Register& rd, const Register& rn, const Operand& operand);
+
+ // Add with carry bit and update status flags.
+ void adcs(const Register& rd, const Register& rn, const Operand& operand);
+
+ // Subtract with carry bit.
+ void sbc(const Register& rd, const Register& rn, const Operand& operand);
+
+ // Subtract with carry bit and update status flags.
+ void sbcs(const Register& rd, const Register& rn, const Operand& operand);
+
+ // Negate with carry bit.
+ void ngc(const Register& rd, const Operand& operand);
+
+ // Negate with carry bit and update status flags.
+ void ngcs(const Register& rd, const Operand& operand);
+
+ // Logical instructions.
+ // Bitwise and (A & B).
+ void and_(const Register& rd, const Register& rn, const Operand& operand);
+
+ // Bitwise and (A & B) and update status flags.
+ void ands(const Register& rd, const Register& rn, const Operand& operand);
+
+ // Bit test, and set flags.
+ void tst(const Register& rn, const Operand& operand);
+
+ // Bit clear (A & ~B).
+ void bic(const Register& rd, const Register& rn, const Operand& operand);
+
+ // Bit clear (A & ~B) and update status flags.
+ void bics(const Register& rd, const Register& rn, const Operand& operand);
+
+ // Bitwise and.
+ void and_(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Bit clear immediate.
+ void bic(const VRegister& vd, const int imm8, const int left_shift = 0);
+
+ // Bit clear.
+ void bic(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Bitwise insert if false.
+ void bif(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Bitwise insert if true.
+ void bit(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Bitwise select.
+ void bsl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Polynomial multiply.
+ void pmul(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Vector move immediate.
+ void movi(const VRegister& vd, const uint64_t imm, Shift shift = LSL,
+ const int shift_amount = 0);
+
+ // Bitwise not.
+ void mvn(const VRegister& vd, const VRegister& vn);
+
+ // Vector move inverted immediate.
+ void mvni(const VRegister& vd, const int imm8, Shift shift = LSL,
+ const int shift_amount = 0);
+
+ // Signed saturating accumulate of unsigned value.
+ void suqadd(const VRegister& vd, const VRegister& vn);
+
+ // Unsigned saturating accumulate of signed value.
+ void usqadd(const VRegister& vd, const VRegister& vn);
+
+ // Absolute value.
+ void abs(const VRegister& vd, const VRegister& vn);
+
+ // Signed saturating absolute value.
+ void sqabs(const VRegister& vd, const VRegister& vn);
+
+ // Negate.
+ void neg(const VRegister& vd, const VRegister& vn);
+
+ // Signed saturating negate.
+ void sqneg(const VRegister& vd, const VRegister& vn);
+
+ // Bitwise not.
+ void not_(const VRegister& vd, const VRegister& vn);
+
+ // Extract narrow.
+ void xtn(const VRegister& vd, const VRegister& vn);
+
+ // Extract narrow (second part).
+ void xtn2(const VRegister& vd, const VRegister& vn);
+
+ // Signed saturating extract narrow.
+ void sqxtn(const VRegister& vd, const VRegister& vn);
+
+ // Signed saturating extract narrow (second part).
+ void sqxtn2(const VRegister& vd, const VRegister& vn);
+
+ // Unsigned saturating extract narrow.
+ void uqxtn(const VRegister& vd, const VRegister& vn);
+
+ // Unsigned saturating extract narrow (second part).
+ void uqxtn2(const VRegister& vd, const VRegister& vn);
+
+ // Signed saturating extract unsigned narrow.
+ void sqxtun(const VRegister& vd, const VRegister& vn);
+
+ // Signed saturating extract unsigned narrow (second part).
+ void sqxtun2(const VRegister& vd, const VRegister& vn);
+
+ // Move register to register.
+ void mov(const VRegister& vd, const VRegister& vn);
+
+ // Bitwise not or.
+ void orn(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Bitwise exclusive or.
+ void eor(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Bitwise or (A | B).
+ void orr(const Register& rd, const Register& rn, const Operand& operand);
+
+ // Bitwise or.
+ void orr(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Bitwise or immediate.
+ void orr(const VRegister& vd, const int imm8, const int left_shift = 0);
+
+ // Bitwise nor (A | ~B).
+ void orn(const Register& rd, const Register& rn, const Operand& operand);
+
+ // Bitwise eor/xor (A ^ B).
+ void eor(const Register& rd, const Register& rn, const Operand& operand);
+
+ // Bitwise enor/xnor (A ^ ~B).
+ void eon(const Register& rd, const Register& rn, const Operand& operand);
+
+ // Logical shift left variable.
+ void lslv(const Register& rd, const Register& rn, const Register& rm);
+
+ // Logical shift right variable.
+ void lsrv(const Register& rd, const Register& rn, const Register& rm);
+
+ // Arithmetic shift right variable.
+ void asrv(const Register& rd, const Register& rn, const Register& rm);
+
+ // Rotate right variable.
+ void rorv(const Register& rd, const Register& rn, const Register& rm);
+
+ // Bitfield instructions.
+ // Bitfield move.
+ void bfm(const Register& rd, const Register& rn, int immr, int imms);
+
+ // Signed bitfield move.
+ void sbfm(const Register& rd, const Register& rn, int immr, int imms);
+
+ // Unsigned bitfield move.
+ void ubfm(const Register& rd, const Register& rn, int immr, int imms);
+
+ // Bfm aliases.
+ // Bitfield insert.
+ void bfi(const Register& rd, const Register& rn, int lsb, int width) {
+ DCHECK_GE(width, 1);
+ DCHECK(lsb + width <= rn.SizeInBits());
+ bfm(rd, rn, (rd.SizeInBits() - lsb) & (rd.SizeInBits() - 1), width - 1);
+ }
+
+ // Bitfield extract and insert low.
+ void bfxil(const Register& rd, const Register& rn, int lsb, int width) {
+ DCHECK_GE(width, 1);
+ DCHECK(lsb + width <= rn.SizeInBits());
+ bfm(rd, rn, lsb, lsb + width - 1);
+ }
+
+ // Sbfm aliases.
+ // Arithmetic shift right.
+ void asr(const Register& rd, const Register& rn, int shift) {
+ DCHECK(shift < rd.SizeInBits());
+ sbfm(rd, rn, shift, rd.SizeInBits() - 1);
+ }
+
+ // Signed bitfield insert in zero.
+ void sbfiz(const Register& rd, const Register& rn, int lsb, int width) {
+ DCHECK_GE(width, 1);
+ DCHECK(lsb + width <= rn.SizeInBits());
+ sbfm(rd, rn, (rd.SizeInBits() - lsb) & (rd.SizeInBits() - 1), width - 1);
+ }
+
+ // Signed bitfield extract.
+ void sbfx(const Register& rd, const Register& rn, int lsb, int width) {
+ DCHECK_GE(width, 1);
+ DCHECK(lsb + width <= rn.SizeInBits());
+ sbfm(rd, rn, lsb, lsb + width - 1);
+ }
+
+ // Signed extend byte.
+ void sxtb(const Register& rd, const Register& rn) { sbfm(rd, rn, 0, 7); }
+
+ // Signed extend halfword.
+ void sxth(const Register& rd, const Register& rn) { sbfm(rd, rn, 0, 15); }
+
+ // Signed extend word.
+ void sxtw(const Register& rd, const Register& rn) { sbfm(rd, rn, 0, 31); }
+
+ // Ubfm aliases.
+ // Logical shift left.
+ void lsl(const Register& rd, const Register& rn, int shift) {
+ int reg_size = rd.SizeInBits();
+ DCHECK(shift < reg_size);
+ ubfm(rd, rn, (reg_size - shift) % reg_size, reg_size - shift - 1);
+ }
+
+ // Logical shift right.
+ void lsr(const Register& rd, const Register& rn, int shift) {
+ DCHECK(shift < rd.SizeInBits());
+ ubfm(rd, rn, shift, rd.SizeInBits() - 1);
+ }
+
+ // Unsigned bitfield insert in zero.
+ void ubfiz(const Register& rd, const Register& rn, int lsb, int width) {
+ DCHECK_GE(width, 1);
+ DCHECK(lsb + width <= rn.SizeInBits());
+ ubfm(rd, rn, (rd.SizeInBits() - lsb) & (rd.SizeInBits() - 1), width - 1);
+ }
+
+ // Unsigned bitfield extract.
+ void ubfx(const Register& rd, const Register& rn, int lsb, int width) {
+ DCHECK_GE(width, 1);
+ DCHECK(lsb + width <= rn.SizeInBits());
+ ubfm(rd, rn, lsb, lsb + width - 1);
+ }
+
+ // Unsigned extend byte.
+ void uxtb(const Register& rd, const Register& rn) { ubfm(rd, rn, 0, 7); }
+
+ // Unsigned extend halfword.
+ void uxth(const Register& rd, const Register& rn) { ubfm(rd, rn, 0, 15); }
+
+ // Unsigned extend word.
+ void uxtw(const Register& rd, const Register& rn) { ubfm(rd, rn, 0, 31); }
+
+ // Extract.
+ void extr(const Register& rd, const Register& rn, const Register& rm,
+ int lsb);
+
+ // Conditional select: rd = cond ? rn : rm.
+ void csel(const Register& rd, const Register& rn, const Register& rm,
+ Condition cond);
+
+ // Conditional select increment: rd = cond ? rn : rm + 1.
+ void csinc(const Register& rd, const Register& rn, const Register& rm,
+ Condition cond);
+
+ // Conditional select inversion: rd = cond ? rn : ~rm.
+ void csinv(const Register& rd, const Register& rn, const Register& rm,
+ Condition cond);
+
+ // Conditional select negation: rd = cond ? rn : -rm.
+ void csneg(const Register& rd, const Register& rn, const Register& rm,
+ Condition cond);
+
+ // Conditional set: rd = cond ? 1 : 0.
+ void cset(const Register& rd, Condition cond);
+
+ // Conditional set minus: rd = cond ? -1 : 0.
+ void csetm(const Register& rd, Condition cond);
+
+ // Conditional increment: rd = cond ? rn + 1 : rn.
+ void cinc(const Register& rd, const Register& rn, Condition cond);
+
+ // Conditional invert: rd = cond ? ~rn : rn.
+ void cinv(const Register& rd, const Register& rn, Condition cond);
+
+ // Conditional negate: rd = cond ? -rn : rn.
+ void cneg(const Register& rd, const Register& rn, Condition cond);
+
+ // Extr aliases.
+ void ror(const Register& rd, const Register& rs, unsigned shift) {
+ extr(rd, rs, rs, shift);
+ }
+
+ // Conditional comparison.
+ // Conditional compare negative.
+ void ccmn(const Register& rn, const Operand& operand, StatusFlags nzcv,
+ Condition cond);
+
+ // Conditional compare.
+ void ccmp(const Register& rn, const Operand& operand, StatusFlags nzcv,
+ Condition cond);
+
+ // Multiplication.
+ // 32 x 32 -> 32-bit and 64 x 64 -> 64-bit multiply.
+ void mul(const Register& rd, const Register& rn, const Register& rm);
+
+ // 32 + 32 x 32 -> 32-bit and 64 + 64 x 64 -> 64-bit multiply accumulate.
+ void madd(const Register& rd, const Register& rn, const Register& rm,
+ const Register& ra);
+
+ // -(32 x 32) -> 32-bit and -(64 x 64) -> 64-bit multiply.
+ void mneg(const Register& rd, const Register& rn, const Register& rm);
+
+ // 32 - 32 x 32 -> 32-bit and 64 - 64 x 64 -> 64-bit multiply subtract.
+ void msub(const Register& rd, const Register& rn, const Register& rm,
+ const Register& ra);
+
+ // 32 x 32 -> 64-bit multiply.
+ void smull(const Register& rd, const Register& rn, const Register& rm);
+
+ // Xd = bits<127:64> of Xn * Xm.
+ void smulh(const Register& rd, const Register& rn, const Register& rm);
+
+ // Signed 32 x 32 -> 64-bit multiply and accumulate.
+ void smaddl(const Register& rd, const Register& rn, const Register& rm,
+ const Register& ra);
+
+ // Unsigned 32 x 32 -> 64-bit multiply and accumulate.
+ void umaddl(const Register& rd, const Register& rn, const Register& rm,
+ const Register& ra);
+
+ // Signed 32 x 32 -> 64-bit multiply and subtract.
+ void smsubl(const Register& rd, const Register& rn, const Register& rm,
+ const Register& ra);
+
+ // Unsigned 32 x 32 -> 64-bit multiply and subtract.
+ void umsubl(const Register& rd, const Register& rn, const Register& rm,
+ const Register& ra);
+
+ // Signed integer divide.
+ void sdiv(const Register& rd, const Register& rn, const Register& rm);
+
+ // Unsigned integer divide.
+ void udiv(const Register& rd, const Register& rn, const Register& rm);
+
+ // Bit count, bit reverse and endian reverse.
+ void rbit(const Register& rd, const Register& rn);
+ void rev16(const Register& rd, const Register& rn);
+ void rev32(const Register& rd, const Register& rn);
+ void rev(const Register& rd, const Register& rn);
+ void clz(const Register& rd, const Register& rn);
+ void cls(const Register& rd, const Register& rn);
+
+ // Pointer Authentication Code for Instruction address, using key B, with
+ // address in x17 and modifier in x16 [Armv8.3].
+ void pacib1716();
+
+ // Pointer Authentication Code for Instruction address, using key B, with
+ // address in LR and modifier in SP [Armv8.3].
+ void pacibsp();
+
+ // Authenticate Instruction address, using key B, with address in x17 and
+ // modifier in x16 [Armv8.3].
+ void autib1716();
+
+ // Authenticate Instruction address, using key B, with address in LR and
+ // modifier in SP [Armv8.3].
+ void autibsp();
+
+ // Memory instructions.
+
+ // Load integer or FP register.
+ void ldr(const CPURegister& rt, const MemOperand& src);
+
+ // Store integer or FP register.
+ void str(const CPURegister& rt, const MemOperand& dst);
+
+ // Load word with sign extension.
+ void ldrsw(const Register& rt, const MemOperand& src);
+
+ // Load byte.
+ void ldrb(const Register& rt, const MemOperand& src);
+
+ // Store byte.
+ void strb(const Register& rt, const MemOperand& dst);
+
+ // Load byte with sign extension.
+ void ldrsb(const Register& rt, const MemOperand& src);
+
+ // Load half-word.
+ void ldrh(const Register& rt, const MemOperand& src);
+
+ // Store half-word.
+ void strh(const Register& rt, const MemOperand& dst);
+
+ // Load half-word with sign extension.
+ void ldrsh(const Register& rt, const MemOperand& src);
+
+ // Load integer or FP register pair.
+ void ldp(const CPURegister& rt, const CPURegister& rt2,
+ const MemOperand& src);
+
+ // Store integer or FP register pair.
+ void stp(const CPURegister& rt, const CPURegister& rt2,
+ const MemOperand& dst);
+
+ // Load word pair with sign extension.
+ void ldpsw(const Register& rt, const Register& rt2, const MemOperand& src);
+
+ // Load literal to register from a pc relative address.
+ void ldr_pcrel(const CPURegister& rt, int imm19);
+
+ // Load literal to register.
+ void ldr(const CPURegister& rt, const Immediate& imm);
+ void ldr(const CPURegister& rt, const Operand& operand);
+
+ // Load-acquire word.
+ void ldar(const Register& rt, const Register& rn);
+
+ // Load-acquire exclusive word.
+ void ldaxr(const Register& rt, const Register& rn);
+
+ // Store-release word.
+ void stlr(const Register& rt, const Register& rn);
+
+ // Store-release exclusive word.
+ void stlxr(const Register& rs, const Register& rt, const Register& rn);
+
+ // Load-acquire byte.
+ void ldarb(const Register& rt, const Register& rn);
+
+ // Load-acquire exclusive byte.
+ void ldaxrb(const Register& rt, const Register& rn);
+
+ // Store-release byte.
+ void stlrb(const Register& rt, const Register& rn);
+
+ // Store-release exclusive byte.
+ void stlxrb(const Register& rs, const Register& rt, const Register& rn);
+
+ // Load-acquire half-word.
+ void ldarh(const Register& rt, const Register& rn);
+
+ // Load-acquire exclusive half-word.
+ void ldaxrh(const Register& rt, const Register& rn);
+
+ // Store-release half-word.
+ void stlrh(const Register& rt, const Register& rn);
+
+ // Store-release exclusive half-word.
+ void stlxrh(const Register& rs, const Register& rt, const Register& rn);
+
+ // Move instructions. The default shift of -1 indicates that the move
+ // instruction will calculate an appropriate 16-bit immediate and left shift
+ // that is equal to the 64-bit immediate argument. If an explicit left shift
+ // is specified (0, 16, 32 or 48), the immediate must be a 16-bit value.
+ //
+ // For movk, an explicit shift can be used to indicate which half word should
+ // be overwritten, eg. movk(x0, 0, 0) will overwrite the least-significant
+ // half word with zero, whereas movk(x0, 0, 48) will overwrite the
+ // most-significant.
+
+ // Move and keep.
+ void movk(const Register& rd, uint64_t imm, int shift = -1) {
+ MoveWide(rd, imm, shift, MOVK);
+ }
+
+ // Move with non-zero.
+ void movn(const Register& rd, uint64_t imm, int shift = -1) {
+ MoveWide(rd, imm, shift, MOVN);
+ }
+
+ // Move with zero.
+ void movz(const Register& rd, uint64_t imm, int shift = -1) {
+ MoveWide(rd, imm, shift, MOVZ);
+ }
+
+ // Misc instructions.
+ // Monitor debug-mode breakpoint.
+ void brk(int code);
+
+ // Halting debug-mode breakpoint.
+ void hlt(int code);
+
+ // Move register to register.
+ void mov(const Register& rd, const Register& rn);
+
+ // Move NOT(operand) to register.
+ void mvn(const Register& rd, const Operand& operand);
+
+ // System instructions.
+ // Move to register from system register.
+ void mrs(const Register& rt, SystemRegister sysreg);
+
+ // Move from register to system register.
+ void msr(SystemRegister sysreg, const Register& rt);
+
+ // System hint.
+ void hint(SystemHint code);
+
+ // Data memory barrier
+ void dmb(BarrierDomain domain, BarrierType type);
+
+ // Data synchronization barrier
+ void dsb(BarrierDomain domain, BarrierType type);
+
+ // Instruction synchronization barrier
+ void isb();
+
+ // Conditional speculation barrier.
+ void csdb();
+
+ // Branch target identification.
+ void bti(BranchTargetIdentifier id);
+
+ // No-op.
+ void nop() { hint(NOP); }
+
+ // Different nop operations are used by the code generator to detect certain
+ // states of the generated code.
+ enum NopMarkerTypes {
+ DEBUG_BREAK_NOP,
+ INTERRUPT_CODE_NOP,
+ ADR_FAR_NOP,
+ FIRST_NOP_MARKER = DEBUG_BREAK_NOP,
+ LAST_NOP_MARKER = ADR_FAR_NOP
+ };
+
+ void nop(NopMarkerTypes n);
+
+ // Add.
+ void add(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned halving add.
+ void uhadd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Subtract.
+ void sub(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed halving add.
+ void shadd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Multiply by scalar element.
+ void mul(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // Multiply-add by scalar element.
+ void mla(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // Multiply-subtract by scalar element.
+ void mls(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // Signed long multiply-add by scalar element.
+ void smlal(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // Signed long multiply-add by scalar element (second part).
+ void smlal2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // Unsigned long multiply-add by scalar element.
+ void umlal(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // Unsigned long multiply-add by scalar element (second part).
+ void umlal2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // Signed long multiply-sub by scalar element.
+ void smlsl(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // Signed long multiply-sub by scalar element (second part).
+ void smlsl2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // Unsigned long multiply-sub by scalar element.
+ void umlsl(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // Unsigned long multiply-sub by scalar element (second part).
+ void umlsl2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // Signed long multiply by scalar element.
+ void smull(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // Signed long multiply by scalar element (second part).
+ void smull2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // Unsigned long multiply by scalar element.
+ void umull(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // Unsigned long multiply by scalar element (second part).
+ void umull2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // Add narrow returning high half.
+ void addhn(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Add narrow returning high half (second part).
+ void addhn2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed saturating double long multiply by element.
+ void sqdmull(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // Signed saturating double long multiply by element (second part).
+ void sqdmull2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // Signed saturating doubling long multiply-add by element.
+ void sqdmlal(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // Signed saturating doubling long multiply-add by element (second part).
+ void sqdmlal2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // Signed saturating doubling long multiply-sub by element.
+ void sqdmlsl(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // Signed saturating doubling long multiply-sub by element (second part).
+ void sqdmlsl2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // Compare bitwise to zero.
+ void cmeq(const VRegister& vd, const VRegister& vn, int value);
+
+ // Compare signed greater than or equal to zero.
+ void cmge(const VRegister& vd, const VRegister& vn, int value);
+
+ // Compare signed greater than zero.
+ void cmgt(const VRegister& vd, const VRegister& vn, int value);
+
+ // Compare signed less than or equal to zero.
+ void cmle(const VRegister& vd, const VRegister& vn, int value);
+
+ // Compare signed less than zero.
+ void cmlt(const VRegister& vd, const VRegister& vn, int value);
+
+ // Unsigned rounding halving add.
+ void urhadd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Compare equal.
+ void cmeq(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Compare signed greater than or equal.
+ void cmge(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Compare signed greater than.
+ void cmgt(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Compare unsigned higher.
+ void cmhi(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Compare unsigned higher or same.
+ void cmhs(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Compare bitwise test bits nonzero.
+ void cmtst(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed shift left by register.
+ void sshl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned shift left by register.
+ void ushl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed saturating doubling long multiply-subtract.
+ void sqdmlsl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed saturating doubling long multiply-subtract (second part).
+ void sqdmlsl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed saturating doubling long multiply.
+ void sqdmull(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed saturating doubling long multiply (second part).
+ void sqdmull2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed saturating doubling multiply returning high half.
+ void sqdmulh(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed saturating rounding doubling multiply returning high half.
+ void sqrdmulh(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed saturating doubling multiply element returning high half.
+ void sqdmulh(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // Signed saturating rounding doubling multiply element returning high half.
+ void sqrdmulh(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // Unsigned long multiply long.
+ void umull(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned long multiply (second part).
+ void umull2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Rounding add narrow returning high half.
+ void raddhn(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Subtract narrow returning high half.
+ void subhn(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Subtract narrow returning high half (second part).
+ void subhn2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Rounding add narrow returning high half (second part).
+ void raddhn2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Rounding subtract narrow returning high half.
+ void rsubhn(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Rounding subtract narrow returning high half (second part).
+ void rsubhn2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed saturating shift left by register.
+ void sqshl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned saturating shift left by register.
+ void uqshl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed rounding shift left by register.
+ void srshl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned rounding shift left by register.
+ void urshl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed saturating rounding shift left by register.
+ void sqrshl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned saturating rounding shift left by register.
+ void uqrshl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed absolute difference.
+ void sabd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned absolute difference and accumulate.
+ void uaba(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Shift left by immediate and insert.
+ void sli(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Shift right by immediate and insert.
+ void sri(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Signed maximum.
+ void smax(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed pairwise maximum.
+ void smaxp(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Add across vector.
+ void addv(const VRegister& vd, const VRegister& vn);
+
+ // Signed add long across vector.
+ void saddlv(const VRegister& vd, const VRegister& vn);
+
+ // Unsigned add long across vector.
+ void uaddlv(const VRegister& vd, const VRegister& vn);
+
+ // FP maximum number across vector.
+ void fmaxnmv(const VRegister& vd, const VRegister& vn);
+
+ // FP maximum across vector.
+ void fmaxv(const VRegister& vd, const VRegister& vn);
+
+ // FP minimum number across vector.
+ void fminnmv(const VRegister& vd, const VRegister& vn);
+
+ // FP minimum across vector.
+ void fminv(const VRegister& vd, const VRegister& vn);
+
+ // Signed maximum across vector.
+ void smaxv(const VRegister& vd, const VRegister& vn);
+
+ // Signed minimum.
+ void smin(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed minimum pairwise.
+ void sminp(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed minimum across vector.
+ void sminv(const VRegister& vd, const VRegister& vn);
+
+ // One-element structure store from one register.
+ void st1(const VRegister& vt, const MemOperand& src);
+
+ // One-element structure store from two registers.
+ void st1(const VRegister& vt, const VRegister& vt2, const MemOperand& src);
+
+ // One-element structure store from three registers.
+ void st1(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ const MemOperand& src);
+
+ // One-element structure store from four registers.
+ void st1(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ const VRegister& vt4, const MemOperand& src);
+
+ // One-element single structure store from one lane.
+ void st1(const VRegister& vt, int lane, const MemOperand& src);
+
+ // Two-element structure store from two registers.
+ void st2(const VRegister& vt, const VRegister& vt2, const MemOperand& src);
+
+ // Two-element single structure store from two lanes.
+ void st2(const VRegister& vt, const VRegister& vt2, int lane,
+ const MemOperand& src);
+
+ // Three-element structure store from three registers.
+ void st3(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ const MemOperand& src);
+
+ // Three-element single structure store from three lanes.
+ void st3(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ int lane, const MemOperand& src);
+
+ // Four-element structure store from four registers.
+ void st4(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ const VRegister& vt4, const MemOperand& src);
+
+ // Four-element single structure store from four lanes.
+ void st4(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ const VRegister& vt4, int lane, const MemOperand& src);
+
+ // Unsigned add long.
+ void uaddl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned add long (second part).
+ void uaddl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned add wide.
+ void uaddw(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned add wide (second part).
+ void uaddw2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed add long.
+ void saddl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed add long (second part).
+ void saddl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed add wide.
+ void saddw(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed add wide (second part).
+ void saddw2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned subtract long.
+ void usubl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned subtract long (second part).
+ void usubl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned subtract wide.
+ void usubw(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed subtract long.
+ void ssubl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed subtract long (second part).
+ void ssubl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed integer subtract wide.
+ void ssubw(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed integer subtract wide (second part).
+ void ssubw2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned subtract wide (second part).
+ void usubw2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned maximum.
+ void umax(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned pairwise maximum.
+ void umaxp(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned maximum across vector.
+ void umaxv(const VRegister& vd, const VRegister& vn);
+
+ // Unsigned minimum.
+ void umin(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned pairwise minimum.
+ void uminp(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned minimum across vector.
+ void uminv(const VRegister& vd, const VRegister& vn);
+
+ // Transpose vectors (primary).
+ void trn1(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Transpose vectors (secondary).
+ void trn2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unzip vectors (primary).
+ void uzp1(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unzip vectors (secondary).
+ void uzp2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Zip vectors (primary).
+ void zip1(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Zip vectors (secondary).
+ void zip2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed shift right by immediate.
+ void sshr(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Unsigned shift right by immediate.
+ void ushr(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Signed rounding shift right by immediate.
+ void srshr(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Unsigned rounding shift right by immediate.
+ void urshr(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Signed shift right by immediate and accumulate.
+ void ssra(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Unsigned shift right by immediate and accumulate.
+ void usra(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Signed rounding shift right by immediate and accumulate.
+ void srsra(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Unsigned rounding shift right by immediate and accumulate.
+ void ursra(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Shift right narrow by immediate.
+ void shrn(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Shift right narrow by immediate (second part).
+ void shrn2(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Rounding shift right narrow by immediate.
+ void rshrn(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Rounding shift right narrow by immediate (second part).
+ void rshrn2(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Unsigned saturating shift right narrow by immediate.
+ void uqshrn(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Unsigned saturating shift right narrow by immediate (second part).
+ void uqshrn2(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Unsigned saturating rounding shift right narrow by immediate.
+ void uqrshrn(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Unsigned saturating rounding shift right narrow by immediate (second part).
+ void uqrshrn2(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Signed saturating shift right narrow by immediate.
+ void sqshrn(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Signed saturating shift right narrow by immediate (second part).
+ void sqshrn2(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Signed saturating rounded shift right narrow by immediate.
+ void sqrshrn(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Signed saturating rounded shift right narrow by immediate (second part).
+ void sqrshrn2(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Signed saturating shift right unsigned narrow by immediate.
+ void sqshrun(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Signed saturating shift right unsigned narrow by immediate (second part).
+ void sqshrun2(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Signed sat rounded shift right unsigned narrow by immediate.
+ void sqrshrun(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Signed sat rounded shift right unsigned narrow by immediate (second part).
+ void sqrshrun2(const VRegister& vd, const VRegister& vn, int shift);
+
+ // FP reciprocal step.
+ void frecps(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP reciprocal estimate.
+ void frecpe(const VRegister& vd, const VRegister& vn);
+
+ // FP reciprocal square root estimate.
+ void frsqrte(const VRegister& vd, const VRegister& vn);
+
+ // FP reciprocal square root step.
+ void frsqrts(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed absolute difference and accumulate long.
+ void sabal(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed absolute difference and accumulate long (second part).
+ void sabal2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned absolute difference and accumulate long.
+ void uabal(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned absolute difference and accumulate long (second part).
+ void uabal2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed absolute difference long.
+ void sabdl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed absolute difference long (second part).
+ void sabdl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned absolute difference long.
+ void uabdl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned absolute difference long (second part).
+ void uabdl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Polynomial multiply long.
+ void pmull(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Polynomial multiply long (second part).
+ void pmull2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed long multiply-add.
+ void smlal(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed long multiply-add (second part).
+ void smlal2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned long multiply-add.
+ void umlal(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned long multiply-add (second part).
+ void umlal2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed long multiply-sub.
+ void smlsl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed long multiply-sub (second part).
+ void smlsl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned long multiply-sub.
+ void umlsl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned long multiply-sub (second part).
+ void umlsl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed long multiply.
+ void smull(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed long multiply (second part).
+ void smull2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed saturating doubling long multiply-add.
+ void sqdmlal(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed saturating doubling long multiply-add (second part).
+ void sqdmlal2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned absolute difference.
+ void uabd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed absolute difference and accumulate.
+ void saba(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP instructions.
+ // Move immediate to FP register.
+ void fmov(const VRegister& fd, double imm);
+ void fmov(const VRegister& fd, float imm);
+
+ // Move FP register to register.
+ void fmov(const Register& rd, const VRegister& fn);
+
+ // Move register to FP register.
+ void fmov(const VRegister& fd, const Register& rn);
+
+ // Move FP register to FP register.
+ void fmov(const VRegister& fd, const VRegister& fn);
+
+ // Move 64-bit register to top half of 128-bit FP register.
+ void fmov(const VRegister& vd, int index, const Register& rn);
+
+ // Move top half of 128-bit FP register to 64-bit register.
+ void fmov(const Register& rd, const VRegister& vn, int index);
+
+ // FP add.
+ void fadd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP subtract.
+ void fsub(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP multiply.
+ void fmul(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP compare equal to zero.
+ void fcmeq(const VRegister& vd, const VRegister& vn, double imm);
+
+ // FP greater than zero.
+ void fcmgt(const VRegister& vd, const VRegister& vn, double imm);
+
+ // FP greater than or equal to zero.
+ void fcmge(const VRegister& vd, const VRegister& vn, double imm);
+
+ // FP less than or equal to zero.
+ void fcmle(const VRegister& vd, const VRegister& vn, double imm);
+
+ // FP less than to zero.
+ void fcmlt(const VRegister& vd, const VRegister& vn, double imm);
+
+ // FP absolute difference.
+ void fabd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP pairwise add vector.
+ void faddp(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP pairwise add scalar.
+ void faddp(const VRegister& vd, const VRegister& vn);
+
+ // FP pairwise maximum scalar.
+ void fmaxp(const VRegister& vd, const VRegister& vn);
+
+ // FP pairwise maximum number scalar.
+ void fmaxnmp(const VRegister& vd, const VRegister& vn);
+
+ // FP pairwise minimum number scalar.
+ void fminnmp(const VRegister& vd, const VRegister& vn);
+
+ // FP vector multiply accumulate.
+ void fmla(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP vector multiply subtract.
+ void fmls(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP vector multiply extended.
+ void fmulx(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP absolute greater than or equal.
+ void facge(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP absolute greater than.
+ void facgt(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP multiply by element.
+ void fmul(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // FP fused multiply-add to accumulator by element.
+ void fmla(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // FP fused multiply-sub from accumulator by element.
+ void fmls(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // FP multiply extended by element.
+ void fmulx(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int vm_index);
+
+ // FP compare equal.
+ void fcmeq(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP greater than.
+ void fcmgt(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP greater than or equal.
+ void fcmge(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP pairwise maximum vector.
+ void fmaxp(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP pairwise minimum vector.
+ void fminp(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP pairwise minimum scalar.
+ void fminp(const VRegister& vd, const VRegister& vn);
+
+ // FP pairwise maximum number vector.
+ void fmaxnmp(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP pairwise minimum number vector.
+ void fminnmp(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP fused multiply-add.
+ void fmadd(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ const VRegister& va);
+
+ // FP fused multiply-subtract.
+ void fmsub(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ const VRegister& va);
+
+ // FP fused multiply-add and negate.
+ void fnmadd(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ const VRegister& va);
+
+ // FP fused multiply-subtract and negate.
+ void fnmsub(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ const VRegister& va);
+
+ // FP multiply-negate scalar.
+ void fnmul(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP reciprocal exponent scalar.
+ void frecpx(const VRegister& vd, const VRegister& vn);
+
+ // FP divide.
+ void fdiv(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP maximum.
+ void fmax(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP minimum.
+ void fmin(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP maximum.
+ void fmaxnm(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP minimum.
+ void fminnm(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP absolute.
+ void fabs(const VRegister& vd, const VRegister& vn);
+
+ // FP negate.
+ void fneg(const VRegister& vd, const VRegister& vn);
+
+ // FP square root.
+ void fsqrt(const VRegister& vd, const VRegister& vn);
+
+ // FP round to integer nearest with ties to away.
+ void frinta(const VRegister& vd, const VRegister& vn);
+
+ // FP round to integer, implicit rounding.
+ void frinti(const VRegister& vd, const VRegister& vn);
+
+ // FP round to integer toward minus infinity.
+ void frintm(const VRegister& vd, const VRegister& vn);
+
+ // FP round to integer nearest with ties to even.
+ void frintn(const VRegister& vd, const VRegister& vn);
+
+ // FP round to integer towards plus infinity.
+ void frintp(const VRegister& vd, const VRegister& vn);
+
+ // FP round to integer, exact, implicit rounding.
+ void frintx(const VRegister& vd, const VRegister& vn);
+
+ // FP round to integer towards zero.
+ void frintz(const VRegister& vd, const VRegister& vn);
+
+ // FP compare registers.
+ void fcmp(const VRegister& vn, const VRegister& vm);
+
+ // FP compare immediate.
+ void fcmp(const VRegister& vn, double value);
+
+ // FP conditional compare.
+ void fccmp(const VRegister& vn, const VRegister& vm, StatusFlags nzcv,
+ Condition cond);
+
+ // FP conditional select.
+ void fcsel(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ Condition cond);
+
+ // Common FP Convert functions.
+ void NEONFPConvertToInt(const Register& rd, const VRegister& vn, Instr op);
+ void NEONFPConvertToInt(const VRegister& vd, const VRegister& vn, Instr op);
+
+ // FP convert between precisions.
+ void fcvt(const VRegister& vd, const VRegister& vn);
+
+ // FP convert to higher precision.
+ void fcvtl(const VRegister& vd, const VRegister& vn);
+
+ // FP convert to higher precision (second part).
+ void fcvtl2(const VRegister& vd, const VRegister& vn);
+
+ // FP convert to lower precision.
+ void fcvtn(const VRegister& vd, const VRegister& vn);
+
+ // FP convert to lower prevision (second part).
+ void fcvtn2(const VRegister& vd, const VRegister& vn);
+
+ // FP convert to lower precision, rounding to odd.
+ void fcvtxn(const VRegister& vd, const VRegister& vn);
+
+ // FP convert to lower precision, rounding to odd (second part).
+ void fcvtxn2(const VRegister& vd, const VRegister& vn);
+
+ // FP convert to signed integer, nearest with ties to away.
+ void fcvtas(const Register& rd, const VRegister& vn);
+
+ // FP convert to unsigned integer, nearest with ties to away.
+ void fcvtau(const Register& rd, const VRegister& vn);
+
+ // FP convert to signed integer, nearest with ties to away.
+ void fcvtas(const VRegister& vd, const VRegister& vn);
+
+ // FP convert to unsigned integer, nearest with ties to away.
+ void fcvtau(const VRegister& vd, const VRegister& vn);
+
+ // FP convert to signed integer, round towards -infinity.
+ void fcvtms(const Register& rd, const VRegister& vn);
+
+ // FP convert to unsigned integer, round towards -infinity.
+ void fcvtmu(const Register& rd, const VRegister& vn);
+
+ // FP convert to signed integer, round towards -infinity.
+ void fcvtms(const VRegister& vd, const VRegister& vn);
+
+ // FP convert to unsigned integer, round towards -infinity.
+ void fcvtmu(const VRegister& vd, const VRegister& vn);
+
+ // FP convert to signed integer, nearest with ties to even.
+ void fcvtns(const Register& rd, const VRegister& vn);
+
+ // FP JavaScript convert to signed integer, rounding toward zero [Armv8.3].
+ void fjcvtzs(const Register& rd, const VRegister& vn);
+
+ // FP convert to unsigned integer, nearest with ties to even.
+ void fcvtnu(const Register& rd, const VRegister& vn);
+
+ // FP convert to signed integer, nearest with ties to even.
+ void fcvtns(const VRegister& rd, const VRegister& vn);
+
+ // FP convert to unsigned integer, nearest with ties to even.
+ void fcvtnu(const VRegister& rd, const VRegister& vn);
+
+ // FP convert to signed integer or fixed-point, round towards zero.
+ void fcvtzs(const Register& rd, const VRegister& vn, int fbits = 0);
+
+ // FP convert to unsigned integer or fixed-point, round towards zero.
+ void fcvtzu(const Register& rd, const VRegister& vn, int fbits = 0);
+
+ // FP convert to signed integer or fixed-point, round towards zero.
+ void fcvtzs(const VRegister& vd, const VRegister& vn, int fbits = 0);
+
+ // FP convert to unsigned integer or fixed-point, round towards zero.
+ void fcvtzu(const VRegister& vd, const VRegister& vn, int fbits = 0);
+
+ // FP convert to signed integer, round towards +infinity.
+ void fcvtps(const Register& rd, const VRegister& vn);
+
+ // FP convert to unsigned integer, round towards +infinity.
+ void fcvtpu(const Register& rd, const VRegister& vn);
+
+ // FP convert to signed integer, round towards +infinity.
+ void fcvtps(const VRegister& vd, const VRegister& vn);
+
+ // FP convert to unsigned integer, round towards +infinity.
+ void fcvtpu(const VRegister& vd, const VRegister& vn);
+
+ // Convert signed integer or fixed point to FP.
+ void scvtf(const VRegister& fd, const Register& rn, int fbits = 0);
+
+ // Convert unsigned integer or fixed point to FP.
+ void ucvtf(const VRegister& fd, const Register& rn, int fbits = 0);
+
+ // Convert signed integer or fixed-point to FP.
+ void scvtf(const VRegister& fd, const VRegister& vn, int fbits = 0);
+
+ // Convert unsigned integer or fixed-point to FP.
+ void ucvtf(const VRegister& fd, const VRegister& vn, int fbits = 0);
+
+ // Extract vector from pair of vectors.
+ void ext(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int index);
+
+ // Duplicate vector element to vector or scalar.
+ void dup(const VRegister& vd, const VRegister& vn, int vn_index);
+
+ // Duplicate general-purpose register to vector.
+ void dup(const VRegister& vd, const Register& rn);
+
+ // Insert vector element from general-purpose register.
+ void ins(const VRegister& vd, int vd_index, const Register& rn);
+
+ // Move general-purpose register to a vector element.
+ void mov(const VRegister& vd, int vd_index, const Register& rn);
+
+ // Unsigned move vector element to general-purpose register.
+ void umov(const Register& rd, const VRegister& vn, int vn_index);
+
+ // Move vector element to general-purpose register.
+ void mov(const Register& rd, const VRegister& vn, int vn_index);
+
+ // Move vector element to scalar.
+ void mov(const VRegister& vd, const VRegister& vn, int vn_index);
+
+ // Insert vector element from another vector element.
+ void ins(const VRegister& vd, int vd_index, const VRegister& vn,
+ int vn_index);
+
+ // Move vector element to another vector element.
+ void mov(const VRegister& vd, int vd_index, const VRegister& vn,
+ int vn_index);
+
+ // Signed move vector element to general-purpose register.
+ void smov(const Register& rd, const VRegister& vn, int vn_index);
+
+ // One-element structure load to one register.
+ void ld1(const VRegister& vt, const MemOperand& src);
+
+ // One-element structure load to two registers.
+ void ld1(const VRegister& vt, const VRegister& vt2, const MemOperand& src);
+
+ // One-element structure load to three registers.
+ void ld1(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ const MemOperand& src);
+
+ // One-element structure load to four registers.
+ void ld1(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ const VRegister& vt4, const MemOperand& src);
+
+ // One-element single structure load to one lane.
+ void ld1(const VRegister& vt, int lane, const MemOperand& src);
+
+ // One-element single structure load to all lanes.
+ void ld1r(const VRegister& vt, const MemOperand& src);
+
+ // Two-element structure load.
+ void ld2(const VRegister& vt, const VRegister& vt2, const MemOperand& src);
+
+ // Two-element single structure load to one lane.
+ void ld2(const VRegister& vt, const VRegister& vt2, int lane,
+ const MemOperand& src);
+
+ // Two-element single structure load to all lanes.
+ void ld2r(const VRegister& vt, const VRegister& vt2, const MemOperand& src);
+
+ // Three-element structure load.
+ void ld3(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ const MemOperand& src);
+
+ // Three-element single structure load to one lane.
+ void ld3(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ int lane, const MemOperand& src);
+
+ // Three-element single structure load to all lanes.
+ void ld3r(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ const MemOperand& src);
+
+ // Four-element structure load.
+ void ld4(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ const VRegister& vt4, const MemOperand& src);
+
+ // Four-element single structure load to one lane.
+ void ld4(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ const VRegister& vt4, int lane, const MemOperand& src);
+
+ // Four-element single structure load to all lanes.
+ void ld4r(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ const VRegister& vt4, const MemOperand& src);
+
+ // Count leading sign bits.
+ void cls(const VRegister& vd, const VRegister& vn);
+
+ // Count leading zero bits (vector).
+ void clz(const VRegister& vd, const VRegister& vn);
+
+ // Population count per byte.
+ void cnt(const VRegister& vd, const VRegister& vn);
+
+ // Reverse bit order.
+ void rbit(const VRegister& vd, const VRegister& vn);
+
+ // Reverse elements in 16-bit halfwords.
+ void rev16(const VRegister& vd, const VRegister& vn);
+
+ // Reverse elements in 32-bit words.
+ void rev32(const VRegister& vd, const VRegister& vn);
+
+ // Reverse elements in 64-bit doublewords.
+ void rev64(const VRegister& vd, const VRegister& vn);
+
+ // Unsigned reciprocal square root estimate.
+ void ursqrte(const VRegister& vd, const VRegister& vn);
+
+ // Unsigned reciprocal estimate.
+ void urecpe(const VRegister& vd, const VRegister& vn);
+
+ // Signed pairwise long add and accumulate.
+ void sadalp(const VRegister& vd, const VRegister& vn);
+
+ // Signed pairwise long add.
+ void saddlp(const VRegister& vd, const VRegister& vn);
+
+ // Unsigned pairwise long add.
+ void uaddlp(const VRegister& vd, const VRegister& vn);
+
+ // Unsigned pairwise long add and accumulate.
+ void uadalp(const VRegister& vd, const VRegister& vn);
+
+ // Shift left by immediate.
+ void shl(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Signed saturating shift left by immediate.
+ void sqshl(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Signed saturating shift left unsigned by immediate.
+ void sqshlu(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Unsigned saturating shift left by immediate.
+ void uqshl(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Signed shift left long by immediate.
+ void sshll(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Signed shift left long by immediate (second part).
+ void sshll2(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Signed extend long.
+ void sxtl(const VRegister& vd, const VRegister& vn);
+
+ // Signed extend long (second part).
+ void sxtl2(const VRegister& vd, const VRegister& vn);
+
+ // Unsigned shift left long by immediate.
+ void ushll(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Unsigned shift left long by immediate (second part).
+ void ushll2(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Shift left long by element size.
+ void shll(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Shift left long by element size (second part).
+ void shll2(const VRegister& vd, const VRegister& vn, int shift);
+
+ // Unsigned extend long.
+ void uxtl(const VRegister& vd, const VRegister& vn);
+
+ // Unsigned extend long (second part).
+ void uxtl2(const VRegister& vd, const VRegister& vn);
+
+ // Signed rounding halving add.
+ void srhadd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned halving sub.
+ void uhsub(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed halving sub.
+ void shsub(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned saturating add.
+ void uqadd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed saturating add.
+ void sqadd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Unsigned saturating subtract.
+ void uqsub(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Signed saturating subtract.
+ void sqsub(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Add pairwise.
+ void addp(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Add pair of elements scalar.
+ void addp(const VRegister& vd, const VRegister& vn);
+
+ // Multiply-add to accumulator.
+ void mla(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Multiply-subtract to accumulator.
+ void mls(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Multiply.
+ void mul(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Table lookup from one register.
+ void tbl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Table lookup from two registers.
+ void tbl(const VRegister& vd, const VRegister& vn, const VRegister& vn2,
+ const VRegister& vm);
+
+ // Table lookup from three registers.
+ void tbl(const VRegister& vd, const VRegister& vn, const VRegister& vn2,
+ const VRegister& vn3, const VRegister& vm);
+
+ // Table lookup from four registers.
+ void tbl(const VRegister& vd, const VRegister& vn, const VRegister& vn2,
+ const VRegister& vn3, const VRegister& vn4, const VRegister& vm);
+
+ // Table lookup extension from one register.
+ void tbx(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // Table lookup extension from two registers.
+ void tbx(const VRegister& vd, const VRegister& vn, const VRegister& vn2,
+ const VRegister& vm);
+
+ // Table lookup extension from three registers.
+ void tbx(const VRegister& vd, const VRegister& vn, const VRegister& vn2,
+ const VRegister& vn3, const VRegister& vm);
+
+ // Table lookup extension from four registers.
+ void tbx(const VRegister& vd, const VRegister& vn, const VRegister& vn2,
+ const VRegister& vn3, const VRegister& vn4, const VRegister& vm);
+
+ // Instruction functions used only for test, debug, and patching.
+ // Emit raw instructions in the instruction stream.
+ void dci(Instr raw_inst) { Emit(raw_inst); }
+
+ // Emit 8 bits of data in the instruction stream.
+ void dc8(uint8_t data) { EmitData(&data, sizeof(data)); }
+
+ // Emit 32 bits of data in the instruction stream.
+ void dc32(uint32_t data) { EmitData(&data, sizeof(data)); }
+
+ // Emit 64 bits of data in the instruction stream.
+ void dc64(uint64_t data) { EmitData(&data, sizeof(data)); }
+
+ // Emit an address in the instruction stream.
+ void dcptr(Label* label);
+
+ // Copy a string into the instruction stream, including the terminating
+ // nullptr character. The instruction pointer (pc_) is then aligned correctly
+ // for subsequent instructions.
+ void EmitStringData(const char* string);
+
+ // Pseudo-instructions ------------------------------------------------------
+
+ // Parameters are described in arm64/instructions-arm64.h.
+ void debug(const char* message, uint32_t code, Instr params = BREAK);
+
+ // Required by V8.
+ void dd(uint32_t data) { dc32(data); }
+ void db(uint8_t data) { dc8(data); }
+ void dq(uint64_t data) { dc64(data); }
+ void dp(uintptr_t data) { dc64(data); }
+
+ // Code generation helpers --------------------------------------------------
+
+ Instruction* pc() const { return Instruction::Cast(pc_); }
+
+ Instruction* InstructionAt(ptrdiff_t offset) const {
+ return reinterpret_cast<Instruction*>(buffer_start_ + offset);
+ }
+
+ ptrdiff_t InstructionOffset(Instruction* instr) const {
+ return reinterpret_cast<byte*>(instr) - buffer_start_;
+ }
+
+ // Register encoding.
+ static Instr Rd(CPURegister rd) {
+ DCHECK_NE(rd.code(), kSPRegInternalCode);
+ return rd.code() << Rd_offset;
+ }
+
+ static Instr Rn(CPURegister rn) {
+ DCHECK_NE(rn.code(), kSPRegInternalCode);
+ return rn.code() << Rn_offset;
+ }
+
+ static Instr Rm(CPURegister rm) {
+ DCHECK_NE(rm.code(), kSPRegInternalCode);
+ return rm.code() << Rm_offset;
+ }
+
+ static Instr RmNot31(CPURegister rm) {
+ DCHECK_NE(rm.code(), kSPRegInternalCode);
+ DCHECK(!rm.IsZero());
+ return Rm(rm);
+ }
+
+ static Instr Ra(CPURegister ra) {
+ DCHECK_NE(ra.code(), kSPRegInternalCode);
+ return ra.code() << Ra_offset;
+ }
+
+ static Instr Rt(CPURegister rt) {
+ DCHECK_NE(rt.code(), kSPRegInternalCode);
+ return rt.code() << Rt_offset;
+ }
+
+ static Instr Rt2(CPURegister rt2) {
+ DCHECK_NE(rt2.code(), kSPRegInternalCode);
+ return rt2.code() << Rt2_offset;
+ }
+
+ static Instr Rs(CPURegister rs) {
+ DCHECK_NE(rs.code(), kSPRegInternalCode);
+ return rs.code() << Rs_offset;
+ }
+
+ // These encoding functions allow the stack pointer to be encoded, and
+ // disallow the zero register.
+ static Instr RdSP(Register rd) {
+ DCHECK(!rd.IsZero());
+ return (rd.code() & kRegCodeMask) << Rd_offset;
+ }
+
+ static Instr RnSP(Register rn) {
+ DCHECK(!rn.IsZero());
+ return (rn.code() & kRegCodeMask) << Rn_offset;
+ }
+
+ // Flags encoding.
+ inline static Instr Flags(FlagsUpdate S);
+ inline static Instr Cond(Condition cond);
+
+ // PC-relative address encoding.
+ inline static Instr ImmPCRelAddress(int imm21);
+
+ // Branch encoding.
+ inline static Instr ImmUncondBranch(int imm26);
+ inline static Instr ImmCondBranch(int imm19);
+ inline static Instr ImmCmpBranch(int imm19);
+ inline static Instr ImmTestBranch(int imm14);
+ inline static Instr ImmTestBranchBit(unsigned bit_pos);
+
+ // Data Processing encoding.
+ inline static Instr SF(Register rd);
+ inline static Instr ImmAddSub(int imm);
+ inline static Instr ImmS(unsigned imms, unsigned reg_size);
+ inline static Instr ImmR(unsigned immr, unsigned reg_size);
+ inline static Instr ImmSetBits(unsigned imms, unsigned reg_size);
+ inline static Instr ImmRotate(unsigned immr, unsigned reg_size);
+ inline static Instr ImmLLiteral(int imm19);
+ inline static Instr BitN(unsigned bitn, unsigned reg_size);
+ inline static Instr ShiftDP(Shift shift);
+ inline static Instr ImmDPShift(unsigned amount);
+ inline static Instr ExtendMode(Extend extend);
+ inline static Instr ImmExtendShift(unsigned left_shift);
+ inline static Instr ImmCondCmp(unsigned imm);
+ inline static Instr Nzcv(StatusFlags nzcv);
+
+ static bool IsImmAddSub(int64_t immediate);
+ static bool IsImmLogical(uint64_t value, unsigned width, unsigned* n,
+ unsigned* imm_s, unsigned* imm_r);
+
+ // MemOperand offset encoding.
+ inline static Instr ImmLSUnsigned(int imm12);
+ inline static Instr ImmLS(int imm9);
+ inline static Instr ImmLSPair(int imm7, unsigned size);
+ inline static Instr ImmShiftLS(unsigned shift_amount);
+ inline static Instr ImmException(int imm16);
+ inline static Instr ImmSystemRegister(int imm15);
+ inline static Instr ImmHint(int imm7);
+ inline static Instr ImmBarrierDomain(int imm2);
+ inline static Instr ImmBarrierType(int imm2);
+ inline static unsigned CalcLSDataSize(LoadStoreOp op);
+
+ // Instruction bits for vector format in data processing operations.
+ static Instr VFormat(VRegister vd) {
+ if (vd.Is64Bits()) {
+ switch (vd.LaneCount()) {
+ case 2:
+ return NEON_2S;
+ case 4:
+ return NEON_4H;
+ case 8:
+ return NEON_8B;
+ default:
+ UNREACHABLE();
+ }
+ } else {
+ DCHECK(vd.Is128Bits());
+ switch (vd.LaneCount()) {
+ case 2:
+ return NEON_2D;
+ case 4:
+ return NEON_4S;
+ case 8:
+ return NEON_8H;
+ case 16:
+ return NEON_16B;
+ default:
+ UNREACHABLE();
+ }
+ }
+ }
+
+ // Instruction bits for vector format in floating point data processing
+ // operations.
+ static Instr FPFormat(VRegister vd) {
+ if (vd.LaneCount() == 1) {
+ // Floating point scalar formats.
+ DCHECK(vd.Is32Bits() || vd.Is64Bits());
+ return vd.Is64Bits() ? FP64 : FP32;
+ }
+
+ // Two lane floating point vector formats.
+ if (vd.LaneCount() == 2) {
+ DCHECK(vd.Is64Bits() || vd.Is128Bits());
+ return vd.Is128Bits() ? NEON_FP_2D : NEON_FP_2S;
+ }
+
+ // Four lane floating point vector format.
+ DCHECK((vd.LaneCount() == 4) && vd.Is128Bits());
+ return NEON_FP_4S;
+ }
+
+ // Instruction bits for vector format in load and store operations.
+ static Instr LSVFormat(VRegister vd) {
+ if (vd.Is64Bits()) {
+ switch (vd.LaneCount()) {
+ case 1:
+ return LS_NEON_1D;
+ case 2:
+ return LS_NEON_2S;
+ case 4:
+ return LS_NEON_4H;
+ case 8:
+ return LS_NEON_8B;
+ default:
+ UNREACHABLE();
+ }
+ } else {
+ DCHECK(vd.Is128Bits());
+ switch (vd.LaneCount()) {
+ case 2:
+ return LS_NEON_2D;
+ case 4:
+ return LS_NEON_4S;
+ case 8:
+ return LS_NEON_8H;
+ case 16:
+ return LS_NEON_16B;
+ default:
+ UNREACHABLE();
+ }
+ }
+ }
+
+ // Instruction bits for scalar format in data processing operations.
+ static Instr SFormat(VRegister vd) {
+ DCHECK(vd.IsScalar());
+ switch (vd.SizeInBytes()) {
+ case 1:
+ return NEON_B;
+ case 2:
+ return NEON_H;
+ case 4:
+ return NEON_S;
+ case 8:
+ return NEON_D;
+ default:
+ UNREACHABLE();
+ }
+ }
+
+ static Instr ImmNEONHLM(int index, int num_bits) {
+ int h, l, m;
+ if (num_bits == 3) {
+ DCHECK(is_uint3(index));
+ h = (index >> 2) & 1;
+ l = (index >> 1) & 1;
+ m = (index >> 0) & 1;
+ } else if (num_bits == 2) {
+ DCHECK(is_uint2(index));
+ h = (index >> 1) & 1;
+ l = (index >> 0) & 1;
+ m = 0;
+ } else {
+ DCHECK(is_uint1(index) && (num_bits == 1));
+ h = (index >> 0) & 1;
+ l = 0;
+ m = 0;
+ }
+ return (h << NEONH_offset) | (l << NEONL_offset) | (m << NEONM_offset);
+ }
+
+ static Instr ImmNEONExt(int imm4) {
+ DCHECK(is_uint4(imm4));
+ return imm4 << ImmNEONExt_offset;
+ }
+
+ static Instr ImmNEON5(Instr format, int index) {
+ DCHECK(is_uint4(index));
+ int s = LaneSizeInBytesLog2FromFormat(static_cast<VectorFormat>(format));
+ int imm5 = (index << (s + 1)) | (1 << s);
+ return imm5 << ImmNEON5_offset;
+ }
+
+ static Instr ImmNEON4(Instr format, int index) {
+ DCHECK(is_uint4(index));
+ int s = LaneSizeInBytesLog2FromFormat(static_cast<VectorFormat>(format));
+ int imm4 = index << s;
+ return imm4 << ImmNEON4_offset;
+ }
+
+ static Instr ImmNEONabcdefgh(int imm8) {
+ DCHECK(is_uint8(imm8));
+ Instr instr;
+ instr = ((imm8 >> 5) & 7) << ImmNEONabc_offset;
+ instr |= (imm8 & 0x1f) << ImmNEONdefgh_offset;
+ return instr;
+ }
+
+ static Instr NEONCmode(int cmode) {
+ DCHECK(is_uint4(cmode));
+ return cmode << NEONCmode_offset;
+ }
+
+ static Instr NEONModImmOp(int op) {
+ DCHECK(is_uint1(op));
+ return op << NEONModImmOp_offset;
+ }
+
+ static bool IsImmLSUnscaled(int64_t offset);
+ static bool IsImmLSScaled(int64_t offset, unsigned size);
+ static bool IsImmLLiteral(int64_t offset);
+
+ // Move immediates encoding.
+ inline static Instr ImmMoveWide(int imm);
+ inline static Instr ShiftMoveWide(int shift);
+
+ // FP Immediates.
+ static Instr ImmFP(double imm);
+ static Instr ImmNEONFP(double imm);
+ inline static Instr FPScale(unsigned scale);
+
+ // FP register type.
+ inline static Instr FPType(VRegister fd);
+
+ // Unused on this architecture.
+ void MaybeEmitOutOfLineConstantPool() {}
+
+ void ForceConstantPoolEmissionWithoutJump() {
+ constpool_.Check(Emission::kForced, Jump::kOmitted);
+ }
+ void ForceConstantPoolEmissionWithJump() {
+ constpool_.Check(Emission::kForced, Jump::kRequired);
+ }
+ // Check if the const pool needs to be emitted while pretending that {margin}
+ // more bytes of instructions have already been emitted.
+ void EmitConstPoolWithJumpIfNeeded(size_t margin = 0) {
+ constpool_.Check(Emission::kIfNeeded, Jump::kRequired, margin);
+ }
+
+ // Returns true if we should emit a veneer as soon as possible for a branch
+ // which can at most reach to specified pc.
+ bool ShouldEmitVeneer(int max_reachable_pc,
+ size_t margin = kVeneerDistanceMargin);
+ bool ShouldEmitVeneers(size_t margin = kVeneerDistanceMargin) {
+ return ShouldEmitVeneer(unresolved_branches_first_limit(), margin);
+ }
+
+ // The maximum code size generated for a veneer. Currently one branch
+ // instruction. This is for code size checking purposes, and can be extended
+ // in the future for example if we decide to add nops between the veneers.
+ static constexpr int kMaxVeneerCodeSize = 1 * kInstrSize;
+
+ void RecordVeneerPool(int location_offset, int size);
+ // Emits veneers for branches that are approaching their maximum range.
+ // If need_protection is true, the veneers are protected by a branch jumping
+ // over the code.
+ void EmitVeneers(bool force_emit, bool need_protection,
+ size_t margin = kVeneerDistanceMargin);
+ void EmitVeneersGuard() { EmitPoolGuard(); }
+ // Checks whether veneers need to be emitted at this point.
+ // If force_emit is set, a veneer is generated for *all* unresolved branches.
+ void CheckVeneerPool(bool force_emit, bool require_jump,
+ size_t margin = kVeneerDistanceMargin);
+
+ using BlockConstPoolScope = ConstantPool::BlockScope;
+
+ class BlockPoolsScope {
+ public:
+ // Block veneer and constant pool. Emits pools if necessary to ensure that
+ // {margin} more bytes can be emitted without triggering pool emission.
+ explicit BlockPoolsScope(Assembler* assem, size_t margin = 0)
+ : assem_(assem), block_const_pool_(assem, margin) {
+ assem_->CheckVeneerPool(false, true, margin);
+ assem_->StartBlockVeneerPool();
+ }
+
+ BlockPoolsScope(Assembler* assem, PoolEmissionCheck check)
+ : assem_(assem), block_const_pool_(assem, check) {
+ assem_->StartBlockVeneerPool();
+ }
+ ~BlockPoolsScope() { assem_->EndBlockVeneerPool(); }
+
+ private:
+ Assembler* assem_;
+ BlockConstPoolScope block_const_pool_;
+ DISALLOW_IMPLICIT_CONSTRUCTORS(BlockPoolsScope);
+ };
+
+#if defined(V8_OS_WIN)
+ win64_unwindinfo::XdataEncoder* GetXdataEncoder() {
+ return xdata_encoder_.get();
+ }
+
+ win64_unwindinfo::BuiltinUnwindInfo GetUnwindInfo() const;
+#endif
+
+ protected:
+ inline const Register& AppropriateZeroRegFor(const CPURegister& reg) const;
+
+ void LoadStore(const CPURegister& rt, const MemOperand& addr, LoadStoreOp op);
+ void LoadStorePair(const CPURegister& rt, const CPURegister& rt2,
+ const MemOperand& addr, LoadStorePairOp op);
+ void LoadStoreStruct(const VRegister& vt, const MemOperand& addr,
+ NEONLoadStoreMultiStructOp op);
+ void LoadStoreStruct1(const VRegister& vt, int reg_count,
+ const MemOperand& addr);
+ void LoadStoreStructSingle(const VRegister& vt, uint32_t lane,
+ const MemOperand& addr,
+ NEONLoadStoreSingleStructOp op);
+ void LoadStoreStructSingleAllLanes(const VRegister& vt,
+ const MemOperand& addr,
+ NEONLoadStoreSingleStructOp op);
+ void LoadStoreStructVerify(const VRegister& vt, const MemOperand& addr,
+ Instr op);
+
+ static bool IsImmLSPair(int64_t offset, unsigned size);
+
+ void Logical(const Register& rd, const Register& rn, const Operand& operand,
+ LogicalOp op);
+ void LogicalImmediate(const Register& rd, const Register& rn, unsigned n,
+ unsigned imm_s, unsigned imm_r, LogicalOp op);
+
+ void ConditionalCompare(const Register& rn, const Operand& operand,
+ StatusFlags nzcv, Condition cond,
+ ConditionalCompareOp op);
+ static bool IsImmConditionalCompare(int64_t immediate);
+
+ void AddSubWithCarry(const Register& rd, const Register& rn,
+ const Operand& operand, FlagsUpdate S,
+ AddSubWithCarryOp op);
+
+ // Functions for emulating operands not directly supported by the instruction
+ // set.
+ void EmitShift(const Register& rd, const Register& rn, Shift shift,
+ unsigned amount);
+ void EmitExtendShift(const Register& rd, const Register& rn, Extend extend,
+ unsigned left_shift);
+
+ void AddSub(const Register& rd, const Register& rn, const Operand& operand,
+ FlagsUpdate S, AddSubOp op);
+
+ static bool IsImmFP32(float imm);
+ static bool IsImmFP64(double imm);
+
+ // Find an appropriate LoadStoreOp or LoadStorePairOp for the specified
+ // registers. Only simple loads are supported; sign- and zero-extension (such
+ // as in LDPSW_x or LDRB_w) are not supported.
+ static inline LoadStoreOp LoadOpFor(const CPURegister& rt);
+ static inline LoadStorePairOp LoadPairOpFor(const CPURegister& rt,
+ const CPURegister& rt2);
+ static inline LoadStoreOp StoreOpFor(const CPURegister& rt);
+ static inline LoadStorePairOp StorePairOpFor(const CPURegister& rt,
+ const CPURegister& rt2);
+ static inline LoadLiteralOp LoadLiteralOpFor(const CPURegister& rt);
+
+ // Remove the specified branch from the unbound label link chain.
+ // If available, a veneer for this label can be used for other branches in the
+ // chain if the link chain cannot be fixed up without this branch.
+ void RemoveBranchFromLabelLinkChain(Instruction* branch, Label* label,
+ Instruction* label_veneer = nullptr);
+
+ private:
+ static uint32_t FPToImm8(double imm);
+
+ // Instruction helpers.
+ void MoveWide(const Register& rd, uint64_t imm, int shift,
+ MoveWideImmediateOp mov_op);
+ void DataProcShiftedRegister(const Register& rd, const Register& rn,
+ const Operand& operand, FlagsUpdate S, Instr op);
+ void DataProcExtendedRegister(const Register& rd, const Register& rn,
+ const Operand& operand, FlagsUpdate S,
+ Instr op);
+ void ConditionalSelect(const Register& rd, const Register& rn,
+ const Register& rm, Condition cond,
+ ConditionalSelectOp op);
+ void DataProcessing1Source(const Register& rd, const Register& rn,
+ DataProcessing1SourceOp op);
+ void DataProcessing3Source(const Register& rd, const Register& rn,
+ const Register& rm, const Register& ra,
+ DataProcessing3SourceOp op);
+ void FPDataProcessing1Source(const VRegister& fd, const VRegister& fn,
+ FPDataProcessing1SourceOp op);
+ void FPDataProcessing2Source(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm,
+ FPDataProcessing2SourceOp op);
+ void FPDataProcessing3Source(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm, const VRegister& fa,
+ FPDataProcessing3SourceOp op);
+ void NEONAcrossLanesL(const VRegister& vd, const VRegister& vn,
+ NEONAcrossLanesOp op);
+ void NEONAcrossLanes(const VRegister& vd, const VRegister& vn,
+ NEONAcrossLanesOp op);
+ void NEONModifiedImmShiftLsl(const VRegister& vd, const int imm8,
+ const int left_shift,
+ NEONModifiedImmediateOp op);
+ void NEONModifiedImmShiftMsl(const VRegister& vd, const int imm8,
+ const int shift_amount,
+ NEONModifiedImmediateOp op);
+ void NEON3Same(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ NEON3SameOp vop);
+ void NEONFP3Same(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm, Instr op);
+ void NEON3DifferentL(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm, NEON3DifferentOp vop);
+ void NEON3DifferentW(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm, NEON3DifferentOp vop);
+ void NEON3DifferentHN(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm, NEON3DifferentOp vop);
+ void NEONFP2RegMisc(const VRegister& vd, const VRegister& vn,
+ NEON2RegMiscOp vop, double value = 0.0);
+ void NEON2RegMisc(const VRegister& vd, const VRegister& vn,
+ NEON2RegMiscOp vop, int value = 0);
+ void NEONFP2RegMisc(const VRegister& vd, const VRegister& vn, Instr op);
+ void NEONAddlp(const VRegister& vd, const VRegister& vn, NEON2RegMiscOp op);
+ void NEONPerm(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ NEONPermOp op);
+ void NEONFPByElement(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm, int vm_index,
+ NEONByIndexedElementOp op);
+ void NEONByElement(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm, int vm_index,
+ NEONByIndexedElementOp op);
+ void NEONByElementL(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm, int vm_index,
+ NEONByIndexedElementOp op);
+ void NEONShiftImmediate(const VRegister& vd, const VRegister& vn,
+ NEONShiftImmediateOp op, int immh_immb);
+ void NEONShiftLeftImmediate(const VRegister& vd, const VRegister& vn,
+ int shift, NEONShiftImmediateOp op);
+ void NEONShiftRightImmediate(const VRegister& vd, const VRegister& vn,
+ int shift, NEONShiftImmediateOp op);
+ void NEONShiftImmediateL(const VRegister& vd, const VRegister& vn, int shift,
+ NEONShiftImmediateOp op);
+ void NEONShiftImmediateN(const VRegister& vd, const VRegister& vn, int shift,
+ NEONShiftImmediateOp op);
+ void NEONXtn(const VRegister& vd, const VRegister& vn, NEON2RegMiscOp vop);
+ void NEONTable(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ NEONTableOp op);
+
+ Instr LoadStoreStructAddrModeField(const MemOperand& addr);
+
+ // Label helpers.
+
+ // Return an offset for a label-referencing instruction, typically a branch.
+ int LinkAndGetByteOffsetTo(Label* label);
+
+ // This is the same as LinkAndGetByteOffsetTo, but return an offset
+ // suitable for fields that take instruction offsets.
+ inline int LinkAndGetInstructionOffsetTo(Label* label);
+
+ static constexpr int kStartOfLabelLinkChain = 0;
+
+ // Verify that a label's link chain is intact.
+ void CheckLabelLinkChain(Label const* label);
+
+ // Emit the instruction at pc_.
+ void Emit(Instr instruction) {
+ STATIC_ASSERT(sizeof(*pc_) == 1);
+ STATIC_ASSERT(sizeof(instruction) == kInstrSize);
+ DCHECK_LE(pc_ + sizeof(instruction), buffer_start_ + buffer_->size());
+
+ memcpy(pc_, &instruction, sizeof(instruction));
+ pc_ += sizeof(instruction);
+ CheckBuffer();
+ }
+
+ // Emit data inline in the instruction stream.
+ void EmitData(void const* data, unsigned size) {
+ DCHECK_EQ(sizeof(*pc_), 1);
+ DCHECK_LE(pc_ + size, buffer_start_ + buffer_->size());
+
+ // TODO(all): Somehow register we have some data here. Then we can
+ // disassemble it correctly.
+ memcpy(pc_, data, size);
+ pc_ += size;
+ CheckBuffer();
+ }
+
+ void GrowBuffer();
+ void CheckBufferSpace();
+ void CheckBuffer();
+
+ // Emission of the veneer pools may be blocked in some code sequences.
+ int veneer_pool_blocked_nesting_; // Block emission if this is not zero.
+
+ // Relocation info generation
+ // Each relocation is encoded as a variable size value
+ static constexpr int kMaxRelocSize = RelocInfoWriter::kMaxSize;
+ RelocInfoWriter reloc_info_writer;
+
+ // Internal reference positions, required for (potential) patching in
+ // GrowBuffer(); contains only those internal references whose labels
+ // are already bound.
+ std::deque<int> internal_reference_positions_;
+
+ protected:
+ // Code generation
+ // The relocation writer's position is at least kGap bytes below the end of
+ // the generated instructions. This is so that multi-instruction sequences do
+ // not have to check for overflow. The same is true for writes of large
+ // relocation info entries, and debug strings encoded in the instruction
+ // stream.
+ static constexpr int kGap = 64;
+ STATIC_ASSERT(AssemblerBase::kMinimalBufferSize >= 2 * kGap);
+
+ public:
+#ifdef DEBUG
+ // Functions used for testing.
+ size_t GetConstantPoolEntriesSizeForTesting() const {
+ // Do not include branch over the pool.
+ return constpool_.Entry32Count() * kInt32Size +
+ constpool_.Entry64Count() * kInt64Size;
+ }
+
+ static size_t GetCheckConstPoolIntervalForTesting() {
+ return ConstantPool::kCheckInterval;
+ }
+
+ static size_t GetApproxMaxDistToConstPoolForTesting() {
+ return ConstantPool::kApproxDistToPool64;
+ }
+#endif
+
+ class FarBranchInfo {
+ public:
+ FarBranchInfo(int offset, Label* label)
+ : pc_offset_(offset), label_(label) {}
+ // Offset of the branch in the code generation buffer.
+ int pc_offset_;
+ // The label branched to.
+ Label* label_;
+ };
+
+ protected:
+ // Information about unresolved (forward) branches.
+ // The Assembler is only allowed to delete out-of-date information from here
+ // after a label is bound. The MacroAssembler uses this information to
+ // generate veneers.
+ //
+ // The second member gives information about the unresolved branch. The first
+ // member of the pair is the maximum offset that the branch can reach in the
+ // buffer. The map is sorted according to this reachable offset, allowing to
+ // easily check when veneers need to be emitted.
+ // Note that the maximum reachable offset (first member of the pairs) should
+ // always be positive but has the same type as the return value for
+ // pc_offset() for convenience.
+ std::multimap<int, FarBranchInfo> unresolved_branches_;
+
+ // We generate a veneer for a branch if we reach within this distance of the
+ // limit of the range.
+ static constexpr int kVeneerDistanceMargin = 1 * KB;
+ // The factor of 2 is a finger in the air guess. With a default margin of
+ // 1KB, that leaves us an addional 256 instructions to avoid generating a
+ // protective branch.
+ static constexpr int kVeneerNoProtectionFactor = 2;
+ static constexpr int kVeneerDistanceCheckMargin =
+ kVeneerNoProtectionFactor * kVeneerDistanceMargin;
+ int unresolved_branches_first_limit() const {
+ DCHECK(!unresolved_branches_.empty());
+ return unresolved_branches_.begin()->first;
+ }
+ // This PC-offset of the next veneer pool check helps reduce the overhead
+ // of checking for veneer pools.
+ // It is maintained to the closest unresolved branch limit minus the maximum
+ // veneer margin (or kMaxInt if there are no unresolved branches).
+ int next_veneer_pool_check_;
+
+#if defined(V8_OS_WIN)
+ std::unique_ptr<win64_unwindinfo::XdataEncoder> xdata_encoder_;
+#endif
+
+ private:
+ // Avoid overflows for displacements etc.
+ static const int kMaximalBufferSize = 512 * MB;
+
+ // If a veneer is emitted for a branch instruction, that instruction must be
+ // removed from the associated label's link chain so that the assembler does
+ // not later attempt (likely unsuccessfully) to patch it to branch directly to
+ // the label.
+ void DeleteUnresolvedBranchInfoForLabel(Label* label);
+ // This function deletes the information related to the label by traversing
+ // the label chain, and for each PC-relative instruction in the chain checking
+ // if pending unresolved information exists. Its complexity is proportional to
+ // the length of the label chain.
+ void DeleteUnresolvedBranchInfoForLabelTraverse(Label* label);
+
+ void AllocateAndInstallRequestedHeapObjects(Isolate* isolate);
+
+ int WriteCodeComments();
+
+ // The pending constant pool.
+ ConstantPool constpool_;
+
+ friend class EnsureSpace;
+ friend class ConstantPool;
+};
+
+class PatchingAssembler : public Assembler {
+ public:
+ // Create an Assembler with a buffer starting at 'start'.
+ // The buffer size is
+ // size of instructions to patch + kGap
+ // Where kGap is the distance from which the Assembler tries to grow the
+ // buffer.
+ // If more or fewer instructions than expected are generated or if some
+ // relocation information takes space in the buffer, the PatchingAssembler
+ // will crash trying to grow the buffer.
+ // Note that the instruction cache will not be flushed.
+ PatchingAssembler(const AssemblerOptions& options, byte* start,
+ unsigned count)
+ : Assembler(options,
+ ExternalAssemblerBuffer(start, count * kInstrSize + kGap)),
+ block_constant_pool_emission_scope(this) {}
+
+ ~PatchingAssembler() {
+ // Verify we have generated the number of instruction we expected.
+ DCHECK_EQ(pc_offset() + kGap, buffer_->size());
+ }
+
+ // See definition of PatchAdrFar() for details.
+ static constexpr int kAdrFarPatchableNNops = 2;
+ static constexpr int kAdrFarPatchableNInstrs = kAdrFarPatchableNNops + 2;
+ void PatchAdrFar(int64_t target_offset);
+ void PatchSubSp(uint32_t immediate);
+
+ private:
+ BlockPoolsScope block_constant_pool_emission_scope;
+};
+
+class EnsureSpace {
+ public:
+ explicit EnsureSpace(Assembler* assembler) : block_pools_scope_(assembler) {
+ assembler->CheckBufferSpace();
+ }
+
+ private:
+ Assembler::BlockPoolsScope block_pools_scope_;
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_ARM64_ASSEMBLER_ARM64_H_
diff --git a/src/codegen/arm64/constants-arm64.h b/src/codegen/arm64/constants-arm64.h
new file mode 100644
index 0000000..52790b9
--- /dev/null
+++ b/src/codegen/arm64/constants-arm64.h
@@ -0,0 +1,2125 @@
+// Copyright 2013 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_ARM64_CONSTANTS_ARM64_H_
+#define V8_CODEGEN_ARM64_CONSTANTS_ARM64_H_
+
+#include "src/base/macros.h"
+#include "src/common/globals.h"
+
+// Assert that this is an LP64 system, or LLP64 on Windows.
+STATIC_ASSERT(sizeof(int) == sizeof(int32_t));
+#if defined(V8_OS_WIN)
+STATIC_ASSERT(sizeof(1L) == sizeof(int32_t));
+#else
+STATIC_ASSERT(sizeof(long) == sizeof(int64_t)); // NOLINT(runtime/int)
+STATIC_ASSERT(sizeof(1L) == sizeof(int64_t));
+#endif
+STATIC_ASSERT(sizeof(void*) == sizeof(int64_t));
+STATIC_ASSERT(sizeof(1) == sizeof(int32_t));
+
+// Get the standard printf format macros for C99 stdint types.
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+#include <inttypes.h>
+
+namespace v8 {
+namespace internal {
+
+constexpr size_t kMaxPCRelativeCodeRangeInMB = 128;
+
+constexpr uint8_t kInstrSize = 4;
+constexpr uint8_t kInstrSizeLog2 = 2;
+constexpr uint8_t kLoadLiteralScaleLog2 = 2;
+constexpr uint8_t kLoadLiteralScale = 1 << kLoadLiteralScaleLog2;
+constexpr int kMaxLoadLiteralRange = 1 * MB;
+
+const int kNumberOfRegisters = 32;
+const int kNumberOfVRegisters = 32;
+// Callee saved registers are x19-x28.
+const int kNumberOfCalleeSavedRegisters = 10;
+// Callee saved FP registers are d8-d15.
+const int kNumberOfCalleeSavedVRegisters = 8;
+const int kWRegSizeInBits = 32;
+const int kWRegSizeInBitsLog2 = 5;
+const int kWRegSize = kWRegSizeInBits >> 3;
+const int kWRegSizeLog2 = kWRegSizeInBitsLog2 - 3;
+const int kXRegSizeInBits = 64;
+const int kXRegSizeInBitsLog2 = 6;
+const int kXRegSize = kXRegSizeInBits >> 3;
+const int kXRegSizeLog2 = kXRegSizeInBitsLog2 - 3;
+const int kSRegSizeInBits = 32;
+const int kSRegSizeInBitsLog2 = 5;
+const int kSRegSize = kSRegSizeInBits >> 3;
+const int kSRegSizeLog2 = kSRegSizeInBitsLog2 - 3;
+const int kDRegSizeInBits = 64;
+const int kDRegSizeInBitsLog2 = 6;
+const int kDRegSize = kDRegSizeInBits >> 3;
+const int kDRegSizeLog2 = kDRegSizeInBitsLog2 - 3;
+const int kDRegSizeInBytesLog2 = kDRegSizeInBitsLog2 - 3;
+const int kBRegSizeInBits = 8;
+const int kBRegSize = kBRegSizeInBits >> 3;
+const int kHRegSizeInBits = 16;
+const int kHRegSize = kHRegSizeInBits >> 3;
+const int kQRegSizeInBits = 128;
+const int kQRegSizeInBitsLog2 = 7;
+const int kQRegSize = kQRegSizeInBits >> 3;
+const int kQRegSizeLog2 = kQRegSizeInBitsLog2 - 3;
+const int kVRegSizeInBits = kQRegSizeInBits;
+const int kVRegSize = kVRegSizeInBits >> 3;
+const int64_t kWRegMask = 0x00000000ffffffffL;
+const int64_t kXRegMask = 0xffffffffffffffffL;
+const int64_t kSRegMask = 0x00000000ffffffffL;
+const int64_t kDRegMask = 0xffffffffffffffffL;
+// TODO(all) check if the expression below works on all compilers or if it
+// triggers an overflow error.
+const int64_t kDSignBit = 63;
+const int64_t kDSignMask = 0x1LL << kDSignBit;
+const int64_t kSSignBit = 31;
+const int64_t kSSignMask = 0x1LL << kSSignBit;
+const int64_t kXSignBit = 63;
+const int64_t kXSignMask = 0x1LL << kXSignBit;
+const int64_t kWSignBit = 31;
+const int64_t kWSignMask = 0x1LL << kWSignBit;
+const int64_t kDQuietNanBit = 51;
+const int64_t kDQuietNanMask = 0x1LL << kDQuietNanBit;
+const int64_t kSQuietNanBit = 22;
+const int64_t kSQuietNanMask = 0x1LL << kSQuietNanBit;
+const int64_t kByteMask = 0xffL;
+const int64_t kHalfWordMask = 0xffffL;
+const int64_t kWordMask = 0xffffffffL;
+const uint64_t kXMaxUInt = 0xffffffffffffffffUL;
+const uint64_t kWMaxUInt = 0xffffffffUL;
+const int64_t kXMaxInt = 0x7fffffffffffffffL;
+const int64_t kXMinInt = 0x8000000000000000L;
+const int32_t kWMaxInt = 0x7fffffff;
+const int32_t kWMinInt = 0x80000000;
+const int kIp0Code = 16;
+const int kIp1Code = 17;
+const int kFramePointerRegCode = 29;
+const int kLinkRegCode = 30;
+const int kZeroRegCode = 31;
+const int kSPRegInternalCode = 63;
+const unsigned kRegCodeMask = 0x1f;
+const unsigned kShiftAmountWRegMask = 0x1f;
+const unsigned kShiftAmountXRegMask = 0x3f;
+// Standard machine types defined by AAPCS64.
+const unsigned kHalfWordSize = 16;
+const unsigned kHalfWordSizeLog2 = 4;
+const unsigned kHalfWordSizeInBytes = kHalfWordSize >> 3;
+const unsigned kHalfWordSizeInBytesLog2 = kHalfWordSizeLog2 - 3;
+const unsigned kWordSize = 32;
+const unsigned kWordSizeLog2 = 5;
+const unsigned kWordSizeInBytes = kWordSize >> 3;
+const unsigned kWordSizeInBytesLog2 = kWordSizeLog2 - 3;
+const unsigned kDoubleWordSize = 64;
+const unsigned kDoubleWordSizeInBytes = kDoubleWordSize >> 3;
+const unsigned kQuadWordSize = 128;
+const unsigned kQuadWordSizeInBytes = kQuadWordSize >> 3;
+const int kMaxLanesPerVector = 16;
+
+const unsigned kAddressTagOffset = 56;
+const unsigned kAddressTagWidth = 8;
+const uint64_t kAddressTagMask = ((UINT64_C(1) << kAddressTagWidth) - 1)
+ << kAddressTagOffset;
+static_assert(kAddressTagMask == UINT64_C(0xff00000000000000),
+ "AddressTagMask must represent most-significant eight bits.");
+
+const uint64_t kTTBRMask = UINT64_C(1) << 55;
+
+// AArch64 floating-point specifics. These match IEEE-754.
+const unsigned kDoubleMantissaBits = 52;
+const unsigned kDoubleExponentBits = 11;
+const unsigned kDoubleExponentBias = 1023;
+const unsigned kFloatMantissaBits = 23;
+const unsigned kFloatExponentBits = 8;
+const unsigned kFloatExponentBias = 127;
+const unsigned kFloat16MantissaBits = 10;
+const unsigned kFloat16ExponentBits = 5;
+const unsigned kFloat16ExponentBias = 15;
+
+// Actual value of root register is offset from the root array's start
+// to take advantage of negative displacement values.
+// TODO(sigurds): Choose best value.
+// TODO(ishell): Choose best value for ptr-compr.
+constexpr int kRootRegisterBias = kSystemPointerSize == kTaggedSize ? 256 : 0;
+
+using float16 = uint16_t;
+
+#define INSTRUCTION_FIELDS_LIST(V_) \
+ /* Register fields */ \
+ V_(Rd, 4, 0, Bits) /* Destination register. */ \
+ V_(Rn, 9, 5, Bits) /* First source register. */ \
+ V_(Rm, 20, 16, Bits) /* Second source register. */ \
+ V_(Ra, 14, 10, Bits) /* Third source register. */ \
+ V_(Rt, 4, 0, Bits) /* Load dest / store source. */ \
+ V_(Rt2, 14, 10, Bits) /* Load second dest / */ \
+ /* store second source. */ \
+ V_(Rs, 20, 16, Bits) /* Store-exclusive status */ \
+ V_(PrefetchMode, 4, 0, Bits) \
+ \
+ /* Common bits */ \
+ V_(SixtyFourBits, 31, 31, Bits) \
+ V_(FlagsUpdate, 29, 29, Bits) \
+ \
+ /* PC relative addressing */ \
+ V_(ImmPCRelHi, 23, 5, SignedBits) \
+ V_(ImmPCRelLo, 30, 29, Bits) \
+ \
+ /* Add/subtract/logical shift register */ \
+ V_(ShiftDP, 23, 22, Bits) \
+ V_(ImmDPShift, 15, 10, Bits) \
+ \
+ /* Add/subtract immediate */ \
+ V_(ImmAddSub, 21, 10, Bits) \
+ V_(ShiftAddSub, 23, 22, Bits) \
+ \
+ /* Add/subtract extend */ \
+ V_(ImmExtendShift, 12, 10, Bits) \
+ V_(ExtendMode, 15, 13, Bits) \
+ \
+ /* Move wide */ \
+ V_(ImmMoveWide, 20, 5, Bits) \
+ V_(ShiftMoveWide, 22, 21, Bits) \
+ \
+ /* Logical immediate, bitfield and extract */ \
+ V_(BitN, 22, 22, Bits) \
+ V_(ImmRotate, 21, 16, Bits) \
+ V_(ImmSetBits, 15, 10, Bits) \
+ V_(ImmR, 21, 16, Bits) \
+ V_(ImmS, 15, 10, Bits) \
+ \
+ /* Test and branch immediate */ \
+ V_(ImmTestBranch, 18, 5, SignedBits) \
+ V_(ImmTestBranchBit40, 23, 19, Bits) \
+ V_(ImmTestBranchBit5, 31, 31, Bits) \
+ \
+ /* Conditionals */ \
+ V_(Condition, 15, 12, Bits) \
+ V_(ConditionBranch, 3, 0, Bits) \
+ V_(Nzcv, 3, 0, Bits) \
+ V_(ImmCondCmp, 20, 16, Bits) \
+ V_(ImmCondBranch, 23, 5, SignedBits) \
+ \
+ /* Floating point */ \
+ V_(FPType, 23, 22, Bits) \
+ V_(ImmFP, 20, 13, Bits) \
+ V_(FPScale, 15, 10, Bits) \
+ \
+ /* Load Store */ \
+ V_(ImmLS, 20, 12, SignedBits) \
+ V_(ImmLSUnsigned, 21, 10, Bits) \
+ V_(ImmLSPair, 21, 15, SignedBits) \
+ V_(ImmShiftLS, 12, 12, Bits) \
+ V_(LSOpc, 23, 22, Bits) \
+ V_(LSVector, 26, 26, Bits) \
+ V_(LSSize, 31, 30, Bits) \
+ \
+ /* NEON generic fields */ \
+ V_(NEONQ, 30, 30, Bits) \
+ V_(NEONSize, 23, 22, Bits) \
+ V_(NEONLSSize, 11, 10, Bits) \
+ V_(NEONS, 12, 12, Bits) \
+ V_(NEONL, 21, 21, Bits) \
+ V_(NEONM, 20, 20, Bits) \
+ V_(NEONH, 11, 11, Bits) \
+ V_(ImmNEONExt, 14, 11, Bits) \
+ V_(ImmNEON5, 20, 16, Bits) \
+ V_(ImmNEON4, 14, 11, Bits) \
+ \
+ /* Other immediates */ \
+ V_(ImmUncondBranch, 25, 0, SignedBits) \
+ V_(ImmCmpBranch, 23, 5, SignedBits) \
+ V_(ImmLLiteral, 23, 5, SignedBits) \
+ V_(ImmException, 20, 5, Bits) \
+ V_(ImmHint, 11, 5, Bits) \
+ V_(ImmBarrierDomain, 11, 10, Bits) \
+ V_(ImmBarrierType, 9, 8, Bits) \
+ \
+ /* System (MRS, MSR) */ \
+ V_(ImmSystemRegister, 19, 5, Bits) \
+ V_(SysO0, 19, 19, Bits) \
+ V_(SysOp1, 18, 16, Bits) \
+ V_(SysOp2, 7, 5, Bits) \
+ V_(CRn, 15, 12, Bits) \
+ V_(CRm, 11, 8, Bits) \
+ \
+ /* Load-/store-exclusive */ \
+ V_(LoadStoreXLoad, 22, 22, Bits) \
+ V_(LoadStoreXNotExclusive, 23, 23, Bits) \
+ V_(LoadStoreXAcquireRelease, 15, 15, Bits) \
+ V_(LoadStoreXSizeLog2, 31, 30, Bits) \
+ V_(LoadStoreXPair, 21, 21, Bits) \
+ \
+ /* NEON load/store */ \
+ V_(NEONLoad, 22, 22, Bits) \
+ \
+ /* NEON Modified Immediate fields */ \
+ V_(ImmNEONabc, 18, 16, Bits) \
+ V_(ImmNEONdefgh, 9, 5, Bits) \
+ V_(NEONModImmOp, 29, 29, Bits) \
+ V_(NEONCmode, 15, 12, Bits) \
+ \
+ /* NEON Shift Immediate fields */ \
+ V_(ImmNEONImmhImmb, 22, 16, Bits) \
+ V_(ImmNEONImmh, 22, 19, Bits) \
+ V_(ImmNEONImmb, 18, 16, Bits)
+
+#define SYSTEM_REGISTER_FIELDS_LIST(V_, M_) \
+ /* NZCV */ \
+ V_(Flags, 31, 28, Bits, uint32_t) \
+ V_(N, 31, 31, Bits, bool) \
+ V_(Z, 30, 30, Bits, bool) \
+ V_(C, 29, 29, Bits, bool) \
+ V_(V, 28, 28, Bits, bool) \
+ M_(NZCV, Flags_mask) \
+ \
+ /* FPCR */ \
+ V_(AHP, 26, 26, Bits, bool) \
+ V_(DN, 25, 25, Bits, bool) \
+ V_(FZ, 24, 24, Bits, bool) \
+ V_(RMode, 23, 22, Bits, FPRounding) \
+ M_(FPCR, AHP_mask | DN_mask | FZ_mask | RMode_mask)
+
+// Fields offsets.
+#define DECLARE_FIELDS_OFFSETS(Name, HighBit, LowBit, unused_1, unused_2) \
+ const int Name##_offset = LowBit; \
+ const int Name##_width = HighBit - LowBit + 1; \
+ const uint32_t Name##_mask = ((1 << Name##_width) - 1) << LowBit;
+#define DECLARE_INSTRUCTION_FIELDS_OFFSETS(Name, HighBit, LowBit, unused_1) \
+ DECLARE_FIELDS_OFFSETS(Name, HighBit, LowBit, unused_1, unused_2)
+INSTRUCTION_FIELDS_LIST(DECLARE_INSTRUCTION_FIELDS_OFFSETS)
+SYSTEM_REGISTER_FIELDS_LIST(DECLARE_FIELDS_OFFSETS, NOTHING)
+#undef DECLARE_FIELDS_OFFSETS
+#undef DECLARE_INSTRUCTION_FIELDS_OFFSETS
+
+// ImmPCRel is a compound field (not present in INSTRUCTION_FIELDS_LIST), formed
+// from ImmPCRelLo and ImmPCRelHi.
+const int ImmPCRel_mask = ImmPCRelLo_mask | ImmPCRelHi_mask;
+
+// Condition codes.
+enum Condition {
+ eq = 0,
+ ne = 1,
+ hs = 2,
+ cs = hs,
+ lo = 3,
+ cc = lo,
+ mi = 4,
+ pl = 5,
+ vs = 6,
+ vc = 7,
+ hi = 8,
+ ls = 9,
+ ge = 10,
+ lt = 11,
+ gt = 12,
+ le = 13,
+ al = 14,
+ nv = 15 // Behaves as always/al.
+};
+
+inline Condition NegateCondition(Condition cond) {
+ // Conditions al and nv behave identically, as "always true". They can't be
+ // inverted, because there is no never condition.
+ DCHECK((cond != al) && (cond != nv));
+ return static_cast<Condition>(cond ^ 1);
+}
+
+enum FlagsUpdate { SetFlags = 1, LeaveFlags = 0 };
+
+enum StatusFlags {
+ NoFlag = 0,
+
+ // Derive the flag combinations from the system register bit descriptions.
+ NFlag = N_mask,
+ ZFlag = Z_mask,
+ CFlag = C_mask,
+ VFlag = V_mask,
+ NZFlag = NFlag | ZFlag,
+ NCFlag = NFlag | CFlag,
+ NVFlag = NFlag | VFlag,
+ ZCFlag = ZFlag | CFlag,
+ ZVFlag = ZFlag | VFlag,
+ CVFlag = CFlag | VFlag,
+ NZCFlag = NFlag | ZFlag | CFlag,
+ NZVFlag = NFlag | ZFlag | VFlag,
+ NCVFlag = NFlag | CFlag | VFlag,
+ ZCVFlag = ZFlag | CFlag | VFlag,
+ NZCVFlag = NFlag | ZFlag | CFlag | VFlag,
+
+ // Floating-point comparison results.
+ FPEqualFlag = ZCFlag,
+ FPLessThanFlag = NFlag,
+ FPGreaterThanFlag = CFlag,
+ FPUnorderedFlag = CVFlag
+};
+
+enum Shift {
+ NO_SHIFT = -1,
+ LSL = 0x0,
+ LSR = 0x1,
+ ASR = 0x2,
+ ROR = 0x3,
+ MSL = 0x4
+};
+
+enum Extend {
+ NO_EXTEND = -1,
+ UXTB = 0,
+ UXTH = 1,
+ UXTW = 2,
+ UXTX = 3,
+ SXTB = 4,
+ SXTH = 5,
+ SXTW = 6,
+ SXTX = 7
+};
+
+enum SystemHint {
+ NOP = 0,
+ YIELD = 1,
+ WFE = 2,
+ WFI = 3,
+ SEV = 4,
+ SEVL = 5,
+ CSDB = 20,
+ BTI = 32,
+ BTI_c = 34,
+ BTI_j = 36,
+ BTI_jc = 38
+};
+
+// In a guarded page, only BTI and PACI[AB]SP instructions are allowed to be
+// the target of indirect branches. Details on which kinds of branches each
+// instruction allows follow in the comments below:
+enum class BranchTargetIdentifier {
+ // Do not emit a BTI instruction.
+ kNone,
+
+ // Emit a BTI instruction. Cannot be the target of indirect jumps/calls.
+ kBti,
+
+ // Emit a "BTI c" instruction. Can be the target of indirect jumps (BR) with
+ // x16/x17 as the target register, or indirect calls (BLR).
+ kBtiCall,
+
+ // Emit a "BTI j" instruction. Can be the target of indirect jumps (BR).
+ kBtiJump,
+
+ // Emit a "BTI jc" instruction, which is a combination of "BTI j" and "BTI c".
+ kBtiJumpCall,
+
+ // Emit a PACIBSP instruction, which acts like a "BTI c" or a "BTI jc",
+ // based on the value of SCTLR_EL1.BT0.
+ kPacibsp
+};
+
+enum BarrierDomain {
+ OuterShareable = 0,
+ NonShareable = 1,
+ InnerShareable = 2,
+ FullSystem = 3
+};
+
+enum BarrierType {
+ BarrierOther = 0,
+ BarrierReads = 1,
+ BarrierWrites = 2,
+ BarrierAll = 3
+};
+
+// System/special register names.
+// This information is not encoded as one field but as the concatenation of
+// multiple fields (Op0<0>, Op1, Crn, Crm, Op2).
+enum SystemRegister {
+ NZCV = ((0x1 << SysO0_offset) | (0x3 << SysOp1_offset) | (0x4 << CRn_offset) |
+ (0x2 << CRm_offset) | (0x0 << SysOp2_offset)) >>
+ ImmSystemRegister_offset,
+ FPCR = ((0x1 << SysO0_offset) | (0x3 << SysOp1_offset) | (0x4 << CRn_offset) |
+ (0x4 << CRm_offset) | (0x0 << SysOp2_offset)) >>
+ ImmSystemRegister_offset
+};
+
+// Instruction enumerations.
+//
+// These are the masks that define a class of instructions, and the list of
+// instructions within each class. Each enumeration has a Fixed, FMask and
+// Mask value.
+//
+// Fixed: The fixed bits in this instruction class.
+// FMask: The mask used to extract the fixed bits in the class.
+// Mask: The mask used to identify the instructions within a class.
+//
+// The enumerations can be used like this:
+//
+// DCHECK(instr->Mask(PCRelAddressingFMask) == PCRelAddressingFixed);
+// switch(instr->Mask(PCRelAddressingMask)) {
+// case ADR: Format("adr 'Xd, 'AddrPCRelByte"); break;
+// case ADRP: Format("adrp 'Xd, 'AddrPCRelPage"); break;
+// default: printf("Unknown instruction\n");
+// }
+
+// Used to corrupt encodings by setting all bits when orred. Although currently
+// unallocated in AArch64, this encoding is not guaranteed to be undefined
+// indefinitely.
+const uint32_t kUnallocatedInstruction = 0xffffffff;
+
+// Generic fields.
+enum GenericInstrField : uint32_t {
+ SixtyFourBits = 0x80000000,
+ ThirtyTwoBits = 0x00000000,
+ FP32 = 0x00000000,
+ FP64 = 0x00400000
+};
+
+enum NEONFormatField : uint32_t {
+ NEONFormatFieldMask = 0x40C00000,
+ NEON_Q = 0x40000000,
+ NEON_8B = 0x00000000,
+ NEON_16B = NEON_8B | NEON_Q,
+ NEON_4H = 0x00400000,
+ NEON_8H = NEON_4H | NEON_Q,
+ NEON_2S = 0x00800000,
+ NEON_4S = NEON_2S | NEON_Q,
+ NEON_1D = 0x00C00000,
+ NEON_2D = 0x00C00000 | NEON_Q
+};
+
+enum NEONFPFormatField : uint32_t {
+ NEONFPFormatFieldMask = 0x40400000,
+ NEON_FP_2S = FP32,
+ NEON_FP_4S = FP32 | NEON_Q,
+ NEON_FP_2D = FP64 | NEON_Q
+};
+
+enum NEONLSFormatField : uint32_t {
+ NEONLSFormatFieldMask = 0x40000C00,
+ LS_NEON_8B = 0x00000000,
+ LS_NEON_16B = LS_NEON_8B | NEON_Q,
+ LS_NEON_4H = 0x00000400,
+ LS_NEON_8H = LS_NEON_4H | NEON_Q,
+ LS_NEON_2S = 0x00000800,
+ LS_NEON_4S = LS_NEON_2S | NEON_Q,
+ LS_NEON_1D = 0x00000C00,
+ LS_NEON_2D = LS_NEON_1D | NEON_Q
+};
+
+enum NEONScalarFormatField : uint32_t {
+ NEONScalarFormatFieldMask = 0x00C00000,
+ NEONScalar = 0x10000000,
+ NEON_B = 0x00000000,
+ NEON_H = 0x00400000,
+ NEON_S = 0x00800000,
+ NEON_D = 0x00C00000
+};
+
+// PC relative addressing.
+enum PCRelAddressingOp : uint32_t {
+ PCRelAddressingFixed = 0x10000000,
+ PCRelAddressingFMask = 0x1F000000,
+ PCRelAddressingMask = 0x9F000000,
+ ADR = PCRelAddressingFixed | 0x00000000,
+ ADRP = PCRelAddressingFixed | 0x80000000
+};
+
+// Add/sub (immediate, shifted and extended.)
+const int kSFOffset = 31;
+enum AddSubOp : uint32_t {
+ AddSubOpMask = 0x60000000,
+ AddSubSetFlagsBit = 0x20000000,
+ ADD = 0x00000000,
+ ADDS = ADD | AddSubSetFlagsBit,
+ SUB = 0x40000000,
+ SUBS = SUB | AddSubSetFlagsBit
+};
+
+#define ADD_SUB_OP_LIST(V) V(ADD), V(ADDS), V(SUB), V(SUBS)
+
+enum AddSubImmediateOp : uint32_t {
+ AddSubImmediateFixed = 0x11000000,
+ AddSubImmediateFMask = 0x1F000000,
+ AddSubImmediateMask = 0xFF000000,
+#define ADD_SUB_IMMEDIATE(A) \
+ A##_w_imm = AddSubImmediateFixed | A, \
+ A##_x_imm = AddSubImmediateFixed | A | SixtyFourBits
+ ADD_SUB_OP_LIST(ADD_SUB_IMMEDIATE)
+#undef ADD_SUB_IMMEDIATE
+};
+
+enum AddSubShiftedOp : uint32_t {
+ AddSubShiftedFixed = 0x0B000000,
+ AddSubShiftedFMask = 0x1F200000,
+ AddSubShiftedMask = 0xFF200000,
+#define ADD_SUB_SHIFTED(A) \
+ A##_w_shift = AddSubShiftedFixed | A, \
+ A##_x_shift = AddSubShiftedFixed | A | SixtyFourBits
+ ADD_SUB_OP_LIST(ADD_SUB_SHIFTED)
+#undef ADD_SUB_SHIFTED
+};
+
+enum AddSubExtendedOp : uint32_t {
+ AddSubExtendedFixed = 0x0B200000,
+ AddSubExtendedFMask = 0x1F200000,
+ AddSubExtendedMask = 0xFFE00000,
+#define ADD_SUB_EXTENDED(A) \
+ A##_w_ext = AddSubExtendedFixed | A, \
+ A##_x_ext = AddSubExtendedFixed | A | SixtyFourBits
+ ADD_SUB_OP_LIST(ADD_SUB_EXTENDED)
+#undef ADD_SUB_EXTENDED
+};
+
+// Add/sub with carry.
+enum AddSubWithCarryOp : uint32_t {
+ AddSubWithCarryFixed = 0x1A000000,
+ AddSubWithCarryFMask = 0x1FE00000,
+ AddSubWithCarryMask = 0xFFE0FC00,
+ ADC_w = AddSubWithCarryFixed | ADD,
+ ADC_x = AddSubWithCarryFixed | ADD | SixtyFourBits,
+ ADC = ADC_w,
+ ADCS_w = AddSubWithCarryFixed | ADDS,
+ ADCS_x = AddSubWithCarryFixed | ADDS | SixtyFourBits,
+ SBC_w = AddSubWithCarryFixed | SUB,
+ SBC_x = AddSubWithCarryFixed | SUB | SixtyFourBits,
+ SBC = SBC_w,
+ SBCS_w = AddSubWithCarryFixed | SUBS,
+ SBCS_x = AddSubWithCarryFixed | SUBS | SixtyFourBits
+};
+
+// Logical (immediate and shifted register).
+enum LogicalOp : uint32_t {
+ LogicalOpMask = 0x60200000,
+ NOT = 0x00200000,
+ AND = 0x00000000,
+ BIC = AND | NOT,
+ ORR = 0x20000000,
+ ORN = ORR | NOT,
+ EOR = 0x40000000,
+ EON = EOR | NOT,
+ ANDS = 0x60000000,
+ BICS = ANDS | NOT
+};
+
+// Logical immediate.
+enum LogicalImmediateOp : uint32_t {
+ LogicalImmediateFixed = 0x12000000,
+ LogicalImmediateFMask = 0x1F800000,
+ LogicalImmediateMask = 0xFF800000,
+ AND_w_imm = LogicalImmediateFixed | AND,
+ AND_x_imm = LogicalImmediateFixed | AND | SixtyFourBits,
+ ORR_w_imm = LogicalImmediateFixed | ORR,
+ ORR_x_imm = LogicalImmediateFixed | ORR | SixtyFourBits,
+ EOR_w_imm = LogicalImmediateFixed | EOR,
+ EOR_x_imm = LogicalImmediateFixed | EOR | SixtyFourBits,
+ ANDS_w_imm = LogicalImmediateFixed | ANDS,
+ ANDS_x_imm = LogicalImmediateFixed | ANDS | SixtyFourBits
+};
+
+// Logical shifted register.
+enum LogicalShiftedOp : uint32_t {
+ LogicalShiftedFixed = 0x0A000000,
+ LogicalShiftedFMask = 0x1F000000,
+ LogicalShiftedMask = 0xFF200000,
+ AND_w = LogicalShiftedFixed | AND,
+ AND_x = LogicalShiftedFixed | AND | SixtyFourBits,
+ AND_shift = AND_w,
+ BIC_w = LogicalShiftedFixed | BIC,
+ BIC_x = LogicalShiftedFixed | BIC | SixtyFourBits,
+ BIC_shift = BIC_w,
+ ORR_w = LogicalShiftedFixed | ORR,
+ ORR_x = LogicalShiftedFixed | ORR | SixtyFourBits,
+ ORR_shift = ORR_w,
+ ORN_w = LogicalShiftedFixed | ORN,
+ ORN_x = LogicalShiftedFixed | ORN | SixtyFourBits,
+ ORN_shift = ORN_w,
+ EOR_w = LogicalShiftedFixed | EOR,
+ EOR_x = LogicalShiftedFixed | EOR | SixtyFourBits,
+ EOR_shift = EOR_w,
+ EON_w = LogicalShiftedFixed | EON,
+ EON_x = LogicalShiftedFixed | EON | SixtyFourBits,
+ EON_shift = EON_w,
+ ANDS_w = LogicalShiftedFixed | ANDS,
+ ANDS_x = LogicalShiftedFixed | ANDS | SixtyFourBits,
+ ANDS_shift = ANDS_w,
+ BICS_w = LogicalShiftedFixed | BICS,
+ BICS_x = LogicalShiftedFixed | BICS | SixtyFourBits,
+ BICS_shift = BICS_w
+};
+
+// Move wide immediate.
+enum MoveWideImmediateOp : uint32_t {
+ MoveWideImmediateFixed = 0x12800000,
+ MoveWideImmediateFMask = 0x1F800000,
+ MoveWideImmediateMask = 0xFF800000,
+ MOVN = 0x00000000,
+ MOVZ = 0x40000000,
+ MOVK = 0x60000000,
+ MOVN_w = MoveWideImmediateFixed | MOVN,
+ MOVN_x = MoveWideImmediateFixed | MOVN | SixtyFourBits,
+ MOVZ_w = MoveWideImmediateFixed | MOVZ,
+ MOVZ_x = MoveWideImmediateFixed | MOVZ | SixtyFourBits,
+ MOVK_w = MoveWideImmediateFixed | MOVK,
+ MOVK_x = MoveWideImmediateFixed | MOVK | SixtyFourBits
+};
+
+// Bitfield.
+const int kBitfieldNOffset = 22;
+enum BitfieldOp : uint32_t {
+ BitfieldFixed = 0x13000000,
+ BitfieldFMask = 0x1F800000,
+ BitfieldMask = 0xFF800000,
+ SBFM_w = BitfieldFixed | 0x00000000,
+ SBFM_x = BitfieldFixed | 0x80000000,
+ SBFM = SBFM_w,
+ BFM_w = BitfieldFixed | 0x20000000,
+ BFM_x = BitfieldFixed | 0xA0000000,
+ BFM = BFM_w,
+ UBFM_w = BitfieldFixed | 0x40000000,
+ UBFM_x = BitfieldFixed | 0xC0000000,
+ UBFM = UBFM_w
+ // Bitfield N field.
+};
+
+// Extract.
+enum ExtractOp : uint32_t {
+ ExtractFixed = 0x13800000,
+ ExtractFMask = 0x1F800000,
+ ExtractMask = 0xFFA00000,
+ EXTR_w = ExtractFixed | 0x00000000,
+ EXTR_x = ExtractFixed | 0x80000000,
+ EXTR = EXTR_w
+};
+
+// Unconditional branch.
+enum UnconditionalBranchOp : uint32_t {
+ UnconditionalBranchFixed = 0x14000000,
+ UnconditionalBranchFMask = 0x7C000000,
+ UnconditionalBranchMask = 0xFC000000,
+ B = UnconditionalBranchFixed | 0x00000000,
+ BL = UnconditionalBranchFixed | 0x80000000
+};
+
+// Unconditional branch to register.
+enum UnconditionalBranchToRegisterOp : uint32_t {
+ UnconditionalBranchToRegisterFixed = 0xD6000000,
+ UnconditionalBranchToRegisterFMask = 0xFE000000,
+ UnconditionalBranchToRegisterMask = 0xFFFFFC1F,
+ BR = UnconditionalBranchToRegisterFixed | 0x001F0000,
+ BLR = UnconditionalBranchToRegisterFixed | 0x003F0000,
+ RET = UnconditionalBranchToRegisterFixed | 0x005F0000
+};
+
+// Compare and branch.
+enum CompareBranchOp : uint32_t {
+ CompareBranchFixed = 0x34000000,
+ CompareBranchFMask = 0x7E000000,
+ CompareBranchMask = 0xFF000000,
+ CBZ_w = CompareBranchFixed | 0x00000000,
+ CBZ_x = CompareBranchFixed | 0x80000000,
+ CBZ = CBZ_w,
+ CBNZ_w = CompareBranchFixed | 0x01000000,
+ CBNZ_x = CompareBranchFixed | 0x81000000,
+ CBNZ = CBNZ_w
+};
+
+// Test and branch.
+enum TestBranchOp : uint32_t {
+ TestBranchFixed = 0x36000000,
+ TestBranchFMask = 0x7E000000,
+ TestBranchMask = 0x7F000000,
+ TBZ = TestBranchFixed | 0x00000000,
+ TBNZ = TestBranchFixed | 0x01000000
+};
+
+// Conditional branch.
+enum ConditionalBranchOp : uint32_t {
+ ConditionalBranchFixed = 0x54000000,
+ ConditionalBranchFMask = 0xFE000000,
+ ConditionalBranchMask = 0xFF000010,
+ B_cond = ConditionalBranchFixed | 0x00000000
+};
+
+// System.
+// System instruction encoding is complicated because some instructions use op
+// and CR fields to encode parameters. To handle this cleanly, the system
+// instructions are split into more than one enum.
+
+enum SystemOp : uint32_t { SystemFixed = 0xD5000000, SystemFMask = 0xFFC00000 };
+
+enum SystemSysRegOp : uint32_t {
+ SystemSysRegFixed = 0xD5100000,
+ SystemSysRegFMask = 0xFFD00000,
+ SystemSysRegMask = 0xFFF00000,
+ MRS = SystemSysRegFixed | 0x00200000,
+ MSR = SystemSysRegFixed | 0x00000000
+};
+
+enum SystemHintOp : uint32_t {
+ SystemHintFixed = 0xD503201F,
+ SystemHintFMask = 0xFFFFF01F,
+ SystemHintMask = 0xFFFFF01F,
+ HINT = SystemHintFixed | 0x00000000
+};
+
+// Exception.
+enum ExceptionOp : uint32_t {
+ ExceptionFixed = 0xD4000000,
+ ExceptionFMask = 0xFF000000,
+ ExceptionMask = 0xFFE0001F,
+ HLT = ExceptionFixed | 0x00400000,
+ BRK = ExceptionFixed | 0x00200000,
+ SVC = ExceptionFixed | 0x00000001,
+ HVC = ExceptionFixed | 0x00000002,
+ SMC = ExceptionFixed | 0x00000003,
+ DCPS1 = ExceptionFixed | 0x00A00001,
+ DCPS2 = ExceptionFixed | 0x00A00002,
+ DCPS3 = ExceptionFixed | 0x00A00003
+};
+// Code used to spot hlt instructions that should not be hit.
+const int kHltBadCode = 0xbad;
+
+enum MemBarrierOp : uint32_t {
+ MemBarrierFixed = 0xD503309F,
+ MemBarrierFMask = 0xFFFFF09F,
+ MemBarrierMask = 0xFFFFF0FF,
+ DSB = MemBarrierFixed | 0x00000000,
+ DMB = MemBarrierFixed | 0x00000020,
+ ISB = MemBarrierFixed | 0x00000040
+};
+
+enum SystemPAuthOp : uint32_t {
+ SystemPAuthFixed = 0xD503211F,
+ SystemPAuthFMask = 0xFFFFFD1F,
+ SystemPAuthMask = 0xFFFFFFFF,
+ PACIB1716 = SystemPAuthFixed | 0x00000140,
+ AUTIB1716 = SystemPAuthFixed | 0x000001C0,
+ PACIBSP = SystemPAuthFixed | 0x00000360,
+ AUTIBSP = SystemPAuthFixed | 0x000003E0
+};
+
+// Any load or store (including pair).
+enum LoadStoreAnyOp : uint32_t {
+ LoadStoreAnyFMask = 0x0a000000,
+ LoadStoreAnyFixed = 0x08000000
+};
+
+// Any load pair or store pair.
+enum LoadStorePairAnyOp : uint32_t {
+ LoadStorePairAnyFMask = 0x3a000000,
+ LoadStorePairAnyFixed = 0x28000000
+};
+
+#define LOAD_STORE_PAIR_OP_LIST(V) \
+ V(STP, w, 0x00000000) \
+ , V(LDP, w, 0x00400000), V(LDPSW, x, 0x40400000), V(STP, x, 0x80000000), \
+ V(LDP, x, 0x80400000), V(STP, s, 0x04000000), V(LDP, s, 0x04400000), \
+ V(STP, d, 0x44000000), V(LDP, d, 0x44400000), V(STP, q, 0x84000000), \
+ V(LDP, q, 0x84400000)
+
+// Load/store pair (post, pre and offset.)
+enum LoadStorePairOp : uint32_t {
+ LoadStorePairMask = 0xC4400000,
+ LoadStorePairLBit = 1 << 22,
+#define LOAD_STORE_PAIR(A, B, C) A##_##B = C
+ LOAD_STORE_PAIR_OP_LIST(LOAD_STORE_PAIR)
+#undef LOAD_STORE_PAIR
+};
+
+enum LoadStorePairPostIndexOp : uint32_t {
+ LoadStorePairPostIndexFixed = 0x28800000,
+ LoadStorePairPostIndexFMask = 0x3B800000,
+ LoadStorePairPostIndexMask = 0xFFC00000,
+#define LOAD_STORE_PAIR_POST_INDEX(A, B, C) \
+ A##_##B##_post = LoadStorePairPostIndexFixed | A##_##B
+ LOAD_STORE_PAIR_OP_LIST(LOAD_STORE_PAIR_POST_INDEX)
+#undef LOAD_STORE_PAIR_POST_INDEX
+};
+
+enum LoadStorePairPreIndexOp : uint32_t {
+ LoadStorePairPreIndexFixed = 0x29800000,
+ LoadStorePairPreIndexFMask = 0x3B800000,
+ LoadStorePairPreIndexMask = 0xFFC00000,
+#define LOAD_STORE_PAIR_PRE_INDEX(A, B, C) \
+ A##_##B##_pre = LoadStorePairPreIndexFixed | A##_##B
+ LOAD_STORE_PAIR_OP_LIST(LOAD_STORE_PAIR_PRE_INDEX)
+#undef LOAD_STORE_PAIR_PRE_INDEX
+};
+
+enum LoadStorePairOffsetOp : uint32_t {
+ LoadStorePairOffsetFixed = 0x29000000,
+ LoadStorePairOffsetFMask = 0x3B800000,
+ LoadStorePairOffsetMask = 0xFFC00000,
+#define LOAD_STORE_PAIR_OFFSET(A, B, C) \
+ A##_##B##_off = LoadStorePairOffsetFixed | A##_##B
+ LOAD_STORE_PAIR_OP_LIST(LOAD_STORE_PAIR_OFFSET)
+#undef LOAD_STORE_PAIR_OFFSET
+};
+
+// Load literal.
+enum LoadLiteralOp : uint32_t {
+ LoadLiteralFixed = 0x18000000,
+ LoadLiteralFMask = 0x3B000000,
+ LoadLiteralMask = 0xFF000000,
+ LDR_w_lit = LoadLiteralFixed | 0x00000000,
+ LDR_x_lit = LoadLiteralFixed | 0x40000000,
+ LDRSW_x_lit = LoadLiteralFixed | 0x80000000,
+ PRFM_lit = LoadLiteralFixed | 0xC0000000,
+ LDR_s_lit = LoadLiteralFixed | 0x04000000,
+ LDR_d_lit = LoadLiteralFixed | 0x44000000
+};
+
+// clang-format off
+
+#define LOAD_STORE_OP_LIST(V) \
+ V(ST, RB, w, 0x00000000), \
+ V(ST, RH, w, 0x40000000), \
+ V(ST, R, w, 0x80000000), \
+ V(ST, R, x, 0xC0000000), \
+ V(LD, RB, w, 0x00400000), \
+ V(LD, RH, w, 0x40400000), \
+ V(LD, R, w, 0x80400000), \
+ V(LD, R, x, 0xC0400000), \
+ V(LD, RSB, x, 0x00800000), \
+ V(LD, RSH, x, 0x40800000), \
+ V(LD, RSW, x, 0x80800000), \
+ V(LD, RSB, w, 0x00C00000), \
+ V(LD, RSH, w, 0x40C00000), \
+ V(ST, R, b, 0x04000000), \
+ V(ST, R, h, 0x44000000), \
+ V(ST, R, s, 0x84000000), \
+ V(ST, R, d, 0xC4000000), \
+ V(ST, R, q, 0x04800000), \
+ V(LD, R, b, 0x04400000), \
+ V(LD, R, h, 0x44400000), \
+ V(LD, R, s, 0x84400000), \
+ V(LD, R, d, 0xC4400000), \
+ V(LD, R, q, 0x04C00000)
+
+// clang-format on
+
+// Load/store unscaled offset.
+enum LoadStoreUnscaledOffsetOp : uint32_t {
+ LoadStoreUnscaledOffsetFixed = 0x38000000,
+ LoadStoreUnscaledOffsetFMask = 0x3B200C00,
+ LoadStoreUnscaledOffsetMask = 0xFFE00C00,
+#define LOAD_STORE_UNSCALED(A, B, C, D) \
+ A##U##B##_##C = LoadStoreUnscaledOffsetFixed | D
+ LOAD_STORE_OP_LIST(LOAD_STORE_UNSCALED)
+#undef LOAD_STORE_UNSCALED
+};
+
+// Load/store (post, pre, offset and unsigned.)
+enum LoadStoreOp : uint32_t {
+ LoadStoreMask = 0xC4C00000,
+#define LOAD_STORE(A, B, C, D) A##B##_##C = D
+ LOAD_STORE_OP_LIST(LOAD_STORE),
+#undef LOAD_STORE
+ PRFM = 0xC0800000
+};
+
+// Load/store post index.
+enum LoadStorePostIndex : uint32_t {
+ LoadStorePostIndexFixed = 0x38000400,
+ LoadStorePostIndexFMask = 0x3B200C00,
+ LoadStorePostIndexMask = 0xFFE00C00,
+#define LOAD_STORE_POST_INDEX(A, B, C, D) \
+ A##B##_##C##_post = LoadStorePostIndexFixed | D
+ LOAD_STORE_OP_LIST(LOAD_STORE_POST_INDEX)
+#undef LOAD_STORE_POST_INDEX
+};
+
+// Load/store pre index.
+enum LoadStorePreIndex : uint32_t {
+ LoadStorePreIndexFixed = 0x38000C00,
+ LoadStorePreIndexFMask = 0x3B200C00,
+ LoadStorePreIndexMask = 0xFFE00C00,
+#define LOAD_STORE_PRE_INDEX(A, B, C, D) \
+ A##B##_##C##_pre = LoadStorePreIndexFixed | D
+ LOAD_STORE_OP_LIST(LOAD_STORE_PRE_INDEX)
+#undef LOAD_STORE_PRE_INDEX
+};
+
+// Load/store unsigned offset.
+enum LoadStoreUnsignedOffset : uint32_t {
+ LoadStoreUnsignedOffsetFixed = 0x39000000,
+ LoadStoreUnsignedOffsetFMask = 0x3B000000,
+ LoadStoreUnsignedOffsetMask = 0xFFC00000,
+ PRFM_unsigned = LoadStoreUnsignedOffsetFixed | PRFM,
+#define LOAD_STORE_UNSIGNED_OFFSET(A, B, C, D) \
+ A##B##_##C##_unsigned = LoadStoreUnsignedOffsetFixed | D
+ LOAD_STORE_OP_LIST(LOAD_STORE_UNSIGNED_OFFSET)
+#undef LOAD_STORE_UNSIGNED_OFFSET
+};
+
+// Load/store register offset.
+enum LoadStoreRegisterOffset : uint32_t {
+ LoadStoreRegisterOffsetFixed = 0x38200800,
+ LoadStoreRegisterOffsetFMask = 0x3B200C00,
+ LoadStoreRegisterOffsetMask = 0xFFE00C00,
+ PRFM_reg = LoadStoreRegisterOffsetFixed | PRFM,
+#define LOAD_STORE_REGISTER_OFFSET(A, B, C, D) \
+ A##B##_##C##_reg = LoadStoreRegisterOffsetFixed | D
+ LOAD_STORE_OP_LIST(LOAD_STORE_REGISTER_OFFSET)
+#undef LOAD_STORE_REGISTER_OFFSET
+};
+
+// Load/store acquire/release.
+enum LoadStoreAcquireReleaseOp : uint32_t {
+ LoadStoreAcquireReleaseFixed = 0x08000000,
+ LoadStoreAcquireReleaseFMask = 0x3F000000,
+ LoadStoreAcquireReleaseMask = 0xCFC08000,
+ STLXR_b = LoadStoreAcquireReleaseFixed | 0x00008000,
+ LDAXR_b = LoadStoreAcquireReleaseFixed | 0x00408000,
+ STLR_b = LoadStoreAcquireReleaseFixed | 0x00808000,
+ LDAR_b = LoadStoreAcquireReleaseFixed | 0x00C08000,
+ STLXR_h = LoadStoreAcquireReleaseFixed | 0x40008000,
+ LDAXR_h = LoadStoreAcquireReleaseFixed | 0x40408000,
+ STLR_h = LoadStoreAcquireReleaseFixed | 0x40808000,
+ LDAR_h = LoadStoreAcquireReleaseFixed | 0x40C08000,
+ STLXR_w = LoadStoreAcquireReleaseFixed | 0x80008000,
+ LDAXR_w = LoadStoreAcquireReleaseFixed | 0x80408000,
+ STLR_w = LoadStoreAcquireReleaseFixed | 0x80808000,
+ LDAR_w = LoadStoreAcquireReleaseFixed | 0x80C08000,
+ STLXR_x = LoadStoreAcquireReleaseFixed | 0xC0008000,
+ LDAXR_x = LoadStoreAcquireReleaseFixed | 0xC0408000,
+ STLR_x = LoadStoreAcquireReleaseFixed | 0xC0808000,
+ LDAR_x = LoadStoreAcquireReleaseFixed | 0xC0C08000,
+};
+
+// Conditional compare.
+enum ConditionalCompareOp : uint32_t {
+ ConditionalCompareMask = 0x60000000,
+ CCMN = 0x20000000,
+ CCMP = 0x60000000
+};
+
+// Conditional compare register.
+enum ConditionalCompareRegisterOp : uint32_t {
+ ConditionalCompareRegisterFixed = 0x1A400000,
+ ConditionalCompareRegisterFMask = 0x1FE00800,
+ ConditionalCompareRegisterMask = 0xFFE00C10,
+ CCMN_w = ConditionalCompareRegisterFixed | CCMN,
+ CCMN_x = ConditionalCompareRegisterFixed | SixtyFourBits | CCMN,
+ CCMP_w = ConditionalCompareRegisterFixed | CCMP,
+ CCMP_x = ConditionalCompareRegisterFixed | SixtyFourBits | CCMP
+};
+
+// Conditional compare immediate.
+enum ConditionalCompareImmediateOp : uint32_t {
+ ConditionalCompareImmediateFixed = 0x1A400800,
+ ConditionalCompareImmediateFMask = 0x1FE00800,
+ ConditionalCompareImmediateMask = 0xFFE00C10,
+ CCMN_w_imm = ConditionalCompareImmediateFixed | CCMN,
+ CCMN_x_imm = ConditionalCompareImmediateFixed | SixtyFourBits | CCMN,
+ CCMP_w_imm = ConditionalCompareImmediateFixed | CCMP,
+ CCMP_x_imm = ConditionalCompareImmediateFixed | SixtyFourBits | CCMP
+};
+
+// Conditional select.
+enum ConditionalSelectOp : uint32_t {
+ ConditionalSelectFixed = 0x1A800000,
+ ConditionalSelectFMask = 0x1FE00000,
+ ConditionalSelectMask = 0xFFE00C00,
+ CSEL_w = ConditionalSelectFixed | 0x00000000,
+ CSEL_x = ConditionalSelectFixed | 0x80000000,
+ CSEL = CSEL_w,
+ CSINC_w = ConditionalSelectFixed | 0x00000400,
+ CSINC_x = ConditionalSelectFixed | 0x80000400,
+ CSINC = CSINC_w,
+ CSINV_w = ConditionalSelectFixed | 0x40000000,
+ CSINV_x = ConditionalSelectFixed | 0xC0000000,
+ CSINV = CSINV_w,
+ CSNEG_w = ConditionalSelectFixed | 0x40000400,
+ CSNEG_x = ConditionalSelectFixed | 0xC0000400,
+ CSNEG = CSNEG_w
+};
+
+// Data processing 1 source.
+enum DataProcessing1SourceOp : uint32_t {
+ DataProcessing1SourceFixed = 0x5AC00000,
+ DataProcessing1SourceFMask = 0x5FE00000,
+ DataProcessing1SourceMask = 0xFFFFFC00,
+ RBIT = DataProcessing1SourceFixed | 0x00000000,
+ RBIT_w = RBIT,
+ RBIT_x = RBIT | SixtyFourBits,
+ REV16 = DataProcessing1SourceFixed | 0x00000400,
+ REV16_w = REV16,
+ REV16_x = REV16 | SixtyFourBits,
+ REV = DataProcessing1SourceFixed | 0x00000800,
+ REV_w = REV,
+ REV32_x = REV | SixtyFourBits,
+ REV_x = DataProcessing1SourceFixed | SixtyFourBits | 0x00000C00,
+ CLZ = DataProcessing1SourceFixed | 0x00001000,
+ CLZ_w = CLZ,
+ CLZ_x = CLZ | SixtyFourBits,
+ CLS = DataProcessing1SourceFixed | 0x00001400,
+ CLS_w = CLS,
+ CLS_x = CLS | SixtyFourBits
+};
+
+// Data processing 2 source.
+enum DataProcessing2SourceOp : uint32_t {
+ DataProcessing2SourceFixed = 0x1AC00000,
+ DataProcessing2SourceFMask = 0x5FE00000,
+ DataProcessing2SourceMask = 0xFFE0FC00,
+ UDIV_w = DataProcessing2SourceFixed | 0x00000800,
+ UDIV_x = DataProcessing2SourceFixed | 0x80000800,
+ UDIV = UDIV_w,
+ SDIV_w = DataProcessing2SourceFixed | 0x00000C00,
+ SDIV_x = DataProcessing2SourceFixed | 0x80000C00,
+ SDIV = SDIV_w,
+ LSLV_w = DataProcessing2SourceFixed | 0x00002000,
+ LSLV_x = DataProcessing2SourceFixed | 0x80002000,
+ LSLV = LSLV_w,
+ LSRV_w = DataProcessing2SourceFixed | 0x00002400,
+ LSRV_x = DataProcessing2SourceFixed | 0x80002400,
+ LSRV = LSRV_w,
+ ASRV_w = DataProcessing2SourceFixed | 0x00002800,
+ ASRV_x = DataProcessing2SourceFixed | 0x80002800,
+ ASRV = ASRV_w,
+ RORV_w = DataProcessing2SourceFixed | 0x00002C00,
+ RORV_x = DataProcessing2SourceFixed | 0x80002C00,
+ RORV = RORV_w,
+ CRC32B = DataProcessing2SourceFixed | 0x00004000,
+ CRC32H = DataProcessing2SourceFixed | 0x00004400,
+ CRC32W = DataProcessing2SourceFixed | 0x00004800,
+ CRC32X = DataProcessing2SourceFixed | SixtyFourBits | 0x00004C00,
+ CRC32CB = DataProcessing2SourceFixed | 0x00005000,
+ CRC32CH = DataProcessing2SourceFixed | 0x00005400,
+ CRC32CW = DataProcessing2SourceFixed | 0x00005800,
+ CRC32CX = DataProcessing2SourceFixed | SixtyFourBits | 0x00005C00
+};
+
+// Data processing 3 source.
+enum DataProcessing3SourceOp : uint32_t {
+ DataProcessing3SourceFixed = 0x1B000000,
+ DataProcessing3SourceFMask = 0x1F000000,
+ DataProcessing3SourceMask = 0xFFE08000,
+ MADD_w = DataProcessing3SourceFixed | 0x00000000,
+ MADD_x = DataProcessing3SourceFixed | 0x80000000,
+ MADD = MADD_w,
+ MSUB_w = DataProcessing3SourceFixed | 0x00008000,
+ MSUB_x = DataProcessing3SourceFixed | 0x80008000,
+ MSUB = MSUB_w,
+ SMADDL_x = DataProcessing3SourceFixed | 0x80200000,
+ SMSUBL_x = DataProcessing3SourceFixed | 0x80208000,
+ SMULH_x = DataProcessing3SourceFixed | 0x80400000,
+ UMADDL_x = DataProcessing3SourceFixed | 0x80A00000,
+ UMSUBL_x = DataProcessing3SourceFixed | 0x80A08000,
+ UMULH_x = DataProcessing3SourceFixed | 0x80C00000
+};
+
+// Floating point compare.
+enum FPCompareOp : uint32_t {
+ FPCompareFixed = 0x1E202000,
+ FPCompareFMask = 0x5F203C00,
+ FPCompareMask = 0xFFE0FC1F,
+ FCMP_s = FPCompareFixed | 0x00000000,
+ FCMP_d = FPCompareFixed | FP64 | 0x00000000,
+ FCMP = FCMP_s,
+ FCMP_s_zero = FPCompareFixed | 0x00000008,
+ FCMP_d_zero = FPCompareFixed | FP64 | 0x00000008,
+ FCMP_zero = FCMP_s_zero,
+ FCMPE_s = FPCompareFixed | 0x00000010,
+ FCMPE_d = FPCompareFixed | FP64 | 0x00000010,
+ FCMPE_s_zero = FPCompareFixed | 0x00000018,
+ FCMPE_d_zero = FPCompareFixed | FP64 | 0x00000018
+};
+
+// Floating point conditional compare.
+enum FPConditionalCompareOp : uint32_t {
+ FPConditionalCompareFixed = 0x1E200400,
+ FPConditionalCompareFMask = 0x5F200C00,
+ FPConditionalCompareMask = 0xFFE00C10,
+ FCCMP_s = FPConditionalCompareFixed | 0x00000000,
+ FCCMP_d = FPConditionalCompareFixed | FP64 | 0x00000000,
+ FCCMP = FCCMP_s,
+ FCCMPE_s = FPConditionalCompareFixed | 0x00000010,
+ FCCMPE_d = FPConditionalCompareFixed | FP64 | 0x00000010,
+ FCCMPE = FCCMPE_s
+};
+
+// Floating point conditional select.
+enum FPConditionalSelectOp : uint32_t {
+ FPConditionalSelectFixed = 0x1E200C00,
+ FPConditionalSelectFMask = 0x5F200C00,
+ FPConditionalSelectMask = 0xFFE00C00,
+ FCSEL_s = FPConditionalSelectFixed | 0x00000000,
+ FCSEL_d = FPConditionalSelectFixed | FP64 | 0x00000000,
+ FCSEL = FCSEL_s
+};
+
+// Floating point immediate.
+enum FPImmediateOp : uint32_t {
+ FPImmediateFixed = 0x1E201000,
+ FPImmediateFMask = 0x5F201C00,
+ FPImmediateMask = 0xFFE01C00,
+ FMOV_s_imm = FPImmediateFixed | 0x00000000,
+ FMOV_d_imm = FPImmediateFixed | FP64 | 0x00000000
+};
+
+// Floating point data processing 1 source.
+enum FPDataProcessing1SourceOp : uint32_t {
+ FPDataProcessing1SourceFixed = 0x1E204000,
+ FPDataProcessing1SourceFMask = 0x5F207C00,
+ FPDataProcessing1SourceMask = 0xFFFFFC00,
+ FMOV_s = FPDataProcessing1SourceFixed | 0x00000000,
+ FMOV_d = FPDataProcessing1SourceFixed | FP64 | 0x00000000,
+ FMOV = FMOV_s,
+ FABS_s = FPDataProcessing1SourceFixed | 0x00008000,
+ FABS_d = FPDataProcessing1SourceFixed | FP64 | 0x00008000,
+ FABS = FABS_s,
+ FNEG_s = FPDataProcessing1SourceFixed | 0x00010000,
+ FNEG_d = FPDataProcessing1SourceFixed | FP64 | 0x00010000,
+ FNEG = FNEG_s,
+ FSQRT_s = FPDataProcessing1SourceFixed | 0x00018000,
+ FSQRT_d = FPDataProcessing1SourceFixed | FP64 | 0x00018000,
+ FSQRT = FSQRT_s,
+ FCVT_ds = FPDataProcessing1SourceFixed | 0x00028000,
+ FCVT_sd = FPDataProcessing1SourceFixed | FP64 | 0x00020000,
+ FCVT_hs = FPDataProcessing1SourceFixed | 0x00038000,
+ FCVT_hd = FPDataProcessing1SourceFixed | FP64 | 0x00038000,
+ FCVT_sh = FPDataProcessing1SourceFixed | 0x00C20000,
+ FCVT_dh = FPDataProcessing1SourceFixed | 0x00C28000,
+ FRINTN_s = FPDataProcessing1SourceFixed | 0x00040000,
+ FRINTN_d = FPDataProcessing1SourceFixed | FP64 | 0x00040000,
+ FRINTN = FRINTN_s,
+ FRINTP_s = FPDataProcessing1SourceFixed | 0x00048000,
+ FRINTP_d = FPDataProcessing1SourceFixed | FP64 | 0x00048000,
+ FRINTP = FRINTP_s,
+ FRINTM_s = FPDataProcessing1SourceFixed | 0x00050000,
+ FRINTM_d = FPDataProcessing1SourceFixed | FP64 | 0x00050000,
+ FRINTM = FRINTM_s,
+ FRINTZ_s = FPDataProcessing1SourceFixed | 0x00058000,
+ FRINTZ_d = FPDataProcessing1SourceFixed | FP64 | 0x00058000,
+ FRINTZ = FRINTZ_s,
+ FRINTA_s = FPDataProcessing1SourceFixed | 0x00060000,
+ FRINTA_d = FPDataProcessing1SourceFixed | FP64 | 0x00060000,
+ FRINTA = FRINTA_s,
+ FRINTX_s = FPDataProcessing1SourceFixed | 0x00070000,
+ FRINTX_d = FPDataProcessing1SourceFixed | FP64 | 0x00070000,
+ FRINTX = FRINTX_s,
+ FRINTI_s = FPDataProcessing1SourceFixed | 0x00078000,
+ FRINTI_d = FPDataProcessing1SourceFixed | FP64 | 0x00078000,
+ FRINTI = FRINTI_s
+};
+
+// Floating point data processing 2 source.
+enum FPDataProcessing2SourceOp : uint32_t {
+ FPDataProcessing2SourceFixed = 0x1E200800,
+ FPDataProcessing2SourceFMask = 0x5F200C00,
+ FPDataProcessing2SourceMask = 0xFFE0FC00,
+ FMUL = FPDataProcessing2SourceFixed | 0x00000000,
+ FMUL_s = FMUL,
+ FMUL_d = FMUL | FP64,
+ FDIV = FPDataProcessing2SourceFixed | 0x00001000,
+ FDIV_s = FDIV,
+ FDIV_d = FDIV | FP64,
+ FADD = FPDataProcessing2SourceFixed | 0x00002000,
+ FADD_s = FADD,
+ FADD_d = FADD | FP64,
+ FSUB = FPDataProcessing2SourceFixed | 0x00003000,
+ FSUB_s = FSUB,
+ FSUB_d = FSUB | FP64,
+ FMAX = FPDataProcessing2SourceFixed | 0x00004000,
+ FMAX_s = FMAX,
+ FMAX_d = FMAX | FP64,
+ FMIN = FPDataProcessing2SourceFixed | 0x00005000,
+ FMIN_s = FMIN,
+ FMIN_d = FMIN | FP64,
+ FMAXNM = FPDataProcessing2SourceFixed | 0x00006000,
+ FMAXNM_s = FMAXNM,
+ FMAXNM_d = FMAXNM | FP64,
+ FMINNM = FPDataProcessing2SourceFixed | 0x00007000,
+ FMINNM_s = FMINNM,
+ FMINNM_d = FMINNM | FP64,
+ FNMUL = FPDataProcessing2SourceFixed | 0x00008000,
+ FNMUL_s = FNMUL,
+ FNMUL_d = FNMUL | FP64
+};
+
+// Floating point data processing 3 source.
+enum FPDataProcessing3SourceOp : uint32_t {
+ FPDataProcessing3SourceFixed = 0x1F000000,
+ FPDataProcessing3SourceFMask = 0x5F000000,
+ FPDataProcessing3SourceMask = 0xFFE08000,
+ FMADD_s = FPDataProcessing3SourceFixed | 0x00000000,
+ FMSUB_s = FPDataProcessing3SourceFixed | 0x00008000,
+ FNMADD_s = FPDataProcessing3SourceFixed | 0x00200000,
+ FNMSUB_s = FPDataProcessing3SourceFixed | 0x00208000,
+ FMADD_d = FPDataProcessing3SourceFixed | 0x00400000,
+ FMSUB_d = FPDataProcessing3SourceFixed | 0x00408000,
+ FNMADD_d = FPDataProcessing3SourceFixed | 0x00600000,
+ FNMSUB_d = FPDataProcessing3SourceFixed | 0x00608000
+};
+
+// Conversion between floating point and integer.
+enum FPIntegerConvertOp : uint32_t {
+ FPIntegerConvertFixed = 0x1E200000,
+ FPIntegerConvertFMask = 0x5F20FC00,
+ FPIntegerConvertMask = 0xFFFFFC00,
+ FCVTNS = FPIntegerConvertFixed | 0x00000000,
+ FCVTNS_ws = FCVTNS,
+ FCVTNS_xs = FCVTNS | SixtyFourBits,
+ FCVTNS_wd = FCVTNS | FP64,
+ FCVTNS_xd = FCVTNS | SixtyFourBits | FP64,
+ FCVTNU = FPIntegerConvertFixed | 0x00010000,
+ FCVTNU_ws = FCVTNU,
+ FCVTNU_xs = FCVTNU | SixtyFourBits,
+ FCVTNU_wd = FCVTNU | FP64,
+ FCVTNU_xd = FCVTNU | SixtyFourBits | FP64,
+ FCVTPS = FPIntegerConvertFixed | 0x00080000,
+ FCVTPS_ws = FCVTPS,
+ FCVTPS_xs = FCVTPS | SixtyFourBits,
+ FCVTPS_wd = FCVTPS | FP64,
+ FCVTPS_xd = FCVTPS | SixtyFourBits | FP64,
+ FCVTPU = FPIntegerConvertFixed | 0x00090000,
+ FCVTPU_ws = FCVTPU,
+ FCVTPU_xs = FCVTPU | SixtyFourBits,
+ FCVTPU_wd = FCVTPU | FP64,
+ FCVTPU_xd = FCVTPU | SixtyFourBits | FP64,
+ FCVTMS = FPIntegerConvertFixed | 0x00100000,
+ FCVTMS_ws = FCVTMS,
+ FCVTMS_xs = FCVTMS | SixtyFourBits,
+ FCVTMS_wd = FCVTMS | FP64,
+ FCVTMS_xd = FCVTMS | SixtyFourBits | FP64,
+ FCVTMU = FPIntegerConvertFixed | 0x00110000,
+ FCVTMU_ws = FCVTMU,
+ FCVTMU_xs = FCVTMU | SixtyFourBits,
+ FCVTMU_wd = FCVTMU | FP64,
+ FCVTMU_xd = FCVTMU | SixtyFourBits | FP64,
+ FCVTZS = FPIntegerConvertFixed | 0x00180000,
+ FCVTZS_ws = FCVTZS,
+ FCVTZS_xs = FCVTZS | SixtyFourBits,
+ FCVTZS_wd = FCVTZS | FP64,
+ FCVTZS_xd = FCVTZS | SixtyFourBits | FP64,
+ FCVTZU = FPIntegerConvertFixed | 0x00190000,
+ FCVTZU_ws = FCVTZU,
+ FCVTZU_xs = FCVTZU | SixtyFourBits,
+ FCVTZU_wd = FCVTZU | FP64,
+ FCVTZU_xd = FCVTZU | SixtyFourBits | FP64,
+ SCVTF = FPIntegerConvertFixed | 0x00020000,
+ SCVTF_sw = SCVTF,
+ SCVTF_sx = SCVTF | SixtyFourBits,
+ SCVTF_dw = SCVTF | FP64,
+ SCVTF_dx = SCVTF | SixtyFourBits | FP64,
+ UCVTF = FPIntegerConvertFixed | 0x00030000,
+ UCVTF_sw = UCVTF,
+ UCVTF_sx = UCVTF | SixtyFourBits,
+ UCVTF_dw = UCVTF | FP64,
+ UCVTF_dx = UCVTF | SixtyFourBits | FP64,
+ FCVTAS = FPIntegerConvertFixed | 0x00040000,
+ FCVTAS_ws = FCVTAS,
+ FCVTAS_xs = FCVTAS | SixtyFourBits,
+ FCVTAS_wd = FCVTAS | FP64,
+ FCVTAS_xd = FCVTAS | SixtyFourBits | FP64,
+ FCVTAU = FPIntegerConvertFixed | 0x00050000,
+ FCVTAU_ws = FCVTAU,
+ FCVTAU_xs = FCVTAU | SixtyFourBits,
+ FCVTAU_wd = FCVTAU | FP64,
+ FCVTAU_xd = FCVTAU | SixtyFourBits | FP64,
+ FMOV_ws = FPIntegerConvertFixed | 0x00060000,
+ FMOV_sw = FPIntegerConvertFixed | 0x00070000,
+ FMOV_xd = FMOV_ws | SixtyFourBits | FP64,
+ FMOV_dx = FMOV_sw | SixtyFourBits | FP64,
+ FMOV_d1_x = FPIntegerConvertFixed | SixtyFourBits | 0x008F0000,
+ FMOV_x_d1 = FPIntegerConvertFixed | SixtyFourBits | 0x008E0000,
+ FJCVTZS = FPIntegerConvertFixed | FP64 | 0x001E0000
+};
+
+// Conversion between fixed point and floating point.
+enum FPFixedPointConvertOp : uint32_t {
+ FPFixedPointConvertFixed = 0x1E000000,
+ FPFixedPointConvertFMask = 0x5F200000,
+ FPFixedPointConvertMask = 0xFFFF0000,
+ FCVTZS_fixed = FPFixedPointConvertFixed | 0x00180000,
+ FCVTZS_ws_fixed = FCVTZS_fixed,
+ FCVTZS_xs_fixed = FCVTZS_fixed | SixtyFourBits,
+ FCVTZS_wd_fixed = FCVTZS_fixed | FP64,
+ FCVTZS_xd_fixed = FCVTZS_fixed | SixtyFourBits | FP64,
+ FCVTZU_fixed = FPFixedPointConvertFixed | 0x00190000,
+ FCVTZU_ws_fixed = FCVTZU_fixed,
+ FCVTZU_xs_fixed = FCVTZU_fixed | SixtyFourBits,
+ FCVTZU_wd_fixed = FCVTZU_fixed | FP64,
+ FCVTZU_xd_fixed = FCVTZU_fixed | SixtyFourBits | FP64,
+ SCVTF_fixed = FPFixedPointConvertFixed | 0x00020000,
+ SCVTF_sw_fixed = SCVTF_fixed,
+ SCVTF_sx_fixed = SCVTF_fixed | SixtyFourBits,
+ SCVTF_dw_fixed = SCVTF_fixed | FP64,
+ SCVTF_dx_fixed = SCVTF_fixed | SixtyFourBits | FP64,
+ UCVTF_fixed = FPFixedPointConvertFixed | 0x00030000,
+ UCVTF_sw_fixed = UCVTF_fixed,
+ UCVTF_sx_fixed = UCVTF_fixed | SixtyFourBits,
+ UCVTF_dw_fixed = UCVTF_fixed | FP64,
+ UCVTF_dx_fixed = UCVTF_fixed | SixtyFourBits | FP64
+};
+
+// NEON instructions with two register operands.
+enum NEON2RegMiscOp : uint32_t {
+ NEON2RegMiscFixed = 0x0E200800,
+ NEON2RegMiscFMask = 0x9F3E0C00,
+ NEON2RegMiscMask = 0xBF3FFC00,
+ NEON2RegMiscUBit = 0x20000000,
+ NEON_REV64 = NEON2RegMiscFixed | 0x00000000,
+ NEON_REV32 = NEON2RegMiscFixed | 0x20000000,
+ NEON_REV16 = NEON2RegMiscFixed | 0x00001000,
+ NEON_SADDLP = NEON2RegMiscFixed | 0x00002000,
+ NEON_UADDLP = NEON_SADDLP | NEON2RegMiscUBit,
+ NEON_SUQADD = NEON2RegMiscFixed | 0x00003000,
+ NEON_USQADD = NEON_SUQADD | NEON2RegMiscUBit,
+ NEON_CLS = NEON2RegMiscFixed | 0x00004000,
+ NEON_CLZ = NEON2RegMiscFixed | 0x20004000,
+ NEON_CNT = NEON2RegMiscFixed | 0x00005000,
+ NEON_RBIT_NOT = NEON2RegMiscFixed | 0x20005000,
+ NEON_SADALP = NEON2RegMiscFixed | 0x00006000,
+ NEON_UADALP = NEON_SADALP | NEON2RegMiscUBit,
+ NEON_SQABS = NEON2RegMiscFixed | 0x00007000,
+ NEON_SQNEG = NEON2RegMiscFixed | 0x20007000,
+ NEON_CMGT_zero = NEON2RegMiscFixed | 0x00008000,
+ NEON_CMGE_zero = NEON2RegMiscFixed | 0x20008000,
+ NEON_CMEQ_zero = NEON2RegMiscFixed | 0x00009000,
+ NEON_CMLE_zero = NEON2RegMiscFixed | 0x20009000,
+ NEON_CMLT_zero = NEON2RegMiscFixed | 0x0000A000,
+ NEON_ABS = NEON2RegMiscFixed | 0x0000B000,
+ NEON_NEG = NEON2RegMiscFixed | 0x2000B000,
+ NEON_XTN = NEON2RegMiscFixed | 0x00012000,
+ NEON_SQXTUN = NEON2RegMiscFixed | 0x20012000,
+ NEON_SHLL = NEON2RegMiscFixed | 0x20013000,
+ NEON_SQXTN = NEON2RegMiscFixed | 0x00014000,
+ NEON_UQXTN = NEON_SQXTN | NEON2RegMiscUBit,
+
+ NEON2RegMiscOpcode = 0x0001F000,
+ NEON_RBIT_NOT_opcode = NEON_RBIT_NOT & NEON2RegMiscOpcode,
+ NEON_NEG_opcode = NEON_NEG & NEON2RegMiscOpcode,
+ NEON_XTN_opcode = NEON_XTN & NEON2RegMiscOpcode,
+ NEON_UQXTN_opcode = NEON_UQXTN & NEON2RegMiscOpcode,
+
+ // These instructions use only one bit of the size field. The other bit is
+ // used to distinguish between instructions.
+ NEON2RegMiscFPMask = NEON2RegMiscMask | 0x00800000,
+ NEON_FABS = NEON2RegMiscFixed | 0x0080F000,
+ NEON_FNEG = NEON2RegMiscFixed | 0x2080F000,
+ NEON_FCVTN = NEON2RegMiscFixed | 0x00016000,
+ NEON_FCVTXN = NEON2RegMiscFixed | 0x20016000,
+ NEON_FCVTL = NEON2RegMiscFixed | 0x00017000,
+ NEON_FRINTN = NEON2RegMiscFixed | 0x00018000,
+ NEON_FRINTA = NEON2RegMiscFixed | 0x20018000,
+ NEON_FRINTP = NEON2RegMiscFixed | 0x00818000,
+ NEON_FRINTM = NEON2RegMiscFixed | 0x00019000,
+ NEON_FRINTX = NEON2RegMiscFixed | 0x20019000,
+ NEON_FRINTZ = NEON2RegMiscFixed | 0x00819000,
+ NEON_FRINTI = NEON2RegMiscFixed | 0x20819000,
+ NEON_FCVTNS = NEON2RegMiscFixed | 0x0001A000,
+ NEON_FCVTNU = NEON_FCVTNS | NEON2RegMiscUBit,
+ NEON_FCVTPS = NEON2RegMiscFixed | 0x0081A000,
+ NEON_FCVTPU = NEON_FCVTPS | NEON2RegMiscUBit,
+ NEON_FCVTMS = NEON2RegMiscFixed | 0x0001B000,
+ NEON_FCVTMU = NEON_FCVTMS | NEON2RegMiscUBit,
+ NEON_FCVTZS = NEON2RegMiscFixed | 0x0081B000,
+ NEON_FCVTZU = NEON_FCVTZS | NEON2RegMiscUBit,
+ NEON_FCVTAS = NEON2RegMiscFixed | 0x0001C000,
+ NEON_FCVTAU = NEON_FCVTAS | NEON2RegMiscUBit,
+ NEON_FSQRT = NEON2RegMiscFixed | 0x2081F000,
+ NEON_SCVTF = NEON2RegMiscFixed | 0x0001D000,
+ NEON_UCVTF = NEON_SCVTF | NEON2RegMiscUBit,
+ NEON_URSQRTE = NEON2RegMiscFixed | 0x2081C000,
+ NEON_URECPE = NEON2RegMiscFixed | 0x0081C000,
+ NEON_FRSQRTE = NEON2RegMiscFixed | 0x2081D000,
+ NEON_FRECPE = NEON2RegMiscFixed | 0x0081D000,
+ NEON_FCMGT_zero = NEON2RegMiscFixed | 0x0080C000,
+ NEON_FCMGE_zero = NEON2RegMiscFixed | 0x2080C000,
+ NEON_FCMEQ_zero = NEON2RegMiscFixed | 0x0080D000,
+ NEON_FCMLE_zero = NEON2RegMiscFixed | 0x2080D000,
+ NEON_FCMLT_zero = NEON2RegMiscFixed | 0x0080E000,
+
+ NEON_FCVTL_opcode = NEON_FCVTL & NEON2RegMiscOpcode,
+ NEON_FCVTN_opcode = NEON_FCVTN & NEON2RegMiscOpcode
+};
+
+// NEON instructions with three same-type operands.
+enum NEON3SameOp : uint32_t {
+ NEON3SameFixed = 0x0E200400,
+ NEON3SameFMask = 0x9F200400,
+ NEON3SameMask = 0xBF20FC00,
+ NEON3SameUBit = 0x20000000,
+ NEON_ADD = NEON3SameFixed | 0x00008000,
+ NEON_ADDP = NEON3SameFixed | 0x0000B800,
+ NEON_SHADD = NEON3SameFixed | 0x00000000,
+ NEON_SHSUB = NEON3SameFixed | 0x00002000,
+ NEON_SRHADD = NEON3SameFixed | 0x00001000,
+ NEON_CMEQ = NEON3SameFixed | NEON3SameUBit | 0x00008800,
+ NEON_CMGE = NEON3SameFixed | 0x00003800,
+ NEON_CMGT = NEON3SameFixed | 0x00003000,
+ NEON_CMHI = NEON3SameFixed | NEON3SameUBit | NEON_CMGT,
+ NEON_CMHS = NEON3SameFixed | NEON3SameUBit | NEON_CMGE,
+ NEON_CMTST = NEON3SameFixed | 0x00008800,
+ NEON_MLA = NEON3SameFixed | 0x00009000,
+ NEON_MLS = NEON3SameFixed | 0x20009000,
+ NEON_MUL = NEON3SameFixed | 0x00009800,
+ NEON_PMUL = NEON3SameFixed | 0x20009800,
+ NEON_SRSHL = NEON3SameFixed | 0x00005000,
+ NEON_SQSHL = NEON3SameFixed | 0x00004800,
+ NEON_SQRSHL = NEON3SameFixed | 0x00005800,
+ NEON_SSHL = NEON3SameFixed | 0x00004000,
+ NEON_SMAX = NEON3SameFixed | 0x00006000,
+ NEON_SMAXP = NEON3SameFixed | 0x0000A000,
+ NEON_SMIN = NEON3SameFixed | 0x00006800,
+ NEON_SMINP = NEON3SameFixed | 0x0000A800,
+ NEON_SABD = NEON3SameFixed | 0x00007000,
+ NEON_SABA = NEON3SameFixed | 0x00007800,
+ NEON_UABD = NEON3SameFixed | NEON3SameUBit | NEON_SABD,
+ NEON_UABA = NEON3SameFixed | NEON3SameUBit | NEON_SABA,
+ NEON_SQADD = NEON3SameFixed | 0x00000800,
+ NEON_SQSUB = NEON3SameFixed | 0x00002800,
+ NEON_SUB = NEON3SameFixed | NEON3SameUBit | 0x00008000,
+ NEON_UHADD = NEON3SameFixed | NEON3SameUBit | NEON_SHADD,
+ NEON_UHSUB = NEON3SameFixed | NEON3SameUBit | NEON_SHSUB,
+ NEON_URHADD = NEON3SameFixed | NEON3SameUBit | NEON_SRHADD,
+ NEON_UMAX = NEON3SameFixed | NEON3SameUBit | NEON_SMAX,
+ NEON_UMAXP = NEON3SameFixed | NEON3SameUBit | NEON_SMAXP,
+ NEON_UMIN = NEON3SameFixed | NEON3SameUBit | NEON_SMIN,
+ NEON_UMINP = NEON3SameFixed | NEON3SameUBit | NEON_SMINP,
+ NEON_URSHL = NEON3SameFixed | NEON3SameUBit | NEON_SRSHL,
+ NEON_UQADD = NEON3SameFixed | NEON3SameUBit | NEON_SQADD,
+ NEON_UQRSHL = NEON3SameFixed | NEON3SameUBit | NEON_SQRSHL,
+ NEON_UQSHL = NEON3SameFixed | NEON3SameUBit | NEON_SQSHL,
+ NEON_UQSUB = NEON3SameFixed | NEON3SameUBit | NEON_SQSUB,
+ NEON_USHL = NEON3SameFixed | NEON3SameUBit | NEON_SSHL,
+ NEON_SQDMULH = NEON3SameFixed | 0x0000B000,
+ NEON_SQRDMULH = NEON3SameFixed | 0x2000B000,
+
+ // NEON floating point instructions with three same-type operands.
+ NEON3SameFPFixed = NEON3SameFixed | 0x0000C000,
+ NEON3SameFPFMask = NEON3SameFMask | 0x0000C000,
+ NEON3SameFPMask = NEON3SameMask | 0x00800000,
+ NEON_FADD = NEON3SameFixed | 0x0000D000,
+ NEON_FSUB = NEON3SameFixed | 0x0080D000,
+ NEON_FMUL = NEON3SameFixed | 0x2000D800,
+ NEON_FDIV = NEON3SameFixed | 0x2000F800,
+ NEON_FMAX = NEON3SameFixed | 0x0000F000,
+ NEON_FMAXNM = NEON3SameFixed | 0x0000C000,
+ NEON_FMAXP = NEON3SameFixed | 0x2000F000,
+ NEON_FMAXNMP = NEON3SameFixed | 0x2000C000,
+ NEON_FMIN = NEON3SameFixed | 0x0080F000,
+ NEON_FMINNM = NEON3SameFixed | 0x0080C000,
+ NEON_FMINP = NEON3SameFixed | 0x2080F000,
+ NEON_FMINNMP = NEON3SameFixed | 0x2080C000,
+ NEON_FMLA = NEON3SameFixed | 0x0000C800,
+ NEON_FMLS = NEON3SameFixed | 0x0080C800,
+ NEON_FMULX = NEON3SameFixed | 0x0000D800,
+ NEON_FRECPS = NEON3SameFixed | 0x0000F800,
+ NEON_FRSQRTS = NEON3SameFixed | 0x0080F800,
+ NEON_FABD = NEON3SameFixed | 0x2080D000,
+ NEON_FADDP = NEON3SameFixed | 0x2000D000,
+ NEON_FCMEQ = NEON3SameFixed | 0x0000E000,
+ NEON_FCMGE = NEON3SameFixed | 0x2000E000,
+ NEON_FCMGT = NEON3SameFixed | 0x2080E000,
+ NEON_FACGE = NEON3SameFixed | 0x2000E800,
+ NEON_FACGT = NEON3SameFixed | 0x2080E800,
+
+ // NEON logical instructions with three same-type operands.
+ NEON3SameLogicalFixed = NEON3SameFixed | 0x00001800,
+ NEON3SameLogicalFMask = NEON3SameFMask | 0x0000F800,
+ NEON3SameLogicalMask = 0xBFE0FC00,
+ NEON3SameLogicalFormatMask = NEON_Q,
+ NEON_AND = NEON3SameLogicalFixed | 0x00000000,
+ NEON_ORR = NEON3SameLogicalFixed | 0x00A00000,
+ NEON_ORN = NEON3SameLogicalFixed | 0x00C00000,
+ NEON_EOR = NEON3SameLogicalFixed | 0x20000000,
+ NEON_BIC = NEON3SameLogicalFixed | 0x00400000,
+ NEON_BIF = NEON3SameLogicalFixed | 0x20C00000,
+ NEON_BIT = NEON3SameLogicalFixed | 0x20800000,
+ NEON_BSL = NEON3SameLogicalFixed | 0x20400000
+};
+
+// NEON instructions with three different-type operands.
+enum NEON3DifferentOp : uint32_t {
+ NEON3DifferentFixed = 0x0E200000,
+ NEON3DifferentFMask = 0x9F200C00,
+ NEON3DifferentMask = 0xFF20FC00,
+ NEON_ADDHN = NEON3DifferentFixed | 0x00004000,
+ NEON_ADDHN2 = NEON_ADDHN | NEON_Q,
+ NEON_PMULL = NEON3DifferentFixed | 0x0000E000,
+ NEON_PMULL2 = NEON_PMULL | NEON_Q,
+ NEON_RADDHN = NEON3DifferentFixed | 0x20004000,
+ NEON_RADDHN2 = NEON_RADDHN | NEON_Q,
+ NEON_RSUBHN = NEON3DifferentFixed | 0x20006000,
+ NEON_RSUBHN2 = NEON_RSUBHN | NEON_Q,
+ NEON_SABAL = NEON3DifferentFixed | 0x00005000,
+ NEON_SABAL2 = NEON_SABAL | NEON_Q,
+ NEON_SABDL = NEON3DifferentFixed | 0x00007000,
+ NEON_SABDL2 = NEON_SABDL | NEON_Q,
+ NEON_SADDL = NEON3DifferentFixed | 0x00000000,
+ NEON_SADDL2 = NEON_SADDL | NEON_Q,
+ NEON_SADDW = NEON3DifferentFixed | 0x00001000,
+ NEON_SADDW2 = NEON_SADDW | NEON_Q,
+ NEON_SMLAL = NEON3DifferentFixed | 0x00008000,
+ NEON_SMLAL2 = NEON_SMLAL | NEON_Q,
+ NEON_SMLSL = NEON3DifferentFixed | 0x0000A000,
+ NEON_SMLSL2 = NEON_SMLSL | NEON_Q,
+ NEON_SMULL = NEON3DifferentFixed | 0x0000C000,
+ NEON_SMULL2 = NEON_SMULL | NEON_Q,
+ NEON_SSUBL = NEON3DifferentFixed | 0x00002000,
+ NEON_SSUBL2 = NEON_SSUBL | NEON_Q,
+ NEON_SSUBW = NEON3DifferentFixed | 0x00003000,
+ NEON_SSUBW2 = NEON_SSUBW | NEON_Q,
+ NEON_SQDMLAL = NEON3DifferentFixed | 0x00009000,
+ NEON_SQDMLAL2 = NEON_SQDMLAL | NEON_Q,
+ NEON_SQDMLSL = NEON3DifferentFixed | 0x0000B000,
+ NEON_SQDMLSL2 = NEON_SQDMLSL | NEON_Q,
+ NEON_SQDMULL = NEON3DifferentFixed | 0x0000D000,
+ NEON_SQDMULL2 = NEON_SQDMULL | NEON_Q,
+ NEON_SUBHN = NEON3DifferentFixed | 0x00006000,
+ NEON_SUBHN2 = NEON_SUBHN | NEON_Q,
+ NEON_UABAL = NEON_SABAL | NEON3SameUBit,
+ NEON_UABAL2 = NEON_UABAL | NEON_Q,
+ NEON_UABDL = NEON_SABDL | NEON3SameUBit,
+ NEON_UABDL2 = NEON_UABDL | NEON_Q,
+ NEON_UADDL = NEON_SADDL | NEON3SameUBit,
+ NEON_UADDL2 = NEON_UADDL | NEON_Q,
+ NEON_UADDW = NEON_SADDW | NEON3SameUBit,
+ NEON_UADDW2 = NEON_UADDW | NEON_Q,
+ NEON_UMLAL = NEON_SMLAL | NEON3SameUBit,
+ NEON_UMLAL2 = NEON_UMLAL | NEON_Q,
+ NEON_UMLSL = NEON_SMLSL | NEON3SameUBit,
+ NEON_UMLSL2 = NEON_UMLSL | NEON_Q,
+ NEON_UMULL = NEON_SMULL | NEON3SameUBit,
+ NEON_UMULL2 = NEON_UMULL | NEON_Q,
+ NEON_USUBL = NEON_SSUBL | NEON3SameUBit,
+ NEON_USUBL2 = NEON_USUBL | NEON_Q,
+ NEON_USUBW = NEON_SSUBW | NEON3SameUBit,
+ NEON_USUBW2 = NEON_USUBW | NEON_Q
+};
+
+// NEON instructions operating across vectors.
+enum NEONAcrossLanesOp : uint32_t {
+ NEONAcrossLanesFixed = 0x0E300800,
+ NEONAcrossLanesFMask = 0x9F3E0C00,
+ NEONAcrossLanesMask = 0xBF3FFC00,
+ NEON_ADDV = NEONAcrossLanesFixed | 0x0001B000,
+ NEON_SADDLV = NEONAcrossLanesFixed | 0x00003000,
+ NEON_UADDLV = NEONAcrossLanesFixed | 0x20003000,
+ NEON_SMAXV = NEONAcrossLanesFixed | 0x0000A000,
+ NEON_SMINV = NEONAcrossLanesFixed | 0x0001A000,
+ NEON_UMAXV = NEONAcrossLanesFixed | 0x2000A000,
+ NEON_UMINV = NEONAcrossLanesFixed | 0x2001A000,
+
+ // NEON floating point across instructions.
+ NEONAcrossLanesFPFixed = NEONAcrossLanesFixed | 0x0000C000,
+ NEONAcrossLanesFPFMask = NEONAcrossLanesFMask | 0x0000C000,
+ NEONAcrossLanesFPMask = NEONAcrossLanesMask | 0x00800000,
+
+ NEON_FMAXV = NEONAcrossLanesFPFixed | 0x2000F000,
+ NEON_FMINV = NEONAcrossLanesFPFixed | 0x2080F000,
+ NEON_FMAXNMV = NEONAcrossLanesFPFixed | 0x2000C000,
+ NEON_FMINNMV = NEONAcrossLanesFPFixed | 0x2080C000
+};
+
+// NEON instructions with indexed element operand.
+enum NEONByIndexedElementOp : uint32_t {
+ NEONByIndexedElementFixed = 0x0F000000,
+ NEONByIndexedElementFMask = 0x9F000400,
+ NEONByIndexedElementMask = 0xBF00F400,
+ NEON_MUL_byelement = NEONByIndexedElementFixed | 0x00008000,
+ NEON_MLA_byelement = NEONByIndexedElementFixed | 0x20000000,
+ NEON_MLS_byelement = NEONByIndexedElementFixed | 0x20004000,
+ NEON_SMULL_byelement = NEONByIndexedElementFixed | 0x0000A000,
+ NEON_SMLAL_byelement = NEONByIndexedElementFixed | 0x00002000,
+ NEON_SMLSL_byelement = NEONByIndexedElementFixed | 0x00006000,
+ NEON_UMULL_byelement = NEONByIndexedElementFixed | 0x2000A000,
+ NEON_UMLAL_byelement = NEONByIndexedElementFixed | 0x20002000,
+ NEON_UMLSL_byelement = NEONByIndexedElementFixed | 0x20006000,
+ NEON_SQDMULL_byelement = NEONByIndexedElementFixed | 0x0000B000,
+ NEON_SQDMLAL_byelement = NEONByIndexedElementFixed | 0x00003000,
+ NEON_SQDMLSL_byelement = NEONByIndexedElementFixed | 0x00007000,
+ NEON_SQDMULH_byelement = NEONByIndexedElementFixed | 0x0000C000,
+ NEON_SQRDMULH_byelement = NEONByIndexedElementFixed | 0x0000D000,
+
+ // Floating point instructions.
+ NEONByIndexedElementFPFixed = NEONByIndexedElementFixed | 0x00800000,
+ NEONByIndexedElementFPMask = NEONByIndexedElementMask | 0x00800000,
+ NEON_FMLA_byelement = NEONByIndexedElementFPFixed | 0x00001000,
+ NEON_FMLS_byelement = NEONByIndexedElementFPFixed | 0x00005000,
+ NEON_FMUL_byelement = NEONByIndexedElementFPFixed | 0x00009000,
+ NEON_FMULX_byelement = NEONByIndexedElementFPFixed | 0x20009000
+};
+
+// NEON modified immediate.
+enum NEONModifiedImmediateOp : uint32_t {
+ NEONModifiedImmediateFixed = 0x0F000400,
+ NEONModifiedImmediateFMask = 0x9FF80400,
+ NEONModifiedImmediateOpBit = 0x20000000,
+ NEONModifiedImmediate_MOVI = NEONModifiedImmediateFixed | 0x00000000,
+ NEONModifiedImmediate_MVNI = NEONModifiedImmediateFixed | 0x20000000,
+ NEONModifiedImmediate_ORR = NEONModifiedImmediateFixed | 0x00001000,
+ NEONModifiedImmediate_BIC = NEONModifiedImmediateFixed | 0x20001000
+};
+
+// NEON extract.
+enum NEONExtractOp : uint32_t {
+ NEONExtractFixed = 0x2E000000,
+ NEONExtractFMask = 0xBF208400,
+ NEONExtractMask = 0xBFE08400,
+ NEON_EXT = NEONExtractFixed | 0x00000000
+};
+
+enum NEONLoadStoreMultiOp : uint32_t {
+ NEONLoadStoreMultiL = 0x00400000,
+ NEONLoadStoreMulti1_1v = 0x00007000,
+ NEONLoadStoreMulti1_2v = 0x0000A000,
+ NEONLoadStoreMulti1_3v = 0x00006000,
+ NEONLoadStoreMulti1_4v = 0x00002000,
+ NEONLoadStoreMulti2 = 0x00008000,
+ NEONLoadStoreMulti3 = 0x00004000,
+ NEONLoadStoreMulti4 = 0x00000000
+};
+
+// NEON load/store multiple structures.
+enum NEONLoadStoreMultiStructOp : uint32_t {
+ NEONLoadStoreMultiStructFixed = 0x0C000000,
+ NEONLoadStoreMultiStructFMask = 0xBFBF0000,
+ NEONLoadStoreMultiStructMask = 0xBFFFF000,
+ NEONLoadStoreMultiStructStore = NEONLoadStoreMultiStructFixed,
+ NEONLoadStoreMultiStructLoad =
+ NEONLoadStoreMultiStructFixed | NEONLoadStoreMultiL,
+ NEON_LD1_1v = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti1_1v,
+ NEON_LD1_2v = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti1_2v,
+ NEON_LD1_3v = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti1_3v,
+ NEON_LD1_4v = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti1_4v,
+ NEON_LD2 = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti2,
+ NEON_LD3 = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti3,
+ NEON_LD4 = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti4,
+ NEON_ST1_1v = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti1_1v,
+ NEON_ST1_2v = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti1_2v,
+ NEON_ST1_3v = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti1_3v,
+ NEON_ST1_4v = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti1_4v,
+ NEON_ST2 = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti2,
+ NEON_ST3 = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti3,
+ NEON_ST4 = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti4
+};
+
+// NEON load/store multiple structures with post-index addressing.
+enum NEONLoadStoreMultiStructPostIndexOp : uint32_t {
+ NEONLoadStoreMultiStructPostIndexFixed = 0x0C800000,
+ NEONLoadStoreMultiStructPostIndexFMask = 0xBFA00000,
+ NEONLoadStoreMultiStructPostIndexMask = 0xBFE0F000,
+ NEONLoadStoreMultiStructPostIndex = 0x00800000,
+ NEON_LD1_1v_post = NEON_LD1_1v | NEONLoadStoreMultiStructPostIndex,
+ NEON_LD1_2v_post = NEON_LD1_2v | NEONLoadStoreMultiStructPostIndex,
+ NEON_LD1_3v_post = NEON_LD1_3v | NEONLoadStoreMultiStructPostIndex,
+ NEON_LD1_4v_post = NEON_LD1_4v | NEONLoadStoreMultiStructPostIndex,
+ NEON_LD2_post = NEON_LD2 | NEONLoadStoreMultiStructPostIndex,
+ NEON_LD3_post = NEON_LD3 | NEONLoadStoreMultiStructPostIndex,
+ NEON_LD4_post = NEON_LD4 | NEONLoadStoreMultiStructPostIndex,
+ NEON_ST1_1v_post = NEON_ST1_1v | NEONLoadStoreMultiStructPostIndex,
+ NEON_ST1_2v_post = NEON_ST1_2v | NEONLoadStoreMultiStructPostIndex,
+ NEON_ST1_3v_post = NEON_ST1_3v | NEONLoadStoreMultiStructPostIndex,
+ NEON_ST1_4v_post = NEON_ST1_4v | NEONLoadStoreMultiStructPostIndex,
+ NEON_ST2_post = NEON_ST2 | NEONLoadStoreMultiStructPostIndex,
+ NEON_ST3_post = NEON_ST3 | NEONLoadStoreMultiStructPostIndex,
+ NEON_ST4_post = NEON_ST4 | NEONLoadStoreMultiStructPostIndex
+};
+
+enum NEONLoadStoreSingleOp : uint32_t {
+ NEONLoadStoreSingle1 = 0x00000000,
+ NEONLoadStoreSingle2 = 0x00200000,
+ NEONLoadStoreSingle3 = 0x00002000,
+ NEONLoadStoreSingle4 = 0x00202000,
+ NEONLoadStoreSingleL = 0x00400000,
+ NEONLoadStoreSingle_b = 0x00000000,
+ NEONLoadStoreSingle_h = 0x00004000,
+ NEONLoadStoreSingle_s = 0x00008000,
+ NEONLoadStoreSingle_d = 0x00008400,
+ NEONLoadStoreSingleAllLanes = 0x0000C000,
+ NEONLoadStoreSingleLenMask = 0x00202000
+};
+
+// NEON load/store single structure.
+enum NEONLoadStoreSingleStructOp : uint32_t {
+ NEONLoadStoreSingleStructFixed = 0x0D000000,
+ NEONLoadStoreSingleStructFMask = 0xBF9F0000,
+ NEONLoadStoreSingleStructMask = 0xBFFFE000,
+ NEONLoadStoreSingleStructStore = NEONLoadStoreSingleStructFixed,
+ NEONLoadStoreSingleStructLoad =
+ NEONLoadStoreSingleStructFixed | NEONLoadStoreSingleL,
+ NEONLoadStoreSingleStructLoad1 =
+ NEONLoadStoreSingle1 | NEONLoadStoreSingleStructLoad,
+ NEONLoadStoreSingleStructLoad2 =
+ NEONLoadStoreSingle2 | NEONLoadStoreSingleStructLoad,
+ NEONLoadStoreSingleStructLoad3 =
+ NEONLoadStoreSingle3 | NEONLoadStoreSingleStructLoad,
+ NEONLoadStoreSingleStructLoad4 =
+ NEONLoadStoreSingle4 | NEONLoadStoreSingleStructLoad,
+ NEONLoadStoreSingleStructStore1 =
+ NEONLoadStoreSingle1 | NEONLoadStoreSingleStructFixed,
+ NEONLoadStoreSingleStructStore2 =
+ NEONLoadStoreSingle2 | NEONLoadStoreSingleStructFixed,
+ NEONLoadStoreSingleStructStore3 =
+ NEONLoadStoreSingle3 | NEONLoadStoreSingleStructFixed,
+ NEONLoadStoreSingleStructStore4 =
+ NEONLoadStoreSingle4 | NEONLoadStoreSingleStructFixed,
+ NEON_LD1_b = NEONLoadStoreSingleStructLoad1 | NEONLoadStoreSingle_b,
+ NEON_LD1_h = NEONLoadStoreSingleStructLoad1 | NEONLoadStoreSingle_h,
+ NEON_LD1_s = NEONLoadStoreSingleStructLoad1 | NEONLoadStoreSingle_s,
+ NEON_LD1_d = NEONLoadStoreSingleStructLoad1 | NEONLoadStoreSingle_d,
+ NEON_LD1R = NEONLoadStoreSingleStructLoad1 | NEONLoadStoreSingleAllLanes,
+ NEON_ST1_b = NEONLoadStoreSingleStructStore1 | NEONLoadStoreSingle_b,
+ NEON_ST1_h = NEONLoadStoreSingleStructStore1 | NEONLoadStoreSingle_h,
+ NEON_ST1_s = NEONLoadStoreSingleStructStore1 | NEONLoadStoreSingle_s,
+ NEON_ST1_d = NEONLoadStoreSingleStructStore1 | NEONLoadStoreSingle_d,
+
+ NEON_LD2_b = NEONLoadStoreSingleStructLoad2 | NEONLoadStoreSingle_b,
+ NEON_LD2_h = NEONLoadStoreSingleStructLoad2 | NEONLoadStoreSingle_h,
+ NEON_LD2_s = NEONLoadStoreSingleStructLoad2 | NEONLoadStoreSingle_s,
+ NEON_LD2_d = NEONLoadStoreSingleStructLoad2 | NEONLoadStoreSingle_d,
+ NEON_LD2R = NEONLoadStoreSingleStructLoad2 | NEONLoadStoreSingleAllLanes,
+ NEON_ST2_b = NEONLoadStoreSingleStructStore2 | NEONLoadStoreSingle_b,
+ NEON_ST2_h = NEONLoadStoreSingleStructStore2 | NEONLoadStoreSingle_h,
+ NEON_ST2_s = NEONLoadStoreSingleStructStore2 | NEONLoadStoreSingle_s,
+ NEON_ST2_d = NEONLoadStoreSingleStructStore2 | NEONLoadStoreSingle_d,
+
+ NEON_LD3_b = NEONLoadStoreSingleStructLoad3 | NEONLoadStoreSingle_b,
+ NEON_LD3_h = NEONLoadStoreSingleStructLoad3 | NEONLoadStoreSingle_h,
+ NEON_LD3_s = NEONLoadStoreSingleStructLoad3 | NEONLoadStoreSingle_s,
+ NEON_LD3_d = NEONLoadStoreSingleStructLoad3 | NEONLoadStoreSingle_d,
+ NEON_LD3R = NEONLoadStoreSingleStructLoad3 | NEONLoadStoreSingleAllLanes,
+ NEON_ST3_b = NEONLoadStoreSingleStructStore3 | NEONLoadStoreSingle_b,
+ NEON_ST3_h = NEONLoadStoreSingleStructStore3 | NEONLoadStoreSingle_h,
+ NEON_ST3_s = NEONLoadStoreSingleStructStore3 | NEONLoadStoreSingle_s,
+ NEON_ST3_d = NEONLoadStoreSingleStructStore3 | NEONLoadStoreSingle_d,
+
+ NEON_LD4_b = NEONLoadStoreSingleStructLoad4 | NEONLoadStoreSingle_b,
+ NEON_LD4_h = NEONLoadStoreSingleStructLoad4 | NEONLoadStoreSingle_h,
+ NEON_LD4_s = NEONLoadStoreSingleStructLoad4 | NEONLoadStoreSingle_s,
+ NEON_LD4_d = NEONLoadStoreSingleStructLoad4 | NEONLoadStoreSingle_d,
+ NEON_LD4R = NEONLoadStoreSingleStructLoad4 | NEONLoadStoreSingleAllLanes,
+ NEON_ST4_b = NEONLoadStoreSingleStructStore4 | NEONLoadStoreSingle_b,
+ NEON_ST4_h = NEONLoadStoreSingleStructStore4 | NEONLoadStoreSingle_h,
+ NEON_ST4_s = NEONLoadStoreSingleStructStore4 | NEONLoadStoreSingle_s,
+ NEON_ST4_d = NEONLoadStoreSingleStructStore4 | NEONLoadStoreSingle_d
+};
+
+// NEON load/store single structure with post-index addressing.
+enum NEONLoadStoreSingleStructPostIndexOp : uint32_t {
+ NEONLoadStoreSingleStructPostIndexFixed = 0x0D800000,
+ NEONLoadStoreSingleStructPostIndexFMask = 0xBF800000,
+ NEONLoadStoreSingleStructPostIndexMask = 0xBFE0E000,
+ NEONLoadStoreSingleStructPostIndex = 0x00800000,
+ NEON_LD1_b_post = NEON_LD1_b | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD1_h_post = NEON_LD1_h | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD1_s_post = NEON_LD1_s | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD1_d_post = NEON_LD1_d | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD1R_post = NEON_LD1R | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST1_b_post = NEON_ST1_b | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST1_h_post = NEON_ST1_h | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST1_s_post = NEON_ST1_s | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST1_d_post = NEON_ST1_d | NEONLoadStoreSingleStructPostIndex,
+
+ NEON_LD2_b_post = NEON_LD2_b | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD2_h_post = NEON_LD2_h | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD2_s_post = NEON_LD2_s | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD2_d_post = NEON_LD2_d | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD2R_post = NEON_LD2R | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST2_b_post = NEON_ST2_b | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST2_h_post = NEON_ST2_h | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST2_s_post = NEON_ST2_s | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST2_d_post = NEON_ST2_d | NEONLoadStoreSingleStructPostIndex,
+
+ NEON_LD3_b_post = NEON_LD3_b | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD3_h_post = NEON_LD3_h | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD3_s_post = NEON_LD3_s | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD3_d_post = NEON_LD3_d | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD3R_post = NEON_LD3R | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST3_b_post = NEON_ST3_b | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST3_h_post = NEON_ST3_h | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST3_s_post = NEON_ST3_s | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST3_d_post = NEON_ST3_d | NEONLoadStoreSingleStructPostIndex,
+
+ NEON_LD4_b_post = NEON_LD4_b | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD4_h_post = NEON_LD4_h | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD4_s_post = NEON_LD4_s | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD4_d_post = NEON_LD4_d | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD4R_post = NEON_LD4R | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST4_b_post = NEON_ST4_b | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST4_h_post = NEON_ST4_h | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST4_s_post = NEON_ST4_s | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST4_d_post = NEON_ST4_d | NEONLoadStoreSingleStructPostIndex
+};
+
+// NEON register copy.
+enum NEONCopyOp : uint32_t {
+ NEONCopyFixed = 0x0E000400,
+ NEONCopyFMask = 0x9FE08400,
+ NEONCopyMask = 0x3FE08400,
+ NEONCopyInsElementMask = NEONCopyMask | 0x40000000,
+ NEONCopyInsGeneralMask = NEONCopyMask | 0x40007800,
+ NEONCopyDupElementMask = NEONCopyMask | 0x20007800,
+ NEONCopyDupGeneralMask = NEONCopyDupElementMask,
+ NEONCopyUmovMask = NEONCopyMask | 0x20007800,
+ NEONCopySmovMask = NEONCopyMask | 0x20007800,
+ NEON_INS_ELEMENT = NEONCopyFixed | 0x60000000,
+ NEON_INS_GENERAL = NEONCopyFixed | 0x40001800,
+ NEON_DUP_ELEMENT = NEONCopyFixed | 0x00000000,
+ NEON_DUP_GENERAL = NEONCopyFixed | 0x00000800,
+ NEON_SMOV = NEONCopyFixed | 0x00002800,
+ NEON_UMOV = NEONCopyFixed | 0x00003800
+};
+
+// NEON scalar instructions with indexed element operand.
+enum NEONScalarByIndexedElementOp : uint32_t {
+ NEONScalarByIndexedElementFixed = 0x5F000000,
+ NEONScalarByIndexedElementFMask = 0xDF000400,
+ NEONScalarByIndexedElementMask = 0xFF00F400,
+ NEON_SQDMLAL_byelement_scalar = NEON_Q | NEONScalar | NEON_SQDMLAL_byelement,
+ NEON_SQDMLSL_byelement_scalar = NEON_Q | NEONScalar | NEON_SQDMLSL_byelement,
+ NEON_SQDMULL_byelement_scalar = NEON_Q | NEONScalar | NEON_SQDMULL_byelement,
+ NEON_SQDMULH_byelement_scalar = NEON_Q | NEONScalar | NEON_SQDMULH_byelement,
+ NEON_SQRDMULH_byelement_scalar =
+ NEON_Q | NEONScalar | NEON_SQRDMULH_byelement,
+
+ // Floating point instructions.
+ NEONScalarByIndexedElementFPFixed =
+ NEONScalarByIndexedElementFixed | 0x00800000,
+ NEONScalarByIndexedElementFPMask =
+ NEONScalarByIndexedElementMask | 0x00800000,
+ NEON_FMLA_byelement_scalar = NEON_Q | NEONScalar | NEON_FMLA_byelement,
+ NEON_FMLS_byelement_scalar = NEON_Q | NEONScalar | NEON_FMLS_byelement,
+ NEON_FMUL_byelement_scalar = NEON_Q | NEONScalar | NEON_FMUL_byelement,
+ NEON_FMULX_byelement_scalar = NEON_Q | NEONScalar | NEON_FMULX_byelement
+};
+
+// NEON shift immediate.
+enum NEONShiftImmediateOp : uint32_t {
+ NEONShiftImmediateFixed = 0x0F000400,
+ NEONShiftImmediateFMask = 0x9F800400,
+ NEONShiftImmediateMask = 0xBF80FC00,
+ NEONShiftImmediateUBit = 0x20000000,
+ NEON_SHL = NEONShiftImmediateFixed | 0x00005000,
+ NEON_SSHLL = NEONShiftImmediateFixed | 0x0000A000,
+ NEON_USHLL = NEONShiftImmediateFixed | 0x2000A000,
+ NEON_SLI = NEONShiftImmediateFixed | 0x20005000,
+ NEON_SRI = NEONShiftImmediateFixed | 0x20004000,
+ NEON_SHRN = NEONShiftImmediateFixed | 0x00008000,
+ NEON_RSHRN = NEONShiftImmediateFixed | 0x00008800,
+ NEON_UQSHRN = NEONShiftImmediateFixed | 0x20009000,
+ NEON_UQRSHRN = NEONShiftImmediateFixed | 0x20009800,
+ NEON_SQSHRN = NEONShiftImmediateFixed | 0x00009000,
+ NEON_SQRSHRN = NEONShiftImmediateFixed | 0x00009800,
+ NEON_SQSHRUN = NEONShiftImmediateFixed | 0x20008000,
+ NEON_SQRSHRUN = NEONShiftImmediateFixed | 0x20008800,
+ NEON_SSHR = NEONShiftImmediateFixed | 0x00000000,
+ NEON_SRSHR = NEONShiftImmediateFixed | 0x00002000,
+ NEON_USHR = NEONShiftImmediateFixed | 0x20000000,
+ NEON_URSHR = NEONShiftImmediateFixed | 0x20002000,
+ NEON_SSRA = NEONShiftImmediateFixed | 0x00001000,
+ NEON_SRSRA = NEONShiftImmediateFixed | 0x00003000,
+ NEON_USRA = NEONShiftImmediateFixed | 0x20001000,
+ NEON_URSRA = NEONShiftImmediateFixed | 0x20003000,
+ NEON_SQSHLU = NEONShiftImmediateFixed | 0x20006000,
+ NEON_SCVTF_imm = NEONShiftImmediateFixed | 0x0000E000,
+ NEON_UCVTF_imm = NEONShiftImmediateFixed | 0x2000E000,
+ NEON_FCVTZS_imm = NEONShiftImmediateFixed | 0x0000F800,
+ NEON_FCVTZU_imm = NEONShiftImmediateFixed | 0x2000F800,
+ NEON_SQSHL_imm = NEONShiftImmediateFixed | 0x00007000,
+ NEON_UQSHL_imm = NEONShiftImmediateFixed | 0x20007000
+};
+
+// NEON scalar register copy.
+enum NEONScalarCopyOp : uint32_t {
+ NEONScalarCopyFixed = 0x5E000400,
+ NEONScalarCopyFMask = 0xDFE08400,
+ NEONScalarCopyMask = 0xFFE0FC00,
+ NEON_DUP_ELEMENT_scalar = NEON_Q | NEONScalar | NEON_DUP_ELEMENT
+};
+
+// NEON scalar pairwise instructions.
+enum NEONScalarPairwiseOp : uint32_t {
+ NEONScalarPairwiseFixed = 0x5E300800,
+ NEONScalarPairwiseFMask = 0xDF3E0C00,
+ NEONScalarPairwiseMask = 0xFFB1F800,
+ NEON_ADDP_scalar = NEONScalarPairwiseFixed | 0x0081B000,
+ NEON_FMAXNMP_scalar = NEONScalarPairwiseFixed | 0x2000C000,
+ NEON_FMINNMP_scalar = NEONScalarPairwiseFixed | 0x2080C000,
+ NEON_FADDP_scalar = NEONScalarPairwiseFixed | 0x2000D000,
+ NEON_FMAXP_scalar = NEONScalarPairwiseFixed | 0x2000F000,
+ NEON_FMINP_scalar = NEONScalarPairwiseFixed | 0x2080F000
+};
+
+// NEON scalar shift immediate.
+enum NEONScalarShiftImmediateOp : uint32_t {
+ NEONScalarShiftImmediateFixed = 0x5F000400,
+ NEONScalarShiftImmediateFMask = 0xDF800400,
+ NEONScalarShiftImmediateMask = 0xFF80FC00,
+ NEON_SHL_scalar = NEON_Q | NEONScalar | NEON_SHL,
+ NEON_SLI_scalar = NEON_Q | NEONScalar | NEON_SLI,
+ NEON_SRI_scalar = NEON_Q | NEONScalar | NEON_SRI,
+ NEON_SSHR_scalar = NEON_Q | NEONScalar | NEON_SSHR,
+ NEON_USHR_scalar = NEON_Q | NEONScalar | NEON_USHR,
+ NEON_SRSHR_scalar = NEON_Q | NEONScalar | NEON_SRSHR,
+ NEON_URSHR_scalar = NEON_Q | NEONScalar | NEON_URSHR,
+ NEON_SSRA_scalar = NEON_Q | NEONScalar | NEON_SSRA,
+ NEON_USRA_scalar = NEON_Q | NEONScalar | NEON_USRA,
+ NEON_SRSRA_scalar = NEON_Q | NEONScalar | NEON_SRSRA,
+ NEON_URSRA_scalar = NEON_Q | NEONScalar | NEON_URSRA,
+ NEON_UQSHRN_scalar = NEON_Q | NEONScalar | NEON_UQSHRN,
+ NEON_UQRSHRN_scalar = NEON_Q | NEONScalar | NEON_UQRSHRN,
+ NEON_SQSHRN_scalar = NEON_Q | NEONScalar | NEON_SQSHRN,
+ NEON_SQRSHRN_scalar = NEON_Q | NEONScalar | NEON_SQRSHRN,
+ NEON_SQSHRUN_scalar = NEON_Q | NEONScalar | NEON_SQSHRUN,
+ NEON_SQRSHRUN_scalar = NEON_Q | NEONScalar | NEON_SQRSHRUN,
+ NEON_SQSHLU_scalar = NEON_Q | NEONScalar | NEON_SQSHLU,
+ NEON_SQSHL_imm_scalar = NEON_Q | NEONScalar | NEON_SQSHL_imm,
+ NEON_UQSHL_imm_scalar = NEON_Q | NEONScalar | NEON_UQSHL_imm,
+ NEON_SCVTF_imm_scalar = NEON_Q | NEONScalar | NEON_SCVTF_imm,
+ NEON_UCVTF_imm_scalar = NEON_Q | NEONScalar | NEON_UCVTF_imm,
+ NEON_FCVTZS_imm_scalar = NEON_Q | NEONScalar | NEON_FCVTZS_imm,
+ NEON_FCVTZU_imm_scalar = NEON_Q | NEONScalar | NEON_FCVTZU_imm
+};
+
+// NEON table.
+enum NEONTableOp : uint32_t {
+ NEONTableFixed = 0x0E000000,
+ NEONTableFMask = 0xBF208C00,
+ NEONTableExt = 0x00001000,
+ NEONTableMask = 0xBF20FC00,
+ NEON_TBL_1v = NEONTableFixed | 0x00000000,
+ NEON_TBL_2v = NEONTableFixed | 0x00002000,
+ NEON_TBL_3v = NEONTableFixed | 0x00004000,
+ NEON_TBL_4v = NEONTableFixed | 0x00006000,
+ NEON_TBX_1v = NEON_TBL_1v | NEONTableExt,
+ NEON_TBX_2v = NEON_TBL_2v | NEONTableExt,
+ NEON_TBX_3v = NEON_TBL_3v | NEONTableExt,
+ NEON_TBX_4v = NEON_TBL_4v | NEONTableExt
+};
+
+// NEON perm.
+enum NEONPermOp : uint32_t {
+ NEONPermFixed = 0x0E000800,
+ NEONPermFMask = 0xBF208C00,
+ NEONPermMask = 0x3F20FC00,
+ NEON_UZP1 = NEONPermFixed | 0x00001000,
+ NEON_TRN1 = NEONPermFixed | 0x00002000,
+ NEON_ZIP1 = NEONPermFixed | 0x00003000,
+ NEON_UZP2 = NEONPermFixed | 0x00005000,
+ NEON_TRN2 = NEONPermFixed | 0x00006000,
+ NEON_ZIP2 = NEONPermFixed | 0x00007000
+};
+
+// NEON scalar instructions with two register operands.
+enum NEONScalar2RegMiscOp : uint32_t {
+ NEONScalar2RegMiscFixed = 0x5E200800,
+ NEONScalar2RegMiscFMask = 0xDF3E0C00,
+ NEONScalar2RegMiscMask = NEON_Q | NEONScalar | NEON2RegMiscMask,
+ NEON_CMGT_zero_scalar = NEON_Q | NEONScalar | NEON_CMGT_zero,
+ NEON_CMEQ_zero_scalar = NEON_Q | NEONScalar | NEON_CMEQ_zero,
+ NEON_CMLT_zero_scalar = NEON_Q | NEONScalar | NEON_CMLT_zero,
+ NEON_CMGE_zero_scalar = NEON_Q | NEONScalar | NEON_CMGE_zero,
+ NEON_CMLE_zero_scalar = NEON_Q | NEONScalar | NEON_CMLE_zero,
+ NEON_ABS_scalar = NEON_Q | NEONScalar | NEON_ABS,
+ NEON_SQABS_scalar = NEON_Q | NEONScalar | NEON_SQABS,
+ NEON_NEG_scalar = NEON_Q | NEONScalar | NEON_NEG,
+ NEON_SQNEG_scalar = NEON_Q | NEONScalar | NEON_SQNEG,
+ NEON_SQXTN_scalar = NEON_Q | NEONScalar | NEON_SQXTN,
+ NEON_UQXTN_scalar = NEON_Q | NEONScalar | NEON_UQXTN,
+ NEON_SQXTUN_scalar = NEON_Q | NEONScalar | NEON_SQXTUN,
+ NEON_SUQADD_scalar = NEON_Q | NEONScalar | NEON_SUQADD,
+ NEON_USQADD_scalar = NEON_Q | NEONScalar | NEON_USQADD,
+
+ NEONScalar2RegMiscOpcode = NEON2RegMiscOpcode,
+ NEON_NEG_scalar_opcode = NEON_NEG_scalar & NEONScalar2RegMiscOpcode,
+
+ NEONScalar2RegMiscFPMask = NEONScalar2RegMiscMask | 0x00800000,
+ NEON_FRSQRTE_scalar = NEON_Q | NEONScalar | NEON_FRSQRTE,
+ NEON_FRECPE_scalar = NEON_Q | NEONScalar | NEON_FRECPE,
+ NEON_SCVTF_scalar = NEON_Q | NEONScalar | NEON_SCVTF,
+ NEON_UCVTF_scalar = NEON_Q | NEONScalar | NEON_UCVTF,
+ NEON_FCMGT_zero_scalar = NEON_Q | NEONScalar | NEON_FCMGT_zero,
+ NEON_FCMEQ_zero_scalar = NEON_Q | NEONScalar | NEON_FCMEQ_zero,
+ NEON_FCMLT_zero_scalar = NEON_Q | NEONScalar | NEON_FCMLT_zero,
+ NEON_FCMGE_zero_scalar = NEON_Q | NEONScalar | NEON_FCMGE_zero,
+ NEON_FCMLE_zero_scalar = NEON_Q | NEONScalar | NEON_FCMLE_zero,
+ NEON_FRECPX_scalar = NEONScalar2RegMiscFixed | 0x0081F000,
+ NEON_FCVTNS_scalar = NEON_Q | NEONScalar | NEON_FCVTNS,
+ NEON_FCVTNU_scalar = NEON_Q | NEONScalar | NEON_FCVTNU,
+ NEON_FCVTPS_scalar = NEON_Q | NEONScalar | NEON_FCVTPS,
+ NEON_FCVTPU_scalar = NEON_Q | NEONScalar | NEON_FCVTPU,
+ NEON_FCVTMS_scalar = NEON_Q | NEONScalar | NEON_FCVTMS,
+ NEON_FCVTMU_scalar = NEON_Q | NEONScalar | NEON_FCVTMU,
+ NEON_FCVTZS_scalar = NEON_Q | NEONScalar | NEON_FCVTZS,
+ NEON_FCVTZU_scalar = NEON_Q | NEONScalar | NEON_FCVTZU,
+ NEON_FCVTAS_scalar = NEON_Q | NEONScalar | NEON_FCVTAS,
+ NEON_FCVTAU_scalar = NEON_Q | NEONScalar | NEON_FCVTAU,
+ NEON_FCVTXN_scalar = NEON_Q | NEONScalar | NEON_FCVTXN
+};
+
+// NEON scalar instructions with three same-type operands.
+enum NEONScalar3SameOp : uint32_t {
+ NEONScalar3SameFixed = 0x5E200400,
+ NEONScalar3SameFMask = 0xDF200400,
+ NEONScalar3SameMask = 0xFF20FC00,
+ NEON_ADD_scalar = NEON_Q | NEONScalar | NEON_ADD,
+ NEON_CMEQ_scalar = NEON_Q | NEONScalar | NEON_CMEQ,
+ NEON_CMGE_scalar = NEON_Q | NEONScalar | NEON_CMGE,
+ NEON_CMGT_scalar = NEON_Q | NEONScalar | NEON_CMGT,
+ NEON_CMHI_scalar = NEON_Q | NEONScalar | NEON_CMHI,
+ NEON_CMHS_scalar = NEON_Q | NEONScalar | NEON_CMHS,
+ NEON_CMTST_scalar = NEON_Q | NEONScalar | NEON_CMTST,
+ NEON_SUB_scalar = NEON_Q | NEONScalar | NEON_SUB,
+ NEON_UQADD_scalar = NEON_Q | NEONScalar | NEON_UQADD,
+ NEON_SQADD_scalar = NEON_Q | NEONScalar | NEON_SQADD,
+ NEON_UQSUB_scalar = NEON_Q | NEONScalar | NEON_UQSUB,
+ NEON_SQSUB_scalar = NEON_Q | NEONScalar | NEON_SQSUB,
+ NEON_USHL_scalar = NEON_Q | NEONScalar | NEON_USHL,
+ NEON_SSHL_scalar = NEON_Q | NEONScalar | NEON_SSHL,
+ NEON_UQSHL_scalar = NEON_Q | NEONScalar | NEON_UQSHL,
+ NEON_SQSHL_scalar = NEON_Q | NEONScalar | NEON_SQSHL,
+ NEON_URSHL_scalar = NEON_Q | NEONScalar | NEON_URSHL,
+ NEON_SRSHL_scalar = NEON_Q | NEONScalar | NEON_SRSHL,
+ NEON_UQRSHL_scalar = NEON_Q | NEONScalar | NEON_UQRSHL,
+ NEON_SQRSHL_scalar = NEON_Q | NEONScalar | NEON_SQRSHL,
+ NEON_SQDMULH_scalar = NEON_Q | NEONScalar | NEON_SQDMULH,
+ NEON_SQRDMULH_scalar = NEON_Q | NEONScalar | NEON_SQRDMULH,
+
+ // NEON floating point scalar instructions with three same-type operands.
+ NEONScalar3SameFPFixed = NEONScalar3SameFixed | 0x0000C000,
+ NEONScalar3SameFPFMask = NEONScalar3SameFMask | 0x0000C000,
+ NEONScalar3SameFPMask = NEONScalar3SameMask | 0x00800000,
+ NEON_FACGE_scalar = NEON_Q | NEONScalar | NEON_FACGE,
+ NEON_FACGT_scalar = NEON_Q | NEONScalar | NEON_FACGT,
+ NEON_FCMEQ_scalar = NEON_Q | NEONScalar | NEON_FCMEQ,
+ NEON_FCMGE_scalar = NEON_Q | NEONScalar | NEON_FCMGE,
+ NEON_FCMGT_scalar = NEON_Q | NEONScalar | NEON_FCMGT,
+ NEON_FMULX_scalar = NEON_Q | NEONScalar | NEON_FMULX,
+ NEON_FRECPS_scalar = NEON_Q | NEONScalar | NEON_FRECPS,
+ NEON_FRSQRTS_scalar = NEON_Q | NEONScalar | NEON_FRSQRTS,
+ NEON_FABD_scalar = NEON_Q | NEONScalar | NEON_FABD
+};
+
+// NEON scalar instructions with three different-type operands.
+enum NEONScalar3DiffOp : uint32_t {
+ NEONScalar3DiffFixed = 0x5E200000,
+ NEONScalar3DiffFMask = 0xDF200C00,
+ NEONScalar3DiffMask = NEON_Q | NEONScalar | NEON3DifferentMask,
+ NEON_SQDMLAL_scalar = NEON_Q | NEONScalar | NEON_SQDMLAL,
+ NEON_SQDMLSL_scalar = NEON_Q | NEONScalar | NEON_SQDMLSL,
+ NEON_SQDMULL_scalar = NEON_Q | NEONScalar | NEON_SQDMULL
+};
+
+// Unimplemented and unallocated instructions. These are defined to make fixed
+// bit assertion easier.
+enum UnimplementedOp : uint32_t {
+ UnimplementedFixed = 0x00000000,
+ UnimplementedFMask = 0x00000000
+};
+
+enum UnallocatedOp : uint32_t {
+ UnallocatedFixed = 0x00000000,
+ UnallocatedFMask = 0x00000000
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_ARM64_CONSTANTS_ARM64_H_
diff --git a/src/codegen/arm64/cpu-arm64.cc b/src/codegen/arm64/cpu-arm64.cc
new file mode 100644
index 0000000..d7bd483
--- /dev/null
+++ b/src/codegen/arm64/cpu-arm64.cc
@@ -0,0 +1,122 @@
+// Copyright 2013 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// CPU specific code for arm independent of OS goes here.
+
+#if V8_TARGET_ARCH_ARM64
+
+#include "src/codegen/arm64/utils-arm64.h"
+#include "src/codegen/cpu-features.h"
+
+#if V8_OS_MACOSX
+#include <libkern/OSCacheControl.h>
+#endif
+
+namespace v8 {
+namespace internal {
+
+class CacheLineSizes {
+ public:
+ CacheLineSizes() {
+#if !defined(V8_HOST_ARCH_ARM64) || defined(V8_OS_WIN) || defined(__APPLE__)
+ cache_type_register_ = 0;
+#else
+ // Copy the content of the cache type register to a core register.
+ __asm__ __volatile__("mrs %x[ctr], ctr_el0" // NOLINT
+ : [ctr] "=r"(cache_type_register_));
+#endif
+ }
+
+ uint32_t icache_line_size() const { return ExtractCacheLineSize(0); }
+ uint32_t dcache_line_size() const { return ExtractCacheLineSize(16); }
+
+ private:
+ uint32_t ExtractCacheLineSize(int cache_line_size_shift) const {
+ // The cache type register holds the size of cache lines in words as a
+ // power of two.
+ return 4 << ((cache_type_register_ >> cache_line_size_shift) & 0xF);
+ }
+
+ uint32_t cache_type_register_;
+};
+
+void CpuFeatures::FlushICache(void* address, size_t length) {
+#if defined(V8_HOST_ARCH_ARM64)
+#if defined(V8_OS_WIN)
+ ::FlushInstructionCache(GetCurrentProcess(), address, length);
+#elif defined(V8_OS_MACOSX)
+ sys_icache_invalidate(address, length);
+#else
+ // The code below assumes user space cache operations are allowed. The goal
+ // of this routine is to make sure the code generated is visible to the I
+ // side of the CPU.
+
+ uintptr_t start = reinterpret_cast<uintptr_t>(address);
+ // Sizes will be used to generate a mask big enough to cover a pointer.
+ CacheLineSizes sizes;
+ uintptr_t dsize = sizes.dcache_line_size();
+ uintptr_t isize = sizes.icache_line_size();
+ // Cache line sizes are always a power of 2.
+ DCHECK_EQ(CountSetBits(dsize, 64), 1);
+ DCHECK_EQ(CountSetBits(isize, 64), 1);
+ uintptr_t dstart = start & ~(dsize - 1);
+ uintptr_t istart = start & ~(isize - 1);
+ uintptr_t end = start + length;
+
+ __asm__ __volatile__( // NOLINT
+ // Clean every line of the D cache containing the
+ // target data.
+ "0: \n\t"
+ // dc : Data Cache maintenance
+ // c : Clean
+ // i : Invalidate
+ // va : by (Virtual) Address
+ // c : to the point of Coherency
+ // See ARM DDI 0406B page B2-12 for more information.
+ // We would prefer to use "cvau" (clean to the point of unification) here
+ // but we use "civac" to work around Cortex-A53 errata 819472, 826319,
+ // 827319 and 824069.
+ "dc civac, %[dline] \n\t"
+ "add %[dline], %[dline], %[dsize] \n\t"
+ "cmp %[dline], %[end] \n\t"
+ "b.lt 0b \n\t"
+ // Barrier to make sure the effect of the code above is visible to the
+ // rest of the world. dsb : Data Synchronisation Barrier
+ // ish : Inner SHareable domain
+ // The point of unification for an Inner Shareable shareability domain is
+ // the point by which the instruction and data caches of all the
+ // processors in that Inner Shareable shareability domain are guaranteed
+ // to see the same copy of a memory location. See ARM DDI 0406B page
+ // B2-12 for more information.
+ "dsb ish \n\t"
+ // Invalidate every line of the I cache containing the target data.
+ "1: \n\t"
+ // ic : instruction cache maintenance
+ // i : invalidate
+ // va : by address
+ // u : to the point of unification
+ "ic ivau, %[iline] \n\t"
+ "add %[iline], %[iline], %[isize] \n\t"
+ "cmp %[iline], %[end] \n\t"
+ "b.lt 1b \n\t"
+ // Barrier to make sure the effect of the code above is visible to the
+ // rest of the world.
+ "dsb ish \n\t"
+ // Barrier to ensure any prefetching which happened before this code is
+ // discarded.
+ // isb : Instruction Synchronisation Barrier
+ "isb \n\t"
+ : [dline] "+r"(dstart), [iline] "+r"(istart)
+ : [dsize] "r"(dsize), [isize] "r"(isize), [end] "r"(end)
+ // This code does not write to memory but without the dependency gcc might
+ // move this code before the code is generated.
+ : "cc", "memory"); // NOLINT
+#endif // V8_OS_WIN
+#endif // V8_HOST_ARCH_ARM64
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_ARM64
diff --git a/src/codegen/arm64/decoder-arm64-inl.h b/src/codegen/arm64/decoder-arm64-inl.h
new file mode 100644
index 0000000..1a7d483
--- /dev/null
+++ b/src/codegen/arm64/decoder-arm64-inl.h
@@ -0,0 +1,809 @@
+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_ARM64_DECODER_ARM64_INL_H_
+#define V8_CODEGEN_ARM64_DECODER_ARM64_INL_H_
+
+#include "src/codegen/arm64/decoder-arm64.h"
+#include "src/common/globals.h"
+#include "src/utils/utils.h"
+
+namespace v8 {
+namespace internal {
+
+// Top-level instruction decode function.
+template <typename V>
+void Decoder<V>::Decode(Instruction* instr) {
+ if (instr->Bits(28, 27) == 0) {
+ V::VisitUnallocated(instr);
+ } else {
+ switch (instr->Bits(27, 24)) {
+ // 0: PC relative addressing.
+ case 0x0:
+ DecodePCRelAddressing(instr);
+ break;
+
+ // 1: Add/sub immediate.
+ case 0x1:
+ DecodeAddSubImmediate(instr);
+ break;
+
+ // A: Logical shifted register.
+ // Add/sub with carry.
+ // Conditional compare register.
+ // Conditional compare immediate.
+ // Conditional select.
+ // Data processing 1 source.
+ // Data processing 2 source.
+ // B: Add/sub shifted register.
+ // Add/sub extended register.
+ // Data processing 3 source.
+ case 0xA:
+ case 0xB:
+ DecodeDataProcessing(instr);
+ break;
+
+ // 2: Logical immediate.
+ // Move wide immediate.
+ case 0x2:
+ DecodeLogical(instr);
+ break;
+
+ // 3: Bitfield.
+ // Extract.
+ case 0x3:
+ DecodeBitfieldExtract(instr);
+ break;
+
+ // 4: Unconditional branch immediate.
+ // Exception generation.
+ // Compare and branch immediate.
+ // 5: Compare and branch immediate.
+ // Conditional branch.
+ // System.
+ // 6,7: Unconditional branch.
+ // Test and branch immediate.
+ case 0x4:
+ case 0x5:
+ case 0x6:
+ case 0x7:
+ DecodeBranchSystemException(instr);
+ break;
+
+ // 8,9: Load/store register pair post-index.
+ // Load register literal.
+ // Load/store register unscaled immediate.
+ // Load/store register immediate post-index.
+ // Load/store register immediate pre-index.
+ // Load/store register offset.
+ // C,D: Load/store register pair offset.
+ // Load/store register pair pre-index.
+ // Load/store register unsigned immediate.
+ // Advanced SIMD.
+ case 0x8:
+ case 0x9:
+ case 0xC:
+ case 0xD:
+ DecodeLoadStore(instr);
+ break;
+
+ // E: FP fixed point conversion.
+ // FP integer conversion.
+ // FP data processing 1 source.
+ // FP compare.
+ // FP immediate.
+ // FP data processing 2 source.
+ // FP conditional compare.
+ // FP conditional select.
+ // Advanced SIMD.
+ // F: FP data processing 3 source.
+ // Advanced SIMD.
+ case 0xE:
+ case 0xF:
+ DecodeFP(instr);
+ break;
+ }
+ }
+}
+
+template <typename V>
+void Decoder<V>::DecodePCRelAddressing(Instruction* instr) {
+ DCHECK_EQ(0x0, instr->Bits(27, 24));
+ // We know bit 28 is set, as <b28:b27> = 0 is filtered out at the top level
+ // decode.
+ DCHECK_EQ(0x1, instr->Bit(28));
+ V::VisitPCRelAddressing(instr);
+}
+
+template <typename V>
+void Decoder<V>::DecodeBranchSystemException(Instruction* instr) {
+ DCHECK_EQ(0x4, instr->Bits(27, 24) & 0xC); // 0x4, 0x5, 0x6, 0x7
+
+ switch (instr->Bits(31, 29)) {
+ case 0:
+ case 4: {
+ V::VisitUnconditionalBranch(instr);
+ break;
+ }
+ case 1:
+ case 5: {
+ if (instr->Bit(25) == 0) {
+ V::VisitCompareBranch(instr);
+ } else {
+ V::VisitTestBranch(instr);
+ }
+ break;
+ }
+ case 2: {
+ if (instr->Bit(25) == 0) {
+ if ((instr->Bit(24) == 0x1) ||
+ (instr->Mask(0x01000010) == 0x00000010)) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitConditionalBranch(instr);
+ }
+ } else {
+ V::VisitUnallocated(instr);
+ }
+ break;
+ }
+ case 6: {
+ if (instr->Bit(25) == 0) {
+ if (instr->Bit(24) == 0) {
+ if ((instr->Bits(4, 2) != 0) ||
+ (instr->Mask(0x00E0001D) == 0x00200001) ||
+ (instr->Mask(0x00E0001D) == 0x00400001) ||
+ (instr->Mask(0x00E0001E) == 0x00200002) ||
+ (instr->Mask(0x00E0001E) == 0x00400002) ||
+ (instr->Mask(0x00E0001C) == 0x00600000) ||
+ (instr->Mask(0x00E0001C) == 0x00800000) ||
+ (instr->Mask(0x00E0001F) == 0x00A00000) ||
+ (instr->Mask(0x00C0001C) == 0x00C00000)) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitException(instr);
+ }
+ } else {
+ if (instr->Bits(23, 22) == 0) {
+ const Instr masked_003FF0E0 = instr->Mask(0x003FF0E0);
+ if ((instr->Bits(21, 19) == 0x4) ||
+ (masked_003FF0E0 == 0x00033000) ||
+ (masked_003FF0E0 == 0x003FF020) ||
+ (masked_003FF0E0 == 0x003FF060) ||
+ (masked_003FF0E0 == 0x003FF0E0) ||
+ (instr->Mask(0x00388000) == 0x00008000) ||
+ (instr->Mask(0x0038E000) == 0x00000000) ||
+ (instr->Mask(0x0039E000) == 0x00002000) ||
+ (instr->Mask(0x003AE000) == 0x00002000) ||
+ (instr->Mask(0x003CE000) == 0x00042000) ||
+ (instr->Mask(0x0038F000) == 0x00005000) ||
+ (instr->Mask(0x0038E000) == 0x00006000)) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitSystem(instr);
+ }
+ } else {
+ V::VisitUnallocated(instr);
+ }
+ }
+ } else {
+ if ((instr->Bit(24) == 0x1) || (instr->Bits(20, 16) != 0x1F) ||
+ (instr->Bits(15, 10) != 0) || (instr->Bits(4, 0) != 0) ||
+ (instr->Bits(24, 21) == 0x3) || (instr->Bits(24, 22) == 0x3)) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitUnconditionalBranchToRegister(instr);
+ }
+ }
+ break;
+ }
+ case 3:
+ case 7: {
+ V::VisitUnallocated(instr);
+ break;
+ }
+ }
+}
+
+template <typename V>
+void Decoder<V>::DecodeLoadStore(Instruction* instr) {
+ DCHECK_EQ(0x8, instr->Bits(27, 24) & 0xA); // 0x8, 0x9, 0xC, 0xD
+
+ if ((instr->Bit(28) == 0) && (instr->Bit(29) == 0) && (instr->Bit(26) == 1)) {
+ DecodeNEONLoadStore(instr);
+ return;
+ }
+
+ if (instr->Bit(24) == 0) {
+ if (instr->Bit(28) == 0) {
+ if (instr->Bit(29) == 0) {
+ if (instr->Bit(26) == 0) {
+ if (instr->Mask(0xA08000) == 0x800000 ||
+ instr->Mask(0xA00000) == 0xA00000) {
+ V::VisitUnallocated(instr);
+ } else if (instr->Mask(0x808000) == 0) {
+ // Load/Store exclusive without acquire/release are unimplemented.
+ V::VisitUnimplemented(instr);
+ } else {
+ V::VisitLoadStoreAcquireRelease(instr);
+ }
+ }
+ } else {
+ if ((instr->Bits(31, 30) == 0x3) ||
+ (instr->Mask(0xC4400000) == 0x40000000)) {
+ V::VisitUnallocated(instr);
+ } else {
+ if (instr->Bit(23) == 0) {
+ if (instr->Mask(0xC4400000) == 0xC0400000) {
+ V::VisitUnallocated(instr);
+ } else {
+ // Nontemporals are unimplemented.
+ V::VisitUnimplemented(instr);
+ }
+ } else {
+ V::VisitLoadStorePairPostIndex(instr);
+ }
+ }
+ }
+ } else {
+ if (instr->Bit(29) == 0) {
+ if (instr->Mask(0xC4000000) == 0xC4000000) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitLoadLiteral(instr);
+ }
+ } else {
+ if ((instr->Mask(0x84C00000) == 0x80C00000) ||
+ (instr->Mask(0x44800000) == 0x44800000) ||
+ (instr->Mask(0x84800000) == 0x84800000)) {
+ V::VisitUnallocated(instr);
+ } else {
+ if (instr->Bit(21) == 0) {
+ switch (instr->Bits(11, 10)) {
+ case 0: {
+ V::VisitLoadStoreUnscaledOffset(instr);
+ break;
+ }
+ case 1: {
+ if (instr->Mask(0xC4C00000) == 0xC0800000) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitLoadStorePostIndex(instr);
+ }
+ break;
+ }
+ case 2: {
+ // TODO(all): VisitLoadStoreRegisterOffsetUnpriv.
+ V::VisitUnimplemented(instr);
+ break;
+ }
+ case 3: {
+ if (instr->Mask(0xC4C00000) == 0xC0800000) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitLoadStorePreIndex(instr);
+ }
+ break;
+ }
+ }
+ } else {
+ if (instr->Bits(11, 10) == 0x2) {
+ if (instr->Bit(14) == 0) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitLoadStoreRegisterOffset(instr);
+ }
+ } else {
+ V::VisitUnallocated(instr);
+ }
+ }
+ }
+ }
+ }
+ } else {
+ if (instr->Bit(28) == 0) {
+ if (instr->Bit(29) == 0) {
+ V::VisitUnallocated(instr);
+ } else {
+ if ((instr->Bits(31, 30) == 0x3) ||
+ (instr->Mask(0xC4400000) == 0x40000000)) {
+ V::VisitUnallocated(instr);
+ } else {
+ if (instr->Bit(23) == 0) {
+ V::VisitLoadStorePairOffset(instr);
+ } else {
+ V::VisitLoadStorePairPreIndex(instr);
+ }
+ }
+ }
+ } else {
+ if (instr->Bit(29) == 0) {
+ V::VisitUnallocated(instr);
+ } else {
+ if ((instr->Mask(0x84C00000) == 0x80C00000) ||
+ (instr->Mask(0x44800000) == 0x44800000) ||
+ (instr->Mask(0x84800000) == 0x84800000)) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitLoadStoreUnsignedOffset(instr);
+ }
+ }
+ }
+ }
+}
+
+template <typename V>
+void Decoder<V>::DecodeLogical(Instruction* instr) {
+ DCHECK_EQ(0x2, instr->Bits(27, 24));
+
+ if (instr->Mask(0x80400000) == 0x00400000) {
+ V::VisitUnallocated(instr);
+ } else {
+ if (instr->Bit(23) == 0) {
+ V::VisitLogicalImmediate(instr);
+ } else {
+ if (instr->Bits(30, 29) == 0x1) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitMoveWideImmediate(instr);
+ }
+ }
+ }
+}
+
+template <typename V>
+void Decoder<V>::DecodeBitfieldExtract(Instruction* instr) {
+ DCHECK_EQ(0x3, instr->Bits(27, 24));
+
+ if ((instr->Mask(0x80400000) == 0x80000000) ||
+ (instr->Mask(0x80400000) == 0x00400000) ||
+ (instr->Mask(0x80008000) == 0x00008000)) {
+ V::VisitUnallocated(instr);
+ } else if (instr->Bit(23) == 0) {
+ if ((instr->Mask(0x80200000) == 0x00200000) ||
+ (instr->Mask(0x60000000) == 0x60000000)) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitBitfield(instr);
+ }
+ } else {
+ if ((instr->Mask(0x60200000) == 0x00200000) ||
+ (instr->Mask(0x60000000) != 0x00000000)) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitExtract(instr);
+ }
+ }
+}
+
+template <typename V>
+void Decoder<V>::DecodeAddSubImmediate(Instruction* instr) {
+ DCHECK_EQ(0x1, instr->Bits(27, 24));
+ if (instr->Bit(23) == 1) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitAddSubImmediate(instr);
+ }
+}
+
+template <typename V>
+void Decoder<V>::DecodeDataProcessing(Instruction* instr) {
+ DCHECK((instr->Bits(27, 24) == 0xA) || (instr->Bits(27, 24) == 0xB));
+
+ if (instr->Bit(24) == 0) {
+ if (instr->Bit(28) == 0) {
+ if (instr->Mask(0x80008000) == 0x00008000) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitLogicalShifted(instr);
+ }
+ } else {
+ switch (instr->Bits(23, 21)) {
+ case 0: {
+ if (instr->Mask(0x0000FC00) != 0) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitAddSubWithCarry(instr);
+ }
+ break;
+ }
+ case 2: {
+ if ((instr->Bit(29) == 0) || (instr->Mask(0x00000410) != 0)) {
+ V::VisitUnallocated(instr);
+ } else {
+ if (instr->Bit(11) == 0) {
+ V::VisitConditionalCompareRegister(instr);
+ } else {
+ V::VisitConditionalCompareImmediate(instr);
+ }
+ }
+ break;
+ }
+ case 4: {
+ if (instr->Mask(0x20000800) != 0x00000000) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitConditionalSelect(instr);
+ }
+ break;
+ }
+ case 6: {
+ if (instr->Bit(29) == 0x1) {
+ V::VisitUnallocated(instr);
+ } else {
+ if (instr->Bit(30) == 0) {
+ if ((instr->Bit(15) == 0x1) || (instr->Bits(15, 11) == 0) ||
+ (instr->Bits(15, 12) == 0x1) ||
+ (instr->Bits(15, 12) == 0x3) ||
+ (instr->Bits(15, 13) == 0x3) ||
+ (instr->Mask(0x8000EC00) == 0x00004C00) ||
+ (instr->Mask(0x8000E800) == 0x80004000) ||
+ (instr->Mask(0x8000E400) == 0x80004000)) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitDataProcessing2Source(instr);
+ }
+ } else {
+ if ((instr->Bit(13) == 1) || (instr->Bits(20, 16) != 0) ||
+ (instr->Bits(15, 14) != 0) ||
+ (instr->Mask(0xA01FFC00) == 0x00000C00) ||
+ (instr->Mask(0x201FF800) == 0x00001800)) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitDataProcessing1Source(instr);
+ }
+ }
+ break;
+ }
+ V8_FALLTHROUGH;
+ }
+ case 1:
+ case 3:
+ case 5:
+ case 7:
+ V::VisitUnallocated(instr);
+ break;
+ }
+ }
+ } else {
+ if (instr->Bit(28) == 0) {
+ if (instr->Bit(21) == 0) {
+ if ((instr->Bits(23, 22) == 0x3) ||
+ (instr->Mask(0x80008000) == 0x00008000)) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitAddSubShifted(instr);
+ }
+ } else {
+ if ((instr->Mask(0x00C00000) != 0x00000000) ||
+ (instr->Mask(0x00001400) == 0x00001400) ||
+ (instr->Mask(0x00001800) == 0x00001800)) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitAddSubExtended(instr);
+ }
+ }
+ } else {
+ if ((instr->Bit(30) == 0x1) || (instr->Bits(30, 29) == 0x1) ||
+ (instr->Mask(0xE0600000) == 0x00200000) ||
+ (instr->Mask(0xE0608000) == 0x00400000) ||
+ (instr->Mask(0x60608000) == 0x00408000) ||
+ (instr->Mask(0x60E00000) == 0x00E00000) ||
+ (instr->Mask(0x60E00000) == 0x00800000) ||
+ (instr->Mask(0x60E00000) == 0x00600000)) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitDataProcessing3Source(instr);
+ }
+ }
+ }
+}
+
+template <typename V>
+void Decoder<V>::DecodeFP(Instruction* instr) {
+ DCHECK((instr->Bits(27, 24) == 0xE) || (instr->Bits(27, 24) == 0xF));
+
+ if (instr->Bit(28) == 0) {
+ DecodeNEONVectorDataProcessing(instr);
+ } else {
+ if (instr->Bits(31, 30) == 0x3) {
+ V::VisitUnallocated(instr);
+ } else if (instr->Bits(31, 30) == 0x1) {
+ DecodeNEONScalarDataProcessing(instr);
+ } else {
+ if (instr->Bit(29) == 0) {
+ if (instr->Bit(24) == 0) {
+ if (instr->Bit(21) == 0) {
+ if ((instr->Bit(23) == 1) || (instr->Bit(18) == 1) ||
+ (instr->Mask(0x80008000) == 0x00000000) ||
+ (instr->Mask(0x000E0000) == 0x00000000) ||
+ (instr->Mask(0x000E0000) == 0x000A0000) ||
+ (instr->Mask(0x00160000) == 0x00000000) ||
+ (instr->Mask(0x00160000) == 0x00120000)) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitFPFixedPointConvert(instr);
+ }
+ } else {
+ if (instr->Bits(15, 10) == 32) {
+ V::VisitUnallocated(instr);
+ } else if (instr->Bits(15, 10) == 0) {
+ if ((instr->Bits(23, 22) == 0x3) ||
+ (instr->Mask(0x000E0000) == 0x000A0000) ||
+ (instr->Mask(0x000E0000) == 0x000C0000) ||
+ (instr->Mask(0x00160000) == 0x00120000) ||
+ (instr->Mask(0x00160000) == 0x00140000) ||
+ (instr->Mask(0x20C40000) == 0x00800000) ||
+ (instr->Mask(0x20C60000) == 0x00840000) ||
+ (instr->Mask(0xA0C60000) == 0x80060000) ||
+ (instr->Mask(0xA0C60000) == 0x00860000) ||
+ (instr->Mask(0xA0CE0000) == 0x80860000) ||
+ (instr->Mask(0xA0CE0000) == 0x804E0000) ||
+ (instr->Mask(0xA0CE0000) == 0x000E0000) ||
+ (instr->Mask(0xA0D60000) == 0x00160000) ||
+ (instr->Mask(0xA0D60000) == 0x80560000) ||
+ (instr->Mask(0xA0D60000) == 0x80960000)) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitFPIntegerConvert(instr);
+ }
+ } else if (instr->Bits(14, 10) == 16) {
+ const Instr masked_A0DF8000 = instr->Mask(0xA0DF8000);
+ if ((instr->Mask(0x80180000) != 0) ||
+ (masked_A0DF8000 == 0x00020000) ||
+ (masked_A0DF8000 == 0x00030000) ||
+ (masked_A0DF8000 == 0x00068000) ||
+ (masked_A0DF8000 == 0x00428000) ||
+ (masked_A0DF8000 == 0x00430000) ||
+ (masked_A0DF8000 == 0x00468000) ||
+ (instr->Mask(0xA0D80000) == 0x00800000) ||
+ (instr->Mask(0xA0DE0000) == 0x00C00000) ||
+ (instr->Mask(0xA0DF0000) == 0x00C30000) ||
+ (instr->Mask(0xA0DC0000) == 0x00C40000)) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitFPDataProcessing1Source(instr);
+ }
+ } else if (instr->Bits(13, 10) == 8) {
+ if ((instr->Bits(15, 14) != 0) || (instr->Bits(2, 0) != 0) ||
+ (instr->Mask(0x80800000) != 0x00000000)) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitFPCompare(instr);
+ }
+ } else if (instr->Bits(12, 10) == 4) {
+ if ((instr->Bits(9, 5) != 0) ||
+ (instr->Mask(0x80800000) != 0x00000000)) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitFPImmediate(instr);
+ }
+ } else {
+ if (instr->Mask(0x80800000) != 0x00000000) {
+ V::VisitUnallocated(instr);
+ } else {
+ switch (instr->Bits(11, 10)) {
+ case 1: {
+ V::VisitFPConditionalCompare(instr);
+ break;
+ }
+ case 2: {
+ if ((instr->Bits(15, 14) == 0x3) ||
+ (instr->Mask(0x00009000) == 0x00009000) ||
+ (instr->Mask(0x0000A000) == 0x0000A000)) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitFPDataProcessing2Source(instr);
+ }
+ break;
+ }
+ case 3: {
+ V::VisitFPConditionalSelect(instr);
+ break;
+ }
+ default:
+ UNREACHABLE();
+ }
+ }
+ }
+ }
+ } else {
+ // Bit 30 == 1 has been handled earlier.
+ DCHECK_EQ(0, instr->Bit(30));
+ if (instr->Mask(0xA0800000) != 0) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitFPDataProcessing3Source(instr);
+ }
+ }
+ } else {
+ V::VisitUnallocated(instr);
+ }
+ }
+ }
+}
+
+template <typename V>
+void Decoder<V>::DecodeNEONLoadStore(Instruction* instr) {
+ DCHECK_EQ(0x6, instr->Bits(29, 25));
+ if (instr->Bit(31) == 0) {
+ if ((instr->Bit(24) == 0) && (instr->Bit(21) == 1)) {
+ V::VisitUnallocated(instr);
+ return;
+ }
+
+ if (instr->Bit(23) == 0) {
+ if (instr->Bits(20, 16) == 0) {
+ if (instr->Bit(24) == 0) {
+ V::VisitNEONLoadStoreMultiStruct(instr);
+ } else {
+ V::VisitNEONLoadStoreSingleStruct(instr);
+ }
+ } else {
+ V::VisitUnallocated(instr);
+ }
+ } else {
+ if (instr->Bit(24) == 0) {
+ V::VisitNEONLoadStoreMultiStructPostIndex(instr);
+ } else {
+ V::VisitNEONLoadStoreSingleStructPostIndex(instr);
+ }
+ }
+ } else {
+ V::VisitUnallocated(instr);
+ }
+}
+
+template <typename V>
+void Decoder<V>::DecodeNEONVectorDataProcessing(Instruction* instr) {
+ DCHECK_EQ(0x7, instr->Bits(28, 25));
+ if (instr->Bit(31) == 0) {
+ if (instr->Bit(24) == 0) {
+ if (instr->Bit(21) == 0) {
+ if (instr->Bit(15) == 0) {
+ if (instr->Bit(10) == 0) {
+ if (instr->Bit(29) == 0) {
+ if (instr->Bit(11) == 0) {
+ V::VisitNEONTable(instr);
+ } else {
+ V::VisitNEONPerm(instr);
+ }
+ } else {
+ V::VisitNEONExtract(instr);
+ }
+ } else {
+ if (instr->Bits(23, 22) == 0) {
+ V::VisitNEONCopy(instr);
+ } else {
+ V::VisitUnallocated(instr);
+ }
+ }
+ } else {
+ V::VisitUnallocated(instr);
+ }
+ } else {
+ if (instr->Bit(10) == 0) {
+ if (instr->Bit(11) == 0) {
+ V::VisitNEON3Different(instr);
+ } else {
+ if (instr->Bits(18, 17) == 0) {
+ if (instr->Bit(20) == 0) {
+ if (instr->Bit(19) == 0) {
+ V::VisitNEON2RegMisc(instr);
+ } else {
+ if (instr->Bits(30, 29) == 0x2) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitUnallocated(instr);
+ }
+ }
+ } else {
+ if (instr->Bit(19) == 0) {
+ V::VisitNEONAcrossLanes(instr);
+ } else {
+ V::VisitUnallocated(instr);
+ }
+ }
+ } else {
+ V::VisitUnallocated(instr);
+ }
+ }
+ } else {
+ V::VisitNEON3Same(instr);
+ }
+ }
+ } else {
+ if (instr->Bit(10) == 0) {
+ V::VisitNEONByIndexedElement(instr);
+ } else {
+ if (instr->Bit(23) == 0) {
+ if (instr->Bits(22, 19) == 0) {
+ V::VisitNEONModifiedImmediate(instr);
+ } else {
+ V::VisitNEONShiftImmediate(instr);
+ }
+ } else {
+ V::VisitUnallocated(instr);
+ }
+ }
+ }
+ } else {
+ V::VisitUnallocated(instr);
+ }
+}
+
+template <typename V>
+void Decoder<V>::DecodeNEONScalarDataProcessing(Instruction* instr) {
+ DCHECK_EQ(0xF, instr->Bits(28, 25));
+ if (instr->Bit(24) == 0) {
+ if (instr->Bit(21) == 0) {
+ if (instr->Bit(15) == 0) {
+ if (instr->Bit(10) == 0) {
+ if (instr->Bit(29) == 0) {
+ if (instr->Bit(11) == 0) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitUnallocated(instr);
+ }
+ } else {
+ V::VisitUnallocated(instr);
+ }
+ } else {
+ if (instr->Bits(23, 22) == 0) {
+ V::VisitNEONScalarCopy(instr);
+ } else {
+ V::VisitUnallocated(instr);
+ }
+ }
+ } else {
+ V::VisitUnallocated(instr);
+ }
+ } else {
+ if (instr->Bit(10) == 0) {
+ if (instr->Bit(11) == 0) {
+ V::VisitNEONScalar3Diff(instr);
+ } else {
+ if (instr->Bits(18, 17) == 0) {
+ if (instr->Bit(20) == 0) {
+ if (instr->Bit(19) == 0) {
+ V::VisitNEONScalar2RegMisc(instr);
+ } else {
+ if (instr->Bit(29) == 0) {
+ V::VisitUnallocated(instr);
+ } else {
+ V::VisitUnallocated(instr);
+ }
+ }
+ } else {
+ if (instr->Bit(19) == 0) {
+ V::VisitNEONScalarPairwise(instr);
+ } else {
+ V::VisitUnallocated(instr);
+ }
+ }
+ } else {
+ V::VisitUnallocated(instr);
+ }
+ }
+ } else {
+ V::VisitNEONScalar3Same(instr);
+ }
+ }
+ } else {
+ if (instr->Bit(10) == 0) {
+ V::VisitNEONScalarByIndexedElement(instr);
+ } else {
+ if (instr->Bit(23) == 0) {
+ V::VisitNEONScalarShiftImmediate(instr);
+ } else {
+ V::VisitUnallocated(instr);
+ }
+ }
+ }
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_ARM64_DECODER_ARM64_INL_H_
diff --git a/src/codegen/arm64/decoder-arm64.cc b/src/codegen/arm64/decoder-arm64.cc
new file mode 100644
index 0000000..af8bbf9
--- /dev/null
+++ b/src/codegen/arm64/decoder-arm64.cc
@@ -0,0 +1,77 @@
+// Copyright 2013 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#if V8_TARGET_ARCH_ARM64
+
+#include "src/codegen/arm64/decoder-arm64.h"
+#include "src/common/globals.h"
+#include "src/utils/utils.h"
+
+namespace v8 {
+namespace internal {
+
+void DispatchingDecoderVisitor::AppendVisitor(DecoderVisitor* new_visitor) {
+ visitors_.remove(new_visitor);
+ visitors_.push_back(new_visitor);
+}
+
+void DispatchingDecoderVisitor::PrependVisitor(DecoderVisitor* new_visitor) {
+ visitors_.remove(new_visitor);
+ visitors_.push_front(new_visitor);
+}
+
+void DispatchingDecoderVisitor::InsertVisitorBefore(
+ DecoderVisitor* new_visitor, DecoderVisitor* registered_visitor) {
+ visitors_.remove(new_visitor);
+ std::list<DecoderVisitor*>::iterator it;
+ for (it = visitors_.begin(); it != visitors_.end(); it++) {
+ if (*it == registered_visitor) {
+ visitors_.insert(it, new_visitor);
+ return;
+ }
+ }
+ // We reached the end of the list. The last element must be
+ // registered_visitor.
+ DCHECK(*it == registered_visitor);
+ visitors_.insert(it, new_visitor);
+}
+
+void DispatchingDecoderVisitor::InsertVisitorAfter(
+ DecoderVisitor* new_visitor, DecoderVisitor* registered_visitor) {
+ visitors_.remove(new_visitor);
+ std::list<DecoderVisitor*>::iterator it;
+ for (it = visitors_.begin(); it != visitors_.end(); it++) {
+ if (*it == registered_visitor) {
+ it++;
+ visitors_.insert(it, new_visitor);
+ return;
+ }
+ }
+ // We reached the end of the list. The last element must be
+ // registered_visitor.
+ DCHECK(*it == registered_visitor);
+ visitors_.push_back(new_visitor);
+}
+
+void DispatchingDecoderVisitor::RemoveVisitor(DecoderVisitor* visitor) {
+ visitors_.remove(visitor);
+}
+
+#define DEFINE_VISITOR_CALLERS(A) \
+ void DispatchingDecoderVisitor::Visit##A(Instruction* instr) { \
+ if (!(instr->Mask(A##FMask) == A##Fixed)) { \
+ DCHECK(instr->Mask(A##FMask) == A##Fixed); \
+ } \
+ std::list<DecoderVisitor*>::iterator it; \
+ for (it = visitors_.begin(); it != visitors_.end(); it++) { \
+ (*it)->Visit##A(instr); \
+ } \
+ }
+VISITOR_LIST(DEFINE_VISITOR_CALLERS)
+#undef DEFINE_VISITOR_CALLERS
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_ARM64
diff --git a/src/codegen/arm64/decoder-arm64.h b/src/codegen/arm64/decoder-arm64.h
new file mode 100644
index 0000000..7621c51
--- /dev/null
+++ b/src/codegen/arm64/decoder-arm64.h
@@ -0,0 +1,213 @@
+// Copyright 2013 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_ARM64_DECODER_ARM64_H_
+#define V8_CODEGEN_ARM64_DECODER_ARM64_H_
+
+#include <list>
+
+#include "src/codegen/arm64/instructions-arm64.h"
+#include "src/common/globals.h"
+
+namespace v8 {
+namespace internal {
+
+// List macro containing all visitors needed by the decoder class.
+
+#define VISITOR_LIST(V) \
+ V(PCRelAddressing) \
+ V(AddSubImmediate) \
+ V(LogicalImmediate) \
+ V(MoveWideImmediate) \
+ V(Bitfield) \
+ V(Extract) \
+ V(UnconditionalBranch) \
+ V(UnconditionalBranchToRegister) \
+ V(CompareBranch) \
+ V(TestBranch) \
+ V(ConditionalBranch) \
+ V(System) \
+ V(Exception) \
+ V(LoadStorePairPostIndex) \
+ V(LoadStorePairOffset) \
+ V(LoadStorePairPreIndex) \
+ V(LoadLiteral) \
+ V(LoadStoreUnscaledOffset) \
+ V(LoadStorePostIndex) \
+ V(LoadStorePreIndex) \
+ V(LoadStoreRegisterOffset) \
+ V(LoadStoreUnsignedOffset) \
+ V(LoadStoreAcquireRelease) \
+ V(LogicalShifted) \
+ V(AddSubShifted) \
+ V(AddSubExtended) \
+ V(AddSubWithCarry) \
+ V(ConditionalCompareRegister) \
+ V(ConditionalCompareImmediate) \
+ V(ConditionalSelect) \
+ V(DataProcessing1Source) \
+ V(DataProcessing2Source) \
+ V(DataProcessing3Source) \
+ V(FPCompare) \
+ V(FPConditionalCompare) \
+ V(FPConditionalSelect) \
+ V(FPImmediate) \
+ V(FPDataProcessing1Source) \
+ V(FPDataProcessing2Source) \
+ V(FPDataProcessing3Source) \
+ V(FPIntegerConvert) \
+ V(FPFixedPointConvert) \
+ V(NEON2RegMisc) \
+ V(NEON3Different) \
+ V(NEON3Same) \
+ V(NEONAcrossLanes) \
+ V(NEONByIndexedElement) \
+ V(NEONCopy) \
+ V(NEONExtract) \
+ V(NEONLoadStoreMultiStruct) \
+ V(NEONLoadStoreMultiStructPostIndex) \
+ V(NEONLoadStoreSingleStruct) \
+ V(NEONLoadStoreSingleStructPostIndex) \
+ V(NEONModifiedImmediate) \
+ V(NEONScalar2RegMisc) \
+ V(NEONScalar3Diff) \
+ V(NEONScalar3Same) \
+ V(NEONScalarByIndexedElement) \
+ V(NEONScalarCopy) \
+ V(NEONScalarPairwise) \
+ V(NEONScalarShiftImmediate) \
+ V(NEONShiftImmediate) \
+ V(NEONTable) \
+ V(NEONPerm) \
+ V(Unallocated) \
+ V(Unimplemented)
+
+// The Visitor interface. Disassembler and simulator (and other tools)
+// must provide implementations for all of these functions.
+class V8_EXPORT_PRIVATE DecoderVisitor {
+ public:
+ virtual ~DecoderVisitor() {}
+
+#define DECLARE(A) virtual void Visit##A(Instruction* instr) = 0;
+ VISITOR_LIST(DECLARE)
+#undef DECLARE
+};
+
+// A visitor that dispatches to a list of visitors.
+class V8_EXPORT_PRIVATE DispatchingDecoderVisitor : public DecoderVisitor {
+ public:
+ DispatchingDecoderVisitor() {}
+ virtual ~DispatchingDecoderVisitor() {}
+
+ // Register a new visitor class with the decoder.
+ // Decode() will call the corresponding visitor method from all registered
+ // visitor classes when decoding reaches the leaf node of the instruction
+ // decode tree.
+ // Visitors are called in the order.
+ // A visitor can only be registered once.
+ // Registering an already registered visitor will update its position.
+ //
+ // d.AppendVisitor(V1);
+ // d.AppendVisitor(V2);
+ // d.PrependVisitor(V2); // Move V2 at the start of the list.
+ // d.InsertVisitorBefore(V3, V2);
+ // d.AppendVisitor(V4);
+ // d.AppendVisitor(V4); // No effect.
+ //
+ // d.Decode(i);
+ //
+ // will call in order visitor methods in V3, V2, V1, V4.
+ void AppendVisitor(DecoderVisitor* visitor);
+ void PrependVisitor(DecoderVisitor* visitor);
+ void InsertVisitorBefore(DecoderVisitor* new_visitor,
+ DecoderVisitor* registered_visitor);
+ void InsertVisitorAfter(DecoderVisitor* new_visitor,
+ DecoderVisitor* registered_visitor);
+
+ // Remove a previously registered visitor class from the list of visitors
+ // stored by the decoder.
+ void RemoveVisitor(DecoderVisitor* visitor);
+
+ void VisitNEONShiftImmediate(const Instruction* instr);
+
+#define DECLARE(A) void Visit##A(Instruction* instr);
+ VISITOR_LIST(DECLARE)
+#undef DECLARE
+
+ private:
+ // Visitors are registered in a list.
+ std::list<DecoderVisitor*> visitors_;
+};
+
+template <typename V>
+class Decoder : public V {
+ public:
+ Decoder() {}
+ virtual ~Decoder() {}
+
+ // Top-level instruction decoder function. Decodes an instruction and calls
+ // the visitor functions registered with the Decoder class.
+ virtual void Decode(Instruction* instr);
+
+ private:
+ // Decode the PC relative addressing instruction, and call the corresponding
+ // visitors.
+ // On entry, instruction bits 27:24 = 0x0.
+ void DecodePCRelAddressing(Instruction* instr);
+
+ // Decode the add/subtract immediate instruction, and call the corresponding
+ // visitors.
+ // On entry, instruction bits 27:24 = 0x1.
+ void DecodeAddSubImmediate(Instruction* instr);
+
+ // Decode the branch, system command, and exception generation parts of
+ // the instruction tree, and call the corresponding visitors.
+ // On entry, instruction bits 27:24 = {0x4, 0x5, 0x6, 0x7}.
+ void DecodeBranchSystemException(Instruction* instr);
+
+ // Decode the load and store parts of the instruction tree, and call
+ // the corresponding visitors.
+ // On entry, instruction bits 27:24 = {0x8, 0x9, 0xC, 0xD}.
+ void DecodeLoadStore(Instruction* instr);
+
+ // Decode the logical immediate and move wide immediate parts of the
+ // instruction tree, and call the corresponding visitors.
+ // On entry, instruction bits 27:24 = 0x2.
+ void DecodeLogical(Instruction* instr);
+
+ // Decode the bitfield and extraction parts of the instruction tree,
+ // and call the corresponding visitors.
+ // On entry, instruction bits 27:24 = 0x3.
+ void DecodeBitfieldExtract(Instruction* instr);
+
+ // Decode the data processing parts of the instruction tree, and call the
+ // corresponding visitors.
+ // On entry, instruction bits 27:24 = {0x1, 0xA, 0xB}.
+ void DecodeDataProcessing(Instruction* instr);
+
+ // Decode the floating point parts of the instruction tree, and call the
+ // corresponding visitors.
+ // On entry, instruction bits 27:24 = {0xE, 0xF}.
+ void DecodeFP(Instruction* instr);
+
+ // Decode the Advanced SIMD (NEON) load/store part of the instruction tree,
+ // and call the corresponding visitors.
+ // On entry, instruction bits 29:25 = 0x6.
+ void DecodeNEONLoadStore(Instruction* instr);
+
+ // Decode the Advanced SIMD (NEON) data processing part of the instruction
+ // tree, and call the corresponding visitors.
+ // On entry, instruction bits 27:25 = 0x7.
+ void DecodeNEONVectorDataProcessing(Instruction* instr);
+
+ // Decode the Advanced SIMD (NEON) scalar data processing part of the
+ // instruction tree, and call the corresponding visitors.
+ // On entry, instruction bits 28:25 = 0xF.
+ void DecodeNEONScalarDataProcessing(Instruction* instr);
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_ARM64_DECODER_ARM64_H_
diff --git a/src/codegen/arm64/instructions-arm64-constants.cc b/src/codegen/arm64/instructions-arm64-constants.cc
new file mode 100644
index 0000000..5c0d42a
--- /dev/null
+++ b/src/codegen/arm64/instructions-arm64-constants.cc
@@ -0,0 +1,58 @@
+// Copyright 2017 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <cstdint>
+#include "include/v8config.h"
+#include "src/base/macros.h"
+
+namespace v8 {
+namespace internal {
+
+// ISA constants. --------------------------------------------------------------
+
+// The following code initializes float/double variables with bit patterns
+// without using static initializers (which is surprisingly difficult in
+// C++). These variables are used by client code as extern float16,
+// extern float and extern double types, which works because (I think) the
+// linker ignores the types. This is kept in a separate source file to
+// avoid breaking jumbo builds.
+//
+// TODO(mostynb): replace these with std::numeric_limits constexpr's where
+// possible, and figure out how to replace *DefaultNaN with something clean,
+// then move this code back into instructions-arm64.cc with the same types
+// that client code uses.
+
+#if defined(V8_OS_WIN)
+extern "C" {
+#endif
+
+extern const uint16_t kFP16PositiveInfinity = 0x7C00;
+extern const uint16_t kFP16NegativeInfinity = 0xFC00;
+V8_EXPORT_PRIVATE extern const uint32_t kFP32PositiveInfinity = 0x7F800000;
+V8_EXPORT_PRIVATE extern const uint32_t kFP32NegativeInfinity = 0xFF800000;
+V8_EXPORT_PRIVATE extern const uint64_t kFP64PositiveInfinity =
+ 0x7FF0000000000000UL;
+V8_EXPORT_PRIVATE extern const uint64_t kFP64NegativeInfinity =
+ 0xFFF0000000000000UL;
+
+// This value is a signalling NaN as both a double and as a float (taking the
+// least-significant word).
+V8_EXPORT_PRIVATE extern const uint64_t kFP64SignallingNaN = 0x7FF000007F800001;
+V8_EXPORT_PRIVATE extern const uint32_t kFP32SignallingNaN = 0x7F800001;
+
+// A similar value, but as a quiet NaN.
+V8_EXPORT_PRIVATE extern const uint64_t kFP64QuietNaN = 0x7FF800007FC00001;
+V8_EXPORT_PRIVATE extern const uint32_t kFP32QuietNaN = 0x7FC00001;
+
+// The default NaN values (for FPCR.DN=1).
+V8_EXPORT_PRIVATE extern const uint64_t kFP64DefaultNaN = 0x7FF8000000000000UL;
+V8_EXPORT_PRIVATE extern const uint32_t kFP32DefaultNaN = 0x7FC00000;
+extern const uint16_t kFP16DefaultNaN = 0x7E00;
+
+#if defined(V8_OS_WIN)
+} // end of extern "C"
+#endif
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/arm64/instructions-arm64.cc b/src/codegen/arm64/instructions-arm64.cc
new file mode 100644
index 0000000..7d986f2
--- /dev/null
+++ b/src/codegen/arm64/instructions-arm64.cc
@@ -0,0 +1,439 @@
+// Copyright 2013 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#if V8_TARGET_ARCH_ARM64
+
+#include "src/codegen/arm64/instructions-arm64.h"
+#include "src/codegen/arm64/assembler-arm64-inl.h"
+
+namespace v8 {
+namespace internal {
+
+bool Instruction::IsLoad() const {
+ if (Mask(LoadStoreAnyFMask) != LoadStoreAnyFixed) {
+ return false;
+ }
+
+ if (Mask(LoadStorePairAnyFMask) == LoadStorePairAnyFixed) {
+ return Mask(LoadStorePairLBit) != 0;
+ } else {
+ LoadStoreOp op = static_cast<LoadStoreOp>(Mask(LoadStoreMask));
+ switch (op) {
+ case LDRB_w:
+ case LDRH_w:
+ case LDR_w:
+ case LDR_x:
+ case LDRSB_w:
+ case LDRSB_x:
+ case LDRSH_w:
+ case LDRSH_x:
+ case LDRSW_x:
+ case LDR_b:
+ case LDR_h:
+ case LDR_s:
+ case LDR_d:
+ case LDR_q:
+ return true;
+ default:
+ return false;
+ }
+ }
+}
+
+bool Instruction::IsStore() const {
+ if (Mask(LoadStoreAnyFMask) != LoadStoreAnyFixed) {
+ return false;
+ }
+
+ if (Mask(LoadStorePairAnyFMask) == LoadStorePairAnyFixed) {
+ return Mask(LoadStorePairLBit) == 0;
+ } else {
+ LoadStoreOp op = static_cast<LoadStoreOp>(Mask(LoadStoreMask));
+ switch (op) {
+ case STRB_w:
+ case STRH_w:
+ case STR_w:
+ case STR_x:
+ case STR_b:
+ case STR_h:
+ case STR_s:
+ case STR_d:
+ case STR_q:
+ return true;
+ default:
+ return false;
+ }
+ }
+}
+
+static uint64_t RotateRight(uint64_t value, unsigned int rotate,
+ unsigned int width) {
+ DCHECK_LE(width, 64);
+ rotate &= 63;
+ if (rotate == 0) return value;
+ return ((value & ((1ULL << rotate) - 1ULL)) << (width - rotate)) |
+ (value >> rotate);
+}
+
+static uint64_t RepeatBitsAcrossReg(unsigned reg_size, uint64_t value,
+ unsigned width) {
+ DCHECK((width == 2) || (width == 4) || (width == 8) || (width == 16) ||
+ (width == 32));
+ DCHECK((reg_size == kWRegSizeInBits) || (reg_size == kXRegSizeInBits));
+ uint64_t result = value & ((1ULL << width) - 1ULL);
+ for (unsigned i = width; i < reg_size; i *= 2) {
+ result |= (result << i);
+ }
+ return result;
+}
+
+// Logical immediates can't encode zero, so a return value of zero is used to
+// indicate a failure case. Specifically, where the constraints on imm_s are not
+// met.
+uint64_t Instruction::ImmLogical() {
+ unsigned reg_size = SixtyFourBits() ? kXRegSizeInBits : kWRegSizeInBits;
+ int32_t n = BitN();
+ int32_t imm_s = ImmSetBits();
+ int32_t imm_r = ImmRotate();
+
+ // An integer is constructed from the n, imm_s and imm_r bits according to
+ // the following table:
+ //
+ // N imms immr size S R
+ // 1 ssssss rrrrrr 64 UInt(ssssss) UInt(rrrrrr)
+ // 0 0sssss xrrrrr 32 UInt(sssss) UInt(rrrrr)
+ // 0 10ssss xxrrrr 16 UInt(ssss) UInt(rrrr)
+ // 0 110sss xxxrrr 8 UInt(sss) UInt(rrr)
+ // 0 1110ss xxxxrr 4 UInt(ss) UInt(rr)
+ // 0 11110s xxxxxr 2 UInt(s) UInt(r)
+ // (s bits must not be all set)
+ //
+ // A pattern is constructed of size bits, where the least significant S+1
+ // bits are set. The pattern is rotated right by R, and repeated across a
+ // 32 or 64-bit value, depending on destination register width.
+ //
+
+ if (n == 1) {
+ if (imm_s == 0x3F) {
+ return 0;
+ }
+ uint64_t bits = (1ULL << (imm_s + 1)) - 1;
+ return RotateRight(bits, imm_r, 64);
+ } else {
+ if ((imm_s >> 1) == 0x1F) {
+ return 0;
+ }
+ for (int width = 0x20; width >= 0x2; width >>= 1) {
+ if ((imm_s & width) == 0) {
+ int mask = width - 1;
+ if ((imm_s & mask) == mask) {
+ return 0;
+ }
+ uint64_t bits = (1ULL << ((imm_s & mask) + 1)) - 1;
+ return RepeatBitsAcrossReg(
+ reg_size, RotateRight(bits, imm_r & mask, width), width);
+ }
+ }
+ }
+ UNREACHABLE();
+}
+
+uint32_t Instruction::ImmNEONabcdefgh() const {
+ return ImmNEONabc() << 5 | ImmNEONdefgh();
+}
+
+float Instruction::ImmFP32() { return Imm8ToFP32(ImmFP()); }
+
+double Instruction::ImmFP64() { return Imm8ToFP64(ImmFP()); }
+
+float Instruction::ImmNEONFP32() const { return Imm8ToFP32(ImmNEONabcdefgh()); }
+
+double Instruction::ImmNEONFP64() const {
+ return Imm8ToFP64(ImmNEONabcdefgh());
+}
+
+unsigned CalcLSDataSize(LoadStoreOp op) {
+ DCHECK_EQ(static_cast<unsigned>(LSSize_offset + LSSize_width),
+ kInstrSize * 8);
+ unsigned size = static_cast<Instr>(op) >> LSSize_offset;
+ if ((op & LSVector_mask) != 0) {
+ // Vector register memory operations encode the access size in the "size"
+ // and "opc" fields.
+ if ((size == 0) && ((op & LSOpc_mask) >> LSOpc_offset) >= 2) {
+ size = kQRegSizeLog2;
+ }
+ }
+ return size;
+}
+
+unsigned CalcLSPairDataSize(LoadStorePairOp op) {
+ static_assert(kXRegSize == kDRegSize, "X and D registers must be same size.");
+ static_assert(kWRegSize == kSRegSize, "W and S registers must be same size.");
+ switch (op) {
+ case STP_q:
+ case LDP_q:
+ return kQRegSizeLog2;
+ case STP_x:
+ case LDP_x:
+ case STP_d:
+ case LDP_d:
+ return kXRegSizeLog2;
+ default:
+ return kWRegSizeLog2;
+ }
+}
+
+int64_t Instruction::ImmPCOffset() {
+ int64_t offset;
+ if (IsPCRelAddressing()) {
+ // PC-relative addressing. Only ADR is supported.
+ offset = ImmPCRel();
+ } else if (BranchType() != UnknownBranchType) {
+ // All PC-relative branches.
+ // Relative branch offsets are instruction-size-aligned.
+ offset = ImmBranch() * kInstrSize;
+ } else if (IsUnresolvedInternalReference()) {
+ // Internal references are always word-aligned.
+ offset = ImmUnresolvedInternalReference() * kInstrSize;
+ } else {
+ // Load literal (offset from PC).
+ DCHECK(IsLdrLiteral());
+ // The offset is always shifted by 2 bits, even for loads to 64-bits
+ // registers.
+ offset = ImmLLiteral() * kInstrSize;
+ }
+ return offset;
+}
+
+Instruction* Instruction::ImmPCOffsetTarget() {
+ return InstructionAtOffset(ImmPCOffset());
+}
+
+bool Instruction::IsValidImmPCOffset(ImmBranchType branch_type,
+ ptrdiff_t offset) {
+ DCHECK_EQ(offset % kInstrSize, 0);
+ return is_intn(offset / kInstrSize, ImmBranchRangeBitwidth(branch_type));
+}
+
+bool Instruction::IsTargetInImmPCOffsetRange(Instruction* target) {
+ return IsValidImmPCOffset(BranchType(), DistanceTo(target));
+}
+
+void Instruction::SetImmPCOffsetTarget(const AssemblerOptions& options,
+ Instruction* target) {
+ if (IsPCRelAddressing()) {
+ SetPCRelImmTarget(options, target);
+ } else if (BranchType() != UnknownBranchType) {
+ SetBranchImmTarget(target);
+ } else if (IsUnresolvedInternalReference()) {
+ SetUnresolvedInternalReferenceImmTarget(options, target);
+ } else {
+ // Load literal (offset from PC).
+ SetImmLLiteral(target);
+ }
+}
+
+void Instruction::SetPCRelImmTarget(const AssemblerOptions& options,
+ Instruction* target) {
+ // ADRP is not supported, so 'this' must point to an ADR instruction.
+ DCHECK(IsAdr());
+
+ ptrdiff_t target_offset = DistanceTo(target);
+ Instr imm;
+ if (Instruction::IsValidPCRelOffset(target_offset)) {
+ imm = Assembler::ImmPCRelAddress(static_cast<int>(target_offset));
+ SetInstructionBits(Mask(~ImmPCRel_mask) | imm);
+ } else {
+ PatchingAssembler patcher(options, reinterpret_cast<byte*>(this),
+ PatchingAssembler::kAdrFarPatchableNInstrs);
+ patcher.PatchAdrFar(target_offset);
+ }
+}
+
+void Instruction::SetBranchImmTarget(Instruction* target) {
+ DCHECK(IsAligned(DistanceTo(target), kInstrSize));
+ DCHECK(IsValidImmPCOffset(BranchType(), DistanceTo(target)));
+ int offset = static_cast<int>(DistanceTo(target) >> kInstrSizeLog2);
+ Instr branch_imm = 0;
+ uint32_t imm_mask = 0;
+ switch (BranchType()) {
+ case CondBranchType: {
+ branch_imm = Assembler::ImmCondBranch(offset);
+ imm_mask = ImmCondBranch_mask;
+ break;
+ }
+ case UncondBranchType: {
+ branch_imm = Assembler::ImmUncondBranch(offset);
+ imm_mask = ImmUncondBranch_mask;
+ break;
+ }
+ case CompareBranchType: {
+ branch_imm = Assembler::ImmCmpBranch(offset);
+ imm_mask = ImmCmpBranch_mask;
+ break;
+ }
+ case TestBranchType: {
+ branch_imm = Assembler::ImmTestBranch(offset);
+ imm_mask = ImmTestBranch_mask;
+ break;
+ }
+ default:
+ UNREACHABLE();
+ }
+ SetInstructionBits(Mask(~imm_mask) | branch_imm);
+}
+
+void Instruction::SetUnresolvedInternalReferenceImmTarget(
+ const AssemblerOptions& options, Instruction* target) {
+ DCHECK(IsUnresolvedInternalReference());
+ DCHECK(IsAligned(DistanceTo(target), kInstrSize));
+ DCHECK(is_int32(DistanceTo(target) >> kInstrSizeLog2));
+ int32_t target_offset =
+ static_cast<int32_t>(DistanceTo(target) >> kInstrSizeLog2);
+ uint32_t high16 = unsigned_bitextract_32(31, 16, target_offset);
+ uint32_t low16 = unsigned_bitextract_32(15, 0, target_offset);
+
+ PatchingAssembler patcher(options, reinterpret_cast<byte*>(this), 2);
+ patcher.brk(high16);
+ patcher.brk(low16);
+}
+
+void Instruction::SetImmLLiteral(Instruction* source) {
+ DCHECK(IsLdrLiteral());
+ DCHECK(IsAligned(DistanceTo(source), kInstrSize));
+ DCHECK(Assembler::IsImmLLiteral(DistanceTo(source)));
+ Instr imm = Assembler::ImmLLiteral(
+ static_cast<int>(DistanceTo(source) >> kLoadLiteralScaleLog2));
+ Instr mask = ImmLLiteral_mask;
+
+ SetInstructionBits(Mask(~mask) | imm);
+}
+
+NEONFormatDecoder::NEONFormatDecoder(const Instruction* instr) {
+ instrbits_ = instr->InstructionBits();
+ SetFormatMaps(IntegerFormatMap());
+}
+
+NEONFormatDecoder::NEONFormatDecoder(const Instruction* instr,
+ const NEONFormatMap* format) {
+ instrbits_ = instr->InstructionBits();
+ SetFormatMaps(format);
+}
+
+NEONFormatDecoder::NEONFormatDecoder(const Instruction* instr,
+ const NEONFormatMap* format0,
+ const NEONFormatMap* format1) {
+ instrbits_ = instr->InstructionBits();
+ SetFormatMaps(format0, format1);
+}
+
+NEONFormatDecoder::NEONFormatDecoder(const Instruction* instr,
+ const NEONFormatMap* format0,
+ const NEONFormatMap* format1,
+ const NEONFormatMap* format2) {
+ instrbits_ = instr->InstructionBits();
+ SetFormatMaps(format0, format1, format2);
+}
+
+void NEONFormatDecoder::SetFormatMaps(const NEONFormatMap* format0,
+ const NEONFormatMap* format1,
+ const NEONFormatMap* format2) {
+ DCHECK_NOT_NULL(format0);
+ formats_[0] = format0;
+ formats_[1] = (format1 == nullptr) ? formats_[0] : format1;
+ formats_[2] = (format2 == nullptr) ? formats_[1] : format2;
+ // Support four parameters form (e.i. ld4r)
+ // to avoid using positional arguments in DisassemblingDecoder.
+ // See: https://crbug.com/v8/10365
+ formats_[3] = formats_[2];
+}
+
+void NEONFormatDecoder::SetFormatMap(unsigned index,
+ const NEONFormatMap* format) {
+ DCHECK_LT(index, arraysize(formats_));
+ DCHECK_NOT_NULL(format);
+ formats_[index] = format;
+}
+
+const char* NEONFormatDecoder::SubstitutePlaceholders(const char* string) {
+ return Substitute(string, kPlaceholder, kPlaceholder, kPlaceholder,
+ kPlaceholder);
+}
+
+const char* NEONFormatDecoder::Substitute(const char* string,
+ SubstitutionMode mode0,
+ SubstitutionMode mode1,
+ SubstitutionMode mode2,
+ SubstitutionMode mode3) {
+ snprintf(form_buffer_, sizeof(form_buffer_), string, GetSubstitute(0, mode0),
+ GetSubstitute(1, mode1), GetSubstitute(2, mode2),
+ GetSubstitute(3, mode3));
+ return form_buffer_;
+}
+
+const char* NEONFormatDecoder::Mnemonic(const char* mnemonic) {
+ if ((instrbits_ & NEON_Q) != 0) {
+ snprintf(mne_buffer_, sizeof(mne_buffer_), "%s2", mnemonic);
+ return mne_buffer_;
+ }
+ return mnemonic;
+}
+
+VectorFormat NEONFormatDecoder::GetVectorFormat(int format_index) {
+ return GetVectorFormat(formats_[format_index]);
+}
+
+VectorFormat NEONFormatDecoder::GetVectorFormat(
+ const NEONFormatMap* format_map) {
+ static const VectorFormat vform[] = {
+ kFormatUndefined, kFormat8B, kFormat16B, kFormat4H, kFormat8H,
+ kFormat2S, kFormat4S, kFormat1D, kFormat2D, kFormatB,
+ kFormatH, kFormatS, kFormatD};
+ DCHECK_LT(GetNEONFormat(format_map), arraysize(vform));
+ return vform[GetNEONFormat(format_map)];
+}
+
+const char* NEONFormatDecoder::GetSubstitute(int index, SubstitutionMode mode) {
+ if (mode == kFormat) {
+ return NEONFormatAsString(GetNEONFormat(formats_[index]));
+ }
+ DCHECK_EQ(mode, kPlaceholder);
+ return NEONFormatAsPlaceholder(GetNEONFormat(formats_[index]));
+}
+
+NEONFormat NEONFormatDecoder::GetNEONFormat(const NEONFormatMap* format_map) {
+ return format_map->map[PickBits(format_map->bits)];
+}
+
+const char* NEONFormatDecoder::NEONFormatAsString(NEONFormat format) {
+ static const char* formats[] = {"undefined", "8b", "16b", "4h", "8h",
+ "2s", "4s", "1d", "2d", "b",
+ "h", "s", "d"};
+ DCHECK_LT(format, arraysize(formats));
+ return formats[format];
+}
+
+const char* NEONFormatDecoder::NEONFormatAsPlaceholder(NEONFormat format) {
+ DCHECK((format == NF_B) || (format == NF_H) || (format == NF_S) ||
+ (format == NF_D) || (format == NF_UNDEF));
+ static const char* formats[] = {
+ "undefined", "undefined", "undefined", "undefined", "undefined",
+ "undefined", "undefined", "undefined", "undefined", "'B",
+ "'H", "'S", "'D"};
+ return formats[format];
+}
+
+uint8_t NEONFormatDecoder::PickBits(const uint8_t bits[]) {
+ uint8_t result = 0;
+ for (unsigned b = 0; b < kNEONFormatMaxBits; b++) {
+ if (bits[b] == 0) break;
+ result <<= 1;
+ result |= ((instrbits_ & (1 << bits[b])) == 0) ? 0 : 1;
+ }
+ return result;
+}
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_ARM64
diff --git a/src/codegen/arm64/instructions-arm64.h b/src/codegen/arm64/instructions-arm64.h
new file mode 100644
index 0000000..b8335e0
--- /dev/null
+++ b/src/codegen/arm64/instructions-arm64.h
@@ -0,0 +1,749 @@
+// Copyright 2013 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_ARM64_INSTRUCTIONS_ARM64_H_
+#define V8_CODEGEN_ARM64_INSTRUCTIONS_ARM64_H_
+
+#include "src/base/memory.h"
+#include "src/codegen/arm64/constants-arm64.h"
+#include "src/codegen/arm64/register-arm64.h"
+#include "src/codegen/arm64/utils-arm64.h"
+#include "src/common/globals.h"
+#include "src/utils/utils.h"
+
+namespace v8 {
+namespace internal {
+
+struct AssemblerOptions;
+
+// ISA constants. --------------------------------------------------------------
+
+using Instr = uint32_t;
+
+#if defined(V8_OS_WIN)
+extern "C" {
+#endif
+
+extern const float16 kFP16PositiveInfinity;
+extern const float16 kFP16NegativeInfinity;
+V8_EXPORT_PRIVATE extern const float kFP32PositiveInfinity;
+V8_EXPORT_PRIVATE extern const float kFP32NegativeInfinity;
+V8_EXPORT_PRIVATE extern const double kFP64PositiveInfinity;
+V8_EXPORT_PRIVATE extern const double kFP64NegativeInfinity;
+
+// This value is a signalling NaN as both a double and as a float (taking the
+// least-significant word).
+V8_EXPORT_PRIVATE extern const double kFP64SignallingNaN;
+V8_EXPORT_PRIVATE extern const float kFP32SignallingNaN;
+
+// A similar value, but as a quiet NaN.
+V8_EXPORT_PRIVATE extern const double kFP64QuietNaN;
+V8_EXPORT_PRIVATE extern const float kFP32QuietNaN;
+
+// The default NaN values (for FPCR.DN=1).
+V8_EXPORT_PRIVATE extern const double kFP64DefaultNaN;
+V8_EXPORT_PRIVATE extern const float kFP32DefaultNaN;
+extern const float16 kFP16DefaultNaN;
+
+#if defined(V8_OS_WIN)
+} // end of extern "C"
+#endif
+
+unsigned CalcLSDataSize(LoadStoreOp op);
+unsigned CalcLSPairDataSize(LoadStorePairOp op);
+
+enum ImmBranchType {
+ UnknownBranchType = 0,
+ CondBranchType = 1,
+ UncondBranchType = 2,
+ CompareBranchType = 3,
+ TestBranchType = 4
+};
+
+enum AddrMode { Offset, PreIndex, PostIndex };
+
+enum FPRounding {
+ // The first four values are encodable directly by FPCR<RMode>.
+ FPTieEven = 0x0,
+ FPPositiveInfinity = 0x1,
+ FPNegativeInfinity = 0x2,
+ FPZero = 0x3,
+
+ // The final rounding modes are only available when explicitly specified by
+ // the instruction (such as with fcvta). They cannot be set in FPCR.
+ FPTieAway,
+ FPRoundOdd
+};
+
+enum Reg31Mode { Reg31IsStackPointer, Reg31IsZeroRegister };
+
+// Instructions. ---------------------------------------------------------------
+
+class Instruction {
+ public:
+ V8_INLINE Instr InstructionBits() const {
+ // Usually this is aligned, but when de/serializing that's not guaranteed.
+ return base::ReadUnalignedValue<Instr>(reinterpret_cast<Address>(this));
+ }
+
+ V8_INLINE void SetInstructionBits(Instr new_instr) {
+ // Usually this is aligned, but when de/serializing that's not guaranteed.
+ base::WriteUnalignedValue(reinterpret_cast<Address>(this), new_instr);
+ }
+
+ int Bit(int pos) const { return (InstructionBits() >> pos) & 1; }
+
+ uint32_t Bits(int msb, int lsb) const {
+ return unsigned_bitextract_32(msb, lsb, InstructionBits());
+ }
+
+ int32_t SignedBits(int msb, int lsb) const {
+ // Usually this is aligned, but when de/serializing that's not guaranteed.
+ int32_t bits =
+ base::ReadUnalignedValue<int32_t>(reinterpret_cast<Address>(this));
+ return signed_bitextract_32(msb, lsb, bits);
+ }
+
+ Instr Mask(uint32_t mask) const { return InstructionBits() & mask; }
+
+ V8_INLINE const Instruction* following(int count = 1) const {
+ return InstructionAtOffset(count * static_cast<int>(kInstrSize));
+ }
+
+ V8_INLINE Instruction* following(int count = 1) {
+ return InstructionAtOffset(count * static_cast<int>(kInstrSize));
+ }
+
+ V8_INLINE const Instruction* preceding(int count = 1) const {
+ return following(-count);
+ }
+
+ V8_INLINE Instruction* preceding(int count = 1) { return following(-count); }
+
+#define DEFINE_GETTER(Name, HighBit, LowBit, Func) \
+ int32_t Name() const { return Func(HighBit, LowBit); }
+ INSTRUCTION_FIELDS_LIST(DEFINE_GETTER)
+#undef DEFINE_GETTER
+
+ // ImmPCRel is a compound field (not present in INSTRUCTION_FIELDS_LIST),
+ // formed from ImmPCRelLo and ImmPCRelHi.
+ int ImmPCRel() const {
+ DCHECK(IsPCRelAddressing());
+ int offset = (static_cast<uint32_t>(ImmPCRelHi()) << ImmPCRelLo_width) |
+ ImmPCRelLo();
+ int width = ImmPCRelLo_width + ImmPCRelHi_width;
+ return signed_bitextract_32(width - 1, 0, offset);
+ }
+
+ uint64_t ImmLogical();
+ unsigned ImmNEONabcdefgh() const;
+ float ImmFP32();
+ double ImmFP64();
+ float ImmNEONFP32() const;
+ double ImmNEONFP64() const;
+
+ unsigned SizeLS() const {
+ return CalcLSDataSize(static_cast<LoadStoreOp>(Mask(LoadStoreMask)));
+ }
+
+ unsigned SizeLSPair() const {
+ return CalcLSPairDataSize(
+ static_cast<LoadStorePairOp>(Mask(LoadStorePairMask)));
+ }
+
+ int NEONLSIndex(int access_size_shift) const {
+ int q = NEONQ();
+ int s = NEONS();
+ int size = NEONLSSize();
+ int index = (q << 3) | (s << 2) | size;
+ return index >> access_size_shift;
+ }
+
+ // Helpers.
+ bool IsCondBranchImm() const {
+ return Mask(ConditionalBranchFMask) == ConditionalBranchFixed;
+ }
+
+ bool IsUncondBranchImm() const {
+ return Mask(UnconditionalBranchFMask) == UnconditionalBranchFixed;
+ }
+
+ bool IsCompareBranch() const {
+ return Mask(CompareBranchFMask) == CompareBranchFixed;
+ }
+
+ bool IsTestBranch() const { return Mask(TestBranchFMask) == TestBranchFixed; }
+
+ bool IsImmBranch() const { return BranchType() != UnknownBranchType; }
+
+ static float Imm8ToFP32(uint32_t imm8) {
+ // Imm8: abcdefgh (8 bits)
+ // Single: aBbb.bbbc.defg.h000.0000.0000.0000.0000 (32 bits)
+ // where B is b ^ 1
+ uint32_t bits = imm8;
+ uint32_t bit7 = (bits >> 7) & 0x1;
+ uint32_t bit6 = (bits >> 6) & 0x1;
+ uint32_t bit5_to_0 = bits & 0x3f;
+ uint32_t result = (bit7 << 31) | ((32 - bit6) << 25) | (bit5_to_0 << 19);
+
+ return bit_cast<float>(result);
+ }
+
+ static double Imm8ToFP64(uint32_t imm8) {
+ // Imm8: abcdefgh (8 bits)
+ // Double: aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
+ // 0000.0000.0000.0000.0000.0000.0000.0000 (64 bits)
+ // where B is b ^ 1
+ uint32_t bits = imm8;
+ uint64_t bit7 = (bits >> 7) & 0x1;
+ uint64_t bit6 = (bits >> 6) & 0x1;
+ uint64_t bit5_to_0 = bits & 0x3f;
+ uint64_t result = (bit7 << 63) | ((256 - bit6) << 54) | (bit5_to_0 << 48);
+
+ return bit_cast<double>(result);
+ }
+
+ bool IsLdrLiteral() const {
+ return Mask(LoadLiteralFMask) == LoadLiteralFixed;
+ }
+
+ bool IsLdrLiteralX() const { return Mask(LoadLiteralMask) == LDR_x_lit; }
+ bool IsLdrLiteralW() const { return Mask(LoadLiteralMask) == LDR_w_lit; }
+
+ bool IsPCRelAddressing() const {
+ return Mask(PCRelAddressingFMask) == PCRelAddressingFixed;
+ }
+
+ bool IsAdr() const { return Mask(PCRelAddressingMask) == ADR; }
+
+ bool IsBrk() const { return Mask(ExceptionMask) == BRK; }
+
+ bool IsUnresolvedInternalReference() const {
+ // Unresolved internal references are encoded as two consecutive brk
+ // instructions.
+ return IsBrk() && following()->IsBrk();
+ }
+
+ bool IsLogicalImmediate() const {
+ return Mask(LogicalImmediateFMask) == LogicalImmediateFixed;
+ }
+
+ bool IsAddSubImmediate() const {
+ return Mask(AddSubImmediateFMask) == AddSubImmediateFixed;
+ }
+
+ bool IsAddSubShifted() const {
+ return Mask(AddSubShiftedFMask) == AddSubShiftedFixed;
+ }
+
+ bool IsAddSubExtended() const {
+ return Mask(AddSubExtendedFMask) == AddSubExtendedFixed;
+ }
+
+ // Match any loads or stores, including pairs.
+ bool IsLoadOrStore() const {
+ return Mask(LoadStoreAnyFMask) == LoadStoreAnyFixed;
+ }
+
+ // Match any loads, including pairs.
+ bool IsLoad() const;
+ // Match any stores, including pairs.
+ bool IsStore() const;
+
+ // Indicate whether Rd can be the stack pointer or the zero register. This
+ // does not check that the instruction actually has an Rd field.
+ Reg31Mode RdMode() const {
+ // The following instructions use sp or wsp as Rd:
+ // Add/sub (immediate) when not setting the flags.
+ // Add/sub (extended) when not setting the flags.
+ // Logical (immediate) when not setting the flags.
+ // Otherwise, r31 is the zero register.
+ if (IsAddSubImmediate() || IsAddSubExtended()) {
+ if (Mask(AddSubSetFlagsBit)) {
+ return Reg31IsZeroRegister;
+ } else {
+ return Reg31IsStackPointer;
+ }
+ }
+ if (IsLogicalImmediate()) {
+ // Of the logical (immediate) instructions, only ANDS (and its aliases)
+ // can set the flags. The others can all write into sp.
+ // Note that some logical operations are not available to
+ // immediate-operand instructions, so we have to combine two masks here.
+ if (Mask(LogicalImmediateMask & LogicalOpMask) == ANDS) {
+ return Reg31IsZeroRegister;
+ } else {
+ return Reg31IsStackPointer;
+ }
+ }
+ return Reg31IsZeroRegister;
+ }
+
+ // Indicate whether Rn can be the stack pointer or the zero register. This
+ // does not check that the instruction actually has an Rn field.
+ Reg31Mode RnMode() const {
+ // The following instructions use sp or wsp as Rn:
+ // All loads and stores.
+ // Add/sub (immediate).
+ // Add/sub (extended).
+ // Otherwise, r31 is the zero register.
+ if (IsLoadOrStore() || IsAddSubImmediate() || IsAddSubExtended()) {
+ return Reg31IsStackPointer;
+ }
+ return Reg31IsZeroRegister;
+ }
+
+ ImmBranchType BranchType() const {
+ if (IsCondBranchImm()) {
+ return CondBranchType;
+ } else if (IsUncondBranchImm()) {
+ return UncondBranchType;
+ } else if (IsCompareBranch()) {
+ return CompareBranchType;
+ } else if (IsTestBranch()) {
+ return TestBranchType;
+ } else {
+ return UnknownBranchType;
+ }
+ }
+
+ static int ImmBranchRangeBitwidth(ImmBranchType branch_type) {
+ switch (branch_type) {
+ case UncondBranchType:
+ return ImmUncondBranch_width;
+ case CondBranchType:
+ return ImmCondBranch_width;
+ case CompareBranchType:
+ return ImmCmpBranch_width;
+ case TestBranchType:
+ return ImmTestBranch_width;
+ default:
+ UNREACHABLE();
+ }
+ }
+
+ // The range of the branch instruction, expressed as 'instr +- range'.
+ static int32_t ImmBranchRange(ImmBranchType branch_type) {
+ return (1 << (ImmBranchRangeBitwidth(branch_type) + kInstrSizeLog2)) / 2 -
+ kInstrSize;
+ }
+
+ int ImmBranch() const {
+ switch (BranchType()) {
+ case CondBranchType:
+ return ImmCondBranch();
+ case UncondBranchType:
+ return ImmUncondBranch();
+ case CompareBranchType:
+ return ImmCmpBranch();
+ case TestBranchType:
+ return ImmTestBranch();
+ default:
+ UNREACHABLE();
+ }
+ return 0;
+ }
+
+ int ImmUnresolvedInternalReference() const {
+ DCHECK(IsUnresolvedInternalReference());
+ // Unresolved references are encoded as two consecutive brk instructions.
+ // The associated immediate is made of the two 16-bit payloads.
+ int32_t high16 = ImmException();
+ int32_t low16 = following()->ImmException();
+ return (high16 << 16) | low16;
+ }
+
+ bool IsUnconditionalBranch() const {
+ return Mask(UnconditionalBranchMask) == B;
+ }
+
+ bool IsBranchAndLink() const { return Mask(UnconditionalBranchMask) == BL; }
+
+ bool IsBranchAndLinkToRegister() const {
+ return Mask(UnconditionalBranchToRegisterMask) == BLR;
+ }
+
+ bool IsMovz() const {
+ return (Mask(MoveWideImmediateMask) == MOVZ_x) ||
+ (Mask(MoveWideImmediateMask) == MOVZ_w);
+ }
+
+ bool IsMovk() const {
+ return (Mask(MoveWideImmediateMask) == MOVK_x) ||
+ (Mask(MoveWideImmediateMask) == MOVK_w);
+ }
+
+ bool IsMovn() const {
+ return (Mask(MoveWideImmediateMask) == MOVN_x) ||
+ (Mask(MoveWideImmediateMask) == MOVN_w);
+ }
+
+ bool IsException() const { return Mask(ExceptionFMask) == ExceptionFixed; }
+
+ bool IsPAuth() const { return Mask(SystemPAuthFMask) == SystemPAuthFixed; }
+
+ bool IsBti() const {
+ if (Mask(SystemHintFMask) == SystemHintFixed) {
+ int imm_hint = ImmHint();
+ switch (imm_hint) {
+ case BTI:
+ case BTI_c:
+ case BTI_j:
+ case BTI_jc:
+ return true;
+ }
+ }
+ return false;
+ }
+
+ bool IsNop(int n) {
+ // A marking nop is an instruction
+ // mov r<n>, r<n>
+ // which is encoded as
+ // orr r<n>, xzr, r<n>
+ return (Mask(LogicalShiftedMask) == ORR_x) && (Rd() == Rm()) && (Rd() == n);
+ }
+
+ // Find the PC offset encoded in this instruction. 'this' may be a branch or
+ // a PC-relative addressing instruction.
+ // The offset returned is unscaled.
+ V8_EXPORT_PRIVATE int64_t ImmPCOffset();
+
+ // Find the target of this instruction. 'this' may be a branch or a
+ // PC-relative addressing instruction.
+ V8_EXPORT_PRIVATE Instruction* ImmPCOffsetTarget();
+
+ // Check if the offset is in range of a given branch type. The offset is
+ // a byte offset, unscaled.
+ static bool IsValidImmPCOffset(ImmBranchType branch_type, ptrdiff_t offset);
+ bool IsTargetInImmPCOffsetRange(Instruction* target);
+ // Patch a PC-relative offset to refer to 'target'. 'this' may be a branch or
+ // a PC-relative addressing instruction.
+ void SetImmPCOffsetTarget(const AssemblerOptions& options,
+ Instruction* target);
+ void SetUnresolvedInternalReferenceImmTarget(const AssemblerOptions& options,
+ Instruction* target);
+ // Patch a literal load instruction to load from 'source'.
+ void SetImmLLiteral(Instruction* source);
+
+ uintptr_t LiteralAddress() {
+ int offset = ImmLLiteral() * kLoadLiteralScale;
+ return reinterpret_cast<uintptr_t>(this) + offset;
+ }
+
+ enum CheckAlignment { NO_CHECK, CHECK_ALIGNMENT };
+
+ V8_INLINE const Instruction* InstructionAtOffset(
+ int64_t offset, CheckAlignment check = CHECK_ALIGNMENT) const {
+ // The FUZZ_disasm test relies on no check being done.
+ DCHECK(check == NO_CHECK || IsAligned(offset, kInstrSize));
+ return this + offset;
+ }
+
+ V8_INLINE Instruction* InstructionAtOffset(
+ int64_t offset, CheckAlignment check = CHECK_ALIGNMENT) {
+ // The FUZZ_disasm test relies on no check being done.
+ DCHECK(check == NO_CHECK || IsAligned(offset, kInstrSize));
+ return this + offset;
+ }
+
+ template <typename T>
+ V8_INLINE static Instruction* Cast(T src) {
+ return reinterpret_cast<Instruction*>(src);
+ }
+
+ V8_INLINE ptrdiff_t DistanceTo(Instruction* target) {
+ return reinterpret_cast<Address>(target) - reinterpret_cast<Address>(this);
+ }
+
+ static const int ImmPCRelRangeBitwidth = 21;
+ static bool IsValidPCRelOffset(ptrdiff_t offset) { return is_int21(offset); }
+ void SetPCRelImmTarget(const AssemblerOptions& options, Instruction* target);
+ V8_EXPORT_PRIVATE void SetBranchImmTarget(Instruction* target);
+};
+
+// Simulator/Debugger debug instructions ---------------------------------------
+// Each debug marker is represented by a HLT instruction. The immediate comment
+// field in the instruction is used to identify the type of debug marker. Each
+// marker encodes arguments in a different way, as described below.
+
+// Indicate to the Debugger that the instruction is a redirected call.
+const Instr kImmExceptionIsRedirectedCall = 0xca11;
+
+// Represent unreachable code. This is used as a guard in parts of the code that
+// should not be reachable, such as in data encoded inline in the instructions.
+const Instr kImmExceptionIsUnreachable = 0xdebf;
+
+// A pseudo 'printf' instruction. The arguments will be passed to the platform
+// printf method.
+const Instr kImmExceptionIsPrintf = 0xdeb1;
+// Most parameters are stored in ARM64 registers as if the printf
+// pseudo-instruction was a call to the real printf method:
+// x0: The format string.
+// x1-x7: Optional arguments.
+// d0-d7: Optional arguments.
+//
+// Also, the argument layout is described inline in the instructions:
+// - arg_count: The number of arguments.
+// - arg_pattern: A set of PrintfArgPattern values, packed into two-bit fields.
+//
+// Floating-point and integer arguments are passed in separate sets of registers
+// in AAPCS64 (even for varargs functions), so it is not possible to determine
+// the type of each argument without some information about the values that were
+// passed in. This information could be retrieved from the printf format string,
+// but the format string is not trivial to parse so we encode the relevant
+// information with the HLT instruction.
+const unsigned kPrintfArgCountOffset = 1 * kInstrSize;
+const unsigned kPrintfArgPatternListOffset = 2 * kInstrSize;
+const unsigned kPrintfLength = 3 * kInstrSize;
+
+const unsigned kPrintfMaxArgCount = 4;
+
+// The argument pattern is a set of two-bit-fields, each with one of the
+// following values:
+enum PrintfArgPattern {
+ kPrintfArgW = 1,
+ kPrintfArgX = 2,
+ // There is no kPrintfArgS because floats are always converted to doubles in C
+ // varargs calls.
+ kPrintfArgD = 3
+};
+static const unsigned kPrintfArgPatternBits = 2;
+
+// A pseudo 'debug' instruction.
+const Instr kImmExceptionIsDebug = 0xdeb0;
+// Parameters are inlined in the code after a debug pseudo-instruction:
+// - Debug code.
+// - Debug parameters.
+// - Debug message string. This is a nullptr-terminated ASCII string, padded to
+// kInstrSize so that subsequent instructions are correctly aligned.
+// - A kImmExceptionIsUnreachable marker, to catch accidental execution of the
+// string data.
+const unsigned kDebugCodeOffset = 1 * kInstrSize;
+const unsigned kDebugParamsOffset = 2 * kInstrSize;
+const unsigned kDebugMessageOffset = 3 * kInstrSize;
+
+// Debug parameters.
+// Used without a TRACE_ option, the Debugger will print the arguments only
+// once. Otherwise TRACE_ENABLE and TRACE_DISABLE will enable or disable tracing
+// before every instruction for the specified LOG_ parameters.
+//
+// TRACE_OVERRIDE enables the specified LOG_ parameters, and disabled any
+// others that were not specified.
+//
+// For example:
+//
+// __ debug("print registers and fp registers", 0, LOG_REGS | LOG_VREGS);
+// will print the registers and fp registers only once.
+//
+// __ debug("trace disasm", 1, TRACE_ENABLE | LOG_DISASM);
+// starts disassembling the code.
+//
+// __ debug("trace rets", 2, TRACE_ENABLE | LOG_REGS);
+// adds the general purpose registers to the trace.
+//
+// __ debug("stop regs", 3, TRACE_DISABLE | LOG_REGS);
+// stops tracing the registers.
+const unsigned kDebuggerTracingDirectivesMask = 3 << 6;
+enum DebugParameters {
+ NO_PARAM = 0,
+ BREAK = 1 << 0,
+ LOG_DISASM = 1 << 1, // Use only with TRACE. Disassemble the code.
+ LOG_REGS = 1 << 2, // Log general purpose registers.
+ LOG_VREGS = 1 << 3, // Log NEON and floating-point registers.
+ LOG_SYS_REGS = 1 << 4, // Log the status flags.
+ LOG_WRITE = 1 << 5, // Log any memory write.
+
+ LOG_NONE = 0,
+ LOG_STATE = LOG_REGS | LOG_VREGS | LOG_SYS_REGS,
+ LOG_ALL = LOG_DISASM | LOG_STATE | LOG_WRITE,
+
+ // Trace control.
+ TRACE_ENABLE = 1 << 6,
+ TRACE_DISABLE = 2 << 6,
+ TRACE_OVERRIDE = 3 << 6
+};
+
+enum NEONFormat {
+ NF_UNDEF = 0,
+ NF_8B = 1,
+ NF_16B = 2,
+ NF_4H = 3,
+ NF_8H = 4,
+ NF_2S = 5,
+ NF_4S = 6,
+ NF_1D = 7,
+ NF_2D = 8,
+ NF_B = 9,
+ NF_H = 10,
+ NF_S = 11,
+ NF_D = 12
+};
+
+static const unsigned kNEONFormatMaxBits = 6;
+
+struct NEONFormatMap {
+ // The bit positions in the instruction to consider.
+ uint8_t bits[kNEONFormatMaxBits];
+
+ // Mapping from concatenated bits to format.
+ NEONFormat map[1 << kNEONFormatMaxBits];
+};
+
+class NEONFormatDecoder {
+ public:
+ enum SubstitutionMode { kPlaceholder, kFormat };
+
+ // Construct a format decoder with increasingly specific format maps for each
+ // substitution. If no format map is specified, the default is the integer
+ // format map.
+ explicit NEONFormatDecoder(const Instruction* instr);
+ NEONFormatDecoder(const Instruction* instr, const NEONFormatMap* format);
+ NEONFormatDecoder(const Instruction* instr, const NEONFormatMap* format0,
+ const NEONFormatMap* format1);
+ NEONFormatDecoder(const Instruction* instr, const NEONFormatMap* format0,
+ const NEONFormatMap* format1, const NEONFormatMap* format2);
+
+ // Set the format mapping for all or individual substitutions.
+ void SetFormatMaps(const NEONFormatMap* format0,
+ const NEONFormatMap* format1 = nullptr,
+ const NEONFormatMap* format2 = nullptr);
+ void SetFormatMap(unsigned index, const NEONFormatMap* format);
+
+ // Substitute %s in the input string with the placeholder string for each
+ // register, ie. "'B", "'H", etc.
+ const char* SubstitutePlaceholders(const char* string);
+
+ // Substitute %s in the input string with a new string based on the
+ // substitution mode.
+ const char* Substitute(const char* string, SubstitutionMode mode0 = kFormat,
+ SubstitutionMode mode1 = kFormat,
+ SubstitutionMode mode2 = kFormat,
+ SubstitutionMode mode3 = kFormat);
+
+ // Append a "2" to a mnemonic string based of the state of the Q bit.
+ const char* Mnemonic(const char* mnemonic);
+
+ VectorFormat GetVectorFormat(int format_index = 0);
+ VectorFormat GetVectorFormat(const NEONFormatMap* format_map);
+
+ // Built in mappings for common cases.
+
+ // The integer format map uses three bits (Q, size<1:0>) to encode the
+ // "standard" set of NEON integer vector formats.
+ static const NEONFormatMap* IntegerFormatMap() {
+ static const NEONFormatMap map = {
+ {23, 22, 30},
+ {NF_8B, NF_16B, NF_4H, NF_8H, NF_2S, NF_4S, NF_UNDEF, NF_2D}};
+ return ↦
+ }
+
+ // The long integer format map uses two bits (size<1:0>) to encode the
+ // long set of NEON integer vector formats. These are used in narrow, wide
+ // and long operations.
+ static const NEONFormatMap* LongIntegerFormatMap() {
+ static const NEONFormatMap map = {{23, 22}, {NF_8H, NF_4S, NF_2D}};
+ return ↦
+ }
+
+ // The FP format map uses two bits (Q, size<0>) to encode the NEON FP vector
+ // formats: NF_2S, NF_4S, NF_2D.
+ static const NEONFormatMap* FPFormatMap() {
+ // The FP format map assumes two bits (Q, size<0>) are used to encode the
+ // NEON FP vector formats: NF_2S, NF_4S, NF_2D.
+ static const NEONFormatMap map = {{22, 30},
+ {NF_2S, NF_4S, NF_UNDEF, NF_2D}};
+ return ↦
+ }
+
+ // The load/store format map uses three bits (Q, 11, 10) to encode the
+ // set of NEON vector formats.
+ static const NEONFormatMap* LoadStoreFormatMap() {
+ static const NEONFormatMap map = {
+ {11, 10, 30},
+ {NF_8B, NF_16B, NF_4H, NF_8H, NF_2S, NF_4S, NF_1D, NF_2D}};
+ return ↦
+ }
+
+ // The logical format map uses one bit (Q) to encode the NEON vector format:
+ // NF_8B, NF_16B.
+ static const NEONFormatMap* LogicalFormatMap() {
+ static const NEONFormatMap map = {{30}, {NF_8B, NF_16B}};
+ return ↦
+ }
+
+ // The triangular format map uses between two and five bits to encode the NEON
+ // vector format:
+ // xxx10->8B, xxx11->16B, xx100->4H, xx101->8H
+ // x1000->2S, x1001->4S, 10001->2D, all others undefined.
+ static const NEONFormatMap* TriangularFormatMap() {
+ static const NEONFormatMap map = {
+ {19, 18, 17, 16, 30},
+ {NF_UNDEF, NF_UNDEF, NF_8B, NF_16B, NF_4H, NF_8H, NF_8B, NF_16B,
+ NF_2S, NF_4S, NF_8B, NF_16B, NF_4H, NF_8H, NF_8B, NF_16B,
+ NF_UNDEF, NF_2D, NF_8B, NF_16B, NF_4H, NF_8H, NF_8B, NF_16B,
+ NF_2S, NF_4S, NF_8B, NF_16B, NF_4H, NF_8H, NF_8B, NF_16B}};
+ return ↦
+ }
+
+ // The scalar format map uses two bits (size<1:0>) to encode the NEON scalar
+ // formats: NF_B, NF_H, NF_S, NF_D.
+ static const NEONFormatMap* ScalarFormatMap() {
+ static const NEONFormatMap map = {{23, 22}, {NF_B, NF_H, NF_S, NF_D}};
+ return ↦
+ }
+
+ // The long scalar format map uses two bits (size<1:0>) to encode the longer
+ // NEON scalar formats: NF_H, NF_S, NF_D.
+ static const NEONFormatMap* LongScalarFormatMap() {
+ static const NEONFormatMap map = {{23, 22}, {NF_H, NF_S, NF_D}};
+ return ↦
+ }
+
+ // The FP scalar format map assumes one bit (size<0>) is used to encode the
+ // NEON FP scalar formats: NF_S, NF_D.
+ static const NEONFormatMap* FPScalarFormatMap() {
+ static const NEONFormatMap map = {{22}, {NF_S, NF_D}};
+ return ↦
+ }
+
+ // The triangular scalar format map uses between one and four bits to encode
+ // the NEON FP scalar formats:
+ // xxx1->B, xx10->H, x100->S, 1000->D, all others undefined.
+ static const NEONFormatMap* TriangularScalarFormatMap() {
+ static const NEONFormatMap map = {
+ {19, 18, 17, 16},
+ {NF_UNDEF, NF_B, NF_H, NF_B, NF_S, NF_B, NF_H, NF_B, NF_D, NF_B, NF_H,
+ NF_B, NF_S, NF_B, NF_H, NF_B}};
+ return ↦
+ }
+
+ private:
+ // Get a pointer to a string that represents the format or placeholder for
+ // the specified substitution index, based on the format map and instruction.
+ const char* GetSubstitute(int index, SubstitutionMode mode);
+
+ // Get the NEONFormat enumerated value for bits obtained from the
+ // instruction based on the specified format mapping.
+ NEONFormat GetNEONFormat(const NEONFormatMap* format_map);
+
+ // Convert a NEONFormat into a string.
+ static const char* NEONFormatAsString(NEONFormat format);
+
+ // Convert a NEONFormat into a register placeholder string.
+ static const char* NEONFormatAsPlaceholder(NEONFormat format);
+
+ // Select bits from instrbits_ defined by the bits array, concatenate them,
+ // and return the value.
+ uint8_t PickBits(const uint8_t bits[]);
+
+ Instr instrbits_;
+ const NEONFormatMap* formats_[4];
+ char form_buffer_[64];
+ char mne_buffer_[16];
+};
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_ARM64_INSTRUCTIONS_ARM64_H_
diff --git a/src/codegen/arm64/interface-descriptors-arm64.cc b/src/codegen/arm64/interface-descriptors-arm64.cc
new file mode 100644
index 0000000..f7bccfd
--- /dev/null
+++ b/src/codegen/arm64/interface-descriptors-arm64.cc
@@ -0,0 +1,288 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#if V8_TARGET_ARCH_ARM64
+
+#include "src/codegen/interface-descriptors.h"
+
+#include "src/execution/frames.h"
+
+namespace v8 {
+namespace internal {
+
+const Register CallInterfaceDescriptor::ContextRegister() { return cp; }
+
+void CallInterfaceDescriptor::DefaultInitializePlatformSpecific(
+ CallInterfaceDescriptorData* data, int register_parameter_count) {
+ const Register default_stub_registers[] = {x0, x1, x2, x3, x4};
+ CHECK_LE(static_cast<size_t>(register_parameter_count),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(register_parameter_count,
+ default_stub_registers);
+}
+
+void RecordWriteDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ const Register default_stub_registers[] = {x0, x1, x2, x3, x4};
+
+ data->RestrictAllocatableRegisters(default_stub_registers,
+ arraysize(default_stub_registers));
+
+ CHECK_LE(static_cast<size_t>(kParameterCount),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(kParameterCount, default_stub_registers);
+}
+
+void EphemeronKeyBarrierDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ const Register default_stub_registers[] = {x0, x1, x2, x3, x4};
+
+ data->RestrictAllocatableRegisters(default_stub_registers,
+ arraysize(default_stub_registers));
+
+ CHECK_LE(static_cast<size_t>(kParameterCount),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(kParameterCount, default_stub_registers);
+}
+
+const Register LoadDescriptor::ReceiverRegister() { return x1; }
+const Register LoadDescriptor::NameRegister() { return x2; }
+const Register LoadDescriptor::SlotRegister() { return x0; }
+
+const Register LoadWithVectorDescriptor::VectorRegister() { return x3; }
+
+const Register
+LoadWithReceiverAndVectorDescriptor::LookupStartObjectRegister() {
+ return x4;
+}
+
+const Register StoreDescriptor::ReceiverRegister() { return x1; }
+const Register StoreDescriptor::NameRegister() { return x2; }
+const Register StoreDescriptor::ValueRegister() { return x0; }
+const Register StoreDescriptor::SlotRegister() { return x4; }
+
+const Register StoreWithVectorDescriptor::VectorRegister() { return x3; }
+
+const Register StoreTransitionDescriptor::SlotRegister() { return x4; }
+const Register StoreTransitionDescriptor::VectorRegister() { return x3; }
+const Register StoreTransitionDescriptor::MapRegister() { return x5; }
+
+const Register ApiGetterDescriptor::HolderRegister() { return x0; }
+const Register ApiGetterDescriptor::CallbackRegister() { return x3; }
+
+const Register GrowArrayElementsDescriptor::ObjectRegister() { return x0; }
+const Register GrowArrayElementsDescriptor::KeyRegister() { return x3; }
+
+// static
+const Register TypeConversionDescriptor::ArgumentRegister() { return x0; }
+
+void TypeofDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {x3};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallTrampolineDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // x1: target
+ // x0: number of arguments
+ Register registers[] = {x1, x0};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // x0 : number of arguments (on the stack, not including receiver)
+ // x1 : the target to call
+ // x4 : arguments list length (untagged)
+ // x2 : arguments list (FixedArray)
+ Register registers[] = {x1, x0, x4, x2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallForwardVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // x1: target
+ // x0: number of arguments
+ // x2: start index (to supported rest parameters)
+ Register registers[] = {x1, x0, x2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallFunctionTemplateDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // x1 : function template info
+ // x2 : number of arguments (on the stack, not including receiver)
+ Register registers[] = {x1, x2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallWithSpreadDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // x0 : number of arguments (on the stack, not including receiver)
+ // x1 : the target to call
+ // x2 : the object to spread
+ Register registers[] = {x1, x0, x2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallWithArrayLikeDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // x1 : the target to call
+ // x2 : the arguments list
+ Register registers[] = {x1, x2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // x0 : number of arguments (on the stack, not including receiver)
+ // x1 : the target to call
+ // x3 : the new target
+ // x4 : arguments list length (untagged)
+ // x2 : arguments list (FixedArray)
+ Register registers[] = {x1, x3, x0, x4, x2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructForwardVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // x3: new target
+ // x1: target
+ // x0: number of arguments
+ // x2: start index (to supported rest parameters)
+ Register registers[] = {x1, x3, x0, x2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructWithSpreadDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // x0 : number of arguments (on the stack, not including receiver)
+ // x1 : the target to call
+ // x3 : the new target
+ // x2 : the object to spread
+ Register registers[] = {x1, x3, x0, x2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructWithArrayLikeDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // x1 : the target to call
+ // x3 : the new target
+ // x2 : the arguments list
+ Register registers[] = {x1, x3, x2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructStubDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // x3: new target
+ // x1: target
+ // x0: number of arguments
+ // x2: allocation site or undefined
+ Register registers[] = {x1, x3, x0, x2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void AbortDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {x1};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CompareDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // x1: left operand
+ // x0: right operand
+ Register registers[] = {x1, x0};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void BinaryOpDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // x1: left operand
+ // x0: right operand
+ Register registers[] = {x1, x0};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ArgumentsAdaptorDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ x1, // JSFunction
+ x3, // the new target
+ x0, // actual number of arguments
+ x2, // expected number of arguments
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ApiCallbackDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ x1, // kApiFunctionAddress
+ x2, // kArgc
+ x3, // kCallData
+ x0, // kHolder
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void InterpreterDispatchDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ kInterpreterAccumulatorRegister, kInterpreterBytecodeOffsetRegister,
+ kInterpreterBytecodeArrayRegister, kInterpreterDispatchTableRegister};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void InterpreterPushArgsThenCallDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ x0, // argument count (not including receiver)
+ x2, // address of first argument
+ x1 // the target callable to be call
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void InterpreterPushArgsThenConstructDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ x0, // argument count (not including receiver)
+ x4, // address of the first argument
+ x1, // constructor to call
+ x3, // new target
+ x2, // allocation site feedback if available, undefined otherwise
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ResumeGeneratorDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ x0, // the value to pass to the generator
+ x1 // the JSGeneratorObject to resume
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void FrameDropperTrampolineDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ x1, // loaded new FP
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void RunMicrotasksEntryDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {x0, x1};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_ARM64
diff --git a/src/codegen/arm64/macro-assembler-arm64-inl.h b/src/codegen/arm64/macro-assembler-arm64-inl.h
new file mode 100644
index 0000000..56be646
--- /dev/null
+++ b/src/codegen/arm64/macro-assembler-arm64-inl.h
@@ -0,0 +1,1465 @@
+// Copyright 2013 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_ARM64_MACRO_ASSEMBLER_ARM64_INL_H_
+#define V8_CODEGEN_ARM64_MACRO_ASSEMBLER_ARM64_INL_H_
+
+#include <ctype.h>
+
+#include "src/common/globals.h"
+
+#include "src/base/bits.h"
+#include "src/codegen/arm64/assembler-arm64-inl.h"
+#include "src/codegen/arm64/assembler-arm64.h"
+#include "src/codegen/macro-assembler.h"
+
+namespace v8 {
+namespace internal {
+
+MemOperand FieldMemOperand(Register object, int offset) {
+ return MemOperand(object, offset - kHeapObjectTag);
+}
+
+void TurboAssembler::And(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ LogicalMacro(rd, rn, operand, AND);
+}
+
+void TurboAssembler::Ands(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ LogicalMacro(rd, rn, operand, ANDS);
+}
+
+void TurboAssembler::Tst(const Register& rn, const Operand& operand) {
+ DCHECK(allow_macro_instructions());
+ LogicalMacro(AppropriateZeroRegFor(rn), rn, operand, ANDS);
+}
+
+void TurboAssembler::Bic(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ LogicalMacro(rd, rn, operand, BIC);
+}
+
+void MacroAssembler::Bics(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ LogicalMacro(rd, rn, operand, BICS);
+}
+
+void TurboAssembler::Orr(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ LogicalMacro(rd, rn, operand, ORR);
+}
+
+void TurboAssembler::Orn(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ LogicalMacro(rd, rn, operand, ORN);
+}
+
+void TurboAssembler::Eor(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ LogicalMacro(rd, rn, operand, EOR);
+}
+
+void TurboAssembler::Eon(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ LogicalMacro(rd, rn, operand, EON);
+}
+
+void TurboAssembler::Ccmp(const Register& rn, const Operand& operand,
+ StatusFlags nzcv, Condition cond) {
+ DCHECK(allow_macro_instructions());
+ if (operand.IsImmediate() && (operand.ImmediateValue() < 0)) {
+ ConditionalCompareMacro(rn, -operand.ImmediateValue(), nzcv, cond, CCMN);
+ } else {
+ ConditionalCompareMacro(rn, operand, nzcv, cond, CCMP);
+ }
+}
+
+void TurboAssembler::CcmpTagged(const Register& rn, const Operand& operand,
+ StatusFlags nzcv, Condition cond) {
+ if (COMPRESS_POINTERS_BOOL) {
+ Ccmp(rn.W(), operand.ToW(), nzcv, cond);
+ } else {
+ Ccmp(rn, operand, nzcv, cond);
+ }
+}
+
+void MacroAssembler::Ccmn(const Register& rn, const Operand& operand,
+ StatusFlags nzcv, Condition cond) {
+ DCHECK(allow_macro_instructions());
+ if (operand.IsImmediate() && (operand.ImmediateValue() < 0)) {
+ ConditionalCompareMacro(rn, -operand.ImmediateValue(), nzcv, cond, CCMP);
+ } else {
+ ConditionalCompareMacro(rn, operand, nzcv, cond, CCMN);
+ }
+}
+
+void TurboAssembler::Add(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ DCHECK(allow_macro_instructions());
+ if (operand.IsImmediate() && (operand.ImmediateValue() < 0) &&
+ IsImmAddSub(-operand.ImmediateValue())) {
+ AddSubMacro(rd, rn, -operand.ImmediateValue(), LeaveFlags, SUB);
+ } else {
+ AddSubMacro(rd, rn, operand, LeaveFlags, ADD);
+ }
+}
+
+void TurboAssembler::Adds(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ DCHECK(allow_macro_instructions());
+ if (operand.IsImmediate() && (operand.ImmediateValue() < 0) &&
+ IsImmAddSub(-operand.ImmediateValue())) {
+ AddSubMacro(rd, rn, -operand.ImmediateValue(), SetFlags, SUB);
+ } else {
+ AddSubMacro(rd, rn, operand, SetFlags, ADD);
+ }
+}
+
+void TurboAssembler::Sub(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ DCHECK(allow_macro_instructions());
+ if (operand.IsImmediate() && (operand.ImmediateValue() < 0) &&
+ IsImmAddSub(-operand.ImmediateValue())) {
+ AddSubMacro(rd, rn, -operand.ImmediateValue(), LeaveFlags, ADD);
+ } else {
+ AddSubMacro(rd, rn, operand, LeaveFlags, SUB);
+ }
+}
+
+void TurboAssembler::Subs(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ DCHECK(allow_macro_instructions());
+ if (operand.IsImmediate() && (operand.ImmediateValue() < 0) &&
+ IsImmAddSub(-operand.ImmediateValue())) {
+ AddSubMacro(rd, rn, -operand.ImmediateValue(), SetFlags, ADD);
+ } else {
+ AddSubMacro(rd, rn, operand, SetFlags, SUB);
+ }
+}
+
+void TurboAssembler::Cmn(const Register& rn, const Operand& operand) {
+ DCHECK(allow_macro_instructions());
+ Adds(AppropriateZeroRegFor(rn), rn, operand);
+}
+
+void TurboAssembler::Cmp(const Register& rn, const Operand& operand) {
+ DCHECK(allow_macro_instructions());
+ Subs(AppropriateZeroRegFor(rn), rn, operand);
+}
+
+void TurboAssembler::CmpTagged(const Register& rn, const Operand& operand) {
+ if (COMPRESS_POINTERS_BOOL) {
+ Cmp(rn.W(), operand.ToW());
+ } else {
+ Cmp(rn, operand);
+ }
+}
+
+void TurboAssembler::Neg(const Register& rd, const Operand& operand) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ if (operand.IsImmediate()) {
+ Mov(rd, -operand.ImmediateValue());
+ } else {
+ Sub(rd, AppropriateZeroRegFor(rd), operand);
+ }
+}
+
+void TurboAssembler::Negs(const Register& rd, const Operand& operand) {
+ DCHECK(allow_macro_instructions());
+ Subs(rd, AppropriateZeroRegFor(rd), operand);
+}
+
+void TurboAssembler::Adc(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ AddSubWithCarryMacro(rd, rn, operand, LeaveFlags, ADC);
+}
+
+void MacroAssembler::Adcs(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ AddSubWithCarryMacro(rd, rn, operand, SetFlags, ADC);
+}
+
+void MacroAssembler::Sbc(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ AddSubWithCarryMacro(rd, rn, operand, LeaveFlags, SBC);
+}
+
+void MacroAssembler::Sbcs(const Register& rd, const Register& rn,
+ const Operand& operand) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ AddSubWithCarryMacro(rd, rn, operand, SetFlags, SBC);
+}
+
+void MacroAssembler::Ngc(const Register& rd, const Operand& operand) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ Register zr = AppropriateZeroRegFor(rd);
+ Sbc(rd, zr, operand);
+}
+
+void MacroAssembler::Ngcs(const Register& rd, const Operand& operand) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ Register zr = AppropriateZeroRegFor(rd);
+ Sbcs(rd, zr, operand);
+}
+
+void TurboAssembler::Mvn(const Register& rd, uint64_t imm) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ Mov(rd, ~imm);
+}
+
+#define DEFINE_FUNCTION(FN, REGTYPE, REG, OP) \
+ void TurboAssembler::FN(const REGTYPE REG, const MemOperand& addr) { \
+ DCHECK(allow_macro_instructions()); \
+ LoadStoreMacro(REG, addr, OP); \
+ }
+LS_MACRO_LIST(DEFINE_FUNCTION)
+#undef DEFINE_FUNCTION
+
+#define DEFINE_FUNCTION(FN, REGTYPE, REG, REG2, OP) \
+ void TurboAssembler::FN(const REGTYPE REG, const REGTYPE REG2, \
+ const MemOperand& addr) { \
+ DCHECK(allow_macro_instructions()); \
+ LoadStorePairMacro(REG, REG2, addr, OP); \
+ }
+LSPAIR_MACRO_LIST(DEFINE_FUNCTION)
+#undef DEFINE_FUNCTION
+
+#define DECLARE_FUNCTION(FN, OP) \
+ void TurboAssembler::FN(const Register& rt, const Register& rn) { \
+ DCHECK(allow_macro_instructions()); \
+ OP(rt, rn); \
+ }
+LDA_STL_MACRO_LIST(DECLARE_FUNCTION)
+#undef DECLARE_FUNCTION
+
+#define DECLARE_FUNCTION(FN, OP) \
+ void MacroAssembler::FN(const Register& rs, const Register& rt, \
+ const Register& rn) { \
+ DCHECK(allow_macro_instructions()); \
+ OP(rs, rt, rn); \
+ }
+STLX_MACRO_LIST(DECLARE_FUNCTION)
+#undef DECLARE_FUNCTION
+
+void TurboAssembler::Asr(const Register& rd, const Register& rn,
+ unsigned shift) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ asr(rd, rn, shift);
+}
+
+void TurboAssembler::Asr(const Register& rd, const Register& rn,
+ const Register& rm) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ asrv(rd, rn, rm);
+}
+
+void TurboAssembler::B(Label* label) {
+ DCHECK(allow_macro_instructions());
+ b(label);
+ CheckVeneerPool(false, false);
+}
+
+void TurboAssembler::B(Condition cond, Label* label) {
+ DCHECK(allow_macro_instructions());
+ B(label, cond);
+}
+
+void TurboAssembler::Bfi(const Register& rd, const Register& rn, unsigned lsb,
+ unsigned width) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ bfi(rd, rn, lsb, width);
+}
+
+void MacroAssembler::Bfxil(const Register& rd, const Register& rn, unsigned lsb,
+ unsigned width) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ bfxil(rd, rn, lsb, width);
+}
+
+void TurboAssembler::Bind(Label* label, BranchTargetIdentifier id) {
+ DCHECK(allow_macro_instructions());
+ if (id == BranchTargetIdentifier::kNone) {
+ bind(label);
+ } else {
+ // Emit this inside an InstructionAccurateScope to ensure there are no extra
+ // instructions between the bind and the target identifier instruction.
+ InstructionAccurateScope scope(this, 1);
+ bind(label);
+ if (id == BranchTargetIdentifier::kPacibsp) {
+ pacibsp();
+ } else {
+ bti(id);
+ }
+ }
+}
+
+void TurboAssembler::CodeEntry() { CallTarget(); }
+
+void TurboAssembler::ExceptionHandler() { JumpTarget(); }
+
+void TurboAssembler::BindExceptionHandler(Label* label) {
+ BindJumpTarget(label);
+}
+
+void TurboAssembler::JumpTarget() {
+#ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
+ bti(BranchTargetIdentifier::kBtiJump);
+#endif
+}
+
+void TurboAssembler::BindJumpTarget(Label* label) {
+#ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
+ Bind(label, BranchTargetIdentifier::kBtiJump);
+#else
+ Bind(label);
+#endif
+}
+
+void TurboAssembler::CallTarget() {
+#ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
+ bti(BranchTargetIdentifier::kBtiCall);
+#endif
+}
+
+void TurboAssembler::JumpOrCallTarget() {
+#ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
+ bti(BranchTargetIdentifier::kBtiJumpCall);
+#endif
+}
+
+void TurboAssembler::BindJumpOrCallTarget(Label* label) {
+#ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
+ Bind(label, BranchTargetIdentifier::kBtiJumpCall);
+#else
+ Bind(label);
+#endif
+}
+
+void TurboAssembler::Bl(Label* label) {
+ DCHECK(allow_macro_instructions());
+ bl(label);
+}
+
+void TurboAssembler::Blr(const Register& xn) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!xn.IsZero());
+ blr(xn);
+}
+
+void TurboAssembler::Br(const Register& xn) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!xn.IsZero());
+ br(xn);
+}
+
+void TurboAssembler::Brk(int code) {
+ DCHECK(allow_macro_instructions());
+ brk(code);
+}
+
+void MacroAssembler::Cinc(const Register& rd, const Register& rn,
+ Condition cond) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ DCHECK((cond != al) && (cond != nv));
+ cinc(rd, rn, cond);
+}
+
+void MacroAssembler::Cinv(const Register& rd, const Register& rn,
+ Condition cond) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ DCHECK((cond != al) && (cond != nv));
+ cinv(rd, rn, cond);
+}
+
+void TurboAssembler::Cls(const Register& rd, const Register& rn) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ cls(rd, rn);
+}
+
+void TurboAssembler::Clz(const Register& rd, const Register& rn) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ clz(rd, rn);
+}
+
+void TurboAssembler::Cneg(const Register& rd, const Register& rn,
+ Condition cond) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ DCHECK((cond != al) && (cond != nv));
+ cneg(rd, rn, cond);
+}
+
+// Conditionally zero the destination register. Only X registers are supported
+// due to the truncation side-effect when used on W registers.
+void MacroAssembler::CzeroX(const Register& rd, Condition cond) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsSP() && rd.Is64Bits());
+ DCHECK((cond != al) && (cond != nv));
+ csel(rd, xzr, rd, cond);
+}
+
+// Conditionally move a value into the destination register. Only X registers
+// are supported due to the truncation side-effect when used on W registers.
+void TurboAssembler::CmovX(const Register& rd, const Register& rn,
+ Condition cond) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsSP());
+ DCHECK(rd.Is64Bits() && rn.Is64Bits());
+ DCHECK((cond != al) && (cond != nv));
+ if (rd != rn) {
+ csel(rd, rn, rd, cond);
+ }
+}
+
+void TurboAssembler::Csdb() {
+ DCHECK(allow_macro_instructions());
+ csdb();
+}
+
+void TurboAssembler::Cset(const Register& rd, Condition cond) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ DCHECK((cond != al) && (cond != nv));
+ cset(rd, cond);
+}
+
+void TurboAssembler::Csetm(const Register& rd, Condition cond) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ DCHECK((cond != al) && (cond != nv));
+ csetm(rd, cond);
+}
+
+void TurboAssembler::Csinc(const Register& rd, const Register& rn,
+ const Register& rm, Condition cond) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ DCHECK((cond != al) && (cond != nv));
+ csinc(rd, rn, rm, cond);
+}
+
+void MacroAssembler::Csinv(const Register& rd, const Register& rn,
+ const Register& rm, Condition cond) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ DCHECK((cond != al) && (cond != nv));
+ csinv(rd, rn, rm, cond);
+}
+
+void MacroAssembler::Csneg(const Register& rd, const Register& rn,
+ const Register& rm, Condition cond) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ DCHECK((cond != al) && (cond != nv));
+ csneg(rd, rn, rm, cond);
+}
+
+void TurboAssembler::Dmb(BarrierDomain domain, BarrierType type) {
+ DCHECK(allow_macro_instructions());
+ dmb(domain, type);
+}
+
+void TurboAssembler::Dsb(BarrierDomain domain, BarrierType type) {
+ DCHECK(allow_macro_instructions());
+ dsb(domain, type);
+}
+
+void TurboAssembler::Debug(const char* message, uint32_t code, Instr params) {
+ DCHECK(allow_macro_instructions());
+ debug(message, code, params);
+}
+
+void MacroAssembler::Extr(const Register& rd, const Register& rn,
+ const Register& rm, unsigned lsb) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ extr(rd, rn, rm, lsb);
+}
+
+void TurboAssembler::Fabs(const VRegister& fd, const VRegister& fn) {
+ DCHECK(allow_macro_instructions());
+ fabs(fd, fn);
+}
+
+void TurboAssembler::Fadd(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm) {
+ DCHECK(allow_macro_instructions());
+ fadd(fd, fn, fm);
+}
+
+void TurboAssembler::Fccmp(const VRegister& fn, const VRegister& fm,
+ StatusFlags nzcv, Condition cond) {
+ DCHECK(allow_macro_instructions());
+ DCHECK((cond != al) && (cond != nv));
+ fccmp(fn, fm, nzcv, cond);
+}
+
+void TurboAssembler::Fcmp(const VRegister& fn, const VRegister& fm) {
+ DCHECK(allow_macro_instructions());
+ fcmp(fn, fm);
+}
+
+void TurboAssembler::Fcmp(const VRegister& fn, double value) {
+ DCHECK(allow_macro_instructions());
+ if (value != 0.0) {
+ UseScratchRegisterScope temps(this);
+ VRegister tmp = temps.AcquireSameSizeAs(fn);
+ Fmov(tmp, value);
+ fcmp(fn, tmp);
+ } else {
+ fcmp(fn, value);
+ }
+}
+
+void MacroAssembler::Fcsel(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm, Condition cond) {
+ DCHECK(allow_macro_instructions());
+ DCHECK((cond != al) && (cond != nv));
+ fcsel(fd, fn, fm, cond);
+}
+
+void TurboAssembler::Fcvt(const VRegister& fd, const VRegister& fn) {
+ DCHECK(allow_macro_instructions());
+ fcvt(fd, fn);
+}
+
+void TurboAssembler::Fcvtas(const Register& rd, const VRegister& fn) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ fcvtas(rd, fn);
+}
+
+void TurboAssembler::Fcvtau(const Register& rd, const VRegister& fn) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ fcvtau(rd, fn);
+}
+
+void TurboAssembler::Fcvtms(const Register& rd, const VRegister& fn) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ fcvtms(rd, fn);
+}
+
+void TurboAssembler::Fcvtmu(const Register& rd, const VRegister& fn) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ fcvtmu(rd, fn);
+}
+
+void TurboAssembler::Fcvtns(const Register& rd, const VRegister& fn) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ fcvtns(rd, fn);
+}
+
+void TurboAssembler::Fcvtnu(const Register& rd, const VRegister& fn) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ fcvtnu(rd, fn);
+}
+
+void TurboAssembler::Fcvtzs(const Register& rd, const VRegister& fn) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ fcvtzs(rd, fn);
+}
+void TurboAssembler::Fcvtzu(const Register& rd, const VRegister& fn) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ fcvtzu(rd, fn);
+}
+
+void TurboAssembler::Fdiv(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm) {
+ DCHECK(allow_macro_instructions());
+ fdiv(fd, fn, fm);
+}
+
+void MacroAssembler::Fmadd(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm, const VRegister& fa) {
+ DCHECK(allow_macro_instructions());
+ fmadd(fd, fn, fm, fa);
+}
+
+void TurboAssembler::Fmax(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm) {
+ DCHECK(allow_macro_instructions());
+ fmax(fd, fn, fm);
+}
+
+void MacroAssembler::Fmaxnm(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm) {
+ DCHECK(allow_macro_instructions());
+ fmaxnm(fd, fn, fm);
+}
+
+void TurboAssembler::Fmin(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm) {
+ DCHECK(allow_macro_instructions());
+ fmin(fd, fn, fm);
+}
+
+void MacroAssembler::Fminnm(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm) {
+ DCHECK(allow_macro_instructions());
+ fminnm(fd, fn, fm);
+}
+
+void TurboAssembler::Fmov(VRegister fd, VRegister fn) {
+ DCHECK(allow_macro_instructions());
+ // Only emit an instruction if fd and fn are different, and they are both D
+ // registers. fmov(s0, s0) is not a no-op because it clears the top word of
+ // d0. Technically, fmov(d0, d0) is not a no-op either because it clears the
+ // top of q0, but VRegister does not currently support Q registers.
+ if (fd != fn || !fd.Is64Bits()) {
+ fmov(fd, fn);
+ }
+}
+
+void TurboAssembler::Fmov(VRegister fd, Register rn) {
+ DCHECK(allow_macro_instructions());
+ fmov(fd, rn);
+}
+
+void TurboAssembler::Fmov(VRegister vd, double imm) {
+ DCHECK(allow_macro_instructions());
+
+ if (vd.Is1S() || vd.Is2S() || vd.Is4S()) {
+ Fmov(vd, static_cast<float>(imm));
+ return;
+ }
+
+ DCHECK(vd.Is1D() || vd.Is2D());
+ if (IsImmFP64(imm)) {
+ fmov(vd, imm);
+ } else {
+ uint64_t bits = bit_cast<uint64_t>(imm);
+ if (vd.IsScalar()) {
+ if (bits == 0) {
+ fmov(vd, xzr);
+ } else {
+ UseScratchRegisterScope temps(this);
+ Register tmp = temps.AcquireX();
+ Mov(tmp, bits);
+ fmov(vd, tmp);
+ }
+ } else {
+ Movi(vd, bits);
+ }
+ }
+}
+
+void TurboAssembler::Fmov(VRegister vd, float imm) {
+ DCHECK(allow_macro_instructions());
+ if (vd.Is1D() || vd.Is2D()) {
+ Fmov(vd, static_cast<double>(imm));
+ return;
+ }
+
+ DCHECK(vd.Is1S() || vd.Is2S() || vd.Is4S());
+ if (IsImmFP32(imm)) {
+ fmov(vd, imm);
+ } else {
+ uint32_t bits = bit_cast<uint32_t>(imm);
+ if (vd.IsScalar()) {
+ if (bits == 0) {
+ fmov(vd, wzr);
+ } else {
+ UseScratchRegisterScope temps(this);
+ Register tmp = temps.AcquireW();
+ Mov(tmp, bits);
+ Fmov(vd, tmp);
+ }
+ } else {
+ Movi(vd, bits);
+ }
+ }
+}
+
+void TurboAssembler::Fmov(Register rd, VRegister fn) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ fmov(rd, fn);
+}
+
+void MacroAssembler::Fmsub(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm, const VRegister& fa) {
+ DCHECK(allow_macro_instructions());
+ fmsub(fd, fn, fm, fa);
+}
+
+void TurboAssembler::Fmul(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm) {
+ DCHECK(allow_macro_instructions());
+ fmul(fd, fn, fm);
+}
+
+void MacroAssembler::Fnmadd(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm, const VRegister& fa) {
+ DCHECK(allow_macro_instructions());
+ fnmadd(fd, fn, fm, fa);
+}
+
+void MacroAssembler::Fnmsub(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm, const VRegister& fa) {
+ DCHECK(allow_macro_instructions());
+ fnmsub(fd, fn, fm, fa);
+}
+
+void TurboAssembler::Fsub(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm) {
+ DCHECK(allow_macro_instructions());
+ fsub(fd, fn, fm);
+}
+
+void MacroAssembler::Hint(SystemHint code) {
+ DCHECK(allow_macro_instructions());
+ hint(code);
+}
+
+void MacroAssembler::Hlt(int code) {
+ DCHECK(allow_macro_instructions());
+ hlt(code);
+}
+
+void TurboAssembler::Isb() {
+ DCHECK(allow_macro_instructions());
+ isb();
+}
+
+void TurboAssembler::Ldr(const CPURegister& rt, const Operand& operand) {
+ DCHECK(allow_macro_instructions());
+ ldr(rt, operand);
+}
+
+void TurboAssembler::Lsl(const Register& rd, const Register& rn,
+ unsigned shift) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ lsl(rd, rn, shift);
+}
+
+void TurboAssembler::Lsl(const Register& rd, const Register& rn,
+ const Register& rm) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ lslv(rd, rn, rm);
+}
+
+void TurboAssembler::Lsr(const Register& rd, const Register& rn,
+ unsigned shift) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ lsr(rd, rn, shift);
+}
+
+void TurboAssembler::Lsr(const Register& rd, const Register& rn,
+ const Register& rm) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ lsrv(rd, rn, rm);
+}
+
+void TurboAssembler::Madd(const Register& rd, const Register& rn,
+ const Register& rm, const Register& ra) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ madd(rd, rn, rm, ra);
+}
+
+void TurboAssembler::Mneg(const Register& rd, const Register& rn,
+ const Register& rm) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ mneg(rd, rn, rm);
+}
+
+void MacroAssembler::Movk(const Register& rd, uint64_t imm, int shift) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ movk(rd, imm, shift);
+}
+
+void TurboAssembler::Mrs(const Register& rt, SystemRegister sysreg) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rt.IsZero());
+ mrs(rt, sysreg);
+}
+
+void TurboAssembler::Msr(SystemRegister sysreg, const Register& rt) {
+ DCHECK(allow_macro_instructions());
+ msr(sysreg, rt);
+}
+
+void TurboAssembler::Msub(const Register& rd, const Register& rn,
+ const Register& rm, const Register& ra) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ msub(rd, rn, rm, ra);
+}
+
+void TurboAssembler::Mul(const Register& rd, const Register& rn,
+ const Register& rm) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ mul(rd, rn, rm);
+}
+
+void TurboAssembler::Rbit(const Register& rd, const Register& rn) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ rbit(rd, rn);
+}
+
+void TurboAssembler::Rev(const Register& rd, const Register& rn) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ rev(rd, rn);
+}
+
+void TurboAssembler::Ret(const Register& xn) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!xn.IsZero());
+ ret(xn);
+ CheckVeneerPool(false, false);
+}
+
+void MacroAssembler::Rev(const Register& rd, const Register& rn) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ rev(rd, rn);
+}
+
+void TurboAssembler::Rev16(const Register& rd, const Register& rn) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ rev16(rd, rn);
+}
+
+void TurboAssembler::Rev32(const Register& rd, const Register& rn) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ rev32(rd, rn);
+}
+
+void TurboAssembler::Ror(const Register& rd, const Register& rs,
+ unsigned shift) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ ror(rd, rs, shift);
+}
+
+void TurboAssembler::Ror(const Register& rd, const Register& rn,
+ const Register& rm) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ rorv(rd, rn, rm);
+}
+
+void MacroAssembler::Sbfiz(const Register& rd, const Register& rn, unsigned lsb,
+ unsigned width) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ sbfiz(rd, rn, lsb, width);
+}
+
+void TurboAssembler::Sbfx(const Register& rd, const Register& rn, unsigned lsb,
+ unsigned width) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ sbfx(rd, rn, lsb, width);
+}
+
+void TurboAssembler::Scvtf(const VRegister& fd, const Register& rn,
+ unsigned fbits) {
+ DCHECK(allow_macro_instructions());
+ scvtf(fd, rn, fbits);
+}
+
+void TurboAssembler::Sdiv(const Register& rd, const Register& rn,
+ const Register& rm) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ sdiv(rd, rn, rm);
+}
+
+void MacroAssembler::Smaddl(const Register& rd, const Register& rn,
+ const Register& rm, const Register& ra) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ smaddl(rd, rn, rm, ra);
+}
+
+void MacroAssembler::Smsubl(const Register& rd, const Register& rn,
+ const Register& rm, const Register& ra) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ smsubl(rd, rn, rm, ra);
+}
+
+void TurboAssembler::Smull(const Register& rd, const Register& rn,
+ const Register& rm) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ smull(rd, rn, rm);
+}
+
+void MacroAssembler::Smulh(const Register& rd, const Register& rn,
+ const Register& rm) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ smulh(rd, rn, rm);
+}
+
+void TurboAssembler::Umull(const Register& rd, const Register& rn,
+ const Register& rm) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ umaddl(rd, rn, rm, xzr);
+}
+
+void TurboAssembler::Sxtb(const Register& rd, const Register& rn) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ sxtb(rd, rn);
+}
+
+void TurboAssembler::Sxth(const Register& rd, const Register& rn) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ sxth(rd, rn);
+}
+
+void TurboAssembler::Sxtw(const Register& rd, const Register& rn) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ sxtw(rd, rn);
+}
+
+void TurboAssembler::Ubfiz(const Register& rd, const Register& rn, unsigned lsb,
+ unsigned width) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ ubfiz(rd, rn, lsb, width);
+}
+
+void TurboAssembler::Ubfx(const Register& rd, const Register& rn, unsigned lsb,
+ unsigned width) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ ubfx(rd, rn, lsb, width);
+}
+
+void TurboAssembler::Ucvtf(const VRegister& fd, const Register& rn,
+ unsigned fbits) {
+ DCHECK(allow_macro_instructions());
+ ucvtf(fd, rn, fbits);
+}
+
+void TurboAssembler::Udiv(const Register& rd, const Register& rn,
+ const Register& rm) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ udiv(rd, rn, rm);
+}
+
+void MacroAssembler::Umaddl(const Register& rd, const Register& rn,
+ const Register& rm, const Register& ra) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ umaddl(rd, rn, rm, ra);
+}
+
+void MacroAssembler::Umsubl(const Register& rd, const Register& rn,
+ const Register& rm, const Register& ra) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ umsubl(rd, rn, rm, ra);
+}
+
+void TurboAssembler::Uxtb(const Register& rd, const Register& rn) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ uxtb(rd, rn);
+}
+
+void TurboAssembler::Uxth(const Register& rd, const Register& rn) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ uxth(rd, rn);
+}
+
+void TurboAssembler::Uxtw(const Register& rd, const Register& rn) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ uxtw(rd, rn);
+}
+
+void TurboAssembler::InitializeRootRegister() {
+ ExternalReference isolate_root = ExternalReference::isolate_root(isolate());
+ Mov(kRootRegister, Operand(isolate_root));
+}
+
+void MacroAssembler::SmiTag(Register dst, Register src) {
+ DCHECK(dst.Is64Bits() && src.Is64Bits());
+ DCHECK(SmiValuesAre32Bits() || SmiValuesAre31Bits());
+ Lsl(dst, src, kSmiShift);
+}
+
+void MacroAssembler::SmiTag(Register smi) { SmiTag(smi, smi); }
+
+void TurboAssembler::SmiUntag(Register dst, Register src) {
+ DCHECK(dst.Is64Bits() && src.Is64Bits());
+ if (FLAG_enable_slow_asserts) {
+ AssertSmi(src);
+ }
+ DCHECK(SmiValuesAre32Bits() || SmiValuesAre31Bits());
+ if (COMPRESS_POINTERS_BOOL) {
+ Asr(dst.W(), src.W(), kSmiShift);
+ Sxtw(dst, dst);
+ } else {
+ Asr(dst, src, kSmiShift);
+ }
+}
+
+void TurboAssembler::SmiUntag(Register dst, const MemOperand& src) {
+ DCHECK(dst.Is64Bits());
+ if (SmiValuesAre32Bits()) {
+ if (src.IsImmediateOffset() && src.shift_amount() == 0) {
+ // Load value directly from the upper half-word.
+ // Assumes that Smis are shifted by 32 bits and little endianness.
+ DCHECK_EQ(kSmiShift, 32);
+ Ldrsw(dst,
+ MemOperand(src.base(), src.offset() + (kSmiShift / kBitsPerByte),
+ src.addrmode()));
+
+ } else {
+ Ldr(dst, src);
+ SmiUntag(dst);
+ }
+ } else {
+ DCHECK(SmiValuesAre31Bits());
+ if (COMPRESS_POINTERS_BOOL) {
+ Ldr(dst.W(), src);
+ } else {
+ Ldr(dst, src);
+ }
+ SmiUntag(dst);
+ }
+}
+
+void TurboAssembler::SmiUntag(Register smi) { SmiUntag(smi, smi); }
+
+void TurboAssembler::JumpIfSmi(Register value, Label* smi_label,
+ Label* not_smi_label) {
+ STATIC_ASSERT((kSmiTagSize == 1) && (kSmiTag == 0));
+ // Check if the tag bit is set.
+ if (smi_label) {
+ Tbz(value, 0, smi_label);
+ if (not_smi_label) {
+ B(not_smi_label);
+ }
+ } else {
+ DCHECK(not_smi_label);
+ Tbnz(value, 0, not_smi_label);
+ }
+}
+
+void TurboAssembler::JumpIfEqual(Register x, int32_t y, Label* dest) {
+ CompareAndBranch(x, y, eq, dest);
+}
+
+void TurboAssembler::JumpIfLessThan(Register x, int32_t y, Label* dest) {
+ CompareAndBranch(x, y, lt, dest);
+}
+
+void MacroAssembler::JumpIfNotSmi(Register value, Label* not_smi_label) {
+ JumpIfSmi(value, nullptr, not_smi_label);
+}
+
+void TurboAssembler::jmp(Label* L) { B(L); }
+
+template <TurboAssembler::StoreLRMode lr_mode>
+void TurboAssembler::Push(const CPURegister& src0, const CPURegister& src1,
+ const CPURegister& src2, const CPURegister& src3) {
+ DCHECK(AreSameSizeAndType(src0, src1, src2, src3));
+ DCHECK_IMPLIES((lr_mode == kSignLR), ((src0 == lr) || (src1 == lr) ||
+ (src2 == lr) || (src3 == lr)));
+ DCHECK_IMPLIES((lr_mode == kDontStoreLR), ((src0 != lr) && (src1 != lr) &&
+ (src2 != lr) && (src3 != lr)));
+
+#ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
+ if (lr_mode == kSignLR) {
+ Pacibsp();
+ }
+#endif
+
+ int count = 1 + src1.is_valid() + src2.is_valid() + src3.is_valid();
+ int size = src0.SizeInBytes();
+ DCHECK_EQ(0, (size * count) % 16);
+
+ PushHelper(count, size, src0, src1, src2, src3);
+}
+
+template <TurboAssembler::StoreLRMode lr_mode>
+void TurboAssembler::Push(const Register& src0, const VRegister& src1) {
+ DCHECK_IMPLIES((lr_mode == kSignLR), ((src0 == lr) || (src1 == lr)));
+ DCHECK_IMPLIES((lr_mode == kDontStoreLR), ((src0 != lr) && (src1 != lr)));
+#ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
+ if (lr_mode == kSignLR) {
+ Pacibsp();
+ }
+#endif
+
+ int size = src0.SizeInBytes() + src1.SizeInBytes();
+ DCHECK_EQ(0, size % 16);
+
+ // Reserve room for src0 and push src1.
+ str(src1, MemOperand(sp, -size, PreIndex));
+ // Fill the gap with src0.
+ str(src0, MemOperand(sp, src1.SizeInBytes()));
+}
+
+template <TurboAssembler::LoadLRMode lr_mode>
+void TurboAssembler::Pop(const CPURegister& dst0, const CPURegister& dst1,
+ const CPURegister& dst2, const CPURegister& dst3) {
+ // It is not valid to pop into the same register more than once in one
+ // instruction, not even into the zero register.
+ DCHECK(!AreAliased(dst0, dst1, dst2, dst3));
+ DCHECK(AreSameSizeAndType(dst0, dst1, dst2, dst3));
+ DCHECK(dst0.is_valid());
+
+ int count = 1 + dst1.is_valid() + dst2.is_valid() + dst3.is_valid();
+ int size = dst0.SizeInBytes();
+ DCHECK_EQ(0, (size * count) % 16);
+
+ PopHelper(count, size, dst0, dst1, dst2, dst3);
+
+ DCHECK_IMPLIES((lr_mode == kAuthLR), ((dst0 == lr) || (dst1 == lr) ||
+ (dst2 == lr) || (dst3 == lr)));
+ DCHECK_IMPLIES((lr_mode == kDontLoadLR), ((dst0 != lr) && (dst1 != lr)) &&
+ (dst2 != lr) && (dst3 != lr));
+
+#ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
+ if (lr_mode == kAuthLR) {
+ Autibsp();
+ }
+#endif
+}
+
+template <TurboAssembler::StoreLRMode lr_mode>
+void TurboAssembler::Poke(const CPURegister& src, const Operand& offset) {
+ DCHECK_IMPLIES((lr_mode == kSignLR), (src == lr));
+ DCHECK_IMPLIES((lr_mode == kDontStoreLR), (src != lr));
+#ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
+ if (lr_mode == kSignLR) {
+ Pacibsp();
+ }
+#endif
+
+ if (offset.IsImmediate()) {
+ DCHECK_GE(offset.ImmediateValue(), 0);
+ } else if (emit_debug_code()) {
+ Cmp(xzr, offset);
+ Check(le, AbortReason::kStackAccessBelowStackPointer);
+ }
+
+ Str(src, MemOperand(sp, offset));
+}
+
+template <TurboAssembler::LoadLRMode lr_mode>
+void TurboAssembler::Peek(const CPURegister& dst, const Operand& offset) {
+ if (offset.IsImmediate()) {
+ DCHECK_GE(offset.ImmediateValue(), 0);
+ } else if (emit_debug_code()) {
+ Cmp(xzr, offset);
+ Check(le, AbortReason::kStackAccessBelowStackPointer);
+ }
+
+ Ldr(dst, MemOperand(sp, offset));
+
+ DCHECK_IMPLIES((lr_mode == kAuthLR), (dst == lr));
+ DCHECK_IMPLIES((lr_mode == kDontLoadLR), (dst != lr));
+#ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
+ if (lr_mode == kAuthLR) {
+ Autibsp();
+ }
+#endif
+}
+
+template <TurboAssembler::StoreLRMode lr_mode>
+void TurboAssembler::PushCPURegList(CPURegList registers) {
+ DCHECK_IMPLIES((lr_mode == kDontStoreLR), !registers.IncludesAliasOf(lr));
+#ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
+ if (lr_mode == kSignLR && registers.IncludesAliasOf(lr)) {
+ Pacibsp();
+ }
+#endif
+
+ int size = registers.RegisterSizeInBytes();
+ DCHECK_EQ(0, (size * registers.Count()) % 16);
+
+ // Push up to four registers at a time.
+ while (!registers.IsEmpty()) {
+ int count_before = registers.Count();
+ const CPURegister& src0 = registers.PopHighestIndex();
+ const CPURegister& src1 = registers.PopHighestIndex();
+ const CPURegister& src2 = registers.PopHighestIndex();
+ const CPURegister& src3 = registers.PopHighestIndex();
+ int count = count_before - registers.Count();
+ PushHelper(count, size, src0, src1, src2, src3);
+ }
+}
+
+template <TurboAssembler::LoadLRMode lr_mode>
+void TurboAssembler::PopCPURegList(CPURegList registers) {
+ int size = registers.RegisterSizeInBytes();
+ DCHECK_EQ(0, (size * registers.Count()) % 16);
+
+#ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
+ bool contains_lr = registers.IncludesAliasOf(lr);
+ DCHECK_IMPLIES((lr_mode == kDontLoadLR), !contains_lr);
+#endif
+
+ // Pop up to four registers at a time.
+ while (!registers.IsEmpty()) {
+ int count_before = registers.Count();
+ const CPURegister& dst0 = registers.PopLowestIndex();
+ const CPURegister& dst1 = registers.PopLowestIndex();
+ const CPURegister& dst2 = registers.PopLowestIndex();
+ const CPURegister& dst3 = registers.PopLowestIndex();
+ int count = count_before - registers.Count();
+ PopHelper(count, size, dst0, dst1, dst2, dst3);
+ }
+
+#ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
+ if (lr_mode == kAuthLR && contains_lr) {
+ Autibsp();
+ }
+#endif
+}
+
+void TurboAssembler::Push(Handle<HeapObject> handle) {
+ UseScratchRegisterScope temps(this);
+ Register tmp = temps.AcquireX();
+ Mov(tmp, Operand(handle));
+ // This is only used in test-heap.cc, for generating code that is not
+ // executed. Push a padding slot together with the handle here, to
+ // satisfy the alignment requirement.
+ Push(padreg, tmp);
+}
+
+void TurboAssembler::Push(Smi smi) {
+ UseScratchRegisterScope temps(this);
+ Register tmp = temps.AcquireX();
+ Mov(tmp, Operand(smi));
+ Push(tmp);
+}
+
+void TurboAssembler::Claim(int64_t count, uint64_t unit_size) {
+ DCHECK_GE(count, 0);
+ uint64_t size = count * unit_size;
+
+ if (size == 0) {
+ return;
+ }
+ DCHECK_EQ(size % 16, 0);
+#if V8_OS_WIN
+ while (size > kStackPageSize) {
+ Sub(sp, sp, kStackPageSize);
+ Str(xzr, MemOperand(sp));
+ size -= kStackPageSize;
+ }
+#endif
+ Sub(sp, sp, size);
+}
+
+void TurboAssembler::Claim(const Register& count, uint64_t unit_size) {
+ if (unit_size == 0) return;
+ DCHECK(base::bits::IsPowerOfTwo(unit_size));
+
+ const int shift = base::bits::CountTrailingZeros(unit_size);
+ const Operand size(count, LSL, shift);
+
+ if (size.IsZero()) {
+ return;
+ }
+ AssertPositiveOrZero(count);
+
+#if V8_OS_WIN
+ // "Functions that allocate 4k or more worth of stack must ensure that each
+ // page prior to the final page is touched in order." Source:
+ // https://docs.microsoft.com/en-us/cpp/build/arm64-windows-abi-conventions?view=vs-2019#stack
+
+ // Callers expect count register to not be clobbered, so copy it.
+ UseScratchRegisterScope temps(this);
+ Register bytes_scratch = temps.AcquireX();
+ Mov(bytes_scratch, size);
+
+ Label check_offset;
+ Label touch_next_page;
+ B(&check_offset);
+ Bind(&touch_next_page);
+ Sub(sp, sp, kStackPageSize);
+ // Just to touch the page, before we increment further.
+ Str(xzr, MemOperand(sp));
+ Sub(bytes_scratch, bytes_scratch, kStackPageSize);
+
+ Bind(&check_offset);
+ Cmp(bytes_scratch, kStackPageSize);
+ B(gt, &touch_next_page);
+
+ Sub(sp, sp, bytes_scratch);
+#else
+ Sub(sp, sp, size);
+#endif
+}
+
+void TurboAssembler::Drop(int64_t count, uint64_t unit_size) {
+ DCHECK_GE(count, 0);
+ uint64_t size = count * unit_size;
+
+ if (size == 0) {
+ return;
+ }
+
+ Add(sp, sp, size);
+ DCHECK_EQ(size % 16, 0);
+}
+
+void TurboAssembler::Drop(const Register& count, uint64_t unit_size) {
+ if (unit_size == 0) return;
+ DCHECK(base::bits::IsPowerOfTwo(unit_size));
+
+ const int shift = base::bits::CountTrailingZeros(unit_size);
+ const Operand size(count, LSL, shift);
+
+ if (size.IsZero()) {
+ return;
+ }
+
+ AssertPositiveOrZero(count);
+ Add(sp, sp, size);
+}
+
+void TurboAssembler::DropArguments(const Register& count,
+ ArgumentsCountMode mode) {
+ int extra_slots = 1; // Padding slot.
+ if (mode == kCountExcludesReceiver) {
+ // Add a slot for the receiver.
+ ++extra_slots;
+ }
+ UseScratchRegisterScope temps(this);
+ Register tmp = temps.AcquireX();
+ Add(tmp, count, extra_slots);
+ Bic(tmp, tmp, 1);
+ Drop(tmp, kXRegSize);
+}
+
+void TurboAssembler::DropArguments(int64_t count, ArgumentsCountMode mode) {
+ if (mode == kCountExcludesReceiver) {
+ // Add a slot for the receiver.
+ ++count;
+ }
+ Drop(RoundUp(count, 2), kXRegSize);
+}
+
+void TurboAssembler::DropSlots(int64_t count) {
+ Drop(RoundUp(count, 2), kXRegSize);
+}
+
+void TurboAssembler::PushArgument(const Register& arg) { Push(padreg, arg); }
+
+void TurboAssembler::CompareAndBranch(const Register& lhs, const Operand& rhs,
+ Condition cond, Label* label) {
+ if (rhs.IsImmediate() && (rhs.ImmediateValue() == 0) &&
+ ((cond == eq) || (cond == ne))) {
+ if (cond == eq) {
+ Cbz(lhs, label);
+ } else {
+ Cbnz(lhs, label);
+ }
+ } else {
+ Cmp(lhs, rhs);
+ B(cond, label);
+ }
+}
+
+void TurboAssembler::CompareTaggedAndBranch(const Register& lhs,
+ const Operand& rhs, Condition cond,
+ Label* label) {
+ if (COMPRESS_POINTERS_BOOL) {
+ CompareAndBranch(lhs.W(), rhs.ToW(), cond, label);
+ } else {
+ CompareAndBranch(lhs, rhs, cond, label);
+ }
+}
+
+void TurboAssembler::TestAndBranchIfAnySet(const Register& reg,
+ const uint64_t bit_pattern,
+ Label* label) {
+ int bits = reg.SizeInBits();
+ DCHECK_GT(CountSetBits(bit_pattern, bits), 0);
+ if (CountSetBits(bit_pattern, bits) == 1) {
+ Tbnz(reg, MaskToBit(bit_pattern), label);
+ } else {
+ Tst(reg, bit_pattern);
+ B(ne, label);
+ }
+}
+
+void TurboAssembler::TestAndBranchIfAllClear(const Register& reg,
+ const uint64_t bit_pattern,
+ Label* label) {
+ int bits = reg.SizeInBits();
+ DCHECK_GT(CountSetBits(bit_pattern, bits), 0);
+ if (CountSetBits(bit_pattern, bits) == 1) {
+ Tbz(reg, MaskToBit(bit_pattern), label);
+ } else {
+ Tst(reg, bit_pattern);
+ B(eq, label);
+ }
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_ARM64_MACRO_ASSEMBLER_ARM64_INL_H_
diff --git a/src/codegen/arm64/macro-assembler-arm64.cc b/src/codegen/arm64/macro-assembler-arm64.cc
new file mode 100644
index 0000000..6924248
--- /dev/null
+++ b/src/codegen/arm64/macro-assembler-arm64.cc
@@ -0,0 +1,3405 @@
+// Copyright 2013 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#if V8_TARGET_ARCH_ARM64
+
+#include "src/base/bits.h"
+#include "src/base/division-by-constant.h"
+#include "src/codegen/assembler.h"
+#include "src/codegen/callable.h"
+#include "src/codegen/code-factory.h"
+#include "src/codegen/external-reference-table.h"
+#include "src/codegen/macro-assembler-inl.h"
+#include "src/codegen/register-configuration.h"
+#include "src/debug/debug.h"
+#include "src/deoptimizer/deoptimizer.h"
+#include "src/execution/frame-constants.h"
+#include "src/execution/frames-inl.h"
+#include "src/heap/memory-chunk.h"
+#include "src/init/bootstrapper.h"
+#include "src/logging/counters.h"
+#include "src/runtime/runtime.h"
+#include "src/snapshot/embedded/embedded-data.h"
+#include "src/snapshot/snapshot.h"
+#include "src/wasm/wasm-code-manager.h"
+
+// Satisfy cpplint check, but don't include platform-specific header. It is
+// included recursively via macro-assembler.h.
+#if 0
+#include "src/codegen/arm64/macro-assembler-arm64.h"
+#endif
+
+namespace v8 {
+namespace internal {
+
+CPURegList TurboAssembler::DefaultTmpList() { return CPURegList(ip0, ip1); }
+
+CPURegList TurboAssembler::DefaultFPTmpList() {
+ return CPURegList(fp_scratch1, fp_scratch2);
+}
+
+int TurboAssembler::RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
+ Register exclusion) const {
+ auto list = kCallerSaved;
+ list.Remove(exclusion);
+ list.Align();
+
+ int bytes = list.Count() * kXRegSizeInBits / 8;
+
+ if (fp_mode == kSaveFPRegs) {
+ DCHECK_EQ(kCallerSavedV.Count() % 2, 0);
+ bytes += kCallerSavedV.Count() * kDRegSizeInBits / 8;
+ }
+ return bytes;
+}
+
+int TurboAssembler::PushCallerSaved(SaveFPRegsMode fp_mode,
+ Register exclusion) {
+ auto list = kCallerSaved;
+ list.Remove(exclusion);
+ list.Align();
+
+ PushCPURegList<kDontStoreLR>(list);
+
+ int bytes = list.Count() * kXRegSizeInBits / 8;
+
+ if (fp_mode == kSaveFPRegs) {
+ DCHECK_EQ(kCallerSavedV.Count() % 2, 0);
+ PushCPURegList(kCallerSavedV);
+ bytes += kCallerSavedV.Count() * kDRegSizeInBits / 8;
+ }
+ return bytes;
+}
+
+int TurboAssembler::PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion) {
+ int bytes = 0;
+ if (fp_mode == kSaveFPRegs) {
+ DCHECK_EQ(kCallerSavedV.Count() % 2, 0);
+ PopCPURegList(kCallerSavedV);
+ bytes += kCallerSavedV.Count() * kDRegSizeInBits / 8;
+ }
+
+ auto list = kCallerSaved;
+ list.Remove(exclusion);
+ list.Align();
+
+ PopCPURegList<kDontLoadLR>(list);
+ bytes += list.Count() * kXRegSizeInBits / 8;
+
+ return bytes;
+}
+
+void TurboAssembler::LogicalMacro(const Register& rd, const Register& rn,
+ const Operand& operand, LogicalOp op) {
+ UseScratchRegisterScope temps(this);
+
+ if (operand.NeedsRelocation(this)) {
+ Register temp = temps.AcquireX();
+ Ldr(temp, operand.immediate());
+ Logical(rd, rn, temp, op);
+
+ } else if (operand.IsImmediate()) {
+ int64_t immediate = operand.ImmediateValue();
+ unsigned reg_size = rd.SizeInBits();
+
+ // If the operation is NOT, invert the operation and immediate.
+ if ((op & NOT) == NOT) {
+ op = static_cast<LogicalOp>(op & ~NOT);
+ immediate = ~immediate;
+ }
+
+ // Ignore the top 32 bits of an immediate if we're moving to a W register.
+ if (rd.Is32Bits()) {
+ // Check that the top 32 bits are consistent.
+ DCHECK(((immediate >> kWRegSizeInBits) == 0) ||
+ ((immediate >> kWRegSizeInBits) == -1));
+ immediate &= kWRegMask;
+ }
+
+ DCHECK(rd.Is64Bits() || is_uint32(immediate));
+
+ // Special cases for all set or all clear immediates.
+ if (immediate == 0) {
+ switch (op) {
+ case AND:
+ Mov(rd, 0);
+ return;
+ case ORR: // Fall through.
+ case EOR:
+ Mov(rd, rn);
+ return;
+ case ANDS: // Fall through.
+ case BICS:
+ break;
+ default:
+ UNREACHABLE();
+ }
+ } else if ((rd.Is64Bits() && (immediate == -1L)) ||
+ (rd.Is32Bits() && (immediate == 0xFFFFFFFFL))) {
+ switch (op) {
+ case AND:
+ Mov(rd, rn);
+ return;
+ case ORR:
+ Mov(rd, immediate);
+ return;
+ case EOR:
+ Mvn(rd, rn);
+ return;
+ case ANDS: // Fall through.
+ case BICS:
+ break;
+ default:
+ UNREACHABLE();
+ }
+ }
+
+ unsigned n, imm_s, imm_r;
+ if (IsImmLogical(immediate, reg_size, &n, &imm_s, &imm_r)) {
+ // Immediate can be encoded in the instruction.
+ LogicalImmediate(rd, rn, n, imm_s, imm_r, op);
+ } else {
+ // Immediate can't be encoded: synthesize using move immediate.
+ Register temp = temps.AcquireSameSizeAs(rn);
+
+ // If the left-hand input is the stack pointer, we can't pre-shift the
+ // immediate, as the encoding won't allow the subsequent post shift.
+ PreShiftImmMode mode = rn == sp ? kNoShift : kAnyShift;
+ Operand imm_operand = MoveImmediateForShiftedOp(temp, immediate, mode);
+
+ if (rd.IsSP()) {
+ // If rd is the stack pointer we cannot use it as the destination
+ // register so we use the temp register as an intermediate again.
+ Logical(temp, rn, imm_operand, op);
+ Mov(sp, temp);
+ } else {
+ Logical(rd, rn, imm_operand, op);
+ }
+ }
+
+ } else if (operand.IsExtendedRegister()) {
+ DCHECK(operand.reg().SizeInBits() <= rd.SizeInBits());
+ // Add/sub extended supports shift <= 4. We want to support exactly the
+ // same modes here.
+ DCHECK_LE(operand.shift_amount(), 4);
+ DCHECK(operand.reg().Is64Bits() ||
+ ((operand.extend() != UXTX) && (operand.extend() != SXTX)));
+ Register temp = temps.AcquireSameSizeAs(rn);
+ EmitExtendShift(temp, operand.reg(), operand.extend(),
+ operand.shift_amount());
+ Logical(rd, rn, temp, op);
+
+ } else {
+ // The operand can be encoded in the instruction.
+ DCHECK(operand.IsShiftedRegister());
+ Logical(rd, rn, operand, op);
+ }
+}
+
+void TurboAssembler::Mov(const Register& rd, uint64_t imm) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(is_uint32(imm) || is_int32(imm) || rd.Is64Bits());
+ DCHECK(!rd.IsZero());
+
+ // TODO(all) extend to support more immediates.
+ //
+ // Immediates on Aarch64 can be produced using an initial value, and zero to
+ // three move keep operations.
+ //
+ // Initial values can be generated with:
+ // 1. 64-bit move zero (movz).
+ // 2. 32-bit move inverted (movn).
+ // 3. 64-bit move inverted.
+ // 4. 32-bit orr immediate.
+ // 5. 64-bit orr immediate.
+ // Move-keep may then be used to modify each of the 16-bit half-words.
+ //
+ // The code below supports all five initial value generators, and
+ // applying move-keep operations to move-zero and move-inverted initial
+ // values.
+
+ // Try to move the immediate in one instruction, and if that fails, switch to
+ // using multiple instructions.
+ if (!TryOneInstrMoveImmediate(rd, imm)) {
+ unsigned reg_size = rd.SizeInBits();
+
+ // Generic immediate case. Imm will be represented by
+ // [imm3, imm2, imm1, imm0], where each imm is 16 bits.
+ // A move-zero or move-inverted is generated for the first non-zero or
+ // non-0xFFFF immX, and a move-keep for subsequent non-zero immX.
+
+ uint64_t ignored_halfword = 0;
+ bool invert_move = false;
+ // If the number of 0xFFFF halfwords is greater than the number of 0x0000
+ // halfwords, it's more efficient to use move-inverted.
+ if (CountClearHalfWords(~imm, reg_size) >
+ CountClearHalfWords(imm, reg_size)) {
+ ignored_halfword = 0xFFFFL;
+ invert_move = true;
+ }
+
+ // Mov instructions can't move immediate values into the stack pointer, so
+ // set up a temporary register, if needed.
+ UseScratchRegisterScope temps(this);
+ Register temp = rd.IsSP() ? temps.AcquireSameSizeAs(rd) : rd;
+
+ // Iterate through the halfwords. Use movn/movz for the first non-ignored
+ // halfword, and movk for subsequent halfwords.
+ DCHECK_EQ(reg_size % 16, 0);
+ bool first_mov_done = false;
+ for (int i = 0; i < (rd.SizeInBits() / 16); i++) {
+ uint64_t imm16 = (imm >> (16 * i)) & 0xFFFFL;
+ if (imm16 != ignored_halfword) {
+ if (!first_mov_done) {
+ if (invert_move) {
+ movn(temp, (~imm16) & 0xFFFFL, 16 * i);
+ } else {
+ movz(temp, imm16, 16 * i);
+ }
+ first_mov_done = true;
+ } else {
+ // Construct a wider constant.
+ movk(temp, imm16, 16 * i);
+ }
+ }
+ }
+ DCHECK(first_mov_done);
+
+ // Move the temporary if the original destination register was the stack
+ // pointer.
+ if (rd.IsSP()) {
+ mov(rd, temp);
+ }
+ }
+}
+
+void TurboAssembler::Mov(const Register& rd, const Operand& operand,
+ DiscardMoveMode discard_mode) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+
+ // Provide a swap register for instructions that need to write into the
+ // system stack pointer (and can't do this inherently).
+ UseScratchRegisterScope temps(this);
+ Register dst = (rd.IsSP()) ? temps.AcquireSameSizeAs(rd) : rd;
+
+ if (operand.NeedsRelocation(this)) {
+ // TODO(jgruber,v8:8887): Also consider a root-relative load when generating
+ // non-isolate-independent code. In many cases it might be cheaper than
+ // embedding the relocatable value.
+ if (root_array_available_ && options().isolate_independent_code) {
+ if (operand.ImmediateRMode() == RelocInfo::EXTERNAL_REFERENCE) {
+ Address addr = static_cast<Address>(operand.ImmediateValue());
+ ExternalReference reference = bit_cast<ExternalReference>(addr);
+ IndirectLoadExternalReference(rd, reference);
+ return;
+ } else if (RelocInfo::IsEmbeddedObjectMode(operand.ImmediateRMode())) {
+ Handle<HeapObject> x(
+ reinterpret_cast<Address*>(operand.ImmediateValue()));
+ // TODO(v8:9706): Fix-it! This load will always uncompress the value
+ // even when we are loading a compressed embedded object.
+ IndirectLoadConstant(rd.X(), x);
+ return;
+ }
+ }
+ Ldr(dst, operand);
+ } else if (operand.IsImmediate()) {
+ // Call the macro assembler for generic immediates.
+ Mov(dst, operand.ImmediateValue());
+ } else if (operand.IsShiftedRegister() && (operand.shift_amount() != 0)) {
+ // Emit a shift instruction if moving a shifted register. This operation
+ // could also be achieved using an orr instruction (like orn used by Mvn),
+ // but using a shift instruction makes the disassembly clearer.
+ EmitShift(dst, operand.reg(), operand.shift(), operand.shift_amount());
+ } else if (operand.IsExtendedRegister()) {
+ // Emit an extend instruction if moving an extended register. This handles
+ // extend with post-shift operations, too.
+ EmitExtendShift(dst, operand.reg(), operand.extend(),
+ operand.shift_amount());
+ } else {
+ // Otherwise, emit a register move only if the registers are distinct, or
+ // if they are not X registers.
+ //
+ // Note that mov(w0, w0) is not a no-op because it clears the top word of
+ // x0. A flag is provided (kDiscardForSameWReg) if a move between the same W
+ // registers is not required to clear the top word of the X register. In
+ // this case, the instruction is discarded.
+ //
+ // If sp is an operand, add #0 is emitted, otherwise, orr #0.
+ if (rd != operand.reg() ||
+ (rd.Is32Bits() && (discard_mode == kDontDiscardForSameWReg))) {
+ Assembler::mov(rd, operand.reg());
+ }
+ // This case can handle writes into the system stack pointer directly.
+ dst = rd;
+ }
+
+ // Copy the result to the system stack pointer.
+ if (dst != rd) {
+ DCHECK(rd.IsSP());
+ Assembler::mov(rd, dst);
+ }
+}
+
+void TurboAssembler::Mov(const Register& rd, Smi smi) {
+ return Mov(rd, Operand(smi));
+}
+
+void TurboAssembler::Movi16bitHelper(const VRegister& vd, uint64_t imm) {
+ DCHECK(is_uint16(imm));
+ int byte1 = (imm & 0xFF);
+ int byte2 = ((imm >> 8) & 0xFF);
+ if (byte1 == byte2) {
+ movi(vd.Is64Bits() ? vd.V8B() : vd.V16B(), byte1);
+ } else if (byte1 == 0) {
+ movi(vd, byte2, LSL, 8);
+ } else if (byte2 == 0) {
+ movi(vd, byte1);
+ } else if (byte1 == 0xFF) {
+ mvni(vd, ~byte2 & 0xFF, LSL, 8);
+ } else if (byte2 == 0xFF) {
+ mvni(vd, ~byte1 & 0xFF);
+ } else {
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireW();
+ movz(temp, imm);
+ dup(vd, temp);
+ }
+}
+
+void TurboAssembler::Movi32bitHelper(const VRegister& vd, uint64_t imm) {
+ DCHECK(is_uint32(imm));
+
+ uint8_t bytes[sizeof(imm)];
+ memcpy(bytes, &imm, sizeof(imm));
+
+ // All bytes are either 0x00 or 0xFF.
+ {
+ bool all0orff = true;
+ for (int i = 0; i < 4; ++i) {
+ if ((bytes[i] != 0) && (bytes[i] != 0xFF)) {
+ all0orff = false;
+ break;
+ }
+ }
+
+ if (all0orff == true) {
+ movi(vd.Is64Bits() ? vd.V1D() : vd.V2D(), ((imm << 32) | imm));
+ return;
+ }
+ }
+
+ // Of the 4 bytes, only one byte is non-zero.
+ for (int i = 0; i < 4; i++) {
+ if ((imm & (0xFF << (i * 8))) == imm) {
+ movi(vd, bytes[i], LSL, i * 8);
+ return;
+ }
+ }
+
+ // Of the 4 bytes, only one byte is not 0xFF.
+ for (int i = 0; i < 4; i++) {
+ uint32_t mask = ~(0xFF << (i * 8));
+ if ((imm & mask) == mask) {
+ mvni(vd, ~bytes[i] & 0xFF, LSL, i * 8);
+ return;
+ }
+ }
+
+ // Immediate is of the form 0x00MMFFFF.
+ if ((imm & 0xFF00FFFF) == 0x0000FFFF) {
+ movi(vd, bytes[2], MSL, 16);
+ return;
+ }
+
+ // Immediate is of the form 0x0000MMFF.
+ if ((imm & 0xFFFF00FF) == 0x000000FF) {
+ movi(vd, bytes[1], MSL, 8);
+ return;
+ }
+
+ // Immediate is of the form 0xFFMM0000.
+ if ((imm & 0xFF00FFFF) == 0xFF000000) {
+ mvni(vd, ~bytes[2] & 0xFF, MSL, 16);
+ return;
+ }
+ // Immediate is of the form 0xFFFFMM00.
+ if ((imm & 0xFFFF00FF) == 0xFFFF0000) {
+ mvni(vd, ~bytes[1] & 0xFF, MSL, 8);
+ return;
+ }
+
+ // Top and bottom 16-bits are equal.
+ if (((imm >> 16) & 0xFFFF) == (imm & 0xFFFF)) {
+ Movi16bitHelper(vd.Is64Bits() ? vd.V4H() : vd.V8H(), imm & 0xFFFF);
+ return;
+ }
+
+ // Default case.
+ {
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireW();
+ Mov(temp, imm);
+ dup(vd, temp);
+ }
+}
+
+void TurboAssembler::Movi64bitHelper(const VRegister& vd, uint64_t imm) {
+ // All bytes are either 0x00 or 0xFF.
+ {
+ bool all0orff = true;
+ for (int i = 0; i < 8; ++i) {
+ int byteval = (imm >> (i * 8)) & 0xFF;
+ if (byteval != 0 && byteval != 0xFF) {
+ all0orff = false;
+ break;
+ }
+ }
+ if (all0orff == true) {
+ movi(vd, imm);
+ return;
+ }
+ }
+
+ // Top and bottom 32-bits are equal.
+ if (((imm >> 32) & 0xFFFFFFFF) == (imm & 0xFFFFFFFF)) {
+ Movi32bitHelper(vd.Is64Bits() ? vd.V2S() : vd.V4S(), imm & 0xFFFFFFFF);
+ return;
+ }
+
+ // Default case.
+ {
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireX();
+ Mov(temp, imm);
+ if (vd.Is1D()) {
+ mov(vd.D(), 0, temp);
+ } else {
+ dup(vd.V2D(), temp);
+ }
+ }
+}
+
+void TurboAssembler::Movi(const VRegister& vd, uint64_t imm, Shift shift,
+ int shift_amount) {
+ DCHECK(allow_macro_instructions());
+ if (shift_amount != 0 || shift != LSL) {
+ movi(vd, imm, shift, shift_amount);
+ } else if (vd.Is8B() || vd.Is16B()) {
+ // 8-bit immediate.
+ DCHECK(is_uint8(imm));
+ movi(vd, imm);
+ } else if (vd.Is4H() || vd.Is8H()) {
+ // 16-bit immediate.
+ Movi16bitHelper(vd, imm);
+ } else if (vd.Is2S() || vd.Is4S()) {
+ // 32-bit immediate.
+ Movi32bitHelper(vd, imm);
+ } else {
+ // 64-bit immediate.
+ Movi64bitHelper(vd, imm);
+ }
+}
+
+void TurboAssembler::Movi(const VRegister& vd, uint64_t hi, uint64_t lo) {
+ // TODO(v8:11033): Move 128-bit values in a more efficient way.
+ DCHECK(vd.Is128Bits());
+ Movi(vd.V2D(), lo);
+ if (lo != hi) {
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireX();
+ Mov(temp, hi);
+ Ins(vd.V2D(), 1, temp);
+ }
+}
+
+void TurboAssembler::Mvn(const Register& rd, const Operand& operand) {
+ DCHECK(allow_macro_instructions());
+
+ if (operand.NeedsRelocation(this)) {
+ Ldr(rd, operand.immediate());
+ mvn(rd, rd);
+
+ } else if (operand.IsImmediate()) {
+ // Call the macro assembler for generic immediates.
+ Mov(rd, ~operand.ImmediateValue());
+
+ } else if (operand.IsExtendedRegister()) {
+ // Emit two instructions for the extend case. This differs from Mov, as
+ // the extend and invert can't be achieved in one instruction.
+ EmitExtendShift(rd, operand.reg(), operand.extend(),
+ operand.shift_amount());
+ mvn(rd, rd);
+
+ } else {
+ mvn(rd, operand);
+ }
+}
+
+unsigned TurboAssembler::CountClearHalfWords(uint64_t imm, unsigned reg_size) {
+ DCHECK_EQ(reg_size % 8, 0);
+ int count = 0;
+ for (unsigned i = 0; i < (reg_size / 16); i++) {
+ if ((imm & 0xFFFF) == 0) {
+ count++;
+ }
+ imm >>= 16;
+ }
+ return count;
+}
+
+// The movz instruction can generate immediates containing an arbitrary 16-bit
+// half-word, with remaining bits clear, eg. 0x00001234, 0x0000123400000000.
+bool TurboAssembler::IsImmMovz(uint64_t imm, unsigned reg_size) {
+ DCHECK((reg_size == kXRegSizeInBits) || (reg_size == kWRegSizeInBits));
+ return CountClearHalfWords(imm, reg_size) >= ((reg_size / 16) - 1);
+}
+
+// The movn instruction can generate immediates containing an arbitrary 16-bit
+// half-word, with remaining bits set, eg. 0xFFFF1234, 0xFFFF1234FFFFFFFF.
+bool TurboAssembler::IsImmMovn(uint64_t imm, unsigned reg_size) {
+ return IsImmMovz(~imm, reg_size);
+}
+
+void TurboAssembler::ConditionalCompareMacro(const Register& rn,
+ const Operand& operand,
+ StatusFlags nzcv, Condition cond,
+ ConditionalCompareOp op) {
+ DCHECK((cond != al) && (cond != nv));
+ if (operand.NeedsRelocation(this)) {
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireX();
+ Ldr(temp, operand.immediate());
+ ConditionalCompareMacro(rn, temp, nzcv, cond, op);
+
+ } else if ((operand.IsShiftedRegister() && (operand.shift_amount() == 0)) ||
+ (operand.IsImmediate() &&
+ IsImmConditionalCompare(operand.ImmediateValue()))) {
+ // The immediate can be encoded in the instruction, or the operand is an
+ // unshifted register: call the assembler.
+ ConditionalCompare(rn, operand, nzcv, cond, op);
+
+ } else {
+ // The operand isn't directly supported by the instruction: perform the
+ // operation on a temporary register.
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireSameSizeAs(rn);
+ Mov(temp, operand);
+ ConditionalCompare(rn, temp, nzcv, cond, op);
+ }
+}
+
+void TurboAssembler::Csel(const Register& rd, const Register& rn,
+ const Operand& operand, Condition cond) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ DCHECK((cond != al) && (cond != nv));
+ if (operand.IsImmediate()) {
+ // Immediate argument. Handle special cases of 0, 1 and -1 using zero
+ // register.
+ int64_t imm = operand.ImmediateValue();
+ Register zr = AppropriateZeroRegFor(rn);
+ if (imm == 0) {
+ csel(rd, rn, zr, cond);
+ } else if (imm == 1) {
+ csinc(rd, rn, zr, cond);
+ } else if (imm == -1) {
+ csinv(rd, rn, zr, cond);
+ } else {
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireSameSizeAs(rn);
+ Mov(temp, imm);
+ csel(rd, rn, temp, cond);
+ }
+ } else if (operand.IsShiftedRegister() && (operand.shift_amount() == 0)) {
+ // Unshifted register argument.
+ csel(rd, rn, operand.reg(), cond);
+ } else {
+ // All other arguments.
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireSameSizeAs(rn);
+ Mov(temp, operand);
+ csel(rd, rn, temp, cond);
+ }
+}
+
+bool TurboAssembler::TryOneInstrMoveImmediate(const Register& dst,
+ int64_t imm) {
+ unsigned n, imm_s, imm_r;
+ int reg_size = dst.SizeInBits();
+ if (IsImmMovz(imm, reg_size) && !dst.IsSP()) {
+ // Immediate can be represented in a move zero instruction. Movz can't write
+ // to the stack pointer.
+ movz(dst, imm);
+ return true;
+ } else if (IsImmMovn(imm, reg_size) && !dst.IsSP()) {
+ // Immediate can be represented in a move not instruction. Movn can't write
+ // to the stack pointer.
+ movn(dst, dst.Is64Bits() ? ~imm : (~imm & kWRegMask));
+ return true;
+ } else if (IsImmLogical(imm, reg_size, &n, &imm_s, &imm_r)) {
+ // Immediate can be represented in a logical orr instruction.
+ LogicalImmediate(dst, AppropriateZeroRegFor(dst), n, imm_s, imm_r, ORR);
+ return true;
+ }
+ return false;
+}
+
+Operand TurboAssembler::MoveImmediateForShiftedOp(const Register& dst,
+ int64_t imm,
+ PreShiftImmMode mode) {
+ int reg_size = dst.SizeInBits();
+ // Encode the immediate in a single move instruction, if possible.
+ if (TryOneInstrMoveImmediate(dst, imm)) {
+ // The move was successful; nothing to do here.
+ } else {
+ // Pre-shift the immediate to the least-significant bits of the register.
+ int shift_low;
+ if (reg_size == 64) {
+ shift_low = base::bits::CountTrailingZeros(imm);
+ } else {
+ DCHECK_EQ(reg_size, 32);
+ shift_low = base::bits::CountTrailingZeros(static_cast<uint32_t>(imm));
+ }
+
+ if (mode == kLimitShiftForSP) {
+ // When applied to the stack pointer, the subsequent arithmetic operation
+ // can use the extend form to shift left by a maximum of four bits. Right
+ // shifts are not allowed, so we filter them out later before the new
+ // immediate is tested.
+ shift_low = std::min(shift_low, 4);
+ }
+ int64_t imm_low = imm >> shift_low;
+
+ // Pre-shift the immediate to the most-significant bits of the register. We
+ // insert set bits in the least-significant bits, as this creates a
+ // different immediate that may be encodable using movn or orr-immediate.
+ // If this new immediate is encodable, the set bits will be eliminated by
+ // the post shift on the following instruction.
+ int shift_high = CountLeadingZeros(imm, reg_size);
+ int64_t imm_high = (imm << shift_high) | ((INT64_C(1) << shift_high) - 1);
+
+ if ((mode != kNoShift) && TryOneInstrMoveImmediate(dst, imm_low)) {
+ // The new immediate has been moved into the destination's low bits:
+ // return a new leftward-shifting operand.
+ return Operand(dst, LSL, shift_low);
+ } else if ((mode == kAnyShift) && TryOneInstrMoveImmediate(dst, imm_high)) {
+ // The new immediate has been moved into the destination's high bits:
+ // return a new rightward-shifting operand.
+ return Operand(dst, LSR, shift_high);
+ } else {
+ // Use the generic move operation to set up the immediate.
+ Mov(dst, imm);
+ }
+ }
+ return Operand(dst);
+}
+
+void TurboAssembler::AddSubMacro(const Register& rd, const Register& rn,
+ const Operand& operand, FlagsUpdate S,
+ AddSubOp op) {
+ if (operand.IsZero() && rd == rn && rd.Is64Bits() && rn.Is64Bits() &&
+ !operand.NeedsRelocation(this) && (S == LeaveFlags)) {
+ // The instruction would be a nop. Avoid generating useless code.
+ return;
+ }
+
+ if (operand.NeedsRelocation(this)) {
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireX();
+ Ldr(temp, operand.immediate());
+ AddSubMacro(rd, rn, temp, S, op);
+ } else if ((operand.IsImmediate() &&
+ !IsImmAddSub(operand.ImmediateValue())) ||
+ (rn.IsZero() && !operand.IsShiftedRegister()) ||
+ (operand.IsShiftedRegister() && (operand.shift() == ROR))) {
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireSameSizeAs(rn);
+ if (operand.IsImmediate()) {
+ PreShiftImmMode mode = kAnyShift;
+
+ // If the destination or source register is the stack pointer, we can
+ // only pre-shift the immediate right by values supported in the add/sub
+ // extend encoding.
+ if (rd == sp) {
+ // If the destination is SP and flags will be set, we can't pre-shift
+ // the immediate at all.
+ mode = (S == SetFlags) ? kNoShift : kLimitShiftForSP;
+ } else if (rn == sp) {
+ mode = kLimitShiftForSP;
+ }
+
+ Operand imm_operand =
+ MoveImmediateForShiftedOp(temp, operand.ImmediateValue(), mode);
+ AddSub(rd, rn, imm_operand, S, op);
+ } else {
+ Mov(temp, operand);
+ AddSub(rd, rn, temp, S, op);
+ }
+ } else {
+ AddSub(rd, rn, operand, S, op);
+ }
+}
+
+void TurboAssembler::AddSubWithCarryMacro(const Register& rd,
+ const Register& rn,
+ const Operand& operand, FlagsUpdate S,
+ AddSubWithCarryOp op) {
+ DCHECK(rd.SizeInBits() == rn.SizeInBits());
+ UseScratchRegisterScope temps(this);
+
+ if (operand.NeedsRelocation(this)) {
+ Register temp = temps.AcquireX();
+ Ldr(temp, operand.immediate());
+ AddSubWithCarryMacro(rd, rn, temp, S, op);
+
+ } else if (operand.IsImmediate() ||
+ (operand.IsShiftedRegister() && (operand.shift() == ROR))) {
+ // Add/sub with carry (immediate or ROR shifted register.)
+ Register temp = temps.AcquireSameSizeAs(rn);
+ Mov(temp, operand);
+ AddSubWithCarry(rd, rn, temp, S, op);
+
+ } else if (operand.IsShiftedRegister() && (operand.shift_amount() != 0)) {
+ // Add/sub with carry (shifted register).
+ DCHECK(operand.reg().SizeInBits() == rd.SizeInBits());
+ DCHECK(operand.shift() != ROR);
+ DCHECK(is_uintn(operand.shift_amount(), rd.SizeInBits() == kXRegSizeInBits
+ ? kXRegSizeInBitsLog2
+ : kWRegSizeInBitsLog2));
+ Register temp = temps.AcquireSameSizeAs(rn);
+ EmitShift(temp, operand.reg(), operand.shift(), operand.shift_amount());
+ AddSubWithCarry(rd, rn, temp, S, op);
+
+ } else if (operand.IsExtendedRegister()) {
+ // Add/sub with carry (extended register).
+ DCHECK(operand.reg().SizeInBits() <= rd.SizeInBits());
+ // Add/sub extended supports a shift <= 4. We want to support exactly the
+ // same modes.
+ DCHECK_LE(operand.shift_amount(), 4);
+ DCHECK(operand.reg().Is64Bits() ||
+ ((operand.extend() != UXTX) && (operand.extend() != SXTX)));
+ Register temp = temps.AcquireSameSizeAs(rn);
+ EmitExtendShift(temp, operand.reg(), operand.extend(),
+ operand.shift_amount());
+ AddSubWithCarry(rd, rn, temp, S, op);
+
+ } else {
+ // The addressing mode is directly supported by the instruction.
+ AddSubWithCarry(rd, rn, operand, S, op);
+ }
+}
+
+void TurboAssembler::LoadStoreMacro(const CPURegister& rt,
+ const MemOperand& addr, LoadStoreOp op) {
+ int64_t offset = addr.offset();
+ unsigned size = CalcLSDataSize(op);
+
+ // Check if an immediate offset fits in the immediate field of the
+ // appropriate instruction. If not, emit two instructions to perform
+ // the operation.
+ if (addr.IsImmediateOffset() && !IsImmLSScaled(offset, size) &&
+ !IsImmLSUnscaled(offset)) {
+ // Immediate offset that can't be encoded using unsigned or unscaled
+ // addressing modes.
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireSameSizeAs(addr.base());
+ Mov(temp, addr.offset());
+ LoadStore(rt, MemOperand(addr.base(), temp), op);
+ } else if (addr.IsPostIndex() && !IsImmLSUnscaled(offset)) {
+ // Post-index beyond unscaled addressing range.
+ LoadStore(rt, MemOperand(addr.base()), op);
+ add(addr.base(), addr.base(), offset);
+ } else if (addr.IsPreIndex() && !IsImmLSUnscaled(offset)) {
+ // Pre-index beyond unscaled addressing range.
+ add(addr.base(), addr.base(), offset);
+ LoadStore(rt, MemOperand(addr.base()), op);
+ } else {
+ // Encodable in one load/store instruction.
+ LoadStore(rt, addr, op);
+ }
+}
+
+void TurboAssembler::LoadStorePairMacro(const CPURegister& rt,
+ const CPURegister& rt2,
+ const MemOperand& addr,
+ LoadStorePairOp op) {
+ // TODO(all): Should we support register offset for load-store-pair?
+ DCHECK(!addr.IsRegisterOffset());
+
+ int64_t offset = addr.offset();
+ unsigned size = CalcLSPairDataSize(op);
+
+ // Check if the offset fits in the immediate field of the appropriate
+ // instruction. If not, emit two instructions to perform the operation.
+ if (IsImmLSPair(offset, size)) {
+ // Encodable in one load/store pair instruction.
+ LoadStorePair(rt, rt2, addr, op);
+ } else {
+ Register base = addr.base();
+ if (addr.IsImmediateOffset()) {
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireSameSizeAs(base);
+ Add(temp, base, offset);
+ LoadStorePair(rt, rt2, MemOperand(temp), op);
+ } else if (addr.IsPostIndex()) {
+ LoadStorePair(rt, rt2, MemOperand(base), op);
+ Add(base, base, offset);
+ } else {
+ DCHECK(addr.IsPreIndex());
+ Add(base, base, offset);
+ LoadStorePair(rt, rt2, MemOperand(base), op);
+ }
+ }
+}
+
+bool TurboAssembler::NeedExtraInstructionsOrRegisterBranch(
+ Label* label, ImmBranchType b_type) {
+ bool need_longer_range = false;
+ // There are two situations in which we care about the offset being out of
+ // range:
+ // - The label is bound but too far away.
+ // - The label is not bound but linked, and the previous branch
+ // instruction in the chain is too far away.
+ if (label->is_bound() || label->is_linked()) {
+ need_longer_range =
+ !Instruction::IsValidImmPCOffset(b_type, label->pos() - pc_offset());
+ }
+ if (!need_longer_range && !label->is_bound()) {
+ int max_reachable_pc = pc_offset() + Instruction::ImmBranchRange(b_type);
+ unresolved_branches_.insert(std::pair<int, FarBranchInfo>(
+ max_reachable_pc, FarBranchInfo(pc_offset(), label)));
+ // Also maintain the next pool check.
+ next_veneer_pool_check_ = std::min(
+ next_veneer_pool_check_, max_reachable_pc - kVeneerDistanceCheckMargin);
+ }
+ return need_longer_range;
+}
+
+void TurboAssembler::Adr(const Register& rd, Label* label, AdrHint hint) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+
+ if (hint == kAdrNear) {
+ adr(rd, label);
+ return;
+ }
+
+ DCHECK_EQ(hint, kAdrFar);
+ if (label->is_bound()) {
+ int label_offset = label->pos() - pc_offset();
+ if (Instruction::IsValidPCRelOffset(label_offset)) {
+ adr(rd, label);
+ } else {
+ DCHECK_LE(label_offset, 0);
+ int min_adr_offset = -(1 << (Instruction::ImmPCRelRangeBitwidth - 1));
+ adr(rd, min_adr_offset);
+ Add(rd, rd, label_offset - min_adr_offset);
+ }
+ } else {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.AcquireX();
+
+ InstructionAccurateScope scope(this,
+ PatchingAssembler::kAdrFarPatchableNInstrs);
+ adr(rd, label);
+ for (int i = 0; i < PatchingAssembler::kAdrFarPatchableNNops; ++i) {
+ nop(ADR_FAR_NOP);
+ }
+ movz(scratch, 0);
+ }
+}
+
+void TurboAssembler::B(Label* label, BranchType type, Register reg, int bit) {
+ DCHECK((reg == NoReg || type >= kBranchTypeFirstUsingReg) &&
+ (bit == -1 || type >= kBranchTypeFirstUsingBit));
+ if (kBranchTypeFirstCondition <= type && type <= kBranchTypeLastCondition) {
+ B(static_cast<Condition>(type), label);
+ } else {
+ switch (type) {
+ case always:
+ B(label);
+ break;
+ case never:
+ break;
+ case reg_zero:
+ Cbz(reg, label);
+ break;
+ case reg_not_zero:
+ Cbnz(reg, label);
+ break;
+ case reg_bit_clear:
+ Tbz(reg, bit, label);
+ break;
+ case reg_bit_set:
+ Tbnz(reg, bit, label);
+ break;
+ default:
+ UNREACHABLE();
+ }
+ }
+}
+
+void TurboAssembler::B(Label* label, Condition cond) {
+ DCHECK(allow_macro_instructions());
+ DCHECK((cond != al) && (cond != nv));
+
+ Label done;
+ bool need_extra_instructions =
+ NeedExtraInstructionsOrRegisterBranch(label, CondBranchType);
+
+ if (need_extra_instructions) {
+ b(&done, NegateCondition(cond));
+ B(label);
+ } else {
+ b(label, cond);
+ }
+ bind(&done);
+}
+
+void TurboAssembler::Tbnz(const Register& rt, unsigned bit_pos, Label* label) {
+ DCHECK(allow_macro_instructions());
+
+ Label done;
+ bool need_extra_instructions =
+ NeedExtraInstructionsOrRegisterBranch(label, TestBranchType);
+
+ if (need_extra_instructions) {
+ tbz(rt, bit_pos, &done);
+ B(label);
+ } else {
+ tbnz(rt, bit_pos, label);
+ }
+ bind(&done);
+}
+
+void TurboAssembler::Tbz(const Register& rt, unsigned bit_pos, Label* label) {
+ DCHECK(allow_macro_instructions());
+
+ Label done;
+ bool need_extra_instructions =
+ NeedExtraInstructionsOrRegisterBranch(label, TestBranchType);
+
+ if (need_extra_instructions) {
+ tbnz(rt, bit_pos, &done);
+ B(label);
+ } else {
+ tbz(rt, bit_pos, label);
+ }
+ bind(&done);
+}
+
+void TurboAssembler::Cbnz(const Register& rt, Label* label) {
+ DCHECK(allow_macro_instructions());
+
+ Label done;
+ bool need_extra_instructions =
+ NeedExtraInstructionsOrRegisterBranch(label, CompareBranchType);
+
+ if (need_extra_instructions) {
+ cbz(rt, &done);
+ B(label);
+ } else {
+ cbnz(rt, label);
+ }
+ bind(&done);
+}
+
+void TurboAssembler::Cbz(const Register& rt, Label* label) {
+ DCHECK(allow_macro_instructions());
+
+ Label done;
+ bool need_extra_instructions =
+ NeedExtraInstructionsOrRegisterBranch(label, CompareBranchType);
+
+ if (need_extra_instructions) {
+ cbnz(rt, &done);
+ B(label);
+ } else {
+ cbz(rt, label);
+ }
+ bind(&done);
+}
+
+// Pseudo-instructions.
+
+void TurboAssembler::Abs(const Register& rd, const Register& rm,
+ Label* is_not_representable, Label* is_representable) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(AreSameSizeAndType(rd, rm));
+
+ Cmp(rm, 1);
+ Cneg(rd, rm, lt);
+
+ // If the comparison sets the v flag, the input was the smallest value
+ // representable by rm, and the mathematical result of abs(rm) is not
+ // representable using two's complement.
+ if ((is_not_representable != nullptr) && (is_representable != nullptr)) {
+ B(is_not_representable, vs);
+ B(is_representable);
+ } else if (is_not_representable != nullptr) {
+ B(is_not_representable, vs);
+ } else if (is_representable != nullptr) {
+ B(is_representable, vc);
+ }
+}
+
+// Abstracted stack operations.
+
+void TurboAssembler::Push(const CPURegister& src0, const CPURegister& src1,
+ const CPURegister& src2, const CPURegister& src3,
+ const CPURegister& src4, const CPURegister& src5,
+ const CPURegister& src6, const CPURegister& src7) {
+ DCHECK(AreSameSizeAndType(src0, src1, src2, src3, src4, src5, src6, src7));
+
+ int count = 5 + src5.is_valid() + src6.is_valid() + src6.is_valid();
+ int size = src0.SizeInBytes();
+ DCHECK_EQ(0, (size * count) % 16);
+
+ PushHelper(4, size, src0, src1, src2, src3);
+ PushHelper(count - 4, size, src4, src5, src6, src7);
+}
+
+void TurboAssembler::Pop(const CPURegister& dst0, const CPURegister& dst1,
+ const CPURegister& dst2, const CPURegister& dst3,
+ const CPURegister& dst4, const CPURegister& dst5,
+ const CPURegister& dst6, const CPURegister& dst7) {
+ // It is not valid to pop into the same register more than once in one
+ // instruction, not even into the zero register.
+ DCHECK(!AreAliased(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7));
+ DCHECK(AreSameSizeAndType(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7));
+ DCHECK(dst0.is_valid());
+
+ int count = 5 + dst5.is_valid() + dst6.is_valid() + dst7.is_valid();
+ int size = dst0.SizeInBytes();
+ DCHECK_EQ(0, (size * count) % 16);
+
+ PopHelper(4, size, dst0, dst1, dst2, dst3);
+ PopHelper(count - 4, size, dst4, dst5, dst6, dst7);
+}
+
+void MacroAssembler::PushMultipleTimes(CPURegister src, Register count) {
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireSameSizeAs(count);
+
+ Label loop, leftover2, leftover1, done;
+
+ Subs(temp, count, 4);
+ B(mi, &leftover2);
+
+ // Push groups of four first.
+ Bind(&loop);
+ Subs(temp, temp, 4);
+ PushHelper(4, src.SizeInBytes(), src, src, src, src);
+ B(pl, &loop);
+
+ // Push groups of two.
+ Bind(&leftover2);
+ Tbz(count, 1, &leftover1);
+ PushHelper(2, src.SizeInBytes(), src, src, NoReg, NoReg);
+
+ // Push the last one (if required).
+ Bind(&leftover1);
+ Tbz(count, 0, &done);
+ PushHelper(1, src.SizeInBytes(), src, NoReg, NoReg, NoReg);
+
+ Bind(&done);
+}
+
+void TurboAssembler::PushHelper(int count, int size, const CPURegister& src0,
+ const CPURegister& src1,
+ const CPURegister& src2,
+ const CPURegister& src3) {
+ // Ensure that we don't unintentially modify scratch or debug registers.
+ InstructionAccurateScope scope(this);
+
+ DCHECK(AreSameSizeAndType(src0, src1, src2, src3));
+ DCHECK(size == src0.SizeInBytes());
+
+ // When pushing multiple registers, the store order is chosen such that
+ // Push(a, b) is equivalent to Push(a) followed by Push(b).
+ switch (count) {
+ case 1:
+ DCHECK(src1.IsNone() && src2.IsNone() && src3.IsNone());
+ str(src0, MemOperand(sp, -1 * size, PreIndex));
+ break;
+ case 2:
+ DCHECK(src2.IsNone() && src3.IsNone());
+ stp(src1, src0, MemOperand(sp, -2 * size, PreIndex));
+ break;
+ case 3:
+ DCHECK(src3.IsNone());
+ stp(src2, src1, MemOperand(sp, -3 * size, PreIndex));
+ str(src0, MemOperand(sp, 2 * size));
+ break;
+ case 4:
+ // Skip over 4 * size, then fill in the gap. This allows four W registers
+ // to be pushed using sp, whilst maintaining 16-byte alignment for sp
+ // at all times.
+ stp(src3, src2, MemOperand(sp, -4 * size, PreIndex));
+ stp(src1, src0, MemOperand(sp, 2 * size));
+ break;
+ default:
+ UNREACHABLE();
+ }
+}
+
+void TurboAssembler::PopHelper(int count, int size, const CPURegister& dst0,
+ const CPURegister& dst1, const CPURegister& dst2,
+ const CPURegister& dst3) {
+ // Ensure that we don't unintentially modify scratch or debug registers.
+ InstructionAccurateScope scope(this);
+
+ DCHECK(AreSameSizeAndType(dst0, dst1, dst2, dst3));
+ DCHECK(size == dst0.SizeInBytes());
+
+ // When popping multiple registers, the load order is chosen such that
+ // Pop(a, b) is equivalent to Pop(a) followed by Pop(b).
+ switch (count) {
+ case 1:
+ DCHECK(dst1.IsNone() && dst2.IsNone() && dst3.IsNone());
+ ldr(dst0, MemOperand(sp, 1 * size, PostIndex));
+ break;
+ case 2:
+ DCHECK(dst2.IsNone() && dst3.IsNone());
+ ldp(dst0, dst1, MemOperand(sp, 2 * size, PostIndex));
+ break;
+ case 3:
+ DCHECK(dst3.IsNone());
+ ldr(dst2, MemOperand(sp, 2 * size));
+ ldp(dst0, dst1, MemOperand(sp, 3 * size, PostIndex));
+ break;
+ case 4:
+ // Load the higher addresses first, then load the lower addresses and
+ // skip the whole block in the second instruction. This allows four W
+ // registers to be popped using sp, whilst maintaining 16-byte alignment
+ // for sp at all times.
+ ldp(dst2, dst3, MemOperand(sp, 2 * size));
+ ldp(dst0, dst1, MemOperand(sp, 4 * size, PostIndex));
+ break;
+ default:
+ UNREACHABLE();
+ }
+}
+
+void TurboAssembler::PokePair(const CPURegister& src1, const CPURegister& src2,
+ int offset) {
+ DCHECK(AreSameSizeAndType(src1, src2));
+ DCHECK((offset >= 0) && ((offset % src1.SizeInBytes()) == 0));
+ Stp(src1, src2, MemOperand(sp, offset));
+}
+
+void MacroAssembler::PeekPair(const CPURegister& dst1, const CPURegister& dst2,
+ int offset) {
+ DCHECK(AreSameSizeAndType(dst1, dst2));
+ DCHECK((offset >= 0) && ((offset % dst1.SizeInBytes()) == 0));
+ Ldp(dst1, dst2, MemOperand(sp, offset));
+}
+
+void MacroAssembler::PushCalleeSavedRegisters() {
+#ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
+ Pacibsp();
+#endif
+
+ {
+ // Ensure that the macro-assembler doesn't use any scratch registers.
+ InstructionAccurateScope scope(this);
+
+ MemOperand tos(sp, -2 * static_cast<int>(kXRegSize), PreIndex);
+
+ stp(d14, d15, tos);
+ stp(d12, d13, tos);
+ stp(d10, d11, tos);
+ stp(d8, d9, tos);
+
+ STATIC_ASSERT(
+ EntryFrameConstants::kCalleeSavedRegisterBytesPushedBeforeFpLrPair ==
+ 8 * kSystemPointerSize);
+ stp(x29, x30, tos); // fp, lr
+
+ STATIC_ASSERT(
+ EntryFrameConstants::kCalleeSavedRegisterBytesPushedAfterFpLrPair ==
+ 10 * kSystemPointerSize);
+
+ stp(x27, x28, tos);
+ stp(x25, x26, tos);
+ stp(x23, x24, tos);
+ stp(x21, x22, tos);
+ stp(x19, x20, tos);
+ }
+}
+
+void MacroAssembler::PopCalleeSavedRegisters() {
+ {
+ // Ensure that the macro-assembler doesn't use any scratch registers.
+ InstructionAccurateScope scope(this);
+
+ MemOperand tos(sp, 2 * kXRegSize, PostIndex);
+
+ ldp(x19, x20, tos);
+ ldp(x21, x22, tos);
+ ldp(x23, x24, tos);
+ ldp(x25, x26, tos);
+ ldp(x27, x28, tos);
+ ldp(x29, x30, tos);
+
+ ldp(d8, d9, tos);
+ ldp(d10, d11, tos);
+ ldp(d12, d13, tos);
+ ldp(d14, d15, tos);
+ }
+
+#ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
+ Autibsp();
+#endif
+}
+
+void TurboAssembler::AssertSpAligned() {
+ if (emit_debug_code()) {
+ HardAbortScope hard_abort(this); // Avoid calls to Abort.
+ // Arm64 requires the stack pointer to be 16-byte aligned prior to address
+ // calculation.
+ UseScratchRegisterScope scope(this);
+ Register temp = scope.AcquireX();
+ Mov(temp, sp);
+ Tst(temp, 15);
+ Check(eq, AbortReason::kUnexpectedStackPointer);
+ }
+}
+
+void TurboAssembler::CopySlots(int dst, Register src, Register slot_count) {
+ DCHECK(!src.IsZero());
+ UseScratchRegisterScope scope(this);
+ Register dst_reg = scope.AcquireX();
+ SlotAddress(dst_reg, dst);
+ SlotAddress(src, src);
+ CopyDoubleWords(dst_reg, src, slot_count);
+}
+
+void TurboAssembler::CopySlots(Register dst, Register src,
+ Register slot_count) {
+ DCHECK(!dst.IsZero() && !src.IsZero());
+ SlotAddress(dst, dst);
+ SlotAddress(src, src);
+ CopyDoubleWords(dst, src, slot_count);
+}
+
+void TurboAssembler::CopyDoubleWords(Register dst, Register src, Register count,
+ CopyDoubleWordsMode mode) {
+ DCHECK(!AreAliased(dst, src, count));
+
+ if (emit_debug_code()) {
+ Register pointer1 = dst;
+ Register pointer2 = src;
+ if (mode == kSrcLessThanDst) {
+ pointer1 = src;
+ pointer2 = dst;
+ }
+ // Copy requires pointer1 < pointer2 || (pointer1 - pointer2) >= count.
+ Label pointer1_below_pointer2;
+ Subs(pointer1, pointer1, pointer2);
+ B(lt, &pointer1_below_pointer2);
+ Cmp(pointer1, count);
+ Check(ge, AbortReason::kOffsetOutOfRange);
+ Bind(&pointer1_below_pointer2);
+ Add(pointer1, pointer1, pointer2);
+ }
+ static_assert(kSystemPointerSize == kDRegSize,
+ "pointers must be the same size as doubles");
+
+ if (mode == kDstLessThanSrcAndReverse) {
+ Add(src, src, Operand(count, LSL, kSystemPointerSizeLog2));
+ Sub(src, src, kSystemPointerSize);
+ }
+
+ int src_direction = (mode == kDstLessThanSrc) ? 1 : -1;
+ int dst_direction = (mode == kSrcLessThanDst) ? -1 : 1;
+
+ UseScratchRegisterScope scope(this);
+ VRegister temp0 = scope.AcquireD();
+ VRegister temp1 = scope.AcquireD();
+
+ Label pairs, loop, done;
+
+ Tbz(count, 0, &pairs);
+ Ldr(temp0, MemOperand(src, src_direction * kSystemPointerSize, PostIndex));
+ Sub(count, count, 1);
+ Str(temp0, MemOperand(dst, dst_direction * kSystemPointerSize, PostIndex));
+
+ Bind(&pairs);
+ if (mode == kSrcLessThanDst) {
+ // Adjust pointers for post-index ldp/stp with negative offset:
+ Sub(dst, dst, kSystemPointerSize);
+ Sub(src, src, kSystemPointerSize);
+ } else if (mode == kDstLessThanSrcAndReverse) {
+ Sub(src, src, kSystemPointerSize);
+ }
+ Bind(&loop);
+ Cbz(count, &done);
+ Ldp(temp0, temp1,
+ MemOperand(src, 2 * src_direction * kSystemPointerSize, PostIndex));
+ Sub(count, count, 2);
+ if (mode == kDstLessThanSrcAndReverse) {
+ Stp(temp1, temp0,
+ MemOperand(dst, 2 * dst_direction * kSystemPointerSize, PostIndex));
+ } else {
+ Stp(temp0, temp1,
+ MemOperand(dst, 2 * dst_direction * kSystemPointerSize, PostIndex));
+ }
+ B(&loop);
+
+ // TODO(all): large copies may benefit from using temporary Q registers
+ // to copy four double words per iteration.
+
+ Bind(&done);
+}
+
+void TurboAssembler::SlotAddress(Register dst, int slot_offset) {
+ Add(dst, sp, slot_offset << kSystemPointerSizeLog2);
+}
+
+void TurboAssembler::SlotAddress(Register dst, Register slot_offset) {
+ Add(dst, sp, Operand(slot_offset, LSL, kSystemPointerSizeLog2));
+}
+
+void TurboAssembler::AssertFPCRState(Register fpcr) {
+ if (emit_debug_code()) {
+ Label unexpected_mode, done;
+ UseScratchRegisterScope temps(this);
+ if (fpcr.IsNone()) {
+ fpcr = temps.AcquireX();
+ Mrs(fpcr, FPCR);
+ }
+
+ // Settings left to their default values:
+ // - Assert that flush-to-zero is not set.
+ Tbnz(fpcr, FZ_offset, &unexpected_mode);
+ // - Assert that the rounding mode is nearest-with-ties-to-even.
+ STATIC_ASSERT(FPTieEven == 0);
+ Tst(fpcr, RMode_mask);
+ B(eq, &done);
+
+ Bind(&unexpected_mode);
+ Abort(AbortReason::kUnexpectedFPCRMode);
+
+ Bind(&done);
+ }
+}
+
+void TurboAssembler::CanonicalizeNaN(const VRegister& dst,
+ const VRegister& src) {
+ AssertFPCRState();
+
+ // Subtracting 0.0 preserves all inputs except for signalling NaNs, which
+ // become quiet NaNs. We use fsub rather than fadd because fsub preserves -0.0
+ // inputs: -0.0 + 0.0 = 0.0, but -0.0 - 0.0 = -0.0.
+ Fsub(dst, src, fp_zero);
+}
+
+void TurboAssembler::LoadRoot(Register destination, RootIndex index) {
+ // TODO(jbramley): Most root values are constants, and can be synthesized
+ // without a load. Refer to the ARM back end for details.
+ Ldr(destination,
+ MemOperand(kRootRegister, RootRegisterOffsetForRootIndex(index)));
+}
+
+void TurboAssembler::Move(Register dst, Smi src) { Mov(dst, src); }
+
+void TurboAssembler::MovePair(Register dst0, Register src0, Register dst1,
+ Register src1) {
+ DCHECK_NE(dst0, dst1);
+ if (dst0 != src1) {
+ Mov(dst0, src0);
+ Mov(dst1, src1);
+ } else if (dst1 != src0) {
+ // Swap the order of the moves to resolve the overlap.
+ Mov(dst1, src1);
+ Mov(dst0, src0);
+ } else {
+ // Worse case scenario, this is a swap.
+ Swap(dst0, src0);
+ }
+}
+
+void TurboAssembler::Swap(Register lhs, Register rhs) {
+ DCHECK(lhs.IsSameSizeAndType(rhs));
+ DCHECK_NE(lhs, rhs);
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireX();
+ Mov(temp, rhs);
+ Mov(rhs, lhs);
+ Mov(lhs, temp);
+}
+
+void TurboAssembler::Swap(VRegister lhs, VRegister rhs) {
+ DCHECK(lhs.IsSameSizeAndType(rhs));
+ DCHECK_NE(lhs, rhs);
+ UseScratchRegisterScope temps(this);
+ VRegister temp = VRegister::no_reg();
+ if (lhs.IsS()) {
+ temp = temps.AcquireS();
+ } else if (lhs.IsD()) {
+ temp = temps.AcquireD();
+ } else {
+ DCHECK(lhs.IsQ());
+ temp = temps.AcquireQ();
+ }
+ Mov(temp, rhs);
+ Mov(rhs, lhs);
+ Mov(lhs, temp);
+}
+
+void TurboAssembler::AssertSmi(Register object, AbortReason reason) {
+ if (emit_debug_code()) {
+ STATIC_ASSERT(kSmiTag == 0);
+ Tst(object, kSmiTagMask);
+ Check(eq, reason);
+ }
+}
+
+void MacroAssembler::AssertNotSmi(Register object, AbortReason reason) {
+ if (emit_debug_code()) {
+ STATIC_ASSERT(kSmiTag == 0);
+ Tst(object, kSmiTagMask);
+ Check(ne, reason);
+ }
+}
+
+void MacroAssembler::AssertConstructor(Register object) {
+ if (emit_debug_code()) {
+ AssertNotSmi(object, AbortReason::kOperandIsASmiAndNotAConstructor);
+
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireX();
+
+ LoadMap(temp, object);
+ Ldrb(temp, FieldMemOperand(temp, Map::kBitFieldOffset));
+ Tst(temp, Operand(Map::Bits1::IsConstructorBit::kMask));
+
+ Check(ne, AbortReason::kOperandIsNotAConstructor);
+ }
+}
+
+void MacroAssembler::AssertFunction(Register object) {
+ if (emit_debug_code()) {
+ AssertNotSmi(object, AbortReason::kOperandIsASmiAndNotAFunction);
+
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireX();
+
+ CompareObjectType(object, temp, temp, JS_FUNCTION_TYPE);
+ Check(eq, AbortReason::kOperandIsNotAFunction);
+ }
+}
+
+void MacroAssembler::AssertBoundFunction(Register object) {
+ if (emit_debug_code()) {
+ AssertNotSmi(object, AbortReason::kOperandIsASmiAndNotABoundFunction);
+
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireX();
+
+ CompareObjectType(object, temp, temp, JS_BOUND_FUNCTION_TYPE);
+ Check(eq, AbortReason::kOperandIsNotABoundFunction);
+ }
+}
+
+void MacroAssembler::AssertGeneratorObject(Register object) {
+ if (!emit_debug_code()) return;
+ AssertNotSmi(object, AbortReason::kOperandIsASmiAndNotAGeneratorObject);
+
+ // Load map
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireX();
+ LoadMap(temp, object);
+
+ Label do_check;
+ // Load instance type and check if JSGeneratorObject
+ CompareInstanceType(temp, temp, JS_GENERATOR_OBJECT_TYPE);
+ B(eq, &do_check);
+
+ // Check if JSAsyncFunctionObject
+ Cmp(temp, JS_ASYNC_FUNCTION_OBJECT_TYPE);
+ B(eq, &do_check);
+
+ // Check if JSAsyncGeneratorObject
+ Cmp(temp, JS_ASYNC_GENERATOR_OBJECT_TYPE);
+
+ bind(&do_check);
+ // Restore generator object to register and perform assertion
+ Check(eq, AbortReason::kOperandIsNotAGeneratorObject);
+}
+
+void MacroAssembler::AssertUndefinedOrAllocationSite(Register object) {
+ if (emit_debug_code()) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.AcquireX();
+ Label done_checking;
+ AssertNotSmi(object);
+ JumpIfRoot(object, RootIndex::kUndefinedValue, &done_checking);
+ LoadMap(scratch, object);
+ CompareInstanceType(scratch, scratch, ALLOCATION_SITE_TYPE);
+ Assert(eq, AbortReason::kExpectedUndefinedOrCell);
+ Bind(&done_checking);
+ }
+}
+
+void TurboAssembler::AssertPositiveOrZero(Register value) {
+ if (emit_debug_code()) {
+ Label done;
+ int sign_bit = value.Is64Bits() ? kXSignBit : kWSignBit;
+ Tbz(value, sign_bit, &done);
+ Abort(AbortReason::kUnexpectedNegativeValue);
+ Bind(&done);
+ }
+}
+
+void MacroAssembler::CallRuntime(const Runtime::Function* f, int num_arguments,
+ SaveFPRegsMode save_doubles) {
+ // All arguments must be on the stack before this function is called.
+ // x0 holds the return value after the call.
+
+ // Check that the number of arguments matches what the function expects.
+ // If f->nargs is -1, the function can accept a variable number of arguments.
+ CHECK(f->nargs < 0 || f->nargs == num_arguments);
+
+ // Place the necessary arguments.
+ Mov(x0, num_arguments);
+ Mov(x1, ExternalReference::Create(f));
+
+ Handle<Code> code =
+ CodeFactory::CEntry(isolate(), f->result_size, save_doubles);
+ Call(code, RelocInfo::CODE_TARGET);
+}
+
+void MacroAssembler::JumpToExternalReference(const ExternalReference& builtin,
+ bool builtin_exit_frame) {
+ Mov(x1, builtin);
+ Handle<Code> code = CodeFactory::CEntry(isolate(), 1, kDontSaveFPRegs,
+ kArgvOnStack, builtin_exit_frame);
+ Jump(code, RelocInfo::CODE_TARGET);
+}
+
+void MacroAssembler::JumpToInstructionStream(Address entry) {
+ Ldr(kOffHeapTrampolineRegister, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+ Br(kOffHeapTrampolineRegister);
+}
+
+void MacroAssembler::TailCallRuntime(Runtime::FunctionId fid) {
+ const Runtime::Function* function = Runtime::FunctionForId(fid);
+ DCHECK_EQ(1, function->result_size);
+ if (function->nargs >= 0) {
+ // TODO(1236192): Most runtime routines don't need the number of
+ // arguments passed in because it is constant. At some point we
+ // should remove this need and make the runtime routine entry code
+ // smarter.
+ Mov(x0, function->nargs);
+ }
+ JumpToExternalReference(ExternalReference::Create(fid));
+}
+
+int TurboAssembler::ActivationFrameAlignment() {
+#if V8_HOST_ARCH_ARM64
+ // Running on the real platform. Use the alignment as mandated by the local
+ // environment.
+ // Note: This will break if we ever start generating snapshots on one ARM
+ // platform for another ARM platform with a different alignment.
+ return base::OS::ActivationFrameAlignment();
+#else // V8_HOST_ARCH_ARM64
+ // If we are using the simulator then we should always align to the expected
+ // alignment. As the simulator is used to generate snapshots we do not know
+ // if the target platform will need alignment, so this is controlled from a
+ // flag.
+ return FLAG_sim_stack_alignment;
+#endif // V8_HOST_ARCH_ARM64
+}
+
+void TurboAssembler::CallCFunction(ExternalReference function,
+ int num_of_reg_args) {
+ CallCFunction(function, num_of_reg_args, 0);
+}
+
+void TurboAssembler::CallCFunction(ExternalReference function,
+ int num_of_reg_args,
+ int num_of_double_args) {
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireX();
+ Mov(temp, function);
+ CallCFunction(temp, num_of_reg_args, num_of_double_args);
+}
+
+static const int kRegisterPassedArguments = 8;
+
+void TurboAssembler::CallCFunction(Register function, int num_of_reg_args,
+ int num_of_double_args) {
+ DCHECK_LE(num_of_reg_args + num_of_double_args, kMaxCParameters);
+ DCHECK(has_frame());
+
+ // If we're passing doubles, we're limited to the following prototypes
+ // (defined by ExternalReference::Type):
+ // BUILTIN_COMPARE_CALL: int f(double, double)
+ // BUILTIN_FP_FP_CALL: double f(double, double)
+ // BUILTIN_FP_CALL: double f(double)
+ // BUILTIN_FP_INT_CALL: double f(double, int)
+ if (num_of_double_args > 0) {
+ DCHECK_LE(num_of_reg_args, 1);
+ DCHECK_LE(num_of_double_args + num_of_reg_args, 2);
+ }
+
+ // Save the frame pointer and PC so that the stack layout remains iterable,
+ // even without an ExitFrame which normally exists between JS and C frames.
+ Register pc_scratch = x4;
+ Register addr_scratch = x5;
+ Push(pc_scratch, addr_scratch);
+
+ Label get_pc;
+ Bind(&get_pc);
+ Adr(pc_scratch, &get_pc);
+
+ // See x64 code for reasoning about how to address the isolate data fields.
+ if (root_array_available()) {
+ Str(pc_scratch,
+ MemOperand(kRootRegister, IsolateData::fast_c_call_caller_pc_offset()));
+ Str(fp,
+ MemOperand(kRootRegister, IsolateData::fast_c_call_caller_fp_offset()));
+ } else {
+ DCHECK_NOT_NULL(isolate());
+ Mov(addr_scratch,
+ ExternalReference::fast_c_call_caller_pc_address(isolate()));
+ Str(pc_scratch, MemOperand(addr_scratch));
+ Mov(addr_scratch,
+ ExternalReference::fast_c_call_caller_fp_address(isolate()));
+ Str(fp, MemOperand(addr_scratch));
+ }
+
+ Pop(addr_scratch, pc_scratch);
+
+ // Call directly. The function called cannot cause a GC, or allow preemption,
+ // so the return address in the link register stays correct.
+ Call(function);
+
+ // We don't unset the PC; the FP is the source of truth.
+ if (root_array_available()) {
+ Str(xzr,
+ MemOperand(kRootRegister, IsolateData::fast_c_call_caller_fp_offset()));
+ } else {
+ DCHECK_NOT_NULL(isolate());
+ Push(addr_scratch, xzr);
+ Mov(addr_scratch,
+ ExternalReference::fast_c_call_caller_fp_address(isolate()));
+ Str(xzr, MemOperand(addr_scratch));
+ Pop(xzr, addr_scratch);
+ }
+
+ if (num_of_reg_args > kRegisterPassedArguments) {
+ // Drop the register passed arguments.
+ int claim_slots = RoundUp(num_of_reg_args - kRegisterPassedArguments, 2);
+ Drop(claim_slots);
+ }
+}
+
+void TurboAssembler::LoadFromConstantsTable(Register destination,
+ int constant_index) {
+ DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kBuiltinsConstantsTable));
+ LoadRoot(destination, RootIndex::kBuiltinsConstantsTable);
+ LoadTaggedPointerField(
+ destination, FieldMemOperand(destination, FixedArray::OffsetOfElementAt(
+ constant_index)));
+}
+
+void TurboAssembler::LoadRootRelative(Register destination, int32_t offset) {
+ Ldr(destination, MemOperand(kRootRegister, offset));
+}
+
+void TurboAssembler::LoadRootRegisterOffset(Register destination,
+ intptr_t offset) {
+ if (offset == 0) {
+ Mov(destination, kRootRegister);
+ } else {
+ Add(destination, kRootRegister, offset);
+ }
+}
+
+void TurboAssembler::Jump(Register target, Condition cond) {
+ if (cond == nv) return;
+ Label done;
+ if (cond != al) B(NegateCondition(cond), &done);
+ Br(target);
+ Bind(&done);
+}
+
+void TurboAssembler::JumpHelper(int64_t offset, RelocInfo::Mode rmode,
+ Condition cond) {
+ if (cond == nv) return;
+ Label done;
+ if (cond != al) B(NegateCondition(cond), &done);
+ if (CanUseNearCallOrJump(rmode)) {
+ DCHECK(IsNearCallOffset(offset));
+ near_jump(static_cast<int>(offset), rmode);
+ } else {
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireX();
+ uint64_t imm = reinterpret_cast<uint64_t>(pc_) + offset * kInstrSize;
+ Mov(temp, Immediate(imm, rmode));
+ Br(temp);
+ }
+ Bind(&done);
+}
+
+namespace {
+
+// The calculated offset is either:
+// * the 'target' input unmodified if this is a Wasm call, or
+// * the offset of the target from the current PC, in instructions, for any
+// other type of call.
+static int64_t CalculateTargetOffset(Address target, RelocInfo::Mode rmode,
+ byte* pc) {
+ int64_t offset = static_cast<int64_t>(target);
+ // The target of WebAssembly calls is still an index instead of an actual
+ // address at this point, and needs to be encoded as-is.
+ if (rmode != RelocInfo::WASM_CALL && rmode != RelocInfo::WASM_STUB_CALL) {
+ offset -= reinterpret_cast<int64_t>(pc);
+ DCHECK_EQ(offset % kInstrSize, 0);
+ offset = offset / static_cast<int>(kInstrSize);
+ }
+ return offset;
+}
+} // namespace
+
+void TurboAssembler::Jump(Address target, RelocInfo::Mode rmode,
+ Condition cond) {
+ JumpHelper(CalculateTargetOffset(target, rmode, pc_), rmode, cond);
+}
+
+void TurboAssembler::Jump(Handle<Code> code, RelocInfo::Mode rmode,
+ Condition cond) {
+ DCHECK(RelocInfo::IsCodeTarget(rmode));
+ DCHECK_IMPLIES(options().isolate_independent_code,
+ Builtins::IsIsolateIndependentBuiltin(*code));
+
+ if (options().inline_offheap_trampolines) {
+ int builtin_index = Builtins::kNoBuiltinId;
+ if (isolate()->builtins()->IsBuiltinHandle(code, &builtin_index)) {
+ // Inline the trampoline.
+ RecordCommentForOffHeapTrampoline(builtin_index);
+ CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.AcquireX();
+ EmbeddedData d = EmbeddedData::FromBlob();
+ Address entry = d.InstructionStartOfBuiltin(builtin_index);
+ Ldr(scratch, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+ Jump(scratch, cond);
+ return;
+ }
+ }
+
+ if (CanUseNearCallOrJump(rmode)) {
+ EmbeddedObjectIndex index = AddEmbeddedObject(code);
+ DCHECK(is_int32(index));
+ JumpHelper(static_cast<int64_t>(index), rmode, cond);
+ } else {
+ Jump(code.address(), rmode, cond);
+ }
+}
+
+void TurboAssembler::Jump(const ExternalReference& reference) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.AcquireX();
+ Mov(scratch, reference);
+ Jump(scratch);
+}
+
+void TurboAssembler::Call(Register target) {
+ BlockPoolsScope scope(this);
+ Blr(target);
+}
+
+void TurboAssembler::Call(Address target, RelocInfo::Mode rmode) {
+ BlockPoolsScope scope(this);
+
+ if (CanUseNearCallOrJump(rmode)) {
+ int64_t offset = CalculateTargetOffset(target, rmode, pc_);
+ DCHECK(IsNearCallOffset(offset));
+ near_call(static_cast<int>(offset), rmode);
+ } else {
+ IndirectCall(target, rmode);
+ }
+}
+
+void TurboAssembler::Call(Handle<Code> code, RelocInfo::Mode rmode) {
+ DCHECK_IMPLIES(options().isolate_independent_code,
+ Builtins::IsIsolateIndependentBuiltin(*code));
+ BlockPoolsScope scope(this);
+
+ if (options().inline_offheap_trampolines) {
+ int builtin_index = Builtins::kNoBuiltinId;
+ if (isolate()->builtins()->IsBuiltinHandle(code, &builtin_index)) {
+ // Inline the trampoline.
+ CallBuiltin(builtin_index);
+ return;
+ }
+ }
+
+ DCHECK(code->IsExecutable());
+ if (CanUseNearCallOrJump(rmode)) {
+ EmbeddedObjectIndex index = AddEmbeddedObject(code);
+ DCHECK(is_int32(index));
+ near_call(static_cast<int32_t>(index), rmode);
+ } else {
+ IndirectCall(code.address(), rmode);
+ }
+}
+
+void TurboAssembler::Call(ExternalReference target) {
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireX();
+ Mov(temp, target);
+ Call(temp);
+}
+
+void TurboAssembler::LoadEntryFromBuiltinIndex(Register builtin_index) {
+ // The builtin_index register contains the builtin index as a Smi.
+ // Untagging is folded into the indexing operand below.
+ if (SmiValuesAre32Bits()) {
+ Asr(builtin_index, builtin_index, kSmiShift - kSystemPointerSizeLog2);
+ Add(builtin_index, builtin_index,
+ IsolateData::builtin_entry_table_offset());
+ Ldr(builtin_index, MemOperand(kRootRegister, builtin_index));
+ } else {
+ DCHECK(SmiValuesAre31Bits());
+ if (COMPRESS_POINTERS_BOOL) {
+ Add(builtin_index, kRootRegister,
+ Operand(builtin_index.W(), SXTW, kSystemPointerSizeLog2 - kSmiShift));
+ } else {
+ Add(builtin_index, kRootRegister,
+ Operand(builtin_index, LSL, kSystemPointerSizeLog2 - kSmiShift));
+ }
+ Ldr(builtin_index,
+ MemOperand(builtin_index, IsolateData::builtin_entry_table_offset()));
+ }
+}
+
+void TurboAssembler::LoadEntryFromBuiltinIndex(Builtins::Name builtin_index,
+ Register destination) {
+ Ldr(destination,
+ MemOperand(kRootRegister,
+ IsolateData::builtin_entry_slot_offset(builtin_index)));
+}
+
+void TurboAssembler::CallBuiltinByIndex(Register builtin_index) {
+ LoadEntryFromBuiltinIndex(builtin_index);
+ Call(builtin_index);
+}
+
+void TurboAssembler::CallBuiltin(int builtin_index) {
+ DCHECK(Builtins::IsBuiltinId(builtin_index));
+ RecordCommentForOffHeapTrampoline(builtin_index);
+ CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.AcquireX();
+ EmbeddedData d = EmbeddedData::FromBlob();
+ Address entry = d.InstructionStartOfBuiltin(builtin_index);
+ Ldr(scratch, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+ Call(scratch);
+}
+
+void TurboAssembler::LoadCodeObjectEntry(Register destination,
+ Register code_object) {
+ // Code objects are called differently depending on whether we are generating
+ // builtin code (which will later be embedded into the binary) or compiling
+ // user JS code at runtime.
+ // * Builtin code runs in --jitless mode and thus must not call into on-heap
+ // Code targets. Instead, we dispatch through the builtins entry table.
+ // * Codegen at runtime does not have this restriction and we can use the
+ // shorter, branchless instruction sequence. The assumption here is that
+ // targets are usually generated code and not builtin Code objects.
+
+ if (options().isolate_independent_code) {
+ DCHECK(root_array_available());
+ Label if_code_is_off_heap, out;
+
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.AcquireX();
+
+ DCHECK(!AreAliased(destination, scratch));
+ DCHECK(!AreAliased(code_object, scratch));
+
+ // Check whether the Code object is an off-heap trampoline. If so, call its
+ // (off-heap) entry point directly without going through the (on-heap)
+ // trampoline. Otherwise, just call the Code object as always.
+
+ Ldrsw(scratch, FieldMemOperand(code_object, Code::kFlagsOffset));
+ Tst(scratch, Operand(Code::IsOffHeapTrampoline::kMask));
+ B(ne, &if_code_is_off_heap);
+
+ // Not an off-heap trampoline object, the entry point is at
+ // Code::raw_instruction_start().
+ Add(destination, code_object, Code::kHeaderSize - kHeapObjectTag);
+ B(&out);
+
+ // An off-heap trampoline, the entry point is loaded from the builtin entry
+ // table.
+ bind(&if_code_is_off_heap);
+ Ldrsw(scratch, FieldMemOperand(code_object, Code::kBuiltinIndexOffset));
+ Lsl(destination, scratch, kSystemPointerSizeLog2);
+ Add(destination, destination, kRootRegister);
+ Ldr(destination,
+ MemOperand(destination, IsolateData::builtin_entry_table_offset()));
+
+ bind(&out);
+ } else {
+ Add(destination, code_object, Code::kHeaderSize - kHeapObjectTag);
+ }
+}
+
+void TurboAssembler::CallCodeObject(Register code_object) {
+ LoadCodeObjectEntry(code_object, code_object);
+ Call(code_object);
+}
+
+void TurboAssembler::JumpCodeObject(Register code_object) {
+ LoadCodeObjectEntry(code_object, code_object);
+
+ UseScratchRegisterScope temps(this);
+ if (code_object != x17) {
+ temps.Exclude(x17);
+ Mov(x17, code_object);
+ }
+ Jump(x17);
+}
+
+void TurboAssembler::StoreReturnAddressAndCall(Register target) {
+ // This generates the final instruction sequence for calls to C functions
+ // once an exit frame has been constructed.
+ //
+ // Note that this assumes the caller code (i.e. the Code object currently
+ // being generated) is immovable or that the callee function cannot trigger
+ // GC, since the callee function will return to it.
+
+ UseScratchRegisterScope temps(this);
+ temps.Exclude(x16, x17);
+
+ Label return_location;
+ Adr(x17, &return_location);
+#ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
+ Add(x16, sp, kSystemPointerSize);
+ Pacib1716();
+#endif
+ Poke(x17, 0);
+
+ if (emit_debug_code()) {
+ // Verify that the slot below fp[kSPOffset]-8 points to the signed return
+ // location.
+ Ldr(x16, MemOperand(fp, ExitFrameConstants::kSPOffset));
+ Ldr(x16, MemOperand(x16, -static_cast<int64_t>(kXRegSize)));
+ Cmp(x16, x17);
+ Check(eq, AbortReason::kReturnAddressNotFoundInFrame);
+ }
+
+ Blr(target);
+ Bind(&return_location);
+}
+
+void TurboAssembler::IndirectCall(Address target, RelocInfo::Mode rmode) {
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireX();
+ Mov(temp, Immediate(target, rmode));
+ Blr(temp);
+}
+
+bool TurboAssembler::IsNearCallOffset(int64_t offset) {
+ return is_int26(offset);
+}
+
+void TurboAssembler::CallForDeoptimization(
+ Builtins::Name target, int deopt_id, Label* exit, DeoptimizeKind kind,
+ Label* jump_deoptimization_entry_label) {
+ BlockPoolsScope scope(this);
+ bl(jump_deoptimization_entry_label);
+ DCHECK_EQ(SizeOfCodeGeneratedSince(exit),
+ (kind == DeoptimizeKind::kLazy)
+ ? Deoptimizer::kLazyDeoptExitSize
+ : Deoptimizer::kNonLazyDeoptExitSize);
+ USE(exit, kind);
+}
+
+void TurboAssembler::PrepareForTailCall(Register callee_args_count,
+ Register caller_args_count,
+ Register scratch0, Register scratch1) {
+ DCHECK(!AreAliased(callee_args_count, caller_args_count, scratch0, scratch1));
+
+ // Calculate the end of destination area where we will put the arguments
+ // after we drop current frame. We add kSystemPointerSize to count the
+ // receiver argument which is not included into formal parameters count.
+ Register dst_reg = scratch0;
+ Add(dst_reg, fp, Operand(caller_args_count, LSL, kSystemPointerSizeLog2));
+ Add(dst_reg, dst_reg,
+ StandardFrameConstants::kCallerSPOffset + kSystemPointerSize);
+ // Round dst_reg up to a multiple of 16 bytes, so that we overwrite any
+ // potential padding.
+ Add(dst_reg, dst_reg, 15);
+ Bic(dst_reg, dst_reg, 15);
+
+ Register src_reg = caller_args_count;
+ // Calculate the end of source area. +kSystemPointerSize is for the receiver.
+ Add(src_reg, sp, Operand(callee_args_count, LSL, kSystemPointerSizeLog2));
+ Add(src_reg, src_reg, kSystemPointerSize);
+
+ // Round src_reg up to a multiple of 16 bytes, so we include any potential
+ // padding in the copy.
+ Add(src_reg, src_reg, 15);
+ Bic(src_reg, src_reg, 15);
+
+ if (FLAG_debug_code) {
+ Cmp(src_reg, dst_reg);
+ Check(lo, AbortReason::kStackAccessBelowStackPointer);
+ }
+
+ // Restore caller's frame pointer and return address now as they will be
+ // overwritten by the copying loop.
+ RestoreFPAndLR();
+
+ // Now copy callee arguments to the caller frame going backwards to avoid
+ // callee arguments corruption (source and destination areas could overlap).
+
+ // Both src_reg and dst_reg are pointing to the word after the one to copy,
+ // so they must be pre-decremented in the loop.
+ Register tmp_reg = scratch1;
+ Label loop, entry;
+ B(&entry);
+ bind(&loop);
+ Ldr(tmp_reg, MemOperand(src_reg, -kSystemPointerSize, PreIndex));
+ Str(tmp_reg, MemOperand(dst_reg, -kSystemPointerSize, PreIndex));
+ bind(&entry);
+ Cmp(sp, src_reg);
+ B(ne, &loop);
+
+ // Leave current frame.
+ Mov(sp, dst_reg);
+}
+
+void MacroAssembler::LoadStackLimit(Register destination, StackLimitKind kind) {
+ DCHECK(root_array_available());
+ Isolate* isolate = this->isolate();
+ ExternalReference limit =
+ kind == StackLimitKind::kRealStackLimit
+ ? ExternalReference::address_of_real_jslimit(isolate)
+ : ExternalReference::address_of_jslimit(isolate);
+ DCHECK(TurboAssembler::IsAddressableThroughRootRegister(isolate, limit));
+
+ intptr_t offset =
+ TurboAssembler::RootRegisterOffsetForExternalReference(isolate, limit);
+ Ldr(destination, MemOperand(kRootRegister, offset));
+}
+
+void MacroAssembler::StackOverflowCheck(Register num_args,
+ Label* stack_overflow) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.AcquireX();
+
+ // Check the stack for overflow.
+ // We are not trying to catch interruptions (e.g. debug break and
+ // preemption) here, so the "real stack limit" is checked.
+
+ LoadStackLimit(scratch, StackLimitKind::kRealStackLimit);
+ // Make scratch the space we have left. The stack might already be overflowed
+ // here which will cause scratch to become negative.
+ Sub(scratch, sp, scratch);
+ // Check if the arguments will overflow the stack.
+ Cmp(scratch, Operand(num_args, LSL, kSystemPointerSizeLog2));
+ B(le, stack_overflow);
+}
+
+void MacroAssembler::InvokePrologue(Register formal_parameter_count,
+ Register actual_argument_count, Label* done,
+ InvokeFlag flag) {
+ // x0: actual arguments count.
+ // x1: function (passed through to callee).
+ // x2: expected arguments count.
+ // x3: new target
+ Label regular_invoke;
+ DCHECK_EQ(actual_argument_count, x0);
+ DCHECK_EQ(formal_parameter_count, x2);
+
+#ifdef V8_NO_ARGUMENTS_ADAPTOR
+ // If the formal parameter count is equal to the adaptor sentinel, no need
+ // to push undefined value as arguments.
+ Cmp(formal_parameter_count, Operand(kDontAdaptArgumentsSentinel));
+ B(eq, ®ular_invoke);
+
+ // If overapplication or if the actual argument count is equal to the
+ // formal parameter count, no need to push extra undefined values.
+ Register extra_argument_count = x2;
+ Subs(extra_argument_count, formal_parameter_count, actual_argument_count);
+ B(le, ®ular_invoke);
+
+ // The stack pointer in arm64 needs to be 16-byte aligned. We might need to
+ // (1) add an extra padding or (2) remove (re-use) the extra padding already
+ // in the stack. Let {slots_to_copy} be the number of slots (arguments) to
+ // move up in the stack and let {slots_to_claim} be the number of extra stack
+ // slots to claim.
+ Label even_extra_count, skip_move;
+ Register slots_to_copy = x4;
+ Register slots_to_claim = x5;
+
+ Add(slots_to_copy, actual_argument_count, 1); // Copy with receiver.
+ Mov(slots_to_claim, extra_argument_count);
+ Tbz(extra_argument_count, 0, &even_extra_count);
+
+ // Calculate {slots_to_claim} when {extra_argument_count} is odd.
+ // If {actual_argument_count} is even, we need one extra padding slot
+ // {slots_to_claim = extra_argument_count + 1}.
+ // If {actual_argument_count} is odd, we know that the
+ // original arguments will have a padding slot that we can reuse
+ // {slots_to_claim = extra_argument_count - 1}.
+ {
+ Register scratch = x11;
+ Add(slots_to_claim, extra_argument_count, 1);
+ And(scratch, actual_argument_count, 1);
+ Eor(scratch, scratch, 1);
+ Sub(slots_to_claim, slots_to_claim, Operand(scratch, LSL, 1));
+ }
+
+ Bind(&even_extra_count);
+ Cbz(slots_to_claim, &skip_move);
+
+ Label stack_overflow;
+ StackOverflowCheck(slots_to_claim, &stack_overflow);
+ Claim(slots_to_claim);
+
+ // Move the arguments already in the stack including the receiver.
+ {
+ Register src = x6;
+ Register dst = x7;
+ SlotAddress(src, slots_to_claim);
+ SlotAddress(dst, 0);
+ CopyDoubleWords(dst, src, slots_to_copy);
+ }
+
+ Bind(&skip_move);
+ Register actual_argument_with_receiver = x4;
+ Register pointer_next_value = x5;
+ Add(actual_argument_with_receiver, actual_argument_count,
+ 1); // {slots_to_copy} was scratched.
+
+ // Copy extra arguments as undefined values.
+ {
+ Label loop;
+ Register undefined_value = x6;
+ Register count = x7;
+ LoadRoot(undefined_value, RootIndex::kUndefinedValue);
+ SlotAddress(pointer_next_value, actual_argument_with_receiver);
+ Mov(count, extra_argument_count);
+ Bind(&loop);
+ Str(undefined_value,
+ MemOperand(pointer_next_value, kSystemPointerSize, PostIndex));
+ Subs(count, count, 1);
+ Cbnz(count, &loop);
+ }
+
+ // Set padding if needed.
+ {
+ Label skip;
+ Register total_args_slots = x4;
+ Add(total_args_slots, actual_argument_with_receiver, extra_argument_count);
+ Tbz(total_args_slots, 0, &skip);
+ Str(padreg, MemOperand(pointer_next_value));
+ Bind(&skip);
+ }
+ B(®ular_invoke);
+
+ bind(&stack_overflow);
+ {
+ FrameScope frame(this,
+ has_frame() ? StackFrame::NONE : StackFrame::INTERNAL);
+ CallRuntime(Runtime::kThrowStackOverflow);
+ Unreachable();
+ }
+#else
+ // Check whether the expected and actual arguments count match. The registers
+ // are set up according to contract with ArgumentsAdaptorTrampoline.ct.
+ // If actual == expected perform a regular invocation.
+ Cmp(formal_parameter_count, actual_argument_count);
+ B(eq, ®ular_invoke);
+
+ // The argument counts mismatch, generate a call to the argument adaptor.
+ Handle<Code> adaptor = BUILTIN_CODE(isolate(), ArgumentsAdaptorTrampoline);
+ if (flag == CALL_FUNCTION) {
+ Call(adaptor);
+ // If the arg counts don't match, no extra code is emitted by
+ // MAsm::InvokeFunctionCode and we can just fall through.
+ B(done);
+ } else {
+ Jump(adaptor, RelocInfo::CODE_TARGET);
+ }
+#endif
+
+ Bind(®ular_invoke);
+}
+
+void MacroAssembler::CallDebugOnFunctionCall(Register fun, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count) {
+ // Load receiver to pass it later to DebugOnFunctionCall hook.
+ Peek(x4, ReceiverOperand(actual_parameter_count));
+ FrameScope frame(this, has_frame() ? StackFrame::NONE : StackFrame::INTERNAL);
+
+ if (!new_target.is_valid()) new_target = padreg;
+
+ // Save values on stack.
+ SmiTag(expected_parameter_count);
+ SmiTag(actual_parameter_count);
+ Push(expected_parameter_count, actual_parameter_count, new_target, fun);
+ Push(fun, x4);
+ CallRuntime(Runtime::kDebugOnFunctionCall);
+
+ // Restore values from stack.
+ Pop(fun, new_target, actual_parameter_count, expected_parameter_count);
+ SmiUntag(actual_parameter_count);
+ SmiUntag(expected_parameter_count);
+}
+
+void MacroAssembler::InvokeFunctionCode(Register function, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count,
+ InvokeFlag flag) {
+ // You can't call a function without a valid frame.
+ DCHECK_IMPLIES(flag == CALL_FUNCTION, has_frame());
+ DCHECK_EQ(function, x1);
+ DCHECK_IMPLIES(new_target.is_valid(), new_target == x3);
+
+ // On function call, call into the debugger if necessary.
+ Label debug_hook, continue_after_hook;
+ {
+ Mov(x4, ExternalReference::debug_hook_on_function_call_address(isolate()));
+ Ldrsb(x4, MemOperand(x4));
+ Cbnz(x4, &debug_hook);
+ }
+ bind(&continue_after_hook);
+
+ // Clear the new.target register if not given.
+ if (!new_target.is_valid()) {
+ LoadRoot(x3, RootIndex::kUndefinedValue);
+ }
+
+ Label done;
+ InvokePrologue(expected_parameter_count, actual_parameter_count, &done, flag);
+
+ // If actual != expected, InvokePrologue will have handled the call through
+ // the argument adaptor mechanism.
+ // The called function expects the call kind in x5.
+ // We call indirectly through the code field in the function to
+ // allow recompilation to take effect without changing any of the
+ // call sites.
+ Register code = kJavaScriptCallCodeStartRegister;
+ LoadTaggedPointerField(code,
+ FieldMemOperand(function, JSFunction::kCodeOffset));
+ if (flag == CALL_FUNCTION) {
+ CallCodeObject(code);
+ } else {
+ DCHECK(flag == JUMP_FUNCTION);
+ JumpCodeObject(code);
+ }
+ B(&done);
+
+ // Deferred debug hook.
+ bind(&debug_hook);
+ CallDebugOnFunctionCall(function, new_target, expected_parameter_count,
+ actual_parameter_count);
+ B(&continue_after_hook);
+
+ // Continue here if InvokePrologue does handle the invocation due to
+ // mismatched parameter counts.
+ Bind(&done);
+}
+
+Operand MacroAssembler::ReceiverOperand(Register arg_count) {
+ return Operand(0);
+}
+
+void MacroAssembler::InvokeFunctionWithNewTarget(
+ Register function, Register new_target, Register actual_parameter_count,
+ InvokeFlag flag) {
+ // You can't call a function without a valid frame.
+ DCHECK(flag == JUMP_FUNCTION || has_frame());
+
+ // Contract with called JS functions requires that function is passed in x1.
+ // (See FullCodeGenerator::Generate().)
+ DCHECK_EQ(function, x1);
+
+ Register expected_parameter_count = x2;
+
+ LoadTaggedPointerField(cp,
+ FieldMemOperand(function, JSFunction::kContextOffset));
+ // The number of arguments is stored as an int32_t, and -1 is a marker
+ // (kDontAdaptArgumentsSentinel), so we need sign
+ // extension to correctly handle it.
+ LoadTaggedPointerField(
+ expected_parameter_count,
+ FieldMemOperand(function, JSFunction::kSharedFunctionInfoOffset));
+ Ldrh(expected_parameter_count,
+ FieldMemOperand(expected_parameter_count,
+ SharedFunctionInfo::kFormalParameterCountOffset));
+
+ InvokeFunctionCode(function, new_target, expected_parameter_count,
+ actual_parameter_count, flag);
+}
+
+void MacroAssembler::InvokeFunction(Register function,
+ Register expected_parameter_count,
+ Register actual_parameter_count,
+ InvokeFlag flag) {
+ // You can't call a function without a valid frame.
+ DCHECK(flag == JUMP_FUNCTION || has_frame());
+
+ // Contract with called JS functions requires that function is passed in x1.
+ // (See FullCodeGenerator::Generate().)
+ DCHECK_EQ(function, x1);
+
+ // Set up the context.
+ LoadTaggedPointerField(cp,
+ FieldMemOperand(function, JSFunction::kContextOffset));
+
+ InvokeFunctionCode(function, no_reg, expected_parameter_count,
+ actual_parameter_count, flag);
+}
+
+void TurboAssembler::TryConvertDoubleToInt64(Register result,
+ DoubleRegister double_input,
+ Label* done) {
+ // Try to convert with an FPU convert instruction. It's trivial to compute
+ // the modulo operation on an integer register so we convert to a 64-bit
+ // integer.
+ //
+ // Fcvtzs will saturate to INT64_MIN (0x800...00) or INT64_MAX (0x7FF...FF)
+ // when the double is out of range. NaNs and infinities will be converted to 0
+ // (as ECMA-262 requires).
+ Fcvtzs(result.X(), double_input);
+
+ // The values INT64_MIN (0x800...00) or INT64_MAX (0x7FF...FF) are not
+ // representable using a double, so if the result is one of those then we know
+ // that saturation occurred, and we need to manually handle the conversion.
+ //
+ // It is easy to detect INT64_MIN and INT64_MAX because adding or subtracting
+ // 1 will cause signed overflow.
+ Cmp(result.X(), 1);
+ Ccmp(result.X(), -1, VFlag, vc);
+
+ B(vc, done);
+}
+
+void TurboAssembler::TruncateDoubleToI(Isolate* isolate, Zone* zone,
+ Register result,
+ DoubleRegister double_input,
+ StubCallMode stub_mode,
+ LinkRegisterStatus lr_status) {
+ if (CpuFeatures::IsSupported(JSCVT)) {
+ Fjcvtzs(result.W(), double_input);
+ return;
+ }
+
+ Label done;
+
+ // Try to convert the double to an int64. If successful, the bottom 32 bits
+ // contain our truncated int32 result.
+ TryConvertDoubleToInt64(result, double_input, &done);
+
+ // If we fell through then inline version didn't succeed - call stub instead.
+ if (lr_status == kLRHasNotBeenSaved) {
+ Push<TurboAssembler::kSignLR>(lr, double_input);
+ } else {
+ Push<TurboAssembler::kDontStoreLR>(xzr, double_input);
+ }
+
+ // DoubleToI preserves any registers it needs to clobber.
+ if (stub_mode == StubCallMode::kCallWasmRuntimeStub) {
+ Call(wasm::WasmCode::kDoubleToI, RelocInfo::WASM_STUB_CALL);
+ } else if (options().inline_offheap_trampolines) {
+ CallBuiltin(Builtins::kDoubleToI);
+ } else {
+ Call(BUILTIN_CODE(isolate, DoubleToI), RelocInfo::CODE_TARGET);
+ }
+ Ldr(result, MemOperand(sp, 0));
+
+ DCHECK_EQ(xzr.SizeInBytes(), double_input.SizeInBytes());
+
+ if (lr_status == kLRHasNotBeenSaved) {
+ // Pop into xzr here to drop the double input on the stack:
+ Pop<TurboAssembler::kAuthLR>(xzr, lr);
+ } else {
+ Drop(2);
+ }
+
+ Bind(&done);
+ // Keep our invariant that the upper 32 bits are zero.
+ Uxtw(result.W(), result.W());
+}
+
+void TurboAssembler::Prologue() {
+ Push<TurboAssembler::kSignLR>(lr, fp);
+ mov(fp, sp);
+ STATIC_ASSERT(kExtraSlotClaimedByPrologue == 1);
+ Push(cp, kJSFunctionRegister, kJavaScriptCallArgCountRegister, padreg);
+}
+
+void TurboAssembler::EnterFrame(StackFrame::Type type) {
+ UseScratchRegisterScope temps(this);
+
+ if (type == StackFrame::INTERNAL || type == StackFrame::WASM_DEBUG_BREAK) {
+ Register type_reg = temps.AcquireX();
+ Mov(type_reg, StackFrame::TypeToMarker(type));
+ Push<TurboAssembler::kSignLR>(lr, fp, type_reg, padreg);
+ const int kFrameSize =
+ TypedFrameConstants::kFixedFrameSizeFromFp + kSystemPointerSize;
+ Add(fp, sp, kFrameSize);
+ // sp[3] : lr
+ // sp[2] : fp
+ // sp[1] : type
+ // sp[0] : for alignment
+ } else if (type == StackFrame::WASM ||
+ type == StackFrame::WASM_COMPILE_LAZY ||
+ type == StackFrame::WASM_EXIT) {
+ Register type_reg = temps.AcquireX();
+ Mov(type_reg, StackFrame::TypeToMarker(type));
+ Push<TurboAssembler::kSignLR>(lr, fp);
+ Mov(fp, sp);
+ Push(type_reg, padreg);
+ // sp[3] : lr
+ // sp[2] : fp
+ // sp[1] : type
+ // sp[0] : for alignment
+ } else {
+ DCHECK_EQ(type, StackFrame::CONSTRUCT);
+ Register type_reg = temps.AcquireX();
+ Mov(type_reg, StackFrame::TypeToMarker(type));
+
+ // Users of this frame type push a context pointer after the type field,
+ // so do it here to keep the stack pointer aligned.
+ Push<TurboAssembler::kSignLR>(lr, fp, type_reg, cp);
+
+ // The context pointer isn't part of the fixed frame, so add an extra slot
+ // to account for it.
+ Add(fp, sp,
+ TypedFrameConstants::kFixedFrameSizeFromFp + kSystemPointerSize);
+ // sp[3] : lr
+ // sp[2] : fp
+ // sp[1] : type
+ // sp[0] : cp
+ }
+}
+
+void TurboAssembler::LeaveFrame(StackFrame::Type type) {
+ // Drop the execution stack down to the frame pointer and restore
+ // the caller frame pointer and return address.
+ Mov(sp, fp);
+ Pop<TurboAssembler::kAuthLR>(fp, lr);
+}
+
+void MacroAssembler::ExitFramePreserveFPRegs() {
+ DCHECK_EQ(kCallerSavedV.Count() % 2, 0);
+ PushCPURegList(kCallerSavedV);
+}
+
+void MacroAssembler::ExitFrameRestoreFPRegs() {
+ // Read the registers from the stack without popping them. The stack pointer
+ // will be reset as part of the unwinding process.
+ CPURegList saved_fp_regs = kCallerSavedV;
+ DCHECK_EQ(saved_fp_regs.Count() % 2, 0);
+
+ int offset = ExitFrameConstants::kLastExitFrameField;
+ while (!saved_fp_regs.IsEmpty()) {
+ const CPURegister& dst0 = saved_fp_regs.PopHighestIndex();
+ const CPURegister& dst1 = saved_fp_regs.PopHighestIndex();
+ offset -= 2 * kDRegSize;
+ Ldp(dst1, dst0, MemOperand(fp, offset));
+ }
+}
+
+void MacroAssembler::EnterExitFrame(bool save_doubles, const Register& scratch,
+ int extra_space,
+ StackFrame::Type frame_type) {
+ DCHECK(frame_type == StackFrame::EXIT ||
+ frame_type == StackFrame::BUILTIN_EXIT);
+
+ // Set up the new stack frame.
+ Push<TurboAssembler::kSignLR>(lr, fp);
+ Mov(fp, sp);
+ Mov(scratch, StackFrame::TypeToMarker(frame_type));
+ Push(scratch, xzr);
+ // fp[8]: CallerPC (lr)
+ // fp -> fp[0]: CallerFP (old fp)
+ // fp[-8]: STUB marker
+ // sp -> fp[-16]: Space reserved for SPOffset.
+ STATIC_ASSERT((2 * kSystemPointerSize) ==
+ ExitFrameConstants::kCallerSPOffset);
+ STATIC_ASSERT((1 * kSystemPointerSize) ==
+ ExitFrameConstants::kCallerPCOffset);
+ STATIC_ASSERT((0 * kSystemPointerSize) ==
+ ExitFrameConstants::kCallerFPOffset);
+ STATIC_ASSERT((-2 * kSystemPointerSize) == ExitFrameConstants::kSPOffset);
+
+ // Save the frame pointer and context pointer in the top frame.
+ Mov(scratch,
+ ExternalReference::Create(IsolateAddressId::kCEntryFPAddress, isolate()));
+ Str(fp, MemOperand(scratch));
+ Mov(scratch,
+ ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
+ Str(cp, MemOperand(scratch));
+
+ STATIC_ASSERT((-2 * kSystemPointerSize) ==
+ ExitFrameConstants::kLastExitFrameField);
+ if (save_doubles) {
+ ExitFramePreserveFPRegs();
+ }
+
+ // Round the number of space we need to claim to a multiple of two.
+ int slots_to_claim = RoundUp(extra_space + 1, 2);
+
+ // Reserve space for the return address and for user requested memory.
+ // We do this before aligning to make sure that we end up correctly
+ // aligned with the minimum of wasted space.
+ Claim(slots_to_claim, kXRegSize);
+ // fp[8]: CallerPC (lr)
+ // fp -> fp[0]: CallerFP (old fp)
+ // fp[-8]: STUB marker
+ // fp[-16]: Space reserved for SPOffset.
+ // fp[-16 - fp_size]: Saved doubles (if save_doubles is true).
+ // sp[8]: Extra space reserved for caller (if extra_space != 0).
+ // sp -> sp[0]: Space reserved for the return address.
+
+ // ExitFrame::GetStateForFramePointer expects to find the return address at
+ // the memory address immediately below the pointer stored in SPOffset.
+ // It is not safe to derive much else from SPOffset, because the size of the
+ // padding can vary.
+ Add(scratch, sp, kXRegSize);
+ Str(scratch, MemOperand(fp, ExitFrameConstants::kSPOffset));
+}
+
+// Leave the current exit frame.
+void MacroAssembler::LeaveExitFrame(bool restore_doubles,
+ const Register& scratch,
+ const Register& scratch2) {
+ if (restore_doubles) {
+ ExitFrameRestoreFPRegs();
+ }
+
+ // Restore the context pointer from the top frame.
+ Mov(scratch,
+ ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
+ Ldr(cp, MemOperand(scratch));
+
+ if (emit_debug_code()) {
+ // Also emit debug code to clear the cp in the top frame.
+ Mov(scratch2, Operand(Context::kInvalidContext));
+ Mov(scratch, ExternalReference::Create(IsolateAddressId::kContextAddress,
+ isolate()));
+ Str(scratch2, MemOperand(scratch));
+ }
+ // Clear the frame pointer from the top frame.
+ Mov(scratch,
+ ExternalReference::Create(IsolateAddressId::kCEntryFPAddress, isolate()));
+ Str(xzr, MemOperand(scratch));
+
+ // Pop the exit frame.
+ // fp[8]: CallerPC (lr)
+ // fp -> fp[0]: CallerFP (old fp)
+ // fp[...]: The rest of the frame.
+ Mov(sp, fp);
+ Pop<TurboAssembler::kAuthLR>(fp, lr);
+}
+
+void MacroAssembler::LoadGlobalProxy(Register dst) {
+ LoadNativeContextSlot(Context::GLOBAL_PROXY_INDEX, dst);
+}
+
+void MacroAssembler::LoadWeakValue(Register out, Register in,
+ Label* target_if_cleared) {
+ CompareAndBranch(in.W(), Operand(kClearedWeakHeapObjectLower32), eq,
+ target_if_cleared);
+
+ and_(out, in, Operand(~kWeakHeapObjectMask));
+}
+
+void MacroAssembler::IncrementCounter(StatsCounter* counter, int value,
+ Register scratch1, Register scratch2) {
+ DCHECK_NE(value, 0);
+ if (FLAG_native_code_counters && counter->Enabled()) {
+ // This operation has to be exactly 32-bit wide in case the external
+ // reference table redirects the counter to a uint32_t dummy_stats_counter_
+ // field.
+ Mov(scratch2, ExternalReference::Create(counter));
+ Ldr(scratch1.W(), MemOperand(scratch2));
+ Add(scratch1.W(), scratch1.W(), value);
+ Str(scratch1.W(), MemOperand(scratch2));
+ }
+}
+
+void MacroAssembler::DecrementCounter(StatsCounter* counter, int value,
+ Register scratch1, Register scratch2) {
+ IncrementCounter(counter, -value, scratch1, scratch2);
+}
+
+void MacroAssembler::MaybeDropFrames() {
+ // Check whether we need to drop frames to restart a function on the stack.
+ Mov(x1, ExternalReference::debug_restart_fp_address(isolate()));
+ Ldr(x1, MemOperand(x1));
+ Tst(x1, x1);
+ Jump(BUILTIN_CODE(isolate(), FrameDropperTrampoline), RelocInfo::CODE_TARGET,
+ ne);
+}
+
+void MacroAssembler::JumpIfObjectType(Register object, Register map,
+ Register type_reg, InstanceType type,
+ Label* if_cond_pass, Condition cond) {
+ CompareObjectType(object, map, type_reg, type);
+ B(cond, if_cond_pass);
+}
+
+// Sets condition flags based on comparison, and returns type in type_reg.
+void MacroAssembler::CompareObjectType(Register object, Register map,
+ Register type_reg, InstanceType type) {
+ LoadMap(map, object);
+ CompareInstanceType(map, type_reg, type);
+}
+
+void MacroAssembler::LoadMap(Register dst, Register object) {
+ LoadTaggedPointerField(dst, FieldMemOperand(object, HeapObject::kMapOffset));
+}
+
+// Sets condition flags based on comparison, and returns type in type_reg.
+void MacroAssembler::CompareInstanceType(Register map, Register type_reg,
+ InstanceType type) {
+ Ldrh(type_reg, FieldMemOperand(map, Map::kInstanceTypeOffset));
+ Cmp(type_reg, type);
+}
+
+void MacroAssembler::LoadElementsKindFromMap(Register result, Register map) {
+ // Load the map's "bit field 2".
+ Ldrb(result, FieldMemOperand(map, Map::kBitField2Offset));
+ // Retrieve elements_kind from bit field 2.
+ DecodeField<Map::Bits2::ElementsKindBits>(result);
+}
+
+void MacroAssembler::CompareRoot(const Register& obj, RootIndex index) {
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireX();
+ DCHECK(!AreAliased(obj, temp));
+ LoadRoot(temp, index);
+ CmpTagged(obj, temp);
+}
+
+void MacroAssembler::JumpIfRoot(const Register& obj, RootIndex index,
+ Label* if_equal) {
+ CompareRoot(obj, index);
+ B(eq, if_equal);
+}
+
+void MacroAssembler::JumpIfNotRoot(const Register& obj, RootIndex index,
+ Label* if_not_equal) {
+ CompareRoot(obj, index);
+ B(ne, if_not_equal);
+}
+
+void MacroAssembler::JumpIfIsInRange(const Register& value,
+ unsigned lower_limit,
+ unsigned higher_limit,
+ Label* on_in_range) {
+ if (lower_limit != 0) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.AcquireW();
+ Sub(scratch, value, Operand(lower_limit));
+ CompareAndBranch(scratch, Operand(higher_limit - lower_limit), ls,
+ on_in_range);
+ } else {
+ CompareAndBranch(value, Operand(higher_limit - lower_limit), ls,
+ on_in_range);
+ }
+}
+
+void TurboAssembler::LoadTaggedPointerField(const Register& destination,
+ const MemOperand& field_operand) {
+ if (COMPRESS_POINTERS_BOOL) {
+ DecompressTaggedPointer(destination, field_operand);
+ } else {
+ Ldr(destination, field_operand);
+ }
+}
+
+void TurboAssembler::LoadAnyTaggedField(const Register& destination,
+ const MemOperand& field_operand) {
+ if (COMPRESS_POINTERS_BOOL) {
+ DecompressAnyTagged(destination, field_operand);
+ } else {
+ Ldr(destination, field_operand);
+ }
+}
+
+void TurboAssembler::SmiUntagField(Register dst, const MemOperand& src) {
+ SmiUntag(dst, src);
+}
+
+void TurboAssembler::StoreTaggedField(const Register& value,
+ const MemOperand& dst_field_operand) {
+ if (COMPRESS_POINTERS_BOOL) {
+ Str(value.W(), dst_field_operand);
+ } else {
+ Str(value, dst_field_operand);
+ }
+}
+
+void TurboAssembler::DecompressTaggedSigned(const Register& destination,
+ const MemOperand& field_operand) {
+ RecordComment("[ DecompressTaggedSigned");
+ Ldr(destination.W(), field_operand);
+ if (FLAG_debug_code) {
+ // Corrupt the top 32 bits. Made up of 16 fixed bits and 16 pc offset bits.
+ Add(destination, destination,
+ ((kDebugZapValue << 16) | (pc_offset() & 0xffff)) << 32);
+ }
+ RecordComment("]");
+}
+
+void TurboAssembler::DecompressTaggedPointer(const Register& destination,
+ const MemOperand& field_operand) {
+ RecordComment("[ DecompressTaggedPointer");
+ Ldr(destination.W(), field_operand);
+ Add(destination, kRootRegister, destination);
+ RecordComment("]");
+}
+
+void TurboAssembler::DecompressTaggedPointer(const Register& destination,
+ const Register& source) {
+ RecordComment("[ DecompressTaggedPointer");
+ Add(destination, kRootRegister, Operand(source, UXTW));
+ RecordComment("]");
+}
+
+void TurboAssembler::DecompressAnyTagged(const Register& destination,
+ const MemOperand& field_operand) {
+ RecordComment("[ DecompressAnyTagged");
+ Ldr(destination.W(), field_operand);
+ Add(destination, kRootRegister, destination);
+ RecordComment("]");
+}
+
+void TurboAssembler::CheckPageFlag(const Register& object, int mask,
+ Condition cc, Label* condition_met) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.AcquireX();
+ And(scratch, object, ~kPageAlignmentMask);
+ Ldr(scratch, MemOperand(scratch, BasicMemoryChunk::kFlagsOffset));
+ if (cc == eq) {
+ TestAndBranchIfAnySet(scratch, mask, condition_met);
+ } else {
+ DCHECK_EQ(cc, ne);
+ TestAndBranchIfAllClear(scratch, mask, condition_met);
+ }
+}
+
+void MacroAssembler::RecordWriteField(Register object, int offset,
+ Register value,
+ LinkRegisterStatus lr_status,
+ SaveFPRegsMode save_fp,
+ RememberedSetAction remembered_set_action,
+ SmiCheck smi_check) {
+ // First, check if a write barrier is even needed. The tests below
+ // catch stores of Smis.
+ Label done;
+
+ // Skip the barrier if writing a smi.
+ if (smi_check == INLINE_SMI_CHECK) {
+ JumpIfSmi(value, &done);
+ }
+
+ // Although the object register is tagged, the offset is relative to the start
+ // of the object, so offset must be a multiple of kTaggedSize.
+ DCHECK(IsAligned(offset, kTaggedSize));
+
+ if (emit_debug_code()) {
+ Label ok;
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.AcquireX();
+ Add(scratch, object, offset - kHeapObjectTag);
+ Tst(scratch, kTaggedSize - 1);
+ B(eq, &ok);
+ Abort(AbortReason::kUnalignedCellInWriteBarrier);
+ Bind(&ok);
+ }
+
+ RecordWrite(object, Operand(offset - kHeapObjectTag), value, lr_status,
+ save_fp, remembered_set_action, OMIT_SMI_CHECK);
+
+ Bind(&done);
+}
+
+void TurboAssembler::SaveRegisters(RegList registers) {
+ DCHECK_GT(NumRegs(registers), 0);
+ CPURegList regs(CPURegister::kRegister, kXRegSizeInBits, registers);
+ // If we were saving LR, we might need to sign it.
+ DCHECK(!regs.IncludesAliasOf(lr));
+ regs.Align();
+ PushCPURegList(regs);
+}
+
+void TurboAssembler::RestoreRegisters(RegList registers) {
+ DCHECK_GT(NumRegs(registers), 0);
+ CPURegList regs(CPURegister::kRegister, kXRegSizeInBits, registers);
+ // If we were saving LR, we might need to sign it.
+ DCHECK(!regs.IncludesAliasOf(lr));
+ regs.Align();
+ PopCPURegList(regs);
+}
+
+void TurboAssembler::CallEphemeronKeyBarrier(Register object, Operand offset,
+ SaveFPRegsMode fp_mode) {
+ EphemeronKeyBarrierDescriptor descriptor;
+ RegList registers = descriptor.allocatable_registers();
+
+ SaveRegisters(registers);
+
+ Register object_parameter(
+ descriptor.GetRegisterParameter(EphemeronKeyBarrierDescriptor::kObject));
+ Register slot_parameter(descriptor.GetRegisterParameter(
+ EphemeronKeyBarrierDescriptor::kSlotAddress));
+ Register fp_mode_parameter(
+ descriptor.GetRegisterParameter(EphemeronKeyBarrierDescriptor::kFPMode));
+
+ MoveObjectAndSlot(object_parameter, slot_parameter, object, offset);
+
+ Mov(fp_mode_parameter, Smi::FromEnum(fp_mode));
+ Call(isolate()->builtins()->builtin_handle(Builtins::kEphemeronKeyBarrier),
+ RelocInfo::CODE_TARGET);
+ RestoreRegisters(registers);
+}
+
+void TurboAssembler::CallRecordWriteStub(
+ Register object, Operand offset, RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode) {
+ CallRecordWriteStub(
+ object, offset, remembered_set_action, fp_mode,
+ isolate()->builtins()->builtin_handle(Builtins::kRecordWrite),
+ kNullAddress);
+}
+
+void TurboAssembler::CallRecordWriteStub(
+ Register object, Operand offset, RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode, Address wasm_target) {
+ CallRecordWriteStub(object, offset, remembered_set_action, fp_mode,
+ Handle<Code>::null(), wasm_target);
+}
+
+void TurboAssembler::CallRecordWriteStub(
+ Register object, Operand offset, RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode, Handle<Code> code_target, Address wasm_target) {
+ DCHECK_NE(code_target.is_null(), wasm_target == kNullAddress);
+ // TODO(albertnetymk): For now we ignore remembered_set_action and fp_mode,
+ // i.e. always emit remember set and save FP registers in RecordWriteStub. If
+ // large performance regression is observed, we should use these values to
+ // avoid unnecessary work.
+
+ RecordWriteDescriptor descriptor;
+ RegList registers = descriptor.allocatable_registers();
+
+ SaveRegisters(registers);
+
+ Register object_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kObject));
+ Register slot_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kSlot));
+ Register remembered_set_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kRememberedSet));
+ Register fp_mode_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kFPMode));
+
+ MoveObjectAndSlot(object_parameter, slot_parameter, object, offset);
+
+ Mov(remembered_set_parameter, Smi::FromEnum(remembered_set_action));
+ Mov(fp_mode_parameter, Smi::FromEnum(fp_mode));
+ if (code_target.is_null()) {
+ Call(wasm_target, RelocInfo::WASM_STUB_CALL);
+ } else {
+ Call(code_target, RelocInfo::CODE_TARGET);
+ }
+
+ RestoreRegisters(registers);
+}
+
+void TurboAssembler::MoveObjectAndSlot(Register dst_object, Register dst_slot,
+ Register object, Operand offset) {
+ DCHECK_NE(dst_object, dst_slot);
+ // If `offset` is a register, it cannot overlap with `object`.
+ DCHECK_IMPLIES(!offset.IsImmediate(), offset.reg() != object);
+
+ // If the slot register does not overlap with the object register, we can
+ // overwrite it.
+ if (dst_slot != object) {
+ Add(dst_slot, object, offset);
+ Mov(dst_object, object);
+ return;
+ }
+
+ DCHECK_EQ(dst_slot, object);
+
+ // If the destination object register does not overlap with the offset
+ // register, we can overwrite it.
+ if (offset.IsImmediate() || (offset.reg() != dst_object)) {
+ Mov(dst_object, dst_slot);
+ Add(dst_slot, dst_slot, offset);
+ return;
+ }
+
+ DCHECK_EQ(dst_object, offset.reg());
+
+ // We only have `dst_slot` and `dst_object` left as distinct registers so we
+ // have to swap them. We write this as a add+sub sequence to avoid using a
+ // scratch register.
+ Add(dst_slot, dst_slot, dst_object);
+ Sub(dst_object, dst_slot, dst_object);
+}
+
+// If lr_status is kLRHasBeenSaved, lr will be clobbered.
+//
+// The register 'object' contains a heap object pointer. The heap object tag is
+// shifted away.
+void MacroAssembler::RecordWrite(Register object, Operand offset,
+ Register value, LinkRegisterStatus lr_status,
+ SaveFPRegsMode fp_mode,
+ RememberedSetAction remembered_set_action,
+ SmiCheck smi_check) {
+ ASM_LOCATION_IN_ASSEMBLER("MacroAssembler::RecordWrite");
+ DCHECK(!AreAliased(object, value));
+
+ if (emit_debug_code()) {
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireX();
+
+ Add(temp, object, offset);
+ LoadTaggedPointerField(temp, MemOperand(temp));
+ Cmp(temp, value);
+ Check(eq, AbortReason::kWrongAddressOrValuePassedToRecordWrite);
+ }
+
+ if ((remembered_set_action == OMIT_REMEMBERED_SET &&
+ !FLAG_incremental_marking) ||
+ FLAG_disable_write_barriers) {
+ return;
+ }
+
+ // First, check if a write barrier is even needed. The tests below
+ // catch stores of smis and stores into the young generation.
+ Label done;
+
+ if (smi_check == INLINE_SMI_CHECK) {
+ DCHECK_EQ(0, kSmiTag);
+ JumpIfSmi(value, &done);
+ }
+ CheckPageFlag(value, MemoryChunk::kPointersToHereAreInterestingMask, ne,
+ &done);
+
+ CheckPageFlag(object, MemoryChunk::kPointersFromHereAreInterestingMask, ne,
+ &done);
+
+ // Record the actual write.
+ if (lr_status == kLRHasNotBeenSaved) {
+ Push<TurboAssembler::kSignLR>(padreg, lr);
+ }
+ CallRecordWriteStub(object, offset, remembered_set_action, fp_mode);
+ if (lr_status == kLRHasNotBeenSaved) {
+ Pop<TurboAssembler::kAuthLR>(lr, padreg);
+ }
+
+ Bind(&done);
+}
+
+void TurboAssembler::Assert(Condition cond, AbortReason reason) {
+ if (emit_debug_code()) {
+ Check(cond, reason);
+ }
+}
+
+void TurboAssembler::AssertUnreachable(AbortReason reason) {
+ if (emit_debug_code()) Abort(reason);
+}
+
+void TurboAssembler::Check(Condition cond, AbortReason reason) {
+ Label ok;
+ B(cond, &ok);
+ Abort(reason);
+ // Will not return here.
+ Bind(&ok);
+}
+
+void TurboAssembler::Trap() { Brk(0); }
+void TurboAssembler::DebugBreak() { Debug("DebugBreak", 0, BREAK); }
+
+void TurboAssembler::Abort(AbortReason reason) {
+#ifdef DEBUG
+ RecordComment("Abort message: ");
+ RecordComment(GetAbortReason(reason));
+#endif
+
+ // Avoid emitting call to builtin if requested.
+ if (trap_on_abort()) {
+ Brk(0);
+ return;
+ }
+
+ // We need some scratch registers for the MacroAssembler, so make sure we have
+ // some. This is safe here because Abort never returns.
+ RegList old_tmp_list = TmpList()->list();
+ TmpList()->Combine(MacroAssembler::DefaultTmpList());
+
+ if (should_abort_hard()) {
+ // We don't care if we constructed a frame. Just pretend we did.
+ FrameScope assume_frame(this, StackFrame::NONE);
+ Mov(w0, static_cast<int>(reason));
+ Call(ExternalReference::abort_with_reason());
+ return;
+ }
+
+ // Avoid infinite recursion; Push contains some assertions that use Abort.
+ HardAbortScope hard_aborts(this);
+
+ Mov(x1, Smi::FromInt(static_cast<int>(reason)));
+
+ if (!has_frame_) {
+ // We don't actually want to generate a pile of code for this, so just
+ // claim there is a stack frame, without generating one.
+ FrameScope scope(this, StackFrame::NONE);
+ Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
+ } else {
+ Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
+ }
+
+ TmpList()->set_list(old_tmp_list);
+}
+
+void MacroAssembler::LoadNativeContextSlot(int index, Register dst) {
+ LoadMap(dst, cp);
+ LoadTaggedPointerField(
+ dst, FieldMemOperand(
+ dst, Map::kConstructorOrBackPointerOrNativeContextOffset));
+ LoadTaggedPointerField(dst, MemOperand(dst, Context::SlotOffset(index)));
+}
+
+// This is the main Printf implementation. All other Printf variants call
+// PrintfNoPreserve after setting up one or more PreserveRegisterScopes.
+void TurboAssembler::PrintfNoPreserve(const char* format,
+ const CPURegister& arg0,
+ const CPURegister& arg1,
+ const CPURegister& arg2,
+ const CPURegister& arg3) {
+ // We cannot handle a caller-saved stack pointer. It doesn't make much sense
+ // in most cases anyway, so this restriction shouldn't be too serious.
+ DCHECK(!kCallerSaved.IncludesAliasOf(sp));
+
+ // The provided arguments, and their proper procedure-call standard registers.
+ CPURegister args[kPrintfMaxArgCount] = {arg0, arg1, arg2, arg3};
+ CPURegister pcs[kPrintfMaxArgCount] = {NoReg, NoReg, NoReg, NoReg};
+
+ int arg_count = kPrintfMaxArgCount;
+
+ // The PCS varargs registers for printf. Note that x0 is used for the printf
+ // format string.
+ static const CPURegList kPCSVarargs =
+ CPURegList(CPURegister::kRegister, kXRegSizeInBits, 1, arg_count);
+ static const CPURegList kPCSVarargsFP =
+ CPURegList(CPURegister::kVRegister, kDRegSizeInBits, 0, arg_count - 1);
+
+ // We can use caller-saved registers as scratch values, except for the
+ // arguments and the PCS registers where they might need to go.
+ CPURegList tmp_list = kCallerSaved;
+ tmp_list.Remove(x0); // Used to pass the format string.
+ tmp_list.Remove(kPCSVarargs);
+ tmp_list.Remove(arg0, arg1, arg2, arg3);
+
+ CPURegList fp_tmp_list = kCallerSavedV;
+ fp_tmp_list.Remove(kPCSVarargsFP);
+ fp_tmp_list.Remove(arg0, arg1, arg2, arg3);
+
+ // Override the TurboAssembler's scratch register list. The lists will be
+ // reset automatically at the end of the UseScratchRegisterScope.
+ UseScratchRegisterScope temps(this);
+ TmpList()->set_list(tmp_list.list());
+ FPTmpList()->set_list(fp_tmp_list.list());
+
+ // Copies of the printf vararg registers that we can pop from.
+ CPURegList pcs_varargs = kPCSVarargs;
+#ifndef V8_OS_WIN
+ CPURegList pcs_varargs_fp = kPCSVarargsFP;
+#endif
+
+ // Place the arguments. There are lots of clever tricks and optimizations we
+ // could use here, but Printf is a debug tool so instead we just try to keep
+ // it simple: Move each input that isn't already in the right place to a
+ // scratch register, then move everything back.
+ for (unsigned i = 0; i < kPrintfMaxArgCount; i++) {
+ // Work out the proper PCS register for this argument.
+ if (args[i].IsRegister()) {
+ pcs[i] = pcs_varargs.PopLowestIndex().X();
+ // We might only need a W register here. We need to know the size of the
+ // argument so we can properly encode it for the simulator call.
+ if (args[i].Is32Bits()) pcs[i] = pcs[i].W();
+ } else if (args[i].IsVRegister()) {
+ // In C, floats are always cast to doubles for varargs calls.
+#ifdef V8_OS_WIN
+ // In case of variadic functions SIMD and Floating-point registers
+ // aren't used. The general x0-x7 should be used instead.
+ // https://docs.microsoft.com/en-us/cpp/build/arm64-windows-abi-conventions
+ pcs[i] = pcs_varargs.PopLowestIndex().X();
+#else
+ pcs[i] = pcs_varargs_fp.PopLowestIndex().D();
+#endif
+ } else {
+ DCHECK(args[i].IsNone());
+ arg_count = i;
+ break;
+ }
+
+ // If the argument is already in the right place, leave it where it is.
+ if (args[i].Aliases(pcs[i])) continue;
+
+ // Otherwise, if the argument is in a PCS argument register, allocate an
+ // appropriate scratch register and then move it out of the way.
+ if (kPCSVarargs.IncludesAliasOf(args[i]) ||
+ kPCSVarargsFP.IncludesAliasOf(args[i])) {
+ if (args[i].IsRegister()) {
+ Register old_arg = args[i].Reg();
+ Register new_arg = temps.AcquireSameSizeAs(old_arg);
+ Mov(new_arg, old_arg);
+ args[i] = new_arg;
+ } else {
+ VRegister old_arg = args[i].VReg();
+ VRegister new_arg = temps.AcquireSameSizeAs(old_arg);
+ Fmov(new_arg, old_arg);
+ args[i] = new_arg;
+ }
+ }
+ }
+
+ // Do a second pass to move values into their final positions and perform any
+ // conversions that may be required.
+ for (int i = 0; i < arg_count; i++) {
+#ifdef V8_OS_WIN
+ if (args[i].IsVRegister()) {
+ if (pcs[i].SizeInBytes() != args[i].SizeInBytes()) {
+ // If the argument is half- or single-precision
+ // converts to double-precision before that is
+ // moved into the one of X scratch register.
+ VRegister temp0 = temps.AcquireD();
+ Fcvt(temp0.VReg(), args[i].VReg());
+ Fmov(pcs[i].Reg(), temp0);
+ } else {
+ Fmov(pcs[i].Reg(), args[i].VReg());
+ }
+ } else {
+ Mov(pcs[i].Reg(), args[i].Reg(), kDiscardForSameWReg);
+ }
+#else
+ DCHECK(pcs[i].type() == args[i].type());
+ if (pcs[i].IsRegister()) {
+ Mov(pcs[i].Reg(), args[i].Reg(), kDiscardForSameWReg);
+ } else {
+ DCHECK(pcs[i].IsVRegister());
+ if (pcs[i].SizeInBytes() == args[i].SizeInBytes()) {
+ Fmov(pcs[i].VReg(), args[i].VReg());
+ } else {
+ Fcvt(pcs[i].VReg(), args[i].VReg());
+ }
+ }
+#endif
+ }
+
+ // Load the format string into x0, as per the procedure-call standard.
+ //
+ // To make the code as portable as possible, the format string is encoded
+ // directly in the instruction stream. It might be cleaner to encode it in a
+ // literal pool, but since Printf is usually used for debugging, it is
+ // beneficial for it to be minimally dependent on other features.
+ Label format_address;
+ Adr(x0, &format_address);
+
+ // Emit the format string directly in the instruction stream.
+ {
+ BlockPoolsScope scope(this);
+ Label after_data;
+ B(&after_data);
+ Bind(&format_address);
+ EmitStringData(format);
+ Unreachable();
+ Bind(&after_data);
+ }
+
+ CallPrintf(arg_count, pcs);
+}
+
+void TurboAssembler::CallPrintf(int arg_count, const CPURegister* args) {
+ // A call to printf needs special handling for the simulator, since the system
+ // printf function will use a different instruction set and the procedure-call
+ // standard will not be compatible.
+ if (options().enable_simulator_code) {
+ InstructionAccurateScope scope(this, kPrintfLength / kInstrSize);
+ hlt(kImmExceptionIsPrintf);
+ dc32(arg_count); // kPrintfArgCountOffset
+
+ // Determine the argument pattern.
+ uint32_t arg_pattern_list = 0;
+ for (int i = 0; i < arg_count; i++) {
+ uint32_t arg_pattern;
+ if (args[i].IsRegister()) {
+ arg_pattern = args[i].Is32Bits() ? kPrintfArgW : kPrintfArgX;
+ } else {
+ DCHECK(args[i].Is64Bits());
+ arg_pattern = kPrintfArgD;
+ }
+ DCHECK(arg_pattern < (1 << kPrintfArgPatternBits));
+ arg_pattern_list |= (arg_pattern << (kPrintfArgPatternBits * i));
+ }
+ dc32(arg_pattern_list); // kPrintfArgPatternListOffset
+ return;
+ }
+
+ Call(ExternalReference::printf_function());
+}
+
+void TurboAssembler::Printf(const char* format, CPURegister arg0,
+ CPURegister arg1, CPURegister arg2,
+ CPURegister arg3) {
+ // Printf is expected to preserve all registers, so make sure that none are
+ // available as scratch registers until we've preserved them.
+ RegList old_tmp_list = TmpList()->list();
+ RegList old_fp_tmp_list = FPTmpList()->list();
+ TmpList()->set_list(0);
+ FPTmpList()->set_list(0);
+
+ CPURegList saved_registers = kCallerSaved;
+ saved_registers.Align();
+
+ // Preserve all caller-saved registers as well as NZCV.
+ // PushCPURegList asserts that the size of each list is a multiple of 16
+ // bytes.
+ PushCPURegList<kDontStoreLR>(saved_registers);
+ PushCPURegList(kCallerSavedV);
+
+ // We can use caller-saved registers as scratch values (except for argN).
+ CPURegList tmp_list = saved_registers;
+ CPURegList fp_tmp_list = kCallerSavedV;
+ tmp_list.Remove(arg0, arg1, arg2, arg3);
+ fp_tmp_list.Remove(arg0, arg1, arg2, arg3);
+ TmpList()->set_list(tmp_list.list());
+ FPTmpList()->set_list(fp_tmp_list.list());
+
+ {
+ UseScratchRegisterScope temps(this);
+ // If any of the arguments are the current stack pointer, allocate a new
+ // register for them, and adjust the value to compensate for pushing the
+ // caller-saved registers.
+ bool arg0_sp = arg0.is_valid() && sp.Aliases(arg0);
+ bool arg1_sp = arg1.is_valid() && sp.Aliases(arg1);
+ bool arg2_sp = arg2.is_valid() && sp.Aliases(arg2);
+ bool arg3_sp = arg3.is_valid() && sp.Aliases(arg3);
+ if (arg0_sp || arg1_sp || arg2_sp || arg3_sp) {
+ // Allocate a register to hold the original stack pointer value, to pass
+ // to PrintfNoPreserve as an argument.
+ Register arg_sp = temps.AcquireX();
+ Add(arg_sp, sp,
+ saved_registers.TotalSizeInBytes() +
+ kCallerSavedV.TotalSizeInBytes());
+ if (arg0_sp) arg0 = Register::Create(arg_sp.code(), arg0.SizeInBits());
+ if (arg1_sp) arg1 = Register::Create(arg_sp.code(), arg1.SizeInBits());
+ if (arg2_sp) arg2 = Register::Create(arg_sp.code(), arg2.SizeInBits());
+ if (arg3_sp) arg3 = Register::Create(arg_sp.code(), arg3.SizeInBits());
+ }
+
+ // Preserve NZCV.
+ {
+ UseScratchRegisterScope temps(this);
+ Register tmp = temps.AcquireX();
+ Mrs(tmp, NZCV);
+ Push(tmp, xzr);
+ }
+
+ PrintfNoPreserve(format, arg0, arg1, arg2, arg3);
+
+ // Restore NZCV.
+ {
+ UseScratchRegisterScope temps(this);
+ Register tmp = temps.AcquireX();
+ Pop(xzr, tmp);
+ Msr(NZCV, tmp);
+ }
+ }
+
+ PopCPURegList(kCallerSavedV);
+ PopCPURegList<kDontLoadLR>(saved_registers);
+
+ TmpList()->set_list(old_tmp_list);
+ FPTmpList()->set_list(old_fp_tmp_list);
+}
+
+UseScratchRegisterScope::~UseScratchRegisterScope() {
+ available_->set_list(old_available_);
+ availablefp_->set_list(old_availablefp_);
+}
+
+Register UseScratchRegisterScope::AcquireSameSizeAs(const Register& reg) {
+ int code = AcquireNextAvailable(available_).code();
+ return Register::Create(code, reg.SizeInBits());
+}
+
+VRegister UseScratchRegisterScope::AcquireSameSizeAs(const VRegister& reg) {
+ int code = AcquireNextAvailable(availablefp_).code();
+ return VRegister::Create(code, reg.SizeInBits());
+}
+
+CPURegister UseScratchRegisterScope::AcquireNextAvailable(
+ CPURegList* available) {
+ CHECK(!available->IsEmpty());
+ CPURegister result = available->PopLowestIndex();
+ DCHECK(!AreAliased(result, xzr, sp));
+ return result;
+}
+
+void TurboAssembler::ComputeCodeStartAddress(const Register& rd) {
+ // We can use adr to load a pc relative location.
+ adr(rd, -pc_offset());
+}
+
+void TurboAssembler::ResetSpeculationPoisonRegister() {
+ Mov(kSpeculationPoisonRegister, -1);
+}
+
+void TurboAssembler::RestoreFPAndLR() {
+ static_assert(StandardFrameConstants::kCallerFPOffset + kSystemPointerSize ==
+ StandardFrameConstants::kCallerPCOffset,
+ "Offsets must be consecutive for ldp!");
+#ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
+ // Make sure we can use x16 and x17.
+ UseScratchRegisterScope temps(this);
+ temps.Exclude(x16, x17);
+ // We can load the return address directly into x17.
+ Add(x16, fp, StandardFrameConstants::kCallerSPOffset);
+ Ldp(fp, x17, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
+ Autib1716();
+ Mov(lr, x17);
+#else
+ Ldp(fp, lr, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
+#endif
+}
+
+void TurboAssembler::StoreReturnAddressInWasmExitFrame(Label* return_location) {
+ UseScratchRegisterScope temps(this);
+ temps.Exclude(x16, x17);
+ Adr(x17, return_location);
+#ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
+ Add(x16, fp, WasmExitFrameConstants::kCallingPCOffset + kSystemPointerSize);
+ Pacib1716();
+#endif
+ Str(x17, MemOperand(fp, WasmExitFrameConstants::kCallingPCOffset));
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_ARM64
diff --git a/src/codegen/arm64/macro-assembler-arm64.h b/src/codegen/arm64/macro-assembler-arm64.h
new file mode 100644
index 0000000..b453a17
--- /dev/null
+++ b/src/codegen/arm64/macro-assembler-arm64.h
@@ -0,0 +1,2140 @@
+// Copyright 2013 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef INCLUDED_FROM_MACRO_ASSEMBLER_H
+#error This header must be included via macro-assembler.h
+#endif
+
+#ifndef V8_CODEGEN_ARM64_MACRO_ASSEMBLER_ARM64_H_
+#define V8_CODEGEN_ARM64_MACRO_ASSEMBLER_ARM64_H_
+
+#include <vector>
+
+#include "src/base/bits.h"
+#include "src/codegen/arm64/assembler-arm64.h"
+#include "src/codegen/bailout-reason.h"
+#include "src/common/globals.h"
+
+// Simulator specific helpers.
+#if USE_SIMULATOR
+#if DEBUG
+#define ASM_LOCATION(message) __ Debug("LOCATION: " message, __LINE__, NO_PARAM)
+#define ASM_LOCATION_IN_ASSEMBLER(message) \
+ Debug("LOCATION: " message, __LINE__, NO_PARAM)
+#else
+#define ASM_LOCATION(message)
+#define ASM_LOCATION_IN_ASSEMBLER(message)
+#endif
+#else
+#define ASM_LOCATION(message)
+#define ASM_LOCATION_IN_ASSEMBLER(message)
+#endif
+
+namespace v8 {
+namespace internal {
+
+#define LS_MACRO_LIST(V) \
+ V(Ldrb, Register&, rt, LDRB_w) \
+ V(Strb, Register&, rt, STRB_w) \
+ V(Ldrsb, Register&, rt, rt.Is64Bits() ? LDRSB_x : LDRSB_w) \
+ V(Ldrh, Register&, rt, LDRH_w) \
+ V(Strh, Register&, rt, STRH_w) \
+ V(Ldrsh, Register&, rt, rt.Is64Bits() ? LDRSH_x : LDRSH_w) \
+ V(Ldr, CPURegister&, rt, LoadOpFor(rt)) \
+ V(Str, CPURegister&, rt, StoreOpFor(rt)) \
+ V(Ldrsw, Register&, rt, LDRSW_x)
+
+#define LSPAIR_MACRO_LIST(V) \
+ V(Ldp, CPURegister&, rt, rt2, LoadPairOpFor(rt, rt2)) \
+ V(Stp, CPURegister&, rt, rt2, StorePairOpFor(rt, rt2)) \
+ V(Ldpsw, CPURegister&, rt, rt2, LDPSW_x)
+
+#define LDA_STL_MACRO_LIST(V) \
+ V(Ldarb, ldarb) \
+ V(Ldarh, ldarh) \
+ V(Ldar, ldar) \
+ V(Ldaxrb, ldaxrb) \
+ V(Ldaxrh, ldaxrh) \
+ V(Ldaxr, ldaxr) \
+ V(Stlrb, stlrb) \
+ V(Stlrh, stlrh) \
+ V(Stlr, stlr)
+
+#define STLX_MACRO_LIST(V) \
+ V(Stlxrb, stlxrb) \
+ V(Stlxrh, stlxrh) \
+ V(Stlxr, stlxr)
+
+// ----------------------------------------------------------------------------
+// Static helper functions
+
+// Generate a MemOperand for loading a field from an object.
+inline MemOperand FieldMemOperand(Register object, int offset);
+
+// ----------------------------------------------------------------------------
+// MacroAssembler
+
+enum BranchType {
+ // Copies of architectural conditions.
+ // The associated conditions can be used in place of those, the code will
+ // take care of reinterpreting them with the correct type.
+ integer_eq = eq,
+ integer_ne = ne,
+ integer_hs = hs,
+ integer_lo = lo,
+ integer_mi = mi,
+ integer_pl = pl,
+ integer_vs = vs,
+ integer_vc = vc,
+ integer_hi = hi,
+ integer_ls = ls,
+ integer_ge = ge,
+ integer_lt = lt,
+ integer_gt = gt,
+ integer_le = le,
+ integer_al = al,
+ integer_nv = nv,
+
+ // These two are *different* from the architectural codes al and nv.
+ // 'always' is used to generate unconditional branches.
+ // 'never' is used to not generate a branch (generally as the inverse
+ // branch type of 'always).
+ always,
+ never,
+ // cbz and cbnz
+ reg_zero,
+ reg_not_zero,
+ // tbz and tbnz
+ reg_bit_clear,
+ reg_bit_set,
+
+ // Aliases.
+ kBranchTypeFirstCondition = eq,
+ kBranchTypeLastCondition = nv,
+ kBranchTypeFirstUsingReg = reg_zero,
+ kBranchTypeFirstUsingBit = reg_bit_clear
+};
+
+inline BranchType InvertBranchType(BranchType type) {
+ if (kBranchTypeFirstCondition <= type && type <= kBranchTypeLastCondition) {
+ return static_cast<BranchType>(
+ NegateCondition(static_cast<Condition>(type)));
+ } else {
+ return static_cast<BranchType>(type ^ 1);
+ }
+}
+
+enum RememberedSetAction { EMIT_REMEMBERED_SET, OMIT_REMEMBERED_SET };
+enum SmiCheck { INLINE_SMI_CHECK, OMIT_SMI_CHECK };
+enum LinkRegisterStatus { kLRHasNotBeenSaved, kLRHasBeenSaved };
+enum DiscardMoveMode { kDontDiscardForSameWReg, kDiscardForSameWReg };
+
+// The macro assembler supports moving automatically pre-shifted immediates for
+// arithmetic and logical instructions, and then applying a post shift in the
+// instruction to undo the modification, in order to reduce the code emitted for
+// an operation. For example:
+//
+// Add(x0, x0, 0x1f7de) => movz x16, 0xfbef; add x0, x0, x16, lsl #1.
+//
+// This optimisation can be only partially applied when the stack pointer is an
+// operand or destination, so this enumeration is used to control the shift.
+enum PreShiftImmMode {
+ kNoShift, // Don't pre-shift.
+ kLimitShiftForSP, // Limit pre-shift for add/sub extend use.
+ kAnyShift // Allow any pre-shift.
+};
+
+// TODO(victorgomes): Move definition to macro-assembler.h, once all other
+// platforms are updated.
+enum class StackLimitKind { kInterruptStackLimit, kRealStackLimit };
+
+class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase {
+ public:
+ using TurboAssemblerBase::TurboAssemblerBase;
+
+#if DEBUG
+ void set_allow_macro_instructions(bool value) {
+ allow_macro_instructions_ = value;
+ }
+ bool allow_macro_instructions() const { return allow_macro_instructions_; }
+#endif
+
+ // We should not use near calls or jumps for calls to external references,
+ // since the code spaces are not guaranteed to be close to each other.
+ bool CanUseNearCallOrJump(RelocInfo::Mode rmode) {
+ return rmode != RelocInfo::EXTERNAL_REFERENCE;
+ }
+
+ static bool IsNearCallOffset(int64_t offset);
+
+ // Activation support.
+ void EnterFrame(StackFrame::Type type);
+ void EnterFrame(StackFrame::Type type, bool load_constant_pool_pointer_reg) {
+ // Out-of-line constant pool not implemented on arm64.
+ UNREACHABLE();
+ }
+ void LeaveFrame(StackFrame::Type type);
+
+ inline void InitializeRootRegister();
+
+ void Mov(const Register& rd, const Operand& operand,
+ DiscardMoveMode discard_mode = kDontDiscardForSameWReg);
+ void Mov(const Register& rd, uint64_t imm);
+ void Mov(const VRegister& vd, int vd_index, const VRegister& vn,
+ int vn_index) {
+ DCHECK(allow_macro_instructions());
+ mov(vd, vd_index, vn, vn_index);
+ }
+ void Mov(const Register& rd, Smi smi);
+ void Mov(const VRegister& vd, const VRegister& vn, int index) {
+ DCHECK(allow_macro_instructions());
+ mov(vd, vn, index);
+ }
+ void Mov(const VRegister& vd, int vd_index, const Register& rn) {
+ DCHECK(allow_macro_instructions());
+ mov(vd, vd_index, rn);
+ }
+ void Mov(const Register& rd, const VRegister& vn, int vn_index) {
+ DCHECK(allow_macro_instructions());
+ mov(rd, vn, vn_index);
+ }
+
+ // This is required for compatibility with architecture independent code.
+ // Remove if not needed.
+ void Move(Register dst, Smi src);
+
+ // Move src0 to dst0 and src1 to dst1, handling possible overlaps.
+ void MovePair(Register dst0, Register src0, Register dst1, Register src1);
+
+ // Register swap. Note that the register operands should be distinct.
+ void Swap(Register lhs, Register rhs);
+ void Swap(VRegister lhs, VRegister rhs);
+
+// NEON by element instructions.
+#define NEON_BYELEMENT_MACRO_LIST(V) \
+ V(fmla, Fmla) \
+ V(fmls, Fmls) \
+ V(fmul, Fmul) \
+ V(fmulx, Fmulx) \
+ V(mul, Mul) \
+ V(mla, Mla) \
+ V(mls, Mls) \
+ V(sqdmulh, Sqdmulh) \
+ V(sqrdmulh, Sqrdmulh) \
+ V(sqdmull, Sqdmull) \
+ V(sqdmull2, Sqdmull2) \
+ V(sqdmlal, Sqdmlal) \
+ V(sqdmlal2, Sqdmlal2) \
+ V(sqdmlsl, Sqdmlsl) \
+ V(sqdmlsl2, Sqdmlsl2) \
+ V(smull, Smull) \
+ V(smull2, Smull2) \
+ V(smlal, Smlal) \
+ V(smlal2, Smlal2) \
+ V(smlsl, Smlsl) \
+ V(smlsl2, Smlsl2) \
+ V(umull, Umull) \
+ V(umull2, Umull2) \
+ V(umlal, Umlal) \
+ V(umlal2, Umlal2) \
+ V(umlsl, Umlsl) \
+ V(umlsl2, Umlsl2)
+
+#define DEFINE_MACRO_ASM_FUNC(ASM, MASM) \
+ void MASM(const VRegister& vd, const VRegister& vn, const VRegister& vm, \
+ int vm_index) { \
+ DCHECK(allow_macro_instructions()); \
+ ASM(vd, vn, vm, vm_index); \
+ }
+ NEON_BYELEMENT_MACRO_LIST(DEFINE_MACRO_ASM_FUNC)
+#undef DEFINE_MACRO_ASM_FUNC
+
+// NEON 2 vector register instructions.
+#define NEON_2VREG_MACRO_LIST(V) \
+ V(abs, Abs) \
+ V(addp, Addp) \
+ V(addv, Addv) \
+ V(cls, Cls) \
+ V(clz, Clz) \
+ V(cnt, Cnt) \
+ V(faddp, Faddp) \
+ V(fcvtas, Fcvtas) \
+ V(fcvtau, Fcvtau) \
+ V(fcvtms, Fcvtms) \
+ V(fcvtmu, Fcvtmu) \
+ V(fcvtns, Fcvtns) \
+ V(fcvtnu, Fcvtnu) \
+ V(fcvtps, Fcvtps) \
+ V(fcvtpu, Fcvtpu) \
+ V(fmaxnmp, Fmaxnmp) \
+ V(fmaxnmv, Fmaxnmv) \
+ V(fmaxp, Fmaxp) \
+ V(fmaxv, Fmaxv) \
+ V(fminnmp, Fminnmp) \
+ V(fminnmv, Fminnmv) \
+ V(fminp, Fminp) \
+ V(fminv, Fminv) \
+ V(fneg, Fneg) \
+ V(frecpe, Frecpe) \
+ V(frecpx, Frecpx) \
+ V(frinta, Frinta) \
+ V(frinti, Frinti) \
+ V(frintm, Frintm) \
+ V(frintn, Frintn) \
+ V(frintp, Frintp) \
+ V(frintx, Frintx) \
+ V(frintz, Frintz) \
+ V(frsqrte, Frsqrte) \
+ V(fsqrt, Fsqrt) \
+ V(mov, Mov) \
+ V(mvn, Mvn) \
+ V(neg, Neg) \
+ V(not_, Not) \
+ V(rbit, Rbit) \
+ V(rev16, Rev16) \
+ V(rev32, Rev32) \
+ V(rev64, Rev64) \
+ V(sadalp, Sadalp) \
+ V(saddlp, Saddlp) \
+ V(saddlv, Saddlv) \
+ V(smaxv, Smaxv) \
+ V(sminv, Sminv) \
+ V(sqabs, Sqabs) \
+ V(sqneg, Sqneg) \
+ V(sqxtn2, Sqxtn2) \
+ V(sqxtn, Sqxtn) \
+ V(sqxtun2, Sqxtun2) \
+ V(sqxtun, Sqxtun) \
+ V(suqadd, Suqadd) \
+ V(sxtl2, Sxtl2) \
+ V(sxtl, Sxtl) \
+ V(uadalp, Uadalp) \
+ V(uaddlp, Uaddlp) \
+ V(uaddlv, Uaddlv) \
+ V(umaxv, Umaxv) \
+ V(uminv, Uminv) \
+ V(uqxtn2, Uqxtn2) \
+ V(uqxtn, Uqxtn) \
+ V(urecpe, Urecpe) \
+ V(ursqrte, Ursqrte) \
+ V(usqadd, Usqadd) \
+ V(uxtl2, Uxtl2) \
+ V(uxtl, Uxtl) \
+ V(xtn2, Xtn2) \
+ V(xtn, Xtn)
+
+#define DEFINE_MACRO_ASM_FUNC(ASM, MASM) \
+ void MASM(const VRegister& vd, const VRegister& vn) { \
+ DCHECK(allow_macro_instructions()); \
+ ASM(vd, vn); \
+ }
+ NEON_2VREG_MACRO_LIST(DEFINE_MACRO_ASM_FUNC)
+#undef DEFINE_MACRO_ASM_FUNC
+#undef NEON_2VREG_MACRO_LIST
+
+// NEON 2 vector register with immediate instructions.
+#define NEON_2VREG_FPIMM_MACRO_LIST(V) \
+ V(fcmeq, Fcmeq) \
+ V(fcmge, Fcmge) \
+ V(fcmgt, Fcmgt) \
+ V(fcmle, Fcmle) \
+ V(fcmlt, Fcmlt)
+
+#define DEFINE_MACRO_ASM_FUNC(ASM, MASM) \
+ void MASM(const VRegister& vd, const VRegister& vn, double imm) { \
+ DCHECK(allow_macro_instructions()); \
+ ASM(vd, vn, imm); \
+ }
+ NEON_2VREG_FPIMM_MACRO_LIST(DEFINE_MACRO_ASM_FUNC)
+#undef DEFINE_MACRO_ASM_FUNC
+
+// NEON 3 vector register instructions.
+#define NEON_3VREG_MACRO_LIST(V) \
+ V(add, Add) \
+ V(addhn2, Addhn2) \
+ V(addhn, Addhn) \
+ V(addp, Addp) \
+ V(and_, And) \
+ V(bic, Bic) \
+ V(bif, Bif) \
+ V(bit, Bit) \
+ V(bsl, Bsl) \
+ V(cmeq, Cmeq) \
+ V(cmge, Cmge) \
+ V(cmgt, Cmgt) \
+ V(cmhi, Cmhi) \
+ V(cmhs, Cmhs) \
+ V(cmtst, Cmtst) \
+ V(eor, Eor) \
+ V(fabd, Fabd) \
+ V(facge, Facge) \
+ V(facgt, Facgt) \
+ V(faddp, Faddp) \
+ V(fcmeq, Fcmeq) \
+ V(fcmge, Fcmge) \
+ V(fcmgt, Fcmgt) \
+ V(fmaxnmp, Fmaxnmp) \
+ V(fmaxp, Fmaxp) \
+ V(fminnmp, Fminnmp) \
+ V(fminp, Fminp) \
+ V(fmla, Fmla) \
+ V(fmls, Fmls) \
+ V(fmulx, Fmulx) \
+ V(fnmul, Fnmul) \
+ V(frecps, Frecps) \
+ V(frsqrts, Frsqrts) \
+ V(mla, Mla) \
+ V(mls, Mls) \
+ V(mul, Mul) \
+ V(orn, Orn) \
+ V(orr, Orr) \
+ V(pmull2, Pmull2) \
+ V(pmull, Pmull) \
+ V(pmul, Pmul) \
+ V(raddhn2, Raddhn2) \
+ V(raddhn, Raddhn) \
+ V(rsubhn2, Rsubhn2) \
+ V(rsubhn, Rsubhn) \
+ V(sabal2, Sabal2) \
+ V(sabal, Sabal) \
+ V(saba, Saba) \
+ V(sabdl2, Sabdl2) \
+ V(sabdl, Sabdl) \
+ V(sabd, Sabd) \
+ V(saddl2, Saddl2) \
+ V(saddl, Saddl) \
+ V(saddw2, Saddw2) \
+ V(saddw, Saddw) \
+ V(shadd, Shadd) \
+ V(shsub, Shsub) \
+ V(smaxp, Smaxp) \
+ V(smax, Smax) \
+ V(sminp, Sminp) \
+ V(smin, Smin) \
+ V(smlal2, Smlal2) \
+ V(smlal, Smlal) \
+ V(smlsl2, Smlsl2) \
+ V(smlsl, Smlsl) \
+ V(smull2, Smull2) \
+ V(smull, Smull) \
+ V(sqadd, Sqadd) \
+ V(sqdmlal2, Sqdmlal2) \
+ V(sqdmlal, Sqdmlal) \
+ V(sqdmlsl2, Sqdmlsl2) \
+ V(sqdmlsl, Sqdmlsl) \
+ V(sqdmulh, Sqdmulh) \
+ V(sqdmull2, Sqdmull2) \
+ V(sqdmull, Sqdmull) \
+ V(sqrdmulh, Sqrdmulh) \
+ V(sqrshl, Sqrshl) \
+ V(sqshl, Sqshl) \
+ V(sqsub, Sqsub) \
+ V(srhadd, Srhadd) \
+ V(srshl, Srshl) \
+ V(sshl, Sshl) \
+ V(ssubl2, Ssubl2) \
+ V(ssubl, Ssubl) \
+ V(ssubw2, Ssubw2) \
+ V(ssubw, Ssubw) \
+ V(subhn2, Subhn2) \
+ V(subhn, Subhn) \
+ V(sub, Sub) \
+ V(trn1, Trn1) \
+ V(trn2, Trn2) \
+ V(uabal2, Uabal2) \
+ V(uabal, Uabal) \
+ V(uaba, Uaba) \
+ V(uabdl2, Uabdl2) \
+ V(uabdl, Uabdl) \
+ V(uabd, Uabd) \
+ V(uaddl2, Uaddl2) \
+ V(uaddl, Uaddl) \
+ V(uaddw2, Uaddw2) \
+ V(uaddw, Uaddw) \
+ V(uhadd, Uhadd) \
+ V(uhsub, Uhsub) \
+ V(umaxp, Umaxp) \
+ V(umax, Umax) \
+ V(uminp, Uminp) \
+ V(umin, Umin) \
+ V(umlal2, Umlal2) \
+ V(umlal, Umlal) \
+ V(umlsl2, Umlsl2) \
+ V(umlsl, Umlsl) \
+ V(umull2, Umull2) \
+ V(umull, Umull) \
+ V(uqadd, Uqadd) \
+ V(uqrshl, Uqrshl) \
+ V(uqshl, Uqshl) \
+ V(uqsub, Uqsub) \
+ V(urhadd, Urhadd) \
+ V(urshl, Urshl) \
+ V(ushl, Ushl) \
+ V(usubl2, Usubl2) \
+ V(usubl, Usubl) \
+ V(usubw2, Usubw2) \
+ V(usubw, Usubw) \
+ V(uzp1, Uzp1) \
+ V(uzp2, Uzp2) \
+ V(zip1, Zip1) \
+ V(zip2, Zip2)
+
+#define DEFINE_MACRO_ASM_FUNC(ASM, MASM) \
+ void MASM(const VRegister& vd, const VRegister& vn, const VRegister& vm) { \
+ DCHECK(allow_macro_instructions()); \
+ ASM(vd, vn, vm); \
+ }
+ NEON_3VREG_MACRO_LIST(DEFINE_MACRO_ASM_FUNC)
+#undef DEFINE_MACRO_ASM_FUNC
+
+ void Bic(const VRegister& vd, const int imm8, const int left_shift = 0) {
+ DCHECK(allow_macro_instructions());
+ bic(vd, imm8, left_shift);
+ }
+
+ // This is required for compatibility in architecture independent code.
+ inline void jmp(Label* L);
+
+ void B(Label* label, BranchType type, Register reg = NoReg, int bit = -1);
+ inline void B(Label* label);
+ inline void B(Condition cond, Label* label);
+ void B(Label* label, Condition cond);
+
+ void Tbnz(const Register& rt, unsigned bit_pos, Label* label);
+ void Tbz(const Register& rt, unsigned bit_pos, Label* label);
+
+ void Cbnz(const Register& rt, Label* label);
+ void Cbz(const Register& rt, Label* label);
+
+ void Pacibsp() {
+ DCHECK(allow_macro_instructions_);
+ pacibsp();
+ }
+ void Autibsp() {
+ DCHECK(allow_macro_instructions_);
+ autibsp();
+ }
+
+ // The 1716 pac and aut instructions encourage people to use x16 and x17
+ // directly, perhaps without realising that this is forbidden. For example:
+ //
+ // UseScratchRegisterScope temps(&masm);
+ // Register temp = temps.AcquireX(); // temp will be x16
+ // __ Mov(x17, ptr);
+ // __ Mov(x16, modifier); // Will override temp!
+ // __ Pacib1716();
+ //
+ // To work around this issue, you must exclude x16 and x17 from the scratch
+ // register list. You may need to replace them with other registers:
+ //
+ // UseScratchRegisterScope temps(&masm);
+ // temps.Exclude(x16, x17);
+ // temps.Include(x10, x11);
+ // __ Mov(x17, ptr);
+ // __ Mov(x16, modifier);
+ // __ Pacib1716();
+ void Pacib1716() {
+ DCHECK(allow_macro_instructions_);
+ DCHECK(!TmpList()->IncludesAliasOf(x16));
+ DCHECK(!TmpList()->IncludesAliasOf(x17));
+ pacib1716();
+ }
+ void Autib1716() {
+ DCHECK(allow_macro_instructions_);
+ DCHECK(!TmpList()->IncludesAliasOf(x16));
+ DCHECK(!TmpList()->IncludesAliasOf(x17));
+ autib1716();
+ }
+
+ inline void Dmb(BarrierDomain domain, BarrierType type);
+ inline void Dsb(BarrierDomain domain, BarrierType type);
+ inline void Isb();
+ inline void Csdb();
+
+ // Removes current frame and its arguments from the stack preserving
+ // the arguments and a return address pushed to the stack for the next call.
+ // Both |callee_args_count| and |caller_args_count| do not include
+ // receiver. |callee_args_count| is not modified. |caller_args_count| is
+ // trashed.
+ void PrepareForTailCall(Register callee_args_count,
+ Register caller_args_count, Register scratch0,
+ Register scratch1);
+
+ inline void SmiUntag(Register dst, Register src);
+ inline void SmiUntag(Register dst, const MemOperand& src);
+ inline void SmiUntag(Register smi);
+
+ // Calls Abort(msg) if the condition cond is not satisfied.
+ // Use --debug_code to enable.
+ void Assert(Condition cond, AbortReason reason);
+
+ // Like Assert(), but without condition.
+ // Use --debug_code to enable.
+ void AssertUnreachable(AbortReason reason);
+
+ void AssertSmi(Register object,
+ AbortReason reason = AbortReason::kOperandIsNotASmi);
+
+ // Like Assert(), but always enabled.
+ void Check(Condition cond, AbortReason reason);
+
+ inline void Debug(const char* message, uint32_t code, Instr params = BREAK);
+
+ void Trap() override;
+ void DebugBreak() override;
+
+ // Print a message to stderr and abort execution.
+ void Abort(AbortReason reason);
+
+ // Like printf, but print at run-time from generated code.
+ //
+ // The caller must ensure that arguments for floating-point placeholders
+ // (such as %e, %f or %g) are VRegisters, and that arguments for integer
+ // placeholders are Registers.
+ //
+ // Format placeholders that refer to more than one argument, or to a specific
+ // argument, are not supported. This includes formats like "%1$d" or "%.*d".
+ //
+ // This function automatically preserves caller-saved registers so that
+ // calling code can use Printf at any point without having to worry about
+ // corruption. The preservation mechanism generates a lot of code. If this is
+ // a problem, preserve the important registers manually and then call
+ // PrintfNoPreserve. Callee-saved registers are not used by Printf, and are
+ // implicitly preserved.
+ void Printf(const char* format, CPURegister arg0 = NoCPUReg,
+ CPURegister arg1 = NoCPUReg, CPURegister arg2 = NoCPUReg,
+ CPURegister arg3 = NoCPUReg);
+
+ // Like Printf, but don't preserve any caller-saved registers, not even 'lr'.
+ //
+ // The return code from the system printf call will be returned in x0.
+ void PrintfNoPreserve(const char* format, const CPURegister& arg0 = NoCPUReg,
+ const CPURegister& arg1 = NoCPUReg,
+ const CPURegister& arg2 = NoCPUReg,
+ const CPURegister& arg3 = NoCPUReg);
+
+ // Remaining instructions are simple pass-through calls to the assembler.
+ inline void Asr(const Register& rd, const Register& rn, unsigned shift);
+ inline void Asr(const Register& rd, const Register& rn, const Register& rm);
+
+ // Try to move an immediate into the destination register in a single
+ // instruction. Returns true for success, and updates the contents of dst.
+ // Returns false, otherwise.
+ bool TryOneInstrMoveImmediate(const Register& dst, int64_t imm);
+
+ inline void Bind(Label* label,
+ BranchTargetIdentifier id = BranchTargetIdentifier::kNone);
+
+ // Control-flow integrity:
+
+ // Define a function entrypoint.
+ inline void CodeEntry();
+ // Define an exception handler.
+ inline void ExceptionHandler();
+ // Define an exception handler and bind a label.
+ inline void BindExceptionHandler(Label* label);
+
+ // Control-flow integrity:
+
+ // Define a jump (BR) target.
+ inline void JumpTarget();
+ // Define a jump (BR) target and bind a label.
+ inline void BindJumpTarget(Label* label);
+ // Define a call (BLR) target. The target also allows tail calls (via BR)
+ // when the target is x16 or x17.
+ inline void CallTarget();
+ // Define a jump/call target.
+ inline void JumpOrCallTarget();
+ // Define a jump/call target and bind a label.
+ inline void BindJumpOrCallTarget(Label* label);
+
+ static unsigned CountClearHalfWords(uint64_t imm, unsigned reg_size);
+
+ CPURegList* TmpList() { return &tmp_list_; }
+ CPURegList* FPTmpList() { return &fptmp_list_; }
+
+ static CPURegList DefaultTmpList();
+ static CPURegList DefaultFPTmpList();
+
+ // Move macros.
+ inline void Mvn(const Register& rd, uint64_t imm);
+ void Mvn(const Register& rd, const Operand& operand);
+ static bool IsImmMovn(uint64_t imm, unsigned reg_size);
+ static bool IsImmMovz(uint64_t imm, unsigned reg_size);
+
+ void LogicalMacro(const Register& rd, const Register& rn,
+ const Operand& operand, LogicalOp op);
+ void AddSubMacro(const Register& rd, const Register& rn,
+ const Operand& operand, FlagsUpdate S, AddSubOp op);
+ inline void Orr(const Register& rd, const Register& rn,
+ const Operand& operand);
+ void Orr(const VRegister& vd, const int imm8, const int left_shift = 0) {
+ DCHECK(allow_macro_instructions());
+ orr(vd, imm8, left_shift);
+ }
+ inline void Orn(const Register& rd, const Register& rn,
+ const Operand& operand);
+ inline void Eor(const Register& rd, const Register& rn,
+ const Operand& operand);
+ inline void Eon(const Register& rd, const Register& rn,
+ const Operand& operand);
+ inline void And(const Register& rd, const Register& rn,
+ const Operand& operand);
+ inline void Ands(const Register& rd, const Register& rn,
+ const Operand& operand);
+ inline void Tst(const Register& rn, const Operand& operand);
+ inline void Bic(const Register& rd, const Register& rn,
+ const Operand& operand);
+ inline void Blr(const Register& xn);
+ inline void Cmp(const Register& rn, const Operand& operand);
+ inline void CmpTagged(const Register& rn, const Operand& operand);
+ inline void Subs(const Register& rd, const Register& rn,
+ const Operand& operand);
+ void Csel(const Register& rd, const Register& rn, const Operand& operand,
+ Condition cond);
+
+ // Emits a runtime assert that the stack pointer is aligned.
+ void AssertSpAligned();
+
+ // Copy slot_count stack slots from the stack offset specified by src to
+ // the stack offset specified by dst. The offsets and count are expressed in
+ // slot-sized units. Offset dst must be less than src, or the gap between
+ // them must be greater than or equal to slot_count, otherwise the result is
+ // unpredictable. The function may corrupt its register arguments. The
+ // registers must not alias each other.
+ void CopySlots(int dst, Register src, Register slot_count);
+ void CopySlots(Register dst, Register src, Register slot_count);
+
+ // Copy count double words from the address in register src to the address
+ // in register dst. There are three modes for this function:
+ // 1) Address dst must be less than src, or the gap between them must be
+ // greater than or equal to count double words, otherwise the result is
+ // unpredictable. This is the default mode.
+ // 2) Address src must be less than dst, or the gap between them must be
+ // greater than or equal to count double words, otherwise the result is
+ // undpredictable. In this mode, src and dst specify the last (highest)
+ // address of the regions to copy from and to.
+ // 3) The same as mode 1, but the words are copied in the reversed order.
+ // The case where src == dst is not supported.
+ // The function may corrupt its register arguments. The registers must not
+ // alias each other.
+ enum CopyDoubleWordsMode {
+ kDstLessThanSrc,
+ kSrcLessThanDst,
+ kDstLessThanSrcAndReverse
+ };
+ void CopyDoubleWords(Register dst, Register src, Register count,
+ CopyDoubleWordsMode mode = kDstLessThanSrc);
+
+ // Calculate the address of a double word-sized slot at slot_offset from the
+ // stack pointer, and write it to dst. Positive slot_offsets are at addresses
+ // greater than sp, with slot zero at sp.
+ void SlotAddress(Register dst, int slot_offset);
+ void SlotAddress(Register dst, Register slot_offset);
+
+ // Load a literal from the inline constant pool.
+ inline void Ldr(const CPURegister& rt, const Operand& imm);
+
+ // Claim or drop stack space.
+ //
+ // On Windows, Claim will write a value every 4k, as is required by the stack
+ // expansion mechanism.
+ //
+ // The stack pointer must be aligned to 16 bytes and the size claimed or
+ // dropped must be a multiple of 16 bytes.
+ //
+ // Note that unit_size must be specified in bytes. For variants which take a
+ // Register count, the unit size must be a power of two.
+ inline void Claim(int64_t count, uint64_t unit_size = kXRegSize);
+ inline void Claim(const Register& count, uint64_t unit_size = kXRegSize);
+ inline void Drop(int64_t count, uint64_t unit_size = kXRegSize);
+ inline void Drop(const Register& count, uint64_t unit_size = kXRegSize);
+
+ // Drop 'count' arguments from the stack, rounded up to a multiple of two,
+ // without actually accessing memory.
+ // We assume the size of the arguments is the pointer size.
+ // An optional mode argument is passed, which can indicate we need to
+ // explicitly add the receiver to the count.
+ enum ArgumentsCountMode { kCountIncludesReceiver, kCountExcludesReceiver };
+ inline void DropArguments(const Register& count,
+ ArgumentsCountMode mode = kCountIncludesReceiver);
+ inline void DropArguments(int64_t count,
+ ArgumentsCountMode mode = kCountIncludesReceiver);
+
+ // Drop 'count' slots from stack, rounded up to a multiple of two, without
+ // actually accessing memory.
+ inline void DropSlots(int64_t count);
+
+ // Push a single argument, with padding, to the stack.
+ inline void PushArgument(const Register& arg);
+
+ // Add and sub macros.
+ inline void Add(const Register& rd, const Register& rn,
+ const Operand& operand);
+ inline void Adds(const Register& rd, const Register& rn,
+ const Operand& operand);
+ inline void Sub(const Register& rd, const Register& rn,
+ const Operand& operand);
+
+ // Abort execution if argument is not a positive or zero integer, enabled via
+ // --debug-code.
+ void AssertPositiveOrZero(Register value);
+
+#define DECLARE_FUNCTION(FN, REGTYPE, REG, OP) \
+ inline void FN(const REGTYPE REG, const MemOperand& addr);
+ LS_MACRO_LIST(DECLARE_FUNCTION)
+#undef DECLARE_FUNCTION
+
+ // Push or pop up to 4 registers of the same width to or from the stack.
+ //
+ // If an argument register is 'NoReg', all further arguments are also assumed
+ // to be 'NoReg', and are thus not pushed or popped.
+ //
+ // Arguments are ordered such that "Push(a, b);" is functionally equivalent
+ // to "Push(a); Push(b);".
+ //
+ // It is valid to push the same register more than once, and there is no
+ // restriction on the order in which registers are specified.
+ //
+ // It is not valid to pop into the same register more than once in one
+ // operation, not even into the zero register.
+ //
+ // The stack pointer must be aligned to 16 bytes on entry and the total size
+ // of the specified registers must also be a multiple of 16 bytes.
+ //
+ // Other than the registers passed into Pop, the stack pointer, (possibly)
+ // the system stack pointer and (possibly) the link register, these methods
+ // do not modify any other registers.
+ //
+ // Some of the methods take an optional LoadLRMode or StoreLRMode template
+ // argument, which specifies whether we need to sign the link register at the
+ // start of the operation, or authenticate it at the end of the operation,
+ // when control flow integrity measures are enabled.
+ // When the mode is kDontLoadLR or kDontStoreLR, LR must not be passed as an
+ // argument to the operation.
+ enum LoadLRMode { kAuthLR, kDontLoadLR };
+ enum StoreLRMode { kSignLR, kDontStoreLR };
+ template <StoreLRMode lr_mode = kDontStoreLR>
+ void Push(const CPURegister& src0, const CPURegister& src1 = NoReg,
+ const CPURegister& src2 = NoReg, const CPURegister& src3 = NoReg);
+ void Push(const CPURegister& src0, const CPURegister& src1,
+ const CPURegister& src2, const CPURegister& src3,
+ const CPURegister& src4, const CPURegister& src5 = NoReg,
+ const CPURegister& src6 = NoReg, const CPURegister& src7 = NoReg);
+ template <LoadLRMode lr_mode = kDontLoadLR>
+ void Pop(const CPURegister& dst0, const CPURegister& dst1 = NoReg,
+ const CPURegister& dst2 = NoReg, const CPURegister& dst3 = NoReg);
+ void Pop(const CPURegister& dst0, const CPURegister& dst1,
+ const CPURegister& dst2, const CPURegister& dst3,
+ const CPURegister& dst4, const CPURegister& dst5 = NoReg,
+ const CPURegister& dst6 = NoReg, const CPURegister& dst7 = NoReg);
+ template <StoreLRMode lr_mode = kDontStoreLR>
+ void Push(const Register& src0, const VRegister& src1);
+
+ // This is a convenience method for pushing a single Handle<Object>.
+ inline void Push(Handle<HeapObject> object);
+ inline void Push(Smi smi);
+
+ // Aliases of Push and Pop, required for V8 compatibility.
+ inline void push(Register src) { Push(src); }
+ inline void pop(Register dst) { Pop(dst); }
+
+ void SaveRegisters(RegList registers);
+ void RestoreRegisters(RegList registers);
+
+ void CallRecordWriteStub(Register object, Operand offset,
+ RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode);
+ void CallRecordWriteStub(Register object, Operand offset,
+ RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode, Address wasm_target);
+ void CallEphemeronKeyBarrier(Register object, Operand offset,
+ SaveFPRegsMode fp_mode);
+
+ // For a given |object| and |offset|:
+ // - Move |object| to |dst_object|.
+ // - Compute the address of the slot pointed to by |offset| in |object| and
+ // write it to |dst_slot|.
+ // This method makes sure |object| and |offset| are allowed to overlap with
+ // the destination registers.
+ void MoveObjectAndSlot(Register dst_object, Register dst_slot,
+ Register object, Operand offset);
+
+ // Alternative forms of Push and Pop, taking a RegList or CPURegList that
+ // specifies the registers that are to be pushed or popped. Higher-numbered
+ // registers are associated with higher memory addresses (as in the A32 push
+ // and pop instructions).
+ //
+ // (Push|Pop)SizeRegList allow you to specify the register size as a
+ // parameter. Only kXRegSizeInBits, kWRegSizeInBits, kDRegSizeInBits and
+ // kSRegSizeInBits are supported.
+ //
+ // Otherwise, (Push|Pop)(CPU|X|W|D|S)RegList is preferred.
+ //
+ // The methods take an optional LoadLRMode or StoreLRMode template argument.
+ // When control flow integrity measures are enabled and the link register is
+ // included in 'registers', passing kSignLR to PushCPURegList will sign the
+ // link register before pushing the list, and passing kAuthLR to
+ // PopCPURegList will authenticate it after popping the list.
+ template <StoreLRMode lr_mode = kDontStoreLR>
+ void PushCPURegList(CPURegList registers);
+ template <LoadLRMode lr_mode = kDontLoadLR>
+ void PopCPURegList(CPURegList registers);
+
+ // Calculate how much stack space (in bytes) are required to store caller
+ // registers excluding those specified in the arguments.
+ int RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
+ Register exclusion) const;
+
+ // Push caller saved registers on the stack, and return the number of bytes
+ // stack pointer is adjusted.
+ int PushCallerSaved(SaveFPRegsMode fp_mode, Register exclusion = no_reg);
+
+ // Restore caller saved registers from the stack, and return the number of
+ // bytes stack pointer is adjusted.
+ int PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion = no_reg);
+
+ // Move an immediate into register dst, and return an Operand object for use
+ // with a subsequent instruction that accepts a shift. The value moved into
+ // dst is not necessarily equal to imm; it may have had a shifting operation
+ // applied to it that will be subsequently undone by the shift applied in the
+ // Operand.
+ Operand MoveImmediateForShiftedOp(const Register& dst, int64_t imm,
+ PreShiftImmMode mode);
+
+ void CheckPageFlag(const Register& object, int mask, Condition cc,
+ Label* condition_met);
+
+ // Compare a register with an operand, and branch to label depending on the
+ // condition. May corrupt the status flags.
+ inline void CompareAndBranch(const Register& lhs, const Operand& rhs,
+ Condition cond, Label* label);
+ inline void CompareTaggedAndBranch(const Register& lhs, const Operand& rhs,
+ Condition cond, Label* label);
+
+ // Test the bits of register defined by bit_pattern, and branch if ANY of
+ // those bits are set. May corrupt the status flags.
+ inline void TestAndBranchIfAnySet(const Register& reg,
+ const uint64_t bit_pattern, Label* label);
+
+ // Test the bits of register defined by bit_pattern, and branch if ALL of
+ // those bits are clear (ie. not set.) May corrupt the status flags.
+ inline void TestAndBranchIfAllClear(const Register& reg,
+ const uint64_t bit_pattern, Label* label);
+
+ inline void Brk(int code);
+
+ inline void JumpIfSmi(Register value, Label* smi_label,
+ Label* not_smi_label = nullptr);
+
+ inline void JumpIfEqual(Register x, int32_t y, Label* dest);
+ inline void JumpIfLessThan(Register x, int32_t y, Label* dest);
+
+ inline void Fmov(VRegister fd, VRegister fn);
+ inline void Fmov(VRegister fd, Register rn);
+ // Provide explicit double and float interfaces for FP immediate moves, rather
+ // than relying on implicit C++ casts. This allows signalling NaNs to be
+ // preserved when the immediate matches the format of fd. Most systems convert
+ // signalling NaNs to quiet NaNs when converting between float and double.
+ inline void Fmov(VRegister fd, double imm);
+ inline void Fmov(VRegister fd, float imm);
+ // Provide a template to allow other types to be converted automatically.
+ template <typename T>
+ void Fmov(VRegister fd, T imm) {
+ DCHECK(allow_macro_instructions());
+ Fmov(fd, static_cast<double>(imm));
+ }
+ inline void Fmov(Register rd, VRegister fn);
+
+ void Movi(const VRegister& vd, uint64_t imm, Shift shift = LSL,
+ int shift_amount = 0);
+ void Movi(const VRegister& vd, uint64_t hi, uint64_t lo);
+
+ void LoadFromConstantsTable(Register destination,
+ int constant_index) override;
+ void LoadRootRegisterOffset(Register destination, intptr_t offset) override;
+ void LoadRootRelative(Register destination, int32_t offset) override;
+
+ void Jump(Register target, Condition cond = al);
+ void Jump(Address target, RelocInfo::Mode rmode, Condition cond = al);
+ void Jump(Handle<Code> code, RelocInfo::Mode rmode, Condition cond = al);
+ void Jump(const ExternalReference& reference) override;
+
+ void Call(Register target);
+ void Call(Address target, RelocInfo::Mode rmode);
+ void Call(Handle<Code> code, RelocInfo::Mode rmode = RelocInfo::CODE_TARGET);
+ void Call(ExternalReference target);
+
+ // Generate an indirect call (for when a direct call's range is not adequate).
+ void IndirectCall(Address target, RelocInfo::Mode rmode);
+
+ // Load the builtin given by the Smi in |builtin_index| into the same
+ // register.
+ void LoadEntryFromBuiltinIndex(Register builtin_index);
+ void LoadEntryFromBuiltinIndex(Builtins::Name builtin_index,
+ Register destination);
+ void CallBuiltinByIndex(Register builtin_index) override;
+ void CallBuiltin(int builtin_index);
+
+ void LoadCodeObjectEntry(Register destination, Register code_object) override;
+ void CallCodeObject(Register code_object) override;
+ void JumpCodeObject(Register code_object) override;
+
+ // Generates an instruction sequence s.t. the return address points to the
+ // instruction following the call.
+ // The return address on the stack is used by frame iteration.
+ void StoreReturnAddressAndCall(Register target);
+
+ void CallForDeoptimization(Builtins::Name target, int deopt_id, Label* exit,
+ DeoptimizeKind kind,
+ Label* jump_deoptimization_entry_label);
+
+ // Calls a C function.
+ // The called function is not allowed to trigger a
+ // garbage collection, since that might move the code and invalidate the
+ // return address (unless this is somehow accounted for by the called
+ // function).
+ void CallCFunction(ExternalReference function, int num_reg_arguments);
+ void CallCFunction(ExternalReference function, int num_reg_arguments,
+ int num_double_arguments);
+ void CallCFunction(Register function, int num_reg_arguments,
+ int num_double_arguments);
+
+ // Performs a truncating conversion of a floating point number as used by
+ // the JS bitwise operations. See ECMA-262 9.5: ToInt32.
+ // Exits with 'result' holding the answer.
+ void TruncateDoubleToI(Isolate* isolate, Zone* zone, Register result,
+ DoubleRegister double_input, StubCallMode stub_mode,
+ LinkRegisterStatus lr_status);
+
+ inline void Mul(const Register& rd, const Register& rn, const Register& rm);
+
+ inline void Fcvtzs(const Register& rd, const VRegister& fn);
+ void Fcvtzs(const VRegister& vd, const VRegister& vn, int fbits = 0) {
+ DCHECK(allow_macro_instructions());
+ fcvtzs(vd, vn, fbits);
+ }
+
+ void Fjcvtzs(const Register& rd, const VRegister& vn) {
+ DCHECK(allow_macro_instructions());
+ DCHECK(!rd.IsZero());
+ fjcvtzs(rd, vn);
+ }
+
+ inline void Fcvtzu(const Register& rd, const VRegister& fn);
+ void Fcvtzu(const VRegister& vd, const VRegister& vn, int fbits = 0) {
+ DCHECK(allow_macro_instructions());
+ fcvtzu(vd, vn, fbits);
+ }
+
+ inline void Madd(const Register& rd, const Register& rn, const Register& rm,
+ const Register& ra);
+ inline void Mneg(const Register& rd, const Register& rn, const Register& rm);
+ inline void Sdiv(const Register& rd, const Register& rn, const Register& rm);
+ inline void Udiv(const Register& rd, const Register& rn, const Register& rm);
+ inline void Msub(const Register& rd, const Register& rn, const Register& rm,
+ const Register& ra);
+
+ inline void Lsl(const Register& rd, const Register& rn, unsigned shift);
+ inline void Lsl(const Register& rd, const Register& rn, const Register& rm);
+ inline void Umull(const Register& rd, const Register& rn, const Register& rm);
+ inline void Smull(const Register& rd, const Register& rn, const Register& rm);
+
+ inline void Sxtb(const Register& rd, const Register& rn);
+ inline void Sxth(const Register& rd, const Register& rn);
+ inline void Sxtw(const Register& rd, const Register& rn);
+ inline void Ubfiz(const Register& rd, const Register& rn, unsigned lsb,
+ unsigned width);
+ inline void Ubfx(const Register& rd, const Register& rn, unsigned lsb,
+ unsigned width);
+ inline void Lsr(const Register& rd, const Register& rn, unsigned shift);
+ inline void Lsr(const Register& rd, const Register& rn, const Register& rm);
+ inline void Ror(const Register& rd, const Register& rs, unsigned shift);
+ inline void Ror(const Register& rd, const Register& rn, const Register& rm);
+ inline void Cmn(const Register& rn, const Operand& operand);
+ inline void Fadd(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm);
+ inline void Fcmp(const VRegister& fn, const VRegister& fm);
+ inline void Fcmp(const VRegister& fn, double value);
+ inline void Fabs(const VRegister& fd, const VRegister& fn);
+ inline void Fmul(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm);
+ inline void Fsub(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm);
+ inline void Fdiv(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm);
+ inline void Fmax(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm);
+ inline void Fmin(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm);
+ inline void Rbit(const Register& rd, const Register& rn);
+ inline void Rev(const Register& rd, const Register& rn);
+
+ enum AdrHint {
+ // The target must be within the immediate range of adr.
+ kAdrNear,
+ // The target may be outside of the immediate range of adr. Additional
+ // instructions may be emitted.
+ kAdrFar
+ };
+ void Adr(const Register& rd, Label* label, AdrHint = kAdrNear);
+
+ // Add/sub with carry macros.
+ inline void Adc(const Register& rd, const Register& rn,
+ const Operand& operand);
+
+ // Conditional macros.
+ inline void Ccmp(const Register& rn, const Operand& operand, StatusFlags nzcv,
+ Condition cond);
+ inline void CcmpTagged(const Register& rn, const Operand& operand,
+ StatusFlags nzcv, Condition cond);
+
+ inline void Clz(const Register& rd, const Register& rn);
+
+ // Poke 'src' onto the stack. The offset is in bytes. The stack pointer must
+ // be 16 byte aligned.
+ // When the optional template argument is kSignLR and control flow integrity
+ // measures are enabled, we sign the link register before poking it onto the
+ // stack. 'src' must be lr in this case.
+ template <StoreLRMode lr_mode = kDontStoreLR>
+ void Poke(const CPURegister& src, const Operand& offset);
+
+ // Peek at a value on the stack, and put it in 'dst'. The offset is in bytes.
+ // The stack pointer must be aligned to 16 bytes.
+ // When the optional template argument is kAuthLR and control flow integrity
+ // measures are enabled, we authenticate the link register after peeking the
+ // value. 'dst' must be lr in this case.
+ template <LoadLRMode lr_mode = kDontLoadLR>
+ void Peek(const CPURegister& dst, const Operand& offset);
+
+ // Poke 'src1' and 'src2' onto the stack. The values written will be adjacent
+ // with 'src2' at a higher address than 'src1'. The offset is in bytes. The
+ // stack pointer must be 16 byte aligned.
+ void PokePair(const CPURegister& src1, const CPURegister& src2, int offset);
+
+ inline void Sbfx(const Register& rd, const Register& rn, unsigned lsb,
+ unsigned width);
+
+ inline void Bfi(const Register& rd, const Register& rn, unsigned lsb,
+ unsigned width);
+
+ inline void Scvtf(const VRegister& fd, const Register& rn,
+ unsigned fbits = 0);
+ void Scvtf(const VRegister& vd, const VRegister& vn, int fbits = 0) {
+ DCHECK(allow_macro_instructions());
+ scvtf(vd, vn, fbits);
+ }
+ inline void Ucvtf(const VRegister& fd, const Register& rn,
+ unsigned fbits = 0);
+ void Ucvtf(const VRegister& vd, const VRegister& vn, int fbits = 0) {
+ DCHECK(allow_macro_instructions());
+ ucvtf(vd, vn, fbits);
+ }
+
+ void AssertFPCRState(Register fpcr = NoReg);
+ void CanonicalizeNaN(const VRegister& dst, const VRegister& src);
+ void CanonicalizeNaN(const VRegister& reg) { CanonicalizeNaN(reg, reg); }
+
+ inline void CmovX(const Register& rd, const Register& rn, Condition cond);
+ inline void Cset(const Register& rd, Condition cond);
+ inline void Csetm(const Register& rd, Condition cond);
+ inline void Fccmp(const VRegister& fn, const VRegister& fm, StatusFlags nzcv,
+ Condition cond);
+ inline void Csinc(const Register& rd, const Register& rn, const Register& rm,
+ Condition cond);
+
+ inline void Fcvt(const VRegister& fd, const VRegister& fn);
+
+ int ActivationFrameAlignment();
+
+ void Ins(const VRegister& vd, int vd_index, const VRegister& vn,
+ int vn_index) {
+ DCHECK(allow_macro_instructions());
+ ins(vd, vd_index, vn, vn_index);
+ }
+ void Ins(const VRegister& vd, int vd_index, const Register& rn) {
+ DCHECK(allow_macro_instructions());
+ ins(vd, vd_index, rn);
+ }
+
+ inline void Bl(Label* label);
+ inline void Br(const Register& xn);
+
+ inline void Uxtb(const Register& rd, const Register& rn);
+ inline void Uxth(const Register& rd, const Register& rn);
+ inline void Uxtw(const Register& rd, const Register& rn);
+
+ void Dup(const VRegister& vd, const VRegister& vn, int index) {
+ DCHECK(allow_macro_instructions());
+ dup(vd, vn, index);
+ }
+ void Dup(const VRegister& vd, const Register& rn) {
+ DCHECK(allow_macro_instructions());
+ dup(vd, rn);
+ }
+
+#define DECLARE_FUNCTION(FN, REGTYPE, REG, REG2, OP) \
+ inline void FN(const REGTYPE REG, const REGTYPE REG2, const MemOperand& addr);
+ LSPAIR_MACRO_LIST(DECLARE_FUNCTION)
+#undef DECLARE_FUNCTION
+
+#define NEON_2VREG_SHIFT_MACRO_LIST(V) \
+ V(rshrn, Rshrn) \
+ V(rshrn2, Rshrn2) \
+ V(shl, Shl) \
+ V(shll, Shll) \
+ V(shll2, Shll2) \
+ V(shrn, Shrn) \
+ V(shrn2, Shrn2) \
+ V(sli, Sli) \
+ V(sqrshrn, Sqrshrn) \
+ V(sqrshrn2, Sqrshrn2) \
+ V(sqrshrun, Sqrshrun) \
+ V(sqrshrun2, Sqrshrun2) \
+ V(sqshl, Sqshl) \
+ V(sqshlu, Sqshlu) \
+ V(sqshrn, Sqshrn) \
+ V(sqshrn2, Sqshrn2) \
+ V(sqshrun, Sqshrun) \
+ V(sqshrun2, Sqshrun2) \
+ V(sri, Sri) \
+ V(srshr, Srshr) \
+ V(srsra, Srsra) \
+ V(sshll, Sshll) \
+ V(sshll2, Sshll2) \
+ V(sshr, Sshr) \
+ V(ssra, Ssra) \
+ V(uqrshrn, Uqrshrn) \
+ V(uqrshrn2, Uqrshrn2) \
+ V(uqshl, Uqshl) \
+ V(uqshrn, Uqshrn) \
+ V(uqshrn2, Uqshrn2) \
+ V(urshr, Urshr) \
+ V(ursra, Ursra) \
+ V(ushll, Ushll) \
+ V(ushll2, Ushll2) \
+ V(ushr, Ushr) \
+ V(usra, Usra)
+
+#define DEFINE_MACRO_ASM_FUNC(ASM, MASM) \
+ void MASM(const VRegister& vd, const VRegister& vn, int shift) { \
+ DCHECK(allow_macro_instructions()); \
+ ASM(vd, vn, shift); \
+ }
+ NEON_2VREG_SHIFT_MACRO_LIST(DEFINE_MACRO_ASM_FUNC)
+#undef DEFINE_MACRO_ASM_FUNC
+
+ void Umov(const Register& rd, const VRegister& vn, int vn_index) {
+ DCHECK(allow_macro_instructions());
+ umov(rd, vn, vn_index);
+ }
+ void Tbl(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
+ DCHECK(allow_macro_instructions());
+ tbl(vd, vn, vm);
+ }
+ void Tbl(const VRegister& vd, const VRegister& vn, const VRegister& vn2,
+ const VRegister& vm) {
+ DCHECK(allow_macro_instructions());
+ tbl(vd, vn, vn2, vm);
+ }
+ void Tbl(const VRegister& vd, const VRegister& vn, const VRegister& vn2,
+ const VRegister& vn3, const VRegister& vm) {
+ DCHECK(allow_macro_instructions());
+ tbl(vd, vn, vn2, vn3, vm);
+ }
+ void Tbl(const VRegister& vd, const VRegister& vn, const VRegister& vn2,
+ const VRegister& vn3, const VRegister& vn4, const VRegister& vm) {
+ DCHECK(allow_macro_instructions());
+ tbl(vd, vn, vn2, vn3, vn4, vm);
+ }
+ void Ext(const VRegister& vd, const VRegister& vn, const VRegister& vm,
+ int index) {
+ DCHECK(allow_macro_instructions());
+ ext(vd, vn, vm, index);
+ }
+
+ void Smov(const Register& rd, const VRegister& vn, int vn_index) {
+ DCHECK(allow_macro_instructions());
+ smov(rd, vn, vn_index);
+ }
+
+// Load-acquire/store-release macros.
+#define DECLARE_FUNCTION(FN, OP) \
+ inline void FN(const Register& rt, const Register& rn);
+ LDA_STL_MACRO_LIST(DECLARE_FUNCTION)
+#undef DECLARE_FUNCTION
+
+ // Load an object from the root table.
+ void LoadRoot(Register destination, RootIndex index) override;
+
+ inline void Ret(const Register& xn = lr);
+
+ // Perform a conversion from a double to a signed int64. If the input fits in
+ // range of the 64-bit result, execution branches to done. Otherwise,
+ // execution falls through, and the sign of the result can be used to
+ // determine if overflow was towards positive or negative infinity.
+ //
+ // On successful conversion, the least significant 32 bits of the result are
+ // equivalent to the ECMA-262 operation "ToInt32".
+ void TryConvertDoubleToInt64(Register result, DoubleRegister input,
+ Label* done);
+
+ inline void Mrs(const Register& rt, SystemRegister sysreg);
+ inline void Msr(SystemRegister sysreg, const Register& rt);
+
+ // Prologue claims an extra slot due to arm64's alignement constraints.
+ static constexpr int kExtraSlotClaimedByPrologue = 1;
+ // Generates function prologue code.
+ void Prologue();
+
+ void Cmgt(const VRegister& vd, const VRegister& vn, int imm) {
+ DCHECK(allow_macro_instructions());
+ cmgt(vd, vn, imm);
+ }
+ void Cmge(const VRegister& vd, const VRegister& vn, int imm) {
+ DCHECK(allow_macro_instructions());
+ cmge(vd, vn, imm);
+ }
+ void Cmeq(const VRegister& vd, const VRegister& vn, int imm) {
+ DCHECK(allow_macro_instructions());
+ cmeq(vd, vn, imm);
+ }
+
+ inline void Neg(const Register& rd, const Operand& operand);
+ inline void Negs(const Register& rd, const Operand& operand);
+
+ // Compute rd = abs(rm).
+ // This function clobbers the condition flags. On output the overflow flag is
+ // set iff the negation overflowed.
+ //
+ // If rm is the minimum representable value, the result is not representable.
+ // Handlers for each case can be specified using the relevant labels.
+ void Abs(const Register& rd, const Register& rm,
+ Label* is_not_representable = nullptr,
+ Label* is_representable = nullptr);
+
+ inline void Cls(const Register& rd, const Register& rn);
+ inline void Cneg(const Register& rd, const Register& rn, Condition cond);
+ inline void Rev16(const Register& rd, const Register& rn);
+ inline void Rev32(const Register& rd, const Register& rn);
+ inline void Fcvtns(const Register& rd, const VRegister& fn);
+ inline void Fcvtnu(const Register& rd, const VRegister& fn);
+ inline void Fcvtms(const Register& rd, const VRegister& fn);
+ inline void Fcvtmu(const Register& rd, const VRegister& fn);
+ inline void Fcvtas(const Register& rd, const VRegister& fn);
+ inline void Fcvtau(const Register& rd, const VRegister& fn);
+
+ // Compute the start of the generated instruction stream from the current PC.
+ // This is an alternative to embedding the {CodeObject} handle as a reference.
+ void ComputeCodeStartAddress(const Register& rd);
+
+ void ResetSpeculationPoisonRegister();
+
+ // ---------------------------------------------------------------------------
+ // Pointer compression Support
+
+ // Loads a field containing a HeapObject and decompresses it if pointer
+ // compression is enabled.
+ void LoadTaggedPointerField(const Register& destination,
+ const MemOperand& field_operand);
+
+ // Loads a field containing any tagged value and decompresses it if necessary.
+ void LoadAnyTaggedField(const Register& destination,
+ const MemOperand& field_operand);
+
+ // Loads a field containing smi value and untags it.
+ void SmiUntagField(Register dst, const MemOperand& src);
+
+ // Compresses and stores tagged value to given on-heap location.
+ void StoreTaggedField(const Register& value,
+ const MemOperand& dst_field_operand);
+
+ void DecompressTaggedSigned(const Register& destination,
+ const MemOperand& field_operand);
+ void DecompressTaggedPointer(const Register& destination,
+ const MemOperand& field_operand);
+ void DecompressTaggedPointer(const Register& destination,
+ const Register& source);
+ void DecompressAnyTagged(const Register& destination,
+ const MemOperand& field_operand);
+
+ // Restore FP and LR from the values stored in the current frame. This will
+ // authenticate the LR when pointer authentication is enabled.
+ void RestoreFPAndLR();
+
+ void StoreReturnAddressInWasmExitFrame(Label* return_location);
+
+ protected:
+ // The actual Push and Pop implementations. These don't generate any code
+ // other than that required for the push or pop. This allows
+ // (Push|Pop)CPURegList to bundle together run-time assertions for a large
+ // block of registers.
+ //
+ // Note that size is per register, and is specified in bytes.
+ void PushHelper(int count, int size, const CPURegister& src0,
+ const CPURegister& src1, const CPURegister& src2,
+ const CPURegister& src3);
+ void PopHelper(int count, int size, const CPURegister& dst0,
+ const CPURegister& dst1, const CPURegister& dst2,
+ const CPURegister& dst3);
+
+ void ConditionalCompareMacro(const Register& rn, const Operand& operand,
+ StatusFlags nzcv, Condition cond,
+ ConditionalCompareOp op);
+
+ void AddSubWithCarryMacro(const Register& rd, const Register& rn,
+ const Operand& operand, FlagsUpdate S,
+ AddSubWithCarryOp op);
+
+ // Call Printf. On a native build, a simple call will be generated, but if the
+ // simulator is being used then a suitable pseudo-instruction is used. The
+ // arguments and stack must be prepared by the caller as for a normal AAPCS64
+ // call to 'printf'.
+ //
+ // The 'args' argument should point to an array of variable arguments in their
+ // proper PCS registers (and in calling order). The argument registers can
+ // have mixed types. The format string (x0) should not be included.
+ void CallPrintf(int arg_count = 0, const CPURegister* args = nullptr);
+
+ private:
+#if DEBUG
+ // Tell whether any of the macro instruction can be used. When false the
+ // MacroAssembler will assert if a method which can emit a variable number
+ // of instructions is called.
+ bool allow_macro_instructions_ = true;
+#endif
+
+ // Scratch registers available for use by the MacroAssembler.
+ CPURegList tmp_list_ = DefaultTmpList();
+ CPURegList fptmp_list_ = DefaultFPTmpList();
+
+ // Helps resolve branching to labels potentially out of range.
+ // If the label is not bound, it registers the information necessary to later
+ // be able to emit a veneer for this branch if necessary.
+ // If the label is bound, it returns true if the label (or the previous link
+ // in the label chain) is out of range. In that case the caller is responsible
+ // for generating appropriate code.
+ // Otherwise it returns false.
+ // This function also checks wether veneers need to be emitted.
+ bool NeedExtraInstructionsOrRegisterBranch(Label* label,
+ ImmBranchType branch_type);
+
+ void Movi16bitHelper(const VRegister& vd, uint64_t imm);
+ void Movi32bitHelper(const VRegister& vd, uint64_t imm);
+ void Movi64bitHelper(const VRegister& vd, uint64_t imm);
+
+ void LoadStoreMacro(const CPURegister& rt, const MemOperand& addr,
+ LoadStoreOp op);
+
+ void LoadStorePairMacro(const CPURegister& rt, const CPURegister& rt2,
+ const MemOperand& addr, LoadStorePairOp op);
+
+ void JumpHelper(int64_t offset, RelocInfo::Mode rmode, Condition cond = al);
+
+ void CallRecordWriteStub(Register object, Operand offset,
+ RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode, Handle<Code> code_target,
+ Address wasm_target);
+};
+
+class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler {
+ public:
+ using TurboAssembler::TurboAssembler;
+
+ // Instruction set functions ------------------------------------------------
+ // Logical macros.
+ inline void Bics(const Register& rd, const Register& rn,
+ const Operand& operand);
+
+ inline void Adcs(const Register& rd, const Register& rn,
+ const Operand& operand);
+ inline void Sbc(const Register& rd, const Register& rn,
+ const Operand& operand);
+ inline void Sbcs(const Register& rd, const Register& rn,
+ const Operand& operand);
+ inline void Ngc(const Register& rd, const Operand& operand);
+ inline void Ngcs(const Register& rd, const Operand& operand);
+
+ inline void Ccmn(const Register& rn, const Operand& operand, StatusFlags nzcv,
+ Condition cond);
+
+#define DECLARE_FUNCTION(FN, OP) \
+ inline void FN(const Register& rs, const Register& rt, const Register& rn);
+ STLX_MACRO_LIST(DECLARE_FUNCTION)
+#undef DECLARE_FUNCTION
+
+ // Branch type inversion relies on these relations.
+ STATIC_ASSERT((reg_zero == (reg_not_zero ^ 1)) &&
+ (reg_bit_clear == (reg_bit_set ^ 1)) &&
+ (always == (never ^ 1)));
+
+ inline void Bfxil(const Register& rd, const Register& rn, unsigned lsb,
+ unsigned width);
+ inline void Cinc(const Register& rd, const Register& rn, Condition cond);
+ inline void Cinv(const Register& rd, const Register& rn, Condition cond);
+ inline void CzeroX(const Register& rd, Condition cond);
+ inline void Csinv(const Register& rd, const Register& rn, const Register& rm,
+ Condition cond);
+ inline void Csneg(const Register& rd, const Register& rn, const Register& rm,
+ Condition cond);
+ inline void Extr(const Register& rd, const Register& rn, const Register& rm,
+ unsigned lsb);
+ inline void Fcsel(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm, Condition cond);
+ void Fcvtl(const VRegister& vd, const VRegister& vn) {
+ DCHECK(allow_macro_instructions());
+ fcvtl(vd, vn);
+ }
+ void Fcvtl2(const VRegister& vd, const VRegister& vn) {
+ DCHECK(allow_macro_instructions());
+ fcvtl2(vd, vn);
+ }
+ void Fcvtn(const VRegister& vd, const VRegister& vn) {
+ DCHECK(allow_macro_instructions());
+ fcvtn(vd, vn);
+ }
+ void Fcvtn2(const VRegister& vd, const VRegister& vn) {
+ DCHECK(allow_macro_instructions());
+ fcvtn2(vd, vn);
+ }
+ void Fcvtxn(const VRegister& vd, const VRegister& vn) {
+ DCHECK(allow_macro_instructions());
+ fcvtxn(vd, vn);
+ }
+ void Fcvtxn2(const VRegister& vd, const VRegister& vn) {
+ DCHECK(allow_macro_instructions());
+ fcvtxn2(vd, vn);
+ }
+ inline void Fmadd(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm, const VRegister& fa);
+ inline void Fmaxnm(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm);
+ inline void Fminnm(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm);
+ inline void Fmsub(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm, const VRegister& fa);
+ inline void Fnmadd(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm, const VRegister& fa);
+ inline void Fnmsub(const VRegister& fd, const VRegister& fn,
+ const VRegister& fm, const VRegister& fa);
+ inline void Hint(SystemHint code);
+ inline void Hlt(int code);
+ inline void Ldnp(const CPURegister& rt, const CPURegister& rt2,
+ const MemOperand& src);
+ inline void Movk(const Register& rd, uint64_t imm, int shift = -1);
+ inline void Nop() { nop(); }
+ void Mvni(const VRegister& vd, const int imm8, Shift shift = LSL,
+ const int shift_amount = 0) {
+ DCHECK(allow_macro_instructions());
+ mvni(vd, imm8, shift, shift_amount);
+ }
+ inline void Rev(const Register& rd, const Register& rn);
+ inline void Sbfiz(const Register& rd, const Register& rn, unsigned lsb,
+ unsigned width);
+ inline void Smaddl(const Register& rd, const Register& rn, const Register& rm,
+ const Register& ra);
+ inline void Smsubl(const Register& rd, const Register& rn, const Register& rm,
+ const Register& ra);
+ inline void Smulh(const Register& rd, const Register& rn, const Register& rm);
+ inline void Stnp(const CPURegister& rt, const CPURegister& rt2,
+ const MemOperand& dst);
+ inline void Umaddl(const Register& rd, const Register& rn, const Register& rm,
+ const Register& ra);
+ inline void Umsubl(const Register& rd, const Register& rn, const Register& rm,
+ const Register& ra);
+
+ void Cmle(const VRegister& vd, const VRegister& vn, int imm) {
+ DCHECK(allow_macro_instructions());
+ cmle(vd, vn, imm);
+ }
+ void Cmlt(const VRegister& vd, const VRegister& vn, int imm) {
+ DCHECK(allow_macro_instructions());
+ cmlt(vd, vn, imm);
+ }
+
+ void Ld1(const VRegister& vt, const MemOperand& src) {
+ DCHECK(allow_macro_instructions());
+ ld1(vt, src);
+ }
+ void Ld1(const VRegister& vt, const VRegister& vt2, const MemOperand& src) {
+ DCHECK(allow_macro_instructions());
+ ld1(vt, vt2, src);
+ }
+ void Ld1(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ const MemOperand& src) {
+ DCHECK(allow_macro_instructions());
+ ld1(vt, vt2, vt3, src);
+ }
+ void Ld1(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ const VRegister& vt4, const MemOperand& src) {
+ DCHECK(allow_macro_instructions());
+ ld1(vt, vt2, vt3, vt4, src);
+ }
+ void Ld1(const VRegister& vt, int lane, const MemOperand& src) {
+ DCHECK(allow_macro_instructions());
+ ld1(vt, lane, src);
+ }
+ void Ld1r(const VRegister& vt, const MemOperand& src) {
+ DCHECK(allow_macro_instructions());
+ ld1r(vt, src);
+ }
+ void Ld2(const VRegister& vt, const VRegister& vt2, const MemOperand& src) {
+ DCHECK(allow_macro_instructions());
+ ld2(vt, vt2, src);
+ }
+ void Ld2(const VRegister& vt, const VRegister& vt2, int lane,
+ const MemOperand& src) {
+ DCHECK(allow_macro_instructions());
+ ld2(vt, vt2, lane, src);
+ }
+ void Ld2r(const VRegister& vt, const VRegister& vt2, const MemOperand& src) {
+ DCHECK(allow_macro_instructions());
+ ld2r(vt, vt2, src);
+ }
+ void Ld3(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ const MemOperand& src) {
+ DCHECK(allow_macro_instructions());
+ ld3(vt, vt2, vt3, src);
+ }
+ void Ld3(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ int lane, const MemOperand& src) {
+ DCHECK(allow_macro_instructions());
+ ld3(vt, vt2, vt3, lane, src);
+ }
+ void Ld3r(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ const MemOperand& src) {
+ DCHECK(allow_macro_instructions());
+ ld3r(vt, vt2, vt3, src);
+ }
+ void Ld4(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ const VRegister& vt4, const MemOperand& src) {
+ DCHECK(allow_macro_instructions());
+ ld4(vt, vt2, vt3, vt4, src);
+ }
+ void Ld4(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ const VRegister& vt4, int lane, const MemOperand& src) {
+ DCHECK(allow_macro_instructions());
+ ld4(vt, vt2, vt3, vt4, lane, src);
+ }
+ void Ld4r(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ const VRegister& vt4, const MemOperand& src) {
+ DCHECK(allow_macro_instructions());
+ ld4r(vt, vt2, vt3, vt4, src);
+ }
+ void St1(const VRegister& vt, const MemOperand& dst) {
+ DCHECK(allow_macro_instructions());
+ st1(vt, dst);
+ }
+ void St1(const VRegister& vt, const VRegister& vt2, const MemOperand& dst) {
+ DCHECK(allow_macro_instructions());
+ st1(vt, vt2, dst);
+ }
+ void St1(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ const MemOperand& dst) {
+ DCHECK(allow_macro_instructions());
+ st1(vt, vt2, vt3, dst);
+ }
+ void St1(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ const VRegister& vt4, const MemOperand& dst) {
+ DCHECK(allow_macro_instructions());
+ st1(vt, vt2, vt3, vt4, dst);
+ }
+ void St1(const VRegister& vt, int lane, const MemOperand& dst) {
+ DCHECK(allow_macro_instructions());
+ st1(vt, lane, dst);
+ }
+ void St2(const VRegister& vt, const VRegister& vt2, const MemOperand& dst) {
+ DCHECK(allow_macro_instructions());
+ st2(vt, vt2, dst);
+ }
+ void St3(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ const MemOperand& dst) {
+ DCHECK(allow_macro_instructions());
+ st3(vt, vt2, vt3, dst);
+ }
+ void St4(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ const VRegister& vt4, const MemOperand& dst) {
+ DCHECK(allow_macro_instructions());
+ st4(vt, vt2, vt3, vt4, dst);
+ }
+ void St2(const VRegister& vt, const VRegister& vt2, int lane,
+ const MemOperand& dst) {
+ DCHECK(allow_macro_instructions());
+ st2(vt, vt2, lane, dst);
+ }
+ void St3(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ int lane, const MemOperand& dst) {
+ DCHECK(allow_macro_instructions());
+ st3(vt, vt2, vt3, lane, dst);
+ }
+ void St4(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
+ const VRegister& vt4, int lane, const MemOperand& dst) {
+ DCHECK(allow_macro_instructions());
+ st4(vt, vt2, vt3, vt4, lane, dst);
+ }
+ void Tbx(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
+ DCHECK(allow_macro_instructions());
+ tbx(vd, vn, vm);
+ }
+ void Tbx(const VRegister& vd, const VRegister& vn, const VRegister& vn2,
+ const VRegister& vm) {
+ DCHECK(allow_macro_instructions());
+ tbx(vd, vn, vn2, vm);
+ }
+ void Tbx(const VRegister& vd, const VRegister& vn, const VRegister& vn2,
+ const VRegister& vn3, const VRegister& vm) {
+ DCHECK(allow_macro_instructions());
+ tbx(vd, vn, vn2, vn3, vm);
+ }
+ void Tbx(const VRegister& vd, const VRegister& vn, const VRegister& vn2,
+ const VRegister& vn3, const VRegister& vn4, const VRegister& vm) {
+ DCHECK(allow_macro_instructions());
+ tbx(vd, vn, vn2, vn3, vn4, vm);
+ }
+
+ // For the 'lr_mode' template argument of the following methods, see
+ // PushCPURegList/PopCPURegList.
+ template <StoreLRMode lr_mode = kDontStoreLR>
+ inline void PushSizeRegList(
+ RegList registers, unsigned reg_size,
+ CPURegister::RegisterType type = CPURegister::kRegister) {
+ PushCPURegList<lr_mode>(CPURegList(type, reg_size, registers));
+ }
+ template <LoadLRMode lr_mode = kDontLoadLR>
+ inline void PopSizeRegList(
+ RegList registers, unsigned reg_size,
+ CPURegister::RegisterType type = CPURegister::kRegister) {
+ PopCPURegList<lr_mode>(CPURegList(type, reg_size, registers));
+ }
+ template <StoreLRMode lr_mode = kDontStoreLR>
+ inline void PushXRegList(RegList regs) {
+ PushSizeRegList<lr_mode>(regs, kXRegSizeInBits);
+ }
+ template <LoadLRMode lr_mode = kDontLoadLR>
+ inline void PopXRegList(RegList regs) {
+ PopSizeRegList<lr_mode>(regs, kXRegSizeInBits);
+ }
+ inline void PushWRegList(RegList regs) {
+ PushSizeRegList(regs, kWRegSizeInBits);
+ }
+ inline void PopWRegList(RegList regs) {
+ PopSizeRegList(regs, kWRegSizeInBits);
+ }
+ inline void PushQRegList(RegList regs) {
+ PushSizeRegList(regs, kQRegSizeInBits, CPURegister::kVRegister);
+ }
+ inline void PopQRegList(RegList regs) {
+ PopSizeRegList(regs, kQRegSizeInBits, CPURegister::kVRegister);
+ }
+ inline void PushDRegList(RegList regs) {
+ PushSizeRegList(regs, kDRegSizeInBits, CPURegister::kVRegister);
+ }
+ inline void PopDRegList(RegList regs) {
+ PopSizeRegList(regs, kDRegSizeInBits, CPURegister::kVRegister);
+ }
+ inline void PushSRegList(RegList regs) {
+ PushSizeRegList(regs, kSRegSizeInBits, CPURegister::kVRegister);
+ }
+ inline void PopSRegList(RegList regs) {
+ PopSizeRegList(regs, kSRegSizeInBits, CPURegister::kVRegister);
+ }
+
+ // Push the specified register 'count' times.
+ void PushMultipleTimes(CPURegister src, Register count);
+
+ // Peek at two values on the stack, and put them in 'dst1' and 'dst2'. The
+ // values peeked will be adjacent, with the value in 'dst2' being from a
+ // higher address than 'dst1'. The offset is in bytes. The stack pointer must
+ // be aligned to 16 bytes.
+ void PeekPair(const CPURegister& dst1, const CPURegister& dst2, int offset);
+
+ // Preserve the callee-saved registers (as defined by AAPCS64).
+ //
+ // Higher-numbered registers are pushed before lower-numbered registers, and
+ // thus get higher addresses.
+ // Floating-point registers are pushed before general-purpose registers, and
+ // thus get higher addresses.
+ //
+ // When control flow integrity measures are enabled, this method signs the
+ // link register before pushing it.
+ //
+ // Note that registers are not checked for invalid values. Use this method
+ // only if you know that the GC won't try to examine the values on the stack.
+ void PushCalleeSavedRegisters();
+
+ // Restore the callee-saved registers (as defined by AAPCS64).
+ //
+ // Higher-numbered registers are popped after lower-numbered registers, and
+ // thus come from higher addresses.
+ // Floating-point registers are popped after general-purpose registers, and
+ // thus come from higher addresses.
+ //
+ // When control flow integrity measures are enabled, this method
+ // authenticates the link register after popping it.
+ void PopCalleeSavedRegisters();
+
+ // Helpers ------------------------------------------------------------------
+
+ template <typename Field>
+ void DecodeField(Register dst, Register src) {
+ static const int shift = Field::kShift;
+ static const int setbits = CountSetBits(Field::kMask, 32);
+ Ubfx(dst, src, shift, setbits);
+ }
+
+ template <typename Field>
+ void DecodeField(Register reg) {
+ DecodeField<Field>(reg, reg);
+ }
+
+ Operand ReceiverOperand(const Register arg_count);
+
+ // ---- SMI and Number Utilities ----
+
+ inline void SmiTag(Register dst, Register src);
+ inline void SmiTag(Register smi);
+
+ inline void JumpIfNotSmi(Register value, Label* not_smi_label);
+
+ // Abort execution if argument is a smi, enabled via --debug-code.
+ void AssertNotSmi(Register object,
+ AbortReason reason = AbortReason::kOperandIsASmi);
+
+ // Abort execution if argument is not a Constructor, enabled via --debug-code.
+ void AssertConstructor(Register object);
+
+ // Abort execution if argument is not a JSFunction, enabled via --debug-code.
+ void AssertFunction(Register object);
+
+ // Abort execution if argument is not a JSGeneratorObject (or subclass),
+ // enabled via --debug-code.
+ void AssertGeneratorObject(Register object);
+
+ // Abort execution if argument is not a JSBoundFunction,
+ // enabled via --debug-code.
+ void AssertBoundFunction(Register object);
+
+ // Abort execution if argument is not undefined or an AllocationSite, enabled
+ // via --debug-code.
+ void AssertUndefinedOrAllocationSite(Register object);
+
+ // ---- Calling / Jumping helpers ----
+
+ void CallRuntime(const Runtime::Function* f, int num_arguments,
+ SaveFPRegsMode save_doubles = kDontSaveFPRegs);
+
+ // Convenience function: Same as above, but takes the fid instead.
+ void CallRuntime(Runtime::FunctionId fid, int num_arguments,
+ SaveFPRegsMode save_doubles = kDontSaveFPRegs) {
+ CallRuntime(Runtime::FunctionForId(fid), num_arguments, save_doubles);
+ }
+
+ // Convenience function: Same as above, but takes the fid instead.
+ void CallRuntime(Runtime::FunctionId fid,
+ SaveFPRegsMode save_doubles = kDontSaveFPRegs) {
+ const Runtime::Function* function = Runtime::FunctionForId(fid);
+ CallRuntime(function, function->nargs, save_doubles);
+ }
+
+ void TailCallRuntime(Runtime::FunctionId fid);
+
+ // Jump to a runtime routine.
+ void JumpToExternalReference(const ExternalReference& builtin,
+ bool builtin_exit_frame = false);
+
+ // Generates a trampoline to jump to the off-heap instruction stream.
+ void JumpToInstructionStream(Address entry);
+
+ // Registers used through the invocation chain are hard-coded.
+ // We force passing the parameters to ensure the contracts are correctly
+ // honoured by the caller.
+ // 'function' must be x1.
+ // 'actual' must use an immediate or x0.
+ // 'expected' must use an immediate or x2.
+ // 'call_kind' must be x5.
+ void InvokePrologue(Register expected_parameter_count,
+ Register actual_parameter_count, Label* done,
+ InvokeFlag flag);
+
+ // On function call, call into the debugger.
+ void CallDebugOnFunctionCall(Register fun, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count);
+ void InvokeFunctionCode(Register function, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count, InvokeFlag flag);
+ // Invoke the JavaScript function in the given register.
+ // Changes the current context to the context in the function before invoking.
+ void InvokeFunctionWithNewTarget(Register function, Register new_target,
+ Register actual_parameter_count,
+ InvokeFlag flag);
+ void InvokeFunction(Register function, Register expected_parameter_count,
+ Register actual_parameter_count, InvokeFlag flag);
+
+ // ---- Code generation helpers ----
+
+ // Frame restart support
+ void MaybeDropFrames();
+
+ // ---------------------------------------------------------------------------
+ // Support functions.
+
+ // Compare object type for heap object. heap_object contains a non-Smi
+ // whose object type should be compared with the given type. This both
+ // sets the flags and leaves the object type in the type_reg register.
+ // It leaves the map in the map register (unless the type_reg and map register
+ // are the same register). It leaves the heap object in the heap_object
+ // register unless the heap_object register is the same register as one of the
+ // other registers.
+ void CompareObjectType(Register heap_object, Register map, Register type_reg,
+ InstanceType type);
+
+ // Compare object type for heap object, and branch if equal (or not.)
+ // heap_object contains a non-Smi whose object type should be compared with
+ // the given type. This both sets the flags and leaves the object type in
+ // the type_reg register. It leaves the map in the map register (unless the
+ // type_reg and map register are the same register). It leaves the heap
+ // object in the heap_object register unless the heap_object register is the
+ // same register as one of the other registers.
+ void JumpIfObjectType(Register object, Register map, Register type_reg,
+ InstanceType type, Label* if_cond_pass,
+ Condition cond = eq);
+
+ // Compare instance type in a map. map contains a valid map object whose
+ // object type should be compared with the given type. This both
+ // sets the flags and leaves the object type in the type_reg register.
+ void CompareInstanceType(Register map, Register type_reg, InstanceType type);
+
+ // Load the elements kind field from a map, and return it in the result
+ // register.
+ void LoadElementsKindFromMap(Register result, Register map);
+
+ // Compare the object in a register to a value from the root list.
+ void CompareRoot(const Register& obj, RootIndex index);
+
+ // Compare the object in a register to a value and jump if they are equal.
+ void JumpIfRoot(const Register& obj, RootIndex index, Label* if_equal);
+
+ // Compare the object in a register to a value and jump if they are not equal.
+ void JumpIfNotRoot(const Register& obj, RootIndex index, Label* if_not_equal);
+
+ // Checks if value is in range [lower_limit, higher_limit] using a single
+ // comparison.
+ void JumpIfIsInRange(const Register& value, unsigned lower_limit,
+ unsigned higher_limit, Label* on_in_range);
+
+ // ---------------------------------------------------------------------------
+ // Frames.
+
+ void ExitFramePreserveFPRegs();
+ void ExitFrameRestoreFPRegs();
+
+ // Enter exit frame. Exit frames are used when calling C code from generated
+ // (JavaScript) code.
+ //
+ // The only registers modified by this function are the provided scratch
+ // register, the frame pointer and the stack pointer.
+ //
+ // The 'extra_space' argument can be used to allocate some space in the exit
+ // frame that will be ignored by the GC. This space will be reserved in the
+ // bottom of the frame immediately above the return address slot.
+ //
+ // Set up a stack frame and registers as follows:
+ // fp[8]: CallerPC (lr)
+ // fp -> fp[0]: CallerFP (old fp)
+ // fp[-8]: SPOffset (new sp)
+ // fp[-16]: CodeObject()
+ // fp[-16 - fp-size]: Saved doubles, if saved_doubles is true.
+ // sp[8]: Memory reserved for the caller if extra_space != 0.
+ // Alignment padding, if necessary.
+ // sp -> sp[0]: Space reserved for the return address.
+ //
+ // This function also stores the new frame information in the top frame, so
+ // that the new frame becomes the current frame.
+ void EnterExitFrame(bool save_doubles, const Register& scratch,
+ int extra_space = 0,
+ StackFrame::Type frame_type = StackFrame::EXIT);
+
+ // Leave the current exit frame, after a C function has returned to generated
+ // (JavaScript) code.
+ //
+ // This effectively unwinds the operation of EnterExitFrame:
+ // * Preserved doubles are restored (if restore_doubles is true).
+ // * The frame information is removed from the top frame.
+ // * The exit frame is dropped.
+ void LeaveExitFrame(bool save_doubles, const Register& scratch,
+ const Register& scratch2);
+
+ void LoadMap(Register dst, Register object);
+
+ // Load the global proxy from the current context.
+ void LoadGlobalProxy(Register dst);
+
+ // ---------------------------------------------------------------------------
+ // In-place weak references.
+ void LoadWeakValue(Register out, Register in, Label* target_if_cleared);
+
+ // ---------------------------------------------------------------------------
+ // StatsCounter support
+
+ void IncrementCounter(StatsCounter* counter, int value, Register scratch1,
+ Register scratch2);
+ void DecrementCounter(StatsCounter* counter, int value, Register scratch1,
+ Register scratch2);
+
+ // ---------------------------------------------------------------------------
+ // Stack limit utilities
+ void LoadStackLimit(Register destination, StackLimitKind kind);
+ void StackOverflowCheck(Register num_args, Label* stack_overflow);
+
+ // ---------------------------------------------------------------------------
+ // Garbage collector support (GC).
+
+ // Notify the garbage collector that we wrote a pointer into an object.
+ // |object| is the object being stored into, |value| is the object being
+ // stored.
+ // The offset is the offset from the start of the object, not the offset from
+ // the tagged HeapObject pointer. For use with FieldMemOperand(reg, off).
+ void RecordWriteField(
+ Register object, int offset, Register value, LinkRegisterStatus lr_status,
+ SaveFPRegsMode save_fp,
+ RememberedSetAction remembered_set_action = EMIT_REMEMBERED_SET,
+ SmiCheck smi_check = INLINE_SMI_CHECK);
+
+ // For a given |object| notify the garbage collector that the slot at |offset|
+ // has been written. |value| is the object being stored.
+ void RecordWrite(
+ Register object, Operand offset, Register value,
+ LinkRegisterStatus lr_status, SaveFPRegsMode save_fp,
+ RememberedSetAction remembered_set_action = EMIT_REMEMBERED_SET,
+ SmiCheck smi_check = INLINE_SMI_CHECK);
+
+ // ---------------------------------------------------------------------------
+ // Debugging.
+
+ void LoadNativeContextSlot(int index, Register dst);
+
+ DISALLOW_IMPLICIT_CONSTRUCTORS(MacroAssembler);
+};
+
+// Use this scope when you need a one-to-one mapping between methods and
+// instructions. This scope prevents the MacroAssembler from being called and
+// literal pools from being emitted. It also asserts the number of instructions
+// emitted is what you specified when creating the scope.
+class InstructionAccurateScope {
+ public:
+ explicit InstructionAccurateScope(TurboAssembler* tasm, size_t count = 0)
+ : tasm_(tasm),
+ block_pool_(tasm, count * kInstrSize)
+#ifdef DEBUG
+ ,
+ size_(count * kInstrSize)
+#endif
+ {
+ tasm_->CheckVeneerPool(false, true, count * kInstrSize);
+ tasm_->StartBlockVeneerPool();
+#ifdef DEBUG
+ if (count != 0) {
+ tasm_->bind(&start_);
+ }
+ previous_allow_macro_instructions_ = tasm_->allow_macro_instructions();
+ tasm_->set_allow_macro_instructions(false);
+#endif
+ }
+
+ ~InstructionAccurateScope() {
+ tasm_->EndBlockVeneerPool();
+#ifdef DEBUG
+ if (start_.is_bound()) {
+ DCHECK(tasm_->SizeOfCodeGeneratedSince(&start_) == size_);
+ }
+ tasm_->set_allow_macro_instructions(previous_allow_macro_instructions_);
+#endif
+ }
+
+ private:
+ TurboAssembler* tasm_;
+ TurboAssembler::BlockConstPoolScope block_pool_;
+#ifdef DEBUG
+ size_t size_;
+ Label start_;
+ bool previous_allow_macro_instructions_;
+#endif
+};
+
+// This scope utility allows scratch registers to be managed safely. The
+// TurboAssembler's TmpList() (and FPTmpList()) is used as a pool of scratch
+// registers. These registers can be allocated on demand, and will be returned
+// at the end of the scope.
+//
+// When the scope ends, the MacroAssembler's lists will be restored to their
+// original state, even if the lists were modified by some other means. Note
+// that this scope can be nested but the destructors need to run in the opposite
+// order as the constructors. We do not have assertions for this.
+class UseScratchRegisterScope {
+ public:
+ explicit UseScratchRegisterScope(TurboAssembler* tasm)
+ : available_(tasm->TmpList()),
+ availablefp_(tasm->FPTmpList()),
+ old_available_(available_->list()),
+ old_availablefp_(availablefp_->list()) {
+ DCHECK_EQ(available_->type(), CPURegister::kRegister);
+ DCHECK_EQ(availablefp_->type(), CPURegister::kVRegister);
+ }
+
+ V8_EXPORT_PRIVATE ~UseScratchRegisterScope();
+
+ // Take a register from the appropriate temps list. It will be returned
+ // automatically when the scope ends.
+ Register AcquireW() { return AcquireNextAvailable(available_).W(); }
+ Register AcquireX() { return AcquireNextAvailable(available_).X(); }
+ VRegister AcquireS() { return AcquireNextAvailable(availablefp_).S(); }
+ VRegister AcquireD() { return AcquireNextAvailable(availablefp_).D(); }
+ VRegister AcquireQ() { return AcquireNextAvailable(availablefp_).Q(); }
+ VRegister AcquireV(VectorFormat format) {
+ return VRegister::Create(AcquireNextAvailable(availablefp_).code(), format);
+ }
+
+ Register AcquireSameSizeAs(const Register& reg);
+ V8_EXPORT_PRIVATE VRegister AcquireSameSizeAs(const VRegister& reg);
+
+ void Include(const CPURegList& list) { available_->Combine(list); }
+ void Exclude(const CPURegList& list) {
+#if DEBUG
+ CPURegList copy(list);
+ while (!copy.IsEmpty()) {
+ const CPURegister& reg = copy.PopHighestIndex();
+ DCHECK(available_->IncludesAliasOf(reg));
+ }
+#endif
+ available_->Remove(list);
+ }
+ void Include(const Register& reg1, const Register& reg2 = NoReg) {
+ CPURegList list(reg1, reg2);
+ Include(list);
+ }
+ void Exclude(const Register& reg1, const Register& reg2 = NoReg) {
+ CPURegList list(reg1, reg2);
+ Exclude(list);
+ }
+
+ private:
+ V8_EXPORT_PRIVATE static CPURegister AcquireNextAvailable(
+ CPURegList* available);
+
+ // Available scratch registers.
+ CPURegList* available_; // kRegister
+ CPURegList* availablefp_; // kVRegister
+
+ // The state of the available lists at the start of this scope.
+ RegList old_available_; // kRegister
+ RegList old_availablefp_; // kVRegister
+};
+
+} // namespace internal
+} // namespace v8
+
+#define ACCESS_MASM(masm) masm->
+
+#endif // V8_CODEGEN_ARM64_MACRO_ASSEMBLER_ARM64_H_
diff --git a/src/codegen/arm64/register-arm64.cc b/src/codegen/arm64/register-arm64.cc
new file mode 100644
index 0000000..9144884
--- /dev/null
+++ b/src/codegen/arm64/register-arm64.cc
@@ -0,0 +1,302 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#if V8_TARGET_ARCH_ARM64
+
+#include "src/codegen/arm64/register-arm64.h"
+
+namespace v8 {
+namespace internal {
+
+VectorFormat VectorFormatHalfWidth(VectorFormat vform) {
+ DCHECK(vform == kFormat8H || vform == kFormat4S || vform == kFormat2D ||
+ vform == kFormatH || vform == kFormatS || vform == kFormatD);
+ switch (vform) {
+ case kFormat8H:
+ return kFormat8B;
+ case kFormat4S:
+ return kFormat4H;
+ case kFormat2D:
+ return kFormat2S;
+ case kFormatH:
+ return kFormatB;
+ case kFormatS:
+ return kFormatH;
+ case kFormatD:
+ return kFormatS;
+ default:
+ UNREACHABLE();
+ }
+}
+
+VectorFormat VectorFormatDoubleWidth(VectorFormat vform) {
+ DCHECK(vform == kFormat8B || vform == kFormat4H || vform == kFormat2S ||
+ vform == kFormatB || vform == kFormatH || vform == kFormatS);
+ switch (vform) {
+ case kFormat8B:
+ return kFormat8H;
+ case kFormat4H:
+ return kFormat4S;
+ case kFormat2S:
+ return kFormat2D;
+ case kFormatB:
+ return kFormatH;
+ case kFormatH:
+ return kFormatS;
+ case kFormatS:
+ return kFormatD;
+ default:
+ UNREACHABLE();
+ }
+}
+
+VectorFormat VectorFormatFillQ(VectorFormat vform) {
+ switch (vform) {
+ case kFormatB:
+ case kFormat8B:
+ case kFormat16B:
+ return kFormat16B;
+ case kFormatH:
+ case kFormat4H:
+ case kFormat8H:
+ return kFormat8H;
+ case kFormatS:
+ case kFormat2S:
+ case kFormat4S:
+ return kFormat4S;
+ case kFormatD:
+ case kFormat1D:
+ case kFormat2D:
+ return kFormat2D;
+ default:
+ UNREACHABLE();
+ }
+}
+
+VectorFormat VectorFormatHalfWidthDoubleLanes(VectorFormat vform) {
+ switch (vform) {
+ case kFormat4H:
+ return kFormat8B;
+ case kFormat8H:
+ return kFormat16B;
+ case kFormat2S:
+ return kFormat4H;
+ case kFormat4S:
+ return kFormat8H;
+ case kFormat1D:
+ return kFormat2S;
+ case kFormat2D:
+ return kFormat4S;
+ default:
+ UNREACHABLE();
+ }
+}
+
+VectorFormat VectorFormatDoubleLanes(VectorFormat vform) {
+ DCHECK(vform == kFormat8B || vform == kFormat4H || vform == kFormat2S);
+ switch (vform) {
+ case kFormat8B:
+ return kFormat16B;
+ case kFormat4H:
+ return kFormat8H;
+ case kFormat2S:
+ return kFormat4S;
+ default:
+ UNREACHABLE();
+ }
+}
+
+VectorFormat VectorFormatHalfLanes(VectorFormat vform) {
+ DCHECK(vform == kFormat16B || vform == kFormat8H || vform == kFormat4S);
+ switch (vform) {
+ case kFormat16B:
+ return kFormat8B;
+ case kFormat8H:
+ return kFormat4H;
+ case kFormat4S:
+ return kFormat2S;
+ default:
+ UNREACHABLE();
+ }
+}
+
+VectorFormat ScalarFormatFromLaneSize(int laneSize) {
+ switch (laneSize) {
+ case 8:
+ return kFormatB;
+ case 16:
+ return kFormatH;
+ case 32:
+ return kFormatS;
+ case 64:
+ return kFormatD;
+ default:
+ UNREACHABLE();
+ }
+}
+
+VectorFormat VectorFormatFillQ(int laneSize) {
+ return VectorFormatFillQ(ScalarFormatFromLaneSize(laneSize));
+}
+
+VectorFormat ScalarFormatFromFormat(VectorFormat vform) {
+ return ScalarFormatFromLaneSize(LaneSizeInBitsFromFormat(vform));
+}
+
+unsigned RegisterSizeInBytesFromFormat(VectorFormat vform) {
+ return RegisterSizeInBitsFromFormat(vform) / 8;
+}
+
+unsigned RegisterSizeInBitsFromFormat(VectorFormat vform) {
+ DCHECK_NE(vform, kFormatUndefined);
+ switch (vform) {
+ case kFormatB:
+ return kBRegSizeInBits;
+ case kFormatH:
+ return kHRegSizeInBits;
+ case kFormatS:
+ return kSRegSizeInBits;
+ case kFormatD:
+ return kDRegSizeInBits;
+ case kFormat8B:
+ case kFormat4H:
+ case kFormat2S:
+ case kFormat1D:
+ return kDRegSizeInBits;
+ default:
+ return kQRegSizeInBits;
+ }
+}
+
+unsigned LaneSizeInBitsFromFormat(VectorFormat vform) {
+ DCHECK_NE(vform, kFormatUndefined);
+ switch (vform) {
+ case kFormatB:
+ case kFormat8B:
+ case kFormat16B:
+ return 8;
+ case kFormatH:
+ case kFormat4H:
+ case kFormat8H:
+ return 16;
+ case kFormatS:
+ case kFormat2S:
+ case kFormat4S:
+ return 32;
+ case kFormatD:
+ case kFormat1D:
+ case kFormat2D:
+ return 64;
+ default:
+ UNREACHABLE();
+ }
+}
+
+int LaneSizeInBytesFromFormat(VectorFormat vform) {
+ return LaneSizeInBitsFromFormat(vform) / 8;
+}
+
+int LaneSizeInBytesLog2FromFormat(VectorFormat vform) {
+ DCHECK_NE(vform, kFormatUndefined);
+ switch (vform) {
+ case kFormatB:
+ case kFormat8B:
+ case kFormat16B:
+ return 0;
+ case kFormatH:
+ case kFormat4H:
+ case kFormat8H:
+ return 1;
+ case kFormatS:
+ case kFormat2S:
+ case kFormat4S:
+ return 2;
+ case kFormatD:
+ case kFormat1D:
+ case kFormat2D:
+ return 3;
+ default:
+ UNREACHABLE();
+ }
+}
+
+int LaneCountFromFormat(VectorFormat vform) {
+ DCHECK_NE(vform, kFormatUndefined);
+ switch (vform) {
+ case kFormat16B:
+ return 16;
+ case kFormat8B:
+ case kFormat8H:
+ return 8;
+ case kFormat4H:
+ case kFormat4S:
+ return 4;
+ case kFormat2S:
+ case kFormat2D:
+ return 2;
+ case kFormat1D:
+ case kFormatB:
+ case kFormatH:
+ case kFormatS:
+ case kFormatD:
+ return 1;
+ default:
+ UNREACHABLE();
+ }
+}
+
+int MaxLaneCountFromFormat(VectorFormat vform) {
+ DCHECK_NE(vform, kFormatUndefined);
+ switch (vform) {
+ case kFormatB:
+ case kFormat8B:
+ case kFormat16B:
+ return 16;
+ case kFormatH:
+ case kFormat4H:
+ case kFormat8H:
+ return 8;
+ case kFormatS:
+ case kFormat2S:
+ case kFormat4S:
+ return 4;
+ case kFormatD:
+ case kFormat1D:
+ case kFormat2D:
+ return 2;
+ default:
+ UNREACHABLE();
+ }
+}
+
+// Does 'vform' indicate a vector format or a scalar format?
+bool IsVectorFormat(VectorFormat vform) {
+ DCHECK_NE(vform, kFormatUndefined);
+ switch (vform) {
+ case kFormatB:
+ case kFormatH:
+ case kFormatS:
+ case kFormatD:
+ return false;
+ default:
+ return true;
+ }
+}
+
+int64_t MaxIntFromFormat(VectorFormat vform) {
+ return INT64_MAX >> (64 - LaneSizeInBitsFromFormat(vform));
+}
+
+int64_t MinIntFromFormat(VectorFormat vform) {
+ return INT64_MIN >> (64 - LaneSizeInBitsFromFormat(vform));
+}
+
+uint64_t MaxUintFromFormat(VectorFormat vform) {
+ return UINT64_MAX >> (64 - LaneSizeInBitsFromFormat(vform));
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_ARM64
diff --git a/src/codegen/arm64/register-arm64.h b/src/codegen/arm64/register-arm64.h
new file mode 100644
index 0000000..fbbb0a1
--- /dev/null
+++ b/src/codegen/arm64/register-arm64.h
@@ -0,0 +1,704 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_ARM64_REGISTER_ARM64_H_
+#define V8_CODEGEN_ARM64_REGISTER_ARM64_H_
+
+#include "src/codegen/arm64/utils-arm64.h"
+#include "src/codegen/register.h"
+#include "src/codegen/reglist.h"
+#include "src/common/globals.h"
+
+namespace v8 {
+namespace internal {
+
+// -----------------------------------------------------------------------------
+// Registers.
+// clang-format off
+#define GENERAL_REGISTER_CODE_LIST(R) \
+ R(0) R(1) R(2) R(3) R(4) R(5) R(6) R(7) \
+ R(8) R(9) R(10) R(11) R(12) R(13) R(14) R(15) \
+ R(16) R(17) R(18) R(19) R(20) R(21) R(22) R(23) \
+ R(24) R(25) R(26) R(27) R(28) R(29) R(30) R(31)
+
+#define GENERAL_REGISTERS(R) \
+ R(x0) R(x1) R(x2) R(x3) R(x4) R(x5) R(x6) R(x7) \
+ R(x8) R(x9) R(x10) R(x11) R(x12) R(x13) R(x14) R(x15) \
+ R(x16) R(x17) R(x18) R(x19) R(x20) R(x21) R(x22) R(x23) \
+ R(x24) R(x25) R(x26) R(x27) R(x28) R(x29) R(x30) R(x31)
+
+// x18 is the platform register and is reserved for the use of platform ABIs.
+// It is known to be reserved by the OS at least on Windows and iOS.
+#define ALLOCATABLE_GENERAL_REGISTERS(R) \
+ R(x0) R(x1) R(x2) R(x3) R(x4) R(x5) R(x6) R(x7) \
+ R(x8) R(x9) R(x10) R(x11) R(x12) R(x13) R(x14) R(x15) \
+ R(x19) R(x20) R(x21) R(x22) R(x23) R(x24) R(x25) \
+ R(x27) R(x28)
+
+#define FLOAT_REGISTERS(V) \
+ V(s0) V(s1) V(s2) V(s3) V(s4) V(s5) V(s6) V(s7) \
+ V(s8) V(s9) V(s10) V(s11) V(s12) V(s13) V(s14) V(s15) \
+ V(s16) V(s17) V(s18) V(s19) V(s20) V(s21) V(s22) V(s23) \
+ V(s24) V(s25) V(s26) V(s27) V(s28) V(s29) V(s30) V(s31)
+
+#define DOUBLE_REGISTERS(R) \
+ R(d0) R(d1) R(d2) R(d3) R(d4) R(d5) R(d6) R(d7) \
+ R(d8) R(d9) R(d10) R(d11) R(d12) R(d13) R(d14) R(d15) \
+ R(d16) R(d17) R(d18) R(d19) R(d20) R(d21) R(d22) R(d23) \
+ R(d24) R(d25) R(d26) R(d27) R(d28) R(d29) R(d30) R(d31)
+
+#define SIMD128_REGISTERS(V) \
+ V(q0) V(q1) V(q2) V(q3) V(q4) V(q5) V(q6) V(q7) \
+ V(q8) V(q9) V(q10) V(q11) V(q12) V(q13) V(q14) V(q15) \
+ V(q16) V(q17) V(q18) V(q19) V(q20) V(q21) V(q22) V(q23) \
+ V(q24) V(q25) V(q26) V(q27) V(q28) V(q29) V(q30) V(q31)
+
+#define VECTOR_REGISTERS(V) \
+ V(v0) V(v1) V(v2) V(v3) V(v4) V(v5) V(v6) V(v7) \
+ V(v8) V(v9) V(v10) V(v11) V(v12) V(v13) V(v14) V(v15) \
+ V(v16) V(v17) V(v18) V(v19) V(v20) V(v21) V(v22) V(v23) \
+ V(v24) V(v25) V(v26) V(v27) V(v28) V(v29) V(v30) V(v31)
+
+// Register d29 could be allocated, but we keep an even length list here, in
+// order to make stack alignment easier for save and restore.
+#define ALLOCATABLE_DOUBLE_REGISTERS(R) \
+ R(d0) R(d1) R(d2) R(d3) R(d4) R(d5) R(d6) R(d7) \
+ R(d8) R(d9) R(d10) R(d11) R(d12) R(d13) R(d14) R(d16) \
+ R(d17) R(d18) R(d19) R(d20) R(d21) R(d22) R(d23) R(d24) \
+ R(d25) R(d26) R(d27) R(d28)
+// clang-format on
+
+constexpr int kRegListSizeInBits = sizeof(RegList) * kBitsPerByte;
+
+// Some CPURegister methods can return Register and VRegister types, so we
+// need to declare them in advance.
+class Register;
+class VRegister;
+
+enum RegisterCode {
+#define REGISTER_CODE(R) kRegCode_##R,
+ GENERAL_REGISTERS(REGISTER_CODE)
+#undef REGISTER_CODE
+ kRegAfterLast
+};
+
+class CPURegister : public RegisterBase<CPURegister, kRegAfterLast> {
+ public:
+ enum RegisterType { kRegister, kVRegister, kNoRegister };
+
+ static constexpr CPURegister no_reg() {
+ return CPURegister{kCode_no_reg, 0, kNoRegister};
+ }
+
+ static constexpr CPURegister Create(int code, int size, RegisterType type) {
+ CONSTEXPR_DCHECK(IsValid(code, size, type));
+ return CPURegister{code, size, type};
+ }
+
+ RegisterType type() const { return reg_type_; }
+ int SizeInBits() const {
+ DCHECK(is_valid());
+ return reg_size_;
+ }
+ int SizeInBytes() const {
+ DCHECK(is_valid());
+ DCHECK_EQ(SizeInBits() % 8, 0);
+ return reg_size_ / 8;
+ }
+ bool Is8Bits() const {
+ DCHECK(is_valid());
+ return reg_size_ == 8;
+ }
+ bool Is16Bits() const {
+ DCHECK(is_valid());
+ return reg_size_ == 16;
+ }
+ bool Is32Bits() const {
+ DCHECK(is_valid());
+ return reg_size_ == 32;
+ }
+ bool Is64Bits() const {
+ DCHECK(is_valid());
+ return reg_size_ == 64;
+ }
+ bool Is128Bits() const {
+ DCHECK(is_valid());
+ return reg_size_ == 128;
+ }
+ bool IsNone() const { return reg_type_ == kNoRegister; }
+ constexpr bool Aliases(const CPURegister& other) const {
+ return RegisterBase::operator==(other) && reg_type_ == other.reg_type_;
+ }
+
+ constexpr bool operator==(const CPURegister& other) const {
+ return RegisterBase::operator==(other) && reg_size_ == other.reg_size_ &&
+ reg_type_ == other.reg_type_;
+ }
+ constexpr bool operator!=(const CPURegister& other) const {
+ return !operator==(other);
+ }
+
+ bool IsZero() const;
+ bool IsSP() const;
+
+ bool IsRegister() const { return reg_type_ == kRegister; }
+ bool IsVRegister() const { return reg_type_ == kVRegister; }
+
+ bool IsFPRegister() const { return IsS() || IsD(); }
+
+ bool IsW() const { return IsRegister() && Is32Bits(); }
+ bool IsX() const { return IsRegister() && Is64Bits(); }
+
+ // These assertions ensure that the size and type of the register are as
+ // described. They do not consider the number of lanes that make up a vector.
+ // So, for example, Is8B() implies IsD(), and Is1D() implies IsD, but IsD()
+ // does not imply Is1D() or Is8B().
+ // Check the number of lanes, ie. the format of the vector, using methods such
+ // as Is8B(), Is1D(), etc. in the VRegister class.
+ bool IsV() const { return IsVRegister(); }
+ bool IsB() const { return IsV() && Is8Bits(); }
+ bool IsH() const { return IsV() && Is16Bits(); }
+ bool IsS() const { return IsV() && Is32Bits(); }
+ bool IsD() const { return IsV() && Is64Bits(); }
+ bool IsQ() const { return IsV() && Is128Bits(); }
+
+ Register Reg() const;
+ VRegister VReg() const;
+
+ Register X() const;
+ Register W() const;
+ VRegister V() const;
+ VRegister B() const;
+ VRegister H() const;
+ VRegister D() const;
+ VRegister S() const;
+ VRegister Q() const;
+
+ bool IsSameSizeAndType(const CPURegister& other) const;
+
+ protected:
+ int reg_size_;
+ RegisterType reg_type_;
+
+#if defined(V8_OS_WIN) && !defined(__clang__)
+ // MSVC has problem to parse template base class as friend class.
+ friend RegisterBase;
+#else
+ friend class RegisterBase;
+#endif
+
+ constexpr CPURegister(int code, int size, RegisterType type)
+ : RegisterBase(code), reg_size_(size), reg_type_(type) {}
+
+ static constexpr bool IsValidRegister(int code, int size) {
+ return (size == kWRegSizeInBits || size == kXRegSizeInBits) &&
+ (code < kNumberOfRegisters || code == kSPRegInternalCode);
+ }
+
+ static constexpr bool IsValidVRegister(int code, int size) {
+ return (size == kBRegSizeInBits || size == kHRegSizeInBits ||
+ size == kSRegSizeInBits || size == kDRegSizeInBits ||
+ size == kQRegSizeInBits) &&
+ code < kNumberOfVRegisters;
+ }
+
+ static constexpr bool IsValid(int code, int size, RegisterType type) {
+ return (type == kRegister && IsValidRegister(code, size)) ||
+ (type == kVRegister && IsValidVRegister(code, size));
+ }
+
+ static constexpr bool IsNone(int code, int size, RegisterType type) {
+ return type == kNoRegister && code == 0 && size == 0;
+ }
+};
+
+ASSERT_TRIVIALLY_COPYABLE(CPURegister);
+
+class Register : public CPURegister {
+ public:
+ static constexpr Register no_reg() { return Register(CPURegister::no_reg()); }
+
+ static constexpr Register Create(int code, int size) {
+ return Register(CPURegister::Create(code, size, CPURegister::kRegister));
+ }
+
+ static Register XRegFromCode(unsigned code);
+ static Register WRegFromCode(unsigned code);
+
+ static constexpr Register from_code(int code) {
+ // Always return an X register.
+ return Register::Create(code, kXRegSizeInBits);
+ }
+
+ static const char* GetSpecialRegisterName(int code) {
+ return (code == kSPRegInternalCode) ? "sp" : "UNKNOWN";
+ }
+
+ private:
+ constexpr explicit Register(const CPURegister& r) : CPURegister(r) {}
+};
+
+ASSERT_TRIVIALLY_COPYABLE(Register);
+
+constexpr bool kPadArguments = true;
+constexpr bool kSimpleFPAliasing = true;
+constexpr bool kSimdMaskRegisters = false;
+
+enum DoubleRegisterCode {
+#define REGISTER_CODE(R) kDoubleCode_##R,
+ DOUBLE_REGISTERS(REGISTER_CODE)
+#undef REGISTER_CODE
+ kDoubleAfterLast
+};
+
+// Functions for handling NEON vector format information.
+enum VectorFormat {
+ kFormatUndefined = 0xffffffff,
+ kFormat8B = NEON_8B,
+ kFormat16B = NEON_16B,
+ kFormat4H = NEON_4H,
+ kFormat8H = NEON_8H,
+ kFormat2S = NEON_2S,
+ kFormat4S = NEON_4S,
+ kFormat1D = NEON_1D,
+ kFormat2D = NEON_2D,
+
+ // Scalar formats. We add the scalar bit to distinguish between scalar and
+ // vector enumerations; the bit is always set in the encoding of scalar ops
+ // and always clear for vector ops. Although kFormatD and kFormat1D appear
+ // to be the same, their meaning is subtly different. The first is a scalar
+ // operation, the second a vector operation that only affects one lane.
+ kFormatB = NEON_B | NEONScalar,
+ kFormatH = NEON_H | NEONScalar,
+ kFormatS = NEON_S | NEONScalar,
+ kFormatD = NEON_D | NEONScalar
+};
+
+VectorFormat VectorFormatHalfWidth(VectorFormat vform);
+VectorFormat VectorFormatDoubleWidth(VectorFormat vform);
+VectorFormat VectorFormatDoubleLanes(VectorFormat vform);
+VectorFormat VectorFormatHalfLanes(VectorFormat vform);
+VectorFormat ScalarFormatFromLaneSize(int lanesize);
+VectorFormat VectorFormatHalfWidthDoubleLanes(VectorFormat vform);
+VectorFormat VectorFormatFillQ(int laneSize);
+VectorFormat VectorFormatFillQ(VectorFormat vform);
+VectorFormat ScalarFormatFromFormat(VectorFormat vform);
+V8_EXPORT_PRIVATE unsigned RegisterSizeInBitsFromFormat(VectorFormat vform);
+unsigned RegisterSizeInBytesFromFormat(VectorFormat vform);
+int LaneSizeInBytesFromFormat(VectorFormat vform);
+unsigned LaneSizeInBitsFromFormat(VectorFormat vform);
+int LaneSizeInBytesLog2FromFormat(VectorFormat vform);
+V8_EXPORT_PRIVATE int LaneCountFromFormat(VectorFormat vform);
+int MaxLaneCountFromFormat(VectorFormat vform);
+V8_EXPORT_PRIVATE bool IsVectorFormat(VectorFormat vform);
+int64_t MaxIntFromFormat(VectorFormat vform);
+int64_t MinIntFromFormat(VectorFormat vform);
+uint64_t MaxUintFromFormat(VectorFormat vform);
+
+class VRegister : public CPURegister {
+ public:
+ static constexpr VRegister no_reg() {
+ return VRegister(CPURegister::no_reg(), 0);
+ }
+
+ static constexpr VRegister Create(int code, int size, int lane_count = 1) {
+ CONSTEXPR_DCHECK(IsValidLaneCount(lane_count));
+ return VRegister(CPURegister::Create(code, size, CPURegister::kVRegister),
+ lane_count);
+ }
+
+ static VRegister Create(int reg_code, VectorFormat format) {
+ int reg_size = RegisterSizeInBitsFromFormat(format);
+ int reg_count = IsVectorFormat(format) ? LaneCountFromFormat(format) : 1;
+ return VRegister::Create(reg_code, reg_size, reg_count);
+ }
+
+ static VRegister BRegFromCode(unsigned code);
+ static VRegister HRegFromCode(unsigned code);
+ static VRegister SRegFromCode(unsigned code);
+ static VRegister DRegFromCode(unsigned code);
+ static VRegister QRegFromCode(unsigned code);
+ static VRegister VRegFromCode(unsigned code);
+
+ VRegister V8B() const {
+ return VRegister::Create(code(), kDRegSizeInBits, 8);
+ }
+ VRegister V16B() const {
+ return VRegister::Create(code(), kQRegSizeInBits, 16);
+ }
+ VRegister V4H() const {
+ return VRegister::Create(code(), kDRegSizeInBits, 4);
+ }
+ VRegister V8H() const {
+ return VRegister::Create(code(), kQRegSizeInBits, 8);
+ }
+ VRegister V2S() const {
+ return VRegister::Create(code(), kDRegSizeInBits, 2);
+ }
+ VRegister V4S() const {
+ return VRegister::Create(code(), kQRegSizeInBits, 4);
+ }
+ VRegister V2D() const {
+ return VRegister::Create(code(), kQRegSizeInBits, 2);
+ }
+ VRegister V1D() const {
+ return VRegister::Create(code(), kDRegSizeInBits, 1);
+ }
+
+ VRegister Format(VectorFormat f) const {
+ return VRegister::Create(code(), f);
+ }
+
+ bool Is8B() const { return (Is64Bits() && (lane_count_ == 8)); }
+ bool Is16B() const { return (Is128Bits() && (lane_count_ == 16)); }
+ bool Is4H() const { return (Is64Bits() && (lane_count_ == 4)); }
+ bool Is8H() const { return (Is128Bits() && (lane_count_ == 8)); }
+ bool Is2S() const { return (Is64Bits() && (lane_count_ == 2)); }
+ bool Is4S() const { return (Is128Bits() && (lane_count_ == 4)); }
+ bool Is1D() const { return (Is64Bits() && (lane_count_ == 1)); }
+ bool Is2D() const { return (Is128Bits() && (lane_count_ == 2)); }
+
+ // For consistency, we assert the number of lanes of these scalar registers,
+ // even though there are no vectors of equivalent total size with which they
+ // could alias.
+ bool Is1B() const {
+ DCHECK(!(Is8Bits() && IsVector()));
+ return Is8Bits();
+ }
+ bool Is1H() const {
+ DCHECK(!(Is16Bits() && IsVector()));
+ return Is16Bits();
+ }
+ bool Is1S() const {
+ DCHECK(!(Is32Bits() && IsVector()));
+ return Is32Bits();
+ }
+
+ bool IsLaneSizeB() const { return LaneSizeInBits() == kBRegSizeInBits; }
+ bool IsLaneSizeH() const { return LaneSizeInBits() == kHRegSizeInBits; }
+ bool IsLaneSizeS() const { return LaneSizeInBits() == kSRegSizeInBits; }
+ bool IsLaneSizeD() const { return LaneSizeInBits() == kDRegSizeInBits; }
+
+ bool IsScalar() const { return lane_count_ == 1; }
+ bool IsVector() const { return lane_count_ > 1; }
+
+ bool IsSameFormat(const VRegister& other) const {
+ return (reg_size_ == other.reg_size_) && (lane_count_ == other.lane_count_);
+ }
+
+ int LaneCount() const { return lane_count_; }
+
+ unsigned LaneSizeInBytes() const { return SizeInBytes() / lane_count_; }
+
+ unsigned LaneSizeInBits() const { return LaneSizeInBytes() * 8; }
+
+ static constexpr int kMaxNumRegisters = kNumberOfVRegisters;
+ STATIC_ASSERT(kMaxNumRegisters == kDoubleAfterLast);
+
+ static VRegister from_code(int code) {
+ // Always return a D register.
+ return VRegister::Create(code, kDRegSizeInBits);
+ }
+
+ private:
+ int lane_count_;
+
+ constexpr explicit VRegister(const CPURegister& r, int lane_count)
+ : CPURegister(r), lane_count_(lane_count) {}
+
+ static constexpr bool IsValidLaneCount(int lane_count) {
+ return base::bits::IsPowerOfTwo(lane_count) && lane_count <= 16;
+ }
+};
+
+ASSERT_TRIVIALLY_COPYABLE(VRegister);
+
+// No*Reg is used to indicate an unused argument, or an error case. Note that
+// these all compare equal. The Register and VRegister variants are provided for
+// convenience.
+constexpr Register NoReg = Register::no_reg();
+constexpr VRegister NoVReg = VRegister::no_reg();
+constexpr CPURegister NoCPUReg = CPURegister::no_reg();
+constexpr Register no_reg = NoReg;
+constexpr VRegister no_dreg = NoVReg;
+
+#define DEFINE_REGISTER(register_class, name, ...) \
+ constexpr register_class name = register_class::Create(__VA_ARGS__)
+#define ALIAS_REGISTER(register_class, alias, name) \
+ constexpr register_class alias = name
+
+#define DEFINE_REGISTERS(N) \
+ DEFINE_REGISTER(Register, w##N, N, kWRegSizeInBits); \
+ DEFINE_REGISTER(Register, x##N, N, kXRegSizeInBits);
+GENERAL_REGISTER_CODE_LIST(DEFINE_REGISTERS)
+#undef DEFINE_REGISTERS
+
+DEFINE_REGISTER(Register, wsp, kSPRegInternalCode, kWRegSizeInBits);
+DEFINE_REGISTER(Register, sp, kSPRegInternalCode, kXRegSizeInBits);
+
+#define DEFINE_VREGISTERS(N) \
+ DEFINE_REGISTER(VRegister, b##N, N, kBRegSizeInBits); \
+ DEFINE_REGISTER(VRegister, h##N, N, kHRegSizeInBits); \
+ DEFINE_REGISTER(VRegister, s##N, N, kSRegSizeInBits); \
+ DEFINE_REGISTER(VRegister, d##N, N, kDRegSizeInBits); \
+ DEFINE_REGISTER(VRegister, q##N, N, kQRegSizeInBits); \
+ DEFINE_REGISTER(VRegister, v##N, N, kQRegSizeInBits);
+GENERAL_REGISTER_CODE_LIST(DEFINE_VREGISTERS)
+#undef DEFINE_VREGISTERS
+
+#undef DEFINE_REGISTER
+
+// Registers aliases.
+ALIAS_REGISTER(VRegister, v8_, v8); // Avoid conflicts with namespace v8.
+ALIAS_REGISTER(Register, ip0, x16);
+ALIAS_REGISTER(Register, ip1, x17);
+ALIAS_REGISTER(Register, wip0, w16);
+ALIAS_REGISTER(Register, wip1, w17);
+// Root register.
+ALIAS_REGISTER(Register, kRootRegister, x26);
+ALIAS_REGISTER(Register, rr, x26);
+// Context pointer register.
+ALIAS_REGISTER(Register, cp, x27);
+ALIAS_REGISTER(Register, fp, x29);
+ALIAS_REGISTER(Register, lr, x30);
+ALIAS_REGISTER(Register, xzr, x31);
+ALIAS_REGISTER(Register, wzr, w31);
+
+// Register used for padding stack slots.
+ALIAS_REGISTER(Register, padreg, x31);
+
+// Keeps the 0 double value.
+ALIAS_REGISTER(VRegister, fp_zero, d15);
+// MacroAssembler fixed V Registers.
+// d29 is not part of ALLOCATABLE_DOUBLE_REGISTERS, so use 27 and 28.
+ALIAS_REGISTER(VRegister, fp_fixed1, d27);
+ALIAS_REGISTER(VRegister, fp_fixed2, d28);
+
+// MacroAssembler scratch V registers.
+ALIAS_REGISTER(VRegister, fp_scratch, d30);
+ALIAS_REGISTER(VRegister, fp_scratch1, d30);
+ALIAS_REGISTER(VRegister, fp_scratch2, d31);
+
+#undef ALIAS_REGISTER
+
+// AreAliased returns true if any of the named registers overlap. Arguments set
+// to NoReg are ignored. The system stack pointer may be specified.
+V8_EXPORT_PRIVATE bool AreAliased(
+ const CPURegister& reg1, const CPURegister& reg2,
+ const CPURegister& reg3 = NoReg, const CPURegister& reg4 = NoReg,
+ const CPURegister& reg5 = NoReg, const CPURegister& reg6 = NoReg,
+ const CPURegister& reg7 = NoReg, const CPURegister& reg8 = NoReg);
+
+// AreSameSizeAndType returns true if all of the specified registers have the
+// same size, and are of the same type. The system stack pointer may be
+// specified. Arguments set to NoReg are ignored, as are any subsequent
+// arguments. At least one argument (reg1) must be valid (not NoCPUReg).
+V8_EXPORT_PRIVATE bool AreSameSizeAndType(
+ const CPURegister& reg1, const CPURegister& reg2 = NoCPUReg,
+ const CPURegister& reg3 = NoCPUReg, const CPURegister& reg4 = NoCPUReg,
+ const CPURegister& reg5 = NoCPUReg, const CPURegister& reg6 = NoCPUReg,
+ const CPURegister& reg7 = NoCPUReg, const CPURegister& reg8 = NoCPUReg);
+
+// AreSameFormat returns true if all of the specified VRegisters have the same
+// vector format. Arguments set to NoVReg are ignored, as are any subsequent
+// arguments. At least one argument (reg1) must be valid (not NoVReg).
+bool AreSameFormat(const VRegister& reg1, const VRegister& reg2,
+ const VRegister& reg3 = NoVReg,
+ const VRegister& reg4 = NoVReg);
+
+// AreConsecutive returns true if all of the specified VRegisters are
+// consecutive in the register file. Arguments may be set to NoVReg, and if so,
+// subsequent arguments must also be NoVReg. At least one argument (reg1) must
+// be valid (not NoVReg).
+V8_EXPORT_PRIVATE bool AreConsecutive(const VRegister& reg1,
+ const VRegister& reg2,
+ const VRegister& reg3 = NoVReg,
+ const VRegister& reg4 = NoVReg);
+
+using FloatRegister = VRegister;
+using DoubleRegister = VRegister;
+using Simd128Register = VRegister;
+
+// -----------------------------------------------------------------------------
+// Lists of registers.
+class V8_EXPORT_PRIVATE CPURegList {
+ public:
+ template <typename... CPURegisters>
+ explicit CPURegList(CPURegister reg0, CPURegisters... regs)
+ : list_(CPURegister::ListOf(reg0, regs...)),
+ size_(reg0.SizeInBits()),
+ type_(reg0.type()) {
+ DCHECK(AreSameSizeAndType(reg0, regs...));
+ DCHECK(is_valid());
+ }
+
+ CPURegList(CPURegister::RegisterType type, int size, RegList list)
+ : list_(list), size_(size), type_(type) {
+ DCHECK(is_valid());
+ }
+
+ CPURegList(CPURegister::RegisterType type, int size, int first_reg,
+ int last_reg)
+ : size_(size), type_(type) {
+ DCHECK(
+ ((type == CPURegister::kRegister) && (last_reg < kNumberOfRegisters)) ||
+ ((type == CPURegister::kVRegister) &&
+ (last_reg < kNumberOfVRegisters)));
+ DCHECK(last_reg >= first_reg);
+ list_ = (1ULL << (last_reg + 1)) - 1;
+ list_ &= ~((1ULL << first_reg) - 1);
+ DCHECK(is_valid());
+ }
+
+ CPURegister::RegisterType type() const {
+ return type_;
+ }
+
+ RegList list() const {
+ return list_;
+ }
+
+ inline void set_list(RegList new_list) {
+ list_ = new_list;
+ DCHECK(is_valid());
+ }
+
+ // Combine another CPURegList into this one. Registers that already exist in
+ // this list are left unchanged. The type and size of the registers in the
+ // 'other' list must match those in this list.
+ void Combine(const CPURegList& other);
+
+ // Remove every register in the other CPURegList from this one. Registers that
+ // do not exist in this list are ignored. The type of the registers in the
+ // 'other' list must match those in this list.
+ void Remove(const CPURegList& other);
+
+ // Variants of Combine and Remove which take CPURegisters.
+ void Combine(const CPURegister& other);
+ void Remove(const CPURegister& other1, const CPURegister& other2 = NoCPUReg,
+ const CPURegister& other3 = NoCPUReg,
+ const CPURegister& other4 = NoCPUReg);
+
+ // Variants of Combine and Remove which take a single register by its code;
+ // the type and size of the register is inferred from this list.
+ void Combine(int code);
+ void Remove(int code);
+
+ // Align the list to 16 bytes.
+ void Align();
+
+ CPURegister PopLowestIndex();
+ CPURegister PopHighestIndex();
+
+ // AAPCS64 callee-saved registers.
+ static CPURegList GetCalleeSaved(int size = kXRegSizeInBits);
+ static CPURegList GetCalleeSavedV(int size = kDRegSizeInBits);
+
+ // AAPCS64 caller-saved registers. Note that this includes lr.
+ // TODO(all): Determine how we handle d8-d15 being callee-saved, but the top
+ // 64-bits being caller-saved.
+ static CPURegList GetCallerSaved(int size = kXRegSizeInBits);
+ static CPURegList GetCallerSavedV(int size = kDRegSizeInBits);
+
+ bool IsEmpty() const {
+ return list_ == 0;
+ }
+
+ bool IncludesAliasOf(const CPURegister& other1,
+ const CPURegister& other2 = NoCPUReg,
+ const CPURegister& other3 = NoCPUReg,
+ const CPURegister& other4 = NoCPUReg) const {
+ RegList list = 0;
+ if (!other1.IsNone() && (other1.type() == type_)) list |= other1.bit();
+ if (!other2.IsNone() && (other2.type() == type_)) list |= other2.bit();
+ if (!other3.IsNone() && (other3.type() == type_)) list |= other3.bit();
+ if (!other4.IsNone() && (other4.type() == type_)) list |= other4.bit();
+ return (list_ & list) != 0;
+ }
+
+ int Count() const {
+ return CountSetBits(list_, kRegListSizeInBits);
+ }
+
+ int RegisterSizeInBits() const {
+ return size_;
+ }
+
+ int RegisterSizeInBytes() const {
+ int size_in_bits = RegisterSizeInBits();
+ DCHECK_EQ(size_in_bits % kBitsPerByte, 0);
+ return size_in_bits / kBitsPerByte;
+ }
+
+ int TotalSizeInBytes() const {
+ return RegisterSizeInBytes() * Count();
+ }
+
+ private:
+ RegList list_;
+ int size_;
+ CPURegister::RegisterType type_;
+
+ bool is_valid() const {
+ constexpr RegList kValidRegisters{0x8000000ffffffff};
+ constexpr RegList kValidVRegisters{0x0000000ffffffff};
+ switch (type_) {
+ case CPURegister::kRegister:
+ return (list_ & kValidRegisters) == list_;
+ case CPURegister::kVRegister:
+ return (list_ & kValidVRegisters) == list_;
+ case CPURegister::kNoRegister:
+ return list_ == 0;
+ default:
+ UNREACHABLE();
+ }
+ }
+};
+
+// AAPCS64 callee-saved registers.
+#define kCalleeSaved CPURegList::GetCalleeSaved()
+#define kCalleeSavedV CPURegList::GetCalleeSavedV()
+
+// AAPCS64 caller-saved registers. Note that this includes lr.
+#define kCallerSaved CPURegList::GetCallerSaved()
+#define kCallerSavedV CPURegList::GetCallerSavedV()
+
+// Define a {RegisterName} method for {Register} and {VRegister}.
+DEFINE_REGISTER_NAMES(Register, GENERAL_REGISTERS)
+DEFINE_REGISTER_NAMES(VRegister, VECTOR_REGISTERS)
+
+// Give alias names to registers for calling conventions.
+constexpr Register kReturnRegister0 = x0;
+constexpr Register kReturnRegister1 = x1;
+constexpr Register kReturnRegister2 = x2;
+constexpr Register kJSFunctionRegister = x1;
+constexpr Register kContextRegister = cp;
+constexpr Register kAllocateSizeRegister = x1;
+
+constexpr Register kSpeculationPoisonRegister = x23;
+
+constexpr Register kInterpreterAccumulatorRegister = x0;
+constexpr Register kInterpreterBytecodeOffsetRegister = x19;
+constexpr Register kInterpreterBytecodeArrayRegister = x20;
+constexpr Register kInterpreterDispatchTableRegister = x21;
+
+constexpr Register kJavaScriptCallArgCountRegister = x0;
+constexpr Register kJavaScriptCallCodeStartRegister = x2;
+constexpr Register kJavaScriptCallTargetRegister = kJSFunctionRegister;
+constexpr Register kJavaScriptCallNewTargetRegister = x3;
+constexpr Register kJavaScriptCallExtraArg1Register = x2;
+
+constexpr Register kOffHeapTrampolineRegister = ip0;
+constexpr Register kRuntimeCallFunctionRegister = x1;
+constexpr Register kRuntimeCallArgCountRegister = x0;
+constexpr Register kRuntimeCallArgvRegister = x11;
+constexpr Register kWasmInstanceRegister = x7;
+constexpr Register kWasmCompileLazyFuncIndexRegister = x8;
+
+constexpr DoubleRegister kFPReturnRegister0 = d0;
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_ARM64_REGISTER_ARM64_H_
diff --git a/src/codegen/arm64/utils-arm64.cc b/src/codegen/arm64/utils-arm64.cc
new file mode 100644
index 0000000..dba2eeb
--- /dev/null
+++ b/src/codegen/arm64/utils-arm64.cc
@@ -0,0 +1,127 @@
+// Copyright 2013 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#if V8_TARGET_ARCH_ARM64
+
+#include "src/codegen/arm64/utils-arm64.h"
+
+namespace v8 {
+namespace internal {
+
+#define __ assm->
+
+uint32_t float_sign(float val) {
+ uint32_t bits = bit_cast<uint32_t>(val);
+ return unsigned_bitextract_32(31, 31, bits);
+}
+
+uint32_t float_exp(float val) {
+ uint32_t bits = bit_cast<uint32_t>(val);
+ return unsigned_bitextract_32(30, 23, bits);
+}
+
+uint32_t float_mantissa(float val) {
+ uint32_t bits = bit_cast<uint32_t>(val);
+ return unsigned_bitextract_32(22, 0, bits);
+}
+
+uint32_t double_sign(double val) {
+ uint64_t bits = bit_cast<uint64_t>(val);
+ return static_cast<uint32_t>(unsigned_bitextract_64(63, 63, bits));
+}
+
+uint32_t double_exp(double val) {
+ uint64_t bits = bit_cast<uint64_t>(val);
+ return static_cast<uint32_t>(unsigned_bitextract_64(62, 52, bits));
+}
+
+uint64_t double_mantissa(double val) {
+ uint64_t bits = bit_cast<uint64_t>(val);
+ return unsigned_bitextract_64(51, 0, bits);
+}
+
+float float_pack(uint32_t sign, uint32_t exp, uint32_t mantissa) {
+ uint32_t bits = sign << kFloatExponentBits | exp;
+ return bit_cast<float>((bits << kFloatMantissaBits) | mantissa);
+}
+
+double double_pack(uint64_t sign, uint64_t exp, uint64_t mantissa) {
+ uint64_t bits = sign << kDoubleExponentBits | exp;
+ return bit_cast<double>((bits << kDoubleMantissaBits) | mantissa);
+}
+
+int float16classify(float16 value) {
+ const uint16_t exponent_max = (1 << kFloat16ExponentBits) - 1;
+ const uint16_t exponent_mask = exponent_max << kFloat16MantissaBits;
+ const uint16_t mantissa_mask = (1 << kFloat16MantissaBits) - 1;
+
+ const uint16_t exponent = (value & exponent_mask) >> kFloat16MantissaBits;
+ const uint16_t mantissa = value & mantissa_mask;
+ if (exponent == 0) {
+ if (mantissa == 0) {
+ return FP_ZERO;
+ }
+ return FP_SUBNORMAL;
+ } else if (exponent == exponent_max) {
+ if (mantissa == 0) {
+ return FP_INFINITE;
+ }
+ return FP_NAN;
+ }
+ return FP_NORMAL;
+}
+
+int CountLeadingZeros(uint64_t value, int width) {
+ DCHECK(base::bits::IsPowerOfTwo(width) && (width <= 64));
+ if (value == 0) {
+ return width;
+ }
+ return base::bits::CountLeadingZeros64(value << (64 - width));
+}
+
+int CountLeadingSignBits(int64_t value, int width) {
+ DCHECK(base::bits::IsPowerOfTwo(width) && (width <= 64));
+ if (value >= 0) {
+ return CountLeadingZeros(value, width) - 1;
+ } else {
+ return CountLeadingZeros(~value, width) - 1;
+ }
+}
+
+int CountSetBits(uint64_t value, int width) {
+ DCHECK((width == 32) || (width == 64));
+ if (width == 64) {
+ return static_cast<int>(base::bits::CountPopulation(value));
+ }
+ return static_cast<int>(
+ base::bits::CountPopulation(static_cast<uint32_t>(value & 0xFFFFFFFFF)));
+}
+
+int LowestSetBitPosition(uint64_t value) {
+ DCHECK_NE(value, 0U);
+ return base::bits::CountTrailingZeros(value) + 1;
+}
+
+int HighestSetBitPosition(uint64_t value) {
+ DCHECK_NE(value, 0U);
+ return 63 - CountLeadingZeros(value, 64);
+}
+
+uint64_t LargestPowerOf2Divisor(uint64_t value) {
+ // Simulate two's complement (instead of casting to signed and negating) to
+ // avoid undefined behavior on signed overflow.
+ return value & ((~value) + 1);
+}
+
+int MaskToBit(uint64_t mask) {
+ DCHECK_EQ(CountSetBits(mask, 64), 1);
+ return base::bits::CountTrailingZeros(mask);
+}
+
+#undef __
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_ARM64
diff --git a/src/codegen/arm64/utils-arm64.h b/src/codegen/arm64/utils-arm64.h
new file mode 100644
index 0000000..182d781
--- /dev/null
+++ b/src/codegen/arm64/utils-arm64.h
@@ -0,0 +1,122 @@
+// Copyright 2013 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_ARM64_UTILS_ARM64_H_
+#define V8_CODEGEN_ARM64_UTILS_ARM64_H_
+
+#include <cmath>
+
+#include "src/codegen/arm64/constants-arm64.h"
+#include "src/utils/utils.h"
+
+namespace v8 {
+namespace internal {
+
+// These are global assumptions in v8.
+STATIC_ASSERT((static_cast<int32_t>(-1) >> 1) == -1);
+STATIC_ASSERT((static_cast<uint32_t>(-1) >> 1) == 0x7FFFFFFF);
+
+uint32_t float_sign(float val);
+uint32_t float_exp(float val);
+uint32_t float_mantissa(float val);
+uint32_t double_sign(double val);
+uint32_t double_exp(double val);
+uint64_t double_mantissa(double val);
+
+float float_pack(uint32_t sign, uint32_t exp, uint32_t mantissa);
+double double_pack(uint64_t sign, uint64_t exp, uint64_t mantissa);
+
+// An fpclassify() function for 16-bit half-precision floats.
+int float16classify(float16 value);
+
+// Bit counting.
+int CountLeadingZeros(uint64_t value, int width);
+int CountLeadingSignBits(int64_t value, int width);
+V8_EXPORT_PRIVATE int CountSetBits(uint64_t value, int width);
+int LowestSetBitPosition(uint64_t value);
+int HighestSetBitPosition(uint64_t value);
+uint64_t LargestPowerOf2Divisor(uint64_t value);
+int MaskToBit(uint64_t mask);
+
+template <typename T>
+T ReverseBytes(T value, int block_bytes_log2) {
+ DCHECK((sizeof(value) == 4) || (sizeof(value) == 8));
+ DCHECK((1ULL << block_bytes_log2) <= sizeof(value));
+ // Split the 64-bit value into an 8-bit array, where b[0] is the least
+ // significant byte, and b[7] is the most significant.
+ uint8_t bytes[8];
+ uint64_t mask = 0xff00000000000000;
+ for (int i = 7; i >= 0; i--) {
+ bytes[i] = (static_cast<uint64_t>(value) & mask) >> (i * 8);
+ mask >>= 8;
+ }
+
+ // Permutation tables for REV instructions.
+ // permute_table[0] is used by REV16_x, REV16_w
+ // permute_table[1] is used by REV32_x, REV_w
+ // permute_table[2] is used by REV_x
+ DCHECK((0 < block_bytes_log2) && (block_bytes_log2 < 4));
+ static const uint8_t permute_table[3][8] = {{6, 7, 4, 5, 2, 3, 0, 1},
+ {4, 5, 6, 7, 0, 1, 2, 3},
+ {0, 1, 2, 3, 4, 5, 6, 7}};
+ typename std::make_unsigned<T>::type result = 0;
+ for (int i = 0; i < 8; i++) {
+ result <<= 8;
+ result |= bytes[permute_table[block_bytes_log2 - 1][i]];
+ }
+ return result;
+}
+
+// NaN tests.
+inline bool IsSignallingNaN(double num) {
+ uint64_t raw = bit_cast<uint64_t>(num);
+ if (std::isnan(num) && ((raw & kDQuietNanMask) == 0)) {
+ return true;
+ }
+ return false;
+}
+
+inline bool IsSignallingNaN(float num) {
+ uint32_t raw = bit_cast<uint32_t>(num);
+ if (std::isnan(num) && ((raw & kSQuietNanMask) == 0)) {
+ return true;
+ }
+ return false;
+}
+
+inline bool IsSignallingNaN(float16 num) {
+ const uint16_t kFP16QuietNaNMask = 0x0200;
+ return (float16classify(num) == FP_NAN) && ((num & kFP16QuietNaNMask) == 0);
+}
+
+template <typename T>
+inline bool IsQuietNaN(T num) {
+ return std::isnan(num) && !IsSignallingNaN(num);
+}
+
+// Convert the NaN in 'num' to a quiet NaN.
+inline double ToQuietNaN(double num) {
+ DCHECK(std::isnan(num));
+ return bit_cast<double>(bit_cast<uint64_t>(num) | kDQuietNanMask);
+}
+
+inline float ToQuietNaN(float num) {
+ DCHECK(std::isnan(num));
+ return bit_cast<float>(bit_cast<uint32_t>(num) |
+ static_cast<uint32_t>(kSQuietNanMask));
+}
+
+// Fused multiply-add.
+inline double FusedMultiplyAdd(double op1, double op2, double a) {
+ return fma(op1, op2, a);
+}
+
+inline float FusedMultiplyAdd(float op1, float op2, float a) {
+ return fmaf(op1, op2, a);
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_ARM64_UTILS_ARM64_H_
diff --git a/src/codegen/assembler-arch.h b/src/codegen/assembler-arch.h
new file mode 100644
index 0000000..d56b372
--- /dev/null
+++ b/src/codegen/assembler-arch.h
@@ -0,0 +1,30 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_ASSEMBLER_ARCH_H_
+#define V8_CODEGEN_ASSEMBLER_ARCH_H_
+
+#include "src/codegen/assembler.h"
+
+#if V8_TARGET_ARCH_IA32
+#include "src/codegen/ia32/assembler-ia32.h"
+#elif V8_TARGET_ARCH_X64
+#include "src/codegen/x64/assembler-x64.h"
+#elif V8_TARGET_ARCH_ARM64
+#include "src/codegen/arm64/assembler-arm64.h"
+#elif V8_TARGET_ARCH_ARM
+#include "src/codegen/arm/assembler-arm.h"
+#elif V8_TARGET_ARCH_PPC || V8_TARGET_ARCH_PPC64
+#include "src/codegen/ppc/assembler-ppc.h"
+#elif V8_TARGET_ARCH_MIPS
+#include "src/codegen/mips/assembler-mips.h"
+#elif V8_TARGET_ARCH_MIPS64
+#include "src/codegen/mips64/assembler-mips64.h"
+#elif V8_TARGET_ARCH_S390
+#include "src/codegen/s390/assembler-s390.h"
+#else
+#error Unknown architecture.
+#endif
+
+#endif // V8_CODEGEN_ASSEMBLER_ARCH_H_
diff --git a/src/codegen/assembler-inl.h b/src/codegen/assembler-inl.h
new file mode 100644
index 0000000..8c81315
--- /dev/null
+++ b/src/codegen/assembler-inl.h
@@ -0,0 +1,30 @@
+// Copyright 2016 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_ASSEMBLER_INL_H_
+#define V8_CODEGEN_ASSEMBLER_INL_H_
+
+#include "src/codegen/assembler.h"
+
+#if V8_TARGET_ARCH_IA32
+#include "src/codegen/ia32/assembler-ia32-inl.h"
+#elif V8_TARGET_ARCH_X64
+#include "src/codegen/x64/assembler-x64-inl.h"
+#elif V8_TARGET_ARCH_ARM64
+#include "src/codegen/arm64/assembler-arm64-inl.h"
+#elif V8_TARGET_ARCH_ARM
+#include "src/codegen/arm/assembler-arm-inl.h"
+#elif V8_TARGET_ARCH_PPC || V8_TARGET_ARCH_PPC64
+#include "src/codegen/ppc/assembler-ppc-inl.h"
+#elif V8_TARGET_ARCH_MIPS
+#include "src/codegen/mips/assembler-mips-inl.h"
+#elif V8_TARGET_ARCH_MIPS64
+#include "src/codegen/mips64/assembler-mips64-inl.h"
+#elif V8_TARGET_ARCH_S390
+#include "src/codegen/s390/assembler-s390-inl.h"
+#else
+#error Unknown architecture.
+#endif
+
+#endif // V8_CODEGEN_ASSEMBLER_INL_H_
diff --git a/src/codegen/assembler.cc b/src/codegen/assembler.cc
new file mode 100644
index 0000000..f23dccb
--- /dev/null
+++ b/src/codegen/assembler.cc
@@ -0,0 +1,278 @@
+// Copyright (c) 1994-2006 Sun Microsystems Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// - Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// - Redistribution in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// - Neither the name of Sun Microsystems or the names of contributors may
+// be used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// The original source code covered by the above license above has been
+// modified significantly by Google Inc.
+// Copyright 2012 the V8 project authors. All rights reserved.
+
+#include "src/codegen/assembler.h"
+
+#include "src/codegen/assembler-inl.h"
+#include "src/codegen/string-constants.h"
+#include "src/deoptimizer/deoptimizer.h"
+#include "src/diagnostics/disassembler.h"
+#include "src/execution/isolate.h"
+#include "src/heap/heap-inl.h" // For MemoryAllocator. TODO(jkummerow): Drop.
+#include "src/snapshot/embedded/embedded-data.h"
+#include "src/snapshot/snapshot.h"
+#include "src/utils/ostreams.h"
+#include "src/utils/vector.h"
+
+namespace v8 {
+namespace internal {
+
+AssemblerOptions AssemblerOptions::Default(Isolate* isolate) {
+ AssemblerOptions options;
+ const bool serializer = isolate->serializer_enabled();
+ const bool generating_embedded_builtin =
+ isolate->IsGeneratingEmbeddedBuiltins();
+ options.record_reloc_info_for_serialization = serializer;
+ options.enable_root_array_delta_access =
+ !serializer && !generating_embedded_builtin;
+#ifdef USE_SIMULATOR
+ // Even though the simulator is enabled, we may still need to generate code
+ // that may need to run on both the simulator and real hardware. For example,
+ // if we are cross-compiling and embedding a script into the snapshot, the
+ // script will need to run on the host causing the embedded builtins to run in
+ // the simulator. While the final cross-compiled V8 will not have a simulator.
+
+ // So here we enable simulator specific code if not generating the snapshot or
+ // if we are but we are targetting the simulator *only*.
+ options.enable_simulator_code = !serializer || FLAG_target_is_simulator;
+#endif
+ options.inline_offheap_trampolines &= !generating_embedded_builtin;
+#if V8_TARGET_ARCH_X64 || V8_TARGET_ARCH_ARM64
+ const base::AddressRegion& code_range = isolate->heap()->code_range();
+ DCHECK_IMPLIES(code_range.begin() != kNullAddress, !code_range.is_empty());
+ options.code_range_start = code_range.begin();
+#endif
+ return options;
+}
+
+AssemblerOptions AssemblerOptions::DefaultForOffHeapTrampoline(
+ Isolate* isolate) {
+ AssemblerOptions options = AssemblerOptions::Default(isolate);
+ // Off-heap trampolines may not contain any metadata since their metadata
+ // offsets refer to the off-heap metadata area.
+ options.emit_code_comments = false;
+ return options;
+}
+
+namespace {
+
+class DefaultAssemblerBuffer : public AssemblerBuffer {
+ public:
+ explicit DefaultAssemblerBuffer(int size)
+ : buffer_(OwnedVector<uint8_t>::NewForOverwrite(size)) {
+#ifdef DEBUG
+ ZapCode(reinterpret_cast<Address>(buffer_.start()), size);
+#endif
+ }
+
+ byte* start() const override { return buffer_.start(); }
+
+ int size() const override { return static_cast<int>(buffer_.size()); }
+
+ std::unique_ptr<AssemblerBuffer> Grow(int new_size) override {
+ DCHECK_LT(size(), new_size);
+ return std::make_unique<DefaultAssemblerBuffer>(new_size);
+ }
+
+ private:
+ OwnedVector<uint8_t> buffer_;
+};
+
+class ExternalAssemblerBufferImpl : public AssemblerBuffer {
+ public:
+ ExternalAssemblerBufferImpl(byte* start, int size)
+ : start_(start), size_(size) {}
+
+ byte* start() const override { return start_; }
+
+ int size() const override { return size_; }
+
+ std::unique_ptr<AssemblerBuffer> Grow(int new_size) override {
+ FATAL("Cannot grow external assembler buffer");
+ }
+
+ private:
+ byte* const start_;
+ const int size_;
+};
+
+} // namespace
+
+std::unique_ptr<AssemblerBuffer> ExternalAssemblerBuffer(void* start,
+ int size) {
+ return std::make_unique<ExternalAssemblerBufferImpl>(
+ reinterpret_cast<byte*>(start), size);
+}
+
+std::unique_ptr<AssemblerBuffer> NewAssemblerBuffer(int size) {
+ return std::make_unique<DefaultAssemblerBuffer>(size);
+}
+
+// -----------------------------------------------------------------------------
+// Implementation of AssemblerBase
+
+AssemblerBase::AssemblerBase(const AssemblerOptions& options,
+ std::unique_ptr<AssemblerBuffer> buffer)
+ : buffer_(std::move(buffer)),
+ options_(options),
+ enabled_cpu_features_(0),
+ emit_debug_code_(FLAG_debug_code),
+ predictable_code_size_(false),
+ constant_pool_available_(false),
+ jump_optimization_info_(nullptr) {
+ if (!buffer_) buffer_ = NewAssemblerBuffer(kDefaultBufferSize);
+ buffer_start_ = buffer_->start();
+ pc_ = buffer_start_;
+}
+
+AssemblerBase::~AssemblerBase() = default;
+
+void AssemblerBase::Print(Isolate* isolate) {
+ StdoutStream os;
+ v8::internal::Disassembler::Decode(isolate, &os, buffer_start_, pc_);
+}
+
+// -----------------------------------------------------------------------------
+// Implementation of CpuFeatureScope
+
+#ifdef DEBUG
+CpuFeatureScope::CpuFeatureScope(AssemblerBase* assembler, CpuFeature f,
+ CheckPolicy check)
+ : assembler_(assembler) {
+ DCHECK_IMPLIES(check == kCheckSupported, CpuFeatures::IsSupported(f));
+ old_enabled_ = assembler_->enabled_cpu_features();
+ assembler_->EnableCpuFeature(f);
+}
+
+CpuFeatureScope::~CpuFeatureScope() {
+ assembler_->set_enabled_cpu_features(old_enabled_);
+}
+#endif
+
+bool CpuFeatures::initialized_ = false;
+unsigned CpuFeatures::supported_ = 0;
+unsigned CpuFeatures::icache_line_size_ = 0;
+unsigned CpuFeatures::dcache_line_size_ = 0;
+
+HeapObjectRequest::HeapObjectRequest(double heap_number, int offset)
+ : kind_(kHeapNumber), offset_(offset) {
+ value_.heap_number = heap_number;
+ DCHECK(!IsSmiDouble(value_.heap_number));
+}
+
+HeapObjectRequest::HeapObjectRequest(const StringConstantBase* string,
+ int offset)
+ : kind_(kStringConstant), offset_(offset) {
+ value_.string = string;
+ DCHECK_NOT_NULL(value_.string);
+}
+
+// Platform specific but identical code for all the platforms.
+
+void Assembler::RecordDeoptReason(DeoptimizeReason reason,
+ SourcePosition position, int id) {
+ EnsureSpace ensure_space(this);
+ RecordRelocInfo(RelocInfo::DEOPT_SCRIPT_OFFSET, position.ScriptOffset());
+ RecordRelocInfo(RelocInfo::DEOPT_INLINING_ID, position.InliningId());
+ RecordRelocInfo(RelocInfo::DEOPT_REASON, static_cast<int>(reason));
+ RecordRelocInfo(RelocInfo::DEOPT_ID, id);
+}
+
+void Assembler::DataAlign(int m) {
+ DCHECK(m >= 2 && base::bits::IsPowerOfTwo(m));
+ while ((pc_offset() & (m - 1)) != 0) {
+ // Pad with 0xcc (= int3 on ia32 and x64); the primary motivation is that
+ // the disassembler expects to find valid instructions, but this is also
+ // nice from a security point of view.
+ db(0xcc);
+ }
+}
+
+void AssemblerBase::RequestHeapObject(HeapObjectRequest request) {
+ request.set_offset(pc_offset());
+ heap_object_requests_.push_front(request);
+}
+
+int AssemblerBase::AddCodeTarget(Handle<Code> target) {
+ int current = static_cast<int>(code_targets_.size());
+ if (current > 0 && !target.is_null() &&
+ code_targets_.back().address() == target.address()) {
+ // Optimization if we keep jumping to the same code target.
+ return current - 1;
+ } else {
+ code_targets_.push_back(target);
+ return current;
+ }
+}
+
+Handle<Code> AssemblerBase::GetCodeTarget(intptr_t code_target_index) const {
+ DCHECK_LT(static_cast<size_t>(code_target_index), code_targets_.size());
+ return code_targets_[code_target_index];
+}
+
+AssemblerBase::EmbeddedObjectIndex AssemblerBase::AddEmbeddedObject(
+ Handle<HeapObject> object) {
+ EmbeddedObjectIndex current = embedded_objects_.size();
+ // Do not deduplicate invalid handles, they are to heap object requests.
+ if (!object.is_null()) {
+ auto entry = embedded_objects_map_.find(object);
+ if (entry != embedded_objects_map_.end()) {
+ return entry->second;
+ }
+ embedded_objects_map_[object] = current;
+ }
+ embedded_objects_.push_back(object);
+ return current;
+}
+
+Handle<HeapObject> AssemblerBase::GetEmbeddedObject(
+ EmbeddedObjectIndex index) const {
+ DCHECK_LT(index, embedded_objects_.size());
+ return embedded_objects_[index];
+}
+
+
+int Assembler::WriteCodeComments() {
+ CHECK_IMPLIES(code_comments_writer_.entry_count() > 0,
+ options().emit_code_comments);
+ if (code_comments_writer_.entry_count() == 0) return 0;
+ int offset = pc_offset();
+ code_comments_writer_.Emit(this);
+ int size = pc_offset() - offset;
+ DCHECK_EQ(size, code_comments_writer_.section_size());
+ return size;
+}
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/assembler.h b/src/codegen/assembler.h
new file mode 100644
index 0000000..626bd04
--- /dev/null
+++ b/src/codegen/assembler.h
@@ -0,0 +1,431 @@
+// Copyright (c) 1994-2006 Sun Microsystems Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// - Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// - Redistribution in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// - Neither the name of Sun Microsystems or the names of contributors may
+// be used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// The original source code covered by the above license above has been
+// modified significantly by Google Inc.
+// Copyright 2012 the V8 project authors. All rights reserved.
+
+#ifndef V8_CODEGEN_ASSEMBLER_H_
+#define V8_CODEGEN_ASSEMBLER_H_
+
+#include <forward_list>
+#include <memory>
+#include <unordered_map>
+
+#include "src/base/memory.h"
+#include "src/codegen/code-comments.h"
+#include "src/codegen/cpu-features.h"
+#include "src/codegen/external-reference.h"
+#include "src/codegen/reglist.h"
+#include "src/codegen/reloc-info.h"
+#include "src/common/globals.h"
+#include "src/deoptimizer/deoptimize-reason.h"
+#include "src/flags/flags.h"
+#include "src/handles/handles.h"
+#include "src/objects/objects.h"
+
+namespace v8 {
+
+// Forward declarations.
+class ApiFunction;
+
+namespace internal {
+
+using base::Memory;
+using base::ReadUnalignedValue;
+using base::WriteUnalignedValue;
+
+// Forward declarations.
+class EmbeddedData;
+class InstructionStream;
+class Isolate;
+class SCTableReference;
+class SourcePosition;
+class StatsCounter;
+class StringConstantBase;
+
+// -----------------------------------------------------------------------------
+// Optimization for far-jmp like instructions that can be replaced by shorter.
+
+class JumpOptimizationInfo {
+ public:
+ bool is_collecting() const { return stage_ == kCollection; }
+ bool is_optimizing() const { return stage_ == kOptimization; }
+ void set_optimizing() {
+ DCHECK(is_optimizable());
+ stage_ = kOptimization;
+ }
+
+ bool is_optimizable() const { return optimizable_; }
+ void set_optimizable() {
+ DCHECK(is_collecting());
+ optimizable_ = true;
+ }
+
+ // Used to verify the instruction sequence is always the same in two stages.
+ size_t hash_code() const { return hash_code_; }
+ void set_hash_code(size_t hash_code) { hash_code_ = hash_code; }
+
+ std::vector<uint32_t>& farjmp_bitmap() { return farjmp_bitmap_; }
+
+ private:
+ enum { kCollection, kOptimization } stage_ = kCollection;
+ bool optimizable_ = false;
+ std::vector<uint32_t> farjmp_bitmap_;
+ size_t hash_code_ = 0u;
+};
+
+class HeapObjectRequest {
+ public:
+ explicit HeapObjectRequest(double heap_number, int offset = -1);
+ explicit HeapObjectRequest(const StringConstantBase* string, int offset = -1);
+
+ enum Kind { kHeapNumber, kStringConstant };
+ Kind kind() const { return kind_; }
+
+ double heap_number() const {
+ DCHECK_EQ(kind(), kHeapNumber);
+ return value_.heap_number;
+ }
+
+ const StringConstantBase* string() const {
+ DCHECK_EQ(kind(), kStringConstant);
+ return value_.string;
+ }
+
+ // The code buffer offset at the time of the request.
+ int offset() const {
+ DCHECK_GE(offset_, 0);
+ return offset_;
+ }
+ void set_offset(int offset) {
+ DCHECK_LT(offset_, 0);
+ offset_ = offset;
+ DCHECK_GE(offset_, 0);
+ }
+
+ private:
+ Kind kind_;
+
+ union {
+ double heap_number;
+ const StringConstantBase* string;
+ } value_;
+
+ int offset_;
+};
+
+// -----------------------------------------------------------------------------
+// Platform independent assembler base class.
+
+enum class CodeObjectRequired { kNo, kYes };
+
+struct V8_EXPORT_PRIVATE AssemblerOptions {
+ // Recording reloc info for external references and off-heap targets is
+ // needed whenever code is serialized, e.g. into the snapshot or as a Wasm
+ // module. This flag allows this reloc info to be disabled for code that
+ // will not survive process destruction.
+ bool record_reloc_info_for_serialization = true;
+ // Recording reloc info can be disabled wholesale. This is needed when the
+ // assembler is used on existing code directly (e.g. JumpTableAssembler)
+ // without any buffer to hold reloc information.
+ bool disable_reloc_info_for_patching = false;
+ // Enables access to exrefs by computing a delta from the root array.
+ // Only valid if code will not survive the process.
+ bool enable_root_array_delta_access = false;
+ // Enables specific assembler sequences only used for the simulator.
+ bool enable_simulator_code = false;
+ // Enables use of isolate-independent constants, indirected through the
+ // root array.
+ // (macro assembler feature).
+ bool isolate_independent_code = false;
+ // Enables the use of isolate-independent builtins through an off-heap
+ // trampoline. (macro assembler feature).
+ bool inline_offheap_trampolines = true;
+ // On some platforms, all code is within a given range in the process,
+ // and the start of this range is configured here.
+ Address code_range_start = 0;
+ // Enable pc-relative calls/jumps on platforms that support it. When setting
+ // this flag, the code range must be small enough to fit all offsets into
+ // the instruction immediates.
+ bool use_pc_relative_calls_and_jumps = false;
+ // Enables the collection of information useful for the generation of unwind
+ // info. This is useful in some platform (Win64) where the unwind info depends
+ // on a function prologue/epilogue.
+ bool collect_win64_unwind_info = false;
+ // Whether to emit code comments.
+ bool emit_code_comments = FLAG_code_comments;
+
+ static AssemblerOptions Default(Isolate* isolate);
+ static AssemblerOptions DefaultForOffHeapTrampoline(Isolate* isolate);
+};
+
+class AssemblerBuffer {
+ public:
+ virtual ~AssemblerBuffer() = default;
+ virtual byte* start() const = 0;
+ virtual int size() const = 0;
+ // Return a grown copy of this buffer. The contained data is uninitialized.
+ // The data in {this} will still be read afterwards (until {this} is
+ // destructed), but not written.
+ virtual std::unique_ptr<AssemblerBuffer> Grow(int new_size)
+ V8_WARN_UNUSED_RESULT = 0;
+};
+
+// Allocate an AssemblerBuffer which uses an existing buffer. This buffer cannot
+// grow, so it must be large enough for all code emitted by the Assembler.
+V8_EXPORT_PRIVATE
+std::unique_ptr<AssemblerBuffer> ExternalAssemblerBuffer(void* buffer,
+ int size);
+
+// Allocate a new growable AssemblerBuffer with a given initial size.
+V8_EXPORT_PRIVATE
+std::unique_ptr<AssemblerBuffer> NewAssemblerBuffer(int size);
+
+class V8_EXPORT_PRIVATE AssemblerBase : public Malloced {
+ public:
+ AssemblerBase(const AssemblerOptions& options,
+ std::unique_ptr<AssemblerBuffer>);
+ virtual ~AssemblerBase();
+
+ const AssemblerOptions& options() const { return options_; }
+
+ bool emit_debug_code() const { return emit_debug_code_; }
+ void set_emit_debug_code(bool value) { emit_debug_code_ = value; }
+
+ bool predictable_code_size() const { return predictable_code_size_; }
+ void set_predictable_code_size(bool value) { predictable_code_size_ = value; }
+
+ uint64_t enabled_cpu_features() const { return enabled_cpu_features_; }
+ void set_enabled_cpu_features(uint64_t features) {
+ enabled_cpu_features_ = features;
+ }
+ // Features are usually enabled by CpuFeatureScope, which also asserts that
+ // the features are supported before they are enabled.
+ // IMPORTANT: IsEnabled() should only be used by DCHECKs. For real feature
+ // detection, use IsSupported().
+ bool IsEnabled(CpuFeature f) {
+ return (enabled_cpu_features_ & (static_cast<uint64_t>(1) << f)) != 0;
+ }
+ void EnableCpuFeature(CpuFeature f) {
+ enabled_cpu_features_ |= (static_cast<uint64_t>(1) << f);
+ }
+
+ bool is_constant_pool_available() const {
+ if (FLAG_enable_embedded_constant_pool) {
+ // We need to disable constant pool here for embeded builtins
+ // because the metadata section is not adjacent to instructions
+ return constant_pool_available_ && !options().isolate_independent_code;
+ } else {
+ // Embedded constant pool not supported on this architecture.
+ UNREACHABLE();
+ }
+ }
+
+ JumpOptimizationInfo* jump_optimization_info() {
+ return jump_optimization_info_;
+ }
+ void set_jump_optimization_info(JumpOptimizationInfo* jump_opt) {
+ jump_optimization_info_ = jump_opt;
+ }
+
+ void FinalizeJumpOptimizationInfo() {}
+
+ // Overwrite a host NaN with a quiet target NaN. Used by mksnapshot for
+ // cross-snapshotting.
+ static void QuietNaN(HeapObject nan) {}
+
+ int pc_offset() const { return static_cast<int>(pc_ - buffer_start_); }
+
+ int pc_offset_for_safepoint() {
+#if defined(V8_TARGET_ARCH_MIPS) || defined(V8_TARGET_ARCH_MIPS64)
+ // Mips needs it's own implementation to avoid trampoline's influence.
+ UNREACHABLE();
+#else
+ return pc_offset();
+#endif
+ }
+
+ byte* buffer_start() const { return buffer_->start(); }
+ int buffer_size() const { return buffer_->size(); }
+ int instruction_size() const { return pc_offset(); }
+
+ // This function is called when code generation is aborted, so that
+ // the assembler could clean up internal data structures.
+ virtual void AbortedCodeGeneration() {}
+
+ // Debugging
+ void Print(Isolate* isolate);
+
+ // Record an inline code comment that can be used by a disassembler.
+ // Use --code-comments to enable.
+ void RecordComment(const char* msg) {
+ if (options().emit_code_comments) {
+ code_comments_writer_.Add(pc_offset(), std::string(msg));
+ }
+ }
+
+ // The minimum buffer size. Should be at least two times the platform-specific
+ // {Assembler::kGap}.
+ static constexpr int kMinimalBufferSize = 128;
+
+ // The default buffer size used if we do not know the final size of the
+ // generated code.
+ static constexpr int kDefaultBufferSize = 4 * KB;
+
+ protected:
+ // Add 'target' to the {code_targets_} vector, if necessary, and return the
+ // offset at which it is stored.
+ int AddCodeTarget(Handle<Code> target);
+ Handle<Code> GetCodeTarget(intptr_t code_target_index) const;
+
+ // Add 'object' to the {embedded_objects_} vector and return the index at
+ // which it is stored.
+ using EmbeddedObjectIndex = size_t;
+ EmbeddedObjectIndex AddEmbeddedObject(Handle<HeapObject> object);
+ Handle<HeapObject> GetEmbeddedObject(EmbeddedObjectIndex index) const;
+
+ // The buffer into which code and relocation info are generated.
+ std::unique_ptr<AssemblerBuffer> buffer_;
+ // Cached from {buffer_->start()}, for faster access.
+ byte* buffer_start_;
+ std::forward_list<HeapObjectRequest> heap_object_requests_;
+ // The program counter, which points into the buffer above and moves forward.
+ // TODO(jkummerow): This should probably have type {Address}.
+ byte* pc_;
+
+ void set_constant_pool_available(bool available) {
+ if (FLAG_enable_embedded_constant_pool) {
+ constant_pool_available_ = available;
+ } else {
+ // Embedded constant pool not supported on this architecture.
+ UNREACHABLE();
+ }
+ }
+
+ // {RequestHeapObject} records the need for a future heap number allocation,
+ // code stub generation or string allocation. After code assembly, each
+ // platform's {Assembler::AllocateAndInstallRequestedHeapObjects} will
+ // allocate these objects and place them where they are expected (determined
+ // by the pc offset associated with each request).
+ void RequestHeapObject(HeapObjectRequest request);
+
+ bool ShouldRecordRelocInfo(RelocInfo::Mode rmode) const {
+ DCHECK(!RelocInfo::IsNone(rmode));
+ if (options().disable_reloc_info_for_patching) return false;
+ if (RelocInfo::IsOnlyForSerializer(rmode) &&
+ !options().record_reloc_info_for_serialization && !emit_debug_code()) {
+ return false;
+ }
+ return true;
+ }
+
+ CodeCommentsWriter code_comments_writer_;
+
+ private:
+ // Before we copy code into the code space, we sometimes cannot encode
+ // call/jump code targets as we normally would, as the difference between the
+ // instruction's location in the temporary buffer and the call target is not
+ // guaranteed to fit in the instruction's offset field. We keep track of the
+ // code handles we encounter in calls in this vector, and encode the index of
+ // the code handle in the vector instead.
+ std::vector<Handle<Code>> code_targets_;
+
+ // If an assembler needs a small number to refer to a heap object handle
+ // (for example, because there are only 32bit available on a 64bit arch), the
+ // assembler adds the object into this vector using AddEmbeddedObject, and
+ // may then refer to the heap object using the handle's index in this vector.
+ std::vector<Handle<HeapObject>> embedded_objects_;
+
+ // Embedded objects are deduplicated based on handle location. This is a
+ // compromise that is almost as effective as deduplication based on actual
+ // heap object addresses maintains GC safety.
+ std::unordered_map<Handle<HeapObject>, EmbeddedObjectIndex,
+ Handle<HeapObject>::hash, Handle<HeapObject>::equal_to>
+ embedded_objects_map_;
+
+ const AssemblerOptions options_;
+ uint64_t enabled_cpu_features_;
+ bool emit_debug_code_;
+ bool predictable_code_size_;
+
+ // Indicates whether the constant pool can be accessed, which is only possible
+ // if the pp register points to the current code object's constant pool.
+ bool constant_pool_available_;
+
+ JumpOptimizationInfo* jump_optimization_info_;
+
+ // Constant pool.
+ friend class FrameAndConstantPoolScope;
+ friend class ConstantPoolUnavailableScope;
+};
+
+// Avoids emitting debug code during the lifetime of this scope object.
+class DontEmitDebugCodeScope {
+ public:
+ explicit DontEmitDebugCodeScope(AssemblerBase* assembler)
+ : assembler_(assembler), old_value_(assembler->emit_debug_code()) {
+ assembler_->set_emit_debug_code(false);
+ }
+ ~DontEmitDebugCodeScope() { assembler_->set_emit_debug_code(old_value_); }
+
+ private:
+ AssemblerBase* assembler_;
+ bool old_value_;
+};
+
+// Enable a specified feature within a scope.
+class V8_EXPORT_PRIVATE CpuFeatureScope {
+ public:
+ enum CheckPolicy {
+ kCheckSupported,
+ kDontCheckSupported,
+ };
+
+#ifdef DEBUG
+ CpuFeatureScope(AssemblerBase* assembler, CpuFeature f,
+ CheckPolicy check = kCheckSupported);
+ ~CpuFeatureScope();
+
+ private:
+ AssemblerBase* assembler_;
+ uint64_t old_enabled_;
+#else
+ CpuFeatureScope(AssemblerBase* assembler, CpuFeature f,
+ CheckPolicy check = kCheckSupported) {}
+ ~CpuFeatureScope() { // NOLINT (modernize-use-equals-default)
+ // Define a destructor to avoid unused variable warnings.
+ }
+#endif
+};
+
+} // namespace internal
+} // namespace v8
+#endif // V8_CODEGEN_ASSEMBLER_H_
diff --git a/src/codegen/bailout-reason.cc b/src/codegen/bailout-reason.cc
new file mode 100644
index 0000000..f4573fb
--- /dev/null
+++ b/src/codegen/bailout-reason.cc
@@ -0,0 +1,36 @@
+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/bailout-reason.h"
+#include "src/base/logging.h"
+
+namespace v8 {
+namespace internal {
+
+#define ERROR_MESSAGES_TEXTS(C, T) T,
+
+const char* GetBailoutReason(BailoutReason reason) {
+ DCHECK_LT(reason, BailoutReason::kLastErrorMessage);
+ DCHECK_GE(reason, BailoutReason::kNoReason);
+ static const char* error_messages_[] = {
+ BAILOUT_MESSAGES_LIST(ERROR_MESSAGES_TEXTS)};
+ return error_messages_[static_cast<int>(reason)];
+}
+
+const char* GetAbortReason(AbortReason reason) {
+ DCHECK_LT(reason, AbortReason::kLastErrorMessage);
+ DCHECK_GE(reason, AbortReason::kNoReason);
+ static const char* error_messages_[] = {
+ ABORT_MESSAGES_LIST(ERROR_MESSAGES_TEXTS)};
+ return error_messages_[static_cast<int>(reason)];
+}
+
+bool IsValidAbortReason(int reason_id) {
+ return reason_id >= static_cast<int>(AbortReason::kNoReason) &&
+ reason_id < static_cast<int>(AbortReason::kLastErrorMessage);
+}
+
+#undef ERROR_MESSAGES_TEXTS
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/bailout-reason.h b/src/codegen/bailout-reason.h
new file mode 100644
index 0000000..267beb5
--- /dev/null
+++ b/src/codegen/bailout-reason.h
@@ -0,0 +1,123 @@
+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_BAILOUT_REASON_H_
+#define V8_CODEGEN_BAILOUT_REASON_H_
+
+#include <cstdint>
+
+namespace v8 {
+namespace internal {
+
+#define ABORT_MESSAGES_LIST(V) \
+ V(kNoReason, "no reason") \
+ \
+ V(k32BitValueInRegisterIsNotZeroExtended, \
+ "32 bit value in register is not zero-extended") \
+ V(kAPICallReturnedInvalidObject, "API call returned invalid object") \
+ V(kAllocatingNonEmptyPackedArray, "Allocating non-empty packed array") \
+ V(kAllocationIsNotDoubleAligned, "Allocation is not double aligned") \
+ V(kExpectedOptimizationSentinel, \
+ "Expected optimized code cell or optimization sentinel") \
+ V(kExpectedUndefinedOrCell, "Expected undefined or cell in register") \
+ V(kFunctionDataShouldBeBytecodeArrayOnInterpreterEntry, \
+ "The function_data field should be a BytecodeArray on interpreter entry") \
+ V(kInputStringTooLong, "Input string too long") \
+ V(kInvalidBytecode, "Invalid bytecode") \
+ V(kInvalidBytecodeAdvance, "Cannot advance current bytecode, ") \
+ V(kInvalidHandleScopeLevel, "Invalid HandleScope level") \
+ V(kInvalidJumpTableIndex, "Invalid jump table index") \
+ V(kInvalidParametersAndRegistersInGenerator, \
+ "invalid parameters and registers in generator") \
+ V(kMissingBytecodeArray, "Missing bytecode array from function") \
+ V(kObjectNotTagged, "The object is not tagged") \
+ V(kObjectTagged, "The object is tagged") \
+ V(kOffsetOutOfRange, "Offset out of range") \
+ V(kOperandIsASmi, "Operand is a smi") \
+ V(kOperandIsASmiAndNotABoundFunction, \
+ "Operand is a smi and not a bound function") \
+ V(kOperandIsASmiAndNotAConstructor, \
+ "Operand is a smi and not a constructor") \
+ V(kOperandIsASmiAndNotAFunction, "Operand is a smi and not a function") \
+ V(kOperandIsASmiAndNotAGeneratorObject, \
+ "Operand is a smi and not a generator object") \
+ V(kOperandIsNotABoundFunction, "Operand is not a bound function") \
+ V(kOperandIsNotAConstructor, "Operand is not a constructor") \
+ V(kOperandIsNotAFixedArray, "Operand is not a fixed array") \
+ V(kOperandIsNotAFunction, "Operand is not a function") \
+ V(kOperandIsNotAGeneratorObject, "Operand is not a generator object") \
+ V(kOperandIsNotASmi, "Operand is not a smi") \
+ V(kPromiseAlreadySettled, "Promise already settled") \
+ V(kReceivedInvalidReturnAddress, "Received invalid return address") \
+ V(kRegisterDidNotMatchExpectedRoot, "Register did not match expected root") \
+ V(kReturnAddressNotFoundInFrame, "Return address not found in frame") \
+ V(kShouldNotDirectlyEnterOsrFunction, \
+ "Should not directly enter OSR-compiled function") \
+ V(kStackAccessBelowStackPointer, "Stack access below stack pointer") \
+ V(kStackFrameTypesMustMatch, "Stack frame types must match") \
+ V(kUnalignedCellInWriteBarrier, "Unaligned cell in write barrier") \
+ V(kUnexpectedAdditionalPopValue, "Unexpected additional pop value") \
+ V(kUnexpectedElementsKindInArrayConstructor, \
+ "Unexpected ElementsKind in array constructor") \
+ V(kUnexpectedFPCRMode, "Unexpected FPCR mode.") \
+ V(kUnexpectedFunctionIDForInvokeIntrinsic, \
+ "Unexpected runtime function id for the InvokeIntrinsic bytecode") \
+ V(kUnexpectedInitialMapForArrayFunction, \
+ "Unexpected initial map for Array function") \
+ V(kUnexpectedLevelAfterReturnFromApiCall, \
+ "Unexpected level after return from api call") \
+ V(kUnexpectedNegativeValue, "Unexpected negative value") \
+ V(kUnexpectedReturnFromFrameDropper, \
+ "Unexpectedly returned from dropping frames") \
+ V(kUnexpectedReturnFromThrow, "Unexpectedly returned from a throw") \
+ V(kUnexpectedReturnFromWasmTrap, \
+ "Should not return after throwing a wasm trap") \
+ V(kUnexpectedStackPointer, "The stack pointer is not the expected value") \
+ V(kUnexpectedValue, "Unexpected value") \
+ V(kUnsupportedModuleOperation, "Unsupported module operation") \
+ V(kUnsupportedNonPrimitiveCompare, "Unsupported non-primitive compare") \
+ V(kWrongAddressOrValuePassedToRecordWrite, \
+ "Wrong address or value passed to RecordWrite") \
+ V(kWrongArgumentCountForInvokeIntrinsic, \
+ "Wrong number of arguments for intrinsic") \
+ V(kWrongFunctionCodeStart, "Wrong value in code start register passed") \
+ V(kWrongFunctionContext, "Wrong context passed to function") \
+ V(kUnexpectedThreadInWasmSet, "thread_in_wasm flag was already set") \
+ V(kUnexpectedThreadInWasmUnset, "thread_in_wasm flag was not set")
+
+#define BAILOUT_MESSAGES_LIST(V) \
+ V(kNoReason, "no reason") \
+ \
+ V(kBailedOutDueToDependencyChange, "Bailed out due to dependency change") \
+ V(kCodeGenerationFailed, "Code generation failed") \
+ V(kCyclicObjectStateDetectedInEscapeAnalysis, \
+ "Cyclic object state detected by escape analysis") \
+ V(kFunctionBeingDebugged, "Function is being debugged") \
+ V(kGraphBuildingFailed, "Optimized graph construction failed") \
+ V(kFunctionTooBig, "Function is too big to be optimized") \
+ V(kLiveEdit, "LiveEdit") \
+ V(kNativeFunctionLiteral, "Native function literal") \
+ V(kNotEnoughVirtualRegistersRegalloc, \
+ "Not enough virtual registers (regalloc)") \
+ V(kOptimizationDisabled, "Optimization disabled") \
+ V(kNeverOptimize, "Optimization is always disabled")
+
+#define ERROR_MESSAGES_CONSTANTS(C, T) C,
+enum class BailoutReason : uint8_t {
+ BAILOUT_MESSAGES_LIST(ERROR_MESSAGES_CONSTANTS) kLastErrorMessage
+};
+
+enum class AbortReason : uint8_t {
+ ABORT_MESSAGES_LIST(ERROR_MESSAGES_CONSTANTS) kLastErrorMessage
+};
+#undef ERROR_MESSAGES_CONSTANTS
+
+const char* GetBailoutReason(BailoutReason reason);
+const char* GetAbortReason(AbortReason reason);
+bool IsValidAbortReason(int reason_id);
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_BAILOUT_REASON_H_
diff --git a/src/codegen/callable.h b/src/codegen/callable.h
new file mode 100644
index 0000000..49ee707
--- /dev/null
+++ b/src/codegen/callable.h
@@ -0,0 +1,33 @@
+// Copyright 2017 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_CALLABLE_H_
+#define V8_CODEGEN_CALLABLE_H_
+
+#include "src/codegen/interface-descriptors.h"
+#include "src/utils/allocation.h"
+
+namespace v8 {
+namespace internal {
+
+class Code;
+
+// Associates a body of code with an interface descriptor.
+class Callable final {
+ public:
+ Callable(Handle<Code> code, CallInterfaceDescriptor descriptor)
+ : code_(code), descriptor_(descriptor) {}
+
+ Handle<Code> code() const { return code_; }
+ CallInterfaceDescriptor descriptor() const { return descriptor_; }
+
+ private:
+ const Handle<Code> code_;
+ const CallInterfaceDescriptor descriptor_;
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_CALLABLE_H_
diff --git a/src/codegen/code-comments.cc b/src/codegen/code-comments.cc
new file mode 100644
index 0000000..b0271a0
--- /dev/null
+++ b/src/codegen/code-comments.cc
@@ -0,0 +1,106 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <cstring>
+#include <iomanip>
+
+#include "src/codegen/assembler-inl.h"
+#include "src/codegen/code-comments.h"
+
+namespace v8 {
+namespace internal {
+
+namespace {
+static constexpr uint8_t kOffsetToFirstCommentEntry = kUInt32Size;
+static constexpr uint8_t kOffsetToPCOffset = 0;
+static constexpr uint8_t kOffsetToCommentSize = kOffsetToPCOffset + kUInt32Size;
+static constexpr uint8_t kOffsetToCommentString =
+ kOffsetToCommentSize + kUInt32Size;
+} // namespace
+
+uint32_t CodeCommentEntry::comment_length() const {
+ return static_cast<uint32_t>(comment.size() + 1);
+}
+
+uint32_t CodeCommentEntry::size() const {
+ return kOffsetToCommentString + comment_length();
+}
+
+CodeCommentsIterator::CodeCommentsIterator(Address code_comments_start,
+ uint32_t code_comments_size)
+ : code_comments_start_(code_comments_start),
+ code_comments_size_(code_comments_size),
+ current_entry_(code_comments_start + kOffsetToFirstCommentEntry) {
+ DCHECK_NE(kNullAddress, code_comments_start);
+ DCHECK_IMPLIES(code_comments_size,
+ code_comments_size ==
+ base::ReadUnalignedValue<uint32_t>(code_comments_start_));
+}
+
+uint32_t CodeCommentsIterator::size() const { return code_comments_size_; }
+
+const char* CodeCommentsIterator::GetComment() const {
+ const char* comment_string =
+ reinterpret_cast<const char*>(current_entry_ + kOffsetToCommentString);
+ CHECK_EQ(GetCommentSize(), strlen(comment_string) + 1);
+ return comment_string;
+}
+
+uint32_t CodeCommentsIterator::GetCommentSize() const {
+ return ReadUnalignedValue<uint32_t>(current_entry_ + kOffsetToCommentSize);
+}
+
+uint32_t CodeCommentsIterator::GetPCOffset() const {
+ return ReadUnalignedValue<uint32_t>(current_entry_ + kOffsetToPCOffset);
+}
+
+void CodeCommentsIterator::Next() {
+ current_entry_ += kOffsetToCommentString + GetCommentSize();
+}
+
+bool CodeCommentsIterator::HasCurrent() const {
+ return current_entry_ < code_comments_start_ + size();
+}
+
+void CodeCommentsWriter::Emit(Assembler* assm) {
+ assm->dd(section_size());
+ for (auto i = comments_.begin(); i != comments_.end(); ++i) {
+ assm->dd(i->pc_offset);
+ assm->dd(i->comment_length());
+ for (char c : i->comment) {
+ EnsureSpace ensure_space(assm);
+ assm->db(c);
+ }
+ assm->db('\0');
+ }
+}
+
+void CodeCommentsWriter::Add(uint32_t pc_offset, std::string comment) {
+ CodeCommentEntry entry = {pc_offset, std::move(comment)};
+ byte_count_ += entry.size();
+ comments_.push_back(std::move(entry));
+}
+
+size_t CodeCommentsWriter::entry_count() const { return comments_.size(); }
+uint32_t CodeCommentsWriter::section_size() const {
+ return kOffsetToFirstCommentEntry + static_cast<uint32_t>(byte_count_);
+}
+
+void PrintCodeCommentsSection(std::ostream& out, Address code_comments_start,
+ uint32_t code_comments_size) {
+ CodeCommentsIterator it(code_comments_start, code_comments_size);
+ out << "CodeComments (size = " << it.size() << ")\n";
+ if (it.HasCurrent()) {
+ out << std::setw(6) << "pc" << std::setw(6) << "len"
+ << " comment\n";
+ }
+ for (; it.HasCurrent(); it.Next()) {
+ out << std::hex << std::setw(6) << it.GetPCOffset() << std::dec
+ << std::setw(6) << it.GetCommentSize() << " (" << it.GetComment()
+ << ")\n";
+ }
+}
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/code-comments.h b/src/codegen/code-comments.h
new file mode 100644
index 0000000..5866296
--- /dev/null
+++ b/src/codegen/code-comments.h
@@ -0,0 +1,71 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_CODE_COMMENTS_H_
+#define V8_CODEGEN_CODE_COMMENTS_H_
+
+#include <ostream>
+#include <string>
+#include <vector>
+
+#include "include/v8-internal.h"
+#include "src/base/macros.h"
+
+namespace v8 {
+namespace internal {
+
+class Assembler;
+
+// Code comments section layout:
+// byte count content
+// ------------------------------------------------------------------------
+// 4 size as uint32_t (only for a check)
+// [Inline array of CodeCommentEntry in increasing pc_offset order]
+// ┌ 4 pc_offset of entry as uint32_t
+// ├ 4 length of the comment including terminating '\0'
+// â”” <variable length> characters of the comment including terminating '\0'
+
+struct CodeCommentEntry {
+ uint32_t pc_offset;
+ std::string comment;
+ uint32_t comment_length() const;
+ uint32_t size() const;
+};
+
+class CodeCommentsWriter {
+ public:
+ V8_EXPORT_PRIVATE void Add(uint32_t pc_offset, std::string comment);
+ void Emit(Assembler* assm);
+ size_t entry_count() const;
+ uint32_t section_size() const;
+
+ private:
+ uint32_t byte_count_ = 0;
+ std::vector<CodeCommentEntry> comments_;
+};
+
+class V8_EXPORT_PRIVATE CodeCommentsIterator {
+ public:
+ CodeCommentsIterator(Address code_comments_start,
+ uint32_t code_comments_size);
+ uint32_t size() const;
+ const char* GetComment() const;
+ uint32_t GetCommentSize() const;
+ uint32_t GetPCOffset() const;
+ void Next();
+ bool HasCurrent() const;
+
+ private:
+ Address code_comments_start_;
+ uint32_t code_comments_size_;
+ Address current_entry_;
+};
+
+void PrintCodeCommentsSection(std::ostream& out, Address code_comments_start,
+ uint32_t code_comments_size);
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_CODE_COMMENTS_H_
diff --git a/src/codegen/code-desc.cc b/src/codegen/code-desc.cc
new file mode 100644
index 0000000..ea2e471
--- /dev/null
+++ b/src/codegen/code-desc.cc
@@ -0,0 +1,73 @@
+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/code-desc.h"
+
+#include "src/codegen/assembler-inl.h"
+
+namespace v8 {
+namespace internal {
+
+// static
+void CodeDesc::Initialize(CodeDesc* desc, Assembler* assembler,
+ int safepoint_table_offset, int handler_table_offset,
+ int constant_pool_offset, int code_comments_offset,
+ int reloc_info_offset) {
+ desc->buffer = assembler->buffer_start();
+ desc->buffer_size = assembler->buffer_size();
+ desc->instr_size = assembler->instruction_size();
+
+ desc->code_comments_offset = code_comments_offset;
+ desc->code_comments_size = desc->instr_size - code_comments_offset;
+
+ desc->constant_pool_offset = constant_pool_offset;
+ desc->constant_pool_size = desc->code_comments_offset - constant_pool_offset;
+
+ desc->handler_table_offset = handler_table_offset;
+ desc->handler_table_size = desc->constant_pool_offset - handler_table_offset;
+
+ desc->safepoint_table_offset = safepoint_table_offset;
+ desc->safepoint_table_size =
+ desc->handler_table_offset - safepoint_table_offset;
+
+ desc->reloc_offset = reloc_info_offset;
+ desc->reloc_size = desc->buffer_size - reloc_info_offset;
+
+ desc->unwinding_info_size = 0;
+ desc->unwinding_info = nullptr;
+
+ desc->origin = assembler;
+
+ CodeDesc::Verify(desc);
+}
+
+#ifdef DEBUG
+// static
+void CodeDesc::Verify(const CodeDesc* desc) {
+ // Zero-size code objects upset the system.
+ DCHECK_GT(desc->instr_size, 0);
+ DCHECK_NOT_NULL(desc->buffer);
+
+ // Instruction area layout invariants.
+ DCHECK_GE(desc->safepoint_table_size, 0);
+ DCHECK_EQ(desc->safepoint_table_size + desc->safepoint_table_offset,
+ desc->handler_table_offset);
+ DCHECK_GE(desc->handler_table_size, 0);
+ DCHECK_EQ(desc->handler_table_size + desc->handler_table_offset,
+ desc->constant_pool_offset);
+ DCHECK_GE(desc->constant_pool_size, 0);
+ DCHECK_EQ(desc->constant_pool_size + desc->constant_pool_offset,
+ desc->code_comments_offset);
+ DCHECK_GE(desc->code_comments_size, 0);
+ DCHECK_EQ(desc->code_comments_size + desc->code_comments_offset,
+ desc->instr_size);
+
+ DCHECK_GE(desc->reloc_offset, 0);
+ DCHECK_GE(desc->reloc_size, 0);
+ DCHECK_GE(desc->unwinding_info_size, 0);
+}
+#endif
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/code-desc.h b/src/codegen/code-desc.h
new file mode 100644
index 0000000..e051bb4
--- /dev/null
+++ b/src/codegen/code-desc.h
@@ -0,0 +1,105 @@
+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_CODE_DESC_H_
+#define V8_CODEGEN_CODE_DESC_H_
+
+#include "src/common/globals.h"
+
+namespace v8 {
+namespace internal {
+
+// A CodeDesc describes a buffer holding instructions and relocation
+// information. The instructions start at the beginning of the buffer
+// and grow forward, the relocation information starts at the end of
+// the buffer and grows backward. Inlined metadata sections may exist
+// at the end of the instructions.
+//
+// |<--------------- buffer_size ----------------------------------->|
+// |<---------------- instr_size ------------->| |<-reloc_size->|
+// |--------------+----------------------------+------+--------------|
+// | instructions | data | free | reloc info |
+// +--------------+----------------------------+------+--------------+
+
+// TODO(jgruber): Add a single chokepoint for specifying the instruction area
+// layout (i.e. the order of inlined metadata fields).
+// TODO(jgruber): Systematically maintain inlined metadata offsets and sizes
+// to simplify CodeDesc initialization.
+
+class CodeDesc {
+ public:
+ static void Initialize(CodeDesc* desc, Assembler* assembler,
+ int safepoint_table_offset, int handler_table_offset,
+ int constant_pool_offset, int code_comments_offset,
+ int reloc_info_offset);
+
+#ifdef DEBUG
+ static void Verify(const CodeDesc* desc);
+#else
+ inline static void Verify(const CodeDesc* desc) {}
+#endif
+
+ public:
+ byte* buffer = nullptr;
+ int buffer_size = 0;
+
+ // The instruction area contains executable code plus inlined metadata.
+
+ int instr_size = 0;
+
+ // Metadata packed into the instructions area.
+
+ int safepoint_table_offset = 0;
+ int safepoint_table_size = 0;
+
+ int handler_table_offset = 0;
+ int handler_table_size = 0;
+
+ int constant_pool_offset = 0;
+ int constant_pool_size = 0;
+
+ int code_comments_offset = 0;
+ int code_comments_size = 0;
+
+ // TODO(jgruber,v8:11036): Remove these functions once CodeDesc fields have
+ // been made consistent with Code layout.
+ int body_size() const { return instr_size + unwinding_info_size; }
+ int instruction_size() const { return safepoint_table_offset; }
+ int metadata_size() const { return body_size() - instruction_size(); }
+ int safepoint_table_offset_relative() const {
+ return safepoint_table_offset - instruction_size();
+ }
+ int handler_table_offset_relative() const {
+ return handler_table_offset - instruction_size();
+ }
+ int constant_pool_offset_relative() const {
+ return constant_pool_offset - instruction_size();
+ }
+ int code_comments_offset_relative() const {
+ return code_comments_offset - instruction_size();
+ }
+
+ // Relocation info is located at the end of the buffer and not part of the
+ // instructions area.
+
+ int reloc_offset = 0;
+ int reloc_size = 0;
+
+ // Unwinding information.
+
+ byte* unwinding_info = nullptr;
+ int unwinding_info_size = 0;
+ int unwinding_info_offset_relative() const {
+ // TODO(jgruber,v8:11036): Remove this function once unwinding_info setup
+ // is more consistent with other metadata tables.
+ return code_comments_offset_relative() + code_comments_size;
+ }
+
+ Assembler* origin = nullptr;
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_CODE_DESC_H_
diff --git a/src/codegen/code-factory.cc b/src/codegen/code-factory.cc
new file mode 100644
index 0000000..006b6be
--- /dev/null
+++ b/src/codegen/code-factory.cc
@@ -0,0 +1,481 @@
+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/code-factory.h"
+
+#include "src/builtins/builtins-descriptors.h"
+#include "src/ic/ic.h"
+#include "src/init/bootstrapper.h"
+#include "src/objects/allocation-site-inl.h"
+#include "src/objects/objects-inl.h"
+
+namespace v8 {
+namespace internal {
+
+// static
+Handle<Code> CodeFactory::RuntimeCEntry(Isolate* isolate, int result_size) {
+ return CodeFactory::CEntry(isolate, result_size);
+}
+
+#define CENTRY_CODE(RS, SD, AM, BE) \
+ BUILTIN_CODE(isolate, CEntry_##RS##_##SD##_##AM##_##BE)
+
+// static
+Handle<Code> CodeFactory::CEntry(Isolate* isolate, int result_size,
+ SaveFPRegsMode save_doubles,
+ ArgvMode argv_mode, bool builtin_exit_frame) {
+ // Aliases for readability below.
+ const int rs = result_size;
+ const SaveFPRegsMode sd = save_doubles;
+ const ArgvMode am = argv_mode;
+ const bool be = builtin_exit_frame;
+
+ if (rs == 1 && sd == kDontSaveFPRegs && am == kArgvOnStack && !be) {
+ return CENTRY_CODE(Return1, DontSaveFPRegs, ArgvOnStack, NoBuiltinExit);
+ } else if (rs == 1 && sd == kDontSaveFPRegs && am == kArgvOnStack && be) {
+ return CENTRY_CODE(Return1, DontSaveFPRegs, ArgvOnStack, BuiltinExit);
+ } else if (rs == 1 && sd == kDontSaveFPRegs && am == kArgvInRegister && !be) {
+ return CENTRY_CODE(Return1, DontSaveFPRegs, ArgvInRegister, NoBuiltinExit);
+ } else if (rs == 1 && sd == kSaveFPRegs && am == kArgvOnStack && !be) {
+ return CENTRY_CODE(Return1, SaveFPRegs, ArgvOnStack, NoBuiltinExit);
+ } else if (rs == 1 && sd == kSaveFPRegs && am == kArgvOnStack && be) {
+ return CENTRY_CODE(Return1, SaveFPRegs, ArgvOnStack, BuiltinExit);
+ } else if (rs == 2 && sd == kDontSaveFPRegs && am == kArgvOnStack && !be) {
+ return CENTRY_CODE(Return2, DontSaveFPRegs, ArgvOnStack, NoBuiltinExit);
+ } else if (rs == 2 && sd == kDontSaveFPRegs && am == kArgvOnStack && be) {
+ return CENTRY_CODE(Return2, DontSaveFPRegs, ArgvOnStack, BuiltinExit);
+ } else if (rs == 2 && sd == kDontSaveFPRegs && am == kArgvInRegister && !be) {
+ return CENTRY_CODE(Return2, DontSaveFPRegs, ArgvInRegister, NoBuiltinExit);
+ } else if (rs == 2 && sd == kSaveFPRegs && am == kArgvOnStack && !be) {
+ return CENTRY_CODE(Return2, SaveFPRegs, ArgvOnStack, NoBuiltinExit);
+ } else if (rs == 2 && sd == kSaveFPRegs && am == kArgvOnStack && be) {
+ return CENTRY_CODE(Return2, SaveFPRegs, ArgvOnStack, BuiltinExit);
+ }
+
+ UNREACHABLE();
+}
+
+#undef CENTRY_CODE
+
+// static
+Callable CodeFactory::ApiGetter(Isolate* isolate) {
+ return Builtins::CallableFor(isolate, Builtins::kCallApiGetter);
+}
+
+// static
+Callable CodeFactory::CallApiCallback(Isolate* isolate) {
+ return Builtins::CallableFor(isolate, Builtins::kCallApiCallback);
+}
+
+// static
+Callable CodeFactory::LoadGlobalIC(Isolate* isolate, TypeofMode typeof_mode) {
+ return typeof_mode == NOT_INSIDE_TYPEOF
+ ? Builtins::CallableFor(isolate, Builtins::kLoadGlobalICTrampoline)
+ : Builtins::CallableFor(
+ isolate, Builtins::kLoadGlobalICInsideTypeofTrampoline);
+}
+
+// static
+Callable CodeFactory::LoadGlobalICInOptimizedCode(Isolate* isolate,
+ TypeofMode typeof_mode) {
+ return typeof_mode == NOT_INSIDE_TYPEOF
+ ? Builtins::CallableFor(isolate, Builtins::kLoadGlobalIC)
+ : Builtins::CallableFor(isolate,
+ Builtins::kLoadGlobalICInsideTypeof);
+}
+
+Callable CodeFactory::StoreOwnIC(Isolate* isolate) {
+ // TODO(ishell): Currently we use StoreOwnIC only for storing properties that
+ // already exist in the boilerplate therefore we can use StoreIC.
+ return Builtins::CallableFor(isolate, Builtins::kStoreICTrampoline);
+}
+
+Callable CodeFactory::StoreOwnICInOptimizedCode(Isolate* isolate) {
+ // TODO(ishell): Currently we use StoreOwnIC only for storing properties that
+ // already exist in the boilerplate therefore we can use StoreIC.
+ return Builtins::CallableFor(isolate, Builtins::kStoreIC);
+}
+
+Callable CodeFactory::KeyedStoreIC_SloppyArguments(Isolate* isolate,
+ KeyedAccessStoreMode mode) {
+ Builtins::Name builtin_index;
+ switch (mode) {
+ case STANDARD_STORE:
+ builtin_index = Builtins::kKeyedStoreIC_SloppyArguments_Standard;
+ break;
+ case STORE_AND_GROW_HANDLE_COW:
+ builtin_index =
+ Builtins::kKeyedStoreIC_SloppyArguments_GrowNoTransitionHandleCOW;
+ break;
+ case STORE_IGNORE_OUT_OF_BOUNDS:
+ builtin_index =
+ Builtins::kKeyedStoreIC_SloppyArguments_NoTransitionIgnoreOOB;
+ break;
+ case STORE_HANDLE_COW:
+ builtin_index =
+ Builtins::kKeyedStoreIC_SloppyArguments_NoTransitionHandleCOW;
+ break;
+ default:
+ UNREACHABLE();
+ }
+ return isolate->builtins()->CallableFor(isolate, builtin_index);
+}
+
+Callable CodeFactory::ElementsTransitionAndStore(Isolate* isolate,
+ KeyedAccessStoreMode mode) {
+ Builtins::Name builtin_index;
+ switch (mode) {
+ case STANDARD_STORE:
+ builtin_index = Builtins::kElementsTransitionAndStore_Standard;
+ break;
+ case STORE_AND_GROW_HANDLE_COW:
+ builtin_index =
+ Builtins::kElementsTransitionAndStore_GrowNoTransitionHandleCOW;
+ break;
+ case STORE_IGNORE_OUT_OF_BOUNDS:
+ builtin_index =
+ Builtins::kElementsTransitionAndStore_NoTransitionIgnoreOOB;
+ break;
+ case STORE_HANDLE_COW:
+ builtin_index =
+ Builtins::kElementsTransitionAndStore_NoTransitionHandleCOW;
+ break;
+ default:
+ UNREACHABLE();
+ }
+ return isolate->builtins()->CallableFor(isolate, builtin_index);
+}
+
+Callable CodeFactory::StoreFastElementIC(Isolate* isolate,
+ KeyedAccessStoreMode mode) {
+ Builtins::Name builtin_index;
+ switch (mode) {
+ case STANDARD_STORE:
+ builtin_index = Builtins::kStoreFastElementIC_Standard;
+ break;
+ case STORE_AND_GROW_HANDLE_COW:
+ builtin_index = Builtins::kStoreFastElementIC_GrowNoTransitionHandleCOW;
+ break;
+ case STORE_IGNORE_OUT_OF_BOUNDS:
+ builtin_index = Builtins::kStoreFastElementIC_NoTransitionIgnoreOOB;
+ break;
+ case STORE_HANDLE_COW:
+ builtin_index = Builtins::kStoreFastElementIC_NoTransitionHandleCOW;
+ break;
+ default:
+ UNREACHABLE();
+ }
+ return isolate->builtins()->CallableFor(isolate, builtin_index);
+}
+
+// static
+Callable CodeFactory::BinaryOperation(Isolate* isolate, Operation op) {
+ switch (op) {
+ case Operation::kShiftRight:
+ return Builtins::CallableFor(isolate, Builtins::kShiftRight);
+ case Operation::kShiftLeft:
+ return Builtins::CallableFor(isolate, Builtins::kShiftLeft);
+ case Operation::kShiftRightLogical:
+ return Builtins::CallableFor(isolate, Builtins::kShiftRightLogical);
+ case Operation::kAdd:
+ return Builtins::CallableFor(isolate, Builtins::kAdd);
+ case Operation::kSubtract:
+ return Builtins::CallableFor(isolate, Builtins::kSubtract);
+ case Operation::kMultiply:
+ return Builtins::CallableFor(isolate, Builtins::kMultiply);
+ case Operation::kDivide:
+ return Builtins::CallableFor(isolate, Builtins::kDivide);
+ case Operation::kModulus:
+ return Builtins::CallableFor(isolate, Builtins::kModulus);
+ case Operation::kBitwiseOr:
+ return Builtins::CallableFor(isolate, Builtins::kBitwiseOr);
+ case Operation::kBitwiseAnd:
+ return Builtins::CallableFor(isolate, Builtins::kBitwiseAnd);
+ case Operation::kBitwiseXor:
+ return Builtins::CallableFor(isolate, Builtins::kBitwiseXor);
+ default:
+ break;
+ }
+ UNREACHABLE();
+}
+
+// static
+Callable CodeFactory::NonPrimitiveToPrimitive(Isolate* isolate,
+ ToPrimitiveHint hint) {
+ return Callable(isolate->builtins()->NonPrimitiveToPrimitive(hint),
+ TypeConversionDescriptor{});
+}
+
+// static
+Callable CodeFactory::OrdinaryToPrimitive(Isolate* isolate,
+ OrdinaryToPrimitiveHint hint) {
+ return Callable(isolate->builtins()->OrdinaryToPrimitive(hint),
+ TypeConversionDescriptor{});
+}
+
+// static
+Callable CodeFactory::StringAdd(Isolate* isolate, StringAddFlags flags) {
+ switch (flags) {
+ case STRING_ADD_CHECK_NONE:
+ return Builtins::CallableFor(isolate, Builtins::kStringAdd_CheckNone);
+ case STRING_ADD_CONVERT_LEFT:
+ return Builtins::CallableFor(isolate, Builtins::kStringAddConvertLeft);
+ case STRING_ADD_CONVERT_RIGHT:
+ return Builtins::CallableFor(isolate, Builtins::kStringAddConvertRight);
+ }
+ UNREACHABLE();
+}
+
+// static
+Callable CodeFactory::ResumeGenerator(Isolate* isolate) {
+ return Builtins::CallableFor(isolate, Builtins::kResumeGeneratorTrampoline);
+}
+
+// static
+Callable CodeFactory::FrameDropperTrampoline(Isolate* isolate) {
+ return Builtins::CallableFor(isolate, Builtins::kFrameDropperTrampoline);
+}
+
+// static
+Callable CodeFactory::HandleDebuggerStatement(Isolate* isolate) {
+ return Builtins::CallableFor(isolate, Builtins::kHandleDebuggerStatement);
+}
+
+// static
+Callable CodeFactory::FastNewFunctionContext(Isolate* isolate,
+ ScopeType scope_type) {
+ switch (scope_type) {
+ case ScopeType::EVAL_SCOPE:
+ return Builtins::CallableFor(isolate,
+ Builtins::kFastNewFunctionContextEval);
+ case ScopeType::FUNCTION_SCOPE:
+ return Builtins::CallableFor(isolate,
+ Builtins::kFastNewFunctionContextFunction);
+ default:
+ UNREACHABLE();
+ }
+}
+
+// static
+Callable CodeFactory::ArgumentAdaptor(Isolate* isolate) {
+ return Builtins::CallableFor(isolate, Builtins::kArgumentsAdaptorTrampoline);
+}
+
+// static
+Callable CodeFactory::Call(Isolate* isolate, ConvertReceiverMode mode) {
+ return Callable(isolate->builtins()->Call(mode), CallTrampolineDescriptor{});
+}
+
+// static
+Callable CodeFactory::Call_WithFeedback(Isolate* isolate,
+ ConvertReceiverMode mode) {
+ switch (mode) {
+ case ConvertReceiverMode::kNullOrUndefined:
+ return Builtins::CallableFor(
+ isolate, Builtins::kCall_ReceiverIsNullOrUndefined_WithFeedback);
+ case ConvertReceiverMode::kNotNullOrUndefined:
+ return Builtins::CallableFor(
+ isolate, Builtins::kCall_ReceiverIsNotNullOrUndefined_WithFeedback);
+ case ConvertReceiverMode::kAny:
+ return Builtins::CallableFor(isolate,
+ Builtins::kCall_ReceiverIsAny_WithFeedback);
+ }
+ UNREACHABLE();
+}
+
+// static
+Callable CodeFactory::CallWithArrayLike(Isolate* isolate) {
+ return Builtins::CallableFor(isolate, Builtins::kCallWithArrayLike);
+}
+
+// static
+Callable CodeFactory::CallWithSpread(Isolate* isolate) {
+ return Builtins::CallableFor(isolate, Builtins::kCallWithSpread);
+}
+
+// static
+Callable CodeFactory::CallFunction(Isolate* isolate, ConvertReceiverMode mode) {
+ return Callable(isolate->builtins()->CallFunction(mode),
+ CallTrampolineDescriptor{});
+}
+
+// static
+Callable CodeFactory::CallVarargs(Isolate* isolate) {
+ return Builtins::CallableFor(isolate, Builtins::kCallVarargs);
+}
+
+// static
+Callable CodeFactory::CallForwardVarargs(Isolate* isolate) {
+ return Builtins::CallableFor(isolate, Builtins::kCallForwardVarargs);
+}
+
+// static
+Callable CodeFactory::CallFunctionForwardVarargs(Isolate* isolate) {
+ return Builtins::CallableFor(isolate, Builtins::kCallFunctionForwardVarargs);
+}
+
+// static
+Callable CodeFactory::Construct(Isolate* isolate) {
+ return Builtins::CallableFor(isolate, Builtins::kConstruct);
+}
+
+// static
+Callable CodeFactory::ConstructWithSpread(Isolate* isolate) {
+ return Builtins::CallableFor(isolate, Builtins::kConstructWithSpread);
+}
+
+// static
+Callable CodeFactory::ConstructFunction(Isolate* isolate) {
+ return Builtins::CallableFor(isolate, Builtins::kConstructFunction);
+}
+
+// static
+Callable CodeFactory::ConstructVarargs(Isolate* isolate) {
+ return Builtins::CallableFor(isolate, Builtins::kConstructVarargs);
+}
+
+// static
+Callable CodeFactory::ConstructForwardVarargs(Isolate* isolate) {
+ return Builtins::CallableFor(isolate, Builtins::kConstructForwardVarargs);
+}
+
+// static
+Callable CodeFactory::ConstructFunctionForwardVarargs(Isolate* isolate) {
+ return Builtins::CallableFor(isolate,
+ Builtins::kConstructFunctionForwardVarargs);
+}
+
+// static
+Callable CodeFactory::InterpreterPushArgsThenCall(
+ Isolate* isolate, ConvertReceiverMode receiver_mode,
+ InterpreterPushArgsMode mode) {
+ switch (mode) {
+ case InterpreterPushArgsMode::kArrayFunction:
+ // There is no special-case handling of calls to Array. They will all go
+ // through the kOther case below.
+ UNREACHABLE();
+ case InterpreterPushArgsMode::kWithFinalSpread:
+ return Builtins::CallableFor(
+ isolate, Builtins::kInterpreterPushArgsThenCallWithFinalSpread);
+ case InterpreterPushArgsMode::kOther:
+ switch (receiver_mode) {
+ case ConvertReceiverMode::kNullOrUndefined:
+ return Builtins::CallableFor(
+ isolate, Builtins::kInterpreterPushUndefinedAndArgsThenCall);
+ case ConvertReceiverMode::kNotNullOrUndefined:
+ case ConvertReceiverMode::kAny:
+ return Builtins::CallableFor(isolate,
+ Builtins::kInterpreterPushArgsThenCall);
+ }
+ }
+ UNREACHABLE();
+}
+
+// static
+Callable CodeFactory::InterpreterPushArgsThenConstruct(
+ Isolate* isolate, InterpreterPushArgsMode mode) {
+ switch (mode) {
+ case InterpreterPushArgsMode::kArrayFunction:
+ return Builtins::CallableFor(
+ isolate, Builtins::kInterpreterPushArgsThenConstructArrayFunction);
+ case InterpreterPushArgsMode::kWithFinalSpread:
+ return Builtins::CallableFor(
+ isolate, Builtins::kInterpreterPushArgsThenConstructWithFinalSpread);
+ case InterpreterPushArgsMode::kOther:
+ return Builtins::CallableFor(isolate,
+ Builtins::kInterpreterPushArgsThenConstruct);
+ }
+ UNREACHABLE();
+}
+
+// static
+Callable CodeFactory::InterpreterCEntry(Isolate* isolate, int result_size) {
+ // Note: If we ever use fpregs in the interpreter then we will need to
+ // save fpregs too.
+ Handle<Code> code = CodeFactory::CEntry(isolate, result_size, kDontSaveFPRegs,
+ kArgvInRegister);
+ if (result_size == 1) {
+ return Callable(code, InterpreterCEntry1Descriptor{});
+ } else {
+ DCHECK_EQ(result_size, 2);
+ return Callable(code, InterpreterCEntry2Descriptor{});
+ }
+}
+
+// static
+Callable CodeFactory::InterpreterOnStackReplacement(Isolate* isolate) {
+ return Builtins::CallableFor(isolate,
+ Builtins::kInterpreterOnStackReplacement);
+}
+
+// static
+Callable CodeFactory::ArrayNoArgumentConstructor(
+ Isolate* isolate, ElementsKind kind,
+ AllocationSiteOverrideMode override_mode) {
+#define CASE(kind_caps, kind_camel, mode_camel) \
+ case kind_caps: \
+ return Builtins::CallableFor( \
+ isolate, \
+ Builtins::kArrayNoArgumentConstructor_##kind_camel##_##mode_camel);
+ if (override_mode == DONT_OVERRIDE && AllocationSite::ShouldTrack(kind)) {
+ DCHECK(IsSmiElementsKind(kind));
+ switch (kind) {
+ CASE(PACKED_SMI_ELEMENTS, PackedSmi, DontOverride);
+ CASE(HOLEY_SMI_ELEMENTS, HoleySmi, DontOverride);
+ default:
+ UNREACHABLE();
+ }
+ } else {
+ DCHECK(override_mode == DISABLE_ALLOCATION_SITES ||
+ !AllocationSite::ShouldTrack(kind));
+ switch (kind) {
+ CASE(PACKED_SMI_ELEMENTS, PackedSmi, DisableAllocationSites);
+ CASE(HOLEY_SMI_ELEMENTS, HoleySmi, DisableAllocationSites);
+ CASE(PACKED_ELEMENTS, Packed, DisableAllocationSites);
+ CASE(HOLEY_ELEMENTS, Holey, DisableAllocationSites);
+ CASE(PACKED_DOUBLE_ELEMENTS, PackedDouble, DisableAllocationSites);
+ CASE(HOLEY_DOUBLE_ELEMENTS, HoleyDouble, DisableAllocationSites);
+ default:
+ UNREACHABLE();
+ }
+ }
+#undef CASE
+}
+
+// static
+Callable CodeFactory::ArraySingleArgumentConstructor(
+ Isolate* isolate, ElementsKind kind,
+ AllocationSiteOverrideMode override_mode) {
+#define CASE(kind_caps, kind_camel, mode_camel) \
+ case kind_caps: \
+ return Builtins::CallableFor( \
+ isolate, \
+ Builtins::kArraySingleArgumentConstructor_##kind_camel##_##mode_camel)
+ if (override_mode == DONT_OVERRIDE && AllocationSite::ShouldTrack(kind)) {
+ DCHECK(IsSmiElementsKind(kind));
+ switch (kind) {
+ CASE(PACKED_SMI_ELEMENTS, PackedSmi, DontOverride);
+ CASE(HOLEY_SMI_ELEMENTS, HoleySmi, DontOverride);
+ default:
+ UNREACHABLE();
+ }
+ } else {
+ DCHECK(override_mode == DISABLE_ALLOCATION_SITES ||
+ !AllocationSite::ShouldTrack(kind));
+ switch (kind) {
+ CASE(PACKED_SMI_ELEMENTS, PackedSmi, DisableAllocationSites);
+ CASE(HOLEY_SMI_ELEMENTS, HoleySmi, DisableAllocationSites);
+ CASE(PACKED_ELEMENTS, Packed, DisableAllocationSites);
+ CASE(HOLEY_ELEMENTS, Holey, DisableAllocationSites);
+ CASE(PACKED_DOUBLE_ELEMENTS, PackedDouble, DisableAllocationSites);
+ CASE(HOLEY_DOUBLE_ELEMENTS, HoleyDouble, DisableAllocationSites);
+ default:
+ UNREACHABLE();
+ }
+ }
+#undef CASE
+}
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/code-factory.h b/src/codegen/code-factory.h
new file mode 100644
index 0000000..02fc7e4
--- /dev/null
+++ b/src/codegen/code-factory.h
@@ -0,0 +1,108 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_CODE_FACTORY_H_
+#define V8_CODEGEN_CODE_FACTORY_H_
+
+#include "src/codegen/callable.h"
+#include "src/codegen/interface-descriptors.h"
+#include "src/common/globals.h"
+#include "src/objects/type-hints.h"
+#include "src/utils/allocation.h"
+
+namespace v8 {
+namespace internal {
+
+// For ArrayNoArgumentConstructor and ArraySingleArgumentConstructor.
+enum AllocationSiteOverrideMode {
+ DONT_OVERRIDE,
+ DISABLE_ALLOCATION_SITES,
+};
+
+class V8_EXPORT_PRIVATE CodeFactory final {
+ public:
+ // CEntry has var-args semantics (all the arguments are passed on the
+ // stack and the arguments count is passed via register) which currently
+ // can't be expressed in CallInterfaceDescriptor. Therefore only the code
+ // is exported here.
+ static Handle<Code> RuntimeCEntry(Isolate* isolate, int result_size = 1);
+
+ static Handle<Code> CEntry(Isolate* isolate, int result_size = 1,
+ SaveFPRegsMode save_doubles = kDontSaveFPRegs,
+ ArgvMode argv_mode = kArgvOnStack,
+ bool builtin_exit_frame = false);
+
+ // Initial states for ICs.
+ static Callable LoadGlobalIC(Isolate* isolate, TypeofMode typeof_mode);
+ static Callable LoadGlobalICInOptimizedCode(Isolate* isolate,
+ TypeofMode typeof_mode);
+ static Callable StoreOwnIC(Isolate* isolate);
+ static Callable StoreOwnICInOptimizedCode(Isolate* isolate);
+
+ static Callable KeyedStoreIC_SloppyArguments(Isolate* isolate,
+ KeyedAccessStoreMode mode);
+ static Callable ElementsTransitionAndStore(Isolate* isolate,
+ KeyedAccessStoreMode mode);
+ static Callable StoreFastElementIC(Isolate* isolate,
+ KeyedAccessStoreMode mode);
+
+ static Callable ResumeGenerator(Isolate* isolate);
+
+ static Callable FrameDropperTrampoline(Isolate* isolate);
+ static Callable HandleDebuggerStatement(Isolate* isolate);
+
+ static Callable BinaryOperation(Isolate* isolate, Operation op);
+
+ static Callable ApiGetter(Isolate* isolate);
+ static Callable CallApiCallback(Isolate* isolate);
+
+ static Callable NonPrimitiveToPrimitive(
+ Isolate* isolate, ToPrimitiveHint hint = ToPrimitiveHint::kDefault);
+ static Callable OrdinaryToPrimitive(Isolate* isolate,
+ OrdinaryToPrimitiveHint hint);
+
+ static Callable StringAdd(Isolate* isolate,
+ StringAddFlags flags = STRING_ADD_CHECK_NONE);
+
+ static Callable FastNewFunctionContext(Isolate* isolate,
+ ScopeType scope_type);
+
+ static Callable ArgumentAdaptor(Isolate* isolate);
+ static Callable Call(Isolate* isolate,
+ ConvertReceiverMode mode = ConvertReceiverMode::kAny);
+ static Callable Call_WithFeedback(Isolate* isolate, ConvertReceiverMode mode);
+ static Callable CallWithArrayLike(Isolate* isolate);
+ static Callable CallWithSpread(Isolate* isolate);
+ static Callable CallFunction(
+ Isolate* isolate, ConvertReceiverMode mode = ConvertReceiverMode::kAny);
+ static Callable CallVarargs(Isolate* isolate);
+ static Callable CallForwardVarargs(Isolate* isolate);
+ static Callable CallFunctionForwardVarargs(Isolate* isolate);
+ static Callable Construct(Isolate* isolate);
+ static Callable ConstructWithSpread(Isolate* isolate);
+ static Callable ConstructFunction(Isolate* isolate);
+ static Callable ConstructVarargs(Isolate* isolate);
+ static Callable ConstructForwardVarargs(Isolate* isolate);
+ static Callable ConstructFunctionForwardVarargs(Isolate* isolate);
+
+ static Callable InterpreterPushArgsThenCall(Isolate* isolate,
+ ConvertReceiverMode receiver_mode,
+ InterpreterPushArgsMode mode);
+ static Callable InterpreterPushArgsThenConstruct(
+ Isolate* isolate, InterpreterPushArgsMode mode);
+ static Callable InterpreterCEntry(Isolate* isolate, int result_size = 1);
+ static Callable InterpreterOnStackReplacement(Isolate* isolate);
+
+ static Callable ArrayNoArgumentConstructor(
+ Isolate* isolate, ElementsKind kind,
+ AllocationSiteOverrideMode override_mode);
+ static Callable ArraySingleArgumentConstructor(
+ Isolate* isolate, ElementsKind kind,
+ AllocationSiteOverrideMode override_mode);
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_CODE_FACTORY_H_
diff --git a/src/codegen/code-reference.cc b/src/codegen/code-reference.cc
new file mode 100644
index 0000000..63c8d37
--- /dev/null
+++ b/src/codegen/code-reference.cc
@@ -0,0 +1,107 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/code-reference.h"
+
+#include "src/codegen/code-desc.h"
+#include "src/common/globals.h"
+#include "src/handles/handles-inl.h"
+#include "src/objects/objects-inl.h"
+#include "src/wasm/wasm-code-manager.h"
+
+namespace v8 {
+namespace internal {
+
+namespace {
+struct JSOps {
+ Handle<Code> code;
+
+ Address constant_pool() const { return code->constant_pool(); }
+ Address instruction_start() const { return code->InstructionStart(); }
+ Address instruction_end() const { return code->InstructionEnd(); }
+ int instruction_size() const { return code->InstructionSize(); }
+ const byte* relocation_start() const { return code->relocation_start(); }
+ const byte* relocation_end() const { return code->relocation_end(); }
+ int relocation_size() const { return code->relocation_size(); }
+ Address code_comments() const { return code->code_comments(); }
+ int code_comments_size() const { return code->code_comments_size(); }
+};
+
+struct WasmOps {
+ const wasm::WasmCode* code;
+
+ Address constant_pool() const { return code->constant_pool(); }
+ Address instruction_start() const {
+ return reinterpret_cast<Address>(code->instructions().begin());
+ }
+ Address instruction_end() const {
+ return reinterpret_cast<Address>(code->instructions().begin() +
+ code->instructions().size());
+ }
+ int instruction_size() const { return code->instructions().length(); }
+ const byte* relocation_start() const { return code->reloc_info().begin(); }
+ const byte* relocation_end() const {
+ return code->reloc_info().begin() + code->reloc_info().length();
+ }
+ int relocation_size() const { return code->reloc_info().length(); }
+ Address code_comments() const { return code->code_comments(); }
+ int code_comments_size() const { return code->code_comments_size(); }
+};
+
+struct CodeDescOps {
+ const CodeDesc* code_desc;
+
+ Address constant_pool() const {
+ return instruction_start() + code_desc->constant_pool_offset;
+ }
+ Address instruction_start() const {
+ return reinterpret_cast<Address>(code_desc->buffer);
+ }
+ Address instruction_end() const {
+ return instruction_start() + code_desc->instr_size;
+ }
+ int instruction_size() const { return code_desc->instr_size; }
+ const byte* relocation_start() const {
+ return code_desc->buffer + code_desc->reloc_offset;
+ }
+ const byte* relocation_end() const {
+ return code_desc->buffer + code_desc->buffer_size;
+ }
+ int relocation_size() const { return code_desc->reloc_size; }
+ Address code_comments() const {
+ return instruction_start() + code_desc->code_comments_offset;
+ }
+ int code_comments_size() const { return code_desc->code_comments_size; }
+};
+} // namespace
+
+#define DISPATCH(ret, method) \
+ ret CodeReference::method() const { \
+ DCHECK(!is_null()); \
+ switch (kind_) { \
+ case JS: \
+ return JSOps{js_code_}.method(); \
+ case WASM: \
+ return WasmOps{wasm_code_}.method(); \
+ case CODE_DESC: \
+ return CodeDescOps{code_desc_}.method(); \
+ default: \
+ UNREACHABLE(); \
+ } \
+ }
+
+DISPATCH(Address, constant_pool)
+DISPATCH(Address, instruction_start)
+DISPATCH(Address, instruction_end)
+DISPATCH(int, instruction_size)
+DISPATCH(const byte*, relocation_start)
+DISPATCH(const byte*, relocation_end)
+DISPATCH(int, relocation_size)
+DISPATCH(Address, code_comments)
+DISPATCH(int, code_comments_size)
+
+#undef DISPATCH
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/code-reference.h b/src/codegen/code-reference.h
new file mode 100644
index 0000000..8ff3581
--- /dev/null
+++ b/src/codegen/code-reference.h
@@ -0,0 +1,70 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_CODE_REFERENCE_H_
+#define V8_CODEGEN_CODE_REFERENCE_H_
+
+#include "src/handles/handles.h"
+#include "src/objects/code.h"
+
+namespace v8 {
+namespace internal {
+
+class Code;
+class CodeDesc;
+
+namespace wasm {
+class WasmCode;
+} // namespace wasm
+
+class CodeReference {
+ public:
+ CodeReference() : kind_(NONE), null_(nullptr) {}
+ explicit CodeReference(const wasm::WasmCode* wasm_code)
+ : kind_(WASM), wasm_code_(wasm_code) {}
+ explicit CodeReference(const CodeDesc* code_desc)
+ : kind_(CODE_DESC), code_desc_(code_desc) {}
+ explicit CodeReference(Handle<Code> js_code) : kind_(JS), js_code_(js_code) {}
+
+ Address constant_pool() const;
+ Address instruction_start() const;
+ Address instruction_end() const;
+ int instruction_size() const;
+ const byte* relocation_start() const;
+ const byte* relocation_end() const;
+ int relocation_size() const;
+ Address code_comments() const;
+ int code_comments_size() const;
+
+ bool is_null() const { return kind_ == NONE; }
+ bool is_js() const { return kind_ == JS; }
+ bool is_wasm_code() const { return kind_ == WASM; }
+
+ Handle<Code> as_js_code() const {
+ DCHECK_EQ(JS, kind_);
+ return js_code_;
+ }
+
+ const wasm::WasmCode* as_wasm_code() const {
+ DCHECK_EQ(WASM, kind_);
+ return wasm_code_;
+ }
+
+ private:
+ enum { NONE, JS, WASM, CODE_DESC } kind_;
+ union {
+ std::nullptr_t null_;
+ const wasm::WasmCode* wasm_code_;
+ const CodeDesc* code_desc_;
+ Handle<Code> js_code_;
+ };
+
+ DISALLOW_NEW_AND_DELETE()
+};
+ASSERT_TRIVIALLY_COPYABLE(CodeReference);
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_CODE_REFERENCE_H_
diff --git a/src/codegen/code-stub-assembler.cc b/src/codegen/code-stub-assembler.cc
new file mode 100644
index 0000000..ca340c6
--- /dev/null
+++ b/src/codegen/code-stub-assembler.cc
@@ -0,0 +1,13534 @@
+// Copyright 2016 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/code-stub-assembler.h"
+
+#include "include/v8-internal.h"
+#include "src/base/macros.h"
+#include "src/codegen/code-factory.h"
+#include "src/codegen/tnode.h"
+#include "src/common/globals.h"
+#include "src/execution/frames-inl.h"
+#include "src/execution/frames.h"
+#include "src/execution/protectors.h"
+#include "src/heap/heap-inl.h" // For MemoryChunk. TODO(jkummerow): Drop.
+#include "src/heap/memory-chunk.h"
+#include "src/logging/counters.h"
+#include "src/objects/api-callbacks.h"
+#include "src/objects/cell.h"
+#include "src/objects/descriptor-array.h"
+#include "src/objects/function-kind.h"
+#include "src/objects/heap-number.h"
+#include "src/objects/js-generator.h"
+#include "src/objects/oddball.h"
+#include "src/objects/ordered-hash-table-inl.h"
+#include "src/objects/property-cell.h"
+#include "src/roots/roots.h"
+#include "src/wasm/wasm-objects.h"
+
+namespace v8 {
+namespace internal {
+
+using compiler::Node;
+
+CodeStubAssembler::CodeStubAssembler(compiler::CodeAssemblerState* state)
+ : compiler::CodeAssembler(state),
+ TorqueGeneratedExportedMacrosAssembler(state) {
+ if (DEBUG_BOOL && FLAG_csa_trap_on_node != nullptr) {
+ HandleBreakOnNode();
+ }
+}
+
+void CodeStubAssembler::HandleBreakOnNode() {
+ // FLAG_csa_trap_on_node should be in a form "STUB,NODE" where STUB is a
+ // string specifying the name of a stub and NODE is number specifying node id.
+ const char* name = state()->name();
+ size_t name_length = strlen(name);
+ if (strncmp(FLAG_csa_trap_on_node, name, name_length) != 0) {
+ // Different name.
+ return;
+ }
+ size_t option_length = strlen(FLAG_csa_trap_on_node);
+ if (option_length < name_length + 2 ||
+ FLAG_csa_trap_on_node[name_length] != ',') {
+ // Option is too short.
+ return;
+ }
+ const char* start = &FLAG_csa_trap_on_node[name_length + 1];
+ char* end;
+ int node_id = static_cast<int>(strtol(start, &end, 10));
+ if (start == end) {
+ // Bad node id.
+ return;
+ }
+ BreakOnNode(node_id);
+}
+
+void CodeStubAssembler::Assert(const BranchGenerator& branch,
+ const char* message, const char* file, int line,
+ std::initializer_list<ExtraNode> extra_nodes) {
+#if defined(DEBUG)
+ if (FLAG_debug_code) {
+ Check(branch, message, file, line, extra_nodes);
+ }
+#endif
+}
+
+void CodeStubAssembler::Assert(const NodeGenerator<BoolT>& condition_body,
+ const char* message, const char* file, int line,
+ std::initializer_list<ExtraNode> extra_nodes) {
+#if defined(DEBUG)
+ if (FLAG_debug_code) {
+ Check(condition_body, message, file, line, extra_nodes);
+ }
+#endif
+}
+
+void CodeStubAssembler::Assert(TNode<Word32T> condition_node,
+ const char* message, const char* file, int line,
+ std::initializer_list<ExtraNode> extra_nodes) {
+#if defined(DEBUG)
+ if (FLAG_debug_code) {
+ Check(condition_node, message, file, line, extra_nodes);
+ }
+#endif
+}
+
+void CodeStubAssembler::Check(const BranchGenerator& branch,
+ const char* message, const char* file, int line,
+ std::initializer_list<ExtraNode> extra_nodes) {
+ Label ok(this);
+ Label not_ok(this, Label::kDeferred);
+ if (message != nullptr && FLAG_code_comments) {
+ Comment("[ Assert: ", message);
+ } else {
+ Comment("[ Assert");
+ }
+ branch(&ok, ¬_ok);
+
+ BIND(¬_ok);
+ std::vector<FileAndLine> file_and_line;
+ if (file != nullptr) {
+ file_and_line.push_back({file, line});
+ }
+ FailAssert(message, file_and_line, extra_nodes);
+
+ BIND(&ok);
+ Comment("] Assert");
+}
+
+void CodeStubAssembler::Check(const NodeGenerator<BoolT>& condition_body,
+ const char* message, const char* file, int line,
+ std::initializer_list<ExtraNode> extra_nodes) {
+ BranchGenerator branch = [=](Label* ok, Label* not_ok) {
+ TNode<BoolT> condition = condition_body();
+ Branch(condition, ok, not_ok);
+ };
+
+ Check(branch, message, file, line, extra_nodes);
+}
+
+void CodeStubAssembler::Check(TNode<Word32T> condition_node,
+ const char* message, const char* file, int line,
+ std::initializer_list<ExtraNode> extra_nodes) {
+ BranchGenerator branch = [=](Label* ok, Label* not_ok) {
+ Branch(condition_node, ok, not_ok);
+ };
+
+ Check(branch, message, file, line, extra_nodes);
+}
+
+void CodeStubAssembler::IncrementCallCount(
+ TNode<FeedbackVector> feedback_vector, TNode<UintPtrT> slot_id) {
+ Comment("increment call count");
+ TNode<Smi> call_count =
+ CAST(LoadFeedbackVectorSlot(feedback_vector, slot_id, kTaggedSize));
+ // The lowest {FeedbackNexus::CallCountField::kShift} bits of the call
+ // count are used as flags. To increment the call count by 1 we hence
+ // have to increment by 1 << {FeedbackNexus::CallCountField::kShift}.
+ TNode<Smi> new_count = SmiAdd(
+ call_count, SmiConstant(1 << FeedbackNexus::CallCountField::kShift));
+ // Count is Smi, so we don't need a write barrier.
+ StoreFeedbackVectorSlot(feedback_vector, slot_id, new_count,
+ SKIP_WRITE_BARRIER, kTaggedSize);
+}
+
+void CodeStubAssembler::FastCheck(TNode<BoolT> condition) {
+ Label ok(this), not_ok(this, Label::kDeferred);
+ Branch(condition, &ok, ¬_ok);
+ BIND(¬_ok);
+ Unreachable();
+ BIND(&ok);
+}
+
+void CodeStubAssembler::FailAssert(
+ const char* message, const std::vector<FileAndLine>& files_and_lines,
+ std::initializer_list<ExtraNode> extra_nodes) {
+ DCHECK_NOT_NULL(message);
+ EmbeddedVector<char, 1024> chars;
+ std::stringstream stream;
+ for (auto it = files_and_lines.rbegin(); it != files_and_lines.rend(); ++it) {
+ if (it->first != nullptr) {
+ stream << " [" << it->first << ":" << it->second << "]";
+#ifndef DEBUG
+ // To limit the size of these strings in release builds, we include only
+ // the innermost macro's file name and line number.
+ break;
+#endif
+ }
+ }
+ std::string files_and_lines_text = stream.str();
+ if (files_and_lines_text.size() != 0) {
+ SNPrintF(chars, "%s%s", message, files_and_lines_text.c_str());
+ message = chars.begin();
+ }
+ TNode<String> message_node = StringConstant(message);
+
+#ifdef DEBUG
+ // Only print the extra nodes in debug builds.
+ for (auto& node : extra_nodes) {
+ CallRuntime(Runtime::kPrintWithNameForAssert, SmiConstant(0),
+ StringConstant(node.second), node.first);
+ }
+#endif
+
+ AbortCSAAssert(message_node);
+ Unreachable();
+}
+
+TNode<Int32T> CodeStubAssembler::SelectInt32Constant(TNode<BoolT> condition,
+ int true_value,
+ int false_value) {
+ return SelectConstant<Int32T>(condition, Int32Constant(true_value),
+ Int32Constant(false_value));
+}
+
+TNode<IntPtrT> CodeStubAssembler::SelectIntPtrConstant(TNode<BoolT> condition,
+ int true_value,
+ int false_value) {
+ return SelectConstant<IntPtrT>(condition, IntPtrConstant(true_value),
+ IntPtrConstant(false_value));
+}
+
+TNode<Oddball> CodeStubAssembler::SelectBooleanConstant(
+ TNode<BoolT> condition) {
+ return SelectConstant<Oddball>(condition, TrueConstant(), FalseConstant());
+}
+
+TNode<Smi> CodeStubAssembler::SelectSmiConstant(TNode<BoolT> condition,
+ Smi true_value,
+ Smi false_value) {
+ return SelectConstant<Smi>(condition, SmiConstant(true_value),
+ SmiConstant(false_value));
+}
+
+TNode<Smi> CodeStubAssembler::NoContextConstant() {
+ return SmiConstant(Context::kNoContext);
+}
+
+#define HEAP_CONSTANT_ACCESSOR(rootIndexName, rootAccessorName, name) \
+ TNode<std::remove_pointer<std::remove_reference<decltype( \
+ std::declval<Heap>().rootAccessorName())>::type>::type> \
+ CodeStubAssembler::name##Constant() { \
+ return UncheckedCast<std::remove_pointer<std::remove_reference<decltype( \
+ std::declval<Heap>().rootAccessorName())>::type>::type>( \
+ LoadRoot(RootIndex::k##rootIndexName)); \
+ }
+HEAP_MUTABLE_IMMOVABLE_OBJECT_LIST(HEAP_CONSTANT_ACCESSOR)
+#undef HEAP_CONSTANT_ACCESSOR
+
+#define HEAP_CONSTANT_ACCESSOR(rootIndexName, rootAccessorName, name) \
+ TNode<std::remove_pointer<std::remove_reference<decltype( \
+ std::declval<ReadOnlyRoots>().rootAccessorName())>::type>::type> \
+ CodeStubAssembler::name##Constant() { \
+ return UncheckedCast<std::remove_pointer<std::remove_reference<decltype( \
+ std::declval<ReadOnlyRoots>().rootAccessorName())>::type>::type>( \
+ LoadRoot(RootIndex::k##rootIndexName)); \
+ }
+HEAP_IMMUTABLE_IMMOVABLE_OBJECT_LIST(HEAP_CONSTANT_ACCESSOR)
+#undef HEAP_CONSTANT_ACCESSOR
+
+#define HEAP_CONSTANT_TEST(rootIndexName, rootAccessorName, name) \
+ TNode<BoolT> CodeStubAssembler::Is##name(SloppyTNode<Object> value) { \
+ return TaggedEqual(value, name##Constant()); \
+ } \
+ TNode<BoolT> CodeStubAssembler::IsNot##name(SloppyTNode<Object> value) { \
+ return TaggedNotEqual(value, name##Constant()); \
+ }
+HEAP_IMMOVABLE_OBJECT_LIST(HEAP_CONSTANT_TEST)
+#undef HEAP_CONSTANT_TEST
+
+TNode<BInt> CodeStubAssembler::BIntConstant(int value) {
+#if defined(BINT_IS_SMI)
+ return SmiConstant(value);
+#elif defined(BINT_IS_INTPTR)
+ return IntPtrConstant(value);
+#else
+#error Unknown architecture.
+#endif
+}
+
+template <>
+TNode<Smi> CodeStubAssembler::IntPtrOrSmiConstant<Smi>(int value) {
+ return SmiConstant(value);
+}
+
+template <>
+TNode<IntPtrT> CodeStubAssembler::IntPtrOrSmiConstant<IntPtrT>(int value) {
+ return IntPtrConstant(value);
+}
+
+template <>
+TNode<UintPtrT> CodeStubAssembler::IntPtrOrSmiConstant<UintPtrT>(int value) {
+ return Unsigned(IntPtrConstant(value));
+}
+
+template <>
+TNode<RawPtrT> CodeStubAssembler::IntPtrOrSmiConstant<RawPtrT>(int value) {
+ return ReinterpretCast<RawPtrT>(IntPtrConstant(value));
+}
+
+bool CodeStubAssembler::TryGetIntPtrOrSmiConstantValue(
+ TNode<Smi> maybe_constant, int* value) {
+ Smi smi_constant;
+ if (ToSmiConstant(maybe_constant, &smi_constant)) {
+ *value = Smi::ToInt(smi_constant);
+ return true;
+ }
+ return false;
+}
+
+bool CodeStubAssembler::TryGetIntPtrOrSmiConstantValue(
+ TNode<IntPtrT> maybe_constant, int* value) {
+ int32_t int32_constant;
+ if (ToInt32Constant(maybe_constant, &int32_constant)) {
+ *value = int32_constant;
+ return true;
+ }
+ return false;
+}
+
+TNode<IntPtrT> CodeStubAssembler::IntPtrRoundUpToPowerOfTwo32(
+ TNode<IntPtrT> value) {
+ Comment("IntPtrRoundUpToPowerOfTwo32");
+ CSA_ASSERT(this, UintPtrLessThanOrEqual(value, IntPtrConstant(0x80000000u)));
+ value = Signed(IntPtrSub(value, IntPtrConstant(1)));
+ for (int i = 1; i <= 16; i *= 2) {
+ value = Signed(WordOr(value, WordShr(value, IntPtrConstant(i))));
+ }
+ return Signed(IntPtrAdd(value, IntPtrConstant(1)));
+}
+
+TNode<BoolT> CodeStubAssembler::WordIsPowerOfTwo(SloppyTNode<IntPtrT> value) {
+ intptr_t constant;
+ if (ToIntPtrConstant(value, &constant)) {
+ return BoolConstant(base::bits::IsPowerOfTwo(constant));
+ }
+ // value && !(value & (value - 1))
+ return IntPtrEqual(
+ Select<IntPtrT>(
+ IntPtrEqual(value, IntPtrConstant(0)),
+ [=] { return IntPtrConstant(1); },
+ [=] { return WordAnd(value, IntPtrSub(value, IntPtrConstant(1))); }),
+ IntPtrConstant(0));
+}
+
+TNode<Float64T> CodeStubAssembler::Float64Round(SloppyTNode<Float64T> x) {
+ TNode<Float64T> one = Float64Constant(1.0);
+ TNode<Float64T> one_half = Float64Constant(0.5);
+
+ Label return_x(this);
+
+ // Round up {x} towards Infinity.
+ TVARIABLE(Float64T, var_x, Float64Ceil(x));
+
+ GotoIf(Float64LessThanOrEqual(Float64Sub(var_x.value(), one_half), x),
+ &return_x);
+ var_x = Float64Sub(var_x.value(), one);
+ Goto(&return_x);
+
+ BIND(&return_x);
+ return TNode<Float64T>::UncheckedCast(var_x.value());
+}
+
+TNode<Float64T> CodeStubAssembler::Float64Ceil(SloppyTNode<Float64T> x) {
+ if (IsFloat64RoundUpSupported()) {
+ return Float64RoundUp(x);
+ }
+
+ TNode<Float64T> one = Float64Constant(1.0);
+ TNode<Float64T> zero = Float64Constant(0.0);
+ TNode<Float64T> two_52 = Float64Constant(4503599627370496.0E0);
+ TNode<Float64T> minus_two_52 = Float64Constant(-4503599627370496.0E0);
+
+ TVARIABLE(Float64T, var_x, x);
+ Label return_x(this), return_minus_x(this);
+
+ // Check if {x} is greater than zero.
+ Label if_xgreaterthanzero(this), if_xnotgreaterthanzero(this);
+ Branch(Float64GreaterThan(x, zero), &if_xgreaterthanzero,
+ &if_xnotgreaterthanzero);
+
+ BIND(&if_xgreaterthanzero);
+ {
+ // Just return {x} unless it's in the range ]0,2^52[.
+ GotoIf(Float64GreaterThanOrEqual(x, two_52), &return_x);
+
+ // Round positive {x} towards Infinity.
+ var_x = Float64Sub(Float64Add(two_52, x), two_52);
+ GotoIfNot(Float64LessThan(var_x.value(), x), &return_x);
+ var_x = Float64Add(var_x.value(), one);
+ Goto(&return_x);
+ }
+
+ BIND(&if_xnotgreaterthanzero);
+ {
+ // Just return {x} unless it's in the range ]-2^52,0[
+ GotoIf(Float64LessThanOrEqual(x, minus_two_52), &return_x);
+ GotoIfNot(Float64LessThan(x, zero), &return_x);
+
+ // Round negated {x} towards Infinity and return the result negated.
+ TNode<Float64T> minus_x = Float64Neg(x);
+ var_x = Float64Sub(Float64Add(two_52, minus_x), two_52);
+ GotoIfNot(Float64GreaterThan(var_x.value(), minus_x), &return_minus_x);
+ var_x = Float64Sub(var_x.value(), one);
+ Goto(&return_minus_x);
+ }
+
+ BIND(&return_minus_x);
+ var_x = Float64Neg(var_x.value());
+ Goto(&return_x);
+
+ BIND(&return_x);
+ return TNode<Float64T>::UncheckedCast(var_x.value());
+}
+
+TNode<Float64T> CodeStubAssembler::Float64Floor(SloppyTNode<Float64T> x) {
+ if (IsFloat64RoundDownSupported()) {
+ return Float64RoundDown(x);
+ }
+
+ TNode<Float64T> one = Float64Constant(1.0);
+ TNode<Float64T> zero = Float64Constant(0.0);
+ TNode<Float64T> two_52 = Float64Constant(4503599627370496.0E0);
+ TNode<Float64T> minus_two_52 = Float64Constant(-4503599627370496.0E0);
+
+ TVARIABLE(Float64T, var_x, x);
+ Label return_x(this), return_minus_x(this);
+
+ // Check if {x} is greater than zero.
+ Label if_xgreaterthanzero(this), if_xnotgreaterthanzero(this);
+ Branch(Float64GreaterThan(x, zero), &if_xgreaterthanzero,
+ &if_xnotgreaterthanzero);
+
+ BIND(&if_xgreaterthanzero);
+ {
+ // Just return {x} unless it's in the range ]0,2^52[.
+ GotoIf(Float64GreaterThanOrEqual(x, two_52), &return_x);
+
+ // Round positive {x} towards -Infinity.
+ var_x = Float64Sub(Float64Add(two_52, x), two_52);
+ GotoIfNot(Float64GreaterThan(var_x.value(), x), &return_x);
+ var_x = Float64Sub(var_x.value(), one);
+ Goto(&return_x);
+ }
+
+ BIND(&if_xnotgreaterthanzero);
+ {
+ // Just return {x} unless it's in the range ]-2^52,0[
+ GotoIf(Float64LessThanOrEqual(x, minus_two_52), &return_x);
+ GotoIfNot(Float64LessThan(x, zero), &return_x);
+
+ // Round negated {x} towards -Infinity and return the result negated.
+ TNode<Float64T> minus_x = Float64Neg(x);
+ var_x = Float64Sub(Float64Add(two_52, minus_x), two_52);
+ GotoIfNot(Float64LessThan(var_x.value(), minus_x), &return_minus_x);
+ var_x = Float64Add(var_x.value(), one);
+ Goto(&return_minus_x);
+ }
+
+ BIND(&return_minus_x);
+ var_x = Float64Neg(var_x.value());
+ Goto(&return_x);
+
+ BIND(&return_x);
+ return TNode<Float64T>::UncheckedCast(var_x.value());
+}
+
+TNode<Float64T> CodeStubAssembler::Float64RoundToEven(SloppyTNode<Float64T> x) {
+ if (IsFloat64RoundTiesEvenSupported()) {
+ return Float64RoundTiesEven(x);
+ }
+ // See ES#sec-touint8clamp for details.
+ TNode<Float64T> f = Float64Floor(x);
+ TNode<Float64T> f_and_half = Float64Add(f, Float64Constant(0.5));
+
+ TVARIABLE(Float64T, var_result);
+ Label return_f(this), return_f_plus_one(this), done(this);
+
+ GotoIf(Float64LessThan(f_and_half, x), &return_f_plus_one);
+ GotoIf(Float64LessThan(x, f_and_half), &return_f);
+ {
+ TNode<Float64T> f_mod_2 = Float64Mod(f, Float64Constant(2.0));
+ Branch(Float64Equal(f_mod_2, Float64Constant(0.0)), &return_f,
+ &return_f_plus_one);
+ }
+
+ BIND(&return_f);
+ var_result = f;
+ Goto(&done);
+
+ BIND(&return_f_plus_one);
+ var_result = Float64Add(f, Float64Constant(1.0));
+ Goto(&done);
+
+ BIND(&done);
+ return TNode<Float64T>::UncheckedCast(var_result.value());
+}
+
+TNode<Float64T> CodeStubAssembler::Float64Trunc(SloppyTNode<Float64T> x) {
+ if (IsFloat64RoundTruncateSupported()) {
+ return Float64RoundTruncate(x);
+ }
+
+ TNode<Float64T> one = Float64Constant(1.0);
+ TNode<Float64T> zero = Float64Constant(0.0);
+ TNode<Float64T> two_52 = Float64Constant(4503599627370496.0E0);
+ TNode<Float64T> minus_two_52 = Float64Constant(-4503599627370496.0E0);
+
+ TVARIABLE(Float64T, var_x, x);
+ Label return_x(this), return_minus_x(this);
+
+ // Check if {x} is greater than 0.
+ Label if_xgreaterthanzero(this), if_xnotgreaterthanzero(this);
+ Branch(Float64GreaterThan(x, zero), &if_xgreaterthanzero,
+ &if_xnotgreaterthanzero);
+
+ BIND(&if_xgreaterthanzero);
+ {
+ if (IsFloat64RoundDownSupported()) {
+ var_x = Float64RoundDown(x);
+ } else {
+ // Just return {x} unless it's in the range ]0,2^52[.
+ GotoIf(Float64GreaterThanOrEqual(x, two_52), &return_x);
+
+ // Round positive {x} towards -Infinity.
+ var_x = Float64Sub(Float64Add(two_52, x), two_52);
+ GotoIfNot(Float64GreaterThan(var_x.value(), x), &return_x);
+ var_x = Float64Sub(var_x.value(), one);
+ }
+ Goto(&return_x);
+ }
+
+ BIND(&if_xnotgreaterthanzero);
+ {
+ if (IsFloat64RoundUpSupported()) {
+ var_x = Float64RoundUp(x);
+ Goto(&return_x);
+ } else {
+ // Just return {x} unless its in the range ]-2^52,0[.
+ GotoIf(Float64LessThanOrEqual(x, minus_two_52), &return_x);
+ GotoIfNot(Float64LessThan(x, zero), &return_x);
+
+ // Round negated {x} towards -Infinity and return result negated.
+ TNode<Float64T> minus_x = Float64Neg(x);
+ var_x = Float64Sub(Float64Add(two_52, minus_x), two_52);
+ GotoIfNot(Float64GreaterThan(var_x.value(), minus_x), &return_minus_x);
+ var_x = Float64Sub(var_x.value(), one);
+ Goto(&return_minus_x);
+ }
+ }
+
+ BIND(&return_minus_x);
+ var_x = Float64Neg(var_x.value());
+ Goto(&return_x);
+
+ BIND(&return_x);
+ return TNode<Float64T>::UncheckedCast(var_x.value());
+}
+
+template <>
+TNode<Smi> CodeStubAssembler::TaggedToParameter(TNode<Smi> value) {
+ return value;
+}
+
+template <>
+TNode<IntPtrT> CodeStubAssembler::TaggedToParameter(TNode<Smi> value) {
+ return SmiUntag(value);
+}
+
+TNode<IntPtrT> CodeStubAssembler::TaggedIndexToIntPtr(
+ TNode<TaggedIndex> value) {
+ return Signed(WordSarShiftOutZeros(BitcastTaggedToWordForTagAndSmiBits(value),
+ IntPtrConstant(kSmiTagSize)));
+}
+
+TNode<TaggedIndex> CodeStubAssembler::IntPtrToTaggedIndex(
+ TNode<IntPtrT> value) {
+ return ReinterpretCast<TaggedIndex>(
+ BitcastWordToTaggedSigned(WordShl(value, IntPtrConstant(kSmiTagSize))));
+}
+
+TNode<Smi> CodeStubAssembler::TaggedIndexToSmi(TNode<TaggedIndex> value) {
+ if (SmiValuesAre32Bits()) {
+ DCHECK_EQ(kSmiShiftSize, 31);
+ return BitcastWordToTaggedSigned(
+ WordShl(BitcastTaggedToWordForTagAndSmiBits(value),
+ IntPtrConstant(kSmiShiftSize)));
+ }
+ DCHECK(SmiValuesAre31Bits());
+ DCHECK_EQ(kSmiShiftSize, 0);
+ return ReinterpretCast<Smi>(value);
+}
+
+TNode<TaggedIndex> CodeStubAssembler::SmiToTaggedIndex(TNode<Smi> value) {
+ if (kSystemPointerSize == kInt32Size) {
+ return ReinterpretCast<TaggedIndex>(value);
+ }
+ if (SmiValuesAre32Bits()) {
+ DCHECK_EQ(kSmiShiftSize, 31);
+ return ReinterpretCast<TaggedIndex>(BitcastWordToTaggedSigned(
+ WordSar(BitcastTaggedToWordForTagAndSmiBits(value),
+ IntPtrConstant(kSmiShiftSize))));
+ }
+ DCHECK(SmiValuesAre31Bits());
+ DCHECK_EQ(kSmiShiftSize, 0);
+ // Just sign-extend the lower 32 bits.
+ TNode<Int32T> raw =
+ TruncateWordToInt32(BitcastTaggedToWordForTagAndSmiBits(value));
+ return ReinterpretCast<TaggedIndex>(
+ BitcastWordToTaggedSigned(ChangeInt32ToIntPtr(raw)));
+}
+
+TNode<Smi> CodeStubAssembler::NormalizeSmiIndex(TNode<Smi> smi_index) {
+ if (COMPRESS_POINTERS_BOOL) {
+ TNode<Int32T> raw =
+ TruncateWordToInt32(BitcastTaggedToWordForTagAndSmiBits(smi_index));
+ smi_index = BitcastWordToTaggedSigned(ChangeInt32ToIntPtr(raw));
+ }
+ return smi_index;
+}
+
+TNode<Smi> CodeStubAssembler::SmiFromInt32(SloppyTNode<Int32T> value) {
+ if (COMPRESS_POINTERS_BOOL) {
+ static_assert(!COMPRESS_POINTERS_BOOL || (kSmiShiftSize + kSmiTagSize == 1),
+ "Use shifting instead of add");
+ return BitcastWordToTaggedSigned(
+ ChangeUint32ToWord(Int32Add(value, value)));
+ }
+ return SmiTag(ChangeInt32ToIntPtr(value));
+}
+
+TNode<Smi> CodeStubAssembler::SmiFromUint32(TNode<Uint32T> value) {
+ CSA_ASSERT(this, IntPtrLessThan(ChangeUint32ToWord(value),
+ IntPtrConstant(Smi::kMaxValue)));
+ return SmiFromInt32(Signed(value));
+}
+
+TNode<BoolT> CodeStubAssembler::IsValidPositiveSmi(TNode<IntPtrT> value) {
+ intptr_t constant_value;
+ if (ToIntPtrConstant(value, &constant_value)) {
+ return (static_cast<uintptr_t>(constant_value) <=
+ static_cast<uintptr_t>(Smi::kMaxValue))
+ ? Int32TrueConstant()
+ : Int32FalseConstant();
+ }
+
+ return UintPtrLessThanOrEqual(value, IntPtrConstant(Smi::kMaxValue));
+}
+
+TNode<Smi> CodeStubAssembler::SmiTag(SloppyTNode<IntPtrT> value) {
+ int32_t constant_value;
+ if (ToInt32Constant(value, &constant_value) && Smi::IsValid(constant_value)) {
+ return SmiConstant(constant_value);
+ }
+ if (COMPRESS_POINTERS_BOOL) {
+ return SmiFromInt32(TruncateIntPtrToInt32(value));
+ }
+ TNode<Smi> smi =
+ BitcastWordToTaggedSigned(WordShl(value, SmiShiftBitsConstant()));
+ return smi;
+}
+
+TNode<IntPtrT> CodeStubAssembler::SmiUntag(SloppyTNode<Smi> value) {
+ intptr_t constant_value;
+ if (ToIntPtrConstant(value, &constant_value)) {
+ return IntPtrConstant(constant_value >> (kSmiShiftSize + kSmiTagSize));
+ }
+ TNode<IntPtrT> raw_bits = BitcastTaggedToWordForTagAndSmiBits(value);
+ if (COMPRESS_POINTERS_BOOL) {
+ // Clear the upper half using sign-extension.
+ raw_bits = ChangeInt32ToIntPtr(TruncateIntPtrToInt32(raw_bits));
+ }
+ return Signed(WordSarShiftOutZeros(raw_bits, SmiShiftBitsConstant()));
+}
+
+TNode<Int32T> CodeStubAssembler::SmiToInt32(SloppyTNode<Smi> value) {
+ if (COMPRESS_POINTERS_BOOL) {
+ return Signed(Word32SarShiftOutZeros(
+ TruncateIntPtrToInt32(BitcastTaggedToWordForTagAndSmiBits(value)),
+ SmiShiftBitsConstant32()));
+ }
+ TNode<IntPtrT> result = SmiUntag(value);
+ return TruncateIntPtrToInt32(result);
+}
+
+TNode<Float64T> CodeStubAssembler::SmiToFloat64(SloppyTNode<Smi> value) {
+ return ChangeInt32ToFloat64(SmiToInt32(value));
+}
+
+TNode<Smi> CodeStubAssembler::SmiMax(TNode<Smi> a, TNode<Smi> b) {
+ return SelectConstant<Smi>(SmiLessThan(a, b), b, a);
+}
+
+TNode<Smi> CodeStubAssembler::SmiMin(TNode<Smi> a, TNode<Smi> b) {
+ return SelectConstant<Smi>(SmiLessThan(a, b), a, b);
+}
+
+TNode<IntPtrT> CodeStubAssembler::TryIntPtrAdd(TNode<IntPtrT> a,
+ TNode<IntPtrT> b,
+ Label* if_overflow) {
+ TNode<PairT<IntPtrT, BoolT>> pair = IntPtrAddWithOverflow(a, b);
+ TNode<BoolT> overflow = Projection<1>(pair);
+ GotoIf(overflow, if_overflow);
+ return Projection<0>(pair);
+}
+
+TNode<IntPtrT> CodeStubAssembler::TryIntPtrSub(TNode<IntPtrT> a,
+ TNode<IntPtrT> b,
+ Label* if_overflow) {
+ TNode<PairT<IntPtrT, BoolT>> pair = IntPtrSubWithOverflow(a, b);
+ TNode<BoolT> overflow = Projection<1>(pair);
+ GotoIf(overflow, if_overflow);
+ return Projection<0>(pair);
+}
+
+TNode<Int32T> CodeStubAssembler::TryInt32Mul(TNode<Int32T> a, TNode<Int32T> b,
+ Label* if_overflow) {
+ TNode<PairT<Int32T, BoolT>> pair = Int32MulWithOverflow(a, b);
+ TNode<BoolT> overflow = Projection<1>(pair);
+ GotoIf(overflow, if_overflow);
+ return Projection<0>(pair);
+}
+
+TNode<Smi> CodeStubAssembler::TrySmiAdd(TNode<Smi> lhs, TNode<Smi> rhs,
+ Label* if_overflow) {
+ if (SmiValuesAre32Bits()) {
+ return BitcastWordToTaggedSigned(
+ TryIntPtrAdd(BitcastTaggedToWordForTagAndSmiBits(lhs),
+ BitcastTaggedToWordForTagAndSmiBits(rhs), if_overflow));
+ } else {
+ DCHECK(SmiValuesAre31Bits());
+ TNode<PairT<Int32T, BoolT>> pair = Int32AddWithOverflow(
+ TruncateIntPtrToInt32(BitcastTaggedToWordForTagAndSmiBits(lhs)),
+ TruncateIntPtrToInt32(BitcastTaggedToWordForTagAndSmiBits(rhs)));
+ TNode<BoolT> overflow = Projection<1>(pair);
+ GotoIf(overflow, if_overflow);
+ TNode<Int32T> result = Projection<0>(pair);
+ return BitcastWordToTaggedSigned(ChangeInt32ToIntPtr(result));
+ }
+}
+
+TNode<Smi> CodeStubAssembler::TrySmiSub(TNode<Smi> lhs, TNode<Smi> rhs,
+ Label* if_overflow) {
+ if (SmiValuesAre32Bits()) {
+ TNode<PairT<IntPtrT, BoolT>> pair =
+ IntPtrSubWithOverflow(BitcastTaggedToWordForTagAndSmiBits(lhs),
+ BitcastTaggedToWordForTagAndSmiBits(rhs));
+ TNode<BoolT> overflow = Projection<1>(pair);
+ GotoIf(overflow, if_overflow);
+ TNode<IntPtrT> result = Projection<0>(pair);
+ return BitcastWordToTaggedSigned(result);
+ } else {
+ DCHECK(SmiValuesAre31Bits());
+ TNode<PairT<Int32T, BoolT>> pair = Int32SubWithOverflow(
+ TruncateIntPtrToInt32(BitcastTaggedToWordForTagAndSmiBits(lhs)),
+ TruncateIntPtrToInt32(BitcastTaggedToWordForTagAndSmiBits(rhs)));
+ TNode<BoolT> overflow = Projection<1>(pair);
+ GotoIf(overflow, if_overflow);
+ TNode<Int32T> result = Projection<0>(pair);
+ return BitcastWordToTaggedSigned(ChangeInt32ToIntPtr(result));
+ }
+}
+
+TNode<Smi> CodeStubAssembler::TrySmiAbs(TNode<Smi> a, Label* if_overflow) {
+ if (SmiValuesAre32Bits()) {
+ TNode<PairT<IntPtrT, BoolT>> pair =
+ IntPtrAbsWithOverflow(BitcastTaggedToWordForTagAndSmiBits(a));
+ TNode<BoolT> overflow = Projection<1>(pair);
+ GotoIf(overflow, if_overflow);
+ TNode<IntPtrT> result = Projection<0>(pair);
+ return BitcastWordToTaggedSigned(result);
+ } else {
+ CHECK(SmiValuesAre31Bits());
+ CHECK(IsInt32AbsWithOverflowSupported());
+ TNode<PairT<Int32T, BoolT>> pair = Int32AbsWithOverflow(
+ TruncateIntPtrToInt32(BitcastTaggedToWordForTagAndSmiBits(a)));
+ TNode<BoolT> overflow = Projection<1>(pair);
+ GotoIf(overflow, if_overflow);
+ TNode<Int32T> result = Projection<0>(pair);
+ return BitcastWordToTaggedSigned(ChangeInt32ToIntPtr(result));
+ }
+}
+
+TNode<Number> CodeStubAssembler::NumberMax(TNode<Number> a, TNode<Number> b) {
+ // TODO(danno): This could be optimized by specifically handling smi cases.
+ TVARIABLE(Number, result);
+ Label done(this), greater_than_equal_a(this), greater_than_equal_b(this);
+ GotoIfNumberGreaterThanOrEqual(a, b, &greater_than_equal_a);
+ GotoIfNumberGreaterThanOrEqual(b, a, &greater_than_equal_b);
+ result = NanConstant();
+ Goto(&done);
+ BIND(&greater_than_equal_a);
+ result = a;
+ Goto(&done);
+ BIND(&greater_than_equal_b);
+ result = b;
+ Goto(&done);
+ BIND(&done);
+ return result.value();
+}
+
+TNode<Number> CodeStubAssembler::NumberMin(TNode<Number> a, TNode<Number> b) {
+ // TODO(danno): This could be optimized by specifically handling smi cases.
+ TVARIABLE(Number, result);
+ Label done(this), greater_than_equal_a(this), greater_than_equal_b(this);
+ GotoIfNumberGreaterThanOrEqual(a, b, &greater_than_equal_a);
+ GotoIfNumberGreaterThanOrEqual(b, a, &greater_than_equal_b);
+ result = NanConstant();
+ Goto(&done);
+ BIND(&greater_than_equal_a);
+ result = b;
+ Goto(&done);
+ BIND(&greater_than_equal_b);
+ result = a;
+ Goto(&done);
+ BIND(&done);
+ return result.value();
+}
+
+TNode<Number> CodeStubAssembler::SmiMod(TNode<Smi> a, TNode<Smi> b) {
+ TVARIABLE(Number, var_result);
+ Label return_result(this, &var_result),
+ return_minuszero(this, Label::kDeferred),
+ return_nan(this, Label::kDeferred);
+
+ // Untag {a} and {b}.
+ TNode<Int32T> int_a = SmiToInt32(a);
+ TNode<Int32T> int_b = SmiToInt32(b);
+
+ // Return NaN if {b} is zero.
+ GotoIf(Word32Equal(int_b, Int32Constant(0)), &return_nan);
+
+ // Check if {a} is non-negative.
+ Label if_aisnotnegative(this), if_aisnegative(this, Label::kDeferred);
+ Branch(Int32LessThanOrEqual(Int32Constant(0), int_a), &if_aisnotnegative,
+ &if_aisnegative);
+
+ BIND(&if_aisnotnegative);
+ {
+ // Fast case, don't need to check any other edge cases.
+ TNode<Int32T> r = Int32Mod(int_a, int_b);
+ var_result = SmiFromInt32(r);
+ Goto(&return_result);
+ }
+
+ BIND(&if_aisnegative);
+ {
+ if (SmiValuesAre32Bits()) {
+ // Check if {a} is kMinInt and {b} is -1 (only relevant if the
+ // kMinInt is actually representable as a Smi).
+ Label join(this);
+ GotoIfNot(Word32Equal(int_a, Int32Constant(kMinInt)), &join);
+ GotoIf(Word32Equal(int_b, Int32Constant(-1)), &return_minuszero);
+ Goto(&join);
+ BIND(&join);
+ }
+
+ // Perform the integer modulus operation.
+ TNode<Int32T> r = Int32Mod(int_a, int_b);
+
+ // Check if {r} is zero, and if so return -0, because we have to
+ // take the sign of the left hand side {a}, which is negative.
+ GotoIf(Word32Equal(r, Int32Constant(0)), &return_minuszero);
+
+ // The remainder {r} can be outside the valid Smi range on 32bit
+ // architectures, so we cannot just say SmiFromInt32(r) here.
+ var_result = ChangeInt32ToTagged(r);
+ Goto(&return_result);
+ }
+
+ BIND(&return_minuszero);
+ var_result = MinusZeroConstant();
+ Goto(&return_result);
+
+ BIND(&return_nan);
+ var_result = NanConstant();
+ Goto(&return_result);
+
+ BIND(&return_result);
+ return var_result.value();
+}
+
+TNode<Number> CodeStubAssembler::SmiMul(TNode<Smi> a, TNode<Smi> b) {
+ TVARIABLE(Number, var_result);
+ TVARIABLE(Float64T, var_lhs_float64);
+ TVARIABLE(Float64T, var_rhs_float64);
+ Label return_result(this, &var_result);
+
+ // Both {a} and {b} are Smis. Convert them to integers and multiply.
+ TNode<Int32T> lhs32 = SmiToInt32(a);
+ TNode<Int32T> rhs32 = SmiToInt32(b);
+ auto pair = Int32MulWithOverflow(lhs32, rhs32);
+
+ TNode<BoolT> overflow = Projection<1>(pair);
+
+ // Check if the multiplication overflowed.
+ Label if_overflow(this, Label::kDeferred), if_notoverflow(this);
+ Branch(overflow, &if_overflow, &if_notoverflow);
+ BIND(&if_notoverflow);
+ {
+ // If the answer is zero, we may need to return -0.0, depending on the
+ // input.
+ Label answer_zero(this), answer_not_zero(this);
+ TNode<Int32T> answer = Projection<0>(pair);
+ TNode<Int32T> zero = Int32Constant(0);
+ Branch(Word32Equal(answer, zero), &answer_zero, &answer_not_zero);
+ BIND(&answer_not_zero);
+ {
+ var_result = ChangeInt32ToTagged(answer);
+ Goto(&return_result);
+ }
+ BIND(&answer_zero);
+ {
+ TNode<Int32T> or_result = Word32Or(lhs32, rhs32);
+ Label if_should_be_negative_zero(this), if_should_be_zero(this);
+ Branch(Int32LessThan(or_result, zero), &if_should_be_negative_zero,
+ &if_should_be_zero);
+ BIND(&if_should_be_negative_zero);
+ {
+ var_result = MinusZeroConstant();
+ Goto(&return_result);
+ }
+ BIND(&if_should_be_zero);
+ {
+ var_result = SmiConstant(0);
+ Goto(&return_result);
+ }
+ }
+ }
+ BIND(&if_overflow);
+ {
+ var_lhs_float64 = SmiToFloat64(a);
+ var_rhs_float64 = SmiToFloat64(b);
+ TNode<Float64T> value =
+ Float64Mul(var_lhs_float64.value(), var_rhs_float64.value());
+ var_result = AllocateHeapNumberWithValue(value);
+ Goto(&return_result);
+ }
+
+ BIND(&return_result);
+ return var_result.value();
+}
+
+TNode<Smi> CodeStubAssembler::TrySmiDiv(TNode<Smi> dividend, TNode<Smi> divisor,
+ Label* bailout) {
+ // Both {a} and {b} are Smis. Bailout to floating point division if {divisor}
+ // is zero.
+ GotoIf(TaggedEqual(divisor, SmiConstant(0)), bailout);
+
+ // Do floating point division if {dividend} is zero and {divisor} is
+ // negative.
+ Label dividend_is_zero(this), dividend_is_not_zero(this);
+ Branch(TaggedEqual(dividend, SmiConstant(0)), ÷nd_is_zero,
+ ÷nd_is_not_zero);
+
+ BIND(÷nd_is_zero);
+ {
+ GotoIf(SmiLessThan(divisor, SmiConstant(0)), bailout);
+ Goto(÷nd_is_not_zero);
+ }
+ BIND(÷nd_is_not_zero);
+
+ TNode<Int32T> untagged_divisor = SmiToInt32(divisor);
+ TNode<Int32T> untagged_dividend = SmiToInt32(dividend);
+
+ // Do floating point division if {dividend} is kMinInt (or kMinInt - 1
+ // if the Smi size is 31) and {divisor} is -1.
+ Label divisor_is_minus_one(this), divisor_is_not_minus_one(this);
+ Branch(Word32Equal(untagged_divisor, Int32Constant(-1)),
+ &divisor_is_minus_one, &divisor_is_not_minus_one);
+
+ BIND(&divisor_is_minus_one);
+ {
+ GotoIf(Word32Equal(
+ untagged_dividend,
+ Int32Constant(kSmiValueSize == 32 ? kMinInt : (kMinInt >> 1))),
+ bailout);
+ Goto(&divisor_is_not_minus_one);
+ }
+ BIND(&divisor_is_not_minus_one);
+
+ TNode<Int32T> untagged_result = Int32Div(untagged_dividend, untagged_divisor);
+ TNode<Int32T> truncated = Int32Mul(untagged_result, untagged_divisor);
+
+ // Do floating point division if the remainder is not 0.
+ GotoIf(Word32NotEqual(untagged_dividend, truncated), bailout);
+
+ return SmiFromInt32(untagged_result);
+}
+
+TNode<Smi> CodeStubAssembler::SmiLexicographicCompare(TNode<Smi> x,
+ TNode<Smi> y) {
+ TNode<ExternalReference> smi_lexicographic_compare =
+ ExternalConstant(ExternalReference::smi_lexicographic_compare_function());
+ TNode<ExternalReference> isolate_ptr =
+ ExternalConstant(ExternalReference::isolate_address(isolate()));
+ return CAST(CallCFunction(smi_lexicographic_compare, MachineType::AnyTagged(),
+ std::make_pair(MachineType::Pointer(), isolate_ptr),
+ std::make_pair(MachineType::AnyTagged(), x),
+ std::make_pair(MachineType::AnyTagged(), y)));
+}
+
+TNode<Int32T> CodeStubAssembler::TruncateWordToInt32(SloppyTNode<WordT> value) {
+ if (Is64()) {
+ return TruncateInt64ToInt32(ReinterpretCast<Int64T>(value));
+ }
+ return ReinterpretCast<Int32T>(value);
+}
+
+TNode<Int32T> CodeStubAssembler::TruncateIntPtrToInt32(
+ SloppyTNode<IntPtrT> value) {
+ if (Is64()) {
+ return TruncateInt64ToInt32(ReinterpretCast<Int64T>(value));
+ }
+ return ReinterpretCast<Int32T>(value);
+}
+
+TNode<BoolT> CodeStubAssembler::TaggedIsSmi(TNode<MaybeObject> a) {
+ STATIC_ASSERT(kSmiTagMask < kMaxUInt32);
+ return Word32Equal(
+ Word32And(TruncateIntPtrToInt32(BitcastTaggedToWordForTagAndSmiBits(a)),
+ Int32Constant(kSmiTagMask)),
+ Int32Constant(0));
+}
+
+TNode<BoolT> CodeStubAssembler::TaggedIsNotSmi(TNode<MaybeObject> a) {
+ return Word32BinaryNot(TaggedIsSmi(a));
+}
+
+TNode<BoolT> CodeStubAssembler::TaggedIsPositiveSmi(SloppyTNode<Object> a) {
+#if defined(V8_HOST_ARCH_32_BIT) || defined(V8_31BIT_SMIS_ON_64BIT_ARCH)
+ return Word32Equal(
+ Word32And(
+ TruncateIntPtrToInt32(BitcastTaggedToWordForTagAndSmiBits(a)),
+ Uint32Constant(static_cast<uint32_t>(kSmiTagMask | kSmiSignMask))),
+ Int32Constant(0));
+#else
+ return WordEqual(WordAnd(BitcastTaggedToWordForTagAndSmiBits(a),
+ IntPtrConstant(kSmiTagMask | kSmiSignMask)),
+ IntPtrConstant(0));
+#endif
+}
+
+TNode<BoolT> CodeStubAssembler::WordIsAligned(SloppyTNode<WordT> word,
+ size_t alignment) {
+ DCHECK(base::bits::IsPowerOfTwo(alignment));
+ DCHECK_LE(alignment, kMaxUInt32);
+ return Word32Equal(
+ Int32Constant(0),
+ Word32And(TruncateWordToInt32(word),
+ Uint32Constant(static_cast<uint32_t>(alignment) - 1)));
+}
+
+#if DEBUG
+void CodeStubAssembler::Bind(Label* label, AssemblerDebugInfo debug_info) {
+ CodeAssembler::Bind(label, debug_info);
+}
+#endif // DEBUG
+
+void CodeStubAssembler::Bind(Label* label) { CodeAssembler::Bind(label); }
+
+TNode<Float64T> CodeStubAssembler::LoadDoubleWithHoleCheck(
+ TNode<FixedDoubleArray> array, TNode<IntPtrT> index, Label* if_hole) {
+ return LoadFixedDoubleArrayElement(array, index, if_hole);
+}
+
+void CodeStubAssembler::BranchIfJSReceiver(SloppyTNode<Object> object,
+ Label* if_true, Label* if_false) {
+ GotoIf(TaggedIsSmi(object), if_false);
+ STATIC_ASSERT(LAST_JS_RECEIVER_TYPE == LAST_TYPE);
+ Branch(IsJSReceiver(CAST(object)), if_true, if_false);
+}
+
+void CodeStubAssembler::GotoIfForceSlowPath(Label* if_true) {
+#ifdef V8_ENABLE_FORCE_SLOW_PATH
+ const TNode<ExternalReference> force_slow_path_addr =
+ ExternalConstant(ExternalReference::force_slow_path(isolate()));
+ const TNode<Uint8T> force_slow = Load<Uint8T>(force_slow_path_addr);
+
+ GotoIf(force_slow, if_true);
+#endif
+}
+
+TNode<HeapObject> CodeStubAssembler::AllocateRaw(TNode<IntPtrT> size_in_bytes,
+ AllocationFlags flags,
+ TNode<RawPtrT> top_address,
+ TNode<RawPtrT> limit_address) {
+ Label if_out_of_memory(this, Label::kDeferred);
+
+ // TODO(jgruber,jkummerow): Extract the slow paths (= probably everything
+ // but bump pointer allocation) into a builtin to save code space. The
+ // size_in_bytes check may be moved there as well since a non-smi
+ // size_in_bytes probably doesn't fit into the bump pointer region
+ // (double-check that).
+
+ intptr_t size_in_bytes_constant;
+ bool size_in_bytes_is_constant = false;
+ if (ToIntPtrConstant(size_in_bytes, &size_in_bytes_constant)) {
+ size_in_bytes_is_constant = true;
+ CHECK(Internals::IsValidSmi(size_in_bytes_constant));
+ CHECK_GT(size_in_bytes_constant, 0);
+ } else {
+ GotoIfNot(IsValidPositiveSmi(size_in_bytes), &if_out_of_memory);
+ }
+
+ TNode<RawPtrT> top = Load<RawPtrT>(top_address);
+ TNode<RawPtrT> limit = Load<RawPtrT>(limit_address);
+
+ // If there's not enough space, call the runtime.
+ TVARIABLE(Object, result);
+ Label runtime_call(this, Label::kDeferred), no_runtime_call(this), out(this);
+
+ bool needs_double_alignment = flags & kDoubleAlignment;
+ bool allow_large_object_allocation = flags & kAllowLargeObjectAllocation;
+
+ if (allow_large_object_allocation) {
+ Label next(this);
+ GotoIf(IsRegularHeapObjectSize(size_in_bytes), &next);
+
+ TNode<Smi> runtime_flags = SmiConstant(Smi::FromInt(
+ AllocateDoubleAlignFlag::encode(needs_double_alignment) |
+ AllowLargeObjectAllocationFlag::encode(allow_large_object_allocation)));
+ if (FLAG_young_generation_large_objects) {
+ result =
+ CallRuntime(Runtime::kAllocateInYoungGeneration, NoContextConstant(),
+ SmiTag(size_in_bytes), runtime_flags);
+ } else {
+ result =
+ CallRuntime(Runtime::kAllocateInOldGeneration, NoContextConstant(),
+ SmiTag(size_in_bytes), runtime_flags);
+ }
+ Goto(&out);
+
+ BIND(&next);
+ }
+
+ TVARIABLE(IntPtrT, adjusted_size, size_in_bytes);
+
+ if (needs_double_alignment) {
+ Label next(this);
+ GotoIfNot(WordAnd(top, IntPtrConstant(kDoubleAlignmentMask)), &next);
+
+ adjusted_size = IntPtrAdd(size_in_bytes, IntPtrConstant(4));
+ Goto(&next);
+
+ BIND(&next);
+ }
+
+ TNode<IntPtrT> new_top =
+ IntPtrAdd(UncheckedCast<IntPtrT>(top), adjusted_size.value());
+
+ Branch(UintPtrGreaterThanOrEqual(new_top, limit), &runtime_call,
+ &no_runtime_call);
+
+ BIND(&runtime_call);
+ {
+ TNode<Smi> runtime_flags = SmiConstant(Smi::FromInt(
+ AllocateDoubleAlignFlag::encode(needs_double_alignment) |
+ AllowLargeObjectAllocationFlag::encode(allow_large_object_allocation)));
+ if (flags & kPretenured) {
+ result =
+ CallRuntime(Runtime::kAllocateInOldGeneration, NoContextConstant(),
+ SmiTag(size_in_bytes), runtime_flags);
+ } else {
+ result =
+ CallRuntime(Runtime::kAllocateInYoungGeneration, NoContextConstant(),
+ SmiTag(size_in_bytes), runtime_flags);
+ }
+ Goto(&out);
+ }
+
+ // When there is enough space, return `top' and bump it up.
+ BIND(&no_runtime_call);
+ {
+ StoreNoWriteBarrier(MachineType::PointerRepresentation(), top_address,
+ new_top);
+
+ TVARIABLE(IntPtrT, address, UncheckedCast<IntPtrT>(top));
+
+ if (needs_double_alignment) {
+ Label next(this);
+ GotoIf(IntPtrEqual(adjusted_size.value(), size_in_bytes), &next);
+
+ // Store a filler and increase the address by 4.
+ StoreNoWriteBarrier(MachineRepresentation::kTagged, top,
+ OnePointerFillerMapConstant());
+ address = IntPtrAdd(UncheckedCast<IntPtrT>(top), IntPtrConstant(4));
+ Goto(&next);
+
+ BIND(&next);
+ }
+
+ result = BitcastWordToTagged(
+ IntPtrAdd(address.value(), IntPtrConstant(kHeapObjectTag)));
+ Goto(&out);
+ }
+
+ if (!size_in_bytes_is_constant) {
+ BIND(&if_out_of_memory);
+ CallRuntime(Runtime::kFatalProcessOutOfMemoryInAllocateRaw,
+ NoContextConstant());
+ Unreachable();
+ }
+
+ BIND(&out);
+ return UncheckedCast<HeapObject>(result.value());
+}
+
+TNode<HeapObject> CodeStubAssembler::AllocateRawUnaligned(
+ TNode<IntPtrT> size_in_bytes, AllocationFlags flags,
+ TNode<RawPtrT> top_address, TNode<RawPtrT> limit_address) {
+ DCHECK_EQ(flags & kDoubleAlignment, 0);
+ return AllocateRaw(size_in_bytes, flags, top_address, limit_address);
+}
+
+TNode<HeapObject> CodeStubAssembler::AllocateRawDoubleAligned(
+ TNode<IntPtrT> size_in_bytes, AllocationFlags flags,
+ TNode<RawPtrT> top_address, TNode<RawPtrT> limit_address) {
+#if defined(V8_HOST_ARCH_32_BIT)
+ return AllocateRaw(size_in_bytes, flags | kDoubleAlignment, top_address,
+ limit_address);
+#elif defined(V8_HOST_ARCH_64_BIT)
+#ifdef V8_COMPRESS_POINTERS
+ // TODO(ishell, v8:8875): Consider using aligned allocations once the
+ // allocation alignment inconsistency is fixed. For now we keep using
+ // unaligned access since both x64 and arm64 architectures (where pointer
+ // compression is supported) allow unaligned access to doubles and full words.
+#endif // V8_COMPRESS_POINTERS
+ // Allocation on 64 bit machine is naturally double aligned
+ return AllocateRaw(size_in_bytes, flags & ~kDoubleAlignment, top_address,
+ limit_address);
+#else
+#error Architecture not supported
+#endif
+}
+
+TNode<HeapObject> CodeStubAssembler::AllocateInNewSpace(
+ TNode<IntPtrT> size_in_bytes, AllocationFlags flags) {
+ DCHECK(flags == kNone || flags == kDoubleAlignment);
+ CSA_ASSERT(this, IsRegularHeapObjectSize(size_in_bytes));
+ return Allocate(size_in_bytes, flags);
+}
+
+TNode<HeapObject> CodeStubAssembler::Allocate(TNode<IntPtrT> size_in_bytes,
+ AllocationFlags flags) {
+ Comment("Allocate");
+ bool const new_space = !(flags & kPretenured);
+ bool const allow_large_objects = flags & kAllowLargeObjectAllocation;
+ // For optimized allocations, we don't allow the allocation to happen in a
+ // different generation than requested.
+ bool const always_allocated_in_requested_space =
+ !new_space || !allow_large_objects || FLAG_young_generation_large_objects;
+ if (!allow_large_objects) {
+ intptr_t size_constant;
+ if (ToIntPtrConstant(size_in_bytes, &size_constant)) {
+ CHECK_LE(size_constant, kMaxRegularHeapObjectSize);
+ } else {
+ CSA_ASSERT(this, IsRegularHeapObjectSize(size_in_bytes));
+ }
+ }
+ if (!(flags & kDoubleAlignment) && always_allocated_in_requested_space) {
+ return OptimizedAllocate(
+ size_in_bytes,
+ new_space ? AllocationType::kYoung : AllocationType::kOld,
+ allow_large_objects ? AllowLargeObjects::kTrue
+ : AllowLargeObjects::kFalse);
+ }
+ TNode<ExternalReference> top_address = ExternalConstant(
+ new_space
+ ? ExternalReference::new_space_allocation_top_address(isolate())
+ : ExternalReference::old_space_allocation_top_address(isolate()));
+ DCHECK_EQ(kSystemPointerSize,
+ ExternalReference::new_space_allocation_limit_address(isolate())
+ .address() -
+ ExternalReference::new_space_allocation_top_address(isolate())
+ .address());
+ DCHECK_EQ(kSystemPointerSize,
+ ExternalReference::old_space_allocation_limit_address(isolate())
+ .address() -
+ ExternalReference::old_space_allocation_top_address(isolate())
+ .address());
+ TNode<IntPtrT> limit_address =
+ IntPtrAdd(ReinterpretCast<IntPtrT>(top_address),
+ IntPtrConstant(kSystemPointerSize));
+
+ if (flags & kDoubleAlignment) {
+ return AllocateRawDoubleAligned(size_in_bytes, flags,
+ ReinterpretCast<RawPtrT>(top_address),
+ ReinterpretCast<RawPtrT>(limit_address));
+ } else {
+ return AllocateRawUnaligned(size_in_bytes, flags,
+ ReinterpretCast<RawPtrT>(top_address),
+ ReinterpretCast<RawPtrT>(limit_address));
+ }
+}
+
+TNode<HeapObject> CodeStubAssembler::AllocateInNewSpace(int size_in_bytes,
+ AllocationFlags flags) {
+ CHECK(flags == kNone || flags == kDoubleAlignment);
+ DCHECK_LE(size_in_bytes, kMaxRegularHeapObjectSize);
+ return CodeStubAssembler::Allocate(IntPtrConstant(size_in_bytes), flags);
+}
+
+TNode<HeapObject> CodeStubAssembler::Allocate(int size_in_bytes,
+ AllocationFlags flags) {
+ return CodeStubAssembler::Allocate(IntPtrConstant(size_in_bytes), flags);
+}
+
+TNode<HeapObject> CodeStubAssembler::InnerAllocate(TNode<HeapObject> previous,
+ TNode<IntPtrT> offset) {
+ return UncheckedCast<HeapObject>(
+ BitcastWordToTagged(IntPtrAdd(BitcastTaggedToWord(previous), offset)));
+}
+
+TNode<HeapObject> CodeStubAssembler::InnerAllocate(TNode<HeapObject> previous,
+ int offset) {
+ return InnerAllocate(previous, IntPtrConstant(offset));
+}
+
+TNode<BoolT> CodeStubAssembler::IsRegularHeapObjectSize(TNode<IntPtrT> size) {
+ return UintPtrLessThanOrEqual(size,
+ IntPtrConstant(kMaxRegularHeapObjectSize));
+}
+
+void CodeStubAssembler::BranchIfToBooleanIsTrue(SloppyTNode<Object> value,
+ Label* if_true,
+ Label* if_false) {
+ Label if_smi(this), if_notsmi(this), if_heapnumber(this, Label::kDeferred),
+ if_bigint(this, Label::kDeferred);
+ // Rule out false {value}.
+ GotoIf(TaggedEqual(value, FalseConstant()), if_false);
+
+ // Check if {value} is a Smi or a HeapObject.
+ Branch(TaggedIsSmi(value), &if_smi, &if_notsmi);
+
+ BIND(&if_smi);
+ {
+ // The {value} is a Smi, only need to check against zero.
+ BranchIfSmiEqual(CAST(value), SmiConstant(0), if_false, if_true);
+ }
+
+ BIND(&if_notsmi);
+ {
+ TNode<HeapObject> value_heapobject = CAST(value);
+
+ // Check if {value} is the empty string.
+ GotoIf(IsEmptyString(value_heapobject), if_false);
+
+ // The {value} is a HeapObject, load its map.
+ TNode<Map> value_map = LoadMap(value_heapobject);
+
+ // Only null, undefined and document.all have the undetectable bit set,
+ // so we can return false immediately when that bit is set.
+ GotoIf(IsUndetectableMap(value_map), if_false);
+
+ // We still need to handle numbers specially, but all other {value}s
+ // that make it here yield true.
+ GotoIf(IsHeapNumberMap(value_map), &if_heapnumber);
+ Branch(IsBigInt(value_heapobject), &if_bigint, if_true);
+
+ BIND(&if_heapnumber);
+ {
+ // Load the floating point value of {value}.
+ TNode<Float64T> value_value =
+ LoadObjectField<Float64T>(value_heapobject, HeapNumber::kValueOffset);
+
+ // Check if the floating point {value} is neither 0.0, -0.0 nor NaN.
+ Branch(Float64LessThan(Float64Constant(0.0), Float64Abs(value_value)),
+ if_true, if_false);
+ }
+
+ BIND(&if_bigint);
+ {
+ TNode<BigInt> bigint = CAST(value);
+ TNode<Word32T> bitfield = LoadBigIntBitfield(bigint);
+ TNode<Uint32T> length = DecodeWord32<BigIntBase::LengthBits>(bitfield);
+ Branch(Word32Equal(length, Int32Constant(0)), if_false, if_true);
+ }
+ }
+}
+
+TNode<ExternalPointerT> CodeStubAssembler::ChangeUint32ToExternalPointer(
+ TNode<Uint32T> value) {
+ STATIC_ASSERT(kExternalPointerSize == kSystemPointerSize);
+ return ReinterpretCast<ExternalPointerT>(ChangeUint32ToWord(value));
+}
+
+TNode<Uint32T> CodeStubAssembler::ChangeExternalPointerToUint32(
+ TNode<ExternalPointerT> value) {
+ STATIC_ASSERT(kExternalPointerSize == kSystemPointerSize);
+ return Unsigned(TruncateWordToInt32(ReinterpretCast<UintPtrT>(value)));
+}
+
+void CodeStubAssembler::InitializeExternalPointerField(TNode<HeapObject> object,
+ TNode<IntPtrT> offset) {
+#ifdef V8_HEAP_SANDBOX
+ TNode<ExternalReference> external_pointer_table_address = ExternalConstant(
+ ExternalReference::external_pointer_table_address(isolate()));
+ TNode<Uint32T> table_length = UncheckedCast<Uint32T>(
+ Load(MachineType::Uint32(), external_pointer_table_address,
+ UintPtrConstant(Internals::kExternalPointerTableLengthOffset)));
+ TNode<Uint32T> table_capacity = UncheckedCast<Uint32T>(
+ Load(MachineType::Uint32(), external_pointer_table_address,
+ UintPtrConstant(Internals::kExternalPointerTableCapacityOffset)));
+
+ Label grow_table(this, Label::kDeferred), finish(this);
+
+ TNode<BoolT> compare = Uint32LessThan(table_length, table_capacity);
+ Branch(compare, &finish, &grow_table);
+
+ BIND(&grow_table);
+ {
+ TNode<ExternalReference> table_grow_function = ExternalConstant(
+ ExternalReference::external_pointer_table_grow_table_function());
+ CallCFunction(
+ table_grow_function, MachineType::Pointer(),
+ std::make_pair(MachineType::Pointer(), external_pointer_table_address));
+ Goto(&finish);
+ }
+ BIND(&finish);
+
+ TNode<Uint32T> new_table_length = Uint32Add(table_length, Uint32Constant(1));
+ StoreNoWriteBarrier(
+ MachineRepresentation::kWord32, external_pointer_table_address,
+ UintPtrConstant(Internals::kExternalPointerTableLengthOffset),
+ new_table_length);
+
+ TNode<Uint32T> index = table_length;
+ TNode<ExternalPointerT> encoded = ChangeUint32ToExternalPointer(index);
+ StoreObjectFieldNoWriteBarrier<ExternalPointerT>(object, offset, encoded);
+#endif
+}
+
+TNode<RawPtrT> CodeStubAssembler::LoadExternalPointerFromObject(
+ TNode<HeapObject> object, TNode<IntPtrT> offset,
+ ExternalPointerTag external_pointer_tag) {
+#ifdef V8_HEAP_SANDBOX
+ TNode<ExternalReference> external_pointer_table_address = ExternalConstant(
+ ExternalReference::external_pointer_table_address(isolate()));
+ TNode<RawPtrT> table = UncheckedCast<RawPtrT>(
+ Load(MachineType::Pointer(), external_pointer_table_address,
+ UintPtrConstant(Internals::kExternalPointerTableBufferOffset)));
+
+ TNode<ExternalPointerT> encoded =
+ LoadObjectField<ExternalPointerT>(object, offset);
+ TNode<Word32T> index = ChangeExternalPointerToUint32(encoded);
+ // TODO(v8:10391, saelo): bounds check if table is not caged
+ TNode<IntPtrT> table_offset = ElementOffsetFromIndex(
+ ChangeUint32ToWord(index), SYSTEM_POINTER_ELEMENTS, 0);
+
+ TNode<UintPtrT> entry = Load<UintPtrT>(table, table_offset);
+ if (external_pointer_tag != 0) {
+ TNode<UintPtrT> tag = UintPtrConstant(external_pointer_tag);
+ entry = UncheckedCast<UintPtrT>(WordXor(entry, tag));
+ }
+ return UncheckedCast<RawPtrT>(UncheckedCast<WordT>(entry));
+#else
+ return LoadObjectField<RawPtrT>(object, offset);
+#endif // V8_HEAP_SANDBOX
+}
+
+void CodeStubAssembler::StoreExternalPointerToObject(
+ TNode<HeapObject> object, TNode<IntPtrT> offset, TNode<RawPtrT> pointer,
+ ExternalPointerTag external_pointer_tag) {
+#ifdef V8_HEAP_SANDBOX
+ TNode<ExternalReference> external_pointer_table_address = ExternalConstant(
+ ExternalReference::external_pointer_table_address(isolate()));
+ TNode<RawPtrT> table = UncheckedCast<RawPtrT>(
+ Load(MachineType::Pointer(), external_pointer_table_address,
+ UintPtrConstant(Internals::kExternalPointerTableBufferOffset)));
+
+ TNode<ExternalPointerT> encoded =
+ LoadObjectField<ExternalPointerT>(object, offset);
+ TNode<Word32T> index = ChangeExternalPointerToUint32(encoded);
+ // TODO(v8:10391, saelo): bounds check if table is not caged
+ TNode<IntPtrT> table_offset = ElementOffsetFromIndex(
+ ChangeUint32ToWord(index), SYSTEM_POINTER_ELEMENTS, 0);
+
+ TNode<UintPtrT> value = UncheckedCast<UintPtrT>(pointer);
+ if (external_pointer_tag != 0) {
+ TNode<UintPtrT> tag = UintPtrConstant(external_pointer_tag);
+ value = UncheckedCast<UintPtrT>(WordXor(pointer, tag));
+ }
+ StoreNoWriteBarrier(MachineType::PointerRepresentation(), table, table_offset,
+ value);
+#else
+ StoreObjectFieldNoWriteBarrier<RawPtrT>(object, offset, pointer);
+#endif // V8_HEAP_SANDBOX
+}
+
+TNode<Object> CodeStubAssembler::LoadFromParentFrame(int offset) {
+ TNode<RawPtrT> frame_pointer = LoadParentFramePointer();
+ return LoadFullTagged(frame_pointer, IntPtrConstant(offset));
+}
+
+TNode<IntPtrT> CodeStubAssembler::LoadAndUntagObjectField(
+ TNode<HeapObject> object, int offset) {
+ if (SmiValuesAre32Bits()) {
+#if V8_TARGET_LITTLE_ENDIAN
+ offset += 4;
+#endif
+ return ChangeInt32ToIntPtr(LoadObjectField<Int32T>(object, offset));
+ } else {
+ return SmiToIntPtr(LoadObjectField<Smi>(object, offset));
+ }
+}
+
+TNode<Int32T> CodeStubAssembler::LoadAndUntagToWord32ObjectField(
+ TNode<HeapObject> object, int offset) {
+ if (SmiValuesAre32Bits()) {
+#if V8_TARGET_LITTLE_ENDIAN
+ offset += 4;
+#endif
+ return LoadObjectField<Int32T>(object, offset);
+ } else {
+ return SmiToInt32(LoadObjectField<Smi>(object, offset));
+ }
+}
+
+TNode<Float64T> CodeStubAssembler::LoadHeapNumberValue(
+ TNode<HeapObject> object) {
+ CSA_ASSERT(this, Word32Or(IsHeapNumber(object), IsOddball(object)));
+ STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
+ return LoadObjectField<Float64T>(object, HeapNumber::kValueOffset);
+}
+
+TNode<Map> CodeStubAssembler::GetInstanceTypeMap(InstanceType instance_type) {
+ Handle<Map> map_handle(
+ Map::GetInstanceTypeMap(ReadOnlyRoots(isolate()), instance_type),
+ isolate());
+ return HeapConstant(map_handle);
+}
+
+TNode<Map> CodeStubAssembler::LoadMap(TNode<HeapObject> object) {
+ return LoadObjectField<Map>(object, HeapObject::kMapOffset);
+}
+
+TNode<Uint16T> CodeStubAssembler::LoadInstanceType(TNode<HeapObject> object) {
+ return LoadMapInstanceType(LoadMap(object));
+}
+
+TNode<BoolT> CodeStubAssembler::HasInstanceType(TNode<HeapObject> object,
+ InstanceType instance_type) {
+ return InstanceTypeEqual(LoadInstanceType(object), instance_type);
+}
+
+TNode<BoolT> CodeStubAssembler::DoesntHaveInstanceType(
+ TNode<HeapObject> object, InstanceType instance_type) {
+ return Word32NotEqual(LoadInstanceType(object), Int32Constant(instance_type));
+}
+
+TNode<BoolT> CodeStubAssembler::TaggedDoesntHaveInstanceType(
+ TNode<HeapObject> any_tagged, InstanceType type) {
+ /* return Phi <TaggedIsSmi(val), DoesntHaveInstanceType(val, type)> */
+ TNode<BoolT> tagged_is_smi = TaggedIsSmi(any_tagged);
+ return Select<BoolT>(
+ tagged_is_smi, [=]() { return tagged_is_smi; },
+ [=]() { return DoesntHaveInstanceType(any_tagged, type); });
+}
+
+TNode<BoolT> CodeStubAssembler::IsSpecialReceiverMap(TNode<Map> map) {
+ TNode<BoolT> is_special =
+ IsSpecialReceiverInstanceType(LoadMapInstanceType(map));
+ uint32_t mask = Map::Bits1::HasNamedInterceptorBit::kMask |
+ Map::Bits1::IsAccessCheckNeededBit::kMask;
+ USE(mask);
+ // Interceptors or access checks imply special receiver.
+ CSA_ASSERT(this,
+ SelectConstant<BoolT>(IsSetWord32(LoadMapBitField(map), mask),
+ is_special, Int32TrueConstant()));
+ return is_special;
+}
+
+TNode<Word32T> CodeStubAssembler::IsStringWrapperElementsKind(TNode<Map> map) {
+ TNode<Int32T> kind = LoadMapElementsKind(map);
+ return Word32Or(
+ Word32Equal(kind, Int32Constant(FAST_STRING_WRAPPER_ELEMENTS)),
+ Word32Equal(kind, Int32Constant(SLOW_STRING_WRAPPER_ELEMENTS)));
+}
+
+void CodeStubAssembler::GotoIfMapHasSlowProperties(TNode<Map> map,
+ Label* if_slow) {
+ GotoIf(IsStringWrapperElementsKind(map), if_slow);
+ GotoIf(IsSpecialReceiverMap(map), if_slow);
+ GotoIf(IsDictionaryMap(map), if_slow);
+}
+
+TNode<HeapObject> CodeStubAssembler::LoadFastProperties(
+ TNode<JSReceiver> object) {
+ CSA_SLOW_ASSERT(this, Word32BinaryNot(IsDictionaryMap(LoadMap(object))));
+ TNode<Object> properties = LoadJSReceiverPropertiesOrHash(object);
+ return Select<HeapObject>(
+ TaggedIsSmi(properties), [=] { return EmptyFixedArrayConstant(); },
+ [=] { return CAST(properties); });
+}
+
+TNode<HeapObject> CodeStubAssembler::LoadSlowProperties(
+ TNode<JSReceiver> object) {
+ CSA_SLOW_ASSERT(this, IsDictionaryMap(LoadMap(object)));
+ TNode<Object> properties = LoadJSReceiverPropertiesOrHash(object);
+ return Select<HeapObject>(
+ TaggedIsSmi(properties),
+ [=] { return EmptyPropertyDictionaryConstant(); },
+ [=] { return CAST(properties); });
+}
+
+TNode<Object> CodeStubAssembler::LoadJSArgumentsObjectLength(
+ TNode<Context> context, TNode<JSArgumentsObject> array) {
+ CSA_ASSERT(this, IsJSArgumentsObjectWithLength(context, array));
+ constexpr int offset = JSStrictArgumentsObject::kLengthOffset;
+ STATIC_ASSERT(offset == JSSloppyArgumentsObject::kLengthOffset);
+ return LoadObjectField(array, offset);
+}
+
+TNode<Smi> CodeStubAssembler::LoadFastJSArrayLength(TNode<JSArray> array) {
+ TNode<Number> length = LoadJSArrayLength(array);
+ CSA_ASSERT(this, Word32Or(IsFastElementsKind(LoadElementsKind(array)),
+ IsElementsKindInRange(
+ LoadElementsKind(array),
+ FIRST_ANY_NONEXTENSIBLE_ELEMENTS_KIND,
+ LAST_ANY_NONEXTENSIBLE_ELEMENTS_KIND)));
+ // JSArray length is always a positive Smi for fast arrays.
+ CSA_SLOW_ASSERT(this, TaggedIsPositiveSmi(length));
+ return CAST(length);
+}
+
+TNode<Smi> CodeStubAssembler::LoadFixedArrayBaseLength(
+ TNode<FixedArrayBase> array) {
+ CSA_SLOW_ASSERT(this, IsNotWeakFixedArraySubclass(array));
+ return LoadObjectField<Smi>(array, FixedArrayBase::kLengthOffset);
+}
+
+TNode<IntPtrT> CodeStubAssembler::LoadAndUntagFixedArrayBaseLength(
+ TNode<FixedArrayBase> array) {
+ return LoadAndUntagObjectField(array, FixedArrayBase::kLengthOffset);
+}
+
+TNode<IntPtrT> CodeStubAssembler::LoadFeedbackVectorLength(
+ TNode<FeedbackVector> vector) {
+ return ChangeInt32ToIntPtr(
+ LoadObjectField<Int32T>(vector, FeedbackVector::kLengthOffset));
+}
+
+TNode<Smi> CodeStubAssembler::LoadWeakFixedArrayLength(
+ TNode<WeakFixedArray> array) {
+ return LoadObjectField<Smi>(array, WeakFixedArray::kLengthOffset);
+}
+
+TNode<IntPtrT> CodeStubAssembler::LoadAndUntagWeakFixedArrayLength(
+ TNode<WeakFixedArray> array) {
+ return LoadAndUntagObjectField(array, WeakFixedArray::kLengthOffset);
+}
+
+TNode<Int32T> CodeStubAssembler::LoadNumberOfDescriptors(
+ TNode<DescriptorArray> array) {
+ return UncheckedCast<Int32T>(LoadObjectField<Int16T>(
+ array, DescriptorArray::kNumberOfDescriptorsOffset));
+}
+
+TNode<Int32T> CodeStubAssembler::LoadNumberOfOwnDescriptors(TNode<Map> map) {
+ TNode<Uint32T> bit_field3 = LoadMapBitField3(map);
+ return UncheckedCast<Int32T>(
+ DecodeWord32<Map::Bits3::NumberOfOwnDescriptorsBits>(bit_field3));
+}
+
+TNode<Int32T> CodeStubAssembler::LoadMapBitField(TNode<Map> map) {
+ return UncheckedCast<Int32T>(
+ LoadObjectField<Uint8T>(map, Map::kBitFieldOffset));
+}
+
+TNode<Int32T> CodeStubAssembler::LoadMapBitField2(TNode<Map> map) {
+ return UncheckedCast<Int32T>(
+ LoadObjectField<Uint8T>(map, Map::kBitField2Offset));
+}
+
+TNode<Uint32T> CodeStubAssembler::LoadMapBitField3(TNode<Map> map) {
+ return LoadObjectField<Uint32T>(map, Map::kBitField3Offset);
+}
+
+TNode<Uint16T> CodeStubAssembler::LoadMapInstanceType(TNode<Map> map) {
+ return LoadObjectField<Uint16T>(map, Map::kInstanceTypeOffset);
+}
+
+TNode<Int32T> CodeStubAssembler::LoadMapElementsKind(TNode<Map> map) {
+ TNode<Int32T> bit_field2 = LoadMapBitField2(map);
+ return Signed(DecodeWord32<Map::Bits2::ElementsKindBits>(bit_field2));
+}
+
+TNode<Int32T> CodeStubAssembler::LoadElementsKind(TNode<HeapObject> object) {
+ return LoadMapElementsKind(LoadMap(object));
+}
+
+TNode<DescriptorArray> CodeStubAssembler::LoadMapDescriptors(TNode<Map> map) {
+ return LoadObjectField<DescriptorArray>(map, Map::kInstanceDescriptorsOffset);
+}
+
+TNode<HeapObject> CodeStubAssembler::LoadMapPrototype(TNode<Map> map) {
+ return LoadObjectField<HeapObject>(map, Map::kPrototypeOffset);
+}
+
+TNode<IntPtrT> CodeStubAssembler::LoadMapInstanceSizeInWords(TNode<Map> map) {
+ return ChangeInt32ToIntPtr(
+ LoadObjectField<Uint8T>(map, Map::kInstanceSizeInWordsOffset));
+}
+
+TNode<IntPtrT> CodeStubAssembler::LoadMapInobjectPropertiesStartInWords(
+ TNode<Map> map) {
+ // See Map::GetInObjectPropertiesStartInWords() for details.
+ CSA_ASSERT(this, IsJSObjectMap(map));
+ return ChangeInt32ToIntPtr(LoadObjectField<Uint8T>(
+ map, Map::kInObjectPropertiesStartOrConstructorFunctionIndexOffset));
+}
+
+TNode<IntPtrT> CodeStubAssembler::LoadMapConstructorFunctionIndex(
+ TNode<Map> map) {
+ // See Map::GetConstructorFunctionIndex() for details.
+ CSA_ASSERT(this, IsPrimitiveInstanceType(LoadMapInstanceType(map)));
+ return ChangeInt32ToIntPtr(LoadObjectField<Uint8T>(
+ map, Map::kInObjectPropertiesStartOrConstructorFunctionIndexOffset));
+}
+
+TNode<Object> CodeStubAssembler::LoadMapConstructor(TNode<Map> map) {
+ TVARIABLE(Object, result,
+ LoadObjectField(
+ map, Map::kConstructorOrBackPointerOrNativeContextOffset));
+
+ Label done(this), loop(this, &result);
+ Goto(&loop);
+ BIND(&loop);
+ {
+ GotoIf(TaggedIsSmi(result.value()), &done);
+ TNode<BoolT> is_map_type =
+ InstanceTypeEqual(LoadInstanceType(CAST(result.value())), MAP_TYPE);
+ GotoIfNot(is_map_type, &done);
+ result =
+ LoadObjectField(CAST(result.value()),
+ Map::kConstructorOrBackPointerOrNativeContextOffset);
+ Goto(&loop);
+ }
+ BIND(&done);
+ return result.value();
+}
+
+TNode<WordT> CodeStubAssembler::LoadMapEnumLength(TNode<Map> map) {
+ TNode<Uint32T> bit_field3 = LoadMapBitField3(map);
+ return DecodeWordFromWord32<Map::Bits3::EnumLengthBits>(bit_field3);
+}
+
+TNode<Object> CodeStubAssembler::LoadMapBackPointer(TNode<Map> map) {
+ TNode<HeapObject> object = CAST(LoadObjectField(
+ map, Map::kConstructorOrBackPointerOrNativeContextOffset));
+ return Select<Object>(
+ IsMap(object), [=] { return object; },
+ [=] { return UndefinedConstant(); });
+}
+
+TNode<Uint32T> CodeStubAssembler::EnsureOnlyHasSimpleProperties(
+ TNode<Map> map, TNode<Int32T> instance_type, Label* bailout) {
+ // This check can have false positives, since it applies to any
+ // JSPrimitiveWrapper type.
+ GotoIf(IsCustomElementsReceiverInstanceType(instance_type), bailout);
+
+ TNode<Uint32T> bit_field3 = LoadMapBitField3(map);
+ GotoIf(IsSetWord32(bit_field3, Map::Bits3::IsDictionaryMapBit::kMask),
+ bailout);
+
+ return bit_field3;
+}
+
+TNode<IntPtrT> CodeStubAssembler::LoadJSReceiverIdentityHash(
+ SloppyTNode<Object> receiver, Label* if_no_hash) {
+ TVARIABLE(IntPtrT, var_hash);
+ Label done(this), if_smi(this), if_property_array(this),
+ if_property_dictionary(this), if_fixed_array(this);
+
+ TNode<Object> properties_or_hash =
+ LoadObjectField(TNode<HeapObject>::UncheckedCast(receiver),
+ JSReceiver::kPropertiesOrHashOffset);
+ GotoIf(TaggedIsSmi(properties_or_hash), &if_smi);
+
+ TNode<HeapObject> properties =
+ TNode<HeapObject>::UncheckedCast(properties_or_hash);
+ TNode<Uint16T> properties_instance_type = LoadInstanceType(properties);
+
+ GotoIf(InstanceTypeEqual(properties_instance_type, PROPERTY_ARRAY_TYPE),
+ &if_property_array);
+ Branch(InstanceTypeEqual(properties_instance_type, NAME_DICTIONARY_TYPE),
+ &if_property_dictionary, &if_fixed_array);
+
+ BIND(&if_fixed_array);
+ {
+ var_hash = IntPtrConstant(PropertyArray::kNoHashSentinel);
+ Goto(&done);
+ }
+
+ BIND(&if_smi);
+ {
+ var_hash = SmiUntag(TNode<Smi>::UncheckedCast(properties_or_hash));
+ Goto(&done);
+ }
+
+ BIND(&if_property_array);
+ {
+ TNode<IntPtrT> length_and_hash = LoadAndUntagObjectField(
+ properties, PropertyArray::kLengthAndHashOffset);
+ var_hash = TNode<IntPtrT>::UncheckedCast(
+ DecodeWord<PropertyArray::HashField>(length_and_hash));
+ Goto(&done);
+ }
+
+ BIND(&if_property_dictionary);
+ {
+ var_hash = SmiUntag(CAST(LoadFixedArrayElement(
+ CAST(properties), NameDictionary::kObjectHashIndex)));
+ Goto(&done);
+ }
+
+ BIND(&done);
+ if (if_no_hash != nullptr) {
+ GotoIf(IntPtrEqual(var_hash.value(),
+ IntPtrConstant(PropertyArray::kNoHashSentinel)),
+ if_no_hash);
+ }
+ return var_hash.value();
+}
+
+TNode<Uint32T> CodeStubAssembler::LoadNameHashAssumeComputed(TNode<Name> name) {
+ TNode<Uint32T> hash_field = LoadNameHashField(name);
+ CSA_ASSERT(this, IsClearWord32(hash_field, Name::kHashNotComputedMask));
+ return Unsigned(Word32Shr(hash_field, Int32Constant(Name::kHashShift)));
+}
+
+TNode<Uint32T> CodeStubAssembler::LoadNameHash(TNode<Name> name,
+ Label* if_hash_not_computed) {
+ TNode<Uint32T> hash_field = LoadNameHashField(name);
+ if (if_hash_not_computed != nullptr) {
+ GotoIf(IsSetWord32(hash_field, Name::kHashNotComputedMask),
+ if_hash_not_computed);
+ }
+ return Unsigned(Word32Shr(hash_field, Int32Constant(Name::kHashShift)));
+}
+
+TNode<Smi> CodeStubAssembler::LoadStringLengthAsSmi(TNode<String> string) {
+ return SmiFromIntPtr(LoadStringLengthAsWord(string));
+}
+
+TNode<IntPtrT> CodeStubAssembler::LoadStringLengthAsWord(TNode<String> string) {
+ return Signed(ChangeUint32ToWord(LoadStringLengthAsWord32(string)));
+}
+
+TNode<Uint32T> CodeStubAssembler::LoadStringLengthAsWord32(
+ TNode<String> string) {
+ return LoadObjectField<Uint32T>(string, String::kLengthOffset);
+}
+
+TNode<Object> CodeStubAssembler::LoadJSPrimitiveWrapperValue(
+ TNode<JSPrimitiveWrapper> object) {
+ return LoadObjectField(object, JSPrimitiveWrapper::kValueOffset);
+}
+
+void CodeStubAssembler::DispatchMaybeObject(TNode<MaybeObject> maybe_object,
+ Label* if_smi, Label* if_cleared,
+ Label* if_weak, Label* if_strong,
+ TVariable<Object>* extracted) {
+ Label inner_if_smi(this), inner_if_strong(this);
+
+ GotoIf(TaggedIsSmi(maybe_object), &inner_if_smi);
+
+ GotoIf(IsCleared(maybe_object), if_cleared);
+
+ GotoIf(IsStrong(maybe_object), &inner_if_strong);
+
+ *extracted = GetHeapObjectAssumeWeak(maybe_object);
+ Goto(if_weak);
+
+ BIND(&inner_if_smi);
+ *extracted = CAST(maybe_object);
+ Goto(if_smi);
+
+ BIND(&inner_if_strong);
+ *extracted = CAST(maybe_object);
+ Goto(if_strong);
+}
+
+TNode<BoolT> CodeStubAssembler::IsStrong(TNode<MaybeObject> value) {
+ return Word32Equal(Word32And(TruncateIntPtrToInt32(
+ BitcastTaggedToWordForTagAndSmiBits(value)),
+ Int32Constant(kHeapObjectTagMask)),
+ Int32Constant(kHeapObjectTag));
+}
+
+TNode<HeapObject> CodeStubAssembler::GetHeapObjectIfStrong(
+ TNode<MaybeObject> value, Label* if_not_strong) {
+ GotoIfNot(IsStrong(value), if_not_strong);
+ return CAST(value);
+}
+
+TNode<BoolT> CodeStubAssembler::IsWeakOrCleared(TNode<MaybeObject> value) {
+ return Word32Equal(Word32And(TruncateIntPtrToInt32(
+ BitcastTaggedToWordForTagAndSmiBits(value)),
+ Int32Constant(kHeapObjectTagMask)),
+ Int32Constant(kWeakHeapObjectTag));
+}
+
+TNode<BoolT> CodeStubAssembler::IsCleared(TNode<MaybeObject> value) {
+ return Word32Equal(TruncateIntPtrToInt32(BitcastMaybeObjectToWord(value)),
+ Int32Constant(kClearedWeakHeapObjectLower32));
+}
+
+TNode<HeapObject> CodeStubAssembler::GetHeapObjectAssumeWeak(
+ TNode<MaybeObject> value) {
+ CSA_ASSERT(this, IsWeakOrCleared(value));
+ CSA_ASSERT(this, IsNotCleared(value));
+ return UncheckedCast<HeapObject>(BitcastWordToTagged(WordAnd(
+ BitcastMaybeObjectToWord(value), IntPtrConstant(~kWeakHeapObjectMask))));
+}
+
+TNode<HeapObject> CodeStubAssembler::GetHeapObjectAssumeWeak(
+ TNode<MaybeObject> value, Label* if_cleared) {
+ GotoIf(IsCleared(value), if_cleared);
+ return GetHeapObjectAssumeWeak(value);
+}
+
+// This version generates
+// (maybe_object & ~mask) == value
+// It works for non-Smi |maybe_object| and for both Smi and HeapObject values
+// but requires a big constant for ~mask.
+TNode<BoolT> CodeStubAssembler::IsWeakReferenceToObject(
+ TNode<MaybeObject> maybe_object, TNode<Object> value) {
+ CSA_ASSERT(this, TaggedIsNotSmi(maybe_object));
+ if (COMPRESS_POINTERS_BOOL) {
+ return Word32Equal(
+ Word32And(TruncateWordToInt32(BitcastMaybeObjectToWord(maybe_object)),
+ Uint32Constant(~static_cast<uint32_t>(kWeakHeapObjectMask))),
+ TruncateWordToInt32(BitcastTaggedToWord(value)));
+ } else {
+ return WordEqual(WordAnd(BitcastMaybeObjectToWord(maybe_object),
+ IntPtrConstant(~kWeakHeapObjectMask)),
+ BitcastTaggedToWord(value));
+ }
+}
+
+// This version generates
+// maybe_object == (heap_object | mask)
+// It works for any |maybe_object| values and generates a better code because it
+// uses a small constant for mask.
+TNode<BoolT> CodeStubAssembler::IsWeakReferenceTo(
+ TNode<MaybeObject> maybe_object, TNode<HeapObject> heap_object) {
+ if (COMPRESS_POINTERS_BOOL) {
+ return Word32Equal(
+ TruncateWordToInt32(BitcastMaybeObjectToWord(maybe_object)),
+ Word32Or(TruncateWordToInt32(BitcastTaggedToWord(heap_object)),
+ Int32Constant(kWeakHeapObjectMask)));
+ } else {
+ return WordEqual(BitcastMaybeObjectToWord(maybe_object),
+ WordOr(BitcastTaggedToWord(heap_object),
+ IntPtrConstant(kWeakHeapObjectMask)));
+ }
+}
+
+TNode<MaybeObject> CodeStubAssembler::MakeWeak(TNode<HeapObject> value) {
+ return ReinterpretCast<MaybeObject>(BitcastWordToTagged(
+ WordOr(BitcastTaggedToWord(value), IntPtrConstant(kWeakHeapObjectTag))));
+}
+
+template <>
+TNode<IntPtrT> CodeStubAssembler::LoadArrayLength(TNode<FixedArray> array) {
+ return LoadAndUntagFixedArrayBaseLength(array);
+}
+
+template <>
+TNode<IntPtrT> CodeStubAssembler::LoadArrayLength(TNode<WeakFixedArray> array) {
+ return LoadAndUntagWeakFixedArrayLength(array);
+}
+
+template <>
+TNode<IntPtrT> CodeStubAssembler::LoadArrayLength(TNode<PropertyArray> array) {
+ return LoadPropertyArrayLength(array);
+}
+
+template <>
+TNode<IntPtrT> CodeStubAssembler::LoadArrayLength(
+ TNode<DescriptorArray> array) {
+ return IntPtrMul(ChangeInt32ToIntPtr(LoadNumberOfDescriptors(array)),
+ IntPtrConstant(DescriptorArray::kEntrySize));
+}
+
+template <>
+TNode<IntPtrT> CodeStubAssembler::LoadArrayLength(
+ TNode<TransitionArray> array) {
+ return LoadAndUntagWeakFixedArrayLength(array);
+}
+
+template <typename Array, typename TIndex, typename TValue>
+TNode<TValue> CodeStubAssembler::LoadArrayElement(
+ TNode<Array> array, int array_header_size, TNode<TIndex> index_node,
+ int additional_offset, LoadSensitivity needs_poisoning) {
+ // TODO(v8:9708): Do we want to keep both IntPtrT and UintPtrT variants?
+ static_assert(std::is_same<TIndex, Smi>::value ||
+ std::is_same<TIndex, UintPtrT>::value ||
+ std::is_same<TIndex, IntPtrT>::value,
+ "Only Smi, UintPtrT or IntPtrT indices are allowed");
+ CSA_ASSERT(this, IntPtrGreaterThanOrEqual(ParameterToIntPtr(index_node),
+ IntPtrConstant(0)));
+ DCHECK(IsAligned(additional_offset, kTaggedSize));
+ int32_t header_size = array_header_size + additional_offset - kHeapObjectTag;
+ TNode<IntPtrT> offset =
+ ElementOffsetFromIndex(index_node, HOLEY_ELEMENTS, header_size);
+ CSA_ASSERT(this, IsOffsetInBounds(offset, LoadArrayLength(array),
+ array_header_size));
+ constexpr MachineType machine_type = MachineTypeOf<TValue>::value;
+ // TODO(gsps): Remove the Load case once LoadFromObject supports poisoning
+ if (needs_poisoning == LoadSensitivity::kSafe) {
+ return UncheckedCast<TValue>(LoadFromObject(machine_type, array, offset));
+ } else {
+ return UncheckedCast<TValue>(
+ Load(machine_type, array, offset, needs_poisoning));
+ }
+}
+
+template V8_EXPORT_PRIVATE TNode<MaybeObject>
+CodeStubAssembler::LoadArrayElement<TransitionArray, IntPtrT>(
+ TNode<TransitionArray>, int, TNode<IntPtrT>, int, LoadSensitivity);
+
+template <typename TIndex>
+TNode<Object> CodeStubAssembler::LoadFixedArrayElement(
+ TNode<FixedArray> object, TNode<TIndex> index, int additional_offset,
+ LoadSensitivity needs_poisoning, CheckBounds check_bounds) {
+ // TODO(v8:9708): Do we want to keep both IntPtrT and UintPtrT variants?
+ static_assert(std::is_same<TIndex, Smi>::value ||
+ std::is_same<TIndex, UintPtrT>::value ||
+ std::is_same<TIndex, IntPtrT>::value,
+ "Only Smi, UintPtrT or IntPtrT indexes are allowed");
+ CSA_ASSERT(this, IsFixedArraySubclass(object));
+ CSA_ASSERT(this, IsNotWeakFixedArraySubclass(object));
+
+ if (NeedsBoundsCheck(check_bounds)) {
+ FixedArrayBoundsCheck(object, index, additional_offset);
+ }
+ TNode<MaybeObject> element =
+ LoadArrayElement(object, FixedArray::kHeaderSize, index,
+ additional_offset, needs_poisoning);
+ return CAST(element);
+}
+
+template V8_EXPORT_PRIVATE TNode<Object>
+CodeStubAssembler::LoadFixedArrayElement<Smi>(TNode<FixedArray>, TNode<Smi>,
+ int, LoadSensitivity,
+ CheckBounds);
+template V8_EXPORT_PRIVATE TNode<Object>
+CodeStubAssembler::LoadFixedArrayElement<UintPtrT>(TNode<FixedArray>,
+ TNode<UintPtrT>, int,
+ LoadSensitivity,
+ CheckBounds);
+template V8_EXPORT_PRIVATE TNode<Object>
+CodeStubAssembler::LoadFixedArrayElement<IntPtrT>(TNode<FixedArray>,
+ TNode<IntPtrT>, int,
+ LoadSensitivity, CheckBounds);
+
+void CodeStubAssembler::FixedArrayBoundsCheck(TNode<FixedArrayBase> array,
+ TNode<Smi> index,
+ int additional_offset) {
+ if (!FLAG_fixed_array_bounds_checks) return;
+ DCHECK(IsAligned(additional_offset, kTaggedSize));
+ TNode<Smi> effective_index;
+ Smi constant_index;
+ bool index_is_constant = ToSmiConstant(index, &constant_index);
+ if (index_is_constant) {
+ effective_index = SmiConstant(Smi::ToInt(constant_index) +
+ additional_offset / kTaggedSize);
+ } else {
+ effective_index =
+ SmiAdd(index, SmiConstant(additional_offset / kTaggedSize));
+ }
+ CSA_CHECK(this, SmiBelow(effective_index, LoadFixedArrayBaseLength(array)));
+}
+
+void CodeStubAssembler::FixedArrayBoundsCheck(TNode<FixedArrayBase> array,
+ TNode<IntPtrT> index,
+ int additional_offset) {
+ if (!FLAG_fixed_array_bounds_checks) return;
+ DCHECK(IsAligned(additional_offset, kTaggedSize));
+ // IntPtrAdd does constant-folding automatically.
+ TNode<IntPtrT> effective_index =
+ IntPtrAdd(index, IntPtrConstant(additional_offset / kTaggedSize));
+ CSA_CHECK(this, UintPtrLessThan(effective_index,
+ LoadAndUntagFixedArrayBaseLength(array)));
+}
+
+TNode<Object> CodeStubAssembler::LoadPropertyArrayElement(
+ TNode<PropertyArray> object, SloppyTNode<IntPtrT> index) {
+ int additional_offset = 0;
+ LoadSensitivity needs_poisoning = LoadSensitivity::kSafe;
+ return CAST(LoadArrayElement(object, PropertyArray::kHeaderSize, index,
+ additional_offset, needs_poisoning));
+}
+
+TNode<IntPtrT> CodeStubAssembler::LoadPropertyArrayLength(
+ TNode<PropertyArray> object) {
+ TNode<IntPtrT> value =
+ LoadAndUntagObjectField(object, PropertyArray::kLengthAndHashOffset);
+ return Signed(DecodeWord<PropertyArray::LengthField>(value));
+}
+
+TNode<RawPtrT> CodeStubAssembler::LoadJSTypedArrayDataPtr(
+ TNode<JSTypedArray> typed_array) {
+ // Data pointer = external_pointer + static_cast<Tagged_t>(base_pointer).
+ TNode<RawPtrT> external_pointer =
+ LoadJSTypedArrayExternalPointerPtr(typed_array);
+
+ TNode<IntPtrT> base_pointer;
+ if (COMPRESS_POINTERS_BOOL) {
+ TNode<Int32T> compressed_base =
+ LoadObjectField<Int32T>(typed_array, JSTypedArray::kBasePointerOffset);
+ // Zero-extend TaggedT to WordT according to current compression scheme
+ // so that the addition with |external_pointer| (which already contains
+ // compensated offset value) below will decompress the tagged value.
+ // See JSTypedArray::ExternalPointerCompensationForOnHeapArray() for
+ // details.
+ base_pointer = Signed(ChangeUint32ToWord(compressed_base));
+ } else {
+ base_pointer =
+ LoadObjectField<IntPtrT>(typed_array, JSTypedArray::kBasePointerOffset);
+ }
+ return RawPtrAdd(external_pointer, base_pointer);
+}
+
+TNode<BigInt> CodeStubAssembler::LoadFixedBigInt64ArrayElementAsTagged(
+ SloppyTNode<RawPtrT> data_pointer, SloppyTNode<IntPtrT> offset) {
+ if (Is64()) {
+ TNode<IntPtrT> value = Load<IntPtrT>(data_pointer, offset);
+ return BigIntFromInt64(value);
+ } else {
+ DCHECK(!Is64());
+#if defined(V8_TARGET_BIG_ENDIAN)
+ TNode<IntPtrT> high = Load<IntPtrT>(data_pointer, offset);
+ TNode<IntPtrT> low = Load<IntPtrT>(
+ data_pointer, IntPtrAdd(offset, IntPtrConstant(kSystemPointerSize)));
+#else
+ TNode<IntPtrT> low = Load<IntPtrT>(data_pointer, offset);
+ TNode<IntPtrT> high = Load<IntPtrT>(
+ data_pointer, IntPtrAdd(offset, IntPtrConstant(kSystemPointerSize)));
+#endif
+ return BigIntFromInt32Pair(low, high);
+ }
+}
+
+TNode<BigInt> CodeStubAssembler::BigIntFromInt32Pair(TNode<IntPtrT> low,
+ TNode<IntPtrT> high) {
+ DCHECK(!Is64());
+ TVARIABLE(BigInt, var_result);
+ TVARIABLE(Word32T, var_sign, Int32Constant(BigInt::SignBits::encode(false)));
+ TVARIABLE(IntPtrT, var_high, high);
+ TVARIABLE(IntPtrT, var_low, low);
+ Label high_zero(this), negative(this), allocate_one_digit(this),
+ allocate_two_digits(this), if_zero(this), done(this);
+
+ GotoIf(IntPtrEqual(var_high.value(), IntPtrConstant(0)), &high_zero);
+ Branch(IntPtrLessThan(var_high.value(), IntPtrConstant(0)), &negative,
+ &allocate_two_digits);
+
+ BIND(&high_zero);
+ Branch(IntPtrEqual(var_low.value(), IntPtrConstant(0)), &if_zero,
+ &allocate_one_digit);
+
+ BIND(&negative);
+ {
+ var_sign = Int32Constant(BigInt::SignBits::encode(true));
+ // We must negate the value by computing "0 - (high|low)", performing
+ // both parts of the subtraction separately and manually taking care
+ // of the carry bit (which is 1 iff low != 0).
+ var_high = IntPtrSub(IntPtrConstant(0), var_high.value());
+ Label carry(this), no_carry(this);
+ Branch(IntPtrEqual(var_low.value(), IntPtrConstant(0)), &no_carry, &carry);
+ BIND(&carry);
+ var_high = IntPtrSub(var_high.value(), IntPtrConstant(1));
+ Goto(&no_carry);
+ BIND(&no_carry);
+ var_low = IntPtrSub(IntPtrConstant(0), var_low.value());
+ // var_high was non-zero going into this block, but subtracting the
+ // carry bit from it could bring us back onto the "one digit" path.
+ Branch(IntPtrEqual(var_high.value(), IntPtrConstant(0)),
+ &allocate_one_digit, &allocate_two_digits);
+ }
+
+ BIND(&allocate_one_digit);
+ {
+ var_result = AllocateRawBigInt(IntPtrConstant(1));
+ StoreBigIntBitfield(var_result.value(),
+ Word32Or(var_sign.value(),
+ Int32Constant(BigInt::LengthBits::encode(1))));
+ StoreBigIntDigit(var_result.value(), 0, Unsigned(var_low.value()));
+ Goto(&done);
+ }
+
+ BIND(&allocate_two_digits);
+ {
+ var_result = AllocateRawBigInt(IntPtrConstant(2));
+ StoreBigIntBitfield(var_result.value(),
+ Word32Or(var_sign.value(),
+ Int32Constant(BigInt::LengthBits::encode(2))));
+ StoreBigIntDigit(var_result.value(), 0, Unsigned(var_low.value()));
+ StoreBigIntDigit(var_result.value(), 1, Unsigned(var_high.value()));
+ Goto(&done);
+ }
+
+ BIND(&if_zero);
+ var_result = AllocateBigInt(IntPtrConstant(0));
+ Goto(&done);
+
+ BIND(&done);
+ return var_result.value();
+}
+
+TNode<BigInt> CodeStubAssembler::BigIntFromInt64(TNode<IntPtrT> value) {
+ DCHECK(Is64());
+ TVARIABLE(BigInt, var_result);
+ Label done(this), if_positive(this), if_negative(this), if_zero(this);
+ GotoIf(IntPtrEqual(value, IntPtrConstant(0)), &if_zero);
+ var_result = AllocateRawBigInt(IntPtrConstant(1));
+ Branch(IntPtrGreaterThan(value, IntPtrConstant(0)), &if_positive,
+ &if_negative);
+
+ BIND(&if_positive);
+ {
+ StoreBigIntBitfield(var_result.value(),
+ Int32Constant(BigInt::SignBits::encode(false) |
+ BigInt::LengthBits::encode(1)));
+ StoreBigIntDigit(var_result.value(), 0, Unsigned(value));
+ Goto(&done);
+ }
+
+ BIND(&if_negative);
+ {
+ StoreBigIntBitfield(var_result.value(),
+ Int32Constant(BigInt::SignBits::encode(true) |
+ BigInt::LengthBits::encode(1)));
+ StoreBigIntDigit(var_result.value(), 0,
+ Unsigned(IntPtrSub(IntPtrConstant(0), value)));
+ Goto(&done);
+ }
+
+ BIND(&if_zero);
+ {
+ var_result = AllocateBigInt(IntPtrConstant(0));
+ Goto(&done);
+ }
+
+ BIND(&done);
+ return var_result.value();
+}
+
+TNode<BigInt> CodeStubAssembler::LoadFixedBigUint64ArrayElementAsTagged(
+ SloppyTNode<RawPtrT> data_pointer, SloppyTNode<IntPtrT> offset) {
+ Label if_zero(this), done(this);
+ if (Is64()) {
+ TNode<UintPtrT> value = Load<UintPtrT>(data_pointer, offset);
+ return BigIntFromUint64(value);
+ } else {
+ DCHECK(!Is64());
+#if defined(V8_TARGET_BIG_ENDIAN)
+ TNode<UintPtrT> high = Load<UintPtrT>(data_pointer, offset);
+ TNode<UintPtrT> low = Load<UintPtrT>(
+ data_pointer, IntPtrAdd(offset, IntPtrConstant(kSystemPointerSize)));
+#else
+ TNode<UintPtrT> low = Load<UintPtrT>(data_pointer, offset);
+ TNode<UintPtrT> high = Load<UintPtrT>(
+ data_pointer, IntPtrAdd(offset, IntPtrConstant(kSystemPointerSize)));
+#endif
+ return BigIntFromUint32Pair(low, high);
+ }
+}
+
+TNode<BigInt> CodeStubAssembler::BigIntFromUint32Pair(TNode<UintPtrT> low,
+ TNode<UintPtrT> high) {
+ DCHECK(!Is64());
+ TVARIABLE(BigInt, var_result);
+ Label high_zero(this), if_zero(this), done(this);
+
+ GotoIf(IntPtrEqual(high, IntPtrConstant(0)), &high_zero);
+ var_result = AllocateBigInt(IntPtrConstant(2));
+ StoreBigIntDigit(var_result.value(), 0, low);
+ StoreBigIntDigit(var_result.value(), 1, high);
+ Goto(&done);
+
+ BIND(&high_zero);
+ GotoIf(IntPtrEqual(low, IntPtrConstant(0)), &if_zero);
+ var_result = AllocateBigInt(IntPtrConstant(1));
+ StoreBigIntDigit(var_result.value(), 0, low);
+ Goto(&done);
+
+ BIND(&if_zero);
+ var_result = AllocateBigInt(IntPtrConstant(0));
+ Goto(&done);
+
+ BIND(&done);
+ return var_result.value();
+}
+
+TNode<BigInt> CodeStubAssembler::BigIntFromUint64(TNode<UintPtrT> value) {
+ DCHECK(Is64());
+ TVARIABLE(BigInt, var_result);
+ Label done(this), if_zero(this);
+ GotoIf(IntPtrEqual(value, IntPtrConstant(0)), &if_zero);
+ var_result = AllocateBigInt(IntPtrConstant(1));
+ StoreBigIntDigit(var_result.value(), 0, value);
+ Goto(&done);
+
+ BIND(&if_zero);
+ var_result = AllocateBigInt(IntPtrConstant(0));
+ Goto(&done);
+ BIND(&done);
+ return var_result.value();
+}
+
+TNode<Numeric> CodeStubAssembler::LoadFixedTypedArrayElementAsTagged(
+ TNode<RawPtrT> data_pointer, TNode<UintPtrT> index,
+ ElementsKind elements_kind) {
+ TNode<IntPtrT> offset =
+ ElementOffsetFromIndex(Signed(index), elements_kind, 0);
+ switch (elements_kind) {
+ case UINT8_ELEMENTS: /* fall through */
+ case UINT8_CLAMPED_ELEMENTS:
+ return SmiFromInt32(Load<Uint8T>(data_pointer, offset));
+ case INT8_ELEMENTS:
+ return SmiFromInt32(Load<Int8T>(data_pointer, offset));
+ case UINT16_ELEMENTS:
+ return SmiFromInt32(Load<Uint16T>(data_pointer, offset));
+ case INT16_ELEMENTS:
+ return SmiFromInt32(Load<Int16T>(data_pointer, offset));
+ case UINT32_ELEMENTS:
+ return ChangeUint32ToTagged(Load<Uint32T>(data_pointer, offset));
+ case INT32_ELEMENTS:
+ return ChangeInt32ToTagged(Load<Int32T>(data_pointer, offset));
+ case FLOAT32_ELEMENTS:
+ return AllocateHeapNumberWithValue(
+ ChangeFloat32ToFloat64(Load<Float32T>(data_pointer, offset)));
+ case FLOAT64_ELEMENTS:
+ return AllocateHeapNumberWithValue(Load<Float64T>(data_pointer, offset));
+ case BIGINT64_ELEMENTS:
+ return LoadFixedBigInt64ArrayElementAsTagged(data_pointer, offset);
+ case BIGUINT64_ELEMENTS:
+ return LoadFixedBigUint64ArrayElementAsTagged(data_pointer, offset);
+ default:
+ UNREACHABLE();
+ }
+}
+
+TNode<Numeric> CodeStubAssembler::LoadFixedTypedArrayElementAsTagged(
+ TNode<RawPtrT> data_pointer, TNode<UintPtrT> index,
+ TNode<Int32T> elements_kind) {
+ TVARIABLE(Numeric, var_result);
+ Label done(this), if_unknown_type(this, Label::kDeferred);
+ int32_t elements_kinds[] = {
+#define TYPED_ARRAY_CASE(Type, type, TYPE, ctype) TYPE##_ELEMENTS,
+ TYPED_ARRAYS(TYPED_ARRAY_CASE)
+#undef TYPED_ARRAY_CASE
+ };
+
+#define TYPED_ARRAY_CASE(Type, type, TYPE, ctype) Label if_##type##array(this);
+ TYPED_ARRAYS(TYPED_ARRAY_CASE)
+#undef TYPED_ARRAY_CASE
+
+ Label* elements_kind_labels[] = {
+#define TYPED_ARRAY_CASE(Type, type, TYPE, ctype) &if_##type##array,
+ TYPED_ARRAYS(TYPED_ARRAY_CASE)
+#undef TYPED_ARRAY_CASE
+ };
+ STATIC_ASSERT(arraysize(elements_kinds) == arraysize(elements_kind_labels));
+
+ Switch(elements_kind, &if_unknown_type, elements_kinds, elements_kind_labels,
+ arraysize(elements_kinds));
+
+ BIND(&if_unknown_type);
+ Unreachable();
+
+#define TYPED_ARRAY_CASE(Type, type, TYPE, ctype) \
+ BIND(&if_##type##array); \
+ { \
+ var_result = LoadFixedTypedArrayElementAsTagged(data_pointer, index, \
+ TYPE##_ELEMENTS); \
+ Goto(&done); \
+ }
+ TYPED_ARRAYS(TYPED_ARRAY_CASE)
+#undef TYPED_ARRAY_CASE
+
+ BIND(&done);
+ return var_result.value();
+}
+
+template <typename TIndex>
+TNode<MaybeObject> CodeStubAssembler::LoadFeedbackVectorSlot(
+ TNode<FeedbackVector> feedback_vector, TNode<TIndex> slot,
+ int additional_offset) {
+ int32_t header_size = FeedbackVector::kRawFeedbackSlotsOffset +
+ additional_offset - kHeapObjectTag;
+ TNode<IntPtrT> offset =
+ ElementOffsetFromIndex(slot, HOLEY_ELEMENTS, header_size);
+ CSA_SLOW_ASSERT(
+ this, IsOffsetInBounds(offset, LoadFeedbackVectorLength(feedback_vector),
+ FeedbackVector::kHeaderSize));
+ return Load<MaybeObject>(feedback_vector, offset);
+}
+
+template TNode<MaybeObject> CodeStubAssembler::LoadFeedbackVectorSlot(
+ TNode<FeedbackVector> feedback_vector, TNode<TaggedIndex> slot,
+ int additional_offset);
+template TNode<MaybeObject> CodeStubAssembler::LoadFeedbackVectorSlot(
+ TNode<FeedbackVector> feedback_vector, TNode<IntPtrT> slot,
+ int additional_offset);
+template TNode<MaybeObject> CodeStubAssembler::LoadFeedbackVectorSlot(
+ TNode<FeedbackVector> feedback_vector, TNode<UintPtrT> slot,
+ int additional_offset);
+
+template <typename Array>
+TNode<Int32T> CodeStubAssembler::LoadAndUntagToWord32ArrayElement(
+ TNode<Array> object, int array_header_size, TNode<IntPtrT> index,
+ int additional_offset) {
+ DCHECK(IsAligned(additional_offset, kTaggedSize));
+ int endian_correction = 0;
+#if V8_TARGET_LITTLE_ENDIAN
+ if (SmiValuesAre32Bits()) endian_correction = 4;
+#endif
+ int32_t header_size = array_header_size + additional_offset - kHeapObjectTag +
+ endian_correction;
+ TNode<IntPtrT> offset =
+ ElementOffsetFromIndex(index, HOLEY_ELEMENTS, header_size);
+ CSA_ASSERT(this, IsOffsetInBounds(offset, LoadArrayLength(object),
+ array_header_size + endian_correction));
+ if (SmiValuesAre32Bits()) {
+ return Load<Int32T>(object, offset);
+ } else {
+ return SmiToInt32(Load(MachineType::TaggedSigned(), object, offset));
+ }
+}
+
+TNode<Int32T> CodeStubAssembler::LoadAndUntagToWord32FixedArrayElement(
+ TNode<FixedArray> object, TNode<IntPtrT> index, int additional_offset) {
+ CSA_SLOW_ASSERT(this, IsFixedArraySubclass(object));
+ return LoadAndUntagToWord32ArrayElement(object, FixedArray::kHeaderSize,
+ index, additional_offset);
+}
+
+TNode<MaybeObject> CodeStubAssembler::LoadWeakFixedArrayElement(
+ TNode<WeakFixedArray> object, TNode<IntPtrT> index, int additional_offset) {
+ return LoadArrayElement(object, WeakFixedArray::kHeaderSize, index,
+ additional_offset, LoadSensitivity::kSafe);
+}
+
+TNode<Float64T> CodeStubAssembler::LoadFixedDoubleArrayElement(
+ TNode<FixedDoubleArray> object, TNode<IntPtrT> index, Label* if_hole,
+ MachineType machine_type) {
+ int32_t header_size = FixedDoubleArray::kHeaderSize - kHeapObjectTag;
+ TNode<IntPtrT> offset =
+ ElementOffsetFromIndex(index, HOLEY_DOUBLE_ELEMENTS, header_size);
+ CSA_ASSERT(this, IsOffsetInBounds(
+ offset, LoadAndUntagFixedArrayBaseLength(object),
+ FixedDoubleArray::kHeaderSize, HOLEY_DOUBLE_ELEMENTS));
+ return LoadDoubleWithHoleCheck(object, offset, if_hole, machine_type);
+}
+
+TNode<Object> CodeStubAssembler::LoadFixedArrayBaseElementAsTagged(
+ TNode<FixedArrayBase> elements, TNode<IntPtrT> index,
+ TNode<Int32T> elements_kind, Label* if_accessor, Label* if_hole) {
+ TVARIABLE(Object, var_result);
+ Label done(this), if_packed(this), if_holey(this), if_packed_double(this),
+ if_holey_double(this), if_dictionary(this, Label::kDeferred);
+
+ int32_t kinds[] = {
+ // Handled by if_packed.
+ PACKED_SMI_ELEMENTS, PACKED_ELEMENTS, PACKED_NONEXTENSIBLE_ELEMENTS,
+ PACKED_SEALED_ELEMENTS, PACKED_FROZEN_ELEMENTS,
+ // Handled by if_holey.
+ HOLEY_SMI_ELEMENTS, HOLEY_ELEMENTS, HOLEY_NONEXTENSIBLE_ELEMENTS,
+ HOLEY_SEALED_ELEMENTS, HOLEY_FROZEN_ELEMENTS,
+ // Handled by if_packed_double.
+ PACKED_DOUBLE_ELEMENTS,
+ // Handled by if_holey_double.
+ HOLEY_DOUBLE_ELEMENTS};
+ Label* labels[] = {// PACKED_{SMI,}_ELEMENTS
+ &if_packed, &if_packed, &if_packed, &if_packed, &if_packed,
+ // HOLEY_{SMI,}_ELEMENTS
+ &if_holey, &if_holey, &if_holey, &if_holey, &if_holey,
+ // PACKED_DOUBLE_ELEMENTS
+ &if_packed_double,
+ // HOLEY_DOUBLE_ELEMENTS
+ &if_holey_double};
+ Switch(elements_kind, &if_dictionary, kinds, labels, arraysize(kinds));
+
+ BIND(&if_packed);
+ {
+ var_result = LoadFixedArrayElement(CAST(elements), index, 0);
+ Goto(&done);
+ }
+
+ BIND(&if_holey);
+ {
+ var_result = LoadFixedArrayElement(CAST(elements), index);
+ Branch(TaggedEqual(var_result.value(), TheHoleConstant()), if_hole, &done);
+ }
+
+ BIND(&if_packed_double);
+ {
+ var_result = AllocateHeapNumberWithValue(
+ LoadFixedDoubleArrayElement(CAST(elements), index));
+ Goto(&done);
+ }
+
+ BIND(&if_holey_double);
+ {
+ var_result = AllocateHeapNumberWithValue(
+ LoadFixedDoubleArrayElement(CAST(elements), index, if_hole));
+ Goto(&done);
+ }
+
+ BIND(&if_dictionary);
+ {
+ CSA_ASSERT(this, IsDictionaryElementsKind(elements_kind));
+ var_result = BasicLoadNumberDictionaryElement(CAST(elements), index,
+ if_accessor, if_hole);
+ Goto(&done);
+ }
+
+ BIND(&done);
+ return var_result.value();
+}
+
+TNode<BoolT> CodeStubAssembler::IsDoubleHole(TNode<Object> base,
+ TNode<IntPtrT> offset) {
+ // TODO(ishell): Compare only the upper part for the hole once the
+ // compiler is able to fold addition of already complex |offset| with
+ // |kIeeeDoubleExponentWordOffset| into one addressing mode.
+ if (Is64()) {
+ TNode<Uint64T> element = Load<Uint64T>(base, offset);
+ return Word64Equal(element, Int64Constant(kHoleNanInt64));
+ } else {
+ TNode<Uint32T> element_upper = Load<Uint32T>(
+ base, IntPtrAdd(offset, IntPtrConstant(kIeeeDoubleExponentWordOffset)));
+ return Word32Equal(element_upper, Int32Constant(kHoleNanUpper32));
+ }
+}
+
+TNode<Float64T> CodeStubAssembler::LoadDoubleWithHoleCheck(
+ TNode<Object> base, TNode<IntPtrT> offset, Label* if_hole,
+ MachineType machine_type) {
+ if (if_hole) {
+ GotoIf(IsDoubleHole(base, offset), if_hole);
+ }
+ if (machine_type.IsNone()) {
+ // This means the actual value is not needed.
+ return TNode<Float64T>();
+ }
+ return UncheckedCast<Float64T>(Load(machine_type, base, offset));
+}
+
+TNode<ScopeInfo> CodeStubAssembler::LoadScopeInfo(TNode<Context> context) {
+ return CAST(LoadContextElement(context, Context::SCOPE_INFO_INDEX));
+}
+
+TNode<BoolT> CodeStubAssembler::LoadScopeInfoHasExtensionField(
+ TNode<ScopeInfo> scope_info) {
+ TNode<IntPtrT> value =
+ LoadAndUntagObjectField(scope_info, ScopeInfo::kFlagsOffset);
+ return IsSetWord<ScopeInfo::HasContextExtensionSlotBit>(value);
+}
+
+void CodeStubAssembler::StoreContextElementNoWriteBarrier(
+ TNode<Context> context, int slot_index, SloppyTNode<Object> value) {
+ int offset = Context::SlotOffset(slot_index);
+ StoreNoWriteBarrier(MachineRepresentation::kTagged, context,
+ IntPtrConstant(offset), value);
+}
+
+TNode<NativeContext> CodeStubAssembler::LoadNativeContext(
+ TNode<Context> context) {
+ TNode<Map> map = LoadMap(context);
+ return CAST(LoadObjectField(
+ map, Map::kConstructorOrBackPointerOrNativeContextOffset));
+}
+
+TNode<Context> CodeStubAssembler::LoadModuleContext(TNode<Context> context) {
+ TNode<NativeContext> native_context = LoadNativeContext(context);
+ TNode<Map> module_map = CAST(
+ LoadContextElement(native_context, Context::MODULE_CONTEXT_MAP_INDEX));
+ TVariable<Object> cur_context(context, this);
+
+ Label context_found(this);
+
+ Label context_search(this, &cur_context);
+
+ // Loop until cur_context->map() is module_map.
+ Goto(&context_search);
+ BIND(&context_search);
+ {
+ CSA_ASSERT(this, Word32BinaryNot(
+ TaggedEqual(cur_context.value(), native_context)));
+ GotoIf(TaggedEqual(LoadMap(CAST(cur_context.value())), module_map),
+ &context_found);
+
+ cur_context =
+ LoadContextElement(CAST(cur_context.value()), Context::PREVIOUS_INDEX);
+ Goto(&context_search);
+ }
+
+ BIND(&context_found);
+ return UncheckedCast<Context>(cur_context.value());
+}
+
+TNode<Map> CodeStubAssembler::LoadObjectFunctionInitialMap(
+ TNode<NativeContext> native_context) {
+ TNode<JSFunction> object_function =
+ CAST(LoadContextElement(native_context, Context::OBJECT_FUNCTION_INDEX));
+ return CAST(LoadJSFunctionPrototypeOrInitialMap(object_function));
+}
+
+TNode<Map> CodeStubAssembler::LoadSlowObjectWithNullPrototypeMap(
+ TNode<NativeContext> native_context) {
+ TNode<Map> map = CAST(LoadContextElement(
+ native_context, Context::SLOW_OBJECT_WITH_NULL_PROTOTYPE_MAP));
+ return map;
+}
+
+TNode<Map> CodeStubAssembler::LoadJSArrayElementsMap(
+ SloppyTNode<Int32T> kind, TNode<NativeContext> native_context) {
+ CSA_ASSERT(this, IsFastElementsKind(kind));
+ TNode<IntPtrT> offset =
+ IntPtrAdd(IntPtrConstant(Context::FIRST_JS_ARRAY_MAP_SLOT),
+ ChangeInt32ToIntPtr(kind));
+ return UncheckedCast<Map>(LoadContextElement(native_context, offset));
+}
+
+TNode<Map> CodeStubAssembler::LoadJSArrayElementsMap(
+ ElementsKind kind, TNode<NativeContext> native_context) {
+ return UncheckedCast<Map>(
+ LoadContextElement(native_context, Context::ArrayMapIndex(kind)));
+}
+
+TNode<BoolT> CodeStubAssembler::IsGeneratorFunction(
+ TNode<JSFunction> function) {
+ const TNode<SharedFunctionInfo> shared_function_info =
+ LoadObjectField<SharedFunctionInfo>(
+ function, JSFunction::kSharedFunctionInfoOffset);
+
+ const TNode<Uint32T> function_kind =
+ DecodeWord32<SharedFunctionInfo::FunctionKindBits>(
+ LoadObjectField<Uint32T>(shared_function_info,
+ SharedFunctionInfo::kFlagsOffset));
+
+ // See IsGeneratorFunction(FunctionKind kind).
+ return IsInRange(function_kind, FunctionKind::kAsyncConciseGeneratorMethod,
+ FunctionKind::kConciseGeneratorMethod);
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSFunctionWithPrototypeSlot(
+ TNode<HeapObject> object) {
+ // Only JSFunction maps may have HasPrototypeSlotBit set.
+ return TNode<BoolT>::UncheckedCast(
+ IsSetWord32<Map::Bits1::HasPrototypeSlotBit>(
+ LoadMapBitField(LoadMap(object))));
+}
+
+void CodeStubAssembler::BranchIfHasPrototypeProperty(
+ TNode<JSFunction> function, TNode<Int32T> function_map_bit_field,
+ Label* if_true, Label* if_false) {
+ // (has_prototype_slot() && IsConstructor()) ||
+ // IsGeneratorFunction(shared()->kind())
+ uint32_t mask = Map::Bits1::HasPrototypeSlotBit::kMask |
+ Map::Bits1::IsConstructorBit::kMask;
+
+ GotoIf(IsAllSetWord32(function_map_bit_field, mask), if_true);
+ Branch(IsGeneratorFunction(function), if_true, if_false);
+}
+
+void CodeStubAssembler::GotoIfPrototypeRequiresRuntimeLookup(
+ TNode<JSFunction> function, TNode<Map> map, Label* runtime) {
+ // !has_prototype_property() || has_non_instance_prototype()
+ TNode<Int32T> map_bit_field = LoadMapBitField(map);
+ Label next_check(this);
+ BranchIfHasPrototypeProperty(function, map_bit_field, &next_check, runtime);
+ BIND(&next_check);
+ GotoIf(IsSetWord32<Map::Bits1::HasNonInstancePrototypeBit>(map_bit_field),
+ runtime);
+}
+
+TNode<HeapObject> CodeStubAssembler::LoadJSFunctionPrototype(
+ TNode<JSFunction> function, Label* if_bailout) {
+ CSA_ASSERT(this, IsFunctionWithPrototypeSlotMap(LoadMap(function)));
+ CSA_ASSERT(this, IsClearWord32<Map::Bits1::HasNonInstancePrototypeBit>(
+ LoadMapBitField(LoadMap(function))));
+ TNode<HeapObject> proto_or_map = LoadObjectField<HeapObject>(
+ function, JSFunction::kPrototypeOrInitialMapOffset);
+ GotoIf(IsTheHole(proto_or_map), if_bailout);
+
+ TVARIABLE(HeapObject, var_result, proto_or_map);
+ Label done(this, &var_result);
+ GotoIfNot(IsMap(proto_or_map), &done);
+
+ var_result = LoadMapPrototype(CAST(proto_or_map));
+ Goto(&done);
+
+ BIND(&done);
+ return var_result.value();
+}
+
+TNode<BytecodeArray> CodeStubAssembler::LoadSharedFunctionInfoBytecodeArray(
+ TNode<SharedFunctionInfo> shared) {
+ TNode<HeapObject> function_data = LoadObjectField<HeapObject>(
+ shared, SharedFunctionInfo::kFunctionDataOffset);
+
+ TVARIABLE(HeapObject, var_result, function_data);
+ Label done(this, &var_result);
+
+ GotoIfNot(HasInstanceType(function_data, INTERPRETER_DATA_TYPE), &done);
+ TNode<BytecodeArray> bytecode_array = LoadObjectField<BytecodeArray>(
+ function_data, InterpreterData::kBytecodeArrayOffset);
+ var_result = bytecode_array;
+ Goto(&done);
+
+ BIND(&done);
+ return CAST(var_result.value());
+}
+
+void CodeStubAssembler::StoreObjectByteNoWriteBarrier(TNode<HeapObject> object,
+ int offset,
+ TNode<Word32T> value) {
+ StoreNoWriteBarrier(MachineRepresentation::kWord8, object,
+ IntPtrConstant(offset - kHeapObjectTag), value);
+}
+
+void CodeStubAssembler::StoreHeapNumberValue(SloppyTNode<HeapNumber> object,
+ SloppyTNode<Float64T> value) {
+ StoreObjectFieldNoWriteBarrier(object, HeapNumber::kValueOffset, value);
+}
+
+void CodeStubAssembler::StoreObjectField(TNode<HeapObject> object, int offset,
+ TNode<Object> value) {
+ DCHECK_NE(HeapObject::kMapOffset, offset); // Use StoreMap instead.
+
+ OptimizedStoreField(MachineRepresentation::kTagged,
+ UncheckedCast<HeapObject>(object), offset, value);
+}
+
+void CodeStubAssembler::StoreObjectField(TNode<HeapObject> object,
+ TNode<IntPtrT> offset,
+ TNode<Object> value) {
+ int const_offset;
+ if (ToInt32Constant(offset, &const_offset)) {
+ StoreObjectField(object, const_offset, value);
+ } else {
+ Store(object, IntPtrSub(offset, IntPtrConstant(kHeapObjectTag)), value);
+ }
+}
+
+void CodeStubAssembler::UnsafeStoreObjectFieldNoWriteBarrier(
+ TNode<HeapObject> object, int offset, TNode<Object> value) {
+ OptimizedStoreFieldUnsafeNoWriteBarrier(MachineRepresentation::kTagged,
+ object, offset, value);
+}
+
+void CodeStubAssembler::StoreMap(TNode<HeapObject> object, TNode<Map> map) {
+ OptimizedStoreMap(object, map);
+}
+
+void CodeStubAssembler::StoreMapNoWriteBarrier(TNode<HeapObject> object,
+ RootIndex map_root_index) {
+ StoreMapNoWriteBarrier(object, CAST(LoadRoot(map_root_index)));
+}
+
+void CodeStubAssembler::StoreMapNoWriteBarrier(TNode<HeapObject> object,
+ TNode<Map> map) {
+ OptimizedStoreFieldAssertNoWriteBarrier(MachineRepresentation::kTaggedPointer,
+ object, HeapObject::kMapOffset, map);
+}
+
+void CodeStubAssembler::StoreObjectFieldRoot(TNode<HeapObject> object,
+ int offset, RootIndex root_index) {
+ if (RootsTable::IsImmortalImmovable(root_index)) {
+ StoreObjectFieldNoWriteBarrier(object, offset, LoadRoot(root_index));
+ } else {
+ StoreObjectField(object, offset, LoadRoot(root_index));
+ }
+}
+
+template <typename TIndex>
+void CodeStubAssembler::StoreFixedArrayOrPropertyArrayElement(
+ TNode<UnionT<FixedArray, PropertyArray>> object, TNode<TIndex> index_node,
+ TNode<Object> value, WriteBarrierMode barrier_mode, int additional_offset) {
+ // TODO(v8:9708): Do we want to keep both IntPtrT and UintPtrT variants?
+ static_assert(std::is_same<TIndex, Smi>::value ||
+ std::is_same<TIndex, UintPtrT>::value ||
+ std::is_same<TIndex, IntPtrT>::value,
+ "Only Smi, UintPtrT or IntPtrT index is allowed");
+ DCHECK(barrier_mode == SKIP_WRITE_BARRIER ||
+ barrier_mode == UNSAFE_SKIP_WRITE_BARRIER ||
+ barrier_mode == UPDATE_WRITE_BARRIER ||
+ barrier_mode == UPDATE_EPHEMERON_KEY_WRITE_BARRIER);
+ DCHECK(IsAligned(additional_offset, kTaggedSize));
+ STATIC_ASSERT(static_cast<int>(FixedArray::kHeaderSize) ==
+ static_cast<int>(PropertyArray::kHeaderSize));
+ int header_size =
+ FixedArray::kHeaderSize + additional_offset - kHeapObjectTag;
+ TNode<IntPtrT> offset =
+ ElementOffsetFromIndex(index_node, HOLEY_ELEMENTS, header_size);
+ STATIC_ASSERT(static_cast<int>(FixedArrayBase::kLengthOffset) ==
+ static_cast<int>(WeakFixedArray::kLengthOffset));
+ STATIC_ASSERT(static_cast<int>(FixedArrayBase::kLengthOffset) ==
+ static_cast<int>(PropertyArray::kLengthAndHashOffset));
+ // Check that index_node + additional_offset <= object.length.
+ // TODO(cbruni): Use proper LoadXXLength helpers
+ CSA_ASSERT(
+ this,
+ IsOffsetInBounds(
+ offset,
+ Select<IntPtrT>(
+ IsPropertyArray(object),
+ [=] {
+ TNode<IntPtrT> length_and_hash = LoadAndUntagObjectField(
+ object, PropertyArray::kLengthAndHashOffset);
+ return TNode<IntPtrT>::UncheckedCast(
+ DecodeWord<PropertyArray::LengthField>(length_and_hash));
+ },
+ [=] {
+ return LoadAndUntagObjectField(object,
+ FixedArrayBase::kLengthOffset);
+ }),
+ FixedArray::kHeaderSize));
+ if (barrier_mode == SKIP_WRITE_BARRIER) {
+ StoreNoWriteBarrier(MachineRepresentation::kTagged, object, offset, value);
+ } else if (barrier_mode == UNSAFE_SKIP_WRITE_BARRIER) {
+ UnsafeStoreNoWriteBarrier(MachineRepresentation::kTagged, object, offset,
+ value);
+ } else if (barrier_mode == UPDATE_EPHEMERON_KEY_WRITE_BARRIER) {
+ StoreEphemeronKey(object, offset, value);
+ } else {
+ Store(object, offset, value);
+ }
+}
+
+template V8_EXPORT_PRIVATE void
+CodeStubAssembler::StoreFixedArrayOrPropertyArrayElement<Smi>(
+ TNode<UnionT<FixedArray, PropertyArray>>, TNode<Smi>, TNode<Object>,
+ WriteBarrierMode, int);
+
+template V8_EXPORT_PRIVATE void
+CodeStubAssembler::StoreFixedArrayOrPropertyArrayElement<IntPtrT>(
+ TNode<UnionT<FixedArray, PropertyArray>>, TNode<IntPtrT>, TNode<Object>,
+ WriteBarrierMode, int);
+
+template V8_EXPORT_PRIVATE void
+CodeStubAssembler::StoreFixedArrayOrPropertyArrayElement<UintPtrT>(
+ TNode<UnionT<FixedArray, PropertyArray>>, TNode<UintPtrT>, TNode<Object>,
+ WriteBarrierMode, int);
+
+template <typename TIndex>
+void CodeStubAssembler::StoreFixedDoubleArrayElement(
+ TNode<FixedDoubleArray> object, TNode<TIndex> index, TNode<Float64T> value,
+ CheckBounds check_bounds) {
+ // TODO(v8:9708): Do we want to keep both IntPtrT and UintPtrT variants?
+ static_assert(std::is_same<TIndex, Smi>::value ||
+ std::is_same<TIndex, UintPtrT>::value ||
+ std::is_same<TIndex, IntPtrT>::value,
+ "Only Smi, UintPtrT or IntPtrT index is allowed");
+ if (NeedsBoundsCheck(check_bounds)) {
+ FixedArrayBoundsCheck(object, index, 0);
+ }
+ TNode<IntPtrT> offset = ElementOffsetFromIndex(
+ index, PACKED_DOUBLE_ELEMENTS, FixedArray::kHeaderSize - kHeapObjectTag);
+ MachineRepresentation rep = MachineRepresentation::kFloat64;
+ // Make sure we do not store signalling NaNs into double arrays.
+ TNode<Float64T> value_silenced = Float64SilenceNaN(value);
+ StoreNoWriteBarrier(rep, object, offset, value_silenced);
+}
+
+// Export the Smi version which is used outside of code-stub-assembler.
+template V8_EXPORT_PRIVATE void CodeStubAssembler::StoreFixedDoubleArrayElement<
+ Smi>(TNode<FixedDoubleArray>, TNode<Smi>, TNode<Float64T>, CheckBounds);
+
+void CodeStubAssembler::StoreFeedbackVectorSlot(
+ TNode<FeedbackVector> feedback_vector, TNode<UintPtrT> slot,
+ TNode<AnyTaggedT> value, WriteBarrierMode barrier_mode,
+ int additional_offset) {
+ DCHECK(IsAligned(additional_offset, kTaggedSize));
+ DCHECK(barrier_mode == SKIP_WRITE_BARRIER ||
+ barrier_mode == UNSAFE_SKIP_WRITE_BARRIER ||
+ barrier_mode == UPDATE_WRITE_BARRIER);
+ int header_size = FeedbackVector::kRawFeedbackSlotsOffset +
+ additional_offset - kHeapObjectTag;
+ TNode<IntPtrT> offset =
+ ElementOffsetFromIndex(Signed(slot), HOLEY_ELEMENTS, header_size);
+ // Check that slot <= feedback_vector.length.
+ CSA_ASSERT(this,
+ IsOffsetInBounds(offset, LoadFeedbackVectorLength(feedback_vector),
+ FeedbackVector::kHeaderSize));
+ if (barrier_mode == SKIP_WRITE_BARRIER) {
+ StoreNoWriteBarrier(MachineRepresentation::kTagged, feedback_vector, offset,
+ value);
+ } else if (barrier_mode == UNSAFE_SKIP_WRITE_BARRIER) {
+ UnsafeStoreNoWriteBarrier(MachineRepresentation::kTagged, feedback_vector,
+ offset, value);
+ } else {
+ Store(feedback_vector, offset, value);
+ }
+}
+
+TNode<Int32T> CodeStubAssembler::EnsureArrayPushable(TNode<Context> context,
+ TNode<Map> map,
+ Label* bailout) {
+ // Disallow pushing onto prototypes. It might be the JSArray prototype.
+ // Disallow pushing onto non-extensible objects.
+ Comment("Disallow pushing onto prototypes");
+ GotoIfNot(IsExtensibleNonPrototypeMap(map), bailout);
+
+ EnsureArrayLengthWritable(context, map, bailout);
+
+ TNode<Uint32T> kind =
+ DecodeWord32<Map::Bits2::ElementsKindBits>(LoadMapBitField2(map));
+ return Signed(kind);
+}
+
+void CodeStubAssembler::PossiblyGrowElementsCapacity(
+ ElementsKind kind, TNode<HeapObject> array, TNode<BInt> length,
+ TVariable<FixedArrayBase>* var_elements, TNode<BInt> growth,
+ Label* bailout) {
+ Label fits(this, var_elements);
+ TNode<BInt> capacity =
+ TaggedToParameter<BInt>(LoadFixedArrayBaseLength(var_elements->value()));
+
+ TNode<BInt> new_length = IntPtrOrSmiAdd(growth, length);
+ GotoIfNot(IntPtrOrSmiGreaterThan(new_length, capacity), &fits);
+ TNode<BInt> new_capacity = CalculateNewElementsCapacity(new_length);
+ *var_elements = GrowElementsCapacity(array, var_elements->value(), kind, kind,
+ capacity, new_capacity, bailout);
+ Goto(&fits);
+ BIND(&fits);
+}
+
+TNode<Smi> CodeStubAssembler::BuildAppendJSArray(ElementsKind kind,
+ TNode<JSArray> array,
+ CodeStubArguments* args,
+ TVariable<IntPtrT>* arg_index,
+ Label* bailout) {
+ Comment("BuildAppendJSArray: ", ElementsKindToString(kind));
+ Label pre_bailout(this);
+ Label success(this);
+ TVARIABLE(Smi, var_tagged_length);
+ TVARIABLE(BInt, var_length, SmiToBInt(LoadFastJSArrayLength(array)));
+ TVARIABLE(FixedArrayBase, var_elements, LoadElements(array));
+
+ // Resize the capacity of the fixed array if it doesn't fit.
+ TNode<IntPtrT> first = arg_index->value();
+ TNode<BInt> growth = IntPtrToBInt(IntPtrSub(args->GetLength(), first));
+ PossiblyGrowElementsCapacity(kind, array, var_length.value(), &var_elements,
+ growth, &pre_bailout);
+
+ // Push each argument onto the end of the array now that there is enough
+ // capacity.
+ CodeStubAssembler::VariableList push_vars({&var_length}, zone());
+ TNode<FixedArrayBase> elements = var_elements.value();
+ args->ForEach(
+ push_vars,
+ [&](TNode<Object> arg) {
+ TryStoreArrayElement(kind, &pre_bailout, elements, var_length.value(),
+ arg);
+ Increment(&var_length);
+ },
+ first);
+ {
+ TNode<Smi> length = BIntToSmi(var_length.value());
+ var_tagged_length = length;
+ StoreObjectFieldNoWriteBarrier(array, JSArray::kLengthOffset, length);
+ Goto(&success);
+ }
+
+ BIND(&pre_bailout);
+ {
+ TNode<Smi> length = ParameterToTagged(var_length.value());
+ var_tagged_length = length;
+ TNode<Smi> diff = SmiSub(length, LoadFastJSArrayLength(array));
+ StoreObjectFieldNoWriteBarrier(array, JSArray::kLengthOffset, length);
+ *arg_index = IntPtrAdd(arg_index->value(), SmiUntag(diff));
+ Goto(bailout);
+ }
+
+ BIND(&success);
+ return var_tagged_length.value();
+}
+
+void CodeStubAssembler::TryStoreArrayElement(ElementsKind kind, Label* bailout,
+ TNode<FixedArrayBase> elements,
+ TNode<BInt> index,
+ TNode<Object> value) {
+ if (IsSmiElementsKind(kind)) {
+ GotoIf(TaggedIsNotSmi(value), bailout);
+ } else if (IsDoubleElementsKind(kind)) {
+ GotoIfNotNumber(value, bailout);
+ }
+
+ if (IsDoubleElementsKind(kind)) {
+ StoreElement(elements, kind, index, ChangeNumberToFloat64(CAST(value)));
+ } else {
+ StoreElement(elements, kind, index, value);
+ }
+}
+
+void CodeStubAssembler::BuildAppendJSArray(ElementsKind kind,
+ TNode<JSArray> array,
+ TNode<Object> value,
+ Label* bailout) {
+ Comment("BuildAppendJSArray: ", ElementsKindToString(kind));
+ TVARIABLE(BInt, var_length, SmiToBInt(LoadFastJSArrayLength(array)));
+ TVARIABLE(FixedArrayBase, var_elements, LoadElements(array));
+
+ // Resize the capacity of the fixed array if it doesn't fit.
+ TNode<BInt> growth = IntPtrOrSmiConstant<BInt>(1);
+ PossiblyGrowElementsCapacity(kind, array, var_length.value(), &var_elements,
+ growth, bailout);
+
+ // Push each argument onto the end of the array now that there is enough
+ // capacity.
+ TryStoreArrayElement(kind, bailout, var_elements.value(), var_length.value(),
+ value);
+ Increment(&var_length);
+
+ TNode<Smi> length = BIntToSmi(var_length.value());
+ StoreObjectFieldNoWriteBarrier(array, JSArray::kLengthOffset, length);
+}
+
+TNode<Cell> CodeStubAssembler::AllocateCellWithValue(TNode<Object> value,
+ WriteBarrierMode mode) {
+ TNode<HeapObject> result = Allocate(Cell::kSize, kNone);
+ StoreMapNoWriteBarrier(result, RootIndex::kCellMap);
+ TNode<Cell> cell = CAST(result);
+ StoreCellValue(cell, value, mode);
+ return cell;
+}
+
+TNode<Object> CodeStubAssembler::LoadCellValue(TNode<Cell> cell) {
+ return LoadObjectField(cell, Cell::kValueOffset);
+}
+
+void CodeStubAssembler::StoreCellValue(TNode<Cell> cell, TNode<Object> value,
+ WriteBarrierMode mode) {
+ DCHECK(mode == SKIP_WRITE_BARRIER || mode == UPDATE_WRITE_BARRIER);
+
+ if (mode == UPDATE_WRITE_BARRIER) {
+ StoreObjectField(cell, Cell::kValueOffset, value);
+ } else {
+ StoreObjectFieldNoWriteBarrier(cell, Cell::kValueOffset, value);
+ }
+}
+
+TNode<HeapNumber> CodeStubAssembler::AllocateHeapNumber() {
+ TNode<HeapObject> result = Allocate(HeapNumber::kSize, kNone);
+ RootIndex heap_map_index = RootIndex::kHeapNumberMap;
+ StoreMapNoWriteBarrier(result, heap_map_index);
+ return UncheckedCast<HeapNumber>(result);
+}
+
+TNode<HeapNumber> CodeStubAssembler::AllocateHeapNumberWithValue(
+ SloppyTNode<Float64T> value) {
+ TNode<HeapNumber> result = AllocateHeapNumber();
+ StoreHeapNumberValue(result, value);
+ return result;
+}
+
+TNode<Object> CodeStubAssembler::CloneIfMutablePrimitive(TNode<Object> object) {
+ TVARIABLE(Object, result, object);
+ Label done(this);
+
+ GotoIf(TaggedIsSmi(object), &done);
+ // TODO(leszeks): Read the field descriptor to decide if this heap number is
+ // mutable or not.
+ GotoIfNot(IsHeapNumber(UncheckedCast<HeapObject>(object)), &done);
+ {
+ // Mutable heap number found --- allocate a clone.
+ TNode<Float64T> value =
+ LoadHeapNumberValue(UncheckedCast<HeapNumber>(object));
+ result = AllocateHeapNumberWithValue(value);
+ Goto(&done);
+ }
+
+ BIND(&done);
+ return result.value();
+}
+
+TNode<BigInt> CodeStubAssembler::AllocateBigInt(TNode<IntPtrT> length) {
+ TNode<BigInt> result = AllocateRawBigInt(length);
+ StoreBigIntBitfield(result,
+ Word32Shl(TruncateIntPtrToInt32(length),
+ Int32Constant(BigInt::LengthBits::kShift)));
+ return result;
+}
+
+TNode<BigInt> CodeStubAssembler::AllocateRawBigInt(TNode<IntPtrT> length) {
+ TNode<IntPtrT> size =
+ IntPtrAdd(IntPtrConstant(BigInt::kHeaderSize),
+ Signed(WordShl(length, kSystemPointerSizeLog2)));
+ TNode<HeapObject> raw_result = Allocate(size, kAllowLargeObjectAllocation);
+ StoreMapNoWriteBarrier(raw_result, RootIndex::kBigIntMap);
+ if (FIELD_SIZE(BigInt::kOptionalPaddingOffset) != 0) {
+ DCHECK_EQ(4, FIELD_SIZE(BigInt::kOptionalPaddingOffset));
+ StoreObjectFieldNoWriteBarrier(raw_result, BigInt::kOptionalPaddingOffset,
+ Int32Constant(0));
+ }
+ return UncheckedCast<BigInt>(raw_result);
+}
+
+void CodeStubAssembler::StoreBigIntBitfield(TNode<BigInt> bigint,
+ TNode<Word32T> bitfield) {
+ StoreObjectFieldNoWriteBarrier(bigint, BigInt::kBitfieldOffset, bitfield);
+}
+
+void CodeStubAssembler::StoreBigIntDigit(TNode<BigInt> bigint,
+ intptr_t digit_index,
+ TNode<UintPtrT> digit) {
+ CHECK_LE(0, digit_index);
+ CHECK_LT(digit_index, BigInt::kMaxLength);
+ StoreObjectFieldNoWriteBarrier(
+ bigint,
+ BigInt::kDigitsOffset +
+ static_cast<int>(digit_index) * kSystemPointerSize,
+ digit);
+}
+
+void CodeStubAssembler::StoreBigIntDigit(TNode<BigInt> bigint,
+ TNode<IntPtrT> digit_index,
+ TNode<UintPtrT> digit) {
+ TNode<IntPtrT> offset =
+ IntPtrAdd(IntPtrConstant(BigInt::kDigitsOffset),
+ IntPtrMul(digit_index, IntPtrConstant(kSystemPointerSize)));
+ StoreObjectFieldNoWriteBarrier(bigint, offset, digit);
+}
+
+TNode<Word32T> CodeStubAssembler::LoadBigIntBitfield(TNode<BigInt> bigint) {
+ return UncheckedCast<Word32T>(
+ LoadObjectField<Uint32T>(bigint, BigInt::kBitfieldOffset));
+}
+
+TNode<UintPtrT> CodeStubAssembler::LoadBigIntDigit(TNode<BigInt> bigint,
+ intptr_t digit_index) {
+ CHECK_LE(0, digit_index);
+ CHECK_LT(digit_index, BigInt::kMaxLength);
+ return LoadObjectField<UintPtrT>(
+ bigint, BigInt::kDigitsOffset +
+ static_cast<int>(digit_index) * kSystemPointerSize);
+}
+
+TNode<UintPtrT> CodeStubAssembler::LoadBigIntDigit(TNode<BigInt> bigint,
+ TNode<IntPtrT> digit_index) {
+ TNode<IntPtrT> offset =
+ IntPtrAdd(IntPtrConstant(BigInt::kDigitsOffset),
+ IntPtrMul(digit_index, IntPtrConstant(kSystemPointerSize)));
+ return LoadObjectField<UintPtrT>(bigint, offset);
+}
+
+TNode<ByteArray> CodeStubAssembler::AllocateByteArray(TNode<UintPtrT> length,
+ AllocationFlags flags) {
+ Comment("AllocateByteArray");
+ TVARIABLE(Object, var_result);
+
+ // Compute the ByteArray size and check if it fits into new space.
+ Label if_lengthiszero(this), if_sizeissmall(this),
+ if_notsizeissmall(this, Label::kDeferred), if_join(this);
+ GotoIf(WordEqual(length, UintPtrConstant(0)), &if_lengthiszero);
+
+ TNode<IntPtrT> raw_size =
+ GetArrayAllocationSize(Signed(length), UINT8_ELEMENTS,
+ ByteArray::kHeaderSize + kObjectAlignmentMask);
+ TNode<IntPtrT> size =
+ WordAnd(raw_size, IntPtrConstant(~kObjectAlignmentMask));
+ Branch(IntPtrLessThanOrEqual(size, IntPtrConstant(kMaxRegularHeapObjectSize)),
+ &if_sizeissmall, &if_notsizeissmall);
+
+ BIND(&if_sizeissmall);
+ {
+ // Just allocate the ByteArray in new space.
+ TNode<HeapObject> result =
+ AllocateInNewSpace(UncheckedCast<IntPtrT>(size), flags);
+ DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kByteArrayMap));
+ StoreMapNoWriteBarrier(result, RootIndex::kByteArrayMap);
+ StoreObjectFieldNoWriteBarrier(result, ByteArray::kLengthOffset,
+ SmiTag(Signed(length)));
+ var_result = result;
+ Goto(&if_join);
+ }
+
+ BIND(&if_notsizeissmall);
+ {
+ // We might need to allocate in large object space, go to the runtime.
+ TNode<Object> result =
+ CallRuntime(Runtime::kAllocateByteArray, NoContextConstant(),
+ ChangeUintPtrToTagged(length));
+ var_result = result;
+ Goto(&if_join);
+ }
+
+ BIND(&if_lengthiszero);
+ {
+ var_result = EmptyByteArrayConstant();
+ Goto(&if_join);
+ }
+
+ BIND(&if_join);
+ return CAST(var_result.value());
+}
+
+TNode<String> CodeStubAssembler::AllocateSeqOneByteString(
+ uint32_t length, AllocationFlags flags) {
+ Comment("AllocateSeqOneByteString");
+ if (length == 0) {
+ return EmptyStringConstant();
+ }
+ TNode<HeapObject> result = Allocate(SeqOneByteString::SizeFor(length), flags);
+ DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kOneByteStringMap));
+ StoreMapNoWriteBarrier(result, RootIndex::kOneByteStringMap);
+ StoreObjectFieldNoWriteBarrier(result, SeqOneByteString::kLengthOffset,
+ Uint32Constant(length));
+ StoreObjectFieldNoWriteBarrier(result, SeqOneByteString::kHashFieldOffset,
+ Int32Constant(String::kEmptyHashField));
+ return CAST(result);
+}
+
+TNode<BoolT> CodeStubAssembler::IsZeroOrContext(SloppyTNode<Object> object) {
+ return Select<BoolT>(
+ TaggedEqual(object, SmiConstant(0)), [=] { return Int32TrueConstant(); },
+ [=] { return IsContext(CAST(object)); });
+}
+
+TNode<String> CodeStubAssembler::AllocateSeqTwoByteString(
+ uint32_t length, AllocationFlags flags) {
+ Comment("AllocateSeqTwoByteString");
+ if (length == 0) {
+ return EmptyStringConstant();
+ }
+ TNode<HeapObject> result = Allocate(SeqTwoByteString::SizeFor(length), flags);
+ DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kStringMap));
+ StoreMapNoWriteBarrier(result, RootIndex::kStringMap);
+ StoreObjectFieldNoWriteBarrier(result, SeqTwoByteString::kLengthOffset,
+ Uint32Constant(length));
+ StoreObjectFieldNoWriteBarrier(result, SeqTwoByteString::kHashFieldOffset,
+ Int32Constant(String::kEmptyHashField));
+ return CAST(result);
+}
+
+TNode<String> CodeStubAssembler::AllocateSlicedString(RootIndex map_root_index,
+ TNode<Uint32T> length,
+ TNode<String> parent,
+ TNode<Smi> offset) {
+ DCHECK(map_root_index == RootIndex::kSlicedOneByteStringMap ||
+ map_root_index == RootIndex::kSlicedStringMap);
+ TNode<HeapObject> result = Allocate(SlicedString::kSize);
+ DCHECK(RootsTable::IsImmortalImmovable(map_root_index));
+ StoreMapNoWriteBarrier(result, map_root_index);
+ StoreObjectFieldNoWriteBarrier(result, SlicedString::kHashFieldOffset,
+ Int32Constant(String::kEmptyHashField));
+ StoreObjectFieldNoWriteBarrier(result, SlicedString::kLengthOffset, length);
+ StoreObjectFieldNoWriteBarrier(result, SlicedString::kParentOffset, parent);
+ StoreObjectFieldNoWriteBarrier(result, SlicedString::kOffsetOffset, offset);
+ return CAST(result);
+}
+
+TNode<String> CodeStubAssembler::AllocateSlicedOneByteString(
+ TNode<Uint32T> length, TNode<String> parent, TNode<Smi> offset) {
+ return AllocateSlicedString(RootIndex::kSlicedOneByteStringMap, length,
+ parent, offset);
+}
+
+TNode<String> CodeStubAssembler::AllocateSlicedTwoByteString(
+ TNode<Uint32T> length, TNode<String> parent, TNode<Smi> offset) {
+ return AllocateSlicedString(RootIndex::kSlicedStringMap, length, parent,
+ offset);
+}
+
+TNode<NameDictionary> CodeStubAssembler::AllocateNameDictionary(
+ int at_least_space_for) {
+ return AllocateNameDictionary(IntPtrConstant(at_least_space_for));
+}
+
+TNode<NameDictionary> CodeStubAssembler::AllocateNameDictionary(
+ TNode<IntPtrT> at_least_space_for, AllocationFlags flags) {
+ CSA_ASSERT(this, UintPtrLessThanOrEqual(
+ at_least_space_for,
+ IntPtrConstant(NameDictionary::kMaxCapacity)));
+ TNode<IntPtrT> capacity = HashTableComputeCapacity(at_least_space_for);
+ return AllocateNameDictionaryWithCapacity(capacity, flags);
+}
+
+TNode<NameDictionary> CodeStubAssembler::AllocateNameDictionaryWithCapacity(
+ TNode<IntPtrT> capacity, AllocationFlags flags) {
+ CSA_ASSERT(this, WordIsPowerOfTwo(capacity));
+ CSA_ASSERT(this, IntPtrGreaterThan(capacity, IntPtrConstant(0)));
+ TNode<IntPtrT> length = EntryToIndex<NameDictionary>(capacity);
+ TNode<IntPtrT> store_size = IntPtrAdd(
+ TimesTaggedSize(length), IntPtrConstant(NameDictionary::kHeaderSize));
+
+ TNode<NameDictionary> result =
+ UncheckedCast<NameDictionary>(Allocate(store_size, flags));
+
+ // Initialize FixedArray fields.
+ {
+ DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kNameDictionaryMap));
+ StoreMapNoWriteBarrier(result, RootIndex::kNameDictionaryMap);
+ StoreObjectFieldNoWriteBarrier(result, FixedArray::kLengthOffset,
+ SmiFromIntPtr(length));
+ }
+
+ // Initialized HashTable fields.
+ {
+ TNode<Smi> zero = SmiConstant(0);
+ StoreFixedArrayElement(result, NameDictionary::kNumberOfElementsIndex, zero,
+ SKIP_WRITE_BARRIER);
+ StoreFixedArrayElement(result,
+ NameDictionary::kNumberOfDeletedElementsIndex, zero,
+ SKIP_WRITE_BARRIER);
+ StoreFixedArrayElement(result, NameDictionary::kCapacityIndex,
+ SmiTag(capacity), SKIP_WRITE_BARRIER);
+ // Initialize Dictionary fields.
+ StoreFixedArrayElement(result, NameDictionary::kNextEnumerationIndexIndex,
+ SmiConstant(PropertyDetails::kInitialIndex),
+ SKIP_WRITE_BARRIER);
+ StoreFixedArrayElement(result, NameDictionary::kObjectHashIndex,
+ SmiConstant(PropertyArray::kNoHashSentinel),
+ SKIP_WRITE_BARRIER);
+ }
+
+ // Initialize NameDictionary elements.
+ {
+ TNode<IntPtrT> result_word = BitcastTaggedToWord(result);
+ TNode<IntPtrT> start_address = IntPtrAdd(
+ result_word, IntPtrConstant(NameDictionary::OffsetOfElementAt(
+ NameDictionary::kElementsStartIndex) -
+ kHeapObjectTag));
+ TNode<IntPtrT> end_address = IntPtrAdd(
+ result_word, IntPtrSub(store_size, IntPtrConstant(kHeapObjectTag)));
+
+ TNode<Oddball> filler = UndefinedConstant();
+ DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kUndefinedValue));
+
+ StoreFieldsNoWriteBarrier(start_address, end_address, filler);
+ }
+
+ return result;
+}
+
+TNode<NameDictionary> CodeStubAssembler::CopyNameDictionary(
+ TNode<NameDictionary> dictionary, Label* large_object_fallback) {
+ Comment("Copy boilerplate property dict");
+ TNode<IntPtrT> capacity = SmiUntag(GetCapacity<NameDictionary>(dictionary));
+ CSA_ASSERT(this, IntPtrGreaterThanOrEqual(capacity, IntPtrConstant(0)));
+ GotoIf(UintPtrGreaterThan(
+ capacity, IntPtrConstant(NameDictionary::kMaxRegularCapacity)),
+ large_object_fallback);
+ TNode<NameDictionary> properties =
+ AllocateNameDictionaryWithCapacity(capacity);
+ TNode<IntPtrT> length = SmiUntag(LoadFixedArrayBaseLength(dictionary));
+ CopyFixedArrayElements(PACKED_ELEMENTS, dictionary, properties, length,
+ SKIP_WRITE_BARRIER);
+ return properties;
+}
+
+template <typename CollectionType>
+TNode<CollectionType> CodeStubAssembler::AllocateOrderedHashTable() {
+ static const int kCapacity = CollectionType::kInitialCapacity;
+ static const int kBucketCount = kCapacity / CollectionType::kLoadFactor;
+ static const int kDataTableLength = kCapacity * CollectionType::kEntrySize;
+ static const int kFixedArrayLength =
+ CollectionType::HashTableStartIndex() + kBucketCount + kDataTableLength;
+ static const int kDataTableStartIndex =
+ CollectionType::HashTableStartIndex() + kBucketCount;
+
+ STATIC_ASSERT(base::bits::IsPowerOfTwo(kCapacity));
+ STATIC_ASSERT(kCapacity <= CollectionType::MaxCapacity());
+
+ // Allocate the table and add the proper map.
+ const ElementsKind elements_kind = HOLEY_ELEMENTS;
+ TNode<IntPtrT> length_intptr = IntPtrConstant(kFixedArrayLength);
+ TNode<Map> fixed_array_map =
+ HeapConstant(CollectionType::GetMap(ReadOnlyRoots(isolate())));
+ TNode<CollectionType> table =
+ CAST(AllocateFixedArray(elements_kind, length_intptr,
+ kAllowLargeObjectAllocation, fixed_array_map));
+
+ // Initialize the OrderedHashTable fields.
+ const WriteBarrierMode barrier_mode = SKIP_WRITE_BARRIER;
+ StoreFixedArrayElement(table, CollectionType::NumberOfElementsIndex(),
+ SmiConstant(0), barrier_mode);
+ StoreFixedArrayElement(table, CollectionType::NumberOfDeletedElementsIndex(),
+ SmiConstant(0), barrier_mode);
+ StoreFixedArrayElement(table, CollectionType::NumberOfBucketsIndex(),
+ SmiConstant(kBucketCount), barrier_mode);
+
+ // Fill the buckets with kNotFound.
+ TNode<Smi> not_found = SmiConstant(CollectionType::kNotFound);
+ STATIC_ASSERT(CollectionType::HashTableStartIndex() ==
+ CollectionType::NumberOfBucketsIndex() + 1);
+ STATIC_ASSERT((CollectionType::HashTableStartIndex() + kBucketCount) ==
+ kDataTableStartIndex);
+ for (int i = 0; i < kBucketCount; i++) {
+ StoreFixedArrayElement(table, CollectionType::HashTableStartIndex() + i,
+ not_found, barrier_mode);
+ }
+
+ // Fill the data table with undefined.
+ STATIC_ASSERT(kDataTableStartIndex + kDataTableLength == kFixedArrayLength);
+ for (int i = 0; i < kDataTableLength; i++) {
+ StoreFixedArrayElement(table, kDataTableStartIndex + i, UndefinedConstant(),
+ barrier_mode);
+ }
+
+ return table;
+}
+
+template TNode<OrderedHashMap>
+CodeStubAssembler::AllocateOrderedHashTable<OrderedHashMap>();
+template TNode<OrderedHashSet>
+CodeStubAssembler::AllocateOrderedHashTable<OrderedHashSet>();
+
+TNode<JSObject> CodeStubAssembler::AllocateJSObjectFromMap(
+ TNode<Map> map, base::Optional<TNode<HeapObject>> properties,
+ base::Optional<TNode<FixedArray>> elements, AllocationFlags flags,
+ SlackTrackingMode slack_tracking_mode) {
+ CSA_ASSERT(this, Word32BinaryNot(IsJSFunctionMap(map)));
+ CSA_ASSERT(this, Word32BinaryNot(InstanceTypeEqual(LoadMapInstanceType(map),
+ JS_GLOBAL_OBJECT_TYPE)));
+ TNode<IntPtrT> instance_size =
+ TimesTaggedSize(LoadMapInstanceSizeInWords(map));
+ TNode<HeapObject> object = AllocateInNewSpace(instance_size, flags);
+ StoreMapNoWriteBarrier(object, map);
+ InitializeJSObjectFromMap(object, map, instance_size, properties, elements,
+ slack_tracking_mode);
+ return CAST(object);
+}
+
+void CodeStubAssembler::InitializeJSObjectFromMap(
+ TNode<HeapObject> object, TNode<Map> map, TNode<IntPtrT> instance_size,
+ base::Optional<TNode<HeapObject>> properties,
+ base::Optional<TNode<FixedArray>> elements,
+ SlackTrackingMode slack_tracking_mode) {
+ // This helper assumes that the object is in new-space, as guarded by the
+ // check in AllocatedJSObjectFromMap.
+ if (!properties) {
+ CSA_ASSERT(this, Word32BinaryNot(IsDictionaryMap((map))));
+ StoreObjectFieldRoot(object, JSObject::kPropertiesOrHashOffset,
+ RootIndex::kEmptyFixedArray);
+ } else {
+ CSA_ASSERT(this, Word32Or(Word32Or(IsPropertyArray(*properties),
+ IsNameDictionary(*properties)),
+ IsEmptyFixedArray(*properties)));
+ StoreObjectFieldNoWriteBarrier(object, JSObject::kPropertiesOrHashOffset,
+ *properties);
+ }
+ if (!elements) {
+ StoreObjectFieldRoot(object, JSObject::kElementsOffset,
+ RootIndex::kEmptyFixedArray);
+ } else {
+ StoreObjectFieldNoWriteBarrier(object, JSObject::kElementsOffset,
+ *elements);
+ }
+ if (slack_tracking_mode == kNoSlackTracking) {
+ InitializeJSObjectBodyNoSlackTracking(object, map, instance_size);
+ } else {
+ DCHECK_EQ(slack_tracking_mode, kWithSlackTracking);
+ InitializeJSObjectBodyWithSlackTracking(object, map, instance_size);
+ }
+}
+
+void CodeStubAssembler::InitializeJSObjectBodyNoSlackTracking(
+ TNode<HeapObject> object, TNode<Map> map,
+ SloppyTNode<IntPtrT> instance_size, int start_offset) {
+ STATIC_ASSERT(Map::kNoSlackTracking == 0);
+ CSA_ASSERT(this, IsClearWord32<Map::Bits3::ConstructionCounterBits>(
+ LoadMapBitField3(map)));
+ InitializeFieldsWithRoot(object, IntPtrConstant(start_offset), instance_size,
+ RootIndex::kUndefinedValue);
+}
+
+void CodeStubAssembler::InitializeJSObjectBodyWithSlackTracking(
+ TNode<HeapObject> object, TNode<Map> map,
+ SloppyTNode<IntPtrT> instance_size) {
+ Comment("InitializeJSObjectBodyNoSlackTracking");
+
+ // Perform in-object slack tracking if requested.
+ int start_offset = JSObject::kHeaderSize;
+ TNode<Uint32T> bit_field3 = LoadMapBitField3(map);
+ Label end(this), slack_tracking(this), complete(this, Label::kDeferred);
+ STATIC_ASSERT(Map::kNoSlackTracking == 0);
+ GotoIf(IsSetWord32<Map::Bits3::ConstructionCounterBits>(bit_field3),
+ &slack_tracking);
+ Comment("No slack tracking");
+ InitializeJSObjectBodyNoSlackTracking(object, map, instance_size);
+ Goto(&end);
+
+ BIND(&slack_tracking);
+ {
+ Comment("Decrease construction counter");
+ // Slack tracking is only done on initial maps.
+ CSA_ASSERT(this, IsUndefined(LoadMapBackPointer(map)));
+ STATIC_ASSERT(Map::Bits3::ConstructionCounterBits::kLastUsedBit == 31);
+ TNode<Word32T> new_bit_field3 = Int32Sub(
+ bit_field3,
+ Int32Constant(1 << Map::Bits3::ConstructionCounterBits::kShift));
+ StoreObjectFieldNoWriteBarrier(map, Map::kBitField3Offset, new_bit_field3);
+ STATIC_ASSERT(Map::kSlackTrackingCounterEnd == 1);
+
+ // The object still has in-object slack therefore the |unsed_or_unused|
+ // field contain the "used" value.
+ TNode<IntPtrT> used_size =
+ Signed(TimesTaggedSize(ChangeUint32ToWord(LoadObjectField<Uint8T>(
+ map, Map::kUsedOrUnusedInstanceSizeInWordsOffset))));
+
+ Comment("iInitialize filler fields");
+ InitializeFieldsWithRoot(object, used_size, instance_size,
+ RootIndex::kOnePointerFillerMap);
+
+ Comment("Initialize undefined fields");
+ InitializeFieldsWithRoot(object, IntPtrConstant(start_offset), used_size,
+ RootIndex::kUndefinedValue);
+
+ STATIC_ASSERT(Map::kNoSlackTracking == 0);
+ GotoIf(IsClearWord32<Map::Bits3::ConstructionCounterBits>(new_bit_field3),
+ &complete);
+ Goto(&end);
+ }
+
+ // Finalize the instance size.
+ BIND(&complete);
+ {
+ // ComplextInobjectSlackTracking doesn't allocate and thus doesn't need a
+ // context.
+ CallRuntime(Runtime::kCompleteInobjectSlackTrackingForMap,
+ NoContextConstant(), map);
+ Goto(&end);
+ }
+
+ BIND(&end);
+}
+
+void CodeStubAssembler::StoreFieldsNoWriteBarrier(TNode<IntPtrT> start_address,
+ TNode<IntPtrT> end_address,
+ TNode<Object> value) {
+ Comment("StoreFieldsNoWriteBarrier");
+ CSA_ASSERT(this, WordIsAligned(start_address, kTaggedSize));
+ CSA_ASSERT(this, WordIsAligned(end_address, kTaggedSize));
+ BuildFastLoop<IntPtrT>(
+ start_address, end_address,
+ [=](TNode<IntPtrT> current) {
+ UnsafeStoreNoWriteBarrier(MachineRepresentation::kTagged, current,
+ value);
+ },
+ kTaggedSize, IndexAdvanceMode::kPost);
+}
+
+void CodeStubAssembler::MakeFixedArrayCOW(TNode<FixedArray> array) {
+ CSA_ASSERT(this, IsFixedArrayMap(LoadMap(array)));
+ Label done(this);
+ // The empty fixed array is not modifiable anyway. And we shouldn't change its
+ // Map.
+ GotoIf(TaggedEqual(array, EmptyFixedArrayConstant()), &done);
+ StoreMap(array, FixedCOWArrayMapConstant());
+ Goto(&done);
+ BIND(&done);
+}
+
+TNode<BoolT> CodeStubAssembler::IsValidFastJSArrayCapacity(
+ TNode<IntPtrT> capacity) {
+ return UintPtrLessThanOrEqual(capacity,
+ UintPtrConstant(JSArray::kMaxFastArrayLength));
+}
+
+TNode<JSArray> CodeStubAssembler::AllocateJSArray(
+ TNode<Map> array_map, TNode<FixedArrayBase> elements, TNode<Smi> length,
+ base::Optional<TNode<AllocationSite>> allocation_site,
+ int array_header_size) {
+ Comment("begin allocation of JSArray passing in elements");
+ CSA_SLOW_ASSERT(this, TaggedIsPositiveSmi(length));
+
+ int base_size = array_header_size;
+ if (allocation_site) {
+ base_size += AllocationMemento::kSize;
+ }
+
+ TNode<IntPtrT> size = IntPtrConstant(base_size);
+ TNode<JSArray> result =
+ AllocateUninitializedJSArray(array_map, length, allocation_site, size);
+ StoreObjectFieldNoWriteBarrier(result, JSArray::kElementsOffset, elements);
+ return result;
+}
+
+std::pair<TNode<JSArray>, TNode<FixedArrayBase>>
+CodeStubAssembler::AllocateUninitializedJSArrayWithElements(
+ ElementsKind kind, TNode<Map> array_map, TNode<Smi> length,
+ base::Optional<TNode<AllocationSite>> allocation_site,
+ TNode<IntPtrT> capacity, AllocationFlags allocation_flags,
+ int array_header_size) {
+ Comment("begin allocation of JSArray with elements");
+ CHECK_EQ(allocation_flags & ~kAllowLargeObjectAllocation, 0);
+ CSA_SLOW_ASSERT(this, TaggedIsPositiveSmi(length));
+
+ TVARIABLE(JSArray, array);
+ TVARIABLE(FixedArrayBase, elements);
+
+ Label out(this), empty(this), nonempty(this);
+
+ int capacity_int;
+ if (ToInt32Constant(capacity, &capacity_int)) {
+ if (capacity_int == 0) {
+ TNode<FixedArray> empty_array = EmptyFixedArrayConstant();
+ array = AllocateJSArray(array_map, empty_array, length, allocation_site,
+ array_header_size);
+ return {array.value(), empty_array};
+ } else {
+ Goto(&nonempty);
+ }
+ } else {
+ Branch(WordEqual(capacity, IntPtrConstant(0)), &empty, &nonempty);
+
+ BIND(&empty);
+ {
+ TNode<FixedArray> empty_array = EmptyFixedArrayConstant();
+ array = AllocateJSArray(array_map, empty_array, length, allocation_site,
+ array_header_size);
+ elements = empty_array;
+ Goto(&out);
+ }
+ }
+
+ BIND(&nonempty);
+ {
+ int base_size = array_header_size;
+ if (allocation_site) {
+ base_size += AllocationMemento::kSize;
+ }
+
+ const int elements_offset = base_size;
+
+ // Compute space for elements
+ base_size += FixedArray::kHeaderSize;
+ TNode<IntPtrT> size = ElementOffsetFromIndex(capacity, kind, base_size);
+
+ // For very large arrays in which the requested allocation exceeds the
+ // maximal size of a regular heap object, we cannot use the allocation
+ // folding trick. Instead, we first allocate the elements in large object
+ // space, and then allocate the JSArray (and possibly the allocation
+ // memento) in new space.
+ if (allocation_flags & kAllowLargeObjectAllocation) {
+ Label next(this);
+ GotoIf(IsRegularHeapObjectSize(size), &next);
+
+ CSA_CHECK(this, IsValidFastJSArrayCapacity(capacity));
+
+ // Allocate and initialize the elements first. Full initialization is
+ // needed because the upcoming JSArray allocation could trigger GC.
+ elements = AllocateFixedArray(kind, capacity, allocation_flags);
+
+ if (IsDoubleElementsKind(kind)) {
+ FillFixedDoubleArrayWithZero(CAST(elements.value()), capacity);
+ } else {
+ FillFixedArrayWithSmiZero(CAST(elements.value()), capacity);
+ }
+
+ // The JSArray and possibly allocation memento next. Note that
+ // allocation_flags are *not* passed on here and the resulting JSArray
+ // will always be in new space.
+ array = AllocateJSArray(array_map, elements.value(), length,
+ allocation_site, array_header_size);
+
+ Goto(&out);
+
+ BIND(&next);
+ }
+
+ // Fold all objects into a single new space allocation.
+ array =
+ AllocateUninitializedJSArray(array_map, length, allocation_site, size);
+ elements = UncheckedCast<FixedArrayBase>(
+ InnerAllocate(array.value(), elements_offset));
+
+ StoreObjectFieldNoWriteBarrier(array.value(), JSObject::kElementsOffset,
+ elements.value());
+
+ // Setup elements object.
+ STATIC_ASSERT(FixedArrayBase::kHeaderSize == 2 * kTaggedSize);
+ RootIndex elements_map_index = IsDoubleElementsKind(kind)
+ ? RootIndex::kFixedDoubleArrayMap
+ : RootIndex::kFixedArrayMap;
+ DCHECK(RootsTable::IsImmortalImmovable(elements_map_index));
+ StoreMapNoWriteBarrier(elements.value(), elements_map_index);
+
+ CSA_ASSERT(this, WordNotEqual(capacity, IntPtrConstant(0)));
+ TNode<Smi> capacity_smi = SmiTag(capacity);
+ StoreObjectFieldNoWriteBarrier(elements.value(), FixedArray::kLengthOffset,
+ capacity_smi);
+ Goto(&out);
+ }
+
+ BIND(&out);
+ return {array.value(), elements.value()};
+}
+
+TNode<JSArray> CodeStubAssembler::AllocateUninitializedJSArray(
+ TNode<Map> array_map, TNode<Smi> length,
+ base::Optional<TNode<AllocationSite>> allocation_site,
+ TNode<IntPtrT> size_in_bytes) {
+ CSA_SLOW_ASSERT(this, TaggedIsPositiveSmi(length));
+
+ // Allocate space for the JSArray and the elements FixedArray in one go.
+ TNode<HeapObject> array = AllocateInNewSpace(size_in_bytes);
+
+ StoreMapNoWriteBarrier(array, array_map);
+ StoreObjectFieldNoWriteBarrier(array, JSArray::kLengthOffset, length);
+ StoreObjectFieldRoot(array, JSArray::kPropertiesOrHashOffset,
+ RootIndex::kEmptyFixedArray);
+
+ if (allocation_site) {
+ InitializeAllocationMemento(array, IntPtrConstant(JSArray::kHeaderSize),
+ *allocation_site);
+ }
+
+ return CAST(array);
+}
+
+TNode<JSArray> CodeStubAssembler::AllocateJSArray(
+ ElementsKind kind, TNode<Map> array_map, TNode<IntPtrT> capacity,
+ TNode<Smi> length, base::Optional<TNode<AllocationSite>> allocation_site,
+ AllocationFlags allocation_flags) {
+ CSA_SLOW_ASSERT(this, TaggedIsPositiveSmi(length));
+
+ TNode<JSArray> array;
+ TNode<FixedArrayBase> elements;
+
+ std::tie(array, elements) = AllocateUninitializedJSArrayWithElements(
+ kind, array_map, length, allocation_site, capacity, allocation_flags);
+
+ Label out(this), nonempty(this);
+
+ Branch(WordEqual(capacity, IntPtrConstant(0)), &out, &nonempty);
+
+ BIND(&nonempty);
+ {
+ FillFixedArrayWithValue(kind, elements, IntPtrConstant(0), capacity,
+ RootIndex::kTheHoleValue);
+ Goto(&out);
+ }
+
+ BIND(&out);
+ return array;
+}
+
+TNode<JSArray> CodeStubAssembler::ExtractFastJSArray(TNode<Context> context,
+ TNode<JSArray> array,
+ TNode<BInt> begin,
+ TNode<BInt> count) {
+ TNode<Map> original_array_map = LoadMap(array);
+ TNode<Int32T> elements_kind = LoadMapElementsKind(original_array_map);
+
+ // Use the canonical map for the Array's ElementsKind
+ TNode<NativeContext> native_context = LoadNativeContext(context);
+ TNode<Map> array_map = LoadJSArrayElementsMap(elements_kind, native_context);
+
+ TNode<FixedArrayBase> new_elements = ExtractFixedArray(
+ LoadElements(array), base::Optional<TNode<BInt>>(begin),
+ base::Optional<TNode<BInt>>(count),
+ base::Optional<TNode<BInt>>(base::nullopt),
+ ExtractFixedArrayFlag::kAllFixedArrays, nullptr, elements_kind);
+
+ TNode<JSArray> result = AllocateJSArray(
+ array_map, new_elements, ParameterToTagged(count), base::nullopt);
+ return result;
+}
+
+TNode<JSArray> CodeStubAssembler::CloneFastJSArray(
+ TNode<Context> context, TNode<JSArray> array,
+ base::Optional<TNode<AllocationSite>> allocation_site,
+ HoleConversionMode convert_holes) {
+ // TODO(dhai): we should be able to assert IsFastJSArray(array) here, but this
+ // function is also used to copy boilerplates even when the no-elements
+ // protector is invalid. This function should be renamed to reflect its uses.
+
+ TNode<Number> length = LoadJSArrayLength(array);
+ TNode<FixedArrayBase> new_elements;
+ TVARIABLE(FixedArrayBase, var_new_elements);
+ TVARIABLE(Int32T, var_elements_kind, LoadMapElementsKind(LoadMap(array)));
+
+ Label allocate_jsarray(this), holey_extract(this),
+ allocate_jsarray_main(this);
+
+ bool need_conversion =
+ convert_holes == HoleConversionMode::kConvertToUndefined;
+ if (need_conversion) {
+ // We need to take care of holes, if the array is of holey elements kind.
+ GotoIf(IsHoleyFastElementsKindForRead(var_elements_kind.value()),
+ &holey_extract);
+ }
+
+ // Simple extraction that preserves holes.
+ new_elements = ExtractFixedArray(
+ LoadElements(array),
+ base::Optional<TNode<BInt>>(IntPtrOrSmiConstant<BInt>(0)),
+ base::Optional<TNode<BInt>>(TaggedToParameter<BInt>(CAST(length))),
+ base::Optional<TNode<BInt>>(base::nullopt),
+ ExtractFixedArrayFlag::kAllFixedArraysDontCopyCOW, nullptr,
+ var_elements_kind.value());
+ var_new_elements = new_elements;
+ Goto(&allocate_jsarray);
+
+ if (need_conversion) {
+ BIND(&holey_extract);
+ // Convert holes to undefined.
+ TVARIABLE(BoolT, var_holes_converted, Int32FalseConstant());
+ // Copy |array|'s elements store. The copy will be compatible with the
+ // original elements kind unless there are holes in the source. Any holes
+ // get converted to undefined, hence in that case the copy is compatible
+ // only with PACKED_ELEMENTS and HOLEY_ELEMENTS, and we will choose
+ // PACKED_ELEMENTS. Also, if we want to replace holes, we must not use
+ // ExtractFixedArrayFlag::kDontCopyCOW.
+ new_elements = ExtractFixedArray(
+ LoadElements(array),
+ base::Optional<TNode<BInt>>(IntPtrOrSmiConstant<BInt>(0)),
+ base::Optional<TNode<BInt>>(TaggedToParameter<BInt>(CAST(length))),
+ base::Optional<TNode<BInt>>(base::nullopt),
+ ExtractFixedArrayFlag::kAllFixedArrays, &var_holes_converted);
+ var_new_elements = new_elements;
+ // If the array type didn't change, use the original elements kind.
+ GotoIfNot(var_holes_converted.value(), &allocate_jsarray);
+ // Otherwise use PACKED_ELEMENTS for the target's elements kind.
+ var_elements_kind = Int32Constant(PACKED_ELEMENTS);
+ Goto(&allocate_jsarray);
+ }
+
+ BIND(&allocate_jsarray);
+
+ // Handle any nonextensible elements kinds
+ CSA_ASSERT(this, IsElementsKindLessThanOrEqual(
+ var_elements_kind.value(),
+ LAST_ANY_NONEXTENSIBLE_ELEMENTS_KIND));
+ GotoIf(IsElementsKindLessThanOrEqual(var_elements_kind.value(),
+ LAST_FAST_ELEMENTS_KIND),
+ &allocate_jsarray_main);
+ var_elements_kind = Int32Constant(PACKED_ELEMENTS);
+ Goto(&allocate_jsarray_main);
+
+ BIND(&allocate_jsarray_main);
+ // Use the cannonical map for the chosen elements kind.
+ TNode<NativeContext> native_context = LoadNativeContext(context);
+ TNode<Map> array_map =
+ LoadJSArrayElementsMap(var_elements_kind.value(), native_context);
+
+ TNode<JSArray> result = AllocateJSArray(array_map, var_new_elements.value(),
+ CAST(length), allocation_site);
+ return result;
+}
+
+template <typename TIndex>
+TNode<FixedArrayBase> CodeStubAssembler::AllocateFixedArray(
+ ElementsKind kind, TNode<TIndex> capacity, AllocationFlags flags,
+ base::Optional<TNode<Map>> fixed_array_map) {
+ static_assert(
+ std::is_same<TIndex, Smi>::value || std::is_same<TIndex, IntPtrT>::value,
+ "Only Smi or IntPtrT capacity is allowed");
+ Comment("AllocateFixedArray");
+ CSA_ASSERT(this,
+ IntPtrOrSmiGreaterThan(capacity, IntPtrOrSmiConstant<TIndex>(0)));
+
+ const intptr_t kMaxLength = IsDoubleElementsKind(kind)
+ ? FixedDoubleArray::kMaxLength
+ : FixedArray::kMaxLength;
+ intptr_t capacity_constant;
+ if (ToParameterConstant(capacity, &capacity_constant)) {
+ CHECK_LE(capacity_constant, kMaxLength);
+ } else {
+ Label if_out_of_memory(this, Label::kDeferred), next(this);
+ Branch(IntPtrOrSmiGreaterThan(capacity, IntPtrOrSmiConstant<TIndex>(
+ static_cast<int>(kMaxLength))),
+ &if_out_of_memory, &next);
+
+ BIND(&if_out_of_memory);
+ CallRuntime(Runtime::kFatalProcessOutOfMemoryInvalidArrayLength,
+ NoContextConstant());
+ Unreachable();
+
+ BIND(&next);
+ }
+
+ TNode<IntPtrT> total_size = GetFixedArrayAllocationSize(capacity, kind);
+
+ if (IsDoubleElementsKind(kind)) flags |= kDoubleAlignment;
+ // Allocate both array and elements object, and initialize the JSArray.
+ TNode<HeapObject> array = Allocate(total_size, flags);
+ if (fixed_array_map) {
+ // Conservatively only skip the write barrier if there are no allocation
+ // flags, this ensures that the object hasn't ended up in LOS. Note that the
+ // fixed array map is currently always immortal and technically wouldn't
+ // need the write barrier even in LOS, but it's better to not take chances
+ // in case this invariant changes later, since it's difficult to enforce
+ // locally here.
+ if (flags == CodeStubAssembler::kNone) {
+ StoreMapNoWriteBarrier(array, *fixed_array_map);
+ } else {
+ StoreMap(array, *fixed_array_map);
+ }
+ } else {
+ RootIndex map_index = IsDoubleElementsKind(kind)
+ ? RootIndex::kFixedDoubleArrayMap
+ : RootIndex::kFixedArrayMap;
+ DCHECK(RootsTable::IsImmortalImmovable(map_index));
+ StoreMapNoWriteBarrier(array, map_index);
+ }
+ StoreObjectFieldNoWriteBarrier(array, FixedArrayBase::kLengthOffset,
+ ParameterToTagged(capacity));
+ return UncheckedCast<FixedArrayBase>(array);
+}
+
+// There is no need to export the Smi version since it is only used inside
+// code-stub-assembler.
+template V8_EXPORT_PRIVATE TNode<FixedArrayBase>
+ CodeStubAssembler::AllocateFixedArray<IntPtrT>(ElementsKind, TNode<IntPtrT>,
+ AllocationFlags,
+ base::Optional<TNode<Map>>);
+
+template <typename TIndex>
+TNode<FixedArray> CodeStubAssembler::ExtractToFixedArray(
+ TNode<FixedArrayBase> source, TNode<TIndex> first, TNode<TIndex> count,
+ TNode<TIndex> capacity, TNode<Map> source_map, ElementsKind from_kind,
+ AllocationFlags allocation_flags, ExtractFixedArrayFlags extract_flags,
+ HoleConversionMode convert_holes, TVariable<BoolT>* var_holes_converted,
+ base::Optional<TNode<Int32T>> source_elements_kind) {
+ static_assert(
+ std::is_same<TIndex, Smi>::value || std::is_same<TIndex, IntPtrT>::value,
+ "Only Smi or IntPtrT first, count, and capacity are allowed");
+
+ DCHECK(extract_flags & ExtractFixedArrayFlag::kFixedArrays);
+ CSA_ASSERT(this,
+ IntPtrOrSmiNotEqual(IntPtrOrSmiConstant<TIndex>(0), capacity));
+ CSA_ASSERT(this, TaggedEqual(source_map, LoadMap(source)));
+
+ TVARIABLE(FixedArrayBase, var_result);
+ TVARIABLE(Map, var_target_map, source_map);
+
+ Label done(this, {&var_result}), is_cow(this),
+ new_space_check(this, {&var_target_map});
+
+ // If source_map is either FixedDoubleArrayMap, or FixedCOWArrayMap but
+ // we can't just use COW, use FixedArrayMap as the target map. Otherwise, use
+ // source_map as the target map.
+ if (IsDoubleElementsKind(from_kind)) {
+ CSA_ASSERT(this, IsFixedDoubleArrayMap(source_map));
+ var_target_map = FixedArrayMapConstant();
+ Goto(&new_space_check);
+ } else {
+ CSA_ASSERT(this, Word32BinaryNot(IsFixedDoubleArrayMap(source_map)));
+ Branch(TaggedEqual(var_target_map.value(), FixedCOWArrayMapConstant()),
+ &is_cow, &new_space_check);
+
+ BIND(&is_cow);
+ {
+ // |source| is a COW array, so we don't actually need to allocate a new
+ // array unless:
+ // 1) |extract_flags| forces us to, or
+ // 2) we're asked to extract only part of the |source| (|first| != 0).
+ if (extract_flags & ExtractFixedArrayFlag::kDontCopyCOW) {
+ Branch(IntPtrOrSmiNotEqual(IntPtrOrSmiConstant<TIndex>(0), first),
+ &new_space_check, [&] {
+ var_result = source;
+ Goto(&done);
+ });
+ } else {
+ var_target_map = FixedArrayMapConstant();
+ Goto(&new_space_check);
+ }
+ }
+ }
+
+ BIND(&new_space_check);
+ {
+ bool handle_old_space = !FLAG_young_generation_large_objects;
+ if (handle_old_space) {
+ if (extract_flags & ExtractFixedArrayFlag::kNewSpaceAllocationOnly) {
+ handle_old_space = false;
+ CSA_ASSERT(this, Word32BinaryNot(FixedArraySizeDoesntFitInNewSpace(
+ count, FixedArray::kHeaderSize)));
+ } else {
+ int constant_count;
+ handle_old_space =
+ !TryGetIntPtrOrSmiConstantValue(count, &constant_count) ||
+ (constant_count >
+ FixedArray::GetMaxLengthForNewSpaceAllocation(PACKED_ELEMENTS));
+ }
+ }
+
+ Label old_space(this, Label::kDeferred);
+ if (handle_old_space) {
+ GotoIfFixedArraySizeDoesntFitInNewSpace(capacity, &old_space,
+ FixedArray::kHeaderSize);
+ }
+
+ Comment("Copy FixedArray in young generation");
+ // We use PACKED_ELEMENTS to tell AllocateFixedArray and
+ // CopyFixedArrayElements that we want a FixedArray.
+ const ElementsKind to_kind = PACKED_ELEMENTS;
+ TNode<FixedArrayBase> to_elements = AllocateFixedArray(
+ to_kind, capacity, allocation_flags, var_target_map.value());
+ var_result = to_elements;
+
+#ifndef V8_ENABLE_SINGLE_GENERATION
+#ifdef DEBUG
+ TNode<IntPtrT> object_word = BitcastTaggedToWord(to_elements);
+ TNode<IntPtrT> object_page = PageFromAddress(object_word);
+ TNode<IntPtrT> page_flags =
+ Load<IntPtrT>(object_page, IntPtrConstant(Page::kFlagsOffset));
+ CSA_ASSERT(
+ this,
+ WordNotEqual(
+ WordAnd(page_flags,
+ IntPtrConstant(MemoryChunk::kIsInYoungGenerationMask)),
+ IntPtrConstant(0)));
+#endif
+#endif
+
+ if (convert_holes == HoleConversionMode::kDontConvert &&
+ !IsDoubleElementsKind(from_kind)) {
+ // We can use CopyElements (memcpy) because we don't need to replace or
+ // convert any values. Since {to_elements} is in new-space, CopyElements
+ // will efficiently use memcpy.
+ FillFixedArrayWithValue(to_kind, to_elements, count, capacity,
+ RootIndex::kTheHoleValue);
+ CopyElements(to_kind, to_elements, IntPtrConstant(0), source,
+ ParameterToIntPtr(first), ParameterToIntPtr(count),
+ SKIP_WRITE_BARRIER);
+ } else {
+ CopyFixedArrayElements(from_kind, source, to_kind, to_elements, first,
+ count, capacity, SKIP_WRITE_BARRIER, convert_holes,
+ var_holes_converted);
+ }
+ Goto(&done);
+
+ if (handle_old_space) {
+ BIND(&old_space);
+ {
+ Comment("Copy FixedArray in old generation");
+ Label copy_one_by_one(this);
+
+ // Try to use memcpy if we don't need to convert holes to undefined.
+ if (convert_holes == HoleConversionMode::kDontConvert &&
+ source_elements_kind) {
+ // Only try memcpy if we're not copying object pointers.
+ GotoIfNot(IsFastSmiElementsKind(*source_elements_kind),
+ ©_one_by_one);
+
+ const ElementsKind to_smi_kind = PACKED_SMI_ELEMENTS;
+ to_elements = AllocateFixedArray(
+ to_smi_kind, capacity, allocation_flags, var_target_map.value());
+ var_result = to_elements;
+
+ FillFixedArrayWithValue(to_smi_kind, to_elements, count, capacity,
+ RootIndex::kTheHoleValue);
+ // CopyElements will try to use memcpy if it's not conflicting with
+ // GC. Otherwise it will copy elements by elements, but skip write
+ // barriers (since we're copying smis to smis).
+ CopyElements(to_smi_kind, to_elements, IntPtrConstant(0), source,
+ ParameterToIntPtr(first), ParameterToIntPtr(count),
+ SKIP_WRITE_BARRIER);
+ Goto(&done);
+ } else {
+ Goto(©_one_by_one);
+ }
+
+ BIND(©_one_by_one);
+ {
+ to_elements = AllocateFixedArray(to_kind, capacity, allocation_flags,
+ var_target_map.value());
+ var_result = to_elements;
+ CopyFixedArrayElements(from_kind, source, to_kind, to_elements, first,
+ count, capacity, UPDATE_WRITE_BARRIER,
+ convert_holes, var_holes_converted);
+ Goto(&done);
+ }
+ }
+ }
+ }
+
+ BIND(&done);
+ return UncheckedCast<FixedArray>(var_result.value());
+}
+
+template <typename TIndex>
+TNode<FixedArrayBase> CodeStubAssembler::ExtractFixedDoubleArrayFillingHoles(
+ TNode<FixedArrayBase> from_array, TNode<TIndex> first, TNode<TIndex> count,
+ TNode<TIndex> capacity, TNode<Map> fixed_array_map,
+ TVariable<BoolT>* var_holes_converted, AllocationFlags allocation_flags,
+ ExtractFixedArrayFlags extract_flags) {
+ static_assert(
+ std::is_same<TIndex, Smi>::value || std::is_same<TIndex, IntPtrT>::value,
+ "Only Smi or IntPtrT first, count, and capacity are allowed");
+
+ DCHECK_NE(var_holes_converted, nullptr);
+ CSA_ASSERT(this, IsFixedDoubleArrayMap(fixed_array_map));
+
+ TVARIABLE(FixedArrayBase, var_result);
+ const ElementsKind kind = PACKED_DOUBLE_ELEMENTS;
+ TNode<FixedArrayBase> to_elements =
+ AllocateFixedArray(kind, capacity, allocation_flags, fixed_array_map);
+ var_result = to_elements;
+ // We first try to copy the FixedDoubleArray to a new FixedDoubleArray.
+ // |var_holes_converted| is set to False preliminarily.
+ *var_holes_converted = Int32FalseConstant();
+
+ // The construction of the loop and the offsets for double elements is
+ // extracted from CopyFixedArrayElements.
+ CSA_SLOW_ASSERT(this, IsFixedArrayWithKindOrEmpty(from_array, kind));
+ STATIC_ASSERT(FixedArray::kHeaderSize == FixedDoubleArray::kHeaderSize);
+
+ Comment("[ ExtractFixedDoubleArrayFillingHoles");
+
+ // This copy can trigger GC, so we pre-initialize the array with holes.
+ FillFixedArrayWithValue(kind, to_elements, IntPtrOrSmiConstant<TIndex>(0),
+ capacity, RootIndex::kTheHoleValue);
+
+ const int first_element_offset = FixedArray::kHeaderSize - kHeapObjectTag;
+ TNode<IntPtrT> first_from_element_offset =
+ ElementOffsetFromIndex(first, kind, 0);
+ TNode<IntPtrT> limit_offset = IntPtrAdd(first_from_element_offset,
+ IntPtrConstant(first_element_offset));
+ TVARIABLE(IntPtrT, var_from_offset,
+ ElementOffsetFromIndex(IntPtrOrSmiAdd(first, count), kind,
+ first_element_offset));
+
+ Label decrement(this, {&var_from_offset}), done(this);
+ TNode<IntPtrT> to_array_adjusted =
+ IntPtrSub(BitcastTaggedToWord(to_elements), first_from_element_offset);
+
+ Branch(WordEqual(var_from_offset.value(), limit_offset), &done, &decrement);
+
+ BIND(&decrement);
+ {
+ TNode<IntPtrT> from_offset =
+ IntPtrSub(var_from_offset.value(), IntPtrConstant(kDoubleSize));
+ var_from_offset = from_offset;
+
+ TNode<IntPtrT> to_offset = from_offset;
+
+ Label if_hole(this);
+
+ TNode<Float64T> value = LoadDoubleWithHoleCheck(
+ from_array, var_from_offset.value(), &if_hole, MachineType::Float64());
+
+ StoreNoWriteBarrier(MachineRepresentation::kFloat64, to_array_adjusted,
+ to_offset, value);
+
+ TNode<BoolT> compare = WordNotEqual(from_offset, limit_offset);
+ Branch(compare, &decrement, &done);
+
+ BIND(&if_hole);
+ // We are unlucky: there are holes! We need to restart the copy, this time
+ // we will copy the FixedDoubleArray to a new FixedArray with undefined
+ // replacing holes. We signal this to the caller through
+ // |var_holes_converted|.
+ *var_holes_converted = Int32TrueConstant();
+ to_elements =
+ ExtractToFixedArray(from_array, first, count, capacity, fixed_array_map,
+ kind, allocation_flags, extract_flags,
+ HoleConversionMode::kConvertToUndefined);
+ var_result = to_elements;
+ Goto(&done);
+ }
+
+ BIND(&done);
+ Comment("] ExtractFixedDoubleArrayFillingHoles");
+ return var_result.value();
+}
+
+template <typename TIndex>
+TNode<FixedArrayBase> CodeStubAssembler::ExtractFixedArray(
+ TNode<FixedArrayBase> source, base::Optional<TNode<TIndex>> first,
+ base::Optional<TNode<TIndex>> count, base::Optional<TNode<TIndex>> capacity,
+ ExtractFixedArrayFlags extract_flags, TVariable<BoolT>* var_holes_converted,
+ base::Optional<TNode<Int32T>> source_elements_kind) {
+ static_assert(
+ std::is_same<TIndex, Smi>::value || std::is_same<TIndex, IntPtrT>::value,
+ "Only Smi or IntPtrT first, count, and capacity are allowed");
+ DCHECK(extract_flags & ExtractFixedArrayFlag::kFixedArrays ||
+ extract_flags & ExtractFixedArrayFlag::kFixedDoubleArrays);
+ // If we want to replace holes, ExtractFixedArrayFlag::kDontCopyCOW should
+ // not be used, because that disables the iteration which detects holes.
+ DCHECK_IMPLIES(var_holes_converted != nullptr,
+ !(extract_flags & ExtractFixedArrayFlag::kDontCopyCOW));
+ HoleConversionMode convert_holes =
+ var_holes_converted != nullptr ? HoleConversionMode::kConvertToUndefined
+ : HoleConversionMode::kDontConvert;
+ TVARIABLE(FixedArrayBase, var_result);
+ const AllocationFlags allocation_flags =
+ (extract_flags & ExtractFixedArrayFlag::kNewSpaceAllocationOnly)
+ ? CodeStubAssembler::kNone
+ : CodeStubAssembler::kAllowLargeObjectAllocation;
+ if (!first) {
+ first = IntPtrOrSmiConstant<TIndex>(0);
+ }
+ if (!count) {
+ count = IntPtrOrSmiSub(
+ TaggedToParameter<TIndex>(LoadFixedArrayBaseLength(source)), *first);
+
+ CSA_ASSERT(this, IntPtrOrSmiLessThanOrEqual(IntPtrOrSmiConstant<TIndex>(0),
+ *count));
+ }
+ if (!capacity) {
+ capacity = *count;
+ } else {
+ CSA_ASSERT(this, Word32BinaryNot(IntPtrOrSmiGreaterThan(
+ IntPtrOrSmiAdd(*first, *count), *capacity)));
+ }
+
+ Label if_fixed_double_array(this), empty(this), done(this, &var_result);
+ TNode<Map> source_map = LoadMap(source);
+ GotoIf(IntPtrOrSmiEqual(IntPtrOrSmiConstant<TIndex>(0), *capacity), &empty);
+
+ if (extract_flags & ExtractFixedArrayFlag::kFixedDoubleArrays) {
+ if (extract_flags & ExtractFixedArrayFlag::kFixedArrays) {
+ GotoIf(IsFixedDoubleArrayMap(source_map), &if_fixed_double_array);
+ } else {
+ CSA_ASSERT(this, IsFixedDoubleArrayMap(source_map));
+ }
+ }
+
+ if (extract_flags & ExtractFixedArrayFlag::kFixedArrays) {
+ // Here we can only get |source| as FixedArray, never FixedDoubleArray.
+ // PACKED_ELEMENTS is used to signify that the source is a FixedArray.
+ TNode<FixedArray> to_elements = ExtractToFixedArray(
+ source, *first, *count, *capacity, source_map, PACKED_ELEMENTS,
+ allocation_flags, extract_flags, convert_holes, var_holes_converted,
+ source_elements_kind);
+ var_result = to_elements;
+ Goto(&done);
+ }
+
+ if (extract_flags & ExtractFixedArrayFlag::kFixedDoubleArrays) {
+ BIND(&if_fixed_double_array);
+ Comment("Copy FixedDoubleArray");
+
+ if (convert_holes == HoleConversionMode::kConvertToUndefined) {
+ TNode<FixedArrayBase> to_elements = ExtractFixedDoubleArrayFillingHoles(
+ source, *first, *count, *capacity, source_map, var_holes_converted,
+ allocation_flags, extract_flags);
+ var_result = to_elements;
+ } else {
+ // We use PACKED_DOUBLE_ELEMENTS to signify that both the source and
+ // the target are FixedDoubleArray. That it is PACKED or HOLEY does not
+ // matter.
+ ElementsKind kind = PACKED_DOUBLE_ELEMENTS;
+ TNode<FixedArrayBase> to_elements =
+ AllocateFixedArray(kind, *capacity, allocation_flags, source_map);
+ FillFixedArrayWithValue(kind, to_elements, *count, *capacity,
+ RootIndex::kTheHoleValue);
+ CopyElements(kind, to_elements, IntPtrConstant(0), source,
+ ParameterToIntPtr(*first), ParameterToIntPtr(*count));
+ var_result = to_elements;
+ }
+
+ Goto(&done);
+ }
+
+ BIND(&empty);
+ {
+ Comment("Copy empty array");
+
+ var_result = EmptyFixedArrayConstant();
+ Goto(&done);
+ }
+
+ BIND(&done);
+ return var_result.value();
+}
+
+template V8_EXPORT_PRIVATE TNode<FixedArrayBase>
+CodeStubAssembler::ExtractFixedArray<Smi>(
+ TNode<FixedArrayBase>, base::Optional<TNode<Smi>>,
+ base::Optional<TNode<Smi>>, base::Optional<TNode<Smi>>,
+ ExtractFixedArrayFlags, TVariable<BoolT>*, base::Optional<TNode<Int32T>>);
+
+template V8_EXPORT_PRIVATE TNode<FixedArrayBase>
+CodeStubAssembler::ExtractFixedArray<IntPtrT>(
+ TNode<FixedArrayBase>, base::Optional<TNode<IntPtrT>>,
+ base::Optional<TNode<IntPtrT>>, base::Optional<TNode<IntPtrT>>,
+ ExtractFixedArrayFlags, TVariable<BoolT>*, base::Optional<TNode<Int32T>>);
+
+void CodeStubAssembler::InitializePropertyArrayLength(
+ TNode<PropertyArray> property_array, TNode<IntPtrT> length) {
+ CSA_ASSERT(this, IntPtrGreaterThan(length, IntPtrConstant(0)));
+ CSA_ASSERT(this,
+ IntPtrLessThanOrEqual(
+ length, IntPtrConstant(PropertyArray::LengthField::kMax)));
+ StoreObjectFieldNoWriteBarrier(
+ property_array, PropertyArray::kLengthAndHashOffset, SmiTag(length));
+}
+
+TNode<PropertyArray> CodeStubAssembler::AllocatePropertyArray(
+ TNode<IntPtrT> capacity) {
+ CSA_ASSERT(this, IntPtrGreaterThan(capacity, IntPtrConstant(0)));
+ TNode<IntPtrT> total_size = GetPropertyArrayAllocationSize(capacity);
+
+ TNode<HeapObject> array = Allocate(total_size, kNone);
+ RootIndex map_index = RootIndex::kPropertyArrayMap;
+ DCHECK(RootsTable::IsImmortalImmovable(map_index));
+ StoreMapNoWriteBarrier(array, map_index);
+ TNode<PropertyArray> property_array = CAST(array);
+ InitializePropertyArrayLength(property_array, capacity);
+ return property_array;
+}
+
+void CodeStubAssembler::FillPropertyArrayWithUndefined(
+ TNode<PropertyArray> array, TNode<IntPtrT> from_index,
+ TNode<IntPtrT> to_index) {
+ ElementsKind kind = PACKED_ELEMENTS;
+ TNode<Oddball> value = UndefinedConstant();
+ BuildFastArrayForEach(
+ array, kind, from_index, to_index,
+ [this, value](TNode<HeapObject> array, TNode<IntPtrT> offset) {
+ StoreNoWriteBarrier(MachineRepresentation::kTagged, array, offset,
+ value);
+ });
+}
+
+template <typename TIndex>
+void CodeStubAssembler::FillFixedArrayWithValue(ElementsKind kind,
+ TNode<FixedArrayBase> array,
+ TNode<TIndex> from_index,
+ TNode<TIndex> to_index,
+ RootIndex value_root_index) {
+ static_assert(
+ std::is_same<TIndex, Smi>::value || std::is_same<TIndex, IntPtrT>::value,
+ "Only Smi or IntPtrT from and to are allowed");
+ CSA_SLOW_ASSERT(this, IsFixedArrayWithKind(array, kind));
+ DCHECK(value_root_index == RootIndex::kTheHoleValue ||
+ value_root_index == RootIndex::kUndefinedValue);
+
+ // Determine the value to initialize the {array} based
+ // on the {value_root_index} and the elements {kind}.
+ TNode<Object> value = LoadRoot(value_root_index);
+ TNode<Float64T> float_value;
+ if (IsDoubleElementsKind(kind)) {
+ float_value = LoadHeapNumberValue(CAST(value));
+ }
+
+ BuildFastArrayForEach(
+ array, kind, from_index, to_index,
+ [this, value, float_value, kind](TNode<HeapObject> array,
+ TNode<IntPtrT> offset) {
+ if (IsDoubleElementsKind(kind)) {
+ StoreNoWriteBarrier(MachineRepresentation::kFloat64, array, offset,
+ float_value);
+ } else {
+ StoreNoWriteBarrier(MachineRepresentation::kTagged, array, offset,
+ value);
+ }
+ });
+}
+
+template V8_EXPORT_PRIVATE void
+ CodeStubAssembler::FillFixedArrayWithValue<IntPtrT>(ElementsKind,
+ TNode<FixedArrayBase>,
+ TNode<IntPtrT>,
+ TNode<IntPtrT>,
+ RootIndex);
+template V8_EXPORT_PRIVATE void CodeStubAssembler::FillFixedArrayWithValue<Smi>(
+ ElementsKind, TNode<FixedArrayBase>, TNode<Smi>, TNode<Smi>, RootIndex);
+
+void CodeStubAssembler::StoreDoubleHole(TNode<HeapObject> object,
+ TNode<IntPtrT> offset) {
+ TNode<UintPtrT> double_hole =
+ Is64() ? ReinterpretCast<UintPtrT>(Int64Constant(kHoleNanInt64))
+ : ReinterpretCast<UintPtrT>(Int32Constant(kHoleNanLower32));
+ // TODO(danno): When we have a Float32/Float64 wrapper class that
+ // preserves double bits during manipulation, remove this code/change
+ // this to an indexed Float64 store.
+ if (Is64()) {
+ StoreNoWriteBarrier(MachineRepresentation::kWord64, object, offset,
+ double_hole);
+ } else {
+ StoreNoWriteBarrier(MachineRepresentation::kWord32, object, offset,
+ double_hole);
+ StoreNoWriteBarrier(MachineRepresentation::kWord32, object,
+ IntPtrAdd(offset, IntPtrConstant(kInt32Size)),
+ double_hole);
+ }
+}
+
+void CodeStubAssembler::StoreFixedDoubleArrayHole(TNode<FixedDoubleArray> array,
+ TNode<IntPtrT> index) {
+ TNode<IntPtrT> offset = ElementOffsetFromIndex(
+ index, PACKED_DOUBLE_ELEMENTS, FixedArray::kHeaderSize - kHeapObjectTag);
+ CSA_ASSERT(this, IsOffsetInBounds(
+ offset, LoadAndUntagFixedArrayBaseLength(array),
+ FixedDoubleArray::kHeaderSize, PACKED_DOUBLE_ELEMENTS));
+ StoreDoubleHole(array, offset);
+}
+
+void CodeStubAssembler::FillFixedArrayWithSmiZero(TNode<FixedArray> array,
+ TNode<IntPtrT> length) {
+ CSA_ASSERT(this, WordEqual(length, LoadAndUntagFixedArrayBaseLength(array)));
+
+ TNode<IntPtrT> byte_length = TimesTaggedSize(length);
+ CSA_ASSERT(this, UintPtrLessThan(length, byte_length));
+
+ static const int32_t fa_base_data_offset =
+ FixedArray::kHeaderSize - kHeapObjectTag;
+ TNode<IntPtrT> backing_store = IntPtrAdd(BitcastTaggedToWord(array),
+ IntPtrConstant(fa_base_data_offset));
+
+ // Call out to memset to perform initialization.
+ TNode<ExternalReference> memset =
+ ExternalConstant(ExternalReference::libc_memset_function());
+ STATIC_ASSERT(kSizetSize == kIntptrSize);
+ CallCFunction(memset, MachineType::Pointer(),
+ std::make_pair(MachineType::Pointer(), backing_store),
+ std::make_pair(MachineType::IntPtr(), IntPtrConstant(0)),
+ std::make_pair(MachineType::UintPtr(), byte_length));
+}
+
+void CodeStubAssembler::FillFixedDoubleArrayWithZero(
+ TNode<FixedDoubleArray> array, TNode<IntPtrT> length) {
+ CSA_ASSERT(this, WordEqual(length, LoadAndUntagFixedArrayBaseLength(array)));
+
+ TNode<IntPtrT> byte_length = TimesDoubleSize(length);
+ CSA_ASSERT(this, UintPtrLessThan(length, byte_length));
+
+ static const int32_t fa_base_data_offset =
+ FixedDoubleArray::kHeaderSize - kHeapObjectTag;
+ TNode<IntPtrT> backing_store = IntPtrAdd(BitcastTaggedToWord(array),
+ IntPtrConstant(fa_base_data_offset));
+
+ // Call out to memset to perform initialization.
+ TNode<ExternalReference> memset =
+ ExternalConstant(ExternalReference::libc_memset_function());
+ STATIC_ASSERT(kSizetSize == kIntptrSize);
+ CallCFunction(memset, MachineType::Pointer(),
+ std::make_pair(MachineType::Pointer(), backing_store),
+ std::make_pair(MachineType::IntPtr(), IntPtrConstant(0)),
+ std::make_pair(MachineType::UintPtr(), byte_length));
+}
+
+void CodeStubAssembler::JumpIfPointersFromHereAreInteresting(
+ TNode<Object> object, Label* interesting) {
+ Label finished(this);
+ TNode<IntPtrT> object_word = BitcastTaggedToWord(object);
+ TNode<IntPtrT> object_page = PageFromAddress(object_word);
+ TNode<IntPtrT> page_flags = UncheckedCast<IntPtrT>(Load(
+ MachineType::IntPtr(), object_page, IntPtrConstant(Page::kFlagsOffset)));
+ Branch(
+ WordEqual(WordAnd(page_flags,
+ IntPtrConstant(
+ MemoryChunk::kPointersFromHereAreInterestingMask)),
+ IntPtrConstant(0)),
+ &finished, interesting);
+ BIND(&finished);
+}
+
+void CodeStubAssembler::MoveElements(ElementsKind kind,
+ TNode<FixedArrayBase> elements,
+ TNode<IntPtrT> dst_index,
+ TNode<IntPtrT> src_index,
+ TNode<IntPtrT> length) {
+ Label finished(this);
+ Label needs_barrier(this);
+ const bool needs_barrier_check = !IsDoubleElementsKind(kind);
+
+ DCHECK(IsFastElementsKind(kind));
+ CSA_ASSERT(this, IsFixedArrayWithKind(elements, kind));
+ CSA_ASSERT(this,
+ IntPtrLessThanOrEqual(IntPtrAdd(dst_index, length),
+ LoadAndUntagFixedArrayBaseLength(elements)));
+ CSA_ASSERT(this,
+ IntPtrLessThanOrEqual(IntPtrAdd(src_index, length),
+ LoadAndUntagFixedArrayBaseLength(elements)));
+
+ // The write barrier can be ignored if {dst_elements} is in new space, or if
+ // the elements pointer is FixedDoubleArray.
+ if (needs_barrier_check) {
+ JumpIfPointersFromHereAreInteresting(elements, &needs_barrier);
+ }
+
+ const TNode<IntPtrT> source_byte_length =
+ IntPtrMul(length, IntPtrConstant(ElementsKindToByteSize(kind)));
+ static const int32_t fa_base_data_offset =
+ FixedArrayBase::kHeaderSize - kHeapObjectTag;
+ TNode<IntPtrT> elements_intptr = BitcastTaggedToWord(elements);
+ TNode<IntPtrT> target_data_ptr =
+ IntPtrAdd(elements_intptr,
+ ElementOffsetFromIndex(dst_index, kind, fa_base_data_offset));
+ TNode<IntPtrT> source_data_ptr =
+ IntPtrAdd(elements_intptr,
+ ElementOffsetFromIndex(src_index, kind, fa_base_data_offset));
+ TNode<ExternalReference> memmove =
+ ExternalConstant(ExternalReference::libc_memmove_function());
+ CallCFunction(memmove, MachineType::Pointer(),
+ std::make_pair(MachineType::Pointer(), target_data_ptr),
+ std::make_pair(MachineType::Pointer(), source_data_ptr),
+ std::make_pair(MachineType::UintPtr(), source_byte_length));
+
+ if (needs_barrier_check) {
+ Goto(&finished);
+
+ BIND(&needs_barrier);
+ {
+ const TNode<IntPtrT> begin = src_index;
+ const TNode<IntPtrT> end = IntPtrAdd(begin, length);
+
+ // If dst_index is less than src_index, then walk forward.
+ const TNode<IntPtrT> delta =
+ IntPtrMul(IntPtrSub(dst_index, begin),
+ IntPtrConstant(ElementsKindToByteSize(kind)));
+ auto loop_body = [&](TNode<HeapObject> array, TNode<IntPtrT> offset) {
+ const TNode<AnyTaggedT> element = Load<AnyTaggedT>(array, offset);
+ const TNode<WordT> delta_offset = IntPtrAdd(offset, delta);
+ Store(array, delta_offset, element);
+ };
+
+ Label iterate_forward(this);
+ Label iterate_backward(this);
+ Branch(IntPtrLessThan(delta, IntPtrConstant(0)), &iterate_forward,
+ &iterate_backward);
+ BIND(&iterate_forward);
+ {
+ // Make a loop for the stores.
+ BuildFastArrayForEach(elements, kind, begin, end, loop_body,
+ ForEachDirection::kForward);
+ Goto(&finished);
+ }
+
+ BIND(&iterate_backward);
+ {
+ BuildFastArrayForEach(elements, kind, begin, end, loop_body,
+ ForEachDirection::kReverse);
+ Goto(&finished);
+ }
+ }
+ BIND(&finished);
+ }
+}
+
+void CodeStubAssembler::CopyElements(ElementsKind kind,
+ TNode<FixedArrayBase> dst_elements,
+ TNode<IntPtrT> dst_index,
+ TNode<FixedArrayBase> src_elements,
+ TNode<IntPtrT> src_index,
+ TNode<IntPtrT> length,
+ WriteBarrierMode write_barrier) {
+ Label finished(this);
+ Label needs_barrier(this);
+ const bool needs_barrier_check = !IsDoubleElementsKind(kind);
+
+ DCHECK(IsFastElementsKind(kind));
+ CSA_ASSERT(this, IsFixedArrayWithKind(dst_elements, kind));
+ CSA_ASSERT(this, IsFixedArrayWithKind(src_elements, kind));
+ CSA_ASSERT(this, IntPtrLessThanOrEqual(
+ IntPtrAdd(dst_index, length),
+ LoadAndUntagFixedArrayBaseLength(dst_elements)));
+ CSA_ASSERT(this, IntPtrLessThanOrEqual(
+ IntPtrAdd(src_index, length),
+ LoadAndUntagFixedArrayBaseLength(src_elements)));
+ CSA_ASSERT(this, Word32Or(TaggedNotEqual(dst_elements, src_elements),
+ IntPtrEqual(length, IntPtrConstant(0))));
+
+ // The write barrier can be ignored if {dst_elements} is in new space, or if
+ // the elements pointer is FixedDoubleArray.
+ if (needs_barrier_check) {
+ JumpIfPointersFromHereAreInteresting(dst_elements, &needs_barrier);
+ }
+
+ TNode<IntPtrT> source_byte_length =
+ IntPtrMul(length, IntPtrConstant(ElementsKindToByteSize(kind)));
+ static const int32_t fa_base_data_offset =
+ FixedArrayBase::kHeaderSize - kHeapObjectTag;
+ TNode<IntPtrT> src_offset_start =
+ ElementOffsetFromIndex(src_index, kind, fa_base_data_offset);
+ TNode<IntPtrT> dst_offset_start =
+ ElementOffsetFromIndex(dst_index, kind, fa_base_data_offset);
+ TNode<IntPtrT> src_elements_intptr = BitcastTaggedToWord(src_elements);
+ TNode<IntPtrT> source_data_ptr =
+ IntPtrAdd(src_elements_intptr, src_offset_start);
+ TNode<IntPtrT> dst_elements_intptr = BitcastTaggedToWord(dst_elements);
+ TNode<IntPtrT> dst_data_ptr =
+ IntPtrAdd(dst_elements_intptr, dst_offset_start);
+ TNode<ExternalReference> memcpy =
+ ExternalConstant(ExternalReference::libc_memcpy_function());
+ CallCFunction(memcpy, MachineType::Pointer(),
+ std::make_pair(MachineType::Pointer(), dst_data_ptr),
+ std::make_pair(MachineType::Pointer(), source_data_ptr),
+ std::make_pair(MachineType::UintPtr(), source_byte_length));
+
+ if (needs_barrier_check) {
+ Goto(&finished);
+
+ BIND(&needs_barrier);
+ {
+ const TNode<IntPtrT> begin = src_index;
+ const TNode<IntPtrT> end = IntPtrAdd(begin, length);
+ const TNode<IntPtrT> delta =
+ IntPtrMul(IntPtrSub(dst_index, src_index),
+ IntPtrConstant(ElementsKindToByteSize(kind)));
+ BuildFastArrayForEach(
+ src_elements, kind, begin, end,
+ [&](TNode<HeapObject> array, TNode<IntPtrT> offset) {
+ const TNode<AnyTaggedT> element = Load<AnyTaggedT>(array, offset);
+ const TNode<WordT> delta_offset = IntPtrAdd(offset, delta);
+ if (write_barrier == SKIP_WRITE_BARRIER) {
+ StoreNoWriteBarrier(MachineRepresentation::kTagged, dst_elements,
+ delta_offset, element);
+ } else {
+ Store(dst_elements, delta_offset, element);
+ }
+ },
+ ForEachDirection::kForward);
+ Goto(&finished);
+ }
+ BIND(&finished);
+ }
+}
+
+template <typename TIndex>
+void CodeStubAssembler::CopyFixedArrayElements(
+ ElementsKind from_kind, TNode<FixedArrayBase> from_array,
+ ElementsKind to_kind, TNode<FixedArrayBase> to_array,
+ TNode<TIndex> first_element, TNode<TIndex> element_count,
+ TNode<TIndex> capacity, WriteBarrierMode barrier_mode,
+ HoleConversionMode convert_holes, TVariable<BoolT>* var_holes_converted) {
+ DCHECK_IMPLIES(var_holes_converted != nullptr,
+ convert_holes == HoleConversionMode::kConvertToUndefined);
+ CSA_SLOW_ASSERT(this, IsFixedArrayWithKindOrEmpty(from_array, from_kind));
+ CSA_SLOW_ASSERT(this, IsFixedArrayWithKindOrEmpty(to_array, to_kind));
+ STATIC_ASSERT(FixedArray::kHeaderSize == FixedDoubleArray::kHeaderSize);
+ static_assert(
+ std::is_same<TIndex, Smi>::value || std::is_same<TIndex, IntPtrT>::value,
+ "Only Smi or IntPtrT indices are allowed");
+
+ const int first_element_offset = FixedArray::kHeaderSize - kHeapObjectTag;
+ Comment("[ CopyFixedArrayElements");
+
+ // Typed array elements are not supported.
+ DCHECK(!IsTypedArrayElementsKind(from_kind));
+ DCHECK(!IsTypedArrayElementsKind(to_kind));
+
+ Label done(this);
+ bool from_double_elements = IsDoubleElementsKind(from_kind);
+ bool to_double_elements = IsDoubleElementsKind(to_kind);
+ bool doubles_to_objects_conversion =
+ IsDoubleElementsKind(from_kind) && IsObjectElementsKind(to_kind);
+ bool needs_write_barrier =
+ doubles_to_objects_conversion ||
+ (barrier_mode == UPDATE_WRITE_BARRIER && IsObjectElementsKind(to_kind));
+ bool element_offset_matches =
+ !needs_write_barrier &&
+ (kTaggedSize == kDoubleSize ||
+ IsDoubleElementsKind(from_kind) == IsDoubleElementsKind(to_kind));
+ TNode<UintPtrT> double_hole =
+ Is64() ? ReinterpretCast<UintPtrT>(Int64Constant(kHoleNanInt64))
+ : ReinterpretCast<UintPtrT>(Int32Constant(kHoleNanLower32));
+
+ // If copying might trigger a GC, we pre-initialize the FixedArray such that
+ // it's always in a consistent state.
+ if (convert_holes == HoleConversionMode::kConvertToUndefined) {
+ DCHECK(IsObjectElementsKind(to_kind));
+ // Use undefined for the part that we copy and holes for the rest.
+ // Later if we run into a hole in the source we can just skip the writing
+ // to the target and are still guaranteed that we get an undefined.
+ FillFixedArrayWithValue(to_kind, to_array, IntPtrOrSmiConstant<TIndex>(0),
+ element_count, RootIndex::kUndefinedValue);
+ FillFixedArrayWithValue(to_kind, to_array, element_count, capacity,
+ RootIndex::kTheHoleValue);
+ } else if (doubles_to_objects_conversion) {
+ // Pre-initialized the target with holes so later if we run into a hole in
+ // the source we can just skip the writing to the target.
+ FillFixedArrayWithValue(to_kind, to_array, IntPtrOrSmiConstant<TIndex>(0),
+ capacity, RootIndex::kTheHoleValue);
+ } else if (element_count != capacity) {
+ FillFixedArrayWithValue(to_kind, to_array, element_count, capacity,
+ RootIndex::kTheHoleValue);
+ }
+
+ TNode<IntPtrT> first_from_element_offset =
+ ElementOffsetFromIndex(first_element, from_kind, 0);
+ TNode<IntPtrT> limit_offset = Signed(IntPtrAdd(
+ first_from_element_offset, IntPtrConstant(first_element_offset)));
+ TVARIABLE(IntPtrT, var_from_offset,
+ ElementOffsetFromIndex(IntPtrOrSmiAdd(first_element, element_count),
+ from_kind, first_element_offset));
+ // This second variable is used only when the element sizes of source and
+ // destination arrays do not match.
+ TVARIABLE(IntPtrT, var_to_offset);
+ if (element_offset_matches) {
+ var_to_offset = var_from_offset.value();
+ } else {
+ var_to_offset =
+ ElementOffsetFromIndex(element_count, to_kind, first_element_offset);
+ }
+
+ VariableList vars({&var_from_offset, &var_to_offset}, zone());
+ if (var_holes_converted != nullptr) vars.push_back(var_holes_converted);
+ Label decrement(this, vars);
+
+ TNode<IntPtrT> to_array_adjusted =
+ element_offset_matches
+ ? IntPtrSub(BitcastTaggedToWord(to_array), first_from_element_offset)
+ : ReinterpretCast<IntPtrT>(to_array);
+
+ Branch(WordEqual(var_from_offset.value(), limit_offset), &done, &decrement);
+
+ BIND(&decrement);
+ {
+ TNode<IntPtrT> from_offset = Signed(IntPtrSub(
+ var_from_offset.value(),
+ IntPtrConstant(from_double_elements ? kDoubleSize : kTaggedSize)));
+ var_from_offset = from_offset;
+
+ TNode<IntPtrT> to_offset;
+ if (element_offset_matches) {
+ to_offset = from_offset;
+ } else {
+ to_offset = IntPtrSub(
+ var_to_offset.value(),
+ IntPtrConstant(to_double_elements ? kDoubleSize : kTaggedSize));
+ var_to_offset = to_offset;
+ }
+
+ Label next_iter(this), store_double_hole(this), signal_hole(this);
+ Label* if_hole;
+ if (convert_holes == HoleConversionMode::kConvertToUndefined) {
+ // The target elements array is already preinitialized with undefined
+ // so we only need to signal that a hole was found and continue the loop.
+ if_hole = &signal_hole;
+ } else if (doubles_to_objects_conversion) {
+ // The target elements array is already preinitialized with holes, so we
+ // can just proceed with the next iteration.
+ if_hole = &next_iter;
+ } else if (IsDoubleElementsKind(to_kind)) {
+ if_hole = &store_double_hole;
+ } else {
+ // In all the other cases don't check for holes and copy the data as is.
+ if_hole = nullptr;
+ }
+
+ Node* value = LoadElementAndPrepareForStore(
+ from_array, var_from_offset.value(), from_kind, to_kind, if_hole);
+
+ if (needs_write_barrier) {
+ CHECK_EQ(to_array, to_array_adjusted);
+ Store(to_array_adjusted, to_offset, value);
+ } else if (to_double_elements) {
+ StoreNoWriteBarrier(MachineRepresentation::kFloat64, to_array_adjusted,
+ to_offset, value);
+ } else {
+ UnsafeStoreNoWriteBarrier(MachineRepresentation::kTagged,
+ to_array_adjusted, to_offset, value);
+ }
+ Goto(&next_iter);
+
+ if (if_hole == &store_double_hole) {
+ BIND(&store_double_hole);
+ // Don't use doubles to store the hole double, since manipulating the
+ // signaling NaN used for the hole in C++, e.g. with bit_cast, will
+ // change its value on ia32 (the x87 stack is used to return values
+ // and stores to the stack silently clear the signalling bit).
+ //
+ // TODO(danno): When we have a Float32/Float64 wrapper class that
+ // preserves double bits during manipulation, remove this code/change
+ // this to an indexed Float64 store.
+ if (Is64()) {
+ StoreNoWriteBarrier(MachineRepresentation::kWord64, to_array_adjusted,
+ to_offset, double_hole);
+ } else {
+ StoreNoWriteBarrier(MachineRepresentation::kWord32, to_array_adjusted,
+ to_offset, double_hole);
+ StoreNoWriteBarrier(MachineRepresentation::kWord32, to_array_adjusted,
+ IntPtrAdd(to_offset, IntPtrConstant(kInt32Size)),
+ double_hole);
+ }
+ Goto(&next_iter);
+ } else if (if_hole == &signal_hole) {
+ // This case happens only when IsObjectElementsKind(to_kind).
+ BIND(&signal_hole);
+ if (var_holes_converted != nullptr) {
+ *var_holes_converted = Int32TrueConstant();
+ }
+ Goto(&next_iter);
+ }
+
+ BIND(&next_iter);
+ TNode<BoolT> compare = WordNotEqual(from_offset, limit_offset);
+ Branch(compare, &decrement, &done);
+ }
+
+ BIND(&done);
+ Comment("] CopyFixedArrayElements");
+}
+
+TNode<FixedArray> CodeStubAssembler::HeapObjectToFixedArray(
+ TNode<HeapObject> base, Label* cast_fail) {
+ Label fixed_array(this);
+ TNode<Map> map = LoadMap(base);
+ GotoIf(TaggedEqual(map, FixedArrayMapConstant()), &fixed_array);
+ GotoIf(TaggedNotEqual(map, FixedCOWArrayMapConstant()), cast_fail);
+ Goto(&fixed_array);
+ BIND(&fixed_array);
+ return UncheckedCast<FixedArray>(base);
+}
+
+void CodeStubAssembler::CopyPropertyArrayValues(TNode<HeapObject> from_array,
+ TNode<PropertyArray> to_array,
+ TNode<IntPtrT> property_count,
+ WriteBarrierMode barrier_mode,
+ DestroySource destroy_source) {
+ CSA_SLOW_ASSERT(this, Word32Or(IsPropertyArray(from_array),
+ IsEmptyFixedArray(from_array)));
+ Comment("[ CopyPropertyArrayValues");
+
+ bool needs_write_barrier = barrier_mode == UPDATE_WRITE_BARRIER;
+
+ if (destroy_source == DestroySource::kNo) {
+ // PropertyArray may contain mutable HeapNumbers, which will be cloned on
+ // the heap, requiring a write barrier.
+ needs_write_barrier = true;
+ }
+
+ TNode<IntPtrT> start = IntPtrConstant(0);
+ ElementsKind kind = PACKED_ELEMENTS;
+ BuildFastArrayForEach(
+ from_array, kind, start, property_count,
+ [this, to_array, needs_write_barrier, destroy_source](
+ TNode<HeapObject> array, TNode<IntPtrT> offset) {
+ TNode<AnyTaggedT> value = Load<AnyTaggedT>(array, offset);
+
+ if (destroy_source == DestroySource::kNo) {
+ value = CloneIfMutablePrimitive(CAST(value));
+ }
+
+ if (needs_write_barrier) {
+ Store(to_array, offset, value);
+ } else {
+ StoreNoWriteBarrier(MachineRepresentation::kTagged, to_array, offset,
+ value);
+ }
+ });
+
+#ifdef DEBUG
+ // Zap {from_array} if the copying above has made it invalid.
+ if (destroy_source == DestroySource::kYes) {
+ Label did_zap(this);
+ GotoIf(IsEmptyFixedArray(from_array), &did_zap);
+ FillPropertyArrayWithUndefined(CAST(from_array), start, property_count);
+
+ Goto(&did_zap);
+ BIND(&did_zap);
+ }
+#endif
+ Comment("] CopyPropertyArrayValues");
+}
+
+TNode<FixedArrayBase> CodeStubAssembler::CloneFixedArray(
+ TNode<FixedArrayBase> source, ExtractFixedArrayFlags flags) {
+ return ExtractFixedArray(
+ source, base::Optional<TNode<BInt>>(IntPtrOrSmiConstant<BInt>(0)),
+ base::Optional<TNode<BInt>>(base::nullopt),
+ base::Optional<TNode<BInt>>(base::nullopt), flags);
+}
+
+Node* CodeStubAssembler::LoadElementAndPrepareForStore(
+ TNode<FixedArrayBase> array, TNode<IntPtrT> offset, ElementsKind from_kind,
+ ElementsKind to_kind, Label* if_hole) {
+ CSA_ASSERT(this, IsFixedArrayWithKind(array, from_kind));
+ if (IsDoubleElementsKind(from_kind)) {
+ TNode<Float64T> value =
+ LoadDoubleWithHoleCheck(array, offset, if_hole, MachineType::Float64());
+ if (!IsDoubleElementsKind(to_kind)) {
+ return AllocateHeapNumberWithValue(value);
+ }
+ return value;
+
+ } else {
+ TNode<Object> value = Load<Object>(array, offset);
+ if (if_hole) {
+ GotoIf(TaggedEqual(value, TheHoleConstant()), if_hole);
+ }
+ if (IsDoubleElementsKind(to_kind)) {
+ if (IsSmiElementsKind(from_kind)) {
+ return SmiToFloat64(CAST(value));
+ }
+ return LoadHeapNumberValue(CAST(value));
+ }
+ return value;
+ }
+}
+
+template <typename TIndex>
+TNode<TIndex> CodeStubAssembler::CalculateNewElementsCapacity(
+ TNode<TIndex> old_capacity) {
+ static_assert(
+ std::is_same<TIndex, Smi>::value || std::is_same<TIndex, IntPtrT>::value,
+ "Only Smi or IntPtrT old_capacity is allowed");
+ Comment("TryGrowElementsCapacity");
+ TNode<TIndex> half_old_capacity = WordOrSmiShr(old_capacity, 1);
+ TNode<TIndex> new_capacity = IntPtrOrSmiAdd(half_old_capacity, old_capacity);
+ TNode<TIndex> padding =
+ IntPtrOrSmiConstant<TIndex>(JSObject::kMinAddedElementsCapacity);
+ return IntPtrOrSmiAdd(new_capacity, padding);
+}
+
+template V8_EXPORT_PRIVATE TNode<IntPtrT>
+ CodeStubAssembler::CalculateNewElementsCapacity<IntPtrT>(TNode<IntPtrT>);
+template V8_EXPORT_PRIVATE TNode<Smi>
+ CodeStubAssembler::CalculateNewElementsCapacity<Smi>(TNode<Smi>);
+
+TNode<FixedArrayBase> CodeStubAssembler::TryGrowElementsCapacity(
+ TNode<HeapObject> object, TNode<FixedArrayBase> elements, ElementsKind kind,
+ TNode<Smi> key, Label* bailout) {
+ CSA_SLOW_ASSERT(this, IsFixedArrayWithKindOrEmpty(elements, kind));
+ TNode<Smi> capacity = LoadFixedArrayBaseLength(elements);
+
+ return TryGrowElementsCapacity(object, elements, kind,
+ TaggedToParameter<BInt>(key),
+ TaggedToParameter<BInt>(capacity), bailout);
+}
+
+template <typename TIndex>
+TNode<FixedArrayBase> CodeStubAssembler::TryGrowElementsCapacity(
+ TNode<HeapObject> object, TNode<FixedArrayBase> elements, ElementsKind kind,
+ TNode<TIndex> key, TNode<TIndex> capacity, Label* bailout) {
+ static_assert(
+ std::is_same<TIndex, Smi>::value || std::is_same<TIndex, IntPtrT>::value,
+ "Only Smi or IntPtrT key and capacity nodes are allowed");
+ Comment("TryGrowElementsCapacity");
+ CSA_SLOW_ASSERT(this, IsFixedArrayWithKindOrEmpty(elements, kind));
+
+ // If the gap growth is too big, fall back to the runtime.
+ TNode<TIndex> max_gap = IntPtrOrSmiConstant<TIndex>(JSObject::kMaxGap);
+ TNode<TIndex> max_capacity = IntPtrOrSmiAdd(capacity, max_gap);
+ GotoIf(UintPtrOrSmiGreaterThanOrEqual(key, max_capacity), bailout);
+
+ // Calculate the capacity of the new backing store.
+ TNode<TIndex> new_capacity = CalculateNewElementsCapacity(
+ IntPtrOrSmiAdd(key, IntPtrOrSmiConstant<TIndex>(1)));
+
+ return GrowElementsCapacity(object, elements, kind, kind, capacity,
+ new_capacity, bailout);
+}
+
+template <typename TIndex>
+TNode<FixedArrayBase> CodeStubAssembler::GrowElementsCapacity(
+ TNode<HeapObject> object, TNode<FixedArrayBase> elements,
+ ElementsKind from_kind, ElementsKind to_kind, TNode<TIndex> capacity,
+ TNode<TIndex> new_capacity, Label* bailout) {
+ static_assert(
+ std::is_same<TIndex, Smi>::value || std::is_same<TIndex, IntPtrT>::value,
+ "Only Smi or IntPtrT capacities are allowed");
+ Comment("[ GrowElementsCapacity");
+ CSA_SLOW_ASSERT(this, IsFixedArrayWithKindOrEmpty(elements, from_kind));
+
+ // If size of the allocation for the new capacity doesn't fit in a page
+ // that we can bump-pointer allocate from, fall back to the runtime.
+ int max_size = FixedArrayBase::GetMaxLengthForNewSpaceAllocation(to_kind);
+ GotoIf(UintPtrOrSmiGreaterThanOrEqual(new_capacity,
+ IntPtrOrSmiConstant<TIndex>(max_size)),
+ bailout);
+
+ // Allocate the new backing store.
+ TNode<FixedArrayBase> new_elements =
+ AllocateFixedArray(to_kind, new_capacity);
+
+ // Copy the elements from the old elements store to the new.
+ // The size-check above guarantees that the |new_elements| is allocated
+ // in new space so we can skip the write barrier.
+ CopyFixedArrayElements(from_kind, elements, to_kind, new_elements, capacity,
+ new_capacity, SKIP_WRITE_BARRIER);
+
+ StoreObjectField(object, JSObject::kElementsOffset, new_elements);
+ Comment("] GrowElementsCapacity");
+ return new_elements;
+}
+
+void CodeStubAssembler::InitializeAllocationMemento(
+ TNode<HeapObject> base, TNode<IntPtrT> base_allocation_size,
+ TNode<AllocationSite> allocation_site) {
+ Comment("[Initialize AllocationMemento");
+ TNode<HeapObject> memento = InnerAllocate(base, base_allocation_size);
+ StoreMapNoWriteBarrier(memento, RootIndex::kAllocationMementoMap);
+ StoreObjectFieldNoWriteBarrier(
+ memento, AllocationMemento::kAllocationSiteOffset, allocation_site);
+ if (FLAG_allocation_site_pretenuring) {
+ TNode<Int32T> count = LoadObjectField<Int32T>(
+ allocation_site, AllocationSite::kPretenureCreateCountOffset);
+
+ TNode<Int32T> incremented_count = Int32Add(count, Int32Constant(1));
+ StoreObjectFieldNoWriteBarrier(allocation_site,
+ AllocationSite::kPretenureCreateCountOffset,
+ incremented_count);
+ }
+ Comment("]");
+}
+
+TNode<Float64T> CodeStubAssembler::TryTaggedToFloat64(
+ TNode<Object> value, Label* if_valueisnotnumber) {
+ return Select<Float64T>(
+ TaggedIsSmi(value), [&]() { return SmiToFloat64(CAST(value)); },
+ [&]() {
+ GotoIfNot(IsHeapNumber(CAST(value)), if_valueisnotnumber);
+ return LoadHeapNumberValue(CAST(value));
+ });
+}
+
+TNode<Float64T> CodeStubAssembler::TruncateTaggedToFloat64(
+ TNode<Context> context, SloppyTNode<Object> value) {
+ // We might need to loop once due to ToNumber conversion.
+ TVARIABLE(Object, var_value, value);
+ TVARIABLE(Float64T, var_result);
+ Label loop(this, &var_value), done_loop(this, &var_result);
+ Goto(&loop);
+ BIND(&loop);
+ {
+ Label if_valueisnotnumber(this, Label::kDeferred);
+
+ // Load the current {value}.
+ value = var_value.value();
+
+ // Convert {value} to Float64 if it is a number and convert it to a number
+ // otherwise.
+ var_result = TryTaggedToFloat64(value, &if_valueisnotnumber);
+ Goto(&done_loop);
+
+ BIND(&if_valueisnotnumber);
+ {
+ // Convert the {value} to a Number first.
+ var_value = CallBuiltin(Builtins::kNonNumberToNumber, context, value);
+ Goto(&loop);
+ }
+ }
+ BIND(&done_loop);
+ return var_result.value();
+}
+
+TNode<Word32T> CodeStubAssembler::TruncateTaggedToWord32(
+ TNode<Context> context, SloppyTNode<Object> value) {
+ TVARIABLE(Word32T, var_result);
+ Label done(this);
+ TaggedToWord32OrBigIntImpl<Object::Conversion::kToNumber>(context, value,
+ &done, &var_result);
+ BIND(&done);
+ return var_result.value();
+}
+
+// Truncate {value} to word32 and jump to {if_number} if it is a Number,
+// or find that it is a BigInt and jump to {if_bigint}.
+void CodeStubAssembler::TaggedToWord32OrBigInt(
+ TNode<Context> context, TNode<Object> value, Label* if_number,
+ TVariable<Word32T>* var_word32, Label* if_bigint,
+ TVariable<BigInt>* var_maybe_bigint) {
+ TaggedToWord32OrBigIntImpl<Object::Conversion::kToNumeric>(
+ context, value, if_number, var_word32, if_bigint, var_maybe_bigint);
+}
+
+// Truncate {value} to word32 and jump to {if_number} if it is a Number,
+// or find that it is a BigInt and jump to {if_bigint}. In either case,
+// store the type feedback in {var_feedback}.
+void CodeStubAssembler::TaggedToWord32OrBigIntWithFeedback(
+ TNode<Context> context, TNode<Object> value, Label* if_number,
+ TVariable<Word32T>* var_word32, Label* if_bigint,
+ TVariable<BigInt>* var_maybe_bigint, TVariable<Smi>* var_feedback) {
+ TaggedToWord32OrBigIntImpl<Object::Conversion::kToNumeric>(
+ context, value, if_number, var_word32, if_bigint, var_maybe_bigint,
+ var_feedback);
+}
+
+template <Object::Conversion conversion>
+void CodeStubAssembler::TaggedToWord32OrBigIntImpl(
+ TNode<Context> context, TNode<Object> value, Label* if_number,
+ TVariable<Word32T>* var_word32, Label* if_bigint,
+ TVariable<BigInt>* var_maybe_bigint, TVariable<Smi>* var_feedback) {
+ // We might need to loop after conversion.
+ TVARIABLE(Object, var_value, value);
+ OverwriteFeedback(var_feedback, BinaryOperationFeedback::kNone);
+ VariableList loop_vars({&var_value}, zone());
+ if (var_feedback != nullptr) loop_vars.push_back(var_feedback);
+ Label loop(this, loop_vars);
+ Goto(&loop);
+ BIND(&loop);
+ {
+ value = var_value.value();
+ Label not_smi(this), is_heap_number(this), is_oddball(this),
+ is_bigint(this);
+ GotoIf(TaggedIsNotSmi(value), ¬_smi);
+
+ // {value} is a Smi.
+ *var_word32 = SmiToInt32(CAST(value));
+ CombineFeedback(var_feedback, BinaryOperationFeedback::kSignedSmall);
+ Goto(if_number);
+
+ BIND(¬_smi);
+ TNode<HeapObject> value_heap_object = CAST(value);
+ TNode<Map> map = LoadMap(value_heap_object);
+ GotoIf(IsHeapNumberMap(map), &is_heap_number);
+ TNode<Uint16T> instance_type = LoadMapInstanceType(map);
+ if (conversion == Object::Conversion::kToNumeric) {
+ GotoIf(IsBigIntInstanceType(instance_type), &is_bigint);
+ }
+
+ // Not HeapNumber (or BigInt if conversion == kToNumeric).
+ {
+ if (var_feedback != nullptr) {
+ // We do not require an Or with earlier feedback here because once we
+ // convert the value to a Numeric, we cannot reach this path. We can
+ // only reach this path on the first pass when the feedback is kNone.
+ CSA_ASSERT(this, SmiEqual(var_feedback->value(),
+ SmiConstant(BinaryOperationFeedback::kNone)));
+ }
+ GotoIf(InstanceTypeEqual(instance_type, ODDBALL_TYPE), &is_oddball);
+ // Not an oddball either -> convert.
+ auto builtin = conversion == Object::Conversion::kToNumeric
+ ? Builtins::kNonNumberToNumeric
+ : Builtins::kNonNumberToNumber;
+ var_value = CallBuiltin(builtin, context, value);
+ OverwriteFeedback(var_feedback, BinaryOperationFeedback::kAny);
+ Goto(&loop);
+
+ BIND(&is_oddball);
+ var_value = LoadObjectField(value_heap_object, Oddball::kToNumberOffset);
+ OverwriteFeedback(var_feedback,
+ BinaryOperationFeedback::kNumberOrOddball);
+ Goto(&loop);
+ }
+
+ BIND(&is_heap_number);
+ *var_word32 = TruncateHeapNumberValueToWord32(CAST(value));
+ CombineFeedback(var_feedback, BinaryOperationFeedback::kNumber);
+ Goto(if_number);
+
+ if (conversion == Object::Conversion::kToNumeric) {
+ BIND(&is_bigint);
+ *var_maybe_bigint = CAST(value);
+ CombineFeedback(var_feedback, BinaryOperationFeedback::kBigInt);
+ Goto(if_bigint);
+ }
+ }
+}
+
+TNode<Int32T> CodeStubAssembler::TruncateNumberToWord32(TNode<Number> number) {
+ TVARIABLE(Int32T, var_result);
+ Label done(this), if_heapnumber(this);
+ GotoIfNot(TaggedIsSmi(number), &if_heapnumber);
+ var_result = SmiToInt32(CAST(number));
+ Goto(&done);
+
+ BIND(&if_heapnumber);
+ TNode<Float64T> value = LoadHeapNumberValue(CAST(number));
+ var_result = Signed(TruncateFloat64ToWord32(value));
+ Goto(&done);
+
+ BIND(&done);
+ return var_result.value();
+}
+
+TNode<Int32T> CodeStubAssembler::TruncateHeapNumberValueToWord32(
+ TNode<HeapNumber> object) {
+ TNode<Float64T> value = LoadHeapNumberValue(object);
+ return Signed(TruncateFloat64ToWord32(value));
+}
+
+void CodeStubAssembler::TryHeapNumberToSmi(TNode<HeapNumber> number,
+ TVariable<Smi>* var_result_smi,
+ Label* if_smi) {
+ TNode<Float64T> value = LoadHeapNumberValue(number);
+ TryFloat64ToSmi(value, var_result_smi, if_smi);
+}
+
+void CodeStubAssembler::TryFloat32ToSmi(TNode<Float32T> value,
+ TVariable<Smi>* var_result_smi,
+ Label* if_smi) {
+ TNode<Int32T> ivalue = TruncateFloat32ToInt32(value);
+ TNode<Float32T> fvalue = RoundInt32ToFloat32(ivalue);
+
+ Label if_int32(this), if_heap_number(this);
+
+ GotoIfNot(Float32Equal(value, fvalue), &if_heap_number);
+ GotoIfNot(Word32Equal(ivalue, Int32Constant(0)), &if_int32);
+ Branch(Int32LessThan(UncheckedCast<Int32T>(BitcastFloat32ToInt32(value)),
+ Int32Constant(0)),
+ &if_heap_number, &if_int32);
+
+ TVARIABLE(Number, var_result);
+ BIND(&if_int32);
+ {
+ if (SmiValuesAre32Bits()) {
+ *var_result_smi = SmiTag(ChangeInt32ToIntPtr(ivalue));
+ } else {
+ DCHECK(SmiValuesAre31Bits());
+ TNode<PairT<Int32T, BoolT>> pair = Int32AddWithOverflow(ivalue, ivalue);
+ TNode<BoolT> overflow = Projection<1>(pair);
+ GotoIf(overflow, &if_heap_number);
+ *var_result_smi =
+ BitcastWordToTaggedSigned(ChangeInt32ToIntPtr(Projection<0>(pair)));
+ }
+ Goto(if_smi);
+ }
+ BIND(&if_heap_number);
+}
+
+void CodeStubAssembler::TryFloat64ToSmi(TNode<Float64T> value,
+ TVariable<Smi>* var_result_smi,
+ Label* if_smi) {
+ TNode<Int32T> value32 = RoundFloat64ToInt32(value);
+ TNode<Float64T> value64 = ChangeInt32ToFloat64(value32);
+
+ Label if_int32(this), if_heap_number(this, Label::kDeferred);
+
+ GotoIfNot(Float64Equal(value, value64), &if_heap_number);
+ GotoIfNot(Word32Equal(value32, Int32Constant(0)), &if_int32);
+ Branch(Int32LessThan(UncheckedCast<Int32T>(Float64ExtractHighWord32(value)),
+ Int32Constant(0)),
+ &if_heap_number, &if_int32);
+
+ TVARIABLE(Number, var_result);
+ BIND(&if_int32);
+ {
+ if (SmiValuesAre32Bits()) {
+ *var_result_smi = SmiTag(ChangeInt32ToIntPtr(value32));
+ } else {
+ DCHECK(SmiValuesAre31Bits());
+ TNode<PairT<Int32T, BoolT>> pair = Int32AddWithOverflow(value32, value32);
+ TNode<BoolT> overflow = Projection<1>(pair);
+ GotoIf(overflow, &if_heap_number);
+ *var_result_smi =
+ BitcastWordToTaggedSigned(ChangeInt32ToIntPtr(Projection<0>(pair)));
+ }
+ Goto(if_smi);
+ }
+ BIND(&if_heap_number);
+}
+
+TNode<Number> CodeStubAssembler::ChangeFloat32ToTagged(TNode<Float32T> value) {
+ Label if_smi(this), done(this);
+ TVARIABLE(Smi, var_smi_result);
+ TVARIABLE(Number, var_result);
+ TryFloat32ToSmi(value, &var_smi_result, &if_smi);
+
+ var_result = AllocateHeapNumberWithValue(ChangeFloat32ToFloat64(value));
+ Goto(&done);
+
+ BIND(&if_smi);
+ {
+ var_result = var_smi_result.value();
+ Goto(&done);
+ }
+ BIND(&done);
+ return var_result.value();
+}
+
+TNode<Number> CodeStubAssembler::ChangeFloat64ToTagged(
+ SloppyTNode<Float64T> value) {
+ Label if_smi(this), done(this);
+ TVARIABLE(Smi, var_smi_result);
+ TVARIABLE(Number, var_result);
+ TryFloat64ToSmi(value, &var_smi_result, &if_smi);
+
+ var_result = AllocateHeapNumberWithValue(value);
+ Goto(&done);
+
+ BIND(&if_smi);
+ {
+ var_result = var_smi_result.value();
+ Goto(&done);
+ }
+ BIND(&done);
+ return var_result.value();
+}
+
+TNode<Number> CodeStubAssembler::ChangeInt32ToTagged(
+ SloppyTNode<Int32T> value) {
+ if (SmiValuesAre32Bits()) {
+ return SmiTag(ChangeInt32ToIntPtr(value));
+ }
+ DCHECK(SmiValuesAre31Bits());
+ TVARIABLE(Number, var_result);
+ TNode<PairT<Int32T, BoolT>> pair = Int32AddWithOverflow(value, value);
+ TNode<BoolT> overflow = Projection<1>(pair);
+ Label if_overflow(this, Label::kDeferred), if_notoverflow(this),
+ if_join(this);
+ Branch(overflow, &if_overflow, &if_notoverflow);
+ BIND(&if_overflow);
+ {
+ TNode<Float64T> value64 = ChangeInt32ToFloat64(value);
+ TNode<HeapNumber> result = AllocateHeapNumberWithValue(value64);
+ var_result = result;
+ Goto(&if_join);
+ }
+ BIND(&if_notoverflow);
+ {
+ TNode<IntPtrT> almost_tagged_value =
+ ChangeInt32ToIntPtr(Projection<0>(pair));
+ TNode<Smi> result = BitcastWordToTaggedSigned(almost_tagged_value);
+ var_result = result;
+ Goto(&if_join);
+ }
+ BIND(&if_join);
+ return var_result.value();
+}
+
+TNode<Number> CodeStubAssembler::ChangeUint32ToTagged(
+ SloppyTNode<Uint32T> value) {
+ Label if_overflow(this, Label::kDeferred), if_not_overflow(this),
+ if_join(this);
+ TVARIABLE(Number, var_result);
+ // If {value} > 2^31 - 1, we need to store it in a HeapNumber.
+ Branch(Uint32LessThan(Uint32Constant(Smi::kMaxValue), value), &if_overflow,
+ &if_not_overflow);
+
+ BIND(&if_not_overflow);
+ {
+ // The {value} is definitely in valid Smi range.
+ var_result = SmiTag(Signed(ChangeUint32ToWord(value)));
+ }
+ Goto(&if_join);
+
+ BIND(&if_overflow);
+ {
+ TNode<Float64T> float64_value = ChangeUint32ToFloat64(value);
+ var_result = AllocateHeapNumberWithValue(float64_value);
+ }
+ Goto(&if_join);
+
+ BIND(&if_join);
+ return var_result.value();
+}
+
+TNode<Number> CodeStubAssembler::ChangeUintPtrToTagged(TNode<UintPtrT> value) {
+ Label if_overflow(this, Label::kDeferred), if_not_overflow(this),
+ if_join(this);
+ TVARIABLE(Number, var_result);
+ // If {value} > 2^31 - 1, we need to store it in a HeapNumber.
+ Branch(UintPtrLessThan(UintPtrConstant(Smi::kMaxValue), value), &if_overflow,
+ &if_not_overflow);
+
+ BIND(&if_not_overflow);
+ {
+ // The {value} is definitely in valid Smi range.
+ var_result = SmiTag(Signed(value));
+ }
+ Goto(&if_join);
+
+ BIND(&if_overflow);
+ {
+ TNode<Float64T> float64_value = ChangeUintPtrToFloat64(value);
+ var_result = AllocateHeapNumberWithValue(float64_value);
+ }
+ Goto(&if_join);
+
+ BIND(&if_join);
+ return var_result.value();
+}
+
+TNode<String> CodeStubAssembler::ToThisString(TNode<Context> context,
+ TNode<Object> value,
+ TNode<String> method_name) {
+ TVARIABLE(Object, var_value, value);
+
+ // Check if the {value} is a Smi or a HeapObject.
+ Label if_valueissmi(this, Label::kDeferred), if_valueisnotsmi(this),
+ if_valueisstring(this);
+ Branch(TaggedIsSmi(value), &if_valueissmi, &if_valueisnotsmi);
+ BIND(&if_valueisnotsmi);
+ {
+ // Load the instance type of the {value}.
+ TNode<Uint16T> value_instance_type = LoadInstanceType(CAST(value));
+
+ // Check if the {value} is already String.
+ Label if_valueisnotstring(this, Label::kDeferred);
+ Branch(IsStringInstanceType(value_instance_type), &if_valueisstring,
+ &if_valueisnotstring);
+ BIND(&if_valueisnotstring);
+ {
+ // Check if the {value} is null.
+ Label if_valueisnullorundefined(this, Label::kDeferred);
+ GotoIf(IsNullOrUndefined(value), &if_valueisnullorundefined);
+ // Convert the {value} to a String.
+ var_value = CallBuiltin(Builtins::kToString, context, value);
+ Goto(&if_valueisstring);
+
+ BIND(&if_valueisnullorundefined);
+ {
+ // The {value} is either null or undefined.
+ ThrowTypeError(context, MessageTemplate::kCalledOnNullOrUndefined,
+ method_name);
+ }
+ }
+ }
+ BIND(&if_valueissmi);
+ {
+ // The {value} is a Smi, convert it to a String.
+ var_value = CallBuiltin(Builtins::kNumberToString, context, value);
+ Goto(&if_valueisstring);
+ }
+ BIND(&if_valueisstring);
+ return CAST(var_value.value());
+}
+
+TNode<Uint32T> CodeStubAssembler::ChangeNumberToUint32(TNode<Number> value) {
+ TVARIABLE(Uint32T, var_result);
+ Label if_smi(this), if_heapnumber(this, Label::kDeferred), done(this);
+ Branch(TaggedIsSmi(value), &if_smi, &if_heapnumber);
+ BIND(&if_smi);
+ {
+ var_result = Unsigned(SmiToInt32(CAST(value)));
+ Goto(&done);
+ }
+ BIND(&if_heapnumber);
+ {
+ var_result = ChangeFloat64ToUint32(LoadHeapNumberValue(CAST(value)));
+ Goto(&done);
+ }
+ BIND(&done);
+ return var_result.value();
+}
+
+TNode<Float64T> CodeStubAssembler::ChangeNumberToFloat64(TNode<Number> value) {
+ TVARIABLE(Float64T, result);
+ Label smi(this);
+ Label done(this, &result);
+ GotoIf(TaggedIsSmi(value), &smi);
+ result = LoadHeapNumberValue(CAST(value));
+ Goto(&done);
+
+ BIND(&smi);
+ {
+ result = SmiToFloat64(CAST(value));
+ Goto(&done);
+ }
+
+ BIND(&done);
+ return result.value();
+}
+
+TNode<Int32T> CodeStubAssembler::ChangeTaggedNonSmiToInt32(
+ TNode<Context> context, TNode<HeapObject> input) {
+ return Select<Int32T>(
+ IsHeapNumber(input),
+ [=] {
+ return Signed(TruncateFloat64ToWord32(LoadHeapNumberValue(input)));
+ },
+ [=] {
+ return TruncateNumberToWord32(
+ CAST(CallBuiltin(Builtins::kNonNumberToNumber, context, input)));
+ });
+}
+
+TNode<Float64T> CodeStubAssembler::ChangeTaggedToFloat64(TNode<Context> context,
+ TNode<Object> input) {
+ TVARIABLE(Float64T, var_result);
+ Label end(this), not_smi(this);
+
+ GotoIfNot(TaggedIsSmi(input), ¬_smi);
+ var_result = SmiToFloat64(CAST(input));
+ Goto(&end);
+
+ BIND(¬_smi);
+ var_result = Select<Float64T>(
+ IsHeapNumber(CAST(input)),
+ [=] { return LoadHeapNumberValue(CAST(input)); },
+ [=] {
+ return ChangeNumberToFloat64(
+ CAST(CallBuiltin(Builtins::kNonNumberToNumber, context, input)));
+ });
+ Goto(&end);
+
+ BIND(&end);
+ return var_result.value();
+}
+
+TNode<WordT> CodeStubAssembler::TimesSystemPointerSize(
+ SloppyTNode<WordT> value) {
+ return WordShl(value, kSystemPointerSizeLog2);
+}
+
+TNode<WordT> CodeStubAssembler::TimesTaggedSize(SloppyTNode<WordT> value) {
+ return WordShl(value, kTaggedSizeLog2);
+}
+
+TNode<WordT> CodeStubAssembler::TimesDoubleSize(SloppyTNode<WordT> value) {
+ return WordShl(value, kDoubleSizeLog2);
+}
+
+TNode<Object> CodeStubAssembler::ToThisValue(TNode<Context> context,
+ TNode<Object> value,
+ PrimitiveType primitive_type,
+ char const* method_name) {
+ // We might need to loop once due to JSPrimitiveWrapper unboxing.
+ TVARIABLE(Object, var_value, value);
+ Label loop(this, &var_value), done_loop(this),
+ done_throw(this, Label::kDeferred);
+ Goto(&loop);
+ BIND(&loop);
+ {
+ // Check if the {value} is a Smi or a HeapObject.
+ GotoIf(
+ TaggedIsSmi(var_value.value()),
+ (primitive_type == PrimitiveType::kNumber) ? &done_loop : &done_throw);
+
+ TNode<HeapObject> value = CAST(var_value.value());
+
+ // Load the map of the {value}.
+ TNode<Map> value_map = LoadMap(value);
+
+ // Load the instance type of the {value}.
+ TNode<Uint16T> value_instance_type = LoadMapInstanceType(value_map);
+
+ // Check if {value} is a JSPrimitiveWrapper.
+ Label if_valueiswrapper(this, Label::kDeferred), if_valueisnotwrapper(this);
+ Branch(InstanceTypeEqual(value_instance_type, JS_PRIMITIVE_WRAPPER_TYPE),
+ &if_valueiswrapper, &if_valueisnotwrapper);
+
+ BIND(&if_valueiswrapper);
+ {
+ // Load the actual value from the {value}.
+ var_value = LoadObjectField(value, JSPrimitiveWrapper::kValueOffset);
+ Goto(&loop);
+ }
+
+ BIND(&if_valueisnotwrapper);
+ {
+ switch (primitive_type) {
+ case PrimitiveType::kBoolean:
+ GotoIf(TaggedEqual(value_map, BooleanMapConstant()), &done_loop);
+ break;
+ case PrimitiveType::kNumber:
+ GotoIf(TaggedEqual(value_map, HeapNumberMapConstant()), &done_loop);
+ break;
+ case PrimitiveType::kString:
+ GotoIf(IsStringInstanceType(value_instance_type), &done_loop);
+ break;
+ case PrimitiveType::kSymbol:
+ GotoIf(TaggedEqual(value_map, SymbolMapConstant()), &done_loop);
+ break;
+ }
+ Goto(&done_throw);
+ }
+ }
+
+ BIND(&done_throw);
+ {
+ const char* primitive_name = nullptr;
+ switch (primitive_type) {
+ case PrimitiveType::kBoolean:
+ primitive_name = "Boolean";
+ break;
+ case PrimitiveType::kNumber:
+ primitive_name = "Number";
+ break;
+ case PrimitiveType::kString:
+ primitive_name = "String";
+ break;
+ case PrimitiveType::kSymbol:
+ primitive_name = "Symbol";
+ break;
+ }
+ CHECK_NOT_NULL(primitive_name);
+
+ // The {value} is not a compatible receiver for this method.
+ ThrowTypeError(context, MessageTemplate::kNotGeneric, method_name,
+ primitive_name);
+ }
+
+ BIND(&done_loop);
+ return var_value.value();
+}
+
+void CodeStubAssembler::ThrowIfNotInstanceType(TNode<Context> context,
+ TNode<Object> value,
+ InstanceType instance_type,
+ char const* method_name) {
+ Label out(this), throw_exception(this, Label::kDeferred);
+
+ GotoIf(TaggedIsSmi(value), &throw_exception);
+
+ // Load the instance type of the {value}.
+ TNode<Map> map = LoadMap(CAST(value));
+ const TNode<Uint16T> value_instance_type = LoadMapInstanceType(map);
+
+ Branch(Word32Equal(value_instance_type, Int32Constant(instance_type)), &out,
+ &throw_exception);
+
+ // The {value} is not a compatible receiver for this method.
+ BIND(&throw_exception);
+ ThrowTypeError(context, MessageTemplate::kIncompatibleMethodReceiver,
+ StringConstant(method_name), value);
+
+ BIND(&out);
+}
+
+void CodeStubAssembler::ThrowIfNotJSReceiver(TNode<Context> context,
+ TNode<Object> value,
+ MessageTemplate msg_template,
+ const char* method_name) {
+ Label done(this), throw_exception(this, Label::kDeferred);
+
+ GotoIf(TaggedIsSmi(value), &throw_exception);
+
+ // Load the instance type of the {value}.
+ TNode<Map> value_map = LoadMap(CAST(value));
+ const TNode<Uint16T> value_instance_type = LoadMapInstanceType(value_map);
+
+ Branch(IsJSReceiverInstanceType(value_instance_type), &done,
+ &throw_exception);
+
+ // The {value} is not a compatible receiver for this method.
+ BIND(&throw_exception);
+ ThrowTypeError(context, msg_template, StringConstant(method_name), value);
+
+ BIND(&done);
+}
+
+void CodeStubAssembler::ThrowIfNotCallable(TNode<Context> context,
+ TNode<Object> value,
+ const char* method_name) {
+ Label out(this), throw_exception(this, Label::kDeferred);
+
+ GotoIf(TaggedIsSmi(value), &throw_exception);
+ Branch(IsCallable(CAST(value)), &out, &throw_exception);
+
+ // The {value} is not a compatible receiver for this method.
+ BIND(&throw_exception);
+ ThrowTypeError(context, MessageTemplate::kCalledNonCallable, method_name);
+
+ BIND(&out);
+}
+
+void CodeStubAssembler::ThrowRangeError(TNode<Context> context,
+ MessageTemplate message,
+ base::Optional<TNode<Object>> arg0,
+ base::Optional<TNode<Object>> arg1,
+ base::Optional<TNode<Object>> arg2) {
+ TNode<Smi> template_index = SmiConstant(static_cast<int>(message));
+ if (!arg0) {
+ CallRuntime(Runtime::kThrowRangeError, context, template_index);
+ } else if (!arg1) {
+ CallRuntime(Runtime::kThrowRangeError, context, template_index, *arg0);
+ } else if (!arg2) {
+ CallRuntime(Runtime::kThrowRangeError, context, template_index, *arg0,
+ *arg1);
+ } else {
+ CallRuntime(Runtime::kThrowRangeError, context, template_index, *arg0,
+ *arg1, *arg2);
+ }
+ Unreachable();
+}
+
+void CodeStubAssembler::ThrowTypeError(TNode<Context> context,
+ MessageTemplate message,
+ char const* arg0, char const* arg1) {
+ base::Optional<TNode<Object>> arg0_node;
+ if (arg0) arg0_node = StringConstant(arg0);
+ base::Optional<TNode<Object>> arg1_node;
+ if (arg1) arg1_node = StringConstant(arg1);
+ ThrowTypeError(context, message, arg0_node, arg1_node);
+}
+
+void CodeStubAssembler::ThrowTypeError(TNode<Context> context,
+ MessageTemplate message,
+ base::Optional<TNode<Object>> arg0,
+ base::Optional<TNode<Object>> arg1,
+ base::Optional<TNode<Object>> arg2) {
+ TNode<Smi> template_index = SmiConstant(static_cast<int>(message));
+ if (!arg0) {
+ CallRuntime(Runtime::kThrowTypeError, context, template_index);
+ } else if (!arg1) {
+ CallRuntime(Runtime::kThrowTypeError, context, template_index, *arg0);
+ } else if (!arg2) {
+ CallRuntime(Runtime::kThrowTypeError, context, template_index, *arg0,
+ *arg1);
+ } else {
+ CallRuntime(Runtime::kThrowTypeError, context, template_index, *arg0, *arg1,
+ *arg2);
+ }
+ Unreachable();
+}
+
+TNode<BoolT> CodeStubAssembler::InstanceTypeEqual(
+ SloppyTNode<Int32T> instance_type, int type) {
+ return Word32Equal(instance_type, Int32Constant(type));
+}
+
+TNode<BoolT> CodeStubAssembler::IsDictionaryMap(TNode<Map> map) {
+ return IsSetWord32<Map::Bits3::IsDictionaryMapBit>(LoadMapBitField3(map));
+}
+
+TNode<BoolT> CodeStubAssembler::IsExtensibleMap(TNode<Map> map) {
+ return IsSetWord32<Map::Bits3::IsExtensibleBit>(LoadMapBitField3(map));
+}
+
+TNode<BoolT> CodeStubAssembler::IsExtensibleNonPrototypeMap(TNode<Map> map) {
+ int kMask =
+ Map::Bits3::IsExtensibleBit::kMask | Map::Bits3::IsPrototypeMapBit::kMask;
+ int kExpected = Map::Bits3::IsExtensibleBit::kMask;
+ return Word32Equal(Word32And(LoadMapBitField3(map), Int32Constant(kMask)),
+ Int32Constant(kExpected));
+}
+
+TNode<BoolT> CodeStubAssembler::IsCallableMap(TNode<Map> map) {
+ return IsSetWord32<Map::Bits1::IsCallableBit>(LoadMapBitField(map));
+}
+
+TNode<BoolT> CodeStubAssembler::IsDeprecatedMap(TNode<Map> map) {
+ return IsSetWord32<Map::Bits3::IsDeprecatedBit>(LoadMapBitField3(map));
+}
+
+TNode<BoolT> CodeStubAssembler::IsUndetectableMap(TNode<Map> map) {
+ return IsSetWord32<Map::Bits1::IsUndetectableBit>(LoadMapBitField(map));
+}
+
+TNode<BoolT> CodeStubAssembler::IsNoElementsProtectorCellInvalid() {
+ TNode<Smi> invalid = SmiConstant(Protectors::kProtectorInvalid);
+ TNode<PropertyCell> cell = NoElementsProtectorConstant();
+ TNode<Object> cell_value = LoadObjectField(cell, PropertyCell::kValueOffset);
+ return TaggedEqual(cell_value, invalid);
+}
+
+TNode<BoolT> CodeStubAssembler::IsArrayIteratorProtectorCellInvalid() {
+ TNode<Smi> invalid = SmiConstant(Protectors::kProtectorInvalid);
+ TNode<PropertyCell> cell = ArrayIteratorProtectorConstant();
+ TNode<Object> cell_value = LoadObjectField(cell, PropertyCell::kValueOffset);
+ return TaggedEqual(cell_value, invalid);
+}
+
+TNode<BoolT> CodeStubAssembler::IsPromiseResolveProtectorCellInvalid() {
+ TNode<Smi> invalid = SmiConstant(Protectors::kProtectorInvalid);
+ TNode<PropertyCell> cell = PromiseResolveProtectorConstant();
+ TNode<Object> cell_value = LoadObjectField(cell, PropertyCell::kValueOffset);
+ return TaggedEqual(cell_value, invalid);
+}
+
+TNode<BoolT> CodeStubAssembler::IsPromiseThenProtectorCellInvalid() {
+ TNode<Smi> invalid = SmiConstant(Protectors::kProtectorInvalid);
+ TNode<PropertyCell> cell = PromiseThenProtectorConstant();
+ TNode<Object> cell_value = LoadObjectField(cell, PropertyCell::kValueOffset);
+ return TaggedEqual(cell_value, invalid);
+}
+
+TNode<BoolT> CodeStubAssembler::IsArraySpeciesProtectorCellInvalid() {
+ TNode<Smi> invalid = SmiConstant(Protectors::kProtectorInvalid);
+ TNode<PropertyCell> cell = ArraySpeciesProtectorConstant();
+ TNode<Object> cell_value = LoadObjectField(cell, PropertyCell::kValueOffset);
+ return TaggedEqual(cell_value, invalid);
+}
+
+TNode<BoolT> CodeStubAssembler::IsTypedArraySpeciesProtectorCellInvalid() {
+ TNode<Smi> invalid = SmiConstant(Protectors::kProtectorInvalid);
+ TNode<PropertyCell> cell = TypedArraySpeciesProtectorConstant();
+ TNode<Object> cell_value = LoadObjectField(cell, PropertyCell::kValueOffset);
+ return TaggedEqual(cell_value, invalid);
+}
+
+TNode<BoolT> CodeStubAssembler::IsRegExpSpeciesProtectorCellInvalid() {
+ TNode<Smi> invalid = SmiConstant(Protectors::kProtectorInvalid);
+ TNode<PropertyCell> cell = RegExpSpeciesProtectorConstant();
+ TNode<Object> cell_value = LoadObjectField(cell, PropertyCell::kValueOffset);
+ return TaggedEqual(cell_value, invalid);
+}
+
+TNode<BoolT> CodeStubAssembler::IsPromiseSpeciesProtectorCellInvalid() {
+ TNode<Smi> invalid = SmiConstant(Protectors::kProtectorInvalid);
+ TNode<PropertyCell> cell = PromiseSpeciesProtectorConstant();
+ TNode<Object> cell_value = LoadObjectField(cell, PropertyCell::kValueOffset);
+ return TaggedEqual(cell_value, invalid);
+}
+
+TNode<BoolT> CodeStubAssembler::IsPrototypeInitialArrayPrototype(
+ TNode<Context> context, TNode<Map> map) {
+ const TNode<NativeContext> native_context = LoadNativeContext(context);
+ const TNode<Object> initial_array_prototype = LoadContextElement(
+ native_context, Context::INITIAL_ARRAY_PROTOTYPE_INDEX);
+ TNode<HeapObject> proto = LoadMapPrototype(map);
+ return TaggedEqual(proto, initial_array_prototype);
+}
+
+TNode<BoolT> CodeStubAssembler::IsPrototypeTypedArrayPrototype(
+ TNode<Context> context, TNode<Map> map) {
+ const TNode<NativeContext> native_context = LoadNativeContext(context);
+ const TNode<Object> typed_array_prototype =
+ LoadContextElement(native_context, Context::TYPED_ARRAY_PROTOTYPE_INDEX);
+ TNode<HeapObject> proto = LoadMapPrototype(map);
+ TNode<HeapObject> proto_of_proto = Select<HeapObject>(
+ IsJSObject(proto), [=] { return LoadMapPrototype(LoadMap(proto)); },
+ [=] { return NullConstant(); });
+ return TaggedEqual(proto_of_proto, typed_array_prototype);
+}
+
+TNode<BoolT> CodeStubAssembler::IsFastAliasedArgumentsMap(
+ TNode<Context> context, TNode<Map> map) {
+ const TNode<NativeContext> native_context = LoadNativeContext(context);
+ const TNode<Object> arguments_map = LoadContextElement(
+ native_context, Context::FAST_ALIASED_ARGUMENTS_MAP_INDEX);
+ return TaggedEqual(arguments_map, map);
+}
+
+TNode<BoolT> CodeStubAssembler::IsSlowAliasedArgumentsMap(
+ TNode<Context> context, TNode<Map> map) {
+ const TNode<NativeContext> native_context = LoadNativeContext(context);
+ const TNode<Object> arguments_map = LoadContextElement(
+ native_context, Context::SLOW_ALIASED_ARGUMENTS_MAP_INDEX);
+ return TaggedEqual(arguments_map, map);
+}
+
+TNode<BoolT> CodeStubAssembler::IsSloppyArgumentsMap(TNode<Context> context,
+ TNode<Map> map) {
+ const TNode<NativeContext> native_context = LoadNativeContext(context);
+ const TNode<Object> arguments_map =
+ LoadContextElement(native_context, Context::SLOPPY_ARGUMENTS_MAP_INDEX);
+ return TaggedEqual(arguments_map, map);
+}
+
+TNode<BoolT> CodeStubAssembler::IsStrictArgumentsMap(TNode<Context> context,
+ TNode<Map> map) {
+ const TNode<NativeContext> native_context = LoadNativeContext(context);
+ const TNode<Object> arguments_map =
+ LoadContextElement(native_context, Context::STRICT_ARGUMENTS_MAP_INDEX);
+ return TaggedEqual(arguments_map, map);
+}
+
+TNode<BoolT> CodeStubAssembler::TaggedIsCallable(TNode<Object> object) {
+ return Select<BoolT>(
+ TaggedIsSmi(object), [=] { return Int32FalseConstant(); },
+ [=] {
+ return IsCallableMap(LoadMap(UncheckedCast<HeapObject>(object)));
+ });
+}
+
+TNode<BoolT> CodeStubAssembler::IsCallable(TNode<HeapObject> object) {
+ return IsCallableMap(LoadMap(object));
+}
+
+TNode<BoolT> CodeStubAssembler::IsConstructorMap(TNode<Map> map) {
+ return IsSetWord32<Map::Bits1::IsConstructorBit>(LoadMapBitField(map));
+}
+
+TNode<BoolT> CodeStubAssembler::IsConstructor(TNode<HeapObject> object) {
+ return IsConstructorMap(LoadMap(object));
+}
+
+TNode<BoolT> CodeStubAssembler::IsFunctionWithPrototypeSlotMap(TNode<Map> map) {
+ return IsSetWord32<Map::Bits1::HasPrototypeSlotBit>(LoadMapBitField(map));
+}
+
+TNode<BoolT> CodeStubAssembler::IsSpecialReceiverInstanceType(
+ TNode<Int32T> instance_type) {
+ STATIC_ASSERT(JS_GLOBAL_OBJECT_TYPE <= LAST_SPECIAL_RECEIVER_TYPE);
+ return Int32LessThanOrEqual(instance_type,
+ Int32Constant(LAST_SPECIAL_RECEIVER_TYPE));
+}
+
+TNode<BoolT> CodeStubAssembler::IsCustomElementsReceiverInstanceType(
+ TNode<Int32T> instance_type) {
+ return Int32LessThanOrEqual(instance_type,
+ Int32Constant(LAST_CUSTOM_ELEMENTS_RECEIVER));
+}
+
+TNode<BoolT> CodeStubAssembler::IsStringInstanceType(
+ SloppyTNode<Int32T> instance_type) {
+ STATIC_ASSERT(INTERNALIZED_STRING_TYPE == FIRST_TYPE);
+ return Int32LessThan(instance_type, Int32Constant(FIRST_NONSTRING_TYPE));
+}
+
+TNode<BoolT> CodeStubAssembler::IsOneByteStringInstanceType(
+ TNode<Int32T> instance_type) {
+ CSA_ASSERT(this, IsStringInstanceType(instance_type));
+ return Word32Equal(
+ Word32And(instance_type, Int32Constant(kStringEncodingMask)),
+ Int32Constant(kOneByteStringTag));
+}
+
+TNode<BoolT> CodeStubAssembler::IsSequentialStringInstanceType(
+ SloppyTNode<Int32T> instance_type) {
+ CSA_ASSERT(this, IsStringInstanceType(instance_type));
+ return Word32Equal(
+ Word32And(instance_type, Int32Constant(kStringRepresentationMask)),
+ Int32Constant(kSeqStringTag));
+}
+
+TNode<BoolT> CodeStubAssembler::IsSeqOneByteStringInstanceType(
+ TNode<Int32T> instance_type) {
+ CSA_ASSERT(this, IsStringInstanceType(instance_type));
+ return Word32Equal(
+ Word32And(instance_type,
+ Int32Constant(kStringRepresentationMask | kStringEncodingMask)),
+ Int32Constant(kSeqStringTag | kOneByteStringTag));
+}
+
+TNode<BoolT> CodeStubAssembler::IsConsStringInstanceType(
+ SloppyTNode<Int32T> instance_type) {
+ CSA_ASSERT(this, IsStringInstanceType(instance_type));
+ return Word32Equal(
+ Word32And(instance_type, Int32Constant(kStringRepresentationMask)),
+ Int32Constant(kConsStringTag));
+}
+
+TNode<BoolT> CodeStubAssembler::IsIndirectStringInstanceType(
+ SloppyTNode<Int32T> instance_type) {
+ CSA_ASSERT(this, IsStringInstanceType(instance_type));
+ STATIC_ASSERT(kIsIndirectStringMask == 0x1);
+ STATIC_ASSERT(kIsIndirectStringTag == 0x1);
+ return UncheckedCast<BoolT>(
+ Word32And(instance_type, Int32Constant(kIsIndirectStringMask)));
+}
+
+TNode<BoolT> CodeStubAssembler::IsExternalStringInstanceType(
+ SloppyTNode<Int32T> instance_type) {
+ CSA_ASSERT(this, IsStringInstanceType(instance_type));
+ return Word32Equal(
+ Word32And(instance_type, Int32Constant(kStringRepresentationMask)),
+ Int32Constant(kExternalStringTag));
+}
+
+TNode<BoolT> CodeStubAssembler::IsUncachedExternalStringInstanceType(
+ SloppyTNode<Int32T> instance_type) {
+ CSA_ASSERT(this, IsStringInstanceType(instance_type));
+ STATIC_ASSERT(kUncachedExternalStringTag != 0);
+ return IsSetWord32(instance_type, kUncachedExternalStringMask);
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSReceiverInstanceType(
+ SloppyTNode<Int32T> instance_type) {
+ STATIC_ASSERT(LAST_JS_RECEIVER_TYPE == LAST_TYPE);
+ return Int32GreaterThanOrEqual(instance_type,
+ Int32Constant(FIRST_JS_RECEIVER_TYPE));
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSReceiverMap(TNode<Map> map) {
+ return IsJSReceiverInstanceType(LoadMapInstanceType(map));
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSReceiver(TNode<HeapObject> object) {
+ return IsJSReceiverMap(LoadMap(object));
+}
+
+TNode<BoolT> CodeStubAssembler::IsNullOrJSReceiver(TNode<HeapObject> object) {
+ return UncheckedCast<BoolT>(Word32Or(IsJSReceiver(object), IsNull(object)));
+}
+
+TNode<BoolT> CodeStubAssembler::IsNullOrUndefined(SloppyTNode<Object> value) {
+ return UncheckedCast<BoolT>(Word32Or(IsUndefined(value), IsNull(value)));
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSGlobalProxyInstanceType(
+ SloppyTNode<Int32T> instance_type) {
+ return InstanceTypeEqual(instance_type, JS_GLOBAL_PROXY_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSGlobalProxyMap(TNode<Map> map) {
+ return IsJSGlobalProxyInstanceType(LoadMapInstanceType(map));
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSGlobalProxy(TNode<HeapObject> object) {
+ return IsJSGlobalProxyMap(LoadMap(object));
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSGeneratorMap(TNode<Map> map) {
+ return InstanceTypeEqual(LoadMapInstanceType(map), JS_GENERATOR_OBJECT_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSObjectInstanceType(
+ SloppyTNode<Int32T> instance_type) {
+ STATIC_ASSERT(LAST_JS_OBJECT_TYPE == LAST_TYPE);
+ return Int32GreaterThanOrEqual(instance_type,
+ Int32Constant(FIRST_JS_OBJECT_TYPE));
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSObjectMap(TNode<Map> map) {
+ return IsJSObjectInstanceType(LoadMapInstanceType(map));
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSObject(TNode<HeapObject> object) {
+ return IsJSObjectMap(LoadMap(object));
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSFinalizationRegistryMap(TNode<Map> map) {
+ return InstanceTypeEqual(LoadMapInstanceType(map),
+ JS_FINALIZATION_REGISTRY_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSFinalizationRegistry(
+ TNode<HeapObject> object) {
+ return IsJSFinalizationRegistryMap(LoadMap(object));
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSPromiseMap(TNode<Map> map) {
+ return InstanceTypeEqual(LoadMapInstanceType(map), JS_PROMISE_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSPromise(TNode<HeapObject> object) {
+ return IsJSPromiseMap(LoadMap(object));
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSProxy(TNode<HeapObject> object) {
+ return HasInstanceType(object, JS_PROXY_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSStringIterator(TNode<HeapObject> object) {
+ return HasInstanceType(object, JS_STRING_ITERATOR_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSRegExpStringIterator(
+ TNode<HeapObject> object) {
+ return HasInstanceType(object, JS_REG_EXP_STRING_ITERATOR_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsMap(TNode<HeapObject> map) {
+ return IsMetaMap(LoadMap(map));
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSPrimitiveWrapperInstanceType(
+ SloppyTNode<Int32T> instance_type) {
+ return InstanceTypeEqual(instance_type, JS_PRIMITIVE_WRAPPER_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSPrimitiveWrapper(TNode<HeapObject> object) {
+ return IsJSPrimitiveWrapperMap(LoadMap(object));
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSPrimitiveWrapperMap(TNode<Map> map) {
+ return IsJSPrimitiveWrapperInstanceType(LoadMapInstanceType(map));
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSArrayInstanceType(
+ SloppyTNode<Int32T> instance_type) {
+ return InstanceTypeEqual(instance_type, JS_ARRAY_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSArray(TNode<HeapObject> object) {
+ return IsJSArrayMap(LoadMap(object));
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSArrayMap(TNode<Map> map) {
+ return IsJSArrayInstanceType(LoadMapInstanceType(map));
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSArrayIterator(TNode<HeapObject> object) {
+ return HasInstanceType(object, JS_ARRAY_ITERATOR_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSAsyncGeneratorObject(
+ TNode<HeapObject> object) {
+ return HasInstanceType(object, JS_ASYNC_GENERATOR_OBJECT_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsFixedArray(TNode<HeapObject> object) {
+ return HasInstanceType(object, FIXED_ARRAY_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsFixedArraySubclass(TNode<HeapObject> object) {
+ TNode<Uint16T> instance_type = LoadInstanceType(object);
+ return UncheckedCast<BoolT>(
+ Word32And(Int32GreaterThanOrEqual(instance_type,
+ Int32Constant(FIRST_FIXED_ARRAY_TYPE)),
+ Int32LessThanOrEqual(instance_type,
+ Int32Constant(LAST_FIXED_ARRAY_TYPE))));
+}
+
+TNode<BoolT> CodeStubAssembler::IsNotWeakFixedArraySubclass(
+ TNode<HeapObject> object) {
+ TNode<Uint16T> instance_type = LoadInstanceType(object);
+ return UncheckedCast<BoolT>(Word32Or(
+ Int32LessThan(instance_type, Int32Constant(FIRST_WEAK_FIXED_ARRAY_TYPE)),
+ Int32GreaterThan(instance_type,
+ Int32Constant(LAST_WEAK_FIXED_ARRAY_TYPE))));
+}
+
+TNode<BoolT> CodeStubAssembler::IsPropertyArray(TNode<HeapObject> object) {
+ return HasInstanceType(object, PROPERTY_ARRAY_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsPromiseReactionJobTask(
+ TNode<HeapObject> object) {
+ TNode<Uint16T> instance_type = LoadInstanceType(object);
+ return IsInRange(instance_type, FIRST_PROMISE_REACTION_JOB_TASK_TYPE,
+ LAST_PROMISE_REACTION_JOB_TASK_TYPE);
+}
+
+// This complicated check is due to elements oddities. If a smi array is empty
+// after Array.p.shift, it is replaced by the empty array constant. If it is
+// later filled with a double element, we try to grow it but pass in a double
+// elements kind. Usually this would cause a size mismatch (since the source
+// fixed array has HOLEY_ELEMENTS and destination has
+// HOLEY_DOUBLE_ELEMENTS), but we don't have to worry about it when the
+// source array is empty.
+// TODO(jgruber): It might we worth creating an empty_double_array constant to
+// simplify this case.
+TNode<BoolT> CodeStubAssembler::IsFixedArrayWithKindOrEmpty(
+ TNode<FixedArrayBase> object, ElementsKind kind) {
+ Label out(this);
+ TVARIABLE(BoolT, var_result, Int32TrueConstant());
+
+ GotoIf(IsFixedArrayWithKind(object, kind), &out);
+
+ const TNode<Smi> length = LoadFixedArrayBaseLength(object);
+ GotoIf(SmiEqual(length, SmiConstant(0)), &out);
+
+ var_result = Int32FalseConstant();
+ Goto(&out);
+
+ BIND(&out);
+ return var_result.value();
+}
+
+TNode<BoolT> CodeStubAssembler::IsFixedArrayWithKind(TNode<HeapObject> object,
+ ElementsKind kind) {
+ if (IsDoubleElementsKind(kind)) {
+ return IsFixedDoubleArray(object);
+ } else {
+ DCHECK(IsSmiOrObjectElementsKind(kind) || IsSealedElementsKind(kind) ||
+ IsNonextensibleElementsKind(kind));
+ return IsFixedArraySubclass(object);
+ }
+}
+
+TNode<BoolT> CodeStubAssembler::IsBoolean(TNode<HeapObject> object) {
+ return IsBooleanMap(LoadMap(object));
+}
+
+TNode<BoolT> CodeStubAssembler::IsPropertyCell(TNode<HeapObject> object) {
+ return IsPropertyCellMap(LoadMap(object));
+}
+
+TNode<BoolT> CodeStubAssembler::IsHeapNumberInstanceType(
+ SloppyTNode<Int32T> instance_type) {
+ return InstanceTypeEqual(instance_type, HEAP_NUMBER_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsOddball(TNode<HeapObject> object) {
+ return IsOddballInstanceType(LoadInstanceType(object));
+}
+
+TNode<BoolT> CodeStubAssembler::IsOddballInstanceType(
+ SloppyTNode<Int32T> instance_type) {
+ return InstanceTypeEqual(instance_type, ODDBALL_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsName(TNode<HeapObject> object) {
+ return IsNameInstanceType(LoadInstanceType(object));
+}
+
+TNode<BoolT> CodeStubAssembler::IsNameInstanceType(
+ SloppyTNode<Int32T> instance_type) {
+ return Int32LessThanOrEqual(instance_type, Int32Constant(LAST_NAME_TYPE));
+}
+
+TNode<BoolT> CodeStubAssembler::IsString(TNode<HeapObject> object) {
+ return IsStringInstanceType(LoadInstanceType(object));
+}
+
+TNode<BoolT> CodeStubAssembler::IsSeqOneByteString(TNode<HeapObject> object) {
+ return IsSeqOneByteStringInstanceType(LoadInstanceType(object));
+}
+
+TNode<BoolT> CodeStubAssembler::IsSymbolInstanceType(
+ SloppyTNode<Int32T> instance_type) {
+ return InstanceTypeEqual(instance_type, SYMBOL_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsInternalizedStringInstanceType(
+ TNode<Int32T> instance_type) {
+ STATIC_ASSERT(kNotInternalizedTag != 0);
+ return Word32Equal(
+ Word32And(instance_type,
+ Int32Constant(kIsNotStringMask | kIsNotInternalizedMask)),
+ Int32Constant(kStringTag | kInternalizedTag));
+}
+
+TNode<BoolT> CodeStubAssembler::IsUniqueName(TNode<HeapObject> object) {
+ TNode<Uint16T> instance_type = LoadInstanceType(object);
+ return Select<BoolT>(
+ IsInternalizedStringInstanceType(instance_type),
+ [=] { return Int32TrueConstant(); },
+ [=] { return IsSymbolInstanceType(instance_type); });
+}
+
+// Semantics: guaranteed not to be an integer index (i.e. contains non-digit
+// characters, or is outside MAX_SAFE_INTEGER/size_t range). Note that for
+// non-TypedArray receivers, there are additional strings that must be treated
+// as named property keys, namely the range [0xFFFFFFFF, MAX_SAFE_INTEGER].
+TNode<BoolT> CodeStubAssembler::IsUniqueNameNoIndex(TNode<HeapObject> object) {
+ TNode<Uint16T> instance_type = LoadInstanceType(object);
+ return Select<BoolT>(
+ IsInternalizedStringInstanceType(instance_type),
+ [=] {
+ return IsSetWord32(LoadNameHashField(CAST(object)),
+ Name::kIsNotIntegerIndexMask);
+ },
+ [=] { return IsSymbolInstanceType(instance_type); });
+}
+
+// Semantics: {object} is a Symbol, or a String that doesn't have a cached
+// index. This returns {true} for strings containing representations of
+// integers in the range above 9999999 (per kMaxCachedArrayIndexLength)
+// and below MAX_SAFE_INTEGER. For CSA_ASSERTs ensuring correct usage, this is
+// better than no checking; and we don't have a good/fast way to accurately
+// check such strings for being within "array index" (uint32_t) range.
+TNode<BoolT> CodeStubAssembler::IsUniqueNameNoCachedIndex(
+ TNode<HeapObject> object) {
+ TNode<Uint16T> instance_type = LoadInstanceType(object);
+ return Select<BoolT>(
+ IsInternalizedStringInstanceType(instance_type),
+ [=] {
+ return IsSetWord32(LoadNameHashField(CAST(object)),
+ Name::kDoesNotContainCachedArrayIndexMask);
+ },
+ [=] { return IsSymbolInstanceType(instance_type); });
+}
+
+TNode<BoolT> CodeStubAssembler::IsBigIntInstanceType(
+ SloppyTNode<Int32T> instance_type) {
+ return InstanceTypeEqual(instance_type, BIGINT_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsBigInt(TNode<HeapObject> object) {
+ return IsBigIntInstanceType(LoadInstanceType(object));
+}
+
+TNode<BoolT> CodeStubAssembler::IsPrimitiveInstanceType(
+ SloppyTNode<Int32T> instance_type) {
+ return Int32LessThanOrEqual(instance_type,
+ Int32Constant(LAST_PRIMITIVE_HEAP_OBJECT_TYPE));
+}
+
+TNode<BoolT> CodeStubAssembler::IsPrivateName(SloppyTNode<Symbol> symbol) {
+ TNode<Uint32T> flags = LoadObjectField<Uint32T>(symbol, Symbol::kFlagsOffset);
+ return IsSetWord32<Symbol::IsPrivateNameBit>(flags);
+}
+
+TNode<BoolT> CodeStubAssembler::IsHashTable(TNode<HeapObject> object) {
+ TNode<Uint16T> instance_type = LoadInstanceType(object);
+ return UncheckedCast<BoolT>(
+ Word32And(Int32GreaterThanOrEqual(instance_type,
+ Int32Constant(FIRST_HASH_TABLE_TYPE)),
+ Int32LessThanOrEqual(instance_type,
+ Int32Constant(LAST_HASH_TABLE_TYPE))));
+}
+
+TNode<BoolT> CodeStubAssembler::IsEphemeronHashTable(TNode<HeapObject> object) {
+ return HasInstanceType(object, EPHEMERON_HASH_TABLE_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsNameDictionary(TNode<HeapObject> object) {
+ return HasInstanceType(object, NAME_DICTIONARY_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsGlobalDictionary(TNode<HeapObject> object) {
+ return HasInstanceType(object, GLOBAL_DICTIONARY_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsNumberDictionary(TNode<HeapObject> object) {
+ return HasInstanceType(object, NUMBER_DICTIONARY_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSGeneratorObject(TNode<HeapObject> object) {
+ return HasInstanceType(object, JS_GENERATOR_OBJECT_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSFunctionInstanceType(
+ SloppyTNode<Int32T> instance_type) {
+ return InstanceTypeEqual(instance_type, JS_FUNCTION_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSFunction(TNode<HeapObject> object) {
+ return IsJSFunctionMap(LoadMap(object));
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSBoundFunction(TNode<HeapObject> object) {
+ return HasInstanceType(object, JS_BOUND_FUNCTION_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSFunctionMap(TNode<Map> map) {
+ return IsJSFunctionInstanceType(LoadMapInstanceType(map));
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSTypedArrayInstanceType(
+ SloppyTNode<Int32T> instance_type) {
+ return InstanceTypeEqual(instance_type, JS_TYPED_ARRAY_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSTypedArrayMap(TNode<Map> map) {
+ return IsJSTypedArrayInstanceType(LoadMapInstanceType(map));
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSTypedArray(TNode<HeapObject> object) {
+ return IsJSTypedArrayMap(LoadMap(object));
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSArrayBuffer(TNode<HeapObject> object) {
+ return HasInstanceType(object, JS_ARRAY_BUFFER_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSDataView(TNode<HeapObject> object) {
+ return HasInstanceType(object, JS_DATA_VIEW_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsJSRegExp(TNode<HeapObject> object) {
+ return HasInstanceType(object, JS_REG_EXP_TYPE);
+}
+
+TNode<BoolT> CodeStubAssembler::IsNumeric(SloppyTNode<Object> object) {
+ return Select<BoolT>(
+ TaggedIsSmi(object), [=] { return Int32TrueConstant(); },
+ [=] {
+ return UncheckedCast<BoolT>(
+ Word32Or(IsHeapNumber(CAST(object)), IsBigInt(CAST(object))));
+ });
+}
+
+TNode<BoolT> CodeStubAssembler::IsNumberNormalized(TNode<Number> number) {
+ TVARIABLE(BoolT, var_result, Int32TrueConstant());
+ Label out(this);
+
+ GotoIf(TaggedIsSmi(number), &out);
+
+ TNode<Float64T> value = LoadHeapNumberValue(CAST(number));
+ TNode<Float64T> smi_min =
+ Float64Constant(static_cast<double>(Smi::kMinValue));
+ TNode<Float64T> smi_max =
+ Float64Constant(static_cast<double>(Smi::kMaxValue));
+
+ GotoIf(Float64LessThan(value, smi_min), &out);
+ GotoIf(Float64GreaterThan(value, smi_max), &out);
+ GotoIfNot(Float64Equal(value, value), &out); // NaN.
+
+ var_result = Int32FalseConstant();
+ Goto(&out);
+
+ BIND(&out);
+ return var_result.value();
+}
+
+TNode<BoolT> CodeStubAssembler::IsNumberPositive(TNode<Number> number) {
+ return Select<BoolT>(
+ TaggedIsSmi(number), [=] { return TaggedIsPositiveSmi(number); },
+ [=] { return IsHeapNumberPositive(CAST(number)); });
+}
+
+// TODO(cbruni): Use TNode<HeapNumber> instead of custom name.
+TNode<BoolT> CodeStubAssembler::IsHeapNumberPositive(TNode<HeapNumber> number) {
+ TNode<Float64T> value = LoadHeapNumberValue(number);
+ TNode<Float64T> float_zero = Float64Constant(0.);
+ return Float64GreaterThanOrEqual(value, float_zero);
+}
+
+TNode<BoolT> CodeStubAssembler::IsNumberNonNegativeSafeInteger(
+ TNode<Number> number) {
+ return Select<BoolT>(
+ // TODO(cbruni): Introduce TaggedIsNonNegateSmi to avoid confusion.
+ TaggedIsSmi(number), [=] { return TaggedIsPositiveSmi(number); },
+ [=] {
+ TNode<HeapNumber> heap_number = CAST(number);
+ return Select<BoolT>(
+ IsInteger(heap_number),
+ [=] { return IsHeapNumberPositive(heap_number); },
+ [=] { return Int32FalseConstant(); });
+ });
+}
+
+TNode<BoolT> CodeStubAssembler::IsSafeInteger(TNode<Object> number) {
+ return Select<BoolT>(
+ TaggedIsSmi(number), [=] { return Int32TrueConstant(); },
+ [=] {
+ return Select<BoolT>(
+ IsHeapNumber(CAST(number)),
+ [=] { return IsSafeInteger(UncheckedCast<HeapNumber>(number)); },
+ [=] { return Int32FalseConstant(); });
+ });
+}
+
+TNode<BoolT> CodeStubAssembler::IsSafeInteger(TNode<HeapNumber> number) {
+ // Load the actual value of {number}.
+ TNode<Float64T> number_value = LoadHeapNumberValue(number);
+ // Truncate the value of {number} to an integer (or an infinity).
+ TNode<Float64T> integer = Float64Trunc(number_value);
+
+ return Select<BoolT>(
+ // Check if {number}s value matches the integer (ruling out the
+ // infinities).
+ Float64Equal(Float64Sub(number_value, integer), Float64Constant(0.0)),
+ [=] {
+ // Check if the {integer} value is in safe integer range.
+ return Float64LessThanOrEqual(Float64Abs(integer),
+ Float64Constant(kMaxSafeInteger));
+ },
+ [=] { return Int32FalseConstant(); });
+}
+
+TNode<BoolT> CodeStubAssembler::IsInteger(TNode<Object> number) {
+ return Select<BoolT>(
+ TaggedIsSmi(number), [=] { return Int32TrueConstant(); },
+ [=] {
+ return Select<BoolT>(
+ IsHeapNumber(CAST(number)),
+ [=] { return IsInteger(UncheckedCast<HeapNumber>(number)); },
+ [=] { return Int32FalseConstant(); });
+ });
+}
+
+TNode<BoolT> CodeStubAssembler::IsInteger(TNode<HeapNumber> number) {
+ TNode<Float64T> number_value = LoadHeapNumberValue(number);
+ // Truncate the value of {number} to an integer (or an infinity).
+ TNode<Float64T> integer = Float64Trunc(number_value);
+ // Check if {number}s value matches the integer (ruling out the infinities).
+ return Float64Equal(Float64Sub(number_value, integer), Float64Constant(0.0));
+}
+
+TNode<BoolT> CodeStubAssembler::IsHeapNumberUint32(TNode<HeapNumber> number) {
+ // Check that the HeapNumber is a valid uint32
+ return Select<BoolT>(
+ IsHeapNumberPositive(number),
+ [=] {
+ TNode<Float64T> value = LoadHeapNumberValue(number);
+ TNode<Uint32T> int_value = TruncateFloat64ToWord32(value);
+ return Float64Equal(value, ChangeUint32ToFloat64(int_value));
+ },
+ [=] { return Int32FalseConstant(); });
+}
+
+TNode<BoolT> CodeStubAssembler::IsNumberArrayIndex(TNode<Number> number) {
+ return Select<BoolT>(
+ TaggedIsSmi(number), [=] { return TaggedIsPositiveSmi(number); },
+ [=] { return IsHeapNumberUint32(CAST(number)); });
+}
+
+template <typename TIndex>
+TNode<BoolT> CodeStubAssembler::FixedArraySizeDoesntFitInNewSpace(
+ TNode<TIndex> element_count, int base_size) {
+ static_assert(
+ std::is_same<TIndex, Smi>::value || std::is_same<TIndex, IntPtrT>::value,
+ "Only Smi or IntPtrT element_count is allowed");
+ int max_newspace_elements =
+ (kMaxRegularHeapObjectSize - base_size) / kTaggedSize;
+ return IntPtrOrSmiGreaterThan(
+ element_count, IntPtrOrSmiConstant<TIndex>(max_newspace_elements));
+}
+
+TNode<Int32T> CodeStubAssembler::StringCharCodeAt(TNode<String> string,
+ TNode<UintPtrT> index) {
+ CSA_ASSERT(this, UintPtrLessThan(index, LoadStringLengthAsWord(string)));
+
+ TVARIABLE(Int32T, var_result);
+
+ Label return_result(this), if_runtime(this, Label::kDeferred),
+ if_stringistwobyte(this), if_stringisonebyte(this);
+
+ ToDirectStringAssembler to_direct(state(), string);
+ to_direct.TryToDirect(&if_runtime);
+ const TNode<UintPtrT> offset =
+ UintPtrAdd(index, Unsigned(to_direct.offset()));
+ const TNode<Int32T> instance_type = to_direct.instance_type();
+ const TNode<RawPtrT> string_data = to_direct.PointerToData(&if_runtime);
+
+ // Check if the {string} is a TwoByteSeqString or a OneByteSeqString.
+ Branch(IsOneByteStringInstanceType(instance_type), &if_stringisonebyte,
+ &if_stringistwobyte);
+
+ BIND(&if_stringisonebyte);
+ {
+ var_result = UncheckedCast<Int32T>(Load<Uint8T>(string_data, offset));
+ Goto(&return_result);
+ }
+
+ BIND(&if_stringistwobyte);
+ {
+ var_result = UncheckedCast<Int32T>(
+ Load<Uint16T>(string_data, WordShl(offset, IntPtrConstant(1))));
+ Goto(&return_result);
+ }
+
+ BIND(&if_runtime);
+ {
+ TNode<Object> result =
+ CallRuntime(Runtime::kStringCharCodeAt, NoContextConstant(), string,
+ ChangeUintPtrToTagged(index));
+ var_result = SmiToInt32(CAST(result));
+ Goto(&return_result);
+ }
+
+ BIND(&return_result);
+ return var_result.value();
+}
+
+TNode<String> CodeStubAssembler::StringFromSingleCharCode(TNode<Int32T> code) {
+ TVARIABLE(String, var_result);
+
+ // Check if the {code} is a one-byte char code.
+ Label if_codeisonebyte(this), if_codeistwobyte(this, Label::kDeferred),
+ if_done(this);
+ Branch(Int32LessThanOrEqual(code, Int32Constant(String::kMaxOneByteCharCode)),
+ &if_codeisonebyte, &if_codeistwobyte);
+ BIND(&if_codeisonebyte);
+ {
+ // Load the isolate wide single character string cache.
+ TNode<FixedArray> cache = SingleCharacterStringCacheConstant();
+ TNode<IntPtrT> code_index = Signed(ChangeUint32ToWord(code));
+
+ // Check if we have an entry for the {code} in the single character string
+ // cache already.
+ Label if_entryisundefined(this, Label::kDeferred),
+ if_entryisnotundefined(this);
+ TNode<Object> entry = UnsafeLoadFixedArrayElement(cache, code_index);
+ Branch(IsUndefined(entry), &if_entryisundefined, &if_entryisnotundefined);
+
+ BIND(&if_entryisundefined);
+ {
+ // Allocate a new SeqOneByteString for {code} and store it in the {cache}.
+ TNode<String> result = AllocateSeqOneByteString(1);
+ StoreNoWriteBarrier(
+ MachineRepresentation::kWord8, result,
+ IntPtrConstant(SeqOneByteString::kHeaderSize - kHeapObjectTag), code);
+ StoreFixedArrayElement(cache, code_index, result);
+ var_result = result;
+ Goto(&if_done);
+ }
+
+ BIND(&if_entryisnotundefined);
+ {
+ // Return the entry from the {cache}.
+ var_result = CAST(entry);
+ Goto(&if_done);
+ }
+ }
+
+ BIND(&if_codeistwobyte);
+ {
+ // Allocate a new SeqTwoByteString for {code}.
+ TNode<String> result = AllocateSeqTwoByteString(1);
+ StoreNoWriteBarrier(
+ MachineRepresentation::kWord16, result,
+ IntPtrConstant(SeqTwoByteString::kHeaderSize - kHeapObjectTag), code);
+ var_result = result;
+ Goto(&if_done);
+ }
+
+ BIND(&if_done);
+ return var_result.value();
+}
+
+ToDirectStringAssembler::ToDirectStringAssembler(
+ compiler::CodeAssemblerState* state, TNode<String> string, Flags flags)
+ : CodeStubAssembler(state),
+ var_string_(string, this),
+ var_instance_type_(LoadInstanceType(string), this),
+ var_offset_(IntPtrConstant(0), this),
+ var_is_external_(Int32Constant(0), this),
+ flags_(flags) {}
+
+TNode<String> ToDirectStringAssembler::TryToDirect(Label* if_bailout) {
+ Label dispatch(this, {&var_string_, &var_offset_, &var_instance_type_});
+ Label if_iscons(this);
+ Label if_isexternal(this);
+ Label if_issliced(this);
+ Label if_isthin(this);
+ Label out(this);
+
+ Branch(IsSequentialStringInstanceType(var_instance_type_.value()), &out,
+ &dispatch);
+
+ // Dispatch based on string representation.
+ BIND(&dispatch);
+ {
+ int32_t values[] = {
+ kSeqStringTag, kConsStringTag, kExternalStringTag,
+ kSlicedStringTag, kThinStringTag,
+ };
+ Label* labels[] = {
+ &out, &if_iscons, &if_isexternal, &if_issliced, &if_isthin,
+ };
+ STATIC_ASSERT(arraysize(values) == arraysize(labels));
+
+ const TNode<Int32T> representation = Word32And(
+ var_instance_type_.value(), Int32Constant(kStringRepresentationMask));
+ Switch(representation, if_bailout, values, labels, arraysize(values));
+ }
+
+ // Cons string. Check whether it is flat, then fetch first part.
+ // Flat cons strings have an empty second part.
+ BIND(&if_iscons);
+ {
+ const TNode<String> string = var_string_.value();
+ GotoIfNot(IsEmptyString(
+ LoadObjectField<String>(string, ConsString::kSecondOffset)),
+ if_bailout);
+
+ const TNode<String> lhs =
+ LoadObjectField<String>(string, ConsString::kFirstOffset);
+ var_string_ = lhs;
+ var_instance_type_ = LoadInstanceType(lhs);
+
+ Goto(&dispatch);
+ }
+
+ // Sliced string. Fetch parent and correct start index by offset.
+ BIND(&if_issliced);
+ {
+ if (!FLAG_string_slices || (flags_ & kDontUnpackSlicedStrings)) {
+ Goto(if_bailout);
+ } else {
+ const TNode<String> string = var_string_.value();
+ const TNode<IntPtrT> sliced_offset =
+ LoadAndUntagObjectField(string, SlicedString::kOffsetOffset);
+ var_offset_ = IntPtrAdd(var_offset_.value(), sliced_offset);
+
+ const TNode<String> parent =
+ LoadObjectField<String>(string, SlicedString::kParentOffset);
+ var_string_ = parent;
+ var_instance_type_ = LoadInstanceType(parent);
+
+ Goto(&dispatch);
+ }
+ }
+
+ // Thin string. Fetch the actual string.
+ BIND(&if_isthin);
+ {
+ const TNode<String> string = var_string_.value();
+ const TNode<String> actual_string =
+ LoadObjectField<String>(string, ThinString::kActualOffset);
+ const TNode<Uint16T> actual_instance_type = LoadInstanceType(actual_string);
+
+ var_string_ = actual_string;
+ var_instance_type_ = actual_instance_type;
+
+ Goto(&dispatch);
+ }
+
+ // External string.
+ BIND(&if_isexternal);
+ var_is_external_ = Int32Constant(1);
+ Goto(&out);
+
+ BIND(&out);
+ return var_string_.value();
+}
+
+TNode<RawPtrT> ToDirectStringAssembler::TryToSequential(
+ StringPointerKind ptr_kind, Label* if_bailout) {
+ CHECK(ptr_kind == PTR_TO_DATA || ptr_kind == PTR_TO_STRING);
+
+ TVARIABLE(RawPtrT, var_result);
+ Label out(this), if_issequential(this), if_isexternal(this, Label::kDeferred);
+ Branch(is_external(), &if_isexternal, &if_issequential);
+
+ BIND(&if_issequential);
+ {
+ STATIC_ASSERT(SeqOneByteString::kHeaderSize ==
+ SeqTwoByteString::kHeaderSize);
+ TNode<RawPtrT> result =
+ ReinterpretCast<RawPtrT>(BitcastTaggedToWord(var_string_.value()));
+ if (ptr_kind == PTR_TO_DATA) {
+ result = RawPtrAdd(result, IntPtrConstant(SeqOneByteString::kHeaderSize -
+ kHeapObjectTag));
+ }
+ var_result = result;
+ Goto(&out);
+ }
+
+ BIND(&if_isexternal);
+ {
+ GotoIf(IsUncachedExternalStringInstanceType(var_instance_type_.value()),
+ if_bailout);
+
+ TNode<String> string = var_string_.value();
+ TNode<RawPtrT> result = LoadExternalStringResourceDataPtr(CAST(string));
+ if (ptr_kind == PTR_TO_STRING) {
+ result = RawPtrSub(result, IntPtrConstant(SeqOneByteString::kHeaderSize -
+ kHeapObjectTag));
+ }
+ var_result = result;
+ Goto(&out);
+ }
+
+ BIND(&out);
+ return var_result.value();
+}
+
+TNode<Number> CodeStubAssembler::StringToNumber(TNode<String> input) {
+ Label runtime(this, Label::kDeferred);
+ Label end(this);
+
+ TVARIABLE(Number, var_result);
+
+ // Check if string has a cached array index.
+ TNode<Uint32T> hash = LoadNameHashField(input);
+ GotoIf(IsSetWord32(hash, Name::kDoesNotContainCachedArrayIndexMask),
+ &runtime);
+
+ var_result =
+ SmiTag(Signed(DecodeWordFromWord32<String::ArrayIndexValueBits>(hash)));
+ Goto(&end);
+
+ BIND(&runtime);
+ {
+ var_result =
+ CAST(CallRuntime(Runtime::kStringToNumber, NoContextConstant(), input));
+ Goto(&end);
+ }
+
+ BIND(&end);
+ return var_result.value();
+}
+
+TNode<String> CodeStubAssembler::NumberToString(TNode<Number> input,
+ Label* bailout) {
+ TVARIABLE(String, result);
+ TVARIABLE(Smi, smi_input);
+ Label if_smi(this), if_heap_number(this), done(this, &result);
+
+ // Load the number string cache.
+ TNode<FixedArray> number_string_cache = NumberStringCacheConstant();
+
+ // Make the hash mask from the length of the number string cache. It
+ // contains two elements (number and string) for each cache entry.
+ TNode<IntPtrT> number_string_cache_length =
+ LoadAndUntagFixedArrayBaseLength(number_string_cache);
+ TNode<Int32T> one = Int32Constant(1);
+ TNode<Word32T> mask = Int32Sub(
+ Word32Shr(TruncateWordToInt32(number_string_cache_length), one), one);
+
+ GotoIfNot(TaggedIsSmi(input), &if_heap_number);
+ smi_input = CAST(input);
+ Goto(&if_smi);
+
+ BIND(&if_heap_number);
+ {
+ Comment("NumberToString - HeapNumber");
+ TNode<HeapNumber> heap_number_input = CAST(input);
+ // Try normalizing the HeapNumber.
+ TryHeapNumberToSmi(heap_number_input, &smi_input, &if_smi);
+
+ // Make a hash from the two 32-bit values of the double.
+ TNode<Int32T> low =
+ LoadObjectField<Int32T>(heap_number_input, HeapNumber::kValueOffset);
+ TNode<Int32T> high = LoadObjectField<Int32T>(
+ heap_number_input, HeapNumber::kValueOffset + kIntSize);
+ TNode<Word32T> hash = Word32And(Word32Xor(low, high), mask);
+ TNode<IntPtrT> entry_index =
+ Signed(ChangeUint32ToWord(Int32Add(hash, hash)));
+
+ // Cache entry's key must be a heap number
+ TNode<Object> number_key =
+ UnsafeLoadFixedArrayElement(number_string_cache, entry_index);
+ GotoIf(TaggedIsSmi(number_key), bailout);
+ TNode<HeapObject> number_key_heap_object = CAST(number_key);
+ GotoIfNot(IsHeapNumber(number_key_heap_object), bailout);
+
+ // Cache entry's key must match the heap number value we're looking for.
+ TNode<Int32T> low_compare = LoadObjectField<Int32T>(
+ number_key_heap_object, HeapNumber::kValueOffset);
+ TNode<Int32T> high_compare = LoadObjectField<Int32T>(
+ number_key_heap_object, HeapNumber::kValueOffset + kIntSize);
+ GotoIfNot(Word32Equal(low, low_compare), bailout);
+ GotoIfNot(Word32Equal(high, high_compare), bailout);
+
+ // Heap number match, return value from cache entry.
+ result = CAST(UnsafeLoadFixedArrayElement(number_string_cache, entry_index,
+ kTaggedSize));
+ Goto(&done);
+ }
+
+ BIND(&if_smi);
+ {
+ Comment("NumberToString - Smi");
+ // Load the smi key, make sure it matches the smi we're looking for.
+ TNode<Word32T> hash = Word32And(SmiToInt32(smi_input.value()), mask);
+ TNode<IntPtrT> entry_index =
+ Signed(ChangeUint32ToWord(Int32Add(hash, hash)));
+ TNode<Object> smi_key =
+ UnsafeLoadFixedArrayElement(number_string_cache, entry_index);
+ Label if_smi_cache_missed(this);
+ GotoIf(TaggedNotEqual(smi_key, smi_input.value()), &if_smi_cache_missed);
+
+ // Smi match, return value from cache entry.
+ result = CAST(UnsafeLoadFixedArrayElement(number_string_cache, entry_index,
+ kTaggedSize));
+ Goto(&done);
+
+ BIND(&if_smi_cache_missed);
+ {
+ Label store_to_cache(this);
+
+ // Bailout when the cache is not full-size.
+ const int kFullCacheSize =
+ isolate()->heap()->MaxNumberToStringCacheSize();
+ Branch(IntPtrLessThan(number_string_cache_length,
+ IntPtrConstant(kFullCacheSize)),
+ bailout, &store_to_cache);
+
+ BIND(&store_to_cache);
+ {
+ // Generate string and update string hash field.
+ result = NumberToStringSmi(SmiToInt32(smi_input.value()),
+ Int32Constant(10), bailout);
+
+ // Store string into cache.
+ StoreFixedArrayElement(number_string_cache, entry_index,
+ smi_input.value());
+ StoreFixedArrayElement(number_string_cache,
+ IntPtrAdd(entry_index, IntPtrConstant(1)),
+ result.value());
+ Goto(&done);
+ }
+ }
+ }
+ BIND(&done);
+ return result.value();
+}
+
+TNode<String> CodeStubAssembler::NumberToString(TNode<Number> input) {
+ TVARIABLE(String, result);
+ Label runtime(this, Label::kDeferred), done(this, &result);
+
+ GotoIfForceSlowPath(&runtime);
+
+ result = NumberToString(input, &runtime);
+ Goto(&done);
+
+ BIND(&runtime);
+ {
+ // No cache entry, go to the runtime.
+ result = CAST(
+ CallRuntime(Runtime::kNumberToStringSlow, NoContextConstant(), input));
+ Goto(&done);
+ }
+ BIND(&done);
+ return result.value();
+}
+
+TNode<Numeric> CodeStubAssembler::NonNumberToNumberOrNumeric(
+ TNode<Context> context, TNode<HeapObject> input, Object::Conversion mode,
+ BigIntHandling bigint_handling) {
+ CSA_ASSERT(this, Word32BinaryNot(IsHeapNumber(input)));
+
+ TVARIABLE(HeapObject, var_input, input);
+ TVARIABLE(Numeric, var_result);
+ TVARIABLE(Uint16T, instance_type, LoadInstanceType(var_input.value()));
+ Label end(this), if_inputisreceiver(this, Label::kDeferred),
+ if_inputisnotreceiver(this);
+
+ // We need to handle JSReceiver first since we might need to do two
+ // conversions due to ToPritmive.
+ Branch(IsJSReceiverInstanceType(instance_type.value()), &if_inputisreceiver,
+ &if_inputisnotreceiver);
+
+ BIND(&if_inputisreceiver);
+ {
+ // The {var_input.value()} is a JSReceiver, we need to convert it to a
+ // Primitive first using the ToPrimitive type conversion, preferably
+ // yielding a Number.
+ Callable callable = CodeFactory::NonPrimitiveToPrimitive(
+ isolate(), ToPrimitiveHint::kNumber);
+ TNode<Object> result = CallStub(callable, context, var_input.value());
+
+ // Check if the {result} is already a Number/Numeric.
+ Label if_done(this), if_notdone(this);
+ Branch(mode == Object::Conversion::kToNumber ? IsNumber(result)
+ : IsNumeric(result),
+ &if_done, &if_notdone);
+
+ BIND(&if_done);
+ {
+ // The ToPrimitive conversion already gave us a Number/Numeric, so
+ // we're done.
+ var_result = CAST(result);
+ Goto(&end);
+ }
+
+ BIND(&if_notdone);
+ {
+ // We now have a Primitive {result}, but it's not yet a
+ // Number/Numeric.
+ var_input = CAST(result);
+ // We have a new input. Redo the check and reload instance_type.
+ CSA_ASSERT(this, Word32BinaryNot(IsHeapNumber(var_input.value())));
+ instance_type = LoadInstanceType(var_input.value());
+ Goto(&if_inputisnotreceiver);
+ }
+ }
+
+ BIND(&if_inputisnotreceiver);
+ {
+ Label not_plain_primitive(this), if_inputisbigint(this),
+ if_inputisother(this, Label::kDeferred);
+
+ // String and Oddball cases.
+ TVARIABLE(Number, var_result_number);
+ TryPlainPrimitiveNonNumberToNumber(var_input.value(), &var_result_number,
+ ¬_plain_primitive);
+ var_result = var_result_number.value();
+ Goto(&end);
+
+ BIND(¬_plain_primitive);
+ {
+ Branch(IsBigIntInstanceType(instance_type.value()), &if_inputisbigint,
+ &if_inputisother);
+
+ BIND(&if_inputisbigint);
+ {
+ if (mode == Object::Conversion::kToNumeric) {
+ var_result = CAST(var_input.value());
+ Goto(&end);
+ } else {
+ DCHECK_EQ(mode, Object::Conversion::kToNumber);
+ if (bigint_handling == BigIntHandling::kThrow) {
+ Goto(&if_inputisother);
+ } else {
+ DCHECK_EQ(bigint_handling, BigIntHandling::kConvertToNumber);
+ var_result = CAST(CallRuntime(Runtime::kBigIntToNumber, context,
+ var_input.value()));
+ Goto(&end);
+ }
+ }
+ }
+
+ BIND(&if_inputisother);
+ {
+ // The {var_input.value()} is something else (e.g. Symbol), let the
+ // runtime figure out the correct exception. Note: We cannot tail call
+ // to the runtime here, as js-to-wasm trampolines also use this code
+ // currently, and they declare all outgoing parameters as untagged,
+ // while we would push a tagged object here.
+ auto function_id = mode == Object::Conversion::kToNumber
+ ? Runtime::kToNumber
+ : Runtime::kToNumeric;
+ var_result = CAST(CallRuntime(function_id, context, var_input.value()));
+ Goto(&end);
+ }
+ }
+ }
+
+ BIND(&end);
+ if (mode == Object::Conversion::kToNumber) {
+ CSA_ASSERT(this, IsNumber(var_result.value()));
+ }
+ return var_result.value();
+}
+
+TNode<Number> CodeStubAssembler::NonNumberToNumber(
+ TNode<Context> context, TNode<HeapObject> input,
+ BigIntHandling bigint_handling) {
+ return CAST(NonNumberToNumberOrNumeric(
+ context, input, Object::Conversion::kToNumber, bigint_handling));
+}
+
+void CodeStubAssembler::TryPlainPrimitiveNonNumberToNumber(
+ TNode<HeapObject> input, TVariable<Number>* var_result, Label* if_bailout) {
+ CSA_ASSERT(this, Word32BinaryNot(IsHeapNumber(input)));
+ Label done(this);
+
+ // Dispatch on the {input} instance type.
+ TNode<Uint16T> input_instance_type = LoadInstanceType(input);
+ Label if_inputisstring(this);
+ GotoIf(IsStringInstanceType(input_instance_type), &if_inputisstring);
+ GotoIfNot(InstanceTypeEqual(input_instance_type, ODDBALL_TYPE), if_bailout);
+
+ // The {input} is an Oddball, we just need to load the Number value of it.
+ *var_result = LoadObjectField<Number>(input, Oddball::kToNumberOffset);
+ Goto(&done);
+
+ BIND(&if_inputisstring);
+ {
+ // The {input} is a String, use the fast stub to convert it to a Number.
+ *var_result = StringToNumber(CAST(input));
+ Goto(&done);
+ }
+
+ BIND(&done);
+}
+
+TNode<Numeric> CodeStubAssembler::NonNumberToNumeric(TNode<Context> context,
+ TNode<HeapObject> input) {
+ return NonNumberToNumberOrNumeric(context, input,
+ Object::Conversion::kToNumeric);
+}
+
+TNode<Number> CodeStubAssembler::ToNumber_Inline(TNode<Context> context,
+ SloppyTNode<Object> input) {
+ TVARIABLE(Number, var_result);
+ Label end(this), not_smi(this, Label::kDeferred);
+
+ GotoIfNot(TaggedIsSmi(input), ¬_smi);
+ var_result = CAST(input);
+ Goto(&end);
+
+ BIND(¬_smi);
+ {
+ var_result = Select<Number>(
+ IsHeapNumber(CAST(input)), [=] { return CAST(input); },
+ [=] {
+ return CAST(
+ CallBuiltin(Builtins::kNonNumberToNumber, context, input));
+ });
+ Goto(&end);
+ }
+
+ BIND(&end);
+ return var_result.value();
+}
+
+TNode<Number> CodeStubAssembler::ToNumber(TNode<Context> context,
+ SloppyTNode<Object> input,
+ BigIntHandling bigint_handling) {
+ TVARIABLE(Number, var_result);
+ Label end(this);
+
+ Label not_smi(this, Label::kDeferred);
+ GotoIfNot(TaggedIsSmi(input), ¬_smi);
+ TNode<Smi> input_smi = CAST(input);
+ var_result = input_smi;
+ Goto(&end);
+
+ BIND(¬_smi);
+ {
+ Label not_heap_number(this, Label::kDeferred);
+ TNode<HeapObject> input_ho = CAST(input);
+ GotoIfNot(IsHeapNumber(input_ho), ¬_heap_number);
+
+ TNode<HeapNumber> input_hn = CAST(input_ho);
+ var_result = input_hn;
+ Goto(&end);
+
+ BIND(¬_heap_number);
+ {
+ var_result = NonNumberToNumber(context, input_ho, bigint_handling);
+ Goto(&end);
+ }
+ }
+
+ BIND(&end);
+ return var_result.value();
+}
+
+TNode<Number> CodeStubAssembler::PlainPrimitiveToNumber(TNode<Object> input) {
+ TVARIABLE(Number, var_result);
+ Label end(this), fallback(this);
+
+ Label not_smi(this, Label::kDeferred);
+ GotoIfNot(TaggedIsSmi(input), ¬_smi);
+ TNode<Smi> input_smi = CAST(input);
+ var_result = input_smi;
+ Goto(&end);
+
+ BIND(¬_smi);
+ {
+ Label not_heap_number(this, Label::kDeferred);
+ TNode<HeapObject> input_ho = CAST(input);
+ GotoIfNot(IsHeapNumber(input_ho), ¬_heap_number);
+
+ TNode<HeapNumber> input_hn = CAST(input_ho);
+ var_result = input_hn;
+ Goto(&end);
+
+ BIND(¬_heap_number);
+ {
+ TryPlainPrimitiveNonNumberToNumber(input_ho, &var_result, &fallback);
+ Goto(&end);
+ BIND(&fallback);
+ Unreachable();
+ }
+ }
+
+ BIND(&end);
+ return var_result.value();
+}
+
+TNode<BigInt> CodeStubAssembler::ToBigInt(TNode<Context> context,
+ TNode<Object> input) {
+ TVARIABLE(BigInt, var_result);
+ Label if_bigint(this), done(this), if_throw(this);
+
+ GotoIf(TaggedIsSmi(input), &if_throw);
+ GotoIf(IsBigInt(CAST(input)), &if_bigint);
+ var_result = CAST(CallRuntime(Runtime::kToBigInt, context, input));
+ Goto(&done);
+
+ BIND(&if_bigint);
+ var_result = CAST(input);
+ Goto(&done);
+
+ BIND(&if_throw);
+ ThrowTypeError(context, MessageTemplate::kBigIntFromObject, input);
+
+ BIND(&done);
+ return var_result.value();
+}
+
+void CodeStubAssembler::TaggedToNumeric(TNode<Context> context,
+ TNode<Object> value,
+ TVariable<Numeric>* var_numeric) {
+ TaggedToNumeric(context, value, var_numeric, nullptr);
+}
+
+void CodeStubAssembler::TaggedToNumericWithFeedback(
+ TNode<Context> context, TNode<Object> value,
+ TVariable<Numeric>* var_numeric, TVariable<Smi>* var_feedback) {
+ DCHECK_NOT_NULL(var_feedback);
+ TaggedToNumeric(context, value, var_numeric, var_feedback);
+}
+
+void CodeStubAssembler::TaggedToNumeric(TNode<Context> context,
+ TNode<Object> value,
+ TVariable<Numeric>* var_numeric,
+ TVariable<Smi>* var_feedback) {
+ Label done(this), if_smi(this), if_heapnumber(this), if_bigint(this),
+ if_oddball(this);
+ GotoIf(TaggedIsSmi(value), &if_smi);
+ TNode<HeapObject> heap_object_value = CAST(value);
+ TNode<Map> map = LoadMap(heap_object_value);
+ GotoIf(IsHeapNumberMap(map), &if_heapnumber);
+ TNode<Uint16T> instance_type = LoadMapInstanceType(map);
+ GotoIf(IsBigIntInstanceType(instance_type), &if_bigint);
+
+ // {heap_object_value} is not a Numeric yet.
+ GotoIf(Word32Equal(instance_type, Int32Constant(ODDBALL_TYPE)), &if_oddball);
+ *var_numeric = CAST(
+ CallBuiltin(Builtins::kNonNumberToNumeric, context, heap_object_value));
+ OverwriteFeedback(var_feedback, BinaryOperationFeedback::kAny);
+ Goto(&done);
+
+ BIND(&if_smi);
+ *var_numeric = CAST(value);
+ OverwriteFeedback(var_feedback, BinaryOperationFeedback::kSignedSmall);
+ Goto(&done);
+
+ BIND(&if_heapnumber);
+ *var_numeric = CAST(value);
+ OverwriteFeedback(var_feedback, BinaryOperationFeedback::kNumber);
+ Goto(&done);
+
+ BIND(&if_bigint);
+ *var_numeric = CAST(value);
+ OverwriteFeedback(var_feedback, BinaryOperationFeedback::kBigInt);
+ Goto(&done);
+
+ BIND(&if_oddball);
+ OverwriteFeedback(var_feedback, BinaryOperationFeedback::kNumberOrOddball);
+ *var_numeric =
+ CAST(LoadObjectField(heap_object_value, Oddball::kToNumberOffset));
+ Goto(&done);
+
+ Bind(&done);
+}
+
+// ES#sec-touint32
+TNode<Number> CodeStubAssembler::ToUint32(TNode<Context> context,
+ SloppyTNode<Object> input) {
+ const TNode<Float64T> float_zero = Float64Constant(0.0);
+ const TNode<Float64T> float_two_32 =
+ Float64Constant(static_cast<double>(1ULL << 32));
+
+ Label out(this);
+
+ TVARIABLE(Object, var_result, input);
+
+ // Early exit for positive smis.
+ {
+ // TODO(jgruber): This branch and the recheck below can be removed once we
+ // have a ToNumber with multiple exits.
+ Label next(this, Label::kDeferred);
+ Branch(TaggedIsPositiveSmi(input), &out, &next);
+ BIND(&next);
+ }
+
+ const TNode<Number> number = ToNumber(context, input);
+ var_result = number;
+
+ // Perhaps we have a positive smi now.
+ {
+ Label next(this, Label::kDeferred);
+ Branch(TaggedIsPositiveSmi(number), &out, &next);
+ BIND(&next);
+ }
+
+ Label if_isnegativesmi(this), if_isheapnumber(this);
+ Branch(TaggedIsSmi(number), &if_isnegativesmi, &if_isheapnumber);
+
+ BIND(&if_isnegativesmi);
+ {
+ const TNode<Int32T> uint32_value = SmiToInt32(CAST(number));
+ TNode<Float64T> float64_value = ChangeUint32ToFloat64(uint32_value);
+ var_result = AllocateHeapNumberWithValue(float64_value);
+ Goto(&out);
+ }
+
+ BIND(&if_isheapnumber);
+ {
+ Label return_zero(this);
+ const TNode<Float64T> value = LoadHeapNumberValue(CAST(number));
+
+ {
+ // +-0.
+ Label next(this);
+ Branch(Float64Equal(value, float_zero), &return_zero, &next);
+ BIND(&next);
+ }
+
+ {
+ // NaN.
+ Label next(this);
+ Branch(Float64Equal(value, value), &next, &return_zero);
+ BIND(&next);
+ }
+
+ {
+ // +Infinity.
+ Label next(this);
+ const TNode<Float64T> positive_infinity =
+ Float64Constant(std::numeric_limits<double>::infinity());
+ Branch(Float64Equal(value, positive_infinity), &return_zero, &next);
+ BIND(&next);
+ }
+
+ {
+ // -Infinity.
+ Label next(this);
+ const TNode<Float64T> negative_infinity =
+ Float64Constant(-1.0 * std::numeric_limits<double>::infinity());
+ Branch(Float64Equal(value, negative_infinity), &return_zero, &next);
+ BIND(&next);
+ }
+
+ // * Let int be the mathematical value that is the same sign as number and
+ // whose magnitude is floor(abs(number)).
+ // * Let int32bit be int modulo 2^32.
+ // * Return int32bit.
+ {
+ TNode<Float64T> x = Float64Trunc(value);
+ x = Float64Mod(x, float_two_32);
+ x = Float64Add(x, float_two_32);
+ x = Float64Mod(x, float_two_32);
+
+ const TNode<Number> result = ChangeFloat64ToTagged(x);
+ var_result = result;
+ Goto(&out);
+ }
+
+ BIND(&return_zero);
+ {
+ var_result = SmiConstant(0);
+ Goto(&out);
+ }
+ }
+
+ BIND(&out);
+ return CAST(var_result.value());
+}
+
+TNode<String> CodeStubAssembler::ToString_Inline(TNode<Context> context,
+ SloppyTNode<Object> input) {
+ TVARIABLE(Object, var_result, input);
+ Label stub_call(this, Label::kDeferred), out(this);
+
+ GotoIf(TaggedIsSmi(input), &stub_call);
+ Branch(IsString(CAST(input)), &out, &stub_call);
+
+ BIND(&stub_call);
+ var_result = CallBuiltin(Builtins::kToString, context, input);
+ Goto(&out);
+
+ BIND(&out);
+ return CAST(var_result.value());
+}
+
+TNode<JSReceiver> CodeStubAssembler::ToObject(TNode<Context> context,
+ SloppyTNode<Object> input) {
+ return CAST(CallBuiltin(Builtins::kToObject, context, input));
+}
+
+TNode<JSReceiver> CodeStubAssembler::ToObject_Inline(TNode<Context> context,
+ TNode<Object> input) {
+ TVARIABLE(JSReceiver, result);
+ Label if_isreceiver(this), if_isnotreceiver(this, Label::kDeferred);
+ Label done(this);
+
+ BranchIfJSReceiver(input, &if_isreceiver, &if_isnotreceiver);
+
+ BIND(&if_isreceiver);
+ {
+ result = CAST(input);
+ Goto(&done);
+ }
+
+ BIND(&if_isnotreceiver);
+ {
+ result = ToObject(context, input);
+ Goto(&done);
+ }
+
+ BIND(&done);
+ return result.value();
+}
+
+TNode<Number> CodeStubAssembler::ToLength_Inline(TNode<Context> context,
+ SloppyTNode<Object> input) {
+ TNode<Smi> smi_zero = SmiConstant(0);
+ return Select<Number>(
+ TaggedIsSmi(input), [=] { return SmiMax(CAST(input), smi_zero); },
+ [=] { return CAST(CallBuiltin(Builtins::kToLength, context, input)); });
+}
+
+TNode<Object> CodeStubAssembler::OrdinaryToPrimitive(
+ TNode<Context> context, TNode<Object> input, OrdinaryToPrimitiveHint hint) {
+ Callable callable = CodeFactory::OrdinaryToPrimitive(isolate(), hint);
+ return CallStub(callable, context, input);
+}
+
+TNode<Uint32T> CodeStubAssembler::DecodeWord32(TNode<Word32T> word32,
+ uint32_t shift, uint32_t mask) {
+ DCHECK_EQ((mask >> shift) << shift, mask);
+ return Unsigned(Word32And(Word32Shr(word32, static_cast<int>(shift)),
+ Int32Constant(mask >> shift)));
+}
+
+TNode<UintPtrT> CodeStubAssembler::DecodeWord(SloppyTNode<WordT> word,
+ uint32_t shift, uintptr_t mask) {
+ DCHECK_EQ((mask >> shift) << shift, mask);
+ return Unsigned(WordAnd(WordShr(word, static_cast<int>(shift)),
+ IntPtrConstant(mask >> shift)));
+}
+
+TNode<Word32T> CodeStubAssembler::UpdateWord32(TNode<Word32T> word,
+ TNode<Uint32T> value,
+ uint32_t shift, uint32_t mask,
+ bool starts_as_zero) {
+ DCHECK_EQ((mask >> shift) << shift, mask);
+ // Ensure the {value} fits fully in the mask.
+ CSA_ASSERT(this, Uint32LessThanOrEqual(value, Uint32Constant(mask >> shift)));
+ TNode<Word32T> encoded_value = Word32Shl(value, Int32Constant(shift));
+ TNode<Word32T> masked_word;
+ if (starts_as_zero) {
+ CSA_ASSERT(this, Word32Equal(Word32And(word, Int32Constant(~mask)), word));
+ masked_word = word;
+ } else {
+ masked_word = Word32And(word, Int32Constant(~mask));
+ }
+ return Word32Or(masked_word, encoded_value);
+}
+
+TNode<WordT> CodeStubAssembler::UpdateWord(TNode<WordT> word,
+ TNode<UintPtrT> value,
+ uint32_t shift, uintptr_t mask,
+ bool starts_as_zero) {
+ DCHECK_EQ((mask >> shift) << shift, mask);
+ // Ensure the {value} fits fully in the mask.
+ CSA_ASSERT(this,
+ UintPtrLessThanOrEqual(value, UintPtrConstant(mask >> shift)));
+ TNode<WordT> encoded_value = WordShl(value, static_cast<int>(shift));
+ TNode<WordT> masked_word;
+ if (starts_as_zero) {
+ CSA_ASSERT(this, WordEqual(WordAnd(word, UintPtrConstant(~mask)), word));
+ masked_word = word;
+ } else {
+ masked_word = WordAnd(word, UintPtrConstant(~mask));
+ }
+ return WordOr(masked_word, encoded_value);
+}
+
+void CodeStubAssembler::SetCounter(StatsCounter* counter, int value) {
+ if (FLAG_native_code_counters && counter->Enabled()) {
+ TNode<ExternalReference> counter_address =
+ ExternalConstant(ExternalReference::Create(counter));
+ StoreNoWriteBarrier(MachineRepresentation::kWord32, counter_address,
+ Int32Constant(value));
+ }
+}
+
+void CodeStubAssembler::IncrementCounter(StatsCounter* counter, int delta) {
+ DCHECK_GT(delta, 0);
+ if (FLAG_native_code_counters && counter->Enabled()) {
+ TNode<ExternalReference> counter_address =
+ ExternalConstant(ExternalReference::Create(counter));
+ // This operation has to be exactly 32-bit wide in case the external
+ // reference table redirects the counter to a uint32_t dummy_stats_counter_
+ // field.
+ TNode<Int32T> value = Load<Int32T>(counter_address);
+ value = Int32Add(value, Int32Constant(delta));
+ StoreNoWriteBarrier(MachineRepresentation::kWord32, counter_address, value);
+ }
+}
+
+void CodeStubAssembler::DecrementCounter(StatsCounter* counter, int delta) {
+ DCHECK_GT(delta, 0);
+ if (FLAG_native_code_counters && counter->Enabled()) {
+ TNode<ExternalReference> counter_address =
+ ExternalConstant(ExternalReference::Create(counter));
+ // This operation has to be exactly 32-bit wide in case the external
+ // reference table redirects the counter to a uint32_t dummy_stats_counter_
+ // field.
+ TNode<Int32T> value = Load<Int32T>(counter_address);
+ value = Int32Sub(value, Int32Constant(delta));
+ StoreNoWriteBarrier(MachineRepresentation::kWord32, counter_address, value);
+ }
+}
+
+template <typename TIndex>
+void CodeStubAssembler::Increment(TVariable<TIndex>* variable, int value) {
+ *variable =
+ IntPtrOrSmiAdd(variable->value(), IntPtrOrSmiConstant<TIndex>(value));
+}
+
+// Instantiate Increment for Smi and IntPtrT.
+// TODO(v8:9708): Consider renaming to [Smi|IntPtrT|RawPtrT]Increment.
+template void CodeStubAssembler::Increment<Smi>(TVariable<Smi>* variable,
+ int value);
+template void CodeStubAssembler::Increment<IntPtrT>(
+ TVariable<IntPtrT>* variable, int value);
+template void CodeStubAssembler::Increment<RawPtrT>(
+ TVariable<RawPtrT>* variable, int value);
+
+void CodeStubAssembler::Use(Label* label) {
+ GotoIf(Word32Equal(Int32Constant(0), Int32Constant(1)), label);
+}
+
+void CodeStubAssembler::TryToName(SloppyTNode<Object> key, Label* if_keyisindex,
+ TVariable<IntPtrT>* var_index,
+ Label* if_keyisunique,
+ TVariable<Name>* var_unique,
+ Label* if_bailout,
+ Label* if_notinternalized) {
+ Comment("TryToName");
+
+ TVARIABLE(Int32T, var_instance_type);
+ Label if_keyisnotindex(this);
+ *var_index = TryToIntptr(key, &if_keyisnotindex, &var_instance_type);
+ Goto(if_keyisindex);
+
+ BIND(&if_keyisnotindex);
+ {
+ Label if_symbol(this), if_string(this),
+ if_keyisother(this, Label::kDeferred);
+
+ // Symbols are unique.
+ GotoIf(IsSymbolInstanceType(var_instance_type.value()), &if_symbol);
+
+ // Miss if |key| is not a String.
+ STATIC_ASSERT(FIRST_NAME_TYPE == FIRST_TYPE);
+ Branch(IsStringInstanceType(var_instance_type.value()), &if_string,
+ &if_keyisother);
+
+ // Symbols are unique.
+ BIND(&if_symbol);
+ {
+ *var_unique = CAST(key);
+ Goto(if_keyisunique);
+ }
+
+ BIND(&if_string);
+ {
+ Label if_thinstring(this), if_has_cached_index(this);
+
+ TNode<Uint32T> hash = LoadNameHashField(CAST(key));
+ GotoIf(IsClearWord32(hash, Name::kDoesNotContainCachedArrayIndexMask),
+ &if_has_cached_index);
+ // No cached array index. If the string knows that it contains an index,
+ // then it must be an uncacheable index. Handle this case in the runtime.
+ GotoIf(IsClearWord32(hash, Name::kIsNotIntegerIndexMask), if_bailout);
+
+ GotoIf(InstanceTypeEqual(var_instance_type.value(), THIN_STRING_TYPE),
+ &if_thinstring);
+ GotoIf(InstanceTypeEqual(var_instance_type.value(),
+ THIN_ONE_BYTE_STRING_TYPE),
+ &if_thinstring);
+ // Finally, check if |key| is internalized.
+ STATIC_ASSERT(kNotInternalizedTag != 0);
+ GotoIf(IsSetWord32(var_instance_type.value(), kIsNotInternalizedMask),
+ if_notinternalized != nullptr ? if_notinternalized : if_bailout);
+
+ *var_unique = CAST(key);
+ Goto(if_keyisunique);
+
+ BIND(&if_thinstring);
+ {
+ *var_unique =
+ LoadObjectField<String>(CAST(key), ThinString::kActualOffset);
+ Goto(if_keyisunique);
+ }
+
+ BIND(&if_has_cached_index);
+ {
+ TNode<IntPtrT> index =
+ Signed(DecodeWordFromWord32<String::ArrayIndexValueBits>(hash));
+ CSA_ASSERT(this, IntPtrLessThan(index, IntPtrConstant(INT_MAX)));
+ *var_index = index;
+ Goto(if_keyisindex);
+ }
+ }
+
+ BIND(&if_keyisother);
+ {
+ GotoIfNot(InstanceTypeEqual(var_instance_type.value(), ODDBALL_TYPE),
+ if_bailout);
+ *var_unique =
+ LoadObjectField<String>(CAST(key), Oddball::kToStringOffset);
+ Goto(if_keyisunique);
+ }
+ }
+}
+
+void CodeStubAssembler::TryInternalizeString(
+ TNode<String> string, Label* if_index, TVariable<IntPtrT>* var_index,
+ Label* if_internalized, TVariable<Name>* var_internalized,
+ Label* if_not_internalized, Label* if_bailout) {
+ TNode<ExternalReference> function = ExternalConstant(
+ ExternalReference::try_string_to_index_or_lookup_existing());
+ const TNode<ExternalReference> isolate_ptr =
+ ExternalConstant(ExternalReference::isolate_address(isolate()));
+ TNode<Object> result =
+ CAST(CallCFunction(function, MachineType::AnyTagged(),
+ std::make_pair(MachineType::Pointer(), isolate_ptr),
+ std::make_pair(MachineType::AnyTagged(), string)));
+ Label internalized(this);
+ GotoIf(TaggedIsNotSmi(result), &internalized);
+ TNode<IntPtrT> word_result = SmiUntag(CAST(result));
+ GotoIf(IntPtrEqual(word_result, IntPtrConstant(ResultSentinel::kNotFound)),
+ if_not_internalized);
+ GotoIf(IntPtrEqual(word_result, IntPtrConstant(ResultSentinel::kUnsupported)),
+ if_bailout);
+ *var_index = word_result;
+ Goto(if_index);
+
+ BIND(&internalized);
+ *var_internalized = CAST(result);
+ Goto(if_internalized);
+}
+
+template <typename Dictionary>
+TNode<IntPtrT> CodeStubAssembler::EntryToIndex(TNode<IntPtrT> entry,
+ int field_index) {
+ TNode<IntPtrT> entry_index =
+ IntPtrMul(entry, IntPtrConstant(Dictionary::kEntrySize));
+ return IntPtrAdd(entry_index, IntPtrConstant(Dictionary::kElementsStartIndex +
+ field_index));
+}
+
+template <typename T>
+TNode<T> CodeStubAssembler::LoadDescriptorArrayElement(
+ TNode<DescriptorArray> object, TNode<IntPtrT> index,
+ int additional_offset) {
+ return LoadArrayElement<DescriptorArray, IntPtrT, T>(
+ object, DescriptorArray::kHeaderSize, index, additional_offset);
+}
+
+TNode<Name> CodeStubAssembler::LoadKeyByKeyIndex(
+ TNode<DescriptorArray> container, TNode<IntPtrT> key_index) {
+ return CAST(LoadDescriptorArrayElement<HeapObject>(container, key_index, 0));
+}
+
+TNode<Uint32T> CodeStubAssembler::LoadDetailsByKeyIndex(
+ TNode<DescriptorArray> container, TNode<IntPtrT> key_index) {
+ const int kKeyToDetailsOffset =
+ DescriptorArray::kEntryDetailsOffset - DescriptorArray::kEntryKeyOffset;
+ return Unsigned(LoadAndUntagToWord32ArrayElement(
+ container, DescriptorArray::kHeaderSize, key_index, kKeyToDetailsOffset));
+}
+
+TNode<Object> CodeStubAssembler::LoadValueByKeyIndex(
+ TNode<DescriptorArray> container, TNode<IntPtrT> key_index) {
+ const int kKeyToValueOffset =
+ DescriptorArray::kEntryValueOffset - DescriptorArray::kEntryKeyOffset;
+ return LoadDescriptorArrayElement<Object>(container, key_index,
+ kKeyToValueOffset);
+}
+
+TNode<MaybeObject> CodeStubAssembler::LoadFieldTypeByKeyIndex(
+ TNode<DescriptorArray> container, TNode<IntPtrT> key_index) {
+ const int kKeyToValueOffset =
+ DescriptorArray::kEntryValueOffset - DescriptorArray::kEntryKeyOffset;
+ return LoadDescriptorArrayElement<MaybeObject>(container, key_index,
+ kKeyToValueOffset);
+}
+
+TNode<IntPtrT> CodeStubAssembler::DescriptorEntryToIndex(
+ TNode<IntPtrT> descriptor_entry) {
+ return IntPtrMul(descriptor_entry,
+ IntPtrConstant(DescriptorArray::kEntrySize));
+}
+
+TNode<Name> CodeStubAssembler::LoadKeyByDescriptorEntry(
+ TNode<DescriptorArray> container, TNode<IntPtrT> descriptor_entry) {
+ return CAST(LoadDescriptorArrayElement<HeapObject>(
+ container, DescriptorEntryToIndex(descriptor_entry),
+ DescriptorArray::ToKeyIndex(0) * kTaggedSize));
+}
+
+TNode<Name> CodeStubAssembler::LoadKeyByDescriptorEntry(
+ TNode<DescriptorArray> container, int descriptor_entry) {
+ return CAST(LoadDescriptorArrayElement<HeapObject>(
+ container, IntPtrConstant(0),
+ DescriptorArray::ToKeyIndex(descriptor_entry) * kTaggedSize));
+}
+
+TNode<Uint32T> CodeStubAssembler::LoadDetailsByDescriptorEntry(
+ TNode<DescriptorArray> container, TNode<IntPtrT> descriptor_entry) {
+ return Unsigned(LoadAndUntagToWord32ArrayElement(
+ container, DescriptorArray::kHeaderSize,
+ DescriptorEntryToIndex(descriptor_entry),
+ DescriptorArray::ToDetailsIndex(0) * kTaggedSize));
+}
+
+TNode<Uint32T> CodeStubAssembler::LoadDetailsByDescriptorEntry(
+ TNode<DescriptorArray> container, int descriptor_entry) {
+ return Unsigned(LoadAndUntagToWord32ArrayElement(
+ container, DescriptorArray::kHeaderSize, IntPtrConstant(0),
+ DescriptorArray::ToDetailsIndex(descriptor_entry) * kTaggedSize));
+}
+
+TNode<Object> CodeStubAssembler::LoadValueByDescriptorEntry(
+ TNode<DescriptorArray> container, int descriptor_entry) {
+ return LoadDescriptorArrayElement<Object>(
+ container, IntPtrConstant(0),
+ DescriptorArray::ToValueIndex(descriptor_entry) * kTaggedSize);
+}
+
+TNode<MaybeObject> CodeStubAssembler::LoadFieldTypeByDescriptorEntry(
+ TNode<DescriptorArray> container, TNode<IntPtrT> descriptor_entry) {
+ return LoadDescriptorArrayElement<MaybeObject>(
+ container, DescriptorEntryToIndex(descriptor_entry),
+ DescriptorArray::ToValueIndex(0) * kTaggedSize);
+}
+
+template V8_EXPORT_PRIVATE TNode<IntPtrT>
+CodeStubAssembler::EntryToIndex<NameDictionary>(TNode<IntPtrT>, int);
+template V8_EXPORT_PRIVATE TNode<IntPtrT>
+CodeStubAssembler::EntryToIndex<GlobalDictionary>(TNode<IntPtrT>, int);
+template V8_EXPORT_PRIVATE TNode<IntPtrT>
+CodeStubAssembler::EntryToIndex<NumberDictionary>(TNode<IntPtrT>, int);
+
+// This must be kept in sync with HashTableBase::ComputeCapacity().
+TNode<IntPtrT> CodeStubAssembler::HashTableComputeCapacity(
+ TNode<IntPtrT> at_least_space_for) {
+ TNode<IntPtrT> capacity = IntPtrRoundUpToPowerOfTwo32(
+ IntPtrAdd(at_least_space_for, WordShr(at_least_space_for, 1)));
+ return IntPtrMax(capacity, IntPtrConstant(HashTableBase::kMinCapacity));
+}
+
+TNode<IntPtrT> CodeStubAssembler::IntPtrMax(SloppyTNode<IntPtrT> left,
+ SloppyTNode<IntPtrT> right) {
+ intptr_t left_constant;
+ intptr_t right_constant;
+ if (ToIntPtrConstant(left, &left_constant) &&
+ ToIntPtrConstant(right, &right_constant)) {
+ return IntPtrConstant(std::max(left_constant, right_constant));
+ }
+ return SelectConstant<IntPtrT>(IntPtrGreaterThanOrEqual(left, right), left,
+ right);
+}
+
+TNode<IntPtrT> CodeStubAssembler::IntPtrMin(SloppyTNode<IntPtrT> left,
+ SloppyTNode<IntPtrT> right) {
+ intptr_t left_constant;
+ intptr_t right_constant;
+ if (ToIntPtrConstant(left, &left_constant) &&
+ ToIntPtrConstant(right, &right_constant)) {
+ return IntPtrConstant(std::min(left_constant, right_constant));
+ }
+ return SelectConstant<IntPtrT>(IntPtrLessThanOrEqual(left, right), left,
+ right);
+}
+
+TNode<UintPtrT> CodeStubAssembler::UintPtrMin(TNode<UintPtrT> left,
+ TNode<UintPtrT> right) {
+ intptr_t left_constant;
+ intptr_t right_constant;
+ if (ToIntPtrConstant(left, &left_constant) &&
+ ToIntPtrConstant(right, &right_constant)) {
+ return UintPtrConstant(std::min(static_cast<uintptr_t>(left_constant),
+ static_cast<uintptr_t>(right_constant)));
+ }
+ return SelectConstant<UintPtrT>(UintPtrLessThanOrEqual(left, right), left,
+ right);
+}
+
+template <>
+TNode<HeapObject> CodeStubAssembler::LoadName<NameDictionary>(
+ TNode<HeapObject> key) {
+ CSA_ASSERT(this, Word32Or(IsTheHole(key), IsName(key)));
+ return key;
+}
+
+template <>
+TNode<HeapObject> CodeStubAssembler::LoadName<GlobalDictionary>(
+ TNode<HeapObject> key) {
+ TNode<PropertyCell> property_cell = CAST(key);
+ return CAST(LoadObjectField(property_cell, PropertyCell::kNameOffset));
+}
+
+template <typename Dictionary>
+void CodeStubAssembler::NameDictionaryLookup(
+ TNode<Dictionary> dictionary, TNode<Name> unique_name, Label* if_found,
+ TVariable<IntPtrT>* var_name_index, Label* if_not_found, LookupMode mode) {
+ static_assert(std::is_same<Dictionary, NameDictionary>::value ||
+ std::is_same<Dictionary, GlobalDictionary>::value,
+ "Unexpected NameDictionary");
+ DCHECK_EQ(MachineType::PointerRepresentation(), var_name_index->rep());
+ DCHECK_IMPLIES(mode == kFindInsertionIndex, if_found == nullptr);
+ Comment("NameDictionaryLookup");
+ CSA_ASSERT(this, IsUniqueName(unique_name));
+
+ TNode<IntPtrT> capacity = SmiUntag(GetCapacity<Dictionary>(dictionary));
+ TNode<IntPtrT> mask = IntPtrSub(capacity, IntPtrConstant(1));
+ TNode<UintPtrT> hash = ChangeUint32ToWord(LoadNameHash(unique_name));
+
+ // See Dictionary::FirstProbe().
+ TNode<IntPtrT> count = IntPtrConstant(0);
+ TNode<IntPtrT> entry = Signed(WordAnd(hash, mask));
+ TNode<Oddball> undefined = UndefinedConstant();
+
+ // Appease the variable merging algorithm for "Goto(&loop)" below.
+ *var_name_index = IntPtrConstant(0);
+
+ TVARIABLE(IntPtrT, var_count, count);
+ TVARIABLE(IntPtrT, var_entry, entry);
+ Label loop(this, {&var_count, &var_entry, var_name_index});
+ Goto(&loop);
+ BIND(&loop);
+ {
+ Label next_probe(this);
+ TNode<IntPtrT> entry = var_entry.value();
+
+ TNode<IntPtrT> index = EntryToIndex<Dictionary>(entry);
+ *var_name_index = index;
+
+ TNode<HeapObject> current =
+ CAST(UnsafeLoadFixedArrayElement(dictionary, index));
+ GotoIf(TaggedEqual(current, undefined), if_not_found);
+ if (mode == kFindExisting) {
+ if (Dictionary::ShapeT::kMatchNeedsHoleCheck) {
+ GotoIf(TaggedEqual(current, TheHoleConstant()), &next_probe);
+ }
+ current = LoadName<Dictionary>(current);
+ GotoIf(TaggedEqual(current, unique_name), if_found);
+ } else {
+ DCHECK_EQ(kFindInsertionIndex, mode);
+ GotoIf(TaggedEqual(current, TheHoleConstant()), if_not_found);
+ }
+ Goto(&next_probe);
+
+ BIND(&next_probe);
+ // See Dictionary::NextProbe().
+ Increment(&var_count);
+ entry = Signed(WordAnd(IntPtrAdd(entry, var_count.value()), mask));
+
+ var_entry = entry;
+ Goto(&loop);
+ }
+}
+
+// Instantiate template methods to workaround GCC compilation issue.
+template V8_EXPORT_PRIVATE void
+CodeStubAssembler::NameDictionaryLookup<NameDictionary>(TNode<NameDictionary>,
+ TNode<Name>, Label*,
+ TVariable<IntPtrT>*,
+ Label*, LookupMode);
+template V8_EXPORT_PRIVATE void CodeStubAssembler::NameDictionaryLookup<
+ GlobalDictionary>(TNode<GlobalDictionary>, TNode<Name>, Label*,
+ TVariable<IntPtrT>*, Label*, LookupMode);
+
+TNode<Word32T> CodeStubAssembler::ComputeSeededHash(TNode<IntPtrT> key) {
+ const TNode<ExternalReference> function_addr =
+ ExternalConstant(ExternalReference::compute_integer_hash());
+ const TNode<ExternalReference> isolate_ptr =
+ ExternalConstant(ExternalReference::isolate_address(isolate()));
+
+ MachineType type_ptr = MachineType::Pointer();
+ MachineType type_uint32 = MachineType::Uint32();
+ MachineType type_int32 = MachineType::Int32();
+
+ return UncheckedCast<Word32T>(CallCFunction(
+ function_addr, type_uint32, std::make_pair(type_ptr, isolate_ptr),
+ std::make_pair(type_int32, TruncateIntPtrToInt32(key))));
+}
+
+void CodeStubAssembler::NumberDictionaryLookup(
+ TNode<NumberDictionary> dictionary, TNode<IntPtrT> intptr_index,
+ Label* if_found, TVariable<IntPtrT>* var_entry, Label* if_not_found) {
+ CSA_ASSERT(this, IsNumberDictionary(dictionary));
+ DCHECK_EQ(MachineType::PointerRepresentation(), var_entry->rep());
+ Comment("NumberDictionaryLookup");
+
+ TNode<IntPtrT> capacity = SmiUntag(GetCapacity<NumberDictionary>(dictionary));
+ TNode<IntPtrT> mask = IntPtrSub(capacity, IntPtrConstant(1));
+
+ TNode<UintPtrT> hash = ChangeUint32ToWord(ComputeSeededHash(intptr_index));
+ TNode<Float64T> key_as_float64 = RoundIntPtrToFloat64(intptr_index);
+
+ // See Dictionary::FirstProbe().
+ TNode<IntPtrT> count = IntPtrConstant(0);
+ TNode<IntPtrT> entry = Signed(WordAnd(hash, mask));
+
+ TNode<Oddball> undefined = UndefinedConstant();
+ TNode<Oddball> the_hole = TheHoleConstant();
+
+ TVARIABLE(IntPtrT, var_count, count);
+ Label loop(this, {&var_count, var_entry});
+ *var_entry = entry;
+ Goto(&loop);
+ BIND(&loop);
+ {
+ TNode<IntPtrT> entry = var_entry->value();
+
+ TNode<IntPtrT> index = EntryToIndex<NumberDictionary>(entry);
+ TNode<Object> current = UnsafeLoadFixedArrayElement(dictionary, index);
+ GotoIf(TaggedEqual(current, undefined), if_not_found);
+ Label next_probe(this);
+ {
+ Label if_currentissmi(this), if_currentisnotsmi(this);
+ Branch(TaggedIsSmi(current), &if_currentissmi, &if_currentisnotsmi);
+ BIND(&if_currentissmi);
+ {
+ TNode<IntPtrT> current_value = SmiUntag(CAST(current));
+ Branch(WordEqual(current_value, intptr_index), if_found, &next_probe);
+ }
+ BIND(&if_currentisnotsmi);
+ {
+ GotoIf(TaggedEqual(current, the_hole), &next_probe);
+ // Current must be the Number.
+ TNode<Float64T> current_value = LoadHeapNumberValue(CAST(current));
+ Branch(Float64Equal(current_value, key_as_float64), if_found,
+ &next_probe);
+ }
+ }
+
+ BIND(&next_probe);
+ // See Dictionary::NextProbe().
+ Increment(&var_count);
+ entry = Signed(WordAnd(IntPtrAdd(entry, var_count.value()), mask));
+
+ *var_entry = entry;
+ Goto(&loop);
+ }
+}
+
+TNode<Object> CodeStubAssembler::BasicLoadNumberDictionaryElement(
+ TNode<NumberDictionary> dictionary, TNode<IntPtrT> intptr_index,
+ Label* not_data, Label* if_hole) {
+ TVARIABLE(IntPtrT, var_entry);
+ Label if_found(this);
+ NumberDictionaryLookup(dictionary, intptr_index, &if_found, &var_entry,
+ if_hole);
+ BIND(&if_found);
+
+ // Check that the value is a data property.
+ TNode<IntPtrT> index = EntryToIndex<NumberDictionary>(var_entry.value());
+ TNode<Uint32T> details = LoadDetailsByKeyIndex(dictionary, index);
+ TNode<Uint32T> kind = DecodeWord32<PropertyDetails::KindField>(details);
+ // TODO(jkummerow): Support accessors without missing?
+ GotoIfNot(Word32Equal(kind, Int32Constant(kData)), not_data);
+ // Finally, load the value.
+ return LoadValueByKeyIndex(dictionary, index);
+}
+
+template <class Dictionary>
+void CodeStubAssembler::FindInsertionEntry(TNode<Dictionary> dictionary,
+ TNode<Name> key,
+ TVariable<IntPtrT>* var_key_index) {
+ UNREACHABLE();
+}
+
+template <>
+void CodeStubAssembler::FindInsertionEntry<NameDictionary>(
+ TNode<NameDictionary> dictionary, TNode<Name> key,
+ TVariable<IntPtrT>* var_key_index) {
+ Label done(this);
+ NameDictionaryLookup<NameDictionary>(dictionary, key, nullptr, var_key_index,
+ &done, kFindInsertionIndex);
+ BIND(&done);
+}
+
+template <class Dictionary>
+void CodeStubAssembler::InsertEntry(TNode<Dictionary> dictionary,
+ TNode<Name> key, TNode<Object> value,
+ TNode<IntPtrT> index,
+ TNode<Smi> enum_index) {
+ UNREACHABLE(); // Use specializations instead.
+}
+
+template <>
+void CodeStubAssembler::InsertEntry<NameDictionary>(
+ TNode<NameDictionary> dictionary, TNode<Name> name, TNode<Object> value,
+ TNode<IntPtrT> index, TNode<Smi> enum_index) {
+ // Store name and value.
+ StoreFixedArrayElement(dictionary, index, name);
+ StoreValueByKeyIndex<NameDictionary>(dictionary, index, value);
+
+ // Prepare details of the new property.
+ PropertyDetails d(kData, NONE, PropertyCellType::kNoCell);
+ enum_index =
+ SmiShl(enum_index, PropertyDetails::DictionaryStorageField::kShift);
+ // We OR over the actual index below, so we expect the initial value to be 0.
+ DCHECK_EQ(0, d.dictionary_index());
+ TVARIABLE(Smi, var_details, SmiOr(SmiConstant(d.AsSmi()), enum_index));
+
+ // Private names must be marked non-enumerable.
+ Label not_private(this, &var_details);
+ GotoIfNot(IsPrivateSymbol(name), ¬_private);
+ TNode<Smi> dont_enum =
+ SmiShl(SmiConstant(DONT_ENUM), PropertyDetails::AttributesField::kShift);
+ var_details = SmiOr(var_details.value(), dont_enum);
+ Goto(¬_private);
+ BIND(¬_private);
+
+ // Finally, store the details.
+ StoreDetailsByKeyIndex<NameDictionary>(dictionary, index,
+ var_details.value());
+}
+
+template <>
+void CodeStubAssembler::InsertEntry<GlobalDictionary>(
+ TNode<GlobalDictionary> dictionary, TNode<Name> key, TNode<Object> value,
+ TNode<IntPtrT> index, TNode<Smi> enum_index) {
+ UNIMPLEMENTED();
+}
+
+template <class Dictionary>
+void CodeStubAssembler::Add(TNode<Dictionary> dictionary, TNode<Name> key,
+ TNode<Object> value, Label* bailout) {
+ CSA_ASSERT(this, Word32BinaryNot(IsEmptyPropertyDictionary(dictionary)));
+ TNode<Smi> capacity = GetCapacity<Dictionary>(dictionary);
+ TNode<Smi> nof = GetNumberOfElements<Dictionary>(dictionary);
+ TNode<Smi> new_nof = SmiAdd(nof, SmiConstant(1));
+ // Require 33% to still be free after adding additional_elements.
+ // Computing "x + (x >> 1)" on a Smi x does not return a valid Smi!
+ // But that's OK here because it's only used for a comparison.
+ TNode<Smi> required_capacity_pseudo_smi = SmiAdd(new_nof, SmiShr(new_nof, 1));
+ GotoIf(SmiBelow(capacity, required_capacity_pseudo_smi), bailout);
+ // Require rehashing if more than 50% of free elements are deleted elements.
+ TNode<Smi> deleted = GetNumberOfDeletedElements<Dictionary>(dictionary);
+ CSA_ASSERT(this, SmiAbove(capacity, new_nof));
+ TNode<Smi> half_of_free_elements = SmiShr(SmiSub(capacity, new_nof), 1);
+ GotoIf(SmiAbove(deleted, half_of_free_elements), bailout);
+
+ TNode<Smi> enum_index = GetNextEnumerationIndex<Dictionary>(dictionary);
+ TNode<Smi> new_enum_index = SmiAdd(enum_index, SmiConstant(1));
+ TNode<Smi> max_enum_index =
+ SmiConstant(PropertyDetails::DictionaryStorageField::kMax);
+ GotoIf(SmiAbove(new_enum_index, max_enum_index), bailout);
+
+ // No more bailouts after this point.
+ // Operations from here on can have side effects.
+
+ SetNextEnumerationIndex<Dictionary>(dictionary, new_enum_index);
+ SetNumberOfElements<Dictionary>(dictionary, new_nof);
+
+ TVARIABLE(IntPtrT, var_key_index);
+ FindInsertionEntry<Dictionary>(dictionary, key, &var_key_index);
+ InsertEntry<Dictionary>(dictionary, key, value, var_key_index.value(),
+ enum_index);
+}
+
+template void CodeStubAssembler::Add<NameDictionary>(TNode<NameDictionary>,
+ TNode<Name>, TNode<Object>,
+ Label*);
+
+template <typename Array>
+void CodeStubAssembler::LookupLinear(TNode<Name> unique_name,
+ TNode<Array> array,
+ TNode<Uint32T> number_of_valid_entries,
+ Label* if_found,
+ TVariable<IntPtrT>* var_name_index,
+ Label* if_not_found) {
+ static_assert(std::is_base_of<FixedArray, Array>::value ||
+ std::is_base_of<WeakFixedArray, Array>::value ||
+ std::is_base_of<DescriptorArray, Array>::value,
+ "T must be a descendant of FixedArray or a WeakFixedArray");
+ Comment("LookupLinear");
+ CSA_ASSERT(this, IsUniqueName(unique_name));
+ TNode<IntPtrT> first_inclusive = IntPtrConstant(Array::ToKeyIndex(0));
+ TNode<IntPtrT> factor = IntPtrConstant(Array::kEntrySize);
+ TNode<IntPtrT> last_exclusive = IntPtrAdd(
+ first_inclusive,
+ IntPtrMul(ChangeInt32ToIntPtr(number_of_valid_entries), factor));
+
+ BuildFastLoop<IntPtrT>(
+ last_exclusive, first_inclusive,
+ [=](TNode<IntPtrT> name_index) {
+ TNode<MaybeObject> element =
+ LoadArrayElement(array, Array::kHeaderSize, name_index);
+ TNode<Name> candidate_name = CAST(element);
+ *var_name_index = name_index;
+ GotoIf(TaggedEqual(candidate_name, unique_name), if_found);
+ },
+ -Array::kEntrySize, IndexAdvanceMode::kPre);
+ Goto(if_not_found);
+}
+
+template <>
+TNode<Uint32T> CodeStubAssembler::NumberOfEntries<DescriptorArray>(
+ TNode<DescriptorArray> descriptors) {
+ return Unsigned(LoadNumberOfDescriptors(descriptors));
+}
+
+template <>
+TNode<Uint32T> CodeStubAssembler::NumberOfEntries<TransitionArray>(
+ TNode<TransitionArray> transitions) {
+ TNode<IntPtrT> length = LoadAndUntagWeakFixedArrayLength(transitions);
+ return Select<Uint32T>(
+ UintPtrLessThan(length, IntPtrConstant(TransitionArray::kFirstIndex)),
+ [=] { return Unsigned(Int32Constant(0)); },
+ [=] {
+ return Unsigned(LoadAndUntagToWord32ArrayElement(
+ transitions, WeakFixedArray::kHeaderSize,
+ IntPtrConstant(TransitionArray::kTransitionLengthIndex)));
+ });
+}
+
+template <typename Array>
+TNode<IntPtrT> CodeStubAssembler::EntryIndexToIndex(
+ TNode<Uint32T> entry_index) {
+ TNode<Int32T> entry_size = Int32Constant(Array::kEntrySize);
+ TNode<Word32T> index = Int32Mul(entry_index, entry_size);
+ return ChangeInt32ToIntPtr(index);
+}
+
+template <typename Array>
+TNode<IntPtrT> CodeStubAssembler::ToKeyIndex(TNode<Uint32T> entry_index) {
+ return IntPtrAdd(IntPtrConstant(Array::ToKeyIndex(0)),
+ EntryIndexToIndex<Array>(entry_index));
+}
+
+template TNode<IntPtrT> CodeStubAssembler::ToKeyIndex<DescriptorArray>(
+ TNode<Uint32T>);
+template TNode<IntPtrT> CodeStubAssembler::ToKeyIndex<TransitionArray>(
+ TNode<Uint32T>);
+
+template <>
+TNode<Uint32T> CodeStubAssembler::GetSortedKeyIndex<DescriptorArray>(
+ TNode<DescriptorArray> descriptors, TNode<Uint32T> descriptor_number) {
+ TNode<Uint32T> details =
+ DescriptorArrayGetDetails(descriptors, descriptor_number);
+ return DecodeWord32<PropertyDetails::DescriptorPointer>(details);
+}
+
+template <>
+TNode<Uint32T> CodeStubAssembler::GetSortedKeyIndex<TransitionArray>(
+ TNode<TransitionArray> transitions, TNode<Uint32T> transition_number) {
+ return transition_number;
+}
+
+template <typename Array>
+TNode<Name> CodeStubAssembler::GetKey(TNode<Array> array,
+ TNode<Uint32T> entry_index) {
+ static_assert(std::is_base_of<TransitionArray, Array>::value ||
+ std::is_base_of<DescriptorArray, Array>::value,
+ "T must be a descendant of DescriptorArray or TransitionArray");
+ const int key_offset = Array::ToKeyIndex(0) * kTaggedSize;
+ TNode<MaybeObject> element =
+ LoadArrayElement(array, Array::kHeaderSize,
+ EntryIndexToIndex<Array>(entry_index), key_offset);
+ return CAST(element);
+}
+
+template TNode<Name> CodeStubAssembler::GetKey<DescriptorArray>(
+ TNode<DescriptorArray>, TNode<Uint32T>);
+template TNode<Name> CodeStubAssembler::GetKey<TransitionArray>(
+ TNode<TransitionArray>, TNode<Uint32T>);
+
+TNode<Uint32T> CodeStubAssembler::DescriptorArrayGetDetails(
+ TNode<DescriptorArray> descriptors, TNode<Uint32T> descriptor_number) {
+ const int details_offset = DescriptorArray::ToDetailsIndex(0) * kTaggedSize;
+ return Unsigned(LoadAndUntagToWord32ArrayElement(
+ descriptors, DescriptorArray::kHeaderSize,
+ EntryIndexToIndex<DescriptorArray>(descriptor_number), details_offset));
+}
+
+template <typename Array>
+void CodeStubAssembler::LookupBinary(TNode<Name> unique_name,
+ TNode<Array> array,
+ TNode<Uint32T> number_of_valid_entries,
+ Label* if_found,
+ TVariable<IntPtrT>* var_name_index,
+ Label* if_not_found) {
+ Comment("LookupBinary");
+ TVARIABLE(Uint32T, var_low, Unsigned(Int32Constant(0)));
+ TNode<Uint32T> limit =
+ Unsigned(Int32Sub(NumberOfEntries<Array>(array), Int32Constant(1)));
+ TVARIABLE(Uint32T, var_high, limit);
+ TNode<Uint32T> hash = LoadNameHashAssumeComputed(unique_name);
+ CSA_ASSERT(this, Word32NotEqual(hash, Int32Constant(0)));
+
+ // Assume non-empty array.
+ CSA_ASSERT(this, Uint32LessThanOrEqual(var_low.value(), var_high.value()));
+
+ Label binary_loop(this, {&var_high, &var_low});
+ Goto(&binary_loop);
+ BIND(&binary_loop);
+ {
+ // mid = low + (high - low) / 2 (to avoid overflow in "(low + high) / 2").
+ TNode<Uint32T> mid = Unsigned(
+ Int32Add(var_low.value(),
+ Word32Shr(Int32Sub(var_high.value(), var_low.value()), 1)));
+ // mid_name = array->GetSortedKey(mid).
+ TNode<Uint32T> sorted_key_index = GetSortedKeyIndex<Array>(array, mid);
+ TNode<Name> mid_name = GetKey<Array>(array, sorted_key_index);
+
+ TNode<Uint32T> mid_hash = LoadNameHashAssumeComputed(mid_name);
+
+ Label mid_greater(this), mid_less(this), merge(this);
+ Branch(Uint32GreaterThanOrEqual(mid_hash, hash), &mid_greater, &mid_less);
+ BIND(&mid_greater);
+ {
+ var_high = mid;
+ Goto(&merge);
+ }
+ BIND(&mid_less);
+ {
+ var_low = Unsigned(Int32Add(mid, Int32Constant(1)));
+ Goto(&merge);
+ }
+ BIND(&merge);
+ GotoIf(Word32NotEqual(var_low.value(), var_high.value()), &binary_loop);
+ }
+
+ Label scan_loop(this, &var_low);
+ Goto(&scan_loop);
+ BIND(&scan_loop);
+ {
+ GotoIf(Int32GreaterThan(var_low.value(), limit), if_not_found);
+
+ TNode<Uint32T> sort_index =
+ GetSortedKeyIndex<Array>(array, var_low.value());
+ TNode<Name> current_name = GetKey<Array>(array, sort_index);
+ TNode<Uint32T> current_hash = LoadNameHashAssumeComputed(current_name);
+ GotoIf(Word32NotEqual(current_hash, hash), if_not_found);
+ Label next(this);
+ GotoIf(TaggedNotEqual(current_name, unique_name), &next);
+ GotoIf(Uint32GreaterThanOrEqual(sort_index, number_of_valid_entries),
+ if_not_found);
+ *var_name_index = ToKeyIndex<Array>(sort_index);
+ Goto(if_found);
+
+ BIND(&next);
+ var_low = Unsigned(Int32Add(var_low.value(), Int32Constant(1)));
+ Goto(&scan_loop);
+ }
+}
+
+void CodeStubAssembler::ForEachEnumerableOwnProperty(
+ TNode<Context> context, TNode<Map> map, TNode<JSObject> object,
+ ForEachEnumerationMode mode, const ForEachKeyValueFunction& body,
+ Label* bailout) {
+ TNode<Uint16T> type = LoadMapInstanceType(map);
+ TNode<Uint32T> bit_field3 = EnsureOnlyHasSimpleProperties(map, type, bailout);
+
+ TVARIABLE(DescriptorArray, var_descriptors, LoadMapDescriptors(map));
+ TNode<Uint32T> nof_descriptors =
+ DecodeWord32<Map::Bits3::NumberOfOwnDescriptorsBits>(bit_field3);
+
+ TVARIABLE(BoolT, var_stable, Int32TrueConstant());
+
+ TVARIABLE(BoolT, var_has_symbol, Int32FalseConstant());
+ // false - iterate only string properties, true - iterate only symbol
+ // properties
+ TVARIABLE(BoolT, var_is_symbol_processing_loop, Int32FalseConstant());
+ TVARIABLE(IntPtrT, var_start_key_index,
+ ToKeyIndex<DescriptorArray>(Unsigned(Int32Constant(0))));
+ // Note: var_end_key_index is exclusive for the loop
+ TVARIABLE(IntPtrT, var_end_key_index,
+ ToKeyIndex<DescriptorArray>(nof_descriptors));
+ VariableList list({&var_descriptors, &var_stable, &var_has_symbol,
+ &var_is_symbol_processing_loop, &var_start_key_index,
+ &var_end_key_index},
+ zone());
+ Label descriptor_array_loop(this, list);
+
+ Goto(&descriptor_array_loop);
+ BIND(&descriptor_array_loop);
+
+ BuildFastLoop<IntPtrT>(
+ list, var_start_key_index.value(), var_end_key_index.value(),
+ [&](TNode<IntPtrT> descriptor_key_index) {
+ TNode<Name> next_key =
+ LoadKeyByKeyIndex(var_descriptors.value(), descriptor_key_index);
+
+ TVARIABLE(Object, var_value, SmiConstant(0));
+ Label callback(this), next_iteration(this);
+
+ if (mode == kEnumerationOrder) {
+ // |next_key| is either a string or a symbol
+ // Skip strings or symbols depending on
+ // |var_is_symbol_processing_loop|.
+ Label if_string(this), if_symbol(this), if_name_ok(this);
+ Branch(IsSymbol(next_key), &if_symbol, &if_string);
+ BIND(&if_symbol);
+ {
+ // Process symbol property when |var_is_symbol_processing_loop| is
+ // true.
+ GotoIf(var_is_symbol_processing_loop.value(), &if_name_ok);
+ // First iteration need to calculate smaller range for processing
+ // symbols
+ Label if_first_symbol(this);
+ // var_end_key_index is still inclusive at this point.
+ var_end_key_index = descriptor_key_index;
+ Branch(var_has_symbol.value(), &next_iteration, &if_first_symbol);
+ BIND(&if_first_symbol);
+ {
+ var_start_key_index = descriptor_key_index;
+ var_has_symbol = Int32TrueConstant();
+ Goto(&next_iteration);
+ }
+ }
+ BIND(&if_string);
+ {
+ CSA_ASSERT(this, IsString(next_key));
+ // Process string property when |var_is_symbol_processing_loop| is
+ // false.
+ Branch(var_is_symbol_processing_loop.value(), &next_iteration,
+ &if_name_ok);
+ }
+ BIND(&if_name_ok);
+ }
+ {
+ TVARIABLE(Map, var_map);
+ TVARIABLE(HeapObject, var_meta_storage);
+ TVARIABLE(IntPtrT, var_entry);
+ TVARIABLE(Uint32T, var_details);
+ Label if_found(this);
+
+ Label if_found_fast(this), if_found_dict(this);
+
+ Label if_stable(this), if_not_stable(this);
+ Branch(var_stable.value(), &if_stable, &if_not_stable);
+ BIND(&if_stable);
+ {
+ // Directly decode from the descriptor array if |object| did not
+ // change shape.
+ var_map = map;
+ var_meta_storage = var_descriptors.value();
+ var_entry = Signed(descriptor_key_index);
+ Goto(&if_found_fast);
+ }
+ BIND(&if_not_stable);
+ {
+ // If the map did change, do a slower lookup. We are still
+ // guaranteed that the object has a simple shape, and that the key
+ // is a name.
+ var_map = LoadMap(object);
+ TryLookupPropertyInSimpleObject(
+ object, var_map.value(), next_key, &if_found_fast,
+ &if_found_dict, &var_meta_storage, &var_entry, &next_iteration);
+ }
+
+ BIND(&if_found_fast);
+ {
+ TNode<DescriptorArray> descriptors = CAST(var_meta_storage.value());
+ TNode<IntPtrT> name_index = var_entry.value();
+
+ // Skip non-enumerable properties.
+ var_details = LoadDetailsByKeyIndex(descriptors, name_index);
+ GotoIf(IsSetWord32(var_details.value(),
+ PropertyDetails::kAttributesDontEnumMask),
+ &next_iteration);
+
+ LoadPropertyFromFastObject(object, var_map.value(), descriptors,
+ name_index, var_details.value(),
+ &var_value);
+ Goto(&if_found);
+ }
+ BIND(&if_found_dict);
+ {
+ TNode<NameDictionary> dictionary = CAST(var_meta_storage.value());
+ TNode<IntPtrT> entry = var_entry.value();
+
+ TNode<Uint32T> details = LoadDetailsByKeyIndex(dictionary, entry);
+ // Skip non-enumerable properties.
+ GotoIf(
+ IsSetWord32(details, PropertyDetails::kAttributesDontEnumMask),
+ &next_iteration);
+
+ var_details = details;
+ var_value = LoadValueByKeyIndex<NameDictionary>(dictionary, entry);
+ Goto(&if_found);
+ }
+
+ // Here we have details and value which could be an accessor.
+ BIND(&if_found);
+ {
+ Label slow_load(this, Label::kDeferred);
+
+ var_value = CallGetterIfAccessor(var_value.value(), object,
+ var_details.value(), context,
+ object, &slow_load, kCallJSGetter);
+ Goto(&callback);
+
+ BIND(&slow_load);
+ var_value =
+ CallRuntime(Runtime::kGetProperty, context, object, next_key);
+ Goto(&callback);
+
+ BIND(&callback);
+ body(next_key, var_value.value());
+
+ // Check if |object| is still stable, i.e. the descriptors in the
+ // preloaded |descriptors| are still the same modulo in-place
+ // representation changes.
+ GotoIfNot(var_stable.value(), &next_iteration);
+ var_stable = TaggedEqual(LoadMap(object), map);
+ // Reload the descriptors just in case the actual array changed, and
+ // any of the field representations changed in-place.
+ var_descriptors = LoadMapDescriptors(map);
+
+ Goto(&next_iteration);
+ }
+ }
+ BIND(&next_iteration);
+ },
+ DescriptorArray::kEntrySize, IndexAdvanceMode::kPost);
+
+ if (mode == kEnumerationOrder) {
+ Label done(this);
+ GotoIf(var_is_symbol_processing_loop.value(), &done);
+ GotoIfNot(var_has_symbol.value(), &done);
+ // All string properties are processed, now process symbol properties.
+ var_is_symbol_processing_loop = Int32TrueConstant();
+ // Add DescriptorArray::kEntrySize to make the var_end_key_index exclusive
+ // as BuildFastLoop() expects.
+ Increment(&var_end_key_index, DescriptorArray::kEntrySize);
+ Goto(&descriptor_array_loop);
+
+ BIND(&done);
+ }
+}
+
+TNode<Object> CodeStubAssembler::GetConstructor(TNode<Map> map) {
+ TVARIABLE(HeapObject, var_maybe_constructor);
+ var_maybe_constructor = map;
+ Label loop(this, &var_maybe_constructor), done(this);
+ GotoIfNot(IsMap(var_maybe_constructor.value()), &done);
+ Goto(&loop);
+
+ BIND(&loop);
+ {
+ var_maybe_constructor = CAST(
+ LoadObjectField(var_maybe_constructor.value(),
+ Map::kConstructorOrBackPointerOrNativeContextOffset));
+ GotoIf(IsMap(var_maybe_constructor.value()), &loop);
+ Goto(&done);
+ }
+
+ BIND(&done);
+ return var_maybe_constructor.value();
+}
+
+TNode<NativeContext> CodeStubAssembler::GetCreationContext(
+ TNode<JSReceiver> receiver, Label* if_bailout) {
+ TNode<Map> receiver_map = LoadMap(receiver);
+ TNode<Object> constructor = GetConstructor(receiver_map);
+
+ TVARIABLE(JSFunction, var_function);
+
+ Label done(this), if_jsfunction(this), if_jsgenerator(this);
+ GotoIf(TaggedIsSmi(constructor), if_bailout);
+
+ TNode<Map> function_map = LoadMap(CAST(constructor));
+ GotoIf(IsJSFunctionMap(function_map), &if_jsfunction);
+ GotoIf(IsJSGeneratorMap(function_map), &if_jsgenerator);
+ // Remote objects don't have a creation context.
+ GotoIf(IsFunctionTemplateInfoMap(function_map), if_bailout);
+
+ CSA_ASSERT(this, IsJSFunctionMap(receiver_map));
+ var_function = CAST(receiver);
+ Goto(&done);
+
+ BIND(&if_jsfunction);
+ {
+ var_function = CAST(constructor);
+ Goto(&done);
+ }
+
+ BIND(&if_jsgenerator);
+ {
+ var_function = LoadJSGeneratorObjectFunction(CAST(receiver));
+ Goto(&done);
+ }
+
+ BIND(&done);
+ TNode<Context> context = LoadJSFunctionContext(var_function.value());
+
+ GotoIfNot(IsContext(context), if_bailout);
+
+ TNode<NativeContext> native_context = LoadNativeContext(context);
+ return native_context;
+}
+
+void CodeStubAssembler::DescriptorLookup(TNode<Name> unique_name,
+ TNode<DescriptorArray> descriptors,
+ TNode<Uint32T> bitfield3,
+ Label* if_found,
+ TVariable<IntPtrT>* var_name_index,
+ Label* if_not_found) {
+ Comment("DescriptorArrayLookup");
+ TNode<Uint32T> nof =
+ DecodeWord32<Map::Bits3::NumberOfOwnDescriptorsBits>(bitfield3);
+ Lookup<DescriptorArray>(unique_name, descriptors, nof, if_found,
+ var_name_index, if_not_found);
+}
+
+void CodeStubAssembler::TransitionLookup(TNode<Name> unique_name,
+ TNode<TransitionArray> transitions,
+ Label* if_found,
+ TVariable<IntPtrT>* var_name_index,
+ Label* if_not_found) {
+ Comment("TransitionArrayLookup");
+ TNode<Uint32T> number_of_valid_transitions =
+ NumberOfEntries<TransitionArray>(transitions);
+ Lookup<TransitionArray>(unique_name, transitions, number_of_valid_transitions,
+ if_found, var_name_index, if_not_found);
+}
+
+template <typename Array>
+void CodeStubAssembler::Lookup(TNode<Name> unique_name, TNode<Array> array,
+ TNode<Uint32T> number_of_valid_entries,
+ Label* if_found,
+ TVariable<IntPtrT>* var_name_index,
+ Label* if_not_found) {
+ Comment("ArrayLookup");
+ if (!number_of_valid_entries) {
+ number_of_valid_entries = NumberOfEntries(array);
+ }
+ GotoIf(Word32Equal(number_of_valid_entries, Int32Constant(0)), if_not_found);
+ Label linear_search(this), binary_search(this);
+ const int kMaxElementsForLinearSearch = 32;
+ Branch(Uint32LessThanOrEqual(number_of_valid_entries,
+ Int32Constant(kMaxElementsForLinearSearch)),
+ &linear_search, &binary_search);
+ BIND(&linear_search);
+ {
+ LookupLinear<Array>(unique_name, array, number_of_valid_entries, if_found,
+ var_name_index, if_not_found);
+ }
+ BIND(&binary_search);
+ {
+ LookupBinary<Array>(unique_name, array, number_of_valid_entries, if_found,
+ var_name_index, if_not_found);
+ }
+}
+
+void CodeStubAssembler::TryLookupPropertyInSimpleObject(
+ TNode<JSObject> object, TNode<Map> map, TNode<Name> unique_name,
+ Label* if_found_fast, Label* if_found_dict,
+ TVariable<HeapObject>* var_meta_storage, TVariable<IntPtrT>* var_name_index,
+ Label* if_not_found) {
+ CSA_ASSERT(this, IsSimpleObjectMap(map));
+ CSA_ASSERT(this, IsUniqueNameNoCachedIndex(unique_name));
+
+ TNode<Uint32T> bit_field3 = LoadMapBitField3(map);
+ Label if_isfastmap(this), if_isslowmap(this);
+ Branch(IsSetWord32<Map::Bits3::IsDictionaryMapBit>(bit_field3), &if_isslowmap,
+ &if_isfastmap);
+ BIND(&if_isfastmap);
+ {
+ TNode<DescriptorArray> descriptors = LoadMapDescriptors(map);
+ *var_meta_storage = descriptors;
+
+ DescriptorLookup(unique_name, descriptors, bit_field3, if_found_fast,
+ var_name_index, if_not_found);
+ }
+ BIND(&if_isslowmap);
+ {
+ TNode<NameDictionary> dictionary = CAST(LoadSlowProperties(object));
+ *var_meta_storage = dictionary;
+
+ NameDictionaryLookup<NameDictionary>(dictionary, unique_name, if_found_dict,
+ var_name_index, if_not_found);
+ }
+}
+
+void CodeStubAssembler::TryLookupProperty(
+ TNode<HeapObject> object, TNode<Map> map, SloppyTNode<Int32T> instance_type,
+ TNode<Name> unique_name, Label* if_found_fast, Label* if_found_dict,
+ Label* if_found_global, TVariable<HeapObject>* var_meta_storage,
+ TVariable<IntPtrT>* var_name_index, Label* if_not_found,
+ Label* if_bailout) {
+ Label if_objectisspecial(this);
+ GotoIf(IsSpecialReceiverInstanceType(instance_type), &if_objectisspecial);
+
+ TryLookupPropertyInSimpleObject(CAST(object), map, unique_name, if_found_fast,
+ if_found_dict, var_meta_storage,
+ var_name_index, if_not_found);
+
+ BIND(&if_objectisspecial);
+ {
+ // Handle global object here and bailout for other special objects.
+ GotoIfNot(InstanceTypeEqual(instance_type, JS_GLOBAL_OBJECT_TYPE),
+ if_bailout);
+
+ // Handle interceptors and access checks in runtime.
+ TNode<Int32T> bit_field = LoadMapBitField(map);
+ int mask = Map::Bits1::HasNamedInterceptorBit::kMask |
+ Map::Bits1::IsAccessCheckNeededBit::kMask;
+ GotoIf(IsSetWord32(bit_field, mask), if_bailout);
+
+ TNode<GlobalDictionary> dictionary = CAST(LoadSlowProperties(CAST(object)));
+ *var_meta_storage = dictionary;
+
+ NameDictionaryLookup<GlobalDictionary>(
+ dictionary, unique_name, if_found_global, var_name_index, if_not_found);
+ }
+}
+
+void CodeStubAssembler::TryHasOwnProperty(TNode<HeapObject> object,
+ TNode<Map> map,
+ TNode<Int32T> instance_type,
+ TNode<Name> unique_name,
+ Label* if_found, Label* if_not_found,
+ Label* if_bailout) {
+ Comment("TryHasOwnProperty");
+ CSA_ASSERT(this, IsUniqueNameNoCachedIndex(unique_name));
+ TVARIABLE(HeapObject, var_meta_storage);
+ TVARIABLE(IntPtrT, var_name_index);
+
+ Label if_found_global(this);
+ TryLookupProperty(object, map, instance_type, unique_name, if_found, if_found,
+ &if_found_global, &var_meta_storage, &var_name_index,
+ if_not_found, if_bailout);
+
+ BIND(&if_found_global);
+ {
+ TVARIABLE(Object, var_value);
+ TVARIABLE(Uint32T, var_details);
+ // Check if the property cell is not deleted.
+ LoadPropertyFromGlobalDictionary(CAST(var_meta_storage.value()),
+ var_name_index.value(), &var_details,
+ &var_value, if_not_found);
+ Goto(if_found);
+ }
+}
+
+TNode<Object> CodeStubAssembler::GetMethod(TNode<Context> context,
+ TNode<Object> object,
+ Handle<Name> name,
+ Label* if_null_or_undefined) {
+ TNode<Object> method = GetProperty(context, object, name);
+
+ GotoIf(IsUndefined(method), if_null_or_undefined);
+ GotoIf(IsNull(method), if_null_or_undefined);
+
+ return method;
+}
+
+TNode<Object> CodeStubAssembler::GetIteratorMethod(
+ TNode<Context> context, TNode<HeapObject> heap_obj,
+ Label* if_iteratorundefined) {
+ return GetMethod(context, heap_obj, isolate()->factory()->iterator_symbol(),
+ if_iteratorundefined);
+}
+
+void CodeStubAssembler::LoadPropertyFromFastObject(
+ TNode<HeapObject> object, TNode<Map> map,
+ TNode<DescriptorArray> descriptors, TNode<IntPtrT> name_index,
+ TVariable<Uint32T>* var_details, TVariable<Object>* var_value) {
+ TNode<Uint32T> details = LoadDetailsByKeyIndex(descriptors, name_index);
+ *var_details = details;
+
+ LoadPropertyFromFastObject(object, map, descriptors, name_index, details,
+ var_value);
+}
+
+void CodeStubAssembler::LoadPropertyFromFastObject(
+ TNode<HeapObject> object, TNode<Map> map,
+ TNode<DescriptorArray> descriptors, TNode<IntPtrT> name_index,
+ TNode<Uint32T> details, TVariable<Object>* var_value) {
+ Comment("[ LoadPropertyFromFastObject");
+
+ TNode<Uint32T> location =
+ DecodeWord32<PropertyDetails::LocationField>(details);
+
+ Label if_in_field(this), if_in_descriptor(this), done(this);
+ Branch(Word32Equal(location, Int32Constant(kField)), &if_in_field,
+ &if_in_descriptor);
+ BIND(&if_in_field);
+ {
+ TNode<IntPtrT> field_index =
+ Signed(DecodeWordFromWord32<PropertyDetails::FieldIndexField>(details));
+ TNode<Uint32T> representation =
+ DecodeWord32<PropertyDetails::RepresentationField>(details);
+
+ field_index =
+ IntPtrAdd(field_index, LoadMapInobjectPropertiesStartInWords(map));
+ TNode<IntPtrT> instance_size_in_words = LoadMapInstanceSizeInWords(map);
+
+ Label if_inobject(this), if_backing_store(this);
+ TVARIABLE(Float64T, var_double_value);
+ Label rebox_double(this, &var_double_value);
+ Branch(UintPtrLessThan(field_index, instance_size_in_words), &if_inobject,
+ &if_backing_store);
+ BIND(&if_inobject);
+ {
+ Comment("if_inobject");
+ TNode<IntPtrT> field_offset = TimesTaggedSize(field_index);
+
+ Label if_double(this), if_tagged(this);
+ Branch(Word32NotEqual(representation,
+ Int32Constant(Representation::kDouble)),
+ &if_tagged, &if_double);
+ BIND(&if_tagged);
+ {
+ *var_value = LoadObjectField(object, field_offset);
+ Goto(&done);
+ }
+ BIND(&if_double);
+ {
+ if (FLAG_unbox_double_fields) {
+ var_double_value = LoadObjectField<Float64T>(object, field_offset);
+ } else {
+ TNode<HeapNumber> heap_number =
+ CAST(LoadObjectField(object, field_offset));
+ var_double_value = LoadHeapNumberValue(heap_number);
+ }
+ Goto(&rebox_double);
+ }
+ }
+ BIND(&if_backing_store);
+ {
+ Comment("if_backing_store");
+ TNode<HeapObject> properties = LoadFastProperties(CAST(object));
+ field_index = Signed(IntPtrSub(field_index, instance_size_in_words));
+ TNode<Object> value =
+ LoadPropertyArrayElement(CAST(properties), field_index);
+
+ Label if_double(this), if_tagged(this);
+ Branch(Word32NotEqual(representation,
+ Int32Constant(Representation::kDouble)),
+ &if_tagged, &if_double);
+ BIND(&if_tagged);
+ {
+ *var_value = value;
+ Goto(&done);
+ }
+ BIND(&if_double);
+ {
+ var_double_value = LoadHeapNumberValue(CAST(value));
+ Goto(&rebox_double);
+ }
+ }
+ BIND(&rebox_double);
+ {
+ Comment("rebox_double");
+ TNode<HeapNumber> heap_number =
+ AllocateHeapNumberWithValue(var_double_value.value());
+ *var_value = heap_number;
+ Goto(&done);
+ }
+ }
+ BIND(&if_in_descriptor);
+ {
+ *var_value = LoadValueByKeyIndex(descriptors, name_index);
+ Goto(&done);
+ }
+ BIND(&done);
+
+ Comment("] LoadPropertyFromFastObject");
+}
+
+void CodeStubAssembler::LoadPropertyFromNameDictionary(
+ TNode<NameDictionary> dictionary, TNode<IntPtrT> name_index,
+ TVariable<Uint32T>* var_details, TVariable<Object>* var_value) {
+ Comment("LoadPropertyFromNameDictionary");
+ *var_details = LoadDetailsByKeyIndex(dictionary, name_index);
+ *var_value = LoadValueByKeyIndex(dictionary, name_index);
+
+ Comment("] LoadPropertyFromNameDictionary");
+}
+
+void CodeStubAssembler::LoadPropertyFromGlobalDictionary(
+ TNode<GlobalDictionary> dictionary, TNode<IntPtrT> name_index,
+ TVariable<Uint32T>* var_details, TVariable<Object>* var_value,
+ Label* if_deleted) {
+ Comment("[ LoadPropertyFromGlobalDictionary");
+ TNode<PropertyCell> property_cell =
+ CAST(LoadFixedArrayElement(dictionary, name_index));
+
+ TNode<Object> value =
+ LoadObjectField(property_cell, PropertyCell::kValueOffset);
+ GotoIf(TaggedEqual(value, TheHoleConstant()), if_deleted);
+
+ *var_value = value;
+
+ TNode<Uint32T> details = Unsigned(LoadAndUntagToWord32ObjectField(
+ property_cell, PropertyCell::kPropertyDetailsRawOffset));
+ *var_details = details;
+
+ Comment("] LoadPropertyFromGlobalDictionary");
+}
+
+// |value| is the property backing store's contents, which is either a value or
+// an accessor pair, as specified by |details|. |holder| is a JSObject or a
+// PropertyCell (TODO: use UnionT). Returns either the original value, or the
+// result of the getter call.
+TNode<Object> CodeStubAssembler::CallGetterIfAccessor(
+ TNode<Object> value, TNode<HeapObject> holder, TNode<Uint32T> details,
+ TNode<Context> context, TNode<Object> receiver, Label* if_bailout,
+ GetOwnPropertyMode mode) {
+ TVARIABLE(Object, var_value, value);
+ Label done(this), if_accessor_info(this, Label::kDeferred);
+
+ TNode<Uint32T> kind = DecodeWord32<PropertyDetails::KindField>(details);
+ GotoIf(Word32Equal(kind, Int32Constant(kData)), &done);
+
+ // Accessor case.
+ GotoIfNot(IsAccessorPair(CAST(value)), &if_accessor_info);
+
+ // AccessorPair case.
+ {
+ if (mode == kCallJSGetter) {
+ Label if_callable(this), if_function_template_info(this);
+ TNode<AccessorPair> accessor_pair = CAST(value);
+ TNode<HeapObject> getter =
+ CAST(LoadObjectField(accessor_pair, AccessorPair::kGetterOffset));
+ TNode<Map> getter_map = LoadMap(getter);
+
+ GotoIf(IsCallableMap(getter_map), &if_callable);
+ GotoIf(IsFunctionTemplateInfoMap(getter_map), &if_function_template_info);
+
+ // Return undefined if the {getter} is not callable.
+ var_value = UndefinedConstant();
+ Goto(&done);
+
+ BIND(&if_callable);
+ {
+ // Call the accessor.
+ var_value = Call(context, getter, receiver);
+ Goto(&done);
+ }
+
+ BIND(&if_function_template_info);
+ {
+ TNode<HeapObject> cached_property_name = LoadObjectField<HeapObject>(
+ getter, FunctionTemplateInfo::kCachedPropertyNameOffset);
+ GotoIfNot(IsTheHole(cached_property_name), if_bailout);
+
+ TNode<NativeContext> creation_context =
+ GetCreationContext(CAST(holder), if_bailout);
+ var_value = CallBuiltin(
+ Builtins::kCallFunctionTemplate_CheckAccessAndCompatibleReceiver,
+ creation_context, getter, IntPtrConstant(0), receiver);
+ Goto(&done);
+ }
+ } else {
+ Goto(&done);
+ }
+ }
+
+ // AccessorInfo case.
+ BIND(&if_accessor_info);
+ {
+ TNode<AccessorInfo> accessor_info = CAST(value);
+ Label if_array(this), if_function(this), if_wrapper(this);
+
+ // Dispatch based on {holder} instance type.
+ TNode<Map> holder_map = LoadMap(holder);
+ TNode<Uint16T> holder_instance_type = LoadMapInstanceType(holder_map);
+ GotoIf(IsJSArrayInstanceType(holder_instance_type), &if_array);
+ GotoIf(IsJSFunctionInstanceType(holder_instance_type), &if_function);
+ Branch(IsJSPrimitiveWrapperInstanceType(holder_instance_type), &if_wrapper,
+ if_bailout);
+
+ // JSArray AccessorInfo case.
+ BIND(&if_array);
+ {
+ // We only deal with the "length" accessor on JSArray.
+ GotoIfNot(IsLengthString(
+ LoadObjectField(accessor_info, AccessorInfo::kNameOffset)),
+ if_bailout);
+ TNode<JSArray> array = CAST(holder);
+ var_value = LoadJSArrayLength(array);
+ Goto(&done);
+ }
+
+ // JSFunction AccessorInfo case.
+ BIND(&if_function);
+ {
+ // We only deal with the "prototype" accessor on JSFunction here.
+ GotoIfNot(IsPrototypeString(
+ LoadObjectField(accessor_info, AccessorInfo::kNameOffset)),
+ if_bailout);
+
+ TNode<JSFunction> function = CAST(holder);
+ GotoIfPrototypeRequiresRuntimeLookup(function, holder_map, if_bailout);
+ var_value = LoadJSFunctionPrototype(function, if_bailout);
+ Goto(&done);
+ }
+
+ // JSPrimitiveWrapper AccessorInfo case.
+ BIND(&if_wrapper);
+ {
+ // We only deal with the "length" accessor on JSPrimitiveWrapper string
+ // wrappers.
+ GotoIfNot(IsLengthString(
+ LoadObjectField(accessor_info, AccessorInfo::kNameOffset)),
+ if_bailout);
+ TNode<Object> holder_value = LoadJSPrimitiveWrapperValue(CAST(holder));
+ GotoIfNot(TaggedIsNotSmi(holder_value), if_bailout);
+ GotoIfNot(IsString(CAST(holder_value)), if_bailout);
+ var_value = LoadStringLengthAsSmi(CAST(holder_value));
+ Goto(&done);
+ }
+ }
+
+ BIND(&done);
+ return var_value.value();
+}
+
+void CodeStubAssembler::TryGetOwnProperty(
+ TNode<Context> context, TNode<Object> receiver, TNode<JSReceiver> object,
+ TNode<Map> map, TNode<Int32T> instance_type, TNode<Name> unique_name,
+ Label* if_found_value, TVariable<Object>* var_value, Label* if_not_found,
+ Label* if_bailout) {
+ TryGetOwnProperty(context, receiver, object, map, instance_type, unique_name,
+ if_found_value, var_value, nullptr, nullptr, if_not_found,
+ if_bailout, kCallJSGetter);
+}
+
+void CodeStubAssembler::TryGetOwnProperty(
+ TNode<Context> context, TNode<Object> receiver, TNode<JSReceiver> object,
+ TNode<Map> map, TNode<Int32T> instance_type, TNode<Name> unique_name,
+ Label* if_found_value, TVariable<Object>* var_value,
+ TVariable<Uint32T>* var_details, TVariable<Object>* var_raw_value,
+ Label* if_not_found, Label* if_bailout, GetOwnPropertyMode mode) {
+ DCHECK_EQ(MachineRepresentation::kTagged, var_value->rep());
+ Comment("TryGetOwnProperty");
+ CSA_ASSERT(this, IsUniqueNameNoCachedIndex(unique_name));
+ TVARIABLE(HeapObject, var_meta_storage);
+ TVARIABLE(IntPtrT, var_entry);
+
+ Label if_found_fast(this), if_found_dict(this), if_found_global(this);
+
+ TVARIABLE(Uint32T, local_var_details);
+ if (!var_details) {
+ var_details = &local_var_details;
+ }
+ Label if_found(this);
+
+ TryLookupProperty(object, map, instance_type, unique_name, &if_found_fast,
+ &if_found_dict, &if_found_global, &var_meta_storage,
+ &var_entry, if_not_found, if_bailout);
+ BIND(&if_found_fast);
+ {
+ TNode<DescriptorArray> descriptors = CAST(var_meta_storage.value());
+ TNode<IntPtrT> name_index = var_entry.value();
+
+ LoadPropertyFromFastObject(object, map, descriptors, name_index,
+ var_details, var_value);
+ Goto(&if_found);
+ }
+ BIND(&if_found_dict);
+ {
+ TNode<NameDictionary> dictionary = CAST(var_meta_storage.value());
+ TNode<IntPtrT> entry = var_entry.value();
+ LoadPropertyFromNameDictionary(dictionary, entry, var_details, var_value);
+ Goto(&if_found);
+ }
+ BIND(&if_found_global);
+ {
+ TNode<GlobalDictionary> dictionary = CAST(var_meta_storage.value());
+ TNode<IntPtrT> entry = var_entry.value();
+
+ LoadPropertyFromGlobalDictionary(dictionary, entry, var_details, var_value,
+ if_not_found);
+ Goto(&if_found);
+ }
+ // Here we have details and value which could be an accessor.
+ BIND(&if_found);
+ {
+ // TODO(ishell): Execute C++ accessor in case of accessor info
+ if (var_raw_value) {
+ *var_raw_value = *var_value;
+ }
+ TNode<Object> value =
+ CallGetterIfAccessor(var_value->value(), object, var_details->value(),
+ context, receiver, if_bailout, mode);
+ *var_value = value;
+ Goto(if_found_value);
+ }
+}
+
+void CodeStubAssembler::TryLookupElement(
+ TNode<HeapObject> object, TNode<Map> map, SloppyTNode<Int32T> instance_type,
+ SloppyTNode<IntPtrT> intptr_index, Label* if_found, Label* if_absent,
+ Label* if_not_found, Label* if_bailout) {
+ // Handle special objects in runtime.
+ GotoIf(IsSpecialReceiverInstanceType(instance_type), if_bailout);
+
+ TNode<Int32T> elements_kind = LoadMapElementsKind(map);
+
+ // TODO(verwaest): Support other elements kinds as well.
+ Label if_isobjectorsmi(this), if_isdouble(this), if_isdictionary(this),
+ if_isfaststringwrapper(this), if_isslowstringwrapper(this), if_oob(this),
+ if_typedarray(this);
+ // clang-format off
+ int32_t values[] = {
+ // Handled by {if_isobjectorsmi}.
+ PACKED_SMI_ELEMENTS, HOLEY_SMI_ELEMENTS, PACKED_ELEMENTS, HOLEY_ELEMENTS,
+ PACKED_NONEXTENSIBLE_ELEMENTS, PACKED_SEALED_ELEMENTS,
+ HOLEY_NONEXTENSIBLE_ELEMENTS, HOLEY_SEALED_ELEMENTS,
+ PACKED_FROZEN_ELEMENTS, HOLEY_FROZEN_ELEMENTS,
+ // Handled by {if_isdouble}.
+ PACKED_DOUBLE_ELEMENTS, HOLEY_DOUBLE_ELEMENTS,
+ // Handled by {if_isdictionary}.
+ DICTIONARY_ELEMENTS,
+ // Handled by {if_isfaststringwrapper}.
+ FAST_STRING_WRAPPER_ELEMENTS,
+ // Handled by {if_isslowstringwrapper}.
+ SLOW_STRING_WRAPPER_ELEMENTS,
+ // Handled by {if_not_found}.
+ NO_ELEMENTS,
+ // Handled by {if_typed_array}.
+ UINT8_ELEMENTS,
+ INT8_ELEMENTS,
+ UINT16_ELEMENTS,
+ INT16_ELEMENTS,
+ UINT32_ELEMENTS,
+ INT32_ELEMENTS,
+ FLOAT32_ELEMENTS,
+ FLOAT64_ELEMENTS,
+ UINT8_CLAMPED_ELEMENTS,
+ BIGUINT64_ELEMENTS,
+ BIGINT64_ELEMENTS,
+ };
+ Label* labels[] = {
+ &if_isobjectorsmi, &if_isobjectorsmi, &if_isobjectorsmi,
+ &if_isobjectorsmi, &if_isobjectorsmi, &if_isobjectorsmi,
+ &if_isobjectorsmi, &if_isobjectorsmi, &if_isobjectorsmi,
+ &if_isobjectorsmi,
+ &if_isdouble, &if_isdouble,
+ &if_isdictionary,
+ &if_isfaststringwrapper,
+ &if_isslowstringwrapper,
+ if_not_found,
+ &if_typedarray,
+ &if_typedarray,
+ &if_typedarray,
+ &if_typedarray,
+ &if_typedarray,
+ &if_typedarray,
+ &if_typedarray,
+ &if_typedarray,
+ &if_typedarray,
+ &if_typedarray,
+ &if_typedarray,
+ };
+ // clang-format on
+ STATIC_ASSERT(arraysize(values) == arraysize(labels));
+ Switch(elements_kind, if_bailout, values, labels, arraysize(values));
+
+ BIND(&if_isobjectorsmi);
+ {
+ TNode<FixedArray> elements = CAST(LoadElements(CAST(object)));
+ TNode<IntPtrT> length = LoadAndUntagFixedArrayBaseLength(elements);
+
+ GotoIfNot(UintPtrLessThan(intptr_index, length), &if_oob);
+
+ TNode<Object> element = UnsafeLoadFixedArrayElement(elements, intptr_index);
+ TNode<Oddball> the_hole = TheHoleConstant();
+ Branch(TaggedEqual(element, the_hole), if_not_found, if_found);
+ }
+ BIND(&if_isdouble);
+ {
+ TNode<FixedArrayBase> elements = LoadElements(CAST(object));
+ TNode<IntPtrT> length = LoadAndUntagFixedArrayBaseLength(elements);
+
+ GotoIfNot(UintPtrLessThan(intptr_index, length), &if_oob);
+
+ // Check if the element is a double hole, but don't load it.
+ LoadFixedDoubleArrayElement(CAST(elements), intptr_index, if_not_found,
+ MachineType::None());
+ Goto(if_found);
+ }
+ BIND(&if_isdictionary);
+ {
+ // Negative and too-large keys must be converted to property names.
+ if (Is64()) {
+ GotoIf(UintPtrLessThan(IntPtrConstant(JSArray::kMaxArrayIndex),
+ intptr_index),
+ if_bailout);
+ } else {
+ GotoIf(IntPtrLessThan(intptr_index, IntPtrConstant(0)), if_bailout);
+ }
+
+ TVARIABLE(IntPtrT, var_entry);
+ TNode<NumberDictionary> elements = CAST(LoadElements(CAST(object)));
+ NumberDictionaryLookup(elements, intptr_index, if_found, &var_entry,
+ if_not_found);
+ }
+ BIND(&if_isfaststringwrapper);
+ {
+ TNode<String> string = CAST(LoadJSPrimitiveWrapperValue(CAST(object)));
+ TNode<IntPtrT> length = LoadStringLengthAsWord(string);
+ GotoIf(UintPtrLessThan(intptr_index, length), if_found);
+ Goto(&if_isobjectorsmi);
+ }
+ BIND(&if_isslowstringwrapper);
+ {
+ TNode<String> string = CAST(LoadJSPrimitiveWrapperValue(CAST(object)));
+ TNode<IntPtrT> length = LoadStringLengthAsWord(string);
+ GotoIf(UintPtrLessThan(intptr_index, length), if_found);
+ Goto(&if_isdictionary);
+ }
+ BIND(&if_typedarray);
+ {
+ TNode<JSArrayBuffer> buffer = LoadJSArrayBufferViewBuffer(CAST(object));
+ GotoIf(IsDetachedBuffer(buffer), if_absent);
+
+ TNode<UintPtrT> length = LoadJSTypedArrayLength(CAST(object));
+ Branch(UintPtrLessThan(intptr_index, length), if_found, if_absent);
+ }
+ BIND(&if_oob);
+ {
+ // Positive OOB indices mean "not found", negative indices and indices
+ // out of array index range must be converted to property names.
+ if (Is64()) {
+ GotoIf(UintPtrLessThan(IntPtrConstant(JSArray::kMaxArrayIndex),
+ intptr_index),
+ if_bailout);
+ } else {
+ GotoIf(IntPtrLessThan(intptr_index, IntPtrConstant(0)), if_bailout);
+ }
+ Goto(if_not_found);
+ }
+}
+
+void CodeStubAssembler::BranchIfMaybeSpecialIndex(TNode<String> name_string,
+ Label* if_maybe_special_index,
+ Label* if_not_special_index) {
+ // TODO(cwhan.tunz): Implement fast cases more.
+
+ // If a name is empty or too long, it's not a special index
+ // Max length of canonical double: -X.XXXXXXXXXXXXXXXXX-eXXX
+ const int kBufferSize = 24;
+ TNode<Smi> string_length = LoadStringLengthAsSmi(name_string);
+ GotoIf(SmiEqual(string_length, SmiConstant(0)), if_not_special_index);
+ GotoIf(SmiGreaterThan(string_length, SmiConstant(kBufferSize)),
+ if_not_special_index);
+
+ // If the first character of name is not a digit or '-', or we can't match it
+ // to Infinity or NaN, then this is not a special index.
+ TNode<Int32T> first_char = StringCharCodeAt(name_string, UintPtrConstant(0));
+ // If the name starts with '-', it can be a negative index.
+ GotoIf(Word32Equal(first_char, Int32Constant('-')), if_maybe_special_index);
+ // If the name starts with 'I', it can be "Infinity".
+ GotoIf(Word32Equal(first_char, Int32Constant('I')), if_maybe_special_index);
+ // If the name starts with 'N', it can be "NaN".
+ GotoIf(Word32Equal(first_char, Int32Constant('N')), if_maybe_special_index);
+ // Finally, if the first character is not a digit either, then we are sure
+ // that the name is not a special index.
+ GotoIf(Uint32LessThan(first_char, Int32Constant('0')), if_not_special_index);
+ GotoIf(Uint32LessThan(Int32Constant('9'), first_char), if_not_special_index);
+ Goto(if_maybe_special_index);
+}
+
+void CodeStubAssembler::TryPrototypeChainLookup(
+ TNode<Object> receiver, TNode<Object> object_arg, TNode<Object> key,
+ const LookupPropertyInHolder& lookup_property_in_holder,
+ const LookupElementInHolder& lookup_element_in_holder, Label* if_end,
+ Label* if_bailout, Label* if_proxy) {
+ // Ensure receiver is JSReceiver, otherwise bailout.
+ GotoIf(TaggedIsSmi(receiver), if_bailout);
+ TNode<HeapObject> object = CAST(object_arg);
+
+ TNode<Map> map = LoadMap(object);
+ TNode<Uint16T> instance_type = LoadMapInstanceType(map);
+ {
+ Label if_objectisreceiver(this);
+ STATIC_ASSERT(LAST_JS_RECEIVER_TYPE == LAST_TYPE);
+ STATIC_ASSERT(FIRST_JS_RECEIVER_TYPE == JS_PROXY_TYPE);
+ Branch(IsJSReceiverInstanceType(instance_type), &if_objectisreceiver,
+ if_bailout);
+ BIND(&if_objectisreceiver);
+
+ GotoIf(InstanceTypeEqual(instance_type, JS_PROXY_TYPE), if_proxy);
+ }
+
+ TVARIABLE(IntPtrT, var_index);
+ TVARIABLE(Name, var_unique);
+
+ Label if_keyisindex(this), if_iskeyunique(this);
+ TryToName(key, &if_keyisindex, &var_index, &if_iskeyunique, &var_unique,
+ if_bailout);
+
+ BIND(&if_iskeyunique);
+ {
+ TVARIABLE(HeapObject, var_holder, object);
+ TVARIABLE(Map, var_holder_map, map);
+ TVARIABLE(Int32T, var_holder_instance_type, instance_type);
+
+ Label loop(this, {&var_holder, &var_holder_map, &var_holder_instance_type});
+ Goto(&loop);
+ BIND(&loop);
+ {
+ TNode<Map> holder_map = var_holder_map.value();
+ TNode<Int32T> holder_instance_type = var_holder_instance_type.value();
+
+ Label next_proto(this), check_integer_indexed_exotic(this);
+ lookup_property_in_holder(CAST(receiver), var_holder.value(), holder_map,
+ holder_instance_type, var_unique.value(),
+ &check_integer_indexed_exotic, if_bailout);
+
+ BIND(&check_integer_indexed_exotic);
+ {
+ // Bailout if it can be an integer indexed exotic case.
+ GotoIfNot(InstanceTypeEqual(holder_instance_type, JS_TYPED_ARRAY_TYPE),
+ &next_proto);
+ GotoIfNot(IsString(var_unique.value()), &next_proto);
+ BranchIfMaybeSpecialIndex(CAST(var_unique.value()), if_bailout,
+ &next_proto);
+ }
+
+ BIND(&next_proto);
+
+ TNode<HeapObject> proto = LoadMapPrototype(holder_map);
+
+ GotoIf(IsNull(proto), if_end);
+
+ TNode<Map> map = LoadMap(proto);
+ TNode<Uint16T> instance_type = LoadMapInstanceType(map);
+
+ var_holder = proto;
+ var_holder_map = map;
+ var_holder_instance_type = instance_type;
+ Goto(&loop);
+ }
+ }
+ BIND(&if_keyisindex);
+ {
+ TVARIABLE(HeapObject, var_holder, object);
+ TVARIABLE(Map, var_holder_map, map);
+ TVARIABLE(Int32T, var_holder_instance_type, instance_type);
+
+ Label loop(this, {&var_holder, &var_holder_map, &var_holder_instance_type});
+ Goto(&loop);
+ BIND(&loop);
+ {
+ Label next_proto(this);
+ lookup_element_in_holder(CAST(receiver), var_holder.value(),
+ var_holder_map.value(),
+ var_holder_instance_type.value(),
+ var_index.value(), &next_proto, if_bailout);
+ BIND(&next_proto);
+
+ TNode<HeapObject> proto = LoadMapPrototype(var_holder_map.value());
+
+ GotoIf(IsNull(proto), if_end);
+
+ TNode<Map> map = LoadMap(proto);
+ TNode<Uint16T> instance_type = LoadMapInstanceType(map);
+
+ var_holder = proto;
+ var_holder_map = map;
+ var_holder_instance_type = instance_type;
+ Goto(&loop);
+ }
+ }
+}
+
+TNode<Oddball> CodeStubAssembler::HasInPrototypeChain(TNode<Context> context,
+ TNode<HeapObject> object,
+ TNode<Object> prototype) {
+ TVARIABLE(Oddball, var_result);
+ Label return_false(this), return_true(this),
+ return_runtime(this, Label::kDeferred), return_result(this);
+
+ // Loop through the prototype chain looking for the {prototype}.
+ TVARIABLE(Map, var_object_map, LoadMap(object));
+ Label loop(this, &var_object_map);
+ Goto(&loop);
+ BIND(&loop);
+ {
+ // Check if we can determine the prototype directly from the {object_map}.
+ Label if_objectisdirect(this), if_objectisspecial(this, Label::kDeferred);
+ TNode<Map> object_map = var_object_map.value();
+ TNode<Uint16T> object_instance_type = LoadMapInstanceType(object_map);
+ Branch(IsSpecialReceiverInstanceType(object_instance_type),
+ &if_objectisspecial, &if_objectisdirect);
+ BIND(&if_objectisspecial);
+ {
+ // The {object_map} is a special receiver map or a primitive map, check
+ // if we need to use the if_objectisspecial path in the runtime.
+ GotoIf(InstanceTypeEqual(object_instance_type, JS_PROXY_TYPE),
+ &return_runtime);
+ TNode<Int32T> object_bitfield = LoadMapBitField(object_map);
+ int mask = Map::Bits1::HasNamedInterceptorBit::kMask |
+ Map::Bits1::IsAccessCheckNeededBit::kMask;
+ Branch(IsSetWord32(object_bitfield, mask), &return_runtime,
+ &if_objectisdirect);
+ }
+ BIND(&if_objectisdirect);
+
+ // Check the current {object} prototype.
+ TNode<HeapObject> object_prototype = LoadMapPrototype(object_map);
+ GotoIf(IsNull(object_prototype), &return_false);
+ GotoIf(TaggedEqual(object_prototype, prototype), &return_true);
+
+ // Continue with the prototype.
+ CSA_ASSERT(this, TaggedIsNotSmi(object_prototype));
+ var_object_map = LoadMap(object_prototype);
+ Goto(&loop);
+ }
+
+ BIND(&return_true);
+ var_result = TrueConstant();
+ Goto(&return_result);
+
+ BIND(&return_false);
+ var_result = FalseConstant();
+ Goto(&return_result);
+
+ BIND(&return_runtime);
+ {
+ // Fallback to the runtime implementation.
+ var_result = CAST(
+ CallRuntime(Runtime::kHasInPrototypeChain, context, object, prototype));
+ }
+ Goto(&return_result);
+
+ BIND(&return_result);
+ return var_result.value();
+}
+
+TNode<Oddball> CodeStubAssembler::OrdinaryHasInstance(
+ TNode<Context> context, TNode<Object> callable_maybe_smi,
+ TNode<Object> object_maybe_smi) {
+ TVARIABLE(Oddball, var_result);
+ Label return_runtime(this, Label::kDeferred), return_result(this);
+
+ GotoIfForceSlowPath(&return_runtime);
+
+ // Goto runtime if {object} is a Smi.
+ GotoIf(TaggedIsSmi(object_maybe_smi), &return_runtime);
+
+ // Goto runtime if {callable} is a Smi.
+ GotoIf(TaggedIsSmi(callable_maybe_smi), &return_runtime);
+
+ {
+ // Load map of {callable}.
+ TNode<HeapObject> object = CAST(object_maybe_smi);
+ TNode<HeapObject> callable = CAST(callable_maybe_smi);
+ TNode<Map> callable_map = LoadMap(callable);
+
+ // Goto runtime if {callable} is not a JSFunction.
+ TNode<Uint16T> callable_instance_type = LoadMapInstanceType(callable_map);
+ GotoIfNot(InstanceTypeEqual(callable_instance_type, JS_FUNCTION_TYPE),
+ &return_runtime);
+
+ GotoIfPrototypeRequiresRuntimeLookup(CAST(callable), callable_map,
+ &return_runtime);
+
+ // Get the "prototype" (or initial map) of the {callable}.
+ TNode<HeapObject> callable_prototype = LoadObjectField<HeapObject>(
+ callable, JSFunction::kPrototypeOrInitialMapOffset);
+ {
+ Label no_initial_map(this), walk_prototype_chain(this);
+ TVARIABLE(HeapObject, var_callable_prototype, callable_prototype);
+
+ // Resolve the "prototype" if the {callable} has an initial map.
+ GotoIfNot(IsMap(callable_prototype), &no_initial_map);
+ var_callable_prototype = LoadObjectField<HeapObject>(
+ callable_prototype, Map::kPrototypeOffset);
+ Goto(&walk_prototype_chain);
+
+ BIND(&no_initial_map);
+ // {callable_prototype} is the hole if the "prototype" property hasn't
+ // been requested so far.
+ Branch(TaggedEqual(callable_prototype, TheHoleConstant()),
+ &return_runtime, &walk_prototype_chain);
+
+ BIND(&walk_prototype_chain);
+ callable_prototype = var_callable_prototype.value();
+ }
+
+ // Loop through the prototype chain looking for the {callable} prototype.
+ var_result = HasInPrototypeChain(context, object, callable_prototype);
+ Goto(&return_result);
+ }
+
+ BIND(&return_runtime);
+ {
+ // Fallback to the runtime implementation.
+ var_result = CAST(CallRuntime(Runtime::kOrdinaryHasInstance, context,
+ callable_maybe_smi, object_maybe_smi));
+ }
+ Goto(&return_result);
+
+ BIND(&return_result);
+ return var_result.value();
+}
+
+template <typename TIndex>
+TNode<IntPtrT> CodeStubAssembler::ElementOffsetFromIndex(
+ TNode<TIndex> index_node, ElementsKind kind, int base_size) {
+ // TODO(v8:9708): Remove IntPtrT variant in favor of UintPtrT.
+ static_assert(std::is_same<TIndex, Smi>::value ||
+ std::is_same<TIndex, TaggedIndex>::value ||
+ std::is_same<TIndex, IntPtrT>::value ||
+ std::is_same<TIndex, UintPtrT>::value,
+ "Only Smi, UintPtrT or IntPtrT index nodes are allowed");
+ int element_size_shift = ElementsKindToShiftSize(kind);
+ int element_size = 1 << element_size_shift;
+ intptr_t index = 0;
+ TNode<IntPtrT> intptr_index_node;
+ bool constant_index = false;
+ if (std::is_same<TIndex, Smi>::value) {
+ TNode<Smi> smi_index_node = ReinterpretCast<Smi>(index_node);
+ int const kSmiShiftBits = kSmiShiftSize + kSmiTagSize;
+ element_size_shift -= kSmiShiftBits;
+ Smi smi_index;
+ constant_index = ToSmiConstant(smi_index_node, &smi_index);
+ if (constant_index) {
+ index = smi_index.value();
+ } else {
+ if (COMPRESS_POINTERS_BOOL) {
+ smi_index_node = NormalizeSmiIndex(smi_index_node);
+ }
+ }
+ intptr_index_node = BitcastTaggedToWordForTagAndSmiBits(smi_index_node);
+ } else if (std::is_same<TIndex, TaggedIndex>::value) {
+ TNode<TaggedIndex> tagged_index_node =
+ ReinterpretCast<TaggedIndex>(index_node);
+ element_size_shift -= kSmiTagSize;
+ intptr_index_node = BitcastTaggedToWordForTagAndSmiBits(tagged_index_node);
+ constant_index = ToIntPtrConstant(intptr_index_node, &index);
+ } else {
+ intptr_index_node = ReinterpretCast<IntPtrT>(index_node);
+ constant_index = ToIntPtrConstant(intptr_index_node, &index);
+ }
+ if (constant_index) {
+ return IntPtrConstant(base_size + element_size * index);
+ }
+
+ TNode<IntPtrT> shifted_index =
+ (element_size_shift == 0)
+ ? intptr_index_node
+ : ((element_size_shift > 0)
+ ? WordShl(intptr_index_node,
+ IntPtrConstant(element_size_shift))
+ : WordSar(intptr_index_node,
+ IntPtrConstant(-element_size_shift)));
+ return IntPtrAdd(IntPtrConstant(base_size), Signed(shifted_index));
+}
+
+// Instantiate ElementOffsetFromIndex for Smi and IntPtrT.
+template V8_EXPORT_PRIVATE TNode<IntPtrT>
+CodeStubAssembler::ElementOffsetFromIndex<Smi>(TNode<Smi> index_node,
+ ElementsKind kind,
+ int base_size);
+template V8_EXPORT_PRIVATE TNode<IntPtrT>
+CodeStubAssembler::ElementOffsetFromIndex<TaggedIndex>(
+ TNode<TaggedIndex> index_node, ElementsKind kind, int base_size);
+template V8_EXPORT_PRIVATE TNode<IntPtrT>
+CodeStubAssembler::ElementOffsetFromIndex<IntPtrT>(TNode<IntPtrT> index_node,
+ ElementsKind kind,
+ int base_size);
+
+TNode<BoolT> CodeStubAssembler::IsOffsetInBounds(SloppyTNode<IntPtrT> offset,
+ SloppyTNode<IntPtrT> length,
+ int header_size,
+ ElementsKind kind) {
+ // Make sure we point to the last field.
+ int element_size = 1 << ElementsKindToShiftSize(kind);
+ int correction = header_size - kHeapObjectTag - element_size;
+ TNode<IntPtrT> last_offset = ElementOffsetFromIndex(length, kind, correction);
+ return IntPtrLessThanOrEqual(offset, last_offset);
+}
+
+TNode<HeapObject> CodeStubAssembler::LoadFeedbackCellValue(
+ TNode<JSFunction> closure) {
+ TNode<FeedbackCell> feedback_cell =
+ LoadObjectField<FeedbackCell>(closure, JSFunction::kFeedbackCellOffset);
+ return LoadObjectField<HeapObject>(feedback_cell, FeedbackCell::kValueOffset);
+}
+
+TNode<HeapObject> CodeStubAssembler::LoadFeedbackVector(
+ TNode<JSFunction> closure) {
+ TVARIABLE(HeapObject, maybe_vector, LoadFeedbackCellValue(closure));
+ Label done(this);
+
+ // If the closure doesn't have a feedback vector allocated yet, return
+ // undefined. FeedbackCell can contain Undefined / FixedArray (for lazy
+ // allocations) / FeedbackVector.
+ GotoIf(IsFeedbackVector(maybe_vector.value()), &done);
+
+ // In all other cases return Undefined.
+ maybe_vector = UndefinedConstant();
+ Goto(&done);
+
+ BIND(&done);
+ return maybe_vector.value();
+}
+
+TNode<ClosureFeedbackCellArray> CodeStubAssembler::LoadClosureFeedbackArray(
+ TNode<JSFunction> closure) {
+ TVARIABLE(HeapObject, feedback_cell_array, LoadFeedbackCellValue(closure));
+ Label end(this);
+
+ // When feedback vectors are not yet allocated feedback cell contains a
+ // an array of feedback cells used by create closures.
+ GotoIf(HasInstanceType(feedback_cell_array.value(),
+ CLOSURE_FEEDBACK_CELL_ARRAY_TYPE),
+ &end);
+
+ // Load FeedbackCellArray from feedback vector.
+ TNode<FeedbackVector> vector = CAST(feedback_cell_array.value());
+ feedback_cell_array = CAST(
+ LoadObjectField(vector, FeedbackVector::kClosureFeedbackCellArrayOffset));
+ Goto(&end);
+
+ BIND(&end);
+ return CAST(feedback_cell_array.value());
+}
+
+TNode<FeedbackVector> CodeStubAssembler::LoadFeedbackVectorForStub() {
+ TNode<JSFunction> function =
+ CAST(LoadFromParentFrame(StandardFrameConstants::kFunctionOffset));
+ return CAST(LoadFeedbackVector(function));
+}
+
+void CodeStubAssembler::UpdateFeedback(TNode<Smi> feedback,
+ TNode<HeapObject> maybe_vector,
+ TNode<UintPtrT> slot_id) {
+ Label end(this);
+ // If feedback_vector is not valid, then nothing to do.
+ GotoIf(IsUndefined(maybe_vector), &end);
+
+ // This method is used for binary op and compare feedback. These
+ // vector nodes are initialized with a smi 0, so we can simply OR
+ // our new feedback in place.
+ TNode<FeedbackVector> feedback_vector = CAST(maybe_vector);
+ TNode<MaybeObject> feedback_element =
+ LoadFeedbackVectorSlot(feedback_vector, slot_id);
+ TNode<Smi> previous_feedback = CAST(feedback_element);
+ TNode<Smi> combined_feedback = SmiOr(previous_feedback, feedback);
+
+ GotoIf(SmiEqual(previous_feedback, combined_feedback), &end);
+ {
+ StoreFeedbackVectorSlot(feedback_vector, slot_id, combined_feedback,
+ SKIP_WRITE_BARRIER);
+ ReportFeedbackUpdate(feedback_vector, slot_id, "UpdateFeedback");
+ Goto(&end);
+ }
+
+ BIND(&end);
+}
+
+void CodeStubAssembler::ReportFeedbackUpdate(
+ TNode<FeedbackVector> feedback_vector, SloppyTNode<UintPtrT> slot_id,
+ const char* reason) {
+ // Reset profiler ticks.
+ StoreObjectFieldNoWriteBarrier(
+ feedback_vector, FeedbackVector::kProfilerTicksOffset, Int32Constant(0));
+
+#ifdef V8_TRACE_FEEDBACK_UPDATES
+ // Trace the update.
+ CallRuntime(Runtime::kInterpreterTraceUpdateFeedback, NoContextConstant(),
+ LoadFromParentFrame(StandardFrameConstants::kFunctionOffset),
+ SmiTag(Signed(slot_id)), StringConstant(reason));
+#endif // V8_TRACE_FEEDBACK_UPDATES
+}
+
+void CodeStubAssembler::OverwriteFeedback(TVariable<Smi>* existing_feedback,
+ int new_feedback) {
+ if (existing_feedback == nullptr) return;
+ *existing_feedback = SmiConstant(new_feedback);
+}
+
+void CodeStubAssembler::CombineFeedback(TVariable<Smi>* existing_feedback,
+ int feedback) {
+ if (existing_feedback == nullptr) return;
+ *existing_feedback = SmiOr(existing_feedback->value(), SmiConstant(feedback));
+}
+
+void CodeStubAssembler::CombineFeedback(TVariable<Smi>* existing_feedback,
+ TNode<Smi> feedback) {
+ if (existing_feedback == nullptr) return;
+ *existing_feedback = SmiOr(existing_feedback->value(), feedback);
+}
+
+void CodeStubAssembler::CheckForAssociatedProtector(TNode<Name> name,
+ Label* if_protector) {
+ // This list must be kept in sync with LookupIterator::UpdateProtector!
+ // TODO(jkummerow): Would it be faster to have a bit in Symbol::flags()?
+ GotoIf(TaggedEqual(name, ConstructorStringConstant()), if_protector);
+ GotoIf(TaggedEqual(name, IteratorSymbolConstant()), if_protector);
+ GotoIf(TaggedEqual(name, NextStringConstant()), if_protector);
+ GotoIf(TaggedEqual(name, SpeciesSymbolConstant()), if_protector);
+ GotoIf(TaggedEqual(name, IsConcatSpreadableSymbolConstant()), if_protector);
+ GotoIf(TaggedEqual(name, ResolveStringConstant()), if_protector);
+ GotoIf(TaggedEqual(name, ThenStringConstant()), if_protector);
+ // Fall through if no case matched.
+}
+
+TNode<Map> CodeStubAssembler::LoadReceiverMap(SloppyTNode<Object> receiver) {
+ return Select<Map>(
+ TaggedIsSmi(receiver), [=] { return HeapNumberMapConstant(); },
+ [=] { return LoadMap(UncheckedCast<HeapObject>(receiver)); });
+}
+
+TNode<IntPtrT> CodeStubAssembler::TryToIntptr(
+ SloppyTNode<Object> key, Label* if_not_intptr,
+ TVariable<Int32T>* var_instance_type) {
+ TVARIABLE(IntPtrT, var_intptr_key);
+ Label done(this, &var_intptr_key), key_is_smi(this), key_is_heapnumber(this);
+ GotoIf(TaggedIsSmi(key), &key_is_smi);
+
+ TNode<Int32T> instance_type = LoadInstanceType(CAST(key));
+ if (var_instance_type != nullptr) {
+ *var_instance_type = instance_type;
+ }
+
+ Branch(IsHeapNumberInstanceType(instance_type), &key_is_heapnumber,
+ if_not_intptr);
+
+ BIND(&key_is_smi);
+ {
+ var_intptr_key = SmiUntag(CAST(key));
+ Goto(&done);
+ }
+
+ BIND(&key_is_heapnumber);
+ {
+ TNode<Float64T> value = LoadHeapNumberValue(CAST(key));
+ TNode<IntPtrT> int_value = ChangeFloat64ToIntPtr(value);
+ GotoIfNot(Float64Equal(value, RoundIntPtrToFloat64(int_value)),
+ if_not_intptr);
+#if V8_TARGET_ARCH_64_BIT
+ // We can't rely on Is64() alone because 32-bit compilers rightly complain
+ // about kMaxSafeIntegerUint64 not fitting into an intptr_t.
+ DCHECK(Is64());
+ // TODO(jkummerow): Investigate whether we can drop support for
+ // negative indices.
+ GotoIfNot(IsInRange(int_value, static_cast<intptr_t>(-kMaxSafeInteger),
+ static_cast<intptr_t>(kMaxSafeIntegerUint64)),
+ if_not_intptr);
+#else
+ DCHECK(!Is64());
+#endif
+ var_intptr_key = int_value;
+ Goto(&done);
+ }
+
+ BIND(&done);
+ return var_intptr_key.value();
+}
+
+TNode<Context> CodeStubAssembler::LoadScriptContext(
+ TNode<Context> context, TNode<IntPtrT> context_index) {
+ TNode<NativeContext> native_context = LoadNativeContext(context);
+ TNode<ScriptContextTable> script_context_table = CAST(
+ LoadContextElement(native_context, Context::SCRIPT_CONTEXT_TABLE_INDEX));
+
+ TNode<Context> script_context = CAST(LoadFixedArrayElement(
+ script_context_table, context_index,
+ ScriptContextTable::kFirstContextSlotIndex * kTaggedSize));
+ return script_context;
+}
+
+namespace {
+
+// Converts typed array elements kind to a machine representations.
+MachineRepresentation ElementsKindToMachineRepresentation(ElementsKind kind) {
+ switch (kind) {
+ case UINT8_CLAMPED_ELEMENTS:
+ case UINT8_ELEMENTS:
+ case INT8_ELEMENTS:
+ return MachineRepresentation::kWord8;
+ case UINT16_ELEMENTS:
+ case INT16_ELEMENTS:
+ return MachineRepresentation::kWord16;
+ case UINT32_ELEMENTS:
+ case INT32_ELEMENTS:
+ return MachineRepresentation::kWord32;
+ case FLOAT32_ELEMENTS:
+ return MachineRepresentation::kFloat32;
+ case FLOAT64_ELEMENTS:
+ return MachineRepresentation::kFloat64;
+ default:
+ UNREACHABLE();
+ }
+}
+
+} // namespace
+
+template <typename TArray, typename TIndex>
+void CodeStubAssembler::StoreElementBigIntOrTypedArray(TNode<TArray> elements,
+ ElementsKind kind,
+ TNode<TIndex> index,
+ Node* value) {
+ // TODO(v8:9708): Do we want to keep both IntPtrT and UintPtrT variants?
+ static_assert(std::is_same<TIndex, Smi>::value ||
+ std::is_same<TIndex, UintPtrT>::value ||
+ std::is_same<TIndex, IntPtrT>::value,
+ "Only Smi, UintPtrT or IntPtrT index is allowed");
+ static_assert(std::is_same<TArray, RawPtrT>::value ||
+ std::is_same<TArray, FixedArrayBase>::value,
+ "Only RawPtrT or FixedArrayBase elements are allowed");
+ if (kind == BIGINT64_ELEMENTS || kind == BIGUINT64_ELEMENTS) {
+ TNode<IntPtrT> offset = ElementOffsetFromIndex(index, kind, 0);
+ TVARIABLE(UintPtrT, var_low);
+ // Only used on 32-bit platforms.
+ TVARIABLE(UintPtrT, var_high);
+ BigIntToRawBytes(CAST(value), &var_low, &var_high);
+
+ MachineRepresentation rep = WordT::kMachineRepresentation;
+#if defined(V8_TARGET_BIG_ENDIAN)
+ if (!Is64()) {
+ StoreNoWriteBarrier(rep, elements, offset, var_high.value());
+ StoreNoWriteBarrier(rep, elements,
+ IntPtrAdd(offset, IntPtrConstant(kSystemPointerSize)),
+ var_low.value());
+ } else {
+ StoreNoWriteBarrier(rep, elements, offset, var_low.value());
+ }
+#else
+ StoreNoWriteBarrier(rep, elements, offset, var_low.value());
+ if (!Is64()) {
+ StoreNoWriteBarrier(rep, elements,
+ IntPtrAdd(offset, IntPtrConstant(kSystemPointerSize)),
+ var_high.value());
+ }
+#endif
+ } else {
+ DCHECK(IsTypedArrayElementsKind(kind));
+ if (kind == UINT8_CLAMPED_ELEMENTS) {
+ CSA_ASSERT(this, Word32Equal(UncheckedCast<Word32T>(value),
+ Word32And(Int32Constant(0xFF), value)));
+ }
+ TNode<IntPtrT> offset = ElementOffsetFromIndex(index, kind, 0);
+ // TODO(cbruni): Add OOB check once typed.
+ MachineRepresentation rep = ElementsKindToMachineRepresentation(kind);
+ StoreNoWriteBarrier(rep, elements, offset, value);
+ }
+}
+
+template <typename TIndex>
+void CodeStubAssembler::StoreElement(TNode<FixedArrayBase> elements,
+ ElementsKind kind, TNode<TIndex> index,
+ Node* value) {
+ if (kind == BIGINT64_ELEMENTS || kind == BIGUINT64_ELEMENTS ||
+ IsTypedArrayElementsKind(kind)) {
+ StoreElementBigIntOrTypedArray(elements, kind, index, value);
+ } else if (IsDoubleElementsKind(kind)) {
+ TNode<Float64T> value_float64 = UncheckedCast<Float64T>(value);
+ StoreFixedDoubleArrayElement(CAST(elements), index, value_float64);
+ } else {
+ WriteBarrierMode barrier_mode = IsSmiElementsKind(kind)
+ ? UNSAFE_SKIP_WRITE_BARRIER
+ : UPDATE_WRITE_BARRIER;
+ StoreFixedArrayElement(CAST(elements), index, value, barrier_mode, 0);
+ }
+}
+
+template <typename TIndex>
+void CodeStubAssembler::StoreElement(TNode<RawPtrT> elements, ElementsKind kind,
+ TNode<TIndex> index, Node* value) {
+ DCHECK(kind == BIGINT64_ELEMENTS || kind == BIGUINT64_ELEMENTS ||
+ IsTypedArrayElementsKind(kind));
+ StoreElementBigIntOrTypedArray(elements, kind, index, value);
+}
+template V8_EXPORT_PRIVATE void CodeStubAssembler::StoreElement<UintPtrT>(
+ TNode<RawPtrT>, ElementsKind, TNode<UintPtrT>, Node*);
+
+TNode<Uint8T> CodeStubAssembler::Int32ToUint8Clamped(
+ TNode<Int32T> int32_value) {
+ Label done(this);
+ TNode<Int32T> int32_zero = Int32Constant(0);
+ TNode<Int32T> int32_255 = Int32Constant(255);
+ TVARIABLE(Word32T, var_value, int32_value);
+ GotoIf(Uint32LessThanOrEqual(int32_value, int32_255), &done);
+ var_value = int32_zero;
+ GotoIf(Int32LessThan(int32_value, int32_zero), &done);
+ var_value = int32_255;
+ Goto(&done);
+ BIND(&done);
+ return UncheckedCast<Uint8T>(var_value.value());
+}
+
+TNode<Uint8T> CodeStubAssembler::Float64ToUint8Clamped(
+ TNode<Float64T> float64_value) {
+ Label done(this);
+ TVARIABLE(Word32T, var_value, Int32Constant(0));
+ GotoIf(Float64LessThanOrEqual(float64_value, Float64Constant(0.0)), &done);
+ var_value = Int32Constant(255);
+ GotoIf(Float64LessThanOrEqual(Float64Constant(255.0), float64_value), &done);
+ {
+ TNode<Float64T> rounded_value = Float64RoundToEven(float64_value);
+ var_value = TruncateFloat64ToWord32(rounded_value);
+ Goto(&done);
+ }
+ BIND(&done);
+ return UncheckedCast<Uint8T>(var_value.value());
+}
+
+template <>
+TNode<Word32T> CodeStubAssembler::PrepareValueForWriteToTypedArray<Word32T>(
+ TNode<Object> input, ElementsKind elements_kind, TNode<Context> context) {
+ DCHECK(IsTypedArrayElementsKind(elements_kind));
+
+ switch (elements_kind) {
+ case UINT8_ELEMENTS:
+ case INT8_ELEMENTS:
+ case UINT16_ELEMENTS:
+ case INT16_ELEMENTS:
+ case UINT32_ELEMENTS:
+ case INT32_ELEMENTS:
+ case UINT8_CLAMPED_ELEMENTS:
+ break;
+ default:
+ UNREACHABLE();
+ }
+
+ TVARIABLE(Word32T, var_result);
+ TVARIABLE(Object, var_input, input);
+ Label done(this, &var_result), if_smi(this), if_heapnumber_or_oddball(this),
+ convert(this), loop(this, &var_input);
+ Goto(&loop);
+ BIND(&loop);
+ GotoIf(TaggedIsSmi(var_input.value()), &if_smi);
+ // We can handle both HeapNumber and Oddball here, since Oddball has the
+ // same layout as the HeapNumber for the HeapNumber::value field. This
+ // way we can also properly optimize stores of oddballs to typed arrays.
+ TNode<HeapObject> heap_object = CAST(var_input.value());
+ GotoIf(IsHeapNumber(heap_object), &if_heapnumber_or_oddball);
+ STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset,
+ Oddball::kToNumberRawOffset);
+ Branch(HasInstanceType(heap_object, ODDBALL_TYPE), &if_heapnumber_or_oddball,
+ &convert);
+
+ BIND(&if_heapnumber_or_oddball);
+ {
+ TNode<Float64T> value =
+ LoadObjectField<Float64T>(heap_object, HeapNumber::kValueOffset);
+ if (elements_kind == UINT8_CLAMPED_ELEMENTS) {
+ var_result = Float64ToUint8Clamped(value);
+ } else {
+ var_result = TruncateFloat64ToWord32(value);
+ }
+ Goto(&done);
+ }
+
+ BIND(&if_smi);
+ {
+ TNode<Int32T> value = SmiToInt32(CAST(var_input.value()));
+ if (elements_kind == UINT8_CLAMPED_ELEMENTS) {
+ var_result = Int32ToUint8Clamped(value);
+ } else {
+ var_result = value;
+ }
+ Goto(&done);
+ }
+
+ BIND(&convert);
+ {
+ var_input = CallBuiltin(Builtins::kNonNumberToNumber, context, input);
+ Goto(&loop);
+ }
+
+ BIND(&done);
+ return var_result.value();
+}
+
+template <>
+TNode<Float32T> CodeStubAssembler::PrepareValueForWriteToTypedArray<Float32T>(
+ TNode<Object> input, ElementsKind elements_kind, TNode<Context> context) {
+ DCHECK(IsTypedArrayElementsKind(elements_kind));
+ CHECK_EQ(elements_kind, FLOAT32_ELEMENTS);
+
+ TVARIABLE(Float32T, var_result);
+ TVARIABLE(Object, var_input, input);
+ Label done(this, &var_result), if_smi(this), if_heapnumber_or_oddball(this),
+ convert(this), loop(this, &var_input);
+ Goto(&loop);
+ BIND(&loop);
+ GotoIf(TaggedIsSmi(var_input.value()), &if_smi);
+ // We can handle both HeapNumber and Oddball here, since Oddball has the
+ // same layout as the HeapNumber for the HeapNumber::value field. This
+ // way we can also properly optimize stores of oddballs to typed arrays.
+ TNode<HeapObject> heap_object = CAST(var_input.value());
+ GotoIf(IsHeapNumber(heap_object), &if_heapnumber_or_oddball);
+ STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset,
+ Oddball::kToNumberRawOffset);
+ Branch(HasInstanceType(heap_object, ODDBALL_TYPE), &if_heapnumber_or_oddball,
+ &convert);
+
+ BIND(&if_heapnumber_or_oddball);
+ {
+ TNode<Float64T> value =
+ LoadObjectField<Float64T>(heap_object, HeapNumber::kValueOffset);
+ var_result = TruncateFloat64ToFloat32(value);
+ Goto(&done);
+ }
+
+ BIND(&if_smi);
+ {
+ TNode<Int32T> value = SmiToInt32(CAST(var_input.value()));
+ var_result = RoundInt32ToFloat32(value);
+ Goto(&done);
+ }
+
+ BIND(&convert);
+ {
+ var_input = CallBuiltin(Builtins::kNonNumberToNumber, context, input);
+ Goto(&loop);
+ }
+
+ BIND(&done);
+ return var_result.value();
+}
+
+template <>
+TNode<Float64T> CodeStubAssembler::PrepareValueForWriteToTypedArray<Float64T>(
+ TNode<Object> input, ElementsKind elements_kind, TNode<Context> context) {
+ DCHECK(IsTypedArrayElementsKind(elements_kind));
+ CHECK_EQ(elements_kind, FLOAT64_ELEMENTS);
+
+ TVARIABLE(Float64T, var_result);
+ TVARIABLE(Object, var_input, input);
+ Label done(this, &var_result), if_smi(this), if_heapnumber_or_oddball(this),
+ convert(this), loop(this, &var_input);
+ Goto(&loop);
+ BIND(&loop);
+ GotoIf(TaggedIsSmi(var_input.value()), &if_smi);
+ // We can handle both HeapNumber and Oddball here, since Oddball has the
+ // same layout as the HeapNumber for the HeapNumber::value field. This
+ // way we can also properly optimize stores of oddballs to typed arrays.
+ TNode<HeapObject> heap_object = CAST(var_input.value());
+ GotoIf(IsHeapNumber(heap_object), &if_heapnumber_or_oddball);
+ STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset,
+ Oddball::kToNumberRawOffset);
+ Branch(HasInstanceType(heap_object, ODDBALL_TYPE), &if_heapnumber_or_oddball,
+ &convert);
+
+ BIND(&if_heapnumber_or_oddball);
+ {
+ var_result =
+ LoadObjectField<Float64T>(heap_object, HeapNumber::kValueOffset);
+ Goto(&done);
+ }
+
+ BIND(&if_smi);
+ {
+ TNode<Int32T> value = SmiToInt32(CAST(var_input.value()));
+ var_result = ChangeInt32ToFloat64(value);
+ Goto(&done);
+ }
+
+ BIND(&convert);
+ {
+ var_input = CallBuiltin(Builtins::kNonNumberToNumber, context, input);
+ Goto(&loop);
+ }
+
+ BIND(&done);
+ return var_result.value();
+}
+
+Node* CodeStubAssembler::PrepareValueForWriteToTypedArray(
+ TNode<Object> input, ElementsKind elements_kind, TNode<Context> context) {
+ DCHECK(IsTypedArrayElementsKind(elements_kind));
+
+ switch (elements_kind) {
+ case UINT8_ELEMENTS:
+ case INT8_ELEMENTS:
+ case UINT16_ELEMENTS:
+ case INT16_ELEMENTS:
+ case UINT32_ELEMENTS:
+ case INT32_ELEMENTS:
+ case UINT8_CLAMPED_ELEMENTS:
+ return PrepareValueForWriteToTypedArray<Word32T>(input, elements_kind,
+ context);
+ case FLOAT32_ELEMENTS:
+ return PrepareValueForWriteToTypedArray<Float32T>(input, elements_kind,
+ context);
+ case FLOAT64_ELEMENTS:
+ return PrepareValueForWriteToTypedArray<Float64T>(input, elements_kind,
+ context);
+ case BIGINT64_ELEMENTS:
+ case BIGUINT64_ELEMENTS:
+ return ToBigInt(context, input);
+ default:
+ UNREACHABLE();
+ }
+}
+
+void CodeStubAssembler::BigIntToRawBytes(TNode<BigInt> bigint,
+ TVariable<UintPtrT>* var_low,
+ TVariable<UintPtrT>* var_high) {
+ Label done(this);
+ *var_low = Unsigned(IntPtrConstant(0));
+ *var_high = Unsigned(IntPtrConstant(0));
+ TNode<Word32T> bitfield = LoadBigIntBitfield(bigint);
+ TNode<Uint32T> length = DecodeWord32<BigIntBase::LengthBits>(bitfield);
+ TNode<Uint32T> sign = DecodeWord32<BigIntBase::SignBits>(bitfield);
+ GotoIf(Word32Equal(length, Int32Constant(0)), &done);
+ *var_low = LoadBigIntDigit(bigint, 0);
+ if (!Is64()) {
+ Label load_done(this);
+ GotoIf(Word32Equal(length, Int32Constant(1)), &load_done);
+ *var_high = LoadBigIntDigit(bigint, 1);
+ Goto(&load_done);
+ BIND(&load_done);
+ }
+ GotoIf(Word32Equal(sign, Int32Constant(0)), &done);
+ // Negative value. Simulate two's complement.
+ if (!Is64()) {
+ *var_high = Unsigned(IntPtrSub(IntPtrConstant(0), var_high->value()));
+ Label no_carry(this);
+ GotoIf(IntPtrEqual(var_low->value(), IntPtrConstant(0)), &no_carry);
+ *var_high = Unsigned(IntPtrSub(var_high->value(), IntPtrConstant(1)));
+ Goto(&no_carry);
+ BIND(&no_carry);
+ }
+ *var_low = Unsigned(IntPtrSub(IntPtrConstant(0), var_low->value()));
+ Goto(&done);
+ BIND(&done);
+}
+
+void CodeStubAssembler::EmitElementStore(
+ TNode<JSObject> object, TNode<Object> key, TNode<Object> value,
+ ElementsKind elements_kind, KeyedAccessStoreMode store_mode, Label* bailout,
+ TNode<Context> context, TVariable<Object>* maybe_converted_value) {
+ CSA_ASSERT(this, Word32BinaryNot(IsJSProxy(object)));
+
+ TNode<FixedArrayBase> elements = LoadElements(object);
+ if (!(IsSmiOrObjectElementsKind(elements_kind) ||
+ IsSealedElementsKind(elements_kind) ||
+ IsNonextensibleElementsKind(elements_kind))) {
+ CSA_ASSERT(this, Word32BinaryNot(IsFixedCOWArrayMap(LoadMap(elements))));
+ } else if (!IsCOWHandlingStoreMode(store_mode)) {
+ GotoIf(IsFixedCOWArrayMap(LoadMap(elements)), bailout);
+ }
+
+ // TODO(ishell): introduce TryToIntPtrOrSmi() and use BInt.
+ TNode<IntPtrT> intptr_key = TryToIntptr(key, bailout);
+
+ // TODO(rmcilroy): TNodify the converted value once this funciton and
+ // StoreElement are templated based on the type elements_kind type.
+ Node* converted_value = value;
+ if (IsTypedArrayElementsKind(elements_kind)) {
+ Label done(this), update_value_and_bailout(this, Label::kDeferred);
+
+ // IntegerIndexedElementSet converts value to a Number/BigInt prior to the
+ // bounds check.
+ converted_value =
+ PrepareValueForWriteToTypedArray(value, elements_kind, context);
+ TNode<JSTypedArray> typed_array = CAST(object);
+
+ // There must be no allocations between the buffer load and
+ // and the actual store to backing store, because GC may decide that
+ // the buffer is not alive or move the elements.
+ // TODO(ishell): introduce DisallowHeapAllocationCode scope here.
+
+ // Check if buffer has been detached.
+ TNode<JSArrayBuffer> buffer = LoadJSArrayBufferViewBuffer(typed_array);
+ if (maybe_converted_value) {
+ GotoIf(IsDetachedBuffer(buffer), &update_value_and_bailout);
+ } else {
+ GotoIf(IsDetachedBuffer(buffer), bailout);
+ }
+
+ // Bounds check.
+ TNode<UintPtrT> length = LoadJSTypedArrayLength(typed_array);
+
+ if (store_mode == STORE_IGNORE_OUT_OF_BOUNDS) {
+ // Skip the store if we write beyond the length or
+ // to a property with a negative integer index.
+ GotoIfNot(UintPtrLessThan(intptr_key, length), &done);
+ } else {
+ DCHECK_EQ(store_mode, STANDARD_STORE);
+ GotoIfNot(UintPtrLessThan(intptr_key, length), &update_value_and_bailout);
+ }
+
+ TNode<RawPtrT> data_ptr = LoadJSTypedArrayDataPtr(typed_array);
+ StoreElement(data_ptr, elements_kind, intptr_key, converted_value);
+ Goto(&done);
+
+ BIND(&update_value_and_bailout);
+ // We already prepared the incoming value for storing into a typed array.
+ // This might involve calling ToNumber in some cases. We shouldn't call
+ // ToNumber again in the runtime so pass the converted value to the runtime.
+ // The prepared value is an untagged value. Convert it to a tagged value
+ // to pass it to runtime. It is not possible to do the detached buffer check
+ // before we prepare the value, since ToNumber can detach the ArrayBuffer.
+ // The spec specifies the order of these operations.
+ if (maybe_converted_value != nullptr) {
+ switch (elements_kind) {
+ case UINT8_ELEMENTS:
+ case INT8_ELEMENTS:
+ case UINT16_ELEMENTS:
+ case INT16_ELEMENTS:
+ case UINT8_CLAMPED_ELEMENTS:
+ *maybe_converted_value = SmiFromInt32(converted_value);
+ break;
+ case UINT32_ELEMENTS:
+ *maybe_converted_value = ChangeUint32ToTagged(converted_value);
+ break;
+ case INT32_ELEMENTS:
+ *maybe_converted_value = ChangeInt32ToTagged(converted_value);
+ break;
+ case FLOAT32_ELEMENTS: {
+ Label dont_allocate_heap_number(this), end(this);
+ GotoIf(TaggedIsSmi(value), &dont_allocate_heap_number);
+ GotoIf(IsHeapNumber(CAST(value)), &dont_allocate_heap_number);
+ {
+ *maybe_converted_value = AllocateHeapNumberWithValue(
+ ChangeFloat32ToFloat64(converted_value));
+ Goto(&end);
+ }
+ BIND(&dont_allocate_heap_number);
+ {
+ *maybe_converted_value = value;
+ Goto(&end);
+ }
+ BIND(&end);
+ break;
+ }
+ case FLOAT64_ELEMENTS: {
+ Label dont_allocate_heap_number(this), end(this);
+ GotoIf(TaggedIsSmi(value), &dont_allocate_heap_number);
+ GotoIf(IsHeapNumber(CAST(value)), &dont_allocate_heap_number);
+ {
+ *maybe_converted_value =
+ AllocateHeapNumberWithValue(converted_value);
+ Goto(&end);
+ }
+ BIND(&dont_allocate_heap_number);
+ {
+ *maybe_converted_value = value;
+ Goto(&end);
+ }
+ BIND(&end);
+ break;
+ }
+ case BIGINT64_ELEMENTS:
+ case BIGUINT64_ELEMENTS:
+ *maybe_converted_value = CAST(converted_value);
+ break;
+ default:
+ UNREACHABLE();
+ }
+ }
+ Goto(bailout);
+
+ BIND(&done);
+ return;
+ }
+ DCHECK(IsFastElementsKind(elements_kind) ||
+ IsSealedElementsKind(elements_kind) ||
+ IsNonextensibleElementsKind(elements_kind));
+
+ // In case value is stored into a fast smi array, assure that the value is
+ // a smi before manipulating the backing store. Otherwise the backing store
+ // may be left in an invalid state.
+ if (IsSmiElementsKind(elements_kind)) {
+ GotoIfNot(TaggedIsSmi(value), bailout);
+ } else if (IsDoubleElementsKind(elements_kind)) {
+ converted_value = TryTaggedToFloat64(value, bailout);
+ }
+
+ TNode<Smi> smi_length = Select<Smi>(
+ IsJSArray(object),
+ [=]() {
+ // This is casting Number -> Smi which may not actually be safe.
+ return CAST(LoadJSArrayLength(CAST(object)));
+ },
+ [=]() { return LoadFixedArrayBaseLength(elements); });
+
+ TNode<UintPtrT> length = Unsigned(SmiUntag(smi_length));
+ if (IsGrowStoreMode(store_mode) &&
+ !(IsSealedElementsKind(elements_kind) ||
+ IsNonextensibleElementsKind(elements_kind))) {
+ elements = CheckForCapacityGrow(object, elements, elements_kind, length,
+ intptr_key, bailout);
+ } else {
+ GotoIfNot(UintPtrLessThan(Unsigned(intptr_key), length), bailout);
+ }
+
+ // Cannot store to a hole in holey sealed elements so bailout.
+ if (elements_kind == HOLEY_SEALED_ELEMENTS ||
+ elements_kind == HOLEY_NONEXTENSIBLE_ELEMENTS) {
+ TNode<Object> target_value =
+ LoadFixedArrayElement(CAST(elements), intptr_key);
+ GotoIf(IsTheHole(target_value), bailout);
+ }
+
+ // If we didn't grow {elements}, it might still be COW, in which case we
+ // copy it now.
+ if (!(IsSmiOrObjectElementsKind(elements_kind) ||
+ IsSealedElementsKind(elements_kind) ||
+ IsNonextensibleElementsKind(elements_kind))) {
+ CSA_ASSERT(this, Word32BinaryNot(IsFixedCOWArrayMap(LoadMap(elements))));
+ } else if (IsCOWHandlingStoreMode(store_mode)) {
+ elements = CopyElementsOnWrite(object, elements, elements_kind,
+ Signed(length), bailout);
+ }
+
+ CSA_ASSERT(this, Word32BinaryNot(IsFixedCOWArrayMap(LoadMap(elements))));
+ StoreElement(elements, elements_kind, intptr_key, converted_value);
+}
+
+TNode<FixedArrayBase> CodeStubAssembler::CheckForCapacityGrow(
+ TNode<JSObject> object, TNode<FixedArrayBase> elements, ElementsKind kind,
+ TNode<UintPtrT> length, TNode<IntPtrT> key, Label* bailout) {
+ DCHECK(IsFastElementsKind(kind));
+ TVARIABLE(FixedArrayBase, checked_elements);
+ Label grow_case(this), no_grow_case(this), done(this),
+ grow_bailout(this, Label::kDeferred);
+
+ TNode<BoolT> condition;
+ if (IsHoleyElementsKind(kind)) {
+ condition = UintPtrGreaterThanOrEqual(key, length);
+ } else {
+ // We don't support growing here unless the value is being appended.
+ condition = WordEqual(key, length);
+ }
+ Branch(condition, &grow_case, &no_grow_case);
+
+ BIND(&grow_case);
+ {
+ TNode<IntPtrT> current_capacity =
+ SmiUntag(LoadFixedArrayBaseLength(elements));
+ checked_elements = elements;
+ Label fits_capacity(this);
+ // If key is negative, we will notice in Runtime::kGrowArrayElements.
+ GotoIf(UintPtrLessThan(key, current_capacity), &fits_capacity);
+
+ {
+ TNode<FixedArrayBase> new_elements = TryGrowElementsCapacity(
+ object, elements, kind, key, current_capacity, &grow_bailout);
+ checked_elements = new_elements;
+ Goto(&fits_capacity);
+ }
+
+ BIND(&grow_bailout);
+ {
+ GotoIf(IntPtrLessThan(key, IntPtrConstant(0)), bailout);
+ TNode<Number> tagged_key = ChangeUintPtrToTagged(Unsigned(key));
+ TNode<Object> maybe_elements = CallRuntime(
+ Runtime::kGrowArrayElements, NoContextConstant(), object, tagged_key);
+ GotoIf(TaggedIsSmi(maybe_elements), bailout);
+ TNode<FixedArrayBase> new_elements = CAST(maybe_elements);
+ CSA_ASSERT(this, IsFixedArrayWithKind(new_elements, kind));
+ checked_elements = new_elements;
+ Goto(&fits_capacity);
+ }
+
+ BIND(&fits_capacity);
+ GotoIfNot(IsJSArray(object), &done);
+
+ TNode<IntPtrT> new_length = IntPtrAdd(key, IntPtrConstant(1));
+ StoreObjectFieldNoWriteBarrier(object, JSArray::kLengthOffset,
+ SmiTag(new_length));
+ Goto(&done);
+ }
+
+ BIND(&no_grow_case);
+ {
+ GotoIfNot(UintPtrLessThan(key, length), bailout);
+ checked_elements = elements;
+ Goto(&done);
+ }
+
+ BIND(&done);
+ return checked_elements.value();
+}
+
+TNode<FixedArrayBase> CodeStubAssembler::CopyElementsOnWrite(
+ TNode<HeapObject> object, TNode<FixedArrayBase> elements, ElementsKind kind,
+ TNode<IntPtrT> length, Label* bailout) {
+ TVARIABLE(FixedArrayBase, new_elements_var, elements);
+ Label done(this);
+
+ GotoIfNot(IsFixedCOWArrayMap(LoadMap(elements)), &done);
+ {
+ TNode<IntPtrT> capacity = SmiUntag(LoadFixedArrayBaseLength(elements));
+ TNode<FixedArrayBase> new_elements = GrowElementsCapacity(
+ object, elements, kind, kind, length, capacity, bailout);
+ new_elements_var = new_elements;
+ Goto(&done);
+ }
+
+ BIND(&done);
+ return new_elements_var.value();
+}
+
+void CodeStubAssembler::TransitionElementsKind(TNode<JSObject> object,
+ TNode<Map> map,
+ ElementsKind from_kind,
+ ElementsKind to_kind,
+ Label* bailout) {
+ DCHECK(!IsHoleyElementsKind(from_kind) || IsHoleyElementsKind(to_kind));
+ if (AllocationSite::ShouldTrack(from_kind, to_kind)) {
+ TrapAllocationMemento(object, bailout);
+ }
+
+ if (!IsSimpleMapChangeTransition(from_kind, to_kind)) {
+ Comment("Non-simple map transition");
+ TNode<FixedArrayBase> elements = LoadElements(object);
+
+ Label done(this);
+ GotoIf(TaggedEqual(elements, EmptyFixedArrayConstant()), &done);
+
+ // TODO(ishell): Use BInt for elements_length and array_length.
+ TNode<IntPtrT> elements_length =
+ SmiUntag(LoadFixedArrayBaseLength(elements));
+ TNode<IntPtrT> array_length = Select<IntPtrT>(
+ IsJSArray(object),
+ [=]() {
+ CSA_ASSERT(this, IsFastElementsKind(LoadElementsKind(object)));
+ return SmiUntag(LoadFastJSArrayLength(CAST(object)));
+ },
+ [=]() { return elements_length; });
+
+ CSA_ASSERT(this, WordNotEqual(elements_length, IntPtrConstant(0)));
+
+ GrowElementsCapacity(object, elements, from_kind, to_kind, array_length,
+ elements_length, bailout);
+ Goto(&done);
+ BIND(&done);
+ }
+
+ StoreMap(object, map);
+}
+
+void CodeStubAssembler::TrapAllocationMemento(TNode<JSObject> object,
+ Label* memento_found) {
+ Comment("[ TrapAllocationMemento");
+ Label no_memento_found(this);
+ Label top_check(this), map_check(this);
+
+ TNode<ExternalReference> new_space_top_address = ExternalConstant(
+ ExternalReference::new_space_allocation_top_address(isolate()));
+ const int kMementoMapOffset = JSArray::kHeaderSize;
+ const int kMementoLastWordOffset =
+ kMementoMapOffset + AllocationMemento::kSize - kTaggedSize;
+
+ // Bail out if the object is not in new space.
+ TNode<IntPtrT> object_word = BitcastTaggedToWord(object);
+ TNode<IntPtrT> object_page = PageFromAddress(object_word);
+ {
+ TNode<IntPtrT> page_flags =
+ Load<IntPtrT>(object_page, IntPtrConstant(Page::kFlagsOffset));
+ GotoIf(WordEqual(
+ WordAnd(page_flags,
+ IntPtrConstant(MemoryChunk::kIsInYoungGenerationMask)),
+ IntPtrConstant(0)),
+ &no_memento_found);
+ // TODO(ulan): Support allocation memento for a large object by allocating
+ // additional word for the memento after the large object.
+ GotoIf(WordNotEqual(WordAnd(page_flags,
+ IntPtrConstant(MemoryChunk::kIsLargePageMask)),
+ IntPtrConstant(0)),
+ &no_memento_found);
+ }
+
+ TNode<IntPtrT> memento_last_word = IntPtrAdd(
+ object_word, IntPtrConstant(kMementoLastWordOffset - kHeapObjectTag));
+ TNode<IntPtrT> memento_last_word_page = PageFromAddress(memento_last_word);
+
+ TNode<IntPtrT> new_space_top = Load<IntPtrT>(new_space_top_address);
+ TNode<IntPtrT> new_space_top_page = PageFromAddress(new_space_top);
+
+ // If the object is in new space, we need to check whether respective
+ // potential memento object is on the same page as the current top.
+ GotoIf(WordEqual(memento_last_word_page, new_space_top_page), &top_check);
+
+ // The object is on a different page than allocation top. Bail out if the
+ // object sits on the page boundary as no memento can follow and we cannot
+ // touch the memory following it.
+ Branch(WordEqual(object_page, memento_last_word_page), &map_check,
+ &no_memento_found);
+
+ // If top is on the same page as the current object, we need to check whether
+ // we are below top.
+ BIND(&top_check);
+ {
+ Branch(UintPtrGreaterThanOrEqual(memento_last_word, new_space_top),
+ &no_memento_found, &map_check);
+ }
+
+ // Memento map check.
+ BIND(&map_check);
+ {
+ TNode<Object> memento_map = LoadObjectField(object, kMementoMapOffset);
+ Branch(TaggedEqual(memento_map, AllocationMementoMapConstant()),
+ memento_found, &no_memento_found);
+ }
+ BIND(&no_memento_found);
+ Comment("] TrapAllocationMemento");
+}
+
+TNode<IntPtrT> CodeStubAssembler::PageFromAddress(TNode<IntPtrT> address) {
+ return WordAnd(address, IntPtrConstant(~kPageAlignmentMask));
+}
+
+TNode<AllocationSite> CodeStubAssembler::CreateAllocationSiteInFeedbackVector(
+ TNode<FeedbackVector> feedback_vector, TNode<UintPtrT> slot) {
+ TNode<IntPtrT> size = IntPtrConstant(AllocationSite::kSizeWithWeakNext);
+ TNode<HeapObject> site = Allocate(size, CodeStubAssembler::kPretenured);
+ StoreMapNoWriteBarrier(site, RootIndex::kAllocationSiteWithWeakNextMap);
+ // Should match AllocationSite::Initialize.
+ TNode<WordT> field = UpdateWord<AllocationSite::ElementsKindBits>(
+ IntPtrConstant(0), UintPtrConstant(GetInitialFastElementsKind()));
+ StoreObjectFieldNoWriteBarrier(
+ site, AllocationSite::kTransitionInfoOrBoilerplateOffset,
+ SmiTag(Signed(field)));
+
+ // Unlike literals, constructed arrays don't have nested sites
+ TNode<Smi> zero = SmiConstant(0);
+ StoreObjectFieldNoWriteBarrier(site, AllocationSite::kNestedSiteOffset, zero);
+
+ // Pretenuring calculation field.
+ StoreObjectFieldNoWriteBarrier(site, AllocationSite::kPretenureDataOffset,
+ Int32Constant(0));
+
+ // Pretenuring memento creation count field.
+ StoreObjectFieldNoWriteBarrier(
+ site, AllocationSite::kPretenureCreateCountOffset, Int32Constant(0));
+
+ // Store an empty fixed array for the code dependency.
+ StoreObjectFieldRoot(site, AllocationSite::kDependentCodeOffset,
+ RootIndex::kEmptyWeakFixedArray);
+
+ // Link the object to the allocation site list
+ TNode<ExternalReference> site_list = ExternalConstant(
+ ExternalReference::allocation_sites_list_address(isolate()));
+ TNode<Object> next_site =
+ LoadBufferObject(ReinterpretCast<RawPtrT>(site_list), 0);
+
+ // TODO(mvstanton): This is a store to a weak pointer, which we may want to
+ // mark as such in order to skip the write barrier, once we have a unified
+ // system for weakness. For now we decided to keep it like this because having
+ // an initial write barrier backed store makes this pointer strong until the
+ // next GC, and allocation sites are designed to survive several GCs anyway.
+ StoreObjectField(site, AllocationSite::kWeakNextOffset, next_site);
+ StoreFullTaggedNoWriteBarrier(site_list, site);
+
+ StoreFeedbackVectorSlot(feedback_vector, slot, site);
+ return CAST(site);
+}
+
+TNode<MaybeObject> CodeStubAssembler::StoreWeakReferenceInFeedbackVector(
+ TNode<FeedbackVector> feedback_vector, TNode<UintPtrT> slot,
+ TNode<HeapObject> value, int additional_offset) {
+ TNode<MaybeObject> weak_value = MakeWeak(value);
+ StoreFeedbackVectorSlot(feedback_vector, slot, weak_value,
+ UPDATE_WRITE_BARRIER, additional_offset);
+ return weak_value;
+}
+
+TNode<BoolT> CodeStubAssembler::HasBoilerplate(
+ TNode<Object> maybe_literal_site) {
+ return TaggedIsNotSmi(maybe_literal_site);
+}
+
+TNode<Smi> CodeStubAssembler::LoadTransitionInfo(
+ TNode<AllocationSite> allocation_site) {
+ TNode<Smi> transition_info = CAST(LoadObjectField(
+ allocation_site, AllocationSite::kTransitionInfoOrBoilerplateOffset));
+ return transition_info;
+}
+
+TNode<JSObject> CodeStubAssembler::LoadBoilerplate(
+ TNode<AllocationSite> allocation_site) {
+ TNode<JSObject> boilerplate = CAST(LoadObjectField(
+ allocation_site, AllocationSite::kTransitionInfoOrBoilerplateOffset));
+ return boilerplate;
+}
+
+TNode<Int32T> CodeStubAssembler::LoadElementsKind(
+ TNode<AllocationSite> allocation_site) {
+ TNode<Smi> transition_info = LoadTransitionInfo(allocation_site);
+ TNode<Int32T> elements_kind =
+ Signed(DecodeWord32<AllocationSite::ElementsKindBits>(
+ SmiToInt32(transition_info)));
+ CSA_ASSERT(this, IsFastElementsKind(elements_kind));
+ return elements_kind;
+}
+
+template <typename TIndex>
+TNode<TIndex> CodeStubAssembler::BuildFastLoop(const VariableList& vars,
+ TNode<TIndex> start_index,
+ TNode<TIndex> end_index,
+ const FastLoopBody<TIndex>& body,
+ int increment,
+ IndexAdvanceMode advance_mode) {
+ TVARIABLE(TIndex, var, start_index);
+ VariableList vars_copy(vars.begin(), vars.end(), zone());
+ vars_copy.push_back(&var);
+ Label loop(this, vars_copy);
+ Label after_loop(this);
+ // Introduce an explicit second check of the termination condition before the
+ // loop that helps turbofan generate better code. If there's only a single
+ // check, then the CodeStubAssembler forces it to be at the beginning of the
+ // loop requiring a backwards branch at the end of the loop (it's not possible
+ // to force the loop header check at the end of the loop and branch forward to
+ // it from the pre-header). The extra branch is slower in the case that the
+ // loop actually iterates.
+ TNode<BoolT> first_check = IntPtrOrSmiEqual(var.value(), end_index);
+ int32_t first_check_val;
+ if (ToInt32Constant(first_check, &first_check_val)) {
+ if (first_check_val) return var.value();
+ Goto(&loop);
+ } else {
+ Branch(first_check, &after_loop, &loop);
+ }
+
+ BIND(&loop);
+ {
+ if (advance_mode == IndexAdvanceMode::kPre) {
+ Increment(&var, increment);
+ }
+ body(var.value());
+ if (advance_mode == IndexAdvanceMode::kPost) {
+ Increment(&var, increment);
+ }
+ Branch(IntPtrOrSmiNotEqual(var.value(), end_index), &loop, &after_loop);
+ }
+ BIND(&after_loop);
+ return var.value();
+}
+
+// Instantiate BuildFastLoop for IntPtrT and UintPtrT.
+template V8_EXPORT_PRIVATE TNode<IntPtrT>
+CodeStubAssembler::BuildFastLoop<IntPtrT>(const VariableList& vars,
+ TNode<IntPtrT> start_index,
+ TNode<IntPtrT> end_index,
+ const FastLoopBody<IntPtrT>& body,
+ int increment,
+ IndexAdvanceMode advance_mode);
+template V8_EXPORT_PRIVATE TNode<UintPtrT>
+CodeStubAssembler::BuildFastLoop<UintPtrT>(const VariableList& vars,
+ TNode<UintPtrT> start_index,
+ TNode<UintPtrT> end_index,
+ const FastLoopBody<UintPtrT>& body,
+ int increment,
+ IndexAdvanceMode advance_mode);
+
+template <typename TIndex>
+void CodeStubAssembler::BuildFastArrayForEach(
+ TNode<UnionT<UnionT<FixedArray, PropertyArray>, HeapObject>> array,
+ ElementsKind kind, TNode<TIndex> first_element_inclusive,
+ TNode<TIndex> last_element_exclusive, const FastArrayForEachBody& body,
+ ForEachDirection direction) {
+ STATIC_ASSERT(FixedArray::kHeaderSize == FixedDoubleArray::kHeaderSize);
+ CSA_SLOW_ASSERT(this, Word32Or(IsFixedArrayWithKind(array, kind),
+ IsPropertyArray(array)));
+
+ int32_t first_val;
+ bool constant_first = ToInt32Constant(first_element_inclusive, &first_val);
+ int32_t last_val;
+ bool constent_last = ToInt32Constant(last_element_exclusive, &last_val);
+ if (constant_first && constent_last) {
+ int delta = last_val - first_val;
+ DCHECK_GE(delta, 0);
+ if (delta <= kElementLoopUnrollThreshold) {
+ if (direction == ForEachDirection::kForward) {
+ for (int i = first_val; i < last_val; ++i) {
+ TNode<IntPtrT> index = IntPtrConstant(i);
+ TNode<IntPtrT> offset = ElementOffsetFromIndex(
+ index, kind, FixedArray::kHeaderSize - kHeapObjectTag);
+ body(array, offset);
+ }
+ } else {
+ for (int i = last_val - 1; i >= first_val; --i) {
+ TNode<IntPtrT> index = IntPtrConstant(i);
+ TNode<IntPtrT> offset = ElementOffsetFromIndex(
+ index, kind, FixedArray::kHeaderSize - kHeapObjectTag);
+ body(array, offset);
+ }
+ }
+ return;
+ }
+ }
+
+ TNode<IntPtrT> start = ElementOffsetFromIndex(
+ first_element_inclusive, kind, FixedArray::kHeaderSize - kHeapObjectTag);
+ TNode<IntPtrT> limit = ElementOffsetFromIndex(
+ last_element_exclusive, kind, FixedArray::kHeaderSize - kHeapObjectTag);
+ if (direction == ForEachDirection::kReverse) std::swap(start, limit);
+
+ int increment = IsDoubleElementsKind(kind) ? kDoubleSize : kTaggedSize;
+ BuildFastLoop<IntPtrT>(
+ start, limit, [&](TNode<IntPtrT> offset) { body(array, offset); },
+ direction == ForEachDirection::kReverse ? -increment : increment,
+ direction == ForEachDirection::kReverse ? IndexAdvanceMode::kPre
+ : IndexAdvanceMode::kPost);
+}
+
+template <typename TIndex>
+void CodeStubAssembler::GotoIfFixedArraySizeDoesntFitInNewSpace(
+ TNode<TIndex> element_count, Label* doesnt_fit, int base_size) {
+ GotoIf(FixedArraySizeDoesntFitInNewSpace(element_count, base_size),
+ doesnt_fit);
+}
+
+void CodeStubAssembler::InitializeFieldsWithRoot(TNode<HeapObject> object,
+ TNode<IntPtrT> start_offset,
+ TNode<IntPtrT> end_offset,
+ RootIndex root_index) {
+ CSA_SLOW_ASSERT(this, TaggedIsNotSmi(object));
+ start_offset = IntPtrAdd(start_offset, IntPtrConstant(-kHeapObjectTag));
+ end_offset = IntPtrAdd(end_offset, IntPtrConstant(-kHeapObjectTag));
+ TNode<Object> root_value = LoadRoot(root_index);
+ BuildFastLoop<IntPtrT>(
+ end_offset, start_offset,
+ [=](TNode<IntPtrT> current) {
+ StoreNoWriteBarrier(MachineRepresentation::kTagged, object, current,
+ root_value);
+ },
+ -kTaggedSize, CodeStubAssembler::IndexAdvanceMode::kPre);
+}
+
+void CodeStubAssembler::BranchIfNumberRelationalComparison(Operation op,
+ TNode<Number> left,
+ TNode<Number> right,
+ Label* if_true,
+ Label* if_false) {
+ Label do_float_comparison(this);
+ TVARIABLE(Float64T, var_left_float);
+ TVARIABLE(Float64T, var_right_float);
+
+ Branch(
+ TaggedIsSmi(left),
+ [&] {
+ TNode<Smi> smi_left = CAST(left);
+
+ Branch(
+ TaggedIsSmi(right),
+ [&] {
+ TNode<Smi> smi_right = CAST(right);
+
+ // Both {left} and {right} are Smi, so just perform a fast
+ // Smi comparison.
+ switch (op) {
+ case Operation::kEqual:
+ BranchIfSmiEqual(smi_left, smi_right, if_true, if_false);
+ break;
+ case Operation::kLessThan:
+ BranchIfSmiLessThan(smi_left, smi_right, if_true, if_false);
+ break;
+ case Operation::kLessThanOrEqual:
+ BranchIfSmiLessThanOrEqual(smi_left, smi_right, if_true,
+ if_false);
+ break;
+ case Operation::kGreaterThan:
+ BranchIfSmiLessThan(smi_right, smi_left, if_true, if_false);
+ break;
+ case Operation::kGreaterThanOrEqual:
+ BranchIfSmiLessThanOrEqual(smi_right, smi_left, if_true,
+ if_false);
+ break;
+ default:
+ UNREACHABLE();
+ }
+ },
+ [&] {
+ var_left_float = SmiToFloat64(smi_left);
+ var_right_float = LoadHeapNumberValue(CAST(right));
+ Goto(&do_float_comparison);
+ });
+ },
+ [&] {
+ var_left_float = LoadHeapNumberValue(CAST(left));
+
+ Branch(
+ TaggedIsSmi(right),
+ [&] {
+ var_right_float = SmiToFloat64(CAST(right));
+ Goto(&do_float_comparison);
+ },
+ [&] {
+ var_right_float = LoadHeapNumberValue(CAST(right));
+ Goto(&do_float_comparison);
+ });
+ });
+
+ BIND(&do_float_comparison);
+ {
+ switch (op) {
+ case Operation::kEqual:
+ Branch(Float64Equal(var_left_float.value(), var_right_float.value()),
+ if_true, if_false);
+ break;
+ case Operation::kLessThan:
+ Branch(Float64LessThan(var_left_float.value(), var_right_float.value()),
+ if_true, if_false);
+ break;
+ case Operation::kLessThanOrEqual:
+ Branch(Float64LessThanOrEqual(var_left_float.value(),
+ var_right_float.value()),
+ if_true, if_false);
+ break;
+ case Operation::kGreaterThan:
+ Branch(
+ Float64GreaterThan(var_left_float.value(), var_right_float.value()),
+ if_true, if_false);
+ break;
+ case Operation::kGreaterThanOrEqual:
+ Branch(Float64GreaterThanOrEqual(var_left_float.value(),
+ var_right_float.value()),
+ if_true, if_false);
+ break;
+ default:
+ UNREACHABLE();
+ }
+ }
+}
+
+void CodeStubAssembler::GotoIfNumberGreaterThanOrEqual(TNode<Number> left,
+ TNode<Number> right,
+ Label* if_true) {
+ Label if_false(this);
+ BranchIfNumberRelationalComparison(Operation::kGreaterThanOrEqual, left,
+ right, if_true, &if_false);
+ BIND(&if_false);
+}
+
+namespace {
+Operation Reverse(Operation op) {
+ switch (op) {
+ case Operation::kLessThan:
+ return Operation::kGreaterThan;
+ case Operation::kLessThanOrEqual:
+ return Operation::kGreaterThanOrEqual;
+ case Operation::kGreaterThan:
+ return Operation::kLessThan;
+ case Operation::kGreaterThanOrEqual:
+ return Operation::kLessThanOrEqual;
+ default:
+ break;
+ }
+ UNREACHABLE();
+}
+} // anonymous namespace
+
+TNode<Oddball> CodeStubAssembler::RelationalComparison(
+ Operation op, TNode<Object> left, TNode<Object> right,
+ TNode<Context> context, TVariable<Smi>* var_type_feedback) {
+ Label return_true(this), return_false(this), do_float_comparison(this),
+ end(this);
+ TVARIABLE(Oddball, var_result); // Actually only "true" or "false".
+ TVARIABLE(Float64T, var_left_float);
+ TVARIABLE(Float64T, var_right_float);
+
+ // We might need to loop several times due to ToPrimitive and/or ToNumeric
+ // conversions.
+ TVARIABLE(Object, var_left, left);
+ TVARIABLE(Object, var_right, right);
+ VariableList loop_variable_list({&var_left, &var_right}, zone());
+ if (var_type_feedback != nullptr) {
+ // Initialize the type feedback to None. The current feedback is combined
+ // with the previous feedback.
+ *var_type_feedback = SmiConstant(CompareOperationFeedback::kNone);
+ loop_variable_list.push_back(var_type_feedback);
+ }
+ Label loop(this, loop_variable_list);
+ Goto(&loop);
+ BIND(&loop);
+ {
+ left = var_left.value();
+ right = var_right.value();
+
+ Label if_left_smi(this), if_left_not_smi(this);
+ Branch(TaggedIsSmi(left), &if_left_smi, &if_left_not_smi);
+
+ BIND(&if_left_smi);
+ {
+ TNode<Smi> smi_left = CAST(left);
+ Label if_right_smi(this), if_right_heapnumber(this),
+ if_right_bigint(this, Label::kDeferred),
+ if_right_not_numeric(this, Label::kDeferred);
+ GotoIf(TaggedIsSmi(right), &if_right_smi);
+ TNode<Map> right_map = LoadMap(CAST(right));
+ GotoIf(IsHeapNumberMap(right_map), &if_right_heapnumber);
+ TNode<Uint16T> right_instance_type = LoadMapInstanceType(right_map);
+ Branch(IsBigIntInstanceType(right_instance_type), &if_right_bigint,
+ &if_right_not_numeric);
+
+ BIND(&if_right_smi);
+ {
+ TNode<Smi> smi_right = CAST(right);
+ CombineFeedback(var_type_feedback,
+ CompareOperationFeedback::kSignedSmall);
+ switch (op) {
+ case Operation::kLessThan:
+ BranchIfSmiLessThan(smi_left, smi_right, &return_true,
+ &return_false);
+ break;
+ case Operation::kLessThanOrEqual:
+ BranchIfSmiLessThanOrEqual(smi_left, smi_right, &return_true,
+ &return_false);
+ break;
+ case Operation::kGreaterThan:
+ BranchIfSmiLessThan(smi_right, smi_left, &return_true,
+ &return_false);
+ break;
+ case Operation::kGreaterThanOrEqual:
+ BranchIfSmiLessThanOrEqual(smi_right, smi_left, &return_true,
+ &return_false);
+ break;
+ default:
+ UNREACHABLE();
+ }
+ }
+
+ BIND(&if_right_heapnumber);
+ {
+ CombineFeedback(var_type_feedback, CompareOperationFeedback::kNumber);
+ var_left_float = SmiToFloat64(smi_left);
+ var_right_float = LoadHeapNumberValue(CAST(right));
+ Goto(&do_float_comparison);
+ }
+
+ BIND(&if_right_bigint);
+ {
+ OverwriteFeedback(var_type_feedback, CompareOperationFeedback::kAny);
+ var_result = CAST(CallRuntime(Runtime::kBigIntCompareToNumber,
+ NoContextConstant(),
+ SmiConstant(Reverse(op)), right, left));
+ Goto(&end);
+ }
+
+ BIND(&if_right_not_numeric);
+ {
+ OverwriteFeedback(var_type_feedback, CompareOperationFeedback::kAny);
+ // Convert {right} to a Numeric; we don't need to perform the
+ // dedicated ToPrimitive(right, hint Number) operation, as the
+ // ToNumeric(right) will by itself already invoke ToPrimitive with
+ // a Number hint.
+ var_right = CallBuiltin(Builtins::kNonNumberToNumeric, context, right);
+ Goto(&loop);
+ }
+ }
+
+ BIND(&if_left_not_smi);
+ {
+ TNode<Map> left_map = LoadMap(CAST(left));
+
+ Label if_right_smi(this), if_right_not_smi(this);
+ Branch(TaggedIsSmi(right), &if_right_smi, &if_right_not_smi);
+
+ BIND(&if_right_smi);
+ {
+ Label if_left_heapnumber(this), if_left_bigint(this, Label::kDeferred),
+ if_left_not_numeric(this, Label::kDeferred);
+ GotoIf(IsHeapNumberMap(left_map), &if_left_heapnumber);
+ TNode<Uint16T> left_instance_type = LoadMapInstanceType(left_map);
+ Branch(IsBigIntInstanceType(left_instance_type), &if_left_bigint,
+ &if_left_not_numeric);
+
+ BIND(&if_left_heapnumber);
+ {
+ CombineFeedback(var_type_feedback, CompareOperationFeedback::kNumber);
+ var_left_float = LoadHeapNumberValue(CAST(left));
+ var_right_float = SmiToFloat64(CAST(right));
+ Goto(&do_float_comparison);
+ }
+
+ BIND(&if_left_bigint);
+ {
+ OverwriteFeedback(var_type_feedback, CompareOperationFeedback::kAny);
+ var_result = CAST(CallRuntime(Runtime::kBigIntCompareToNumber,
+ NoContextConstant(), SmiConstant(op),
+ left, right));
+ Goto(&end);
+ }
+
+ BIND(&if_left_not_numeric);
+ {
+ OverwriteFeedback(var_type_feedback, CompareOperationFeedback::kAny);
+ // Convert {left} to a Numeric; we don't need to perform the
+ // dedicated ToPrimitive(left, hint Number) operation, as the
+ // ToNumeric(left) will by itself already invoke ToPrimitive with
+ // a Number hint.
+ var_left = CallBuiltin(Builtins::kNonNumberToNumeric, context, left);
+ Goto(&loop);
+ }
+ }
+
+ BIND(&if_right_not_smi);
+ {
+ TNode<Map> right_map = LoadMap(CAST(right));
+
+ Label if_left_heapnumber(this), if_left_bigint(this, Label::kDeferred),
+ if_left_string(this, Label::kDeferred),
+ if_left_other(this, Label::kDeferred);
+ GotoIf(IsHeapNumberMap(left_map), &if_left_heapnumber);
+ TNode<Uint16T> left_instance_type = LoadMapInstanceType(left_map);
+ GotoIf(IsBigIntInstanceType(left_instance_type), &if_left_bigint);
+ Branch(IsStringInstanceType(left_instance_type), &if_left_string,
+ &if_left_other);
+
+ BIND(&if_left_heapnumber);
+ {
+ Label if_right_heapnumber(this),
+ if_right_bigint(this, Label::kDeferred),
+ if_right_not_numeric(this, Label::kDeferred);
+ GotoIf(TaggedEqual(right_map, left_map), &if_right_heapnumber);
+ TNode<Uint16T> right_instance_type = LoadMapInstanceType(right_map);
+ Branch(IsBigIntInstanceType(right_instance_type), &if_right_bigint,
+ &if_right_not_numeric);
+
+ BIND(&if_right_heapnumber);
+ {
+ CombineFeedback(var_type_feedback,
+ CompareOperationFeedback::kNumber);
+ var_left_float = LoadHeapNumberValue(CAST(left));
+ var_right_float = LoadHeapNumberValue(CAST(right));
+ Goto(&do_float_comparison);
+ }
+
+ BIND(&if_right_bigint);
+ {
+ OverwriteFeedback(var_type_feedback,
+ CompareOperationFeedback::kAny);
+ var_result = CAST(CallRuntime(
+ Runtime::kBigIntCompareToNumber, NoContextConstant(),
+ SmiConstant(Reverse(op)), right, left));
+ Goto(&end);
+ }
+
+ BIND(&if_right_not_numeric);
+ {
+ OverwriteFeedback(var_type_feedback,
+ CompareOperationFeedback::kAny);
+ // Convert {right} to a Numeric; we don't need to perform
+ // dedicated ToPrimitive(right, hint Number) operation, as the
+ // ToNumeric(right) will by itself already invoke ToPrimitive with
+ // a Number hint.
+ var_right =
+ CallBuiltin(Builtins::kNonNumberToNumeric, context, right);
+ Goto(&loop);
+ }
+ }
+
+ BIND(&if_left_bigint);
+ {
+ Label if_right_heapnumber(this), if_right_bigint(this),
+ if_right_string(this), if_right_other(this);
+ GotoIf(IsHeapNumberMap(right_map), &if_right_heapnumber);
+ TNode<Uint16T> right_instance_type = LoadMapInstanceType(right_map);
+ GotoIf(IsBigIntInstanceType(right_instance_type), &if_right_bigint);
+ Branch(IsStringInstanceType(right_instance_type), &if_right_string,
+ &if_right_other);
+
+ BIND(&if_right_heapnumber);
+ {
+ OverwriteFeedback(var_type_feedback,
+ CompareOperationFeedback::kAny);
+ var_result = CAST(CallRuntime(Runtime::kBigIntCompareToNumber,
+ NoContextConstant(), SmiConstant(op),
+ left, right));
+ Goto(&end);
+ }
+
+ BIND(&if_right_bigint);
+ {
+ CombineFeedback(var_type_feedback,
+ CompareOperationFeedback::kBigInt);
+ var_result = CAST(CallRuntime(Runtime::kBigIntCompareToBigInt,
+ NoContextConstant(), SmiConstant(op),
+ left, right));
+ Goto(&end);
+ }
+
+ BIND(&if_right_string);
+ {
+ OverwriteFeedback(var_type_feedback,
+ CompareOperationFeedback::kAny);
+ var_result = CAST(CallRuntime(Runtime::kBigIntCompareToString,
+ NoContextConstant(), SmiConstant(op),
+ left, right));
+ Goto(&end);
+ }
+
+ // {right} is not a Number, BigInt, or String.
+ BIND(&if_right_other);
+ {
+ OverwriteFeedback(var_type_feedback,
+ CompareOperationFeedback::kAny);
+ // Convert {right} to a Numeric; we don't need to perform
+ // dedicated ToPrimitive(right, hint Number) operation, as the
+ // ToNumeric(right) will by itself already invoke ToPrimitive with
+ // a Number hint.
+ var_right =
+ CallBuiltin(Builtins::kNonNumberToNumeric, context, right);
+ Goto(&loop);
+ }
+ }
+
+ BIND(&if_left_string);
+ {
+ TNode<Uint16T> right_instance_type = LoadMapInstanceType(right_map);
+
+ Label if_right_not_string(this, Label::kDeferred);
+ GotoIfNot(IsStringInstanceType(right_instance_type),
+ &if_right_not_string);
+
+ // Both {left} and {right} are strings.
+ CombineFeedback(var_type_feedback, CompareOperationFeedback::kString);
+ Builtins::Name builtin;
+ switch (op) {
+ case Operation::kLessThan:
+ builtin = Builtins::kStringLessThan;
+ break;
+ case Operation::kLessThanOrEqual:
+ builtin = Builtins::kStringLessThanOrEqual;
+ break;
+ case Operation::kGreaterThan:
+ builtin = Builtins::kStringGreaterThan;
+ break;
+ case Operation::kGreaterThanOrEqual:
+ builtin = Builtins::kStringGreaterThanOrEqual;
+ break;
+ default:
+ UNREACHABLE();
+ }
+ var_result = CAST(CallBuiltin(builtin, context, left, right));
+ Goto(&end);
+
+ BIND(&if_right_not_string);
+ {
+ OverwriteFeedback(var_type_feedback,
+ CompareOperationFeedback::kAny);
+ // {left} is a String, while {right} isn't. Check if {right} is
+ // a BigInt, otherwise call ToPrimitive(right, hint Number) if
+ // {right} is a receiver, or ToNumeric(left) and then
+ // ToNumeric(right) in the other cases.
+ STATIC_ASSERT(LAST_JS_RECEIVER_TYPE == LAST_TYPE);
+ Label if_right_bigint(this),
+ if_right_receiver(this, Label::kDeferred);
+ GotoIf(IsBigIntInstanceType(right_instance_type), &if_right_bigint);
+ GotoIf(IsJSReceiverInstanceType(right_instance_type),
+ &if_right_receiver);
+
+ var_left =
+ CallBuiltin(Builtins::kNonNumberToNumeric, context, left);
+ var_right = CallBuiltin(Builtins::kToNumeric, context, right);
+ Goto(&loop);
+
+ BIND(&if_right_bigint);
+ {
+ var_result = CAST(CallRuntime(
+ Runtime::kBigIntCompareToString, NoContextConstant(),
+ SmiConstant(Reverse(op)), right, left));
+ Goto(&end);
+ }
+
+ BIND(&if_right_receiver);
+ {
+ Callable callable = CodeFactory::NonPrimitiveToPrimitive(
+ isolate(), ToPrimitiveHint::kNumber);
+ var_right = CallStub(callable, context, right);
+ Goto(&loop);
+ }
+ }
+ }
+
+ BIND(&if_left_other);
+ {
+ // {left} is neither a Numeric nor a String, and {right} is not a Smi.
+ if (var_type_feedback != nullptr) {
+ // Collect NumberOrOddball feedback if {left} is an Oddball
+ // and {right} is either a HeapNumber or Oddball. Otherwise collect
+ // Any feedback.
+ Label collect_any_feedback(this), collect_oddball_feedback(this),
+ collect_feedback_done(this);
+ GotoIfNot(InstanceTypeEqual(left_instance_type, ODDBALL_TYPE),
+ &collect_any_feedback);
+
+ GotoIf(IsHeapNumberMap(right_map), &collect_oddball_feedback);
+ TNode<Uint16T> right_instance_type = LoadMapInstanceType(right_map);
+ Branch(InstanceTypeEqual(right_instance_type, ODDBALL_TYPE),
+ &collect_oddball_feedback, &collect_any_feedback);
+
+ BIND(&collect_oddball_feedback);
+ {
+ CombineFeedback(var_type_feedback,
+ CompareOperationFeedback::kNumberOrOddball);
+ Goto(&collect_feedback_done);
+ }
+
+ BIND(&collect_any_feedback);
+ {
+ OverwriteFeedback(var_type_feedback,
+ CompareOperationFeedback::kAny);
+ Goto(&collect_feedback_done);
+ }
+
+ BIND(&collect_feedback_done);
+ }
+
+ // If {left} is a receiver, call ToPrimitive(left, hint Number).
+ // Otherwise call ToNumeric(right) and then ToNumeric(left), the
+ // order here is important as it's observable by user code.
+ STATIC_ASSERT(LAST_JS_RECEIVER_TYPE == LAST_TYPE);
+ Label if_left_receiver(this, Label::kDeferred);
+ GotoIf(IsJSReceiverInstanceType(left_instance_type),
+ &if_left_receiver);
+
+ var_right = CallBuiltin(Builtins::kToNumeric, context, right);
+ var_left = CallBuiltin(Builtins::kNonNumberToNumeric, context, left);
+ Goto(&loop);
+
+ BIND(&if_left_receiver);
+ {
+ Callable callable = CodeFactory::NonPrimitiveToPrimitive(
+ isolate(), ToPrimitiveHint::kNumber);
+ var_left = CallStub(callable, context, left);
+ Goto(&loop);
+ }
+ }
+ }
+ }
+ }
+
+ BIND(&do_float_comparison);
+ {
+ switch (op) {
+ case Operation::kLessThan:
+ Branch(Float64LessThan(var_left_float.value(), var_right_float.value()),
+ &return_true, &return_false);
+ break;
+ case Operation::kLessThanOrEqual:
+ Branch(Float64LessThanOrEqual(var_left_float.value(),
+ var_right_float.value()),
+ &return_true, &return_false);
+ break;
+ case Operation::kGreaterThan:
+ Branch(
+ Float64GreaterThan(var_left_float.value(), var_right_float.value()),
+ &return_true, &return_false);
+ break;
+ case Operation::kGreaterThanOrEqual:
+ Branch(Float64GreaterThanOrEqual(var_left_float.value(),
+ var_right_float.value()),
+ &return_true, &return_false);
+ break;
+ default:
+ UNREACHABLE();
+ }
+ }
+
+ BIND(&return_true);
+ {
+ var_result = TrueConstant();
+ Goto(&end);
+ }
+
+ BIND(&return_false);
+ {
+ var_result = FalseConstant();
+ Goto(&end);
+ }
+
+ BIND(&end);
+ return var_result.value();
+}
+
+TNode<Smi> CodeStubAssembler::CollectFeedbackForString(
+ SloppyTNode<Int32T> instance_type) {
+ TNode<Smi> feedback = SelectSmiConstant(
+ Word32Equal(
+ Word32And(instance_type, Int32Constant(kIsNotInternalizedMask)),
+ Int32Constant(kInternalizedTag)),
+ CompareOperationFeedback::kInternalizedString,
+ CompareOperationFeedback::kString);
+ return feedback;
+}
+
+void CodeStubAssembler::GenerateEqual_Same(SloppyTNode<Object> value,
+ Label* if_equal, Label* if_notequal,
+ TVariable<Smi>* var_type_feedback) {
+ // In case of abstract or strict equality checks, we need additional checks
+ // for NaN values because they are not considered equal, even if both the
+ // left and the right hand side reference exactly the same value.
+
+ Label if_smi(this), if_heapnumber(this);
+ GotoIf(TaggedIsSmi(value), &if_smi);
+
+ TNode<HeapObject> value_heapobject = CAST(value);
+ TNode<Map> value_map = LoadMap(value_heapobject);
+ GotoIf(IsHeapNumberMap(value_map), &if_heapnumber);
+
+ // For non-HeapNumbers, all we do is collect type feedback.
+ if (var_type_feedback != nullptr) {
+ TNode<Uint16T> instance_type = LoadMapInstanceType(value_map);
+
+ Label if_string(this), if_receiver(this), if_oddball(this), if_symbol(this),
+ if_bigint(this);
+ GotoIf(IsStringInstanceType(instance_type), &if_string);
+ GotoIf(IsJSReceiverInstanceType(instance_type), &if_receiver);
+ GotoIf(IsOddballInstanceType(instance_type), &if_oddball);
+ Branch(IsBigIntInstanceType(instance_type), &if_bigint, &if_symbol);
+
+ BIND(&if_string);
+ {
+ CSA_ASSERT(this, IsString(value_heapobject));
+ CombineFeedback(var_type_feedback,
+ CollectFeedbackForString(instance_type));
+ Goto(if_equal);
+ }
+
+ BIND(&if_symbol);
+ {
+ CSA_ASSERT(this, IsSymbol(value_heapobject));
+ CombineFeedback(var_type_feedback, CompareOperationFeedback::kSymbol);
+ Goto(if_equal);
+ }
+
+ BIND(&if_receiver);
+ {
+ CSA_ASSERT(this, IsJSReceiver(value_heapobject));
+ CombineFeedback(var_type_feedback, CompareOperationFeedback::kReceiver);
+ Goto(if_equal);
+ }
+
+ BIND(&if_bigint);
+ {
+ CSA_ASSERT(this, IsBigInt(value_heapobject));
+ CombineFeedback(var_type_feedback, CompareOperationFeedback::kBigInt);
+ Goto(if_equal);
+ }
+
+ BIND(&if_oddball);
+ {
+ CSA_ASSERT(this, IsOddball(value_heapobject));
+ Label if_boolean(this), if_not_boolean(this);
+ Branch(IsBooleanMap(value_map), &if_boolean, &if_not_boolean);
+
+ BIND(&if_boolean);
+ {
+ CombineFeedback(var_type_feedback, CompareOperationFeedback::kBoolean);
+ Goto(if_equal);
+ }
+
+ BIND(&if_not_boolean);
+ {
+ CSA_ASSERT(this, IsNullOrUndefined(value_heapobject));
+ CombineFeedback(var_type_feedback,
+ CompareOperationFeedback::kReceiverOrNullOrUndefined);
+ Goto(if_equal);
+ }
+ }
+ } else {
+ Goto(if_equal);
+ }
+
+ BIND(&if_heapnumber);
+ {
+ CombineFeedback(var_type_feedback, CompareOperationFeedback::kNumber);
+ TNode<Float64T> number_value = LoadHeapNumberValue(value_heapobject);
+ BranchIfFloat64IsNaN(number_value, if_notequal, if_equal);
+ }
+
+ BIND(&if_smi);
+ {
+ CombineFeedback(var_type_feedback, CompareOperationFeedback::kSignedSmall);
+ Goto(if_equal);
+ }
+}
+
+// ES6 section 7.2.12 Abstract Equality Comparison
+TNode<Oddball> CodeStubAssembler::Equal(SloppyTNode<Object> left,
+ SloppyTNode<Object> right,
+ TNode<Context> context,
+ TVariable<Smi>* var_type_feedback) {
+ // This is a slightly optimized version of Object::Equals. Whenever you
+ // change something functionality wise in here, remember to update the
+ // Object::Equals method as well.
+
+ Label if_equal(this), if_notequal(this), do_float_comparison(this),
+ do_right_stringtonumber(this, Label::kDeferred), end(this);
+ TVARIABLE(Oddball, result);
+ TVARIABLE(Float64T, var_left_float);
+ TVARIABLE(Float64T, var_right_float);
+
+ // We can avoid code duplication by exploiting the fact that abstract equality
+ // is symmetric.
+ Label use_symmetry(this);
+
+ // We might need to loop several times due to ToPrimitive and/or ToNumber
+ // conversions.
+ TVARIABLE(Object, var_left, left);
+ TVARIABLE(Object, var_right, right);
+ VariableList loop_variable_list({&var_left, &var_right}, zone());
+ if (var_type_feedback != nullptr) {
+ // Initialize the type feedback to None. The current feedback will be
+ // combined with the previous feedback.
+ OverwriteFeedback(var_type_feedback, CompareOperationFeedback::kNone);
+ loop_variable_list.push_back(var_type_feedback);
+ }
+ Label loop(this, loop_variable_list);
+ Goto(&loop);
+ BIND(&loop);
+ {
+ left = var_left.value();
+ right = var_right.value();
+
+ Label if_notsame(this);
+ GotoIf(TaggedNotEqual(left, right), &if_notsame);
+ {
+ // {left} and {right} reference the exact same value, yet we need special
+ // treatment for HeapNumber, as NaN is not equal to NaN.
+ GenerateEqual_Same(left, &if_equal, &if_notequal, var_type_feedback);
+ }
+
+ BIND(&if_notsame);
+ Label if_left_smi(this), if_left_not_smi(this);
+ Branch(TaggedIsSmi(left), &if_left_smi, &if_left_not_smi);
+
+ BIND(&if_left_smi);
+ {
+ Label if_right_smi(this), if_right_not_smi(this);
+ CombineFeedback(var_type_feedback,
+ CompareOperationFeedback::kSignedSmall);
+ Branch(TaggedIsSmi(right), &if_right_smi, &if_right_not_smi);
+
+ BIND(&if_right_smi);
+ {
+ // We have already checked for {left} and {right} being the same value,
+ // so when we get here they must be different Smis.
+ Goto(&if_notequal);
+ }
+
+ BIND(&if_right_not_smi);
+ {
+ TNode<Map> right_map = LoadMap(CAST(right));
+ Label if_right_heapnumber(this), if_right_boolean(this),
+ if_right_oddball(this), if_right_bigint(this, Label::kDeferred),
+ if_right_receiver(this, Label::kDeferred);
+ GotoIf(IsHeapNumberMap(right_map), &if_right_heapnumber);
+
+ // {left} is Smi and {right} is not HeapNumber or Smi.
+ TNode<Uint16T> right_type = LoadMapInstanceType(right_map);
+ GotoIf(IsStringInstanceType(right_type), &do_right_stringtonumber);
+ GotoIf(IsOddballInstanceType(right_type), &if_right_oddball);
+ GotoIf(IsBigIntInstanceType(right_type), &if_right_bigint);
+ GotoIf(IsJSReceiverInstanceType(right_type), &if_right_receiver);
+ CombineFeedback(var_type_feedback, CompareOperationFeedback::kAny);
+ Goto(&if_notequal);
+
+ BIND(&if_right_heapnumber);
+ {
+ CombineFeedback(var_type_feedback, CompareOperationFeedback::kNumber);
+ var_left_float = SmiToFloat64(CAST(left));
+ var_right_float = LoadHeapNumberValue(CAST(right));
+ Goto(&do_float_comparison);
+ }
+
+ BIND(&if_right_oddball);
+ {
+ Label if_right_boolean(this);
+ GotoIf(IsBooleanMap(right_map), &if_right_boolean);
+ CombineFeedback(var_type_feedback,
+ CompareOperationFeedback::kOddball);
+ Goto(&if_notequal);
+
+ BIND(&if_right_boolean);
+ {
+ CombineFeedback(var_type_feedback,
+ CompareOperationFeedback::kBoolean);
+ var_right = LoadObjectField(CAST(right), Oddball::kToNumberOffset);
+ Goto(&loop);
+ }
+ }
+
+ BIND(&if_right_bigint);
+ {
+ CombineFeedback(var_type_feedback, CompareOperationFeedback::kBigInt);
+ result = CAST(CallRuntime(Runtime::kBigIntEqualToNumber,
+ NoContextConstant(), right, left));
+ Goto(&end);
+ }
+
+ BIND(&if_right_receiver);
+ {
+ CombineFeedback(var_type_feedback,
+ CompareOperationFeedback::kReceiver);
+ Callable callable = CodeFactory::NonPrimitiveToPrimitive(isolate());
+ var_right = CallStub(callable, context, right);
+ Goto(&loop);
+ }
+ }
+ }
+
+ BIND(&if_left_not_smi);
+ {
+ GotoIf(TaggedIsSmi(right), &use_symmetry);
+
+ Label if_left_symbol(this), if_left_number(this),
+ if_left_string(this, Label::kDeferred),
+ if_left_bigint(this, Label::kDeferred), if_left_oddball(this),
+ if_left_receiver(this);
+
+ TNode<Map> left_map = LoadMap(CAST(left));
+ TNode<Map> right_map = LoadMap(CAST(right));
+ TNode<Uint16T> left_type = LoadMapInstanceType(left_map);
+ TNode<Uint16T> right_type = LoadMapInstanceType(right_map);
+
+ GotoIf(IsStringInstanceType(left_type), &if_left_string);
+ GotoIf(IsSymbolInstanceType(left_type), &if_left_symbol);
+ GotoIf(IsHeapNumberInstanceType(left_type), &if_left_number);
+ GotoIf(IsOddballInstanceType(left_type), &if_left_oddball);
+ Branch(IsBigIntInstanceType(left_type), &if_left_bigint,
+ &if_left_receiver);
+
+ BIND(&if_left_string);
+ {
+ GotoIfNot(IsStringInstanceType(right_type), &use_symmetry);
+ result =
+ CAST(CallBuiltin(Builtins::kStringEqual, context, left, right));
+ CombineFeedback(var_type_feedback,
+ SmiOr(CollectFeedbackForString(left_type),
+ CollectFeedbackForString(right_type)));
+ Goto(&end);
+ }
+
+ BIND(&if_left_number);
+ {
+ Label if_right_not_number(this);
+
+ CombineFeedback(var_type_feedback, CompareOperationFeedback::kNumber);
+ GotoIf(Word32NotEqual(left_type, right_type), &if_right_not_number);
+
+ var_left_float = LoadHeapNumberValue(CAST(left));
+ var_right_float = LoadHeapNumberValue(CAST(right));
+ Goto(&do_float_comparison);
+
+ BIND(&if_right_not_number);
+ {
+ Label if_right_oddball(this);
+
+ GotoIf(IsStringInstanceType(right_type), &do_right_stringtonumber);
+ GotoIf(IsOddballInstanceType(right_type), &if_right_oddball);
+ GotoIf(IsBigIntInstanceType(right_type), &use_symmetry);
+ GotoIf(IsJSReceiverInstanceType(right_type), &use_symmetry);
+ CombineFeedback(var_type_feedback, CompareOperationFeedback::kAny);
+ Goto(&if_notequal);
+
+ BIND(&if_right_oddball);
+ {
+ Label if_right_boolean(this);
+ GotoIf(IsBooleanMap(right_map), &if_right_boolean);
+ CombineFeedback(var_type_feedback,
+ CompareOperationFeedback::kOddball);
+ Goto(&if_notequal);
+
+ BIND(&if_right_boolean);
+ {
+ CombineFeedback(var_type_feedback,
+ CompareOperationFeedback::kBoolean);
+ var_right =
+ LoadObjectField(CAST(right), Oddball::kToNumberOffset);
+ Goto(&loop);
+ }
+ }
+ }
+ }
+
+ BIND(&if_left_bigint);
+ {
+ Label if_right_heapnumber(this), if_right_bigint(this),
+ if_right_string(this), if_right_boolean(this);
+ CombineFeedback(var_type_feedback, CompareOperationFeedback::kBigInt);
+
+ GotoIf(IsHeapNumberMap(right_map), &if_right_heapnumber);
+ GotoIf(IsBigIntInstanceType(right_type), &if_right_bigint);
+ GotoIf(IsStringInstanceType(right_type), &if_right_string);
+ GotoIf(IsBooleanMap(right_map), &if_right_boolean);
+ Branch(IsJSReceiverInstanceType(right_type), &use_symmetry,
+ &if_notequal);
+
+ BIND(&if_right_heapnumber);
+ {
+ CombineFeedback(var_type_feedback, CompareOperationFeedback::kNumber);
+ result = CAST(CallRuntime(Runtime::kBigIntEqualToNumber,
+ NoContextConstant(), left, right));
+ Goto(&end);
+ }
+
+ BIND(&if_right_bigint);
+ {
+ // We already have BigInt feedback.
+ result = CAST(CallRuntime(Runtime::kBigIntEqualToBigInt,
+ NoContextConstant(), left, right));
+ Goto(&end);
+ }
+
+ BIND(&if_right_string);
+ {
+ CombineFeedback(var_type_feedback, CompareOperationFeedback::kString);
+ result = CAST(CallRuntime(Runtime::kBigIntEqualToString,
+ NoContextConstant(), left, right));
+ Goto(&end);
+ }
+
+ BIND(&if_right_boolean);
+ {
+ CombineFeedback(var_type_feedback,
+ CompareOperationFeedback::kBoolean);
+ var_right = LoadObjectField(CAST(right), Oddball::kToNumberOffset);
+ Goto(&loop);
+ }
+ }
+
+ BIND(&if_left_oddball);
+ {
+ Label if_left_boolean(this), if_left_not_boolean(this);
+ GotoIf(IsBooleanMap(left_map), &if_left_boolean);
+ if (var_type_feedback != nullptr) {
+ CombineFeedback(var_type_feedback,
+ CompareOperationFeedback::kNullOrUndefined);
+ GotoIf(IsUndetectableMap(left_map), &if_left_not_boolean);
+ }
+ Goto(&if_left_not_boolean);
+
+ BIND(&if_left_not_boolean);
+ {
+ // {left} is either Null or Undefined. Check if {right} is
+ // undetectable (which includes Null and Undefined).
+ Label if_right_undetectable(this), if_right_number(this),
+ if_right_oddball(this),
+ if_right_not_number_or_oddball_or_undetectable(this);
+ GotoIf(IsUndetectableMap(right_map), &if_right_undetectable);
+ GotoIf(IsHeapNumberInstanceType(right_type), &if_right_number);
+ GotoIf(IsOddballInstanceType(right_type), &if_right_oddball);
+ Goto(&if_right_not_number_or_oddball_or_undetectable);
+
+ BIND(&if_right_undetectable);
+ {
+ // If {right} is undetectable, it must be either also
+ // Null or Undefined, or a Receiver (aka document.all).
+ CombineFeedback(
+ var_type_feedback,
+ CompareOperationFeedback::kReceiverOrNullOrUndefined);
+ Goto(&if_equal);
+ }
+
+ BIND(&if_right_number);
+ {
+ CombineFeedback(var_type_feedback,
+ CompareOperationFeedback::kNumber);
+ Goto(&if_notequal);
+ }
+
+ BIND(&if_right_oddball);
+ {
+ CombineFeedback(var_type_feedback,
+ CompareOperationFeedback::kOddball);
+ Goto(&if_notequal);
+ }
+
+ BIND(&if_right_not_number_or_oddball_or_undetectable);
+ {
+ if (var_type_feedback != nullptr) {
+ // Track whether {right} is Null, Undefined or Receiver.
+ CombineFeedback(
+ var_type_feedback,
+ CompareOperationFeedback::kReceiverOrNullOrUndefined);
+ GotoIf(IsJSReceiverInstanceType(right_type), &if_notequal);
+ CombineFeedback(var_type_feedback,
+ CompareOperationFeedback::kAny);
+ }
+ Goto(&if_notequal);
+ }
+ }
+
+ BIND(&if_left_boolean);
+ {
+ CombineFeedback(var_type_feedback,
+ CompareOperationFeedback::kBoolean);
+
+ // If {right} is a Boolean too, it must be a different Boolean.
+ GotoIf(TaggedEqual(right_map, left_map), &if_notequal);
+
+ // Otherwise, convert {left} to number and try again.
+ var_left = LoadObjectField(CAST(left), Oddball::kToNumberOffset);
+ Goto(&loop);
+ }
+ }
+
+ BIND(&if_left_symbol);
+ {
+ Label if_right_receiver(this);
+ GotoIf(IsJSReceiverInstanceType(right_type), &if_right_receiver);
+ // {right} is not a JSReceiver and also not the same Symbol as {left},
+ // so the result is "not equal".
+ if (var_type_feedback != nullptr) {
+ Label if_right_symbol(this);
+ GotoIf(IsSymbolInstanceType(right_type), &if_right_symbol);
+ *var_type_feedback = SmiConstant(CompareOperationFeedback::kAny);
+ Goto(&if_notequal);
+
+ BIND(&if_right_symbol);
+ {
+ CombineFeedback(var_type_feedback,
+ CompareOperationFeedback::kSymbol);
+ Goto(&if_notequal);
+ }
+ } else {
+ Goto(&if_notequal);
+ }
+
+ BIND(&if_right_receiver);
+ {
+ // {left} is a Primitive and {right} is a JSReceiver, so swapping
+ // the order is not observable.
+ if (var_type_feedback != nullptr) {
+ *var_type_feedback = SmiConstant(CompareOperationFeedback::kAny);
+ }
+ Goto(&use_symmetry);
+ }
+ }
+
+ BIND(&if_left_receiver);
+ {
+ CSA_ASSERT(this, IsJSReceiverInstanceType(left_type));
+ Label if_right_receiver(this), if_right_not_receiver(this);
+ Branch(IsJSReceiverInstanceType(right_type), &if_right_receiver,
+ &if_right_not_receiver);
+
+ BIND(&if_right_receiver);
+ {
+ // {left} and {right} are different JSReceiver references.
+ CombineFeedback(var_type_feedback,
+ CompareOperationFeedback::kReceiver);
+ Goto(&if_notequal);
+ }
+
+ BIND(&if_right_not_receiver);
+ {
+ // Check if {right} is undetectable, which means it must be Null
+ // or Undefined, since we already ruled out Receiver for {right}.
+ Label if_right_undetectable(this),
+ if_right_not_undetectable(this, Label::kDeferred);
+ Branch(IsUndetectableMap(right_map), &if_right_undetectable,
+ &if_right_not_undetectable);
+
+ BIND(&if_right_undetectable);
+ {
+ // When we get here, {right} must be either Null or Undefined.
+ CSA_ASSERT(this, IsNullOrUndefined(right));
+ if (var_type_feedback != nullptr) {
+ *var_type_feedback = SmiConstant(
+ CompareOperationFeedback::kReceiverOrNullOrUndefined);
+ }
+ Branch(IsUndetectableMap(left_map), &if_equal, &if_notequal);
+ }
+
+ BIND(&if_right_not_undetectable);
+ {
+ // {right} is a Primitive, and neither Null or Undefined;
+ // convert {left} to Primitive too.
+ CombineFeedback(var_type_feedback, CompareOperationFeedback::kAny);
+ Callable callable = CodeFactory::NonPrimitiveToPrimitive(isolate());
+ var_left = CallStub(callable, context, left);
+ Goto(&loop);
+ }
+ }
+ }
+ }
+
+ BIND(&do_right_stringtonumber);
+ {
+ if (var_type_feedback != nullptr) {
+ TNode<Map> right_map = LoadMap(CAST(right));
+ TNode<Uint16T> right_type = LoadMapInstanceType(right_map);
+ CombineFeedback(var_type_feedback,
+ CollectFeedbackForString(right_type));
+ }
+ var_right = CallBuiltin(Builtins::kStringToNumber, context, right);
+ Goto(&loop);
+ }
+
+ BIND(&use_symmetry);
+ {
+ var_left = right;
+ var_right = left;
+ Goto(&loop);
+ }
+ }
+
+ BIND(&do_float_comparison);
+ {
+ Branch(Float64Equal(var_left_float.value(), var_right_float.value()),
+ &if_equal, &if_notequal);
+ }
+
+ BIND(&if_equal);
+ {
+ result = TrueConstant();
+ Goto(&end);
+ }
+
+ BIND(&if_notequal);
+ {
+ result = FalseConstant();
+ Goto(&end);
+ }
+
+ BIND(&end);
+ return result.value();
+}
+
+TNode<Oddball> CodeStubAssembler::StrictEqual(
+ SloppyTNode<Object> lhs, SloppyTNode<Object> rhs,
+ TVariable<Smi>* var_type_feedback) {
+ // Pseudo-code for the algorithm below:
+ //
+ // if (lhs == rhs) {
+ // if (lhs->IsHeapNumber()) return HeapNumber::cast(lhs)->value() != NaN;
+ // return true;
+ // }
+ // if (!lhs->IsSmi()) {
+ // if (lhs->IsHeapNumber()) {
+ // if (rhs->IsSmi()) {
+ // return Smi::ToInt(rhs) == HeapNumber::cast(lhs)->value();
+ // } else if (rhs->IsHeapNumber()) {
+ // return HeapNumber::cast(rhs)->value() ==
+ // HeapNumber::cast(lhs)->value();
+ // } else {
+ // return false;
+ // }
+ // } else {
+ // if (rhs->IsSmi()) {
+ // return false;
+ // } else {
+ // if (lhs->IsString()) {
+ // if (rhs->IsString()) {
+ // return %StringEqual(lhs, rhs);
+ // } else {
+ // return false;
+ // }
+ // } else if (lhs->IsBigInt()) {
+ // if (rhs->IsBigInt()) {
+ // return %BigIntEqualToBigInt(lhs, rhs);
+ // } else {
+ // return false;
+ // }
+ // } else {
+ // return false;
+ // }
+ // }
+ // }
+ // } else {
+ // if (rhs->IsSmi()) {
+ // return false;
+ // } else {
+ // if (rhs->IsHeapNumber()) {
+ // return Smi::ToInt(lhs) == HeapNumber::cast(rhs)->value();
+ // } else {
+ // return false;
+ // }
+ // }
+ // }
+
+ Label if_equal(this), if_notequal(this), if_not_equivalent_types(this),
+ end(this);
+ TVARIABLE(Oddball, result);
+
+ OverwriteFeedback(var_type_feedback, CompareOperationFeedback::kNone);
+
+ // Check if {lhs} and {rhs} refer to the same object.
+ Label if_same(this), if_notsame(this);
+ Branch(TaggedEqual(lhs, rhs), &if_same, &if_notsame);
+
+ BIND(&if_same);
+ {
+ // The {lhs} and {rhs} reference the exact same value, yet we need special
+ // treatment for HeapNumber, as NaN is not equal to NaN.
+ GenerateEqual_Same(lhs, &if_equal, &if_notequal, var_type_feedback);
+ }
+
+ BIND(&if_notsame);
+ {
+ // The {lhs} and {rhs} reference different objects, yet for Smi, HeapNumber,
+ // BigInt and String they can still be considered equal.
+
+ // Check if {lhs} is a Smi or a HeapObject.
+ Label if_lhsissmi(this), if_lhsisnotsmi(this);
+ Branch(TaggedIsSmi(lhs), &if_lhsissmi, &if_lhsisnotsmi);
+
+ BIND(&if_lhsisnotsmi);
+ {
+ // Load the map of {lhs}.
+ TNode<Map> lhs_map = LoadMap(CAST(lhs));
+
+ // Check if {lhs} is a HeapNumber.
+ Label if_lhsisnumber(this), if_lhsisnotnumber(this);
+ Branch(IsHeapNumberMap(lhs_map), &if_lhsisnumber, &if_lhsisnotnumber);
+
+ BIND(&if_lhsisnumber);
+ {
+ // Check if {rhs} is a Smi or a HeapObject.
+ Label if_rhsissmi(this), if_rhsisnotsmi(this);
+ Branch(TaggedIsSmi(rhs), &if_rhsissmi, &if_rhsisnotsmi);
+
+ BIND(&if_rhsissmi);
+ {
+ // Convert {lhs} and {rhs} to floating point values.
+ TNode<Float64T> lhs_value = LoadHeapNumberValue(CAST(lhs));
+ TNode<Float64T> rhs_value = SmiToFloat64(CAST(rhs));
+
+ CombineFeedback(var_type_feedback, CompareOperationFeedback::kNumber);
+
+ // Perform a floating point comparison of {lhs} and {rhs}.
+ Branch(Float64Equal(lhs_value, rhs_value), &if_equal, &if_notequal);
+ }
+
+ BIND(&if_rhsisnotsmi);
+ {
+ TNode<HeapObject> rhs_ho = CAST(rhs);
+ // Load the map of {rhs}.
+ TNode<Map> rhs_map = LoadMap(rhs_ho);
+
+ // Check if {rhs} is also a HeapNumber.
+ Label if_rhsisnumber(this), if_rhsisnotnumber(this);
+ Branch(IsHeapNumberMap(rhs_map), &if_rhsisnumber, &if_rhsisnotnumber);
+
+ BIND(&if_rhsisnumber);
+ {
+ // Convert {lhs} and {rhs} to floating point values.
+ TNode<Float64T> lhs_value = LoadHeapNumberValue(CAST(lhs));
+ TNode<Float64T> rhs_value = LoadHeapNumberValue(CAST(rhs));
+
+ CombineFeedback(var_type_feedback,
+ CompareOperationFeedback::kNumber);
+
+ // Perform a floating point comparison of {lhs} and {rhs}.
+ Branch(Float64Equal(lhs_value, rhs_value), &if_equal, &if_notequal);
+ }
+
+ BIND(&if_rhsisnotnumber);
+ Goto(&if_not_equivalent_types);
+ }
+ }
+
+ BIND(&if_lhsisnotnumber);
+ {
+ // Check if {rhs} is a Smi or a HeapObject.
+ Label if_rhsissmi(this), if_rhsisnotsmi(this);
+ Branch(TaggedIsSmi(rhs), &if_rhsissmi, &if_rhsisnotsmi);
+
+ BIND(&if_rhsissmi);
+ Goto(&if_not_equivalent_types);
+
+ BIND(&if_rhsisnotsmi);
+ {
+ // Load the instance type of {lhs}.
+ TNode<Uint16T> lhs_instance_type = LoadMapInstanceType(lhs_map);
+
+ // Check if {lhs} is a String.
+ Label if_lhsisstring(this, Label::kDeferred), if_lhsisnotstring(this);
+ Branch(IsStringInstanceType(lhs_instance_type), &if_lhsisstring,
+ &if_lhsisnotstring);
+
+ BIND(&if_lhsisstring);
+ {
+ // Load the instance type of {rhs}.
+ TNode<Uint16T> rhs_instance_type = LoadInstanceType(CAST(rhs));
+
+ // Check if {rhs} is also a String.
+ Label if_rhsisstring(this, Label::kDeferred),
+ if_rhsisnotstring(this);
+ Branch(IsStringInstanceType(rhs_instance_type), &if_rhsisstring,
+ &if_rhsisnotstring);
+
+ BIND(&if_rhsisstring);
+ {
+ if (var_type_feedback != nullptr) {
+ TNode<Smi> lhs_feedback =
+ CollectFeedbackForString(lhs_instance_type);
+ TNode<Smi> rhs_feedback =
+ CollectFeedbackForString(rhs_instance_type);
+ *var_type_feedback = SmiOr(lhs_feedback, rhs_feedback);
+ }
+ result = CAST(CallBuiltin(Builtins::kStringEqual,
+ NoContextConstant(), lhs, rhs));
+ Goto(&end);
+ }
+
+ BIND(&if_rhsisnotstring);
+ Goto(&if_not_equivalent_types);
+ }
+
+ BIND(&if_lhsisnotstring);
+ {
+ // Check if {lhs} is a BigInt.
+ Label if_lhsisbigint(this), if_lhsisnotbigint(this);
+ Branch(IsBigIntInstanceType(lhs_instance_type), &if_lhsisbigint,
+ &if_lhsisnotbigint);
+
+ BIND(&if_lhsisbigint);
+ {
+ // Load the instance type of {rhs}.
+ TNode<Uint16T> rhs_instance_type = LoadInstanceType(CAST(rhs));
+
+ // Check if {rhs} is also a BigInt.
+ Label if_rhsisbigint(this, Label::kDeferred),
+ if_rhsisnotbigint(this);
+ Branch(IsBigIntInstanceType(rhs_instance_type), &if_rhsisbigint,
+ &if_rhsisnotbigint);
+
+ BIND(&if_rhsisbigint);
+ {
+ CombineFeedback(var_type_feedback,
+ CompareOperationFeedback::kBigInt);
+ result = CAST(CallRuntime(Runtime::kBigIntEqualToBigInt,
+ NoContextConstant(), lhs, rhs));
+ Goto(&end);
+ }
+
+ BIND(&if_rhsisnotbigint);
+ Goto(&if_not_equivalent_types);
+ }
+
+ BIND(&if_lhsisnotbigint);
+ if (var_type_feedback != nullptr) {
+ // Load the instance type of {rhs}.
+ TNode<Map> rhs_map = LoadMap(CAST(rhs));
+ TNode<Uint16T> rhs_instance_type = LoadMapInstanceType(rhs_map);
+
+ Label if_lhsissymbol(this), if_lhsisreceiver(this),
+ if_lhsisoddball(this);
+ GotoIf(IsJSReceiverInstanceType(lhs_instance_type),
+ &if_lhsisreceiver);
+ GotoIf(IsBooleanMap(lhs_map), &if_not_equivalent_types);
+ GotoIf(IsOddballInstanceType(lhs_instance_type),
+ &if_lhsisoddball);
+ Branch(IsSymbolInstanceType(lhs_instance_type), &if_lhsissymbol,
+ &if_not_equivalent_types);
+
+ BIND(&if_lhsisreceiver);
+ {
+ GotoIf(IsBooleanMap(rhs_map), &if_not_equivalent_types);
+ OverwriteFeedback(var_type_feedback,
+ CompareOperationFeedback::kReceiver);
+ GotoIf(IsJSReceiverInstanceType(rhs_instance_type),
+ &if_notequal);
+ OverwriteFeedback(
+ var_type_feedback,
+ CompareOperationFeedback::kReceiverOrNullOrUndefined);
+ GotoIf(IsOddballInstanceType(rhs_instance_type), &if_notequal);
+ Goto(&if_not_equivalent_types);
+ }
+
+ BIND(&if_lhsisoddball);
+ {
+ Label if_lhsisboolean(this), if_lhsisnotboolean(this);
+ Branch(IsBooleanMap(lhs_map), &if_lhsisboolean,
+ &if_lhsisnotboolean);
+
+ BIND(&if_lhsisboolean);
+ {
+ OverwriteFeedback(var_type_feedback,
+ CompareOperationFeedback::kNumberOrOddball);
+ GotoIf(IsBooleanMap(rhs_map), &if_notequal);
+ Goto(&if_not_equivalent_types);
+ }
+
+ BIND(&if_lhsisnotboolean);
+ {
+ Label if_rhsisheapnumber(this), if_rhsisnotheapnumber(this);
+
+ STATIC_ASSERT(LAST_PRIMITIVE_HEAP_OBJECT_TYPE ==
+ ODDBALL_TYPE);
+ GotoIf(Int32LessThan(rhs_instance_type,
+ Int32Constant(ODDBALL_TYPE)),
+ &if_not_equivalent_types);
+
+ Branch(IsHeapNumberMap(rhs_map), &if_rhsisheapnumber,
+ &if_rhsisnotheapnumber);
+
+ BIND(&if_rhsisheapnumber);
+ {
+ OverwriteFeedback(
+ var_type_feedback,
+ CompareOperationFeedback::kNumberOrOddball);
+ Goto(&if_not_equivalent_types);
+ }
+
+ BIND(&if_rhsisnotheapnumber);
+ {
+ OverwriteFeedback(
+ var_type_feedback,
+ CompareOperationFeedback::kReceiverOrNullOrUndefined);
+ Goto(&if_notequal);
+ }
+ }
+ }
+
+ BIND(&if_lhsissymbol);
+ {
+ GotoIfNot(IsSymbolInstanceType(rhs_instance_type),
+ &if_not_equivalent_types);
+ OverwriteFeedback(var_type_feedback,
+ CompareOperationFeedback::kSymbol);
+ Goto(&if_notequal);
+ }
+ } else {
+ Goto(&if_notequal);
+ }
+ }
+ }
+ }
+ }
+
+ BIND(&if_lhsissmi);
+ {
+ // We already know that {lhs} and {rhs} are not reference equal, and {lhs}
+ // is a Smi; so {lhs} and {rhs} can only be strictly equal if {rhs} is a
+ // HeapNumber with an equal floating point value.
+
+ // Check if {rhs} is a Smi or a HeapObject.
+ Label if_rhsissmi(this), if_rhsisnotsmi(this);
+ Branch(TaggedIsSmi(rhs), &if_rhsissmi, &if_rhsisnotsmi);
+
+ BIND(&if_rhsissmi);
+ CombineFeedback(var_type_feedback,
+ CompareOperationFeedback::kSignedSmall);
+ Goto(&if_notequal);
+
+ BIND(&if_rhsisnotsmi);
+ {
+ // Load the map of the {rhs}.
+ TNode<Map> rhs_map = LoadMap(CAST(rhs));
+
+ // The {rhs} could be a HeapNumber with the same value as {lhs}.
+ Label if_rhsisnumber(this), if_rhsisnotnumber(this);
+ Branch(IsHeapNumberMap(rhs_map), &if_rhsisnumber, &if_rhsisnotnumber);
+
+ BIND(&if_rhsisnumber);
+ {
+ // Convert {lhs} and {rhs} to floating point values.
+ TNode<Float64T> lhs_value = SmiToFloat64(CAST(lhs));
+ TNode<Float64T> rhs_value = LoadHeapNumberValue(CAST(rhs));
+
+ CombineFeedback(var_type_feedback, CompareOperationFeedback::kNumber);
+
+ // Perform a floating point comparison of {lhs} and {rhs}.
+ Branch(Float64Equal(lhs_value, rhs_value), &if_equal, &if_notequal);
+ }
+
+ BIND(&if_rhsisnotnumber);
+ {
+ TNode<Uint16T> rhs_instance_type = LoadMapInstanceType(rhs_map);
+ GotoIfNot(IsOddballInstanceType(rhs_instance_type),
+ &if_not_equivalent_types);
+ OverwriteFeedback(var_type_feedback,
+ CompareOperationFeedback::kNumberOrOddball);
+ Goto(&if_notequal);
+ }
+ }
+ }
+ }
+
+ BIND(&if_equal);
+ {
+ result = TrueConstant();
+ Goto(&end);
+ }
+
+ BIND(&if_not_equivalent_types);
+ {
+ OverwriteFeedback(var_type_feedback, CompareOperationFeedback::kAny);
+ Goto(&if_notequal);
+ }
+
+ BIND(&if_notequal);
+ {
+ result = FalseConstant();
+ Goto(&end);
+ }
+
+ BIND(&end);
+ return result.value();
+}
+
+// ECMA#sec-samevalue
+// This algorithm differs from the Strict Equality Comparison Algorithm in its
+// treatment of signed zeroes and NaNs.
+void CodeStubAssembler::BranchIfSameValue(SloppyTNode<Object> lhs,
+ SloppyTNode<Object> rhs,
+ Label* if_true, Label* if_false,
+ SameValueMode mode) {
+ TVARIABLE(Float64T, var_lhs_value);
+ TVARIABLE(Float64T, var_rhs_value);
+ Label do_fcmp(this);
+
+ // Immediately jump to {if_true} if {lhs} == {rhs}, because - unlike
+ // StrictEqual - SameValue considers two NaNs to be equal.
+ GotoIf(TaggedEqual(lhs, rhs), if_true);
+
+ // Check if the {lhs} is a Smi.
+ Label if_lhsissmi(this), if_lhsisheapobject(this);
+ Branch(TaggedIsSmi(lhs), &if_lhsissmi, &if_lhsisheapobject);
+
+ BIND(&if_lhsissmi);
+ {
+ // Since {lhs} is a Smi, the comparison can only yield true
+ // iff the {rhs} is a HeapNumber with the same float64 value.
+ Branch(TaggedIsSmi(rhs), if_false, [&] {
+ GotoIfNot(IsHeapNumber(CAST(rhs)), if_false);
+ var_lhs_value = SmiToFloat64(CAST(lhs));
+ var_rhs_value = LoadHeapNumberValue(CAST(rhs));
+ Goto(&do_fcmp);
+ });
+ }
+
+ BIND(&if_lhsisheapobject);
+ {
+ // Check if the {rhs} is a Smi.
+ Branch(
+ TaggedIsSmi(rhs),
+ [&] {
+ // Since {rhs} is a Smi, the comparison can only yield true
+ // iff the {lhs} is a HeapNumber with the same float64 value.
+ GotoIfNot(IsHeapNumber(CAST(lhs)), if_false);
+ var_lhs_value = LoadHeapNumberValue(CAST(lhs));
+ var_rhs_value = SmiToFloat64(CAST(rhs));
+ Goto(&do_fcmp);
+ },
+ [&] {
+ // Now this can only yield true if either both {lhs} and {rhs} are
+ // HeapNumbers with the same value, or both are Strings with the
+ // same character sequence, or both are BigInts with the same
+ // value.
+ Label if_lhsisheapnumber(this), if_lhsisstring(this),
+ if_lhsisbigint(this);
+ const TNode<Map> lhs_map = LoadMap(CAST(lhs));
+ GotoIf(IsHeapNumberMap(lhs_map), &if_lhsisheapnumber);
+ if (mode != SameValueMode::kNumbersOnly) {
+ const TNode<Uint16T> lhs_instance_type =
+ LoadMapInstanceType(lhs_map);
+ GotoIf(IsStringInstanceType(lhs_instance_type), &if_lhsisstring);
+ GotoIf(IsBigIntInstanceType(lhs_instance_type), &if_lhsisbigint);
+ }
+ Goto(if_false);
+
+ BIND(&if_lhsisheapnumber);
+ {
+ GotoIfNot(IsHeapNumber(CAST(rhs)), if_false);
+ var_lhs_value = LoadHeapNumberValue(CAST(lhs));
+ var_rhs_value = LoadHeapNumberValue(CAST(rhs));
+ Goto(&do_fcmp);
+ }
+
+ if (mode != SameValueMode::kNumbersOnly) {
+ BIND(&if_lhsisstring);
+ {
+ // Now we can only yield true if {rhs} is also a String
+ // with the same sequence of characters.
+ GotoIfNot(IsString(CAST(rhs)), if_false);
+ const TNode<Object> result = CallBuiltin(
+ Builtins::kStringEqual, NoContextConstant(), lhs, rhs);
+ Branch(IsTrue(result), if_true, if_false);
+ }
+
+ BIND(&if_lhsisbigint);
+ {
+ GotoIfNot(IsBigInt(CAST(rhs)), if_false);
+ const TNode<Object> result = CallRuntime(
+ Runtime::kBigIntEqualToBigInt, NoContextConstant(), lhs, rhs);
+ Branch(IsTrue(result), if_true, if_false);
+ }
+ }
+ });
+ }
+
+ BIND(&do_fcmp);
+ {
+ TNode<Float64T> lhs_value = UncheckedCast<Float64T>(var_lhs_value.value());
+ TNode<Float64T> rhs_value = UncheckedCast<Float64T>(var_rhs_value.value());
+ BranchIfSameNumberValue(lhs_value, rhs_value, if_true, if_false);
+ }
+}
+
+void CodeStubAssembler::BranchIfSameNumberValue(TNode<Float64T> lhs_value,
+ TNode<Float64T> rhs_value,
+ Label* if_true,
+ Label* if_false) {
+ Label if_equal(this), if_notequal(this);
+ Branch(Float64Equal(lhs_value, rhs_value), &if_equal, &if_notequal);
+
+ BIND(&if_equal);
+ {
+ // We still need to handle the case when {lhs} and {rhs} are -0.0 and
+ // 0.0 (or vice versa). Compare the high word to
+ // distinguish between the two.
+ const TNode<Uint32T> lhs_hi_word = Float64ExtractHighWord32(lhs_value);
+ const TNode<Uint32T> rhs_hi_word = Float64ExtractHighWord32(rhs_value);
+
+ // If x is +0 and y is -0, return false.
+ // If x is -0 and y is +0, return false.
+ Branch(Word32Equal(lhs_hi_word, rhs_hi_word), if_true, if_false);
+ }
+
+ BIND(&if_notequal);
+ {
+ // Return true iff both {rhs} and {lhs} are NaN.
+ GotoIf(Float64Equal(lhs_value, lhs_value), if_false);
+ Branch(Float64Equal(rhs_value, rhs_value), if_false, if_true);
+ }
+}
+
+TNode<Oddball> CodeStubAssembler::HasProperty(TNode<Context> context,
+ SloppyTNode<Object> object,
+ SloppyTNode<Object> key,
+ HasPropertyLookupMode mode) {
+ Label call_runtime(this, Label::kDeferred), return_true(this),
+ return_false(this), end(this), if_proxy(this, Label::kDeferred);
+
+ CodeStubAssembler::LookupPropertyInHolder lookup_property_in_holder =
+ [this, &return_true](
+ TNode<HeapObject> receiver, TNode<HeapObject> holder,
+ TNode<Map> holder_map, TNode<Int32T> holder_instance_type,
+ TNode<Name> unique_name, Label* next_holder, Label* if_bailout) {
+ TryHasOwnProperty(holder, holder_map, holder_instance_type, unique_name,
+ &return_true, next_holder, if_bailout);
+ };
+
+ CodeStubAssembler::LookupElementInHolder lookup_element_in_holder =
+ [this, &return_true, &return_false](
+ TNode<HeapObject> receiver, TNode<HeapObject> holder,
+ TNode<Map> holder_map, TNode<Int32T> holder_instance_type,
+ TNode<IntPtrT> index, Label* next_holder, Label* if_bailout) {
+ TryLookupElement(holder, holder_map, holder_instance_type, index,
+ &return_true, &return_false, next_holder, if_bailout);
+ };
+
+ TryPrototypeChainLookup(object, object, key, lookup_property_in_holder,
+ lookup_element_in_holder, &return_false,
+ &call_runtime, &if_proxy);
+
+ TVARIABLE(Oddball, result);
+
+ BIND(&if_proxy);
+ {
+ TNode<Name> name = CAST(CallBuiltin(Builtins::kToName, context, key));
+ switch (mode) {
+ case kHasProperty:
+ GotoIf(IsPrivateSymbol(name), &return_false);
+
+ result = CAST(
+ CallBuiltin(Builtins::kProxyHasProperty, context, object, name));
+ Goto(&end);
+ break;
+ case kForInHasProperty:
+ Goto(&call_runtime);
+ break;
+ }
+ }
+
+ BIND(&return_true);
+ {
+ result = TrueConstant();
+ Goto(&end);
+ }
+
+ BIND(&return_false);
+ {
+ result = FalseConstant();
+ Goto(&end);
+ }
+
+ BIND(&call_runtime);
+ {
+ Runtime::FunctionId fallback_runtime_function_id;
+ switch (mode) {
+ case kHasProperty:
+ fallback_runtime_function_id = Runtime::kHasProperty;
+ break;
+ case kForInHasProperty:
+ fallback_runtime_function_id = Runtime::kForInHasProperty;
+ break;
+ }
+
+ result =
+ CAST(CallRuntime(fallback_runtime_function_id, context, object, key));
+ Goto(&end);
+ }
+
+ BIND(&end);
+ CSA_ASSERT(this, IsBoolean(result.value()));
+ return result.value();
+}
+
+void CodeStubAssembler::ForInPrepare(TNode<HeapObject> enumerator,
+ TNode<UintPtrT> slot,
+ TNode<HeapObject> maybe_feedback_vector,
+ TNode<FixedArray>* cache_array_out,
+ TNode<Smi>* cache_length_out) {
+ // Check if we're using an enum cache.
+ TVARIABLE(FixedArray, cache_array);
+ TVARIABLE(Smi, cache_length);
+ Label if_fast(this), if_slow(this, Label::kDeferred), out(this);
+ Branch(IsMap(enumerator), &if_fast, &if_slow);
+
+ BIND(&if_fast);
+ {
+ // Load the enumeration length and cache from the {enumerator}.
+ TNode<Map> map_enumerator = CAST(enumerator);
+ TNode<WordT> enum_length = LoadMapEnumLength(map_enumerator);
+ CSA_ASSERT(this, WordNotEqual(enum_length,
+ IntPtrConstant(kInvalidEnumCacheSentinel)));
+ TNode<DescriptorArray> descriptors = LoadMapDescriptors(map_enumerator);
+ TNode<EnumCache> enum_cache = LoadObjectField<EnumCache>(
+ descriptors, DescriptorArray::kEnumCacheOffset);
+ TNode<FixedArray> enum_keys =
+ LoadObjectField<FixedArray>(enum_cache, EnumCache::kKeysOffset);
+
+ // Check if we have enum indices available.
+ TNode<FixedArray> enum_indices =
+ LoadObjectField<FixedArray>(enum_cache, EnumCache::kIndicesOffset);
+ TNode<IntPtrT> enum_indices_length =
+ LoadAndUntagFixedArrayBaseLength(enum_indices);
+ TNode<Smi> feedback = SelectSmiConstant(
+ IntPtrLessThanOrEqual(enum_length, enum_indices_length),
+ static_cast<int>(ForInFeedback::kEnumCacheKeysAndIndices),
+ static_cast<int>(ForInFeedback::kEnumCacheKeys));
+ UpdateFeedback(feedback, maybe_feedback_vector, slot);
+
+ cache_array = enum_keys;
+ cache_length = SmiTag(Signed(enum_length));
+ Goto(&out);
+ }
+
+ BIND(&if_slow);
+ {
+ // The {enumerator} is a FixedArray with all the keys to iterate.
+ TNode<FixedArray> array_enumerator = CAST(enumerator);
+
+ // Record the fact that we hit the for-in slow-path.
+ UpdateFeedback(SmiConstant(ForInFeedback::kAny), maybe_feedback_vector,
+ slot);
+
+ cache_array = array_enumerator;
+ cache_length = LoadFixedArrayBaseLength(array_enumerator);
+ Goto(&out);
+ }
+
+ BIND(&out);
+ *cache_array_out = cache_array.value();
+ *cache_length_out = cache_length.value();
+}
+
+TNode<FixedArray> CodeStubAssembler::ForInPrepareForTorque(
+ TNode<HeapObject> enumerator, TNode<UintPtrT> slot,
+ TNode<HeapObject> maybe_feedback_vector) {
+ TNode<FixedArray> cache_array;
+ TNode<Smi> cache_length;
+ ForInPrepare(enumerator, slot, maybe_feedback_vector, &cache_array,
+ &cache_length);
+
+ TNode<FixedArray> result = AllocateUninitializedFixedArray(2);
+ StoreFixedArrayElement(result, 0, cache_array);
+ StoreFixedArrayElement(result, 1, cache_length);
+
+ return result;
+}
+
+TNode<String> CodeStubAssembler::Typeof(SloppyTNode<Object> value) {
+ TVARIABLE(String, result_var);
+
+ Label return_number(this, Label::kDeferred), if_oddball(this),
+ return_function(this), return_undefined(this), return_object(this),
+ return_string(this), return_bigint(this), return_result(this);
+
+ GotoIf(TaggedIsSmi(value), &return_number);
+
+ TNode<HeapObject> value_heap_object = CAST(value);
+ TNode<Map> map = LoadMap(value_heap_object);
+
+ GotoIf(IsHeapNumberMap(map), &return_number);
+
+ TNode<Uint16T> instance_type = LoadMapInstanceType(map);
+
+ GotoIf(InstanceTypeEqual(instance_type, ODDBALL_TYPE), &if_oddball);
+
+ TNode<Int32T> callable_or_undetectable_mask =
+ Word32And(LoadMapBitField(map),
+ Int32Constant(Map::Bits1::IsCallableBit::kMask |
+ Map::Bits1::IsUndetectableBit::kMask));
+
+ GotoIf(Word32Equal(callable_or_undetectable_mask,
+ Int32Constant(Map::Bits1::IsCallableBit::kMask)),
+ &return_function);
+
+ GotoIfNot(Word32Equal(callable_or_undetectable_mask, Int32Constant(0)),
+ &return_undefined);
+
+ GotoIf(IsJSReceiverInstanceType(instance_type), &return_object);
+
+ GotoIf(IsStringInstanceType(instance_type), &return_string);
+
+ GotoIf(IsBigIntInstanceType(instance_type), &return_bigint);
+
+ CSA_ASSERT(this, InstanceTypeEqual(instance_type, SYMBOL_TYPE));
+ result_var = HeapConstant(isolate()->factory()->symbol_string());
+ Goto(&return_result);
+
+ BIND(&return_number);
+ {
+ result_var = HeapConstant(isolate()->factory()->number_string());
+ Goto(&return_result);
+ }
+
+ BIND(&if_oddball);
+ {
+ TNode<String> type =
+ CAST(LoadObjectField(value_heap_object, Oddball::kTypeOfOffset));
+ result_var = type;
+ Goto(&return_result);
+ }
+
+ BIND(&return_function);
+ {
+ result_var = HeapConstant(isolate()->factory()->function_string());
+ Goto(&return_result);
+ }
+
+ BIND(&return_undefined);
+ {
+ result_var = HeapConstant(isolate()->factory()->undefined_string());
+ Goto(&return_result);
+ }
+
+ BIND(&return_object);
+ {
+ result_var = HeapConstant(isolate()->factory()->object_string());
+ Goto(&return_result);
+ }
+
+ BIND(&return_string);
+ {
+ result_var = HeapConstant(isolate()->factory()->string_string());
+ Goto(&return_result);
+ }
+
+ BIND(&return_bigint);
+ {
+ result_var = HeapConstant(isolate()->factory()->bigint_string());
+ Goto(&return_result);
+ }
+
+ BIND(&return_result);
+ return result_var.value();
+}
+
+TNode<HeapObject> CodeStubAssembler::GetSuperConstructor(
+ TNode<JSFunction> active_function) {
+ TNode<Map> map = LoadMap(active_function);
+ return LoadMapPrototype(map);
+}
+
+TNode<JSReceiver> CodeStubAssembler::SpeciesConstructor(
+ TNode<Context> context, SloppyTNode<Object> object,
+ TNode<JSReceiver> default_constructor) {
+ Isolate* isolate = this->isolate();
+ TVARIABLE(JSReceiver, var_result, default_constructor);
+
+ // 2. Let C be ? Get(O, "constructor").
+ TNode<Object> constructor =
+ GetProperty(context, object, isolate->factory()->constructor_string());
+
+ // 3. If C is undefined, return defaultConstructor.
+ Label out(this);
+ GotoIf(IsUndefined(constructor), &out);
+
+ // 4. If Type(C) is not Object, throw a TypeError exception.
+ ThrowIfNotJSReceiver(context, constructor,
+ MessageTemplate::kConstructorNotReceiver, "");
+
+ // 5. Let S be ? Get(C, @@species).
+ TNode<Object> species =
+ GetProperty(context, constructor, isolate->factory()->species_symbol());
+
+ // 6. If S is either undefined or null, return defaultConstructor.
+ GotoIf(IsNullOrUndefined(species), &out);
+
+ // 7. If IsConstructor(S) is true, return S.
+ Label throw_error(this);
+ GotoIf(TaggedIsSmi(species), &throw_error);
+ GotoIfNot(IsConstructorMap(LoadMap(CAST(species))), &throw_error);
+ var_result = CAST(species);
+ Goto(&out);
+
+ // 8. Throw a TypeError exception.
+ BIND(&throw_error);
+ ThrowTypeError(context, MessageTemplate::kSpeciesNotConstructor);
+
+ BIND(&out);
+ return var_result.value();
+}
+
+TNode<Oddball> CodeStubAssembler::InstanceOf(TNode<Object> object,
+ TNode<Object> callable,
+ TNode<Context> context) {
+ TVARIABLE(Oddball, var_result);
+ Label if_notcallable(this, Label::kDeferred),
+ if_notreceiver(this, Label::kDeferred), if_otherhandler(this),
+ if_nohandler(this, Label::kDeferred), return_true(this),
+ return_false(this), return_result(this, &var_result);
+
+ // Ensure that the {callable} is actually a JSReceiver.
+ GotoIf(TaggedIsSmi(callable), &if_notreceiver);
+ GotoIfNot(IsJSReceiver(CAST(callable)), &if_notreceiver);
+
+ // Load the @@hasInstance property from {callable}.
+ TNode<Object> inst_of_handler =
+ GetProperty(context, callable, HasInstanceSymbolConstant());
+
+ // Optimize for the likely case where {inst_of_handler} is the builtin
+ // Function.prototype[@@hasInstance] method, and emit a direct call in
+ // that case without any additional checking.
+ TNode<NativeContext> native_context = LoadNativeContext(context);
+ TNode<Object> function_has_instance =
+ LoadContextElement(native_context, Context::FUNCTION_HAS_INSTANCE_INDEX);
+ GotoIfNot(TaggedEqual(inst_of_handler, function_has_instance),
+ &if_otherhandler);
+ {
+ // Call to Function.prototype[@@hasInstance] directly.
+ Callable builtin(BUILTIN_CODE(isolate(), FunctionPrototypeHasInstance),
+ CallTrampolineDescriptor{});
+ var_result =
+ CAST(CallJS(builtin, context, inst_of_handler, callable, object));
+ Goto(&return_result);
+ }
+
+ BIND(&if_otherhandler);
+ {
+ // Check if there's actually an {inst_of_handler}.
+ GotoIf(IsNull(inst_of_handler), &if_nohandler);
+ GotoIf(IsUndefined(inst_of_handler), &if_nohandler);
+
+ // Call the {inst_of_handler} for {callable} and {object}.
+ TNode<Object> result = Call(context, inst_of_handler, callable, object);
+
+ // Convert the {result} to a Boolean.
+ BranchIfToBooleanIsTrue(result, &return_true, &return_false);
+ }
+
+ BIND(&if_nohandler);
+ {
+ // Ensure that the {callable} is actually Callable.
+ GotoIfNot(IsCallable(CAST(callable)), &if_notcallable);
+
+ // Use the OrdinaryHasInstance algorithm.
+ var_result = CAST(
+ CallBuiltin(Builtins::kOrdinaryHasInstance, context, callable, object));
+ Goto(&return_result);
+ }
+
+ BIND(&if_notcallable);
+ { ThrowTypeError(context, MessageTemplate::kNonCallableInInstanceOfCheck); }
+
+ BIND(&if_notreceiver);
+ { ThrowTypeError(context, MessageTemplate::kNonObjectInInstanceOfCheck); }
+
+ BIND(&return_true);
+ var_result = TrueConstant();
+ Goto(&return_result);
+
+ BIND(&return_false);
+ var_result = FalseConstant();
+ Goto(&return_result);
+
+ BIND(&return_result);
+ return var_result.value();
+}
+
+TNode<Number> CodeStubAssembler::NumberInc(TNode<Number> value) {
+ TVARIABLE(Number, var_result);
+ TVARIABLE(Float64T, var_finc_value);
+ Label if_issmi(this), if_isnotsmi(this), do_finc(this), end(this);
+ Branch(TaggedIsSmi(value), &if_issmi, &if_isnotsmi);
+
+ BIND(&if_issmi);
+ {
+ Label if_overflow(this);
+ TNode<Smi> smi_value = CAST(value);
+ TNode<Smi> one = SmiConstant(1);
+ var_result = TrySmiAdd(smi_value, one, &if_overflow);
+ Goto(&end);
+
+ BIND(&if_overflow);
+ {
+ var_finc_value = SmiToFloat64(smi_value);
+ Goto(&do_finc);
+ }
+ }
+
+ BIND(&if_isnotsmi);
+ {
+ TNode<HeapNumber> heap_number_value = CAST(value);
+
+ // Load the HeapNumber value.
+ var_finc_value = LoadHeapNumberValue(heap_number_value);
+ Goto(&do_finc);
+ }
+
+ BIND(&do_finc);
+ {
+ TNode<Float64T> finc_value = var_finc_value.value();
+ TNode<Float64T> one = Float64Constant(1.0);
+ TNode<Float64T> finc_result = Float64Add(finc_value, one);
+ var_result = AllocateHeapNumberWithValue(finc_result);
+ Goto(&end);
+ }
+
+ BIND(&end);
+ return var_result.value();
+}
+
+TNode<Number> CodeStubAssembler::NumberDec(TNode<Number> value) {
+ TVARIABLE(Number, var_result);
+ TVARIABLE(Float64T, var_fdec_value);
+ Label if_issmi(this), if_isnotsmi(this), do_fdec(this), end(this);
+ Branch(TaggedIsSmi(value), &if_issmi, &if_isnotsmi);
+
+ BIND(&if_issmi);
+ {
+ TNode<Smi> smi_value = CAST(value);
+ TNode<Smi> one = SmiConstant(1);
+ Label if_overflow(this);
+ var_result = TrySmiSub(smi_value, one, &if_overflow);
+ Goto(&end);
+
+ BIND(&if_overflow);
+ {
+ var_fdec_value = SmiToFloat64(smi_value);
+ Goto(&do_fdec);
+ }
+ }
+
+ BIND(&if_isnotsmi);
+ {
+ TNode<HeapNumber> heap_number_value = CAST(value);
+
+ // Load the HeapNumber value.
+ var_fdec_value = LoadHeapNumberValue(heap_number_value);
+ Goto(&do_fdec);
+ }
+
+ BIND(&do_fdec);
+ {
+ TNode<Float64T> fdec_value = var_fdec_value.value();
+ TNode<Float64T> minus_one = Float64Constant(-1.0);
+ TNode<Float64T> fdec_result = Float64Add(fdec_value, minus_one);
+ var_result = AllocateHeapNumberWithValue(fdec_result);
+ Goto(&end);
+ }
+
+ BIND(&end);
+ return var_result.value();
+}
+
+TNode<Number> CodeStubAssembler::NumberAdd(TNode<Number> a, TNode<Number> b) {
+ TVARIABLE(Number, var_result);
+ Label float_add(this, Label::kDeferred), end(this);
+ GotoIf(TaggedIsNotSmi(a), &float_add);
+ GotoIf(TaggedIsNotSmi(b), &float_add);
+
+ // Try fast Smi addition first.
+ var_result = TrySmiAdd(CAST(a), CAST(b), &float_add);
+ Goto(&end);
+
+ BIND(&float_add);
+ {
+ var_result = ChangeFloat64ToTagged(
+ Float64Add(ChangeNumberToFloat64(a), ChangeNumberToFloat64(b)));
+ Goto(&end);
+ }
+
+ BIND(&end);
+ return var_result.value();
+}
+
+TNode<Number> CodeStubAssembler::NumberSub(TNode<Number> a, TNode<Number> b) {
+ TVARIABLE(Number, var_result);
+ Label float_sub(this, Label::kDeferred), end(this);
+ GotoIf(TaggedIsNotSmi(a), &float_sub);
+ GotoIf(TaggedIsNotSmi(b), &float_sub);
+
+ // Try fast Smi subtraction first.
+ var_result = TrySmiSub(CAST(a), CAST(b), &float_sub);
+ Goto(&end);
+
+ BIND(&float_sub);
+ {
+ var_result = ChangeFloat64ToTagged(
+ Float64Sub(ChangeNumberToFloat64(a), ChangeNumberToFloat64(b)));
+ Goto(&end);
+ }
+
+ BIND(&end);
+ return var_result.value();
+}
+
+void CodeStubAssembler::GotoIfNotNumber(TNode<Object> input,
+ Label* is_not_number) {
+ Label is_number(this);
+ GotoIf(TaggedIsSmi(input), &is_number);
+ Branch(IsHeapNumber(CAST(input)), &is_number, is_not_number);
+ BIND(&is_number);
+}
+
+void CodeStubAssembler::GotoIfNumber(TNode<Object> input, Label* is_number) {
+ GotoIf(TaggedIsSmi(input), is_number);
+ GotoIf(IsHeapNumber(CAST(input)), is_number);
+}
+
+TNode<Number> CodeStubAssembler::BitwiseOp(TNode<Word32T> left32,
+ TNode<Word32T> right32,
+ Operation bitwise_op) {
+ switch (bitwise_op) {
+ case Operation::kBitwiseAnd:
+ return ChangeInt32ToTagged(Signed(Word32And(left32, right32)));
+ case Operation::kBitwiseOr:
+ return ChangeInt32ToTagged(Signed(Word32Or(left32, right32)));
+ case Operation::kBitwiseXor:
+ return ChangeInt32ToTagged(Signed(Word32Xor(left32, right32)));
+ case Operation::kShiftLeft:
+ if (!Word32ShiftIsSafe()) {
+ right32 = Word32And(right32, Int32Constant(0x1F));
+ }
+ return ChangeInt32ToTagged(Signed(Word32Shl(left32, right32)));
+ case Operation::kShiftRight:
+ if (!Word32ShiftIsSafe()) {
+ right32 = Word32And(right32, Int32Constant(0x1F));
+ }
+ return ChangeInt32ToTagged(Signed(Word32Sar(left32, right32)));
+ case Operation::kShiftRightLogical:
+ if (!Word32ShiftIsSafe()) {
+ right32 = Word32And(right32, Int32Constant(0x1F));
+ }
+ return ChangeUint32ToTagged(Unsigned(Word32Shr(left32, right32)));
+ default:
+ break;
+ }
+ UNREACHABLE();
+}
+
+TNode<JSObject> CodeStubAssembler::AllocateJSIteratorResult(
+ TNode<Context> context, SloppyTNode<Object> value,
+ SloppyTNode<Oddball> done) {
+ CSA_ASSERT(this, IsBoolean(done));
+ TNode<NativeContext> native_context = LoadNativeContext(context);
+ TNode<Map> map = CAST(
+ LoadContextElement(native_context, Context::ITERATOR_RESULT_MAP_INDEX));
+ TNode<HeapObject> result = Allocate(JSIteratorResult::kSize);
+ StoreMapNoWriteBarrier(result, map);
+ StoreObjectFieldRoot(result, JSIteratorResult::kPropertiesOrHashOffset,
+ RootIndex::kEmptyFixedArray);
+ StoreObjectFieldRoot(result, JSIteratorResult::kElementsOffset,
+ RootIndex::kEmptyFixedArray);
+ StoreObjectFieldNoWriteBarrier(result, JSIteratorResult::kValueOffset, value);
+ StoreObjectFieldNoWriteBarrier(result, JSIteratorResult::kDoneOffset, done);
+ return CAST(result);
+}
+
+TNode<JSObject> CodeStubAssembler::AllocateJSIteratorResultForEntry(
+ TNode<Context> context, TNode<Object> key, SloppyTNode<Object> value) {
+ TNode<NativeContext> native_context = LoadNativeContext(context);
+ TNode<Smi> length = SmiConstant(2);
+ int const elements_size = FixedArray::SizeFor(2);
+ TNode<FixedArray> elements = UncheckedCast<FixedArray>(
+ Allocate(elements_size + JSArray::kHeaderSize + JSIteratorResult::kSize));
+ StoreObjectFieldRoot(elements, FixedArray::kMapOffset,
+ RootIndex::kFixedArrayMap);
+ StoreObjectFieldNoWriteBarrier(elements, FixedArray::kLengthOffset, length);
+ StoreFixedArrayElement(elements, 0, key);
+ StoreFixedArrayElement(elements, 1, value);
+ TNode<Map> array_map = CAST(LoadContextElement(
+ native_context, Context::JS_ARRAY_PACKED_ELEMENTS_MAP_INDEX));
+ TNode<HeapObject> array = InnerAllocate(elements, elements_size);
+ StoreMapNoWriteBarrier(array, array_map);
+ StoreObjectFieldRoot(array, JSArray::kPropertiesOrHashOffset,
+ RootIndex::kEmptyFixedArray);
+ StoreObjectFieldNoWriteBarrier(array, JSArray::kElementsOffset, elements);
+ StoreObjectFieldNoWriteBarrier(array, JSArray::kLengthOffset, length);
+ TNode<Map> iterator_map = CAST(
+ LoadContextElement(native_context, Context::ITERATOR_RESULT_MAP_INDEX));
+ TNode<HeapObject> result = InnerAllocate(array, JSArray::kHeaderSize);
+ StoreMapNoWriteBarrier(result, iterator_map);
+ StoreObjectFieldRoot(result, JSIteratorResult::kPropertiesOrHashOffset,
+ RootIndex::kEmptyFixedArray);
+ StoreObjectFieldRoot(result, JSIteratorResult::kElementsOffset,
+ RootIndex::kEmptyFixedArray);
+ StoreObjectFieldNoWriteBarrier(result, JSIteratorResult::kValueOffset, array);
+ StoreObjectFieldRoot(result, JSIteratorResult::kDoneOffset,
+ RootIndex::kFalseValue);
+ return CAST(result);
+}
+
+TNode<JSReceiver> CodeStubAssembler::ArraySpeciesCreate(TNode<Context> context,
+ TNode<Object> o,
+ TNode<Number> len) {
+ TNode<JSReceiver> constructor =
+ CAST(CallRuntime(Runtime::kArraySpeciesConstructor, context, o));
+ return Construct(context, constructor, len);
+}
+
+void CodeStubAssembler::ThrowIfArrayBufferIsDetached(
+ TNode<Context> context, TNode<JSArrayBuffer> array_buffer,
+ const char* method_name) {
+ Label if_detached(this, Label::kDeferred), if_not_detached(this);
+ Branch(IsDetachedBuffer(array_buffer), &if_detached, &if_not_detached);
+ BIND(&if_detached);
+ ThrowTypeError(context, MessageTemplate::kDetachedOperation, method_name);
+ BIND(&if_not_detached);
+}
+
+void CodeStubAssembler::ThrowIfArrayBufferViewBufferIsDetached(
+ TNode<Context> context, TNode<JSArrayBufferView> array_buffer_view,
+ const char* method_name) {
+ TNode<JSArrayBuffer> buffer = LoadJSArrayBufferViewBuffer(array_buffer_view);
+ ThrowIfArrayBufferIsDetached(context, buffer, method_name);
+}
+
+TNode<RawPtrT> CodeStubAssembler::LoadJSArrayBufferBackingStorePtr(
+ TNode<JSArrayBuffer> array_buffer) {
+ return LoadExternalPointerFromObject(array_buffer,
+ JSArrayBuffer::kBackingStoreOffset,
+ kArrayBufferBackingStoreTag);
+}
+
+TNode<JSArrayBuffer> CodeStubAssembler::LoadJSArrayBufferViewBuffer(
+ TNode<JSArrayBufferView> array_buffer_view) {
+ return LoadObjectField<JSArrayBuffer>(array_buffer_view,
+ JSArrayBufferView::kBufferOffset);
+}
+
+TNode<UintPtrT> CodeStubAssembler::LoadJSArrayBufferViewByteLength(
+ TNode<JSArrayBufferView> array_buffer_view) {
+ return LoadObjectField<UintPtrT>(array_buffer_view,
+ JSArrayBufferView::kByteLengthOffset);
+}
+
+TNode<UintPtrT> CodeStubAssembler::LoadJSArrayBufferViewByteOffset(
+ TNode<JSArrayBufferView> array_buffer_view) {
+ return LoadObjectField<UintPtrT>(array_buffer_view,
+ JSArrayBufferView::kByteOffsetOffset);
+}
+
+TNode<UintPtrT> CodeStubAssembler::LoadJSTypedArrayLength(
+ TNode<JSTypedArray> typed_array) {
+ return LoadObjectField<UintPtrT>(typed_array, JSTypedArray::kLengthOffset);
+}
+
+TNode<JSArrayBuffer> CodeStubAssembler::GetTypedArrayBuffer(
+ TNode<Context> context, TNode<JSTypedArray> array) {
+ Label call_runtime(this), done(this);
+ TVARIABLE(Object, var_result);
+
+ TNode<JSArrayBuffer> buffer = LoadJSArrayBufferViewBuffer(array);
+ GotoIf(IsDetachedBuffer(buffer), &call_runtime);
+ TNode<RawPtrT> backing_store = LoadJSArrayBufferBackingStorePtr(buffer);
+ GotoIf(WordEqual(backing_store, IntPtrConstant(0)), &call_runtime);
+ var_result = buffer;
+ Goto(&done);
+
+ BIND(&call_runtime);
+ {
+ var_result = CallRuntime(Runtime::kTypedArrayGetBuffer, context, array);
+ Goto(&done);
+ }
+
+ BIND(&done);
+ return CAST(var_result.value());
+}
+
+CodeStubArguments::CodeStubArguments(CodeStubAssembler* assembler,
+ TNode<IntPtrT> argc, TNode<RawPtrT> fp)
+ : assembler_(assembler),
+ argc_(argc),
+ base_(),
+ fp_(fp != nullptr ? fp : assembler_->LoadFramePointer()) {
+ TNode<IntPtrT> offset = assembler_->IntPtrConstant(
+ (StandardFrameConstants::kFixedSlotCountAboveFp + 1) *
+ kSystemPointerSize);
+ // base_ points to the first argument, not the receiver
+ // whether present or not.
+ base_ = assembler_->RawPtrAdd(fp_, offset);
+}
+
+TNode<Object> CodeStubArguments::GetReceiver() const {
+ intptr_t offset = -kSystemPointerSize;
+ return assembler_->LoadFullTagged(base_, assembler_->IntPtrConstant(offset));
+}
+
+void CodeStubArguments::SetReceiver(TNode<Object> object) const {
+ intptr_t offset = -kSystemPointerSize;
+ assembler_->StoreFullTaggedNoWriteBarrier(
+ base_, assembler_->IntPtrConstant(offset), object);
+}
+
+TNode<RawPtrT> CodeStubArguments::AtIndexPtr(TNode<IntPtrT> index) const {
+ TNode<IntPtrT> offset =
+ assembler_->ElementOffsetFromIndex(index, SYSTEM_POINTER_ELEMENTS, 0);
+ return assembler_->RawPtrAdd(base_, offset);
+}
+
+TNode<Object> CodeStubArguments::AtIndex(TNode<IntPtrT> index) const {
+ CSA_ASSERT(assembler_, assembler_->UintPtrOrSmiLessThan(index, GetLength()));
+ return assembler_->UncheckedCast<Object>(
+ assembler_->LoadFullTagged(AtIndexPtr(index)));
+}
+
+TNode<Object> CodeStubArguments::AtIndex(int index) const {
+ return AtIndex(assembler_->IntPtrConstant(index));
+}
+
+TNode<Object> CodeStubArguments::GetOptionalArgumentValue(
+ TNode<IntPtrT> index, TNode<Object> default_value) {
+ CodeStubAssembler::TVariable<Object> result(assembler_);
+ CodeStubAssembler::Label argument_missing(assembler_),
+ argument_done(assembler_, &result);
+
+ assembler_->GotoIf(assembler_->UintPtrGreaterThanOrEqual(index, argc_),
+ &argument_missing);
+ result = AtIndex(index);
+ assembler_->Goto(&argument_done);
+
+ assembler_->BIND(&argument_missing);
+ result = default_value;
+ assembler_->Goto(&argument_done);
+
+ assembler_->BIND(&argument_done);
+ return result.value();
+}
+
+void CodeStubArguments::ForEach(
+ const CodeStubAssembler::VariableList& vars,
+ const CodeStubArguments::ForEachBodyFunction& body, TNode<IntPtrT> first,
+ TNode<IntPtrT> last) const {
+ assembler_->Comment("CodeStubArguments::ForEach");
+ if (first == nullptr) {
+ first = assembler_->IntPtrConstant(0);
+ }
+ if (last == nullptr) {
+ last = argc_;
+ }
+ TNode<RawPtrT> start = AtIndexPtr(first);
+ TNode<RawPtrT> end = AtIndexPtr(last);
+ const int increment = kSystemPointerSize;
+ assembler_->BuildFastLoop<RawPtrT>(
+ vars, start, end,
+ [&](TNode<RawPtrT> current) {
+ TNode<Object> arg = assembler_->LoadFullTagged(current);
+ body(arg);
+ },
+ increment, CodeStubAssembler::IndexAdvanceMode::kPost);
+}
+
+void CodeStubArguments::PopAndReturn(TNode<Object> value) {
+ TNode<IntPtrT> pop_count =
+ assembler_->IntPtrAdd(argc_, assembler_->IntPtrConstant(1));
+ assembler_->PopAndReturn(pop_count, value);
+}
+
+TNode<BoolT> CodeStubAssembler::IsFastElementsKind(
+ TNode<Int32T> elements_kind) {
+ STATIC_ASSERT(FIRST_ELEMENTS_KIND == FIRST_FAST_ELEMENTS_KIND);
+ return Uint32LessThanOrEqual(elements_kind,
+ Int32Constant(LAST_FAST_ELEMENTS_KIND));
+}
+
+TNode<BoolT> CodeStubAssembler::IsFastOrNonExtensibleOrSealedElementsKind(
+ TNode<Int32T> elements_kind) {
+ STATIC_ASSERT(FIRST_ELEMENTS_KIND == FIRST_FAST_ELEMENTS_KIND);
+ STATIC_ASSERT(LAST_FAST_ELEMENTS_KIND + 1 == PACKED_NONEXTENSIBLE_ELEMENTS);
+ STATIC_ASSERT(PACKED_NONEXTENSIBLE_ELEMENTS + 1 ==
+ HOLEY_NONEXTENSIBLE_ELEMENTS);
+ STATIC_ASSERT(HOLEY_NONEXTENSIBLE_ELEMENTS + 1 == PACKED_SEALED_ELEMENTS);
+ STATIC_ASSERT(PACKED_SEALED_ELEMENTS + 1 == HOLEY_SEALED_ELEMENTS);
+ return Uint32LessThanOrEqual(elements_kind,
+ Int32Constant(HOLEY_SEALED_ELEMENTS));
+}
+
+TNode<BoolT> CodeStubAssembler::IsDoubleElementsKind(
+ TNode<Int32T> elements_kind) {
+ STATIC_ASSERT(FIRST_ELEMENTS_KIND == FIRST_FAST_ELEMENTS_KIND);
+ STATIC_ASSERT((PACKED_DOUBLE_ELEMENTS & 1) == 0);
+ STATIC_ASSERT(PACKED_DOUBLE_ELEMENTS + 1 == HOLEY_DOUBLE_ELEMENTS);
+ return Word32Equal(Word32Shr(elements_kind, Int32Constant(1)),
+ Int32Constant(PACKED_DOUBLE_ELEMENTS / 2));
+}
+
+TNode<BoolT> CodeStubAssembler::IsFastSmiOrTaggedElementsKind(
+ TNode<Int32T> elements_kind) {
+ STATIC_ASSERT(FIRST_ELEMENTS_KIND == FIRST_FAST_ELEMENTS_KIND);
+ STATIC_ASSERT(PACKED_DOUBLE_ELEMENTS > TERMINAL_FAST_ELEMENTS_KIND);
+ STATIC_ASSERT(HOLEY_DOUBLE_ELEMENTS > TERMINAL_FAST_ELEMENTS_KIND);
+ return Uint32LessThanOrEqual(elements_kind,
+ Int32Constant(TERMINAL_FAST_ELEMENTS_KIND));
+}
+
+TNode<BoolT> CodeStubAssembler::IsFastSmiElementsKind(
+ SloppyTNode<Int32T> elements_kind) {
+ return Uint32LessThanOrEqual(elements_kind,
+ Int32Constant(HOLEY_SMI_ELEMENTS));
+}
+
+TNode<BoolT> CodeStubAssembler::IsHoleyFastElementsKind(
+ TNode<Int32T> elements_kind) {
+ CSA_ASSERT(this, IsFastElementsKind(elements_kind));
+
+ STATIC_ASSERT(HOLEY_SMI_ELEMENTS == (PACKED_SMI_ELEMENTS | 1));
+ STATIC_ASSERT(HOLEY_ELEMENTS == (PACKED_ELEMENTS | 1));
+ STATIC_ASSERT(HOLEY_DOUBLE_ELEMENTS == (PACKED_DOUBLE_ELEMENTS | 1));
+ return IsSetWord32(elements_kind, 1);
+}
+
+TNode<BoolT> CodeStubAssembler::IsHoleyFastElementsKindForRead(
+ TNode<Int32T> elements_kind) {
+ CSA_ASSERT(this, Uint32LessThanOrEqual(
+ elements_kind,
+ Int32Constant(LAST_ANY_NONEXTENSIBLE_ELEMENTS_KIND)));
+
+ STATIC_ASSERT(HOLEY_SMI_ELEMENTS == (PACKED_SMI_ELEMENTS | 1));
+ STATIC_ASSERT(HOLEY_ELEMENTS == (PACKED_ELEMENTS | 1));
+ STATIC_ASSERT(HOLEY_DOUBLE_ELEMENTS == (PACKED_DOUBLE_ELEMENTS | 1));
+ STATIC_ASSERT(HOLEY_NONEXTENSIBLE_ELEMENTS ==
+ (PACKED_NONEXTENSIBLE_ELEMENTS | 1));
+ STATIC_ASSERT(HOLEY_SEALED_ELEMENTS == (PACKED_SEALED_ELEMENTS | 1));
+ STATIC_ASSERT(HOLEY_FROZEN_ELEMENTS == (PACKED_FROZEN_ELEMENTS | 1));
+ return IsSetWord32(elements_kind, 1);
+}
+
+TNode<BoolT> CodeStubAssembler::IsElementsKindGreaterThan(
+ TNode<Int32T> target_kind, ElementsKind reference_kind) {
+ return Int32GreaterThan(target_kind, Int32Constant(reference_kind));
+}
+
+TNode<BoolT> CodeStubAssembler::IsElementsKindLessThanOrEqual(
+ TNode<Int32T> target_kind, ElementsKind reference_kind) {
+ return Int32LessThanOrEqual(target_kind, Int32Constant(reference_kind));
+}
+
+TNode<BoolT> CodeStubAssembler::IsDebugActive() {
+ TNode<Uint8T> is_debug_active = Load<Uint8T>(
+ ExternalConstant(ExternalReference::debug_is_active_address(isolate())));
+ return Word32NotEqual(is_debug_active, Int32Constant(0));
+}
+
+TNode<BoolT> CodeStubAssembler::IsPromiseHookEnabled() {
+ const TNode<RawPtrT> promise_hook = Load<RawPtrT>(
+ ExternalConstant(ExternalReference::promise_hook_address(isolate())));
+ return WordNotEqual(promise_hook, IntPtrConstant(0));
+}
+
+TNode<BoolT> CodeStubAssembler::HasAsyncEventDelegate() {
+ const TNode<RawPtrT> async_event_delegate = Load<RawPtrT>(ExternalConstant(
+ ExternalReference::async_event_delegate_address(isolate())));
+ return WordNotEqual(async_event_delegate, IntPtrConstant(0));
+}
+
+TNode<BoolT> CodeStubAssembler::IsPromiseHookEnabledOrHasAsyncEventDelegate() {
+ const TNode<Uint8T> promise_hook_or_async_event_delegate =
+ Load<Uint8T>(ExternalConstant(
+ ExternalReference::promise_hook_or_async_event_delegate_address(
+ isolate())));
+ return Word32NotEqual(promise_hook_or_async_event_delegate, Int32Constant(0));
+}
+
+TNode<BoolT> CodeStubAssembler::
+ IsPromiseHookEnabledOrDebugIsActiveOrHasAsyncEventDelegate() {
+ const TNode<Uint8T> promise_hook_or_debug_is_active_or_async_event_delegate =
+ Load<Uint8T>(ExternalConstant(
+ ExternalReference::
+ promise_hook_or_debug_is_active_or_async_event_delegate_address(
+ isolate())));
+ return Word32NotEqual(promise_hook_or_debug_is_active_or_async_event_delegate,
+ Int32Constant(0));
+}
+
+TNode<Code> CodeStubAssembler::LoadBuiltin(TNode<Smi> builtin_id) {
+ CSA_ASSERT(this, SmiBelow(builtin_id, SmiConstant(Builtins::builtin_count)));
+
+ TNode<IntPtrT> offset =
+ ElementOffsetFromIndex(SmiToBInt(builtin_id), SYSTEM_POINTER_ELEMENTS);
+
+ return CAST(BitcastWordToTagged(
+ Load(MachineType::Pointer(),
+ ExternalConstant(ExternalReference::builtins_address(isolate())),
+ offset)));
+}
+
+TNode<Code> CodeStubAssembler::GetSharedFunctionInfoCode(
+ TNode<SharedFunctionInfo> shared_info, Label* if_compile_lazy) {
+ TNode<Object> sfi_data =
+ LoadObjectField(shared_info, SharedFunctionInfo::kFunctionDataOffset);
+
+ TVARIABLE(Code, sfi_code);
+
+ Label done(this);
+ Label check_instance_type(this);
+
+ // IsSmi: Is builtin
+ GotoIf(TaggedIsNotSmi(sfi_data), &check_instance_type);
+ if (if_compile_lazy) {
+ GotoIf(SmiEqual(CAST(sfi_data), SmiConstant(Builtins::kCompileLazy)),
+ if_compile_lazy);
+ }
+ sfi_code = LoadBuiltin(CAST(sfi_data));
+ Goto(&done);
+
+ // Switch on data's instance type.
+ BIND(&check_instance_type);
+ TNode<Uint16T> data_type = LoadInstanceType(CAST(sfi_data));
+
+ int32_t case_values[] = {BYTECODE_ARRAY_TYPE,
+ WASM_EXPORTED_FUNCTION_DATA_TYPE,
+ ASM_WASM_DATA_TYPE,
+ UNCOMPILED_DATA_WITHOUT_PREPARSE_DATA_TYPE,
+ UNCOMPILED_DATA_WITH_PREPARSE_DATA_TYPE,
+ FUNCTION_TEMPLATE_INFO_TYPE,
+ WASM_JS_FUNCTION_DATA_TYPE,
+ WASM_CAPI_FUNCTION_DATA_TYPE};
+ Label check_is_bytecode_array(this);
+ Label check_is_exported_function_data(this);
+ Label check_is_asm_wasm_data(this);
+ Label check_is_uncompiled_data_without_preparse_data(this);
+ Label check_is_uncompiled_data_with_preparse_data(this);
+ Label check_is_function_template_info(this);
+ Label check_is_interpreter_data(this);
+ Label check_is_wasm_js_function_data(this);
+ Label check_is_wasm_capi_function_data(this);
+ Label* case_labels[] = {&check_is_bytecode_array,
+ &check_is_exported_function_data,
+ &check_is_asm_wasm_data,
+ &check_is_uncompiled_data_without_preparse_data,
+ &check_is_uncompiled_data_with_preparse_data,
+ &check_is_function_template_info,
+ &check_is_wasm_js_function_data,
+ &check_is_wasm_capi_function_data};
+ STATIC_ASSERT(arraysize(case_values) == arraysize(case_labels));
+ Switch(data_type, &check_is_interpreter_data, case_values, case_labels,
+ arraysize(case_labels));
+
+ // IsBytecodeArray: Interpret bytecode
+ BIND(&check_is_bytecode_array);
+ sfi_code = HeapConstant(BUILTIN_CODE(isolate(), InterpreterEntryTrampoline));
+ Goto(&done);
+
+ // IsWasmExportedFunctionData: Use the wrapper code
+ BIND(&check_is_exported_function_data);
+ sfi_code = CAST(LoadObjectField(
+ CAST(sfi_data), WasmExportedFunctionData::kWrapperCodeOffset));
+ Goto(&done);
+
+ // IsAsmWasmData: Instantiate using AsmWasmData
+ BIND(&check_is_asm_wasm_data);
+ sfi_code = HeapConstant(BUILTIN_CODE(isolate(), InstantiateAsmJs));
+ Goto(&done);
+
+ // IsUncompiledDataWithPreparseData | IsUncompiledDataWithoutPreparseData:
+ // Compile lazy
+ BIND(&check_is_uncompiled_data_with_preparse_data);
+ Goto(&check_is_uncompiled_data_without_preparse_data);
+ BIND(&check_is_uncompiled_data_without_preparse_data);
+ sfi_code = HeapConstant(BUILTIN_CODE(isolate(), CompileLazy));
+ Goto(if_compile_lazy ? if_compile_lazy : &done);
+
+ // IsFunctionTemplateInfo: API call
+ BIND(&check_is_function_template_info);
+ sfi_code = HeapConstant(BUILTIN_CODE(isolate(), HandleApiCall));
+ Goto(&done);
+
+ // IsInterpreterData: Interpret bytecode
+ BIND(&check_is_interpreter_data);
+ // This is the default branch, so assert that we have the expected data type.
+ CSA_ASSERT(this,
+ Word32Equal(data_type, Int32Constant(INTERPRETER_DATA_TYPE)));
+ sfi_code = CAST(LoadObjectField(
+ CAST(sfi_data), InterpreterData::kInterpreterTrampolineOffset));
+ Goto(&done);
+
+ // IsWasmJSFunctionData: Use the wrapper code.
+ BIND(&check_is_wasm_js_function_data);
+ sfi_code = CAST(
+ LoadObjectField(CAST(sfi_data), WasmJSFunctionData::kWrapperCodeOffset));
+ Goto(&done);
+
+ // IsWasmCapiFunctionData: Use the wrapper code.
+ BIND(&check_is_wasm_capi_function_data);
+ sfi_code = CAST(LoadObjectField(CAST(sfi_data),
+ WasmCapiFunctionData::kWrapperCodeOffset));
+ Goto(&done);
+
+ BIND(&done);
+ return sfi_code.value();
+}
+
+TNode<JSFunction> CodeStubAssembler::AllocateFunctionWithMapAndContext(
+ TNode<Map> map, TNode<SharedFunctionInfo> shared_info,
+ TNode<Context> context) {
+ const TNode<Code> code = GetSharedFunctionInfoCode(shared_info);
+
+ // TODO(ishell): All the callers of this function pass map loaded from
+ // Context::STRICT_FUNCTION_WITHOUT_PROTOTYPE_MAP_INDEX. So we can remove
+ // map parameter.
+ CSA_ASSERT(this, Word32BinaryNot(IsConstructorMap(map)));
+ CSA_ASSERT(this, Word32BinaryNot(IsFunctionWithPrototypeSlotMap(map)));
+ const TNode<HeapObject> fun = Allocate(JSFunction::kSizeWithoutPrototype);
+ STATIC_ASSERT(JSFunction::kSizeWithoutPrototype == 7 * kTaggedSize);
+ StoreMapNoWriteBarrier(fun, map);
+ StoreObjectFieldRoot(fun, JSObject::kPropertiesOrHashOffset,
+ RootIndex::kEmptyFixedArray);
+ StoreObjectFieldRoot(fun, JSObject::kElementsOffset,
+ RootIndex::kEmptyFixedArray);
+ StoreObjectFieldRoot(fun, JSFunction::kFeedbackCellOffset,
+ RootIndex::kManyClosuresCell);
+ StoreObjectFieldNoWriteBarrier(fun, JSFunction::kSharedFunctionInfoOffset,
+ shared_info);
+ StoreObjectFieldNoWriteBarrier(fun, JSFunction::kContextOffset, context);
+ StoreObjectFieldNoWriteBarrier(fun, JSFunction::kCodeOffset, code);
+ return CAST(fun);
+}
+
+void CodeStubAssembler::CheckPrototypeEnumCache(TNode<JSReceiver> receiver,
+ TNode<Map> receiver_map,
+ Label* if_fast,
+ Label* if_slow) {
+ TVARIABLE(JSReceiver, var_object, receiver);
+ TVARIABLE(Map, object_map, receiver_map);
+
+ Label loop(this, {&var_object, &object_map}), done_loop(this);
+ Goto(&loop);
+ BIND(&loop);
+ {
+ // Check that there are no elements on the current {var_object}.
+ Label if_no_elements(this);
+
+ // The following relies on the elements only aliasing with JSProxy::target,
+ // which is a JavaScript value and hence cannot be confused with an elements
+ // backing store.
+ STATIC_ASSERT(static_cast<int>(JSObject::kElementsOffset) ==
+ static_cast<int>(JSProxy::kTargetOffset));
+ TNode<Object> object_elements =
+ LoadObjectField(var_object.value(), JSObject::kElementsOffset);
+ GotoIf(IsEmptyFixedArray(object_elements), &if_no_elements);
+ GotoIf(IsEmptySlowElementDictionary(object_elements), &if_no_elements);
+
+ // It might still be an empty JSArray.
+ GotoIfNot(IsJSArrayMap(object_map.value()), if_slow);
+ TNode<Number> object_length = LoadJSArrayLength(CAST(var_object.value()));
+ Branch(TaggedEqual(object_length, SmiConstant(0)), &if_no_elements,
+ if_slow);
+
+ // Continue with {var_object}'s prototype.
+ BIND(&if_no_elements);
+ TNode<HeapObject> object = LoadMapPrototype(object_map.value());
+ GotoIf(IsNull(object), if_fast);
+
+ // For all {object}s but the {receiver}, check that the cache is empty.
+ var_object = CAST(object);
+ object_map = LoadMap(object);
+ TNode<WordT> object_enum_length = LoadMapEnumLength(object_map.value());
+ Branch(WordEqual(object_enum_length, IntPtrConstant(0)), &loop, if_slow);
+ }
+}
+
+TNode<Map> CodeStubAssembler::CheckEnumCache(TNode<JSReceiver> receiver,
+ Label* if_empty,
+ Label* if_runtime) {
+ Label if_fast(this), if_cache(this), if_no_cache(this, Label::kDeferred);
+ TNode<Map> receiver_map = LoadMap(receiver);
+
+ // Check if the enum length field of the {receiver} is properly initialized,
+ // indicating that there is an enum cache.
+ TNode<WordT> receiver_enum_length = LoadMapEnumLength(receiver_map);
+ Branch(WordEqual(receiver_enum_length,
+ IntPtrConstant(kInvalidEnumCacheSentinel)),
+ &if_no_cache, &if_cache);
+
+ BIND(&if_no_cache);
+ {
+ // Avoid runtime-call for empty dictionary receivers.
+ GotoIfNot(IsDictionaryMap(receiver_map), if_runtime);
+ TNode<HashTableBase> properties =
+ UncheckedCast<HashTableBase>(LoadSlowProperties(receiver));
+ CSA_ASSERT(this, Word32Or(IsNameDictionary(properties),
+ IsGlobalDictionary(properties)));
+ STATIC_ASSERT(static_cast<int>(NameDictionary::kNumberOfElementsIndex) ==
+ static_cast<int>(GlobalDictionary::kNumberOfElementsIndex));
+ TNode<Smi> length = GetNumberOfElements(properties);
+ GotoIfNot(TaggedEqual(length, SmiConstant(0)), if_runtime);
+ // Check that there are no elements on the {receiver} and its prototype
+ // chain. Given that we do not create an EnumCache for dict-mode objects,
+ // directly jump to {if_empty} if there are no elements and no properties
+ // on the {receiver}.
+ CheckPrototypeEnumCache(receiver, receiver_map, if_empty, if_runtime);
+ }
+
+ // Check that there are no elements on the fast {receiver} and its
+ // prototype chain.
+ BIND(&if_cache);
+ CheckPrototypeEnumCache(receiver, receiver_map, &if_fast, if_runtime);
+
+ BIND(&if_fast);
+ return receiver_map;
+}
+
+TNode<Object> CodeStubAssembler::GetArgumentValue(TorqueStructArguments args,
+ TNode<IntPtrT> index) {
+ return CodeStubArguments(this, args).GetOptionalArgumentValue(index);
+}
+
+TorqueStructArguments CodeStubAssembler::GetFrameArguments(
+ TNode<RawPtrT> frame, TNode<IntPtrT> argc) {
+ return CodeStubArguments(this, argc, frame).GetTorqueArguments();
+}
+
+void CodeStubAssembler::Print(const char* s) {
+ std::string formatted(s);
+ formatted += "\n";
+ CallRuntime(Runtime::kGlobalPrint, NoContextConstant(),
+ StringConstant(formatted.c_str()));
+}
+
+void CodeStubAssembler::Print(const char* prefix,
+ TNode<MaybeObject> tagged_value) {
+ if (prefix != nullptr) {
+ std::string formatted(prefix);
+ formatted += ": ";
+ Handle<String> string = isolate()->factory()->NewStringFromAsciiChecked(
+ formatted.c_str(), AllocationType::kOld);
+ CallRuntime(Runtime::kGlobalPrint, NoContextConstant(),
+ HeapConstant(string));
+ }
+ // CallRuntime only accepts Objects, so do an UncheckedCast to object.
+ // DebugPrint explicitly checks whether the tagged value is a MaybeObject.
+ TNode<Object> arg = UncheckedCast<Object>(tagged_value);
+ CallRuntime(Runtime::kDebugPrint, NoContextConstant(), arg);
+}
+
+void CodeStubAssembler::PerformStackCheck(TNode<Context> context) {
+ Label ok(this), stack_check_interrupt(this, Label::kDeferred);
+
+ TNode<UintPtrT> stack_limit = UncheckedCast<UintPtrT>(
+ Load(MachineType::Pointer(),
+ ExternalConstant(ExternalReference::address_of_jslimit(isolate()))));
+ TNode<BoolT> sp_within_limit = StackPointerGreaterThan(stack_limit);
+
+ Branch(sp_within_limit, &ok, &stack_check_interrupt);
+
+ BIND(&stack_check_interrupt);
+ CallRuntime(Runtime::kStackGuard, context);
+ Goto(&ok);
+
+ BIND(&ok);
+}
+
+TNode<Object> CodeStubAssembler::CallApiCallback(
+ TNode<Object> context, TNode<RawPtrT> callback, TNode<IntPtrT> argc,
+ TNode<Object> data, TNode<Object> holder, TNode<Object> receiver) {
+ Callable callable = CodeFactory::CallApiCallback(isolate());
+ return CallStub(callable, context, callback, argc, data, holder, receiver);
+}
+
+TNode<Object> CodeStubAssembler::CallApiCallback(
+ TNode<Object> context, TNode<RawPtrT> callback, TNode<IntPtrT> argc,
+ TNode<Object> data, TNode<Object> holder, TNode<Object> receiver,
+ TNode<Object> value) {
+ Callable callable = CodeFactory::CallApiCallback(isolate());
+ return CallStub(callable, context, callback, argc, data, holder, receiver,
+ value);
+}
+
+TNode<Object> CodeStubAssembler::CallRuntimeNewArray(
+ TNode<Context> context, TNode<Object> receiver, TNode<Object> length,
+ TNode<Object> new_target, TNode<Object> allocation_site) {
+ // Runtime_NewArray receives arguments in the JS order (to avoid unnecessary
+ // copy). Except the last two (new_target and allocation_site) which are add
+ // on top of the stack later.
+ return CallRuntime(Runtime::kNewArray, context, length, receiver, new_target,
+ allocation_site);
+}
+
+void CodeStubAssembler::TailCallRuntimeNewArray(TNode<Context> context,
+ TNode<Object> receiver,
+ TNode<Object> length,
+ TNode<Object> new_target,
+ TNode<Object> allocation_site) {
+ // Runtime_NewArray receives arguments in the JS order (to avoid unnecessary
+ // copy). Except the last two (new_target and allocation_site) which are add
+ // on top of the stack later.
+ return TailCallRuntime(Runtime::kNewArray, context, length, receiver,
+ new_target, allocation_site);
+}
+
+TNode<JSArray> CodeStubAssembler::ArrayCreate(TNode<Context> context,
+ TNode<Number> length) {
+ TVARIABLE(JSArray, array);
+ Label allocate_js_array(this);
+
+ Label done(this), next(this), runtime(this, Label::kDeferred);
+ TNode<Smi> limit = SmiConstant(JSArray::kInitialMaxFastElementArray);
+ CSA_ASSERT_BRANCH(this, [=](Label* ok, Label* not_ok) {
+ BranchIfNumberRelationalComparison(Operation::kGreaterThanOrEqual, length,
+ SmiConstant(0), ok, not_ok);
+ });
+ // This check also transitively covers the case where length is too big
+ // to be representable by a SMI and so is not usable with
+ // AllocateJSArray.
+ BranchIfNumberRelationalComparison(Operation::kGreaterThanOrEqual, length,
+ limit, &runtime, &next);
+
+ BIND(&runtime);
+ {
+ TNode<NativeContext> native_context = LoadNativeContext(context);
+ TNode<JSFunction> array_function =
+ CAST(LoadContextElement(native_context, Context::ARRAY_FUNCTION_INDEX));
+ array = CAST(CallRuntimeNewArray(context, array_function, length,
+ array_function, UndefinedConstant()));
+ Goto(&done);
+ }
+
+ BIND(&next);
+ TNode<Smi> length_smi = CAST(length);
+
+ TNode<Map> array_map = CAST(LoadContextElement(
+ context, Context::JS_ARRAY_PACKED_SMI_ELEMENTS_MAP_INDEX));
+
+ // TODO(delphick): Consider using
+ // AllocateUninitializedJSArrayWithElements to avoid initializing an
+ // array and then writing over it.
+ array = AllocateJSArray(PACKED_SMI_ELEMENTS, array_map, length_smi,
+ SmiConstant(0));
+ Goto(&done);
+
+ BIND(&done);
+ return array.value();
+}
+
+void CodeStubAssembler::SetPropertyLength(TNode<Context> context,
+ TNode<Object> array,
+ TNode<Number> length) {
+ Label fast(this), runtime(this), done(this);
+ // There's no need to set the length, if
+ // 1) the array is a fast JS array and
+ // 2) the new length is equal to the old length.
+ // as the set is not observable. Otherwise fall back to the run-time.
+
+ // 1) Check that the array has fast elements.
+ // TODO(delphick): Consider changing this since it does an an unnecessary
+ // check for SMIs.
+ // TODO(delphick): Also we could hoist this to after the array construction
+ // and copy the args into array in the same way as the Array constructor.
+ BranchIfFastJSArray(array, context, &fast, &runtime);
+
+ BIND(&fast);
+ {
+ TNode<JSArray> fast_array = CAST(array);
+
+ TNode<Smi> length_smi = CAST(length);
+ TNode<Smi> old_length = LoadFastJSArrayLength(fast_array);
+ CSA_ASSERT(this, TaggedIsPositiveSmi(old_length));
+
+ // 2) If the created array's length matches the required length, then
+ // there's nothing else to do. Otherwise use the runtime to set the
+ // property as that will insert holes into excess elements or shrink
+ // the backing store as appropriate.
+ Branch(SmiNotEqual(length_smi, old_length), &runtime, &done);
+ }
+
+ BIND(&runtime);
+ {
+ SetPropertyStrict(context, array, CodeStubAssembler::LengthStringConstant(),
+ length);
+ Goto(&done);
+ }
+
+ BIND(&done);
+}
+
+TNode<Smi> CodeStubAssembler::RefillMathRandom(
+ TNode<NativeContext> native_context) {
+ // Cache exhausted, populate the cache. Return value is the new index.
+ const TNode<ExternalReference> refill_math_random =
+ ExternalConstant(ExternalReference::refill_math_random());
+ const TNode<ExternalReference> isolate_ptr =
+ ExternalConstant(ExternalReference::isolate_address(isolate()));
+ MachineType type_tagged = MachineType::AnyTagged();
+ MachineType type_ptr = MachineType::Pointer();
+
+ return CAST(CallCFunction(refill_math_random, type_tagged,
+ std::make_pair(type_ptr, isolate_ptr),
+ std::make_pair(type_tagged, native_context)));
+}
+
+TNode<String> CodeStubAssembler::TaggedToDirectString(TNode<Object> value,
+ Label* fail) {
+ ToDirectStringAssembler to_direct(state(), CAST(value));
+ to_direct.TryToDirect(fail);
+ to_direct.PointerToData(fail);
+ return CAST(value);
+}
+
+void CodeStubAssembler::RemoveFinalizationRegistryCellFromUnregisterTokenMap(
+ TNode<JSFinalizationRegistry> finalization_registry,
+ TNode<WeakCell> weak_cell) {
+ const TNode<ExternalReference> remove_cell = ExternalConstant(
+ ExternalReference::
+ js_finalization_registry_remove_cell_from_unregister_token_map());
+ const TNode<ExternalReference> isolate_ptr =
+ ExternalConstant(ExternalReference::isolate_address(isolate()));
+
+ CallCFunction(remove_cell, MachineType::Pointer(),
+ std::make_pair(MachineType::Pointer(), isolate_ptr),
+ std::make_pair(MachineType::AnyTagged(), finalization_registry),
+ std::make_pair(MachineType::AnyTagged(), weak_cell));
+}
+
+PrototypeCheckAssembler::PrototypeCheckAssembler(
+ compiler::CodeAssemblerState* state, Flags flags,
+ TNode<NativeContext> native_context, TNode<Map> initial_prototype_map,
+ Vector<DescriptorIndexNameValue> properties)
+ : CodeStubAssembler(state),
+ flags_(flags),
+ native_context_(native_context),
+ initial_prototype_map_(initial_prototype_map),
+ properties_(properties) {}
+
+void PrototypeCheckAssembler::CheckAndBranch(TNode<HeapObject> prototype,
+ Label* if_unmodified,
+ Label* if_modified) {
+ TNode<Map> prototype_map = LoadMap(prototype);
+ TNode<DescriptorArray> descriptors = LoadMapDescriptors(prototype_map);
+
+ // The continuation of a failed fast check: if property identity checks are
+ // enabled, we continue there (since they may still classify the prototype as
+ // fast), otherwise we bail out.
+ Label property_identity_check(this, Label::kDeferred);
+ Label* if_fast_check_failed =
+ ((flags_ & kCheckPrototypePropertyIdentity) == 0)
+ ? if_modified
+ : &property_identity_check;
+
+ if ((flags_ & kCheckPrototypePropertyConstness) != 0) {
+ // A simple prototype map identity check. Note that map identity does not
+ // guarantee unmodified properties. It does guarantee that no new properties
+ // have been added, or old properties deleted.
+
+ GotoIfNot(TaggedEqual(prototype_map, initial_prototype_map_),
+ if_fast_check_failed);
+
+ // We need to make sure that relevant properties in the prototype have
+ // not been tampered with. We do this by checking that their slots
+ // in the prototype's descriptor array are still marked as const.
+
+ TNode<Uint32T> combined_details;
+ for (int i = 0; i < properties_.length(); i++) {
+ // Assert the descriptor index is in-bounds.
+ int descriptor = properties_[i].descriptor_index;
+ CSA_ASSERT(this, Int32LessThan(Int32Constant(descriptor),
+ LoadNumberOfDescriptors(descriptors)));
+
+ // Assert that the name is correct. This essentially checks that
+ // the descriptor index corresponds to the insertion order in
+ // the bootstrapper.
+ CSA_ASSERT(
+ this,
+ TaggedEqual(LoadKeyByDescriptorEntry(descriptors, descriptor),
+ CodeAssembler::LoadRoot(properties_[i].name_root_index)));
+
+ TNode<Uint32T> details =
+ DescriptorArrayGetDetails(descriptors, Uint32Constant(descriptor));
+
+ if (i == 0) {
+ combined_details = details;
+ } else {
+ combined_details = Word32And(combined_details, details);
+ }
+ }
+
+ TNode<Uint32T> constness =
+ DecodeWord32<PropertyDetails::ConstnessField>(combined_details);
+
+ Branch(
+ Word32Equal(constness,
+ Int32Constant(static_cast<int>(PropertyConstness::kConst))),
+ if_unmodified, if_fast_check_failed);
+ }
+
+ if ((flags_ & kCheckPrototypePropertyIdentity) != 0) {
+ // The above checks have failed, for whatever reason (maybe the prototype
+ // map has changed, or a property is no longer const). This block implements
+ // a more thorough check that can also accept maps which 1. do not have the
+ // initial map, 2. have mutable relevant properties, but 3. still match the
+ // expected value for all relevant properties.
+
+ BIND(&property_identity_check);
+
+ int max_descriptor_index = -1;
+ for (int i = 0; i < properties_.length(); i++) {
+ max_descriptor_index =
+ std::max(max_descriptor_index, properties_[i].descriptor_index);
+ }
+
+ // If the greatest descriptor index is out of bounds, the map cannot be
+ // fast.
+ GotoIfNot(Int32LessThan(Int32Constant(max_descriptor_index),
+ LoadNumberOfDescriptors(descriptors)),
+ if_modified);
+
+ // Logic below only handles maps with fast properties.
+ GotoIfMapHasSlowProperties(prototype_map, if_modified);
+
+ for (int i = 0; i < properties_.length(); i++) {
+ const DescriptorIndexNameValue& p = properties_[i];
+ const int descriptor = p.descriptor_index;
+
+ // Check if the name is correct. This essentially checks that
+ // the descriptor index corresponds to the insertion order in
+ // the bootstrapper.
+ GotoIfNot(TaggedEqual(LoadKeyByDescriptorEntry(descriptors, descriptor),
+ CodeAssembler::LoadRoot(p.name_root_index)),
+ if_modified);
+
+ // Finally, check whether the actual value equals the expected value.
+ TNode<Uint32T> details =
+ DescriptorArrayGetDetails(descriptors, Uint32Constant(descriptor));
+ TVARIABLE(Uint32T, var_details, details);
+ TVARIABLE(Object, var_value);
+
+ const int key_index = DescriptorArray::ToKeyIndex(descriptor);
+ LoadPropertyFromFastObject(prototype, prototype_map, descriptors,
+ IntPtrConstant(key_index), &var_details,
+ &var_value);
+
+ TNode<Object> actual_value = var_value.value();
+ TNode<Object> expected_value =
+ LoadContextElement(native_context_, p.expected_value_context_index);
+ GotoIfNot(TaggedEqual(actual_value, expected_value), if_modified);
+ }
+
+ Goto(if_unmodified);
+ }
+}
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/code-stub-assembler.h b/src/codegen/code-stub-assembler.h
new file mode 100644
index 0000000..89e9556
--- /dev/null
+++ b/src/codegen/code-stub-assembler.h
@@ -0,0 +1,3929 @@
+// Copyright 2016 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_CODE_STUB_ASSEMBLER_H_
+#define V8_CODEGEN_CODE_STUB_ASSEMBLER_H_
+
+#include <functional>
+
+#include "src/base/macros.h"
+#include "src/codegen/bailout-reason.h"
+#include "src/common/external-pointer.h"
+#include "src/common/globals.h"
+#include "src/common/message-template.h"
+#include "src/compiler/code-assembler.h"
+#include "src/objects/arguments.h"
+#include "src/objects/bigint.h"
+#include "src/objects/feedback-vector.h"
+#include "src/objects/js-function.h"
+#include "src/objects/objects.h"
+#include "src/objects/promise.h"
+#include "src/objects/shared-function-info.h"
+#include "src/objects/smi.h"
+#include "src/objects/tagged-index.h"
+#include "src/roots/roots.h"
+#include "torque-generated/exported-macros-assembler.h"
+
+namespace v8 {
+namespace internal {
+
+class CallInterfaceDescriptor;
+class CodeStubArguments;
+class CodeStubAssembler;
+class StatsCounter;
+class StubCache;
+
+enum class PrimitiveType { kBoolean, kNumber, kString, kSymbol };
+
+#define HEAP_MUTABLE_IMMOVABLE_OBJECT_LIST(V) \
+ V(ArrayIteratorProtector, array_iterator_protector, ArrayIteratorProtector) \
+ V(ArraySpeciesProtector, array_species_protector, ArraySpeciesProtector) \
+ V(AsyncFunctionAwaitRejectSharedFun, async_function_await_reject_shared_fun, \
+ AsyncFunctionAwaitRejectSharedFun) \
+ V(AsyncFunctionAwaitResolveSharedFun, \
+ async_function_await_resolve_shared_fun, \
+ AsyncFunctionAwaitResolveSharedFun) \
+ V(AsyncGeneratorAwaitRejectSharedFun, \
+ async_generator_await_reject_shared_fun, \
+ AsyncGeneratorAwaitRejectSharedFun) \
+ V(AsyncGeneratorAwaitResolveSharedFun, \
+ async_generator_await_resolve_shared_fun, \
+ AsyncGeneratorAwaitResolveSharedFun) \
+ V(AsyncGeneratorReturnClosedRejectSharedFun, \
+ async_generator_return_closed_reject_shared_fun, \
+ AsyncGeneratorReturnClosedRejectSharedFun) \
+ V(AsyncGeneratorReturnClosedResolveSharedFun, \
+ async_generator_return_closed_resolve_shared_fun, \
+ AsyncGeneratorReturnClosedResolveSharedFun) \
+ V(AsyncGeneratorReturnResolveSharedFun, \
+ async_generator_return_resolve_shared_fun, \
+ AsyncGeneratorReturnResolveSharedFun) \
+ V(AsyncGeneratorYieldResolveSharedFun, \
+ async_generator_yield_resolve_shared_fun, \
+ AsyncGeneratorYieldResolveSharedFun) \
+ V(AsyncIteratorValueUnwrapSharedFun, async_iterator_value_unwrap_shared_fun, \
+ AsyncIteratorValueUnwrapSharedFun) \
+ V(MapIteratorProtector, map_iterator_protector, MapIteratorProtector) \
+ V(NoElementsProtector, no_elements_protector, NoElementsProtector) \
+ V(NumberStringCache, number_string_cache, NumberStringCache) \
+ V(PromiseAllResolveElementSharedFun, promise_all_resolve_element_shared_fun, \
+ PromiseAllResolveElementSharedFun) \
+ V(PromiseAllSettledRejectElementSharedFun, \
+ promise_all_settled_reject_element_shared_fun, \
+ PromiseAllSettledRejectElementSharedFun) \
+ V(PromiseAllSettledResolveElementSharedFun, \
+ promise_all_settled_resolve_element_shared_fun, \
+ PromiseAllSettledResolveElementSharedFun) \
+ V(PromiseAnyRejectElementSharedFun, promise_any_reject_element_shared_fun, \
+ PromiseAnyRejectElementSharedFun) \
+ V(PromiseCapabilityDefaultRejectSharedFun, \
+ promise_capability_default_reject_shared_fun, \
+ PromiseCapabilityDefaultRejectSharedFun) \
+ V(PromiseCapabilityDefaultResolveSharedFun, \
+ promise_capability_default_resolve_shared_fun, \
+ PromiseCapabilityDefaultResolveSharedFun) \
+ V(PromiseCatchFinallySharedFun, promise_catch_finally_shared_fun, \
+ PromiseCatchFinallySharedFun) \
+ V(PromiseGetCapabilitiesExecutorSharedFun, \
+ promise_get_capabilities_executor_shared_fun, \
+ PromiseGetCapabilitiesExecutorSharedFun) \
+ V(PromiseResolveProtector, promise_resolve_protector, \
+ PromiseResolveProtector) \
+ V(PromiseSpeciesProtector, promise_species_protector, \
+ PromiseSpeciesProtector) \
+ V(PromiseThenFinallySharedFun, promise_then_finally_shared_fun, \
+ PromiseThenFinallySharedFun) \
+ V(PromiseThenProtector, promise_then_protector, PromiseThenProtector) \
+ V(PromiseThrowerFinallySharedFun, promise_thrower_finally_shared_fun, \
+ PromiseThrowerFinallySharedFun) \
+ V(PromiseValueThunkFinallySharedFun, promise_value_thunk_finally_shared_fun, \
+ PromiseValueThunkFinallySharedFun) \
+ V(ProxyRevokeSharedFun, proxy_revoke_shared_fun, ProxyRevokeSharedFun) \
+ V(RegExpSpeciesProtector, regexp_species_protector, RegExpSpeciesProtector) \
+ V(SetIteratorProtector, set_iterator_protector, SetIteratorProtector) \
+ V(SingleCharacterStringCache, single_character_string_cache, \
+ SingleCharacterStringCache) \
+ V(StringIteratorProtector, string_iterator_protector, \
+ StringIteratorProtector) \
+ V(TypedArraySpeciesProtector, typed_array_species_protector, \
+ TypedArraySpeciesProtector)
+
+#define UNIQUE_INSTANCE_TYPE_IMMUTABLE_IMMOVABLE_MAP_ADAPTER( \
+ V, rootIndexName, rootAccessorName, class_name) \
+ V(rootIndexName, rootAccessorName, class_name##Map)
+
+#define HEAP_IMMUTABLE_IMMOVABLE_OBJECT_LIST(V) \
+ V(AllocationSiteWithoutWeakNextMap, allocation_site_without_weaknext_map, \
+ AllocationSiteWithoutWeakNextMap) \
+ V(AllocationSiteWithWeakNextMap, allocation_site_map, AllocationSiteMap) \
+ V(arguments_to_string, arguments_to_string, ArgumentsToString) \
+ V(Array_string, Array_string, ArrayString) \
+ V(array_to_string, array_to_string, ArrayToString) \
+ V(BooleanMap, boolean_map, BooleanMap) \
+ V(boolean_to_string, boolean_to_string, BooleanToString) \
+ V(ConsOneByteStringMap, cons_one_byte_string_map, ConsOneByteStringMap) \
+ V(ConsStringMap, cons_string_map, ConsStringMap) \
+ V(constructor_string, constructor_string, ConstructorString) \
+ V(date_to_string, date_to_string, DateToString) \
+ V(default_string, default_string, DefaultString) \
+ V(EmptyByteArray, empty_byte_array, EmptyByteArray) \
+ V(EmptyFixedArray, empty_fixed_array, EmptyFixedArray) \
+ V(EmptyScopeInfo, empty_scope_info, EmptyScopeInfo) \
+ V(EmptyPropertyDictionary, empty_property_dictionary, \
+ EmptyPropertyDictionary) \
+ V(EmptySlowElementDictionary, empty_slow_element_dictionary, \
+ EmptySlowElementDictionary) \
+ V(empty_string, empty_string, EmptyString) \
+ V(error_to_string, error_to_string, ErrorToString) \
+ V(errors_string, errors_string, ErrorsString) \
+ V(FalseValue, false_value, False) \
+ V(FixedArrayMap, fixed_array_map, FixedArrayMap) \
+ V(FixedCOWArrayMap, fixed_cow_array_map, FixedCOWArrayMap) \
+ V(Function_string, function_string, FunctionString) \
+ V(function_to_string, function_to_string, FunctionToString) \
+ V(GlobalPropertyCellMap, global_property_cell_map, PropertyCellMap) \
+ V(has_instance_symbol, has_instance_symbol, HasInstanceSymbol) \
+ V(Infinity_string, Infinity_string, InfinityString) \
+ V(is_concat_spreadable_symbol, is_concat_spreadable_symbol, \
+ IsConcatSpreadableSymbol) \
+ V(iterator_symbol, iterator_symbol, IteratorSymbol) \
+ V(length_string, length_string, LengthString) \
+ V(ManyClosuresCellMap, many_closures_cell_map, ManyClosuresCellMap) \
+ V(match_symbol, match_symbol, MatchSymbol) \
+ V(megamorphic_symbol, megamorphic_symbol, MegamorphicSymbol) \
+ V(message_string, message_string, MessageString) \
+ V(minus_Infinity_string, minus_Infinity_string, MinusInfinityString) \
+ V(MinusZeroValue, minus_zero_value, MinusZero) \
+ V(name_string, name_string, NameString) \
+ V(NanValue, nan_value, Nan) \
+ V(NaN_string, NaN_string, NaNString) \
+ V(next_string, next_string, NextString) \
+ V(NoClosuresCellMap, no_closures_cell_map, NoClosuresCellMap) \
+ V(null_to_string, null_to_string, NullToString) \
+ V(NullValue, null_value, Null) \
+ V(number_string, number_string, NumberString) \
+ V(number_to_string, number_to_string, NumberToString) \
+ V(Object_string, Object_string, ObjectString) \
+ V(object_to_string, object_to_string, ObjectToString) \
+ V(OneByteStringMap, one_byte_string_map, OneByteStringMap) \
+ V(OneClosureCellMap, one_closure_cell_map, OneClosureCellMap) \
+ V(OnePointerFillerMap, one_pointer_filler_map, OnePointerFillerMap) \
+ V(PromiseCapabilityMap, promise_capability_map, PromiseCapabilityMap) \
+ V(promise_forwarding_handler_symbol, promise_forwarding_handler_symbol, \
+ PromiseForwardingHandlerSymbol) \
+ V(PromiseFulfillReactionJobTaskMap, promise_fulfill_reaction_job_task_map, \
+ PromiseFulfillReactionJobTaskMap) \
+ V(promise_handled_by_symbol, promise_handled_by_symbol, \
+ PromiseHandledBySymbol) \
+ V(PromiseReactionMap, promise_reaction_map, PromiseReactionMap) \
+ V(PromiseRejectReactionJobTaskMap, promise_reject_reaction_job_task_map, \
+ PromiseRejectReactionJobTaskMap) \
+ V(PromiseResolveThenableJobTaskMap, promise_resolve_thenable_job_task_map, \
+ PromiseResolveThenableJobTaskMap) \
+ V(prototype_string, prototype_string, PrototypeString) \
+ V(replace_symbol, replace_symbol, ReplaceSymbol) \
+ V(regexp_to_string, regexp_to_string, RegexpToString) \
+ V(resolve_string, resolve_string, ResolveString) \
+ V(return_string, return_string, ReturnString) \
+ V(species_symbol, species_symbol, SpeciesSymbol) \
+ V(StaleRegister, stale_register, StaleRegister) \
+ V(StoreHandler0Map, store_handler0_map, StoreHandler0Map) \
+ V(string_string, string_string, StringString) \
+ V(string_to_string, string_to_string, StringToString) \
+ V(StringMap, string_map, StringMap) \
+ V(TheHoleValue, the_hole_value, TheHole) \
+ V(then_string, then_string, ThenString) \
+ V(toString_string, toString_string, ToStringString) \
+ V(to_primitive_symbol, to_primitive_symbol, ToPrimitiveSymbol) \
+ V(to_string_tag_symbol, to_string_tag_symbol, ToStringTagSymbol) \
+ V(TrueValue, true_value, True) \
+ V(undefined_to_string, undefined_to_string, UndefinedToString) \
+ V(UndefinedValue, undefined_value, Undefined) \
+ V(uninitialized_symbol, uninitialized_symbol, UninitializedSymbol) \
+ V(valueOf_string, valueOf_string, ValueOfString) \
+ V(wasm_wrapped_object_symbol, wasm_wrapped_object_symbol, \
+ WasmWrappedObjectSymbol) \
+ V(zero_string, zero_string, ZeroString) \
+ UNIQUE_INSTANCE_TYPE_MAP_LIST_GENERATOR( \
+ UNIQUE_INSTANCE_TYPE_IMMUTABLE_IMMOVABLE_MAP_ADAPTER, V)
+
+#define HEAP_IMMOVABLE_OBJECT_LIST(V) \
+ HEAP_MUTABLE_IMMOVABLE_OBJECT_LIST(V) \
+ HEAP_IMMUTABLE_IMMOVABLE_OBJECT_LIST(V)
+
+#ifdef DEBUG
+#define CSA_CHECK(csa, x) \
+ (csa)->Check([&]() -> TNode<BoolT> { return x; }, #x, __FILE__, __LINE__)
+#else
+#define CSA_CHECK(csa, x) (csa)->FastCheck(x)
+#endif
+
+#ifdef DEBUG
+// CSA_ASSERT_ARGS generates an
+// std::initializer_list<CodeStubAssembler::ExtraNode> from __VA_ARGS__. It
+// currently supports between 0 and 2 arguments.
+
+// clang-format off
+#define CSA_ASSERT_0_ARGS(...) {}
+#define CSA_ASSERT_1_ARG(a, ...) {{a, #a}}
+#define CSA_ASSERT_2_ARGS(a, b, ...) {{a, #a}, {b, #b}}
+// clang-format on
+#define SWITCH_CSA_ASSERT_ARGS(dummy, a, b, FUNC, ...) FUNC(a, b)
+#define CSA_ASSERT_ARGS(...) \
+ CALL(SWITCH_CSA_ASSERT_ARGS, (, ##__VA_ARGS__, CSA_ASSERT_2_ARGS, \
+ CSA_ASSERT_1_ARG, CSA_ASSERT_0_ARGS))
+// Workaround for MSVC to skip comma in empty __VA_ARGS__.
+#define CALL(x, y) x y
+
+// CSA_ASSERT(csa, <condition>, <extra values to print...>)
+
+#define CSA_ASSERT(csa, condition_node, ...) \
+ (csa)->Assert(condition_node, #condition_node, __FILE__, __LINE__, \
+ CSA_ASSERT_ARGS(__VA_ARGS__))
+
+// CSA_ASSERT_BRANCH(csa, [](Label* ok, Label* not_ok) {...},
+// <extra values to print...>)
+
+#define CSA_ASSERT_BRANCH(csa, gen, ...) \
+ (csa)->Assert(gen, #gen, __FILE__, __LINE__, CSA_ASSERT_ARGS(__VA_ARGS__))
+
+#define CSA_ASSERT_JS_ARGC_OP(csa, Op, op, expected) \
+ (csa)->Assert( \
+ [&]() -> TNode<BoolT> { \
+ const TNode<Word32T> argc = (csa)->UncheckedParameter<Word32T>( \
+ Descriptor::kJSActualArgumentsCount); \
+ return (csa)->Op(argc, (csa)->Int32Constant(expected)); \
+ }, \
+ "argc " #op " " #expected, __FILE__, __LINE__, \
+ {{SmiFromInt32((csa)->UncheckedParameter<Int32T>( \
+ Descriptor::kJSActualArgumentsCount)), \
+ "argc"}})
+
+#define CSA_ASSERT_JS_ARGC_EQ(csa, expected) \
+ CSA_ASSERT_JS_ARGC_OP(csa, Word32Equal, ==, expected)
+
+#define CSA_DEBUG_INFO(name) \
+ { #name, __FILE__, __LINE__ }
+#define BIND(label) Bind(label, CSA_DEBUG_INFO(label))
+#define TYPED_VARIABLE_DEF(type, name, ...) \
+ TVariable<type> name(CSA_DEBUG_INFO(name), __VA_ARGS__)
+#define TYPED_VARIABLE_CONSTRUCTOR(name, ...) \
+ name(CSA_DEBUG_INFO(name), __VA_ARGS__)
+#else // DEBUG
+#define CSA_ASSERT(csa, ...) ((void)0)
+#define CSA_ASSERT_BRANCH(csa, ...) ((void)0)
+#define CSA_ASSERT_JS_ARGC_EQ(csa, expected) ((void)0)
+#define BIND(label) Bind(label)
+#define TYPED_VARIABLE_DEF(type, name, ...) TVariable<type> name(__VA_ARGS__)
+#define TYPED_VARIABLE_CONSTRUCTOR(name, ...) name(__VA_ARGS__)
+#endif // DEBUG
+
+#define TVARIABLE(...) EXPAND(TYPED_VARIABLE_DEF(__VA_ARGS__, this))
+#define TVARIABLE_CONSTRUCTOR(...) \
+ EXPAND(TYPED_VARIABLE_CONSTRUCTOR(__VA_ARGS__, this))
+
+#ifdef ENABLE_SLOW_DCHECKS
+#define CSA_SLOW_ASSERT(csa, ...) \
+ if (FLAG_enable_slow_asserts) { \
+ CSA_ASSERT(csa, __VA_ARGS__); \
+ }
+#else
+#define CSA_SLOW_ASSERT(csa, ...) ((void)0)
+#endif
+
+// Provides a constexpr boolean to be used inside Torque.
+#ifdef V8_NO_ARGUMENTS_ADAPTOR
+constexpr bool kNoArgumentsAdaptor = true;
+#else
+constexpr bool kNoArgumentsAdaptor = false;
+#endif
+
+// Provides JavaScript-specific "macro-assembler" functionality on top of the
+// CodeAssembler. By factoring the JavaScript-isms out of the CodeAssembler,
+// it's possible to add JavaScript-specific useful CodeAssembler "macros"
+// without modifying files in the compiler directory (and requiring a review
+// from a compiler directory OWNER).
+class V8_EXPORT_PRIVATE CodeStubAssembler
+ : public compiler::CodeAssembler,
+ public TorqueGeneratedExportedMacrosAssembler {
+ public:
+ using Node = compiler::Node;
+ using ScopedExceptionHandler = compiler::ScopedExceptionHandler;
+
+ template <typename T>
+ using LazyNode = std::function<TNode<T>()>;
+
+ explicit CodeStubAssembler(compiler::CodeAssemblerState* state);
+
+ enum AllocationFlag : uint8_t {
+ kNone = 0,
+ kDoubleAlignment = 1,
+ kPretenured = 1 << 1,
+ kAllowLargeObjectAllocation = 1 << 2,
+ };
+
+ enum SlackTrackingMode { kWithSlackTracking, kNoSlackTracking };
+
+ using AllocationFlags = base::Flags<AllocationFlag>;
+
+ TNode<IntPtrT> ParameterToIntPtr(TNode<Smi> value) { return SmiUntag(value); }
+ TNode<IntPtrT> ParameterToIntPtr(TNode<IntPtrT> value) { return value; }
+ TNode<IntPtrT> ParameterToIntPtr(TNode<UintPtrT> value) {
+ return Signed(value);
+ }
+
+ TNode<Smi> ParameterToTagged(TNode<Smi> value) { return value; }
+
+ TNode<Smi> ParameterToTagged(TNode<IntPtrT> value) { return SmiTag(value); }
+
+ template <typename TIndex>
+ TNode<TIndex> TaggedToParameter(TNode<Smi> value);
+
+ bool ToParameterConstant(TNode<Smi> node, intptr_t* out) {
+ Smi constant;
+ if (ToSmiConstant(node, &constant)) {
+ *out = static_cast<intptr_t>(constant.value());
+ return true;
+ }
+ return false;
+ }
+
+ bool ToParameterConstant(TNode<IntPtrT> node, intptr_t* out) {
+ intptr_t constant;
+ if (ToIntPtrConstant(node, &constant)) {
+ *out = constant;
+ return true;
+ }
+ return false;
+ }
+
+#if defined(BINT_IS_SMI)
+ TNode<Smi> BIntToSmi(TNode<BInt> source) { return source; }
+ TNode<IntPtrT> BIntToIntPtr(TNode<BInt> source) {
+ return SmiToIntPtr(source);
+ }
+ TNode<BInt> SmiToBInt(TNode<Smi> source) { return source; }
+ TNode<BInt> IntPtrToBInt(TNode<IntPtrT> source) {
+ return SmiFromIntPtr(source);
+ }
+#elif defined(BINT_IS_INTPTR)
+ TNode<Smi> BIntToSmi(TNode<BInt> source) { return SmiFromIntPtr(source); }
+ TNode<IntPtrT> BIntToIntPtr(TNode<BInt> source) { return source; }
+ TNode<BInt> SmiToBInt(TNode<Smi> source) { return SmiToIntPtr(source); }
+ TNode<BInt> IntPtrToBInt(TNode<IntPtrT> source) { return source; }
+#else
+#error Unknown architecture.
+#endif
+
+ TNode<IntPtrT> TaggedIndexToIntPtr(TNode<TaggedIndex> value);
+ TNode<TaggedIndex> IntPtrToTaggedIndex(TNode<IntPtrT> value);
+ // TODO(v8:10047): Get rid of these convertions eventually.
+ TNode<Smi> TaggedIndexToSmi(TNode<TaggedIndex> value);
+ TNode<TaggedIndex> SmiToTaggedIndex(TNode<Smi> value);
+
+ // Pointer compression specific. Ensures that the upper 32 bits of a Smi
+ // contain the sign of a lower 32 bits so that the Smi can be directly used
+ // as an index in element offset computation.
+ TNode<Smi> NormalizeSmiIndex(TNode<Smi> smi_index);
+
+ TNode<Smi> TaggedToSmi(TNode<Object> value, Label* fail) {
+ GotoIf(TaggedIsNotSmi(value), fail);
+ return UncheckedCast<Smi>(value);
+ }
+
+ TNode<Smi> TaggedToPositiveSmi(TNode<Object> value, Label* fail) {
+ GotoIfNot(TaggedIsPositiveSmi(value), fail);
+ return UncheckedCast<Smi>(value);
+ }
+
+ TNode<String> TaggedToDirectString(TNode<Object> value, Label* fail);
+
+ TNode<HeapObject> TaggedToHeapObject(TNode<Object> value, Label* fail) {
+ GotoIf(TaggedIsSmi(value), fail);
+ return UncheckedCast<HeapObject>(value);
+ }
+
+ TNode<Uint16T> Uint16Constant(uint16_t t) {
+ return UncheckedCast<Uint16T>(Int32Constant(t));
+ }
+
+ TNode<JSDataView> HeapObjectToJSDataView(TNode<HeapObject> heap_object,
+ Label* fail) {
+ GotoIfNot(IsJSDataView(heap_object), fail);
+ return CAST(heap_object);
+ }
+
+ TNode<JSProxy> HeapObjectToJSProxy(TNode<HeapObject> heap_object,
+ Label* fail) {
+ GotoIfNot(IsJSProxy(heap_object), fail);
+ return CAST(heap_object);
+ }
+
+ TNode<JSStringIterator> HeapObjectToJSStringIterator(
+ TNode<HeapObject> heap_object, Label* fail) {
+ GotoIfNot(IsJSStringIterator(heap_object), fail);
+ return CAST(heap_object);
+ }
+
+ TNode<JSReceiver> HeapObjectToCallable(TNode<HeapObject> heap_object,
+ Label* fail) {
+ GotoIfNot(IsCallable(heap_object), fail);
+ return CAST(heap_object);
+ }
+
+ TNode<String> HeapObjectToString(TNode<HeapObject> heap_object, Label* fail) {
+ GotoIfNot(IsString(heap_object), fail);
+ return CAST(heap_object);
+ }
+
+ TNode<JSReceiver> HeapObjectToConstructor(TNode<HeapObject> heap_object,
+ Label* fail) {
+ GotoIfNot(IsConstructor(heap_object), fail);
+ return CAST(heap_object);
+ }
+
+ TNode<JSFunction> HeapObjectToJSFunctionWithPrototypeSlot(
+ TNode<HeapObject> heap_object, Label* fail) {
+ GotoIfNot(IsJSFunctionWithPrototypeSlot(heap_object), fail);
+ return CAST(heap_object);
+ }
+
+#define PARAMETER_BINOP(OpName, IntPtrOpName, SmiOpName) \
+ TNode<Smi> OpName(TNode<Smi> a, TNode<Smi> b) { return SmiOpName(a, b); } \
+ TNode<IntPtrT> OpName(TNode<IntPtrT> a, TNode<IntPtrT> b) { \
+ return IntPtrOpName(a, b); \
+ } \
+ TNode<UintPtrT> OpName(TNode<UintPtrT> a, TNode<UintPtrT> b) { \
+ return Unsigned(IntPtrOpName(Signed(a), Signed(b))); \
+ } \
+ TNode<RawPtrT> OpName(TNode<RawPtrT> a, TNode<RawPtrT> b) { \
+ return ReinterpretCast<RawPtrT>(IntPtrOpName( \
+ ReinterpretCast<IntPtrT>(a), ReinterpretCast<IntPtrT>(b))); \
+ }
+ // TODO(v8:9708): Define BInt operations once all uses are ported.
+ PARAMETER_BINOP(IntPtrOrSmiAdd, IntPtrAdd, SmiAdd)
+ PARAMETER_BINOP(IntPtrOrSmiSub, IntPtrSub, SmiSub)
+#undef PARAMETER_BINOP
+
+#define PARAMETER_BINOP(OpName, IntPtrOpName, SmiOpName) \
+ TNode<BoolT> OpName(TNode<Smi> a, TNode<Smi> b) { return SmiOpName(a, b); } \
+ TNode<BoolT> OpName(TNode<IntPtrT> a, TNode<IntPtrT> b) { \
+ return IntPtrOpName(a, b); \
+ } \
+ TNode<BoolT> OpName(TNode<UintPtrT> a, TNode<UintPtrT> b) { \
+ return IntPtrOpName(Signed(a), Signed(b)); \
+ } \
+ TNode<BoolT> OpName(TNode<RawPtrT> a, TNode<RawPtrT> b) { \
+ return IntPtrOpName(a, b); \
+ }
+ // TODO(v8:9708): Define BInt operations once all uses are ported.
+ PARAMETER_BINOP(IntPtrOrSmiEqual, WordEqual, SmiEqual)
+ PARAMETER_BINOP(IntPtrOrSmiNotEqual, WordNotEqual, SmiNotEqual)
+ PARAMETER_BINOP(IntPtrOrSmiLessThanOrEqual, IntPtrLessThanOrEqual,
+ SmiLessThanOrEqual)
+ PARAMETER_BINOP(IntPtrOrSmiGreaterThan, IntPtrGreaterThan, SmiGreaterThan)
+ PARAMETER_BINOP(UintPtrOrSmiLessThan, UintPtrLessThan, SmiBelow)
+ PARAMETER_BINOP(UintPtrOrSmiGreaterThanOrEqual, UintPtrGreaterThanOrEqual,
+ SmiAboveOrEqual)
+#undef PARAMETER_BINOP
+
+ uintptr_t ConstexprUintPtrShl(uintptr_t a, int32_t b) { return a << b; }
+ uintptr_t ConstexprUintPtrShr(uintptr_t a, int32_t b) { return a >> b; }
+ intptr_t ConstexprIntPtrAdd(intptr_t a, intptr_t b) { return a + b; }
+ uintptr_t ConstexprUintPtrAdd(uintptr_t a, uintptr_t b) { return a + b; }
+ intptr_t ConstexprWordNot(intptr_t a) { return ~a; }
+ uintptr_t ConstexprWordNot(uintptr_t a) { return ~a; }
+
+ TNode<BoolT> TaggedEqual(TNode<AnyTaggedT> a, TNode<AnyTaggedT> b) {
+ if (COMPRESS_POINTERS_BOOL) {
+ return Word32Equal(ReinterpretCast<Word32T>(a),
+ ReinterpretCast<Word32T>(b));
+ } else {
+ return WordEqual(ReinterpretCast<WordT>(a), ReinterpretCast<WordT>(b));
+ }
+ }
+
+ TNode<BoolT> TaggedNotEqual(TNode<AnyTaggedT> a, TNode<AnyTaggedT> b) {
+ return Word32BinaryNot(TaggedEqual(a, b));
+ }
+
+ TNode<Smi> NoContextConstant();
+
+#define HEAP_CONSTANT_ACCESSOR(rootIndexName, rootAccessorName, name) \
+ TNode<std::remove_pointer<std::remove_reference<decltype( \
+ std::declval<ReadOnlyRoots>().rootAccessorName())>::type>::type> \
+ name##Constant();
+ HEAP_IMMUTABLE_IMMOVABLE_OBJECT_LIST(HEAP_CONSTANT_ACCESSOR)
+#undef HEAP_CONSTANT_ACCESSOR
+
+#define HEAP_CONSTANT_ACCESSOR(rootIndexName, rootAccessorName, name) \
+ TNode<std::remove_pointer<std::remove_reference<decltype( \
+ std::declval<Heap>().rootAccessorName())>::type>::type> \
+ name##Constant();
+ HEAP_MUTABLE_IMMOVABLE_OBJECT_LIST(HEAP_CONSTANT_ACCESSOR)
+#undef HEAP_CONSTANT_ACCESSOR
+
+#define HEAP_CONSTANT_TEST(rootIndexName, rootAccessorName, name) \
+ TNode<BoolT> Is##name(SloppyTNode<Object> value); \
+ TNode<BoolT> IsNot##name(SloppyTNode<Object> value);
+ HEAP_IMMOVABLE_OBJECT_LIST(HEAP_CONSTANT_TEST)
+#undef HEAP_CONSTANT_TEST
+
+ TNode<BInt> BIntConstant(int value);
+
+ template <typename TIndex>
+ TNode<TIndex> IntPtrOrSmiConstant(int value);
+
+ bool TryGetIntPtrOrSmiConstantValue(TNode<Smi> maybe_constant, int* value);
+ bool TryGetIntPtrOrSmiConstantValue(TNode<IntPtrT> maybe_constant,
+ int* value);
+
+ // Round the 32bits payload of the provided word up to the next power of two.
+ TNode<IntPtrT> IntPtrRoundUpToPowerOfTwo32(TNode<IntPtrT> value);
+ // Select the maximum of the two provided IntPtr values.
+ TNode<IntPtrT> IntPtrMax(SloppyTNode<IntPtrT> left,
+ SloppyTNode<IntPtrT> right);
+ // Select the minimum of the two provided IntPtr values.
+ TNode<IntPtrT> IntPtrMin(SloppyTNode<IntPtrT> left,
+ SloppyTNode<IntPtrT> right);
+ TNode<UintPtrT> UintPtrMin(TNode<UintPtrT> left, TNode<UintPtrT> right);
+
+ // Float64 operations.
+ TNode<Float64T> Float64Ceil(SloppyTNode<Float64T> x);
+ TNode<Float64T> Float64Floor(SloppyTNode<Float64T> x);
+ TNode<Float64T> Float64Round(SloppyTNode<Float64T> x);
+ TNode<Float64T> Float64RoundToEven(SloppyTNode<Float64T> x);
+ TNode<Float64T> Float64Trunc(SloppyTNode<Float64T> x);
+ // Select the minimum of the two provided Number values.
+ TNode<Number> NumberMax(TNode<Number> left, TNode<Number> right);
+ // Select the minimum of the two provided Number values.
+ TNode<Number> NumberMin(TNode<Number> left, TNode<Number> right);
+
+ // Returns true iff the given value fits into smi range and is >= 0.
+ TNode<BoolT> IsValidPositiveSmi(TNode<IntPtrT> value);
+
+ // Tag an IntPtr as a Smi value.
+ TNode<Smi> SmiTag(SloppyTNode<IntPtrT> value);
+ // Untag a Smi value as an IntPtr.
+ TNode<IntPtrT> SmiUntag(SloppyTNode<Smi> value);
+
+ // Smi conversions.
+ TNode<Float64T> SmiToFloat64(SloppyTNode<Smi> value);
+ TNode<Smi> SmiFromIntPtr(SloppyTNode<IntPtrT> value) { return SmiTag(value); }
+ TNode<Smi> SmiFromInt32(SloppyTNode<Int32T> value);
+ TNode<Smi> SmiFromUint32(TNode<Uint32T> value);
+ TNode<IntPtrT> SmiToIntPtr(SloppyTNode<Smi> value) { return SmiUntag(value); }
+ TNode<Int32T> SmiToInt32(SloppyTNode<Smi> value);
+
+ // Smi operations.
+#define SMI_ARITHMETIC_BINOP(SmiOpName, IntPtrOpName, Int32OpName) \
+ TNode<Smi> SmiOpName(TNode<Smi> a, TNode<Smi> b) { \
+ if (SmiValuesAre32Bits()) { \
+ return BitcastWordToTaggedSigned( \
+ IntPtrOpName(BitcastTaggedToWordForTagAndSmiBits(a), \
+ BitcastTaggedToWordForTagAndSmiBits(b))); \
+ } else { \
+ DCHECK(SmiValuesAre31Bits()); \
+ return BitcastWordToTaggedSigned(ChangeInt32ToIntPtr(Int32OpName( \
+ TruncateIntPtrToInt32(BitcastTaggedToWordForTagAndSmiBits(a)), \
+ TruncateIntPtrToInt32(BitcastTaggedToWordForTagAndSmiBits(b))))); \
+ } \
+ }
+ SMI_ARITHMETIC_BINOP(SmiAdd, IntPtrAdd, Int32Add)
+ SMI_ARITHMETIC_BINOP(SmiSub, IntPtrSub, Int32Sub)
+ SMI_ARITHMETIC_BINOP(SmiAnd, WordAnd, Word32And)
+ SMI_ARITHMETIC_BINOP(SmiOr, WordOr, Word32Or)
+#undef SMI_ARITHMETIC_BINOP
+
+ TNode<IntPtrT> TryIntPtrAdd(TNode<IntPtrT> a, TNode<IntPtrT> b,
+ Label* if_overflow);
+ TNode<IntPtrT> TryIntPtrSub(TNode<IntPtrT> a, TNode<IntPtrT> b,
+ Label* if_overflow);
+ TNode<Int32T> TryInt32Mul(TNode<Int32T> a, TNode<Int32T> b,
+ Label* if_overflow);
+ TNode<Smi> TrySmiAdd(TNode<Smi> a, TNode<Smi> b, Label* if_overflow);
+ TNode<Smi> TrySmiSub(TNode<Smi> a, TNode<Smi> b, Label* if_overflow);
+ TNode<Smi> TrySmiAbs(TNode<Smi> a, Label* if_overflow);
+
+ TNode<Smi> SmiShl(TNode<Smi> a, int shift) {
+ return BitcastWordToTaggedSigned(
+ WordShl(BitcastTaggedToWordForTagAndSmiBits(a), shift));
+ }
+
+ TNode<Smi> SmiShr(TNode<Smi> a, int shift) {
+ if (kTaggedSize == kInt64Size) {
+ return BitcastWordToTaggedSigned(
+ WordAnd(WordShr(BitcastTaggedToWordForTagAndSmiBits(a), shift),
+ BitcastTaggedToWordForTagAndSmiBits(SmiConstant(-1))));
+ } else {
+ // For pointer compressed Smis, we want to make sure that we truncate to
+ // int32 before shifting, to avoid the values of the top 32-bits from
+ // leaking into the sign bit of the smi.
+ return BitcastWordToTaggedSigned(WordAnd(
+ ChangeInt32ToIntPtr(Word32Shr(
+ TruncateWordToInt32(BitcastTaggedToWordForTagAndSmiBits(a)),
+ shift)),
+ BitcastTaggedToWordForTagAndSmiBits(SmiConstant(-1))));
+ }
+ }
+
+ TNode<Smi> SmiSar(TNode<Smi> a, int shift) {
+ if (kTaggedSize == kInt64Size) {
+ return BitcastWordToTaggedSigned(
+ WordAnd(WordSar(BitcastTaggedToWordForTagAndSmiBits(a), shift),
+ BitcastTaggedToWordForTagAndSmiBits(SmiConstant(-1))));
+ } else {
+ // For pointer compressed Smis, we want to make sure that we truncate to
+ // int32 before shifting, to avoid the values of the top 32-bits from
+ // changing the sign bit of the smi.
+ return BitcastWordToTaggedSigned(WordAnd(
+ ChangeInt32ToIntPtr(Word32Sar(
+ TruncateWordToInt32(BitcastTaggedToWordForTagAndSmiBits(a)),
+ shift)),
+ BitcastTaggedToWordForTagAndSmiBits(SmiConstant(-1))));
+ }
+ }
+
+ TNode<Smi> WordOrSmiShr(TNode<Smi> a, int shift) { return SmiShr(a, shift); }
+
+ TNode<IntPtrT> WordOrSmiShr(TNode<IntPtrT> a, int shift) {
+ return WordShr(a, shift);
+ }
+
+#define SMI_COMPARISON_OP(SmiOpName, IntPtrOpName, Int32OpName) \
+ TNode<BoolT> SmiOpName(TNode<Smi> a, TNode<Smi> b) { \
+ if (kTaggedSize == kInt64Size) { \
+ return IntPtrOpName(BitcastTaggedToWordForTagAndSmiBits(a), \
+ BitcastTaggedToWordForTagAndSmiBits(b)); \
+ } else { \
+ DCHECK_EQ(kTaggedSize, kInt32Size); \
+ DCHECK(SmiValuesAre31Bits()); \
+ return Int32OpName( \
+ TruncateIntPtrToInt32(BitcastTaggedToWordForTagAndSmiBits(a)), \
+ TruncateIntPtrToInt32(BitcastTaggedToWordForTagAndSmiBits(b))); \
+ } \
+ }
+ SMI_COMPARISON_OP(SmiEqual, WordEqual, Word32Equal)
+ SMI_COMPARISON_OP(SmiNotEqual, WordNotEqual, Word32NotEqual)
+ SMI_COMPARISON_OP(SmiAbove, UintPtrGreaterThan, Uint32GreaterThan)
+ SMI_COMPARISON_OP(SmiAboveOrEqual, UintPtrGreaterThanOrEqual,
+ Uint32GreaterThanOrEqual)
+ SMI_COMPARISON_OP(SmiBelow, UintPtrLessThan, Uint32LessThan)
+ SMI_COMPARISON_OP(SmiLessThan, IntPtrLessThan, Int32LessThan)
+ SMI_COMPARISON_OP(SmiLessThanOrEqual, IntPtrLessThanOrEqual,
+ Int32LessThanOrEqual)
+ SMI_COMPARISON_OP(SmiGreaterThan, IntPtrGreaterThan, Int32GreaterThan)
+ SMI_COMPARISON_OP(SmiGreaterThanOrEqual, IntPtrGreaterThanOrEqual,
+ Int32GreaterThanOrEqual)
+#undef SMI_COMPARISON_OP
+ TNode<Smi> SmiMax(TNode<Smi> a, TNode<Smi> b);
+ TNode<Smi> SmiMin(TNode<Smi> a, TNode<Smi> b);
+ // Computes a % b for Smi inputs a and b; result is not necessarily a Smi.
+ TNode<Number> SmiMod(TNode<Smi> a, TNode<Smi> b);
+ // Computes a * b for Smi inputs a and b; result is not necessarily a Smi.
+ TNode<Number> SmiMul(TNode<Smi> a, TNode<Smi> b);
+ // Tries to compute dividend / divisor for Smi inputs; branching to bailout
+ // if the division needs to be performed as a floating point operation.
+ TNode<Smi> TrySmiDiv(TNode<Smi> dividend, TNode<Smi> divisor, Label* bailout);
+
+ // Compares two Smis a and b as if they were converted to strings and then
+ // compared lexicographically. Returns:
+ // -1 iff x < y.
+ // 0 iff x == y.
+ // 1 iff x > y.
+ TNode<Smi> SmiLexicographicCompare(TNode<Smi> x, TNode<Smi> y);
+
+#ifdef BINT_IS_SMI
+#define BINT_COMPARISON_OP(BIntOpName, SmiOpName, IntPtrOpName) \
+ TNode<BoolT> BIntOpName(TNode<BInt> a, TNode<BInt> b) { \
+ return SmiOpName(a, b); \
+ }
+#else
+#define BINT_COMPARISON_OP(BIntOpName, SmiOpName, IntPtrOpName) \
+ TNode<BoolT> BIntOpName(TNode<BInt> a, TNode<BInt> b) { \
+ return IntPtrOpName(a, b); \
+ }
+#endif
+ BINT_COMPARISON_OP(BIntEqual, SmiEqual, WordEqual)
+ BINT_COMPARISON_OP(BIntNotEqual, SmiNotEqual, WordNotEqual)
+ BINT_COMPARISON_OP(BIntAbove, SmiAbove, UintPtrGreaterThan)
+ BINT_COMPARISON_OP(BIntAboveOrEqual, SmiAboveOrEqual,
+ UintPtrGreaterThanOrEqual)
+ BINT_COMPARISON_OP(BIntBelow, SmiBelow, UintPtrLessThan)
+ BINT_COMPARISON_OP(BIntLessThan, SmiLessThan, IntPtrLessThan)
+ BINT_COMPARISON_OP(BIntLessThanOrEqual, SmiLessThanOrEqual,
+ IntPtrLessThanOrEqual)
+ BINT_COMPARISON_OP(BIntGreaterThan, SmiGreaterThan, IntPtrGreaterThan)
+ BINT_COMPARISON_OP(BIntGreaterThanOrEqual, SmiGreaterThanOrEqual,
+ IntPtrGreaterThanOrEqual)
+#undef BINT_COMPARISON_OP
+
+ // Smi | HeapNumber operations.
+ TNode<Number> NumberInc(TNode<Number> value);
+ TNode<Number> NumberDec(TNode<Number> value);
+ TNode<Number> NumberAdd(TNode<Number> a, TNode<Number> b);
+ TNode<Number> NumberSub(TNode<Number> a, TNode<Number> b);
+ void GotoIfNotNumber(TNode<Object> value, Label* is_not_number);
+ void GotoIfNumber(TNode<Object> value, Label* is_number);
+ TNode<Number> SmiToNumber(TNode<Smi> v) { return v; }
+
+ TNode<Number> BitwiseOp(TNode<Word32T> left32, TNode<Word32T> right32,
+ Operation bitwise_op);
+
+ // Allocate an object of the given size.
+ TNode<HeapObject> AllocateInNewSpace(TNode<IntPtrT> size,
+ AllocationFlags flags = kNone);
+ TNode<HeapObject> AllocateInNewSpace(int size, AllocationFlags flags = kNone);
+ TNode<HeapObject> Allocate(TNode<IntPtrT> size,
+ AllocationFlags flags = kNone);
+
+ TNode<HeapObject> Allocate(int size, AllocationFlags flags = kNone);
+ TNode<HeapObject> InnerAllocate(TNode<HeapObject> previous, int offset);
+ TNode<HeapObject> InnerAllocate(TNode<HeapObject> previous,
+ TNode<IntPtrT> offset);
+
+ TNode<BoolT> IsRegularHeapObjectSize(TNode<IntPtrT> size);
+
+ using BranchGenerator = std::function<void(Label*, Label*)>;
+ template <typename T>
+ using NodeGenerator = std::function<TNode<T>()>;
+ using ExtraNode = std::pair<TNode<Object>, const char*>;
+
+ void Assert(const BranchGenerator& branch, const char* message,
+ const char* file, int line,
+ std::initializer_list<ExtraNode> extra_nodes = {});
+ void Assert(const NodeGenerator<BoolT>& condition_body, const char* message,
+ const char* file, int line,
+ std::initializer_list<ExtraNode> extra_nodes = {});
+ void Assert(TNode<Word32T> condition_node, const char* message,
+ const char* file, int line,
+ std::initializer_list<ExtraNode> extra_nodes = {});
+ void Check(const BranchGenerator& branch, const char* message,
+ const char* file, int line,
+ std::initializer_list<ExtraNode> extra_nodes = {});
+ void Check(const NodeGenerator<BoolT>& condition_body, const char* message,
+ const char* file, int line,
+ std::initializer_list<ExtraNode> extra_nodes = {});
+ void Check(TNode<Word32T> condition_node, const char* message,
+ const char* file, int line,
+ std::initializer_list<ExtraNode> extra_nodes = {});
+ void FailAssert(const char* message,
+ const std::vector<FileAndLine>& files_and_lines,
+ std::initializer_list<ExtraNode> extra_nodes = {});
+
+ void FastCheck(TNode<BoolT> condition);
+
+ // The following Call wrappers call an object according to the semantics that
+ // one finds in the EcmaScript spec, operating on an Callable (e.g. a
+ // JSFunction or proxy) rather than a Code object.
+ template <class... TArgs>
+ TNode<Object> Call(TNode<Context> context, TNode<Object> callable,
+ TNode<JSReceiver> receiver, TArgs... args) {
+ return UncheckedCast<Object>(CallJS(
+ CodeFactory::Call(isolate(), ConvertReceiverMode::kNotNullOrUndefined),
+ context, callable, receiver, args...));
+ }
+ template <class... TArgs>
+ TNode<Object> Call(TNode<Context> context, TNode<Object> callable,
+ TNode<Object> receiver, TArgs... args) {
+ if (IsUndefinedConstant(receiver) || IsNullConstant(receiver)) {
+ return UncheckedCast<Object>(CallJS(
+ CodeFactory::Call(isolate(), ConvertReceiverMode::kNullOrUndefined),
+ context, callable, receiver, args...));
+ }
+ return UncheckedCast<Object>(CallJS(CodeFactory::Call(isolate()), context,
+ callable, receiver, args...));
+ }
+
+ TNode<Object> CallApiCallback(TNode<Object> context, TNode<RawPtrT> callback,
+ TNode<IntPtrT> argc, TNode<Object> data,
+ TNode<Object> holder, TNode<Object> receiver);
+
+ TNode<Object> CallApiCallback(TNode<Object> context, TNode<RawPtrT> callback,
+ TNode<IntPtrT> argc, TNode<Object> data,
+ TNode<Object> holder, TNode<Object> receiver,
+ TNode<Object> value);
+
+ TNode<Object> CallRuntimeNewArray(TNode<Context> context,
+ TNode<Object> receiver,
+ TNode<Object> length,
+ TNode<Object> new_target,
+ TNode<Object> allocation_site);
+
+ void TailCallRuntimeNewArray(TNode<Context> context, TNode<Object> receiver,
+ TNode<Object> length, TNode<Object> new_target,
+ TNode<Object> allocation_site);
+
+ template <class... TArgs>
+ TNode<JSReceiver> ConstructWithTarget(TNode<Context> context,
+ TNode<JSReceiver> target,
+ TNode<JSReceiver> new_target,
+ TArgs... args) {
+ return CAST(ConstructJSWithTarget(CodeFactory::Construct(isolate()),
+ context, target, new_target,
+ implicit_cast<TNode<Object>>(args)...));
+ }
+ template <class... TArgs>
+ TNode<JSReceiver> Construct(TNode<Context> context,
+ TNode<JSReceiver> new_target, TArgs... args) {
+ return ConstructWithTarget(context, new_target, new_target, args...);
+ }
+
+ template <typename T>
+ TNode<T> Select(TNode<BoolT> condition, const NodeGenerator<T>& true_body,
+ const NodeGenerator<T>& false_body) {
+ TVARIABLE(T, value);
+ Label vtrue(this), vfalse(this), end(this);
+ Branch(condition, &vtrue, &vfalse);
+
+ BIND(&vtrue);
+ {
+ value = true_body();
+ Goto(&end);
+ }
+ BIND(&vfalse);
+ {
+ value = false_body();
+ Goto(&end);
+ }
+
+ BIND(&end);
+ return value.value();
+ }
+
+ template <class A>
+ TNode<A> SelectConstant(TNode<BoolT> condition, TNode<A> true_value,
+ TNode<A> false_value) {
+ return Select<A>(
+ condition, [=] { return true_value; }, [=] { return false_value; });
+ }
+
+ TNode<Int32T> SelectInt32Constant(TNode<BoolT> condition, int true_value,
+ int false_value);
+ TNode<IntPtrT> SelectIntPtrConstant(TNode<BoolT> condition, int true_value,
+ int false_value);
+ TNode<Oddball> SelectBooleanConstant(TNode<BoolT> condition);
+ TNode<Smi> SelectSmiConstant(TNode<BoolT> condition, Smi true_value,
+ Smi false_value);
+ TNode<Smi> SelectSmiConstant(TNode<BoolT> condition, int true_value,
+ Smi false_value) {
+ return SelectSmiConstant(condition, Smi::FromInt(true_value), false_value);
+ }
+ TNode<Smi> SelectSmiConstant(TNode<BoolT> condition, Smi true_value,
+ int false_value) {
+ return SelectSmiConstant(condition, true_value, Smi::FromInt(false_value));
+ }
+ TNode<Smi> SelectSmiConstant(TNode<BoolT> condition, int true_value,
+ int false_value) {
+ return SelectSmiConstant(condition, Smi::FromInt(true_value),
+ Smi::FromInt(false_value));
+ }
+
+ TNode<String> SingleCharacterStringConstant(char const* single_char) {
+ DCHECK_EQ(strlen(single_char), 1);
+ return HeapConstant(
+ isolate()->factory()->LookupSingleCharacterStringFromCode(
+ single_char[0]));
+ }
+
+ TNode<Int32T> TruncateWordToInt32(SloppyTNode<WordT> value);
+ TNode<Int32T> TruncateIntPtrToInt32(SloppyTNode<IntPtrT> value);
+
+ // Check a value for smi-ness
+ TNode<BoolT> TaggedIsSmi(TNode<MaybeObject> a);
+ TNode<BoolT> TaggedIsNotSmi(TNode<MaybeObject> a);
+
+ // Check that the value is a non-negative smi.
+ TNode<BoolT> TaggedIsPositiveSmi(SloppyTNode<Object> a);
+ // Check that a word has a word-aligned address.
+ TNode<BoolT> WordIsAligned(SloppyTNode<WordT> word, size_t alignment);
+ TNode<BoolT> WordIsPowerOfTwo(SloppyTNode<IntPtrT> value);
+
+ // Check if lower_limit <= value <= higher_limit.
+ template <typename U>
+ TNode<BoolT> IsInRange(TNode<Word32T> value, U lower_limit, U higher_limit) {
+ DCHECK_LE(lower_limit, higher_limit);
+ STATIC_ASSERT(sizeof(U) <= kInt32Size);
+ return Uint32LessThanOrEqual(Int32Sub(value, Int32Constant(lower_limit)),
+ Int32Constant(higher_limit - lower_limit));
+ }
+
+ TNode<BoolT> IsInRange(TNode<WordT> value, intptr_t lower_limit,
+ intptr_t higher_limit) {
+ DCHECK_LE(lower_limit, higher_limit);
+ return UintPtrLessThanOrEqual(IntPtrSub(value, IntPtrConstant(lower_limit)),
+ IntPtrConstant(higher_limit - lower_limit));
+ }
+
+#if DEBUG
+ void Bind(Label* label, AssemblerDebugInfo debug_info);
+#endif // DEBUG
+ void Bind(Label* label);
+
+ template <class... T>
+ void Bind(compiler::CodeAssemblerParameterizedLabel<T...>* label,
+ TNode<T>*... phis) {
+ CodeAssembler::Bind(label, phis...);
+ }
+
+ void BranchIfSmiEqual(TNode<Smi> a, TNode<Smi> b, Label* if_true,
+ Label* if_false) {
+ Branch(SmiEqual(a, b), if_true, if_false);
+ }
+
+ void BranchIfSmiLessThan(TNode<Smi> a, TNode<Smi> b, Label* if_true,
+ Label* if_false) {
+ Branch(SmiLessThan(a, b), if_true, if_false);
+ }
+
+ void BranchIfSmiLessThanOrEqual(TNode<Smi> a, TNode<Smi> b, Label* if_true,
+ Label* if_false) {
+ Branch(SmiLessThanOrEqual(a, b), if_true, if_false);
+ }
+
+ void BranchIfFloat64IsNaN(TNode<Float64T> value, Label* if_true,
+ Label* if_false) {
+ Branch(Float64Equal(value, value), if_false, if_true);
+ }
+
+ // Branches to {if_true} if ToBoolean applied to {value} yields true,
+ // otherwise goes to {if_false}.
+ void BranchIfToBooleanIsTrue(SloppyTNode<Object> value, Label* if_true,
+ Label* if_false);
+
+ // Branches to {if_false} if ToBoolean applied to {value} yields false,
+ // otherwise goes to {if_true}.
+ void BranchIfToBooleanIsFalse(SloppyTNode<Object> value, Label* if_false,
+ Label* if_true) {
+ BranchIfToBooleanIsTrue(value, if_true, if_false);
+ }
+
+ void BranchIfJSReceiver(SloppyTNode<Object> object, Label* if_true,
+ Label* if_false);
+
+ // Branches to {if_true} when --force-slow-path flag has been passed.
+ // It's used for testing to ensure that slow path implementation behave
+ // equivalent to corresponding fast paths (where applicable).
+ //
+ // Works only with V8_ENABLE_FORCE_SLOW_PATH compile time flag. Nop otherwise.
+ void GotoIfForceSlowPath(Label* if_true);
+
+ //
+ // ExternalPointerT-related functionality.
+ //
+
+ TNode<ExternalPointerT> ChangeUint32ToExternalPointer(TNode<Uint32T> value);
+ TNode<Uint32T> ChangeExternalPointerToUint32(TNode<ExternalPointerT> value);
+
+ // Initialize an external pointer field in an object.
+ void InitializeExternalPointerField(TNode<HeapObject> object, int offset) {
+ InitializeExternalPointerField(object, IntPtrConstant(offset));
+ }
+ void InitializeExternalPointerField(TNode<HeapObject> object,
+ TNode<IntPtrT> offset);
+
+ // Initialize an external pointer field in an object with given value.
+ void InitializeExternalPointerField(TNode<HeapObject> object, int offset,
+ TNode<RawPtrT> pointer,
+ ExternalPointerTag tag) {
+ InitializeExternalPointerField(object, IntPtrConstant(offset), pointer,
+ tag);
+ }
+
+ void InitializeExternalPointerField(TNode<HeapObject> object,
+ TNode<IntPtrT> offset,
+ TNode<RawPtrT> pointer,
+ ExternalPointerTag tag) {
+ InitializeExternalPointerField(object, offset);
+ StoreExternalPointerToObject(object, offset, pointer, tag);
+ }
+
+ // Load an external pointer value from an object.
+ TNode<RawPtrT> LoadExternalPointerFromObject(TNode<HeapObject> object,
+ int offset,
+ ExternalPointerTag tag) {
+ return LoadExternalPointerFromObject(object, IntPtrConstant(offset), tag);
+ }
+
+ TNode<RawPtrT> LoadExternalPointerFromObject(TNode<HeapObject> object,
+ TNode<IntPtrT> offset,
+ ExternalPointerTag tag);
+
+ // Store external object pointer to object.
+ void StoreExternalPointerToObject(TNode<HeapObject> object, int offset,
+ TNode<RawPtrT> pointer,
+ ExternalPointerTag tag) {
+ StoreExternalPointerToObject(object, IntPtrConstant(offset), pointer, tag);
+ }
+
+ void StoreExternalPointerToObject(TNode<HeapObject> object,
+ TNode<IntPtrT> offset,
+ TNode<RawPtrT> pointer,
+ ExternalPointerTag tag);
+
+ TNode<RawPtrT> LoadForeignForeignAddressPtr(TNode<Foreign> object) {
+ return LoadExternalPointerFromObject(object, Foreign::kForeignAddressOffset,
+ kForeignForeignAddressTag);
+ }
+
+ TNode<RawPtrT> LoadExternalStringResourcePtr(TNode<ExternalString> object) {
+ return LoadExternalPointerFromObject(
+ object, ExternalString::kResourceOffset, kExternalStringResourceTag);
+ }
+
+ TNode<RawPtrT> LoadExternalStringResourceDataPtr(
+ TNode<ExternalString> object) {
+ return LoadExternalPointerFromObject(object,
+ ExternalString::kResourceDataOffset,
+ kExternalStringResourceDataTag);
+ }
+
+ TNode<RawPtrT> LoadJSTypedArrayExternalPointerPtr(
+ TNode<JSTypedArray> holder) {
+ return LoadExternalPointerFromObject(holder,
+ JSTypedArray::kExternalPointerOffset,
+ kTypedArrayExternalPointerTag);
+ }
+
+ void StoreJSTypedArrayExternalPointerPtr(TNode<JSTypedArray> holder,
+ TNode<RawPtrT> value) {
+ StoreExternalPointerToObject(holder, JSTypedArray::kExternalPointerOffset,
+ value, kTypedArrayExternalPointerTag);
+ }
+
+ // Load value from current parent frame by given offset in bytes.
+ TNode<Object> LoadFromParentFrame(int offset);
+
+ // Load an object pointer from a buffer that isn't in the heap.
+ TNode<Object> LoadBufferObject(TNode<RawPtrT> buffer, int offset) {
+ return LoadFullTagged(buffer, IntPtrConstant(offset));
+ }
+ template <typename T>
+ TNode<T> LoadBufferData(TNode<RawPtrT> buffer, int offset) {
+ return UncheckedCast<T>(
+ Load(MachineTypeOf<T>::value, buffer, IntPtrConstant(offset)));
+ }
+ TNode<RawPtrT> LoadBufferPointer(TNode<RawPtrT> buffer, int offset) {
+ return LoadBufferData<RawPtrT>(buffer, offset);
+ }
+ TNode<Smi> LoadBufferSmi(TNode<RawPtrT> buffer, int offset) {
+ return CAST(LoadBufferObject(buffer, offset));
+ }
+ TNode<IntPtrT> LoadBufferIntptr(TNode<RawPtrT> buffer, int offset) {
+ return LoadBufferData<IntPtrT>(buffer, offset);
+ }
+ // Load a field from an object on the heap.
+ template <class T, typename std::enable_if<
+ std::is_convertible<TNode<T>, TNode<Object>>::value,
+ int>::type = 0>
+ TNode<T> LoadObjectField(TNode<HeapObject> object, int offset) {
+ return CAST(LoadFromObject(MachineTypeOf<T>::value, object,
+ IntPtrConstant(offset - kHeapObjectTag)));
+ }
+ template <class T, typename std::enable_if<
+ std::is_convertible<TNode<T>, TNode<UntaggedT>>::value,
+ int>::type = 0>
+ TNode<T> LoadObjectField(TNode<HeapObject> object, int offset) {
+ return UncheckedCast<T>(
+ LoadFromObject(MachineTypeOf<T>::value, object,
+ IntPtrConstant(offset - kHeapObjectTag)));
+ }
+ TNode<Object> LoadObjectField(TNode<HeapObject> object, int offset) {
+ return UncheckedCast<Object>(
+ LoadFromObject(MachineType::AnyTagged(), object,
+ IntPtrConstant(offset - kHeapObjectTag)));
+ }
+ TNode<Object> LoadObjectField(TNode<HeapObject> object,
+ TNode<IntPtrT> offset) {
+ return UncheckedCast<Object>(
+ LoadFromObject(MachineType::AnyTagged(), object,
+ IntPtrSub(offset, IntPtrConstant(kHeapObjectTag))));
+ }
+ template <class T, typename std::enable_if<
+ std::is_convertible<TNode<T>, TNode<UntaggedT>>::value,
+ int>::type = 0>
+ TNode<T> LoadObjectField(TNode<HeapObject> object, TNode<IntPtrT> offset) {
+ return UncheckedCast<T>(
+ LoadFromObject(MachineTypeOf<T>::value, object,
+ IntPtrSub(offset, IntPtrConstant(kHeapObjectTag))));
+ }
+ // Load a SMI field and untag it.
+ TNode<IntPtrT> LoadAndUntagObjectField(TNode<HeapObject> object, int offset);
+ // Load a SMI field, untag it, and convert to Word32.
+ TNode<Int32T> LoadAndUntagToWord32ObjectField(TNode<HeapObject> object,
+ int offset);
+
+ TNode<MaybeObject> LoadMaybeWeakObjectField(TNode<HeapObject> object,
+ int offset) {
+ return UncheckedCast<MaybeObject>(LoadObjectField(object, offset));
+ }
+
+ TNode<Object> LoadConstructorOrBackPointer(TNode<Map> map) {
+ return LoadObjectField(map,
+ Map::kConstructorOrBackPointerOrNativeContextOffset);
+ }
+
+ // Reference is the CSA-equivalent of a Torque reference value,
+ // representing an inner pointer into a HeapObject.
+ // TODO(gsps): Remove in favor of flattened {Load,Store}Reference interface
+ struct Reference {
+ TNode<HeapObject> object;
+ TNode<IntPtrT> offset;
+
+ std::tuple<TNode<HeapObject>, TNode<IntPtrT>> Flatten() const {
+ return std::make_tuple(object, offset);
+ }
+ };
+
+ template <class T, typename std::enable_if<
+ std::is_convertible<TNode<T>, TNode<Object>>::value,
+ int>::type = 0>
+ TNode<T> LoadReference(Reference reference) {
+ TNode<IntPtrT> offset =
+ IntPtrSub(reference.offset, IntPtrConstant(kHeapObjectTag));
+ return CAST(
+ LoadFromObject(MachineTypeOf<T>::value, reference.object, offset));
+ }
+ template <class T,
+ typename std::enable_if<
+ std::is_convertible<TNode<T>, TNode<UntaggedT>>::value ||
+ std::is_same<T, MaybeObject>::value,
+ int>::type = 0>
+ TNode<T> LoadReference(Reference reference) {
+ TNode<IntPtrT> offset =
+ IntPtrSub(reference.offset, IntPtrConstant(kHeapObjectTag));
+ return UncheckedCast<T>(
+ LoadFromObject(MachineTypeOf<T>::value, reference.object, offset));
+ }
+ template <class T, typename std::enable_if<
+ std::is_convertible<TNode<T>, TNode<Object>>::value ||
+ std::is_same<T, MaybeObject>::value,
+ int>::type = 0>
+ void StoreReference(Reference reference, TNode<T> value) {
+ MachineRepresentation rep = MachineRepresentationOf<T>::value;
+ StoreToObjectWriteBarrier write_barrier = StoreToObjectWriteBarrier::kFull;
+ if (std::is_same<T, Smi>::value) {
+ write_barrier = StoreToObjectWriteBarrier::kNone;
+ } else if (std::is_same<T, Map>::value) {
+ write_barrier = StoreToObjectWriteBarrier::kMap;
+ }
+ TNode<IntPtrT> offset =
+ IntPtrSub(reference.offset, IntPtrConstant(kHeapObjectTag));
+ StoreToObject(rep, reference.object, offset, value, write_barrier);
+ }
+ template <class T, typename std::enable_if<
+ std::is_convertible<TNode<T>, TNode<UntaggedT>>::value,
+ int>::type = 0>
+ void StoreReference(Reference reference, TNode<T> value) {
+ TNode<IntPtrT> offset =
+ IntPtrSub(reference.offset, IntPtrConstant(kHeapObjectTag));
+ StoreToObject(MachineRepresentationOf<T>::value, reference.object, offset,
+ value, StoreToObjectWriteBarrier::kNone);
+ }
+
+ // Load the floating point value of a HeapNumber.
+ TNode<Float64T> LoadHeapNumberValue(TNode<HeapObject> object);
+ // Load the Map of an HeapObject.
+ TNode<Map> LoadMap(TNode<HeapObject> object);
+ // Load the instance type of an HeapObject.
+ TNode<Uint16T> LoadInstanceType(TNode<HeapObject> object);
+ // Compare the instance the type of the object against the provided one.
+ TNode<BoolT> HasInstanceType(TNode<HeapObject> object, InstanceType type);
+ TNode<BoolT> DoesntHaveInstanceType(TNode<HeapObject> object,
+ InstanceType type);
+ TNode<BoolT> TaggedDoesntHaveInstanceType(TNode<HeapObject> any_tagged,
+ InstanceType type);
+
+ TNode<Word32T> IsStringWrapperElementsKind(TNode<Map> map);
+ void GotoIfMapHasSlowProperties(TNode<Map> map, Label* if_slow);
+
+ // Load the properties backing store of a JSReceiver.
+ TNode<HeapObject> LoadSlowProperties(TNode<JSReceiver> object);
+ TNode<HeapObject> LoadFastProperties(TNode<JSReceiver> object);
+ // Load the elements backing store of a JSObject.
+ TNode<FixedArrayBase> LoadElements(TNode<JSObject> object) {
+ return LoadJSObjectElements(object);
+ }
+ // Load the length of a JSArray instance.
+ TNode<Object> LoadJSArgumentsObjectLength(TNode<Context> context,
+ TNode<JSArgumentsObject> array);
+ // Load the length of a fast JSArray instance. Returns a positive Smi.
+ TNode<Smi> LoadFastJSArrayLength(TNode<JSArray> array);
+ // Load the length of a fixed array base instance.
+ TNode<Smi> LoadFixedArrayBaseLength(TNode<FixedArrayBase> array);
+ // Load the length of a fixed array base instance.
+ TNode<IntPtrT> LoadAndUntagFixedArrayBaseLength(TNode<FixedArrayBase> array);
+ // Load the length of a WeakFixedArray.
+ TNode<Smi> LoadWeakFixedArrayLength(TNode<WeakFixedArray> array);
+ TNode<IntPtrT> LoadAndUntagWeakFixedArrayLength(TNode<WeakFixedArray> array);
+ // Load the number of descriptors in DescriptorArray.
+ TNode<Int32T> LoadNumberOfDescriptors(TNode<DescriptorArray> array);
+ // Load the number of own descriptors of a map.
+ TNode<Int32T> LoadNumberOfOwnDescriptors(TNode<Map> map);
+ // Load the bit field of a Map.
+ TNode<Int32T> LoadMapBitField(TNode<Map> map);
+ // Load bit field 2 of a map.
+ TNode<Int32T> LoadMapBitField2(TNode<Map> map);
+ // Load bit field 3 of a map.
+ TNode<Uint32T> LoadMapBitField3(TNode<Map> map);
+ // Load the instance type of a map.
+ TNode<Uint16T> LoadMapInstanceType(TNode<Map> map);
+ // Load the ElementsKind of a map.
+ TNode<Int32T> LoadMapElementsKind(TNode<Map> map);
+ TNode<Int32T> LoadElementsKind(TNode<HeapObject> object);
+ // Load the instance descriptors of a map.
+ TNode<DescriptorArray> LoadMapDescriptors(TNode<Map> map);
+ // Load the prototype of a map.
+ TNode<HeapObject> LoadMapPrototype(TNode<Map> map);
+ // Load the instance size of a Map.
+ TNode<IntPtrT> LoadMapInstanceSizeInWords(TNode<Map> map);
+ // Load the inobject properties start of a Map (valid only for JSObjects).
+ TNode<IntPtrT> LoadMapInobjectPropertiesStartInWords(TNode<Map> map);
+ // Load the constructor function index of a Map (only for primitive maps).
+ TNode<IntPtrT> LoadMapConstructorFunctionIndex(TNode<Map> map);
+ // Load the constructor of a Map (equivalent to Map::GetConstructor()).
+ TNode<Object> LoadMapConstructor(TNode<Map> map);
+ // Load the EnumLength of a Map.
+ TNode<WordT> LoadMapEnumLength(TNode<Map> map);
+ // Load the back-pointer of a Map.
+ TNode<Object> LoadMapBackPointer(TNode<Map> map);
+ // Checks that |map| has only simple properties, returns bitfield3.
+ TNode<Uint32T> EnsureOnlyHasSimpleProperties(TNode<Map> map,
+ TNode<Int32T> instance_type,
+ Label* bailout);
+ // Load the identity hash of a JSRececiver.
+ TNode<IntPtrT> LoadJSReceiverIdentityHash(SloppyTNode<Object> receiver,
+ Label* if_no_hash = nullptr);
+
+ // This is only used on a newly allocated PropertyArray which
+ // doesn't have an existing hash.
+ void InitializePropertyArrayLength(TNode<PropertyArray> property_array,
+ TNode<IntPtrT> length);
+
+ // Check if the map is set for slow properties.
+ TNode<BoolT> IsDictionaryMap(TNode<Map> map);
+
+ // Load the Name::hash() value of a name as an uint32 value.
+ // If {if_hash_not_computed} label is specified then it also checks if
+ // hash is actually computed.
+ TNode<Uint32T> LoadNameHash(TNode<Name> name,
+ Label* if_hash_not_computed = nullptr);
+ TNode<Uint32T> LoadNameHashAssumeComputed(TNode<Name> name);
+
+ // Load length field of a String object as Smi value.
+ TNode<Smi> LoadStringLengthAsSmi(TNode<String> string);
+ // Load length field of a String object as intptr_t value.
+ TNode<IntPtrT> LoadStringLengthAsWord(TNode<String> string);
+ // Load length field of a String object as uint32_t value.
+ TNode<Uint32T> LoadStringLengthAsWord32(TNode<String> string);
+ // Load value field of a JSPrimitiveWrapper object.
+ TNode<Object> LoadJSPrimitiveWrapperValue(TNode<JSPrimitiveWrapper> object);
+
+ // Figures out whether the value of maybe_object is:
+ // - a SMI (jump to "if_smi", "extracted" will be the SMI value)
+ // - a cleared weak reference (jump to "if_cleared", "extracted" will be
+ // untouched)
+ // - a weak reference (jump to "if_weak", "extracted" will be the object
+ // pointed to)
+ // - a strong reference (jump to "if_strong", "extracted" will be the object
+ // pointed to)
+ void DispatchMaybeObject(TNode<MaybeObject> maybe_object, Label* if_smi,
+ Label* if_cleared, Label* if_weak, Label* if_strong,
+ TVariable<Object>* extracted);
+ // See MaybeObject for semantics of these functions.
+ TNode<BoolT> IsStrong(TNode<MaybeObject> value);
+ TNode<HeapObject> GetHeapObjectIfStrong(TNode<MaybeObject> value,
+ Label* if_not_strong);
+
+ TNode<BoolT> IsWeakOrCleared(TNode<MaybeObject> value);
+ TNode<BoolT> IsCleared(TNode<MaybeObject> value);
+ TNode<BoolT> IsNotCleared(TNode<MaybeObject> value) {
+ return Word32BinaryNot(IsCleared(value));
+ }
+
+ // Removes the weak bit + asserts it was set.
+ TNode<HeapObject> GetHeapObjectAssumeWeak(TNode<MaybeObject> value);
+
+ TNode<HeapObject> GetHeapObjectAssumeWeak(TNode<MaybeObject> value,
+ Label* if_cleared);
+
+ // Checks if |maybe_object| is a weak reference to given |heap_object|.
+ // Works for both any tagged |maybe_object| values.
+ TNode<BoolT> IsWeakReferenceTo(TNode<MaybeObject> maybe_object,
+ TNode<HeapObject> heap_object);
+ // Returns true if the |object| is a HeapObject and |maybe_object| is a weak
+ // reference to |object|.
+ // The |maybe_object| must not be a Smi.
+ TNode<BoolT> IsWeakReferenceToObject(TNode<MaybeObject> maybe_object,
+ TNode<Object> object);
+
+ TNode<MaybeObject> MakeWeak(TNode<HeapObject> value);
+
+ void FixedArrayBoundsCheck(TNode<FixedArrayBase> array, TNode<Smi> index,
+ int additional_offset);
+
+ void FixedArrayBoundsCheck(TNode<FixedArrayBase> array, TNode<IntPtrT> index,
+ int additional_offset);
+
+ void FixedArrayBoundsCheck(TNode<FixedArrayBase> array, TNode<UintPtrT> index,
+ int additional_offset) {
+ FixedArrayBoundsCheck(array, Signed(index), additional_offset);
+ }
+
+ // Array is any array-like type that has a fixed header followed by
+ // tagged elements.
+ template <typename Array>
+ TNode<IntPtrT> LoadArrayLength(TNode<Array> array);
+
+ // Array is any array-like type that has a fixed header followed by
+ // tagged elements.
+ template <typename Array, typename TIndex, typename TValue = MaybeObject>
+ TNode<TValue> LoadArrayElement(
+ TNode<Array> array, int array_header_size, TNode<TIndex> index,
+ int additional_offset = 0,
+ LoadSensitivity needs_poisoning = LoadSensitivity::kSafe);
+
+ template <typename TIndex>
+ TNode<Object> LoadFixedArrayElement(
+ TNode<FixedArray> object, TNode<TIndex> index, int additional_offset = 0,
+ LoadSensitivity needs_poisoning = LoadSensitivity::kSafe,
+ CheckBounds check_bounds = CheckBounds::kAlways);
+
+ // This doesn't emit a bounds-check. As part of the security-performance
+ // tradeoff, only use it if it is performance critical.
+ TNode<Object> UnsafeLoadFixedArrayElement(
+ TNode<FixedArray> object, TNode<IntPtrT> index, int additional_offset = 0,
+ LoadSensitivity needs_poisoning = LoadSensitivity::kSafe) {
+ return LoadFixedArrayElement(object, index, additional_offset,
+ needs_poisoning, CheckBounds::kDebugOnly);
+ }
+
+ TNode<Object> LoadFixedArrayElement(
+ TNode<FixedArray> object, int index, int additional_offset = 0,
+ LoadSensitivity needs_poisoning = LoadSensitivity::kSafe) {
+ return LoadFixedArrayElement(object, IntPtrConstant(index),
+ additional_offset, needs_poisoning);
+ }
+ // This doesn't emit a bounds-check. As part of the security-performance
+ // tradeoff, only use it if it is performance critical.
+ TNode<Object> UnsafeLoadFixedArrayElement(
+ TNode<FixedArray> object, int index, int additional_offset = 0,
+ LoadSensitivity needs_poisoning = LoadSensitivity::kSafe) {
+ return LoadFixedArrayElement(object, IntPtrConstant(index),
+ additional_offset, needs_poisoning,
+ CheckBounds::kDebugOnly);
+ }
+
+ TNode<Object> LoadPropertyArrayElement(TNode<PropertyArray> object,
+ SloppyTNode<IntPtrT> index);
+ TNode<IntPtrT> LoadPropertyArrayLength(TNode<PropertyArray> object);
+
+ // Load an element from an array and untag it and return it as Word32.
+ // Array is any array-like type that has a fixed header followed by
+ // tagged elements.
+ template <typename Array>
+ TNode<Int32T> LoadAndUntagToWord32ArrayElement(TNode<Array> array,
+ int array_header_size,
+ TNode<IntPtrT> index,
+ int additional_offset = 0);
+
+ // Load an array element from a FixedArray, untag it and return it as Word32.
+ TNode<Int32T> LoadAndUntagToWord32FixedArrayElement(
+ TNode<FixedArray> object, TNode<IntPtrT> index,
+ int additional_offset = 0);
+
+ // Load an array element from a WeakFixedArray.
+ TNode<MaybeObject> LoadWeakFixedArrayElement(TNode<WeakFixedArray> object,
+ TNode<IntPtrT> index,
+ int additional_offset = 0);
+
+ // Load an array element from a FixedDoubleArray.
+ TNode<Float64T> LoadFixedDoubleArrayElement(
+ TNode<FixedDoubleArray> object, TNode<IntPtrT> index,
+ Label* if_hole = nullptr,
+ MachineType machine_type = MachineType::Float64());
+
+ // Load an array element from a FixedArray, FixedDoubleArray or a
+ // NumberDictionary (depending on the |elements_kind|) and return
+ // it as a tagged value. Assumes that the |index| passed a length
+ // check before. Bails out to |if_accessor| if the element that
+ // was found is an accessor, or to |if_hole| if the element at
+ // the given |index| is not found in |elements|.
+ TNode<Object> LoadFixedArrayBaseElementAsTagged(
+ TNode<FixedArrayBase> elements, TNode<IntPtrT> index,
+ TNode<Int32T> elements_kind, Label* if_accessor, Label* if_hole);
+
+ // Load a feedback slot from a FeedbackVector.
+ template <typename TIndex>
+ TNode<MaybeObject> LoadFeedbackVectorSlot(
+ TNode<FeedbackVector> feedback_vector, TNode<TIndex> slot,
+ int additional_offset = 0);
+
+ TNode<IntPtrT> LoadFeedbackVectorLength(TNode<FeedbackVector>);
+ TNode<Float64T> LoadDoubleWithHoleCheck(TNode<FixedDoubleArray> array,
+ TNode<IntPtrT> index,
+ Label* if_hole = nullptr);
+
+ TNode<BoolT> IsDoubleHole(TNode<Object> base, TNode<IntPtrT> offset);
+ // Load Float64 value by |base| + |offset| address. If the value is a double
+ // hole then jump to |if_hole|. If |machine_type| is None then only the hole
+ // check is generated.
+ TNode<Float64T> LoadDoubleWithHoleCheck(
+ TNode<Object> base, TNode<IntPtrT> offset, Label* if_hole,
+ MachineType machine_type = MachineType::Float64());
+ TNode<Numeric> LoadFixedTypedArrayElementAsTagged(TNode<RawPtrT> data_pointer,
+ TNode<UintPtrT> index,
+ ElementsKind elements_kind);
+ TNode<Numeric> LoadFixedTypedArrayElementAsTagged(
+ TNode<RawPtrT> data_pointer, TNode<UintPtrT> index,
+ TNode<Int32T> elements_kind);
+ // Parts of the above, factored out for readability:
+ TNode<BigInt> LoadFixedBigInt64ArrayElementAsTagged(
+ SloppyTNode<RawPtrT> data_pointer, SloppyTNode<IntPtrT> offset);
+ TNode<BigInt> LoadFixedBigUint64ArrayElementAsTagged(
+ SloppyTNode<RawPtrT> data_pointer, SloppyTNode<IntPtrT> offset);
+ // 64-bit platforms only:
+ TNode<BigInt> BigIntFromInt64(TNode<IntPtrT> value);
+ TNode<BigInt> BigIntFromUint64(TNode<UintPtrT> value);
+ // 32-bit platforms only:
+ TNode<BigInt> BigIntFromInt32Pair(TNode<IntPtrT> low, TNode<IntPtrT> high);
+ TNode<BigInt> BigIntFromUint32Pair(TNode<UintPtrT> low, TNode<UintPtrT> high);
+
+ // ScopeInfo:
+ TNode<ScopeInfo> LoadScopeInfo(TNode<Context> context);
+ TNode<BoolT> LoadScopeInfoHasExtensionField(TNode<ScopeInfo> scope_info);
+
+ // Context manipulation:
+ void StoreContextElementNoWriteBarrier(TNode<Context> context, int slot_index,
+ SloppyTNode<Object> value);
+ TNode<NativeContext> LoadNativeContext(TNode<Context> context);
+ // Calling this is only valid if there's a module context in the chain.
+ TNode<Context> LoadModuleContext(TNode<Context> context);
+
+ void GotoIfContextElementEqual(SloppyTNode<Object> value,
+ TNode<NativeContext> native_context,
+ int slot_index, Label* if_equal) {
+ GotoIf(TaggedEqual(value, LoadContextElement(native_context, slot_index)),
+ if_equal);
+ }
+
+ // Loads the initial map of the the Object constructor.
+ TNode<Map> LoadObjectFunctionInitialMap(TNode<NativeContext> native_context);
+ TNode<Map> LoadSlowObjectWithNullPrototypeMap(
+ TNode<NativeContext> native_context);
+
+ TNode<Map> LoadJSArrayElementsMap(ElementsKind kind,
+ TNode<NativeContext> native_context);
+ TNode<Map> LoadJSArrayElementsMap(SloppyTNode<Int32T> kind,
+ TNode<NativeContext> native_context);
+
+ TNode<BoolT> IsJSFunctionWithPrototypeSlot(TNode<HeapObject> object);
+ TNode<BoolT> IsGeneratorFunction(TNode<JSFunction> function);
+ void BranchIfHasPrototypeProperty(TNode<JSFunction> function,
+ TNode<Int32T> function_map_bit_field,
+ Label* if_true, Label* if_false);
+ void GotoIfPrototypeRequiresRuntimeLookup(TNode<JSFunction> function,
+ TNode<Map> map, Label* runtime);
+ // Load the "prototype" property of a JSFunction.
+ TNode<HeapObject> LoadJSFunctionPrototype(TNode<JSFunction> function,
+ Label* if_bailout);
+
+ TNode<BytecodeArray> LoadSharedFunctionInfoBytecodeArray(
+ TNode<SharedFunctionInfo> shared);
+
+ void StoreObjectByteNoWriteBarrier(TNode<HeapObject> object, int offset,
+ TNode<Word32T> value);
+
+ // Store the floating point value of a HeapNumber.
+ void StoreHeapNumberValue(SloppyTNode<HeapNumber> object,
+ SloppyTNode<Float64T> value);
+ // Store a field to an object on the heap.
+ void StoreObjectField(TNode<HeapObject> object, int offset,
+ TNode<Object> value);
+ void StoreObjectField(TNode<HeapObject> object, TNode<IntPtrT> offset,
+ TNode<Object> value);
+ template <class T>
+ void StoreObjectFieldNoWriteBarrier(TNode<HeapObject> object,
+ SloppyTNode<IntPtrT> offset,
+ TNode<T> value) {
+ int const_offset;
+ if (ToInt32Constant(offset, &const_offset)) {
+ return StoreObjectFieldNoWriteBarrier<T>(object, const_offset, value);
+ }
+ StoreNoWriteBarrier(MachineRepresentationOf<T>::value, object,
+ IntPtrSub(offset, IntPtrConstant(kHeapObjectTag)),
+ value);
+ }
+ template <class T>
+ void StoreObjectFieldNoWriteBarrier(TNode<HeapObject> object, int offset,
+ TNode<T> value) {
+ if (CanBeTaggedPointer(MachineRepresentationOf<T>::value)) {
+ OptimizedStoreFieldAssertNoWriteBarrier(MachineRepresentationOf<T>::value,
+ object, offset, value);
+ } else {
+ OptimizedStoreFieldUnsafeNoWriteBarrier(MachineRepresentationOf<T>::value,
+ object, offset, value);
+ }
+ }
+
+ void UnsafeStoreObjectFieldNoWriteBarrier(TNode<HeapObject> object,
+ int offset, TNode<Object> value);
+
+ // Store the Map of an HeapObject.
+ void StoreMap(TNode<HeapObject> object, TNode<Map> map);
+ void StoreMapNoWriteBarrier(TNode<HeapObject> object,
+ RootIndex map_root_index);
+ void StoreMapNoWriteBarrier(TNode<HeapObject> object, TNode<Map> map);
+ void StoreObjectFieldRoot(TNode<HeapObject> object, int offset,
+ RootIndex root);
+ // Store an array element to a FixedArray.
+ void StoreFixedArrayElement(
+ TNode<FixedArray> object, int index, SloppyTNode<Object> value,
+ WriteBarrierMode barrier_mode = UPDATE_WRITE_BARRIER,
+ CheckBounds check_bounds = CheckBounds::kAlways) {
+ return StoreFixedArrayElement(object, IntPtrConstant(index), value,
+ barrier_mode, 0, check_bounds);
+ }
+ // This doesn't emit a bounds-check. As part of the security-performance
+ // tradeoff, only use it if it is performance critical.
+ void UnsafeStoreFixedArrayElement(
+ TNode<FixedArray> object, int index, TNode<Object> value,
+ WriteBarrierMode barrier_mode = UPDATE_WRITE_BARRIER) {
+ return StoreFixedArrayElement(object, index, value, barrier_mode,
+ CheckBounds::kDebugOnly);
+ }
+ void UnsafeStoreFixedArrayElement(TNode<FixedArray> object, int index,
+ TNode<Smi> value) {
+ return StoreFixedArrayElement(object, index, value,
+ UNSAFE_SKIP_WRITE_BARRIER,
+ CheckBounds::kDebugOnly);
+ }
+ void StoreFixedArrayElement(TNode<FixedArray> object, int index,
+ TNode<Smi> value,
+ CheckBounds check_bounds = CheckBounds::kAlways) {
+ return StoreFixedArrayElement(object, IntPtrConstant(index), value,
+ UNSAFE_SKIP_WRITE_BARRIER, 0, check_bounds);
+ }
+ template <typename TIndex>
+ void StoreFixedArrayElement(
+ TNode<FixedArray> array, TNode<TIndex> index, SloppyTNode<Object> value,
+ WriteBarrierMode barrier_mode = UPDATE_WRITE_BARRIER,
+ int additional_offset = 0,
+ CheckBounds check_bounds = CheckBounds::kAlways) {
+ // TODO(v8:9708): Do we want to keep both IntPtrT and UintPtrT variants?
+ static_assert(std::is_same<TIndex, Smi>::value ||
+ std::is_same<TIndex, UintPtrT>::value ||
+ std::is_same<TIndex, IntPtrT>::value,
+ "Only Smi, UintPtrT or IntPtrT index is allowed");
+ if (NeedsBoundsCheck(check_bounds)) {
+ FixedArrayBoundsCheck(array, index, additional_offset);
+ }
+ StoreFixedArrayOrPropertyArrayElement(array, index, value, barrier_mode,
+ additional_offset);
+ }
+ // This doesn't emit a bounds-check. As part of the security-performance
+ // tradeoff, only use it if it is performance critical.
+ void UnsafeStoreFixedArrayElement(
+ TNode<FixedArray> array, TNode<IntPtrT> index, TNode<Object> value,
+ WriteBarrierMode barrier_mode = UPDATE_WRITE_BARRIER,
+ int additional_offset = 0) {
+ return StoreFixedArrayElement(array, index, value, barrier_mode,
+ additional_offset, CheckBounds::kDebugOnly);
+ }
+
+ void UnsafeStoreFixedArrayElement(TNode<FixedArray> array,
+ TNode<IntPtrT> index, TNode<Smi> value,
+ int additional_offset) {
+ return StoreFixedArrayElement(array, index, value,
+ UNSAFE_SKIP_WRITE_BARRIER, additional_offset,
+ CheckBounds::kDebugOnly);
+ }
+
+ void StorePropertyArrayElement(TNode<PropertyArray> array,
+ TNode<IntPtrT> index, TNode<Object> value) {
+ StoreFixedArrayOrPropertyArrayElement(array, index, value,
+ UPDATE_WRITE_BARRIER);
+ }
+
+ void StoreFixedArrayElement(
+ TNode<FixedArray> array, TNode<Smi> index, TNode<Object> value,
+ WriteBarrierMode barrier_mode = UPDATE_WRITE_BARRIER) {
+ StoreFixedArrayElement(array, index, value, barrier_mode, 0);
+ }
+ void StoreFixedArrayElement(
+ TNode<FixedArray> array, TNode<IntPtrT> index, TNode<Smi> value,
+ WriteBarrierMode barrier_mode = SKIP_WRITE_BARRIER,
+ int additional_offset = 0) {
+ DCHECK_EQ(SKIP_WRITE_BARRIER, barrier_mode);
+ StoreFixedArrayElement(array, index, TNode<Object>{value},
+ UNSAFE_SKIP_WRITE_BARRIER, additional_offset);
+ }
+ void StoreFixedArrayElement(
+ TNode<FixedArray> array, TNode<Smi> index, TNode<Smi> value,
+ WriteBarrierMode barrier_mode = SKIP_WRITE_BARRIER,
+ int additional_offset = 0) {
+ DCHECK_EQ(SKIP_WRITE_BARRIER, barrier_mode);
+ StoreFixedArrayElement(array, index, TNode<Object>{value},
+ UNSAFE_SKIP_WRITE_BARRIER, additional_offset);
+ }
+
+ template <typename TIndex>
+ void StoreFixedDoubleArrayElement(
+ TNode<FixedDoubleArray> object, TNode<TIndex> index,
+ TNode<Float64T> value, CheckBounds check_bounds = CheckBounds::kAlways);
+
+ void StoreDoubleHole(TNode<HeapObject> object, TNode<IntPtrT> offset);
+ void StoreFixedDoubleArrayHole(TNode<FixedDoubleArray> array,
+ TNode<IntPtrT> index);
+ void StoreFeedbackVectorSlot(
+ TNode<FeedbackVector> feedback_vector, TNode<UintPtrT> slot,
+ TNode<AnyTaggedT> value,
+ WriteBarrierMode barrier_mode = UPDATE_WRITE_BARRIER,
+ int additional_offset = 0);
+
+ // EnsureArrayPushable verifies that receiver with this map is:
+ // 1. Is not a prototype.
+ // 2. Is not a dictionary.
+ // 3. Has a writeable length property.
+ // It returns ElementsKind as a node for further division into cases.
+ TNode<Int32T> EnsureArrayPushable(TNode<Context> context, TNode<Map> map,
+ Label* bailout);
+
+ void TryStoreArrayElement(ElementsKind kind, Label* bailout,
+ TNode<FixedArrayBase> elements, TNode<BInt> index,
+ TNode<Object> value);
+ // Consumes args into the array, and returns tagged new length.
+ TNode<Smi> BuildAppendJSArray(ElementsKind kind, TNode<JSArray> array,
+ CodeStubArguments* args,
+ TVariable<IntPtrT>* arg_index, Label* bailout);
+ // Pushes value onto the end of array.
+ void BuildAppendJSArray(ElementsKind kind, TNode<JSArray> array,
+ TNode<Object> value, Label* bailout);
+
+ void StoreFieldsNoWriteBarrier(TNode<IntPtrT> start_address,
+ TNode<IntPtrT> end_address,
+ TNode<Object> value);
+
+ // Marks the FixedArray copy-on-write without moving it.
+ void MakeFixedArrayCOW(TNode<FixedArray> array);
+
+ TNode<Cell> AllocateCellWithValue(
+ TNode<Object> value, WriteBarrierMode mode = UPDATE_WRITE_BARRIER);
+ TNode<Cell> AllocateSmiCell(int value = 0) {
+ return AllocateCellWithValue(SmiConstant(value), SKIP_WRITE_BARRIER);
+ }
+
+ TNode<Object> LoadCellValue(TNode<Cell> cell);
+
+ void StoreCellValue(TNode<Cell> cell, TNode<Object> value,
+ WriteBarrierMode mode = UPDATE_WRITE_BARRIER);
+
+ // Allocate a HeapNumber without initializing its value.
+ TNode<HeapNumber> AllocateHeapNumber();
+ // Allocate a HeapNumber with a specific value.
+ TNode<HeapNumber> AllocateHeapNumberWithValue(SloppyTNode<Float64T> value);
+ TNode<HeapNumber> AllocateHeapNumberWithValue(double value) {
+ return AllocateHeapNumberWithValue(Float64Constant(value));
+ }
+
+ // Allocate a BigInt with {length} digits. Sets the sign bit to {false}.
+ // Does not initialize the digits.
+ TNode<BigInt> AllocateBigInt(TNode<IntPtrT> length);
+ // Like above, but allowing custom bitfield initialization.
+ TNode<BigInt> AllocateRawBigInt(TNode<IntPtrT> length);
+ void StoreBigIntBitfield(TNode<BigInt> bigint, TNode<Word32T> bitfield);
+ void StoreBigIntDigit(TNode<BigInt> bigint, intptr_t digit_index,
+ TNode<UintPtrT> digit);
+ void StoreBigIntDigit(TNode<BigInt> bigint, TNode<IntPtrT> digit_index,
+ TNode<UintPtrT> digit);
+
+ TNode<Word32T> LoadBigIntBitfield(TNode<BigInt> bigint);
+ TNode<UintPtrT> LoadBigIntDigit(TNode<BigInt> bigint, intptr_t digit_index);
+ TNode<UintPtrT> LoadBigIntDigit(TNode<BigInt> bigint,
+ TNode<IntPtrT> digit_index);
+
+ // Allocate a ByteArray with the given length.
+ TNode<ByteArray> AllocateByteArray(TNode<UintPtrT> length,
+ AllocationFlags flags = kNone);
+
+ // Allocate a SeqOneByteString with the given length.
+ TNode<String> AllocateSeqOneByteString(uint32_t length,
+ AllocationFlags flags = kNone);
+ using TorqueGeneratedExportedMacrosAssembler::AllocateSeqOneByteString;
+
+ // Allocate a SeqTwoByteString with the given length.
+ TNode<String> AllocateSeqTwoByteString(uint32_t length,
+ AllocationFlags flags = kNone);
+ using TorqueGeneratedExportedMacrosAssembler::AllocateSeqTwoByteString;
+
+ // Allocate a SlicedOneByteString with the given length, parent and offset.
+ // |length| and |offset| are expected to be tagged.
+
+ TNode<String> AllocateSlicedOneByteString(TNode<Uint32T> length,
+ TNode<String> parent,
+ TNode<Smi> offset);
+ // Allocate a SlicedTwoByteString with the given length, parent and offset.
+ // |length| and |offset| are expected to be tagged.
+ TNode<String> AllocateSlicedTwoByteString(TNode<Uint32T> length,
+ TNode<String> parent,
+ TNode<Smi> offset);
+
+ TNode<NameDictionary> AllocateNameDictionary(int at_least_space_for);
+ TNode<NameDictionary> AllocateNameDictionary(
+ TNode<IntPtrT> at_least_space_for, AllocationFlags = kNone);
+ TNode<NameDictionary> AllocateNameDictionaryWithCapacity(
+ TNode<IntPtrT> capacity, AllocationFlags = kNone);
+ TNode<NameDictionary> CopyNameDictionary(TNode<NameDictionary> dictionary,
+ Label* large_object_fallback);
+
+ template <typename CollectionType>
+ TNode<CollectionType> AllocateOrderedHashTable();
+
+ TNode<JSObject> AllocateJSObjectFromMap(
+ TNode<Map> map,
+ base::Optional<TNode<HeapObject>> properties = base::nullopt,
+ base::Optional<TNode<FixedArray>> elements = base::nullopt,
+ AllocationFlags flags = kNone,
+ SlackTrackingMode slack_tracking_mode = kNoSlackTracking);
+
+ void InitializeJSObjectFromMap(
+ TNode<HeapObject> object, TNode<Map> map, TNode<IntPtrT> instance_size,
+ base::Optional<TNode<HeapObject>> properties = base::nullopt,
+ base::Optional<TNode<FixedArray>> elements = base::nullopt,
+ SlackTrackingMode slack_tracking_mode = kNoSlackTracking);
+
+ void InitializeJSObjectBodyWithSlackTracking(
+ TNode<HeapObject> object, TNode<Map> map,
+ SloppyTNode<IntPtrT> instance_size);
+ void InitializeJSObjectBodyNoSlackTracking(
+ TNode<HeapObject> object, TNode<Map> map,
+ SloppyTNode<IntPtrT> instance_size,
+ int start_offset = JSObject::kHeaderSize);
+
+ TNode<BoolT> IsValidFastJSArrayCapacity(TNode<IntPtrT> capacity);
+
+ //
+ // Allocate and return a JSArray with initialized header fields and its
+ // uninitialized elements.
+ std::pair<TNode<JSArray>, TNode<FixedArrayBase>>
+ AllocateUninitializedJSArrayWithElements(
+ ElementsKind kind, TNode<Map> array_map, TNode<Smi> length,
+ base::Optional<TNode<AllocationSite>> allocation_site,
+ TNode<IntPtrT> capacity, AllocationFlags allocation_flags = kNone,
+ int array_header_size = JSArray::kHeaderSize);
+
+ // Allocate a JSArray and fill elements with the hole.
+ TNode<JSArray> AllocateJSArray(
+ ElementsKind kind, TNode<Map> array_map, TNode<IntPtrT> capacity,
+ TNode<Smi> length, base::Optional<TNode<AllocationSite>> allocation_site,
+ AllocationFlags allocation_flags = kNone);
+ TNode<JSArray> AllocateJSArray(
+ ElementsKind kind, TNode<Map> array_map, TNode<Smi> capacity,
+ TNode<Smi> length, base::Optional<TNode<AllocationSite>> allocation_site,
+ AllocationFlags allocation_flags = kNone) {
+ return AllocateJSArray(kind, array_map, SmiUntag(capacity), length,
+ allocation_site, allocation_flags);
+ }
+ TNode<JSArray> AllocateJSArray(ElementsKind kind, TNode<Map> array_map,
+ TNode<Smi> capacity, TNode<Smi> length,
+ AllocationFlags allocation_flags = kNone) {
+ return AllocateJSArray(kind, array_map, SmiUntag(capacity), length,
+ base::nullopt, allocation_flags);
+ }
+ TNode<JSArray> AllocateJSArray(ElementsKind kind, TNode<Map> array_map,
+ TNode<IntPtrT> capacity, TNode<Smi> length,
+ AllocationFlags allocation_flags = kNone) {
+ return AllocateJSArray(kind, array_map, capacity, length, base::nullopt,
+ allocation_flags);
+ }
+
+ // Allocate a JSArray and initialize the header fields.
+ TNode<JSArray> AllocateJSArray(
+ TNode<Map> array_map, TNode<FixedArrayBase> elements, TNode<Smi> length,
+ base::Optional<TNode<AllocationSite>> allocation_site = base::nullopt,
+ int array_header_size = JSArray::kHeaderSize);
+
+ enum class HoleConversionMode { kDontConvert, kConvertToUndefined };
+ // Clone a fast JSArray |array| into a new fast JSArray.
+ // |convert_holes| tells the function to convert holes into undefined or not.
+ // If |convert_holes| is set to kConvertToUndefined, but the function did not
+ // find any hole in |array|, the resulting array will have the same elements
+ // kind as |array|. If the function did find a hole, it will convert holes in
+ // |array| to undefined in the resulting array, who will now have
+ // PACKED_ELEMENTS kind.
+ // If |convert_holes| is set kDontConvert, holes are also copied to the
+ // resulting array, who will have the same elements kind as |array|. The
+ // function generates significantly less code in this case.
+ TNode<JSArray> CloneFastJSArray(
+ TNode<Context> context, TNode<JSArray> array,
+ base::Optional<TNode<AllocationSite>> allocation_site = base::nullopt,
+ HoleConversionMode convert_holes = HoleConversionMode::kDontConvert);
+
+ TNode<JSArray> ExtractFastJSArray(TNode<Context> context,
+ TNode<JSArray> array, TNode<BInt> begin,
+ TNode<BInt> count);
+
+ template <typename TIndex>
+ TNode<FixedArrayBase> AllocateFixedArray(
+ ElementsKind kind, TNode<TIndex> capacity, AllocationFlags flags = kNone,
+ base::Optional<TNode<Map>> fixed_array_map = base::nullopt);
+
+ TNode<NativeContext> GetCreationContext(TNode<JSReceiver> receiver,
+ Label* if_bailout);
+ TNode<Object> GetConstructor(TNode<Map> map);
+
+ TNode<Map> GetInstanceTypeMap(InstanceType instance_type);
+
+ TNode<FixedArray> AllocateUninitializedFixedArray(intptr_t capacity) {
+ return UncheckedCast<FixedArray>(AllocateFixedArray(
+ PACKED_ELEMENTS, IntPtrConstant(capacity), AllocationFlag::kNone));
+ }
+
+ TNode<FixedArray> AllocateZeroedFixedArray(TNode<IntPtrT> capacity) {
+ TNode<FixedArray> result = UncheckedCast<FixedArray>(
+ AllocateFixedArray(PACKED_ELEMENTS, capacity,
+ AllocationFlag::kAllowLargeObjectAllocation));
+ FillFixedArrayWithSmiZero(result, capacity);
+ return result;
+ }
+
+ TNode<FixedDoubleArray> AllocateZeroedFixedDoubleArray(
+ TNode<IntPtrT> capacity) {
+ TNode<FixedDoubleArray> result = UncheckedCast<FixedDoubleArray>(
+ AllocateFixedArray(PACKED_DOUBLE_ELEMENTS, capacity,
+ AllocationFlag::kAllowLargeObjectAllocation));
+ FillFixedDoubleArrayWithZero(result, capacity);
+ return result;
+ }
+
+ TNode<FixedArray> AllocateFixedArrayWithHoles(TNode<IntPtrT> capacity,
+ AllocationFlags flags) {
+ TNode<FixedArray> result = UncheckedCast<FixedArray>(
+ AllocateFixedArray(PACKED_ELEMENTS, capacity, flags));
+ FillFixedArrayWithValue(PACKED_ELEMENTS, result, IntPtrConstant(0),
+ capacity, RootIndex::kTheHoleValue);
+ return result;
+ }
+
+ TNode<FixedDoubleArray> AllocateFixedDoubleArrayWithHoles(
+ TNode<IntPtrT> capacity, AllocationFlags flags) {
+ TNode<FixedDoubleArray> result = UncheckedCast<FixedDoubleArray>(
+ AllocateFixedArray(PACKED_DOUBLE_ELEMENTS, capacity, flags));
+ FillFixedArrayWithValue(PACKED_DOUBLE_ELEMENTS, result, IntPtrConstant(0),
+ capacity, RootIndex::kTheHoleValue);
+ return result;
+ }
+
+ TNode<PropertyArray> AllocatePropertyArray(TNode<IntPtrT> capacity);
+
+ // TODO(v8:9722): Return type should be JSIteratorResult
+ TNode<JSObject> AllocateJSIteratorResult(TNode<Context> context,
+ SloppyTNode<Object> value,
+ SloppyTNode<Oddball> done);
+
+ // TODO(v8:9722): Return type should be JSIteratorResult
+ TNode<JSObject> AllocateJSIteratorResultForEntry(TNode<Context> context,
+ TNode<Object> key,
+ SloppyTNode<Object> value);
+
+ TNode<JSReceiver> ArraySpeciesCreate(TNode<Context> context,
+ TNode<Object> originalArray,
+ TNode<Number> len);
+
+ template <typename TIndex>
+ void FillFixedArrayWithValue(ElementsKind kind, TNode<FixedArrayBase> array,
+ TNode<TIndex> from_index, TNode<TIndex> to_index,
+ RootIndex value_root_index);
+
+ // Uses memset to effectively initialize the given FixedArray with zeroes.
+ void FillFixedArrayWithSmiZero(TNode<FixedArray> array,
+ TNode<IntPtrT> length);
+ void FillFixedDoubleArrayWithZero(TNode<FixedDoubleArray> array,
+ TNode<IntPtrT> length);
+
+ void FillPropertyArrayWithUndefined(TNode<PropertyArray> array,
+ TNode<IntPtrT> from_index,
+ TNode<IntPtrT> to_index);
+
+ enum class DestroySource { kNo, kYes };
+
+ // Increment the call count for a CALL_IC or construct call.
+ // The call count is located at feedback_vector[slot_id + 1].
+ void IncrementCallCount(TNode<FeedbackVector> feedback_vector,
+ TNode<UintPtrT> slot_id);
+
+ // Specify DestroySource::kYes if {from_array} is being supplanted by
+ // {to_array}. This offers a slight performance benefit by simply copying the
+ // array word by word. The source may be destroyed at the end of this macro.
+ //
+ // Otherwise, specify DestroySource::kNo for operations where an Object is
+ // being cloned, to ensure that mutable HeapNumbers are unique between the
+ // source and cloned object.
+ void CopyPropertyArrayValues(TNode<HeapObject> from_array,
+ TNode<PropertyArray> to_array,
+ TNode<IntPtrT> length,
+ WriteBarrierMode barrier_mode,
+ DestroySource destroy_source);
+
+ // Copies all elements from |from_array| of |length| size to
+ // |to_array| of the same size respecting the elements kind.
+ template <typename TIndex>
+ void CopyFixedArrayElements(
+ ElementsKind kind, TNode<FixedArrayBase> from_array,
+ TNode<FixedArrayBase> to_array, TNode<TIndex> length,
+ WriteBarrierMode barrier_mode = UPDATE_WRITE_BARRIER) {
+ CopyFixedArrayElements(kind, from_array, kind, to_array,
+ IntPtrOrSmiConstant<TIndex>(0), length, length,
+ barrier_mode);
+ }
+
+ // Copies |element_count| elements from |from_array| starting from element
+ // zero to |to_array| of |capacity| size respecting both array's elements
+ // kinds.
+ template <typename TIndex>
+ void CopyFixedArrayElements(
+ ElementsKind from_kind, TNode<FixedArrayBase> from_array,
+ ElementsKind to_kind, TNode<FixedArrayBase> to_array,
+ TNode<TIndex> element_count, TNode<TIndex> capacity,
+ WriteBarrierMode barrier_mode = UPDATE_WRITE_BARRIER) {
+ CopyFixedArrayElements(from_kind, from_array, to_kind, to_array,
+ IntPtrOrSmiConstant<TIndex>(0), element_count,
+ capacity, barrier_mode);
+ }
+
+ // Copies |element_count| elements from |from_array| starting from element
+ // |first_element| to |to_array| of |capacity| size respecting both array's
+ // elements kinds.
+ // |convert_holes| tells the function whether to convert holes to undefined.
+ // |var_holes_converted| can be used to signify that the conversion happened
+ // (i.e. that there were holes). If |convert_holes_to_undefined| is
+ // HoleConversionMode::kConvertToUndefined, then it must not be the case that
+ // IsDoubleElementsKind(to_kind).
+ template <typename TIndex>
+ void CopyFixedArrayElements(
+ ElementsKind from_kind, TNode<FixedArrayBase> from_array,
+ ElementsKind to_kind, TNode<FixedArrayBase> to_array,
+ TNode<TIndex> first_element, TNode<TIndex> element_count,
+ TNode<TIndex> capacity,
+ WriteBarrierMode barrier_mode = UPDATE_WRITE_BARRIER,
+ HoleConversionMode convert_holes = HoleConversionMode::kDontConvert,
+ TVariable<BoolT>* var_holes_converted = nullptr);
+
+ void JumpIfPointersFromHereAreInteresting(TNode<Object> object,
+ Label* interesting);
+
+ // Efficiently copy elements within a single array. The regions
+ // [src_index, src_index + length) and [dst_index, dst_index + length)
+ // can be overlapping.
+ void MoveElements(ElementsKind kind, TNode<FixedArrayBase> elements,
+ TNode<IntPtrT> dst_index, TNode<IntPtrT> src_index,
+ TNode<IntPtrT> length);
+
+ // Efficiently copy elements from one array to another. The ElementsKind
+ // needs to be the same. Copy from src_elements at
+ // [src_index, src_index + length) to dst_elements at
+ // [dst_index, dst_index + length).
+ // The function decides whether it can use memcpy. In case it cannot,
+ // |write_barrier| can help it to skip write barrier. SKIP_WRITE_BARRIER is
+ // only safe when copying to new space, or when copying to old space and the
+ // array does not contain object pointers.
+ void CopyElements(ElementsKind kind, TNode<FixedArrayBase> dst_elements,
+ TNode<IntPtrT> dst_index,
+ TNode<FixedArrayBase> src_elements,
+ TNode<IntPtrT> src_index, TNode<IntPtrT> length,
+ WriteBarrierMode write_barrier = UPDATE_WRITE_BARRIER);
+
+ TNode<FixedArray> HeapObjectToFixedArray(TNode<HeapObject> base,
+ Label* cast_fail);
+
+ TNode<FixedDoubleArray> HeapObjectToFixedDoubleArray(TNode<HeapObject> base,
+ Label* cast_fail) {
+ GotoIf(TaggedNotEqual(LoadMap(base), FixedDoubleArrayMapConstant()),
+ cast_fail);
+ return UncheckedCast<FixedDoubleArray>(base);
+ }
+
+ TNode<Int32T> ConvertElementsKindToInt(TNode<Int32T> elements_kind) {
+ return UncheckedCast<Int32T>(elements_kind);
+ }
+
+ template <typename T>
+ bool ClassHasMapConstant() {
+ return false;
+ }
+
+ template <typename T>
+ TNode<Map> GetClassMapConstant() {
+ UNREACHABLE();
+ return TNode<Map>();
+ }
+
+ enum class ExtractFixedArrayFlag {
+ kFixedArrays = 1,
+ kFixedDoubleArrays = 2,
+ kDontCopyCOW = 4,
+ kNewSpaceAllocationOnly = 8,
+ kAllFixedArrays = kFixedArrays | kFixedDoubleArrays,
+ kAllFixedArraysDontCopyCOW = kAllFixedArrays | kDontCopyCOW
+ };
+
+ using ExtractFixedArrayFlags = base::Flags<ExtractFixedArrayFlag>;
+
+ // Copy a portion of an existing FixedArray or FixedDoubleArray into a new
+ // array, including special appropriate handling for empty arrays and COW
+ // arrays. The result array will be of the same type as the original array.
+ //
+ // * |source| is either a FixedArray or FixedDoubleArray from which to copy
+ // elements.
+ // * |first| is the starting element index to copy from, if nullptr is passed
+ // then index zero is used by default.
+ // * |count| is the number of elements to copy out of the source array
+ // starting from and including the element indexed by |start|. If |count| is
+ // nullptr, then all of the elements from |start| to the end of |source| are
+ // copied.
+ // * |capacity| determines the size of the allocated result array, with
+ // |capacity| >= |count|. If |capacity| is nullptr, then |count| is used as
+ // the destination array's capacity.
+ // * |extract_flags| determines whether FixedArrays, FixedDoubleArrays or both
+ // are detected and copied. Although it's always correct to pass
+ // kAllFixedArrays, the generated code is more compact and efficient if the
+ // caller can specify whether only FixedArrays or FixedDoubleArrays will be
+ // passed as the |source| parameter.
+ // * |parameter_mode| determines the parameter mode of |first|, |count| and
+ // |capacity|.
+ // * If |var_holes_converted| is given, any holes will be converted to
+ // undefined and the variable will be set according to whether or not there
+ // were any hole.
+ // * If |source_elements_kind| is given, the function will try to use the
+ // runtime elements kind of source to make copy faster. More specifically, it
+ // can skip write barriers.
+ template <typename TIndex>
+ TNode<FixedArrayBase> ExtractFixedArray(
+ TNode<FixedArrayBase> source, base::Optional<TNode<TIndex>> first,
+ base::Optional<TNode<TIndex>> count = base::nullopt,
+ base::Optional<TNode<TIndex>> capacity = base::nullopt,
+ ExtractFixedArrayFlags extract_flags =
+ ExtractFixedArrayFlag::kAllFixedArrays,
+ TVariable<BoolT>* var_holes_converted = nullptr,
+ base::Optional<TNode<Int32T>> source_elements_kind = base::nullopt);
+
+ // Copy a portion of an existing FixedArray or FixedDoubleArray into a new
+ // FixedArray, including special appropriate handling for COW arrays.
+ // * |source| is either a FixedArray or FixedDoubleArray from which to copy
+ // elements. |source| is assumed to be non-empty.
+ // * |first| is the starting element index to copy from.
+ // * |count| is the number of elements to copy out of the source array
+ // starting from and including the element indexed by |start|.
+ // * |capacity| determines the size of the allocated result array, with
+ // |capacity| >= |count|.
+ // * |source_map| is the map of the |source|.
+ // * |from_kind| is the elements kind that is consistent with |source| being
+ // a FixedArray or FixedDoubleArray. This function only cares about double vs.
+ // non-double, so as to distinguish FixedDoubleArray vs. FixedArray. It does
+ // not care about holeyness. For example, when |source| is a FixedArray,
+ // PACKED/HOLEY_ELEMENTS can be used, but not PACKED_DOUBLE_ELEMENTS.
+ // * |allocation_flags| and |extract_flags| influence how the target
+ // FixedArray is allocated.
+ // * |convert_holes| is used to signify that the target array should use
+ // undefined in places of holes.
+ // * If |convert_holes| is true and |var_holes_converted| not nullptr, then
+ // |var_holes_converted| is used to signal whether any holes were found and
+ // converted. The caller should use this information to decide which map is
+ // compatible with the result array. For example, if the input was of
+ // HOLEY_SMI_ELEMENTS kind, and a conversion took place, the result will be
+ // compatible only with HOLEY_ELEMENTS and PACKED_ELEMENTS.
+ template <typename TIndex>
+ TNode<FixedArray> ExtractToFixedArray(
+ TNode<FixedArrayBase> source, TNode<TIndex> first, TNode<TIndex> count,
+ TNode<TIndex> capacity, TNode<Map> source_map, ElementsKind from_kind,
+ AllocationFlags allocation_flags, ExtractFixedArrayFlags extract_flags,
+ HoleConversionMode convert_holes,
+ TVariable<BoolT>* var_holes_converted = nullptr,
+ base::Optional<TNode<Int32T>> source_runtime_kind = base::nullopt);
+
+ // Attempt to copy a FixedDoubleArray to another FixedDoubleArray. In the case
+ // where the source array has a hole, produce a FixedArray instead where holes
+ // are replaced with undefined.
+ // * |source| is a FixedDoubleArray from which to copy elements.
+ // * |first| is the starting element index to copy from.
+ // * |count| is the number of elements to copy out of the source array
+ // starting from and including the element indexed by |start|.
+ // * |capacity| determines the size of the allocated result array, with
+ // |capacity| >= |count|.
+ // * |source_map| is the map of |source|. It will be used as the map of the
+ // target array if the target can stay a FixedDoubleArray. Otherwise if the
+ // target array needs to be a FixedArray, the FixedArrayMap will be used.
+ // * |var_holes_converted| is used to signal whether a FixedAray
+ // is produced or not.
+ // * |allocation_flags| and |extract_flags| influence how the target array is
+ // allocated.
+ template <typename TIndex>
+ TNode<FixedArrayBase> ExtractFixedDoubleArrayFillingHoles(
+ TNode<FixedArrayBase> source, TNode<TIndex> first, TNode<TIndex> count,
+ TNode<TIndex> capacity, TNode<Map> source_map,
+ TVariable<BoolT>* var_holes_converted, AllocationFlags allocation_flags,
+ ExtractFixedArrayFlags extract_flags);
+
+ // Copy the entire contents of a FixedArray or FixedDoubleArray to a new
+ // array, including special appropriate handling for empty arrays and COW
+ // arrays.
+ //
+ // * |source| is either a FixedArray or FixedDoubleArray from which to copy
+ // elements.
+ // * |extract_flags| determines whether FixedArrays, FixedDoubleArrays or both
+ // are detected and copied. Although it's always correct to pass
+ // kAllFixedArrays, the generated code is more compact and efficient if the
+ // caller can specify whether only FixedArrays or FixedDoubleArrays will be
+ // passed as the |source| parameter.
+ TNode<FixedArrayBase> CloneFixedArray(
+ TNode<FixedArrayBase> source,
+ ExtractFixedArrayFlags flags =
+ ExtractFixedArrayFlag::kAllFixedArraysDontCopyCOW);
+
+ // Loads an element from |array| of |from_kind| elements by given |offset|
+ // (NOTE: not index!), does a hole check if |if_hole| is provided and
+ // converts the value so that it becomes ready for storing to array of
+ // |to_kind| elements.
+ Node* LoadElementAndPrepareForStore(TNode<FixedArrayBase> array,
+ TNode<IntPtrT> offset,
+ ElementsKind from_kind,
+ ElementsKind to_kind, Label* if_hole);
+
+ template <typename TIndex>
+ TNode<TIndex> CalculateNewElementsCapacity(TNode<TIndex> old_capacity);
+
+ // Tries to grow the |elements| array of given |object| to store the |key|
+ // or bails out if the growing gap is too big. Returns new elements.
+ TNode<FixedArrayBase> TryGrowElementsCapacity(TNode<HeapObject> object,
+ TNode<FixedArrayBase> elements,
+ ElementsKind kind,
+ TNode<Smi> key, Label* bailout);
+
+ // Tries to grow the |capacity|-length |elements| array of given |object|
+ // to store the |key| or bails out if the growing gap is too big. Returns
+ // new elements.
+ template <typename TIndex>
+ TNode<FixedArrayBase> TryGrowElementsCapacity(TNode<HeapObject> object,
+ TNode<FixedArrayBase> elements,
+ ElementsKind kind,
+ TNode<TIndex> key,
+ TNode<TIndex> capacity,
+ Label* bailout);
+
+ // Grows elements capacity of given object. Returns new elements.
+ template <typename TIndex>
+ TNode<FixedArrayBase> GrowElementsCapacity(
+ TNode<HeapObject> object, TNode<FixedArrayBase> elements,
+ ElementsKind from_kind, ElementsKind to_kind, TNode<TIndex> capacity,
+ TNode<TIndex> new_capacity, Label* bailout);
+
+ // Given a need to grow by |growth|, allocate an appropriate new capacity
+ // if necessary, and return a new elements FixedArray object. Label |bailout|
+ // is followed for allocation failure.
+ void PossiblyGrowElementsCapacity(ElementsKind kind, TNode<HeapObject> array,
+ TNode<BInt> length,
+ TVariable<FixedArrayBase>* var_elements,
+ TNode<BInt> growth, Label* bailout);
+
+ // Allocation site manipulation
+ void InitializeAllocationMemento(TNode<HeapObject> base,
+ TNode<IntPtrT> base_allocation_size,
+ TNode<AllocationSite> allocation_site);
+
+ TNode<Float64T> TryTaggedToFloat64(TNode<Object> value,
+ Label* if_valueisnotnumber);
+ TNode<Float64T> TruncateTaggedToFloat64(TNode<Context> context,
+ SloppyTNode<Object> value);
+ TNode<Word32T> TruncateTaggedToWord32(TNode<Context> context,
+ SloppyTNode<Object> value);
+ void TaggedToWord32OrBigInt(TNode<Context> context, TNode<Object> value,
+ Label* if_number, TVariable<Word32T>* var_word32,
+ Label* if_bigint,
+ TVariable<BigInt>* var_maybe_bigint);
+ void TaggedToWord32OrBigIntWithFeedback(TNode<Context> context,
+ TNode<Object> value, Label* if_number,
+ TVariable<Word32T>* var_word32,
+ Label* if_bigint,
+ TVariable<BigInt>* var_maybe_bigint,
+ TVariable<Smi>* var_feedback);
+
+ TNode<Int32T> TruncateNumberToWord32(TNode<Number> value);
+ // Truncate the floating point value of a HeapNumber to an Int32.
+ TNode<Int32T> TruncateHeapNumberValueToWord32(TNode<HeapNumber> object);
+
+ // Conversions.
+ void TryHeapNumberToSmi(TNode<HeapNumber> number, TVariable<Smi>* output,
+ Label* if_smi);
+ void TryFloat32ToSmi(TNode<Float32T> number, TVariable<Smi>* output,
+ Label* if_smi);
+ void TryFloat64ToSmi(TNode<Float64T> number, TVariable<Smi>* output,
+ Label* if_smi);
+ TNode<Number> ChangeFloat32ToTagged(TNode<Float32T> value);
+ TNode<Number> ChangeFloat64ToTagged(SloppyTNode<Float64T> value);
+ TNode<Number> ChangeInt32ToTagged(SloppyTNode<Int32T> value);
+ TNode<Number> ChangeUint32ToTagged(SloppyTNode<Uint32T> value);
+ TNode<Number> ChangeUintPtrToTagged(TNode<UintPtrT> value);
+ TNode<Uint32T> ChangeNumberToUint32(TNode<Number> value);
+ TNode<Float64T> ChangeNumberToFloat64(TNode<Number> value);
+
+ TNode<Int32T> ChangeTaggedNonSmiToInt32(TNode<Context> context,
+ TNode<HeapObject> input);
+ TNode<Float64T> ChangeTaggedToFloat64(TNode<Context> context,
+ TNode<Object> input);
+
+ void TaggedToNumeric(TNode<Context> context, TNode<Object> value,
+ TVariable<Numeric>* var_numeric);
+ void TaggedToNumericWithFeedback(TNode<Context> context, TNode<Object> value,
+ TVariable<Numeric>* var_numeric,
+ TVariable<Smi>* var_feedback);
+
+ TNode<WordT> TimesSystemPointerSize(SloppyTNode<WordT> value);
+ TNode<IntPtrT> TimesSystemPointerSize(TNode<IntPtrT> value) {
+ return Signed(TimesSystemPointerSize(implicit_cast<TNode<WordT>>(value)));
+ }
+ TNode<UintPtrT> TimesSystemPointerSize(TNode<UintPtrT> value) {
+ return Unsigned(TimesSystemPointerSize(implicit_cast<TNode<WordT>>(value)));
+ }
+
+ TNode<WordT> TimesTaggedSize(SloppyTNode<WordT> value);
+ TNode<IntPtrT> TimesTaggedSize(TNode<IntPtrT> value) {
+ return Signed(TimesTaggedSize(implicit_cast<TNode<WordT>>(value)));
+ }
+ TNode<UintPtrT> TimesTaggedSize(TNode<UintPtrT> value) {
+ return Unsigned(TimesTaggedSize(implicit_cast<TNode<WordT>>(value)));
+ }
+
+ TNode<WordT> TimesDoubleSize(SloppyTNode<WordT> value);
+ TNode<UintPtrT> TimesDoubleSize(TNode<UintPtrT> value) {
+ return Unsigned(TimesDoubleSize(implicit_cast<TNode<WordT>>(value)));
+ }
+ TNode<IntPtrT> TimesDoubleSize(TNode<IntPtrT> value) {
+ return Signed(TimesDoubleSize(implicit_cast<TNode<WordT>>(value)));
+ }
+
+ // Type conversions.
+ // Throws a TypeError for {method_name} if {value} is not coercible to Object,
+ // or returns the {value} converted to a String otherwise.
+ TNode<String> ToThisString(TNode<Context> context, TNode<Object> value,
+ TNode<String> method_name);
+ TNode<String> ToThisString(TNode<Context> context, TNode<Object> value,
+ char const* method_name) {
+ return ToThisString(context, value, StringConstant(method_name));
+ }
+
+ // Throws a TypeError for {method_name} if {value} is neither of the given
+ // {primitive_type} nor a JSPrimitiveWrapper wrapping a value of
+ // {primitive_type}, or returns the {value} (or wrapped value) otherwise.
+ TNode<Object> ToThisValue(TNode<Context> context, TNode<Object> value,
+ PrimitiveType primitive_type,
+ char const* method_name);
+
+ // Throws a TypeError for {method_name} if {value} is not of the given
+ // instance type.
+ void ThrowIfNotInstanceType(TNode<Context> context, TNode<Object> value,
+ InstanceType instance_type,
+ char const* method_name);
+ // Throws a TypeError for {method_name} if {value} is not a JSReceiver.
+ void ThrowIfNotJSReceiver(TNode<Context> context, TNode<Object> value,
+ MessageTemplate msg_template,
+ const char* method_name);
+ void ThrowIfNotCallable(TNode<Context> context, TNode<Object> value,
+ const char* method_name);
+
+ void ThrowRangeError(TNode<Context> context, MessageTemplate message,
+ base::Optional<TNode<Object>> arg0 = base::nullopt,
+ base::Optional<TNode<Object>> arg1 = base::nullopt,
+ base::Optional<TNode<Object>> arg2 = base::nullopt);
+ void ThrowTypeError(TNode<Context> context, MessageTemplate message,
+ char const* arg0 = nullptr, char const* arg1 = nullptr);
+ void ThrowTypeError(TNode<Context> context, MessageTemplate message,
+ base::Optional<TNode<Object>> arg0,
+ base::Optional<TNode<Object>> arg1 = base::nullopt,
+ base::Optional<TNode<Object>> arg2 = base::nullopt);
+
+ // Type checks.
+ // Check whether the map is for an object with special properties, such as a
+ // JSProxy or an object with interceptors.
+ TNode<BoolT> InstanceTypeEqual(SloppyTNode<Int32T> instance_type, int type);
+ TNode<BoolT> IsNoElementsProtectorCellInvalid();
+ TNode<BoolT> IsArrayIteratorProtectorCellInvalid();
+ TNode<BoolT> IsBigIntInstanceType(SloppyTNode<Int32T> instance_type);
+ TNode<BoolT> IsBigInt(TNode<HeapObject> object);
+ TNode<BoolT> IsBoolean(TNode<HeapObject> object);
+ TNode<BoolT> IsCallableMap(TNode<Map> map);
+ TNode<BoolT> IsCallable(TNode<HeapObject> object);
+ TNode<BoolT> TaggedIsCallable(TNode<Object> object);
+ TNode<BoolT> IsConsStringInstanceType(SloppyTNode<Int32T> instance_type);
+ TNode<BoolT> IsConstructorMap(TNode<Map> map);
+ TNode<BoolT> IsConstructor(TNode<HeapObject> object);
+ TNode<BoolT> IsDeprecatedMap(TNode<Map> map);
+ TNode<BoolT> IsNameDictionary(TNode<HeapObject> object);
+ TNode<BoolT> IsGlobalDictionary(TNode<HeapObject> object);
+ TNode<BoolT> IsExtensibleMap(TNode<Map> map);
+ TNode<BoolT> IsExtensibleNonPrototypeMap(TNode<Map> map);
+ TNode<BoolT> IsExternalStringInstanceType(SloppyTNode<Int32T> instance_type);
+ TNode<BoolT> IsFixedArray(TNode<HeapObject> object);
+ TNode<BoolT> IsFixedArraySubclass(TNode<HeapObject> object);
+ TNode<BoolT> IsFixedArrayWithKind(TNode<HeapObject> object,
+ ElementsKind kind);
+ TNode<BoolT> IsFixedArrayWithKindOrEmpty(TNode<FixedArrayBase> object,
+ ElementsKind kind);
+ TNode<BoolT> IsFunctionWithPrototypeSlotMap(TNode<Map> map);
+ TNode<BoolT> IsHashTable(TNode<HeapObject> object);
+ TNode<BoolT> IsEphemeronHashTable(TNode<HeapObject> object);
+ TNode<BoolT> IsHeapNumberInstanceType(SloppyTNode<Int32T> instance_type);
+ TNode<BoolT> IsOddball(TNode<HeapObject> object);
+ TNode<BoolT> IsOddballInstanceType(SloppyTNode<Int32T> instance_type);
+ TNode<BoolT> IsIndirectStringInstanceType(SloppyTNode<Int32T> instance_type);
+ TNode<BoolT> IsJSArrayBuffer(TNode<HeapObject> object);
+ TNode<BoolT> IsJSDataView(TNode<HeapObject> object);
+ TNode<BoolT> IsJSArrayInstanceType(SloppyTNode<Int32T> instance_type);
+ TNode<BoolT> IsJSArrayMap(TNode<Map> map);
+ TNode<BoolT> IsJSArray(TNode<HeapObject> object);
+ TNode<BoolT> IsJSArrayIterator(TNode<HeapObject> object);
+ TNode<BoolT> IsJSAsyncGeneratorObject(TNode<HeapObject> object);
+ TNode<BoolT> IsJSFunctionInstanceType(SloppyTNode<Int32T> instance_type);
+ TNode<BoolT> IsJSFunctionMap(TNode<Map> map);
+ TNode<BoolT> IsJSFunction(TNode<HeapObject> object);
+ TNode<BoolT> IsJSBoundFunction(TNode<HeapObject> object);
+ TNode<BoolT> IsJSGeneratorObject(TNode<HeapObject> object);
+ TNode<BoolT> IsJSGlobalProxyInstanceType(SloppyTNode<Int32T> instance_type);
+ TNode<BoolT> IsJSGlobalProxyMap(TNode<Map> map);
+ TNode<BoolT> IsJSGlobalProxy(TNode<HeapObject> object);
+ TNode<BoolT> IsJSObjectInstanceType(SloppyTNode<Int32T> instance_type);
+ TNode<BoolT> IsJSObjectMap(TNode<Map> map);
+ TNode<BoolT> IsJSObject(TNode<HeapObject> object);
+ TNode<BoolT> IsJSFinalizationRegistryMap(TNode<Map> map);
+ TNode<BoolT> IsJSFinalizationRegistry(TNode<HeapObject> object);
+ TNode<BoolT> IsJSPromiseMap(TNode<Map> map);
+ TNode<BoolT> IsJSPromise(TNode<HeapObject> object);
+ TNode<BoolT> IsJSProxy(TNode<HeapObject> object);
+ TNode<BoolT> IsJSStringIterator(TNode<HeapObject> object);
+ TNode<BoolT> IsJSRegExpStringIterator(TNode<HeapObject> object);
+ TNode<BoolT> IsJSReceiverInstanceType(SloppyTNode<Int32T> instance_type);
+ TNode<BoolT> IsJSReceiverMap(TNode<Map> map);
+ TNode<BoolT> IsJSReceiver(TNode<HeapObject> object);
+ TNode<BoolT> IsJSRegExp(TNode<HeapObject> object);
+ TNode<BoolT> IsJSTypedArrayInstanceType(SloppyTNode<Int32T> instance_type);
+ TNode<BoolT> IsJSTypedArrayMap(TNode<Map> map);
+ TNode<BoolT> IsJSTypedArray(TNode<HeapObject> object);
+ TNode<BoolT> IsJSGeneratorMap(TNode<Map> map);
+ TNode<BoolT> IsJSPrimitiveWrapperInstanceType(
+ SloppyTNode<Int32T> instance_type);
+ TNode<BoolT> IsJSPrimitiveWrapperMap(TNode<Map> map);
+ TNode<BoolT> IsJSPrimitiveWrapper(TNode<HeapObject> object);
+ TNode<BoolT> IsMap(TNode<HeapObject> object);
+ TNode<BoolT> IsName(TNode<HeapObject> object);
+ TNode<BoolT> IsNameInstanceType(SloppyTNode<Int32T> instance_type);
+ TNode<BoolT> IsNullOrJSReceiver(TNode<HeapObject> object);
+ TNode<BoolT> IsNullOrUndefined(SloppyTNode<Object> object);
+ TNode<BoolT> IsNumberDictionary(TNode<HeapObject> object);
+ TNode<BoolT> IsOneByteStringInstanceType(TNode<Int32T> instance_type);
+ TNode<BoolT> IsSeqOneByteStringInstanceType(TNode<Int32T> instance_type);
+ TNode<BoolT> IsPrimitiveInstanceType(SloppyTNode<Int32T> instance_type);
+ TNode<BoolT> IsPrivateName(SloppyTNode<Symbol> symbol);
+ TNode<BoolT> IsPropertyArray(TNode<HeapObject> object);
+ TNode<BoolT> IsPropertyCell(TNode<HeapObject> object);
+ TNode<BoolT> IsPromiseReactionJobTask(TNode<HeapObject> object);
+ TNode<BoolT> IsPrototypeInitialArrayPrototype(TNode<Context> context,
+ TNode<Map> map);
+ TNode<BoolT> IsPrototypeTypedArrayPrototype(TNode<Context> context,
+ TNode<Map> map);
+
+ TNode<BoolT> IsFastAliasedArgumentsMap(TNode<Context> context,
+ TNode<Map> map);
+ TNode<BoolT> IsSlowAliasedArgumentsMap(TNode<Context> context,
+ TNode<Map> map);
+ TNode<BoolT> IsSloppyArgumentsMap(TNode<Context> context, TNode<Map> map);
+ TNode<BoolT> IsStrictArgumentsMap(TNode<Context> context, TNode<Map> map);
+
+ TNode<BoolT> IsSequentialStringInstanceType(
+ SloppyTNode<Int32T> instance_type);
+ TNode<BoolT> IsUncachedExternalStringInstanceType(
+ SloppyTNode<Int32T> instance_type);
+ TNode<BoolT> IsSpecialReceiverInstanceType(TNode<Int32T> instance_type);
+ TNode<BoolT> IsCustomElementsReceiverInstanceType(
+ TNode<Int32T> instance_type);
+ TNode<BoolT> IsSpecialReceiverMap(TNode<Map> map);
+ TNode<BoolT> IsStringInstanceType(SloppyTNode<Int32T> instance_type);
+ TNode<BoolT> IsString(TNode<HeapObject> object);
+ TNode<BoolT> IsSeqOneByteString(TNode<HeapObject> object);
+
+ TNode<BoolT> IsSymbolInstanceType(SloppyTNode<Int32T> instance_type);
+ TNode<BoolT> IsInternalizedStringInstanceType(TNode<Int32T> instance_type);
+ TNode<BoolT> IsUniqueName(TNode<HeapObject> object);
+ TNode<BoolT> IsUniqueNameNoIndex(TNode<HeapObject> object);
+ TNode<BoolT> IsUniqueNameNoCachedIndex(TNode<HeapObject> object);
+ TNode<BoolT> IsUndetectableMap(TNode<Map> map);
+ TNode<BoolT> IsNotWeakFixedArraySubclass(TNode<HeapObject> object);
+ TNode<BoolT> IsZeroOrContext(SloppyTNode<Object> object);
+
+ TNode<BoolT> IsPromiseResolveProtectorCellInvalid();
+ TNode<BoolT> IsPromiseThenProtectorCellInvalid();
+ TNode<BoolT> IsArraySpeciesProtectorCellInvalid();
+ TNode<BoolT> IsTypedArraySpeciesProtectorCellInvalid();
+ TNode<BoolT> IsRegExpSpeciesProtectorCellInvalid();
+ TNode<BoolT> IsPromiseSpeciesProtectorCellInvalid();
+
+ TNode<BoolT> IsMockArrayBufferAllocatorFlag() {
+ TNode<Word32T> flag_value = UncheckedCast<Word32T>(Load(
+ MachineType::Uint8(),
+ ExternalConstant(
+ ExternalReference::address_of_mock_arraybuffer_allocator_flag())));
+ return Word32NotEqual(Word32And(flag_value, Int32Constant(0xFF)),
+ Int32Constant(0));
+ }
+
+ // True iff |object| is a Smi or a HeapNumber or a BigInt.
+ TNode<BoolT> IsNumeric(SloppyTNode<Object> object);
+
+ // True iff |number| is either a Smi, or a HeapNumber whose value is not
+ // within Smi range.
+ TNode<BoolT> IsNumberNormalized(TNode<Number> number);
+ TNode<BoolT> IsNumberPositive(TNode<Number> number);
+ TNode<BoolT> IsHeapNumberPositive(TNode<HeapNumber> number);
+
+ // True iff {number} is non-negative and less or equal than 2**53-1.
+ TNode<BoolT> IsNumberNonNegativeSafeInteger(TNode<Number> number);
+
+ // True iff {number} represents an integer value.
+ TNode<BoolT> IsInteger(TNode<Object> number);
+ TNode<BoolT> IsInteger(TNode<HeapNumber> number);
+
+ // True iff abs({number}) <= 2**53 -1
+ TNode<BoolT> IsSafeInteger(TNode<Object> number);
+ TNode<BoolT> IsSafeInteger(TNode<HeapNumber> number);
+
+ // True iff {number} represents a valid uint32t value.
+ TNode<BoolT> IsHeapNumberUint32(TNode<HeapNumber> number);
+
+ // True iff {number} is a positive number and a valid array index in the range
+ // [0, 2^32-1).
+ TNode<BoolT> IsNumberArrayIndex(TNode<Number> number);
+
+ template <typename TIndex>
+ TNode<BoolT> FixedArraySizeDoesntFitInNewSpace(TNode<TIndex> element_count,
+ int base_size);
+
+ TNode<BoolT> IsMetaMap(TNode<HeapObject> o) { return IsMapMap(o); }
+
+ // ElementsKind helpers:
+ TNode<BoolT> ElementsKindEqual(TNode<Int32T> a, TNode<Int32T> b) {
+ return Word32Equal(a, b);
+ }
+ bool ElementsKindEqual(ElementsKind a, ElementsKind b) { return a == b; }
+ TNode<BoolT> IsFastElementsKind(TNode<Int32T> elements_kind);
+ bool IsFastElementsKind(ElementsKind kind) {
+ return v8::internal::IsFastElementsKind(kind);
+ }
+ TNode<BoolT> IsFastOrNonExtensibleOrSealedElementsKind(
+ TNode<Int32T> elements_kind);
+
+ TNode<BoolT> IsDictionaryElementsKind(TNode<Int32T> elements_kind) {
+ return ElementsKindEqual(elements_kind, Int32Constant(DICTIONARY_ELEMENTS));
+ }
+ TNode<BoolT> IsDoubleElementsKind(TNode<Int32T> elements_kind);
+ bool IsDoubleElementsKind(ElementsKind kind) {
+ return v8::internal::IsDoubleElementsKind(kind);
+ }
+ TNode<BoolT> IsFastSmiOrTaggedElementsKind(TNode<Int32T> elements_kind);
+ TNode<BoolT> IsFastSmiElementsKind(SloppyTNode<Int32T> elements_kind);
+ TNode<BoolT> IsHoleyFastElementsKind(TNode<Int32T> elements_kind);
+ TNode<BoolT> IsHoleyFastElementsKindForRead(TNode<Int32T> elements_kind);
+ TNode<BoolT> IsElementsKindGreaterThan(TNode<Int32T> target_kind,
+ ElementsKind reference_kind);
+ TNode<BoolT> IsElementsKindLessThanOrEqual(TNode<Int32T> target_kind,
+ ElementsKind reference_kind);
+ // Check if lower_reference_kind <= target_kind <= higher_reference_kind.
+ TNode<BoolT> IsElementsKindInRange(TNode<Int32T> target_kind,
+ ElementsKind lower_reference_kind,
+ ElementsKind higher_reference_kind) {
+ return IsInRange(target_kind, lower_reference_kind, higher_reference_kind);
+ }
+
+ // String helpers.
+ // Load a character from a String (might flatten a ConsString).
+ TNode<Int32T> StringCharCodeAt(TNode<String> string, TNode<UintPtrT> index);
+ // Return the single character string with only {code}.
+ TNode<String> StringFromSingleCharCode(TNode<Int32T> code);
+
+ // Type conversion helpers.
+ enum class BigIntHandling { kConvertToNumber, kThrow };
+ // Convert a String to a Number.
+ TNode<Number> StringToNumber(TNode<String> input);
+ // Convert a Number to a String.
+ TNode<String> NumberToString(TNode<Number> input);
+ TNode<String> NumberToString(TNode<Number> input, Label* bailout);
+
+ // Convert a Non-Number object to a Number.
+ TNode<Number> NonNumberToNumber(
+ TNode<Context> context, TNode<HeapObject> input,
+ BigIntHandling bigint_handling = BigIntHandling::kThrow);
+ // Convert a Non-Number object to a Numeric.
+ TNode<Numeric> NonNumberToNumeric(TNode<Context> context,
+ TNode<HeapObject> input);
+ // Convert any object to a Number.
+ // Conforms to ES#sec-tonumber if {bigint_handling} == kThrow.
+ // With {bigint_handling} == kConvertToNumber, matches behavior of
+ // tc39.github.io/proposal-bigint/#sec-number-constructor-number-value.
+ TNode<Number> ToNumber(
+ TNode<Context> context, SloppyTNode<Object> input,
+ BigIntHandling bigint_handling = BigIntHandling::kThrow);
+ TNode<Number> ToNumber_Inline(TNode<Context> context,
+ SloppyTNode<Object> input);
+ // Convert any plain primitive to a Number. No need to handle BigInts since
+ // they are not plain primitives.
+ TNode<Number> PlainPrimitiveToNumber(TNode<Object> input);
+
+ // Try to convert an object to a BigInt. Throws on failure (e.g. for Numbers).
+ // https://tc39.github.io/proposal-bigint/#sec-to-bigint
+ TNode<BigInt> ToBigInt(TNode<Context> context, TNode<Object> input);
+
+ // Converts |input| to one of 2^32 integer values in the range 0 through
+ // 2^32-1, inclusive.
+ // ES#sec-touint32
+ TNode<Number> ToUint32(TNode<Context> context, SloppyTNode<Object> input);
+
+ // Convert any object to a String.
+ TNode<String> ToString_Inline(TNode<Context> context,
+ SloppyTNode<Object> input);
+
+ TNode<JSReceiver> ToObject(TNode<Context> context, SloppyTNode<Object> input);
+
+ // Same as ToObject but avoids the Builtin call if |input| is already a
+ // JSReceiver.
+ TNode<JSReceiver> ToObject_Inline(TNode<Context> context,
+ TNode<Object> input);
+
+ // ES6 7.1.15 ToLength, but with inlined fast path.
+ TNode<Number> ToLength_Inline(TNode<Context> context,
+ SloppyTNode<Object> input);
+
+ TNode<Object> OrdinaryToPrimitive(TNode<Context> context, TNode<Object> input,
+ OrdinaryToPrimitiveHint hint);
+
+ // Returns a node that contains a decoded (unsigned!) value of a bit
+ // field |BitField| in |word32|. Returns result as an uint32 node.
+ template <typename BitField>
+ TNode<Uint32T> DecodeWord32(TNode<Word32T> word32) {
+ return DecodeWord32(word32, BitField::kShift, BitField::kMask);
+ }
+
+ // Returns a node that contains a decoded (unsigned!) value of a bit
+ // field |BitField| in |word|. Returns result as a word-size node.
+ template <typename BitField>
+ TNode<UintPtrT> DecodeWord(SloppyTNode<WordT> word) {
+ return DecodeWord(word, BitField::kShift, BitField::kMask);
+ }
+
+ // Returns a node that contains a decoded (unsigned!) value of a bit
+ // field |BitField| in |word32|. Returns result as a word-size node.
+ template <typename BitField>
+ TNode<UintPtrT> DecodeWordFromWord32(TNode<Word32T> word32) {
+ return DecodeWord<BitField>(ChangeUint32ToWord(word32));
+ }
+
+ // Returns a node that contains a decoded (unsigned!) value of a bit
+ // field |BitField| in |word|. Returns result as an uint32 node.
+ template <typename BitField>
+ TNode<Uint32T> DecodeWord32FromWord(SloppyTNode<WordT> word) {
+ return UncheckedCast<Uint32T>(
+ TruncateIntPtrToInt32(Signed(DecodeWord<BitField>(word))));
+ }
+
+ // Decodes an unsigned (!) value from |word32| to an uint32 node.
+ TNode<Uint32T> DecodeWord32(TNode<Word32T> word32, uint32_t shift,
+ uint32_t mask);
+
+ // Decodes an unsigned (!) value from |word| to a word-size node.
+ TNode<UintPtrT> DecodeWord(SloppyTNode<WordT> word, uint32_t shift,
+ uintptr_t mask);
+
+ // Returns a node that contains the updated values of a |BitField|.
+ template <typename BitField>
+ TNode<Word32T> UpdateWord32(TNode<Word32T> word, TNode<Uint32T> value,
+ bool starts_as_zero = false) {
+ return UpdateWord32(word, value, BitField::kShift, BitField::kMask,
+ starts_as_zero);
+ }
+
+ // Returns a node that contains the updated values of a |BitField|.
+ template <typename BitField>
+ TNode<WordT> UpdateWord(TNode<WordT> word, TNode<UintPtrT> value,
+ bool starts_as_zero = false) {
+ return UpdateWord(word, value, BitField::kShift, BitField::kMask,
+ starts_as_zero);
+ }
+
+ // Returns a node that contains the updated values of a |BitField|.
+ template <typename BitField>
+ TNode<Word32T> UpdateWordInWord32(TNode<Word32T> word, TNode<UintPtrT> value,
+ bool starts_as_zero = false) {
+ return UncheckedCast<Uint32T>(
+ TruncateIntPtrToInt32(Signed(UpdateWord<BitField>(
+ ChangeUint32ToWord(word), value, starts_as_zero))));
+ }
+
+ // Returns a node that contains the updated values of a |BitField|.
+ template <typename BitField>
+ TNode<WordT> UpdateWord32InWord(TNode<WordT> word, TNode<Uint32T> value,
+ bool starts_as_zero = false) {
+ return UpdateWord<BitField>(word, ChangeUint32ToWord(value),
+ starts_as_zero);
+ }
+
+ // Returns a node that contains the updated {value} inside {word} starting
+ // at {shift} and fitting in {mask}.
+ TNode<Word32T> UpdateWord32(TNode<Word32T> word, TNode<Uint32T> value,
+ uint32_t shift, uint32_t mask,
+ bool starts_as_zero = false);
+
+ // Returns a node that contains the updated {value} inside {word} starting
+ // at {shift} and fitting in {mask}.
+ TNode<WordT> UpdateWord(TNode<WordT> word, TNode<UintPtrT> value,
+ uint32_t shift, uintptr_t mask,
+ bool starts_as_zero = false);
+
+ // Returns true if any of the |T|'s bits in given |word32| are set.
+ template <typename T>
+ TNode<BoolT> IsSetWord32(TNode<Word32T> word32) {
+ return IsSetWord32(word32, T::kMask);
+ }
+
+ // Returns true if any of the mask's bits in given |word32| are set.
+ TNode<BoolT> IsSetWord32(TNode<Word32T> word32, uint32_t mask) {
+ return Word32NotEqual(Word32And(word32, Int32Constant(mask)),
+ Int32Constant(0));
+ }
+
+ // Returns true if none of the mask's bits in given |word32| are set.
+ TNode<BoolT> IsNotSetWord32(TNode<Word32T> word32, uint32_t mask) {
+ return Word32Equal(Word32And(word32, Int32Constant(mask)),
+ Int32Constant(0));
+ }
+
+ // Returns true if all of the mask's bits in a given |word32| are set.
+ TNode<BoolT> IsAllSetWord32(TNode<Word32T> word32, uint32_t mask) {
+ TNode<Int32T> const_mask = Int32Constant(mask);
+ return Word32Equal(Word32And(word32, const_mask), const_mask);
+ }
+
+ // Returns true if the bit field |BitField| in |word32| is equal to a given.
+ // constant |value|. Avoids a shift compared to using DecodeWord32.
+ template <typename BitField>
+ TNode<BoolT> IsEqualInWord32(TNode<Word32T> word32,
+ typename BitField::FieldType value) {
+ TNode<Word32T> masked_word32 =
+ Word32And(word32, Int32Constant(BitField::kMask));
+ return Word32Equal(masked_word32, Int32Constant(BitField::encode(value)));
+ }
+
+ // Returns true if any of the |T|'s bits in given |word| are set.
+ template <typename T>
+ TNode<BoolT> IsSetWord(SloppyTNode<WordT> word) {
+ return IsSetWord(word, T::kMask);
+ }
+
+ // Returns true if any of the mask's bits in given |word| are set.
+ TNode<BoolT> IsSetWord(SloppyTNode<WordT> word, uint32_t mask) {
+ return WordNotEqual(WordAnd(word, IntPtrConstant(mask)), IntPtrConstant(0));
+ }
+
+ // Returns true if any of the mask's bit are set in the given Smi.
+ // Smi-encoding of the mask is performed implicitly!
+ TNode<BoolT> IsSetSmi(SloppyTNode<Smi> smi, int untagged_mask) {
+ intptr_t mask_word = bit_cast<intptr_t>(Smi::FromInt(untagged_mask));
+ return WordNotEqual(WordAnd(BitcastTaggedToWordForTagAndSmiBits(smi),
+ IntPtrConstant(mask_word)),
+ IntPtrConstant(0));
+ }
+
+ // Returns true if all of the |T|'s bits in given |word32| are clear.
+ template <typename T>
+ TNode<BoolT> IsClearWord32(TNode<Word32T> word32) {
+ return IsClearWord32(word32, T::kMask);
+ }
+
+ // Returns true if all of the mask's bits in given |word32| are clear.
+ TNode<BoolT> IsClearWord32(TNode<Word32T> word32, uint32_t mask) {
+ return Word32Equal(Word32And(word32, Int32Constant(mask)),
+ Int32Constant(0));
+ }
+
+ // Returns true if all of the |T|'s bits in given |word| are clear.
+ template <typename T>
+ TNode<BoolT> IsClearWord(SloppyTNode<WordT> word) {
+ return IsClearWord(word, T::kMask);
+ }
+
+ // Returns true if all of the mask's bits in given |word| are clear.
+ TNode<BoolT> IsClearWord(SloppyTNode<WordT> word, uint32_t mask) {
+ return IntPtrEqual(WordAnd(word, IntPtrConstant(mask)), IntPtrConstant(0));
+ }
+
+ void SetCounter(StatsCounter* counter, int value);
+ void IncrementCounter(StatsCounter* counter, int delta);
+ void DecrementCounter(StatsCounter* counter, int delta);
+
+ template <typename TIndex>
+ void Increment(TVariable<TIndex>* variable, int value = 1);
+
+ template <typename TIndex>
+ void Decrement(TVariable<TIndex>* variable, int value = 1) {
+ Increment(variable, -value);
+ }
+
+ // Generates "if (false) goto label" code. Useful for marking a label as
+ // "live" to avoid assertion failures during graph building. In the resulting
+ // code this check will be eliminated.
+ void Use(Label* label);
+
+ // Various building blocks for stubs doing property lookups.
+
+ // |if_notinternalized| is optional; |if_bailout| will be used by default.
+ // Note: If |key| does not yet have a hash, |if_notinternalized| will be taken
+ // even if |key| is an array index. |if_keyisunique| will never
+ // be taken for array indices.
+ void TryToName(SloppyTNode<Object> key, Label* if_keyisindex,
+ TVariable<IntPtrT>* var_index, Label* if_keyisunique,
+ TVariable<Name>* var_unique, Label* if_bailout,
+ Label* if_notinternalized = nullptr);
+
+ // Performs a hash computation and string table lookup for the given string,
+ // and jumps to:
+ // - |if_index| if the string is an array index like "123"; |var_index|
+ // will contain the intptr representation of that index.
+ // - |if_internalized| if the string exists in the string table; the
+ // internalized version will be in |var_internalized|.
+ // - |if_not_internalized| if the string is not in the string table (but
+ // does not add it).
+ // - |if_bailout| for unsupported cases (e.g. uncachable array index).
+ void TryInternalizeString(TNode<String> string, Label* if_index,
+ TVariable<IntPtrT>* var_index,
+ Label* if_internalized,
+ TVariable<Name>* var_internalized,
+ Label* if_not_internalized, Label* if_bailout);
+
+ // Calculates array index for given dictionary entry and entry field.
+ // See Dictionary::EntryToIndex().
+ template <typename Dictionary>
+ TNode<IntPtrT> EntryToIndex(TNode<IntPtrT> entry, int field_index);
+ template <typename Dictionary>
+ TNode<IntPtrT> EntryToIndex(TNode<IntPtrT> entry) {
+ return EntryToIndex<Dictionary>(entry, Dictionary::kEntryKeyIndex);
+ }
+
+ // Loads the details for the entry with the given key_index.
+ // Returns an untagged int32.
+ template <class ContainerType>
+ TNode<Uint32T> LoadDetailsByKeyIndex(TNode<ContainerType> container,
+ TNode<IntPtrT> key_index) {
+ static_assert(!std::is_same<ContainerType, DescriptorArray>::value,
+ "Use the non-templatized version for DescriptorArray");
+ const int kKeyToDetailsOffset =
+ (ContainerType::kEntryDetailsIndex - ContainerType::kEntryKeyIndex) *
+ kTaggedSize;
+ return Unsigned(LoadAndUntagToWord32FixedArrayElement(container, key_index,
+ kKeyToDetailsOffset));
+ }
+
+ // Loads the value for the entry with the given key_index.
+ // Returns a tagged value.
+ template <class ContainerType>
+ TNode<Object> LoadValueByKeyIndex(TNode<ContainerType> container,
+ TNode<IntPtrT> key_index) {
+ static_assert(!std::is_same<ContainerType, DescriptorArray>::value,
+ "Use the non-templatized version for DescriptorArray");
+ const int kKeyToValueOffset =
+ (ContainerType::kEntryValueIndex - ContainerType::kEntryKeyIndex) *
+ kTaggedSize;
+ return LoadFixedArrayElement(container, key_index, kKeyToValueOffset);
+ }
+
+ // Stores the details for the entry with the given key_index.
+ // |details| must be a Smi.
+ template <class ContainerType>
+ void StoreDetailsByKeyIndex(TNode<ContainerType> container,
+ TNode<IntPtrT> key_index, TNode<Smi> details) {
+ const int kKeyToDetailsOffset =
+ (ContainerType::kEntryDetailsIndex - ContainerType::kEntryKeyIndex) *
+ kTaggedSize;
+ StoreFixedArrayElement(container, key_index, details, SKIP_WRITE_BARRIER,
+ kKeyToDetailsOffset);
+ }
+
+ // Stores the value for the entry with the given key_index.
+ template <class ContainerType>
+ void StoreValueByKeyIndex(
+ TNode<ContainerType> container, TNode<IntPtrT> key_index,
+ TNode<Object> value,
+ WriteBarrierMode write_barrier = UPDATE_WRITE_BARRIER) {
+ const int kKeyToValueOffset =
+ (ContainerType::kEntryValueIndex - ContainerType::kEntryKeyIndex) *
+ kTaggedSize;
+ StoreFixedArrayElement(container, key_index, value, write_barrier,
+ kKeyToValueOffset);
+ }
+
+ // Calculate a valid size for the a hash table.
+ TNode<IntPtrT> HashTableComputeCapacity(TNode<IntPtrT> at_least_space_for);
+
+ template <class Dictionary>
+ TNode<Smi> GetNumberOfElements(TNode<Dictionary> dictionary) {
+ return CAST(
+ LoadFixedArrayElement(dictionary, Dictionary::kNumberOfElementsIndex));
+ }
+
+ TNode<Smi> GetNumberDictionaryNumberOfElements(
+ TNode<NumberDictionary> dictionary) {
+ return GetNumberOfElements<NumberDictionary>(dictionary);
+ }
+
+ template <class Dictionary>
+ void SetNumberOfElements(TNode<Dictionary> dictionary,
+ TNode<Smi> num_elements_smi) {
+ StoreFixedArrayElement(dictionary, Dictionary::kNumberOfElementsIndex,
+ num_elements_smi, SKIP_WRITE_BARRIER);
+ }
+
+ template <class Dictionary>
+ TNode<Smi> GetNumberOfDeletedElements(TNode<Dictionary> dictionary) {
+ return CAST(LoadFixedArrayElement(
+ dictionary, Dictionary::kNumberOfDeletedElementsIndex));
+ }
+
+ template <class Dictionary>
+ void SetNumberOfDeletedElements(TNode<Dictionary> dictionary,
+ TNode<Smi> num_deleted_smi) {
+ StoreFixedArrayElement(dictionary,
+ Dictionary::kNumberOfDeletedElementsIndex,
+ num_deleted_smi, SKIP_WRITE_BARRIER);
+ }
+
+ template <class Dictionary>
+ TNode<Smi> GetCapacity(TNode<Dictionary> dictionary) {
+ return CAST(
+ UnsafeLoadFixedArrayElement(dictionary, Dictionary::kCapacityIndex));
+ }
+
+ template <class Dictionary>
+ TNode<Smi> GetNextEnumerationIndex(TNode<Dictionary> dictionary) {
+ return CAST(LoadFixedArrayElement(dictionary,
+ Dictionary::kNextEnumerationIndexIndex));
+ }
+
+ template <class Dictionary>
+ void SetNextEnumerationIndex(TNode<Dictionary> dictionary,
+ TNode<Smi> next_enum_index_smi) {
+ StoreFixedArrayElement(dictionary, Dictionary::kNextEnumerationIndexIndex,
+ next_enum_index_smi, SKIP_WRITE_BARRIER);
+ }
+
+ // Looks up an entry in a NameDictionaryBase successor. If the entry is found
+ // control goes to {if_found} and {var_name_index} contains an index of the
+ // key field of the entry found. If the key is not found control goes to
+ // {if_not_found}.
+ enum LookupMode { kFindExisting, kFindInsertionIndex };
+
+ template <typename Dictionary>
+ TNode<HeapObject> LoadName(TNode<HeapObject> key);
+
+ template <typename Dictionary>
+ void NameDictionaryLookup(TNode<Dictionary> dictionary,
+ TNode<Name> unique_name, Label* if_found,
+ TVariable<IntPtrT>* var_name_index,
+ Label* if_not_found,
+ LookupMode mode = kFindExisting);
+
+ TNode<Word32T> ComputeSeededHash(TNode<IntPtrT> key);
+
+ void NumberDictionaryLookup(TNode<NumberDictionary> dictionary,
+ TNode<IntPtrT> intptr_index, Label* if_found,
+ TVariable<IntPtrT>* var_entry,
+ Label* if_not_found);
+
+ TNode<Object> BasicLoadNumberDictionaryElement(
+ TNode<NumberDictionary> dictionary, TNode<IntPtrT> intptr_index,
+ Label* not_data, Label* if_hole);
+
+ template <class Dictionary>
+ void FindInsertionEntry(TNode<Dictionary> dictionary, TNode<Name> key,
+ TVariable<IntPtrT>* var_key_index);
+
+ template <class Dictionary>
+ void InsertEntry(TNode<Dictionary> dictionary, TNode<Name> key,
+ TNode<Object> value, TNode<IntPtrT> index,
+ TNode<Smi> enum_index);
+
+ template <class Dictionary>
+ void Add(TNode<Dictionary> dictionary, TNode<Name> key, TNode<Object> value,
+ Label* bailout);
+
+ // Tries to check if {object} has own {unique_name} property.
+ void TryHasOwnProperty(TNode<HeapObject> object, TNode<Map> map,
+ TNode<Int32T> instance_type, TNode<Name> unique_name,
+ Label* if_found, Label* if_not_found,
+ Label* if_bailout);
+
+ // Operating mode for TryGetOwnProperty and CallGetterIfAccessor
+ // kReturnAccessorPair is used when we're only getting the property descriptor
+ enum GetOwnPropertyMode { kCallJSGetter, kReturnAccessorPair };
+ // Tries to get {object}'s own {unique_name} property value. If the property
+ // is an accessor then it also calls a getter. If the property is a double
+ // field it re-wraps value in an immutable heap number. {unique_name} must be
+ // a unique name (Symbol or InternalizedString) that is not an array index.
+ void TryGetOwnProperty(TNode<Context> context, TNode<Object> receiver,
+ TNode<JSReceiver> object, TNode<Map> map,
+ TNode<Int32T> instance_type, TNode<Name> unique_name,
+ Label* if_found_value, TVariable<Object>* var_value,
+ Label* if_not_found, Label* if_bailout);
+ void TryGetOwnProperty(TNode<Context> context, TNode<Object> receiver,
+ TNode<JSReceiver> object, TNode<Map> map,
+ TNode<Int32T> instance_type, TNode<Name> unique_name,
+ Label* if_found_value, TVariable<Object>* var_value,
+ TVariable<Uint32T>* var_details,
+ TVariable<Object>* var_raw_value, Label* if_not_found,
+ Label* if_bailout, GetOwnPropertyMode mode);
+
+ TNode<Object> GetProperty(TNode<Context> context,
+ SloppyTNode<Object> receiver, Handle<Name> name) {
+ return GetProperty(context, receiver, HeapConstant(name));
+ }
+
+ TNode<Object> GetProperty(TNode<Context> context,
+ SloppyTNode<Object> receiver,
+ SloppyTNode<Object> name) {
+ return CallBuiltin(Builtins::kGetProperty, context, receiver, name);
+ }
+
+ TNode<Object> SetPropertyStrict(TNode<Context> context,
+ TNode<Object> receiver, TNode<Object> key,
+ TNode<Object> value) {
+ return CallBuiltin(Builtins::kSetProperty, context, receiver, key, value);
+ }
+
+ TNode<Object> SetPropertyInLiteral(TNode<Context> context,
+ TNode<JSObject> receiver,
+ TNode<Object> key, TNode<Object> value) {
+ return CallBuiltin(Builtins::kSetPropertyInLiteral, context, receiver, key,
+ value);
+ }
+
+ TNode<Object> GetMethod(TNode<Context> context, TNode<Object> object,
+ Handle<Name> name, Label* if_null_or_undefined);
+
+ TNode<Object> GetIteratorMethod(TNode<Context> context,
+ TNode<HeapObject> heap_obj,
+ Label* if_iteratorundefined);
+
+ template <class... TArgs>
+ TNode<Object> CallBuiltin(Builtins::Name id, SloppyTNode<Object> context,
+ TArgs... args) {
+ return CallStub<Object>(Builtins::CallableFor(isolate(), id), context,
+ args...);
+ }
+
+ template <class... TArgs>
+ void TailCallBuiltin(Builtins::Name id, SloppyTNode<Object> context,
+ TArgs... args) {
+ return TailCallStub(Builtins::CallableFor(isolate(), id), context, args...);
+ }
+
+ void LoadPropertyFromFastObject(TNode<HeapObject> object, TNode<Map> map,
+ TNode<DescriptorArray> descriptors,
+ TNode<IntPtrT> name_index,
+ TVariable<Uint32T>* var_details,
+ TVariable<Object>* var_value);
+
+ void LoadPropertyFromFastObject(TNode<HeapObject> object, TNode<Map> map,
+ TNode<DescriptorArray> descriptors,
+ TNode<IntPtrT> name_index, TNode<Uint32T>,
+ TVariable<Object>* var_value);
+
+ void LoadPropertyFromNameDictionary(TNode<NameDictionary> dictionary,
+ TNode<IntPtrT> name_index,
+ TVariable<Uint32T>* var_details,
+ TVariable<Object>* var_value);
+ void LoadPropertyFromGlobalDictionary(TNode<GlobalDictionary> dictionary,
+ TNode<IntPtrT> name_index,
+ TVariable<Uint32T>* var_details,
+ TVariable<Object>* var_value,
+ Label* if_deleted);
+
+ // Generic property lookup generator. If the {object} is fast and
+ // {unique_name} property is found then the control goes to {if_found_fast}
+ // label and {var_meta_storage} and {var_name_index} will contain
+ // DescriptorArray and an index of the descriptor's name respectively.
+ // If the {object} is slow or global then the control goes to {if_found_dict}
+ // or {if_found_global} and the {var_meta_storage} and {var_name_index} will
+ // contain a dictionary and an index of the key field of the found entry.
+ // If property is not found or given lookup is not supported then
+ // the control goes to {if_not_found} or {if_bailout} respectively.
+ //
+ // Note: this code does not check if the global dictionary points to deleted
+ // entry! This has to be done by the caller.
+ void TryLookupProperty(TNode<HeapObject> object, TNode<Map> map,
+ SloppyTNode<Int32T> instance_type,
+ TNode<Name> unique_name, Label* if_found_fast,
+ Label* if_found_dict, Label* if_found_global,
+ TVariable<HeapObject>* var_meta_storage,
+ TVariable<IntPtrT>* var_name_index,
+ Label* if_not_found, Label* if_bailout);
+
+ // This is a building block for TryLookupProperty() above. Supports only
+ // non-special fast and dictionary objects.
+ void TryLookupPropertyInSimpleObject(TNode<JSObject> object, TNode<Map> map,
+ TNode<Name> unique_name,
+ Label* if_found_fast,
+ Label* if_found_dict,
+ TVariable<HeapObject>* var_meta_storage,
+ TVariable<IntPtrT>* var_name_index,
+ Label* if_not_found);
+
+ // This method jumps to if_found if the element is known to exist. To
+ // if_absent if it's known to not exist. To if_not_found if the prototype
+ // chain needs to be checked. And if_bailout if the lookup is unsupported.
+ void TryLookupElement(TNode<HeapObject> object, TNode<Map> map,
+ SloppyTNode<Int32T> instance_type,
+ SloppyTNode<IntPtrT> intptr_index, Label* if_found,
+ Label* if_absent, Label* if_not_found,
+ Label* if_bailout);
+
+ // For integer indexed exotic cases, check if the given string cannot be a
+ // special index. If we are not sure that the given string is not a special
+ // index with a simple check, return False. Note that "False" return value
+ // does not mean that the name_string is a special index in the current
+ // implementation.
+ void BranchIfMaybeSpecialIndex(TNode<String> name_string,
+ Label* if_maybe_special_index,
+ Label* if_not_special_index);
+
+ // This is a type of a lookup property in holder generator function. The {key}
+ // is guaranteed to be an unique name.
+ using LookupPropertyInHolder = std::function<void(
+ TNode<HeapObject> receiver, TNode<HeapObject> holder, TNode<Map> map,
+ TNode<Int32T> instance_type, TNode<Name> key, Label* next_holder,
+ Label* if_bailout)>;
+
+ // This is a type of a lookup element in holder generator function. The {key}
+ // is an Int32 index.
+ using LookupElementInHolder = std::function<void(
+ TNode<HeapObject> receiver, TNode<HeapObject> holder, TNode<Map> map,
+ TNode<Int32T> instance_type, TNode<IntPtrT> key, Label* next_holder,
+ Label* if_bailout)>;
+
+ // Generic property prototype chain lookup generator.
+ // For properties it generates lookup using given {lookup_property_in_holder}
+ // and for elements it uses {lookup_element_in_holder}.
+ // Upon reaching the end of prototype chain the control goes to {if_end}.
+ // If it can't handle the case {receiver}/{key} case then the control goes
+ // to {if_bailout}.
+ // If {if_proxy} is nullptr, proxies go to if_bailout.
+ void TryPrototypeChainLookup(
+ TNode<Object> receiver, TNode<Object> object, TNode<Object> key,
+ const LookupPropertyInHolder& lookup_property_in_holder,
+ const LookupElementInHolder& lookup_element_in_holder, Label* if_end,
+ Label* if_bailout, Label* if_proxy);
+
+ // Instanceof helpers.
+ // Returns true if {object} has {prototype} somewhere in it's prototype
+ // chain, otherwise false is returned. Might cause arbitrary side effects
+ // due to [[GetPrototypeOf]] invocations.
+ TNode<Oddball> HasInPrototypeChain(TNode<Context> context,
+ TNode<HeapObject> object,
+ TNode<Object> prototype);
+ // ES6 section 7.3.19 OrdinaryHasInstance (C, O)
+ TNode<Oddball> OrdinaryHasInstance(TNode<Context> context,
+ TNode<Object> callable,
+ TNode<Object> object);
+
+ // Load type feedback vector from the stub caller's frame.
+ TNode<FeedbackVector> LoadFeedbackVectorForStub();
+
+ // Load the value from closure's feedback cell.
+ TNode<HeapObject> LoadFeedbackCellValue(TNode<JSFunction> closure);
+
+ // Load the object from feedback vector cell for the given closure.
+ // The returned object could be undefined if the closure does not have
+ // a feedback vector associated with it.
+ TNode<HeapObject> LoadFeedbackVector(TNode<JSFunction> closure);
+
+ // Load the ClosureFeedbackCellArray that contains the feedback cells
+ // used when creating closures from this function. This array could be
+ // directly hanging off the FeedbackCell when there is no feedback vector
+ // or available from the feedback vector's header.
+ TNode<ClosureFeedbackCellArray> LoadClosureFeedbackArray(
+ TNode<JSFunction> closure);
+
+ // Update the type feedback vector.
+ void UpdateFeedback(TNode<Smi> feedback,
+ TNode<HeapObject> maybe_feedback_vector,
+ TNode<UintPtrT> slot_id);
+
+ // Report that there was a feedback update, performing any tasks that should
+ // be done after a feedback update.
+ void ReportFeedbackUpdate(TNode<FeedbackVector> feedback_vector,
+ SloppyTNode<UintPtrT> slot_id, const char* reason);
+
+ // Combine the new feedback with the existing_feedback. Do nothing if
+ // existing_feedback is nullptr.
+ void CombineFeedback(TVariable<Smi>* existing_feedback, int feedback);
+ void CombineFeedback(TVariable<Smi>* existing_feedback, TNode<Smi> feedback);
+
+ // Overwrite the existing feedback with new_feedback. Do nothing if
+ // existing_feedback is nullptr.
+ void OverwriteFeedback(TVariable<Smi>* existing_feedback, int new_feedback);
+
+ // Check if a property name might require protector invalidation when it is
+ // used for a property store or deletion.
+ void CheckForAssociatedProtector(TNode<Name> name, Label* if_protector);
+
+ TNode<Map> LoadReceiverMap(SloppyTNode<Object> receiver);
+
+ // Loads script context from the script context table.
+ TNode<Context> LoadScriptContext(TNode<Context> context,
+ TNode<IntPtrT> context_index);
+
+ TNode<Uint8T> Int32ToUint8Clamped(TNode<Int32T> int32_value);
+ TNode<Uint8T> Float64ToUint8Clamped(TNode<Float64T> float64_value);
+
+ Node* PrepareValueForWriteToTypedArray(TNode<Object> input,
+ ElementsKind elements_kind,
+ TNode<Context> context);
+
+ template <typename T>
+ TNode<T> PrepareValueForWriteToTypedArray(TNode<Object> input,
+ ElementsKind elements_kind,
+ TNode<Context> context);
+
+ // Store value to an elements array with given elements kind.
+ // TODO(turbofan): For BIGINT64_ELEMENTS and BIGUINT64_ELEMENTS
+ // we pass {value} as BigInt object instead of int64_t. We should
+ // teach TurboFan to handle int64_t on 32-bit platforms eventually.
+ template <typename TIndex>
+ void StoreElement(TNode<RawPtrT> elements, ElementsKind kind,
+ TNode<TIndex> index, Node* value);
+
+ // Implements the BigInt part of
+ // https://tc39.github.io/proposal-bigint/#sec-numbertorawbytes,
+ // including truncation to 64 bits (i.e. modulo 2^64).
+ // {var_high} is only used on 32-bit platforms.
+ void BigIntToRawBytes(TNode<BigInt> bigint, TVariable<UintPtrT>* var_low,
+ TVariable<UintPtrT>* var_high);
+
+ void EmitElementStore(TNode<JSObject> object, TNode<Object> key,
+ TNode<Object> value, ElementsKind elements_kind,
+ KeyedAccessStoreMode store_mode, Label* bailout,
+ TNode<Context> context,
+ TVariable<Object>* maybe_converted_value = nullptr);
+
+ TNode<FixedArrayBase> CheckForCapacityGrow(
+ TNode<JSObject> object, TNode<FixedArrayBase> elements, ElementsKind kind,
+ TNode<UintPtrT> length, TNode<IntPtrT> key, Label* bailout);
+
+ TNode<FixedArrayBase> CopyElementsOnWrite(TNode<HeapObject> object,
+ TNode<FixedArrayBase> elements,
+ ElementsKind kind,
+ TNode<IntPtrT> length,
+ Label* bailout);
+
+ void TransitionElementsKind(TNode<JSObject> object, TNode<Map> map,
+ ElementsKind from_kind, ElementsKind to_kind,
+ Label* bailout);
+
+ void TrapAllocationMemento(TNode<JSObject> object, Label* memento_found);
+
+ TNode<IntPtrT> PageFromAddress(TNode<IntPtrT> address);
+
+ // Store a weak in-place reference into the FeedbackVector.
+ TNode<MaybeObject> StoreWeakReferenceInFeedbackVector(
+ TNode<FeedbackVector> feedback_vector, TNode<UintPtrT> slot,
+ TNode<HeapObject> value, int additional_offset = 0);
+
+ // Create a new AllocationSite and install it into a feedback vector.
+ TNode<AllocationSite> CreateAllocationSiteInFeedbackVector(
+ TNode<FeedbackVector> feedback_vector, TNode<UintPtrT> slot);
+
+ TNode<BoolT> HasBoilerplate(TNode<Object> maybe_literal_site);
+ TNode<Smi> LoadTransitionInfo(TNode<AllocationSite> allocation_site);
+ TNode<JSObject> LoadBoilerplate(TNode<AllocationSite> allocation_site);
+ TNode<Int32T> LoadElementsKind(TNode<AllocationSite> allocation_site);
+
+ enum class IndexAdvanceMode { kPre, kPost };
+
+ template <typename TIndex>
+ using FastLoopBody = std::function<void(TNode<TIndex> index)>;
+
+ template <typename TIndex>
+ TNode<TIndex> BuildFastLoop(
+ const VariableList& var_list, TNode<TIndex> start_index,
+ TNode<TIndex> end_index, const FastLoopBody<TIndex>& body, int increment,
+ IndexAdvanceMode advance_mode = IndexAdvanceMode::kPre);
+
+ template <typename TIndex>
+ TNode<TIndex> BuildFastLoop(
+ TNode<TIndex> start_index, TNode<TIndex> end_index,
+ const FastLoopBody<TIndex>& body, int increment,
+ IndexAdvanceMode advance_mode = IndexAdvanceMode::kPre) {
+ return BuildFastLoop(VariableList(0, zone()), start_index, end_index, body,
+ increment, advance_mode);
+ }
+
+ enum class ForEachDirection { kForward, kReverse };
+
+ using FastArrayForEachBody =
+ std::function<void(TNode<HeapObject> array, TNode<IntPtrT> offset)>;
+
+ template <typename TIndex>
+ void BuildFastArrayForEach(
+ TNode<UnionT<UnionT<FixedArray, PropertyArray>, HeapObject>> array,
+ ElementsKind kind, TNode<TIndex> first_element_inclusive,
+ TNode<TIndex> last_element_exclusive, const FastArrayForEachBody& body,
+ ForEachDirection direction = ForEachDirection::kReverse);
+
+ template <typename TIndex>
+ TNode<IntPtrT> GetArrayAllocationSize(TNode<TIndex> element_count,
+ ElementsKind kind, int header_size) {
+ return ElementOffsetFromIndex(element_count, kind, header_size);
+ }
+
+ template <typename TIndex>
+ TNode<IntPtrT> GetFixedArrayAllocationSize(TNode<TIndex> element_count,
+ ElementsKind kind) {
+ return GetArrayAllocationSize(element_count, kind, FixedArray::kHeaderSize);
+ }
+
+ TNode<IntPtrT> GetPropertyArrayAllocationSize(TNode<IntPtrT> element_count) {
+ return GetArrayAllocationSize(element_count, PACKED_ELEMENTS,
+ PropertyArray::kHeaderSize);
+ }
+
+ template <typename TIndex>
+ void GotoIfFixedArraySizeDoesntFitInNewSpace(TNode<TIndex> element_count,
+ Label* doesnt_fit,
+ int base_size);
+
+ void InitializeFieldsWithRoot(TNode<HeapObject> object,
+ TNode<IntPtrT> start_offset,
+ TNode<IntPtrT> end_offset, RootIndex root);
+
+ TNode<Oddball> RelationalComparison(
+ Operation op, TNode<Object> left, TNode<Object> right,
+ TNode<Context> context, TVariable<Smi>* var_type_feedback = nullptr);
+
+ void BranchIfNumberRelationalComparison(Operation op, TNode<Number> left,
+ TNode<Number> right, Label* if_true,
+ Label* if_false);
+
+ void BranchIfNumberEqual(TNode<Number> left, TNode<Number> right,
+ Label* if_true, Label* if_false) {
+ BranchIfNumberRelationalComparison(Operation::kEqual, left, right, if_true,
+ if_false);
+ }
+
+ void BranchIfNumberNotEqual(TNode<Number> left, TNode<Number> right,
+ Label* if_true, Label* if_false) {
+ BranchIfNumberEqual(left, right, if_false, if_true);
+ }
+
+ void BranchIfNumberLessThan(TNode<Number> left, TNode<Number> right,
+ Label* if_true, Label* if_false) {
+ BranchIfNumberRelationalComparison(Operation::kLessThan, left, right,
+ if_true, if_false);
+ }
+
+ void BranchIfNumberLessThanOrEqual(TNode<Number> left, TNode<Number> right,
+ Label* if_true, Label* if_false) {
+ BranchIfNumberRelationalComparison(Operation::kLessThanOrEqual, left, right,
+ if_true, if_false);
+ }
+
+ void BranchIfNumberGreaterThan(TNode<Number> left, TNode<Number> right,
+ Label* if_true, Label* if_false) {
+ BranchIfNumberRelationalComparison(Operation::kGreaterThan, left, right,
+ if_true, if_false);
+ }
+
+ void BranchIfNumberGreaterThanOrEqual(TNode<Number> left, TNode<Number> right,
+ Label* if_true, Label* if_false) {
+ BranchIfNumberRelationalComparison(Operation::kGreaterThanOrEqual, left,
+ right, if_true, if_false);
+ }
+
+ void BranchIfAccessorPair(TNode<Object> value, Label* if_accessor_pair,
+ Label* if_not_accessor_pair) {
+ GotoIf(TaggedIsSmi(value), if_not_accessor_pair);
+ Branch(IsAccessorPair(CAST(value)), if_accessor_pair, if_not_accessor_pair);
+ }
+
+ void GotoIfNumberGreaterThanOrEqual(TNode<Number> left, TNode<Number> right,
+ Label* if_false);
+
+ TNode<Oddball> Equal(SloppyTNode<Object> lhs, SloppyTNode<Object> rhs,
+ TNode<Context> context,
+ TVariable<Smi>* var_type_feedback = nullptr);
+
+ TNode<Oddball> StrictEqual(SloppyTNode<Object> lhs, SloppyTNode<Object> rhs,
+ TVariable<Smi>* var_type_feedback = nullptr);
+
+ // ECMA#sec-samevalue
+ // Similar to StrictEqual except that NaNs are treated as equal and minus zero
+ // differs from positive zero.
+ enum class SameValueMode { kNumbersOnly, kFull };
+ void BranchIfSameValue(SloppyTNode<Object> lhs, SloppyTNode<Object> rhs,
+ Label* if_true, Label* if_false,
+ SameValueMode mode = SameValueMode::kFull);
+ // A part of BranchIfSameValue() that handles two double values.
+ // Treats NaN == NaN and +0 != -0.
+ void BranchIfSameNumberValue(TNode<Float64T> lhs_value,
+ TNode<Float64T> rhs_value, Label* if_true,
+ Label* if_false);
+
+ enum HasPropertyLookupMode { kHasProperty, kForInHasProperty };
+
+ TNode<Oddball> HasProperty(TNode<Context> context, SloppyTNode<Object> object,
+ SloppyTNode<Object> key,
+ HasPropertyLookupMode mode);
+
+ // Due to naming conflict with the builtin function namespace.
+ TNode<Oddball> HasProperty_Inline(TNode<Context> context,
+ TNode<JSReceiver> object,
+ TNode<Object> key) {
+ return HasProperty(context, object, key,
+ HasPropertyLookupMode::kHasProperty);
+ }
+
+ void ForInPrepare(TNode<HeapObject> enumerator, TNode<UintPtrT> slot,
+ TNode<HeapObject> maybe_feedback_vector,
+ TNode<FixedArray>* cache_array_out,
+ TNode<Smi>* cache_length_out);
+ // Returns {cache_array} and {cache_length} in a fixed array of length 2.
+ // TODO(jgruber): Tuple2 would be a slightly better fit as the return type,
+ // but FixedArray has better support and there are no effective drawbacks to
+ // using it instead of Tuple2 in practice.
+ TNode<FixedArray> ForInPrepareForTorque(
+ TNode<HeapObject> enumerator, TNode<UintPtrT> slot,
+ TNode<HeapObject> maybe_feedback_vector);
+
+ TNode<String> Typeof(SloppyTNode<Object> value);
+
+ TNode<HeapObject> GetSuperConstructor(TNode<JSFunction> active_function);
+
+ TNode<JSReceiver> SpeciesConstructor(TNode<Context> context,
+ SloppyTNode<Object> object,
+ TNode<JSReceiver> default_constructor);
+
+ TNode<Oddball> InstanceOf(TNode<Object> object, TNode<Object> callable,
+ TNode<Context> context);
+
+ // Debug helpers
+ TNode<BoolT> IsDebugActive();
+
+ // JSArrayBuffer helpers
+ TNode<RawPtrT> LoadJSArrayBufferBackingStorePtr(
+ TNode<JSArrayBuffer> array_buffer);
+ void ThrowIfArrayBufferIsDetached(TNode<Context> context,
+ TNode<JSArrayBuffer> array_buffer,
+ const char* method_name);
+
+ // JSArrayBufferView helpers
+ TNode<JSArrayBuffer> LoadJSArrayBufferViewBuffer(
+ TNode<JSArrayBufferView> array_buffer_view);
+ TNode<UintPtrT> LoadJSArrayBufferViewByteLength(
+ TNode<JSArrayBufferView> array_buffer_view);
+ TNode<UintPtrT> LoadJSArrayBufferViewByteOffset(
+ TNode<JSArrayBufferView> array_buffer_view);
+ void ThrowIfArrayBufferViewBufferIsDetached(
+ TNode<Context> context, TNode<JSArrayBufferView> array_buffer_view,
+ const char* method_name);
+
+ // JSTypedArray helpers
+ TNode<UintPtrT> LoadJSTypedArrayLength(TNode<JSTypedArray> typed_array);
+ TNode<RawPtrT> LoadJSTypedArrayDataPtr(TNode<JSTypedArray> typed_array);
+ TNode<JSArrayBuffer> GetTypedArrayBuffer(TNode<Context> context,
+ TNode<JSTypedArray> array);
+
+ template <typename TIndex>
+ TNode<IntPtrT> ElementOffsetFromIndex(TNode<TIndex> index, ElementsKind kind,
+ int base_size = 0);
+
+ // Check that a field offset is within the bounds of the an object.
+ TNode<BoolT> IsOffsetInBounds(SloppyTNode<IntPtrT> offset,
+ SloppyTNode<IntPtrT> length, int header_size,
+ ElementsKind kind = HOLEY_ELEMENTS);
+
+ // Load a builtin's code from the builtin array in the isolate.
+ TNode<Code> LoadBuiltin(TNode<Smi> builtin_id);
+
+ // Figure out the SFI's code object using its data field.
+ // If |if_compile_lazy| is provided then the execution will go to the given
+ // label in case of an CompileLazy code object.
+ TNode<Code> GetSharedFunctionInfoCode(TNode<SharedFunctionInfo> shared_info,
+ Label* if_compile_lazy = nullptr);
+
+ TNode<JSFunction> AllocateFunctionWithMapAndContext(
+ TNode<Map> map, TNode<SharedFunctionInfo> shared_info,
+ TNode<Context> context);
+
+ // Promise helpers
+ TNode<BoolT> IsPromiseHookEnabled();
+ TNode<BoolT> HasAsyncEventDelegate();
+ TNode<BoolT> IsPromiseHookEnabledOrHasAsyncEventDelegate();
+ TNode<BoolT> IsPromiseHookEnabledOrDebugIsActiveOrHasAsyncEventDelegate();
+
+ // for..in helpers
+ void CheckPrototypeEnumCache(TNode<JSReceiver> receiver,
+ TNode<Map> receiver_map, Label* if_fast,
+ Label* if_slow);
+ TNode<Map> CheckEnumCache(TNode<JSReceiver> receiver, Label* if_empty,
+ Label* if_runtime);
+
+ TNode<Object> GetArgumentValue(TorqueStructArguments args,
+ TNode<IntPtrT> index);
+
+ TorqueStructArguments GetFrameArguments(TNode<RawPtrT> frame,
+ TNode<IntPtrT> argc);
+
+ // Support for printf-style debugging
+ void Print(const char* s);
+ void Print(const char* prefix, TNode<MaybeObject> tagged_value);
+ void Print(TNode<MaybeObject> tagged_value) {
+ return Print(nullptr, tagged_value);
+ }
+
+ template <class... TArgs>
+ TNode<HeapObject> MakeTypeError(MessageTemplate message,
+ TNode<Context> context, TArgs... args) {
+ STATIC_ASSERT(sizeof...(TArgs) <= 3);
+ return CAST(CallRuntime(Runtime::kNewTypeError, context,
+ SmiConstant(message), args...));
+ }
+
+ void Abort(AbortReason reason) {
+ CallRuntime(Runtime::kAbort, NoContextConstant(), SmiConstant(reason));
+ Unreachable();
+ }
+
+ bool ConstexprBoolNot(bool value) { return !value; }
+
+ bool ConstexprInt31Equal(int31_t a, int31_t b) { return a == b; }
+ bool ConstexprInt31NotEqual(int31_t a, int31_t b) { return a != b; }
+ bool ConstexprInt31GreaterThanEqual(int31_t a, int31_t b) { return a >= b; }
+ bool ConstexprUint32Equal(uint32_t a, uint32_t b) { return a == b; }
+ bool ConstexprUint32NotEqual(uint32_t a, uint32_t b) { return a != b; }
+ bool ConstexprInt32Equal(int32_t a, int32_t b) { return a == b; }
+ bool ConstexprInt32NotEqual(int32_t a, int32_t b) { return a != b; }
+ bool ConstexprInt32GreaterThanEqual(int32_t a, int32_t b) { return a >= b; }
+ uint32_t ConstexprUint32Add(uint32_t a, uint32_t b) { return a + b; }
+ int32_t ConstexprUint32Sub(uint32_t a, uint32_t b) { return a - b; }
+ int31_t ConstexprInt31Add(int31_t a, int31_t b) {
+ int32_t val;
+ CHECK(!base::bits::SignedAddOverflow32(a, b, &val));
+ return val;
+ }
+ int31_t ConstexprInt31Mul(int31_t a, int31_t b) {
+ int32_t val;
+ CHECK(!base::bits::SignedMulOverflow32(a, b, &val));
+ return val;
+ }
+
+ int32_t ConstexprWord32Or(int32_t a, int32_t b) { return a | b; }
+
+ bool ConstexprUintPtrLessThan(uintptr_t a, uintptr_t b) { return a < b; }
+
+ // CSA does not support 64-bit types on 32-bit platforms so as a workaround
+ // the kMaxSafeIntegerUint64 is defined as uintptr and allowed to be used only
+ // inside if constexpr (Is64()) i.e. on 64-bit architectures.
+ static uintptr_t MaxSafeIntegerUintPtr() {
+#if defined(V8_HOST_ARCH_64_BIT)
+ // This ifdef is required to avoid build issues on 32-bit MSVC which
+ // complains about static_cast<uintptr_t>(kMaxSafeIntegerUint64).
+ return kMaxSafeIntegerUint64;
+#else
+ UNREACHABLE();
+#endif
+ }
+
+ void PerformStackCheck(TNode<Context> context);
+
+ void SetPropertyLength(TNode<Context> context, TNode<Object> array,
+ TNode<Number> length);
+
+ // Implements DescriptorArray::Search().
+ void DescriptorLookup(TNode<Name> unique_name,
+ TNode<DescriptorArray> descriptors,
+ TNode<Uint32T> bitfield3, Label* if_found,
+ TVariable<IntPtrT>* var_name_index,
+ Label* if_not_found);
+
+ // Implements TransitionArray::SearchName() - searches for first transition
+ // entry with given name (note that there could be multiple entries with
+ // the same name).
+ void TransitionLookup(TNode<Name> unique_name,
+ TNode<TransitionArray> transitions, Label* if_found,
+ TVariable<IntPtrT>* var_name_index,
+ Label* if_not_found);
+
+ // Implements generic search procedure like i::Search<Array>().
+ template <typename Array>
+ void Lookup(TNode<Name> unique_name, TNode<Array> array,
+ TNode<Uint32T> number_of_valid_entries, Label* if_found,
+ TVariable<IntPtrT>* var_name_index, Label* if_not_found);
+
+ // Implements generic linear search procedure like i::LinearSearch<Array>().
+ template <typename Array>
+ void LookupLinear(TNode<Name> unique_name, TNode<Array> array,
+ TNode<Uint32T> number_of_valid_entries, Label* if_found,
+ TVariable<IntPtrT>* var_name_index, Label* if_not_found);
+
+ // Implements generic binary search procedure like i::BinarySearch<Array>().
+ template <typename Array>
+ void LookupBinary(TNode<Name> unique_name, TNode<Array> array,
+ TNode<Uint32T> number_of_valid_entries, Label* if_found,
+ TVariable<IntPtrT>* var_name_index, Label* if_not_found);
+
+ // Converts [Descriptor/Transition]Array entry number to a fixed array index.
+ template <typename Array>
+ TNode<IntPtrT> EntryIndexToIndex(TNode<Uint32T> entry_index);
+
+ // Implements [Descriptor/Transition]Array::ToKeyIndex.
+ template <typename Array>
+ TNode<IntPtrT> ToKeyIndex(TNode<Uint32T> entry_index);
+
+ // Implements [Descriptor/Transition]Array::GetKey.
+ template <typename Array>
+ TNode<Name> GetKey(TNode<Array> array, TNode<Uint32T> entry_index);
+
+ // Implements DescriptorArray::GetDetails.
+ TNode<Uint32T> DescriptorArrayGetDetails(TNode<DescriptorArray> descriptors,
+ TNode<Uint32T> descriptor_number);
+
+ using ForEachDescriptorBodyFunction =
+ std::function<void(TNode<IntPtrT> descriptor_key_index)>;
+
+ // Descriptor array accessors based on key_index, which is equal to
+ // DescriptorArray::ToKeyIndex(descriptor).
+ TNode<Name> LoadKeyByKeyIndex(TNode<DescriptorArray> container,
+ TNode<IntPtrT> key_index);
+ TNode<Uint32T> LoadDetailsByKeyIndex(TNode<DescriptorArray> container,
+ TNode<IntPtrT> key_index);
+ TNode<Object> LoadValueByKeyIndex(TNode<DescriptorArray> container,
+ TNode<IntPtrT> key_index);
+ TNode<MaybeObject> LoadFieldTypeByKeyIndex(TNode<DescriptorArray> container,
+ TNode<IntPtrT> key_index);
+
+ TNode<IntPtrT> DescriptorEntryToIndex(TNode<IntPtrT> descriptor);
+
+ // Descriptor array accessors based on descriptor.
+ TNode<Name> LoadKeyByDescriptorEntry(TNode<DescriptorArray> descriptors,
+ TNode<IntPtrT> descriptor);
+ TNode<Name> LoadKeyByDescriptorEntry(TNode<DescriptorArray> descriptors,
+ int descriptor);
+ TNode<Uint32T> LoadDetailsByDescriptorEntry(
+ TNode<DescriptorArray> descriptors, TNode<IntPtrT> descriptor);
+ TNode<Uint32T> LoadDetailsByDescriptorEntry(
+ TNode<DescriptorArray> descriptors, int descriptor);
+ TNode<Object> LoadValueByDescriptorEntry(TNode<DescriptorArray> descriptors,
+ int descriptor);
+ TNode<MaybeObject> LoadFieldTypeByDescriptorEntry(
+ TNode<DescriptorArray> descriptors, TNode<IntPtrT> descriptor);
+
+ using ForEachKeyValueFunction =
+ std::function<void(TNode<Name> key, TNode<Object> value)>;
+
+ enum ForEachEnumerationMode {
+ // String and then Symbol properties according to the spec
+ // ES#sec-object.assign
+ kEnumerationOrder,
+ // Order of property addition
+ kPropertyAdditionOrder,
+ };
+
+ // For each JSObject property (in DescriptorArray order), check if the key is
+ // enumerable, and if so, load the value from the receiver and evaluate the
+ // closure.
+ void ForEachEnumerableOwnProperty(TNode<Context> context, TNode<Map> map,
+ TNode<JSObject> object,
+ ForEachEnumerationMode mode,
+ const ForEachKeyValueFunction& body,
+ Label* bailout);
+
+ TNode<Object> CallGetterIfAccessor(TNode<Object> value,
+ TNode<HeapObject> holder,
+ TNode<Uint32T> details,
+ TNode<Context> context,
+ TNode<Object> receiver, Label* if_bailout,
+ GetOwnPropertyMode mode = kCallJSGetter);
+
+ TNode<IntPtrT> TryToIntptr(SloppyTNode<Object> key, Label* if_not_intptr,
+ TVariable<Int32T>* var_instance_type = nullptr);
+
+ TNode<JSArray> ArrayCreate(TNode<Context> context, TNode<Number> length);
+
+ // Allocate a clone of a mutable primitive, if {object} is a mutable
+ // HeapNumber.
+ TNode<Object> CloneIfMutablePrimitive(TNode<Object> object);
+
+ TNode<Smi> RefillMathRandom(TNode<NativeContext> native_context);
+
+ void RemoveFinalizationRegistryCellFromUnregisterTokenMap(
+ TNode<JSFinalizationRegistry> finalization_registry,
+ TNode<WeakCell> weak_cell);
+
+ TNode<IntPtrT> FeedbackIteratorSizeFor(int number_of_entries) {
+ return IntPtrConstant(FeedbackIterator::SizeFor(number_of_entries));
+ }
+
+ TNode<IntPtrT> FeedbackIteratorMapIndexForEntry(int entry) {
+ return IntPtrConstant(FeedbackIterator::MapIndexForEntry(entry));
+ }
+
+ TNode<IntPtrT> FeedbackIteratorHandlerIndexForEntry(int entry) {
+ return IntPtrConstant(FeedbackIterator::HandlerIndexForEntry(entry));
+ }
+
+ private:
+ friend class CodeStubArguments;
+
+ void HandleBreakOnNode();
+
+ TNode<HeapObject> AllocateRawDoubleAligned(TNode<IntPtrT> size_in_bytes,
+ AllocationFlags flags,
+ TNode<RawPtrT> top_address,
+ TNode<RawPtrT> limit_address);
+ TNode<HeapObject> AllocateRawUnaligned(TNode<IntPtrT> size_in_bytes,
+ AllocationFlags flags,
+ TNode<RawPtrT> top_address,
+ TNode<RawPtrT> limit_address);
+ TNode<HeapObject> AllocateRaw(TNode<IntPtrT> size_in_bytes,
+ AllocationFlags flags,
+ TNode<RawPtrT> top_address,
+ TNode<RawPtrT> limit_address);
+
+ // Allocate and return a JSArray of given total size in bytes with header
+ // fields initialized.
+ TNode<JSArray> AllocateUninitializedJSArray(
+ TNode<Map> array_map, TNode<Smi> length,
+ base::Optional<TNode<AllocationSite>> allocation_site,
+ TNode<IntPtrT> size_in_bytes);
+
+ TNode<IntPtrT> SmiShiftBitsConstant() {
+ return IntPtrConstant(kSmiShiftSize + kSmiTagSize);
+ }
+ TNode<Int32T> SmiShiftBitsConstant32() {
+ return Int32Constant(kSmiShiftSize + kSmiTagSize);
+ }
+
+ TNode<String> AllocateSlicedString(RootIndex map_root_index,
+ TNode<Uint32T> length,
+ TNode<String> parent, TNode<Smi> offset);
+
+ // Implements [Descriptor/Transition]Array::number_of_entries.
+ template <typename Array>
+ TNode<Uint32T> NumberOfEntries(TNode<Array> array);
+
+ // Implements [Descriptor/Transition]Array::GetSortedKeyIndex.
+ template <typename Array>
+ TNode<Uint32T> GetSortedKeyIndex(TNode<Array> descriptors,
+ TNode<Uint32T> entry_index);
+
+ TNode<Smi> CollectFeedbackForString(SloppyTNode<Int32T> instance_type);
+ void GenerateEqual_Same(SloppyTNode<Object> value, Label* if_equal,
+ Label* if_notequal,
+ TVariable<Smi>* var_type_feedback = nullptr);
+
+ static const int kElementLoopUnrollThreshold = 8;
+
+ // {convert_bigint} is only meaningful when {mode} == kToNumber.
+ TNode<Numeric> NonNumberToNumberOrNumeric(
+ TNode<Context> context, TNode<HeapObject> input, Object::Conversion mode,
+ BigIntHandling bigint_handling = BigIntHandling::kThrow);
+
+ void TaggedToNumeric(TNode<Context> context, TNode<Object> value,
+ TVariable<Numeric>* var_numeric,
+ TVariable<Smi>* var_feedback);
+
+ template <Object::Conversion conversion>
+ void TaggedToWord32OrBigIntImpl(TNode<Context> context, TNode<Object> value,
+ Label* if_number,
+ TVariable<Word32T>* var_word32,
+ Label* if_bigint = nullptr,
+ TVariable<BigInt>* var_maybe_bigint = nullptr,
+ TVariable<Smi>* var_feedback = nullptr);
+
+ // Low-level accessors for Descriptor arrays.
+ template <typename T>
+ TNode<T> LoadDescriptorArrayElement(TNode<DescriptorArray> object,
+ TNode<IntPtrT> index,
+ int additional_offset);
+
+ // Hide LoadRoot for subclasses of CodeStubAssembler. If you get an error
+ // complaining about this method, don't make it public, add your root to
+ // HEAP_(IM)MUTABLE_IMMOVABLE_OBJECT_LIST instead. If you *really* need
+ // LoadRoot, use CodeAssembler::LoadRoot.
+ TNode<Object> LoadRoot(RootIndex root_index) {
+ return CodeAssembler::LoadRoot(root_index);
+ }
+
+ template <typename TIndex>
+ void StoreFixedArrayOrPropertyArrayElement(
+ TNode<UnionT<FixedArray, PropertyArray>> array, TNode<TIndex> index,
+ TNode<Object> value, WriteBarrierMode barrier_mode = UPDATE_WRITE_BARRIER,
+ int additional_offset = 0);
+
+ // Store value to an elements array with given elements kind.
+ // TODO(turbofan): For BIGINT64_ELEMENTS and BIGUINT64_ELEMENTS
+ // we pass {value} as BigInt object instead of int64_t. We should
+ // teach TurboFan to handle int64_t on 32-bit platforms eventually.
+ // TODO(solanes): This method can go away and simplify into only one version
+ // of StoreElement once we have "if constexpr" available to use.
+ template <typename TArray, typename TIndex>
+ void StoreElementBigIntOrTypedArray(TNode<TArray> elements, ElementsKind kind,
+ TNode<TIndex> index, Node* value);
+
+ template <typename TIndex>
+ void StoreElement(TNode<FixedArrayBase> elements, ElementsKind kind,
+ TNode<TIndex> index, Node* value);
+
+ // Converts {input} to a number if {input} is a plain primitve (i.e. String or
+ // Oddball) and stores the result in {var_result}. Otherwise, it bails out to
+ // {if_bailout}.
+ void TryPlainPrimitiveNonNumberToNumber(TNode<HeapObject> input,
+ TVariable<Number>* var_result,
+ Label* if_bailout);
+};
+
+class V8_EXPORT_PRIVATE CodeStubArguments {
+ public:
+ using Node = compiler::Node;
+
+ // |argc| specifies the number of arguments passed to the builtin excluding
+ // the receiver. The arguments include the receiver.
+ CodeStubArguments(CodeStubAssembler* assembler, TNode<IntPtrT> argc)
+ : CodeStubArguments(assembler, argc, TNode<RawPtrT>()) {}
+ CodeStubArguments(CodeStubAssembler* assembler, TNode<Int32T> argc)
+ : CodeStubArguments(assembler, assembler->ChangeInt32ToIntPtr(argc)) {}
+ CodeStubArguments(CodeStubAssembler* assembler, TNode<IntPtrT> argc,
+ TNode<RawPtrT> fp);
+
+ // Used by Torque to construct arguments based on a Torque-defined
+ // struct of values.
+ CodeStubArguments(CodeStubAssembler* assembler,
+ TorqueStructArguments torque_arguments)
+ : assembler_(assembler),
+ argc_(torque_arguments.length),
+ base_(torque_arguments.base),
+ fp_(torque_arguments.frame) {}
+
+ TNode<Object> GetReceiver() const;
+ // Replaces receiver argument on the expression stack. Should be used only
+ // for manipulating arguments in trampoline builtins before tail calling
+ // further with passing all the JS arguments as is.
+ void SetReceiver(TNode<Object> object) const;
+
+ // Computes address of the index'th argument.
+ TNode<RawPtrT> AtIndexPtr(TNode<IntPtrT> index) const;
+
+ // |index| is zero-based and does not include the receiver
+ TNode<Object> AtIndex(TNode<IntPtrT> index) const;
+ TNode<Object> AtIndex(int index) const;
+
+ TNode<IntPtrT> GetLength() const { return argc_; }
+
+ TorqueStructArguments GetTorqueArguments() const {
+ return TorqueStructArguments{fp_, base_, argc_};
+ }
+
+ TNode<Object> GetOptionalArgumentValue(TNode<IntPtrT> index,
+ TNode<Object> default_value);
+ TNode<Object> GetOptionalArgumentValue(TNode<IntPtrT> index) {
+ return GetOptionalArgumentValue(index, assembler_->UndefinedConstant());
+ }
+ TNode<Object> GetOptionalArgumentValue(int index) {
+ return GetOptionalArgumentValue(assembler_->IntPtrConstant(index));
+ }
+
+ // Iteration doesn't include the receiver. |first| and |last| are zero-based.
+ using ForEachBodyFunction = std::function<void(TNode<Object> arg)>;
+ void ForEach(const ForEachBodyFunction& body, TNode<IntPtrT> first = {},
+ TNode<IntPtrT> last = {}) const {
+ CodeStubAssembler::VariableList list(0, assembler_->zone());
+ ForEach(list, body, first, last);
+ }
+ void ForEach(const CodeStubAssembler::VariableList& vars,
+ const ForEachBodyFunction& body, TNode<IntPtrT> first = {},
+ TNode<IntPtrT> last = {}) const;
+
+ void PopAndReturn(TNode<Object> value);
+
+ private:
+ CodeStubAssembler* assembler_;
+ TNode<IntPtrT> argc_;
+ TNode<RawPtrT> base_;
+ TNode<RawPtrT> fp_;
+};
+
+class ToDirectStringAssembler : public CodeStubAssembler {
+ private:
+ enum StringPointerKind { PTR_TO_DATA, PTR_TO_STRING };
+
+ public:
+ enum Flag {
+ kDontUnpackSlicedStrings = 1 << 0,
+ };
+ using Flags = base::Flags<Flag>;
+
+ ToDirectStringAssembler(compiler::CodeAssemblerState* state,
+ TNode<String> string, Flags flags = Flags());
+
+ // Converts flat cons, thin, and sliced strings and returns the direct
+ // string. The result can be either a sequential or external string.
+ // Jumps to if_bailout if the string if the string is indirect and cannot
+ // be unpacked.
+ TNode<String> TryToDirect(Label* if_bailout);
+
+ // Returns a pointer to the beginning of the string data.
+ // Jumps to if_bailout if the external string cannot be unpacked.
+ TNode<RawPtrT> PointerToData(Label* if_bailout) {
+ return TryToSequential(PTR_TO_DATA, if_bailout);
+ }
+
+ // Returns a pointer that, offset-wise, looks like a String.
+ // Jumps to if_bailout if the external string cannot be unpacked.
+ TNode<RawPtrT> PointerToString(Label* if_bailout) {
+ return TryToSequential(PTR_TO_STRING, if_bailout);
+ }
+
+ TNode<String> string() { return var_string_.value(); }
+ TNode<Int32T> instance_type() { return var_instance_type_.value(); }
+ TNode<IntPtrT> offset() { return var_offset_.value(); }
+ TNode<Word32T> is_external() { return var_is_external_.value(); }
+
+ private:
+ TNode<RawPtrT> TryToSequential(StringPointerKind ptr_kind, Label* if_bailout);
+
+ TVariable<String> var_string_;
+ TVariable<Int32T> var_instance_type_;
+ // TODO(v8:9880): Use UintPtrT here.
+ TVariable<IntPtrT> var_offset_;
+ TVariable<Word32T> var_is_external_;
+
+ const Flags flags_;
+};
+
+// Performs checks on a given prototype (e.g. map identity, property
+// verification), intended for use in fast path checks.
+class PrototypeCheckAssembler : public CodeStubAssembler {
+ public:
+ enum Flag {
+ kCheckPrototypePropertyConstness = 1 << 0,
+ kCheckPrototypePropertyIdentity = 1 << 1,
+ kCheckFull =
+ kCheckPrototypePropertyConstness | kCheckPrototypePropertyIdentity,
+ };
+ using Flags = base::Flags<Flag>;
+
+ // A tuple describing a relevant property. It contains the descriptor index of
+ // the property (within the descriptor array), the property's expected name
+ // (stored as a root), and the property's expected value (stored on the native
+ // context).
+ struct DescriptorIndexNameValue {
+ int descriptor_index;
+ RootIndex name_root_index;
+ int expected_value_context_index;
+ };
+
+ PrototypeCheckAssembler(compiler::CodeAssemblerState* state, Flags flags,
+ TNode<NativeContext> native_context,
+ TNode<Map> initial_prototype_map,
+ Vector<DescriptorIndexNameValue> properties);
+
+ void CheckAndBranch(TNode<HeapObject> prototype, Label* if_unmodified,
+ Label* if_modified);
+
+ private:
+ const Flags flags_;
+ const TNode<NativeContext> native_context_;
+ const TNode<Map> initial_prototype_map_;
+ const Vector<DescriptorIndexNameValue> properties_;
+};
+
+DEFINE_OPERATORS_FOR_FLAGS(CodeStubAssembler::AllocationFlags)
+
+#define CLASS_MAP_CONSTANT_ADAPTER(V, rootIndexName, rootAccessorName, \
+ class_name) \
+ template <> \
+ inline bool CodeStubAssembler::ClassHasMapConstant<class_name>() { \
+ return true; \
+ } \
+ template <> \
+ inline TNode<Map> CodeStubAssembler::GetClassMapConstant<class_name>() { \
+ return class_name##MapConstant(); \
+ }
+
+UNIQUE_INSTANCE_TYPE_MAP_LIST_GENERATOR(CLASS_MAP_CONSTANT_ADAPTER, _)
+
+} // namespace internal
+} // namespace v8
+#endif // V8_CODEGEN_CODE_STUB_ASSEMBLER_H_
diff --git a/src/codegen/compilation-cache.cc b/src/codegen/compilation-cache.cc
new file mode 100644
index 0000000..f5e9bb8
--- /dev/null
+++ b/src/codegen/compilation-cache.cc
@@ -0,0 +1,454 @@
+// Copyright 2011 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/compilation-cache.h"
+
+#include "src/common/globals.h"
+#include "src/heap/factory.h"
+#include "src/logging/counters.h"
+#include "src/logging/log.h"
+#include "src/objects/compilation-cache-table-inl.h"
+#include "src/objects/objects-inl.h"
+#include "src/objects/slots.h"
+#include "src/objects/visitors.h"
+#include "src/utils/ostreams.h"
+
+namespace v8 {
+namespace internal {
+
+// The number of generations for each sub cache.
+static const int kRegExpGenerations = 2;
+
+// Initial size of each compilation cache table allocated.
+static const int kInitialCacheSize = 64;
+
+CompilationCache::CompilationCache(Isolate* isolate)
+ : isolate_(isolate),
+ script_(isolate),
+ eval_global_(isolate),
+ eval_contextual_(isolate),
+ reg_exp_(isolate, kRegExpGenerations),
+ code_(isolate),
+ enabled_script_and_eval_(true) {
+ CompilationSubCache* subcaches[kSubCacheCount] = {
+ &script_, &eval_global_, &eval_contextual_, ®_exp_, &code_};
+ for (int i = 0; i < kSubCacheCount; ++i) {
+ subcaches_[i] = subcaches[i];
+ }
+}
+
+Handle<CompilationCacheTable> CompilationSubCache::GetTable(int generation) {
+ DCHECK_LT(generation, generations());
+ Handle<CompilationCacheTable> result;
+ if (tables_[generation].IsUndefined(isolate())) {
+ result = CompilationCacheTable::New(isolate(), kInitialCacheSize);
+ tables_[generation] = *result;
+ } else {
+ CompilationCacheTable table =
+ CompilationCacheTable::cast(tables_[generation]);
+ result = Handle<CompilationCacheTable>(table, isolate());
+ }
+ return result;
+}
+
+// static
+void CompilationSubCache::AgeByGeneration(CompilationSubCache* c) {
+ DCHECK_GT(c->generations(), 1);
+
+ // Age the generations implicitly killing off the oldest.
+ for (int i = c->generations() - 1; i > 0; i--) {
+ c->tables_[i] = c->tables_[i - 1];
+ }
+
+ // Set the first generation as unborn.
+ c->tables_[0] = ReadOnlyRoots(c->isolate()).undefined_value();
+}
+
+// static
+void CompilationSubCache::AgeCustom(CompilationSubCache* c) {
+ DCHECK_EQ(c->generations(), 1);
+ if (c->tables_[0].IsUndefined(c->isolate())) return;
+ CompilationCacheTable::cast(c->tables_[0]).Age();
+}
+
+void CompilationCacheScript::Age() {
+ if (FLAG_isolate_script_cache_ageing) AgeCustom(this);
+}
+void CompilationCacheEval::Age() { AgeCustom(this); }
+void CompilationCacheRegExp::Age() { AgeByGeneration(this); }
+void CompilationCacheCode::Age() {
+ if (FLAG_turbo_nci_cache_ageing) {
+ if (FLAG_trace_turbo_nci) CompilationCacheCode::TraceAgeing();
+ AgeByGeneration(this);
+ }
+}
+
+void CompilationSubCache::Iterate(RootVisitor* v) {
+ v->VisitRootPointers(Root::kCompilationCache, nullptr,
+ FullObjectSlot(&tables_[0]),
+ FullObjectSlot(&tables_[generations()]));
+}
+
+void CompilationSubCache::Clear() {
+ MemsetPointer(reinterpret_cast<Address*>(tables_),
+ ReadOnlyRoots(isolate()).undefined_value().ptr(),
+ generations());
+}
+
+void CompilationSubCache::Remove(Handle<SharedFunctionInfo> function_info) {
+ // Probe the script generation tables. Make sure not to leak handles
+ // into the caller's handle scope.
+ {
+ HandleScope scope(isolate());
+ for (int generation = 0; generation < generations(); generation++) {
+ Handle<CompilationCacheTable> table = GetTable(generation);
+ table->Remove(*function_info);
+ }
+ }
+}
+
+CompilationCacheScript::CompilationCacheScript(Isolate* isolate)
+ : CompilationSubCache(isolate, 1) {}
+
+// We only re-use a cached function for some script source code if the
+// script originates from the same place. This is to avoid issues
+// when reporting errors, etc.
+bool CompilationCacheScript::HasOrigin(Handle<SharedFunctionInfo> function_info,
+ MaybeHandle<Object> maybe_name,
+ int line_offset, int column_offset,
+ ScriptOriginOptions resource_options) {
+ Handle<Script> script =
+ Handle<Script>(Script::cast(function_info->script()), isolate());
+ // If the script name isn't set, the boilerplate script should have
+ // an undefined name to have the same origin.
+ Handle<Object> name;
+ if (!maybe_name.ToHandle(&name)) {
+ return script->name().IsUndefined(isolate());
+ }
+ // Do the fast bailout checks first.
+ if (line_offset != script->line_offset()) return false;
+ if (column_offset != script->column_offset()) return false;
+ // Check that both names are strings. If not, no match.
+ if (!name->IsString() || !script->name().IsString()) return false;
+ // Are the origin_options same?
+ if (resource_options.Flags() != script->origin_options().Flags())
+ return false;
+ // Compare the two name strings for equality.
+ return String::Equals(
+ isolate(), Handle<String>::cast(name),
+ Handle<String>(String::cast(script->name()), isolate()));
+}
+
+// TODO(245): Need to allow identical code from different contexts to
+// be cached in the same script generation. Currently the first use
+// will be cached, but subsequent code from different source / line
+// won't.
+MaybeHandle<SharedFunctionInfo> CompilationCacheScript::Lookup(
+ Handle<String> source, MaybeHandle<Object> name, int line_offset,
+ int column_offset, ScriptOriginOptions resource_options,
+ Handle<Context> native_context, LanguageMode language_mode) {
+ MaybeHandle<SharedFunctionInfo> result;
+
+ // Probe the script generation tables. Make sure not to leak handles
+ // into the caller's handle scope.
+ {
+ HandleScope scope(isolate());
+ const int generation = 0;
+ DCHECK_EQ(generations(), 1);
+ Handle<CompilationCacheTable> table = GetTable(generation);
+ MaybeHandle<SharedFunctionInfo> probe = CompilationCacheTable::LookupScript(
+ table, source, native_context, language_mode);
+ Handle<SharedFunctionInfo> function_info;
+ if (probe.ToHandle(&function_info)) {
+ // Break when we've found a suitable shared function info that
+ // matches the origin.
+ if (HasOrigin(function_info, name, line_offset, column_offset,
+ resource_options)) {
+ result = scope.CloseAndEscape(function_info);
+ }
+ }
+ }
+
+ // Once outside the manacles of the handle scope, we need to recheck
+ // to see if we actually found a cached script. If so, we return a
+ // handle created in the caller's handle scope.
+ Handle<SharedFunctionInfo> function_info;
+ if (result.ToHandle(&function_info)) {
+#ifdef DEBUG
+ // Since HasOrigin can allocate, we need to protect the SharedFunctionInfo
+ // with handles during the call.
+ DCHECK(HasOrigin(function_info, name, line_offset, column_offset,
+ resource_options));
+#endif
+ isolate()->counters()->compilation_cache_hits()->Increment();
+ LOG(isolate(), CompilationCacheEvent("hit", "script", *function_info));
+ } else {
+ isolate()->counters()->compilation_cache_misses()->Increment();
+ }
+ return result;
+}
+
+void CompilationCacheScript::Put(Handle<String> source,
+ Handle<Context> native_context,
+ LanguageMode language_mode,
+ Handle<SharedFunctionInfo> function_info) {
+ HandleScope scope(isolate());
+ Handle<CompilationCacheTable> table = GetFirstTable();
+ SetFirstTable(CompilationCacheTable::PutScript(table, source, native_context,
+ language_mode, function_info));
+}
+
+InfoCellPair CompilationCacheEval::Lookup(Handle<String> source,
+ Handle<SharedFunctionInfo> outer_info,
+ Handle<Context> native_context,
+ LanguageMode language_mode,
+ int position) {
+ HandleScope scope(isolate());
+ // Make sure not to leak the table into the surrounding handle
+ // scope. Otherwise, we risk keeping old tables around even after
+ // having cleared the cache.
+ InfoCellPair result;
+ const int generation = 0;
+ DCHECK_EQ(generations(), 1);
+ Handle<CompilationCacheTable> table = GetTable(generation);
+ result = CompilationCacheTable::LookupEval(
+ table, source, outer_info, native_context, language_mode, position);
+ if (result.has_shared()) {
+ isolate()->counters()->compilation_cache_hits()->Increment();
+ } else {
+ isolate()->counters()->compilation_cache_misses()->Increment();
+ }
+ return result;
+}
+
+void CompilationCacheEval::Put(Handle<String> source,
+ Handle<SharedFunctionInfo> outer_info,
+ Handle<SharedFunctionInfo> function_info,
+ Handle<Context> native_context,
+ Handle<FeedbackCell> feedback_cell,
+ int position) {
+ HandleScope scope(isolate());
+ Handle<CompilationCacheTable> table = GetFirstTable();
+ table =
+ CompilationCacheTable::PutEval(table, source, outer_info, function_info,
+ native_context, feedback_cell, position);
+ SetFirstTable(table);
+}
+
+MaybeHandle<FixedArray> CompilationCacheRegExp::Lookup(Handle<String> source,
+ JSRegExp::Flags flags) {
+ HandleScope scope(isolate());
+ // Make sure not to leak the table into the surrounding handle
+ // scope. Otherwise, we risk keeping old tables around even after
+ // having cleared the cache.
+ Handle<Object> result = isolate()->factory()->undefined_value();
+ int generation;
+ for (generation = 0; generation < generations(); generation++) {
+ Handle<CompilationCacheTable> table = GetTable(generation);
+ result = table->LookupRegExp(source, flags);
+ if (result->IsFixedArray()) break;
+ }
+ if (result->IsFixedArray()) {
+ Handle<FixedArray> data = Handle<FixedArray>::cast(result);
+ if (generation != 0) {
+ Put(source, flags, data);
+ }
+ isolate()->counters()->compilation_cache_hits()->Increment();
+ return scope.CloseAndEscape(data);
+ } else {
+ isolate()->counters()->compilation_cache_misses()->Increment();
+ return MaybeHandle<FixedArray>();
+ }
+}
+
+void CompilationCacheRegExp::Put(Handle<String> source, JSRegExp::Flags flags,
+ Handle<FixedArray> data) {
+ HandleScope scope(isolate());
+ Handle<CompilationCacheTable> table = GetFirstTable();
+ SetFirstTable(
+ CompilationCacheTable::PutRegExp(isolate(), table, source, flags, data));
+}
+
+MaybeHandle<Code> CompilationCacheCode::Lookup(Handle<SharedFunctionInfo> key) {
+ // Make sure not to leak the table into the surrounding handle
+ // scope. Otherwise, we risk keeping old tables around even after
+ // having cleared the cache.
+ HandleScope scope(isolate());
+ MaybeHandle<Code> maybe_value;
+ int generation = 0;
+ for (; generation < generations(); generation++) {
+ Handle<CompilationCacheTable> table = GetTable(generation);
+ maybe_value = table->LookupCode(key);
+ if (!maybe_value.is_null()) break;
+ }
+
+ if (maybe_value.is_null()) {
+ isolate()->counters()->compilation_cache_misses()->Increment();
+ return MaybeHandle<Code>();
+ }
+
+ Handle<Code> value = maybe_value.ToHandleChecked();
+ if (generation != 0) Put(key, value); // Add to the first generation.
+ isolate()->counters()->compilation_cache_hits()->Increment();
+ return scope.CloseAndEscape(value);
+}
+
+void CompilationCacheCode::Put(Handle<SharedFunctionInfo> key,
+ Handle<Code> value) {
+ HandleScope scope(isolate());
+ Handle<CompilationCacheTable> table = GetFirstTable();
+ SetFirstTable(CompilationCacheTable::PutCode(isolate(), table, key, value));
+}
+
+void CompilationCacheCode::TraceAgeing() {
+ DCHECK(FLAG_trace_turbo_nci);
+ StdoutStream os;
+ os << "NCI cache ageing: Removing oldest generation" << std::endl;
+}
+
+void CompilationCacheCode::TraceInsertion(Handle<SharedFunctionInfo> key,
+ Handle<Code> value) {
+ DCHECK(FLAG_trace_turbo_nci);
+ StdoutStream os;
+ os << "NCI cache insertion: " << Brief(*key) << ", " << Brief(*value)
+ << std::endl;
+}
+
+void CompilationCacheCode::TraceHit(Handle<SharedFunctionInfo> key,
+ Handle<Code> value) {
+ DCHECK(FLAG_trace_turbo_nci);
+ StdoutStream os;
+ os << "NCI cache hit: " << Brief(*key) << ", " << Brief(*value) << std::endl;
+}
+
+void CompilationCache::Remove(Handle<SharedFunctionInfo> function_info) {
+ if (!IsEnabledScriptAndEval()) return;
+
+ eval_global_.Remove(function_info);
+ eval_contextual_.Remove(function_info);
+ script_.Remove(function_info);
+}
+
+MaybeHandle<SharedFunctionInfo> CompilationCache::LookupScript(
+ Handle<String> source, MaybeHandle<Object> name, int line_offset,
+ int column_offset, ScriptOriginOptions resource_options,
+ Handle<Context> native_context, LanguageMode language_mode) {
+ if (!IsEnabledScriptAndEval()) return MaybeHandle<SharedFunctionInfo>();
+
+ return script_.Lookup(source, name, line_offset, column_offset,
+ resource_options, native_context, language_mode);
+}
+
+InfoCellPair CompilationCache::LookupEval(Handle<String> source,
+ Handle<SharedFunctionInfo> outer_info,
+ Handle<Context> context,
+ LanguageMode language_mode,
+ int position) {
+ InfoCellPair result;
+ if (!IsEnabledScriptAndEval()) return result;
+
+ const char* cache_type;
+
+ if (context->IsNativeContext()) {
+ result = eval_global_.Lookup(source, outer_info, context, language_mode,
+ position);
+ cache_type = "eval-global";
+
+ } else {
+ DCHECK_NE(position, kNoSourcePosition);
+ Handle<Context> native_context(context->native_context(), isolate());
+ result = eval_contextual_.Lookup(source, outer_info, native_context,
+ language_mode, position);
+ cache_type = "eval-contextual";
+ }
+
+ if (result.has_shared()) {
+ LOG(isolate(), CompilationCacheEvent("hit", cache_type, result.shared()));
+ }
+
+ return result;
+}
+
+MaybeHandle<FixedArray> CompilationCache::LookupRegExp(Handle<String> source,
+ JSRegExp::Flags flags) {
+ return reg_exp_.Lookup(source, flags);
+}
+
+MaybeHandle<Code> CompilationCache::LookupCode(Handle<SharedFunctionInfo> sfi) {
+ return code_.Lookup(sfi);
+}
+
+void CompilationCache::PutScript(Handle<String> source,
+ Handle<Context> native_context,
+ LanguageMode language_mode,
+ Handle<SharedFunctionInfo> function_info) {
+ if (!IsEnabledScriptAndEval()) return;
+ LOG(isolate(), CompilationCacheEvent("put", "script", *function_info));
+
+ script_.Put(source, native_context, language_mode, function_info);
+}
+
+void CompilationCache::PutEval(Handle<String> source,
+ Handle<SharedFunctionInfo> outer_info,
+ Handle<Context> context,
+ Handle<SharedFunctionInfo> function_info,
+ Handle<FeedbackCell> feedback_cell,
+ int position) {
+ if (!IsEnabledScriptAndEval()) return;
+
+ const char* cache_type;
+ HandleScope scope(isolate());
+ if (context->IsNativeContext()) {
+ eval_global_.Put(source, outer_info, function_info, context, feedback_cell,
+ position);
+ cache_type = "eval-global";
+ } else {
+ DCHECK_NE(position, kNoSourcePosition);
+ Handle<Context> native_context(context->native_context(), isolate());
+ eval_contextual_.Put(source, outer_info, function_info, native_context,
+ feedback_cell, position);
+ cache_type = "eval-contextual";
+ }
+ LOG(isolate(), CompilationCacheEvent("put", cache_type, *function_info));
+}
+
+void CompilationCache::PutRegExp(Handle<String> source, JSRegExp::Flags flags,
+ Handle<FixedArray> data) {
+ reg_exp_.Put(source, flags, data);
+}
+
+void CompilationCache::PutCode(Handle<SharedFunctionInfo> shared,
+ Handle<Code> code) {
+ code_.Put(shared, code);
+}
+
+void CompilationCache::Clear() {
+ for (int i = 0; i < kSubCacheCount; i++) {
+ subcaches_[i]->Clear();
+ }
+}
+
+void CompilationCache::Iterate(RootVisitor* v) {
+ for (int i = 0; i < kSubCacheCount; i++) {
+ subcaches_[i]->Iterate(v);
+ }
+}
+
+void CompilationCache::MarkCompactPrologue() {
+ for (int i = 0; i < kSubCacheCount; i++) {
+ subcaches_[i]->Age();
+ }
+}
+
+void CompilationCache::EnableScriptAndEval() {
+ enabled_script_and_eval_ = true;
+}
+
+void CompilationCache::DisableScriptAndEval() {
+ enabled_script_and_eval_ = false;
+ Clear();
+}
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/compilation-cache.h b/src/codegen/compilation-cache.h
new file mode 100644
index 0000000..56d90a3
--- /dev/null
+++ b/src/codegen/compilation-cache.h
@@ -0,0 +1,288 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_COMPILATION_CACHE_H_
+#define V8_CODEGEN_COMPILATION_CACHE_H_
+
+#include "src/base/hashmap.h"
+#include "src/objects/compilation-cache-table.h"
+#include "src/utils/allocation.h"
+
+namespace v8 {
+namespace internal {
+
+template <typename T>
+class Handle;
+
+class RootVisitor;
+
+// The compilation cache consists of several generational sub-caches which uses
+// this class as a base class. A sub-cache contains a compilation cache tables
+// for each generation of the sub-cache. Since the same source code string has
+// different compiled code for scripts and evals, we use separate sub-caches
+// for different compilation modes, to avoid retrieving the wrong result.
+class CompilationSubCache {
+ public:
+ CompilationSubCache(Isolate* isolate, int generations)
+ : isolate_(isolate), generations_(generations) {
+ DCHECK_LE(generations, kMaxGenerations);
+ }
+
+ static constexpr int kFirstGeneration = 0;
+ static constexpr int kMaxGenerations = 2;
+
+ // Get the compilation cache tables for a specific generation.
+ Handle<CompilationCacheTable> GetTable(int generation);
+
+ // Accessors for first generation.
+ Handle<CompilationCacheTable> GetFirstTable() {
+ return GetTable(kFirstGeneration);
+ }
+ void SetFirstTable(Handle<CompilationCacheTable> value) {
+ DCHECK_LT(kFirstGeneration, generations_);
+ tables_[kFirstGeneration] = *value;
+ }
+
+ // Age the sub-cache by evicting the oldest generation and creating a new
+ // young generation.
+ virtual void Age() = 0;
+
+ // GC support.
+ void Iterate(RootVisitor* v);
+
+ // Clear this sub-cache evicting all its content.
+ void Clear();
+
+ // Remove given shared function info from sub-cache.
+ void Remove(Handle<SharedFunctionInfo> function_info);
+
+ // Number of generations in this sub-cache.
+ int generations() const { return generations_; }
+
+ protected:
+ Isolate* isolate() const { return isolate_; }
+
+ // Ageing occurs either by removing the oldest generation, or with
+ // custom logic implemented in CompilationCacheTable::Age.
+ static void AgeByGeneration(CompilationSubCache* c);
+ static void AgeCustom(CompilationSubCache* c);
+
+ private:
+ Isolate* const isolate_;
+ const int generations_;
+ Object tables_[kMaxGenerations]; // One for each generation.
+
+ DISALLOW_IMPLICIT_CONSTRUCTORS(CompilationSubCache);
+};
+
+// Sub-cache for scripts.
+class CompilationCacheScript : public CompilationSubCache {
+ public:
+ explicit CompilationCacheScript(Isolate* isolate);
+
+ MaybeHandle<SharedFunctionInfo> Lookup(Handle<String> source,
+ MaybeHandle<Object> name,
+ int line_offset, int column_offset,
+ ScriptOriginOptions resource_options,
+ Handle<Context> native_context,
+ LanguageMode language_mode);
+
+ void Put(Handle<String> source, Handle<Context> context,
+ LanguageMode language_mode,
+ Handle<SharedFunctionInfo> function_info);
+
+ void Age() override;
+
+ private:
+ bool HasOrigin(Handle<SharedFunctionInfo> function_info,
+ MaybeHandle<Object> name, int line_offset, int column_offset,
+ ScriptOriginOptions resource_options);
+
+ DISALLOW_IMPLICIT_CONSTRUCTORS(CompilationCacheScript);
+};
+
+// Sub-cache for eval scripts. Two caches for eval are used. One for eval calls
+// in native contexts and one for eval calls in other contexts. The cache
+// considers the following pieces of information when checking for matching
+// entries:
+// 1. The source string.
+// 2. The shared function info of the calling function.
+// 3. Whether the source should be compiled as strict code or as sloppy code.
+// Note: Currently there are clients of CompileEval that always compile
+// sloppy code even if the calling function is a strict mode function.
+// More specifically these are the CompileString, DebugEvaluate and
+// DebugEvaluateGlobal runtime functions.
+// 4. The start position of the calling scope.
+class CompilationCacheEval : public CompilationSubCache {
+ public:
+ explicit CompilationCacheEval(Isolate* isolate)
+ : CompilationSubCache(isolate, 1) {}
+
+ InfoCellPair Lookup(Handle<String> source,
+ Handle<SharedFunctionInfo> outer_info,
+ Handle<Context> native_context,
+ LanguageMode language_mode, int position);
+
+ void Put(Handle<String> source, Handle<SharedFunctionInfo> outer_info,
+ Handle<SharedFunctionInfo> function_info,
+ Handle<Context> native_context, Handle<FeedbackCell> feedback_cell,
+ int position);
+
+ void Age() override;
+
+ private:
+ DISALLOW_IMPLICIT_CONSTRUCTORS(CompilationCacheEval);
+};
+
+// Sub-cache for regular expressions.
+class CompilationCacheRegExp : public CompilationSubCache {
+ public:
+ CompilationCacheRegExp(Isolate* isolate, int generations)
+ : CompilationSubCache(isolate, generations) {}
+
+ MaybeHandle<FixedArray> Lookup(Handle<String> source, JSRegExp::Flags flags);
+
+ void Put(Handle<String> source, JSRegExp::Flags flags,
+ Handle<FixedArray> data);
+
+ void Age() override;
+
+ private:
+ DISALLOW_IMPLICIT_CONSTRUCTORS(CompilationCacheRegExp);
+};
+
+// Sub-cache for Code objects. All code inserted into this cache must
+// be usable across different native contexts.
+class CompilationCacheCode : public CompilationSubCache {
+ public:
+ explicit CompilationCacheCode(Isolate* isolate)
+ : CompilationSubCache(isolate, kGenerations) {}
+
+ MaybeHandle<Code> Lookup(Handle<SharedFunctionInfo> key);
+ void Put(Handle<SharedFunctionInfo> key, Handle<Code> value);
+
+ void Age() override;
+
+ // TODO(jgruber,v8:8888): For simplicity we use the generational
+ // approach here, but could consider something else (or more
+ // generations) in the future.
+ static constexpr int kGenerations = 2;
+
+ static void TraceAgeing();
+ static void TraceInsertion(Handle<SharedFunctionInfo> key,
+ Handle<Code> value);
+ static void TraceHit(Handle<SharedFunctionInfo> key, Handle<Code> value);
+
+ private:
+ DISALLOW_IMPLICIT_CONSTRUCTORS(CompilationCacheCode);
+};
+
+// The compilation cache keeps shared function infos for compiled
+// scripts and evals. The shared function infos are looked up using
+// the source string as the key. For regular expressions the
+// compilation data is cached.
+class V8_EXPORT_PRIVATE CompilationCache {
+ public:
+ // Finds the script shared function info for a source
+ // string. Returns an empty handle if the cache doesn't contain a
+ // script for the given source string with the right origin.
+ MaybeHandle<SharedFunctionInfo> LookupScript(
+ Handle<String> source, MaybeHandle<Object> name, int line_offset,
+ int column_offset, ScriptOriginOptions resource_options,
+ Handle<Context> native_context, LanguageMode language_mode);
+
+ // Finds the shared function info for a source string for eval in a
+ // given context. Returns an empty handle if the cache doesn't
+ // contain a script for the given source string.
+ InfoCellPair LookupEval(Handle<String> source,
+ Handle<SharedFunctionInfo> outer_info,
+ Handle<Context> context, LanguageMode language_mode,
+ int position);
+
+ // Returns the regexp data associated with the given regexp if it
+ // is in cache, otherwise an empty handle.
+ MaybeHandle<FixedArray> LookupRegExp(Handle<String> source,
+ JSRegExp::Flags flags);
+
+ MaybeHandle<Code> LookupCode(Handle<SharedFunctionInfo> sfi);
+
+ // Associate the (source, kind) pair to the shared function
+ // info. This may overwrite an existing mapping.
+ void PutScript(Handle<String> source, Handle<Context> native_context,
+ LanguageMode language_mode,
+ Handle<SharedFunctionInfo> function_info);
+
+ // Associate the (source, context->closure()->shared(), kind) triple
+ // with the shared function info. This may overwrite an existing mapping.
+ void PutEval(Handle<String> source, Handle<SharedFunctionInfo> outer_info,
+ Handle<Context> context,
+ Handle<SharedFunctionInfo> function_info,
+ Handle<FeedbackCell> feedback_cell, int position);
+
+ // Associate the (source, flags) pair to the given regexp data.
+ // This may overwrite an existing mapping.
+ void PutRegExp(Handle<String> source, JSRegExp::Flags flags,
+ Handle<FixedArray> data);
+
+ void PutCode(Handle<SharedFunctionInfo> shared, Handle<Code> code);
+
+ // Clear the cache - also used to initialize the cache at startup.
+ void Clear();
+
+ // Remove given shared function info from all caches.
+ void Remove(Handle<SharedFunctionInfo> function_info);
+
+ // GC support.
+ void Iterate(RootVisitor* v);
+
+ // Notify the cache that a mark-sweep garbage collection is about to
+ // take place. This is used to retire entries from the cache to
+ // avoid keeping them alive too long without using them.
+ void MarkCompactPrologue();
+
+ // Enable/disable compilation cache. Used by debugger to disable compilation
+ // cache during debugging so that eval and new scripts are always compiled.
+ // TODO(bmeurer, chromium:992277): The RegExp cache cannot be enabled and/or
+ // disabled, since it doesn't affect debugging. However ideally the other
+ // caches should also be always on, even in the presence of the debugger,
+ // but at this point there are too many unclear invariants, and so I decided
+ // to just fix the pressing performance problem for RegExp individually first.
+ void EnableScriptAndEval();
+ void DisableScriptAndEval();
+
+ private:
+ explicit CompilationCache(Isolate* isolate);
+ ~CompilationCache() = default;
+
+ base::HashMap* EagerOptimizingSet();
+
+ bool IsEnabledScriptAndEval() const {
+ return FLAG_compilation_cache && enabled_script_and_eval_;
+ }
+
+ Isolate* isolate() const { return isolate_; }
+
+ Isolate* isolate_;
+
+ CompilationCacheScript script_;
+ CompilationCacheEval eval_global_;
+ CompilationCacheEval eval_contextual_;
+ CompilationCacheRegExp reg_exp_;
+ CompilationCacheCode code_;
+
+ static constexpr int kSubCacheCount = 5;
+ CompilationSubCache* subcaches_[kSubCacheCount];
+
+ // Current enable state of the compilation cache for scripts and eval.
+ bool enabled_script_and_eval_;
+
+ friend class Isolate;
+
+ DISALLOW_COPY_AND_ASSIGN(CompilationCache);
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_COMPILATION_CACHE_H_
diff --git a/src/codegen/compiler.cc b/src/codegen/compiler.cc
new file mode 100644
index 0000000..bb51b3b
--- /dev/null
+++ b/src/codegen/compiler.cc
@@ -0,0 +1,3139 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/compiler.h"
+
+#include <algorithm>
+#include <memory>
+
+#include "src/api/api-inl.h"
+#include "src/asmjs/asm-js.h"
+#include "src/ast/prettyprinter.h"
+#include "src/ast/scopes.h"
+#include "src/base/logging.h"
+#include "src/base/optional.h"
+#include "src/codegen/assembler-inl.h"
+#include "src/codegen/compilation-cache.h"
+#include "src/codegen/optimized-compilation-info.h"
+#include "src/codegen/pending-optimization-table.h"
+#include "src/codegen/unoptimized-compilation-info.h"
+#include "src/common/assert-scope.h"
+#include "src/common/globals.h"
+#include "src/common/message-template.h"
+#include "src/compiler-dispatcher/compiler-dispatcher.h"
+#include "src/compiler-dispatcher/optimizing-compile-dispatcher.h"
+#include "src/compiler/pipeline.h"
+#include "src/debug/debug.h"
+#include "src/debug/liveedit.h"
+#include "src/diagnostics/code-tracer.h"
+#include "src/execution/frames-inl.h"
+#include "src/execution/isolate-inl.h"
+#include "src/execution/isolate.h"
+#include "src/execution/runtime-profiler.h"
+#include "src/execution/vm-state-inl.h"
+#include "src/handles/maybe-handles.h"
+#include "src/heap/heap-inl.h"
+#include "src/heap/local-factory-inl.h"
+#include "src/heap/local-heap-inl.h"
+#include "src/heap/local-heap.h"
+#include "src/init/bootstrapper.h"
+#include "src/interpreter/interpreter.h"
+#include "src/logging/log-inl.h"
+#include "src/objects/feedback-cell-inl.h"
+#include "src/objects/js-function-inl.h"
+#include "src/objects/map.h"
+#include "src/objects/object-list-macros.h"
+#include "src/objects/shared-function-info.h"
+#include "src/objects/string.h"
+#include "src/parsing/parse-info.h"
+#include "src/parsing/parser.h"
+#include "src/parsing/parsing.h"
+#include "src/parsing/pending-compilation-error-handler.h"
+#include "src/parsing/scanner-character-streams.h"
+#include "src/snapshot/code-serializer.h"
+#include "src/utils/ostreams.h"
+#include "src/zone/zone-list-inl.h" // crbug.com/v8/8816
+
+namespace v8 {
+namespace internal {
+
+namespace {
+
+bool IsForNativeContextIndependentCachingOnly(CodeKind kind) {
+ // NCI code is only cached (and not installed on the JSFunction upon
+ // successful compilation), unless the testing-only
+ // FLAG_turbo_nci_as_midtier is enabled.
+ return CodeKindIsNativeContextIndependentJSFunction(kind) &&
+ !FLAG_turbo_nci_as_midtier;
+}
+
+// This predicate is currently needed only because the nci-as-midtier testing
+// configuration is special. A quick summary of compilation configurations:
+//
+// - Turbofan (and currently Turboprop) uses both the optimization marker and
+// the optimized code cache (underneath, the marker and the cache share the same
+// slot on the feedback vector).
+// - Native context independent (NCI) code uses neither the marker nor the
+// cache.
+// - The NCI-as-midtier testing configuration uses the marker, but not the
+// cache.
+//
+// This predicate supports that last case. In the near future, this last case is
+// expected to change s.t. code kinds use the marker iff they use the optimized
+// code cache (details still TBD). In that case, the existing
+// CodeKindIsStoredInOptimizedCodeCache is sufficient and this extra predicate
+// can be removed.
+// TODO(jgruber,rmcilroy,v8:8888): Remove this predicate once that has happened.
+bool UsesOptimizationMarker(CodeKind kind) {
+ return !IsForNativeContextIndependentCachingOnly(kind);
+}
+
+class CompilerTracer : public AllStatic {
+ public:
+ static void PrintTracePrefix(const CodeTracer::Scope& scope,
+ const char* header,
+ OptimizedCompilationInfo* info) {
+ PrintTracePrefix(scope, header, info->closure(), info->code_kind());
+ }
+
+ static void PrintTracePrefix(const CodeTracer::Scope& scope,
+ const char* header, Handle<JSFunction> function,
+ CodeKind code_kind) {
+ PrintF(scope.file(), "[%s ", header);
+ function->ShortPrint(scope.file());
+ PrintF(scope.file(), " (target %s)", CodeKindToString(code_kind));
+ }
+
+ static void PrintTraceSuffix(const CodeTracer::Scope& scope) {
+ PrintF(scope.file(), "]\n");
+ }
+
+ static void TracePrepareJob(Isolate* isolate, OptimizedCompilationInfo* info,
+ const char* compiler_name) {
+ if (!FLAG_trace_opt || !info->IsOptimizing()) return;
+ CodeTracer::Scope scope(isolate->GetCodeTracer());
+ PrintTracePrefix(scope, "compiling method", info);
+ PrintF(scope.file(), " using %s%s", compiler_name,
+ info->is_osr() ? " OSR" : "");
+ PrintTraceSuffix(scope);
+ }
+
+ static void TraceCompilationStats(Isolate* isolate,
+ OptimizedCompilationInfo* info,
+ double ms_creategraph, double ms_optimize,
+ double ms_codegen) {
+ if (!FLAG_trace_opt || !info->IsOptimizing()) return;
+ CodeTracer::Scope scope(isolate->GetCodeTracer());
+ PrintTracePrefix(scope, "optimizing", info);
+ PrintF(scope.file(), " - took %0.3f, %0.3f, %0.3f ms", ms_creategraph,
+ ms_optimize, ms_codegen);
+ PrintTraceSuffix(scope);
+ }
+
+ static void TraceCompletedJob(Isolate* isolate,
+ OptimizedCompilationInfo* info) {
+ if (!FLAG_trace_opt) return;
+ CodeTracer::Scope scope(isolate->GetCodeTracer());
+ PrintTracePrefix(scope, "completed optimizing", info);
+ PrintTraceSuffix(scope);
+ }
+
+ static void TraceAbortedJob(Isolate* isolate,
+ OptimizedCompilationInfo* info) {
+ if (!FLAG_trace_opt) return;
+ CodeTracer::Scope scope(isolate->GetCodeTracer());
+ PrintTracePrefix(scope, "aborted optimizing", info);
+ PrintF(scope.file(), " because: %s",
+ GetBailoutReason(info->bailout_reason()));
+ PrintTraceSuffix(scope);
+ }
+
+ static void TraceOptimizedCodeCacheHit(Isolate* isolate,
+ Handle<JSFunction> function,
+ BailoutId osr_offset,
+ CodeKind code_kind) {
+ if (!FLAG_trace_opt) return;
+ CodeTracer::Scope scope(isolate->GetCodeTracer());
+ PrintTracePrefix(scope, "found optimized code for", function, code_kind);
+ if (!osr_offset.IsNone()) {
+ PrintF(scope.file(), " at OSR AST id %d", osr_offset.ToInt());
+ }
+ PrintTraceSuffix(scope);
+ }
+
+ static void TraceOptimizeForAlwaysOpt(Isolate* isolate,
+ Handle<JSFunction> function,
+ CodeKind code_kind) {
+ if (!FLAG_trace_opt) return;
+ CodeTracer::Scope scope(isolate->GetCodeTracer());
+ PrintTracePrefix(scope, "optimizing", function, code_kind);
+ PrintF(scope.file(), " because --always-opt");
+ PrintTraceSuffix(scope);
+ }
+
+ static void TraceMarkForAlwaysOpt(Isolate* isolate,
+ Handle<JSFunction> function) {
+ if (!FLAG_trace_opt) return;
+ CodeTracer::Scope scope(isolate->GetCodeTracer());
+ PrintF(scope.file(), "[marking ");
+ function->ShortPrint(scope.file());
+ PrintF(scope.file(), " for optimized recompilation because --always-opt");
+ PrintF(scope.file(), "]\n");
+ }
+};
+
+} // namespace
+
+// Helper that times a scoped region and records the elapsed time.
+struct ScopedTimer {
+ explicit ScopedTimer(base::TimeDelta* location) : location_(location) {
+ DCHECK_NOT_NULL(location_);
+ timer_.Start();
+ }
+
+ ~ScopedTimer() { *location_ += timer_.Elapsed(); }
+
+ base::ElapsedTimer timer_;
+ base::TimeDelta* location_;
+};
+
+namespace {
+
+void LogFunctionCompilation(CodeEventListener::LogEventsAndTags tag,
+ Handle<SharedFunctionInfo> shared,
+ Handle<Script> script,
+ Handle<AbstractCode> abstract_code, bool optimizing,
+ double time_taken_ms, Isolate* isolate) {
+ DCHECK(!abstract_code.is_null());
+ DCHECK(!abstract_code.is_identical_to(BUILTIN_CODE(isolate, CompileLazy)));
+
+ // Log the code generation. If source information is available include
+ // script name and line number. Check explicitly whether logging is
+ // enabled as finding the line number is not free.
+ if (!isolate->logger()->is_listening_to_code_events() &&
+ !isolate->is_profiling() && !FLAG_log_function_events &&
+ !isolate->code_event_dispatcher()->IsListeningToCodeEvents()) {
+ return;
+ }
+
+ int line_num = Script::GetLineNumber(script, shared->StartPosition()) + 1;
+ int column_num = Script::GetColumnNumber(script, shared->StartPosition()) + 1;
+ Handle<String> script_name(script->name().IsString()
+ ? String::cast(script->name())
+ : ReadOnlyRoots(isolate).empty_string(),
+ isolate);
+ CodeEventListener::LogEventsAndTags log_tag =
+ Logger::ToNativeByScript(tag, *script);
+ PROFILE(isolate, CodeCreateEvent(log_tag, abstract_code, shared, script_name,
+ line_num, column_num));
+ if (!FLAG_log_function_events) return;
+
+ DisallowHeapAllocation no_gc;
+
+ std::string name = optimizing ? "optimize" : "compile";
+ switch (tag) {
+ case CodeEventListener::EVAL_TAG:
+ name += "-eval";
+ break;
+ case CodeEventListener::SCRIPT_TAG:
+ break;
+ case CodeEventListener::LAZY_COMPILE_TAG:
+ name += "-lazy";
+ break;
+ case CodeEventListener::FUNCTION_TAG:
+ break;
+ default:
+ UNREACHABLE();
+ }
+
+ LOG(isolate, FunctionEvent(name.c_str(), script->id(), time_taken_ms,
+ shared->StartPosition(), shared->EndPosition(),
+ shared->DebugName()));
+}
+
+ScriptOriginOptions OriginOptionsForEval(Object script) {
+ if (!script.IsScript()) return ScriptOriginOptions();
+
+ const auto outer_origin_options = Script::cast(script).origin_options();
+ return ScriptOriginOptions(outer_origin_options.IsSharedCrossOrigin(),
+ outer_origin_options.IsOpaque());
+}
+
+} // namespace
+
+// ----------------------------------------------------------------------------
+// Implementation of UnoptimizedCompilationJob
+
+CompilationJob::Status UnoptimizedCompilationJob::ExecuteJob() {
+ DisallowHeapAccess no_heap_access;
+ // Delegate to the underlying implementation.
+ DCHECK_EQ(state(), State::kReadyToExecute);
+ ScopedTimer t(&time_taken_to_execute_);
+ return UpdateState(ExecuteJobImpl(), State::kReadyToFinalize);
+}
+
+CompilationJob::Status UnoptimizedCompilationJob::FinalizeJob(
+ Handle<SharedFunctionInfo> shared_info, Isolate* isolate) {
+ DCHECK_EQ(ThreadId::Current(), isolate->thread_id());
+ DisallowCodeDependencyChange no_dependency_change;
+ DisallowJavascriptExecution no_js(isolate);
+
+ // Delegate to the underlying implementation.
+ DCHECK_EQ(state(), State::kReadyToFinalize);
+ ScopedTimer t(&time_taken_to_finalize_);
+ return UpdateState(FinalizeJobImpl(shared_info, isolate), State::kSucceeded);
+}
+
+CompilationJob::Status UnoptimizedCompilationJob::FinalizeJob(
+ Handle<SharedFunctionInfo> shared_info, LocalIsolate* isolate) {
+ // Delegate to the underlying implementation.
+ DCHECK_EQ(state(), State::kReadyToFinalize);
+ ScopedTimer t(&time_taken_to_finalize_);
+ return UpdateState(FinalizeJobImpl(shared_info, isolate), State::kSucceeded);
+}
+
+namespace {
+
+void RecordUnoptimizedCompilationStats(Isolate* isolate,
+ Handle<SharedFunctionInfo> shared_info) {
+ int code_size;
+ if (shared_info->HasBytecodeArray()) {
+ code_size = shared_info->GetBytecodeArray().SizeIncludingMetadata();
+ } else {
+ code_size = shared_info->asm_wasm_data().Size();
+ }
+
+ Counters* counters = isolate->counters();
+ // TODO(4280): Rename counters from "baseline" to "unoptimized" eventually.
+ counters->total_baseline_code_size()->Increment(code_size);
+ counters->total_baseline_compile_count()->Increment(1);
+
+ // TODO(5203): Add timers for each phase of compilation.
+ // Also add total time (there's now already timer_ on the base class).
+}
+
+void RecordUnoptimizedFunctionCompilation(
+ Isolate* isolate, CodeEventListener::LogEventsAndTags tag,
+ Handle<SharedFunctionInfo> shared, base::TimeDelta time_taken_to_execute,
+ base::TimeDelta time_taken_to_finalize) {
+ Handle<AbstractCode> abstract_code;
+ if (shared->HasBytecodeArray()) {
+ abstract_code =
+ handle(AbstractCode::cast(shared->GetBytecodeArray()), isolate);
+ } else {
+ DCHECK(shared->HasAsmWasmData());
+ abstract_code =
+ Handle<AbstractCode>::cast(BUILTIN_CODE(isolate, InstantiateAsmJs));
+ }
+
+ double time_taken_ms = time_taken_to_execute.InMillisecondsF() +
+ time_taken_to_finalize.InMillisecondsF();
+
+ Handle<Script> script(Script::cast(shared->script()), isolate);
+ LogFunctionCompilation(tag, shared, script, abstract_code, false,
+ time_taken_ms, isolate);
+}
+
+} // namespace
+
+// ----------------------------------------------------------------------------
+// Implementation of OptimizedCompilationJob
+
+CompilationJob::Status OptimizedCompilationJob::PrepareJob(Isolate* isolate) {
+ DCHECK_EQ(ThreadId::Current(), isolate->thread_id());
+ DisallowJavascriptExecution no_js(isolate);
+ CompilerTracer::TracePrepareJob(isolate, compilation_info(), compiler_name_);
+
+ // Delegate to the underlying implementation.
+ DCHECK_EQ(state(), State::kReadyToPrepare);
+ ScopedTimer t(&time_taken_to_prepare_);
+ return UpdateState(PrepareJobImpl(isolate), State::kReadyToExecute);
+}
+
+CompilationJob::Status OptimizedCompilationJob::ExecuteJob(
+ RuntimeCallStats* stats, LocalIsolate* local_isolate) {
+ DisallowHeapAccess no_heap_access;
+ // Delegate to the underlying implementation.
+ DCHECK_EQ(state(), State::kReadyToExecute);
+ ScopedTimer t(&time_taken_to_execute_);
+ return UpdateState(ExecuteJobImpl(stats, local_isolate),
+ State::kReadyToFinalize);
+}
+
+CompilationJob::Status OptimizedCompilationJob::FinalizeJob(Isolate* isolate) {
+ DCHECK_EQ(ThreadId::Current(), isolate->thread_id());
+ DisallowJavascriptExecution no_js(isolate);
+
+ // Delegate to the underlying implementation.
+ DCHECK_EQ(state(), State::kReadyToFinalize);
+ ScopedTimer t(&time_taken_to_finalize_);
+ return UpdateState(FinalizeJobImpl(isolate), State::kSucceeded);
+}
+
+CompilationJob::Status OptimizedCompilationJob::RetryOptimization(
+ BailoutReason reason) {
+ DCHECK(compilation_info_->IsOptimizing());
+ compilation_info_->RetryOptimization(reason);
+ return UpdateState(FAILED, State::kFailed);
+}
+
+CompilationJob::Status OptimizedCompilationJob::AbortOptimization(
+ BailoutReason reason) {
+ DCHECK(compilation_info_->IsOptimizing());
+ compilation_info_->AbortOptimization(reason);
+ return UpdateState(FAILED, State::kFailed);
+}
+
+void OptimizedCompilationJob::RecordCompilationStats(CompilationMode mode,
+ Isolate* isolate) const {
+ DCHECK(compilation_info()->IsOptimizing());
+ Handle<JSFunction> function = compilation_info()->closure();
+ double ms_creategraph = time_taken_to_prepare_.InMillisecondsF();
+ double ms_optimize = time_taken_to_execute_.InMillisecondsF();
+ double ms_codegen = time_taken_to_finalize_.InMillisecondsF();
+ CompilerTracer::TraceCompilationStats(
+ isolate, compilation_info(), ms_creategraph, ms_optimize, ms_codegen);
+ if (FLAG_trace_opt_stats) {
+ static double compilation_time = 0.0;
+ static int compiled_functions = 0;
+ static int code_size = 0;
+
+ compilation_time += (ms_creategraph + ms_optimize + ms_codegen);
+ compiled_functions++;
+ code_size += function->shared().SourceSize();
+ PrintF("Compiled: %d functions with %d byte source size in %fms.\n",
+ compiled_functions, code_size, compilation_time);
+ }
+ // Don't record samples from machines without high-resolution timers,
+ // as that can cause serious reporting issues. See the thread at
+ // http://g/chrome-metrics-team/NwwJEyL8odU/discussion for more details.
+ if (base::TimeTicks::IsHighResolution()) {
+ Counters* const counters = isolate->counters();
+ if (compilation_info()->is_osr()) {
+ counters->turbofan_osr_prepare()->AddSample(
+ static_cast<int>(time_taken_to_prepare_.InMicroseconds()));
+ counters->turbofan_osr_execute()->AddSample(
+ static_cast<int>(time_taken_to_execute_.InMicroseconds()));
+ counters->turbofan_osr_finalize()->AddSample(
+ static_cast<int>(time_taken_to_finalize_.InMicroseconds()));
+ counters->turbofan_osr_total_time()->AddSample(
+ static_cast<int>(ElapsedTime().InMicroseconds()));
+ } else {
+ counters->turbofan_optimize_prepare()->AddSample(
+ static_cast<int>(time_taken_to_prepare_.InMicroseconds()));
+ counters->turbofan_optimize_execute()->AddSample(
+ static_cast<int>(time_taken_to_execute_.InMicroseconds()));
+ counters->turbofan_optimize_finalize()->AddSample(
+ static_cast<int>(time_taken_to_finalize_.InMicroseconds()));
+ counters->turbofan_optimize_total_time()->AddSample(
+ static_cast<int>(ElapsedTime().InMicroseconds()));
+
+ // Compute foreground / background time.
+ base::TimeDelta time_background;
+ base::TimeDelta time_foreground =
+ time_taken_to_prepare_ + time_taken_to_finalize_;
+ switch (mode) {
+ case OptimizedCompilationJob::kConcurrent:
+ time_background += time_taken_to_execute_;
+ counters->turbofan_optimize_concurrent_total_time()->AddSample(
+ static_cast<int>(ElapsedTime().InMicroseconds()));
+ break;
+ case OptimizedCompilationJob::kSynchronous:
+ counters->turbofan_optimize_non_concurrent_total_time()->AddSample(
+ static_cast<int>(ElapsedTime().InMicroseconds()));
+ time_foreground += time_taken_to_execute_;
+ break;
+ }
+ counters->turbofan_optimize_total_background()->AddSample(
+ static_cast<int>(time_background.InMicroseconds()));
+ counters->turbofan_optimize_total_foreground()->AddSample(
+ static_cast<int>(time_foreground.InMicroseconds()));
+ }
+ counters->turbofan_ticks()->AddSample(static_cast<int>(
+ compilation_info()->tick_counter().CurrentTicks() / 1000));
+ }
+}
+
+void OptimizedCompilationJob::RecordFunctionCompilation(
+ CodeEventListener::LogEventsAndTags tag, Isolate* isolate) const {
+ Handle<AbstractCode> abstract_code =
+ Handle<AbstractCode>::cast(compilation_info()->code());
+
+ double time_taken_ms = time_taken_to_prepare_.InMillisecondsF() +
+ time_taken_to_execute_.InMillisecondsF() +
+ time_taken_to_finalize_.InMillisecondsF();
+
+ Handle<Script> script(
+ Script::cast(compilation_info()->shared_info()->script()), isolate);
+ LogFunctionCompilation(tag, compilation_info()->shared_info(), script,
+ abstract_code, true, time_taken_ms, isolate);
+}
+
+// ----------------------------------------------------------------------------
+// Local helper methods that make up the compilation pipeline.
+
+namespace {
+
+bool UseAsmWasm(FunctionLiteral* literal, bool asm_wasm_broken) {
+ // Check whether asm.js validation is enabled.
+ if (!FLAG_validate_asm) return false;
+
+ // Modules that have validated successfully, but were subsequently broken by
+ // invalid module instantiation attempts are off limit forever.
+ if (asm_wasm_broken) return false;
+
+ // In stress mode we want to run the validator on everything.
+ if (FLAG_stress_validate_asm) return true;
+
+ // In general, we respect the "use asm" directive.
+ return literal->scope()->IsAsmModule();
+}
+
+void InstallInterpreterTrampolineCopy(Isolate* isolate,
+ Handle<SharedFunctionInfo> shared_info) {
+ DCHECK(FLAG_interpreted_frames_native_stack);
+ if (!shared_info->function_data(kAcquireLoad).IsBytecodeArray()) {
+ DCHECK(!shared_info->HasBytecodeArray());
+ return;
+ }
+ Handle<BytecodeArray> bytecode_array(shared_info->GetBytecodeArray(),
+ isolate);
+
+ Handle<Code> code = isolate->factory()->CopyCode(Handle<Code>::cast(
+ isolate->factory()->interpreter_entry_trampoline_for_profiling()));
+
+ Handle<InterpreterData> interpreter_data =
+ Handle<InterpreterData>::cast(isolate->factory()->NewStruct(
+ INTERPRETER_DATA_TYPE, AllocationType::kOld));
+
+ interpreter_data->set_bytecode_array(*bytecode_array);
+ interpreter_data->set_interpreter_trampoline(*code);
+
+ shared_info->set_interpreter_data(*interpreter_data);
+
+ Handle<Script> script(Script::cast(shared_info->script()), isolate);
+ Handle<AbstractCode> abstract_code = Handle<AbstractCode>::cast(code);
+ int line_num =
+ Script::GetLineNumber(script, shared_info->StartPosition()) + 1;
+ int column_num =
+ Script::GetColumnNumber(script, shared_info->StartPosition()) + 1;
+ Handle<String> script_name =
+ handle(script->name().IsString() ? String::cast(script->name())
+ : ReadOnlyRoots(isolate).empty_string(),
+ isolate);
+ CodeEventListener::LogEventsAndTags log_tag = Logger::ToNativeByScript(
+ CodeEventListener::INTERPRETED_FUNCTION_TAG, *script);
+ PROFILE(isolate, CodeCreateEvent(log_tag, abstract_code, shared_info,
+ script_name, line_num, column_num));
+}
+
+template <typename LocalIsolate>
+void InstallUnoptimizedCode(UnoptimizedCompilationInfo* compilation_info,
+ Handle<SharedFunctionInfo> shared_info,
+ LocalIsolate* isolate) {
+ if (compilation_info->has_bytecode_array()) {
+ DCHECK(!shared_info->HasBytecodeArray()); // Only compiled once.
+ DCHECK(!compilation_info->has_asm_wasm_data());
+ DCHECK(!shared_info->HasFeedbackMetadata());
+
+ // If the function failed asm-wasm compilation, mark asm_wasm as broken
+ // to ensure we don't try to compile as asm-wasm.
+ if (compilation_info->literal()->scope()->IsAsmModule()) {
+ shared_info->set_is_asm_wasm_broken(true);
+ }
+
+ shared_info->set_bytecode_array(*compilation_info->bytecode_array());
+
+ Handle<FeedbackMetadata> feedback_metadata = FeedbackMetadata::New(
+ isolate, compilation_info->feedback_vector_spec());
+ shared_info->set_feedback_metadata(*feedback_metadata);
+ } else {
+ DCHECK(compilation_info->has_asm_wasm_data());
+ // We should only have asm/wasm data when finalizing on the main thread.
+ DCHECK((std::is_same<LocalIsolate, Isolate>::value));
+ shared_info->set_asm_wasm_data(*compilation_info->asm_wasm_data());
+ shared_info->set_feedback_metadata(
+ ReadOnlyRoots(isolate).empty_feedback_metadata());
+ }
+}
+
+void LogUnoptimizedCompilation(Isolate* isolate,
+ Handle<SharedFunctionInfo> shared_info,
+ UnoptimizedCompileFlags flags,
+ base::TimeDelta time_taken_to_execute,
+ base::TimeDelta time_taken_to_finalize) {
+ CodeEventListener::LogEventsAndTags log_tag;
+ if (flags.is_toplevel()) {
+ log_tag = flags.is_eval() ? CodeEventListener::EVAL_TAG
+ : CodeEventListener::SCRIPT_TAG;
+ } else {
+ log_tag = flags.is_lazy_compile() ? CodeEventListener::LAZY_COMPILE_TAG
+ : CodeEventListener::FUNCTION_TAG;
+ }
+
+ RecordUnoptimizedFunctionCompilation(isolate, log_tag, shared_info,
+ time_taken_to_execute,
+ time_taken_to_finalize);
+ RecordUnoptimizedCompilationStats(isolate, shared_info);
+}
+
+template <typename LocalIsolate>
+void EnsureSharedFunctionInfosArrayOnScript(Handle<Script> script,
+ ParseInfo* parse_info,
+ LocalIsolate* isolate) {
+ DCHECK(parse_info->flags().is_toplevel());
+ if (script->shared_function_infos().length() > 0) {
+ DCHECK_EQ(script->shared_function_infos().length(),
+ parse_info->max_function_literal_id() + 1);
+ return;
+ }
+ Handle<WeakFixedArray> infos(isolate->factory()->NewWeakFixedArray(
+ parse_info->max_function_literal_id() + 1, AllocationType::kOld));
+ script->set_shared_function_infos(*infos);
+}
+
+void UpdateSharedFunctionFlagsAfterCompilation(FunctionLiteral* literal,
+ SharedFunctionInfo shared_info) {
+ DCHECK_EQ(shared_info.language_mode(), literal->language_mode());
+
+ shared_info.set_has_duplicate_parameters(literal->has_duplicate_parameters());
+ shared_info.set_is_oneshot_iife(literal->is_oneshot_iife());
+ shared_info.UpdateAndFinalizeExpectedNofPropertiesFromEstimate(literal);
+ if (literal->dont_optimize_reason() != BailoutReason::kNoReason) {
+ shared_info.DisableOptimization(literal->dont_optimize_reason());
+ }
+
+ shared_info.set_class_scope_has_private_brand(
+ literal->class_scope_has_private_brand());
+ shared_info.set_has_static_private_methods_or_accessors(
+ literal->has_static_private_methods_or_accessors());
+
+ shared_info.SetScopeInfo(*literal->scope()->scope_info());
+}
+
+// Finalize a single compilation job. This function can return
+// RETRY_ON_MAIN_THREAD if the job cannot be finalized off-thread, in which case
+// it should be safe to call it again on the main thread with the same job.
+template <typename LocalIsolate>
+CompilationJob::Status FinalizeSingleUnoptimizedCompilationJob(
+ UnoptimizedCompilationJob* job, Handle<SharedFunctionInfo> shared_info,
+ LocalIsolate* isolate,
+ FinalizeUnoptimizedCompilationDataList*
+ finalize_unoptimized_compilation_data_list) {
+ UnoptimizedCompilationInfo* compilation_info = job->compilation_info();
+
+ CompilationJob::Status status = job->FinalizeJob(shared_info, isolate);
+ if (status == CompilationJob::SUCCEEDED) {
+ InstallUnoptimizedCode(compilation_info, shared_info, isolate);
+
+ MaybeHandle<CoverageInfo> coverage_info;
+ if (compilation_info->has_coverage_info() &&
+ !shared_info->HasCoverageInfo()) {
+ coverage_info = compilation_info->coverage_info();
+ }
+
+ finalize_unoptimized_compilation_data_list->emplace_back(
+ isolate, shared_info, coverage_info, job->time_taken_to_execute(),
+ job->time_taken_to_finalize());
+ }
+ DCHECK_IMPLIES(
+ status == CompilationJob::RETRY_ON_MAIN_THREAD,
+ (std::is_same<LocalIsolate, v8::internal::LocalIsolate>::value));
+ return status;
+}
+
+std::unique_ptr<UnoptimizedCompilationJob>
+ExecuteSingleUnoptimizedCompilationJob(
+ ParseInfo* parse_info, FunctionLiteral* literal,
+ AccountingAllocator* allocator,
+ std::vector<FunctionLiteral*>* eager_inner_literals) {
+ if (UseAsmWasm(literal, parse_info->flags().is_asm_wasm_broken())) {
+ std::unique_ptr<UnoptimizedCompilationJob> asm_job(
+ AsmJs::NewCompilationJob(parse_info, literal, allocator));
+ if (asm_job->ExecuteJob() == CompilationJob::SUCCEEDED) {
+ return asm_job;
+ }
+ // asm.js validation failed, fall through to standard unoptimized compile.
+ // Note: we rely on the fact that AsmJs jobs have done all validation in the
+ // PrepareJob and ExecuteJob phases and can't fail in FinalizeJob with
+ // with a validation error or another error that could be solve by falling
+ // through to standard unoptimized compile.
+ }
+ std::unique_ptr<UnoptimizedCompilationJob> job(
+ interpreter::Interpreter::NewCompilationJob(
+ parse_info, literal, allocator, eager_inner_literals));
+
+ if (job->ExecuteJob() != CompilationJob::SUCCEEDED) {
+ // Compilation failed, return null.
+ return std::unique_ptr<UnoptimizedCompilationJob>();
+ }
+
+ return job;
+}
+
+bool RecursivelyExecuteUnoptimizedCompilationJobs(
+ ParseInfo* parse_info, FunctionLiteral* literal,
+ AccountingAllocator* allocator,
+ UnoptimizedCompilationJobList* function_jobs) {
+ std::vector<FunctionLiteral*> eager_inner_literals;
+ std::unique_ptr<UnoptimizedCompilationJob> job =
+ ExecuteSingleUnoptimizedCompilationJob(parse_info, literal, allocator,
+ &eager_inner_literals);
+
+ if (!job) return false;
+
+ // Recursively compile eager inner literals.
+ for (FunctionLiteral* inner_literal : eager_inner_literals) {
+ if (!RecursivelyExecuteUnoptimizedCompilationJobs(
+ parse_info, inner_literal, allocator, function_jobs)) {
+ return false;
+ }
+ }
+
+ function_jobs->emplace_front(std::move(job));
+ return true;
+}
+
+template <typename LocalIsolate>
+bool IterativelyExecuteAndFinalizeUnoptimizedCompilationJobs(
+ LocalIsolate* isolate, Handle<SharedFunctionInfo> outer_shared_info,
+ Handle<Script> script, ParseInfo* parse_info,
+ AccountingAllocator* allocator, IsCompiledScope* is_compiled_scope,
+ FinalizeUnoptimizedCompilationDataList*
+ finalize_unoptimized_compilation_data_list,
+ DeferredFinalizationJobDataList*
+ jobs_to_retry_finalization_on_main_thread) {
+ DeclarationScope::AllocateScopeInfos(parse_info, isolate);
+
+ std::vector<FunctionLiteral*> functions_to_compile;
+ functions_to_compile.push_back(parse_info->literal());
+
+ while (!functions_to_compile.empty()) {
+ FunctionLiteral* literal = functions_to_compile.back();
+ functions_to_compile.pop_back();
+ Handle<SharedFunctionInfo> shared_info =
+ Compiler::GetSharedFunctionInfo(literal, script, isolate);
+ if (shared_info->is_compiled()) continue;
+
+ std::unique_ptr<UnoptimizedCompilationJob> job =
+ ExecuteSingleUnoptimizedCompilationJob(parse_info, literal, allocator,
+ &functions_to_compile);
+ if (!job) return false;
+
+ UpdateSharedFunctionFlagsAfterCompilation(literal, *shared_info);
+
+ auto finalization_status = FinalizeSingleUnoptimizedCompilationJob(
+ job.get(), shared_info, isolate,
+ finalize_unoptimized_compilation_data_list);
+
+ switch (finalization_status) {
+ case CompilationJob::SUCCEEDED:
+ if (shared_info.is_identical_to(outer_shared_info)) {
+ // Ensure that the top level function is retained.
+ *is_compiled_scope = shared_info->is_compiled_scope(isolate);
+ DCHECK(is_compiled_scope->is_compiled());
+ }
+ break;
+
+ case CompilationJob::FAILED:
+ return false;
+
+ case CompilationJob::RETRY_ON_MAIN_THREAD:
+ // This should not happen on the main thread.
+ DCHECK((!std::is_same<LocalIsolate, Isolate>::value));
+ DCHECK_NOT_NULL(jobs_to_retry_finalization_on_main_thread);
+
+ // Clear the literal and ParseInfo to prevent further attempts to access
+ // them.
+ job->compilation_info()->ClearLiteral();
+ job->ClearParseInfo();
+ jobs_to_retry_finalization_on_main_thread->emplace_back(
+ isolate, shared_info, std::move(job));
+ break;
+ }
+ }
+
+ // Report any warnings generated during compilation.
+ if (parse_info->pending_error_handler()->has_pending_warnings()) {
+ parse_info->pending_error_handler()->PrepareWarnings(isolate);
+ }
+
+ return true;
+}
+
+bool FinalizeAllUnoptimizedCompilationJobs(
+ ParseInfo* parse_info, Isolate* isolate, Handle<Script> script,
+ UnoptimizedCompilationJobList* compilation_jobs,
+ FinalizeUnoptimizedCompilationDataList*
+ finalize_unoptimized_compilation_data_list) {
+ DCHECK(AllowCompilation::IsAllowed(isolate));
+ DCHECK(!compilation_jobs->empty());
+
+ // TODO(rmcilroy): Clear native context in debug once AsmJS generates doesn't
+ // rely on accessing native context during finalization.
+
+ // Allocate scope infos for the literal.
+ DeclarationScope::AllocateScopeInfos(parse_info, isolate);
+
+ // Finalize the functions' compilation jobs.
+ for (auto&& job : *compilation_jobs) {
+ FunctionLiteral* literal = job->compilation_info()->literal();
+ Handle<SharedFunctionInfo> shared_info =
+ Compiler::GetSharedFunctionInfo(literal, script, isolate);
+ // The inner function might be compiled already if compiling for debug.
+ if (shared_info->is_compiled()) continue;
+ UpdateSharedFunctionFlagsAfterCompilation(literal, *shared_info);
+ if (FinalizeSingleUnoptimizedCompilationJob(
+ job.get(), shared_info, isolate,
+ finalize_unoptimized_compilation_data_list) !=
+ CompilationJob::SUCCEEDED) {
+ return false;
+ }
+ }
+
+ // Report any warnings generated during compilation.
+ if (parse_info->pending_error_handler()->has_pending_warnings()) {
+ parse_info->pending_error_handler()->PrepareWarnings(isolate);
+ }
+
+ return true;
+}
+
+bool FinalizeDeferredUnoptimizedCompilationJobs(
+ Isolate* isolate, Handle<Script> script,
+ DeferredFinalizationJobDataList* deferred_jobs,
+ PendingCompilationErrorHandler* pending_error_handler,
+ FinalizeUnoptimizedCompilationDataList*
+ finalize_unoptimized_compilation_data_list) {
+ DCHECK(AllowCompilation::IsAllowed(isolate));
+
+ if (deferred_jobs->empty()) return true;
+
+ // TODO(rmcilroy): Clear native context in debug once AsmJS generates doesn't
+ // rely on accessing native context during finalization.
+
+ // Finalize the deferred compilation jobs.
+ for (auto&& job : *deferred_jobs) {
+ Handle<SharedFunctionInfo> shared_info = job.function_handle();
+ if (FinalizeSingleUnoptimizedCompilationJob(
+ job.job(), shared_info, isolate,
+ finalize_unoptimized_compilation_data_list) !=
+ CompilationJob::SUCCEEDED) {
+ return false;
+ }
+ }
+
+ // Report any warnings generated during deferred finalization.
+ if (pending_error_handler->has_pending_warnings()) {
+ pending_error_handler->PrepareWarnings(isolate);
+ }
+
+ return true;
+}
+
+V8_WARN_UNUSED_RESULT MaybeHandle<Code> GetCodeFromOptimizedCodeCache(
+ Handle<JSFunction> function, BailoutId osr_offset, CodeKind code_kind) {
+ RuntimeCallTimerScope runtimeTimer(
+ function->GetIsolate(),
+ RuntimeCallCounterId::kCompileGetFromOptimizedCodeMap);
+ Handle<SharedFunctionInfo> shared(function->shared(), function->GetIsolate());
+ Isolate* isolate = function->GetIsolate();
+ DisallowHeapAllocation no_gc;
+ Code code;
+ if (osr_offset.IsNone() && function->has_feedback_vector()) {
+ FeedbackVector feedback_vector = function->feedback_vector();
+ feedback_vector.EvictOptimizedCodeMarkedForDeoptimization(
+ function->shared(), "GetCodeFromOptimizedCodeCache");
+ code = feedback_vector.optimized_code();
+ } else if (!osr_offset.IsNone()) {
+ code = function->context()
+ .native_context()
+ .GetOSROptimizedCodeCache()
+ .GetOptimizedCode(shared, osr_offset, isolate);
+ }
+ DCHECK_IMPLIES(!code.is_null(), code.kind() <= code_kind);
+ if (!code.is_null() && code.kind() == code_kind) {
+ // Caching of optimized code enabled and optimized code found.
+ DCHECK(!code.marked_for_deoptimization());
+ DCHECK(function->shared().is_compiled());
+ DCHECK(CodeKindIsStoredInOptimizedCodeCache(code.kind()));
+ DCHECK_IMPLIES(!osr_offset.IsNone(), CodeKindCanOSR(code.kind()));
+ return Handle<Code>(code, isolate);
+ }
+ return MaybeHandle<Code>();
+}
+
+void ClearOptimizedCodeCache(OptimizedCompilationInfo* compilation_info) {
+ DCHECK(UsesOptimizationMarker(compilation_info->code_kind()));
+ Handle<JSFunction> function = compilation_info->closure();
+ if (compilation_info->osr_offset().IsNone()) {
+ Handle<FeedbackVector> vector =
+ handle(function->feedback_vector(), function->GetIsolate());
+ vector->ClearOptimizationMarker();
+ }
+}
+
+void InsertCodeIntoOptimizedCodeCache(
+ OptimizedCompilationInfo* compilation_info) {
+ const CodeKind kind = compilation_info->code_kind();
+ if (!CodeKindIsStoredInOptimizedCodeCache(kind)) {
+ if (UsesOptimizationMarker(kind)) {
+ ClearOptimizedCodeCache(compilation_info);
+ }
+ return;
+ }
+
+ if (compilation_info->function_context_specializing()) {
+ // Function context specialization folds-in the function context, so no
+ // sharing can occur. Make sure the optimized code cache is cleared.
+ ClearOptimizedCodeCache(compilation_info);
+ return;
+ }
+
+ // Cache optimized code.
+ Handle<Code> code = compilation_info->code();
+ Handle<JSFunction> function = compilation_info->closure();
+ Handle<SharedFunctionInfo> shared(function->shared(), function->GetIsolate());
+ Handle<NativeContext> native_context(function->context().native_context(),
+ function->GetIsolate());
+ if (compilation_info->osr_offset().IsNone()) {
+ Handle<FeedbackVector> vector =
+ handle(function->feedback_vector(), function->GetIsolate());
+ FeedbackVector::SetOptimizedCode(vector, code);
+ } else {
+ DCHECK(CodeKindCanOSR(kind));
+ OSROptimizedCodeCache::AddOptimizedCode(native_context, shared, code,
+ compilation_info->osr_offset());
+ }
+}
+
+void InsertCodeIntoCompilationCache(Isolate* isolate,
+ OptimizedCompilationInfo* info) {
+ if (!CodeKindIsNativeContextIndependentJSFunction(info->code_kind())) return;
+
+ DCHECK(info->osr_offset().IsNone());
+
+ Handle<Code> code = info->code();
+ DCHECK(!info->function_context_specializing());
+
+ Handle<SharedFunctionInfo> sfi = info->shared_info();
+ CompilationCache* cache = isolate->compilation_cache();
+ cache->PutCode(sfi, code);
+ DCHECK(!cache->LookupCode(sfi).is_null());
+
+ sfi->set_may_have_cached_code(true);
+
+ if (FLAG_trace_turbo_nci) CompilationCacheCode::TraceInsertion(sfi, code);
+}
+
+V8_WARN_UNUSED_RESULT MaybeHandle<Code> GetCodeFromCompilationCache(
+ Isolate* isolate, Handle<SharedFunctionInfo> shared) {
+ if (!shared->may_have_cached_code()) return {};
+ return shared->TryGetCachedCode(isolate);
+}
+
+// Runs PrepareJob in the proper compilation & canonical scopes. Handles will be
+// allocated in a persistent handle scope that is detached and handed off to the
+// {compilation_info} after PrepareJob.
+bool PrepareJobWithHandleScope(OptimizedCompilationJob* job, Isolate* isolate,
+ OptimizedCompilationInfo* compilation_info) {
+ CompilationHandleScope compilation(isolate, compilation_info);
+ CanonicalHandleScope canonical(isolate, compilation_info);
+ compilation_info->ReopenHandlesInNewHandleScope(isolate);
+ return job->PrepareJob(isolate) == CompilationJob::SUCCEEDED;
+}
+
+bool GetOptimizedCodeNow(OptimizedCompilationJob* job, Isolate* isolate,
+ OptimizedCompilationInfo* compilation_info) {
+ TimerEventScope<TimerEventRecompileSynchronous> timer(isolate);
+ RuntimeCallTimerScope runtimeTimer(
+ isolate, RuntimeCallCounterId::kOptimizeNonConcurrent);
+ TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.compile"),
+ "V8.OptimizeNonConcurrent");
+
+ if (!PrepareJobWithHandleScope(job, isolate, compilation_info)) {
+ CompilerTracer::TraceAbortedJob(isolate, compilation_info);
+ return false;
+ }
+
+ {
+ LocalIsolate local_isolate(isolate, ThreadKind::kMain);
+ if (job->ExecuteJob(isolate->counters()->runtime_call_stats(),
+ &local_isolate)) {
+ CompilerTracer::TraceAbortedJob(isolate, compilation_info);
+ return false;
+ }
+ }
+
+ if (job->FinalizeJob(isolate) != CompilationJob::SUCCEEDED) {
+ CompilerTracer::TraceAbortedJob(isolate, compilation_info);
+ return false;
+ }
+
+ // Success!
+ job->RecordCompilationStats(OptimizedCompilationJob::kSynchronous, isolate);
+ DCHECK(!isolate->has_pending_exception());
+ InsertCodeIntoOptimizedCodeCache(compilation_info);
+ job->RecordFunctionCompilation(CodeEventListener::LAZY_COMPILE_TAG, isolate);
+ return true;
+}
+
+bool GetOptimizedCodeLater(std::unique_ptr<OptimizedCompilationJob> job,
+ Isolate* isolate,
+ OptimizedCompilationInfo* compilation_info,
+ CodeKind code_kind, Handle<JSFunction> function) {
+ if (!isolate->optimizing_compile_dispatcher()->IsQueueAvailable()) {
+ if (FLAG_trace_concurrent_recompilation) {
+ PrintF(" ** Compilation queue full, will retry optimizing ");
+ compilation_info->closure()->ShortPrint();
+ PrintF(" later.\n");
+ }
+ return false;
+ }
+
+ if (isolate->heap()->HighMemoryPressure()) {
+ if (FLAG_trace_concurrent_recompilation) {
+ PrintF(" ** High memory pressure, will retry optimizing ");
+ compilation_info->closure()->ShortPrint();
+ PrintF(" later.\n");
+ }
+ return false;
+ }
+
+ TimerEventScope<TimerEventRecompileSynchronous> timer(isolate);
+ RuntimeCallTimerScope runtimeTimer(
+ isolate, RuntimeCallCounterId::kOptimizeConcurrentPrepare);
+ TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.compile"),
+ "V8.OptimizeConcurrentPrepare");
+
+ if (!PrepareJobWithHandleScope(job.get(), isolate, compilation_info)) {
+ return false;
+ }
+
+ // The background recompile will own this job.
+ isolate->optimizing_compile_dispatcher()->QueueForOptimization(job.get());
+ job.release();
+
+ if (FLAG_trace_concurrent_recompilation) {
+ PrintF(" ** Queued ");
+ compilation_info->closure()->ShortPrint();
+ PrintF(" for concurrent optimization.\n");
+ }
+
+ if (CodeKindIsStoredInOptimizedCodeCache(code_kind)) {
+ function->SetOptimizationMarker(OptimizationMarker::kInOptimizationQueue);
+ }
+
+ // Note: Usually the active tier is expected to be Ignition or NCI at this
+ // point (in other words we don't expect to optimize if the function is
+ // already TF-optimized). There is a special case for OSR though, for which
+ // we *can* reach this point even if we've already generated non-OSR'd TF
+ // code.
+ DCHECK(function->shared().HasBytecodeArray());
+ return true;
+}
+
+// Returns the code object at which execution continues after a concurrent
+// optimization job has been started (but not finished).
+Handle<Code> ContinuationForConcurrentOptimization(
+ Isolate* isolate, Handle<JSFunction> function) {
+ Handle<Code> cached_code;
+ if (FLAG_turbo_nci && function->NextTier() == CodeKindForTopTier() &&
+ GetCodeFromCompilationCache(isolate, handle(function->shared(), isolate))
+ .ToHandle(&cached_code)) {
+ // Tiering up to Turbofan and cached optimized code exists. Continue
+ // execution there until TF optimization has finished.
+ return cached_code;
+ } else if (FLAG_turboprop_as_midtier &&
+ function->HasAvailableOptimizedCode()) {
+ DCHECK(function->NextTier() == CodeKind::TURBOFAN);
+ // It is possible that we have marked a closure for TurboFan optimization
+ // but the marker is processed by another closure that doesn't have
+ // optimized code yet. So heal the closure here and return the optimized
+ // code.
+ if (!function->HasAttachedOptimizedCode()) {
+ DCHECK(function->feedback_vector().has_optimized_code());
+ function->set_code(function->feedback_vector().optimized_code());
+ }
+ return handle(function->code(), isolate);
+ }
+ return BUILTIN_CODE(isolate, InterpreterEntryTrampoline);
+}
+
+MaybeHandle<Code> GetOptimizedCode(Handle<JSFunction> function,
+ ConcurrencyMode mode, CodeKind code_kind,
+ BailoutId osr_offset = BailoutId::None(),
+ JavaScriptFrame* osr_frame = nullptr) {
+ DCHECK(CodeKindIsOptimizedJSFunction(code_kind));
+
+ Isolate* isolate = function->GetIsolate();
+ Handle<SharedFunctionInfo> shared(function->shared(), isolate);
+
+ // Make sure we clear the optimization marker on the function so that we
+ // don't try to re-optimize.
+ // If compiling for NCI caching only (which does not use the optimization
+ // marker), don't touch the marker to avoid interfering with Turbofan
+ // compilation.
+ if (UsesOptimizationMarker(code_kind) && function->HasOptimizationMarker()) {
+ function->ClearOptimizationMarker();
+ }
+
+ if (shared->optimization_disabled() &&
+ shared->disable_optimization_reason() == BailoutReason::kNeverOptimize) {
+ return {};
+ }
+
+ // Do not optimize when debugger needs to hook into every call.
+ if (isolate->debug()->needs_check_on_function_call()) return {};
+
+ // Do not use TurboFan if we need to be able to set break points.
+ if (shared->HasBreakInfo()) return {};
+
+ // Do not use TurboFan if optimization is disabled or function doesn't pass
+ // turbo_filter.
+ if (!FLAG_opt || !shared->PassesFilter(FLAG_turbo_filter)) return {};
+
+ // If code was pending optimization for testing, remove the entry from the
+ // table that was preventing the bytecode from being flushed.
+ if (V8_UNLIKELY(FLAG_testing_d8_test_runner)) {
+ PendingOptimizationTable::FunctionWasOptimized(isolate, function);
+ }
+
+ // Check the optimized code cache (stored on the SharedFunctionInfo).
+ if (CodeKindIsStoredInOptimizedCodeCache(code_kind)) {
+ Handle<Code> cached_code;
+ if (GetCodeFromOptimizedCodeCache(function, osr_offset, code_kind)
+ .ToHandle(&cached_code)) {
+ CompilerTracer::TraceOptimizedCodeCacheHit(isolate, function, osr_offset,
+ code_kind);
+ return cached_code;
+ }
+ }
+
+ // Reset profiler ticks, function is no longer considered hot.
+ DCHECK(shared->is_compiled());
+ function->feedback_vector().set_profiler_ticks(0);
+
+ // Check the compilation cache (stored on the Isolate, shared between native
+ // contexts).
+ if (CodeKindIsNativeContextIndependentJSFunction(code_kind)) {
+ DCHECK(osr_offset.IsNone());
+ DCHECK(FLAG_turbo_nci_as_midtier || !FLAG_turbo_nci_delayed_codegen ||
+ shared->has_optimized_at_least_once());
+
+ Handle<Code> cached_code;
+ if (GetCodeFromCompilationCache(isolate, shared).ToHandle(&cached_code)) {
+ CHECK_EQ(cached_code->kind(), CodeKind::NATIVE_CONTEXT_INDEPENDENT);
+ if (FLAG_trace_turbo_nci) {
+ CompilationCacheCode::TraceHit(shared, cached_code);
+ }
+ return cached_code;
+ }
+ }
+
+ VMState<COMPILER> state(isolate);
+ TimerEventScope<TimerEventOptimizeCode> optimize_code_timer(isolate);
+ RuntimeCallTimerScope runtimeTimer(isolate,
+ RuntimeCallCounterId::kOptimizeCode);
+ TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.compile"), "V8.OptimizeCode");
+
+ DCHECK(!isolate->has_pending_exception());
+ PostponeInterruptsScope postpone(isolate);
+ bool has_script = shared->script().IsScript();
+ // BUG(5946): This DCHECK is necessary to make certain that we won't
+ // tolerate the lack of a script without bytecode.
+ DCHECK_IMPLIES(!has_script, shared->HasBytecodeArray());
+ std::unique_ptr<OptimizedCompilationJob> job(
+ compiler::Pipeline::NewCompilationJob(isolate, function, code_kind,
+ has_script, osr_offset, osr_frame));
+ OptimizedCompilationInfo* compilation_info = job->compilation_info();
+
+ // Prepare the job and launch concurrent compilation, or compile now.
+ if (mode == ConcurrencyMode::kConcurrent) {
+ if (GetOptimizedCodeLater(std::move(job), isolate, compilation_info,
+ code_kind, function)) {
+ return ContinuationForConcurrentOptimization(isolate, function);
+ }
+ } else {
+ DCHECK_EQ(mode, ConcurrencyMode::kNotConcurrent);
+ if (GetOptimizedCodeNow(job.get(), isolate, compilation_info)) {
+ InsertCodeIntoCompilationCache(isolate, compilation_info);
+ return compilation_info->code();
+ }
+ }
+
+ if (isolate->has_pending_exception()) isolate->clear_pending_exception();
+ return {};
+}
+
+bool FailAndClearPendingException(Isolate* isolate) {
+ isolate->clear_pending_exception();
+ return false;
+}
+
+template <typename LocalIsolate>
+bool PreparePendingException(LocalIsolate* isolate, ParseInfo* parse_info) {
+ if (parse_info->pending_error_handler()->has_pending_error()) {
+ parse_info->pending_error_handler()->PrepareErrors(
+ isolate, parse_info->ast_value_factory());
+ }
+ return false;
+}
+
+bool FailWithPreparedPendingException(
+ Isolate* isolate, Handle<Script> script,
+ const PendingCompilationErrorHandler* pending_error_handler) {
+ if (!isolate->has_pending_exception()) {
+ if (pending_error_handler->has_pending_error()) {
+ pending_error_handler->ReportErrors(isolate, script);
+ } else {
+ isolate->StackOverflow();
+ }
+ }
+ return false;
+}
+
+bool FailWithPendingException(Isolate* isolate, Handle<Script> script,
+ ParseInfo* parse_info,
+ Compiler::ClearExceptionFlag flag) {
+ if (flag == Compiler::CLEAR_EXCEPTION) {
+ return FailAndClearPendingException(isolate);
+ }
+
+ PreparePendingException(isolate, parse_info);
+ return FailWithPreparedPendingException(isolate, script,
+ parse_info->pending_error_handler());
+}
+
+void FinalizeUnoptimizedCompilation(
+ Isolate* isolate, Handle<Script> script,
+ const UnoptimizedCompileFlags& flags,
+ const UnoptimizedCompileState* compile_state,
+ const FinalizeUnoptimizedCompilationDataList&
+ finalize_unoptimized_compilation_data_list) {
+ if (compile_state->pending_error_handler()->has_pending_warnings()) {
+ compile_state->pending_error_handler()->ReportWarnings(isolate, script);
+ }
+
+ bool need_source_positions = FLAG_stress_lazy_source_positions ||
+ (!flags.collect_source_positions() &&
+ isolate->NeedsSourcePositionsForProfiling());
+
+ for (const auto& finalize_data : finalize_unoptimized_compilation_data_list) {
+ Handle<SharedFunctionInfo> shared_info = finalize_data.function_handle();
+ // It's unlikely, but possible, that the bytecode was flushed between being
+ // allocated and now, so guard against that case, and against it being
+ // flushed in the middle of this loop.
+ IsCompiledScope is_compiled_scope(*shared_info, isolate);
+ if (!is_compiled_scope.is_compiled()) continue;
+
+ if (need_source_positions) {
+ SharedFunctionInfo::EnsureSourcePositionsAvailable(isolate, shared_info);
+ }
+ if (FLAG_interpreted_frames_native_stack) {
+ InstallInterpreterTrampolineCopy(isolate, shared_info);
+ }
+ Handle<CoverageInfo> coverage_info;
+ if (finalize_data.coverage_info().ToHandle(&coverage_info)) {
+ isolate->debug()->InstallCoverageInfo(shared_info, coverage_info);
+ }
+
+ LogUnoptimizedCompilation(isolate, shared_info, flags,
+ finalize_data.time_taken_to_execute(),
+ finalize_data.time_taken_to_finalize());
+ }
+}
+
+void FinalizeUnoptimizedScriptCompilation(
+ Isolate* isolate, Handle<Script> script,
+ const UnoptimizedCompileFlags& flags,
+ const UnoptimizedCompileState* compile_state,
+ const FinalizeUnoptimizedCompilationDataList&
+ finalize_unoptimized_compilation_data_list) {
+ FinalizeUnoptimizedCompilation(isolate, script, flags, compile_state,
+ finalize_unoptimized_compilation_data_list);
+
+ script->set_compilation_state(Script::COMPILATION_STATE_COMPILED);
+
+ UnoptimizedCompileState::ParallelTasks* parallel_tasks =
+ compile_state->parallel_tasks();
+ if (parallel_tasks) {
+ CompilerDispatcher* dispatcher = parallel_tasks->dispatcher();
+ for (auto& it : *parallel_tasks) {
+ FunctionLiteral* literal = it.first;
+ CompilerDispatcher::JobId job_id = it.second;
+ MaybeHandle<SharedFunctionInfo> maybe_shared_for_task =
+ script->FindSharedFunctionInfo(isolate,
+ literal->function_literal_id());
+ Handle<SharedFunctionInfo> shared_for_task;
+ if (maybe_shared_for_task.ToHandle(&shared_for_task)) {
+ dispatcher->RegisterSharedFunctionInfo(job_id, *shared_for_task);
+ } else {
+ dispatcher->AbortJob(job_id);
+ }
+ }
+ }
+
+ if (isolate->NeedsSourcePositionsForProfiling()) {
+ Script::InitLineEnds(isolate, script);
+ }
+}
+
+// Create shared function info for top level and shared function infos array for
+// inner functions.
+template <typename LocalIsolate>
+Handle<SharedFunctionInfo> CreateTopLevelSharedFunctionInfo(
+ ParseInfo* parse_info, Handle<Script> script, LocalIsolate* isolate) {
+ EnsureSharedFunctionInfosArrayOnScript(script, parse_info, isolate);
+ DCHECK_EQ(kNoSourcePosition,
+ parse_info->literal()->function_token_position());
+ return isolate->factory()->NewSharedFunctionInfoForLiteral(
+ parse_info->literal(), script, true);
+}
+
+MaybeHandle<SharedFunctionInfo> CompileToplevel(
+ ParseInfo* parse_info, Handle<Script> script,
+ MaybeHandle<ScopeInfo> maybe_outer_scope_info, Isolate* isolate,
+ IsCompiledScope* is_compiled_scope) {
+ TimerEventScope<TimerEventCompileCode> top_level_timer(isolate);
+ TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.compile"), "V8.CompileCode");
+ DCHECK_EQ(ThreadId::Current(), isolate->thread_id());
+
+ PostponeInterruptsScope postpone(isolate);
+ DCHECK(!isolate->native_context().is_null());
+ RuntimeCallTimerScope runtimeTimer(
+ isolate, parse_info->flags().is_eval()
+ ? RuntimeCallCounterId::kCompileEval
+ : RuntimeCallCounterId::kCompileScript);
+ VMState<BYTECODE_COMPILER> state(isolate);
+ if (parse_info->literal() == nullptr &&
+ !parsing::ParseProgram(parse_info, script, maybe_outer_scope_info,
+ isolate, parsing::ReportStatisticsMode::kYes)) {
+ FailWithPendingException(isolate, script, parse_info,
+ Compiler::ClearExceptionFlag::KEEP_EXCEPTION);
+ return MaybeHandle<SharedFunctionInfo>();
+ }
+ // Measure how long it takes to do the compilation; only take the
+ // rest of the function into account to avoid overlap with the
+ // parsing statistics.
+ HistogramTimer* rate = parse_info->flags().is_eval()
+ ? isolate->counters()->compile_eval()
+ : isolate->counters()->compile();
+ HistogramTimerScope timer(rate);
+ TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.compile"),
+ parse_info->flags().is_eval() ? "V8.CompileEval" : "V8.Compile");
+
+ // Prepare and execute compilation of the outer-most function.
+
+ // Create the SharedFunctionInfo and add it to the script's list.
+ Handle<SharedFunctionInfo> shared_info =
+ CreateTopLevelSharedFunctionInfo(parse_info, script, isolate);
+
+ FinalizeUnoptimizedCompilationDataList
+ finalize_unoptimized_compilation_data_list;
+
+ if (!IterativelyExecuteAndFinalizeUnoptimizedCompilationJobs(
+ isolate, shared_info, script, parse_info, isolate->allocator(),
+ is_compiled_scope, &finalize_unoptimized_compilation_data_list,
+ nullptr)) {
+ FailWithPendingException(isolate, script, parse_info,
+ Compiler::ClearExceptionFlag::KEEP_EXCEPTION);
+ return MaybeHandle<SharedFunctionInfo>();
+ }
+
+ // Character stream shouldn't be used again.
+ parse_info->ResetCharacterStream();
+
+ FinalizeUnoptimizedScriptCompilation(
+ isolate, script, parse_info->flags(), parse_info->state(),
+ finalize_unoptimized_compilation_data_list);
+ return shared_info;
+}
+
+RuntimeCallCounterId RuntimeCallCounterIdForCompileBackground(
+ ParseInfo* parse_info) {
+ if (parse_info->flags().is_toplevel()) {
+ if (parse_info->flags().is_eval()) {
+ return RuntimeCallCounterId::kCompileBackgroundEval;
+ }
+ return RuntimeCallCounterId::kCompileBackgroundScript;
+ }
+ return RuntimeCallCounterId::kCompileBackgroundFunction;
+}
+
+MaybeHandle<SharedFunctionInfo> CompileAndFinalizeOnBackgroundThread(
+ ParseInfo* parse_info, AccountingAllocator* allocator,
+ Handle<Script> script, LocalIsolate* isolate,
+ FinalizeUnoptimizedCompilationDataList*
+ finalize_unoptimized_compilation_data_list,
+ DeferredFinalizationJobDataList* jobs_to_retry_finalization_on_main_thread,
+ IsCompiledScope* is_compiled_scope) {
+ TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.compile"),
+ "V8.CompileCodeBackground");
+ RuntimeCallTimerScope runtimeTimer(
+ parse_info->runtime_call_stats(),
+ RuntimeCallCounterIdForCompileBackground(parse_info));
+
+ Handle<SharedFunctionInfo> shared_info =
+ CreateTopLevelSharedFunctionInfo(parse_info, script, isolate);
+
+ if (!IterativelyExecuteAndFinalizeUnoptimizedCompilationJobs(
+ isolate, shared_info, script, parse_info, allocator,
+ is_compiled_scope, finalize_unoptimized_compilation_data_list,
+ jobs_to_retry_finalization_on_main_thread)) {
+ return kNullMaybeHandle;
+ }
+
+ // Character stream shouldn't be used again.
+ parse_info->ResetCharacterStream();
+
+ return shared_info;
+}
+
+// TODO(leszeks): Remove this once off-thread finalization is always on.
+void CompileOnBackgroundThread(ParseInfo* parse_info,
+ AccountingAllocator* allocator,
+ UnoptimizedCompilationJobList* jobs) {
+ DisallowHeapAccess no_heap_access;
+ TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.compile"),
+ "V8.CompileCodeBackground");
+ RuntimeCallTimerScope runtimeTimer(
+ parse_info->runtime_call_stats(),
+ RuntimeCallCounterIdForCompileBackground(parse_info));
+
+ // Generate the unoptimized bytecode or asm-js data.
+ DCHECK(jobs->empty());
+
+ bool success = RecursivelyExecuteUnoptimizedCompilationJobs(
+ parse_info, parse_info->literal(), allocator, jobs);
+
+ USE(success);
+ DCHECK_EQ(success, !jobs->empty());
+
+ // Character stream shouldn't be used again.
+ parse_info->ResetCharacterStream();
+}
+
+MaybeHandle<SharedFunctionInfo> CompileToplevel(
+ ParseInfo* parse_info, Handle<Script> script, Isolate* isolate,
+ IsCompiledScope* is_compiled_scope) {
+ return CompileToplevel(parse_info, script, kNullMaybeHandle, isolate,
+ is_compiled_scope);
+}
+
+} // namespace
+
+CompilationHandleScope::~CompilationHandleScope() {
+ info_->set_persistent_handles(persistent_.Detach());
+}
+
+FinalizeUnoptimizedCompilationData::FinalizeUnoptimizedCompilationData(
+ LocalIsolate* isolate, Handle<SharedFunctionInfo> function_handle,
+ MaybeHandle<CoverageInfo> coverage_info,
+ base::TimeDelta time_taken_to_execute,
+ base::TimeDelta time_taken_to_finalize)
+ : time_taken_to_execute_(time_taken_to_execute),
+ time_taken_to_finalize_(time_taken_to_finalize),
+ function_handle_(isolate->heap()->NewPersistentHandle(function_handle)),
+ coverage_info_(isolate->heap()->NewPersistentMaybeHandle(coverage_info)) {
+}
+
+DeferredFinalizationJobData::DeferredFinalizationJobData(
+ LocalIsolate* isolate, Handle<SharedFunctionInfo> function_handle,
+ std::unique_ptr<UnoptimizedCompilationJob> job)
+ : function_handle_(isolate->heap()->NewPersistentHandle(function_handle)),
+ job_(std::move(job)) {}
+
+BackgroundCompileTask::BackgroundCompileTask(ScriptStreamingData* streamed_data,
+ Isolate* isolate)
+ : flags_(UnoptimizedCompileFlags::ForToplevelCompile(
+ isolate, true, construct_language_mode(FLAG_use_strict),
+ REPLMode::kNo)),
+ compile_state_(isolate),
+ info_(std::make_unique<ParseInfo>(isolate, flags_, &compile_state_)),
+ isolate_for_local_isolate_(isolate),
+ start_position_(0),
+ end_position_(0),
+ function_literal_id_(kFunctionLiteralIdTopLevel),
+ stack_size_(i::FLAG_stack_size),
+ worker_thread_runtime_call_stats_(
+ isolate->counters()->worker_thread_runtime_call_stats()),
+ timer_(isolate->counters()->compile_script_on_background()),
+ language_mode_(info_->language_mode()) {
+ VMState<PARSER> state(isolate);
+
+ // Prepare the data for the internalization phase and compilation phase, which
+ // will happen in the main thread after parsing.
+
+ LOG(isolate, ScriptEvent(Logger::ScriptEventType::kStreamingCompile,
+ info_->flags().script_id()));
+
+ std::unique_ptr<Utf16CharacterStream> stream(ScannerStream::For(
+ streamed_data->source_stream.get(), streamed_data->encoding));
+ info_->set_character_stream(std::move(stream));
+}
+
+BackgroundCompileTask::BackgroundCompileTask(
+ const ParseInfo* outer_parse_info, const AstRawString* function_name,
+ const FunctionLiteral* function_literal,
+ WorkerThreadRuntimeCallStats* worker_thread_runtime_stats,
+ TimedHistogram* timer, int max_stack_size)
+ : flags_(UnoptimizedCompileFlags::ForToplevelFunction(
+ outer_parse_info->flags(), function_literal)),
+ compile_state_(*outer_parse_info->state()),
+ info_(ParseInfo::ForToplevelFunction(flags_, &compile_state_,
+ function_literal, function_name)),
+ isolate_for_local_isolate_(nullptr),
+ start_position_(function_literal->start_position()),
+ end_position_(function_literal->end_position()),
+ function_literal_id_(function_literal->function_literal_id()),
+ stack_size_(max_stack_size),
+ worker_thread_runtime_call_stats_(worker_thread_runtime_stats),
+ timer_(timer),
+ language_mode_(info_->language_mode()) {
+ DCHECK_EQ(outer_parse_info->parameters_end_pos(), kNoSourcePosition);
+ DCHECK_NULL(outer_parse_info->extension());
+
+ DCHECK(!function_literal->is_toplevel());
+
+ // Clone the character stream so both can be accessed independently.
+ std::unique_ptr<Utf16CharacterStream> character_stream =
+ outer_parse_info->character_stream()->Clone();
+ character_stream->Seek(start_position_);
+ info_->set_character_stream(std::move(character_stream));
+
+ // Get preparsed scope data from the function literal.
+ if (function_literal->produced_preparse_data()) {
+ ZonePreparseData* serialized_data =
+ function_literal->produced_preparse_data()->Serialize(info_->zone());
+ info_->set_consumed_preparse_data(
+ ConsumedPreparseData::For(info_->zone(), serialized_data));
+ }
+}
+
+BackgroundCompileTask::~BackgroundCompileTask() = default;
+
+namespace {
+
+// A scope object that ensures a parse info's runtime call stats and stack limit
+// are set correctly during worker-thread compile, and restores it after going
+// out of scope.
+class OffThreadParseInfoScope {
+ public:
+ OffThreadParseInfoScope(
+ ParseInfo* parse_info,
+ WorkerThreadRuntimeCallStats* worker_thread_runtime_stats, int stack_size)
+ : parse_info_(parse_info),
+ original_runtime_call_stats_(parse_info_->runtime_call_stats()),
+ original_stack_limit_(parse_info_->stack_limit()),
+ worker_thread_scope_(worker_thread_runtime_stats) {
+ parse_info_->SetPerThreadState(GetCurrentStackPosition() - stack_size * KB,
+ worker_thread_scope_.Get());
+ }
+
+ ~OffThreadParseInfoScope() {
+ DCHECK_NOT_NULL(parse_info_);
+ parse_info_->SetPerThreadState(original_stack_limit_,
+ original_runtime_call_stats_);
+ }
+
+ private:
+ ParseInfo* parse_info_;
+ RuntimeCallStats* original_runtime_call_stats_;
+ uintptr_t original_stack_limit_;
+ WorkerThreadRuntimeCallStatsScope worker_thread_scope_;
+
+ DISALLOW_COPY_AND_ASSIGN(OffThreadParseInfoScope);
+};
+
+} // namespace
+
+void BackgroundCompileTask::Run() {
+ TimedHistogramScope timer(timer_);
+ base::Optional<OffThreadParseInfoScope> off_thread_scope(
+ base::in_place, info_.get(), worker_thread_runtime_call_stats_,
+ stack_size_);
+ TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.compile"),
+ "BackgroundCompileTask::Run");
+ RuntimeCallTimerScope runtimeTimer(
+ info_->runtime_call_stats(),
+ RuntimeCallCounterId::kCompileBackgroundCompileTask);
+
+ // Update the character stream's runtime call stats.
+ info_->character_stream()->set_runtime_call_stats(
+ info_->runtime_call_stats());
+
+ // Parser needs to stay alive for finalizing the parsing on the main
+ // thread.
+ parser_.reset(new Parser(info_.get()));
+ parser_->InitializeEmptyScopeChain(info_.get());
+
+ parser_->ParseOnBackground(info_.get(), start_position_, end_position_,
+ function_literal_id_);
+
+ // Save the language mode.
+ language_mode_ = info_->language_mode();
+
+ if (!FLAG_finalize_streaming_on_background) {
+ if (info_->literal() != nullptr) {
+ CompileOnBackgroundThread(info_.get(), compile_state_.allocator(),
+ &compilation_jobs_);
+ }
+ } else {
+ DCHECK(info_->flags().is_toplevel());
+
+ LocalIsolate isolate(isolate_for_local_isolate_, ThreadKind::kBackground);
+ UnparkedScope unparked_scope(isolate.heap());
+ LocalHandleScope handle_scope(&isolate);
+
+ info_->ast_value_factory()->Internalize(&isolate);
+
+ // We don't have the script source, origin, or details yet, so use default
+ // values for them. These will be fixed up during the main-thread merge.
+ Handle<Script> script =
+ info_->CreateScript(&isolate, isolate.factory()->empty_string(),
+ kNullMaybeHandle, ScriptOriginOptions());
+
+ parser_->HandleSourceURLComments(&isolate, script);
+
+ MaybeHandle<SharedFunctionInfo> maybe_result;
+ if (info_->literal() != nullptr) {
+ maybe_result = CompileAndFinalizeOnBackgroundThread(
+ info_.get(), compile_state_.allocator(), script, &isolate,
+ &finalize_unoptimized_compilation_data_,
+ &jobs_to_retry_finalization_on_main_thread_, &is_compiled_scope_);
+ } else {
+ DCHECK(compile_state_.pending_error_handler()->has_pending_error());
+ PreparePendingException(&isolate, info_.get());
+ }
+
+ outer_function_sfi_ =
+ isolate.heap()->NewPersistentMaybeHandle(maybe_result);
+ script_ = isolate.heap()->NewPersistentHandle(script);
+
+ persistent_handles_ = isolate.heap()->DetachPersistentHandles();
+
+ {
+ TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.compile"),
+ "V8.FinalizeCodeBackground.ReleaseParser");
+ DCHECK_EQ(language_mode_, info_->language_mode());
+ off_thread_scope.reset();
+ parser_.reset();
+ info_.reset();
+ }
+ }
+}
+
+MaybeHandle<SharedFunctionInfo> BackgroundCompileTask::GetOuterFunctionSfi(
+ Isolate* isolate) {
+ // outer_function_sfi_ is a persistent Handle, tied to the lifetime of the
+ // persistent_handles_ member, so create a new Handle to let it outlive
+ // the BackgroundCompileTask.
+ Handle<SharedFunctionInfo> result;
+ if (outer_function_sfi_.ToHandle(&result)) {
+ return handle(*result, isolate);
+ }
+ return kNullMaybeHandle;
+}
+
+Handle<Script> BackgroundCompileTask::GetScript(Isolate* isolate) {
+ // script_ is a persistent Handle, tied to the lifetime of the
+ // persistent_handles_ member, so create a new Handle to let it outlive
+ // the BackgroundCompileTask.
+ return handle(*script_, isolate);
+}
+
+// ----------------------------------------------------------------------------
+// Implementation of Compiler
+
+// static
+bool Compiler::CollectSourcePositions(Isolate* isolate,
+ Handle<SharedFunctionInfo> shared_info) {
+ DCHECK(shared_info->is_compiled());
+ DCHECK(shared_info->HasBytecodeArray());
+ DCHECK(!shared_info->GetBytecodeArray().HasSourcePositionTable());
+
+ // Source position collection should be context independent.
+ NullContextScope null_context_scope(isolate);
+
+ // Collecting source positions requires allocating a new source position
+ // table.
+ DCHECK(AllowHeapAllocation::IsAllowed());
+ DCHECK(AllowGarbageCollection::IsAllowed());
+
+ Handle<BytecodeArray> bytecode =
+ handle(shared_info->GetBytecodeArray(), isolate);
+
+ // TODO(v8:8510): Push the CLEAR_EXCEPTION flag or something like it down into
+ // the parser so it aborts without setting a pending exception, which then
+ // gets thrown. This would avoid the situation where potentially we'd reparse
+ // several times (running out of stack each time) before hitting this limit.
+ if (GetCurrentStackPosition() < isolate->stack_guard()->real_climit()) {
+ // Stack is already exhausted.
+ bytecode->SetSourcePositionsFailedToCollect();
+ return false;
+ }
+
+ DCHECK(AllowCompilation::IsAllowed(isolate));
+ DCHECK_EQ(ThreadId::Current(), isolate->thread_id());
+ DCHECK(!isolate->has_pending_exception());
+ VMState<BYTECODE_COMPILER> state(isolate);
+ PostponeInterruptsScope postpone(isolate);
+ RuntimeCallTimerScope runtimeTimer(
+ isolate, RuntimeCallCounterId::kCompileCollectSourcePositions);
+ TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.compile"),
+ "V8.CollectSourcePositions");
+ HistogramTimerScope timer(isolate->counters()->collect_source_positions());
+
+ // Set up parse info.
+ UnoptimizedCompileFlags flags =
+ UnoptimizedCompileFlags::ForFunctionCompile(isolate, *shared_info);
+ flags.set_is_lazy_compile(true);
+ flags.set_collect_source_positions(true);
+ flags.set_allow_natives_syntax(FLAG_allow_natives_syntax);
+
+ UnoptimizedCompileState compile_state(isolate);
+ ParseInfo parse_info(isolate, flags, &compile_state);
+
+ // Parse and update ParseInfo with the results. Don't update parsing
+ // statistics since we've already parsed the code before.
+ if (!parsing::ParseAny(&parse_info, shared_info, isolate,
+ parsing::ReportStatisticsMode::kNo)) {
+ // Parsing failed probably as a result of stack exhaustion.
+ bytecode->SetSourcePositionsFailedToCollect();
+ return FailAndClearPendingException(isolate);
+ }
+
+ // Character stream shouldn't be used again.
+ parse_info.ResetCharacterStream();
+
+ // Generate the unoptimized bytecode.
+ // TODO(v8:8510): Consider forcing preparsing of inner functions to avoid
+ // wasting time fully parsing them when they won't ever be used.
+ std::unique_ptr<UnoptimizedCompilationJob> job;
+ {
+ job = interpreter::Interpreter::NewSourcePositionCollectionJob(
+ &parse_info, parse_info.literal(), bytecode, isolate->allocator());
+
+ if (!job || job->ExecuteJob() != CompilationJob::SUCCEEDED ||
+ job->FinalizeJob(shared_info, isolate) != CompilationJob::SUCCEEDED) {
+ // Recompiling failed probably as a result of stack exhaustion.
+ bytecode->SetSourcePositionsFailedToCollect();
+ return FailAndClearPendingException(isolate);
+ }
+ }
+
+ DCHECK(job->compilation_info()->flags().collect_source_positions());
+
+ // If debugging, make sure that instrumented bytecode has the source position
+ // table set on it as well.
+ if (shared_info->HasDebugInfo() &&
+ shared_info->GetDebugInfo().HasInstrumentedBytecodeArray()) {
+ ByteArray source_position_table =
+ job->compilation_info()->bytecode_array()->SourcePositionTable();
+ shared_info->GetDebugBytecodeArray().set_source_position_table(
+ source_position_table, kReleaseStore);
+ }
+
+ DCHECK(!isolate->has_pending_exception());
+ DCHECK(shared_info->is_compiled_scope(isolate).is_compiled());
+ return true;
+}
+
+// static
+bool Compiler::Compile(Handle<SharedFunctionInfo> shared_info,
+ ClearExceptionFlag flag,
+ IsCompiledScope* is_compiled_scope) {
+ // We should never reach here if the function is already compiled.
+ DCHECK(!shared_info->is_compiled());
+ DCHECK(!is_compiled_scope->is_compiled());
+
+ Isolate* isolate = shared_info->GetIsolate();
+ DCHECK(AllowCompilation::IsAllowed(isolate));
+ DCHECK_EQ(ThreadId::Current(), isolate->thread_id());
+ DCHECK(!isolate->has_pending_exception());
+ DCHECK(!shared_info->HasBytecodeArray());
+ VMState<BYTECODE_COMPILER> state(isolate);
+ PostponeInterruptsScope postpone(isolate);
+ TimerEventScope<TimerEventCompileCode> compile_timer(isolate);
+ RuntimeCallTimerScope runtimeTimer(isolate,
+ RuntimeCallCounterId::kCompileFunction);
+ TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.compile"), "V8.CompileCode");
+ AggregatedHistogramTimerScope timer(isolate->counters()->compile_lazy());
+
+ Handle<Script> script(Script::cast(shared_info->script()), isolate);
+
+ // Set up parse info.
+ UnoptimizedCompileFlags flags =
+ UnoptimizedCompileFlags::ForFunctionCompile(isolate, *shared_info);
+ flags.set_is_lazy_compile(true);
+
+ UnoptimizedCompileState compile_state(isolate);
+ ParseInfo parse_info(isolate, flags, &compile_state);
+
+ // Check if the compiler dispatcher has shared_info enqueued for compile.
+ CompilerDispatcher* dispatcher = isolate->compiler_dispatcher();
+ if (dispatcher->IsEnqueued(shared_info)) {
+ if (!dispatcher->FinishNow(shared_info)) {
+ return FailWithPendingException(isolate, script, &parse_info, flag);
+ }
+ *is_compiled_scope = shared_info->is_compiled_scope(isolate);
+ DCHECK(is_compiled_scope->is_compiled());
+ return true;
+ }
+
+ if (shared_info->HasUncompiledDataWithPreparseData()) {
+ parse_info.set_consumed_preparse_data(ConsumedPreparseData::For(
+ isolate,
+ handle(
+ shared_info->uncompiled_data_with_preparse_data().preparse_data(),
+ isolate)));
+ }
+
+ // Parse and update ParseInfo with the results.
+ if (!parsing::ParseAny(&parse_info, shared_info, isolate,
+ parsing::ReportStatisticsMode::kYes)) {
+ return FailWithPendingException(isolate, script, &parse_info, flag);
+ }
+
+ // Generate the unoptimized bytecode or asm-js data.
+ FinalizeUnoptimizedCompilationDataList
+ finalize_unoptimized_compilation_data_list;
+
+ if (!IterativelyExecuteAndFinalizeUnoptimizedCompilationJobs(
+ isolate, shared_info, script, &parse_info, isolate->allocator(),
+ is_compiled_scope, &finalize_unoptimized_compilation_data_list,
+ nullptr)) {
+ return FailWithPendingException(isolate, script, &parse_info, flag);
+ }
+
+ FinalizeUnoptimizedCompilation(isolate, script, flags, &compile_state,
+ finalize_unoptimized_compilation_data_list);
+
+ DCHECK(!isolate->has_pending_exception());
+ DCHECK(is_compiled_scope->is_compiled());
+ return true;
+}
+
+// static
+bool Compiler::Compile(Handle<JSFunction> function, ClearExceptionFlag flag,
+ IsCompiledScope* is_compiled_scope) {
+ // We should never reach here if the function is already compiled or
+ // optimized.
+ DCHECK(!function->is_compiled());
+ DCHECK(!function->HasOptimizationMarker());
+ DCHECK(!function->HasAvailableOptimizedCode());
+
+ // Reset the JSFunction if we are recompiling due to the bytecode having been
+ // flushed.
+ function->ResetIfBytecodeFlushed();
+
+ Isolate* isolate = function->GetIsolate();
+ Handle<SharedFunctionInfo> shared_info = handle(function->shared(), isolate);
+
+ // Ensure shared function info is compiled.
+ *is_compiled_scope = shared_info->is_compiled_scope(isolate);
+ if (!is_compiled_scope->is_compiled() &&
+ !Compile(shared_info, flag, is_compiled_scope)) {
+ return false;
+ }
+ DCHECK(is_compiled_scope->is_compiled());
+ Handle<Code> code = handle(shared_info->GetCode(), isolate);
+
+ // Initialize the feedback cell for this JSFunction.
+ JSFunction::InitializeFeedbackCell(function, is_compiled_scope);
+
+ // Optimize now if --always-opt is enabled.
+ if (FLAG_always_opt && !function->shared().HasAsmWasmData()) {
+ CompilerTracer::TraceOptimizeForAlwaysOpt(isolate, function,
+ CodeKindForTopTier());
+
+ Handle<Code> maybe_code;
+ if (GetOptimizedCode(function, ConcurrencyMode::kNotConcurrent,
+ CodeKindForTopTier())
+ .ToHandle(&maybe_code)) {
+ code = maybe_code;
+ }
+ }
+
+ // Install code on closure.
+ function->set_code(*code);
+
+ // Check postconditions on success.
+ DCHECK(!isolate->has_pending_exception());
+ DCHECK(function->shared().is_compiled());
+ DCHECK(function->is_compiled());
+ return true;
+}
+
+// static
+bool Compiler::FinalizeBackgroundCompileTask(
+ BackgroundCompileTask* task, Handle<SharedFunctionInfo> shared_info,
+ Isolate* isolate, ClearExceptionFlag flag) {
+ DCHECK(!FLAG_finalize_streaming_on_background);
+
+ TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.compile"),
+ "V8.FinalizeBackgroundCompileTask");
+ RuntimeCallTimerScope runtimeTimer(
+ isolate, RuntimeCallCounterId::kCompileFinalizeBackgroundCompileTask);
+ HandleScope scope(isolate);
+ ParseInfo* parse_info = task->info();
+ DCHECK(!parse_info->flags().is_toplevel());
+ DCHECK(!shared_info->is_compiled());
+
+ Handle<Script> script(Script::cast(shared_info->script()), isolate);
+ parse_info->CheckFlagsForFunctionFromScript(*script);
+
+ task->parser()->UpdateStatistics(isolate, script);
+ task->parser()->HandleSourceURLComments(isolate, script);
+
+ if (task->compilation_jobs()->empty()) {
+ // Parsing or compile failed on background thread - report error messages.
+ return FailWithPendingException(isolate, script, parse_info, flag);
+ }
+
+ // Parsing has succeeded - finalize compilation.
+ parse_info->ast_value_factory()->Internalize(isolate);
+ if (!FinalizeAllUnoptimizedCompilationJobs(
+ parse_info, isolate, script, task->compilation_jobs(),
+ task->finalize_unoptimized_compilation_data())) {
+ // Finalization failed - throw an exception.
+ return FailWithPendingException(isolate, script, parse_info, flag);
+ }
+ FinalizeUnoptimizedCompilation(
+ isolate, script, parse_info->flags(), parse_info->state(),
+ *task->finalize_unoptimized_compilation_data());
+
+ DCHECK(!isolate->has_pending_exception());
+ DCHECK(shared_info->is_compiled());
+ return true;
+}
+
+// static
+bool Compiler::CompileOptimized(Handle<JSFunction> function,
+ ConcurrencyMode mode, CodeKind code_kind) {
+ DCHECK(CodeKindIsOptimizedJSFunction(code_kind));
+
+ // If the requested code kind is already available, do nothing.
+ if (function->HasAvailableCodeKind(code_kind)) return true;
+
+ Isolate* isolate = function->GetIsolate();
+ DCHECK(AllowCompilation::IsAllowed(isolate));
+
+ Handle<Code> code;
+ if (!GetOptimizedCode(function, mode, code_kind).ToHandle(&code)) {
+ // Optimization failed, get unoptimized code. Unoptimized code must exist
+ // already if we are optimizing.
+ DCHECK(!isolate->has_pending_exception());
+ DCHECK(function->shared().is_compiled());
+ DCHECK(function->shared().IsInterpreted());
+ code = BUILTIN_CODE(isolate, InterpreterEntryTrampoline);
+ }
+
+ if (!IsForNativeContextIndependentCachingOnly(code_kind)) {
+ function->set_code(*code);
+ }
+
+ // Check postconditions on success.
+ DCHECK(!isolate->has_pending_exception());
+ DCHECK(function->shared().is_compiled());
+ DCHECK(function->is_compiled());
+ if (UsesOptimizationMarker(code_kind)) {
+ DCHECK_IMPLIES(function->HasOptimizationMarker(),
+ function->IsInOptimizationQueue());
+ DCHECK_IMPLIES(function->HasOptimizationMarker(),
+ function->ChecksOptimizationMarker());
+ DCHECK_IMPLIES(function->IsInOptimizationQueue(),
+ mode == ConcurrencyMode::kConcurrent);
+ }
+ return true;
+}
+
+// static
+MaybeHandle<SharedFunctionInfo> Compiler::CompileForLiveEdit(
+ ParseInfo* parse_info, Handle<Script> script, Isolate* isolate) {
+ IsCompiledScope is_compiled_scope;
+ return CompileToplevel(parse_info, script, isolate, &is_compiled_scope);
+}
+
+// static
+MaybeHandle<JSFunction> Compiler::GetFunctionFromEval(
+ Handle<String> source, Handle<SharedFunctionInfo> outer_info,
+ Handle<Context> context, LanguageMode language_mode,
+ ParseRestriction restriction, int parameters_end_pos,
+ int eval_scope_position, int eval_position) {
+ Isolate* isolate = context->GetIsolate();
+ int source_length = source->length();
+ isolate->counters()->total_eval_size()->Increment(source_length);
+ isolate->counters()->total_compile_size()->Increment(source_length);
+
+ // The cache lookup key needs to be aware of the separation between the
+ // parameters and the body to prevent this valid invocation:
+ // Function("", "function anonymous(\n/**/) {\n}");
+ // from adding an entry that falsely approves this invalid invocation:
+ // Function("\n/**/) {\nfunction anonymous(", "}");
+ // The actual eval_scope_position for indirect eval and CreateDynamicFunction
+ // is unused (just 0), which means it's an available field to use to indicate
+ // this separation. But to make sure we're not causing other false hits, we
+ // negate the scope position.
+ if (restriction == ONLY_SINGLE_FUNCTION_LITERAL &&
+ parameters_end_pos != kNoSourcePosition) {
+ // use the parameters_end_pos as the eval_scope_position in the eval cache.
+ DCHECK_EQ(eval_scope_position, 0);
+ eval_scope_position = -parameters_end_pos;
+ }
+ CompilationCache* compilation_cache = isolate->compilation_cache();
+ InfoCellPair eval_result = compilation_cache->LookupEval(
+ source, outer_info, context, language_mode, eval_scope_position);
+ Handle<FeedbackCell> feedback_cell;
+ if (eval_result.has_feedback_cell()) {
+ feedback_cell = handle(eval_result.feedback_cell(), isolate);
+ }
+
+ Handle<SharedFunctionInfo> shared_info;
+ Handle<Script> script;
+ IsCompiledScope is_compiled_scope;
+ bool allow_eval_cache;
+ if (eval_result.has_shared()) {
+ shared_info = Handle<SharedFunctionInfo>(eval_result.shared(), isolate);
+ script = Handle<Script>(Script::cast(shared_info->script()), isolate);
+ is_compiled_scope = shared_info->is_compiled_scope(isolate);
+ allow_eval_cache = true;
+ } else {
+ UnoptimizedCompileFlags flags = UnoptimizedCompileFlags::ForToplevelCompile(
+ isolate, true, language_mode, REPLMode::kNo);
+ flags.set_is_eval(true);
+ flags.set_parse_restriction(restriction);
+
+ UnoptimizedCompileState compile_state(isolate);
+ ParseInfo parse_info(isolate, flags, &compile_state);
+ parse_info.set_parameters_end_pos(parameters_end_pos);
+ DCHECK(!parse_info.flags().is_module());
+
+ MaybeHandle<ScopeInfo> maybe_outer_scope_info;
+ if (!context->IsNativeContext()) {
+ maybe_outer_scope_info = handle(context->scope_info(), isolate);
+ }
+ script =
+ parse_info.CreateScript(isolate, source, kNullMaybeHandle,
+ OriginOptionsForEval(outer_info->script()));
+ script->set_eval_from_shared(*outer_info);
+ if (eval_position == kNoSourcePosition) {
+ // If the position is missing, attempt to get the code offset by
+ // walking the stack. Do not translate the code offset into source
+ // position, but store it as negative value for lazy translation.
+ StackTraceFrameIterator it(isolate);
+ if (!it.done() && it.is_javascript()) {
+ FrameSummary summary = FrameSummary::GetTop(it.javascript_frame());
+ script->set_eval_from_shared(
+ summary.AsJavaScript().function()->shared());
+ script->set_origin_options(OriginOptionsForEval(*summary.script()));
+ eval_position = -summary.code_offset();
+ } else {
+ eval_position = 0;
+ }
+ }
+ script->set_eval_from_position(eval_position);
+
+ if (!CompileToplevel(&parse_info, script, maybe_outer_scope_info, isolate,
+ &is_compiled_scope)
+ .ToHandle(&shared_info)) {
+ return MaybeHandle<JSFunction>();
+ }
+ allow_eval_cache = parse_info.allow_eval_cache();
+ }
+
+ // If caller is strict mode, the result must be in strict mode as well.
+ DCHECK(is_sloppy(language_mode) || is_strict(shared_info->language_mode()));
+
+ Handle<JSFunction> result;
+ if (eval_result.has_shared()) {
+ if (eval_result.has_feedback_cell()) {
+ result = isolate->factory()->NewFunctionFromSharedFunctionInfo(
+ shared_info, context, feedback_cell, AllocationType::kYoung);
+ } else {
+ result = isolate->factory()->NewFunctionFromSharedFunctionInfo(
+ shared_info, context, AllocationType::kYoung);
+ JSFunction::InitializeFeedbackCell(result, &is_compiled_scope);
+ if (allow_eval_cache) {
+ // Make sure to cache this result.
+ Handle<FeedbackCell> new_feedback_cell(result->raw_feedback_cell(),
+ isolate);
+ compilation_cache->PutEval(source, outer_info, context, shared_info,
+ new_feedback_cell, eval_scope_position);
+ }
+ }
+ } else {
+ result = isolate->factory()->NewFunctionFromSharedFunctionInfo(
+ shared_info, context, AllocationType::kYoung);
+ JSFunction::InitializeFeedbackCell(result, &is_compiled_scope);
+ if (allow_eval_cache) {
+ // Add the SharedFunctionInfo and the LiteralsArray to the eval cache if
+ // we didn't retrieve from there.
+ Handle<FeedbackCell> new_feedback_cell(result->raw_feedback_cell(),
+ isolate);
+ compilation_cache->PutEval(source, outer_info, context, shared_info,
+ new_feedback_cell, eval_scope_position);
+ }
+ }
+ DCHECK(is_compiled_scope.is_compiled());
+
+ return result;
+}
+
+// Check whether embedder allows code generation in this context.
+// (via v8::Isolate::SetAllowCodeGenerationFromStringsCallback)
+bool CodeGenerationFromStringsAllowed(Isolate* isolate, Handle<Context> context,
+ Handle<String> source) {
+ DCHECK(context->allow_code_gen_from_strings().IsFalse(isolate));
+ DCHECK(isolate->allow_code_gen_callback());
+
+ // Callback set. Let it decide if code generation is allowed.
+ VMState<EXTERNAL> state(isolate);
+ RuntimeCallTimerScope timer(
+ isolate, RuntimeCallCounterId::kCodeGenerationFromStringsCallbacks);
+ AllowCodeGenerationFromStringsCallback callback =
+ isolate->allow_code_gen_callback();
+ return callback(v8::Utils::ToLocal(context), v8::Utils::ToLocal(source));
+}
+
+// Check whether embedder allows code generation in this context.
+// (via v8::Isolate::SetModifyCodeGenerationFromStringsCallback
+// or v8::Isolate::SetModifyCodeGenerationFromStringsCallback2)
+bool ModifyCodeGenerationFromStrings(Isolate* isolate, Handle<Context> context,
+ Handle<i::Object>* source,
+ bool is_code_like) {
+ DCHECK(isolate->modify_code_gen_callback() ||
+ isolate->modify_code_gen_callback2());
+ DCHECK(source);
+
+ // Callback set. Run it, and use the return value as source, or block
+ // execution if it's not set.
+ VMState<EXTERNAL> state(isolate);
+ RuntimeCallTimerScope timer(
+ isolate, RuntimeCallCounterId::kCodeGenerationFromStringsCallbacks);
+ ModifyCodeGenerationFromStringsResult result =
+ isolate->modify_code_gen_callback()
+ ? isolate->modify_code_gen_callback()(v8::Utils::ToLocal(context),
+ v8::Utils::ToLocal(*source))
+ : isolate->modify_code_gen_callback2()(v8::Utils::ToLocal(context),
+ v8::Utils::ToLocal(*source),
+ is_code_like);
+ if (result.codegen_allowed && !result.modified_source.IsEmpty()) {
+ // Use the new source (which might be the same as the old source).
+ *source =
+ Utils::OpenHandle(*result.modified_source.ToLocalChecked(), false);
+ }
+ return result.codegen_allowed;
+}
+
+// Run Embedder-mandated checks before generating code from a string.
+//
+// Returns a string to be used for compilation, or a flag that an object type
+// was encountered that is neither a string, nor something the embedder knows
+// how to handle.
+//
+// Returns: (assuming: std::tie(source, unknown_object))
+// - !source.is_null(): compilation allowed, source contains the source string.
+// - unknown_object is true: compilation allowed, but we don't know how to
+// deal with source_object.
+// - source.is_null() && !unknown_object: compilation should be blocked.
+//
+// - !source_is_null() and unknown_object can't be true at the same time.
+
+// static
+std::pair<MaybeHandle<String>, bool> Compiler::ValidateDynamicCompilationSource(
+ Isolate* isolate, Handle<Context> context,
+ Handle<i::Object> original_source, bool is_code_like) {
+ // Check if the context unconditionally allows code gen from strings.
+ // allow_code_gen_from_strings can be many things, so we'll always check
+ // against the 'false' literal, so that e.g. undefined and 'true' are treated
+ // the same.
+ if (!context->allow_code_gen_from_strings().IsFalse(isolate) &&
+ original_source->IsString()) {
+ return {Handle<String>::cast(original_source), false};
+ }
+
+ // Check if the context allows code generation for this string.
+ // allow_code_gen_callback only allows proper strings.
+ // (I.e., let allow_code_gen_callback decide, if it has been set.)
+ if (isolate->allow_code_gen_callback()) {
+ // If we run into this condition, the embedder has marked some object
+ // templates as "code like", but has given us a callback that only accepts
+ // strings. That makes no sense.
+ DCHECK(!original_source->IsCodeLike(isolate));
+
+ if (!original_source->IsString()) {
+ return {MaybeHandle<String>(), true};
+ }
+ Handle<String> string_source = Handle<String>::cast(original_source);
+ if (!CodeGenerationFromStringsAllowed(isolate, context, string_source)) {
+ return {MaybeHandle<String>(), false};
+ }
+ return {string_source, false};
+ }
+
+ // Check if the context wants to block or modify this source object.
+ // Double-check that we really have a string now.
+ // (Let modify_code_gen_callback decide, if it's been set.)
+ if (isolate->modify_code_gen_callback() ||
+ isolate->modify_code_gen_callback2()) {
+ Handle<i::Object> modified_source = original_source;
+ if (!ModifyCodeGenerationFromStrings(isolate, context, &modified_source,
+ is_code_like)) {
+ return {MaybeHandle<String>(), false};
+ }
+ if (!modified_source->IsString()) {
+ return {MaybeHandle<String>(), true};
+ }
+ return {Handle<String>::cast(modified_source), false};
+ }
+
+ if (!context->allow_code_gen_from_strings().IsFalse(isolate) &&
+ original_source->IsCodeLike(isolate)) {
+ // Codegen is unconditionally allowed, and we're been given a CodeLike
+ // object. Stringify.
+ MaybeHandle<String> stringified_source =
+ Object::ToString(isolate, original_source);
+ return {stringified_source, stringified_source.is_null()};
+ }
+
+ // If unconditional codegen was disabled, and no callback defined, we block
+ // strings and allow all other objects.
+ return {MaybeHandle<String>(), !original_source->IsString()};
+}
+
+// static
+MaybeHandle<JSFunction> Compiler::GetFunctionFromValidatedString(
+ Handle<Context> context, MaybeHandle<String> source,
+ ParseRestriction restriction, int parameters_end_pos) {
+ Isolate* const isolate = context->GetIsolate();
+ Handle<Context> native_context(context->native_context(), isolate);
+
+ // Raise an EvalError if we did not receive a string.
+ if (source.is_null()) {
+ Handle<Object> error_message =
+ native_context->ErrorMessageForCodeGenerationFromStrings();
+ THROW_NEW_ERROR(
+ isolate,
+ NewEvalError(MessageTemplate::kCodeGenFromStrings, error_message),
+ JSFunction);
+ }
+
+ // Compile source string in the native context.
+ int eval_scope_position = 0;
+ int eval_position = kNoSourcePosition;
+ Handle<SharedFunctionInfo> outer_info(
+ native_context->empty_function().shared(), isolate);
+ return Compiler::GetFunctionFromEval(source.ToHandleChecked(), outer_info,
+ native_context, LanguageMode::kSloppy,
+ restriction, parameters_end_pos,
+ eval_scope_position, eval_position);
+}
+
+// static
+MaybeHandle<JSFunction> Compiler::GetFunctionFromString(
+ Handle<Context> context, Handle<Object> source,
+ ParseRestriction restriction, int parameters_end_pos, bool is_code_like) {
+ Isolate* const isolate = context->GetIsolate();
+ MaybeHandle<String> validated_source =
+ ValidateDynamicCompilationSource(isolate, context, source, is_code_like)
+ .first;
+ return GetFunctionFromValidatedString(context, validated_source, restriction,
+ parameters_end_pos);
+}
+
+namespace {
+
+struct ScriptCompileTimerScope {
+ public:
+ // TODO(leszeks): There are too many blink-specific entries in this enum,
+ // figure out a way to push produce/hit-isolate-cache/consume/consume-failed
+ // back up the API and log them in blink instead.
+ enum class CacheBehaviour {
+ kProduceCodeCache,
+ kHitIsolateCacheWhenNoCache,
+ kConsumeCodeCache,
+ kConsumeCodeCacheFailed,
+ kNoCacheBecauseInlineScript,
+ kNoCacheBecauseScriptTooSmall,
+ kNoCacheBecauseCacheTooCold,
+ kNoCacheNoReason,
+ kNoCacheBecauseNoResource,
+ kNoCacheBecauseInspector,
+ kNoCacheBecauseCachingDisabled,
+ kNoCacheBecauseModule,
+ kNoCacheBecauseStreamingSource,
+ kNoCacheBecauseV8Extension,
+ kHitIsolateCacheWhenProduceCodeCache,
+ kHitIsolateCacheWhenConsumeCodeCache,
+ kNoCacheBecauseExtensionModule,
+ kNoCacheBecausePacScript,
+ kNoCacheBecauseInDocumentWrite,
+ kNoCacheBecauseResourceWithNoCacheHandler,
+ kHitIsolateCacheWhenStreamingSource,
+ kCount
+ };
+
+ explicit ScriptCompileTimerScope(
+ Isolate* isolate, ScriptCompiler::NoCacheReason no_cache_reason)
+ : isolate_(isolate),
+ all_scripts_histogram_scope_(isolate->counters()->compile_script(),
+ true),
+ no_cache_reason_(no_cache_reason),
+ hit_isolate_cache_(false),
+ producing_code_cache_(false),
+ consuming_code_cache_(false),
+ consuming_code_cache_failed_(false) {}
+
+ ~ScriptCompileTimerScope() {
+ CacheBehaviour cache_behaviour = GetCacheBehaviour();
+
+ Histogram* cache_behaviour_histogram =
+ isolate_->counters()->compile_script_cache_behaviour();
+ // Sanity check that the histogram has exactly one bin per enum entry.
+ DCHECK_EQ(0, cache_behaviour_histogram->min());
+ DCHECK_EQ(static_cast<int>(CacheBehaviour::kCount),
+ cache_behaviour_histogram->max() + 1);
+ DCHECK_EQ(static_cast<int>(CacheBehaviour::kCount),
+ cache_behaviour_histogram->num_buckets());
+ cache_behaviour_histogram->AddSample(static_cast<int>(cache_behaviour));
+
+ histogram_scope_.set_histogram(
+ GetCacheBehaviourTimedHistogram(cache_behaviour));
+ }
+
+ void set_hit_isolate_cache() { hit_isolate_cache_ = true; }
+
+ void set_producing_code_cache() { producing_code_cache_ = true; }
+
+ void set_consuming_code_cache() { consuming_code_cache_ = true; }
+
+ void set_consuming_code_cache_failed() {
+ consuming_code_cache_failed_ = true;
+ }
+
+ private:
+ Isolate* isolate_;
+ LazyTimedHistogramScope histogram_scope_;
+ // TODO(leszeks): This timer is the sum of the other times, consider removing
+ // it to save space.
+ HistogramTimerScope all_scripts_histogram_scope_;
+ ScriptCompiler::NoCacheReason no_cache_reason_;
+ bool hit_isolate_cache_;
+ bool producing_code_cache_;
+ bool consuming_code_cache_;
+ bool consuming_code_cache_failed_;
+
+ CacheBehaviour GetCacheBehaviour() {
+ if (producing_code_cache_) {
+ if (hit_isolate_cache_) {
+ return CacheBehaviour::kHitIsolateCacheWhenProduceCodeCache;
+ } else {
+ return CacheBehaviour::kProduceCodeCache;
+ }
+ }
+
+ if (consuming_code_cache_) {
+ if (hit_isolate_cache_) {
+ return CacheBehaviour::kHitIsolateCacheWhenConsumeCodeCache;
+ } else if (consuming_code_cache_failed_) {
+ return CacheBehaviour::kConsumeCodeCacheFailed;
+ }
+ return CacheBehaviour::kConsumeCodeCache;
+ }
+
+ if (hit_isolate_cache_) {
+ if (no_cache_reason_ == ScriptCompiler::kNoCacheBecauseStreamingSource) {
+ return CacheBehaviour::kHitIsolateCacheWhenStreamingSource;
+ }
+ return CacheBehaviour::kHitIsolateCacheWhenNoCache;
+ }
+
+ switch (no_cache_reason_) {
+ case ScriptCompiler::kNoCacheBecauseInlineScript:
+ return CacheBehaviour::kNoCacheBecauseInlineScript;
+ case ScriptCompiler::kNoCacheBecauseScriptTooSmall:
+ return CacheBehaviour::kNoCacheBecauseScriptTooSmall;
+ case ScriptCompiler::kNoCacheBecauseCacheTooCold:
+ return CacheBehaviour::kNoCacheBecauseCacheTooCold;
+ case ScriptCompiler::kNoCacheNoReason:
+ return CacheBehaviour::kNoCacheNoReason;
+ case ScriptCompiler::kNoCacheBecauseNoResource:
+ return CacheBehaviour::kNoCacheBecauseNoResource;
+ case ScriptCompiler::kNoCacheBecauseInspector:
+ return CacheBehaviour::kNoCacheBecauseInspector;
+ case ScriptCompiler::kNoCacheBecauseCachingDisabled:
+ return CacheBehaviour::kNoCacheBecauseCachingDisabled;
+ case ScriptCompiler::kNoCacheBecauseModule:
+ return CacheBehaviour::kNoCacheBecauseModule;
+ case ScriptCompiler::kNoCacheBecauseStreamingSource:
+ return CacheBehaviour::kNoCacheBecauseStreamingSource;
+ case ScriptCompiler::kNoCacheBecauseV8Extension:
+ return CacheBehaviour::kNoCacheBecauseV8Extension;
+ case ScriptCompiler::kNoCacheBecauseExtensionModule:
+ return CacheBehaviour::kNoCacheBecauseExtensionModule;
+ case ScriptCompiler::kNoCacheBecausePacScript:
+ return CacheBehaviour::kNoCacheBecausePacScript;
+ case ScriptCompiler::kNoCacheBecauseInDocumentWrite:
+ return CacheBehaviour::kNoCacheBecauseInDocumentWrite;
+ case ScriptCompiler::kNoCacheBecauseResourceWithNoCacheHandler:
+ return CacheBehaviour::kNoCacheBecauseResourceWithNoCacheHandler;
+ case ScriptCompiler::kNoCacheBecauseDeferredProduceCodeCache: {
+ if (hit_isolate_cache_) {
+ return CacheBehaviour::kHitIsolateCacheWhenProduceCodeCache;
+ } else {
+ return CacheBehaviour::kProduceCodeCache;
+ }
+ }
+ }
+ UNREACHABLE();
+ }
+
+ TimedHistogram* GetCacheBehaviourTimedHistogram(
+ CacheBehaviour cache_behaviour) {
+ switch (cache_behaviour) {
+ case CacheBehaviour::kProduceCodeCache:
+ // Even if we hit the isolate's compilation cache, we currently recompile
+ // when we want to produce the code cache.
+ case CacheBehaviour::kHitIsolateCacheWhenProduceCodeCache:
+ return isolate_->counters()->compile_script_with_produce_cache();
+ case CacheBehaviour::kHitIsolateCacheWhenNoCache:
+ case CacheBehaviour::kHitIsolateCacheWhenConsumeCodeCache:
+ case CacheBehaviour::kHitIsolateCacheWhenStreamingSource:
+ return isolate_->counters()->compile_script_with_isolate_cache_hit();
+ case CacheBehaviour::kConsumeCodeCacheFailed:
+ return isolate_->counters()->compile_script_consume_failed();
+ case CacheBehaviour::kConsumeCodeCache:
+ return isolate_->counters()->compile_script_with_consume_cache();
+
+ // Note that this only counts the finalization part of streaming, the
+ // actual streaming compile is counted by BackgroundCompileTask into
+ // "compile_script_on_background".
+ case CacheBehaviour::kNoCacheBecauseStreamingSource:
+ return isolate_->counters()->compile_script_streaming_finalization();
+
+ case CacheBehaviour::kNoCacheBecauseInlineScript:
+ return isolate_->counters()
+ ->compile_script_no_cache_because_inline_script();
+ case CacheBehaviour::kNoCacheBecauseScriptTooSmall:
+ return isolate_->counters()
+ ->compile_script_no_cache_because_script_too_small();
+ case CacheBehaviour::kNoCacheBecauseCacheTooCold:
+ return isolate_->counters()
+ ->compile_script_no_cache_because_cache_too_cold();
+
+ // Aggregate all the other "no cache" counters into a single histogram, to
+ // save space.
+ case CacheBehaviour::kNoCacheNoReason:
+ case CacheBehaviour::kNoCacheBecauseNoResource:
+ case CacheBehaviour::kNoCacheBecauseInspector:
+ case CacheBehaviour::kNoCacheBecauseCachingDisabled:
+ // TODO(leszeks): Consider counting separately once modules are more
+ // common.
+ case CacheBehaviour::kNoCacheBecauseModule:
+ case CacheBehaviour::kNoCacheBecauseV8Extension:
+ case CacheBehaviour::kNoCacheBecauseExtensionModule:
+ case CacheBehaviour::kNoCacheBecausePacScript:
+ case CacheBehaviour::kNoCacheBecauseInDocumentWrite:
+ case CacheBehaviour::kNoCacheBecauseResourceWithNoCacheHandler:
+ return isolate_->counters()->compile_script_no_cache_other();
+
+ case CacheBehaviour::kCount:
+ UNREACHABLE();
+ }
+ UNREACHABLE();
+ }
+};
+
+void SetScriptFieldsFromDetails(Isolate* isolate, Script script,
+ Compiler::ScriptDetails script_details,
+ DisallowHeapAllocation* no_gc) {
+ Handle<Object> script_name;
+ if (script_details.name_obj.ToHandle(&script_name)) {
+ script.set_name(*script_name);
+ script.set_line_offset(script_details.line_offset);
+ script.set_column_offset(script_details.column_offset);
+ }
+ // The API can provide a source map URL, but a source map URL could also have
+ // been inferred by the parser from a magic comment. The latter takes
+ // preference over the former, so we don't want to override the source mapping
+ // URL if it already exists.
+ Handle<Object> source_map_url;
+ if (script_details.source_map_url.ToHandle(&source_map_url) &&
+ script.source_mapping_url(isolate).IsUndefined(isolate)) {
+ script.set_source_mapping_url(*source_map_url);
+ }
+ Handle<FixedArray> host_defined_options;
+ if (script_details.host_defined_options.ToHandle(&host_defined_options)) {
+ script.set_host_defined_options(*host_defined_options);
+ }
+}
+
+Handle<Script> NewScript(
+ Isolate* isolate, ParseInfo* parse_info, Handle<String> source,
+ Compiler::ScriptDetails script_details, ScriptOriginOptions origin_options,
+ NativesFlag natives,
+ MaybeHandle<FixedArray> maybe_wrapped_arguments = kNullMaybeHandle) {
+ // Create a script object describing the script to be compiled.
+ Handle<Script> script = parse_info->CreateScript(
+ isolate, source, maybe_wrapped_arguments, origin_options, natives);
+ DisallowHeapAllocation no_gc;
+ SetScriptFieldsFromDetails(isolate, *script, script_details, &no_gc);
+ LOG(isolate, ScriptDetails(*script));
+ return script;
+}
+
+MaybeHandle<SharedFunctionInfo> CompileScriptOnMainThread(
+ const UnoptimizedCompileFlags flags, Handle<String> source,
+ const Compiler::ScriptDetails& script_details,
+ ScriptOriginOptions origin_options, NativesFlag natives,
+ v8::Extension* extension, Isolate* isolate,
+ IsCompiledScope* is_compiled_scope) {
+ UnoptimizedCompileState compile_state(isolate);
+ ParseInfo parse_info(isolate, flags, &compile_state);
+ parse_info.set_extension(extension);
+
+ Handle<Script> script = NewScript(isolate, &parse_info, source,
+ script_details, origin_options, natives);
+ DCHECK_IMPLIES(parse_info.flags().collect_type_profile(),
+ script->IsUserJavaScript());
+ DCHECK_EQ(parse_info.flags().is_repl_mode(), script->is_repl_mode());
+
+ return CompileToplevel(&parse_info, script, isolate, is_compiled_scope);
+}
+
+class StressBackgroundCompileThread : public base::Thread {
+ public:
+ StressBackgroundCompileThread(Isolate* isolate, Handle<String> source)
+ : base::Thread(
+ base::Thread::Options("StressBackgroundCompileThread", 2 * i::MB)),
+ source_(source),
+ streamed_source_(std::make_unique<SourceStream>(source, isolate),
+ v8::ScriptCompiler::StreamedSource::UTF8) {
+ data()->task = std::make_unique<i::BackgroundCompileTask>(data(), isolate);
+ }
+
+ void Run() override { data()->task->Run(); }
+
+ ScriptStreamingData* data() { return streamed_source_.impl(); }
+
+ private:
+ // Dummy external source stream which returns the whole source in one go.
+ // TODO(leszeks): Also test chunking the data.
+ class SourceStream : public v8::ScriptCompiler::ExternalSourceStream {
+ public:
+ SourceStream(Handle<String> source, Isolate* isolate) : done_(false) {
+ source_buffer_ = source->ToCString(ALLOW_NULLS, FAST_STRING_TRAVERSAL,
+ &source_length_);
+ }
+
+ size_t GetMoreData(const uint8_t** src) override {
+ if (done_) {
+ return 0;
+ }
+ *src = reinterpret_cast<uint8_t*>(source_buffer_.release());
+ done_ = true;
+
+ return source_length_;
+ }
+
+ private:
+ int source_length_;
+ std::unique_ptr<char[]> source_buffer_;
+ bool done_;
+ };
+
+ Handle<String> source_;
+ v8::ScriptCompiler::StreamedSource streamed_source_;
+};
+
+bool CanBackgroundCompile(const Compiler::ScriptDetails& script_details,
+ ScriptOriginOptions origin_options,
+ v8::Extension* extension,
+ ScriptCompiler::CompileOptions compile_options,
+ NativesFlag natives) {
+ // TODO(leszeks): Remove the module check once background compilation of
+ // modules is supported.
+ return !origin_options.IsModule() && !extension &&
+ script_details.repl_mode == REPLMode::kNo &&
+ compile_options == ScriptCompiler::kNoCompileOptions &&
+ natives == NOT_NATIVES_CODE;
+}
+
+bool CompilationExceptionIsRangeError(Isolate* isolate, Handle<Object> obj) {
+ if (!obj->IsJSError(isolate)) return false;
+ Handle<JSReceiver> js_obj = Handle<JSReceiver>::cast(obj);
+ Handle<JSReceiver> constructor;
+ if (!JSReceiver::GetConstructor(js_obj).ToHandle(&constructor)) {
+ return false;
+ }
+ return *constructor == *isolate->range_error_function();
+}
+
+MaybeHandle<SharedFunctionInfo> CompileScriptOnBothBackgroundAndMainThread(
+ Handle<String> source, const Compiler::ScriptDetails& script_details,
+ ScriptOriginOptions origin_options, Isolate* isolate,
+ IsCompiledScope* is_compiled_scope) {
+ // Start a background thread compiling the script.
+ StressBackgroundCompileThread background_compile_thread(isolate, source);
+
+ UnoptimizedCompileFlags flags_copy =
+ background_compile_thread.data()->task->flags();
+
+ CHECK(background_compile_thread.Start());
+ MaybeHandle<SharedFunctionInfo> main_thread_maybe_result;
+ bool main_thread_had_stack_overflow = false;
+ // In parallel, compile on the main thread to flush out any data races.
+ {
+ IsCompiledScope inner_is_compiled_scope;
+ // The background thread should also create any relevant exceptions, so we
+ // can ignore the main-thread created ones.
+ // TODO(leszeks): Maybe verify that any thrown (or unthrown) exceptions are
+ // equivalent.
+ TryCatch ignore_try_catch(reinterpret_cast<v8::Isolate*>(isolate));
+ flags_copy.set_script_id(Script::kTemporaryScriptId);
+ main_thread_maybe_result = CompileScriptOnMainThread(
+ flags_copy, source, script_details, origin_options, NOT_NATIVES_CODE,
+ nullptr, isolate, &inner_is_compiled_scope);
+ if (main_thread_maybe_result.is_null()) {
+ // Assume all range errors are stack overflows.
+ main_thread_had_stack_overflow = CompilationExceptionIsRangeError(
+ isolate, handle(isolate->pending_exception(), isolate));
+ isolate->clear_pending_exception();
+ }
+ }
+
+ // Join with background thread and finalize compilation.
+ background_compile_thread.Join();
+ MaybeHandle<SharedFunctionInfo> maybe_result =
+ Compiler::GetSharedFunctionInfoForStreamedScript(
+ isolate, source, script_details, origin_options,
+ background_compile_thread.data());
+
+ // Either both compiles should succeed, or both should fail. The one exception
+ // to this is that the main-thread compilation might stack overflow while the
+ // background compilation doesn't, so relax the check to include this case.
+ // TODO(leszeks): Compare the contents of the results of the two compiles.
+ if (main_thread_had_stack_overflow) {
+ CHECK(main_thread_maybe_result.is_null());
+ } else {
+ CHECK_EQ(maybe_result.is_null(), main_thread_maybe_result.is_null());
+ }
+
+ Handle<SharedFunctionInfo> result;
+ if (maybe_result.ToHandle(&result)) {
+ // The BackgroundCompileTask's IsCompiledScope will keep the result alive
+ // until it dies at the end of this function, after which this new
+ // IsCompiledScope can take over.
+ *is_compiled_scope = result->is_compiled_scope(isolate);
+ }
+
+ return maybe_result;
+}
+
+} // namespace
+
+// static
+MaybeHandle<SharedFunctionInfo> Compiler::GetSharedFunctionInfoForScript(
+ Isolate* isolate, Handle<String> source,
+ const Compiler::ScriptDetails& script_details,
+ ScriptOriginOptions origin_options, v8::Extension* extension,
+ ScriptData* cached_data, ScriptCompiler::CompileOptions compile_options,
+ ScriptCompiler::NoCacheReason no_cache_reason, NativesFlag natives) {
+ ScriptCompileTimerScope compile_timer(isolate, no_cache_reason);
+
+ if (compile_options == ScriptCompiler::kNoCompileOptions ||
+ compile_options == ScriptCompiler::kEagerCompile) {
+ DCHECK_NULL(cached_data);
+ } else {
+ DCHECK(compile_options == ScriptCompiler::kConsumeCodeCache);
+ DCHECK(cached_data);
+ DCHECK_NULL(extension);
+ }
+ int source_length = source->length();
+ isolate->counters()->total_load_size()->Increment(source_length);
+ isolate->counters()->total_compile_size()->Increment(source_length);
+
+ LanguageMode language_mode = construct_language_mode(FLAG_use_strict);
+ CompilationCache* compilation_cache = isolate->compilation_cache();
+
+ // For extensions or REPL mode scripts neither do a compilation cache lookup,
+ // nor put the compilation result back into the cache.
+ const bool use_compilation_cache =
+ extension == nullptr && script_details.repl_mode == REPLMode::kNo;
+ MaybeHandle<SharedFunctionInfo> maybe_result;
+ IsCompiledScope is_compiled_scope;
+ if (use_compilation_cache) {
+ bool can_consume_code_cache =
+ compile_options == ScriptCompiler::kConsumeCodeCache;
+ if (can_consume_code_cache) {
+ compile_timer.set_consuming_code_cache();
+ }
+
+ // First check per-isolate compilation cache.
+ maybe_result = compilation_cache->LookupScript(
+ source, script_details.name_obj, script_details.line_offset,
+ script_details.column_offset, origin_options, isolate->native_context(),
+ language_mode);
+ if (!maybe_result.is_null()) {
+ compile_timer.set_hit_isolate_cache();
+ } else if (can_consume_code_cache) {
+ compile_timer.set_consuming_code_cache();
+ // Then check cached code provided by embedder.
+ HistogramTimerScope timer(isolate->counters()->compile_deserialize());
+ RuntimeCallTimerScope runtimeTimer(
+ isolate, RuntimeCallCounterId::kCompileDeserialize);
+ TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.compile"),
+ "V8.CompileDeserialize");
+ Handle<SharedFunctionInfo> inner_result;
+ if (CodeSerializer::Deserialize(isolate, cached_data, source,
+ origin_options)
+ .ToHandle(&inner_result) &&
+ inner_result->is_compiled()) {
+ // Promote to per-isolate compilation cache.
+ is_compiled_scope = inner_result->is_compiled_scope(isolate);
+ DCHECK(is_compiled_scope.is_compiled());
+ compilation_cache->PutScript(source, isolate->native_context(),
+ language_mode, inner_result);
+ Handle<Script> script(Script::cast(inner_result->script()), isolate);
+ maybe_result = inner_result;
+ } else {
+ // Deserializer failed. Fall through to compile.
+ compile_timer.set_consuming_code_cache_failed();
+ }
+ }
+ }
+
+ if (maybe_result.is_null()) {
+ // No cache entry found compile the script.
+ if (FLAG_stress_background_compile &&
+ CanBackgroundCompile(script_details, origin_options, extension,
+ compile_options, natives)) {
+ // If the --stress-background-compile flag is set, do the actual
+ // compilation on a background thread, and wait for its result.
+ maybe_result = CompileScriptOnBothBackgroundAndMainThread(
+ source, script_details, origin_options, isolate, &is_compiled_scope);
+ } else {
+ UnoptimizedCompileFlags flags =
+ UnoptimizedCompileFlags::ForToplevelCompile(
+ isolate, natives == NOT_NATIVES_CODE, language_mode,
+ script_details.repl_mode);
+
+ flags.set_is_eager(compile_options == ScriptCompiler::kEagerCompile);
+ flags.set_is_module(origin_options.IsModule());
+
+ maybe_result = CompileScriptOnMainThread(
+ flags, source, script_details, origin_options, natives, extension,
+ isolate, &is_compiled_scope);
+ }
+
+ // Add the result to the isolate cache.
+ Handle<SharedFunctionInfo> result;
+ if (use_compilation_cache && maybe_result.ToHandle(&result)) {
+ DCHECK(is_compiled_scope.is_compiled());
+ compilation_cache->PutScript(source, isolate->native_context(),
+ language_mode, result);
+ } else if (maybe_result.is_null() && natives != EXTENSION_CODE) {
+ isolate->ReportPendingMessages();
+ }
+ }
+
+ return maybe_result;
+}
+
+// static
+MaybeHandle<JSFunction> Compiler::GetWrappedFunction(
+ Handle<String> source, Handle<FixedArray> arguments,
+ Handle<Context> context, const Compiler::ScriptDetails& script_details,
+ ScriptOriginOptions origin_options, ScriptData* cached_data,
+ v8::ScriptCompiler::CompileOptions compile_options,
+ v8::ScriptCompiler::NoCacheReason no_cache_reason) {
+ Isolate* isolate = context->GetIsolate();
+ ScriptCompileTimerScope compile_timer(isolate, no_cache_reason);
+
+ if (compile_options == ScriptCompiler::kNoCompileOptions ||
+ compile_options == ScriptCompiler::kEagerCompile) {
+ DCHECK_NULL(cached_data);
+ } else {
+ DCHECK(compile_options == ScriptCompiler::kConsumeCodeCache);
+ DCHECK(cached_data);
+ }
+
+ int source_length = source->length();
+ isolate->counters()->total_compile_size()->Increment(source_length);
+
+ LanguageMode language_mode = construct_language_mode(FLAG_use_strict);
+
+ MaybeHandle<SharedFunctionInfo> maybe_result;
+ bool can_consume_code_cache =
+ compile_options == ScriptCompiler::kConsumeCodeCache;
+ if (can_consume_code_cache) {
+ compile_timer.set_consuming_code_cache();
+ // Then check cached code provided by embedder.
+ HistogramTimerScope timer(isolate->counters()->compile_deserialize());
+ RuntimeCallTimerScope runtimeTimer(
+ isolate, RuntimeCallCounterId::kCompileDeserialize);
+ TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.compile"),
+ "V8.CompileDeserialize");
+ maybe_result = CodeSerializer::Deserialize(isolate, cached_data, source,
+ origin_options);
+ if (maybe_result.is_null()) {
+ // Deserializer failed. Fall through to compile.
+ compile_timer.set_consuming_code_cache_failed();
+ }
+ }
+
+ Handle<SharedFunctionInfo> wrapped;
+ Handle<Script> script;
+ IsCompiledScope is_compiled_scope;
+ if (!maybe_result.ToHandle(&wrapped)) {
+ UnoptimizedCompileFlags flags = UnoptimizedCompileFlags::ForToplevelCompile(
+ isolate, true, language_mode, script_details.repl_mode);
+ flags.set_is_eval(true); // Use an eval scope as declaration scope.
+ flags.set_function_syntax_kind(FunctionSyntaxKind::kWrapped);
+ // TODO(delphick): Remove this and instead make the wrapped and wrapper
+ // functions fully non-lazy instead thus preventing source positions from
+ // being omitted.
+ flags.set_collect_source_positions(true);
+ // flags.set_eager(compile_options == ScriptCompiler::kEagerCompile);
+
+ UnoptimizedCompileState compile_state(isolate);
+ ParseInfo parse_info(isolate, flags, &compile_state);
+
+ MaybeHandle<ScopeInfo> maybe_outer_scope_info;
+ if (!context->IsNativeContext()) {
+ maybe_outer_scope_info = handle(context->scope_info(), isolate);
+ }
+
+ script = NewScript(isolate, &parse_info, source, script_details,
+ origin_options, NOT_NATIVES_CODE, arguments);
+
+ Handle<SharedFunctionInfo> top_level;
+ maybe_result = CompileToplevel(&parse_info, script, maybe_outer_scope_info,
+ isolate, &is_compiled_scope);
+ if (maybe_result.is_null()) isolate->ReportPendingMessages();
+ ASSIGN_RETURN_ON_EXCEPTION(isolate, top_level, maybe_result, JSFunction);
+
+ SharedFunctionInfo::ScriptIterator infos(isolate, *script);
+ for (SharedFunctionInfo info = infos.Next(); !info.is_null();
+ info = infos.Next()) {
+ if (info.is_wrapped()) {
+ wrapped = Handle<SharedFunctionInfo>(info, isolate);
+ break;
+ }
+ }
+ DCHECK(!wrapped.is_null());
+ } else {
+ is_compiled_scope = wrapped->is_compiled_scope(isolate);
+ script = Handle<Script>(Script::cast(wrapped->script()), isolate);
+ }
+ DCHECK(is_compiled_scope.is_compiled());
+
+ return isolate->factory()->NewFunctionFromSharedFunctionInfo(
+ wrapped, context, AllocationType::kYoung);
+}
+
+// static
+MaybeHandle<SharedFunctionInfo>
+Compiler::GetSharedFunctionInfoForStreamedScript(
+ Isolate* isolate, Handle<String> source,
+ const ScriptDetails& script_details, ScriptOriginOptions origin_options,
+ ScriptStreamingData* streaming_data) {
+ DCHECK(!origin_options.IsModule());
+ DCHECK(!origin_options.IsWasm());
+
+ ScriptCompileTimerScope compile_timer(
+ isolate, ScriptCompiler::kNoCacheBecauseStreamingSource);
+ PostponeInterruptsScope postpone(isolate);
+
+ int source_length = source->length();
+ isolate->counters()->total_load_size()->Increment(source_length);
+ isolate->counters()->total_compile_size()->Increment(source_length);
+
+ BackgroundCompileTask* task = streaming_data->task.get();
+
+ MaybeHandle<SharedFunctionInfo> maybe_result;
+ // Check if compile cache already holds the SFI, if so no need to finalize
+ // the code compiled on the background thread.
+ CompilationCache* compilation_cache = isolate->compilation_cache();
+ {
+ TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.compile"),
+ "V8.StreamingFinalization.CheckCache");
+ maybe_result = compilation_cache->LookupScript(
+ source, script_details.name_obj, script_details.line_offset,
+ script_details.column_offset, origin_options, isolate->native_context(),
+ task->language_mode());
+ if (!maybe_result.is_null()) {
+ compile_timer.set_hit_isolate_cache();
+ }
+ }
+
+ if (maybe_result.is_null()) {
+ // No cache entry found, finalize compilation of the script and add it to
+ // the isolate cache.
+
+ Handle<Script> script;
+ if (FLAG_finalize_streaming_on_background) {
+ RuntimeCallTimerScope runtimeTimerScope(
+ isolate, RuntimeCallCounterId::kCompilePublishBackgroundFinalization);
+ TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.compile"),
+ "V8.OffThreadFinalization.Publish");
+
+ script = task->GetScript(isolate);
+
+ // We might not have been able to finalize all jobs on the background
+ // thread (e.g. asm.js jobs), so finalize those deferred jobs now.
+ if (FinalizeDeferredUnoptimizedCompilationJobs(
+ isolate, script,
+ task->jobs_to_retry_finalization_on_main_thread(),
+ task->compile_state()->pending_error_handler(),
+ task->finalize_unoptimized_compilation_data())) {
+ maybe_result = task->GetOuterFunctionSfi(isolate);
+ }
+
+ script->set_source(*source);
+ script->set_origin_options(origin_options);
+
+ // The one post-hoc fix-up: Add the script to the script list.
+ Handle<WeakArrayList> scripts = isolate->factory()->script_list();
+ scripts = WeakArrayList::Append(isolate, scripts,
+ MaybeObjectHandle::Weak(script));
+ isolate->heap()->SetRootScriptList(*scripts);
+ } else {
+ ParseInfo* parse_info = task->info();
+ DCHECK(parse_info->flags().is_toplevel());
+
+ script = parse_info->CreateScript(isolate, source, kNullMaybeHandle,
+ origin_options);
+
+ task->parser()->UpdateStatistics(isolate, script);
+ task->parser()->HandleSourceURLComments(isolate, script);
+
+ if (!task->compilation_jobs()->empty()) {
+ // Off-thread parse & compile has succeeded - finalize compilation.
+ DCHECK_NOT_NULL(parse_info->literal());
+
+ parse_info->ast_value_factory()->Internalize(isolate);
+
+ Handle<SharedFunctionInfo> shared_info =
+ CreateTopLevelSharedFunctionInfo(parse_info, script, isolate);
+ if (FinalizeAllUnoptimizedCompilationJobs(
+ parse_info, isolate, script, task->compilation_jobs(),
+ task->finalize_unoptimized_compilation_data())) {
+ maybe_result = shared_info;
+ }
+ }
+
+ if (maybe_result.is_null()) {
+ // Compilation failed - prepare to throw an exception after script
+ // fields have been set.
+ PreparePendingException(isolate, parse_info);
+ }
+ }
+
+ // Set the script fields after finalization, to keep this path the same
+ // between main-thread and off-thread finalization.
+ {
+ DisallowHeapAllocation no_gc;
+ SetScriptFieldsFromDetails(isolate, *script, script_details, &no_gc);
+ LOG(isolate, ScriptDetails(*script));
+ }
+
+ Handle<SharedFunctionInfo> result;
+ if (!maybe_result.ToHandle(&result)) {
+ FailWithPreparedPendingException(
+ isolate, script, task->compile_state()->pending_error_handler());
+ } else {
+ FinalizeUnoptimizedScriptCompilation(
+ isolate, script, task->flags(), task->compile_state(),
+ *task->finalize_unoptimized_compilation_data());
+
+ // Add compiled code to the isolate cache.
+ TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.compile"),
+ "V8.StreamingFinalization.AddToCache");
+ compilation_cache->PutScript(source, isolate->native_context(),
+ task->language_mode(), result);
+ }
+ }
+
+ TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.compile"),
+ "V8.StreamingFinalization.Release");
+ streaming_data->Release();
+ return maybe_result;
+}
+
+// static
+template <typename LocalIsolate>
+Handle<SharedFunctionInfo> Compiler::GetSharedFunctionInfo(
+ FunctionLiteral* literal, Handle<Script> script, LocalIsolate* isolate) {
+ // Precondition: code has been parsed and scopes have been analyzed.
+ MaybeHandle<SharedFunctionInfo> maybe_existing;
+
+ // Find any previously allocated shared function info for the given literal.
+ maybe_existing =
+ script->FindSharedFunctionInfo(isolate, literal->function_literal_id());
+
+ // If we found an existing shared function info, return it.
+ Handle<SharedFunctionInfo> existing;
+ if (maybe_existing.ToHandle(&existing)) {
+ // If the function has been uncompiled (bytecode flushed) it will have lost
+ // any preparsed data. If we produced preparsed data during this compile for
+ // this function, replace the uncompiled data with one that includes it.
+ if (literal->produced_preparse_data() != nullptr &&
+ existing->HasUncompiledDataWithoutPreparseData()) {
+ Handle<UncompiledData> existing_uncompiled_data =
+ handle(existing->uncompiled_data(), isolate);
+ DCHECK_EQ(literal->start_position(),
+ existing_uncompiled_data->start_position());
+ DCHECK_EQ(literal->end_position(),
+ existing_uncompiled_data->end_position());
+ // Use existing uncompiled data's inferred name as it may be more
+ // accurate than the literal we preparsed.
+ Handle<String> inferred_name =
+ handle(existing_uncompiled_data->inferred_name(), isolate);
+ Handle<PreparseData> preparse_data =
+ literal->produced_preparse_data()->Serialize(isolate);
+ Handle<UncompiledData> new_uncompiled_data =
+ isolate->factory()->NewUncompiledDataWithPreparseData(
+ inferred_name, existing_uncompiled_data->start_position(),
+ existing_uncompiled_data->end_position(), preparse_data);
+ existing->set_uncompiled_data(*new_uncompiled_data);
+ }
+ return existing;
+ }
+
+ // Allocate a shared function info object which will be compiled lazily.
+ Handle<SharedFunctionInfo> result =
+ isolate->factory()->NewSharedFunctionInfoForLiteral(literal, script,
+ false);
+ return result;
+}
+
+template Handle<SharedFunctionInfo> Compiler::GetSharedFunctionInfo(
+ FunctionLiteral* literal, Handle<Script> script, Isolate* isolate);
+template Handle<SharedFunctionInfo> Compiler::GetSharedFunctionInfo(
+ FunctionLiteral* literal, Handle<Script> script, LocalIsolate* isolate);
+
+// static
+MaybeHandle<Code> Compiler::GetOptimizedCodeForOSR(Handle<JSFunction> function,
+ BailoutId osr_offset,
+ JavaScriptFrame* osr_frame) {
+ DCHECK(!osr_offset.IsNone());
+ DCHECK_NOT_NULL(osr_frame);
+ return GetOptimizedCode(function, ConcurrencyMode::kNotConcurrent,
+ CodeKindForTopTier(), osr_offset, osr_frame);
+}
+
+// static
+bool Compiler::FinalizeOptimizedCompilationJob(OptimizedCompilationJob* job,
+ Isolate* isolate) {
+ VMState<COMPILER> state(isolate);
+ // Take ownership of the job. Deleting the job also tears down the zone.
+ std::unique_ptr<OptimizedCompilationJob> job_scope(job);
+ OptimizedCompilationInfo* compilation_info = job->compilation_info();
+
+ TimerEventScope<TimerEventRecompileSynchronous> timer(isolate);
+ RuntimeCallTimerScope runtimeTimer(
+ isolate, RuntimeCallCounterId::kOptimizeConcurrentFinalize);
+ TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.compile"),
+ "V8.OptimizeConcurrentFinalize");
+
+ Handle<SharedFunctionInfo> shared = compilation_info->shared_info();
+
+ CodeKind code_kind = compilation_info->code_kind();
+ const bool should_install_code_on_function =
+ !IsForNativeContextIndependentCachingOnly(code_kind);
+ if (should_install_code_on_function) {
+ // Reset profiler ticks, function is no longer considered hot.
+ compilation_info->closure()->feedback_vector().set_profiler_ticks(0);
+ }
+
+ DCHECK(!shared->HasBreakInfo());
+
+ // 1) Optimization on the concurrent thread may have failed.
+ // 2) The function may have already been optimized by OSR. Simply continue.
+ // Except when OSR already disabled optimization for some reason.
+ // 3) The code may have already been invalidated due to dependency change.
+ // 4) Code generation may have failed.
+ if (job->state() == CompilationJob::State::kReadyToFinalize) {
+ if (shared->optimization_disabled()) {
+ job->RetryOptimization(BailoutReason::kOptimizationDisabled);
+ } else if (job->FinalizeJob(isolate) == CompilationJob::SUCCEEDED) {
+ job->RecordCompilationStats(OptimizedCompilationJob::kConcurrent,
+ isolate);
+ job->RecordFunctionCompilation(CodeEventListener::LAZY_COMPILE_TAG,
+ isolate);
+ InsertCodeIntoOptimizedCodeCache(compilation_info);
+ InsertCodeIntoCompilationCache(isolate, compilation_info);
+ CompilerTracer::TraceCompletedJob(isolate, compilation_info);
+ if (should_install_code_on_function) {
+ compilation_info->closure()->set_code(*compilation_info->code());
+ }
+ return CompilationJob::SUCCEEDED;
+ }
+ }
+
+ DCHECK_EQ(job->state(), CompilationJob::State::kFailed);
+ CompilerTracer::TraceAbortedJob(isolate, compilation_info);
+ compilation_info->closure()->set_code(shared->GetCode());
+ // Clear the InOptimizationQueue marker, if it exists.
+ if (UsesOptimizationMarker(code_kind) &&
+ compilation_info->closure()->IsInOptimizationQueue()) {
+ compilation_info->closure()->ClearOptimizationMarker();
+ }
+ return CompilationJob::FAILED;
+}
+
+// static
+void Compiler::PostInstantiation(Handle<JSFunction> function) {
+ Isolate* isolate = function->GetIsolate();
+ Handle<SharedFunctionInfo> shared(function->shared(), isolate);
+ IsCompiledScope is_compiled_scope(shared->is_compiled_scope(isolate));
+
+ // If code is compiled to bytecode (i.e., isn't asm.js), then allocate a
+ // feedback and check for optimized code.
+ if (is_compiled_scope.is_compiled() && shared->HasBytecodeArray()) {
+ JSFunction::InitializeFeedbackCell(function, &is_compiled_scope);
+
+ Code code = function->has_feedback_vector()
+ ? function->feedback_vector().optimized_code()
+ : Code();
+ if (!code.is_null()) {
+ // Caching of optimized code enabled and optimized code found.
+ DCHECK(!code.marked_for_deoptimization());
+ DCHECK(function->shared().is_compiled());
+ function->set_code(code);
+ }
+
+ if (FLAG_always_opt && shared->allows_lazy_compilation() &&
+ !shared->optimization_disabled() &&
+ !function->HasAvailableOptimizedCode()) {
+ CompilerTracer::TraceMarkForAlwaysOpt(isolate, function);
+ JSFunction::EnsureFeedbackVector(function, &is_compiled_scope);
+ function->MarkForOptimization(ConcurrencyMode::kNotConcurrent);
+ }
+ }
+
+ if (shared->is_toplevel() || shared->is_wrapped()) {
+ // If it's a top-level script, report compilation to the debugger.
+ Handle<Script> script(Script::cast(shared->script()), isolate);
+ isolate->debug()->OnAfterCompile(script);
+ }
+}
+
+// ----------------------------------------------------------------------------
+// Implementation of ScriptStreamingData
+
+ScriptStreamingData::ScriptStreamingData(
+ std::unique_ptr<ScriptCompiler::ExternalSourceStream> source_stream,
+ ScriptCompiler::StreamedSource::Encoding encoding)
+ : source_stream(std::move(source_stream)), encoding(encoding) {}
+
+ScriptStreamingData::~ScriptStreamingData() = default;
+
+void ScriptStreamingData::Release() { task.reset(); }
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/compiler.h b/src/codegen/compiler.h
new file mode 100644
index 0000000..c599841
--- /dev/null
+++ b/src/codegen/compiler.h
@@ -0,0 +1,563 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_COMPILER_H_
+#define V8_CODEGEN_COMPILER_H_
+
+#include <forward_list>
+#include <memory>
+
+#include "src/base/platform/elapsed-timer.h"
+#include "src/codegen/bailout-reason.h"
+#include "src/common/globals.h"
+#include "src/execution/isolate.h"
+#include "src/execution/local-isolate.h"
+#include "src/handles/persistent-handles.h"
+#include "src/logging/code-events.h"
+#include "src/objects/contexts.h"
+#include "src/objects/debug-objects.h"
+#include "src/parsing/parse-info.h"
+#include "src/parsing/pending-compilation-error-handler.h"
+#include "src/utils/allocation.h"
+#include "src/zone/zone.h"
+
+namespace v8 {
+namespace internal {
+
+// Forward declarations.
+class AstRawString;
+class BackgroundCompileTask;
+class IsCompiledScope;
+class JavaScriptFrame;
+class OptimizedCompilationInfo;
+class OptimizedCompilationJob;
+class ParseInfo;
+class Parser;
+class RuntimeCallStats;
+class ScriptData;
+struct ScriptStreamingData;
+class TimedHistogram;
+class UnoptimizedCompilationInfo;
+class UnoptimizedCompilationJob;
+class WorkerThreadRuntimeCallStats;
+
+using UnoptimizedCompilationJobList =
+ std::forward_list<std::unique_ptr<UnoptimizedCompilationJob>>;
+
+// The V8 compiler API.
+//
+// This is the central hub for dispatching to the various compilers within V8.
+// Logic for which compiler to choose and how to wire compilation results into
+// the object heap should be kept inside this class.
+//
+// General strategy: Scripts are translated into anonymous functions w/o
+// parameters which then can be executed. If the source code contains other
+// functions, they might be compiled and allocated as part of the compilation
+// of the source code or deferred for lazy compilation at a later point.
+class V8_EXPORT_PRIVATE Compiler : public AllStatic {
+ public:
+ enum ClearExceptionFlag { KEEP_EXCEPTION, CLEAR_EXCEPTION };
+
+ // ===========================================================================
+ // The following family of methods ensures a given function is compiled. The
+ // general contract is that failures will be reported by returning {false},
+ // whereas successful compilation ensures the {is_compiled} predicate on the
+ // given function holds (except for live-edit, which compiles the world).
+
+ static bool Compile(Handle<SharedFunctionInfo> shared,
+ ClearExceptionFlag flag,
+ IsCompiledScope* is_compiled_scope);
+ static bool Compile(Handle<JSFunction> function, ClearExceptionFlag flag,
+ IsCompiledScope* is_compiled_scope);
+ static bool CompileOptimized(Handle<JSFunction> function,
+ ConcurrencyMode mode, CodeKind code_kind);
+
+ // Collect source positions for a function that has already been compiled to
+ // bytecode, but for which source positions were not collected (e.g. because
+ // they were not immediately needed).
+ static bool CollectSourcePositions(Isolate* isolate,
+ Handle<SharedFunctionInfo> shared);
+
+ V8_WARN_UNUSED_RESULT static MaybeHandle<SharedFunctionInfo>
+ CompileForLiveEdit(ParseInfo* parse_info, Handle<Script> script,
+ Isolate* isolate);
+
+ // Finalize and install code from previously run background compile task.
+ static bool FinalizeBackgroundCompileTask(
+ BackgroundCompileTask* task, Handle<SharedFunctionInfo> shared_info,
+ Isolate* isolate, ClearExceptionFlag flag);
+
+ // Finalize and install optimized code from previously run job.
+ static bool FinalizeOptimizedCompilationJob(OptimizedCompilationJob* job,
+ Isolate* isolate);
+
+ // Give the compiler a chance to perform low-latency initialization tasks of
+ // the given {function} on its instantiation. Note that only the runtime will
+ // offer this chance, optimized closure instantiation will not call this.
+ static void PostInstantiation(Handle<JSFunction> function);
+
+ // ===========================================================================
+ // The following family of methods instantiates new functions for scripts or
+ // function literals. The decision whether those functions will be compiled,
+ // is left to the discretion of the compiler.
+ //
+ // Please note this interface returns shared function infos. This means you
+ // need to call Factory::NewFunctionFromSharedFunctionInfo before you have a
+ // real function with a context.
+
+ // Create a (bound) function for a String source within a context for eval.
+ V8_WARN_UNUSED_RESULT static MaybeHandle<JSFunction> GetFunctionFromEval(
+ Handle<String> source, Handle<SharedFunctionInfo> outer_info,
+ Handle<Context> context, LanguageMode language_mode,
+ ParseRestriction restriction, int parameters_end_pos,
+ int eval_scope_position, int eval_position);
+
+ struct ScriptDetails {
+ ScriptDetails()
+ : line_offset(0), column_offset(0), repl_mode(REPLMode::kNo) {}
+ explicit ScriptDetails(Handle<Object> script_name)
+ : line_offset(0),
+ column_offset(0),
+ name_obj(script_name),
+ repl_mode(REPLMode::kNo) {}
+
+ int line_offset;
+ int column_offset;
+ i::MaybeHandle<i::Object> name_obj;
+ i::MaybeHandle<i::Object> source_map_url;
+ i::MaybeHandle<i::FixedArray> host_defined_options;
+ REPLMode repl_mode;
+ };
+
+ // Create a function that results from wrapping |source| in a function,
+ // with |arguments| being a list of parameters for that function.
+ V8_WARN_UNUSED_RESULT static MaybeHandle<JSFunction> GetWrappedFunction(
+ Handle<String> source, Handle<FixedArray> arguments,
+ Handle<Context> context, const ScriptDetails& script_details,
+ ScriptOriginOptions origin_options, ScriptData* cached_data,
+ v8::ScriptCompiler::CompileOptions compile_options,
+ v8::ScriptCompiler::NoCacheReason no_cache_reason);
+
+ // Create a (bound) function for a String source within a context for eval.
+ V8_WARN_UNUSED_RESULT static MaybeHandle<JSFunction> GetFunctionFromString(
+ Handle<Context> context, Handle<i::Object> source,
+ ParseRestriction restriction, int parameters_end_pos, bool is_code_like);
+
+ // Decompose GetFunctionFromString into two functions, to allow callers to
+ // deal seperately with a case of object not handled by the embedder.
+ V8_WARN_UNUSED_RESULT static std::pair<MaybeHandle<String>, bool>
+ ValidateDynamicCompilationSource(Isolate* isolate, Handle<Context> context,
+ Handle<i::Object> source_object,
+ bool is_code_like = false);
+ V8_WARN_UNUSED_RESULT static MaybeHandle<JSFunction>
+ GetFunctionFromValidatedString(Handle<Context> context,
+ MaybeHandle<String> source,
+ ParseRestriction restriction,
+ int parameters_end_pos);
+
+ // Create a shared function info object for a String source.
+ static MaybeHandle<SharedFunctionInfo> GetSharedFunctionInfoForScript(
+ Isolate* isolate, Handle<String> source,
+ const ScriptDetails& script_details, ScriptOriginOptions origin_options,
+ v8::Extension* extension, ScriptData* cached_data,
+ ScriptCompiler::CompileOptions compile_options,
+ ScriptCompiler::NoCacheReason no_cache_reason,
+ NativesFlag is_natives_code);
+
+ // Create a shared function info object for a Script source that has already
+ // been parsed and possibly compiled on a background thread while being loaded
+ // from a streamed source. On return, the data held by |streaming_data| will
+ // have been released, however the object itself isn't freed and is still
+ // owned by the caller.
+ static MaybeHandle<SharedFunctionInfo> GetSharedFunctionInfoForStreamedScript(
+ Isolate* isolate, Handle<String> source,
+ const ScriptDetails& script_details, ScriptOriginOptions origin_options,
+ ScriptStreamingData* streaming_data);
+
+ // Create a shared function info object for the given function literal
+ // node (the code may be lazily compiled).
+ template <typename LocalIsolate>
+ static Handle<SharedFunctionInfo> GetSharedFunctionInfo(
+ FunctionLiteral* node, Handle<Script> script, LocalIsolate* isolate);
+
+ // ===========================================================================
+ // The following family of methods provides support for OSR. Code generated
+ // for entry via OSR might not be suitable for normal entry, hence will be
+ // returned directly to the caller.
+ //
+ // Please note this interface is the only part dealing with {Code} objects
+ // directly. Other methods are agnostic to {Code} and can use an interpreter
+ // instead of generating JIT code for a function at all.
+
+ // Generate and return optimized code for OSR, or empty handle on failure.
+ V8_WARN_UNUSED_RESULT static MaybeHandle<Code> GetOptimizedCodeForOSR(
+ Handle<JSFunction> function, BailoutId osr_offset,
+ JavaScriptFrame* osr_frame);
+};
+
+// A base class for compilation jobs intended to run concurrent to the main
+// thread. The current state of the job can be checked using {state()}.
+class V8_EXPORT_PRIVATE CompilationJob {
+ public:
+ enum Status { SUCCEEDED, FAILED, RETRY_ON_MAIN_THREAD };
+ enum class State {
+ kReadyToPrepare,
+ kReadyToExecute,
+ kReadyToFinalize,
+ kSucceeded,
+ kFailed,
+ };
+
+ explicit CompilationJob(State initial_state) : state_(initial_state) {
+ timer_.Start();
+ }
+ virtual ~CompilationJob() = default;
+
+ State state() const { return state_; }
+
+ protected:
+ V8_WARN_UNUSED_RESULT base::TimeDelta ElapsedTime() const {
+ return timer_.Elapsed();
+ }
+
+ V8_WARN_UNUSED_RESULT Status UpdateState(Status status, State next_state) {
+ switch (status) {
+ case SUCCEEDED:
+ state_ = next_state;
+ break;
+ case FAILED:
+ state_ = State::kFailed;
+ break;
+ case RETRY_ON_MAIN_THREAD:
+ // Don't change the state, we'll re-try on the main thread.
+ break;
+ }
+ return status;
+ }
+
+ private:
+ State state_;
+ base::ElapsedTimer timer_;
+};
+
+// A base class for unoptimized compilation jobs.
+//
+// The job is split into two phases which are called in sequence on
+// different threads and with different limitations:
+// 1) ExecuteJob: Runs concurrently. No heap allocation or handle derefs.
+// 2) FinalizeJob: Runs on main thread. No dependency changes.
+//
+// Either of phases can either fail or succeed.
+class UnoptimizedCompilationJob : public CompilationJob {
+ public:
+ UnoptimizedCompilationJob(uintptr_t stack_limit, ParseInfo* parse_info,
+ UnoptimizedCompilationInfo* compilation_info)
+ : CompilationJob(State::kReadyToExecute),
+ stack_limit_(stack_limit),
+ parse_info_(parse_info),
+ compilation_info_(compilation_info) {}
+
+ // Executes the compile job. Can be called on a background thread.
+ V8_WARN_UNUSED_RESULT Status ExecuteJob();
+
+ // Finalizes the compile job. Must be called on the main thread.
+ V8_WARN_UNUSED_RESULT Status
+ FinalizeJob(Handle<SharedFunctionInfo> shared_info, Isolate* isolate);
+
+ // Finalizes the compile job. Can be called on a background thread, and might
+ // return RETRY_ON_MAIN_THREAD if the finalization can't be run on the
+ // background thread, and should instead be retried on the foreground thread.
+ V8_WARN_UNUSED_RESULT Status
+ FinalizeJob(Handle<SharedFunctionInfo> shared_info, LocalIsolate* isolate);
+
+ void RecordCompilationStats(Isolate* isolate) const;
+ void RecordFunctionCompilation(CodeEventListener::LogEventsAndTags tag,
+ Handle<SharedFunctionInfo> shared,
+ Isolate* isolate) const;
+
+ ParseInfo* parse_info() const {
+ DCHECK_NOT_NULL(parse_info_);
+ return parse_info_;
+ }
+ UnoptimizedCompilationInfo* compilation_info() const {
+ return compilation_info_;
+ }
+
+ uintptr_t stack_limit() const { return stack_limit_; }
+
+ base::TimeDelta time_taken_to_execute() const {
+ return time_taken_to_execute_;
+ }
+ base::TimeDelta time_taken_to_finalize() const {
+ return time_taken_to_finalize_;
+ }
+
+ void ClearParseInfo() { parse_info_ = nullptr; }
+
+ protected:
+ // Overridden by the actual implementation.
+ virtual Status ExecuteJobImpl() = 0;
+ virtual Status FinalizeJobImpl(Handle<SharedFunctionInfo> shared_info,
+ Isolate* isolate) = 0;
+ virtual Status FinalizeJobImpl(Handle<SharedFunctionInfo> shared_info,
+ LocalIsolate* isolate) = 0;
+
+ private:
+ uintptr_t stack_limit_;
+ ParseInfo* parse_info_;
+ UnoptimizedCompilationInfo* compilation_info_;
+ base::TimeDelta time_taken_to_execute_;
+ base::TimeDelta time_taken_to_finalize_;
+};
+
+// A base class for optimized compilation jobs.
+//
+// The job is split into three phases which are called in sequence on
+// different threads and with different limitations:
+// 1) PrepareJob: Runs on main thread. No major limitations.
+// 2) ExecuteJob: Runs concurrently. No heap allocation or handle derefs.
+// 3) FinalizeJob: Runs on main thread. No dependency changes.
+//
+// Each of the three phases can either fail or succeed.
+class OptimizedCompilationJob : public CompilationJob {
+ public:
+ OptimizedCompilationJob(OptimizedCompilationInfo* compilation_info,
+ const char* compiler_name,
+ State initial_state = State::kReadyToPrepare)
+ : CompilationJob(initial_state),
+ compilation_info_(compilation_info),
+ compiler_name_(compiler_name) {}
+
+ // Prepare the compile job. Must be called on the main thread.
+ V8_WARN_UNUSED_RESULT Status PrepareJob(Isolate* isolate);
+
+ // Executes the compile job. Can be called on a background thread if
+ // can_execute_on_background_thread() returns true.
+ V8_WARN_UNUSED_RESULT Status
+ ExecuteJob(RuntimeCallStats* stats, LocalIsolate* local_isolate = nullptr);
+
+ // Finalizes the compile job. Must be called on the main thread.
+ V8_WARN_UNUSED_RESULT Status FinalizeJob(Isolate* isolate);
+
+ // Report a transient failure, try again next time. Should only be called on
+ // optimization compilation jobs.
+ Status RetryOptimization(BailoutReason reason);
+
+ // Report a persistent failure, disable future optimization on the function.
+ // Should only be called on optimization compilation jobs.
+ Status AbortOptimization(BailoutReason reason);
+
+ enum CompilationMode { kConcurrent, kSynchronous };
+ void RecordCompilationStats(CompilationMode mode, Isolate* isolate) const;
+ void RecordFunctionCompilation(CodeEventListener::LogEventsAndTags tag,
+ Isolate* isolate) const;
+
+ OptimizedCompilationInfo* compilation_info() const {
+ return compilation_info_;
+ }
+
+ protected:
+ // Overridden by the actual implementation.
+ virtual Status PrepareJobImpl(Isolate* isolate) = 0;
+ virtual Status ExecuteJobImpl(RuntimeCallStats* stats,
+ LocalIsolate* local_heap) = 0;
+ virtual Status FinalizeJobImpl(Isolate* isolate) = 0;
+
+ private:
+ OptimizedCompilationInfo* compilation_info_;
+ base::TimeDelta time_taken_to_prepare_;
+ base::TimeDelta time_taken_to_execute_;
+ base::TimeDelta time_taken_to_finalize_;
+ const char* compiler_name_;
+};
+
+class FinalizeUnoptimizedCompilationData {
+ public:
+ FinalizeUnoptimizedCompilationData(Isolate* isolate,
+ Handle<SharedFunctionInfo> function_handle,
+ MaybeHandle<CoverageInfo> coverage_info,
+ base::TimeDelta time_taken_to_execute,
+ base::TimeDelta time_taken_to_finalize)
+ : time_taken_to_execute_(time_taken_to_execute),
+ time_taken_to_finalize_(time_taken_to_finalize),
+ function_handle_(function_handle),
+ coverage_info_(coverage_info) {}
+
+ FinalizeUnoptimizedCompilationData(LocalIsolate* isolate,
+ Handle<SharedFunctionInfo> function_handle,
+ MaybeHandle<CoverageInfo> coverage_info,
+ base::TimeDelta time_taken_to_execute,
+ base::TimeDelta time_taken_to_finalize);
+
+ Handle<SharedFunctionInfo> function_handle() const {
+ return function_handle_;
+ }
+
+ MaybeHandle<CoverageInfo> coverage_info() const { return coverage_info_; }
+
+ base::TimeDelta time_taken_to_execute() const {
+ return time_taken_to_execute_;
+ }
+ base::TimeDelta time_taken_to_finalize() const {
+ return time_taken_to_finalize_;
+ }
+
+ private:
+ base::TimeDelta time_taken_to_execute_;
+ base::TimeDelta time_taken_to_finalize_;
+ Handle<SharedFunctionInfo> function_handle_;
+ MaybeHandle<CoverageInfo> coverage_info_;
+};
+
+using FinalizeUnoptimizedCompilationDataList =
+ std::vector<FinalizeUnoptimizedCompilationData>;
+
+class DeferredFinalizationJobData {
+ public:
+ DeferredFinalizationJobData(Isolate* isolate,
+ Handle<SharedFunctionInfo> function_handle,
+ std::unique_ptr<UnoptimizedCompilationJob> job) {
+ UNREACHABLE();
+ }
+ DeferredFinalizationJobData(LocalIsolate* isolate,
+ Handle<SharedFunctionInfo> function_handle,
+ std::unique_ptr<UnoptimizedCompilationJob> job);
+
+ Handle<SharedFunctionInfo> function_handle() const {
+ return function_handle_;
+ }
+
+ UnoptimizedCompilationJob* job() const { return job_.get(); }
+
+ private:
+ Handle<SharedFunctionInfo> function_handle_;
+ std::unique_ptr<UnoptimizedCompilationJob> job_;
+};
+
+// A wrapper around a OptimizedCompilationInfo that detaches the Handles from
+// the underlying PersistentHandlesScope and stores them in info_ on
+// destruction.
+class CompilationHandleScope final {
+ public:
+ explicit CompilationHandleScope(Isolate* isolate,
+ OptimizedCompilationInfo* info)
+ : persistent_(isolate), info_(info) {}
+ ~CompilationHandleScope();
+
+ private:
+ PersistentHandlesScope persistent_;
+ OptimizedCompilationInfo* info_;
+};
+
+using DeferredFinalizationJobDataList =
+ std::vector<DeferredFinalizationJobData>;
+
+class V8_EXPORT_PRIVATE BackgroundCompileTask {
+ public:
+ // Creates a new task that when run will parse and compile the streamed
+ // script associated with |data| and can be finalized with
+ // Compiler::GetSharedFunctionInfoForStreamedScript.
+ // Note: does not take ownership of |data|.
+ BackgroundCompileTask(ScriptStreamingData* data, Isolate* isolate);
+ ~BackgroundCompileTask();
+
+ // Creates a new task that when run will parse and compile the
+ // |function_literal| and can be finalized with
+ // Compiler::FinalizeBackgroundCompileTask.
+ BackgroundCompileTask(
+ const ParseInfo* outer_parse_info, const AstRawString* function_name,
+ const FunctionLiteral* function_literal,
+ WorkerThreadRuntimeCallStats* worker_thread_runtime_stats,
+ TimedHistogram* timer, int max_stack_size);
+
+ void Run();
+
+ ParseInfo* info() {
+ DCHECK_NOT_NULL(info_);
+ return info_.get();
+ }
+ Parser* parser() { return parser_.get(); }
+ UnoptimizedCompilationJobList* compilation_jobs() {
+ return &compilation_jobs_;
+ }
+ UnoptimizedCompileFlags flags() const { return flags_; }
+ UnoptimizedCompileState* compile_state() { return &compile_state_; }
+ LanguageMode language_mode() { return language_mode_; }
+ FinalizeUnoptimizedCompilationDataList*
+ finalize_unoptimized_compilation_data() {
+ return &finalize_unoptimized_compilation_data_;
+ }
+
+ // Jobs which could not be finalized in the background task, and need to be
+ // finalized on the main thread.
+ DeferredFinalizationJobDataList* jobs_to_retry_finalization_on_main_thread() {
+ return &jobs_to_retry_finalization_on_main_thread_;
+ }
+
+ // Getters for the off-thread finalization results, that create main-thread
+ // handles to the objects.
+ MaybeHandle<SharedFunctionInfo> GetOuterFunctionSfi(Isolate* isolate);
+ Handle<Script> GetScript(Isolate* isolate);
+
+ private:
+ // Data needed for parsing, and data needed to to be passed between thread
+ // between parsing and compilation. These need to be initialized before the
+ // compilation starts.
+ UnoptimizedCompileFlags flags_;
+ UnoptimizedCompileState compile_state_;
+ std::unique_ptr<ParseInfo> info_;
+ std::unique_ptr<Parser> parser_;
+
+ // Data needed for finalizing compilation after background compilation.
+ UnoptimizedCompilationJobList compilation_jobs_;
+
+ // Data needed for merging onto the main thread after background finalization.
+ // TODO(leszeks): When these are available, the above fields are not. We
+ // should add some stricter type-safety or DCHECKs to ensure that the user of
+ // the task knows this.
+ Isolate* isolate_for_local_isolate_;
+ std::unique_ptr<PersistentHandles> persistent_handles_;
+ MaybeHandle<SharedFunctionInfo> outer_function_sfi_;
+ Handle<Script> script_;
+ IsCompiledScope is_compiled_scope_;
+ FinalizeUnoptimizedCompilationDataList finalize_unoptimized_compilation_data_;
+ DeferredFinalizationJobDataList jobs_to_retry_finalization_on_main_thread_;
+
+ // Single function data for top-level function compilation.
+ int start_position_;
+ int end_position_;
+ int function_literal_id_;
+
+ int stack_size_;
+ WorkerThreadRuntimeCallStats* worker_thread_runtime_call_stats_;
+ TimedHistogram* timer_;
+ LanguageMode language_mode_;
+
+ DISALLOW_COPY_AND_ASSIGN(BackgroundCompileTask);
+};
+
+// Contains all data which needs to be transmitted between threads for
+// background parsing and compiling and finalizing it on the main thread.
+struct ScriptStreamingData {
+ ScriptStreamingData(
+ std::unique_ptr<ScriptCompiler::ExternalSourceStream> source_stream,
+ ScriptCompiler::StreamedSource::Encoding encoding);
+ ~ScriptStreamingData();
+
+ void Release();
+
+ // Internal implementation of v8::ScriptCompiler::StreamedSource.
+ std::unique_ptr<ScriptCompiler::ExternalSourceStream> source_stream;
+ ScriptCompiler::StreamedSource::Encoding encoding;
+
+ // Task that performs background parsing and compilation.
+ std::unique_ptr<BackgroundCompileTask> task;
+
+ DISALLOW_COPY_AND_ASSIGN(ScriptStreamingData);
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_COMPILER_H_
diff --git a/src/codegen/constant-pool.cc b/src/codegen/constant-pool.cc
new file mode 100644
index 0000000..1a67678
--- /dev/null
+++ b/src/codegen/constant-pool.cc
@@ -0,0 +1,463 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/constant-pool.h"
+#include "src/codegen/assembler-arch.h"
+#include "src/codegen/assembler-inl.h"
+
+namespace v8 {
+namespace internal {
+
+#if defined(V8_TARGET_ARCH_PPC) || defined(V8_TARGET_ARCH_PPC64)
+
+ConstantPoolBuilder::ConstantPoolBuilder(int ptr_reach_bits,
+ int double_reach_bits) {
+ info_[ConstantPoolEntry::INTPTR].entries.reserve(64);
+ info_[ConstantPoolEntry::INTPTR].regular_reach_bits = ptr_reach_bits;
+ info_[ConstantPoolEntry::DOUBLE].regular_reach_bits = double_reach_bits;
+}
+
+ConstantPoolEntry::Access ConstantPoolBuilder::NextAccess(
+ ConstantPoolEntry::Type type) const {
+ const PerTypeEntryInfo& info = info_[type];
+
+ if (info.overflow()) return ConstantPoolEntry::OVERFLOWED;
+
+ int dbl_count = info_[ConstantPoolEntry::DOUBLE].regular_count;
+ int dbl_offset = dbl_count * kDoubleSize;
+ int ptr_count = info_[ConstantPoolEntry::INTPTR].regular_count;
+ int ptr_offset = ptr_count * kSystemPointerSize + dbl_offset;
+
+ if (type == ConstantPoolEntry::DOUBLE) {
+ // Double overflow detection must take into account the reach for both types
+ int ptr_reach_bits = info_[ConstantPoolEntry::INTPTR].regular_reach_bits;
+ if (!is_uintn(dbl_offset, info.regular_reach_bits) ||
+ (ptr_count > 0 &&
+ !is_uintn(ptr_offset + kDoubleSize - kSystemPointerSize,
+ ptr_reach_bits))) {
+ return ConstantPoolEntry::OVERFLOWED;
+ }
+ } else {
+ DCHECK(type == ConstantPoolEntry::INTPTR);
+ if (!is_uintn(ptr_offset, info.regular_reach_bits)) {
+ return ConstantPoolEntry::OVERFLOWED;
+ }
+ }
+
+ return ConstantPoolEntry::REGULAR;
+}
+
+ConstantPoolEntry::Access ConstantPoolBuilder::AddEntry(
+ ConstantPoolEntry* entry, ConstantPoolEntry::Type type) {
+ DCHECK(!emitted_label_.is_bound());
+ PerTypeEntryInfo& info = info_[type];
+ const int entry_size = ConstantPoolEntry::size(type);
+ bool merged = false;
+
+ if (entry->sharing_ok()) {
+ // Try to merge entries
+ std::vector<ConstantPoolEntry>::iterator it = info.shared_entries.begin();
+ int end = static_cast<int>(info.shared_entries.size());
+ for (int i = 0; i < end; i++, it++) {
+ if ((entry_size == kSystemPointerSize)
+ ? entry->value() == it->value()
+ : entry->value64() == it->value64()) {
+ // Merge with found entry.
+ entry->set_merged_index(i);
+ merged = true;
+ break;
+ }
+ }
+ }
+
+ // By definition, merged entries have regular access.
+ DCHECK(!merged || entry->merged_index() < info.regular_count);
+ ConstantPoolEntry::Access access =
+ (merged ? ConstantPoolEntry::REGULAR : NextAccess(type));
+
+ // Enforce an upper bound on search time by limiting the search to
+ // unique sharable entries which fit in the regular section.
+ if (entry->sharing_ok() && !merged && access == ConstantPoolEntry::REGULAR) {
+ info.shared_entries.push_back(*entry);
+ } else {
+ info.entries.push_back(*entry);
+ }
+
+ // We're done if we found a match or have already triggered the
+ // overflow state.
+ if (merged || info.overflow()) return access;
+
+ if (access == ConstantPoolEntry::REGULAR) {
+ info.regular_count++;
+ } else {
+ info.overflow_start = static_cast<int>(info.entries.size()) - 1;
+ }
+
+ return access;
+}
+
+void ConstantPoolBuilder::EmitSharedEntries(Assembler* assm,
+ ConstantPoolEntry::Type type) {
+ PerTypeEntryInfo& info = info_[type];
+ std::vector<ConstantPoolEntry>& shared_entries = info.shared_entries;
+ const int entry_size = ConstantPoolEntry::size(type);
+ int base = emitted_label_.pos();
+ DCHECK_GT(base, 0);
+ int shared_end = static_cast<int>(shared_entries.size());
+ std::vector<ConstantPoolEntry>::iterator shared_it = shared_entries.begin();
+ for (int i = 0; i < shared_end; i++, shared_it++) {
+ int offset = assm->pc_offset() - base;
+ shared_it->set_offset(offset); // Save offset for merged entries.
+ if (entry_size == kSystemPointerSize) {
+ assm->dp(shared_it->value());
+ } else {
+ assm->dq(shared_it->value64());
+ }
+ DCHECK(is_uintn(offset, info.regular_reach_bits));
+
+ // Patch load sequence with correct offset.
+ assm->PatchConstantPoolAccessInstruction(shared_it->position(), offset,
+ ConstantPoolEntry::REGULAR, type);
+ }
+}
+
+void ConstantPoolBuilder::EmitGroup(Assembler* assm,
+ ConstantPoolEntry::Access access,
+ ConstantPoolEntry::Type type) {
+ PerTypeEntryInfo& info = info_[type];
+ const bool overflow = info.overflow();
+ std::vector<ConstantPoolEntry>& entries = info.entries;
+ std::vector<ConstantPoolEntry>& shared_entries = info.shared_entries;
+ const int entry_size = ConstantPoolEntry::size(type);
+ int base = emitted_label_.pos();
+ DCHECK_GT(base, 0);
+ int begin;
+ int end;
+
+ if (access == ConstantPoolEntry::REGULAR) {
+ // Emit any shared entries first
+ EmitSharedEntries(assm, type);
+ }
+
+ if (access == ConstantPoolEntry::REGULAR) {
+ begin = 0;
+ end = overflow ? info.overflow_start : static_cast<int>(entries.size());
+ } else {
+ DCHECK(access == ConstantPoolEntry::OVERFLOWED);
+ if (!overflow) return;
+ begin = info.overflow_start;
+ end = static_cast<int>(entries.size());
+ }
+
+ std::vector<ConstantPoolEntry>::iterator it = entries.begin();
+ if (begin > 0) std::advance(it, begin);
+ for (int i = begin; i < end; i++, it++) {
+ // Update constant pool if necessary and get the entry's offset.
+ int offset;
+ ConstantPoolEntry::Access entry_access;
+ if (!it->is_merged()) {
+ // Emit new entry
+ offset = assm->pc_offset() - base;
+ entry_access = access;
+ if (entry_size == kSystemPointerSize) {
+ assm->dp(it->value());
+ } else {
+ assm->dq(it->value64());
+ }
+ } else {
+ // Retrieve offset from shared entry.
+ offset = shared_entries[it->merged_index()].offset();
+ entry_access = ConstantPoolEntry::REGULAR;
+ }
+
+ DCHECK(entry_access == ConstantPoolEntry::OVERFLOWED ||
+ is_uintn(offset, info.regular_reach_bits));
+
+ // Patch load sequence with correct offset.
+ assm->PatchConstantPoolAccessInstruction(it->position(), offset,
+ entry_access, type);
+ }
+}
+
+// Emit and return size of pool.
+int ConstantPoolBuilder::Emit(Assembler* assm) {
+ bool emitted = emitted_label_.is_bound();
+ bool empty = IsEmpty();
+
+ if (!emitted) {
+ // Mark start of constant pool. Align if necessary.
+ if (!empty) assm->DataAlign(kDoubleSize);
+ assm->bind(&emitted_label_);
+ if (!empty) {
+ // Emit in groups based on access and type.
+ // Emit doubles first for alignment purposes.
+ EmitGroup(assm, ConstantPoolEntry::REGULAR, ConstantPoolEntry::DOUBLE);
+ EmitGroup(assm, ConstantPoolEntry::REGULAR, ConstantPoolEntry::INTPTR);
+ if (info_[ConstantPoolEntry::DOUBLE].overflow()) {
+ assm->DataAlign(kDoubleSize);
+ EmitGroup(assm, ConstantPoolEntry::OVERFLOWED,
+ ConstantPoolEntry::DOUBLE);
+ }
+ if (info_[ConstantPoolEntry::INTPTR].overflow()) {
+ EmitGroup(assm, ConstantPoolEntry::OVERFLOWED,
+ ConstantPoolEntry::INTPTR);
+ }
+ }
+ }
+
+ return !empty ? (assm->pc_offset() - emitted_label_.pos()) : 0;
+}
+
+#endif // defined(V8_TARGET_ARCH_PPC) || defined(V8_TARGET_ARCH_PPC64)
+
+#if defined(V8_TARGET_ARCH_ARM64)
+
+// Constant Pool.
+
+ConstantPool::ConstantPool(Assembler* assm) : assm_(assm) {}
+ConstantPool::~ConstantPool() { DCHECK_EQ(blocked_nesting_, 0); }
+
+RelocInfoStatus ConstantPool::RecordEntry(uint32_t data,
+ RelocInfo::Mode rmode) {
+ ConstantPoolKey key(data, rmode);
+ CHECK(key.is_value32());
+ return RecordKey(std::move(key), assm_->pc_offset());
+}
+
+RelocInfoStatus ConstantPool::RecordEntry(uint64_t data,
+ RelocInfo::Mode rmode) {
+ ConstantPoolKey key(data, rmode);
+ CHECK(!key.is_value32());
+ return RecordKey(std::move(key), assm_->pc_offset());
+}
+
+RelocInfoStatus ConstantPool::RecordKey(ConstantPoolKey key, int offset) {
+ RelocInfoStatus write_reloc_info = GetRelocInfoStatusFor(key);
+ if (write_reloc_info == RelocInfoStatus::kMustRecord) {
+ if (key.is_value32()) {
+ if (entry32_count_ == 0) first_use_32_ = offset;
+ ++entry32_count_;
+ } else {
+ if (entry64_count_ == 0) first_use_64_ = offset;
+ ++entry64_count_;
+ }
+ }
+ entries_.insert(std::make_pair(key, offset));
+
+ if (Entry32Count() + Entry64Count() > ConstantPool::kApproxMaxEntryCount) {
+ // Request constant pool emission after the next instruction.
+ SetNextCheckIn(1);
+ }
+
+ return write_reloc_info;
+}
+
+RelocInfoStatus ConstantPool::GetRelocInfoStatusFor(
+ const ConstantPoolKey& key) {
+ if (key.AllowsDeduplication()) {
+ auto existing = entries_.find(key);
+ if (existing != entries_.end()) {
+ return RelocInfoStatus::kMustOmitForDuplicate;
+ }
+ }
+ return RelocInfoStatus::kMustRecord;
+}
+
+void ConstantPool::EmitAndClear(Jump require_jump) {
+ DCHECK(!IsBlocked());
+ // Prevent recursive pool emission.
+ Assembler::BlockPoolsScope block_pools(assm_, PoolEmissionCheck::kSkip);
+ Alignment require_alignment =
+ IsAlignmentRequiredIfEmittedAt(require_jump, assm_->pc_offset());
+ int size = ComputeSize(require_jump, require_alignment);
+ Label size_check;
+ assm_->bind(&size_check);
+ assm_->RecordConstPool(size);
+
+ // Emit the constant pool. It is preceded by an optional branch if
+ // {require_jump} and a header which will:
+ // 1) Encode the size of the constant pool, for use by the disassembler.
+ // 2) Terminate the program, to try to prevent execution from accidentally
+ // flowing into the constant pool.
+ // 3) align the 64bit pool entries to 64-bit.
+ // TODO(all): Make the alignment part less fragile. Currently code is
+ // allocated as a byte array so there are no guarantees the alignment will
+ // be preserved on compaction. Currently it works as allocation seems to be
+ // 64-bit aligned.
+
+ Label after_pool;
+ if (require_jump == Jump::kRequired) assm_->b(&after_pool);
+
+ assm_->RecordComment("[ Constant Pool");
+ EmitPrologue(require_alignment);
+ if (require_alignment == Alignment::kRequired) assm_->Align(kInt64Size);
+ EmitEntries();
+ assm_->RecordComment("]");
+
+ if (after_pool.is_linked()) assm_->bind(&after_pool);
+
+ DCHECK_EQ(assm_->SizeOfCodeGeneratedSince(&size_check), size);
+ Clear();
+}
+
+void ConstantPool::Clear() {
+ entries_.clear();
+ first_use_32_ = -1;
+ first_use_64_ = -1;
+ entry32_count_ = 0;
+ entry64_count_ = 0;
+ next_check_ = 0;
+}
+
+void ConstantPool::StartBlock() {
+ if (blocked_nesting_ == 0) {
+ // Prevent constant pool checks from happening by setting the next check to
+ // the biggest possible offset.
+ next_check_ = kMaxInt;
+ }
+ ++blocked_nesting_;
+}
+
+void ConstantPool::EndBlock() {
+ --blocked_nesting_;
+ if (blocked_nesting_ == 0) {
+ DCHECK(IsInImmRangeIfEmittedAt(assm_->pc_offset()));
+ // Make sure a check happens quickly after getting unblocked.
+ next_check_ = 0;
+ }
+}
+
+bool ConstantPool::IsBlocked() const { return blocked_nesting_ > 0; }
+
+void ConstantPool::SetNextCheckIn(size_t instructions) {
+ next_check_ =
+ assm_->pc_offset() + static_cast<int>(instructions * kInstrSize);
+}
+
+void ConstantPool::EmitEntries() {
+ for (auto iter = entries_.begin(); iter != entries_.end();) {
+ DCHECK(iter->first.is_value32() || IsAligned(assm_->pc_offset(), 8));
+ auto range = entries_.equal_range(iter->first);
+ bool shared = iter->first.AllowsDeduplication();
+ for (auto it = range.first; it != range.second; ++it) {
+ SetLoadOffsetToConstPoolEntry(it->second, assm_->pc(), it->first);
+ if (!shared) Emit(it->first);
+ }
+ if (shared) Emit(iter->first);
+ iter = range.second;
+ }
+}
+
+void ConstantPool::Emit(const ConstantPoolKey& key) {
+ if (key.is_value32()) {
+ assm_->dd(key.value32());
+ } else {
+ assm_->dq(key.value64());
+ }
+}
+
+bool ConstantPool::ShouldEmitNow(Jump require_jump, size_t margin) const {
+ if (IsEmpty()) return false;
+ if (Entry32Count() + Entry64Count() > ConstantPool::kApproxMaxEntryCount) {
+ return true;
+ }
+ // We compute {dist32/64}, i.e. the distance from the first instruction
+ // accessing a 32bit/64bit entry in the constant pool to any of the
+ // 32bit/64bit constant pool entries, respectively. This is required because
+ // we do not guarantee that entries are emitted in order of reference, i.e. it
+ // is possible that the entry with the earliest reference is emitted last.
+ // The constant pool should be emitted if either of the following is true:
+ // (A) {dist32/64} will be out of range at the next check in.
+ // (B) Emission can be done behind an unconditional branch and {dist32/64}
+ // exceeds {kOpportunityDist*}.
+ // (C) {dist32/64} exceeds the desired approximate distance to the pool.
+ int worst_case_size = ComputeSize(Jump::kRequired, Alignment::kRequired);
+ size_t pool_end_32 = assm_->pc_offset() + margin + worst_case_size;
+ size_t pool_end_64 = pool_end_32 - Entry32Count() * kInt32Size;
+ if (Entry64Count() != 0) {
+ // The 64-bit constants are always emitted before the 32-bit constants, so
+ // we subtract the size of the 32-bit constants from {size}.
+ size_t dist64 = pool_end_64 - first_use_64_;
+ bool next_check_too_late = dist64 + 2 * kCheckInterval >= kMaxDistToPool64;
+ bool opportune_emission_without_jump =
+ require_jump == Jump::kOmitted && (dist64 >= kOpportunityDistToPool64);
+ bool approximate_distance_exceeded = dist64 >= kApproxDistToPool64;
+ if (next_check_too_late || opportune_emission_without_jump ||
+ approximate_distance_exceeded) {
+ return true;
+ }
+ }
+ if (Entry32Count() != 0) {
+ size_t dist32 = pool_end_32 - first_use_32_;
+ bool next_check_too_late = dist32 + 2 * kCheckInterval >= kMaxDistToPool32;
+ bool opportune_emission_without_jump =
+ require_jump == Jump::kOmitted && (dist32 >= kOpportunityDistToPool32);
+ bool approximate_distance_exceeded = dist32 >= kApproxDistToPool32;
+ if (next_check_too_late || opportune_emission_without_jump ||
+ approximate_distance_exceeded) {
+ return true;
+ }
+ }
+ return false;
+}
+
+int ConstantPool::ComputeSize(Jump require_jump,
+ Alignment require_alignment) const {
+ int size_up_to_marker = PrologueSize(require_jump);
+ int alignment = require_alignment == Alignment::kRequired ? kInstrSize : 0;
+ size_t size_after_marker =
+ Entry32Count() * kInt32Size + alignment + Entry64Count() * kInt64Size;
+ return size_up_to_marker + static_cast<int>(size_after_marker);
+}
+
+Alignment ConstantPool::IsAlignmentRequiredIfEmittedAt(Jump require_jump,
+ int pc_offset) const {
+ int size_up_to_marker = PrologueSize(require_jump);
+ if (Entry64Count() != 0 &&
+ !IsAligned(pc_offset + size_up_to_marker, kInt64Size)) {
+ return Alignment::kRequired;
+ }
+ return Alignment::kOmitted;
+}
+
+bool ConstantPool::IsInImmRangeIfEmittedAt(int pc_offset) {
+ // Check that all entries are in range if the pool is emitted at {pc_offset}.
+ // This ignores kPcLoadDelta (conservatively, since all offsets are positive),
+ // and over-estimates the last entry's address with the pool's end.
+ Alignment require_alignment =
+ IsAlignmentRequiredIfEmittedAt(Jump::kRequired, pc_offset);
+ size_t pool_end_32 =
+ pc_offset + ComputeSize(Jump::kRequired, require_alignment);
+ size_t pool_end_64 = pool_end_32 - Entry32Count() * kInt32Size;
+ bool entries_in_range_32 =
+ Entry32Count() == 0 || (pool_end_32 < first_use_32_ + kMaxDistToPool32);
+ bool entries_in_range_64 =
+ Entry64Count() == 0 || (pool_end_64 < first_use_64_ + kMaxDistToPool64);
+ return entries_in_range_32 && entries_in_range_64;
+}
+
+ConstantPool::BlockScope::BlockScope(Assembler* assm, size_t margin)
+ : pool_(&assm->constpool_) {
+ pool_->assm_->EmitConstPoolWithJumpIfNeeded(margin);
+ pool_->StartBlock();
+}
+
+ConstantPool::BlockScope::BlockScope(Assembler* assm, PoolEmissionCheck check)
+ : pool_(&assm->constpool_) {
+ DCHECK_EQ(check, PoolEmissionCheck::kSkip);
+ pool_->StartBlock();
+}
+
+ConstantPool::BlockScope::~BlockScope() { pool_->EndBlock(); }
+
+void ConstantPool::MaybeCheck() {
+ if (assm_->pc_offset() >= next_check_) {
+ Check(Emission::kIfNeeded, Jump::kRequired);
+ }
+}
+
+#endif // defined(V8_TARGET_ARCH_ARM64)
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/constant-pool.h b/src/codegen/constant-pool.h
new file mode 100644
index 0000000..581644b
--- /dev/null
+++ b/src/codegen/constant-pool.h
@@ -0,0 +1,352 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_CONSTANT_POOL_H_
+#define V8_CODEGEN_CONSTANT_POOL_H_
+
+#include <map>
+
+#include "src/codegen/label.h"
+#include "src/codegen/reloc-info.h"
+#include "src/common/globals.h"
+#include "src/numbers/double.h"
+
+namespace v8 {
+namespace internal {
+
+class Instruction;
+
+// -----------------------------------------------------------------------------
+// Constant pool support
+
+class ConstantPoolEntry {
+ public:
+ ConstantPoolEntry() = default;
+ ConstantPoolEntry(int position, intptr_t value, bool sharing_ok,
+ RelocInfo::Mode rmode = RelocInfo::NONE)
+ : position_(position),
+ merged_index_(sharing_ok ? SHARING_ALLOWED : SHARING_PROHIBITED),
+ value_(value),
+ rmode_(rmode) {}
+ ConstantPoolEntry(int position, Double value,
+ RelocInfo::Mode rmode = RelocInfo::NONE)
+ : position_(position),
+ merged_index_(SHARING_ALLOWED),
+ value64_(value.AsUint64()),
+ rmode_(rmode) {}
+
+ int position() const { return position_; }
+ bool sharing_ok() const { return merged_index_ != SHARING_PROHIBITED; }
+ bool is_merged() const { return merged_index_ >= 0; }
+ int merged_index() const {
+ DCHECK(is_merged());
+ return merged_index_;
+ }
+ void set_merged_index(int index) {
+ DCHECK(sharing_ok());
+ merged_index_ = index;
+ DCHECK(is_merged());
+ }
+ int offset() const {
+ DCHECK_GE(merged_index_, 0);
+ return merged_index_;
+ }
+ void set_offset(int offset) {
+ DCHECK_GE(offset, 0);
+ merged_index_ = offset;
+ }
+ intptr_t value() const { return value_; }
+ uint64_t value64() const { return value64_; }
+ RelocInfo::Mode rmode() const { return rmode_; }
+
+ enum Type { INTPTR, DOUBLE, NUMBER_OF_TYPES };
+
+ static int size(Type type) {
+ return (type == INTPTR) ? kSystemPointerSize : kDoubleSize;
+ }
+
+ enum Access { REGULAR, OVERFLOWED };
+
+ private:
+ int position_;
+ int merged_index_;
+ union {
+ intptr_t value_;
+ uint64_t value64_;
+ };
+ // TODO(leszeks): The way we use this, it could probably be packed into
+ // merged_index_ if size is a concern.
+ RelocInfo::Mode rmode_;
+ enum { SHARING_PROHIBITED = -2, SHARING_ALLOWED = -1 };
+};
+
+#if defined(V8_TARGET_ARCH_PPC) || defined(V8_TARGET_ARCH_PPC64)
+
+// -----------------------------------------------------------------------------
+// Embedded constant pool support
+
+class ConstantPoolBuilder {
+ public:
+ ConstantPoolBuilder(int ptr_reach_bits, int double_reach_bits);
+
+#ifdef DEBUG
+ ~ConstantPoolBuilder() {
+ // Unused labels to prevent DCHECK failures.
+ emitted_label_.Unuse();
+ emitted_label_.UnuseNear();
+ }
+#endif
+
+ // Add pointer-sized constant to the embedded constant pool
+ ConstantPoolEntry::Access AddEntry(int position, intptr_t value,
+ bool sharing_ok) {
+ ConstantPoolEntry entry(position, value, sharing_ok);
+ return AddEntry(&entry, ConstantPoolEntry::INTPTR);
+ }
+
+ // Add double constant to the embedded constant pool
+ ConstantPoolEntry::Access AddEntry(int position, Double value) {
+ ConstantPoolEntry entry(position, value);
+ return AddEntry(&entry, ConstantPoolEntry::DOUBLE);
+ }
+
+ // Add double constant to the embedded constant pool
+ ConstantPoolEntry::Access AddEntry(int position, double value) {
+ return AddEntry(position, Double(value));
+ }
+
+ // Previews the access type required for the next new entry to be added.
+ ConstantPoolEntry::Access NextAccess(ConstantPoolEntry::Type type) const;
+
+ bool IsEmpty() {
+ return info_[ConstantPoolEntry::INTPTR].entries.empty() &&
+ info_[ConstantPoolEntry::INTPTR].shared_entries.empty() &&
+ info_[ConstantPoolEntry::DOUBLE].entries.empty() &&
+ info_[ConstantPoolEntry::DOUBLE].shared_entries.empty();
+ }
+
+ // Emit the constant pool. Invoke only after all entries have been
+ // added and all instructions have been emitted.
+ // Returns position of the emitted pool (zero implies no constant pool).
+ int Emit(Assembler* assm);
+
+ // Returns the label associated with the start of the constant pool.
+ // Linking to this label in the function prologue may provide an
+ // efficient means of constant pool pointer register initialization
+ // on some architectures.
+ inline Label* EmittedPosition() { return &emitted_label_; }
+
+ private:
+ ConstantPoolEntry::Access AddEntry(ConstantPoolEntry* entry,
+ ConstantPoolEntry::Type type);
+ void EmitSharedEntries(Assembler* assm, ConstantPoolEntry::Type type);
+ void EmitGroup(Assembler* assm, ConstantPoolEntry::Access access,
+ ConstantPoolEntry::Type type);
+
+ struct PerTypeEntryInfo {
+ PerTypeEntryInfo() : regular_count(0), overflow_start(-1) {}
+ bool overflow() const {
+ return (overflow_start >= 0 &&
+ overflow_start < static_cast<int>(entries.size()));
+ }
+ int regular_reach_bits;
+ int regular_count;
+ int overflow_start;
+ std::vector<ConstantPoolEntry> entries;
+ std::vector<ConstantPoolEntry> shared_entries;
+ };
+
+ Label emitted_label_; // Records pc_offset of emitted pool
+ PerTypeEntryInfo info_[ConstantPoolEntry::NUMBER_OF_TYPES];
+};
+
+#endif // defined(V8_TARGET_ARCH_PPC) || defined(V8_TARGET_ARCH_PPC64)
+
+#if defined(V8_TARGET_ARCH_ARM64)
+
+class ConstantPoolKey {
+ public:
+ explicit ConstantPoolKey(uint64_t value,
+ RelocInfo::Mode rmode = RelocInfo::NONE)
+ : is_value32_(false), value64_(value), rmode_(rmode) {}
+
+ explicit ConstantPoolKey(uint32_t value,
+ RelocInfo::Mode rmode = RelocInfo::NONE)
+ : is_value32_(true), value32_(value), rmode_(rmode) {}
+
+ uint64_t value64() const {
+ CHECK(!is_value32_);
+ return value64_;
+ }
+ uint32_t value32() const {
+ CHECK(is_value32_);
+ return value32_;
+ }
+
+ bool is_value32() const { return is_value32_; }
+ RelocInfo::Mode rmode() const { return rmode_; }
+
+ bool AllowsDeduplication() const {
+ DCHECK(rmode_ != RelocInfo::CONST_POOL &&
+ rmode_ != RelocInfo::VENEER_POOL &&
+ rmode_ != RelocInfo::DEOPT_SCRIPT_OFFSET &&
+ rmode_ != RelocInfo::DEOPT_INLINING_ID &&
+ rmode_ != RelocInfo::DEOPT_REASON && rmode_ != RelocInfo::DEOPT_ID);
+ // CODE_TARGETs can be shared because they aren't patched anymore,
+ // and we make sure we emit only one reloc info for them (thus delta
+ // patching) will apply the delta only once. At the moment, we do not dedup
+ // code targets if they are wrapped in a heap object request (value == 0).
+ bool is_sharable_code_target =
+ rmode_ == RelocInfo::CODE_TARGET &&
+ (is_value32() ? (value32() != 0) : (value64() != 0));
+ bool is_sharable_embedded_object = RelocInfo::IsEmbeddedObjectMode(rmode_);
+ return RelocInfo::IsShareableRelocMode(rmode_) || is_sharable_code_target ||
+ is_sharable_embedded_object;
+ }
+
+ private:
+ bool is_value32_;
+ union {
+ uint64_t value64_;
+ uint32_t value32_;
+ };
+ RelocInfo::Mode rmode_;
+};
+
+// Order for pool entries. 64bit entries go first.
+inline bool operator<(const ConstantPoolKey& a, const ConstantPoolKey& b) {
+ if (a.is_value32() < b.is_value32()) return true;
+ if (a.is_value32() > b.is_value32()) return false;
+ if (a.rmode() < b.rmode()) return true;
+ if (a.rmode() > b.rmode()) return false;
+ if (a.is_value32()) return a.value32() < b.value32();
+ return a.value64() < b.value64();
+}
+
+inline bool operator==(const ConstantPoolKey& a, const ConstantPoolKey& b) {
+ if (a.rmode() != b.rmode() || a.is_value32() != b.is_value32()) {
+ return false;
+ }
+ if (a.is_value32()) return a.value32() == b.value32();
+ return a.value64() == b.value64();
+}
+
+// Constant pool generation
+enum class Jump { kOmitted, kRequired };
+enum class Emission { kIfNeeded, kForced };
+enum class Alignment { kOmitted, kRequired };
+enum class RelocInfoStatus { kMustRecord, kMustOmitForDuplicate };
+enum class PoolEmissionCheck { kSkip };
+
+// Pools are emitted in the instruction stream, preferably after unconditional
+// jumps or after returns from functions (in dead code locations).
+// If a long code sequence does not contain unconditional jumps, it is
+// necessary to emit the constant pool before the pool gets too far from the
+// location it is accessed from. In this case, we emit a jump over the emitted
+// constant pool.
+// Constants in the pool may be addresses of functions that gets relocated;
+// if so, a relocation info entry is associated to the constant pool entry.
+class ConstantPool {
+ public:
+ explicit ConstantPool(Assembler* assm);
+ ~ConstantPool();
+
+ // Returns true when we need to write RelocInfo and false when we do not.
+ RelocInfoStatus RecordEntry(uint32_t data, RelocInfo::Mode rmode);
+ RelocInfoStatus RecordEntry(uint64_t data, RelocInfo::Mode rmode);
+
+ size_t Entry32Count() const { return entry32_count_; }
+ size_t Entry64Count() const { return entry64_count_; }
+ bool IsEmpty() const { return entries_.empty(); }
+ // Check if pool will be out of range at {pc_offset}.
+ bool IsInImmRangeIfEmittedAt(int pc_offset);
+ // Size in bytes of the constant pool. Depending on parameters, the size will
+ // include the branch over the pool and alignment padding.
+ int ComputeSize(Jump require_jump, Alignment require_alignment) const;
+
+ // Emit the pool at the current pc with a branch over the pool if requested.
+ void EmitAndClear(Jump require);
+ bool ShouldEmitNow(Jump require_jump, size_t margin = 0) const;
+ V8_EXPORT_PRIVATE void Check(Emission force_emission, Jump require_jump,
+ size_t margin = 0);
+
+ V8_EXPORT_PRIVATE void MaybeCheck();
+ void Clear();
+
+ // Constant pool emisssion can be blocked temporarily.
+ bool IsBlocked() const;
+
+ // Repeated checking whether the constant pool should be emitted is expensive;
+ // only check once a number of instructions have been generated.
+ void SetNextCheckIn(size_t instructions);
+
+ // Class for scoping postponing the constant pool generation.
+ class V8_EXPORT_PRIVATE BlockScope {
+ public:
+ // BlockScope immediatelly emits the pool if necessary to ensure that
+ // during the block scope at least {margin} bytes can be emitted without
+ // pool emission becomming necessary.
+ explicit BlockScope(Assembler* pool, size_t margin = 0);
+ BlockScope(Assembler* pool, PoolEmissionCheck);
+ ~BlockScope();
+
+ private:
+ ConstantPool* pool_;
+ DISALLOW_IMPLICIT_CONSTRUCTORS(BlockScope);
+ };
+
+ // Hard limit to the const pool which must not be exceeded.
+ static const size_t kMaxDistToPool32;
+ static const size_t kMaxDistToPool64;
+ // Approximate distance where the pool should be emitted.
+ static const size_t kApproxDistToPool32;
+ V8_EXPORT_PRIVATE static const size_t kApproxDistToPool64;
+ // Approximate distance where the pool may be emitted if
+ // no jump is required (due to a recent unconditional jump).
+ static const size_t kOpportunityDistToPool32;
+ static const size_t kOpportunityDistToPool64;
+ // PC distance between constant pool checks.
+ V8_EXPORT_PRIVATE static const size_t kCheckInterval;
+ // Number of entries in the pool which trigger a check.
+ static const size_t kApproxMaxEntryCount;
+
+ private:
+ void StartBlock();
+ void EndBlock();
+
+ void EmitEntries();
+ void EmitPrologue(Alignment require_alignment);
+ int PrologueSize(Jump require_jump) const;
+ RelocInfoStatus RecordKey(ConstantPoolKey key, int offset);
+ RelocInfoStatus GetRelocInfoStatusFor(const ConstantPoolKey& key);
+ void Emit(const ConstantPoolKey& key);
+ void SetLoadOffsetToConstPoolEntry(int load_offset, Instruction* entry_offset,
+ const ConstantPoolKey& key);
+ Alignment IsAlignmentRequiredIfEmittedAt(Jump require_jump,
+ int pc_offset) const;
+
+ Assembler* assm_;
+ // Keep track of the first instruction requiring a constant pool entry
+ // since the previous constant pool was emitted.
+ int first_use_32_ = -1;
+ int first_use_64_ = -1;
+ // We sort not according to insertion order, but since we do not insert
+ // addresses (for heap objects we insert an index which is created in
+ // increasing order), the order is deterministic. We map each entry to the
+ // pc offset of the load. We use a multimap because we need to record the
+ // pc offset of each load of the same constant so that the immediate of the
+ // loads can be back-patched when the pool is emitted.
+ std::multimap<ConstantPoolKey, int> entries_;
+ size_t entry32_count_ = 0;
+ size_t entry64_count_ = 0;
+ int next_check_ = 0;
+ int blocked_nesting_ = 0;
+};
+
+#endif // defined(V8_TARGET_ARCH_ARM64)
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_CONSTANT_POOL_H_
diff --git a/src/codegen/constants-arch.h b/src/codegen/constants-arch.h
new file mode 100644
index 0000000..7a222c9
--- /dev/null
+++ b/src/codegen/constants-arch.h
@@ -0,0 +1,28 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_CONSTANTS_ARCH_H_
+#define V8_CODEGEN_CONSTANTS_ARCH_H_
+
+#if V8_TARGET_ARCH_ARM
+#include "src/codegen/arm/constants-arm.h" // NOLINT
+#elif V8_TARGET_ARCH_ARM64
+#include "src/codegen/arm64/constants-arm64.h" // NOLINT
+#elif V8_TARGET_ARCH_IA32
+#include "src/codegen/ia32/constants-ia32.h" // NOLINT
+#elif V8_TARGET_ARCH_MIPS
+#include "src/codegen/mips/constants-mips.h" // NOLINT
+#elif V8_TARGET_ARCH_MIPS64
+#include "src/codegen/mips64/constants-mips64.h" // NOLINT
+#elif V8_TARGET_ARCH_PPC || V8_TARGET_ARCH_PPC64
+#include "src/codegen/ppc/constants-ppc.h" // NOLINT
+#elif V8_TARGET_ARCH_S390
+#include "src/codegen/s390/constants-s390.h" // NOLINT
+#elif V8_TARGET_ARCH_X64
+#include "src/codegen/x64/constants-x64.h" // NOLINT
+#else
+#error Unsupported target architecture.
+#endif
+
+#endif // V8_CODEGEN_CONSTANTS_ARCH_H_
diff --git a/src/codegen/cpu-features.h b/src/codegen/cpu-features.h
new file mode 100644
index 0000000..eef98f7
--- /dev/null
+++ b/src/codegen/cpu-features.h
@@ -0,0 +1,137 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_CPU_FEATURES_H_
+#define V8_CODEGEN_CPU_FEATURES_H_
+
+#include "src/common/globals.h"
+
+namespace v8 {
+
+namespace internal {
+
+// CPU feature flags.
+enum CpuFeature {
+#if V8_TARGET_ARCH_IA32 || V8_TARGET_ARCH_X64
+ SSE4_2,
+ SSE4_1,
+ SSSE3,
+ SSE3,
+ SAHF,
+ AVX,
+ FMA3,
+ BMI1,
+ BMI2,
+ LZCNT,
+ POPCNT,
+ ATOM,
+
+#elif V8_TARGET_ARCH_ARM
+ // - Standard configurations. The baseline is ARMv6+VFPv2.
+ ARMv7, // ARMv7-A + VFPv3-D32 + NEON
+ ARMv7_SUDIV, // ARMv7-A + VFPv4-D32 + NEON + SUDIV
+ ARMv8, // ARMv8-A (+ all of the above)
+
+ // ARM feature aliases (based on the standard configurations above).
+ VFPv3 = ARMv7,
+ NEON = ARMv7,
+ VFP32DREGS = ARMv7,
+ SUDIV = ARMv7_SUDIV,
+
+#elif V8_TARGET_ARCH_ARM64
+ JSCVT,
+
+#elif V8_TARGET_ARCH_MIPS || V8_TARGET_ARCH_MIPS64
+ FPU,
+ FP64FPU,
+ MIPSr1,
+ MIPSr2,
+ MIPSr6,
+ MIPS_SIMD, // MSA instructions
+
+#elif V8_TARGET_ARCH_PPC || V8_TARGET_ARCH_PPC64
+ FPU,
+ FPR_GPR_MOV,
+ LWSYNC,
+ ISELECT,
+ VSX,
+ MODULO,
+
+#elif V8_TARGET_ARCH_S390X
+ FPU,
+ DISTINCT_OPS,
+ GENERAL_INSTR_EXT,
+ FLOATING_POINT_EXT,
+ VECTOR_FACILITY,
+ VECTOR_ENHANCE_FACILITY_1,
+ VECTOR_ENHANCE_FACILITY_2,
+ MISC_INSTR_EXT2,
+#endif
+
+ NUMBER_OF_CPU_FEATURES
+};
+
+// CpuFeatures keeps track of which features are supported by the target CPU.
+// Supported features must be enabled by a CpuFeatureScope before use.
+// Example:
+// if (assembler->IsSupported(SSE3)) {
+// CpuFeatureScope fscope(assembler, SSE3);
+// // Generate code containing SSE3 instructions.
+// } else {
+// // Generate alternative code.
+// }
+class V8_EXPORT_PRIVATE CpuFeatures : public AllStatic {
+ public:
+ static void Probe(bool cross_compile) {
+ STATIC_ASSERT(NUMBER_OF_CPU_FEATURES <= kBitsPerInt);
+ if (initialized_) return;
+ initialized_ = true;
+ ProbeImpl(cross_compile);
+ }
+
+ static unsigned SupportedFeatures() {
+ Probe(false);
+ return supported_;
+ }
+
+ static bool IsSupported(CpuFeature f) {
+ return (supported_ & (1u << f)) != 0;
+ }
+
+ static inline bool SupportsOptimizer();
+
+ static inline bool SupportsWasmSimd128();
+
+ static inline unsigned icache_line_size() {
+ DCHECK_NE(icache_line_size_, 0);
+ return icache_line_size_;
+ }
+
+ static inline unsigned dcache_line_size() {
+ DCHECK_NE(dcache_line_size_, 0);
+ return dcache_line_size_;
+ }
+
+ static void PrintTarget();
+ static void PrintFeatures();
+
+ private:
+ friend void V8_EXPORT_PRIVATE FlushInstructionCache(void*, size_t);
+ friend class ExternalReference;
+ // Flush instruction cache.
+ static void FlushICache(void* start, size_t size);
+
+ // Platform-dependent implementation.
+ static void ProbeImpl(bool cross_compile);
+
+ static unsigned supported_;
+ static unsigned icache_line_size_;
+ static unsigned dcache_line_size_;
+ static bool initialized_;
+ DISALLOW_COPY_AND_ASSIGN(CpuFeatures);
+};
+
+} // namespace internal
+} // namespace v8
+#endif // V8_CODEGEN_CPU_FEATURES_H_
diff --git a/src/codegen/external-reference-encoder.cc b/src/codegen/external-reference-encoder.cc
new file mode 100644
index 0000000..0dfe3e9
--- /dev/null
+++ b/src/codegen/external-reference-encoder.cc
@@ -0,0 +1,97 @@
+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/external-reference-encoder.h"
+
+#include "src/codegen/external-reference-table.h"
+#include "src/execution/isolate.h"
+
+namespace v8 {
+namespace internal {
+
+ExternalReferenceEncoder::ExternalReferenceEncoder(Isolate* isolate) {
+#ifdef DEBUG
+ api_references_ = isolate->api_external_references();
+ if (api_references_ != nullptr) {
+ for (uint32_t i = 0; api_references_[i] != 0; ++i) count_.push_back(0);
+ }
+#endif // DEBUG
+ map_ = isolate->external_reference_map();
+ if (map_ != nullptr) return;
+ map_ = new AddressToIndexHashMap();
+ isolate->set_external_reference_map(map_);
+ // Add V8's external references.
+ ExternalReferenceTable* table = isolate->external_reference_table();
+ for (uint32_t i = 0; i < ExternalReferenceTable::kSize; ++i) {
+ Address addr = table->address(i);
+ // Ignore duplicate references.
+ // This can happen due to ICF. See http://crbug.com/726896.
+ if (map_->Get(addr).IsNothing()) map_->Set(addr, Value::Encode(i, false));
+ DCHECK(map_->Get(addr).IsJust());
+ }
+ // Add external references provided by the embedder.
+ const intptr_t* api_references = isolate->api_external_references();
+ if (api_references == nullptr) return;
+ for (uint32_t i = 0; api_references[i] != 0; ++i) {
+ Address addr = static_cast<Address>(api_references[i]);
+ // Ignore duplicate references.
+ // This can happen due to ICF. See http://crbug.com/726896.
+ if (map_->Get(addr).IsNothing()) map_->Set(addr, Value::Encode(i, true));
+ DCHECK(map_->Get(addr).IsJust());
+ }
+}
+
+#ifdef DEBUG
+ExternalReferenceEncoder::~ExternalReferenceEncoder() {
+ if (!i::FLAG_external_reference_stats) return;
+ if (api_references_ == nullptr) return;
+ for (uint32_t i = 0; api_references_[i] != 0; ++i) {
+ Address addr = static_cast<Address>(api_references_[i]);
+ DCHECK(map_->Get(addr).IsJust());
+ v8::base::OS::Print(
+ "index=%5d count=%5d %-60s\n", i, count_[i],
+ ExternalReferenceTable::ResolveSymbol(reinterpret_cast<void*>(addr)));
+ }
+}
+#endif // DEBUG
+
+Maybe<ExternalReferenceEncoder::Value> ExternalReferenceEncoder::TryEncode(
+ Address address) {
+ Maybe<uint32_t> maybe_index = map_->Get(address);
+ if (maybe_index.IsNothing()) return Nothing<Value>();
+ Value result(maybe_index.FromJust());
+#ifdef DEBUG
+ if (result.is_from_api()) count_[result.index()]++;
+#endif // DEBUG
+ return Just<Value>(result);
+}
+
+ExternalReferenceEncoder::Value ExternalReferenceEncoder::Encode(
+ Address address) {
+ Maybe<uint32_t> maybe_index = map_->Get(address);
+ if (maybe_index.IsNothing()) {
+ void* addr = reinterpret_cast<void*>(address);
+ v8::base::OS::PrintError("Unknown external reference %p.\n", addr);
+ v8::base::OS::PrintError("%s\n",
+ ExternalReferenceTable::ResolveSymbol(addr));
+ v8::base::OS::Abort();
+ }
+ Value result(maybe_index.FromJust());
+#ifdef DEBUG
+ if (result.is_from_api()) count_[result.index()]++;
+#endif // DEBUG
+ return result;
+}
+
+const char* ExternalReferenceEncoder::NameOfAddress(Isolate* isolate,
+ Address address) const {
+ Maybe<uint32_t> maybe_index = map_->Get(address);
+ if (maybe_index.IsNothing()) return "<unknown>";
+ Value value(maybe_index.FromJust());
+ if (value.is_from_api()) return "<from api>";
+ return isolate->external_reference_table()->name(value.index());
+}
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/external-reference-encoder.h b/src/codegen/external-reference-encoder.h
new file mode 100644
index 0000000..7c41206
--- /dev/null
+++ b/src/codegen/external-reference-encoder.h
@@ -0,0 +1,60 @@
+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_EXTERNAL_REFERENCE_ENCODER_H_
+#define V8_CODEGEN_EXTERNAL_REFERENCE_ENCODER_H_
+
+#include "src/base/bit-field.h"
+#include "src/common/globals.h"
+#include "src/utils/address-map.h"
+
+namespace v8 {
+namespace internal {
+
+class Isolate;
+
+class ExternalReferenceEncoder {
+ public:
+ class Value {
+ public:
+ explicit Value(uint32_t raw) : value_(raw) {}
+ Value() : value_(0) {}
+ static uint32_t Encode(uint32_t index, bool is_from_api) {
+ return Index::encode(index) | IsFromAPI::encode(is_from_api);
+ }
+
+ bool is_from_api() const { return IsFromAPI::decode(value_); }
+ uint32_t index() const { return Index::decode(value_); }
+
+ private:
+ using Index = base::BitField<uint32_t, 0, 31>;
+ using IsFromAPI = base::BitField<bool, 31, 1>;
+ uint32_t value_;
+ };
+
+ explicit ExternalReferenceEncoder(Isolate* isolate);
+#ifdef DEBUG
+ ~ExternalReferenceEncoder();
+#endif // DEBUG
+
+ Value Encode(Address key);
+ Maybe<Value> TryEncode(Address key);
+
+ const char* NameOfAddress(Isolate* isolate, Address address) const;
+
+ private:
+ AddressToIndexHashMap* map_;
+
+#ifdef DEBUG
+ std::vector<int> count_;
+ const intptr_t* api_references_;
+#endif // DEBUG
+
+ DISALLOW_COPY_AND_ASSIGN(ExternalReferenceEncoder);
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_EXTERNAL_REFERENCE_ENCODER_H_
diff --git a/src/codegen/external-reference-table.cc b/src/codegen/external-reference-table.cc
new file mode 100644
index 0000000..b43f1a2
--- /dev/null
+++ b/src/codegen/external-reference-table.cc
@@ -0,0 +1,274 @@
+// Copyright 2016 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/external-reference-table.h"
+
+#include "src/builtins/accessors.h"
+#include "src/codegen/external-reference.h"
+#include "src/ic/stub-cache.h"
+#include "src/logging/counters.h"
+
+#if defined(DEBUG) && defined(V8_OS_LINUX) && !defined(V8_OS_ANDROID)
+#define SYMBOLIZE_FUNCTION
+#include <execinfo.h>
+#include <vector>
+#endif // DEBUG && V8_OS_LINUX && !V8_OS_ANDROID
+
+namespace v8 {
+namespace internal {
+
+#define ADD_EXT_REF_NAME(name, desc) desc,
+#define ADD_BUILTIN_NAME(Name, ...) "Builtin_" #Name,
+#define ADD_RUNTIME_FUNCTION(name, ...) "Runtime::" #name,
+#define ADD_ISOLATE_ADDR(Name, name) "Isolate::" #name "_address",
+#define ADD_ACCESSOR_INFO_NAME(_, __, AccessorName, ...) \
+ "Accessors::" #AccessorName "Getter",
+#define ADD_ACCESSOR_SETTER_NAME(name) "Accessors::" #name,
+#define ADD_STATS_COUNTER_NAME(name, ...) "StatsCounter::" #name,
+// static
+// clang-format off
+const char* const
+ ExternalReferenceTable::ref_name_[ExternalReferenceTable::kSize] = {
+ // Special references:
+ "nullptr",
+ // External references:
+ EXTERNAL_REFERENCE_LIST(ADD_EXT_REF_NAME)
+ EXTERNAL_REFERENCE_LIST_WITH_ISOLATE(ADD_EXT_REF_NAME)
+ // Builtins:
+ BUILTIN_LIST_C(ADD_BUILTIN_NAME)
+ // Runtime functions:
+ FOR_EACH_INTRINSIC(ADD_RUNTIME_FUNCTION)
+ // Isolate addresses:
+ FOR_EACH_ISOLATE_ADDRESS_NAME(ADD_ISOLATE_ADDR)
+ // Accessors:
+ ACCESSOR_INFO_LIST_GENERATOR(ADD_ACCESSOR_INFO_NAME, /* not used */)
+ ACCESSOR_SETTER_LIST(ADD_ACCESSOR_SETTER_NAME)
+ // Stub cache:
+ "Load StubCache::primary_->key",
+ "Load StubCache::primary_->value",
+ "Load StubCache::primary_->map",
+ "Load StubCache::secondary_->key",
+ "Load StubCache::secondary_->value",
+ "Load StubCache::secondary_->map",
+ "Store StubCache::primary_->key",
+ "Store StubCache::primary_->value",
+ "Store StubCache::primary_->map",
+ "Store StubCache::secondary_->key",
+ "Store StubCache::secondary_->value",
+ "Store StubCache::secondary_->map",
+ // Native code counters:
+ STATS_COUNTER_NATIVE_CODE_LIST(ADD_STATS_COUNTER_NAME)
+};
+// clang-format on
+#undef ADD_EXT_REF_NAME
+#undef ADD_BUILTIN_NAME
+#undef ADD_RUNTIME_FUNCTION
+#undef ADD_ISOLATE_ADDR
+#undef ADD_ACCESSOR_INFO_NAME
+#undef ADD_ACCESSOR_SETTER_NAME
+#undef ADD_STATS_COUNTER_NAME
+
+// Forward declarations for C++ builtins.
+#define FORWARD_DECLARE(Name) \
+ Address Builtin_##Name(int argc, Address* args, Isolate* isolate);
+BUILTIN_LIST_C(FORWARD_DECLARE)
+#undef FORWARD_DECLARE
+
+void ExternalReferenceTable::Init(Isolate* isolate) {
+ int index = 0;
+
+ // kNullAddress is preserved through serialization/deserialization.
+ Add(kNullAddress, &index);
+ AddReferences(isolate, &index);
+ AddBuiltins(&index);
+ AddRuntimeFunctions(&index);
+ AddIsolateAddresses(isolate, &index);
+ AddAccessors(&index);
+ AddStubCache(isolate, &index);
+ AddNativeCodeStatsCounters(isolate, &index);
+ is_initialized_ = static_cast<uint32_t>(true);
+
+ CHECK_EQ(kSize, index);
+}
+
+const char* ExternalReferenceTable::ResolveSymbol(void* address) {
+#ifdef SYMBOLIZE_FUNCTION
+ char** names = backtrace_symbols(&address, 1);
+ const char* name = names[0];
+ // The array of names is malloc'ed. However, each name string is static
+ // and do not need to be freed.
+ free(names);
+ return name;
+#else
+ return "<unresolved>";
+#endif // SYMBOLIZE_FUNCTION
+}
+
+void ExternalReferenceTable::Add(Address address, int* index) {
+ ref_addr_[(*index)++] = address;
+}
+
+void ExternalReferenceTable::AddReferences(Isolate* isolate, int* index) {
+ CHECK_EQ(kSpecialReferenceCount, *index);
+
+#define ADD_EXTERNAL_REFERENCE(name, desc) \
+ Add(ExternalReference::name().address(), index);
+ EXTERNAL_REFERENCE_LIST(ADD_EXTERNAL_REFERENCE)
+#undef ADD_EXTERNAL_REFERENCE
+
+#define ADD_EXTERNAL_REFERENCE(name, desc) \
+ Add(ExternalReference::name(isolate).address(), index);
+ EXTERNAL_REFERENCE_LIST_WITH_ISOLATE(ADD_EXTERNAL_REFERENCE)
+#undef ADD_EXTERNAL_REFERENCE
+
+ CHECK_EQ(kSpecialReferenceCount + kExternalReferenceCount, *index);
+}
+
+void ExternalReferenceTable::AddBuiltins(int* index) {
+ CHECK_EQ(kSpecialReferenceCount + kExternalReferenceCount, *index);
+
+ static const Address c_builtins[] = {
+#define DEF_ENTRY(Name, ...) FUNCTION_ADDR(&Builtin_##Name),
+ BUILTIN_LIST_C(DEF_ENTRY)
+#undef DEF_ENTRY
+ };
+ for (Address addr : c_builtins) {
+ Add(ExternalReference::Create(addr).address(), index);
+ }
+
+ CHECK_EQ(kSpecialReferenceCount + kExternalReferenceCount +
+ kBuiltinsReferenceCount,
+ *index);
+}
+
+void ExternalReferenceTable::AddRuntimeFunctions(int* index) {
+ CHECK_EQ(kSpecialReferenceCount + kExternalReferenceCount +
+ kBuiltinsReferenceCount,
+ *index);
+
+ static constexpr Runtime::FunctionId runtime_functions[] = {
+#define RUNTIME_ENTRY(name, ...) Runtime::k##name,
+ FOR_EACH_INTRINSIC(RUNTIME_ENTRY)
+#undef RUNTIME_ENTRY
+ };
+
+ for (Runtime::FunctionId fId : runtime_functions) {
+ Add(ExternalReference::Create(fId).address(), index);
+ }
+
+ CHECK_EQ(kSpecialReferenceCount + kExternalReferenceCount +
+ kBuiltinsReferenceCount + kRuntimeReferenceCount,
+ *index);
+}
+
+void ExternalReferenceTable::AddIsolateAddresses(Isolate* isolate, int* index) {
+ CHECK_EQ(kSpecialReferenceCount + kExternalReferenceCount +
+ kBuiltinsReferenceCount + kRuntimeReferenceCount,
+ *index);
+
+ for (int i = 0; i < IsolateAddressId::kIsolateAddressCount; ++i) {
+ Add(isolate->get_address_from_id(static_cast<IsolateAddressId>(i)), index);
+ }
+
+ CHECK_EQ(kSpecialReferenceCount + kExternalReferenceCount +
+ kBuiltinsReferenceCount + kRuntimeReferenceCount +
+ kIsolateAddressReferenceCount,
+ *index);
+}
+
+void ExternalReferenceTable::AddAccessors(int* index) {
+ CHECK_EQ(kSpecialReferenceCount + kExternalReferenceCount +
+ kBuiltinsReferenceCount + kRuntimeReferenceCount +
+ kIsolateAddressReferenceCount,
+ *index);
+
+ static const Address accessors[] = {
+ // Getters:
+#define ACCESSOR_INFO_DECLARATION(_, __, AccessorName, ...) \
+ FUNCTION_ADDR(&Accessors::AccessorName##Getter),
+ ACCESSOR_INFO_LIST_GENERATOR(ACCESSOR_INFO_DECLARATION, /* not used */)
+#undef ACCESSOR_INFO_DECLARATION
+ // Setters:
+#define ACCESSOR_SETTER_DECLARATION(name) FUNCTION_ADDR(&Accessors::name),
+ ACCESSOR_SETTER_LIST(ACCESSOR_SETTER_DECLARATION)
+#undef ACCESSOR_SETTER_DECLARATION
+ };
+
+ for (Address addr : accessors) {
+ Add(addr, index);
+ }
+
+ CHECK_EQ(kSpecialReferenceCount + kExternalReferenceCount +
+ kBuiltinsReferenceCount + kRuntimeReferenceCount +
+ kIsolateAddressReferenceCount + kAccessorReferenceCount,
+ *index);
+}
+
+void ExternalReferenceTable::AddStubCache(Isolate* isolate, int* index) {
+ CHECK_EQ(kSpecialReferenceCount + kExternalReferenceCount +
+ kBuiltinsReferenceCount + kRuntimeReferenceCount +
+ kIsolateAddressReferenceCount + kAccessorReferenceCount,
+ *index);
+
+ StubCache* load_stub_cache = isolate->load_stub_cache();
+
+ // Stub cache tables
+ Add(load_stub_cache->key_reference(StubCache::kPrimary).address(), index);
+ Add(load_stub_cache->value_reference(StubCache::kPrimary).address(), index);
+ Add(load_stub_cache->map_reference(StubCache::kPrimary).address(), index);
+ Add(load_stub_cache->key_reference(StubCache::kSecondary).address(), index);
+ Add(load_stub_cache->value_reference(StubCache::kSecondary).address(), index);
+ Add(load_stub_cache->map_reference(StubCache::kSecondary).address(), index);
+
+ StubCache* store_stub_cache = isolate->store_stub_cache();
+
+ // Stub cache tables
+ Add(store_stub_cache->key_reference(StubCache::kPrimary).address(), index);
+ Add(store_stub_cache->value_reference(StubCache::kPrimary).address(), index);
+ Add(store_stub_cache->map_reference(StubCache::kPrimary).address(), index);
+ Add(store_stub_cache->key_reference(StubCache::kSecondary).address(), index);
+ Add(store_stub_cache->value_reference(StubCache::kSecondary).address(),
+ index);
+ Add(store_stub_cache->map_reference(StubCache::kSecondary).address(), index);
+
+ CHECK_EQ(kSpecialReferenceCount + kExternalReferenceCount +
+ kBuiltinsReferenceCount + kRuntimeReferenceCount +
+ kIsolateAddressReferenceCount + kAccessorReferenceCount +
+ kStubCacheReferenceCount,
+ *index);
+}
+
+Address ExternalReferenceTable::GetStatsCounterAddress(StatsCounter* counter) {
+ int* address = counter->Enabled()
+ ? counter->GetInternalPointer()
+ : reinterpret_cast<int*>(&dummy_stats_counter_);
+ return reinterpret_cast<Address>(address);
+}
+
+void ExternalReferenceTable::AddNativeCodeStatsCounters(Isolate* isolate,
+ int* index) {
+ CHECK_EQ(kSpecialReferenceCount + kExternalReferenceCount +
+ kBuiltinsReferenceCount + kRuntimeReferenceCount +
+ kIsolateAddressReferenceCount + kAccessorReferenceCount +
+ kStubCacheReferenceCount,
+ *index);
+
+ Counters* counters = isolate->counters();
+
+#define SC(name, caption) Add(GetStatsCounterAddress(counters->name()), index);
+ STATS_COUNTER_NATIVE_CODE_LIST(SC)
+#undef SC
+
+ CHECK_EQ(kSpecialReferenceCount + kExternalReferenceCount +
+ kBuiltinsReferenceCount + kRuntimeReferenceCount +
+ kIsolateAddressReferenceCount + kAccessorReferenceCount +
+ kStubCacheReferenceCount + kStatsCountersReferenceCount,
+ *index);
+ CHECK_EQ(kSize, *index);
+}
+
+} // namespace internal
+} // namespace v8
+
+#undef SYMBOLIZE_FUNCTION
diff --git a/src/codegen/external-reference-table.h b/src/codegen/external-reference-table.h
new file mode 100644
index 0000000..798859b
--- /dev/null
+++ b/src/codegen/external-reference-table.h
@@ -0,0 +1,111 @@
+// Copyright 2016 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_EXTERNAL_REFERENCE_TABLE_H_
+#define V8_CODEGEN_EXTERNAL_REFERENCE_TABLE_H_
+
+#include <vector>
+
+#include "src/builtins/accessors.h"
+#include "src/builtins/builtins.h"
+#include "src/codegen/external-reference.h"
+#include "src/logging/counters-definitions.h"
+
+namespace v8 {
+namespace internal {
+
+class Isolate;
+
+// ExternalReferenceTable is a helper class that defines the relationship
+// between external references and their encodings. It is used to build
+// hashmaps in ExternalReferenceEncoder and ExternalReferenceDecoder.
+class ExternalReferenceTable {
+ public:
+ // For the nullptr ref, see the constructor.
+ static constexpr int kSpecialReferenceCount = 1;
+ static constexpr int kExternalReferenceCount =
+ ExternalReference::kExternalReferenceCount;
+ static constexpr int kBuiltinsReferenceCount =
+#define COUNT_C_BUILTIN(...) +1
+ BUILTIN_LIST_C(COUNT_C_BUILTIN);
+#undef COUNT_C_BUILTIN
+ static constexpr int kRuntimeReferenceCount =
+ Runtime::kNumFunctions -
+ Runtime::kNumInlineFunctions; // Don't count dupe kInline... functions.
+ static constexpr int kIsolateAddressReferenceCount = kIsolateAddressCount;
+ static constexpr int kAccessorReferenceCount =
+ Accessors::kAccessorInfoCount + Accessors::kAccessorSetterCount;
+ // The number of stub cache external references, see AddStubCache.
+ static constexpr int kStubCacheReferenceCount = 12;
+ static constexpr int kStatsCountersReferenceCount =
+#define SC(...) +1
+ STATS_COUNTER_NATIVE_CODE_LIST(SC);
+#undef SC
+ static constexpr int kSize =
+ kSpecialReferenceCount + kExternalReferenceCount +
+ kBuiltinsReferenceCount + kRuntimeReferenceCount +
+ kIsolateAddressReferenceCount + kAccessorReferenceCount +
+ kStubCacheReferenceCount + kStatsCountersReferenceCount;
+ static constexpr uint32_t kEntrySize =
+ static_cast<uint32_t>(kSystemPointerSize);
+ static constexpr uint32_t kSizeInBytes = kSize * kEntrySize + 2 * kUInt32Size;
+
+ Address address(uint32_t i) const { return ref_addr_[i]; }
+ const char* name(uint32_t i) const { return ref_name_[i]; }
+
+ bool is_initialized() const { return is_initialized_ != 0; }
+
+ static const char* ResolveSymbol(void* address);
+
+ static constexpr uint32_t OffsetOfEntry(uint32_t i) {
+ // Used in CodeAssembler::LookupExternalReference.
+ return i * kEntrySize;
+ }
+
+ const char* NameFromOffset(uint32_t offset) {
+ DCHECK_EQ(offset % kEntrySize, 0);
+ DCHECK_LT(offset, kSizeInBytes);
+ int index = offset / kEntrySize;
+ return name(index);
+ }
+
+ ExternalReferenceTable() = default;
+ void Init(Isolate* isolate);
+
+ private:
+ void Add(Address address, int* index);
+
+ void AddReferences(Isolate* isolate, int* index);
+ void AddBuiltins(int* index);
+ void AddRuntimeFunctions(int* index);
+ void AddIsolateAddresses(Isolate* isolate, int* index);
+ void AddAccessors(int* index);
+ void AddStubCache(Isolate* isolate, int* index);
+
+ Address GetStatsCounterAddress(StatsCounter* counter);
+ void AddNativeCodeStatsCounters(Isolate* isolate, int* index);
+
+ STATIC_ASSERT(sizeof(Address) == kEntrySize);
+ Address ref_addr_[kSize];
+ static const char* const ref_name_[kSize];
+
+ // Not bool to guarantee deterministic size.
+ uint32_t is_initialized_ = 0;
+
+ // Redirect disabled stats counters to this field. This is done to make sure
+ // we can have a snapshot that includes native counters even when the embedder
+ // isn't collecting them.
+ // This field is uint32_t since the MacroAssembler and CodeStubAssembler
+ // accesses this field as a uint32_t.
+ uint32_t dummy_stats_counter_ = 0;
+
+ DISALLOW_COPY_AND_ASSIGN(ExternalReferenceTable);
+};
+
+STATIC_ASSERT(ExternalReferenceTable::kSizeInBytes ==
+ sizeof(ExternalReferenceTable));
+
+} // namespace internal
+} // namespace v8
+#endif // V8_CODEGEN_EXTERNAL_REFERENCE_TABLE_H_
diff --git a/src/codegen/external-reference.cc b/src/codegen/external-reference.cc
new file mode 100644
index 0000000..499e5c5
--- /dev/null
+++ b/src/codegen/external-reference.cc
@@ -0,0 +1,1001 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/external-reference.h"
+
+#include "src/api/api.h"
+#include "src/base/ieee754.h"
+#include "src/codegen/cpu-features.h"
+#include "src/compiler/code-assembler.h"
+#include "src/date/date.h"
+#include "src/debug/debug.h"
+#include "src/deoptimizer/deoptimizer.h"
+#include "src/execution/isolate.h"
+#include "src/execution/microtask-queue.h"
+#include "src/execution/simulator-base.h"
+#include "src/heap/heap-inl.h"
+#include "src/heap/heap.h"
+#include "src/ic/stub-cache.h"
+#include "src/interpreter/interpreter.h"
+#include "src/logging/counters.h"
+#include "src/logging/log.h"
+#include "src/numbers/hash-seed-inl.h"
+#include "src/numbers/math-random.h"
+#include "src/objects/elements.h"
+#include "src/objects/objects-inl.h"
+#include "src/objects/ordered-hash-table.h"
+#include "src/regexp/experimental/experimental.h"
+#include "src/regexp/regexp-interpreter.h"
+#include "src/regexp/regexp-macro-assembler-arch.h"
+#include "src/regexp/regexp-stack.h"
+#include "src/strings/string-search.h"
+#include "src/wasm/wasm-external-refs.h"
+
+#ifdef V8_INTL_SUPPORT
+#include "src/objects/intl-objects.h"
+#endif // V8_INTL_SUPPORT
+
+namespace v8 {
+namespace internal {
+
+// -----------------------------------------------------------------------------
+// Common double constants.
+
+constexpr double double_min_int_constant = kMinInt;
+constexpr double double_one_half_constant = 0.5;
+constexpr uint64_t double_the_hole_nan_constant = kHoleNanInt64;
+constexpr double double_uint32_bias_constant =
+ static_cast<double>(kMaxUInt32) + 1;
+
+constexpr struct alignas(16) {
+ uint32_t a;
+ uint32_t b;
+ uint32_t c;
+ uint32_t d;
+} float_absolute_constant = {0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF};
+
+constexpr struct alignas(16) {
+ uint32_t a;
+ uint32_t b;
+ uint32_t c;
+ uint32_t d;
+} float_negate_constant = {0x80000000, 0x80000000, 0x80000000, 0x80000000};
+
+constexpr struct alignas(16) {
+ uint64_t a;
+ uint64_t b;
+} double_absolute_constant = {uint64_t{0x7FFFFFFFFFFFFFFF},
+ uint64_t{0x7FFFFFFFFFFFFFFF}};
+
+constexpr struct alignas(16) {
+ uint64_t a;
+ uint64_t b;
+} double_negate_constant = {uint64_t{0x8000000000000000},
+ uint64_t{0x8000000000000000}};
+
+// Implementation of ExternalReference
+
+static ExternalReference::Type BuiltinCallTypeForResultSize(int result_size) {
+ switch (result_size) {
+ case 1:
+ return ExternalReference::BUILTIN_CALL;
+ case 2:
+ return ExternalReference::BUILTIN_CALL_PAIR;
+ }
+ UNREACHABLE();
+}
+
+// static
+ExternalReference ExternalReference::Create(
+ ApiFunction* fun, Type type = ExternalReference::BUILTIN_CALL) {
+ return ExternalReference(Redirect(fun->address(), type));
+}
+
+// static
+ExternalReference ExternalReference::Create(Runtime::FunctionId id) {
+ return Create(Runtime::FunctionForId(id));
+}
+
+// static
+ExternalReference ExternalReference::Create(const Runtime::Function* f) {
+ return ExternalReference(
+ Redirect(f->entry, BuiltinCallTypeForResultSize(f->result_size)));
+}
+
+// static
+ExternalReference ExternalReference::Create(Address address) {
+ return ExternalReference(Redirect(address));
+}
+
+ExternalReference ExternalReference::isolate_address(Isolate* isolate) {
+ return ExternalReference(isolate);
+}
+
+ExternalReference ExternalReference::builtins_address(Isolate* isolate) {
+ return ExternalReference(isolate->heap()->builtin_address(0));
+}
+
+ExternalReference ExternalReference::handle_scope_implementer_address(
+ Isolate* isolate) {
+ return ExternalReference(isolate->handle_scope_implementer_address());
+}
+
+#ifdef V8_HEAP_SANDBOX
+ExternalReference ExternalReference::external_pointer_table_address(
+ Isolate* isolate) {
+ return ExternalReference(isolate->external_pointer_table_address());
+}
+#endif
+
+ExternalReference ExternalReference::interpreter_dispatch_table_address(
+ Isolate* isolate) {
+ return ExternalReference(isolate->interpreter()->dispatch_table_address());
+}
+
+ExternalReference ExternalReference::interpreter_dispatch_counters(
+ Isolate* isolate) {
+ return ExternalReference(
+ isolate->interpreter()->bytecode_dispatch_counters_table());
+}
+
+ExternalReference
+ExternalReference::address_of_interpreter_entry_trampoline_instruction_start(
+ Isolate* isolate) {
+ return ExternalReference(
+ isolate->interpreter()
+ ->address_of_interpreter_entry_trampoline_instruction_start());
+}
+
+ExternalReference ExternalReference::bytecode_size_table_address() {
+ return ExternalReference(
+ interpreter::Bytecodes::bytecode_size_table_address());
+}
+
+// static
+ExternalReference ExternalReference::Create(StatsCounter* counter) {
+ return ExternalReference(
+ reinterpret_cast<Address>(counter->GetInternalPointer()));
+}
+
+// static
+ExternalReference ExternalReference::Create(IsolateAddressId id,
+ Isolate* isolate) {
+ return ExternalReference(isolate->get_address_from_id(id));
+}
+
+// static
+ExternalReference ExternalReference::Create(const SCTableReference& table_ref) {
+ return ExternalReference(table_ref.address());
+}
+
+namespace {
+
+// Helper function to verify that all types in a list of types are scalar.
+// This includes primitive types (int, Address) and pointer types. We also
+// allow void.
+template <typename T>
+constexpr bool AllScalar() {
+ return std::is_scalar<T>::value || std::is_void<T>::value;
+}
+
+template <typename T1, typename T2, typename... Rest>
+constexpr bool AllScalar() {
+ return AllScalar<T1>() && AllScalar<T2, Rest...>();
+}
+
+// Checks a function pointer's type for compatibility with the
+// ExternalReference calling mechanism. Specifically, all arguments
+// as well as the result type must pass the AllScalar check above,
+// because we expect each item to fit into one register or stack slot.
+template <typename T>
+struct IsValidExternalReferenceType;
+
+template <typename Result, typename... Args>
+struct IsValidExternalReferenceType<Result (*)(Args...)> {
+ static const bool value = AllScalar<Result, Args...>();
+};
+
+template <typename Result, typename Class, typename... Args>
+struct IsValidExternalReferenceType<Result (Class::*)(Args...)> {
+ static const bool value = AllScalar<Result, Args...>();
+};
+
+} // namespace
+
+#define FUNCTION_REFERENCE(Name, Target) \
+ ExternalReference ExternalReference::Name() { \
+ STATIC_ASSERT(IsValidExternalReferenceType<decltype(&Target)>::value); \
+ return ExternalReference(Redirect(FUNCTION_ADDR(Target))); \
+ }
+
+#define FUNCTION_REFERENCE_WITH_ISOLATE(Name, Target) \
+ ExternalReference ExternalReference::Name(Isolate* isolate) { \
+ STATIC_ASSERT(IsValidExternalReferenceType<decltype(&Target)>::value); \
+ return ExternalReference(Redirect(FUNCTION_ADDR(Target))); \
+ }
+
+#define FUNCTION_REFERENCE_WITH_TYPE(Name, Target, Type) \
+ ExternalReference ExternalReference::Name() { \
+ STATIC_ASSERT(IsValidExternalReferenceType<decltype(&Target)>::value); \
+ return ExternalReference(Redirect(FUNCTION_ADDR(Target), Type)); \
+ }
+
+FUNCTION_REFERENCE(write_barrier_marking_from_code_function,
+ WriteBarrier::MarkingFromCode)
+
+FUNCTION_REFERENCE(insert_remembered_set_function,
+ Heap::InsertIntoRememberedSetFromCode)
+
+FUNCTION_REFERENCE(delete_handle_scope_extensions,
+ HandleScope::DeleteExtensions)
+
+FUNCTION_REFERENCE(ephemeron_key_write_barrier_function,
+ Heap::EphemeronKeyWriteBarrierFromCode)
+
+FUNCTION_REFERENCE(get_date_field_function, JSDate::GetField)
+
+ExternalReference ExternalReference::date_cache_stamp(Isolate* isolate) {
+ return ExternalReference(isolate->date_cache()->stamp_address());
+}
+
+// static
+ExternalReference
+ExternalReference::runtime_function_table_address_for_unittests(
+ Isolate* isolate) {
+ return runtime_function_table_address(isolate);
+}
+
+// static
+Address ExternalReference::Redirect(Address address, Type type) {
+#ifdef USE_SIMULATOR
+ return SimulatorBase::RedirectExternalReference(address, type);
+#else
+ return address;
+#endif
+}
+
+ExternalReference ExternalReference::stress_deopt_count(Isolate* isolate) {
+ return ExternalReference(isolate->stress_deopt_count_address());
+}
+
+ExternalReference ExternalReference::force_slow_path(Isolate* isolate) {
+ return ExternalReference(isolate->force_slow_path_address());
+}
+
+FUNCTION_REFERENCE(new_deoptimizer_function, Deoptimizer::New)
+
+FUNCTION_REFERENCE(compute_output_frames_function,
+ Deoptimizer::ComputeOutputFrames)
+
+FUNCTION_REFERENCE(wasm_f32_trunc, wasm::f32_trunc_wrapper)
+FUNCTION_REFERENCE(wasm_f32_floor, wasm::f32_floor_wrapper)
+FUNCTION_REFERENCE(wasm_f32_ceil, wasm::f32_ceil_wrapper)
+FUNCTION_REFERENCE(wasm_f32_nearest_int, wasm::f32_nearest_int_wrapper)
+FUNCTION_REFERENCE(wasm_f64_trunc, wasm::f64_trunc_wrapper)
+FUNCTION_REFERENCE(wasm_f64_floor, wasm::f64_floor_wrapper)
+FUNCTION_REFERENCE(wasm_f64_ceil, wasm::f64_ceil_wrapper)
+FUNCTION_REFERENCE(wasm_f64_nearest_int, wasm::f64_nearest_int_wrapper)
+FUNCTION_REFERENCE(wasm_int64_to_float32, wasm::int64_to_float32_wrapper)
+FUNCTION_REFERENCE(wasm_uint64_to_float32, wasm::uint64_to_float32_wrapper)
+FUNCTION_REFERENCE(wasm_int64_to_float64, wasm::int64_to_float64_wrapper)
+FUNCTION_REFERENCE(wasm_uint64_to_float64, wasm::uint64_to_float64_wrapper)
+FUNCTION_REFERENCE(wasm_float32_to_int64, wasm::float32_to_int64_wrapper)
+FUNCTION_REFERENCE(wasm_float32_to_uint64, wasm::float32_to_uint64_wrapper)
+FUNCTION_REFERENCE(wasm_float64_to_int64, wasm::float64_to_int64_wrapper)
+FUNCTION_REFERENCE(wasm_float64_to_uint64, wasm::float64_to_uint64_wrapper)
+FUNCTION_REFERENCE(wasm_float32_to_int64_sat,
+ wasm::float32_to_int64_sat_wrapper)
+FUNCTION_REFERENCE(wasm_float32_to_uint64_sat,
+ wasm::float32_to_uint64_sat_wrapper)
+FUNCTION_REFERENCE(wasm_float64_to_int64_sat,
+ wasm::float64_to_int64_sat_wrapper)
+FUNCTION_REFERENCE(wasm_float64_to_uint64_sat,
+ wasm::float64_to_uint64_sat_wrapper)
+FUNCTION_REFERENCE(wasm_int64_div, wasm::int64_div_wrapper)
+FUNCTION_REFERENCE(wasm_int64_mod, wasm::int64_mod_wrapper)
+FUNCTION_REFERENCE(wasm_uint64_div, wasm::uint64_div_wrapper)
+FUNCTION_REFERENCE(wasm_uint64_mod, wasm::uint64_mod_wrapper)
+FUNCTION_REFERENCE(wasm_word32_ctz, wasm::word32_ctz_wrapper)
+FUNCTION_REFERENCE(wasm_word64_ctz, wasm::word64_ctz_wrapper)
+FUNCTION_REFERENCE(wasm_word32_popcnt, wasm::word32_popcnt_wrapper)
+FUNCTION_REFERENCE(wasm_word64_popcnt, wasm::word64_popcnt_wrapper)
+FUNCTION_REFERENCE(wasm_word32_rol, wasm::word32_rol_wrapper)
+FUNCTION_REFERENCE(wasm_word32_ror, wasm::word32_ror_wrapper)
+FUNCTION_REFERENCE(wasm_word64_rol, wasm::word64_rol_wrapper)
+FUNCTION_REFERENCE(wasm_word64_ror, wasm::word64_ror_wrapper)
+FUNCTION_REFERENCE(wasm_f64x2_ceil, wasm::f64x2_ceil_wrapper)
+FUNCTION_REFERENCE(wasm_f64x2_floor, wasm::f64x2_floor_wrapper)
+FUNCTION_REFERENCE(wasm_f64x2_trunc, wasm::f64x2_trunc_wrapper)
+FUNCTION_REFERENCE(wasm_f64x2_nearest_int, wasm::f64x2_nearest_int_wrapper)
+FUNCTION_REFERENCE(wasm_f32x4_ceil, wasm::f32x4_ceil_wrapper)
+FUNCTION_REFERENCE(wasm_f32x4_floor, wasm::f32x4_floor_wrapper)
+FUNCTION_REFERENCE(wasm_f32x4_trunc, wasm::f32x4_trunc_wrapper)
+FUNCTION_REFERENCE(wasm_f32x4_nearest_int, wasm::f32x4_nearest_int_wrapper)
+FUNCTION_REFERENCE(wasm_memory_init, wasm::memory_init_wrapper)
+FUNCTION_REFERENCE(wasm_memory_copy, wasm::memory_copy_wrapper)
+FUNCTION_REFERENCE(wasm_memory_fill, wasm::memory_fill_wrapper)
+
+static void f64_acos_wrapper(Address data) {
+ double input = ReadUnalignedValue<double>(data);
+ WriteUnalignedValue(data, base::ieee754::acos(input));
+}
+
+FUNCTION_REFERENCE(f64_acos_wrapper_function, f64_acos_wrapper)
+
+static void f64_asin_wrapper(Address data) {
+ double input = ReadUnalignedValue<double>(data);
+ WriteUnalignedValue<double>(data, base::ieee754::asin(input));
+}
+
+FUNCTION_REFERENCE(f64_asin_wrapper_function, f64_asin_wrapper)
+
+FUNCTION_REFERENCE(wasm_float64_pow, wasm::float64_pow_wrapper)
+
+static void f64_mod_wrapper(Address data) {
+ double dividend = ReadUnalignedValue<double>(data);
+ double divisor = ReadUnalignedValue<double>(data + sizeof(dividend));
+ WriteUnalignedValue<double>(data, Modulo(dividend, divisor));
+}
+
+FUNCTION_REFERENCE(f64_mod_wrapper_function, f64_mod_wrapper)
+
+FUNCTION_REFERENCE(wasm_call_trap_callback_for_testing,
+ wasm::call_trap_callback_for_testing)
+
+ExternalReference ExternalReference::isolate_root(Isolate* isolate) {
+ return ExternalReference(isolate->isolate_root());
+}
+
+ExternalReference ExternalReference::allocation_sites_list_address(
+ Isolate* isolate) {
+ return ExternalReference(isolate->heap()->allocation_sites_list_address());
+}
+
+ExternalReference ExternalReference::address_of_jslimit(Isolate* isolate) {
+ Address address = isolate->stack_guard()->address_of_jslimit();
+ // For efficient generated code, this should be root-register-addressable.
+ DCHECK(isolate->root_register_addressable_region().contains(address));
+ return ExternalReference(address);
+}
+
+ExternalReference ExternalReference::address_of_real_jslimit(Isolate* isolate) {
+ Address address = isolate->stack_guard()->address_of_real_jslimit();
+ // For efficient generated code, this should be root-register-addressable.
+ DCHECK(isolate->root_register_addressable_region().contains(address));
+ return ExternalReference(address);
+}
+
+ExternalReference ExternalReference::heap_is_marking_flag_address(
+ Isolate* isolate) {
+ return ExternalReference(isolate->heap()->IsMarkingFlagAddress());
+}
+
+ExternalReference ExternalReference::new_space_allocation_top_address(
+ Isolate* isolate) {
+ return ExternalReference(isolate->heap()->NewSpaceAllocationTopAddress());
+}
+
+ExternalReference ExternalReference::new_space_allocation_limit_address(
+ Isolate* isolate) {
+ return ExternalReference(isolate->heap()->NewSpaceAllocationLimitAddress());
+}
+
+ExternalReference ExternalReference::old_space_allocation_top_address(
+ Isolate* isolate) {
+ return ExternalReference(isolate->heap()->OldSpaceAllocationTopAddress());
+}
+
+ExternalReference ExternalReference::old_space_allocation_limit_address(
+ Isolate* isolate) {
+ return ExternalReference(isolate->heap()->OldSpaceAllocationLimitAddress());
+}
+
+ExternalReference ExternalReference::handle_scope_level_address(
+ Isolate* isolate) {
+ return ExternalReference(HandleScope::current_level_address(isolate));
+}
+
+ExternalReference ExternalReference::handle_scope_next_address(
+ Isolate* isolate) {
+ return ExternalReference(HandleScope::current_next_address(isolate));
+}
+
+ExternalReference ExternalReference::handle_scope_limit_address(
+ Isolate* isolate) {
+ return ExternalReference(HandleScope::current_limit_address(isolate));
+}
+
+ExternalReference ExternalReference::scheduled_exception_address(
+ Isolate* isolate) {
+ return ExternalReference(isolate->scheduled_exception_address());
+}
+
+ExternalReference ExternalReference::address_of_pending_message_obj(
+ Isolate* isolate) {
+ return ExternalReference(isolate->pending_message_obj_address());
+}
+
+FUNCTION_REFERENCE(abort_with_reason, i::abort_with_reason)
+
+ExternalReference ExternalReference::address_of_min_int() {
+ return ExternalReference(reinterpret_cast<Address>(&double_min_int_constant));
+}
+
+ExternalReference
+ExternalReference::address_of_mock_arraybuffer_allocator_flag() {
+ return ExternalReference(&FLAG_mock_arraybuffer_allocator);
+}
+
+ExternalReference ExternalReference::address_of_runtime_stats_flag() {
+ return ExternalReference(&TracingFlags::runtime_stats);
+}
+
+ExternalReference ExternalReference::address_of_load_from_stack_count(
+ const char* function_name) {
+ return ExternalReference(
+ Isolate::load_from_stack_count_address(function_name));
+}
+
+ExternalReference ExternalReference::address_of_store_to_stack_count(
+ const char* function_name) {
+ return ExternalReference(
+ Isolate::store_to_stack_count_address(function_name));
+}
+
+ExternalReference ExternalReference::address_of_one_half() {
+ return ExternalReference(
+ reinterpret_cast<Address>(&double_one_half_constant));
+}
+
+ExternalReference ExternalReference::address_of_the_hole_nan() {
+ return ExternalReference(
+ reinterpret_cast<Address>(&double_the_hole_nan_constant));
+}
+
+ExternalReference ExternalReference::address_of_uint32_bias() {
+ return ExternalReference(
+ reinterpret_cast<Address>(&double_uint32_bias_constant));
+}
+
+ExternalReference ExternalReference::address_of_float_abs_constant() {
+ return ExternalReference(reinterpret_cast<Address>(&float_absolute_constant));
+}
+
+ExternalReference ExternalReference::address_of_float_neg_constant() {
+ return ExternalReference(reinterpret_cast<Address>(&float_negate_constant));
+}
+
+ExternalReference ExternalReference::address_of_double_abs_constant() {
+ return ExternalReference(
+ reinterpret_cast<Address>(&double_absolute_constant));
+}
+
+ExternalReference ExternalReference::address_of_double_neg_constant() {
+ return ExternalReference(reinterpret_cast<Address>(&double_negate_constant));
+}
+
+ExternalReference
+ExternalReference::address_of_enable_experimental_regexp_engine() {
+ return ExternalReference(&FLAG_enable_experimental_regexp_engine);
+}
+
+ExternalReference ExternalReference::is_profiling_address(Isolate* isolate) {
+ return ExternalReference(isolate->is_profiling_address());
+}
+
+ExternalReference ExternalReference::invoke_function_callback() {
+ Address thunk_address = FUNCTION_ADDR(&InvokeFunctionCallback);
+ ExternalReference::Type thunk_type = ExternalReference::PROFILING_API_CALL;
+ ApiFunction thunk_fun(thunk_address);
+ return ExternalReference::Create(&thunk_fun, thunk_type);
+}
+
+ExternalReference ExternalReference::invoke_accessor_getter_callback() {
+ Address thunk_address = FUNCTION_ADDR(&InvokeAccessorGetterCallback);
+ ExternalReference::Type thunk_type = ExternalReference::PROFILING_GETTER_CALL;
+ ApiFunction thunk_fun(thunk_address);
+ return ExternalReference::Create(&thunk_fun, thunk_type);
+}
+
+#if V8_TARGET_ARCH_X64
+#define re_stack_check_func RegExpMacroAssemblerX64::CheckStackGuardState
+#elif V8_TARGET_ARCH_IA32
+#define re_stack_check_func RegExpMacroAssemblerIA32::CheckStackGuardState
+#elif V8_TARGET_ARCH_ARM64
+#define re_stack_check_func RegExpMacroAssemblerARM64::CheckStackGuardState
+#elif V8_TARGET_ARCH_ARM
+#define re_stack_check_func RegExpMacroAssemblerARM::CheckStackGuardState
+#elif V8_TARGET_ARCH_PPC || V8_TARGET_ARCH_PPC64
+#define re_stack_check_func RegExpMacroAssemblerPPC::CheckStackGuardState
+#elif V8_TARGET_ARCH_MIPS
+#define re_stack_check_func RegExpMacroAssemblerMIPS::CheckStackGuardState
+#elif V8_TARGET_ARCH_MIPS64
+#define re_stack_check_func RegExpMacroAssemblerMIPS::CheckStackGuardState
+#elif V8_TARGET_ARCH_S390
+#define re_stack_check_func RegExpMacroAssemblerS390::CheckStackGuardState
+#else
+UNREACHABLE();
+#endif
+
+FUNCTION_REFERENCE_WITH_ISOLATE(re_check_stack_guard_state, re_stack_check_func)
+#undef re_stack_check_func
+
+FUNCTION_REFERENCE_WITH_ISOLATE(re_grow_stack,
+ NativeRegExpMacroAssembler::GrowStack)
+
+FUNCTION_REFERENCE(re_match_for_call_from_js,
+ IrregexpInterpreter::MatchForCallFromJs)
+
+FUNCTION_REFERENCE(re_experimental_match_for_call_from_js,
+ ExperimentalRegExp::MatchForCallFromJs)
+
+FUNCTION_REFERENCE_WITH_ISOLATE(
+ re_case_insensitive_compare_unicode,
+ NativeRegExpMacroAssembler::CaseInsensitiveCompareUnicode)
+
+FUNCTION_REFERENCE_WITH_ISOLATE(
+ re_case_insensitive_compare_non_unicode,
+ NativeRegExpMacroAssembler::CaseInsensitiveCompareNonUnicode)
+
+ExternalReference ExternalReference::re_word_character_map(Isolate* isolate) {
+ return ExternalReference(
+ NativeRegExpMacroAssembler::word_character_map_address());
+}
+
+ExternalReference ExternalReference::address_of_static_offsets_vector(
+ Isolate* isolate) {
+ return ExternalReference(
+ reinterpret_cast<Address>(isolate->jsregexp_static_offsets_vector()));
+}
+
+ExternalReference ExternalReference::address_of_regexp_stack_limit_address(
+ Isolate* isolate) {
+ return ExternalReference(isolate->regexp_stack()->limit_address_address());
+}
+
+ExternalReference ExternalReference::address_of_regexp_stack_memory_top_address(
+ Isolate* isolate) {
+ return ExternalReference(
+ isolate->regexp_stack()->memory_top_address_address());
+}
+
+FUNCTION_REFERENCE_WITH_TYPE(ieee754_acos_function, base::ieee754::acos,
+ BUILTIN_FP_CALL)
+FUNCTION_REFERENCE_WITH_TYPE(ieee754_acosh_function, base::ieee754::acosh,
+ BUILTIN_FP_CALL)
+FUNCTION_REFERENCE_WITH_TYPE(ieee754_asin_function, base::ieee754::asin,
+ BUILTIN_FP_CALL)
+FUNCTION_REFERENCE_WITH_TYPE(ieee754_asinh_function, base::ieee754::asinh,
+ BUILTIN_FP_CALL)
+FUNCTION_REFERENCE_WITH_TYPE(ieee754_atan_function, base::ieee754::atan,
+ BUILTIN_FP_CALL)
+FUNCTION_REFERENCE_WITH_TYPE(ieee754_atanh_function, base::ieee754::atanh,
+ BUILTIN_FP_CALL)
+FUNCTION_REFERENCE_WITH_TYPE(ieee754_atan2_function, base::ieee754::atan2,
+ BUILTIN_FP_FP_CALL)
+FUNCTION_REFERENCE_WITH_TYPE(ieee754_cbrt_function, base::ieee754::cbrt,
+ BUILTIN_FP_CALL)
+FUNCTION_REFERENCE_WITH_TYPE(ieee754_cos_function, base::ieee754::cos,
+ BUILTIN_FP_CALL)
+FUNCTION_REFERENCE_WITH_TYPE(ieee754_cosh_function, base::ieee754::cosh,
+ BUILTIN_FP_CALL)
+FUNCTION_REFERENCE_WITH_TYPE(ieee754_exp_function, base::ieee754::exp,
+ BUILTIN_FP_CALL)
+FUNCTION_REFERENCE_WITH_TYPE(ieee754_expm1_function, base::ieee754::expm1,
+ BUILTIN_FP_CALL)
+FUNCTION_REFERENCE_WITH_TYPE(ieee754_log_function, base::ieee754::log,
+ BUILTIN_FP_CALL)
+FUNCTION_REFERENCE_WITH_TYPE(ieee754_log1p_function, base::ieee754::log1p,
+ BUILTIN_FP_CALL)
+FUNCTION_REFERENCE_WITH_TYPE(ieee754_log10_function, base::ieee754::log10,
+ BUILTIN_FP_CALL)
+FUNCTION_REFERENCE_WITH_TYPE(ieee754_log2_function, base::ieee754::log2,
+ BUILTIN_FP_CALL)
+FUNCTION_REFERENCE_WITH_TYPE(ieee754_sin_function, base::ieee754::sin,
+ BUILTIN_FP_CALL)
+FUNCTION_REFERENCE_WITH_TYPE(ieee754_sinh_function, base::ieee754::sinh,
+ BUILTIN_FP_CALL)
+FUNCTION_REFERENCE_WITH_TYPE(ieee754_tan_function, base::ieee754::tan,
+ BUILTIN_FP_CALL)
+FUNCTION_REFERENCE_WITH_TYPE(ieee754_tanh_function, base::ieee754::tanh,
+ BUILTIN_FP_CALL)
+FUNCTION_REFERENCE_WITH_TYPE(ieee754_pow_function, base::ieee754::pow,
+ BUILTIN_FP_FP_CALL)
+
+void* libc_memchr(void* string, int character, size_t search_length) {
+ return memchr(string, character, search_length);
+}
+
+FUNCTION_REFERENCE(libc_memchr_function, libc_memchr)
+
+void* libc_memcpy(void* dest, const void* src, size_t n) {
+ return memcpy(dest, src, n);
+}
+
+FUNCTION_REFERENCE(libc_memcpy_function, libc_memcpy)
+
+void* libc_memmove(void* dest, const void* src, size_t n) {
+ return memmove(dest, src, n);
+}
+
+FUNCTION_REFERENCE(libc_memmove_function, libc_memmove)
+
+void* libc_memset(void* dest, int value, size_t n) {
+ DCHECK_EQ(static_cast<byte>(value), value);
+ return memset(dest, value, n);
+}
+
+FUNCTION_REFERENCE(libc_memset_function, libc_memset)
+
+ExternalReference ExternalReference::printf_function() {
+ return ExternalReference(Redirect(FUNCTION_ADDR(std::printf)));
+}
+
+FUNCTION_REFERENCE(refill_math_random, MathRandom::RefillCache)
+
+template <typename SubjectChar, typename PatternChar>
+ExternalReference ExternalReference::search_string_raw() {
+ auto f = SearchStringRaw<SubjectChar, PatternChar>;
+ return ExternalReference(Redirect(FUNCTION_ADDR(f)));
+}
+
+FUNCTION_REFERENCE(jsarray_array_join_concat_to_sequential_string,
+ JSArray::ArrayJoinConcatToSequentialString)
+
+ExternalReference ExternalReference::search_string_raw_one_one() {
+ return search_string_raw<const uint8_t, const uint8_t>();
+}
+
+ExternalReference ExternalReference::search_string_raw_one_two() {
+ return search_string_raw<const uint8_t, const uc16>();
+}
+
+ExternalReference ExternalReference::search_string_raw_two_one() {
+ return search_string_raw<const uc16, const uint8_t>();
+}
+
+ExternalReference ExternalReference::search_string_raw_two_two() {
+ return search_string_raw<const uc16, const uc16>();
+}
+
+FUNCTION_REFERENCE(orderedhashmap_gethash_raw, OrderedHashMap::GetHash)
+
+Address GetOrCreateHash(Isolate* isolate, Address raw_key) {
+ DisallowHeapAllocation no_gc;
+ return Object(raw_key).GetOrCreateHash(isolate).ptr();
+}
+
+FUNCTION_REFERENCE(get_or_create_hash_raw, GetOrCreateHash)
+
+static Address JSReceiverCreateIdentityHash(Isolate* isolate, Address raw_key) {
+ JSReceiver key = JSReceiver::cast(Object(raw_key));
+ return JSReceiver::CreateIdentityHash(isolate, key).ptr();
+}
+
+FUNCTION_REFERENCE(jsreceiver_create_identity_hash,
+ JSReceiverCreateIdentityHash)
+
+static uint32_t ComputeSeededIntegerHash(Isolate* isolate, int32_t key) {
+ DisallowHeapAllocation no_gc;
+ return ComputeSeededHash(static_cast<uint32_t>(key), HashSeed(isolate));
+}
+
+FUNCTION_REFERENCE(compute_integer_hash, ComputeSeededIntegerHash)
+FUNCTION_REFERENCE(copy_fast_number_jsarray_elements_to_typed_array,
+ CopyFastNumberJSArrayElementsToTypedArray)
+FUNCTION_REFERENCE(copy_typed_array_elements_to_typed_array,
+ CopyTypedArrayElementsToTypedArray)
+FUNCTION_REFERENCE(copy_typed_array_elements_slice, CopyTypedArrayElementsSlice)
+FUNCTION_REFERENCE(try_string_to_index_or_lookup_existing,
+ StringTable::TryStringToIndexOrLookupExisting)
+FUNCTION_REFERENCE(string_to_array_index_function, String::ToArrayIndex)
+
+static Address LexicographicCompareWrapper(Isolate* isolate, Address smi_x,
+ Address smi_y) {
+ Smi x(smi_x);
+ Smi y(smi_y);
+ return Smi::LexicographicCompare(isolate, x, y);
+}
+
+FUNCTION_REFERENCE(smi_lexicographic_compare_function,
+ LexicographicCompareWrapper)
+
+FUNCTION_REFERENCE(mutable_big_int_absolute_add_and_canonicalize_function,
+ MutableBigInt_AbsoluteAddAndCanonicalize)
+
+FUNCTION_REFERENCE(mutable_big_int_absolute_compare_function,
+ MutableBigInt_AbsoluteCompare)
+
+FUNCTION_REFERENCE(mutable_big_int_absolute_sub_and_canonicalize_function,
+ MutableBigInt_AbsoluteSubAndCanonicalize)
+
+FUNCTION_REFERENCE(check_object_type, CheckObjectType)
+
+#ifdef V8_INTL_SUPPORT
+
+static Address ConvertOneByteToLower(Address raw_src, Address raw_dst) {
+ String src = String::cast(Object(raw_src));
+ String dst = String::cast(Object(raw_dst));
+ return Intl::ConvertOneByteToLower(src, dst).ptr();
+}
+FUNCTION_REFERENCE(intl_convert_one_byte_to_lower, ConvertOneByteToLower)
+
+ExternalReference ExternalReference::intl_to_latin1_lower_table() {
+ uint8_t* ptr = const_cast<uint8_t*>(Intl::ToLatin1LowerTable());
+ return ExternalReference(reinterpret_cast<Address>(ptr));
+}
+#endif // V8_INTL_SUPPORT
+
+// Explicit instantiations for all combinations of 1- and 2-byte strings.
+template ExternalReference
+ExternalReference::search_string_raw<const uint8_t, const uint8_t>();
+template ExternalReference
+ExternalReference::search_string_raw<const uint8_t, const uc16>();
+template ExternalReference
+ExternalReference::search_string_raw<const uc16, const uint8_t>();
+template ExternalReference
+ExternalReference::search_string_raw<const uc16, const uc16>();
+
+ExternalReference ExternalReference::FromRawAddress(Address address) {
+ return ExternalReference(address);
+}
+
+ExternalReference ExternalReference::cpu_features() {
+ DCHECK(CpuFeatures::initialized_);
+ return ExternalReference(&CpuFeatures::supported_);
+}
+
+ExternalReference ExternalReference::promise_hook_address(Isolate* isolate) {
+ return ExternalReference(isolate->promise_hook_address());
+}
+
+ExternalReference ExternalReference::async_event_delegate_address(
+ Isolate* isolate) {
+ return ExternalReference(isolate->async_event_delegate_address());
+}
+
+ExternalReference
+ExternalReference::promise_hook_or_async_event_delegate_address(
+ Isolate* isolate) {
+ return ExternalReference(
+ isolate->promise_hook_or_async_event_delegate_address());
+}
+
+ExternalReference ExternalReference::
+ promise_hook_or_debug_is_active_or_async_event_delegate_address(
+ Isolate* isolate) {
+ return ExternalReference(
+ isolate
+ ->promise_hook_or_debug_is_active_or_async_event_delegate_address());
+}
+
+ExternalReference ExternalReference::debug_execution_mode_address(
+ Isolate* isolate) {
+ return ExternalReference(isolate->debug_execution_mode_address());
+}
+
+ExternalReference ExternalReference::debug_is_active_address(Isolate* isolate) {
+ return ExternalReference(isolate->debug()->is_active_address());
+}
+
+ExternalReference ExternalReference::debug_hook_on_function_call_address(
+ Isolate* isolate) {
+ return ExternalReference(isolate->debug()->hook_on_function_call_address());
+}
+
+ExternalReference ExternalReference::runtime_function_table_address(
+ Isolate* isolate) {
+ return ExternalReference(
+ const_cast<Runtime::Function*>(Runtime::RuntimeFunctionTable(isolate)));
+}
+
+static Address InvalidatePrototypeChainsWrapper(Address raw_map) {
+ Map map = Map::cast(Object(raw_map));
+ return JSObject::InvalidatePrototypeChains(map).ptr();
+}
+
+FUNCTION_REFERENCE(invalidate_prototype_chains_function,
+ InvalidatePrototypeChainsWrapper)
+
+double modulo_double_double(double x, double y) { return Modulo(x, y); }
+
+FUNCTION_REFERENCE_WITH_TYPE(mod_two_doubles_operation, modulo_double_double,
+ BUILTIN_FP_FP_CALL)
+
+ExternalReference ExternalReference::debug_suspended_generator_address(
+ Isolate* isolate) {
+ return ExternalReference(isolate->debug()->suspended_generator_address());
+}
+
+ExternalReference ExternalReference::debug_restart_fp_address(
+ Isolate* isolate) {
+ return ExternalReference(isolate->debug()->restart_fp_address());
+}
+
+ExternalReference ExternalReference::fast_c_call_caller_fp_address(
+ Isolate* isolate) {
+ return ExternalReference(
+ isolate->isolate_data()->fast_c_call_caller_fp_address());
+}
+
+ExternalReference ExternalReference::fast_c_call_caller_pc_address(
+ Isolate* isolate) {
+ return ExternalReference(
+ isolate->isolate_data()->fast_c_call_caller_pc_address());
+}
+
+ExternalReference ExternalReference::stack_is_iterable_address(
+ Isolate* isolate) {
+ return ExternalReference(
+ isolate->isolate_data()->stack_is_iterable_address());
+}
+
+FUNCTION_REFERENCE(call_enqueue_microtask_function,
+ MicrotaskQueue::CallEnqueueMicrotask)
+
+static int64_t atomic_pair_load(intptr_t address) {
+ return std::atomic_load(reinterpret_cast<std::atomic<int64_t>*>(address));
+}
+
+ExternalReference ExternalReference::atomic_pair_load_function() {
+ return ExternalReference(Redirect(FUNCTION_ADDR(atomic_pair_load)));
+}
+
+static void atomic_pair_store(intptr_t address, int value_low, int value_high) {
+ int64_t value =
+ static_cast<int64_t>(value_high) << 32 | (value_low & 0xFFFFFFFF);
+ std::atomic_store(reinterpret_cast<std::atomic<int64_t>*>(address), value);
+}
+
+ExternalReference ExternalReference::atomic_pair_store_function() {
+ return ExternalReference(Redirect(FUNCTION_ADDR(atomic_pair_store)));
+}
+
+static int64_t atomic_pair_add(intptr_t address, int value_low,
+ int value_high) {
+ int64_t value =
+ static_cast<int64_t>(value_high) << 32 | (value_low & 0xFFFFFFFF);
+ return std::atomic_fetch_add(reinterpret_cast<std::atomic<int64_t>*>(address),
+ value);
+}
+
+ExternalReference ExternalReference::atomic_pair_add_function() {
+ return ExternalReference(Redirect(FUNCTION_ADDR(atomic_pair_add)));
+}
+
+static int64_t atomic_pair_sub(intptr_t address, int value_low,
+ int value_high) {
+ int64_t value =
+ static_cast<int64_t>(value_high) << 32 | (value_low & 0xFFFFFFFF);
+ return std::atomic_fetch_sub(reinterpret_cast<std::atomic<int64_t>*>(address),
+ value);
+}
+
+ExternalReference ExternalReference::atomic_pair_sub_function() {
+ return ExternalReference(Redirect(FUNCTION_ADDR(atomic_pair_sub)));
+}
+
+static int64_t atomic_pair_and(intptr_t address, int value_low,
+ int value_high) {
+ int64_t value =
+ static_cast<int64_t>(value_high) << 32 | (value_low & 0xFFFFFFFF);
+ return std::atomic_fetch_and(reinterpret_cast<std::atomic<int64_t>*>(address),
+ value);
+}
+
+ExternalReference ExternalReference::atomic_pair_and_function() {
+ return ExternalReference(Redirect(FUNCTION_ADDR(atomic_pair_and)));
+}
+
+static int64_t atomic_pair_or(intptr_t address, int value_low, int value_high) {
+ int64_t value =
+ static_cast<int64_t>(value_high) << 32 | (value_low & 0xFFFFFFFF);
+ return std::atomic_fetch_or(reinterpret_cast<std::atomic<int64_t>*>(address),
+ value);
+}
+
+ExternalReference ExternalReference::atomic_pair_or_function() {
+ return ExternalReference(Redirect(FUNCTION_ADDR(atomic_pair_or)));
+}
+
+static int64_t atomic_pair_xor(intptr_t address, int value_low,
+ int value_high) {
+ int64_t value =
+ static_cast<int64_t>(value_high) << 32 | (value_low & 0xFFFFFFFF);
+ return std::atomic_fetch_xor(reinterpret_cast<std::atomic<int64_t>*>(address),
+ value);
+}
+
+ExternalReference ExternalReference::atomic_pair_xor_function() {
+ return ExternalReference(Redirect(FUNCTION_ADDR(atomic_pair_xor)));
+}
+
+static int64_t atomic_pair_exchange(intptr_t address, int value_low,
+ int value_high) {
+ int64_t value =
+ static_cast<int64_t>(value_high) << 32 | (value_low & 0xFFFFFFFF);
+ return std::atomic_exchange(reinterpret_cast<std::atomic<int64_t>*>(address),
+ value);
+}
+
+ExternalReference ExternalReference::atomic_pair_exchange_function() {
+ return ExternalReference(Redirect(FUNCTION_ADDR(atomic_pair_exchange)));
+}
+
+static uint64_t atomic_pair_compare_exchange(intptr_t address,
+ int old_value_low,
+ int old_value_high,
+ int new_value_low,
+ int new_value_high) {
+ uint64_t old_value = static_cast<uint64_t>(old_value_high) << 32 |
+ (old_value_low & 0xFFFFFFFF);
+ uint64_t new_value = static_cast<uint64_t>(new_value_high) << 32 |
+ (new_value_low & 0xFFFFFFFF);
+ std::atomic_compare_exchange_strong(
+ reinterpret_cast<std::atomic<uint64_t>*>(address), &old_value, new_value);
+ return old_value;
+}
+
+FUNCTION_REFERENCE(atomic_pair_compare_exchange_function,
+ atomic_pair_compare_exchange)
+
+static int EnterMicrotaskContextWrapper(HandleScopeImplementer* hsi,
+ Address raw_context) {
+ Context context = Context::cast(Object(raw_context));
+ hsi->EnterMicrotaskContext(context);
+ return 0;
+}
+
+FUNCTION_REFERENCE(call_enter_context_function, EnterMicrotaskContextWrapper)
+
+FUNCTION_REFERENCE(
+ js_finalization_registry_remove_cell_from_unregister_token_map,
+ JSFinalizationRegistry::RemoveCellFromUnregisterTokenMap)
+
+#ifdef V8_HEAP_SANDBOX
+FUNCTION_REFERENCE(external_pointer_table_grow_table_function,
+ ExternalPointerTable::GrowTable)
+#endif
+
+bool operator==(ExternalReference lhs, ExternalReference rhs) {
+ return lhs.address() == rhs.address();
+}
+
+bool operator!=(ExternalReference lhs, ExternalReference rhs) {
+ return !(lhs == rhs);
+}
+
+size_t hash_value(ExternalReference reference) {
+ if (FLAG_predictable) {
+ // Avoid ASLR non-determinism in predictable mode. For this, just take the
+ // lowest 12 bit corresponding to a 4K page size.
+ return base::hash<Address>()(reference.address() & 0xfff);
+ }
+ return base::hash<Address>()(reference.address());
+}
+
+std::ostream& operator<<(std::ostream& os, ExternalReference reference) {
+ os << reinterpret_cast<const void*>(reference.address());
+ const Runtime::Function* fn = Runtime::FunctionForEntry(reference.address());
+ if (fn) os << "<" << fn->name << ".entry>";
+ return os;
+}
+
+void abort_with_reason(int reason) {
+ if (IsValidAbortReason(reason)) {
+ const char* message = GetAbortReason(static_cast<AbortReason>(reason));
+ base::OS::PrintError("abort: %s\n", message);
+ } else {
+ base::OS::PrintError("abort: <unknown reason: %d>\n", reason);
+ }
+ base::OS::Abort();
+ UNREACHABLE();
+}
+
+#undef FUNCTION_REFERENCE
+#undef FUNCTION_REFERENCE_WITH_ISOLATE
+#undef FUNCTION_REFERENCE_WITH_TYPE
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/external-reference.h b/src/codegen/external-reference.h
new file mode 100644
index 0000000..72a3397
--- /dev/null
+++ b/src/codegen/external-reference.h
@@ -0,0 +1,384 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_EXTERNAL_REFERENCE_H_
+#define V8_CODEGEN_EXTERNAL_REFERENCE_H_
+
+#include "src/common/globals.h"
+#include "src/runtime/runtime.h"
+
+namespace v8 {
+
+class ApiFunction;
+
+namespace internal {
+
+class Isolate;
+class Page;
+class SCTableReference;
+class StatsCounter;
+
+//------------------------------------------------------------------------------
+// External references
+
+#define EXTERNAL_REFERENCE_LIST_WITH_ISOLATE(V) \
+ V(isolate_address, "isolate") \
+ V(builtins_address, "builtins") \
+ V(handle_scope_implementer_address, \
+ "Isolate::handle_scope_implementer_address") \
+ V(address_of_interpreter_entry_trampoline_instruction_start, \
+ "Address of the InterpreterEntryTrampoline instruction start") \
+ V(interpreter_dispatch_counters, "Interpreter::dispatch_counters") \
+ V(interpreter_dispatch_table_address, "Interpreter::dispatch_table_address") \
+ V(date_cache_stamp, "date_cache_stamp") \
+ V(stress_deopt_count, "Isolate::stress_deopt_count_address()") \
+ V(force_slow_path, "Isolate::force_slow_path_address()") \
+ V(isolate_root, "Isolate::isolate_root()") \
+ V(allocation_sites_list_address, "Heap::allocation_sites_list_address()") \
+ V(address_of_jslimit, "StackGuard::address_of_jslimit()") \
+ V(address_of_real_jslimit, "StackGuard::address_of_real_jslimit()") \
+ V(heap_is_marking_flag_address, "heap_is_marking_flag_address") \
+ V(new_space_allocation_top_address, "Heap::NewSpaceAllocationTopAddress()") \
+ V(new_space_allocation_limit_address, \
+ "Heap::NewSpaceAllocationLimitAddress()") \
+ V(old_space_allocation_top_address, "Heap::OldSpaceAllocationTopAddress") \
+ V(old_space_allocation_limit_address, \
+ "Heap::OldSpaceAllocationLimitAddress") \
+ V(handle_scope_level_address, "HandleScope::level") \
+ V(handle_scope_next_address, "HandleScope::next") \
+ V(handle_scope_limit_address, "HandleScope::limit") \
+ V(scheduled_exception_address, "Isolate::scheduled_exception") \
+ V(address_of_pending_message_obj, "address_of_pending_message_obj") \
+ V(promise_hook_address, "Isolate::promise_hook_address()") \
+ V(async_event_delegate_address, "Isolate::async_event_delegate_address()") \
+ V(promise_hook_or_async_event_delegate_address, \
+ "Isolate::promise_hook_or_async_event_delegate_address()") \
+ V(promise_hook_or_debug_is_active_or_async_event_delegate_address, \
+ "Isolate::promise_hook_or_debug_is_active_or_async_event_delegate_" \
+ "address()") \
+ V(debug_execution_mode_address, "Isolate::debug_execution_mode_address()") \
+ V(debug_is_active_address, "Debug::is_active_address()") \
+ V(debug_hook_on_function_call_address, \
+ "Debug::hook_on_function_call_address()") \
+ V(runtime_function_table_address, \
+ "Runtime::runtime_function_table_address()") \
+ V(is_profiling_address, "Isolate::is_profiling") \
+ V(debug_suspended_generator_address, \
+ "Debug::step_suspended_generator_address()") \
+ V(debug_restart_fp_address, "Debug::restart_fp_address()") \
+ V(fast_c_call_caller_fp_address, \
+ "IsolateData::fast_c_call_caller_fp_address") \
+ V(fast_c_call_caller_pc_address, \
+ "IsolateData::fast_c_call_caller_pc_address") \
+ V(stack_is_iterable_address, "IsolateData::stack_is_iterable_address") \
+ V(address_of_regexp_stack_limit_address, \
+ "RegExpStack::limit_address_address()") \
+ V(address_of_regexp_stack_memory_top_address, \
+ "RegExpStack::memory_top_address_address()") \
+ V(address_of_static_offsets_vector, "OffsetsVector::static_offsets_vector") \
+ V(re_case_insensitive_compare_unicode, \
+ "NativeRegExpMacroAssembler::CaseInsensitiveCompareUnicode()") \
+ V(re_case_insensitive_compare_non_unicode, \
+ "NativeRegExpMacroAssembler::CaseInsensitiveCompareNonUnicode()") \
+ V(re_check_stack_guard_state, \
+ "RegExpMacroAssembler*::CheckStackGuardState()") \
+ V(re_grow_stack, "NativeRegExpMacroAssembler::GrowStack()") \
+ V(re_word_character_map, "NativeRegExpMacroAssembler::word_character_map") \
+ EXTERNAL_REFERENCE_LIST_WITH_ISOLATE_HEAP_SANDBOX(V)
+
+#ifdef V8_HEAP_SANDBOX
+#define EXTERNAL_REFERENCE_LIST_WITH_ISOLATE_HEAP_SANDBOX(V) \
+ V(external_pointer_table_address, \
+ "Isolate::external_pointer_table_address(" \
+ ")")
+#else
+#define EXTERNAL_REFERENCE_LIST_WITH_ISOLATE_HEAP_SANDBOX(V)
+#endif // V8_HEAP_SANDBOX
+
+#define EXTERNAL_REFERENCE_LIST(V) \
+ V(abort_with_reason, "abort_with_reason") \
+ V(address_of_double_abs_constant, "double_absolute_constant") \
+ V(address_of_double_neg_constant, "double_negate_constant") \
+ V(address_of_enable_experimental_regexp_engine, \
+ "address_of_enable_experimental_regexp_engine") \
+ V(address_of_float_abs_constant, "float_absolute_constant") \
+ V(address_of_float_neg_constant, "float_negate_constant") \
+ V(address_of_min_int, "LDoubleConstant::min_int") \
+ V(address_of_mock_arraybuffer_allocator_flag, \
+ "FLAG_mock_arraybuffer_allocator") \
+ V(address_of_one_half, "LDoubleConstant::one_half") \
+ V(address_of_runtime_stats_flag, "TracingFlags::runtime_stats") \
+ V(address_of_the_hole_nan, "the_hole_nan") \
+ V(address_of_uint32_bias, "uint32_bias") \
+ V(bytecode_size_table_address, "Bytecodes::bytecode_size_table_address") \
+ V(check_object_type, "check_object_type") \
+ V(compute_integer_hash, "ComputeSeededHash") \
+ V(compute_output_frames_function, "Deoptimizer::ComputeOutputFrames()") \
+ V(copy_fast_number_jsarray_elements_to_typed_array, \
+ "copy_fast_number_jsarray_elements_to_typed_array") \
+ V(copy_typed_array_elements_slice, "copy_typed_array_elements_slice") \
+ V(copy_typed_array_elements_to_typed_array, \
+ "copy_typed_array_elements_to_typed_array") \
+ V(cpu_features, "cpu_features") \
+ V(delete_handle_scope_extensions, "HandleScope::DeleteExtensions") \
+ V(ephemeron_key_write_barrier_function, \
+ "Heap::EphemeronKeyWriteBarrierFromCode") \
+ V(f64_acos_wrapper_function, "f64_acos_wrapper") \
+ V(f64_asin_wrapper_function, "f64_asin_wrapper") \
+ V(f64_mod_wrapper_function, "f64_mod_wrapper") \
+ V(get_date_field_function, "JSDate::GetField") \
+ V(get_or_create_hash_raw, "get_or_create_hash_raw") \
+ V(ieee754_acos_function, "base::ieee754::acos") \
+ V(ieee754_acosh_function, "base::ieee754::acosh") \
+ V(ieee754_asin_function, "base::ieee754::asin") \
+ V(ieee754_asinh_function, "base::ieee754::asinh") \
+ V(ieee754_atan_function, "base::ieee754::atan") \
+ V(ieee754_atan2_function, "base::ieee754::atan2") \
+ V(ieee754_atanh_function, "base::ieee754::atanh") \
+ V(ieee754_cbrt_function, "base::ieee754::cbrt") \
+ V(ieee754_cos_function, "base::ieee754::cos") \
+ V(ieee754_cosh_function, "base::ieee754::cosh") \
+ V(ieee754_exp_function, "base::ieee754::exp") \
+ V(ieee754_expm1_function, "base::ieee754::expm1") \
+ V(ieee754_log_function, "base::ieee754::log") \
+ V(ieee754_log10_function, "base::ieee754::log10") \
+ V(ieee754_log1p_function, "base::ieee754::log1p") \
+ V(ieee754_log2_function, "base::ieee754::log2") \
+ V(ieee754_pow_function, "base::ieee754::pow") \
+ V(ieee754_sin_function, "base::ieee754::sin") \
+ V(ieee754_sinh_function, "base::ieee754::sinh") \
+ V(ieee754_tan_function, "base::ieee754::tan") \
+ V(ieee754_tanh_function, "base::ieee754::tanh") \
+ V(insert_remembered_set_function, "Heap::InsertIntoRememberedSetFromCode") \
+ V(invalidate_prototype_chains_function, \
+ "JSObject::InvalidatePrototypeChains()") \
+ V(invoke_accessor_getter_callback, "InvokeAccessorGetterCallback") \
+ V(invoke_function_callback, "InvokeFunctionCallback") \
+ V(jsarray_array_join_concat_to_sequential_string, \
+ "jsarray_array_join_concat_to_sequential_string") \
+ V(jsreceiver_create_identity_hash, "jsreceiver_create_identity_hash") \
+ V(libc_memchr_function, "libc_memchr") \
+ V(libc_memcpy_function, "libc_memcpy") \
+ V(libc_memmove_function, "libc_memmove") \
+ V(libc_memset_function, "libc_memset") \
+ V(mod_two_doubles_operation, "mod_two_doubles") \
+ V(mutable_big_int_absolute_add_and_canonicalize_function, \
+ "MutableBigInt_AbsoluteAddAndCanonicalize") \
+ V(mutable_big_int_absolute_compare_function, \
+ "MutableBigInt_AbsoluteCompare") \
+ V(mutable_big_int_absolute_sub_and_canonicalize_function, \
+ "MutableBigInt_AbsoluteSubAndCanonicalize") \
+ V(new_deoptimizer_function, "Deoptimizer::New()") \
+ V(orderedhashmap_gethash_raw, "orderedhashmap_gethash_raw") \
+ V(printf_function, "printf") \
+ V(refill_math_random, "MathRandom::RefillCache") \
+ V(search_string_raw_one_one, "search_string_raw_one_one") \
+ V(search_string_raw_one_two, "search_string_raw_one_two") \
+ V(search_string_raw_two_one, "search_string_raw_two_one") \
+ V(search_string_raw_two_two, "search_string_raw_two_two") \
+ V(smi_lexicographic_compare_function, "smi_lexicographic_compare_function") \
+ V(string_to_array_index_function, "String::ToArrayIndex") \
+ V(try_string_to_index_or_lookup_existing, \
+ "try_string_to_index_or_lookup_existing") \
+ V(wasm_call_trap_callback_for_testing, \
+ "wasm::call_trap_callback_for_testing") \
+ V(wasm_f32_ceil, "wasm::f32_ceil_wrapper") \
+ V(wasm_f32_floor, "wasm::f32_floor_wrapper") \
+ V(wasm_f32_nearest_int, "wasm::f32_nearest_int_wrapper") \
+ V(wasm_f32_trunc, "wasm::f32_trunc_wrapper") \
+ V(wasm_f64_ceil, "wasm::f64_ceil_wrapper") \
+ V(wasm_f64_floor, "wasm::f64_floor_wrapper") \
+ V(wasm_f64_nearest_int, "wasm::f64_nearest_int_wrapper") \
+ V(wasm_f64_trunc, "wasm::f64_trunc_wrapper") \
+ V(wasm_float32_to_int64, "wasm::float32_to_int64_wrapper") \
+ V(wasm_float32_to_uint64, "wasm::float32_to_uint64_wrapper") \
+ V(wasm_float32_to_int64_sat, "wasm::float32_to_int64_sat_wrapper") \
+ V(wasm_float32_to_uint64_sat, "wasm::float32_to_uint64_sat_wrapper") \
+ V(wasm_float64_pow, "wasm::float64_pow") \
+ V(wasm_float64_to_int64, "wasm::float64_to_int64_wrapper") \
+ V(wasm_float64_to_uint64, "wasm::float64_to_uint64_wrapper") \
+ V(wasm_float64_to_int64_sat, "wasm::float64_to_int64_sat_wrapper") \
+ V(wasm_float64_to_uint64_sat, "wasm::float64_to_uint64_sat_wrapper") \
+ V(wasm_int64_div, "wasm::int64_div") \
+ V(wasm_int64_mod, "wasm::int64_mod") \
+ V(wasm_int64_to_float32, "wasm::int64_to_float32_wrapper") \
+ V(wasm_int64_to_float64, "wasm::int64_to_float64_wrapper") \
+ V(wasm_uint64_div, "wasm::uint64_div") \
+ V(wasm_uint64_mod, "wasm::uint64_mod") \
+ V(wasm_uint64_to_float32, "wasm::uint64_to_float32_wrapper") \
+ V(wasm_uint64_to_float64, "wasm::uint64_to_float64_wrapper") \
+ V(wasm_word32_ctz, "wasm::word32_ctz") \
+ V(wasm_word32_popcnt, "wasm::word32_popcnt") \
+ V(wasm_word32_rol, "wasm::word32_rol") \
+ V(wasm_word32_ror, "wasm::word32_ror") \
+ V(wasm_word64_rol, "wasm::word64_rol") \
+ V(wasm_word64_ror, "wasm::word64_ror") \
+ V(wasm_word64_ctz, "wasm::word64_ctz") \
+ V(wasm_word64_popcnt, "wasm::word64_popcnt") \
+ V(wasm_f64x2_ceil, "wasm::f64x2_ceil_wrapper") \
+ V(wasm_f64x2_floor, "wasm::f64x2_floor_wrapper") \
+ V(wasm_f64x2_trunc, "wasm::f64x2_trunc_wrapper") \
+ V(wasm_f64x2_nearest_int, "wasm::f64x2_nearest_int_wrapper") \
+ V(wasm_f32x4_ceil, "wasm::f32x4_ceil_wrapper") \
+ V(wasm_f32x4_floor, "wasm::f32x4_floor_wrapper") \
+ V(wasm_f32x4_trunc, "wasm::f32x4_trunc_wrapper") \
+ V(wasm_f32x4_nearest_int, "wasm::f32x4_nearest_int_wrapper") \
+ V(wasm_memory_init, "wasm::memory_init") \
+ V(wasm_memory_copy, "wasm::memory_copy") \
+ V(wasm_memory_fill, "wasm::memory_fill") \
+ V(write_barrier_marking_from_code_function, "WriteBarrier::MarkingFromCode") \
+ V(call_enqueue_microtask_function, "MicrotaskQueue::CallEnqueueMicrotask") \
+ V(call_enter_context_function, "call_enter_context_function") \
+ V(atomic_pair_load_function, "atomic_pair_load_function") \
+ V(atomic_pair_store_function, "atomic_pair_store_function") \
+ V(atomic_pair_add_function, "atomic_pair_add_function") \
+ V(atomic_pair_sub_function, "atomic_pair_sub_function") \
+ V(atomic_pair_and_function, "atomic_pair_and_function") \
+ V(atomic_pair_or_function, "atomic_pair_or_function") \
+ V(atomic_pair_xor_function, "atomic_pair_xor_function") \
+ V(atomic_pair_exchange_function, "atomic_pair_exchange_function") \
+ V(atomic_pair_compare_exchange_function, \
+ "atomic_pair_compare_exchange_function") \
+ V(js_finalization_registry_remove_cell_from_unregister_token_map, \
+ "JSFinalizationRegistry::RemoveCellFromUnregisterTokenMap") \
+ V(re_match_for_call_from_js, "IrregexpInterpreter::MatchForCallFromJs") \
+ V(re_experimental_match_for_call_from_js, \
+ "ExperimentalRegExp::MatchForCallFromJs") \
+ EXTERNAL_REFERENCE_LIST_INTL(V) \
+ EXTERNAL_REFERENCE_LIST_HEAP_SANDBOX(V)
+
+#ifdef V8_INTL_SUPPORT
+#define EXTERNAL_REFERENCE_LIST_INTL(V) \
+ V(intl_convert_one_byte_to_lower, "intl_convert_one_byte_to_lower") \
+ V(intl_to_latin1_lower_table, "intl_to_latin1_lower_table")
+#else
+#define EXTERNAL_REFERENCE_LIST_INTL(V)
+#endif // V8_INTL_SUPPORT
+
+#ifdef V8_HEAP_SANDBOX
+#define EXTERNAL_REFERENCE_LIST_HEAP_SANDBOX(V) \
+ V(external_pointer_table_grow_table_function, \
+ "ExternalPointerTable::GrowTable")
+#else
+#define EXTERNAL_REFERENCE_LIST_HEAP_SANDBOX(V)
+#endif // V8_HEAP_SANDBOX
+
+// An ExternalReference represents a C++ address used in the generated
+// code. All references to C++ functions and variables must be encapsulated
+// in an ExternalReference instance. This is done in order to track the
+// origin of all external references in the code so that they can be bound
+// to the correct addresses when deserializing a heap.
+class ExternalReference {
+ public:
+ // Used in the simulator to support different native api calls.
+ enum Type {
+ // Builtin call.
+ // Address f(v8::internal::Arguments).
+ BUILTIN_CALL, // default
+
+ // Builtin call returning object pair.
+ // ObjectPair f(v8::internal::Arguments).
+ BUILTIN_CALL_PAIR,
+
+ // Builtin that takes float arguments and returns an int.
+ // int f(double, double).
+ BUILTIN_COMPARE_CALL,
+
+ // Builtin call that returns floating point.
+ // double f(double, double).
+ BUILTIN_FP_FP_CALL,
+
+ // Builtin call that returns floating point.
+ // double f(double).
+ BUILTIN_FP_CALL,
+
+ // Builtin call that returns floating point.
+ // double f(double, int).
+ BUILTIN_FP_INT_CALL,
+
+ // Direct call to API function callback.
+ // void f(v8::FunctionCallbackInfo&)
+ DIRECT_API_CALL,
+
+ // Call to function callback via InvokeFunctionCallback.
+ // void f(v8::FunctionCallbackInfo&, v8::FunctionCallback)
+ PROFILING_API_CALL,
+
+ // Direct call to accessor getter callback.
+ // void f(Local<Name> property, PropertyCallbackInfo& info)
+ DIRECT_GETTER_CALL,
+
+ // Call to accessor getter callback via InvokeAccessorGetterCallback.
+ // void f(Local<Name> property, PropertyCallbackInfo& info,
+ // AccessorNameGetterCallback callback)
+ PROFILING_GETTER_CALL
+ };
+
+ static constexpr int kExternalReferenceCount =
+#define COUNT_EXTERNAL_REFERENCE(name, desc) +1
+ EXTERNAL_REFERENCE_LIST(COUNT_EXTERNAL_REFERENCE)
+ EXTERNAL_REFERENCE_LIST_WITH_ISOLATE(COUNT_EXTERNAL_REFERENCE);
+#undef COUNT_EXTERNAL_REFERENCE
+
+ ExternalReference() : address_(kNullAddress) {}
+ static ExternalReference Create(const SCTableReference& table_ref);
+ static ExternalReference Create(StatsCounter* counter);
+ static V8_EXPORT_PRIVATE ExternalReference Create(ApiFunction* ptr,
+ Type type);
+ static ExternalReference Create(const Runtime::Function* f);
+ static ExternalReference Create(IsolateAddressId id, Isolate* isolate);
+ static ExternalReference Create(Runtime::FunctionId id);
+ static V8_EXPORT_PRIVATE ExternalReference Create(Address address);
+
+ template <typename SubjectChar, typename PatternChar>
+ static ExternalReference search_string_raw();
+
+ V8_EXPORT_PRIVATE static ExternalReference FromRawAddress(Address address);
+
+#define DECL_EXTERNAL_REFERENCE(name, desc) \
+ V8_EXPORT_PRIVATE static ExternalReference name();
+ EXTERNAL_REFERENCE_LIST(DECL_EXTERNAL_REFERENCE)
+#undef DECL_EXTERNAL_REFERENCE
+
+#define DECL_EXTERNAL_REFERENCE(name, desc) \
+ static V8_EXPORT_PRIVATE ExternalReference name(Isolate* isolate);
+ EXTERNAL_REFERENCE_LIST_WITH_ISOLATE(DECL_EXTERNAL_REFERENCE)
+#undef DECL_EXTERNAL_REFERENCE
+
+ V8_EXPORT_PRIVATE V8_NOINLINE static ExternalReference
+ runtime_function_table_address_for_unittests(Isolate* isolate);
+
+ static V8_EXPORT_PRIVATE ExternalReference
+ address_of_load_from_stack_count(const char* function_name);
+ static V8_EXPORT_PRIVATE ExternalReference
+ address_of_store_to_stack_count(const char* function_name);
+
+ Address address() const { return address_; }
+
+ private:
+ explicit ExternalReference(Address address) : address_(address) {}
+
+ explicit ExternalReference(void* address)
+ : address_(reinterpret_cast<Address>(address)) {}
+
+ static Address Redirect(Address address_arg,
+ Type type = ExternalReference::BUILTIN_CALL);
+
+ Address address_;
+};
+ASSERT_TRIVIALLY_COPYABLE(ExternalReference);
+
+V8_EXPORT_PRIVATE bool operator==(ExternalReference, ExternalReference);
+bool operator!=(ExternalReference, ExternalReference);
+
+size_t hash_value(ExternalReference);
+
+V8_EXPORT_PRIVATE std::ostream& operator<<(std::ostream&, ExternalReference);
+
+void abort_with_reason(int reason);
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_EXTERNAL_REFERENCE_H_
diff --git a/src/codegen/flush-instruction-cache.cc b/src/codegen/flush-instruction-cache.cc
new file mode 100644
index 0000000..cb4088a
--- /dev/null
+++ b/src/codegen/flush-instruction-cache.cc
@@ -0,0 +1,27 @@
+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/flush-instruction-cache.h"
+
+#include "src/base/platform/mutex.h"
+#include "src/codegen/cpu-features.h"
+#include "src/execution/simulator.h"
+
+namespace v8 {
+namespace internal {
+
+void FlushInstructionCache(void* start, size_t size) {
+ if (size == 0) return;
+ if (FLAG_jitless) return;
+
+#if defined(USE_SIMULATOR)
+ base::MutexGuard lock_guard(Simulator::i_cache_mutex());
+ Simulator::FlushICache(Simulator::i_cache(), start, size);
+#else
+ CpuFeatures::FlushICache(start, size);
+#endif // USE_SIMULATOR
+}
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/flush-instruction-cache.h b/src/codegen/flush-instruction-cache.h
new file mode 100644
index 0000000..88e5bd3
--- /dev/null
+++ b/src/codegen/flush-instruction-cache.h
@@ -0,0 +1,23 @@
+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_FLUSH_INSTRUCTION_CACHE_H_
+#define V8_CODEGEN_FLUSH_INSTRUCTION_CACHE_H_
+
+#include "include/v8-internal.h"
+#include "src/base/macros.h"
+
+namespace v8 {
+namespace internal {
+
+V8_EXPORT_PRIVATE void FlushInstructionCache(void* start, size_t size);
+V8_EXPORT_PRIVATE V8_INLINE void FlushInstructionCache(Address start,
+ size_t size) {
+ return FlushInstructionCache(reinterpret_cast<void*>(start), size);
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_FLUSH_INSTRUCTION_CACHE_H_
diff --git a/src/codegen/handler-table.cc b/src/codegen/handler-table.cc
new file mode 100644
index 0000000..8aec047
--- /dev/null
+++ b/src/codegen/handler-table.cc
@@ -0,0 +1,261 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/handler-table.h"
+
+#include <algorithm>
+#include <iomanip>
+
+#include "src/base/iterator.h"
+#include "src/codegen/assembler-inl.h"
+#include "src/objects/code-inl.h"
+#include "src/objects/objects-inl.h"
+#include "src/wasm/wasm-code-manager.h"
+
+namespace v8 {
+namespace internal {
+
+HandlerTable::HandlerTable(Code code)
+ : HandlerTable(code.HandlerTableAddress(), code.handler_table_size(),
+ kReturnAddressBasedEncoding) {}
+
+HandlerTable::HandlerTable(const wasm::WasmCode* code)
+ : HandlerTable(code->handler_table(), code->handler_table_size(),
+ kReturnAddressBasedEncoding) {}
+
+HandlerTable::HandlerTable(BytecodeArray bytecode_array)
+ : HandlerTable(bytecode_array.handler_table()) {}
+
+HandlerTable::HandlerTable(ByteArray byte_array)
+ : HandlerTable(reinterpret_cast<Address>(byte_array.GetDataStartAddress()),
+ byte_array.length(), kRangeBasedEncoding) {}
+
+HandlerTable::HandlerTable(Address handler_table, int handler_table_size,
+ EncodingMode encoding_mode)
+ : number_of_entries_(handler_table_size / EntrySizeFromMode(encoding_mode) /
+ sizeof(int32_t)),
+#ifdef DEBUG
+ mode_(encoding_mode),
+#endif
+ raw_encoded_data_(handler_table) {
+ // Check padding.
+ static_assert(4 < kReturnEntrySize * sizeof(int32_t), "allowed padding");
+ // For return address encoding, maximum padding is 4; otherwise, there should
+ // be no padding.
+ DCHECK_GE(kReturnAddressBasedEncoding == encoding_mode ? 4 : 0,
+ handler_table_size %
+ (EntrySizeFromMode(encoding_mode) * sizeof(int32_t)));
+}
+
+// static
+int HandlerTable::EntrySizeFromMode(EncodingMode mode) {
+ switch (mode) {
+ case kReturnAddressBasedEncoding:
+ return kReturnEntrySize;
+ case kRangeBasedEncoding:
+ return kRangeEntrySize;
+ }
+ UNREACHABLE();
+}
+
+int HandlerTable::GetRangeStart(int index) const {
+ DCHECK_EQ(kRangeBasedEncoding, mode_);
+ DCHECK_LT(index, NumberOfRangeEntries());
+ int offset = index * kRangeEntrySize + kRangeStartIndex;
+ return Memory<int32_t>(raw_encoded_data_ + offset * sizeof(int32_t));
+}
+
+int HandlerTable::GetRangeEnd(int index) const {
+ DCHECK_EQ(kRangeBasedEncoding, mode_);
+ DCHECK_LT(index, NumberOfRangeEntries());
+ int offset = index * kRangeEntrySize + kRangeEndIndex;
+ return Memory<int32_t>(raw_encoded_data_ + offset * sizeof(int32_t));
+}
+
+int HandlerTable::GetRangeHandler(int index) const {
+ DCHECK_EQ(kRangeBasedEncoding, mode_);
+ DCHECK_LT(index, NumberOfRangeEntries());
+ int offset = index * kRangeEntrySize + kRangeHandlerIndex;
+ return HandlerOffsetField::decode(
+ Memory<int32_t>(raw_encoded_data_ + offset * sizeof(int32_t)));
+}
+
+int HandlerTable::GetRangeData(int index) const {
+ DCHECK_EQ(kRangeBasedEncoding, mode_);
+ DCHECK_LT(index, NumberOfRangeEntries());
+ int offset = index * kRangeEntrySize + kRangeDataIndex;
+ return Memory<int32_t>(raw_encoded_data_ + offset * sizeof(int32_t));
+}
+
+HandlerTable::CatchPrediction HandlerTable::GetRangePrediction(
+ int index) const {
+ DCHECK_EQ(kRangeBasedEncoding, mode_);
+ DCHECK_LT(index, NumberOfRangeEntries());
+ int offset = index * kRangeEntrySize + kRangeHandlerIndex;
+ return HandlerPredictionField::decode(
+ Memory<int32_t>(raw_encoded_data_ + offset * sizeof(int32_t)));
+}
+
+int HandlerTable::GetReturnOffset(int index) const {
+ DCHECK_EQ(kReturnAddressBasedEncoding, mode_);
+ DCHECK_LT(index, NumberOfReturnEntries());
+ int offset = index * kReturnEntrySize + kReturnOffsetIndex;
+ return Memory<int32_t>(raw_encoded_data_ + offset * sizeof(int32_t));
+}
+
+int HandlerTable::GetReturnHandler(int index) const {
+ DCHECK_EQ(kReturnAddressBasedEncoding, mode_);
+ DCHECK_LT(index, NumberOfReturnEntries());
+ int offset = index * kReturnEntrySize + kReturnHandlerIndex;
+ return HandlerOffsetField::decode(
+ Memory<int32_t>(raw_encoded_data_ + offset * sizeof(int32_t)));
+}
+
+void HandlerTable::SetRangeStart(int index, int value) {
+ int offset = index * kRangeEntrySize + kRangeStartIndex;
+ Memory<int32_t>(raw_encoded_data_ + offset * sizeof(int32_t)) = value;
+}
+
+void HandlerTable::SetRangeEnd(int index, int value) {
+ int offset = index * kRangeEntrySize + kRangeEndIndex;
+ Memory<int32_t>(raw_encoded_data_ + offset * sizeof(int32_t)) = value;
+}
+
+void HandlerTable::SetRangeHandler(int index, int handler_offset,
+ CatchPrediction prediction) {
+ int value = HandlerOffsetField::encode(handler_offset) |
+ HandlerPredictionField::encode(prediction);
+ int offset = index * kRangeEntrySize + kRangeHandlerIndex;
+ Memory<int32_t>(raw_encoded_data_ + offset * sizeof(int32_t)) = value;
+}
+
+void HandlerTable::SetRangeData(int index, int value) {
+ int offset = index * kRangeEntrySize + kRangeDataIndex;
+ Memory<int32_t>(raw_encoded_data_ + offset * sizeof(int32_t)) = value;
+}
+
+// static
+int HandlerTable::LengthForRange(int entries) {
+ return entries * kRangeEntrySize * sizeof(int32_t);
+}
+
+// static
+int HandlerTable::EmitReturnTableStart(Assembler* masm) {
+ masm->DataAlign(Code::kMetadataAlignment);
+ masm->RecordComment(";;; Exception handler table.");
+ int table_start = masm->pc_offset();
+ return table_start;
+}
+
+// static
+void HandlerTable::EmitReturnEntry(Assembler* masm, int offset, int handler) {
+ masm->dd(offset);
+ masm->dd(HandlerOffsetField::encode(handler));
+}
+
+int HandlerTable::NumberOfRangeEntries() const {
+ DCHECK_EQ(kRangeBasedEncoding, mode_);
+ return number_of_entries_;
+}
+
+int HandlerTable::NumberOfReturnEntries() const {
+ DCHECK_EQ(kReturnAddressBasedEncoding, mode_);
+ return number_of_entries_;
+}
+
+int HandlerTable::LookupRange(int pc_offset, int* data_out,
+ CatchPrediction* prediction_out) {
+ int innermost_handler = -1;
+#ifdef DEBUG
+ // Assuming that ranges are well nested, we don't need to track the innermost
+ // offsets. This is just to verify that the table is actually well nested.
+ int innermost_start = std::numeric_limits<int>::min();
+ int innermost_end = std::numeric_limits<int>::max();
+#endif
+ for (int i = 0; i < NumberOfRangeEntries(); ++i) {
+ int start_offset = GetRangeStart(i);
+ int end_offset = GetRangeEnd(i);
+ int handler_offset = GetRangeHandler(i);
+ int handler_data = GetRangeData(i);
+ CatchPrediction prediction = GetRangePrediction(i);
+ if (pc_offset >= start_offset && pc_offset < end_offset) {
+ DCHECK_GE(start_offset, innermost_start);
+ DCHECK_LT(end_offset, innermost_end);
+ innermost_handler = handler_offset;
+#ifdef DEBUG
+ innermost_start = start_offset;
+ innermost_end = end_offset;
+#endif
+ if (data_out) *data_out = handler_data;
+ if (prediction_out) *prediction_out = prediction;
+ }
+ }
+ return innermost_handler;
+}
+
+int HandlerTable::LookupReturn(int pc_offset) {
+ // We only implement the methods needed by the standard libraries we care
+ // about. This is not technically a full random access iterator by the spec.
+ struct Iterator : base::iterator<std::random_access_iterator_tag, int> {
+ Iterator(HandlerTable* tbl, int idx) : table(tbl), index(idx) {}
+ value_type operator*() const { return table->GetReturnOffset(index); }
+ bool operator!=(const Iterator& other) const { return !(*this == other); }
+ bool operator==(const Iterator& other) const {
+ return index == other.index;
+ }
+ Iterator& operator++() {
+ index++;
+ return *this;
+ }
+ Iterator& operator--() {
+ index--;
+ return *this;
+ }
+ Iterator& operator+=(difference_type offset) {
+ index += offset;
+ return *this;
+ }
+ difference_type operator-(const Iterator& other) const {
+ return index - other.index;
+ }
+ HandlerTable* table;
+ int index;
+ };
+ Iterator begin{this, 0}, end{this, NumberOfReturnEntries()};
+ SLOW_DCHECK(std::is_sorted(begin, end)); // Must be sorted.
+ Iterator result = std::lower_bound(begin, end, pc_offset);
+ bool exact_match = result != end && *result == pc_offset;
+ return exact_match ? GetReturnHandler(result.index) : -1;
+}
+
+#ifdef ENABLE_DISASSEMBLER
+
+void HandlerTable::HandlerTableRangePrint(std::ostream& os) {
+ os << " from to hdlr (prediction, data)\n";
+ for (int i = 0; i < NumberOfRangeEntries(); ++i) {
+ int pc_start = GetRangeStart(i);
+ int pc_end = GetRangeEnd(i);
+ int handler_offset = GetRangeHandler(i);
+ int handler_data = GetRangeData(i);
+ CatchPrediction prediction = GetRangePrediction(i);
+ os << " (" << std::setw(4) << pc_start << "," << std::setw(4) << pc_end
+ << ") -> " << std::setw(4) << handler_offset
+ << " (prediction=" << prediction << ", data=" << handler_data << ")\n";
+ }
+}
+
+void HandlerTable::HandlerTableReturnPrint(std::ostream& os) {
+ os << " offset handler\n";
+ for (int i = 0; i < NumberOfReturnEntries(); ++i) {
+ int pc_offset = GetReturnOffset(i);
+ int handler_offset = GetReturnHandler(i);
+ os << std::hex << " " << std::setw(4) << pc_offset << " -> "
+ << std::setw(4) << handler_offset << std::dec << "\n";
+ }
+}
+
+#endif // ENABLE_DISASSEMBLER
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/handler-table.h b/src/codegen/handler-table.h
new file mode 100644
index 0000000..0445b68
--- /dev/null
+++ b/src/codegen/handler-table.h
@@ -0,0 +1,148 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_HANDLER_TABLE_H_
+#define V8_CODEGEN_HANDLER_TABLE_H_
+
+#include "src/base/bit-field.h"
+#include "src/common/assert-scope.h"
+#include "src/common/globals.h"
+
+namespace v8 {
+namespace internal {
+
+class Assembler;
+class ByteArray;
+class BytecodeArray;
+
+namespace wasm {
+class WasmCode;
+} // namespace wasm
+
+// HandlerTable is a byte array containing entries for exception handlers in
+// the code object it is associated with. The tables come in two flavors:
+// 1) Based on ranges: Used for unoptimized code. Stored in a {ByteArray} that
+// is attached to each {BytecodeArray}. Contains one entry per exception
+// handler and a range representing the try-block covered by that handler.
+// Layout looks as follows:
+// [ range-start , range-end , handler-offset , handler-data ]
+// 2) Based on return addresses: Used for turbofanned code. Stored directly in
+// the instruction stream of the {Code} object. Contains one entry per
+// call-site that could throw an exception. Layout looks as follows:
+// [ return-address-offset , handler-offset ]
+class V8_EXPORT_PRIVATE HandlerTable {
+ public:
+ // Conservative prediction whether a given handler will locally catch an
+ // exception or cause a re-throw to outside the code boundary. Since this is
+ // undecidable it is merely an approximation (e.g. useful for debugger).
+ enum CatchPrediction {
+ UNCAUGHT, // The handler will (likely) rethrow the exception.
+ CAUGHT, // The exception will be caught by the handler.
+ PROMISE, // The exception will be caught and cause a promise rejection.
+ DESUGARING, // The exception will be caught, but both the exception and
+ // the catching are part of a desugaring and should therefore
+ // not be visible to the user (we won't notify the debugger of
+ // such exceptions).
+ ASYNC_AWAIT, // The exception will be caught and cause a promise rejection
+ // in the desugaring of an async function, so special
+ // async/await handling in the debugger can take place.
+ UNCAUGHT_ASYNC_AWAIT, // The exception will be caught and cause a promise
+ // rejection in the desugaring of an async REPL
+ // script. The corresponding message object needs to
+ // be kept alive on the Isolate though.
+ };
+
+ enum EncodingMode { kRangeBasedEncoding, kReturnAddressBasedEncoding };
+
+ // Constructors for the various encodings.
+ explicit HandlerTable(Code code);
+ explicit HandlerTable(ByteArray byte_array);
+ explicit HandlerTable(const wasm::WasmCode* code);
+ explicit HandlerTable(BytecodeArray bytecode_array);
+ HandlerTable(Address handler_table, int handler_table_size,
+ EncodingMode encoding_mode);
+
+ // Getters for handler table based on ranges.
+ int GetRangeStart(int index) const;
+ int GetRangeEnd(int index) const;
+ int GetRangeHandler(int index) const;
+ int GetRangeData(int index) const;
+
+ // Setters for handler table based on ranges.
+ void SetRangeStart(int index, int value);
+ void SetRangeEnd(int index, int value);
+ void SetRangeHandler(int index, int offset, CatchPrediction pred);
+ void SetRangeData(int index, int value);
+
+ // Returns the required length of the underlying byte array.
+ static int LengthForRange(int entries);
+
+ // Emitters for handler table based on return addresses.
+ static int EmitReturnTableStart(Assembler* masm);
+ static void EmitReturnEntry(Assembler* masm, int offset, int handler);
+
+ // Lookup handler in a table based on ranges. The {pc_offset} is an offset to
+ // the start of the potentially throwing instruction (using return addresses
+ // for this value would be invalid).
+ int LookupRange(int pc_offset, int* data, CatchPrediction* prediction);
+
+ // Lookup handler in a table based on return addresses.
+ int LookupReturn(int pc_offset);
+
+ // Returns the number of entries in the table.
+ int NumberOfRangeEntries() const;
+ int NumberOfReturnEntries() const;
+
+#ifdef ENABLE_DISASSEMBLER
+ void HandlerTableRangePrint(std::ostream& os); // NOLINT
+ void HandlerTableReturnPrint(std::ostream& os); // NOLINT
+#endif
+
+ private:
+ // Getters for handler table based on ranges.
+ CatchPrediction GetRangePrediction(int index) const;
+
+ // Gets entry size based on mode.
+ static int EntrySizeFromMode(EncodingMode mode);
+
+ // Getters for handler table based on return addresses.
+ int GetReturnOffset(int index) const;
+ int GetReturnHandler(int index) const;
+
+ // Number of entries in the loaded handler table.
+ const int number_of_entries_;
+
+#ifdef DEBUG
+ // The encoding mode of the table. Mostly useful for debugging to check that
+ // used accessors and constructors fit together.
+ const EncodingMode mode_;
+#endif
+
+ // Direct pointer into the encoded data. This pointer potentially points into
+ // objects on the GC heap (either {ByteArray} or {Code}) and could become
+ // stale during a collection. Hence we disallow any allocation.
+ const Address raw_encoded_data_;
+ DISALLOW_HEAP_ALLOCATION(no_gc_)
+
+ // Layout description for handler table based on ranges.
+ static const int kRangeStartIndex = 0;
+ static const int kRangeEndIndex = 1;
+ static const int kRangeHandlerIndex = 2;
+ static const int kRangeDataIndex = 3;
+ static const int kRangeEntrySize = 4;
+
+ // Layout description for handler table based on return addresses.
+ static const int kReturnOffsetIndex = 0;
+ static const int kReturnHandlerIndex = 1;
+ static const int kReturnEntrySize = 2;
+
+ // Encoding of the {handler} field.
+ using HandlerPredictionField = base::BitField<CatchPrediction, 0, 3>;
+ using HandlerOffsetField = base::BitField<int, 3, 29>;
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_HANDLER_TABLE_H_
diff --git a/src/codegen/ia32/assembler-ia32-inl.h b/src/codegen/ia32/assembler-ia32-inl.h
new file mode 100644
index 0000000..174a483
--- /dev/null
+++ b/src/codegen/ia32/assembler-ia32-inl.h
@@ -0,0 +1,305 @@
+// Copyright (c) 1994-2006 Sun Microsystems Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// - Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// - Redistribution in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// - Neither the name of Sun Microsystems or the names of contributors may
+// be used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// The original source code covered by the above license above has been
+// modified significantly by Google Inc.
+// Copyright 2012 the V8 project authors. All rights reserved.
+
+// A light-weight IA32 Assembler.
+
+#ifndef V8_CODEGEN_IA32_ASSEMBLER_IA32_INL_H_
+#define V8_CODEGEN_IA32_ASSEMBLER_IA32_INL_H_
+
+#include "src/codegen/ia32/assembler-ia32.h"
+
+#include "src/base/memory.h"
+#include "src/codegen/assembler.h"
+#include "src/debug/debug.h"
+#include "src/objects/objects-inl.h"
+
+namespace v8 {
+namespace internal {
+
+bool CpuFeatures::SupportsOptimizer() { return true; }
+
+bool CpuFeatures::SupportsWasmSimd128() { return IsSupported(SSE4_1); }
+
+// The modes possibly affected by apply must be in kApplyMask.
+void RelocInfo::apply(intptr_t delta) {
+ DCHECK_EQ(kApplyMask, (RelocInfo::ModeMask(RelocInfo::CODE_TARGET) |
+ RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE) |
+ RelocInfo::ModeMask(RelocInfo::OFF_HEAP_TARGET) |
+ RelocInfo::ModeMask(RelocInfo::RUNTIME_ENTRY)));
+ if (IsRuntimeEntry(rmode_) || IsCodeTarget(rmode_) ||
+ IsOffHeapTarget(rmode_)) {
+ base::WriteUnalignedValue(pc_,
+ base::ReadUnalignedValue<int32_t>(pc_) - delta);
+ } else if (IsInternalReference(rmode_)) {
+ // Absolute code pointer inside code object moves with the code object.
+ base::WriteUnalignedValue(pc_,
+ base::ReadUnalignedValue<int32_t>(pc_) + delta);
+ }
+}
+
+Address RelocInfo::target_address() {
+ DCHECK(IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_) || IsWasmCall(rmode_));
+ return Assembler::target_address_at(pc_, constant_pool_);
+}
+
+Address RelocInfo::target_address_address() {
+ DCHECK(HasTargetAddressAddress());
+ return pc_;
+}
+
+Address RelocInfo::constant_pool_entry_address() { UNREACHABLE(); }
+
+int RelocInfo::target_address_size() { return Assembler::kSpecialTargetSize; }
+
+HeapObject RelocInfo::target_object() {
+ DCHECK(IsCodeTarget(rmode_) || rmode_ == FULL_EMBEDDED_OBJECT);
+ return HeapObject::cast(Object(ReadUnalignedValue<Address>(pc_)));
+}
+
+HeapObject RelocInfo::target_object_no_host(Isolate* isolate) {
+ return target_object();
+}
+
+Handle<HeapObject> RelocInfo::target_object_handle(Assembler* origin) {
+ DCHECK(IsCodeTarget(rmode_) || rmode_ == FULL_EMBEDDED_OBJECT);
+ return Handle<HeapObject>::cast(ReadUnalignedValue<Handle<Object>>(pc_));
+}
+
+void RelocInfo::set_target_object(Heap* heap, HeapObject target,
+ WriteBarrierMode write_barrier_mode,
+ ICacheFlushMode icache_flush_mode) {
+ DCHECK(IsCodeTarget(rmode_) || rmode_ == FULL_EMBEDDED_OBJECT);
+ WriteUnalignedValue(pc_, target.ptr());
+ if (icache_flush_mode != SKIP_ICACHE_FLUSH) {
+ FlushInstructionCache(pc_, sizeof(Address));
+ }
+ if (write_barrier_mode == UPDATE_WRITE_BARRIER && !host().is_null() &&
+ !FLAG_disable_write_barriers) {
+ WriteBarrierForCode(host(), this, target);
+ }
+}
+
+Address RelocInfo::target_external_reference() {
+ DCHECK(rmode_ == RelocInfo::EXTERNAL_REFERENCE);
+ return ReadUnalignedValue<Address>(pc_);
+}
+
+void RelocInfo::set_target_external_reference(
+ Address target, ICacheFlushMode icache_flush_mode) {
+ DCHECK(rmode_ == RelocInfo::EXTERNAL_REFERENCE);
+ WriteUnalignedValue(pc_, target);
+ if (icache_flush_mode != SKIP_ICACHE_FLUSH) {
+ FlushInstructionCache(pc_, sizeof(Address));
+ }
+}
+
+Address RelocInfo::target_internal_reference() {
+ DCHECK(rmode_ == INTERNAL_REFERENCE);
+ return ReadUnalignedValue<Address>(pc_);
+}
+
+Address RelocInfo::target_internal_reference_address() {
+ DCHECK(rmode_ == INTERNAL_REFERENCE);
+ return pc_;
+}
+
+Address RelocInfo::target_runtime_entry(Assembler* origin) {
+ DCHECK(IsRuntimeEntry(rmode_));
+ return ReadUnalignedValue<Address>(pc_);
+}
+
+void RelocInfo::set_target_runtime_entry(Address target,
+ WriteBarrierMode write_barrier_mode,
+ ICacheFlushMode icache_flush_mode) {
+ DCHECK(IsRuntimeEntry(rmode_));
+ if (target_address() != target) {
+ set_target_address(target, write_barrier_mode, icache_flush_mode);
+ }
+}
+
+Address RelocInfo::target_off_heap_target() {
+ DCHECK(IsOffHeapTarget(rmode_));
+ return Assembler::target_address_at(pc_, constant_pool_);
+}
+
+void RelocInfo::WipeOut() {
+ if (IsFullEmbeddedObject(rmode_) || IsExternalReference(rmode_) ||
+ IsInternalReference(rmode_)) {
+ WriteUnalignedValue(pc_, kNullAddress);
+ } else if (IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_) ||
+ IsOffHeapTarget(rmode_)) {
+ // Effectively write zero into the relocation.
+ Assembler::set_target_address_at(pc_, constant_pool_,
+ pc_ + sizeof(int32_t));
+ } else {
+ UNREACHABLE();
+ }
+}
+
+void Assembler::emit(uint32_t x) {
+ WriteUnalignedValue(reinterpret_cast<Address>(pc_), x);
+ pc_ += sizeof(uint32_t);
+}
+
+void Assembler::emit_q(uint64_t x) {
+ WriteUnalignedValue(reinterpret_cast<Address>(pc_), x);
+ pc_ += sizeof(uint64_t);
+}
+
+void Assembler::emit(Handle<HeapObject> handle) {
+ emit(handle.address(), RelocInfo::FULL_EMBEDDED_OBJECT);
+}
+
+void Assembler::emit(uint32_t x, RelocInfo::Mode rmode) {
+ if (!RelocInfo::IsNone(rmode)) {
+ RecordRelocInfo(rmode);
+ }
+ emit(x);
+}
+
+void Assembler::emit(Handle<Code> code, RelocInfo::Mode rmode) {
+ emit(code.address(), rmode);
+}
+
+void Assembler::emit(const Immediate& x) {
+ if (x.rmode_ == RelocInfo::INTERNAL_REFERENCE) {
+ Label* label = reinterpret_cast<Label*>(x.immediate());
+ emit_code_relative_offset(label);
+ return;
+ }
+ if (!RelocInfo::IsNone(x.rmode_)) RecordRelocInfo(x.rmode_);
+ if (x.is_heap_object_request()) {
+ RequestHeapObject(x.heap_object_request());
+ emit(0);
+ } else {
+ emit(x.immediate());
+ }
+}
+
+void Assembler::emit_code_relative_offset(Label* label) {
+ if (label->is_bound()) {
+ int32_t pos;
+ pos = label->pos() + Code::kHeaderSize - kHeapObjectTag;
+ emit(pos);
+ } else {
+ emit_disp(label, Displacement::CODE_RELATIVE);
+ }
+}
+
+void Assembler::emit_b(Immediate x) {
+ DCHECK(x.is_int8() || x.is_uint8());
+ uint8_t value = static_cast<uint8_t>(x.immediate());
+ *pc_++ = value;
+}
+
+void Assembler::emit_w(const Immediate& x) {
+ DCHECK(RelocInfo::IsNone(x.rmode_));
+ uint16_t value = static_cast<uint16_t>(x.immediate());
+ WriteUnalignedValue(reinterpret_cast<Address>(pc_), value);
+ pc_ += sizeof(uint16_t);
+}
+
+Address Assembler::target_address_at(Address pc, Address constant_pool) {
+ return pc + sizeof(int32_t) + ReadUnalignedValue<int32_t>(pc);
+}
+
+void Assembler::set_target_address_at(Address pc, Address constant_pool,
+ Address target,
+ ICacheFlushMode icache_flush_mode) {
+ WriteUnalignedValue(pc, target - (pc + sizeof(int32_t)));
+ if (icache_flush_mode != SKIP_ICACHE_FLUSH) {
+ FlushInstructionCache(pc, sizeof(int32_t));
+ }
+}
+
+void Assembler::deserialization_set_special_target_at(
+ Address instruction_payload, Code code, Address target) {
+ set_target_address_at(instruction_payload,
+ !code.is_null() ? code.constant_pool() : kNullAddress,
+ target);
+}
+
+int Assembler::deserialization_special_target_size(
+ Address instruction_payload) {
+ return kSpecialTargetSize;
+}
+
+Displacement Assembler::disp_at(Label* L) {
+ return Displacement(long_at(L->pos()));
+}
+
+void Assembler::disp_at_put(Label* L, Displacement disp) {
+ long_at_put(L->pos(), disp.data());
+}
+
+void Assembler::emit_disp(Label* L, Displacement::Type type) {
+ Displacement disp(L, type);
+ L->link_to(pc_offset());
+ emit(static_cast<int>(disp.data()));
+}
+
+void Assembler::emit_near_disp(Label* L) {
+ byte disp = 0x00;
+ if (L->is_near_linked()) {
+ int offset = L->near_link_pos() - pc_offset();
+ DCHECK(is_int8(offset));
+ disp = static_cast<byte>(offset & 0xFF);
+ }
+ L->link_to(pc_offset(), Label::kNear);
+ *pc_++ = disp;
+}
+
+void Assembler::deserialization_set_target_internal_reference_at(
+ Address pc, Address target, RelocInfo::Mode mode) {
+ WriteUnalignedValue(pc, target);
+}
+
+void Operand::set_sib(ScaleFactor scale, Register index, Register base) {
+ DCHECK_EQ(len_, 1);
+ DCHECK_EQ(scale & -4, 0);
+ // Use SIB with no index register only for base esp.
+ DCHECK(index != esp || base == esp);
+ buf_[1] = scale << 6 | index.code() << 3 | base.code();
+ len_ = 2;
+}
+
+void Operand::set_disp8(int8_t disp) {
+ DCHECK(len_ == 1 || len_ == 2);
+ *reinterpret_cast<int8_t*>(&buf_[len_++]) = disp;
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_IA32_ASSEMBLER_IA32_INL_H_
diff --git a/src/codegen/ia32/assembler-ia32.cc b/src/codegen/ia32/assembler-ia32.cc
new file mode 100644
index 0000000..f19c8dd
--- /dev/null
+++ b/src/codegen/ia32/assembler-ia32.cc
@@ -0,0 +1,3295 @@
+// Copyright (c) 1994-2006 Sun Microsystems Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//
+// - Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// - Redistribution in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the
+// distribution.
+//
+// - Neither the name of Sun Microsystems or the names of contributors may
+// be used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+// COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+// OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// The original source code covered by the above license above has been modified
+// significantly by Google Inc.
+// Copyright 2012 the V8 project authors. All rights reserved.
+
+#include "src/codegen/ia32/assembler-ia32.h"
+
+#include <cstring>
+
+#if V8_TARGET_ARCH_IA32
+
+#if V8_LIBC_MSVCRT
+#include <intrin.h> // _xgetbv()
+#endif
+#if V8_OS_MACOSX
+#include <sys/sysctl.h>
+#endif
+
+#include "src/base/bits.h"
+#include "src/base/cpu.h"
+#include "src/codegen/assembler-inl.h"
+#include "src/codegen/macro-assembler.h"
+#include "src/codegen/string-constants.h"
+#include "src/deoptimizer/deoptimizer.h"
+#include "src/diagnostics/disassembler.h"
+#include "src/init/v8.h"
+#include "src/numbers/conversions-inl.h"
+
+namespace v8 {
+namespace internal {
+
+Immediate Immediate::EmbeddedNumber(double value) {
+ int32_t smi;
+ if (DoubleToSmiInteger(value, &smi)) return Immediate(Smi::FromInt(smi));
+ Immediate result(0, RelocInfo::FULL_EMBEDDED_OBJECT);
+ result.is_heap_object_request_ = true;
+ result.value_.heap_object_request = HeapObjectRequest(value);
+ return result;
+}
+
+Immediate Immediate::EmbeddedStringConstant(const StringConstantBase* str) {
+ Immediate result(0, RelocInfo::FULL_EMBEDDED_OBJECT);
+ result.is_heap_object_request_ = true;
+ result.value_.heap_object_request = HeapObjectRequest(str);
+ return result;
+}
+
+// -----------------------------------------------------------------------------
+// Implementation of CpuFeatures
+
+namespace {
+
+#if !V8_LIBC_MSVCRT
+
+V8_INLINE uint64_t _xgetbv(unsigned int xcr) {
+ unsigned eax, edx;
+ // Check xgetbv; this uses a .byte sequence instead of the instruction
+ // directly because older assemblers do not include support for xgetbv and
+ // there is no easy way to conditionally compile based on the assembler
+ // used.
+ __asm__ volatile(".byte 0x0F, 0x01, 0xD0" : "=a"(eax), "=d"(edx) : "c"(xcr));
+ return static_cast<uint64_t>(eax) | (static_cast<uint64_t>(edx) << 32);
+}
+
+#define _XCR_XFEATURE_ENABLED_MASK 0
+
+#endif // !V8_LIBC_MSVCRT
+
+bool OSHasAVXSupport() {
+#if V8_OS_MACOSX
+ // Mac OS X up to 10.9 has a bug where AVX transitions were indeed being
+ // caused by ISRs, so we detect that here and disable AVX in that case.
+ char buffer[128];
+ size_t buffer_size = arraysize(buffer);
+ int ctl_name[] = {CTL_KERN, KERN_OSRELEASE};
+ if (sysctl(ctl_name, 2, buffer, &buffer_size, nullptr, 0) != 0) {
+ FATAL("V8 failed to get kernel version");
+ }
+ // The buffer now contains a string of the form XX.YY.ZZ, where
+ // XX is the major kernel version component.
+ char* period_pos = strchr(buffer, '.');
+ DCHECK_NOT_NULL(period_pos);
+ *period_pos = '\0';
+ long kernel_version_major = strtol(buffer, nullptr, 10); // NOLINT
+ if (kernel_version_major <= 13) return false;
+#endif // V8_OS_MACOSX
+ // Check whether OS claims to support AVX.
+ uint64_t feature_mask = _xgetbv(_XCR_XFEATURE_ENABLED_MASK);
+ return (feature_mask & 0x6) == 0x6;
+}
+
+#undef _XCR_XFEATURE_ENABLED_MASK
+
+} // namespace
+
+void CpuFeatures::ProbeImpl(bool cross_compile) {
+ base::CPU cpu;
+ CHECK(cpu.has_sse2()); // SSE2 support is mandatory.
+ CHECK(cpu.has_cmov()); // CMOV support is mandatory.
+
+ // Only use statically determined features for cross compile (snapshot).
+ if (cross_compile) return;
+
+ if (cpu.has_sse41() && FLAG_enable_sse4_1) supported_ |= 1u << SSE4_1;
+ if (cpu.has_ssse3() && FLAG_enable_ssse3) supported_ |= 1u << SSSE3;
+ if (cpu.has_sse3() && FLAG_enable_sse3) supported_ |= 1u << SSE3;
+ if (cpu.has_avx() && FLAG_enable_avx && cpu.has_osxsave() &&
+ OSHasAVXSupport()) {
+ supported_ |= 1u << AVX;
+ }
+ if (cpu.has_fma3() && FLAG_enable_fma3 && cpu.has_osxsave() &&
+ OSHasAVXSupport()) {
+ supported_ |= 1u << FMA3;
+ }
+ if (cpu.has_bmi1() && FLAG_enable_bmi1) supported_ |= 1u << BMI1;
+ if (cpu.has_bmi2() && FLAG_enable_bmi2) supported_ |= 1u << BMI2;
+ if (cpu.has_lzcnt() && FLAG_enable_lzcnt) supported_ |= 1u << LZCNT;
+ if (cpu.has_popcnt() && FLAG_enable_popcnt) supported_ |= 1u << POPCNT;
+ if (strcmp(FLAG_mcpu, "auto") == 0) {
+ if (cpu.is_atom()) supported_ |= 1u << ATOM;
+ } else if (strcmp(FLAG_mcpu, "atom") == 0) {
+ supported_ |= 1u << ATOM;
+ }
+}
+
+void CpuFeatures::PrintTarget() {}
+void CpuFeatures::PrintFeatures() {
+ printf(
+ "SSE3=%d SSSE3=%d SSE4_1=%d AVX=%d FMA3=%d BMI1=%d BMI2=%d LZCNT=%d "
+ "POPCNT=%d ATOM=%d\n",
+ CpuFeatures::IsSupported(SSE3), CpuFeatures::IsSupported(SSSE3),
+ CpuFeatures::IsSupported(SSE4_1), CpuFeatures::IsSupported(AVX),
+ CpuFeatures::IsSupported(FMA3), CpuFeatures::IsSupported(BMI1),
+ CpuFeatures::IsSupported(BMI2), CpuFeatures::IsSupported(LZCNT),
+ CpuFeatures::IsSupported(POPCNT), CpuFeatures::IsSupported(ATOM));
+}
+
+// -----------------------------------------------------------------------------
+// Implementation of Displacement
+
+void Displacement::init(Label* L, Type type) {
+ DCHECK(!L->is_bound());
+ int next = 0;
+ if (L->is_linked()) {
+ next = L->pos();
+ DCHECK_GT(next, 0); // Displacements must be at positions > 0
+ }
+ // Ensure that we _never_ overflow the next field.
+ DCHECK(NextField::is_valid(Assembler::kMaximalBufferSize));
+ data_ = NextField::encode(next) | TypeField::encode(type);
+}
+
+// -----------------------------------------------------------------------------
+// Implementation of RelocInfo
+
+const int RelocInfo::kApplyMask =
+ RelocInfo::ModeMask(RelocInfo::CODE_TARGET) |
+ RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE) |
+ RelocInfo::ModeMask(RelocInfo::OFF_HEAP_TARGET) |
+ RelocInfo::ModeMask(RelocInfo::RUNTIME_ENTRY);
+
+bool RelocInfo::IsCodedSpecially() {
+ // The deserializer needs to know whether a pointer is specially coded. Being
+ // specially coded on IA32 means that it is a relative address, as used by
+ // branch instructions. These are also the ones that need changing when a
+ // code object moves.
+ return RelocInfo::ModeMask(rmode_) & kApplyMask;
+}
+
+bool RelocInfo::IsInConstantPool() { return false; }
+
+uint32_t RelocInfo::wasm_call_tag() const {
+ DCHECK(rmode_ == WASM_CALL || rmode_ == WASM_STUB_CALL);
+ return ReadUnalignedValue<uint32_t>(pc_);
+}
+
+// -----------------------------------------------------------------------------
+// Implementation of Operand
+
+Operand::Operand(Register base, int32_t disp, RelocInfo::Mode rmode) {
+ // [base + disp/r]
+ if (disp == 0 && RelocInfo::IsNone(rmode) && base != ebp) {
+ // [base]
+ set_modrm(0, base);
+ if (base == esp) set_sib(times_1, esp, base);
+ } else if (is_int8(disp) && RelocInfo::IsNone(rmode)) {
+ // [base + disp8]
+ set_modrm(1, base);
+ if (base == esp) set_sib(times_1, esp, base);
+ set_disp8(disp);
+ } else {
+ // [base + disp/r]
+ set_modrm(2, base);
+ if (base == esp) set_sib(times_1, esp, base);
+ set_dispr(disp, rmode);
+ }
+}
+
+Operand::Operand(Register base, Register index, ScaleFactor scale, int32_t disp,
+ RelocInfo::Mode rmode) {
+ DCHECK(index != esp); // illegal addressing mode
+ // [base + index*scale + disp/r]
+ if (disp == 0 && RelocInfo::IsNone(rmode) && base != ebp) {
+ // [base + index*scale]
+ set_modrm(0, esp);
+ set_sib(scale, index, base);
+ } else if (is_int8(disp) && RelocInfo::IsNone(rmode)) {
+ // [base + index*scale + disp8]
+ set_modrm(1, esp);
+ set_sib(scale, index, base);
+ set_disp8(disp);
+ } else {
+ // [base + index*scale + disp/r]
+ set_modrm(2, esp);
+ set_sib(scale, index, base);
+ set_dispr(disp, rmode);
+ }
+}
+
+Operand::Operand(Register index, ScaleFactor scale, int32_t disp,
+ RelocInfo::Mode rmode) {
+ DCHECK(index != esp); // illegal addressing mode
+ // [index*scale + disp/r]
+ set_modrm(0, esp);
+ set_sib(scale, index, ebp);
+ set_dispr(disp, rmode);
+}
+
+bool Operand::is_reg_only() const {
+ return (buf_[0] & 0xF8) == 0xC0; // Addressing mode is register only.
+}
+
+Register Operand::reg() const {
+ DCHECK(is_reg_only());
+ return Register::from_code(buf_[0] & 0x07);
+}
+
+void Assembler::AllocateAndInstallRequestedHeapObjects(Isolate* isolate) {
+ DCHECK_IMPLIES(isolate == nullptr, heap_object_requests_.empty());
+ for (auto& request : heap_object_requests_) {
+ Handle<HeapObject> object;
+ switch (request.kind()) {
+ case HeapObjectRequest::kHeapNumber:
+ object = isolate->factory()->NewHeapNumber<AllocationType::kOld>(
+ request.heap_number());
+ break;
+ case HeapObjectRequest::kStringConstant: {
+ const StringConstantBase* str = request.string();
+ CHECK_NOT_NULL(str);
+ object = str->AllocateStringConstant(isolate);
+ break;
+ }
+ }
+ Address pc = reinterpret_cast<Address>(buffer_start_) + request.offset();
+ WriteUnalignedValue(pc, object);
+ }
+}
+
+// -----------------------------------------------------------------------------
+// Implementation of Assembler.
+
+// Emit a single byte. Must always be inlined.
+#define EMIT(x) *pc_++ = (x)
+
+Assembler::Assembler(const AssemblerOptions& options,
+ std::unique_ptr<AssemblerBuffer> buffer)
+ : AssemblerBase(options, std::move(buffer)) {
+ reloc_info_writer.Reposition(buffer_start_ + buffer_->size(), pc_);
+}
+
+void Assembler::GetCode(Isolate* isolate, CodeDesc* desc,
+ SafepointTableBuilder* safepoint_table_builder,
+ int handler_table_offset) {
+ // As a crutch to avoid having to add manual Align calls wherever we use a
+ // raw workflow to create Code objects (mostly in tests), add another Align
+ // call here. It does no harm - the end of the Code object is aligned to the
+ // (larger) kCodeAlignment anyways.
+ // TODO(jgruber): Consider moving responsibility for proper alignment to
+ // metadata table builders (safepoint, handler, constant pool, code
+ // comments).
+ DataAlign(Code::kMetadataAlignment);
+
+ const int code_comments_size = WriteCodeComments();
+
+ // Finalize code (at this point overflow() may be true, but the gap ensures
+ // that we are still not overlapping instructions and relocation info).
+ DCHECK(pc_ <= reloc_info_writer.pos()); // No overlap.
+
+ AllocateAndInstallRequestedHeapObjects(isolate);
+
+ // Set up code descriptor.
+ // TODO(jgruber): Reconsider how these offsets and sizes are maintained up to
+ // this point to make CodeDesc initialization less fiddly.
+
+ static constexpr int kConstantPoolSize = 0;
+ const int instruction_size = pc_offset();
+ const int code_comments_offset = instruction_size - code_comments_size;
+ const int constant_pool_offset = code_comments_offset - kConstantPoolSize;
+ const int handler_table_offset2 = (handler_table_offset == kNoHandlerTable)
+ ? constant_pool_offset
+ : handler_table_offset;
+ const int safepoint_table_offset =
+ (safepoint_table_builder == kNoSafepointTable)
+ ? handler_table_offset2
+ : safepoint_table_builder->GetCodeOffset();
+ const int reloc_info_offset =
+ static_cast<int>(reloc_info_writer.pos() - buffer_->start());
+ CodeDesc::Initialize(desc, this, safepoint_table_offset,
+ handler_table_offset2, constant_pool_offset,
+ code_comments_offset, reloc_info_offset);
+}
+
+void Assembler::FinalizeJumpOptimizationInfo() {
+ // Collection stage
+ auto jump_opt = jump_optimization_info();
+ if (jump_opt && jump_opt->is_collecting()) {
+ auto& bitmap = jump_opt->farjmp_bitmap();
+ int num = static_cast<int>(farjmp_positions_.size());
+ if (num && bitmap.empty()) {
+ bool can_opt = false;
+
+ bitmap.resize((num + 31) / 32, 0);
+ for (int i = 0; i < num; i++) {
+ int disp_pos = farjmp_positions_[i];
+ int disp = long_at(disp_pos);
+ if (is_int8(disp)) {
+ bitmap[i / 32] |= 1 << (i & 31);
+ can_opt = true;
+ }
+ }
+ if (can_opt) {
+ jump_opt->set_optimizable();
+ }
+ }
+ }
+}
+
+void Assembler::Align(int m) {
+ DCHECK(base::bits::IsPowerOfTwo(m));
+ int mask = m - 1;
+ int addr = pc_offset();
+ Nop((m - (addr & mask)) & mask);
+}
+
+bool Assembler::IsNop(Address addr) {
+ byte* a = reinterpret_cast<byte*>(addr);
+ while (*a == 0x66) a++;
+ if (*a == 0x90) return true;
+ if (a[0] == 0xF && a[1] == 0x1F) return true;
+ return false;
+}
+
+void Assembler::Nop(int bytes) {
+ EnsureSpace ensure_space(this);
+ // Multi byte nops from http://support.amd.com/us/Processor_TechDocs/40546.pdf
+ while (bytes > 0) {
+ switch (bytes) {
+ case 2:
+ EMIT(0x66);
+ V8_FALLTHROUGH;
+ case 1:
+ EMIT(0x90);
+ return;
+ case 3:
+ EMIT(0xF);
+ EMIT(0x1F);
+ EMIT(0);
+ return;
+ case 4:
+ EMIT(0xF);
+ EMIT(0x1F);
+ EMIT(0x40);
+ EMIT(0);
+ return;
+ case 6:
+ EMIT(0x66);
+ V8_FALLTHROUGH;
+ case 5:
+ EMIT(0xF);
+ EMIT(0x1F);
+ EMIT(0x44);
+ EMIT(0);
+ EMIT(0);
+ return;
+ case 7:
+ EMIT(0xF);
+ EMIT(0x1F);
+ EMIT(0x80);
+ EMIT(0);
+ EMIT(0);
+ EMIT(0);
+ EMIT(0);
+ return;
+ default:
+ case 11:
+ EMIT(0x66);
+ bytes--;
+ V8_FALLTHROUGH;
+ case 10:
+ EMIT(0x66);
+ bytes--;
+ V8_FALLTHROUGH;
+ case 9:
+ EMIT(0x66);
+ bytes--;
+ V8_FALLTHROUGH;
+ case 8:
+ EMIT(0xF);
+ EMIT(0x1F);
+ EMIT(0x84);
+ EMIT(0);
+ EMIT(0);
+ EMIT(0);
+ EMIT(0);
+ EMIT(0);
+ bytes -= 8;
+ }
+ }
+}
+
+void Assembler::CodeTargetAlign() {
+ Align(16); // Preferred alignment of jump targets on ia32.
+}
+
+void Assembler::cpuid() {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0xA2);
+}
+
+void Assembler::pushad() {
+ EnsureSpace ensure_space(this);
+ EMIT(0x60);
+}
+
+void Assembler::popad() {
+ EnsureSpace ensure_space(this);
+ EMIT(0x61);
+}
+
+void Assembler::pushfd() {
+ EnsureSpace ensure_space(this);
+ EMIT(0x9C);
+}
+
+void Assembler::popfd() {
+ EnsureSpace ensure_space(this);
+ EMIT(0x9D);
+}
+
+void Assembler::push(const Immediate& x) {
+ EnsureSpace ensure_space(this);
+ if (x.is_int8()) {
+ EMIT(0x6A);
+ EMIT(x.immediate());
+ } else {
+ EMIT(0x68);
+ emit(x);
+ }
+}
+
+void Assembler::push_imm32(int32_t imm32) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x68);
+ emit(imm32);
+}
+
+void Assembler::push(Register src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x50 | src.code());
+}
+
+void Assembler::push(Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xFF);
+ emit_operand(esi, src);
+}
+
+void Assembler::pop(Register dst) {
+ DCHECK_NOT_NULL(reloc_info_writer.last_pc());
+ EnsureSpace ensure_space(this);
+ EMIT(0x58 | dst.code());
+}
+
+void Assembler::pop(Operand dst) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x8F);
+ emit_operand(eax, dst);
+}
+
+void Assembler::leave() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xC9);
+}
+
+void Assembler::mov_b(Register dst, Operand src) {
+ CHECK(dst.is_byte_register());
+ EnsureSpace ensure_space(this);
+ EMIT(0x8A);
+ emit_operand(dst, src);
+}
+
+void Assembler::mov_b(Operand dst, const Immediate& src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xC6);
+ emit_operand(eax, dst);
+ EMIT(static_cast<int8_t>(src.immediate()));
+}
+
+void Assembler::mov_b(Operand dst, Register src) {
+ CHECK(src.is_byte_register());
+ EnsureSpace ensure_space(this);
+ EMIT(0x88);
+ emit_operand(src, dst);
+}
+
+void Assembler::mov_w(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x8B);
+ emit_operand(dst, src);
+}
+
+void Assembler::mov_w(Operand dst, Register src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x89);
+ emit_operand(src, dst);
+}
+
+void Assembler::mov_w(Operand dst, const Immediate& src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0xC7);
+ emit_operand(eax, dst);
+ EMIT(static_cast<int8_t>(src.immediate() & 0xFF));
+ EMIT(static_cast<int8_t>(src.immediate() >> 8));
+}
+
+void Assembler::mov(Register dst, int32_t imm32) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xB8 | dst.code());
+ emit(imm32);
+}
+
+void Assembler::mov(Register dst, const Immediate& x) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xB8 | dst.code());
+ emit(x);
+}
+
+void Assembler::mov(Register dst, Handle<HeapObject> handle) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xB8 | dst.code());
+ emit(handle);
+}
+
+void Assembler::mov(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x8B);
+ emit_operand(dst, src);
+}
+
+void Assembler::mov(Register dst, Register src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x89);
+ EMIT(0xC0 | src.code() << 3 | dst.code());
+}
+
+void Assembler::mov(Operand dst, const Immediate& x) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xC7);
+ emit_operand(eax, dst);
+ emit(x);
+}
+
+void Assembler::mov(Operand dst, Address src, RelocInfo::Mode rmode) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xC7);
+ emit_operand(eax, dst);
+ emit(src, rmode);
+}
+
+void Assembler::mov(Operand dst, Handle<HeapObject> handle) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xC7);
+ emit_operand(eax, dst);
+ emit(handle);
+}
+
+void Assembler::mov(Operand dst, Register src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x89);
+ emit_operand(src, dst);
+}
+
+void Assembler::movsx_b(Register dst, Operand src) {
+ DCHECK_IMPLIES(src.is_reg_only(), src.reg().is_byte_register());
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0xBE);
+ emit_operand(dst, src);
+}
+
+void Assembler::movsx_w(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0xBF);
+ emit_operand(dst, src);
+}
+
+void Assembler::movzx_b(Register dst, Operand src) {
+ DCHECK_IMPLIES(src.is_reg_only(), src.reg().is_byte_register());
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0xB6);
+ emit_operand(dst, src);
+}
+
+void Assembler::movzx_w(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0xB7);
+ emit_operand(dst, src);
+}
+
+void Assembler::movq(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF3);
+ EMIT(0x0F);
+ EMIT(0x7E);
+ emit_operand(dst, src);
+}
+
+void Assembler::cmov(Condition cc, Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ // Opcode: 0f 40 + cc /r.
+ EMIT(0x0F);
+ EMIT(0x40 + cc);
+ emit_operand(dst, src);
+}
+
+void Assembler::cld() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xFC);
+}
+
+void Assembler::rep_movs() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF3);
+ EMIT(0xA5);
+}
+
+void Assembler::rep_stos() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF3);
+ EMIT(0xAB);
+}
+
+void Assembler::stos() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xAB);
+}
+
+void Assembler::xadd(Operand dst, Register src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0xC1);
+ emit_operand(src, dst);
+}
+
+void Assembler::xadd_b(Operand dst, Register src) {
+ DCHECK(src.is_byte_register());
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0xC0);
+ emit_operand(src, dst);
+}
+
+void Assembler::xadd_w(Operand dst, Register src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0xC1);
+ emit_operand(src, dst);
+}
+
+void Assembler::xchg(Register dst, Register src) {
+ EnsureSpace ensure_space(this);
+ if (src == eax || dst == eax) { // Single-byte encoding.
+ EMIT(0x90 | (src == eax ? dst.code() : src.code()));
+ } else {
+ EMIT(0x87);
+ EMIT(0xC0 | src.code() << 3 | dst.code());
+ }
+}
+
+void Assembler::xchg(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x87);
+ emit_operand(dst, src);
+}
+
+void Assembler::xchg_b(Register reg, Operand op) {
+ DCHECK(reg.is_byte_register());
+ EnsureSpace ensure_space(this);
+ EMIT(0x86);
+ emit_operand(reg, op);
+}
+
+void Assembler::xchg_w(Register reg, Operand op) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x87);
+ emit_operand(reg, op);
+}
+
+void Assembler::lock() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF0);
+}
+
+void Assembler::cmpxchg(Operand dst, Register src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0xB1);
+ emit_operand(src, dst);
+}
+
+void Assembler::cmpxchg_b(Operand dst, Register src) {
+ DCHECK(src.is_byte_register());
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0xB0);
+ emit_operand(src, dst);
+}
+
+void Assembler::cmpxchg_w(Operand dst, Register src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0xB1);
+ emit_operand(src, dst);
+}
+
+void Assembler::cmpxchg8b(Operand dst) {
+ EnsureSpace enure_space(this);
+ EMIT(0x0F);
+ EMIT(0xC7);
+ emit_operand(ecx, dst);
+}
+
+void Assembler::mfence() {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0xAE);
+ EMIT(0xF0);
+}
+
+void Assembler::lfence() {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0xAE);
+ EMIT(0xE8);
+}
+
+void Assembler::pause() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF3);
+ EMIT(0x90);
+}
+
+void Assembler::adc(Register dst, int32_t imm32) {
+ EnsureSpace ensure_space(this);
+ emit_arith(2, Operand(dst), Immediate(imm32));
+}
+
+void Assembler::adc(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x13);
+ emit_operand(dst, src);
+}
+
+void Assembler::add(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x03);
+ emit_operand(dst, src);
+}
+
+void Assembler::add(Operand dst, Register src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x01);
+ emit_operand(src, dst);
+}
+
+void Assembler::add(Operand dst, const Immediate& x) {
+ DCHECK_NOT_NULL(reloc_info_writer.last_pc());
+ EnsureSpace ensure_space(this);
+ emit_arith(0, dst, x);
+}
+
+void Assembler::and_(Register dst, int32_t imm32) {
+ and_(dst, Immediate(imm32));
+}
+
+void Assembler::and_(Register dst, const Immediate& x) {
+ EnsureSpace ensure_space(this);
+ emit_arith(4, Operand(dst), x);
+}
+
+void Assembler::and_(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x23);
+ emit_operand(dst, src);
+}
+
+void Assembler::and_(Operand dst, const Immediate& x) {
+ EnsureSpace ensure_space(this);
+ emit_arith(4, dst, x);
+}
+
+void Assembler::and_(Operand dst, Register src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x21);
+ emit_operand(src, dst);
+}
+
+void Assembler::cmpb(Operand op, Immediate imm8) {
+ DCHECK(imm8.is_int8() || imm8.is_uint8());
+ EnsureSpace ensure_space(this);
+ if (op.is_reg(eax)) {
+ EMIT(0x3C);
+ } else {
+ EMIT(0x80);
+ emit_operand(edi, op); // edi == 7
+ }
+ emit_b(imm8);
+}
+
+void Assembler::cmpb(Operand op, Register reg) {
+ CHECK(reg.is_byte_register());
+ EnsureSpace ensure_space(this);
+ EMIT(0x38);
+ emit_operand(reg, op);
+}
+
+void Assembler::cmpb(Register reg, Operand op) {
+ CHECK(reg.is_byte_register());
+ EnsureSpace ensure_space(this);
+ EMIT(0x3A);
+ emit_operand(reg, op);
+}
+
+void Assembler::cmpw(Operand op, Immediate imm16) {
+ DCHECK(imm16.is_int16() || imm16.is_uint16());
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x81);
+ emit_operand(edi, op);
+ emit_w(imm16);
+}
+
+void Assembler::cmpw(Register reg, Operand op) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x3B);
+ emit_operand(reg, op);
+}
+
+void Assembler::cmpw(Operand op, Register reg) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x39);
+ emit_operand(reg, op);
+}
+
+void Assembler::cmp(Register reg, int32_t imm32) {
+ EnsureSpace ensure_space(this);
+ emit_arith(7, Operand(reg), Immediate(imm32));
+}
+
+void Assembler::cmp(Register reg, Handle<HeapObject> handle) {
+ EnsureSpace ensure_space(this);
+ emit_arith(7, Operand(reg), Immediate(handle));
+}
+
+void Assembler::cmp(Register reg, Operand op) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x3B);
+ emit_operand(reg, op);
+}
+
+void Assembler::cmp(Operand op, Register reg) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x39);
+ emit_operand(reg, op);
+}
+
+void Assembler::cmp(Operand op, const Immediate& imm) {
+ EnsureSpace ensure_space(this);
+ emit_arith(7, op, imm);
+}
+
+void Assembler::cmp(Operand op, Handle<HeapObject> handle) {
+ EnsureSpace ensure_space(this);
+ emit_arith(7, op, Immediate(handle));
+}
+
+void Assembler::cmpb_al(Operand op) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x38); // CMP r/m8, r8
+ emit_operand(eax, op); // eax has same code as register al.
+}
+
+void Assembler::cmpw_ax(Operand op) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x39); // CMP r/m16, r16
+ emit_operand(eax, op); // eax has same code as register ax.
+}
+
+void Assembler::dec_b(Register dst) {
+ CHECK(dst.is_byte_register());
+ EnsureSpace ensure_space(this);
+ EMIT(0xFE);
+ EMIT(0xC8 | dst.code());
+}
+
+void Assembler::dec_b(Operand dst) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xFE);
+ emit_operand(ecx, dst);
+}
+
+void Assembler::dec(Register dst) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x48 | dst.code());
+}
+
+void Assembler::dec(Operand dst) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xFF);
+ emit_operand(ecx, dst);
+}
+
+void Assembler::cdq() {
+ EnsureSpace ensure_space(this);
+ EMIT(0x99);
+}
+
+void Assembler::idiv(Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF7);
+ emit_operand(edi, src);
+}
+
+void Assembler::div(Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF7);
+ emit_operand(esi, src);
+}
+
+void Assembler::imul(Register reg) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF7);
+ EMIT(0xE8 | reg.code());
+}
+
+void Assembler::imul(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0xAF);
+ emit_operand(dst, src);
+}
+
+void Assembler::imul(Register dst, Register src, int32_t imm32) {
+ imul(dst, Operand(src), imm32);
+}
+
+void Assembler::imul(Register dst, Operand src, int32_t imm32) {
+ EnsureSpace ensure_space(this);
+ if (is_int8(imm32)) {
+ EMIT(0x6B);
+ emit_operand(dst, src);
+ EMIT(imm32);
+ } else {
+ EMIT(0x69);
+ emit_operand(dst, src);
+ emit(imm32);
+ }
+}
+
+void Assembler::inc(Register dst) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x40 | dst.code());
+}
+
+void Assembler::inc(Operand dst) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xFF);
+ emit_operand(eax, dst);
+}
+
+void Assembler::lea(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x8D);
+ emit_operand(dst, src);
+}
+
+void Assembler::mul(Register src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF7);
+ EMIT(0xE0 | src.code());
+}
+
+void Assembler::neg(Register dst) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF7);
+ EMIT(0xD8 | dst.code());
+}
+
+void Assembler::neg(Operand dst) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF7);
+ emit_operand(ebx, dst);
+}
+
+void Assembler::not_(Register dst) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF7);
+ EMIT(0xD0 | dst.code());
+}
+
+void Assembler::not_(Operand dst) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF7);
+ emit_operand(edx, dst);
+}
+
+void Assembler::or_(Register dst, int32_t imm32) {
+ EnsureSpace ensure_space(this);
+ emit_arith(1, Operand(dst), Immediate(imm32));
+}
+
+void Assembler::or_(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0B);
+ emit_operand(dst, src);
+}
+
+void Assembler::or_(Operand dst, const Immediate& x) {
+ EnsureSpace ensure_space(this);
+ emit_arith(1, dst, x);
+}
+
+void Assembler::or_(Operand dst, Register src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x09);
+ emit_operand(src, dst);
+}
+
+void Assembler::rcl(Register dst, uint8_t imm8) {
+ EnsureSpace ensure_space(this);
+ DCHECK(is_uint5(imm8)); // illegal shift count
+ if (imm8 == 1) {
+ EMIT(0xD1);
+ EMIT(0xD0 | dst.code());
+ } else {
+ EMIT(0xC1);
+ EMIT(0xD0 | dst.code());
+ EMIT(imm8);
+ }
+}
+
+void Assembler::rcr(Register dst, uint8_t imm8) {
+ EnsureSpace ensure_space(this);
+ DCHECK(is_uint5(imm8)); // illegal shift count
+ if (imm8 == 1) {
+ EMIT(0xD1);
+ EMIT(0xD8 | dst.code());
+ } else {
+ EMIT(0xC1);
+ EMIT(0xD8 | dst.code());
+ EMIT(imm8);
+ }
+}
+
+void Assembler::rol(Operand dst, uint8_t imm8) {
+ EnsureSpace ensure_space(this);
+ DCHECK(is_uint5(imm8)); // illegal shift count
+ if (imm8 == 1) {
+ EMIT(0xD1);
+ emit_operand(eax, dst);
+ } else {
+ EMIT(0xC1);
+ emit_operand(eax, dst);
+ EMIT(imm8);
+ }
+}
+
+void Assembler::rol_cl(Operand dst) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xD3);
+ emit_operand(eax, dst);
+}
+
+void Assembler::ror(Operand dst, uint8_t imm8) {
+ EnsureSpace ensure_space(this);
+ DCHECK(is_uint5(imm8)); // illegal shift count
+ if (imm8 == 1) {
+ EMIT(0xD1);
+ emit_operand(ecx, dst);
+ } else {
+ EMIT(0xC1);
+ emit_operand(ecx, dst);
+ EMIT(imm8);
+ }
+}
+
+void Assembler::ror_cl(Operand dst) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xD3);
+ emit_operand(ecx, dst);
+}
+
+void Assembler::sar(Operand dst, uint8_t imm8) {
+ EnsureSpace ensure_space(this);
+ DCHECK(is_uint5(imm8)); // illegal shift count
+ if (imm8 == 1) {
+ EMIT(0xD1);
+ emit_operand(edi, dst);
+ } else {
+ EMIT(0xC1);
+ emit_operand(edi, dst);
+ EMIT(imm8);
+ }
+}
+
+void Assembler::sar_cl(Operand dst) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xD3);
+ emit_operand(edi, dst);
+}
+
+void Assembler::sbb(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x1B);
+ emit_operand(dst, src);
+}
+
+void Assembler::shld(Register dst, Register src, uint8_t shift) {
+ DCHECK(is_uint5(shift));
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0xA4);
+ emit_operand(src, Operand(dst));
+ EMIT(shift);
+}
+
+void Assembler::shld_cl(Register dst, Register src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0xA5);
+ emit_operand(src, Operand(dst));
+}
+
+void Assembler::shl(Operand dst, uint8_t imm8) {
+ EnsureSpace ensure_space(this);
+ DCHECK(is_uint5(imm8)); // illegal shift count
+ if (imm8 == 1) {
+ EMIT(0xD1);
+ emit_operand(esp, dst);
+ } else {
+ EMIT(0xC1);
+ emit_operand(esp, dst);
+ EMIT(imm8);
+ }
+}
+
+void Assembler::shl_cl(Operand dst) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xD3);
+ emit_operand(esp, dst);
+}
+
+void Assembler::shr(Operand dst, uint8_t imm8) {
+ EnsureSpace ensure_space(this);
+ DCHECK(is_uint5(imm8)); // illegal shift count
+ if (imm8 == 1) {
+ EMIT(0xD1);
+ emit_operand(ebp, dst);
+ } else {
+ EMIT(0xC1);
+ emit_operand(ebp, dst);
+ EMIT(imm8);
+ }
+}
+
+void Assembler::shr_cl(Operand dst) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xD3);
+ emit_operand(ebp, dst);
+}
+
+void Assembler::shrd(Register dst, Register src, uint8_t shift) {
+ DCHECK(is_uint5(shift));
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0xAC);
+ emit_operand(src, Operand(dst));
+ EMIT(shift);
+}
+
+void Assembler::shrd_cl(Operand dst, Register src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0xAD);
+ emit_operand(src, dst);
+}
+
+void Assembler::sub(Operand dst, const Immediate& x) {
+ EnsureSpace ensure_space(this);
+ emit_arith(5, dst, x);
+}
+
+void Assembler::sub(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x2B);
+ emit_operand(dst, src);
+}
+
+void Assembler::sub(Operand dst, Register src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x29);
+ emit_operand(src, dst);
+}
+
+void Assembler::sub_sp_32(uint32_t imm) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x81); // using a literal 32-bit immediate.
+ static constexpr Register ireg = Register::from_code(5);
+ emit_operand(ireg, Operand(esp));
+ emit(imm);
+}
+
+void Assembler::test(Register reg, const Immediate& imm) {
+ if (imm.is_uint8()) {
+ test_b(reg, imm);
+ return;
+ }
+
+ EnsureSpace ensure_space(this);
+ // This is not using emit_arith because test doesn't support
+ // sign-extension of 8-bit operands.
+ if (reg == eax) {
+ EMIT(0xA9);
+ } else {
+ EMIT(0xF7);
+ EMIT(0xC0 | reg.code());
+ }
+ emit(imm);
+}
+
+void Assembler::test(Register reg, Operand op) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x85);
+ emit_operand(reg, op);
+}
+
+void Assembler::test_b(Register reg, Operand op) {
+ CHECK(reg.is_byte_register());
+ EnsureSpace ensure_space(this);
+ EMIT(0x84);
+ emit_operand(reg, op);
+}
+
+void Assembler::test(Operand op, const Immediate& imm) {
+ if (op.is_reg_only()) {
+ test(op.reg(), imm);
+ return;
+ }
+ if (imm.is_uint8()) {
+ return test_b(op, imm);
+ }
+ EnsureSpace ensure_space(this);
+ EMIT(0xF7);
+ emit_operand(eax, op);
+ emit(imm);
+}
+
+void Assembler::test_b(Register reg, Immediate imm8) {
+ DCHECK(imm8.is_uint8());
+ EnsureSpace ensure_space(this);
+ // Only use test against byte for registers that have a byte
+ // variant: eax, ebx, ecx, and edx.
+ if (reg == eax) {
+ EMIT(0xA8);
+ emit_b(imm8);
+ } else if (reg.is_byte_register()) {
+ emit_arith_b(0xF6, 0xC0, reg, static_cast<uint8_t>(imm8.immediate()));
+ } else {
+ EMIT(0x66);
+ EMIT(0xF7);
+ EMIT(0xC0 | reg.code());
+ emit_w(imm8);
+ }
+}
+
+void Assembler::test_b(Operand op, Immediate imm8) {
+ if (op.is_reg_only()) {
+ test_b(op.reg(), imm8);
+ return;
+ }
+ EnsureSpace ensure_space(this);
+ EMIT(0xF6);
+ emit_operand(eax, op);
+ emit_b(imm8);
+}
+
+void Assembler::test_w(Register reg, Immediate imm16) {
+ DCHECK(imm16.is_int16() || imm16.is_uint16());
+ EnsureSpace ensure_space(this);
+ if (reg == eax) {
+ EMIT(0xA9);
+ emit_w(imm16);
+ } else {
+ EMIT(0x66);
+ EMIT(0xF7);
+ EMIT(0xC0 | reg.code());
+ emit_w(imm16);
+ }
+}
+
+void Assembler::test_w(Register reg, Operand op) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x85);
+ emit_operand(reg, op);
+}
+
+void Assembler::test_w(Operand op, Immediate imm16) {
+ DCHECK(imm16.is_int16() || imm16.is_uint16());
+ if (op.is_reg_only()) {
+ test_w(op.reg(), imm16);
+ return;
+ }
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0xF7);
+ emit_operand(eax, op);
+ emit_w(imm16);
+}
+
+void Assembler::xor_(Register dst, int32_t imm32) {
+ EnsureSpace ensure_space(this);
+ emit_arith(6, Operand(dst), Immediate(imm32));
+}
+
+void Assembler::xor_(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x33);
+ emit_operand(dst, src);
+}
+
+void Assembler::xor_(Operand dst, Register src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x31);
+ emit_operand(src, dst);
+}
+
+void Assembler::xor_(Operand dst, const Immediate& x) {
+ EnsureSpace ensure_space(this);
+ emit_arith(6, dst, x);
+}
+
+void Assembler::bswap(Register dst) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0xC8 + dst.code());
+}
+
+void Assembler::bt(Operand dst, Register src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0xA3);
+ emit_operand(src, dst);
+}
+
+void Assembler::bts(Operand dst, Register src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0xAB);
+ emit_operand(src, dst);
+}
+
+void Assembler::bsr(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0xBD);
+ emit_operand(dst, src);
+}
+
+void Assembler::bsf(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0xBC);
+ emit_operand(dst, src);
+}
+
+void Assembler::hlt() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF4);
+}
+
+void Assembler::int3() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xCC);
+}
+
+void Assembler::nop() {
+ EnsureSpace ensure_space(this);
+ EMIT(0x90);
+}
+
+void Assembler::ret(int imm16) {
+ EnsureSpace ensure_space(this);
+ DCHECK(is_uint16(imm16));
+ if (imm16 == 0) {
+ EMIT(0xC3);
+ } else {
+ EMIT(0xC2);
+ EMIT(imm16 & 0xFF);
+ EMIT((imm16 >> 8) & 0xFF);
+ }
+}
+
+void Assembler::ud2() {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0x0B);
+}
+
+// Labels refer to positions in the (to be) generated code.
+// There are bound, linked, and unused labels.
+//
+// Bound labels refer to known positions in the already
+// generated code. pos() is the position the label refers to.
+//
+// Linked labels refer to unknown positions in the code
+// to be generated; pos() is the position of the 32bit
+// Displacement of the last instruction using the label.
+
+void Assembler::print(const Label* L) {
+ if (L->is_unused()) {
+ PrintF("unused label\n");
+ } else if (L->is_bound()) {
+ PrintF("bound label to %d\n", L->pos());
+ } else if (L->is_linked()) {
+ Label l;
+ l.link_to(L->pos());
+ PrintF("unbound label");
+ while (l.is_linked()) {
+ Displacement disp = disp_at(&l);
+ PrintF("@ %d ", l.pos());
+ disp.print();
+ PrintF("\n");
+ disp.next(&l);
+ }
+ } else {
+ PrintF("label in inconsistent state (pos = %d)\n", L->pos_);
+ }
+}
+
+void Assembler::bind_to(Label* L, int pos) {
+ EnsureSpace ensure_space(this);
+ DCHECK(0 <= pos && pos <= pc_offset()); // must have a valid binding position
+ while (L->is_linked()) {
+ Displacement disp = disp_at(L);
+ int fixup_pos = L->pos();
+ if (disp.type() == Displacement::CODE_ABSOLUTE) {
+ long_at_put(fixup_pos, reinterpret_cast<int>(buffer_start_ + pos));
+ internal_reference_positions_.push_back(fixup_pos);
+ } else if (disp.type() == Displacement::CODE_RELATIVE) {
+ // Relative to Code heap object pointer.
+ long_at_put(fixup_pos, pos + Code::kHeaderSize - kHeapObjectTag);
+ } else {
+ if (disp.type() == Displacement::UNCONDITIONAL_JUMP) {
+ DCHECK_EQ(byte_at(fixup_pos - 1), 0xE9); // jmp expected
+ }
+ // Relative address, relative to point after address.
+ int imm32 = pos - (fixup_pos + sizeof(int32_t));
+ long_at_put(fixup_pos, imm32);
+ }
+ disp.next(L);
+ }
+ while (L->is_near_linked()) {
+ int fixup_pos = L->near_link_pos();
+ int offset_to_next =
+ static_cast<int>(*reinterpret_cast<int8_t*>(addr_at(fixup_pos)));
+ DCHECK_LE(offset_to_next, 0);
+ // Relative address, relative to point after address.
+ int disp = pos - fixup_pos - sizeof(int8_t);
+ CHECK(0 <= disp && disp <= 127);
+ set_byte_at(fixup_pos, disp);
+ if (offset_to_next < 0) {
+ L->link_to(fixup_pos + offset_to_next, Label::kNear);
+ } else {
+ L->UnuseNear();
+ }
+ }
+
+ // Optimization stage
+ auto jump_opt = jump_optimization_info();
+ if (jump_opt && jump_opt->is_optimizing()) {
+ auto it = label_farjmp_maps_.find(L);
+ if (it != label_farjmp_maps_.end()) {
+ auto& pos_vector = it->second;
+ for (auto fixup_pos : pos_vector) {
+ int disp = pos - (fixup_pos + sizeof(int8_t));
+ CHECK(is_int8(disp));
+ set_byte_at(fixup_pos, disp);
+ }
+ label_farjmp_maps_.erase(it);
+ }
+ }
+ L->bind_to(pos);
+}
+
+void Assembler::bind(Label* L) {
+ EnsureSpace ensure_space(this);
+ DCHECK(!L->is_bound()); // label can only be bound once
+ bind_to(L, pc_offset());
+}
+
+void Assembler::record_farjmp_position(Label* L, int pos) {
+ auto& pos_vector = label_farjmp_maps_[L];
+ pos_vector.push_back(pos);
+}
+
+bool Assembler::is_optimizable_farjmp(int idx) {
+ if (predictable_code_size()) return false;
+
+ auto jump_opt = jump_optimization_info();
+ CHECK(jump_opt->is_optimizing());
+
+ auto& bitmap = jump_opt->farjmp_bitmap();
+ CHECK(idx < static_cast<int>(bitmap.size() * 32));
+ return !!(bitmap[idx / 32] & (1 << (idx & 31)));
+}
+
+void Assembler::call(Label* L) {
+ EnsureSpace ensure_space(this);
+ if (L->is_bound()) {
+ const int long_size = 5;
+ int offs = L->pos() - pc_offset();
+ DCHECK_LE(offs, 0);
+ // 1110 1000 #32-bit disp.
+ EMIT(0xE8);
+ emit(offs - long_size);
+ } else {
+ // 1110 1000 #32-bit disp.
+ EMIT(0xE8);
+ emit_disp(L, Displacement::OTHER);
+ }
+}
+
+void Assembler::call(Address entry, RelocInfo::Mode rmode) {
+ EnsureSpace ensure_space(this);
+ DCHECK(!RelocInfo::IsCodeTarget(rmode));
+ EMIT(0xE8);
+ if (RelocInfo::IsRuntimeEntry(rmode)) {
+ emit(entry, rmode);
+ } else {
+ emit(entry - (reinterpret_cast<Address>(pc_) + sizeof(int32_t)), rmode);
+ }
+}
+
+void Assembler::wasm_call(Address entry, RelocInfo::Mode rmode) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xE8);
+ emit(entry, rmode);
+}
+
+void Assembler::call(Operand adr) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xFF);
+ emit_operand(edx, adr);
+}
+
+void Assembler::call(Handle<Code> code, RelocInfo::Mode rmode) {
+ EnsureSpace ensure_space(this);
+ DCHECK(RelocInfo::IsCodeTarget(rmode));
+ DCHECK(code->IsExecutable());
+ EMIT(0xE8);
+ emit(code, rmode);
+}
+
+void Assembler::jmp_rel(int offset) {
+ EnsureSpace ensure_space(this);
+ const int short_size = 2;
+ const int long_size = 5;
+ if (is_int8(offset - short_size)) {
+ // 1110 1011 #8-bit disp.
+ EMIT(0xEB);
+ EMIT((offset - short_size) & 0xFF);
+ } else {
+ // 1110 1001 #32-bit disp.
+ EMIT(0xE9);
+ emit(offset - long_size);
+ }
+}
+
+void Assembler::jmp(Label* L, Label::Distance distance) {
+ if (L->is_bound()) {
+ int offset = L->pos() - pc_offset();
+ DCHECK_LE(offset, 0); // backward jump.
+ jmp_rel(offset);
+ return;
+ }
+
+ EnsureSpace ensure_space(this);
+ if (distance == Label::kNear) {
+ EMIT(0xEB);
+ emit_near_disp(L);
+ } else {
+ auto jump_opt = jump_optimization_info();
+ if (V8_UNLIKELY(jump_opt)) {
+ if (jump_opt->is_optimizing() && is_optimizable_farjmp(farjmp_num_++)) {
+ EMIT(0xEB);
+ record_farjmp_position(L, pc_offset());
+ EMIT(0);
+ return;
+ }
+ if (jump_opt->is_collecting()) {
+ farjmp_positions_.push_back(pc_offset() + 1);
+ }
+ }
+ // 1110 1001 #32-bit disp.
+ EMIT(0xE9);
+ emit_disp(L, Displacement::UNCONDITIONAL_JUMP);
+ }
+}
+
+void Assembler::jmp(Address entry, RelocInfo::Mode rmode) {
+ EnsureSpace ensure_space(this);
+ DCHECK(!RelocInfo::IsCodeTarget(rmode));
+ EMIT(0xE9);
+ if (RelocInfo::IsRuntimeEntry(rmode) || RelocInfo::IsWasmCall(rmode)) {
+ emit(entry, rmode);
+ } else {
+ emit(entry - (reinterpret_cast<Address>(pc_) + sizeof(int32_t)), rmode);
+ }
+}
+
+void Assembler::jmp(Operand adr) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xFF);
+ emit_operand(esp, adr);
+}
+
+void Assembler::jmp(Handle<Code> code, RelocInfo::Mode rmode) {
+ EnsureSpace ensure_space(this);
+ DCHECK(RelocInfo::IsCodeTarget(rmode));
+ EMIT(0xE9);
+ emit(code, rmode);
+}
+
+void Assembler::j(Condition cc, Label* L, Label::Distance distance) {
+ EnsureSpace ensure_space(this);
+ DCHECK(0 <= cc && static_cast<int>(cc) < 16);
+ if (L->is_bound()) {
+ const int short_size = 2;
+ const int long_size = 6;
+ int offs = L->pos() - pc_offset();
+ DCHECK_LE(offs, 0);
+ if (is_int8(offs - short_size)) {
+ // 0111 tttn #8-bit disp
+ EMIT(0x70 | cc);
+ EMIT((offs - short_size) & 0xFF);
+ } else {
+ // 0000 1111 1000 tttn #32-bit disp
+ EMIT(0x0F);
+ EMIT(0x80 | cc);
+ emit(offs - long_size);
+ }
+ } else if (distance == Label::kNear) {
+ EMIT(0x70 | cc);
+ emit_near_disp(L);
+ } else {
+ auto jump_opt = jump_optimization_info();
+ if (V8_UNLIKELY(jump_opt)) {
+ if (jump_opt->is_optimizing() && is_optimizable_farjmp(farjmp_num_++)) {
+ // 0111 tttn #8-bit disp
+ EMIT(0x70 | cc);
+ record_farjmp_position(L, pc_offset());
+ EMIT(0);
+ return;
+ }
+ if (jump_opt->is_collecting()) {
+ farjmp_positions_.push_back(pc_offset() + 2);
+ }
+ }
+ // 0000 1111 1000 tttn #32-bit disp
+ // Note: could eliminate cond. jumps to this jump if condition
+ // is the same however, seems to be rather unlikely case.
+ EMIT(0x0F);
+ EMIT(0x80 | cc);
+ emit_disp(L, Displacement::OTHER);
+ }
+}
+
+void Assembler::j(Condition cc, byte* entry, RelocInfo::Mode rmode) {
+ EnsureSpace ensure_space(this);
+ DCHECK((0 <= cc) && (static_cast<int>(cc) < 16));
+ // 0000 1111 1000 tttn #32-bit disp.
+ EMIT(0x0F);
+ EMIT(0x80 | cc);
+ if (RelocInfo::IsRuntimeEntry(rmode)) {
+ emit(reinterpret_cast<uint32_t>(entry), rmode);
+ } else {
+ emit(entry - (pc_ + sizeof(int32_t)), rmode);
+ }
+}
+
+void Assembler::j(Condition cc, Handle<Code> code, RelocInfo::Mode rmode) {
+ EnsureSpace ensure_space(this);
+ // 0000 1111 1000 tttn #32-bit disp
+ EMIT(0x0F);
+ EMIT(0x80 | cc);
+ emit(code, rmode);
+}
+
+// FPU instructions.
+
+void Assembler::fld(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xD9, 0xC0, i);
+}
+
+void Assembler::fstp(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xDD, 0xD8, i);
+}
+
+void Assembler::fld1() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xD9);
+ EMIT(0xE8);
+}
+
+void Assembler::fldpi() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xD9);
+ EMIT(0xEB);
+}
+
+void Assembler::fldz() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xD9);
+ EMIT(0xEE);
+}
+
+void Assembler::fldln2() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xD9);
+ EMIT(0xED);
+}
+
+void Assembler::fld_s(Operand adr) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xD9);
+ emit_operand(eax, adr);
+}
+
+void Assembler::fld_d(Operand adr) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xDD);
+ emit_operand(eax, adr);
+}
+
+void Assembler::fstp_s(Operand adr) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xD9);
+ emit_operand(ebx, adr);
+}
+
+void Assembler::fst_s(Operand adr) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xD9);
+ emit_operand(edx, adr);
+}
+
+void Assembler::fstp_d(Operand adr) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xDD);
+ emit_operand(ebx, adr);
+}
+
+void Assembler::fst_d(Operand adr) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xDD);
+ emit_operand(edx, adr);
+}
+
+void Assembler::fild_s(Operand adr) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xDB);
+ emit_operand(eax, adr);
+}
+
+void Assembler::fild_d(Operand adr) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xDF);
+ emit_operand(ebp, adr);
+}
+
+void Assembler::fistp_s(Operand adr) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xDB);
+ emit_operand(ebx, adr);
+}
+
+void Assembler::fisttp_s(Operand adr) {
+ DCHECK(IsEnabled(SSE3));
+ EnsureSpace ensure_space(this);
+ EMIT(0xDB);
+ emit_operand(ecx, adr);
+}
+
+void Assembler::fisttp_d(Operand adr) {
+ DCHECK(IsEnabled(SSE3));
+ EnsureSpace ensure_space(this);
+ EMIT(0xDD);
+ emit_operand(ecx, adr);
+}
+
+void Assembler::fist_s(Operand adr) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xDB);
+ emit_operand(edx, adr);
+}
+
+void Assembler::fistp_d(Operand adr) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xDF);
+ emit_operand(edi, adr);
+}
+
+void Assembler::fabs() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xD9);
+ EMIT(0xE1);
+}
+
+void Assembler::fchs() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xD9);
+ EMIT(0xE0);
+}
+
+void Assembler::fcos() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xD9);
+ EMIT(0xFF);
+}
+
+void Assembler::fsin() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xD9);
+ EMIT(0xFE);
+}
+
+void Assembler::fptan() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xD9);
+ EMIT(0xF2);
+}
+
+void Assembler::fyl2x() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xD9);
+ EMIT(0xF1);
+}
+
+void Assembler::f2xm1() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xD9);
+ EMIT(0xF0);
+}
+
+void Assembler::fscale() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xD9);
+ EMIT(0xFD);
+}
+
+void Assembler::fninit() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xDB);
+ EMIT(0xE3);
+}
+
+void Assembler::fadd(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xDC, 0xC0, i);
+}
+
+void Assembler::fadd_i(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xD8, 0xC0, i);
+}
+
+void Assembler::fsub(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xDC, 0xE8, i);
+}
+
+void Assembler::fsub_i(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xD8, 0xE0, i);
+}
+
+void Assembler::fisub_s(Operand adr) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xDA);
+ emit_operand(esp, adr);
+}
+
+void Assembler::fmul_i(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xD8, 0xC8, i);
+}
+
+void Assembler::fmul(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xDC, 0xC8, i);
+}
+
+void Assembler::fdiv(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xDC, 0xF8, i);
+}
+
+void Assembler::fdiv_i(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xD8, 0xF0, i);
+}
+
+void Assembler::faddp(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xDE, 0xC0, i);
+}
+
+void Assembler::fsubp(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xDE, 0xE8, i);
+}
+
+void Assembler::fsubrp(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xDE, 0xE0, i);
+}
+
+void Assembler::fmulp(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xDE, 0xC8, i);
+}
+
+void Assembler::fdivp(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xDE, 0xF8, i);
+}
+
+void Assembler::fprem() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xD9);
+ EMIT(0xF8);
+}
+
+void Assembler::fprem1() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xD9);
+ EMIT(0xF5);
+}
+
+void Assembler::fxch(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xD9, 0xC8, i);
+}
+
+void Assembler::fincstp() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xD9);
+ EMIT(0xF7);
+}
+
+void Assembler::ffree(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xDD, 0xC0, i);
+}
+
+void Assembler::ftst() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xD9);
+ EMIT(0xE4);
+}
+
+void Assembler::fucomp(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xDD, 0xE8, i);
+}
+
+void Assembler::fucompp() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xDA);
+ EMIT(0xE9);
+}
+
+void Assembler::fucomi(int i) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xDB);
+ EMIT(0xE8 + i);
+}
+
+void Assembler::fucomip() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xDF);
+ EMIT(0xE9);
+}
+
+void Assembler::fcompp() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xDE);
+ EMIT(0xD9);
+}
+
+void Assembler::fnstsw_ax() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xDF);
+ EMIT(0xE0);
+}
+
+void Assembler::fwait() {
+ EnsureSpace ensure_space(this);
+ EMIT(0x9B);
+}
+
+void Assembler::frndint() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xD9);
+ EMIT(0xFC);
+}
+
+void Assembler::fnclex() {
+ EnsureSpace ensure_space(this);
+ EMIT(0xDB);
+ EMIT(0xE2);
+}
+
+void Assembler::sahf() {
+ EnsureSpace ensure_space(this);
+ EMIT(0x9E);
+}
+
+void Assembler::setcc(Condition cc, Register reg) {
+ DCHECK(reg.is_byte_register());
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0x90 | cc);
+ EMIT(0xC0 | reg.code());
+}
+
+void Assembler::cvttss2si(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ // The [src] might contain ebx's register code, but in
+ // this case, it refers to xmm3, so it is OK to emit.
+ EMIT(0xF3);
+ EMIT(0x0F);
+ EMIT(0x2C);
+ emit_operand(dst, src);
+}
+
+void Assembler::cvttsd2si(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ // The [src] might contain ebx's register code, but in
+ // this case, it refers to xmm3, so it is OK to emit.
+ EMIT(0xF2);
+ EMIT(0x0F);
+ EMIT(0x2C);
+ emit_operand(dst, src);
+}
+
+void Assembler::cvtsd2si(Register dst, XMMRegister src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF2);
+ EMIT(0x0F);
+ EMIT(0x2D);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cvtsi2ss(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF3);
+ EMIT(0x0F);
+ EMIT(0x2A);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cvtsi2sd(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF2);
+ EMIT(0x0F);
+ EMIT(0x2A);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cvtss2sd(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF3);
+ EMIT(0x0F);
+ EMIT(0x5A);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cvtsd2ss(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF2);
+ EMIT(0x0F);
+ EMIT(0x5A);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cvtdq2ps(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0x5B);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cvttps2dq(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF3);
+ EMIT(0x0F);
+ EMIT(0x5B);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::addsd(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF2);
+ EMIT(0x0F);
+ EMIT(0x58);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::mulsd(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF2);
+ EMIT(0x0F);
+ EMIT(0x59);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::subsd(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF2);
+ EMIT(0x0F);
+ EMIT(0x5C);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::divsd(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF2);
+ EMIT(0x0F);
+ EMIT(0x5E);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::rcpps(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0x53);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::sqrtps(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0x51);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::rsqrtps(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0x52);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cmpps(XMMRegister dst, Operand src, uint8_t cmp) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0xC2);
+ emit_sse_operand(dst, src);
+ EMIT(cmp);
+}
+
+void Assembler::cmppd(XMMRegister dst, Operand src, uint8_t cmp) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0xC2);
+ emit_sse_operand(dst, src);
+ EMIT(cmp);
+}
+
+void Assembler::sqrtsd(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF2);
+ EMIT(0x0F);
+ EMIT(0x51);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::haddps(XMMRegister dst, Operand src) {
+ DCHECK(IsEnabled(SSE3));
+ EnsureSpace ensure_space(this);
+ EMIT(0xF2);
+ EMIT(0x0F);
+ EMIT(0x7C);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::ucomisd(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x2E);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::roundps(XMMRegister dst, XMMRegister src, RoundingMode mode) {
+ DCHECK(IsEnabled(SSE4_1));
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x3A);
+ EMIT(0x08);
+ emit_sse_operand(dst, src);
+ // Mask precision exeption.
+ EMIT(static_cast<byte>(mode) | 0x8);
+}
+
+void Assembler::roundpd(XMMRegister dst, XMMRegister src, RoundingMode mode) {
+ DCHECK(IsEnabled(SSE4_1));
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x3A);
+ EMIT(0x09);
+ emit_sse_operand(dst, src);
+ // Mask precision exeption.
+ EMIT(static_cast<byte>(mode) | 0x8);
+}
+
+void Assembler::roundss(XMMRegister dst, XMMRegister src, RoundingMode mode) {
+ DCHECK(IsEnabled(SSE4_1));
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x3A);
+ EMIT(0x0A);
+ emit_sse_operand(dst, src);
+ // Mask precision exeption.
+ EMIT(static_cast<byte>(mode) | 0x8);
+}
+
+void Assembler::roundsd(XMMRegister dst, XMMRegister src, RoundingMode mode) {
+ DCHECK(IsEnabled(SSE4_1));
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x3A);
+ EMIT(0x0B);
+ emit_sse_operand(dst, src);
+ // Mask precision exeption.
+ EMIT(static_cast<byte>(mode) | 0x8);
+}
+
+void Assembler::movmskpd(Register dst, XMMRegister src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x50);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movmskps(Register dst, XMMRegister src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0x50);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::pmovmskb(Register dst, XMMRegister src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0xD7);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::maxsd(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF2);
+ EMIT(0x0F);
+ EMIT(0x5F);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::minsd(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF2);
+ EMIT(0x0F);
+ EMIT(0x5D);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cmpltsd(XMMRegister dst, XMMRegister src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF2);
+ EMIT(0x0F);
+ EMIT(0xC2);
+ emit_sse_operand(dst, src);
+ EMIT(1); // LT == 1
+}
+
+void Assembler::movaps(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0x28);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movups(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0x10);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movups(Operand dst, XMMRegister src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0x11);
+ emit_sse_operand(src, dst);
+}
+
+void Assembler::movddup(XMMRegister dst, Operand src) {
+ DCHECK(IsEnabled(SSE3));
+ EnsureSpace ensure_space(this);
+ EMIT(0xF2);
+ EMIT(0x0F);
+ EMIT(0x12);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::shufps(XMMRegister dst, XMMRegister src, byte imm8) {
+ DCHECK(is_uint8(imm8));
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0xC6);
+ emit_sse_operand(dst, src);
+ EMIT(imm8);
+}
+
+void Assembler::shufpd(XMMRegister dst, XMMRegister src, byte imm8) {
+ DCHECK(is_uint8(imm8));
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0xC6);
+ emit_sse_operand(dst, src);
+ EMIT(imm8);
+}
+
+void Assembler::movdqa(Operand dst, XMMRegister src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x7F);
+ emit_sse_operand(src, dst);
+}
+
+void Assembler::movdqa(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x6F);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movdqu(Operand dst, XMMRegister src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF3);
+ EMIT(0x0F);
+ EMIT(0x7F);
+ emit_sse_operand(src, dst);
+}
+
+void Assembler::movdqu(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF3);
+ EMIT(0x0F);
+ EMIT(0x6F);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::prefetch(Operand src, int level) {
+ DCHECK(is_uint2(level));
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0x18);
+ // Emit hint number in Reg position of RegR/M.
+ XMMRegister code = XMMRegister::from_code(level);
+ emit_sse_operand(code, src);
+}
+
+void Assembler::movsd(Operand dst, XMMRegister src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF2); // double
+ EMIT(0x0F);
+ EMIT(0x11); // store
+ emit_sse_operand(src, dst);
+}
+
+void Assembler::movsd(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF2); // double
+ EMIT(0x0F);
+ EMIT(0x10); // load
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movss(Operand dst, XMMRegister src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF3); // float
+ EMIT(0x0F);
+ EMIT(0x11); // store
+ emit_sse_operand(src, dst);
+}
+
+void Assembler::movss(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF3); // float
+ EMIT(0x0F);
+ EMIT(0x10); // load
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movd(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x6E);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movd(Operand dst, XMMRegister src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x7E);
+ emit_sse_operand(src, dst);
+}
+
+void Assembler::extractps(Register dst, XMMRegister src, byte imm8) {
+ DCHECK(IsEnabled(SSE4_1));
+ DCHECK(is_uint8(imm8));
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x3A);
+ EMIT(0x17);
+ emit_sse_operand(src, dst);
+ EMIT(imm8);
+}
+
+void Assembler::psllw(XMMRegister reg, uint8_t shift) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x71);
+ emit_sse_operand(esi, reg); // esi == 6
+ EMIT(shift);
+}
+
+void Assembler::pslld(XMMRegister reg, uint8_t shift) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x72);
+ emit_sse_operand(esi, reg); // esi == 6
+ EMIT(shift);
+}
+
+void Assembler::psrlw(XMMRegister reg, uint8_t shift) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x71);
+ emit_sse_operand(edx, reg); // edx == 2
+ EMIT(shift);
+}
+
+void Assembler::psrld(XMMRegister reg, uint8_t shift) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x72);
+ emit_sse_operand(edx, reg); // edx == 2
+ EMIT(shift);
+}
+
+void Assembler::psraw(XMMRegister reg, uint8_t shift) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x71);
+ emit_sse_operand(esp, reg); // esp == 4
+ EMIT(shift);
+}
+
+void Assembler::psrad(XMMRegister reg, uint8_t shift) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x72);
+ emit_sse_operand(esp, reg); // esp == 4
+ EMIT(shift);
+}
+
+void Assembler::psllq(XMMRegister reg, uint8_t shift) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x73);
+ emit_sse_operand(esi, reg); // esi == 6
+ EMIT(shift);
+}
+
+void Assembler::psrlq(XMMRegister reg, uint8_t shift) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x73);
+ emit_sse_operand(edx, reg); // edx == 2
+ EMIT(shift);
+}
+
+void Assembler::pshufhw(XMMRegister dst, Operand src, uint8_t shuffle) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF3);
+ EMIT(0x0F);
+ EMIT(0x70);
+ emit_sse_operand(dst, src);
+ EMIT(shuffle);
+}
+
+void Assembler::pshuflw(XMMRegister dst, Operand src, uint8_t shuffle) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF2);
+ EMIT(0x0F);
+ EMIT(0x70);
+ emit_sse_operand(dst, src);
+ EMIT(shuffle);
+}
+
+void Assembler::pshufd(XMMRegister dst, Operand src, uint8_t shuffle) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x70);
+ emit_sse_operand(dst, src);
+ EMIT(shuffle);
+}
+
+void Assembler::pblendw(XMMRegister dst, Operand src, uint8_t mask) {
+ DCHECK(IsEnabled(SSE4_1));
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x3A);
+ EMIT(0x0E);
+ emit_sse_operand(dst, src);
+ EMIT(mask);
+}
+
+void Assembler::palignr(XMMRegister dst, Operand src, uint8_t mask) {
+ DCHECK(IsEnabled(SSSE3));
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x3A);
+ EMIT(0x0F);
+ emit_sse_operand(dst, src);
+ EMIT(mask);
+}
+
+void Assembler::pextrb(Operand dst, XMMRegister src, uint8_t offset) {
+ DCHECK(IsEnabled(SSE4_1));
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x3A);
+ EMIT(0x14);
+ emit_sse_operand(src, dst);
+ EMIT(offset);
+}
+
+void Assembler::pextrw(Operand dst, XMMRegister src, uint8_t offset) {
+ DCHECK(IsEnabled(SSE4_1));
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x3A);
+ EMIT(0x15);
+ emit_sse_operand(src, dst);
+ EMIT(offset);
+}
+
+void Assembler::pextrd(Operand dst, XMMRegister src, uint8_t offset) {
+ DCHECK(IsEnabled(SSE4_1));
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x3A);
+ EMIT(0x16);
+ emit_sse_operand(src, dst);
+ EMIT(offset);
+}
+
+void Assembler::insertps(XMMRegister dst, Operand src, uint8_t offset) {
+ DCHECK(IsEnabled(SSE4_1));
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x3A);
+ EMIT(0x21);
+ emit_sse_operand(dst, src);
+ EMIT(offset);
+}
+
+void Assembler::pinsrb(XMMRegister dst, Operand src, uint8_t offset) {
+ DCHECK(IsEnabled(SSE4_1));
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x3A);
+ EMIT(0x20);
+ emit_sse_operand(dst, src);
+ EMIT(offset);
+}
+
+void Assembler::pinsrw(XMMRegister dst, Operand src, uint8_t offset) {
+ DCHECK(is_uint8(offset));
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0xC4);
+ emit_sse_operand(dst, src);
+ EMIT(offset);
+}
+
+void Assembler::pinsrd(XMMRegister dst, Operand src, uint8_t offset) {
+ DCHECK(IsEnabled(SSE4_1));
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(0x3A);
+ EMIT(0x22);
+ emit_sse_operand(dst, src);
+ EMIT(offset);
+}
+
+void Assembler::addss(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF3);
+ EMIT(0x0F);
+ EMIT(0x58);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::subss(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF3);
+ EMIT(0x0F);
+ EMIT(0x5C);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::mulss(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF3);
+ EMIT(0x0F);
+ EMIT(0x59);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::divss(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF3);
+ EMIT(0x0F);
+ EMIT(0x5E);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::sqrtss(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF3);
+ EMIT(0x0F);
+ EMIT(0x51);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::ucomiss(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(0x2E);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::maxss(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF3);
+ EMIT(0x0F);
+ EMIT(0x5F);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::minss(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0xF3);
+ EMIT(0x0F);
+ EMIT(0x5D);
+ emit_sse_operand(dst, src);
+}
+
+// Packed single-precision floating-point SSE instructions.
+void Assembler::ps(byte opcode, XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x0F);
+ EMIT(opcode);
+ emit_sse_operand(dst, src);
+}
+
+// Packed double-precision floating-point SSE instructions.
+void Assembler::pd(byte opcode, XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ EMIT(0x66);
+ EMIT(0x0F);
+ EMIT(opcode);
+ emit_sse_operand(dst, src);
+}
+
+// AVX instructions
+void Assembler::vfmasd(byte op, XMMRegister dst, XMMRegister src1,
+ Operand src2) {
+ DCHECK(IsEnabled(FMA3));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(src1, kLIG, k66, k0F38, kW1);
+ EMIT(op);
+ emit_sse_operand(dst, src2);
+}
+
+void Assembler::vfmass(byte op, XMMRegister dst, XMMRegister src1,
+ Operand src2) {
+ DCHECK(IsEnabled(FMA3));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(src1, kLIG, k66, k0F38, kW0);
+ EMIT(op);
+ emit_sse_operand(dst, src2);
+}
+
+void Assembler::vsd(byte op, XMMRegister dst, XMMRegister src1, Operand src2) {
+ vinstr(op, dst, src1, src2, kF2, k0F, kWIG);
+}
+
+void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1, Operand src2) {
+ vinstr(op, dst, src1, src2, kF3, k0F, kWIG);
+}
+
+void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1, Operand src2) {
+ vinstr(op, dst, src1, src2, kNone, k0F, kWIG);
+}
+
+void Assembler::vpd(byte op, XMMRegister dst, XMMRegister src1, Operand src2) {
+ vinstr(op, dst, src1, src2, k66, k0F, kWIG);
+}
+
+void Assembler::vshufpd(XMMRegister dst, XMMRegister src1, Operand src2,
+ byte imm8) {
+ DCHECK(is_uint8(imm8));
+ vpd(0xC6, dst, src1, src2);
+ EMIT(imm8);
+}
+
+void Assembler::vcmpps(XMMRegister dst, XMMRegister src1, Operand src2,
+ uint8_t cmp) {
+ vps(0xC2, dst, src1, src2);
+ EMIT(cmp);
+}
+
+void Assembler::vcmppd(XMMRegister dst, XMMRegister src1, Operand src2,
+ uint8_t cmp) {
+ vpd(0xC2, dst, src1, src2);
+ EMIT(cmp);
+}
+
+void Assembler::vshufps(XMMRegister dst, XMMRegister src1, Operand src2,
+ byte imm8) {
+ DCHECK(is_uint8(imm8));
+ vps(0xC6, dst, src1, src2);
+ EMIT(imm8);
+}
+
+void Assembler::vpsllw(XMMRegister dst, XMMRegister src, uint8_t imm8) {
+ XMMRegister iop = XMMRegister::from_code(6);
+ vinstr(0x71, iop, dst, Operand(src), k66, k0F, kWIG);
+ EMIT(imm8);
+}
+
+void Assembler::vpslld(XMMRegister dst, XMMRegister src, uint8_t imm8) {
+ XMMRegister iop = XMMRegister::from_code(6);
+ vinstr(0x72, iop, dst, Operand(src), k66, k0F, kWIG);
+ EMIT(imm8);
+}
+
+void Assembler::vpsllq(XMMRegister dst, XMMRegister src, uint8_t imm8) {
+ XMMRegister iop = XMMRegister::from_code(6);
+ vinstr(0x73, iop, dst, Operand(src), k66, k0F, kWIG);
+ EMIT(imm8);
+}
+
+void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, uint8_t imm8) {
+ XMMRegister iop = XMMRegister::from_code(2);
+ vinstr(0x71, iop, dst, Operand(src), k66, k0F, kWIG);
+ EMIT(imm8);
+}
+
+void Assembler::vpsrld(XMMRegister dst, XMMRegister src, uint8_t imm8) {
+ XMMRegister iop = XMMRegister::from_code(2);
+ vinstr(0x72, iop, dst, Operand(src), k66, k0F, kWIG);
+ EMIT(imm8);
+}
+
+void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, uint8_t imm8) {
+ XMMRegister iop = XMMRegister::from_code(2);
+ vinstr(0x73, iop, dst, Operand(src), k66, k0F, kWIG);
+ EMIT(imm8);
+}
+
+void Assembler::vpsraw(XMMRegister dst, XMMRegister src, uint8_t imm8) {
+ XMMRegister iop = XMMRegister::from_code(4);
+ vinstr(0x71, iop, dst, Operand(src), k66, k0F, kWIG);
+ EMIT(imm8);
+}
+
+void Assembler::vpsrad(XMMRegister dst, XMMRegister src, uint8_t imm8) {
+ XMMRegister iop = XMMRegister::from_code(4);
+ vinstr(0x72, iop, dst, Operand(src), k66, k0F, kWIG);
+ EMIT(imm8);
+}
+
+void Assembler::vpshufhw(XMMRegister dst, Operand src, uint8_t shuffle) {
+ vinstr(0x70, dst, xmm0, src, kF3, k0F, kWIG);
+ EMIT(shuffle);
+}
+
+void Assembler::vpshuflw(XMMRegister dst, Operand src, uint8_t shuffle) {
+ vinstr(0x70, dst, xmm0, src, kF2, k0F, kWIG);
+ EMIT(shuffle);
+}
+
+void Assembler::vpshufd(XMMRegister dst, Operand src, uint8_t shuffle) {
+ vinstr(0x70, dst, xmm0, src, k66, k0F, kWIG);
+ EMIT(shuffle);
+}
+
+void Assembler::vpblendw(XMMRegister dst, XMMRegister src1, Operand src2,
+ uint8_t mask) {
+ vinstr(0x0E, dst, src1, src2, k66, k0F3A, kWIG);
+ EMIT(mask);
+}
+
+void Assembler::vpalignr(XMMRegister dst, XMMRegister src1, Operand src2,
+ uint8_t mask) {
+ vinstr(0x0F, dst, src1, src2, k66, k0F3A, kWIG);
+ EMIT(mask);
+}
+
+void Assembler::vpextrb(Operand dst, XMMRegister src, uint8_t offset) {
+ vinstr(0x14, src, xmm0, dst, k66, k0F3A, kWIG);
+ EMIT(offset);
+}
+
+void Assembler::vpextrw(Operand dst, XMMRegister src, uint8_t offset) {
+ vinstr(0x15, src, xmm0, dst, k66, k0F3A, kWIG);
+ EMIT(offset);
+}
+
+void Assembler::vpextrd(Operand dst, XMMRegister src, uint8_t offset) {
+ vinstr(0x16, src, xmm0, dst, k66, k0F3A, kWIG);
+ EMIT(offset);
+}
+
+void Assembler::vinsertps(XMMRegister dst, XMMRegister src1, Operand src2,
+ uint8_t offset) {
+ vinstr(0x21, dst, src1, src2, k66, k0F3A, kWIG);
+ EMIT(offset);
+}
+
+void Assembler::vpinsrb(XMMRegister dst, XMMRegister src1, Operand src2,
+ uint8_t offset) {
+ vinstr(0x20, dst, src1, src2, k66, k0F3A, kWIG);
+ EMIT(offset);
+}
+
+void Assembler::vpinsrw(XMMRegister dst, XMMRegister src1, Operand src2,
+ uint8_t offset) {
+ vinstr(0xC4, dst, src1, src2, k66, k0F, kWIG);
+ EMIT(offset);
+}
+
+void Assembler::vpinsrd(XMMRegister dst, XMMRegister src1, Operand src2,
+ uint8_t offset) {
+ vinstr(0x22, dst, src1, src2, k66, k0F3A, kWIG);
+ EMIT(offset);
+}
+
+void Assembler::vroundps(XMMRegister dst, XMMRegister src, RoundingMode mode) {
+ vinstr(0x08, dst, xmm0, Operand(src), k66, k0F3A, kWIG);
+ EMIT(static_cast<byte>(mode) | 0x8); // Mask precision exception.
+}
+void Assembler::vroundpd(XMMRegister dst, XMMRegister src, RoundingMode mode) {
+ vinstr(0x09, dst, xmm0, Operand(src), k66, k0F3A, kWIG);
+ EMIT(static_cast<byte>(mode) | 0x8); // Mask precision exception.
+}
+
+void Assembler::vmovmskps(Register dst, XMMRegister src) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(xmm0, kL128, kNone, k0F, kWIG);
+ EMIT(0x50);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::vpmovmskb(Register dst, XMMRegister src) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(xmm0, kL128, k66, k0F, kWIG);
+ EMIT(0xD7);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::bmi1(byte op, Register reg, Register vreg, Operand rm) {
+ DCHECK(IsEnabled(BMI1));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(vreg, kLZ, kNone, k0F38, kW0);
+ EMIT(op);
+ emit_operand(reg, rm);
+}
+
+void Assembler::tzcnt(Register dst, Operand src) {
+ DCHECK(IsEnabled(BMI1));
+ EnsureSpace ensure_space(this);
+ EMIT(0xF3);
+ EMIT(0x0F);
+ EMIT(0xBC);
+ emit_operand(dst, src);
+}
+
+void Assembler::lzcnt(Register dst, Operand src) {
+ DCHECK(IsEnabled(LZCNT));
+ EnsureSpace ensure_space(this);
+ EMIT(0xF3);
+ EMIT(0x0F);
+ EMIT(0xBD);
+ emit_operand(dst, src);
+}
+
+void Assembler::popcnt(Register dst, Operand src) {
+ DCHECK(IsEnabled(POPCNT));
+ EnsureSpace ensure_space(this);
+ EMIT(0xF3);
+ EMIT(0x0F);
+ EMIT(0xB8);
+ emit_operand(dst, src);
+}
+
+void Assembler::bmi2(SIMDPrefix pp, byte op, Register reg, Register vreg,
+ Operand rm) {
+ DCHECK(IsEnabled(BMI2));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(vreg, kLZ, pp, k0F38, kW0);
+ EMIT(op);
+ emit_operand(reg, rm);
+}
+
+void Assembler::rorx(Register dst, Operand src, byte imm8) {
+ DCHECK(IsEnabled(BMI2));
+ DCHECK(is_uint8(imm8));
+ Register vreg = Register::from_code(0); // VEX.vvvv unused
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(vreg, kLZ, kF2, k0F3A, kW0);
+ EMIT(0xF0);
+ emit_operand(dst, src);
+ EMIT(imm8);
+}
+
+void Assembler::sse2_instr(XMMRegister dst, Operand src, byte prefix,
+ byte escape, byte opcode) {
+ EnsureSpace ensure_space(this);
+ EMIT(prefix);
+ EMIT(escape);
+ EMIT(opcode);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::ssse3_instr(XMMRegister dst, Operand src, byte prefix,
+ byte escape1, byte escape2, byte opcode) {
+ DCHECK(IsEnabled(SSSE3));
+ EnsureSpace ensure_space(this);
+ EMIT(prefix);
+ EMIT(escape1);
+ EMIT(escape2);
+ EMIT(opcode);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::sse4_instr(XMMRegister dst, Operand src, byte prefix,
+ byte escape1, byte escape2, byte opcode) {
+ DCHECK(IsEnabled(SSE4_1));
+ EnsureSpace ensure_space(this);
+ EMIT(prefix);
+ EMIT(escape1);
+ EMIT(escape2);
+ EMIT(opcode);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::vinstr(byte op, XMMRegister dst, XMMRegister src1, Operand src2,
+ SIMDPrefix pp, LeadingOpcode m, VexW w) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(src1, kL128, pp, m, w);
+ EMIT(op);
+ emit_sse_operand(dst, src2);
+}
+
+void Assembler::emit_sse_operand(XMMRegister reg, Operand adr) {
+ Register ireg = Register::from_code(reg.code());
+ emit_operand(ireg, adr);
+}
+
+void Assembler::emit_sse_operand(XMMRegister dst, XMMRegister src) {
+ EMIT(0xC0 | dst.code() << 3 | src.code());
+}
+
+void Assembler::emit_sse_operand(Register dst, XMMRegister src) {
+ EMIT(0xC0 | dst.code() << 3 | src.code());
+}
+
+void Assembler::emit_sse_operand(XMMRegister dst, Register src) {
+ EMIT(0xC0 | (dst.code() << 3) | src.code());
+}
+
+void Assembler::emit_vex_prefix(XMMRegister vreg, VectorLength l, SIMDPrefix pp,
+ LeadingOpcode mm, VexW w) {
+ if (mm != k0F || w != kW0) {
+ EMIT(0xC4);
+ // Change RXB from "110" to "111" to align with gdb disassembler.
+ EMIT(0xE0 | mm);
+ EMIT(w | ((~vreg.code() & 0xF) << 3) | l | pp);
+ } else {
+ EMIT(0xC5);
+ EMIT(((~vreg.code()) << 3) | l | pp);
+ }
+}
+
+void Assembler::emit_vex_prefix(Register vreg, VectorLength l, SIMDPrefix pp,
+ LeadingOpcode mm, VexW w) {
+ XMMRegister ivreg = XMMRegister::from_code(vreg.code());
+ emit_vex_prefix(ivreg, l, pp, mm, w);
+}
+
+void Assembler::GrowBuffer() {
+ DCHECK(buffer_overflow());
+ DCHECK_EQ(buffer_start_, buffer_->start());
+
+ // Compute new buffer size.
+ int old_size = buffer_->size();
+ int new_size = 2 * old_size;
+
+ // Some internal data structures overflow for very large buffers,
+ // they must ensure that kMaximalBufferSize is not too large.
+ if (new_size > kMaximalBufferSize) {
+ V8::FatalProcessOutOfMemory(nullptr, "Assembler::GrowBuffer");
+ }
+
+ // Set up new buffer.
+ std::unique_ptr<AssemblerBuffer> new_buffer = buffer_->Grow(new_size);
+ DCHECK_EQ(new_size, new_buffer->size());
+ byte* new_start = new_buffer->start();
+
+ // Copy the data.
+ intptr_t pc_delta = new_start - buffer_start_;
+ intptr_t rc_delta = (new_start + new_size) - (buffer_start_ + old_size);
+ size_t reloc_size = (buffer_start_ + old_size) - reloc_info_writer.pos();
+ MemMove(new_start, buffer_start_, pc_offset());
+ MemMove(rc_delta + reloc_info_writer.pos(), reloc_info_writer.pos(),
+ reloc_size);
+
+ // Switch buffers.
+ buffer_ = std::move(new_buffer);
+ buffer_start_ = new_start;
+ pc_ += pc_delta;
+ reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta,
+ reloc_info_writer.last_pc() + pc_delta);
+
+ // Relocate internal references.
+ for (auto pos : internal_reference_positions_) {
+ Address p = reinterpret_cast<Address>(buffer_start_ + pos);
+ WriteUnalignedValue(p, ReadUnalignedValue<int>(p) + pc_delta);
+ }
+
+ // Relocate pc-relative references.
+ int mode_mask = RelocInfo::ModeMask(RelocInfo::OFF_HEAP_TARGET);
+ DCHECK_EQ(mode_mask, RelocInfo::kApplyMask & mode_mask);
+ Vector<byte> instructions{buffer_start_, static_cast<size_t>(pc_offset())};
+ Vector<const byte> reloc_info{reloc_info_writer.pos(), reloc_size};
+ for (RelocIterator it(instructions, reloc_info, 0, mode_mask); !it.done();
+ it.next()) {
+ it.rinfo()->apply(pc_delta);
+ }
+
+ DCHECK(!buffer_overflow());
+}
+
+void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
+ DCHECK(is_uint8(op1) && is_uint8(op2)); // wrong opcode
+ DCHECK(is_uint8(imm8));
+ DCHECK_EQ(op1 & 0x01, 0); // should be 8bit operation
+ EMIT(op1);
+ EMIT(op2 | dst.code());
+ EMIT(imm8);
+}
+
+void Assembler::emit_arith(int sel, Operand dst, const Immediate& x) {
+ DCHECK((0 <= sel) && (sel <= 7));
+ Register ireg = Register::from_code(sel);
+ if (x.is_int8()) {
+ EMIT(0x83); // using a sign-extended 8-bit immediate.
+ emit_operand(ireg, dst);
+ EMIT(x.immediate() & 0xFF);
+ } else if (dst.is_reg(eax)) {
+ EMIT((sel << 3) | 0x05); // short form if the destination is eax.
+ emit(x);
+ } else {
+ EMIT(0x81); // using a literal 32-bit immediate.
+ emit_operand(ireg, dst);
+ emit(x);
+ }
+}
+
+void Assembler::emit_operand(Register reg, Operand adr) {
+ emit_operand(reg.code(), adr);
+}
+
+void Assembler::emit_operand(XMMRegister reg, Operand adr) {
+ Register ireg = Register::from_code(reg.code());
+ emit_operand(ireg, adr);
+}
+
+void Assembler::emit_operand(int code, Operand adr) {
+ // Isolate-independent code may not embed relocatable addresses.
+ DCHECK(!options().isolate_independent_code ||
+ adr.rmode_ != RelocInfo::CODE_TARGET);
+ DCHECK(!options().isolate_independent_code ||
+ adr.rmode_ != RelocInfo::FULL_EMBEDDED_OBJECT);
+ DCHECK(!options().isolate_independent_code ||
+ adr.rmode_ != RelocInfo::EXTERNAL_REFERENCE);
+
+ const unsigned length = adr.len_;
+ DCHECK_GT(length, 0);
+
+ // Emit updated ModRM byte containing the given register.
+ EMIT((adr.buf_[0] & ~0x38) | (code << 3));
+
+ // Emit the rest of the encoded operand.
+ for (unsigned i = 1; i < length; i++) EMIT(adr.buf_[i]);
+
+ // Emit relocation information if necessary.
+ if (length >= sizeof(int32_t) && !RelocInfo::IsNone(adr.rmode_)) {
+ pc_ -= sizeof(int32_t); // pc_ must be *at* disp32
+ RecordRelocInfo(adr.rmode_);
+ if (adr.rmode_ == RelocInfo::INTERNAL_REFERENCE) { // Fixup for labels
+ emit_label(ReadUnalignedValue<Label*>(reinterpret_cast<Address>(pc_)));
+ } else {
+ pc_ += sizeof(int32_t);
+ }
+ }
+}
+
+void Assembler::emit_label(Label* label) {
+ if (label->is_bound()) {
+ internal_reference_positions_.push_back(pc_offset());
+ emit(reinterpret_cast<uint32_t>(buffer_start_ + label->pos()));
+ } else {
+ emit_disp(label, Displacement::CODE_ABSOLUTE);
+ }
+}
+
+void Assembler::emit_farith(int b1, int b2, int i) {
+ DCHECK(is_uint8(b1) && is_uint8(b2)); // wrong opcode
+ DCHECK(0 <= i && i < 8); // illegal stack offset
+ EMIT(b1);
+ EMIT(b2 + i);
+}
+
+void Assembler::db(uint8_t data) {
+ EnsureSpace ensure_space(this);
+ EMIT(data);
+}
+
+void Assembler::dd(uint32_t data) {
+ EnsureSpace ensure_space(this);
+ emit(data);
+}
+
+void Assembler::dq(uint64_t data) {
+ EnsureSpace ensure_space(this);
+ emit_q(data);
+}
+
+void Assembler::dd(Label* label) {
+ EnsureSpace ensure_space(this);
+ RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE);
+ emit_label(label);
+}
+
+void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) {
+ if (!ShouldRecordRelocInfo(rmode)) return;
+ RelocInfo rinfo(reinterpret_cast<Address>(pc_), rmode, data, Code());
+ reloc_info_writer.Write(&rinfo);
+}
+
+#undef EMIT
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_IA32
diff --git a/src/codegen/ia32/assembler-ia32.h b/src/codegen/ia32/assembler-ia32.h
new file mode 100644
index 0000000..333daf6
--- /dev/null
+++ b/src/codegen/ia32/assembler-ia32.h
@@ -0,0 +1,1873 @@
+// Copyright (c) 1994-2006 Sun Microsystems Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// - Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// - Redistribution in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// - Neither the name of Sun Microsystems or the names of contributors may
+// be used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// The original source code covered by the above license above has been
+// modified significantly by Google Inc.
+// Copyright 2011 the V8 project authors. All rights reserved.
+
+// A light-weight IA32 Assembler.
+
+#ifndef V8_CODEGEN_IA32_ASSEMBLER_IA32_H_
+#define V8_CODEGEN_IA32_ASSEMBLER_IA32_H_
+
+#include <deque>
+#include <memory>
+
+#include "src/codegen/assembler.h"
+#include "src/codegen/ia32/constants-ia32.h"
+#include "src/codegen/ia32/register-ia32.h"
+#include "src/codegen/ia32/sse-instr.h"
+#include "src/codegen/label.h"
+#include "src/execution/isolate.h"
+#include "src/objects/smi.h"
+#include "src/utils/utils.h"
+
+namespace v8 {
+namespace internal {
+
+class SafepointTableBuilder;
+
+enum Condition {
+ // any value < 0 is considered no_condition
+ no_condition = -1,
+
+ overflow = 0,
+ no_overflow = 1,
+ below = 2,
+ above_equal = 3,
+ equal = 4,
+ not_equal = 5,
+ below_equal = 6,
+ above = 7,
+ negative = 8,
+ positive = 9,
+ parity_even = 10,
+ parity_odd = 11,
+ less = 12,
+ greater_equal = 13,
+ less_equal = 14,
+ greater = 15,
+
+ // aliases
+ carry = below,
+ not_carry = above_equal,
+ zero = equal,
+ not_zero = not_equal,
+ sign = negative,
+ not_sign = positive
+};
+
+// Returns the equivalent of !cc.
+// Negation of the default no_condition (-1) results in a non-default
+// no_condition value (-2). As long as tests for no_condition check
+// for condition < 0, this will work as expected.
+inline Condition NegateCondition(Condition cc) {
+ return static_cast<Condition>(cc ^ 1);
+}
+
+enum RoundingMode {
+ kRoundToNearest = 0x0,
+ kRoundDown = 0x1,
+ kRoundUp = 0x2,
+ kRoundToZero = 0x3
+};
+
+// -----------------------------------------------------------------------------
+// Machine instruction Immediates
+
+class Immediate {
+ public:
+ // Calls where x is an Address (uintptr_t) resolve to this overload.
+ inline explicit Immediate(int x, RelocInfo::Mode rmode = RelocInfo::NONE) {
+ value_.immediate = x;
+ rmode_ = rmode;
+ }
+ inline explicit Immediate(const ExternalReference& ext)
+ : Immediate(ext.address(), RelocInfo::EXTERNAL_REFERENCE) {}
+ inline explicit Immediate(Handle<HeapObject> handle)
+ : Immediate(handle.address(), RelocInfo::FULL_EMBEDDED_OBJECT) {}
+ inline explicit Immediate(Smi value)
+ : Immediate(static_cast<intptr_t>(value.ptr())) {}
+
+ static Immediate EmbeddedNumber(double number); // Smi or HeapNumber.
+ static Immediate EmbeddedStringConstant(const StringConstantBase* str);
+
+ static Immediate CodeRelativeOffset(Label* label) { return Immediate(label); }
+
+ bool is_heap_object_request() const {
+ DCHECK_IMPLIES(is_heap_object_request_,
+ rmode_ == RelocInfo::FULL_EMBEDDED_OBJECT ||
+ rmode_ == RelocInfo::CODE_TARGET);
+ return is_heap_object_request_;
+ }
+
+ HeapObjectRequest heap_object_request() const {
+ DCHECK(is_heap_object_request());
+ return value_.heap_object_request;
+ }
+
+ int immediate() const {
+ DCHECK(!is_heap_object_request());
+ return value_.immediate;
+ }
+
+ bool is_embedded_object() const {
+ return !is_heap_object_request() &&
+ rmode() == RelocInfo::FULL_EMBEDDED_OBJECT;
+ }
+
+ Handle<HeapObject> embedded_object() const {
+ return Handle<HeapObject>(reinterpret_cast<Address*>(immediate()));
+ }
+
+ bool is_external_reference() const {
+ return rmode() == RelocInfo::EXTERNAL_REFERENCE;
+ }
+
+ ExternalReference external_reference() const {
+ DCHECK(is_external_reference());
+ return bit_cast<ExternalReference>(immediate());
+ }
+
+ bool is_zero() const { return RelocInfo::IsNone(rmode_) && immediate() == 0; }
+ bool is_int8() const {
+ return RelocInfo::IsNone(rmode_) && i::is_int8(immediate());
+ }
+ bool is_uint8() const {
+ return RelocInfo::IsNone(rmode_) && i::is_uint8(immediate());
+ }
+ bool is_int16() const {
+ return RelocInfo::IsNone(rmode_) && i::is_int16(immediate());
+ }
+
+ bool is_uint16() const {
+ return RelocInfo::IsNone(rmode_) && i::is_uint16(immediate());
+ }
+
+ RelocInfo::Mode rmode() const { return rmode_; }
+
+ private:
+ inline explicit Immediate(Label* value) {
+ value_.immediate = reinterpret_cast<int32_t>(value);
+ rmode_ = RelocInfo::INTERNAL_REFERENCE;
+ }
+
+ union Value {
+ Value() {}
+ HeapObjectRequest heap_object_request;
+ int immediate;
+ } value_;
+ bool is_heap_object_request_ = false;
+ RelocInfo::Mode rmode_;
+
+ friend class Operand;
+ friend class Assembler;
+ friend class MacroAssembler;
+};
+
+// -----------------------------------------------------------------------------
+// Machine instruction Operands
+
+enum ScaleFactor {
+ times_1 = 0,
+ times_2 = 1,
+ times_4 = 2,
+ times_8 = 3,
+ times_int_size = times_4,
+
+ times_half_system_pointer_size = times_2,
+ times_system_pointer_size = times_4,
+
+ times_tagged_size = times_4,
+};
+
+class V8_EXPORT_PRIVATE Operand {
+ public:
+ // reg
+ V8_INLINE explicit Operand(Register reg) { set_modrm(3, reg); }
+
+ // XMM reg
+ V8_INLINE explicit Operand(XMMRegister xmm_reg) {
+ Register reg = Register::from_code(xmm_reg.code());
+ set_modrm(3, reg);
+ }
+
+ // [disp/r]
+ V8_INLINE explicit Operand(int32_t disp, RelocInfo::Mode rmode) {
+ set_modrm(0, ebp);
+ set_dispr(disp, rmode);
+ }
+
+ // [disp/r]
+ V8_INLINE explicit Operand(Immediate imm) {
+ set_modrm(0, ebp);
+ set_dispr(imm.immediate(), imm.rmode_);
+ }
+
+ // [base + disp/r]
+ explicit Operand(Register base, int32_t disp,
+ RelocInfo::Mode rmode = RelocInfo::NONE);
+
+ // [base + index*scale + disp/r]
+ explicit Operand(Register base, Register index, ScaleFactor scale,
+ int32_t disp, RelocInfo::Mode rmode = RelocInfo::NONE);
+
+ // [index*scale + disp/r]
+ explicit Operand(Register index, ScaleFactor scale, int32_t disp,
+ RelocInfo::Mode rmode = RelocInfo::NONE);
+
+ static Operand JumpTable(Register index, ScaleFactor scale, Label* table) {
+ return Operand(index, scale, reinterpret_cast<int32_t>(table),
+ RelocInfo::INTERNAL_REFERENCE);
+ }
+
+ static Operand ForRegisterPlusImmediate(Register base, Immediate imm) {
+ return Operand(base, imm.value_.immediate, imm.rmode_);
+ }
+
+ // Returns true if this Operand is a wrapper for the specified register.
+ bool is_reg(Register reg) const { return is_reg(reg.code()); }
+ bool is_reg(XMMRegister reg) const { return is_reg(reg.code()); }
+
+ // Returns true if this Operand is a wrapper for one register.
+ bool is_reg_only() const;
+
+ // Asserts that this Operand is a wrapper for one register and returns the
+ // register.
+ Register reg() const;
+
+ private:
+ // Set the ModRM byte without an encoded 'reg' register. The
+ // register is encoded later as part of the emit_operand operation.
+ inline void set_modrm(int mod, Register rm) {
+ DCHECK_EQ(mod & -4, 0);
+ buf_[0] = mod << 6 | rm.code();
+ len_ = 1;
+ }
+
+ inline void set_sib(ScaleFactor scale, Register index, Register base);
+ inline void set_disp8(int8_t disp);
+ inline void set_dispr(int32_t disp, RelocInfo::Mode rmode) {
+ DCHECK(len_ == 1 || len_ == 2);
+ Address p = reinterpret_cast<Address>(&buf_[len_]);
+ WriteUnalignedValue(p, disp);
+ len_ += sizeof(int32_t);
+ rmode_ = rmode;
+ }
+
+ inline bool is_reg(int reg_code) const {
+ return ((buf_[0] & 0xF8) == 0xC0) // addressing mode is register only.
+ && ((buf_[0] & 0x07) == reg_code); // register codes match.
+ }
+
+ byte buf_[6];
+ // The number of bytes in buf_.
+ uint8_t len_ = 0;
+ // Only valid if len_ > 4.
+ RelocInfo::Mode rmode_ = RelocInfo::NONE;
+
+ // TODO(clemensb): Get rid of this friendship, or make Operand immutable.
+ friend class Assembler;
+};
+ASSERT_TRIVIALLY_COPYABLE(Operand);
+static_assert(sizeof(Operand) <= 2 * kSystemPointerSize,
+ "Operand must be small enough to pass it by value");
+
+// -----------------------------------------------------------------------------
+// A Displacement describes the 32bit immediate field of an instruction which
+// may be used together with a Label in order to refer to a yet unknown code
+// position. Displacements stored in the instruction stream are used to describe
+// the instruction and to chain a list of instructions using the same Label.
+// A Displacement contains 2 different fields:
+//
+// next field: position of next displacement in the chain (0 = end of list)
+// type field: instruction type
+//
+// A next value of null (0) indicates the end of a chain (note that there can
+// be no displacement at position zero, because there is always at least one
+// instruction byte before the displacement).
+//
+// Displacement _data field layout
+//
+// |31.....2|1......0|
+// [ next | type |
+
+class Displacement {
+ public:
+ enum Type { UNCONDITIONAL_JUMP, CODE_RELATIVE, OTHER, CODE_ABSOLUTE };
+
+ int data() const { return data_; }
+ Type type() const { return TypeField::decode(data_); }
+ void next(Label* L) const {
+ int n = NextField::decode(data_);
+ n > 0 ? L->link_to(n) : L->Unuse();
+ }
+ void link_to(Label* L) { init(L, type()); }
+
+ explicit Displacement(int data) { data_ = data; }
+
+ Displacement(Label* L, Type type) { init(L, type); }
+
+ void print() {
+ PrintF("%s (%x) ", (type() == UNCONDITIONAL_JUMP ? "jmp" : "[other]"),
+ NextField::decode(data_));
+ }
+
+ private:
+ int data_;
+
+ using TypeField = base::BitField<Type, 0, 2>;
+ using NextField = base::BitField<int, 2, 32 - 2>;
+
+ void init(Label* L, Type type);
+};
+
+class V8_EXPORT_PRIVATE Assembler : public AssemblerBase {
+ private:
+ // We check before assembling an instruction that there is sufficient
+ // space to write an instruction and its relocation information.
+ // The relocation writer's position must be kGap bytes above the end of
+ // the generated instructions. This leaves enough space for the
+ // longest possible ia32 instruction, 15 bytes, and the longest possible
+ // relocation information encoding, RelocInfoWriter::kMaxLength == 16.
+ // (There is a 15 byte limit on ia32 instruction length that rules out some
+ // otherwise valid instructions.)
+ // This allows for a single, fast space check per instruction.
+ static constexpr int kGap = 32;
+ STATIC_ASSERT(AssemblerBase::kMinimalBufferSize >= 2 * kGap);
+
+ public:
+ // Create an assembler. Instructions and relocation information are emitted
+ // into a buffer, with the instructions starting from the beginning and the
+ // relocation information starting from the end of the buffer. See CodeDesc
+ // for a detailed comment on the layout (globals.h).
+ //
+ // If the provided buffer is nullptr, the assembler allocates and grows its
+ // own buffer. Otherwise it takes ownership of the provided buffer.
+ explicit Assembler(const AssemblerOptions&,
+ std::unique_ptr<AssemblerBuffer> = {});
+
+ // GetCode emits any pending (non-emitted) code and fills the descriptor desc.
+ static constexpr int kNoHandlerTable = 0;
+ static constexpr SafepointTableBuilder* kNoSafepointTable = nullptr;
+ void GetCode(Isolate* isolate, CodeDesc* desc,
+ SafepointTableBuilder* safepoint_table_builder,
+ int handler_table_offset);
+
+ // Convenience wrapper for code without safepoint or handler tables.
+ void GetCode(Isolate* isolate, CodeDesc* desc) {
+ GetCode(isolate, desc, kNoSafepointTable, kNoHandlerTable);
+ }
+
+ void FinalizeJumpOptimizationInfo();
+
+ // Unused on this architecture.
+ void MaybeEmitOutOfLineConstantPool() {}
+
+ // Read/Modify the code target in the branch/call instruction at pc.
+ // The isolate argument is unused (and may be nullptr) when skipping flushing.
+ inline static Address target_address_at(Address pc, Address constant_pool);
+ inline static void set_target_address_at(
+ Address pc, Address constant_pool, Address target,
+ ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED);
+
+ // This sets the branch destination (which is in the instruction on x86).
+ // This is for calls and branches within generated code.
+ inline static void deserialization_set_special_target_at(
+ Address instruction_payload, Code code, Address target);
+
+ // Get the size of the special target encoded at 'instruction_payload'.
+ inline static int deserialization_special_target_size(
+ Address instruction_payload);
+
+ // This sets the internal reference at the pc.
+ inline static void deserialization_set_target_internal_reference_at(
+ Address pc, Address target,
+ RelocInfo::Mode mode = RelocInfo::INTERNAL_REFERENCE);
+
+ static constexpr int kSpecialTargetSize = kSystemPointerSize;
+
+ // One byte opcode for test al, 0xXX.
+ static constexpr byte kTestAlByte = 0xA8;
+ // One byte opcode for nop.
+ static constexpr byte kNopByte = 0x90;
+
+ // One byte opcode for a short unconditional jump.
+ static constexpr byte kJmpShortOpcode = 0xEB;
+ // One byte prefix for a short conditional jump.
+ static constexpr byte kJccShortPrefix = 0x70;
+ static constexpr byte kJncShortOpcode = kJccShortPrefix | not_carry;
+ static constexpr byte kJcShortOpcode = kJccShortPrefix | carry;
+ static constexpr byte kJnzShortOpcode = kJccShortPrefix | not_zero;
+ static constexpr byte kJzShortOpcode = kJccShortPrefix | zero;
+
+ // ---------------------------------------------------------------------------
+ // Code generation
+ //
+ // - function names correspond one-to-one to ia32 instruction mnemonics
+ // - unless specified otherwise, instructions operate on 32bit operands
+ // - instructions on 8bit (byte) operands/registers have a trailing '_b'
+ // - instructions on 16bit (word) operands/registers have a trailing '_w'
+ // - naming conflicts with C++ keywords are resolved via a trailing '_'
+
+ // NOTE ON INTERFACE: Currently, the interface is not very consistent
+ // in the sense that some operations (e.g. mov()) can be called in more
+ // the one way to generate the same instruction: The Register argument
+ // can in some cases be replaced with an Operand(Register) argument.
+ // This should be cleaned up and made more orthogonal. The questions
+ // is: should we always use Operands instead of Registers where an
+ // Operand is possible, or should we have a Register (overloaded) form
+ // instead? We must be careful to make sure that the selected instruction
+ // is obvious from the parameters to avoid hard-to-find code generation
+ // bugs.
+
+ // Insert the smallest number of nop instructions
+ // possible to align the pc offset to a multiple
+ // of m. m must be a power of 2.
+ void Align(int m);
+ // Insert the smallest number of zero bytes possible to align the pc offset
+ // to a mulitple of m. m must be a power of 2 (>= 2).
+ void DataAlign(int m);
+ void Nop(int bytes = 1);
+ // Aligns code to something that's optimal for a jump target for the platform.
+ void CodeTargetAlign();
+
+ // Stack
+ void pushad();
+ void popad();
+
+ void pushfd();
+ void popfd();
+
+ void push(const Immediate& x);
+ void push_imm32(int32_t imm32);
+ void push(Register src);
+ void push(Operand src);
+
+ void pop(Register dst);
+ void pop(Operand dst);
+
+ void leave();
+
+ // Moves
+ void mov_b(Register dst, Register src) { mov_b(dst, Operand(src)); }
+ void mov_b(Register dst, Operand src);
+ void mov_b(Register dst, int8_t imm8) { mov_b(Operand(dst), imm8); }
+ void mov_b(Operand dst, int8_t src) { mov_b(dst, Immediate(src)); }
+ void mov_b(Operand dst, const Immediate& src);
+ void mov_b(Operand dst, Register src);
+
+ void mov_w(Register dst, Operand src);
+ void mov_w(Operand dst, int16_t src) { mov_w(dst, Immediate(src)); }
+ void mov_w(Operand dst, const Immediate& src);
+ void mov_w(Operand dst, Register src);
+
+ void mov(Register dst, int32_t imm32);
+ void mov(Register dst, const Immediate& x);
+ void mov(Register dst, Handle<HeapObject> handle);
+ void mov(Register dst, Operand src);
+ void mov(Register dst, Register src);
+ void mov(Operand dst, const Immediate& x);
+ void mov(Operand dst, Handle<HeapObject> handle);
+ void mov(Operand dst, Register src);
+ void mov(Operand dst, Address src, RelocInfo::Mode);
+
+ void movsx_b(Register dst, Register src) { movsx_b(dst, Operand(src)); }
+ void movsx_b(Register dst, Operand src);
+
+ void movsx_w(Register dst, Register src) { movsx_w(dst, Operand(src)); }
+ void movsx_w(Register dst, Operand src);
+
+ void movzx_b(Register dst, Register src) { movzx_b(dst, Operand(src)); }
+ void movzx_b(Register dst, Operand src);
+
+ void movzx_w(Register dst, Register src) { movzx_w(dst, Operand(src)); }
+ void movzx_w(Register dst, Operand src);
+
+ void movq(XMMRegister dst, Operand src);
+
+ // Conditional moves
+ void cmov(Condition cc, Register dst, Register src) {
+ cmov(cc, dst, Operand(src));
+ }
+ void cmov(Condition cc, Register dst, Operand src);
+
+ // Flag management.
+ void cld();
+
+ // Repetitive string instructions.
+ void rep_movs();
+ void rep_stos();
+ void stos();
+
+ void xadd(Operand dst, Register src);
+ void xadd_b(Operand dst, Register src);
+ void xadd_w(Operand dst, Register src);
+
+ // Exchange
+ void xchg(Register dst, Register src);
+ void xchg(Register dst, Operand src);
+ void xchg_b(Register reg, Operand op);
+ void xchg_w(Register reg, Operand op);
+
+ // Lock prefix
+ void lock();
+
+ // CompareExchange
+ void cmpxchg(Operand dst, Register src);
+ void cmpxchg_b(Operand dst, Register src);
+ void cmpxchg_w(Operand dst, Register src);
+ void cmpxchg8b(Operand dst);
+
+ // Memory Fence
+ void mfence();
+ void lfence();
+
+ void pause();
+
+ // Arithmetics
+ void adc(Register dst, int32_t imm32);
+ void adc(Register dst, Register src) { adc(dst, Operand(src)); }
+ void adc(Register dst, Operand src);
+
+ void add(Register dst, Register src) { add(dst, Operand(src)); }
+ void add(Register dst, Operand src);
+ void add(Operand dst, Register src);
+ void add(Register dst, const Immediate& imm) { add(Operand(dst), imm); }
+ void add(Operand dst, const Immediate& x);
+
+ void and_(Register dst, int32_t imm32);
+ void and_(Register dst, const Immediate& x);
+ void and_(Register dst, Register src) { and_(dst, Operand(src)); }
+ void and_(Register dst, Operand src);
+ void and_(Operand dst, Register src);
+ void and_(Operand dst, const Immediate& x);
+
+ void cmpb(Register reg, Immediate imm8) {
+ DCHECK(reg.is_byte_register());
+ cmpb(Operand(reg), imm8);
+ }
+ void cmpb(Operand op, Immediate imm8);
+ void cmpb(Register reg, Operand op);
+ void cmpb(Operand op, Register reg);
+ void cmpb(Register dst, Register src) { cmpb(Operand(dst), src); }
+ void cmpb_al(Operand op);
+ void cmpw_ax(Operand op);
+ void cmpw(Operand dst, Immediate src);
+ void cmpw(Register dst, Immediate src) { cmpw(Operand(dst), src); }
+ void cmpw(Register dst, Operand src);
+ void cmpw(Register dst, Register src) { cmpw(Operand(dst), src); }
+ void cmpw(Operand dst, Register src);
+ void cmp(Register reg, int32_t imm32);
+ void cmp(Register reg, Handle<HeapObject> handle);
+ void cmp(Register reg0, Register reg1) { cmp(reg0, Operand(reg1)); }
+ void cmp(Register reg, Operand op);
+ void cmp(Register reg, const Immediate& imm) { cmp(Operand(reg), imm); }
+ void cmp(Operand op, Register reg);
+ void cmp(Operand op, const Immediate& imm);
+ void cmp(Operand op, Handle<HeapObject> handle);
+
+ void dec_b(Register dst);
+ void dec_b(Operand dst);
+
+ void dec(Register dst);
+ void dec(Operand dst);
+
+ void cdq();
+
+ void idiv(Register src) { idiv(Operand(src)); }
+ void idiv(Operand src);
+ void div(Register src) { div(Operand(src)); }
+ void div(Operand src);
+
+ // Signed multiply instructions.
+ void imul(Register src); // edx:eax = eax * src.
+ void imul(Register dst, Register src) { imul(dst, Operand(src)); }
+ void imul(Register dst, Operand src); // dst = dst * src.
+ void imul(Register dst, Register src, int32_t imm32); // dst = src * imm32.
+ void imul(Register dst, Operand src, int32_t imm32);
+
+ void inc(Register dst);
+ void inc(Operand dst);
+
+ void lea(Register dst, Operand src);
+
+ // Unsigned multiply instruction.
+ void mul(Register src); // edx:eax = eax * reg.
+
+ void neg(Register dst);
+ void neg(Operand dst);
+
+ void not_(Register dst);
+ void not_(Operand dst);
+
+ void or_(Register dst, int32_t imm32);
+ void or_(Register dst, Register src) { or_(dst, Operand(src)); }
+ void or_(Register dst, Operand src);
+ void or_(Operand dst, Register src);
+ void or_(Register dst, const Immediate& imm) { or_(Operand(dst), imm); }
+ void or_(Operand dst, const Immediate& x);
+
+ void rcl(Register dst, uint8_t imm8);
+ void rcr(Register dst, uint8_t imm8);
+
+ void rol(Register dst, uint8_t imm8) { rol(Operand(dst), imm8); }
+ void rol(Operand dst, uint8_t imm8);
+ void rol_cl(Register dst) { rol_cl(Operand(dst)); }
+ void rol_cl(Operand dst);
+
+ void ror(Register dst, uint8_t imm8) { ror(Operand(dst), imm8); }
+ void ror(Operand dst, uint8_t imm8);
+ void ror_cl(Register dst) { ror_cl(Operand(dst)); }
+ void ror_cl(Operand dst);
+
+ void sar(Register dst, uint8_t imm8) { sar(Operand(dst), imm8); }
+ void sar(Operand dst, uint8_t imm8);
+ void sar_cl(Register dst) { sar_cl(Operand(dst)); }
+ void sar_cl(Operand dst);
+
+ void sbb(Register dst, Register src) { sbb(dst, Operand(src)); }
+ void sbb(Register dst, Operand src);
+
+ void shl(Register dst, uint8_t imm8) { shl(Operand(dst), imm8); }
+ void shl(Operand dst, uint8_t imm8);
+ void shl_cl(Register dst) { shl_cl(Operand(dst)); }
+ void shl_cl(Operand dst);
+ void shld(Register dst, Register src, uint8_t shift);
+ void shld_cl(Register dst, Register src);
+
+ void shr(Register dst, uint8_t imm8) { shr(Operand(dst), imm8); }
+ void shr(Operand dst, uint8_t imm8);
+ void shr_cl(Register dst) { shr_cl(Operand(dst)); }
+ void shr_cl(Operand dst);
+ void shrd(Register dst, Register src, uint8_t shift);
+ void shrd_cl(Register dst, Register src) { shrd_cl(Operand(dst), src); }
+ void shrd_cl(Operand dst, Register src);
+
+ void sub(Register dst, const Immediate& imm) { sub(Operand(dst), imm); }
+ void sub(Operand dst, const Immediate& x);
+ void sub(Register dst, Register src) { sub(dst, Operand(src)); }
+ void sub(Register dst, Operand src);
+ void sub(Operand dst, Register src);
+ void sub_sp_32(uint32_t imm);
+
+ void test(Register reg, const Immediate& imm);
+ void test(Register reg0, Register reg1) { test(reg0, Operand(reg1)); }
+ void test(Register reg, Operand op);
+ void test(Operand op, const Immediate& imm);
+ void test(Operand op, Register reg) { test(reg, op); }
+ void test_b(Register reg, Operand op);
+ void test_b(Register reg, Immediate imm8);
+ void test_b(Operand op, Immediate imm8);
+ void test_b(Operand op, Register reg) { test_b(reg, op); }
+ void test_b(Register dst, Register src) { test_b(dst, Operand(src)); }
+ void test_w(Register reg, Operand op);
+ void test_w(Register reg, Immediate imm16);
+ void test_w(Operand op, Immediate imm16);
+ void test_w(Operand op, Register reg) { test_w(reg, op); }
+ void test_w(Register dst, Register src) { test_w(dst, Operand(src)); }
+
+ void xor_(Register dst, int32_t imm32);
+ void xor_(Register dst, Register src) { xor_(dst, Operand(src)); }
+ void xor_(Register dst, Operand src);
+ void xor_(Operand dst, Register src);
+ void xor_(Register dst, const Immediate& imm) { xor_(Operand(dst), imm); }
+ void xor_(Operand dst, const Immediate& x);
+
+ // Bit operations.
+ void bswap(Register dst);
+ void bt(Operand dst, Register src);
+ void bts(Register dst, Register src) { bts(Operand(dst), src); }
+ void bts(Operand dst, Register src);
+ void bsr(Register dst, Register src) { bsr(dst, Operand(src)); }
+ void bsr(Register dst, Operand src);
+ void bsf(Register dst, Register src) { bsf(dst, Operand(src)); }
+ void bsf(Register dst, Operand src);
+
+ // Miscellaneous
+ void hlt();
+ void int3();
+ void nop();
+ void ret(int imm16);
+ void ud2();
+
+ // Label operations & relative jumps (PPUM Appendix D)
+ //
+ // Takes a branch opcode (cc) and a label (L) and generates
+ // either a backward branch or a forward branch and links it
+ // to the label fixup chain. Usage:
+ //
+ // Label L; // unbound label
+ // j(cc, &L); // forward branch to unbound label
+ // bind(&L); // bind label to the current pc
+ // j(cc, &L); // backward branch to bound label
+ // bind(&L); // illegal: a label may be bound only once
+ //
+ // Note: The same Label can be used for forward and backward branches
+ // but it may be bound only once.
+
+ void bind(Label* L); // binds an unbound label L to the current code position
+
+ // Calls
+ void call(Label* L);
+ void call(Address entry, RelocInfo::Mode rmode);
+ void call(Register reg) { call(Operand(reg)); }
+ void call(Operand adr);
+ void call(Handle<Code> code, RelocInfo::Mode rmode);
+ void wasm_call(Address address, RelocInfo::Mode rmode);
+
+ // Jumps
+ // unconditional jump to L
+ void jmp(Label* L, Label::Distance distance = Label::kFar);
+ void jmp(Address entry, RelocInfo::Mode rmode);
+ void jmp(Register reg) { jmp(Operand(reg)); }
+ void jmp(Operand adr);
+ void jmp(Handle<Code> code, RelocInfo::Mode rmode);
+ // Unconditional jump relative to the current address. Low-level routine,
+ // use with caution!
+ void jmp_rel(int offset);
+
+ // Conditional jumps
+ void j(Condition cc, Label* L, Label::Distance distance = Label::kFar);
+ void j(Condition cc, byte* entry, RelocInfo::Mode rmode);
+ void j(Condition cc, Handle<Code> code,
+ RelocInfo::Mode rmode = RelocInfo::CODE_TARGET);
+
+ // Floating-point operations
+ void fld(int i);
+ void fstp(int i);
+
+ void fld1();
+ void fldz();
+ void fldpi();
+ void fldln2();
+
+ void fld_s(Operand adr);
+ void fld_d(Operand adr);
+
+ void fstp_s(Operand adr);
+ void fst_s(Operand adr);
+ void fstp_d(Operand adr);
+ void fst_d(Operand adr);
+
+ void fild_s(Operand adr);
+ void fild_d(Operand adr);
+
+ void fist_s(Operand adr);
+
+ void fistp_s(Operand adr);
+ void fistp_d(Operand adr);
+
+ // The fisttp instructions require SSE3.
+ void fisttp_s(Operand adr);
+ void fisttp_d(Operand adr);
+
+ void fabs();
+ void fchs();
+ void fcos();
+ void fsin();
+ void fptan();
+ void fyl2x();
+ void f2xm1();
+ void fscale();
+ void fninit();
+
+ void fadd(int i);
+ void fadd_i(int i);
+ void fsub(int i);
+ void fsub_i(int i);
+ void fmul(int i);
+ void fmul_i(int i);
+ void fdiv(int i);
+ void fdiv_i(int i);
+
+ void fisub_s(Operand adr);
+
+ void faddp(int i = 1);
+ void fsubp(int i = 1);
+ void fsubrp(int i = 1);
+ void fmulp(int i = 1);
+ void fdivp(int i = 1);
+ void fprem();
+ void fprem1();
+
+ void fxch(int i = 1);
+ void fincstp();
+ void ffree(int i = 0);
+
+ void ftst();
+ void fucomp(int i);
+ void fucompp();
+ void fucomi(int i);
+ void fucomip();
+ void fcompp();
+ void fnstsw_ax();
+ void fwait();
+ void fnclex();
+
+ void frndint();
+
+ void sahf();
+ void setcc(Condition cc, Register reg);
+
+ void cpuid();
+
+ // SSE instructions
+ void addss(XMMRegister dst, XMMRegister src) { addss(dst, Operand(src)); }
+ void addss(XMMRegister dst, Operand src);
+ void subss(XMMRegister dst, XMMRegister src) { subss(dst, Operand(src)); }
+ void subss(XMMRegister dst, Operand src);
+ void mulss(XMMRegister dst, XMMRegister src) { mulss(dst, Operand(src)); }
+ void mulss(XMMRegister dst, Operand src);
+ void divss(XMMRegister dst, XMMRegister src) { divss(dst, Operand(src)); }
+ void divss(XMMRegister dst, Operand src);
+ void sqrtss(XMMRegister dst, XMMRegister src) { sqrtss(dst, Operand(src)); }
+ void sqrtss(XMMRegister dst, Operand src);
+
+ void ucomiss(XMMRegister dst, XMMRegister src) { ucomiss(dst, Operand(src)); }
+ void ucomiss(XMMRegister dst, Operand src);
+ void movaps(XMMRegister dst, XMMRegister src) { movaps(dst, Operand(src)); }
+ void movaps(XMMRegister dst, Operand src);
+ void movups(XMMRegister dst, XMMRegister src) { movups(dst, Operand(src)); }
+ void movups(XMMRegister dst, Operand src);
+ void movups(Operand dst, XMMRegister src);
+ void shufps(XMMRegister dst, XMMRegister src, byte imm8);
+ void shufpd(XMMRegister dst, XMMRegister src, byte imm8);
+
+ void maxss(XMMRegister dst, XMMRegister src) { maxss(dst, Operand(src)); }
+ void maxss(XMMRegister dst, Operand src);
+ void minss(XMMRegister dst, XMMRegister src) { minss(dst, Operand(src)); }
+ void minss(XMMRegister dst, Operand src);
+
+ void rcpps(XMMRegister dst, Operand src);
+ void rcpps(XMMRegister dst, XMMRegister src) { rcpps(dst, Operand(src)); }
+ void sqrtps(XMMRegister dst, Operand src);
+ void sqrtps(XMMRegister dst, XMMRegister src) { sqrtps(dst, Operand(src)); }
+ void rsqrtps(XMMRegister dst, Operand src);
+ void rsqrtps(XMMRegister dst, XMMRegister src) { rsqrtps(dst, Operand(src)); }
+ void haddps(XMMRegister dst, Operand src);
+ void haddps(XMMRegister dst, XMMRegister src) { haddps(dst, Operand(src)); }
+ void sqrtpd(XMMRegister dst, Operand src) {
+ sse2_instr(dst, src, 0x66, 0x0F, 0x51);
+ }
+ void sqrtpd(XMMRegister dst, XMMRegister src) { sqrtpd(dst, Operand(src)); }
+
+ void cmpps(XMMRegister dst, Operand src, uint8_t cmp);
+ void cmpps(XMMRegister dst, XMMRegister src, uint8_t cmp) {
+ cmpps(dst, Operand(src), cmp);
+ }
+ void cmppd(XMMRegister dst, Operand src, uint8_t cmp);
+ void cmppd(XMMRegister dst, XMMRegister src, uint8_t cmp) {
+ cmppd(dst, Operand(src), cmp);
+ }
+
+// Packed floating-point comparison operations.
+#define PACKED_CMP_LIST(V) \
+ V(cmpeq, 0x0) \
+ V(cmplt, 0x1) \
+ V(cmple, 0x2) \
+ V(cmpunord, 0x3) \
+ V(cmpneq, 0x4)
+
+#define SSE_CMP_P(instr, imm8) \
+ void instr##ps(XMMRegister dst, XMMRegister src) { \
+ cmpps(dst, Operand(src), imm8); \
+ } \
+ void instr##ps(XMMRegister dst, Operand src) { cmpps(dst, src, imm8); } \
+ void instr##pd(XMMRegister dst, XMMRegister src) { \
+ cmppd(dst, Operand(src), imm8); \
+ } \
+ void instr##pd(XMMRegister dst, Operand src) { cmppd(dst, src, imm8); }
+
+ PACKED_CMP_LIST(SSE_CMP_P)
+#undef SSE_CMP_P
+
+ // SSE2 instructions
+ void cvttss2si(Register dst, Operand src);
+ void cvttss2si(Register dst, XMMRegister src) {
+ cvttss2si(dst, Operand(src));
+ }
+ void cvttsd2si(Register dst, Operand src);
+ void cvttsd2si(Register dst, XMMRegister src) {
+ cvttsd2si(dst, Operand(src));
+ }
+ void cvtsd2si(Register dst, XMMRegister src);
+
+ void cvtsi2ss(XMMRegister dst, Register src) { cvtsi2ss(dst, Operand(src)); }
+ void cvtsi2ss(XMMRegister dst, Operand src);
+ void cvtsi2sd(XMMRegister dst, Register src) { cvtsi2sd(dst, Operand(src)); }
+ void cvtsi2sd(XMMRegister dst, Operand src);
+ void cvtss2sd(XMMRegister dst, Operand src);
+ void cvtss2sd(XMMRegister dst, XMMRegister src) {
+ cvtss2sd(dst, Operand(src));
+ }
+ void cvtsd2ss(XMMRegister dst, Operand src);
+ void cvtsd2ss(XMMRegister dst, XMMRegister src) {
+ cvtsd2ss(dst, Operand(src));
+ }
+ void cvtdq2ps(XMMRegister dst, XMMRegister src) {
+ cvtdq2ps(dst, Operand(src));
+ }
+ void cvtdq2ps(XMMRegister dst, Operand src);
+ void cvttps2dq(XMMRegister dst, XMMRegister src) {
+ cvttps2dq(dst, Operand(src));
+ }
+ void cvttps2dq(XMMRegister dst, Operand src);
+
+ void addsd(XMMRegister dst, XMMRegister src) { addsd(dst, Operand(src)); }
+ void addsd(XMMRegister dst, Operand src);
+ void subsd(XMMRegister dst, XMMRegister src) { subsd(dst, Operand(src)); }
+ void subsd(XMMRegister dst, Operand src);
+ void mulsd(XMMRegister dst, XMMRegister src) { mulsd(dst, Operand(src)); }
+ void mulsd(XMMRegister dst, Operand src);
+ void divsd(XMMRegister dst, XMMRegister src) { divsd(dst, Operand(src)); }
+ void divsd(XMMRegister dst, Operand src);
+ void sqrtsd(XMMRegister dst, XMMRegister src) { sqrtsd(dst, Operand(src)); }
+ void sqrtsd(XMMRegister dst, Operand src);
+
+ void ucomisd(XMMRegister dst, XMMRegister src) { ucomisd(dst, Operand(src)); }
+ void ucomisd(XMMRegister dst, Operand src);
+
+ void roundss(XMMRegister dst, XMMRegister src, RoundingMode mode);
+ void roundsd(XMMRegister dst, XMMRegister src, RoundingMode mode);
+
+ void movapd(XMMRegister dst, XMMRegister src) { movapd(dst, Operand(src)); }
+ void movapd(XMMRegister dst, Operand src) {
+ sse2_instr(dst, src, 0x66, 0x0F, 0x28);
+ }
+ void movupd(XMMRegister dst, Operand src) {
+ sse2_instr(dst, src, 0x66, 0x0F, 0x10);
+ }
+
+ void movmskpd(Register dst, XMMRegister src);
+ void movmskps(Register dst, XMMRegister src);
+
+ void pmovmskb(Register dst, XMMRegister src);
+
+ void cmpltsd(XMMRegister dst, XMMRegister src);
+
+ void maxsd(XMMRegister dst, XMMRegister src) { maxsd(dst, Operand(src)); }
+ void maxsd(XMMRegister dst, Operand src);
+ void minsd(XMMRegister dst, XMMRegister src) { minsd(dst, Operand(src)); }
+ void minsd(XMMRegister dst, Operand src);
+
+ void movdqa(XMMRegister dst, Operand src);
+ void movdqa(Operand dst, XMMRegister src);
+ void movdqu(XMMRegister dst, Operand src);
+ void movdqu(Operand dst, XMMRegister src);
+ void movdq(bool aligned, XMMRegister dst, Operand src) {
+ if (aligned) {
+ movdqa(dst, src);
+ } else {
+ movdqu(dst, src);
+ }
+ }
+
+ void movd(XMMRegister dst, Register src) { movd(dst, Operand(src)); }
+ void movd(XMMRegister dst, Operand src);
+ void movd(Register dst, XMMRegister src) { movd(Operand(dst), src); }
+ void movd(Operand dst, XMMRegister src);
+ void movsd(XMMRegister dst, XMMRegister src) { movsd(dst, Operand(src)); }
+ void movsd(XMMRegister dst, Operand src);
+ void movsd(Operand dst, XMMRegister src);
+
+ void movss(XMMRegister dst, Operand src);
+ void movss(Operand dst, XMMRegister src);
+ void movss(XMMRegister dst, XMMRegister src) { movss(dst, Operand(src)); }
+ void extractps(Register dst, XMMRegister src, byte imm8);
+
+ void psllw(XMMRegister reg, uint8_t shift);
+ void pslld(XMMRegister reg, uint8_t shift);
+ void psrlw(XMMRegister reg, uint8_t shift);
+ void psrld(XMMRegister reg, uint8_t shift);
+ void psraw(XMMRegister reg, uint8_t shift);
+ void psrad(XMMRegister reg, uint8_t shift);
+ void psllq(XMMRegister reg, uint8_t shift);
+ void psrlq(XMMRegister reg, uint8_t shift);
+
+ void pshufhw(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
+ pshufhw(dst, Operand(src), shuffle);
+ }
+ void pshufhw(XMMRegister dst, Operand src, uint8_t shuffle);
+ void pshuflw(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
+ pshuflw(dst, Operand(src), shuffle);
+ }
+ void pshuflw(XMMRegister dst, Operand src, uint8_t shuffle);
+ void pshufd(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
+ pshufd(dst, Operand(src), shuffle);
+ }
+ void pshufd(XMMRegister dst, Operand src, uint8_t shuffle);
+
+ void pblendw(XMMRegister dst, XMMRegister src, uint8_t mask) {
+ pblendw(dst, Operand(src), mask);
+ }
+ void pblendw(XMMRegister dst, Operand src, uint8_t mask);
+
+ void palignr(XMMRegister dst, XMMRegister src, uint8_t mask) {
+ palignr(dst, Operand(src), mask);
+ }
+ void palignr(XMMRegister dst, Operand src, uint8_t mask);
+
+ void pextrb(Register dst, XMMRegister src, uint8_t offset) {
+ pextrb(Operand(dst), src, offset);
+ }
+ void pextrb(Operand dst, XMMRegister src, uint8_t offset);
+ // SSE3 instructions
+ void movddup(XMMRegister dst, Operand src);
+ void movddup(XMMRegister dst, XMMRegister src) { movddup(dst, Operand(src)); }
+
+ // Use SSE4_1 encoding for pextrw reg, xmm, imm8 for consistency
+ void pextrw(Register dst, XMMRegister src, uint8_t offset) {
+ pextrw(Operand(dst), src, offset);
+ }
+ void pextrw(Operand dst, XMMRegister src, uint8_t offset);
+ void pextrd(Register dst, XMMRegister src, uint8_t offset) {
+ pextrd(Operand(dst), src, offset);
+ }
+ void pextrd(Operand dst, XMMRegister src, uint8_t offset);
+
+ void insertps(XMMRegister dst, XMMRegister src, uint8_t offset) {
+ insertps(dst, Operand(src), offset);
+ }
+ void insertps(XMMRegister dst, Operand src, uint8_t offset);
+ void pinsrb(XMMRegister dst, Register src, uint8_t offset) {
+ pinsrb(dst, Operand(src), offset);
+ }
+ void pinsrb(XMMRegister dst, Operand src, uint8_t offset);
+ void pinsrw(XMMRegister dst, Register src, uint8_t offset) {
+ pinsrw(dst, Operand(src), offset);
+ }
+ void pinsrw(XMMRegister dst, Operand src, uint8_t offset);
+ void pinsrd(XMMRegister dst, Register src, uint8_t offset) {
+ pinsrd(dst, Operand(src), offset);
+ }
+ void pinsrd(XMMRegister dst, Operand src, uint8_t offset);
+
+ void roundps(XMMRegister dst, XMMRegister src, RoundingMode mode);
+ void roundpd(XMMRegister dst, XMMRegister src, RoundingMode mode);
+
+ // AVX instructions
+ void vfmadd132sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vfmadd132sd(dst, src1, Operand(src2));
+ }
+ void vfmadd213sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vfmadd213sd(dst, src1, Operand(src2));
+ }
+ void vfmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vfmadd231sd(dst, src1, Operand(src2));
+ }
+ void vfmadd132sd(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vfmasd(0x99, dst, src1, src2);
+ }
+ void vfmadd213sd(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vfmasd(0xa9, dst, src1, src2);
+ }
+ void vfmadd231sd(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vfmasd(0xb9, dst, src1, src2);
+ }
+ void vfmsub132sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vfmsub132sd(dst, src1, Operand(src2));
+ }
+ void vfmsub213sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vfmsub213sd(dst, src1, Operand(src2));
+ }
+ void vfmsub231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vfmsub231sd(dst, src1, Operand(src2));
+ }
+ void vfmsub132sd(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vfmasd(0x9b, dst, src1, src2);
+ }
+ void vfmsub213sd(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vfmasd(0xab, dst, src1, src2);
+ }
+ void vfmsub231sd(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vfmasd(0xbb, dst, src1, src2);
+ }
+ void vfnmadd132sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vfnmadd132sd(dst, src1, Operand(src2));
+ }
+ void vfnmadd213sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vfnmadd213sd(dst, src1, Operand(src2));
+ }
+ void vfnmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vfnmadd231sd(dst, src1, Operand(src2));
+ }
+ void vfnmadd132sd(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vfmasd(0x9d, dst, src1, src2);
+ }
+ void vfnmadd213sd(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vfmasd(0xad, dst, src1, src2);
+ }
+ void vfnmadd231sd(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vfmasd(0xbd, dst, src1, src2);
+ }
+ void vfnmsub132sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vfnmsub132sd(dst, src1, Operand(src2));
+ }
+ void vfnmsub213sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vfnmsub213sd(dst, src1, Operand(src2));
+ }
+ void vfnmsub231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vfnmsub231sd(dst, src1, Operand(src2));
+ }
+ void vfnmsub132sd(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vfmasd(0x9f, dst, src1, src2);
+ }
+ void vfnmsub213sd(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vfmasd(0xaf, dst, src1, src2);
+ }
+ void vfnmsub231sd(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vfmasd(0xbf, dst, src1, src2);
+ }
+ void vfmasd(byte op, XMMRegister dst, XMMRegister src1, Operand src2);
+
+ void vfmadd132ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vfmadd132ss(dst, src1, Operand(src2));
+ }
+ void vfmadd213ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vfmadd213ss(dst, src1, Operand(src2));
+ }
+ void vfmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vfmadd231ss(dst, src1, Operand(src2));
+ }
+ void vfmadd132ss(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vfmass(0x99, dst, src1, src2);
+ }
+ void vfmadd213ss(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vfmass(0xa9, dst, src1, src2);
+ }
+ void vfmadd231ss(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vfmass(0xb9, dst, src1, src2);
+ }
+ void vfmsub132ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vfmsub132ss(dst, src1, Operand(src2));
+ }
+ void vfmsub213ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vfmsub213ss(dst, src1, Operand(src2));
+ }
+ void vfmsub231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vfmsub231ss(dst, src1, Operand(src2));
+ }
+ void vfmsub132ss(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vfmass(0x9b, dst, src1, src2);
+ }
+ void vfmsub213ss(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vfmass(0xab, dst, src1, src2);
+ }
+ void vfmsub231ss(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vfmass(0xbb, dst, src1, src2);
+ }
+ void vfnmadd132ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vfnmadd132ss(dst, src1, Operand(src2));
+ }
+ void vfnmadd213ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vfnmadd213ss(dst, src1, Operand(src2));
+ }
+ void vfnmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vfnmadd231ss(dst, src1, Operand(src2));
+ }
+ void vfnmadd132ss(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vfmass(0x9d, dst, src1, src2);
+ }
+ void vfnmadd213ss(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vfmass(0xad, dst, src1, src2);
+ }
+ void vfnmadd231ss(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vfmass(0xbd, dst, src1, src2);
+ }
+ void vfnmsub132ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vfnmsub132ss(dst, src1, Operand(src2));
+ }
+ void vfnmsub213ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vfnmsub213ss(dst, src1, Operand(src2));
+ }
+ void vfnmsub231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vfnmsub231ss(dst, src1, Operand(src2));
+ }
+ void vfnmsub132ss(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vfmass(0x9f, dst, src1, src2);
+ }
+ void vfnmsub213ss(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vfmass(0xaf, dst, src1, src2);
+ }
+ void vfnmsub231ss(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vfmass(0xbf, dst, src1, src2);
+ }
+ void vfmass(byte op, XMMRegister dst, XMMRegister src1, Operand src2);
+
+ void vaddsd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vaddsd(dst, src1, Operand(src2));
+ }
+ void vaddsd(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vsd(0x58, dst, src1, src2);
+ }
+ void vsubsd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vsubsd(dst, src1, Operand(src2));
+ }
+ void vsubsd(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vsd(0x5c, dst, src1, src2);
+ }
+ void vmulsd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vmulsd(dst, src1, Operand(src2));
+ }
+ void vmulsd(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vsd(0x59, dst, src1, src2);
+ }
+ void vdivsd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vdivsd(dst, src1, Operand(src2));
+ }
+ void vdivsd(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vsd(0x5e, dst, src1, src2);
+ }
+ void vmaxsd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vmaxsd(dst, src1, Operand(src2));
+ }
+ void vmaxsd(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vsd(0x5f, dst, src1, src2);
+ }
+ void vminsd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vminsd(dst, src1, Operand(src2));
+ }
+ void vminsd(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vsd(0x5d, dst, src1, src2);
+ }
+ void vsqrtsd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vsqrtsd(dst, src1, Operand(src2));
+ }
+ void vsqrtsd(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vsd(0x51, dst, src1, src2);
+ }
+ void vsd(byte op, XMMRegister dst, XMMRegister src1, Operand src2);
+
+ void vaddss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vaddss(dst, src1, Operand(src2));
+ }
+ void vaddss(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vss(0x58, dst, src1, src2);
+ }
+ void vsubss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vsubss(dst, src1, Operand(src2));
+ }
+ void vsubss(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vss(0x5c, dst, src1, src2);
+ }
+ void vmulss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vmulss(dst, src1, Operand(src2));
+ }
+ void vmulss(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vss(0x59, dst, src1, src2);
+ }
+ void vdivss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vdivss(dst, src1, Operand(src2));
+ }
+ void vdivss(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vss(0x5e, dst, src1, src2);
+ }
+ void vmaxss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vmaxss(dst, src1, Operand(src2));
+ }
+ void vmaxss(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vss(0x5f, dst, src1, src2);
+ }
+ void vminss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vminss(dst, src1, Operand(src2));
+ }
+ void vminss(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vss(0x5d, dst, src1, src2);
+ }
+ void vsqrtss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vsqrtss(dst, src1, Operand(src2));
+ }
+ void vsqrtss(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vss(0x51, dst, src1, src2);
+ }
+ void vss(byte op, XMMRegister dst, XMMRegister src1, Operand src2);
+
+ void vrcpps(XMMRegister dst, XMMRegister src) { vrcpps(dst, Operand(src)); }
+ void vrcpps(XMMRegister dst, Operand src) {
+ vinstr(0x53, dst, xmm0, src, kNone, k0F, kWIG);
+ }
+ void vsqrtps(XMMRegister dst, XMMRegister src) { vsqrtps(dst, Operand(src)); }
+ void vsqrtps(XMMRegister dst, Operand src) {
+ vinstr(0x51, dst, xmm0, src, kNone, k0F, kWIG);
+ }
+ void vrsqrtps(XMMRegister dst, XMMRegister src) {
+ vrsqrtps(dst, Operand(src));
+ }
+ void vrsqrtps(XMMRegister dst, Operand src) {
+ vinstr(0x52, dst, xmm0, src, kNone, k0F, kWIG);
+ }
+ void vhaddps(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vhaddps(dst, src1, Operand(src2));
+ }
+ void vhaddps(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vinstr(0x7C, dst, src1, src2, kF2, k0F, kWIG);
+ }
+ void vsqrtpd(XMMRegister dst, XMMRegister src) { vsqrtpd(dst, Operand(src)); }
+ void vsqrtpd(XMMRegister dst, Operand src) {
+ vinstr(0x51, dst, xmm0, src, k66, k0F, kWIG);
+ }
+ void vmovaps(XMMRegister dst, XMMRegister src) { vmovaps(dst, Operand(src)); }
+ void vmovaps(XMMRegister dst, Operand src) { vps(0x28, dst, xmm0, src); }
+ void vmovapd(XMMRegister dst, XMMRegister src) { vmovapd(dst, Operand(src)); }
+ void vmovapd(XMMRegister dst, Operand src) { vpd(0x28, dst, xmm0, src); }
+ void vmovups(XMMRegister dst, XMMRegister src) { vmovups(dst, Operand(src)); }
+ void vmovups(XMMRegister dst, Operand src) { vps(0x10, dst, xmm0, src); }
+ void vmovupd(XMMRegister dst, Operand src) { vpd(0x10, dst, xmm0, src); }
+ void vshufps(XMMRegister dst, XMMRegister src1, XMMRegister src2, byte imm8) {
+ vshufps(dst, src1, Operand(src2), imm8);
+ }
+ void vshufps(XMMRegister dst, XMMRegister src1, Operand src2, byte imm8);
+ void vshufpd(XMMRegister dst, XMMRegister src1, XMMRegister src2, byte imm8) {
+ vshufpd(dst, src1, Operand(src2), imm8);
+ }
+ void vshufpd(XMMRegister dst, XMMRegister src1, Operand src2, byte imm8);
+
+ void vpsllw(XMMRegister dst, XMMRegister src, uint8_t imm8);
+ void vpslld(XMMRegister dst, XMMRegister src, uint8_t imm8);
+ void vpsllq(XMMRegister dst, XMMRegister src, uint8_t imm8);
+ void vpsrlw(XMMRegister dst, XMMRegister src, uint8_t imm8);
+ void vpsrld(XMMRegister dst, XMMRegister src, uint8_t imm8);
+ void vpsraw(XMMRegister dst, XMMRegister src, uint8_t imm8);
+ void vpsrad(XMMRegister dst, XMMRegister src, uint8_t imm8);
+ void vpsrlq(XMMRegister dst, XMMRegister src, uint8_t imm8);
+
+ void vpshufhw(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
+ vpshufhw(dst, Operand(src), shuffle);
+ }
+ void vpshufhw(XMMRegister dst, Operand src, uint8_t shuffle);
+ void vpshuflw(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
+ vpshuflw(dst, Operand(src), shuffle);
+ }
+ void vpshuflw(XMMRegister dst, Operand src, uint8_t shuffle);
+ void vpshufd(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
+ vpshufd(dst, Operand(src), shuffle);
+ }
+ void vpshufd(XMMRegister dst, Operand src, uint8_t shuffle);
+
+ void vpblendw(XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ uint8_t mask) {
+ vpblendw(dst, src1, Operand(src2), mask);
+ }
+ void vpblendw(XMMRegister dst, XMMRegister src1, Operand src2, uint8_t mask);
+
+ void vpalignr(XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ uint8_t mask) {
+ vpalignr(dst, src1, Operand(src2), mask);
+ }
+ void vpalignr(XMMRegister dst, XMMRegister src1, Operand src2, uint8_t mask);
+
+ void vpextrb(Register dst, XMMRegister src, uint8_t offset) {
+ vpextrb(Operand(dst), src, offset);
+ }
+ void vpextrb(Operand dst, XMMRegister src, uint8_t offset);
+ void vpextrw(Register dst, XMMRegister src, uint8_t offset) {
+ vpextrw(Operand(dst), src, offset);
+ }
+ void vpextrw(Operand dst, XMMRegister src, uint8_t offset);
+ void vpextrd(Register dst, XMMRegister src, uint8_t offset) {
+ vpextrd(Operand(dst), src, offset);
+ }
+ void vpextrd(Operand dst, XMMRegister src, uint8_t offset);
+
+ void vinsertps(XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ uint8_t offset) {
+ vinsertps(dst, src1, Operand(src2), offset);
+ }
+ void vinsertps(XMMRegister dst, XMMRegister src1, Operand src2,
+ uint8_t offset);
+ void vpinsrb(XMMRegister dst, XMMRegister src1, Register src2,
+ uint8_t offset) {
+ vpinsrb(dst, src1, Operand(src2), offset);
+ }
+ void vpinsrb(XMMRegister dst, XMMRegister src1, Operand src2, uint8_t offset);
+ void vpinsrw(XMMRegister dst, XMMRegister src1, Register src2,
+ uint8_t offset) {
+ vpinsrw(dst, src1, Operand(src2), offset);
+ }
+ void vpinsrw(XMMRegister dst, XMMRegister src1, Operand src2, uint8_t offset);
+ void vpinsrd(XMMRegister dst, XMMRegister src1, Register src2,
+ uint8_t offset) {
+ vpinsrd(dst, src1, Operand(src2), offset);
+ }
+ void vpinsrd(XMMRegister dst, XMMRegister src1, Operand src2, uint8_t offset);
+
+ void vroundps(XMMRegister dst, XMMRegister src, RoundingMode mode);
+ void vroundpd(XMMRegister dst, XMMRegister src, RoundingMode mode);
+
+ void vcvtdq2ps(XMMRegister dst, XMMRegister src) {
+ vcvtdq2ps(dst, Operand(src));
+ }
+ void vcvtdq2ps(XMMRegister dst, Operand src) {
+ vinstr(0x5B, dst, xmm0, src, kNone, k0F, kWIG);
+ }
+ void vcvttps2dq(XMMRegister dst, XMMRegister src) {
+ vcvttps2dq(dst, Operand(src));
+ }
+ void vcvttps2dq(XMMRegister dst, Operand src) {
+ vinstr(0x5B, dst, xmm0, src, kF3, k0F, kWIG);
+ }
+
+ void vmovddup(XMMRegister dst, Operand src) {
+ vinstr(0x12, dst, xmm0, src, kF2, k0F, kWIG);
+ }
+ void vmovddup(XMMRegister dst, XMMRegister src) {
+ vmovddup(dst, Operand(src));
+ }
+ void vbroadcastss(XMMRegister dst, Operand src) {
+ vinstr(0x18, dst, xmm0, src, k66, k0F38, kW0);
+ }
+ void vmovdqu(XMMRegister dst, Operand src) {
+ vinstr(0x6F, dst, xmm0, src, kF3, k0F, kWIG);
+ }
+ void vmovdqu(Operand dst, XMMRegister src) {
+ vinstr(0x7F, src, xmm0, dst, kF3, k0F, kWIG);
+ }
+ void vmovd(XMMRegister dst, Register src) { vmovd(dst, Operand(src)); }
+ void vmovd(XMMRegister dst, Operand src) {
+ vinstr(0x6E, dst, xmm0, src, k66, k0F, kWIG);
+ }
+ void vmovd(Register dst, XMMRegister src) { movd(Operand(dst), src); }
+ void vmovd(Operand dst, XMMRegister src) {
+ vinstr(0x7E, src, xmm0, dst, k66, k0F, kWIG);
+ }
+
+ void vmovmskps(Register dst, XMMRegister src);
+
+ void vpmovmskb(Register dst, XMMRegister src);
+
+ // BMI instruction
+ void andn(Register dst, Register src1, Register src2) {
+ andn(dst, src1, Operand(src2));
+ }
+ void andn(Register dst, Register src1, Operand src2) {
+ bmi1(0xf2, dst, src1, src2);
+ }
+ void bextr(Register dst, Register src1, Register src2) {
+ bextr(dst, Operand(src1), src2);
+ }
+ void bextr(Register dst, Operand src1, Register src2) {
+ bmi1(0xf7, dst, src2, src1);
+ }
+ void blsi(Register dst, Register src) { blsi(dst, Operand(src)); }
+ void blsi(Register dst, Operand src) { bmi1(0xf3, ebx, dst, src); }
+ void blsmsk(Register dst, Register src) { blsmsk(dst, Operand(src)); }
+ void blsmsk(Register dst, Operand src) { bmi1(0xf3, edx, dst, src); }
+ void blsr(Register dst, Register src) { blsr(dst, Operand(src)); }
+ void blsr(Register dst, Operand src) { bmi1(0xf3, ecx, dst, src); }
+ void tzcnt(Register dst, Register src) { tzcnt(dst, Operand(src)); }
+ void tzcnt(Register dst, Operand src);
+
+ void lzcnt(Register dst, Register src) { lzcnt(dst, Operand(src)); }
+ void lzcnt(Register dst, Operand src);
+
+ void popcnt(Register dst, Register src) { popcnt(dst, Operand(src)); }
+ void popcnt(Register dst, Operand src);
+
+ void bzhi(Register dst, Register src1, Register src2) {
+ bzhi(dst, Operand(src1), src2);
+ }
+ void bzhi(Register dst, Operand src1, Register src2) {
+ bmi2(kNone, 0xf5, dst, src2, src1);
+ }
+ void mulx(Register dst1, Register dst2, Register src) {
+ mulx(dst1, dst2, Operand(src));
+ }
+ void mulx(Register dst1, Register dst2, Operand src) {
+ bmi2(kF2, 0xf6, dst1, dst2, src);
+ }
+ void pdep(Register dst, Register src1, Register src2) {
+ pdep(dst, src1, Operand(src2));
+ }
+ void pdep(Register dst, Register src1, Operand src2) {
+ bmi2(kF2, 0xf5, dst, src1, src2);
+ }
+ void pext(Register dst, Register src1, Register src2) {
+ pext(dst, src1, Operand(src2));
+ }
+ void pext(Register dst, Register src1, Operand src2) {
+ bmi2(kF3, 0xf5, dst, src1, src2);
+ }
+ void sarx(Register dst, Register src1, Register src2) {
+ sarx(dst, Operand(src1), src2);
+ }
+ void sarx(Register dst, Operand src1, Register src2) {
+ bmi2(kF3, 0xf7, dst, src2, src1);
+ }
+ void shlx(Register dst, Register src1, Register src2) {
+ shlx(dst, Operand(src1), src2);
+ }
+ void shlx(Register dst, Operand src1, Register src2) {
+ bmi2(k66, 0xf7, dst, src2, src1);
+ }
+ void shrx(Register dst, Register src1, Register src2) {
+ shrx(dst, Operand(src1), src2);
+ }
+ void shrx(Register dst, Operand src1, Register src2) {
+ bmi2(kF2, 0xf7, dst, src2, src1);
+ }
+ void rorx(Register dst, Register src, byte imm8) {
+ rorx(dst, Operand(src), imm8);
+ }
+ void rorx(Register dst, Operand src, byte imm8);
+
+ // Implementation of packed single-precision floating-point SSE instructions.
+ void ps(byte op, XMMRegister dst, Operand src);
+ // Implementation of packed double-precision floating-point SSE instructions.
+ void pd(byte op, XMMRegister dst, Operand src);
+
+#define PACKED_OP_LIST(V) \
+ V(and, 0x54) \
+ V(andn, 0x55) \
+ V(or, 0x56) \
+ V(xor, 0x57) \
+ V(add, 0x58) \
+ V(mul, 0x59) \
+ V(sub, 0x5c) \
+ V(min, 0x5d) \
+ V(div, 0x5e) \
+ V(max, 0x5f)
+
+#define SSE_PACKED_OP_DECLARE(name, opcode) \
+ void name##ps(XMMRegister dst, XMMRegister src) { \
+ ps(opcode, dst, Operand(src)); \
+ } \
+ void name##ps(XMMRegister dst, Operand src) { ps(opcode, dst, src); } \
+ void name##pd(XMMRegister dst, XMMRegister src) { \
+ pd(opcode, dst, Operand(src)); \
+ } \
+ void name##pd(XMMRegister dst, Operand src) { pd(opcode, dst, src); }
+
+ PACKED_OP_LIST(SSE_PACKED_OP_DECLARE)
+#undef SSE_PACKED_OP_DECLARE
+
+#define AVX_PACKED_OP_DECLARE(name, opcode) \
+ void v##name##ps(XMMRegister dst, XMMRegister src1, XMMRegister src2) { \
+ vps(opcode, dst, src1, Operand(src2)); \
+ } \
+ void v##name##ps(XMMRegister dst, XMMRegister src1, Operand src2) { \
+ vps(opcode, dst, src1, src2); \
+ } \
+ void v##name##pd(XMMRegister dst, XMMRegister src1, XMMRegister src2) { \
+ vpd(opcode, dst, src1, Operand(src2)); \
+ } \
+ void v##name##pd(XMMRegister dst, XMMRegister src1, Operand src2) { \
+ vpd(opcode, dst, src1, src2); \
+ }
+
+ PACKED_OP_LIST(AVX_PACKED_OP_DECLARE)
+#undef AVX_PACKED_OP_DECLARE
+#undef PACKED_OP_LIST
+
+ void vps(byte op, XMMRegister dst, XMMRegister src1, Operand src2);
+ void vpd(byte op, XMMRegister dst, XMMRegister src1, Operand src2);
+
+ void vcmpps(XMMRegister dst, XMMRegister src1, Operand src2, uint8_t cmp);
+ void vcmppd(XMMRegister dst, XMMRegister src1, Operand src2, uint8_t cmp);
+
+#define AVX_CMP_P(instr, imm8) \
+ void v##instr##ps(XMMRegister dst, XMMRegister src1, XMMRegister src2) { \
+ vcmpps(dst, src1, Operand(src2), imm8); \
+ } \
+ void v##instr##ps(XMMRegister dst, XMMRegister src1, Operand src2) { \
+ vcmpps(dst, src1, src2, imm8); \
+ } \
+ void v##instr##pd(XMMRegister dst, XMMRegister src1, XMMRegister src2) { \
+ vcmppd(dst, src1, Operand(src2), imm8); \
+ } \
+ void v##instr##pd(XMMRegister dst, XMMRegister src1, Operand src2) { \
+ vcmppd(dst, src1, src2, imm8); \
+ }
+
+ PACKED_CMP_LIST(AVX_CMP_P)
+#undef AVX_CMP_P
+#undef PACKED_CMP_LIST
+
+// Other SSE and AVX instructions
+#define DECLARE_SSE2_INSTRUCTION(instruction, prefix, escape, opcode) \
+ void instruction(XMMRegister dst, XMMRegister src) { \
+ instruction(dst, Operand(src)); \
+ } \
+ void instruction(XMMRegister dst, Operand src) { \
+ sse2_instr(dst, src, 0x##prefix, 0x##escape, 0x##opcode); \
+ }
+
+ SSE2_INSTRUCTION_LIST(DECLARE_SSE2_INSTRUCTION)
+#undef DECLARE_SSE2_INSTRUCTION
+
+#define DECLARE_SSE2_AVX_INSTRUCTION(instruction, prefix, escape, opcode) \
+ void v##instruction(XMMRegister dst, XMMRegister src1, XMMRegister src2) { \
+ v##instruction(dst, src1, Operand(src2)); \
+ } \
+ void v##instruction(XMMRegister dst, XMMRegister src1, Operand src2) { \
+ vinstr(0x##opcode, dst, src1, src2, k##prefix, k##escape, kW0); \
+ }
+
+ SSE2_INSTRUCTION_LIST(DECLARE_SSE2_AVX_INSTRUCTION)
+#undef DECLARE_SSE2_AVX_INSTRUCTION
+
+#define DECLARE_SSSE3_INSTRUCTION(instruction, prefix, escape1, escape2, \
+ opcode) \
+ void instruction(XMMRegister dst, XMMRegister src) { \
+ instruction(dst, Operand(src)); \
+ } \
+ void instruction(XMMRegister dst, Operand src) { \
+ ssse3_instr(dst, src, 0x##prefix, 0x##escape1, 0x##escape2, 0x##opcode); \
+ }
+
+ SSSE3_INSTRUCTION_LIST(DECLARE_SSSE3_INSTRUCTION)
+ SSSE3_UNOP_INSTRUCTION_LIST(DECLARE_SSSE3_INSTRUCTION)
+#undef DECLARE_SSSE3_INSTRUCTION
+
+#define DECLARE_SSE4_INSTRUCTION(instruction, prefix, escape1, escape2, \
+ opcode) \
+ void instruction(XMMRegister dst, XMMRegister src) { \
+ instruction(dst, Operand(src)); \
+ } \
+ void instruction(XMMRegister dst, Operand src) { \
+ sse4_instr(dst, src, 0x##prefix, 0x##escape1, 0x##escape2, 0x##opcode); \
+ }
+
+ SSE4_INSTRUCTION_LIST(DECLARE_SSE4_INSTRUCTION)
+ SSE4_RM_INSTRUCTION_LIST(DECLARE_SSE4_INSTRUCTION)
+#undef DECLARE_SSE4_INSTRUCTION
+
+#define DECLARE_SSE34_AVX_INSTRUCTION(instruction, prefix, escape1, escape2, \
+ opcode) \
+ void v##instruction(XMMRegister dst, XMMRegister src1, XMMRegister src2) { \
+ v##instruction(dst, src1, Operand(src2)); \
+ } \
+ void v##instruction(XMMRegister dst, XMMRegister src1, Operand src2) { \
+ vinstr(0x##opcode, dst, src1, src2, k##prefix, k##escape1##escape2, kW0); \
+ }
+
+ SSSE3_INSTRUCTION_LIST(DECLARE_SSE34_AVX_INSTRUCTION)
+ SSE4_INSTRUCTION_LIST(DECLARE_SSE34_AVX_INSTRUCTION)
+#undef DECLARE_SSE34_AVX_INSTRUCTION
+
+#define DECLARE_SSE4_AVX_RM_INSTRUCTION(instruction, prefix, escape1, escape2, \
+ opcode) \
+ void v##instruction(XMMRegister dst, XMMRegister src) { \
+ v##instruction(dst, Operand(src)); \
+ } \
+ void v##instruction(XMMRegister dst, Operand src) { \
+ vinstr(0x##opcode, dst, xmm0, src, k##prefix, k##escape1##escape2, kW0); \
+ }
+
+ SSSE3_UNOP_INSTRUCTION_LIST(DECLARE_SSE4_AVX_RM_INSTRUCTION)
+ SSE4_RM_INSTRUCTION_LIST(DECLARE_SSE4_AVX_RM_INSTRUCTION)
+#undef DECLARE_SSE4_AVX_RM_INSTRUCTION
+
+ // Prefetch src position into cache level.
+ // Level 1, 2 or 3 specifies CPU cache level. Level 0 specifies a
+ // non-temporal
+ void prefetch(Operand src, int level);
+ // TODO(lrn): Need SFENCE for movnt?
+
+ // Check the code size generated from label to here.
+ int SizeOfCodeGeneratedSince(Label* label) {
+ return pc_offset() - label->pos();
+ }
+
+ // Record a deoptimization reason that can be used by a log or cpu profiler.
+ // Use --trace-deopt to enable.
+ void RecordDeoptReason(DeoptimizeReason reason, SourcePosition position,
+ int id);
+
+ // Writes a single byte or word of data in the code stream. Used for
+ // inline tables, e.g., jump-tables.
+ void db(uint8_t data);
+ void dd(uint32_t data);
+ void dq(uint64_t data);
+ void dp(uintptr_t data) { dd(data); }
+ void dd(Label* label);
+
+ // Check if there is less than kGap bytes available in the buffer.
+ // If this is the case, we need to grow the buffer before emitting
+ // an instruction or relocation information.
+ inline bool buffer_overflow() const {
+ return pc_ >= reloc_info_writer.pos() - kGap;
+ }
+
+ // Get the number of bytes available in the buffer.
+ inline int available_space() const { return reloc_info_writer.pos() - pc_; }
+
+ static bool IsNop(Address addr);
+
+ int relocation_writer_size() {
+ return (buffer_start_ + buffer_->size()) - reloc_info_writer.pos();
+ }
+
+ // Avoid overflows for displacements etc.
+ static constexpr int kMaximalBufferSize = 512 * MB;
+
+ byte byte_at(int pos) { return buffer_start_[pos]; }
+ void set_byte_at(int pos, byte value) { buffer_start_[pos] = value; }
+
+ protected:
+ void emit_sse_operand(XMMRegister reg, Operand adr);
+ void emit_sse_operand(XMMRegister dst, XMMRegister src);
+ void emit_sse_operand(Register dst, XMMRegister src);
+ void emit_sse_operand(XMMRegister dst, Register src);
+
+ Address addr_at(int pos) {
+ return reinterpret_cast<Address>(buffer_start_ + pos);
+ }
+
+ private:
+ uint32_t long_at(int pos) {
+ return ReadUnalignedValue<uint32_t>(addr_at(pos));
+ }
+ void long_at_put(int pos, uint32_t x) {
+ WriteUnalignedValue(addr_at(pos), x);
+ }
+
+ // code emission
+ void GrowBuffer();
+ inline void emit(uint32_t x);
+ inline void emit(Handle<HeapObject> handle);
+ inline void emit(uint32_t x, RelocInfo::Mode rmode);
+ inline void emit(Handle<Code> code, RelocInfo::Mode rmode);
+ inline void emit(const Immediate& x);
+ inline void emit_b(Immediate x);
+ inline void emit_w(const Immediate& x);
+ inline void emit_q(uint64_t x);
+
+ // Emit the code-object-relative offset of the label's position
+ inline void emit_code_relative_offset(Label* label);
+
+ // instruction generation
+ void emit_arith_b(int op1, int op2, Register dst, int imm8);
+
+ // Emit a basic arithmetic instruction (i.e. first byte of the family is 0x81)
+ // with a given destination expression and an immediate operand. It attempts
+ // to use the shortest encoding possible.
+ // sel specifies the /n in the modrm byte (see the Intel PRM).
+ void emit_arith(int sel, Operand dst, const Immediate& x);
+
+ void emit_operand(int code, Operand adr);
+ void emit_operand(Register reg, Operand adr);
+ void emit_operand(XMMRegister reg, Operand adr);
+
+ void emit_label(Label* label);
+
+ void emit_farith(int b1, int b2, int i);
+
+ // Emit vex prefix
+ enum SIMDPrefix { kNone = 0x0, k66 = 0x1, kF3 = 0x2, kF2 = 0x3 };
+ enum VectorLength { kL128 = 0x0, kL256 = 0x4, kLIG = kL128, kLZ = kL128 };
+ enum VexW { kW0 = 0x0, kW1 = 0x80, kWIG = kW0 };
+ enum LeadingOpcode { k0F = 0x1, k0F38 = 0x2, k0F3A = 0x3 };
+ inline void emit_vex_prefix(XMMRegister v, VectorLength l, SIMDPrefix pp,
+ LeadingOpcode m, VexW w);
+ inline void emit_vex_prefix(Register v, VectorLength l, SIMDPrefix pp,
+ LeadingOpcode m, VexW w);
+
+ // labels
+ void print(const Label* L);
+ void bind_to(Label* L, int pos);
+
+ // displacements
+ inline Displacement disp_at(Label* L);
+ inline void disp_at_put(Label* L, Displacement disp);
+ inline void emit_disp(Label* L, Displacement::Type type);
+ inline void emit_near_disp(Label* L);
+
+ void sse2_instr(XMMRegister dst, Operand src, byte prefix, byte escape,
+ byte opcode);
+ void ssse3_instr(XMMRegister dst, Operand src, byte prefix, byte escape1,
+ byte escape2, byte opcode);
+ void sse4_instr(XMMRegister dst, Operand src, byte prefix, byte escape1,
+ byte escape2, byte opcode);
+ void vinstr(byte op, XMMRegister dst, XMMRegister src1, Operand src2,
+ SIMDPrefix pp, LeadingOpcode m, VexW w);
+ // Most BMI instructions are similar.
+ void bmi1(byte op, Register reg, Register vreg, Operand rm);
+ void bmi2(SIMDPrefix pp, byte op, Register reg, Register vreg, Operand rm);
+
+ // record reloc info for current pc_
+ void RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data = 0);
+
+ // record the position of jmp/jcc instruction
+ void record_farjmp_position(Label* L, int pos);
+
+ bool is_optimizable_farjmp(int idx);
+
+ void AllocateAndInstallRequestedHeapObjects(Isolate* isolate);
+
+ int WriteCodeComments();
+
+ friend class EnsureSpace;
+
+ // Internal reference positions, required for (potential) patching in
+ // GrowBuffer(); contains only those internal references whose labels
+ // are already bound.
+ std::deque<int> internal_reference_positions_;
+
+ // code generation
+ RelocInfoWriter reloc_info_writer;
+
+ // Variables for this instance of assembler
+ int farjmp_num_ = 0;
+ std::deque<int> farjmp_positions_;
+ std::map<Label*, std::vector<int>> label_farjmp_maps_;
+};
+
+// Helper class that ensures that there is enough space for generating
+// instructions and relocation information. The constructor makes
+// sure that there is enough space and (in debug mode) the destructor
+// checks that we did not generate too much.
+class EnsureSpace {
+ public:
+ explicit EnsureSpace(Assembler* assembler) : assembler_(assembler) {
+ if (assembler_->buffer_overflow()) assembler_->GrowBuffer();
+#ifdef DEBUG
+ space_before_ = assembler_->available_space();
+#endif
+ }
+
+#ifdef DEBUG
+ ~EnsureSpace() {
+ int bytes_generated = space_before_ - assembler_->available_space();
+ DCHECK(bytes_generated < assembler_->kGap);
+ }
+#endif
+
+ private:
+ Assembler* assembler_;
+#ifdef DEBUG
+ int space_before_;
+#endif
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_IA32_ASSEMBLER_IA32_H_
diff --git a/src/codegen/ia32/constants-ia32.h b/src/codegen/ia32/constants-ia32.h
new file mode 100644
index 0000000..af3bd09
--- /dev/null
+++ b/src/codegen/ia32/constants-ia32.h
@@ -0,0 +1,23 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_IA32_CONSTANTS_IA32_H_
+#define V8_CODEGEN_IA32_CONSTANTS_IA32_H_
+
+#include "src/common/globals.h"
+
+namespace v8 {
+namespace internal {
+// Actual value of root register is offset from the root array's start
+// to take advantage of negative displacement values.
+// For x86, this value is provided for uniformity with other platforms, although
+// currently no root register is present.
+constexpr int kRootRegisterBias = 0;
+
+// TODO(sigurds): Change this value once we use relative jumps.
+constexpr size_t kMaxPCRelativeCodeRangeInMB = 0;
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_IA32_CONSTANTS_IA32_H_
diff --git a/src/codegen/ia32/cpu-ia32.cc b/src/codegen/ia32/cpu-ia32.cc
new file mode 100644
index 0000000..5e6d8a6
--- /dev/null
+++ b/src/codegen/ia32/cpu-ia32.cc
@@ -0,0 +1,42 @@
+// Copyright 2011 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// CPU specific code for ia32 independent of OS goes here.
+
+#ifdef __GNUC__
+#include "src/third_party/valgrind/valgrind.h"
+#endif
+
+#if V8_TARGET_ARCH_IA32
+
+#include "src/codegen/cpu-features.h"
+
+namespace v8 {
+namespace internal {
+
+void CpuFeatures::FlushICache(void* start, size_t size) {
+ // No need to flush the instruction cache on Intel. On Intel instruction
+ // cache flushing is only necessary when multiple cores running the same
+ // code simultaneously. V8 (and JavaScript) is single threaded and when code
+ // is patched on an intel CPU the core performing the patching will have its
+ // own instruction cache updated automatically.
+
+ // If flushing of the instruction cache becomes necessary Windows has the
+ // API function FlushInstructionCache.
+
+ // By default, valgrind only checks the stack for writes that might need to
+ // invalidate already cached translated code. This leads to random
+ // instability when code patches or moves are sometimes unnoticed. One
+ // solution is to run valgrind with --smc-check=all, but this comes at a big
+ // performance cost. We can notify valgrind to invalidate its cache.
+#ifdef VALGRIND_DISCARD_TRANSLATIONS
+ unsigned res = VALGRIND_DISCARD_TRANSLATIONS(start, size);
+ USE(res);
+#endif
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_IA32
diff --git a/src/codegen/ia32/interface-descriptors-ia32.cc b/src/codegen/ia32/interface-descriptors-ia32.cc
new file mode 100644
index 0000000..ee9c3aa
--- /dev/null
+++ b/src/codegen/ia32/interface-descriptors-ia32.cc
@@ -0,0 +1,300 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#if V8_TARGET_ARCH_IA32
+
+#include "src/codegen/interface-descriptors.h"
+
+#include "src/execution/frames.h"
+
+namespace v8 {
+namespace internal {
+
+const Register CallInterfaceDescriptor::ContextRegister() { return esi; }
+
+void CallInterfaceDescriptor::DefaultInitializePlatformSpecific(
+ CallInterfaceDescriptorData* data, int register_parameter_count) {
+ constexpr Register default_stub_registers[] = {eax, ecx, edx, edi};
+ STATIC_ASSERT(arraysize(default_stub_registers) == kMaxBuiltinRegisterParams);
+ CHECK_LE(static_cast<size_t>(register_parameter_count),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(register_parameter_count,
+ default_stub_registers);
+}
+
+void RecordWriteDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ static const Register default_stub_registers[] = {ecx, edx, esi, edi,
+ kReturnRegister0};
+
+ data->RestrictAllocatableRegisters(default_stub_registers,
+ arraysize(default_stub_registers));
+
+ CHECK_LE(static_cast<size_t>(kParameterCount),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(kParameterCount, default_stub_registers);
+}
+
+void EphemeronKeyBarrierDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ static const Register default_stub_registers[] = {ecx, edx, esi, edi,
+ kReturnRegister0};
+
+ data->RestrictAllocatableRegisters(default_stub_registers,
+ arraysize(default_stub_registers));
+
+ CHECK_LE(static_cast<size_t>(kParameterCount),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(kParameterCount, default_stub_registers);
+}
+
+const Register LoadDescriptor::ReceiverRegister() { return edx; }
+const Register LoadDescriptor::NameRegister() { return ecx; }
+const Register LoadDescriptor::SlotRegister() { return eax; }
+
+const Register LoadWithVectorDescriptor::VectorRegister() { return no_reg; }
+
+const Register
+LoadWithReceiverAndVectorDescriptor::LookupStartObjectRegister() {
+ return edi;
+}
+
+const Register StoreDescriptor::ReceiverRegister() { return edx; }
+const Register StoreDescriptor::NameRegister() { return ecx; }
+const Register StoreDescriptor::ValueRegister() { return no_reg; }
+const Register StoreDescriptor::SlotRegister() { return no_reg; }
+
+const Register StoreWithVectorDescriptor::VectorRegister() { return no_reg; }
+
+const Register StoreTransitionDescriptor::SlotRegister() { return no_reg; }
+const Register StoreTransitionDescriptor::VectorRegister() { return no_reg; }
+const Register StoreTransitionDescriptor::MapRegister() { return edi; }
+
+const Register ApiGetterDescriptor::HolderRegister() { return ecx; }
+const Register ApiGetterDescriptor::CallbackRegister() { return eax; }
+
+const Register GrowArrayElementsDescriptor::ObjectRegister() { return eax; }
+const Register GrowArrayElementsDescriptor::KeyRegister() { return ecx; }
+
+// static
+const Register TypeConversionDescriptor::ArgumentRegister() { return eax; }
+
+void TypeofDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {ecx};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallTrampolineDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // eax : number of arguments
+ // edi : the target to call
+ Register registers[] = {edi, eax};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // eax : number of arguments (on the stack, not including receiver)
+ // edi : the target to call
+ // ecx : arguments list length (untagged)
+ // On the stack : arguments list (FixedArray)
+ Register registers[] = {edi, eax, ecx};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallForwardVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // eax : number of arguments
+ // ecx : start index (to support rest parameters)
+ // edi : the target to call
+ Register registers[] = {edi, eax, ecx};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallFunctionTemplateDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // edx : function template info
+ // ecx : number of arguments (on the stack, not including receiver)
+ Register registers[] = {edx, ecx};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallWithSpreadDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // eax : number of arguments (on the stack, not including receiver)
+ // edi : the target to call
+ // ecx : the object to spread
+ Register registers[] = {edi, eax, ecx};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallWithArrayLikeDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // edi : the target to call
+ // edx : the arguments list
+ Register registers[] = {edi, edx};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // eax : number of arguments (on the stack, not including receiver)
+ // edi : the target to call
+ // edx : the new target
+ // ecx : arguments list length (untagged)
+ // On the stack : arguments list (FixedArray)
+ Register registers[] = {edi, edx, eax, ecx};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructForwardVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // eax : number of arguments
+ // edx : the new target
+ // ecx : start index (to support rest parameters)
+ // edi : the target to call
+ Register registers[] = {edi, edx, eax, ecx};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructWithSpreadDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // eax : number of arguments (on the stack, not including receiver)
+ // edi : the target to call
+ // edx : the new target
+ // ecx : the object to spread
+ Register registers[] = {edi, edx, eax, ecx};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructWithArrayLikeDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // edi : the target to call
+ // edx : the new target
+ // ecx : the arguments list
+ Register registers[] = {edi, edx, ecx};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructStubDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // eax : number of arguments
+ // edx : the new target
+ // edi : the target to call
+ // ecx : allocation site or undefined
+ // TODO(jgruber): Remove the unused allocation site parameter.
+ Register registers[] = {edi, edx, eax, ecx};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void AbortDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {edx};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CompareDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {edx, eax};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void BinaryOpDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {edx, eax};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ArgumentsAdaptorDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ edi, // JSFunction
+ edx, // the new target
+ eax, // actual number of arguments
+ ecx, // expected number of arguments
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ApiCallbackDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ edx, // kApiFunctionAddress
+ ecx, // kArgc
+ eax, // kCallData
+ edi, // kHolder
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void InterpreterDispatchDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ kInterpreterAccumulatorRegister, kInterpreterBytecodeOffsetRegister,
+ kInterpreterBytecodeArrayRegister, kInterpreterDispatchTableRegister};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void InterpreterPushArgsThenCallDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ eax, // argument count (not including receiver)
+ ecx, // address of first argument
+ edi // the target callable to be call
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void InterpreterPushArgsThenConstructDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ eax, // argument count (not including receiver)
+ ecx, // address of first argument
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ResumeGeneratorDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ eax, // the value to pass to the generator
+ edx // the JSGeneratorObject to resume
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void FrameDropperTrampolineDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ eax, // loaded new FP
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void RunMicrotasksEntryDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ data->InitializePlatformSpecific(0, nullptr);
+}
+
+void WasmFloat32ToNumberDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // Work around using eax, whose register code is 0, and leads to the FP
+ // parameter being passed via xmm0, which is not allocatable on ia32.
+ Register registers[] = {ecx};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void WasmFloat64ToNumberDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // Work around using eax, whose register code is 0, and leads to the FP
+ // parameter being passed via xmm0, which is not allocatable on ia32.
+ Register registers[] = {ecx};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_IA32
diff --git a/src/codegen/ia32/macro-assembler-ia32.cc b/src/codegen/ia32/macro-assembler-ia32.cc
new file mode 100644
index 0000000..b615c59
--- /dev/null
+++ b/src/codegen/ia32/macro-assembler-ia32.cc
@@ -0,0 +1,2212 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#if V8_TARGET_ARCH_IA32
+
+#include "src/base/bits.h"
+#include "src/base/division-by-constant.h"
+#include "src/base/utils/random-number-generator.h"
+#include "src/codegen/callable.h"
+#include "src/codegen/code-factory.h"
+#include "src/codegen/external-reference-table.h"
+#include "src/codegen/ia32/assembler-ia32-inl.h"
+#include "src/codegen/macro-assembler.h"
+#include "src/debug/debug.h"
+#include "src/execution/frame-constants.h"
+#include "src/execution/frames-inl.h"
+#include "src/heap/memory-chunk.h"
+#include "src/init/bootstrapper.h"
+#include "src/logging/counters.h"
+#include "src/runtime/runtime.h"
+#include "src/snapshot/embedded/embedded-data.h"
+#include "src/snapshot/snapshot.h"
+
+// Satisfy cpplint check, but don't include platform-specific header. It is
+// included recursively via macro-assembler.h.
+#if 0
+#include "src/codegen/ia32/macro-assembler-ia32.h"
+#endif
+
+namespace v8 {
+namespace internal {
+
+Operand StackArgumentsAccessor::GetArgumentOperand(int index) const {
+ DCHECK_GE(index, 0);
+ // arg[0] = esp + kPCOnStackSize;
+ // arg[i] = arg[0] + i * kSystemPointerSize;
+ return Operand(esp, kPCOnStackSize + index * kSystemPointerSize);
+}
+
+// -------------------------------------------------------------------------
+// MacroAssembler implementation.
+
+void TurboAssembler::InitializeRootRegister() {
+ ExternalReference isolate_root = ExternalReference::isolate_root(isolate());
+ Move(kRootRegister, Immediate(isolate_root));
+}
+
+void TurboAssembler::LoadRoot(Register destination, RootIndex index) {
+ if (root_array_available()) {
+ mov(destination,
+ Operand(kRootRegister, RootRegisterOffsetForRootIndex(index)));
+ return;
+ }
+
+ if (RootsTable::IsImmortalImmovable(index)) {
+ Handle<Object> object = isolate()->root_handle(index);
+ if (object->IsSmi()) {
+ mov(destination, Immediate(Smi::cast(*object)));
+ return;
+ } else {
+ DCHECK(object->IsHeapObject());
+ mov(destination, Handle<HeapObject>::cast(object));
+ return;
+ }
+ }
+
+ ExternalReference isolate_root = ExternalReference::isolate_root(isolate());
+ lea(destination,
+ Operand(isolate_root.address(), RelocInfo::EXTERNAL_REFERENCE));
+ mov(destination, Operand(destination, RootRegisterOffsetForRootIndex(index)));
+}
+
+void TurboAssembler::CompareRoot(Register with, Register scratch,
+ RootIndex index) {
+ if (root_array_available()) {
+ CompareRoot(with, index);
+ } else {
+ ExternalReference isolate_root = ExternalReference::isolate_root(isolate());
+ lea(scratch,
+ Operand(isolate_root.address(), RelocInfo::EXTERNAL_REFERENCE));
+ cmp(with, Operand(scratch, RootRegisterOffsetForRootIndex(index)));
+ }
+}
+
+void TurboAssembler::CompareRoot(Register with, RootIndex index) {
+ if (root_array_available()) {
+ cmp(with, Operand(kRootRegister, RootRegisterOffsetForRootIndex(index)));
+ return;
+ }
+
+ DCHECK(RootsTable::IsImmortalImmovable(index));
+ Handle<Object> object = isolate()->root_handle(index);
+ if (object->IsHeapObject()) {
+ cmp(with, Handle<HeapObject>::cast(object));
+ } else {
+ cmp(with, Immediate(Smi::cast(*object)));
+ }
+}
+
+void MacroAssembler::PushRoot(RootIndex index) {
+ if (root_array_available()) {
+ DCHECK(RootsTable::IsImmortalImmovable(index));
+ push(Operand(kRootRegister, RootRegisterOffsetForRootIndex(index)));
+ return;
+ }
+
+ // TODO(v8:6666): Add a scratch register or remove all uses.
+ DCHECK(RootsTable::IsImmortalImmovable(index));
+ Handle<Object> object = isolate()->root_handle(index);
+ if (object->IsHeapObject()) {
+ Push(Handle<HeapObject>::cast(object));
+ } else {
+ Push(Smi::cast(*object));
+ }
+}
+
+void MacroAssembler::JumpIfIsInRange(Register value, unsigned lower_limit,
+ unsigned higher_limit, Register scratch,
+ Label* on_in_range,
+ Label::Distance near_jump) {
+ if (lower_limit != 0) {
+ lea(scratch, Operand(value, 0u - lower_limit));
+ cmp(scratch, Immediate(higher_limit - lower_limit));
+ } else {
+ cmp(value, Immediate(higher_limit));
+ }
+ j(below_equal, on_in_range, near_jump);
+}
+
+void TurboAssembler::PushArray(Register array, Register size, Register scratch,
+ PushArrayOrder order) {
+ DCHECK(!AreAliased(array, size, scratch));
+ Register counter = scratch;
+ Label loop, entry;
+ if (order == PushArrayOrder::kReverse) {
+ mov(counter, 0);
+ jmp(&entry);
+ bind(&loop);
+ Push(Operand(array, counter, times_system_pointer_size, 0));
+ inc(counter);
+ bind(&entry);
+ cmp(counter, size);
+ j(less, &loop, Label::kNear);
+ } else {
+ mov(counter, size);
+ jmp(&entry);
+ bind(&loop);
+ Push(Operand(array, counter, times_system_pointer_size, 0));
+ bind(&entry);
+ dec(counter);
+ j(greater_equal, &loop, Label::kNear);
+ }
+}
+
+Operand TurboAssembler::ExternalReferenceAsOperand(ExternalReference reference,
+ Register scratch) {
+ // TODO(jgruber): Add support for enable_root_array_delta_access.
+ if (root_array_available() && options().isolate_independent_code) {
+ if (IsAddressableThroughRootRegister(isolate(), reference)) {
+ // Some external references can be efficiently loaded as an offset from
+ // kRootRegister.
+ intptr_t offset =
+ RootRegisterOffsetForExternalReference(isolate(), reference);
+ return Operand(kRootRegister, offset);
+ } else {
+ // Otherwise, do a memory load from the external reference table.
+ mov(scratch, Operand(kRootRegister,
+ RootRegisterOffsetForExternalReferenceTableEntry(
+ isolate(), reference)));
+ return Operand(scratch, 0);
+ }
+ }
+ Move(scratch, Immediate(reference));
+ return Operand(scratch, 0);
+}
+
+// TODO(v8:6666): If possible, refactor into a platform-independent function in
+// TurboAssembler.
+Operand TurboAssembler::ExternalReferenceAddressAsOperand(
+ ExternalReference reference) {
+ DCHECK(root_array_available());
+ DCHECK(options().isolate_independent_code);
+ return Operand(
+ kRootRegister,
+ RootRegisterOffsetForExternalReferenceTableEntry(isolate(), reference));
+}
+
+// TODO(v8:6666): If possible, refactor into a platform-independent function in
+// TurboAssembler.
+Operand TurboAssembler::HeapObjectAsOperand(Handle<HeapObject> object) {
+ DCHECK(root_array_available());
+
+ int builtin_index;
+ RootIndex root_index;
+ if (isolate()->roots_table().IsRootHandle(object, &root_index)) {
+ return Operand(kRootRegister, RootRegisterOffsetForRootIndex(root_index));
+ } else if (isolate()->builtins()->IsBuiltinHandle(object, &builtin_index)) {
+ return Operand(kRootRegister,
+ RootRegisterOffsetForBuiltinIndex(builtin_index));
+ } else if (object.is_identical_to(code_object_) &&
+ Builtins::IsBuiltinId(maybe_builtin_index_)) {
+ return Operand(kRootRegister,
+ RootRegisterOffsetForBuiltinIndex(maybe_builtin_index_));
+ } else {
+ // Objects in the constants table need an additional indirection, which
+ // cannot be represented as a single Operand.
+ UNREACHABLE();
+ }
+}
+
+void TurboAssembler::LoadFromConstantsTable(Register destination,
+ int constant_index) {
+ DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kBuiltinsConstantsTable));
+ LoadRoot(destination, RootIndex::kBuiltinsConstantsTable);
+ mov(destination,
+ FieldOperand(destination, FixedArray::OffsetOfElementAt(constant_index)));
+}
+
+void TurboAssembler::LoadRootRegisterOffset(Register destination,
+ intptr_t offset) {
+ DCHECK(is_int32(offset));
+ DCHECK(root_array_available());
+ if (offset == 0) {
+ mov(destination, kRootRegister);
+ } else {
+ lea(destination, Operand(kRootRegister, static_cast<int32_t>(offset)));
+ }
+}
+
+void TurboAssembler::LoadRootRelative(Register destination, int32_t offset) {
+ DCHECK(root_array_available());
+ mov(destination, Operand(kRootRegister, offset));
+}
+
+void TurboAssembler::LoadAddress(Register destination,
+ ExternalReference source) {
+ // TODO(jgruber): Add support for enable_root_array_delta_access.
+ if (root_array_available() && options().isolate_independent_code) {
+ IndirectLoadExternalReference(destination, source);
+ return;
+ }
+ mov(destination, Immediate(source));
+}
+
+static constexpr Register saved_regs[] = {eax, ecx, edx};
+
+static constexpr int kNumberOfSavedRegs = sizeof(saved_regs) / sizeof(Register);
+
+int TurboAssembler::RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
+ Register exclusion1,
+ Register exclusion2,
+ Register exclusion3) const {
+ int bytes = 0;
+ for (int i = 0; i < kNumberOfSavedRegs; i++) {
+ Register reg = saved_regs[i];
+ if (reg != exclusion1 && reg != exclusion2 && reg != exclusion3) {
+ bytes += kSystemPointerSize;
+ }
+ }
+
+ if (fp_mode == kSaveFPRegs) {
+ // Count all XMM registers except XMM0.
+ bytes += kDoubleSize * (XMMRegister::kNumRegisters - 1);
+ }
+
+ return bytes;
+}
+
+int TurboAssembler::PushCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1,
+ Register exclusion2, Register exclusion3) {
+ // We don't allow a GC during a store buffer overflow so there is no need to
+ // store the registers in any particular way, but we do have to store and
+ // restore them.
+ int bytes = 0;
+ for (int i = 0; i < kNumberOfSavedRegs; i++) {
+ Register reg = saved_regs[i];
+ if (reg != exclusion1 && reg != exclusion2 && reg != exclusion3) {
+ push(reg);
+ bytes += kSystemPointerSize;
+ }
+ }
+
+ if (fp_mode == kSaveFPRegs) {
+ // Save all XMM registers except XMM0.
+ int delta = kDoubleSize * (XMMRegister::kNumRegisters - 1);
+ AllocateStackSpace(delta);
+ for (int i = XMMRegister::kNumRegisters - 1; i > 0; i--) {
+ XMMRegister reg = XMMRegister::from_code(i);
+ movsd(Operand(esp, (i - 1) * kDoubleSize), reg);
+ }
+ bytes += delta;
+ }
+
+ return bytes;
+}
+
+int TurboAssembler::PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1,
+ Register exclusion2, Register exclusion3) {
+ int bytes = 0;
+ if (fp_mode == kSaveFPRegs) {
+ // Restore all XMM registers except XMM0.
+ int delta = kDoubleSize * (XMMRegister::kNumRegisters - 1);
+ for (int i = XMMRegister::kNumRegisters - 1; i > 0; i--) {
+ XMMRegister reg = XMMRegister::from_code(i);
+ movsd(reg, Operand(esp, (i - 1) * kDoubleSize));
+ }
+ add(esp, Immediate(delta));
+ bytes += delta;
+ }
+
+ for (int i = kNumberOfSavedRegs - 1; i >= 0; i--) {
+ Register reg = saved_regs[i];
+ if (reg != exclusion1 && reg != exclusion2 && reg != exclusion3) {
+ pop(reg);
+ bytes += kSystemPointerSize;
+ }
+ }
+
+ return bytes;
+}
+
+void MacroAssembler::RecordWriteField(Register object, int offset,
+ Register value, Register dst,
+ SaveFPRegsMode save_fp,
+ RememberedSetAction remembered_set_action,
+ SmiCheck smi_check) {
+ // First, check if a write barrier is even needed. The tests below
+ // catch stores of Smis.
+ Label done;
+
+ // Skip barrier if writing a smi.
+ if (smi_check == INLINE_SMI_CHECK) {
+ JumpIfSmi(value, &done);
+ }
+
+ // Although the object register is tagged, the offset is relative to the start
+ // of the object, so so offset must be a multiple of kTaggedSize.
+ DCHECK(IsAligned(offset, kTaggedSize));
+
+ lea(dst, FieldOperand(object, offset));
+ if (emit_debug_code()) {
+ Label ok;
+ test_b(dst, Immediate(kTaggedSize - 1));
+ j(zero, &ok, Label::kNear);
+ int3();
+ bind(&ok);
+ }
+
+ RecordWrite(object, dst, value, save_fp, remembered_set_action,
+ OMIT_SMI_CHECK);
+
+ bind(&done);
+
+ // Clobber clobbered input registers when running with the debug-code flag
+ // turned on to provoke errors.
+ if (emit_debug_code()) {
+ mov(value, Immediate(bit_cast<int32_t>(kZapValue)));
+ mov(dst, Immediate(bit_cast<int32_t>(kZapValue)));
+ }
+}
+
+void TurboAssembler::SaveRegisters(RegList registers) {
+ DCHECK_GT(NumRegs(registers), 0);
+ for (int i = 0; i < Register::kNumRegisters; ++i) {
+ if ((registers >> i) & 1u) {
+ push(Register::from_code(i));
+ }
+ }
+}
+
+void TurboAssembler::RestoreRegisters(RegList registers) {
+ DCHECK_GT(NumRegs(registers), 0);
+ for (int i = Register::kNumRegisters - 1; i >= 0; --i) {
+ if ((registers >> i) & 1u) {
+ pop(Register::from_code(i));
+ }
+ }
+}
+
+void TurboAssembler::CallEphemeronKeyBarrier(Register object, Register address,
+ SaveFPRegsMode fp_mode) {
+ EphemeronKeyBarrierDescriptor descriptor;
+ RegList registers = descriptor.allocatable_registers();
+
+ SaveRegisters(registers);
+
+ Register object_parameter(
+ descriptor.GetRegisterParameter(EphemeronKeyBarrierDescriptor::kObject));
+ Register slot_parameter(descriptor.GetRegisterParameter(
+ EphemeronKeyBarrierDescriptor::kSlotAddress));
+ Register fp_mode_parameter(
+ descriptor.GetRegisterParameter(EphemeronKeyBarrierDescriptor::kFPMode));
+
+ push(object);
+ push(address);
+
+ pop(slot_parameter);
+ pop(object_parameter);
+
+ Move(fp_mode_parameter, Smi::FromEnum(fp_mode));
+ Call(isolate()->builtins()->builtin_handle(Builtins::kEphemeronKeyBarrier),
+ RelocInfo::CODE_TARGET);
+
+ RestoreRegisters(registers);
+}
+
+void TurboAssembler::CallRecordWriteStub(
+ Register object, Register address,
+ RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode) {
+ CallRecordWriteStub(
+ object, address, remembered_set_action, fp_mode,
+ isolate()->builtins()->builtin_handle(Builtins::kRecordWrite),
+ kNullAddress);
+}
+
+void TurboAssembler::CallRecordWriteStub(
+ Register object, Register address,
+ RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode,
+ Address wasm_target) {
+ CallRecordWriteStub(object, address, remembered_set_action, fp_mode,
+ Handle<Code>::null(), wasm_target);
+}
+
+void TurboAssembler::CallRecordWriteStub(
+ Register object, Register address,
+ RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode,
+ Handle<Code> code_target, Address wasm_target) {
+ DCHECK_NE(code_target.is_null(), wasm_target == kNullAddress);
+ // TODO(albertnetymk): For now we ignore remembered_set_action and fp_mode,
+ // i.e. always emit remember set and save FP registers in RecordWriteStub. If
+ // large performance regression is observed, we should use these values to
+ // avoid unnecessary work.
+
+ RecordWriteDescriptor descriptor;
+ RegList registers = descriptor.allocatable_registers();
+
+ SaveRegisters(registers);
+
+ Register object_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kObject));
+ Register slot_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kSlot));
+ Register remembered_set_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kRememberedSet));
+ Register fp_mode_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kFPMode));
+
+ push(object);
+ push(address);
+
+ pop(slot_parameter);
+ pop(object_parameter);
+
+ Move(remembered_set_parameter, Smi::FromEnum(remembered_set_action));
+ Move(fp_mode_parameter, Smi::FromEnum(fp_mode));
+ if (code_target.is_null()) {
+ // Use {wasm_call} for direct Wasm call within a module.
+ wasm_call(wasm_target, RelocInfo::WASM_STUB_CALL);
+ } else {
+ Call(code_target, RelocInfo::CODE_TARGET);
+ }
+
+ RestoreRegisters(registers);
+}
+
+void MacroAssembler::RecordWrite(Register object, Register address,
+ Register value, SaveFPRegsMode fp_mode,
+ RememberedSetAction remembered_set_action,
+ SmiCheck smi_check) {
+ DCHECK(object != value);
+ DCHECK(object != address);
+ DCHECK(value != address);
+ AssertNotSmi(object);
+
+ if ((remembered_set_action == OMIT_REMEMBERED_SET &&
+ !FLAG_incremental_marking) ||
+ FLAG_disable_write_barriers) {
+ return;
+ }
+
+ if (emit_debug_code()) {
+ Label ok;
+ cmp(value, Operand(address, 0));
+ j(equal, &ok, Label::kNear);
+ int3();
+ bind(&ok);
+ }
+
+ // First, check if a write barrier is even needed. The tests below
+ // catch stores of Smis and stores into young gen.
+ Label done;
+
+ if (smi_check == INLINE_SMI_CHECK) {
+ // Skip barrier if writing a smi.
+ JumpIfSmi(value, &done, Label::kNear);
+ }
+
+ CheckPageFlag(value,
+ value, // Used as scratch.
+ MemoryChunk::kPointersToHereAreInterestingMask, zero, &done,
+ Label::kNear);
+ CheckPageFlag(object,
+ value, // Used as scratch.
+ MemoryChunk::kPointersFromHereAreInterestingMask, zero, &done,
+ Label::kNear);
+
+ CallRecordWriteStub(object, address, remembered_set_action, fp_mode);
+
+ bind(&done);
+
+ // Clobber clobbered registers when running with the debug-code flag
+ // turned on to provoke errors.
+ if (emit_debug_code()) {
+ mov(address, Immediate(bit_cast<int32_t>(kZapValue)));
+ mov(value, Immediate(bit_cast<int32_t>(kZapValue)));
+ }
+}
+
+void MacroAssembler::MaybeDropFrames() {
+ // Check whether we need to drop frames to restart a function on the stack.
+ Label dont_drop;
+ ExternalReference restart_fp =
+ ExternalReference::debug_restart_fp_address(isolate());
+ mov(eax, ExternalReferenceAsOperand(restart_fp, eax));
+ test(eax, eax);
+ j(zero, &dont_drop, Label::kNear);
+
+ Jump(BUILTIN_CODE(isolate(), FrameDropperTrampoline), RelocInfo::CODE_TARGET);
+ bind(&dont_drop);
+}
+
+void TurboAssembler::Cvtsi2ss(XMMRegister dst, Operand src) {
+ xorps(dst, dst);
+ cvtsi2ss(dst, src);
+}
+
+void TurboAssembler::Cvtsi2sd(XMMRegister dst, Operand src) {
+ xorpd(dst, dst);
+ cvtsi2sd(dst, src);
+}
+
+void TurboAssembler::Cvtui2ss(XMMRegister dst, Operand src, Register tmp) {
+ Label done;
+ Register src_reg = src.is_reg_only() ? src.reg() : tmp;
+ if (src_reg == tmp) mov(tmp, src);
+ cvtsi2ss(dst, src_reg);
+ test(src_reg, src_reg);
+ j(positive, &done, Label::kNear);
+
+ // Compute {src/2 | (src&1)} (retain the LSB to avoid rounding errors).
+ if (src_reg != tmp) mov(tmp, src_reg);
+ shr(tmp, 1);
+ // The LSB is shifted into CF. If it is set, set the LSB in {tmp}.
+ Label msb_not_set;
+ j(not_carry, &msb_not_set, Label::kNear);
+ or_(tmp, Immediate(1));
+ bind(&msb_not_set);
+ cvtsi2ss(dst, tmp);
+ addss(dst, dst);
+ bind(&done);
+}
+
+void TurboAssembler::Cvttss2ui(Register dst, Operand src, XMMRegister tmp) {
+ Label done;
+ cvttss2si(dst, src);
+ test(dst, dst);
+ j(positive, &done);
+ Move(tmp, static_cast<float>(INT32_MIN));
+ addss(tmp, src);
+ cvttss2si(dst, tmp);
+ or_(dst, Immediate(0x80000000));
+ bind(&done);
+}
+
+void TurboAssembler::Cvtui2sd(XMMRegister dst, Operand src, Register scratch) {
+ Label done;
+ cmp(src, Immediate(0));
+ ExternalReference uint32_bias = ExternalReference::address_of_uint32_bias();
+ Cvtsi2sd(dst, src);
+ j(not_sign, &done, Label::kNear);
+ addsd(dst, ExternalReferenceAsOperand(uint32_bias, scratch));
+ bind(&done);
+}
+
+void TurboAssembler::Cvttsd2ui(Register dst, Operand src, XMMRegister tmp) {
+ Move(tmp, -2147483648.0);
+ addsd(tmp, src);
+ cvttsd2si(dst, tmp);
+ add(dst, Immediate(0x80000000));
+}
+
+void TurboAssembler::Roundps(XMMRegister dst, XMMRegister src,
+ RoundingMode mode) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vroundps(dst, src, mode);
+ } else {
+ CpuFeatureScope scope(this, SSE4_1);
+ roundps(dst, src, mode);
+ }
+}
+
+void TurboAssembler::Roundpd(XMMRegister dst, XMMRegister src,
+ RoundingMode mode) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vroundpd(dst, src, mode);
+ } else {
+ CpuFeatureScope scope(this, SSE4_1);
+ roundpd(dst, src, mode);
+ }
+}
+
+void TurboAssembler::ShlPair(Register high, Register low, uint8_t shift) {
+ DCHECK_GE(63, shift);
+ if (shift >= 32) {
+ mov(high, low);
+ if (shift != 32) shl(high, shift - 32);
+ xor_(low, low);
+ } else {
+ shld(high, low, shift);
+ shl(low, shift);
+ }
+}
+
+void TurboAssembler::ShlPair_cl(Register high, Register low) {
+ shld_cl(high, low);
+ shl_cl(low);
+ Label done;
+ test(ecx, Immediate(0x20));
+ j(equal, &done, Label::kNear);
+ mov(high, low);
+ xor_(low, low);
+ bind(&done);
+}
+
+void TurboAssembler::ShrPair(Register high, Register low, uint8_t shift) {
+ DCHECK_GE(63, shift);
+ if (shift >= 32) {
+ mov(low, high);
+ if (shift != 32) shr(low, shift - 32);
+ xor_(high, high);
+ } else {
+ shrd(low, high, shift);
+ shr(high, shift);
+ }
+}
+
+void TurboAssembler::ShrPair_cl(Register high, Register low) {
+ shrd_cl(low, high);
+ shr_cl(high);
+ Label done;
+ test(ecx, Immediate(0x20));
+ j(equal, &done, Label::kNear);
+ mov(low, high);
+ xor_(high, high);
+ bind(&done);
+}
+
+void TurboAssembler::SarPair(Register high, Register low, uint8_t shift) {
+ DCHECK_GE(63, shift);
+ if (shift >= 32) {
+ mov(low, high);
+ if (shift != 32) sar(low, shift - 32);
+ sar(high, 31);
+ } else {
+ shrd(low, high, shift);
+ sar(high, shift);
+ }
+}
+
+void TurboAssembler::SarPair_cl(Register high, Register low) {
+ shrd_cl(low, high);
+ sar_cl(high);
+ Label done;
+ test(ecx, Immediate(0x20));
+ j(equal, &done, Label::kNear);
+ mov(low, high);
+ sar(high, 31);
+ bind(&done);
+}
+
+void TurboAssembler::LoadMap(Register destination, Register object) {
+ mov(destination, FieldOperand(object, HeapObject::kMapOffset));
+}
+
+void MacroAssembler::CmpObjectType(Register heap_object, InstanceType type,
+ Register map) {
+ LoadMap(map, heap_object);
+ CmpInstanceType(map, type);
+}
+
+void MacroAssembler::CmpInstanceType(Register map, InstanceType type) {
+ cmpw(FieldOperand(map, Map::kInstanceTypeOffset), Immediate(type));
+}
+
+void MacroAssembler::AssertSmi(Register object) {
+ if (emit_debug_code()) {
+ test(object, Immediate(kSmiTagMask));
+ Check(equal, AbortReason::kOperandIsNotASmi);
+ }
+}
+
+void MacroAssembler::AssertConstructor(Register object) {
+ if (emit_debug_code()) {
+ test(object, Immediate(kSmiTagMask));
+ Check(not_equal, AbortReason::kOperandIsASmiAndNotAConstructor);
+ Push(object);
+ LoadMap(object, object);
+ test_b(FieldOperand(object, Map::kBitFieldOffset),
+ Immediate(Map::Bits1::IsConstructorBit::kMask));
+ Pop(object);
+ Check(not_zero, AbortReason::kOperandIsNotAConstructor);
+ }
+}
+
+void MacroAssembler::AssertFunction(Register object) {
+ if (emit_debug_code()) {
+ test(object, Immediate(kSmiTagMask));
+ Check(not_equal, AbortReason::kOperandIsASmiAndNotAFunction);
+ Push(object);
+ CmpObjectType(object, JS_FUNCTION_TYPE, object);
+ Pop(object);
+ Check(equal, AbortReason::kOperandIsNotAFunction);
+ }
+}
+
+void MacroAssembler::AssertBoundFunction(Register object) {
+ if (emit_debug_code()) {
+ test(object, Immediate(kSmiTagMask));
+ Check(not_equal, AbortReason::kOperandIsASmiAndNotABoundFunction);
+ Push(object);
+ CmpObjectType(object, JS_BOUND_FUNCTION_TYPE, object);
+ Pop(object);
+ Check(equal, AbortReason::kOperandIsNotABoundFunction);
+ }
+}
+
+void MacroAssembler::AssertGeneratorObject(Register object) {
+ if (!emit_debug_code()) return;
+
+ test(object, Immediate(kSmiTagMask));
+ Check(not_equal, AbortReason::kOperandIsASmiAndNotAGeneratorObject);
+
+ {
+ Push(object);
+ Register map = object;
+
+ LoadMap(map, object);
+
+ Label do_check;
+ // Check if JSGeneratorObject
+ CmpInstanceType(map, JS_GENERATOR_OBJECT_TYPE);
+ j(equal, &do_check, Label::kNear);
+
+ // Check if JSAsyncFunctionObject.
+ CmpInstanceType(map, JS_ASYNC_FUNCTION_OBJECT_TYPE);
+ j(equal, &do_check, Label::kNear);
+
+ // Check if JSAsyncGeneratorObject
+ CmpInstanceType(map, JS_ASYNC_GENERATOR_OBJECT_TYPE);
+
+ bind(&do_check);
+ Pop(object);
+ }
+
+ Check(equal, AbortReason::kOperandIsNotAGeneratorObject);
+}
+
+void MacroAssembler::AssertUndefinedOrAllocationSite(Register object,
+ Register scratch) {
+ if (emit_debug_code()) {
+ Label done_checking;
+ AssertNotSmi(object);
+ CompareRoot(object, scratch, RootIndex::kUndefinedValue);
+ j(equal, &done_checking);
+ LoadRoot(scratch, RootIndex::kAllocationSiteWithWeakNextMap);
+ cmp(FieldOperand(object, 0), scratch);
+ Assert(equal, AbortReason::kExpectedUndefinedOrCell);
+ bind(&done_checking);
+ }
+}
+
+void MacroAssembler::AssertNotSmi(Register object) {
+ if (emit_debug_code()) {
+ test(object, Immediate(kSmiTagMask));
+ Check(not_equal, AbortReason::kOperandIsASmi);
+ }
+}
+
+void TurboAssembler::StubPrologue(StackFrame::Type type) {
+ push(ebp); // Caller's frame pointer.
+ mov(ebp, esp);
+ push(Immediate(StackFrame::TypeToMarker(type)));
+}
+
+void TurboAssembler::Prologue() {
+ push(ebp); // Caller's frame pointer.
+ mov(ebp, esp);
+ push(kContextRegister); // Callee's context.
+ push(kJSFunctionRegister); // Callee's JS function.
+ push(kJavaScriptCallArgCountRegister); // Actual argument count.
+}
+
+void TurboAssembler::EnterFrame(StackFrame::Type type) {
+ push(ebp);
+ mov(ebp, esp);
+ push(Immediate(StackFrame::TypeToMarker(type)));
+}
+
+void TurboAssembler::LeaveFrame(StackFrame::Type type) {
+ if (emit_debug_code()) {
+ cmp(Operand(ebp, CommonFrameConstants::kContextOrFrameTypeOffset),
+ Immediate(StackFrame::TypeToMarker(type)));
+ Check(equal, AbortReason::kStackFrameTypesMustMatch);
+ }
+ leave();
+}
+
+#ifdef V8_OS_WIN
+void TurboAssembler::AllocateStackSpace(Register bytes_scratch) {
+ // In windows, we cannot increment the stack size by more than one page
+ // (minimum page size is 4KB) without accessing at least one byte on the
+ // page. Check this:
+ // https://msdn.microsoft.com/en-us/library/aa227153(v=vs.60).aspx.
+ Label check_offset;
+ Label touch_next_page;
+ jmp(&check_offset);
+ bind(&touch_next_page);
+ sub(esp, Immediate(kStackPageSize));
+ // Just to touch the page, before we increment further.
+ mov(Operand(esp, 0), Immediate(0));
+ sub(bytes_scratch, Immediate(kStackPageSize));
+
+ bind(&check_offset);
+ cmp(bytes_scratch, kStackPageSize);
+ j(greater, &touch_next_page);
+
+ sub(esp, bytes_scratch);
+}
+
+void TurboAssembler::AllocateStackSpace(int bytes) {
+ while (bytes > kStackPageSize) {
+ sub(esp, Immediate(kStackPageSize));
+ mov(Operand(esp, 0), Immediate(0));
+ bytes -= kStackPageSize;
+ }
+ sub(esp, Immediate(bytes));
+}
+#endif
+
+void MacroAssembler::EnterExitFramePrologue(StackFrame::Type frame_type,
+ Register scratch) {
+ DCHECK(frame_type == StackFrame::EXIT ||
+ frame_type == StackFrame::BUILTIN_EXIT);
+
+ // Set up the frame structure on the stack.
+ DCHECK_EQ(+2 * kSystemPointerSize, ExitFrameConstants::kCallerSPDisplacement);
+ DCHECK_EQ(+1 * kSystemPointerSize, ExitFrameConstants::kCallerPCOffset);
+ DCHECK_EQ(0 * kSystemPointerSize, ExitFrameConstants::kCallerFPOffset);
+ push(ebp);
+ mov(ebp, esp);
+
+ // Reserve room for entry stack pointer.
+ push(Immediate(StackFrame::TypeToMarker(frame_type)));
+ DCHECK_EQ(-2 * kSystemPointerSize, ExitFrameConstants::kSPOffset);
+ push(Immediate(0)); // Saved entry sp, patched before call.
+
+ STATIC_ASSERT(edx == kRuntimeCallFunctionRegister);
+ STATIC_ASSERT(esi == kContextRegister);
+
+ // Save the frame pointer and the context in top.
+ ExternalReference c_entry_fp_address =
+ ExternalReference::Create(IsolateAddressId::kCEntryFPAddress, isolate());
+ ExternalReference context_address =
+ ExternalReference::Create(IsolateAddressId::kContextAddress, isolate());
+ ExternalReference c_function_address =
+ ExternalReference::Create(IsolateAddressId::kCFunctionAddress, isolate());
+
+ DCHECK(!AreAliased(scratch, ebp, esi, edx));
+ mov(ExternalReferenceAsOperand(c_entry_fp_address, scratch), ebp);
+ mov(ExternalReferenceAsOperand(context_address, scratch), esi);
+ mov(ExternalReferenceAsOperand(c_function_address, scratch), edx);
+}
+
+void MacroAssembler::EnterExitFrameEpilogue(int argc, bool save_doubles) {
+ // Optionally save all XMM registers.
+ if (save_doubles) {
+ int space =
+ XMMRegister::kNumRegisters * kDoubleSize + argc * kSystemPointerSize;
+ AllocateStackSpace(space);
+ const int offset = -ExitFrameConstants::kFixedFrameSizeFromFp;
+ for (int i = 0; i < XMMRegister::kNumRegisters; i++) {
+ XMMRegister reg = XMMRegister::from_code(i);
+ movsd(Operand(ebp, offset - ((i + 1) * kDoubleSize)), reg);
+ }
+ } else {
+ AllocateStackSpace(argc * kSystemPointerSize);
+ }
+
+ // Get the required frame alignment for the OS.
+ const int kFrameAlignment = base::OS::ActivationFrameAlignment();
+ if (kFrameAlignment > 0) {
+ DCHECK(base::bits::IsPowerOfTwo(kFrameAlignment));
+ and_(esp, -kFrameAlignment);
+ }
+
+ // Patch the saved entry sp.
+ mov(Operand(ebp, ExitFrameConstants::kSPOffset), esp);
+}
+
+void MacroAssembler::EnterExitFrame(int argc, bool save_doubles,
+ StackFrame::Type frame_type) {
+ EnterExitFramePrologue(frame_type, edi);
+
+ // Set up argc and argv in callee-saved registers.
+ int offset = StandardFrameConstants::kCallerSPOffset - kSystemPointerSize;
+ mov(edi, eax);
+ lea(esi, Operand(ebp, eax, times_system_pointer_size, offset));
+
+ // Reserve space for argc, argv and isolate.
+ EnterExitFrameEpilogue(argc, save_doubles);
+}
+
+void MacroAssembler::EnterApiExitFrame(int argc, Register scratch) {
+ EnterExitFramePrologue(StackFrame::EXIT, scratch);
+ EnterExitFrameEpilogue(argc, false);
+}
+
+void MacroAssembler::LeaveExitFrame(bool save_doubles, bool pop_arguments) {
+ // Optionally restore all XMM registers.
+ if (save_doubles) {
+ const int offset = -ExitFrameConstants::kFixedFrameSizeFromFp;
+ for (int i = 0; i < XMMRegister::kNumRegisters; i++) {
+ XMMRegister reg = XMMRegister::from_code(i);
+ movsd(reg, Operand(ebp, offset - ((i + 1) * kDoubleSize)));
+ }
+ }
+
+ if (pop_arguments) {
+ // Get the return address from the stack and restore the frame pointer.
+ mov(ecx, Operand(ebp, 1 * kSystemPointerSize));
+ mov(ebp, Operand(ebp, 0 * kSystemPointerSize));
+
+ // Pop the arguments and the receiver from the caller stack.
+ lea(esp, Operand(esi, 1 * kSystemPointerSize));
+
+ // Push the return address to get ready to return.
+ push(ecx);
+ } else {
+ // Otherwise just leave the exit frame.
+ leave();
+ }
+
+ LeaveExitFrameEpilogue();
+}
+
+void MacroAssembler::LeaveExitFrameEpilogue() {
+ // Clear the top frame.
+ ExternalReference c_entry_fp_address =
+ ExternalReference::Create(IsolateAddressId::kCEntryFPAddress, isolate());
+ mov(ExternalReferenceAsOperand(c_entry_fp_address, esi), Immediate(0));
+
+ // Restore current context from top and clear it in debug mode.
+ ExternalReference context_address =
+ ExternalReference::Create(IsolateAddressId::kContextAddress, isolate());
+ mov(esi, ExternalReferenceAsOperand(context_address, esi));
+#ifdef DEBUG
+ push(eax);
+ mov(ExternalReferenceAsOperand(context_address, eax),
+ Immediate(Context::kInvalidContext));
+ pop(eax);
+#endif
+}
+
+void MacroAssembler::LeaveApiExitFrame() {
+ mov(esp, ebp);
+ pop(ebp);
+
+ LeaveExitFrameEpilogue();
+}
+
+void MacroAssembler::PushStackHandler(Register scratch) {
+ // Adjust this code if not the case.
+ STATIC_ASSERT(StackHandlerConstants::kSize == 2 * kSystemPointerSize);
+ STATIC_ASSERT(StackHandlerConstants::kNextOffset == 0);
+
+ push(Immediate(0)); // Padding.
+
+ // Link the current handler as the next handler.
+ ExternalReference handler_address =
+ ExternalReference::Create(IsolateAddressId::kHandlerAddress, isolate());
+ push(ExternalReferenceAsOperand(handler_address, scratch));
+
+ // Set this new handler as the current one.
+ mov(ExternalReferenceAsOperand(handler_address, scratch), esp);
+}
+
+void MacroAssembler::PopStackHandler(Register scratch) {
+ STATIC_ASSERT(StackHandlerConstants::kNextOffset == 0);
+ ExternalReference handler_address =
+ ExternalReference::Create(IsolateAddressId::kHandlerAddress, isolate());
+ pop(ExternalReferenceAsOperand(handler_address, scratch));
+ add(esp, Immediate(StackHandlerConstants::kSize - kSystemPointerSize));
+}
+
+void MacroAssembler::CallRuntime(const Runtime::Function* f, int num_arguments,
+ SaveFPRegsMode save_doubles) {
+ // If the expected number of arguments of the runtime function is
+ // constant, we check that the actual number of arguments match the
+ // expectation.
+ CHECK(f->nargs < 0 || f->nargs == num_arguments);
+
+ // TODO(1236192): Most runtime routines don't need the number of
+ // arguments passed in because it is constant. At some point we
+ // should remove this need and make the runtime routine entry code
+ // smarter.
+ Move(kRuntimeCallArgCountRegister, Immediate(num_arguments));
+ Move(kRuntimeCallFunctionRegister, Immediate(ExternalReference::Create(f)));
+ Handle<Code> code =
+ CodeFactory::CEntry(isolate(), f->result_size, save_doubles);
+ Call(code, RelocInfo::CODE_TARGET);
+}
+
+void MacroAssembler::TailCallRuntime(Runtime::FunctionId fid) {
+ // ----------- S t a t e -------------
+ // -- esp[0] : return address
+ // -- esp[8] : argument num_arguments - 1
+ // ...
+ // -- esp[8 * num_arguments] : argument 0 (receiver)
+ //
+ // For runtime functions with variable arguments:
+ // -- eax : number of arguments
+ // -----------------------------------
+
+ const Runtime::Function* function = Runtime::FunctionForId(fid);
+ DCHECK_EQ(1, function->result_size);
+ if (function->nargs >= 0) {
+ // TODO(1236192): Most runtime routines don't need the number of
+ // arguments passed in because it is constant. At some point we
+ // should remove this need and make the runtime routine entry code
+ // smarter.
+ Move(kRuntimeCallArgCountRegister, Immediate(function->nargs));
+ }
+ JumpToExternalReference(ExternalReference::Create(fid));
+}
+
+void MacroAssembler::JumpToExternalReference(const ExternalReference& ext,
+ bool builtin_exit_frame) {
+ // Set the entry point and jump to the C entry runtime stub.
+ Move(kRuntimeCallFunctionRegister, Immediate(ext));
+ Handle<Code> code = CodeFactory::CEntry(isolate(), 1, kDontSaveFPRegs,
+ kArgvOnStack, builtin_exit_frame);
+ Jump(code, RelocInfo::CODE_TARGET);
+}
+
+void MacroAssembler::JumpToInstructionStream(Address entry) {
+ jmp(entry, RelocInfo::OFF_HEAP_TARGET);
+}
+
+void TurboAssembler::PrepareForTailCall(
+ Register callee_args_count, Register caller_args_count, Register scratch0,
+ Register scratch1, int number_of_temp_values_after_return_address) {
+ DCHECK(!AreAliased(callee_args_count, caller_args_count, scratch0, scratch1));
+
+ // Calculate the destination address where we will put the return address
+ // after we drop current frame.
+ Register new_sp_reg = scratch0;
+ sub(caller_args_count, callee_args_count);
+ lea(new_sp_reg, Operand(ebp, caller_args_count, times_system_pointer_size,
+ StandardFrameConstants::kCallerPCOffset -
+ number_of_temp_values_after_return_address *
+ kSystemPointerSize));
+
+ if (FLAG_debug_code) {
+ cmp(esp, new_sp_reg);
+ Check(below, AbortReason::kStackAccessBelowStackPointer);
+ }
+
+ // Copy return address from caller's frame to current frame's return address
+ // to avoid its trashing and let the following loop copy it to the right
+ // place.
+ Register tmp_reg = scratch1;
+ mov(tmp_reg, Operand(ebp, StandardFrameConstants::kCallerPCOffset));
+ mov(Operand(esp,
+ number_of_temp_values_after_return_address * kSystemPointerSize),
+ tmp_reg);
+
+ // Restore caller's frame pointer now as it could be overwritten by
+ // the copying loop.
+ mov(ebp, Operand(ebp, StandardFrameConstants::kCallerFPOffset));
+
+ // +2 here is to copy both receiver and return address.
+ Register count_reg = caller_args_count;
+ lea(count_reg, Operand(callee_args_count,
+ 2 + number_of_temp_values_after_return_address));
+
+ // Now copy callee arguments to the caller frame going backwards to avoid
+ // callee arguments corruption (source and destination areas could overlap).
+ Label loop, entry;
+ jmp(&entry, Label::kNear);
+ bind(&loop);
+ dec(count_reg);
+ mov(tmp_reg, Operand(esp, count_reg, times_system_pointer_size, 0));
+ mov(Operand(new_sp_reg, count_reg, times_system_pointer_size, 0), tmp_reg);
+ bind(&entry);
+ cmp(count_reg, Immediate(0));
+ j(not_equal, &loop, Label::kNear);
+
+ // Leave current frame.
+ mov(esp, new_sp_reg);
+}
+
+void MacroAssembler::CompareStackLimit(Register with, StackLimitKind kind) {
+ DCHECK(root_array_available());
+ Isolate* isolate = this->isolate();
+ // Address through the root register. No load is needed.
+ ExternalReference limit =
+ kind == StackLimitKind::kRealStackLimit
+ ? ExternalReference::address_of_real_jslimit(isolate)
+ : ExternalReference::address_of_jslimit(isolate);
+ DCHECK(TurboAssembler::IsAddressableThroughRootRegister(isolate, limit));
+
+ intptr_t offset =
+ TurboAssembler::RootRegisterOffsetForExternalReference(isolate, limit);
+ cmp(with, Operand(kRootRegister, offset));
+}
+
+void MacroAssembler::StackOverflowCheck(Register num_args, Register scratch,
+ Label* stack_overflow,
+ bool include_receiver) {
+ DCHECK_NE(num_args, scratch);
+ // Check the stack for overflow. We are not trying to catch
+ // interruptions (e.g. debug break and preemption) here, so the "real stack
+ // limit" is checked.
+ ExternalReference real_stack_limit =
+ ExternalReference::address_of_real_jslimit(isolate());
+ // Compute the space that is left as a negative number in scratch. If
+ // we already overflowed, this will be a positive number.
+ mov(scratch, ExternalReferenceAsOperand(real_stack_limit, scratch));
+ sub(scratch, esp);
+ // TODO(victorgomes): Remove {include_receiver} and always require one extra
+ // word of the stack space.
+ lea(scratch, Operand(scratch, num_args, times_system_pointer_size, 0));
+ if (include_receiver) {
+ add(scratch, Immediate(kSystemPointerSize));
+ }
+ // See if we overflowed, i.e. scratch is positive.
+ cmp(scratch, Immediate(0));
+ // TODO(victorgomes): Save some bytes in the builtins that use stack checks
+ // by jumping to a builtin that throws the exception.
+ j(greater, stack_overflow); // Signed comparison.
+}
+
+void MacroAssembler::InvokePrologue(Register expected_parameter_count,
+ Register actual_parameter_count,
+ Label* done, InvokeFlag flag) {
+ if (expected_parameter_count != actual_parameter_count) {
+ DCHECK_EQ(actual_parameter_count, eax);
+ DCHECK_EQ(expected_parameter_count, ecx);
+ Label regular_invoke;
+#ifdef V8_NO_ARGUMENTS_ADAPTOR
+ // If the expected parameter count is equal to the adaptor sentinel, no need
+ // to push undefined value as arguments.
+ cmp(expected_parameter_count, Immediate(kDontAdaptArgumentsSentinel));
+ j(equal, ®ular_invoke, Label::kFar);
+
+ // If overapplication or if the actual argument count is equal to the
+ // formal parameter count, no need to push extra undefined values.
+ sub(expected_parameter_count, actual_parameter_count);
+ j(less_equal, ®ular_invoke, Label::kFar);
+
+ // We need to preserve edx, edi, esi and ebx.
+ movd(xmm0, edx);
+ movd(xmm1, edi);
+ movd(xmm2, esi);
+ movd(xmm3, ebx);
+
+ Label stack_overflow;
+ StackOverflowCheck(expected_parameter_count, edx, &stack_overflow);
+
+ Register scratch = esi;
+
+ // Underapplication. Move the arguments already in the stack, including the
+ // receiver and the return address.
+ {
+ Label copy, check;
+ Register src = edx, dest = esp, num = edi, current = ebx;
+ mov(src, esp);
+ lea(scratch,
+ Operand(expected_parameter_count, times_system_pointer_size, 0));
+ AllocateStackSpace(scratch);
+ // Extra words are the receiver and the return address (if a jump).
+ int extra_words = flag == CALL_FUNCTION ? 1 : 2;
+ lea(num, Operand(eax, extra_words)); // Number of words to copy.
+ Set(current, 0);
+ // Fall-through to the loop body because there are non-zero words to copy.
+ bind(©);
+ mov(scratch, Operand(src, current, times_system_pointer_size, 0));
+ mov(Operand(dest, current, times_system_pointer_size, 0), scratch);
+ inc(current);
+ bind(&check);
+ cmp(current, num);
+ j(less, ©);
+ lea(edx, Operand(esp, num, times_system_pointer_size, 0));
+ }
+
+ // Fill remaining expected arguments with undefined values.
+ movd(ebx, xmm3); // Restore root.
+ LoadRoot(scratch, RootIndex::kUndefinedValue);
+ {
+ Label loop;
+ bind(&loop);
+ dec(expected_parameter_count);
+ mov(Operand(edx, expected_parameter_count, times_system_pointer_size, 0),
+ scratch);
+ j(greater, &loop, Label::kNear);
+ }
+
+ // Restore remaining registers.
+ movd(esi, xmm2);
+ movd(edi, xmm1);
+ movd(edx, xmm0);
+
+ jmp(®ular_invoke);
+
+ bind(&stack_overflow);
+ {
+ FrameScope frame(this,
+ has_frame() ? StackFrame::NONE : StackFrame::INTERNAL);
+ CallRuntime(Runtime::kThrowStackOverflow);
+ int3(); // This should be unreachable.
+ }
+#else
+ cmp(expected_parameter_count, actual_parameter_count);
+ j(equal, ®ular_invoke);
+ Handle<Code> adaptor = BUILTIN_CODE(isolate(), ArgumentsAdaptorTrampoline);
+ if (flag == CALL_FUNCTION) {
+ Call(adaptor, RelocInfo::CODE_TARGET);
+ jmp(done, Label::kNear);
+ } else {
+ Jump(adaptor, RelocInfo::CODE_TARGET);
+ }
+#endif
+ bind(®ular_invoke);
+ }
+}
+
+void MacroAssembler::CallDebugOnFunctionCall(Register fun, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count) {
+ FrameScope frame(this, has_frame() ? StackFrame::NONE : StackFrame::INTERNAL);
+ SmiTag(expected_parameter_count);
+ Push(expected_parameter_count);
+
+ SmiTag(actual_parameter_count);
+ Push(actual_parameter_count);
+ SmiUntag(actual_parameter_count);
+
+ if (new_target.is_valid()) {
+ Push(new_target);
+ }
+ Push(fun);
+ Push(fun);
+ // Arguments are located 2 words below the base pointer.
+ Operand receiver_op = Operand(ebp, kSystemPointerSize * 2);
+ Push(receiver_op);
+ CallRuntime(Runtime::kDebugOnFunctionCall);
+ Pop(fun);
+ if (new_target.is_valid()) {
+ Pop(new_target);
+ }
+ Pop(actual_parameter_count);
+ SmiUntag(actual_parameter_count);
+
+ Pop(expected_parameter_count);
+ SmiUntag(expected_parameter_count);
+}
+
+void MacroAssembler::InvokeFunctionCode(Register function, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count,
+ InvokeFlag flag) {
+ // You can't call a function without a valid frame.
+ DCHECK_IMPLIES(flag == CALL_FUNCTION, has_frame());
+ DCHECK_EQ(function, edi);
+ DCHECK_IMPLIES(new_target.is_valid(), new_target == edx);
+ DCHECK(expected_parameter_count == ecx || expected_parameter_count == eax);
+ DCHECK_EQ(actual_parameter_count, eax);
+
+ // On function call, call into the debugger if necessary.
+ Label debug_hook, continue_after_hook;
+ {
+ ExternalReference debug_hook_active =
+ ExternalReference::debug_hook_on_function_call_address(isolate());
+ push(eax);
+ cmpb(ExternalReferenceAsOperand(debug_hook_active, eax), Immediate(0));
+ pop(eax);
+ j(not_equal, &debug_hook);
+ }
+ bind(&continue_after_hook);
+
+ // Clear the new.target register if not given.
+ if (!new_target.is_valid()) {
+ Move(edx, isolate()->factory()->undefined_value());
+ }
+
+ Label done;
+ InvokePrologue(expected_parameter_count, actual_parameter_count, &done, flag);
+ // We call indirectly through the code field in the function to
+ // allow recompilation to take effect without changing any of the
+ // call sites.
+ static_assert(kJavaScriptCallCodeStartRegister == ecx, "ABI mismatch");
+ mov(ecx, FieldOperand(function, JSFunction::kCodeOffset));
+ if (flag == CALL_FUNCTION) {
+ CallCodeObject(ecx);
+ } else {
+ DCHECK(flag == JUMP_FUNCTION);
+ JumpCodeObject(ecx);
+ }
+ jmp(&done, Label::kNear);
+
+ // Deferred debug hook.
+ bind(&debug_hook);
+ CallDebugOnFunctionCall(function, new_target, expected_parameter_count,
+ actual_parameter_count);
+ jmp(&continue_after_hook);
+
+ bind(&done);
+}
+
+void MacroAssembler::InvokeFunction(Register fun, Register new_target,
+ Register actual_parameter_count,
+ InvokeFlag flag) {
+ // You can't call a function without a valid frame.
+ DCHECK(flag == JUMP_FUNCTION || has_frame());
+
+ DCHECK(fun == edi);
+ mov(ecx, FieldOperand(edi, JSFunction::kSharedFunctionInfoOffset));
+ mov(esi, FieldOperand(edi, JSFunction::kContextOffset));
+ movzx_w(ecx,
+ FieldOperand(ecx, SharedFunctionInfo::kFormalParameterCountOffset));
+
+ InvokeFunctionCode(edi, new_target, ecx, actual_parameter_count, flag);
+}
+
+void MacroAssembler::LoadGlobalProxy(Register dst) {
+ LoadNativeContextSlot(dst, Context::GLOBAL_PROXY_INDEX);
+}
+
+void MacroAssembler::LoadNativeContextSlot(Register destination, int index) {
+ // Load the native context from the current context.
+ LoadMap(destination, esi);
+ mov(destination,
+ FieldOperand(destination,
+ Map::kConstructorOrBackPointerOrNativeContextOffset));
+ // Load the function from the native context.
+ mov(destination, Operand(destination, Context::SlotOffset(index)));
+}
+
+int MacroAssembler::SafepointRegisterStackIndex(int reg_code) {
+ // The registers are pushed starting with the lowest encoding,
+ // which means that lowest encodings are furthest away from
+ // the stack pointer.
+ DCHECK(reg_code >= 0 && reg_code < kNumSafepointRegisters);
+ return kNumSafepointRegisters - reg_code - 1;
+}
+
+void TurboAssembler::Ret() { ret(0); }
+
+void TurboAssembler::Ret(int bytes_dropped, Register scratch) {
+ if (is_uint16(bytes_dropped)) {
+ ret(bytes_dropped);
+ } else {
+ pop(scratch);
+ add(esp, Immediate(bytes_dropped));
+ push(scratch);
+ ret(0);
+ }
+}
+
+void TurboAssembler::Push(Immediate value) {
+ if (root_array_available() && options().isolate_independent_code) {
+ if (value.is_embedded_object()) {
+ Push(HeapObjectAsOperand(value.embedded_object()));
+ return;
+ } else if (value.is_external_reference()) {
+ Push(ExternalReferenceAddressAsOperand(value.external_reference()));
+ return;
+ }
+ }
+ push(value);
+}
+
+void MacroAssembler::Drop(int stack_elements) {
+ if (stack_elements > 0) {
+ add(esp, Immediate(stack_elements * kSystemPointerSize));
+ }
+}
+
+void TurboAssembler::Move(Register dst, Register src) {
+ if (dst != src) {
+ mov(dst, src);
+ }
+}
+
+void TurboAssembler::Move(Register dst, const Immediate& src) {
+ if (!src.is_heap_object_request() && src.is_zero()) {
+ xor_(dst, dst); // Shorter than mov of 32-bit immediate 0.
+ } else if (src.is_external_reference()) {
+ LoadAddress(dst, src.external_reference());
+ } else {
+ mov(dst, src);
+ }
+}
+
+void TurboAssembler::Move(Operand dst, const Immediate& src) {
+ // Since there's no scratch register available, take a detour through the
+ // stack.
+ if (root_array_available() && options().isolate_independent_code) {
+ if (src.is_embedded_object() || src.is_external_reference() ||
+ src.is_heap_object_request()) {
+ Push(src);
+ pop(dst);
+ return;
+ }
+ }
+
+ if (src.is_embedded_object()) {
+ mov(dst, src.embedded_object());
+ } else {
+ mov(dst, src);
+ }
+}
+
+void TurboAssembler::Move(Register dst, Handle<HeapObject> src) {
+ if (root_array_available() && options().isolate_independent_code) {
+ IndirectLoadConstant(dst, src);
+ return;
+ }
+ mov(dst, src);
+}
+
+void TurboAssembler::Move(XMMRegister dst, uint32_t src) {
+ if (src == 0) {
+ pxor(dst, dst);
+ } else {
+ unsigned cnt = base::bits::CountPopulation(src);
+ unsigned nlz = base::bits::CountLeadingZeros32(src);
+ unsigned ntz = base::bits::CountTrailingZeros32(src);
+ if (nlz + cnt + ntz == 32) {
+ pcmpeqd(dst, dst);
+ if (ntz == 0) {
+ psrld(dst, 32 - cnt);
+ } else {
+ pslld(dst, 32 - cnt);
+ if (nlz != 0) psrld(dst, nlz);
+ }
+ } else {
+ push(eax);
+ mov(eax, Immediate(src));
+ movd(dst, Operand(eax));
+ pop(eax);
+ }
+ }
+}
+
+void TurboAssembler::Move(XMMRegister dst, uint64_t src) {
+ if (src == 0) {
+ pxor(dst, dst);
+ } else {
+ uint32_t lower = static_cast<uint32_t>(src);
+ uint32_t upper = static_cast<uint32_t>(src >> 32);
+ unsigned cnt = base::bits::CountPopulation(src);
+ unsigned nlz = base::bits::CountLeadingZeros64(src);
+ unsigned ntz = base::bits::CountTrailingZeros64(src);
+ if (nlz + cnt + ntz == 64) {
+ pcmpeqd(dst, dst);
+ if (ntz == 0) {
+ psrlq(dst, 64 - cnt);
+ } else {
+ psllq(dst, 64 - cnt);
+ if (nlz != 0) psrlq(dst, nlz);
+ }
+ } else if (lower == 0) {
+ Move(dst, upper);
+ psllq(dst, 32);
+ } else if (CpuFeatures::IsSupported(SSE4_1)) {
+ CpuFeatureScope scope(this, SSE4_1);
+ push(eax);
+ Move(eax, Immediate(lower));
+ movd(dst, Operand(eax));
+ if (upper != lower) {
+ Move(eax, Immediate(upper));
+ }
+ pinsrd(dst, Operand(eax), 1);
+ pop(eax);
+ } else {
+ push(Immediate(upper));
+ push(Immediate(lower));
+ movsd(dst, Operand(esp, 0));
+ add(esp, Immediate(kDoubleSize));
+ }
+ }
+}
+
+void TurboAssembler::Pshufhw(XMMRegister dst, Operand src, uint8_t shuffle) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vpshufhw(dst, src, shuffle);
+ } else {
+ pshufhw(dst, src, shuffle);
+ }
+}
+
+void TurboAssembler::Pshuflw(XMMRegister dst, Operand src, uint8_t shuffle) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vpshuflw(dst, src, shuffle);
+ } else {
+ pshuflw(dst, src, shuffle);
+ }
+}
+
+void TurboAssembler::Pshufd(XMMRegister dst, Operand src, uint8_t shuffle) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vpshufd(dst, src, shuffle);
+ } else {
+ pshufd(dst, src, shuffle);
+ }
+}
+
+void TurboAssembler::Psraw(XMMRegister dst, uint8_t shift) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vpsraw(dst, dst, shift);
+ } else {
+ psraw(dst, shift);
+ }
+}
+
+void TurboAssembler::Psrlw(XMMRegister dst, uint8_t shift) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vpsrlw(dst, dst, shift);
+ } else {
+ psrlw(dst, shift);
+ }
+}
+
+void TurboAssembler::Psrlq(XMMRegister dst, uint8_t shift) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vpsrlq(dst, dst, shift);
+ } else {
+ psrlq(dst, shift);
+ }
+}
+
+void TurboAssembler::Psignb(XMMRegister dst, Operand src) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vpsignb(dst, dst, src);
+ return;
+ }
+ if (CpuFeatures::IsSupported(SSSE3)) {
+ CpuFeatureScope sse_scope(this, SSSE3);
+ psignb(dst, src);
+ return;
+ }
+ FATAL("no AVX or SSE3 support");
+}
+
+void TurboAssembler::Psignw(XMMRegister dst, Operand src) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vpsignw(dst, dst, src);
+ return;
+ }
+ if (CpuFeatures::IsSupported(SSSE3)) {
+ CpuFeatureScope sse_scope(this, SSSE3);
+ psignw(dst, src);
+ return;
+ }
+ FATAL("no AVX or SSE3 support");
+}
+
+void TurboAssembler::Psignd(XMMRegister dst, Operand src) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vpsignd(dst, dst, src);
+ return;
+ }
+ if (CpuFeatures::IsSupported(SSSE3)) {
+ CpuFeatureScope sse_scope(this, SSSE3);
+ psignd(dst, src);
+ return;
+ }
+ FATAL("no AVX or SSE3 support");
+}
+
+void TurboAssembler::Pshufb(XMMRegister dst, XMMRegister src, Operand mask) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vpshufb(dst, src, mask);
+ return;
+ }
+ if (CpuFeatures::IsSupported(SSSE3)) {
+ // Make sure these are different so that we won't overwrite mask.
+ DCHECK(!mask.is_reg(dst));
+ CpuFeatureScope sse_scope(this, SSSE3);
+ if (dst != src) {
+ movapd(dst, src);
+ }
+ pshufb(dst, mask);
+ return;
+ }
+ FATAL("no AVX or SSE3 support");
+}
+
+void TurboAssembler::Pblendw(XMMRegister dst, Operand src, uint8_t imm8) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vpblendw(dst, dst, src, imm8);
+ return;
+ }
+ if (CpuFeatures::IsSupported(SSE4_1)) {
+ CpuFeatureScope sse_scope(this, SSE4_1);
+ pblendw(dst, src, imm8);
+ return;
+ }
+ FATAL("no AVX or SSE4.1 support");
+}
+
+void TurboAssembler::Palignr(XMMRegister dst, Operand src, uint8_t imm8) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vpalignr(dst, dst, src, imm8);
+ return;
+ }
+ if (CpuFeatures::IsSupported(SSSE3)) {
+ CpuFeatureScope sse_scope(this, SSSE3);
+ palignr(dst, src, imm8);
+ return;
+ }
+ FATAL("no AVX or SSE3 support");
+}
+
+void TurboAssembler::Pextrb(Register dst, XMMRegister src, uint8_t imm8) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vpextrb(dst, src, imm8);
+ return;
+ }
+ if (CpuFeatures::IsSupported(SSE4_1)) {
+ CpuFeatureScope sse_scope(this, SSE4_1);
+ pextrb(dst, src, imm8);
+ return;
+ }
+ FATAL("no AVX or SSE4.1 support");
+}
+
+void TurboAssembler::Pextrw(Register dst, XMMRegister src, uint8_t imm8) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vpextrw(dst, src, imm8);
+ return;
+ }
+ if (CpuFeatures::IsSupported(SSE4_1)) {
+ CpuFeatureScope sse_scope(this, SSE4_1);
+ pextrw(dst, src, imm8);
+ return;
+ }
+ FATAL("no AVX or SSE4.1 support");
+}
+
+void TurboAssembler::Pextrd(Register dst, XMMRegister src, uint8_t imm8) {
+ if (imm8 == 0) {
+ Movd(dst, src);
+ return;
+ }
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vpextrd(dst, src, imm8);
+ return;
+ }
+ if (CpuFeatures::IsSupported(SSE4_1)) {
+ CpuFeatureScope sse_scope(this, SSE4_1);
+ pextrd(dst, src, imm8);
+ return;
+ }
+ // Without AVX or SSE, we can only have 64-bit values in xmm registers.
+ // We don't have an xmm scratch register, so move the data via the stack. This
+ // path is rarely required, so it's acceptable to be slow.
+ DCHECK_LT(imm8, 2);
+ AllocateStackSpace(kDoubleSize);
+ movsd(Operand(esp, 0), src);
+ mov(dst, Operand(esp, imm8 * kUInt32Size));
+ add(esp, Immediate(kDoubleSize));
+}
+
+void TurboAssembler::Pinsrb(XMMRegister dst, Operand src, int8_t imm8) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vpinsrb(dst, dst, src, imm8);
+ return;
+ }
+ if (CpuFeatures::IsSupported(SSE4_1)) {
+ CpuFeatureScope sse_scope(this, SSE4_1);
+ pinsrb(dst, src, imm8);
+ return;
+ }
+ FATAL("no AVX or SSE4.1 support");
+}
+
+void TurboAssembler::Pinsrd(XMMRegister dst, Operand src, uint8_t imm8) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vpinsrd(dst, dst, src, imm8);
+ return;
+ }
+ if (CpuFeatures::IsSupported(SSE4_1)) {
+ CpuFeatureScope sse_scope(this, SSE4_1);
+ pinsrd(dst, src, imm8);
+ return;
+ }
+ // Without AVX or SSE, we can only have 64-bit values in xmm registers.
+ // We don't have an xmm scratch register, so move the data via the stack. This
+ // path is rarely required, so it's acceptable to be slow.
+ DCHECK_LT(imm8, 2);
+ AllocateStackSpace(kDoubleSize);
+ // Write original content of {dst} to the stack.
+ movsd(Operand(esp, 0), dst);
+ // Overwrite the portion specified in {imm8}.
+ if (src.is_reg_only()) {
+ mov(Operand(esp, imm8 * kUInt32Size), src.reg());
+ } else {
+ movss(dst, src);
+ movss(Operand(esp, imm8 * kUInt32Size), dst);
+ }
+ // Load back the full value into {dst}.
+ movsd(dst, Operand(esp, 0));
+ add(esp, Immediate(kDoubleSize));
+}
+
+void TurboAssembler::Pinsrw(XMMRegister dst, Operand src, int8_t imm8) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vpinsrw(dst, dst, src, imm8);
+ return;
+ } else {
+ pinsrw(dst, src, imm8);
+ return;
+ }
+}
+
+void TurboAssembler::Vbroadcastss(XMMRegister dst, Operand src) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope avx_scope(this, AVX);
+ vbroadcastss(dst, src);
+ return;
+ }
+ movss(dst, src);
+ shufps(dst, dst, static_cast<byte>(0));
+}
+
+void TurboAssembler::Lzcnt(Register dst, Operand src) {
+ if (CpuFeatures::IsSupported(LZCNT)) {
+ CpuFeatureScope scope(this, LZCNT);
+ lzcnt(dst, src);
+ return;
+ }
+ Label not_zero_src;
+ bsr(dst, src);
+ j(not_zero, ¬_zero_src, Label::kNear);
+ mov(dst, 63); // 63^31 == 32
+ bind(¬_zero_src);
+ xor_(dst, Immediate(31)); // for x in [0..31], 31^x == 31-x.
+}
+
+void TurboAssembler::Tzcnt(Register dst, Operand src) {
+ if (CpuFeatures::IsSupported(BMI1)) {
+ CpuFeatureScope scope(this, BMI1);
+ tzcnt(dst, src);
+ return;
+ }
+ Label not_zero_src;
+ bsf(dst, src);
+ j(not_zero, ¬_zero_src, Label::kNear);
+ mov(dst, 32); // The result of tzcnt is 32 if src = 0.
+ bind(¬_zero_src);
+}
+
+void TurboAssembler::Popcnt(Register dst, Operand src) {
+ if (CpuFeatures::IsSupported(POPCNT)) {
+ CpuFeatureScope scope(this, POPCNT);
+ popcnt(dst, src);
+ return;
+ }
+ FATAL("no POPCNT support");
+}
+
+void MacroAssembler::LoadWeakValue(Register in_out, Label* target_if_cleared) {
+ cmp(in_out, Immediate(kClearedWeakHeapObjectLower32));
+ j(equal, target_if_cleared);
+
+ and_(in_out, Immediate(~kWeakHeapObjectMask));
+}
+
+void MacroAssembler::IncrementCounter(StatsCounter* counter, int value,
+ Register scratch) {
+ DCHECK_GT(value, 0);
+ if (FLAG_native_code_counters && counter->Enabled()) {
+ Operand operand =
+ ExternalReferenceAsOperand(ExternalReference::Create(counter), scratch);
+ if (value == 1) {
+ inc(operand);
+ } else {
+ add(operand, Immediate(value));
+ }
+ }
+}
+
+void MacroAssembler::DecrementCounter(StatsCounter* counter, int value,
+ Register scratch) {
+ DCHECK_GT(value, 0);
+ if (FLAG_native_code_counters && counter->Enabled()) {
+ Operand operand =
+ ExternalReferenceAsOperand(ExternalReference::Create(counter), scratch);
+ if (value == 1) {
+ dec(operand);
+ } else {
+ sub(operand, Immediate(value));
+ }
+ }
+}
+
+void TurboAssembler::Assert(Condition cc, AbortReason reason) {
+ if (emit_debug_code()) Check(cc, reason);
+}
+
+void TurboAssembler::AssertUnreachable(AbortReason reason) {
+ if (emit_debug_code()) Abort(reason);
+}
+
+void TurboAssembler::Check(Condition cc, AbortReason reason) {
+ Label L;
+ j(cc, &L);
+ Abort(reason);
+ // will not return here
+ bind(&L);
+}
+
+void TurboAssembler::CheckStackAlignment() {
+ int frame_alignment = base::OS::ActivationFrameAlignment();
+ int frame_alignment_mask = frame_alignment - 1;
+ if (frame_alignment > kSystemPointerSize) {
+ DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
+ Label alignment_as_expected;
+ test(esp, Immediate(frame_alignment_mask));
+ j(zero, &alignment_as_expected);
+ // Abort if stack is not aligned.
+ int3();
+ bind(&alignment_as_expected);
+ }
+}
+
+void TurboAssembler::Abort(AbortReason reason) {
+#ifdef DEBUG
+ const char* msg = GetAbortReason(reason);
+ RecordComment("Abort message: ");
+ RecordComment(msg);
+#endif
+
+ // Avoid emitting call to builtin if requested.
+ if (trap_on_abort()) {
+ int3();
+ return;
+ }
+
+ if (should_abort_hard()) {
+ // We don't care if we constructed a frame. Just pretend we did.
+ FrameScope assume_frame(this, StackFrame::NONE);
+ PrepareCallCFunction(1, eax);
+ mov(Operand(esp, 0), Immediate(static_cast<int>(reason)));
+ CallCFunction(ExternalReference::abort_with_reason(), 1);
+ return;
+ }
+
+ Move(edx, Smi::FromInt(static_cast<int>(reason)));
+
+ // Disable stub call restrictions to always allow calls to abort.
+ if (!has_frame()) {
+ // We don't actually want to generate a pile of code for this, so just
+ // claim there is a stack frame, without generating one.
+ FrameScope scope(this, StackFrame::NONE);
+ Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
+ } else {
+ Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
+ }
+ // will not return here
+ int3();
+}
+
+void TurboAssembler::PrepareCallCFunction(int num_arguments, Register scratch) {
+ int frame_alignment = base::OS::ActivationFrameAlignment();
+ if (frame_alignment != 0) {
+ // Make stack end at alignment and make room for num_arguments words
+ // and the original value of esp.
+ mov(scratch, esp);
+ AllocateStackSpace((num_arguments + 1) * kSystemPointerSize);
+ DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
+ and_(esp, -frame_alignment);
+ mov(Operand(esp, num_arguments * kSystemPointerSize), scratch);
+ } else {
+ AllocateStackSpace(num_arguments * kSystemPointerSize);
+ }
+}
+
+void TurboAssembler::CallCFunction(ExternalReference function,
+ int num_arguments) {
+ // Trashing eax is ok as it will be the return value.
+ Move(eax, Immediate(function));
+ CallCFunction(eax, num_arguments);
+}
+
+void TurboAssembler::CallCFunction(Register function, int num_arguments) {
+ DCHECK_LE(num_arguments, kMaxCParameters);
+ DCHECK(has_frame());
+ // Check stack alignment.
+ if (emit_debug_code()) {
+ CheckStackAlignment();
+ }
+
+ // Save the frame pointer and PC so that the stack layout remains iterable,
+ // even without an ExitFrame which normally exists between JS and C frames.
+ // Find two caller-saved scratch registers.
+ Register pc_scratch = eax;
+ Register scratch = ecx;
+ if (function == eax) pc_scratch = edx;
+ if (function == ecx) scratch = edx;
+ PushPC();
+ pop(pc_scratch);
+
+ // See x64 code for reasoning about how to address the isolate data fields.
+ DCHECK_IMPLIES(!root_array_available(), isolate() != nullptr);
+ mov(root_array_available()
+ ? Operand(kRootRegister, IsolateData::fast_c_call_caller_pc_offset())
+ : ExternalReferenceAsOperand(
+ ExternalReference::fast_c_call_caller_pc_address(isolate()),
+ scratch),
+ pc_scratch);
+ mov(root_array_available()
+ ? Operand(kRootRegister, IsolateData::fast_c_call_caller_fp_offset())
+ : ExternalReferenceAsOperand(
+ ExternalReference::fast_c_call_caller_fp_address(isolate()),
+ scratch),
+ ebp);
+
+ call(function);
+
+ // We don't unset the PC; the FP is the source of truth.
+ mov(root_array_available()
+ ? Operand(kRootRegister, IsolateData::fast_c_call_caller_fp_offset())
+ : ExternalReferenceAsOperand(
+ ExternalReference::fast_c_call_caller_fp_address(isolate()),
+ scratch),
+ Immediate(0));
+
+ if (base::OS::ActivationFrameAlignment() != 0) {
+ mov(esp, Operand(esp, num_arguments * kSystemPointerSize));
+ } else {
+ add(esp, Immediate(num_arguments * kSystemPointerSize));
+ }
+}
+
+void TurboAssembler::PushPC() {
+ // Push the current PC onto the stack as "return address" via calling
+ // the next instruction.
+ Label get_pc;
+ call(&get_pc);
+ bind(&get_pc);
+}
+
+void TurboAssembler::Call(Handle<Code> code_object, RelocInfo::Mode rmode) {
+ DCHECK_IMPLIES(options().isolate_independent_code,
+ Builtins::IsIsolateIndependentBuiltin(*code_object));
+ if (options().inline_offheap_trampolines) {
+ int builtin_index = Builtins::kNoBuiltinId;
+ if (isolate()->builtins()->IsBuiltinHandle(code_object, &builtin_index)) {
+ // Inline the trampoline.
+ CallBuiltin(builtin_index);
+ return;
+ }
+ }
+ DCHECK(RelocInfo::IsCodeTarget(rmode));
+ call(code_object, rmode);
+}
+
+void TurboAssembler::LoadEntryFromBuiltinIndex(Register builtin_index) {
+ STATIC_ASSERT(kSystemPointerSize == 4);
+ STATIC_ASSERT(kSmiShiftSize == 0);
+ STATIC_ASSERT(kSmiTagSize == 1);
+ STATIC_ASSERT(kSmiTag == 0);
+
+ // The builtin_index register contains the builtin index as a Smi.
+ // Untagging is folded into the indexing operand below (we use
+ // times_half_system_pointer_size instead of times_system_pointer_size since
+ // smis are already shifted by one).
+ mov(builtin_index,
+ Operand(kRootRegister, builtin_index, times_half_system_pointer_size,
+ IsolateData::builtin_entry_table_offset()));
+}
+
+void TurboAssembler::CallBuiltinByIndex(Register builtin_index) {
+ LoadEntryFromBuiltinIndex(builtin_index);
+ call(builtin_index);
+}
+
+void TurboAssembler::CallBuiltin(int builtin_index) {
+ DCHECK(Builtins::IsBuiltinId(builtin_index));
+ RecordCommentForOffHeapTrampoline(builtin_index);
+ CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
+ EmbeddedData d = EmbeddedData::FromBlob();
+ Address entry = d.InstructionStartOfBuiltin(builtin_index);
+ call(entry, RelocInfo::OFF_HEAP_TARGET);
+}
+
+void TurboAssembler::LoadCodeObjectEntry(Register destination,
+ Register code_object) {
+ // Code objects are called differently depending on whether we are generating
+ // builtin code (which will later be embedded into the binary) or compiling
+ // user JS code at runtime.
+ // * Builtin code runs in --jitless mode and thus must not call into on-heap
+ // Code targets. Instead, we dispatch through the builtins entry table.
+ // * Codegen at runtime does not have this restriction and we can use the
+ // shorter, branchless instruction sequence. The assumption here is that
+ // targets are usually generated code and not builtin Code objects.
+
+ if (options().isolate_independent_code) {
+ DCHECK(root_array_available());
+ Label if_code_is_off_heap, out;
+
+ // Check whether the Code object is an off-heap trampoline. If so, call its
+ // (off-heap) entry point directly without going through the (on-heap)
+ // trampoline. Otherwise, just call the Code object as always.
+ test(FieldOperand(code_object, Code::kFlagsOffset),
+ Immediate(Code::IsOffHeapTrampoline::kMask));
+ j(not_equal, &if_code_is_off_heap);
+
+ // Not an off-heap trampoline, the entry point is at
+ // Code::raw_instruction_start().
+ Move(destination, code_object);
+ add(destination, Immediate(Code::kHeaderSize - kHeapObjectTag));
+ jmp(&out);
+
+ // An off-heap trampoline, the entry point is loaded from the builtin entry
+ // table.
+ bind(&if_code_is_off_heap);
+ mov(destination, FieldOperand(code_object, Code::kBuiltinIndexOffset));
+ mov(destination,
+ Operand(kRootRegister, destination, times_system_pointer_size,
+ IsolateData::builtin_entry_table_offset()));
+
+ bind(&out);
+ } else {
+ Move(destination, code_object);
+ add(destination, Immediate(Code::kHeaderSize - kHeapObjectTag));
+ }
+}
+
+void TurboAssembler::CallCodeObject(Register code_object) {
+ LoadCodeObjectEntry(code_object, code_object);
+ call(code_object);
+}
+
+void TurboAssembler::JumpCodeObject(Register code_object) {
+ LoadCodeObjectEntry(code_object, code_object);
+ jmp(code_object);
+}
+
+void TurboAssembler::Jump(const ExternalReference& reference) {
+ DCHECK(root_array_available());
+ jmp(Operand(kRootRegister, RootRegisterOffsetForExternalReferenceTableEntry(
+ isolate(), reference)));
+}
+
+void TurboAssembler::Jump(Handle<Code> code_object, RelocInfo::Mode rmode) {
+ DCHECK_IMPLIES(options().isolate_independent_code,
+ Builtins::IsIsolateIndependentBuiltin(*code_object));
+ if (options().inline_offheap_trampolines) {
+ int builtin_index = Builtins::kNoBuiltinId;
+ if (isolate()->builtins()->IsBuiltinHandle(code_object, &builtin_index)) {
+ // Inline the trampoline.
+ RecordCommentForOffHeapTrampoline(builtin_index);
+ CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
+ EmbeddedData d = EmbeddedData::FromBlob();
+ Address entry = d.InstructionStartOfBuiltin(builtin_index);
+ jmp(entry, RelocInfo::OFF_HEAP_TARGET);
+ return;
+ }
+ }
+ DCHECK(RelocInfo::IsCodeTarget(rmode));
+ jmp(code_object, rmode);
+}
+
+void TurboAssembler::RetpolineCall(Register reg) {
+ Label setup_return, setup_target, inner_indirect_branch, capture_spec;
+
+ jmp(&setup_return); // Jump past the entire retpoline below.
+
+ bind(&inner_indirect_branch);
+ call(&setup_target);
+
+ bind(&capture_spec);
+ pause();
+ jmp(&capture_spec);
+
+ bind(&setup_target);
+ mov(Operand(esp, 0), reg);
+ ret(0);
+
+ bind(&setup_return);
+ call(&inner_indirect_branch); // Callee will return after this instruction.
+}
+
+void TurboAssembler::RetpolineCall(Address destination, RelocInfo::Mode rmode) {
+ Label setup_return, setup_target, inner_indirect_branch, capture_spec;
+
+ jmp(&setup_return); // Jump past the entire retpoline below.
+
+ bind(&inner_indirect_branch);
+ call(&setup_target);
+
+ bind(&capture_spec);
+ pause();
+ jmp(&capture_spec);
+
+ bind(&setup_target);
+ mov(Operand(esp, 0), destination, rmode);
+ ret(0);
+
+ bind(&setup_return);
+ call(&inner_indirect_branch); // Callee will return after this instruction.
+}
+
+void TurboAssembler::RetpolineJump(Register reg) {
+ Label setup_target, capture_spec;
+
+ call(&setup_target);
+
+ bind(&capture_spec);
+ pause();
+ jmp(&capture_spec);
+
+ bind(&setup_target);
+ mov(Operand(esp, 0), reg);
+ ret(0);
+}
+
+void TurboAssembler::CheckPageFlag(Register object, Register scratch, int mask,
+ Condition cc, Label* condition_met,
+ Label::Distance condition_met_distance) {
+ DCHECK(cc == zero || cc == not_zero);
+ if (scratch == object) {
+ and_(scratch, Immediate(~kPageAlignmentMask));
+ } else {
+ mov(scratch, Immediate(~kPageAlignmentMask));
+ and_(scratch, object);
+ }
+ if (mask < (1 << kBitsPerByte)) {
+ test_b(Operand(scratch, BasicMemoryChunk::kFlagsOffset), Immediate(mask));
+ } else {
+ test(Operand(scratch, BasicMemoryChunk::kFlagsOffset), Immediate(mask));
+ }
+ j(cc, condition_met, condition_met_distance);
+}
+
+void TurboAssembler::ComputeCodeStartAddress(Register dst) {
+ // In order to get the address of the current instruction, we first need
+ // to use a call and then use a pop, thus pushing the return address to
+ // the stack and then popping it into the register.
+ Label current;
+ call(¤t);
+ int pc = pc_offset();
+ bind(¤t);
+ pop(dst);
+ if (pc != 0) {
+ sub(dst, Immediate(pc));
+ }
+}
+
+void TurboAssembler::CallForDeoptimization(Builtins::Name target, int,
+ Label* exit, DeoptimizeKind kind,
+ Label*) {
+ CallBuiltin(target);
+ DCHECK_EQ(SizeOfCodeGeneratedSince(exit),
+ (kind == DeoptimizeKind::kLazy)
+ ? Deoptimizer::kLazyDeoptExitSize
+ : Deoptimizer::kNonLazyDeoptExitSize);
+ USE(exit, kind);
+}
+
+void TurboAssembler::Trap() { int3(); }
+void TurboAssembler::DebugBreak() { int3(); }
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_IA32
diff --git a/src/codegen/ia32/macro-assembler-ia32.h b/src/codegen/ia32/macro-assembler-ia32.h
new file mode 100644
index 0000000..33635b0
--- /dev/null
+++ b/src/codegen/ia32/macro-assembler-ia32.h
@@ -0,0 +1,897 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef INCLUDED_FROM_MACRO_ASSEMBLER_H
+#error This header must be included via macro-assembler.h
+#endif
+
+#ifndef V8_CODEGEN_IA32_MACRO_ASSEMBLER_IA32_H_
+#define V8_CODEGEN_IA32_MACRO_ASSEMBLER_IA32_H_
+
+#include "src/codegen/assembler.h"
+#include "src/codegen/bailout-reason.h"
+#include "src/codegen/ia32/assembler-ia32.h"
+#include "src/common/globals.h"
+
+namespace v8 {
+namespace internal {
+
+// Convenience for platform-independent signatures. We do not normally
+// distinguish memory operands from other operands on ia32.
+using MemOperand = Operand;
+
+enum RememberedSetAction { EMIT_REMEMBERED_SET, OMIT_REMEMBERED_SET };
+enum SmiCheck { INLINE_SMI_CHECK, OMIT_SMI_CHECK };
+
+// TODO(victorgomes): Move definition to macro-assembler.h, once all other
+// platforms are updated.
+enum class StackLimitKind { kInterruptStackLimit, kRealStackLimit };
+
+// Convenient class to access arguments below the stack pointer.
+class StackArgumentsAccessor {
+ public:
+ // argc = the number of arguments not including the receiver.
+ explicit StackArgumentsAccessor(Register argc) : argc_(argc) {
+ DCHECK_NE(argc_, no_reg);
+ }
+
+ // Argument 0 is the receiver (despite argc not including the receiver).
+ Operand operator[](int index) const { return GetArgumentOperand(index); }
+
+ Operand GetArgumentOperand(int index) const;
+ Operand GetReceiverOperand() const { return GetArgumentOperand(0); }
+
+ private:
+ const Register argc_;
+
+ DISALLOW_IMPLICIT_CONSTRUCTORS(StackArgumentsAccessor);
+};
+
+class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase {
+ public:
+ using TurboAssemblerBase::TurboAssemblerBase;
+
+ void CheckPageFlag(Register object, Register scratch, int mask, Condition cc,
+ Label* condition_met,
+ Label::Distance condition_met_distance = Label::kFar);
+
+ // Activation support.
+ void EnterFrame(StackFrame::Type type);
+ void EnterFrame(StackFrame::Type type, bool load_constant_pool_pointer_reg) {
+ // Out-of-line constant pool not implemented on ia32.
+ UNREACHABLE();
+ }
+ void LeaveFrame(StackFrame::Type type);
+
+// Allocate stack space of given size (i.e. decrement {esp} by the value
+// stored in the given register, or by a constant). If you need to perform a
+// stack check, do it before calling this function because this function may
+// write into the newly allocated space. It may also overwrite the given
+// register's value, in the version that takes a register.
+#ifdef V8_OS_WIN
+ void AllocateStackSpace(Register bytes_scratch);
+ void AllocateStackSpace(int bytes);
+#else
+ void AllocateStackSpace(Register bytes) { sub(esp, bytes); }
+ void AllocateStackSpace(int bytes) { sub(esp, Immediate(bytes)); }
+#endif
+
+ // Print a message to stdout and abort execution.
+ void Abort(AbortReason reason);
+
+ // Calls Abort(msg) if the condition cc is not satisfied.
+ // Use --debug_code to enable.
+ void Assert(Condition cc, AbortReason reason);
+
+ // Like Assert(), but without condition.
+ // Use --debug_code to enable.
+ void AssertUnreachable(AbortReason reason);
+
+ // Like Assert(), but always enabled.
+ void Check(Condition cc, AbortReason reason);
+
+ // Check that the stack is aligned.
+ void CheckStackAlignment();
+
+ // Move a constant into a destination using the most efficient encoding.
+ void Move(Register dst, const Immediate& src);
+ void Move(Register dst, Smi src) { Move(dst, Immediate(src)); }
+ void Move(Register dst, Handle<HeapObject> src);
+ void Move(Register dst, Register src);
+ void Move(Operand dst, const Immediate& src);
+
+ // Move an immediate into an XMM register.
+ void Move(XMMRegister dst, uint32_t src);
+ void Move(XMMRegister dst, uint64_t src);
+ void Move(XMMRegister dst, float src) { Move(dst, bit_cast<uint32_t>(src)); }
+ void Move(XMMRegister dst, double src) { Move(dst, bit_cast<uint64_t>(src)); }
+
+ void Call(Register reg) { call(reg); }
+ void Call(Label* target) { call(target); }
+ void Call(Handle<Code> code_object, RelocInfo::Mode rmode);
+
+ // Load the builtin given by the Smi in |builtin_index| into the same
+ // register.
+ void LoadEntryFromBuiltinIndex(Register builtin_index);
+ void CallBuiltinByIndex(Register builtin_index) override;
+ void CallBuiltin(int builtin_index);
+
+ void LoadCodeObjectEntry(Register destination, Register code_object) override;
+ void CallCodeObject(Register code_object) override;
+ void JumpCodeObject(Register code_object) override;
+ void Jump(const ExternalReference& reference) override;
+
+ void RetpolineCall(Register reg);
+ void RetpolineCall(Address destination, RelocInfo::Mode rmode);
+
+ void Jump(Handle<Code> code_object, RelocInfo::Mode rmode);
+
+ void LoadMap(Register destination, Register object);
+
+ void RetpolineJump(Register reg);
+
+ void Trap() override;
+ void DebugBreak() override;
+
+ void CallForDeoptimization(Builtins::Name target, int deopt_id, Label* exit,
+ DeoptimizeKind kind,
+ Label* jump_deoptimization_entry_label);
+
+ // Jump the register contains a smi.
+ inline void JumpIfSmi(Register value, Label* smi_label,
+ Label::Distance distance = Label::kFar) {
+ test(value, Immediate(kSmiTagMask));
+ j(zero, smi_label, distance);
+ }
+ // Jump if the operand is a smi.
+ inline void JumpIfSmi(Operand value, Label* smi_label,
+ Label::Distance distance = Label::kFar) {
+ test(value, Immediate(kSmiTagMask));
+ j(zero, smi_label, distance);
+ }
+
+ void JumpIfEqual(Register a, int32_t b, Label* dest) {
+ cmp(a, Immediate(b));
+ j(equal, dest);
+ }
+
+ void JumpIfLessThan(Register a, int32_t b, Label* dest) {
+ cmp(a, Immediate(b));
+ j(less, dest);
+ }
+
+ void SmiUntag(Register reg) { sar(reg, kSmiTagSize); }
+
+ // Removes current frame and its arguments from the stack preserving the
+ // arguments and a return address pushed to the stack for the next call. Both
+ // |callee_args_count| and |caller_args_count| do not include receiver.
+ // |callee_args_count| is not modified. |caller_args_count| is trashed.
+ // |number_of_temp_values_after_return_address| specifies the number of words
+ // pushed to the stack after the return address. This is to allow "allocation"
+ // of scratch registers that this function requires by saving their values on
+ // the stack.
+ void PrepareForTailCall(Register callee_args_count,
+ Register caller_args_count, Register scratch0,
+ Register scratch1,
+ int number_of_temp_values_after_return_address);
+
+ // Before calling a C-function from generated code, align arguments on stack.
+ // After aligning the frame, arguments must be stored in esp[0], esp[4],
+ // etc., not pushed. The argument count assumes all arguments are word sized.
+ // Some compilers/platforms require the stack to be aligned when calling
+ // C++ code.
+ // Needs a scratch register to do some arithmetic. This register will be
+ // trashed.
+ void PrepareCallCFunction(int num_arguments, Register scratch);
+
+ // Calls a C function and cleans up the space for arguments allocated
+ // by PrepareCallCFunction. The called function is not allowed to trigger a
+ // garbage collection, since that might move the code and invalidate the
+ // return address (unless this is somehow accounted for by the called
+ // function).
+ void CallCFunction(ExternalReference function, int num_arguments);
+ void CallCFunction(Register function, int num_arguments);
+
+ void ShlPair(Register high, Register low, uint8_t imm8);
+ void ShlPair_cl(Register high, Register low);
+ void ShrPair(Register high, Register low, uint8_t imm8);
+ void ShrPair_cl(Register high, Register low);
+ void SarPair(Register high, Register low, uint8_t imm8);
+ void SarPair_cl(Register high, Register low);
+
+ // Generates function and stub prologue code.
+ void StubPrologue(StackFrame::Type type);
+ void Prologue();
+
+ void Lzcnt(Register dst, Register src) { Lzcnt(dst, Operand(src)); }
+ void Lzcnt(Register dst, Operand src);
+
+ void Tzcnt(Register dst, Register src) { Tzcnt(dst, Operand(src)); }
+ void Tzcnt(Register dst, Operand src);
+
+ void Popcnt(Register dst, Register src) { Popcnt(dst, Operand(src)); }
+ void Popcnt(Register dst, Operand src);
+
+ void PushReturnAddressFrom(Register src) { push(src); }
+ void PopReturnAddressTo(Register dst) { pop(dst); }
+
+ void Ret();
+
+ // Root register utility functions.
+
+ void InitializeRootRegister();
+
+ void LoadRoot(Register destination, RootIndex index) override;
+
+ // Indirect root-relative loads.
+ void LoadFromConstantsTable(Register destination,
+ int constant_index) override;
+ void LoadRootRegisterOffset(Register destination, intptr_t offset) override;
+ void LoadRootRelative(Register destination, int32_t offset) override;
+
+ void PushPC();
+
+ enum class PushArrayOrder { kNormal, kReverse };
+ // `array` points to the first element (the lowest address).
+ // `array` and `size` are not modified.
+ void PushArray(Register array, Register size, Register scratch,
+ PushArrayOrder order = PushArrayOrder::kNormal);
+
+ // Operand pointing to an external reference.
+ // May emit code to set up the scratch register. The operand is
+ // only guaranteed to be correct as long as the scratch register
+ // isn't changed.
+ // If the operand is used more than once, use a scratch register
+ // that is guaranteed not to be clobbered.
+ Operand ExternalReferenceAsOperand(ExternalReference reference,
+ Register scratch);
+ Operand ExternalReferenceAddressAsOperand(ExternalReference reference);
+ Operand HeapObjectAsOperand(Handle<HeapObject> object);
+
+ void LoadAddress(Register destination, ExternalReference source);
+
+ void CompareRoot(Register with, RootIndex index);
+ void CompareRoot(Register with, Register scratch, RootIndex index);
+
+ // Return and drop arguments from stack, where the number of arguments
+ // may be bigger than 2^16 - 1. Requires a scratch register.
+ void Ret(int bytes_dropped, Register scratch);
+
+ void Pshufhw(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
+ Pshufhw(dst, Operand(src), shuffle);
+ }
+ void Pshufhw(XMMRegister dst, Operand src, uint8_t shuffle);
+ void Pshuflw(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
+ Pshuflw(dst, Operand(src), shuffle);
+ }
+ void Pshuflw(XMMRegister dst, Operand src, uint8_t shuffle);
+ void Pshufd(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
+ Pshufd(dst, Operand(src), shuffle);
+ }
+ void Pshufd(XMMRegister dst, Operand src, uint8_t shuffle);
+ void Psraw(XMMRegister dst, uint8_t shift);
+ void Psrlw(XMMRegister dst, uint8_t shift);
+ void Psrlq(XMMRegister dst, uint8_t shift);
+
+// SSE/SSE2 instructions with AVX version.
+#define AVX_OP2_WITH_TYPE(macro_name, name, dst_type, src_type) \
+ void macro_name(dst_type dst, src_type src) { \
+ if (CpuFeatures::IsSupported(AVX)) { \
+ CpuFeatureScope scope(this, AVX); \
+ v##name(dst, src); \
+ } else { \
+ name(dst, src); \
+ } \
+ }
+
+ AVX_OP2_WITH_TYPE(Rcpps, rcpps, XMMRegister, const Operand&)
+ AVX_OP2_WITH_TYPE(Rsqrtps, rsqrtps, XMMRegister, const Operand&)
+ AVX_OP2_WITH_TYPE(Movdqu, movdqu, XMMRegister, Operand)
+ AVX_OP2_WITH_TYPE(Movdqu, movdqu, Operand, XMMRegister)
+ AVX_OP2_WITH_TYPE(Movd, movd, XMMRegister, Register)
+ AVX_OP2_WITH_TYPE(Movd, movd, XMMRegister, Operand)
+ AVX_OP2_WITH_TYPE(Movd, movd, Register, XMMRegister)
+ AVX_OP2_WITH_TYPE(Movd, movd, Operand, XMMRegister)
+ AVX_OP2_WITH_TYPE(Cvtdq2ps, cvtdq2ps, XMMRegister, Operand)
+ AVX_OP2_WITH_TYPE(Cvtdq2ps, cvtdq2ps, XMMRegister, XMMRegister)
+ AVX_OP2_WITH_TYPE(Cvttps2dq, cvttps2dq, XMMRegister, XMMRegister)
+ AVX_OP2_WITH_TYPE(Sqrtps, sqrtps, XMMRegister, XMMRegister)
+ AVX_OP2_WITH_TYPE(Sqrtpd, sqrtpd, XMMRegister, XMMRegister)
+ AVX_OP2_WITH_TYPE(Sqrtpd, sqrtpd, XMMRegister, const Operand&)
+ AVX_OP2_WITH_TYPE(Movaps, movaps, XMMRegister, XMMRegister)
+ AVX_OP2_WITH_TYPE(Movapd, movapd, XMMRegister, XMMRegister)
+ AVX_OP2_WITH_TYPE(Movapd, movapd, XMMRegister, const Operand&)
+ AVX_OP2_WITH_TYPE(Movupd, movupd, XMMRegister, const Operand&)
+ AVX_OP2_WITH_TYPE(Pmovmskb, pmovmskb, Register, XMMRegister)
+ AVX_OP2_WITH_TYPE(Movmskps, movmskps, Register, XMMRegister)
+
+#undef AVX_OP2_WITH_TYPE
+
+// Only use these macros when non-destructive source of AVX version is not
+// needed.
+#define AVX_OP3_WITH_TYPE(macro_name, name, dst_type, src_type) \
+ void macro_name(dst_type dst, src_type src) { \
+ if (CpuFeatures::IsSupported(AVX)) { \
+ CpuFeatureScope scope(this, AVX); \
+ v##name(dst, dst, src); \
+ } else { \
+ name(dst, src); \
+ } \
+ }
+#define AVX_OP3_XO(macro_name, name) \
+ AVX_OP3_WITH_TYPE(macro_name, name, XMMRegister, XMMRegister) \
+ AVX_OP3_WITH_TYPE(macro_name, name, XMMRegister, Operand)
+
+ AVX_OP3_XO(Packsswb, packsswb)
+ AVX_OP3_XO(Packuswb, packuswb)
+ AVX_OP3_XO(Paddusb, paddusb)
+ AVX_OP3_XO(Pand, pand)
+ AVX_OP3_XO(Pcmpeqb, pcmpeqb)
+ AVX_OP3_XO(Pcmpeqw, pcmpeqw)
+ AVX_OP3_XO(Pcmpeqd, pcmpeqd)
+ AVX_OP3_XO(Por, por)
+ AVX_OP3_XO(Psubb, psubb)
+ AVX_OP3_XO(Psubw, psubw)
+ AVX_OP3_XO(Psubd, psubd)
+ AVX_OP3_XO(Psubq, psubq)
+ AVX_OP3_XO(Punpcklbw, punpcklbw)
+ AVX_OP3_XO(Punpckhbw, punpckhbw)
+ AVX_OP3_XO(Punpckldq, punpckldq)
+ AVX_OP3_XO(Punpcklqdq, punpcklqdq)
+ AVX_OP3_XO(Pxor, pxor)
+ AVX_OP3_XO(Andps, andps)
+ AVX_OP3_XO(Andnps, andnps)
+ AVX_OP3_XO(Andpd, andpd)
+ AVX_OP3_XO(Xorps, xorps)
+ AVX_OP3_XO(Xorpd, xorpd)
+ AVX_OP3_XO(Sqrtss, sqrtss)
+ AVX_OP3_XO(Sqrtsd, sqrtsd)
+ AVX_OP3_XO(Orps, orps)
+ AVX_OP3_XO(Orpd, orpd)
+ AVX_OP3_XO(Andnpd, andnpd)
+
+#undef AVX_OP3_XO
+#undef AVX_OP3_WITH_TYPE
+
+// Only use this macro when dst and src1 is the same in SSE case.
+#define AVX_PACKED_OP3_WITH_TYPE(macro_name, name, dst_type, src_type) \
+ void macro_name(dst_type dst, dst_type src1, src_type src2) { \
+ if (CpuFeatures::IsSupported(AVX)) { \
+ CpuFeatureScope scope(this, AVX); \
+ v##name(dst, src1, src2); \
+ } else { \
+ DCHECK_EQ(dst, src1); \
+ name(dst, src2); \
+ } \
+ }
+#define AVX_PACKED_OP3(macro_name, name) \
+ AVX_PACKED_OP3_WITH_TYPE(macro_name, name, XMMRegister, XMMRegister) \
+ AVX_PACKED_OP3_WITH_TYPE(macro_name, name, XMMRegister, Operand)
+
+ AVX_PACKED_OP3(Addps, addps)
+ AVX_PACKED_OP3(Addpd, addpd)
+ AVX_PACKED_OP3(Subps, subps)
+ AVX_PACKED_OP3(Subpd, subpd)
+ AVX_PACKED_OP3(Mulpd, mulpd)
+ AVX_PACKED_OP3(Divpd, divpd)
+ AVX_PACKED_OP3(Cmpeqpd, cmpeqpd)
+ AVX_PACKED_OP3(Cmpneqpd, cmpneqpd)
+ AVX_PACKED_OP3(Cmpltpd, cmpltpd)
+ AVX_PACKED_OP3(Cmpleps, cmpleps)
+ AVX_PACKED_OP3(Cmplepd, cmplepd)
+ AVX_PACKED_OP3(Minps, minps)
+ AVX_PACKED_OP3(Minpd, minpd)
+ AVX_PACKED_OP3(Maxps, maxps)
+ AVX_PACKED_OP3(Maxpd, maxpd)
+ AVX_PACKED_OP3(Cmpunordps, cmpunordps)
+ AVX_PACKED_OP3(Cmpunordpd, cmpunordpd)
+ AVX_PACKED_OP3(Psllw, psllw)
+ AVX_PACKED_OP3(Pslld, pslld)
+ AVX_PACKED_OP3(Psllq, psllq)
+ AVX_PACKED_OP3(Psrlw, psrlw)
+ AVX_PACKED_OP3(Psrld, psrld)
+ AVX_PACKED_OP3(Psrlq, psrlq)
+ AVX_PACKED_OP3(Psraw, psraw)
+ AVX_PACKED_OP3(Psrad, psrad)
+ AVX_PACKED_OP3(Pmaddwd, pmaddwd)
+ AVX_PACKED_OP3(Paddd, paddd)
+ AVX_PACKED_OP3(Paddq, paddq)
+ AVX_PACKED_OP3(Psubq, psubq)
+ AVX_PACKED_OP3(Pmuludq, pmuludq)
+ AVX_PACKED_OP3(Pavgb, pavgb)
+ AVX_PACKED_OP3(Pavgw, pavgw)
+#undef AVX_PACKED_OP3
+
+ AVX_PACKED_OP3_WITH_TYPE(Psllw, psllw, XMMRegister, uint8_t)
+ AVX_PACKED_OP3_WITH_TYPE(Pslld, pslld, XMMRegister, uint8_t)
+ AVX_PACKED_OP3_WITH_TYPE(Psllq, psllq, XMMRegister, uint8_t)
+ AVX_PACKED_OP3_WITH_TYPE(Psrlw, psrlw, XMMRegister, uint8_t)
+ AVX_PACKED_OP3_WITH_TYPE(Psrld, psrld, XMMRegister, uint8_t)
+ AVX_PACKED_OP3_WITH_TYPE(Psrlq, psrlq, XMMRegister, uint8_t)
+ AVX_PACKED_OP3_WITH_TYPE(Psraw, psraw, XMMRegister, uint8_t)
+ AVX_PACKED_OP3_WITH_TYPE(Psrad, psrad, XMMRegister, uint8_t)
+#undef AVX_PACKED_OP3_WITH_TYPE
+
+// Non-SSE2 instructions.
+#define AVX_OP2_WITH_TYPE_SCOPE(macro_name, name, dst_type, src_type, \
+ sse_scope) \
+ void macro_name(dst_type dst, src_type src) { \
+ if (CpuFeatures::IsSupported(AVX)) { \
+ CpuFeatureScope scope(this, AVX); \
+ v##name(dst, src); \
+ return; \
+ } \
+ if (CpuFeatures::IsSupported(sse_scope)) { \
+ CpuFeatureScope scope(this, sse_scope); \
+ name(dst, src); \
+ return; \
+ } \
+ UNREACHABLE(); \
+ }
+#define AVX_OP2_XO_SSE3(macro_name, name) \
+ AVX_OP2_WITH_TYPE_SCOPE(macro_name, name, XMMRegister, XMMRegister, SSE3) \
+ AVX_OP2_WITH_TYPE_SCOPE(macro_name, name, XMMRegister, Operand, SSE3)
+ AVX_OP2_XO_SSE3(Movddup, movddup)
+
+#undef AVX_OP2_XO_SSE3
+
+#define AVX_OP2_XO_SSSE3(macro_name, name) \
+ AVX_OP2_WITH_TYPE_SCOPE(macro_name, name, XMMRegister, XMMRegister, SSSE3) \
+ AVX_OP2_WITH_TYPE_SCOPE(macro_name, name, XMMRegister, Operand, SSSE3)
+ AVX_OP2_XO_SSSE3(Pabsb, pabsb)
+ AVX_OP2_XO_SSSE3(Pabsw, pabsw)
+ AVX_OP2_XO_SSSE3(Pabsd, pabsd)
+
+#undef AVX_OP2_XO_SSE3
+
+#define AVX_OP2_XO_SSE4(macro_name, name) \
+ AVX_OP2_WITH_TYPE_SCOPE(macro_name, name, XMMRegister, XMMRegister, SSE4_1) \
+ AVX_OP2_WITH_TYPE_SCOPE(macro_name, name, XMMRegister, Operand, SSE4_1)
+
+ AVX_OP2_XO_SSE4(Ptest, ptest)
+ AVX_OP2_XO_SSE4(Pmovsxbw, pmovsxbw)
+ AVX_OP2_XO_SSE4(Pmovsxwd, pmovsxwd)
+ AVX_OP2_XO_SSE4(Pmovsxdq, pmovsxdq)
+ AVX_OP2_XO_SSE4(Pmovzxbw, pmovzxbw)
+ AVX_OP2_XO_SSE4(Pmovzxwd, pmovzxwd)
+ AVX_OP2_XO_SSE4(Pmovzxdq, pmovzxdq)
+
+#undef AVX_OP2_WITH_TYPE_SCOPE
+#undef AVX_OP2_XO_SSE4
+
+#define AVX_OP3_WITH_TYPE_SCOPE(macro_name, name, dst_type, src_type, \
+ sse_scope) \
+ void macro_name(dst_type dst, src_type src) { \
+ if (CpuFeatures::IsSupported(AVX)) { \
+ CpuFeatureScope scope(this, AVX); \
+ v##name(dst, dst, src); \
+ return; \
+ } \
+ if (CpuFeatures::IsSupported(sse_scope)) { \
+ CpuFeatureScope scope(this, sse_scope); \
+ name(dst, src); \
+ return; \
+ } \
+ UNREACHABLE(); \
+ }
+#define AVX_OP3_XO_SSE4(macro_name, name) \
+ AVX_OP3_WITH_TYPE_SCOPE(macro_name, name, XMMRegister, XMMRegister, SSE4_1) \
+ AVX_OP3_WITH_TYPE_SCOPE(macro_name, name, XMMRegister, Operand, SSE4_1)
+
+ AVX_OP3_XO_SSE4(Pmaxsd, pmaxsd)
+
+#undef AVX_OP3_XO_SSE4
+#undef AVX_OP3_WITH_TYPE_SCOPE
+
+ void Pshufb(XMMRegister dst, XMMRegister src) { Pshufb(dst, dst, src); }
+ void Pshufb(XMMRegister dst, Operand src) { Pshufb(dst, dst, src); }
+ // Handles SSE and AVX. On SSE, moves src to dst if they are not equal.
+ void Pshufb(XMMRegister dst, XMMRegister src, XMMRegister mask) {
+ Pshufb(dst, src, Operand(mask));
+ }
+ void Pshufb(XMMRegister dst, XMMRegister src, Operand mask);
+ void Pblendw(XMMRegister dst, XMMRegister src, uint8_t imm8) {
+ Pblendw(dst, Operand(src), imm8);
+ }
+ void Pblendw(XMMRegister dst, Operand src, uint8_t imm8);
+
+ void Psignb(XMMRegister dst, XMMRegister src) { Psignb(dst, Operand(src)); }
+ void Psignb(XMMRegister dst, Operand src);
+ void Psignw(XMMRegister dst, XMMRegister src) { Psignw(dst, Operand(src)); }
+ void Psignw(XMMRegister dst, Operand src);
+ void Psignd(XMMRegister dst, XMMRegister src) { Psignd(dst, Operand(src)); }
+ void Psignd(XMMRegister dst, Operand src);
+
+ void Palignr(XMMRegister dst, XMMRegister src, uint8_t imm8) {
+ Palignr(dst, Operand(src), imm8);
+ }
+ void Palignr(XMMRegister dst, Operand src, uint8_t imm8);
+
+ void Pextrb(Register dst, XMMRegister src, uint8_t imm8);
+ void Pextrw(Register dst, XMMRegister src, uint8_t imm8);
+ void Pextrd(Register dst, XMMRegister src, uint8_t imm8);
+ void Pinsrb(XMMRegister dst, Register src, int8_t imm8) {
+ Pinsrb(dst, Operand(src), imm8);
+ }
+ void Pinsrb(XMMRegister dst, Operand src, int8_t imm8);
+ void Pinsrd(XMMRegister dst, Register src, uint8_t imm8) {
+ Pinsrd(dst, Operand(src), imm8);
+ }
+ void Pinsrd(XMMRegister dst, Operand src, uint8_t imm8);
+ void Pinsrw(XMMRegister dst, Register src, int8_t imm8) {
+ Pinsrw(dst, Operand(src), imm8);
+ }
+ void Pinsrw(XMMRegister dst, Operand src, int8_t imm8);
+ void Vbroadcastss(XMMRegister dst, Operand src);
+
+ // Expression support
+ // cvtsi2sd instruction only writes to the low 64-bit of dst register, which
+ // hinders register renaming and makes dependence chains longer. So we use
+ // xorps to clear the dst register before cvtsi2sd to solve this issue.
+ void Cvtsi2ss(XMMRegister dst, Register src) { Cvtsi2ss(dst, Operand(src)); }
+ void Cvtsi2ss(XMMRegister dst, Operand src);
+ void Cvtsi2sd(XMMRegister dst, Register src) { Cvtsi2sd(dst, Operand(src)); }
+ void Cvtsi2sd(XMMRegister dst, Operand src);
+
+ void Cvtui2ss(XMMRegister dst, Register src, Register tmp) {
+ Cvtui2ss(dst, Operand(src), tmp);
+ }
+ void Cvtui2ss(XMMRegister dst, Operand src, Register tmp);
+ void Cvttss2ui(Register dst, XMMRegister src, XMMRegister tmp) {
+ Cvttss2ui(dst, Operand(src), tmp);
+ }
+ void Cvttss2ui(Register dst, Operand src, XMMRegister tmp);
+ void Cvtui2sd(XMMRegister dst, Register src, Register scratch) {
+ Cvtui2sd(dst, Operand(src), scratch);
+ }
+ void Cvtui2sd(XMMRegister dst, Operand src, Register scratch);
+ void Cvttsd2ui(Register dst, XMMRegister src, XMMRegister tmp) {
+ Cvttsd2ui(dst, Operand(src), tmp);
+ }
+ void Cvttsd2ui(Register dst, Operand src, XMMRegister tmp);
+
+ void Roundps(XMMRegister dst, XMMRegister src, RoundingMode mode);
+ void Roundpd(XMMRegister dst, XMMRegister src, RoundingMode mode);
+
+ void Push(Register src) { push(src); }
+ void Push(Operand src) { push(src); }
+ void Push(Immediate value);
+ void Push(Handle<HeapObject> handle) { push(Immediate(handle)); }
+ void Push(Smi smi) { Push(Immediate(smi)); }
+
+ void SaveRegisters(RegList registers);
+ void RestoreRegisters(RegList registers);
+
+ void CallRecordWriteStub(Register object, Register address,
+ RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode);
+ void CallRecordWriteStub(Register object, Register address,
+ RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode, Address wasm_target);
+ void CallEphemeronKeyBarrier(Register object, Register address,
+ SaveFPRegsMode fp_mode);
+
+ // Calculate how much stack space (in bytes) are required to store caller
+ // registers excluding those specified in the arguments.
+ int RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
+ Register exclusion1 = no_reg,
+ Register exclusion2 = no_reg,
+ Register exclusion3 = no_reg) const;
+
+ // PushCallerSaved and PopCallerSaved do not arrange the registers in any
+ // particular order so they are not useful for calls that can cause a GC.
+ // The caller can exclude up to 3 registers that do not need to be saved and
+ // restored.
+
+ // Push caller saved registers on the stack, and return the number of bytes
+ // stack pointer is adjusted.
+ int PushCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1 = no_reg,
+ Register exclusion2 = no_reg,
+ Register exclusion3 = no_reg);
+ // Restore caller saved registers from the stack, and return the number of
+ // bytes stack pointer is adjusted.
+ int PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1 = no_reg,
+ Register exclusion2 = no_reg,
+ Register exclusion3 = no_reg);
+
+ // Compute the start of the generated instruction stream from the current PC.
+ // This is an alternative to embedding the {CodeObject} handle as a reference.
+ void ComputeCodeStartAddress(Register dst);
+
+ // TODO(860429): Remove remaining poisoning infrastructure on ia32.
+ void ResetSpeculationPoisonRegister() { UNREACHABLE(); }
+
+ // Control-flow integrity:
+
+ // Define a function entrypoint. This doesn't emit any code for this
+ // architecture, as control-flow integrity is not supported for it.
+ void CodeEntry() {}
+ // Define an exception handler.
+ void ExceptionHandler() {}
+ // Define an exception handler and bind a label.
+ void BindExceptionHandler(Label* label) { bind(label); }
+
+ void CallRecordWriteStub(Register object, Register address,
+ RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode, Handle<Code> code_target,
+ Address wasm_target);
+};
+
+// MacroAssembler implements a collection of frequently used macros.
+class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler {
+ public:
+ using TurboAssembler::TurboAssembler;
+
+ // Load a register with a long value as efficiently as possible.
+ void Set(Register dst, int32_t x) {
+ if (x == 0) {
+ xor_(dst, dst);
+ } else {
+ mov(dst, Immediate(x));
+ }
+ }
+
+ void PushRoot(RootIndex index);
+
+ // Compare the object in a register to a value and jump if they are equal.
+ void JumpIfRoot(Register with, RootIndex index, Label* if_equal,
+ Label::Distance if_equal_distance = Label::kFar) {
+ CompareRoot(with, index);
+ j(equal, if_equal, if_equal_distance);
+ }
+
+ // Compare the object in a register to a value and jump if they are not equal.
+ void JumpIfNotRoot(Register with, RootIndex index, Label* if_not_equal,
+ Label::Distance if_not_equal_distance = Label::kFar) {
+ CompareRoot(with, index);
+ j(not_equal, if_not_equal, if_not_equal_distance);
+ }
+
+ // Checks if value is in range [lower_limit, higher_limit] using a single
+ // comparison.
+ void JumpIfIsInRange(Register value, unsigned lower_limit,
+ unsigned higher_limit, Register scratch,
+ Label* on_in_range,
+ Label::Distance near_jump = Label::kFar);
+
+ // ---------------------------------------------------------------------------
+ // GC Support
+ // Notify the garbage collector that we wrote a pointer into an object.
+ // |object| is the object being stored into, |value| is the object being
+ // stored. value and scratch registers are clobbered by the operation.
+ // The offset is the offset from the start of the object, not the offset from
+ // the tagged HeapObject pointer. For use with FieldOperand(reg, off).
+ void RecordWriteField(
+ Register object, int offset, Register value, Register scratch,
+ SaveFPRegsMode save_fp,
+ RememberedSetAction remembered_set_action = EMIT_REMEMBERED_SET,
+ SmiCheck smi_check = INLINE_SMI_CHECK);
+
+ // For page containing |object| mark region covering |address|
+ // dirty. |object| is the object being stored into, |value| is the
+ // object being stored. The address and value registers are clobbered by the
+ // operation. RecordWrite filters out smis so it does not update the
+ // write barrier if the value is a smi.
+ void RecordWrite(
+ Register object, Register address, Register value, SaveFPRegsMode save_fp,
+ RememberedSetAction remembered_set_action = EMIT_REMEMBERED_SET,
+ SmiCheck smi_check = INLINE_SMI_CHECK);
+
+ // Frame restart support
+ void MaybeDropFrames();
+
+ // Enter specific kind of exit frame. Expects the number of
+ // arguments in register eax and sets up the number of arguments in
+ // register edi and the pointer to the first argument in register
+ // esi.
+ void EnterExitFrame(int argc, bool save_doubles, StackFrame::Type frame_type);
+
+ void EnterApiExitFrame(int argc, Register scratch);
+
+ // Leave the current exit frame. Expects the return value in
+ // register eax:edx (untouched) and the pointer to the first
+ // argument in register esi (if pop_arguments == true).
+ void LeaveExitFrame(bool save_doubles, bool pop_arguments = true);
+
+ // Leave the current exit frame. Expects the return value in
+ // register eax (untouched).
+ void LeaveApiExitFrame();
+
+ // Load the global proxy from the current context.
+ void LoadGlobalProxy(Register dst);
+
+ // Load a value from the native context with a given index.
+ void LoadNativeContextSlot(Register dst, int index);
+
+ // ---------------------------------------------------------------------------
+ // JavaScript invokes
+
+ // Invoke the JavaScript function code by either calling or jumping.
+
+ void InvokeFunctionCode(Register function, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count, InvokeFlag flag);
+
+ // On function call, call into the debugger.
+ // This may clobber ecx.
+ void CallDebugOnFunctionCall(Register fun, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count);
+
+ // Invoke the JavaScript function in the given register. Changes the
+ // current context to the context in the function before invoking.
+ void InvokeFunction(Register function, Register new_target,
+ Register actual_parameter_count, InvokeFlag flag);
+
+ // Compare object type for heap object.
+ // Incoming register is heap_object and outgoing register is map.
+ void CmpObjectType(Register heap_object, InstanceType type, Register map);
+
+ // Compare instance type for map.
+ void CmpInstanceType(Register map, InstanceType type);
+
+ // Smi tagging support.
+ void SmiTag(Register reg) {
+ STATIC_ASSERT(kSmiTag == 0);
+ STATIC_ASSERT(kSmiTagSize == 1);
+ add(reg, reg);
+ }
+
+ // Jump if register contain a non-smi.
+ inline void JumpIfNotSmi(Register value, Label* not_smi_label,
+ Label::Distance distance = Label::kFar) {
+ test(value, Immediate(kSmiTagMask));
+ j(not_zero, not_smi_label, distance);
+ }
+ // Jump if the operand is not a smi.
+ inline void JumpIfNotSmi(Operand value, Label* smi_label,
+ Label::Distance distance = Label::kFar) {
+ test(value, Immediate(kSmiTagMask));
+ j(not_zero, smi_label, distance);
+ }
+
+ template <typename Field>
+ void DecodeField(Register reg) {
+ static const int shift = Field::kShift;
+ static const int mask = Field::kMask >> Field::kShift;
+ if (shift != 0) {
+ sar(reg, shift);
+ }
+ and_(reg, Immediate(mask));
+ }
+
+ // Abort execution if argument is not a smi, enabled via --debug-code.
+ void AssertSmi(Register object);
+
+ // Abort execution if argument is a smi, enabled via --debug-code.
+ void AssertNotSmi(Register object);
+
+ // Abort execution if argument is not a JSFunction, enabled via --debug-code.
+ void AssertFunction(Register object);
+
+ // Abort execution if argument is not a Constructor, enabled via --debug-code.
+ void AssertConstructor(Register object);
+
+ // Abort execution if argument is not a JSBoundFunction,
+ // enabled via --debug-code.
+ void AssertBoundFunction(Register object);
+
+ // Abort execution if argument is not a JSGeneratorObject (or subclass),
+ // enabled via --debug-code.
+ void AssertGeneratorObject(Register object);
+
+ // Abort execution if argument is not undefined or an AllocationSite, enabled
+ // via --debug-code.
+ void AssertUndefinedOrAllocationSite(Register object, Register scratch);
+
+ // ---------------------------------------------------------------------------
+ // Exception handling
+
+ // Push a new stack handler and link it into stack handler chain.
+ void PushStackHandler(Register scratch);
+
+ // Unlink the stack handler on top of the stack from the stack handler chain.
+ void PopStackHandler(Register scratch);
+
+ // ---------------------------------------------------------------------------
+ // Runtime calls
+
+ // Call a runtime routine.
+ void CallRuntime(const Runtime::Function* f, int num_arguments,
+ SaveFPRegsMode save_doubles = kDontSaveFPRegs);
+
+ // Convenience function: Same as above, but takes the fid instead.
+ void CallRuntime(Runtime::FunctionId fid,
+ SaveFPRegsMode save_doubles = kDontSaveFPRegs) {
+ const Runtime::Function* function = Runtime::FunctionForId(fid);
+ CallRuntime(function, function->nargs, save_doubles);
+ }
+
+ // Convenience function: Same as above, but takes the fid instead.
+ void CallRuntime(Runtime::FunctionId fid, int num_arguments,
+ SaveFPRegsMode save_doubles = kDontSaveFPRegs) {
+ CallRuntime(Runtime::FunctionForId(fid), num_arguments, save_doubles);
+ }
+
+ // Convenience function: tail call a runtime routine (jump).
+ void TailCallRuntime(Runtime::FunctionId fid);
+
+ // Jump to a runtime routine.
+ void JumpToExternalReference(const ExternalReference& ext,
+ bool builtin_exit_frame = false);
+
+ // Generates a trampoline to jump to the off-heap instruction stream.
+ void JumpToInstructionStream(Address entry);
+
+ // ---------------------------------------------------------------------------
+ // Utilities
+
+ // Emit code to discard a non-negative number of pointer-sized elements
+ // from the stack, clobbering only the esp register.
+ void Drop(int element_count);
+
+ void Pop(Register dst) { pop(dst); }
+ void Pop(Operand dst) { pop(dst); }
+
+ // ---------------------------------------------------------------------------
+ // In-place weak references.
+ void LoadWeakValue(Register in_out, Label* target_if_cleared);
+
+ // ---------------------------------------------------------------------------
+ // StatsCounter support
+
+ void IncrementCounter(StatsCounter* counter, int value, Register scratch);
+ void DecrementCounter(StatsCounter* counter, int value, Register scratch);
+
+ // ---------------------------------------------------------------------------
+ // Stack limit utilities
+ void CompareStackLimit(Register with, StackLimitKind kind);
+ void StackOverflowCheck(Register num_args, Register scratch,
+ Label* stack_overflow, bool include_receiver = false);
+
+ static int SafepointRegisterStackIndex(Register reg) {
+ return SafepointRegisterStackIndex(reg.code());
+ }
+
+ private:
+ // Helper functions for generating invokes.
+ void InvokePrologue(Register expected_parameter_count,
+ Register actual_parameter_count, Label* done,
+ InvokeFlag flag);
+
+ void EnterExitFramePrologue(StackFrame::Type frame_type, Register scratch);
+ void EnterExitFrameEpilogue(int argc, bool save_doubles);
+
+ void LeaveExitFrameEpilogue();
+
+ // Compute memory operands for safepoint stack slots.
+ static int SafepointRegisterStackIndex(int reg_code);
+
+ // Needs access to SafepointRegisterStackIndex for compiled frame
+ // traversal.
+ friend class CommonFrame;
+
+ DISALLOW_IMPLICIT_CONSTRUCTORS(MacroAssembler);
+};
+
+// -----------------------------------------------------------------------------
+// Static helper functions.
+
+// Generate an Operand for loading a field from an object.
+inline Operand FieldOperand(Register object, int offset) {
+ return Operand(object, offset - kHeapObjectTag);
+}
+
+// Generate an Operand for loading an indexed field from an object.
+inline Operand FieldOperand(Register object, Register index, ScaleFactor scale,
+ int offset) {
+ return Operand(object, index, scale, offset - kHeapObjectTag);
+}
+
+#define ACCESS_MASM(masm) masm->
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_IA32_MACRO_ASSEMBLER_IA32_H_
diff --git a/src/codegen/ia32/register-ia32.h b/src/codegen/ia32/register-ia32.h
new file mode 100644
index 0000000..df3117e
--- /dev/null
+++ b/src/codegen/ia32/register-ia32.h
@@ -0,0 +1,167 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_IA32_REGISTER_IA32_H_
+#define V8_CODEGEN_IA32_REGISTER_IA32_H_
+
+#include "src/codegen/register.h"
+#include "src/codegen/reglist.h"
+
+namespace v8 {
+namespace internal {
+
+#define GENERAL_REGISTERS(V) \
+ V(eax) \
+ V(ecx) \
+ V(edx) \
+ V(ebx) \
+ V(esp) \
+ V(ebp) \
+ V(esi) \
+ V(edi)
+
+#define ALLOCATABLE_GENERAL_REGISTERS(V) \
+ V(eax) \
+ V(ecx) \
+ V(edx) \
+ V(esi) \
+ V(edi)
+
+#define DOUBLE_REGISTERS(V) \
+ V(xmm0) \
+ V(xmm1) \
+ V(xmm2) \
+ V(xmm3) \
+ V(xmm4) \
+ V(xmm5) \
+ V(xmm6) \
+ V(xmm7)
+
+#define FLOAT_REGISTERS DOUBLE_REGISTERS
+#define SIMD128_REGISTERS DOUBLE_REGISTERS
+
+#define ALLOCATABLE_DOUBLE_REGISTERS(V) \
+ V(xmm1) \
+ V(xmm2) \
+ V(xmm3) \
+ V(xmm4) \
+ V(xmm5) \
+ V(xmm6) \
+ V(xmm7)
+
+enum RegisterCode {
+#define REGISTER_CODE(R) kRegCode_##R,
+ GENERAL_REGISTERS(REGISTER_CODE)
+#undef REGISTER_CODE
+ kRegAfterLast
+};
+
+class Register : public RegisterBase<Register, kRegAfterLast> {
+ public:
+ bool is_byte_register() const { return code() <= 3; }
+
+ private:
+ friend class RegisterBase<Register, kRegAfterLast>;
+ explicit constexpr Register(int code) : RegisterBase(code) {}
+};
+
+ASSERT_TRIVIALLY_COPYABLE(Register);
+static_assert(sizeof(Register) == sizeof(int),
+ "Register can efficiently be passed by value");
+
+#define DEFINE_REGISTER(R) \
+ constexpr Register R = Register::from_code(kRegCode_##R);
+GENERAL_REGISTERS(DEFINE_REGISTER)
+#undef DEFINE_REGISTER
+constexpr Register no_reg = Register::no_reg();
+
+constexpr bool kPadArguments = false;
+constexpr bool kSimpleFPAliasing = true;
+constexpr bool kSimdMaskRegisters = false;
+
+enum DoubleCode {
+#define REGISTER_CODE(R) kDoubleCode_##R,
+ DOUBLE_REGISTERS(REGISTER_CODE)
+#undef REGISTER_CODE
+ kDoubleAfterLast
+};
+
+class XMMRegister : public RegisterBase<XMMRegister, kDoubleAfterLast> {
+ friend class RegisterBase<XMMRegister, kDoubleAfterLast>;
+ explicit constexpr XMMRegister(int code) : RegisterBase(code) {}
+};
+
+using FloatRegister = XMMRegister;
+
+using DoubleRegister = XMMRegister;
+
+using Simd128Register = XMMRegister;
+
+#define DEFINE_REGISTER(R) \
+ constexpr DoubleRegister R = DoubleRegister::from_code(kDoubleCode_##R);
+DOUBLE_REGISTERS(DEFINE_REGISTER)
+#undef DEFINE_REGISTER
+constexpr DoubleRegister no_dreg = DoubleRegister::no_reg();
+
+// Note that the bit values must match those used in actual instruction encoding
+constexpr int kNumRegs = 8;
+
+// Caller-saved registers
+constexpr RegList kJSCallerSaved =
+ Register::ListOf(eax, ecx, edx,
+ ebx, // used as caller-saved register in JavaScript code
+ edi); // callee function
+
+constexpr int kNumJSCallerSaved = 5;
+
+// Number of registers for which space is reserved in safepoints.
+constexpr int kNumSafepointRegisters = 8;
+
+// Define {RegisterName} methods for the register types.
+DEFINE_REGISTER_NAMES(Register, GENERAL_REGISTERS)
+DEFINE_REGISTER_NAMES(XMMRegister, DOUBLE_REGISTERS)
+
+// Give alias names to registers for calling conventions.
+constexpr Register kReturnRegister0 = eax;
+constexpr Register kReturnRegister1 = edx;
+constexpr Register kReturnRegister2 = edi;
+constexpr Register kJSFunctionRegister = edi;
+constexpr Register kContextRegister = esi;
+constexpr Register kAllocateSizeRegister = edx;
+constexpr Register kInterpreterAccumulatorRegister = eax;
+constexpr Register kInterpreterBytecodeOffsetRegister = edx;
+constexpr Register kInterpreterBytecodeArrayRegister = edi;
+constexpr Register kInterpreterDispatchTableRegister = esi;
+
+constexpr Register kJavaScriptCallArgCountRegister = eax;
+constexpr Register kJavaScriptCallCodeStartRegister = ecx;
+constexpr Register kJavaScriptCallTargetRegister = kJSFunctionRegister;
+constexpr Register kJavaScriptCallNewTargetRegister = edx;
+
+// The ExtraArg1Register not part of the real JS calling convention and is
+// mostly there to simplify consistent interface descriptor definitions across
+// platforms. Note that on ia32 it aliases kJavaScriptCallCodeStartRegister.
+constexpr Register kJavaScriptCallExtraArg1Register = ecx;
+
+// The off-heap trampoline does not need a register on ia32 (it uses a
+// pc-relative call instead).
+constexpr Register kOffHeapTrampolineRegister = no_reg;
+
+constexpr Register kRuntimeCallFunctionRegister = edx;
+constexpr Register kRuntimeCallArgCountRegister = eax;
+constexpr Register kRuntimeCallArgvRegister = ecx;
+constexpr Register kWasmInstanceRegister = esi;
+constexpr Register kWasmCompileLazyFuncIndexRegister = edi;
+
+constexpr Register kRootRegister = ebx;
+
+// TODO(860429): Remove remaining poisoning infrastructure on ia32.
+constexpr Register kSpeculationPoisonRegister = no_reg;
+
+constexpr DoubleRegister kFPReturnRegister0 = xmm1; // xmm0 isn't allocatable.
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_IA32_REGISTER_IA32_H_
diff --git a/src/codegen/ia32/sse-instr.h b/src/codegen/ia32/sse-instr.h
new file mode 100644
index 0000000..a56dc13
--- /dev/null
+++ b/src/codegen/ia32/sse-instr.h
@@ -0,0 +1,98 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_IA32_SSE_INSTR_H_
+#define V8_CODEGEN_IA32_SSE_INSTR_H_
+
+#define SSE2_INSTRUCTION_LIST(V) \
+ V(packsswb, 66, 0F, 63) \
+ V(packssdw, 66, 0F, 6B) \
+ V(packuswb, 66, 0F, 67) \
+ V(pmaddwd, 66, 0F, F5) \
+ V(paddb, 66, 0F, FC) \
+ V(paddw, 66, 0F, FD) \
+ V(paddd, 66, 0F, FE) \
+ V(paddq, 66, 0F, D4) \
+ V(paddsb, 66, 0F, EC) \
+ V(paddsw, 66, 0F, ED) \
+ V(paddusb, 66, 0F, DC) \
+ V(paddusw, 66, 0F, DD) \
+ V(pand, 66, 0F, DB) \
+ V(pcmpeqb, 66, 0F, 74) \
+ V(pcmpeqw, 66, 0F, 75) \
+ V(pcmpeqd, 66, 0F, 76) \
+ V(pcmpgtb, 66, 0F, 64) \
+ V(pcmpgtw, 66, 0F, 65) \
+ V(pcmpgtd, 66, 0F, 66) \
+ V(pmaxsw, 66, 0F, EE) \
+ V(pmaxub, 66, 0F, DE) \
+ V(pminsw, 66, 0F, EA) \
+ V(pminub, 66, 0F, DA) \
+ V(pmullw, 66, 0F, D5) \
+ V(por, 66, 0F, EB) \
+ V(psllw, 66, 0F, F1) \
+ V(pslld, 66, 0F, F2) \
+ V(psllq, 66, 0F, F3) \
+ V(pmuludq, 66, 0F, F4) \
+ V(pavgb, 66, 0F, E0) \
+ V(psraw, 66, 0F, E1) \
+ V(psrad, 66, 0F, E2) \
+ V(pavgw, 66, 0F, E3) \
+ V(psrlw, 66, 0F, D1) \
+ V(psrld, 66, 0F, D2) \
+ V(psrlq, 66, 0F, D3) \
+ V(psubb, 66, 0F, F8) \
+ V(psubw, 66, 0F, F9) \
+ V(psubd, 66, 0F, FA) \
+ V(psubq, 66, 0F, FB) \
+ V(psubsb, 66, 0F, E8) \
+ V(psubsw, 66, 0F, E9) \
+ V(psubusb, 66, 0F, D8) \
+ V(psubusw, 66, 0F, D9) \
+ V(punpcklbw, 66, 0F, 60) \
+ V(punpcklwd, 66, 0F, 61) \
+ V(punpckldq, 66, 0F, 62) \
+ V(punpcklqdq, 66, 0F, 6C) \
+ V(punpckhbw, 66, 0F, 68) \
+ V(punpckhwd, 66, 0F, 69) \
+ V(punpckhdq, 66, 0F, 6A) \
+ V(punpckhqdq, 66, 0F, 6D) \
+ V(pxor, 66, 0F, EF)
+
+#define SSSE3_INSTRUCTION_LIST(V) \
+ V(phaddd, 66, 0F, 38, 02) \
+ V(phaddw, 66, 0F, 38, 01) \
+ V(pshufb, 66, 0F, 38, 00) \
+ V(psignb, 66, 0F, 38, 08) \
+ V(psignw, 66, 0F, 38, 09) \
+ V(psignd, 66, 0F, 38, 0A)
+
+// SSSE3 instructions whose AVX version has two operands.
+#define SSSE3_UNOP_INSTRUCTION_LIST(V) \
+ V(pabsb, 66, 0F, 38, 1C) \
+ V(pabsw, 66, 0F, 38, 1D) \
+ V(pabsd, 66, 0F, 38, 1E)
+
+#define SSE4_INSTRUCTION_LIST(V) \
+ V(packusdw, 66, 0F, 38, 2B) \
+ V(pminsb, 66, 0F, 38, 38) \
+ V(pminsd, 66, 0F, 38, 39) \
+ V(pminuw, 66, 0F, 38, 3A) \
+ V(pminud, 66, 0F, 38, 3B) \
+ V(pmaxsb, 66, 0F, 38, 3C) \
+ V(pmaxsd, 66, 0F, 38, 3D) \
+ V(pmaxuw, 66, 0F, 38, 3E) \
+ V(pmaxud, 66, 0F, 38, 3F) \
+ V(pmulld, 66, 0F, 38, 40)
+
+#define SSE4_RM_INSTRUCTION_LIST(V) \
+ V(pmovsxbw, 66, 0F, 38, 20) \
+ V(pmovsxwd, 66, 0F, 38, 23) \
+ V(pmovsxdq, 66, 0F, 38, 25) \
+ V(pmovzxbw, 66, 0F, 38, 30) \
+ V(pmovzxwd, 66, 0F, 38, 33) \
+ V(pmovzxdq, 66, 0F, 38, 35) \
+ V(ptest, 66, 0F, 38, 17)
+
+#endif // V8_CODEGEN_IA32_SSE_INSTR_H_
diff --git a/src/codegen/interface-descriptors.cc b/src/codegen/interface-descriptors.cc
new file mode 100644
index 0000000..79dad84
--- /dev/null
+++ b/src/codegen/interface-descriptors.cc
@@ -0,0 +1,490 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/interface-descriptors.h"
+
+#include "src/codegen/macro-assembler.h"
+
+namespace v8 {
+namespace internal {
+
+void CallInterfaceDescriptorData::InitializePlatformSpecific(
+ int register_parameter_count, const Register* registers) {
+ DCHECK(!IsInitializedPlatformIndependent());
+
+ register_param_count_ = register_parameter_count;
+
+ // UBSan doesn't like creating zero-length arrays.
+ if (register_parameter_count == 0) return;
+
+ // InterfaceDescriptor owns a copy of the registers array.
+ register_params_ = NewArray<Register>(register_parameter_count, no_reg);
+ for (int i = 0; i < register_parameter_count; i++) {
+ // The value of the root register must be reserved, thus any uses
+ // within the calling convention are disallowed.
+ DCHECK_NE(registers[i], kRootRegister);
+ register_params_[i] = registers[i];
+ }
+}
+
+void CallInterfaceDescriptorData::InitializePlatformIndependent(
+ Flags flags, int return_count, int parameter_count,
+ const MachineType* machine_types, int machine_types_length,
+ StackArgumentOrder stack_order) {
+ DCHECK(IsInitializedPlatformSpecific());
+
+ flags_ = flags;
+ stack_order_ = stack_order;
+ return_count_ = return_count;
+ param_count_ = parameter_count;
+ const int types_length = return_count_ + param_count_;
+
+ // Machine types are either fully initialized or null.
+ if (machine_types == nullptr) {
+ machine_types_ =
+ NewArray<MachineType>(types_length, MachineType::AnyTagged());
+ } else {
+ DCHECK_EQ(machine_types_length, types_length);
+ machine_types_ = NewArray<MachineType>(types_length);
+ for (int i = 0; i < types_length; i++) machine_types_[i] = machine_types[i];
+ }
+
+ if (!(flags_ & kNoStackScan)) DCHECK(AllStackParametersAreTagged());
+}
+
+#ifdef DEBUG
+bool CallInterfaceDescriptorData::AllStackParametersAreTagged() const {
+ DCHECK(IsInitialized());
+ const int types_length = return_count_ + param_count_;
+ const int first_stack_param = return_count_ + register_param_count_;
+ for (int i = first_stack_param; i < types_length; i++) {
+ if (!machine_types_[i].IsTagged()) return false;
+ }
+ return true;
+}
+#endif // DEBUG
+
+void CallInterfaceDescriptorData::Reset() {
+ delete[] machine_types_;
+ machine_types_ = nullptr;
+ delete[] register_params_;
+ register_params_ = nullptr;
+}
+
+// static
+CallInterfaceDescriptorData
+ CallDescriptors::call_descriptor_data_[NUMBER_OF_DESCRIPTORS];
+
+void CallDescriptors::InitializeOncePerProcess() {
+#define INTERFACE_DESCRIPTOR(name, ...) \
+ name##Descriptor().Initialize(&call_descriptor_data_[CallDescriptors::name]);
+ INTERFACE_DESCRIPTOR_LIST(INTERFACE_DESCRIPTOR)
+#undef INTERFACE_DESCRIPTOR
+
+ DCHECK(ContextOnlyDescriptor{}.HasContextParameter());
+ DCHECK(!NoContextDescriptor{}.HasContextParameter());
+ DCHECK(!AllocateDescriptor{}.HasContextParameter());
+ DCHECK(!AbortDescriptor{}.HasContextParameter());
+ DCHECK(!WasmFloat32ToNumberDescriptor{}.HasContextParameter());
+ DCHECK(!WasmFloat64ToNumberDescriptor{}.HasContextParameter());
+}
+
+void CallDescriptors::TearDown() {
+ for (CallInterfaceDescriptorData& data : call_descriptor_data_) {
+ data.Reset();
+ }
+}
+
+void CallInterfaceDescriptor::JSDefaultInitializePlatformSpecific(
+ CallInterfaceDescriptorData* data, int non_js_register_parameter_count) {
+ DCHECK_LE(static_cast<unsigned>(non_js_register_parameter_count), 1);
+
+ // 3 is for kTarget, kNewTarget and kActualArgumentsCount
+ int register_parameter_count = 3 + non_js_register_parameter_count;
+
+ DCHECK(!AreAliased(
+ kJavaScriptCallTargetRegister, kJavaScriptCallNewTargetRegister,
+ kJavaScriptCallArgCountRegister, kJavaScriptCallExtraArg1Register));
+
+ const Register default_js_stub_registers[] = {
+ kJavaScriptCallTargetRegister, kJavaScriptCallNewTargetRegister,
+ kJavaScriptCallArgCountRegister, kJavaScriptCallExtraArg1Register};
+
+ CHECK_LE(static_cast<size_t>(register_parameter_count),
+ arraysize(default_js_stub_registers));
+ data->InitializePlatformSpecific(register_parameter_count,
+ default_js_stub_registers);
+}
+
+const char* CallInterfaceDescriptor::DebugName() const {
+ CallDescriptors::Key key = CallDescriptors::GetKey(data_);
+ switch (key) {
+#define DEF_CASE(name, ...) \
+ case CallDescriptors::name: \
+ return #name " Descriptor";
+ INTERFACE_DESCRIPTOR_LIST(DEF_CASE)
+#undef DEF_CASE
+ case CallDescriptors::NUMBER_OF_DESCRIPTORS:
+ break;
+ }
+ return "";
+}
+
+#if !defined(V8_TARGET_ARCH_MIPS) && !defined(V8_TARGET_ARCH_MIPS64)
+bool CallInterfaceDescriptor::IsValidFloatParameterRegister(Register reg) {
+ return true;
+}
+#endif
+
+void VoidDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ data->InitializePlatformSpecific(0, nullptr);
+}
+
+void AllocateDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {kAllocateSizeRegister};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CEntry1ArgvOnStackDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {kRuntimeCallArgCountRegister,
+ kRuntimeCallFunctionRegister};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+namespace {
+
+void InterpreterCEntryDescriptor_InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {kRuntimeCallArgCountRegister,
+ kRuntimeCallArgvRegister,
+ kRuntimeCallFunctionRegister};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+} // namespace
+
+void InterpreterCEntry1Descriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ InterpreterCEntryDescriptor_InitializePlatformSpecific(data);
+}
+
+void InterpreterCEntry2Descriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ InterpreterCEntryDescriptor_InitializePlatformSpecific(data);
+}
+
+void FastNewObjectDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {TargetRegister(), NewTargetRegister()};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+const Register FastNewObjectDescriptor::TargetRegister() {
+ return kJSFunctionRegister;
+}
+
+const Register FastNewObjectDescriptor::NewTargetRegister() {
+ return kJavaScriptCallNewTargetRegister;
+}
+
+void LoadDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {ReceiverRegister(), NameRegister(), SlotRegister()};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void LoadNoFeedbackDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {ReceiverRegister(), NameRegister(), ICKindRegister()};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void LoadGlobalDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {NameRegister(), SlotRegister()};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void LoadGlobalNoFeedbackDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {NameRegister(), ICKindRegister()};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void LoadGlobalWithVectorDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {NameRegister(), SlotRegister(), VectorRegister()};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void LoadWithReceiverAndVectorDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ DCHECK(!AreAliased(ReceiverRegister(), LookupStartObjectRegister(),
+ NameRegister(), SlotRegister(), VectorRegister()));
+ Register registers[] = {ReceiverRegister(), LookupStartObjectRegister(),
+ NameRegister(), SlotRegister(), VectorRegister()};
+ int len = arraysize(registers) - kStackArgumentsCount;
+ data->InitializePlatformSpecific(len, registers);
+}
+
+void StoreGlobalDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {NameRegister(), ValueRegister(), SlotRegister()};
+
+ int len = arraysize(registers) - kStackArgumentsCount;
+ data->InitializePlatformSpecific(len, registers);
+}
+
+void StoreGlobalWithVectorDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {NameRegister(), ValueRegister(), SlotRegister(),
+ VectorRegister()};
+ int len = arraysize(registers) - kStackArgumentsCount;
+ data->InitializePlatformSpecific(len, registers);
+}
+
+void StoreDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {ReceiverRegister(), NameRegister(), ValueRegister(),
+ SlotRegister()};
+
+ int len = arraysize(registers) - kStackArgumentsCount;
+ data->InitializePlatformSpecific(len, registers);
+}
+
+void StoreTransitionDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ ReceiverRegister(), NameRegister(), MapRegister(),
+ ValueRegister(), SlotRegister(), VectorRegister(),
+ };
+ int len = arraysize(registers) - kStackArgumentsCount;
+ data->InitializePlatformSpecific(len, registers);
+}
+
+void StringAtDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ DefaultInitializePlatformSpecific(data, kParameterCount);
+}
+
+void StringAtAsStringDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ DefaultInitializePlatformSpecific(data, kParameterCount);
+}
+
+void StringSubstringDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ DefaultInitializePlatformSpecific(data, kParameterCount);
+}
+
+void TypeConversionDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {ArgumentRegister()};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void TypeConversionNoContextDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {TypeConversionDescriptor::ArgumentRegister()};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void TypeConversionStackParameterDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ data->InitializePlatformSpecific(0, nullptr);
+}
+
+void AsyncFunctionStackParameterDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ data->InitializePlatformSpecific(0, nullptr);
+}
+
+void GetIteratorStackParameterDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ data->InitializePlatformSpecific(0, nullptr);
+}
+
+void LoadWithVectorDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {ReceiverRegister(), NameRegister(), SlotRegister(),
+ VectorRegister()};
+ // TODO(jgruber): This DCHECK could be enabled if RegisterBase::ListOf were
+ // to allow no_reg entries.
+ // DCHECK(!AreAliased(ReceiverRegister(), NameRegister(), SlotRegister(),
+ // VectorRegister(), kRootRegister));
+ int len = arraysize(registers) - kStackArgumentsCount;
+ data->InitializePlatformSpecific(len, registers);
+}
+
+void StoreWithVectorDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {ReceiverRegister(), NameRegister(), ValueRegister(),
+ SlotRegister(), VectorRegister()};
+ // TODO(jgruber): This DCHECK could be enabled if RegisterBase::ListOf were
+ // to allow no_reg entries.
+ // DCHECK(!AreAliased(ReceiverRegister(), NameRegister(), kRootRegister));
+ int len = arraysize(registers) - kStackArgumentsCount;
+ data->InitializePlatformSpecific(len, registers);
+}
+
+const Register ApiGetterDescriptor::ReceiverRegister() {
+ return LoadDescriptor::ReceiverRegister();
+}
+
+void ApiGetterDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {ReceiverRegister(), HolderRegister(),
+ CallbackRegister()};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ContextOnlyDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ data->InitializePlatformSpecific(0, nullptr);
+}
+
+void NoContextDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ data->InitializePlatformSpecific(0, nullptr);
+}
+
+void GrowArrayElementsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {ObjectRegister(), KeyRegister()};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ArrayNoArgumentConstructorDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // This descriptor must use the same set of registers as the
+ // ArrayNArgumentsConstructorDescriptor.
+ ArrayNArgumentsConstructorDescriptor::InitializePlatformSpecific(data);
+}
+
+void ArraySingleArgumentConstructorDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // This descriptor must use the same set of registers as the
+ // ArrayNArgumentsConstructorDescriptor.
+ ArrayNArgumentsConstructorDescriptor::InitializePlatformSpecific(data);
+}
+
+void ArrayNArgumentsConstructorDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // Keep the arguments on the same registers as they were in
+ // ArrayConstructorDescriptor to avoid unnecessary register moves.
+ // kFunction, kAllocationSite, kActualArgumentsCount
+ Register registers[] = {kJavaScriptCallTargetRegister,
+ kJavaScriptCallExtraArg1Register,
+ kJavaScriptCallArgCountRegister};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+#if !V8_TARGET_ARCH_IA32
+// We need a custom descriptor on ia32 to avoid using xmm0.
+void WasmFloat32ToNumberDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ DefaultInitializePlatformSpecific(data, kParameterCount);
+}
+
+// We need a custom descriptor on ia32 to avoid using xmm0.
+void WasmFloat64ToNumberDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ DefaultInitializePlatformSpecific(data, kParameterCount);
+}
+#endif // !V8_TARGET_ARCH_IA32
+
+#if !defined(V8_TARGET_ARCH_MIPS) && !defined(V8_TARGET_ARCH_MIPS64)
+void WasmI32AtomicWait32Descriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ DefaultInitializePlatformSpecific(data, kParameterCount);
+}
+
+void WasmI64AtomicWait32Descriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ DefaultInitializePlatformSpecific(data,
+ kParameterCount - kStackArgumentsCount);
+}
+#endif
+
+void CloneObjectWithVectorDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ DefaultInitializePlatformSpecific(data, kParameterCount);
+}
+
+// static
+Register RunMicrotasksDescriptor::MicrotaskQueueRegister() {
+ return CallDescriptors::call_descriptor_data(CallDescriptors::RunMicrotasks)
+ ->register_param(0);
+}
+
+void RunMicrotasksDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ DefaultInitializePlatformSpecific(data, kParameterCount);
+}
+
+void I64ToBigIntDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ DefaultInitializePlatformSpecific(data, kParameterCount);
+}
+
+void I32PairToBigIntDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ DefaultInitializePlatformSpecific(data, kParameterCount);
+}
+
+void BigIntToI64Descriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ DefaultInitializePlatformSpecific(data, kParameterCount);
+}
+
+void BigIntToI32PairDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ DefaultInitializePlatformSpecific(data, kParameterCount);
+}
+
+void BinaryOp_WithFeedbackDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ DefaultInitializePlatformSpecific(data, 4);
+}
+
+void CallTrampoline_WithFeedbackDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ DefaultInitializePlatformSpecific(data, 4);
+}
+
+void CallWithArrayLike_WithFeedbackDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ DefaultInitializePlatformSpecific(data, 4);
+}
+
+void CallWithSpread_WithFeedbackDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ DefaultInitializePlatformSpecific(data, 4);
+}
+
+void ConstructWithArrayLike_WithFeedbackDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ DefaultInitializePlatformSpecific(data, 4);
+}
+
+void ConstructWithSpread_WithFeedbackDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ DefaultInitializePlatformSpecific(data, 4);
+}
+
+void Compare_WithFeedbackDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ DefaultInitializePlatformSpecific(data, 4);
+}
+
+void UnaryOp_WithFeedbackDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ DefaultInitializePlatformSpecific(data, 3);
+}
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/interface-descriptors.h b/src/codegen/interface-descriptors.h
new file mode 100644
index 0000000..f086f23
--- /dev/null
+++ b/src/codegen/interface-descriptors.h
@@ -0,0 +1,1588 @@
+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_INTERFACE_DESCRIPTORS_H_
+#define V8_CODEGEN_INTERFACE_DESCRIPTORS_H_
+
+#include <memory>
+
+#include "src/codegen/machine-type.h"
+#include "src/codegen/register-arch.h"
+#include "src/codegen/tnode.h"
+#include "src/common/globals.h"
+#include "src/execution/isolate.h"
+
+namespace v8 {
+namespace internal {
+
+#define TORQUE_BUILTIN_LIST_TFC(V) \
+ BUILTIN_LIST_FROM_TORQUE(IGNORE_BUILTIN, IGNORE_BUILTIN, V, IGNORE_BUILTIN, \
+ IGNORE_BUILTIN, IGNORE_BUILTIN)
+
+#define INTERFACE_DESCRIPTOR_LIST(V) \
+ V(Abort) \
+ V(Allocate) \
+ V(ApiCallback) \
+ V(ApiGetter) \
+ V(ArgumentsAdaptor) \
+ V(ArrayConstructor) \
+ V(ArrayNArgumentsConstructor) \
+ V(ArrayNoArgumentConstructor) \
+ V(ArraySingleArgumentConstructor) \
+ V(AsyncFunctionStackParameter) \
+ V(BigIntToI32Pair) \
+ V(BigIntToI64) \
+ V(BinaryOp) \
+ V(BinaryOp_WithFeedback) \
+ V(CallForwardVarargs) \
+ V(CallFunctionTemplate) \
+ V(CallTrampoline) \
+ V(CallTrampoline_WithFeedback) \
+ V(CallVarargs) \
+ V(CallWithArrayLike) \
+ V(CallWithArrayLike_WithFeedback) \
+ V(CallWithSpread) \
+ V(CallWithSpread_WithFeedback) \
+ V(CEntry1ArgvOnStack) \
+ V(CloneObjectWithVector) \
+ V(Compare) \
+ V(Compare_WithFeedback) \
+ V(ConstructForwardVarargs) \
+ V(ConstructStub) \
+ V(ConstructVarargs) \
+ V(ConstructWithArrayLike) \
+ V(ConstructWithArrayLike_WithFeedback) \
+ V(Construct_WithFeedback) \
+ V(ConstructWithSpread) \
+ V(ConstructWithSpread_WithFeedback) \
+ V(ContextOnly) \
+ V(CppBuiltinAdaptor) \
+ V(EphemeronKeyBarrier) \
+ V(FastNewObject) \
+ V(FrameDropperTrampoline) \
+ V(GetIteratorStackParameter) \
+ V(GetProperty) \
+ V(GrowArrayElements) \
+ V(I32PairToBigInt) \
+ V(I64ToBigInt) \
+ V(InterpreterCEntry1) \
+ V(InterpreterCEntry2) \
+ V(InterpreterDispatch) \
+ V(InterpreterPushArgsThenCall) \
+ V(InterpreterPushArgsThenConstruct) \
+ V(JSTrampoline) \
+ V(Load) \
+ V(LoadGlobal) \
+ V(LoadGlobalNoFeedback) \
+ V(LoadGlobalWithVector) \
+ V(LoadNoFeedback) \
+ V(LoadWithVector) \
+ V(LoadWithReceiverAndVector) \
+ V(NoContext) \
+ V(RecordWrite) \
+ V(ResumeGenerator) \
+ V(RunMicrotasks) \
+ V(RunMicrotasksEntry) \
+ V(Store) \
+ V(StoreGlobal) \
+ V(StoreGlobalWithVector) \
+ V(StoreTransition) \
+ V(StoreWithVector) \
+ V(StringAt) \
+ V(StringAtAsString) \
+ V(StringSubstring) \
+ V(TypeConversion) \
+ V(TypeConversionNoContext) \
+ V(TypeConversionStackParameter) \
+ V(Typeof) \
+ V(UnaryOp_WithFeedback) \
+ V(Void) \
+ V(WasmFloat32ToNumber) \
+ V(WasmFloat64ToNumber) \
+ V(WasmI32AtomicWait32) \
+ V(WasmI64AtomicWait32) \
+ BUILTIN_LIST_TFS(V) \
+ TORQUE_BUILTIN_LIST_TFC(V)
+
+enum class StackArgumentOrder {
+ kDefault, // Arguments in the stack are pushed in the default/stub order (the
+ // first argument is pushed first).
+ kJS, // Arguments in the stack are pushed in the same order as the one used
+ // by JS-to-JS function calls. This should be used if calling a
+ // JSFunction or if the builtin is expected to be called directly from a
+ // JSFunction. This order is reversed compared to kDefault.
+};
+
+class V8_EXPORT_PRIVATE CallInterfaceDescriptorData {
+ public:
+ enum Flag {
+ kNoFlags = 0u,
+ kNoContext = 1u << 0,
+ // This indicates that the code uses a special frame that does not scan the
+ // stack arguments, e.g. EntryFrame. And this allows the code to use
+ // untagged stack arguments.
+ kNoStackScan = 1u << 1,
+ // In addition to the specified parameters, additional arguments can be
+ // passed on the stack.
+ // This does not indicate if arguments adaption is used or not.
+ kAllowVarArgs = 1u << 2,
+ };
+ using Flags = base::Flags<Flag>;
+
+ CallInterfaceDescriptorData() = default;
+
+ // A copy of the passed in registers and param_representations is made
+ // and owned by the CallInterfaceDescriptorData.
+
+ void InitializePlatformSpecific(int register_parameter_count,
+ const Register* registers);
+
+ // if machine_types is null, then an array of size
+ // (return_count + parameter_count) will be created with
+ // MachineType::AnyTagged() for each member.
+ //
+ // if machine_types is not null, then it should be of the size
+ // (return_count + parameter_count). Those members of the parameter array will
+ // be initialized from {machine_types}, and the rest initialized to
+ // MachineType::AnyTagged().
+ void InitializePlatformIndependent(Flags flags, int return_count,
+ int parameter_count,
+ const MachineType* machine_types,
+ int machine_types_length,
+ StackArgumentOrder stack_order);
+
+ void Reset();
+
+ bool IsInitialized() const {
+ return IsInitializedPlatformSpecific() &&
+ IsInitializedPlatformIndependent();
+ }
+
+ Flags flags() const { return flags_; }
+ int return_count() const { return return_count_; }
+ int param_count() const { return param_count_; }
+ int register_param_count() const { return register_param_count_; }
+ Register register_param(int index) const { return register_params_[index]; }
+ Register* register_params() const { return register_params_; }
+ MachineType return_type(int index) const {
+ DCHECK_LT(index, return_count_);
+ return machine_types_[index];
+ }
+ MachineType param_type(int index) const {
+ DCHECK_LT(index, param_count_);
+ return machine_types_[return_count_ + index];
+ }
+ StackArgumentOrder stack_order() const { return stack_order_; }
+
+ void RestrictAllocatableRegisters(const Register* registers, int num) {
+ DCHECK_EQ(allocatable_registers_, 0);
+ for (int i = 0; i < num; ++i) {
+ allocatable_registers_ |= registers[i].bit();
+ }
+ DCHECK_GT(NumRegs(allocatable_registers_), 0);
+ }
+
+ RegList allocatable_registers() const { return allocatable_registers_; }
+
+ private:
+ bool IsInitializedPlatformSpecific() const {
+ const bool initialized =
+ (register_param_count_ == 0 && register_params_ == nullptr) ||
+ (register_param_count_ > 0 && register_params_ != nullptr);
+ // Platform-specific initialization happens before platform-independent.
+ return initialized;
+ }
+ bool IsInitializedPlatformIndependent() const {
+ const bool initialized =
+ return_count_ >= 0 && param_count_ >= 0 && machine_types_ != nullptr;
+ // Platform-specific initialization happens before platform-independent.
+ return initialized;
+ }
+
+#ifdef DEBUG
+ bool AllStackParametersAreTagged() const;
+#endif // DEBUG
+
+ int register_param_count_ = -1;
+ int return_count_ = -1;
+ int param_count_ = -1;
+ Flags flags_ = kNoFlags;
+ StackArgumentOrder stack_order_ = StackArgumentOrder::kDefault;
+
+ // Specifying the set of registers that could be used by the register
+ // allocator. Currently, it's only used by RecordWrite code stub.
+ RegList allocatable_registers_ = 0;
+
+ // |registers_params_| defines registers that are used for parameter passing.
+ // |machine_types_| defines machine types for resulting values and incomping
+ // parameters.
+ // Both arrays are allocated dynamically by the InterfaceDescriptor and
+ // freed on destruction. This is because static arrays cause creation of
+ // runtime static initializers which we don't want.
+ Register* register_params_ = nullptr;
+ MachineType* machine_types_ = nullptr;
+
+ DISALLOW_COPY_AND_ASSIGN(CallInterfaceDescriptorData);
+};
+
+class V8_EXPORT_PRIVATE CallDescriptors : public AllStatic {
+ public:
+ enum Key {
+#define DEF_ENUM(name, ...) name,
+ INTERFACE_DESCRIPTOR_LIST(DEF_ENUM)
+#undef DEF_ENUM
+ NUMBER_OF_DESCRIPTORS
+ };
+
+ static void InitializeOncePerProcess();
+ static void TearDown();
+
+ static CallInterfaceDescriptorData* call_descriptor_data(
+ CallDescriptors::Key key) {
+ return &call_descriptor_data_[key];
+ }
+
+ static Key GetKey(const CallInterfaceDescriptorData* data) {
+ ptrdiff_t index = data - call_descriptor_data_;
+ DCHECK_LE(0, index);
+ DCHECK_LT(index, CallDescriptors::NUMBER_OF_DESCRIPTORS);
+ return static_cast<CallDescriptors::Key>(index);
+ }
+
+ private:
+ static CallInterfaceDescriptorData
+ call_descriptor_data_[NUMBER_OF_DESCRIPTORS];
+};
+
+class V8_EXPORT_PRIVATE CallInterfaceDescriptor {
+ public:
+ using Flags = CallInterfaceDescriptorData::Flags;
+
+ CallInterfaceDescriptor() : data_(nullptr) {}
+ virtual ~CallInterfaceDescriptor() = default;
+
+ explicit CallInterfaceDescriptor(CallDescriptors::Key key)
+ : data_(CallDescriptors::call_descriptor_data(key)) {}
+
+ Flags flags() const { return data()->flags(); }
+
+ bool HasContextParameter() const {
+ return (flags() & CallInterfaceDescriptorData::kNoContext) == 0;
+ }
+
+ bool AllowVarArgs() const {
+ return flags() & CallInterfaceDescriptorData::kAllowVarArgs;
+ }
+
+ int GetReturnCount() const { return data()->return_count(); }
+
+ MachineType GetReturnType(int index) const {
+ DCHECK_LT(index, data()->return_count());
+ return data()->return_type(index);
+ }
+
+ int GetParameterCount() const { return data()->param_count(); }
+
+ int GetRegisterParameterCount() const {
+ return data()->register_param_count();
+ }
+
+ int GetStackParameterCount() const {
+ return data()->param_count() - data()->register_param_count();
+ }
+
+ Register GetRegisterParameter(int index) const {
+ return data()->register_param(index);
+ }
+
+ MachineType GetParameterType(int index) const {
+ DCHECK_LT(index, data()->param_count());
+ return data()->param_type(index);
+ }
+
+ RegList allocatable_registers() const {
+ return data()->allocatable_registers();
+ }
+
+ StackArgumentOrder GetStackArgumentOrder() const {
+ return data()->stack_order();
+ }
+
+ static const Register ContextRegister();
+
+ const char* DebugName() const;
+
+ bool operator==(const CallInterfaceDescriptor& other) const {
+ return data() == other.data();
+ }
+
+ protected:
+ const CallInterfaceDescriptorData* data() const { return data_; }
+
+ virtual void InitializePlatformSpecific(CallInterfaceDescriptorData* data) {
+ UNREACHABLE();
+ }
+
+ virtual void InitializePlatformIndependent(
+ CallInterfaceDescriptorData* data) {
+ // Default descriptor configuration: one result, all parameters are passed
+ // in registers and all parameters have MachineType::AnyTagged() type.
+ data->InitializePlatformIndependent(
+ CallInterfaceDescriptorData::kNoFlags, 1, data->register_param_count(),
+ nullptr, 0, StackArgumentOrder::kDefault);
+ }
+
+ // Initializes |data| using the platform dependent default set of registers.
+ // It is intended to be used for TurboFan stubs when particular set of
+ // registers does not matter.
+ static void DefaultInitializePlatformSpecific(
+ CallInterfaceDescriptorData* data, int register_parameter_count);
+
+ // Initializes |data| using the platform dependent default set of registers
+ // for JavaScript-compatible calling convention.
+ // It is intended to be used for TurboFan stubs being called with JavaScript
+ // linkage + additional parameters on registers and stack.
+ static void JSDefaultInitializePlatformSpecific(
+ CallInterfaceDescriptorData* data, int non_js_register_parameter_count);
+
+ // Checks if float parameters are not assigned invalid registers.
+ bool CheckFloatingPointParameters(CallInterfaceDescriptorData* data) {
+ for (int i = 0; i < data->register_param_count(); i++) {
+ if (IsFloatingPoint(data->param_type(i).representation())) {
+ if (!IsValidFloatParameterRegister(data->register_param(i))) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ bool IsValidFloatParameterRegister(Register reg);
+
+ private:
+ // {CallDescriptors} is allowed to call the private {Initialize} method.
+ friend class CallDescriptors;
+
+ const CallInterfaceDescriptorData* data_;
+
+ void Initialize(CallInterfaceDescriptorData* data) {
+ // The passed pointer should be a modifiable pointer to our own data.
+ DCHECK_EQ(data, data_);
+ DCHECK(!data->IsInitialized());
+ InitializePlatformSpecific(data);
+ InitializePlatformIndependent(data);
+ DCHECK(data->IsInitialized());
+ DCHECK(CheckFloatingPointParameters(data));
+ }
+};
+
+#define DECLARE_DESCRIPTOR_WITH_BASE(name, base) \
+ public: \
+ explicit name() : base(key()) {} \
+ static inline CallDescriptors::Key key();
+
+#if defined(V8_TARGET_ARCH_IA32)
+// To support all possible cases, we must limit the number of register args for
+// TFS builtins on ia32 to 3. Out of the 6 allocatable registers, esi is taken
+// as the context register and ebx is the root register. One register must
+// remain available to store the jump/call target. Thus 3 registers remain for
+// arguments. The reason this applies to TFS builtins specifically is because
+// this becomes relevant for builtins used as targets of Torque function
+// pointers (which must have a register available to store the target).
+// TODO(jgruber): Ideally we should just decrement kMaxBuiltinRegisterParams but
+// that comes with its own set of complications. It's possible, but requires
+// refactoring the calling convention of other existing stubs.
+constexpr int kMaxBuiltinRegisterParams = 4;
+constexpr int kMaxTFSBuiltinRegisterParams = 3;
+#else
+constexpr int kMaxBuiltinRegisterParams = 5;
+constexpr int kMaxTFSBuiltinRegisterParams = kMaxBuiltinRegisterParams;
+#endif
+STATIC_ASSERT(kMaxTFSBuiltinRegisterParams <= kMaxBuiltinRegisterParams);
+
+#define DECLARE_DEFAULT_DESCRIPTOR(name, base) \
+ DECLARE_DESCRIPTOR_WITH_BASE(name, base) \
+ protected: \
+ static const int kRegisterParams = \
+ kParameterCount > kMaxTFSBuiltinRegisterParams \
+ ? kMaxTFSBuiltinRegisterParams \
+ : kParameterCount; \
+ static const int kStackParams = kParameterCount - kRegisterParams; \
+ void InitializePlatformSpecific(CallInterfaceDescriptorData* data) \
+ override { \
+ DefaultInitializePlatformSpecific(data, kRegisterParams); \
+ } \
+ void InitializePlatformIndependent(CallInterfaceDescriptorData* data) \
+ override { \
+ data->InitializePlatformIndependent(Flags(kDescriptorFlags), kReturnCount, \
+ kParameterCount, nullptr, 0, \
+ kStackArgumentOrder); \
+ } \
+ name(CallDescriptors::Key key) : base(key) {} \
+ \
+ public:
+
+#define DECLARE_JS_COMPATIBLE_DESCRIPTOR(name, base, \
+ non_js_reg_parameters_count) \
+ DECLARE_DESCRIPTOR_WITH_BASE(name, base) \
+ protected: \
+ void InitializePlatformSpecific(CallInterfaceDescriptorData* data) \
+ override { \
+ JSDefaultInitializePlatformSpecific(data, non_js_reg_parameters_count); \
+ } \
+ name(CallDescriptors::Key key) : base(key) {} \
+ \
+ public:
+
+#define DEFINE_FLAGS_AND_RESULT_AND_PARAMETERS(flags, stack_order, \
+ return_count, ...) \
+ static constexpr int kDescriptorFlags = flags; \
+ static constexpr int kReturnCount = return_count; \
+ static constexpr StackArgumentOrder kStackArgumentOrder = stack_order; \
+ enum ParameterIndices { \
+ __dummy = -1, /* to be able to pass zero arguments */ \
+ ##__VA_ARGS__, \
+ \
+ kParameterCount, \
+ kContext = kParameterCount /* implicit parameter */ \
+ };
+
+#define DEFINE_RESULT_AND_PARAMETERS(return_count, ...) \
+ DEFINE_FLAGS_AND_RESULT_AND_PARAMETERS( \
+ CallInterfaceDescriptorData::kNoFlags, StackArgumentOrder::kDefault, \
+ return_count, ##__VA_ARGS__)
+
+// This is valid only for builtins that use EntryFrame, which does not scan
+// stack arguments on GC.
+#define DEFINE_PARAMETERS_ENTRY(...) \
+ static constexpr int kDescriptorFlags = \
+ CallInterfaceDescriptorData::kNoContext | \
+ CallInterfaceDescriptorData::kNoStackScan; \
+ static constexpr StackArgumentOrder kStackArgumentOrder = \
+ StackArgumentOrder::kDefault; \
+ static constexpr int kReturnCount = 1; \
+ enum ParameterIndices { \
+ __dummy = -1, /* to be able to pass zero arguments */ \
+ ##__VA_ARGS__, \
+ \
+ kParameterCount \
+ };
+
+#define DEFINE_PARAMETERS(...) \
+ DEFINE_FLAGS_AND_RESULT_AND_PARAMETERS( \
+ CallInterfaceDescriptorData::kNoFlags, StackArgumentOrder::kDefault, 1, \
+ ##__VA_ARGS__)
+
+#define DEFINE_PARAMETERS_NO_CONTEXT(...) \
+ DEFINE_FLAGS_AND_RESULT_AND_PARAMETERS( \
+ CallInterfaceDescriptorData::kNoContext, StackArgumentOrder::kDefault, \
+ 1, ##__VA_ARGS__)
+
+#define DEFINE_PARAMETERS_VARARGS(...) \
+ DEFINE_FLAGS_AND_RESULT_AND_PARAMETERS( \
+ CallInterfaceDescriptorData::kAllowVarArgs, StackArgumentOrder::kJS, 1, \
+ ##__VA_ARGS__)
+
+#define DEFINE_RESULT_AND_PARAMETER_TYPES_WITH_FLAG(flag, ...) \
+ void InitializePlatformIndependent(CallInterfaceDescriptorData* data) \
+ override { \
+ MachineType machine_types[] = {__VA_ARGS__}; \
+ static_assert( \
+ kReturnCount + kParameterCount == arraysize(machine_types), \
+ "Parameter names definition is not consistent with parameter types"); \
+ data->InitializePlatformIndependent( \
+ Flags(flag | kDescriptorFlags), kReturnCount, kParameterCount, \
+ machine_types, arraysize(machine_types), kStackArgumentOrder); \
+ }
+
+#define DEFINE_RESULT_AND_PARAMETER_TYPES(...) \
+ DEFINE_RESULT_AND_PARAMETER_TYPES_WITH_FLAG( \
+ CallInterfaceDescriptorData::kNoFlags, __VA_ARGS__)
+
+#define DEFINE_PARAMETER_TYPES(...) \
+ DEFINE_RESULT_AND_PARAMETER_TYPES(MachineType::AnyTagged() /* result */, \
+ ##__VA_ARGS__)
+
+// When the extra arguments described here are located in the stack, they are
+// just above the return address in the frame (first arguments).
+#define DEFINE_JS_PARAMETERS(...) \
+ static constexpr int kDescriptorFlags = \
+ CallInterfaceDescriptorData::kAllowVarArgs; \
+ static constexpr int kReturnCount = 1; \
+ static constexpr StackArgumentOrder kStackArgumentOrder = \
+ StackArgumentOrder::kJS; \
+ enum ParameterIndices { \
+ kTarget, \
+ kNewTarget, \
+ kActualArgumentsCount, \
+ ##__VA_ARGS__, \
+ \
+ kParameterCount, \
+ kContext = kParameterCount /* implicit parameter */ \
+ };
+
+#define DEFINE_JS_PARAMETER_TYPES(...) \
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), /* kTarget */ \
+ MachineType::AnyTagged(), /* kNewTarget */ \
+ MachineType::Int32(), /* kActualArgumentsCount */ \
+ ##__VA_ARGS__)
+
+#define DECLARE_DESCRIPTOR(name, base) \
+ DECLARE_DESCRIPTOR_WITH_BASE(name, base) \
+ protected: \
+ void InitializePlatformSpecific(CallInterfaceDescriptorData* data) override; \
+ name(CallDescriptors::Key key) : base(key) {} \
+ \
+ public:
+
+class V8_EXPORT_PRIVATE VoidDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS()
+ DEFINE_PARAMETER_TYPES()
+ DECLARE_DESCRIPTOR(VoidDescriptor, CallInterfaceDescriptor)
+};
+
+// This class is subclassed by Torque-generated call interface descriptors.
+template <int parameter_count, bool has_context_parameter>
+class TorqueInterfaceDescriptor : public CallInterfaceDescriptor {
+ public:
+ static constexpr int kDescriptorFlags =
+ has_context_parameter ? CallInterfaceDescriptorData::kNoFlags
+ : CallInterfaceDescriptorData::kNoContext;
+ static constexpr int kParameterCount = parameter_count;
+ enum ParameterIndices { kContext = kParameterCount };
+ template <int i>
+ static ParameterIndices ParameterIndex() {
+ STATIC_ASSERT(0 <= i && i < kParameterCount);
+ return static_cast<ParameterIndices>(i);
+ }
+ static constexpr int kReturnCount = 1;
+
+ using CallInterfaceDescriptor::CallInterfaceDescriptor;
+
+ protected:
+ static const int kRegisterParams =
+ kParameterCount > kMaxTFSBuiltinRegisterParams
+ ? kMaxTFSBuiltinRegisterParams
+ : kParameterCount;
+ static const int kStackParams = kParameterCount - kRegisterParams;
+ virtual MachineType ReturnType() = 0;
+ virtual std::array<MachineType, kParameterCount> ParameterTypes() = 0;
+ void InitializePlatformSpecific(CallInterfaceDescriptorData* data) override {
+ DefaultInitializePlatformSpecific(data, kRegisterParams);
+ }
+ void InitializePlatformIndependent(
+ CallInterfaceDescriptorData* data) override {
+ std::vector<MachineType> machine_types = {ReturnType()};
+ auto parameter_types = ParameterTypes();
+ machine_types.insert(machine_types.end(), parameter_types.begin(),
+ parameter_types.end());
+ DCHECK_EQ(kReturnCount + kParameterCount, machine_types.size());
+ data->InitializePlatformIndependent(Flags(kDescriptorFlags), kReturnCount,
+ kParameterCount, machine_types.data(),
+ static_cast<int>(machine_types.size()),
+ StackArgumentOrder::kDefault);
+ }
+};
+
+// Dummy descriptor used to mark builtins that don't yet have their proper
+// descriptor associated.
+using DummyDescriptor = VoidDescriptor;
+
+// Dummy descriptor that marks builtins with C calling convention.
+using CCallDescriptor = VoidDescriptor;
+
+// Marks deoptimization entry builtins. Precise calling conventions currently
+// differ based on the platform.
+// TODO(jgruber): Once this is unified, we could create a better description
+// here.
+using DeoptimizationEntryDescriptor = VoidDescriptor;
+
+class AllocateDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS_NO_CONTEXT(kRequestedSize)
+ DEFINE_RESULT_AND_PARAMETER_TYPES(MachineType::TaggedPointer(), // result 1
+ MachineType::IntPtr()) // kRequestedSize
+ DECLARE_DESCRIPTOR(AllocateDescriptor, CallInterfaceDescriptor)
+};
+
+// This descriptor defines the JavaScript calling convention that can be used
+// by stubs: target, new.target, argc (not including the receiver) and context
+// are passed in registers while receiver and the rest of the JS arguments are
+// passed on the stack.
+class JSTrampolineDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_JS_PARAMETERS()
+ DEFINE_JS_PARAMETER_TYPES()
+
+ DECLARE_JS_COMPATIBLE_DESCRIPTOR(JSTrampolineDescriptor,
+ CallInterfaceDescriptor, 0)
+};
+
+class ContextOnlyDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS()
+ DEFINE_PARAMETER_TYPES()
+ DECLARE_DESCRIPTOR(ContextOnlyDescriptor, CallInterfaceDescriptor)
+};
+
+class NoContextDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS_NO_CONTEXT()
+ DEFINE_PARAMETER_TYPES()
+ DECLARE_DESCRIPTOR(NoContextDescriptor, CallInterfaceDescriptor)
+};
+
+// LoadDescriptor is used by all stubs that implement Load/KeyedLoad ICs.
+class LoadDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kReceiver, kName, kSlot)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kReceiver
+ MachineType::AnyTagged(), // kName
+ MachineType::TaggedSigned()) // kSlot
+ DECLARE_DESCRIPTOR(LoadDescriptor, CallInterfaceDescriptor)
+
+ static const Register ReceiverRegister();
+ static const Register NameRegister();
+ static const Register SlotRegister();
+};
+
+class LoadGlobalNoFeedbackDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kName, kICKind)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kName
+ MachineType::TaggedSigned()) // kICKind
+ DECLARE_DESCRIPTOR(LoadGlobalNoFeedbackDescriptor, CallInterfaceDescriptor)
+
+ static const Register NameRegister() {
+ return LoadDescriptor::NameRegister();
+ }
+
+ static const Register ICKindRegister() {
+ return LoadDescriptor::SlotRegister();
+ }
+};
+
+class LoadNoFeedbackDescriptor : public LoadGlobalNoFeedbackDescriptor {
+ public:
+ DEFINE_PARAMETERS(kReceiver, kName, kICKind)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kReceiver
+ MachineType::AnyTagged(), // kName
+ MachineType::TaggedSigned()) // kICKind
+ DECLARE_DESCRIPTOR(LoadNoFeedbackDescriptor, LoadGlobalNoFeedbackDescriptor)
+
+ static const Register ReceiverRegister() {
+ return LoadDescriptor::ReceiverRegister();
+ }
+
+ static const Register NameRegister() {
+ return LoadGlobalNoFeedbackDescriptor::NameRegister();
+ }
+
+ static const Register ICKindRegister() {
+ return LoadGlobalNoFeedbackDescriptor::ICKindRegister();
+ }
+};
+
+class LoadGlobalDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kName, kSlot)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kName
+ MachineType::TaggedSigned()) // kSlot
+ DECLARE_DESCRIPTOR(LoadGlobalDescriptor, CallInterfaceDescriptor)
+
+ static const Register NameRegister() {
+ return LoadDescriptor::NameRegister();
+ }
+
+ static const Register SlotRegister() {
+ return LoadDescriptor::SlotRegister();
+ }
+};
+
+class StoreDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kReceiver, kName, kValue, kSlot)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kReceiver
+ MachineType::AnyTagged(), // kName
+ MachineType::AnyTagged(), // kValue
+ MachineType::TaggedSigned()) // kSlot
+ DECLARE_DESCRIPTOR(StoreDescriptor, CallInterfaceDescriptor)
+
+ static const Register ReceiverRegister();
+ static const Register NameRegister();
+ static const Register ValueRegister();
+ static const Register SlotRegister();
+
+#if V8_TARGET_ARCH_IA32
+ static const bool kPassLastArgsOnStack = true;
+#else
+ static const bool kPassLastArgsOnStack = false;
+#endif
+
+ // Pass value and slot through the stack.
+ static const int kStackArgumentsCount = kPassLastArgsOnStack ? 2 : 0;
+};
+
+class StoreTransitionDescriptor : public StoreDescriptor {
+ public:
+ DEFINE_PARAMETERS(kReceiver, kName, kMap, kValue, kSlot, kVector)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kReceiver
+ MachineType::AnyTagged(), // kName
+ MachineType::AnyTagged(), // kMap
+ MachineType::AnyTagged(), // kValue
+ MachineType::TaggedSigned(), // kSlot
+ MachineType::AnyTagged()) // kVector
+ DECLARE_DESCRIPTOR(StoreTransitionDescriptor, StoreDescriptor)
+
+ static const Register MapRegister();
+ static const Register SlotRegister();
+ static const Register VectorRegister();
+
+ // Pass value, slot and vector through the stack.
+ static const int kStackArgumentsCount = kPassLastArgsOnStack ? 3 : 0;
+};
+
+class StoreWithVectorDescriptor : public StoreDescriptor {
+ public:
+ DEFINE_PARAMETERS(kReceiver, kName, kValue, kSlot, kVector)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kReceiver
+ MachineType::AnyTagged(), // kName
+ MachineType::AnyTagged(), // kValue
+ MachineType::TaggedSigned(), // kSlot
+ MachineType::AnyTagged()) // kVector
+ DECLARE_DESCRIPTOR(StoreWithVectorDescriptor, StoreDescriptor)
+
+ static const Register VectorRegister();
+
+ // Pass value, slot and vector through the stack.
+ static const int kStackArgumentsCount = kPassLastArgsOnStack ? 3 : 0;
+};
+
+class StoreGlobalDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kName, kValue, kSlot)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kName
+ MachineType::AnyTagged(), // kValue
+ MachineType::TaggedSigned()) // kSlot
+ DECLARE_DESCRIPTOR(StoreGlobalDescriptor, CallInterfaceDescriptor)
+
+ static const bool kPassLastArgsOnStack =
+ StoreDescriptor::kPassLastArgsOnStack;
+ // Pass value and slot through the stack.
+ static const int kStackArgumentsCount = kPassLastArgsOnStack ? 2 : 0;
+
+ static const Register NameRegister() {
+ return StoreDescriptor::NameRegister();
+ }
+
+ static const Register ValueRegister() {
+ return StoreDescriptor::ValueRegister();
+ }
+
+ static const Register SlotRegister() {
+ return StoreDescriptor::SlotRegister();
+ }
+};
+
+class StoreGlobalWithVectorDescriptor : public StoreGlobalDescriptor {
+ public:
+ DEFINE_PARAMETERS(kName, kValue, kSlot, kVector)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kName
+ MachineType::AnyTagged(), // kValue
+ MachineType::TaggedSigned(), // kSlot
+ MachineType::AnyTagged()) // kVector
+ DECLARE_DESCRIPTOR(StoreGlobalWithVectorDescriptor, StoreGlobalDescriptor)
+
+ static const Register VectorRegister() {
+ return StoreWithVectorDescriptor::VectorRegister();
+ }
+
+ // Pass value, slot and vector through the stack.
+ static const int kStackArgumentsCount = kPassLastArgsOnStack ? 3 : 0;
+};
+
+class LoadWithVectorDescriptor : public LoadDescriptor {
+ public:
+ // TODO(v8:9497): Revert the Machine type for kSlot to the
+ // TaggedSigned once Torque can emit better call descriptors
+ DEFINE_PARAMETERS(kReceiver, kName, kSlot, kVector)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kReceiver
+ MachineType::AnyTagged(), // kName
+ MachineType::AnyTagged(), // kSlot
+ MachineType::AnyTagged()) // kVector
+ DECLARE_DESCRIPTOR(LoadWithVectorDescriptor, LoadDescriptor)
+
+ static const Register VectorRegister();
+
+#if V8_TARGET_ARCH_IA32
+ static const bool kPassLastArgsOnStack = true;
+#else
+ static const bool kPassLastArgsOnStack = false;
+#endif
+
+ // Pass vector through the stack.
+ static const int kStackArgumentsCount = kPassLastArgsOnStack ? 1 : 0;
+};
+
+// Like LoadWithVectorDescriptor, except we pass the receiver (the object which
+// should be used as the receiver for accessor function calls) and the lookup
+// start object separately.
+class LoadWithReceiverAndVectorDescriptor : public LoadWithVectorDescriptor {
+ public:
+ // TODO(v8:9497): Revert the Machine type for kSlot to the
+ // TaggedSigned once Torque can emit better call descriptors
+ DEFINE_PARAMETERS(kReceiver, kLookupStartObject, kName, kSlot, kVector)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kReceiver
+ MachineType::AnyTagged(), // kLookupStartObject
+ MachineType::AnyTagged(), // kName
+ MachineType::AnyTagged(), // kSlot
+ MachineType::AnyTagged()) // kVector
+ DECLARE_DESCRIPTOR(LoadWithReceiverAndVectorDescriptor,
+ LoadWithVectorDescriptor)
+
+ static const Register LookupStartObjectRegister();
+
+#if V8_TARGET_ARCH_IA32
+ static const bool kPassLastArgsOnStack = true;
+#else
+ static const bool kPassLastArgsOnStack = false;
+#endif
+
+ // Pass vector through the stack.
+ static const int kStackArgumentsCount = kPassLastArgsOnStack ? 1 : 0;
+};
+
+class LoadGlobalWithVectorDescriptor : public LoadGlobalDescriptor {
+ public:
+ DEFINE_PARAMETERS(kName, kSlot, kVector)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kName
+ MachineType::TaggedSigned(), // kSlot
+ MachineType::AnyTagged()) // kVector
+ DECLARE_DESCRIPTOR(LoadGlobalWithVectorDescriptor, LoadGlobalDescriptor)
+
+#if V8_TARGET_ARCH_IA32
+ // On ia32, LoadWithVectorDescriptor passes vector on the stack and thus we
+ // need to choose a new register here.
+ static const Register VectorRegister() { return edx; }
+#else
+ static const Register VectorRegister() {
+ return LoadWithVectorDescriptor::VectorRegister();
+ }
+#endif
+};
+
+class FastNewObjectDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kTarget, kNewTarget)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kTarget
+ MachineType::AnyTagged()) // kNewTarget
+ DECLARE_DESCRIPTOR(FastNewObjectDescriptor, CallInterfaceDescriptor)
+ static const Register TargetRegister();
+ static const Register NewTargetRegister();
+};
+
+class RecordWriteDescriptor final : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS_NO_CONTEXT(kObject, kSlot, kRememberedSet, kFPMode)
+ DEFINE_PARAMETER_TYPES(MachineType::TaggedPointer(), // kObject
+ MachineType::Pointer(), // kSlot
+ MachineType::TaggedSigned(), // kRememberedSet
+ MachineType::TaggedSigned()) // kFPMode
+
+ DECLARE_DESCRIPTOR(RecordWriteDescriptor, CallInterfaceDescriptor)
+};
+
+class EphemeronKeyBarrierDescriptor final : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS_NO_CONTEXT(kObject, kSlotAddress, kFPMode)
+ DEFINE_PARAMETER_TYPES(MachineType::TaggedPointer(), // kObject
+ MachineType::Pointer(), // kSlotAddress
+ MachineType::TaggedSigned()) // kFPMode
+
+ DECLARE_DESCRIPTOR(EphemeronKeyBarrierDescriptor, CallInterfaceDescriptor)
+};
+
+class TypeConversionDescriptor final : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kArgument)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged())
+ DECLARE_DESCRIPTOR(TypeConversionDescriptor, CallInterfaceDescriptor)
+
+ static const Register ArgumentRegister();
+};
+
+class TypeConversionNoContextDescriptor final : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS_NO_CONTEXT(kArgument)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged())
+ DECLARE_DESCRIPTOR(TypeConversionNoContextDescriptor, CallInterfaceDescriptor)
+};
+
+class TypeConversionStackParameterDescriptor final
+ : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kArgument)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged())
+ DECLARE_DESCRIPTOR(TypeConversionStackParameterDescriptor,
+ CallInterfaceDescriptor)
+};
+
+class AsyncFunctionStackParameterDescriptor final
+ : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kPromise, kResult)
+ DEFINE_PARAMETER_TYPES(MachineType::TaggedPointer(), MachineType::AnyTagged())
+ DECLARE_DESCRIPTOR(AsyncFunctionStackParameterDescriptor,
+ CallInterfaceDescriptor)
+};
+
+class GetIteratorStackParameterDescriptor final
+ : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kReceiver, kCallSlot, kFeedback, kResult)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), MachineType::AnyTagged(),
+ MachineType::AnyTagged(), MachineType::AnyTagged())
+ DECLARE_DESCRIPTOR(GetIteratorStackParameterDescriptor,
+ CallInterfaceDescriptor)
+};
+
+class GetPropertyDescriptor final : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kObject, kKey)
+ DECLARE_DEFAULT_DESCRIPTOR(GetPropertyDescriptor, CallInterfaceDescriptor)
+};
+
+class TypeofDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kObject)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged())
+ DECLARE_DESCRIPTOR(TypeofDescriptor, CallInterfaceDescriptor)
+};
+
+class CallTrampolineDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS_VARARGS(kFunction, kActualArgumentsCount)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kFunction
+ MachineType::Int32()) // kActualArgumentsCount
+ DECLARE_DESCRIPTOR(CallTrampolineDescriptor, CallInterfaceDescriptor)
+};
+
+class CallVarargsDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS_VARARGS(kTarget, kActualArgumentsCount, kArgumentsLength,
+ kArgumentsList)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kTarget
+ MachineType::Int32(), // kActualArgumentsCount
+ MachineType::Int32(), // kArgumentsLength
+ MachineType::AnyTagged()) // kArgumentsList
+ DECLARE_DESCRIPTOR(CallVarargsDescriptor, CallInterfaceDescriptor)
+};
+
+class CallForwardVarargsDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS_VARARGS(kTarget, kActualArgumentsCount, kStartIndex)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kTarget
+ MachineType::Int32(), // kActualArgumentsCount
+ MachineType::Int32()) // kStartIndex
+ DECLARE_DESCRIPTOR(CallForwardVarargsDescriptor, CallInterfaceDescriptor)
+};
+
+class CallFunctionTemplateDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS_VARARGS(kFunctionTemplateInfo, kArgumentsCount)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kFunctionTemplateInfo
+ MachineType::IntPtr()) // kArgumentsCount
+ DECLARE_DESCRIPTOR(CallFunctionTemplateDescriptor, CallInterfaceDescriptor)
+};
+
+class CallWithSpreadDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS_VARARGS(kTarget, kArgumentsCount, kSpread)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kTarget
+ MachineType::Int32(), // kArgumentsCount
+ MachineType::AnyTagged()) // kSpread
+ DECLARE_DESCRIPTOR(CallWithSpreadDescriptor, CallInterfaceDescriptor)
+};
+
+// TODO(jgruber): Pass the slot as UintPtr.
+class CallWithSpread_WithFeedbackDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS_VARARGS(kTarget, kArgumentsCount, kSpread, kSlot,
+ kMaybeFeedbackVector)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kTarget
+ MachineType::Int32(), // kArgumentsCount
+ MachineType::AnyTagged(), // kSpread
+ MachineType::Int32(), // kSlot
+ MachineType::AnyTagged()) // kMaybeFeedbackVector
+ DECLARE_DESCRIPTOR(CallWithSpread_WithFeedbackDescriptor,
+ CallInterfaceDescriptor)
+};
+
+class CallWithArrayLikeDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kTarget, kArgumentsList)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kTarget
+ MachineType::AnyTagged()) // kArgumentsList
+ DECLARE_DESCRIPTOR(CallWithArrayLikeDescriptor, CallInterfaceDescriptor)
+};
+
+// TODO(jgruber): Pass the slot as UintPtr.
+class CallWithArrayLike_WithFeedbackDescriptor
+ : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kTarget, kArgumentsList, kSlot, kMaybeFeedbackVector)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kTarget
+ MachineType::AnyTagged(), // kArgumentsList
+ MachineType::Int32(), // kSlot
+ MachineType::AnyTagged()) // kMaybeFeedbackVector
+ DECLARE_DESCRIPTOR(CallWithArrayLike_WithFeedbackDescriptor,
+ CallInterfaceDescriptor)
+};
+
+class ConstructVarargsDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_JS_PARAMETERS(kArgumentsLength, kArgumentsList)
+ DEFINE_JS_PARAMETER_TYPES(MachineType::Int32(), // kArgumentsLength
+ MachineType::AnyTagged()) // kArgumentsList
+
+ DECLARE_DESCRIPTOR(ConstructVarargsDescriptor, CallInterfaceDescriptor)
+};
+
+class ConstructForwardVarargsDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_JS_PARAMETERS(kStartIndex)
+ DEFINE_JS_PARAMETER_TYPES(MachineType::Int32())
+ DECLARE_DESCRIPTOR(ConstructForwardVarargsDescriptor, CallInterfaceDescriptor)
+};
+
+class ConstructWithSpreadDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_JS_PARAMETERS(kSpread)
+ DEFINE_JS_PARAMETER_TYPES(MachineType::AnyTagged())
+ DECLARE_DESCRIPTOR(ConstructWithSpreadDescriptor, CallInterfaceDescriptor)
+};
+
+// TODO(jgruber): Pass the slot as UintPtr.
+class ConstructWithSpread_WithFeedbackDescriptor
+ : public CallInterfaceDescriptor {
+ public:
+ // Note: kSlot comes before kSpread since as an untagged value it must be
+ // passed in a register.
+ DEFINE_JS_PARAMETERS(kSlot, kSpread, kMaybeFeedbackVector)
+ DEFINE_JS_PARAMETER_TYPES(MachineType::Int32(), // kSlot
+ MachineType::AnyTagged(), // kSpread
+ MachineType::AnyTagged()) // kMaybeFeedbackVector
+ DECLARE_DESCRIPTOR(ConstructWithSpread_WithFeedbackDescriptor,
+ CallInterfaceDescriptor)
+};
+
+class ConstructWithArrayLikeDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kTarget, kNewTarget, kArgumentsList)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kTarget
+ MachineType::AnyTagged(), // kNewTarget
+ MachineType::AnyTagged()) // kArgumentsList
+ DECLARE_DESCRIPTOR(ConstructWithArrayLikeDescriptor, CallInterfaceDescriptor)
+};
+
+// TODO(jgruber): Pass the slot as UintPtr.
+class ConstructWithArrayLike_WithFeedbackDescriptor
+ : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kTarget, kNewTarget, kArgumentsList, kSlot,
+ kMaybeFeedbackVector)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kTarget
+ MachineType::AnyTagged(), // kNewTarget
+ MachineType::AnyTagged(), // kArgumentsList
+ MachineType::Int32(), // kSlot
+ MachineType::AnyTagged()) // kMaybeFeedbackVector
+ DECLARE_DESCRIPTOR(ConstructWithArrayLike_WithFeedbackDescriptor,
+ CallInterfaceDescriptor)
+};
+
+// TODO(ishell): consider merging this with ArrayConstructorDescriptor
+class ConstructStubDescriptor : public CallInterfaceDescriptor {
+ public:
+ // TODO(jgruber): Remove the unused allocation site parameter.
+ DEFINE_JS_PARAMETERS(kAllocationSite)
+ DEFINE_JS_PARAMETER_TYPES(MachineType::AnyTagged())
+
+ // TODO(ishell): Use DECLARE_JS_COMPATIBLE_DESCRIPTOR if registers match
+ DECLARE_DESCRIPTOR(ConstructStubDescriptor, CallInterfaceDescriptor)
+};
+
+class AbortDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS_NO_CONTEXT(kMessageOrMessageId)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged())
+ DECLARE_DESCRIPTOR(AbortDescriptor, CallInterfaceDescriptor)
+};
+
+class ArrayConstructorDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_JS_PARAMETERS(kAllocationSite)
+ DEFINE_JS_PARAMETER_TYPES(MachineType::AnyTagged())
+
+ DECLARE_JS_COMPATIBLE_DESCRIPTOR(ArrayConstructorDescriptor,
+ CallInterfaceDescriptor, 1)
+};
+
+class ArrayNArgumentsConstructorDescriptor : public CallInterfaceDescriptor {
+ public:
+ // This descriptor declares only register arguments while respective number
+ // of JS arguments stay on the expression stack.
+ // The ArrayNArgumentsConstructor builtin does not access stack arguments
+ // directly it just forwards them to the runtime function.
+ DEFINE_PARAMETERS(kFunction, kAllocationSite, kActualArgumentsCount)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kFunction,
+ MachineType::AnyTagged(), // kAllocationSite
+ MachineType::Int32()) // kActualArgumentsCount
+ DECLARE_DESCRIPTOR(ArrayNArgumentsConstructorDescriptor,
+ CallInterfaceDescriptor)
+};
+
+class ArrayNoArgumentConstructorDescriptor
+ : public ArrayNArgumentsConstructorDescriptor {
+ public:
+ // This descriptor declares same register arguments as the parent
+ // ArrayNArgumentsConstructorDescriptor and it declares indices for
+ // JS arguments passed on the expression stack.
+ DEFINE_PARAMETERS(kFunction, kAllocationSite, kActualArgumentsCount,
+ kFunctionParameter)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kFunction
+ MachineType::AnyTagged(), // kAllocationSite
+ MachineType::Int32(), // kActualArgumentsCount
+ MachineType::AnyTagged()) // kFunctionParameter
+ DECLARE_DESCRIPTOR(ArrayNoArgumentConstructorDescriptor,
+ ArrayNArgumentsConstructorDescriptor)
+};
+
+class ArraySingleArgumentConstructorDescriptor
+ : public ArrayNArgumentsConstructorDescriptor {
+ public:
+ // This descriptor declares same register arguments as the parent
+ // ArrayNArgumentsConstructorDescriptor and it declares indices for
+ // JS arguments passed on the expression stack.
+ DEFINE_PARAMETERS(kFunction, kAllocationSite, kActualArgumentsCount,
+ kArraySizeSmiParameter, kReceiverParameter)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kFunction
+ MachineType::AnyTagged(), // kAllocationSite
+ MachineType::Int32(), // kActualArgumentsCount
+ // JS arguments on the stack
+ MachineType::AnyTagged(), // kArraySizeSmiParameter
+ MachineType::AnyTagged()) // kReceiverParameter
+ DECLARE_DESCRIPTOR(ArraySingleArgumentConstructorDescriptor,
+ ArrayNArgumentsConstructorDescriptor)
+};
+
+class CompareDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kLeft, kRight)
+ DECLARE_DESCRIPTOR(CompareDescriptor, CallInterfaceDescriptor)
+};
+
+class BinaryOpDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kLeft, kRight)
+ DECLARE_DESCRIPTOR(BinaryOpDescriptor, CallInterfaceDescriptor)
+};
+
+// This desciptor is shared among String.p.charAt/charCodeAt/codePointAt
+// as they all have the same interface.
+class StringAtDescriptor final : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kReceiver, kPosition)
+ // TODO(turbofan): Return untagged value here.
+ DEFINE_RESULT_AND_PARAMETER_TYPES(MachineType::TaggedSigned(), // result 1
+ MachineType::AnyTagged(), // kReceiver
+ MachineType::IntPtr()) // kPosition
+ DECLARE_DESCRIPTOR(StringAtDescriptor, CallInterfaceDescriptor)
+};
+
+class StringAtAsStringDescriptor final : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kReceiver, kPosition)
+ // TODO(turbofan): Return untagged value here.
+ DEFINE_RESULT_AND_PARAMETER_TYPES(
+ MachineType::TaggedPointer(), // result string
+ MachineType::AnyTagged(), // kReceiver
+ MachineType::IntPtr()) // kPosition
+ DECLARE_DESCRIPTOR(StringAtAsStringDescriptor, CallInterfaceDescriptor)
+};
+
+class StringSubstringDescriptor final : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kString, kFrom, kTo)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kString
+ MachineType::IntPtr(), // kFrom
+ MachineType::IntPtr()) // kTo
+
+ // TODO(turbofan): Allow builtins to return untagged values.
+ DECLARE_DESCRIPTOR(StringSubstringDescriptor, CallInterfaceDescriptor)
+};
+
+class ArgumentsAdaptorDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_JS_PARAMETERS(kExpectedArgumentsCount)
+ DEFINE_JS_PARAMETER_TYPES(MachineType::Int32())
+ DECLARE_DESCRIPTOR(ArgumentsAdaptorDescriptor, CallInterfaceDescriptor)
+};
+
+class CppBuiltinAdaptorDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_JS_PARAMETERS(kCFunction)
+ DEFINE_JS_PARAMETER_TYPES(MachineType::Pointer())
+ DECLARE_JS_COMPATIBLE_DESCRIPTOR(CppBuiltinAdaptorDescriptor,
+ CallInterfaceDescriptor, 1)
+};
+
+class CEntry1ArgvOnStackDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kArity, // register argument
+ kCFunction, // register argument
+ kPadding, // stack argument 1 (just padding)
+ kArgcSmi, // stack argument 2
+ kTargetCopy, // stack argument 3
+ kNewTargetCopy) // stack argument 4
+ DEFINE_PARAMETER_TYPES(MachineType::Int32(), // kArity
+ MachineType::Pointer(), // kCFunction
+ MachineType::AnyTagged(), // kPadding
+ MachineType::AnyTagged(), // kArgcSmi
+ MachineType::AnyTagged(), // kTargetCopy
+ MachineType::AnyTagged()) // kNewTargetCopy
+ DECLARE_DESCRIPTOR(CEntry1ArgvOnStackDescriptor, CallInterfaceDescriptor)
+};
+
+class ApiCallbackDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS_VARARGS(kApiFunctionAddress, kActualArgumentsCount,
+ kCallData, kHolder)
+ // receiver is implicit stack argument 1
+ // argv are implicit stack arguments [2, 2 + kArgc[
+ DEFINE_PARAMETER_TYPES(MachineType::Pointer(), // kApiFunctionAddress
+ MachineType::IntPtr(), // kActualArgumentsCount
+ MachineType::AnyTagged(), // kCallData
+ MachineType::AnyTagged()) // kHolder
+ DECLARE_DESCRIPTOR(ApiCallbackDescriptor, CallInterfaceDescriptor)
+};
+
+class ApiGetterDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kReceiver, kHolder, kCallback)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kReceiver
+ MachineType::AnyTagged(), // kHolder
+ MachineType::AnyTagged()) // kCallback
+ DECLARE_DESCRIPTOR(ApiGetterDescriptor, CallInterfaceDescriptor)
+
+ static const Register ReceiverRegister();
+ static const Register HolderRegister();
+ static const Register CallbackRegister();
+};
+
+// TODO(turbofan): We should probably rename this to GrowFastElementsDescriptor.
+class GrowArrayElementsDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kObject, kKey)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kObject
+ MachineType::AnyTagged()) // kKey
+ DECLARE_DESCRIPTOR(GrowArrayElementsDescriptor, CallInterfaceDescriptor)
+
+ static const Register ObjectRegister();
+ static const Register KeyRegister();
+};
+
+class V8_EXPORT_PRIVATE InterpreterDispatchDescriptor
+ : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kAccumulator, kBytecodeOffset, kBytecodeArray,
+ kDispatchTable)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kAccumulator
+ MachineType::IntPtr(), // kBytecodeOffset
+ MachineType::AnyTagged(), // kBytecodeArray
+ MachineType::IntPtr()) // kDispatchTable
+ DECLARE_DESCRIPTOR(InterpreterDispatchDescriptor, CallInterfaceDescriptor)
+};
+
+class InterpreterPushArgsThenCallDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kNumberOfArguments, kFirstArgument, kFunction)
+ DEFINE_PARAMETER_TYPES(MachineType::Int32(), // kNumberOfArguments
+ MachineType::Pointer(), // kFirstArgument
+ MachineType::AnyTagged()) // kFunction
+ DECLARE_DESCRIPTOR(InterpreterPushArgsThenCallDescriptor,
+ CallInterfaceDescriptor)
+};
+
+class InterpreterPushArgsThenConstructDescriptor
+ : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kNumberOfArguments, kFirstArgument, kConstructor,
+ kNewTarget, kFeedbackElement)
+ DEFINE_PARAMETER_TYPES(MachineType::Int32(), // kNumberOfArguments
+ MachineType::Pointer(), // kFirstArgument
+ MachineType::AnyTagged(), // kConstructor
+ MachineType::AnyTagged(), // kNewTarget
+ MachineType::AnyTagged()) // kFeedbackElement
+ DECLARE_DESCRIPTOR(InterpreterPushArgsThenConstructDescriptor,
+ CallInterfaceDescriptor)
+
+#if V8_TARGET_ARCH_IA32
+ static const bool kPassLastArgsOnStack = true;
+#else
+ static const bool kPassLastArgsOnStack = false;
+#endif
+
+ // Pass constructor, new target and feedback element through the stack.
+ static const int kStackArgumentsCount = kPassLastArgsOnStack ? 3 : 0;
+};
+
+class InterpreterCEntry1Descriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_RESULT_AND_PARAMETERS(1, kNumberOfArguments, kFirstArgument,
+ kFunctionEntry)
+ DEFINE_RESULT_AND_PARAMETER_TYPES(MachineType::AnyTagged(), // result 1
+ MachineType::Int32(), // kNumberOfArguments
+ MachineType::Pointer(), // kFirstArgument
+ MachineType::Pointer()) // kFunctionEntry
+ DECLARE_DESCRIPTOR(InterpreterCEntry1Descriptor, CallInterfaceDescriptor)
+};
+
+class InterpreterCEntry2Descriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_RESULT_AND_PARAMETERS(2, kNumberOfArguments, kFirstArgument,
+ kFunctionEntry)
+ DEFINE_RESULT_AND_PARAMETER_TYPES(MachineType::AnyTagged(), // result 1
+ MachineType::AnyTagged(), // result 2
+ MachineType::Int32(), // kNumberOfArguments
+ MachineType::Pointer(), // kFirstArgument
+ MachineType::Pointer()) // kFunctionEntry
+ DECLARE_DESCRIPTOR(InterpreterCEntry2Descriptor, CallInterfaceDescriptor)
+};
+
+class ResumeGeneratorDescriptor final : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kValue, kGenerator)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kValue
+ MachineType::AnyTagged()) // kGenerator
+ DECLARE_DESCRIPTOR(ResumeGeneratorDescriptor, CallInterfaceDescriptor)
+};
+
+class FrameDropperTrampolineDescriptor final : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kRestartFp)
+ DEFINE_PARAMETER_TYPES(MachineType::Pointer())
+ DECLARE_DESCRIPTOR(FrameDropperTrampolineDescriptor, CallInterfaceDescriptor)
+};
+
+class RunMicrotasksEntryDescriptor final : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS_ENTRY(kRootRegisterValue, kMicrotaskQueue)
+ DEFINE_PARAMETER_TYPES(MachineType::Pointer(), // kRootRegisterValue
+ MachineType::Pointer()) // kMicrotaskQueue
+ DECLARE_DESCRIPTOR(RunMicrotasksEntryDescriptor, CallInterfaceDescriptor)
+};
+
+class RunMicrotasksDescriptor final : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kMicrotaskQueue)
+ DEFINE_PARAMETER_TYPES(MachineType::Pointer())
+ DECLARE_DESCRIPTOR(RunMicrotasksDescriptor, CallInterfaceDescriptor)
+
+ static Register MicrotaskQueueRegister();
+};
+
+class WasmFloat32ToNumberDescriptor final : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS_NO_CONTEXT(kValue)
+ DEFINE_RESULT_AND_PARAMETER_TYPES(MachineType::AnyTagged(), // result
+ MachineType::Float32()) // value
+ DECLARE_DESCRIPTOR(WasmFloat32ToNumberDescriptor, CallInterfaceDescriptor)
+};
+
+class WasmFloat64ToNumberDescriptor final : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS_NO_CONTEXT(kValue)
+ DEFINE_RESULT_AND_PARAMETER_TYPES(MachineType::AnyTagged(), // result
+ MachineType::Float64()) // value
+ DECLARE_DESCRIPTOR(WasmFloat64ToNumberDescriptor, CallInterfaceDescriptor)
+};
+
+class V8_EXPORT_PRIVATE I64ToBigIntDescriptor final
+ : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS_NO_CONTEXT(kArgument)
+ DEFINE_PARAMETER_TYPES(MachineType::Int64()) // kArgument
+ DECLARE_DESCRIPTOR(I64ToBigIntDescriptor, CallInterfaceDescriptor)
+};
+
+// 32 bits version of the I64ToBigIntDescriptor call interface descriptor
+class V8_EXPORT_PRIVATE I32PairToBigIntDescriptor final
+ : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS_NO_CONTEXT(kLow, kHigh)
+ DEFINE_PARAMETER_TYPES(MachineType::Uint32(), // kLow
+ MachineType::Uint32()) // kHigh
+ DECLARE_DESCRIPTOR(I32PairToBigIntDescriptor, CallInterfaceDescriptor)
+};
+
+class V8_EXPORT_PRIVATE BigIntToI64Descriptor final
+ : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kArgument)
+ DEFINE_RESULT_AND_PARAMETER_TYPES(MachineType::Int64(), // result 1
+ MachineType::AnyTagged()) // kArgument
+ DECLARE_DESCRIPTOR(BigIntToI64Descriptor, CallInterfaceDescriptor)
+};
+
+class V8_EXPORT_PRIVATE BigIntToI32PairDescriptor final
+ : public CallInterfaceDescriptor {
+ public:
+ DEFINE_RESULT_AND_PARAMETERS(2, kArgument)
+ DEFINE_RESULT_AND_PARAMETER_TYPES(MachineType::Uint32(), // result 1
+ MachineType::Uint32(), // result 2
+ MachineType::AnyTagged()) // kArgument
+ DECLARE_DESCRIPTOR(BigIntToI32PairDescriptor, CallInterfaceDescriptor)
+};
+
+class WasmI32AtomicWait32Descriptor final : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS_NO_CONTEXT(kAddress, kExpectedValue, kTimeoutLow,
+ kTimeoutHigh)
+ DEFINE_RESULT_AND_PARAMETER_TYPES(MachineType::Uint32(), // result 1
+ MachineType::Uint32(), // kAddress
+ MachineType::Int32(), // kExpectedValue
+ MachineType::Uint32(), // kTimeoutLow
+ MachineType::Uint32()) // kTimeoutHigh
+ DECLARE_DESCRIPTOR(WasmI32AtomicWait32Descriptor, CallInterfaceDescriptor)
+};
+
+class WasmI64AtomicWait32Descriptor final : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS_NO_CONTEXT(kAddress, kExpectedValueLow, kExpectedValueHigh,
+ kTimeoutLow, kTimeoutHigh)
+
+ DEFINE_RESULT_AND_PARAMETER_TYPES_WITH_FLAG(
+ CallInterfaceDescriptorData::kNoStackScan, // allow untagged stack params
+ MachineType::Uint32(), // result 1
+ MachineType::Uint32(), // kAddress
+ MachineType::Uint32(), // kExpectedValueLow
+ MachineType::Uint32(), // kExpectedValueHigh
+ MachineType::Uint32(), // kTimeoutLow
+ MachineType::Uint32()) // kTimeoutHigh
+
+#if V8_TARGET_ARCH_IA32
+ static constexpr bool kPassLastArgOnStack = true;
+#else
+ static constexpr bool kPassLastArgOnStack = false;
+#endif
+
+ // Pass the last parameter through the stack.
+ static constexpr int kStackArgumentsCount = kPassLastArgOnStack ? 1 : 0;
+
+ DECLARE_DESCRIPTOR(WasmI64AtomicWait32Descriptor, CallInterfaceDescriptor)
+};
+
+class CloneObjectWithVectorDescriptor final : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kSource, kFlags, kSlot, kVector)
+ DEFINE_RESULT_AND_PARAMETER_TYPES(MachineType::TaggedPointer(), // result 1
+ MachineType::AnyTagged(), // kSource
+ MachineType::TaggedSigned(), // kFlags
+ MachineType::TaggedSigned(), // kSlot
+ MachineType::AnyTagged()) // kVector
+ DECLARE_DESCRIPTOR(CloneObjectWithVectorDescriptor, CallInterfaceDescriptor)
+};
+
+class BinaryOp_WithFeedbackDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kLeft, kRight, kSlot, kMaybeFeedbackVector)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kLeft
+ MachineType::AnyTagged(), // kRight
+ MachineType::UintPtr(), // kSlot
+ MachineType::AnyTagged()) // kMaybeFeedbackVector
+ DECLARE_DESCRIPTOR(BinaryOp_WithFeedbackDescriptor, CallInterfaceDescriptor)
+};
+
+// TODO(jgruber): Pass the slot as UintPtr.
+class CallTrampoline_WithFeedbackDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS_VARARGS(kFunction, kActualArgumentsCount, kSlot,
+ kMaybeFeedbackVector)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kFunction
+ MachineType::Int32(), // kActualArgumentsCount
+ MachineType::Int32(), // kSlot
+ MachineType::AnyTagged()) // kMaybeFeedbackVector
+ DECLARE_DESCRIPTOR(CallTrampoline_WithFeedbackDescriptor,
+ CallInterfaceDescriptor)
+};
+
+class Compare_WithFeedbackDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kLeft, kRight, kSlot, kMaybeFeedbackVector)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kLeft
+ MachineType::AnyTagged(), // kRight
+ MachineType::UintPtr(), // kSlot
+ MachineType::AnyTagged()) // kMaybeFeedbackVector
+ DECLARE_DESCRIPTOR(Compare_WithFeedbackDescriptor, CallInterfaceDescriptor)
+};
+
+// TODO(jgruber): Pass the slot as UintPtr.
+class Construct_WithFeedbackDescriptor : public CallInterfaceDescriptor {
+ public:
+ // kSlot is passed in a register, kMaybeFeedbackVector on the stack.
+ DEFINE_JS_PARAMETERS(kSlot, kMaybeFeedbackVector)
+ DEFINE_JS_PARAMETER_TYPES(MachineType::Int32(), // kSlot
+ MachineType::AnyTagged()) // kMaybeFeedbackVector
+ DECLARE_JS_COMPATIBLE_DESCRIPTOR(Construct_WithFeedbackDescriptor,
+ CallInterfaceDescriptor, 1)
+};
+
+class UnaryOp_WithFeedbackDescriptor : public CallInterfaceDescriptor {
+ public:
+ DEFINE_PARAMETERS(kValue, kSlot, kMaybeFeedbackVector)
+ DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kValue
+ MachineType::UintPtr(), // kSlot
+ MachineType::AnyTagged()) // kMaybeFeedbackVector
+ DECLARE_DESCRIPTOR(UnaryOp_WithFeedbackDescriptor, CallInterfaceDescriptor)
+};
+
+#define DEFINE_TFS_BUILTIN_DESCRIPTOR(Name, ...) \
+ class Name##Descriptor : public CallInterfaceDescriptor { \
+ public: \
+ DEFINE_PARAMETERS(__VA_ARGS__) \
+ DECLARE_DEFAULT_DESCRIPTOR(Name##Descriptor, CallInterfaceDescriptor) \
+ };
+BUILTIN_LIST_TFS(DEFINE_TFS_BUILTIN_DESCRIPTOR)
+#undef DEFINE_TFS_BUILTIN_DESCRIPTOR
+
+// This file contains interface descriptor class definitions for builtins
+// defined in Torque. It is included here because the class definitions need to
+// precede the definition of name##Descriptor::key() below.
+#include "torque-generated/interface-descriptors.inc"
+
+#undef DECLARE_DEFAULT_DESCRIPTOR
+#undef DECLARE_DESCRIPTOR_WITH_BASE
+#undef DECLARE_DESCRIPTOR
+#undef DECLARE_JS_COMPATIBLE_DESCRIPTOR
+#undef DEFINE_FLAGS_AND_RESULT_AND_PARAMETERS
+#undef DEFINE_RESULT_AND_PARAMETERS
+#undef DEFINE_PARAMETERS
+#undef DEFINE_PARAMETERS_VARARGS
+#undef DEFINE_PARAMETERS_NO_CONTEXT
+#undef DEFINE_RESULT_AND_PARAMETER_TYPES
+#undef DEFINE_PARAMETER_TYPES
+#undef DEFINE_JS_PARAMETERS
+#undef DEFINE_JS_PARAMETER_TYPES
+
+// We define the association between CallDescriptors::Key and the specialized
+// descriptor here to reduce boilerplate and mistakes.
+#define DEF_KEY(name, ...) \
+ CallDescriptors::Key name##Descriptor::key() { return CallDescriptors::name; }
+INTERFACE_DESCRIPTOR_LIST(DEF_KEY)
+#undef DEF_KEY
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_INTERFACE_DESCRIPTORS_H_
diff --git a/src/codegen/label.h b/src/codegen/label.h
new file mode 100644
index 0000000..f45f1e6
--- /dev/null
+++ b/src/codegen/label.h
@@ -0,0 +1,112 @@
+// Copyright 2017 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_LABEL_H_
+#define V8_CODEGEN_LABEL_H_
+
+#include "src/base/macros.h"
+
+namespace v8 {
+namespace internal {
+
+// -----------------------------------------------------------------------------
+// Labels represent pc locations; they are typically jump or call targets.
+// After declaration, a label can be freely used to denote known or (yet)
+// unknown pc location. Assembler::bind() is used to bind a label to the
+// current pc. A label can be bound only once.
+
+class Label {
+ public:
+ enum Distance {
+ kNear, // near jump: 8 bit displacement (signed)
+ kFar // far jump: 32 bit displacement (signed)
+ };
+
+ Label() = default;
+
+// On ARM64, the Assembler keeps track of pointers to Labels to resolve
+// branches to distant targets. Copying labels would confuse the Assembler.
+// On other platforms, allow move construction.
+#if !V8_TARGET_ARCH_ARM64
+// In debug builds, the old Label has to be cleared in order to avoid a DCHECK
+// failure in it's destructor.
+#ifdef DEBUG
+ Label(Label&& other) V8_NOEXCEPT { *this = std::move(other); }
+ Label& operator=(Label&& other) V8_NOEXCEPT {
+ pos_ = other.pos_;
+ near_link_pos_ = other.near_link_pos_;
+ other.Unuse();
+ other.UnuseNear();
+ return *this;
+ }
+#else
+ Label(Label&&) V8_NOEXCEPT = default;
+ Label& operator=(Label&&) V8_NOEXCEPT = default;
+#endif
+#endif
+
+#ifdef DEBUG
+ V8_INLINE ~Label() {
+ DCHECK(!is_linked());
+ DCHECK(!is_near_linked());
+ }
+#endif
+
+ V8_INLINE void Unuse() { pos_ = 0; }
+ V8_INLINE void UnuseNear() { near_link_pos_ = 0; }
+
+ V8_INLINE bool is_bound() const { return pos_ < 0; }
+ V8_INLINE bool is_unused() const { return pos_ == 0 && near_link_pos_ == 0; }
+ V8_INLINE bool is_linked() const { return pos_ > 0; }
+ V8_INLINE bool is_near_linked() const { return near_link_pos_ > 0; }
+
+ // Returns the position of bound or linked labels. Cannot be used
+ // for unused labels.
+ int pos() const {
+ if (pos_ < 0) return -pos_ - 1;
+ if (pos_ > 0) return pos_ - 1;
+ UNREACHABLE();
+ }
+
+ int near_link_pos() const { return near_link_pos_ - 1; }
+
+ private:
+ // pos_ encodes both the binding state (via its sign)
+ // and the binding position (via its value) of a label.
+ //
+ // pos_ < 0 bound label, pos() returns the jump target position
+ // pos_ == 0 unused label
+ // pos_ > 0 linked label, pos() returns the last reference position
+ int pos_ = 0;
+
+ // Behaves like |pos_| in the "> 0" case, but for near jumps to this label.
+ int near_link_pos_ = 0;
+
+ void bind_to(int pos) {
+ pos_ = -pos - 1;
+ DCHECK(is_bound());
+ }
+ void link_to(int pos, Distance distance = kFar) {
+ if (distance == kNear) {
+ near_link_pos_ = pos + 1;
+ DCHECK(is_near_linked());
+ } else {
+ pos_ = pos + 1;
+ DCHECK(is_linked());
+ }
+ }
+
+ friend class Assembler;
+ friend class Displacement;
+ friend class RegExpBytecodeGenerator;
+
+ // Disallow copy construction and assignment, but allow move construction and
+ // move assignment on selected platforms (see above).
+ DISALLOW_COPY_AND_ASSIGN(Label);
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_LABEL_H_
diff --git a/src/codegen/machine-type.cc b/src/codegen/machine-type.cc
new file mode 100644
index 0000000..86fc480
--- /dev/null
+++ b/src/codegen/machine-type.cc
@@ -0,0 +1,98 @@
+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/machine-type.h"
+#include "src/utils/ostreams.h"
+
+namespace v8 {
+namespace internal {
+
+bool IsSubtype(MachineRepresentation rep1, MachineRepresentation rep2) {
+ if (rep1 == rep2) return true;
+ switch (rep1) {
+ case MachineRepresentation::kTaggedSigned: // Fall through.
+ case MachineRepresentation::kTaggedPointer:
+ return rep2 == MachineRepresentation::kTagged;
+ case MachineRepresentation::kCompressedPointer:
+ return rep2 == MachineRepresentation::kCompressed;
+ default:
+ return false;
+ }
+}
+
+std::ostream& operator<<(std::ostream& os, MachineRepresentation rep) {
+ return os << MachineReprToString(rep);
+}
+
+const char* MachineReprToString(MachineRepresentation rep) {
+ switch (rep) {
+ case MachineRepresentation::kNone:
+ return "kMachNone";
+ case MachineRepresentation::kBit:
+ return "kRepBit";
+ case MachineRepresentation::kWord8:
+ return "kRepWord8";
+ case MachineRepresentation::kWord16:
+ return "kRepWord16";
+ case MachineRepresentation::kWord32:
+ return "kRepWord32";
+ case MachineRepresentation::kWord64:
+ return "kRepWord64";
+ case MachineRepresentation::kFloat32:
+ return "kRepFloat32";
+ case MachineRepresentation::kFloat64:
+ return "kRepFloat64";
+ case MachineRepresentation::kSimd128:
+ return "kRepSimd128";
+ case MachineRepresentation::kTaggedSigned:
+ return "kRepTaggedSigned";
+ case MachineRepresentation::kTaggedPointer:
+ return "kRepTaggedPointer";
+ case MachineRepresentation::kTagged:
+ return "kRepTagged";
+ case MachineRepresentation::kCompressedPointer:
+ return "kRepCompressedPointer";
+ case MachineRepresentation::kCompressed:
+ return "kRepCompressed";
+ }
+ UNREACHABLE();
+}
+
+std::ostream& operator<<(std::ostream& os, MachineSemantic type) {
+ switch (type) {
+ case MachineSemantic::kNone:
+ return os << "kMachNone";
+ case MachineSemantic::kBool:
+ return os << "kTypeBool";
+ case MachineSemantic::kInt32:
+ return os << "kTypeInt32";
+ case MachineSemantic::kUint32:
+ return os << "kTypeUint32";
+ case MachineSemantic::kInt64:
+ return os << "kTypeInt64";
+ case MachineSemantic::kUint64:
+ return os << "kTypeUint64";
+ case MachineSemantic::kNumber:
+ return os << "kTypeNumber";
+ case MachineSemantic::kAny:
+ return os << "kTypeAny";
+ }
+ UNREACHABLE();
+}
+
+std::ostream& operator<<(std::ostream& os, MachineType type) {
+ if (type == MachineType::None()) {
+ return os;
+ } else if (type.representation() == MachineRepresentation::kNone) {
+ return os << type.semantic();
+ } else if (type.semantic() == MachineSemantic::kNone) {
+ return os << type.representation();
+ } else {
+ return os << type.representation() << "|" << type.semantic();
+ }
+ return os;
+}
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/machine-type.h b/src/codegen/machine-type.h
new file mode 100644
index 0000000..e7e1020
--- /dev/null
+++ b/src/codegen/machine-type.h
@@ -0,0 +1,336 @@
+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_MACHINE_TYPE_H_
+#define V8_CODEGEN_MACHINE_TYPE_H_
+
+#include <iosfwd>
+
+#include "src/base/bits.h"
+#include "src/common/globals.h"
+#include "src/flags/flags.h"
+
+namespace v8 {
+namespace internal {
+
+enum class MachineRepresentation : uint8_t {
+ kNone,
+ kBit,
+ kWord8,
+ kWord16,
+ kWord32,
+ kWord64,
+ kTaggedSigned, // (uncompressed) Smi
+ kTaggedPointer, // (uncompressed) HeapObject
+ kTagged, // (uncompressed) Object (Smi or HeapObject)
+ kCompressedPointer, // (compressed) HeapObject
+ kCompressed, // (compressed) Object (Smi or HeapObject)
+ // FP representations must be last, and in order of increasing size.
+ kFloat32,
+ kFloat64,
+ kSimd128,
+ kFirstFPRepresentation = kFloat32,
+ kLastRepresentation = kSimd128
+};
+
+bool IsSubtype(MachineRepresentation rep1, MachineRepresentation rep2);
+
+static_assert(static_cast<int>(MachineRepresentation::kLastRepresentation) <
+ kIntSize * kBitsPerByte,
+ "Bit masks of MachineRepresentation should fit in an int");
+
+V8_EXPORT_PRIVATE const char* MachineReprToString(MachineRepresentation);
+
+enum class MachineSemantic : uint8_t {
+ kNone,
+ kBool,
+ kInt32,
+ kUint32,
+ kInt64,
+ kUint64,
+ kNumber,
+ kAny
+};
+
+V8_EXPORT_PRIVATE inline constexpr int ElementSizeLog2Of(MachineRepresentation);
+
+V8_EXPORT_PRIVATE inline constexpr int ElementSizeInBytes(
+ MachineRepresentation);
+
+class MachineType {
+ public:
+ constexpr MachineType()
+ : representation_(MachineRepresentation::kNone),
+ semantic_(MachineSemantic::kNone) {}
+ constexpr MachineType(MachineRepresentation representation,
+ MachineSemantic semantic)
+ : representation_(representation), semantic_(semantic) {}
+
+ constexpr bool operator==(MachineType other) const {
+ return representation() == other.representation() &&
+ semantic() == other.semantic();
+ }
+
+ constexpr bool operator!=(MachineType other) const {
+ return !(*this == other);
+ }
+
+ constexpr MachineRepresentation representation() const {
+ return representation_;
+ }
+ constexpr MachineSemantic semantic() const { return semantic_; }
+
+ constexpr bool IsNone() const {
+ return representation() == MachineRepresentation::kNone;
+ }
+
+ constexpr bool IsSigned() const {
+ return semantic() == MachineSemantic::kInt32 ||
+ semantic() == MachineSemantic::kInt64;
+ }
+ constexpr bool IsUnsigned() const {
+ return semantic() == MachineSemantic::kUint32 ||
+ semantic() == MachineSemantic::kUint64;
+ }
+ constexpr bool IsTagged() const {
+ return representation() == MachineRepresentation::kTaggedPointer ||
+ representation() == MachineRepresentation::kTaggedSigned ||
+ representation() == MachineRepresentation::kTagged;
+ }
+ constexpr bool IsTaggedSigned() const {
+ return representation() == MachineRepresentation::kTaggedSigned;
+ }
+ constexpr bool IsTaggedPointer() const {
+ return representation() == MachineRepresentation::kTaggedPointer;
+ }
+ constexpr bool IsCompressed() const {
+ return representation() == MachineRepresentation::kCompressedPointer ||
+ representation() == MachineRepresentation::kCompressed;
+ }
+ constexpr bool IsCompressedPointer() const {
+ return representation() == MachineRepresentation::kCompressedPointer;
+ }
+ constexpr static MachineRepresentation TaggedRepresentation() {
+ return (kTaggedSize == 4) ? MachineRepresentation::kWord32
+ : MachineRepresentation::kWord64;
+ }
+ constexpr static MachineRepresentation PointerRepresentation() {
+ return (kSystemPointerSize == 4) ? MachineRepresentation::kWord32
+ : MachineRepresentation::kWord64;
+ }
+ constexpr static MachineType UintPtr() {
+ return (kSystemPointerSize == 4) ? Uint32() : Uint64();
+ }
+ constexpr static MachineType IntPtr() {
+ return (kSystemPointerSize == 4) ? Int32() : Int64();
+ }
+ constexpr static MachineType Int8() {
+ return MachineType(MachineRepresentation::kWord8, MachineSemantic::kInt32);
+ }
+ constexpr static MachineType Uint8() {
+ return MachineType(MachineRepresentation::kWord8, MachineSemantic::kUint32);
+ }
+ constexpr static MachineType Int16() {
+ return MachineType(MachineRepresentation::kWord16, MachineSemantic::kInt32);
+ }
+ constexpr static MachineType Uint16() {
+ return MachineType(MachineRepresentation::kWord16,
+ MachineSemantic::kUint32);
+ }
+ constexpr static MachineType Int32() {
+ return MachineType(MachineRepresentation::kWord32, MachineSemantic::kInt32);
+ }
+ constexpr static MachineType Uint32() {
+ return MachineType(MachineRepresentation::kWord32,
+ MachineSemantic::kUint32);
+ }
+ constexpr static MachineType Int64() {
+ return MachineType(MachineRepresentation::kWord64, MachineSemantic::kInt64);
+ }
+ constexpr static MachineType Uint64() {
+ return MachineType(MachineRepresentation::kWord64,
+ MachineSemantic::kUint64);
+ }
+ constexpr static MachineType Float32() {
+ return MachineType(MachineRepresentation::kFloat32,
+ MachineSemantic::kNumber);
+ }
+ constexpr static MachineType Float64() {
+ return MachineType(MachineRepresentation::kFloat64,
+ MachineSemantic::kNumber);
+ }
+ constexpr static MachineType Simd128() {
+ return MachineType(MachineRepresentation::kSimd128, MachineSemantic::kNone);
+ }
+ constexpr static MachineType Pointer() {
+ return MachineType(PointerRepresentation(), MachineSemantic::kNone);
+ }
+ constexpr static MachineType TaggedPointer() {
+ return MachineType(MachineRepresentation::kTaggedPointer,
+ MachineSemantic::kAny);
+ }
+ constexpr static MachineType TaggedSigned() {
+ return MachineType(MachineRepresentation::kTaggedSigned,
+ MachineSemantic::kInt32);
+ }
+ constexpr static MachineType AnyTagged() {
+ return MachineType(MachineRepresentation::kTagged, MachineSemantic::kAny);
+ }
+ constexpr static MachineType CompressedPointer() {
+ return MachineType(MachineRepresentation::kCompressedPointer,
+ MachineSemantic::kAny);
+ }
+ constexpr static MachineType AnyCompressed() {
+ return MachineType(MachineRepresentation::kCompressed,
+ MachineSemantic::kAny);
+ }
+ constexpr static MachineType Bool() {
+ return MachineType(MachineRepresentation::kBit, MachineSemantic::kBool);
+ }
+ constexpr static MachineType None() {
+ return MachineType(MachineRepresentation::kNone, MachineSemantic::kNone);
+ }
+
+ static MachineType TypeForRepresentation(const MachineRepresentation& rep,
+ bool isSigned = true) {
+ switch (rep) {
+ case MachineRepresentation::kNone:
+ return MachineType::None();
+ case MachineRepresentation::kBit:
+ return MachineType::Bool();
+ case MachineRepresentation::kWord8:
+ return isSigned ? MachineType::Int8() : MachineType::Uint8();
+ case MachineRepresentation::kWord16:
+ return isSigned ? MachineType::Int16() : MachineType::Uint16();
+ case MachineRepresentation::kWord32:
+ return isSigned ? MachineType::Int32() : MachineType::Uint32();
+ case MachineRepresentation::kWord64:
+ return isSigned ? MachineType::Int64() : MachineType::Uint64();
+ case MachineRepresentation::kFloat32:
+ return MachineType::Float32();
+ case MachineRepresentation::kFloat64:
+ return MachineType::Float64();
+ case MachineRepresentation::kSimd128:
+ return MachineType::Simd128();
+ case MachineRepresentation::kTagged:
+ return MachineType::AnyTagged();
+ case MachineRepresentation::kTaggedSigned:
+ return MachineType::TaggedSigned();
+ case MachineRepresentation::kTaggedPointer:
+ return MachineType::TaggedPointer();
+ case MachineRepresentation::kCompressed:
+ return MachineType::AnyCompressed();
+ case MachineRepresentation::kCompressedPointer:
+ return MachineType::CompressedPointer();
+ default:
+ UNREACHABLE();
+ }
+ }
+
+ constexpr bool LessThanOrEqualPointerSize() const {
+ return ElementSizeLog2Of(this->representation()) <= kSystemPointerSizeLog2;
+ }
+
+ constexpr byte MemSize() const {
+ return 1 << i::ElementSizeLog2Of(this->representation());
+ }
+
+ private:
+ MachineRepresentation representation_;
+ MachineSemantic semantic_;
+};
+
+V8_INLINE size_t hash_value(MachineRepresentation rep) {
+ return static_cast<size_t>(rep);
+}
+
+V8_INLINE size_t hash_value(MachineType type) {
+ return static_cast<size_t>(type.representation()) +
+ static_cast<size_t>(type.semantic()) * 16;
+}
+
+V8_EXPORT_PRIVATE std::ostream& operator<<(std::ostream& os,
+ MachineRepresentation rep);
+std::ostream& operator<<(std::ostream& os, MachineSemantic type);
+V8_EXPORT_PRIVATE std::ostream& operator<<(std::ostream& os, MachineType type);
+
+inline bool IsFloatingPoint(MachineRepresentation rep) {
+ return rep >= MachineRepresentation::kFirstFPRepresentation;
+}
+
+inline bool CanBeTaggedPointer(MachineRepresentation rep) {
+ return rep == MachineRepresentation::kTagged ||
+ rep == MachineRepresentation::kTaggedPointer;
+}
+
+inline bool CanBeTaggedSigned(MachineRepresentation rep) {
+ return rep == MachineRepresentation::kTagged ||
+ rep == MachineRepresentation::kTaggedSigned;
+}
+
+inline bool IsAnyTagged(MachineRepresentation rep) {
+ return CanBeTaggedPointer(rep) || rep == MachineRepresentation::kTaggedSigned;
+}
+
+inline bool CanBeCompressedPointer(MachineRepresentation rep) {
+ return rep == MachineRepresentation::kCompressed ||
+ rep == MachineRepresentation::kCompressedPointer;
+}
+
+inline bool CanBeTaggedOrCompressedPointer(MachineRepresentation rep) {
+ return CanBeTaggedPointer(rep) || CanBeCompressedPointer(rep);
+}
+
+inline bool IsAnyCompressed(MachineRepresentation rep) {
+ return CanBeCompressedPointer(rep);
+}
+
+// Gets the log2 of the element size in bytes of the machine type.
+V8_EXPORT_PRIVATE inline constexpr int ElementSizeLog2Of(
+ MachineRepresentation rep) {
+ switch (rep) {
+ case MachineRepresentation::kBit:
+ case MachineRepresentation::kWord8:
+ return 0;
+ case MachineRepresentation::kWord16:
+ return 1;
+ case MachineRepresentation::kWord32:
+ case MachineRepresentation::kFloat32:
+ return 2;
+ case MachineRepresentation::kWord64:
+ case MachineRepresentation::kFloat64:
+ return 3;
+ case MachineRepresentation::kSimd128:
+ return 4;
+ case MachineRepresentation::kTaggedSigned:
+ case MachineRepresentation::kTaggedPointer:
+ case MachineRepresentation::kTagged:
+ case MachineRepresentation::kCompressedPointer:
+ case MachineRepresentation::kCompressed:
+ return kTaggedSizeLog2;
+ default:
+#if V8_HAS_CXX14_CONSTEXPR
+ UNREACHABLE();
+#else
+ // Return something for older compilers.
+ return -1;
+#endif
+ }
+}
+
+V8_EXPORT_PRIVATE inline constexpr int ElementSizeInBytes(
+ MachineRepresentation rep) {
+ return 1 << ElementSizeLog2Of(rep);
+}
+
+// Converts representation to bit for representation masks.
+V8_EXPORT_PRIVATE inline constexpr int RepresentationBit(
+ MachineRepresentation rep) {
+ return 1 << static_cast<int>(rep);
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_MACHINE_TYPE_H_
diff --git a/src/codegen/macro-assembler-inl.h b/src/codegen/macro-assembler-inl.h
new file mode 100644
index 0000000..3e237ef
--- /dev/null
+++ b/src/codegen/macro-assembler-inl.h
@@ -0,0 +1,15 @@
+// Copyright 2017 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_MACRO_ASSEMBLER_INL_H_
+#define V8_CODEGEN_MACRO_ASSEMBLER_INL_H_
+
+#include "src/codegen/assembler-inl.h"
+#include "src/codegen/macro-assembler.h"
+
+#if V8_TARGET_ARCH_ARM64
+#include "src/codegen/arm64/macro-assembler-arm64-inl.h"
+#endif
+
+#endif // V8_CODEGEN_MACRO_ASSEMBLER_INL_H_
diff --git a/src/codegen/macro-assembler.h b/src/codegen/macro-assembler.h
new file mode 100644
index 0000000..a213fc5
--- /dev/null
+++ b/src/codegen/macro-assembler.h
@@ -0,0 +1,178 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_MACRO_ASSEMBLER_H_
+#define V8_CODEGEN_MACRO_ASSEMBLER_H_
+
+#include "src/codegen/turbo-assembler.h"
+#include "src/execution/frames.h"
+#include "src/heap/heap.h"
+
+// Helper types to make boolean flag easier to read at call-site.
+enum InvokeFlag { CALL_FUNCTION, JUMP_FUNCTION };
+
+// Flags used for the AllocateInNewSpace functions.
+enum AllocationFlags {
+ // No special flags.
+ NO_ALLOCATION_FLAGS = 0,
+ // The content of the result register already contains the allocation top in
+ // new space.
+ RESULT_CONTAINS_TOP = 1 << 0,
+ // Specify that the requested size of the space to allocate is specified in
+ // words instead of bytes.
+ SIZE_IN_WORDS = 1 << 1,
+ // Align the allocation to a multiple of kDoubleSize
+ DOUBLE_ALIGNMENT = 1 << 2,
+ // Directly allocate in old space
+ PRETENURE = 1 << 3,
+};
+
+// This is the only place allowed to include the platform-specific headers.
+#define INCLUDED_FROM_MACRO_ASSEMBLER_H
+#if V8_TARGET_ARCH_IA32
+#include "src/codegen/ia32/macro-assembler-ia32.h"
+#elif V8_TARGET_ARCH_X64
+#include "src/codegen/x64/macro-assembler-x64.h"
+#elif V8_TARGET_ARCH_ARM64
+#include "src/codegen/arm64/constants-arm64.h"
+#include "src/codegen/arm64/macro-assembler-arm64.h"
+#elif V8_TARGET_ARCH_ARM
+#include "src/codegen/arm/constants-arm.h"
+#include "src/codegen/arm/macro-assembler-arm.h"
+#elif V8_TARGET_ARCH_PPC || V8_TARGET_ARCH_PPC64
+#include "src/codegen/ppc/constants-ppc.h"
+#include "src/codegen/ppc/macro-assembler-ppc.h"
+#elif V8_TARGET_ARCH_MIPS
+#include "src/codegen/mips/constants-mips.h"
+#include "src/codegen/mips/macro-assembler-mips.h"
+#elif V8_TARGET_ARCH_MIPS64
+#include "src/codegen/mips64/constants-mips64.h"
+#include "src/codegen/mips64/macro-assembler-mips64.h"
+#elif V8_TARGET_ARCH_S390
+#include "src/codegen/s390/constants-s390.h"
+#include "src/codegen/s390/macro-assembler-s390.h"
+#else
+#error Unsupported target architecture.
+#endif
+#undef INCLUDED_FROM_MACRO_ASSEMBLER_H
+
+namespace v8 {
+namespace internal {
+
+// Maximum number of parameters supported in calls to C/C++. The C++ standard
+// defines a limit of 256 parameters but in simulator builds we provide only
+// limited support.
+#ifdef USE_SIMULATOR
+static constexpr int kMaxCParameters = 10;
+#else
+static constexpr int kMaxCParameters = 256;
+#endif
+
+class FrameScope {
+ public:
+ explicit FrameScope(TurboAssembler* tasm, StackFrame::Type type)
+ : tasm_(tasm), type_(type), old_has_frame_(tasm->has_frame()) {
+ tasm->set_has_frame(true);
+ if (type != StackFrame::MANUAL && type_ != StackFrame::NONE) {
+ tasm->EnterFrame(type);
+ }
+ }
+
+ ~FrameScope() {
+ if (type_ != StackFrame::MANUAL && type_ != StackFrame::NONE) {
+ tasm_->LeaveFrame(type_);
+ }
+ tasm_->set_has_frame(old_has_frame_);
+ }
+
+ private:
+ TurboAssembler* tasm_;
+ StackFrame::Type type_;
+ bool old_has_frame_;
+};
+
+class FrameAndConstantPoolScope {
+ public:
+ FrameAndConstantPoolScope(MacroAssembler* masm, StackFrame::Type type)
+ : masm_(masm),
+ type_(type),
+ old_has_frame_(masm->has_frame()),
+ old_constant_pool_available_(FLAG_enable_embedded_constant_pool &&
+ masm->is_constant_pool_available()) {
+ masm->set_has_frame(true);
+ if (FLAG_enable_embedded_constant_pool) {
+ masm->set_constant_pool_available(true);
+ }
+ if (type_ != StackFrame::MANUAL && type_ != StackFrame::NONE) {
+ masm->EnterFrame(type, !old_constant_pool_available_);
+ }
+ }
+
+ ~FrameAndConstantPoolScope() {
+ masm_->LeaveFrame(type_);
+ masm_->set_has_frame(old_has_frame_);
+ if (FLAG_enable_embedded_constant_pool) {
+ masm_->set_constant_pool_available(old_constant_pool_available_);
+ }
+ }
+
+ private:
+ MacroAssembler* masm_;
+ StackFrame::Type type_;
+ bool old_has_frame_;
+ bool old_constant_pool_available_;
+
+ DISALLOW_IMPLICIT_CONSTRUCTORS(FrameAndConstantPoolScope);
+};
+
+// Class for scoping the the unavailability of constant pool access.
+class ConstantPoolUnavailableScope {
+ public:
+ explicit ConstantPoolUnavailableScope(Assembler* assembler)
+ : assembler_(assembler),
+ old_constant_pool_available_(FLAG_enable_embedded_constant_pool &&
+ assembler->is_constant_pool_available()) {
+ if (FLAG_enable_embedded_constant_pool) {
+ assembler->set_constant_pool_available(false);
+ }
+ }
+ ~ConstantPoolUnavailableScope() {
+ if (FLAG_enable_embedded_constant_pool) {
+ assembler_->set_constant_pool_available(old_constant_pool_available_);
+ }
+ }
+
+ private:
+ Assembler* assembler_;
+ int old_constant_pool_available_;
+
+ DISALLOW_IMPLICIT_CONSTRUCTORS(ConstantPoolUnavailableScope);
+};
+
+class AllowExternalCallThatCantCauseGC : public FrameScope {
+ public:
+ explicit AllowExternalCallThatCantCauseGC(MacroAssembler* masm)
+ : FrameScope(masm, StackFrame::NONE) {}
+};
+
+// Prevent the use of the RootArray during the lifetime of this
+// scope object.
+class NoRootArrayScope {
+ public:
+ explicit NoRootArrayScope(TurboAssembler* masm)
+ : masm_(masm), old_value_(masm->root_array_available()) {
+ masm->set_root_array_available(false);
+ }
+
+ ~NoRootArrayScope() { masm_->set_root_array_available(old_value_); }
+
+ private:
+ TurboAssembler* masm_;
+ bool old_value_;
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_MACRO_ASSEMBLER_H_
diff --git a/src/codegen/mips/assembler-mips-inl.h b/src/codegen/mips/assembler-mips-inl.h
new file mode 100644
index 0000000..53e6f93
--- /dev/null
+++ b/src/codegen/mips/assembler-mips-inl.h
@@ -0,0 +1,347 @@
+
+// Copyright (c) 1994-2006 Sun Microsystems Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// - Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// - Redistribution in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// - Neither the name of Sun Microsystems or the names of contributors may
+// be used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// The original source code covered by the above license above has been
+// modified significantly by Google Inc.
+// Copyright 2012 the V8 project authors. All rights reserved.
+
+#ifndef V8_CODEGEN_MIPS_ASSEMBLER_MIPS_INL_H_
+#define V8_CODEGEN_MIPS_ASSEMBLER_MIPS_INL_H_
+
+#include "src/codegen/mips/assembler-mips.h"
+
+#include "src/codegen/assembler.h"
+#include "src/debug/debug.h"
+#include "src/objects/objects-inl.h"
+
+namespace v8 {
+namespace internal {
+
+bool CpuFeatures::SupportsOptimizer() { return IsSupported(FPU); }
+
+bool CpuFeatures::SupportsWasmSimd128() { return IsSupported(MIPS_SIMD); }
+
+// -----------------------------------------------------------------------------
+// Operand and MemOperand.
+
+bool Operand::is_reg() const { return rm_.is_valid(); }
+
+int32_t Operand::immediate() const {
+ DCHECK(!is_reg());
+ DCHECK(!IsHeapObjectRequest());
+ return value_.immediate;
+}
+
+// -----------------------------------------------------------------------------
+// RelocInfo.
+
+void RelocInfo::apply(intptr_t delta) {
+ if (IsInternalReference(rmode_) || IsInternalReferenceEncoded(rmode_)) {
+ // Absolute code pointer inside code object moves with the code object.
+ Assembler::RelocateInternalReference(rmode_, pc_, delta);
+ } else if (IsRelativeCodeTarget(rmode_)) {
+ Assembler::RelocateRelativeReference(rmode_, pc_, delta);
+ }
+}
+
+Address RelocInfo::target_address() {
+ DCHECK(IsCodeTargetMode(rmode_) || IsRuntimeEntry(rmode_) ||
+ IsWasmCall(rmode_));
+ return Assembler::target_address_at(pc_, constant_pool_);
+}
+
+Address RelocInfo::target_address_address() {
+ DCHECK(HasTargetAddressAddress());
+ // Read the address of the word containing the target_address in an
+ // instruction stream.
+ // The only architecture-independent user of this function is the serializer.
+ // The serializer uses it to find out how many raw bytes of instruction to
+ // output before the next target.
+ // For an instruction like LUI/ORI where the target bits are mixed into the
+ // instruction bits, the size of the target will be zero, indicating that the
+ // serializer should not step forward in memory after a target is resolved
+ // and written. In this case the target_address_address function should
+ // return the end of the instructions to be patched, allowing the
+ // deserializer to deserialize the instructions as raw bytes and put them in
+ // place, ready to be patched with the target. After jump optimization,
+ // that is the address of the instruction that follows J/JAL/JR/JALR
+ // instruction.
+ if (IsMipsArchVariant(kMips32r6)) {
+ // On R6 we don't move to the end of the instructions to be patched, but one
+ // instruction before, because if these instructions are at the end of the
+ // code object it can cause errors in the deserializer.
+ return pc_ + (Assembler::kInstructionsFor32BitConstant - 1) * kInstrSize;
+ } else {
+ return pc_ + Assembler::kInstructionsFor32BitConstant * kInstrSize;
+ }
+}
+
+Address RelocInfo::constant_pool_entry_address() { UNREACHABLE(); }
+
+int RelocInfo::target_address_size() { return Assembler::kSpecialTargetSize; }
+
+void Assembler::deserialization_set_special_target_at(
+ Address instruction_payload, Code code, Address target) {
+ set_target_address_at(instruction_payload,
+ !code.is_null() ? code.constant_pool() : kNullAddress,
+ target);
+}
+
+int Assembler::deserialization_special_target_size(
+ Address instruction_payload) {
+ return kSpecialTargetSize;
+}
+
+void Assembler::set_target_internal_reference_encoded_at(Address pc,
+ Address target) {
+ Instr instr1 = Assembler::instr_at(pc + 0 * kInstrSize);
+ Instr instr2 = Assembler::instr_at(pc + 1 * kInstrSize);
+ DCHECK(Assembler::IsLui(instr1));
+ DCHECK(Assembler::IsOri(instr2) || Assembler::IsJicOrJialc(instr2));
+ instr1 &= ~kImm16Mask;
+ instr2 &= ~kImm16Mask;
+ int32_t imm = static_cast<int32_t>(target);
+ DCHECK_EQ(imm & 3, 0);
+ if (Assembler::IsJicOrJialc(instr2)) {
+ // Encoded internal references are lui/jic load of 32-bit absolute address.
+ uint32_t lui_offset_u, jic_offset_u;
+ Assembler::UnpackTargetAddressUnsigned(imm, &lui_offset_u, &jic_offset_u);
+
+ Assembler::instr_at_put(pc + 0 * kInstrSize, instr1 | lui_offset_u);
+ Assembler::instr_at_put(pc + 1 * kInstrSize, instr2 | jic_offset_u);
+ } else {
+ // Encoded internal references are lui/ori load of 32-bit absolute address.
+ PatchLuiOriImmediate(pc, imm, instr1, 0 * kInstrSize, instr2,
+ 1 * kInstrSize);
+ }
+
+ // Currently used only by deserializer, and all code will be flushed
+ // after complete deserialization, no need to flush on each reference.
+}
+
+void Assembler::deserialization_set_target_internal_reference_at(
+ Address pc, Address target, RelocInfo::Mode mode) {
+ if (RelocInfo::IsInternalReferenceEncoded(mode)) {
+ DCHECK(IsLui(instr_at(pc)));
+ set_target_internal_reference_encoded_at(pc, target);
+ } else {
+ DCHECK(RelocInfo::IsInternalReference(mode));
+ Memory<Address>(pc) = target;
+ }
+}
+
+HeapObject RelocInfo::target_object() {
+ DCHECK(IsCodeTarget(rmode_) || IsFullEmbeddedObject(rmode_));
+ return HeapObject::cast(
+ Object(Assembler::target_address_at(pc_, constant_pool_)));
+}
+
+HeapObject RelocInfo::target_object_no_host(Isolate* isolate) {
+ return target_object();
+}
+
+Handle<HeapObject> RelocInfo::target_object_handle(Assembler* origin) {
+ if (IsCodeTarget(rmode_) || IsFullEmbeddedObject(rmode_)) {
+ return Handle<HeapObject>(reinterpret_cast<Address*>(
+ Assembler::target_address_at(pc_, constant_pool_)));
+ }
+ DCHECK(IsRelativeCodeTarget(rmode_));
+ return origin->relative_code_target_object_handle_at(pc_);
+}
+
+void RelocInfo::set_target_object(Heap* heap, HeapObject target,
+ WriteBarrierMode write_barrier_mode,
+ ICacheFlushMode icache_flush_mode) {
+ DCHECK(IsCodeTarget(rmode_) || IsFullEmbeddedObject(rmode_));
+ Assembler::set_target_address_at(pc_, constant_pool_, target.ptr(),
+ icache_flush_mode);
+ if (write_barrier_mode == UPDATE_WRITE_BARRIER && !host().is_null() &&
+ !FLAG_disable_write_barriers) {
+ WriteBarrierForCode(host(), this, target);
+ }
+}
+
+Address RelocInfo::target_external_reference() {
+ DCHECK(IsExternalReference(rmode_));
+ return Assembler::target_address_at(pc_, constant_pool_);
+}
+
+void RelocInfo::set_target_external_reference(
+ Address target, ICacheFlushMode icache_flush_mode) {
+ DCHECK(IsExternalReference(rmode_));
+ Assembler::set_target_address_at(pc_, constant_pool_, target,
+ icache_flush_mode);
+}
+
+Address RelocInfo::target_internal_reference() {
+ if (IsInternalReference(rmode_)) {
+ return Memory<Address>(pc_);
+ } else {
+ // Encoded internal references are lui/ori or lui/jic load of 32-bit
+ // absolute address.
+ DCHECK(IsInternalReferenceEncoded(rmode_));
+ Instr instr1 = Assembler::instr_at(pc_ + 0 * kInstrSize);
+ Instr instr2 = Assembler::instr_at(pc_ + 1 * kInstrSize);
+ DCHECK(Assembler::IsLui(instr1));
+ DCHECK(Assembler::IsOri(instr2) || Assembler::IsJicOrJialc(instr2));
+ if (Assembler::IsJicOrJialc(instr2)) {
+ return static_cast<Address>(
+ Assembler::CreateTargetAddress(instr1, instr2));
+ }
+ return static_cast<Address>(Assembler::GetLuiOriImmediate(instr1, instr2));
+ }
+}
+
+Address RelocInfo::target_internal_reference_address() {
+ DCHECK(IsInternalReference(rmode_) || IsInternalReferenceEncoded(rmode_));
+ return pc_;
+}
+
+Address RelocInfo::target_runtime_entry(Assembler* origin) {
+ DCHECK(IsRuntimeEntry(rmode_));
+ return target_address();
+}
+
+void RelocInfo::set_target_runtime_entry(Address target,
+ WriteBarrierMode write_barrier_mode,
+ ICacheFlushMode icache_flush_mode) {
+ DCHECK(IsRuntimeEntry(rmode_));
+ if (target_address() != target)
+ set_target_address(target, write_barrier_mode, icache_flush_mode);
+}
+
+Address RelocInfo::target_off_heap_target() {
+ DCHECK(IsOffHeapTarget(rmode_));
+ return Assembler::target_address_at(pc_, constant_pool_);
+}
+
+void RelocInfo::WipeOut() {
+ DCHECK(IsFullEmbeddedObject(rmode_) || IsCodeTarget(rmode_) ||
+ IsRuntimeEntry(rmode_) || IsExternalReference(rmode_) ||
+ IsInternalReference(rmode_) || IsInternalReferenceEncoded(rmode_) ||
+ IsOffHeapTarget(rmode_));
+ if (IsInternalReference(rmode_)) {
+ Memory<Address>(pc_) = kNullAddress;
+ } else if (IsInternalReferenceEncoded(rmode_)) {
+ Assembler::set_target_internal_reference_encoded_at(pc_, kNullAddress);
+ } else {
+ Assembler::set_target_address_at(pc_, constant_pool_, kNullAddress);
+ }
+}
+
+Handle<Code> Assembler::relative_code_target_object_handle_at(
+ Address pc) const {
+ Instr instr1 = instr_at(pc);
+ Instr instr2 = instr_at(pc + kInstrSize);
+ DCHECK(IsLui(instr1));
+ DCHECK(IsOri(instr2) || IsNal(instr2));
+ DCHECK(IsNal(instr2) || IsNal(instr_at(pc - kInstrSize)));
+ if (IsNal(instr2)) {
+ instr2 = instr_at(pc + 2 * kInstrSize);
+ }
+ // Interpret 2 instructions generated by li (lui/ori).
+ int code_target_index = GetLuiOriImmediate(instr1, instr2);
+ return GetCodeTarget(code_target_index);
+}
+
+// -----------------------------------------------------------------------------
+// Assembler.
+
+void Assembler::CheckBuffer() {
+ if (buffer_space() <= kGap) {
+ GrowBuffer();
+ }
+}
+
+void Assembler::CheckForEmitInForbiddenSlot() {
+ if (!is_buffer_growth_blocked()) {
+ CheckBuffer();
+ }
+ if (IsPrevInstrCompactBranch()) {
+ // Nop instruction to precede a CTI in forbidden slot:
+ Instr nop = SPECIAL | SLL;
+ *reinterpret_cast<Instr*>(pc_) = nop;
+ pc_ += kInstrSize;
+
+ ClearCompactBranchState();
+ }
+}
+
+void Assembler::EmitHelper(Instr x, CompactBranchType is_compact_branch) {
+ if (IsPrevInstrCompactBranch()) {
+ if (Instruction::IsForbiddenAfterBranchInstr(x)) {
+ // Nop instruction to precede a CTI in forbidden slot:
+ Instr nop = SPECIAL | SLL;
+ *reinterpret_cast<Instr*>(pc_) = nop;
+ pc_ += kInstrSize;
+ }
+ ClearCompactBranchState();
+ }
+ *reinterpret_cast<Instr*>(pc_) = x;
+ pc_ += kInstrSize;
+ if (is_compact_branch == CompactBranchType::COMPACT_BRANCH) {
+ EmittedCompactBranchInstruction();
+ }
+ CheckTrampolinePoolQuick();
+}
+
+template <>
+inline void Assembler::EmitHelper(uint8_t x);
+
+template <typename T>
+void Assembler::EmitHelper(T x) {
+ *reinterpret_cast<T*>(pc_) = x;
+ pc_ += sizeof(x);
+ CheckTrampolinePoolQuick();
+}
+
+template <>
+void Assembler::EmitHelper(uint8_t x) {
+ *reinterpret_cast<uint8_t*>(pc_) = x;
+ pc_ += sizeof(x);
+ if (reinterpret_cast<intptr_t>(pc_) % kInstrSize == 0) {
+ CheckTrampolinePoolQuick();
+ }
+}
+
+void Assembler::emit(Instr x, CompactBranchType is_compact_branch) {
+ if (!is_buffer_growth_blocked()) {
+ CheckBuffer();
+ }
+ EmitHelper(x, is_compact_branch);
+}
+
+EnsureSpace::EnsureSpace(Assembler* assembler) { assembler->CheckBuffer(); }
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_MIPS_ASSEMBLER_MIPS_INL_H_
diff --git a/src/codegen/mips/assembler-mips.cc b/src/codegen/mips/assembler-mips.cc
new file mode 100644
index 0000000..df46577
--- /dev/null
+++ b/src/codegen/mips/assembler-mips.cc
@@ -0,0 +1,3839 @@
+// Copyright (c) 1994-2006 Sun Microsystems Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// - Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// - Redistribution in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// - Neither the name of Sun Microsystems or the names of contributors may
+// be used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// The original source code covered by the above license above has been
+// modified significantly by Google Inc.
+// Copyright 2012 the V8 project authors. All rights reserved.
+
+#include "src/codegen/mips/assembler-mips.h"
+
+#if V8_TARGET_ARCH_MIPS
+
+#include "src/base/bits.h"
+#include "src/base/cpu.h"
+#include "src/codegen/mips/assembler-mips-inl.h"
+#include "src/codegen/safepoint-table.h"
+#include "src/codegen/string-constants.h"
+#include "src/deoptimizer/deoptimizer.h"
+#include "src/objects/heap-number-inl.h"
+
+namespace v8 {
+namespace internal {
+
+// Get the CPU features enabled by the build. For cross compilation the
+// preprocessor symbols CAN_USE_FPU_INSTRUCTIONS
+// can be defined to enable FPU instructions when building the
+// snapshot.
+static unsigned CpuFeaturesImpliedByCompiler() {
+ unsigned answer = 0;
+#ifdef CAN_USE_FPU_INSTRUCTIONS
+ answer |= 1u << FPU;
+#endif // def CAN_USE_FPU_INSTRUCTIONS
+
+ // If the compiler is allowed to use FPU then we can use FPU too in our code
+ // generation even when generating snapshots. This won't work for cross
+ // compilation.
+#if defined(__mips__) && defined(__mips_hard_float) && __mips_hard_float != 0
+ answer |= 1u << FPU;
+#endif
+
+ return answer;
+}
+
+void CpuFeatures::ProbeImpl(bool cross_compile) {
+ supported_ |= CpuFeaturesImpliedByCompiler();
+
+ // Only use statically determined features for cross compile (snapshot).
+ if (cross_compile) return;
+
+ // If the compiler is allowed to use fpu then we can use fpu too in our
+ // code generation.
+#ifndef __mips__
+ // For the simulator build, use FPU.
+ supported_ |= 1u << FPU;
+#if defined(_MIPS_ARCH_MIPS32R6)
+ // FP64 mode is implied on r6.
+ supported_ |= 1u << FP64FPU;
+#if defined(_MIPS_MSA)
+ supported_ |= 1u << MIPS_SIMD;
+#endif
+#endif
+#if defined(FPU_MODE_FP64)
+ supported_ |= 1u << FP64FPU;
+#endif
+#else
+ // Probe for additional features at runtime.
+ base::CPU cpu;
+ if (cpu.has_fpu()) supported_ |= 1u << FPU;
+#if defined(FPU_MODE_FPXX)
+ if (cpu.is_fp64_mode()) supported_ |= 1u << FP64FPU;
+#elif defined(FPU_MODE_FP64)
+ supported_ |= 1u << FP64FPU;
+#if defined(_MIPS_ARCH_MIPS32R6)
+#if defined(_MIPS_MSA)
+ supported_ |= 1u << MIPS_SIMD;
+#else
+ if (cpu.has_msa()) supported_ |= 1u << MIPS_SIMD;
+#endif
+#endif
+#endif
+#if defined(_MIPS_ARCH_MIPS32RX)
+ if (cpu.architecture() == 6) {
+ supported_ |= 1u << MIPSr6;
+ } else if (cpu.architecture() == 2) {
+ supported_ |= 1u << MIPSr1;
+ supported_ |= 1u << MIPSr2;
+ } else {
+ supported_ |= 1u << MIPSr1;
+ }
+#endif
+#endif
+}
+
+void CpuFeatures::PrintTarget() {}
+void CpuFeatures::PrintFeatures() {}
+
+int ToNumber(Register reg) {
+ DCHECK(reg.is_valid());
+ const int kNumbers[] = {
+ 0, // zero_reg
+ 1, // at
+ 2, // v0
+ 3, // v1
+ 4, // a0
+ 5, // a1
+ 6, // a2
+ 7, // a3
+ 8, // t0
+ 9, // t1
+ 10, // t2
+ 11, // t3
+ 12, // t4
+ 13, // t5
+ 14, // t6
+ 15, // t7
+ 16, // s0
+ 17, // s1
+ 18, // s2
+ 19, // s3
+ 20, // s4
+ 21, // s5
+ 22, // s6
+ 23, // s7
+ 24, // t8
+ 25, // t9
+ 26, // k0
+ 27, // k1
+ 28, // gp
+ 29, // sp
+ 30, // fp
+ 31, // ra
+ };
+ return kNumbers[reg.code()];
+}
+
+Register ToRegister(int num) {
+ DCHECK(num >= 0 && num < kNumRegisters);
+ const Register kRegisters[] = {
+ zero_reg, at, v0, v1, a0, a1, a2, a3, t0, t1, t2, t3, t4, t5, t6, t7,
+ s0, s1, s2, s3, s4, s5, s6, s7, t8, t9, k0, k1, gp, sp, fp, ra};
+ return kRegisters[num];
+}
+
+// -----------------------------------------------------------------------------
+// Implementation of RelocInfo.
+
+const int RelocInfo::kApplyMask =
+ RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE) |
+ RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE_ENCODED) |
+ RelocInfo::ModeMask(RelocInfo::RELATIVE_CODE_TARGET);
+
+bool RelocInfo::IsCodedSpecially() {
+ // The deserializer needs to know whether a pointer is specially coded. Being
+ // specially coded on MIPS means that it is a lui/ori instruction, and that is
+ // always the case inside code objects.
+ return true;
+}
+
+bool RelocInfo::IsInConstantPool() { return false; }
+
+uint32_t RelocInfo::wasm_call_tag() const {
+ DCHECK(rmode_ == WASM_CALL || rmode_ == WASM_STUB_CALL);
+ return static_cast<uint32_t>(
+ Assembler::target_address_at(pc_, constant_pool_));
+}
+
+// -----------------------------------------------------------------------------
+// Implementation of Operand and MemOperand.
+// See assembler-mips-inl.h for inlined constructors.
+
+Operand::Operand(Handle<HeapObject> handle)
+ : rm_(no_reg), rmode_(RelocInfo::FULL_EMBEDDED_OBJECT) {
+ value_.immediate = static_cast<intptr_t>(handle.address());
+}
+
+Operand Operand::EmbeddedNumber(double value) {
+ int32_t smi;
+ if (DoubleToSmiInteger(value, &smi)) return Operand(Smi::FromInt(smi));
+ Operand result(0, RelocInfo::FULL_EMBEDDED_OBJECT);
+ result.is_heap_object_request_ = true;
+ result.value_.heap_object_request = HeapObjectRequest(value);
+ return result;
+}
+
+Operand Operand::EmbeddedStringConstant(const StringConstantBase* str) {
+ Operand result(0, RelocInfo::FULL_EMBEDDED_OBJECT);
+ result.is_heap_object_request_ = true;
+ result.value_.heap_object_request = HeapObjectRequest(str);
+ return result;
+}
+
+MemOperand::MemOperand(Register rm, int32_t offset) : Operand(rm) {
+ offset_ = offset;
+}
+
+MemOperand::MemOperand(Register rm, int32_t unit, int32_t multiplier,
+ OffsetAddend offset_addend)
+ : Operand(rm) {
+ offset_ = unit * multiplier + offset_addend;
+}
+
+void Assembler::AllocateAndInstallRequestedHeapObjects(Isolate* isolate) {
+ DCHECK_IMPLIES(isolate == nullptr, heap_object_requests_.empty());
+ for (auto& request : heap_object_requests_) {
+ Handle<HeapObject> object;
+ switch (request.kind()) {
+ case HeapObjectRequest::kHeapNumber:
+ object = isolate->factory()->NewHeapNumber<AllocationType::kOld>(
+ request.heap_number());
+ break;
+ case HeapObjectRequest::kStringConstant:
+ const StringConstantBase* str = request.string();
+ CHECK_NOT_NULL(str);
+ object = str->AllocateStringConstant(isolate);
+ break;
+ }
+ Address pc = reinterpret_cast<Address>(buffer_start_) + request.offset();
+ set_target_value_at(pc, reinterpret_cast<uint32_t>(object.location()));
+ }
+}
+
+// -----------------------------------------------------------------------------
+// Specific instructions, constants, and masks.
+
+static const int kNegOffset = 0x00008000;
+// addiu(sp, sp, 4) aka Pop() operation or part of Pop(r)
+// operations as post-increment of sp.
+const Instr kPopInstruction = ADDIU | (sp.code() << kRsShift) |
+ (sp.code() << kRtShift) |
+ (kPointerSize & kImm16Mask); // NOLINT
+// addiu(sp, sp, -4) part of Push(r) operation as pre-decrement of sp.
+const Instr kPushInstruction = ADDIU | (sp.code() << kRsShift) |
+ (sp.code() << kRtShift) |
+ (-kPointerSize & kImm16Mask); // NOLINT
+// sw(r, MemOperand(sp, 0))
+const Instr kPushRegPattern =
+ SW | (sp.code() << kRsShift) | (0 & kImm16Mask); // NOLINT
+// lw(r, MemOperand(sp, 0))
+const Instr kPopRegPattern =
+ LW | (sp.code() << kRsShift) | (0 & kImm16Mask); // NOLINT
+
+const Instr kLwRegFpOffsetPattern =
+ LW | (fp.code() << kRsShift) | (0 & kImm16Mask); // NOLINT
+
+const Instr kSwRegFpOffsetPattern =
+ SW | (fp.code() << kRsShift) | (0 & kImm16Mask); // NOLINT
+
+const Instr kLwRegFpNegOffsetPattern =
+ LW | (fp.code() << kRsShift) | (kNegOffset & kImm16Mask); // NOLINT
+
+const Instr kSwRegFpNegOffsetPattern =
+ SW | (fp.code() << kRsShift) | (kNegOffset & kImm16Mask); // NOLINT
+// A mask for the Rt register for push, pop, lw, sw instructions.
+const Instr kRtMask = kRtFieldMask;
+const Instr kLwSwInstrTypeMask = 0xFFE00000;
+const Instr kLwSwInstrArgumentMask = ~kLwSwInstrTypeMask;
+const Instr kLwSwOffsetMask = kImm16Mask;
+
+Assembler::Assembler(const AssemblerOptions& options,
+ std::unique_ptr<AssemblerBuffer> buffer)
+ : AssemblerBase(options, std::move(buffer)),
+ scratch_register_list_(at.bit()) {
+ reloc_info_writer.Reposition(buffer_start_ + buffer_->size(), pc_);
+
+ last_trampoline_pool_end_ = 0;
+ no_trampoline_pool_before_ = 0;
+ trampoline_pool_blocked_nesting_ = 0;
+ // We leave space (16 * kTrampolineSlotsSize)
+ // for BlockTrampolinePoolScope buffer.
+ next_buffer_check_ = FLAG_force_long_branches
+ ? kMaxInt
+ : kMaxBranchOffset - kTrampolineSlotsSize * 16;
+ internal_trampoline_exception_ = false;
+ last_bound_pos_ = 0;
+
+ trampoline_emitted_ = FLAG_force_long_branches;
+ unbound_labels_count_ = 0;
+ block_buffer_growth_ = false;
+}
+
+void Assembler::GetCode(Isolate* isolate, CodeDesc* desc,
+ SafepointTableBuilder* safepoint_table_builder,
+ int handler_table_offset) {
+ // As a crutch to avoid having to add manual Align calls wherever we use a
+ // raw workflow to create Code objects (mostly in tests), add another Align
+ // call here. It does no harm - the end of the Code object is aligned to the
+ // (larger) kCodeAlignment anyways.
+ // TODO(jgruber): Consider moving responsibility for proper alignment to
+ // metadata table builders (safepoint, handler, constant pool, code
+ // comments).
+ DataAlign(Code::kMetadataAlignment);
+
+ EmitForbiddenSlotInstruction();
+
+ int code_comments_size = WriteCodeComments();
+
+ DCHECK(pc_ <= reloc_info_writer.pos()); // No overlap.
+
+ AllocateAndInstallRequestedHeapObjects(isolate);
+
+ // Set up code descriptor.
+ // TODO(jgruber): Reconsider how these offsets and sizes are maintained up to
+ // this point to make CodeDesc initialization less fiddly.
+
+ static constexpr int kConstantPoolSize = 0;
+ const int instruction_size = pc_offset();
+ const int code_comments_offset = instruction_size - code_comments_size;
+ const int constant_pool_offset = code_comments_offset - kConstantPoolSize;
+ const int handler_table_offset2 = (handler_table_offset == kNoHandlerTable)
+ ? constant_pool_offset
+ : handler_table_offset;
+ const int safepoint_table_offset =
+ (safepoint_table_builder == kNoSafepointTable)
+ ? handler_table_offset2
+ : safepoint_table_builder->GetCodeOffset();
+ const int reloc_info_offset =
+ static_cast<int>(reloc_info_writer.pos() - buffer_->start());
+ CodeDesc::Initialize(desc, this, safepoint_table_offset,
+ handler_table_offset2, constant_pool_offset,
+ code_comments_offset, reloc_info_offset);
+}
+
+void Assembler::Align(int m) {
+ DCHECK(m >= 4 && base::bits::IsPowerOfTwo(m));
+ EmitForbiddenSlotInstruction();
+ while ((pc_offset() & (m - 1)) != 0) {
+ nop();
+ }
+}
+
+void Assembler::CodeTargetAlign() {
+ // No advantage to aligning branch/call targets to more than
+ // single instruction, that I am aware of.
+ Align(4);
+}
+
+Register Assembler::GetRtReg(Instr instr) {
+ return Register::from_code((instr & kRtFieldMask) >> kRtShift);
+}
+
+Register Assembler::GetRsReg(Instr instr) {
+ return Register::from_code((instr & kRsFieldMask) >> kRsShift);
+}
+
+Register Assembler::GetRdReg(Instr instr) {
+ return Register::from_code((instr & kRdFieldMask) >> kRdShift);
+}
+
+uint32_t Assembler::GetRt(Instr instr) {
+ return (instr & kRtFieldMask) >> kRtShift;
+}
+
+uint32_t Assembler::GetRtField(Instr instr) { return instr & kRtFieldMask; }
+
+uint32_t Assembler::GetRs(Instr instr) {
+ return (instr & kRsFieldMask) >> kRsShift;
+}
+
+uint32_t Assembler::GetRsField(Instr instr) { return instr & kRsFieldMask; }
+
+uint32_t Assembler::GetRd(Instr instr) {
+ return (instr & kRdFieldMask) >> kRdShift;
+}
+
+uint32_t Assembler::GetRdField(Instr instr) { return instr & kRdFieldMask; }
+
+uint32_t Assembler::GetSa(Instr instr) {
+ return (instr & kSaFieldMask) >> kSaShift;
+}
+
+uint32_t Assembler::GetSaField(Instr instr) { return instr & kSaFieldMask; }
+
+uint32_t Assembler::GetOpcodeField(Instr instr) { return instr & kOpcodeMask; }
+
+uint32_t Assembler::GetFunction(Instr instr) {
+ return (instr & kFunctionFieldMask) >> kFunctionShift;
+}
+
+uint32_t Assembler::GetFunctionField(Instr instr) {
+ return instr & kFunctionFieldMask;
+}
+
+uint32_t Assembler::GetImmediate16(Instr instr) { return instr & kImm16Mask; }
+
+uint32_t Assembler::GetLabelConst(Instr instr) { return instr & ~kImm16Mask; }
+
+bool Assembler::IsPop(Instr instr) {
+ return (instr & ~kRtMask) == kPopRegPattern;
+}
+
+bool Assembler::IsPush(Instr instr) {
+ return (instr & ~kRtMask) == kPushRegPattern;
+}
+
+bool Assembler::IsSwRegFpOffset(Instr instr) {
+ return ((instr & kLwSwInstrTypeMask) == kSwRegFpOffsetPattern);
+}
+
+bool Assembler::IsLwRegFpOffset(Instr instr) {
+ return ((instr & kLwSwInstrTypeMask) == kLwRegFpOffsetPattern);
+}
+
+bool Assembler::IsSwRegFpNegOffset(Instr instr) {
+ return ((instr & (kLwSwInstrTypeMask | kNegOffset)) ==
+ kSwRegFpNegOffsetPattern);
+}
+
+bool Assembler::IsLwRegFpNegOffset(Instr instr) {
+ return ((instr & (kLwSwInstrTypeMask | kNegOffset)) ==
+ kLwRegFpNegOffsetPattern);
+}
+
+// Labels refer to positions in the (to be) generated code.
+// There are bound, linked, and unused labels.
+//
+// Bound labels refer to known positions in the already
+// generated code. pos() is the position the label refers to.
+//
+// Linked labels refer to unknown positions in the code
+// to be generated; pos() is the position of the last
+// instruction using the label.
+
+// The link chain is terminated by a value in the instruction of -1,
+// which is an otherwise illegal value (branch -1 is inf loop).
+// The instruction 16-bit offset field addresses 32-bit words, but in
+// code is conv to an 18-bit value addressing bytes, hence the -4 value.
+
+const int kEndOfChain = -4;
+// Determines the end of the Jump chain (a subset of the label link chain).
+const int kEndOfJumpChain = 0;
+
+bool Assembler::IsMsaBranch(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ uint32_t rs_field = GetRsField(instr);
+ if (opcode == COP1) {
+ switch (rs_field) {
+ case BZ_V:
+ case BZ_B:
+ case BZ_H:
+ case BZ_W:
+ case BZ_D:
+ case BNZ_V:
+ case BNZ_B:
+ case BNZ_H:
+ case BNZ_W:
+ case BNZ_D:
+ return true;
+ default:
+ return false;
+ }
+ } else {
+ return false;
+ }
+}
+
+bool Assembler::IsBranch(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ uint32_t rt_field = GetRtField(instr);
+ uint32_t rs_field = GetRsField(instr);
+ // Checks if the instruction is a branch.
+ bool isBranch =
+ opcode == BEQ || opcode == BNE || opcode == BLEZ || opcode == BGTZ ||
+ opcode == BEQL || opcode == BNEL || opcode == BLEZL || opcode == BGTZL ||
+ (opcode == REGIMM && (rt_field == BLTZ || rt_field == BGEZ ||
+ rt_field == BLTZAL || rt_field == BGEZAL)) ||
+ (opcode == COP1 && rs_field == BC1) || // Coprocessor branch.
+ (opcode == COP1 && rs_field == BC1EQZ) ||
+ (opcode == COP1 && rs_field == BC1NEZ) || IsMsaBranch(instr);
+ if (!isBranch && IsMipsArchVariant(kMips32r6)) {
+ // All the 3 variants of POP10 (BOVC, BEQC, BEQZALC) and
+ // POP30 (BNVC, BNEC, BNEZALC) are branch ops.
+ isBranch |= opcode == POP10 || opcode == POP30 || opcode == BC ||
+ opcode == BALC ||
+ (opcode == POP66 && rs_field != 0) || // BEQZC
+ (opcode == POP76 && rs_field != 0); // BNEZC
+ }
+ return isBranch;
+}
+
+bool Assembler::IsBc(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ // Checks if the instruction is a BC or BALC.
+ return opcode == BC || opcode == BALC;
+}
+
+bool Assembler::IsNal(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ uint32_t rt_field = GetRtField(instr);
+ uint32_t rs_field = GetRsField(instr);
+ return opcode == REGIMM && rt_field == BLTZAL && rs_field == 0;
+}
+
+bool Assembler::IsBzc(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ // Checks if the instruction is BEQZC or BNEZC.
+ return (opcode == POP66 && GetRsField(instr) != 0) ||
+ (opcode == POP76 && GetRsField(instr) != 0);
+}
+
+bool Assembler::IsEmittedConstant(Instr instr) {
+ uint32_t label_constant = GetLabelConst(instr);
+ return label_constant == 0; // Emitted label const in reg-exp engine.
+}
+
+bool Assembler::IsBeq(Instr instr) { return GetOpcodeField(instr) == BEQ; }
+
+bool Assembler::IsBne(Instr instr) { return GetOpcodeField(instr) == BNE; }
+
+bool Assembler::IsBeqzc(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ return opcode == POP66 && GetRsField(instr) != 0;
+}
+
+bool Assembler::IsBnezc(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ return opcode == POP76 && GetRsField(instr) != 0;
+}
+
+bool Assembler::IsBeqc(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ uint32_t rs = GetRsField(instr);
+ uint32_t rt = GetRtField(instr);
+ return opcode == POP10 && rs != 0 && rs < rt; // && rt != 0
+}
+
+bool Assembler::IsBnec(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ uint32_t rs = GetRsField(instr);
+ uint32_t rt = GetRtField(instr);
+ return opcode == POP30 && rs != 0 && rs < rt; // && rt != 0
+}
+
+bool Assembler::IsJicOrJialc(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ uint32_t rs = GetRsField(instr);
+ return (opcode == POP66 || opcode == POP76) && rs == 0;
+}
+
+bool Assembler::IsJump(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ uint32_t rt_field = GetRtField(instr);
+ uint32_t rd_field = GetRdField(instr);
+ uint32_t function_field = GetFunctionField(instr);
+ // Checks if the instruction is a jump.
+ return opcode == J || opcode == JAL ||
+ (opcode == SPECIAL && rt_field == 0 &&
+ ((function_field == JALR) ||
+ (rd_field == 0 && (function_field == JR))));
+}
+
+bool Assembler::IsJ(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ // Checks if the instruction is a jump.
+ return opcode == J;
+}
+
+bool Assembler::IsJal(Instr instr) { return GetOpcodeField(instr) == JAL; }
+
+bool Assembler::IsJr(Instr instr) {
+ if (!IsMipsArchVariant(kMips32r6)) {
+ return GetOpcodeField(instr) == SPECIAL && GetFunctionField(instr) == JR;
+ } else {
+ return GetOpcodeField(instr) == SPECIAL && GetRdField(instr) == 0 &&
+ GetFunctionField(instr) == JALR;
+ }
+}
+
+bool Assembler::IsJalr(Instr instr) {
+ return GetOpcodeField(instr) == SPECIAL && GetRdField(instr) != 0 &&
+ GetFunctionField(instr) == JALR;
+}
+
+bool Assembler::IsLui(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ // Checks if the instruction is a load upper immediate.
+ return opcode == LUI;
+}
+
+bool Assembler::IsOri(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ // Checks if the instruction is a load upper immediate.
+ return opcode == ORI;
+}
+
+bool Assembler::IsAddu(Instr instr, Register rd, Register rs, Register rt) {
+ uint32_t opcode = GetOpcodeField(instr);
+ uint32_t rd_field = GetRd(instr);
+ uint32_t rs_field = GetRs(instr);
+ uint32_t rt_field = GetRt(instr);
+ uint32_t sa_field = GetSaField(instr);
+ uint32_t rd_reg = static_cast<uint32_t>(rd.code());
+ uint32_t rs_reg = static_cast<uint32_t>(rs.code());
+ uint32_t rt_reg = static_cast<uint32_t>(rt.code());
+ uint32_t function_field = GetFunction(instr);
+ return opcode == SPECIAL && sa_field == 0 && function_field == ADDU &&
+ rd_reg == rd_field && rs_reg == rs_field && rt_reg == rt_field;
+}
+
+bool Assembler::IsMov(Instr instr, Register rd, Register rs) {
+ uint32_t opcode = GetOpcodeField(instr);
+ uint32_t rd_field = GetRd(instr);
+ uint32_t rs_field = GetRs(instr);
+ uint32_t rt_field = GetRt(instr);
+ uint32_t rd_reg = static_cast<uint32_t>(rd.code());
+ uint32_t rs_reg = static_cast<uint32_t>(rs.code());
+ uint32_t function_field = GetFunctionField(instr);
+ // Checks if the instruction is a OR with zero_reg argument (aka MOV).
+ bool res = opcode == SPECIAL && function_field == OR && rd_field == rd_reg &&
+ rs_field == rs_reg && rt_field == 0;
+ return res;
+}
+
+bool Assembler::IsNop(Instr instr, unsigned int type) {
+ // See Assembler::nop(type).
+ DCHECK_LT(type, 32);
+ uint32_t opcode = GetOpcodeField(instr);
+ uint32_t function = GetFunctionField(instr);
+ uint32_t rt = GetRt(instr);
+ uint32_t rd = GetRd(instr);
+ uint32_t sa = GetSa(instr);
+
+ // Traditional mips nop == sll(zero_reg, zero_reg, 0)
+ // When marking non-zero type, use sll(zero_reg, at, type)
+ // to avoid use of mips ssnop and ehb special encodings
+ // of the sll instruction.
+
+ Register nop_rt_reg = (type == 0) ? zero_reg : at;
+ bool ret = (opcode == SPECIAL && function == SLL &&
+ rd == static_cast<uint32_t>(ToNumber(zero_reg)) &&
+ rt == static_cast<uint32_t>(ToNumber(nop_rt_reg)) && sa == type);
+
+ return ret;
+}
+
+int32_t Assembler::GetBranchOffset(Instr instr) {
+ DCHECK(IsBranch(instr));
+ return (static_cast<int16_t>(instr & kImm16Mask)) << 2;
+}
+
+bool Assembler::IsLw(Instr instr) {
+ return (static_cast<uint32_t>(instr & kOpcodeMask) == LW);
+}
+
+int16_t Assembler::GetLwOffset(Instr instr) {
+ DCHECK(IsLw(instr));
+ return ((instr & kImm16Mask));
+}
+
+Instr Assembler::SetLwOffset(Instr instr, int16_t offset) {
+ DCHECK(IsLw(instr));
+
+ // We actually create a new lw instruction based on the original one.
+ Instr temp_instr = LW | (instr & kRsFieldMask) | (instr & kRtFieldMask) |
+ (offset & kImm16Mask);
+
+ return temp_instr;
+}
+
+bool Assembler::IsSw(Instr instr) {
+ return (static_cast<uint32_t>(instr & kOpcodeMask) == SW);
+}
+
+Instr Assembler::SetSwOffset(Instr instr, int16_t offset) {
+ DCHECK(IsSw(instr));
+ return ((instr & ~kImm16Mask) | (offset & kImm16Mask));
+}
+
+bool Assembler::IsAddImmediate(Instr instr) {
+ return ((instr & kOpcodeMask) == ADDIU);
+}
+
+Instr Assembler::SetAddImmediateOffset(Instr instr, int16_t offset) {
+ DCHECK(IsAddImmediate(instr));
+ return ((instr & ~kImm16Mask) | (offset & kImm16Mask));
+}
+
+bool Assembler::IsAndImmediate(Instr instr) {
+ return GetOpcodeField(instr) == ANDI;
+}
+
+static Assembler::OffsetSize OffsetSizeInBits(Instr instr) {
+ if (IsMipsArchVariant(kMips32r6)) {
+ if (Assembler::IsBc(instr)) {
+ return Assembler::OffsetSize::kOffset26;
+ } else if (Assembler::IsBzc(instr)) {
+ return Assembler::OffsetSize::kOffset21;
+ }
+ }
+ return Assembler::OffsetSize::kOffset16;
+}
+
+static inline int32_t AddBranchOffset(int pos, Instr instr) {
+ int bits = OffsetSizeInBits(instr);
+ const int32_t mask = (1 << bits) - 1;
+ bits = 32 - bits;
+
+ // Do NOT change this to <<2. We rely on arithmetic shifts here, assuming
+ // the compiler uses arithmetic shifts for signed integers.
+ int32_t imm = ((instr & mask) << bits) >> (bits - 2);
+
+ if (imm == kEndOfChain) {
+ // EndOfChain sentinel is returned directly, not relative to pc or pos.
+ return kEndOfChain;
+ } else {
+ return pos + Assembler::kBranchPCOffset + imm;
+ }
+}
+
+uint32_t Assembler::CreateTargetAddress(Instr instr_lui, Instr instr_jic) {
+ DCHECK(IsLui(instr_lui) && IsJicOrJialc(instr_jic));
+ int16_t jic_offset = GetImmediate16(instr_jic);
+ int16_t lui_offset = GetImmediate16(instr_lui);
+
+ if (jic_offset < 0) {
+ lui_offset += kImm16Mask;
+ }
+ uint32_t lui_offset_u = (static_cast<uint32_t>(lui_offset)) << kLuiShift;
+ uint32_t jic_offset_u = static_cast<uint32_t>(jic_offset) & kImm16Mask;
+
+ return lui_offset_u | jic_offset_u;
+}
+
+// Use just lui and jic instructions. Insert lower part of the target address in
+// jic offset part. Since jic sign-extends offset and then add it with register,
+// before that addition, difference between upper part of the target address and
+// upper part of the sign-extended offset (0xFFFF or 0x0000), will be inserted
+// in jic register with lui instruction.
+void Assembler::UnpackTargetAddress(uint32_t address, int16_t* lui_offset,
+ int16_t* jic_offset) {
+ *lui_offset = (address & kHiMask) >> kLuiShift;
+ *jic_offset = address & kLoMask;
+
+ if (*jic_offset < 0) {
+ *lui_offset -= kImm16Mask;
+ }
+}
+
+void Assembler::UnpackTargetAddressUnsigned(uint32_t address,
+ uint32_t* lui_offset,
+ uint32_t* jic_offset) {
+ int16_t lui_offset16 = (address & kHiMask) >> kLuiShift;
+ int16_t jic_offset16 = address & kLoMask;
+
+ if (jic_offset16 < 0) {
+ lui_offset16 -= kImm16Mask;
+ }
+ *lui_offset = static_cast<uint32_t>(lui_offset16) & kImm16Mask;
+ *jic_offset = static_cast<uint32_t>(jic_offset16) & kImm16Mask;
+}
+
+void Assembler::PatchLuiOriImmediate(int pc, int32_t imm, Instr instr_lui,
+ Address offset_lui, Instr instr_ori,
+ Address offset_ori) {
+ DCHECK(IsLui(instr_lui));
+ DCHECK(IsOri(instr_ori));
+ instr_at_put(static_cast<int>(pc + offset_lui),
+ instr_lui | ((imm >> kLuiShift) & kImm16Mask));
+ instr_at_put(static_cast<int>(pc + offset_ori),
+ instr_ori | (imm & kImm16Mask));
+}
+
+void Assembler::PatchLuiOriImmediate(Address pc, int32_t imm, Instr instr_lui,
+ Address offset_lui, Instr instr_ori,
+ Address offset_ori) {
+ DCHECK(IsLui(instr_lui));
+ DCHECK(IsOri(instr_ori));
+ instr_at_put(pc + offset_lui, instr_lui | ((imm >> kLuiShift) & kImm16Mask));
+ instr_at_put(pc + offset_ori, instr_ori | (imm & kImm16Mask));
+}
+
+int32_t Assembler::GetLuiOriImmediate(Instr instr_lui, Instr instr_ori) {
+ DCHECK(IsLui(instr_lui));
+ DCHECK(IsOri(instr_ori));
+ int32_t imm;
+ imm = (instr_lui & static_cast<int32_t>(kImm16Mask)) << kLuiShift;
+ imm |= (instr_ori & static_cast<int32_t>(kImm16Mask));
+ return imm;
+}
+
+int Assembler::target_at(int pos, bool is_internal) {
+ Instr instr = instr_at(pos);
+ if (is_internal) {
+ if (instr == 0) {
+ return kEndOfChain;
+ } else {
+ int32_t instr_address = reinterpret_cast<int32_t>(buffer_start_ + pos);
+ int delta = static_cast<int>(instr_address - instr);
+ DCHECK(pos > delta);
+ return pos - delta;
+ }
+ }
+ if ((instr & ~kImm16Mask) == 0) {
+ // Emitted label constant, not part of a branch.
+ if (instr == 0) {
+ return kEndOfChain;
+ } else {
+ int32_t imm18 = ((instr & static_cast<int32_t>(kImm16Mask)) << 16) >> 14;
+ return (imm18 + pos);
+ }
+ }
+ // Check we have a branch or jump instruction.
+ DCHECK(IsBranch(instr) || IsLui(instr) || IsMov(instr, t8, ra));
+ if (IsBranch(instr)) {
+ return AddBranchOffset(pos, instr);
+ } else if (IsMov(instr, t8, ra)) {
+ int32_t imm32;
+ Instr instr_lui = instr_at(pos + 2 * kInstrSize);
+ Instr instr_ori = instr_at(pos + 3 * kInstrSize);
+ imm32 = GetLuiOriImmediate(instr_lui, instr_ori);
+ if (imm32 == kEndOfJumpChain) {
+ // EndOfChain sentinel is returned directly, not relative to pc or pos.
+ return kEndOfChain;
+ }
+ return pos + Assembler::kLongBranchPCOffset + imm32;
+ } else {
+ DCHECK(IsLui(instr));
+ if (IsNal(instr_at(pos + kInstrSize))) {
+ int32_t imm32;
+ Instr instr_lui = instr_at(pos + 0 * kInstrSize);
+ Instr instr_ori = instr_at(pos + 2 * kInstrSize);
+ imm32 = GetLuiOriImmediate(instr_lui, instr_ori);
+ if (imm32 == kEndOfJumpChain) {
+ // EndOfChain sentinel is returned directly, not relative to pc or pos.
+ return kEndOfChain;
+ }
+ return pos + Assembler::kLongBranchPCOffset + imm32;
+ } else {
+ Instr instr1 = instr_at(pos + 0 * kInstrSize);
+ Instr instr2 = instr_at(pos + 1 * kInstrSize);
+ DCHECK(IsOri(instr2) || IsJicOrJialc(instr2));
+ int32_t imm;
+ if (IsJicOrJialc(instr2)) {
+ imm = CreateTargetAddress(instr1, instr2);
+ } else {
+ imm = GetLuiOriImmediate(instr1, instr2);
+ }
+
+ if (imm == kEndOfJumpChain) {
+ // EndOfChain sentinel is returned directly, not relative to pc or pos.
+ return kEndOfChain;
+ } else {
+ uint32_t instr_address = reinterpret_cast<int32_t>(buffer_start_ + pos);
+ int32_t delta = instr_address - imm;
+ DCHECK(pos > delta);
+ return pos - delta;
+ }
+ }
+ }
+ return 0;
+}
+
+static inline Instr SetBranchOffset(int32_t pos, int32_t target_pos,
+ Instr instr) {
+ int32_t bits = OffsetSizeInBits(instr);
+ int32_t imm = target_pos - (pos + Assembler::kBranchPCOffset);
+ DCHECK_EQ(imm & 3, 0);
+ imm >>= 2;
+
+ const int32_t mask = (1 << bits) - 1;
+ instr &= ~mask;
+ DCHECK(is_intn(imm, bits));
+
+ return instr | (imm & mask);
+}
+
+void Assembler::target_at_put(int32_t pos, int32_t target_pos,
+ bool is_internal) {
+ Instr instr = instr_at(pos);
+
+ if (is_internal) {
+ uint32_t imm = reinterpret_cast<uint32_t>(buffer_start_) + target_pos;
+ instr_at_put(pos, imm);
+ return;
+ }
+ if ((instr & ~kImm16Mask) == 0) {
+ DCHECK(target_pos == kEndOfChain || target_pos >= 0);
+ // Emitted label constant, not part of a branch.
+ // Make label relative to Code pointer of generated Code object.
+ instr_at_put(pos, target_pos + (Code::kHeaderSize - kHeapObjectTag));
+ return;
+ }
+
+ DCHECK(IsBranch(instr) || IsLui(instr) || IsMov(instr, t8, ra));
+ if (IsBranch(instr)) {
+ instr = SetBranchOffset(pos, target_pos, instr);
+ instr_at_put(pos, instr);
+ } else if (IsMov(instr, t8, ra)) {
+ Instr instr_lui = instr_at(pos + 2 * kInstrSize);
+ Instr instr_ori = instr_at(pos + 3 * kInstrSize);
+ DCHECK(IsLui(instr_lui));
+ DCHECK(IsOri(instr_ori));
+
+ int32_t imm_short = target_pos - (pos + Assembler::kBranchPCOffset);
+
+ if (is_int16(imm_short)) {
+ // Optimize by converting to regular branch with 16-bit
+ // offset
+ Instr instr_b = BEQ;
+ instr_b = SetBranchOffset(pos, target_pos, instr_b);
+
+ Instr instr_j = instr_at(pos + 5 * kInstrSize);
+ Instr instr_branch_delay;
+
+ if (IsJump(instr_j)) {
+ // Case when branch delay slot is protected.
+ instr_branch_delay = nopInstr;
+ } else {
+ // Case when branch delay slot is used.
+ instr_branch_delay = instr_at(pos + 7 * kInstrSize);
+ }
+ instr_at_put(pos + 0 * kInstrSize, instr_b);
+ instr_at_put(pos + 1 * kInstrSize, instr_branch_delay);
+ } else {
+ int32_t imm = target_pos - (pos + Assembler::kLongBranchPCOffset);
+ DCHECK_EQ(imm & 3, 0);
+
+ instr_lui &= ~kImm16Mask;
+ instr_ori &= ~kImm16Mask;
+
+ PatchLuiOriImmediate(pos, imm, instr_lui, 2 * kInstrSize, instr_ori,
+ 3 * kInstrSize);
+ }
+ } else {
+ DCHECK(IsLui(instr));
+ if (IsNal(instr_at(pos + kInstrSize))) {
+ Instr instr_lui = instr_at(pos + 0 * kInstrSize);
+ Instr instr_ori = instr_at(pos + 2 * kInstrSize);
+ DCHECK(IsLui(instr_lui));
+ DCHECK(IsOri(instr_ori));
+ int32_t imm = target_pos - (pos + Assembler::kLongBranchPCOffset);
+ DCHECK_EQ(imm & 3, 0);
+ if (is_int16(imm + Assembler::kLongBranchPCOffset -
+ Assembler::kBranchPCOffset)) {
+ // Optimize by converting to regular branch and link with 16-bit
+ // offset.
+ Instr instr_b = REGIMM | BGEZAL; // Branch and link.
+ instr_b = SetBranchOffset(pos, target_pos, instr_b);
+ // Correct ra register to point to one instruction after jalr from
+ // TurboAssembler::BranchAndLinkLong.
+ Instr instr_a = ADDIU | ra.code() << kRsShift | ra.code() << kRtShift |
+ kOptimizedBranchAndLinkLongReturnOffset;
+
+ instr_at_put(pos, instr_b);
+ instr_at_put(pos + 1 * kInstrSize, instr_a);
+ } else {
+ instr_lui &= ~kImm16Mask;
+ instr_ori &= ~kImm16Mask;
+ PatchLuiOriImmediate(pos, imm, instr_lui, 0 * kInstrSize, instr_ori,
+ 2 * kInstrSize);
+ }
+ } else {
+ Instr instr1 = instr_at(pos + 0 * kInstrSize);
+ Instr instr2 = instr_at(pos + 1 * kInstrSize);
+ DCHECK(IsOri(instr2) || IsJicOrJialc(instr2));
+ uint32_t imm = reinterpret_cast<uint32_t>(buffer_start_) + target_pos;
+ DCHECK_EQ(imm & 3, 0);
+ DCHECK(IsLui(instr1) && (IsJicOrJialc(instr2) || IsOri(instr2)));
+ instr1 &= ~kImm16Mask;
+ instr2 &= ~kImm16Mask;
+
+ if (IsJicOrJialc(instr2)) {
+ uint32_t lui_offset_u, jic_offset_u;
+ UnpackTargetAddressUnsigned(imm, &lui_offset_u, &jic_offset_u);
+ instr_at_put(pos + 0 * kInstrSize, instr1 | lui_offset_u);
+ instr_at_put(pos + 1 * kInstrSize, instr2 | jic_offset_u);
+ } else {
+ PatchLuiOriImmediate(pos, imm, instr1, 0 * kInstrSize, instr2,
+ 1 * kInstrSize);
+ }
+ }
+ }
+}
+
+void Assembler::print(const Label* L) {
+ if (L->is_unused()) {
+ PrintF("unused label\n");
+ } else if (L->is_bound()) {
+ PrintF("bound label to %d\n", L->pos());
+ } else if (L->is_linked()) {
+ Label l;
+ l.link_to(L->pos());
+ PrintF("unbound label");
+ while (l.is_linked()) {
+ PrintF("@ %d ", l.pos());
+ Instr instr = instr_at(l.pos());
+ if ((instr & ~kImm16Mask) == 0) {
+ PrintF("value\n");
+ } else {
+ PrintF("%d\n", instr);
+ }
+ next(&l, is_internal_reference(&l));
+ }
+ } else {
+ PrintF("label in inconsistent state (pos = %d)\n", L->pos_);
+ }
+}
+
+void Assembler::bind_to(Label* L, int pos) {
+ DCHECK(0 <= pos && pos <= pc_offset()); // Must have valid binding position.
+ int32_t trampoline_pos = kInvalidSlotPos;
+ bool is_internal = false;
+ if (L->is_linked() && !trampoline_emitted_) {
+ unbound_labels_count_--;
+ if (!is_internal_reference(L)) {
+ next_buffer_check_ += kTrampolineSlotsSize;
+ }
+ }
+
+ while (L->is_linked()) {
+ int32_t fixup_pos = L->pos();
+ int32_t dist = pos - fixup_pos;
+ is_internal = is_internal_reference(L);
+ next(L, is_internal); // Call next before overwriting link with target at
+ // fixup_pos.
+ Instr instr = instr_at(fixup_pos);
+ if (is_internal) {
+ target_at_put(fixup_pos, pos, is_internal);
+ } else {
+ if (IsBranch(instr)) {
+ int branch_offset = BranchOffset(instr);
+ if (dist > branch_offset) {
+ if (trampoline_pos == kInvalidSlotPos) {
+ trampoline_pos = get_trampoline_entry(fixup_pos);
+ CHECK_NE(trampoline_pos, kInvalidSlotPos);
+ }
+ CHECK((trampoline_pos - fixup_pos) <= branch_offset);
+ target_at_put(fixup_pos, trampoline_pos, false);
+ fixup_pos = trampoline_pos;
+ }
+ target_at_put(fixup_pos, pos, false);
+ } else {
+ target_at_put(fixup_pos, pos, false);
+ }
+ }
+ }
+ L->bind_to(pos);
+
+ // Keep track of the last bound label so we don't eliminate any instructions
+ // before a bound label.
+ if (pos > last_bound_pos_) last_bound_pos_ = pos;
+}
+
+void Assembler::bind(Label* L) {
+ DCHECK(!L->is_bound()); // Label can only be bound once.
+ bind_to(L, pc_offset());
+}
+
+void Assembler::next(Label* L, bool is_internal) {
+ DCHECK(L->is_linked());
+ int link = target_at(L->pos(), is_internal);
+ if (link == kEndOfChain) {
+ L->Unuse();
+ } else {
+ DCHECK_GE(link, 0);
+ L->link_to(link);
+ }
+}
+
+bool Assembler::is_near(Label* L) {
+ DCHECK(L->is_bound());
+ return pc_offset() - L->pos() < kMaxBranchOffset - 4 * kInstrSize;
+}
+
+bool Assembler::is_near(Label* L, OffsetSize bits) {
+ if (L == nullptr || !L->is_bound()) return true;
+ return pc_offset() - L->pos() < (1 << (bits + 2 - 1)) - 1 - 5 * kInstrSize;
+}
+
+bool Assembler::is_near_branch(Label* L) {
+ DCHECK(L->is_bound());
+ return IsMipsArchVariant(kMips32r6) ? is_near_r6(L) : is_near_pre_r6(L);
+}
+
+int Assembler::BranchOffset(Instr instr) {
+ // At pre-R6 and for other R6 branches the offset is 16 bits.
+ int bits = OffsetSize::kOffset16;
+
+ if (IsMipsArchVariant(kMips32r6)) {
+ uint32_t opcode = GetOpcodeField(instr);
+ switch (opcode) {
+ // Checks BC or BALC.
+ case BC:
+ case BALC:
+ bits = OffsetSize::kOffset26;
+ break;
+
+ // Checks BEQZC or BNEZC.
+ case POP66:
+ case POP76:
+ if (GetRsField(instr) != 0) bits = OffsetSize::kOffset21;
+ break;
+ default:
+ break;
+ }
+ }
+
+ return (1 << (bits + 2 - 1)) - 1;
+}
+
+// We have to use a temporary register for things that can be relocated even
+// if they can be encoded in the MIPS's 16 bits of immediate-offset instruction
+// space. There is no guarantee that the relocated location can be similarly
+// encoded.
+bool Assembler::MustUseReg(RelocInfo::Mode rmode) {
+ return !RelocInfo::IsNone(rmode);
+}
+
+void Assembler::GenInstrRegister(Opcode opcode, Register rs, Register rt,
+ Register rd, uint16_t sa,
+ SecondaryField func) {
+ DCHECK(rd.is_valid() && rs.is_valid() && rt.is_valid() && is_uint5(sa));
+ Instr instr = opcode | (rs.code() << kRsShift) | (rt.code() << kRtShift) |
+ (rd.code() << kRdShift) | (sa << kSaShift) | func;
+ emit(instr);
+}
+
+void Assembler::GenInstrRegister(Opcode opcode, Register rs, Register rt,
+ uint16_t msb, uint16_t lsb,
+ SecondaryField func) {
+ DCHECK(rs.is_valid() && rt.is_valid() && is_uint5(msb) && is_uint5(lsb));
+ Instr instr = opcode | (rs.code() << kRsShift) | (rt.code() << kRtShift) |
+ (msb << kRdShift) | (lsb << kSaShift) | func;
+ emit(instr);
+}
+
+void Assembler::GenInstrRegister(Opcode opcode, SecondaryField fmt,
+ FPURegister ft, FPURegister fs, FPURegister fd,
+ SecondaryField func) {
+ DCHECK(fd.is_valid() && fs.is_valid() && ft.is_valid());
+ Instr instr = opcode | fmt | (ft.code() << kFtShift) |
+ (fs.code() << kFsShift) | (fd.code() << kFdShift) | func;
+ emit(instr);
+}
+
+void Assembler::GenInstrRegister(Opcode opcode, FPURegister fr, FPURegister ft,
+ FPURegister fs, FPURegister fd,
+ SecondaryField func) {
+ DCHECK(fd.is_valid() && fr.is_valid() && fs.is_valid() && ft.is_valid());
+ Instr instr = opcode | (fr.code() << kFrShift) | (ft.code() << kFtShift) |
+ (fs.code() << kFsShift) | (fd.code() << kFdShift) | func;
+ emit(instr);
+}
+
+void Assembler::GenInstrRegister(Opcode opcode, SecondaryField fmt, Register rt,
+ FPURegister fs, FPURegister fd,
+ SecondaryField func) {
+ DCHECK(fd.is_valid() && fs.is_valid() && rt.is_valid());
+ Instr instr = opcode | fmt | (rt.code() << kRtShift) |
+ (fs.code() << kFsShift) | (fd.code() << kFdShift) | func;
+ emit(instr);
+}
+
+void Assembler::GenInstrRegister(Opcode opcode, SecondaryField fmt, Register rt,
+ FPUControlRegister fs, SecondaryField func) {
+ DCHECK(fs.is_valid() && rt.is_valid());
+ Instr instr =
+ opcode | fmt | (rt.code() << kRtShift) | (fs.code() << kFsShift) | func;
+ emit(instr);
+}
+
+// Instructions with immediate value.
+// Registers are in the order of the instruction encoding, from left to right.
+void Assembler::GenInstrImmediate(Opcode opcode, Register rs, Register rt,
+ int32_t j,
+ CompactBranchType is_compact_branch) {
+ DCHECK(rs.is_valid() && rt.is_valid() && (is_int16(j) || is_uint16(j)));
+ Instr instr = opcode | (rs.code() << kRsShift) | (rt.code() << kRtShift) |
+ (j & kImm16Mask);
+ emit(instr, is_compact_branch);
+}
+
+void Assembler::GenInstrImmediate(Opcode opcode, Register base, Register rt,
+ int32_t offset9, int bit6,
+ SecondaryField func) {
+ DCHECK(base.is_valid() && rt.is_valid() && is_int9(offset9) &&
+ is_uint1(bit6));
+ Instr instr = opcode | (base.code() << kBaseShift) | (rt.code() << kRtShift) |
+ ((offset9 << kImm9Shift) & kImm9Mask) | bit6 << kBit6Shift |
+ func;
+ emit(instr);
+}
+
+void Assembler::GenInstrImmediate(Opcode opcode, Register rs, SecondaryField SF,
+ int32_t j,
+ CompactBranchType is_compact_branch) {
+ DCHECK(rs.is_valid() && (is_int16(j) || is_uint16(j)));
+ Instr instr = opcode | (rs.code() << kRsShift) | SF | (j & kImm16Mask);
+ emit(instr, is_compact_branch);
+}
+
+void Assembler::GenInstrImmediate(Opcode opcode, Register rs, FPURegister ft,
+ int32_t j,
+ CompactBranchType is_compact_branch) {
+ DCHECK(rs.is_valid() && ft.is_valid() && (is_int16(j) || is_uint16(j)));
+ Instr instr = opcode | (rs.code() << kRsShift) | (ft.code() << kFtShift) |
+ (j & kImm16Mask);
+ emit(instr, is_compact_branch);
+}
+
+void Assembler::GenInstrImmediate(Opcode opcode, Register rs, int32_t offset21,
+ CompactBranchType is_compact_branch) {
+ DCHECK(rs.is_valid() && (is_int21(offset21)));
+ Instr instr = opcode | (rs.code() << kRsShift) | (offset21 & kImm21Mask);
+ emit(instr, is_compact_branch);
+}
+
+void Assembler::GenInstrImmediate(Opcode opcode, Register rs,
+ uint32_t offset21) {
+ DCHECK(rs.is_valid() && (is_uint21(offset21)));
+ Instr instr = opcode | (rs.code() << kRsShift) | (offset21 & kImm21Mask);
+ emit(instr);
+}
+
+void Assembler::GenInstrImmediate(Opcode opcode, int32_t offset26,
+ CompactBranchType is_compact_branch) {
+ DCHECK(is_int26(offset26));
+ Instr instr = opcode | (offset26 & kImm26Mask);
+ emit(instr, is_compact_branch);
+}
+
+void Assembler::GenInstrJump(Opcode opcode, uint32_t address) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ DCHECK(is_uint26(address));
+ Instr instr = opcode | address;
+ emit(instr);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+// MSA instructions
+void Assembler::GenInstrMsaI8(SecondaryField operation, uint32_t imm8,
+ MSARegister ws, MSARegister wd) {
+ DCHECK(IsMipsArchVariant(kMips32r6) && IsEnabled(MIPS_SIMD));
+ DCHECK(ws.is_valid() && wd.is_valid() && is_uint8(imm8));
+ Instr instr = MSA | operation | ((imm8 & kImm8Mask) << kWtShift) |
+ (ws.code() << kWsShift) | (wd.code() << kWdShift);
+ emit(instr);
+}
+
+void Assembler::GenInstrMsaI5(SecondaryField operation, SecondaryField df,
+ int32_t imm5, MSARegister ws, MSARegister wd) {
+ DCHECK(IsMipsArchVariant(kMips32r6) && IsEnabled(MIPS_SIMD));
+ DCHECK(ws.is_valid() && wd.is_valid());
+ DCHECK((operation == MAXI_S) || (operation == MINI_S) ||
+ (operation == CEQI) || (operation == CLTI_S) ||
+ (operation == CLEI_S)
+ ? is_int5(imm5)
+ : is_uint5(imm5));
+ Instr instr = MSA | operation | df | ((imm5 & kImm5Mask) << kWtShift) |
+ (ws.code() << kWsShift) | (wd.code() << kWdShift);
+ emit(instr);
+}
+
+void Assembler::GenInstrMsaBit(SecondaryField operation, SecondaryField df,
+ uint32_t m, MSARegister ws, MSARegister wd) {
+ DCHECK(IsMipsArchVariant(kMips32r6) && IsEnabled(MIPS_SIMD));
+ DCHECK(ws.is_valid() && wd.is_valid() && is_valid_msa_df_m(df, m));
+ Instr instr = MSA | operation | df | (m << kWtShift) |
+ (ws.code() << kWsShift) | (wd.code() << kWdShift);
+ emit(instr);
+}
+
+void Assembler::GenInstrMsaI10(SecondaryField operation, SecondaryField df,
+ int32_t imm10, MSARegister wd) {
+ DCHECK(IsMipsArchVariant(kMips32r6) && IsEnabled(MIPS_SIMD));
+ DCHECK(wd.is_valid() && is_int10(imm10));
+ Instr instr = MSA | operation | df | ((imm10 & kImm10Mask) << kWsShift) |
+ (wd.code() << kWdShift);
+ emit(instr);
+}
+
+template <typename RegType>
+void Assembler::GenInstrMsa3R(SecondaryField operation, SecondaryField df,
+ RegType t, MSARegister ws, MSARegister wd) {
+ DCHECK(IsMipsArchVariant(kMips32r6) && IsEnabled(MIPS_SIMD));
+ DCHECK(t.is_valid() && ws.is_valid() && wd.is_valid());
+ Instr instr = MSA | operation | df | (t.code() << kWtShift) |
+ (ws.code() << kWsShift) | (wd.code() << kWdShift);
+ emit(instr);
+}
+
+template <typename DstType, typename SrcType>
+void Assembler::GenInstrMsaElm(SecondaryField operation, SecondaryField df,
+ uint32_t n, SrcType src, DstType dst) {
+ DCHECK(IsMipsArchVariant(kMips32r6) && IsEnabled(MIPS_SIMD));
+ DCHECK(src.is_valid() && dst.is_valid() && is_valid_msa_df_n(df, n));
+ Instr instr = MSA | operation | df | (n << kWtShift) |
+ (src.code() << kWsShift) | (dst.code() << kWdShift) |
+ MSA_ELM_MINOR;
+ emit(instr);
+}
+
+void Assembler::GenInstrMsa3RF(SecondaryField operation, uint32_t df,
+ MSARegister wt, MSARegister ws, MSARegister wd) {
+ DCHECK(IsMipsArchVariant(kMips32r6) && IsEnabled(MIPS_SIMD));
+ DCHECK(wt.is_valid() && ws.is_valid() && wd.is_valid());
+ DCHECK_LT(df, 2);
+ Instr instr = MSA | operation | (df << 21) | (wt.code() << kWtShift) |
+ (ws.code() << kWsShift) | (wd.code() << kWdShift);
+ emit(instr);
+}
+
+void Assembler::GenInstrMsaVec(SecondaryField operation, MSARegister wt,
+ MSARegister ws, MSARegister wd) {
+ DCHECK(IsMipsArchVariant(kMips32r6) && IsEnabled(MIPS_SIMD));
+ DCHECK(wt.is_valid() && ws.is_valid() && wd.is_valid());
+ Instr instr = MSA | operation | (wt.code() << kWtShift) |
+ (ws.code() << kWsShift) | (wd.code() << kWdShift) |
+ MSA_VEC_2R_2RF_MINOR;
+ emit(instr);
+}
+
+void Assembler::GenInstrMsaMI10(SecondaryField operation, int32_t s10,
+ Register rs, MSARegister wd) {
+ DCHECK(IsMipsArchVariant(kMips32r6) && IsEnabled(MIPS_SIMD));
+ DCHECK(rs.is_valid() && wd.is_valid() && is_int10(s10));
+ Instr instr = MSA | operation | ((s10 & kImm10Mask) << kWtShift) |
+ (rs.code() << kWsShift) | (wd.code() << kWdShift);
+ emit(instr);
+}
+
+void Assembler::GenInstrMsa2R(SecondaryField operation, SecondaryField df,
+ MSARegister ws, MSARegister wd) {
+ DCHECK(IsMipsArchVariant(kMips32r6) && IsEnabled(MIPS_SIMD));
+ DCHECK(ws.is_valid() && wd.is_valid());
+ Instr instr = MSA | MSA_2R_FORMAT | operation | df | (ws.code() << kWsShift) |
+ (wd.code() << kWdShift) | MSA_VEC_2R_2RF_MINOR;
+ emit(instr);
+}
+
+void Assembler::GenInstrMsa2RF(SecondaryField operation, SecondaryField df,
+ MSARegister ws, MSARegister wd) {
+ DCHECK(IsMipsArchVariant(kMips32r6) && IsEnabled(MIPS_SIMD));
+ DCHECK(ws.is_valid() && wd.is_valid());
+ Instr instr = MSA | MSA_2RF_FORMAT | operation | df |
+ (ws.code() << kWsShift) | (wd.code() << kWdShift) |
+ MSA_VEC_2R_2RF_MINOR;
+ emit(instr);
+}
+
+void Assembler::GenInstrMsaBranch(SecondaryField operation, MSARegister wt,
+ int32_t offset16) {
+ DCHECK(IsMipsArchVariant(kMips32r6) && IsEnabled(MIPS_SIMD));
+ DCHECK(wt.is_valid() && is_int16(offset16));
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Instr instr =
+ COP1 | operation | (wt.code() << kWtShift) | (offset16 & kImm16Mask);
+ emit(instr);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+// Returns the next free trampoline entry.
+int32_t Assembler::get_trampoline_entry(int32_t pos) {
+ int32_t trampoline_entry = kInvalidSlotPos;
+
+ if (!internal_trampoline_exception_) {
+ if (trampoline_.start() > pos) {
+ trampoline_entry = trampoline_.take_slot();
+ }
+
+ if (kInvalidSlotPos == trampoline_entry) {
+ internal_trampoline_exception_ = true;
+ }
+ }
+ return trampoline_entry;
+}
+
+uint32_t Assembler::jump_address(Label* L) {
+ int32_t target_pos;
+
+ if (L->is_bound()) {
+ target_pos = L->pos();
+ } else {
+ if (L->is_linked()) {
+ target_pos = L->pos(); // L's link.
+ L->link_to(pc_offset());
+ } else {
+ L->link_to(pc_offset());
+ return kEndOfJumpChain;
+ }
+ }
+
+ uint32_t imm = reinterpret_cast<uint32_t>(buffer_start_) + target_pos;
+ DCHECK_EQ(imm & 3, 0);
+
+ return imm;
+}
+
+uint32_t Assembler::branch_long_offset(Label* L) {
+ int32_t target_pos;
+
+ if (L->is_bound()) {
+ target_pos = L->pos();
+ } else {
+ if (L->is_linked()) {
+ target_pos = L->pos(); // L's link.
+ L->link_to(pc_offset());
+ } else {
+ L->link_to(pc_offset());
+ return kEndOfJumpChain;
+ }
+ }
+
+ DCHECK(is_int32(static_cast<int64_t>(target_pos) -
+ static_cast<int64_t>(pc_offset() + kLongBranchPCOffset)));
+ int32_t offset = target_pos - (pc_offset() + kLongBranchPCOffset);
+ DCHECK_EQ(offset & 3, 0);
+
+ return offset;
+}
+
+int32_t Assembler::branch_offset_helper(Label* L, OffsetSize bits) {
+ int32_t target_pos;
+ int32_t pad = IsPrevInstrCompactBranch() ? kInstrSize : 0;
+
+ if (L->is_bound()) {
+ target_pos = L->pos();
+ } else {
+ if (L->is_linked()) {
+ target_pos = L->pos();
+ L->link_to(pc_offset() + pad);
+ } else {
+ L->link_to(pc_offset() + pad);
+ if (!trampoline_emitted_) {
+ unbound_labels_count_++;
+ next_buffer_check_ -= kTrampolineSlotsSize;
+ }
+ return kEndOfChain;
+ }
+ }
+
+ int32_t offset = target_pos - (pc_offset() + kBranchPCOffset + pad);
+ DCHECK(is_intn(offset, bits + 2));
+ DCHECK_EQ(offset & 3, 0);
+
+ return offset;
+}
+
+void Assembler::label_at_put(Label* L, int at_offset) {
+ int target_pos;
+ if (L->is_bound()) {
+ target_pos = L->pos();
+ instr_at_put(at_offset, target_pos + (Code::kHeaderSize - kHeapObjectTag));
+ } else {
+ if (L->is_linked()) {
+ target_pos = L->pos(); // L's link.
+ int32_t imm18 = target_pos - at_offset;
+ DCHECK_EQ(imm18 & 3, 0);
+ int32_t imm16 = imm18 >> 2;
+ DCHECK(is_int16(imm16));
+ instr_at_put(at_offset, (imm16 & kImm16Mask));
+ } else {
+ target_pos = kEndOfChain;
+ instr_at_put(at_offset, 0);
+ if (!trampoline_emitted_) {
+ unbound_labels_count_++;
+ next_buffer_check_ -= kTrampolineSlotsSize;
+ }
+ }
+ L->link_to(at_offset);
+ }
+}
+
+//------- Branch and jump instructions --------
+
+void Assembler::b(int16_t offset) { beq(zero_reg, zero_reg, offset); }
+
+void Assembler::bal(int16_t offset) { bgezal(zero_reg, offset); }
+
+void Assembler::bc(int32_t offset) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ GenInstrImmediate(BC, offset, CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::balc(int32_t offset) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ GenInstrImmediate(BALC, offset, CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::beq(Register rs, Register rt, int16_t offset) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ GenInstrImmediate(BEQ, rs, rt, offset);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+void Assembler::bgez(Register rs, int16_t offset) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ GenInstrImmediate(REGIMM, rs, BGEZ, offset);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+void Assembler::bgezc(Register rt, int16_t offset) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK(rt != zero_reg);
+ GenInstrImmediate(BLEZL, rt, rt, offset, CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::bgeuc(Register rs, Register rt, int16_t offset) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK(rs != zero_reg);
+ DCHECK(rt != zero_reg);
+ DCHECK(rs.code() != rt.code());
+ GenInstrImmediate(BLEZ, rs, rt, offset, CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::bgec(Register rs, Register rt, int16_t offset) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK(rs != zero_reg);
+ DCHECK(rt != zero_reg);
+ DCHECK(rs.code() != rt.code());
+ GenInstrImmediate(BLEZL, rs, rt, offset, CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::bgezal(Register rs, int16_t offset) {
+ DCHECK(!IsMipsArchVariant(kMips32r6) || rs == zero_reg);
+ DCHECK(rs != ra);
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ GenInstrImmediate(REGIMM, rs, BGEZAL, offset);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+void Assembler::bgtz(Register rs, int16_t offset) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ GenInstrImmediate(BGTZ, rs, zero_reg, offset);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+void Assembler::bgtzc(Register rt, int16_t offset) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK(rt != zero_reg);
+ GenInstrImmediate(BGTZL, zero_reg, rt, offset,
+ CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::blez(Register rs, int16_t offset) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ GenInstrImmediate(BLEZ, rs, zero_reg, offset);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+void Assembler::blezc(Register rt, int16_t offset) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK(rt != zero_reg);
+ GenInstrImmediate(BLEZL, zero_reg, rt, offset,
+ CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::bltzc(Register rt, int16_t offset) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK(rt != zero_reg);
+ GenInstrImmediate(BGTZL, rt, rt, offset, CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::bltuc(Register rs, Register rt, int16_t offset) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK(rs != zero_reg);
+ DCHECK(rt != zero_reg);
+ DCHECK(rs.code() != rt.code());
+ GenInstrImmediate(BGTZ, rs, rt, offset, CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::bltc(Register rs, Register rt, int16_t offset) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK(rs != zero_reg);
+ DCHECK(rt != zero_reg);
+ DCHECK(rs.code() != rt.code());
+ GenInstrImmediate(BGTZL, rs, rt, offset, CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::bltz(Register rs, int16_t offset) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ GenInstrImmediate(REGIMM, rs, BLTZ, offset);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+void Assembler::bltzal(Register rs, int16_t offset) {
+ DCHECK(!IsMipsArchVariant(kMips32r6) || rs == zero_reg);
+ DCHECK(rs != ra);
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ GenInstrImmediate(REGIMM, rs, BLTZAL, offset);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+void Assembler::bne(Register rs, Register rt, int16_t offset) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ GenInstrImmediate(BNE, rs, rt, offset);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+void Assembler::bovc(Register rs, Register rt, int16_t offset) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ if (rs.code() >= rt.code()) {
+ GenInstrImmediate(ADDI, rs, rt, offset, CompactBranchType::COMPACT_BRANCH);
+ } else {
+ GenInstrImmediate(ADDI, rt, rs, offset, CompactBranchType::COMPACT_BRANCH);
+ }
+}
+
+void Assembler::bnvc(Register rs, Register rt, int16_t offset) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ if (rs.code() >= rt.code()) {
+ GenInstrImmediate(DADDI, rs, rt, offset, CompactBranchType::COMPACT_BRANCH);
+ } else {
+ GenInstrImmediate(DADDI, rt, rs, offset, CompactBranchType::COMPACT_BRANCH);
+ }
+}
+
+void Assembler::blezalc(Register rt, int16_t offset) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK(rt != zero_reg);
+ DCHECK(rt != ra);
+ GenInstrImmediate(BLEZ, zero_reg, rt, offset,
+ CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::bgezalc(Register rt, int16_t offset) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK(rt != zero_reg);
+ DCHECK(rt != ra);
+ GenInstrImmediate(BLEZ, rt, rt, offset, CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::bgezall(Register rs, int16_t offset) {
+ DCHECK(!IsMipsArchVariant(kMips32r6));
+ DCHECK(rs != zero_reg);
+ DCHECK(rs != ra);
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ GenInstrImmediate(REGIMM, rs, BGEZALL, offset);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+void Assembler::bltzalc(Register rt, int16_t offset) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK(rt != zero_reg);
+ DCHECK(rt != ra);
+ GenInstrImmediate(BGTZ, rt, rt, offset, CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::bgtzalc(Register rt, int16_t offset) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK(rt != zero_reg);
+ DCHECK(rt != ra);
+ GenInstrImmediate(BGTZ, zero_reg, rt, offset,
+ CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::beqzalc(Register rt, int16_t offset) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK(rt != zero_reg);
+ DCHECK(rt != ra);
+ GenInstrImmediate(ADDI, zero_reg, rt, offset,
+ CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::bnezalc(Register rt, int16_t offset) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK(rt != zero_reg);
+ DCHECK(rt != ra);
+ GenInstrImmediate(DADDI, zero_reg, rt, offset,
+ CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::beqc(Register rs, Register rt, int16_t offset) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK(rs.code() != rt.code() && rs.code() != 0 && rt.code() != 0);
+ if (rs.code() < rt.code()) {
+ GenInstrImmediate(ADDI, rs, rt, offset, CompactBranchType::COMPACT_BRANCH);
+ } else {
+ GenInstrImmediate(ADDI, rt, rs, offset, CompactBranchType::COMPACT_BRANCH);
+ }
+}
+
+void Assembler::beqzc(Register rs, int32_t offset) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK(rs != zero_reg);
+ GenInstrImmediate(POP66, rs, offset, CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::bnec(Register rs, Register rt, int16_t offset) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK(rs.code() != rt.code() && rs.code() != 0 && rt.code() != 0);
+ if (rs.code() < rt.code()) {
+ GenInstrImmediate(DADDI, rs, rt, offset, CompactBranchType::COMPACT_BRANCH);
+ } else {
+ GenInstrImmediate(DADDI, rt, rs, offset, CompactBranchType::COMPACT_BRANCH);
+ }
+}
+
+void Assembler::bnezc(Register rs, int32_t offset) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK(rs != zero_reg);
+ GenInstrImmediate(POP76, rs, offset, CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::j(int32_t target) {
+#if DEBUG
+ // Get pc of delay slot.
+ uint32_t ipc = reinterpret_cast<uint32_t>(pc_ + 1 * kInstrSize);
+ bool in_range = ((ipc ^ static_cast<uint32_t>(target)) >>
+ (kImm26Bits + kImmFieldShift)) == 0;
+ DCHECK(in_range && ((target & 3) == 0));
+#endif
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ GenInstrJump(J, (target >> 2) & kImm26Mask);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+void Assembler::jr(Register rs) {
+ if (!IsMipsArchVariant(kMips32r6)) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ GenInstrRegister(SPECIAL, rs, zero_reg, zero_reg, 0, JR);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+ } else {
+ jalr(rs, zero_reg);
+ }
+}
+
+void Assembler::jal(int32_t target) {
+#ifdef DEBUG
+ // Get pc of delay slot.
+ uint32_t ipc = reinterpret_cast<uint32_t>(pc_ + 1 * kInstrSize);
+ bool in_range = ((ipc ^ static_cast<uint32_t>(target)) >>
+ (kImm26Bits + kImmFieldShift)) == 0;
+ DCHECK(in_range && ((target & 3) == 0));
+#endif
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ GenInstrJump(JAL, (target >> 2) & kImm26Mask);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+void Assembler::jalr(Register rs, Register rd) {
+ DCHECK(rs.code() != rd.code());
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ GenInstrRegister(SPECIAL, rs, zero_reg, rd, 0, JALR);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+void Assembler::jic(Register rt, int16_t offset) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ GenInstrImmediate(POP66, zero_reg, rt, offset);
+}
+
+void Assembler::jialc(Register rt, int16_t offset) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ GenInstrImmediate(POP76, zero_reg, rt, offset);
+}
+
+// -------Data-processing-instructions---------
+
+// Arithmetic.
+
+void Assembler::addu(Register rd, Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, ADDU);
+}
+
+void Assembler::addiu(Register rd, Register rs, int32_t j) {
+ GenInstrImmediate(ADDIU, rs, rd, j);
+}
+
+void Assembler::subu(Register rd, Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, SUBU);
+}
+
+void Assembler::mul(Register rd, Register rs, Register rt) {
+ if (!IsMipsArchVariant(kMips32r6)) {
+ GenInstrRegister(SPECIAL2, rs, rt, rd, 0, MUL);
+ } else {
+ GenInstrRegister(SPECIAL, rs, rt, rd, MUL_OP, MUL_MUH);
+ }
+}
+
+void Assembler::mulu(Register rd, Register rs, Register rt) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(SPECIAL, rs, rt, rd, MUL_OP, MUL_MUH_U);
+}
+
+void Assembler::muh(Register rd, Register rs, Register rt) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(SPECIAL, rs, rt, rd, MUH_OP, MUL_MUH);
+}
+
+void Assembler::muhu(Register rd, Register rs, Register rt) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(SPECIAL, rs, rt, rd, MUH_OP, MUL_MUH_U);
+}
+
+void Assembler::mod(Register rd, Register rs, Register rt) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(SPECIAL, rs, rt, rd, MOD_OP, DIV_MOD);
+}
+
+void Assembler::modu(Register rd, Register rs, Register rt) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(SPECIAL, rs, rt, rd, MOD_OP, DIV_MOD_U);
+}
+
+void Assembler::mult(Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, zero_reg, 0, MULT);
+}
+
+void Assembler::multu(Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, zero_reg, 0, MULTU);
+}
+
+void Assembler::div(Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, zero_reg, 0, DIV);
+}
+
+void Assembler::div(Register rd, Register rs, Register rt) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(SPECIAL, rs, rt, rd, DIV_OP, DIV_MOD);
+}
+
+void Assembler::divu(Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, zero_reg, 0, DIVU);
+}
+
+void Assembler::divu(Register rd, Register rs, Register rt) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(SPECIAL, rs, rt, rd, DIV_OP, DIV_MOD_U);
+}
+
+// Logical.
+
+void Assembler::and_(Register rd, Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, AND);
+}
+
+void Assembler::andi(Register rt, Register rs, int32_t j) {
+ DCHECK(is_uint16(j));
+ GenInstrImmediate(ANDI, rs, rt, j);
+}
+
+void Assembler::or_(Register rd, Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, OR);
+}
+
+void Assembler::ori(Register rt, Register rs, int32_t j) {
+ DCHECK(is_uint16(j));
+ GenInstrImmediate(ORI, rs, rt, j);
+}
+
+void Assembler::xor_(Register rd, Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, XOR);
+}
+
+void Assembler::xori(Register rt, Register rs, int32_t j) {
+ DCHECK(is_uint16(j));
+ GenInstrImmediate(XORI, rs, rt, j);
+}
+
+void Assembler::nor(Register rd, Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, NOR);
+}
+
+// Shifts.
+void Assembler::sll(Register rd, Register rt, uint16_t sa,
+ bool coming_from_nop) {
+ // Don't allow nop instructions in the form sll zero_reg, zero_reg to be
+ // generated using the sll instruction. They must be generated using
+ // nop(int/NopMarkerTypes).
+ DCHECK(coming_from_nop || !(rd == zero_reg && rt == zero_reg));
+ GenInstrRegister(SPECIAL, zero_reg, rt, rd, sa & 0x1F, SLL);
+}
+
+void Assembler::sllv(Register rd, Register rt, Register rs) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, SLLV);
+}
+
+void Assembler::srl(Register rd, Register rt, uint16_t sa) {
+ GenInstrRegister(SPECIAL, zero_reg, rt, rd, sa & 0x1F, SRL);
+}
+
+void Assembler::srlv(Register rd, Register rt, Register rs) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, SRLV);
+}
+
+void Assembler::sra(Register rd, Register rt, uint16_t sa) {
+ GenInstrRegister(SPECIAL, zero_reg, rt, rd, sa & 0x1F, SRA);
+}
+
+void Assembler::srav(Register rd, Register rt, Register rs) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, SRAV);
+}
+
+void Assembler::rotr(Register rd, Register rt, uint16_t sa) {
+ // Should be called via MacroAssembler::Ror.
+ DCHECK(rd.is_valid() && rt.is_valid() && is_uint5(sa));
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6));
+ Instr instr = SPECIAL | (1 << kRsShift) | (rt.code() << kRtShift) |
+ (rd.code() << kRdShift) | (sa << kSaShift) | SRL;
+ emit(instr);
+}
+
+void Assembler::rotrv(Register rd, Register rt, Register rs) {
+ // Should be called via MacroAssembler::Ror.
+ DCHECK(rd.is_valid() && rt.is_valid() && rs.is_valid());
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6));
+ Instr instr = SPECIAL | (rs.code() << kRsShift) | (rt.code() << kRtShift) |
+ (rd.code() << kRdShift) | (1 << kSaShift) | SRLV;
+ emit(instr);
+}
+
+void Assembler::lsa(Register rd, Register rt, Register rs, uint8_t sa) {
+ DCHECK(rd.is_valid() && rt.is_valid() && rs.is_valid());
+ DCHECK_LE(sa, 3);
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ Instr instr = SPECIAL | rs.code() << kRsShift | rt.code() << kRtShift |
+ rd.code() << kRdShift | sa << kSaShift | LSA;
+ emit(instr);
+}
+
+// ------------Memory-instructions-------------
+
+void Assembler::AdjustBaseAndOffset(MemOperand* src,
+ OffsetAccessType access_type,
+ int second_access_add_to_offset) {
+ // This method is used to adjust the base register and offset pair
+ // for a load/store when the offset doesn't fit into int16_t.
+ // It is assumed that 'base + offset' is sufficiently aligned for memory
+ // operands that are machine word in size or smaller. For doubleword-sized
+ // operands it's assumed that 'base' is a multiple of 8, while 'offset'
+ // may be a multiple of 4 (e.g. 4-byte-aligned long and double arguments
+ // and spilled variables on the stack accessed relative to the stack
+ // pointer register).
+ // We preserve the "alignment" of 'offset' by adjusting it by a multiple of 8.
+
+ bool doubleword_aligned = (src->offset() & (kDoubleSize - 1)) == 0;
+ bool two_accesses = static_cast<bool>(access_type) || !doubleword_aligned;
+ DCHECK_LE(second_access_add_to_offset, 7); // Must be <= 7.
+
+ // is_int16 must be passed a signed value, hence the static cast below.
+ if (is_int16(src->offset()) &&
+ (!two_accesses || is_int16(static_cast<int32_t>(
+ src->offset() + second_access_add_to_offset)))) {
+ // Nothing to do: 'offset' (and, if needed, 'offset + 4', or other specified
+ // value) fits into int16_t.
+ return;
+ }
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(src->rm() != scratch); // Must not overwrite the register 'base'
+ // while loading 'offset'.
+
+#ifdef DEBUG
+ // Remember the "(mis)alignment" of 'offset', it will be checked at the end.
+ uint32_t misalignment = src->offset() & (kDoubleSize - 1);
+#endif
+
+ // Do not load the whole 32-bit 'offset' if it can be represented as
+ // a sum of two 16-bit signed offsets. This can save an instruction or two.
+ // To simplify matters, only do this for a symmetric range of offsets from
+ // about -64KB to about +64KB, allowing further addition of 4 when accessing
+ // 64-bit variables with two 32-bit accesses.
+ constexpr int32_t kMinOffsetForSimpleAdjustment =
+ 0x7FF8; // Max int16_t that's a multiple of 8.
+ constexpr int32_t kMaxOffsetForSimpleAdjustment =
+ 2 * kMinOffsetForSimpleAdjustment;
+ if (0 <= src->offset() && src->offset() <= kMaxOffsetForSimpleAdjustment) {
+ addiu(at, src->rm(), kMinOffsetForSimpleAdjustment);
+ src->offset_ -= kMinOffsetForSimpleAdjustment;
+ } else if (-kMaxOffsetForSimpleAdjustment <= src->offset() &&
+ src->offset() < 0) {
+ addiu(at, src->rm(), -kMinOffsetForSimpleAdjustment);
+ src->offset_ += kMinOffsetForSimpleAdjustment;
+ } else if (IsMipsArchVariant(kMips32r6)) {
+ // On r6 take advantage of the aui instruction, e.g.:
+ // aui at, base, offset_high
+ // lw reg_lo, offset_low(at)
+ // lw reg_hi, (offset_low+4)(at)
+ // or when offset_low+4 overflows int16_t:
+ // aui at, base, offset_high
+ // addiu at, at, 8
+ // lw reg_lo, (offset_low-8)(at)
+ // lw reg_hi, (offset_low-4)(at)
+ int16_t offset_high = static_cast<uint16_t>(src->offset() >> 16);
+ int16_t offset_low = static_cast<uint16_t>(src->offset());
+ offset_high += (offset_low < 0)
+ ? 1
+ : 0; // Account for offset sign extension in load/store.
+ aui(scratch, src->rm(), static_cast<uint16_t>(offset_high));
+ if (two_accesses && !is_int16(static_cast<int32_t>(
+ offset_low + second_access_add_to_offset))) {
+ // Avoid overflow in the 16-bit offset of the load/store instruction when
+ // adding 4.
+ addiu(scratch, scratch, kDoubleSize);
+ offset_low -= kDoubleSize;
+ }
+ src->offset_ = offset_low;
+ } else {
+ // Do not load the whole 32-bit 'offset' if it can be represented as
+ // a sum of three 16-bit signed offsets. This can save an instruction.
+ // To simplify matters, only do this for a symmetric range of offsets from
+ // about -96KB to about +96KB, allowing further addition of 4 when accessing
+ // 64-bit variables with two 32-bit accesses.
+ constexpr int32_t kMinOffsetForMediumAdjustment =
+ 2 * kMinOffsetForSimpleAdjustment;
+ constexpr int32_t kMaxOffsetForMediumAdjustment =
+ 3 * kMinOffsetForSimpleAdjustment;
+ if (0 <= src->offset() && src->offset() <= kMaxOffsetForMediumAdjustment) {
+ addiu(scratch, src->rm(), kMinOffsetForMediumAdjustment / 2);
+ addiu(scratch, scratch, kMinOffsetForMediumAdjustment / 2);
+ src->offset_ -= kMinOffsetForMediumAdjustment;
+ } else if (-kMaxOffsetForMediumAdjustment <= src->offset() &&
+ src->offset() < 0) {
+ addiu(scratch, src->rm(), -kMinOffsetForMediumAdjustment / 2);
+ addiu(scratch, scratch, -kMinOffsetForMediumAdjustment / 2);
+ src->offset_ += kMinOffsetForMediumAdjustment;
+ } else {
+ // Now that all shorter options have been exhausted, load the full 32-bit
+ // offset.
+ int32_t loaded_offset = RoundDown(src->offset(), kDoubleSize);
+ lui(scratch, (loaded_offset >> kLuiShift) & kImm16Mask);
+ ori(scratch, scratch, loaded_offset & kImm16Mask); // Load 32-bit offset.
+ addu(scratch, scratch, src->rm());
+ src->offset_ -= loaded_offset;
+ }
+ }
+ src->rm_ = scratch;
+
+ DCHECK(is_int16(src->offset()));
+ if (two_accesses) {
+ DCHECK(is_int16(
+ static_cast<int32_t>(src->offset() + second_access_add_to_offset)));
+ }
+ DCHECK(misalignment == (src->offset() & (kDoubleSize - 1)));
+}
+
+void Assembler::lb(Register rd, const MemOperand& rs) {
+ MemOperand source = rs;
+ AdjustBaseAndOffset(&source);
+ GenInstrImmediate(LB, source.rm(), rd, source.offset());
+}
+
+void Assembler::lbu(Register rd, const MemOperand& rs) {
+ MemOperand source = rs;
+ AdjustBaseAndOffset(&source);
+ GenInstrImmediate(LBU, source.rm(), rd, source.offset());
+}
+
+void Assembler::lh(Register rd, const MemOperand& rs) {
+ MemOperand source = rs;
+ AdjustBaseAndOffset(&source);
+ GenInstrImmediate(LH, source.rm(), rd, source.offset());
+}
+
+void Assembler::lhu(Register rd, const MemOperand& rs) {
+ MemOperand source = rs;
+ AdjustBaseAndOffset(&source);
+ GenInstrImmediate(LHU, source.rm(), rd, source.offset());
+}
+
+void Assembler::lw(Register rd, const MemOperand& rs) {
+ MemOperand source = rs;
+ AdjustBaseAndOffset(&source);
+ GenInstrImmediate(LW, source.rm(), rd, source.offset());
+}
+
+void Assembler::lwl(Register rd, const MemOperand& rs) {
+ DCHECK(is_int16(rs.offset_));
+ DCHECK(IsMipsArchVariant(kLoongson) || IsMipsArchVariant(kMips32r1) ||
+ IsMipsArchVariant(kMips32r2));
+ GenInstrImmediate(LWL, rs.rm(), rd, rs.offset_);
+}
+
+void Assembler::lwr(Register rd, const MemOperand& rs) {
+ DCHECK(is_int16(rs.offset_));
+ DCHECK(IsMipsArchVariant(kLoongson) || IsMipsArchVariant(kMips32r1) ||
+ IsMipsArchVariant(kMips32r2));
+ GenInstrImmediate(LWR, rs.rm(), rd, rs.offset_);
+}
+
+void Assembler::sb(Register rd, const MemOperand& rs) {
+ MemOperand source = rs;
+ AdjustBaseAndOffset(&source);
+ GenInstrImmediate(SB, source.rm(), rd, source.offset());
+}
+
+void Assembler::sh(Register rd, const MemOperand& rs) {
+ MemOperand source = rs;
+ AdjustBaseAndOffset(&source);
+ GenInstrImmediate(SH, source.rm(), rd, source.offset());
+}
+
+void Assembler::sw(Register rd, const MemOperand& rs) {
+ MemOperand source = rs;
+ AdjustBaseAndOffset(&source);
+ GenInstrImmediate(SW, source.rm(), rd, source.offset());
+}
+
+void Assembler::swl(Register rd, const MemOperand& rs) {
+ DCHECK(is_int16(rs.offset_));
+ DCHECK(IsMipsArchVariant(kLoongson) || IsMipsArchVariant(kMips32r1) ||
+ IsMipsArchVariant(kMips32r2));
+ GenInstrImmediate(SWL, rs.rm(), rd, rs.offset_);
+}
+
+void Assembler::swr(Register rd, const MemOperand& rs) {
+ DCHECK(is_int16(rs.offset_));
+ DCHECK(IsMipsArchVariant(kLoongson) || IsMipsArchVariant(kMips32r1) ||
+ IsMipsArchVariant(kMips32r2));
+ GenInstrImmediate(SWR, rs.rm(), rd, rs.offset_);
+}
+
+void Assembler::ll(Register rd, const MemOperand& rs) {
+ if (IsMipsArchVariant(kMips32r6)) {
+ DCHECK(is_int9(rs.offset_));
+ GenInstrImmediate(SPECIAL3, rs.rm(), rd, rs.offset_, 0, LL_R6);
+ } else {
+ DCHECK(IsMipsArchVariant(kLoongson) || IsMipsArchVariant(kMips32r1) ||
+ IsMipsArchVariant(kMips32r2));
+ DCHECK(is_int16(rs.offset_));
+ GenInstrImmediate(LL, rs.rm(), rd, rs.offset_);
+ }
+}
+
+void Assembler::sc(Register rd, const MemOperand& rs) {
+ if (IsMipsArchVariant(kMips32r6)) {
+ DCHECK(is_int9(rs.offset_));
+ GenInstrImmediate(SPECIAL3, rs.rm(), rd, rs.offset_, 0, SC_R6);
+ } else {
+ DCHECK(IsMipsArchVariant(kLoongson) || IsMipsArchVariant(kMips32r1) ||
+ IsMipsArchVariant(kMips32r2));
+ GenInstrImmediate(SC, rs.rm(), rd, rs.offset_);
+ }
+}
+
+void Assembler::llx(Register rd, const MemOperand& rs) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK(is_int9(rs.offset_));
+ GenInstrImmediate(SPECIAL3, rs.rm(), rd, rs.offset_, 1, LL_R6);
+}
+
+void Assembler::scx(Register rd, const MemOperand& rs) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK(is_int9(rs.offset_));
+ GenInstrImmediate(SPECIAL3, rs.rm(), rd, rs.offset_, 1, SC_R6);
+}
+
+void Assembler::lui(Register rd, int32_t j) {
+ DCHECK(is_uint16(j) || is_int16(j));
+ GenInstrImmediate(LUI, zero_reg, rd, j);
+}
+
+void Assembler::aui(Register rt, Register rs, int32_t j) {
+ // This instruction uses same opcode as 'lui'. The difference in encoding is
+ // 'lui' has zero reg. for rs field.
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK(rs != zero_reg);
+ DCHECK(is_uint16(j));
+ GenInstrImmediate(LUI, rs, rt, j);
+}
+
+// ---------PC-Relative instructions-----------
+
+void Assembler::addiupc(Register rs, int32_t imm19) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK(rs.is_valid() && is_int19(imm19));
+ uint32_t imm21 = ADDIUPC << kImm19Bits | (imm19 & kImm19Mask);
+ GenInstrImmediate(PCREL, rs, imm21);
+}
+
+void Assembler::lwpc(Register rs, int32_t offset19) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK(rs.is_valid() && is_int19(offset19));
+ uint32_t imm21 = LWPC << kImm19Bits | (offset19 & kImm19Mask);
+ GenInstrImmediate(PCREL, rs, imm21);
+}
+
+void Assembler::auipc(Register rs, int16_t imm16) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK(rs.is_valid());
+ uint32_t imm21 = AUIPC << kImm16Bits | (imm16 & kImm16Mask);
+ GenInstrImmediate(PCREL, rs, imm21);
+}
+
+void Assembler::aluipc(Register rs, int16_t imm16) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK(rs.is_valid());
+ uint32_t imm21 = ALUIPC << kImm16Bits | (imm16 & kImm16Mask);
+ GenInstrImmediate(PCREL, rs, imm21);
+}
+
+// -------------Misc-instructions--------------
+
+// Break / Trap instructions.
+void Assembler::break_(uint32_t code, bool break_as_stop) {
+ DCHECK_EQ(code & ~0xFFFFF, 0);
+ // We need to invalidate breaks that could be stops as well because the
+ // simulator expects a char pointer after the stop instruction.
+ // See constants-mips.h for explanation.
+ DCHECK(
+ (break_as_stop && code <= kMaxStopCode && code > kMaxWatchpointCode) ||
+ (!break_as_stop && (code > kMaxStopCode || code <= kMaxWatchpointCode)));
+ Instr break_instr = SPECIAL | BREAK | (code << 6);
+ emit(break_instr);
+}
+
+void Assembler::stop(uint32_t code) {
+ DCHECK_GT(code, kMaxWatchpointCode);
+ DCHECK_LE(code, kMaxStopCode);
+#if V8_HOST_ARCH_MIPS
+ break_(0x54321);
+#else // V8_HOST_ARCH_MIPS
+ break_(code, true);
+#endif
+}
+
+void Assembler::tge(Register rs, Register rt, uint16_t code) {
+ DCHECK(is_uint10(code));
+ Instr instr =
+ SPECIAL | TGE | rs.code() << kRsShift | rt.code() << kRtShift | code << 6;
+ emit(instr);
+}
+
+void Assembler::tgeu(Register rs, Register rt, uint16_t code) {
+ DCHECK(is_uint10(code));
+ Instr instr = SPECIAL | TGEU | rs.code() << kRsShift | rt.code() << kRtShift |
+ code << 6;
+ emit(instr);
+}
+
+void Assembler::tlt(Register rs, Register rt, uint16_t code) {
+ DCHECK(is_uint10(code));
+ Instr instr =
+ SPECIAL | TLT | rs.code() << kRsShift | rt.code() << kRtShift | code << 6;
+ emit(instr);
+}
+
+void Assembler::tltu(Register rs, Register rt, uint16_t code) {
+ DCHECK(is_uint10(code));
+ Instr instr = SPECIAL | TLTU | rs.code() << kRsShift | rt.code() << kRtShift |
+ code << 6;
+ emit(instr);
+}
+
+void Assembler::teq(Register rs, Register rt, uint16_t code) {
+ DCHECK(is_uint10(code));
+ Instr instr =
+ SPECIAL | TEQ | rs.code() << kRsShift | rt.code() << kRtShift | code << 6;
+ emit(instr);
+}
+
+void Assembler::tne(Register rs, Register rt, uint16_t code) {
+ DCHECK(is_uint10(code));
+ Instr instr =
+ SPECIAL | TNE | rs.code() << kRsShift | rt.code() << kRtShift | code << 6;
+ emit(instr);
+}
+
+void Assembler::sync() {
+ Instr sync_instr = SPECIAL | SYNC;
+ emit(sync_instr);
+}
+
+// Move from HI/LO register.
+
+void Assembler::mfhi(Register rd) {
+ GenInstrRegister(SPECIAL, zero_reg, zero_reg, rd, 0, MFHI);
+}
+
+void Assembler::mflo(Register rd) {
+ GenInstrRegister(SPECIAL, zero_reg, zero_reg, rd, 0, MFLO);
+}
+
+// Set on less than instructions.
+void Assembler::slt(Register rd, Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, SLT);
+}
+
+void Assembler::sltu(Register rd, Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, SLTU);
+}
+
+void Assembler::slti(Register rt, Register rs, int32_t j) {
+ GenInstrImmediate(SLTI, rs, rt, j);
+}
+
+void Assembler::sltiu(Register rt, Register rs, int32_t j) {
+ GenInstrImmediate(SLTIU, rs, rt, j);
+}
+
+// Conditional move.
+void Assembler::movz(Register rd, Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, MOVZ);
+}
+
+void Assembler::movn(Register rd, Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, MOVN);
+}
+
+void Assembler::movt(Register rd, Register rs, uint16_t cc) {
+ Register rt = Register::from_code((cc & 0x0007) << 2 | 1);
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, MOVCI);
+}
+
+void Assembler::movf(Register rd, Register rs, uint16_t cc) {
+ Register rt = Register::from_code((cc & 0x0007) << 2 | 0);
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, MOVCI);
+}
+
+void Assembler::seleqz(Register rd, Register rs, Register rt) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, SELEQZ_S);
+}
+
+// Bit twiddling.
+void Assembler::clz(Register rd, Register rs) {
+ if (!IsMipsArchVariant(kMips32r6)) {
+ // Clz instr requires same GPR number in 'rd' and 'rt' fields.
+ GenInstrRegister(SPECIAL2, rs, rd, rd, 0, CLZ);
+ } else {
+ GenInstrRegister(SPECIAL, rs, zero_reg, rd, 1, CLZ_R6);
+ }
+}
+
+void Assembler::ins_(Register rt, Register rs, uint16_t pos, uint16_t size) {
+ // Should be called via MacroAssembler::Ins.
+ // Ins instr has 'rt' field as dest, and two uint5: msb, lsb.
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(SPECIAL3, rs, rt, pos + size - 1, pos, INS);
+}
+
+void Assembler::ext_(Register rt, Register rs, uint16_t pos, uint16_t size) {
+ // Should be called via MacroAssembler::Ext.
+ // Ext instr has 'rt' field as dest, and two uint5: msb, lsb.
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(SPECIAL3, rs, rt, size - 1, pos, EXT);
+}
+
+void Assembler::bitswap(Register rd, Register rt) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(SPECIAL3, zero_reg, rt, rd, 0, BSHFL);
+}
+
+void Assembler::pref(int32_t hint, const MemOperand& rs) {
+ DCHECK(!IsMipsArchVariant(kLoongson));
+ DCHECK(is_uint5(hint) && is_uint16(rs.offset_));
+ Instr instr =
+ PREF | (rs.rm().code() << kRsShift) | (hint << kRtShift) | (rs.offset_);
+ emit(instr);
+}
+
+void Assembler::align(Register rd, Register rs, Register rt, uint8_t bp) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK(is_uint3(bp));
+ uint16_t sa = (ALIGN << kBp2Bits) | bp;
+ GenInstrRegister(SPECIAL3, rs, rt, rd, sa, BSHFL);
+}
+
+// Byte swap.
+void Assembler::wsbh(Register rd, Register rt) {
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(SPECIAL3, zero_reg, rt, rd, WSBH, BSHFL);
+}
+
+void Assembler::seh(Register rd, Register rt) {
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(SPECIAL3, zero_reg, rt, rd, SEH, BSHFL);
+}
+
+void Assembler::seb(Register rd, Register rt) {
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(SPECIAL3, zero_reg, rt, rd, SEB, BSHFL);
+}
+
+// --------Coprocessor-instructions----------------
+
+// Load, store, move.
+void Assembler::lwc1(FPURegister fd, const MemOperand& src) {
+ MemOperand tmp = src;
+ AdjustBaseAndOffset(&tmp);
+ GenInstrImmediate(LWC1, tmp.rm(), fd, tmp.offset());
+}
+
+void Assembler::swc1(FPURegister fd, const MemOperand& src) {
+ MemOperand tmp = src;
+ AdjustBaseAndOffset(&tmp);
+ GenInstrImmediate(SWC1, tmp.rm(), fd, tmp.offset());
+}
+
+void Assembler::mtc1(Register rt, FPURegister fs) {
+ GenInstrRegister(COP1, MTC1, rt, fs, f0);
+}
+
+void Assembler::mthc1(Register rt, FPURegister fs) {
+ GenInstrRegister(COP1, MTHC1, rt, fs, f0);
+}
+
+void Assembler::mfc1(Register rt, FPURegister fs) {
+ GenInstrRegister(COP1, MFC1, rt, fs, f0);
+}
+
+void Assembler::mfhc1(Register rt, FPURegister fs) {
+ GenInstrRegister(COP1, MFHC1, rt, fs, f0);
+}
+
+void Assembler::ctc1(Register rt, FPUControlRegister fs) {
+ GenInstrRegister(COP1, CTC1, rt, fs);
+}
+
+void Assembler::cfc1(Register rt, FPUControlRegister fs) {
+ GenInstrRegister(COP1, CFC1, rt, fs);
+}
+
+void Assembler::movn_s(FPURegister fd, FPURegister fs, Register rt) {
+ DCHECK(!IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(COP1, S, rt, fs, fd, MOVN_C);
+}
+
+void Assembler::movn_d(FPURegister fd, FPURegister fs, Register rt) {
+ DCHECK(!IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(COP1, D, rt, fs, fd, MOVN_C);
+}
+
+void Assembler::sel(SecondaryField fmt, FPURegister fd, FPURegister fs,
+ FPURegister ft) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK((fmt == D) || (fmt == S));
+
+ GenInstrRegister(COP1, fmt, ft, fs, fd, SEL);
+}
+
+void Assembler::sel_s(FPURegister fd, FPURegister fs, FPURegister ft) {
+ sel(S, fd, fs, ft);
+}
+
+void Assembler::sel_d(FPURegister fd, FPURegister fs, FPURegister ft) {
+ sel(D, fd, fs, ft);
+}
+
+void Assembler::seleqz(SecondaryField fmt, FPURegister fd, FPURegister fs,
+ FPURegister ft) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK((fmt == D) || (fmt == S));
+ GenInstrRegister(COP1, fmt, ft, fs, fd, SELEQZ_C);
+}
+
+void Assembler::selnez(Register rd, Register rs, Register rt) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, SELNEZ_S);
+}
+
+void Assembler::selnez(SecondaryField fmt, FPURegister fd, FPURegister fs,
+ FPURegister ft) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK((fmt == D) || (fmt == S));
+ GenInstrRegister(COP1, fmt, ft, fs, fd, SELNEZ_C);
+}
+
+void Assembler::seleqz_d(FPURegister fd, FPURegister fs, FPURegister ft) {
+ seleqz(D, fd, fs, ft);
+}
+
+void Assembler::seleqz_s(FPURegister fd, FPURegister fs, FPURegister ft) {
+ seleqz(S, fd, fs, ft);
+}
+
+void Assembler::selnez_d(FPURegister fd, FPURegister fs, FPURegister ft) {
+ selnez(D, fd, fs, ft);
+}
+
+void Assembler::selnez_s(FPURegister fd, FPURegister fs, FPURegister ft) {
+ selnez(S, fd, fs, ft);
+}
+
+void Assembler::movz_s(FPURegister fd, FPURegister fs, Register rt) {
+ DCHECK(!IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(COP1, S, rt, fs, fd, MOVZ_C);
+}
+
+void Assembler::movz_d(FPURegister fd, FPURegister fs, Register rt) {
+ DCHECK(!IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(COP1, D, rt, fs, fd, MOVZ_C);
+}
+
+void Assembler::movt_s(FPURegister fd, FPURegister fs, uint16_t cc) {
+ DCHECK(!IsMipsArchVariant(kMips32r6));
+ FPURegister ft = FPURegister::from_code((cc & 0x0007) << 2 | 1);
+ GenInstrRegister(COP1, S, ft, fs, fd, MOVF);
+}
+
+void Assembler::movt_d(FPURegister fd, FPURegister fs, uint16_t cc) {
+ DCHECK(!IsMipsArchVariant(kMips32r6));
+ FPURegister ft = FPURegister::from_code((cc & 0x0007) << 2 | 1);
+ GenInstrRegister(COP1, D, ft, fs, fd, MOVF);
+}
+
+void Assembler::movf_s(FPURegister fd, FPURegister fs, uint16_t cc) {
+ DCHECK(!IsMipsArchVariant(kMips32r6));
+ FPURegister ft = FPURegister::from_code((cc & 0x0007) << 2 | 0);
+ GenInstrRegister(COP1, S, ft, fs, fd, MOVF);
+}
+
+void Assembler::movf_d(FPURegister fd, FPURegister fs, uint16_t cc) {
+ DCHECK(!IsMipsArchVariant(kMips32r6));
+ FPURegister ft = FPURegister::from_code((cc & 0x0007) << 2 | 0);
+ GenInstrRegister(COP1, D, ft, fs, fd, MOVF);
+}
+
+// Arithmetic.
+
+void Assembler::add_s(FPURegister fd, FPURegister fs, FPURegister ft) {
+ GenInstrRegister(COP1, S, ft, fs, fd, ADD_S);
+}
+
+void Assembler::add_d(FPURegister fd, FPURegister fs, FPURegister ft) {
+ GenInstrRegister(COP1, D, ft, fs, fd, ADD_D);
+}
+
+void Assembler::sub_s(FPURegister fd, FPURegister fs, FPURegister ft) {
+ GenInstrRegister(COP1, S, ft, fs, fd, SUB_S);
+}
+
+void Assembler::sub_d(FPURegister fd, FPURegister fs, FPURegister ft) {
+ GenInstrRegister(COP1, D, ft, fs, fd, SUB_D);
+}
+
+void Assembler::mul_s(FPURegister fd, FPURegister fs, FPURegister ft) {
+ GenInstrRegister(COP1, S, ft, fs, fd, MUL_S);
+}
+
+void Assembler::mul_d(FPURegister fd, FPURegister fs, FPURegister ft) {
+ GenInstrRegister(COP1, D, ft, fs, fd, MUL_D);
+}
+
+void Assembler::madd_s(FPURegister fd, FPURegister fr, FPURegister fs,
+ FPURegister ft) {
+ DCHECK(IsMipsArchVariant(kMips32r2));
+ GenInstrRegister(COP1X, fr, ft, fs, fd, MADD_S);
+}
+
+void Assembler::madd_d(FPURegister fd, FPURegister fr, FPURegister fs,
+ FPURegister ft) {
+ DCHECK(IsMipsArchVariant(kMips32r2));
+ GenInstrRegister(COP1X, fr, ft, fs, fd, MADD_D);
+}
+
+void Assembler::msub_s(FPURegister fd, FPURegister fr, FPURegister fs,
+ FPURegister ft) {
+ DCHECK(IsMipsArchVariant(kMips32r2));
+ GenInstrRegister(COP1X, fr, ft, fs, fd, MSUB_S);
+}
+
+void Assembler::msub_d(FPURegister fd, FPURegister fr, FPURegister fs,
+ FPURegister ft) {
+ DCHECK(IsMipsArchVariant(kMips32r2));
+ GenInstrRegister(COP1X, fr, ft, fs, fd, MSUB_D);
+}
+
+void Assembler::maddf_s(FPURegister fd, FPURegister fs, FPURegister ft) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(COP1, S, ft, fs, fd, MADDF_S);
+}
+
+void Assembler::maddf_d(FPURegister fd, FPURegister fs, FPURegister ft) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(COP1, D, ft, fs, fd, MADDF_D);
+}
+
+void Assembler::msubf_s(FPURegister fd, FPURegister fs, FPURegister ft) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(COP1, S, ft, fs, fd, MSUBF_S);
+}
+
+void Assembler::msubf_d(FPURegister fd, FPURegister fs, FPURegister ft) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(COP1, D, ft, fs, fd, MSUBF_D);
+}
+
+void Assembler::div_s(FPURegister fd, FPURegister fs, FPURegister ft) {
+ GenInstrRegister(COP1, S, ft, fs, fd, DIV_S);
+}
+
+void Assembler::div_d(FPURegister fd, FPURegister fs, FPURegister ft) {
+ GenInstrRegister(COP1, D, ft, fs, fd, DIV_D);
+}
+
+void Assembler::abs_s(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, S, f0, fs, fd, ABS_S);
+}
+
+void Assembler::abs_d(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, D, f0, fs, fd, ABS_D);
+}
+
+void Assembler::mov_d(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, D, f0, fs, fd, MOV_D);
+}
+
+void Assembler::mov_s(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, S, f0, fs, fd, MOV_S);
+}
+
+void Assembler::neg_s(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, S, f0, fs, fd, NEG_S);
+}
+
+void Assembler::neg_d(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, D, f0, fs, fd, NEG_D);
+}
+
+void Assembler::sqrt_s(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, S, f0, fs, fd, SQRT_S);
+}
+
+void Assembler::sqrt_d(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, D, f0, fs, fd, SQRT_D);
+}
+
+void Assembler::rsqrt_s(FPURegister fd, FPURegister fs) {
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(COP1, S, f0, fs, fd, RSQRT_S);
+}
+
+void Assembler::rsqrt_d(FPURegister fd, FPURegister fs) {
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(COP1, D, f0, fs, fd, RSQRT_D);
+}
+
+void Assembler::recip_d(FPURegister fd, FPURegister fs) {
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(COP1, D, f0, fs, fd, RECIP_D);
+}
+
+void Assembler::recip_s(FPURegister fd, FPURegister fs) {
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(COP1, S, f0, fs, fd, RECIP_S);
+}
+
+// Conversions.
+
+void Assembler::cvt_w_s(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, S, f0, fs, fd, CVT_W_S);
+}
+
+void Assembler::cvt_w_d(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, D, f0, fs, fd, CVT_W_D);
+}
+
+void Assembler::trunc_w_s(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, S, f0, fs, fd, TRUNC_W_S);
+}
+
+void Assembler::trunc_w_d(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, D, f0, fs, fd, TRUNC_W_D);
+}
+
+void Assembler::round_w_s(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, S, f0, fs, fd, ROUND_W_S);
+}
+
+void Assembler::round_w_d(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, D, f0, fs, fd, ROUND_W_D);
+}
+
+void Assembler::floor_w_s(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, S, f0, fs, fd, FLOOR_W_S);
+}
+
+void Assembler::floor_w_d(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, D, f0, fs, fd, FLOOR_W_D);
+}
+
+void Assembler::ceil_w_s(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, S, f0, fs, fd, CEIL_W_S);
+}
+
+void Assembler::ceil_w_d(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, D, f0, fs, fd, CEIL_W_D);
+}
+
+void Assembler::rint_s(FPURegister fd, FPURegister fs) { rint(S, fd, fs); }
+
+void Assembler::rint(SecondaryField fmt, FPURegister fd, FPURegister fs) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK((fmt == D) || (fmt == S));
+ GenInstrRegister(COP1, fmt, f0, fs, fd, RINT);
+}
+
+void Assembler::rint_d(FPURegister fd, FPURegister fs) { rint(D, fd, fs); }
+
+void Assembler::cvt_l_s(FPURegister fd, FPURegister fs) {
+ DCHECK((IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6)) &&
+ IsFp64Mode());
+ GenInstrRegister(COP1, S, f0, fs, fd, CVT_L_S);
+}
+
+void Assembler::cvt_l_d(FPURegister fd, FPURegister fs) {
+ DCHECK((IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6)) &&
+ IsFp64Mode());
+ GenInstrRegister(COP1, D, f0, fs, fd, CVT_L_D);
+}
+
+void Assembler::trunc_l_s(FPURegister fd, FPURegister fs) {
+ DCHECK((IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6)) &&
+ IsFp64Mode());
+ GenInstrRegister(COP1, S, f0, fs, fd, TRUNC_L_S);
+}
+
+void Assembler::trunc_l_d(FPURegister fd, FPURegister fs) {
+ DCHECK((IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6)) &&
+ IsFp64Mode());
+ GenInstrRegister(COP1, D, f0, fs, fd, TRUNC_L_D);
+}
+
+void Assembler::round_l_s(FPURegister fd, FPURegister fs) {
+ DCHECK((IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6)) &&
+ IsFp64Mode());
+ GenInstrRegister(COP1, S, f0, fs, fd, ROUND_L_S);
+}
+
+void Assembler::round_l_d(FPURegister fd, FPURegister fs) {
+ DCHECK((IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6)) &&
+ IsFp64Mode());
+ GenInstrRegister(COP1, D, f0, fs, fd, ROUND_L_D);
+}
+
+void Assembler::floor_l_s(FPURegister fd, FPURegister fs) {
+ DCHECK((IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6)) &&
+ IsFp64Mode());
+ GenInstrRegister(COP1, S, f0, fs, fd, FLOOR_L_S);
+}
+
+void Assembler::floor_l_d(FPURegister fd, FPURegister fs) {
+ DCHECK((IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6)) &&
+ IsFp64Mode());
+ GenInstrRegister(COP1, D, f0, fs, fd, FLOOR_L_D);
+}
+
+void Assembler::ceil_l_s(FPURegister fd, FPURegister fs) {
+ DCHECK((IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6)) &&
+ IsFp64Mode());
+ GenInstrRegister(COP1, S, f0, fs, fd, CEIL_L_S);
+}
+
+void Assembler::ceil_l_d(FPURegister fd, FPURegister fs) {
+ DCHECK((IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6)) &&
+ IsFp64Mode());
+ GenInstrRegister(COP1, D, f0, fs, fd, CEIL_L_D);
+}
+
+void Assembler::class_s(FPURegister fd, FPURegister fs) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(COP1, S, f0, fs, fd, CLASS_S);
+}
+
+void Assembler::class_d(FPURegister fd, FPURegister fs) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ GenInstrRegister(COP1, D, f0, fs, fd, CLASS_D);
+}
+
+void Assembler::min(SecondaryField fmt, FPURegister fd, FPURegister fs,
+ FPURegister ft) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK((fmt == D) || (fmt == S));
+ GenInstrRegister(COP1, fmt, ft, fs, fd, MIN);
+}
+
+void Assembler::mina(SecondaryField fmt, FPURegister fd, FPURegister fs,
+ FPURegister ft) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK((fmt == D) || (fmt == S));
+ GenInstrRegister(COP1, fmt, ft, fs, fd, MINA);
+}
+
+void Assembler::max(SecondaryField fmt, FPURegister fd, FPURegister fs,
+ FPURegister ft) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK((fmt == D) || (fmt == S));
+ GenInstrRegister(COP1, fmt, ft, fs, fd, MAX);
+}
+
+void Assembler::maxa(SecondaryField fmt, FPURegister fd, FPURegister fs,
+ FPURegister ft) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK((fmt == D) || (fmt == S));
+ GenInstrRegister(COP1, fmt, ft, fs, fd, MAXA);
+}
+
+void Assembler::min_s(FPURegister fd, FPURegister fs, FPURegister ft) {
+ min(S, fd, fs, ft);
+}
+
+void Assembler::min_d(FPURegister fd, FPURegister fs, FPURegister ft) {
+ min(D, fd, fs, ft);
+}
+
+void Assembler::max_s(FPURegister fd, FPURegister fs, FPURegister ft) {
+ max(S, fd, fs, ft);
+}
+
+void Assembler::max_d(FPURegister fd, FPURegister fs, FPURegister ft) {
+ max(D, fd, fs, ft);
+}
+
+void Assembler::mina_s(FPURegister fd, FPURegister fs, FPURegister ft) {
+ mina(S, fd, fs, ft);
+}
+
+void Assembler::mina_d(FPURegister fd, FPURegister fs, FPURegister ft) {
+ mina(D, fd, fs, ft);
+}
+
+void Assembler::maxa_s(FPURegister fd, FPURegister fs, FPURegister ft) {
+ maxa(S, fd, fs, ft);
+}
+
+void Assembler::maxa_d(FPURegister fd, FPURegister fs, FPURegister ft) {
+ maxa(D, fd, fs, ft);
+}
+
+void Assembler::cvt_s_w(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, W, f0, fs, fd, CVT_S_W);
+}
+
+void Assembler::cvt_s_l(FPURegister fd, FPURegister fs) {
+ DCHECK((IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6)) &&
+ IsFp64Mode());
+ GenInstrRegister(COP1, L, f0, fs, fd, CVT_S_L);
+}
+
+void Assembler::cvt_s_d(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, D, f0, fs, fd, CVT_S_D);
+}
+
+void Assembler::cvt_d_w(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, W, f0, fs, fd, CVT_D_W);
+}
+
+void Assembler::cvt_d_l(FPURegister fd, FPURegister fs) {
+ DCHECK((IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6)) &&
+ IsFp64Mode());
+ GenInstrRegister(COP1, L, f0, fs, fd, CVT_D_L);
+}
+
+void Assembler::cvt_d_s(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, S, f0, fs, fd, CVT_D_S);
+}
+
+// Conditions for >= MIPSr6.
+void Assembler::cmp(FPUCondition cond, SecondaryField fmt, FPURegister fd,
+ FPURegister fs, FPURegister ft) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ DCHECK_EQ(fmt & ~(31 << kRsShift), 0);
+ Instr instr = COP1 | fmt | ft.code() << kFtShift | fs.code() << kFsShift |
+ fd.code() << kFdShift | (0 << 5) | cond;
+ emit(instr);
+}
+
+void Assembler::cmp_s(FPUCondition cond, FPURegister fd, FPURegister fs,
+ FPURegister ft) {
+ cmp(cond, W, fd, fs, ft);
+}
+
+void Assembler::cmp_d(FPUCondition cond, FPURegister fd, FPURegister fs,
+ FPURegister ft) {
+ cmp(cond, L, fd, fs, ft);
+}
+
+void Assembler::bc1eqz(int16_t offset, FPURegister ft) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Instr instr = COP1 | BC1EQZ | ft.code() << kFtShift | (offset & kImm16Mask);
+ emit(instr);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+void Assembler::bc1nez(int16_t offset, FPURegister ft) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Instr instr = COP1 | BC1NEZ | ft.code() << kFtShift | (offset & kImm16Mask);
+ emit(instr);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+// Conditions for < MIPSr6.
+void Assembler::c(FPUCondition cond, SecondaryField fmt, FPURegister fs,
+ FPURegister ft, uint16_t cc) {
+ DCHECK(is_uint3(cc));
+ DCHECK(fmt == S || fmt == D);
+ DCHECK_EQ(fmt & ~(31 << kRsShift), 0);
+ Instr instr = COP1 | fmt | ft.code() << 16 | fs.code() << kFsShift | cc << 8 |
+ 3 << 4 | cond;
+ emit(instr);
+}
+
+void Assembler::c_s(FPUCondition cond, FPURegister fs, FPURegister ft,
+ uint16_t cc) {
+ c(cond, S, fs, ft, cc);
+}
+
+void Assembler::c_d(FPUCondition cond, FPURegister fs, FPURegister ft,
+ uint16_t cc) {
+ c(cond, D, fs, ft, cc);
+}
+
+void Assembler::fcmp(FPURegister src1, const double src2, FPUCondition cond) {
+ DCHECK_EQ(src2, 0.0);
+ mtc1(zero_reg, f14);
+ cvt_d_w(f14, f14);
+ c(cond, D, src1, f14, 0);
+}
+
+void Assembler::bc1f(int16_t offset, uint16_t cc) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ DCHECK(is_uint3(cc));
+ Instr instr = COP1 | BC1 | cc << 18 | 0 << 16 | (offset & kImm16Mask);
+ emit(instr);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+void Assembler::bc1t(int16_t offset, uint16_t cc) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ DCHECK(is_uint3(cc));
+ Instr instr = COP1 | BC1 | cc << 18 | 1 << 16 | (offset & kImm16Mask);
+ emit(instr);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+// ---------- MSA instructions ------------
+#define MSA_BRANCH_LIST(V) \
+ V(bz_v, BZ_V) \
+ V(bz_b, BZ_B) \
+ V(bz_h, BZ_H) \
+ V(bz_w, BZ_W) \
+ V(bz_d, BZ_D) \
+ V(bnz_v, BNZ_V) \
+ V(bnz_b, BNZ_B) \
+ V(bnz_h, BNZ_H) \
+ V(bnz_w, BNZ_W) \
+ V(bnz_d, BNZ_D)
+
+#define MSA_BRANCH(name, opcode) \
+ void Assembler::name(MSARegister wt, int16_t offset) { \
+ GenInstrMsaBranch(opcode, wt, offset); \
+ }
+
+MSA_BRANCH_LIST(MSA_BRANCH)
+#undef MSA_BRANCH
+#undef MSA_BRANCH_LIST
+
+#define MSA_LD_ST_LIST(V) \
+ V(ld_b, LD_B) \
+ V(ld_h, LD_H) \
+ V(ld_w, LD_W) \
+ V(ld_d, LD_D) \
+ V(st_b, ST_B) \
+ V(st_h, ST_H) \
+ V(st_w, ST_W) \
+ V(st_d, ST_D)
+
+#define MSA_LD_ST(name, opcode) \
+ void Assembler::name(MSARegister wd, const MemOperand& rs) { \
+ MemOperand source = rs; \
+ AdjustBaseAndOffset(&source); \
+ if (is_int10(source.offset())) { \
+ GenInstrMsaMI10(opcode, source.offset(), source.rm(), wd); \
+ } else { \
+ UseScratchRegisterScope temps(this); \
+ Register scratch = temps.Acquire(); \
+ DCHECK(rs.rm() != scratch); \
+ addiu(scratch, source.rm(), source.offset()); \
+ GenInstrMsaMI10(opcode, 0, scratch, wd); \
+ } \
+ }
+
+MSA_LD_ST_LIST(MSA_LD_ST)
+#undef MSA_LD_ST
+#undef MSA_LD_ST_LIST
+
+#define MSA_I10_LIST(V) \
+ V(ldi_b, I5_DF_b) \
+ V(ldi_h, I5_DF_h) \
+ V(ldi_w, I5_DF_w) \
+ V(ldi_d, I5_DF_d)
+
+#define MSA_I10(name, format) \
+ void Assembler::name(MSARegister wd, int32_t imm10) { \
+ GenInstrMsaI10(LDI, format, imm10, wd); \
+ }
+MSA_I10_LIST(MSA_I10)
+#undef MSA_I10
+#undef MSA_I10_LIST
+
+#define MSA_I5_LIST(V) \
+ V(addvi, ADDVI) \
+ V(subvi, SUBVI) \
+ V(maxi_s, MAXI_S) \
+ V(maxi_u, MAXI_U) \
+ V(mini_s, MINI_S) \
+ V(mini_u, MINI_U) \
+ V(ceqi, CEQI) \
+ V(clti_s, CLTI_S) \
+ V(clti_u, CLTI_U) \
+ V(clei_s, CLEI_S) \
+ V(clei_u, CLEI_U)
+
+#define MSA_I5_FORMAT(name, opcode, format) \
+ void Assembler::name##_##format(MSARegister wd, MSARegister ws, \
+ uint32_t imm5) { \
+ GenInstrMsaI5(opcode, I5_DF_##format, imm5, ws, wd); \
+ }
+
+#define MSA_I5(name, opcode) \
+ MSA_I5_FORMAT(name, opcode, b) \
+ MSA_I5_FORMAT(name, opcode, h) \
+ MSA_I5_FORMAT(name, opcode, w) \
+ MSA_I5_FORMAT(name, opcode, d)
+
+MSA_I5_LIST(MSA_I5)
+#undef MSA_I5
+#undef MSA_I5_FORMAT
+#undef MSA_I5_LIST
+
+#define MSA_I8_LIST(V) \
+ V(andi_b, ANDI_B) \
+ V(ori_b, ORI_B) \
+ V(nori_b, NORI_B) \
+ V(xori_b, XORI_B) \
+ V(bmnzi_b, BMNZI_B) \
+ V(bmzi_b, BMZI_B) \
+ V(bseli_b, BSELI_B) \
+ V(shf_b, SHF_B) \
+ V(shf_h, SHF_H) \
+ V(shf_w, SHF_W)
+
+#define MSA_I8(name, opcode) \
+ void Assembler::name(MSARegister wd, MSARegister ws, uint32_t imm8) { \
+ GenInstrMsaI8(opcode, imm8, ws, wd); \
+ }
+
+MSA_I8_LIST(MSA_I8)
+#undef MSA_I8
+#undef MSA_I8_LIST
+
+#define MSA_VEC_LIST(V) \
+ V(and_v, AND_V) \
+ V(or_v, OR_V) \
+ V(nor_v, NOR_V) \
+ V(xor_v, XOR_V) \
+ V(bmnz_v, BMNZ_V) \
+ V(bmz_v, BMZ_V) \
+ V(bsel_v, BSEL_V)
+
+#define MSA_VEC(name, opcode) \
+ void Assembler::name(MSARegister wd, MSARegister ws, MSARegister wt) { \
+ GenInstrMsaVec(opcode, wt, ws, wd); \
+ }
+
+MSA_VEC_LIST(MSA_VEC)
+#undef MSA_VEC
+#undef MSA_VEC_LIST
+
+#define MSA_2R_LIST(V) \
+ V(pcnt, PCNT) \
+ V(nloc, NLOC) \
+ V(nlzc, NLZC)
+
+#define MSA_2R_FORMAT(name, opcode, format) \
+ void Assembler::name##_##format(MSARegister wd, MSARegister ws) { \
+ GenInstrMsa2R(opcode, MSA_2R_DF_##format, ws, wd); \
+ }
+
+#define MSA_2R(name, opcode) \
+ MSA_2R_FORMAT(name, opcode, b) \
+ MSA_2R_FORMAT(name, opcode, h) \
+ MSA_2R_FORMAT(name, opcode, w) \
+ MSA_2R_FORMAT(name, opcode, d)
+
+MSA_2R_LIST(MSA_2R)
+#undef MSA_2R
+#undef MSA_2R_FORMAT
+#undef MSA_2R_LIST
+
+#define MSA_FILL(format) \
+ void Assembler::fill_##format(MSARegister wd, Register rs) { \
+ DCHECK(IsMipsArchVariant(kMips32r6) && IsEnabled(MIPS_SIMD)); \
+ DCHECK(rs.is_valid() && wd.is_valid()); \
+ Instr instr = MSA | MSA_2R_FORMAT | FILL | MSA_2R_DF_##format | \
+ (rs.code() << kWsShift) | (wd.code() << kWdShift) | \
+ MSA_VEC_2R_2RF_MINOR; \
+ emit(instr); \
+ }
+
+MSA_FILL(b)
+MSA_FILL(h)
+MSA_FILL(w)
+#undef MSA_FILL
+
+#define MSA_2RF_LIST(V) \
+ V(fclass, FCLASS) \
+ V(ftrunc_s, FTRUNC_S) \
+ V(ftrunc_u, FTRUNC_U) \
+ V(fsqrt, FSQRT) \
+ V(frsqrt, FRSQRT) \
+ V(frcp, FRCP) \
+ V(frint, FRINT) \
+ V(flog2, FLOG2) \
+ V(fexupl, FEXUPL) \
+ V(fexupr, FEXUPR) \
+ V(ffql, FFQL) \
+ V(ffqr, FFQR) \
+ V(ftint_s, FTINT_S) \
+ V(ftint_u, FTINT_U) \
+ V(ffint_s, FFINT_S) \
+ V(ffint_u, FFINT_U)
+
+#define MSA_2RF_FORMAT(name, opcode, format) \
+ void Assembler::name##_##format(MSARegister wd, MSARegister ws) { \
+ GenInstrMsa2RF(opcode, MSA_2RF_DF_##format, ws, wd); \
+ }
+
+#define MSA_2RF(name, opcode) \
+ MSA_2RF_FORMAT(name, opcode, w) \
+ MSA_2RF_FORMAT(name, opcode, d)
+
+MSA_2RF_LIST(MSA_2RF)
+#undef MSA_2RF
+#undef MSA_2RF_FORMAT
+#undef MSA_2RF_LIST
+
+#define MSA_3R_LIST(V) \
+ V(sll, SLL_MSA) \
+ V(sra, SRA_MSA) \
+ V(srl, SRL_MSA) \
+ V(bclr, BCLR) \
+ V(bset, BSET) \
+ V(bneg, BNEG) \
+ V(binsl, BINSL) \
+ V(binsr, BINSR) \
+ V(addv, ADDV) \
+ V(subv, SUBV) \
+ V(max_s, MAX_S) \
+ V(max_u, MAX_U) \
+ V(min_s, MIN_S) \
+ V(min_u, MIN_U) \
+ V(max_a, MAX_A) \
+ V(min_a, MIN_A) \
+ V(ceq, CEQ) \
+ V(clt_s, CLT_S) \
+ V(clt_u, CLT_U) \
+ V(cle_s, CLE_S) \
+ V(cle_u, CLE_U) \
+ V(add_a, ADD_A) \
+ V(adds_a, ADDS_A) \
+ V(adds_s, ADDS_S) \
+ V(adds_u, ADDS_U) \
+ V(ave_s, AVE_S) \
+ V(ave_u, AVE_U) \
+ V(aver_s, AVER_S) \
+ V(aver_u, AVER_U) \
+ V(subs_s, SUBS_S) \
+ V(subs_u, SUBS_U) \
+ V(subsus_u, SUBSUS_U) \
+ V(subsuu_s, SUBSUU_S) \
+ V(asub_s, ASUB_S) \
+ V(asub_u, ASUB_U) \
+ V(mulv, MULV) \
+ V(maddv, MADDV) \
+ V(msubv, MSUBV) \
+ V(div_s, DIV_S_MSA) \
+ V(div_u, DIV_U) \
+ V(mod_s, MOD_S) \
+ V(mod_u, MOD_U) \
+ V(dotp_s, DOTP_S) \
+ V(dotp_u, DOTP_U) \
+ V(dpadd_s, DPADD_S) \
+ V(dpadd_u, DPADD_U) \
+ V(dpsub_s, DPSUB_S) \
+ V(dpsub_u, DPSUB_U) \
+ V(pckev, PCKEV) \
+ V(pckod, PCKOD) \
+ V(ilvl, ILVL) \
+ V(ilvr, ILVR) \
+ V(ilvev, ILVEV) \
+ V(ilvod, ILVOD) \
+ V(vshf, VSHF) \
+ V(srar, SRAR) \
+ V(srlr, SRLR) \
+ V(hadd_s, HADD_S) \
+ V(hadd_u, HADD_U) \
+ V(hsub_s, HSUB_S) \
+ V(hsub_u, HSUB_U)
+
+#define MSA_3R_FORMAT(name, opcode, format) \
+ void Assembler::name##_##format(MSARegister wd, MSARegister ws, \
+ MSARegister wt) { \
+ GenInstrMsa3R<MSARegister>(opcode, MSA_3R_DF_##format, wt, ws, wd); \
+ }
+
+#define MSA_3R_FORMAT_SLD_SPLAT(name, opcode, format) \
+ void Assembler::name##_##format(MSARegister wd, MSARegister ws, \
+ Register rt) { \
+ GenInstrMsa3R<Register>(opcode, MSA_3R_DF_##format, rt, ws, wd); \
+ }
+
+#define MSA_3R(name, opcode) \
+ MSA_3R_FORMAT(name, opcode, b) \
+ MSA_3R_FORMAT(name, opcode, h) \
+ MSA_3R_FORMAT(name, opcode, w) \
+ MSA_3R_FORMAT(name, opcode, d)
+
+#define MSA_3R_SLD_SPLAT(name, opcode) \
+ MSA_3R_FORMAT_SLD_SPLAT(name, opcode, b) \
+ MSA_3R_FORMAT_SLD_SPLAT(name, opcode, h) \
+ MSA_3R_FORMAT_SLD_SPLAT(name, opcode, w) \
+ MSA_3R_FORMAT_SLD_SPLAT(name, opcode, d)
+
+MSA_3R_LIST(MSA_3R)
+MSA_3R_SLD_SPLAT(sld, SLD)
+MSA_3R_SLD_SPLAT(splat, SPLAT)
+
+#undef MSA_3R
+#undef MSA_3R_FORMAT
+#undef MSA_3R_FORMAT_SLD_SPLAT
+#undef MSA_3R_SLD_SPLAT
+#undef MSA_3R_LIST
+
+#define MSA_3RF_LIST1(V) \
+ V(fcaf, FCAF) \
+ V(fcun, FCUN) \
+ V(fceq, FCEQ) \
+ V(fcueq, FCUEQ) \
+ V(fclt, FCLT) \
+ V(fcult, FCULT) \
+ V(fcle, FCLE) \
+ V(fcule, FCULE) \
+ V(fsaf, FSAF) \
+ V(fsun, FSUN) \
+ V(fseq, FSEQ) \
+ V(fsueq, FSUEQ) \
+ V(fslt, FSLT) \
+ V(fsult, FSULT) \
+ V(fsle, FSLE) \
+ V(fsule, FSULE) \
+ V(fadd, FADD) \
+ V(fsub, FSUB) \
+ V(fmul, FMUL) \
+ V(fdiv, FDIV) \
+ V(fmadd, FMADD) \
+ V(fmsub, FMSUB) \
+ V(fexp2, FEXP2) \
+ V(fmin, FMIN) \
+ V(fmin_a, FMIN_A) \
+ V(fmax, FMAX) \
+ V(fmax_a, FMAX_A) \
+ V(fcor, FCOR) \
+ V(fcune, FCUNE) \
+ V(fcne, FCNE) \
+ V(fsor, FSOR) \
+ V(fsune, FSUNE) \
+ V(fsne, FSNE)
+
+#define MSA_3RF_LIST2(V) \
+ V(fexdo, FEXDO) \
+ V(ftq, FTQ) \
+ V(mul_q, MUL_Q) \
+ V(madd_q, MADD_Q) \
+ V(msub_q, MSUB_Q) \
+ V(mulr_q, MULR_Q) \
+ V(maddr_q, MADDR_Q) \
+ V(msubr_q, MSUBR_Q)
+
+#define MSA_3RF_FORMAT(name, opcode, df, df_c) \
+ void Assembler::name##_##df(MSARegister wd, MSARegister ws, \
+ MSARegister wt) { \
+ GenInstrMsa3RF(opcode, df_c, wt, ws, wd); \
+ }
+
+#define MSA_3RF_1(name, opcode) \
+ MSA_3RF_FORMAT(name, opcode, w, 0) \
+ MSA_3RF_FORMAT(name, opcode, d, 1)
+
+#define MSA_3RF_2(name, opcode) \
+ MSA_3RF_FORMAT(name, opcode, h, 0) \
+ MSA_3RF_FORMAT(name, opcode, w, 1)
+
+MSA_3RF_LIST1(MSA_3RF_1)
+MSA_3RF_LIST2(MSA_3RF_2)
+#undef MSA_3RF_1
+#undef MSA_3RF_2
+#undef MSA_3RF_FORMAT
+#undef MSA_3RF_LIST1
+#undef MSA_3RF_LIST2
+
+void Assembler::sldi_b(MSARegister wd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<MSARegister, MSARegister>(SLDI, ELM_DF_B, n, ws, wd);
+}
+
+void Assembler::sldi_h(MSARegister wd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<MSARegister, MSARegister>(SLDI, ELM_DF_H, n, ws, wd);
+}
+
+void Assembler::sldi_w(MSARegister wd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<MSARegister, MSARegister>(SLDI, ELM_DF_W, n, ws, wd);
+}
+
+void Assembler::sldi_d(MSARegister wd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<MSARegister, MSARegister>(SLDI, ELM_DF_D, n, ws, wd);
+}
+
+void Assembler::splati_b(MSARegister wd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<MSARegister, MSARegister>(SPLATI, ELM_DF_B, n, ws, wd);
+}
+
+void Assembler::splati_h(MSARegister wd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<MSARegister, MSARegister>(SPLATI, ELM_DF_H, n, ws, wd);
+}
+
+void Assembler::splati_w(MSARegister wd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<MSARegister, MSARegister>(SPLATI, ELM_DF_W, n, ws, wd);
+}
+
+void Assembler::splati_d(MSARegister wd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<MSARegister, MSARegister>(SPLATI, ELM_DF_D, n, ws, wd);
+}
+
+void Assembler::copy_s_b(Register rd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<Register, MSARegister>(COPY_S, ELM_DF_B, n, ws, rd);
+}
+
+void Assembler::copy_s_h(Register rd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<Register, MSARegister>(COPY_S, ELM_DF_H, n, ws, rd);
+}
+
+void Assembler::copy_s_w(Register rd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<Register, MSARegister>(COPY_S, ELM_DF_W, n, ws, rd);
+}
+
+void Assembler::copy_u_b(Register rd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<Register, MSARegister>(COPY_U, ELM_DF_B, n, ws, rd);
+}
+
+void Assembler::copy_u_h(Register rd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<Register, MSARegister>(COPY_U, ELM_DF_H, n, ws, rd);
+}
+
+void Assembler::copy_u_w(Register rd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<Register, MSARegister>(COPY_U, ELM_DF_W, n, ws, rd);
+}
+
+void Assembler::insert_b(MSARegister wd, uint32_t n, Register rs) {
+ GenInstrMsaElm<MSARegister, Register>(INSERT, ELM_DF_B, n, rs, wd);
+}
+
+void Assembler::insert_h(MSARegister wd, uint32_t n, Register rs) {
+ GenInstrMsaElm<MSARegister, Register>(INSERT, ELM_DF_H, n, rs, wd);
+}
+
+void Assembler::insert_w(MSARegister wd, uint32_t n, Register rs) {
+ GenInstrMsaElm<MSARegister, Register>(INSERT, ELM_DF_W, n, rs, wd);
+}
+
+void Assembler::insve_b(MSARegister wd, uint32_t n, MSARegister ws) {
+ GenInstrMsaElm<MSARegister, MSARegister>(INSVE, ELM_DF_B, n, ws, wd);
+}
+
+void Assembler::insve_h(MSARegister wd, uint32_t n, MSARegister ws) {
+ GenInstrMsaElm<MSARegister, MSARegister>(INSVE, ELM_DF_H, n, ws, wd);
+}
+
+void Assembler::insve_w(MSARegister wd, uint32_t n, MSARegister ws) {
+ GenInstrMsaElm<MSARegister, MSARegister>(INSVE, ELM_DF_W, n, ws, wd);
+}
+
+void Assembler::insve_d(MSARegister wd, uint32_t n, MSARegister ws) {
+ GenInstrMsaElm<MSARegister, MSARegister>(INSVE, ELM_DF_D, n, ws, wd);
+}
+
+void Assembler::move_v(MSARegister wd, MSARegister ws) {
+ DCHECK(IsMipsArchVariant(kMips32r6) && IsEnabled(MIPS_SIMD));
+ DCHECK(ws.is_valid() && wd.is_valid());
+ Instr instr = MSA | MOVE_V | (ws.code() << kWsShift) |
+ (wd.code() << kWdShift) | MSA_ELM_MINOR;
+ emit(instr);
+}
+
+void Assembler::ctcmsa(MSAControlRegister cd, Register rs) {
+ DCHECK(IsMipsArchVariant(kMips32r6) && IsEnabled(MIPS_SIMD));
+ DCHECK(cd.is_valid() && rs.is_valid());
+ Instr instr = MSA | CTCMSA | (rs.code() << kWsShift) |
+ (cd.code() << kWdShift) | MSA_ELM_MINOR;
+ emit(instr);
+}
+
+void Assembler::cfcmsa(Register rd, MSAControlRegister cs) {
+ DCHECK(IsMipsArchVariant(kMips32r6) && IsEnabled(MIPS_SIMD));
+ DCHECK(rd.is_valid() && cs.is_valid());
+ Instr instr = MSA | CFCMSA | (cs.code() << kWsShift) |
+ (rd.code() << kWdShift) | MSA_ELM_MINOR;
+ emit(instr);
+}
+
+#define MSA_BIT_LIST(V) \
+ V(slli, SLLI) \
+ V(srai, SRAI) \
+ V(srli, SRLI) \
+ V(bclri, BCLRI) \
+ V(bseti, BSETI) \
+ V(bnegi, BNEGI) \
+ V(binsli, BINSLI) \
+ V(binsri, BINSRI) \
+ V(sat_s, SAT_S) \
+ V(sat_u, SAT_U) \
+ V(srari, SRARI) \
+ V(srlri, SRLRI)
+
+#define MSA_BIT_FORMAT(name, opcode, format) \
+ void Assembler::name##_##format(MSARegister wd, MSARegister ws, \
+ uint32_t m) { \
+ GenInstrMsaBit(opcode, BIT_DF_##format, m, ws, wd); \
+ }
+
+#define MSA_BIT(name, opcode) \
+ MSA_BIT_FORMAT(name, opcode, b) \
+ MSA_BIT_FORMAT(name, opcode, h) \
+ MSA_BIT_FORMAT(name, opcode, w) \
+ MSA_BIT_FORMAT(name, opcode, d)
+
+MSA_BIT_LIST(MSA_BIT)
+#undef MSA_BIT
+#undef MSA_BIT_FORMAT
+#undef MSA_BIT_LIST
+
+int Assembler::RelocateInternalReference(RelocInfo::Mode rmode, Address pc,
+ intptr_t pc_delta) {
+ Instr instr = instr_at(pc);
+
+ if (RelocInfo::IsInternalReference(rmode)) {
+ int32_t* p = reinterpret_cast<int32_t*>(pc);
+ if (*p == 0) {
+ return 0; // Number of instructions patched.
+ }
+ *p += pc_delta;
+ return 1; // Number of instructions patched.
+ } else {
+ DCHECK(RelocInfo::IsInternalReferenceEncoded(rmode));
+ if (IsLui(instr)) {
+ Instr instr1 = instr_at(pc + 0 * kInstrSize);
+ Instr instr2 = instr_at(pc + 1 * kInstrSize);
+ DCHECK(IsOri(instr2) || IsJicOrJialc(instr2));
+ int32_t imm;
+ if (IsJicOrJialc(instr2)) {
+ imm = CreateTargetAddress(instr1, instr2);
+ } else {
+ imm = GetLuiOriImmediate(instr1, instr2);
+ }
+
+ if (imm == kEndOfJumpChain) {
+ return 0; // Number of instructions patched.
+ }
+ imm += pc_delta;
+ DCHECK_EQ(imm & 3, 0);
+ instr1 &= ~kImm16Mask;
+ instr2 &= ~kImm16Mask;
+
+ if (IsJicOrJialc(instr2)) {
+ uint32_t lui_offset_u, jic_offset_u;
+ Assembler::UnpackTargetAddressUnsigned(imm,
+ &lui_offset_u, &jic_offset_u);
+ instr_at_put(pc + 0 * kInstrSize, instr1 | lui_offset_u);
+ instr_at_put(pc + 1 * kInstrSize, instr2 | jic_offset_u);
+ } else {
+ PatchLuiOriImmediate(pc, imm, instr1, 0 * kInstrSize, instr2,
+ 1 * kInstrSize);
+ }
+ return 2; // Number of instructions patched.
+ } else {
+ UNREACHABLE();
+ }
+ }
+}
+
+void Assembler::RelocateRelativeReference(RelocInfo::Mode rmode, Address pc,
+ intptr_t pc_delta) {
+ Instr instr = instr_at(pc);
+
+ DCHECK(RelocInfo::IsRelativeCodeTarget(rmode));
+ if (IsLui(instr)) {
+ Instr instr1 = instr_at(pc + 0 * kInstrSize);
+ Instr instr2 = instr_at(pc + 1 * kInstrSize);
+ Instr instr3 = instr_at(pc + 2 * kInstrSize);
+ int32_t imm;
+ Address ori_offset;
+ if (IsNal(instr2)) {
+ instr2 = instr3;
+ ori_offset = 2 * kInstrSize;
+ } else {
+ ori_offset = 1 * kInstrSize;
+ }
+ DCHECK(IsOri(instr2));
+ imm = GetLuiOriImmediate(instr1, instr2);
+ instr1 &= ~kImm16Mask;
+ instr2 &= ~kImm16Mask;
+
+ if (imm == kEndOfJumpChain) {
+ return;
+ }
+ imm -= pc_delta;
+ DCHECK_EQ(imm & 3, 0);
+ PatchLuiOriImmediate(pc, imm, instr1, 0 * kInstrSize, instr2, ori_offset);
+ return;
+ } else {
+ UNREACHABLE();
+ }
+}
+
+void Assembler::GrowBuffer() {
+ // Compute new buffer size.
+ int old_size = buffer_->size();
+ int new_size = std::min(2 * old_size, old_size + 1 * MB);
+
+ // Some internal data structures overflow for very large buffers,
+ // they must ensure that kMaximalBufferSize is not too large.
+ if (new_size > kMaximalBufferSize) {
+ V8::FatalProcessOutOfMemory(nullptr, "Assembler::GrowBuffer");
+ }
+
+ // Set up new buffer.
+ std::unique_ptr<AssemblerBuffer> new_buffer = buffer_->Grow(new_size);
+ DCHECK_EQ(new_size, new_buffer->size());
+ byte* new_start = new_buffer->start();
+
+ // Copy the data.
+ int pc_delta = new_start - buffer_start_;
+ int rc_delta = (new_start + new_size) - (buffer_start_ + old_size);
+ size_t reloc_size = (buffer_start_ + old_size) - reloc_info_writer.pos();
+ MemMove(new_start, buffer_start_, pc_offset());
+ MemMove(reloc_info_writer.pos() + rc_delta, reloc_info_writer.pos(),
+ reloc_size);
+
+ // Switch buffers.
+ buffer_ = std::move(new_buffer);
+ buffer_start_ = new_start;
+ pc_ += pc_delta;
+ last_call_pc_ += pc_delta;
+ reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta,
+ reloc_info_writer.last_pc() + pc_delta);
+
+ // Relocate runtime entries.
+ Vector<byte> instructions{buffer_start_, pc_offset()};
+ Vector<const byte> reloc_info{reloc_info_writer.pos(), reloc_size};
+ for (RelocIterator it(instructions, reloc_info, 0); !it.done(); it.next()) {
+ RelocInfo::Mode rmode = it.rinfo()->rmode();
+ if (rmode == RelocInfo::INTERNAL_REFERENCE_ENCODED ||
+ rmode == RelocInfo::INTERNAL_REFERENCE) {
+ RelocateInternalReference(rmode, it.rinfo()->pc(), pc_delta);
+ }
+ }
+ DCHECK(!overflow());
+}
+
+void Assembler::db(uint8_t data) {
+ CheckForEmitInForbiddenSlot();
+ *reinterpret_cast<uint8_t*>(pc_) = data;
+ pc_ += sizeof(uint8_t);
+}
+
+void Assembler::dd(uint32_t data) {
+ CheckForEmitInForbiddenSlot();
+ *reinterpret_cast<uint32_t*>(pc_) = data;
+ pc_ += sizeof(uint32_t);
+}
+
+void Assembler::dq(uint64_t data) {
+ CheckForEmitInForbiddenSlot();
+ *reinterpret_cast<uint64_t*>(pc_) = data;
+ pc_ += sizeof(uint64_t);
+}
+
+void Assembler::dd(Label* label) {
+ uint32_t data;
+ CheckForEmitInForbiddenSlot();
+ if (label->is_bound()) {
+ data = reinterpret_cast<uint32_t>(buffer_start_ + label->pos());
+ } else {
+ data = jump_address(label);
+ unbound_labels_count_++;
+ internal_reference_positions_.insert(label->pos());
+ }
+ RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE);
+ EmitHelper(data);
+}
+
+void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) {
+ if (!ShouldRecordRelocInfo(rmode)) return;
+ // We do not try to reuse pool constants.
+ RelocInfo rinfo(reinterpret_cast<Address>(pc_), rmode, data, Code());
+ DCHECK_GE(buffer_space(), kMaxRelocSize); // Too late to grow buffer here.
+ reloc_info_writer.Write(&rinfo);
+}
+
+void Assembler::BlockTrampolinePoolFor(int instructions) {
+ CheckTrampolinePoolQuick(instructions);
+ BlockTrampolinePoolBefore(pc_offset() + instructions * kInstrSize);
+}
+
+void Assembler::CheckTrampolinePool() {
+ // Some small sequences of instructions must not be broken up by the
+ // insertion of a trampoline pool; such sequences are protected by setting
+ // either trampoline_pool_blocked_nesting_ or no_trampoline_pool_before_,
+ // which are both checked here. Also, recursive calls to CheckTrampolinePool
+ // are blocked by trampoline_pool_blocked_nesting_.
+ if ((trampoline_pool_blocked_nesting_ > 0) ||
+ (pc_offset() < no_trampoline_pool_before_)) {
+ // Emission is currently blocked; make sure we try again as soon as
+ // possible.
+ if (trampoline_pool_blocked_nesting_ > 0) {
+ next_buffer_check_ = pc_offset() + kInstrSize;
+ } else {
+ next_buffer_check_ = no_trampoline_pool_before_;
+ }
+ return;
+ }
+
+ DCHECK(!trampoline_emitted_);
+ DCHECK_GE(unbound_labels_count_, 0);
+ if (unbound_labels_count_ > 0) {
+ // First we emit jump (2 instructions), then we emit trampoline pool.
+ {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Label after_pool;
+ if (IsMipsArchVariant(kMips32r6)) {
+ bc(&after_pool);
+ } else {
+ b(&after_pool);
+ }
+ nop();
+
+ int pool_start = pc_offset();
+ for (int i = 0; i < unbound_labels_count_; i++) {
+ {
+ if (IsMipsArchVariant(kMips32r6)) {
+ bc(&after_pool);
+ nop();
+ } else {
+ GenPCRelativeJump(t8, t9, 0, RelocInfo::NONE,
+ BranchDelaySlot::PROTECT);
+ }
+ }
+ }
+ // If unbound_labels_count_ is big enough, label after_pool will
+ // need a trampoline too, so we must create the trampoline before
+ // the bind operation to make sure function 'bind' can get this
+ // information.
+ trampoline_ = Trampoline(pool_start, unbound_labels_count_);
+ bind(&after_pool);
+
+ trampoline_emitted_ = true;
+ // As we are only going to emit trampoline once, we need to prevent any
+ // further emission.
+ next_buffer_check_ = kMaxInt;
+ }
+ } else {
+ // Number of branches to unbound label at this point is zero, so we can
+ // move next buffer check to maximum.
+ next_buffer_check_ =
+ pc_offset() + kMaxBranchOffset - kTrampolineSlotsSize * 16;
+ }
+ return;
+}
+
+Address Assembler::target_address_at(Address pc) {
+ Instr instr1 = instr_at(pc);
+ Instr instr2 = instr_at(pc + kInstrSize);
+ Instr instr3 = instr_at(pc + 2 * kInstrSize);
+ // Interpret 2 instructions generated by li (lui/ori) or optimized pairs
+ // lui/jic, aui/jic or lui/jialc.
+ if (IsLui(instr1)) {
+ if (IsOri(instr2)) {
+ Address target_address;
+ // Assemble the 32 bit value.
+ target_address = GetLuiOriImmediate(instr1, instr2);
+ if (IsAddu(instr3, t9, ra, t9)) {
+ target_address += pc + kRelativeJumpForBuiltinsOffset;
+ }
+ return target_address;
+ } else if (IsJicOrJialc(instr2)) {
+ // Assemble the 32 bit value.
+ return static_cast<Address>(CreateTargetAddress(instr1, instr2));
+ } else if (IsNal(instr2)) {
+ DCHECK(IsOri(instr3));
+ Address target_address;
+ target_address = GetLuiOriImmediate(instr1, instr3);
+ return target_address + pc + kRelativeCallForBuiltinsOffset;
+ }
+ }
+
+ // We should never get here, force a bad address if we do.
+ UNREACHABLE();
+}
+
+// On Mips, a target address is stored in a lui/ori instruction pair, each
+// of which load 16 bits of the 32-bit address to a register.
+// Patching the address must replace both instr, and flush the i-cache.
+// On r6, target address is stored in a lui/jic pair, and both instr have to be
+// patched.
+void Assembler::set_target_value_at(Address pc, uint32_t target,
+ ICacheFlushMode icache_flush_mode) {
+ Instr instr1 = instr_at(pc);
+ Instr instr2 = instr_at(pc + kInstrSize);
+
+#ifdef DEBUG
+ // Check we have the result from a li macro-instruction, using instr pair.
+ DCHECK(IsLui(instr1) &&
+ (IsOri(instr2) || IsJicOrJialc(instr2) || IsNal(instr2)));
+#endif
+
+ if (IsJicOrJialc(instr2)) {
+ // Must use 2 instructions to insure patchable code => use lui and jic
+ uint32_t lui_offset, jic_offset;
+ Assembler::UnpackTargetAddressUnsigned(target, &lui_offset, &jic_offset);
+
+ instr1 &= ~kImm16Mask;
+ instr2 &= ~kImm16Mask;
+
+ instr1 |= lui_offset;
+ instr2 |= jic_offset;
+
+ instr_at_put(pc, instr1);
+ instr_at_put(pc + kInstrSize, instr2);
+ } else {
+ Instr instr3 = instr_at(pc + 2 * kInstrSize);
+ // If we are using relative calls/jumps for builtins.
+ if (IsNal(instr2)) {
+ target -= pc + kRelativeCallForBuiltinsOffset;
+ }
+ if (IsAddu(instr3, t9, ra, t9)) {
+ target -= pc + kRelativeJumpForBuiltinsOffset;
+ }
+ // Must use 2 instructions to insure patchable code => just use lui and ori.
+ // lui rt, upper-16.
+ // ori rt rt, lower-16.
+ if (IsNal(instr2)) {
+ instr1 &= ~kImm16Mask;
+ instr3 &= ~kImm16Mask;
+ PatchLuiOriImmediate(pc, target, instr1, 0 * kInstrSize, instr3,
+ 2 * kInstrSize);
+ } else {
+ instr1 &= ~kImm16Mask;
+ instr2 &= ~kImm16Mask;
+ PatchLuiOriImmediate(pc, target, instr1, 0 * kInstrSize, instr2,
+ 1 * kInstrSize);
+ }
+ }
+
+ if (icache_flush_mode != SKIP_ICACHE_FLUSH) {
+ FlushInstructionCache(pc, 2 * sizeof(int32_t));
+ }
+}
+
+void Assembler::GenPCRelativeJump(Register tf, Register ts, int32_t imm32,
+ RelocInfo::Mode rmode,
+ BranchDelaySlot bdslot) {
+ // Order of these instructions is relied upon when patching them
+ // or when changing imm32 that lui/ori pair loads.
+ or_(tf, ra, zero_reg);
+ nal(); // Relative place of nal instruction determines kLongBranchPCOffset.
+ if (!RelocInfo::IsNone(rmode)) {
+ RecordRelocInfo(rmode);
+ }
+ lui(ts, (imm32 & kHiMask) >> kLuiShift);
+ ori(ts, ts, (imm32 & kImm16Mask));
+ addu(ts, ra, ts);
+ if (bdslot == USE_DELAY_SLOT) {
+ or_(ra, tf, zero_reg);
+ }
+ jr(ts);
+ if (bdslot == PROTECT) {
+ or_(ra, tf, zero_reg);
+ }
+}
+
+void Assembler::GenPCRelativeJumpAndLink(Register t, int32_t imm32,
+ RelocInfo::Mode rmode,
+ BranchDelaySlot bdslot) {
+ if (!RelocInfo::IsNone(rmode)) {
+ RecordRelocInfo(rmode);
+ }
+ // Order of these instructions is relied upon when patching them
+ // or when changing imm32 that lui/ori pair loads.
+ lui(t, (imm32 & kHiMask) >> kLuiShift);
+ nal(); // Relative place of nal instruction determines kLongBranchPCOffset.
+ ori(t, t, (imm32 & kImm16Mask));
+ addu(t, ra, t);
+ jalr(t);
+ if (bdslot == PROTECT) nop();
+ set_last_call_pc_(pc_);
+}
+
+UseScratchRegisterScope::UseScratchRegisterScope(Assembler* assembler)
+ : available_(assembler->GetScratchRegisterList()),
+ old_available_(*available_) {}
+
+UseScratchRegisterScope::~UseScratchRegisterScope() {
+ *available_ = old_available_;
+}
+
+Register UseScratchRegisterScope::Acquire() {
+ DCHECK_NOT_NULL(available_);
+ DCHECK_NE(*available_, 0);
+ int index = static_cast<int>(base::bits::CountTrailingZeros32(*available_));
+ *available_ &= ~(1UL << index);
+
+ return Register::from_code(index);
+}
+
+bool UseScratchRegisterScope::hasAvailable() const { return *available_ != 0; }
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_MIPS
diff --git a/src/codegen/mips/assembler-mips.h b/src/codegen/mips/assembler-mips.h
new file mode 100644
index 0000000..248bd1a
--- /dev/null
+++ b/src/codegen/mips/assembler-mips.h
@@ -0,0 +1,1927 @@
+// Copyright (c) 1994-2006 Sun Microsystems Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// - Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// - Redistribution in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// - Neither the name of Sun Microsystems or the names of contributors may
+// be used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// The original source code covered by the above license above has been
+// modified significantly by Google Inc.
+// Copyright 2012 the V8 project authors. All rights reserved.
+
+#ifndef V8_CODEGEN_MIPS_ASSEMBLER_MIPS_H_
+#define V8_CODEGEN_MIPS_ASSEMBLER_MIPS_H_
+
+#include <stdio.h>
+#include <memory>
+
+#include <set>
+
+#include "src/codegen/assembler.h"
+#include "src/codegen/external-reference.h"
+#include "src/codegen/label.h"
+#include "src/codegen/mips/constants-mips.h"
+#include "src/codegen/mips/register-mips.h"
+#include "src/objects/smi.h"
+
+namespace v8 {
+namespace internal {
+
+class SafepointTableBuilder;
+
+// Allow programmer to use Branch Delay Slot of Branches, Jumps, Calls.
+enum BranchDelaySlot { USE_DELAY_SLOT, PROTECT };
+
+// -----------------------------------------------------------------------------
+// Machine instruction Operands.
+
+// Class Operand represents a shifter operand in data processing instructions.
+class Operand {
+ public:
+ // Immediate.
+ V8_INLINE explicit Operand(int32_t immediate,
+ RelocInfo::Mode rmode = RelocInfo::NONE)
+ : rm_(no_reg), rmode_(rmode) {
+ value_.immediate = immediate;
+ }
+ V8_INLINE explicit Operand(const ExternalReference& f)
+ : rm_(no_reg), rmode_(RelocInfo::EXTERNAL_REFERENCE) {
+ value_.immediate = static_cast<int32_t>(f.address());
+ }
+ V8_INLINE explicit Operand(const char* s);
+ explicit Operand(Handle<HeapObject> handle);
+ V8_INLINE explicit Operand(Smi value) : rm_(no_reg), rmode_(RelocInfo::NONE) {
+ value_.immediate = static_cast<intptr_t>(value.ptr());
+ }
+
+ static Operand EmbeddedNumber(double number); // Smi or HeapNumber.
+ static Operand EmbeddedStringConstant(const StringConstantBase* str);
+
+ // Register.
+ V8_INLINE explicit Operand(Register rm) : rm_(rm) {}
+
+ // Return true if this is a register operand.
+ V8_INLINE bool is_reg() const;
+
+ inline int32_t immediate() const;
+
+ bool IsImmediate() const { return !rm_.is_valid(); }
+
+ HeapObjectRequest heap_object_request() const {
+ DCHECK(IsHeapObjectRequest());
+ return value_.heap_object_request;
+ }
+
+ bool IsHeapObjectRequest() const {
+ DCHECK_IMPLIES(is_heap_object_request_, IsImmediate());
+ DCHECK_IMPLIES(is_heap_object_request_,
+ rmode_ == RelocInfo::FULL_EMBEDDED_OBJECT ||
+ rmode_ == RelocInfo::CODE_TARGET);
+ return is_heap_object_request_;
+ }
+
+ Register rm() const { return rm_; }
+
+ RelocInfo::Mode rmode() const { return rmode_; }
+
+ private:
+ Register rm_;
+ union Value {
+ Value() {}
+ HeapObjectRequest heap_object_request; // if is_heap_object_request_
+ int32_t immediate; // otherwise
+ } value_; // valid if rm_ == no_reg
+ bool is_heap_object_request_ = false;
+ RelocInfo::Mode rmode_;
+
+ friend class Assembler;
+ // friend class MacroAssembler;
+};
+
+// On MIPS we have only one addressing mode with base_reg + offset.
+// Class MemOperand represents a memory operand in load and store instructions.
+class V8_EXPORT_PRIVATE MemOperand : public Operand {
+ public:
+ // Immediate value attached to offset.
+ enum OffsetAddend { offset_minus_one = -1, offset_zero = 0 };
+
+ explicit MemOperand(Register rn, int32_t offset = 0);
+ explicit MemOperand(Register rn, int32_t unit, int32_t multiplier,
+ OffsetAddend offset_addend = offset_zero);
+ int32_t offset() const { return offset_; }
+
+ bool OffsetIsInt16Encodable() const { return is_int16(offset_); }
+
+ private:
+ int32_t offset_;
+
+ friend class Assembler;
+};
+
+class V8_EXPORT_PRIVATE Assembler : public AssemblerBase {
+ public:
+ // Create an assembler. Instructions and relocation information are emitted
+ // into a buffer, with the instructions starting from the beginning and the
+ // relocation information starting from the end of the buffer. See CodeDesc
+ // for a detailed comment on the layout (globals.h).
+ //
+ // If the provided buffer is nullptr, the assembler allocates and grows its
+ // own buffer. Otherwise it takes ownership of the provided buffer.
+ explicit Assembler(const AssemblerOptions&,
+ std::unique_ptr<AssemblerBuffer> = {});
+
+ virtual ~Assembler() {}
+
+ // GetCode emits any pending (non-emitted) code and fills the descriptor desc.
+ static constexpr int kNoHandlerTable = 0;
+ static constexpr SafepointTableBuilder* kNoSafepointTable = nullptr;
+ void GetCode(Isolate* isolate, CodeDesc* desc,
+ SafepointTableBuilder* safepoint_table_builder,
+ int handler_table_offset);
+
+ // Convenience wrapper for code without safepoint or handler tables.
+ void GetCode(Isolate* isolate, CodeDesc* desc) {
+ GetCode(isolate, desc, kNoSafepointTable, kNoHandlerTable);
+ }
+
+ // Unused on this architecture.
+ void MaybeEmitOutOfLineConstantPool() {}
+
+ // Mips uses BlockTrampolinePool to prevent generating trampoline inside a
+ // continuous instruction block. For Call instrution, it prevents generating
+ // trampoline between jalr and delay slot instruction. In the destructor of
+ // BlockTrampolinePool, it must check if it needs to generate trampoline
+ // immediately, if it does not do this, the branch range will go beyond the
+ // max branch offset, that means the pc_offset after call CheckTrampolinePool
+ // may be not the Call instruction's location. So we use last_call_pc here for
+ // safepoint record.
+ int pc_offset_for_safepoint() {
+#ifdef DEBUG
+ Instr instr1 =
+ instr_at(static_cast<int>(last_call_pc_ - buffer_start_ - kInstrSize));
+ Instr instr2 = instr_at(
+ static_cast<int>(last_call_pc_ - buffer_start_ - kInstrSize * 2));
+ if (GetOpcodeField(instr1) != SPECIAL) { // instr1 == jialc.
+ DCHECK(IsMipsArchVariant(kMips32r6) && GetOpcodeField(instr1) == POP76 &&
+ GetRs(instr1) == 0);
+ } else {
+ if (GetFunctionField(instr1) == SLL) { // instr1 == nop, instr2 == jalr.
+ DCHECK(GetOpcodeField(instr2) == SPECIAL &&
+ GetFunctionField(instr2) == JALR);
+ } else { // instr1 == jalr.
+ DCHECK(GetFunctionField(instr1) == JALR);
+ }
+ }
+#endif
+ return static_cast<int>(last_call_pc_ - buffer_start_);
+ }
+
+ // Label operations & relative jumps (PPUM Appendix D).
+ //
+ // Takes a branch opcode (cc) and a label (L) and generates
+ // either a backward branch or a forward branch and links it
+ // to the label fixup chain. Usage:
+ //
+ // Label L; // unbound label
+ // j(cc, &L); // forward branch to unbound label
+ // bind(&L); // bind label to the current pc
+ // j(cc, &L); // backward branch to bound label
+ // bind(&L); // illegal: a label may be bound only once
+ //
+ // Note: The same Label can be used for forward and backward branches
+ // but it may be bound only once.
+ void bind(Label* L); // Binds an unbound label L to current code position.
+
+ enum OffsetSize : int { kOffset26 = 26, kOffset21 = 21, kOffset16 = 16 };
+
+ // Determines if Label is bound and near enough so that branch instruction
+ // can be used to reach it, instead of jump instruction.
+ bool is_near(Label* L);
+ bool is_near(Label* L, OffsetSize bits);
+ bool is_near_branch(Label* L);
+ inline bool is_near_pre_r6(Label* L) {
+ DCHECK(!IsMipsArchVariant(kMips32r6));
+ return pc_offset() - L->pos() < kMaxBranchOffset - 4 * kInstrSize;
+ }
+ inline bool is_near_r6(Label* L) {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ return pc_offset() - L->pos() < kMaxCompactBranchOffset - 4 * kInstrSize;
+ }
+
+ int BranchOffset(Instr instr);
+
+ // Returns the branch offset to the given label from the current code
+ // position. Links the label to the current position if it is still unbound.
+ // Manages the jump elimination optimization if the second parameter is true.
+ int32_t branch_offset_helper(Label* L, OffsetSize bits);
+ inline int32_t branch_offset(Label* L) {
+ return branch_offset_helper(L, OffsetSize::kOffset16);
+ }
+ inline int32_t branch_offset21(Label* L) {
+ return branch_offset_helper(L, OffsetSize::kOffset21);
+ }
+ inline int32_t branch_offset26(Label* L) {
+ return branch_offset_helper(L, OffsetSize::kOffset26);
+ }
+ inline int32_t shifted_branch_offset(Label* L) {
+ return branch_offset(L) >> 2;
+ }
+ inline int32_t shifted_branch_offset21(Label* L) {
+ return branch_offset21(L) >> 2;
+ }
+ inline int32_t shifted_branch_offset26(Label* L) {
+ return branch_offset26(L) >> 2;
+ }
+ uint32_t jump_address(Label* L);
+ uint32_t branch_long_offset(Label* L);
+
+ // Puts a labels target address at the given position.
+ // The high 8 bits are set to zero.
+ void label_at_put(Label* L, int at_offset);
+
+ // Read/Modify the code target address in the branch/call instruction at pc.
+ // The isolate argument is unused (and may be nullptr) when skipping flushing.
+ static Address target_address_at(Address pc);
+ V8_INLINE static void set_target_address_at(
+ Address pc, Address target,
+ ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED) {
+ set_target_value_at(pc, static_cast<uint32_t>(target), icache_flush_mode);
+ }
+ // On MIPS there is no Constant Pool so we skip that parameter.
+ V8_INLINE static Address target_address_at(Address pc,
+ Address constant_pool) {
+ return target_address_at(pc);
+ }
+ V8_INLINE static void set_target_address_at(
+ Address pc, Address constant_pool, Address target,
+ ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED) {
+ set_target_address_at(pc, target, icache_flush_mode);
+ }
+
+ static void set_target_value_at(
+ Address pc, uint32_t target,
+ ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED);
+
+ // This sets the branch destination (which gets loaded at the call address).
+ // This is for calls and branches within generated code. The serializer
+ // has already deserialized the lui/ori instructions etc.
+ inline static void deserialization_set_special_target_at(
+ Address instruction_payload, Code code, Address target);
+
+ // Get the size of the special target encoded at 'instruction_payload'.
+ inline static int deserialization_special_target_size(
+ Address instruction_payload);
+
+ // This sets the internal reference at the pc.
+ inline static void deserialization_set_target_internal_reference_at(
+ Address pc, Address target,
+ RelocInfo::Mode mode = RelocInfo::INTERNAL_REFERENCE);
+
+ // Difference between address of current opcode and target address offset.
+ static constexpr int kBranchPCOffset = kInstrSize;
+
+ // Difference between address of current opcode and target address offset,
+ // when we are generatinga sequence of instructions for long relative PC
+ // branches. It is distance between address of the first instruction in
+ // the jump sequence, and the value that ra gets after calling nal().
+ static constexpr int kLongBranchPCOffset = 3 * kInstrSize;
+
+ // Adjust ra register in branch delay slot of bal instruction in order to skip
+ // instructions not needed after optimization of PIC in
+ // TurboAssembler::BranchAndLink method.
+ static constexpr int kOptimizedBranchAndLinkLongReturnOffset = 3 * kInstrSize;
+
+ // Offset of target relative address in calls/jumps for builtins. It is
+ // distance between instruction that is placed just after calling
+ // RecordRelocInfo, and the value that ra gets aftr calling nal().
+ static constexpr int kRelativeJumpForBuiltinsOffset = 1 * kInstrSize;
+ // Relative target address of jumps for builtins when we use lui, ori, dsll,
+ // ori sequence when loading address that cannot fit into 32 bits.
+ static constexpr int kRelativeCallForBuiltinsOffset = 3 * kInstrSize;
+
+ // Here we are patching the address in the LUI/ORI instruction pair.
+ // These values are used in the serialization process and must be zero for
+ // MIPS platform, as Code, Embedded Object or External-reference pointers
+ // are split across two consecutive instructions and don't exist separately
+ // in the code, so the serializer should not step forwards in memory after
+ // a target is resolved and written.
+
+ static constexpr int kSpecialTargetSize = 0;
+
+ // Number of consecutive instructions used to store 32bit constant. This
+ // constant is used in RelocInfo::target_address_address() function to tell
+ // serializer address of the instruction that follows LUI/ORI instruction
+ // pair.
+ static constexpr int kInstructionsFor32BitConstant = 2;
+
+ // Max offset for instructions with 16-bit offset field
+ static constexpr int kMaxBranchOffset = (1 << (18 - 1)) - 1;
+
+ // Max offset for compact branch instructions with 26-bit offset field
+ static constexpr int kMaxCompactBranchOffset = (1 << (28 - 1)) - 1;
+
+ static constexpr int kTrampolineSlotsSize =
+ IsMipsArchVariant(kMips32r6) ? 2 * kInstrSize : 7 * kInstrSize;
+
+ RegList* GetScratchRegisterList() { return &scratch_register_list_; }
+
+ // ---------------------------------------------------------------------------
+ // Code generation.
+
+ // Insert the smallest number of nop instructions
+ // possible to align the pc offset to a multiple
+ // of m. m must be a power of 2 (>= 4).
+ void Align(int m);
+ // Insert the smallest number of zero bytes possible to align the pc offset
+ // to a mulitple of m. m must be a power of 2 (>= 2).
+ void DataAlign(int m);
+ // Aligns code to something that's optimal for a jump target for the platform.
+ void CodeTargetAlign();
+
+ // Different nop operations are used by the code generator to detect certain
+ // states of the generated code.
+ enum NopMarkerTypes {
+ NON_MARKING_NOP = 0,
+ DEBUG_BREAK_NOP,
+ // IC markers.
+ PROPERTY_ACCESS_INLINED,
+ PROPERTY_ACCESS_INLINED_CONTEXT,
+ PROPERTY_ACCESS_INLINED_CONTEXT_DONT_DELETE,
+ // Helper values.
+ LAST_CODE_MARKER,
+ FIRST_IC_MARKER = PROPERTY_ACCESS_INLINED,
+ };
+
+ // Type == 0 is the default non-marking nop. For mips this is a
+ // sll(zero_reg, zero_reg, 0). We use rt_reg == at for non-zero
+ // marking, to avoid conflict with ssnop and ehb instructions.
+ void nop(unsigned int type = 0) {
+ DCHECK_LT(type, 32);
+ Register nop_rt_reg = (type == 0) ? zero_reg : at;
+ sll(zero_reg, nop_rt_reg, type, true);
+ }
+
+ // --------Branch-and-jump-instructions----------
+ // We don't use likely variant of instructions.
+ void b(int16_t offset);
+ inline void b(Label* L) { b(shifted_branch_offset(L)); }
+ void bal(int16_t offset);
+ inline void bal(Label* L) { bal(shifted_branch_offset(L)); }
+ void bc(int32_t offset);
+ inline void bc(Label* L) { bc(shifted_branch_offset26(L)); }
+ void balc(int32_t offset);
+ inline void balc(Label* L) { balc(shifted_branch_offset26(L)); }
+
+ void beq(Register rs, Register rt, int16_t offset);
+ inline void beq(Register rs, Register rt, Label* L) {
+ beq(rs, rt, shifted_branch_offset(L));
+ }
+ void bgez(Register rs, int16_t offset);
+ void bgezc(Register rt, int16_t offset);
+ inline void bgezc(Register rt, Label* L) {
+ bgezc(rt, shifted_branch_offset(L));
+ }
+ void bgeuc(Register rs, Register rt, int16_t offset);
+ inline void bgeuc(Register rs, Register rt, Label* L) {
+ bgeuc(rs, rt, shifted_branch_offset(L));
+ }
+ void bgec(Register rs, Register rt, int16_t offset);
+ inline void bgec(Register rs, Register rt, Label* L) {
+ bgec(rs, rt, shifted_branch_offset(L));
+ }
+ void bgezal(Register rs, int16_t offset);
+ void bgezalc(Register rt, int16_t offset);
+ inline void bgezalc(Register rt, Label* L) {
+ bgezalc(rt, shifted_branch_offset(L));
+ }
+ void bgezall(Register rs, int16_t offset);
+ inline void bgezall(Register rs, Label* L) {
+ bgezall(rs, branch_offset(L) >> 2);
+ }
+ void bgtz(Register rs, int16_t offset);
+ void bgtzc(Register rt, int16_t offset);
+ inline void bgtzc(Register rt, Label* L) {
+ bgtzc(rt, shifted_branch_offset(L));
+ }
+ void blez(Register rs, int16_t offset);
+ void blezc(Register rt, int16_t offset);
+ inline void blezc(Register rt, Label* L) {
+ blezc(rt, shifted_branch_offset(L));
+ }
+ void bltz(Register rs, int16_t offset);
+ void bltzc(Register rt, int16_t offset);
+ inline void bltzc(Register rt, Label* L) {
+ bltzc(rt, shifted_branch_offset(L));
+ }
+ void bltuc(Register rs, Register rt, int16_t offset);
+ inline void bltuc(Register rs, Register rt, Label* L) {
+ bltuc(rs, rt, shifted_branch_offset(L));
+ }
+ void bltc(Register rs, Register rt, int16_t offset);
+ inline void bltc(Register rs, Register rt, Label* L) {
+ bltc(rs, rt, shifted_branch_offset(L));
+ }
+ void bltzal(Register rs, int16_t offset);
+ void nal() { bltzal(zero_reg, 0); }
+ void blezalc(Register rt, int16_t offset);
+ inline void blezalc(Register rt, Label* L) {
+ blezalc(rt, shifted_branch_offset(L));
+ }
+ void bltzalc(Register rt, int16_t offset);
+ inline void bltzalc(Register rt, Label* L) {
+ bltzalc(rt, shifted_branch_offset(L));
+ }
+ void bgtzalc(Register rt, int16_t offset);
+ inline void bgtzalc(Register rt, Label* L) {
+ bgtzalc(rt, shifted_branch_offset(L));
+ }
+ void beqzalc(Register rt, int16_t offset);
+ inline void beqzalc(Register rt, Label* L) {
+ beqzalc(rt, shifted_branch_offset(L));
+ }
+ void beqc(Register rs, Register rt, int16_t offset);
+ inline void beqc(Register rs, Register rt, Label* L) {
+ beqc(rs, rt, shifted_branch_offset(L));
+ }
+ void beqzc(Register rs, int32_t offset);
+ inline void beqzc(Register rs, Label* L) {
+ beqzc(rs, shifted_branch_offset21(L));
+ }
+ void bnezalc(Register rt, int16_t offset);
+ inline void bnezalc(Register rt, Label* L) {
+ bnezalc(rt, shifted_branch_offset(L));
+ }
+ void bnec(Register rs, Register rt, int16_t offset);
+ inline void bnec(Register rs, Register rt, Label* L) {
+ bnec(rs, rt, shifted_branch_offset(L));
+ }
+ void bnezc(Register rt, int32_t offset);
+ inline void bnezc(Register rt, Label* L) {
+ bnezc(rt, shifted_branch_offset21(L));
+ }
+ void bne(Register rs, Register rt, int16_t offset);
+ inline void bne(Register rs, Register rt, Label* L) {
+ bne(rs, rt, shifted_branch_offset(L));
+ }
+ void bovc(Register rs, Register rt, int16_t offset);
+ inline void bovc(Register rs, Register rt, Label* L) {
+ bovc(rs, rt, shifted_branch_offset(L));
+ }
+ void bnvc(Register rs, Register rt, int16_t offset);
+ inline void bnvc(Register rs, Register rt, Label* L) {
+ bnvc(rs, rt, shifted_branch_offset(L));
+ }
+
+ // Never use the int16_t b(l)cond version with a branch offset
+ // instead of using the Label* version.
+
+ // Jump targets must be in the current 256 MB-aligned region. i.e. 28 bits.
+ void j(int32_t target);
+ void jal(int32_t target);
+ void jalr(Register rs, Register rd = ra);
+ void jr(Register target);
+ void jic(Register rt, int16_t offset);
+ void jialc(Register rt, int16_t offset);
+
+ // -------Data-processing-instructions---------
+
+ // Arithmetic.
+ void addu(Register rd, Register rs, Register rt);
+ void subu(Register rd, Register rs, Register rt);
+ void mult(Register rs, Register rt);
+ void multu(Register rs, Register rt);
+ void div(Register rs, Register rt);
+ void divu(Register rs, Register rt);
+ void div(Register rd, Register rs, Register rt);
+ void divu(Register rd, Register rs, Register rt);
+ void mod(Register rd, Register rs, Register rt);
+ void modu(Register rd, Register rs, Register rt);
+ void mul(Register rd, Register rs, Register rt);
+ void muh(Register rd, Register rs, Register rt);
+ void mulu(Register rd, Register rs, Register rt);
+ void muhu(Register rd, Register rs, Register rt);
+
+ void addiu(Register rd, Register rs, int32_t j);
+
+ // Logical.
+ void and_(Register rd, Register rs, Register rt);
+ void or_(Register rd, Register rs, Register rt);
+ void xor_(Register rd, Register rs, Register rt);
+ void nor(Register rd, Register rs, Register rt);
+
+ void andi(Register rd, Register rs, int32_t j);
+ void ori(Register rd, Register rs, int32_t j);
+ void xori(Register rd, Register rs, int32_t j);
+ void lui(Register rd, int32_t j);
+ void aui(Register rs, Register rt, int32_t j);
+
+ // Shifts.
+ // Please note: sll(zero_reg, zero_reg, x) instructions are reserved as nop
+ // and may cause problems in normal code. coming_from_nop makes sure this
+ // doesn't happen.
+ void sll(Register rd, Register rt, uint16_t sa, bool coming_from_nop = false);
+ void sllv(Register rd, Register rt, Register rs);
+ void srl(Register rd, Register rt, uint16_t sa);
+ void srlv(Register rd, Register rt, Register rs);
+ void sra(Register rt, Register rd, uint16_t sa);
+ void srav(Register rt, Register rd, Register rs);
+ void rotr(Register rd, Register rt, uint16_t sa);
+ void rotrv(Register rd, Register rt, Register rs);
+
+ // ------------Memory-instructions-------------
+
+ void lb(Register rd, const MemOperand& rs);
+ void lbu(Register rd, const MemOperand& rs);
+ void lh(Register rd, const MemOperand& rs);
+ void lhu(Register rd, const MemOperand& rs);
+ void lw(Register rd, const MemOperand& rs);
+ void lwl(Register rd, const MemOperand& rs);
+ void lwr(Register rd, const MemOperand& rs);
+ void sb(Register rd, const MemOperand& rs);
+ void sh(Register rd, const MemOperand& rs);
+ void sw(Register rd, const MemOperand& rs);
+ void swl(Register rd, const MemOperand& rs);
+ void swr(Register rd, const MemOperand& rs);
+
+ // ----------Atomic instructions--------------
+
+ void ll(Register rd, const MemOperand& rs);
+ void sc(Register rd, const MemOperand& rs);
+ void llx(Register rd, const MemOperand& rs);
+ void scx(Register rd, const MemOperand& rs);
+
+ // ---------PC-Relative-instructions-----------
+
+ void addiupc(Register rs, int32_t imm19);
+ void lwpc(Register rs, int32_t offset19);
+ void auipc(Register rs, int16_t imm16);
+ void aluipc(Register rs, int16_t imm16);
+
+ // ----------------Prefetch--------------------
+
+ void pref(int32_t hint, const MemOperand& rs);
+
+ // -------------Misc-instructions--------------
+
+ // Break / Trap instructions.
+ void break_(uint32_t code, bool break_as_stop = false);
+ void stop(uint32_t code = kMaxStopCode);
+ void tge(Register rs, Register rt, uint16_t code);
+ void tgeu(Register rs, Register rt, uint16_t code);
+ void tlt(Register rs, Register rt, uint16_t code);
+ void tltu(Register rs, Register rt, uint16_t code);
+ void teq(Register rs, Register rt, uint16_t code);
+ void tne(Register rs, Register rt, uint16_t code);
+
+ // Memory barrier instruction.
+ void sync();
+
+ // Move from HI/LO register.
+ void mfhi(Register rd);
+ void mflo(Register rd);
+
+ // Set on less than.
+ void slt(Register rd, Register rs, Register rt);
+ void sltu(Register rd, Register rs, Register rt);
+ void slti(Register rd, Register rs, int32_t j);
+ void sltiu(Register rd, Register rs, int32_t j);
+
+ // Conditional move.
+ void movz(Register rd, Register rs, Register rt);
+ void movn(Register rd, Register rs, Register rt);
+ void movt(Register rd, Register rs, uint16_t cc = 0);
+ void movf(Register rd, Register rs, uint16_t cc = 0);
+
+ void sel(SecondaryField fmt, FPURegister fd, FPURegister fs, FPURegister ft);
+ void sel_s(FPURegister fd, FPURegister fs, FPURegister ft);
+ void sel_d(FPURegister fd, FPURegister fs, FPURegister ft);
+ void seleqz(Register rd, Register rs, Register rt);
+ void seleqz(SecondaryField fmt, FPURegister fd, FPURegister fs,
+ FPURegister ft);
+ void selnez(Register rd, Register rs, Register rt);
+ void selnez(SecondaryField fmt, FPURegister fd, FPURegister fs,
+ FPURegister ft);
+ void seleqz_d(FPURegister fd, FPURegister fs, FPURegister ft);
+ void seleqz_s(FPURegister fd, FPURegister fs, FPURegister ft);
+ void selnez_d(FPURegister fd, FPURegister fs, FPURegister ft);
+ void selnez_s(FPURegister fd, FPURegister fs, FPURegister ft);
+
+ void movz_s(FPURegister fd, FPURegister fs, Register rt);
+ void movz_d(FPURegister fd, FPURegister fs, Register rt);
+ void movt_s(FPURegister fd, FPURegister fs, uint16_t cc = 0);
+ void movt_d(FPURegister fd, FPURegister fs, uint16_t cc = 0);
+ void movf_s(FPURegister fd, FPURegister fs, uint16_t cc = 0);
+ void movf_d(FPURegister fd, FPURegister fs, uint16_t cc = 0);
+ void movn_s(FPURegister fd, FPURegister fs, Register rt);
+ void movn_d(FPURegister fd, FPURegister fs, Register rt);
+ // Bit twiddling.
+ void clz(Register rd, Register rs);
+ void ins_(Register rt, Register rs, uint16_t pos, uint16_t size);
+ void ext_(Register rt, Register rs, uint16_t pos, uint16_t size);
+ void bitswap(Register rd, Register rt);
+ void align(Register rd, Register rs, Register rt, uint8_t bp);
+
+ void wsbh(Register rd, Register rt);
+ void seh(Register rd, Register rt);
+ void seb(Register rd, Register rt);
+
+ // --------Coprocessor-instructions----------------
+
+ // Load, store, and move.
+ void lwc1(FPURegister fd, const MemOperand& src);
+ void swc1(FPURegister fs, const MemOperand& dst);
+
+ void mtc1(Register rt, FPURegister fs);
+ void mthc1(Register rt, FPURegister fs);
+
+ void mfc1(Register rt, FPURegister fs);
+ void mfhc1(Register rt, FPURegister fs);
+
+ void ctc1(Register rt, FPUControlRegister fs);
+ void cfc1(Register rt, FPUControlRegister fs);
+
+ // Arithmetic.
+ void add_s(FPURegister fd, FPURegister fs, FPURegister ft);
+ void add_d(FPURegister fd, FPURegister fs, FPURegister ft);
+ void sub_s(FPURegister fd, FPURegister fs, FPURegister ft);
+ void sub_d(FPURegister fd, FPURegister fs, FPURegister ft);
+ void mul_s(FPURegister fd, FPURegister fs, FPURegister ft);
+ void mul_d(FPURegister fd, FPURegister fs, FPURegister ft);
+ void madd_s(FPURegister fd, FPURegister fr, FPURegister fs, FPURegister ft);
+ void madd_d(FPURegister fd, FPURegister fr, FPURegister fs, FPURegister ft);
+ void msub_s(FPURegister fd, FPURegister fr, FPURegister fs, FPURegister ft);
+ void msub_d(FPURegister fd, FPURegister fr, FPURegister fs, FPURegister ft);
+ void maddf_s(FPURegister fd, FPURegister fs, FPURegister ft);
+ void maddf_d(FPURegister fd, FPURegister fs, FPURegister ft);
+ void msubf_s(FPURegister fd, FPURegister fs, FPURegister ft);
+ void msubf_d(FPURegister fd, FPURegister fs, FPURegister ft);
+ void div_s(FPURegister fd, FPURegister fs, FPURegister ft);
+ void div_d(FPURegister fd, FPURegister fs, FPURegister ft);
+ void abs_s(FPURegister fd, FPURegister fs);
+ void abs_d(FPURegister fd, FPURegister fs);
+ void mov_d(FPURegister fd, FPURegister fs);
+ void mov_s(FPURegister fd, FPURegister fs);
+ void neg_s(FPURegister fd, FPURegister fs);
+ void neg_d(FPURegister fd, FPURegister fs);
+ void sqrt_s(FPURegister fd, FPURegister fs);
+ void sqrt_d(FPURegister fd, FPURegister fs);
+ void rsqrt_s(FPURegister fd, FPURegister fs);
+ void rsqrt_d(FPURegister fd, FPURegister fs);
+ void recip_d(FPURegister fd, FPURegister fs);
+ void recip_s(FPURegister fd, FPURegister fs);
+
+ // Conversion.
+ void cvt_w_s(FPURegister fd, FPURegister fs);
+ void cvt_w_d(FPURegister fd, FPURegister fs);
+ void trunc_w_s(FPURegister fd, FPURegister fs);
+ void trunc_w_d(FPURegister fd, FPURegister fs);
+ void round_w_s(FPURegister fd, FPURegister fs);
+ void round_w_d(FPURegister fd, FPURegister fs);
+ void floor_w_s(FPURegister fd, FPURegister fs);
+ void floor_w_d(FPURegister fd, FPURegister fs);
+ void ceil_w_s(FPURegister fd, FPURegister fs);
+ void ceil_w_d(FPURegister fd, FPURegister fs);
+ void rint_s(FPURegister fd, FPURegister fs);
+ void rint_d(FPURegister fd, FPURegister fs);
+ void rint(SecondaryField fmt, FPURegister fd, FPURegister fs);
+
+ void cvt_l_s(FPURegister fd, FPURegister fs);
+ void cvt_l_d(FPURegister fd, FPURegister fs);
+ void trunc_l_s(FPURegister fd, FPURegister fs);
+ void trunc_l_d(FPURegister fd, FPURegister fs);
+ void round_l_s(FPURegister fd, FPURegister fs);
+ void round_l_d(FPURegister fd, FPURegister fs);
+ void floor_l_s(FPURegister fd, FPURegister fs);
+ void floor_l_d(FPURegister fd, FPURegister fs);
+ void ceil_l_s(FPURegister fd, FPURegister fs);
+ void ceil_l_d(FPURegister fd, FPURegister fs);
+
+ void class_s(FPURegister fd, FPURegister fs);
+ void class_d(FPURegister fd, FPURegister fs);
+
+ void min(SecondaryField fmt, FPURegister fd, FPURegister fs, FPURegister ft);
+ void mina(SecondaryField fmt, FPURegister fd, FPURegister fs, FPURegister ft);
+ void max(SecondaryField fmt, FPURegister fd, FPURegister fs, FPURegister ft);
+ void maxa(SecondaryField fmt, FPURegister fd, FPURegister fs, FPURegister ft);
+ void min_s(FPURegister fd, FPURegister fs, FPURegister ft);
+ void min_d(FPURegister fd, FPURegister fs, FPURegister ft);
+ void max_s(FPURegister fd, FPURegister fs, FPURegister ft);
+ void max_d(FPURegister fd, FPURegister fs, FPURegister ft);
+ void mina_s(FPURegister fd, FPURegister fs, FPURegister ft);
+ void mina_d(FPURegister fd, FPURegister fs, FPURegister ft);
+ void maxa_s(FPURegister fd, FPURegister fs, FPURegister ft);
+ void maxa_d(FPURegister fd, FPURegister fs, FPURegister ft);
+
+ void cvt_s_w(FPURegister fd, FPURegister fs);
+ void cvt_s_l(FPURegister fd, FPURegister fs);
+ void cvt_s_d(FPURegister fd, FPURegister fs);
+
+ void cvt_d_w(FPURegister fd, FPURegister fs);
+ void cvt_d_l(FPURegister fd, FPURegister fs);
+ void cvt_d_s(FPURegister fd, FPURegister fs);
+
+ // Conditions and branches for MIPSr6.
+ void cmp(FPUCondition cond, SecondaryField fmt, FPURegister fd,
+ FPURegister ft, FPURegister fs);
+ void cmp_s(FPUCondition cond, FPURegister fd, FPURegister fs, FPURegister ft);
+ void cmp_d(FPUCondition cond, FPURegister fd, FPURegister fs, FPURegister ft);
+
+ void bc1eqz(int16_t offset, FPURegister ft);
+ inline void bc1eqz(Label* L, FPURegister ft) {
+ bc1eqz(shifted_branch_offset(L), ft);
+ }
+ void bc1nez(int16_t offset, FPURegister ft);
+ inline void bc1nez(Label* L, FPURegister ft) {
+ bc1nez(shifted_branch_offset(L), ft);
+ }
+
+ // Conditions and branches for non MIPSr6.
+ void c(FPUCondition cond, SecondaryField fmt, FPURegister ft, FPURegister fs,
+ uint16_t cc = 0);
+ void c_s(FPUCondition cond, FPURegister ft, FPURegister fs, uint16_t cc = 0);
+ void c_d(FPUCondition cond, FPURegister ft, FPURegister fs, uint16_t cc = 0);
+
+ void bc1f(int16_t offset, uint16_t cc = 0);
+ inline void bc1f(Label* L, uint16_t cc = 0) {
+ bc1f(shifted_branch_offset(L), cc);
+ }
+ void bc1t(int16_t offset, uint16_t cc = 0);
+ inline void bc1t(Label* L, uint16_t cc = 0) {
+ bc1t(shifted_branch_offset(L), cc);
+ }
+ void fcmp(FPURegister src1, const double src2, FPUCondition cond);
+
+ // MSA instructions
+ void bz_v(MSARegister wt, int16_t offset);
+ inline void bz_v(MSARegister wt, Label* L) {
+ bz_v(wt, shifted_branch_offset(L));
+ }
+ void bz_b(MSARegister wt, int16_t offset);
+ inline void bz_b(MSARegister wt, Label* L) {
+ bz_b(wt, shifted_branch_offset(L));
+ }
+ void bz_h(MSARegister wt, int16_t offset);
+ inline void bz_h(MSARegister wt, Label* L) {
+ bz_h(wt, shifted_branch_offset(L));
+ }
+ void bz_w(MSARegister wt, int16_t offset);
+ inline void bz_w(MSARegister wt, Label* L) {
+ bz_w(wt, shifted_branch_offset(L));
+ }
+ void bz_d(MSARegister wt, int16_t offset);
+ inline void bz_d(MSARegister wt, Label* L) {
+ bz_d(wt, shifted_branch_offset(L));
+ }
+ void bnz_v(MSARegister wt, int16_t offset);
+ inline void bnz_v(MSARegister wt, Label* L) {
+ bnz_v(wt, shifted_branch_offset(L));
+ }
+ void bnz_b(MSARegister wt, int16_t offset);
+ inline void bnz_b(MSARegister wt, Label* L) {
+ bnz_b(wt, shifted_branch_offset(L));
+ }
+ void bnz_h(MSARegister wt, int16_t offset);
+ inline void bnz_h(MSARegister wt, Label* L) {
+ bnz_h(wt, shifted_branch_offset(L));
+ }
+ void bnz_w(MSARegister wt, int16_t offset);
+ inline void bnz_w(MSARegister wt, Label* L) {
+ bnz_w(wt, shifted_branch_offset(L));
+ }
+ void bnz_d(MSARegister wt, int16_t offset);
+ inline void bnz_d(MSARegister wt, Label* L) {
+ bnz_d(wt, shifted_branch_offset(L));
+ }
+
+ void ld_b(MSARegister wd, const MemOperand& rs);
+ void ld_h(MSARegister wd, const MemOperand& rs);
+ void ld_w(MSARegister wd, const MemOperand& rs);
+ void ld_d(MSARegister wd, const MemOperand& rs);
+ void st_b(MSARegister wd, const MemOperand& rs);
+ void st_h(MSARegister wd, const MemOperand& rs);
+ void st_w(MSARegister wd, const MemOperand& rs);
+ void st_d(MSARegister wd, const MemOperand& rs);
+
+ void ldi_b(MSARegister wd, int32_t imm10);
+ void ldi_h(MSARegister wd, int32_t imm10);
+ void ldi_w(MSARegister wd, int32_t imm10);
+ void ldi_d(MSARegister wd, int32_t imm10);
+
+ void addvi_b(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void addvi_h(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void addvi_w(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void addvi_d(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void subvi_b(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void subvi_h(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void subvi_w(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void subvi_d(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void maxi_s_b(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void maxi_s_h(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void maxi_s_w(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void maxi_s_d(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void maxi_u_b(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void maxi_u_h(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void maxi_u_w(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void maxi_u_d(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void mini_s_b(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void mini_s_h(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void mini_s_w(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void mini_s_d(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void mini_u_b(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void mini_u_h(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void mini_u_w(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void mini_u_d(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void ceqi_b(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void ceqi_h(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void ceqi_w(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void ceqi_d(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clti_s_b(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clti_s_h(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clti_s_w(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clti_s_d(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clti_u_b(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clti_u_h(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clti_u_w(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clti_u_d(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clei_s_b(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clei_s_h(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clei_s_w(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clei_s_d(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clei_u_b(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clei_u_h(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clei_u_w(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clei_u_d(MSARegister wd, MSARegister ws, uint32_t imm5);
+
+ void andi_b(MSARegister wd, MSARegister ws, uint32_t imm8);
+ void ori_b(MSARegister wd, MSARegister ws, uint32_t imm8);
+ void nori_b(MSARegister wd, MSARegister ws, uint32_t imm8);
+ void xori_b(MSARegister wd, MSARegister ws, uint32_t imm8);
+ void bmnzi_b(MSARegister wd, MSARegister ws, uint32_t imm8);
+ void bmzi_b(MSARegister wd, MSARegister ws, uint32_t imm8);
+ void bseli_b(MSARegister wd, MSARegister ws, uint32_t imm8);
+ void shf_b(MSARegister wd, MSARegister ws, uint32_t imm8);
+ void shf_h(MSARegister wd, MSARegister ws, uint32_t imm8);
+ void shf_w(MSARegister wd, MSARegister ws, uint32_t imm8);
+
+ void and_v(MSARegister wd, MSARegister ws, MSARegister wt);
+ void or_v(MSARegister wd, MSARegister ws, MSARegister wt);
+ void nor_v(MSARegister wd, MSARegister ws, MSARegister wt);
+ void xor_v(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bmnz_v(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bmz_v(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bsel_v(MSARegister wd, MSARegister ws, MSARegister wt);
+
+ void fill_b(MSARegister wd, Register rs);
+ void fill_h(MSARegister wd, Register rs);
+ void fill_w(MSARegister wd, Register rs);
+ void pcnt_b(MSARegister wd, MSARegister ws);
+ void pcnt_h(MSARegister wd, MSARegister ws);
+ void pcnt_w(MSARegister wd, MSARegister ws);
+ void pcnt_d(MSARegister wd, MSARegister ws);
+ void nloc_b(MSARegister wd, MSARegister ws);
+ void nloc_h(MSARegister wd, MSARegister ws);
+ void nloc_w(MSARegister wd, MSARegister ws);
+ void nloc_d(MSARegister wd, MSARegister ws);
+ void nlzc_b(MSARegister wd, MSARegister ws);
+ void nlzc_h(MSARegister wd, MSARegister ws);
+ void nlzc_w(MSARegister wd, MSARegister ws);
+ void nlzc_d(MSARegister wd, MSARegister ws);
+
+ void fclass_w(MSARegister wd, MSARegister ws);
+ void fclass_d(MSARegister wd, MSARegister ws);
+ void ftrunc_s_w(MSARegister wd, MSARegister ws);
+ void ftrunc_s_d(MSARegister wd, MSARegister ws);
+ void ftrunc_u_w(MSARegister wd, MSARegister ws);
+ void ftrunc_u_d(MSARegister wd, MSARegister ws);
+ void fsqrt_w(MSARegister wd, MSARegister ws);
+ void fsqrt_d(MSARegister wd, MSARegister ws);
+ void frsqrt_w(MSARegister wd, MSARegister ws);
+ void frsqrt_d(MSARegister wd, MSARegister ws);
+ void frcp_w(MSARegister wd, MSARegister ws);
+ void frcp_d(MSARegister wd, MSARegister ws);
+ void frint_w(MSARegister wd, MSARegister ws);
+ void frint_d(MSARegister wd, MSARegister ws);
+ void flog2_w(MSARegister wd, MSARegister ws);
+ void flog2_d(MSARegister wd, MSARegister ws);
+ void fexupl_w(MSARegister wd, MSARegister ws);
+ void fexupl_d(MSARegister wd, MSARegister ws);
+ void fexupr_w(MSARegister wd, MSARegister ws);
+ void fexupr_d(MSARegister wd, MSARegister ws);
+ void ffql_w(MSARegister wd, MSARegister ws);
+ void ffql_d(MSARegister wd, MSARegister ws);
+ void ffqr_w(MSARegister wd, MSARegister ws);
+ void ffqr_d(MSARegister wd, MSARegister ws);
+ void ftint_s_w(MSARegister wd, MSARegister ws);
+ void ftint_s_d(MSARegister wd, MSARegister ws);
+ void ftint_u_w(MSARegister wd, MSARegister ws);
+ void ftint_u_d(MSARegister wd, MSARegister ws);
+ void ffint_s_w(MSARegister wd, MSARegister ws);
+ void ffint_s_d(MSARegister wd, MSARegister ws);
+ void ffint_u_w(MSARegister wd, MSARegister ws);
+ void ffint_u_d(MSARegister wd, MSARegister ws);
+
+ void sll_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void sll_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void sll_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void sll_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void sra_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void sra_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void sra_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void sra_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void srl_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void srl_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void srl_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void srl_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bclr_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bclr_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bclr_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bclr_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bset_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bset_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bset_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bset_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bneg_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bneg_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bneg_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bneg_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void binsl_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void binsl_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void binsl_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void binsl_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void binsr_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void binsr_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void binsr_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void binsr_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void addv_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void addv_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void addv_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void addv_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subv_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subv_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subv_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subv_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void max_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void max_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void max_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void max_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void max_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void max_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void max_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void max_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void min_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void min_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void min_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void min_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void min_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void min_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void min_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void min_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void max_a_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void max_a_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void max_a_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void max_a_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void min_a_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void min_a_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void min_a_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void min_a_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ceq_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ceq_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ceq_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ceq_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void clt_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void clt_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void clt_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void clt_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void clt_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void clt_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void clt_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void clt_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void cle_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void cle_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void cle_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void cle_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void cle_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void cle_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void cle_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void cle_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void add_a_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void add_a_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void add_a_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void add_a_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void adds_a_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void adds_a_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void adds_a_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void adds_a_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void adds_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void adds_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void adds_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void adds_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void adds_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void adds_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void adds_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void adds_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ave_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ave_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ave_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ave_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ave_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ave_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ave_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ave_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void aver_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void aver_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void aver_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void aver_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void aver_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void aver_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void aver_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void aver_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subs_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subs_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subs_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subs_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subs_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subs_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subs_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subs_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsus_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsus_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsus_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsus_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsus_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsus_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsus_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsus_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsuu_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsuu_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsuu_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsuu_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsuu_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsuu_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsuu_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsuu_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void asub_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void asub_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void asub_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void asub_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void asub_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void asub_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void asub_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void asub_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mulv_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mulv_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mulv_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mulv_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void maddv_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void maddv_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void maddv_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void maddv_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void msubv_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void msubv_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void msubv_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void msubv_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void div_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void div_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void div_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void div_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void div_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void div_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void div_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void div_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mod_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mod_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mod_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mod_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mod_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mod_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mod_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mod_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dotp_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dotp_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dotp_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dotp_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dotp_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dotp_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dotp_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dotp_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpadd_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpadd_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpadd_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpadd_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpadd_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpadd_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpadd_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpadd_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpsub_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpsub_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpsub_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpsub_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpsub_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpsub_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpsub_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpsub_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void sld_b(MSARegister wd, MSARegister ws, Register rt);
+ void sld_h(MSARegister wd, MSARegister ws, Register rt);
+ void sld_w(MSARegister wd, MSARegister ws, Register rt);
+ void sld_d(MSARegister wd, MSARegister ws, Register rt);
+ void splat_b(MSARegister wd, MSARegister ws, Register rt);
+ void splat_h(MSARegister wd, MSARegister ws, Register rt);
+ void splat_w(MSARegister wd, MSARegister ws, Register rt);
+ void splat_d(MSARegister wd, MSARegister ws, Register rt);
+ void pckev_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void pckev_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void pckev_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void pckev_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void pckod_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void pckod_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void pckod_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void pckod_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvl_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvl_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvl_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvl_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvr_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvr_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvr_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvr_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvev_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvev_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvev_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvev_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvod_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvod_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvod_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvod_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void vshf_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void vshf_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void vshf_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void vshf_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void srar_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void srar_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void srar_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void srar_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void srlr_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void srlr_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void srlr_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void srlr_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hadd_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hadd_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hadd_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hadd_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hadd_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hadd_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hadd_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hadd_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hsub_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hsub_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hsub_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hsub_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hsub_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hsub_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hsub_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hsub_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+
+ void fcaf_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcaf_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcun_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcun_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fceq_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fceq_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcueq_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcueq_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fclt_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fclt_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcult_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcult_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcle_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcle_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcule_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcule_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsaf_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsaf_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsun_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsun_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fseq_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fseq_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsueq_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsueq_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fslt_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fslt_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsult_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsult_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsle_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsle_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsule_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsule_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fadd_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fadd_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsub_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsub_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmul_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmul_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fdiv_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fdiv_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmadd_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmadd_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmsub_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmsub_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fexp2_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fexp2_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fexdo_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fexdo_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ftq_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ftq_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmin_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmin_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmin_a_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmin_a_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmax_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmax_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmax_a_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmax_a_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcor_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcor_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcune_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcune_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcne_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcne_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mul_q_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mul_q_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void madd_q_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void madd_q_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void msub_q_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void msub_q_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsor_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsor_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsune_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsune_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsne_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsne_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mulr_q_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mulr_q_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void maddr_q_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void maddr_q_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void msubr_q_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void msubr_q_w(MSARegister wd, MSARegister ws, MSARegister wt);
+
+ void sldi_b(MSARegister wd, MSARegister ws, uint32_t n);
+ void sldi_h(MSARegister wd, MSARegister ws, uint32_t n);
+ void sldi_w(MSARegister wd, MSARegister ws, uint32_t n);
+ void sldi_d(MSARegister wd, MSARegister ws, uint32_t n);
+ void splati_b(MSARegister wd, MSARegister ws, uint32_t n);
+ void splati_h(MSARegister wd, MSARegister ws, uint32_t n);
+ void splati_w(MSARegister wd, MSARegister ws, uint32_t n);
+ void splati_d(MSARegister wd, MSARegister ws, uint32_t n);
+ void copy_s_b(Register rd, MSARegister ws, uint32_t n);
+ void copy_s_h(Register rd, MSARegister ws, uint32_t n);
+ void copy_s_w(Register rd, MSARegister ws, uint32_t n);
+ void copy_u_b(Register rd, MSARegister ws, uint32_t n);
+ void copy_u_h(Register rd, MSARegister ws, uint32_t n);
+ void copy_u_w(Register rd, MSARegister ws, uint32_t n);
+ void insert_b(MSARegister wd, uint32_t n, Register rs);
+ void insert_h(MSARegister wd, uint32_t n, Register rs);
+ void insert_w(MSARegister wd, uint32_t n, Register rs);
+ void insve_b(MSARegister wd, uint32_t n, MSARegister ws);
+ void insve_h(MSARegister wd, uint32_t n, MSARegister ws);
+ void insve_w(MSARegister wd, uint32_t n, MSARegister ws);
+ void insve_d(MSARegister wd, uint32_t n, MSARegister ws);
+ void move_v(MSARegister wd, MSARegister ws);
+ void ctcmsa(MSAControlRegister cd, Register rs);
+ void cfcmsa(Register rd, MSAControlRegister cs);
+
+ void slli_b(MSARegister wd, MSARegister ws, uint32_t m);
+ void slli_h(MSARegister wd, MSARegister ws, uint32_t m);
+ void slli_w(MSARegister wd, MSARegister ws, uint32_t m);
+ void slli_d(MSARegister wd, MSARegister ws, uint32_t m);
+ void srai_b(MSARegister wd, MSARegister ws, uint32_t m);
+ void srai_h(MSARegister wd, MSARegister ws, uint32_t m);
+ void srai_w(MSARegister wd, MSARegister ws, uint32_t m);
+ void srai_d(MSARegister wd, MSARegister ws, uint32_t m);
+ void srli_b(MSARegister wd, MSARegister ws, uint32_t m);
+ void srli_h(MSARegister wd, MSARegister ws, uint32_t m);
+ void srli_w(MSARegister wd, MSARegister ws, uint32_t m);
+ void srli_d(MSARegister wd, MSARegister ws, uint32_t m);
+ void bclri_b(MSARegister wd, MSARegister ws, uint32_t m);
+ void bclri_h(MSARegister wd, MSARegister ws, uint32_t m);
+ void bclri_w(MSARegister wd, MSARegister ws, uint32_t m);
+ void bclri_d(MSARegister wd, MSARegister ws, uint32_t m);
+ void bseti_b(MSARegister wd, MSARegister ws, uint32_t m);
+ void bseti_h(MSARegister wd, MSARegister ws, uint32_t m);
+ void bseti_w(MSARegister wd, MSARegister ws, uint32_t m);
+ void bseti_d(MSARegister wd, MSARegister ws, uint32_t m);
+ void bnegi_b(MSARegister wd, MSARegister ws, uint32_t m);
+ void bnegi_h(MSARegister wd, MSARegister ws, uint32_t m);
+ void bnegi_w(MSARegister wd, MSARegister ws, uint32_t m);
+ void bnegi_d(MSARegister wd, MSARegister ws, uint32_t m);
+ void binsli_b(MSARegister wd, MSARegister ws, uint32_t m);
+ void binsli_h(MSARegister wd, MSARegister ws, uint32_t m);
+ void binsli_w(MSARegister wd, MSARegister ws, uint32_t m);
+ void binsli_d(MSARegister wd, MSARegister ws, uint32_t m);
+ void binsri_b(MSARegister wd, MSARegister ws, uint32_t m);
+ void binsri_h(MSARegister wd, MSARegister ws, uint32_t m);
+ void binsri_w(MSARegister wd, MSARegister ws, uint32_t m);
+ void binsri_d(MSARegister wd, MSARegister ws, uint32_t m);
+ void sat_s_b(MSARegister wd, MSARegister ws, uint32_t m);
+ void sat_s_h(MSARegister wd, MSARegister ws, uint32_t m);
+ void sat_s_w(MSARegister wd, MSARegister ws, uint32_t m);
+ void sat_s_d(MSARegister wd, MSARegister ws, uint32_t m);
+ void sat_u_b(MSARegister wd, MSARegister ws, uint32_t m);
+ void sat_u_h(MSARegister wd, MSARegister ws, uint32_t m);
+ void sat_u_w(MSARegister wd, MSARegister ws, uint32_t m);
+ void sat_u_d(MSARegister wd, MSARegister ws, uint32_t m);
+ void srari_b(MSARegister wd, MSARegister ws, uint32_t m);
+ void srari_h(MSARegister wd, MSARegister ws, uint32_t m);
+ void srari_w(MSARegister wd, MSARegister ws, uint32_t m);
+ void srari_d(MSARegister wd, MSARegister ws, uint32_t m);
+ void srlri_b(MSARegister wd, MSARegister ws, uint32_t m);
+ void srlri_h(MSARegister wd, MSARegister ws, uint32_t m);
+ void srlri_w(MSARegister wd, MSARegister ws, uint32_t m);
+ void srlri_d(MSARegister wd, MSARegister ws, uint32_t m);
+
+ // Check the code size generated from label to here.
+ int SizeOfCodeGeneratedSince(Label* label) {
+ return pc_offset() - label->pos();
+ }
+
+ // Check the number of instructions generated from label to here.
+ int InstructionsGeneratedSince(Label* label) {
+ return SizeOfCodeGeneratedSince(label) / kInstrSize;
+ }
+
+ // Class for scoping postponing the trampoline pool generation.
+ class BlockTrampolinePoolScope {
+ public:
+ explicit BlockTrampolinePoolScope(Assembler* assem) : assem_(assem) {
+ assem_->StartBlockTrampolinePool();
+ }
+ ~BlockTrampolinePoolScope() { assem_->EndBlockTrampolinePool(); }
+
+ private:
+ Assembler* assem_;
+
+ DISALLOW_IMPLICIT_CONSTRUCTORS(BlockTrampolinePoolScope);
+ };
+
+ // Class for postponing the assembly buffer growth. Typically used for
+ // sequences of instructions that must be emitted as a unit, before
+ // buffer growth (and relocation) can occur.
+ // This blocking scope is not nestable.
+ class BlockGrowBufferScope {
+ public:
+ explicit BlockGrowBufferScope(Assembler* assem) : assem_(assem) {
+ assem_->StartBlockGrowBuffer();
+ }
+ ~BlockGrowBufferScope() { assem_->EndBlockGrowBuffer(); }
+
+ private:
+ Assembler* assem_;
+
+ DISALLOW_IMPLICIT_CONSTRUCTORS(BlockGrowBufferScope);
+ };
+
+ // Record a deoptimization reason that can be used by a log or cpu profiler.
+ // Use --trace-deopt to enable.
+ void RecordDeoptReason(DeoptimizeReason reason, SourcePosition position,
+ int id);
+
+ static int RelocateInternalReference(RelocInfo::Mode rmode, Address pc,
+ intptr_t pc_delta);
+
+ static void RelocateRelativeReference(RelocInfo::Mode rmode, Address pc,
+ intptr_t pc_delta);
+
+ // Writes a single byte or word of data in the code stream. Used for
+ // inline tables, e.g., jump-tables.
+ void db(uint8_t data);
+ void dd(uint32_t data);
+ void dq(uint64_t data);
+ void dp(uintptr_t data) { dd(data); }
+ void dd(Label* label);
+
+ // Postpone the generation of the trampoline pool for the specified number of
+ // instructions.
+ void BlockTrampolinePoolFor(int instructions);
+
+ // Check if there is less than kGap bytes available in the buffer.
+ // If this is the case, we need to grow the buffer before emitting
+ // an instruction or relocation information.
+ inline bool overflow() const { return pc_ >= reloc_info_writer.pos() - kGap; }
+
+ // Get the number of bytes available in the buffer.
+ inline int available_space() const { return reloc_info_writer.pos() - pc_; }
+
+ // Read/patch instructions.
+ static Instr instr_at(Address pc) { return *reinterpret_cast<Instr*>(pc); }
+ static void instr_at_put(Address pc, Instr instr) {
+ *reinterpret_cast<Instr*>(pc) = instr;
+ }
+ Instr instr_at(int pos) {
+ return *reinterpret_cast<Instr*>(buffer_start_ + pos);
+ }
+ void instr_at_put(int pos, Instr instr) {
+ *reinterpret_cast<Instr*>(buffer_start_ + pos) = instr;
+ }
+
+ // Check if an instruction is a branch of some kind.
+ static bool IsBranch(Instr instr);
+ static bool IsMsaBranch(Instr instr);
+ static bool IsBc(Instr instr);
+ static bool IsNal(Instr instr);
+ static bool IsBzc(Instr instr);
+ static bool IsBeq(Instr instr);
+ static bool IsBne(Instr instr);
+ static bool IsBeqzc(Instr instr);
+ static bool IsBnezc(Instr instr);
+ static bool IsBeqc(Instr instr);
+ static bool IsBnec(Instr instr);
+ static bool IsJicOrJialc(Instr instr);
+ static bool IsMov(Instr instr, Register rd, Register rs);
+
+ static bool IsJump(Instr instr);
+ static bool IsJ(Instr instr);
+ static bool IsLui(Instr instr);
+ static bool IsOri(Instr instr);
+ static bool IsAddu(Instr instr, Register rd, Register rs, Register rt);
+
+ static bool IsJal(Instr instr);
+ static bool IsJr(Instr instr);
+ static bool IsJalr(Instr instr);
+
+ static bool IsNop(Instr instr, unsigned int type);
+ static bool IsPop(Instr instr);
+ static bool IsPush(Instr instr);
+ static bool IsLwRegFpOffset(Instr instr);
+ static bool IsSwRegFpOffset(Instr instr);
+ static bool IsLwRegFpNegOffset(Instr instr);
+ static bool IsSwRegFpNegOffset(Instr instr);
+
+ static Register GetRtReg(Instr instr);
+ static Register GetRsReg(Instr instr);
+ static Register GetRdReg(Instr instr);
+
+ static uint32_t GetRt(Instr instr);
+ static uint32_t GetRtField(Instr instr);
+ static uint32_t GetRs(Instr instr);
+ static uint32_t GetRsField(Instr instr);
+ static uint32_t GetRd(Instr instr);
+ static uint32_t GetRdField(Instr instr);
+ static uint32_t GetSa(Instr instr);
+ static uint32_t GetSaField(Instr instr);
+ static uint32_t GetOpcodeField(Instr instr);
+ static uint32_t GetFunction(Instr instr);
+ static uint32_t GetFunctionField(Instr instr);
+ static uint32_t GetImmediate16(Instr instr);
+ static uint32_t GetLabelConst(Instr instr);
+
+ static int32_t GetBranchOffset(Instr instr);
+ static bool IsLw(Instr instr);
+ static int16_t GetLwOffset(Instr instr);
+ static int16_t GetJicOrJialcOffset(Instr instr);
+ static int16_t GetLuiOffset(Instr instr);
+ static Instr SetLwOffset(Instr instr, int16_t offset);
+
+ static bool IsSw(Instr instr);
+ static Instr SetSwOffset(Instr instr, int16_t offset);
+ static bool IsAddImmediate(Instr instr);
+ static Instr SetAddImmediateOffset(Instr instr, int16_t offset);
+ static uint32_t CreateTargetAddress(Instr instr_lui, Instr instr_jic);
+ static void UnpackTargetAddress(uint32_t address, int16_t* lui_offset,
+ int16_t* jic_offset);
+ static void UnpackTargetAddressUnsigned(uint32_t address,
+ uint32_t* lui_offset,
+ uint32_t* jic_offset);
+
+ static bool IsAndImmediate(Instr instr);
+ static bool IsEmittedConstant(Instr instr);
+
+ void CheckTrampolinePool();
+
+ bool IsPrevInstrCompactBranch() { return prev_instr_compact_branch_; }
+ static bool IsCompactBranchSupported() {
+ return IsMipsArchVariant(kMips32r6);
+ }
+
+ // Get the code target object for a pc-relative call or jump.
+ V8_INLINE Handle<Code> relative_code_target_object_handle_at(
+ Address pc_) const;
+
+ inline int UnboundLabelsCount() { return unbound_labels_count_; }
+
+ protected:
+ // Load Scaled Address instruction.
+ void lsa(Register rd, Register rt, Register rs, uint8_t sa);
+
+ // Readable constants for base and offset adjustment helper, these indicate if
+ // aside from offset, another value like offset + 4 should fit into int16.
+ enum class OffsetAccessType : bool {
+ SINGLE_ACCESS = false,
+ TWO_ACCESSES = true
+ };
+
+ // Helper function for memory load/store using base register and offset.
+ void AdjustBaseAndOffset(
+ MemOperand* src,
+ OffsetAccessType access_type = OffsetAccessType::SINGLE_ACCESS,
+ int second_access_add_to_offset = 4);
+
+ int32_t buffer_space() const { return reloc_info_writer.pos() - pc_; }
+
+ // Decode branch instruction at pos and return branch target pos.
+ int target_at(int pos, bool is_internal);
+
+ // Patch branch instruction at pos to branch to given branch target pos.
+ void target_at_put(int pos, int target_pos, bool is_internal);
+
+ // Say if we need to relocate with this mode.
+ bool MustUseReg(RelocInfo::Mode rmode);
+
+ // Record reloc info for current pc_.
+ void RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data = 0);
+
+ // Read 32-bit immediate from lui, ori pair that is used to load immediate.
+ static int32_t GetLuiOriImmediate(Instr instr1, Instr instr2);
+
+ // Block the emission of the trampoline pool before pc_offset.
+ void BlockTrampolinePoolBefore(int pc_offset) {
+ if (no_trampoline_pool_before_ < pc_offset)
+ no_trampoline_pool_before_ = pc_offset;
+ }
+
+ void StartBlockTrampolinePool() { trampoline_pool_blocked_nesting_++; }
+
+ void EndBlockTrampolinePool() {
+ trampoline_pool_blocked_nesting_--;
+ if (trampoline_pool_blocked_nesting_ == 0) {
+ CheckTrampolinePoolQuick(1);
+ }
+ }
+
+ bool is_trampoline_pool_blocked() const {
+ return trampoline_pool_blocked_nesting_ > 0;
+ }
+
+ bool has_exception() const { return internal_trampoline_exception_; }
+
+ bool is_trampoline_emitted() const { return trampoline_emitted_; }
+
+ // Temporarily block automatic assembly buffer growth.
+ void StartBlockGrowBuffer() {
+ DCHECK(!block_buffer_growth_);
+ block_buffer_growth_ = true;
+ }
+
+ void EndBlockGrowBuffer() {
+ DCHECK(block_buffer_growth_);
+ block_buffer_growth_ = false;
+ }
+
+ bool is_buffer_growth_blocked() const { return block_buffer_growth_; }
+
+ void EmitForbiddenSlotInstruction() {
+ if (IsPrevInstrCompactBranch()) {
+ nop();
+ }
+ }
+
+ inline void CheckTrampolinePoolQuick(int extra_instructions = 0) {
+ if (pc_offset() >= next_buffer_check_ - extra_instructions * kInstrSize) {
+ CheckTrampolinePool();
+ }
+ }
+
+ inline void CheckBuffer();
+
+ RegList scratch_register_list_;
+
+ // Generate common instruction sequence.
+ void GenPCRelativeJump(Register tf, Register ts, int32_t imm32,
+ RelocInfo::Mode rmode, BranchDelaySlot bdslot);
+ void GenPCRelativeJumpAndLink(Register t, int32_t imm32,
+ RelocInfo::Mode rmode, BranchDelaySlot bdslot);
+
+ void set_last_call_pc_(byte* pc) { last_call_pc_ = pc; }
+
+ private:
+ // Avoid overflows for displacements etc.
+ static const int kMaximalBufferSize = 512 * MB;
+
+ inline static void set_target_internal_reference_encoded_at(Address pc,
+ Address target);
+
+ // Buffer size and constant pool distance are checked together at regular
+ // intervals of kBufferCheckInterval emitted bytes.
+ static constexpr int kBufferCheckInterval = 1 * KB / 2;
+
+ // Code generation.
+ // The relocation writer's position is at least kGap bytes below the end of
+ // the generated instructions. This is so that multi-instruction sequences do
+ // not have to check for overflow. The same is true for writes of large
+ // relocation info entries.
+ static constexpr int kGap = 32;
+ STATIC_ASSERT(AssemblerBase::kMinimalBufferSize >= 2 * kGap);
+
+ // Repeated checking whether the trampoline pool should be emitted is rather
+ // expensive. By default we only check again once a number of instructions
+ // has been generated.
+ static constexpr int kCheckConstIntervalInst = 32;
+ static constexpr int kCheckConstInterval =
+ kCheckConstIntervalInst * kInstrSize;
+
+ int next_buffer_check_; // pc offset of next buffer check.
+
+ // Emission of the trampoline pool may be blocked in some code sequences.
+ int trampoline_pool_blocked_nesting_; // Block emission if this is not zero.
+ int no_trampoline_pool_before_; // Block emission before this pc offset.
+
+ // Keep track of the last emitted pool to guarantee a maximal distance.
+ int last_trampoline_pool_end_; // pc offset of the end of the last pool.
+
+ // Automatic growth of the assembly buffer may be blocked for some sequences.
+ bool block_buffer_growth_; // Block growth when true.
+
+ // Relocation information generation.
+ // Each relocation is encoded as a variable size value.
+ static constexpr int kMaxRelocSize = RelocInfoWriter::kMaxSize;
+ RelocInfoWriter reloc_info_writer;
+
+ // The bound position, before this we cannot do instruction elimination.
+ int last_bound_pos_;
+
+ // Readable constants for compact branch handling in emit()
+ enum class CompactBranchType : bool { NO = false, COMPACT_BRANCH = true };
+
+ // Code emission.
+ void GrowBuffer();
+ inline void emit(Instr x,
+ CompactBranchType is_compact_branch = CompactBranchType::NO);
+ inline void emit(uint64_t x);
+ inline void CheckForEmitInForbiddenSlot();
+ template <typename T>
+ inline void EmitHelper(T x);
+ inline void EmitHelper(Instr x, CompactBranchType is_compact_branch);
+
+ // Instruction generation.
+ // We have 3 different kind of encoding layout on MIPS.
+ // However due to many different types of objects encoded in the same fields
+ // we have quite a few aliases for each mode.
+ // Using the same structure to refer to Register and FPURegister would spare a
+ // few aliases, but mixing both does not look clean to me.
+ // Anyway we could surely implement this differently.
+
+ void GenInstrRegister(Opcode opcode, Register rs, Register rt, Register rd,
+ uint16_t sa = 0, SecondaryField func = nullptrSF);
+
+ void GenInstrRegister(Opcode opcode, Register rs, Register rt, uint16_t msb,
+ uint16_t lsb, SecondaryField func);
+
+ void GenInstrRegister(Opcode opcode, SecondaryField fmt, FPURegister ft,
+ FPURegister fs, FPURegister fd,
+ SecondaryField func = nullptrSF);
+
+ void GenInstrRegister(Opcode opcode, FPURegister fr, FPURegister ft,
+ FPURegister fs, FPURegister fd,
+ SecondaryField func = nullptrSF);
+
+ void GenInstrRegister(Opcode opcode, SecondaryField fmt, Register rt,
+ FPURegister fs, FPURegister fd,
+ SecondaryField func = nullptrSF);
+
+ void GenInstrRegister(Opcode opcode, SecondaryField fmt, Register rt,
+ FPUControlRegister fs, SecondaryField func = nullptrSF);
+
+ void GenInstrImmediate(
+ Opcode opcode, Register rs, Register rt, int32_t j,
+ CompactBranchType is_compact_branch = CompactBranchType::NO);
+ void GenInstrImmediate(
+ Opcode opcode, Register rs, SecondaryField SF, int32_t j,
+ CompactBranchType is_compact_branch = CompactBranchType::NO);
+ void GenInstrImmediate(
+ Opcode opcode, Register r1, FPURegister r2, int32_t j,
+ CompactBranchType is_compact_branch = CompactBranchType::NO);
+ void GenInstrImmediate(Opcode opcode, Register base, Register rt,
+ int32_t offset9, int bit6, SecondaryField func);
+ void GenInstrImmediate(
+ Opcode opcode, Register rs, int32_t offset21,
+ CompactBranchType is_compact_branch = CompactBranchType::NO);
+ void GenInstrImmediate(Opcode opcode, Register rs, uint32_t offset21);
+ void GenInstrImmediate(
+ Opcode opcode, int32_t offset26,
+ CompactBranchType is_compact_branch = CompactBranchType::NO);
+
+ void GenInstrJump(Opcode opcode, uint32_t address);
+
+ // MSA
+ void GenInstrMsaI8(SecondaryField operation, uint32_t imm8, MSARegister ws,
+ MSARegister wd);
+
+ void GenInstrMsaI5(SecondaryField operation, SecondaryField df, int32_t imm5,
+ MSARegister ws, MSARegister wd);
+
+ void GenInstrMsaBit(SecondaryField operation, SecondaryField df, uint32_t m,
+ MSARegister ws, MSARegister wd);
+
+ void GenInstrMsaI10(SecondaryField operation, SecondaryField df,
+ int32_t imm10, MSARegister wd);
+
+ template <typename RegType>
+ void GenInstrMsa3R(SecondaryField operation, SecondaryField df, RegType t,
+ MSARegister ws, MSARegister wd);
+
+ template <typename DstType, typename SrcType>
+ void GenInstrMsaElm(SecondaryField operation, SecondaryField df, uint32_t n,
+ SrcType src, DstType dst);
+
+ void GenInstrMsa3RF(SecondaryField operation, uint32_t df, MSARegister wt,
+ MSARegister ws, MSARegister wd);
+
+ void GenInstrMsaVec(SecondaryField operation, MSARegister wt, MSARegister ws,
+ MSARegister wd);
+
+ void GenInstrMsaMI10(SecondaryField operation, int32_t s10, Register rs,
+ MSARegister wd);
+
+ void GenInstrMsa2R(SecondaryField operation, SecondaryField df,
+ MSARegister ws, MSARegister wd);
+
+ void GenInstrMsa2RF(SecondaryField operation, SecondaryField df,
+ MSARegister ws, MSARegister wd);
+
+ void GenInstrMsaBranch(SecondaryField operation, MSARegister wt,
+ int32_t offset16);
+
+ inline bool is_valid_msa_df_m(SecondaryField bit_df, uint32_t m) {
+ switch (bit_df) {
+ case BIT_DF_b:
+ return is_uint3(m);
+ case BIT_DF_h:
+ return is_uint4(m);
+ case BIT_DF_w:
+ return is_uint5(m);
+ case BIT_DF_d:
+ return is_uint6(m);
+ default:
+ return false;
+ }
+ }
+
+ inline bool is_valid_msa_df_n(SecondaryField elm_df, uint32_t n) {
+ switch (elm_df) {
+ case ELM_DF_B:
+ return is_uint4(n);
+ case ELM_DF_H:
+ return is_uint3(n);
+ case ELM_DF_W:
+ return is_uint2(n);
+ case ELM_DF_D:
+ return is_uint1(n);
+ default:
+ return false;
+ }
+ }
+
+ // Labels.
+ void print(const Label* L);
+ void bind_to(Label* L, int pos);
+ void next(Label* L, bool is_internal);
+
+ // Patching lui/ori pair which is commonly used for loading constants.
+ static void PatchLuiOriImmediate(Address pc, int32_t imm, Instr instr1,
+ Address offset_lui, Instr instr2,
+ Address offset_ori);
+ void PatchLuiOriImmediate(int pc, int32_t imm, Instr instr1,
+ Address offset_lui, Instr instr2,
+ Address offset_ori);
+
+ // One trampoline consists of:
+ // - space for trampoline slots,
+ // - space for labels.
+ //
+ // Space for trampoline slots is equal to slot_count * 2 * kInstrSize.
+ // Space for trampoline slots precedes space for labels. Each label is of one
+ // instruction size, so total amount for labels is equal to
+ // label_count * kInstrSize.
+ class Trampoline {
+ public:
+ Trampoline() {
+ start_ = 0;
+ next_slot_ = 0;
+ free_slot_count_ = 0;
+ end_ = 0;
+ }
+ Trampoline(int start, int slot_count) {
+ start_ = start;
+ next_slot_ = start;
+ free_slot_count_ = slot_count;
+ end_ = start + slot_count * kTrampolineSlotsSize;
+ }
+ int start() { return start_; }
+ int end() { return end_; }
+ int take_slot() {
+ int trampoline_slot = kInvalidSlotPos;
+ if (free_slot_count_ <= 0) {
+ // We have run out of space on trampolines.
+ // Make sure we fail in debug mode, so we become aware of each case
+ // when this happens.
+ DCHECK(0);
+ // Internal exception will be caught.
+ } else {
+ trampoline_slot = next_slot_;
+ free_slot_count_--;
+ next_slot_ += kTrampolineSlotsSize;
+ }
+ return trampoline_slot;
+ }
+
+ private:
+ int start_;
+ int end_;
+ int next_slot_;
+ int free_slot_count_;
+ };
+
+ int32_t get_trampoline_entry(int32_t pos);
+ int unbound_labels_count_;
+ // If trampoline is emitted, generated code is becoming large. As this is
+ // already a slow case which can possibly break our code generation for the
+ // extreme case, we use this information to trigger different mode of
+ // branch instruction generation, where we use jump instructions rather
+ // than regular branch instructions.
+ bool trampoline_emitted_;
+ static constexpr int kInvalidSlotPos = -1;
+
+ // Internal reference positions, required for unbounded internal reference
+ // labels.
+ std::set<int> internal_reference_positions_;
+ bool is_internal_reference(Label* L) {
+ return internal_reference_positions_.find(L->pos()) !=
+ internal_reference_positions_.end();
+ }
+
+ void EmittedCompactBranchInstruction() { prev_instr_compact_branch_ = true; }
+ void ClearCompactBranchState() { prev_instr_compact_branch_ = false; }
+ bool prev_instr_compact_branch_ = false;
+
+ Trampoline trampoline_;
+ bool internal_trampoline_exception_;
+
+ // Keep track of the last Call's position to ensure that safepoint can get the
+ // correct information even if there is a trampoline immediately after the
+ // Call.
+ byte* last_call_pc_;
+
+ private:
+ void AllocateAndInstallRequestedHeapObjects(Isolate* isolate);
+
+ int WriteCodeComments();
+
+ friend class RegExpMacroAssemblerMIPS;
+ friend class RelocInfo;
+ friend class BlockTrampolinePoolScope;
+ friend class EnsureSpace;
+};
+
+class EnsureSpace {
+ public:
+ explicit inline EnsureSpace(Assembler* assembler);
+};
+
+class V8_EXPORT_PRIVATE UseScratchRegisterScope {
+ public:
+ explicit UseScratchRegisterScope(Assembler* assembler);
+ ~UseScratchRegisterScope();
+
+ Register Acquire();
+ bool hasAvailable() const;
+
+ private:
+ RegList* available_;
+ RegList old_available_;
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_MIPS_ASSEMBLER_MIPS_H_
diff --git a/src/codegen/mips/constants-mips.cc b/src/codegen/mips/constants-mips.cc
new file mode 100644
index 0000000..4411387
--- /dev/null
+++ b/src/codegen/mips/constants-mips.cc
@@ -0,0 +1,144 @@
+// Copyright 2011 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#if V8_TARGET_ARCH_MIPS
+
+#include "src/codegen/mips/constants-mips.h"
+
+namespace v8 {
+namespace internal {
+
+// -----------------------------------------------------------------------------
+// Registers.
+
+// These register names are defined in a way to match the native disassembler
+// formatting. See for example the command "objdump -d <binary file>".
+const char* Registers::names_[kNumSimuRegisters] = {
+ "zero_reg", "at", "v0", "v1", "a0", "a1", "a2", "a3", "t0",
+ "t1", "t2", "t3", "t4", "t5", "t6", "t7", "s0", "s1",
+ "s2", "s3", "s4", "s5", "s6", "s7", "t8", "t9", "k0",
+ "k1", "gp", "sp", "fp", "ra", "LO", "HI", "pc"};
+
+// List of alias names which can be used when referring to MIPS registers.
+const Registers::RegisterAlias Registers::aliases_[] = {
+ {0, "zero"},
+ {23, "cp"},
+ {30, "s8"},
+ {30, "s8_fp"},
+ {kInvalidRegister, nullptr}};
+
+const char* Registers::Name(int reg) {
+ const char* result;
+ if ((0 <= reg) && (reg < kNumSimuRegisters)) {
+ result = names_[reg];
+ } else {
+ result = "noreg";
+ }
+ return result;
+}
+
+int Registers::Number(const char* name) {
+ // Look through the canonical names.
+ for (int i = 0; i < kNumSimuRegisters; i++) {
+ if (strcmp(names_[i], name) == 0) {
+ return i;
+ }
+ }
+
+ // Look through the alias names.
+ int i = 0;
+ while (aliases_[i].reg != kInvalidRegister) {
+ if (strcmp(aliases_[i].name, name) == 0) {
+ return aliases_[i].reg;
+ }
+ i++;
+ }
+
+ // No register with the reguested name found.
+ return kInvalidRegister;
+}
+
+const char* FPURegisters::names_[kNumFPURegisters] = {
+ "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "f9", "f10",
+ "f11", "f12", "f13", "f14", "f15", "f16", "f17", "f18", "f19", "f20", "f21",
+ "f22", "f23", "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31"};
+
+// List of alias names which can be used when referring to MIPS registers.
+const FPURegisters::RegisterAlias FPURegisters::aliases_[] = {
+ {kInvalidRegister, nullptr}};
+
+const char* FPURegisters::Name(int creg) {
+ const char* result;
+ if ((0 <= creg) && (creg < kNumFPURegisters)) {
+ result = names_[creg];
+ } else {
+ result = "nocreg";
+ }
+ return result;
+}
+
+int FPURegisters::Number(const char* name) {
+ // Look through the canonical names.
+ for (int i = 0; i < kNumFPURegisters; i++) {
+ if (strcmp(names_[i], name) == 0) {
+ return i;
+ }
+ }
+
+ // Look through the alias names.
+ int i = 0;
+ while (aliases_[i].creg != kInvalidRegister) {
+ if (strcmp(aliases_[i].name, name) == 0) {
+ return aliases_[i].creg;
+ }
+ i++;
+ }
+
+ // No Cregister with the reguested name found.
+ return kInvalidFPURegister;
+}
+
+const char* MSARegisters::names_[kNumMSARegisters] = {
+ "w0", "w1", "w2", "w3", "w4", "w5", "w6", "w7", "w8", "w9", "w10",
+ "w11", "w12", "w13", "w14", "w15", "w16", "w17", "w18", "w19", "w20", "w21",
+ "w22", "w23", "w24", "w25", "w26", "w27", "w28", "w29", "w30", "w31"};
+
+const MSARegisters::RegisterAlias MSARegisters::aliases_[] = {
+ {kInvalidRegister, nullptr}};
+
+const char* MSARegisters::Name(int creg) {
+ const char* result;
+ if ((0 <= creg) && (creg < kNumMSARegisters)) {
+ result = names_[creg];
+ } else {
+ result = "nocreg";
+ }
+ return result;
+}
+
+int MSARegisters::Number(const char* name) {
+ // Look through the canonical names.
+ for (int i = 0; i < kNumMSARegisters; i++) {
+ if (strcmp(names_[i], name) == 0) {
+ return i;
+ }
+ }
+
+ // Look through the alias names.
+ int i = 0;
+ while (aliases_[i].creg != kInvalidRegister) {
+ if (strcmp(aliases_[i].name, name) == 0) {
+ return aliases_[i].creg;
+ }
+ i++;
+ }
+
+ // No Cregister with the reguested name found.
+ return kInvalidMSARegister;
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_MIPS
diff --git a/src/codegen/mips/constants-mips.h b/src/codegen/mips/constants-mips.h
new file mode 100644
index 0000000..67d1215
--- /dev/null
+++ b/src/codegen/mips/constants-mips.h
@@ -0,0 +1,1904 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_MIPS_CONSTANTS_MIPS_H_
+#define V8_CODEGEN_MIPS_CONSTANTS_MIPS_H_
+#include "src/codegen/cpu-features.h"
+// UNIMPLEMENTED_ macro for MIPS.
+#ifdef DEBUG
+#define UNIMPLEMENTED_MIPS() \
+ v8::internal::PrintF("%s, \tline %d: \tfunction %s not implemented. \n", \
+ __FILE__, __LINE__, __func__)
+#else
+#define UNIMPLEMENTED_MIPS()
+#endif
+
+#define UNSUPPORTED_MIPS() v8::internal::PrintF("Unsupported instruction.\n")
+
+enum ArchVariants {
+ kMips32r1 = v8::internal::MIPSr1,
+ kMips32r2 = v8::internal::MIPSr2,
+ kMips32r6 = v8::internal::MIPSr6,
+ kLoongson
+};
+
+#ifdef _MIPS_ARCH_MIPS32R2
+static const ArchVariants kArchVariant = kMips32r2;
+#elif _MIPS_ARCH_MIPS32R6
+static const ArchVariants kArchVariant = kMips32r6;
+#elif _MIPS_ARCH_LOONGSON
+// The loongson flag refers to the LOONGSON architectures based on MIPS-III,
+// which predates (and is a subset of) the mips32r2 and r1 architectures.
+static const ArchVariants kArchVariant = kLoongson;
+#elif _MIPS_ARCH_MIPS32RX
+// This flags referred to compatibility mode that creates universal code that
+// can run on any MIPS32 architecture revision. The dynamically generated code
+// by v8 is specialized for the MIPS host detected in runtime probing.
+static const ArchVariants kArchVariant = kMips32r1;
+#else
+static const ArchVariants kArchVariant = kMips32r1;
+#endif
+
+enum Endianness { kLittle, kBig };
+
+#if defined(V8_TARGET_LITTLE_ENDIAN)
+static const Endianness kArchEndian = kLittle;
+#elif defined(V8_TARGET_BIG_ENDIAN)
+static const Endianness kArchEndian = kBig;
+#else
+#error Unknown endianness
+#endif
+
+enum FpuMode { kFP32, kFP64, kFPXX };
+
+#if defined(FPU_MODE_FP32)
+static const FpuMode kFpuMode = kFP32;
+#elif defined(FPU_MODE_FP64)
+static const FpuMode kFpuMode = kFP64;
+#elif defined(FPU_MODE_FPXX)
+#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS32R6)
+static const FpuMode kFpuMode = kFPXX;
+#else
+#error "FPXX is supported only on Mips32R2 and Mips32R6"
+#endif
+#else
+static const FpuMode kFpuMode = kFP32;
+#endif
+
+#if defined(__mips_hard_float) && __mips_hard_float != 0
+// Use floating-point coprocessor instructions. This flag is raised when
+// -mhard-float is passed to the compiler.
+const bool IsMipsSoftFloatABI = false;
+#elif defined(__mips_soft_float) && __mips_soft_float != 0
+// This flag is raised when -msoft-float is passed to the compiler.
+// Although FPU is a base requirement for v8, soft-float ABI is used
+// on soft-float systems with FPU kernel emulation.
+const bool IsMipsSoftFloatABI = true;
+#else
+const bool IsMipsSoftFloatABI = true;
+#endif
+
+#if defined(V8_TARGET_LITTLE_ENDIAN)
+const uint32_t kHoleNanUpper32Offset = 4;
+const uint32_t kHoleNanLower32Offset = 0;
+#elif defined(V8_TARGET_BIG_ENDIAN)
+const uint32_t kHoleNanUpper32Offset = 0;
+const uint32_t kHoleNanLower32Offset = 4;
+#else
+#error Unknown endianness
+#endif
+
+constexpr bool IsFp64Mode() { return kFpuMode == kFP64; }
+constexpr bool IsFp32Mode() { return kFpuMode == kFP32; }
+constexpr bool IsFpxxMode() { return kFpuMode == kFPXX; }
+
+#ifndef _MIPS_ARCH_MIPS32RX
+constexpr bool IsMipsArchVariant(const ArchVariants check) {
+ return kArchVariant == check;
+}
+#else
+bool IsMipsArchVariant(const ArchVariants check) {
+ return CpuFeatures::IsSupported(static_cast<CpuFeature>(check));
+}
+#endif
+
+#if defined(V8_TARGET_LITTLE_ENDIAN)
+const uint32_t kMipsLwrOffset = 0;
+const uint32_t kMipsLwlOffset = 3;
+const uint32_t kMipsSwrOffset = 0;
+const uint32_t kMipsSwlOffset = 3;
+#elif defined(V8_TARGET_BIG_ENDIAN)
+const uint32_t kMipsLwrOffset = 3;
+const uint32_t kMipsLwlOffset = 0;
+const uint32_t kMipsSwrOffset = 3;
+const uint32_t kMipsSwlOffset = 0;
+#else
+#error Unknown endianness
+#endif
+
+#if defined(V8_TARGET_LITTLE_ENDIAN)
+const uint32_t kLeastSignificantByteInInt32Offset = 0;
+#elif defined(V8_TARGET_BIG_ENDIAN)
+const uint32_t kLeastSignificantByteInInt32Offset = 3;
+#else
+#error Unknown endianness
+#endif
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+#include <inttypes.h>
+
+// Defines constants and accessor classes to assemble, disassemble and
+// simulate MIPS32 instructions.
+//
+// See: MIPS32 Architecture For Programmers
+// Volume II: The MIPS32 Instruction Set
+// Try www.cs.cornell.edu/courses/cs3410/2008fa/MIPS_Vol2.pdf.
+
+namespace v8 {
+namespace internal {
+
+constexpr size_t kMaxPCRelativeCodeRangeInMB = 4096;
+
+// -----------------------------------------------------------------------------
+// Registers and FPURegisters.
+
+// Number of general purpose registers.
+const int kNumRegisters = 32;
+const int kInvalidRegister = -1;
+
+// Number of registers with HI, LO, and pc.
+const int kNumSimuRegisters = 35;
+
+// In the simulator, the PC register is simulated as the 34th register.
+const int kPCRegister = 34;
+
+// Number coprocessor registers.
+const int kNumFPURegisters = 32;
+const int kInvalidFPURegister = -1;
+
+// Number of MSA registers
+const int kNumMSARegisters = 32;
+const int kInvalidMSARegister = -1;
+
+const int kInvalidMSAControlRegister = -1;
+const int kMSAIRRegister = 0;
+const int kMSACSRRegister = 1;
+const int kMSARegSize = 128;
+const int kMSALanesByte = kMSARegSize / 8;
+const int kMSALanesHalf = kMSARegSize / 16;
+const int kMSALanesWord = kMSARegSize / 32;
+const int kMSALanesDword = kMSARegSize / 64;
+
+// FPU (coprocessor 1) control registers. Currently only FCSR is implemented.
+const int kFCSRRegister = 31;
+const int kInvalidFPUControlRegister = -1;
+const uint32_t kFPUInvalidResult = static_cast<uint32_t>(1u << 31) - 1;
+const int32_t kFPUInvalidResultNegative = static_cast<int32_t>(1u << 31);
+const uint64_t kFPU64InvalidResult =
+ static_cast<uint64_t>(static_cast<uint64_t>(1) << 63) - 1;
+const int64_t kFPU64InvalidResultNegative =
+ static_cast<int64_t>(static_cast<uint64_t>(1) << 63);
+
+// FCSR constants.
+const uint32_t kFCSRInexactFlagBit = 2;
+const uint32_t kFCSRUnderflowFlagBit = 3;
+const uint32_t kFCSROverflowFlagBit = 4;
+const uint32_t kFCSRDivideByZeroFlagBit = 5;
+const uint32_t kFCSRInvalidOpFlagBit = 6;
+const uint32_t kFCSRNaN2008FlagBit = 18;
+
+const uint32_t kFCSRInexactFlagMask = 1 << kFCSRInexactFlagBit;
+const uint32_t kFCSRUnderflowFlagMask = 1 << kFCSRUnderflowFlagBit;
+const uint32_t kFCSROverflowFlagMask = 1 << kFCSROverflowFlagBit;
+const uint32_t kFCSRDivideByZeroFlagMask = 1 << kFCSRDivideByZeroFlagBit;
+const uint32_t kFCSRInvalidOpFlagMask = 1 << kFCSRInvalidOpFlagBit;
+const uint32_t kFCSRNaN2008FlagMask = 1 << kFCSRNaN2008FlagBit;
+
+const uint32_t kFCSRFlagMask =
+ kFCSRInexactFlagMask | kFCSRUnderflowFlagMask | kFCSROverflowFlagMask |
+ kFCSRDivideByZeroFlagMask | kFCSRInvalidOpFlagMask;
+
+const uint32_t kFCSRExceptionFlagMask = kFCSRFlagMask ^ kFCSRInexactFlagMask;
+
+// 'pref' instruction hints
+const int32_t kPrefHintLoad = 0;
+const int32_t kPrefHintStore = 1;
+const int32_t kPrefHintLoadStreamed = 4;
+const int32_t kPrefHintStoreStreamed = 5;
+const int32_t kPrefHintLoadRetained = 6;
+const int32_t kPrefHintStoreRetained = 7;
+const int32_t kPrefHintWritebackInvalidate = 25;
+const int32_t kPrefHintPrepareForStore = 30;
+
+// Actual value of root register is offset from the root array's start
+// to take advantage of negative displacement values.
+// TODO(sigurds): Choose best value.
+constexpr int kRootRegisterBias = 256;
+
+// Helper functions for converting between register numbers and names.
+class Registers {
+ public:
+ // Return the name of the register.
+ static const char* Name(int reg);
+
+ // Lookup the register number for the name provided.
+ static int Number(const char* name);
+
+ struct RegisterAlias {
+ int reg;
+ const char* name;
+ };
+
+ static const int32_t kMaxValue = 0x7fffffff;
+ static const int32_t kMinValue = 0x80000000;
+
+ private:
+ static const char* names_[kNumSimuRegisters];
+ static const RegisterAlias aliases_[];
+};
+
+// Helper functions for converting between register numbers and names.
+class FPURegisters {
+ public:
+ // Return the name of the register.
+ static const char* Name(int reg);
+
+ // Lookup the register number for the name provided.
+ static int Number(const char* name);
+
+ struct RegisterAlias {
+ int creg;
+ const char* name;
+ };
+
+ private:
+ static const char* names_[kNumFPURegisters];
+ static const RegisterAlias aliases_[];
+};
+
+// Helper functions for converting between register numbers and names.
+class MSARegisters {
+ public:
+ // Return the name of the register.
+ static const char* Name(int reg);
+
+ // Lookup the register number for the name provided.
+ static int Number(const char* name);
+
+ struct RegisterAlias {
+ int creg;
+ const char* name;
+ };
+
+ private:
+ static const char* names_[kNumMSARegisters];
+ static const RegisterAlias aliases_[];
+};
+
+// -----------------------------------------------------------------------------
+// Instructions encoding constants.
+
+// On MIPS all instructions are 32 bits.
+using Instr = int32_t;
+
+// Special Software Interrupt codes when used in the presence of the MIPS
+// simulator.
+enum SoftwareInterruptCodes {
+ // Transition to C code.
+ call_rt_redirected = 0xfffff
+};
+
+// On MIPS Simulator breakpoints can have different codes:
+// - Breaks between 0 and kMaxWatchpointCode are treated as simple watchpoints,
+// the simulator will run through them and print the registers.
+// - Breaks between kMaxWatchpointCode and kMaxStopCode are treated as stop()
+// instructions (see Assembler::stop()).
+// - Breaks larger than kMaxStopCode are simple breaks, dropping you into the
+// debugger.
+const uint32_t kMaxWatchpointCode = 31;
+const uint32_t kMaxStopCode = 127;
+STATIC_ASSERT(kMaxWatchpointCode < kMaxStopCode);
+
+// ----- Fields offset and length.
+const int kOpcodeShift = 26;
+const int kOpcodeBits = 6;
+const int kRsShift = 21;
+const int kRsBits = 5;
+const int kRtShift = 16;
+const int kRtBits = 5;
+const int kRdShift = 11;
+const int kRdBits = 5;
+const int kSaShift = 6;
+const int kSaBits = 5;
+const int kLsaSaBits = 2;
+const int kFunctionShift = 0;
+const int kFunctionBits = 6;
+const int kLuiShift = 16;
+const int kBp2Shift = 6;
+const int kBp2Bits = 2;
+const int kBaseShift = 21;
+const int kBaseBits = 5;
+const int kBit6Shift = 6;
+const int kBit6Bits = 1;
+
+const int kImm9Shift = 7;
+const int kImm9Bits = 9;
+const int kImm16Shift = 0;
+const int kImm16Bits = 16;
+const int kImm18Shift = 0;
+const int kImm18Bits = 18;
+const int kImm19Shift = 0;
+const int kImm19Bits = 19;
+const int kImm21Shift = 0;
+const int kImm21Bits = 21;
+const int kImm26Shift = 0;
+const int kImm26Bits = 26;
+const int kImm28Shift = 0;
+const int kImm28Bits = 28;
+const int kImm32Shift = 0;
+const int kImm32Bits = 32;
+const int kMsaImm8Shift = 16;
+const int kMsaImm8Bits = 8;
+const int kMsaImm5Shift = 16;
+const int kMsaImm5Bits = 5;
+const int kMsaImm10Shift = 11;
+const int kMsaImm10Bits = 10;
+const int kMsaImmMI10Shift = 16;
+const int kMsaImmMI10Bits = 10;
+
+// In branches and jumps immediate fields point to words, not bytes,
+// and are therefore shifted by 2.
+const int kImmFieldShift = 2;
+
+const int kFrBits = 5;
+const int kFrShift = 21;
+const int kFsShift = 11;
+const int kFsBits = 5;
+const int kFtShift = 16;
+const int kFtBits = 5;
+const int kFdShift = 6;
+const int kFdBits = 5;
+const int kFCccShift = 8;
+const int kFCccBits = 3;
+const int kFBccShift = 18;
+const int kFBccBits = 3;
+const int kFBtrueShift = 16;
+const int kFBtrueBits = 1;
+const int kWtBits = 5;
+const int kWtShift = 16;
+const int kWsBits = 5;
+const int kWsShift = 11;
+const int kWdBits = 5;
+const int kWdShift = 6;
+
+// ----- Miscellaneous useful masks.
+// Instruction bit masks.
+const int kOpcodeMask = ((1 << kOpcodeBits) - 1) << kOpcodeShift;
+const int kImm9Mask = ((1 << kImm9Bits) - 1) << kImm9Shift;
+const int kImm16Mask = ((1 << kImm16Bits) - 1) << kImm16Shift;
+const int kImm18Mask = ((1 << kImm18Bits) - 1) << kImm18Shift;
+const int kImm19Mask = ((1 << kImm19Bits) - 1) << kImm19Shift;
+const int kImm21Mask = ((1 << kImm21Bits) - 1) << kImm21Shift;
+const int kImm26Mask = ((1 << kImm26Bits) - 1) << kImm26Shift;
+const int kImm28Mask = ((1 << kImm28Bits) - 1) << kImm28Shift;
+const int kImm5Mask = ((1 << 5) - 1);
+const int kImm8Mask = ((1 << 8) - 1);
+const int kImm10Mask = ((1 << 10) - 1);
+const int kMsaI5I10Mask = ((7U << 23) | ((1 << 6) - 1));
+const int kMsaI8Mask = ((3U << 24) | ((1 << 6) - 1));
+const int kMsaI5Mask = ((7U << 23) | ((1 << 6) - 1));
+const int kMsaMI10Mask = (15U << 2);
+const int kMsaBITMask = ((7U << 23) | ((1 << 6) - 1));
+const int kMsaELMMask = (15U << 22);
+const int kMsaLongerELMMask = kMsaELMMask | (63U << 16);
+const int kMsa3RMask = ((7U << 23) | ((1 << 6) - 1));
+const int kMsa3RFMask = ((15U << 22) | ((1 << 6) - 1));
+const int kMsaVECMask = (23U << 21);
+const int kMsa2RMask = (7U << 18);
+const int kMsa2RFMask = (15U << 17);
+const int kRsFieldMask = ((1 << kRsBits) - 1) << kRsShift;
+const int kRtFieldMask = ((1 << kRtBits) - 1) << kRtShift;
+const int kRdFieldMask = ((1 << kRdBits) - 1) << kRdShift;
+const int kSaFieldMask = ((1 << kSaBits) - 1) << kSaShift;
+const int kFunctionFieldMask = ((1 << kFunctionBits) - 1) << kFunctionShift;
+// Misc masks.
+const int kHiMask = 0xffff << 16;
+const int kLoMask = 0xffff;
+const int kSignMask = 0x80000000;
+const int kJumpAddrMask = (1 << (kImm26Bits + kImmFieldShift)) - 1;
+
+// ----- MIPS Opcodes and Function Fields.
+// We use this presentation to stay close to the table representation in
+// MIPS32 Architecture For Programmers, Volume II: The MIPS32 Instruction Set.
+enum Opcode : uint32_t {
+ SPECIAL = 0U << kOpcodeShift,
+ REGIMM = 1U << kOpcodeShift,
+
+ J = ((0U << 3) + 2) << kOpcodeShift,
+ JAL = ((0U << 3) + 3) << kOpcodeShift,
+ BEQ = ((0U << 3) + 4) << kOpcodeShift,
+ BNE = ((0U << 3) + 5) << kOpcodeShift,
+ BLEZ = ((0U << 3) + 6) << kOpcodeShift,
+ BGTZ = ((0U << 3) + 7) << kOpcodeShift,
+
+ ADDI = ((1U << 3) + 0) << kOpcodeShift,
+ ADDIU = ((1U << 3) + 1) << kOpcodeShift,
+ SLTI = ((1U << 3) + 2) << kOpcodeShift,
+ SLTIU = ((1U << 3) + 3) << kOpcodeShift,
+ ANDI = ((1U << 3) + 4) << kOpcodeShift,
+ ORI = ((1U << 3) + 5) << kOpcodeShift,
+ XORI = ((1U << 3) + 6) << kOpcodeShift,
+ LUI = ((1U << 3) + 7) << kOpcodeShift, // LUI/AUI family.
+
+ BEQC = ((2U << 3) + 0) << kOpcodeShift,
+ COP1 = ((2U << 3) + 1) << kOpcodeShift, // Coprocessor 1 class.
+ BEQL = ((2U << 3) + 4) << kOpcodeShift,
+ BNEL = ((2U << 3) + 5) << kOpcodeShift,
+ BLEZL = ((2U << 3) + 6) << kOpcodeShift,
+ BGTZL = ((2U << 3) + 7) << kOpcodeShift,
+
+ DADDI = ((3U << 3) + 0) << kOpcodeShift, // This is also BNEC.
+ SPECIAL2 = ((3U << 3) + 4) << kOpcodeShift,
+ MSA = ((3U << 3) + 6) << kOpcodeShift,
+ SPECIAL3 = ((3U << 3) + 7) << kOpcodeShift,
+
+ LB = ((4U << 3) + 0) << kOpcodeShift,
+ LH = ((4U << 3) + 1) << kOpcodeShift,
+ LWL = ((4U << 3) + 2) << kOpcodeShift,
+ LW = ((4U << 3) + 3) << kOpcodeShift,
+ LBU = ((4U << 3) + 4) << kOpcodeShift,
+ LHU = ((4U << 3) + 5) << kOpcodeShift,
+ LWR = ((4U << 3) + 6) << kOpcodeShift,
+ SB = ((5U << 3) + 0) << kOpcodeShift,
+ SH = ((5U << 3) + 1) << kOpcodeShift,
+ SWL = ((5U << 3) + 2) << kOpcodeShift,
+ SW = ((5U << 3) + 3) << kOpcodeShift,
+ SWR = ((5U << 3) + 6) << kOpcodeShift,
+
+ LL = ((6U << 3) + 0) << kOpcodeShift,
+ LWC1 = ((6U << 3) + 1) << kOpcodeShift,
+ BC = ((6U << 3) + 2) << kOpcodeShift,
+ LDC1 = ((6U << 3) + 5) << kOpcodeShift,
+ POP66 = ((6U << 3) + 6) << kOpcodeShift, // beqzc, jic
+
+ PREF = ((6U << 3) + 3) << kOpcodeShift,
+
+ SC = ((7U << 3) + 0) << kOpcodeShift,
+ SWC1 = ((7U << 3) + 1) << kOpcodeShift,
+ BALC = ((7U << 3) + 2) << kOpcodeShift,
+ PCREL = ((7U << 3) + 3) << kOpcodeShift,
+ SDC1 = ((7U << 3) + 5) << kOpcodeShift,
+ POP76 = ((7U << 3) + 6) << kOpcodeShift, // bnezc, jialc
+
+ COP1X = ((1U << 4) + 3) << kOpcodeShift,
+
+ // New r6 instruction.
+ POP06 = BLEZ, // bgeuc/bleuc, blezalc, bgezalc
+ POP07 = BGTZ, // bltuc/bgtuc, bgtzalc, bltzalc
+ POP10 = ADDI, // beqzalc, bovc, beqc
+ POP26 = BLEZL, // bgezc, blezc, bgec/blec
+ POP27 = BGTZL, // bgtzc, bltzc, bltc/bgtc
+ POP30 = DADDI, // bnezalc, bnvc, bnec
+};
+
+enum SecondaryField : uint32_t {
+ // SPECIAL Encoding of Function Field.
+ SLL = ((0U << 3) + 0),
+ MOVCI = ((0U << 3) + 1),
+ SRL = ((0U << 3) + 2),
+ SRA = ((0U << 3) + 3),
+ SLLV = ((0U << 3) + 4),
+ LSA = ((0U << 3) + 5),
+ SRLV = ((0U << 3) + 6),
+ SRAV = ((0U << 3) + 7),
+
+ JR = ((1U << 3) + 0),
+ JALR = ((1U << 3) + 1),
+ MOVZ = ((1U << 3) + 2),
+ MOVN = ((1U << 3) + 3),
+ BREAK = ((1U << 3) + 5),
+ SYNC = ((1U << 3) + 7),
+
+ MFHI = ((2U << 3) + 0),
+ CLZ_R6 = ((2U << 3) + 0),
+ CLO_R6 = ((2U << 3) + 1),
+ MFLO = ((2U << 3) + 2),
+
+ MULT = ((3U << 3) + 0),
+ MULTU = ((3U << 3) + 1),
+ DIV = ((3U << 3) + 2),
+ DIVU = ((3U << 3) + 3),
+
+ ADD = ((4U << 3) + 0),
+ ADDU = ((4U << 3) + 1),
+ SUB = ((4U << 3) + 2),
+ SUBU = ((4U << 3) + 3),
+ AND = ((4U << 3) + 4),
+ OR = ((4U << 3) + 5),
+ XOR = ((4U << 3) + 6),
+ NOR = ((4U << 3) + 7),
+
+ SLT = ((5U << 3) + 2),
+ SLTU = ((5U << 3) + 3),
+
+ TGE = ((6U << 3) + 0),
+ TGEU = ((6U << 3) + 1),
+ TLT = ((6U << 3) + 2),
+ TLTU = ((6U << 3) + 3),
+ TEQ = ((6U << 3) + 4),
+ SELEQZ_S = ((6U << 3) + 5),
+ TNE = ((6U << 3) + 6),
+ SELNEZ_S = ((6U << 3) + 7),
+
+ // Multiply integers in r6.
+ MUL_MUH = ((3U << 3) + 0), // MUL, MUH.
+ MUL_MUH_U = ((3U << 3) + 1), // MUL_U, MUH_U.
+ RINT = ((3U << 3) + 2),
+
+ MUL_OP = ((0U << 3) + 2),
+ MUH_OP = ((0U << 3) + 3),
+ DIV_OP = ((0U << 3) + 2),
+ MOD_OP = ((0U << 3) + 3),
+
+ DIV_MOD = ((3U << 3) + 2),
+ DIV_MOD_U = ((3U << 3) + 3),
+
+ // SPECIAL2 Encoding of Function Field.
+ MUL = ((0U << 3) + 2),
+ CLZ = ((4U << 3) + 0),
+ CLO = ((4U << 3) + 1),
+
+ // SPECIAL3 Encoding of Function Field.
+ EXT = ((0U << 3) + 0),
+ INS = ((0U << 3) + 4),
+ BSHFL = ((4U << 3) + 0),
+ SC_R6 = ((4U << 3) + 6),
+ LL_R6 = ((6U << 3) + 6),
+
+ // SPECIAL3 Encoding of sa Field.
+ BITSWAP = ((0U << 3) + 0),
+ ALIGN = ((0U << 3) + 2),
+ WSBH = ((0U << 3) + 2),
+ SEB = ((2U << 3) + 0),
+ SEH = ((3U << 3) + 0),
+
+ // REGIMM encoding of rt Field.
+ BLTZ = ((0U << 3) + 0) << 16,
+ BGEZ = ((0U << 3) + 1) << 16,
+ BLTZAL = ((2U << 3) + 0) << 16,
+ BGEZAL = ((2U << 3) + 1) << 16,
+ BGEZALL = ((2U << 3) + 3) << 16,
+
+ // COP1 Encoding of rs Field.
+ MFC1 = ((0U << 3) + 0) << 21,
+ CFC1 = ((0U << 3) + 2) << 21,
+ MFHC1 = ((0U << 3) + 3) << 21,
+ MTC1 = ((0U << 3) + 4) << 21,
+ CTC1 = ((0U << 3) + 6) << 21,
+ MTHC1 = ((0U << 3) + 7) << 21,
+ BC1 = ((1U << 3) + 0) << 21,
+ S = ((2U << 3) + 0) << 21,
+ D = ((2U << 3) + 1) << 21,
+ W = ((2U << 3) + 4) << 21,
+ L = ((2U << 3) + 5) << 21,
+ PS = ((2U << 3) + 6) << 21,
+ // COP1 Encoding of Function Field When rs=S.
+
+ ADD_S = ((0U << 3) + 0),
+ SUB_S = ((0U << 3) + 1),
+ MUL_S = ((0U << 3) + 2),
+ DIV_S = ((0U << 3) + 3),
+ ABS_S = ((0U << 3) + 5),
+ SQRT_S = ((0U << 3) + 4),
+ MOV_S = ((0U << 3) + 6),
+ NEG_S = ((0U << 3) + 7),
+ ROUND_L_S = ((1U << 3) + 0),
+ TRUNC_L_S = ((1U << 3) + 1),
+ CEIL_L_S = ((1U << 3) + 2),
+ FLOOR_L_S = ((1U << 3) + 3),
+ ROUND_W_S = ((1U << 3) + 4),
+ TRUNC_W_S = ((1U << 3) + 5),
+ CEIL_W_S = ((1U << 3) + 6),
+ FLOOR_W_S = ((1U << 3) + 7),
+ RECIP_S = ((2U << 3) + 5),
+ RSQRT_S = ((2U << 3) + 6),
+ MADDF_S = ((3U << 3) + 0),
+ MSUBF_S = ((3U << 3) + 1),
+ CLASS_S = ((3U << 3) + 3),
+ CVT_D_S = ((4U << 3) + 1),
+ CVT_W_S = ((4U << 3) + 4),
+ CVT_L_S = ((4U << 3) + 5),
+ CVT_PS_S = ((4U << 3) + 6),
+
+ // COP1 Encoding of Function Field When rs=D.
+ ADD_D = ((0U << 3) + 0),
+ SUB_D = ((0U << 3) + 1),
+ MUL_D = ((0U << 3) + 2),
+ DIV_D = ((0U << 3) + 3),
+ SQRT_D = ((0U << 3) + 4),
+ ABS_D = ((0U << 3) + 5),
+ MOV_D = ((0U << 3) + 6),
+ NEG_D = ((0U << 3) + 7),
+ ROUND_L_D = ((1U << 3) + 0),
+ TRUNC_L_D = ((1U << 3) + 1),
+ CEIL_L_D = ((1U << 3) + 2),
+ FLOOR_L_D = ((1U << 3) + 3),
+ ROUND_W_D = ((1U << 3) + 4),
+ TRUNC_W_D = ((1U << 3) + 5),
+ CEIL_W_D = ((1U << 3) + 6),
+ FLOOR_W_D = ((1U << 3) + 7),
+ RECIP_D = ((2U << 3) + 5),
+ RSQRT_D = ((2U << 3) + 6),
+ MADDF_D = ((3U << 3) + 0),
+ MSUBF_D = ((3U << 3) + 1),
+ CLASS_D = ((3U << 3) + 3),
+ MIN = ((3U << 3) + 4),
+ MINA = ((3U << 3) + 5),
+ MAX = ((3U << 3) + 6),
+ MAXA = ((3U << 3) + 7),
+ CVT_S_D = ((4U << 3) + 0),
+ CVT_W_D = ((4U << 3) + 4),
+ CVT_L_D = ((4U << 3) + 5),
+ C_F_D = ((6U << 3) + 0),
+ C_UN_D = ((6U << 3) + 1),
+ C_EQ_D = ((6U << 3) + 2),
+ C_UEQ_D = ((6U << 3) + 3),
+ C_OLT_D = ((6U << 3) + 4),
+ C_ULT_D = ((6U << 3) + 5),
+ C_OLE_D = ((6U << 3) + 6),
+ C_ULE_D = ((6U << 3) + 7),
+
+ // COP1 Encoding of Function Field When rs=W or L.
+ CVT_S_W = ((4U << 3) + 0),
+ CVT_D_W = ((4U << 3) + 1),
+ CVT_S_L = ((4U << 3) + 0),
+ CVT_D_L = ((4U << 3) + 1),
+ BC1EQZ = ((2U << 2) + 1) << 21,
+ BC1NEZ = ((3U << 2) + 1) << 21,
+ // COP1 CMP positive predicates Bit 5..4 = 00.
+ CMP_AF = ((0U << 3) + 0),
+ CMP_UN = ((0U << 3) + 1),
+ CMP_EQ = ((0U << 3) + 2),
+ CMP_UEQ = ((0U << 3) + 3),
+ CMP_LT = ((0U << 3) + 4),
+ CMP_ULT = ((0U << 3) + 5),
+ CMP_LE = ((0U << 3) + 6),
+ CMP_ULE = ((0U << 3) + 7),
+ CMP_SAF = ((1U << 3) + 0),
+ CMP_SUN = ((1U << 3) + 1),
+ CMP_SEQ = ((1U << 3) + 2),
+ CMP_SUEQ = ((1U << 3) + 3),
+ CMP_SSLT = ((1U << 3) + 4),
+ CMP_SSULT = ((1U << 3) + 5),
+ CMP_SLE = ((1U << 3) + 6),
+ CMP_SULE = ((1U << 3) + 7),
+ // COP1 CMP negative predicates Bit 5..4 = 01.
+ CMP_AT = ((2U << 3) + 0), // Reserved, not implemented.
+ CMP_OR = ((2U << 3) + 1),
+ CMP_UNE = ((2U << 3) + 2),
+ CMP_NE = ((2U << 3) + 3),
+ CMP_UGE = ((2U << 3) + 4), // Reserved, not implemented.
+ CMP_OGE = ((2U << 3) + 5), // Reserved, not implemented.
+ CMP_UGT = ((2U << 3) + 6), // Reserved, not implemented.
+ CMP_OGT = ((2U << 3) + 7), // Reserved, not implemented.
+ CMP_SAT = ((3U << 3) + 0), // Reserved, not implemented.
+ CMP_SOR = ((3U << 3) + 1),
+ CMP_SUNE = ((3U << 3) + 2),
+ CMP_SNE = ((3U << 3) + 3),
+ CMP_SUGE = ((3U << 3) + 4), // Reserved, not implemented.
+ CMP_SOGE = ((3U << 3) + 5), // Reserved, not implemented.
+ CMP_SUGT = ((3U << 3) + 6), // Reserved, not implemented.
+ CMP_SOGT = ((3U << 3) + 7), // Reserved, not implemented.
+
+ SEL = ((2U << 3) + 0),
+ MOVZ_C = ((2U << 3) + 2),
+ MOVN_C = ((2U << 3) + 3),
+ SELEQZ_C = ((2U << 3) + 4), // COP1 on FPR registers.
+ MOVF = ((2U << 3) + 1), // Function field for MOVT.fmt and MOVF.fmt
+ SELNEZ_C = ((2U << 3) + 7), // COP1 on FPR registers.
+ // COP1 Encoding of Function Field When rs=PS.
+
+ // COP1X Encoding of Function Field.
+ MADD_S = ((4U << 3) + 0),
+ MADD_D = ((4U << 3) + 1),
+ MSUB_S = ((5U << 3) + 0),
+ MSUB_D = ((5U << 3) + 1),
+
+ // PCREL Encoding of rt Field.
+ ADDIUPC = ((0U << 2) + 0),
+ LWPC = ((0U << 2) + 1),
+ AUIPC = ((3U << 3) + 6),
+ ALUIPC = ((3U << 3) + 7),
+
+ // POP66 Encoding of rs Field.
+ JIC = ((0U << 5) + 0),
+
+ // POP76 Encoding of rs Field.
+ JIALC = ((0U << 5) + 0),
+
+ // COP1 Encoding of rs Field for MSA Branch Instructions
+ BZ_V = (((1U << 3) + 3) << kRsShift),
+ BNZ_V = (((1U << 3) + 7) << kRsShift),
+ BZ_B = (((3U << 3) + 0) << kRsShift),
+ BZ_H = (((3U << 3) + 1) << kRsShift),
+ BZ_W = (((3U << 3) + 2) << kRsShift),
+ BZ_D = (((3U << 3) + 3) << kRsShift),
+ BNZ_B = (((3U << 3) + 4) << kRsShift),
+ BNZ_H = (((3U << 3) + 5) << kRsShift),
+ BNZ_W = (((3U << 3) + 6) << kRsShift),
+ BNZ_D = (((3U << 3) + 7) << kRsShift),
+
+ // MSA: Operation Field for MI10 Instruction Formats
+ MSA_LD = (8U << 2),
+ MSA_ST = (9U << 2),
+ LD_B = ((8U << 2) + 0),
+ LD_H = ((8U << 2) + 1),
+ LD_W = ((8U << 2) + 2),
+ LD_D = ((8U << 2) + 3),
+ ST_B = ((9U << 2) + 0),
+ ST_H = ((9U << 2) + 1),
+ ST_W = ((9U << 2) + 2),
+ ST_D = ((9U << 2) + 3),
+
+ // MSA: Operation Field for I5 Instruction Format
+ ADDVI = ((0U << 23) + 6),
+ SUBVI = ((1U << 23) + 6),
+ MAXI_S = ((2U << 23) + 6),
+ MAXI_U = ((3U << 23) + 6),
+ MINI_S = ((4U << 23) + 6),
+ MINI_U = ((5U << 23) + 6),
+ CEQI = ((0U << 23) + 7),
+ CLTI_S = ((2U << 23) + 7),
+ CLTI_U = ((3U << 23) + 7),
+ CLEI_S = ((4U << 23) + 7),
+ CLEI_U = ((5U << 23) + 7),
+ LDI = ((6U << 23) + 7), // I10 instruction format
+ I5_DF_b = (0U << 21),
+ I5_DF_h = (1U << 21),
+ I5_DF_w = (2U << 21),
+ I5_DF_d = (3U << 21),
+
+ // MSA: Operation Field for I8 Instruction Format
+ ANDI_B = ((0U << 24) + 0),
+ ORI_B = ((1U << 24) + 0),
+ NORI_B = ((2U << 24) + 0),
+ XORI_B = ((3U << 24) + 0),
+ BMNZI_B = ((0U << 24) + 1),
+ BMZI_B = ((1U << 24) + 1),
+ BSELI_B = ((2U << 24) + 1),
+ SHF_B = ((0U << 24) + 2),
+ SHF_H = ((1U << 24) + 2),
+ SHF_W = ((2U << 24) + 2),
+
+ MSA_VEC_2R_2RF_MINOR = ((3U << 3) + 6),
+
+ // MSA: Operation Field for VEC Instruction Formats
+ AND_V = (((0U << 2) + 0) << 21),
+ OR_V = (((0U << 2) + 1) << 21),
+ NOR_V = (((0U << 2) + 2) << 21),
+ XOR_V = (((0U << 2) + 3) << 21),
+ BMNZ_V = (((1U << 2) + 0) << 21),
+ BMZ_V = (((1U << 2) + 1) << 21),
+ BSEL_V = (((1U << 2) + 2) << 21),
+
+ // MSA: Operation Field for 2R Instruction Formats
+ MSA_2R_FORMAT = (((6U << 2) + 0) << 21),
+ FILL = (0U << 18),
+ PCNT = (1U << 18),
+ NLOC = (2U << 18),
+ NLZC = (3U << 18),
+ MSA_2R_DF_b = (0U << 16),
+ MSA_2R_DF_h = (1U << 16),
+ MSA_2R_DF_w = (2U << 16),
+ MSA_2R_DF_d = (3U << 16),
+
+ // MSA: Operation Field for 2RF Instruction Formats
+ MSA_2RF_FORMAT = (((6U << 2) + 1) << 21),
+ FCLASS = (0U << 17),
+ FTRUNC_S = (1U << 17),
+ FTRUNC_U = (2U << 17),
+ FSQRT = (3U << 17),
+ FRSQRT = (4U << 17),
+ FRCP = (5U << 17),
+ FRINT = (6U << 17),
+ FLOG2 = (7U << 17),
+ FEXUPL = (8U << 17),
+ FEXUPR = (9U << 17),
+ FFQL = (10U << 17),
+ FFQR = (11U << 17),
+ FTINT_S = (12U << 17),
+ FTINT_U = (13U << 17),
+ FFINT_S = (14U << 17),
+ FFINT_U = (15U << 17),
+ MSA_2RF_DF_w = (0U << 16),
+ MSA_2RF_DF_d = (1U << 16),
+
+ // MSA: Operation Field for 3R Instruction Format
+ SLL_MSA = ((0U << 23) + 13),
+ SRA_MSA = ((1U << 23) + 13),
+ SRL_MSA = ((2U << 23) + 13),
+ BCLR = ((3U << 23) + 13),
+ BSET = ((4U << 23) + 13),
+ BNEG = ((5U << 23) + 13),
+ BINSL = ((6U << 23) + 13),
+ BINSR = ((7U << 23) + 13),
+ ADDV = ((0U << 23) + 14),
+ SUBV = ((1U << 23) + 14),
+ MAX_S = ((2U << 23) + 14),
+ MAX_U = ((3U << 23) + 14),
+ MIN_S = ((4U << 23) + 14),
+ MIN_U = ((5U << 23) + 14),
+ MAX_A = ((6U << 23) + 14),
+ MIN_A = ((7U << 23) + 14),
+ CEQ = ((0U << 23) + 15),
+ CLT_S = ((2U << 23) + 15),
+ CLT_U = ((3U << 23) + 15),
+ CLE_S = ((4U << 23) + 15),
+ CLE_U = ((5U << 23) + 15),
+ ADD_A = ((0U << 23) + 16),
+ ADDS_A = ((1U << 23) + 16),
+ ADDS_S = ((2U << 23) + 16),
+ ADDS_U = ((3U << 23) + 16),
+ AVE_S = ((4U << 23) + 16),
+ AVE_U = ((5U << 23) + 16),
+ AVER_S = ((6U << 23) + 16),
+ AVER_U = ((7U << 23) + 16),
+ SUBS_S = ((0U << 23) + 17),
+ SUBS_U = ((1U << 23) + 17),
+ SUBSUS_U = ((2U << 23) + 17),
+ SUBSUU_S = ((3U << 23) + 17),
+ ASUB_S = ((4U << 23) + 17),
+ ASUB_U = ((5U << 23) + 17),
+ MULV = ((0U << 23) + 18),
+ MADDV = ((1U << 23) + 18),
+ MSUBV = ((2U << 23) + 18),
+ DIV_S_MSA = ((4U << 23) + 18),
+ DIV_U = ((5U << 23) + 18),
+ MOD_S = ((6U << 23) + 18),
+ MOD_U = ((7U << 23) + 18),
+ DOTP_S = ((0U << 23) + 19),
+ DOTP_U = ((1U << 23) + 19),
+ DPADD_S = ((2U << 23) + 19),
+ DPADD_U = ((3U << 23) + 19),
+ DPSUB_S = ((4U << 23) + 19),
+ DPSUB_U = ((5U << 23) + 19),
+ SLD = ((0U << 23) + 20),
+ SPLAT = ((1U << 23) + 20),
+ PCKEV = ((2U << 23) + 20),
+ PCKOD = ((3U << 23) + 20),
+ ILVL = ((4U << 23) + 20),
+ ILVR = ((5U << 23) + 20),
+ ILVEV = ((6U << 23) + 20),
+ ILVOD = ((7U << 23) + 20),
+ VSHF = ((0U << 23) + 21),
+ SRAR = ((1U << 23) + 21),
+ SRLR = ((2U << 23) + 21),
+ HADD_S = ((4U << 23) + 21),
+ HADD_U = ((5U << 23) + 21),
+ HSUB_S = ((6U << 23) + 21),
+ HSUB_U = ((7U << 23) + 21),
+ MSA_3R_DF_b = (0U << 21),
+ MSA_3R_DF_h = (1U << 21),
+ MSA_3R_DF_w = (2U << 21),
+ MSA_3R_DF_d = (3U << 21),
+
+ // MSA: Operation Field for 3RF Instruction Format
+ FCAF = ((0U << 22) + 26),
+ FCUN = ((1U << 22) + 26),
+ FCEQ = ((2U << 22) + 26),
+ FCUEQ = ((3U << 22) + 26),
+ FCLT = ((4U << 22) + 26),
+ FCULT = ((5U << 22) + 26),
+ FCLE = ((6U << 22) + 26),
+ FCULE = ((7U << 22) + 26),
+ FSAF = ((8U << 22) + 26),
+ FSUN = ((9U << 22) + 26),
+ FSEQ = ((10U << 22) + 26),
+ FSUEQ = ((11U << 22) + 26),
+ FSLT = ((12U << 22) + 26),
+ FSULT = ((13U << 22) + 26),
+ FSLE = ((14U << 22) + 26),
+ FSULE = ((15U << 22) + 26),
+ FADD = ((0U << 22) + 27),
+ FSUB = ((1U << 22) + 27),
+ FMUL = ((2U << 22) + 27),
+ FDIV = ((3U << 22) + 27),
+ FMADD = ((4U << 22) + 27),
+ FMSUB = ((5U << 22) + 27),
+ FEXP2 = ((7U << 22) + 27),
+ FEXDO = ((8U << 22) + 27),
+ FTQ = ((10U << 22) + 27),
+ FMIN = ((12U << 22) + 27),
+ FMIN_A = ((13U << 22) + 27),
+ FMAX = ((14U << 22) + 27),
+ FMAX_A = ((15U << 22) + 27),
+ FCOR = ((1U << 22) + 28),
+ FCUNE = ((2U << 22) + 28),
+ FCNE = ((3U << 22) + 28),
+ MUL_Q = ((4U << 22) + 28),
+ MADD_Q = ((5U << 22) + 28),
+ MSUB_Q = ((6U << 22) + 28),
+ FSOR = ((9U << 22) + 28),
+ FSUNE = ((10U << 22) + 28),
+ FSNE = ((11U << 22) + 28),
+ MULR_Q = ((12U << 22) + 28),
+ MADDR_Q = ((13U << 22) + 28),
+ MSUBR_Q = ((14U << 22) + 28),
+
+ // MSA: Operation Field for ELM Instruction Format
+ MSA_ELM_MINOR = ((3U << 3) + 1),
+ SLDI = (0U << 22),
+ CTCMSA = ((0U << 22) | (62U << 16)),
+ SPLATI = (1U << 22),
+ CFCMSA = ((1U << 22) | (62U << 16)),
+ COPY_S = (2U << 22),
+ MOVE_V = ((2U << 22) | (62U << 16)),
+ COPY_U = (3U << 22),
+ INSERT = (4U << 22),
+ INSVE = (5U << 22),
+ ELM_DF_B = ((0U << 4) << 16),
+ ELM_DF_H = ((4U << 3) << 16),
+ ELM_DF_W = ((12U << 2) << 16),
+ ELM_DF_D = ((28U << 1) << 16),
+
+ // MSA: Operation Field for BIT Instruction Format
+ SLLI = ((0U << 23) + 9),
+ SRAI = ((1U << 23) + 9),
+ SRLI = ((2U << 23) + 9),
+ BCLRI = ((3U << 23) + 9),
+ BSETI = ((4U << 23) + 9),
+ BNEGI = ((5U << 23) + 9),
+ BINSLI = ((6U << 23) + 9),
+ BINSRI = ((7U << 23) + 9),
+ SAT_S = ((0U << 23) + 10),
+ SAT_U = ((1U << 23) + 10),
+ SRARI = ((2U << 23) + 10),
+ SRLRI = ((3U << 23) + 10),
+ BIT_DF_b = ((14U << 3) << 16),
+ BIT_DF_h = ((6U << 4) << 16),
+ BIT_DF_w = ((2U << 5) << 16),
+ BIT_DF_d = ((0U << 6) << 16),
+
+ nullptrSF = 0U
+};
+
+enum MSAMinorOpcode : uint32_t {
+ kMsaMinorUndefined = 0,
+ kMsaMinorI8,
+ kMsaMinorI5,
+ kMsaMinorI10,
+ kMsaMinorBIT,
+ kMsaMinor3R,
+ kMsaMinor3RF,
+ kMsaMinorELM,
+ kMsaMinorVEC,
+ kMsaMinor2R,
+ kMsaMinor2RF,
+ kMsaMinorMI10
+};
+
+// ----- Emulated conditions.
+// On MIPS we use this enum to abstract from conditional branch instructions.
+// The 'U' prefix is used to specify unsigned comparisons.
+// Opposite conditions must be paired as odd/even numbers
+// because 'NegateCondition' function flips LSB to negate condition.
+enum Condition {
+ // Any value < 0 is considered no_condition.
+ kNoCondition = -1,
+ overflow = 0,
+ no_overflow = 1,
+ Uless = 2,
+ Ugreater_equal = 3,
+ Uless_equal = 4,
+ Ugreater = 5,
+ equal = 6,
+ not_equal = 7, // Unordered or Not Equal.
+ negative = 8,
+ positive = 9,
+ parity_even = 10,
+ parity_odd = 11,
+ less = 12,
+ greater_equal = 13,
+ less_equal = 14,
+ greater = 15,
+ ueq = 16, // Unordered or Equal.
+ ogl = 17, // Ordered and Not Equal.
+ cc_always = 18,
+
+ // Aliases.
+ carry = Uless,
+ not_carry = Ugreater_equal,
+ zero = equal,
+ eq = equal,
+ not_zero = not_equal,
+ ne = not_equal,
+ nz = not_equal,
+ sign = negative,
+ not_sign = positive,
+ mi = negative,
+ pl = positive,
+ hi = Ugreater,
+ ls = Uless_equal,
+ ge = greater_equal,
+ lt = less,
+ gt = greater,
+ le = less_equal,
+ hs = Ugreater_equal,
+ lo = Uless,
+ al = cc_always,
+ ult = Uless,
+ uge = Ugreater_equal,
+ ule = Uless_equal,
+ ugt = Ugreater,
+ cc_default = kNoCondition
+};
+
+// Returns the equivalent of !cc.
+// Negation of the default kNoCondition (-1) results in a non-default
+// no_condition value (-2). As long as tests for no_condition check
+// for condition < 0, this will work as expected.
+inline Condition NegateCondition(Condition cc) {
+ DCHECK(cc != cc_always);
+ return static_cast<Condition>(cc ^ 1);
+}
+
+inline Condition NegateFpuCondition(Condition cc) {
+ DCHECK(cc != cc_always);
+ switch (cc) {
+ case ult:
+ return ge;
+ case ugt:
+ return le;
+ case uge:
+ return lt;
+ case ule:
+ return gt;
+ case lt:
+ return uge;
+ case gt:
+ return ule;
+ case ge:
+ return ult;
+ case le:
+ return ugt;
+ case eq:
+ return ne;
+ case ne:
+ return eq;
+ case ueq:
+ return ogl;
+ case ogl:
+ return ueq;
+ default:
+ return cc;
+ }
+}
+
+enum MSABranchCondition {
+ all_not_zero = 0, // Branch If All Elements Are Not Zero
+ one_elem_not_zero, // Branch If At Least One Element of Any Format Is Not
+ // Zero
+ one_elem_zero, // Branch If At Least One Element Is Zero
+ all_zero // Branch If All Elements of Any Format Are Zero
+};
+
+inline MSABranchCondition NegateMSABranchCondition(MSABranchCondition cond) {
+ switch (cond) {
+ case all_not_zero:
+ return one_elem_zero;
+ case one_elem_not_zero:
+ return all_zero;
+ case one_elem_zero:
+ return all_not_zero;
+ case all_zero:
+ return one_elem_not_zero;
+ default:
+ return cond;
+ }
+}
+
+enum MSABranchDF {
+ MSA_BRANCH_B = 0,
+ MSA_BRANCH_H,
+ MSA_BRANCH_W,
+ MSA_BRANCH_D,
+ MSA_BRANCH_V
+};
+
+// ----- Coprocessor conditions.
+enum FPUCondition {
+ kNoFPUCondition = -1,
+
+ F = 0x00, // False.
+ UN = 0x01, // Unordered.
+ EQ = 0x02, // Equal.
+ UEQ = 0x03, // Unordered or Equal.
+ OLT = 0x04, // Ordered or Less Than, on Mips release < 6.
+ LT = 0x04, // Ordered or Less Than, on Mips release >= 6.
+ ULT = 0x05, // Unordered or Less Than.
+ OLE = 0x06, // Ordered or Less Than or Equal, on Mips release < 6.
+ LE = 0x06, // Ordered or Less Than or Equal, on Mips release >= 6.
+ ULE = 0x07, // Unordered or Less Than or Equal.
+
+ // Following constants are available on Mips release >= 6 only.
+ ORD = 0x11, // Ordered, on Mips release >= 6.
+ UNE = 0x12, // Not equal, on Mips release >= 6.
+ NE = 0x13, // Ordered Greater Than or Less Than. on Mips >= 6 only.
+};
+
+// FPU rounding modes.
+enum FPURoundingMode {
+ RN = 0 << 0, // Round to Nearest.
+ RZ = 1 << 0, // Round towards zero.
+ RP = 2 << 0, // Round towards Plus Infinity.
+ RM = 3 << 0, // Round towards Minus Infinity.
+
+ // Aliases.
+ kRoundToNearest = RN,
+ kRoundToZero = RZ,
+ kRoundToPlusInf = RP,
+ kRoundToMinusInf = RM,
+
+ mode_round = RN,
+ mode_ceil = RP,
+ mode_floor = RM,
+ mode_trunc = RZ
+};
+
+const uint32_t kFPURoundingModeMask = 3 << 0;
+
+enum CheckForInexactConversion {
+ kCheckForInexactConversion,
+ kDontCheckForInexactConversion
+};
+
+enum class MaxMinKind : int { kMin = 0, kMax = 1 };
+
+// -----------------------------------------------------------------------------
+// Hints.
+
+// Branch hints are not used on the MIPS. They are defined so that they can
+// appear in shared function signatures, but will be ignored in MIPS
+// implementations.
+enum Hint { no_hint = 0 };
+
+inline Hint NegateHint(Hint hint) { return no_hint; }
+
+// -----------------------------------------------------------------------------
+// Specific instructions, constants, and masks.
+// These constants are declared in assembler-mips.cc, as they use named
+// registers and other constants.
+
+// addiu(sp, sp, 4) aka Pop() operation or part of Pop(r)
+// operations as post-increment of sp.
+extern const Instr kPopInstruction;
+// addiu(sp, sp, -4) part of Push(r) operation as pre-decrement of sp.
+extern const Instr kPushInstruction;
+// sw(r, MemOperand(sp, 0))
+extern const Instr kPushRegPattern;
+// lw(r, MemOperand(sp, 0))
+extern const Instr kPopRegPattern;
+extern const Instr kLwRegFpOffsetPattern;
+extern const Instr kSwRegFpOffsetPattern;
+extern const Instr kLwRegFpNegOffsetPattern;
+extern const Instr kSwRegFpNegOffsetPattern;
+// A mask for the Rt register for push, pop, lw, sw instructions.
+extern const Instr kRtMask;
+extern const Instr kLwSwInstrTypeMask;
+extern const Instr kLwSwInstrArgumentMask;
+extern const Instr kLwSwOffsetMask;
+
+// Break 0xfffff, reserved for redirected real time call.
+const Instr rtCallRedirInstr = SPECIAL | BREAK | call_rt_redirected << 6;
+// A nop instruction. (Encoding of sll 0 0 0).
+const Instr nopInstr = 0;
+
+static constexpr uint64_t OpcodeToBitNumber(Opcode opcode) {
+ return 1ULL << (static_cast<uint32_t>(opcode) >> kOpcodeShift);
+}
+
+constexpr uint8_t kInstrSize = 4;
+constexpr uint8_t kInstrSizeLog2 = 2;
+
+class InstructionBase {
+ public:
+ enum {
+ // On MIPS PC cannot actually be directly accessed. We behave as if PC was
+ // always the value of the current instruction being executed.
+ kPCReadOffset = 0
+ };
+
+ // Instruction type.
+ enum Type { kRegisterType, kImmediateType, kJumpType, kUnsupported = -1 };
+
+ // Get the raw instruction bits.
+ inline Instr InstructionBits() const {
+ return *reinterpret_cast<const Instr*>(this);
+ }
+
+ // Set the raw instruction bits to value.
+ inline void SetInstructionBits(Instr value) {
+ *reinterpret_cast<Instr*>(this) = value;
+ }
+
+ // Read one particular bit out of the instruction bits.
+ inline int Bit(int nr) const { return (InstructionBits() >> nr) & 1; }
+
+ // Read a bit field out of the instruction bits.
+ inline int Bits(int hi, int lo) const {
+ return (InstructionBits() >> lo) & ((2U << (hi - lo)) - 1);
+ }
+
+ static constexpr uint64_t kOpcodeImmediateTypeMask =
+ OpcodeToBitNumber(REGIMM) | OpcodeToBitNumber(BEQ) |
+ OpcodeToBitNumber(BNE) | OpcodeToBitNumber(BLEZ) |
+ OpcodeToBitNumber(BGTZ) | OpcodeToBitNumber(ADDI) |
+ OpcodeToBitNumber(DADDI) | OpcodeToBitNumber(ADDIU) |
+ OpcodeToBitNumber(SLTI) | OpcodeToBitNumber(SLTIU) |
+ OpcodeToBitNumber(ANDI) | OpcodeToBitNumber(ORI) |
+ OpcodeToBitNumber(XORI) | OpcodeToBitNumber(LUI) |
+ OpcodeToBitNumber(BEQL) | OpcodeToBitNumber(BNEL) |
+ OpcodeToBitNumber(BLEZL) | OpcodeToBitNumber(BGTZL) |
+ OpcodeToBitNumber(POP66) | OpcodeToBitNumber(POP76) |
+ OpcodeToBitNumber(LB) | OpcodeToBitNumber(LH) | OpcodeToBitNumber(LWL) |
+ OpcodeToBitNumber(LW) | OpcodeToBitNumber(LBU) | OpcodeToBitNumber(LHU) |
+ OpcodeToBitNumber(LWR) | OpcodeToBitNumber(SB) | OpcodeToBitNumber(SH) |
+ OpcodeToBitNumber(SWL) | OpcodeToBitNumber(SW) | OpcodeToBitNumber(SWR) |
+ OpcodeToBitNumber(LWC1) | OpcodeToBitNumber(LDC1) |
+ OpcodeToBitNumber(SWC1) | OpcodeToBitNumber(SDC1) |
+ OpcodeToBitNumber(PCREL) | OpcodeToBitNumber(BC) |
+ OpcodeToBitNumber(BALC);
+
+#define FunctionFieldToBitNumber(function) (1ULL << function)
+
+ static const uint64_t kFunctionFieldRegisterTypeMask =
+ FunctionFieldToBitNumber(JR) | FunctionFieldToBitNumber(JALR) |
+ FunctionFieldToBitNumber(BREAK) | FunctionFieldToBitNumber(SLL) |
+ FunctionFieldToBitNumber(SRL) | FunctionFieldToBitNumber(SRA) |
+ FunctionFieldToBitNumber(SLLV) | FunctionFieldToBitNumber(SRLV) |
+ FunctionFieldToBitNumber(SRAV) | FunctionFieldToBitNumber(LSA) |
+ FunctionFieldToBitNumber(MFHI) | FunctionFieldToBitNumber(MFLO) |
+ FunctionFieldToBitNumber(MULT) | FunctionFieldToBitNumber(MULTU) |
+ FunctionFieldToBitNumber(DIV) | FunctionFieldToBitNumber(DIVU) |
+ FunctionFieldToBitNumber(ADD) | FunctionFieldToBitNumber(ADDU) |
+ FunctionFieldToBitNumber(SUB) | FunctionFieldToBitNumber(SUBU) |
+ FunctionFieldToBitNumber(AND) | FunctionFieldToBitNumber(OR) |
+ FunctionFieldToBitNumber(XOR) | FunctionFieldToBitNumber(NOR) |
+ FunctionFieldToBitNumber(SLT) | FunctionFieldToBitNumber(SLTU) |
+ FunctionFieldToBitNumber(TGE) | FunctionFieldToBitNumber(TGEU) |
+ FunctionFieldToBitNumber(TLT) | FunctionFieldToBitNumber(TLTU) |
+ FunctionFieldToBitNumber(TEQ) | FunctionFieldToBitNumber(TNE) |
+ FunctionFieldToBitNumber(MOVZ) | FunctionFieldToBitNumber(MOVN) |
+ FunctionFieldToBitNumber(MOVCI) | FunctionFieldToBitNumber(SELEQZ_S) |
+ FunctionFieldToBitNumber(SELNEZ_S) | FunctionFieldToBitNumber(SYNC);
+
+ // Accessors for the different named fields used in the MIPS encoding.
+ inline Opcode OpcodeValue() const {
+ return static_cast<Opcode>(
+ Bits(kOpcodeShift + kOpcodeBits - 1, kOpcodeShift));
+ }
+
+ inline int FunctionFieldRaw() const {
+ return InstructionBits() & kFunctionFieldMask;
+ }
+
+ // Return the fields at their original place in the instruction encoding.
+ inline Opcode OpcodeFieldRaw() const {
+ return static_cast<Opcode>(InstructionBits() & kOpcodeMask);
+ }
+
+ // Safe to call within InstructionType().
+ inline int RsFieldRawNoAssert() const {
+ return InstructionBits() & kRsFieldMask;
+ }
+
+ inline int SaFieldRaw() const { return InstructionBits() & kSaFieldMask; }
+
+ // Get the encoding type of the instruction.
+ inline Type InstructionType() const;
+
+ inline MSAMinorOpcode MSAMinorOpcodeField() const {
+ int op = this->FunctionFieldRaw();
+ switch (op) {
+ case 0:
+ case 1:
+ case 2:
+ return kMsaMinorI8;
+ case 6:
+ return kMsaMinorI5;
+ case 7:
+ return (((this->InstructionBits() & kMsaI5I10Mask) == LDI)
+ ? kMsaMinorI10
+ : kMsaMinorI5);
+ case 9:
+ case 10:
+ return kMsaMinorBIT;
+ case 13:
+ case 14:
+ case 15:
+ case 16:
+ case 17:
+ case 18:
+ case 19:
+ case 20:
+ case 21:
+ return kMsaMinor3R;
+ case 25:
+ return kMsaMinorELM;
+ case 26:
+ case 27:
+ case 28:
+ return kMsaMinor3RF;
+ case 30:
+ switch (this->RsFieldRawNoAssert()) {
+ case MSA_2R_FORMAT:
+ return kMsaMinor2R;
+ case MSA_2RF_FORMAT:
+ return kMsaMinor2RF;
+ default:
+ return kMsaMinorVEC;
+ }
+ break;
+ case 32:
+ case 33:
+ case 34:
+ case 35:
+ case 36:
+ case 37:
+ case 38:
+ case 39:
+ return kMsaMinorMI10;
+ default:
+ return kMsaMinorUndefined;
+ }
+ }
+
+ protected:
+ InstructionBase() {}
+};
+
+template <class T>
+class InstructionGetters : public T {
+ public:
+ inline int RsValue() const {
+ DCHECK(this->InstructionType() == InstructionBase::kRegisterType ||
+ this->InstructionType() == InstructionBase::kImmediateType);
+ return InstructionBase::Bits(kRsShift + kRsBits - 1, kRsShift);
+ }
+
+ inline int RtValue() const {
+ DCHECK(this->InstructionType() == InstructionBase::kRegisterType ||
+ this->InstructionType() == InstructionBase::kImmediateType);
+ return this->Bits(kRtShift + kRtBits - 1, kRtShift);
+ }
+
+ inline int RdValue() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kRegisterType);
+ return this->Bits(kRdShift + kRdBits - 1, kRdShift);
+ }
+
+ inline int BaseValue() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kImmediateType);
+ return this->Bits(kBaseShift + kBaseBits - 1, kBaseShift);
+ }
+
+ inline int SaValue() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kRegisterType);
+ return this->Bits(kSaShift + kSaBits - 1, kSaShift);
+ }
+
+ inline int LsaSaValue() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kRegisterType);
+ return this->Bits(kSaShift + kLsaSaBits - 1, kSaShift);
+ }
+
+ inline int FunctionValue() const {
+ DCHECK(this->InstructionType() == InstructionBase::kRegisterType ||
+ this->InstructionType() == InstructionBase::kImmediateType);
+ return this->Bits(kFunctionShift + kFunctionBits - 1, kFunctionShift);
+ }
+
+ inline int FdValue() const {
+ return this->Bits(kFdShift + kFdBits - 1, kFdShift);
+ }
+
+ inline int FsValue() const {
+ return this->Bits(kFsShift + kFsBits - 1, kFsShift);
+ }
+
+ inline int FtValue() const {
+ return this->Bits(kFtShift + kFtBits - 1, kFtShift);
+ }
+
+ inline int FrValue() const {
+ return this->Bits(kFrShift + kFrBits - 1, kFrShift);
+ }
+
+ inline int WdValue() const {
+ return this->Bits(kWdShift + kWdBits - 1, kWdShift);
+ }
+
+ inline int WsValue() const {
+ return this->Bits(kWsShift + kWsBits - 1, kWsShift);
+ }
+
+ inline int WtValue() const {
+ return this->Bits(kWtShift + kWtBits - 1, kWtShift);
+ }
+
+ inline int Bp2Value() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kRegisterType);
+ return this->Bits(kBp2Shift + kBp2Bits - 1, kBp2Shift);
+ }
+
+ // Float Compare condition code instruction bits.
+ inline int FCccValue() const {
+ return this->Bits(kFCccShift + kFCccBits - 1, kFCccShift);
+ }
+
+ // Float Branch condition code instruction bits.
+ inline int FBccValue() const {
+ return this->Bits(kFBccShift + kFBccBits - 1, kFBccShift);
+ }
+
+ // Float Branch true/false instruction bit.
+ inline int FBtrueValue() const {
+ return this->Bits(kFBtrueShift + kFBtrueBits - 1, kFBtrueShift);
+ }
+
+ // Return the fields at their original place in the instruction encoding.
+ inline Opcode OpcodeFieldRaw() const {
+ return static_cast<Opcode>(this->InstructionBits() & kOpcodeMask);
+ }
+
+ inline int RsFieldRaw() const {
+ DCHECK(this->InstructionType() == InstructionBase::kRegisterType ||
+ this->InstructionType() == InstructionBase::kImmediateType);
+ return this->InstructionBits() & kRsFieldMask;
+ }
+
+ inline int RtFieldRaw() const {
+ DCHECK(this->InstructionType() == InstructionBase::kRegisterType ||
+ this->InstructionType() == InstructionBase::kImmediateType);
+ return this->InstructionBits() & kRtFieldMask;
+ }
+
+ inline int RdFieldRaw() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kRegisterType);
+ return this->InstructionBits() & kRdFieldMask;
+ }
+
+ inline int SaFieldRaw() const {
+ return this->InstructionBits() & kSaFieldMask;
+ }
+
+ inline int FunctionFieldRaw() const {
+ return this->InstructionBits() & kFunctionFieldMask;
+ }
+
+ // Get the secondary field according to the opcode.
+ inline int SecondaryValue() const {
+ Opcode op = this->OpcodeFieldRaw();
+ switch (op) {
+ case SPECIAL:
+ case SPECIAL2:
+ return FunctionValue();
+ case COP1:
+ return RsValue();
+ case REGIMM:
+ return RtValue();
+ default:
+ return nullptrSF;
+ }
+ }
+
+ inline int32_t ImmValue(int bits) const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kImmediateType);
+ return this->Bits(bits - 1, 0);
+ }
+
+ inline int32_t Imm9Value() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kImmediateType);
+ return this->Bits(kImm9Shift + kImm9Bits - 1, kImm9Shift);
+ }
+
+ inline int32_t Imm16Value() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kImmediateType);
+ return this->Bits(kImm16Shift + kImm16Bits - 1, kImm16Shift);
+ }
+
+ inline int32_t Imm18Value() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kImmediateType);
+ return this->Bits(kImm18Shift + kImm18Bits - 1, kImm18Shift);
+ }
+
+ inline int32_t Imm19Value() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kImmediateType);
+ return this->Bits(kImm19Shift + kImm19Bits - 1, kImm19Shift);
+ }
+
+ inline int32_t Imm21Value() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kImmediateType);
+ return this->Bits(kImm21Shift + kImm21Bits - 1, kImm21Shift);
+ }
+
+ inline int32_t Imm26Value() const {
+ DCHECK((this->InstructionType() == InstructionBase::kJumpType) ||
+ (this->InstructionType() == InstructionBase::kImmediateType));
+ return this->Bits(kImm26Shift + kImm26Bits - 1, kImm26Shift);
+ }
+
+ inline int32_t MsaImm8Value() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kImmediateType);
+ return this->Bits(kMsaImm8Shift + kMsaImm8Bits - 1, kMsaImm8Shift);
+ }
+
+ inline int32_t MsaImm5Value() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kImmediateType);
+ return this->Bits(kMsaImm5Shift + kMsaImm5Bits - 1, kMsaImm5Shift);
+ }
+
+ inline int32_t MsaImm10Value() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kImmediateType);
+ return this->Bits(kMsaImm10Shift + kMsaImm10Bits - 1, kMsaImm10Shift);
+ }
+
+ inline int32_t MsaImmMI10Value() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kImmediateType);
+ return this->Bits(kMsaImmMI10Shift + kMsaImmMI10Bits - 1, kMsaImmMI10Shift);
+ }
+
+ inline int32_t MsaBitDf() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kImmediateType);
+ int32_t df_m = this->Bits(22, 16);
+ if (((df_m >> 6) & 1U) == 0) {
+ return 3;
+ } else if (((df_m >> 5) & 3U) == 2) {
+ return 2;
+ } else if (((df_m >> 4) & 7U) == 6) {
+ return 1;
+ } else if (((df_m >> 3) & 15U) == 14) {
+ return 0;
+ } else {
+ return -1;
+ }
+ }
+
+ inline int32_t MsaBitMValue() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kImmediateType);
+ return this->Bits(16 + this->MsaBitDf() + 3, 16);
+ }
+
+ inline int32_t MsaElmDf() const {
+ DCHECK(this->InstructionType() == InstructionBase::kRegisterType ||
+ this->InstructionType() == InstructionBase::kImmediateType);
+ int32_t df_n = this->Bits(21, 16);
+ if (((df_n >> 4) & 3U) == 0) {
+ return 0;
+ } else if (((df_n >> 3) & 7U) == 4) {
+ return 1;
+ } else if (((df_n >> 2) & 15U) == 12) {
+ return 2;
+ } else if (((df_n >> 1) & 31U) == 28) {
+ return 3;
+ } else {
+ return -1;
+ }
+ }
+
+ inline int32_t MsaElmNValue() const {
+ DCHECK(this->InstructionType() == InstructionBase::kRegisterType ||
+ this->InstructionType() == InstructionBase::kImmediateType);
+ return this->Bits(16 + 4 - this->MsaElmDf(), 16);
+ }
+
+ static bool IsForbiddenAfterBranchInstr(Instr instr);
+
+ // Say if the instruction should not be used in a branch delay slot or
+ // immediately after a compact branch.
+ inline bool IsForbiddenAfterBranch() const {
+ return IsForbiddenAfterBranchInstr(this->InstructionBits());
+ }
+
+ inline bool IsForbiddenInBranchDelay() const {
+ return IsForbiddenAfterBranch();
+ }
+
+ // Say if the instruction 'links'. e.g. jal, bal.
+ bool IsLinkingInstruction() const;
+ // Say if the instruction is a break or a trap.
+ bool IsTrap() const;
+
+ inline bool IsMSABranchInstr() const {
+ if (this->OpcodeFieldRaw() == COP1) {
+ switch (this->RsFieldRaw()) {
+ case BZ_V:
+ case BZ_B:
+ case BZ_H:
+ case BZ_W:
+ case BZ_D:
+ case BNZ_V:
+ case BNZ_B:
+ case BNZ_H:
+ case BNZ_W:
+ case BNZ_D:
+ return true;
+ default:
+ return false;
+ }
+ }
+ return false;
+ }
+
+ inline bool IsMSAInstr() const {
+ if (this->IsMSABranchInstr() || (this->OpcodeFieldRaw() == MSA))
+ return true;
+ return false;
+ }
+};
+
+class Instruction : public InstructionGetters<InstructionBase> {
+ public:
+ // Instructions are read of out a code stream. The only way to get a
+ // reference to an instruction is to convert a pointer. There is no way
+ // to allocate or create instances of class Instruction.
+ // Use the At(pc) function to create references to Instruction.
+ static Instruction* At(byte* pc) {
+ return reinterpret_cast<Instruction*>(pc);
+ }
+
+ private:
+ // We need to prevent the creation of instances of class Instruction.
+ DISALLOW_IMPLICIT_CONSTRUCTORS(Instruction);
+};
+
+// -----------------------------------------------------------------------------
+// MIPS assembly various constants.
+
+// C/C++ argument slots size.
+const int kCArgSlotCount = 4;
+const int kCArgsSlotsSize = kCArgSlotCount * kInstrSize;
+
+// JS argument slots size.
+const int kJSArgsSlotsSize = 0 * kInstrSize;
+
+// Assembly builtins argument slots size.
+const int kBArgsSlotsSize = 0 * kInstrSize;
+
+const int kBranchReturnOffset = 2 * kInstrSize;
+
+InstructionBase::Type InstructionBase::InstructionType() const {
+ switch (OpcodeFieldRaw()) {
+ case SPECIAL:
+ if (FunctionFieldToBitNumber(FunctionFieldRaw()) &
+ kFunctionFieldRegisterTypeMask) {
+ return kRegisterType;
+ }
+ return kUnsupported;
+ case SPECIAL2:
+ switch (FunctionFieldRaw()) {
+ case MUL:
+ case CLZ:
+ return kRegisterType;
+ default:
+ return kUnsupported;
+ }
+ break;
+ case SPECIAL3:
+ switch (FunctionFieldRaw()) {
+ case INS:
+ case EXT:
+ return kRegisterType;
+ case BSHFL: {
+ int sa = SaFieldRaw() >> kSaShift;
+ switch (sa) {
+ case BITSWAP:
+ case WSBH:
+ case SEB:
+ case SEH:
+ return kRegisterType;
+ }
+ sa >>= kBp2Bits;
+ switch (sa) {
+ case ALIGN:
+ return kRegisterType;
+ default:
+ return kUnsupported;
+ }
+ }
+ case LL_R6:
+ case SC_R6: {
+ DCHECK(IsMipsArchVariant(kMips32r6));
+ return kImmediateType;
+ }
+ default:
+ return kUnsupported;
+ }
+ break;
+ case COP1: // Coprocessor instructions.
+ switch (RsFieldRawNoAssert()) {
+ case BC1: // Branch on coprocessor condition.
+ case BC1EQZ:
+ case BC1NEZ:
+ return kImmediateType;
+ // MSA Branch instructions
+ case BZ_V:
+ case BNZ_V:
+ case BZ_B:
+ case BZ_H:
+ case BZ_W:
+ case BZ_D:
+ case BNZ_B:
+ case BNZ_H:
+ case BNZ_W:
+ case BNZ_D:
+ return kImmediateType;
+ default:
+ return kRegisterType;
+ }
+ break;
+ case COP1X:
+ return kRegisterType;
+
+ // 26 bits immediate type instructions. e.g.: j imm26.
+ case J:
+ case JAL:
+ return kJumpType;
+
+ case MSA:
+ switch (MSAMinorOpcodeField()) {
+ case kMsaMinor3R:
+ case kMsaMinor3RF:
+ case kMsaMinorVEC:
+ case kMsaMinor2R:
+ case kMsaMinor2RF:
+ return kRegisterType;
+ case kMsaMinorELM:
+ switch (InstructionBits() & kMsaLongerELMMask) {
+ case CFCMSA:
+ case CTCMSA:
+ case MOVE_V:
+ return kRegisterType;
+ default:
+ return kImmediateType;
+ }
+ default:
+ return kImmediateType;
+ }
+
+ default:
+ return kImmediateType;
+ }
+}
+
+#undef OpcodeToBitNumber
+#undef FunctionFieldToBitNumber
+
+// -----------------------------------------------------------------------------
+// Instructions.
+
+template <class P>
+bool InstructionGetters<P>::IsLinkingInstruction() const {
+ uint32_t op = this->OpcodeFieldRaw();
+ switch (op) {
+ case JAL:
+ return true;
+ case POP76:
+ if (this->RsFieldRawNoAssert() == JIALC)
+ return true; // JIALC
+ else
+ return false; // BNEZC
+ case REGIMM:
+ switch (this->RtFieldRaw()) {
+ case BGEZAL:
+ case BLTZAL:
+ return true;
+ default:
+ return false;
+ }
+ case SPECIAL:
+ switch (this->FunctionFieldRaw()) {
+ case JALR:
+ return true;
+ default:
+ return false;
+ }
+ default:
+ return false;
+ }
+}
+
+template <class P>
+bool InstructionGetters<P>::IsTrap() const {
+ if (this->OpcodeFieldRaw() != SPECIAL) {
+ return false;
+ } else {
+ switch (this->FunctionFieldRaw()) {
+ case BREAK:
+ case TGE:
+ case TGEU:
+ case TLT:
+ case TLTU:
+ case TEQ:
+ case TNE:
+ return true;
+ default:
+ return false;
+ }
+ }
+}
+
+// static
+template <class T>
+bool InstructionGetters<T>::IsForbiddenAfterBranchInstr(Instr instr) {
+ Opcode opcode = static_cast<Opcode>(instr & kOpcodeMask);
+ switch (opcode) {
+ case J:
+ case JAL:
+ case BEQ:
+ case BNE:
+ case BLEZ: // POP06 bgeuc/bleuc, blezalc, bgezalc
+ case BGTZ: // POP07 bltuc/bgtuc, bgtzalc, bltzalc
+ case BEQL:
+ case BNEL:
+ case BLEZL: // POP26 bgezc, blezc, bgec/blec
+ case BGTZL: // POP27 bgtzc, bltzc, bltc/bgtc
+ case BC:
+ case BALC:
+ case POP10: // beqzalc, bovc, beqc
+ case POP30: // bnezalc, bnvc, bnec
+ case POP66: // beqzc, jic
+ case POP76: // bnezc, jialc
+ return true;
+ case REGIMM:
+ switch (instr & kRtFieldMask) {
+ case BLTZ:
+ case BGEZ:
+ case BLTZAL:
+ case BGEZAL:
+ return true;
+ default:
+ return false;
+ }
+ break;
+ case SPECIAL:
+ switch (instr & kFunctionFieldMask) {
+ case JR:
+ case JALR:
+ return true;
+ default:
+ return false;
+ }
+ break;
+ case COP1:
+ switch (instr & kRsFieldMask) {
+ case BC1:
+ case BC1EQZ:
+ case BC1NEZ:
+ case BZ_V:
+ case BZ_B:
+ case BZ_H:
+ case BZ_W:
+ case BZ_D:
+ case BNZ_V:
+ case BNZ_B:
+ case BNZ_H:
+ case BNZ_W:
+ case BNZ_D:
+ return true;
+ break;
+ default:
+ return false;
+ }
+ break;
+ default:
+ return false;
+ }
+}
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_MIPS_CONSTANTS_MIPS_H_
diff --git a/src/codegen/mips/cpu-mips.cc b/src/codegen/mips/cpu-mips.cc
new file mode 100644
index 0000000..a7120d1
--- /dev/null
+++ b/src/codegen/mips/cpu-mips.cc
@@ -0,0 +1,45 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// CPU specific code for arm independent of OS goes here.
+
+#include <sys/syscall.h>
+#include <unistd.h>
+
+#ifdef __mips
+#include <asm/cachectl.h>
+#endif // #ifdef __mips
+
+#if V8_TARGET_ARCH_MIPS
+
+#include "src/codegen/cpu-features.h"
+
+namespace v8 {
+namespace internal {
+
+void CpuFeatures::FlushICache(void* start, size_t size) {
+#if !defined(USE_SIMULATOR)
+ // Nothing to do, flushing no instructions.
+ if (size == 0) {
+ return;
+ }
+
+#if defined(ANDROID)
+ // Bionic cacheflush can typically run in userland, avoiding kernel call.
+ char* end = reinterpret_cast<char*>(start) + size;
+ cacheflush(reinterpret_cast<intptr_t>(start), reinterpret_cast<intptr_t>(end),
+ 0);
+#else // ANDROID
+ int res;
+ // See http://www.linux-mips.org/wiki/Cacheflush_Syscall.
+ res = syscall(__NR_cacheflush, start, size, ICACHE);
+ if (res) FATAL("Failed to flush the instruction cache");
+#endif // ANDROID
+#endif // !USE_SIMULATOR.
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_MIPS
diff --git a/src/codegen/mips/interface-descriptors-mips.cc b/src/codegen/mips/interface-descriptors-mips.cc
new file mode 100644
index 0000000..75835e6
--- /dev/null
+++ b/src/codegen/mips/interface-descriptors-mips.cc
@@ -0,0 +1,310 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#if V8_TARGET_ARCH_MIPS
+
+#include "src/codegen/interface-descriptors.h"
+
+#include "src/execution/frames.h"
+
+namespace v8 {
+namespace internal {
+
+const Register CallInterfaceDescriptor::ContextRegister() { return cp; }
+
+void CallInterfaceDescriptor::DefaultInitializePlatformSpecific(
+ CallInterfaceDescriptorData* data, int register_parameter_count) {
+ const Register default_stub_registers[] = {a0, a1, a2, a3, t0};
+ CHECK_LE(static_cast<size_t>(register_parameter_count),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(register_parameter_count,
+ default_stub_registers);
+}
+
+// On MIPS it is not allowed to use odd numbered floating point registers
+// (e.g. f1, f3, etc.) for parameters. This can happen if we use
+// DefaultInitializePlatformSpecific to assign float registers for parameters.
+// E.g if fourth parameter goes to float register, f7 would be assigned for
+// parameter (a3 casted to int is 7).
+bool CallInterfaceDescriptor::IsValidFloatParameterRegister(Register reg) {
+ return reg.code() % 2 == 0;
+}
+
+void WasmI32AtomicWait32Descriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ const Register default_stub_registers[] = {a0, a1, a2, a3};
+ CHECK_EQ(static_cast<size_t>(kParameterCount),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(kParameterCount, default_stub_registers);
+}
+
+void WasmI64AtomicWait32Descriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ const Register default_stub_registers[] = {a0, a1, a2, a3, t0};
+ CHECK_EQ(static_cast<size_t>(kParameterCount - kStackArgumentsCount),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(kParameterCount - kStackArgumentsCount,
+ default_stub_registers);
+}
+
+void RecordWriteDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ const Register default_stub_registers[] = {a0, a1, a2, a3, kReturnRegister0};
+
+ data->RestrictAllocatableRegisters(default_stub_registers,
+ arraysize(default_stub_registers));
+
+ CHECK_LE(static_cast<size_t>(kParameterCount),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(kParameterCount, default_stub_registers);
+}
+
+void EphemeronKeyBarrierDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ const Register default_stub_registers[] = {a0, a1, a2, a3, kReturnRegister0};
+
+ data->RestrictAllocatableRegisters(default_stub_registers,
+ arraysize(default_stub_registers));
+
+ CHECK_LE(static_cast<size_t>(kParameterCount),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(kParameterCount, default_stub_registers);
+}
+
+const Register LoadDescriptor::ReceiverRegister() { return a1; }
+const Register LoadDescriptor::NameRegister() { return a2; }
+const Register LoadDescriptor::SlotRegister() { return a0; }
+
+const Register LoadWithVectorDescriptor::VectorRegister() { return a3; }
+
+const Register
+LoadWithReceiverAndVectorDescriptor::LookupStartObjectRegister() {
+ return t0;
+}
+
+const Register StoreDescriptor::ReceiverRegister() { return a1; }
+const Register StoreDescriptor::NameRegister() { return a2; }
+const Register StoreDescriptor::ValueRegister() { return a0; }
+const Register StoreDescriptor::SlotRegister() { return t0; }
+
+const Register StoreWithVectorDescriptor::VectorRegister() { return a3; }
+
+const Register StoreTransitionDescriptor::SlotRegister() { return t0; }
+const Register StoreTransitionDescriptor::VectorRegister() { return a3; }
+const Register StoreTransitionDescriptor::MapRegister() { return t1; }
+
+const Register ApiGetterDescriptor::HolderRegister() { return a0; }
+const Register ApiGetterDescriptor::CallbackRegister() { return a3; }
+
+const Register GrowArrayElementsDescriptor::ObjectRegister() { return a0; }
+const Register GrowArrayElementsDescriptor::KeyRegister() { return a3; }
+
+// static
+const Register TypeConversionDescriptor::ArgumentRegister() { return a0; }
+
+void TypeofDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {a3};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallTrampolineDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // a1: target
+ // a0: number of arguments
+ Register registers[] = {a1, a0};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // a0 : number of arguments (on the stack, not including receiver)
+ // a1 : the target to call
+ // t0 : arguments list length (untagged)
+ // a2 : arguments list (FixedArray)
+ Register registers[] = {a1, a0, t0, a2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallForwardVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // a1: the target to call
+ // a0: number of arguments
+ // a2: start index (to support rest parameters)
+ Register registers[] = {a1, a0, a2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallFunctionTemplateDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // a1 : function template info
+ // a0 : number of arguments (on the stack, not including receiver)
+ Register registers[] = {a1, a0};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallWithSpreadDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // a0 : number of arguments (on the stack, not including receiver)
+ // a1 : the target to call
+ // a2 : the object to spread
+ Register registers[] = {a1, a0, a2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallWithArrayLikeDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // a1 : the target to call
+ // a2 : the arguments list
+ Register registers[] = {a1, a2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // a0 : number of arguments (on the stack, not including receiver)
+ // a1 : the target to call
+ // a3 : the new target
+ // t0 : arguments list length (untagged)
+ // a2 : arguments list (FixedArray)
+ Register registers[] = {a1, a3, a0, t0, a2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructForwardVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // a1: the target to call
+ // a3: new target
+ // a0: number of arguments
+ // a2: start index (to support rest parameters)
+ Register registers[] = {a1, a3, a0, a2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructWithSpreadDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // a0 : number of arguments (on the stack, not including receiver)
+ // a1 : the target to call
+ // a3 : the new target
+ // a2 : the object to spread
+ Register registers[] = {a1, a3, a0, a2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructWithArrayLikeDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // a1 : the target to call
+ // a3 : the new target
+ // a2 : the arguments list
+ Register registers[] = {a1, a3, a2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructStubDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // a1: target
+ // a3: new target
+ // a0: number of arguments
+ // a2: allocation site or undefined
+ Register registers[] = {a1, a3, a0, a2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void AbortDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {a0};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CompareDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {a1, a0};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void BinaryOpDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {a1, a0};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ArgumentsAdaptorDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ a1, // JSFunction
+ a3, // the new target
+ a0, // actual number of arguments
+ a2, // expected number of arguments
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ApiCallbackDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ a1, // kApiFunctionAddress
+ a2, // kArgc
+ a3, // kCallData
+ a0, // kHolder
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void InterpreterDispatchDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ kInterpreterAccumulatorRegister, kInterpreterBytecodeOffsetRegister,
+ kInterpreterBytecodeArrayRegister, kInterpreterDispatchTableRegister};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void InterpreterPushArgsThenCallDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ a0, // argument count (not including receiver)
+ a2, // address of first argument
+ a1 // the target callable to be call
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void InterpreterPushArgsThenConstructDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ a0, // argument count (not including receiver)
+ t4, // address of the first argument
+ a1, // constructor to call
+ a3, // new target
+ a2, // allocation site feedback if available, undefined otherwise
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ResumeGeneratorDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ v0, // the value to pass to the generator
+ a1 // the JSGeneratorObject to resume
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void FrameDropperTrampolineDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ a1, // loaded new FP
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void RunMicrotasksEntryDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {a0, a1};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_MIPS
diff --git a/src/codegen/mips/macro-assembler-mips.cc b/src/codegen/mips/macro-assembler-mips.cc
new file mode 100644
index 0000000..37a6aca
--- /dev/null
+++ b/src/codegen/mips/macro-assembler-mips.cc
@@ -0,0 +1,5626 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <limits.h> // For LONG_MIN, LONG_MAX.
+
+#if V8_TARGET_ARCH_MIPS
+
+#include "src/base/bits.h"
+#include "src/base/division-by-constant.h"
+#include "src/codegen/assembler-inl.h"
+#include "src/codegen/callable.h"
+#include "src/codegen/code-factory.h"
+#include "src/codegen/external-reference-table.h"
+#include "src/codegen/macro-assembler.h"
+#include "src/codegen/register-configuration.h"
+#include "src/debug/debug.h"
+#include "src/execution/frames-inl.h"
+#include "src/heap/memory-chunk.h"
+#include "src/init/bootstrapper.h"
+#include "src/logging/counters.h"
+#include "src/objects/heap-number.h"
+#include "src/runtime/runtime.h"
+#include "src/snapshot/embedded/embedded-data.h"
+#include "src/snapshot/snapshot.h"
+#include "src/wasm/wasm-code-manager.h"
+
+// Satisfy cpplint check, but don't include platform-specific header. It is
+// included recursively via macro-assembler.h.
+#if 0
+#include "src/codegen/mips/macro-assembler-mips.h"
+#endif
+
+namespace v8 {
+namespace internal {
+
+static inline bool IsZero(const Operand& rt) {
+ if (rt.is_reg()) {
+ return rt.rm() == zero_reg;
+ } else {
+ return rt.immediate() == 0;
+ }
+}
+
+int TurboAssembler::RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
+ Register exclusion1,
+ Register exclusion2,
+ Register exclusion3) const {
+ int bytes = 0;
+ RegList exclusions = 0;
+ if (exclusion1 != no_reg) {
+ exclusions |= exclusion1.bit();
+ if (exclusion2 != no_reg) {
+ exclusions |= exclusion2.bit();
+ if (exclusion3 != no_reg) {
+ exclusions |= exclusion3.bit();
+ }
+ }
+ }
+
+ RegList list = kJSCallerSaved & ~exclusions;
+ bytes += NumRegs(list) * kPointerSize;
+
+ if (fp_mode == kSaveFPRegs) {
+ bytes += NumRegs(kCallerSavedFPU) * kDoubleSize;
+ }
+
+ return bytes;
+}
+
+int TurboAssembler::PushCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1,
+ Register exclusion2, Register exclusion3) {
+ int bytes = 0;
+ RegList exclusions = 0;
+ if (exclusion1 != no_reg) {
+ exclusions |= exclusion1.bit();
+ if (exclusion2 != no_reg) {
+ exclusions |= exclusion2.bit();
+ if (exclusion3 != no_reg) {
+ exclusions |= exclusion3.bit();
+ }
+ }
+ }
+
+ RegList list = kJSCallerSaved & ~exclusions;
+ MultiPush(list);
+ bytes += NumRegs(list) * kPointerSize;
+
+ if (fp_mode == kSaveFPRegs) {
+ MultiPushFPU(kCallerSavedFPU);
+ bytes += NumRegs(kCallerSavedFPU) * kDoubleSize;
+ }
+
+ return bytes;
+}
+
+int TurboAssembler::PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1,
+ Register exclusion2, Register exclusion3) {
+ int bytes = 0;
+ if (fp_mode == kSaveFPRegs) {
+ MultiPopFPU(kCallerSavedFPU);
+ bytes += NumRegs(kCallerSavedFPU) * kDoubleSize;
+ }
+
+ RegList exclusions = 0;
+ if (exclusion1 != no_reg) {
+ exclusions |= exclusion1.bit();
+ if (exclusion2 != no_reg) {
+ exclusions |= exclusion2.bit();
+ if (exclusion3 != no_reg) {
+ exclusions |= exclusion3.bit();
+ }
+ }
+ }
+
+ RegList list = kJSCallerSaved & ~exclusions;
+ MultiPop(list);
+ bytes += NumRegs(list) * kPointerSize;
+
+ return bytes;
+}
+
+void TurboAssembler::LoadRoot(Register destination, RootIndex index) {
+ lw(destination,
+ MemOperand(kRootRegister, RootRegisterOffsetForRootIndex(index)));
+}
+
+void TurboAssembler::LoadRoot(Register destination, RootIndex index,
+ Condition cond, Register src1,
+ const Operand& src2) {
+ Branch(2, NegateCondition(cond), src1, src2);
+ lw(destination,
+ MemOperand(kRootRegister, RootRegisterOffsetForRootIndex(index)));
+}
+
+void TurboAssembler::PushCommonFrame(Register marker_reg) {
+ if (marker_reg.is_valid()) {
+ Push(ra, fp, marker_reg);
+ Addu(fp, sp, Operand(kPointerSize));
+ } else {
+ Push(ra, fp);
+ mov(fp, sp);
+ }
+}
+
+void TurboAssembler::PushStandardFrame(Register function_reg) {
+ int offset = -StandardFrameConstants::kContextOffset;
+ if (function_reg.is_valid()) {
+ Push(ra, fp, cp, function_reg, kJavaScriptCallArgCountRegister);
+ offset += 2 * kPointerSize;
+ } else {
+ Push(ra, fp, cp, kJavaScriptCallArgCountRegister);
+ offset += kPointerSize;
+ }
+ Addu(fp, sp, Operand(offset));
+}
+
+int MacroAssembler::SafepointRegisterStackIndex(int reg_code) {
+ // The registers are pushed starting with the highest encoding,
+ // which means that lowest encodings are closest to the stack pointer.
+ return kSafepointRegisterStackIndexMap[reg_code];
+}
+
+// Clobbers object, dst, value, and ra, if (ra_status == kRAHasBeenSaved)
+// The register 'object' contains a heap object pointer. The heap object
+// tag is shifted away.
+void MacroAssembler::RecordWriteField(Register object, int offset,
+ Register value, Register dst,
+ RAStatus ra_status,
+ SaveFPRegsMode save_fp,
+ RememberedSetAction remembered_set_action,
+ SmiCheck smi_check) {
+ DCHECK(!AreAliased(value, dst, t8, object));
+ // First, check if a write barrier is even needed. The tests below
+ // catch stores of Smis.
+ Label done;
+
+ // Skip barrier if writing a smi.
+ if (smi_check == INLINE_SMI_CHECK) {
+ JumpIfSmi(value, &done);
+ }
+
+ // Although the object register is tagged, the offset is relative to the start
+ // of the object, so so offset must be a multiple of kPointerSize.
+ DCHECK(IsAligned(offset, kPointerSize));
+
+ Addu(dst, object, Operand(offset - kHeapObjectTag));
+ if (emit_debug_code()) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Label ok;
+ And(t8, dst, Operand(kPointerSize - 1));
+ Branch(&ok, eq, t8, Operand(zero_reg));
+ stop();
+ bind(&ok);
+ }
+
+ RecordWrite(object, dst, value, ra_status, save_fp, remembered_set_action,
+ OMIT_SMI_CHECK);
+
+ bind(&done);
+
+ // Clobber clobbered input registers when running with the debug-code flag
+ // turned on to provoke errors.
+ if (emit_debug_code()) {
+ li(value, Operand(bit_cast<int32_t>(kZapValue + 4)));
+ li(dst, Operand(bit_cast<int32_t>(kZapValue + 8)));
+ }
+}
+
+void TurboAssembler::SaveRegisters(RegList registers) {
+ DCHECK_GT(NumRegs(registers), 0);
+ RegList regs = 0;
+ for (int i = 0; i < Register::kNumRegisters; ++i) {
+ if ((registers >> i) & 1u) {
+ regs |= Register::from_code(i).bit();
+ }
+ }
+ MultiPush(regs);
+}
+
+void TurboAssembler::RestoreRegisters(RegList registers) {
+ DCHECK_GT(NumRegs(registers), 0);
+ RegList regs = 0;
+ for (int i = 0; i < Register::kNumRegisters; ++i) {
+ if ((registers >> i) & 1u) {
+ regs |= Register::from_code(i).bit();
+ }
+ }
+ MultiPop(regs);
+}
+
+void TurboAssembler::CallEphemeronKeyBarrier(Register object, Register address,
+ SaveFPRegsMode fp_mode) {
+ EphemeronKeyBarrierDescriptor descriptor;
+ RegList registers = descriptor.allocatable_registers();
+
+ SaveRegisters(registers);
+
+ Register object_parameter(
+ descriptor.GetRegisterParameter(EphemeronKeyBarrierDescriptor::kObject));
+ Register slot_parameter(descriptor.GetRegisterParameter(
+ EphemeronKeyBarrierDescriptor::kSlotAddress));
+ Register fp_mode_parameter(
+ descriptor.GetRegisterParameter(EphemeronKeyBarrierDescriptor::kFPMode));
+
+ Push(object);
+ Push(address);
+
+ Pop(slot_parameter);
+ Pop(object_parameter);
+
+ Move(fp_mode_parameter, Smi::FromEnum(fp_mode));
+ Call(isolate()->builtins()->builtin_handle(Builtins::kEphemeronKeyBarrier),
+ RelocInfo::CODE_TARGET);
+ RestoreRegisters(registers);
+}
+
+void TurboAssembler::CallRecordWriteStub(
+ Register object, Register address,
+ RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode) {
+ CallRecordWriteStub(
+ object, address, remembered_set_action, fp_mode,
+ isolate()->builtins()->builtin_handle(Builtins::kRecordWrite),
+ kNullAddress);
+}
+
+void TurboAssembler::CallRecordWriteStub(
+ Register object, Register address,
+ RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode,
+ Address wasm_target) {
+ CallRecordWriteStub(object, address, remembered_set_action, fp_mode,
+ Handle<Code>::null(), wasm_target);
+}
+
+void TurboAssembler::CallRecordWriteStub(
+ Register object, Register address,
+ RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode,
+ Handle<Code> code_target, Address wasm_target) {
+ DCHECK_NE(code_target.is_null(), wasm_target == kNullAddress);
+ // TODO(albertnetymk): For now we ignore remembered_set_action and fp_mode,
+ // i.e. always emit remember set and save FP registers in RecordWriteStub. If
+ // large performance regression is observed, we should use these values to
+ // avoid unnecessary work.
+
+ RecordWriteDescriptor descriptor;
+ RegList registers = descriptor.allocatable_registers();
+
+ SaveRegisters(registers);
+ Register object_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kObject));
+ Register slot_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kSlot));
+ Register remembered_set_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kRememberedSet));
+ Register fp_mode_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kFPMode));
+
+ Push(object);
+ Push(address);
+
+ Pop(slot_parameter);
+ Pop(object_parameter);
+
+ Move(remembered_set_parameter, Smi::FromEnum(remembered_set_action));
+ Move(fp_mode_parameter, Smi::FromEnum(fp_mode));
+ if (code_target.is_null()) {
+ Call(wasm_target, RelocInfo::WASM_STUB_CALL);
+ } else {
+ Call(code_target, RelocInfo::CODE_TARGET);
+ }
+
+ RestoreRegisters(registers);
+}
+
+// Clobbers object, address, value, and ra, if (ra_status == kRAHasBeenSaved)
+// The register 'object' contains a heap object pointer. The heap object
+// tag is shifted away.
+void MacroAssembler::RecordWrite(Register object, Register address,
+ Register value, RAStatus ra_status,
+ SaveFPRegsMode fp_mode,
+ RememberedSetAction remembered_set_action,
+ SmiCheck smi_check) {
+ DCHECK(!AreAliased(object, address, value, t8));
+ DCHECK(!AreAliased(object, address, value, t9));
+
+ if (emit_debug_code()) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ lw(scratch, MemOperand(address));
+ Assert(eq, AbortReason::kWrongAddressOrValuePassedToRecordWrite, scratch,
+ Operand(value));
+ }
+
+ if ((remembered_set_action == OMIT_REMEMBERED_SET &&
+ !FLAG_incremental_marking) ||
+ FLAG_disable_write_barriers) {
+ return;
+ }
+
+ // First, check if a write barrier is even needed. The tests below
+ // catch stores of smis and stores into the young generation.
+ Label done;
+
+ if (smi_check == INLINE_SMI_CHECK) {
+ DCHECK_EQ(0, kSmiTag);
+ JumpIfSmi(value, &done);
+ }
+
+ CheckPageFlag(value,
+ value, // Used as scratch.
+ MemoryChunk::kPointersToHereAreInterestingMask, eq, &done);
+ CheckPageFlag(object,
+ value, // Used as scratch.
+ MemoryChunk::kPointersFromHereAreInterestingMask, eq, &done);
+
+ // Record the actual write.
+ if (ra_status == kRAHasNotBeenSaved) {
+ push(ra);
+ }
+ CallRecordWriteStub(object, address, remembered_set_action, fp_mode);
+ if (ra_status == kRAHasNotBeenSaved) {
+ pop(ra);
+ }
+
+ bind(&done);
+
+ // Clobber clobbered registers when running with the debug-code flag
+ // turned on to provoke errors.
+ if (emit_debug_code()) {
+ li(address, Operand(bit_cast<int32_t>(kZapValue + 12)));
+ li(value, Operand(bit_cast<int32_t>(kZapValue + 16)));
+ }
+}
+
+// ---------------------------------------------------------------------------
+// Instruction macros.
+
+void TurboAssembler::Addu(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ addu(rd, rs, rt.rm());
+ } else {
+ if (is_int16(rt.immediate()) && !MustUseReg(rt.rmode())) {
+ addiu(rd, rs, rt.immediate());
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ addu(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Subu(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ subu(rd, rs, rt.rm());
+ } else {
+ if (is_int16(-rt.immediate()) && !MustUseReg(rt.rmode())) {
+ addiu(rd, rs, -rt.immediate()); // No subiu instr, use addiu(x, y, -imm).
+ } else if (!(-rt.immediate() & kHiMask) &&
+ !MustUseReg(rt.rmode())) { // Use load
+ // -imm and addu for cases where loading -imm generates one instruction.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, -rt.immediate());
+ addu(rd, rs, scratch);
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ subu(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Mul(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ if (IsMipsArchVariant(kLoongson)) {
+ mult(rs, rt.rm());
+ mflo(rd);
+ } else {
+ mul(rd, rs, rt.rm());
+ }
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ if (IsMipsArchVariant(kLoongson)) {
+ mult(rs, scratch);
+ mflo(rd);
+ } else {
+ mul(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Mul(Register rd_hi, Register rd_lo, Register rs,
+ const Operand& rt) {
+ if (rt.is_reg()) {
+ if (!IsMipsArchVariant(kMips32r6)) {
+ mult(rs, rt.rm());
+ mflo(rd_lo);
+ mfhi(rd_hi);
+ } else {
+ if (rd_lo == rs) {
+ DCHECK(rd_hi != rs);
+ DCHECK(rd_hi != rt.rm() && rd_lo != rt.rm());
+ muh(rd_hi, rs, rt.rm());
+ mul(rd_lo, rs, rt.rm());
+ } else {
+ DCHECK(rd_hi != rt.rm() && rd_lo != rt.rm());
+ mul(rd_lo, rs, rt.rm());
+ muh(rd_hi, rs, rt.rm());
+ }
+ }
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ if (!IsMipsArchVariant(kMips32r6)) {
+ mult(rs, scratch);
+ mflo(rd_lo);
+ mfhi(rd_hi);
+ } else {
+ if (rd_lo == rs) {
+ DCHECK(rd_hi != rs);
+ DCHECK(rd_hi != scratch && rd_lo != scratch);
+ muh(rd_hi, rs, scratch);
+ mul(rd_lo, rs, scratch);
+ } else {
+ DCHECK(rd_hi != scratch && rd_lo != scratch);
+ mul(rd_lo, rs, scratch);
+ muh(rd_hi, rs, scratch);
+ }
+ }
+ }
+}
+
+void TurboAssembler::Mulu(Register rd_hi, Register rd_lo, Register rs,
+ const Operand& rt) {
+ Register reg = no_reg;
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ if (rt.is_reg()) {
+ reg = rt.rm();
+ } else {
+ DCHECK(rs != scratch);
+ reg = scratch;
+ li(reg, rt);
+ }
+
+ if (!IsMipsArchVariant(kMips32r6)) {
+ multu(rs, reg);
+ mflo(rd_lo);
+ mfhi(rd_hi);
+ } else {
+ if (rd_lo == rs) {
+ DCHECK(rd_hi != rs);
+ DCHECK(rd_hi != reg && rd_lo != reg);
+ muhu(rd_hi, rs, reg);
+ mulu(rd_lo, rs, reg);
+ } else {
+ DCHECK(rd_hi != reg && rd_lo != reg);
+ mulu(rd_lo, rs, reg);
+ muhu(rd_hi, rs, reg);
+ }
+ }
+}
+
+void TurboAssembler::Mulh(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ if (!IsMipsArchVariant(kMips32r6)) {
+ mult(rs, rt.rm());
+ mfhi(rd);
+ } else {
+ muh(rd, rs, rt.rm());
+ }
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ if (!IsMipsArchVariant(kMips32r6)) {
+ mult(rs, scratch);
+ mfhi(rd);
+ } else {
+ muh(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Mult(Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ mult(rs, rt.rm());
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ mult(rs, scratch);
+ }
+}
+
+void TurboAssembler::Mulhu(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ if (!IsMipsArchVariant(kMips32r6)) {
+ multu(rs, rt.rm());
+ mfhi(rd);
+ } else {
+ muhu(rd, rs, rt.rm());
+ }
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ if (!IsMipsArchVariant(kMips32r6)) {
+ multu(rs, scratch);
+ mfhi(rd);
+ } else {
+ muhu(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Multu(Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ multu(rs, rt.rm());
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ multu(rs, scratch);
+ }
+}
+
+void TurboAssembler::Div(Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ div(rs, rt.rm());
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ div(rs, scratch);
+ }
+}
+
+void TurboAssembler::Div(Register rem, Register res, Register rs,
+ const Operand& rt) {
+ if (rt.is_reg()) {
+ if (!IsMipsArchVariant(kMips32r6)) {
+ div(rs, rt.rm());
+ mflo(res);
+ mfhi(rem);
+ } else {
+ div(res, rs, rt.rm());
+ mod(rem, rs, rt.rm());
+ }
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ if (!IsMipsArchVariant(kMips32r6)) {
+ div(rs, scratch);
+ mflo(res);
+ mfhi(rem);
+ } else {
+ div(res, rs, scratch);
+ mod(rem, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Div(Register res, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ if (!IsMipsArchVariant(kMips32r6)) {
+ div(rs, rt.rm());
+ mflo(res);
+ } else {
+ div(res, rs, rt.rm());
+ }
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ if (!IsMipsArchVariant(kMips32r6)) {
+ div(rs, scratch);
+ mflo(res);
+ } else {
+ div(res, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Mod(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ if (!IsMipsArchVariant(kMips32r6)) {
+ div(rs, rt.rm());
+ mfhi(rd);
+ } else {
+ mod(rd, rs, rt.rm());
+ }
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ if (!IsMipsArchVariant(kMips32r6)) {
+ div(rs, scratch);
+ mfhi(rd);
+ } else {
+ mod(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Modu(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ if (!IsMipsArchVariant(kMips32r6)) {
+ divu(rs, rt.rm());
+ mfhi(rd);
+ } else {
+ modu(rd, rs, rt.rm());
+ }
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ if (!IsMipsArchVariant(kMips32r6)) {
+ divu(rs, scratch);
+ mfhi(rd);
+ } else {
+ modu(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Divu(Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ divu(rs, rt.rm());
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ divu(rs, scratch);
+ }
+}
+
+void TurboAssembler::Divu(Register res, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ if (!IsMipsArchVariant(kMips32r6)) {
+ divu(rs, rt.rm());
+ mflo(res);
+ } else {
+ divu(res, rs, rt.rm());
+ }
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ if (!IsMipsArchVariant(kMips32r6)) {
+ divu(rs, scratch);
+ mflo(res);
+ } else {
+ divu(res, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::And(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ and_(rd, rs, rt.rm());
+ } else {
+ if (is_uint16(rt.immediate()) && !MustUseReg(rt.rmode())) {
+ andi(rd, rs, rt.immediate());
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ and_(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Or(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ or_(rd, rs, rt.rm());
+ } else {
+ if (is_uint16(rt.immediate()) && !MustUseReg(rt.rmode())) {
+ ori(rd, rs, rt.immediate());
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ or_(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Xor(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ xor_(rd, rs, rt.rm());
+ } else {
+ if (is_uint16(rt.immediate()) && !MustUseReg(rt.rmode())) {
+ xori(rd, rs, rt.immediate());
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ xor_(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Nor(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ nor(rd, rs, rt.rm());
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ nor(rd, rs, scratch);
+ }
+}
+
+void TurboAssembler::Neg(Register rs, const Operand& rt) {
+ subu(rs, zero_reg, rt.rm());
+}
+
+void TurboAssembler::Slt(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ slt(rd, rs, rt.rm());
+ } else {
+ if (is_int16(rt.immediate()) && !MustUseReg(rt.rmode())) {
+ slti(rd, rs, rt.immediate());
+ } else {
+ // li handles the relocation.
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ UseScratchRegisterScope temps(this);
+ Register scratch = rd == at ? t8 : temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ slt(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Sltu(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ sltu(rd, rs, rt.rm());
+ } else {
+ const uint32_t int16_min = std::numeric_limits<int16_t>::min();
+ if (is_uint15(rt.immediate()) && !MustUseReg(rt.rmode())) {
+ // Imm range is: [0, 32767].
+ sltiu(rd, rs, rt.immediate());
+ } else if (is_uint15(rt.immediate() - int16_min) &&
+ !MustUseReg(rt.rmode())) {
+ // Imm range is: [max_unsigned-32767,max_unsigned].
+ sltiu(rd, rs, static_cast<uint16_t>(rt.immediate()));
+ } else {
+ // li handles the relocation.
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ UseScratchRegisterScope temps(this);
+ Register scratch = rd == at ? t8 : temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ sltu(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Sle(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ slt(rd, rt.rm(), rs);
+ } else {
+ // li handles the relocation.
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ slt(rd, scratch, rs);
+ }
+ xori(rd, rd, 1);
+}
+
+void TurboAssembler::Sleu(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ sltu(rd, rt.rm(), rs);
+ } else {
+ // li handles the relocation.
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ sltu(rd, scratch, rs);
+ }
+ xori(rd, rd, 1);
+}
+
+void TurboAssembler::Sge(Register rd, Register rs, const Operand& rt) {
+ Slt(rd, rs, rt);
+ xori(rd, rd, 1);
+}
+
+void TurboAssembler::Sgeu(Register rd, Register rs, const Operand& rt) {
+ Sltu(rd, rs, rt);
+ xori(rd, rd, 1);
+}
+
+void TurboAssembler::Sgt(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ slt(rd, rt.rm(), rs);
+ } else {
+ // li handles the relocation.
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ slt(rd, scratch, rs);
+ }
+}
+
+void TurboAssembler::Sgtu(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ sltu(rd, rt.rm(), rs);
+ } else {
+ // li handles the relocation.
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ sltu(rd, scratch, rs);
+ }
+}
+
+void TurboAssembler::Ror(Register rd, Register rs, const Operand& rt) {
+ if (IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6)) {
+ if (rt.is_reg()) {
+ rotrv(rd, rs, rt.rm());
+ } else {
+ rotr(rd, rs, rt.immediate() & 0x1F);
+ }
+ } else {
+ if (rt.is_reg()) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
+ subu(scratch, zero_reg, rt.rm());
+ sllv(scratch, rs, scratch);
+ srlv(rd, rs, rt.rm());
+ or_(rd, rd, scratch);
+ } else {
+ if (rt.immediate() == 0) {
+ srl(rd, rs, 0);
+ } else {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ srl(scratch, rs, rt.immediate() & 0x1F);
+ sll(rd, rs, (0x20 - (rt.immediate() & 0x1F)) & 0x1F);
+ or_(rd, rd, scratch);
+ }
+ }
+ }
+}
+
+void MacroAssembler::Pref(int32_t hint, const MemOperand& rs) {
+ if (IsMipsArchVariant(kLoongson)) {
+ lw(zero_reg, rs);
+ } else {
+ pref(hint, rs);
+ }
+}
+
+void TurboAssembler::Lsa(Register rd, Register rt, Register rs, uint8_t sa,
+ Register scratch) {
+ DCHECK(sa >= 1 && sa <= 31);
+ if (IsMipsArchVariant(kMips32r6) && sa <= 4) {
+ lsa(rd, rt, rs, sa - 1);
+ } else {
+ Register tmp = rd == rt ? scratch : rd;
+ DCHECK(tmp != rt);
+ sll(tmp, rs, sa);
+ Addu(rd, rt, tmp);
+ }
+}
+
+void TurboAssembler::Bovc(Register rs, Register rt, Label* L) {
+ if (is_trampoline_emitted()) {
+ Label skip;
+ bnvc(rs, rt, &skip);
+ BranchLong(L, PROTECT);
+ bind(&skip);
+ } else {
+ bovc(rs, rt, L);
+ }
+}
+
+void TurboAssembler::Bnvc(Register rs, Register rt, Label* L) {
+ if (is_trampoline_emitted()) {
+ Label skip;
+ bovc(rs, rt, &skip);
+ BranchLong(L, PROTECT);
+ bind(&skip);
+ } else {
+ bnvc(rs, rt, L);
+ }
+}
+
+// ------------Pseudo-instructions-------------
+
+// Word Swap Byte
+void TurboAssembler::ByteSwapSigned(Register dest, Register src,
+ int operand_size) {
+ DCHECK(operand_size == 2 || operand_size == 4);
+
+ if (IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6)) {
+ if (operand_size == 2) {
+ wsbh(dest, src);
+ seh(dest, dest);
+ } else {
+ wsbh(dest, src);
+ rotr(dest, dest, 16);
+ }
+ } else if (IsMipsArchVariant(kMips32r1) || IsMipsArchVariant(kLoongson)) {
+ if (operand_size == 2) {
+ DCHECK(src != at && dest != at);
+ srl(at, src, 8);
+ andi(at, at, 0xFF);
+ sll(dest, src, 8);
+ or_(dest, dest, at);
+
+ // Sign-extension
+ sll(dest, dest, 16);
+ sra(dest, dest, 16);
+ } else {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Register tmp = at;
+ Register tmp2 = t8;
+ DCHECK(dest != tmp && dest != tmp2);
+ DCHECK(src != tmp && src != tmp2);
+
+ andi(tmp2, src, 0xFF);
+ sll(tmp, tmp2, 24);
+
+ andi(tmp2, src, 0xFF00);
+ sll(tmp2, tmp2, 8);
+ or_(tmp, tmp, tmp2);
+
+ srl(tmp2, src, 8);
+ andi(tmp2, tmp2, 0xFF00);
+ or_(tmp, tmp, tmp2);
+
+ srl(tmp2, src, 24);
+ or_(dest, tmp, tmp2);
+ }
+ }
+}
+
+void TurboAssembler::ByteSwapUnsigned(Register dest, Register src,
+ int operand_size) {
+ DCHECK_EQ(operand_size, 2);
+
+ if (IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6)) {
+ wsbh(dest, src);
+ andi(dest, dest, 0xFFFF);
+ } else if (IsMipsArchVariant(kMips32r1) || IsMipsArchVariant(kLoongson)) {
+ DCHECK(src != at && dest != at);
+ srl(at, src, 8);
+ andi(at, at, 0xFF);
+ sll(dest, src, 8);
+ or_(dest, dest, at);
+
+ // Zero-extension
+ andi(dest, dest, 0xFFFF);
+ }
+}
+
+void TurboAssembler::Ulw(Register rd, const MemOperand& rs) {
+ DCHECK(rd != at);
+ DCHECK(rs.rm() != at);
+ if (IsMipsArchVariant(kMips32r6)) {
+ lw(rd, rs);
+ } else {
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r1) ||
+ IsMipsArchVariant(kLoongson));
+ DCHECK(kMipsLwrOffset <= 3 && kMipsLwlOffset <= 3);
+ MemOperand source = rs;
+ // Adjust offset for two accesses and check if offset + 3 fits into int16_t.
+ AdjustBaseAndOffset(&source, OffsetAccessType::TWO_ACCESSES, 3);
+ if (rd != source.rm()) {
+ lwr(rd, MemOperand(source.rm(), source.offset() + kMipsLwrOffset));
+ lwl(rd, MemOperand(source.rm(), source.offset() + kMipsLwlOffset));
+ } else {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ lwr(scratch, MemOperand(rs.rm(), rs.offset() + kMipsLwrOffset));
+ lwl(scratch, MemOperand(rs.rm(), rs.offset() + kMipsLwlOffset));
+ mov(rd, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Usw(Register rd, const MemOperand& rs) {
+ DCHECK(rd != at);
+ DCHECK(rs.rm() != at);
+ DCHECK(rd != rs.rm());
+ if (IsMipsArchVariant(kMips32r6)) {
+ sw(rd, rs);
+ } else {
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r1) ||
+ IsMipsArchVariant(kLoongson));
+ DCHECK(kMipsSwrOffset <= 3 && kMipsSwlOffset <= 3);
+ MemOperand source = rs;
+ // Adjust offset for two accesses and check if offset + 3 fits into int16_t.
+ AdjustBaseAndOffset(&source, OffsetAccessType::TWO_ACCESSES, 3);
+ swr(rd, MemOperand(source.rm(), source.offset() + kMipsSwrOffset));
+ swl(rd, MemOperand(source.rm(), source.offset() + kMipsSwlOffset));
+ }
+}
+
+void TurboAssembler::Ulh(Register rd, const MemOperand& rs) {
+ DCHECK(rd != at);
+ DCHECK(rs.rm() != at);
+ if (IsMipsArchVariant(kMips32r6)) {
+ lh(rd, rs);
+ } else {
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r1) ||
+ IsMipsArchVariant(kLoongson));
+ MemOperand source = rs;
+ // Adjust offset for two accesses and check if offset + 1 fits into int16_t.
+ AdjustBaseAndOffset(&source, OffsetAccessType::TWO_ACCESSES, 1);
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ if (source.rm() == scratch) {
+#if defined(V8_TARGET_LITTLE_ENDIAN)
+ lb(rd, MemOperand(source.rm(), source.offset() + 1));
+ lbu(scratch, source);
+#elif defined(V8_TARGET_BIG_ENDIAN)
+ lb(rd, source);
+ lbu(scratch, MemOperand(source.rm(), source.offset() + 1));
+#endif
+ } else {
+#if defined(V8_TARGET_LITTLE_ENDIAN)
+ lbu(scratch, source);
+ lb(rd, MemOperand(source.rm(), source.offset() + 1));
+#elif defined(V8_TARGET_BIG_ENDIAN)
+ lbu(scratch, MemOperand(source.rm(), source.offset() + 1));
+ lb(rd, source);
+#endif
+ }
+ sll(rd, rd, 8);
+ or_(rd, rd, scratch);
+ }
+}
+
+void TurboAssembler::Ulhu(Register rd, const MemOperand& rs) {
+ DCHECK(rd != at);
+ DCHECK(rs.rm() != at);
+ if (IsMipsArchVariant(kMips32r6)) {
+ lhu(rd, rs);
+ } else {
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r1) ||
+ IsMipsArchVariant(kLoongson));
+ MemOperand source = rs;
+ // Adjust offset for two accesses and check if offset + 1 fits into int16_t.
+ AdjustBaseAndOffset(&source, OffsetAccessType::TWO_ACCESSES, 1);
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ if (source.rm() == scratch) {
+#if defined(V8_TARGET_LITTLE_ENDIAN)
+ lbu(rd, MemOperand(source.rm(), source.offset() + 1));
+ lbu(scratch, source);
+#elif defined(V8_TARGET_BIG_ENDIAN)
+ lbu(rd, source);
+ lbu(scratch, MemOperand(source.rm(), source.offset() + 1));
+#endif
+ } else {
+#if defined(V8_TARGET_LITTLE_ENDIAN)
+ lbu(scratch, source);
+ lbu(rd, MemOperand(source.rm(), source.offset() + 1));
+#elif defined(V8_TARGET_BIG_ENDIAN)
+ lbu(scratch, MemOperand(source.rm(), source.offset() + 1));
+ lbu(rd, source);
+#endif
+ }
+ sll(rd, rd, 8);
+ or_(rd, rd, scratch);
+ }
+}
+
+void TurboAssembler::Ush(Register rd, const MemOperand& rs, Register scratch) {
+ DCHECK(rd != at);
+ DCHECK(rs.rm() != at);
+ DCHECK(rs.rm() != scratch);
+ DCHECK(scratch != at);
+ if (IsMipsArchVariant(kMips32r6)) {
+ sh(rd, rs);
+ } else {
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r1) ||
+ IsMipsArchVariant(kLoongson));
+ MemOperand source = rs;
+ // Adjust offset for two accesses and check if offset + 1 fits into int16_t.
+ AdjustBaseAndOffset(&source, OffsetAccessType::TWO_ACCESSES, 1);
+
+ if (scratch != rd) {
+ mov(scratch, rd);
+ }
+
+#if defined(V8_TARGET_LITTLE_ENDIAN)
+ sb(scratch, source);
+ srl(scratch, scratch, 8);
+ sb(scratch, MemOperand(source.rm(), source.offset() + 1));
+#elif defined(V8_TARGET_BIG_ENDIAN)
+ sb(scratch, MemOperand(source.rm(), source.offset() + 1));
+ srl(scratch, scratch, 8);
+ sb(scratch, source);
+#endif
+ }
+}
+
+void TurboAssembler::Ulwc1(FPURegister fd, const MemOperand& rs,
+ Register scratch) {
+ if (IsMipsArchVariant(kMips32r6)) {
+ lwc1(fd, rs);
+ } else {
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r1) ||
+ IsMipsArchVariant(kLoongson));
+ Ulw(scratch, rs);
+ mtc1(scratch, fd);
+ }
+}
+
+void TurboAssembler::Uswc1(FPURegister fd, const MemOperand& rs,
+ Register scratch) {
+ if (IsMipsArchVariant(kMips32r6)) {
+ swc1(fd, rs);
+ } else {
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r1) ||
+ IsMipsArchVariant(kLoongson));
+ mfc1(scratch, fd);
+ Usw(scratch, rs);
+ }
+}
+
+void TurboAssembler::Uldc1(FPURegister fd, const MemOperand& rs,
+ Register scratch) {
+ DCHECK(scratch != at);
+ if (IsMipsArchVariant(kMips32r6)) {
+ Ldc1(fd, rs);
+ } else {
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r1) ||
+ IsMipsArchVariant(kLoongson));
+ Ulw(scratch, MemOperand(rs.rm(), rs.offset() + Register::kMantissaOffset));
+ mtc1(scratch, fd);
+ Ulw(scratch, MemOperand(rs.rm(), rs.offset() + Register::kExponentOffset));
+ Mthc1(scratch, fd);
+ }
+}
+
+void TurboAssembler::Usdc1(FPURegister fd, const MemOperand& rs,
+ Register scratch) {
+ DCHECK(scratch != at);
+ if (IsMipsArchVariant(kMips32r6)) {
+ Sdc1(fd, rs);
+ } else {
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r1) ||
+ IsMipsArchVariant(kLoongson));
+ mfc1(scratch, fd);
+ Usw(scratch, MemOperand(rs.rm(), rs.offset() + Register::kMantissaOffset));
+ Mfhc1(scratch, fd);
+ Usw(scratch, MemOperand(rs.rm(), rs.offset() + Register::kExponentOffset));
+ }
+}
+
+void TurboAssembler::Ldc1(FPURegister fd, const MemOperand& src) {
+ // Workaround for non-8-byte alignment of HeapNumber, convert 64-bit
+ // load to two 32-bit loads.
+ {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ DCHECK(Register::kMantissaOffset <= 4 && Register::kExponentOffset <= 4);
+ MemOperand tmp = src;
+ AdjustBaseAndOffset(&tmp, OffsetAccessType::TWO_ACCESSES);
+ lwc1(fd, MemOperand(tmp.rm(), tmp.offset() + Register::kMantissaOffset));
+ if (IsFp32Mode()) { // fp32 mode.
+ FPURegister nextfpreg = FPURegister::from_code(fd.code() + 1);
+ lwc1(nextfpreg,
+ MemOperand(tmp.rm(), tmp.offset() + Register::kExponentOffset));
+ } else {
+ DCHECK(IsFp64Mode() || IsFpxxMode());
+ // Currently we support FPXX and FP64 on Mips32r2 and Mips32r6
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6));
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(src.rm() != scratch);
+ lw(scratch,
+ MemOperand(tmp.rm(), tmp.offset() + Register::kExponentOffset));
+ Mthc1(scratch, fd);
+ }
+ }
+ CheckTrampolinePoolQuick(1);
+}
+
+void TurboAssembler::Sdc1(FPURegister fd, const MemOperand& src) {
+ // Workaround for non-8-byte alignment of HeapNumber, convert 64-bit
+ // store to two 32-bit stores.
+ {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ DCHECK(Register::kMantissaOffset <= 4 && Register::kExponentOffset <= 4);
+ MemOperand tmp = src;
+ AdjustBaseAndOffset(&tmp, OffsetAccessType::TWO_ACCESSES);
+ swc1(fd, MemOperand(tmp.rm(), tmp.offset() + Register::kMantissaOffset));
+ if (IsFp32Mode()) { // fp32 mode.
+ FPURegister nextfpreg = FPURegister::from_code(fd.code() + 1);
+ swc1(nextfpreg,
+ MemOperand(tmp.rm(), tmp.offset() + Register::kExponentOffset));
+ } else {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ DCHECK(IsFp64Mode() || IsFpxxMode());
+ // Currently we support FPXX and FP64 on Mips32r2 and Mips32r6
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6));
+ DCHECK(src.rm() != t8);
+ Mfhc1(t8, fd);
+ sw(t8, MemOperand(tmp.rm(), tmp.offset() + Register::kExponentOffset));
+ }
+ }
+ CheckTrampolinePoolQuick(1);
+}
+
+void TurboAssembler::Lw(Register rd, const MemOperand& rs) {
+ MemOperand source = rs;
+ AdjustBaseAndOffset(&source);
+ lw(rd, source);
+}
+
+void TurboAssembler::Sw(Register rd, const MemOperand& rs) {
+ MemOperand dest = rs;
+ AdjustBaseAndOffset(&dest);
+ sw(rd, dest);
+}
+
+void TurboAssembler::Ll(Register rd, const MemOperand& rs) {
+ bool is_one_instruction = IsMipsArchVariant(kMips32r6)
+ ? is_int9(rs.offset())
+ : is_int16(rs.offset());
+ if (is_one_instruction) {
+ ll(rd, rs);
+ } else {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ li(scratch, rs.offset());
+ addu(scratch, scratch, rs.rm());
+ ll(rd, MemOperand(scratch, 0));
+ }
+}
+
+void TurboAssembler::Sc(Register rd, const MemOperand& rs) {
+ bool is_one_instruction = IsMipsArchVariant(kMips32r6)
+ ? is_int9(rs.offset())
+ : is_int16(rs.offset());
+ if (is_one_instruction) {
+ sc(rd, rs);
+ } else {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ li(scratch, rs.offset());
+ addu(scratch, scratch, rs.rm());
+ sc(rd, MemOperand(scratch, 0));
+ }
+}
+
+void TurboAssembler::li(Register dst, Handle<HeapObject> value, LiFlags mode) {
+ // TODO(jgruber,v8:8887): Also consider a root-relative load when generating
+ // non-isolate-independent code. In many cases it might be cheaper than
+ // embedding the relocatable value.
+ if (root_array_available_ && options().isolate_independent_code) {
+ IndirectLoadConstant(dst, value);
+ return;
+ }
+ li(dst, Operand(value), mode);
+}
+
+void TurboAssembler::li(Register dst, ExternalReference value, LiFlags mode) {
+ // TODO(jgruber,v8:8887): Also consider a root-relative load when generating
+ // non-isolate-independent code. In many cases it might be cheaper than
+ // embedding the relocatable value.
+ if (root_array_available_ && options().isolate_independent_code) {
+ IndirectLoadExternalReference(dst, value);
+ return;
+ }
+ li(dst, Operand(value), mode);
+}
+
+void TurboAssembler::li(Register dst, const StringConstantBase* string,
+ LiFlags mode) {
+ li(dst, Operand::EmbeddedStringConstant(string), mode);
+}
+
+void TurboAssembler::li(Register rd, Operand j, LiFlags mode) {
+ DCHECK(!j.is_reg());
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ if (!MustUseReg(j.rmode()) && mode == OPTIMIZE_SIZE) {
+ // Normal load of an immediate value which does not need Relocation Info.
+ if (is_int16(j.immediate())) {
+ addiu(rd, zero_reg, j.immediate());
+ } else if (!(j.immediate() & kHiMask)) {
+ ori(rd, zero_reg, j.immediate());
+ } else {
+ lui(rd, (j.immediate() >> kLuiShift) & kImm16Mask);
+ if (j.immediate() & kImm16Mask) {
+ ori(rd, rd, (j.immediate() & kImm16Mask));
+ }
+ }
+ } else {
+ int32_t immediate;
+ if (j.IsHeapObjectRequest()) {
+ RequestHeapObject(j.heap_object_request());
+ immediate = 0;
+ } else {
+ immediate = j.immediate();
+ }
+
+ if (MustUseReg(j.rmode())) {
+ RecordRelocInfo(j.rmode(), immediate);
+ }
+ // We always need the same number of instructions as we may need to patch
+ // this code to load another value which may need 2 instructions to load.
+
+ lui(rd, (immediate >> kLuiShift) & kImm16Mask);
+ ori(rd, rd, (immediate & kImm16Mask));
+ }
+}
+
+void TurboAssembler::MultiPush(RegList regs) {
+ int16_t num_to_push = base::bits::CountPopulation(regs);
+ int16_t stack_offset = num_to_push * kPointerSize;
+
+ Subu(sp, sp, Operand(stack_offset));
+ for (int16_t i = kNumRegisters - 1; i >= 0; i--) {
+ if ((regs & (1 << i)) != 0) {
+ stack_offset -= kPointerSize;
+ sw(ToRegister(i), MemOperand(sp, stack_offset));
+ }
+ }
+}
+
+void TurboAssembler::MultiPop(RegList regs) {
+ int16_t stack_offset = 0;
+
+ for (int16_t i = 0; i < kNumRegisters; i++) {
+ if ((regs & (1 << i)) != 0) {
+ lw(ToRegister(i), MemOperand(sp, stack_offset));
+ stack_offset += kPointerSize;
+ }
+ }
+ addiu(sp, sp, stack_offset);
+}
+
+void TurboAssembler::MultiPushFPU(RegList regs) {
+ int16_t num_to_push = base::bits::CountPopulation(regs);
+ int16_t stack_offset = num_to_push * kDoubleSize;
+
+ Subu(sp, sp, Operand(stack_offset));
+ for (int16_t i = kNumRegisters - 1; i >= 0; i--) {
+ if ((regs & (1 << i)) != 0) {
+ stack_offset -= kDoubleSize;
+ Sdc1(FPURegister::from_code(i), MemOperand(sp, stack_offset));
+ }
+ }
+}
+
+void TurboAssembler::MultiPopFPU(RegList regs) {
+ int16_t stack_offset = 0;
+
+ for (int16_t i = 0; i < kNumRegisters; i++) {
+ if ((regs & (1 << i)) != 0) {
+ Ldc1(FPURegister::from_code(i), MemOperand(sp, stack_offset));
+ stack_offset += kDoubleSize;
+ }
+ }
+ addiu(sp, sp, stack_offset);
+}
+
+void TurboAssembler::AddPair(Register dst_low, Register dst_high,
+ Register left_low, Register left_high,
+ Register right_low, Register right_high,
+ Register scratch1, Register scratch2) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Register scratch3 = t8;
+ Addu(scratch1, left_low, right_low);
+ Sltu(scratch3, scratch1, left_low);
+ Addu(scratch2, left_high, right_high);
+ Addu(dst_high, scratch2, scratch3);
+ Move(dst_low, scratch1);
+}
+
+void TurboAssembler::AddPair(Register dst_low, Register dst_high,
+ Register left_low, Register left_high, int32_t imm,
+ Register scratch1, Register scratch2) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Register scratch3 = t8;
+ li(dst_low, Operand(imm));
+ sra(dst_high, dst_low, 31);
+ Addu(scratch1, left_low, dst_low);
+ Sltu(scratch3, scratch1, left_low);
+ Addu(scratch2, left_high, dst_high);
+ Addu(dst_high, scratch2, scratch3);
+ Move(dst_low, scratch1);
+}
+
+void TurboAssembler::SubPair(Register dst_low, Register dst_high,
+ Register left_low, Register left_high,
+ Register right_low, Register right_high,
+ Register scratch1, Register scratch2) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Register scratch3 = t8;
+ Sltu(scratch3, left_low, right_low);
+ Subu(scratch1, left_low, right_low);
+ Subu(scratch2, left_high, right_high);
+ Subu(dst_high, scratch2, scratch3);
+ Move(dst_low, scratch1);
+}
+
+void TurboAssembler::AndPair(Register dst_low, Register dst_high,
+ Register left_low, Register left_high,
+ Register right_low, Register right_high) {
+ And(dst_low, left_low, right_low);
+ And(dst_high, left_high, right_high);
+}
+
+void TurboAssembler::OrPair(Register dst_low, Register dst_high,
+ Register left_low, Register left_high,
+ Register right_low, Register right_high) {
+ Or(dst_low, left_low, right_low);
+ Or(dst_high, left_high, right_high);
+}
+void TurboAssembler::XorPair(Register dst_low, Register dst_high,
+ Register left_low, Register left_high,
+ Register right_low, Register right_high) {
+ Xor(dst_low, left_low, right_low);
+ Xor(dst_high, left_high, right_high);
+}
+
+void TurboAssembler::MulPair(Register dst_low, Register dst_high,
+ Register left_low, Register left_high,
+ Register right_low, Register right_high,
+ Register scratch1, Register scratch2) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Register scratch3 = t8;
+ Mulu(scratch2, scratch1, left_low, right_low);
+ Mul(scratch3, left_low, right_high);
+ Addu(scratch2, scratch2, scratch3);
+ Mul(scratch3, left_high, right_low);
+ Addu(dst_high, scratch2, scratch3);
+ Move(dst_low, scratch1);
+}
+
+void TurboAssembler::ShlPair(Register dst_low, Register dst_high,
+ Register src_low, Register src_high,
+ Register shift, Register scratch1,
+ Register scratch2) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Label done;
+ Register scratch3 = t8;
+ And(scratch3, shift, 0x3F);
+ sllv(dst_low, src_low, scratch3);
+ Nor(scratch2, zero_reg, scratch3);
+ srl(scratch1, src_low, 1);
+ srlv(scratch1, scratch1, scratch2);
+ sllv(dst_high, src_high, scratch3);
+ Or(dst_high, dst_high, scratch1);
+ And(scratch1, scratch3, 32);
+ if (IsMipsArchVariant(kLoongson) || IsMipsArchVariant(kMips32r6)) {
+ Branch(&done, eq, scratch1, Operand(zero_reg));
+ mov(dst_high, dst_low);
+ mov(dst_low, zero_reg);
+ } else {
+ movn(dst_high, dst_low, scratch1);
+ movn(dst_low, zero_reg, scratch1);
+ }
+ bind(&done);
+}
+
+void TurboAssembler::ShlPair(Register dst_low, Register dst_high,
+ Register src_low, Register src_high,
+ uint32_t shift, Register scratch) {
+ DCHECK_NE(dst_low, src_low);
+ DCHECK_NE(dst_high, src_low);
+ shift = shift & 0x3F;
+ if (shift == 0) {
+ mov(dst_high, src_high);
+ mov(dst_low, src_low);
+ } else if (shift < 32) {
+ if (IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6)) {
+ DCHECK_NE(dst_high, src_high);
+ srl(dst_high, src_low, 32 - shift);
+ Ins(dst_high, src_high, shift, 32 - shift);
+ sll(dst_low, src_low, shift);
+ } else {
+ sll(dst_high, src_high, shift);
+ sll(dst_low, src_low, shift);
+ srl(scratch, src_low, 32 - shift);
+ Or(dst_high, dst_high, scratch);
+ }
+ } else if (shift == 32) {
+ mov(dst_low, zero_reg);
+ mov(dst_high, src_low);
+ } else {
+ shift = shift - 32;
+ mov(dst_low, zero_reg);
+ sll(dst_high, src_low, shift);
+ }
+}
+
+void TurboAssembler::ShrPair(Register dst_low, Register dst_high,
+ Register src_low, Register src_high,
+ Register shift, Register scratch1,
+ Register scratch2) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Label done;
+ Register scratch3 = t8;
+ And(scratch3, shift, 0x3F);
+ srlv(dst_high, src_high, scratch3);
+ Nor(scratch2, zero_reg, scratch3);
+ sll(scratch1, src_high, 1);
+ sllv(scratch1, scratch1, scratch2);
+ srlv(dst_low, src_low, scratch3);
+ Or(dst_low, dst_low, scratch1);
+ And(scratch1, scratch3, 32);
+ if (IsMipsArchVariant(kLoongson) || IsMipsArchVariant(kMips32r6)) {
+ Branch(&done, eq, scratch1, Operand(zero_reg));
+ mov(dst_low, dst_high);
+ mov(dst_high, zero_reg);
+ } else {
+ movn(dst_low, dst_high, scratch1);
+ movn(dst_high, zero_reg, scratch1);
+ }
+ bind(&done);
+}
+
+void TurboAssembler::ShrPair(Register dst_low, Register dst_high,
+ Register src_low, Register src_high,
+ uint32_t shift, Register scratch) {
+ DCHECK_NE(dst_low, src_high);
+ DCHECK_NE(dst_high, src_high);
+ shift = shift & 0x3F;
+ if (shift == 0) {
+ mov(dst_low, src_low);
+ mov(dst_high, src_high);
+ } else if (shift < 32) {
+ if (IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6)) {
+ srl(dst_low, src_low, shift);
+ Ins(dst_low, src_high, 32 - shift, shift);
+ srl(dst_high, src_high, shift);
+ } else {
+ srl(dst_low, src_low, shift);
+ srl(dst_high, src_high, shift);
+ shift = 32 - shift;
+ sll(scratch, src_high, shift);
+ Or(dst_low, dst_low, scratch);
+ }
+ } else if (shift == 32) {
+ mov(dst_high, zero_reg);
+ mov(dst_low, src_high);
+ } else {
+ shift = shift - 32;
+ mov(dst_high, zero_reg);
+ srl(dst_low, src_high, shift);
+ }
+}
+
+void TurboAssembler::SarPair(Register dst_low, Register dst_high,
+ Register src_low, Register src_high,
+ Register shift, Register scratch1,
+ Register scratch2) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Label done;
+ Register scratch3 = t8;
+ And(scratch3, shift, 0x3F);
+ srav(dst_high, src_high, scratch3);
+ Nor(scratch2, zero_reg, scratch3);
+ sll(scratch1, src_high, 1);
+ sllv(scratch1, scratch1, scratch2);
+ srlv(dst_low, src_low, scratch3);
+ Or(dst_low, dst_low, scratch1);
+ And(scratch1, scratch3, 32);
+ Branch(&done, eq, scratch1, Operand(zero_reg));
+ mov(dst_low, dst_high);
+ sra(dst_high, dst_high, 31);
+ bind(&done);
+}
+
+void TurboAssembler::SarPair(Register dst_low, Register dst_high,
+ Register src_low, Register src_high,
+ uint32_t shift, Register scratch) {
+ DCHECK_NE(dst_low, src_high);
+ DCHECK_NE(dst_high, src_high);
+ shift = shift & 0x3F;
+ if (shift == 0) {
+ mov(dst_low, src_low);
+ mov(dst_high, src_high);
+ } else if (shift < 32) {
+ if (IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6)) {
+ srl(dst_low, src_low, shift);
+ Ins(dst_low, src_high, 32 - shift, shift);
+ sra(dst_high, src_high, shift);
+ } else {
+ srl(dst_low, src_low, shift);
+ sra(dst_high, src_high, shift);
+ shift = 32 - shift;
+ sll(scratch, src_high, shift);
+ Or(dst_low, dst_low, scratch);
+ }
+ } else if (shift == 32) {
+ sra(dst_high, src_high, 31);
+ mov(dst_low, src_high);
+ } else {
+ shift = shift - 32;
+ sra(dst_high, src_high, 31);
+ sra(dst_low, src_high, shift);
+ }
+}
+
+void TurboAssembler::Ext(Register rt, Register rs, uint16_t pos,
+ uint16_t size) {
+ DCHECK_LT(pos, 32);
+ DCHECK_LT(pos + size, 33);
+
+ if (IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6)) {
+ ext_(rt, rs, pos, size);
+ } else {
+ // Move rs to rt and shift it left then right to get the
+ // desired bitfield on the right side and zeroes on the left.
+ int shift_left = 32 - (pos + size);
+ sll(rt, rs, shift_left); // Acts as a move if shift_left == 0.
+
+ int shift_right = 32 - size;
+ if (shift_right > 0) {
+ srl(rt, rt, shift_right);
+ }
+ }
+}
+
+void TurboAssembler::Ins(Register rt, Register rs, uint16_t pos,
+ uint16_t size) {
+ DCHECK_LT(pos, 32);
+ DCHECK_LE(pos + size, 32);
+ DCHECK_NE(size, 0);
+
+ if (IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6)) {
+ ins_(rt, rs, pos, size);
+ } else {
+ DCHECK(rt != t8 && rs != t8);
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ Subu(scratch, zero_reg, Operand(1));
+ srl(scratch, scratch, 32 - size);
+ and_(t8, rs, scratch);
+ sll(t8, t8, pos);
+ sll(scratch, scratch, pos);
+ nor(scratch, scratch, zero_reg);
+ and_(scratch, rt, scratch);
+ or_(rt, t8, scratch);
+ }
+}
+
+void TurboAssembler::ExtractBits(Register dest, Register source, Register pos,
+ int size, bool sign_extend) {
+ srav(dest, source, pos);
+ Ext(dest, dest, 0, size);
+ if (size == 8) {
+ if (sign_extend) {
+ Seb(dest, dest);
+ }
+ } else if (size == 16) {
+ if (sign_extend) {
+ Seh(dest, dest);
+ }
+ } else {
+ UNREACHABLE();
+ }
+}
+
+void TurboAssembler::InsertBits(Register dest, Register source, Register pos,
+ int size) {
+ Ror(dest, dest, pos);
+ Ins(dest, source, 0, size);
+ {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ Subu(scratch, zero_reg, pos);
+ Ror(dest, dest, scratch);
+ }
+}
+
+void TurboAssembler::Seb(Register rd, Register rt) {
+ if (IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6)) {
+ seb(rd, rt);
+ } else {
+ DCHECK(IsMipsArchVariant(kMips32r1) || IsMipsArchVariant(kLoongson));
+ sll(rd, rt, 24);
+ sra(rd, rd, 24);
+ }
+}
+
+void TurboAssembler::Seh(Register rd, Register rt) {
+ if (IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6)) {
+ seh(rd, rt);
+ } else {
+ DCHECK(IsMipsArchVariant(kMips32r1) || IsMipsArchVariant(kLoongson));
+ sll(rd, rt, 16);
+ sra(rd, rd, 16);
+ }
+}
+
+void TurboAssembler::Neg_s(FPURegister fd, FPURegister fs) {
+ if (IsMipsArchVariant(kMips32r6)) {
+ // r6 neg_s changes the sign for NaN-like operands as well.
+ neg_s(fd, fs);
+ } else {
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r1) ||
+ IsMipsArchVariant(kLoongson));
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Label is_nan, done;
+ Register scratch1 = t8;
+ Register scratch2 = t9;
+ CompareIsNanF32(fs, fs);
+ BranchTrueShortF(&is_nan);
+ Branch(USE_DELAY_SLOT, &done);
+ // For NaN input, neg_s will return the same NaN value,
+ // while the sign has to be changed separately.
+ neg_s(fd, fs); // In delay slot.
+ bind(&is_nan);
+ mfc1(scratch1, fs);
+ li(scratch2, kBinary32SignMask);
+ Xor(scratch1, scratch1, scratch2);
+ mtc1(scratch1, fd);
+ bind(&done);
+ }
+}
+
+void TurboAssembler::Neg_d(FPURegister fd, FPURegister fs) {
+ if (IsMipsArchVariant(kMips32r6)) {
+ // r6 neg_d changes the sign for NaN-like operands as well.
+ neg_d(fd, fs);
+ } else {
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r1) ||
+ IsMipsArchVariant(kLoongson));
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Label is_nan, done;
+ Register scratch1 = t8;
+ Register scratch2 = t9;
+ CompareIsNanF64(fs, fs);
+ BranchTrueShortF(&is_nan);
+ Branch(USE_DELAY_SLOT, &done);
+ // For NaN input, neg_d will return the same NaN value,
+ // while the sign has to be changed separately.
+ neg_d(fd, fs); // In delay slot.
+ bind(&is_nan);
+ Move(fd, fs);
+ Mfhc1(scratch1, fd);
+ li(scratch2, HeapNumber::kSignMask);
+ Xor(scratch1, scratch1, scratch2);
+ Mthc1(scratch1, fd);
+ bind(&done);
+ }
+}
+
+void TurboAssembler::Cvt_d_uw(FPURegister fd, Register rs,
+ FPURegister scratch) {
+ // In FP64Mode we do conversion from long.
+ if (IsFp64Mode()) {
+ mtc1(rs, scratch);
+ Mthc1(zero_reg, scratch);
+ cvt_d_l(fd, scratch);
+ } else {
+ // Convert rs to a FP value in fd.
+ DCHECK(fd != scratch);
+ DCHECK(rs != at);
+
+ Label msb_clear, conversion_done;
+ // For a value which is < 2^31, regard it as a signed positve word.
+ Branch(&msb_clear, ge, rs, Operand(zero_reg), USE_DELAY_SLOT);
+ mtc1(rs, fd);
+ {
+ UseScratchRegisterScope temps(this);
+ Register scratch1 = temps.Acquire();
+ li(scratch1, 0x41F00000); // FP value: 2^32.
+
+ // For unsigned inputs > 2^31, we convert to double as a signed int32,
+ // then add 2^32 to move it back to unsigned value in range 2^31..2^31-1.
+ mtc1(zero_reg, scratch);
+ Mthc1(scratch1, scratch);
+ }
+
+ cvt_d_w(fd, fd);
+
+ Branch(USE_DELAY_SLOT, &conversion_done);
+ add_d(fd, fd, scratch);
+
+ bind(&msb_clear);
+ cvt_d_w(fd, fd);
+
+ bind(&conversion_done);
+ }
+}
+
+void TurboAssembler::Trunc_uw_d(FPURegister fd, FPURegister fs,
+ FPURegister scratch) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Trunc_uw_d(t8, fs, scratch);
+ mtc1(t8, fd);
+}
+
+void TurboAssembler::Trunc_uw_s(FPURegister fd, FPURegister fs,
+ FPURegister scratch) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Trunc_uw_s(t8, fs, scratch);
+ mtc1(t8, fd);
+}
+
+void TurboAssembler::Trunc_w_d(FPURegister fd, FPURegister fs) {
+ if (IsMipsArchVariant(kLoongson) && fd == fs) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Mfhc1(t8, fs);
+ trunc_w_d(fd, fs);
+ Mthc1(t8, fs);
+ } else {
+ trunc_w_d(fd, fs);
+ }
+}
+
+void TurboAssembler::Round_w_d(FPURegister fd, FPURegister fs) {
+ if (IsMipsArchVariant(kLoongson) && fd == fs) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Mfhc1(t8, fs);
+ round_w_d(fd, fs);
+ Mthc1(t8, fs);
+ } else {
+ round_w_d(fd, fs);
+ }
+}
+
+void TurboAssembler::Floor_w_d(FPURegister fd, FPURegister fs) {
+ if (IsMipsArchVariant(kLoongson) && fd == fs) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Mfhc1(t8, fs);
+ floor_w_d(fd, fs);
+ Mthc1(t8, fs);
+ } else {
+ floor_w_d(fd, fs);
+ }
+}
+
+void TurboAssembler::Ceil_w_d(FPURegister fd, FPURegister fs) {
+ if (IsMipsArchVariant(kLoongson) && fd == fs) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Mfhc1(t8, fs);
+ ceil_w_d(fd, fs);
+ Mthc1(t8, fs);
+ } else {
+ ceil_w_d(fd, fs);
+ }
+}
+
+void TurboAssembler::Trunc_uw_d(Register rd, FPURegister fs,
+ FPURegister scratch) {
+ DCHECK(fs != scratch);
+ DCHECK(rd != at);
+
+ {
+ // Load 2^31 into scratch as its float representation.
+ UseScratchRegisterScope temps(this);
+ Register scratch1 = temps.Acquire();
+ li(scratch1, 0x41E00000);
+ mtc1(zero_reg, scratch);
+ Mthc1(scratch1, scratch);
+ }
+ // Test if scratch > fs.
+ // If fs < 2^31 we can convert it normally.
+ Label simple_convert;
+ CompareF64(OLT, fs, scratch);
+ BranchTrueShortF(&simple_convert);
+
+ // First we subtract 2^31 from fs, then trunc it to rd
+ // and add 2^31 to rd.
+ sub_d(scratch, fs, scratch);
+ trunc_w_d(scratch, scratch);
+ mfc1(rd, scratch);
+ Or(rd, rd, 1 << 31);
+
+ Label done;
+ Branch(&done);
+ // Simple conversion.
+ bind(&simple_convert);
+ trunc_w_d(scratch, fs);
+ mfc1(rd, scratch);
+
+ bind(&done);
+}
+
+void TurboAssembler::Trunc_uw_s(Register rd, FPURegister fs,
+ FPURegister scratch) {
+ DCHECK(fs != scratch);
+ DCHECK(rd != at);
+
+ {
+ // Load 2^31 into scratch as its float representation.
+ UseScratchRegisterScope temps(this);
+ Register scratch1 = temps.Acquire();
+ li(scratch1, 0x4F000000);
+ mtc1(scratch1, scratch);
+ }
+ // Test if scratch > fs.
+ // If fs < 2^31 we can convert it normally.
+ Label simple_convert;
+ CompareF32(OLT, fs, scratch);
+ BranchTrueShortF(&simple_convert);
+
+ // First we subtract 2^31 from fs, then trunc it to rd
+ // and add 2^31 to rd.
+ sub_s(scratch, fs, scratch);
+ trunc_w_s(scratch, scratch);
+ mfc1(rd, scratch);
+ Or(rd, rd, 1 << 31);
+
+ Label done;
+ Branch(&done);
+ // Simple conversion.
+ bind(&simple_convert);
+ trunc_w_s(scratch, fs);
+ mfc1(rd, scratch);
+
+ bind(&done);
+}
+
+template <typename RoundFunc>
+void TurboAssembler::RoundDouble(FPURegister dst, FPURegister src,
+ FPURoundingMode mode, RoundFunc round) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Register scratch = t8;
+ Register scratch2 = t9;
+ if (IsMipsArchVariant(kMips32r6)) {
+ cfc1(scratch, FCSR);
+ li(at, Operand(mode));
+ ctc1(at, FCSR);
+ rint_d(dst, src);
+ ctc1(scratch, FCSR);
+ } else {
+ Label done;
+ Mfhc1(scratch, src);
+ Ext(at, scratch, HeapNumber::kExponentShift, HeapNumber::kExponentBits);
+ Branch(USE_DELAY_SLOT, &done, hs, at,
+ Operand(HeapNumber::kExponentBias + HeapNumber::kMantissaBits));
+ mov_d(dst, src);
+ round(this, dst, src);
+ Move(at, scratch2, dst);
+ or_(at, at, scratch2);
+ Branch(USE_DELAY_SLOT, &done, ne, at, Operand(zero_reg));
+ cvt_d_l(dst, dst);
+ srl(at, scratch, 31);
+ sll(at, at, 31);
+ Mthc1(at, dst);
+ bind(&done);
+ }
+}
+
+void TurboAssembler::Floor_d_d(FPURegister dst, FPURegister src) {
+ RoundDouble(dst, src, mode_floor,
+ [](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
+ tasm->floor_l_d(dst, src);
+ });
+}
+
+void TurboAssembler::Ceil_d_d(FPURegister dst, FPURegister src) {
+ RoundDouble(dst, src, mode_ceil,
+ [](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
+ tasm->ceil_l_d(dst, src);
+ });
+}
+
+void TurboAssembler::Trunc_d_d(FPURegister dst, FPURegister src) {
+ RoundDouble(dst, src, mode_trunc,
+ [](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
+ tasm->trunc_l_d(dst, src);
+ });
+}
+
+void TurboAssembler::Round_d_d(FPURegister dst, FPURegister src) {
+ RoundDouble(dst, src, mode_round,
+ [](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
+ tasm->round_l_d(dst, src);
+ });
+}
+
+template <typename RoundFunc>
+void TurboAssembler::RoundFloat(FPURegister dst, FPURegister src,
+ FPURoundingMode mode, RoundFunc round) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Register scratch = t8;
+ if (IsMipsArchVariant(kMips32r6)) {
+ cfc1(scratch, FCSR);
+ li(at, Operand(mode));
+ ctc1(at, FCSR);
+ rint_s(dst, src);
+ ctc1(scratch, FCSR);
+ } else {
+ int32_t kFloat32ExponentBias = 127;
+ int32_t kFloat32MantissaBits = 23;
+ int32_t kFloat32ExponentBits = 8;
+ Label done;
+ if (!IsDoubleZeroRegSet()) {
+ Move(kDoubleRegZero, 0.0);
+ }
+ mfc1(scratch, src);
+ Ext(at, scratch, kFloat32MantissaBits, kFloat32ExponentBits);
+ Branch(USE_DELAY_SLOT, &done, hs, at,
+ Operand(kFloat32ExponentBias + kFloat32MantissaBits));
+ // Canonicalize the result.
+ sub_s(dst, src, kDoubleRegZero);
+ round(this, dst, src);
+ mfc1(at, dst);
+ Branch(USE_DELAY_SLOT, &done, ne, at, Operand(zero_reg));
+ cvt_s_w(dst, dst);
+ srl(at, scratch, 31);
+ sll(at, at, 31);
+ mtc1(at, dst);
+ bind(&done);
+ }
+}
+
+void TurboAssembler::Floor_s_s(FPURegister dst, FPURegister src) {
+ RoundFloat(dst, src, mode_floor,
+ [](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
+ tasm->floor_w_s(dst, src);
+ });
+}
+
+void TurboAssembler::Ceil_s_s(FPURegister dst, FPURegister src) {
+ RoundFloat(dst, src, mode_ceil,
+ [](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
+ tasm->ceil_w_s(dst, src);
+ });
+}
+
+void TurboAssembler::Trunc_s_s(FPURegister dst, FPURegister src) {
+ RoundFloat(dst, src, mode_trunc,
+ [](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
+ tasm->trunc_w_s(dst, src);
+ });
+}
+
+void TurboAssembler::Round_s_s(FPURegister dst, FPURegister src) {
+ RoundFloat(dst, src, mode_round,
+ [](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
+ tasm->round_w_s(dst, src);
+ });
+}
+
+void TurboAssembler::Mthc1(Register rt, FPURegister fs) {
+ if (IsFp32Mode()) {
+ mtc1(rt, fs.high());
+ } else {
+ DCHECK(IsFp64Mode() || IsFpxxMode());
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6));
+ mthc1(rt, fs);
+ }
+}
+
+void TurboAssembler::Mfhc1(Register rt, FPURegister fs) {
+ if (IsFp32Mode()) {
+ mfc1(rt, fs.high());
+ } else {
+ DCHECK(IsFp64Mode() || IsFpxxMode());
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6));
+ mfhc1(rt, fs);
+ }
+}
+
+void TurboAssembler::Madd_s(FPURegister fd, FPURegister fr, FPURegister fs,
+ FPURegister ft, FPURegister scratch) {
+ if (IsMipsArchVariant(kMips32r2)) {
+ madd_s(fd, fr, fs, ft);
+ } else {
+ DCHECK(fr != scratch && fs != scratch && ft != scratch);
+ mul_s(scratch, fs, ft);
+ add_s(fd, fr, scratch);
+ }
+}
+
+void TurboAssembler::Madd_d(FPURegister fd, FPURegister fr, FPURegister fs,
+ FPURegister ft, FPURegister scratch) {
+ if (IsMipsArchVariant(kMips32r2)) {
+ madd_d(fd, fr, fs, ft);
+ } else {
+ DCHECK(fr != scratch && fs != scratch && ft != scratch);
+ mul_d(scratch, fs, ft);
+ add_d(fd, fr, scratch);
+ }
+}
+
+void TurboAssembler::Msub_s(FPURegister fd, FPURegister fr, FPURegister fs,
+ FPURegister ft, FPURegister scratch) {
+ if (IsMipsArchVariant(kMips32r2)) {
+ msub_s(fd, fr, fs, ft);
+ } else {
+ DCHECK(fr != scratch && fs != scratch && ft != scratch);
+ mul_s(scratch, fs, ft);
+ sub_s(fd, scratch, fr);
+ }
+}
+
+void TurboAssembler::Msub_d(FPURegister fd, FPURegister fr, FPURegister fs,
+ FPURegister ft, FPURegister scratch) {
+ if (IsMipsArchVariant(kMips32r2)) {
+ msub_d(fd, fr, fs, ft);
+ } else {
+ DCHECK(fr != scratch && fs != scratch && ft != scratch);
+ mul_d(scratch, fs, ft);
+ sub_d(fd, scratch, fr);
+ }
+}
+
+void TurboAssembler::CompareF(SecondaryField sizeField, FPUCondition cc,
+ FPURegister cmp1, FPURegister cmp2) {
+ if (IsMipsArchVariant(kMips32r6)) {
+ sizeField = sizeField == D ? L : W;
+ DCHECK(cmp1 != kDoubleCompareReg && cmp2 != kDoubleCompareReg);
+ cmp(cc, sizeField, kDoubleCompareReg, cmp1, cmp2);
+ } else {
+ c(cc, sizeField, cmp1, cmp2);
+ }
+}
+
+void TurboAssembler::CompareIsNanF(SecondaryField sizeField, FPURegister cmp1,
+ FPURegister cmp2) {
+ CompareF(sizeField, UN, cmp1, cmp2);
+}
+
+void TurboAssembler::BranchTrueShortF(Label* target, BranchDelaySlot bd) {
+ if (IsMipsArchVariant(kMips32r6)) {
+ bc1nez(target, kDoubleCompareReg);
+ } else {
+ bc1t(target);
+ }
+ if (bd == PROTECT) {
+ nop();
+ }
+}
+
+void TurboAssembler::BranchFalseShortF(Label* target, BranchDelaySlot bd) {
+ if (IsMipsArchVariant(kMips32r6)) {
+ bc1eqz(target, kDoubleCompareReg);
+ } else {
+ bc1f(target);
+ }
+ if (bd == PROTECT) {
+ nop();
+ }
+}
+
+void TurboAssembler::BranchTrueF(Label* target, BranchDelaySlot bd) {
+ bool long_branch =
+ target->is_bound() ? !is_near(target) : is_trampoline_emitted();
+ if (long_branch) {
+ Label skip;
+ BranchFalseShortF(&skip);
+ BranchLong(target, bd);
+ bind(&skip);
+ } else {
+ BranchTrueShortF(target, bd);
+ }
+}
+
+void TurboAssembler::BranchFalseF(Label* target, BranchDelaySlot bd) {
+ bool long_branch =
+ target->is_bound() ? !is_near(target) : is_trampoline_emitted();
+ if (long_branch) {
+ Label skip;
+ BranchTrueShortF(&skip);
+ BranchLong(target, bd);
+ bind(&skip);
+ } else {
+ BranchFalseShortF(target, bd);
+ }
+}
+
+void TurboAssembler::BranchMSA(Label* target, MSABranchDF df,
+ MSABranchCondition cond, MSARegister wt,
+ BranchDelaySlot bd) {
+ {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+
+ if (target) {
+ bool long_branch =
+ target->is_bound() ? !is_near(target) : is_trampoline_emitted();
+ if (long_branch) {
+ Label skip;
+ MSABranchCondition neg_cond = NegateMSABranchCondition(cond);
+ BranchShortMSA(df, &skip, neg_cond, wt, bd);
+ BranchLong(target, bd);
+ bind(&skip);
+ } else {
+ BranchShortMSA(df, target, cond, wt, bd);
+ }
+ }
+ }
+}
+
+void TurboAssembler::BranchShortMSA(MSABranchDF df, Label* target,
+ MSABranchCondition cond, MSARegister wt,
+ BranchDelaySlot bd) {
+ if (IsMipsArchVariant(kMips32r6)) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ if (target) {
+ switch (cond) {
+ case all_not_zero:
+ switch (df) {
+ case MSA_BRANCH_D:
+ bnz_d(wt, target);
+ break;
+ case MSA_BRANCH_W:
+ bnz_w(wt, target);
+ break;
+ case MSA_BRANCH_H:
+ bnz_h(wt, target);
+ break;
+ case MSA_BRANCH_B:
+ default:
+ bnz_b(wt, target);
+ }
+ break;
+ case one_elem_not_zero:
+ bnz_v(wt, target);
+ break;
+ case one_elem_zero:
+ switch (df) {
+ case MSA_BRANCH_D:
+ bz_d(wt, target);
+ break;
+ case MSA_BRANCH_W:
+ bz_w(wt, target);
+ break;
+ case MSA_BRANCH_H:
+ bz_h(wt, target);
+ break;
+ case MSA_BRANCH_B:
+ default:
+ bz_b(wt, target);
+ }
+ break;
+ case all_zero:
+ bz_v(wt, target);
+ break;
+ default:
+ UNREACHABLE();
+ }
+ }
+ }
+ if (bd == PROTECT) {
+ nop();
+ }
+}
+
+void TurboAssembler::FmoveLow(FPURegister dst, Register src_low) {
+ if (IsFp32Mode()) {
+ mtc1(src_low, dst);
+ } else {
+ DCHECK(IsFp64Mode() || IsFpxxMode());
+ DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6));
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(src_low != scratch);
+ mfhc1(scratch, dst);
+ mtc1(src_low, dst);
+ mthc1(scratch, dst);
+ }
+}
+
+void TurboAssembler::Move(FPURegister dst, uint32_t src) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ li(scratch, Operand(static_cast<int32_t>(src)));
+ mtc1(scratch, dst);
+}
+
+void TurboAssembler::Move(FPURegister dst, uint64_t src) {
+ // Handle special values first.
+ if (src == bit_cast<uint64_t>(0.0) && has_double_zero_reg_set_) {
+ mov_d(dst, kDoubleRegZero);
+ } else if (src == bit_cast<uint64_t>(-0.0) && has_double_zero_reg_set_) {
+ Neg_d(dst, kDoubleRegZero);
+ } else {
+ uint32_t lo = src & 0xFFFFFFFF;
+ uint32_t hi = src >> 32;
+ // Move the low part of the double into the lower of the corresponding FPU
+ // register of FPU register pair.
+ if (lo != 0) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ li(scratch, Operand(lo));
+ mtc1(scratch, dst);
+ } else {
+ mtc1(zero_reg, dst);
+ }
+ // Move the high part of the double into the higher of the corresponding FPU
+ // register of FPU register pair.
+ if (hi != 0) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ li(scratch, Operand(hi));
+ Mthc1(scratch, dst);
+ } else {
+ Mthc1(zero_reg, dst);
+ }
+ if (dst == kDoubleRegZero) has_double_zero_reg_set_ = true;
+ }
+}
+
+void TurboAssembler::LoadZeroOnCondition(Register rd, Register rs,
+ const Operand& rt, Condition cond) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ switch (cond) {
+ case cc_always:
+ mov(rd, zero_reg);
+ break;
+ case eq:
+ if (rs == zero_reg) {
+ if (rt.is_reg()) {
+ LoadZeroIfConditionZero(rd, rt.rm());
+ } else {
+ if (rt.immediate() == 0) {
+ mov(rd, zero_reg);
+ } else {
+ nop();
+ }
+ }
+ } else if (IsZero(rt)) {
+ LoadZeroIfConditionZero(rd, rs);
+ } else {
+ Subu(t9, rs, rt);
+ LoadZeroIfConditionZero(rd, t9);
+ }
+ break;
+ case ne:
+ if (rs == zero_reg) {
+ if (rt.is_reg()) {
+ LoadZeroIfConditionNotZero(rd, rt.rm());
+ } else {
+ if (rt.immediate() != 0) {
+ mov(rd, zero_reg);
+ } else {
+ nop();
+ }
+ }
+ } else if (IsZero(rt)) {
+ LoadZeroIfConditionNotZero(rd, rs);
+ } else {
+ Subu(t9, rs, rt);
+ LoadZeroIfConditionNotZero(rd, t9);
+ }
+ break;
+
+ // Signed comparison.
+ case greater:
+ Sgt(t9, rs, rt);
+ LoadZeroIfConditionNotZero(rd, t9);
+ break;
+ case greater_equal:
+ Sge(t9, rs, rt);
+ LoadZeroIfConditionNotZero(rd, t9);
+ // rs >= rt
+ break;
+ case less:
+ Slt(t9, rs, rt);
+ LoadZeroIfConditionNotZero(rd, t9);
+ // rs < rt
+ break;
+ case less_equal:
+ Sle(t9, rs, rt);
+ LoadZeroIfConditionNotZero(rd, t9);
+ // rs <= rt
+ break;
+
+ // Unsigned comparison.
+ case Ugreater:
+ Sgtu(t9, rs, rt);
+ LoadZeroIfConditionNotZero(rd, t9);
+ // rs > rt
+ break;
+
+ case Ugreater_equal:
+ Sgeu(t9, rs, rt);
+ LoadZeroIfConditionNotZero(rd, t9);
+ // rs >= rt
+ break;
+ case Uless:
+ Sltu(t9, rs, rt);
+ LoadZeroIfConditionNotZero(rd, t9);
+ // rs < rt
+ break;
+ case Uless_equal:
+ Sleu(t9, rs, rt);
+ LoadZeroIfConditionNotZero(rd, t9);
+ // rs <= rt
+ break;
+ default:
+ UNREACHABLE();
+ }
+}
+
+void TurboAssembler::LoadZeroIfConditionNotZero(Register dest,
+ Register condition) {
+ if (IsMipsArchVariant(kMips32r6)) {
+ seleqz(dest, dest, condition);
+ } else {
+ Movn(dest, zero_reg, condition);
+ }
+}
+
+void TurboAssembler::LoadZeroIfConditionZero(Register dest,
+ Register condition) {
+ if (IsMipsArchVariant(kMips32r6)) {
+ selnez(dest, dest, condition);
+ } else {
+ Movz(dest, zero_reg, condition);
+ }
+}
+
+void TurboAssembler::LoadZeroIfFPUCondition(Register dest) {
+ if (IsMipsArchVariant(kMips32r6)) {
+ mfc1(kScratchReg, kDoubleCompareReg);
+ LoadZeroIfConditionNotZero(dest, kScratchReg);
+ } else {
+ Movt(dest, zero_reg);
+ }
+}
+
+void TurboAssembler::LoadZeroIfNotFPUCondition(Register dest) {
+ if (IsMipsArchVariant(kMips32r6)) {
+ mfc1(kScratchReg, kDoubleCompareReg);
+ LoadZeroIfConditionZero(dest, kScratchReg);
+ } else {
+ Movf(dest, zero_reg);
+ }
+}
+
+void TurboAssembler::Movz(Register rd, Register rs, Register rt) {
+ if (IsMipsArchVariant(kLoongson) || IsMipsArchVariant(kMips32r6)) {
+ Label done;
+ Branch(&done, ne, rt, Operand(zero_reg));
+ mov(rd, rs);
+ bind(&done);
+ } else {
+ movz(rd, rs, rt);
+ }
+}
+
+void TurboAssembler::Movn(Register rd, Register rs, Register rt) {
+ if (IsMipsArchVariant(kLoongson) || IsMipsArchVariant(kMips32r6)) {
+ Label done;
+ Branch(&done, eq, rt, Operand(zero_reg));
+ mov(rd, rs);
+ bind(&done);
+ } else {
+ movn(rd, rs, rt);
+ }
+}
+
+void TurboAssembler::Movt(Register rd, Register rs, uint16_t cc) {
+ if (IsMipsArchVariant(kLoongson)) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ // Tests an FP condition code and then conditionally move rs to rd.
+ // We do not currently use any FPU cc bit other than bit 0.
+ DCHECK_EQ(cc, 0);
+ DCHECK(rs != t8 && rd != t8);
+ Label done;
+ Register scratch = t8;
+ // For testing purposes we need to fetch content of the FCSR register and
+ // than test its cc (floating point condition code) bit (for cc = 0, it is
+ // 24. bit of the FCSR).
+ cfc1(scratch, FCSR);
+ // For the MIPS I, II and III architectures, the contents of scratch is
+ // UNPREDICTABLE for the instruction immediately following CFC1.
+ nop();
+ srl(scratch, scratch, 16);
+ andi(scratch, scratch, 0x0080);
+ Branch(&done, eq, scratch, Operand(zero_reg));
+ mov(rd, rs);
+ bind(&done);
+ } else {
+ movt(rd, rs, cc);
+ }
+}
+
+void TurboAssembler::Movf(Register rd, Register rs, uint16_t cc) {
+ if (IsMipsArchVariant(kLoongson)) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ // Tests an FP condition code and then conditionally move rs to rd.
+ // We do not currently use any FPU cc bit other than bit 0.
+ DCHECK_EQ(cc, 0);
+ DCHECK(rs != t8 && rd != t8);
+ Label done;
+ Register scratch = t8;
+ // For testing purposes we need to fetch content of the FCSR register and
+ // than test its cc (floating point condition code) bit (for cc = 0, it is
+ // 24. bit of the FCSR).
+ cfc1(scratch, FCSR);
+ // For the MIPS I, II and III architectures, the contents of scratch is
+ // UNPREDICTABLE for the instruction immediately following CFC1.
+ nop();
+ srl(scratch, scratch, 16);
+ andi(scratch, scratch, 0x0080);
+ Branch(&done, ne, scratch, Operand(zero_reg));
+ mov(rd, rs);
+ bind(&done);
+ } else {
+ movf(rd, rs, cc);
+ }
+}
+
+void TurboAssembler::Clz(Register rd, Register rs) {
+ if (IsMipsArchVariant(kLoongson)) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ DCHECK(rd != t8 && rd != t9 && rs != t8 && rs != t9);
+ Register mask = t8;
+ Register scratch = t9;
+ Label loop, end;
+ {
+ UseScratchRegisterScope temps(this);
+ Register scratch1 = temps.Acquire();
+ mov(scratch1, rs);
+ mov(rd, zero_reg);
+ lui(mask, 0x8000);
+ bind(&loop);
+ and_(scratch, scratch1, mask);
+ }
+ Branch(&end, ne, scratch, Operand(zero_reg));
+ addiu(rd, rd, 1);
+ Branch(&loop, ne, mask, Operand(zero_reg), USE_DELAY_SLOT);
+ srl(mask, mask, 1);
+ bind(&end);
+ } else {
+ clz(rd, rs);
+ }
+}
+
+void TurboAssembler::Ctz(Register rd, Register rs) {
+ if (IsMipsArchVariant(kMips32r6)) {
+ // We don't have an instruction to count the number of trailing zeroes.
+ // Start by flipping the bits end-for-end so we can count the number of
+ // leading zeroes instead.
+ Ror(rd, rs, 16);
+ wsbh(rd, rd);
+ bitswap(rd, rd);
+ Clz(rd, rd);
+ } else {
+ // Convert trailing zeroes to trailing ones, and bits to their left
+ // to zeroes.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ Addu(scratch, rs, -1);
+ Xor(rd, scratch, rs);
+ And(rd, rd, scratch);
+ // Count number of leading zeroes.
+ Clz(rd, rd);
+ // Subtract number of leading zeroes from 32 to get number of trailing
+ // ones. Remember that the trailing ones were formerly trailing zeroes.
+ li(scratch, 32);
+ Subu(rd, scratch, rd);
+ }
+}
+
+void TurboAssembler::Popcnt(Register rd, Register rs) {
+ // https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
+ //
+ // A generalization of the best bit counting method to integers of
+ // bit-widths up to 128 (parameterized by type T) is this:
+ //
+ // v = v - ((v >> 1) & (T)~(T)0/3); // temp
+ // v = (v & (T)~(T)0/15*3) + ((v >> 2) & (T)~(T)0/15*3); // temp
+ // v = (v + (v >> 4)) & (T)~(T)0/255*15; // temp
+ // c = (T)(v * ((T)~(T)0/255)) >> (sizeof(T) - 1) * BITS_PER_BYTE; //count
+ //
+ // For comparison, for 32-bit quantities, this algorithm can be executed
+ // using 20 MIPS instructions (the calls to LoadConst32() generate two
+ // machine instructions each for the values being used in this algorithm).
+ // A(n unrolled) loop-based algorithm requires 25 instructions.
+ //
+ // For 64-bit quantities, this algorithm gets executed twice, (once
+ // for in_lo, and again for in_hi), but saves a few instructions
+ // because the mask values only have to be loaded once. Using this
+ // algorithm the count for a 64-bit operand can be performed in 29
+ // instructions compared to a loop-based algorithm which requires 47
+ // instructions.
+ uint32_t B0 = 0x55555555; // (T)~(T)0/3
+ uint32_t B1 = 0x33333333; // (T)~(T)0/15*3
+ uint32_t B2 = 0x0F0F0F0F; // (T)~(T)0/255*15
+ uint32_t value = 0x01010101; // (T)~(T)0/255
+ uint32_t shift = 24; // (sizeof(T) - 1) * BITS_PER_BYTE
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ Register scratch2 = t8;
+ srl(scratch, rs, 1);
+ li(scratch2, B0);
+ And(scratch, scratch, scratch2);
+ Subu(scratch, rs, scratch);
+ li(scratch2, B1);
+ And(rd, scratch, scratch2);
+ srl(scratch, scratch, 2);
+ And(scratch, scratch, scratch2);
+ Addu(scratch, rd, scratch);
+ srl(rd, scratch, 4);
+ Addu(rd, rd, scratch);
+ li(scratch2, B2);
+ And(rd, rd, scratch2);
+ li(scratch, value);
+ Mul(rd, rd, scratch);
+ srl(rd, rd, shift);
+}
+
+void MacroAssembler::EmitFPUTruncate(
+ FPURoundingMode rounding_mode, Register result, DoubleRegister double_input,
+ Register scratch, DoubleRegister double_scratch, Register except_flag,
+ CheckForInexactConversion check_inexact) {
+ DCHECK(result != scratch);
+ DCHECK(double_input != double_scratch);
+ DCHECK(except_flag != scratch);
+
+ Label done;
+
+ // Clear the except flag (0 = no exception)
+ mov(except_flag, zero_reg);
+
+ // Test for values that can be exactly represented as a signed 32-bit integer.
+ cvt_w_d(double_scratch, double_input);
+ mfc1(result, double_scratch);
+ cvt_d_w(double_scratch, double_scratch);
+ CompareF64(EQ, double_input, double_scratch);
+ BranchTrueShortF(&done);
+
+ int32_t except_mask = kFCSRFlagMask; // Assume interested in all exceptions.
+
+ if (check_inexact == kDontCheckForInexactConversion) {
+ // Ignore inexact exceptions.
+ except_mask &= ~kFCSRInexactFlagMask;
+ }
+
+ // Save FCSR.
+ cfc1(scratch, FCSR);
+ // Disable FPU exceptions.
+ ctc1(zero_reg, FCSR);
+
+ // Do operation based on rounding mode.
+ switch (rounding_mode) {
+ case kRoundToNearest:
+ Round_w_d(double_scratch, double_input);
+ break;
+ case kRoundToZero:
+ Trunc_w_d(double_scratch, double_input);
+ break;
+ case kRoundToPlusInf:
+ Ceil_w_d(double_scratch, double_input);
+ break;
+ case kRoundToMinusInf:
+ Floor_w_d(double_scratch, double_input);
+ break;
+ } // End of switch-statement.
+
+ // Retrieve FCSR.
+ cfc1(except_flag, FCSR);
+ // Restore FCSR.
+ ctc1(scratch, FCSR);
+ // Move the converted value into the result register.
+ mfc1(result, double_scratch);
+
+ // Check for fpu exceptions.
+ And(except_flag, except_flag, Operand(except_mask));
+
+ bind(&done);
+}
+
+void TurboAssembler::TryInlineTruncateDoubleToI(Register result,
+ DoubleRegister double_input,
+ Label* done) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ DoubleRegister single_scratch = kScratchDoubleReg.low();
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ Register scratch2 = t9;
+
+ // Clear cumulative exception flags and save the FCSR.
+ cfc1(scratch2, FCSR);
+ ctc1(zero_reg, FCSR);
+ // Try a conversion to a signed integer.
+ trunc_w_d(single_scratch, double_input);
+ mfc1(result, single_scratch);
+ // Retrieve and restore the FCSR.
+ cfc1(scratch, FCSR);
+ ctc1(scratch2, FCSR);
+ // Check for overflow and NaNs.
+ And(scratch, scratch,
+ kFCSROverflowFlagMask | kFCSRUnderflowFlagMask | kFCSRInvalidOpFlagMask);
+ // If we had no exceptions we are done.
+ Branch(done, eq, scratch, Operand(zero_reg));
+}
+
+void TurboAssembler::TruncateDoubleToI(Isolate* isolate, Zone* zone,
+ Register result,
+ DoubleRegister double_input,
+ StubCallMode stub_mode) {
+ Label done;
+
+ TryInlineTruncateDoubleToI(result, double_input, &done);
+
+ // If we fell through then inline version didn't succeed - call stub instead.
+ push(ra);
+ Subu(sp, sp, Operand(kDoubleSize)); // Put input on stack.
+ Sdc1(double_input, MemOperand(sp, 0));
+
+ if (stub_mode == StubCallMode::kCallWasmRuntimeStub) {
+ Call(wasm::WasmCode::kDoubleToI, RelocInfo::WASM_STUB_CALL);
+ } else {
+ Call(BUILTIN_CODE(isolate, DoubleToI), RelocInfo::CODE_TARGET);
+ }
+ lw(result, MemOperand(sp, 0));
+
+ Addu(sp, sp, Operand(kDoubleSize));
+ pop(ra);
+
+ bind(&done);
+}
+
+// Emulated condtional branches do not emit a nop in the branch delay slot.
+//
+// BRANCH_ARGS_CHECK checks that conditional jump arguments are correct.
+#define BRANCH_ARGS_CHECK(cond, rs, rt) \
+ DCHECK((cond == cc_always && rs == zero_reg && rt.rm() == zero_reg) || \
+ (cond != cc_always && (rs != zero_reg || rt.rm() != zero_reg)))
+
+void TurboAssembler::Branch(int32_t offset, BranchDelaySlot bdslot) {
+ DCHECK(IsMipsArchVariant(kMips32r6) ? is_int26(offset) : is_int16(offset));
+ BranchShort(offset, bdslot);
+}
+
+void TurboAssembler::Branch(int32_t offset, Condition cond, Register rs,
+ const Operand& rt, BranchDelaySlot bdslot) {
+ bool is_near = BranchShortCheck(offset, nullptr, cond, rs, rt, bdslot);
+ DCHECK(is_near);
+ USE(is_near);
+}
+
+void TurboAssembler::Branch(Label* L, BranchDelaySlot bdslot) {
+ if (L->is_bound()) {
+ if (is_near_branch(L)) {
+ BranchShort(L, bdslot);
+ } else {
+ BranchLong(L, bdslot);
+ }
+ } else {
+ if (is_trampoline_emitted()) {
+ BranchLong(L, bdslot);
+ } else {
+ BranchShort(L, bdslot);
+ }
+ }
+}
+
+void TurboAssembler::Branch(Label* L, Condition cond, Register rs,
+ const Operand& rt, BranchDelaySlot bdslot) {
+ if (L->is_bound()) {
+ if (!BranchShortCheck(0, L, cond, rs, rt, bdslot)) {
+ if (cond != cc_always) {
+ Label skip;
+ Condition neg_cond = NegateCondition(cond);
+ BranchShort(&skip, neg_cond, rs, rt);
+ BranchLong(L, bdslot);
+ bind(&skip);
+ } else {
+ BranchLong(L, bdslot);
+ }
+ }
+ } else {
+ if (is_trampoline_emitted()) {
+ if (cond != cc_always) {
+ Label skip;
+ Condition neg_cond = NegateCondition(cond);
+ BranchShort(&skip, neg_cond, rs, rt);
+ BranchLong(L, bdslot);
+ bind(&skip);
+ } else {
+ BranchLong(L, bdslot);
+ }
+ } else {
+ BranchShort(L, cond, rs, rt, bdslot);
+ }
+ }
+}
+
+void TurboAssembler::Branch(Label* L, Condition cond, Register rs,
+ RootIndex index, BranchDelaySlot bdslot) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ LoadRoot(scratch, index);
+ Branch(L, cond, rs, Operand(scratch), bdslot);
+}
+
+void TurboAssembler::BranchShortHelper(int16_t offset, Label* L,
+ BranchDelaySlot bdslot) {
+ DCHECK(L == nullptr || offset == 0);
+ offset = GetOffset(offset, L, OffsetSize::kOffset16);
+ b(offset);
+
+ // Emit a nop in the branch delay slot if required.
+ if (bdslot == PROTECT) nop();
+}
+
+void TurboAssembler::BranchShortHelperR6(int32_t offset, Label* L) {
+ DCHECK(L == nullptr || offset == 0);
+ offset = GetOffset(offset, L, OffsetSize::kOffset26);
+ bc(offset);
+}
+
+void TurboAssembler::BranchShort(int32_t offset, BranchDelaySlot bdslot) {
+ if (IsMipsArchVariant(kMips32r6) && bdslot == PROTECT) {
+ DCHECK(is_int26(offset));
+ BranchShortHelperR6(offset, nullptr);
+ } else {
+ DCHECK(is_int16(offset));
+ BranchShortHelper(offset, nullptr, bdslot);
+ }
+}
+
+void TurboAssembler::BranchShort(Label* L, BranchDelaySlot bdslot) {
+ if (IsMipsArchVariant(kMips32r6) && bdslot == PROTECT) {
+ BranchShortHelperR6(0, L);
+ } else {
+ BranchShortHelper(0, L, bdslot);
+ }
+}
+
+int32_t TurboAssembler::GetOffset(int32_t offset, Label* L, OffsetSize bits) {
+ if (L) {
+ offset = branch_offset_helper(L, bits) >> 2;
+ } else {
+ DCHECK(is_intn(offset, bits));
+ }
+ return offset;
+}
+
+Register TurboAssembler::GetRtAsRegisterHelper(const Operand& rt,
+ Register scratch) {
+ Register r2 = no_reg;
+ if (rt.is_reg()) {
+ r2 = rt.rm();
+ } else {
+ r2 = scratch;
+ li(r2, rt);
+ }
+
+ return r2;
+}
+
+bool TurboAssembler::CalculateOffset(Label* L, int32_t* offset,
+ OffsetSize bits) {
+ if (!is_near(L, bits)) return false;
+ *offset = GetOffset(*offset, L, bits);
+ return true;
+}
+
+bool TurboAssembler::CalculateOffset(Label* L, int32_t* offset, OffsetSize bits,
+ Register* scratch, const Operand& rt) {
+ if (!is_near(L, bits)) return false;
+ *scratch = GetRtAsRegisterHelper(rt, *scratch);
+ *offset = GetOffset(*offset, L, bits);
+ return true;
+}
+
+bool TurboAssembler::BranchShortHelperR6(int32_t offset, Label* L,
+ Condition cond, Register rs,
+ const Operand& rt) {
+ DCHECK(L == nullptr || offset == 0);
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
+
+ // Be careful to always use shifted_branch_offset only just before the
+ // branch instruction, as the location will be remember for patching the
+ // target.
+ {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ switch (cond) {
+ case cc_always:
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset26)) return false;
+ bc(offset);
+ break;
+ case eq:
+ if (rt.is_reg() && rs.code() == rt.rm().code()) {
+ // Pre R6 beq is used here to make the code patchable. Otherwise bc
+ // should be used which has no condition field so is not patchable.
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ beq(rs, scratch, offset);
+ nop();
+ } else if (IsZero(rt)) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset21)) return false;
+ beqzc(rs, offset);
+ } else {
+ // We don't want any other register but scratch clobbered.
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ beqc(rs, scratch, offset);
+ }
+ break;
+ case ne:
+ if (rt.is_reg() && rs.code() == rt.rm().code()) {
+ // Pre R6 bne is used here to make the code patchable. Otherwise we
+ // should not generate any instruction.
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ bne(rs, scratch, offset);
+ nop();
+ } else if (IsZero(rt)) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset21)) return false;
+ bnezc(rs, offset);
+ } else {
+ // We don't want any other register but scratch clobbered.
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ bnec(rs, scratch, offset);
+ }
+ break;
+
+ // Signed comparison.
+ case greater:
+ // rs > rt
+ if (rt.is_reg() && rs.code() == rt.rm().code()) {
+ break; // No code needs to be emitted.
+ } else if (rs == zero_reg) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ bltzc(scratch, offset);
+ } else if (IsZero(rt)) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16)) return false;
+ bgtzc(rs, offset);
+ } else {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ DCHECK(rs != scratch);
+ bltc(scratch, rs, offset);
+ }
+ break;
+ case greater_equal:
+ // rs >= rt
+ if (rt.is_reg() && rs.code() == rt.rm().code()) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset26)) return false;
+ bc(offset);
+ } else if (rs == zero_reg) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ blezc(scratch, offset);
+ } else if (IsZero(rt)) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16)) return false;
+ bgezc(rs, offset);
+ } else {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ DCHECK(rs != scratch);
+ bgec(rs, scratch, offset);
+ }
+ break;
+ case less:
+ // rs < rt
+ if (rt.is_reg() && rs.code() == rt.rm().code()) {
+ break; // No code needs to be emitted.
+ } else if (rs == zero_reg) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ bgtzc(scratch, offset);
+ } else if (IsZero(rt)) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16)) return false;
+ bltzc(rs, offset);
+ } else {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ DCHECK(rs != scratch);
+ bltc(rs, scratch, offset);
+ }
+ break;
+ case less_equal:
+ // rs <= rt
+ if (rt.is_reg() && rs.code() == rt.rm().code()) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset26)) return false;
+ bc(offset);
+ } else if (rs == zero_reg) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ bgezc(scratch, offset);
+ } else if (IsZero(rt)) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16)) return false;
+ blezc(rs, offset);
+ } else {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ DCHECK(rs != scratch);
+ bgec(scratch, rs, offset);
+ }
+ break;
+
+ // Unsigned comparison.
+ case Ugreater:
+ // rs > rt
+ if (rt.is_reg() && rs.code() == rt.rm().code()) {
+ break; // No code needs to be emitted.
+ } else if (rs == zero_reg) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset21, &scratch, rt))
+ return false;
+ bnezc(scratch, offset);
+ } else if (IsZero(rt)) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset21)) return false;
+ bnezc(rs, offset);
+ } else {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ DCHECK(rs != scratch);
+ bltuc(scratch, rs, offset);
+ }
+ break;
+ case Ugreater_equal:
+ // rs >= rt
+ if (rt.is_reg() && rs.code() == rt.rm().code()) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset26)) return false;
+ bc(offset);
+ } else if (rs == zero_reg) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset21, &scratch, rt))
+ return false;
+ beqzc(scratch, offset);
+ } else if (IsZero(rt)) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset26)) return false;
+ bc(offset);
+ } else {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ DCHECK(rs != scratch);
+ bgeuc(rs, scratch, offset);
+ }
+ break;
+ case Uless:
+ // rs < rt
+ if (rt.is_reg() && rs.code() == rt.rm().code()) {
+ break; // No code needs to be emitted.
+ } else if (rs == zero_reg) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset21, &scratch, rt))
+ return false;
+ bnezc(scratch, offset);
+ } else if (IsZero(rt)) {
+ break; // No code needs to be emitted.
+ } else {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ DCHECK(rs != scratch);
+ bltuc(rs, scratch, offset);
+ }
+ break;
+ case Uless_equal:
+ // rs <= rt
+ if (rt.is_reg() && rs.code() == rt.rm().code()) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset26)) return false;
+ bc(offset);
+ } else if (rs == zero_reg) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset26, &scratch, rt))
+ return false;
+ bc(offset);
+ } else if (IsZero(rt)) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset21)) return false;
+ beqzc(rs, offset);
+ } else {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ DCHECK(rs != scratch);
+ bgeuc(scratch, rs, offset);
+ }
+ break;
+ default:
+ UNREACHABLE();
+ }
+ }
+ CheckTrampolinePoolQuick(1);
+ return true;
+}
+
+bool TurboAssembler::BranchShortHelper(int16_t offset, Label* L, Condition cond,
+ Register rs, const Operand& rt,
+ BranchDelaySlot bdslot) {
+ DCHECK(L == nullptr || offset == 0);
+ if (!is_near(L, OffsetSize::kOffset16)) return false;
+
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
+ int32_t offset32;
+
+ // Be careful to always use shifted_branch_offset only just before the
+ // branch instruction, as the location will be remember for patching the
+ // target.
+ {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ switch (cond) {
+ case cc_always:
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ b(offset32);
+ break;
+ case eq:
+ if (IsZero(rt)) {
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ beq(rs, zero_reg, offset32);
+ } else {
+ // We don't want any other register but scratch clobbered.
+ scratch = GetRtAsRegisterHelper(rt, scratch);
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ beq(rs, scratch, offset32);
+ }
+ break;
+ case ne:
+ if (IsZero(rt)) {
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ bne(rs, zero_reg, offset32);
+ } else {
+ // We don't want any other register but scratch clobbered.
+ scratch = GetRtAsRegisterHelper(rt, scratch);
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ bne(rs, scratch, offset32);
+ }
+ break;
+
+ // Signed comparison.
+ case greater:
+ if (IsZero(rt)) {
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ bgtz(rs, offset32);
+ } else {
+ Slt(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ bne(scratch, zero_reg, offset32);
+ }
+ break;
+ case greater_equal:
+ if (IsZero(rt)) {
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ bgez(rs, offset32);
+ } else {
+ Slt(scratch, rs, rt);
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ beq(scratch, zero_reg, offset32);
+ }
+ break;
+ case less:
+ if (IsZero(rt)) {
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ bltz(rs, offset32);
+ } else {
+ Slt(scratch, rs, rt);
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ bne(scratch, zero_reg, offset32);
+ }
+ break;
+ case less_equal:
+ if (IsZero(rt)) {
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ blez(rs, offset32);
+ } else {
+ Slt(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ beq(scratch, zero_reg, offset32);
+ }
+ break;
+
+ // Unsigned comparison.
+ case Ugreater:
+ if (IsZero(rt)) {
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ bne(rs, zero_reg, offset32);
+ } else {
+ Sltu(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ bne(scratch, zero_reg, offset32);
+ }
+ break;
+ case Ugreater_equal:
+ if (IsZero(rt)) {
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ b(offset32);
+ } else {
+ Sltu(scratch, rs, rt);
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ beq(scratch, zero_reg, offset32);
+ }
+ break;
+ case Uless:
+ if (IsZero(rt)) {
+ return true; // No code needs to be emitted.
+ } else {
+ Sltu(scratch, rs, rt);
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ bne(scratch, zero_reg, offset32);
+ }
+ break;
+ case Uless_equal:
+ if (IsZero(rt)) {
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ beq(rs, zero_reg, offset32);
+ } else {
+ Sltu(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ beq(scratch, zero_reg, offset32);
+ }
+ break;
+ default:
+ UNREACHABLE();
+ }
+ }
+ // Emit a nop in the branch delay slot if required.
+ if (bdslot == PROTECT) nop();
+
+ return true;
+}
+
+bool TurboAssembler::BranchShortCheck(int32_t offset, Label* L, Condition cond,
+ Register rs, const Operand& rt,
+ BranchDelaySlot bdslot) {
+ BRANCH_ARGS_CHECK(cond, rs, rt);
+ if (!L) {
+ if (IsMipsArchVariant(kMips32r6) && bdslot == PROTECT) {
+ DCHECK(is_int26(offset));
+ return BranchShortHelperR6(offset, nullptr, cond, rs, rt);
+ } else {
+ DCHECK(is_int16(offset));
+ return BranchShortHelper(offset, nullptr, cond, rs, rt, bdslot);
+ }
+ } else {
+ DCHECK_EQ(offset, 0);
+ if (IsMipsArchVariant(kMips32r6) && bdslot == PROTECT) {
+ return BranchShortHelperR6(0, L, cond, rs, rt);
+ } else {
+ return BranchShortHelper(0, L, cond, rs, rt, bdslot);
+ }
+ }
+ return false;
+}
+
+void TurboAssembler::BranchShort(int32_t offset, Condition cond, Register rs,
+ const Operand& rt, BranchDelaySlot bdslot) {
+ BranchShortCheck(offset, nullptr, cond, rs, rt, bdslot);
+}
+
+void TurboAssembler::BranchShort(Label* L, Condition cond, Register rs,
+ const Operand& rt, BranchDelaySlot bdslot) {
+ BranchShortCheck(0, L, cond, rs, rt, bdslot);
+}
+
+void TurboAssembler::BranchAndLink(int32_t offset, BranchDelaySlot bdslot) {
+ BranchAndLinkShort(offset, bdslot);
+}
+
+void TurboAssembler::BranchAndLink(int32_t offset, Condition cond, Register rs,
+ const Operand& rt, BranchDelaySlot bdslot) {
+ bool is_near = BranchAndLinkShortCheck(offset, nullptr, cond, rs, rt, bdslot);
+ DCHECK(is_near);
+ USE(is_near);
+}
+
+void TurboAssembler::BranchAndLink(Label* L, BranchDelaySlot bdslot) {
+ if (L->is_bound()) {
+ if (is_near_branch(L)) {
+ BranchAndLinkShort(L, bdslot);
+ } else {
+ BranchAndLinkLong(L, bdslot);
+ }
+ } else {
+ if (is_trampoline_emitted()) {
+ BranchAndLinkLong(L, bdslot);
+ } else {
+ BranchAndLinkShort(L, bdslot);
+ }
+ }
+}
+
+void TurboAssembler::BranchAndLink(Label* L, Condition cond, Register rs,
+ const Operand& rt, BranchDelaySlot bdslot) {
+ if (L->is_bound()) {
+ if (!BranchAndLinkShortCheck(0, L, cond, rs, rt, bdslot)) {
+ Label skip;
+ Condition neg_cond = NegateCondition(cond);
+ BranchShort(&skip, neg_cond, rs, rt);
+ BranchAndLinkLong(L, bdslot);
+ bind(&skip);
+ }
+ } else {
+ if (is_trampoline_emitted()) {
+ Label skip;
+ Condition neg_cond = NegateCondition(cond);
+ BranchShort(&skip, neg_cond, rs, rt);
+ BranchAndLinkLong(L, bdslot);
+ bind(&skip);
+ } else {
+ BranchAndLinkShortCheck(0, L, cond, rs, rt, bdslot);
+ }
+ }
+}
+
+void TurboAssembler::BranchAndLinkShortHelper(int16_t offset, Label* L,
+ BranchDelaySlot bdslot) {
+ DCHECK(L == nullptr || offset == 0);
+ offset = GetOffset(offset, L, OffsetSize::kOffset16);
+ bal(offset);
+
+ // Emit a nop in the branch delay slot if required.
+ if (bdslot == PROTECT) nop();
+}
+
+void TurboAssembler::BranchAndLinkShortHelperR6(int32_t offset, Label* L) {
+ DCHECK(L == nullptr || offset == 0);
+ offset = GetOffset(offset, L, OffsetSize::kOffset26);
+ balc(offset);
+}
+
+void TurboAssembler::BranchAndLinkShort(int32_t offset,
+ BranchDelaySlot bdslot) {
+ if (IsMipsArchVariant(kMips32r6) && bdslot == PROTECT) {
+ DCHECK(is_int26(offset));
+ BranchAndLinkShortHelperR6(offset, nullptr);
+ } else {
+ DCHECK(is_int16(offset));
+ BranchAndLinkShortHelper(offset, nullptr, bdslot);
+ }
+}
+
+void TurboAssembler::BranchAndLinkShort(Label* L, BranchDelaySlot bdslot) {
+ if (IsMipsArchVariant(kMips32r6) && bdslot == PROTECT) {
+ BranchAndLinkShortHelperR6(0, L);
+ } else {
+ BranchAndLinkShortHelper(0, L, bdslot);
+ }
+}
+
+bool TurboAssembler::BranchAndLinkShortHelperR6(int32_t offset, Label* L,
+ Condition cond, Register rs,
+ const Operand& rt) {
+ DCHECK(L == nullptr || offset == 0);
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
+ OffsetSize bits = OffsetSize::kOffset16;
+
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ DCHECK((cond == cc_always && is_int26(offset)) || is_int16(offset));
+ switch (cond) {
+ case cc_always:
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset26)) return false;
+ balc(offset);
+ break;
+ case eq:
+ if (!is_near(L, bits)) return false;
+ Subu(scratch, rs, rt);
+ offset = GetOffset(offset, L, bits);
+ beqzalc(scratch, offset);
+ break;
+ case ne:
+ if (!is_near(L, bits)) return false;
+ Subu(scratch, rs, rt);
+ offset = GetOffset(offset, L, bits);
+ bnezalc(scratch, offset);
+ break;
+
+ // Signed comparison.
+ case greater:
+ // rs > rt
+ if (rs.code() == rt.rm().code()) {
+ break; // No code needs to be emitted.
+ } else if (rs == zero_reg) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ bltzalc(scratch, offset);
+ } else if (IsZero(rt)) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16)) return false;
+ bgtzalc(rs, offset);
+ } else {
+ if (!is_near(L, bits)) return false;
+ Slt(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
+ offset = GetOffset(offset, L, bits);
+ bnezalc(scratch, offset);
+ }
+ break;
+ case greater_equal:
+ // rs >= rt
+ if (rs.code() == rt.rm().code()) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset26)) return false;
+ balc(offset);
+ } else if (rs == zero_reg) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ blezalc(scratch, offset);
+ } else if (IsZero(rt)) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16)) return false;
+ bgezalc(rs, offset);
+ } else {
+ if (!is_near(L, bits)) return false;
+ Slt(scratch, rs, rt);
+ offset = GetOffset(offset, L, bits);
+ beqzalc(scratch, offset);
+ }
+ break;
+ case less:
+ // rs < rt
+ if (rs.code() == rt.rm().code()) {
+ break; // No code needs to be emitted.
+ } else if (rs == zero_reg) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ bgtzalc(scratch, offset);
+ } else if (IsZero(rt)) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16)) return false;
+ bltzalc(rs, offset);
+ } else {
+ if (!is_near(L, bits)) return false;
+ Slt(scratch, rs, rt);
+ offset = GetOffset(offset, L, bits);
+ bnezalc(scratch, offset);
+ }
+ break;
+ case less_equal:
+ // rs <= r2
+ if (rs.code() == rt.rm().code()) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset26)) return false;
+ balc(offset);
+ } else if (rs == zero_reg) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ bgezalc(scratch, offset);
+ } else if (IsZero(rt)) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16)) return false;
+ blezalc(rs, offset);
+ } else {
+ if (!is_near(L, bits)) return false;
+ Slt(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
+ offset = GetOffset(offset, L, bits);
+ beqzalc(scratch, offset);
+ }
+ break;
+
+ // Unsigned comparison.
+ case Ugreater:
+ // rs > r2
+ if (!is_near(L, bits)) return false;
+ Sltu(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
+ offset = GetOffset(offset, L, bits);
+ bnezalc(scratch, offset);
+ break;
+ case Ugreater_equal:
+ // rs >= r2
+ if (!is_near(L, bits)) return false;
+ Sltu(scratch, rs, rt);
+ offset = GetOffset(offset, L, bits);
+ beqzalc(scratch, offset);
+ break;
+ case Uless:
+ // rs < r2
+ if (!is_near(L, bits)) return false;
+ Sltu(scratch, rs, rt);
+ offset = GetOffset(offset, L, bits);
+ bnezalc(scratch, offset);
+ break;
+ case Uless_equal:
+ // rs <= r2
+ if (!is_near(L, bits)) return false;
+ Sltu(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
+ offset = GetOffset(offset, L, bits);
+ beqzalc(scratch, offset);
+ break;
+ default:
+ UNREACHABLE();
+ }
+ return true;
+}
+
+// Pre r6 we need to use a bgezal or bltzal, but they can't be used directly
+// with the slt instructions. We could use sub or add instead but we would miss
+// overflow cases, so we keep slt and add an intermediate third instruction.
+bool TurboAssembler::BranchAndLinkShortHelper(int16_t offset, Label* L,
+ Condition cond, Register rs,
+ const Operand& rt,
+ BranchDelaySlot bdslot) {
+ DCHECK(L == nullptr || offset == 0);
+ if (!is_near(L, OffsetSize::kOffset16)) return false;
+
+ Register scratch = t8;
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+
+ switch (cond) {
+ case cc_always:
+ offset = GetOffset(offset, L, OffsetSize::kOffset16);
+ bal(offset);
+ break;
+ case eq:
+ bne(rs, GetRtAsRegisterHelper(rt, scratch), 2);
+ nop();
+ offset = GetOffset(offset, L, OffsetSize::kOffset16);
+ bal(offset);
+ break;
+ case ne:
+ beq(rs, GetRtAsRegisterHelper(rt, scratch), 2);
+ nop();
+ offset = GetOffset(offset, L, OffsetSize::kOffset16);
+ bal(offset);
+ break;
+
+ // Signed comparison.
+ case greater:
+ Slt(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
+ addiu(scratch, scratch, -1);
+ offset = GetOffset(offset, L, OffsetSize::kOffset16);
+ bgezal(scratch, offset);
+ break;
+ case greater_equal:
+ Slt(scratch, rs, rt);
+ addiu(scratch, scratch, -1);
+ offset = GetOffset(offset, L, OffsetSize::kOffset16);
+ bltzal(scratch, offset);
+ break;
+ case less:
+ Slt(scratch, rs, rt);
+ addiu(scratch, scratch, -1);
+ offset = GetOffset(offset, L, OffsetSize::kOffset16);
+ bgezal(scratch, offset);
+ break;
+ case less_equal:
+ Slt(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
+ addiu(scratch, scratch, -1);
+ offset = GetOffset(offset, L, OffsetSize::kOffset16);
+ bltzal(scratch, offset);
+ break;
+
+ // Unsigned comparison.
+ case Ugreater:
+ Sltu(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
+ addiu(scratch, scratch, -1);
+ offset = GetOffset(offset, L, OffsetSize::kOffset16);
+ bgezal(scratch, offset);
+ break;
+ case Ugreater_equal:
+ Sltu(scratch, rs, rt);
+ addiu(scratch, scratch, -1);
+ offset = GetOffset(offset, L, OffsetSize::kOffset16);
+ bltzal(scratch, offset);
+ break;
+ case Uless:
+ Sltu(scratch, rs, rt);
+ addiu(scratch, scratch, -1);
+ offset = GetOffset(offset, L, OffsetSize::kOffset16);
+ bgezal(scratch, offset);
+ break;
+ case Uless_equal:
+ Sltu(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
+ addiu(scratch, scratch, -1);
+ offset = GetOffset(offset, L, OffsetSize::kOffset16);
+ bltzal(scratch, offset);
+ break;
+
+ default:
+ UNREACHABLE();
+ }
+
+ // Emit a nop in the branch delay slot if required.
+ if (bdslot == PROTECT) nop();
+
+ return true;
+}
+
+bool TurboAssembler::BranchAndLinkShortCheck(int32_t offset, Label* L,
+ Condition cond, Register rs,
+ const Operand& rt,
+ BranchDelaySlot bdslot) {
+ BRANCH_ARGS_CHECK(cond, rs, rt);
+
+ if (!L) {
+ if (IsMipsArchVariant(kMips32r6) && bdslot == PROTECT) {
+ DCHECK(is_int26(offset));
+ return BranchAndLinkShortHelperR6(offset, nullptr, cond, rs, rt);
+ } else {
+ DCHECK(is_int16(offset));
+ return BranchAndLinkShortHelper(offset, nullptr, cond, rs, rt, bdslot);
+ }
+ } else {
+ DCHECK_EQ(offset, 0);
+ if (IsMipsArchVariant(kMips32r6) && bdslot == PROTECT) {
+ return BranchAndLinkShortHelperR6(0, L, cond, rs, rt);
+ } else {
+ return BranchAndLinkShortHelper(0, L, cond, rs, rt, bdslot);
+ }
+ }
+ return false;
+}
+
+void TurboAssembler::LoadFromConstantsTable(Register destination,
+ int constant_index) {
+ DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kBuiltinsConstantsTable));
+ LoadRoot(destination, RootIndex::kBuiltinsConstantsTable);
+ lw(destination,
+ FieldMemOperand(destination,
+ FixedArray::kHeaderSize + constant_index * kPointerSize));
+}
+
+void TurboAssembler::LoadRootRelative(Register destination, int32_t offset) {
+ lw(destination, MemOperand(kRootRegister, offset));
+}
+
+void TurboAssembler::LoadRootRegisterOffset(Register destination,
+ intptr_t offset) {
+ if (offset == 0) {
+ Move(destination, kRootRegister);
+ } else {
+ Addu(destination, kRootRegister, offset);
+ }
+}
+
+void TurboAssembler::Jump(Register target, int16_t offset, Condition cond,
+ Register rs, const Operand& rt, BranchDelaySlot bd) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ DCHECK(is_int16(offset));
+ if (IsMipsArchVariant(kMips32r6) && bd == PROTECT) {
+ if (cond == cc_always) {
+ jic(target, offset);
+ } else {
+ BRANCH_ARGS_CHECK(cond, rs, rt);
+ Branch(2, NegateCondition(cond), rs, rt);
+ jic(target, offset);
+ }
+ } else {
+ if (offset != 0) {
+ Addu(target, target, offset);
+ }
+ if (cond == cc_always) {
+ jr(target);
+ } else {
+ BRANCH_ARGS_CHECK(cond, rs, rt);
+ Branch(2, NegateCondition(cond), rs, rt);
+ jr(target);
+ }
+ // Emit a nop in the branch delay slot if required.
+ if (bd == PROTECT) nop();
+ }
+}
+
+void TurboAssembler::Jump(Register target, Register base, int16_t offset,
+ Condition cond, Register rs, const Operand& rt,
+ BranchDelaySlot bd) {
+ DCHECK(is_int16(offset));
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ if (IsMipsArchVariant(kMips32r6) && bd == PROTECT) {
+ if (cond == cc_always) {
+ jic(base, offset);
+ } else {
+ BRANCH_ARGS_CHECK(cond, rs, rt);
+ Branch(2, NegateCondition(cond), rs, rt);
+ jic(base, offset);
+ }
+ } else {
+ if (offset != 0) {
+ Addu(target, base, offset);
+ } else { // Call through target
+ if (target != base) mov(target, base);
+ }
+ if (cond == cc_always) {
+ jr(target);
+ } else {
+ BRANCH_ARGS_CHECK(cond, rs, rt);
+ Branch(2, NegateCondition(cond), rs, rt);
+ jr(target);
+ }
+ // Emit a nop in the branch delay slot if required.
+ if (bd == PROTECT) nop();
+ }
+}
+
+void TurboAssembler::Jump(Register target, const Operand& offset,
+ Condition cond, Register rs, const Operand& rt,
+ BranchDelaySlot bd) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ if (IsMipsArchVariant(kMips32r6) && bd == PROTECT &&
+ !is_int16(offset.immediate())) {
+ uint32_t aui_offset, jic_offset;
+ Assembler::UnpackTargetAddressUnsigned(offset.immediate(), &aui_offset,
+ &jic_offset);
+ RecordRelocInfo(RelocInfo::EXTERNAL_REFERENCE, offset.immediate());
+ aui(target, target, aui_offset);
+ if (cond == cc_always) {
+ jic(target, jic_offset);
+ } else {
+ BRANCH_ARGS_CHECK(cond, rs, rt);
+ Branch(2, NegateCondition(cond), rs, rt);
+ jic(target, jic_offset);
+ }
+ } else {
+ if (offset.immediate() != 0) {
+ Addu(target, target, offset);
+ }
+ if (cond == cc_always) {
+ jr(target);
+ } else {
+ BRANCH_ARGS_CHECK(cond, rs, rt);
+ Branch(2, NegateCondition(cond), rs, rt);
+ jr(target);
+ }
+ // Emit a nop in the branch delay slot if required.
+ if (bd == PROTECT) nop();
+ }
+}
+
+void TurboAssembler::Jump(intptr_t target, RelocInfo::Mode rmode,
+ Condition cond, Register rs, const Operand& rt,
+ BranchDelaySlot bd) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Label skip;
+ if (cond != cc_always) {
+ Branch(USE_DELAY_SLOT, &skip, NegateCondition(cond), rs, rt);
+ }
+ // The first instruction of 'li' may be placed in the delay slot.
+ // This is not an issue, t9 is expected to be clobbered anyway.
+ if (IsMipsArchVariant(kMips32r6) && bd == PROTECT) {
+ uint32_t lui_offset, jic_offset;
+ UnpackTargetAddressUnsigned(target, &lui_offset, &jic_offset);
+ if (MustUseReg(rmode)) {
+ RecordRelocInfo(rmode, target);
+ }
+ lui(t9, lui_offset);
+ Jump(t9, jic_offset, al, zero_reg, Operand(zero_reg), bd);
+ } else {
+ li(t9, Operand(target, rmode));
+ Jump(t9, 0, al, zero_reg, Operand(zero_reg), bd);
+ }
+ bind(&skip);
+}
+
+void TurboAssembler::Jump(Address target, RelocInfo::Mode rmode, Condition cond,
+ Register rs, const Operand& rt, BranchDelaySlot bd) {
+ DCHECK(!RelocInfo::IsCodeTarget(rmode));
+ Jump(static_cast<intptr_t>(target), rmode, cond, rs, rt, bd);
+}
+
+void TurboAssembler::Jump(Handle<Code> code, RelocInfo::Mode rmode,
+ Condition cond, Register rs, const Operand& rt,
+ BranchDelaySlot bd) {
+ DCHECK(RelocInfo::IsCodeTarget(rmode));
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+
+ int builtin_index = Builtins::kNoBuiltinId;
+ bool target_is_isolate_independent_builtin =
+ isolate()->builtins()->IsBuiltinHandle(code, &builtin_index) &&
+ Builtins::IsIsolateIndependent(builtin_index);
+ if (target_is_isolate_independent_builtin &&
+ options().use_pc_relative_calls_and_jumps) {
+ int32_t code_target_index = AddCodeTarget(code);
+ Label skip;
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ if (cond != cc_always) {
+ // By using delay slot, we always execute first instruction of
+ // GenPcRelativeJump (which is or_(t8, ra, zero_reg)).
+ Branch(USE_DELAY_SLOT, &skip, NegateCondition(cond), rs, rt);
+ }
+ GenPCRelativeJump(t8, t9, code_target_index,
+ RelocInfo::RELATIVE_CODE_TARGET, bd);
+ bind(&skip);
+ return;
+ } else if (root_array_available_ && options().isolate_independent_code) {
+ IndirectLoadConstant(t9, code);
+ Jump(t9, Code::kHeaderSize - kHeapObjectTag, cond, rs, rt, bd);
+ return;
+ } else if (target_is_isolate_independent_builtin &&
+ options().inline_offheap_trampolines) {
+ // Inline the trampoline.
+ RecordCommentForOffHeapTrampoline(builtin_index);
+ CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
+ EmbeddedData d = EmbeddedData::FromBlob();
+ Address entry = d.InstructionStartOfBuiltin(builtin_index);
+ li(t9, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+ Jump(t9, 0, cond, rs, rt, bd);
+ return;
+ }
+
+ Jump(static_cast<intptr_t>(code.address()), rmode, cond, rs, rt, bd);
+}
+
+void TurboAssembler::Jump(const ExternalReference& reference) {
+ li(t9, reference);
+ Jump(t9);
+}
+
+void MacroAssembler::JumpIfIsInRange(Register value, unsigned lower_limit,
+ unsigned higher_limit,
+ Label* on_in_range) {
+ if (lower_limit != 0) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ Subu(scratch, value, Operand(lower_limit));
+ Branch(on_in_range, ls, scratch, Operand(higher_limit - lower_limit));
+ } else {
+ Branch(on_in_range, ls, value, Operand(higher_limit - lower_limit));
+ }
+}
+
+// Note: To call gcc-compiled C code on mips, you must call through t9.
+void TurboAssembler::Call(Register target, int16_t offset, Condition cond,
+ Register rs, const Operand& rt, BranchDelaySlot bd) {
+ DCHECK(is_int16(offset));
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ if (IsMipsArchVariant(kMips32r6) && bd == PROTECT) {
+ if (cond == cc_always) {
+ jialc(target, offset);
+ } else {
+ BRANCH_ARGS_CHECK(cond, rs, rt);
+ Branch(2, NegateCondition(cond), rs, rt);
+ jialc(target, offset);
+ }
+ } else {
+ if (offset != 0) {
+ Addu(target, target, offset);
+ }
+ if (cond == cc_always) {
+ jalr(target);
+ } else {
+ BRANCH_ARGS_CHECK(cond, rs, rt);
+ Branch(2, NegateCondition(cond), rs, rt);
+ jalr(target);
+ }
+ // Emit a nop in the branch delay slot if required.
+ if (bd == PROTECT) nop();
+ }
+ set_last_call_pc_(pc_);
+}
+
+// Note: To call gcc-compiled C code on mips, you must call through t9.
+void TurboAssembler::Call(Register target, Register base, int16_t offset,
+ Condition cond, Register rs, const Operand& rt,
+ BranchDelaySlot bd) {
+ DCHECK(is_uint16(offset));
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ if (IsMipsArchVariant(kMips32r6) && bd == PROTECT) {
+ if (cond == cc_always) {
+ jialc(base, offset);
+ } else {
+ BRANCH_ARGS_CHECK(cond, rs, rt);
+ Branch(2, NegateCondition(cond), rs, rt);
+ jialc(base, offset);
+ }
+ } else {
+ if (offset != 0) {
+ Addu(target, base, offset);
+ } else { // Call through target
+ if (target != base) mov(target, base);
+ }
+ if (cond == cc_always) {
+ jalr(target);
+ } else {
+ BRANCH_ARGS_CHECK(cond, rs, rt);
+ Branch(2, NegateCondition(cond), rs, rt);
+ jalr(target);
+ }
+ // Emit a nop in the branch delay slot if required.
+ if (bd == PROTECT) nop();
+ }
+ set_last_call_pc_(pc_);
+}
+
+void TurboAssembler::Call(Address target, RelocInfo::Mode rmode, Condition cond,
+ Register rs, const Operand& rt, BranchDelaySlot bd) {
+ CheckBuffer();
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ int32_t target_int = static_cast<int32_t>(target);
+ if (IsMipsArchVariant(kMips32r6) && bd == PROTECT && cond == cc_always) {
+ uint32_t lui_offset, jialc_offset;
+ UnpackTargetAddressUnsigned(target_int, &lui_offset, &jialc_offset);
+ if (MustUseReg(rmode)) {
+ RecordRelocInfo(rmode, target_int);
+ }
+ lui(t9, lui_offset);
+ Call(t9, jialc_offset, cond, rs, rt, bd);
+ } else {
+ li(t9, Operand(target_int, rmode), CONSTANT_SIZE);
+ Call(t9, 0, cond, rs, rt, bd);
+ }
+}
+
+void TurboAssembler::Call(Handle<Code> code, RelocInfo::Mode rmode,
+ Condition cond, Register rs, const Operand& rt,
+ BranchDelaySlot bd) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+
+ int builtin_index = Builtins::kNoBuiltinId;
+ bool target_is_isolate_independent_builtin =
+ isolate()->builtins()->IsBuiltinHandle(code, &builtin_index) &&
+ Builtins::IsIsolateIndependent(builtin_index);
+ if (target_is_isolate_independent_builtin &&
+ options().use_pc_relative_calls_and_jumps) {
+ int32_t code_target_index = AddCodeTarget(code);
+ Label skip;
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ if (cond != cc_always) {
+ Branch(PROTECT, &skip, NegateCondition(cond), rs, rt);
+ }
+ GenPCRelativeJumpAndLink(t8, code_target_index,
+ RelocInfo::RELATIVE_CODE_TARGET, bd);
+ bind(&skip);
+ return;
+ } else if (root_array_available_ && options().isolate_independent_code) {
+ IndirectLoadConstant(t9, code);
+ Call(t9, Code::kHeaderSize - kHeapObjectTag, cond, rs, rt, bd);
+ return;
+ } else if (target_is_isolate_independent_builtin &&
+ options().inline_offheap_trampolines) {
+ // Inline the trampoline.
+ RecordCommentForOffHeapTrampoline(builtin_index);
+ CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
+ EmbeddedData d = EmbeddedData::FromBlob();
+ Address entry = d.InstructionStartOfBuiltin(builtin_index);
+ li(t9, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+ Call(t9, 0, cond, rs, rt, bd);
+ return;
+ }
+
+ DCHECK(RelocInfo::IsCodeTarget(rmode));
+ DCHECK(code->IsExecutable());
+ Call(code.address(), rmode, cond, rs, rt, bd);
+}
+
+void TurboAssembler::LoadEntryFromBuiltinIndex(Register builtin_index) {
+ STATIC_ASSERT(kSystemPointerSize == 4);
+ STATIC_ASSERT(kSmiShiftSize == 0);
+ STATIC_ASSERT(kSmiTagSize == 1);
+ STATIC_ASSERT(kSmiTag == 0);
+
+ // The builtin_index register contains the builtin index as a Smi.
+ SmiUntag(builtin_index, builtin_index);
+ Lsa(builtin_index, kRootRegister, builtin_index, kSystemPointerSizeLog2);
+ lw(builtin_index,
+ MemOperand(builtin_index, IsolateData::builtin_entry_table_offset()));
+}
+
+void TurboAssembler::CallBuiltinByIndex(Register builtin_index) {
+ LoadEntryFromBuiltinIndex(builtin_index);
+ Call(builtin_index);
+}
+
+void TurboAssembler::PatchAndJump(Address target) {
+ if (kArchVariant != kMips32r6) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ mov(scratch, ra);
+ bal(1); // jump to lw
+ nop(); // in the delay slot
+ lw(t9, MemOperand(ra, kInstrSize * 3)); // ra == pc_
+ jr(t9);
+ mov(ra, scratch); // in delay slot
+ DCHECK_EQ(reinterpret_cast<uint32_t>(pc_) % 8, 0);
+ *reinterpret_cast<uint32_t*>(pc_) = target;
+ pc_ += sizeof(uint32_t);
+ } else {
+ // TODO(mips r6): Implement.
+ UNIMPLEMENTED();
+ }
+}
+
+void TurboAssembler::StoreReturnAddressAndCall(Register target) {
+ // This generates the final instruction sequence for calls to C functions
+ // once an exit frame has been constructed.
+ //
+ // Note that this assumes the caller code (i.e. the Code object currently
+ // being generated) is immovable or that the callee function cannot trigger
+ // GC, since the callee function will return to it.
+
+ Assembler::BlockTrampolinePoolScope block_trampoline_pool(this);
+ static constexpr int kNumInstructionsToJump = 4;
+ Label find_ra;
+ // Adjust the value in ra to point to the correct return location, 2nd
+ // instruction past the real call into C code (the jalr(t9)), and push it.
+ // This is the return address of the exit frame.
+ if (kArchVariant >= kMips32r6) {
+ addiupc(ra, kNumInstructionsToJump + 1);
+ } else {
+ // This no-op-and-link sequence saves PC + 8 in ra register on pre-r6 MIPS
+ nal(); // nal has branch delay slot.
+ Addu(ra, ra, kNumInstructionsToJump * kInstrSize);
+ }
+ bind(&find_ra);
+
+ // This spot was reserved in EnterExitFrame.
+ sw(ra, MemOperand(sp));
+ // Stack space reservation moved to the branch delay slot below.
+ // Stack is still aligned.
+
+ // Call the C routine.
+ mov(t9, target); // Function pointer to t9 to conform to ABI for PIC.
+ jalr(t9);
+ // Set up sp in the delay slot.
+ addiu(sp, sp, -kCArgsSlotsSize);
+ // Make sure the stored 'ra' points to this position.
+ DCHECK_EQ(kNumInstructionsToJump, InstructionsGeneratedSince(&find_ra));
+}
+
+void TurboAssembler::Ret(Condition cond, Register rs, const Operand& rt,
+ BranchDelaySlot bd) {
+ Jump(ra, 0, cond, rs, rt, bd);
+}
+
+void TurboAssembler::BranchLong(Label* L, BranchDelaySlot bdslot) {
+ if (IsMipsArchVariant(kMips32r6) && bdslot == PROTECT &&
+ (!L->is_bound() || is_near_r6(L))) {
+ BranchShortHelperR6(0, L);
+ } else {
+ // Generate position independent long branch.
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ int32_t imm32;
+ imm32 = branch_long_offset(L);
+ GenPCRelativeJump(t8, t9, imm32, RelocInfo::NONE, bdslot);
+ }
+}
+
+void TurboAssembler::BranchAndLinkLong(Label* L, BranchDelaySlot bdslot) {
+ if (IsMipsArchVariant(kMips32r6) && bdslot == PROTECT &&
+ (!L->is_bound() || is_near_r6(L))) {
+ BranchAndLinkShortHelperR6(0, L);
+ } else {
+ // Generate position independent long branch and link.
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ int32_t imm32;
+ imm32 = branch_long_offset(L);
+ GenPCRelativeJumpAndLink(t8, imm32, RelocInfo::NONE, bdslot);
+ }
+}
+
+void TurboAssembler::DropAndRet(int drop) {
+ int32_t drop_size = drop * kSystemPointerSize;
+ DCHECK(is_int31(drop_size));
+
+ if (is_int16(drop_size)) {
+ Ret(USE_DELAY_SLOT);
+ addiu(sp, sp, drop_size);
+ } else {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ li(scratch, drop_size);
+ Ret(USE_DELAY_SLOT);
+ addu(sp, sp, scratch);
+ }
+}
+
+void TurboAssembler::DropAndRet(int drop, Condition cond, Register r1,
+ const Operand& r2) {
+ // Both Drop and Ret need to be conditional.
+ Label skip;
+ if (cond != cc_always) {
+ Branch(&skip, NegateCondition(cond), r1, r2);
+ }
+
+ Drop(drop);
+ Ret();
+
+ if (cond != cc_always) {
+ bind(&skip);
+ }
+}
+
+void TurboAssembler::Drop(int count, Condition cond, Register reg,
+ const Operand& op) {
+ if (count <= 0) {
+ return;
+ }
+
+ Label skip;
+
+ if (cond != al) {
+ Branch(&skip, NegateCondition(cond), reg, op);
+ }
+
+ Addu(sp, sp, Operand(count * kPointerSize));
+
+ if (cond != al) {
+ bind(&skip);
+ }
+}
+
+void MacroAssembler::Swap(Register reg1, Register reg2, Register scratch) {
+ if (scratch == no_reg) {
+ Xor(reg1, reg1, Operand(reg2));
+ Xor(reg2, reg2, Operand(reg1));
+ Xor(reg1, reg1, Operand(reg2));
+ } else {
+ mov(scratch, reg1);
+ mov(reg1, reg2);
+ mov(reg2, scratch);
+ }
+}
+
+void TurboAssembler::Call(Label* target) { BranchAndLink(target); }
+
+void TurboAssembler::LoadAddress(Register dst, Label* target) {
+ uint32_t address = jump_address(target);
+ li(dst, address);
+}
+
+void TurboAssembler::Push(Handle<HeapObject> handle) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ li(scratch, Operand(handle));
+ push(scratch);
+}
+
+void TurboAssembler::Push(Smi smi) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ li(scratch, Operand(smi));
+ push(scratch);
+}
+
+void TurboAssembler::PushArray(Register array, Register size, Register scratch,
+ Register scratch2, PushArrayOrder order) {
+ DCHECK(!AreAliased(array, size, scratch, scratch2));
+ Label loop, entry;
+ if (order == PushArrayOrder::kReverse) {
+ mov(scratch, zero_reg);
+ jmp(&entry);
+ bind(&loop);
+ Lsa(scratch2, array, scratch, kPointerSizeLog2);
+ Lw(scratch2, MemOperand(scratch2));
+ push(scratch2);
+ Addu(scratch, scratch, Operand(1));
+ bind(&entry);
+ Branch(&loop, less, scratch, Operand(size));
+ } else {
+ mov(scratch, size);
+ jmp(&entry);
+ bind(&loop);
+ Lsa(scratch2, array, scratch, kPointerSizeLog2);
+ Lw(scratch2, MemOperand(scratch2));
+ push(scratch2);
+ bind(&entry);
+ Addu(scratch, scratch, Operand(-1));
+ Branch(&loop, greater_equal, scratch, Operand(zero_reg));
+ }
+}
+
+void MacroAssembler::MaybeDropFrames() {
+ // Check whether we need to drop frames to restart a function on the stack.
+ li(a1, ExternalReference::debug_restart_fp_address(isolate()));
+ lw(a1, MemOperand(a1));
+ Jump(BUILTIN_CODE(isolate(), FrameDropperTrampoline), RelocInfo::CODE_TARGET,
+ ne, a1, Operand(zero_reg));
+}
+
+// ---------------------------------------------------------------------------
+// Exception handling.
+
+void MacroAssembler::PushStackHandler() {
+ // Adjust this code if not the case.
+ STATIC_ASSERT(StackHandlerConstants::kSize == 2 * kPointerSize);
+ STATIC_ASSERT(StackHandlerConstants::kNextOffset == 0 * kPointerSize);
+
+ Push(Smi::zero()); // Padding.
+
+ // Link the current handler as the next handler.
+ li(t2,
+ ExternalReference::Create(IsolateAddressId::kHandlerAddress, isolate()));
+ lw(t1, MemOperand(t2));
+ push(t1);
+
+ // Set this new handler as the current one.
+ sw(sp, MemOperand(t2));
+}
+
+void MacroAssembler::PopStackHandler() {
+ STATIC_ASSERT(StackHandlerConstants::kNextOffset == 0);
+ pop(a1);
+ Addu(sp, sp, Operand(StackHandlerConstants::kSize - kPointerSize));
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ li(scratch,
+ ExternalReference::Create(IsolateAddressId::kHandlerAddress, isolate()));
+ sw(a1, MemOperand(scratch));
+}
+
+void TurboAssembler::FPUCanonicalizeNaN(const DoubleRegister dst,
+ const DoubleRegister src) {
+ sub_d(dst, src, kDoubleRegZero);
+}
+
+void TurboAssembler::MovFromFloatResult(DoubleRegister dst) {
+ if (IsMipsSoftFloatABI) {
+ if (kArchEndian == kLittle) {
+ Move(dst, v0, v1);
+ } else {
+ Move(dst, v1, v0);
+ }
+ } else {
+ Move(dst, f0); // Reg f0 is o32 ABI FP return value.
+ }
+}
+
+void TurboAssembler::MovFromFloatParameter(DoubleRegister dst) {
+ if (IsMipsSoftFloatABI) {
+ if (kArchEndian == kLittle) {
+ Move(dst, a0, a1);
+ } else {
+ Move(dst, a1, a0);
+ }
+ } else {
+ Move(dst, f12); // Reg f12 is o32 ABI FP first argument value.
+ }
+}
+
+void TurboAssembler::MovToFloatParameter(DoubleRegister src) {
+ if (!IsMipsSoftFloatABI) {
+ Move(f12, src);
+ } else {
+ if (kArchEndian == kLittle) {
+ Move(a0, a1, src);
+ } else {
+ Move(a1, a0, src);
+ }
+ }
+}
+
+void TurboAssembler::MovToFloatResult(DoubleRegister src) {
+ if (!IsMipsSoftFloatABI) {
+ Move(f0, src);
+ } else {
+ if (kArchEndian == kLittle) {
+ Move(v0, v1, src);
+ } else {
+ Move(v1, v0, src);
+ }
+ }
+}
+
+void TurboAssembler::MovToFloatParameters(DoubleRegister src1,
+ DoubleRegister src2) {
+ if (!IsMipsSoftFloatABI) {
+ if (src2 == f12) {
+ DCHECK(src1 != f14);
+ Move(f14, src2);
+ Move(f12, src1);
+ } else {
+ Move(f12, src1);
+ Move(f14, src2);
+ }
+ } else {
+ if (kArchEndian == kLittle) {
+ Move(a0, a1, src1);
+ Move(a2, a3, src2);
+ } else {
+ Move(a1, a0, src1);
+ Move(a3, a2, src2);
+ }
+ }
+}
+
+// -----------------------------------------------------------------------------
+// JavaScript invokes.
+
+void TurboAssembler::PrepareForTailCall(Register callee_args_count,
+ Register caller_args_count,
+ Register scratch0, Register scratch1) {
+ // Calculate the end of destination area where we will put the arguments
+ // after we drop current frame. We add kPointerSize to count the receiver
+ // argument which is not included into formal parameters count.
+ Register dst_reg = scratch0;
+ Lsa(dst_reg, fp, caller_args_count, kPointerSizeLog2);
+ Addu(dst_reg, dst_reg,
+ Operand(StandardFrameConstants::kCallerSPOffset + kPointerSize));
+
+ Register src_reg = caller_args_count;
+ // Calculate the end of source area. +kPointerSize is for the receiver.
+ Lsa(src_reg, sp, callee_args_count, kPointerSizeLog2);
+ Addu(src_reg, src_reg, Operand(kPointerSize));
+
+ if (FLAG_debug_code) {
+ Check(lo, AbortReason::kStackAccessBelowStackPointer, src_reg,
+ Operand(dst_reg));
+ }
+
+ // Restore caller's frame pointer and return address now as they will be
+ // overwritten by the copying loop.
+ lw(ra, MemOperand(fp, StandardFrameConstants::kCallerPCOffset));
+ lw(fp, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
+
+ // Now copy callee arguments to the caller frame going backwards to avoid
+ // callee arguments corruption (source and destination areas could overlap).
+
+ // Both src_reg and dst_reg are pointing to the word after the one to copy,
+ // so they must be pre-decremented in the loop.
+ Register tmp_reg = scratch1;
+ Label loop, entry;
+ Branch(&entry);
+ bind(&loop);
+ Subu(src_reg, src_reg, Operand(kPointerSize));
+ Subu(dst_reg, dst_reg, Operand(kPointerSize));
+ lw(tmp_reg, MemOperand(src_reg));
+ sw(tmp_reg, MemOperand(dst_reg));
+ bind(&entry);
+ Branch(&loop, ne, sp, Operand(src_reg));
+
+ // Leave current frame.
+ mov(sp, dst_reg);
+}
+
+void MacroAssembler::LoadStackLimit(Register destination, StackLimitKind kind) {
+ DCHECK(root_array_available());
+ Isolate* isolate = this->isolate();
+ ExternalReference limit =
+ kind == StackLimitKind::kRealStackLimit
+ ? ExternalReference::address_of_real_jslimit(isolate)
+ : ExternalReference::address_of_jslimit(isolate);
+ DCHECK(TurboAssembler::IsAddressableThroughRootRegister(isolate, limit));
+
+ intptr_t offset =
+ TurboAssembler::RootRegisterOffsetForExternalReference(isolate, limit);
+ CHECK(is_int32(offset));
+ Lw(destination, MemOperand(kRootRegister, static_cast<int32_t>(offset)));
+}
+
+void MacroAssembler::StackOverflowCheck(Register num_args, Register scratch1,
+ Register scratch2,
+ Label* stack_overflow) {
+ // Check the stack for overflow. We are not trying to catch
+ // interruptions (e.g. debug break and preemption) here, so the "real stack
+ // limit" is checked.
+
+ LoadStackLimit(scratch1, StackLimitKind::kRealStackLimit);
+ // Make scratch1 the space we have left. The stack might already be overflowed
+ // here which will cause scratch1 to become negative.
+ subu(scratch1, sp, scratch1);
+ // Check if the arguments will overflow the stack.
+ sll(scratch2, num_args, kPointerSizeLog2);
+ // Signed comparison.
+ Branch(stack_overflow, le, scratch1, Operand(scratch2));
+}
+
+void MacroAssembler::InvokePrologue(Register expected_parameter_count,
+ Register actual_parameter_count,
+ Label* done, InvokeFlag flag) {
+ Label regular_invoke;
+
+ // a0: actual arguments count
+ // a1: function (passed through to callee)
+ // a2: expected arguments count
+
+ DCHECK_EQ(actual_parameter_count, a0);
+ DCHECK_EQ(expected_parameter_count, a2);
+
+#ifdef V8_NO_ARGUMENTS_ADAPTOR
+ // If the expected parameter count is equal to the adaptor sentinel, no need
+ // to push undefined value as arguments.
+ Branch(®ular_invoke, eq, expected_parameter_count,
+ Operand(kDontAdaptArgumentsSentinel));
+
+ // If overapplication or if the actual argument count is equal to the
+ // formal parameter count, no need to push extra undefined values.
+ Subu(expected_parameter_count, expected_parameter_count,
+ actual_parameter_count);
+ Branch(®ular_invoke, le, expected_parameter_count, Operand(zero_reg));
+
+ Label stack_overflow;
+ StackOverflowCheck(expected_parameter_count, t0, t1, &stack_overflow);
+ // Underapplication. Move the arguments already in the stack, including the
+ // receiver and the return address.
+ {
+ Label copy;
+ Register src = t3, dest = t4;
+ mov(src, sp);
+ sll(t0, expected_parameter_count, kSystemPointerSizeLog2);
+ Subu(sp, sp, Operand(t0));
+ // Update stack pointer.
+ mov(dest, sp);
+ mov(t0, a0);
+ bind(©);
+ Lw(t1, MemOperand(src, 0));
+ Sw(t1, MemOperand(dest, 0));
+ Subu(t0, t0, Operand(1));
+ Addu(src, src, Operand(kSystemPointerSize));
+ Addu(dest, dest, Operand(kSystemPointerSize));
+ Branch(©, ge, t0, Operand(zero_reg));
+ }
+
+ // Fill remaining expected arguments with undefined values.
+ LoadRoot(t0, RootIndex::kUndefinedValue);
+ {
+ Label loop;
+ bind(&loop);
+ Sw(t0, MemOperand(t4, 0));
+ Subu(expected_parameter_count, expected_parameter_count, Operand(1));
+ Addu(t4, t4, Operand(kSystemPointerSize));
+ Branch(&loop, gt, expected_parameter_count, Operand(zero_reg));
+ }
+ b(®ular_invoke);
+ nop();
+
+ bind(&stack_overflow);
+ {
+ FrameScope frame(this,
+ has_frame() ? StackFrame::NONE : StackFrame::INTERNAL);
+ CallRuntime(Runtime::kThrowStackOverflow);
+ break_(0xCC);
+ }
+#else
+ // Check whether the expected and actual arguments count match. The registers
+ // are set up according to contract with ArgumentsAdaptorTrampoline:
+ Branch(®ular_invoke, eq, expected_parameter_count,
+ Operand(actual_parameter_count));
+
+ Handle<Code> adaptor = BUILTIN_CODE(isolate(), ArgumentsAdaptorTrampoline);
+ if (flag == CALL_FUNCTION) {
+ Call(adaptor);
+ Branch(done);
+ } else {
+ Jump(adaptor, RelocInfo::CODE_TARGET);
+ }
+#endif
+ bind(®ular_invoke);
+}
+
+void MacroAssembler::CheckDebugHook(Register fun, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count) {
+ Label skip_hook;
+ li(t0, ExternalReference::debug_hook_on_function_call_address(isolate()));
+ lb(t0, MemOperand(t0));
+ Branch(&skip_hook, eq, t0, Operand(zero_reg));
+
+ {
+ // Load receiver to pass it later to DebugOnFunctionCall hook.
+ LoadReceiver(t0, actual_parameter_count);
+
+ FrameScope frame(this,
+ has_frame() ? StackFrame::NONE : StackFrame::INTERNAL);
+ SmiTag(expected_parameter_count);
+ Push(expected_parameter_count);
+
+ SmiTag(actual_parameter_count);
+ Push(actual_parameter_count);
+
+ if (new_target.is_valid()) {
+ Push(new_target);
+ }
+ Push(fun);
+ Push(fun);
+ Push(t0);
+ CallRuntime(Runtime::kDebugOnFunctionCall);
+ Pop(fun);
+ if (new_target.is_valid()) {
+ Pop(new_target);
+ }
+
+ Pop(actual_parameter_count);
+ SmiUntag(actual_parameter_count);
+
+ Pop(expected_parameter_count);
+ SmiUntag(expected_parameter_count);
+ }
+ bind(&skip_hook);
+}
+
+void MacroAssembler::InvokeFunctionCode(Register function, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count,
+ InvokeFlag flag) {
+ // You can't call a function without a valid frame.
+ DCHECK_IMPLIES(flag == CALL_FUNCTION, has_frame());
+ DCHECK_EQ(function, a1);
+ DCHECK_IMPLIES(new_target.is_valid(), new_target == a3);
+
+ // On function call, call into the debugger if necessary.
+ CheckDebugHook(function, new_target, expected_parameter_count,
+ actual_parameter_count);
+
+ // Clear the new.target register if not given.
+ if (!new_target.is_valid()) {
+ LoadRoot(a3, RootIndex::kUndefinedValue);
+ }
+
+ Label done;
+ InvokePrologue(expected_parameter_count, actual_parameter_count, &done, flag);
+ // We call indirectly through the code field in the function to
+ // allow recompilation to take effect without changing any of the
+ // call sites.
+ Register code = kJavaScriptCallCodeStartRegister;
+ lw(code, FieldMemOperand(function, JSFunction::kCodeOffset));
+ if (flag == CALL_FUNCTION) {
+ Addu(code, code, Code::kHeaderSize - kHeapObjectTag);
+ Call(code);
+ } else {
+ DCHECK(flag == JUMP_FUNCTION);
+ Addu(code, code, Code::kHeaderSize - kHeapObjectTag);
+ Jump(code);
+ }
+
+ // Continue here if InvokePrologue does handle the invocation due to
+ // mismatched parameter counts.
+ bind(&done);
+}
+
+void MacroAssembler::InvokeFunctionWithNewTarget(
+ Register function, Register new_target, Register actual_parameter_count,
+ InvokeFlag flag) {
+ // You can't call a function without a valid frame.
+ DCHECK_IMPLIES(flag == CALL_FUNCTION, has_frame());
+
+ // Contract with called JS functions requires that function is passed in a1.
+ DCHECK_EQ(function, a1);
+ Register expected_reg = a2;
+ Register temp_reg = t0;
+
+ lw(temp_reg, FieldMemOperand(a1, JSFunction::kSharedFunctionInfoOffset));
+ lw(cp, FieldMemOperand(a1, JSFunction::kContextOffset));
+ lhu(expected_reg,
+ FieldMemOperand(temp_reg,
+ SharedFunctionInfo::kFormalParameterCountOffset));
+
+ InvokeFunctionCode(function, new_target, expected_reg, actual_parameter_count,
+ flag);
+}
+
+void MacroAssembler::InvokeFunction(Register function,
+ Register expected_parameter_count,
+ Register actual_parameter_count,
+ InvokeFlag flag) {
+ // You can't call a function without a valid frame.
+ DCHECK_IMPLIES(flag == CALL_FUNCTION, has_frame());
+
+ // Contract with called JS functions requires that function is passed in a1.
+ DCHECK_EQ(function, a1);
+
+ // Get the function and setup the context.
+ lw(cp, FieldMemOperand(a1, JSFunction::kContextOffset));
+
+ InvokeFunctionCode(a1, no_reg, expected_parameter_count,
+ actual_parameter_count, flag);
+}
+
+// ---------------------------------------------------------------------------
+// Support functions.
+
+void MacroAssembler::GetObjectType(Register object, Register map,
+ Register type_reg) {
+ LoadMap(map, object);
+ lhu(type_reg, FieldMemOperand(map, Map::kInstanceTypeOffset));
+}
+
+// -----------------------------------------------------------------------------
+// Runtime calls.
+
+void TurboAssembler::AddOverflow(Register dst, Register left,
+ const Operand& right, Register overflow) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Register right_reg = no_reg;
+ Register scratch = t8;
+ if (!right.is_reg()) {
+ li(at, Operand(right));
+ right_reg = at;
+ } else {
+ right_reg = right.rm();
+ }
+
+ DCHECK(left != scratch && right_reg != scratch && dst != scratch &&
+ overflow != scratch);
+ DCHECK(overflow != left && overflow != right_reg);
+
+ if (dst == left || dst == right_reg) {
+ addu(scratch, left, right_reg);
+ xor_(overflow, scratch, left);
+ xor_(at, scratch, right_reg);
+ and_(overflow, overflow, at);
+ mov(dst, scratch);
+ } else {
+ addu(dst, left, right_reg);
+ xor_(overflow, dst, left);
+ xor_(at, dst, right_reg);
+ and_(overflow, overflow, at);
+ }
+}
+
+void TurboAssembler::SubOverflow(Register dst, Register left,
+ const Operand& right, Register overflow) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Register right_reg = no_reg;
+ Register scratch = t8;
+ if (!right.is_reg()) {
+ li(at, Operand(right));
+ right_reg = at;
+ } else {
+ right_reg = right.rm();
+ }
+
+ DCHECK(left != scratch && right_reg != scratch && dst != scratch &&
+ overflow != scratch);
+ DCHECK(overflow != left && overflow != right_reg);
+
+ if (dst == left || dst == right_reg) {
+ subu(scratch, left, right_reg);
+ xor_(overflow, left, scratch);
+ xor_(at, left, right_reg);
+ and_(overflow, overflow, at);
+ mov(dst, scratch);
+ } else {
+ subu(dst, left, right_reg);
+ xor_(overflow, left, dst);
+ xor_(at, left, right_reg);
+ and_(overflow, overflow, at);
+ }
+}
+
+void TurboAssembler::MulOverflow(Register dst, Register left,
+ const Operand& right, Register overflow) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Register right_reg = no_reg;
+ Register scratch = t8;
+ Register scratch2 = t9;
+ if (!right.is_reg()) {
+ li(at, Operand(right));
+ right_reg = at;
+ } else {
+ right_reg = right.rm();
+ }
+
+ DCHECK(left != scratch && right_reg != scratch && dst != scratch &&
+ overflow != scratch);
+ DCHECK(overflow != left && overflow != right_reg);
+
+ if (dst == left || dst == right_reg) {
+ Mul(overflow, scratch2, left, right_reg);
+ sra(scratch, scratch2, 31);
+ xor_(overflow, overflow, scratch);
+ mov(dst, scratch2);
+ } else {
+ Mul(overflow, dst, left, right_reg);
+ sra(scratch, dst, 31);
+ xor_(overflow, overflow, scratch);
+ }
+}
+
+void MacroAssembler::CallRuntime(const Runtime::Function* f, int num_arguments,
+ SaveFPRegsMode save_doubles) {
+ // All parameters are on the stack. v0 has the return value after call.
+
+ // If the expected number of arguments of the runtime function is
+ // constant, we check that the actual number of arguments match the
+ // expectation.
+ CHECK(f->nargs < 0 || f->nargs == num_arguments);
+
+ // TODO(1236192): Most runtime routines don't need the number of
+ // arguments passed in because it is constant. At some point we
+ // should remove this need and make the runtime routine entry code
+ // smarter.
+ PrepareCEntryArgs(num_arguments);
+ PrepareCEntryFunction(ExternalReference::Create(f));
+ Handle<Code> code =
+ CodeFactory::CEntry(isolate(), f->result_size, save_doubles);
+ Call(code, RelocInfo::CODE_TARGET);
+}
+
+void MacroAssembler::TailCallRuntime(Runtime::FunctionId fid) {
+ const Runtime::Function* function = Runtime::FunctionForId(fid);
+ DCHECK_EQ(1, function->result_size);
+ if (function->nargs >= 0) {
+ PrepareCEntryArgs(function->nargs);
+ }
+ JumpToExternalReference(ExternalReference::Create(fid));
+}
+
+void MacroAssembler::JumpToExternalReference(const ExternalReference& builtin,
+ BranchDelaySlot bd,
+ bool builtin_exit_frame) {
+ PrepareCEntryFunction(builtin);
+ Handle<Code> code = CodeFactory::CEntry(isolate(), 1, kDontSaveFPRegs,
+ kArgvOnStack, builtin_exit_frame);
+ Jump(code, RelocInfo::CODE_TARGET, al, zero_reg, Operand(zero_reg), bd);
+}
+
+void MacroAssembler::JumpToInstructionStream(Address entry) {
+ li(kOffHeapTrampolineRegister, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+ Jump(kOffHeapTrampolineRegister);
+}
+
+void MacroAssembler::LoadWeakValue(Register out, Register in,
+ Label* target_if_cleared) {
+ Branch(target_if_cleared, eq, in, Operand(kClearedWeakHeapObjectLower32));
+
+ And(out, in, Operand(~kWeakHeapObjectMask));
+}
+
+void MacroAssembler::IncrementCounter(StatsCounter* counter, int value,
+ Register scratch1, Register scratch2) {
+ DCHECK_GT(value, 0);
+ if (FLAG_native_code_counters && counter->Enabled()) {
+ li(scratch2, ExternalReference::Create(counter));
+ lw(scratch1, MemOperand(scratch2));
+ Addu(scratch1, scratch1, Operand(value));
+ sw(scratch1, MemOperand(scratch2));
+ }
+}
+
+void MacroAssembler::DecrementCounter(StatsCounter* counter, int value,
+ Register scratch1, Register scratch2) {
+ DCHECK_GT(value, 0);
+ if (FLAG_native_code_counters && counter->Enabled()) {
+ li(scratch2, ExternalReference::Create(counter));
+ lw(scratch1, MemOperand(scratch2));
+ Subu(scratch1, scratch1, Operand(value));
+ sw(scratch1, MemOperand(scratch2));
+ }
+}
+
+// -----------------------------------------------------------------------------
+// Debugging.
+
+void TurboAssembler::Trap() { stop(); }
+void TurboAssembler::DebugBreak() { stop(); }
+
+void TurboAssembler::Assert(Condition cc, AbortReason reason, Register rs,
+ Operand rt) {
+ if (emit_debug_code()) Check(cc, reason, rs, rt);
+}
+
+void TurboAssembler::Check(Condition cc, AbortReason reason, Register rs,
+ Operand rt) {
+ Label L;
+ Branch(&L, cc, rs, rt);
+ Abort(reason);
+ // Will not return here.
+ bind(&L);
+}
+
+void TurboAssembler::Abort(AbortReason reason) {
+ Label abort_start;
+ bind(&abort_start);
+#ifdef DEBUG
+ const char* msg = GetAbortReason(reason);
+ RecordComment("Abort message: ");
+ RecordComment(msg);
+#endif
+
+ // Avoid emitting call to builtin if requested.
+ if (trap_on_abort()) {
+ stop();
+ return;
+ }
+
+ if (should_abort_hard()) {
+ // We don't care if we constructed a frame. Just pretend we did.
+ FrameScope assume_frame(this, StackFrame::NONE);
+ PrepareCallCFunction(0, a0);
+ li(a0, Operand(static_cast<int>(reason)));
+ CallCFunction(ExternalReference::abort_with_reason(), 1);
+ return;
+ }
+
+ Move(a0, Smi::FromInt(static_cast<int>(reason)));
+
+ // Disable stub call restrictions to always allow calls to abort.
+ if (!has_frame_) {
+ // We don't actually want to generate a pile of code for this, so just
+ // claim there is a stack frame, without generating one.
+ FrameScope scope(this, StackFrame::NONE);
+ Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
+ } else {
+ Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
+ }
+ // Will not return here.
+ if (is_trampoline_pool_blocked()) {
+ // If the calling code cares about the exact number of
+ // instructions generated, we insert padding here to keep the size
+ // of the Abort macro constant.
+ // Currently in debug mode with debug_code enabled the number of
+ // generated instructions is 10, so we use this as a maximum value.
+ static const int kExpectedAbortInstructions = 10;
+ int abort_instructions = InstructionsGeneratedSince(&abort_start);
+ DCHECK_LE(abort_instructions, kExpectedAbortInstructions);
+ while (abort_instructions++ < kExpectedAbortInstructions) {
+ nop();
+ }
+ }
+}
+
+void MacroAssembler::LoadMap(Register destination, Register object) {
+ Lw(destination, FieldMemOperand(object, HeapObject::kMapOffset));
+}
+
+void MacroAssembler::LoadNativeContextSlot(int index, Register dst) {
+ LoadMap(dst, cp);
+ Lw(dst,
+ FieldMemOperand(dst, Map::kConstructorOrBackPointerOrNativeContextOffset));
+ Lw(dst, MemOperand(dst, Context::SlotOffset(index)));
+}
+
+void TurboAssembler::StubPrologue(StackFrame::Type type) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ li(scratch, Operand(StackFrame::TypeToMarker(type)));
+ PushCommonFrame(scratch);
+}
+
+void TurboAssembler::Prologue() { PushStandardFrame(a1); }
+
+void TurboAssembler::EnterFrame(StackFrame::Type type) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ int stack_offset = -3 * kPointerSize;
+ const int fp_offset = 1 * kPointerSize;
+ addiu(sp, sp, stack_offset);
+ stack_offset = -stack_offset - kPointerSize;
+ sw(ra, MemOperand(sp, stack_offset));
+ stack_offset -= kPointerSize;
+ sw(fp, MemOperand(sp, stack_offset));
+ stack_offset -= kPointerSize;
+ li(t9, Operand(StackFrame::TypeToMarker(type)));
+ sw(t9, MemOperand(sp, stack_offset));
+ // Adjust FP to point to saved FP.
+ DCHECK_EQ(stack_offset, 0);
+ Addu(fp, sp, Operand(fp_offset));
+}
+
+void TurboAssembler::LeaveFrame(StackFrame::Type type) {
+ addiu(sp, fp, 2 * kPointerSize);
+ lw(ra, MemOperand(fp, 1 * kPointerSize));
+ lw(fp, MemOperand(fp, 0 * kPointerSize));
+}
+
+void MacroAssembler::EnterExitFrame(bool save_doubles, int stack_space,
+ StackFrame::Type frame_type) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ DCHECK(frame_type == StackFrame::EXIT ||
+ frame_type == StackFrame::BUILTIN_EXIT);
+
+ // Set up the frame structure on the stack.
+ STATIC_ASSERT(2 * kPointerSize == ExitFrameConstants::kCallerSPDisplacement);
+ STATIC_ASSERT(1 * kPointerSize == ExitFrameConstants::kCallerPCOffset);
+ STATIC_ASSERT(0 * kPointerSize == ExitFrameConstants::kCallerFPOffset);
+
+ // This is how the stack will look:
+ // fp + 2 (==kCallerSPDisplacement) - old stack's end
+ // [fp + 1 (==kCallerPCOffset)] - saved old ra
+ // [fp + 0 (==kCallerFPOffset)] - saved old fp
+ // [fp - 1 StackFrame::EXIT Smi
+ // [fp - 2 (==kSPOffset)] - sp of the called function
+ // fp - (2 + stack_space + alignment) == sp == [fp - kSPOffset] - top of the
+ // new stack (will contain saved ra)
+
+ // Save registers and reserve room for saved entry sp.
+ addiu(sp, sp, -2 * kPointerSize - ExitFrameConstants::kFixedFrameSizeFromFp);
+ sw(ra, MemOperand(sp, 3 * kPointerSize));
+ sw(fp, MemOperand(sp, 2 * kPointerSize));
+ {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ li(scratch, Operand(StackFrame::TypeToMarker(frame_type)));
+ sw(scratch, MemOperand(sp, 1 * kPointerSize));
+ }
+ // Set up new frame pointer.
+ addiu(fp, sp, ExitFrameConstants::kFixedFrameSizeFromFp);
+
+ if (emit_debug_code()) {
+ sw(zero_reg, MemOperand(fp, ExitFrameConstants::kSPOffset));
+ }
+
+ // Save the frame pointer and the context in top.
+ li(t8,
+ ExternalReference::Create(IsolateAddressId::kCEntryFPAddress, isolate()));
+ sw(fp, MemOperand(t8));
+ li(t8,
+ ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
+ sw(cp, MemOperand(t8));
+
+ const int frame_alignment = MacroAssembler::ActivationFrameAlignment();
+ if (save_doubles) {
+ // The stack must be align to 0 modulo 8 for stores with sdc1.
+ DCHECK_EQ(kDoubleSize, frame_alignment);
+ if (frame_alignment > 0) {
+ DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
+ And(sp, sp, Operand(-frame_alignment)); // Align stack.
+ }
+ int space = FPURegister::kNumRegisters * kDoubleSize;
+ Subu(sp, sp, Operand(space));
+ // Remember: we only need to save every 2nd double FPU value.
+ for (int i = 0; i < FPURegister::kNumRegisters; i += 2) {
+ FPURegister reg = FPURegister::from_code(i);
+ Sdc1(reg, MemOperand(sp, i * kDoubleSize));
+ }
+ }
+
+ // Reserve place for the return address, stack space and an optional slot
+ // (used by DirectCEntry to hold the return value if a struct is
+ // returned) and align the frame preparing for calling the runtime function.
+ DCHECK_GE(stack_space, 0);
+ Subu(sp, sp, Operand((stack_space + 2) * kPointerSize));
+ if (frame_alignment > 0) {
+ DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
+ And(sp, sp, Operand(-frame_alignment)); // Align stack.
+ }
+
+ // Set the exit frame sp value to point just before the return address
+ // location.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ addiu(scratch, sp, kPointerSize);
+ sw(scratch, MemOperand(fp, ExitFrameConstants::kSPOffset));
+}
+
+void MacroAssembler::LeaveExitFrame(bool save_doubles, Register argument_count,
+ bool do_return,
+ bool argument_count_is_length) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ // Optionally restore all double registers.
+ if (save_doubles) {
+ // Remember: we only need to restore every 2nd double FPU value.
+ lw(t8, MemOperand(fp, ExitFrameConstants::kSPOffset));
+ for (int i = 0; i < FPURegister::kNumRegisters; i += 2) {
+ FPURegister reg = FPURegister::from_code(i);
+ Ldc1(reg, MemOperand(t8, i * kDoubleSize + kPointerSize));
+ }
+ }
+
+ // Clear top frame.
+ li(t8,
+ ExternalReference::Create(IsolateAddressId::kCEntryFPAddress, isolate()));
+ sw(zero_reg, MemOperand(t8));
+
+ // Restore current context from top and clear it in debug mode.
+ li(t8,
+ ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
+ lw(cp, MemOperand(t8));
+
+#ifdef DEBUG
+ li(t8,
+ ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
+ sw(a3, MemOperand(t8));
+#endif
+
+ // Pop the arguments, restore registers, and return.
+ mov(sp, fp); // Respect ABI stack constraint.
+ lw(fp, MemOperand(sp, ExitFrameConstants::kCallerFPOffset));
+ lw(ra, MemOperand(sp, ExitFrameConstants::kCallerPCOffset));
+
+ if (argument_count.is_valid()) {
+ if (argument_count_is_length) {
+ addu(sp, sp, argument_count);
+ } else {
+ Lsa(sp, sp, argument_count, kPointerSizeLog2, t8);
+ }
+ }
+
+ if (do_return) {
+ Ret(USE_DELAY_SLOT);
+ // If returning, the instruction in the delay slot will be the addiu below.
+ }
+ addiu(sp, sp, 8);
+}
+
+int TurboAssembler::ActivationFrameAlignment() {
+#if V8_HOST_ARCH_MIPS
+ // Running on the real platform. Use the alignment as mandated by the local
+ // environment.
+ // Note: This will break if we ever start generating snapshots on one Mips
+ // platform for another Mips platform with a different alignment.
+ return base::OS::ActivationFrameAlignment();
+#else // V8_HOST_ARCH_MIPS
+ // If we are using the simulator then we should always align to the expected
+ // alignment. As the simulator is used to generate snapshots we do not know
+ // if the target platform will need alignment, so this is controlled from a
+ // flag.
+ return FLAG_sim_stack_alignment;
+#endif // V8_HOST_ARCH_MIPS
+}
+
+void MacroAssembler::AssertStackIsAligned() {
+ if (emit_debug_code()) {
+ const int frame_alignment = ActivationFrameAlignment();
+ const int frame_alignment_mask = frame_alignment - 1;
+
+ if (frame_alignment > kPointerSize) {
+ Label alignment_as_expected;
+ DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ andi(scratch, sp, frame_alignment_mask);
+ Branch(&alignment_as_expected, eq, scratch, Operand(zero_reg));
+ // Don't use Check here, as it will call Runtime_Abort re-entering here.
+ stop();
+ bind(&alignment_as_expected);
+ }
+ }
+}
+
+void TurboAssembler::JumpIfSmi(Register value, Label* smi_label,
+ Register scratch, BranchDelaySlot bd) {
+ DCHECK_EQ(0, kSmiTag);
+ andi(scratch, value, kSmiTagMask);
+ Branch(bd, smi_label, eq, scratch, Operand(zero_reg));
+}
+
+void MacroAssembler::JumpIfNotSmi(Register value, Label* not_smi_label,
+ Register scratch, BranchDelaySlot bd) {
+ DCHECK_EQ(0, kSmiTag);
+ andi(scratch, value, kSmiTagMask);
+ Branch(bd, not_smi_label, ne, scratch, Operand(zero_reg));
+}
+
+void MacroAssembler::AssertNotSmi(Register object) {
+ if (emit_debug_code()) {
+ STATIC_ASSERT(kSmiTag == 0);
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ andi(scratch, object, kSmiTagMask);
+ Check(ne, AbortReason::kOperandIsASmi, scratch, Operand(zero_reg));
+ }
+}
+
+void MacroAssembler::AssertSmi(Register object) {
+ if (emit_debug_code()) {
+ STATIC_ASSERT(kSmiTag == 0);
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ andi(scratch, object, kSmiTagMask);
+ Check(eq, AbortReason::kOperandIsASmi, scratch, Operand(zero_reg));
+ }
+}
+
+void MacroAssembler::AssertConstructor(Register object) {
+ if (emit_debug_code()) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ STATIC_ASSERT(kSmiTag == 0);
+ SmiTst(object, t8);
+ Check(ne, AbortReason::kOperandIsASmiAndNotAConstructor, t8,
+ Operand(zero_reg));
+
+ LoadMap(t8, object);
+ lbu(t8, FieldMemOperand(t8, Map::kBitFieldOffset));
+ And(t8, t8, Operand(Map::Bits1::IsConstructorBit::kMask));
+ Check(ne, AbortReason::kOperandIsNotAConstructor, t8, Operand(zero_reg));
+ }
+}
+
+void MacroAssembler::AssertFunction(Register object) {
+ if (emit_debug_code()) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ STATIC_ASSERT(kSmiTag == 0);
+ SmiTst(object, t8);
+ Check(ne, AbortReason::kOperandIsASmiAndNotAFunction, t8,
+ Operand(zero_reg));
+ GetObjectType(object, t8, t8);
+ Check(eq, AbortReason::kOperandIsNotAFunction, t8,
+ Operand(JS_FUNCTION_TYPE));
+ }
+}
+
+void MacroAssembler::AssertBoundFunction(Register object) {
+ if (emit_debug_code()) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ STATIC_ASSERT(kSmiTag == 0);
+ SmiTst(object, t8);
+ Check(ne, AbortReason::kOperandIsASmiAndNotABoundFunction, t8,
+ Operand(zero_reg));
+ GetObjectType(object, t8, t8);
+ Check(eq, AbortReason::kOperandIsNotABoundFunction, t8,
+ Operand(JS_BOUND_FUNCTION_TYPE));
+ }
+}
+
+void MacroAssembler::AssertGeneratorObject(Register object) {
+ if (!emit_debug_code()) return;
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ STATIC_ASSERT(kSmiTag == 0);
+ SmiTst(object, t8);
+ Check(ne, AbortReason::kOperandIsASmiAndNotAGeneratorObject, t8,
+ Operand(zero_reg));
+
+ GetObjectType(object, t8, t8);
+
+ Label done;
+
+ // Check if JSGeneratorObject
+ Branch(&done, eq, t8, Operand(JS_GENERATOR_OBJECT_TYPE));
+
+ // Check if JSAsyncFunctionObject (See MacroAssembler::CompareInstanceType)
+ Branch(&done, eq, t8, Operand(JS_ASYNC_FUNCTION_OBJECT_TYPE));
+
+ // Check if JSAsyncGeneratorObject
+ Branch(&done, eq, t8, Operand(JS_ASYNC_GENERATOR_OBJECT_TYPE));
+
+ Abort(AbortReason::kOperandIsNotAGeneratorObject);
+
+ bind(&done);
+}
+
+void MacroAssembler::AssertUndefinedOrAllocationSite(Register object,
+ Register scratch) {
+ if (emit_debug_code()) {
+ Label done_checking;
+ AssertNotSmi(object);
+ LoadRoot(scratch, RootIndex::kUndefinedValue);
+ Branch(&done_checking, eq, object, Operand(scratch));
+ GetObjectType(object, scratch, scratch);
+ Assert(eq, AbortReason::kExpectedUndefinedOrCell, scratch,
+ Operand(ALLOCATION_SITE_TYPE));
+ bind(&done_checking);
+ }
+}
+
+void TurboAssembler::Float32Max(FPURegister dst, FPURegister src1,
+ FPURegister src2, Label* out_of_line) {
+ if (src1 == src2) {
+ Move_s(dst, src1);
+ return;
+ }
+
+ // Check if one of operands is NaN.
+ CompareIsNanF32(src1, src2);
+ BranchTrueF(out_of_line);
+
+ if (IsMipsArchVariant(kMips32r6)) {
+ max_s(dst, src1, src2);
+ } else {
+ Label return_left, return_right, done;
+
+ CompareF32(OLT, src1, src2);
+ BranchTrueShortF(&return_right);
+ CompareF32(OLT, src2, src1);
+ BranchTrueShortF(&return_left);
+
+ // Operands are equal, but check for +/-0.
+ {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ mfc1(t8, src1);
+ Branch(&return_left, eq, t8, Operand(zero_reg));
+ Branch(&return_right);
+ }
+
+ bind(&return_right);
+ if (src2 != dst) {
+ Move_s(dst, src2);
+ }
+ Branch(&done);
+
+ bind(&return_left);
+ if (src1 != dst) {
+ Move_s(dst, src1);
+ }
+
+ bind(&done);
+ }
+}
+
+void TurboAssembler::Float32MaxOutOfLine(FPURegister dst, FPURegister src1,
+ FPURegister src2) {
+ add_s(dst, src1, src2);
+}
+
+void TurboAssembler::Float32Min(FPURegister dst, FPURegister src1,
+ FPURegister src2, Label* out_of_line) {
+ if (src1 == src2) {
+ Move_s(dst, src1);
+ return;
+ }
+
+ // Check if one of operands is NaN.
+ CompareIsNanF32(src1, src2);
+ BranchTrueF(out_of_line);
+
+ if (IsMipsArchVariant(kMips32r6)) {
+ min_s(dst, src1, src2);
+ } else {
+ Label return_left, return_right, done;
+
+ CompareF32(OLT, src1, src2);
+ BranchTrueShortF(&return_left);
+ CompareF32(OLT, src2, src1);
+ BranchTrueShortF(&return_right);
+
+ // Left equals right => check for -0.
+ {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ mfc1(t8, src1);
+ Branch(&return_right, eq, t8, Operand(zero_reg));
+ Branch(&return_left);
+ }
+
+ bind(&return_right);
+ if (src2 != dst) {
+ Move_s(dst, src2);
+ }
+ Branch(&done);
+
+ bind(&return_left);
+ if (src1 != dst) {
+ Move_s(dst, src1);
+ }
+
+ bind(&done);
+ }
+}
+
+void TurboAssembler::Float32MinOutOfLine(FPURegister dst, FPURegister src1,
+ FPURegister src2) {
+ add_s(dst, src1, src2);
+}
+
+void TurboAssembler::Float64Max(DoubleRegister dst, DoubleRegister src1,
+ DoubleRegister src2, Label* out_of_line) {
+ if (src1 == src2) {
+ Move_d(dst, src1);
+ return;
+ }
+
+ // Check if one of operands is NaN.
+ CompareIsNanF64(src1, src2);
+ BranchTrueF(out_of_line);
+
+ if (IsMipsArchVariant(kMips32r6)) {
+ max_d(dst, src1, src2);
+ } else {
+ Label return_left, return_right, done;
+
+ CompareF64(OLT, src1, src2);
+ BranchTrueShortF(&return_right);
+ CompareF64(OLT, src2, src1);
+ BranchTrueShortF(&return_left);
+
+ // Left equals right => check for -0.
+ {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Mfhc1(t8, src1);
+ Branch(&return_left, eq, t8, Operand(zero_reg));
+ Branch(&return_right);
+ }
+
+ bind(&return_right);
+ if (src2 != dst) {
+ Move_d(dst, src2);
+ }
+ Branch(&done);
+
+ bind(&return_left);
+ if (src1 != dst) {
+ Move_d(dst, src1);
+ }
+
+ bind(&done);
+ }
+}
+
+void TurboAssembler::Float64MaxOutOfLine(DoubleRegister dst,
+ DoubleRegister src1,
+ DoubleRegister src2) {
+ add_d(dst, src1, src2);
+}
+
+void TurboAssembler::Float64Min(DoubleRegister dst, DoubleRegister src1,
+ DoubleRegister src2, Label* out_of_line) {
+ if (src1 == src2) {
+ Move_d(dst, src1);
+ return;
+ }
+
+ // Check if one of operands is NaN.
+ CompareIsNanF64(src1, src2);
+ BranchTrueF(out_of_line);
+
+ if (IsMipsArchVariant(kMips32r6)) {
+ min_d(dst, src1, src2);
+ } else {
+ Label return_left, return_right, done;
+
+ CompareF64(OLT, src1, src2);
+ BranchTrueShortF(&return_left);
+ CompareF64(OLT, src2, src1);
+ BranchTrueShortF(&return_right);
+
+ // Left equals right => check for -0.
+ {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Mfhc1(t8, src1);
+ Branch(&return_right, eq, t8, Operand(zero_reg));
+ Branch(&return_left);
+ }
+
+ bind(&return_right);
+ if (src2 != dst) {
+ Move_d(dst, src2);
+ }
+ Branch(&done);
+
+ bind(&return_left);
+ if (src1 != dst) {
+ Move_d(dst, src1);
+ }
+
+ bind(&done);
+ }
+}
+
+void TurboAssembler::Float64MinOutOfLine(DoubleRegister dst,
+ DoubleRegister src1,
+ DoubleRegister src2) {
+ add_d(dst, src1, src2);
+}
+
+static const int kRegisterPassedArguments = 4;
+
+int TurboAssembler::CalculateStackPassedWords(int num_reg_arguments,
+ int num_double_arguments) {
+ int stack_passed_words = 0;
+ num_reg_arguments += 2 * num_double_arguments;
+
+ // Up to four simple arguments are passed in registers a0..a3.
+ if (num_reg_arguments > kRegisterPassedArguments) {
+ stack_passed_words += num_reg_arguments - kRegisterPassedArguments;
+ }
+ stack_passed_words += kCArgSlotCount;
+ return stack_passed_words;
+}
+
+void TurboAssembler::PrepareCallCFunction(int num_reg_arguments,
+ int num_double_arguments,
+ Register scratch) {
+ int frame_alignment = ActivationFrameAlignment();
+
+ // Up to four simple arguments are passed in registers a0..a3.
+ // Those four arguments must have reserved argument slots on the stack for
+ // mips, even though those argument slots are not normally used.
+ // Remaining arguments are pushed on the stack, above (higher address than)
+ // the argument slots.
+ int stack_passed_arguments =
+ CalculateStackPassedWords(num_reg_arguments, num_double_arguments);
+ if (frame_alignment > kPointerSize) {
+ // Make stack end at alignment and make room for num_arguments - 4 words
+ // and the original value of sp.
+ mov(scratch, sp);
+ Subu(sp, sp, Operand((stack_passed_arguments + 1) * kPointerSize));
+ DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
+ And(sp, sp, Operand(-frame_alignment));
+ sw(scratch, MemOperand(sp, stack_passed_arguments * kPointerSize));
+ } else {
+ Subu(sp, sp, Operand(stack_passed_arguments * kPointerSize));
+ }
+}
+
+void TurboAssembler::PrepareCallCFunction(int num_reg_arguments,
+ Register scratch) {
+ PrepareCallCFunction(num_reg_arguments, 0, scratch);
+}
+
+void TurboAssembler::CallCFunction(ExternalReference function,
+ int num_reg_arguments,
+ int num_double_arguments) {
+ // Linux/MIPS convention demands that register t9 contains
+ // the address of the function being call in case of
+ // Position independent code
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ li(t9, function);
+ CallCFunctionHelper(t9, 0, num_reg_arguments, num_double_arguments);
+}
+
+void TurboAssembler::CallCFunction(Register function, int num_reg_arguments,
+ int num_double_arguments) {
+ CallCFunctionHelper(function, 0, num_reg_arguments, num_double_arguments);
+}
+
+void TurboAssembler::CallCFunction(ExternalReference function,
+ int num_arguments) {
+ CallCFunction(function, num_arguments, 0);
+}
+
+void TurboAssembler::CallCFunction(Register function, int num_arguments) {
+ CallCFunction(function, num_arguments, 0);
+}
+
+void TurboAssembler::CallCFunctionHelper(Register function_base,
+ int16_t function_offset,
+ int num_reg_arguments,
+ int num_double_arguments) {
+ DCHECK_LE(num_reg_arguments + num_double_arguments, kMaxCParameters);
+ DCHECK(has_frame());
+ // Make sure that the stack is aligned before calling a C function unless
+ // running in the simulator. The simulator has its own alignment check which
+ // provides more information.
+ // The argument stots are presumed to have been set up by
+ // PrepareCallCFunction. The C function must be called via t9, for mips ABI.
+
+#if V8_HOST_ARCH_MIPS
+ if (emit_debug_code()) {
+ int frame_alignment = base::OS::ActivationFrameAlignment();
+ int frame_alignment_mask = frame_alignment - 1;
+ if (frame_alignment > kPointerSize) {
+ DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
+ Label alignment_as_expected;
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ And(scratch, sp, Operand(frame_alignment_mask));
+ Branch(&alignment_as_expected, eq, scratch, Operand(zero_reg));
+ // Don't use Check here, as it will call Runtime_Abort possibly
+ // re-entering here.
+ stop();
+ bind(&alignment_as_expected);
+ }
+ }
+#endif // V8_HOST_ARCH_MIPS
+
+ // Just call directly. The function called cannot cause a GC, or
+ // allow preemption, so the return address in the link register
+ // stays correct.
+
+ {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ if (function_base != t9) {
+ mov(t9, function_base);
+ function_base = t9;
+ }
+
+ if (function_offset != 0) {
+ addiu(t9, t9, function_offset);
+ function_offset = 0;
+ }
+
+ // Save the frame pointer and PC so that the stack layout remains iterable,
+ // even without an ExitFrame which normally exists between JS and C frames.
+ // 't' registers are caller-saved so this is safe as a scratch register.
+ Register pc_scratch = t4;
+ Register scratch = t5;
+ DCHECK(!AreAliased(pc_scratch, scratch, function_base));
+
+ mov(scratch, ra);
+ nal();
+ mov(pc_scratch, ra);
+ mov(ra, scratch);
+
+ // See x64 code for reasoning about how to address the isolate data fields.
+ if (root_array_available()) {
+ sw(pc_scratch, MemOperand(kRootRegister,
+ IsolateData::fast_c_call_caller_pc_offset()));
+ sw(fp, MemOperand(kRootRegister,
+ IsolateData::fast_c_call_caller_fp_offset()));
+ } else {
+ DCHECK_NOT_NULL(isolate());
+ li(scratch, ExternalReference::fast_c_call_caller_pc_address(isolate()));
+ sw(pc_scratch, MemOperand(scratch));
+ li(scratch, ExternalReference::fast_c_call_caller_fp_address(isolate()));
+ sw(fp, MemOperand(scratch));
+ }
+
+ Call(function_base, function_offset);
+
+ // We don't unset the PC; the FP is the source of truth.
+ if (root_array_available()) {
+ sw(zero_reg, MemOperand(kRootRegister,
+ IsolateData::fast_c_call_caller_fp_offset()));
+ } else {
+ DCHECK_NOT_NULL(isolate());
+ li(scratch, ExternalReference::fast_c_call_caller_fp_address(isolate()));
+ sw(zero_reg, MemOperand(scratch));
+ }
+ }
+
+ int stack_passed_arguments =
+ CalculateStackPassedWords(num_reg_arguments, num_double_arguments);
+
+ if (base::OS::ActivationFrameAlignment() > kPointerSize) {
+ lw(sp, MemOperand(sp, stack_passed_arguments * kPointerSize));
+ } else {
+ Addu(sp, sp, Operand(stack_passed_arguments * kPointerSize));
+ }
+}
+
+#undef BRANCH_ARGS_CHECK
+
+void TurboAssembler::CheckPageFlag(Register object, Register scratch, int mask,
+ Condition cc, Label* condition_met) {
+ And(scratch, object, Operand(~kPageAlignmentMask));
+ lw(scratch, MemOperand(scratch, BasicMemoryChunk::kFlagsOffset));
+ And(scratch, scratch, Operand(mask));
+ Branch(condition_met, cc, scratch, Operand(zero_reg));
+}
+
+Register GetRegisterThatIsNotOneOf(Register reg1, Register reg2, Register reg3,
+ Register reg4, Register reg5,
+ Register reg6) {
+ RegList regs = 0;
+ if (reg1.is_valid()) regs |= reg1.bit();
+ if (reg2.is_valid()) regs |= reg2.bit();
+ if (reg3.is_valid()) regs |= reg3.bit();
+ if (reg4.is_valid()) regs |= reg4.bit();
+ if (reg5.is_valid()) regs |= reg5.bit();
+ if (reg6.is_valid()) regs |= reg6.bit();
+
+ const RegisterConfiguration* config = RegisterConfiguration::Default();
+ for (int i = 0; i < config->num_allocatable_general_registers(); ++i) {
+ int code = config->GetAllocatableGeneralCode(i);
+ Register candidate = Register::from_code(code);
+ if (regs & candidate.bit()) continue;
+ return candidate;
+ }
+ UNREACHABLE();
+}
+
+void TurboAssembler::ComputeCodeStartAddress(Register dst) {
+ // This push on ra and the pop below together ensure that we restore the
+ // register ra, which is needed while computing the code start address.
+ push(ra);
+
+ // The nal instruction puts the address of the current instruction into
+ // the return address (ra) register, which we can use later on.
+ if (IsMipsArchVariant(kMips32r6)) {
+ addiupc(ra, 1);
+ } else {
+ nal();
+ nop();
+ }
+ int pc = pc_offset();
+ li(dst, pc);
+ subu(dst, ra, dst);
+
+ pop(ra); // Restore ra
+}
+
+void TurboAssembler::ResetSpeculationPoisonRegister() {
+ li(kSpeculationPoisonRegister, -1);
+}
+
+void TurboAssembler::CallForDeoptimization(Builtins::Name target, int,
+ Label* exit, DeoptimizeKind kind,
+ Label*) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Lw(t9,
+ MemOperand(kRootRegister, IsolateData::builtin_entry_slot_offset(target)));
+ Call(t9);
+ DCHECK_EQ(SizeOfCodeGeneratedSince(exit),
+ (kind == DeoptimizeKind::kLazy)
+ ? Deoptimizer::kLazyDeoptExitSize
+ : Deoptimizer::kNonLazyDeoptExitSize);
+ USE(exit, kind);
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_MIPS
diff --git a/src/codegen/mips/macro-assembler-mips.h b/src/codegen/mips/macro-assembler-mips.h
new file mode 100644
index 0000000..d91a4a7
--- /dev/null
+++ b/src/codegen/mips/macro-assembler-mips.h
@@ -0,0 +1,1202 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef INCLUDED_FROM_MACRO_ASSEMBLER_H
+#error This header must be included via macro-assembler.h
+#endif
+
+#ifndef V8_CODEGEN_MIPS_MACRO_ASSEMBLER_MIPS_H_
+#define V8_CODEGEN_MIPS_MACRO_ASSEMBLER_MIPS_H_
+
+#include "src/codegen/assembler.h"
+#include "src/codegen/mips/assembler-mips.h"
+#include "src/common/globals.h"
+#include "src/objects/contexts.h"
+
+namespace v8 {
+namespace internal {
+
+// Forward declarations
+enum class AbortReason : uint8_t;
+
+// Reserved Register Usage Summary.
+//
+// Registers t8, t9, and at are reserved for use by the MacroAssembler.
+//
+// The programmer should know that the MacroAssembler may clobber these three,
+// but won't touch other registers except in special cases.
+//
+// Per the MIPS ABI, register t9 must be used for indirect function call
+// via 'jalr t9' or 'jr t9' instructions. This is relied upon by gcc when
+// trying to update gp register for position-independent-code. Whenever
+// MIPS generated code calls C code, it must be via t9 register.
+
+// Flags used for LeaveExitFrame function.
+enum LeaveExitFrameMode { EMIT_RETURN = true, NO_EMIT_RETURN = false };
+
+// Flags used for the li macro-assembler function.
+enum LiFlags {
+ // If the constant value can be represented in just 16 bits, then
+ // optimize the li to use a single instruction, rather than lui/ori pair.
+ OPTIMIZE_SIZE = 0,
+ // Always use 2 instructions (lui/ori pair), even if the constant could
+ // be loaded with just one, so that this value is patchable later.
+ CONSTANT_SIZE = 1
+};
+
+enum RememberedSetAction { EMIT_REMEMBERED_SET, OMIT_REMEMBERED_SET };
+enum SmiCheck { INLINE_SMI_CHECK, OMIT_SMI_CHECK };
+enum RAStatus { kRAHasNotBeenSaved, kRAHasBeenSaved };
+
+Register GetRegisterThatIsNotOneOf(Register reg1, Register reg2 = no_reg,
+ Register reg3 = no_reg,
+ Register reg4 = no_reg,
+ Register reg5 = no_reg,
+ Register reg6 = no_reg);
+
+// -----------------------------------------------------------------------------
+// Static helper functions.
+// Generate a MemOperand for loading a field from an object.
+inline MemOperand FieldMemOperand(Register object, int offset) {
+ return MemOperand(object, offset - kHeapObjectTag);
+}
+
+// Generate a MemOperand for storing arguments 5..N on the stack
+// when calling CallCFunction().
+inline MemOperand CFunctionArgumentOperand(int index) {
+ DCHECK_GT(index, kCArgSlotCount);
+ // Argument 5 takes the slot just past the four Arg-slots.
+ int offset = (index - 5) * kPointerSize + kCArgsSlotsSize;
+ return MemOperand(sp, offset);
+}
+
+class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase {
+ public:
+ using TurboAssemblerBase::TurboAssemblerBase;
+
+ // Activation support.
+ void EnterFrame(StackFrame::Type type);
+ void EnterFrame(StackFrame::Type type, bool load_constant_pool_pointer_reg) {
+ // Out-of-line constant pool not implemented on mips.
+ UNREACHABLE();
+ }
+ void LeaveFrame(StackFrame::Type type);
+
+ // Generates function and stub prologue code.
+ void StubPrologue(StackFrame::Type type);
+ void Prologue();
+
+ void InitializeRootRegister() {
+ ExternalReference isolate_root = ExternalReference::isolate_root(isolate());
+ li(kRootRegister, Operand(isolate_root));
+ }
+
+ // Jump unconditionally to given label.
+ // We NEED a nop in the branch delay slot, as it used by v8, for example in
+ // CodeGenerator::ProcessDeferred().
+ // Currently the branch delay slot is filled by the MacroAssembler.
+ // Use rather b(Label) for code generation.
+ void jmp(Label* L) { Branch(L); }
+
+ // -------------------------------------------------------------------------
+ // Debugging.
+
+ void Trap() override;
+ void DebugBreak() override;
+
+ // Calls Abort(msg) if the condition cc is not satisfied.
+ // Use --debug_code to enable.
+ void Assert(Condition cc, AbortReason reason, Register rs, Operand rt);
+
+ // Like Assert(), but always enabled.
+ void Check(Condition cc, AbortReason reason, Register rs, Operand rt);
+
+ // Print a message to stdout and abort execution.
+ void Abort(AbortReason msg);
+
+ // Arguments macros.
+#define COND_TYPED_ARGS Condition cond, Register r1, const Operand &r2
+#define COND_ARGS cond, r1, r2
+
+ // Cases when relocation is not needed.
+#define DECLARE_NORELOC_PROTOTYPE(Name, target_type) \
+ void Name(target_type target, BranchDelaySlot bd = PROTECT); \
+ inline void Name(BranchDelaySlot bd, target_type target) { \
+ Name(target, bd); \
+ } \
+ void Name(target_type target, COND_TYPED_ARGS, \
+ BranchDelaySlot bd = PROTECT); \
+ inline void Name(BranchDelaySlot bd, target_type target, COND_TYPED_ARGS) { \
+ Name(target, COND_ARGS, bd); \
+ }
+
+#define DECLARE_BRANCH_PROTOTYPES(Name) \
+ DECLARE_NORELOC_PROTOTYPE(Name, Label*) \
+ DECLARE_NORELOC_PROTOTYPE(Name, int32_t)
+
+ DECLARE_BRANCH_PROTOTYPES(Branch)
+ DECLARE_BRANCH_PROTOTYPES(BranchAndLink)
+ DECLARE_BRANCH_PROTOTYPES(BranchShort)
+
+#undef DECLARE_BRANCH_PROTOTYPES
+#undef COND_TYPED_ARGS
+#undef COND_ARGS
+
+ // Floating point branches
+ void CompareF32(FPUCondition cc, FPURegister cmp1, FPURegister cmp2) {
+ CompareF(S, cc, cmp1, cmp2);
+ }
+
+ void CompareIsNanF32(FPURegister cmp1, FPURegister cmp2) {
+ CompareIsNanF(S, cmp1, cmp2);
+ }
+
+ void CompareF64(FPUCondition cc, FPURegister cmp1, FPURegister cmp2) {
+ CompareF(D, cc, cmp1, cmp2);
+ }
+
+ void CompareIsNanF64(FPURegister cmp1, FPURegister cmp2) {
+ CompareIsNanF(D, cmp1, cmp2);
+ }
+
+ void BranchTrueShortF(Label* target, BranchDelaySlot bd = PROTECT);
+ void BranchFalseShortF(Label* target, BranchDelaySlot bd = PROTECT);
+
+ void BranchTrueF(Label* target, BranchDelaySlot bd = PROTECT);
+ void BranchFalseF(Label* target, BranchDelaySlot bd = PROTECT);
+
+ // MSA Branches
+ void BranchMSA(Label* target, MSABranchDF df, MSABranchCondition cond,
+ MSARegister wt, BranchDelaySlot bd = PROTECT);
+
+ void Branch(Label* L, Condition cond, Register rs, RootIndex index,
+ BranchDelaySlot bdslot = PROTECT);
+
+ // Load int32 in the rd register.
+ void li(Register rd, Operand j, LiFlags mode = OPTIMIZE_SIZE);
+ inline void li(Register rd, int32_t j, LiFlags mode = OPTIMIZE_SIZE) {
+ li(rd, Operand(j), mode);
+ }
+ void li(Register dst, Handle<HeapObject> value, LiFlags mode = OPTIMIZE_SIZE);
+ void li(Register dst, ExternalReference value, LiFlags mode = OPTIMIZE_SIZE);
+ void li(Register dst, const StringConstantBase* string,
+ LiFlags mode = OPTIMIZE_SIZE);
+
+ void LoadFromConstantsTable(Register destination,
+ int constant_index) override;
+ void LoadRootRegisterOffset(Register destination, intptr_t offset) override;
+ void LoadRootRelative(Register destination, int32_t offset) override;
+
+// Jump, Call, and Ret pseudo instructions implementing inter-working.
+#define COND_ARGS \
+ Condition cond = al, Register rs = zero_reg, \
+ const Operand &rt = Operand(zero_reg), \
+ BranchDelaySlot bd = PROTECT
+
+ void Jump(Register target, int16_t offset = 0, COND_ARGS);
+ void Jump(Register target, Register base, int16_t offset = 0, COND_ARGS);
+ void Jump(Register target, const Operand& offset, COND_ARGS);
+ void Jump(intptr_t target, RelocInfo::Mode rmode, COND_ARGS);
+ void Jump(Address target, RelocInfo::Mode rmode, COND_ARGS);
+ // Deffer from li, this method save target to the memory, and then load
+ // it to register use lw, it can be used in wasm jump table for concurrent
+ // patching.
+ void PatchAndJump(Address target);
+ void Jump(Handle<Code> code, RelocInfo::Mode rmode, COND_ARGS);
+ void Jump(const ExternalReference& reference) override;
+ void Call(Register target, int16_t offset = 0, COND_ARGS);
+ void Call(Register target, Register base, int16_t offset = 0, COND_ARGS);
+ void Call(Address target, RelocInfo::Mode rmode, COND_ARGS);
+ void Call(Handle<Code> code, RelocInfo::Mode rmode = RelocInfo::CODE_TARGET,
+ COND_ARGS);
+ void Call(Label* target);
+ void LoadAddress(Register dst, Label* target);
+
+ // Load the builtin given by the Smi in |builtin_index| into the same
+ // register.
+ void LoadEntryFromBuiltinIndex(Register builtin_index);
+ void CallBuiltinByIndex(Register builtin_index) override;
+
+ void LoadCodeObjectEntry(Register destination,
+ Register code_object) override {
+ // TODO(mips): Implement.
+ UNIMPLEMENTED();
+ }
+ void CallCodeObject(Register code_object) override {
+ // TODO(mips): Implement.
+ UNIMPLEMENTED();
+ }
+ void JumpCodeObject(Register code_object) override {
+ // TODO(mips): Implement.
+ UNIMPLEMENTED();
+ }
+
+ // Generates an instruction sequence s.t. the return address points to the
+ // instruction following the call.
+ // The return address on the stack is used by frame iteration.
+ void StoreReturnAddressAndCall(Register target);
+
+ void CallForDeoptimization(Builtins::Name target, int deopt_id, Label* exit,
+ DeoptimizeKind kind,
+ Label* jump_deoptimization_entry_label);
+
+ void Ret(COND_ARGS);
+ inline void Ret(BranchDelaySlot bd, Condition cond = al,
+ Register rs = zero_reg,
+ const Operand& rt = Operand(zero_reg)) {
+ Ret(cond, rs, rt, bd);
+ }
+
+ // Emit code to discard a non-negative number of pointer-sized elements
+ // from the stack, clobbering only the sp register.
+ void Drop(int count, Condition cond = cc_always, Register reg = no_reg,
+ const Operand& op = Operand(no_reg));
+
+ // Trivial case of DropAndRet that utilizes the delay slot.
+ void DropAndRet(int drop);
+
+ void DropAndRet(int drop, Condition cond, Register reg, const Operand& op);
+
+ void Lw(Register rd, const MemOperand& rs);
+ void Sw(Register rd, const MemOperand& rs);
+
+ void push(Register src) {
+ Addu(sp, sp, Operand(-kPointerSize));
+ sw(src, MemOperand(sp, 0));
+ }
+
+ void Push(Register src) { push(src); }
+ void Push(Handle<HeapObject> handle);
+ void Push(Smi smi);
+
+ // Push two registers. Pushes leftmost register first (to highest address).
+ void Push(Register src1, Register src2) {
+ Subu(sp, sp, Operand(2 * kPointerSize));
+ sw(src1, MemOperand(sp, 1 * kPointerSize));
+ sw(src2, MemOperand(sp, 0 * kPointerSize));
+ }
+
+ // Push three registers. Pushes leftmost register first (to highest address).
+ void Push(Register src1, Register src2, Register src3) {
+ Subu(sp, sp, Operand(3 * kPointerSize));
+ sw(src1, MemOperand(sp, 2 * kPointerSize));
+ sw(src2, MemOperand(sp, 1 * kPointerSize));
+ sw(src3, MemOperand(sp, 0 * kPointerSize));
+ }
+
+ // Push four registers. Pushes leftmost register first (to highest address).
+ void Push(Register src1, Register src2, Register src3, Register src4) {
+ Subu(sp, sp, Operand(4 * kPointerSize));
+ sw(src1, MemOperand(sp, 3 * kPointerSize));
+ sw(src2, MemOperand(sp, 2 * kPointerSize));
+ sw(src3, MemOperand(sp, 1 * kPointerSize));
+ sw(src4, MemOperand(sp, 0 * kPointerSize));
+ }
+
+ // Push five registers. Pushes leftmost register first (to highest address).
+ void Push(Register src1, Register src2, Register src3, Register src4,
+ Register src5) {
+ Subu(sp, sp, Operand(5 * kPointerSize));
+ sw(src1, MemOperand(sp, 4 * kPointerSize));
+ sw(src2, MemOperand(sp, 3 * kPointerSize));
+ sw(src3, MemOperand(sp, 2 * kPointerSize));
+ sw(src4, MemOperand(sp, 1 * kPointerSize));
+ sw(src5, MemOperand(sp, 0 * kPointerSize));
+ }
+
+ void Push(Register src, Condition cond, Register tst1, Register tst2) {
+ // Since we don't have conditional execution we use a Branch.
+ Branch(3, cond, tst1, Operand(tst2));
+ Subu(sp, sp, Operand(kPointerSize));
+ sw(src, MemOperand(sp, 0));
+ }
+
+ enum PushArrayOrder { kNormal, kReverse };
+ void PushArray(Register array, Register size, Register scratch,
+ Register scratch2, PushArrayOrder order = kNormal);
+
+ void SaveRegisters(RegList registers);
+ void RestoreRegisters(RegList registers);
+
+ void CallRecordWriteStub(Register object, Register address,
+ RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode);
+ void CallRecordWriteStub(Register object, Register address,
+ RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode, Address wasm_target);
+ void CallEphemeronKeyBarrier(Register object, Register address,
+ SaveFPRegsMode fp_mode);
+
+ // Push multiple registers on the stack.
+ // Registers are saved in numerical order, with higher numbered registers
+ // saved in higher memory addresses.
+ void MultiPush(RegList regs);
+ void MultiPushFPU(RegList regs);
+
+ // Calculate how much stack space (in bytes) are required to store caller
+ // registers excluding those specified in the arguments.
+ int RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
+ Register exclusion1 = no_reg,
+ Register exclusion2 = no_reg,
+ Register exclusion3 = no_reg) const;
+
+ // Push caller saved registers on the stack, and return the number of bytes
+ // stack pointer is adjusted.
+ int PushCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1 = no_reg,
+ Register exclusion2 = no_reg,
+ Register exclusion3 = no_reg);
+ // Restore caller saved registers from the stack, and return the number of
+ // bytes stack pointer is adjusted.
+ int PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1 = no_reg,
+ Register exclusion2 = no_reg,
+ Register exclusion3 = no_reg);
+
+ void pop(Register dst) {
+ lw(dst, MemOperand(sp, 0));
+ Addu(sp, sp, Operand(kPointerSize));
+ }
+
+ void Pop(Register dst) { pop(dst); }
+
+ // Pop two registers. Pops rightmost register first (from lower address).
+ void Pop(Register src1, Register src2) {
+ DCHECK(src1 != src2);
+ lw(src2, MemOperand(sp, 0 * kPointerSize));
+ lw(src1, MemOperand(sp, 1 * kPointerSize));
+ Addu(sp, sp, 2 * kPointerSize);
+ }
+
+ // Pop three registers. Pops rightmost register first (from lower address).
+ void Pop(Register src1, Register src2, Register src3) {
+ lw(src3, MemOperand(sp, 0 * kPointerSize));
+ lw(src2, MemOperand(sp, 1 * kPointerSize));
+ lw(src1, MemOperand(sp, 2 * kPointerSize));
+ Addu(sp, sp, 3 * kPointerSize);
+ }
+
+ void Pop(uint32_t count = 1) { Addu(sp, sp, Operand(count * kPointerSize)); }
+
+ // Pops multiple values from the stack and load them in the
+ // registers specified in regs. Pop order is the opposite as in MultiPush.
+ void MultiPop(RegList regs);
+ void MultiPopFPU(RegList regs);
+
+ // Load Scaled Address instructions. Parameter sa (shift argument) must be
+ // between [1, 31] (inclusive). On pre-r6 architectures the scratch register
+ // may be clobbered.
+ void Lsa(Register rd, Register rs, Register rt, uint8_t sa,
+ Register scratch = at);
+
+#define DEFINE_INSTRUCTION(instr) \
+ void instr(Register rd, Register rs, const Operand& rt); \
+ void instr(Register rd, Register rs, Register rt) { \
+ instr(rd, rs, Operand(rt)); \
+ } \
+ void instr(Register rs, Register rt, int32_t j) { instr(rs, rt, Operand(j)); }
+
+#define DEFINE_INSTRUCTION2(instr) \
+ void instr(Register rs, const Operand& rt); \
+ void instr(Register rs, Register rt) { instr(rs, Operand(rt)); } \
+ void instr(Register rs, int32_t j) { instr(rs, Operand(j)); }
+
+#define DEFINE_INSTRUCTION3(instr) \
+ void instr(Register rd_hi, Register rd_lo, Register rs, const Operand& rt); \
+ void instr(Register rd_hi, Register rd_lo, Register rs, Register rt) { \
+ instr(rd_hi, rd_lo, rs, Operand(rt)); \
+ } \
+ void instr(Register rd_hi, Register rd_lo, Register rs, int32_t j) { \
+ instr(rd_hi, rd_lo, rs, Operand(j)); \
+ }
+
+ DEFINE_INSTRUCTION(Addu)
+ DEFINE_INSTRUCTION(Subu)
+ DEFINE_INSTRUCTION(Mul)
+ DEFINE_INSTRUCTION(Div)
+ DEFINE_INSTRUCTION(Divu)
+ DEFINE_INSTRUCTION(Mod)
+ DEFINE_INSTRUCTION(Modu)
+ DEFINE_INSTRUCTION(Mulh)
+ DEFINE_INSTRUCTION2(Mult)
+ DEFINE_INSTRUCTION(Mulhu)
+ DEFINE_INSTRUCTION2(Multu)
+ DEFINE_INSTRUCTION2(Div)
+ DEFINE_INSTRUCTION2(Divu)
+
+ DEFINE_INSTRUCTION3(Div)
+ DEFINE_INSTRUCTION3(Mul)
+ DEFINE_INSTRUCTION3(Mulu)
+
+ DEFINE_INSTRUCTION(And)
+ DEFINE_INSTRUCTION(Or)
+ DEFINE_INSTRUCTION(Xor)
+ DEFINE_INSTRUCTION(Nor)
+ DEFINE_INSTRUCTION2(Neg)
+
+ DEFINE_INSTRUCTION(Slt)
+ DEFINE_INSTRUCTION(Sltu)
+ DEFINE_INSTRUCTION(Sle)
+ DEFINE_INSTRUCTION(Sleu)
+ DEFINE_INSTRUCTION(Sgt)
+ DEFINE_INSTRUCTION(Sgtu)
+ DEFINE_INSTRUCTION(Sge)
+ DEFINE_INSTRUCTION(Sgeu)
+
+ // MIPS32 R2 instruction macro.
+ DEFINE_INSTRUCTION(Ror)
+
+#undef DEFINE_INSTRUCTION
+#undef DEFINE_INSTRUCTION2
+#undef DEFINE_INSTRUCTION3
+
+ void SmiUntag(Register reg) { sra(reg, reg, kSmiTagSize); }
+
+ void SmiUntag(Register dst, Register src) { sra(dst, src, kSmiTagSize); }
+
+ // Removes current frame and its arguments from the stack preserving
+ // the arguments and a return address pushed to the stack for the next call.
+ // Both |callee_args_count| and |caller_args_count| do not include
+ // receiver. |callee_args_count| is not modified. |caller_args_count|
+ // is trashed.
+ void PrepareForTailCall(Register callee_args_count,
+ Register caller_args_count, Register scratch0,
+ Register scratch1);
+
+ int CalculateStackPassedWords(int num_reg_arguments,
+ int num_double_arguments);
+
+ // Before calling a C-function from generated code, align arguments on stack
+ // and add space for the four mips argument slots.
+ // After aligning the frame, non-register arguments must be stored on the
+ // stack, after the argument-slots using helper: CFunctionArgumentOperand().
+ // The argument count assumes all arguments are word sized.
+ // Some compilers/platforms require the stack to be aligned when calling
+ // C++ code.
+ // Needs a scratch register to do some arithmetic. This register will be
+ // trashed.
+ void PrepareCallCFunction(int num_reg_arguments, int num_double_registers,
+ Register scratch);
+ void PrepareCallCFunction(int num_reg_arguments, Register scratch);
+
+ // Arguments 1-4 are placed in registers a0 through a3 respectively.
+ // Arguments 5..n are stored to stack using following:
+ // sw(t0, CFunctionArgumentOperand(5));
+
+ // Calls a C function and cleans up the space for arguments allocated
+ // by PrepareCallCFunction. The called function is not allowed to trigger a
+ // garbage collection, since that might move the code and invalidate the
+ // return address (unless this is somehow accounted for by the called
+ // function).
+ void CallCFunction(ExternalReference function, int num_arguments);
+ void CallCFunction(Register function, int num_arguments);
+ void CallCFunction(ExternalReference function, int num_reg_arguments,
+ int num_double_arguments);
+ void CallCFunction(Register function, int num_reg_arguments,
+ int num_double_arguments);
+ void MovFromFloatResult(DoubleRegister dst);
+ void MovFromFloatParameter(DoubleRegister dst);
+
+ // There are two ways of passing double arguments on MIPS, depending on
+ // whether soft or hard floating point ABI is used. These functions
+ // abstract parameter passing for the three different ways we call
+ // C functions from generated code.
+ void MovToFloatParameter(DoubleRegister src);
+ void MovToFloatParameters(DoubleRegister src1, DoubleRegister src2);
+ void MovToFloatResult(DoubleRegister src);
+
+ // See comments at the beginning of Builtins::Generate_CEntry.
+ inline void PrepareCEntryArgs(int num_args) { li(a0, num_args); }
+ inline void PrepareCEntryFunction(const ExternalReference& ref) {
+ li(a1, ref);
+ }
+
+ void CheckPageFlag(Register object, Register scratch, int mask, Condition cc,
+ Label* condition_met);
+#undef COND_ARGS
+
+ // Performs a truncating conversion of a floating point number as used by
+ // the JS bitwise operations. See ECMA-262 9.5: ToInt32.
+ // Exits with 'result' holding the answer.
+ void TruncateDoubleToI(Isolate* isolate, Zone* zone, Register result,
+ DoubleRegister double_input, StubCallMode stub_mode);
+
+ // Conditional move.
+ void Movz(Register rd, Register rs, Register rt);
+ void Movn(Register rd, Register rs, Register rt);
+ void Movt(Register rd, Register rs, uint16_t cc = 0);
+ void Movf(Register rd, Register rs, uint16_t cc = 0);
+
+ void LoadZeroIfFPUCondition(Register dest);
+ void LoadZeroIfNotFPUCondition(Register dest);
+
+ void LoadZeroIfConditionNotZero(Register dest, Register condition);
+ void LoadZeroIfConditionZero(Register dest, Register condition);
+ void LoadZeroOnCondition(Register rd, Register rs, const Operand& rt,
+ Condition cond);
+
+ void Clz(Register rd, Register rs);
+ void Ctz(Register rd, Register rs);
+ void Popcnt(Register rd, Register rs);
+
+ // Int64Lowering instructions
+ void AddPair(Register dst_low, Register dst_high, Register left_low,
+ Register left_high, Register right_low, Register right_high,
+ Register scratch1, Register scratch2);
+
+ void AddPair(Register dst_low, Register dst_high, Register left_low,
+ Register left_high, int32_t imm, Register scratch1,
+ Register scratch2);
+
+ void SubPair(Register dst_low, Register dst_high, Register left_low,
+ Register left_high, Register right_low, Register right_high,
+ Register scratch1, Register scratch2);
+
+ void AndPair(Register dst_low, Register dst_high, Register left_low,
+ Register left_high, Register right_low, Register right_high);
+
+ void OrPair(Register dst_low, Register dst_high, Register left_low,
+ Register left_high, Register right_low, Register right_high);
+
+ void XorPair(Register dst_low, Register dst_high, Register left_low,
+ Register left_high, Register right_low, Register right_high);
+
+ void MulPair(Register dst_low, Register dst_high, Register left_low,
+ Register left_high, Register right_low, Register right_high,
+ Register scratch1, Register scratch2);
+
+ void ShlPair(Register dst_low, Register dst_high, Register src_low,
+ Register src_high, Register shift, Register scratch1,
+ Register scratch2);
+
+ void ShlPair(Register dst_low, Register dst_high, Register src_low,
+ Register src_high, uint32_t shift, Register scratch);
+
+ void ShrPair(Register dst_low, Register dst_high, Register src_low,
+ Register src_high, Register shift, Register scratch1,
+ Register scratch2);
+
+ void ShrPair(Register dst_low, Register dst_high, Register src_low,
+ Register src_high, uint32_t shift, Register scratch);
+
+ void SarPair(Register dst_low, Register dst_high, Register src_low,
+ Register src_high, Register shift, Register scratch1,
+ Register scratch2);
+
+ void SarPair(Register dst_low, Register dst_high, Register src_low,
+ Register src_high, uint32_t shift, Register scratch);
+
+ // MIPS32 R2 instruction macro.
+ void Ins(Register rt, Register rs, uint16_t pos, uint16_t size);
+ void Ext(Register rt, Register rs, uint16_t pos, uint16_t size);
+ void ExtractBits(Register dest, Register source, Register pos, int size,
+ bool sign_extend = false);
+ void InsertBits(Register dest, Register source, Register pos, int size);
+
+ void Seb(Register rd, Register rt);
+ void Seh(Register rd, Register rt);
+ void Neg_s(FPURegister fd, FPURegister fs);
+ void Neg_d(FPURegister fd, FPURegister fs);
+
+ // MIPS32 R6 instruction macros.
+ void Bovc(Register rt, Register rs, Label* L);
+ void Bnvc(Register rt, Register rs, Label* L);
+
+ // Convert single to unsigned word.
+ void Trunc_uw_s(FPURegister fd, FPURegister fs, FPURegister scratch);
+ void Trunc_uw_s(Register rd, FPURegister fs, FPURegister scratch);
+
+ void Trunc_w_d(FPURegister fd, FPURegister fs);
+ void Round_w_d(FPURegister fd, FPURegister fs);
+ void Floor_w_d(FPURegister fd, FPURegister fs);
+ void Ceil_w_d(FPURegister fd, FPURegister fs);
+
+ // Round double functions
+ void Trunc_d_d(FPURegister fd, FPURegister fs);
+ void Round_d_d(FPURegister fd, FPURegister fs);
+ void Floor_d_d(FPURegister fd, FPURegister fs);
+ void Ceil_d_d(FPURegister fd, FPURegister fs);
+
+ // Round float functions
+ void Trunc_s_s(FPURegister fd, FPURegister fs);
+ void Round_s_s(FPURegister fd, FPURegister fs);
+ void Floor_s_s(FPURegister fd, FPURegister fs);
+ void Ceil_s_s(FPURegister fd, FPURegister fs);
+
+ // FP32 mode: Move the general purpose register into
+ // the high part of the double-register pair.
+ // FP64 mode: Move the general-purpose register into
+ // the higher 32 bits of the 64-bit coprocessor register,
+ // while leaving the low bits unchanged.
+ void Mthc1(Register rt, FPURegister fs);
+
+ // FP32 mode: move the high part of the double-register pair into
+ // general purpose register.
+ // FP64 mode: Move the higher 32 bits of the 64-bit coprocessor register into
+ // general-purpose register.
+ void Mfhc1(Register rt, FPURegister fs);
+
+ void Madd_s(FPURegister fd, FPURegister fr, FPURegister fs, FPURegister ft,
+ FPURegister scratch);
+ void Madd_d(FPURegister fd, FPURegister fr, FPURegister fs, FPURegister ft,
+ FPURegister scratch);
+ void Msub_s(FPURegister fd, FPURegister fr, FPURegister fs, FPURegister ft,
+ FPURegister scratch);
+ void Msub_d(FPURegister fd, FPURegister fr, FPURegister fs, FPURegister ft,
+ FPURegister scratch);
+
+ // Change endianness
+ void ByteSwapSigned(Register dest, Register src, int operand_size);
+ void ByteSwapUnsigned(Register dest, Register src, int operand_size);
+
+ void Ulh(Register rd, const MemOperand& rs);
+ void Ulhu(Register rd, const MemOperand& rs);
+ void Ush(Register rd, const MemOperand& rs, Register scratch);
+
+ void Ulw(Register rd, const MemOperand& rs);
+ void Usw(Register rd, const MemOperand& rs);
+
+ void Ulwc1(FPURegister fd, const MemOperand& rs, Register scratch);
+ void Uswc1(FPURegister fd, const MemOperand& rs, Register scratch);
+
+ void Uldc1(FPURegister fd, const MemOperand& rs, Register scratch);
+ void Usdc1(FPURegister fd, const MemOperand& rs, Register scratch);
+
+ void Ldc1(FPURegister fd, const MemOperand& src);
+ void Sdc1(FPURegister fs, const MemOperand& dst);
+
+ void Ll(Register rd, const MemOperand& rs);
+ void Sc(Register rd, const MemOperand& rs);
+
+ // Perform a floating-point min or max operation with the
+ // (IEEE-754-compatible) semantics of MIPS32's Release 6 MIN.fmt/MAX.fmt.
+ // Some cases, typically NaNs or +/-0.0, are expected to be rare and are
+ // handled in out-of-line code. The specific behaviour depends on supported
+ // instructions.
+ //
+ // These functions assume (and assert) that src1!=src2. It is permitted
+ // for the result to alias either input register.
+ void Float32Max(FPURegister dst, FPURegister src1, FPURegister src2,
+ Label* out_of_line);
+ void Float32Min(FPURegister dst, FPURegister src1, FPURegister src2,
+ Label* out_of_line);
+ void Float64Max(DoubleRegister dst, DoubleRegister src1, DoubleRegister src2,
+ Label* out_of_line);
+ void Float64Min(DoubleRegister dst, DoubleRegister src1, DoubleRegister src2,
+ Label* out_of_line);
+
+ // Generate out-of-line cases for the macros above.
+ void Float32MaxOutOfLine(FPURegister dst, FPURegister src1, FPURegister src2);
+ void Float32MinOutOfLine(FPURegister dst, FPURegister src1, FPURegister src2);
+ void Float64MaxOutOfLine(DoubleRegister dst, DoubleRegister src1,
+ DoubleRegister src2);
+ void Float64MinOutOfLine(DoubleRegister dst, DoubleRegister src1,
+ DoubleRegister src2);
+
+ bool IsDoubleZeroRegSet() { return has_double_zero_reg_set_; }
+
+ void mov(Register rd, Register rt) { or_(rd, rt, zero_reg); }
+
+ inline void Move(Register dst, Handle<HeapObject> handle) { li(dst, handle); }
+ inline void Move(Register dst, Smi smi) { li(dst, Operand(smi)); }
+
+ inline void Move(Register dst, Register src) {
+ if (dst != src) {
+ mov(dst, src);
+ }
+ }
+
+ inline void Move_d(FPURegister dst, FPURegister src) {
+ if (dst != src) {
+ mov_d(dst, src);
+ }
+ }
+
+ inline void Move_s(FPURegister dst, FPURegister src) {
+ if (dst != src) {
+ mov_s(dst, src);
+ }
+ }
+
+ inline void Move(FPURegister dst, FPURegister src) { Move_d(dst, src); }
+
+ inline void Move(Register dst_low, Register dst_high, FPURegister src) {
+ mfc1(dst_low, src);
+ Mfhc1(dst_high, src);
+ }
+
+ inline void FmoveHigh(Register dst_high, FPURegister src) {
+ Mfhc1(dst_high, src);
+ }
+
+ inline void FmoveHigh(FPURegister dst, Register src_high) {
+ Mthc1(src_high, dst);
+ }
+
+ inline void FmoveLow(Register dst_low, FPURegister src) {
+ mfc1(dst_low, src);
+ }
+
+ void FmoveLow(FPURegister dst, Register src_low);
+
+ inline void Move(FPURegister dst, Register src_low, Register src_high) {
+ mtc1(src_low, dst);
+ Mthc1(src_high, dst);
+ }
+
+ void Move(FPURegister dst, float imm) { Move(dst, bit_cast<uint32_t>(imm)); }
+ void Move(FPURegister dst, double imm) { Move(dst, bit_cast<uint64_t>(imm)); }
+ void Move(FPURegister dst, uint32_t src);
+ void Move(FPURegister dst, uint64_t src);
+
+ // -------------------------------------------------------------------------
+ // Overflow operations.
+
+ // AddOverflow sets overflow register to a negative value if
+ // overflow occured, otherwise it is zero or positive
+ void AddOverflow(Register dst, Register left, const Operand& right,
+ Register overflow);
+ // SubOverflow sets overflow register to a negative value if
+ // overflow occured, otherwise it is zero or positive
+ void SubOverflow(Register dst, Register left, const Operand& right,
+ Register overflow);
+ // MulOverflow sets overflow register to zero if no overflow occured
+ void MulOverflow(Register dst, Register left, const Operand& right,
+ Register overflow);
+
+// Number of instructions needed for calculation of switch table entry address
+#ifdef _MIPS_ARCH_MIPS32R6
+ static constexpr int kSwitchTablePrologueSize = 5;
+#else
+ static constexpr int kSwitchTablePrologueSize = 10;
+#endif
+ // GetLabelFunction must be lambda '[](size_t index) -> Label*' or a
+ // functor/function with 'Label *func(size_t index)' declaration.
+ template <typename Func>
+ void GenerateSwitchTable(Register index, size_t case_count,
+ Func GetLabelFunction);
+
+ // Load an object from the root table.
+ void LoadRoot(Register destination, RootIndex index) override;
+ void LoadRoot(Register destination, RootIndex index, Condition cond,
+ Register src1, const Operand& src2);
+
+ // If the value is a NaN, canonicalize the value else, do nothing.
+ void FPUCanonicalizeNaN(const DoubleRegister dst, const DoubleRegister src);
+
+ // ---------------------------------------------------------------------------
+ // FPU macros. These do not handle special cases like NaN or +- inf.
+
+ // Convert unsigned word to double.
+ void Cvt_d_uw(FPURegister fd, Register rs, FPURegister scratch);
+
+ // Convert double to unsigned word.
+ void Trunc_uw_d(FPURegister fd, FPURegister fs, FPURegister scratch);
+ void Trunc_uw_d(Register rd, FPURegister fs, FPURegister scratch);
+
+ // Jump the register contains a smi.
+ void JumpIfSmi(Register value, Label* smi_label, Register scratch = at,
+ BranchDelaySlot bd = PROTECT);
+
+ void JumpIfEqual(Register a, int32_t b, Label* dest) {
+ li(kScratchReg, Operand(b));
+ Branch(dest, eq, a, Operand(kScratchReg));
+ }
+
+ void JumpIfLessThan(Register a, int32_t b, Label* dest) {
+ li(kScratchReg, Operand(b));
+ Branch(dest, lt, a, Operand(kScratchReg));
+ }
+
+ // Push a standard frame, consisting of ra, fp, context and JS function.
+ void PushStandardFrame(Register function_reg);
+
+ // Get the actual activation frame alignment for target environment.
+ static int ActivationFrameAlignment();
+
+ // Compute the start of the generated instruction stream from the current PC.
+ // This is an alternative to embedding the {CodeObject} handle as a reference.
+ void ComputeCodeStartAddress(Register dst);
+
+ void ResetSpeculationPoisonRegister();
+
+ // Control-flow integrity:
+
+ // Define a function entrypoint. This doesn't emit any code for this
+ // architecture, as control-flow integrity is not supported for it.
+ void CodeEntry() {}
+ // Define an exception handler.
+ void ExceptionHandler() {}
+ // Define an exception handler and bind a label.
+ void BindExceptionHandler(Label* label) { bind(label); }
+
+ protected:
+ void BranchLong(Label* L, BranchDelaySlot bdslot);
+
+ inline Register GetRtAsRegisterHelper(const Operand& rt, Register scratch);
+
+ inline int32_t GetOffset(int32_t offset, Label* L, OffsetSize bits);
+
+ private:
+ bool has_double_zero_reg_set_ = false;
+
+ // Performs a truncating conversion of a floating point number as used by
+ // the JS bitwise operations. See ECMA-262 9.5: ToInt32. Goes to 'done' if it
+ // succeeds, otherwise falls through if result is saturated. On return
+ // 'result' either holds answer, or is clobbered on fall through.
+ void TryInlineTruncateDoubleToI(Register result, DoubleRegister input,
+ Label* done);
+
+ void CallCFunctionHelper(Register function_base, int16_t function_offset,
+ int num_reg_arguments, int num_double_arguments);
+
+ void CompareF(SecondaryField sizeField, FPUCondition cc, FPURegister cmp1,
+ FPURegister cmp2);
+
+ void CompareIsNanF(SecondaryField sizeField, FPURegister cmp1,
+ FPURegister cmp2);
+
+ void BranchShortMSA(MSABranchDF df, Label* target, MSABranchCondition cond,
+ MSARegister wt, BranchDelaySlot bd = PROTECT);
+
+ // TODO(mips) Reorder parameters so out parameters come last.
+ bool CalculateOffset(Label* L, int32_t* offset, OffsetSize bits);
+ bool CalculateOffset(Label* L, int32_t* offset, OffsetSize bits,
+ Register* scratch, const Operand& rt);
+
+ void BranchShortHelperR6(int32_t offset, Label* L);
+ void BranchShortHelper(int16_t offset, Label* L, BranchDelaySlot bdslot);
+ bool BranchShortHelperR6(int32_t offset, Label* L, Condition cond,
+ Register rs, const Operand& rt);
+ bool BranchShortHelper(int16_t offset, Label* L, Condition cond, Register rs,
+ const Operand& rt, BranchDelaySlot bdslot);
+ bool BranchShortCheck(int32_t offset, Label* L, Condition cond, Register rs,
+ const Operand& rt, BranchDelaySlot bdslot);
+
+ void BranchAndLinkShortHelperR6(int32_t offset, Label* L);
+ void BranchAndLinkShortHelper(int16_t offset, Label* L,
+ BranchDelaySlot bdslot);
+ void BranchAndLinkShort(int32_t offset, BranchDelaySlot bdslot = PROTECT);
+ void BranchAndLinkShort(Label* L, BranchDelaySlot bdslot = PROTECT);
+ bool BranchAndLinkShortHelperR6(int32_t offset, Label* L, Condition cond,
+ Register rs, const Operand& rt);
+ bool BranchAndLinkShortHelper(int16_t offset, Label* L, Condition cond,
+ Register rs, const Operand& rt,
+ BranchDelaySlot bdslot);
+ bool BranchAndLinkShortCheck(int32_t offset, Label* L, Condition cond,
+ Register rs, const Operand& rt,
+ BranchDelaySlot bdslot);
+ void BranchAndLinkLong(Label* L, BranchDelaySlot bdslot);
+
+ template <typename RoundFunc>
+ void RoundDouble(FPURegister dst, FPURegister src, FPURoundingMode mode,
+ RoundFunc round);
+
+ template <typename RoundFunc>
+ void RoundFloat(FPURegister dst, FPURegister src, FPURoundingMode mode,
+ RoundFunc round);
+
+ // Push a fixed frame, consisting of ra, fp.
+ void PushCommonFrame(Register marker_reg = no_reg);
+
+ void CallRecordWriteStub(Register object, Register address,
+ RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode, Handle<Code> code_target,
+ Address wasm_target);
+};
+
+// MacroAssembler implements a collection of frequently used macros.
+class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler {
+ public:
+ using TurboAssembler::TurboAssembler;
+
+ // It assumes that the arguments are located below the stack pointer.
+ // argc is the number of arguments not including the receiver.
+ // TODO(victorgomes): Remove this function once we stick with the reversed
+ // arguments order.
+ void LoadReceiver(Register dest, Register argc) {
+ Lw(dest, MemOperand(sp, 0));
+ }
+
+ void StoreReceiver(Register rec, Register argc, Register scratch) {
+ Sw(rec, MemOperand(sp, 0));
+ }
+
+ // Swap two registers. If the scratch register is omitted then a slightly
+ // less efficient form using xor instead of mov is emitted.
+ void Swap(Register reg1, Register reg2, Register scratch = no_reg);
+
+ void PushRoot(RootIndex index) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ LoadRoot(scratch, index);
+ Push(scratch);
+ }
+
+ // Compare the object in a register to a value and jump if they are equal.
+ void JumpIfRoot(Register with, RootIndex index, Label* if_equal) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ LoadRoot(scratch, index);
+ Branch(if_equal, eq, with, Operand(scratch));
+ }
+
+ // Compare the object in a register to a value and jump if they are not equal.
+ void JumpIfNotRoot(Register with, RootIndex index, Label* if_not_equal) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ LoadRoot(scratch, index);
+ Branch(if_not_equal, ne, with, Operand(scratch));
+ }
+
+ // Checks if value is in range [lower_limit, higher_limit] using a single
+ // comparison.
+ void JumpIfIsInRange(Register value, unsigned lower_limit,
+ unsigned higher_limit, Label* on_in_range);
+
+ // ---------------------------------------------------------------------------
+ // GC Support
+
+ // Notify the garbage collector that we wrote a pointer into an object.
+ // |object| is the object being stored into, |value| is the object being
+ // stored. value and scratch registers are clobbered by the operation.
+ // The offset is the offset from the start of the object, not the offset from
+ // the tagged HeapObject pointer. For use with FieldOperand(reg, off).
+ void RecordWriteField(
+ Register object, int offset, Register value, Register scratch,
+ RAStatus ra_status, SaveFPRegsMode save_fp,
+ RememberedSetAction remembered_set_action = EMIT_REMEMBERED_SET,
+ SmiCheck smi_check = INLINE_SMI_CHECK);
+
+ // For a given |object| notify the garbage collector that the slot |address|
+ // has been written. |value| is the object being stored. The value and
+ // address registers are clobbered by the operation.
+ void RecordWrite(
+ Register object, Register address, Register value, RAStatus ra_status,
+ SaveFPRegsMode save_fp,
+ RememberedSetAction remembered_set_action = EMIT_REMEMBERED_SET,
+ SmiCheck smi_check = INLINE_SMI_CHECK);
+
+ void Pref(int32_t hint, const MemOperand& rs);
+
+ // Truncates a double using a specific rounding mode, and writes the value
+ // to the result register.
+ // The except_flag will contain any exceptions caused by the instruction.
+ // If check_inexact is kDontCheckForInexactConversion, then the inexact
+ // exception is masked.
+ void EmitFPUTruncate(
+ FPURoundingMode rounding_mode, Register result,
+ DoubleRegister double_input, Register scratch,
+ DoubleRegister double_scratch, Register except_flag,
+ CheckForInexactConversion check_inexact = kDontCheckForInexactConversion);
+
+ // Enter exit frame.
+ // argc - argument count to be dropped by LeaveExitFrame.
+ // save_doubles - saves FPU registers on stack, currently disabled.
+ // stack_space - extra stack space.
+ void EnterExitFrame(bool save_doubles, int stack_space = 0,
+ StackFrame::Type frame_type = StackFrame::EXIT);
+
+ // Leave the current exit frame.
+ void LeaveExitFrame(bool save_doubles, Register arg_count,
+ bool do_return = NO_EMIT_RETURN,
+ bool argument_count_is_length = false);
+
+ void LoadMap(Register destination, Register object);
+
+ // Make sure the stack is aligned. Only emits code in debug mode.
+ void AssertStackIsAligned();
+
+ // Load the global proxy from the current context.
+ void LoadGlobalProxy(Register dst) {
+ LoadNativeContextSlot(Context::GLOBAL_PROXY_INDEX, dst);
+ }
+
+ void LoadNativeContextSlot(int index, Register dst);
+
+ // -------------------------------------------------------------------------
+ // JavaScript invokes.
+
+ // Invoke the JavaScript function code by either calling or jumping.
+ void InvokeFunctionCode(Register function, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count, InvokeFlag flag);
+
+ // On function call, call into the debugger if necessary.
+ void CheckDebugHook(Register fun, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count);
+
+ // Invoke the JavaScript function in the given register. Changes the
+ // current context to the context in the function before invoking.
+ void InvokeFunctionWithNewTarget(Register function, Register new_target,
+ Register actual_parameter_count,
+ InvokeFlag flag);
+
+ void InvokeFunction(Register function, Register expected_parameter_count,
+ Register actual_parameter_count, InvokeFlag flag);
+
+ // Frame restart support.
+ void MaybeDropFrames();
+
+ // Exception handling.
+
+ // Push a new stack handler and link into stack handler chain.
+ void PushStackHandler();
+
+ // Unlink the stack handler on top of the stack from the stack handler chain.
+ // Must preserve the result register.
+ void PopStackHandler();
+
+ // -------------------------------------------------------------------------
+ // Support functions.
+
+ void GetObjectType(Register function, Register map, Register type_reg);
+
+ // -------------------------------------------------------------------------
+ // Runtime calls.
+
+ // Call a runtime routine.
+ void CallRuntime(const Runtime::Function* f, int num_arguments,
+ SaveFPRegsMode save_doubles = kDontSaveFPRegs);
+
+ // Convenience function: Same as above, but takes the fid instead.
+ void CallRuntime(Runtime::FunctionId fid,
+ SaveFPRegsMode save_doubles = kDontSaveFPRegs) {
+ const Runtime::Function* function = Runtime::FunctionForId(fid);
+ CallRuntime(function, function->nargs, save_doubles);
+ }
+
+ // Convenience function: Same as above, but takes the fid instead.
+ void CallRuntime(Runtime::FunctionId id, int num_arguments,
+ SaveFPRegsMode save_doubles = kDontSaveFPRegs) {
+ CallRuntime(Runtime::FunctionForId(id), num_arguments, save_doubles);
+ }
+
+ // Convenience function: tail call a runtime routine (jump).
+ void TailCallRuntime(Runtime::FunctionId fid);
+
+ // Jump to the builtin routine.
+ void JumpToExternalReference(const ExternalReference& builtin,
+ BranchDelaySlot bd = PROTECT,
+ bool builtin_exit_frame = false);
+
+ // Generates a trampoline to jump to the off-heap instruction stream.
+ void JumpToInstructionStream(Address entry);
+
+ // ---------------------------------------------------------------------------
+ // In-place weak references.
+ void LoadWeakValue(Register out, Register in, Label* target_if_cleared);
+
+ // -------------------------------------------------------------------------
+ // StatsCounter support.
+
+ void IncrementCounter(StatsCounter* counter, int value, Register scratch1,
+ Register scratch2);
+ void DecrementCounter(StatsCounter* counter, int value, Register scratch1,
+ Register scratch2);
+
+ // -------------------------------------------------------------------------
+ // Stack limit utilities
+
+ enum StackLimitKind { kInterruptStackLimit, kRealStackLimit };
+ void LoadStackLimit(Register destination, StackLimitKind kind);
+ void StackOverflowCheck(Register num_args, Register scratch1,
+ Register scratch2, Label* stack_overflow);
+
+ // ---------------------------------------------------------------------------
+ // Smi utilities.
+
+ void SmiTag(Register reg) { Addu(reg, reg, reg); }
+
+ void SmiTag(Register dst, Register src) { Addu(dst, src, src); }
+
+ // Test if the register contains a smi.
+ inline void SmiTst(Register value, Register scratch) {
+ And(scratch, value, Operand(kSmiTagMask));
+ }
+
+ // Jump if the register contains a non-smi.
+ void JumpIfNotSmi(Register value, Label* not_smi_label, Register scratch = at,
+ BranchDelaySlot bd = PROTECT);
+
+ // Abort execution if argument is a smi, enabled via --debug-code.
+ void AssertNotSmi(Register object);
+ void AssertSmi(Register object);
+
+ // Abort execution if argument is not a Constructor, enabled via --debug-code.
+ void AssertConstructor(Register object);
+
+ // Abort execution if argument is not a JSFunction, enabled via --debug-code.
+ void AssertFunction(Register object);
+
+ // Abort execution if argument is not a JSBoundFunction,
+ // enabled via --debug-code.
+ void AssertBoundFunction(Register object);
+
+ // Abort execution if argument is not a JSGeneratorObject (or subclass),
+ // enabled via --debug-code.
+ void AssertGeneratorObject(Register object);
+
+ // Abort execution if argument is not undefined or an AllocationSite, enabled
+ // via --debug-code.
+ void AssertUndefinedOrAllocationSite(Register object, Register scratch);
+
+ template <typename Field>
+ void DecodeField(Register dst, Register src) {
+ Ext(dst, src, Field::kShift, Field::kSize);
+ }
+
+ template <typename Field>
+ void DecodeField(Register reg) {
+ DecodeField<Field>(reg, reg);
+ }
+
+ private:
+ // Helper functions for generating invokes.
+ void InvokePrologue(Register expected_parameter_count,
+ Register actual_parameter_count, Label* done,
+ InvokeFlag flag);
+
+ // Compute memory operands for safepoint stack slots.
+ static int SafepointRegisterStackIndex(int reg_code);
+
+ // Needs access to SafepointRegisterStackIndex for compiled frame
+ // traversal.
+ friend class CommonFrame;
+
+ DISALLOW_IMPLICIT_CONSTRUCTORS(MacroAssembler);
+};
+
+template <typename Func>
+void TurboAssembler::GenerateSwitchTable(Register index, size_t case_count,
+ Func GetLabelFunction) {
+ Label here;
+ BlockTrampolinePoolFor(case_count + kSwitchTablePrologueSize);
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ if (kArchVariant >= kMips32r6) {
+ addiupc(scratch, 5);
+ Lsa(scratch, scratch, index, kPointerSizeLog2);
+ lw(scratch, MemOperand(scratch));
+ } else {
+ push(ra);
+ bal(&here);
+ sll(scratch, index, kPointerSizeLog2); // Branch delay slot.
+ bind(&here);
+ addu(scratch, scratch, ra);
+ pop(ra);
+ lw(scratch, MemOperand(scratch, 6 * v8::internal::kInstrSize));
+ }
+ jr(scratch);
+ nop(); // Branch delay slot nop.
+ for (size_t index = 0; index < case_count; ++index) {
+ dd(GetLabelFunction(index));
+ }
+}
+
+#define ACCESS_MASM(masm) masm->
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_MIPS_MACRO_ASSEMBLER_MIPS_H_
diff --git a/src/codegen/mips/register-mips.h b/src/codegen/mips/register-mips.h
new file mode 100644
index 0000000..2b5f454
--- /dev/null
+++ b/src/codegen/mips/register-mips.h
@@ -0,0 +1,384 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_MIPS_REGISTER_MIPS_H_
+#define V8_CODEGEN_MIPS_REGISTER_MIPS_H_
+
+#include "src/codegen/mips/constants-mips.h"
+#include "src/codegen/register.h"
+#include "src/codegen/reglist.h"
+
+namespace v8 {
+namespace internal {
+
+// clang-format off
+#define GENERAL_REGISTERS(V) \
+ V(zero_reg) V(at) V(v0) V(v1) V(a0) V(a1) V(a2) V(a3) \
+ V(t0) V(t1) V(t2) V(t3) V(t4) V(t5) V(t6) V(t7) \
+ V(s0) V(s1) V(s2) V(s3) V(s4) V(s5) V(s6) V(s7) V(t8) V(t9) \
+ V(k0) V(k1) V(gp) V(sp) V(fp) V(ra)
+
+#define ALLOCATABLE_GENERAL_REGISTERS(V) \
+ V(a0) V(a1) V(a2) V(a3) \
+ V(t0) V(t1) V(t2) V(t3) V(t4) V(t5) V(t6) V(s7) \
+ V(v0) V(v1)
+
+#define DOUBLE_REGISTERS(V) \
+ V(f0) V(f1) V(f2) V(f3) V(f4) V(f5) V(f6) V(f7) \
+ V(f8) V(f9) V(f10) V(f11) V(f12) V(f13) V(f14) V(f15) \
+ V(f16) V(f17) V(f18) V(f19) V(f20) V(f21) V(f22) V(f23) \
+ V(f24) V(f25) V(f26) V(f27) V(f28) V(f29) V(f30) V(f31)
+
+#define FLOAT_REGISTERS DOUBLE_REGISTERS
+#define SIMD128_REGISTERS(V) \
+ V(w0) V(w1) V(w2) V(w3) V(w4) V(w5) V(w6) V(w7) \
+ V(w8) V(w9) V(w10) V(w11) V(w12) V(w13) V(w14) V(w15) \
+ V(w16) V(w17) V(w18) V(w19) V(w20) V(w21) V(w22) V(w23) \
+ V(w24) V(w25) V(w26) V(w27) V(w28) V(w29) V(w30) V(w31)
+
+#define ALLOCATABLE_DOUBLE_REGISTERS(V) \
+ V(f0) V(f2) V(f4) V(f6) V(f8) V(f10) V(f12) V(f14) \
+ V(f16) V(f18) V(f20) V(f22) V(f24)
+// clang-format on
+
+// Register lists.
+// Note that the bit values must match those used in actual instruction
+// encoding.
+const int kNumRegs = 32;
+
+const RegList kJSCallerSaved = 1 << 2 | // v0
+ 1 << 3 | // v1
+ 1 << 4 | // a0
+ 1 << 5 | // a1
+ 1 << 6 | // a2
+ 1 << 7 | // a3
+ 1 << 8 | // t0
+ 1 << 9 | // t1
+ 1 << 10 | // t2
+ 1 << 11 | // t3
+ 1 << 12 | // t4
+ 1 << 13 | // t5
+ 1 << 14 | // t6
+ 1 << 15; // t7
+
+const int kNumJSCallerSaved = 14;
+
+// Callee-saved registers preserved when switching from C to JavaScript.
+const RegList kCalleeSaved = 1 << 16 | // s0
+ 1 << 17 | // s1
+ 1 << 18 | // s2
+ 1 << 19 | // s3
+ 1 << 20 | // s4
+ 1 << 21 | // s5
+ 1 << 22 | // s6 (roots in Javascript code)
+ 1 << 23 | // s7 (cp in Javascript code)
+ 1 << 30; // fp/s8
+
+const int kNumCalleeSaved = 9;
+
+const RegList kCalleeSavedFPU = 1 << 20 | // f20
+ 1 << 22 | // f22
+ 1 << 24 | // f24
+ 1 << 26 | // f26
+ 1 << 28 | // f28
+ 1 << 30; // f30
+
+const int kNumCalleeSavedFPU = 6;
+
+const RegList kCallerSavedFPU = 1 << 0 | // f0
+ 1 << 2 | // f2
+ 1 << 4 | // f4
+ 1 << 6 | // f6
+ 1 << 8 | // f8
+ 1 << 10 | // f10
+ 1 << 12 | // f12
+ 1 << 14 | // f14
+ 1 << 16 | // f16
+ 1 << 18; // f18
+
+// Number of registers for which space is reserved in safepoints. Must be a
+// multiple of 8.
+const int kNumSafepointRegisters = 24;
+
+// Define the list of registers actually saved at safepoints.
+// Note that the number of saved registers may be smaller than the reserved
+// space, i.e. kNumSafepointSavedRegisters <= kNumSafepointRegisters.
+const RegList kSafepointSavedRegisters = kJSCallerSaved | kCalleeSaved;
+const int kNumSafepointSavedRegisters = kNumJSCallerSaved + kNumCalleeSaved;
+
+const int kUndefIndex = -1;
+// Map with indexes on stack that corresponds to codes of saved registers.
+const int kSafepointRegisterStackIndexMap[kNumRegs] = {kUndefIndex, // zero_reg
+ kUndefIndex, // at
+ 0, // v0
+ 1, // v1
+ 2, // a0
+ 3, // a1
+ 4, // a2
+ 5, // a3
+ 6, // t0
+ 7, // t1
+ 8, // t2
+ 9, // t3
+ 10, // t4
+ 11, // t5
+ 12, // t6
+ 13, // t7
+ 14, // s0
+ 15, // s1
+ 16, // s2
+ 17, // s3
+ 18, // s4
+ 19, // s5
+ 20, // s6
+ 21, // s7
+ kUndefIndex, // t8
+ kUndefIndex, // t9
+ kUndefIndex, // k0
+ kUndefIndex, // k1
+ kUndefIndex, // gp
+ kUndefIndex, // sp
+ 22, // fp
+ kUndefIndex};
+
+// CPU Registers.
+//
+// 1) We would prefer to use an enum, but enum values are assignment-
+// compatible with int, which has caused code-generation bugs.
+//
+// 2) We would prefer to use a class instead of a struct but we don't like
+// the register initialization to depend on the particular initialization
+// order (which appears to be different on OS X, Linux, and Windows for the
+// installed versions of C++ we tried). Using a struct permits C-style
+// "initialization". Also, the Register objects cannot be const as this
+// forces initialization stubs in MSVC, making us dependent on initialization
+// order.
+//
+// 3) By not using an enum, we are possibly preventing the compiler from
+// doing certain constant folds, which may significantly reduce the
+// code generated for some assembly instructions (because they boil down
+// to a few constants). If this is a problem, we could change the code
+// such that we use an enum in optimized mode, and the struct in debug
+// mode. This way we get the compile-time error checking in debug mode
+// and best performance in optimized code.
+
+// -----------------------------------------------------------------------------
+// Implementation of Register and FPURegister.
+
+enum RegisterCode {
+#define REGISTER_CODE(R) kRegCode_##R,
+ GENERAL_REGISTERS(REGISTER_CODE)
+#undef REGISTER_CODE
+ kRegAfterLast
+};
+
+class Register : public RegisterBase<Register, kRegAfterLast> {
+ public:
+#if defined(V8_TARGET_LITTLE_ENDIAN)
+ static constexpr int kMantissaOffset = 0;
+ static constexpr int kExponentOffset = 4;
+#elif defined(V8_TARGET_BIG_ENDIAN)
+ static constexpr int kMantissaOffset = 4;
+ static constexpr int kExponentOffset = 0;
+#else
+#error Unknown endianness
+#endif
+
+ private:
+ friend class RegisterBase;
+ explicit constexpr Register(int code) : RegisterBase(code) {}
+};
+
+// s7: context register
+// s3: scratch register
+// s4: scratch register 2
+#define DECLARE_REGISTER(R) \
+ constexpr Register R = Register::from_code(kRegCode_##R);
+GENERAL_REGISTERS(DECLARE_REGISTER)
+#undef DECLARE_REGISTER
+constexpr Register no_reg = Register::no_reg();
+
+int ToNumber(Register reg);
+
+Register ToRegister(int num);
+
+constexpr bool kPadArguments = false;
+constexpr bool kSimpleFPAliasing = true;
+constexpr bool kSimdMaskRegisters = false;
+
+enum DoubleRegisterCode {
+#define REGISTER_CODE(R) kDoubleCode_##R,
+ DOUBLE_REGISTERS(REGISTER_CODE)
+#undef REGISTER_CODE
+ kDoubleAfterLast
+};
+
+// Coprocessor register.
+class FPURegister : public RegisterBase<FPURegister, kDoubleAfterLast> {
+ public:
+ FPURegister low() const {
+ // Find low reg of a Double-reg pair, which is the reg itself.
+ DCHECK_EQ(code() % 2, 0); // Specified Double reg must be even.
+ return FPURegister::from_code(code());
+ }
+ FPURegister high() const {
+ // Find high reg of a Doubel-reg pair, which is reg + 1.
+ DCHECK_EQ(code() % 2, 0); // Specified Double reg must be even.
+ return FPURegister::from_code(code() + 1);
+ }
+
+ private:
+ friend class RegisterBase;
+ explicit constexpr FPURegister(int code) : RegisterBase(code) {}
+};
+
+enum MSARegisterCode {
+#define REGISTER_CODE(R) kMsaCode_##R,
+ SIMD128_REGISTERS(REGISTER_CODE)
+#undef REGISTER_CODE
+ kMsaAfterLast
+};
+
+// MIPS SIMD (MSA) register
+class MSARegister : public RegisterBase<MSARegister, kMsaAfterLast> {
+ friend class RegisterBase;
+ explicit constexpr MSARegister(int code) : RegisterBase(code) {}
+};
+
+// A few double registers are reserved: one as a scratch register and one to
+// hold 0.0.
+// f28: 0.0
+// f30: scratch register.
+
+// V8 now supports the O32 ABI, and the FPU Registers are organized as 32
+// 32-bit registers, f0 through f31. When used as 'double' they are used
+// in pairs, starting with the even numbered register. So a double operation
+// on f0 really uses f0 and f1.
+// (Modern mips hardware also supports 32 64-bit registers, via setting
+// (priviledged) Status Register FR bit to 1. This is used by the N32 ABI,
+// but it is not in common use. Someday we will want to support this in v8.)
+
+// For O32 ABI, Floats and Doubles refer to same set of 32 32-bit registers.
+using FloatRegister = FPURegister;
+
+using DoubleRegister = FPURegister;
+
+#define DECLARE_DOUBLE_REGISTER(R) \
+ constexpr DoubleRegister R = DoubleRegister::from_code(kDoubleCode_##R);
+DOUBLE_REGISTERS(DECLARE_DOUBLE_REGISTER)
+#undef DECLARE_DOUBLE_REGISTER
+
+constexpr DoubleRegister no_dreg = DoubleRegister::no_reg();
+
+// SIMD registers.
+using Simd128Register = MSARegister;
+
+#define DECLARE_SIMD128_REGISTER(R) \
+ constexpr Simd128Register R = Simd128Register::from_code(kMsaCode_##R);
+SIMD128_REGISTERS(DECLARE_SIMD128_REGISTER)
+#undef DECLARE_SIMD128_REGISTER
+
+const Simd128Register no_msareg = Simd128Register::no_reg();
+
+// Register aliases.
+// cp is assumed to be a callee saved register.
+constexpr Register kRootRegister = s6;
+constexpr Register cp = s7;
+constexpr Register kScratchReg = s3;
+constexpr Register kScratchReg2 = s4;
+constexpr DoubleRegister kScratchDoubleReg = f30;
+constexpr DoubleRegister kDoubleRegZero = f28;
+// Used on mips32r6 for compare operations.
+constexpr DoubleRegister kDoubleCompareReg = f26;
+// MSA zero and scratch regs must have the same numbers as FPU zero and scratch
+constexpr Simd128Register kSimd128RegZero = w28;
+constexpr Simd128Register kSimd128ScratchReg = w30;
+
+// FPU (coprocessor 1) control registers.
+// Currently only FCSR (#31) is implemented.
+struct FPUControlRegister {
+ bool is_valid() const { return reg_code == kFCSRRegister; }
+ bool is(FPUControlRegister creg) const { return reg_code == creg.reg_code; }
+ int code() const {
+ DCHECK(is_valid());
+ return reg_code;
+ }
+ int bit() const {
+ DCHECK(is_valid());
+ return 1 << reg_code;
+ }
+ void setcode(int f) {
+ reg_code = f;
+ DCHECK(is_valid());
+ }
+ // Unfortunately we can't make this private in a struct.
+ int reg_code;
+};
+
+constexpr FPUControlRegister no_fpucreg = {kInvalidFPUControlRegister};
+constexpr FPUControlRegister FCSR = {kFCSRRegister};
+
+// MSA control registers
+struct MSAControlRegister {
+ bool is_valid() const {
+ return (reg_code == kMSAIRRegister) || (reg_code == kMSACSRRegister);
+ }
+ bool is(MSAControlRegister creg) const { return reg_code == creg.reg_code; }
+ int code() const {
+ DCHECK(is_valid());
+ return reg_code;
+ }
+ int bit() const {
+ DCHECK(is_valid());
+ return 1 << reg_code;
+ }
+ void setcode(int f) {
+ reg_code = f;
+ DCHECK(is_valid());
+ }
+ // Unfortunately we can't make this private in a struct.
+ int reg_code;
+};
+
+constexpr MSAControlRegister no_msacreg = {kInvalidMSAControlRegister};
+constexpr MSAControlRegister MSAIR = {kMSAIRRegister};
+constexpr MSAControlRegister MSACSR = {kMSACSRRegister};
+
+// Define {RegisterName} methods for the register types.
+DEFINE_REGISTER_NAMES(Register, GENERAL_REGISTERS)
+DEFINE_REGISTER_NAMES(FPURegister, DOUBLE_REGISTERS)
+DEFINE_REGISTER_NAMES(MSARegister, SIMD128_REGISTERS)
+
+// Give alias names to registers for calling conventions.
+constexpr Register kReturnRegister0 = v0;
+constexpr Register kReturnRegister1 = v1;
+constexpr Register kReturnRegister2 = a0;
+constexpr Register kJSFunctionRegister = a1;
+constexpr Register kContextRegister = s7;
+constexpr Register kAllocateSizeRegister = a0;
+constexpr Register kSpeculationPoisonRegister = t3;
+constexpr Register kInterpreterAccumulatorRegister = v0;
+constexpr Register kInterpreterBytecodeOffsetRegister = t4;
+constexpr Register kInterpreterBytecodeArrayRegister = t5;
+constexpr Register kInterpreterDispatchTableRegister = t6;
+
+constexpr Register kJavaScriptCallArgCountRegister = a0;
+constexpr Register kJavaScriptCallCodeStartRegister = a2;
+constexpr Register kJavaScriptCallTargetRegister = kJSFunctionRegister;
+constexpr Register kJavaScriptCallNewTargetRegister = a3;
+constexpr Register kJavaScriptCallExtraArg1Register = a2;
+
+constexpr Register kOffHeapTrampolineRegister = at;
+constexpr Register kRuntimeCallFunctionRegister = a1;
+constexpr Register kRuntimeCallArgCountRegister = a0;
+constexpr Register kRuntimeCallArgvRegister = a2;
+constexpr Register kWasmInstanceRegister = a0;
+constexpr Register kWasmCompileLazyFuncIndexRegister = t0;
+
+constexpr DoubleRegister kFPReturnRegister0 = f0;
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_MIPS_REGISTER_MIPS_H_
diff --git a/src/codegen/mips64/assembler-mips64-inl.h b/src/codegen/mips64/assembler-mips64-inl.h
new file mode 100644
index 0000000..cacdbd8
--- /dev/null
+++ b/src/codegen/mips64/assembler-mips64-inl.h
@@ -0,0 +1,308 @@
+
+// Copyright (c) 1994-2006 Sun Microsystems Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// - Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// - Redistribution in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// - Neither the name of Sun Microsystems or the names of contributors may
+// be used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// The original source code covered by the above license above has been
+// modified significantly by Google Inc.
+// Copyright 2012 the V8 project authors. All rights reserved.
+
+#ifndef V8_CODEGEN_MIPS64_ASSEMBLER_MIPS64_INL_H_
+#define V8_CODEGEN_MIPS64_ASSEMBLER_MIPS64_INL_H_
+
+#include "src/codegen/mips64/assembler-mips64.h"
+
+#include "src/codegen/assembler.h"
+#include "src/debug/debug.h"
+#include "src/objects/objects-inl.h"
+
+namespace v8 {
+namespace internal {
+
+bool CpuFeatures::SupportsOptimizer() { return IsSupported(FPU); }
+
+bool CpuFeatures::SupportsWasmSimd128() { return IsSupported(MIPS_SIMD); }
+
+// -----------------------------------------------------------------------------
+// Operand and MemOperand.
+
+bool Operand::is_reg() const { return rm_.is_valid(); }
+
+int64_t Operand::immediate() const {
+ DCHECK(!is_reg());
+ DCHECK(!IsHeapObjectRequest());
+ return value_.immediate;
+}
+
+// -----------------------------------------------------------------------------
+// RelocInfo.
+
+void RelocInfo::apply(intptr_t delta) {
+ if (IsInternalReference(rmode_) || IsInternalReferenceEncoded(rmode_)) {
+ // Absolute code pointer inside code object moves with the code object.
+ Assembler::RelocateInternalReference(rmode_, pc_, delta);
+ }
+}
+
+Address RelocInfo::target_address() {
+ DCHECK(IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_) || IsWasmCall(rmode_));
+ return Assembler::target_address_at(pc_, constant_pool_);
+}
+
+Address RelocInfo::target_address_address() {
+ DCHECK(HasTargetAddressAddress());
+ // Read the address of the word containing the target_address in an
+ // instruction stream.
+ // The only architecture-independent user of this function is the serializer.
+ // The serializer uses it to find out how many raw bytes of instruction to
+ // output before the next target.
+ // For an instruction like LUI/ORI where the target bits are mixed into the
+ // instruction bits, the size of the target will be zero, indicating that the
+ // serializer should not step forward in memory after a target is resolved
+ // and written. In this case the target_address_address function should
+ // return the end of the instructions to be patched, allowing the
+ // deserializer to deserialize the instructions as raw bytes and put them in
+ // place, ready to be patched with the target. After jump optimization,
+ // that is the address of the instruction that follows J/JAL/JR/JALR
+ // instruction.
+ return pc_ + Assembler::kInstructionsFor64BitConstant * kInstrSize;
+}
+
+Address RelocInfo::constant_pool_entry_address() { UNREACHABLE(); }
+
+int RelocInfo::target_address_size() { return Assembler::kSpecialTargetSize; }
+
+void Assembler::deserialization_set_special_target_at(
+ Address instruction_payload, Code code, Address target) {
+ set_target_address_at(instruction_payload,
+ !code.is_null() ? code.constant_pool() : kNullAddress,
+ target);
+}
+
+int Assembler::deserialization_special_target_size(
+ Address instruction_payload) {
+ return kSpecialTargetSize;
+}
+
+void Assembler::set_target_internal_reference_encoded_at(Address pc,
+ Address target) {
+ // Encoded internal references are j/jal instructions.
+ Instr instr = Assembler::instr_at(pc + 0 * kInstrSize);
+
+ uint64_t imm28 = target & static_cast<uint64_t>(kImm28Mask);
+
+ instr &= ~kImm26Mask;
+ uint64_t imm26 = imm28 >> 2;
+ DCHECK(is_uint26(imm26));
+
+ instr_at_put(pc, instr | (imm26 & kImm26Mask));
+ // Currently used only by deserializer, and all code will be flushed
+ // after complete deserialization, no need to flush on each reference.
+}
+
+void Assembler::deserialization_set_target_internal_reference_at(
+ Address pc, Address target, RelocInfo::Mode mode) {
+ if (mode == RelocInfo::INTERNAL_REFERENCE_ENCODED) {
+ DCHECK(IsJ(instr_at(pc)));
+ set_target_internal_reference_encoded_at(pc, target);
+ } else {
+ DCHECK(mode == RelocInfo::INTERNAL_REFERENCE);
+ Memory<Address>(pc) = target;
+ }
+}
+
+HeapObject RelocInfo::target_object() {
+ DCHECK(IsCodeTarget(rmode_) || IsFullEmbeddedObject(rmode_));
+ return HeapObject::cast(
+ Object(Assembler::target_address_at(pc_, constant_pool_)));
+}
+
+HeapObject RelocInfo::target_object_no_host(Isolate* isolate) {
+ return target_object();
+}
+
+Handle<HeapObject> RelocInfo::target_object_handle(Assembler* origin) {
+ DCHECK(IsCodeTarget(rmode_) || IsFullEmbeddedObject(rmode_));
+ return Handle<HeapObject>(reinterpret_cast<Address*>(
+ Assembler::target_address_at(pc_, constant_pool_)));
+}
+
+void RelocInfo::set_target_object(Heap* heap, HeapObject target,
+ WriteBarrierMode write_barrier_mode,
+ ICacheFlushMode icache_flush_mode) {
+ DCHECK(IsCodeTarget(rmode_) || IsFullEmbeddedObject(rmode_));
+ Assembler::set_target_address_at(pc_, constant_pool_, target.ptr(),
+ icache_flush_mode);
+ if (write_barrier_mode == UPDATE_WRITE_BARRIER && !host().is_null() &&
+ !FLAG_disable_write_barriers) {
+ WriteBarrierForCode(host(), this, target);
+ }
+}
+
+Address RelocInfo::target_external_reference() {
+ DCHECK(rmode_ == EXTERNAL_REFERENCE);
+ return Assembler::target_address_at(pc_, constant_pool_);
+}
+
+void RelocInfo::set_target_external_reference(
+ Address target, ICacheFlushMode icache_flush_mode) {
+ DCHECK(rmode_ == RelocInfo::EXTERNAL_REFERENCE);
+ Assembler::set_target_address_at(pc_, constant_pool_, target,
+ icache_flush_mode);
+}
+
+Address RelocInfo::target_internal_reference() {
+ if (rmode_ == INTERNAL_REFERENCE) {
+ return Memory<Address>(pc_);
+ } else {
+ // Encoded internal references are j/jal instructions.
+ DCHECK(rmode_ == INTERNAL_REFERENCE_ENCODED);
+ Instr instr = Assembler::instr_at(pc_ + 0 * kInstrSize);
+ instr &= kImm26Mask;
+ uint64_t imm28 = instr << 2;
+ uint64_t segment = pc_ & ~static_cast<uint64_t>(kImm28Mask);
+ return static_cast<Address>(segment | imm28);
+ }
+}
+
+Address RelocInfo::target_internal_reference_address() {
+ DCHECK(rmode_ == INTERNAL_REFERENCE || rmode_ == INTERNAL_REFERENCE_ENCODED);
+ return pc_;
+}
+
+Address RelocInfo::target_runtime_entry(Assembler* origin) {
+ DCHECK(IsRuntimeEntry(rmode_));
+ return target_address();
+}
+
+void RelocInfo::set_target_runtime_entry(Address target,
+ WriteBarrierMode write_barrier_mode,
+ ICacheFlushMode icache_flush_mode) {
+ DCHECK(IsRuntimeEntry(rmode_));
+ if (target_address() != target)
+ set_target_address(target, write_barrier_mode, icache_flush_mode);
+}
+
+Address RelocInfo::target_off_heap_target() {
+ DCHECK(IsOffHeapTarget(rmode_));
+ return Assembler::target_address_at(pc_, constant_pool_);
+}
+
+void RelocInfo::WipeOut() {
+ DCHECK(IsFullEmbeddedObject(rmode_) || IsCodeTarget(rmode_) ||
+ IsRuntimeEntry(rmode_) || IsExternalReference(rmode_) ||
+ IsInternalReference(rmode_) || IsInternalReferenceEncoded(rmode_) ||
+ IsOffHeapTarget(rmode_));
+ if (IsInternalReference(rmode_)) {
+ Memory<Address>(pc_) = kNullAddress;
+ } else if (IsInternalReferenceEncoded(rmode_)) {
+ Assembler::set_target_internal_reference_encoded_at(pc_, kNullAddress);
+ } else {
+ Assembler::set_target_address_at(pc_, constant_pool_, kNullAddress);
+ }
+}
+
+// -----------------------------------------------------------------------------
+// Assembler.
+
+void Assembler::CheckBuffer() {
+ if (buffer_space() <= kGap) {
+ GrowBuffer();
+ }
+}
+
+void Assembler::CheckForEmitInForbiddenSlot() {
+ if (!is_buffer_growth_blocked()) {
+ CheckBuffer();
+ }
+ if (IsPrevInstrCompactBranch()) {
+ // Nop instruction to precede a CTI in forbidden slot:
+ Instr nop = SPECIAL | SLL;
+ *reinterpret_cast<Instr*>(pc_) = nop;
+ pc_ += kInstrSize;
+
+ ClearCompactBranchState();
+ }
+}
+
+void Assembler::EmitHelper(Instr x, CompactBranchType is_compact_branch) {
+ if (IsPrevInstrCompactBranch()) {
+ if (Instruction::IsForbiddenAfterBranchInstr(x)) {
+ // Nop instruction to precede a CTI in forbidden slot:
+ Instr nop = SPECIAL | SLL;
+ *reinterpret_cast<Instr*>(pc_) = nop;
+ pc_ += kInstrSize;
+ }
+ ClearCompactBranchState();
+ }
+ *reinterpret_cast<Instr*>(pc_) = x;
+ pc_ += kInstrSize;
+ if (is_compact_branch == CompactBranchType::COMPACT_BRANCH) {
+ EmittedCompactBranchInstruction();
+ }
+ CheckTrampolinePoolQuick();
+}
+
+template <>
+inline void Assembler::EmitHelper(uint8_t x);
+
+template <typename T>
+void Assembler::EmitHelper(T x) {
+ *reinterpret_cast<T*>(pc_) = x;
+ pc_ += sizeof(x);
+ CheckTrampolinePoolQuick();
+}
+
+template <>
+void Assembler::EmitHelper(uint8_t x) {
+ *reinterpret_cast<uint8_t*>(pc_) = x;
+ pc_ += sizeof(x);
+ if (reinterpret_cast<intptr_t>(pc_) % kInstrSize == 0) {
+ CheckTrampolinePoolQuick();
+ }
+}
+
+void Assembler::emit(Instr x, CompactBranchType is_compact_branch) {
+ if (!is_buffer_growth_blocked()) {
+ CheckBuffer();
+ }
+ EmitHelper(x, is_compact_branch);
+}
+
+void Assembler::emit(uint64_t data) {
+ CheckForEmitInForbiddenSlot();
+ EmitHelper(data);
+}
+
+EnsureSpace::EnsureSpace(Assembler* assembler) { assembler->CheckBuffer(); }
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_MIPS64_ASSEMBLER_MIPS64_INL_H_
diff --git a/src/codegen/mips64/assembler-mips64.cc b/src/codegen/mips64/assembler-mips64.cc
new file mode 100644
index 0000000..3b16805
--- /dev/null
+++ b/src/codegen/mips64/assembler-mips64.cc
@@ -0,0 +1,3987 @@
+// Copyright (c) 1994-2006 Sun Microsystems Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// - Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// - Redistribution in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// - Neither the name of Sun Microsystems or the names of contributors may
+// be used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// The original source code covered by the above license above has been
+// modified significantly by Google Inc.
+// Copyright 2012 the V8 project authors. All rights reserved.
+
+#include "src/codegen/mips64/assembler-mips64.h"
+
+#if V8_TARGET_ARCH_MIPS64
+
+#include "src/base/cpu.h"
+#include "src/codegen/mips64/assembler-mips64-inl.h"
+#include "src/codegen/safepoint-table.h"
+#include "src/codegen/string-constants.h"
+#include "src/deoptimizer/deoptimizer.h"
+#include "src/objects/heap-number-inl.h"
+
+namespace v8 {
+namespace internal {
+
+// Get the CPU features enabled by the build. For cross compilation the
+// preprocessor symbols CAN_USE_FPU_INSTRUCTIONS
+// can be defined to enable FPU instructions when building the
+// snapshot.
+static unsigned CpuFeaturesImpliedByCompiler() {
+ unsigned answer = 0;
+#ifdef CAN_USE_FPU_INSTRUCTIONS
+ answer |= 1u << FPU;
+#endif // def CAN_USE_FPU_INSTRUCTIONS
+
+ // If the compiler is allowed to use FPU then we can use FPU too in our code
+ // generation even when generating snapshots. This won't work for cross
+ // compilation.
+#if defined(__mips__) && defined(__mips_hard_float) && __mips_hard_float != 0
+ answer |= 1u << FPU;
+#endif
+
+ return answer;
+}
+
+void CpuFeatures::ProbeImpl(bool cross_compile) {
+ supported_ |= CpuFeaturesImpliedByCompiler();
+
+ // Only use statically determined features for cross compile (snapshot).
+ if (cross_compile) return;
+
+ // If the compiler is allowed to use fpu then we can use fpu too in our
+ // code generation.
+#ifndef __mips__
+ // For the simulator build, use FPU.
+ supported_ |= 1u << FPU;
+#if defined(_MIPS_ARCH_MIPS64R6) && defined(_MIPS_MSA)
+ supported_ |= 1u << MIPS_SIMD;
+#endif
+#else
+ // Probe for additional features at runtime.
+ base::CPU cpu;
+ if (cpu.has_fpu()) supported_ |= 1u << FPU;
+#if defined(_MIPS_MSA)
+ supported_ |= 1u << MIPS_SIMD;
+#else
+ if (cpu.has_msa()) supported_ |= 1u << MIPS_SIMD;
+#endif
+#endif
+}
+
+void CpuFeatures::PrintTarget() {}
+void CpuFeatures::PrintFeatures() {}
+
+int ToNumber(Register reg) {
+ DCHECK(reg.is_valid());
+ const int kNumbers[] = {
+ 0, // zero_reg
+ 1, // at
+ 2, // v0
+ 3, // v1
+ 4, // a0
+ 5, // a1
+ 6, // a2
+ 7, // a3
+ 8, // a4
+ 9, // a5
+ 10, // a6
+ 11, // a7
+ 12, // t0
+ 13, // t1
+ 14, // t2
+ 15, // t3
+ 16, // s0
+ 17, // s1
+ 18, // s2
+ 19, // s3
+ 20, // s4
+ 21, // s5
+ 22, // s6
+ 23, // s7
+ 24, // t8
+ 25, // t9
+ 26, // k0
+ 27, // k1
+ 28, // gp
+ 29, // sp
+ 30, // fp
+ 31, // ra
+ };
+ return kNumbers[reg.code()];
+}
+
+Register ToRegister(int num) {
+ DCHECK(num >= 0 && num < kNumRegisters);
+ const Register kRegisters[] = {
+ zero_reg, at, v0, v1, a0, a1, a2, a3, a4, a5, a6, a7, t0, t1, t2, t3,
+ s0, s1, s2, s3, s4, s5, s6, s7, t8, t9, k0, k1, gp, sp, fp, ra};
+ return kRegisters[num];
+}
+
+// -----------------------------------------------------------------------------
+// Implementation of RelocInfo.
+
+const int RelocInfo::kApplyMask =
+ RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE) |
+ RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE_ENCODED);
+
+bool RelocInfo::IsCodedSpecially() {
+ // The deserializer needs to know whether a pointer is specially coded. Being
+ // specially coded on MIPS means that it is a lui/ori instruction, and that is
+ // always the case inside code objects.
+ return true;
+}
+
+bool RelocInfo::IsInConstantPool() { return false; }
+
+uint32_t RelocInfo::wasm_call_tag() const {
+ DCHECK(rmode_ == WASM_CALL || rmode_ == WASM_STUB_CALL);
+ return static_cast<uint32_t>(
+ Assembler::target_address_at(pc_, constant_pool_));
+}
+
+// -----------------------------------------------------------------------------
+// Implementation of Operand and MemOperand.
+// See assembler-mips-inl.h for inlined constructors.
+
+Operand::Operand(Handle<HeapObject> handle)
+ : rm_(no_reg), rmode_(RelocInfo::FULL_EMBEDDED_OBJECT) {
+ value_.immediate = static_cast<intptr_t>(handle.address());
+}
+
+Operand Operand::EmbeddedNumber(double value) {
+ int32_t smi;
+ if (DoubleToSmiInteger(value, &smi)) return Operand(Smi::FromInt(smi));
+ Operand result(0, RelocInfo::FULL_EMBEDDED_OBJECT);
+ result.is_heap_object_request_ = true;
+ result.value_.heap_object_request = HeapObjectRequest(value);
+ return result;
+}
+
+Operand Operand::EmbeddedStringConstant(const StringConstantBase* str) {
+ Operand result(0, RelocInfo::FULL_EMBEDDED_OBJECT);
+ result.is_heap_object_request_ = true;
+ result.value_.heap_object_request = HeapObjectRequest(str);
+ return result;
+}
+
+MemOperand::MemOperand(Register rm, int32_t offset) : Operand(rm) {
+ offset_ = offset;
+}
+
+MemOperand::MemOperand(Register rm, int32_t unit, int32_t multiplier,
+ OffsetAddend offset_addend)
+ : Operand(rm) {
+ offset_ = unit * multiplier + offset_addend;
+}
+
+void Assembler::AllocateAndInstallRequestedHeapObjects(Isolate* isolate) {
+ DCHECK_IMPLIES(isolate == nullptr, heap_object_requests_.empty());
+ for (auto& request : heap_object_requests_) {
+ Handle<HeapObject> object;
+ switch (request.kind()) {
+ case HeapObjectRequest::kHeapNumber:
+ object = isolate->factory()->NewHeapNumber<AllocationType::kOld>(
+ request.heap_number());
+ break;
+ case HeapObjectRequest::kStringConstant:
+ const StringConstantBase* str = request.string();
+ CHECK_NOT_NULL(str);
+ object = str->AllocateStringConstant(isolate);
+ break;
+ }
+ Address pc = reinterpret_cast<Address>(buffer_start_) + request.offset();
+ set_target_value_at(pc, reinterpret_cast<uint64_t>(object.location()));
+ }
+}
+
+// -----------------------------------------------------------------------------
+// Specific instructions, constants, and masks.
+
+// daddiu(sp, sp, 8) aka Pop() operation or part of Pop(r)
+// operations as post-increment of sp.
+const Instr kPopInstruction = DADDIU | (sp.code() << kRsShift) |
+ (sp.code() << kRtShift) |
+ (kPointerSize & kImm16Mask); // NOLINT
+// daddiu(sp, sp, -8) part of Push(r) operation as pre-decrement of sp.
+const Instr kPushInstruction = DADDIU | (sp.code() << kRsShift) |
+ (sp.code() << kRtShift) |
+ (-kPointerSize & kImm16Mask); // NOLINT
+// Sd(r, MemOperand(sp, 0))
+const Instr kPushRegPattern =
+ SD | (sp.code() << kRsShift) | (0 & kImm16Mask); // NOLINT
+// Ld(r, MemOperand(sp, 0))
+const Instr kPopRegPattern =
+ LD | (sp.code() << kRsShift) | (0 & kImm16Mask); // NOLINT
+
+const Instr kLwRegFpOffsetPattern =
+ LW | (fp.code() << kRsShift) | (0 & kImm16Mask); // NOLINT
+
+const Instr kSwRegFpOffsetPattern =
+ SW | (fp.code() << kRsShift) | (0 & kImm16Mask); // NOLINT
+
+const Instr kLwRegFpNegOffsetPattern =
+ LW | (fp.code() << kRsShift) | (kNegOffset & kImm16Mask); // NOLINT
+
+const Instr kSwRegFpNegOffsetPattern =
+ SW | (fp.code() << kRsShift) | (kNegOffset & kImm16Mask); // NOLINT
+// A mask for the Rt register for push, pop, lw, sw instructions.
+const Instr kRtMask = kRtFieldMask;
+const Instr kLwSwInstrTypeMask = 0xFFE00000;
+const Instr kLwSwInstrArgumentMask = ~kLwSwInstrTypeMask;
+const Instr kLwSwOffsetMask = kImm16Mask;
+
+Assembler::Assembler(const AssemblerOptions& options,
+ std::unique_ptr<AssemblerBuffer> buffer)
+ : AssemblerBase(options, std::move(buffer)),
+ scratch_register_list_(at.bit()) {
+ if (CpuFeatures::IsSupported(MIPS_SIMD)) {
+ EnableCpuFeature(MIPS_SIMD);
+ }
+ reloc_info_writer.Reposition(buffer_start_ + buffer_->size(), pc_);
+
+ last_trampoline_pool_end_ = 0;
+ no_trampoline_pool_before_ = 0;
+ trampoline_pool_blocked_nesting_ = 0;
+ // We leave space (16 * kTrampolineSlotsSize)
+ // for BlockTrampolinePoolScope buffer.
+ next_buffer_check_ = FLAG_force_long_branches
+ ? kMaxInt
+ : kMaxBranchOffset - kTrampolineSlotsSize * 16;
+ internal_trampoline_exception_ = false;
+ last_bound_pos_ = 0;
+
+ trampoline_emitted_ = FLAG_force_long_branches;
+ unbound_labels_count_ = 0;
+ block_buffer_growth_ = false;
+}
+
+void Assembler::GetCode(Isolate* isolate, CodeDesc* desc,
+ SafepointTableBuilder* safepoint_table_builder,
+ int handler_table_offset) {
+ // As a crutch to avoid having to add manual Align calls wherever we use a
+ // raw workflow to create Code objects (mostly in tests), add another Align
+ // call here. It does no harm - the end of the Code object is aligned to the
+ // (larger) kCodeAlignment anyways.
+ // TODO(jgruber): Consider moving responsibility for proper alignment to
+ // metadata table builders (safepoint, handler, constant pool, code
+ // comments).
+ DataAlign(Code::kMetadataAlignment);
+
+ EmitForbiddenSlotInstruction();
+
+ int code_comments_size = WriteCodeComments();
+
+ DCHECK(pc_ <= reloc_info_writer.pos()); // No overlap.
+
+ AllocateAndInstallRequestedHeapObjects(isolate);
+
+ // Set up code descriptor.
+ // TODO(jgruber): Reconsider how these offsets and sizes are maintained up to
+ // this point to make CodeDesc initialization less fiddly.
+
+ static constexpr int kConstantPoolSize = 0;
+ const int instruction_size = pc_offset();
+ const int code_comments_offset = instruction_size - code_comments_size;
+ const int constant_pool_offset = code_comments_offset - kConstantPoolSize;
+ const int handler_table_offset2 = (handler_table_offset == kNoHandlerTable)
+ ? constant_pool_offset
+ : handler_table_offset;
+ const int safepoint_table_offset =
+ (safepoint_table_builder == kNoSafepointTable)
+ ? handler_table_offset2
+ : safepoint_table_builder->GetCodeOffset();
+ const int reloc_info_offset =
+ static_cast<int>(reloc_info_writer.pos() - buffer_->start());
+ CodeDesc::Initialize(desc, this, safepoint_table_offset,
+ handler_table_offset2, constant_pool_offset,
+ code_comments_offset, reloc_info_offset);
+}
+
+void Assembler::Align(int m) {
+ DCHECK(m >= 4 && base::bits::IsPowerOfTwo(m));
+ EmitForbiddenSlotInstruction();
+ while ((pc_offset() & (m - 1)) != 0) {
+ nop();
+ }
+}
+
+void Assembler::CodeTargetAlign() {
+ // No advantage to aligning branch/call targets to more than
+ // single instruction, that I am aware of.
+ Align(4);
+}
+
+Register Assembler::GetRtReg(Instr instr) {
+ return Register::from_code((instr & kRtFieldMask) >> kRtShift);
+}
+
+Register Assembler::GetRsReg(Instr instr) {
+ return Register::from_code((instr & kRsFieldMask) >> kRsShift);
+}
+
+Register Assembler::GetRdReg(Instr instr) {
+ return Register::from_code((instr & kRdFieldMask) >> kRdShift);
+}
+
+uint32_t Assembler::GetRt(Instr instr) {
+ return (instr & kRtFieldMask) >> kRtShift;
+}
+
+uint32_t Assembler::GetRtField(Instr instr) { return instr & kRtFieldMask; }
+
+uint32_t Assembler::GetRs(Instr instr) {
+ return (instr & kRsFieldMask) >> kRsShift;
+}
+
+uint32_t Assembler::GetRsField(Instr instr) { return instr & kRsFieldMask; }
+
+uint32_t Assembler::GetRd(Instr instr) {
+ return (instr & kRdFieldMask) >> kRdShift;
+}
+
+uint32_t Assembler::GetRdField(Instr instr) { return instr & kRdFieldMask; }
+
+uint32_t Assembler::GetSa(Instr instr) {
+ return (instr & kSaFieldMask) >> kSaShift;
+}
+
+uint32_t Assembler::GetSaField(Instr instr) { return instr & kSaFieldMask; }
+
+uint32_t Assembler::GetOpcodeField(Instr instr) { return instr & kOpcodeMask; }
+
+uint32_t Assembler::GetFunction(Instr instr) {
+ return (instr & kFunctionFieldMask) >> kFunctionShift;
+}
+
+uint32_t Assembler::GetFunctionField(Instr instr) {
+ return instr & kFunctionFieldMask;
+}
+
+uint32_t Assembler::GetImmediate16(Instr instr) { return instr & kImm16Mask; }
+
+uint32_t Assembler::GetLabelConst(Instr instr) { return instr & ~kImm16Mask; }
+
+bool Assembler::IsPop(Instr instr) {
+ return (instr & ~kRtMask) == kPopRegPattern;
+}
+
+bool Assembler::IsPush(Instr instr) {
+ return (instr & ~kRtMask) == kPushRegPattern;
+}
+
+bool Assembler::IsSwRegFpOffset(Instr instr) {
+ return ((instr & kLwSwInstrTypeMask) == kSwRegFpOffsetPattern);
+}
+
+bool Assembler::IsLwRegFpOffset(Instr instr) {
+ return ((instr & kLwSwInstrTypeMask) == kLwRegFpOffsetPattern);
+}
+
+bool Assembler::IsSwRegFpNegOffset(Instr instr) {
+ return ((instr & (kLwSwInstrTypeMask | kNegOffset)) ==
+ kSwRegFpNegOffsetPattern);
+}
+
+bool Assembler::IsLwRegFpNegOffset(Instr instr) {
+ return ((instr & (kLwSwInstrTypeMask | kNegOffset)) ==
+ kLwRegFpNegOffsetPattern);
+}
+
+// Labels refer to positions in the (to be) generated code.
+// There are bound, linked, and unused labels.
+//
+// Bound labels refer to known positions in the already
+// generated code. pos() is the position the label refers to.
+//
+// Linked labels refer to unknown positions in the code
+// to be generated; pos() is the position of the last
+// instruction using the label.
+
+// The link chain is terminated by a value in the instruction of -1,
+// which is an otherwise illegal value (branch -1 is inf loop).
+// The instruction 16-bit offset field addresses 32-bit words, but in
+// code is conv to an 18-bit value addressing bytes, hence the -4 value.
+
+const int kEndOfChain = -4;
+// Determines the end of the Jump chain (a subset of the label link chain).
+const int kEndOfJumpChain = 0;
+
+bool Assembler::IsMsaBranch(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ uint32_t rs_field = GetRsField(instr);
+ if (opcode == COP1) {
+ switch (rs_field) {
+ case BZ_V:
+ case BZ_B:
+ case BZ_H:
+ case BZ_W:
+ case BZ_D:
+ case BNZ_V:
+ case BNZ_B:
+ case BNZ_H:
+ case BNZ_W:
+ case BNZ_D:
+ return true;
+ default:
+ return false;
+ }
+ } else {
+ return false;
+ }
+}
+
+bool Assembler::IsBranch(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ uint32_t rt_field = GetRtField(instr);
+ uint32_t rs_field = GetRsField(instr);
+ // Checks if the instruction is a branch.
+ bool isBranch =
+ opcode == BEQ || opcode == BNE || opcode == BLEZ || opcode == BGTZ ||
+ opcode == BEQL || opcode == BNEL || opcode == BLEZL || opcode == BGTZL ||
+ (opcode == REGIMM && (rt_field == BLTZ || rt_field == BGEZ ||
+ rt_field == BLTZAL || rt_field == BGEZAL)) ||
+ (opcode == COP1 && rs_field == BC1) || // Coprocessor branch.
+ (opcode == COP1 && rs_field == BC1EQZ) ||
+ (opcode == COP1 && rs_field == BC1NEZ) || IsMsaBranch(instr);
+ if (!isBranch && kArchVariant == kMips64r6) {
+ // All the 3 variants of POP10 (BOVC, BEQC, BEQZALC) and
+ // POP30 (BNVC, BNEC, BNEZALC) are branch ops.
+ isBranch |= opcode == POP10 || opcode == POP30 || opcode == BC ||
+ opcode == BALC ||
+ (opcode == POP66 && rs_field != 0) || // BEQZC
+ (opcode == POP76 && rs_field != 0); // BNEZC
+ }
+ return isBranch;
+}
+
+bool Assembler::IsBc(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ // Checks if the instruction is a BC or BALC.
+ return opcode == BC || opcode == BALC;
+}
+
+bool Assembler::IsNal(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ uint32_t rt_field = GetRtField(instr);
+ uint32_t rs_field = GetRsField(instr);
+ return opcode == REGIMM && rt_field == BLTZAL && rs_field == 0;
+}
+
+bool Assembler::IsBzc(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ // Checks if the instruction is BEQZC or BNEZC.
+ return (opcode == POP66 && GetRsField(instr) != 0) ||
+ (opcode == POP76 && GetRsField(instr) != 0);
+}
+
+bool Assembler::IsEmittedConstant(Instr instr) {
+ uint32_t label_constant = GetLabelConst(instr);
+ return label_constant == 0; // Emitted label const in reg-exp engine.
+}
+
+bool Assembler::IsBeq(Instr instr) { return GetOpcodeField(instr) == BEQ; }
+
+bool Assembler::IsBne(Instr instr) { return GetOpcodeField(instr) == BNE; }
+
+bool Assembler::IsBeqzc(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ return opcode == POP66 && GetRsField(instr) != 0;
+}
+
+bool Assembler::IsBnezc(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ return opcode == POP76 && GetRsField(instr) != 0;
+}
+
+bool Assembler::IsBeqc(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ uint32_t rs = GetRsField(instr);
+ uint32_t rt = GetRtField(instr);
+ return opcode == POP10 && rs != 0 && rs < rt; // && rt != 0
+}
+
+bool Assembler::IsBnec(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ uint32_t rs = GetRsField(instr);
+ uint32_t rt = GetRtField(instr);
+ return opcode == POP30 && rs != 0 && rs < rt; // && rt != 0
+}
+
+bool Assembler::IsMov(Instr instr, Register rd, Register rs) {
+ uint32_t opcode = GetOpcodeField(instr);
+ uint32_t rd_field = GetRd(instr);
+ uint32_t rs_field = GetRs(instr);
+ uint32_t rt_field = GetRt(instr);
+ uint32_t rd_reg = static_cast<uint32_t>(rd.code());
+ uint32_t rs_reg = static_cast<uint32_t>(rs.code());
+ uint32_t function_field = GetFunctionField(instr);
+ // Checks if the instruction is a OR with zero_reg argument (aka MOV).
+ bool res = opcode == SPECIAL && function_field == OR && rd_field == rd_reg &&
+ rs_field == rs_reg && rt_field == 0;
+ return res;
+}
+
+bool Assembler::IsJump(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ uint32_t rt_field = GetRtField(instr);
+ uint32_t rd_field = GetRdField(instr);
+ uint32_t function_field = GetFunctionField(instr);
+ // Checks if the instruction is a jump.
+ return opcode == J || opcode == JAL ||
+ (opcode == SPECIAL && rt_field == 0 &&
+ ((function_field == JALR) ||
+ (rd_field == 0 && (function_field == JR))));
+}
+
+bool Assembler::IsJ(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ // Checks if the instruction is a jump.
+ return opcode == J;
+}
+
+bool Assembler::IsJal(Instr instr) { return GetOpcodeField(instr) == JAL; }
+
+bool Assembler::IsJr(Instr instr) {
+ return GetOpcodeField(instr) == SPECIAL && GetFunctionField(instr) == JR;
+}
+
+bool Assembler::IsJalr(Instr instr) {
+ return GetOpcodeField(instr) == SPECIAL && GetFunctionField(instr) == JALR;
+}
+
+bool Assembler::IsLui(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ // Checks if the instruction is a load upper immediate.
+ return opcode == LUI;
+}
+
+bool Assembler::IsOri(Instr instr) {
+ uint32_t opcode = GetOpcodeField(instr);
+ // Checks if the instruction is a load upper immediate.
+ return opcode == ORI;
+}
+
+bool Assembler::IsNop(Instr instr, unsigned int type) {
+ // See Assembler::nop(type).
+ DCHECK_LT(type, 32);
+ uint32_t opcode = GetOpcodeField(instr);
+ uint32_t function = GetFunctionField(instr);
+ uint32_t rt = GetRt(instr);
+ uint32_t rd = GetRd(instr);
+ uint32_t sa = GetSa(instr);
+
+ // Traditional mips nop == sll(zero_reg, zero_reg, 0)
+ // When marking non-zero type, use sll(zero_reg, at, type)
+ // to avoid use of mips ssnop and ehb special encodings
+ // of the sll instruction.
+
+ Register nop_rt_reg = (type == 0) ? zero_reg : at;
+ bool ret = (opcode == SPECIAL && function == SLL &&
+ rd == static_cast<uint32_t>(ToNumber(zero_reg)) &&
+ rt == static_cast<uint32_t>(ToNumber(nop_rt_reg)) && sa == type);
+
+ return ret;
+}
+
+int32_t Assembler::GetBranchOffset(Instr instr) {
+ DCHECK(IsBranch(instr));
+ return (static_cast<int16_t>(instr & kImm16Mask)) << 2;
+}
+
+bool Assembler::IsLw(Instr instr) {
+ return (static_cast<uint32_t>(instr & kOpcodeMask) == LW);
+}
+
+int16_t Assembler::GetLwOffset(Instr instr) {
+ DCHECK(IsLw(instr));
+ return ((instr & kImm16Mask));
+}
+
+Instr Assembler::SetLwOffset(Instr instr, int16_t offset) {
+ DCHECK(IsLw(instr));
+
+ // We actually create a new lw instruction based on the original one.
+ Instr temp_instr = LW | (instr & kRsFieldMask) | (instr & kRtFieldMask) |
+ (offset & kImm16Mask);
+
+ return temp_instr;
+}
+
+bool Assembler::IsSw(Instr instr) {
+ return (static_cast<uint32_t>(instr & kOpcodeMask) == SW);
+}
+
+Instr Assembler::SetSwOffset(Instr instr, int16_t offset) {
+ DCHECK(IsSw(instr));
+ return ((instr & ~kImm16Mask) | (offset & kImm16Mask));
+}
+
+bool Assembler::IsAddImmediate(Instr instr) {
+ return ((instr & kOpcodeMask) == ADDIU || (instr & kOpcodeMask) == DADDIU);
+}
+
+Instr Assembler::SetAddImmediateOffset(Instr instr, int16_t offset) {
+ DCHECK(IsAddImmediate(instr));
+ return ((instr & ~kImm16Mask) | (offset & kImm16Mask));
+}
+
+bool Assembler::IsAndImmediate(Instr instr) {
+ return GetOpcodeField(instr) == ANDI;
+}
+
+static Assembler::OffsetSize OffsetSizeInBits(Instr instr) {
+ if (kArchVariant == kMips64r6) {
+ if (Assembler::IsBc(instr)) {
+ return Assembler::OffsetSize::kOffset26;
+ } else if (Assembler::IsBzc(instr)) {
+ return Assembler::OffsetSize::kOffset21;
+ }
+ }
+ return Assembler::OffsetSize::kOffset16;
+}
+
+static inline int32_t AddBranchOffset(int pos, Instr instr) {
+ int bits = OffsetSizeInBits(instr);
+ const int32_t mask = (1 << bits) - 1;
+ bits = 32 - bits;
+
+ // Do NOT change this to <<2. We rely on arithmetic shifts here, assuming
+ // the compiler uses arithmetic shifts for signed integers.
+ int32_t imm = ((instr & mask) << bits) >> (bits - 2);
+
+ if (imm == kEndOfChain) {
+ // EndOfChain sentinel is returned directly, not relative to pc or pos.
+ return kEndOfChain;
+ } else {
+ return pos + Assembler::kBranchPCOffset + imm;
+ }
+}
+
+int Assembler::target_at(int pos, bool is_internal) {
+ if (is_internal) {
+ int64_t* p = reinterpret_cast<int64_t*>(buffer_start_ + pos);
+ int64_t address = *p;
+ if (address == kEndOfJumpChain) {
+ return kEndOfChain;
+ } else {
+ int64_t instr_address = reinterpret_cast<int64_t>(p);
+ DCHECK(instr_address - address < INT_MAX);
+ int delta = static_cast<int>(instr_address - address);
+ DCHECK(pos > delta);
+ return pos - delta;
+ }
+ }
+ Instr instr = instr_at(pos);
+ if ((instr & ~kImm16Mask) == 0) {
+ // Emitted label constant, not part of a branch.
+ if (instr == 0) {
+ return kEndOfChain;
+ } else {
+ int32_t imm18 = ((instr & static_cast<int32_t>(kImm16Mask)) << 16) >> 14;
+ return (imm18 + pos);
+ }
+ }
+ // Check we have a branch or jump instruction.
+ DCHECK(IsBranch(instr) || IsJ(instr) || IsJal(instr) || IsLui(instr) ||
+ IsMov(instr, t8, ra));
+ // Do NOT change this to <<2. We rely on arithmetic shifts here, assuming
+ // the compiler uses arithmetic shifts for signed integers.
+ if (IsBranch(instr)) {
+ return AddBranchOffset(pos, instr);
+ } else if (IsMov(instr, t8, ra)) {
+ int32_t imm32;
+ Instr instr_lui = instr_at(pos + 2 * kInstrSize);
+ Instr instr_ori = instr_at(pos + 3 * kInstrSize);
+ DCHECK(IsLui(instr_lui));
+ DCHECK(IsOri(instr_ori));
+ imm32 = (instr_lui & static_cast<int32_t>(kImm16Mask)) << kLuiShift;
+ imm32 |= (instr_ori & static_cast<int32_t>(kImm16Mask));
+ if (imm32 == kEndOfJumpChain) {
+ // EndOfChain sentinel is returned directly, not relative to pc or pos.
+ return kEndOfChain;
+ }
+ return pos + Assembler::kLongBranchPCOffset + imm32;
+ } else if (IsLui(instr)) {
+ if (IsNal(instr_at(pos + kInstrSize))) {
+ int32_t imm32;
+ Instr instr_lui = instr_at(pos + 0 * kInstrSize);
+ Instr instr_ori = instr_at(pos + 2 * kInstrSize);
+ DCHECK(IsLui(instr_lui));
+ DCHECK(IsOri(instr_ori));
+ imm32 = (instr_lui & static_cast<int32_t>(kImm16Mask)) << kLuiShift;
+ imm32 |= (instr_ori & static_cast<int32_t>(kImm16Mask));
+ if (imm32 == kEndOfJumpChain) {
+ // EndOfChain sentinel is returned directly, not relative to pc or pos.
+ return kEndOfChain;
+ }
+ return pos + Assembler::kLongBranchPCOffset + imm32;
+ } else {
+ Instr instr_lui = instr_at(pos + 0 * kInstrSize);
+ Instr instr_ori = instr_at(pos + 1 * kInstrSize);
+ Instr instr_ori2 = instr_at(pos + 3 * kInstrSize);
+ DCHECK(IsOri(instr_ori));
+ DCHECK(IsOri(instr_ori2));
+
+ // TODO(plind) create named constants for shift values.
+ int64_t imm = static_cast<int64_t>(instr_lui & kImm16Mask) << 48;
+ imm |= static_cast<int64_t>(instr_ori & kImm16Mask) << 32;
+ imm |= static_cast<int64_t>(instr_ori2 & kImm16Mask) << 16;
+ // Sign extend address;
+ imm >>= 16;
+
+ if (imm == kEndOfJumpChain) {
+ // EndOfChain sentinel is returned directly, not relative to pc or pos.
+ return kEndOfChain;
+ } else {
+ uint64_t instr_address = reinterpret_cast<int64_t>(buffer_start_ + pos);
+ DCHECK(instr_address - imm < INT_MAX);
+ int delta = static_cast<int>(instr_address - imm);
+ DCHECK(pos > delta);
+ return pos - delta;
+ }
+ }
+ } else {
+ DCHECK(IsJ(instr) || IsJal(instr));
+ int32_t imm28 = (instr & static_cast<int32_t>(kImm26Mask)) << 2;
+ if (imm28 == kEndOfJumpChain) {
+ // EndOfChain sentinel is returned directly, not relative to pc or pos.
+ return kEndOfChain;
+ } else {
+ // Sign extend 28-bit offset.
+ int32_t delta = static_cast<int32_t>((imm28 << 4) >> 4);
+ return pos + delta;
+ }
+ }
+}
+
+static inline Instr SetBranchOffset(int32_t pos, int32_t target_pos,
+ Instr instr) {
+ int32_t bits = OffsetSizeInBits(instr);
+ int32_t imm = target_pos - (pos + Assembler::kBranchPCOffset);
+ DCHECK_EQ(imm & 3, 0);
+ imm >>= 2;
+
+ const int32_t mask = (1 << bits) - 1;
+ instr &= ~mask;
+ DCHECK(is_intn(imm, bits));
+
+ return instr | (imm & mask);
+}
+
+void Assembler::target_at_put(int pos, int target_pos, bool is_internal) {
+ if (is_internal) {
+ uint64_t imm = reinterpret_cast<uint64_t>(buffer_start_) + target_pos;
+ *reinterpret_cast<uint64_t*>(buffer_start_ + pos) = imm;
+ return;
+ }
+ Instr instr = instr_at(pos);
+ if ((instr & ~kImm16Mask) == 0) {
+ DCHECK(target_pos == kEndOfChain || target_pos >= 0);
+ // Emitted label constant, not part of a branch.
+ // Make label relative to Code pointer of generated Code object.
+ instr_at_put(pos, target_pos + (Code::kHeaderSize - kHeapObjectTag));
+ return;
+ }
+
+ if (IsBranch(instr)) {
+ instr = SetBranchOffset(pos, target_pos, instr);
+ instr_at_put(pos, instr);
+ } else if (IsLui(instr)) {
+ if (IsNal(instr_at(pos + kInstrSize))) {
+ Instr instr_lui = instr_at(pos + 0 * kInstrSize);
+ Instr instr_ori = instr_at(pos + 2 * kInstrSize);
+ DCHECK(IsLui(instr_lui));
+ DCHECK(IsOri(instr_ori));
+ int32_t imm = target_pos - (pos + Assembler::kLongBranchPCOffset);
+ DCHECK_EQ(imm & 3, 0);
+ if (is_int16(imm + Assembler::kLongBranchPCOffset -
+ Assembler::kBranchPCOffset)) {
+ // Optimize by converting to regular branch and link with 16-bit
+ // offset.
+ Instr instr_b = REGIMM | BGEZAL; // Branch and link.
+ instr_b = SetBranchOffset(pos, target_pos, instr_b);
+ // Correct ra register to point to one instruction after jalr from
+ // TurboAssembler::BranchAndLinkLong.
+ Instr instr_a = DADDIU | ra.code() << kRsShift | ra.code() << kRtShift |
+ kOptimizedBranchAndLinkLongReturnOffset;
+
+ instr_at_put(pos, instr_b);
+ instr_at_put(pos + 1 * kInstrSize, instr_a);
+ } else {
+ instr_lui &= ~kImm16Mask;
+ instr_ori &= ~kImm16Mask;
+
+ instr_at_put(pos + 0 * kInstrSize,
+ instr_lui | ((imm >> kLuiShift) & kImm16Mask));
+ instr_at_put(pos + 2 * kInstrSize, instr_ori | (imm & kImm16Mask));
+ }
+ } else {
+ Instr instr_lui = instr_at(pos + 0 * kInstrSize);
+ Instr instr_ori = instr_at(pos + 1 * kInstrSize);
+ Instr instr_ori2 = instr_at(pos + 3 * kInstrSize);
+ DCHECK(IsOri(instr_ori));
+ DCHECK(IsOri(instr_ori2));
+
+ uint64_t imm = reinterpret_cast<uint64_t>(buffer_start_) + target_pos;
+ DCHECK_EQ(imm & 3, 0);
+
+ instr_lui &= ~kImm16Mask;
+ instr_ori &= ~kImm16Mask;
+ instr_ori2 &= ~kImm16Mask;
+
+ instr_at_put(pos + 0 * kInstrSize,
+ instr_lui | ((imm >> 32) & kImm16Mask));
+ instr_at_put(pos + 1 * kInstrSize,
+ instr_ori | ((imm >> 16) & kImm16Mask));
+ instr_at_put(pos + 3 * kInstrSize, instr_ori2 | (imm & kImm16Mask));
+ }
+ } else if (IsMov(instr, t8, ra)) {
+ Instr instr_lui = instr_at(pos + 2 * kInstrSize);
+ Instr instr_ori = instr_at(pos + 3 * kInstrSize);
+ DCHECK(IsLui(instr_lui));
+ DCHECK(IsOri(instr_ori));
+
+ int32_t imm_short = target_pos - (pos + Assembler::kBranchPCOffset);
+
+ if (is_int16(imm_short)) {
+ // Optimize by converting to regular branch with 16-bit
+ // offset
+ Instr instr_b = BEQ;
+ instr_b = SetBranchOffset(pos, target_pos, instr_b);
+
+ Instr instr_j = instr_at(pos + 5 * kInstrSize);
+ Instr instr_branch_delay;
+
+ if (IsJump(instr_j)) {
+ // Case when branch delay slot is protected.
+ instr_branch_delay = nopInstr;
+ } else {
+ // Case when branch delay slot is used.
+ instr_branch_delay = instr_at(pos + 7 * kInstrSize);
+ }
+ instr_at_put(pos, instr_b);
+ instr_at_put(pos + 1 * kInstrSize, instr_branch_delay);
+ } else {
+ int32_t imm = target_pos - (pos + Assembler::kLongBranchPCOffset);
+ DCHECK_EQ(imm & 3, 0);
+
+ instr_lui &= ~kImm16Mask;
+ instr_ori &= ~kImm16Mask;
+
+ instr_at_put(pos + 2 * kInstrSize,
+ instr_lui | ((imm >> kLuiShift) & kImm16Mask));
+ instr_at_put(pos + 3 * kInstrSize, instr_ori | (imm & kImm16Mask));
+ }
+ } else if (IsJ(instr) || IsJal(instr)) {
+ int32_t imm28 = target_pos - pos;
+ DCHECK_EQ(imm28 & 3, 0);
+
+ uint32_t imm26 = static_cast<uint32_t>(imm28 >> 2);
+ DCHECK(is_uint26(imm26));
+ // Place 26-bit signed offset with markings.
+ // When code is committed it will be resolved to j/jal.
+ int32_t mark = IsJ(instr) ? kJRawMark : kJalRawMark;
+ instr_at_put(pos, mark | (imm26 & kImm26Mask));
+ } else {
+ int32_t imm28 = target_pos - pos;
+ DCHECK_EQ(imm28 & 3, 0);
+
+ uint32_t imm26 = static_cast<uint32_t>(imm28 >> 2);
+ DCHECK(is_uint26(imm26));
+ // Place raw 26-bit signed offset.
+ // When code is committed it will be resolved to j/jal.
+ instr &= ~kImm26Mask;
+ instr_at_put(pos, instr | (imm26 & kImm26Mask));
+ }
+}
+
+void Assembler::print(const Label* L) {
+ if (L->is_unused()) {
+ PrintF("unused label\n");
+ } else if (L->is_bound()) {
+ PrintF("bound label to %d\n", L->pos());
+ } else if (L->is_linked()) {
+ Label l;
+ l.link_to(L->pos());
+ PrintF("unbound label");
+ while (l.is_linked()) {
+ PrintF("@ %d ", l.pos());
+ Instr instr = instr_at(l.pos());
+ if ((instr & ~kImm16Mask) == 0) {
+ PrintF("value\n");
+ } else {
+ PrintF("%d\n", instr);
+ }
+ next(&l, is_internal_reference(&l));
+ }
+ } else {
+ PrintF("label in inconsistent state (pos = %d)\n", L->pos_);
+ }
+}
+
+void Assembler::bind_to(Label* L, int pos) {
+ DCHECK(0 <= pos && pos <= pc_offset()); // Must have valid binding position.
+ int trampoline_pos = kInvalidSlotPos;
+ bool is_internal = false;
+ if (L->is_linked() && !trampoline_emitted_) {
+ unbound_labels_count_--;
+ if (!is_internal_reference(L)) {
+ next_buffer_check_ += kTrampolineSlotsSize;
+ }
+ }
+
+ while (L->is_linked()) {
+ int fixup_pos = L->pos();
+ int dist = pos - fixup_pos;
+ is_internal = is_internal_reference(L);
+ next(L, is_internal); // Call next before overwriting link with target at
+ // fixup_pos.
+ Instr instr = instr_at(fixup_pos);
+ if (is_internal) {
+ target_at_put(fixup_pos, pos, is_internal);
+ } else {
+ if (IsBranch(instr)) {
+ int branch_offset = BranchOffset(instr);
+ if (dist > branch_offset) {
+ if (trampoline_pos == kInvalidSlotPos) {
+ trampoline_pos = get_trampoline_entry(fixup_pos);
+ CHECK_NE(trampoline_pos, kInvalidSlotPos);
+ }
+ CHECK((trampoline_pos - fixup_pos) <= branch_offset);
+ target_at_put(fixup_pos, trampoline_pos, false);
+ fixup_pos = trampoline_pos;
+ }
+ target_at_put(fixup_pos, pos, false);
+ } else {
+ DCHECK(IsJ(instr) || IsJal(instr) || IsLui(instr) ||
+ IsEmittedConstant(instr) || IsMov(instr, t8, ra));
+ target_at_put(fixup_pos, pos, false);
+ }
+ }
+ }
+ L->bind_to(pos);
+
+ // Keep track of the last bound label so we don't eliminate any instructions
+ // before a bound label.
+ if (pos > last_bound_pos_) last_bound_pos_ = pos;
+}
+
+void Assembler::bind(Label* L) {
+ DCHECK(!L->is_bound()); // Label can only be bound once.
+ bind_to(L, pc_offset());
+}
+
+void Assembler::next(Label* L, bool is_internal) {
+ DCHECK(L->is_linked());
+ int link = target_at(L->pos(), is_internal);
+ if (link == kEndOfChain) {
+ L->Unuse();
+ } else {
+ DCHECK_GE(link, 0);
+ L->link_to(link);
+ }
+}
+
+bool Assembler::is_near(Label* L) {
+ DCHECK(L->is_bound());
+ return pc_offset() - L->pos() < kMaxBranchOffset - 4 * kInstrSize;
+}
+
+bool Assembler::is_near(Label* L, OffsetSize bits) {
+ if (L == nullptr || !L->is_bound()) return true;
+ return ((pc_offset() - L->pos()) <
+ (1 << (bits + 2 - 1)) - 1 - 5 * kInstrSize);
+}
+
+bool Assembler::is_near_branch(Label* L) {
+ DCHECK(L->is_bound());
+ return kArchVariant == kMips64r6 ? is_near_r6(L) : is_near_pre_r6(L);
+}
+
+int Assembler::BranchOffset(Instr instr) {
+ // At pre-R6 and for other R6 branches the offset is 16 bits.
+ int bits = OffsetSize::kOffset16;
+
+ if (kArchVariant == kMips64r6) {
+ uint32_t opcode = GetOpcodeField(instr);
+ switch (opcode) {
+ // Checks BC or BALC.
+ case BC:
+ case BALC:
+ bits = OffsetSize::kOffset26;
+ break;
+
+ // Checks BEQZC or BNEZC.
+ case POP66:
+ case POP76:
+ if (GetRsField(instr) != 0) bits = OffsetSize::kOffset21;
+ break;
+ default:
+ break;
+ }
+ }
+
+ return (1 << (bits + 2 - 1)) - 1;
+}
+
+// We have to use a temporary register for things that can be relocated even
+// if they can be encoded in the MIPS's 16 bits of immediate-offset instruction
+// space. There is no guarantee that the relocated location can be similarly
+// encoded.
+bool Assembler::MustUseReg(RelocInfo::Mode rmode) {
+ return !RelocInfo::IsNone(rmode);
+}
+
+void Assembler::GenInstrRegister(Opcode opcode, Register rs, Register rt,
+ Register rd, uint16_t sa,
+ SecondaryField func) {
+ DCHECK(rd.is_valid() && rs.is_valid() && rt.is_valid() && is_uint5(sa));
+ Instr instr = opcode | (rs.code() << kRsShift) | (rt.code() << kRtShift) |
+ (rd.code() << kRdShift) | (sa << kSaShift) | func;
+ emit(instr);
+}
+
+void Assembler::GenInstrRegister(Opcode opcode, Register rs, Register rt,
+ uint16_t msb, uint16_t lsb,
+ SecondaryField func) {
+ DCHECK(rs.is_valid() && rt.is_valid() && is_uint5(msb) && is_uint5(lsb));
+ Instr instr = opcode | (rs.code() << kRsShift) | (rt.code() << kRtShift) |
+ (msb << kRdShift) | (lsb << kSaShift) | func;
+ emit(instr);
+}
+
+void Assembler::GenInstrRegister(Opcode opcode, SecondaryField fmt,
+ FPURegister ft, FPURegister fs, FPURegister fd,
+ SecondaryField func) {
+ DCHECK(fd.is_valid() && fs.is_valid() && ft.is_valid());
+ Instr instr = opcode | fmt | (ft.code() << kFtShift) |
+ (fs.code() << kFsShift) | (fd.code() << kFdShift) | func;
+ emit(instr);
+}
+
+void Assembler::GenInstrRegister(Opcode opcode, FPURegister fr, FPURegister ft,
+ FPURegister fs, FPURegister fd,
+ SecondaryField func) {
+ DCHECK(fd.is_valid() && fr.is_valid() && fs.is_valid() && ft.is_valid());
+ Instr instr = opcode | (fr.code() << kFrShift) | (ft.code() << kFtShift) |
+ (fs.code() << kFsShift) | (fd.code() << kFdShift) | func;
+ emit(instr);
+}
+
+void Assembler::GenInstrRegister(Opcode opcode, SecondaryField fmt, Register rt,
+ FPURegister fs, FPURegister fd,
+ SecondaryField func) {
+ DCHECK(fd.is_valid() && fs.is_valid() && rt.is_valid());
+ Instr instr = opcode | fmt | (rt.code() << kRtShift) |
+ (fs.code() << kFsShift) | (fd.code() << kFdShift) | func;
+ emit(instr);
+}
+
+void Assembler::GenInstrRegister(Opcode opcode, SecondaryField fmt, Register rt,
+ FPUControlRegister fs, SecondaryField func) {
+ DCHECK(fs.is_valid() && rt.is_valid());
+ Instr instr =
+ opcode | fmt | (rt.code() << kRtShift) | (fs.code() << kFsShift) | func;
+ emit(instr);
+}
+
+// Instructions with immediate value.
+// Registers are in the order of the instruction encoding, from left to right.
+void Assembler::GenInstrImmediate(Opcode opcode, Register rs, Register rt,
+ int32_t j,
+ CompactBranchType is_compact_branch) {
+ DCHECK(rs.is_valid() && rt.is_valid() && (is_int16(j) || is_uint16(j)));
+ Instr instr = opcode | (rs.code() << kRsShift) | (rt.code() << kRtShift) |
+ (j & kImm16Mask);
+ emit(instr, is_compact_branch);
+}
+
+void Assembler::GenInstrImmediate(Opcode opcode, Register base, Register rt,
+ int32_t offset9, int bit6,
+ SecondaryField func) {
+ DCHECK(base.is_valid() && rt.is_valid() && is_int9(offset9) &&
+ is_uint1(bit6));
+ Instr instr = opcode | (base.code() << kBaseShift) | (rt.code() << kRtShift) |
+ ((offset9 << kImm9Shift) & kImm9Mask) | bit6 << kBit6Shift |
+ func;
+ emit(instr);
+}
+
+void Assembler::GenInstrImmediate(Opcode opcode, Register rs, SecondaryField SF,
+ int32_t j,
+ CompactBranchType is_compact_branch) {
+ DCHECK(rs.is_valid() && (is_int16(j) || is_uint16(j)));
+ Instr instr = opcode | (rs.code() << kRsShift) | SF | (j & kImm16Mask);
+ emit(instr, is_compact_branch);
+}
+
+void Assembler::GenInstrImmediate(Opcode opcode, Register rs, FPURegister ft,
+ int32_t j,
+ CompactBranchType is_compact_branch) {
+ DCHECK(rs.is_valid() && ft.is_valid() && (is_int16(j) || is_uint16(j)));
+ Instr instr = opcode | (rs.code() << kRsShift) | (ft.code() << kFtShift) |
+ (j & kImm16Mask);
+ emit(instr, is_compact_branch);
+}
+
+void Assembler::GenInstrImmediate(Opcode opcode, Register rs, int32_t offset21,
+ CompactBranchType is_compact_branch) {
+ DCHECK(rs.is_valid() && (is_int21(offset21)));
+ Instr instr = opcode | (rs.code() << kRsShift) | (offset21 & kImm21Mask);
+ emit(instr, is_compact_branch);
+}
+
+void Assembler::GenInstrImmediate(Opcode opcode, Register rs,
+ uint32_t offset21) {
+ DCHECK(rs.is_valid() && (is_uint21(offset21)));
+ Instr instr = opcode | (rs.code() << kRsShift) | (offset21 & kImm21Mask);
+ emit(instr);
+}
+
+void Assembler::GenInstrImmediate(Opcode opcode, int32_t offset26,
+ CompactBranchType is_compact_branch) {
+ DCHECK(is_int26(offset26));
+ Instr instr = opcode | (offset26 & kImm26Mask);
+ emit(instr, is_compact_branch);
+}
+
+void Assembler::GenInstrJump(Opcode opcode, uint32_t address) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ DCHECK(is_uint26(address));
+ Instr instr = opcode | address;
+ emit(instr);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+// MSA instructions
+void Assembler::GenInstrMsaI8(SecondaryField operation, uint32_t imm8,
+ MSARegister ws, MSARegister wd) {
+ DCHECK(IsEnabled(MIPS_SIMD));
+ DCHECK(ws.is_valid() && wd.is_valid() && is_uint8(imm8));
+ Instr instr = MSA | operation | ((imm8 & kImm8Mask) << kWtShift) |
+ (ws.code() << kWsShift) | (wd.code() << kWdShift);
+ emit(instr);
+}
+
+void Assembler::GenInstrMsaI5(SecondaryField operation, SecondaryField df,
+ int32_t imm5, MSARegister ws, MSARegister wd) {
+ DCHECK(IsEnabled(MIPS_SIMD));
+ DCHECK(ws.is_valid() && wd.is_valid());
+ DCHECK((operation == MAXI_S) || (operation == MINI_S) ||
+ (operation == CEQI) || (operation == CLTI_S) ||
+ (operation == CLEI_S)
+ ? is_int5(imm5)
+ : is_uint5(imm5));
+ Instr instr = MSA | operation | df | ((imm5 & kImm5Mask) << kWtShift) |
+ (ws.code() << kWsShift) | (wd.code() << kWdShift);
+ emit(instr);
+}
+
+void Assembler::GenInstrMsaBit(SecondaryField operation, SecondaryField df,
+ uint32_t m, MSARegister ws, MSARegister wd) {
+ DCHECK(IsEnabled(MIPS_SIMD));
+ DCHECK(ws.is_valid() && wd.is_valid() && is_valid_msa_df_m(df, m));
+ Instr instr = MSA | operation | df | (m << kWtShift) |
+ (ws.code() << kWsShift) | (wd.code() << kWdShift);
+ emit(instr);
+}
+
+void Assembler::GenInstrMsaI10(SecondaryField operation, SecondaryField df,
+ int32_t imm10, MSARegister wd) {
+ DCHECK(IsEnabled(MIPS_SIMD));
+ DCHECK(wd.is_valid() && is_int10(imm10));
+ Instr instr = MSA | operation | df | ((imm10 & kImm10Mask) << kWsShift) |
+ (wd.code() << kWdShift);
+ emit(instr);
+}
+
+template <typename RegType>
+void Assembler::GenInstrMsa3R(SecondaryField operation, SecondaryField df,
+ RegType t, MSARegister ws, MSARegister wd) {
+ DCHECK(IsEnabled(MIPS_SIMD));
+ DCHECK(t.is_valid() && ws.is_valid() && wd.is_valid());
+ Instr instr = MSA | operation | df | (t.code() << kWtShift) |
+ (ws.code() << kWsShift) | (wd.code() << kWdShift);
+ emit(instr);
+}
+
+template <typename DstType, typename SrcType>
+void Assembler::GenInstrMsaElm(SecondaryField operation, SecondaryField df,
+ uint32_t n, SrcType src, DstType dst) {
+ DCHECK(IsEnabled(MIPS_SIMD));
+ DCHECK(src.is_valid() && dst.is_valid() && is_valid_msa_df_n(df, n));
+ Instr instr = MSA | operation | df | (n << kWtShift) |
+ (src.code() << kWsShift) | (dst.code() << kWdShift) |
+ MSA_ELM_MINOR;
+ emit(instr);
+}
+
+void Assembler::GenInstrMsa3RF(SecondaryField operation, uint32_t df,
+ MSARegister wt, MSARegister ws, MSARegister wd) {
+ DCHECK(IsEnabled(MIPS_SIMD));
+ DCHECK(wt.is_valid() && ws.is_valid() && wd.is_valid());
+ DCHECK_LT(df, 2);
+ Instr instr = MSA | operation | (df << 21) | (wt.code() << kWtShift) |
+ (ws.code() << kWsShift) | (wd.code() << kWdShift);
+ emit(instr);
+}
+
+void Assembler::GenInstrMsaVec(SecondaryField operation, MSARegister wt,
+ MSARegister ws, MSARegister wd) {
+ DCHECK(IsEnabled(MIPS_SIMD));
+ DCHECK(wt.is_valid() && ws.is_valid() && wd.is_valid());
+ Instr instr = MSA | operation | (wt.code() << kWtShift) |
+ (ws.code() << kWsShift) | (wd.code() << kWdShift) |
+ MSA_VEC_2R_2RF_MINOR;
+ emit(instr);
+}
+
+void Assembler::GenInstrMsaMI10(SecondaryField operation, int32_t s10,
+ Register rs, MSARegister wd) {
+ DCHECK(IsEnabled(MIPS_SIMD));
+ DCHECK(rs.is_valid() && wd.is_valid() && is_int10(s10));
+ Instr instr = MSA | operation | ((s10 & kImm10Mask) << kWtShift) |
+ (rs.code() << kWsShift) | (wd.code() << kWdShift);
+ emit(instr);
+}
+
+void Assembler::GenInstrMsa2R(SecondaryField operation, SecondaryField df,
+ MSARegister ws, MSARegister wd) {
+ DCHECK(IsEnabled(MIPS_SIMD));
+ DCHECK(ws.is_valid() && wd.is_valid());
+ Instr instr = MSA | MSA_2R_FORMAT | operation | df | (ws.code() << kWsShift) |
+ (wd.code() << kWdShift) | MSA_VEC_2R_2RF_MINOR;
+ emit(instr);
+}
+
+void Assembler::GenInstrMsa2RF(SecondaryField operation, SecondaryField df,
+ MSARegister ws, MSARegister wd) {
+ DCHECK(IsEnabled(MIPS_SIMD));
+ DCHECK(ws.is_valid() && wd.is_valid());
+ Instr instr = MSA | MSA_2RF_FORMAT | operation | df |
+ (ws.code() << kWsShift) | (wd.code() << kWdShift) |
+ MSA_VEC_2R_2RF_MINOR;
+ emit(instr);
+}
+
+void Assembler::GenInstrMsaBranch(SecondaryField operation, MSARegister wt,
+ int32_t offset16) {
+ DCHECK(IsEnabled(MIPS_SIMD));
+ DCHECK(wt.is_valid() && is_int16(offset16));
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Instr instr =
+ COP1 | operation | (wt.code() << kWtShift) | (offset16 & kImm16Mask);
+ emit(instr);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+// Returns the next free trampoline entry.
+int32_t Assembler::get_trampoline_entry(int32_t pos) {
+ int32_t trampoline_entry = kInvalidSlotPos;
+ if (!internal_trampoline_exception_) {
+ if (trampoline_.start() > pos) {
+ trampoline_entry = trampoline_.take_slot();
+ }
+
+ if (kInvalidSlotPos == trampoline_entry) {
+ internal_trampoline_exception_ = true;
+ }
+ }
+ return trampoline_entry;
+}
+
+uint64_t Assembler::jump_address(Label* L) {
+ int64_t target_pos;
+ if (L->is_bound()) {
+ target_pos = L->pos();
+ } else {
+ if (L->is_linked()) {
+ target_pos = L->pos(); // L's link.
+ L->link_to(pc_offset());
+ } else {
+ L->link_to(pc_offset());
+ return kEndOfJumpChain;
+ }
+ }
+ uint64_t imm = reinterpret_cast<uint64_t>(buffer_start_) + target_pos;
+ DCHECK_EQ(imm & 3, 0);
+
+ return imm;
+}
+
+uint64_t Assembler::jump_offset(Label* L) {
+ int64_t target_pos;
+ int32_t pad = IsPrevInstrCompactBranch() ? kInstrSize : 0;
+
+ if (L->is_bound()) {
+ target_pos = L->pos();
+ } else {
+ if (L->is_linked()) {
+ target_pos = L->pos(); // L's link.
+ L->link_to(pc_offset() + pad);
+ } else {
+ L->link_to(pc_offset() + pad);
+ return kEndOfJumpChain;
+ }
+ }
+ int64_t imm = target_pos - (pc_offset() + pad);
+ DCHECK_EQ(imm & 3, 0);
+
+ return static_cast<uint64_t>(imm);
+}
+
+uint64_t Assembler::branch_long_offset(Label* L) {
+ int64_t target_pos;
+
+ if (L->is_bound()) {
+ target_pos = L->pos();
+ } else {
+ if (L->is_linked()) {
+ target_pos = L->pos(); // L's link.
+ L->link_to(pc_offset());
+ } else {
+ L->link_to(pc_offset());
+ return kEndOfJumpChain;
+ }
+ }
+ int64_t offset = target_pos - (pc_offset() + kLongBranchPCOffset);
+ DCHECK_EQ(offset & 3, 0);
+
+ return static_cast<uint64_t>(offset);
+}
+
+int32_t Assembler::branch_offset_helper(Label* L, OffsetSize bits) {
+ int32_t target_pos;
+ int32_t pad = IsPrevInstrCompactBranch() ? kInstrSize : 0;
+
+ if (L->is_bound()) {
+ target_pos = L->pos();
+ } else {
+ if (L->is_linked()) {
+ target_pos = L->pos();
+ L->link_to(pc_offset() + pad);
+ } else {
+ L->link_to(pc_offset() + pad);
+ if (!trampoline_emitted_) {
+ unbound_labels_count_++;
+ next_buffer_check_ -= kTrampolineSlotsSize;
+ }
+ return kEndOfChain;
+ }
+ }
+
+ int32_t offset = target_pos - (pc_offset() + kBranchPCOffset + pad);
+ DCHECK(is_intn(offset, bits + 2));
+ DCHECK_EQ(offset & 3, 0);
+
+ return offset;
+}
+
+void Assembler::label_at_put(Label* L, int at_offset) {
+ int target_pos;
+ if (L->is_bound()) {
+ target_pos = L->pos();
+ instr_at_put(at_offset, target_pos + (Code::kHeaderSize - kHeapObjectTag));
+ } else {
+ if (L->is_linked()) {
+ target_pos = L->pos(); // L's link.
+ int32_t imm18 = target_pos - at_offset;
+ DCHECK_EQ(imm18 & 3, 0);
+ int32_t imm16 = imm18 >> 2;
+ DCHECK(is_int16(imm16));
+ instr_at_put(at_offset, (imm16 & kImm16Mask));
+ } else {
+ target_pos = kEndOfChain;
+ instr_at_put(at_offset, 0);
+ if (!trampoline_emitted_) {
+ unbound_labels_count_++;
+ next_buffer_check_ -= kTrampolineSlotsSize;
+ }
+ }
+ L->link_to(at_offset);
+ }
+}
+
+//------- Branch and jump instructions --------
+
+void Assembler::b(int16_t offset) { beq(zero_reg, zero_reg, offset); }
+
+void Assembler::bal(int16_t offset) { bgezal(zero_reg, offset); }
+
+void Assembler::bc(int32_t offset) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrImmediate(BC, offset, CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::balc(int32_t offset) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrImmediate(BALC, offset, CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::beq(Register rs, Register rt, int16_t offset) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ GenInstrImmediate(BEQ, rs, rt, offset);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+void Assembler::bgez(Register rs, int16_t offset) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ GenInstrImmediate(REGIMM, rs, BGEZ, offset);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+void Assembler::bgezc(Register rt, int16_t offset) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK(rt != zero_reg);
+ GenInstrImmediate(BLEZL, rt, rt, offset, CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::bgeuc(Register rs, Register rt, int16_t offset) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK(rs != zero_reg);
+ DCHECK(rt != zero_reg);
+ DCHECK(rs.code() != rt.code());
+ GenInstrImmediate(BLEZ, rs, rt, offset, CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::bgec(Register rs, Register rt, int16_t offset) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK(rs != zero_reg);
+ DCHECK(rt != zero_reg);
+ DCHECK(rs.code() != rt.code());
+ GenInstrImmediate(BLEZL, rs, rt, offset, CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::bgezal(Register rs, int16_t offset) {
+ DCHECK(kArchVariant != kMips64r6 || rs == zero_reg);
+ DCHECK(rs != ra);
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ GenInstrImmediate(REGIMM, rs, BGEZAL, offset);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+void Assembler::bgtz(Register rs, int16_t offset) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ GenInstrImmediate(BGTZ, rs, zero_reg, offset);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+void Assembler::bgtzc(Register rt, int16_t offset) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK(rt != zero_reg);
+ GenInstrImmediate(BGTZL, zero_reg, rt, offset,
+ CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::blez(Register rs, int16_t offset) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ GenInstrImmediate(BLEZ, rs, zero_reg, offset);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+void Assembler::blezc(Register rt, int16_t offset) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK(rt != zero_reg);
+ GenInstrImmediate(BLEZL, zero_reg, rt, offset,
+ CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::bltzc(Register rt, int16_t offset) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK(rt != zero_reg);
+ GenInstrImmediate(BGTZL, rt, rt, offset, CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::bltuc(Register rs, Register rt, int16_t offset) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK(rs != zero_reg);
+ DCHECK(rt != zero_reg);
+ DCHECK(rs.code() != rt.code());
+ GenInstrImmediate(BGTZ, rs, rt, offset, CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::bltc(Register rs, Register rt, int16_t offset) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK(rs != zero_reg);
+ DCHECK(rt != zero_reg);
+ DCHECK(rs.code() != rt.code());
+ GenInstrImmediate(BGTZL, rs, rt, offset, CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::bltz(Register rs, int16_t offset) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ GenInstrImmediate(REGIMM, rs, BLTZ, offset);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+void Assembler::bltzal(Register rs, int16_t offset) {
+ DCHECK(kArchVariant != kMips64r6 || rs == zero_reg);
+ DCHECK(rs != ra);
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ GenInstrImmediate(REGIMM, rs, BLTZAL, offset);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+void Assembler::bne(Register rs, Register rt, int16_t offset) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ GenInstrImmediate(BNE, rs, rt, offset);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+void Assembler::bovc(Register rs, Register rt, int16_t offset) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ if (rs.code() >= rt.code()) {
+ GenInstrImmediate(ADDI, rs, rt, offset, CompactBranchType::COMPACT_BRANCH);
+ } else {
+ GenInstrImmediate(ADDI, rt, rs, offset, CompactBranchType::COMPACT_BRANCH);
+ }
+}
+
+void Assembler::bnvc(Register rs, Register rt, int16_t offset) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ if (rs.code() >= rt.code()) {
+ GenInstrImmediate(DADDI, rs, rt, offset, CompactBranchType::COMPACT_BRANCH);
+ } else {
+ GenInstrImmediate(DADDI, rt, rs, offset, CompactBranchType::COMPACT_BRANCH);
+ }
+}
+
+void Assembler::blezalc(Register rt, int16_t offset) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK(rt != zero_reg);
+ DCHECK(rt != ra);
+ GenInstrImmediate(BLEZ, zero_reg, rt, offset,
+ CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::bgezalc(Register rt, int16_t offset) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK(rt != zero_reg);
+ DCHECK(rt != ra);
+ GenInstrImmediate(BLEZ, rt, rt, offset, CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::bgezall(Register rs, int16_t offset) {
+ DCHECK_NE(kArchVariant, kMips64r6);
+ DCHECK(rs != zero_reg);
+ DCHECK(rs != ra);
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ GenInstrImmediate(REGIMM, rs, BGEZALL, offset);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+void Assembler::bltzalc(Register rt, int16_t offset) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK(rt != zero_reg);
+ DCHECK(rt != ra);
+ GenInstrImmediate(BGTZ, rt, rt, offset, CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::bgtzalc(Register rt, int16_t offset) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK(rt != zero_reg);
+ DCHECK(rt != ra);
+ GenInstrImmediate(BGTZ, zero_reg, rt, offset,
+ CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::beqzalc(Register rt, int16_t offset) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK(rt != zero_reg);
+ DCHECK(rt != ra);
+ GenInstrImmediate(ADDI, zero_reg, rt, offset,
+ CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::bnezalc(Register rt, int16_t offset) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK(rt != zero_reg);
+ DCHECK(rt != ra);
+ GenInstrImmediate(DADDI, zero_reg, rt, offset,
+ CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::beqc(Register rs, Register rt, int16_t offset) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK(rs.code() != rt.code() && rs.code() != 0 && rt.code() != 0);
+ if (rs.code() < rt.code()) {
+ GenInstrImmediate(ADDI, rs, rt, offset, CompactBranchType::COMPACT_BRANCH);
+ } else {
+ GenInstrImmediate(ADDI, rt, rs, offset, CompactBranchType::COMPACT_BRANCH);
+ }
+}
+
+void Assembler::beqzc(Register rs, int32_t offset) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK(rs != zero_reg);
+ GenInstrImmediate(POP66, rs, offset, CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::bnec(Register rs, Register rt, int16_t offset) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK(rs.code() != rt.code() && rs.code() != 0 && rt.code() != 0);
+ if (rs.code() < rt.code()) {
+ GenInstrImmediate(DADDI, rs, rt, offset, CompactBranchType::COMPACT_BRANCH);
+ } else {
+ GenInstrImmediate(DADDI, rt, rs, offset, CompactBranchType::COMPACT_BRANCH);
+ }
+}
+
+void Assembler::bnezc(Register rs, int32_t offset) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK(rs != zero_reg);
+ GenInstrImmediate(POP76, rs, offset, CompactBranchType::COMPACT_BRANCH);
+}
+
+void Assembler::j(int64_t target) {
+ // Deprecated. Use PC-relative jumps instead.
+ UNREACHABLE();
+}
+
+void Assembler::j(Label* target) {
+ // Deprecated. Use PC-relative jumps instead.
+ UNREACHABLE();
+}
+
+void Assembler::jal(Label* target) {
+ // Deprecated. Use PC-relative jumps instead.
+ UNREACHABLE();
+}
+
+void Assembler::jal(int64_t target) {
+ // Deprecated. Use PC-relative jumps instead.
+ UNREACHABLE();
+}
+
+void Assembler::jr(Register rs) {
+ if (kArchVariant != kMips64r6) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ GenInstrRegister(SPECIAL, rs, zero_reg, zero_reg, 0, JR);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+ } else {
+ jalr(rs, zero_reg);
+ }
+}
+
+void Assembler::jalr(Register rs, Register rd) {
+ DCHECK(rs.code() != rd.code());
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ GenInstrRegister(SPECIAL, rs, zero_reg, rd, 0, JALR);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+void Assembler::jic(Register rt, int16_t offset) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrImmediate(POP66, zero_reg, rt, offset);
+}
+
+void Assembler::jialc(Register rt, int16_t offset) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrImmediate(POP76, zero_reg, rt, offset);
+}
+
+// -------Data-processing-instructions---------
+
+// Arithmetic.
+
+void Assembler::addu(Register rd, Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, ADDU);
+}
+
+void Assembler::addiu(Register rd, Register rs, int32_t j) {
+ GenInstrImmediate(ADDIU, rs, rd, j);
+}
+
+void Assembler::subu(Register rd, Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, SUBU);
+}
+
+void Assembler::mul(Register rd, Register rs, Register rt) {
+ if (kArchVariant == kMips64r6) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, MUL_OP, MUL_MUH);
+ } else {
+ GenInstrRegister(SPECIAL2, rs, rt, rd, 0, MUL);
+ }
+}
+
+void Assembler::muh(Register rd, Register rs, Register rt) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrRegister(SPECIAL, rs, rt, rd, MUH_OP, MUL_MUH);
+}
+
+void Assembler::mulu(Register rd, Register rs, Register rt) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrRegister(SPECIAL, rs, rt, rd, MUL_OP, MUL_MUH_U);
+}
+
+void Assembler::muhu(Register rd, Register rs, Register rt) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrRegister(SPECIAL, rs, rt, rd, MUH_OP, MUL_MUH_U);
+}
+
+void Assembler::dmul(Register rd, Register rs, Register rt) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrRegister(SPECIAL, rs, rt, rd, MUL_OP, D_MUL_MUH);
+}
+
+void Assembler::dmuh(Register rd, Register rs, Register rt) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrRegister(SPECIAL, rs, rt, rd, MUH_OP, D_MUL_MUH);
+}
+
+void Assembler::dmulu(Register rd, Register rs, Register rt) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrRegister(SPECIAL, rs, rt, rd, MUL_OP, D_MUL_MUH_U);
+}
+
+void Assembler::dmuhu(Register rd, Register rs, Register rt) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrRegister(SPECIAL, rs, rt, rd, MUH_OP, D_MUL_MUH_U);
+}
+
+void Assembler::mult(Register rs, Register rt) {
+ DCHECK_NE(kArchVariant, kMips64r6);
+ GenInstrRegister(SPECIAL, rs, rt, zero_reg, 0, MULT);
+}
+
+void Assembler::multu(Register rs, Register rt) {
+ DCHECK_NE(kArchVariant, kMips64r6);
+ GenInstrRegister(SPECIAL, rs, rt, zero_reg, 0, MULTU);
+}
+
+void Assembler::daddiu(Register rd, Register rs, int32_t j) {
+ GenInstrImmediate(DADDIU, rs, rd, j);
+}
+
+void Assembler::div(Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, zero_reg, 0, DIV);
+}
+
+void Assembler::div(Register rd, Register rs, Register rt) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrRegister(SPECIAL, rs, rt, rd, DIV_OP, DIV_MOD);
+}
+
+void Assembler::mod(Register rd, Register rs, Register rt) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrRegister(SPECIAL, rs, rt, rd, MOD_OP, DIV_MOD);
+}
+
+void Assembler::divu(Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, zero_reg, 0, DIVU);
+}
+
+void Assembler::divu(Register rd, Register rs, Register rt) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrRegister(SPECIAL, rs, rt, rd, DIV_OP, DIV_MOD_U);
+}
+
+void Assembler::modu(Register rd, Register rs, Register rt) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrRegister(SPECIAL, rs, rt, rd, MOD_OP, DIV_MOD_U);
+}
+
+void Assembler::daddu(Register rd, Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, DADDU);
+}
+
+void Assembler::dsubu(Register rd, Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, DSUBU);
+}
+
+void Assembler::dmult(Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, zero_reg, 0, DMULT);
+}
+
+void Assembler::dmultu(Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, zero_reg, 0, DMULTU);
+}
+
+void Assembler::ddiv(Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, zero_reg, 0, DDIV);
+}
+
+void Assembler::ddiv(Register rd, Register rs, Register rt) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrRegister(SPECIAL, rs, rt, rd, DIV_OP, D_DIV_MOD);
+}
+
+void Assembler::dmod(Register rd, Register rs, Register rt) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrRegister(SPECIAL, rs, rt, rd, MOD_OP, D_DIV_MOD);
+}
+
+void Assembler::ddivu(Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, zero_reg, 0, DDIVU);
+}
+
+void Assembler::ddivu(Register rd, Register rs, Register rt) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrRegister(SPECIAL, rs, rt, rd, DIV_OP, D_DIV_MOD_U);
+}
+
+void Assembler::dmodu(Register rd, Register rs, Register rt) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrRegister(SPECIAL, rs, rt, rd, MOD_OP, D_DIV_MOD_U);
+}
+
+// Logical.
+
+void Assembler::and_(Register rd, Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, AND);
+}
+
+void Assembler::andi(Register rt, Register rs, int32_t j) {
+ DCHECK(is_uint16(j));
+ GenInstrImmediate(ANDI, rs, rt, j);
+}
+
+void Assembler::or_(Register rd, Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, OR);
+}
+
+void Assembler::ori(Register rt, Register rs, int32_t j) {
+ DCHECK(is_uint16(j));
+ GenInstrImmediate(ORI, rs, rt, j);
+}
+
+void Assembler::xor_(Register rd, Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, XOR);
+}
+
+void Assembler::xori(Register rt, Register rs, int32_t j) {
+ DCHECK(is_uint16(j));
+ GenInstrImmediate(XORI, rs, rt, j);
+}
+
+void Assembler::nor(Register rd, Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, NOR);
+}
+
+// Shifts.
+void Assembler::sll(Register rd, Register rt, uint16_t sa,
+ bool coming_from_nop) {
+ // Don't allow nop instructions in the form sll zero_reg, zero_reg to be
+ // generated using the sll instruction. They must be generated using
+ // nop(int/NopMarkerTypes).
+ DCHECK(coming_from_nop || (rd != zero_reg && rt != zero_reg));
+ GenInstrRegister(SPECIAL, zero_reg, rt, rd, sa & 0x1F, SLL);
+}
+
+void Assembler::sllv(Register rd, Register rt, Register rs) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, SLLV);
+}
+
+void Assembler::srl(Register rd, Register rt, uint16_t sa) {
+ GenInstrRegister(SPECIAL, zero_reg, rt, rd, sa & 0x1F, SRL);
+}
+
+void Assembler::srlv(Register rd, Register rt, Register rs) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, SRLV);
+}
+
+void Assembler::sra(Register rd, Register rt, uint16_t sa) {
+ GenInstrRegister(SPECIAL, zero_reg, rt, rd, sa & 0x1F, SRA);
+}
+
+void Assembler::srav(Register rd, Register rt, Register rs) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, SRAV);
+}
+
+void Assembler::rotr(Register rd, Register rt, uint16_t sa) {
+ // Should be called via MacroAssembler::Ror.
+ DCHECK(rd.is_valid() && rt.is_valid() && is_uint5(sa));
+ DCHECK(kArchVariant == kMips64r2 || kArchVariant == kMips64r6);
+ Instr instr = SPECIAL | (1 << kRsShift) | (rt.code() << kRtShift) |
+ (rd.code() << kRdShift) | (sa << kSaShift) | SRL;
+ emit(instr);
+}
+
+void Assembler::rotrv(Register rd, Register rt, Register rs) {
+ // Should be called via MacroAssembler::Ror.
+ DCHECK(rd.is_valid() && rt.is_valid() && rs.is_valid());
+ DCHECK(kArchVariant == kMips64r2 || kArchVariant == kMips64r6);
+ Instr instr = SPECIAL | (rs.code() << kRsShift) | (rt.code() << kRtShift) |
+ (rd.code() << kRdShift) | (1 << kSaShift) | SRLV;
+ emit(instr);
+}
+
+void Assembler::dsll(Register rd, Register rt, uint16_t sa) {
+ GenInstrRegister(SPECIAL, zero_reg, rt, rd, sa & 0x1F, DSLL);
+}
+
+void Assembler::dsllv(Register rd, Register rt, Register rs) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, DSLLV);
+}
+
+void Assembler::dsrl(Register rd, Register rt, uint16_t sa) {
+ GenInstrRegister(SPECIAL, zero_reg, rt, rd, sa & 0x1F, DSRL);
+}
+
+void Assembler::dsrlv(Register rd, Register rt, Register rs) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, DSRLV);
+}
+
+void Assembler::drotr(Register rd, Register rt, uint16_t sa) {
+ DCHECK(rd.is_valid() && rt.is_valid() && is_uint5(sa));
+ Instr instr = SPECIAL | (1 << kRsShift) | (rt.code() << kRtShift) |
+ (rd.code() << kRdShift) | (sa << kSaShift) | DSRL;
+ emit(instr);
+}
+
+void Assembler::drotr32(Register rd, Register rt, uint16_t sa) {
+ DCHECK(rd.is_valid() && rt.is_valid() && is_uint5(sa));
+ Instr instr = SPECIAL | (1 << kRsShift) | (rt.code() << kRtShift) |
+ (rd.code() << kRdShift) | (sa << kSaShift) | DSRL32;
+ emit(instr);
+}
+
+void Assembler::drotrv(Register rd, Register rt, Register rs) {
+ DCHECK(rd.is_valid() && rt.is_valid() && rs.is_valid());
+ Instr instr = SPECIAL | (rs.code() << kRsShift) | (rt.code() << kRtShift) |
+ (rd.code() << kRdShift) | (1 << kSaShift) | DSRLV;
+ emit(instr);
+}
+
+void Assembler::dsra(Register rd, Register rt, uint16_t sa) {
+ GenInstrRegister(SPECIAL, zero_reg, rt, rd, sa & 0x1F, DSRA);
+}
+
+void Assembler::dsrav(Register rd, Register rt, Register rs) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, DSRAV);
+}
+
+void Assembler::dsll32(Register rd, Register rt, uint16_t sa) {
+ GenInstrRegister(SPECIAL, zero_reg, rt, rd, sa & 0x1F, DSLL32);
+}
+
+void Assembler::dsrl32(Register rd, Register rt, uint16_t sa) {
+ GenInstrRegister(SPECIAL, zero_reg, rt, rd, sa & 0x1F, DSRL32);
+}
+
+void Assembler::dsra32(Register rd, Register rt, uint16_t sa) {
+ GenInstrRegister(SPECIAL, zero_reg, rt, rd, sa & 0x1F, DSRA32);
+}
+
+void Assembler::lsa(Register rd, Register rt, Register rs, uint8_t sa) {
+ DCHECK(rd.is_valid() && rt.is_valid() && rs.is_valid());
+ DCHECK_LE(sa, 3);
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ Instr instr = SPECIAL | rs.code() << kRsShift | rt.code() << kRtShift |
+ rd.code() << kRdShift | sa << kSaShift | LSA;
+ emit(instr);
+}
+
+void Assembler::dlsa(Register rd, Register rt, Register rs, uint8_t sa) {
+ DCHECK(rd.is_valid() && rt.is_valid() && rs.is_valid());
+ DCHECK_LE(sa, 3);
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ Instr instr = SPECIAL | rs.code() << kRsShift | rt.code() << kRtShift |
+ rd.code() << kRdShift | sa << kSaShift | DLSA;
+ emit(instr);
+}
+
+// ------------Memory-instructions-------------
+
+void Assembler::AdjustBaseAndOffset(MemOperand* src,
+ OffsetAccessType access_type,
+ int second_access_add_to_offset) {
+ // This method is used to adjust the base register and offset pair
+ // for a load/store when the offset doesn't fit into int16_t.
+ // It is assumed that 'base + offset' is sufficiently aligned for memory
+ // operands that are machine word in size or smaller. For doubleword-sized
+ // operands it's assumed that 'base' is a multiple of 8, while 'offset'
+ // may be a multiple of 4 (e.g. 4-byte-aligned long and double arguments
+ // and spilled variables on the stack accessed relative to the stack
+ // pointer register).
+ // We preserve the "alignment" of 'offset' by adjusting it by a multiple of 8.
+
+ bool doubleword_aligned = (src->offset() & (kDoubleSize - 1)) == 0;
+ bool two_accesses = static_cast<bool>(access_type) || !doubleword_aligned;
+ DCHECK_LE(second_access_add_to_offset, 7); // Must be <= 7.
+
+ // is_int16 must be passed a signed value, hence the static cast below.
+ if (is_int16(src->offset()) &&
+ (!two_accesses || is_int16(static_cast<int32_t>(
+ src->offset() + second_access_add_to_offset)))) {
+ // Nothing to do: 'offset' (and, if needed, 'offset + 4', or other specified
+ // value) fits into int16_t.
+ return;
+ }
+
+ DCHECK(src->rm() !=
+ at); // Must not overwrite the register 'base' while loading 'offset'.
+
+#ifdef DEBUG
+ // Remember the "(mis)alignment" of 'offset', it will be checked at the end.
+ uint32_t misalignment = src->offset() & (kDoubleSize - 1);
+#endif
+
+ // Do not load the whole 32-bit 'offset' if it can be represented as
+ // a sum of two 16-bit signed offsets. This can save an instruction or two.
+ // To simplify matters, only do this for a symmetric range of offsets from
+ // about -64KB to about +64KB, allowing further addition of 4 when accessing
+ // 64-bit variables with two 32-bit accesses.
+ constexpr int32_t kMinOffsetForSimpleAdjustment =
+ 0x7FF8; // Max int16_t that's a multiple of 8.
+ constexpr int32_t kMaxOffsetForSimpleAdjustment =
+ 2 * kMinOffsetForSimpleAdjustment;
+
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ if (0 <= src->offset() && src->offset() <= kMaxOffsetForSimpleAdjustment) {
+ daddiu(scratch, src->rm(), kMinOffsetForSimpleAdjustment);
+ src->offset_ -= kMinOffsetForSimpleAdjustment;
+ } else if (-kMaxOffsetForSimpleAdjustment <= src->offset() &&
+ src->offset() < 0) {
+ daddiu(scratch, src->rm(), -kMinOffsetForSimpleAdjustment);
+ src->offset_ += kMinOffsetForSimpleAdjustment;
+ } else if (kArchVariant == kMips64r6) {
+ // On r6 take advantage of the daui instruction, e.g.:
+ // daui at, base, offset_high
+ // [dahi at, 1] // When `offset` is close to +2GB.
+ // lw reg_lo, offset_low(at)
+ // [lw reg_hi, (offset_low+4)(at)] // If misaligned 64-bit load.
+ // or when offset_low+4 overflows int16_t:
+ // daui at, base, offset_high
+ // daddiu at, at, 8
+ // lw reg_lo, (offset_low-8)(at)
+ // lw reg_hi, (offset_low-4)(at)
+ int16_t offset_low = static_cast<uint16_t>(src->offset());
+ int32_t offset_low32 = offset_low;
+ int16_t offset_high = static_cast<uint16_t>(src->offset() >> 16);
+ bool increment_hi16 = offset_low < 0;
+ bool overflow_hi16 = false;
+
+ if (increment_hi16) {
+ offset_high++;
+ overflow_hi16 = (offset_high == -32768);
+ }
+ daui(scratch, src->rm(), static_cast<uint16_t>(offset_high));
+
+ if (overflow_hi16) {
+ dahi(scratch, 1);
+ }
+
+ if (two_accesses && !is_int16(static_cast<int32_t>(
+ offset_low32 + second_access_add_to_offset))) {
+ // Avoid overflow in the 16-bit offset of the load/store instruction when
+ // adding 4.
+ daddiu(scratch, scratch, kDoubleSize);
+ offset_low32 -= kDoubleSize;
+ }
+
+ src->offset_ = offset_low32;
+ } else {
+ // Do not load the whole 32-bit 'offset' if it can be represented as
+ // a sum of three 16-bit signed offsets. This can save an instruction.
+ // To simplify matters, only do this for a symmetric range of offsets from
+ // about -96KB to about +96KB, allowing further addition of 4 when accessing
+ // 64-bit variables with two 32-bit accesses.
+ constexpr int32_t kMinOffsetForMediumAdjustment =
+ 2 * kMinOffsetForSimpleAdjustment;
+ constexpr int32_t kMaxOffsetForMediumAdjustment =
+ 3 * kMinOffsetForSimpleAdjustment;
+ if (0 <= src->offset() && src->offset() <= kMaxOffsetForMediumAdjustment) {
+ daddiu(scratch, src->rm(), kMinOffsetForMediumAdjustment / 2);
+ daddiu(scratch, scratch, kMinOffsetForMediumAdjustment / 2);
+ src->offset_ -= kMinOffsetForMediumAdjustment;
+ } else if (-kMaxOffsetForMediumAdjustment <= src->offset() &&
+ src->offset() < 0) {
+ daddiu(scratch, src->rm(), -kMinOffsetForMediumAdjustment / 2);
+ daddiu(scratch, scratch, -kMinOffsetForMediumAdjustment / 2);
+ src->offset_ += kMinOffsetForMediumAdjustment;
+ } else {
+ // Now that all shorter options have been exhausted, load the full 32-bit
+ // offset.
+ int32_t loaded_offset = RoundDown(src->offset(), kDoubleSize);
+ lui(scratch, (loaded_offset >> kLuiShift) & kImm16Mask);
+ ori(scratch, scratch, loaded_offset & kImm16Mask); // Load 32-bit offset.
+ daddu(scratch, scratch, src->rm());
+ src->offset_ -= loaded_offset;
+ }
+ }
+ src->rm_ = scratch;
+
+ DCHECK(is_int16(src->offset()));
+ if (two_accesses) {
+ DCHECK(is_int16(
+ static_cast<int32_t>(src->offset() + second_access_add_to_offset)));
+ }
+ DCHECK(misalignment == (src->offset() & (kDoubleSize - 1)));
+}
+
+void Assembler::lb(Register rd, const MemOperand& rs) {
+ GenInstrImmediate(LB, rs.rm(), rd, rs.offset_);
+}
+
+void Assembler::lbu(Register rd, const MemOperand& rs) {
+ GenInstrImmediate(LBU, rs.rm(), rd, rs.offset_);
+}
+
+void Assembler::lh(Register rd, const MemOperand& rs) {
+ GenInstrImmediate(LH, rs.rm(), rd, rs.offset_);
+}
+
+void Assembler::lhu(Register rd, const MemOperand& rs) {
+ GenInstrImmediate(LHU, rs.rm(), rd, rs.offset_);
+}
+
+void Assembler::lw(Register rd, const MemOperand& rs) {
+ GenInstrImmediate(LW, rs.rm(), rd, rs.offset_);
+}
+
+void Assembler::lwu(Register rd, const MemOperand& rs) {
+ GenInstrImmediate(LWU, rs.rm(), rd, rs.offset_);
+}
+
+void Assembler::lwl(Register rd, const MemOperand& rs) {
+ DCHECK(is_int16(rs.offset_));
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ GenInstrImmediate(LWL, rs.rm(), rd, rs.offset_);
+}
+
+void Assembler::lwr(Register rd, const MemOperand& rs) {
+ DCHECK(is_int16(rs.offset_));
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ GenInstrImmediate(LWR, rs.rm(), rd, rs.offset_);
+}
+
+void Assembler::sb(Register rd, const MemOperand& rs) {
+ GenInstrImmediate(SB, rs.rm(), rd, rs.offset_);
+}
+
+void Assembler::sh(Register rd, const MemOperand& rs) {
+ GenInstrImmediate(SH, rs.rm(), rd, rs.offset_);
+}
+
+void Assembler::sw(Register rd, const MemOperand& rs) {
+ GenInstrImmediate(SW, rs.rm(), rd, rs.offset_);
+}
+
+void Assembler::swl(Register rd, const MemOperand& rs) {
+ DCHECK(is_int16(rs.offset_));
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ GenInstrImmediate(SWL, rs.rm(), rd, rs.offset_);
+}
+
+void Assembler::swr(Register rd, const MemOperand& rs) {
+ DCHECK(is_int16(rs.offset_));
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ GenInstrImmediate(SWR, rs.rm(), rd, rs.offset_);
+}
+
+void Assembler::ll(Register rd, const MemOperand& rs) {
+ if (kArchVariant == kMips64r6) {
+ DCHECK(is_int9(rs.offset_));
+ GenInstrImmediate(SPECIAL3, rs.rm(), rd, rs.offset_, 0, LL_R6);
+ } else {
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ DCHECK(is_int16(rs.offset_));
+ GenInstrImmediate(LL, rs.rm(), rd, rs.offset_);
+ }
+}
+
+void Assembler::lld(Register rd, const MemOperand& rs) {
+ if (kArchVariant == kMips64r6) {
+ DCHECK(is_int9(rs.offset_));
+ GenInstrImmediate(SPECIAL3, rs.rm(), rd, rs.offset_, 0, LLD_R6);
+ } else {
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ DCHECK(is_int16(rs.offset_));
+ GenInstrImmediate(LLD, rs.rm(), rd, rs.offset_);
+ }
+}
+
+void Assembler::sc(Register rd, const MemOperand& rs) {
+ if (kArchVariant == kMips64r6) {
+ DCHECK(is_int9(rs.offset_));
+ GenInstrImmediate(SPECIAL3, rs.rm(), rd, rs.offset_, 0, SC_R6);
+ } else {
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ GenInstrImmediate(SC, rs.rm(), rd, rs.offset_);
+ }
+}
+
+void Assembler::scd(Register rd, const MemOperand& rs) {
+ if (kArchVariant == kMips64r6) {
+ DCHECK(is_int9(rs.offset_));
+ GenInstrImmediate(SPECIAL3, rs.rm(), rd, rs.offset_, 0, SCD_R6);
+ } else {
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ GenInstrImmediate(SCD, rs.rm(), rd, rs.offset_);
+ }
+}
+
+void Assembler::lui(Register rd, int32_t j) {
+ DCHECK(is_uint16(j) || is_int16(j));
+ GenInstrImmediate(LUI, zero_reg, rd, j);
+}
+
+void Assembler::aui(Register rt, Register rs, int32_t j) {
+ // This instruction uses same opcode as 'lui'. The difference in encoding is
+ // 'lui' has zero reg. for rs field.
+ DCHECK(is_uint16(j));
+ GenInstrImmediate(LUI, rs, rt, j);
+}
+
+void Assembler::daui(Register rt, Register rs, int32_t j) {
+ DCHECK(is_uint16(j));
+ DCHECK(rs != zero_reg);
+ GenInstrImmediate(DAUI, rs, rt, j);
+}
+
+void Assembler::dahi(Register rs, int32_t j) {
+ DCHECK(is_uint16(j));
+ GenInstrImmediate(REGIMM, rs, DAHI, j);
+}
+
+void Assembler::dati(Register rs, int32_t j) {
+ DCHECK(is_uint16(j));
+ GenInstrImmediate(REGIMM, rs, DATI, j);
+}
+
+void Assembler::ldl(Register rd, const MemOperand& rs) {
+ DCHECK(is_int16(rs.offset_));
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ GenInstrImmediate(LDL, rs.rm(), rd, rs.offset_);
+}
+
+void Assembler::ldr(Register rd, const MemOperand& rs) {
+ DCHECK(is_int16(rs.offset_));
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ GenInstrImmediate(LDR, rs.rm(), rd, rs.offset_);
+}
+
+void Assembler::sdl(Register rd, const MemOperand& rs) {
+ DCHECK(is_int16(rs.offset_));
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ GenInstrImmediate(SDL, rs.rm(), rd, rs.offset_);
+}
+
+void Assembler::sdr(Register rd, const MemOperand& rs) {
+ DCHECK(is_int16(rs.offset_));
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ GenInstrImmediate(SDR, rs.rm(), rd, rs.offset_);
+}
+
+void Assembler::ld(Register rd, const MemOperand& rs) {
+ GenInstrImmediate(LD, rs.rm(), rd, rs.offset_);
+}
+
+void Assembler::sd(Register rd, const MemOperand& rs) {
+ GenInstrImmediate(SD, rs.rm(), rd, rs.offset_);
+}
+
+// ---------PC-Relative instructions-----------
+
+void Assembler::addiupc(Register rs, int32_t imm19) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK(rs.is_valid() && is_int19(imm19));
+ uint32_t imm21 = ADDIUPC << kImm19Bits | (imm19 & kImm19Mask);
+ GenInstrImmediate(PCREL, rs, imm21);
+}
+
+void Assembler::lwpc(Register rs, int32_t offset19) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK(rs.is_valid() && is_int19(offset19));
+ uint32_t imm21 = LWPC << kImm19Bits | (offset19 & kImm19Mask);
+ GenInstrImmediate(PCREL, rs, imm21);
+}
+
+void Assembler::lwupc(Register rs, int32_t offset19) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK(rs.is_valid() && is_int19(offset19));
+ uint32_t imm21 = LWUPC << kImm19Bits | (offset19 & kImm19Mask);
+ GenInstrImmediate(PCREL, rs, imm21);
+}
+
+void Assembler::ldpc(Register rs, int32_t offset18) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK(rs.is_valid() && is_int18(offset18));
+ uint32_t imm21 = LDPC << kImm18Bits | (offset18 & kImm18Mask);
+ GenInstrImmediate(PCREL, rs, imm21);
+}
+
+void Assembler::auipc(Register rs, int16_t imm16) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK(rs.is_valid());
+ uint32_t imm21 = AUIPC << kImm16Bits | (imm16 & kImm16Mask);
+ GenInstrImmediate(PCREL, rs, imm21);
+}
+
+void Assembler::aluipc(Register rs, int16_t imm16) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK(rs.is_valid());
+ uint32_t imm21 = ALUIPC << kImm16Bits | (imm16 & kImm16Mask);
+ GenInstrImmediate(PCREL, rs, imm21);
+}
+
+// -------------Misc-instructions--------------
+
+// Break / Trap instructions.
+void Assembler::break_(uint32_t code, bool break_as_stop) {
+ DCHECK_EQ(code & ~0xFFFFF, 0);
+ // We need to invalidate breaks that could be stops as well because the
+ // simulator expects a char pointer after the stop instruction.
+ // See constants-mips.h for explanation.
+ DCHECK(
+ (break_as_stop && code <= kMaxStopCode && code > kMaxWatchpointCode) ||
+ (!break_as_stop && (code > kMaxStopCode || code <= kMaxWatchpointCode)));
+ Instr break_instr = SPECIAL | BREAK | (code << 6);
+ emit(break_instr);
+}
+
+void Assembler::stop(uint32_t code) {
+ DCHECK_GT(code, kMaxWatchpointCode);
+ DCHECK_LE(code, kMaxStopCode);
+#if defined(V8_HOST_ARCH_MIPS) || defined(V8_HOST_ARCH_MIPS64)
+ break_(0x54321);
+#else // V8_HOST_ARCH_MIPS
+ break_(code, true);
+#endif
+}
+
+void Assembler::tge(Register rs, Register rt, uint16_t code) {
+ DCHECK(is_uint10(code));
+ Instr instr =
+ SPECIAL | TGE | rs.code() << kRsShift | rt.code() << kRtShift | code << 6;
+ emit(instr);
+}
+
+void Assembler::tgeu(Register rs, Register rt, uint16_t code) {
+ DCHECK(is_uint10(code));
+ Instr instr = SPECIAL | TGEU | rs.code() << kRsShift | rt.code() << kRtShift |
+ code << 6;
+ emit(instr);
+}
+
+void Assembler::tlt(Register rs, Register rt, uint16_t code) {
+ DCHECK(is_uint10(code));
+ Instr instr =
+ SPECIAL | TLT | rs.code() << kRsShift | rt.code() << kRtShift | code << 6;
+ emit(instr);
+}
+
+void Assembler::tltu(Register rs, Register rt, uint16_t code) {
+ DCHECK(is_uint10(code));
+ Instr instr = SPECIAL | TLTU | rs.code() << kRsShift | rt.code() << kRtShift |
+ code << 6;
+ emit(instr);
+}
+
+void Assembler::teq(Register rs, Register rt, uint16_t code) {
+ DCHECK(is_uint10(code));
+ Instr instr =
+ SPECIAL | TEQ | rs.code() << kRsShift | rt.code() << kRtShift | code << 6;
+ emit(instr);
+}
+
+void Assembler::tne(Register rs, Register rt, uint16_t code) {
+ DCHECK(is_uint10(code));
+ Instr instr =
+ SPECIAL | TNE | rs.code() << kRsShift | rt.code() << kRtShift | code << 6;
+ emit(instr);
+}
+
+void Assembler::sync() {
+ Instr sync_instr = SPECIAL | SYNC;
+ emit(sync_instr);
+}
+
+// Move from HI/LO register.
+
+void Assembler::mfhi(Register rd) {
+ GenInstrRegister(SPECIAL, zero_reg, zero_reg, rd, 0, MFHI);
+}
+
+void Assembler::mflo(Register rd) {
+ GenInstrRegister(SPECIAL, zero_reg, zero_reg, rd, 0, MFLO);
+}
+
+// Set on less than instructions.
+void Assembler::slt(Register rd, Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, SLT);
+}
+
+void Assembler::sltu(Register rd, Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, SLTU);
+}
+
+void Assembler::slti(Register rt, Register rs, int32_t j) {
+ GenInstrImmediate(SLTI, rs, rt, j);
+}
+
+void Assembler::sltiu(Register rt, Register rs, int32_t j) {
+ GenInstrImmediate(SLTIU, rs, rt, j);
+}
+
+// Conditional move.
+void Assembler::movz(Register rd, Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, MOVZ);
+}
+
+void Assembler::movn(Register rd, Register rs, Register rt) {
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, MOVN);
+}
+
+void Assembler::movt(Register rd, Register rs, uint16_t cc) {
+ Register rt = Register::from_code((cc & 0x0007) << 2 | 1);
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, MOVCI);
+}
+
+void Assembler::movf(Register rd, Register rs, uint16_t cc) {
+ Register rt = Register::from_code((cc & 0x0007) << 2 | 0);
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, MOVCI);
+}
+
+void Assembler::min_s(FPURegister fd, FPURegister fs, FPURegister ft) {
+ min(S, fd, fs, ft);
+}
+
+void Assembler::min_d(FPURegister fd, FPURegister fs, FPURegister ft) {
+ min(D, fd, fs, ft);
+}
+
+void Assembler::max_s(FPURegister fd, FPURegister fs, FPURegister ft) {
+ max(S, fd, fs, ft);
+}
+
+void Assembler::max_d(FPURegister fd, FPURegister fs, FPURegister ft) {
+ max(D, fd, fs, ft);
+}
+
+void Assembler::mina_s(FPURegister fd, FPURegister fs, FPURegister ft) {
+ mina(S, fd, fs, ft);
+}
+
+void Assembler::mina_d(FPURegister fd, FPURegister fs, FPURegister ft) {
+ mina(D, fd, fs, ft);
+}
+
+void Assembler::maxa_s(FPURegister fd, FPURegister fs, FPURegister ft) {
+ maxa(S, fd, fs, ft);
+}
+
+void Assembler::maxa_d(FPURegister fd, FPURegister fs, FPURegister ft) {
+ maxa(D, fd, fs, ft);
+}
+
+void Assembler::max(SecondaryField fmt, FPURegister fd, FPURegister fs,
+ FPURegister ft) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK((fmt == D) || (fmt == S));
+ GenInstrRegister(COP1, fmt, ft, fs, fd, MAX);
+}
+
+void Assembler::min(SecondaryField fmt, FPURegister fd, FPURegister fs,
+ FPURegister ft) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK((fmt == D) || (fmt == S));
+ GenInstrRegister(COP1, fmt, ft, fs, fd, MIN);
+}
+
+// GPR.
+void Assembler::seleqz(Register rd, Register rs, Register rt) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, SELEQZ_S);
+}
+
+// GPR.
+void Assembler::selnez(Register rd, Register rs, Register rt) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrRegister(SPECIAL, rs, rt, rd, 0, SELNEZ_S);
+}
+
+// Bit twiddling.
+void Assembler::clz(Register rd, Register rs) {
+ if (kArchVariant != kMips64r6) {
+ // clz instr requires same GPR number in 'rd' and 'rt' fields.
+ GenInstrRegister(SPECIAL2, rs, rd, rd, 0, CLZ);
+ } else {
+ GenInstrRegister(SPECIAL, rs, zero_reg, rd, 1, CLZ_R6);
+ }
+}
+
+void Assembler::dclz(Register rd, Register rs) {
+ if (kArchVariant != kMips64r6) {
+ // dclz instr requires same GPR number in 'rd' and 'rt' fields.
+ GenInstrRegister(SPECIAL2, rs, rd, rd, 0, DCLZ);
+ } else {
+ GenInstrRegister(SPECIAL, rs, zero_reg, rd, 1, DCLZ_R6);
+ }
+}
+
+void Assembler::ins_(Register rt, Register rs, uint16_t pos, uint16_t size) {
+ // Should be called via MacroAssembler::Ins.
+ // ins instr has 'rt' field as dest, and two uint5: msb, lsb.
+ DCHECK((kArchVariant == kMips64r2) || (kArchVariant == kMips64r6));
+ GenInstrRegister(SPECIAL3, rs, rt, pos + size - 1, pos, INS);
+}
+
+void Assembler::dins_(Register rt, Register rs, uint16_t pos, uint16_t size) {
+ // Should be called via MacroAssembler::Dins.
+ // dins instr has 'rt' field as dest, and two uint5: msb, lsb.
+ DCHECK(kArchVariant == kMips64r2 || kArchVariant == kMips64r6);
+ GenInstrRegister(SPECIAL3, rs, rt, pos + size - 1, pos, DINS);
+}
+
+void Assembler::dinsm_(Register rt, Register rs, uint16_t pos, uint16_t size) {
+ // Should be called via MacroAssembler::Dins.
+ // dinsm instr has 'rt' field as dest, and two uint5: msbminus32, lsb.
+ DCHECK(kArchVariant == kMips64r2 || kArchVariant == kMips64r6);
+ GenInstrRegister(SPECIAL3, rs, rt, pos + size - 1 - 32, pos, DINSM);
+}
+
+void Assembler::dinsu_(Register rt, Register rs, uint16_t pos, uint16_t size) {
+ // Should be called via MacroAssembler::Dins.
+ // dinsu instr has 'rt' field as dest, and two uint5: msbminus32, lsbminus32.
+ DCHECK(kArchVariant == kMips64r2 || kArchVariant == kMips64r6);
+ GenInstrRegister(SPECIAL3, rs, rt, pos + size - 1 - 32, pos - 32, DINSU);
+}
+
+void Assembler::ext_(Register rt, Register rs, uint16_t pos, uint16_t size) {
+ // Should be called via MacroAssembler::Ext.
+ // ext instr has 'rt' field as dest, and two uint5: msbd, lsb.
+ DCHECK(kArchVariant == kMips64r2 || kArchVariant == kMips64r6);
+ GenInstrRegister(SPECIAL3, rs, rt, size - 1, pos, EXT);
+}
+
+void Assembler::dext_(Register rt, Register rs, uint16_t pos, uint16_t size) {
+ // Should be called via MacroAssembler::Dext.
+ // dext instr has 'rt' field as dest, and two uint5: msbd, lsb.
+ DCHECK(kArchVariant == kMips64r2 || kArchVariant == kMips64r6);
+ GenInstrRegister(SPECIAL3, rs, rt, size - 1, pos, DEXT);
+}
+
+void Assembler::dextm_(Register rt, Register rs, uint16_t pos, uint16_t size) {
+ // Should be called via MacroAssembler::Dextm.
+ // dextm instr has 'rt' field as dest, and two uint5: msbdminus32, lsb.
+ DCHECK(kArchVariant == kMips64r2 || kArchVariant == kMips64r6);
+ GenInstrRegister(SPECIAL3, rs, rt, size - 1 - 32, pos, DEXTM);
+}
+
+void Assembler::dextu_(Register rt, Register rs, uint16_t pos, uint16_t size) {
+ // Should be called via MacroAssembler::Dextu.
+ // dextu instr has 'rt' field as dest, and two uint5: msbd, lsbminus32.
+ DCHECK(kArchVariant == kMips64r2 || kArchVariant == kMips64r6);
+ GenInstrRegister(SPECIAL3, rs, rt, size - 1, pos - 32, DEXTU);
+}
+
+void Assembler::bitswap(Register rd, Register rt) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrRegister(SPECIAL3, zero_reg, rt, rd, 0, BSHFL);
+}
+
+void Assembler::dbitswap(Register rd, Register rt) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrRegister(SPECIAL3, zero_reg, rt, rd, 0, DBSHFL);
+}
+
+void Assembler::pref(int32_t hint, const MemOperand& rs) {
+ DCHECK(is_uint5(hint) && is_uint16(rs.offset_));
+ Instr instr =
+ PREF | (rs.rm().code() << kRsShift) | (hint << kRtShift) | (rs.offset_);
+ emit(instr);
+}
+
+void Assembler::align(Register rd, Register rs, Register rt, uint8_t bp) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK(is_uint3(bp));
+ uint16_t sa = (ALIGN << kBp2Bits) | bp;
+ GenInstrRegister(SPECIAL3, rs, rt, rd, sa, BSHFL);
+}
+
+void Assembler::dalign(Register rd, Register rs, Register rt, uint8_t bp) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK(is_uint3(bp));
+ uint16_t sa = (DALIGN << kBp3Bits) | bp;
+ GenInstrRegister(SPECIAL3, rs, rt, rd, sa, DBSHFL);
+}
+
+void Assembler::wsbh(Register rd, Register rt) {
+ DCHECK(kArchVariant == kMips64r2 || kArchVariant == kMips64r6);
+ GenInstrRegister(SPECIAL3, zero_reg, rt, rd, WSBH, BSHFL);
+}
+
+void Assembler::dsbh(Register rd, Register rt) {
+ DCHECK(kArchVariant == kMips64r2 || kArchVariant == kMips64r6);
+ GenInstrRegister(SPECIAL3, zero_reg, rt, rd, DSBH, DBSHFL);
+}
+
+void Assembler::dshd(Register rd, Register rt) {
+ DCHECK(kArchVariant == kMips64r2 || kArchVariant == kMips64r6);
+ GenInstrRegister(SPECIAL3, zero_reg, rt, rd, DSHD, DBSHFL);
+}
+
+void Assembler::seh(Register rd, Register rt) {
+ DCHECK(kArchVariant == kMips64r2 || kArchVariant == kMips64r6);
+ GenInstrRegister(SPECIAL3, zero_reg, rt, rd, SEH, BSHFL);
+}
+
+void Assembler::seb(Register rd, Register rt) {
+ DCHECK(kArchVariant == kMips64r2 || kArchVariant == kMips64r6);
+ GenInstrRegister(SPECIAL3, zero_reg, rt, rd, SEB, BSHFL);
+}
+
+// --------Coprocessor-instructions----------------
+
+// Load, store, move.
+void Assembler::lwc1(FPURegister fd, const MemOperand& src) {
+ GenInstrImmediate(LWC1, src.rm(), fd, src.offset_);
+}
+
+void Assembler::ldc1(FPURegister fd, const MemOperand& src) {
+ GenInstrImmediate(LDC1, src.rm(), fd, src.offset_);
+}
+
+void Assembler::swc1(FPURegister fs, const MemOperand& src) {
+ GenInstrImmediate(SWC1, src.rm(), fs, src.offset_);
+}
+
+void Assembler::sdc1(FPURegister fs, const MemOperand& src) {
+ GenInstrImmediate(SDC1, src.rm(), fs, src.offset_);
+}
+
+void Assembler::mtc1(Register rt, FPURegister fs) {
+ GenInstrRegister(COP1, MTC1, rt, fs, f0);
+}
+
+void Assembler::mthc1(Register rt, FPURegister fs) {
+ GenInstrRegister(COP1, MTHC1, rt, fs, f0);
+}
+
+void Assembler::dmtc1(Register rt, FPURegister fs) {
+ GenInstrRegister(COP1, DMTC1, rt, fs, f0);
+}
+
+void Assembler::mfc1(Register rt, FPURegister fs) {
+ GenInstrRegister(COP1, MFC1, rt, fs, f0);
+}
+
+void Assembler::mfhc1(Register rt, FPURegister fs) {
+ GenInstrRegister(COP1, MFHC1, rt, fs, f0);
+}
+
+void Assembler::dmfc1(Register rt, FPURegister fs) {
+ GenInstrRegister(COP1, DMFC1, rt, fs, f0);
+}
+
+void Assembler::ctc1(Register rt, FPUControlRegister fs) {
+ GenInstrRegister(COP1, CTC1, rt, fs);
+}
+
+void Assembler::cfc1(Register rt, FPUControlRegister fs) {
+ GenInstrRegister(COP1, CFC1, rt, fs);
+}
+
+void Assembler::sel(SecondaryField fmt, FPURegister fd, FPURegister fs,
+ FPURegister ft) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK((fmt == D) || (fmt == S));
+
+ GenInstrRegister(COP1, fmt, ft, fs, fd, SEL);
+}
+
+void Assembler::sel_s(FPURegister fd, FPURegister fs, FPURegister ft) {
+ sel(S, fd, fs, ft);
+}
+
+void Assembler::sel_d(FPURegister fd, FPURegister fs, FPURegister ft) {
+ sel(D, fd, fs, ft);
+}
+
+// FPR.
+void Assembler::seleqz(SecondaryField fmt, FPURegister fd, FPURegister fs,
+ FPURegister ft) {
+ DCHECK((fmt == D) || (fmt == S));
+ GenInstrRegister(COP1, fmt, ft, fs, fd, SELEQZ_C);
+}
+
+void Assembler::seleqz_d(FPURegister fd, FPURegister fs, FPURegister ft) {
+ seleqz(D, fd, fs, ft);
+}
+
+void Assembler::seleqz_s(FPURegister fd, FPURegister fs, FPURegister ft) {
+ seleqz(S, fd, fs, ft);
+}
+
+void Assembler::selnez_d(FPURegister fd, FPURegister fs, FPURegister ft) {
+ selnez(D, fd, fs, ft);
+}
+
+void Assembler::selnez_s(FPURegister fd, FPURegister fs, FPURegister ft) {
+ selnez(S, fd, fs, ft);
+}
+
+void Assembler::movz_s(FPURegister fd, FPURegister fs, Register rt) {
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ GenInstrRegister(COP1, S, rt, fs, fd, MOVZ_C);
+}
+
+void Assembler::movz_d(FPURegister fd, FPURegister fs, Register rt) {
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ GenInstrRegister(COP1, D, rt, fs, fd, MOVZ_C);
+}
+
+void Assembler::movt_s(FPURegister fd, FPURegister fs, uint16_t cc) {
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ FPURegister ft = FPURegister::from_code((cc & 0x0007) << 2 | 1);
+ GenInstrRegister(COP1, S, ft, fs, fd, MOVF);
+}
+
+void Assembler::movt_d(FPURegister fd, FPURegister fs, uint16_t cc) {
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ FPURegister ft = FPURegister::from_code((cc & 0x0007) << 2 | 1);
+ GenInstrRegister(COP1, D, ft, fs, fd, MOVF);
+}
+
+void Assembler::movf_s(FPURegister fd, FPURegister fs, uint16_t cc) {
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ FPURegister ft = FPURegister::from_code((cc & 0x0007) << 2 | 0);
+ GenInstrRegister(COP1, S, ft, fs, fd, MOVF);
+}
+
+void Assembler::movf_d(FPURegister fd, FPURegister fs, uint16_t cc) {
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ FPURegister ft = FPURegister::from_code((cc & 0x0007) << 2 | 0);
+ GenInstrRegister(COP1, D, ft, fs, fd, MOVF);
+}
+
+void Assembler::movn_s(FPURegister fd, FPURegister fs, Register rt) {
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ GenInstrRegister(COP1, S, rt, fs, fd, MOVN_C);
+}
+
+void Assembler::movn_d(FPURegister fd, FPURegister fs, Register rt) {
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ GenInstrRegister(COP1, D, rt, fs, fd, MOVN_C);
+}
+
+// FPR.
+void Assembler::selnez(SecondaryField fmt, FPURegister fd, FPURegister fs,
+ FPURegister ft) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK((fmt == D) || (fmt == S));
+ GenInstrRegister(COP1, fmt, ft, fs, fd, SELNEZ_C);
+}
+
+// Arithmetic.
+
+void Assembler::add_s(FPURegister fd, FPURegister fs, FPURegister ft) {
+ GenInstrRegister(COP1, S, ft, fs, fd, ADD_D);
+}
+
+void Assembler::add_d(FPURegister fd, FPURegister fs, FPURegister ft) {
+ GenInstrRegister(COP1, D, ft, fs, fd, ADD_D);
+}
+
+void Assembler::sub_s(FPURegister fd, FPURegister fs, FPURegister ft) {
+ GenInstrRegister(COP1, S, ft, fs, fd, SUB_D);
+}
+
+void Assembler::sub_d(FPURegister fd, FPURegister fs, FPURegister ft) {
+ GenInstrRegister(COP1, D, ft, fs, fd, SUB_D);
+}
+
+void Assembler::mul_s(FPURegister fd, FPURegister fs, FPURegister ft) {
+ GenInstrRegister(COP1, S, ft, fs, fd, MUL_D);
+}
+
+void Assembler::mul_d(FPURegister fd, FPURegister fs, FPURegister ft) {
+ GenInstrRegister(COP1, D, ft, fs, fd, MUL_D);
+}
+
+void Assembler::madd_s(FPURegister fd, FPURegister fr, FPURegister fs,
+ FPURegister ft) {
+ // On Loongson 3A (MIPS64R2), MADD.S instruction is actually fused MADD.S and
+ // this causes failure in some of the tests. Since this optimization is rarely
+ // used, and not used at all on MIPS64R6, this isntruction is removed.
+ UNREACHABLE();
+}
+
+void Assembler::madd_d(FPURegister fd, FPURegister fr, FPURegister fs,
+ FPURegister ft) {
+ // On Loongson 3A (MIPS64R2), MADD.D instruction is actually fused MADD.D and
+ // this causes failure in some of the tests. Since this optimization is rarely
+ // used, and not used at all on MIPS64R6, this isntruction is removed.
+ UNREACHABLE();
+}
+
+void Assembler::msub_s(FPURegister fd, FPURegister fr, FPURegister fs,
+ FPURegister ft) {
+ // See explanation for instruction madd_s.
+ UNREACHABLE();
+}
+
+void Assembler::msub_d(FPURegister fd, FPURegister fr, FPURegister fs,
+ FPURegister ft) {
+ // See explanation for instruction madd_d.
+ UNREACHABLE();
+}
+
+void Assembler::maddf_s(FPURegister fd, FPURegister fs, FPURegister ft) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrRegister(COP1, S, ft, fs, fd, MADDF_S);
+}
+
+void Assembler::maddf_d(FPURegister fd, FPURegister fs, FPURegister ft) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrRegister(COP1, D, ft, fs, fd, MADDF_D);
+}
+
+void Assembler::msubf_s(FPURegister fd, FPURegister fs, FPURegister ft) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrRegister(COP1, S, ft, fs, fd, MSUBF_S);
+}
+
+void Assembler::msubf_d(FPURegister fd, FPURegister fs, FPURegister ft) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrRegister(COP1, D, ft, fs, fd, MSUBF_D);
+}
+
+void Assembler::div_s(FPURegister fd, FPURegister fs, FPURegister ft) {
+ GenInstrRegister(COP1, S, ft, fs, fd, DIV_D);
+}
+
+void Assembler::div_d(FPURegister fd, FPURegister fs, FPURegister ft) {
+ GenInstrRegister(COP1, D, ft, fs, fd, DIV_D);
+}
+
+void Assembler::abs_s(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, S, f0, fs, fd, ABS_D);
+}
+
+void Assembler::abs_d(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, D, f0, fs, fd, ABS_D);
+}
+
+void Assembler::mov_d(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, D, f0, fs, fd, MOV_D);
+}
+
+void Assembler::mov_s(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, S, f0, fs, fd, MOV_S);
+}
+
+void Assembler::neg_s(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, S, f0, fs, fd, NEG_D);
+}
+
+void Assembler::neg_d(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, D, f0, fs, fd, NEG_D);
+}
+
+void Assembler::sqrt_s(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, S, f0, fs, fd, SQRT_D);
+}
+
+void Assembler::sqrt_d(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, D, f0, fs, fd, SQRT_D);
+}
+
+void Assembler::rsqrt_s(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, S, f0, fs, fd, RSQRT_S);
+}
+
+void Assembler::rsqrt_d(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, D, f0, fs, fd, RSQRT_D);
+}
+
+void Assembler::recip_d(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, D, f0, fs, fd, RECIP_D);
+}
+
+void Assembler::recip_s(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, S, f0, fs, fd, RECIP_S);
+}
+
+// Conversions.
+void Assembler::cvt_w_s(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, S, f0, fs, fd, CVT_W_S);
+}
+
+void Assembler::cvt_w_d(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, D, f0, fs, fd, CVT_W_D);
+}
+
+void Assembler::trunc_w_s(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, S, f0, fs, fd, TRUNC_W_S);
+}
+
+void Assembler::trunc_w_d(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, D, f0, fs, fd, TRUNC_W_D);
+}
+
+void Assembler::round_w_s(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, S, f0, fs, fd, ROUND_W_S);
+}
+
+void Assembler::round_w_d(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, D, f0, fs, fd, ROUND_W_D);
+}
+
+void Assembler::floor_w_s(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, S, f0, fs, fd, FLOOR_W_S);
+}
+
+void Assembler::floor_w_d(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, D, f0, fs, fd, FLOOR_W_D);
+}
+
+void Assembler::ceil_w_s(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, S, f0, fs, fd, CEIL_W_S);
+}
+
+void Assembler::ceil_w_d(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, D, f0, fs, fd, CEIL_W_D);
+}
+
+void Assembler::rint_s(FPURegister fd, FPURegister fs) { rint(S, fd, fs); }
+
+void Assembler::rint_d(FPURegister fd, FPURegister fs) { rint(D, fd, fs); }
+
+void Assembler::rint(SecondaryField fmt, FPURegister fd, FPURegister fs) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrRegister(COP1, fmt, f0, fs, fd, RINT);
+}
+
+void Assembler::cvt_l_s(FPURegister fd, FPURegister fs) {
+ DCHECK(kArchVariant == kMips64r2 || kArchVariant == kMips64r6);
+ GenInstrRegister(COP1, S, f0, fs, fd, CVT_L_S);
+}
+
+void Assembler::cvt_l_d(FPURegister fd, FPURegister fs) {
+ DCHECK(kArchVariant == kMips64r2 || kArchVariant == kMips64r6);
+ GenInstrRegister(COP1, D, f0, fs, fd, CVT_L_D);
+}
+
+void Assembler::trunc_l_s(FPURegister fd, FPURegister fs) {
+ DCHECK(kArchVariant == kMips64r2 || kArchVariant == kMips64r6);
+ GenInstrRegister(COP1, S, f0, fs, fd, TRUNC_L_S);
+}
+
+void Assembler::trunc_l_d(FPURegister fd, FPURegister fs) {
+ DCHECK(kArchVariant == kMips64r2 || kArchVariant == kMips64r6);
+ GenInstrRegister(COP1, D, f0, fs, fd, TRUNC_L_D);
+}
+
+void Assembler::round_l_s(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, S, f0, fs, fd, ROUND_L_S);
+}
+
+void Assembler::round_l_d(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, D, f0, fs, fd, ROUND_L_D);
+}
+
+void Assembler::floor_l_s(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, S, f0, fs, fd, FLOOR_L_S);
+}
+
+void Assembler::floor_l_d(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, D, f0, fs, fd, FLOOR_L_D);
+}
+
+void Assembler::ceil_l_s(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, S, f0, fs, fd, CEIL_L_S);
+}
+
+void Assembler::ceil_l_d(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, D, f0, fs, fd, CEIL_L_D);
+}
+
+void Assembler::class_s(FPURegister fd, FPURegister fs) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrRegister(COP1, S, f0, fs, fd, CLASS_S);
+}
+
+void Assembler::class_d(FPURegister fd, FPURegister fs) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ GenInstrRegister(COP1, D, f0, fs, fd, CLASS_D);
+}
+
+void Assembler::mina(SecondaryField fmt, FPURegister fd, FPURegister fs,
+ FPURegister ft) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK((fmt == D) || (fmt == S));
+ GenInstrRegister(COP1, fmt, ft, fs, fd, MINA);
+}
+
+void Assembler::maxa(SecondaryField fmt, FPURegister fd, FPURegister fs,
+ FPURegister ft) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK((fmt == D) || (fmt == S));
+ GenInstrRegister(COP1, fmt, ft, fs, fd, MAXA);
+}
+
+void Assembler::cvt_s_w(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, W, f0, fs, fd, CVT_S_W);
+}
+
+void Assembler::cvt_s_l(FPURegister fd, FPURegister fs) {
+ DCHECK(kArchVariant == kMips64r2 || kArchVariant == kMips64r6);
+ GenInstrRegister(COP1, L, f0, fs, fd, CVT_S_L);
+}
+
+void Assembler::cvt_s_d(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, D, f0, fs, fd, CVT_S_D);
+}
+
+void Assembler::cvt_d_w(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, W, f0, fs, fd, CVT_D_W);
+}
+
+void Assembler::cvt_d_l(FPURegister fd, FPURegister fs) {
+ DCHECK(kArchVariant == kMips64r2 || kArchVariant == kMips64r6);
+ GenInstrRegister(COP1, L, f0, fs, fd, CVT_D_L);
+}
+
+void Assembler::cvt_d_s(FPURegister fd, FPURegister fs) {
+ GenInstrRegister(COP1, S, f0, fs, fd, CVT_D_S);
+}
+
+// Conditions for >= MIPSr6.
+void Assembler::cmp(FPUCondition cond, SecondaryField fmt, FPURegister fd,
+ FPURegister fs, FPURegister ft) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ DCHECK_EQ(fmt & ~(31 << kRsShift), 0);
+ Instr instr = COP1 | fmt | ft.code() << kFtShift | fs.code() << kFsShift |
+ fd.code() << kFdShift | (0 << 5) | cond;
+ emit(instr);
+}
+
+void Assembler::cmp_s(FPUCondition cond, FPURegister fd, FPURegister fs,
+ FPURegister ft) {
+ cmp(cond, W, fd, fs, ft);
+}
+
+void Assembler::cmp_d(FPUCondition cond, FPURegister fd, FPURegister fs,
+ FPURegister ft) {
+ cmp(cond, L, fd, fs, ft);
+}
+
+void Assembler::bc1eqz(int16_t offset, FPURegister ft) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Instr instr = COP1 | BC1EQZ | ft.code() << kFtShift | (offset & kImm16Mask);
+ emit(instr);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+void Assembler::bc1nez(int16_t offset, FPURegister ft) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Instr instr = COP1 | BC1NEZ | ft.code() << kFtShift | (offset & kImm16Mask);
+ emit(instr);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+// Conditions for < MIPSr6.
+void Assembler::c(FPUCondition cond, SecondaryField fmt, FPURegister fs,
+ FPURegister ft, uint16_t cc) {
+ DCHECK_NE(kArchVariant, kMips64r6);
+ DCHECK(is_uint3(cc));
+ DCHECK(fmt == S || fmt == D);
+ DCHECK_EQ(fmt & ~(31 << kRsShift), 0);
+ Instr instr = COP1 | fmt | ft.code() << kFtShift | fs.code() << kFsShift |
+ cc << 8 | 3 << 4 | cond;
+ emit(instr);
+}
+
+void Assembler::c_s(FPUCondition cond, FPURegister fs, FPURegister ft,
+ uint16_t cc) {
+ c(cond, S, fs, ft, cc);
+}
+
+void Assembler::c_d(FPUCondition cond, FPURegister fs, FPURegister ft,
+ uint16_t cc) {
+ c(cond, D, fs, ft, cc);
+}
+
+void Assembler::fcmp(FPURegister src1, const double src2, FPUCondition cond) {
+ DCHECK_EQ(src2, 0.0);
+ mtc1(zero_reg, f14);
+ cvt_d_w(f14, f14);
+ c(cond, D, src1, f14, 0);
+}
+
+void Assembler::bc1f(int16_t offset, uint16_t cc) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ DCHECK(is_uint3(cc));
+ Instr instr = COP1 | BC1 | cc << 18 | 0 << 16 | (offset & kImm16Mask);
+ emit(instr);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+void Assembler::bc1t(int16_t offset, uint16_t cc) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ DCHECK(is_uint3(cc));
+ Instr instr = COP1 | BC1 | cc << 18 | 1 << 16 | (offset & kImm16Mask);
+ emit(instr);
+ BlockTrampolinePoolFor(1); // For associated delay slot.
+}
+
+// ---------- MSA instructions ------------
+#define MSA_BRANCH_LIST(V) \
+ V(bz_v, BZ_V) \
+ V(bz_b, BZ_B) \
+ V(bz_h, BZ_H) \
+ V(bz_w, BZ_W) \
+ V(bz_d, BZ_D) \
+ V(bnz_v, BNZ_V) \
+ V(bnz_b, BNZ_B) \
+ V(bnz_h, BNZ_H) \
+ V(bnz_w, BNZ_W) \
+ V(bnz_d, BNZ_D)
+
+#define MSA_BRANCH(name, opcode) \
+ void Assembler::name(MSARegister wt, int16_t offset) { \
+ GenInstrMsaBranch(opcode, wt, offset); \
+ }
+
+MSA_BRANCH_LIST(MSA_BRANCH)
+#undef MSA_BRANCH
+#undef MSA_BRANCH_LIST
+
+#define MSA_LD_ST_LIST(V) \
+ V(ld_b, LD_B, 1) \
+ V(ld_h, LD_H, 2) \
+ V(ld_w, LD_W, 4) \
+ V(ld_d, LD_D, 8) \
+ V(st_b, ST_B, 1) \
+ V(st_h, ST_H, 2) \
+ V(st_w, ST_W, 4) \
+ V(st_d, ST_D, 8)
+
+#define MSA_LD_ST(name, opcode, b) \
+ void Assembler::name(MSARegister wd, const MemOperand& rs) { \
+ MemOperand source = rs; \
+ AdjustBaseAndOffset(&source); \
+ if (is_int10(source.offset())) { \
+ DCHECK_EQ(source.offset() % b, 0); \
+ GenInstrMsaMI10(opcode, source.offset() / b, source.rm(), wd); \
+ } else { \
+ UseScratchRegisterScope temps(this); \
+ Register scratch = temps.Acquire(); \
+ DCHECK_NE(rs.rm(), scratch); \
+ daddiu(scratch, source.rm(), source.offset()); \
+ GenInstrMsaMI10(opcode, 0, scratch, wd); \
+ } \
+ }
+
+MSA_LD_ST_LIST(MSA_LD_ST)
+#undef MSA_LD_ST
+#undef MSA_LD_ST_LIST
+
+#define MSA_I10_LIST(V) \
+ V(ldi_b, I5_DF_b) \
+ V(ldi_h, I5_DF_h) \
+ V(ldi_w, I5_DF_w) \
+ V(ldi_d, I5_DF_d)
+
+#define MSA_I10(name, format) \
+ void Assembler::name(MSARegister wd, int32_t imm10) { \
+ GenInstrMsaI10(LDI, format, imm10, wd); \
+ }
+MSA_I10_LIST(MSA_I10)
+#undef MSA_I10
+#undef MSA_I10_LIST
+
+#define MSA_I5_LIST(V) \
+ V(addvi, ADDVI) \
+ V(subvi, SUBVI) \
+ V(maxi_s, MAXI_S) \
+ V(maxi_u, MAXI_U) \
+ V(mini_s, MINI_S) \
+ V(mini_u, MINI_U) \
+ V(ceqi, CEQI) \
+ V(clti_s, CLTI_S) \
+ V(clti_u, CLTI_U) \
+ V(clei_s, CLEI_S) \
+ V(clei_u, CLEI_U)
+
+#define MSA_I5_FORMAT(name, opcode, format) \
+ void Assembler::name##_##format(MSARegister wd, MSARegister ws, \
+ uint32_t imm5) { \
+ GenInstrMsaI5(opcode, I5_DF_##format, imm5, ws, wd); \
+ }
+
+#define MSA_I5(name, opcode) \
+ MSA_I5_FORMAT(name, opcode, b) \
+ MSA_I5_FORMAT(name, opcode, h) \
+ MSA_I5_FORMAT(name, opcode, w) \
+ MSA_I5_FORMAT(name, opcode, d)
+
+MSA_I5_LIST(MSA_I5)
+#undef MSA_I5
+#undef MSA_I5_FORMAT
+#undef MSA_I5_LIST
+
+#define MSA_I8_LIST(V) \
+ V(andi_b, ANDI_B) \
+ V(ori_b, ORI_B) \
+ V(nori_b, NORI_B) \
+ V(xori_b, XORI_B) \
+ V(bmnzi_b, BMNZI_B) \
+ V(bmzi_b, BMZI_B) \
+ V(bseli_b, BSELI_B) \
+ V(shf_b, SHF_B) \
+ V(shf_h, SHF_H) \
+ V(shf_w, SHF_W)
+
+#define MSA_I8(name, opcode) \
+ void Assembler::name(MSARegister wd, MSARegister ws, uint32_t imm8) { \
+ GenInstrMsaI8(opcode, imm8, ws, wd); \
+ }
+
+MSA_I8_LIST(MSA_I8)
+#undef MSA_I8
+#undef MSA_I8_LIST
+
+#define MSA_VEC_LIST(V) \
+ V(and_v, AND_V) \
+ V(or_v, OR_V) \
+ V(nor_v, NOR_V) \
+ V(xor_v, XOR_V) \
+ V(bmnz_v, BMNZ_V) \
+ V(bmz_v, BMZ_V) \
+ V(bsel_v, BSEL_V)
+
+#define MSA_VEC(name, opcode) \
+ void Assembler::name(MSARegister wd, MSARegister ws, MSARegister wt) { \
+ GenInstrMsaVec(opcode, wt, ws, wd); \
+ }
+
+MSA_VEC_LIST(MSA_VEC)
+#undef MSA_VEC
+#undef MSA_VEC_LIST
+
+#define MSA_2R_LIST(V) \
+ V(pcnt, PCNT) \
+ V(nloc, NLOC) \
+ V(nlzc, NLZC)
+
+#define MSA_2R_FORMAT(name, opcode, format) \
+ void Assembler::name##_##format(MSARegister wd, MSARegister ws) { \
+ GenInstrMsa2R(opcode, MSA_2R_DF_##format, ws, wd); \
+ }
+
+#define MSA_2R(name, opcode) \
+ MSA_2R_FORMAT(name, opcode, b) \
+ MSA_2R_FORMAT(name, opcode, h) \
+ MSA_2R_FORMAT(name, opcode, w) \
+ MSA_2R_FORMAT(name, opcode, d)
+
+MSA_2R_LIST(MSA_2R)
+#undef MSA_2R
+#undef MSA_2R_FORMAT
+#undef MSA_2R_LIST
+
+#define MSA_FILL(format) \
+ void Assembler::fill_##format(MSARegister wd, Register rs) { \
+ DCHECK(IsEnabled(MIPS_SIMD)); \
+ DCHECK(rs.is_valid() && wd.is_valid()); \
+ Instr instr = MSA | MSA_2R_FORMAT | FILL | MSA_2R_DF_##format | \
+ (rs.code() << kWsShift) | (wd.code() << kWdShift) | \
+ MSA_VEC_2R_2RF_MINOR; \
+ emit(instr); \
+ }
+
+MSA_FILL(b)
+MSA_FILL(h)
+MSA_FILL(w)
+MSA_FILL(d)
+#undef MSA_FILL
+
+#define MSA_2RF_LIST(V) \
+ V(fclass, FCLASS) \
+ V(ftrunc_s, FTRUNC_S) \
+ V(ftrunc_u, FTRUNC_U) \
+ V(fsqrt, FSQRT) \
+ V(frsqrt, FRSQRT) \
+ V(frcp, FRCP) \
+ V(frint, FRINT) \
+ V(flog2, FLOG2) \
+ V(fexupl, FEXUPL) \
+ V(fexupr, FEXUPR) \
+ V(ffql, FFQL) \
+ V(ffqr, FFQR) \
+ V(ftint_s, FTINT_S) \
+ V(ftint_u, FTINT_U) \
+ V(ffint_s, FFINT_S) \
+ V(ffint_u, FFINT_U)
+
+#define MSA_2RF_FORMAT(name, opcode, format) \
+ void Assembler::name##_##format(MSARegister wd, MSARegister ws) { \
+ GenInstrMsa2RF(opcode, MSA_2RF_DF_##format, ws, wd); \
+ }
+
+#define MSA_2RF(name, opcode) \
+ MSA_2RF_FORMAT(name, opcode, w) \
+ MSA_2RF_FORMAT(name, opcode, d)
+
+MSA_2RF_LIST(MSA_2RF)
+#undef MSA_2RF
+#undef MSA_2RF_FORMAT
+#undef MSA_2RF_LIST
+
+#define MSA_3R_LIST(V) \
+ V(sll, SLL_MSA) \
+ V(sra, SRA_MSA) \
+ V(srl, SRL_MSA) \
+ V(bclr, BCLR) \
+ V(bset, BSET) \
+ V(bneg, BNEG) \
+ V(binsl, BINSL) \
+ V(binsr, BINSR) \
+ V(addv, ADDV) \
+ V(subv, SUBV) \
+ V(max_s, MAX_S) \
+ V(max_u, MAX_U) \
+ V(min_s, MIN_S) \
+ V(min_u, MIN_U) \
+ V(max_a, MAX_A) \
+ V(min_a, MIN_A) \
+ V(ceq, CEQ) \
+ V(clt_s, CLT_S) \
+ V(clt_u, CLT_U) \
+ V(cle_s, CLE_S) \
+ V(cle_u, CLE_U) \
+ V(add_a, ADD_A) \
+ V(adds_a, ADDS_A) \
+ V(adds_s, ADDS_S) \
+ V(adds_u, ADDS_U) \
+ V(ave_s, AVE_S) \
+ V(ave_u, AVE_U) \
+ V(aver_s, AVER_S) \
+ V(aver_u, AVER_U) \
+ V(subs_s, SUBS_S) \
+ V(subs_u, SUBS_U) \
+ V(subsus_u, SUBSUS_U) \
+ V(subsuu_s, SUBSUU_S) \
+ V(asub_s, ASUB_S) \
+ V(asub_u, ASUB_U) \
+ V(mulv, MULV) \
+ V(maddv, MADDV) \
+ V(msubv, MSUBV) \
+ V(div_s, DIV_S_MSA) \
+ V(div_u, DIV_U) \
+ V(mod_s, MOD_S) \
+ V(mod_u, MOD_U) \
+ V(dotp_s, DOTP_S) \
+ V(dotp_u, DOTP_U) \
+ V(dpadd_s, DPADD_S) \
+ V(dpadd_u, DPADD_U) \
+ V(dpsub_s, DPSUB_S) \
+ V(dpsub_u, DPSUB_U) \
+ V(pckev, PCKEV) \
+ V(pckod, PCKOD) \
+ V(ilvl, ILVL) \
+ V(ilvr, ILVR) \
+ V(ilvev, ILVEV) \
+ V(ilvod, ILVOD) \
+ V(vshf, VSHF) \
+ V(srar, SRAR) \
+ V(srlr, SRLR) \
+ V(hadd_s, HADD_S) \
+ V(hadd_u, HADD_U) \
+ V(hsub_s, HSUB_S) \
+ V(hsub_u, HSUB_U)
+
+#define MSA_3R_FORMAT(name, opcode, format) \
+ void Assembler::name##_##format(MSARegister wd, MSARegister ws, \
+ MSARegister wt) { \
+ GenInstrMsa3R<MSARegister>(opcode, MSA_3R_DF_##format, wt, ws, wd); \
+ }
+
+#define MSA_3R_FORMAT_SLD_SPLAT(name, opcode, format) \
+ void Assembler::name##_##format(MSARegister wd, MSARegister ws, \
+ Register rt) { \
+ GenInstrMsa3R<Register>(opcode, MSA_3R_DF_##format, rt, ws, wd); \
+ }
+
+#define MSA_3R(name, opcode) \
+ MSA_3R_FORMAT(name, opcode, b) \
+ MSA_3R_FORMAT(name, opcode, h) \
+ MSA_3R_FORMAT(name, opcode, w) \
+ MSA_3R_FORMAT(name, opcode, d)
+
+#define MSA_3R_SLD_SPLAT(name, opcode) \
+ MSA_3R_FORMAT_SLD_SPLAT(name, opcode, b) \
+ MSA_3R_FORMAT_SLD_SPLAT(name, opcode, h) \
+ MSA_3R_FORMAT_SLD_SPLAT(name, opcode, w) \
+ MSA_3R_FORMAT_SLD_SPLAT(name, opcode, d)
+
+MSA_3R_LIST(MSA_3R)
+MSA_3R_SLD_SPLAT(sld, SLD)
+MSA_3R_SLD_SPLAT(splat, SPLAT)
+
+#undef MSA_3R
+#undef MSA_3R_FORMAT
+#undef MSA_3R_FORMAT_SLD_SPLAT
+#undef MSA_3R_SLD_SPLAT
+#undef MSA_3R_LIST
+
+#define MSA_3RF_LIST1(V) \
+ V(fcaf, FCAF) \
+ V(fcun, FCUN) \
+ V(fceq, FCEQ) \
+ V(fcueq, FCUEQ) \
+ V(fclt, FCLT) \
+ V(fcult, FCULT) \
+ V(fcle, FCLE) \
+ V(fcule, FCULE) \
+ V(fsaf, FSAF) \
+ V(fsun, FSUN) \
+ V(fseq, FSEQ) \
+ V(fsueq, FSUEQ) \
+ V(fslt, FSLT) \
+ V(fsult, FSULT) \
+ V(fsle, FSLE) \
+ V(fsule, FSULE) \
+ V(fadd, FADD) \
+ V(fsub, FSUB) \
+ V(fmul, FMUL) \
+ V(fdiv, FDIV) \
+ V(fmadd, FMADD) \
+ V(fmsub, FMSUB) \
+ V(fexp2, FEXP2) \
+ V(fmin, FMIN) \
+ V(fmin_a, FMIN_A) \
+ V(fmax, FMAX) \
+ V(fmax_a, FMAX_A) \
+ V(fcor, FCOR) \
+ V(fcune, FCUNE) \
+ V(fcne, FCNE) \
+ V(fsor, FSOR) \
+ V(fsune, FSUNE) \
+ V(fsne, FSNE)
+
+#define MSA_3RF_LIST2(V) \
+ V(fexdo, FEXDO) \
+ V(ftq, FTQ) \
+ V(mul_q, MUL_Q) \
+ V(madd_q, MADD_Q) \
+ V(msub_q, MSUB_Q) \
+ V(mulr_q, MULR_Q) \
+ V(maddr_q, MADDR_Q) \
+ V(msubr_q, MSUBR_Q)
+
+#define MSA_3RF_FORMAT(name, opcode, df, df_c) \
+ void Assembler::name##_##df(MSARegister wd, MSARegister ws, \
+ MSARegister wt) { \
+ GenInstrMsa3RF(opcode, df_c, wt, ws, wd); \
+ }
+
+#define MSA_3RF_1(name, opcode) \
+ MSA_3RF_FORMAT(name, opcode, w, 0) \
+ MSA_3RF_FORMAT(name, opcode, d, 1)
+
+#define MSA_3RF_2(name, opcode) \
+ MSA_3RF_FORMAT(name, opcode, h, 0) \
+ MSA_3RF_FORMAT(name, opcode, w, 1)
+
+MSA_3RF_LIST1(MSA_3RF_1)
+MSA_3RF_LIST2(MSA_3RF_2)
+#undef MSA_3RF_1
+#undef MSA_3RF_2
+#undef MSA_3RF_FORMAT
+#undef MSA_3RF_LIST1
+#undef MSA_3RF_LIST2
+
+void Assembler::sldi_b(MSARegister wd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<MSARegister, MSARegister>(SLDI, ELM_DF_B, n, ws, wd);
+}
+
+void Assembler::sldi_h(MSARegister wd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<MSARegister, MSARegister>(SLDI, ELM_DF_H, n, ws, wd);
+}
+
+void Assembler::sldi_w(MSARegister wd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<MSARegister, MSARegister>(SLDI, ELM_DF_W, n, ws, wd);
+}
+
+void Assembler::sldi_d(MSARegister wd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<MSARegister, MSARegister>(SLDI, ELM_DF_D, n, ws, wd);
+}
+
+void Assembler::splati_b(MSARegister wd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<MSARegister, MSARegister>(SPLATI, ELM_DF_B, n, ws, wd);
+}
+
+void Assembler::splati_h(MSARegister wd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<MSARegister, MSARegister>(SPLATI, ELM_DF_H, n, ws, wd);
+}
+
+void Assembler::splati_w(MSARegister wd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<MSARegister, MSARegister>(SPLATI, ELM_DF_W, n, ws, wd);
+}
+
+void Assembler::splati_d(MSARegister wd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<MSARegister, MSARegister>(SPLATI, ELM_DF_D, n, ws, wd);
+}
+
+void Assembler::copy_s_b(Register rd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<Register, MSARegister>(COPY_S, ELM_DF_B, n, ws, rd);
+}
+
+void Assembler::copy_s_h(Register rd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<Register, MSARegister>(COPY_S, ELM_DF_H, n, ws, rd);
+}
+
+void Assembler::copy_s_w(Register rd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<Register, MSARegister>(COPY_S, ELM_DF_W, n, ws, rd);
+}
+
+void Assembler::copy_s_d(Register rd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<Register, MSARegister>(COPY_S, ELM_DF_D, n, ws, rd);
+}
+
+void Assembler::copy_u_b(Register rd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<Register, MSARegister>(COPY_U, ELM_DF_B, n, ws, rd);
+}
+
+void Assembler::copy_u_h(Register rd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<Register, MSARegister>(COPY_U, ELM_DF_H, n, ws, rd);
+}
+
+void Assembler::copy_u_w(Register rd, MSARegister ws, uint32_t n) {
+ GenInstrMsaElm<Register, MSARegister>(COPY_U, ELM_DF_W, n, ws, rd);
+}
+
+void Assembler::insert_b(MSARegister wd, uint32_t n, Register rs) {
+ GenInstrMsaElm<MSARegister, Register>(INSERT, ELM_DF_B, n, rs, wd);
+}
+
+void Assembler::insert_h(MSARegister wd, uint32_t n, Register rs) {
+ GenInstrMsaElm<MSARegister, Register>(INSERT, ELM_DF_H, n, rs, wd);
+}
+
+void Assembler::insert_w(MSARegister wd, uint32_t n, Register rs) {
+ GenInstrMsaElm<MSARegister, Register>(INSERT, ELM_DF_W, n, rs, wd);
+}
+
+void Assembler::insert_d(MSARegister wd, uint32_t n, Register rs) {
+ GenInstrMsaElm<MSARegister, Register>(INSERT, ELM_DF_D, n, rs, wd);
+}
+
+void Assembler::insve_b(MSARegister wd, uint32_t n, MSARegister ws) {
+ GenInstrMsaElm<MSARegister, MSARegister>(INSVE, ELM_DF_B, n, ws, wd);
+}
+
+void Assembler::insve_h(MSARegister wd, uint32_t n, MSARegister ws) {
+ GenInstrMsaElm<MSARegister, MSARegister>(INSVE, ELM_DF_H, n, ws, wd);
+}
+
+void Assembler::insve_w(MSARegister wd, uint32_t n, MSARegister ws) {
+ GenInstrMsaElm<MSARegister, MSARegister>(INSVE, ELM_DF_W, n, ws, wd);
+}
+
+void Assembler::insve_d(MSARegister wd, uint32_t n, MSARegister ws) {
+ GenInstrMsaElm<MSARegister, MSARegister>(INSVE, ELM_DF_D, n, ws, wd);
+}
+
+void Assembler::move_v(MSARegister wd, MSARegister ws) {
+ DCHECK(IsEnabled(MIPS_SIMD));
+ DCHECK(ws.is_valid() && wd.is_valid());
+ Instr instr = MSA | MOVE_V | (ws.code() << kWsShift) |
+ (wd.code() << kWdShift) | MSA_ELM_MINOR;
+ emit(instr);
+}
+
+void Assembler::ctcmsa(MSAControlRegister cd, Register rs) {
+ DCHECK(IsEnabled(MIPS_SIMD));
+ DCHECK(cd.is_valid() && rs.is_valid());
+ Instr instr = MSA | CTCMSA | (rs.code() << kWsShift) |
+ (cd.code() << kWdShift) | MSA_ELM_MINOR;
+ emit(instr);
+}
+
+void Assembler::cfcmsa(Register rd, MSAControlRegister cs) {
+ DCHECK(IsEnabled(MIPS_SIMD));
+ DCHECK(rd.is_valid() && cs.is_valid());
+ Instr instr = MSA | CFCMSA | (cs.code() << kWsShift) |
+ (rd.code() << kWdShift) | MSA_ELM_MINOR;
+ emit(instr);
+}
+
+#define MSA_BIT_LIST(V) \
+ V(slli, SLLI) \
+ V(srai, SRAI) \
+ V(srli, SRLI) \
+ V(bclri, BCLRI) \
+ V(bseti, BSETI) \
+ V(bnegi, BNEGI) \
+ V(binsli, BINSLI) \
+ V(binsri, BINSRI) \
+ V(sat_s, SAT_S) \
+ V(sat_u, SAT_U) \
+ V(srari, SRARI) \
+ V(srlri, SRLRI)
+
+#define MSA_BIT_FORMAT(name, opcode, format) \
+ void Assembler::name##_##format(MSARegister wd, MSARegister ws, \
+ uint32_t m) { \
+ GenInstrMsaBit(opcode, BIT_DF_##format, m, ws, wd); \
+ }
+
+#define MSA_BIT(name, opcode) \
+ MSA_BIT_FORMAT(name, opcode, b) \
+ MSA_BIT_FORMAT(name, opcode, h) \
+ MSA_BIT_FORMAT(name, opcode, w) \
+ MSA_BIT_FORMAT(name, opcode, d)
+
+MSA_BIT_LIST(MSA_BIT)
+#undef MSA_BIT
+#undef MSA_BIT_FORMAT
+#undef MSA_BIT_LIST
+
+int Assembler::RelocateInternalReference(RelocInfo::Mode rmode, Address pc,
+ intptr_t pc_delta) {
+ if (RelocInfo::IsInternalReference(rmode)) {
+ int64_t* p = reinterpret_cast<int64_t*>(pc);
+ if (*p == kEndOfJumpChain) {
+ return 0; // Number of instructions patched.
+ }
+ *p += pc_delta;
+ return 2; // Number of instructions patched.
+ }
+ Instr instr = instr_at(pc);
+ DCHECK(RelocInfo::IsInternalReferenceEncoded(rmode));
+ if (IsLui(instr)) {
+ Instr instr_lui = instr_at(pc + 0 * kInstrSize);
+ Instr instr_ori = instr_at(pc + 1 * kInstrSize);
+ Instr instr_ori2 = instr_at(pc + 3 * kInstrSize);
+ DCHECK(IsOri(instr_ori));
+ DCHECK(IsOri(instr_ori2));
+ // TODO(plind): symbolic names for the shifts.
+ int64_t imm = (instr_lui & static_cast<int64_t>(kImm16Mask)) << 48;
+ imm |= (instr_ori & static_cast<int64_t>(kImm16Mask)) << 32;
+ imm |= (instr_ori2 & static_cast<int64_t>(kImm16Mask)) << 16;
+ // Sign extend address.
+ imm >>= 16;
+
+ if (imm == kEndOfJumpChain) {
+ return 0; // Number of instructions patched.
+ }
+ imm += pc_delta;
+ DCHECK_EQ(imm & 3, 0);
+
+ instr_lui &= ~kImm16Mask;
+ instr_ori &= ~kImm16Mask;
+ instr_ori2 &= ~kImm16Mask;
+
+ instr_at_put(pc + 0 * kInstrSize, instr_lui | ((imm >> 32) & kImm16Mask));
+ instr_at_put(pc + 1 * kInstrSize, instr_ori | (imm >> 16 & kImm16Mask));
+ instr_at_put(pc + 3 * kInstrSize, instr_ori2 | (imm & kImm16Mask));
+ return 4; // Number of instructions patched.
+ } else if (IsJ(instr) || IsJal(instr)) {
+ // Regular j/jal relocation.
+ uint32_t imm28 = (instr & static_cast<int32_t>(kImm26Mask)) << 2;
+ imm28 += pc_delta;
+ imm28 &= kImm28Mask;
+ instr &= ~kImm26Mask;
+ DCHECK_EQ(imm28 & 3, 0);
+ uint32_t imm26 = static_cast<uint32_t>(imm28 >> 2);
+ instr_at_put(pc, instr | (imm26 & kImm26Mask));
+ return 1; // Number of instructions patched.
+ } else {
+ DCHECK(((instr & kJumpRawMask) == kJRawMark) ||
+ ((instr & kJumpRawMask) == kJalRawMark));
+ // Unbox raw offset and emit j/jal.
+ int32_t imm28 = (instr & static_cast<int32_t>(kImm26Mask)) << 2;
+ // Sign extend 28-bit offset to 32-bit.
+ imm28 = (imm28 << 4) >> 4;
+ uint64_t target =
+ static_cast<int64_t>(imm28) + reinterpret_cast<uint64_t>(pc);
+ target &= kImm28Mask;
+ DCHECK_EQ(imm28 & 3, 0);
+ uint32_t imm26 = static_cast<uint32_t>(target >> 2);
+ // Check markings whether to emit j or jal.
+ uint32_t unbox = (instr & kJRawMark) ? J : JAL;
+ instr_at_put(pc, unbox | (imm26 & kImm26Mask));
+ return 1; // Number of instructions patched.
+ }
+}
+
+void Assembler::GrowBuffer() {
+ // Compute new buffer size.
+ int old_size = buffer_->size();
+ int new_size = std::min(2 * old_size, old_size + 1 * MB);
+
+ // Some internal data structures overflow for very large buffers,
+ // they must ensure that kMaximalBufferSize is not too large.
+ if (new_size > kMaximalBufferSize) {
+ V8::FatalProcessOutOfMemory(nullptr, "Assembler::GrowBuffer");
+ }
+
+ // Set up new buffer.
+ std::unique_ptr<AssemblerBuffer> new_buffer = buffer_->Grow(new_size);
+ DCHECK_EQ(new_size, new_buffer->size());
+ byte* new_start = new_buffer->start();
+
+ // Copy the data.
+ intptr_t pc_delta = new_start - buffer_start_;
+ intptr_t rc_delta = (new_start + new_size) - (buffer_start_ + old_size);
+ size_t reloc_size = (buffer_start_ + old_size) - reloc_info_writer.pos();
+ MemMove(new_start, buffer_start_, pc_offset());
+ MemMove(reloc_info_writer.pos() + rc_delta, reloc_info_writer.pos(),
+ reloc_size);
+
+ // Switch buffers.
+ buffer_ = std::move(new_buffer);
+ buffer_start_ = new_start;
+ pc_ += pc_delta;
+ last_call_pc_ += pc_delta;
+ reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta,
+ reloc_info_writer.last_pc() + pc_delta);
+
+ // Relocate runtime entries.
+ Vector<byte> instructions{buffer_start_, pc_offset()};
+ Vector<const byte> reloc_info{reloc_info_writer.pos(), reloc_size};
+ for (RelocIterator it(instructions, reloc_info, 0); !it.done(); it.next()) {
+ RelocInfo::Mode rmode = it.rinfo()->rmode();
+ if (rmode == RelocInfo::INTERNAL_REFERENCE) {
+ RelocateInternalReference(rmode, it.rinfo()->pc(), pc_delta);
+ }
+ }
+ DCHECK(!overflow());
+}
+
+void Assembler::db(uint8_t data) {
+ CheckForEmitInForbiddenSlot();
+ *reinterpret_cast<uint8_t*>(pc_) = data;
+ pc_ += sizeof(uint8_t);
+}
+
+void Assembler::dd(uint32_t data) {
+ CheckForEmitInForbiddenSlot();
+ *reinterpret_cast<uint32_t*>(pc_) = data;
+ pc_ += sizeof(uint32_t);
+}
+
+void Assembler::dq(uint64_t data) {
+ CheckForEmitInForbiddenSlot();
+ *reinterpret_cast<uint64_t*>(pc_) = data;
+ pc_ += sizeof(uint64_t);
+}
+
+void Assembler::dd(Label* label) {
+ uint64_t data;
+ CheckForEmitInForbiddenSlot();
+ if (label->is_bound()) {
+ data = reinterpret_cast<uint64_t>(buffer_start_ + label->pos());
+ } else {
+ data = jump_address(label);
+ unbound_labels_count_++;
+ internal_reference_positions_.insert(label->pos());
+ }
+ RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE);
+ EmitHelper(data);
+}
+
+void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) {
+ if (!ShouldRecordRelocInfo(rmode)) return;
+ // We do not try to reuse pool constants.
+ RelocInfo rinfo(reinterpret_cast<Address>(pc_), rmode, data, Code());
+ DCHECK_GE(buffer_space(), kMaxRelocSize); // Too late to grow buffer here.
+ reloc_info_writer.Write(&rinfo);
+}
+
+void Assembler::BlockTrampolinePoolFor(int instructions) {
+ CheckTrampolinePoolQuick(instructions);
+ BlockTrampolinePoolBefore(pc_offset() + instructions * kInstrSize);
+}
+
+void Assembler::CheckTrampolinePool() {
+ // Some small sequences of instructions must not be broken up by the
+ // insertion of a trampoline pool; such sequences are protected by setting
+ // either trampoline_pool_blocked_nesting_ or no_trampoline_pool_before_,
+ // which are both checked here. Also, recursive calls to CheckTrampolinePool
+ // are blocked by trampoline_pool_blocked_nesting_.
+ if ((trampoline_pool_blocked_nesting_ > 0) ||
+ (pc_offset() < no_trampoline_pool_before_)) {
+ // Emission is currently blocked; make sure we try again as soon as
+ // possible.
+ if (trampoline_pool_blocked_nesting_ > 0) {
+ next_buffer_check_ = pc_offset() + kInstrSize;
+ } else {
+ next_buffer_check_ = no_trampoline_pool_before_;
+ }
+ return;
+ }
+
+ DCHECK(!trampoline_emitted_);
+ DCHECK_GE(unbound_labels_count_, 0);
+ if (unbound_labels_count_ > 0) {
+ // First we emit jump (2 instructions), then we emit trampoline pool.
+ {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Label after_pool;
+ if (kArchVariant == kMips64r6) {
+ bc(&after_pool);
+ } else {
+ b(&after_pool);
+ }
+ nop();
+
+ int pool_start = pc_offset();
+ for (int i = 0; i < unbound_labels_count_; i++) {
+ {
+ if (kArchVariant == kMips64r6) {
+ bc(&after_pool);
+ nop();
+ } else {
+ or_(t8, ra, zero_reg);
+ nal(); // Read PC into ra register.
+ lui(t9, 0); // Branch delay slot.
+ ori(t9, t9, 0);
+ daddu(t9, ra, t9);
+ or_(ra, t8, zero_reg);
+ // Instruction jr will take or_ from the next trampoline.
+ // in its branch delay slot. This is the expected behavior
+ // in order to decrease size of trampoline pool.
+ jr(t9);
+ }
+ }
+ }
+ nop();
+ // If unbound_labels_count_ is big enough, label after_pool will
+ // need a trampoline too, so we must create the trampoline before
+ // the bind operation to make sure function 'bind' can get this
+ // information.
+ trampoline_ = Trampoline(pool_start, unbound_labels_count_);
+ bind(&after_pool);
+
+ trampoline_emitted_ = true;
+ // As we are only going to emit trampoline once, we need to prevent any
+ // further emission.
+ next_buffer_check_ = kMaxInt;
+ }
+ } else {
+ // Number of branches to unbound label at this point is zero, so we can
+ // move next buffer check to maximum.
+ next_buffer_check_ =
+ pc_offset() + kMaxBranchOffset - kTrampolineSlotsSize * 16;
+ }
+ return;
+}
+
+Address Assembler::target_address_at(Address pc) {
+ Instr instr0 = instr_at(pc);
+ Instr instr1 = instr_at(pc + 1 * kInstrSize);
+ Instr instr3 = instr_at(pc + 3 * kInstrSize);
+
+ // Interpret 4 instructions for address generated by li: See listing in
+ // Assembler::set_target_address_at() just below.
+ if ((GetOpcodeField(instr0) == LUI) && (GetOpcodeField(instr1) == ORI) &&
+ (GetOpcodeField(instr3) == ORI)) {
+ // Assemble the 48 bit value.
+ int64_t addr =
+ static_cast<int64_t>(((uint64_t)(GetImmediate16(instr0)) << 32) |
+ ((uint64_t)(GetImmediate16(instr1)) << 16) |
+ ((uint64_t)(GetImmediate16(instr3))));
+
+ // Sign extend to get canonical address.
+ addr = (addr << 16) >> 16;
+ return static_cast<Address>(addr);
+ }
+ // We should never get here, force a bad address if we do.
+ UNREACHABLE();
+}
+
+// On Mips64, a target address is stored in a 4-instruction sequence:
+// 0: lui(rd, (j.imm64_ >> 32) & kImm16Mask);
+// 1: ori(rd, rd, (j.imm64_ >> 16) & kImm16Mask);
+// 2: dsll(rd, rd, 16);
+// 3: ori(rd, rd, j.imm32_ & kImm16Mask);
+//
+// Patching the address must replace all the lui & ori instructions,
+// and flush the i-cache.
+//
+// There is an optimization below, which emits a nop when the address
+// fits in just 16 bits. This is unlikely to help, and should be benchmarked,
+// and possibly removed.
+void Assembler::set_target_value_at(Address pc, uint64_t target,
+ ICacheFlushMode icache_flush_mode) {
+ // There is an optimization where only 4 instructions are used to load address
+ // in code on MIP64 because only 48-bits of address is effectively used.
+ // It relies on fact the upper [63:48] bits are not used for virtual address
+ // translation and they have to be set according to value of bit 47 in order
+ // get canonical address.
+ Instr instr1 = instr_at(pc + kInstrSize);
+ uint32_t rt_code = GetRt(instr1);
+ uint32_t* p = reinterpret_cast<uint32_t*>(pc);
+
+#ifdef DEBUG
+ // Check we have the result from a li macro-instruction.
+ Instr instr0 = instr_at(pc);
+ Instr instr3 = instr_at(pc + kInstrSize * 3);
+ DCHECK((GetOpcodeField(instr0) == LUI && GetOpcodeField(instr1) == ORI &&
+ GetOpcodeField(instr3) == ORI));
+#endif
+
+ // Must use 4 instructions to insure patchable code.
+ // lui rt, upper-16.
+ // ori rt, rt, lower-16.
+ // dsll rt, rt, 16.
+ // ori rt rt, lower-16.
+ *p = LUI | (rt_code << kRtShift) | ((target >> 32) & kImm16Mask);
+ *(p + 1) = ORI | (rt_code << kRtShift) | (rt_code << kRsShift) |
+ ((target >> 16) & kImm16Mask);
+ *(p + 3) = ORI | (rt_code << kRsShift) | (rt_code << kRtShift) |
+ (target & kImm16Mask);
+
+ if (icache_flush_mode != SKIP_ICACHE_FLUSH) {
+ FlushInstructionCache(pc, 4 * kInstrSize);
+ }
+}
+
+UseScratchRegisterScope::UseScratchRegisterScope(Assembler* assembler)
+ : available_(assembler->GetScratchRegisterList()),
+ old_available_(*available_) {}
+
+UseScratchRegisterScope::~UseScratchRegisterScope() {
+ *available_ = old_available_;
+}
+
+Register UseScratchRegisterScope::Acquire() {
+ DCHECK_NOT_NULL(available_);
+ DCHECK_NE(*available_, 0);
+ int index = static_cast<int>(base::bits::CountTrailingZeros32(*available_));
+ *available_ &= ~(1UL << index);
+
+ return Register::from_code(index);
+}
+
+bool UseScratchRegisterScope::hasAvailable() const { return *available_ != 0; }
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_MIPS64
diff --git a/src/codegen/mips64/assembler-mips64.h b/src/codegen/mips64/assembler-mips64.h
new file mode 100644
index 0000000..b5edc75
--- /dev/null
+++ b/src/codegen/mips64/assembler-mips64.h
@@ -0,0 +1,1955 @@
+// Copyright (c) 1994-2006 Sun Microsystems Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// - Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// - Redistribution in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// - Neither the name of Sun Microsystems or the names of contributors may
+// be used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// The original source code covered by the above license above has been
+// modified significantly by Google Inc.
+// Copyright 2012 the V8 project authors. All rights reserved.
+
+#ifndef V8_CODEGEN_MIPS64_ASSEMBLER_MIPS64_H_
+#define V8_CODEGEN_MIPS64_ASSEMBLER_MIPS64_H_
+
+#include <stdio.h>
+#include <memory>
+#include <set>
+
+#include "src/codegen/assembler.h"
+#include "src/codegen/external-reference.h"
+#include "src/codegen/label.h"
+#include "src/codegen/mips64/constants-mips64.h"
+#include "src/codegen/mips64/register-mips64.h"
+#include "src/objects/contexts.h"
+#include "src/objects/smi.h"
+
+namespace v8 {
+namespace internal {
+
+class SafepointTableBuilder;
+
+// -----------------------------------------------------------------------------
+// Machine instruction Operands.
+constexpr int kSmiShift = kSmiTagSize + kSmiShiftSize;
+constexpr uint64_t kSmiShiftMask = (1UL << kSmiShift) - 1;
+// Class Operand represents a shifter operand in data processing instructions.
+class Operand {
+ public:
+ // Immediate.
+ V8_INLINE explicit Operand(int64_t immediate,
+ RelocInfo::Mode rmode = RelocInfo::NONE)
+ : rm_(no_reg), rmode_(rmode) {
+ value_.immediate = immediate;
+ }
+ V8_INLINE explicit Operand(const ExternalReference& f)
+ : rm_(no_reg), rmode_(RelocInfo::EXTERNAL_REFERENCE) {
+ value_.immediate = static_cast<int64_t>(f.address());
+ }
+ V8_INLINE explicit Operand(const char* s);
+ explicit Operand(Handle<HeapObject> handle);
+ V8_INLINE explicit Operand(Smi value) : rm_(no_reg), rmode_(RelocInfo::NONE) {
+ value_.immediate = static_cast<intptr_t>(value.ptr());
+ }
+
+ static Operand EmbeddedNumber(double number); // Smi or HeapNumber.
+ static Operand EmbeddedStringConstant(const StringConstantBase* str);
+
+ // Register.
+ V8_INLINE explicit Operand(Register rm) : rm_(rm) {}
+
+ // Return true if this is a register operand.
+ V8_INLINE bool is_reg() const;
+
+ inline int64_t immediate() const;
+
+ bool IsImmediate() const { return !rm_.is_valid(); }
+
+ HeapObjectRequest heap_object_request() const {
+ DCHECK(IsHeapObjectRequest());
+ return value_.heap_object_request;
+ }
+
+ bool IsHeapObjectRequest() const {
+ DCHECK_IMPLIES(is_heap_object_request_, IsImmediate());
+ DCHECK_IMPLIES(is_heap_object_request_,
+ rmode_ == RelocInfo::FULL_EMBEDDED_OBJECT ||
+ rmode_ == RelocInfo::CODE_TARGET);
+ return is_heap_object_request_;
+ }
+
+ Register rm() const { return rm_; }
+
+ RelocInfo::Mode rmode() const { return rmode_; }
+
+ private:
+ Register rm_;
+ union Value {
+ Value() {}
+ HeapObjectRequest heap_object_request; // if is_heap_object_request_
+ int64_t immediate; // otherwise
+ } value_; // valid if rm_ == no_reg
+ bool is_heap_object_request_ = false;
+ RelocInfo::Mode rmode_;
+
+ friend class Assembler;
+ friend class MacroAssembler;
+};
+
+// On MIPS we have only one addressing mode with base_reg + offset.
+// Class MemOperand represents a memory operand in load and store instructions.
+class V8_EXPORT_PRIVATE MemOperand : public Operand {
+ public:
+ // Immediate value attached to offset.
+ enum OffsetAddend { offset_minus_one = -1, offset_zero = 0 };
+
+ explicit MemOperand(Register rn, int32_t offset = 0);
+ explicit MemOperand(Register rn, int32_t unit, int32_t multiplier,
+ OffsetAddend offset_addend = offset_zero);
+ int32_t offset() const { return offset_; }
+
+ bool OffsetIsInt16Encodable() const { return is_int16(offset_); }
+
+ private:
+ int32_t offset_;
+
+ friend class Assembler;
+};
+
+class V8_EXPORT_PRIVATE Assembler : public AssemblerBase {
+ public:
+ // Create an assembler. Instructions and relocation information are emitted
+ // into a buffer, with the instructions starting from the beginning and the
+ // relocation information starting from the end of the buffer. See CodeDesc
+ // for a detailed comment on the layout (globals.h).
+ //
+ // If the provided buffer is nullptr, the assembler allocates and grows its
+ // own buffer. Otherwise it takes ownership of the provided buffer.
+ explicit Assembler(const AssemblerOptions&,
+ std::unique_ptr<AssemblerBuffer> = {});
+
+ virtual ~Assembler() {}
+
+ // GetCode emits any pending (non-emitted) code and fills the descriptor desc.
+ static constexpr int kNoHandlerTable = 0;
+ static constexpr SafepointTableBuilder* kNoSafepointTable = nullptr;
+ void GetCode(Isolate* isolate, CodeDesc* desc,
+ SafepointTableBuilder* safepoint_table_builder,
+ int handler_table_offset);
+
+ // Convenience wrapper for code without safepoint or handler tables.
+ void GetCode(Isolate* isolate, CodeDesc* desc) {
+ GetCode(isolate, desc, kNoSafepointTable, kNoHandlerTable);
+ }
+
+ // Unused on this architecture.
+ void MaybeEmitOutOfLineConstantPool() {}
+
+ // Mips uses BlockTrampolinePool to prevent generating trampoline inside a
+ // continuous instruction block. For Call instruction, it prevents generating
+ // trampoline between jalr and delay slot instruction. In the destructor of
+ // BlockTrampolinePool, it must check if it needs to generate trampoline
+ // immediately, if it does not do this, the branch range will go beyond the
+ // max branch offset, that means the pc_offset after call CheckTrampolinePool
+ // may be not the Call instruction's location. So we use last_call_pc here for
+ // safepoint record.
+ int pc_offset_for_safepoint() {
+#ifdef DEBUG
+ Instr instr1 =
+ instr_at(static_cast<int>(last_call_pc_ - buffer_start_ - kInstrSize));
+ Instr instr2 = instr_at(
+ static_cast<int>(last_call_pc_ - buffer_start_ - kInstrSize * 2));
+ if (GetOpcodeField(instr1) != SPECIAL) { // instr1 == jialc.
+ DCHECK((kArchVariant == kMips64r6) && GetOpcodeField(instr1) == POP76 &&
+ GetRs(instr1) == 0);
+ } else {
+ if (GetFunctionField(instr1) == SLL) { // instr1 == nop, instr2 == jalr.
+ DCHECK(GetOpcodeField(instr2) == SPECIAL &&
+ GetFunctionField(instr2) == JALR);
+ } else { // instr1 == jalr.
+ DCHECK(GetFunctionField(instr1) == JALR);
+ }
+ }
+#endif
+ return static_cast<int>(last_call_pc_ - buffer_start_);
+ }
+
+ // Label operations & relative jumps (PPUM Appendix D).
+ //
+ // Takes a branch opcode (cc) and a label (L) and generates
+ // either a backward branch or a forward branch and links it
+ // to the label fixup chain. Usage:
+ //
+ // Label L; // unbound label
+ // j(cc, &L); // forward branch to unbound label
+ // bind(&L); // bind label to the current pc
+ // j(cc, &L); // backward branch to bound label
+ // bind(&L); // illegal: a label may be bound only once
+ //
+ // Note: The same Label can be used for forward and backward branches
+ // but it may be bound only once.
+ void bind(Label* L); // Binds an unbound label L to current code position.
+
+ enum OffsetSize : int { kOffset26 = 26, kOffset21 = 21, kOffset16 = 16 };
+
+ // Determines if Label is bound and near enough so that branch instruction
+ // can be used to reach it, instead of jump instruction.
+ bool is_near(Label* L);
+ bool is_near(Label* L, OffsetSize bits);
+ bool is_near_branch(Label* L);
+ inline bool is_near_pre_r6(Label* L) {
+ DCHECK(!(kArchVariant == kMips64r6));
+ return pc_offset() - L->pos() < kMaxBranchOffset - 4 * kInstrSize;
+ }
+ inline bool is_near_r6(Label* L) {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ return pc_offset() - L->pos() < kMaxCompactBranchOffset - 4 * kInstrSize;
+ }
+
+ int BranchOffset(Instr instr);
+
+ // Returns the branch offset to the given label from the current code
+ // position. Links the label to the current position if it is still unbound.
+ // Manages the jump elimination optimization if the second parameter is true.
+ int32_t branch_offset_helper(Label* L, OffsetSize bits);
+ inline int32_t branch_offset(Label* L) {
+ return branch_offset_helper(L, OffsetSize::kOffset16);
+ }
+ inline int32_t branch_offset21(Label* L) {
+ return branch_offset_helper(L, OffsetSize::kOffset21);
+ }
+ inline int32_t branch_offset26(Label* L) {
+ return branch_offset_helper(L, OffsetSize::kOffset26);
+ }
+ inline int32_t shifted_branch_offset(Label* L) {
+ return branch_offset(L) >> 2;
+ }
+ inline int32_t shifted_branch_offset21(Label* L) {
+ return branch_offset21(L) >> 2;
+ }
+ inline int32_t shifted_branch_offset26(Label* L) {
+ return branch_offset26(L) >> 2;
+ }
+ uint64_t jump_address(Label* L);
+ uint64_t jump_offset(Label* L);
+ uint64_t branch_long_offset(Label* L);
+
+ // Puts a labels target address at the given position.
+ // The high 8 bits are set to zero.
+ void label_at_put(Label* L, int at_offset);
+
+ // Read/Modify the code target address in the branch/call instruction at pc.
+ // The isolate argument is unused (and may be nullptr) when skipping flushing.
+ static Address target_address_at(Address pc);
+ V8_INLINE static void set_target_address_at(
+ Address pc, Address target,
+ ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED) {
+ set_target_value_at(pc, target, icache_flush_mode);
+ }
+ // On MIPS there is no Constant Pool so we skip that parameter.
+ V8_INLINE static Address target_address_at(Address pc,
+ Address constant_pool) {
+ return target_address_at(pc);
+ }
+ V8_INLINE static void set_target_address_at(
+ Address pc, Address constant_pool, Address target,
+ ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED) {
+ set_target_address_at(pc, target, icache_flush_mode);
+ }
+
+ static void set_target_value_at(
+ Address pc, uint64_t target,
+ ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED);
+
+ static void JumpLabelToJumpRegister(Address pc);
+
+ // This sets the branch destination (which gets loaded at the call address).
+ // This is for calls and branches within generated code. The serializer
+ // has already deserialized the lui/ori instructions etc.
+ inline static void deserialization_set_special_target_at(
+ Address instruction_payload, Code code, Address target);
+
+ // Get the size of the special target encoded at 'instruction_payload'.
+ inline static int deserialization_special_target_size(
+ Address instruction_payload);
+
+ // This sets the internal reference at the pc.
+ inline static void deserialization_set_target_internal_reference_at(
+ Address pc, Address target,
+ RelocInfo::Mode mode = RelocInfo::INTERNAL_REFERENCE);
+
+ // Difference between address of current opcode and target address offset.
+ static constexpr int kBranchPCOffset = kInstrSize;
+
+ // Difference between address of current opcode and target address offset,
+ // when we are generatinga sequence of instructions for long relative PC
+ // branches
+ static constexpr int kLongBranchPCOffset = 3 * kInstrSize;
+
+ // Adjust ra register in branch delay slot of bal instruction so to skip
+ // instructions not needed after optimization of PIC in
+ // TurboAssembler::BranchAndLink method.
+
+ static constexpr int kOptimizedBranchAndLinkLongReturnOffset = 4 * kInstrSize;
+
+ // Here we are patching the address in the LUI/ORI instruction pair.
+ // These values are used in the serialization process and must be zero for
+ // MIPS platform, as Code, Embedded Object or External-reference pointers
+ // are split across two consecutive instructions and don't exist separately
+ // in the code, so the serializer should not step forwards in memory after
+ // a target is resolved and written.
+ static constexpr int kSpecialTargetSize = 0;
+
+ // Number of consecutive instructions used to store 32bit/64bit constant.
+ // This constant was used in RelocInfo::target_address_address() function
+ // to tell serializer address of the instruction that follows
+ // LUI/ORI instruction pair.
+ static constexpr int kInstructionsFor32BitConstant = 2;
+ static constexpr int kInstructionsFor64BitConstant = 4;
+
+ // Difference between address of current opcode and value read from pc
+ // register.
+ static constexpr int kPcLoadDelta = 4;
+
+ // Max offset for instructions with 16-bit offset field
+ static constexpr int kMaxBranchOffset = (1 << (18 - 1)) - 1;
+
+ // Max offset for compact branch instructions with 26-bit offset field
+ static constexpr int kMaxCompactBranchOffset = (1 << (28 - 1)) - 1;
+
+ static constexpr int kTrampolineSlotsSize =
+ kArchVariant == kMips64r6 ? 2 * kInstrSize : 7 * kInstrSize;
+
+ RegList* GetScratchRegisterList() { return &scratch_register_list_; }
+
+ // ---------------------------------------------------------------------------
+ // Code generation.
+
+ // Insert the smallest number of nop instructions
+ // possible to align the pc offset to a multiple
+ // of m. m must be a power of 2 (>= 4).
+ void Align(int m);
+ // Insert the smallest number of zero bytes possible to align the pc offset
+ // to a mulitple of m. m must be a power of 2 (>= 2).
+ void DataAlign(int m);
+ // Aligns code to something that's optimal for a jump target for the platform.
+ void CodeTargetAlign();
+
+ // Different nop operations are used by the code generator to detect certain
+ // states of the generated code.
+ enum NopMarkerTypes {
+ NON_MARKING_NOP = 0,
+ DEBUG_BREAK_NOP,
+ // IC markers.
+ PROPERTY_ACCESS_INLINED,
+ PROPERTY_ACCESS_INLINED_CONTEXT,
+ PROPERTY_ACCESS_INLINED_CONTEXT_DONT_DELETE,
+ // Helper values.
+ LAST_CODE_MARKER,
+ FIRST_IC_MARKER = PROPERTY_ACCESS_INLINED,
+ };
+
+ // Type == 0 is the default non-marking nop. For mips this is a
+ // sll(zero_reg, zero_reg, 0). We use rt_reg == at for non-zero
+ // marking, to avoid conflict with ssnop and ehb instructions.
+ void nop(unsigned int type = 0) {
+ DCHECK_LT(type, 32);
+ Register nop_rt_reg = (type == 0) ? zero_reg : at;
+ sll(zero_reg, nop_rt_reg, type, true);
+ }
+
+ // --------Branch-and-jump-instructions----------
+ // We don't use likely variant of instructions.
+ void b(int16_t offset);
+ inline void b(Label* L) { b(shifted_branch_offset(L)); }
+ void bal(int16_t offset);
+ inline void bal(Label* L) { bal(shifted_branch_offset(L)); }
+ void bc(int32_t offset);
+ inline void bc(Label* L) { bc(shifted_branch_offset26(L)); }
+ void balc(int32_t offset);
+ inline void balc(Label* L) { balc(shifted_branch_offset26(L)); }
+
+ void beq(Register rs, Register rt, int16_t offset);
+ inline void beq(Register rs, Register rt, Label* L) {
+ beq(rs, rt, shifted_branch_offset(L));
+ }
+ void bgez(Register rs, int16_t offset);
+ void bgezc(Register rt, int16_t offset);
+ inline void bgezc(Register rt, Label* L) {
+ bgezc(rt, shifted_branch_offset(L));
+ }
+ void bgeuc(Register rs, Register rt, int16_t offset);
+ inline void bgeuc(Register rs, Register rt, Label* L) {
+ bgeuc(rs, rt, shifted_branch_offset(L));
+ }
+ void bgec(Register rs, Register rt, int16_t offset);
+ inline void bgec(Register rs, Register rt, Label* L) {
+ bgec(rs, rt, shifted_branch_offset(L));
+ }
+ void bgezal(Register rs, int16_t offset);
+ void bgezalc(Register rt, int16_t offset);
+ inline void bgezalc(Register rt, Label* L) {
+ bgezalc(rt, shifted_branch_offset(L));
+ }
+ void bgezall(Register rs, int16_t offset);
+ inline void bgezall(Register rs, Label* L) {
+ bgezall(rs, branch_offset(L) >> 2);
+ }
+ void bgtz(Register rs, int16_t offset);
+ void bgtzc(Register rt, int16_t offset);
+ inline void bgtzc(Register rt, Label* L) {
+ bgtzc(rt, shifted_branch_offset(L));
+ }
+ void blez(Register rs, int16_t offset);
+ void blezc(Register rt, int16_t offset);
+ inline void blezc(Register rt, Label* L) {
+ blezc(rt, shifted_branch_offset(L));
+ }
+ void bltz(Register rs, int16_t offset);
+ void bltzc(Register rt, int16_t offset);
+ inline void bltzc(Register rt, Label* L) {
+ bltzc(rt, shifted_branch_offset(L));
+ }
+ void bltuc(Register rs, Register rt, int16_t offset);
+ inline void bltuc(Register rs, Register rt, Label* L) {
+ bltuc(rs, rt, shifted_branch_offset(L));
+ }
+ void bltc(Register rs, Register rt, int16_t offset);
+ inline void bltc(Register rs, Register rt, Label* L) {
+ bltc(rs, rt, shifted_branch_offset(L));
+ }
+ void bltzal(Register rs, int16_t offset);
+ void nal() { bltzal(zero_reg, 0); }
+ void blezalc(Register rt, int16_t offset);
+ inline void blezalc(Register rt, Label* L) {
+ blezalc(rt, shifted_branch_offset(L));
+ }
+ void bltzalc(Register rt, int16_t offset);
+ inline void bltzalc(Register rt, Label* L) {
+ bltzalc(rt, shifted_branch_offset(L));
+ }
+ void bgtzalc(Register rt, int16_t offset);
+ inline void bgtzalc(Register rt, Label* L) {
+ bgtzalc(rt, shifted_branch_offset(L));
+ }
+ void beqzalc(Register rt, int16_t offset);
+ inline void beqzalc(Register rt, Label* L) {
+ beqzalc(rt, shifted_branch_offset(L));
+ }
+ void beqc(Register rs, Register rt, int16_t offset);
+ inline void beqc(Register rs, Register rt, Label* L) {
+ beqc(rs, rt, shifted_branch_offset(L));
+ }
+ void beqzc(Register rs, int32_t offset);
+ inline void beqzc(Register rs, Label* L) {
+ beqzc(rs, shifted_branch_offset21(L));
+ }
+ void bnezalc(Register rt, int16_t offset);
+ inline void bnezalc(Register rt, Label* L) {
+ bnezalc(rt, shifted_branch_offset(L));
+ }
+ void bnec(Register rs, Register rt, int16_t offset);
+ inline void bnec(Register rs, Register rt, Label* L) {
+ bnec(rs, rt, shifted_branch_offset(L));
+ }
+ void bnezc(Register rt, int32_t offset);
+ inline void bnezc(Register rt, Label* L) {
+ bnezc(rt, shifted_branch_offset21(L));
+ }
+ void bne(Register rs, Register rt, int16_t offset);
+ inline void bne(Register rs, Register rt, Label* L) {
+ bne(rs, rt, shifted_branch_offset(L));
+ }
+ void bovc(Register rs, Register rt, int16_t offset);
+ inline void bovc(Register rs, Register rt, Label* L) {
+ bovc(rs, rt, shifted_branch_offset(L));
+ }
+ void bnvc(Register rs, Register rt, int16_t offset);
+ inline void bnvc(Register rs, Register rt, Label* L) {
+ bnvc(rs, rt, shifted_branch_offset(L));
+ }
+
+ // Never use the int16_t b(l)cond version with a branch offset
+ // instead of using the Label* version.
+
+ void jalr(Register rs, Register rd = ra);
+ void jr(Register target);
+ void jic(Register rt, int16_t offset);
+ void jialc(Register rt, int16_t offset);
+
+ // Following instructions are deprecated and require 256 MB
+ // code alignment. Use PC-relative instructions instead.
+ void j(int64_t target);
+ void jal(int64_t target);
+ void j(Label* target);
+ void jal(Label* target);
+
+ // -------Data-processing-instructions---------
+
+ // Arithmetic.
+ void addu(Register rd, Register rs, Register rt);
+ void subu(Register rd, Register rs, Register rt);
+
+ void div(Register rs, Register rt);
+ void divu(Register rs, Register rt);
+ void ddiv(Register rs, Register rt);
+ void ddivu(Register rs, Register rt);
+ void div(Register rd, Register rs, Register rt);
+ void divu(Register rd, Register rs, Register rt);
+ void ddiv(Register rd, Register rs, Register rt);
+ void ddivu(Register rd, Register rs, Register rt);
+ void mod(Register rd, Register rs, Register rt);
+ void modu(Register rd, Register rs, Register rt);
+ void dmod(Register rd, Register rs, Register rt);
+ void dmodu(Register rd, Register rs, Register rt);
+
+ void mul(Register rd, Register rs, Register rt);
+ void muh(Register rd, Register rs, Register rt);
+ void mulu(Register rd, Register rs, Register rt);
+ void muhu(Register rd, Register rs, Register rt);
+ void mult(Register rs, Register rt);
+ void multu(Register rs, Register rt);
+ void dmul(Register rd, Register rs, Register rt);
+ void dmuh(Register rd, Register rs, Register rt);
+ void dmulu(Register rd, Register rs, Register rt);
+ void dmuhu(Register rd, Register rs, Register rt);
+ void daddu(Register rd, Register rs, Register rt);
+ void dsubu(Register rd, Register rs, Register rt);
+ void dmult(Register rs, Register rt);
+ void dmultu(Register rs, Register rt);
+
+ void addiu(Register rd, Register rs, int32_t j);
+ void daddiu(Register rd, Register rs, int32_t j);
+
+ // Logical.
+ void and_(Register rd, Register rs, Register rt);
+ void or_(Register rd, Register rs, Register rt);
+ void xor_(Register rd, Register rs, Register rt);
+ void nor(Register rd, Register rs, Register rt);
+
+ void andi(Register rd, Register rs, int32_t j);
+ void ori(Register rd, Register rs, int32_t j);
+ void xori(Register rd, Register rs, int32_t j);
+ void lui(Register rd, int32_t j);
+ void aui(Register rt, Register rs, int32_t j);
+ void daui(Register rt, Register rs, int32_t j);
+ void dahi(Register rs, int32_t j);
+ void dati(Register rs, int32_t j);
+
+ // Shifts.
+ // Please note: sll(zero_reg, zero_reg, x) instructions are reserved as nop
+ // and may cause problems in normal code. coming_from_nop makes sure this
+ // doesn't happen.
+ void sll(Register rd, Register rt, uint16_t sa, bool coming_from_nop = false);
+ void sllv(Register rd, Register rt, Register rs);
+ void srl(Register rd, Register rt, uint16_t sa);
+ void srlv(Register rd, Register rt, Register rs);
+ void sra(Register rt, Register rd, uint16_t sa);
+ void srav(Register rt, Register rd, Register rs);
+ void rotr(Register rd, Register rt, uint16_t sa);
+ void rotrv(Register rd, Register rt, Register rs);
+ void dsll(Register rd, Register rt, uint16_t sa);
+ void dsllv(Register rd, Register rt, Register rs);
+ void dsrl(Register rd, Register rt, uint16_t sa);
+ void dsrlv(Register rd, Register rt, Register rs);
+ void drotr(Register rd, Register rt, uint16_t sa);
+ void drotr32(Register rd, Register rt, uint16_t sa);
+ void drotrv(Register rd, Register rt, Register rs);
+ void dsra(Register rt, Register rd, uint16_t sa);
+ void dsrav(Register rd, Register rt, Register rs);
+ void dsll32(Register rt, Register rd, uint16_t sa);
+ void dsrl32(Register rt, Register rd, uint16_t sa);
+ void dsra32(Register rt, Register rd, uint16_t sa);
+
+ // ------------Memory-instructions-------------
+
+ void lb(Register rd, const MemOperand& rs);
+ void lbu(Register rd, const MemOperand& rs);
+ void lh(Register rd, const MemOperand& rs);
+ void lhu(Register rd, const MemOperand& rs);
+ void lw(Register rd, const MemOperand& rs);
+ void lwu(Register rd, const MemOperand& rs);
+ void lwl(Register rd, const MemOperand& rs);
+ void lwr(Register rd, const MemOperand& rs);
+ void sb(Register rd, const MemOperand& rs);
+ void sh(Register rd, const MemOperand& rs);
+ void sw(Register rd, const MemOperand& rs);
+ void swl(Register rd, const MemOperand& rs);
+ void swr(Register rd, const MemOperand& rs);
+ void ldl(Register rd, const MemOperand& rs);
+ void ldr(Register rd, const MemOperand& rs);
+ void sdl(Register rd, const MemOperand& rs);
+ void sdr(Register rd, const MemOperand& rs);
+ void ld(Register rd, const MemOperand& rs);
+ void sd(Register rd, const MemOperand& rs);
+
+ // ----------Atomic instructions--------------
+
+ void ll(Register rd, const MemOperand& rs);
+ void sc(Register rd, const MemOperand& rs);
+ void lld(Register rd, const MemOperand& rs);
+ void scd(Register rd, const MemOperand& rs);
+
+ // ---------PC-Relative-instructions-----------
+
+ void addiupc(Register rs, int32_t imm19);
+ void lwpc(Register rs, int32_t offset19);
+ void lwupc(Register rs, int32_t offset19);
+ void ldpc(Register rs, int32_t offset18);
+ void auipc(Register rs, int16_t imm16);
+ void aluipc(Register rs, int16_t imm16);
+
+ // ----------------Prefetch--------------------
+
+ void pref(int32_t hint, const MemOperand& rs);
+
+ // -------------Misc-instructions--------------
+
+ // Break / Trap instructions.
+ void break_(uint32_t code, bool break_as_stop = false);
+ void stop(uint32_t code = kMaxStopCode);
+ void tge(Register rs, Register rt, uint16_t code);
+ void tgeu(Register rs, Register rt, uint16_t code);
+ void tlt(Register rs, Register rt, uint16_t code);
+ void tltu(Register rs, Register rt, uint16_t code);
+ void teq(Register rs, Register rt, uint16_t code);
+ void tne(Register rs, Register rt, uint16_t code);
+
+ // Memory barrier instruction.
+ void sync();
+
+ // Move from HI/LO register.
+ void mfhi(Register rd);
+ void mflo(Register rd);
+
+ // Set on less than.
+ void slt(Register rd, Register rs, Register rt);
+ void sltu(Register rd, Register rs, Register rt);
+ void slti(Register rd, Register rs, int32_t j);
+ void sltiu(Register rd, Register rs, int32_t j);
+
+ // Conditional move.
+ void movz(Register rd, Register rs, Register rt);
+ void movn(Register rd, Register rs, Register rt);
+ void movt(Register rd, Register rs, uint16_t cc = 0);
+ void movf(Register rd, Register rs, uint16_t cc = 0);
+
+ void sel(SecondaryField fmt, FPURegister fd, FPURegister fs, FPURegister ft);
+ void sel_s(FPURegister fd, FPURegister fs, FPURegister ft);
+ void sel_d(FPURegister fd, FPURegister fs, FPURegister ft);
+ void seleqz(Register rd, Register rs, Register rt);
+ void seleqz(SecondaryField fmt, FPURegister fd, FPURegister fs,
+ FPURegister ft);
+ void selnez(Register rs, Register rt, Register rd);
+ void selnez(SecondaryField fmt, FPURegister fd, FPURegister fs,
+ FPURegister ft);
+ void seleqz_d(FPURegister fd, FPURegister fs, FPURegister ft);
+ void seleqz_s(FPURegister fd, FPURegister fs, FPURegister ft);
+ void selnez_d(FPURegister fd, FPURegister fs, FPURegister ft);
+ void selnez_s(FPURegister fd, FPURegister fs, FPURegister ft);
+
+ void movz_s(FPURegister fd, FPURegister fs, Register rt);
+ void movz_d(FPURegister fd, FPURegister fs, Register rt);
+ void movt_s(FPURegister fd, FPURegister fs, uint16_t cc = 0);
+ void movt_d(FPURegister fd, FPURegister fs, uint16_t cc = 0);
+ void movf_s(FPURegister fd, FPURegister fs, uint16_t cc = 0);
+ void movf_d(FPURegister fd, FPURegister fs, uint16_t cc = 0);
+ void movn_s(FPURegister fd, FPURegister fs, Register rt);
+ void movn_d(FPURegister fd, FPURegister fs, Register rt);
+ // Bit twiddling.
+ void clz(Register rd, Register rs);
+ void dclz(Register rd, Register rs);
+ void ins_(Register rt, Register rs, uint16_t pos, uint16_t size);
+ void ext_(Register rt, Register rs, uint16_t pos, uint16_t size);
+ void dext_(Register rt, Register rs, uint16_t pos, uint16_t size);
+ void dextm_(Register rt, Register rs, uint16_t pos, uint16_t size);
+ void dextu_(Register rt, Register rs, uint16_t pos, uint16_t size);
+ void dins_(Register rt, Register rs, uint16_t pos, uint16_t size);
+ void dinsm_(Register rt, Register rs, uint16_t pos, uint16_t size);
+ void dinsu_(Register rt, Register rs, uint16_t pos, uint16_t size);
+ void bitswap(Register rd, Register rt);
+ void dbitswap(Register rd, Register rt);
+ void align(Register rd, Register rs, Register rt, uint8_t bp);
+ void dalign(Register rd, Register rs, Register rt, uint8_t bp);
+
+ void wsbh(Register rd, Register rt);
+ void dsbh(Register rd, Register rt);
+ void dshd(Register rd, Register rt);
+ void seh(Register rd, Register rt);
+ void seb(Register rd, Register rt);
+
+ // --------Coprocessor-instructions----------------
+
+ // Load, store, and move.
+ void lwc1(FPURegister fd, const MemOperand& src);
+ void ldc1(FPURegister fd, const MemOperand& src);
+
+ void swc1(FPURegister fs, const MemOperand& dst);
+ void sdc1(FPURegister fs, const MemOperand& dst);
+
+ void mtc1(Register rt, FPURegister fs);
+ void mthc1(Register rt, FPURegister fs);
+ void dmtc1(Register rt, FPURegister fs);
+
+ void mfc1(Register rt, FPURegister fs);
+ void mfhc1(Register rt, FPURegister fs);
+ void dmfc1(Register rt, FPURegister fs);
+
+ void ctc1(Register rt, FPUControlRegister fs);
+ void cfc1(Register rt, FPUControlRegister fs);
+
+ // Arithmetic.
+ void add_s(FPURegister fd, FPURegister fs, FPURegister ft);
+ void add_d(FPURegister fd, FPURegister fs, FPURegister ft);
+ void sub_s(FPURegister fd, FPURegister fs, FPURegister ft);
+ void sub_d(FPURegister fd, FPURegister fs, FPURegister ft);
+ void mul_s(FPURegister fd, FPURegister fs, FPURegister ft);
+ void mul_d(FPURegister fd, FPURegister fs, FPURegister ft);
+ void madd_s(FPURegister fd, FPURegister fr, FPURegister fs, FPURegister ft);
+ void madd_d(FPURegister fd, FPURegister fr, FPURegister fs, FPURegister ft);
+ void msub_s(FPURegister fd, FPURegister fr, FPURegister fs, FPURegister ft);
+ void msub_d(FPURegister fd, FPURegister fr, FPURegister fs, FPURegister ft);
+ void maddf_s(FPURegister fd, FPURegister fs, FPURegister ft);
+ void maddf_d(FPURegister fd, FPURegister fs, FPURegister ft);
+ void msubf_s(FPURegister fd, FPURegister fs, FPURegister ft);
+ void msubf_d(FPURegister fd, FPURegister fs, FPURegister ft);
+ void div_s(FPURegister fd, FPURegister fs, FPURegister ft);
+ void div_d(FPURegister fd, FPURegister fs, FPURegister ft);
+ void abs_s(FPURegister fd, FPURegister fs);
+ void abs_d(FPURegister fd, FPURegister fs);
+ void mov_d(FPURegister fd, FPURegister fs);
+ void mov_s(FPURegister fd, FPURegister fs);
+ void neg_s(FPURegister fd, FPURegister fs);
+ void neg_d(FPURegister fd, FPURegister fs);
+ void sqrt_s(FPURegister fd, FPURegister fs);
+ void sqrt_d(FPURegister fd, FPURegister fs);
+ void rsqrt_s(FPURegister fd, FPURegister fs);
+ void rsqrt_d(FPURegister fd, FPURegister fs);
+ void recip_d(FPURegister fd, FPURegister fs);
+ void recip_s(FPURegister fd, FPURegister fs);
+
+ // Conversion.
+ void cvt_w_s(FPURegister fd, FPURegister fs);
+ void cvt_w_d(FPURegister fd, FPURegister fs);
+ void trunc_w_s(FPURegister fd, FPURegister fs);
+ void trunc_w_d(FPURegister fd, FPURegister fs);
+ void round_w_s(FPURegister fd, FPURegister fs);
+ void round_w_d(FPURegister fd, FPURegister fs);
+ void floor_w_s(FPURegister fd, FPURegister fs);
+ void floor_w_d(FPURegister fd, FPURegister fs);
+ void ceil_w_s(FPURegister fd, FPURegister fs);
+ void ceil_w_d(FPURegister fd, FPURegister fs);
+ void rint_s(FPURegister fd, FPURegister fs);
+ void rint_d(FPURegister fd, FPURegister fs);
+ void rint(SecondaryField fmt, FPURegister fd, FPURegister fs);
+
+ void cvt_l_s(FPURegister fd, FPURegister fs);
+ void cvt_l_d(FPURegister fd, FPURegister fs);
+ void trunc_l_s(FPURegister fd, FPURegister fs);
+ void trunc_l_d(FPURegister fd, FPURegister fs);
+ void round_l_s(FPURegister fd, FPURegister fs);
+ void round_l_d(FPURegister fd, FPURegister fs);
+ void floor_l_s(FPURegister fd, FPURegister fs);
+ void floor_l_d(FPURegister fd, FPURegister fs);
+ void ceil_l_s(FPURegister fd, FPURegister fs);
+ void ceil_l_d(FPURegister fd, FPURegister fs);
+
+ void class_s(FPURegister fd, FPURegister fs);
+ void class_d(FPURegister fd, FPURegister fs);
+
+ void min(SecondaryField fmt, FPURegister fd, FPURegister fs, FPURegister ft);
+ void mina(SecondaryField fmt, FPURegister fd, FPURegister fs, FPURegister ft);
+ void max(SecondaryField fmt, FPURegister fd, FPURegister fs, FPURegister ft);
+ void maxa(SecondaryField fmt, FPURegister fd, FPURegister fs, FPURegister ft);
+ void min_s(FPURegister fd, FPURegister fs, FPURegister ft);
+ void min_d(FPURegister fd, FPURegister fs, FPURegister ft);
+ void max_s(FPURegister fd, FPURegister fs, FPURegister ft);
+ void max_d(FPURegister fd, FPURegister fs, FPURegister ft);
+ void mina_s(FPURegister fd, FPURegister fs, FPURegister ft);
+ void mina_d(FPURegister fd, FPURegister fs, FPURegister ft);
+ void maxa_s(FPURegister fd, FPURegister fs, FPURegister ft);
+ void maxa_d(FPURegister fd, FPURegister fs, FPURegister ft);
+
+ void cvt_s_w(FPURegister fd, FPURegister fs);
+ void cvt_s_l(FPURegister fd, FPURegister fs);
+ void cvt_s_d(FPURegister fd, FPURegister fs);
+
+ void cvt_d_w(FPURegister fd, FPURegister fs);
+ void cvt_d_l(FPURegister fd, FPURegister fs);
+ void cvt_d_s(FPURegister fd, FPURegister fs);
+
+ // Conditions and branches for MIPSr6.
+ void cmp(FPUCondition cond, SecondaryField fmt, FPURegister fd,
+ FPURegister ft, FPURegister fs);
+ void cmp_s(FPUCondition cond, FPURegister fd, FPURegister fs, FPURegister ft);
+ void cmp_d(FPUCondition cond, FPURegister fd, FPURegister fs, FPURegister ft);
+
+ void bc1eqz(int16_t offset, FPURegister ft);
+ inline void bc1eqz(Label* L, FPURegister ft) {
+ bc1eqz(shifted_branch_offset(L), ft);
+ }
+ void bc1nez(int16_t offset, FPURegister ft);
+ inline void bc1nez(Label* L, FPURegister ft) {
+ bc1nez(shifted_branch_offset(L), ft);
+ }
+
+ // Conditions and branches for non MIPSr6.
+ void c(FPUCondition cond, SecondaryField fmt, FPURegister ft, FPURegister fs,
+ uint16_t cc = 0);
+ void c_s(FPUCondition cond, FPURegister ft, FPURegister fs, uint16_t cc = 0);
+ void c_d(FPUCondition cond, FPURegister ft, FPURegister fs, uint16_t cc = 0);
+
+ void bc1f(int16_t offset, uint16_t cc = 0);
+ inline void bc1f(Label* L, uint16_t cc = 0) {
+ bc1f(shifted_branch_offset(L), cc);
+ }
+ void bc1t(int16_t offset, uint16_t cc = 0);
+ inline void bc1t(Label* L, uint16_t cc = 0) {
+ bc1t(shifted_branch_offset(L), cc);
+ }
+ void fcmp(FPURegister src1, const double src2, FPUCondition cond);
+
+ // MSA instructions
+ void bz_v(MSARegister wt, int16_t offset);
+ inline void bz_v(MSARegister wt, Label* L) {
+ bz_v(wt, shifted_branch_offset(L));
+ }
+ void bz_b(MSARegister wt, int16_t offset);
+ inline void bz_b(MSARegister wt, Label* L) {
+ bz_b(wt, shifted_branch_offset(L));
+ }
+ void bz_h(MSARegister wt, int16_t offset);
+ inline void bz_h(MSARegister wt, Label* L) {
+ bz_h(wt, shifted_branch_offset(L));
+ }
+ void bz_w(MSARegister wt, int16_t offset);
+ inline void bz_w(MSARegister wt, Label* L) {
+ bz_w(wt, shifted_branch_offset(L));
+ }
+ void bz_d(MSARegister wt, int16_t offset);
+ inline void bz_d(MSARegister wt, Label* L) {
+ bz_d(wt, shifted_branch_offset(L));
+ }
+ void bnz_v(MSARegister wt, int16_t offset);
+ inline void bnz_v(MSARegister wt, Label* L) {
+ bnz_v(wt, shifted_branch_offset(L));
+ }
+ void bnz_b(MSARegister wt, int16_t offset);
+ inline void bnz_b(MSARegister wt, Label* L) {
+ bnz_b(wt, shifted_branch_offset(L));
+ }
+ void bnz_h(MSARegister wt, int16_t offset);
+ inline void bnz_h(MSARegister wt, Label* L) {
+ bnz_h(wt, shifted_branch_offset(L));
+ }
+ void bnz_w(MSARegister wt, int16_t offset);
+ inline void bnz_w(MSARegister wt, Label* L) {
+ bnz_w(wt, shifted_branch_offset(L));
+ }
+ void bnz_d(MSARegister wt, int16_t offset);
+ inline void bnz_d(MSARegister wt, Label* L) {
+ bnz_d(wt, shifted_branch_offset(L));
+ }
+
+ void ld_b(MSARegister wd, const MemOperand& rs);
+ void ld_h(MSARegister wd, const MemOperand& rs);
+ void ld_w(MSARegister wd, const MemOperand& rs);
+ void ld_d(MSARegister wd, const MemOperand& rs);
+ void st_b(MSARegister wd, const MemOperand& rs);
+ void st_h(MSARegister wd, const MemOperand& rs);
+ void st_w(MSARegister wd, const MemOperand& rs);
+ void st_d(MSARegister wd, const MemOperand& rs);
+
+ void ldi_b(MSARegister wd, int32_t imm10);
+ void ldi_h(MSARegister wd, int32_t imm10);
+ void ldi_w(MSARegister wd, int32_t imm10);
+ void ldi_d(MSARegister wd, int32_t imm10);
+
+ void addvi_b(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void addvi_h(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void addvi_w(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void addvi_d(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void subvi_b(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void subvi_h(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void subvi_w(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void subvi_d(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void maxi_s_b(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void maxi_s_h(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void maxi_s_w(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void maxi_s_d(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void maxi_u_b(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void maxi_u_h(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void maxi_u_w(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void maxi_u_d(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void mini_s_b(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void mini_s_h(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void mini_s_w(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void mini_s_d(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void mini_u_b(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void mini_u_h(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void mini_u_w(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void mini_u_d(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void ceqi_b(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void ceqi_h(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void ceqi_w(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void ceqi_d(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clti_s_b(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clti_s_h(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clti_s_w(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clti_s_d(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clti_u_b(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clti_u_h(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clti_u_w(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clti_u_d(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clei_s_b(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clei_s_h(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clei_s_w(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clei_s_d(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clei_u_b(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clei_u_h(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clei_u_w(MSARegister wd, MSARegister ws, uint32_t imm5);
+ void clei_u_d(MSARegister wd, MSARegister ws, uint32_t imm5);
+
+ void andi_b(MSARegister wd, MSARegister ws, uint32_t imm8);
+ void ori_b(MSARegister wd, MSARegister ws, uint32_t imm8);
+ void nori_b(MSARegister wd, MSARegister ws, uint32_t imm8);
+ void xori_b(MSARegister wd, MSARegister ws, uint32_t imm8);
+ void bmnzi_b(MSARegister wd, MSARegister ws, uint32_t imm8);
+ void bmzi_b(MSARegister wd, MSARegister ws, uint32_t imm8);
+ void bseli_b(MSARegister wd, MSARegister ws, uint32_t imm8);
+ void shf_b(MSARegister wd, MSARegister ws, uint32_t imm8);
+ void shf_h(MSARegister wd, MSARegister ws, uint32_t imm8);
+ void shf_w(MSARegister wd, MSARegister ws, uint32_t imm8);
+
+ void and_v(MSARegister wd, MSARegister ws, MSARegister wt);
+ void or_v(MSARegister wd, MSARegister ws, MSARegister wt);
+ void nor_v(MSARegister wd, MSARegister ws, MSARegister wt);
+ void xor_v(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bmnz_v(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bmz_v(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bsel_v(MSARegister wd, MSARegister ws, MSARegister wt);
+
+ void fill_b(MSARegister wd, Register rs);
+ void fill_h(MSARegister wd, Register rs);
+ void fill_w(MSARegister wd, Register rs);
+ void fill_d(MSARegister wd, Register rs);
+ void pcnt_b(MSARegister wd, MSARegister ws);
+ void pcnt_h(MSARegister wd, MSARegister ws);
+ void pcnt_w(MSARegister wd, MSARegister ws);
+ void pcnt_d(MSARegister wd, MSARegister ws);
+ void nloc_b(MSARegister wd, MSARegister ws);
+ void nloc_h(MSARegister wd, MSARegister ws);
+ void nloc_w(MSARegister wd, MSARegister ws);
+ void nloc_d(MSARegister wd, MSARegister ws);
+ void nlzc_b(MSARegister wd, MSARegister ws);
+ void nlzc_h(MSARegister wd, MSARegister ws);
+ void nlzc_w(MSARegister wd, MSARegister ws);
+ void nlzc_d(MSARegister wd, MSARegister ws);
+
+ void fclass_w(MSARegister wd, MSARegister ws);
+ void fclass_d(MSARegister wd, MSARegister ws);
+ void ftrunc_s_w(MSARegister wd, MSARegister ws);
+ void ftrunc_s_d(MSARegister wd, MSARegister ws);
+ void ftrunc_u_w(MSARegister wd, MSARegister ws);
+ void ftrunc_u_d(MSARegister wd, MSARegister ws);
+ void fsqrt_w(MSARegister wd, MSARegister ws);
+ void fsqrt_d(MSARegister wd, MSARegister ws);
+ void frsqrt_w(MSARegister wd, MSARegister ws);
+ void frsqrt_d(MSARegister wd, MSARegister ws);
+ void frcp_w(MSARegister wd, MSARegister ws);
+ void frcp_d(MSARegister wd, MSARegister ws);
+ void frint_w(MSARegister wd, MSARegister ws);
+ void frint_d(MSARegister wd, MSARegister ws);
+ void flog2_w(MSARegister wd, MSARegister ws);
+ void flog2_d(MSARegister wd, MSARegister ws);
+ void fexupl_w(MSARegister wd, MSARegister ws);
+ void fexupl_d(MSARegister wd, MSARegister ws);
+ void fexupr_w(MSARegister wd, MSARegister ws);
+ void fexupr_d(MSARegister wd, MSARegister ws);
+ void ffql_w(MSARegister wd, MSARegister ws);
+ void ffql_d(MSARegister wd, MSARegister ws);
+ void ffqr_w(MSARegister wd, MSARegister ws);
+ void ffqr_d(MSARegister wd, MSARegister ws);
+ void ftint_s_w(MSARegister wd, MSARegister ws);
+ void ftint_s_d(MSARegister wd, MSARegister ws);
+ void ftint_u_w(MSARegister wd, MSARegister ws);
+ void ftint_u_d(MSARegister wd, MSARegister ws);
+ void ffint_s_w(MSARegister wd, MSARegister ws);
+ void ffint_s_d(MSARegister wd, MSARegister ws);
+ void ffint_u_w(MSARegister wd, MSARegister ws);
+ void ffint_u_d(MSARegister wd, MSARegister ws);
+
+ void sll_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void sll_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void sll_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void sll_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void sra_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void sra_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void sra_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void sra_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void srl_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void srl_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void srl_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void srl_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bclr_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bclr_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bclr_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bclr_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bset_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bset_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bset_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bset_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bneg_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bneg_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bneg_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void bneg_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void binsl_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void binsl_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void binsl_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void binsl_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void binsr_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void binsr_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void binsr_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void binsr_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void addv_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void addv_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void addv_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void addv_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subv_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subv_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subv_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subv_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void max_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void max_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void max_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void max_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void max_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void max_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void max_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void max_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void min_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void min_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void min_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void min_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void min_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void min_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void min_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void min_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void max_a_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void max_a_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void max_a_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void max_a_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void min_a_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void min_a_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void min_a_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void min_a_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ceq_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ceq_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ceq_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ceq_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void clt_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void clt_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void clt_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void clt_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void clt_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void clt_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void clt_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void clt_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void cle_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void cle_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void cle_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void cle_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void cle_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void cle_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void cle_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void cle_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void add_a_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void add_a_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void add_a_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void add_a_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void adds_a_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void adds_a_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void adds_a_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void adds_a_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void adds_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void adds_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void adds_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void adds_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void adds_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void adds_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void adds_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void adds_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ave_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ave_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ave_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ave_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ave_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ave_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ave_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ave_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void aver_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void aver_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void aver_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void aver_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void aver_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void aver_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void aver_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void aver_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subs_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subs_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subs_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subs_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subs_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subs_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subs_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subs_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsus_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsus_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsus_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsus_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsus_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsus_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsus_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsus_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsuu_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsuu_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsuu_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsuu_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsuu_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsuu_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsuu_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void subsuu_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void asub_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void asub_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void asub_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void asub_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void asub_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void asub_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void asub_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void asub_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mulv_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mulv_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mulv_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mulv_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void maddv_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void maddv_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void maddv_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void maddv_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void msubv_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void msubv_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void msubv_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void msubv_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void div_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void div_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void div_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void div_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void div_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void div_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void div_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void div_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mod_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mod_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mod_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mod_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mod_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mod_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mod_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mod_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dotp_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dotp_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dotp_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dotp_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dotp_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dotp_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dotp_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dotp_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpadd_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpadd_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpadd_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpadd_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpadd_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpadd_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpadd_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpadd_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpsub_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpsub_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpsub_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpsub_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpsub_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpsub_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpsub_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void dpsub_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void sld_b(MSARegister wd, MSARegister ws, Register rt);
+ void sld_h(MSARegister wd, MSARegister ws, Register rt);
+ void sld_w(MSARegister wd, MSARegister ws, Register rt);
+ void sld_d(MSARegister wd, MSARegister ws, Register rt);
+ void splat_b(MSARegister wd, MSARegister ws, Register rt);
+ void splat_h(MSARegister wd, MSARegister ws, Register rt);
+ void splat_w(MSARegister wd, MSARegister ws, Register rt);
+ void splat_d(MSARegister wd, MSARegister ws, Register rt);
+ void pckev_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void pckev_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void pckev_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void pckev_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void pckod_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void pckod_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void pckod_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void pckod_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvl_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvl_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvl_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvl_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvr_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvr_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvr_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvr_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvev_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvev_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvev_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvev_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvod_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvod_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvod_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ilvod_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void vshf_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void vshf_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void vshf_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void vshf_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void srar_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void srar_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void srar_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void srar_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void srlr_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void srlr_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void srlr_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void srlr_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hadd_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hadd_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hadd_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hadd_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hadd_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hadd_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hadd_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hadd_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hsub_s_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hsub_s_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hsub_s_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hsub_s_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hsub_u_b(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hsub_u_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hsub_u_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void hsub_u_d(MSARegister wd, MSARegister ws, MSARegister wt);
+
+ void fcaf_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcaf_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcun_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcun_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fceq_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fceq_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcueq_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcueq_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fclt_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fclt_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcult_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcult_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcle_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcle_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcule_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcule_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsaf_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsaf_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsun_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsun_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fseq_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fseq_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsueq_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsueq_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fslt_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fslt_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsult_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsult_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsle_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsle_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsule_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsule_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fadd_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fadd_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsub_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsub_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmul_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmul_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fdiv_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fdiv_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmadd_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmadd_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmsub_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmsub_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fexp2_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fexp2_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fexdo_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fexdo_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ftq_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void ftq_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmin_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmin_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmin_a_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmin_a_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmax_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmax_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmax_a_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fmax_a_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcor_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcor_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcune_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcune_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcne_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fcne_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mul_q_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mul_q_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void madd_q_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void madd_q_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void msub_q_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void msub_q_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsor_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsor_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsune_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsune_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsne_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void fsne_d(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mulr_q_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void mulr_q_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void maddr_q_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void maddr_q_w(MSARegister wd, MSARegister ws, MSARegister wt);
+ void msubr_q_h(MSARegister wd, MSARegister ws, MSARegister wt);
+ void msubr_q_w(MSARegister wd, MSARegister ws, MSARegister wt);
+
+ void sldi_b(MSARegister wd, MSARegister ws, uint32_t n);
+ void sldi_h(MSARegister wd, MSARegister ws, uint32_t n);
+ void sldi_w(MSARegister wd, MSARegister ws, uint32_t n);
+ void sldi_d(MSARegister wd, MSARegister ws, uint32_t n);
+ void splati_b(MSARegister wd, MSARegister ws, uint32_t n);
+ void splati_h(MSARegister wd, MSARegister ws, uint32_t n);
+ void splati_w(MSARegister wd, MSARegister ws, uint32_t n);
+ void splati_d(MSARegister wd, MSARegister ws, uint32_t n);
+ void copy_s_b(Register rd, MSARegister ws, uint32_t n);
+ void copy_s_h(Register rd, MSARegister ws, uint32_t n);
+ void copy_s_w(Register rd, MSARegister ws, uint32_t n);
+ void copy_s_d(Register rd, MSARegister ws, uint32_t n);
+ void copy_u_b(Register rd, MSARegister ws, uint32_t n);
+ void copy_u_h(Register rd, MSARegister ws, uint32_t n);
+ void copy_u_w(Register rd, MSARegister ws, uint32_t n);
+ void insert_b(MSARegister wd, uint32_t n, Register rs);
+ void insert_h(MSARegister wd, uint32_t n, Register rs);
+ void insert_w(MSARegister wd, uint32_t n, Register rs);
+ void insert_d(MSARegister wd, uint32_t n, Register rs);
+ void insve_b(MSARegister wd, uint32_t n, MSARegister ws);
+ void insve_h(MSARegister wd, uint32_t n, MSARegister ws);
+ void insve_w(MSARegister wd, uint32_t n, MSARegister ws);
+ void insve_d(MSARegister wd, uint32_t n, MSARegister ws);
+ void move_v(MSARegister wd, MSARegister ws);
+ void ctcmsa(MSAControlRegister cd, Register rs);
+ void cfcmsa(Register rd, MSAControlRegister cs);
+
+ void slli_b(MSARegister wd, MSARegister ws, uint32_t m);
+ void slli_h(MSARegister wd, MSARegister ws, uint32_t m);
+ void slli_w(MSARegister wd, MSARegister ws, uint32_t m);
+ void slli_d(MSARegister wd, MSARegister ws, uint32_t m);
+ void srai_b(MSARegister wd, MSARegister ws, uint32_t m);
+ void srai_h(MSARegister wd, MSARegister ws, uint32_t m);
+ void srai_w(MSARegister wd, MSARegister ws, uint32_t m);
+ void srai_d(MSARegister wd, MSARegister ws, uint32_t m);
+ void srli_b(MSARegister wd, MSARegister ws, uint32_t m);
+ void srli_h(MSARegister wd, MSARegister ws, uint32_t m);
+ void srli_w(MSARegister wd, MSARegister ws, uint32_t m);
+ void srli_d(MSARegister wd, MSARegister ws, uint32_t m);
+ void bclri_b(MSARegister wd, MSARegister ws, uint32_t m);
+ void bclri_h(MSARegister wd, MSARegister ws, uint32_t m);
+ void bclri_w(MSARegister wd, MSARegister ws, uint32_t m);
+ void bclri_d(MSARegister wd, MSARegister ws, uint32_t m);
+ void bseti_b(MSARegister wd, MSARegister ws, uint32_t m);
+ void bseti_h(MSARegister wd, MSARegister ws, uint32_t m);
+ void bseti_w(MSARegister wd, MSARegister ws, uint32_t m);
+ void bseti_d(MSARegister wd, MSARegister ws, uint32_t m);
+ void bnegi_b(MSARegister wd, MSARegister ws, uint32_t m);
+ void bnegi_h(MSARegister wd, MSARegister ws, uint32_t m);
+ void bnegi_w(MSARegister wd, MSARegister ws, uint32_t m);
+ void bnegi_d(MSARegister wd, MSARegister ws, uint32_t m);
+ void binsli_b(MSARegister wd, MSARegister ws, uint32_t m);
+ void binsli_h(MSARegister wd, MSARegister ws, uint32_t m);
+ void binsli_w(MSARegister wd, MSARegister ws, uint32_t m);
+ void binsli_d(MSARegister wd, MSARegister ws, uint32_t m);
+ void binsri_b(MSARegister wd, MSARegister ws, uint32_t m);
+ void binsri_h(MSARegister wd, MSARegister ws, uint32_t m);
+ void binsri_w(MSARegister wd, MSARegister ws, uint32_t m);
+ void binsri_d(MSARegister wd, MSARegister ws, uint32_t m);
+ void sat_s_b(MSARegister wd, MSARegister ws, uint32_t m);
+ void sat_s_h(MSARegister wd, MSARegister ws, uint32_t m);
+ void sat_s_w(MSARegister wd, MSARegister ws, uint32_t m);
+ void sat_s_d(MSARegister wd, MSARegister ws, uint32_t m);
+ void sat_u_b(MSARegister wd, MSARegister ws, uint32_t m);
+ void sat_u_h(MSARegister wd, MSARegister ws, uint32_t m);
+ void sat_u_w(MSARegister wd, MSARegister ws, uint32_t m);
+ void sat_u_d(MSARegister wd, MSARegister ws, uint32_t m);
+ void srari_b(MSARegister wd, MSARegister ws, uint32_t m);
+ void srari_h(MSARegister wd, MSARegister ws, uint32_t m);
+ void srari_w(MSARegister wd, MSARegister ws, uint32_t m);
+ void srari_d(MSARegister wd, MSARegister ws, uint32_t m);
+ void srlri_b(MSARegister wd, MSARegister ws, uint32_t m);
+ void srlri_h(MSARegister wd, MSARegister ws, uint32_t m);
+ void srlri_w(MSARegister wd, MSARegister ws, uint32_t m);
+ void srlri_d(MSARegister wd, MSARegister ws, uint32_t m);
+
+ // Check the code size generated from label to here.
+ int SizeOfCodeGeneratedSince(Label* label) {
+ return pc_offset() - label->pos();
+ }
+
+ // Check the number of instructions generated from label to here.
+ int InstructionsGeneratedSince(Label* label) {
+ return SizeOfCodeGeneratedSince(label) / kInstrSize;
+ }
+
+ // Class for scoping postponing the trampoline pool generation.
+ class BlockTrampolinePoolScope {
+ public:
+ explicit BlockTrampolinePoolScope(Assembler* assem) : assem_(assem) {
+ assem_->StartBlockTrampolinePool();
+ }
+ ~BlockTrampolinePoolScope() { assem_->EndBlockTrampolinePool(); }
+
+ private:
+ Assembler* assem_;
+
+ DISALLOW_IMPLICIT_CONSTRUCTORS(BlockTrampolinePoolScope);
+ };
+
+ // Class for postponing the assembly buffer growth. Typically used for
+ // sequences of instructions that must be emitted as a unit, before
+ // buffer growth (and relocation) can occur.
+ // This blocking scope is not nestable.
+ class BlockGrowBufferScope {
+ public:
+ explicit BlockGrowBufferScope(Assembler* assem) : assem_(assem) {
+ assem_->StartBlockGrowBuffer();
+ }
+ ~BlockGrowBufferScope() { assem_->EndBlockGrowBuffer(); }
+
+ private:
+ Assembler* assem_;
+
+ DISALLOW_IMPLICIT_CONSTRUCTORS(BlockGrowBufferScope);
+ };
+
+ // Record a deoptimization reason that can be used by a log or cpu profiler.
+ // Use --trace-deopt to enable.
+ void RecordDeoptReason(DeoptimizeReason reason, SourcePosition position,
+ int id);
+
+ static int RelocateInternalReference(RelocInfo::Mode rmode, Address pc,
+ intptr_t pc_delta);
+
+ // Writes a single byte or word of data in the code stream. Used for
+ // inline tables, e.g., jump-tables.
+ void db(uint8_t data);
+ void dd(uint32_t data);
+ void dq(uint64_t data);
+ void dp(uintptr_t data) { dq(data); }
+ void dd(Label* label);
+
+ // Postpone the generation of the trampoline pool for the specified number of
+ // instructions.
+ void BlockTrampolinePoolFor(int instructions);
+
+ // Check if there is less than kGap bytes available in the buffer.
+ // If this is the case, we need to grow the buffer before emitting
+ // an instruction or relocation information.
+ inline bool overflow() const { return pc_ >= reloc_info_writer.pos() - kGap; }
+
+ // Get the number of bytes available in the buffer.
+ inline intptr_t available_space() const {
+ return reloc_info_writer.pos() - pc_;
+ }
+
+ // Read/patch instructions.
+ static Instr instr_at(Address pc) { return *reinterpret_cast<Instr*>(pc); }
+ static void instr_at_put(Address pc, Instr instr) {
+ *reinterpret_cast<Instr*>(pc) = instr;
+ }
+ Instr instr_at(int pos) {
+ return *reinterpret_cast<Instr*>(buffer_start_ + pos);
+ }
+ void instr_at_put(int pos, Instr instr) {
+ *reinterpret_cast<Instr*>(buffer_start_ + pos) = instr;
+ }
+
+ // Check if an instruction is a branch of some kind.
+ static bool IsBranch(Instr instr);
+ static bool IsMsaBranch(Instr instr);
+ static bool IsBc(Instr instr);
+ static bool IsNal(Instr instr);
+ static bool IsBzc(Instr instr);
+
+ static bool IsBeq(Instr instr);
+ static bool IsBne(Instr instr);
+ static bool IsBeqzc(Instr instr);
+ static bool IsBnezc(Instr instr);
+ static bool IsBeqc(Instr instr);
+ static bool IsBnec(Instr instr);
+
+ static bool IsJump(Instr instr);
+ static bool IsJ(Instr instr);
+ static bool IsLui(Instr instr);
+ static bool IsOri(Instr instr);
+ static bool IsMov(Instr instr, Register rd, Register rs);
+
+ static bool IsJal(Instr instr);
+ static bool IsJr(Instr instr);
+ static bool IsJalr(Instr instr);
+
+ static bool IsNop(Instr instr, unsigned int type);
+ static bool IsPop(Instr instr);
+ static bool IsPush(Instr instr);
+ static bool IsLwRegFpOffset(Instr instr);
+ static bool IsSwRegFpOffset(Instr instr);
+ static bool IsLwRegFpNegOffset(Instr instr);
+ static bool IsSwRegFpNegOffset(Instr instr);
+
+ static Register GetRtReg(Instr instr);
+ static Register GetRsReg(Instr instr);
+ static Register GetRdReg(Instr instr);
+
+ static uint32_t GetRt(Instr instr);
+ static uint32_t GetRtField(Instr instr);
+ static uint32_t GetRs(Instr instr);
+ static uint32_t GetRsField(Instr instr);
+ static uint32_t GetRd(Instr instr);
+ static uint32_t GetRdField(Instr instr);
+ static uint32_t GetSa(Instr instr);
+ static uint32_t GetSaField(Instr instr);
+ static uint32_t GetOpcodeField(Instr instr);
+ static uint32_t GetFunction(Instr instr);
+ static uint32_t GetFunctionField(Instr instr);
+ static uint32_t GetImmediate16(Instr instr);
+ static uint32_t GetLabelConst(Instr instr);
+
+ static int32_t GetBranchOffset(Instr instr);
+ static bool IsLw(Instr instr);
+ static int16_t GetLwOffset(Instr instr);
+ static Instr SetLwOffset(Instr instr, int16_t offset);
+
+ static bool IsSw(Instr instr);
+ static Instr SetSwOffset(Instr instr, int16_t offset);
+ static bool IsAddImmediate(Instr instr);
+ static Instr SetAddImmediateOffset(Instr instr, int16_t offset);
+
+ static bool IsAndImmediate(Instr instr);
+ static bool IsEmittedConstant(Instr instr);
+
+ void CheckTrampolinePool();
+
+ bool IsPrevInstrCompactBranch() { return prev_instr_compact_branch_; }
+ static bool IsCompactBranchSupported() { return kArchVariant == kMips64r6; }
+
+ inline int UnboundLabelsCount() { return unbound_labels_count_; }
+
+ protected:
+ // Load Scaled Address instructions.
+ void lsa(Register rd, Register rt, Register rs, uint8_t sa);
+ void dlsa(Register rd, Register rt, Register rs, uint8_t sa);
+
+ // Readable constants for base and offset adjustment helper, these indicate if
+ // aside from offset, another value like offset + 4 should fit into int16.
+ enum class OffsetAccessType : bool {
+ SINGLE_ACCESS = false,
+ TWO_ACCESSES = true
+ };
+
+ // Helper function for memory load/store using base register and offset.
+ void AdjustBaseAndOffset(
+ MemOperand* src,
+ OffsetAccessType access_type = OffsetAccessType::SINGLE_ACCESS,
+ int second_access_add_to_offset = 4);
+
+ inline static void set_target_internal_reference_encoded_at(Address pc,
+ Address target);
+
+ int64_t buffer_space() const { return reloc_info_writer.pos() - pc_; }
+
+ // Decode branch instruction at pos and return branch target pos.
+ int target_at(int pos, bool is_internal);
+
+ // Patch branch instruction at pos to branch to given branch target pos.
+ void target_at_put(int pos, int target_pos, bool is_internal);
+
+ // Say if we need to relocate with this mode.
+ bool MustUseReg(RelocInfo::Mode rmode);
+
+ // Record reloc info for current pc_.
+ void RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data = 0);
+
+ // Block the emission of the trampoline pool before pc_offset.
+ void BlockTrampolinePoolBefore(int pc_offset) {
+ if (no_trampoline_pool_before_ < pc_offset)
+ no_trampoline_pool_before_ = pc_offset;
+ }
+
+ void StartBlockTrampolinePool() { trampoline_pool_blocked_nesting_++; }
+
+ void EndBlockTrampolinePool() {
+ trampoline_pool_blocked_nesting_--;
+ if (trampoline_pool_blocked_nesting_ == 0) {
+ CheckTrampolinePoolQuick(1);
+ }
+ }
+
+ bool is_trampoline_pool_blocked() const {
+ return trampoline_pool_blocked_nesting_ > 0;
+ }
+
+ bool has_exception() const { return internal_trampoline_exception_; }
+
+ bool is_trampoline_emitted() const { return trampoline_emitted_; }
+
+ // Temporarily block automatic assembly buffer growth.
+ void StartBlockGrowBuffer() {
+ DCHECK(!block_buffer_growth_);
+ block_buffer_growth_ = true;
+ }
+
+ void EndBlockGrowBuffer() {
+ DCHECK(block_buffer_growth_);
+ block_buffer_growth_ = false;
+ }
+
+ bool is_buffer_growth_blocked() const { return block_buffer_growth_; }
+
+ void EmitForbiddenSlotInstruction() {
+ if (IsPrevInstrCompactBranch()) {
+ nop();
+ }
+ }
+
+ void CheckTrampolinePoolQuick(int extra_instructions = 0) {
+ if (pc_offset() >= next_buffer_check_ - extra_instructions * kInstrSize) {
+ CheckTrampolinePool();
+ }
+ }
+
+ void set_last_call_pc_(byte* pc) { last_call_pc_ = pc; }
+
+ private:
+ // Avoid overflows for displacements etc.
+ static const int kMaximalBufferSize = 512 * MB;
+
+ // Buffer size and constant pool distance are checked together at regular
+ // intervals of kBufferCheckInterval emitted bytes.
+ static constexpr int kBufferCheckInterval = 1 * KB / 2;
+
+ // Code generation.
+ // The relocation writer's position is at least kGap bytes below the end of
+ // the generated instructions. This is so that multi-instruction sequences do
+ // not have to check for overflow. The same is true for writes of large
+ // relocation info entries.
+ static constexpr int kGap = 64;
+ STATIC_ASSERT(AssemblerBase::kMinimalBufferSize >= 2 * kGap);
+
+ // Repeated checking whether the trampoline pool should be emitted is rather
+ // expensive. By default we only check again once a number of instructions
+ // has been generated.
+ static constexpr int kCheckConstIntervalInst = 32;
+ static constexpr int kCheckConstInterval =
+ kCheckConstIntervalInst * kInstrSize;
+
+ int next_buffer_check_; // pc offset of next buffer check.
+
+ // Emission of the trampoline pool may be blocked in some code sequences.
+ int trampoline_pool_blocked_nesting_; // Block emission if this is not zero.
+ int no_trampoline_pool_before_; // Block emission before this pc offset.
+
+ // Keep track of the last emitted pool to guarantee a maximal distance.
+ int last_trampoline_pool_end_; // pc offset of the end of the last pool.
+
+ // Automatic growth of the assembly buffer may be blocked for some sequences.
+ bool block_buffer_growth_; // Block growth when true.
+
+ // Relocation information generation.
+ // Each relocation is encoded as a variable size value.
+ static constexpr int kMaxRelocSize = RelocInfoWriter::kMaxSize;
+ RelocInfoWriter reloc_info_writer;
+
+ // The bound position, before this we cannot do instruction elimination.
+ int last_bound_pos_;
+
+ // Readable constants for compact branch handling in emit()
+ enum class CompactBranchType : bool { NO = false, COMPACT_BRANCH = true };
+
+ // Code emission.
+ inline void CheckBuffer();
+ void GrowBuffer();
+ inline void emit(Instr x,
+ CompactBranchType is_compact_branch = CompactBranchType::NO);
+ inline void emit(uint64_t x);
+ inline void CheckForEmitInForbiddenSlot();
+ template <typename T>
+ inline void EmitHelper(T x);
+ inline void EmitHelper(Instr x, CompactBranchType is_compact_branch);
+
+ // Instruction generation.
+ // We have 3 different kind of encoding layout on MIPS.
+ // However due to many different types of objects encoded in the same fields
+ // we have quite a few aliases for each mode.
+ // Using the same structure to refer to Register and FPURegister would spare a
+ // few aliases, but mixing both does not look clean to me.
+ // Anyway we could surely implement this differently.
+
+ void GenInstrRegister(Opcode opcode, Register rs, Register rt, Register rd,
+ uint16_t sa = 0, SecondaryField func = nullptrSF);
+
+ void GenInstrRegister(Opcode opcode, Register rs, Register rt, uint16_t msb,
+ uint16_t lsb, SecondaryField func);
+
+ void GenInstrRegister(Opcode opcode, SecondaryField fmt, FPURegister ft,
+ FPURegister fs, FPURegister fd,
+ SecondaryField func = nullptrSF);
+
+ void GenInstrRegister(Opcode opcode, FPURegister fr, FPURegister ft,
+ FPURegister fs, FPURegister fd,
+ SecondaryField func = nullptrSF);
+
+ void GenInstrRegister(Opcode opcode, SecondaryField fmt, Register rt,
+ FPURegister fs, FPURegister fd,
+ SecondaryField func = nullptrSF);
+
+ void GenInstrRegister(Opcode opcode, SecondaryField fmt, Register rt,
+ FPUControlRegister fs, SecondaryField func = nullptrSF);
+
+ void GenInstrImmediate(
+ Opcode opcode, Register rs, Register rt, int32_t j,
+ CompactBranchType is_compact_branch = CompactBranchType::NO);
+ void GenInstrImmediate(
+ Opcode opcode, Register rs, SecondaryField SF, int32_t j,
+ CompactBranchType is_compact_branch = CompactBranchType::NO);
+ void GenInstrImmediate(
+ Opcode opcode, Register r1, FPURegister r2, int32_t j,
+ CompactBranchType is_compact_branch = CompactBranchType::NO);
+ void GenInstrImmediate(Opcode opcode, Register base, Register rt,
+ int32_t offset9, int bit6, SecondaryField func);
+ void GenInstrImmediate(
+ Opcode opcode, Register rs, int32_t offset21,
+ CompactBranchType is_compact_branch = CompactBranchType::NO);
+ void GenInstrImmediate(Opcode opcode, Register rs, uint32_t offset21);
+ void GenInstrImmediate(
+ Opcode opcode, int32_t offset26,
+ CompactBranchType is_compact_branch = CompactBranchType::NO);
+
+ void GenInstrJump(Opcode opcode, uint32_t address);
+
+ // MSA
+ void GenInstrMsaI8(SecondaryField operation, uint32_t imm8, MSARegister ws,
+ MSARegister wd);
+
+ void GenInstrMsaI5(SecondaryField operation, SecondaryField df, int32_t imm5,
+ MSARegister ws, MSARegister wd);
+
+ void GenInstrMsaBit(SecondaryField operation, SecondaryField df, uint32_t m,
+ MSARegister ws, MSARegister wd);
+
+ void GenInstrMsaI10(SecondaryField operation, SecondaryField df,
+ int32_t imm10, MSARegister wd);
+
+ template <typename RegType>
+ void GenInstrMsa3R(SecondaryField operation, SecondaryField df, RegType t,
+ MSARegister ws, MSARegister wd);
+
+ template <typename DstType, typename SrcType>
+ void GenInstrMsaElm(SecondaryField operation, SecondaryField df, uint32_t n,
+ SrcType src, DstType dst);
+
+ void GenInstrMsa3RF(SecondaryField operation, uint32_t df, MSARegister wt,
+ MSARegister ws, MSARegister wd);
+
+ void GenInstrMsaVec(SecondaryField operation, MSARegister wt, MSARegister ws,
+ MSARegister wd);
+
+ void GenInstrMsaMI10(SecondaryField operation, int32_t s10, Register rs,
+ MSARegister wd);
+
+ void GenInstrMsa2R(SecondaryField operation, SecondaryField df,
+ MSARegister ws, MSARegister wd);
+
+ void GenInstrMsa2RF(SecondaryField operation, SecondaryField df,
+ MSARegister ws, MSARegister wd);
+
+ void GenInstrMsaBranch(SecondaryField operation, MSARegister wt,
+ int32_t offset16);
+
+ inline bool is_valid_msa_df_m(SecondaryField bit_df, uint32_t m) {
+ switch (bit_df) {
+ case BIT_DF_b:
+ return is_uint3(m);
+ case BIT_DF_h:
+ return is_uint4(m);
+ case BIT_DF_w:
+ return is_uint5(m);
+ case BIT_DF_d:
+ return is_uint6(m);
+ default:
+ return false;
+ }
+ }
+
+ inline bool is_valid_msa_df_n(SecondaryField elm_df, uint32_t n) {
+ switch (elm_df) {
+ case ELM_DF_B:
+ return is_uint4(n);
+ case ELM_DF_H:
+ return is_uint3(n);
+ case ELM_DF_W:
+ return is_uint2(n);
+ case ELM_DF_D:
+ return is_uint1(n);
+ default:
+ return false;
+ }
+ }
+
+ // Labels.
+ void print(const Label* L);
+ void bind_to(Label* L, int pos);
+ void next(Label* L, bool is_internal);
+
+ // One trampoline consists of:
+ // - space for trampoline slots,
+ // - space for labels.
+ //
+ // Space for trampoline slots is equal to slot_count * 2 * kInstrSize.
+ // Space for trampoline slots precedes space for labels. Each label is of one
+ // instruction size, so total amount for labels is equal to
+ // label_count * kInstrSize.
+ class Trampoline {
+ public:
+ Trampoline() {
+ start_ = 0;
+ next_slot_ = 0;
+ free_slot_count_ = 0;
+ end_ = 0;
+ }
+ Trampoline(int start, int slot_count) {
+ start_ = start;
+ next_slot_ = start;
+ free_slot_count_ = slot_count;
+ end_ = start + slot_count * kTrampolineSlotsSize;
+ }
+ int start() { return start_; }
+ int end() { return end_; }
+ int take_slot() {
+ int trampoline_slot = kInvalidSlotPos;
+ if (free_slot_count_ <= 0) {
+ // We have run out of space on trampolines.
+ // Make sure we fail in debug mode, so we become aware of each case
+ // when this happens.
+ DCHECK(0);
+ // Internal exception will be caught.
+ } else {
+ trampoline_slot = next_slot_;
+ free_slot_count_--;
+ next_slot_ += kTrampolineSlotsSize;
+ }
+ return trampoline_slot;
+ }
+
+ private:
+ int start_;
+ int end_;
+ int next_slot_;
+ int free_slot_count_;
+ };
+
+ int32_t get_trampoline_entry(int32_t pos);
+ int unbound_labels_count_;
+ // After trampoline is emitted, long branches are used in generated code for
+ // the forward branches whose target offsets could be beyond reach of branch
+ // instruction. We use this information to trigger different mode of
+ // branch instruction generation, where we use jump instructions rather
+ // than regular branch instructions.
+ bool trampoline_emitted_;
+ static constexpr int kInvalidSlotPos = -1;
+
+ // Internal reference positions, required for unbounded internal reference
+ // labels.
+ std::set<int64_t> internal_reference_positions_;
+ bool is_internal_reference(Label* L) {
+ return internal_reference_positions_.find(L->pos()) !=
+ internal_reference_positions_.end();
+ }
+
+ void EmittedCompactBranchInstruction() { prev_instr_compact_branch_ = true; }
+ void ClearCompactBranchState() { prev_instr_compact_branch_ = false; }
+ bool prev_instr_compact_branch_ = false;
+
+ Trampoline trampoline_;
+ bool internal_trampoline_exception_;
+
+ // Keep track of the last Call's position to ensure that safepoint can get the
+ // correct information even if there is a trampoline immediately after the
+ // Call.
+ byte* last_call_pc_;
+
+ RegList scratch_register_list_;
+
+ private:
+ void AllocateAndInstallRequestedHeapObjects(Isolate* isolate);
+
+ int WriteCodeComments();
+
+ friend class RegExpMacroAssemblerMIPS;
+ friend class RelocInfo;
+ friend class BlockTrampolinePoolScope;
+ friend class EnsureSpace;
+};
+
+class EnsureSpace {
+ public:
+ explicit inline EnsureSpace(Assembler* assembler);
+};
+
+class V8_EXPORT_PRIVATE UseScratchRegisterScope {
+ public:
+ explicit UseScratchRegisterScope(Assembler* assembler);
+ ~UseScratchRegisterScope();
+
+ Register Acquire();
+ bool hasAvailable() const;
+
+ private:
+ RegList* available_;
+ RegList old_available_;
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_MIPS64_ASSEMBLER_MIPS64_H_
diff --git a/src/codegen/mips64/constants-mips64.cc b/src/codegen/mips64/constants-mips64.cc
new file mode 100644
index 0000000..e4ee3c1
--- /dev/null
+++ b/src/codegen/mips64/constants-mips64.cc
@@ -0,0 +1,144 @@
+// Copyright 2011 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#if V8_TARGET_ARCH_MIPS64
+
+#include "src/codegen/mips64/constants-mips64.h"
+
+namespace v8 {
+namespace internal {
+
+// -----------------------------------------------------------------------------
+// Registers.
+
+// These register names are defined in a way to match the native disassembler
+// formatting. See for example the command "objdump -d <binary file>".
+const char* Registers::names_[kNumSimuRegisters] = {
+ "zero_reg", "at", "v0", "v1", "a0", "a1", "a2", "a3", "a4",
+ "a5", "a6", "a7", "t0", "t1", "t2", "t3", "s0", "s1",
+ "s2", "s3", "s4", "s5", "s6", "s7", "t8", "t9", "k0",
+ "k1", "gp", "sp", "fp", "ra", "LO", "HI", "pc"};
+
+// List of alias names which can be used when referring to MIPS registers.
+const Registers::RegisterAlias Registers::aliases_[] = {
+ {0, "zero"},
+ {23, "cp"},
+ {30, "s8"},
+ {30, "s8_fp"},
+ {kInvalidRegister, nullptr}};
+
+const char* Registers::Name(int reg) {
+ const char* result;
+ if ((0 <= reg) && (reg < kNumSimuRegisters)) {
+ result = names_[reg];
+ } else {
+ result = "noreg";
+ }
+ return result;
+}
+
+int Registers::Number(const char* name) {
+ // Look through the canonical names.
+ for (int i = 0; i < kNumSimuRegisters; i++) {
+ if (strcmp(names_[i], name) == 0) {
+ return i;
+ }
+ }
+
+ // Look through the alias names.
+ int i = 0;
+ while (aliases_[i].reg != kInvalidRegister) {
+ if (strcmp(aliases_[i].name, name) == 0) {
+ return aliases_[i].reg;
+ }
+ i++;
+ }
+
+ // No register with the reguested name found.
+ return kInvalidRegister;
+}
+
+const char* FPURegisters::names_[kNumFPURegisters] = {
+ "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "f9", "f10",
+ "f11", "f12", "f13", "f14", "f15", "f16", "f17", "f18", "f19", "f20", "f21",
+ "f22", "f23", "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31"};
+
+// List of alias names which can be used when referring to MIPS registers.
+const FPURegisters::RegisterAlias FPURegisters::aliases_[] = {
+ {kInvalidRegister, nullptr}};
+
+const char* FPURegisters::Name(int creg) {
+ const char* result;
+ if ((0 <= creg) && (creg < kNumFPURegisters)) {
+ result = names_[creg];
+ } else {
+ result = "nocreg";
+ }
+ return result;
+}
+
+int FPURegisters::Number(const char* name) {
+ // Look through the canonical names.
+ for (int i = 0; i < kNumFPURegisters; i++) {
+ if (strcmp(names_[i], name) == 0) {
+ return i;
+ }
+ }
+
+ // Look through the alias names.
+ int i = 0;
+ while (aliases_[i].creg != kInvalidRegister) {
+ if (strcmp(aliases_[i].name, name) == 0) {
+ return aliases_[i].creg;
+ }
+ i++;
+ }
+
+ // No Cregister with the reguested name found.
+ return kInvalidFPURegister;
+}
+
+const char* MSARegisters::names_[kNumMSARegisters] = {
+ "w0", "w1", "w2", "w3", "w4", "w5", "w6", "w7", "w8", "w9", "w10",
+ "w11", "w12", "w13", "w14", "w15", "w16", "w17", "w18", "w19", "w20", "w21",
+ "w22", "w23", "w24", "w25", "w26", "w27", "w28", "w29", "w30", "w31"};
+
+const MSARegisters::RegisterAlias MSARegisters::aliases_[] = {
+ {kInvalidRegister, nullptr}};
+
+const char* MSARegisters::Name(int creg) {
+ const char* result;
+ if ((0 <= creg) && (creg < kNumMSARegisters)) {
+ result = names_[creg];
+ } else {
+ result = "nocreg";
+ }
+ return result;
+}
+
+int MSARegisters::Number(const char* name) {
+ // Look through the canonical names.
+ for (int i = 0; i < kNumMSARegisters; i++) {
+ if (strcmp(names_[i], name) == 0) {
+ return i;
+ }
+ }
+
+ // Look through the alias names.
+ int i = 0;
+ while (aliases_[i].creg != kInvalidRegister) {
+ if (strcmp(aliases_[i].name, name) == 0) {
+ return aliases_[i].creg;
+ }
+ i++;
+ }
+
+ // No Cregister with the reguested name found.
+ return kInvalidMSARegister;
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_MIPS64
diff --git a/src/codegen/mips64/constants-mips64.h b/src/codegen/mips64/constants-mips64.h
new file mode 100644
index 0000000..751fa3c
--- /dev/null
+++ b/src/codegen/mips64/constants-mips64.h
@@ -0,0 +1,1986 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_MIPS64_CONSTANTS_MIPS64_H_
+#define V8_CODEGEN_MIPS64_CONSTANTS_MIPS64_H_
+
+#include "src/base/logging.h"
+#include "src/base/macros.h"
+#include "src/common/globals.h"
+
+// UNIMPLEMENTED_ macro for MIPS.
+#ifdef DEBUG
+#define UNIMPLEMENTED_MIPS() \
+ v8::internal::PrintF("%s, \tline %d: \tfunction %s not implemented. \n", \
+ __FILE__, __LINE__, __func__)
+#else
+#define UNIMPLEMENTED_MIPS()
+#endif
+
+#define UNSUPPORTED_MIPS() v8::internal::PrintF("Unsupported instruction.\n")
+
+enum ArchVariants { kMips64r2, kMips64r6 };
+
+#ifdef _MIPS_ARCH_MIPS64R2
+static const ArchVariants kArchVariant = kMips64r2;
+#elif _MIPS_ARCH_MIPS64R6
+static const ArchVariants kArchVariant = kMips64r6;
+#else
+static const ArchVariants kArchVariant = kMips64r2;
+#endif
+
+enum Endianness { kLittle, kBig };
+
+#if defined(V8_TARGET_LITTLE_ENDIAN)
+static const Endianness kArchEndian = kLittle;
+#elif defined(V8_TARGET_BIG_ENDIAN)
+static const Endianness kArchEndian = kBig;
+#else
+#error Unknown endianness
+#endif
+
+// TODO(plind): consider renaming these ...
+#if defined(__mips_hard_float) && __mips_hard_float != 0
+// Use floating-point coprocessor instructions. This flag is raised when
+// -mhard-float is passed to the compiler.
+const bool IsMipsSoftFloatABI = false;
+#elif defined(__mips_soft_float) && __mips_soft_float != 0
+// This flag is raised when -msoft-float is passed to the compiler.
+// Although FPU is a base requirement for v8, soft-float ABI is used
+// on soft-float systems with FPU kernel emulation.
+const bool IsMipsSoftFloatABI = true;
+#else
+const bool IsMipsSoftFloatABI = true;
+#endif
+
+#if defined(V8_TARGET_LITTLE_ENDIAN)
+const uint32_t kMipsLwrOffset = 0;
+const uint32_t kMipsLwlOffset = 3;
+const uint32_t kMipsSwrOffset = 0;
+const uint32_t kMipsSwlOffset = 3;
+const uint32_t kMipsLdrOffset = 0;
+const uint32_t kMipsLdlOffset = 7;
+const uint32_t kMipsSdrOffset = 0;
+const uint32_t kMipsSdlOffset = 7;
+#elif defined(V8_TARGET_BIG_ENDIAN)
+const uint32_t kMipsLwrOffset = 3;
+const uint32_t kMipsLwlOffset = 0;
+const uint32_t kMipsSwrOffset = 3;
+const uint32_t kMipsSwlOffset = 0;
+const uint32_t kMipsLdrOffset = 7;
+const uint32_t kMipsLdlOffset = 0;
+const uint32_t kMipsSdrOffset = 7;
+const uint32_t kMipsSdlOffset = 0;
+#else
+#error Unknown endianness
+#endif
+
+#if defined(V8_TARGET_LITTLE_ENDIAN)
+const uint32_t kLeastSignificantByteInInt32Offset = 0;
+const uint32_t kLessSignificantWordInDoublewordOffset = 0;
+#elif defined(V8_TARGET_BIG_ENDIAN)
+const uint32_t kLeastSignificantByteInInt32Offset = 3;
+const uint32_t kLessSignificantWordInDoublewordOffset = 4;
+#else
+#error Unknown endianness
+#endif
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+#include <inttypes.h>
+
+// Defines constants and accessor classes to assemble, disassemble and
+// simulate MIPS32 instructions.
+//
+// See: MIPS32 Architecture For Programmers
+// Volume II: The MIPS32 Instruction Set
+// Try www.cs.cornell.edu/courses/cs3410/2008fa/MIPS_Vol2.pdf.
+
+namespace v8 {
+namespace internal {
+
+// TODO(sigurds): Change this value once we use relative jumps.
+constexpr size_t kMaxPCRelativeCodeRangeInMB = 0;
+
+// -----------------------------------------------------------------------------
+// Registers and FPURegisters.
+
+// Number of general purpose registers.
+const int kNumRegisters = 32;
+const int kInvalidRegister = -1;
+
+// Number of registers with HI, LO, and pc.
+const int kNumSimuRegisters = 35;
+
+// In the simulator, the PC register is simulated as the 34th register.
+const int kPCRegister = 34;
+
+// Number coprocessor registers.
+const int kNumFPURegisters = 32;
+const int kInvalidFPURegister = -1;
+
+// Number of MSA registers
+const int kNumMSARegisters = 32;
+const int kInvalidMSARegister = -1;
+
+const int kInvalidMSAControlRegister = -1;
+const int kMSAIRRegister = 0;
+const int kMSACSRRegister = 1;
+const int kMSARegSize = 128;
+const int kMSALanesByte = kMSARegSize / 8;
+const int kMSALanesHalf = kMSARegSize / 16;
+const int kMSALanesWord = kMSARegSize / 32;
+const int kMSALanesDword = kMSARegSize / 64;
+
+// FPU (coprocessor 1) control registers. Currently only FCSR is implemented.
+const int kFCSRRegister = 31;
+const int kInvalidFPUControlRegister = -1;
+const uint32_t kFPUInvalidResult = static_cast<uint32_t>(1u << 31) - 1;
+const int32_t kFPUInvalidResultNegative = static_cast<int32_t>(1u << 31);
+const uint64_t kFPU64InvalidResult =
+ static_cast<uint64_t>(static_cast<uint64_t>(1) << 63) - 1;
+const int64_t kFPU64InvalidResultNegative =
+ static_cast<int64_t>(static_cast<uint64_t>(1) << 63);
+
+// FCSR constants.
+const uint32_t kFCSRInexactFlagBit = 2;
+const uint32_t kFCSRUnderflowFlagBit = 3;
+const uint32_t kFCSROverflowFlagBit = 4;
+const uint32_t kFCSRDivideByZeroFlagBit = 5;
+const uint32_t kFCSRInvalidOpFlagBit = 6;
+const uint32_t kFCSRNaN2008FlagBit = 18;
+
+const uint32_t kFCSRInexactFlagMask = 1 << kFCSRInexactFlagBit;
+const uint32_t kFCSRUnderflowFlagMask = 1 << kFCSRUnderflowFlagBit;
+const uint32_t kFCSROverflowFlagMask = 1 << kFCSROverflowFlagBit;
+const uint32_t kFCSRDivideByZeroFlagMask = 1 << kFCSRDivideByZeroFlagBit;
+const uint32_t kFCSRInvalidOpFlagMask = 1 << kFCSRInvalidOpFlagBit;
+const uint32_t kFCSRNaN2008FlagMask = 1 << kFCSRNaN2008FlagBit;
+
+const uint32_t kFCSRFlagMask =
+ kFCSRInexactFlagMask | kFCSRUnderflowFlagMask | kFCSROverflowFlagMask |
+ kFCSRDivideByZeroFlagMask | kFCSRInvalidOpFlagMask;
+
+const uint32_t kFCSRExceptionFlagMask = kFCSRFlagMask ^ kFCSRInexactFlagMask;
+
+// 'pref' instruction hints
+const int32_t kPrefHintLoad = 0;
+const int32_t kPrefHintStore = 1;
+const int32_t kPrefHintLoadStreamed = 4;
+const int32_t kPrefHintStoreStreamed = 5;
+const int32_t kPrefHintLoadRetained = 6;
+const int32_t kPrefHintStoreRetained = 7;
+const int32_t kPrefHintWritebackInvalidate = 25;
+const int32_t kPrefHintPrepareForStore = 30;
+
+// Actual value of root register is offset from the root array's start
+// to take advantage of negative displacement values.
+// TODO(sigurds): Choose best value.
+constexpr int kRootRegisterBias = 256;
+
+// Helper functions for converting between register numbers and names.
+class Registers {
+ public:
+ // Return the name of the register.
+ static const char* Name(int reg);
+
+ // Lookup the register number for the name provided.
+ static int Number(const char* name);
+
+ struct RegisterAlias {
+ int reg;
+ const char* name;
+ };
+
+ static const int64_t kMaxValue = 0x7fffffffffffffffl;
+ static const int64_t kMinValue = 0x8000000000000000l;
+
+ private:
+ static const char* names_[kNumSimuRegisters];
+ static const RegisterAlias aliases_[];
+};
+
+// Helper functions for converting between register numbers and names.
+class FPURegisters {
+ public:
+ // Return the name of the register.
+ static const char* Name(int reg);
+
+ // Lookup the register number for the name provided.
+ static int Number(const char* name);
+
+ struct RegisterAlias {
+ int creg;
+ const char* name;
+ };
+
+ private:
+ static const char* names_[kNumFPURegisters];
+ static const RegisterAlias aliases_[];
+};
+
+// Helper functions for converting between register numbers and names.
+class MSARegisters {
+ public:
+ // Return the name of the register.
+ static const char* Name(int reg);
+
+ // Lookup the register number for the name provided.
+ static int Number(const char* name);
+
+ struct RegisterAlias {
+ int creg;
+ const char* name;
+ };
+
+ private:
+ static const char* names_[kNumMSARegisters];
+ static const RegisterAlias aliases_[];
+};
+
+// -----------------------------------------------------------------------------
+// Instructions encoding constants.
+
+// On MIPS all instructions are 32 bits.
+using Instr = int32_t;
+
+// Special Software Interrupt codes when used in the presence of the MIPS
+// simulator.
+enum SoftwareInterruptCodes {
+ // Transition to C code.
+ call_rt_redirected = 0xfffff
+};
+
+// On MIPS Simulator breakpoints can have different codes:
+// - Breaks between 0 and kMaxWatchpointCode are treated as simple watchpoints,
+// the simulator will run through them and print the registers.
+// - Breaks between kMaxWatchpointCode and kMaxStopCode are treated as stop()
+// instructions (see Assembler::stop()).
+// - Breaks larger than kMaxStopCode are simple breaks, dropping you into the
+// debugger.
+const uint32_t kMaxWatchpointCode = 31;
+const uint32_t kMaxStopCode = 127;
+STATIC_ASSERT(kMaxWatchpointCode < kMaxStopCode);
+
+// ----- Fields offset and length.
+const int kOpcodeShift = 26;
+const int kOpcodeBits = 6;
+const int kRsShift = 21;
+const int kRsBits = 5;
+const int kRtShift = 16;
+const int kRtBits = 5;
+const int kRdShift = 11;
+const int kRdBits = 5;
+const int kSaShift = 6;
+const int kSaBits = 5;
+const int kLsaSaBits = 2;
+const int kFunctionShift = 0;
+const int kFunctionBits = 6;
+const int kLuiShift = 16;
+const int kBp2Shift = 6;
+const int kBp2Bits = 2;
+const int kBp3Shift = 6;
+const int kBp3Bits = 3;
+const int kBaseShift = 21;
+const int kBaseBits = 5;
+const int kBit6Shift = 6;
+const int kBit6Bits = 1;
+
+const int kImm9Shift = 7;
+const int kImm9Bits = 9;
+const int kImm16Shift = 0;
+const int kImm16Bits = 16;
+const int kImm18Shift = 0;
+const int kImm18Bits = 18;
+const int kImm19Shift = 0;
+const int kImm19Bits = 19;
+const int kImm21Shift = 0;
+const int kImm21Bits = 21;
+const int kImm26Shift = 0;
+const int kImm26Bits = 26;
+const int kImm28Shift = 0;
+const int kImm28Bits = 28;
+const int kImm32Shift = 0;
+const int kImm32Bits = 32;
+const int kMsaImm8Shift = 16;
+const int kMsaImm8Bits = 8;
+const int kMsaImm5Shift = 16;
+const int kMsaImm5Bits = 5;
+const int kMsaImm10Shift = 11;
+const int kMsaImm10Bits = 10;
+const int kMsaImmMI10Shift = 16;
+const int kMsaImmMI10Bits = 10;
+
+// In branches and jumps immediate fields point to words, not bytes,
+// and are therefore shifted by 2.
+const int kImmFieldShift = 2;
+
+const int kFrBits = 5;
+const int kFrShift = 21;
+const int kFsShift = 11;
+const int kFsBits = 5;
+const int kFtShift = 16;
+const int kFtBits = 5;
+const int kFdShift = 6;
+const int kFdBits = 5;
+const int kFCccShift = 8;
+const int kFCccBits = 3;
+const int kFBccShift = 18;
+const int kFBccBits = 3;
+const int kFBtrueShift = 16;
+const int kFBtrueBits = 1;
+const int kWtBits = 5;
+const int kWtShift = 16;
+const int kWsBits = 5;
+const int kWsShift = 11;
+const int kWdBits = 5;
+const int kWdShift = 6;
+
+// ----- Miscellaneous useful masks.
+// Instruction bit masks.
+const int kOpcodeMask = ((1 << kOpcodeBits) - 1) << kOpcodeShift;
+const int kImm9Mask = ((1 << kImm9Bits) - 1) << kImm9Shift;
+const int kImm16Mask = ((1 << kImm16Bits) - 1) << kImm16Shift;
+const int kImm18Mask = ((1 << kImm18Bits) - 1) << kImm18Shift;
+const int kImm19Mask = ((1 << kImm19Bits) - 1) << kImm19Shift;
+const int kImm21Mask = ((1 << kImm21Bits) - 1) << kImm21Shift;
+const int kImm26Mask = ((1 << kImm26Bits) - 1) << kImm26Shift;
+const int kImm28Mask = ((1 << kImm28Bits) - 1) << kImm28Shift;
+const int kImm5Mask = ((1 << 5) - 1);
+const int kImm8Mask = ((1 << 8) - 1);
+const int kImm10Mask = ((1 << 10) - 1);
+const int kMsaI5I10Mask = ((7U << 23) | ((1 << 6) - 1));
+const int kMsaI8Mask = ((3U << 24) | ((1 << 6) - 1));
+const int kMsaI5Mask = ((7U << 23) | ((1 << 6) - 1));
+const int kMsaMI10Mask = (15U << 2);
+const int kMsaBITMask = ((7U << 23) | ((1 << 6) - 1));
+const int kMsaELMMask = (15U << 22);
+const int kMsaLongerELMMask = kMsaELMMask | (63U << 16);
+const int kMsa3RMask = ((7U << 23) | ((1 << 6) - 1));
+const int kMsa3RFMask = ((15U << 22) | ((1 << 6) - 1));
+const int kMsaVECMask = (23U << 21);
+const int kMsa2RMask = (7U << 18);
+const int kMsa2RFMask = (15U << 17);
+const int kRsFieldMask = ((1 << kRsBits) - 1) << kRsShift;
+const int kRtFieldMask = ((1 << kRtBits) - 1) << kRtShift;
+const int kRdFieldMask = ((1 << kRdBits) - 1) << kRdShift;
+const int kSaFieldMask = ((1 << kSaBits) - 1) << kSaShift;
+const int kFunctionFieldMask = ((1 << kFunctionBits) - 1) << kFunctionShift;
+// Misc masks.
+const int kHiMaskOf32 = 0xffff << 16; // Only to be used with 32-bit values
+const int kLoMaskOf32 = 0xffff;
+const int kSignMaskOf32 = 0x80000000; // Only to be used with 32-bit values
+const int kJumpAddrMask = (1 << (kImm26Bits + kImmFieldShift)) - 1;
+const int64_t kTop16MaskOf64 = (int64_t)0xffff << 48;
+const int64_t kHigher16MaskOf64 = (int64_t)0xffff << 32;
+const int64_t kUpper16MaskOf64 = (int64_t)0xffff << 16;
+const int32_t kJalRawMark = 0x00000000;
+const int32_t kJRawMark = 0xf0000000;
+const int32_t kJumpRawMask = 0xf0000000;
+
+// ----- MIPS Opcodes and Function Fields.
+// We use this presentation to stay close to the table representation in
+// MIPS32 Architecture For Programmers, Volume II: The MIPS32 Instruction Set.
+enum Opcode : uint32_t {
+ SPECIAL = 0U << kOpcodeShift,
+ REGIMM = 1U << kOpcodeShift,
+
+ J = ((0U << 3) + 2) << kOpcodeShift,
+ JAL = ((0U << 3) + 3) << kOpcodeShift,
+ BEQ = ((0U << 3) + 4) << kOpcodeShift,
+ BNE = ((0U << 3) + 5) << kOpcodeShift,
+ BLEZ = ((0U << 3) + 6) << kOpcodeShift,
+ BGTZ = ((0U << 3) + 7) << kOpcodeShift,
+
+ ADDI = ((1U << 3) + 0) << kOpcodeShift,
+ ADDIU = ((1U << 3) + 1) << kOpcodeShift,
+ SLTI = ((1U << 3) + 2) << kOpcodeShift,
+ SLTIU = ((1U << 3) + 3) << kOpcodeShift,
+ ANDI = ((1U << 3) + 4) << kOpcodeShift,
+ ORI = ((1U << 3) + 5) << kOpcodeShift,
+ XORI = ((1U << 3) + 6) << kOpcodeShift,
+ LUI = ((1U << 3) + 7) << kOpcodeShift, // LUI/AUI family.
+ DAUI = ((3U << 3) + 5) << kOpcodeShift,
+
+ BEQC = ((2U << 3) + 0) << kOpcodeShift,
+ COP1 = ((2U << 3) + 1) << kOpcodeShift, // Coprocessor 1 class.
+ BEQL = ((2U << 3) + 4) << kOpcodeShift,
+ BNEL = ((2U << 3) + 5) << kOpcodeShift,
+ BLEZL = ((2U << 3) + 6) << kOpcodeShift,
+ BGTZL = ((2U << 3) + 7) << kOpcodeShift,
+
+ DADDI = ((3U << 3) + 0) << kOpcodeShift, // This is also BNEC.
+ DADDIU = ((3U << 3) + 1) << kOpcodeShift,
+ LDL = ((3U << 3) + 2) << kOpcodeShift,
+ LDR = ((3U << 3) + 3) << kOpcodeShift,
+ SPECIAL2 = ((3U << 3) + 4) << kOpcodeShift,
+ MSA = ((3U << 3) + 6) << kOpcodeShift,
+ SPECIAL3 = ((3U << 3) + 7) << kOpcodeShift,
+
+ LB = ((4U << 3) + 0) << kOpcodeShift,
+ LH = ((4U << 3) + 1) << kOpcodeShift,
+ LWL = ((4U << 3) + 2) << kOpcodeShift,
+ LW = ((4U << 3) + 3) << kOpcodeShift,
+ LBU = ((4U << 3) + 4) << kOpcodeShift,
+ LHU = ((4U << 3) + 5) << kOpcodeShift,
+ LWR = ((4U << 3) + 6) << kOpcodeShift,
+ LWU = ((4U << 3) + 7) << kOpcodeShift,
+
+ SB = ((5U << 3) + 0) << kOpcodeShift,
+ SH = ((5U << 3) + 1) << kOpcodeShift,
+ SWL = ((5U << 3) + 2) << kOpcodeShift,
+ SW = ((5U << 3) + 3) << kOpcodeShift,
+ SDL = ((5U << 3) + 4) << kOpcodeShift,
+ SDR = ((5U << 3) + 5) << kOpcodeShift,
+ SWR = ((5U << 3) + 6) << kOpcodeShift,
+
+ LL = ((6U << 3) + 0) << kOpcodeShift,
+ LWC1 = ((6U << 3) + 1) << kOpcodeShift,
+ BC = ((6U << 3) + 2) << kOpcodeShift,
+ LLD = ((6U << 3) + 4) << kOpcodeShift,
+ LDC1 = ((6U << 3) + 5) << kOpcodeShift,
+ POP66 = ((6U << 3) + 6) << kOpcodeShift,
+ LD = ((6U << 3) + 7) << kOpcodeShift,
+
+ PREF = ((6U << 3) + 3) << kOpcodeShift,
+
+ SC = ((7U << 3) + 0) << kOpcodeShift,
+ SWC1 = ((7U << 3) + 1) << kOpcodeShift,
+ BALC = ((7U << 3) + 2) << kOpcodeShift,
+ PCREL = ((7U << 3) + 3) << kOpcodeShift,
+ SCD = ((7U << 3) + 4) << kOpcodeShift,
+ SDC1 = ((7U << 3) + 5) << kOpcodeShift,
+ POP76 = ((7U << 3) + 6) << kOpcodeShift,
+ SD = ((7U << 3) + 7) << kOpcodeShift,
+
+ COP1X = ((1U << 4) + 3) << kOpcodeShift,
+
+ // New r6 instruction.
+ POP06 = BLEZ, // bgeuc/bleuc, blezalc, bgezalc
+ POP07 = BGTZ, // bltuc/bgtuc, bgtzalc, bltzalc
+ POP10 = ADDI, // beqzalc, bovc, beqc
+ POP26 = BLEZL, // bgezc, blezc, bgec/blec
+ POP27 = BGTZL, // bgtzc, bltzc, bltc/bgtc
+ POP30 = DADDI, // bnezalc, bnvc, bnec
+};
+
+enum SecondaryField : uint32_t {
+ // SPECIAL Encoding of Function Field.
+ SLL = ((0U << 3) + 0),
+ MOVCI = ((0U << 3) + 1),
+ SRL = ((0U << 3) + 2),
+ SRA = ((0U << 3) + 3),
+ SLLV = ((0U << 3) + 4),
+ LSA = ((0U << 3) + 5),
+ SRLV = ((0U << 3) + 6),
+ SRAV = ((0U << 3) + 7),
+
+ JR = ((1U << 3) + 0),
+ JALR = ((1U << 3) + 1),
+ MOVZ = ((1U << 3) + 2),
+ MOVN = ((1U << 3) + 3),
+ BREAK = ((1U << 3) + 5),
+ SYNC = ((1U << 3) + 7),
+
+ MFHI = ((2U << 3) + 0),
+ CLZ_R6 = ((2U << 3) + 0),
+ CLO_R6 = ((2U << 3) + 1),
+ MFLO = ((2U << 3) + 2),
+ DCLZ_R6 = ((2U << 3) + 2),
+ DCLO_R6 = ((2U << 3) + 3),
+ DSLLV = ((2U << 3) + 4),
+ DLSA = ((2U << 3) + 5),
+ DSRLV = ((2U << 3) + 6),
+ DSRAV = ((2U << 3) + 7),
+
+ MULT = ((3U << 3) + 0),
+ MULTU = ((3U << 3) + 1),
+ DIV = ((3U << 3) + 2),
+ DIVU = ((3U << 3) + 3),
+ DMULT = ((3U << 3) + 4),
+ DMULTU = ((3U << 3) + 5),
+ DDIV = ((3U << 3) + 6),
+ DDIVU = ((3U << 3) + 7),
+
+ ADD = ((4U << 3) + 0),
+ ADDU = ((4U << 3) + 1),
+ SUB = ((4U << 3) + 2),
+ SUBU = ((4U << 3) + 3),
+ AND = ((4U << 3) + 4),
+ OR = ((4U << 3) + 5),
+ XOR = ((4U << 3) + 6),
+ NOR = ((4U << 3) + 7),
+
+ SLT = ((5U << 3) + 2),
+ SLTU = ((5U << 3) + 3),
+ DADD = ((5U << 3) + 4),
+ DADDU = ((5U << 3) + 5),
+ DSUB = ((5U << 3) + 6),
+ DSUBU = ((5U << 3) + 7),
+
+ TGE = ((6U << 3) + 0),
+ TGEU = ((6U << 3) + 1),
+ TLT = ((6U << 3) + 2),
+ TLTU = ((6U << 3) + 3),
+ TEQ = ((6U << 3) + 4),
+ SELEQZ_S = ((6U << 3) + 5),
+ TNE = ((6U << 3) + 6),
+ SELNEZ_S = ((6U << 3) + 7),
+
+ DSLL = ((7U << 3) + 0),
+ DSRL = ((7U << 3) + 2),
+ DSRA = ((7U << 3) + 3),
+ DSLL32 = ((7U << 3) + 4),
+ DSRL32 = ((7U << 3) + 6),
+ DSRA32 = ((7U << 3) + 7),
+
+ // Multiply integers in r6.
+ MUL_MUH = ((3U << 3) + 0), // MUL, MUH.
+ MUL_MUH_U = ((3U << 3) + 1), // MUL_U, MUH_U.
+ D_MUL_MUH = ((7U << 2) + 0), // DMUL, DMUH.
+ D_MUL_MUH_U = ((7U << 2) + 1), // DMUL_U, DMUH_U.
+ RINT = ((3U << 3) + 2),
+
+ MUL_OP = ((0U << 3) + 2),
+ MUH_OP = ((0U << 3) + 3),
+ DIV_OP = ((0U << 3) + 2),
+ MOD_OP = ((0U << 3) + 3),
+
+ DIV_MOD = ((3U << 3) + 2),
+ DIV_MOD_U = ((3U << 3) + 3),
+ D_DIV_MOD = ((3U << 3) + 6),
+ D_DIV_MOD_U = ((3U << 3) + 7),
+
+ // drotr in special4?
+
+ // SPECIAL2 Encoding of Function Field.
+ MUL = ((0U << 3) + 2),
+ CLZ = ((4U << 3) + 0),
+ CLO = ((4U << 3) + 1),
+ DCLZ = ((4U << 3) + 4),
+ DCLO = ((4U << 3) + 5),
+
+ // SPECIAL3 Encoding of Function Field.
+ EXT = ((0U << 3) + 0),
+ DEXTM = ((0U << 3) + 1),
+ DEXTU = ((0U << 3) + 2),
+ DEXT = ((0U << 3) + 3),
+ INS = ((0U << 3) + 4),
+ DINSM = ((0U << 3) + 5),
+ DINSU = ((0U << 3) + 6),
+ DINS = ((0U << 3) + 7),
+
+ BSHFL = ((4U << 3) + 0),
+ DBSHFL = ((4U << 3) + 4),
+ SC_R6 = ((4U << 3) + 6),
+ SCD_R6 = ((4U << 3) + 7),
+ LL_R6 = ((6U << 3) + 6),
+ LLD_R6 = ((6U << 3) + 7),
+
+ // SPECIAL3 Encoding of sa Field.
+ BITSWAP = ((0U << 3) + 0),
+ ALIGN = ((0U << 3) + 2),
+ WSBH = ((0U << 3) + 2),
+ SEB = ((2U << 3) + 0),
+ SEH = ((3U << 3) + 0),
+
+ DBITSWAP = ((0U << 3) + 0),
+ DALIGN = ((0U << 3) + 1),
+ DBITSWAP_SA = ((0U << 3) + 0) << kSaShift,
+ DSBH = ((0U << 3) + 2),
+ DSHD = ((0U << 3) + 5),
+
+ // REGIMM encoding of rt Field.
+ BLTZ = ((0U << 3) + 0) << 16,
+ BGEZ = ((0U << 3) + 1) << 16,
+ BLTZAL = ((2U << 3) + 0) << 16,
+ BGEZAL = ((2U << 3) + 1) << 16,
+ BGEZALL = ((2U << 3) + 3) << 16,
+ DAHI = ((0U << 3) + 6) << 16,
+ DATI = ((3U << 3) + 6) << 16,
+
+ // COP1 Encoding of rs Field.
+ MFC1 = ((0U << 3) + 0) << 21,
+ DMFC1 = ((0U << 3) + 1) << 21,
+ CFC1 = ((0U << 3) + 2) << 21,
+ MFHC1 = ((0U << 3) + 3) << 21,
+ MTC1 = ((0U << 3) + 4) << 21,
+ DMTC1 = ((0U << 3) + 5) << 21,
+ CTC1 = ((0U << 3) + 6) << 21,
+ MTHC1 = ((0U << 3) + 7) << 21,
+ BC1 = ((1U << 3) + 0) << 21,
+ S = ((2U << 3) + 0) << 21,
+ D = ((2U << 3) + 1) << 21,
+ W = ((2U << 3) + 4) << 21,
+ L = ((2U << 3) + 5) << 21,
+ PS = ((2U << 3) + 6) << 21,
+ // COP1 Encoding of Function Field When rs=S.
+
+ ADD_S = ((0U << 3) + 0),
+ SUB_S = ((0U << 3) + 1),
+ MUL_S = ((0U << 3) + 2),
+ DIV_S = ((0U << 3) + 3),
+ ABS_S = ((0U << 3) + 5),
+ SQRT_S = ((0U << 3) + 4),
+ MOV_S = ((0U << 3) + 6),
+ NEG_S = ((0U << 3) + 7),
+ ROUND_L_S = ((1U << 3) + 0),
+ TRUNC_L_S = ((1U << 3) + 1),
+ CEIL_L_S = ((1U << 3) + 2),
+ FLOOR_L_S = ((1U << 3) + 3),
+ ROUND_W_S = ((1U << 3) + 4),
+ TRUNC_W_S = ((1U << 3) + 5),
+ CEIL_W_S = ((1U << 3) + 6),
+ FLOOR_W_S = ((1U << 3) + 7),
+ RECIP_S = ((2U << 3) + 5),
+ RSQRT_S = ((2U << 3) + 6),
+ MADDF_S = ((3U << 3) + 0),
+ MSUBF_S = ((3U << 3) + 1),
+ CLASS_S = ((3U << 3) + 3),
+ CVT_D_S = ((4U << 3) + 1),
+ CVT_W_S = ((4U << 3) + 4),
+ CVT_L_S = ((4U << 3) + 5),
+ CVT_PS_S = ((4U << 3) + 6),
+ // COP1 Encoding of Function Field When rs=D.
+ ADD_D = ((0U << 3) + 0),
+ SUB_D = ((0U << 3) + 1),
+ MUL_D = ((0U << 3) + 2),
+ DIV_D = ((0U << 3) + 3),
+ SQRT_D = ((0U << 3) + 4),
+ ABS_D = ((0U << 3) + 5),
+ MOV_D = ((0U << 3) + 6),
+ NEG_D = ((0U << 3) + 7),
+ ROUND_L_D = ((1U << 3) + 0),
+ TRUNC_L_D = ((1U << 3) + 1),
+ CEIL_L_D = ((1U << 3) + 2),
+ FLOOR_L_D = ((1U << 3) + 3),
+ ROUND_W_D = ((1U << 3) + 4),
+ TRUNC_W_D = ((1U << 3) + 5),
+ CEIL_W_D = ((1U << 3) + 6),
+ FLOOR_W_D = ((1U << 3) + 7),
+ RECIP_D = ((2U << 3) + 5),
+ RSQRT_D = ((2U << 3) + 6),
+ MADDF_D = ((3U << 3) + 0),
+ MSUBF_D = ((3U << 3) + 1),
+ CLASS_D = ((3U << 3) + 3),
+ MIN = ((3U << 3) + 4),
+ MINA = ((3U << 3) + 5),
+ MAX = ((3U << 3) + 6),
+ MAXA = ((3U << 3) + 7),
+ CVT_S_D = ((4U << 3) + 0),
+ CVT_W_D = ((4U << 3) + 4),
+ CVT_L_D = ((4U << 3) + 5),
+ C_F_D = ((6U << 3) + 0),
+ C_UN_D = ((6U << 3) + 1),
+ C_EQ_D = ((6U << 3) + 2),
+ C_UEQ_D = ((6U << 3) + 3),
+ C_OLT_D = ((6U << 3) + 4),
+ C_ULT_D = ((6U << 3) + 5),
+ C_OLE_D = ((6U << 3) + 6),
+ C_ULE_D = ((6U << 3) + 7),
+
+ // COP1 Encoding of Function Field When rs=W or L.
+ CVT_S_W = ((4U << 3) + 0),
+ CVT_D_W = ((4U << 3) + 1),
+ CVT_S_L = ((4U << 3) + 0),
+ CVT_D_L = ((4U << 3) + 1),
+ BC1EQZ = ((2U << 2) + 1) << 21,
+ BC1NEZ = ((3U << 2) + 1) << 21,
+ // COP1 CMP positive predicates Bit 5..4 = 00.
+ CMP_AF = ((0U << 3) + 0),
+ CMP_UN = ((0U << 3) + 1),
+ CMP_EQ = ((0U << 3) + 2),
+ CMP_UEQ = ((0U << 3) + 3),
+ CMP_LT = ((0U << 3) + 4),
+ CMP_ULT = ((0U << 3) + 5),
+ CMP_LE = ((0U << 3) + 6),
+ CMP_ULE = ((0U << 3) + 7),
+ CMP_SAF = ((1U << 3) + 0),
+ CMP_SUN = ((1U << 3) + 1),
+ CMP_SEQ = ((1U << 3) + 2),
+ CMP_SUEQ = ((1U << 3) + 3),
+ CMP_SSLT = ((1U << 3) + 4),
+ CMP_SSULT = ((1U << 3) + 5),
+ CMP_SLE = ((1U << 3) + 6),
+ CMP_SULE = ((1U << 3) + 7),
+ // COP1 CMP negative predicates Bit 5..4 = 01.
+ CMP_AT = ((2U << 3) + 0), // Reserved, not implemented.
+ CMP_OR = ((2U << 3) + 1),
+ CMP_UNE = ((2U << 3) + 2),
+ CMP_NE = ((2U << 3) + 3),
+ CMP_UGE = ((2U << 3) + 4), // Reserved, not implemented.
+ CMP_OGE = ((2U << 3) + 5), // Reserved, not implemented.
+ CMP_UGT = ((2U << 3) + 6), // Reserved, not implemented.
+ CMP_OGT = ((2U << 3) + 7), // Reserved, not implemented.
+ CMP_SAT = ((3U << 3) + 0), // Reserved, not implemented.
+ CMP_SOR = ((3U << 3) + 1),
+ CMP_SUNE = ((3U << 3) + 2),
+ CMP_SNE = ((3U << 3) + 3),
+ CMP_SUGE = ((3U << 3) + 4), // Reserved, not implemented.
+ CMP_SOGE = ((3U << 3) + 5), // Reserved, not implemented.
+ CMP_SUGT = ((3U << 3) + 6), // Reserved, not implemented.
+ CMP_SOGT = ((3U << 3) + 7), // Reserved, not implemented.
+
+ SEL = ((2U << 3) + 0),
+ MOVF = ((2U << 3) + 1), // Function field for MOVT.fmt and MOVF.fmt
+ MOVZ_C = ((2U << 3) + 2), // COP1 on FPR registers.
+ MOVN_C = ((2U << 3) + 3), // COP1 on FPR registers.
+ SELEQZ_C = ((2U << 3) + 4), // COP1 on FPR registers.
+ SELNEZ_C = ((2U << 3) + 7), // COP1 on FPR registers.
+
+ // COP1 Encoding of Function Field When rs=PS.
+
+ // COP1X Encoding of Function Field.
+ MADD_S = ((4U << 3) + 0),
+ MADD_D = ((4U << 3) + 1),
+ MSUB_S = ((5U << 3) + 0),
+ MSUB_D = ((5U << 3) + 1),
+
+ // PCREL Encoding of rt Field.
+ ADDIUPC = ((0U << 2) + 0),
+ LWPC = ((0U << 2) + 1),
+ LWUPC = ((0U << 2) + 2),
+ LDPC = ((0U << 3) + 6),
+ // reserved ((1U << 3) + 6),
+ AUIPC = ((3U << 3) + 6),
+ ALUIPC = ((3U << 3) + 7),
+
+ // POP66 Encoding of rs Field.
+ JIC = ((0U << 5) + 0),
+
+ // POP76 Encoding of rs Field.
+ JIALC = ((0U << 5) + 0),
+
+ // COP1 Encoding of rs Field for MSA Branch Instructions
+ BZ_V = (((1U << 3) + 3) << kRsShift),
+ BNZ_V = (((1U << 3) + 7) << kRsShift),
+ BZ_B = (((3U << 3) + 0) << kRsShift),
+ BZ_H = (((3U << 3) + 1) << kRsShift),
+ BZ_W = (((3U << 3) + 2) << kRsShift),
+ BZ_D = (((3U << 3) + 3) << kRsShift),
+ BNZ_B = (((3U << 3) + 4) << kRsShift),
+ BNZ_H = (((3U << 3) + 5) << kRsShift),
+ BNZ_W = (((3U << 3) + 6) << kRsShift),
+ BNZ_D = (((3U << 3) + 7) << kRsShift),
+
+ // MSA: Operation Field for MI10 Instruction Formats
+ MSA_LD = (8U << 2),
+ MSA_ST = (9U << 2),
+ LD_B = ((8U << 2) + 0),
+ LD_H = ((8U << 2) + 1),
+ LD_W = ((8U << 2) + 2),
+ LD_D = ((8U << 2) + 3),
+ ST_B = ((9U << 2) + 0),
+ ST_H = ((9U << 2) + 1),
+ ST_W = ((9U << 2) + 2),
+ ST_D = ((9U << 2) + 3),
+
+ // MSA: Operation Field for I5 Instruction Format
+ ADDVI = ((0U << 23) + 6),
+ SUBVI = ((1U << 23) + 6),
+ MAXI_S = ((2U << 23) + 6),
+ MAXI_U = ((3U << 23) + 6),
+ MINI_S = ((4U << 23) + 6),
+ MINI_U = ((5U << 23) + 6),
+ CEQI = ((0U << 23) + 7),
+ CLTI_S = ((2U << 23) + 7),
+ CLTI_U = ((3U << 23) + 7),
+ CLEI_S = ((4U << 23) + 7),
+ CLEI_U = ((5U << 23) + 7),
+ LDI = ((6U << 23) + 7), // I10 instruction format
+ I5_DF_b = (0U << 21),
+ I5_DF_h = (1U << 21),
+ I5_DF_w = (2U << 21),
+ I5_DF_d = (3U << 21),
+
+ // MSA: Operation Field for I8 Instruction Format
+ ANDI_B = ((0U << 24) + 0),
+ ORI_B = ((1U << 24) + 0),
+ NORI_B = ((2U << 24) + 0),
+ XORI_B = ((3U << 24) + 0),
+ BMNZI_B = ((0U << 24) + 1),
+ BMZI_B = ((1U << 24) + 1),
+ BSELI_B = ((2U << 24) + 1),
+ SHF_B = ((0U << 24) + 2),
+ SHF_H = ((1U << 24) + 2),
+ SHF_W = ((2U << 24) + 2),
+
+ MSA_VEC_2R_2RF_MINOR = ((3U << 3) + 6),
+
+ // MSA: Operation Field for VEC Instruction Formats
+ AND_V = (((0U << 2) + 0) << 21),
+ OR_V = (((0U << 2) + 1) << 21),
+ NOR_V = (((0U << 2) + 2) << 21),
+ XOR_V = (((0U << 2) + 3) << 21),
+ BMNZ_V = (((1U << 2) + 0) << 21),
+ BMZ_V = (((1U << 2) + 1) << 21),
+ BSEL_V = (((1U << 2) + 2) << 21),
+
+ // MSA: Operation Field for 2R Instruction Formats
+ MSA_2R_FORMAT = (((6U << 2) + 0) << 21),
+ FILL = (0U << 18),
+ PCNT = (1U << 18),
+ NLOC = (2U << 18),
+ NLZC = (3U << 18),
+ MSA_2R_DF_b = (0U << 16),
+ MSA_2R_DF_h = (1U << 16),
+ MSA_2R_DF_w = (2U << 16),
+ MSA_2R_DF_d = (3U << 16),
+
+ // MSA: Operation Field for 2RF Instruction Formats
+ MSA_2RF_FORMAT = (((6U << 2) + 1) << 21),
+ FCLASS = (0U << 17),
+ FTRUNC_S = (1U << 17),
+ FTRUNC_U = (2U << 17),
+ FSQRT = (3U << 17),
+ FRSQRT = (4U << 17),
+ FRCP = (5U << 17),
+ FRINT = (6U << 17),
+ FLOG2 = (7U << 17),
+ FEXUPL = (8U << 17),
+ FEXUPR = (9U << 17),
+ FFQL = (10U << 17),
+ FFQR = (11U << 17),
+ FTINT_S = (12U << 17),
+ FTINT_U = (13U << 17),
+ FFINT_S = (14U << 17),
+ FFINT_U = (15U << 17),
+ MSA_2RF_DF_w = (0U << 16),
+ MSA_2RF_DF_d = (1U << 16),
+
+ // MSA: Operation Field for 3R Instruction Format
+ SLL_MSA = ((0U << 23) + 13),
+ SRA_MSA = ((1U << 23) + 13),
+ SRL_MSA = ((2U << 23) + 13),
+ BCLR = ((3U << 23) + 13),
+ BSET = ((4U << 23) + 13),
+ BNEG = ((5U << 23) + 13),
+ BINSL = ((6U << 23) + 13),
+ BINSR = ((7U << 23) + 13),
+ ADDV = ((0U << 23) + 14),
+ SUBV = ((1U << 23) + 14),
+ MAX_S = ((2U << 23) + 14),
+ MAX_U = ((3U << 23) + 14),
+ MIN_S = ((4U << 23) + 14),
+ MIN_U = ((5U << 23) + 14),
+ MAX_A = ((6U << 23) + 14),
+ MIN_A = ((7U << 23) + 14),
+ CEQ = ((0U << 23) + 15),
+ CLT_S = ((2U << 23) + 15),
+ CLT_U = ((3U << 23) + 15),
+ CLE_S = ((4U << 23) + 15),
+ CLE_U = ((5U << 23) + 15),
+ ADD_A = ((0U << 23) + 16),
+ ADDS_A = ((1U << 23) + 16),
+ ADDS_S = ((2U << 23) + 16),
+ ADDS_U = ((3U << 23) + 16),
+ AVE_S = ((4U << 23) + 16),
+ AVE_U = ((5U << 23) + 16),
+ AVER_S = ((6U << 23) + 16),
+ AVER_U = ((7U << 23) + 16),
+ SUBS_S = ((0U << 23) + 17),
+ SUBS_U = ((1U << 23) + 17),
+ SUBSUS_U = ((2U << 23) + 17),
+ SUBSUU_S = ((3U << 23) + 17),
+ ASUB_S = ((4U << 23) + 17),
+ ASUB_U = ((5U << 23) + 17),
+ MULV = ((0U << 23) + 18),
+ MADDV = ((1U << 23) + 18),
+ MSUBV = ((2U << 23) + 18),
+ DIV_S_MSA = ((4U << 23) + 18),
+ DIV_U = ((5U << 23) + 18),
+ MOD_S = ((6U << 23) + 18),
+ MOD_U = ((7U << 23) + 18),
+ DOTP_S = ((0U << 23) + 19),
+ DOTP_U = ((1U << 23) + 19),
+ DPADD_S = ((2U << 23) + 19),
+ DPADD_U = ((3U << 23) + 19),
+ DPSUB_S = ((4U << 23) + 19),
+ DPSUB_U = ((5U << 23) + 19),
+ SLD = ((0U << 23) + 20),
+ SPLAT = ((1U << 23) + 20),
+ PCKEV = ((2U << 23) + 20),
+ PCKOD = ((3U << 23) + 20),
+ ILVL = ((4U << 23) + 20),
+ ILVR = ((5U << 23) + 20),
+ ILVEV = ((6U << 23) + 20),
+ ILVOD = ((7U << 23) + 20),
+ VSHF = ((0U << 23) + 21),
+ SRAR = ((1U << 23) + 21),
+ SRLR = ((2U << 23) + 21),
+ HADD_S = ((4U << 23) + 21),
+ HADD_U = ((5U << 23) + 21),
+ HSUB_S = ((6U << 23) + 21),
+ HSUB_U = ((7U << 23) + 21),
+ MSA_3R_DF_b = (0U << 21),
+ MSA_3R_DF_h = (1U << 21),
+ MSA_3R_DF_w = (2U << 21),
+ MSA_3R_DF_d = (3U << 21),
+
+ // MSA: Operation Field for 3RF Instruction Format
+ FCAF = ((0U << 22) + 26),
+ FCUN = ((1U << 22) + 26),
+ FCEQ = ((2U << 22) + 26),
+ FCUEQ = ((3U << 22) + 26),
+ FCLT = ((4U << 22) + 26),
+ FCULT = ((5U << 22) + 26),
+ FCLE = ((6U << 22) + 26),
+ FCULE = ((7U << 22) + 26),
+ FSAF = ((8U << 22) + 26),
+ FSUN = ((9U << 22) + 26),
+ FSEQ = ((10U << 22) + 26),
+ FSUEQ = ((11U << 22) + 26),
+ FSLT = ((12U << 22) + 26),
+ FSULT = ((13U << 22) + 26),
+ FSLE = ((14U << 22) + 26),
+ FSULE = ((15U << 22) + 26),
+ FADD = ((0U << 22) + 27),
+ FSUB = ((1U << 22) + 27),
+ FMUL = ((2U << 22) + 27),
+ FDIV = ((3U << 22) + 27),
+ FMADD = ((4U << 22) + 27),
+ FMSUB = ((5U << 22) + 27),
+ FEXP2 = ((7U << 22) + 27),
+ FEXDO = ((8U << 22) + 27),
+ FTQ = ((10U << 22) + 27),
+ FMIN = ((12U << 22) + 27),
+ FMIN_A = ((13U << 22) + 27),
+ FMAX = ((14U << 22) + 27),
+ FMAX_A = ((15U << 22) + 27),
+ FCOR = ((1U << 22) + 28),
+ FCUNE = ((2U << 22) + 28),
+ FCNE = ((3U << 22) + 28),
+ MUL_Q = ((4U << 22) + 28),
+ MADD_Q = ((5U << 22) + 28),
+ MSUB_Q = ((6U << 22) + 28),
+ FSOR = ((9U << 22) + 28),
+ FSUNE = ((10U << 22) + 28),
+ FSNE = ((11U << 22) + 28),
+ MULR_Q = ((12U << 22) + 28),
+ MADDR_Q = ((13U << 22) + 28),
+ MSUBR_Q = ((14U << 22) + 28),
+
+ // MSA: Operation Field for ELM Instruction Format
+ MSA_ELM_MINOR = ((3U << 3) + 1),
+ SLDI = (0U << 22),
+ CTCMSA = ((0U << 22) | (62U << 16)),
+ SPLATI = (1U << 22),
+ CFCMSA = ((1U << 22) | (62U << 16)),
+ COPY_S = (2U << 22),
+ MOVE_V = ((2U << 22) | (62U << 16)),
+ COPY_U = (3U << 22),
+ INSERT = (4U << 22),
+ INSVE = (5U << 22),
+ ELM_DF_B = ((0U << 4) << 16),
+ ELM_DF_H = ((4U << 3) << 16),
+ ELM_DF_W = ((12U << 2) << 16),
+ ELM_DF_D = ((28U << 1) << 16),
+
+ // MSA: Operation Field for BIT Instruction Format
+ SLLI = ((0U << 23) + 9),
+ SRAI = ((1U << 23) + 9),
+ SRLI = ((2U << 23) + 9),
+ BCLRI = ((3U << 23) + 9),
+ BSETI = ((4U << 23) + 9),
+ BNEGI = ((5U << 23) + 9),
+ BINSLI = ((6U << 23) + 9),
+ BINSRI = ((7U << 23) + 9),
+ SAT_S = ((0U << 23) + 10),
+ SAT_U = ((1U << 23) + 10),
+ SRARI = ((2U << 23) + 10),
+ SRLRI = ((3U << 23) + 10),
+ BIT_DF_b = ((14U << 3) << 16),
+ BIT_DF_h = ((6U << 4) << 16),
+ BIT_DF_w = ((2U << 5) << 16),
+ BIT_DF_d = ((0U << 6) << 16),
+
+ nullptrSF = 0U
+};
+
+enum MSAMinorOpcode : uint32_t {
+ kMsaMinorUndefined = 0,
+ kMsaMinorI8,
+ kMsaMinorI5,
+ kMsaMinorI10,
+ kMsaMinorBIT,
+ kMsaMinor3R,
+ kMsaMinor3RF,
+ kMsaMinorELM,
+ kMsaMinorVEC,
+ kMsaMinor2R,
+ kMsaMinor2RF,
+ kMsaMinorMI10
+};
+
+// ----- Emulated conditions.
+// On MIPS we use this enum to abstract from conditional branch instructions.
+// The 'U' prefix is used to specify unsigned comparisons.
+// Opposite conditions must be paired as odd/even numbers
+// because 'NegateCondition' function flips LSB to negate condition.
+enum Condition {
+ // Any value < 0 is considered no_condition.
+ kNoCondition = -1,
+ overflow = 0,
+ no_overflow = 1,
+ Uless = 2,
+ Ugreater_equal = 3,
+ Uless_equal = 4,
+ Ugreater = 5,
+ equal = 6,
+ not_equal = 7, // Unordered or Not Equal.
+ negative = 8,
+ positive = 9,
+ parity_even = 10,
+ parity_odd = 11,
+ less = 12,
+ greater_equal = 13,
+ less_equal = 14,
+ greater = 15,
+ ueq = 16, // Unordered or Equal.
+ ogl = 17, // Ordered and Not Equal.
+ cc_always = 18,
+
+ // Aliases.
+ carry = Uless,
+ not_carry = Ugreater_equal,
+ zero = equal,
+ eq = equal,
+ not_zero = not_equal,
+ ne = not_equal,
+ nz = not_equal,
+ sign = negative,
+ not_sign = positive,
+ mi = negative,
+ pl = positive,
+ hi = Ugreater,
+ ls = Uless_equal,
+ ge = greater_equal,
+ lt = less,
+ gt = greater,
+ le = less_equal,
+ hs = Ugreater_equal,
+ lo = Uless,
+ al = cc_always,
+ ult = Uless,
+ uge = Ugreater_equal,
+ ule = Uless_equal,
+ ugt = Ugreater,
+ cc_default = kNoCondition
+};
+
+// Returns the equivalent of !cc.
+// Negation of the default kNoCondition (-1) results in a non-default
+// no_condition value (-2). As long as tests for no_condition check
+// for condition < 0, this will work as expected.
+inline Condition NegateCondition(Condition cc) {
+ DCHECK(cc != cc_always);
+ return static_cast<Condition>(cc ^ 1);
+}
+
+inline Condition NegateFpuCondition(Condition cc) {
+ DCHECK(cc != cc_always);
+ switch (cc) {
+ case ult:
+ return ge;
+ case ugt:
+ return le;
+ case uge:
+ return lt;
+ case ule:
+ return gt;
+ case lt:
+ return uge;
+ case gt:
+ return ule;
+ case ge:
+ return ult;
+ case le:
+ return ugt;
+ case eq:
+ return ne;
+ case ne:
+ return eq;
+ case ueq:
+ return ogl;
+ case ogl:
+ return ueq;
+ default:
+ return cc;
+ }
+}
+
+enum MSABranchCondition {
+ all_not_zero = 0, // Branch If All Elements Are Not Zero
+ one_elem_not_zero, // Branch If At Least One Element of Any Format Is Not
+ // Zero
+ one_elem_zero, // Branch If At Least One Element Is Zero
+ all_zero // Branch If All Elements of Any Format Are Zero
+};
+
+inline MSABranchCondition NegateMSABranchCondition(MSABranchCondition cond) {
+ switch (cond) {
+ case all_not_zero:
+ return one_elem_zero;
+ case one_elem_not_zero:
+ return all_zero;
+ case one_elem_zero:
+ return all_not_zero;
+ case all_zero:
+ return one_elem_not_zero;
+ default:
+ return cond;
+ }
+}
+
+enum MSABranchDF {
+ MSA_BRANCH_B = 0,
+ MSA_BRANCH_H,
+ MSA_BRANCH_W,
+ MSA_BRANCH_D,
+ MSA_BRANCH_V
+};
+
+// ----- Coprocessor conditions.
+enum FPUCondition {
+ kNoFPUCondition = -1,
+
+ F = 0x00, // False.
+ UN = 0x01, // Unordered.
+ EQ = 0x02, // Equal.
+ UEQ = 0x03, // Unordered or Equal.
+ OLT = 0x04, // Ordered or Less Than, on Mips release < 6.
+ LT = 0x04, // Ordered or Less Than, on Mips release >= 6.
+ ULT = 0x05, // Unordered or Less Than.
+ OLE = 0x06, // Ordered or Less Than or Equal, on Mips release < 6.
+ LE = 0x06, // Ordered or Less Than or Equal, on Mips release >= 6.
+ ULE = 0x07, // Unordered or Less Than or Equal.
+
+ // Following constants are available on Mips release >= 6 only.
+ ORD = 0x11, // Ordered, on Mips release >= 6.
+ UNE = 0x12, // Not equal, on Mips release >= 6.
+ NE = 0x13, // Ordered Greater Than or Less Than. on Mips >= 6 only.
+};
+
+// FPU rounding modes.
+enum FPURoundingMode {
+ RN = 0 << 0, // Round to Nearest.
+ RZ = 1 << 0, // Round towards zero.
+ RP = 2 << 0, // Round towards Plus Infinity.
+ RM = 3 << 0, // Round towards Minus Infinity.
+
+ // Aliases.
+ kRoundToNearest = RN,
+ kRoundToZero = RZ,
+ kRoundToPlusInf = RP,
+ kRoundToMinusInf = RM,
+
+ mode_round = RN,
+ mode_ceil = RP,
+ mode_floor = RM,
+ mode_trunc = RZ
+};
+
+const uint32_t kFPURoundingModeMask = 3 << 0;
+
+enum CheckForInexactConversion {
+ kCheckForInexactConversion,
+ kDontCheckForInexactConversion
+};
+
+enum class MaxMinKind : int { kMin = 0, kMax = 1 };
+
+// -----------------------------------------------------------------------------
+// Hints.
+
+// Branch hints are not used on the MIPS. They are defined so that they can
+// appear in shared function signatures, but will be ignored in MIPS
+// implementations.
+enum Hint { no_hint = 0 };
+
+inline Hint NegateHint(Hint hint) { return no_hint; }
+
+// -----------------------------------------------------------------------------
+// Specific instructions, constants, and masks.
+// These constants are declared in assembler-mips.cc, as they use named
+// registers and other constants.
+
+// addiu(sp, sp, 4) aka Pop() operation or part of Pop(r)
+// operations as post-increment of sp.
+extern const Instr kPopInstruction;
+// addiu(sp, sp, -4) part of Push(r) operation as pre-decrement of sp.
+extern const Instr kPushInstruction;
+// Sw(r, MemOperand(sp, 0))
+extern const Instr kPushRegPattern;
+// Lw(r, MemOperand(sp, 0))
+extern const Instr kPopRegPattern;
+extern const Instr kLwRegFpOffsetPattern;
+extern const Instr kSwRegFpOffsetPattern;
+extern const Instr kLwRegFpNegOffsetPattern;
+extern const Instr kSwRegFpNegOffsetPattern;
+// A mask for the Rt register for push, pop, lw, sw instructions.
+extern const Instr kRtMask;
+extern const Instr kLwSwInstrTypeMask;
+extern const Instr kLwSwInstrArgumentMask;
+extern const Instr kLwSwOffsetMask;
+
+// Break 0xfffff, reserved for redirected real time call.
+const Instr rtCallRedirInstr = SPECIAL | BREAK | call_rt_redirected << 6;
+// A nop instruction. (Encoding of sll 0 0 0).
+const Instr nopInstr = 0;
+
+static constexpr uint64_t OpcodeToBitNumber(Opcode opcode) {
+ return 1ULL << (static_cast<uint32_t>(opcode) >> kOpcodeShift);
+}
+
+constexpr uint8_t kInstrSize = 4;
+constexpr uint8_t kInstrSizeLog2 = 2;
+
+class InstructionBase {
+ public:
+ enum {
+ // On MIPS PC cannot actually be directly accessed. We behave as if PC was
+ // always the value of the current instruction being executed.
+ kPCReadOffset = 0
+ };
+
+ // Instruction type.
+ enum Type { kRegisterType, kImmediateType, kJumpType, kUnsupported = -1 };
+
+ // Get the raw instruction bits.
+ inline Instr InstructionBits() const {
+ return *reinterpret_cast<const Instr*>(this);
+ }
+
+ // Set the raw instruction bits to value.
+ inline void SetInstructionBits(Instr value) {
+ *reinterpret_cast<Instr*>(this) = value;
+ }
+
+ // Read one particular bit out of the instruction bits.
+ inline int Bit(int nr) const { return (InstructionBits() >> nr) & 1; }
+
+ // Read a bit field out of the instruction bits.
+ inline int Bits(int hi, int lo) const {
+ return (InstructionBits() >> lo) & ((2U << (hi - lo)) - 1);
+ }
+
+ static constexpr uint64_t kOpcodeImmediateTypeMask =
+ OpcodeToBitNumber(REGIMM) | OpcodeToBitNumber(BEQ) |
+ OpcodeToBitNumber(BNE) | OpcodeToBitNumber(BLEZ) |
+ OpcodeToBitNumber(BGTZ) | OpcodeToBitNumber(ADDI) |
+ OpcodeToBitNumber(DADDI) | OpcodeToBitNumber(ADDIU) |
+ OpcodeToBitNumber(DADDIU) | OpcodeToBitNumber(SLTI) |
+ OpcodeToBitNumber(SLTIU) | OpcodeToBitNumber(ANDI) |
+ OpcodeToBitNumber(ORI) | OpcodeToBitNumber(XORI) |
+ OpcodeToBitNumber(LUI) | OpcodeToBitNumber(BEQL) |
+ OpcodeToBitNumber(BNEL) | OpcodeToBitNumber(BLEZL) |
+ OpcodeToBitNumber(BGTZL) | OpcodeToBitNumber(POP66) |
+ OpcodeToBitNumber(POP76) | OpcodeToBitNumber(LB) | OpcodeToBitNumber(LH) |
+ OpcodeToBitNumber(LWL) | OpcodeToBitNumber(LW) | OpcodeToBitNumber(LWU) |
+ OpcodeToBitNumber(LD) | OpcodeToBitNumber(LBU) | OpcodeToBitNumber(LHU) |
+ OpcodeToBitNumber(LDL) | OpcodeToBitNumber(LDR) | OpcodeToBitNumber(LWR) |
+ OpcodeToBitNumber(SDL) | OpcodeToBitNumber(SB) | OpcodeToBitNumber(SH) |
+ OpcodeToBitNumber(SWL) | OpcodeToBitNumber(SW) | OpcodeToBitNumber(SD) |
+ OpcodeToBitNumber(SWR) | OpcodeToBitNumber(SDR) |
+ OpcodeToBitNumber(LWC1) | OpcodeToBitNumber(LDC1) |
+ OpcodeToBitNumber(SWC1) | OpcodeToBitNumber(SDC1) |
+ OpcodeToBitNumber(PCREL) | OpcodeToBitNumber(DAUI) |
+ OpcodeToBitNumber(BC) | OpcodeToBitNumber(BALC);
+
+#define FunctionFieldToBitNumber(function) (1ULL << function)
+
+ // On r6, DCLZ_R6 aliases to existing MFLO.
+ static const uint64_t kFunctionFieldRegisterTypeMask =
+ FunctionFieldToBitNumber(JR) | FunctionFieldToBitNumber(JALR) |
+ FunctionFieldToBitNumber(BREAK) | FunctionFieldToBitNumber(SLL) |
+ FunctionFieldToBitNumber(DSLL) | FunctionFieldToBitNumber(DSLL32) |
+ FunctionFieldToBitNumber(SRL) | FunctionFieldToBitNumber(DSRL) |
+ FunctionFieldToBitNumber(DSRL32) | FunctionFieldToBitNumber(SRA) |
+ FunctionFieldToBitNumber(DSRA) | FunctionFieldToBitNumber(DSRA32) |
+ FunctionFieldToBitNumber(SLLV) | FunctionFieldToBitNumber(DSLLV) |
+ FunctionFieldToBitNumber(SRLV) | FunctionFieldToBitNumber(DSRLV) |
+ FunctionFieldToBitNumber(SRAV) | FunctionFieldToBitNumber(DSRAV) |
+ FunctionFieldToBitNumber(LSA) | FunctionFieldToBitNumber(DLSA) |
+ FunctionFieldToBitNumber(MFHI) | FunctionFieldToBitNumber(MFLO) |
+ FunctionFieldToBitNumber(MULT) | FunctionFieldToBitNumber(DMULT) |
+ FunctionFieldToBitNumber(MULTU) | FunctionFieldToBitNumber(DMULTU) |
+ FunctionFieldToBitNumber(DIV) | FunctionFieldToBitNumber(DDIV) |
+ FunctionFieldToBitNumber(DIVU) | FunctionFieldToBitNumber(DDIVU) |
+ FunctionFieldToBitNumber(ADD) | FunctionFieldToBitNumber(DADD) |
+ FunctionFieldToBitNumber(ADDU) | FunctionFieldToBitNumber(DADDU) |
+ FunctionFieldToBitNumber(SUB) | FunctionFieldToBitNumber(DSUB) |
+ FunctionFieldToBitNumber(SUBU) | FunctionFieldToBitNumber(DSUBU) |
+ FunctionFieldToBitNumber(AND) | FunctionFieldToBitNumber(OR) |
+ FunctionFieldToBitNumber(XOR) | FunctionFieldToBitNumber(NOR) |
+ FunctionFieldToBitNumber(SLT) | FunctionFieldToBitNumber(SLTU) |
+ FunctionFieldToBitNumber(TGE) | FunctionFieldToBitNumber(TGEU) |
+ FunctionFieldToBitNumber(TLT) | FunctionFieldToBitNumber(TLTU) |
+ FunctionFieldToBitNumber(TEQ) | FunctionFieldToBitNumber(TNE) |
+ FunctionFieldToBitNumber(MOVZ) | FunctionFieldToBitNumber(MOVN) |
+ FunctionFieldToBitNumber(MOVCI) | FunctionFieldToBitNumber(SELEQZ_S) |
+ FunctionFieldToBitNumber(SELNEZ_S) | FunctionFieldToBitNumber(SYNC);
+
+ // Accessors for the different named fields used in the MIPS encoding.
+ inline Opcode OpcodeValue() const {
+ return static_cast<Opcode>(
+ Bits(kOpcodeShift + kOpcodeBits - 1, kOpcodeShift));
+ }
+
+ inline int FunctionFieldRaw() const {
+ return InstructionBits() & kFunctionFieldMask;
+ }
+
+ // Return the fields at their original place in the instruction encoding.
+ inline Opcode OpcodeFieldRaw() const {
+ return static_cast<Opcode>(InstructionBits() & kOpcodeMask);
+ }
+
+ // Safe to call within InstructionType().
+ inline int RsFieldRawNoAssert() const {
+ return InstructionBits() & kRsFieldMask;
+ }
+
+ inline int SaFieldRaw() const { return InstructionBits() & kSaFieldMask; }
+
+ // Get the encoding type of the instruction.
+ inline Type InstructionType() const;
+
+ inline MSAMinorOpcode MSAMinorOpcodeField() const {
+ int op = this->FunctionFieldRaw();
+ switch (op) {
+ case 0:
+ case 1:
+ case 2:
+ return kMsaMinorI8;
+ case 6:
+ return kMsaMinorI5;
+ case 7:
+ return (((this->InstructionBits() & kMsaI5I10Mask) == LDI)
+ ? kMsaMinorI10
+ : kMsaMinorI5);
+ case 9:
+ case 10:
+ return kMsaMinorBIT;
+ case 13:
+ case 14:
+ case 15:
+ case 16:
+ case 17:
+ case 18:
+ case 19:
+ case 20:
+ case 21:
+ return kMsaMinor3R;
+ case 25:
+ return kMsaMinorELM;
+ case 26:
+ case 27:
+ case 28:
+ return kMsaMinor3RF;
+ case 30:
+ switch (this->RsFieldRawNoAssert()) {
+ case MSA_2R_FORMAT:
+ return kMsaMinor2R;
+ case MSA_2RF_FORMAT:
+ return kMsaMinor2RF;
+ default:
+ return kMsaMinorVEC;
+ }
+ break;
+ case 32:
+ case 33:
+ case 34:
+ case 35:
+ case 36:
+ case 37:
+ case 38:
+ case 39:
+ return kMsaMinorMI10;
+ default:
+ return kMsaMinorUndefined;
+ }
+ }
+
+ protected:
+ InstructionBase() {}
+};
+
+template <class T>
+class InstructionGetters : public T {
+ public:
+ inline int RsValue() const {
+ DCHECK(this->InstructionType() == InstructionBase::kRegisterType ||
+ this->InstructionType() == InstructionBase::kImmediateType);
+ return this->Bits(kRsShift + kRsBits - 1, kRsShift);
+ }
+
+ inline int RtValue() const {
+ DCHECK(this->InstructionType() == InstructionBase::kRegisterType ||
+ this->InstructionType() == InstructionBase::kImmediateType);
+ return this->Bits(kRtShift + kRtBits - 1, kRtShift);
+ }
+
+ inline int RdValue() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kRegisterType);
+ return this->Bits(kRdShift + kRdBits - 1, kRdShift);
+ }
+
+ inline int BaseValue() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kImmediateType);
+ return this->Bits(kBaseShift + kBaseBits - 1, kBaseShift);
+ }
+
+ inline int SaValue() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kRegisterType);
+ return this->Bits(kSaShift + kSaBits - 1, kSaShift);
+ }
+
+ inline int LsaSaValue() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kRegisterType);
+ return this->Bits(kSaShift + kLsaSaBits - 1, kSaShift);
+ }
+
+ inline int FunctionValue() const {
+ DCHECK(this->InstructionType() == InstructionBase::kRegisterType ||
+ this->InstructionType() == InstructionBase::kImmediateType);
+ return this->Bits(kFunctionShift + kFunctionBits - 1, kFunctionShift);
+ }
+
+ inline int FdValue() const {
+ return this->Bits(kFdShift + kFdBits - 1, kFdShift);
+ }
+
+ inline int FsValue() const {
+ return this->Bits(kFsShift + kFsBits - 1, kFsShift);
+ }
+
+ inline int FtValue() const {
+ return this->Bits(kFtShift + kFtBits - 1, kFtShift);
+ }
+
+ inline int FrValue() const {
+ return this->Bits(kFrShift + kFrBits - 1, kFrShift);
+ }
+
+ inline int WdValue() const {
+ return this->Bits(kWdShift + kWdBits - 1, kWdShift);
+ }
+
+ inline int WsValue() const {
+ return this->Bits(kWsShift + kWsBits - 1, kWsShift);
+ }
+
+ inline int WtValue() const {
+ return this->Bits(kWtShift + kWtBits - 1, kWtShift);
+ }
+
+ inline int Bp2Value() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kRegisterType);
+ return this->Bits(kBp2Shift + kBp2Bits - 1, kBp2Shift);
+ }
+
+ inline int Bp3Value() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kRegisterType);
+ return this->Bits(kBp3Shift + kBp3Bits - 1, kBp3Shift);
+ }
+
+ // Float Compare condition code instruction bits.
+ inline int FCccValue() const {
+ return this->Bits(kFCccShift + kFCccBits - 1, kFCccShift);
+ }
+
+ // Float Branch condition code instruction bits.
+ inline int FBccValue() const {
+ return this->Bits(kFBccShift + kFBccBits - 1, kFBccShift);
+ }
+
+ // Float Branch true/false instruction bit.
+ inline int FBtrueValue() const {
+ return this->Bits(kFBtrueShift + kFBtrueBits - 1, kFBtrueShift);
+ }
+
+ // Return the fields at their original place in the instruction encoding.
+ inline Opcode OpcodeFieldRaw() const {
+ return static_cast<Opcode>(this->InstructionBits() & kOpcodeMask);
+ }
+
+ inline int RsFieldRaw() const {
+ DCHECK(this->InstructionType() == InstructionBase::kRegisterType ||
+ this->InstructionType() == InstructionBase::kImmediateType);
+ return this->InstructionBits() & kRsFieldMask;
+ }
+
+ // Same as above function, but safe to call within InstructionType().
+ inline int RsFieldRawNoAssert() const {
+ return this->InstructionBits() & kRsFieldMask;
+ }
+
+ inline int RtFieldRaw() const {
+ DCHECK(this->InstructionType() == InstructionBase::kRegisterType ||
+ this->InstructionType() == InstructionBase::kImmediateType);
+ return this->InstructionBits() & kRtFieldMask;
+ }
+
+ inline int RdFieldRaw() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kRegisterType);
+ return this->InstructionBits() & kRdFieldMask;
+ }
+
+ inline int SaFieldRaw() const {
+ return this->InstructionBits() & kSaFieldMask;
+ }
+
+ inline int FunctionFieldRaw() const {
+ return this->InstructionBits() & kFunctionFieldMask;
+ }
+
+ // Get the secondary field according to the opcode.
+ inline int SecondaryValue() const {
+ Opcode op = this->OpcodeFieldRaw();
+ switch (op) {
+ case SPECIAL:
+ case SPECIAL2:
+ return FunctionValue();
+ case COP1:
+ return RsValue();
+ case REGIMM:
+ return RtValue();
+ default:
+ return nullptrSF;
+ }
+ }
+
+ inline int32_t ImmValue(int bits) const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kImmediateType);
+ return this->Bits(bits - 1, 0);
+ }
+
+ inline int32_t Imm9Value() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kImmediateType);
+ return this->Bits(kImm9Shift + kImm9Bits - 1, kImm9Shift);
+ }
+
+ inline int32_t Imm16Value() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kImmediateType);
+ return this->Bits(kImm16Shift + kImm16Bits - 1, kImm16Shift);
+ }
+
+ inline int32_t Imm18Value() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kImmediateType);
+ return this->Bits(kImm18Shift + kImm18Bits - 1, kImm18Shift);
+ }
+
+ inline int32_t Imm19Value() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kImmediateType);
+ return this->Bits(kImm19Shift + kImm19Bits - 1, kImm19Shift);
+ }
+
+ inline int32_t Imm21Value() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kImmediateType);
+ return this->Bits(kImm21Shift + kImm21Bits - 1, kImm21Shift);
+ }
+
+ inline int32_t Imm26Value() const {
+ DCHECK((this->InstructionType() == InstructionBase::kJumpType) ||
+ (this->InstructionType() == InstructionBase::kImmediateType));
+ return this->Bits(kImm26Shift + kImm26Bits - 1, kImm26Shift);
+ }
+
+ inline int32_t MsaImm8Value() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kImmediateType);
+ return this->Bits(kMsaImm8Shift + kMsaImm8Bits - 1, kMsaImm8Shift);
+ }
+
+ inline int32_t MsaImm5Value() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kImmediateType);
+ return this->Bits(kMsaImm5Shift + kMsaImm5Bits - 1, kMsaImm5Shift);
+ }
+
+ inline int32_t MsaImm10Value() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kImmediateType);
+ return this->Bits(kMsaImm10Shift + kMsaImm10Bits - 1, kMsaImm10Shift);
+ }
+
+ inline int32_t MsaImmMI10Value() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kImmediateType);
+ return this->Bits(kMsaImmMI10Shift + kMsaImmMI10Bits - 1, kMsaImmMI10Shift);
+ }
+
+ inline int32_t MsaBitDf() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kImmediateType);
+ int32_t df_m = this->Bits(22, 16);
+ if (((df_m >> 6) & 1U) == 0) {
+ return 3;
+ } else if (((df_m >> 5) & 3U) == 2) {
+ return 2;
+ } else if (((df_m >> 4) & 7U) == 6) {
+ return 1;
+ } else if (((df_m >> 3) & 15U) == 14) {
+ return 0;
+ } else {
+ return -1;
+ }
+ }
+
+ inline int32_t MsaBitMValue() const {
+ DCHECK_EQ(this->InstructionType(), InstructionBase::kImmediateType);
+ return this->Bits(16 + this->MsaBitDf() + 3, 16);
+ }
+
+ inline int32_t MsaElmDf() const {
+ DCHECK(this->InstructionType() == InstructionBase::kRegisterType ||
+ this->InstructionType() == InstructionBase::kImmediateType);
+ int32_t df_n = this->Bits(21, 16);
+ if (((df_n >> 4) & 3U) == 0) {
+ return 0;
+ } else if (((df_n >> 3) & 7U) == 4) {
+ return 1;
+ } else if (((df_n >> 2) & 15U) == 12) {
+ return 2;
+ } else if (((df_n >> 1) & 31U) == 28) {
+ return 3;
+ } else {
+ return -1;
+ }
+ }
+
+ inline int32_t MsaElmNValue() const {
+ DCHECK(this->InstructionType() == InstructionBase::kRegisterType ||
+ this->InstructionType() == InstructionBase::kImmediateType);
+ return this->Bits(16 + 4 - this->MsaElmDf(), 16);
+ }
+
+ static bool IsForbiddenAfterBranchInstr(Instr instr);
+
+ // Say if the instruction should not be used in a branch delay slot or
+ // immediately after a compact branch.
+ inline bool IsForbiddenAfterBranch() const {
+ return IsForbiddenAfterBranchInstr(this->InstructionBits());
+ }
+
+ inline bool IsForbiddenInBranchDelay() const {
+ return IsForbiddenAfterBranch();
+ }
+
+ // Say if the instruction 'links'. e.g. jal, bal.
+ bool IsLinkingInstruction() const;
+ // Say if the instruction is a break or a trap.
+ bool IsTrap() const;
+
+ inline bool IsMSABranchInstr() const {
+ if (this->OpcodeFieldRaw() == COP1) {
+ switch (this->RsFieldRaw()) {
+ case BZ_V:
+ case BZ_B:
+ case BZ_H:
+ case BZ_W:
+ case BZ_D:
+ case BNZ_V:
+ case BNZ_B:
+ case BNZ_H:
+ case BNZ_W:
+ case BNZ_D:
+ return true;
+ default:
+ return false;
+ }
+ }
+ return false;
+ }
+
+ inline bool IsMSAInstr() const {
+ if (this->IsMSABranchInstr() || (this->OpcodeFieldRaw() == MSA))
+ return true;
+ return false;
+ }
+};
+
+class Instruction : public InstructionGetters<InstructionBase> {
+ public:
+ // Instructions are read of out a code stream. The only way to get a
+ // reference to an instruction is to convert a pointer. There is no way
+ // to allocate or create instances of class Instruction.
+ // Use the At(pc) function to create references to Instruction.
+ static Instruction* At(byte* pc) {
+ return reinterpret_cast<Instruction*>(pc);
+ }
+
+ private:
+ // We need to prevent the creation of instances of class Instruction.
+ DISALLOW_IMPLICIT_CONSTRUCTORS(Instruction);
+};
+
+// -----------------------------------------------------------------------------
+// MIPS assembly various constants.
+
+// C/C++ argument slots size.
+const int kCArgSlotCount = 0;
+
+// TODO(plind): below should be based on kPointerSize
+// TODO(plind): find all usages and remove the needless instructions for n64.
+const int kCArgsSlotsSize = kCArgSlotCount * kInstrSize * 2;
+
+const int kInvalidStackOffset = -1;
+const int kBranchReturnOffset = 2 * kInstrSize;
+
+static const int kNegOffset = 0x00008000;
+
+InstructionBase::Type InstructionBase::InstructionType() const {
+ switch (OpcodeFieldRaw()) {
+ case SPECIAL:
+ if (FunctionFieldToBitNumber(FunctionFieldRaw()) &
+ kFunctionFieldRegisterTypeMask) {
+ return kRegisterType;
+ }
+ return kUnsupported;
+ case SPECIAL2:
+ switch (FunctionFieldRaw()) {
+ case MUL:
+ case CLZ:
+ case DCLZ:
+ return kRegisterType;
+ default:
+ return kUnsupported;
+ }
+ break;
+ case SPECIAL3:
+ switch (FunctionFieldRaw()) {
+ case INS:
+ case DINS:
+ case DINSM:
+ case DINSU:
+ case EXT:
+ case DEXT:
+ case DEXTM:
+ case DEXTU:
+ return kRegisterType;
+ case BSHFL: {
+ int sa = SaFieldRaw() >> kSaShift;
+ switch (sa) {
+ case BITSWAP:
+ case WSBH:
+ case SEB:
+ case SEH:
+ return kRegisterType;
+ }
+ sa >>= kBp2Bits;
+ switch (sa) {
+ case ALIGN:
+ return kRegisterType;
+ default:
+ return kUnsupported;
+ }
+ }
+ case LL_R6:
+ case LLD_R6:
+ case SC_R6:
+ case SCD_R6: {
+ DCHECK_EQ(kArchVariant, kMips64r6);
+ return kImmediateType;
+ }
+ case DBSHFL: {
+ int sa = SaFieldRaw() >> kSaShift;
+ switch (sa) {
+ case DBITSWAP:
+ case DSBH:
+ case DSHD:
+ return kRegisterType;
+ }
+ sa = SaFieldRaw() >> kSaShift;
+ sa >>= kBp3Bits;
+ switch (sa) {
+ case DALIGN:
+ return kRegisterType;
+ default:
+ return kUnsupported;
+ }
+ }
+ default:
+ return kUnsupported;
+ }
+ break;
+ case COP1: // Coprocessor instructions.
+ switch (RsFieldRawNoAssert()) {
+ case BC1: // Branch on coprocessor condition.
+ case BC1EQZ:
+ case BC1NEZ:
+ return kImmediateType;
+ // MSA Branch instructions
+ case BZ_V:
+ case BNZ_V:
+ case BZ_B:
+ case BZ_H:
+ case BZ_W:
+ case BZ_D:
+ case BNZ_B:
+ case BNZ_H:
+ case BNZ_W:
+ case BNZ_D:
+ return kImmediateType;
+ default:
+ return kRegisterType;
+ }
+ break;
+ case COP1X:
+ return kRegisterType;
+
+ // 26 bits immediate type instructions. e.g.: j imm26.
+ case J:
+ case JAL:
+ return kJumpType;
+
+ case MSA:
+ switch (MSAMinorOpcodeField()) {
+ case kMsaMinor3R:
+ case kMsaMinor3RF:
+ case kMsaMinorVEC:
+ case kMsaMinor2R:
+ case kMsaMinor2RF:
+ return kRegisterType;
+ case kMsaMinorELM:
+ switch (InstructionBits() & kMsaLongerELMMask) {
+ case CFCMSA:
+ case CTCMSA:
+ case MOVE_V:
+ return kRegisterType;
+ default:
+ return kImmediateType;
+ }
+ default:
+ return kImmediateType;
+ }
+
+ default:
+ return kImmediateType;
+ }
+ return kUnsupported;
+}
+#undef OpcodeToBitNumber
+#undef FunctionFieldToBitNumber
+
+// -----------------------------------------------------------------------------
+// Instructions.
+
+template <class P>
+bool InstructionGetters<P>::IsLinkingInstruction() const {
+ switch (OpcodeFieldRaw()) {
+ case JAL:
+ return true;
+ case POP76:
+ if (RsFieldRawNoAssert() == JIALC)
+ return true; // JIALC
+ else
+ return false; // BNEZC
+ case REGIMM:
+ switch (RtFieldRaw()) {
+ case BGEZAL:
+ case BLTZAL:
+ return true;
+ default:
+ return false;
+ }
+ case SPECIAL:
+ switch (FunctionFieldRaw()) {
+ case JALR:
+ return true;
+ default:
+ return false;
+ }
+ default:
+ return false;
+ }
+}
+
+template <class P>
+bool InstructionGetters<P>::IsTrap() const {
+ if (OpcodeFieldRaw() != SPECIAL) {
+ return false;
+ } else {
+ switch (FunctionFieldRaw()) {
+ case BREAK:
+ case TGE:
+ case TGEU:
+ case TLT:
+ case TLTU:
+ case TEQ:
+ case TNE:
+ return true;
+ default:
+ return false;
+ }
+ }
+}
+
+// static
+template <class T>
+bool InstructionGetters<T>::IsForbiddenAfterBranchInstr(Instr instr) {
+ Opcode opcode = static_cast<Opcode>(instr & kOpcodeMask);
+ switch (opcode) {
+ case J:
+ case JAL:
+ case BEQ:
+ case BNE:
+ case BLEZ: // POP06 bgeuc/bleuc, blezalc, bgezalc
+ case BGTZ: // POP07 bltuc/bgtuc, bgtzalc, bltzalc
+ case BEQL:
+ case BNEL:
+ case BLEZL: // POP26 bgezc, blezc, bgec/blec
+ case BGTZL: // POP27 bgtzc, bltzc, bltc/bgtc
+ case BC:
+ case BALC:
+ case POP10: // beqzalc, bovc, beqc
+ case POP30: // bnezalc, bnvc, bnec
+ case POP66: // beqzc, jic
+ case POP76: // bnezc, jialc
+ return true;
+ case REGIMM:
+ switch (instr & kRtFieldMask) {
+ case BLTZ:
+ case BGEZ:
+ case BLTZAL:
+ case BGEZAL:
+ return true;
+ default:
+ return false;
+ }
+ break;
+ case SPECIAL:
+ switch (instr & kFunctionFieldMask) {
+ case JR:
+ case JALR:
+ return true;
+ default:
+ return false;
+ }
+ break;
+ case COP1:
+ switch (instr & kRsFieldMask) {
+ case BC1:
+ case BC1EQZ:
+ case BC1NEZ:
+ case BZ_V:
+ case BZ_B:
+ case BZ_H:
+ case BZ_W:
+ case BZ_D:
+ case BNZ_V:
+ case BNZ_B:
+ case BNZ_H:
+ case BNZ_W:
+ case BNZ_D:
+ return true;
+ break;
+ default:
+ return false;
+ }
+ break;
+ default:
+ return false;
+ }
+}
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_MIPS64_CONSTANTS_MIPS64_H_
diff --git a/src/codegen/mips64/cpu-mips64.cc b/src/codegen/mips64/cpu-mips64.cc
new file mode 100644
index 0000000..0f3713b
--- /dev/null
+++ b/src/codegen/mips64/cpu-mips64.cc
@@ -0,0 +1,45 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// CPU specific code for arm independent of OS goes here.
+
+#include <sys/syscall.h>
+#include <unistd.h>
+
+#ifdef __mips
+#include <asm/cachectl.h>
+#endif // #ifdef __mips
+
+#if V8_TARGET_ARCH_MIPS64
+
+#include "src/codegen/cpu-features.h"
+
+namespace v8 {
+namespace internal {
+
+void CpuFeatures::FlushICache(void* start, size_t size) {
+#if !defined(USE_SIMULATOR)
+ // Nothing to do, flushing no instructions.
+ if (size == 0) {
+ return;
+ }
+
+#if defined(ANDROID) && !defined(__LP64__)
+ // Bionic cacheflush can typically run in userland, avoiding kernel call.
+ char* end = reinterpret_cast<char*>(start) + size;
+ cacheflush(reinterpret_cast<intptr_t>(start), reinterpret_cast<intptr_t>(end),
+ 0);
+#else // ANDROID
+ long res; // NOLINT(runtime/int)
+ // See http://www.linux-mips.org/wiki/Cacheflush_Syscall.
+ res = syscall(__NR_cacheflush, start, size, ICACHE);
+ if (res) FATAL("Failed to flush the instruction cache");
+#endif // ANDROID
+#endif // !USE_SIMULATOR.
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_MIPS64
diff --git a/src/codegen/mips64/interface-descriptors-mips64.cc b/src/codegen/mips64/interface-descriptors-mips64.cc
new file mode 100644
index 0000000..f77d8d4
--- /dev/null
+++ b/src/codegen/mips64/interface-descriptors-mips64.cc
@@ -0,0 +1,310 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#if V8_TARGET_ARCH_MIPS64
+
+#include "src/codegen/interface-descriptors.h"
+
+#include "src/execution/frames.h"
+
+namespace v8 {
+namespace internal {
+
+const Register CallInterfaceDescriptor::ContextRegister() { return cp; }
+
+void CallInterfaceDescriptor::DefaultInitializePlatformSpecific(
+ CallInterfaceDescriptorData* data, int register_parameter_count) {
+ const Register default_stub_registers[] = {a0, a1, a2, a3, a4};
+ CHECK_LE(static_cast<size_t>(register_parameter_count),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(register_parameter_count,
+ default_stub_registers);
+}
+
+// On MIPS it is not allowed to use odd numbered floating point registers
+// (e.g. f1, f3, etc.) for parameters. This can happen if we use
+// DefaultInitializePlatformSpecific to assign float registers for parameters.
+// E.g if fourth parameter goes to float register, f7 would be assigned for
+// parameter (a3 casted to int is 7).
+bool CallInterfaceDescriptor::IsValidFloatParameterRegister(Register reg) {
+ return reg.code() % 2 == 0;
+}
+
+void WasmI32AtomicWait32Descriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ const Register default_stub_registers[] = {a0, a1, a2, a3};
+ CHECK_EQ(static_cast<size_t>(kParameterCount),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(kParameterCount, default_stub_registers);
+}
+
+void WasmI64AtomicWait32Descriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ const Register default_stub_registers[] = {a0, a1, a2, a3, a4};
+ CHECK_EQ(static_cast<size_t>(kParameterCount - kStackArgumentsCount),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(kParameterCount - kStackArgumentsCount,
+ default_stub_registers);
+}
+
+void RecordWriteDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ const Register default_stub_registers[] = {a0, a1, a2, a3, kReturnRegister0};
+
+ data->RestrictAllocatableRegisters(default_stub_registers,
+ arraysize(default_stub_registers));
+
+ CHECK_LE(static_cast<size_t>(kParameterCount),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(kParameterCount, default_stub_registers);
+}
+
+void EphemeronKeyBarrierDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ const Register default_stub_registers[] = {a0, a1, a2, a3, kReturnRegister0};
+
+ data->RestrictAllocatableRegisters(default_stub_registers,
+ arraysize(default_stub_registers));
+
+ CHECK_LE(static_cast<size_t>(kParameterCount),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(kParameterCount, default_stub_registers);
+}
+
+const Register LoadDescriptor::ReceiverRegister() { return a1; }
+const Register LoadDescriptor::NameRegister() { return a2; }
+const Register LoadDescriptor::SlotRegister() { return a0; }
+
+const Register LoadWithVectorDescriptor::VectorRegister() { return a3; }
+
+const Register
+LoadWithReceiverAndVectorDescriptor::LookupStartObjectRegister() {
+ return a4;
+}
+
+const Register StoreDescriptor::ReceiverRegister() { return a1; }
+const Register StoreDescriptor::NameRegister() { return a2; }
+const Register StoreDescriptor::ValueRegister() { return a0; }
+const Register StoreDescriptor::SlotRegister() { return a4; }
+
+const Register StoreWithVectorDescriptor::VectorRegister() { return a3; }
+
+const Register StoreTransitionDescriptor::SlotRegister() { return a4; }
+const Register StoreTransitionDescriptor::VectorRegister() { return a3; }
+const Register StoreTransitionDescriptor::MapRegister() { return a5; }
+
+const Register ApiGetterDescriptor::HolderRegister() { return a0; }
+const Register ApiGetterDescriptor::CallbackRegister() { return a3; }
+
+const Register GrowArrayElementsDescriptor::ObjectRegister() { return a0; }
+const Register GrowArrayElementsDescriptor::KeyRegister() { return a3; }
+
+// static
+const Register TypeConversionDescriptor::ArgumentRegister() { return a0; }
+
+void TypeofDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {a3};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallTrampolineDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // a1: target
+ // a0: number of arguments
+ Register registers[] = {a1, a0};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // a0 : number of arguments (on the stack, not including receiver)
+ // a1 : the target to call
+ // a4 : arguments list length (untagged)
+ // a2 : arguments list (FixedArray)
+ Register registers[] = {a1, a0, a4, a2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallForwardVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // a1: the target to call
+ // a0: number of arguments
+ // a2: start index (to support rest parameters)
+ Register registers[] = {a1, a0, a2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallFunctionTemplateDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // a1 : function template info
+ // a0 : number of arguments (on the stack, not including receiver)
+ Register registers[] = {a1, a0};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallWithSpreadDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // a0 : number of arguments (on the stack, not including receiver)
+ // a1 : the target to call
+ // a2 : the object to spread
+ Register registers[] = {a1, a0, a2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallWithArrayLikeDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // a1 : the target to call
+ // a2 : the arguments list
+ Register registers[] = {a1, a2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // a0 : number of arguments (on the stack, not including receiver)
+ // a1 : the target to call
+ // a3 : the new target
+ // a4 : arguments list length (untagged)
+ // a2 : arguments list (FixedArray)
+ Register registers[] = {a1, a3, a0, a4, a2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructForwardVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // a1: the target to call
+ // a3: new target
+ // a0: number of arguments
+ // a2: start index (to support rest parameters)
+ Register registers[] = {a1, a3, a0, a2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructWithSpreadDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // a0 : number of arguments (on the stack, not including receiver)
+ // a1 : the target to call
+ // a3 : the new target
+ // a2 : the object to spread
+ Register registers[] = {a1, a3, a0, a2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructWithArrayLikeDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // a1 : the target to call
+ // a3 : the new target
+ // a2 : the arguments list
+ Register registers[] = {a1, a3, a2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructStubDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // a1: target
+ // a3: new target
+ // a0: number of arguments
+ // a2: allocation site or undefined
+ Register registers[] = {a1, a3, a0, a2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void AbortDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {a0};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CompareDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {a1, a0};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void BinaryOpDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {a1, a0};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ArgumentsAdaptorDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ a1, // JSFunction
+ a3, // the new target
+ a0, // actual number of arguments
+ a2, // expected number of arguments
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ApiCallbackDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ a1, // kApiFunctionAddress
+ a2, // kArgc
+ a3, // kCallData
+ a0, // kHolder
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void InterpreterDispatchDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ kInterpreterAccumulatorRegister, kInterpreterBytecodeOffsetRegister,
+ kInterpreterBytecodeArrayRegister, kInterpreterDispatchTableRegister};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void InterpreterPushArgsThenCallDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ a0, // argument count (not including receiver)
+ a2, // address of first argument
+ a1 // the target callable to be call
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void InterpreterPushArgsThenConstructDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ a0, // argument count (not including receiver)
+ a4, // address of the first argument
+ a1, // constructor to call
+ a3, // new target
+ a2, // allocation site feedback if available, undefined otherwise
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ResumeGeneratorDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ v0, // the value to pass to the generator
+ a1 // the JSGeneratorObject to resume
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void FrameDropperTrampolineDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ a1, // loaded new FP
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void RunMicrotasksEntryDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {a0, a1};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_MIPS64
diff --git a/src/codegen/mips64/macro-assembler-mips64.cc b/src/codegen/mips64/macro-assembler-mips64.cc
new file mode 100644
index 0000000..249fc91
--- /dev/null
+++ b/src/codegen/mips64/macro-assembler-mips64.cc
@@ -0,0 +1,5987 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <limits.h> // For LONG_MIN, LONG_MAX.
+
+#if V8_TARGET_ARCH_MIPS64
+
+#include "src/base/bits.h"
+#include "src/base/division-by-constant.h"
+#include "src/codegen/assembler-inl.h"
+#include "src/codegen/callable.h"
+#include "src/codegen/code-factory.h"
+#include "src/codegen/external-reference-table.h"
+#include "src/codegen/macro-assembler.h"
+#include "src/codegen/register-configuration.h"
+#include "src/debug/debug.h"
+#include "src/execution/frames-inl.h"
+#include "src/heap/memory-chunk.h"
+#include "src/init/bootstrapper.h"
+#include "src/logging/counters.h"
+#include "src/objects/heap-number.h"
+#include "src/runtime/runtime.h"
+#include "src/snapshot/embedded/embedded-data.h"
+#include "src/snapshot/snapshot.h"
+#include "src/wasm/wasm-code-manager.h"
+
+// Satisfy cpplint check, but don't include platform-specific header. It is
+// included recursively via macro-assembler.h.
+#if 0
+#include "src/codegen/mips64/macro-assembler-mips64.h"
+#endif
+
+namespace v8 {
+namespace internal {
+
+static inline bool IsZero(const Operand& rt) {
+ if (rt.is_reg()) {
+ return rt.rm() == zero_reg;
+ } else {
+ return rt.immediate() == 0;
+ }
+}
+
+int TurboAssembler::RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
+ Register exclusion1,
+ Register exclusion2,
+ Register exclusion3) const {
+ int bytes = 0;
+ RegList exclusions = 0;
+ if (exclusion1 != no_reg) {
+ exclusions |= exclusion1.bit();
+ if (exclusion2 != no_reg) {
+ exclusions |= exclusion2.bit();
+ if (exclusion3 != no_reg) {
+ exclusions |= exclusion3.bit();
+ }
+ }
+ }
+
+ RegList list = kJSCallerSaved & ~exclusions;
+ bytes += NumRegs(list) * kPointerSize;
+
+ if (fp_mode == kSaveFPRegs) {
+ bytes += NumRegs(kCallerSavedFPU) * kDoubleSize;
+ }
+
+ return bytes;
+}
+
+int TurboAssembler::PushCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1,
+ Register exclusion2, Register exclusion3) {
+ int bytes = 0;
+ RegList exclusions = 0;
+ if (exclusion1 != no_reg) {
+ exclusions |= exclusion1.bit();
+ if (exclusion2 != no_reg) {
+ exclusions |= exclusion2.bit();
+ if (exclusion3 != no_reg) {
+ exclusions |= exclusion3.bit();
+ }
+ }
+ }
+
+ RegList list = kJSCallerSaved & ~exclusions;
+ MultiPush(list);
+ bytes += NumRegs(list) * kPointerSize;
+
+ if (fp_mode == kSaveFPRegs) {
+ MultiPushFPU(kCallerSavedFPU);
+ bytes += NumRegs(kCallerSavedFPU) * kDoubleSize;
+ }
+
+ return bytes;
+}
+
+int TurboAssembler::PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1,
+ Register exclusion2, Register exclusion3) {
+ int bytes = 0;
+ if (fp_mode == kSaveFPRegs) {
+ MultiPopFPU(kCallerSavedFPU);
+ bytes += NumRegs(kCallerSavedFPU) * kDoubleSize;
+ }
+
+ RegList exclusions = 0;
+ if (exclusion1 != no_reg) {
+ exclusions |= exclusion1.bit();
+ if (exclusion2 != no_reg) {
+ exclusions |= exclusion2.bit();
+ if (exclusion3 != no_reg) {
+ exclusions |= exclusion3.bit();
+ }
+ }
+ }
+
+ RegList list = kJSCallerSaved & ~exclusions;
+ MultiPop(list);
+ bytes += NumRegs(list) * kPointerSize;
+
+ return bytes;
+}
+
+void TurboAssembler::LoadRoot(Register destination, RootIndex index) {
+ Ld(destination, MemOperand(s6, RootRegisterOffsetForRootIndex(index)));
+}
+
+void TurboAssembler::LoadRoot(Register destination, RootIndex index,
+ Condition cond, Register src1,
+ const Operand& src2) {
+ Branch(2, NegateCondition(cond), src1, src2);
+ Ld(destination, MemOperand(s6, RootRegisterOffsetForRootIndex(index)));
+}
+
+void TurboAssembler::PushCommonFrame(Register marker_reg) {
+ if (marker_reg.is_valid()) {
+ Push(ra, fp, marker_reg);
+ Daddu(fp, sp, Operand(kPointerSize));
+ } else {
+ Push(ra, fp);
+ mov(fp, sp);
+ }
+}
+
+void TurboAssembler::PushStandardFrame(Register function_reg) {
+ int offset = -StandardFrameConstants::kContextOffset;
+ if (function_reg.is_valid()) {
+ Push(ra, fp, cp, function_reg, kJavaScriptCallArgCountRegister);
+ offset += 2 * kPointerSize;
+ } else {
+ Push(ra, fp, cp, kJavaScriptCallArgCountRegister);
+ offset += kPointerSize;
+ }
+ Daddu(fp, sp, Operand(offset));
+}
+
+int MacroAssembler::SafepointRegisterStackIndex(int reg_code) {
+ // The registers are pushed starting with the highest encoding,
+ // which means that lowest encodings are closest to the stack pointer.
+ return kSafepointRegisterStackIndexMap[reg_code];
+}
+
+// Clobbers object, dst, value, and ra, if (ra_status == kRAHasBeenSaved)
+// The register 'object' contains a heap object pointer. The heap object
+// tag is shifted away.
+void MacroAssembler::RecordWriteField(Register object, int offset,
+ Register value, Register dst,
+ RAStatus ra_status,
+ SaveFPRegsMode save_fp,
+ RememberedSetAction remembered_set_action,
+ SmiCheck smi_check) {
+ DCHECK(!AreAliased(value, dst, t8, object));
+ // First, check if a write barrier is even needed. The tests below
+ // catch stores of Smis.
+ Label done;
+
+ // Skip barrier if writing a smi.
+ if (smi_check == INLINE_SMI_CHECK) {
+ JumpIfSmi(value, &done);
+ }
+
+ // Although the object register is tagged, the offset is relative to the start
+ // of the object, so so offset must be a multiple of kPointerSize.
+ DCHECK(IsAligned(offset, kPointerSize));
+
+ Daddu(dst, object, Operand(offset - kHeapObjectTag));
+ if (emit_debug_code()) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Label ok;
+ And(t8, dst, Operand(kPointerSize - 1));
+ Branch(&ok, eq, t8, Operand(zero_reg));
+ stop();
+ bind(&ok);
+ }
+
+ RecordWrite(object, dst, value, ra_status, save_fp, remembered_set_action,
+ OMIT_SMI_CHECK);
+
+ bind(&done);
+
+ // Clobber clobbered input registers when running with the debug-code flag
+ // turned on to provoke errors.
+ if (emit_debug_code()) {
+ li(value, Operand(bit_cast<int64_t>(kZapValue + 4)));
+ li(dst, Operand(bit_cast<int64_t>(kZapValue + 8)));
+ }
+}
+
+void TurboAssembler::SaveRegisters(RegList registers) {
+ DCHECK_GT(NumRegs(registers), 0);
+ RegList regs = 0;
+ for (int i = 0; i < Register::kNumRegisters; ++i) {
+ if ((registers >> i) & 1u) {
+ regs |= Register::from_code(i).bit();
+ }
+ }
+ MultiPush(regs);
+}
+
+void TurboAssembler::RestoreRegisters(RegList registers) {
+ DCHECK_GT(NumRegs(registers), 0);
+ RegList regs = 0;
+ for (int i = 0; i < Register::kNumRegisters; ++i) {
+ if ((registers >> i) & 1u) {
+ regs |= Register::from_code(i).bit();
+ }
+ }
+ MultiPop(regs);
+}
+
+void TurboAssembler::CallEphemeronKeyBarrier(Register object, Register address,
+ SaveFPRegsMode fp_mode) {
+ EphemeronKeyBarrierDescriptor descriptor;
+ RegList registers = descriptor.allocatable_registers();
+
+ SaveRegisters(registers);
+
+ Register object_parameter(
+ descriptor.GetRegisterParameter(EphemeronKeyBarrierDescriptor::kObject));
+ Register slot_parameter(descriptor.GetRegisterParameter(
+ EphemeronKeyBarrierDescriptor::kSlotAddress));
+ Register fp_mode_parameter(
+ descriptor.GetRegisterParameter(EphemeronKeyBarrierDescriptor::kFPMode));
+
+ Push(object);
+ Push(address);
+
+ Pop(slot_parameter);
+ Pop(object_parameter);
+
+ Move(fp_mode_parameter, Smi::FromEnum(fp_mode));
+ Call(isolate()->builtins()->builtin_handle(Builtins::kEphemeronKeyBarrier),
+ RelocInfo::CODE_TARGET);
+ RestoreRegisters(registers);
+}
+
+void TurboAssembler::CallRecordWriteStub(
+ Register object, Register address,
+ RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode) {
+ CallRecordWriteStub(
+ object, address, remembered_set_action, fp_mode,
+ isolate()->builtins()->builtin_handle(Builtins::kRecordWrite),
+ kNullAddress);
+}
+
+void TurboAssembler::CallRecordWriteStub(
+ Register object, Register address,
+ RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode,
+ Address wasm_target) {
+ CallRecordWriteStub(object, address, remembered_set_action, fp_mode,
+ Handle<Code>::null(), wasm_target);
+}
+
+void TurboAssembler::CallRecordWriteStub(
+ Register object, Register address,
+ RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode,
+ Handle<Code> code_target, Address wasm_target) {
+ DCHECK_NE(code_target.is_null(), wasm_target == kNullAddress);
+ // TODO(albertnetymk): For now we ignore remembered_set_action and fp_mode,
+ // i.e. always emit remember set and save FP registers in RecordWriteStub. If
+ // large performance regression is observed, we should use these values to
+ // avoid unnecessary work.
+
+ RecordWriteDescriptor descriptor;
+ RegList registers = descriptor.allocatable_registers();
+
+ SaveRegisters(registers);
+ Register object_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kObject));
+ Register slot_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kSlot));
+ Register remembered_set_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kRememberedSet));
+ Register fp_mode_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kFPMode));
+
+ Push(object);
+ Push(address);
+
+ Pop(slot_parameter);
+ Pop(object_parameter);
+
+ Move(remembered_set_parameter, Smi::FromEnum(remembered_set_action));
+ Move(fp_mode_parameter, Smi::FromEnum(fp_mode));
+ if (code_target.is_null()) {
+ Call(wasm_target, RelocInfo::WASM_STUB_CALL);
+ } else {
+ Call(code_target, RelocInfo::CODE_TARGET);
+ }
+
+ RestoreRegisters(registers);
+}
+
+// Clobbers object, address, value, and ra, if (ra_status == kRAHasBeenSaved)
+// The register 'object' contains a heap object pointer. The heap object
+// tag is shifted away.
+void MacroAssembler::RecordWrite(Register object, Register address,
+ Register value, RAStatus ra_status,
+ SaveFPRegsMode fp_mode,
+ RememberedSetAction remembered_set_action,
+ SmiCheck smi_check) {
+ DCHECK(!AreAliased(object, address, value, t8));
+ DCHECK(!AreAliased(object, address, value, t9));
+
+ if (emit_debug_code()) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ Ld(scratch, MemOperand(address));
+ Assert(eq, AbortReason::kWrongAddressOrValuePassedToRecordWrite, scratch,
+ Operand(value));
+ }
+
+ if ((remembered_set_action == OMIT_REMEMBERED_SET &&
+ !FLAG_incremental_marking) ||
+ FLAG_disable_write_barriers) {
+ return;
+ }
+
+ // First, check if a write barrier is even needed. The tests below
+ // catch stores of smis and stores into the young generation.
+ Label done;
+
+ if (smi_check == INLINE_SMI_CHECK) {
+ DCHECK_EQ(0, kSmiTag);
+ JumpIfSmi(value, &done);
+ }
+
+ CheckPageFlag(value,
+ value, // Used as scratch.
+ MemoryChunk::kPointersToHereAreInterestingMask, eq, &done);
+ CheckPageFlag(object,
+ value, // Used as scratch.
+ MemoryChunk::kPointersFromHereAreInterestingMask, eq, &done);
+
+ // Record the actual write.
+ if (ra_status == kRAHasNotBeenSaved) {
+ push(ra);
+ }
+ CallRecordWriteStub(object, address, remembered_set_action, fp_mode);
+ if (ra_status == kRAHasNotBeenSaved) {
+ pop(ra);
+ }
+
+ bind(&done);
+
+ // Clobber clobbered registers when running with the debug-code flag
+ // turned on to provoke errors.
+ if (emit_debug_code()) {
+ li(address, Operand(bit_cast<int64_t>(kZapValue + 12)));
+ li(value, Operand(bit_cast<int64_t>(kZapValue + 16)));
+ }
+}
+
+// ---------------------------------------------------------------------------
+// Instruction macros.
+
+void TurboAssembler::Addu(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ addu(rd, rs, rt.rm());
+ } else {
+ if (is_int16(rt.immediate()) && !MustUseReg(rt.rmode())) {
+ addiu(rd, rs, static_cast<int32_t>(rt.immediate()));
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ addu(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Daddu(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ daddu(rd, rs, rt.rm());
+ } else {
+ if (is_int16(rt.immediate()) && !MustUseReg(rt.rmode())) {
+ daddiu(rd, rs, static_cast<int32_t>(rt.immediate()));
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ daddu(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Subu(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ subu(rd, rs, rt.rm());
+ } else {
+ DCHECK(is_int32(rt.immediate()));
+ if (is_int16(-rt.immediate()) && !MustUseReg(rt.rmode())) {
+ addiu(rd, rs,
+ static_cast<int32_t>(
+ -rt.immediate())); // No subiu instr, use addiu(x, y, -imm).
+ } else {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ if (-rt.immediate() >> 16 == 0 && !MustUseReg(rt.rmode())) {
+ // Use load -imm and addu when loading -imm generates one instruction.
+ li(scratch, -rt.immediate());
+ addu(rd, rs, scratch);
+ } else {
+ // li handles the relocation.
+ li(scratch, rt);
+ subu(rd, rs, scratch);
+ }
+ }
+ }
+}
+
+void TurboAssembler::Dsubu(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ dsubu(rd, rs, rt.rm());
+ } else if (is_int16(-rt.immediate()) && !MustUseReg(rt.rmode())) {
+ daddiu(rd, rs,
+ static_cast<int32_t>(
+ -rt.immediate())); // No dsubiu instr, use daddiu(x, y, -imm).
+ } else {
+ DCHECK(rs != at);
+ int li_count = InstrCountForLi64Bit(rt.immediate());
+ int li_neg_count = InstrCountForLi64Bit(-rt.immediate());
+ if (li_neg_count < li_count && !MustUseReg(rt.rmode())) {
+ // Use load -imm and daddu when loading -imm generates one instruction.
+ DCHECK(rt.immediate() != std::numeric_limits<int32_t>::min());
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ li(scratch, Operand(-rt.immediate()));
+ Daddu(rd, rs, scratch);
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ li(scratch, rt);
+ dsubu(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Mul(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ mul(rd, rs, rt.rm());
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ mul(rd, rs, scratch);
+ }
+}
+
+void TurboAssembler::Mulh(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ if (kArchVariant != kMips64r6) {
+ mult(rs, rt.rm());
+ mfhi(rd);
+ } else {
+ muh(rd, rs, rt.rm());
+ }
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ if (kArchVariant != kMips64r6) {
+ mult(rs, scratch);
+ mfhi(rd);
+ } else {
+ muh(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Mulhu(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ if (kArchVariant != kMips64r6) {
+ multu(rs, rt.rm());
+ mfhi(rd);
+ } else {
+ muhu(rd, rs, rt.rm());
+ }
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ if (kArchVariant != kMips64r6) {
+ multu(rs, scratch);
+ mfhi(rd);
+ } else {
+ muhu(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Dmul(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ if (kArchVariant == kMips64r6) {
+ dmul(rd, rs, rt.rm());
+ } else {
+ dmult(rs, rt.rm());
+ mflo(rd);
+ }
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ if (kArchVariant == kMips64r6) {
+ dmul(rd, rs, scratch);
+ } else {
+ dmult(rs, scratch);
+ mflo(rd);
+ }
+ }
+}
+
+void TurboAssembler::Dmulh(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ if (kArchVariant == kMips64r6) {
+ dmuh(rd, rs, rt.rm());
+ } else {
+ dmult(rs, rt.rm());
+ mfhi(rd);
+ }
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ if (kArchVariant == kMips64r6) {
+ dmuh(rd, rs, scratch);
+ } else {
+ dmult(rs, scratch);
+ mfhi(rd);
+ }
+ }
+}
+
+void TurboAssembler::Mult(Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ mult(rs, rt.rm());
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ mult(rs, scratch);
+ }
+}
+
+void TurboAssembler::Dmult(Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ dmult(rs, rt.rm());
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ dmult(rs, scratch);
+ }
+}
+
+void TurboAssembler::Multu(Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ multu(rs, rt.rm());
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ multu(rs, scratch);
+ }
+}
+
+void TurboAssembler::Dmultu(Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ dmultu(rs, rt.rm());
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ dmultu(rs, scratch);
+ }
+}
+
+void TurboAssembler::Div(Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ div(rs, rt.rm());
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ div(rs, scratch);
+ }
+}
+
+void TurboAssembler::Div(Register res, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ if (kArchVariant != kMips64r6) {
+ div(rs, rt.rm());
+ mflo(res);
+ } else {
+ div(res, rs, rt.rm());
+ }
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ if (kArchVariant != kMips64r6) {
+ div(rs, scratch);
+ mflo(res);
+ } else {
+ div(res, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Mod(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ if (kArchVariant != kMips64r6) {
+ div(rs, rt.rm());
+ mfhi(rd);
+ } else {
+ mod(rd, rs, rt.rm());
+ }
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ if (kArchVariant != kMips64r6) {
+ div(rs, scratch);
+ mfhi(rd);
+ } else {
+ mod(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Modu(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ if (kArchVariant != kMips64r6) {
+ divu(rs, rt.rm());
+ mfhi(rd);
+ } else {
+ modu(rd, rs, rt.rm());
+ }
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ if (kArchVariant != kMips64r6) {
+ divu(rs, scratch);
+ mfhi(rd);
+ } else {
+ modu(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Ddiv(Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ ddiv(rs, rt.rm());
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ ddiv(rs, scratch);
+ }
+}
+
+void TurboAssembler::Ddiv(Register rd, Register rs, const Operand& rt) {
+ if (kArchVariant != kMips64r6) {
+ if (rt.is_reg()) {
+ ddiv(rs, rt.rm());
+ mflo(rd);
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ ddiv(rs, scratch);
+ mflo(rd);
+ }
+ } else {
+ if (rt.is_reg()) {
+ ddiv(rd, rs, rt.rm());
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ ddiv(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Divu(Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ divu(rs, rt.rm());
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ divu(rs, scratch);
+ }
+}
+
+void TurboAssembler::Divu(Register res, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ if (kArchVariant != kMips64r6) {
+ divu(rs, rt.rm());
+ mflo(res);
+ } else {
+ divu(res, rs, rt.rm());
+ }
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ if (kArchVariant != kMips64r6) {
+ divu(rs, scratch);
+ mflo(res);
+ } else {
+ divu(res, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Ddivu(Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ ddivu(rs, rt.rm());
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ ddivu(rs, scratch);
+ }
+}
+
+void TurboAssembler::Ddivu(Register res, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ if (kArchVariant != kMips64r6) {
+ ddivu(rs, rt.rm());
+ mflo(res);
+ } else {
+ ddivu(res, rs, rt.rm());
+ }
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ if (kArchVariant != kMips64r6) {
+ ddivu(rs, scratch);
+ mflo(res);
+ } else {
+ ddivu(res, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Dmod(Register rd, Register rs, const Operand& rt) {
+ if (kArchVariant != kMips64r6) {
+ if (rt.is_reg()) {
+ ddiv(rs, rt.rm());
+ mfhi(rd);
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ ddiv(rs, scratch);
+ mfhi(rd);
+ }
+ } else {
+ if (rt.is_reg()) {
+ dmod(rd, rs, rt.rm());
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ dmod(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Dmodu(Register rd, Register rs, const Operand& rt) {
+ if (kArchVariant != kMips64r6) {
+ if (rt.is_reg()) {
+ ddivu(rs, rt.rm());
+ mfhi(rd);
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ ddivu(rs, scratch);
+ mfhi(rd);
+ }
+ } else {
+ if (rt.is_reg()) {
+ dmodu(rd, rs, rt.rm());
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ dmodu(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::And(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ and_(rd, rs, rt.rm());
+ } else {
+ if (is_uint16(rt.immediate()) && !MustUseReg(rt.rmode())) {
+ andi(rd, rs, static_cast<int32_t>(rt.immediate()));
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ and_(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Or(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ or_(rd, rs, rt.rm());
+ } else {
+ if (is_uint16(rt.immediate()) && !MustUseReg(rt.rmode())) {
+ ori(rd, rs, static_cast<int32_t>(rt.immediate()));
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ or_(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Xor(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ xor_(rd, rs, rt.rm());
+ } else {
+ if (is_uint16(rt.immediate()) && !MustUseReg(rt.rmode())) {
+ xori(rd, rs, static_cast<int32_t>(rt.immediate()));
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ xor_(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Nor(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ nor(rd, rs, rt.rm());
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ nor(rd, rs, scratch);
+ }
+}
+
+void TurboAssembler::Neg(Register rs, const Operand& rt) {
+ dsubu(rs, zero_reg, rt.rm());
+}
+
+void TurboAssembler::Slt(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ slt(rd, rs, rt.rm());
+ } else {
+ if (is_int16(rt.immediate()) && !MustUseReg(rt.rmode())) {
+ slti(rd, rs, static_cast<int32_t>(rt.immediate()));
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ slt(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Sltu(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ sltu(rd, rs, rt.rm());
+ } else {
+ const uint64_t int16_min = std::numeric_limits<int16_t>::min();
+ if (is_uint15(rt.immediate()) && !MustUseReg(rt.rmode())) {
+ // Imm range is: [0, 32767].
+ sltiu(rd, rs, static_cast<int32_t>(rt.immediate()));
+ } else if (is_uint15(rt.immediate() - int16_min) &&
+ !MustUseReg(rt.rmode())) {
+ // Imm range is: [max_unsigned-32767,max_unsigned].
+ sltiu(rd, rs, static_cast<uint16_t>(rt.immediate()));
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ sltu(rd, rs, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Sle(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ slt(rd, rt.rm(), rs);
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ slt(rd, scratch, rs);
+ }
+ xori(rd, rd, 1);
+}
+
+void TurboAssembler::Sleu(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ sltu(rd, rt.rm(), rs);
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ sltu(rd, scratch, rs);
+ }
+ xori(rd, rd, 1);
+}
+
+void TurboAssembler::Sge(Register rd, Register rs, const Operand& rt) {
+ Slt(rd, rs, rt);
+ xori(rd, rd, 1);
+}
+
+void TurboAssembler::Sgeu(Register rd, Register rs, const Operand& rt) {
+ Sltu(rd, rs, rt);
+ xori(rd, rd, 1);
+}
+
+void TurboAssembler::Sgt(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ slt(rd, rt.rm(), rs);
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ slt(rd, scratch, rs);
+ }
+}
+
+void TurboAssembler::Sgtu(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ sltu(rd, rt.rm(), rs);
+ } else {
+ // li handles the relocation.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ DCHECK(rs != scratch);
+ li(scratch, rt);
+ sltu(rd, scratch, rs);
+ }
+}
+
+void TurboAssembler::Ror(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ rotrv(rd, rs, rt.rm());
+ } else {
+ int64_t ror_value = rt.immediate() % 32;
+ if (ror_value < 0) {
+ ror_value += 32;
+ }
+ rotr(rd, rs, ror_value);
+ }
+}
+
+void TurboAssembler::Dror(Register rd, Register rs, const Operand& rt) {
+ if (rt.is_reg()) {
+ drotrv(rd, rs, rt.rm());
+ } else {
+ int64_t dror_value = rt.immediate() % 64;
+ if (dror_value < 0) dror_value += 64;
+ if (dror_value <= 31) {
+ drotr(rd, rs, dror_value);
+ } else {
+ drotr32(rd, rs, dror_value - 32);
+ }
+ }
+}
+
+void MacroAssembler::Pref(int32_t hint, const MemOperand& rs) {
+ pref(hint, rs);
+}
+
+void TurboAssembler::Lsa(Register rd, Register rt, Register rs, uint8_t sa,
+ Register scratch) {
+ DCHECK(sa >= 1 && sa <= 31);
+ if (kArchVariant == kMips64r6 && sa <= 4) {
+ lsa(rd, rt, rs, sa - 1);
+ } else {
+ Register tmp = rd == rt ? scratch : rd;
+ DCHECK(tmp != rt);
+ sll(tmp, rs, sa);
+ Addu(rd, rt, tmp);
+ }
+}
+
+void TurboAssembler::Dlsa(Register rd, Register rt, Register rs, uint8_t sa,
+ Register scratch) {
+ DCHECK(sa >= 1 && sa <= 31);
+ if (kArchVariant == kMips64r6 && sa <= 4) {
+ dlsa(rd, rt, rs, sa - 1);
+ } else {
+ Register tmp = rd == rt ? scratch : rd;
+ DCHECK(tmp != rt);
+ dsll(tmp, rs, sa);
+ Daddu(rd, rt, tmp);
+ }
+}
+
+void TurboAssembler::Bovc(Register rs, Register rt, Label* L) {
+ if (is_trampoline_emitted()) {
+ Label skip;
+ bnvc(rs, rt, &skip);
+ BranchLong(L, PROTECT);
+ bind(&skip);
+ } else {
+ bovc(rs, rt, L);
+ }
+}
+
+void TurboAssembler::Bnvc(Register rs, Register rt, Label* L) {
+ if (is_trampoline_emitted()) {
+ Label skip;
+ bovc(rs, rt, &skip);
+ BranchLong(L, PROTECT);
+ bind(&skip);
+ } else {
+ bnvc(rs, rt, L);
+ }
+}
+
+// ------------Pseudo-instructions-------------
+
+// Change endianness
+void TurboAssembler::ByteSwapSigned(Register dest, Register src,
+ int operand_size) {
+ DCHECK(operand_size == 2 || operand_size == 4 || operand_size == 8);
+ DCHECK(kArchVariant == kMips64r6 || kArchVariant == kMips64r2);
+ if (operand_size == 2) {
+ wsbh(dest, src);
+ seh(dest, dest);
+ } else if (operand_size == 4) {
+ wsbh(dest, src);
+ rotr(dest, dest, 16);
+ } else {
+ dsbh(dest, src);
+ dshd(dest, dest);
+ }
+}
+
+void TurboAssembler::ByteSwapUnsigned(Register dest, Register src,
+ int operand_size) {
+ DCHECK(operand_size == 2 || operand_size == 4);
+ if (operand_size == 2) {
+ wsbh(dest, src);
+ andi(dest, dest, 0xFFFF);
+ } else {
+ wsbh(dest, src);
+ rotr(dest, dest, 16);
+ dinsu_(dest, zero_reg, 32, 32);
+ }
+}
+
+void TurboAssembler::Ulw(Register rd, const MemOperand& rs) {
+ DCHECK(rd != at);
+ DCHECK(rs.rm() != at);
+ if (kArchVariant == kMips64r6) {
+ Lw(rd, rs);
+ } else {
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ DCHECK(kMipsLwrOffset <= 3 && kMipsLwlOffset <= 3);
+ MemOperand source = rs;
+ // Adjust offset for two accesses and check if offset + 3 fits into int16_t.
+ AdjustBaseAndOffset(&source, OffsetAccessType::TWO_ACCESSES, 3);
+ if (rd != source.rm()) {
+ lwr(rd, MemOperand(source.rm(), source.offset() + kMipsLwrOffset));
+ lwl(rd, MemOperand(source.rm(), source.offset() + kMipsLwlOffset));
+ } else {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ lwr(scratch, MemOperand(rs.rm(), rs.offset() + kMipsLwrOffset));
+ lwl(scratch, MemOperand(rs.rm(), rs.offset() + kMipsLwlOffset));
+ mov(rd, scratch);
+ }
+ }
+}
+
+void TurboAssembler::Ulwu(Register rd, const MemOperand& rs) {
+ if (kArchVariant == kMips64r6) {
+ Lwu(rd, rs);
+ } else {
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ Ulw(rd, rs);
+ Dext(rd, rd, 0, 32);
+ }
+}
+
+void TurboAssembler::Usw(Register rd, const MemOperand& rs) {
+ DCHECK(rd != at);
+ DCHECK(rs.rm() != at);
+ DCHECK(rd != rs.rm());
+ if (kArchVariant == kMips64r6) {
+ Sw(rd, rs);
+ } else {
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ DCHECK(kMipsSwrOffset <= 3 && kMipsSwlOffset <= 3);
+ MemOperand source = rs;
+ // Adjust offset for two accesses and check if offset + 3 fits into int16_t.
+ AdjustBaseAndOffset(&source, OffsetAccessType::TWO_ACCESSES, 3);
+ swr(rd, MemOperand(source.rm(), source.offset() + kMipsSwrOffset));
+ swl(rd, MemOperand(source.rm(), source.offset() + kMipsSwlOffset));
+ }
+}
+
+void TurboAssembler::Ulh(Register rd, const MemOperand& rs) {
+ DCHECK(rd != at);
+ DCHECK(rs.rm() != at);
+ if (kArchVariant == kMips64r6) {
+ Lh(rd, rs);
+ } else {
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ MemOperand source = rs;
+ // Adjust offset for two accesses and check if offset + 1 fits into int16_t.
+ AdjustBaseAndOffset(&source, OffsetAccessType::TWO_ACCESSES, 1);
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ if (source.rm() == scratch) {
+#if defined(V8_TARGET_LITTLE_ENDIAN)
+ Lb(rd, MemOperand(source.rm(), source.offset() + 1));
+ Lbu(scratch, source);
+#elif defined(V8_TARGET_BIG_ENDIAN)
+ Lb(rd, source);
+ Lbu(scratch, MemOperand(source.rm(), source.offset() + 1));
+#endif
+ } else {
+#if defined(V8_TARGET_LITTLE_ENDIAN)
+ Lbu(scratch, source);
+ Lb(rd, MemOperand(source.rm(), source.offset() + 1));
+#elif defined(V8_TARGET_BIG_ENDIAN)
+ Lbu(scratch, MemOperand(source.rm(), source.offset() + 1));
+ Lb(rd, source);
+#endif
+ }
+ dsll(rd, rd, 8);
+ or_(rd, rd, scratch);
+ }
+}
+
+void TurboAssembler::Ulhu(Register rd, const MemOperand& rs) {
+ DCHECK(rd != at);
+ DCHECK(rs.rm() != at);
+ if (kArchVariant == kMips64r6) {
+ Lhu(rd, rs);
+ } else {
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ MemOperand source = rs;
+ // Adjust offset for two accesses and check if offset + 1 fits into int16_t.
+ AdjustBaseAndOffset(&source, OffsetAccessType::TWO_ACCESSES, 1);
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ if (source.rm() == scratch) {
+#if defined(V8_TARGET_LITTLE_ENDIAN)
+ Lbu(rd, MemOperand(source.rm(), source.offset() + 1));
+ Lbu(scratch, source);
+#elif defined(V8_TARGET_BIG_ENDIAN)
+ Lbu(rd, source);
+ Lbu(scratch, MemOperand(source.rm(), source.offset() + 1));
+#endif
+ } else {
+#if defined(V8_TARGET_LITTLE_ENDIAN)
+ Lbu(scratch, source);
+ Lbu(rd, MemOperand(source.rm(), source.offset() + 1));
+#elif defined(V8_TARGET_BIG_ENDIAN)
+ Lbu(scratch, MemOperand(source.rm(), source.offset() + 1));
+ Lbu(rd, source);
+#endif
+ }
+ dsll(rd, rd, 8);
+ or_(rd, rd, scratch);
+ }
+}
+
+void TurboAssembler::Ush(Register rd, const MemOperand& rs, Register scratch) {
+ DCHECK(rd != at);
+ DCHECK(rs.rm() != at);
+ DCHECK(rs.rm() != scratch);
+ DCHECK(scratch != at);
+ if (kArchVariant == kMips64r6) {
+ Sh(rd, rs);
+ } else {
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ MemOperand source = rs;
+ // Adjust offset for two accesses and check if offset + 1 fits into int16_t.
+ AdjustBaseAndOffset(&source, OffsetAccessType::TWO_ACCESSES, 1);
+
+ if (scratch != rd) {
+ mov(scratch, rd);
+ }
+
+#if defined(V8_TARGET_LITTLE_ENDIAN)
+ Sb(scratch, source);
+ srl(scratch, scratch, 8);
+ Sb(scratch, MemOperand(source.rm(), source.offset() + 1));
+#elif defined(V8_TARGET_BIG_ENDIAN)
+ Sb(scratch, MemOperand(source.rm(), source.offset() + 1));
+ srl(scratch, scratch, 8);
+ Sb(scratch, source);
+#endif
+ }
+}
+
+void TurboAssembler::Uld(Register rd, const MemOperand& rs) {
+ DCHECK(rd != at);
+ DCHECK(rs.rm() != at);
+ if (kArchVariant == kMips64r6) {
+ Ld(rd, rs);
+ } else {
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ DCHECK(kMipsLdrOffset <= 7 && kMipsLdlOffset <= 7);
+ MemOperand source = rs;
+ // Adjust offset for two accesses and check if offset + 7 fits into int16_t.
+ AdjustBaseAndOffset(&source, OffsetAccessType::TWO_ACCESSES, 7);
+ if (rd != source.rm()) {
+ ldr(rd, MemOperand(source.rm(), source.offset() + kMipsLdrOffset));
+ ldl(rd, MemOperand(source.rm(), source.offset() + kMipsLdlOffset));
+ } else {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ ldr(scratch, MemOperand(rs.rm(), rs.offset() + kMipsLdrOffset));
+ ldl(scratch, MemOperand(rs.rm(), rs.offset() + kMipsLdlOffset));
+ mov(rd, scratch);
+ }
+ }
+}
+
+// Load consequent 32-bit word pair in 64-bit reg. and put first word in low
+// bits,
+// second word in high bits.
+void MacroAssembler::LoadWordPair(Register rd, const MemOperand& rs,
+ Register scratch) {
+ Lwu(rd, rs);
+ Lw(scratch, MemOperand(rs.rm(), rs.offset() + kPointerSize / 2));
+ dsll32(scratch, scratch, 0);
+ Daddu(rd, rd, scratch);
+}
+
+void TurboAssembler::Usd(Register rd, const MemOperand& rs) {
+ DCHECK(rd != at);
+ DCHECK(rs.rm() != at);
+ if (kArchVariant == kMips64r6) {
+ Sd(rd, rs);
+ } else {
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ DCHECK(kMipsSdrOffset <= 7 && kMipsSdlOffset <= 7);
+ MemOperand source = rs;
+ // Adjust offset for two accesses and check if offset + 7 fits into int16_t.
+ AdjustBaseAndOffset(&source, OffsetAccessType::TWO_ACCESSES, 7);
+ sdr(rd, MemOperand(source.rm(), source.offset() + kMipsSdrOffset));
+ sdl(rd, MemOperand(source.rm(), source.offset() + kMipsSdlOffset));
+ }
+}
+
+// Do 64-bit store as two consequent 32-bit stores to unaligned address.
+void MacroAssembler::StoreWordPair(Register rd, const MemOperand& rs,
+ Register scratch) {
+ Sw(rd, rs);
+ dsrl32(scratch, rd, 0);
+ Sw(scratch, MemOperand(rs.rm(), rs.offset() + kPointerSize / 2));
+}
+
+void TurboAssembler::Ulwc1(FPURegister fd, const MemOperand& rs,
+ Register scratch) {
+ if (kArchVariant == kMips64r6) {
+ Lwc1(fd, rs);
+ } else {
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ Ulw(scratch, rs);
+ mtc1(scratch, fd);
+ }
+}
+
+void TurboAssembler::Uswc1(FPURegister fd, const MemOperand& rs,
+ Register scratch) {
+ if (kArchVariant == kMips64r6) {
+ Swc1(fd, rs);
+ } else {
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ mfc1(scratch, fd);
+ Usw(scratch, rs);
+ }
+}
+
+void TurboAssembler::Uldc1(FPURegister fd, const MemOperand& rs,
+ Register scratch) {
+ DCHECK(scratch != at);
+ if (kArchVariant == kMips64r6) {
+ Ldc1(fd, rs);
+ } else {
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ Uld(scratch, rs);
+ dmtc1(scratch, fd);
+ }
+}
+
+void TurboAssembler::Usdc1(FPURegister fd, const MemOperand& rs,
+ Register scratch) {
+ DCHECK(scratch != at);
+ if (kArchVariant == kMips64r6) {
+ Sdc1(fd, rs);
+ } else {
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ dmfc1(scratch, fd);
+ Usd(scratch, rs);
+ }
+}
+
+void TurboAssembler::Lb(Register rd, const MemOperand& rs) {
+ MemOperand source = rs;
+ AdjustBaseAndOffset(&source);
+ lb(rd, source);
+}
+
+void TurboAssembler::Lbu(Register rd, const MemOperand& rs) {
+ MemOperand source = rs;
+ AdjustBaseAndOffset(&source);
+ lbu(rd, source);
+}
+
+void TurboAssembler::Sb(Register rd, const MemOperand& rs) {
+ MemOperand source = rs;
+ AdjustBaseAndOffset(&source);
+ sb(rd, source);
+}
+
+void TurboAssembler::Lh(Register rd, const MemOperand& rs) {
+ MemOperand source = rs;
+ AdjustBaseAndOffset(&source);
+ lh(rd, source);
+}
+
+void TurboAssembler::Lhu(Register rd, const MemOperand& rs) {
+ MemOperand source = rs;
+ AdjustBaseAndOffset(&source);
+ lhu(rd, source);
+}
+
+void TurboAssembler::Sh(Register rd, const MemOperand& rs) {
+ MemOperand source = rs;
+ AdjustBaseAndOffset(&source);
+ sh(rd, source);
+}
+
+void TurboAssembler::Lw(Register rd, const MemOperand& rs) {
+ MemOperand source = rs;
+ AdjustBaseAndOffset(&source);
+ lw(rd, source);
+}
+
+void TurboAssembler::Lwu(Register rd, const MemOperand& rs) {
+ MemOperand source = rs;
+ AdjustBaseAndOffset(&source);
+ lwu(rd, source);
+}
+
+void TurboAssembler::Sw(Register rd, const MemOperand& rs) {
+ MemOperand source = rs;
+ AdjustBaseAndOffset(&source);
+ sw(rd, source);
+}
+
+void TurboAssembler::Ld(Register rd, const MemOperand& rs) {
+ MemOperand source = rs;
+ AdjustBaseAndOffset(&source);
+ ld(rd, source);
+}
+
+void TurboAssembler::Sd(Register rd, const MemOperand& rs) {
+ MemOperand source = rs;
+ AdjustBaseAndOffset(&source);
+ sd(rd, source);
+}
+
+void TurboAssembler::Lwc1(FPURegister fd, const MemOperand& src) {
+ MemOperand tmp = src;
+ AdjustBaseAndOffset(&tmp);
+ lwc1(fd, tmp);
+}
+
+void TurboAssembler::Swc1(FPURegister fs, const MemOperand& src) {
+ MemOperand tmp = src;
+ AdjustBaseAndOffset(&tmp);
+ swc1(fs, tmp);
+}
+
+void TurboAssembler::Ldc1(FPURegister fd, const MemOperand& src) {
+ MemOperand tmp = src;
+ AdjustBaseAndOffset(&tmp);
+ ldc1(fd, tmp);
+}
+
+void TurboAssembler::Sdc1(FPURegister fs, const MemOperand& src) {
+ MemOperand tmp = src;
+ AdjustBaseAndOffset(&tmp);
+ sdc1(fs, tmp);
+}
+
+void TurboAssembler::Ll(Register rd, const MemOperand& rs) {
+ bool is_one_instruction = (kArchVariant == kMips64r6) ? is_int9(rs.offset())
+ : is_int16(rs.offset());
+ if (is_one_instruction) {
+ ll(rd, rs);
+ } else {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ li(scratch, rs.offset());
+ daddu(scratch, scratch, rs.rm());
+ ll(rd, MemOperand(scratch, 0));
+ }
+}
+
+void TurboAssembler::Lld(Register rd, const MemOperand& rs) {
+ bool is_one_instruction = (kArchVariant == kMips64r6) ? is_int9(rs.offset())
+ : is_int16(rs.offset());
+ if (is_one_instruction) {
+ lld(rd, rs);
+ } else {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ li(scratch, rs.offset());
+ daddu(scratch, scratch, rs.rm());
+ lld(rd, MemOperand(scratch, 0));
+ }
+}
+
+void TurboAssembler::Sc(Register rd, const MemOperand& rs) {
+ bool is_one_instruction = (kArchVariant == kMips64r6) ? is_int9(rs.offset())
+ : is_int16(rs.offset());
+ if (is_one_instruction) {
+ sc(rd, rs);
+ } else {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ li(scratch, rs.offset());
+ daddu(scratch, scratch, rs.rm());
+ sc(rd, MemOperand(scratch, 0));
+ }
+}
+
+void TurboAssembler::Scd(Register rd, const MemOperand& rs) {
+ bool is_one_instruction = (kArchVariant == kMips64r6) ? is_int9(rs.offset())
+ : is_int16(rs.offset());
+ if (is_one_instruction) {
+ scd(rd, rs);
+ } else {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ li(scratch, rs.offset());
+ daddu(scratch, scratch, rs.rm());
+ scd(rd, MemOperand(scratch, 0));
+ }
+}
+
+void TurboAssembler::li(Register dst, Handle<HeapObject> value, LiFlags mode) {
+ // TODO(jgruber,v8:8887): Also consider a root-relative load when generating
+ // non-isolate-independent code. In many cases it might be cheaper than
+ // embedding the relocatable value.
+ if (root_array_available_ && options().isolate_independent_code) {
+ IndirectLoadConstant(dst, value);
+ return;
+ }
+ li(dst, Operand(value), mode);
+}
+
+void TurboAssembler::li(Register dst, ExternalReference value, LiFlags mode) {
+ // TODO(jgruber,v8:8887): Also consider a root-relative load when generating
+ // non-isolate-independent code. In many cases it might be cheaper than
+ // embedding the relocatable value.
+ if (root_array_available_ && options().isolate_independent_code) {
+ IndirectLoadExternalReference(dst, value);
+ return;
+ }
+ li(dst, Operand(value), mode);
+}
+
+void TurboAssembler::li(Register dst, const StringConstantBase* string,
+ LiFlags mode) {
+ li(dst, Operand::EmbeddedStringConstant(string), mode);
+}
+
+static inline int InstrCountForLiLower32Bit(int64_t value) {
+ if (!is_int16(static_cast<int32_t>(value)) && (value & kUpper16MaskOf64) &&
+ (value & kImm16Mask)) {
+ return 2;
+ } else {
+ return 1;
+ }
+}
+
+void TurboAssembler::LiLower32BitHelper(Register rd, Operand j) {
+ if (is_int16(static_cast<int32_t>(j.immediate()))) {
+ daddiu(rd, zero_reg, (j.immediate() & kImm16Mask));
+ } else if (!(j.immediate() & kUpper16MaskOf64)) {
+ ori(rd, zero_reg, j.immediate() & kImm16Mask);
+ } else {
+ lui(rd, j.immediate() >> kLuiShift & kImm16Mask);
+ if (j.immediate() & kImm16Mask) {
+ ori(rd, rd, j.immediate() & kImm16Mask);
+ }
+ }
+}
+
+static inline int InstrCountForLoadReplicatedConst32(int64_t value) {
+ uint32_t x = static_cast<uint32_t>(value);
+ uint32_t y = static_cast<uint32_t>(value >> 32);
+
+ if (x == y) {
+ return (is_uint16(x) || is_int16(x) || (x & kImm16Mask) == 0) ? 2 : 3;
+ }
+
+ return INT_MAX;
+}
+
+int TurboAssembler::InstrCountForLi64Bit(int64_t value) {
+ if (is_int32(value)) {
+ return InstrCountForLiLower32Bit(value);
+ } else {
+ int bit31 = value >> 31 & 0x1;
+ if ((value & kUpper16MaskOf64) == 0 && is_int16(value >> 32) &&
+ kArchVariant == kMips64r6) {
+ return 2;
+ } else if ((value & (kHigher16MaskOf64 | kUpper16MaskOf64)) == 0 &&
+ kArchVariant == kMips64r6) {
+ return 2;
+ } else if ((value & kImm16Mask) == 0 && is_int16((value >> 32) + bit31) &&
+ kArchVariant == kMips64r6) {
+ return 2;
+ } else if ((value & kImm16Mask) == 0 &&
+ ((value >> 31) & 0x1FFFF) == ((0x20000 - bit31) & 0x1FFFF) &&
+ kArchVariant == kMips64r6) {
+ return 2;
+ } else if (is_int16(static_cast<int32_t>(value)) &&
+ is_int16((value >> 32) + bit31) && kArchVariant == kMips64r6) {
+ return 2;
+ } else if (is_int16(static_cast<int32_t>(value)) &&
+ ((value >> 31) & 0x1FFFF) == ((0x20000 - bit31) & 0x1FFFF) &&
+ kArchVariant == kMips64r6) {
+ return 2;
+ } else if (base::bits::IsPowerOfTwo(value + 1) ||
+ value == std::numeric_limits<int64_t>::max()) {
+ return 2;
+ } else {
+ int shift_cnt = base::bits::CountTrailingZeros64(value);
+ int rep32_count = InstrCountForLoadReplicatedConst32(value);
+ int64_t tmp = value >> shift_cnt;
+ if (is_uint16(tmp)) {
+ return 2;
+ } else if (is_int16(tmp)) {
+ return 2;
+ } else if (rep32_count < 3) {
+ return 2;
+ } else if (is_int32(tmp)) {
+ return 3;
+ } else {
+ shift_cnt = 16 + base::bits::CountTrailingZeros64(value >> 16);
+ tmp = value >> shift_cnt;
+ if (is_uint16(tmp)) {
+ return 3;
+ } else if (is_int16(tmp)) {
+ return 3;
+ } else if (rep32_count < 4) {
+ return 3;
+ } else if (kArchVariant == kMips64r6) {
+ int64_t imm = value;
+ int count = InstrCountForLiLower32Bit(imm);
+ imm = (imm >> 32) + bit31;
+ if (imm & kImm16Mask) {
+ count++;
+ }
+ imm = (imm >> 16) + (imm >> 15 & 0x1);
+ if (imm & kImm16Mask) {
+ count++;
+ }
+ return count;
+ } else {
+ if (is_int48(value)) {
+ int64_t k = value >> 16;
+ int count = InstrCountForLiLower32Bit(k) + 1;
+ if (value & kImm16Mask) {
+ count++;
+ }
+ return count;
+ } else {
+ int64_t k = value >> 32;
+ int count = InstrCountForLiLower32Bit(k);
+ if ((value >> 16) & kImm16Mask) {
+ count += 3;
+ if (value & kImm16Mask) {
+ count++;
+ }
+ } else {
+ count++;
+ if (value & kImm16Mask) {
+ count++;
+ }
+ }
+ return count;
+ }
+ }
+ }
+ }
+ }
+ UNREACHABLE();
+ return INT_MAX;
+}
+
+// All changes to if...else conditions here must be added to
+// InstrCountForLi64Bit as well.
+void TurboAssembler::li_optimized(Register rd, Operand j, LiFlags mode) {
+ DCHECK(!j.is_reg());
+ DCHECK(!MustUseReg(j.rmode()));
+ DCHECK(mode == OPTIMIZE_SIZE);
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ // Normal load of an immediate value which does not need Relocation Info.
+ if (is_int32(j.immediate())) {
+ LiLower32BitHelper(rd, j);
+ } else {
+ int bit31 = j.immediate() >> 31 & 0x1;
+ if ((j.immediate() & kUpper16MaskOf64) == 0 &&
+ is_int16(j.immediate() >> 32) && kArchVariant == kMips64r6) {
+ // 64-bit value which consists of an unsigned 16-bit value in its
+ // least significant 32-bits, and a signed 16-bit value in its
+ // most significant 32-bits.
+ ori(rd, zero_reg, j.immediate() & kImm16Mask);
+ dahi(rd, j.immediate() >> 32 & kImm16Mask);
+ } else if ((j.immediate() & (kHigher16MaskOf64 | kUpper16MaskOf64)) == 0 &&
+ kArchVariant == kMips64r6) {
+ // 64-bit value which consists of an unsigned 16-bit value in its
+ // least significant 48-bits, and a signed 16-bit value in its
+ // most significant 16-bits.
+ ori(rd, zero_reg, j.immediate() & kImm16Mask);
+ dati(rd, j.immediate() >> 48 & kImm16Mask);
+ } else if ((j.immediate() & kImm16Mask) == 0 &&
+ is_int16((j.immediate() >> 32) + bit31) &&
+ kArchVariant == kMips64r6) {
+ // 16 LSBs (Least Significant Bits) all set to zero.
+ // 48 MSBs (Most Significant Bits) hold a signed 32-bit value.
+ lui(rd, j.immediate() >> kLuiShift & kImm16Mask);
+ dahi(rd, ((j.immediate() >> 32) + bit31) & kImm16Mask);
+ } else if ((j.immediate() & kImm16Mask) == 0 &&
+ ((j.immediate() >> 31) & 0x1FFFF) ==
+ ((0x20000 - bit31) & 0x1FFFF) &&
+ kArchVariant == kMips64r6) {
+ // 16 LSBs all set to zero.
+ // 48 MSBs hold a signed value which can't be represented by signed
+ // 32-bit number, and the middle 16 bits are all zero, or all one.
+ lui(rd, j.immediate() >> kLuiShift & kImm16Mask);
+ dati(rd, ((j.immediate() >> 48) + bit31) & kImm16Mask);
+ } else if (is_int16(static_cast<int32_t>(j.immediate())) &&
+ is_int16((j.immediate() >> 32) + bit31) &&
+ kArchVariant == kMips64r6) {
+ // 32 LSBs contain a signed 16-bit number.
+ // 32 MSBs contain a signed 16-bit number.
+ daddiu(rd, zero_reg, j.immediate() & kImm16Mask);
+ dahi(rd, ((j.immediate() >> 32) + bit31) & kImm16Mask);
+ } else if (is_int16(static_cast<int32_t>(j.immediate())) &&
+ ((j.immediate() >> 31) & 0x1FFFF) ==
+ ((0x20000 - bit31) & 0x1FFFF) &&
+ kArchVariant == kMips64r6) {
+ // 48 LSBs contain an unsigned 16-bit number.
+ // 16 MSBs contain a signed 16-bit number.
+ daddiu(rd, zero_reg, j.immediate() & kImm16Mask);
+ dati(rd, ((j.immediate() >> 48) + bit31) & kImm16Mask);
+ } else if (base::bits::IsPowerOfTwo(j.immediate() + 1) ||
+ j.immediate() == std::numeric_limits<int64_t>::max()) {
+ // 64-bit values which have their "n" LSBs set to one, and their
+ // "64-n" MSBs set to zero. "n" must meet the restrictions 0 < n < 64.
+ int shift_cnt = 64 - base::bits::CountTrailingZeros64(j.immediate() + 1);
+ daddiu(rd, zero_reg, -1);
+ if (shift_cnt < 32) {
+ dsrl(rd, rd, shift_cnt);
+ } else {
+ dsrl32(rd, rd, shift_cnt & 31);
+ }
+ } else {
+ int shift_cnt = base::bits::CountTrailingZeros64(j.immediate());
+ int rep32_count = InstrCountForLoadReplicatedConst32(j.immediate());
+ int64_t tmp = j.immediate() >> shift_cnt;
+ if (is_uint16(tmp)) {
+ // Value can be computed by loading a 16-bit unsigned value, and
+ // then shifting left.
+ ori(rd, zero_reg, tmp & kImm16Mask);
+ if (shift_cnt < 32) {
+ dsll(rd, rd, shift_cnt);
+ } else {
+ dsll32(rd, rd, shift_cnt & 31);
+ }
+ } else if (is_int16(tmp)) {
+ // Value can be computed by loading a 16-bit signed value, and
+ // then shifting left.
+ daddiu(rd, zero_reg, static_cast<int32_t>(tmp));
+ if (shift_cnt < 32) {
+ dsll(rd, rd, shift_cnt);
+ } else {
+ dsll32(rd, rd, shift_cnt & 31);
+ }
+ } else if (rep32_count < 3) {
+ // Value being loaded has 32 LSBs equal to the 32 MSBs, and the
+ // value loaded into the 32 LSBs can be loaded with a single
+ // MIPS instruction.
+ LiLower32BitHelper(rd, j);
+ Dins(rd, rd, 32, 32);
+ } else if (is_int32(tmp)) {
+ // Loads with 3 instructions.
+ // Value can be computed by loading a 32-bit signed value, and
+ // then shifting left.
+ lui(rd, tmp >> kLuiShift & kImm16Mask);
+ ori(rd, rd, tmp & kImm16Mask);
+ if (shift_cnt < 32) {
+ dsll(rd, rd, shift_cnt);
+ } else {
+ dsll32(rd, rd, shift_cnt & 31);
+ }
+ } else {
+ shift_cnt = 16 + base::bits::CountTrailingZeros64(j.immediate() >> 16);
+ tmp = j.immediate() >> shift_cnt;
+ if (is_uint16(tmp)) {
+ // Value can be computed by loading a 16-bit unsigned value,
+ // shifting left, and "or"ing in another 16-bit unsigned value.
+ ori(rd, zero_reg, tmp & kImm16Mask);
+ if (shift_cnt < 32) {
+ dsll(rd, rd, shift_cnt);
+ } else {
+ dsll32(rd, rd, shift_cnt & 31);
+ }
+ ori(rd, rd, j.immediate() & kImm16Mask);
+ } else if (is_int16(tmp)) {
+ // Value can be computed by loading a 16-bit signed value,
+ // shifting left, and "or"ing in a 16-bit unsigned value.
+ daddiu(rd, zero_reg, static_cast<int32_t>(tmp));
+ if (shift_cnt < 32) {
+ dsll(rd, rd, shift_cnt);
+ } else {
+ dsll32(rd, rd, shift_cnt & 31);
+ }
+ ori(rd, rd, j.immediate() & kImm16Mask);
+ } else if (rep32_count < 4) {
+ // Value being loaded has 32 LSBs equal to the 32 MSBs, and the
+ // value in the 32 LSBs requires 2 MIPS instructions to load.
+ LiLower32BitHelper(rd, j);
+ Dins(rd, rd, 32, 32);
+ } else if (kArchVariant == kMips64r6) {
+ // Loads with 3-4 instructions.
+ // Catch-all case to get any other 64-bit values which aren't
+ // handled by special cases above.
+ int64_t imm = j.immediate();
+ LiLower32BitHelper(rd, j);
+ imm = (imm >> 32) + bit31;
+ if (imm & kImm16Mask) {
+ dahi(rd, imm & kImm16Mask);
+ }
+ imm = (imm >> 16) + (imm >> 15 & 0x1);
+ if (imm & kImm16Mask) {
+ dati(rd, imm & kImm16Mask);
+ }
+ } else {
+ if (is_int48(j.immediate())) {
+ Operand k = Operand(j.immediate() >> 16);
+ LiLower32BitHelper(rd, k);
+ dsll(rd, rd, 16);
+ if (j.immediate() & kImm16Mask) {
+ ori(rd, rd, j.immediate() & kImm16Mask);
+ }
+ } else {
+ Operand k = Operand(j.immediate() >> 32);
+ LiLower32BitHelper(rd, k);
+ if ((j.immediate() >> 16) & kImm16Mask) {
+ dsll(rd, rd, 16);
+ ori(rd, rd, (j.immediate() >> 16) & kImm16Mask);
+ dsll(rd, rd, 16);
+ if (j.immediate() & kImm16Mask) {
+ ori(rd, rd, j.immediate() & kImm16Mask);
+ }
+ } else {
+ dsll32(rd, rd, 0);
+ if (j.immediate() & kImm16Mask) {
+ ori(rd, rd, j.immediate() & kImm16Mask);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+void TurboAssembler::li(Register rd, Operand j, LiFlags mode) {
+ DCHECK(!j.is_reg());
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ if (!MustUseReg(j.rmode()) && mode == OPTIMIZE_SIZE) {
+ int li_count = InstrCountForLi64Bit(j.immediate());
+ int li_neg_count = InstrCountForLi64Bit(-j.immediate());
+ int li_not_count = InstrCountForLi64Bit(~j.immediate());
+ // Loading -MIN_INT64 could cause problems, but loading MIN_INT64 takes only
+ // two instructions so no need to check for this.
+ if (li_neg_count <= li_not_count && li_neg_count < li_count - 1) {
+ DCHECK(j.immediate() != std::numeric_limits<int64_t>::min());
+ li_optimized(rd, Operand(-j.immediate()), mode);
+ Dsubu(rd, zero_reg, rd);
+ } else if (li_neg_count > li_not_count && li_not_count < li_count - 1) {
+ DCHECK(j.immediate() != std::numeric_limits<int64_t>::min());
+ li_optimized(rd, Operand(~j.immediate()), mode);
+ nor(rd, rd, rd);
+ } else {
+ li_optimized(rd, j, mode);
+ }
+ } else if (MustUseReg(j.rmode())) {
+ int64_t immediate;
+ if (j.IsHeapObjectRequest()) {
+ RequestHeapObject(j.heap_object_request());
+ immediate = 0;
+ } else {
+ immediate = j.immediate();
+ }
+
+ RecordRelocInfo(j.rmode(), immediate);
+ lui(rd, (immediate >> 32) & kImm16Mask);
+ ori(rd, rd, (immediate >> 16) & kImm16Mask);
+ dsll(rd, rd, 16);
+ ori(rd, rd, immediate & kImm16Mask);
+ } else if (mode == ADDRESS_LOAD) {
+ // We always need the same number of instructions as we may need to patch
+ // this code to load another value which may need all 4 instructions.
+ lui(rd, (j.immediate() >> 32) & kImm16Mask);
+ ori(rd, rd, (j.immediate() >> 16) & kImm16Mask);
+ dsll(rd, rd, 16);
+ ori(rd, rd, j.immediate() & kImm16Mask);
+ } else { // mode == CONSTANT_SIZE - always emit the same instruction
+ // sequence.
+ if (kArchVariant == kMips64r6) {
+ int64_t imm = j.immediate();
+ lui(rd, imm >> kLuiShift & kImm16Mask);
+ ori(rd, rd, (imm & kImm16Mask));
+ imm = (imm >> 32) + ((imm >> 31) & 0x1);
+ dahi(rd, imm & kImm16Mask & kImm16Mask);
+ imm = (imm >> 16) + ((imm >> 15) & 0x1);
+ dati(rd, imm & kImm16Mask & kImm16Mask);
+ } else {
+ lui(rd, (j.immediate() >> 48) & kImm16Mask);
+ ori(rd, rd, (j.immediate() >> 32) & kImm16Mask);
+ dsll(rd, rd, 16);
+ ori(rd, rd, (j.immediate() >> 16) & kImm16Mask);
+ dsll(rd, rd, 16);
+ ori(rd, rd, j.immediate() & kImm16Mask);
+ }
+ }
+}
+
+void TurboAssembler::MultiPush(RegList regs) {
+ int16_t num_to_push = base::bits::CountPopulation(regs);
+ int16_t stack_offset = num_to_push * kPointerSize;
+
+ Dsubu(sp, sp, Operand(stack_offset));
+ for (int16_t i = kNumRegisters - 1; i >= 0; i--) {
+ if ((regs & (1 << i)) != 0) {
+ stack_offset -= kPointerSize;
+ Sd(ToRegister(i), MemOperand(sp, stack_offset));
+ }
+ }
+}
+
+void TurboAssembler::MultiPop(RegList regs) {
+ int16_t stack_offset = 0;
+
+ for (int16_t i = 0; i < kNumRegisters; i++) {
+ if ((regs & (1 << i)) != 0) {
+ Ld(ToRegister(i), MemOperand(sp, stack_offset));
+ stack_offset += kPointerSize;
+ }
+ }
+ daddiu(sp, sp, stack_offset);
+}
+
+void TurboAssembler::MultiPushFPU(RegList regs) {
+ int16_t num_to_push = base::bits::CountPopulation(regs);
+ int16_t stack_offset = num_to_push * kDoubleSize;
+
+ Dsubu(sp, sp, Operand(stack_offset));
+ for (int16_t i = kNumRegisters - 1; i >= 0; i--) {
+ if ((regs & (1 << i)) != 0) {
+ stack_offset -= kDoubleSize;
+ Sdc1(FPURegister::from_code(i), MemOperand(sp, stack_offset));
+ }
+ }
+}
+
+void TurboAssembler::MultiPopFPU(RegList regs) {
+ int16_t stack_offset = 0;
+
+ for (int16_t i = 0; i < kNumRegisters; i++) {
+ if ((regs & (1 << i)) != 0) {
+ Ldc1(FPURegister::from_code(i), MemOperand(sp, stack_offset));
+ stack_offset += kDoubleSize;
+ }
+ }
+ daddiu(sp, sp, stack_offset);
+}
+
+void TurboAssembler::Ext(Register rt, Register rs, uint16_t pos,
+ uint16_t size) {
+ DCHECK_LT(pos, 32);
+ DCHECK_LT(pos + size, 33);
+ ext_(rt, rs, pos, size);
+}
+
+void TurboAssembler::Dext(Register rt, Register rs, uint16_t pos,
+ uint16_t size) {
+ DCHECK(pos < 64 && 0 < size && size <= 64 && 0 < pos + size &&
+ pos + size <= 64);
+ if (size > 32) {
+ dextm_(rt, rs, pos, size);
+ } else if (pos >= 32) {
+ dextu_(rt, rs, pos, size);
+ } else {
+ dext_(rt, rs, pos, size);
+ }
+}
+
+void TurboAssembler::Ins(Register rt, Register rs, uint16_t pos,
+ uint16_t size) {
+ DCHECK_LT(pos, 32);
+ DCHECK_LE(pos + size, 32);
+ DCHECK_NE(size, 0);
+ ins_(rt, rs, pos, size);
+}
+
+void TurboAssembler::Dins(Register rt, Register rs, uint16_t pos,
+ uint16_t size) {
+ DCHECK(pos < 64 && 0 < size && size <= 64 && 0 < pos + size &&
+ pos + size <= 64);
+ if (pos + size <= 32) {
+ dins_(rt, rs, pos, size);
+ } else if (pos < 32) {
+ dinsm_(rt, rs, pos, size);
+ } else {
+ dinsu_(rt, rs, pos, size);
+ }
+}
+
+void TurboAssembler::ExtractBits(Register dest, Register source, Register pos,
+ int size, bool sign_extend) {
+ dsrav(dest, source, pos);
+ Dext(dest, dest, 0, size);
+ if (sign_extend) {
+ switch (size) {
+ case 8:
+ seb(dest, dest);
+ break;
+ case 16:
+ seh(dest, dest);
+ break;
+ case 32:
+ // sign-extend word
+ sll(dest, dest, 0);
+ break;
+ default:
+ UNREACHABLE();
+ }
+ }
+}
+
+void TurboAssembler::InsertBits(Register dest, Register source, Register pos,
+ int size) {
+ Dror(dest, dest, pos);
+ Dins(dest, source, 0, size);
+ {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ Dsubu(scratch, zero_reg, pos);
+ Dror(dest, dest, scratch);
+ }
+}
+
+void TurboAssembler::Neg_s(FPURegister fd, FPURegister fs) {
+ if (kArchVariant == kMips64r6) {
+ // r6 neg_s changes the sign for NaN-like operands as well.
+ neg_s(fd, fs);
+ } else {
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Label is_nan, done;
+ Register scratch1 = t8;
+ Register scratch2 = t9;
+ CompareIsNanF32(fs, fs);
+ BranchTrueShortF(&is_nan);
+ Branch(USE_DELAY_SLOT, &done);
+ // For NaN input, neg_s will return the same NaN value,
+ // while the sign has to be changed separately.
+ neg_s(fd, fs); // In delay slot.
+ bind(&is_nan);
+ mfc1(scratch1, fs);
+ li(scratch2, kBinary32SignMask);
+ Xor(scratch1, scratch1, scratch2);
+ mtc1(scratch1, fd);
+ bind(&done);
+ }
+}
+
+void TurboAssembler::Neg_d(FPURegister fd, FPURegister fs) {
+ if (kArchVariant == kMips64r6) {
+ // r6 neg_d changes the sign for NaN-like operands as well.
+ neg_d(fd, fs);
+ } else {
+ DCHECK_EQ(kArchVariant, kMips64r2);
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Label is_nan, done;
+ Register scratch1 = t8;
+ Register scratch2 = t9;
+ CompareIsNanF64(fs, fs);
+ BranchTrueShortF(&is_nan);
+ Branch(USE_DELAY_SLOT, &done);
+ // For NaN input, neg_d will return the same NaN value,
+ // while the sign has to be changed separately.
+ neg_d(fd, fs); // In delay slot.
+ bind(&is_nan);
+ dmfc1(scratch1, fs);
+ li(scratch2, Double::kSignMask);
+ Xor(scratch1, scratch1, scratch2);
+ dmtc1(scratch1, fd);
+ bind(&done);
+ }
+}
+
+void TurboAssembler::Cvt_d_uw(FPURegister fd, FPURegister fs) {
+ // Move the data from fs to t8.
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ mfc1(t8, fs);
+ Cvt_d_uw(fd, t8);
+}
+
+void TurboAssembler::Cvt_d_uw(FPURegister fd, Register rs) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+
+ // Convert rs to a FP value in fd.
+ DCHECK(rs != t9);
+ DCHECK(rs != at);
+
+ // Zero extend int32 in rs.
+ Dext(t9, rs, 0, 32);
+ dmtc1(t9, fd);
+ cvt_d_l(fd, fd);
+}
+
+void TurboAssembler::Cvt_d_ul(FPURegister fd, FPURegister fs) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ // Move the data from fs to t8.
+ dmfc1(t8, fs);
+ Cvt_d_ul(fd, t8);
+}
+
+void TurboAssembler::Cvt_d_ul(FPURegister fd, Register rs) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ // Convert rs to a FP value in fd.
+
+ DCHECK(rs != t9);
+ DCHECK(rs != at);
+
+ Label msb_clear, conversion_done;
+
+ Branch(&msb_clear, ge, rs, Operand(zero_reg));
+
+ // Rs >= 2^63
+ andi(t9, rs, 1);
+ dsrl(rs, rs, 1);
+ or_(t9, t9, rs);
+ dmtc1(t9, fd);
+ cvt_d_l(fd, fd);
+ Branch(USE_DELAY_SLOT, &conversion_done);
+ add_d(fd, fd, fd); // In delay slot.
+
+ bind(&msb_clear);
+ // Rs < 2^63, we can do simple conversion.
+ dmtc1(rs, fd);
+ cvt_d_l(fd, fd);
+
+ bind(&conversion_done);
+}
+
+void TurboAssembler::Cvt_s_uw(FPURegister fd, FPURegister fs) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ // Move the data from fs to t8.
+ mfc1(t8, fs);
+ Cvt_s_uw(fd, t8);
+}
+
+void TurboAssembler::Cvt_s_uw(FPURegister fd, Register rs) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ // Convert rs to a FP value in fd.
+ DCHECK(rs != t9);
+ DCHECK(rs != at);
+
+ // Zero extend int32 in rs.
+ Dext(t9, rs, 0, 32);
+ dmtc1(t9, fd);
+ cvt_s_l(fd, fd);
+}
+
+void TurboAssembler::Cvt_s_ul(FPURegister fd, FPURegister fs) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ // Move the data from fs to t8.
+ dmfc1(t8, fs);
+ Cvt_s_ul(fd, t8);
+}
+
+void TurboAssembler::Cvt_s_ul(FPURegister fd, Register rs) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ // Convert rs to a FP value in fd.
+
+ DCHECK(rs != t9);
+ DCHECK(rs != at);
+
+ Label positive, conversion_done;
+
+ Branch(&positive, ge, rs, Operand(zero_reg));
+
+ // Rs >= 2^31.
+ andi(t9, rs, 1);
+ dsrl(rs, rs, 1);
+ or_(t9, t9, rs);
+ dmtc1(t9, fd);
+ cvt_s_l(fd, fd);
+ Branch(USE_DELAY_SLOT, &conversion_done);
+ add_s(fd, fd, fd); // In delay slot.
+
+ bind(&positive);
+ // Rs < 2^31, we can do simple conversion.
+ dmtc1(rs, fd);
+ cvt_s_l(fd, fd);
+
+ bind(&conversion_done);
+}
+
+void MacroAssembler::Round_l_d(FPURegister fd, FPURegister fs) {
+ round_l_d(fd, fs);
+}
+
+void MacroAssembler::Floor_l_d(FPURegister fd, FPURegister fs) {
+ floor_l_d(fd, fs);
+}
+
+void MacroAssembler::Ceil_l_d(FPURegister fd, FPURegister fs) {
+ ceil_l_d(fd, fs);
+}
+
+void MacroAssembler::Trunc_l_d(FPURegister fd, FPURegister fs) {
+ trunc_l_d(fd, fs);
+}
+
+void MacroAssembler::Trunc_l_ud(FPURegister fd, FPURegister fs,
+ FPURegister scratch) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ // Load to GPR.
+ dmfc1(t8, fs);
+ // Reset sign bit.
+ {
+ UseScratchRegisterScope temps(this);
+ Register scratch1 = temps.Acquire();
+ li(scratch1, 0x7FFFFFFFFFFFFFFF);
+ and_(t8, t8, scratch1);
+ }
+ dmtc1(t8, fs);
+ trunc_l_d(fd, fs);
+}
+
+void TurboAssembler::Trunc_uw_d(FPURegister fd, FPURegister fs,
+ FPURegister scratch) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Trunc_uw_d(t8, fs, scratch);
+ mtc1(t8, fd);
+}
+
+void TurboAssembler::Trunc_uw_s(FPURegister fd, FPURegister fs,
+ FPURegister scratch) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Trunc_uw_s(t8, fs, scratch);
+ mtc1(t8, fd);
+}
+
+void TurboAssembler::Trunc_ul_d(FPURegister fd, FPURegister fs,
+ FPURegister scratch, Register result) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Trunc_ul_d(t8, fs, scratch, result);
+ dmtc1(t8, fd);
+}
+
+void TurboAssembler::Trunc_ul_s(FPURegister fd, FPURegister fs,
+ FPURegister scratch, Register result) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Trunc_ul_s(t8, fs, scratch, result);
+ dmtc1(t8, fd);
+}
+
+void MacroAssembler::Trunc_w_d(FPURegister fd, FPURegister fs) {
+ trunc_w_d(fd, fs);
+}
+
+void MacroAssembler::Round_w_d(FPURegister fd, FPURegister fs) {
+ round_w_d(fd, fs);
+}
+
+void MacroAssembler::Floor_w_d(FPURegister fd, FPURegister fs) {
+ floor_w_d(fd, fs);
+}
+
+void MacroAssembler::Ceil_w_d(FPURegister fd, FPURegister fs) {
+ ceil_w_d(fd, fs);
+}
+
+void TurboAssembler::Trunc_uw_d(Register rd, FPURegister fs,
+ FPURegister scratch) {
+ DCHECK(fs != scratch);
+ DCHECK(rd != at);
+
+ {
+ // Load 2^31 into scratch as its float representation.
+ UseScratchRegisterScope temps(this);
+ Register scratch1 = temps.Acquire();
+ li(scratch1, 0x41E00000);
+ mtc1(zero_reg, scratch);
+ mthc1(scratch1, scratch);
+ }
+ // Test if scratch > fd.
+ // If fd < 2^31 we can convert it normally.
+ Label simple_convert;
+ CompareF64(OLT, fs, scratch);
+ BranchTrueShortF(&simple_convert);
+
+ // First we subtract 2^31 from fd, then trunc it to rs
+ // and add 2^31 to rs.
+ sub_d(scratch, fs, scratch);
+ trunc_w_d(scratch, scratch);
+ mfc1(rd, scratch);
+ Or(rd, rd, 1 << 31);
+
+ Label done;
+ Branch(&done);
+ // Simple conversion.
+ bind(&simple_convert);
+ trunc_w_d(scratch, fs);
+ mfc1(rd, scratch);
+
+ bind(&done);
+}
+
+void TurboAssembler::Trunc_uw_s(Register rd, FPURegister fs,
+ FPURegister scratch) {
+ DCHECK(fs != scratch);
+ DCHECK(rd != at);
+
+ {
+ // Load 2^31 into scratch as its float representation.
+ UseScratchRegisterScope temps(this);
+ Register scratch1 = temps.Acquire();
+ li(scratch1, 0x4F000000);
+ mtc1(scratch1, scratch);
+ }
+ // Test if scratch > fs.
+ // If fs < 2^31 we can convert it normally.
+ Label simple_convert;
+ CompareF32(OLT, fs, scratch);
+ BranchTrueShortF(&simple_convert);
+
+ // First we subtract 2^31 from fs, then trunc it to rd
+ // and add 2^31 to rd.
+ sub_s(scratch, fs, scratch);
+ trunc_w_s(scratch, scratch);
+ mfc1(rd, scratch);
+ Or(rd, rd, 1 << 31);
+
+ Label done;
+ Branch(&done);
+ // Simple conversion.
+ bind(&simple_convert);
+ trunc_w_s(scratch, fs);
+ mfc1(rd, scratch);
+
+ bind(&done);
+}
+
+void TurboAssembler::Trunc_ul_d(Register rd, FPURegister fs,
+ FPURegister scratch, Register result) {
+ DCHECK(fs != scratch);
+ DCHECK(result.is_valid() ? !AreAliased(rd, result, at) : !AreAliased(rd, at));
+
+ Label simple_convert, done, fail;
+ if (result.is_valid()) {
+ mov(result, zero_reg);
+ Move(scratch, -1.0);
+ // If fd =< -1 or unordered, then the conversion fails.
+ CompareF64(OLE, fs, scratch);
+ BranchTrueShortF(&fail);
+ CompareIsNanF64(fs, scratch);
+ BranchTrueShortF(&fail);
+ }
+
+ // Load 2^63 into scratch as its double representation.
+ li(at, 0x43E0000000000000);
+ dmtc1(at, scratch);
+
+ // Test if scratch > fs.
+ // If fs < 2^63 we can convert it normally.
+ CompareF64(OLT, fs, scratch);
+ BranchTrueShortF(&simple_convert);
+
+ // First we subtract 2^63 from fs, then trunc it to rd
+ // and add 2^63 to rd.
+ sub_d(scratch, fs, scratch);
+ trunc_l_d(scratch, scratch);
+ dmfc1(rd, scratch);
+ Or(rd, rd, Operand(1UL << 63));
+ Branch(&done);
+
+ // Simple conversion.
+ bind(&simple_convert);
+ trunc_l_d(scratch, fs);
+ dmfc1(rd, scratch);
+
+ bind(&done);
+ if (result.is_valid()) {
+ // Conversion is failed if the result is negative.
+ {
+ UseScratchRegisterScope temps(this);
+ Register scratch1 = temps.Acquire();
+ addiu(scratch1, zero_reg, -1);
+ dsrl(scratch1, scratch1, 1); // Load 2^62.
+ dmfc1(result, scratch);
+ xor_(result, result, scratch1);
+ }
+ Slt(result, zero_reg, result);
+ }
+
+ bind(&fail);
+}
+
+void TurboAssembler::Trunc_ul_s(Register rd, FPURegister fs,
+ FPURegister scratch, Register result) {
+ DCHECK(fs != scratch);
+ DCHECK(result.is_valid() ? !AreAliased(rd, result, at) : !AreAliased(rd, at));
+
+ Label simple_convert, done, fail;
+ if (result.is_valid()) {
+ mov(result, zero_reg);
+ Move(scratch, -1.0f);
+ // If fd =< -1 or unordered, then the conversion fails.
+ CompareF32(OLE, fs, scratch);
+ BranchTrueShortF(&fail);
+ CompareIsNanF32(fs, scratch);
+ BranchTrueShortF(&fail);
+ }
+
+ {
+ // Load 2^63 into scratch as its float representation.
+ UseScratchRegisterScope temps(this);
+ Register scratch1 = temps.Acquire();
+ li(scratch1, 0x5F000000);
+ mtc1(scratch1, scratch);
+ }
+
+ // Test if scratch > fs.
+ // If fs < 2^63 we can convert it normally.
+ CompareF32(OLT, fs, scratch);
+ BranchTrueShortF(&simple_convert);
+
+ // First we subtract 2^63 from fs, then trunc it to rd
+ // and add 2^63 to rd.
+ sub_s(scratch, fs, scratch);
+ trunc_l_s(scratch, scratch);
+ dmfc1(rd, scratch);
+ Or(rd, rd, Operand(1UL << 63));
+ Branch(&done);
+
+ // Simple conversion.
+ bind(&simple_convert);
+ trunc_l_s(scratch, fs);
+ dmfc1(rd, scratch);
+
+ bind(&done);
+ if (result.is_valid()) {
+ // Conversion is failed if the result is negative or unordered.
+ {
+ UseScratchRegisterScope temps(this);
+ Register scratch1 = temps.Acquire();
+ addiu(scratch1, zero_reg, -1);
+ dsrl(scratch1, scratch1, 1); // Load 2^62.
+ dmfc1(result, scratch);
+ xor_(result, result, scratch1);
+ }
+ Slt(result, zero_reg, result);
+ }
+
+ bind(&fail);
+}
+
+template <typename RoundFunc>
+void TurboAssembler::RoundDouble(FPURegister dst, FPURegister src,
+ FPURoundingMode mode, RoundFunc round) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Register scratch = t8;
+ if (kArchVariant == kMips64r6) {
+ cfc1(scratch, FCSR);
+ li(at, Operand(mode));
+ ctc1(at, FCSR);
+ rint_d(dst, src);
+ ctc1(scratch, FCSR);
+ } else {
+ Label done;
+ if (!IsDoubleZeroRegSet()) {
+ Move(kDoubleRegZero, 0.0);
+ }
+ mfhc1(scratch, src);
+ Ext(at, scratch, HeapNumber::kExponentShift, HeapNumber::kExponentBits);
+ Branch(USE_DELAY_SLOT, &done, hs, at,
+ Operand(HeapNumber::kExponentBias + HeapNumber::kMantissaBits));
+ // Canonicalize the result.
+ sub_d(dst, src, kDoubleRegZero);
+ round(this, dst, src);
+ dmfc1(at, dst);
+ Branch(USE_DELAY_SLOT, &done, ne, at, Operand(zero_reg));
+ cvt_d_l(dst, dst);
+ srl(at, scratch, 31);
+ sll(at, at, 31);
+ mthc1(at, dst);
+ bind(&done);
+ }
+}
+
+void TurboAssembler::Floor_d_d(FPURegister dst, FPURegister src) {
+ RoundDouble(dst, src, mode_floor,
+ [](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
+ tasm->floor_l_d(dst, src);
+ });
+}
+
+void TurboAssembler::Ceil_d_d(FPURegister dst, FPURegister src) {
+ RoundDouble(dst, src, mode_ceil,
+ [](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
+ tasm->ceil_l_d(dst, src);
+ });
+}
+
+void TurboAssembler::Trunc_d_d(FPURegister dst, FPURegister src) {
+ RoundDouble(dst, src, mode_trunc,
+ [](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
+ tasm->trunc_l_d(dst, src);
+ });
+}
+
+void TurboAssembler::Round_d_d(FPURegister dst, FPURegister src) {
+ RoundDouble(dst, src, mode_round,
+ [](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
+ tasm->round_l_d(dst, src);
+ });
+}
+
+template <typename RoundFunc>
+void TurboAssembler::RoundFloat(FPURegister dst, FPURegister src,
+ FPURoundingMode mode, RoundFunc round) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Register scratch = t8;
+ if (kArchVariant == kMips64r6) {
+ cfc1(scratch, FCSR);
+ li(at, Operand(mode));
+ ctc1(at, FCSR);
+ rint_s(dst, src);
+ ctc1(scratch, FCSR);
+ } else {
+ int32_t kFloat32ExponentBias = 127;
+ int32_t kFloat32MantissaBits = 23;
+ int32_t kFloat32ExponentBits = 8;
+ Label done;
+ if (!IsDoubleZeroRegSet()) {
+ Move(kDoubleRegZero, 0.0);
+ }
+ mfc1(scratch, src);
+ Ext(at, scratch, kFloat32MantissaBits, kFloat32ExponentBits);
+ Branch(USE_DELAY_SLOT, &done, hs, at,
+ Operand(kFloat32ExponentBias + kFloat32MantissaBits));
+ // Canonicalize the result.
+ sub_s(dst, src, kDoubleRegZero);
+ round(this, dst, src);
+ mfc1(at, dst);
+ Branch(USE_DELAY_SLOT, &done, ne, at, Operand(zero_reg));
+ cvt_s_w(dst, dst);
+ srl(at, scratch, 31);
+ sll(at, at, 31);
+ mtc1(at, dst);
+ bind(&done);
+ }
+}
+
+void TurboAssembler::Floor_s_s(FPURegister dst, FPURegister src) {
+ RoundFloat(dst, src, mode_floor,
+ [](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
+ tasm->floor_w_s(dst, src);
+ });
+}
+
+void TurboAssembler::Ceil_s_s(FPURegister dst, FPURegister src) {
+ RoundFloat(dst, src, mode_ceil,
+ [](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
+ tasm->ceil_w_s(dst, src);
+ });
+}
+
+void TurboAssembler::Trunc_s_s(FPURegister dst, FPURegister src) {
+ RoundFloat(dst, src, mode_trunc,
+ [](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
+ tasm->trunc_w_s(dst, src);
+ });
+}
+
+void TurboAssembler::Round_s_s(FPURegister dst, FPURegister src) {
+ RoundFloat(dst, src, mode_round,
+ [](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
+ tasm->round_w_s(dst, src);
+ });
+}
+
+void TurboAssembler::MSARoundW(MSARegister dst, MSARegister src,
+ FPURoundingMode mode) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Register scratch = t8;
+ Register scratch2 = at;
+ cfcmsa(scratch, MSACSR);
+ if (mode == kRoundToNearest) {
+ scratch2 = zero_reg;
+ } else {
+ li(scratch2, Operand(mode));
+ }
+ ctcmsa(MSACSR, scratch2);
+ frint_w(dst, src);
+ ctcmsa(MSACSR, scratch);
+}
+
+void TurboAssembler::MSARoundD(MSARegister dst, MSARegister src,
+ FPURoundingMode mode) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Register scratch = t8;
+ Register scratch2 = at;
+ cfcmsa(scratch, MSACSR);
+ if (mode == kRoundToNearest) {
+ scratch2 = zero_reg;
+ } else {
+ li(scratch2, Operand(mode));
+ }
+ ctcmsa(MSACSR, scratch2);
+ frint_d(dst, src);
+ ctcmsa(MSACSR, scratch);
+}
+
+void MacroAssembler::Madd_s(FPURegister fd, FPURegister fr, FPURegister fs,
+ FPURegister ft, FPURegister scratch) {
+ DCHECK(fr != scratch && fs != scratch && ft != scratch);
+ mul_s(scratch, fs, ft);
+ add_s(fd, fr, scratch);
+}
+
+void MacroAssembler::Madd_d(FPURegister fd, FPURegister fr, FPURegister fs,
+ FPURegister ft, FPURegister scratch) {
+ DCHECK(fr != scratch && fs != scratch && ft != scratch);
+ mul_d(scratch, fs, ft);
+ add_d(fd, fr, scratch);
+}
+
+void MacroAssembler::Msub_s(FPURegister fd, FPURegister fr, FPURegister fs,
+ FPURegister ft, FPURegister scratch) {
+ DCHECK(fr != scratch && fs != scratch && ft != scratch);
+ mul_s(scratch, fs, ft);
+ sub_s(fd, scratch, fr);
+}
+
+void MacroAssembler::Msub_d(FPURegister fd, FPURegister fr, FPURegister fs,
+ FPURegister ft, FPURegister scratch) {
+ DCHECK(fr != scratch && fs != scratch && ft != scratch);
+ mul_d(scratch, fs, ft);
+ sub_d(fd, scratch, fr);
+}
+
+void TurboAssembler::CompareF(SecondaryField sizeField, FPUCondition cc,
+ FPURegister cmp1, FPURegister cmp2) {
+ if (kArchVariant == kMips64r6) {
+ sizeField = sizeField == D ? L : W;
+ DCHECK(cmp1 != kDoubleCompareReg && cmp2 != kDoubleCompareReg);
+ cmp(cc, sizeField, kDoubleCompareReg, cmp1, cmp2);
+ } else {
+ c(cc, sizeField, cmp1, cmp2);
+ }
+}
+
+void TurboAssembler::CompareIsNanF(SecondaryField sizeField, FPURegister cmp1,
+ FPURegister cmp2) {
+ CompareF(sizeField, UN, cmp1, cmp2);
+}
+
+void TurboAssembler::BranchTrueShortF(Label* target, BranchDelaySlot bd) {
+ if (kArchVariant == kMips64r6) {
+ bc1nez(target, kDoubleCompareReg);
+ } else {
+ bc1t(target);
+ }
+ if (bd == PROTECT) {
+ nop();
+ }
+}
+
+void TurboAssembler::BranchFalseShortF(Label* target, BranchDelaySlot bd) {
+ if (kArchVariant == kMips64r6) {
+ bc1eqz(target, kDoubleCompareReg);
+ } else {
+ bc1f(target);
+ }
+ if (bd == PROTECT) {
+ nop();
+ }
+}
+
+void TurboAssembler::BranchTrueF(Label* target, BranchDelaySlot bd) {
+ bool long_branch =
+ target->is_bound() ? !is_near(target) : is_trampoline_emitted();
+ if (long_branch) {
+ Label skip;
+ BranchFalseShortF(&skip);
+ BranchLong(target, bd);
+ bind(&skip);
+ } else {
+ BranchTrueShortF(target, bd);
+ }
+}
+
+void TurboAssembler::BranchFalseF(Label* target, BranchDelaySlot bd) {
+ bool long_branch =
+ target->is_bound() ? !is_near(target) : is_trampoline_emitted();
+ if (long_branch) {
+ Label skip;
+ BranchTrueShortF(&skip);
+ BranchLong(target, bd);
+ bind(&skip);
+ } else {
+ BranchFalseShortF(target, bd);
+ }
+}
+
+void TurboAssembler::BranchMSA(Label* target, MSABranchDF df,
+ MSABranchCondition cond, MSARegister wt,
+ BranchDelaySlot bd) {
+ {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+
+ if (target) {
+ bool long_branch =
+ target->is_bound() ? !is_near(target) : is_trampoline_emitted();
+ if (long_branch) {
+ Label skip;
+ MSABranchCondition neg_cond = NegateMSABranchCondition(cond);
+ BranchShortMSA(df, &skip, neg_cond, wt, bd);
+ BranchLong(target, bd);
+ bind(&skip);
+ } else {
+ BranchShortMSA(df, target, cond, wt, bd);
+ }
+ }
+ }
+}
+
+void TurboAssembler::BranchShortMSA(MSABranchDF df, Label* target,
+ MSABranchCondition cond, MSARegister wt,
+ BranchDelaySlot bd) {
+ if (IsEnabled(MIPS_SIMD)) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ if (target) {
+ switch (cond) {
+ case all_not_zero:
+ switch (df) {
+ case MSA_BRANCH_D:
+ bnz_d(wt, target);
+ break;
+ case MSA_BRANCH_W:
+ bnz_w(wt, target);
+ break;
+ case MSA_BRANCH_H:
+ bnz_h(wt, target);
+ break;
+ case MSA_BRANCH_B:
+ default:
+ bnz_b(wt, target);
+ }
+ break;
+ case one_elem_not_zero:
+ bnz_v(wt, target);
+ break;
+ case one_elem_zero:
+ switch (df) {
+ case MSA_BRANCH_D:
+ bz_d(wt, target);
+ break;
+ case MSA_BRANCH_W:
+ bz_w(wt, target);
+ break;
+ case MSA_BRANCH_H:
+ bz_h(wt, target);
+ break;
+ case MSA_BRANCH_B:
+ default:
+ bz_b(wt, target);
+ }
+ break;
+ case all_zero:
+ bz_v(wt, target);
+ break;
+ default:
+ UNREACHABLE();
+ }
+ }
+ } else {
+ UNREACHABLE();
+ }
+ if (bd == PROTECT) {
+ nop();
+ }
+}
+
+void TurboAssembler::FmoveLow(FPURegister dst, Register src_low) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ DCHECK(src_low != scratch);
+ mfhc1(scratch, dst);
+ mtc1(src_low, dst);
+ mthc1(scratch, dst);
+}
+
+void TurboAssembler::Move(FPURegister dst, uint32_t src) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ li(scratch, Operand(static_cast<int32_t>(src)));
+ mtc1(scratch, dst);
+}
+
+void TurboAssembler::Move(FPURegister dst, uint64_t src) {
+ // Handle special values first.
+ if (src == bit_cast<uint64_t>(0.0) && has_double_zero_reg_set_) {
+ mov_d(dst, kDoubleRegZero);
+ } else if (src == bit_cast<uint64_t>(-0.0) && has_double_zero_reg_set_) {
+ Neg_d(dst, kDoubleRegZero);
+ } else {
+ uint32_t lo = src & 0xFFFFFFFF;
+ uint32_t hi = src >> 32;
+ // Move the low part of the double into the lower of the corresponding FPU
+ // register of FPU register pair.
+ if (lo != 0) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ li(scratch, Operand(lo));
+ mtc1(scratch, dst);
+ } else {
+ mtc1(zero_reg, dst);
+ }
+ // Move the high part of the double into the higher of the corresponding FPU
+ // register of FPU register pair.
+ if (hi != 0) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ li(scratch, Operand(hi));
+ mthc1(scratch, dst);
+ } else {
+ mthc1(zero_reg, dst);
+ }
+ if (dst == kDoubleRegZero) has_double_zero_reg_set_ = true;
+ }
+}
+
+void TurboAssembler::Movz(Register rd, Register rs, Register rt) {
+ if (kArchVariant == kMips64r6) {
+ Label done;
+ Branch(&done, ne, rt, Operand(zero_reg));
+ mov(rd, rs);
+ bind(&done);
+ } else {
+ movz(rd, rs, rt);
+ }
+}
+
+void TurboAssembler::Movn(Register rd, Register rs, Register rt) {
+ if (kArchVariant == kMips64r6) {
+ Label done;
+ Branch(&done, eq, rt, Operand(zero_reg));
+ mov(rd, rs);
+ bind(&done);
+ } else {
+ movn(rd, rs, rt);
+ }
+}
+
+void TurboAssembler::LoadZeroOnCondition(Register rd, Register rs,
+ const Operand& rt, Condition cond) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ switch (cond) {
+ case cc_always:
+ mov(rd, zero_reg);
+ break;
+ case eq:
+ if (rs == zero_reg) {
+ if (rt.is_reg()) {
+ LoadZeroIfConditionZero(rd, rt.rm());
+ } else {
+ if (rt.immediate() == 0) {
+ mov(rd, zero_reg);
+ } else {
+ nop();
+ }
+ }
+ } else if (IsZero(rt)) {
+ LoadZeroIfConditionZero(rd, rs);
+ } else {
+ Dsubu(t9, rs, rt);
+ LoadZeroIfConditionZero(rd, t9);
+ }
+ break;
+ case ne:
+ if (rs == zero_reg) {
+ if (rt.is_reg()) {
+ LoadZeroIfConditionNotZero(rd, rt.rm());
+ } else {
+ if (rt.immediate() != 0) {
+ mov(rd, zero_reg);
+ } else {
+ nop();
+ }
+ }
+ } else if (IsZero(rt)) {
+ LoadZeroIfConditionNotZero(rd, rs);
+ } else {
+ Dsubu(t9, rs, rt);
+ LoadZeroIfConditionNotZero(rd, t9);
+ }
+ break;
+
+ // Signed comparison.
+ case greater:
+ Sgt(t9, rs, rt);
+ LoadZeroIfConditionNotZero(rd, t9);
+ break;
+ case greater_equal:
+ Sge(t9, rs, rt);
+ LoadZeroIfConditionNotZero(rd, t9);
+ // rs >= rt
+ break;
+ case less:
+ Slt(t9, rs, rt);
+ LoadZeroIfConditionNotZero(rd, t9);
+ // rs < rt
+ break;
+ case less_equal:
+ Sle(t9, rs, rt);
+ LoadZeroIfConditionNotZero(rd, t9);
+ // rs <= rt
+ break;
+
+ // Unsigned comparison.
+ case Ugreater:
+ Sgtu(t9, rs, rt);
+ LoadZeroIfConditionNotZero(rd, t9);
+ // rs > rt
+ break;
+
+ case Ugreater_equal:
+ Sgeu(t9, rs, rt);
+ LoadZeroIfConditionNotZero(rd, t9);
+ // rs >= rt
+ break;
+ case Uless:
+ Sltu(t9, rs, rt);
+ LoadZeroIfConditionNotZero(rd, t9);
+ // rs < rt
+ break;
+ case Uless_equal:
+ Sleu(t9, rs, rt);
+ LoadZeroIfConditionNotZero(rd, t9);
+ // rs <= rt
+ break;
+ default:
+ UNREACHABLE();
+ }
+}
+
+void TurboAssembler::LoadZeroIfConditionNotZero(Register dest,
+ Register condition) {
+ if (kArchVariant == kMips64r6) {
+ seleqz(dest, dest, condition);
+ } else {
+ Movn(dest, zero_reg, condition);
+ }
+}
+
+void TurboAssembler::LoadZeroIfConditionZero(Register dest,
+ Register condition) {
+ if (kArchVariant == kMips64r6) {
+ selnez(dest, dest, condition);
+ } else {
+ Movz(dest, zero_reg, condition);
+ }
+}
+
+void TurboAssembler::LoadZeroIfFPUCondition(Register dest) {
+ if (kArchVariant == kMips64r6) {
+ dmfc1(kScratchReg, kDoubleCompareReg);
+ LoadZeroIfConditionNotZero(dest, kScratchReg);
+ } else {
+ Movt(dest, zero_reg);
+ }
+}
+
+void TurboAssembler::LoadZeroIfNotFPUCondition(Register dest) {
+ if (kArchVariant == kMips64r6) {
+ dmfc1(kScratchReg, kDoubleCompareReg);
+ LoadZeroIfConditionZero(dest, kScratchReg);
+ } else {
+ Movf(dest, zero_reg);
+ }
+}
+
+void TurboAssembler::Movt(Register rd, Register rs, uint16_t cc) {
+ movt(rd, rs, cc);
+}
+
+void TurboAssembler::Movf(Register rd, Register rs, uint16_t cc) {
+ movf(rd, rs, cc);
+}
+
+void TurboAssembler::Clz(Register rd, Register rs) { clz(rd, rs); }
+
+void TurboAssembler::Dclz(Register rd, Register rs) { dclz(rd, rs); }
+
+void TurboAssembler::Ctz(Register rd, Register rs) {
+ if (kArchVariant == kMips64r6) {
+ // We don't have an instruction to count the number of trailing zeroes.
+ // Start by flipping the bits end-for-end so we can count the number of
+ // leading zeroes instead.
+ rotr(rd, rs, 16);
+ wsbh(rd, rd);
+ bitswap(rd, rd);
+ Clz(rd, rd);
+ } else {
+ // Convert trailing zeroes to trailing ones, and bits to their left
+ // to zeroes.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ Daddu(scratch, rs, -1);
+ Xor(rd, scratch, rs);
+ And(rd, rd, scratch);
+ // Count number of leading zeroes.
+ Clz(rd, rd);
+ // Subtract number of leading zeroes from 32 to get number of trailing
+ // ones. Remember that the trailing ones were formerly trailing zeroes.
+ li(scratch, 32);
+ Subu(rd, scratch, rd);
+ }
+}
+
+void TurboAssembler::Dctz(Register rd, Register rs) {
+ if (kArchVariant == kMips64r6) {
+ // We don't have an instruction to count the number of trailing zeroes.
+ // Start by flipping the bits end-for-end so we can count the number of
+ // leading zeroes instead.
+ dsbh(rd, rs);
+ dshd(rd, rd);
+ dbitswap(rd, rd);
+ dclz(rd, rd);
+ } else {
+ // Convert trailing zeroes to trailing ones, and bits to their left
+ // to zeroes.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ Daddu(scratch, rs, -1);
+ Xor(rd, scratch, rs);
+ And(rd, rd, scratch);
+ // Count number of leading zeroes.
+ dclz(rd, rd);
+ // Subtract number of leading zeroes from 64 to get number of trailing
+ // ones. Remember that the trailing ones were formerly trailing zeroes.
+ li(scratch, 64);
+ Dsubu(rd, scratch, rd);
+ }
+}
+
+void TurboAssembler::Popcnt(Register rd, Register rs) {
+ // https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
+ //
+ // A generalization of the best bit counting method to integers of
+ // bit-widths up to 128 (parameterized by type T) is this:
+ //
+ // v = v - ((v >> 1) & (T)~(T)0/3); // temp
+ // v = (v & (T)~(T)0/15*3) + ((v >> 2) & (T)~(T)0/15*3); // temp
+ // v = (v + (v >> 4)) & (T)~(T)0/255*15; // temp
+ // c = (T)(v * ((T)~(T)0/255)) >> (sizeof(T) - 1) * BITS_PER_BYTE; //count
+ //
+ // For comparison, for 32-bit quantities, this algorithm can be executed
+ // using 20 MIPS instructions (the calls to LoadConst32() generate two
+ // machine instructions each for the values being used in this algorithm).
+ // A(n unrolled) loop-based algorithm requires 25 instructions.
+ //
+ // For a 64-bit operand this can be performed in 24 instructions compared
+ // to a(n unrolled) loop based algorithm which requires 38 instructions.
+ //
+ // There are algorithms which are faster in the cases where very few
+ // bits are set but the algorithm here attempts to minimize the total
+ // number of instructions executed even when a large number of bits
+ // are set.
+ uint32_t B0 = 0x55555555; // (T)~(T)0/3
+ uint32_t B1 = 0x33333333; // (T)~(T)0/15*3
+ uint32_t B2 = 0x0F0F0F0F; // (T)~(T)0/255*15
+ uint32_t value = 0x01010101; // (T)~(T)0/255
+ uint32_t shift = 24; // (sizeof(T) - 1) * BITS_PER_BYTE
+
+ UseScratchRegisterScope temps(this);
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Register scratch = temps.Acquire();
+ Register scratch2 = t8;
+ srl(scratch, rs, 1);
+ li(scratch2, B0);
+ And(scratch, scratch, scratch2);
+ Subu(scratch, rs, scratch);
+ li(scratch2, B1);
+ And(rd, scratch, scratch2);
+ srl(scratch, scratch, 2);
+ And(scratch, scratch, scratch2);
+ Addu(scratch, rd, scratch);
+ srl(rd, scratch, 4);
+ Addu(rd, rd, scratch);
+ li(scratch2, B2);
+ And(rd, rd, scratch2);
+ li(scratch, value);
+ Mul(rd, rd, scratch);
+ srl(rd, rd, shift);
+}
+
+void TurboAssembler::Dpopcnt(Register rd, Register rs) {
+ uint64_t B0 = 0x5555555555555555l; // (T)~(T)0/3
+ uint64_t B1 = 0x3333333333333333l; // (T)~(T)0/15*3
+ uint64_t B2 = 0x0F0F0F0F0F0F0F0Fl; // (T)~(T)0/255*15
+ uint64_t value = 0x0101010101010101l; // (T)~(T)0/255
+ uint64_t shift = 24; // (sizeof(T) - 1) * BITS_PER_BYTE
+
+ UseScratchRegisterScope temps(this);
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Register scratch = temps.Acquire();
+ Register scratch2 = t8;
+ dsrl(scratch, rs, 1);
+ li(scratch2, B0);
+ And(scratch, scratch, scratch2);
+ Dsubu(scratch, rs, scratch);
+ li(scratch2, B1);
+ And(rd, scratch, scratch2);
+ dsrl(scratch, scratch, 2);
+ And(scratch, scratch, scratch2);
+ Daddu(scratch, rd, scratch);
+ dsrl(rd, scratch, 4);
+ Daddu(rd, rd, scratch);
+ li(scratch2, B2);
+ And(rd, rd, scratch2);
+ li(scratch, value);
+ Dmul(rd, rd, scratch);
+ dsrl32(rd, rd, shift);
+}
+
+void MacroAssembler::EmitFPUTruncate(
+ FPURoundingMode rounding_mode, Register result, DoubleRegister double_input,
+ Register scratch, DoubleRegister double_scratch, Register except_flag,
+ CheckForInexactConversion check_inexact) {
+ DCHECK(result != scratch);
+ DCHECK(double_input != double_scratch);
+ DCHECK(except_flag != scratch);
+
+ Label done;
+
+ // Clear the except flag (0 = no exception)
+ mov(except_flag, zero_reg);
+
+ // Test for values that can be exactly represented as a signed 32-bit integer.
+ cvt_w_d(double_scratch, double_input);
+ mfc1(result, double_scratch);
+ cvt_d_w(double_scratch, double_scratch);
+ CompareF64(EQ, double_input, double_scratch);
+ BranchTrueShortF(&done);
+
+ int32_t except_mask = kFCSRFlagMask; // Assume interested in all exceptions.
+
+ if (check_inexact == kDontCheckForInexactConversion) {
+ // Ignore inexact exceptions.
+ except_mask &= ~kFCSRInexactFlagMask;
+ }
+
+ // Save FCSR.
+ cfc1(scratch, FCSR);
+ // Disable FPU exceptions.
+ ctc1(zero_reg, FCSR);
+
+ // Do operation based on rounding mode.
+ switch (rounding_mode) {
+ case kRoundToNearest:
+ Round_w_d(double_scratch, double_input);
+ break;
+ case kRoundToZero:
+ Trunc_w_d(double_scratch, double_input);
+ break;
+ case kRoundToPlusInf:
+ Ceil_w_d(double_scratch, double_input);
+ break;
+ case kRoundToMinusInf:
+ Floor_w_d(double_scratch, double_input);
+ break;
+ } // End of switch-statement.
+
+ // Retrieve FCSR.
+ cfc1(except_flag, FCSR);
+ // Restore FCSR.
+ ctc1(scratch, FCSR);
+ // Move the converted value into the result register.
+ mfc1(result, double_scratch);
+
+ // Check for fpu exceptions.
+ And(except_flag, except_flag, Operand(except_mask));
+
+ bind(&done);
+}
+
+void TurboAssembler::TryInlineTruncateDoubleToI(Register result,
+ DoubleRegister double_input,
+ Label* done) {
+ DoubleRegister single_scratch = kScratchDoubleReg.low();
+ UseScratchRegisterScope temps(this);
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Register scratch = temps.Acquire();
+ Register scratch2 = t9;
+
+ // Clear cumulative exception flags and save the FCSR.
+ cfc1(scratch2, FCSR);
+ ctc1(zero_reg, FCSR);
+ // Try a conversion to a signed integer.
+ trunc_w_d(single_scratch, double_input);
+ mfc1(result, single_scratch);
+ // Retrieve and restore the FCSR.
+ cfc1(scratch, FCSR);
+ ctc1(scratch2, FCSR);
+ // Check for overflow and NaNs.
+ And(scratch, scratch,
+ kFCSROverflowFlagMask | kFCSRUnderflowFlagMask | kFCSRInvalidOpFlagMask);
+ // If we had no exceptions we are done.
+ Branch(done, eq, scratch, Operand(zero_reg));
+}
+
+void TurboAssembler::TruncateDoubleToI(Isolate* isolate, Zone* zone,
+ Register result,
+ DoubleRegister double_input,
+ StubCallMode stub_mode) {
+ Label done;
+
+ TryInlineTruncateDoubleToI(result, double_input, &done);
+
+ // If we fell through then inline version didn't succeed - call stub instead.
+ push(ra);
+ Dsubu(sp, sp, Operand(kDoubleSize)); // Put input on stack.
+ Sdc1(double_input, MemOperand(sp, 0));
+
+ if (stub_mode == StubCallMode::kCallWasmRuntimeStub) {
+ Call(wasm::WasmCode::kDoubleToI, RelocInfo::WASM_STUB_CALL);
+ } else {
+ Call(BUILTIN_CODE(isolate, DoubleToI), RelocInfo::CODE_TARGET);
+ }
+ Ld(result, MemOperand(sp, 0));
+
+ Daddu(sp, sp, Operand(kDoubleSize));
+ pop(ra);
+
+ bind(&done);
+}
+
+// Emulated condtional branches do not emit a nop in the branch delay slot.
+//
+// BRANCH_ARGS_CHECK checks that conditional jump arguments are correct.
+#define BRANCH_ARGS_CHECK(cond, rs, rt) \
+ DCHECK((cond == cc_always && rs == zero_reg && rt.rm() == zero_reg) || \
+ (cond != cc_always && (rs != zero_reg || rt.rm() != zero_reg)))
+
+void TurboAssembler::Branch(int32_t offset, BranchDelaySlot bdslot) {
+ DCHECK_EQ(kArchVariant, kMips64r6 ? is_int26(offset) : is_int16(offset));
+ BranchShort(offset, bdslot);
+}
+
+void TurboAssembler::Branch(int32_t offset, Condition cond, Register rs,
+ const Operand& rt, BranchDelaySlot bdslot) {
+ bool is_near = BranchShortCheck(offset, nullptr, cond, rs, rt, bdslot);
+ DCHECK(is_near);
+ USE(is_near);
+}
+
+void TurboAssembler::Branch(Label* L, BranchDelaySlot bdslot) {
+ if (L->is_bound()) {
+ if (is_near_branch(L)) {
+ BranchShort(L, bdslot);
+ } else {
+ BranchLong(L, bdslot);
+ }
+ } else {
+ if (is_trampoline_emitted()) {
+ BranchLong(L, bdslot);
+ } else {
+ BranchShort(L, bdslot);
+ }
+ }
+}
+
+void TurboAssembler::Branch(Label* L, Condition cond, Register rs,
+ const Operand& rt, BranchDelaySlot bdslot) {
+ if (L->is_bound()) {
+ if (!BranchShortCheck(0, L, cond, rs, rt, bdslot)) {
+ if (cond != cc_always) {
+ Label skip;
+ Condition neg_cond = NegateCondition(cond);
+ BranchShort(&skip, neg_cond, rs, rt);
+ BranchLong(L, bdslot);
+ bind(&skip);
+ } else {
+ BranchLong(L, bdslot);
+ }
+ }
+ } else {
+ if (is_trampoline_emitted()) {
+ if (cond != cc_always) {
+ Label skip;
+ Condition neg_cond = NegateCondition(cond);
+ BranchShort(&skip, neg_cond, rs, rt);
+ BranchLong(L, bdslot);
+ bind(&skip);
+ } else {
+ BranchLong(L, bdslot);
+ }
+ } else {
+ BranchShort(L, cond, rs, rt, bdslot);
+ }
+ }
+}
+
+void TurboAssembler::Branch(Label* L, Condition cond, Register rs,
+ RootIndex index, BranchDelaySlot bdslot) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ LoadRoot(scratch, index);
+ Branch(L, cond, rs, Operand(scratch), bdslot);
+}
+
+void TurboAssembler::BranchShortHelper(int16_t offset, Label* L,
+ BranchDelaySlot bdslot) {
+ DCHECK(L == nullptr || offset == 0);
+ offset = GetOffset(offset, L, OffsetSize::kOffset16);
+ b(offset);
+
+ // Emit a nop in the branch delay slot if required.
+ if (bdslot == PROTECT) nop();
+}
+
+void TurboAssembler::BranchShortHelperR6(int32_t offset, Label* L) {
+ DCHECK(L == nullptr || offset == 0);
+ offset = GetOffset(offset, L, OffsetSize::kOffset26);
+ bc(offset);
+}
+
+void TurboAssembler::BranchShort(int32_t offset, BranchDelaySlot bdslot) {
+ if (kArchVariant == kMips64r6 && bdslot == PROTECT) {
+ DCHECK(is_int26(offset));
+ BranchShortHelperR6(offset, nullptr);
+ } else {
+ DCHECK(is_int16(offset));
+ BranchShortHelper(offset, nullptr, bdslot);
+ }
+}
+
+void TurboAssembler::BranchShort(Label* L, BranchDelaySlot bdslot) {
+ if (kArchVariant == kMips64r6 && bdslot == PROTECT) {
+ BranchShortHelperR6(0, L);
+ } else {
+ BranchShortHelper(0, L, bdslot);
+ }
+}
+
+int32_t TurboAssembler::GetOffset(int32_t offset, Label* L, OffsetSize bits) {
+ if (L) {
+ offset = branch_offset_helper(L, bits) >> 2;
+ } else {
+ DCHECK(is_intn(offset, bits));
+ }
+ return offset;
+}
+
+Register TurboAssembler::GetRtAsRegisterHelper(const Operand& rt,
+ Register scratch) {
+ Register r2 = no_reg;
+ if (rt.is_reg()) {
+ r2 = rt.rm();
+ } else {
+ r2 = scratch;
+ li(r2, rt);
+ }
+
+ return r2;
+}
+
+bool TurboAssembler::CalculateOffset(Label* L, int32_t* offset,
+ OffsetSize bits) {
+ if (!is_near(L, bits)) return false;
+ *offset = GetOffset(*offset, L, bits);
+ return true;
+}
+
+bool TurboAssembler::CalculateOffset(Label* L, int32_t* offset, OffsetSize bits,
+ Register* scratch, const Operand& rt) {
+ if (!is_near(L, bits)) return false;
+ *scratch = GetRtAsRegisterHelper(rt, *scratch);
+ *offset = GetOffset(*offset, L, bits);
+ return true;
+}
+
+bool TurboAssembler::BranchShortHelperR6(int32_t offset, Label* L,
+ Condition cond, Register rs,
+ const Operand& rt) {
+ DCHECK(L == nullptr || offset == 0);
+ UseScratchRegisterScope temps(this);
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
+
+ // Be careful to always use shifted_branch_offset only just before the
+ // branch instruction, as the location will be remember for patching the
+ // target.
+ {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ switch (cond) {
+ case cc_always:
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset26)) return false;
+ bc(offset);
+ break;
+ case eq:
+ if (rt.is_reg() && rs.code() == rt.rm().code()) {
+ // Pre R6 beq is used here to make the code patchable. Otherwise bc
+ // should be used which has no condition field so is not patchable.
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ beq(rs, scratch, offset);
+ nop();
+ } else if (IsZero(rt)) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset21)) return false;
+ beqzc(rs, offset);
+ } else {
+ // We don't want any other register but scratch clobbered.
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ beqc(rs, scratch, offset);
+ }
+ break;
+ case ne:
+ if (rt.is_reg() && rs.code() == rt.rm().code()) {
+ // Pre R6 bne is used here to make the code patchable. Otherwise we
+ // should not generate any instruction.
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ bne(rs, scratch, offset);
+ nop();
+ } else if (IsZero(rt)) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset21)) return false;
+ bnezc(rs, offset);
+ } else {
+ // We don't want any other register but scratch clobbered.
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ bnec(rs, scratch, offset);
+ }
+ break;
+
+ // Signed comparison.
+ case greater:
+ // rs > rt
+ if (rt.is_reg() && rs.code() == rt.rm().code()) {
+ break; // No code needs to be emitted.
+ } else if (rs == zero_reg) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ bltzc(scratch, offset);
+ } else if (IsZero(rt)) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16)) return false;
+ bgtzc(rs, offset);
+ } else {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ DCHECK(rs != scratch);
+ bltc(scratch, rs, offset);
+ }
+ break;
+ case greater_equal:
+ // rs >= rt
+ if (rt.is_reg() && rs.code() == rt.rm().code()) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset26)) return false;
+ bc(offset);
+ } else if (rs == zero_reg) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ blezc(scratch, offset);
+ } else if (IsZero(rt)) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16)) return false;
+ bgezc(rs, offset);
+ } else {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ DCHECK(rs != scratch);
+ bgec(rs, scratch, offset);
+ }
+ break;
+ case less:
+ // rs < rt
+ if (rt.is_reg() && rs.code() == rt.rm().code()) {
+ break; // No code needs to be emitted.
+ } else if (rs == zero_reg) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ bgtzc(scratch, offset);
+ } else if (IsZero(rt)) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16)) return false;
+ bltzc(rs, offset);
+ } else {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ DCHECK(rs != scratch);
+ bltc(rs, scratch, offset);
+ }
+ break;
+ case less_equal:
+ // rs <= rt
+ if (rt.is_reg() && rs.code() == rt.rm().code()) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset26)) return false;
+ bc(offset);
+ } else if (rs == zero_reg) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ bgezc(scratch, offset);
+ } else if (IsZero(rt)) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16)) return false;
+ blezc(rs, offset);
+ } else {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ DCHECK(rs != scratch);
+ bgec(scratch, rs, offset);
+ }
+ break;
+
+ // Unsigned comparison.
+ case Ugreater:
+ // rs > rt
+ if (rt.is_reg() && rs.code() == rt.rm().code()) {
+ break; // No code needs to be emitted.
+ } else if (rs == zero_reg) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset21, &scratch, rt))
+ return false;
+ bnezc(scratch, offset);
+ } else if (IsZero(rt)) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset21)) return false;
+ bnezc(rs, offset);
+ } else {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ DCHECK(rs != scratch);
+ bltuc(scratch, rs, offset);
+ }
+ break;
+ case Ugreater_equal:
+ // rs >= rt
+ if (rt.is_reg() && rs.code() == rt.rm().code()) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset26)) return false;
+ bc(offset);
+ } else if (rs == zero_reg) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset21, &scratch, rt))
+ return false;
+ beqzc(scratch, offset);
+ } else if (IsZero(rt)) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset26)) return false;
+ bc(offset);
+ } else {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ DCHECK(rs != scratch);
+ bgeuc(rs, scratch, offset);
+ }
+ break;
+ case Uless:
+ // rs < rt
+ if (rt.is_reg() && rs.code() == rt.rm().code()) {
+ break; // No code needs to be emitted.
+ } else if (rs == zero_reg) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset21, &scratch, rt))
+ return false;
+ bnezc(scratch, offset);
+ } else if (IsZero(rt)) {
+ break; // No code needs to be emitted.
+ } else {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ DCHECK(rs != scratch);
+ bltuc(rs, scratch, offset);
+ }
+ break;
+ case Uless_equal:
+ // rs <= rt
+ if (rt.is_reg() && rs.code() == rt.rm().code()) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset26)) return false;
+ bc(offset);
+ } else if (rs == zero_reg) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset26, &scratch, rt))
+ return false;
+ bc(offset);
+ } else if (IsZero(rt)) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset21)) return false;
+ beqzc(rs, offset);
+ } else {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ DCHECK(rs != scratch);
+ bgeuc(scratch, rs, offset);
+ }
+ break;
+ default:
+ UNREACHABLE();
+ }
+ }
+ CheckTrampolinePoolQuick(1);
+ return true;
+}
+
+bool TurboAssembler::BranchShortHelper(int16_t offset, Label* L, Condition cond,
+ Register rs, const Operand& rt,
+ BranchDelaySlot bdslot) {
+ DCHECK(L == nullptr || offset == 0);
+ if (!is_near(L, OffsetSize::kOffset16)) return false;
+
+ UseScratchRegisterScope temps(this);
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
+ int32_t offset32;
+
+ // Be careful to always use shifted_branch_offset only just before the
+ // branch instruction, as the location will be remember for patching the
+ // target.
+ {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ switch (cond) {
+ case cc_always:
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ b(offset32);
+ break;
+ case eq:
+ if (IsZero(rt)) {
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ beq(rs, zero_reg, offset32);
+ } else {
+ // We don't want any other register but scratch clobbered.
+ scratch = GetRtAsRegisterHelper(rt, scratch);
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ beq(rs, scratch, offset32);
+ }
+ break;
+ case ne:
+ if (IsZero(rt)) {
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ bne(rs, zero_reg, offset32);
+ } else {
+ // We don't want any other register but scratch clobbered.
+ scratch = GetRtAsRegisterHelper(rt, scratch);
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ bne(rs, scratch, offset32);
+ }
+ break;
+
+ // Signed comparison.
+ case greater:
+ if (IsZero(rt)) {
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ bgtz(rs, offset32);
+ } else {
+ Slt(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ bne(scratch, zero_reg, offset32);
+ }
+ break;
+ case greater_equal:
+ if (IsZero(rt)) {
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ bgez(rs, offset32);
+ } else {
+ Slt(scratch, rs, rt);
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ beq(scratch, zero_reg, offset32);
+ }
+ break;
+ case less:
+ if (IsZero(rt)) {
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ bltz(rs, offset32);
+ } else {
+ Slt(scratch, rs, rt);
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ bne(scratch, zero_reg, offset32);
+ }
+ break;
+ case less_equal:
+ if (IsZero(rt)) {
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ blez(rs, offset32);
+ } else {
+ Slt(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ beq(scratch, zero_reg, offset32);
+ }
+ break;
+
+ // Unsigned comparison.
+ case Ugreater:
+ if (IsZero(rt)) {
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ bne(rs, zero_reg, offset32);
+ } else {
+ Sltu(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ bne(scratch, zero_reg, offset32);
+ }
+ break;
+ case Ugreater_equal:
+ if (IsZero(rt)) {
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ b(offset32);
+ } else {
+ Sltu(scratch, rs, rt);
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ beq(scratch, zero_reg, offset32);
+ }
+ break;
+ case Uless:
+ if (IsZero(rt)) {
+ return true; // No code needs to be emitted.
+ } else {
+ Sltu(scratch, rs, rt);
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ bne(scratch, zero_reg, offset32);
+ }
+ break;
+ case Uless_equal:
+ if (IsZero(rt)) {
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ beq(rs, zero_reg, offset32);
+ } else {
+ Sltu(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
+ offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
+ beq(scratch, zero_reg, offset32);
+ }
+ break;
+ default:
+ UNREACHABLE();
+ }
+ }
+
+ // Emit a nop in the branch delay slot if required.
+ if (bdslot == PROTECT) nop();
+
+ return true;
+}
+
+bool TurboAssembler::BranchShortCheck(int32_t offset, Label* L, Condition cond,
+ Register rs, const Operand& rt,
+ BranchDelaySlot bdslot) {
+ BRANCH_ARGS_CHECK(cond, rs, rt);
+
+ if (!L) {
+ if (kArchVariant == kMips64r6 && bdslot == PROTECT) {
+ DCHECK(is_int26(offset));
+ return BranchShortHelperR6(offset, nullptr, cond, rs, rt);
+ } else {
+ DCHECK(is_int16(offset));
+ return BranchShortHelper(offset, nullptr, cond, rs, rt, bdslot);
+ }
+ } else {
+ DCHECK_EQ(offset, 0);
+ if (kArchVariant == kMips64r6 && bdslot == PROTECT) {
+ return BranchShortHelperR6(0, L, cond, rs, rt);
+ } else {
+ return BranchShortHelper(0, L, cond, rs, rt, bdslot);
+ }
+ }
+ return false;
+}
+
+void TurboAssembler::BranchShort(int32_t offset, Condition cond, Register rs,
+ const Operand& rt, BranchDelaySlot bdslot) {
+ BranchShortCheck(offset, nullptr, cond, rs, rt, bdslot);
+}
+
+void TurboAssembler::BranchShort(Label* L, Condition cond, Register rs,
+ const Operand& rt, BranchDelaySlot bdslot) {
+ BranchShortCheck(0, L, cond, rs, rt, bdslot);
+}
+
+void TurboAssembler::BranchAndLink(int32_t offset, BranchDelaySlot bdslot) {
+ BranchAndLinkShort(offset, bdslot);
+}
+
+void TurboAssembler::BranchAndLink(int32_t offset, Condition cond, Register rs,
+ const Operand& rt, BranchDelaySlot bdslot) {
+ bool is_near = BranchAndLinkShortCheck(offset, nullptr, cond, rs, rt, bdslot);
+ DCHECK(is_near);
+ USE(is_near);
+}
+
+void TurboAssembler::BranchAndLink(Label* L, BranchDelaySlot bdslot) {
+ if (L->is_bound()) {
+ if (is_near_branch(L)) {
+ BranchAndLinkShort(L, bdslot);
+ } else {
+ BranchAndLinkLong(L, bdslot);
+ }
+ } else {
+ if (is_trampoline_emitted()) {
+ BranchAndLinkLong(L, bdslot);
+ } else {
+ BranchAndLinkShort(L, bdslot);
+ }
+ }
+}
+
+void TurboAssembler::BranchAndLink(Label* L, Condition cond, Register rs,
+ const Operand& rt, BranchDelaySlot bdslot) {
+ if (L->is_bound()) {
+ if (!BranchAndLinkShortCheck(0, L, cond, rs, rt, bdslot)) {
+ Label skip;
+ Condition neg_cond = NegateCondition(cond);
+ BranchShort(&skip, neg_cond, rs, rt);
+ BranchAndLinkLong(L, bdslot);
+ bind(&skip);
+ }
+ } else {
+ if (is_trampoline_emitted()) {
+ Label skip;
+ Condition neg_cond = NegateCondition(cond);
+ BranchShort(&skip, neg_cond, rs, rt);
+ BranchAndLinkLong(L, bdslot);
+ bind(&skip);
+ } else {
+ BranchAndLinkShortCheck(0, L, cond, rs, rt, bdslot);
+ }
+ }
+}
+
+void TurboAssembler::BranchAndLinkShortHelper(int16_t offset, Label* L,
+ BranchDelaySlot bdslot) {
+ DCHECK(L == nullptr || offset == 0);
+ offset = GetOffset(offset, L, OffsetSize::kOffset16);
+ bal(offset);
+
+ // Emit a nop in the branch delay slot if required.
+ if (bdslot == PROTECT) nop();
+}
+
+void TurboAssembler::BranchAndLinkShortHelperR6(int32_t offset, Label* L) {
+ DCHECK(L == nullptr || offset == 0);
+ offset = GetOffset(offset, L, OffsetSize::kOffset26);
+ balc(offset);
+}
+
+void TurboAssembler::BranchAndLinkShort(int32_t offset,
+ BranchDelaySlot bdslot) {
+ if (kArchVariant == kMips64r6 && bdslot == PROTECT) {
+ DCHECK(is_int26(offset));
+ BranchAndLinkShortHelperR6(offset, nullptr);
+ } else {
+ DCHECK(is_int16(offset));
+ BranchAndLinkShortHelper(offset, nullptr, bdslot);
+ }
+}
+
+void TurboAssembler::BranchAndLinkShort(Label* L, BranchDelaySlot bdslot) {
+ if (kArchVariant == kMips64r6 && bdslot == PROTECT) {
+ BranchAndLinkShortHelperR6(0, L);
+ } else {
+ BranchAndLinkShortHelper(0, L, bdslot);
+ }
+}
+
+bool TurboAssembler::BranchAndLinkShortHelperR6(int32_t offset, Label* L,
+ Condition cond, Register rs,
+ const Operand& rt) {
+ DCHECK(L == nullptr || offset == 0);
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
+ OffsetSize bits = OffsetSize::kOffset16;
+
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ DCHECK((cond == cc_always && is_int26(offset)) || is_int16(offset));
+ switch (cond) {
+ case cc_always:
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset26)) return false;
+ balc(offset);
+ break;
+ case eq:
+ if (!is_near(L, bits)) return false;
+ Subu(scratch, rs, rt);
+ offset = GetOffset(offset, L, bits);
+ beqzalc(scratch, offset);
+ break;
+ case ne:
+ if (!is_near(L, bits)) return false;
+ Subu(scratch, rs, rt);
+ offset = GetOffset(offset, L, bits);
+ bnezalc(scratch, offset);
+ break;
+
+ // Signed comparison.
+ case greater:
+ // rs > rt
+ if (rs.code() == rt.rm().code()) {
+ break; // No code needs to be emitted.
+ } else if (rs == zero_reg) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ bltzalc(scratch, offset);
+ } else if (IsZero(rt)) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16)) return false;
+ bgtzalc(rs, offset);
+ } else {
+ if (!is_near(L, bits)) return false;
+ Slt(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
+ offset = GetOffset(offset, L, bits);
+ bnezalc(scratch, offset);
+ }
+ break;
+ case greater_equal:
+ // rs >= rt
+ if (rs.code() == rt.rm().code()) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset26)) return false;
+ balc(offset);
+ } else if (rs == zero_reg) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ blezalc(scratch, offset);
+ } else if (IsZero(rt)) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16)) return false;
+ bgezalc(rs, offset);
+ } else {
+ if (!is_near(L, bits)) return false;
+ Slt(scratch, rs, rt);
+ offset = GetOffset(offset, L, bits);
+ beqzalc(scratch, offset);
+ }
+ break;
+ case less:
+ // rs < rt
+ if (rs.code() == rt.rm().code()) {
+ break; // No code needs to be emitted.
+ } else if (rs == zero_reg) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ bgtzalc(scratch, offset);
+ } else if (IsZero(rt)) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16)) return false;
+ bltzalc(rs, offset);
+ } else {
+ if (!is_near(L, bits)) return false;
+ Slt(scratch, rs, rt);
+ offset = GetOffset(offset, L, bits);
+ bnezalc(scratch, offset);
+ }
+ break;
+ case less_equal:
+ // rs <= r2
+ if (rs.code() == rt.rm().code()) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset26)) return false;
+ balc(offset);
+ } else if (rs == zero_reg) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16, &scratch, rt))
+ return false;
+ bgezalc(scratch, offset);
+ } else if (IsZero(rt)) {
+ if (!CalculateOffset(L, &offset, OffsetSize::kOffset16)) return false;
+ blezalc(rs, offset);
+ } else {
+ if (!is_near(L, bits)) return false;
+ Slt(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
+ offset = GetOffset(offset, L, bits);
+ beqzalc(scratch, offset);
+ }
+ break;
+
+ // Unsigned comparison.
+ case Ugreater:
+ // rs > r2
+ if (!is_near(L, bits)) return false;
+ Sltu(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
+ offset = GetOffset(offset, L, bits);
+ bnezalc(scratch, offset);
+ break;
+ case Ugreater_equal:
+ // rs >= r2
+ if (!is_near(L, bits)) return false;
+ Sltu(scratch, rs, rt);
+ offset = GetOffset(offset, L, bits);
+ beqzalc(scratch, offset);
+ break;
+ case Uless:
+ // rs < r2
+ if (!is_near(L, bits)) return false;
+ Sltu(scratch, rs, rt);
+ offset = GetOffset(offset, L, bits);
+ bnezalc(scratch, offset);
+ break;
+ case Uless_equal:
+ // rs <= r2
+ if (!is_near(L, bits)) return false;
+ Sltu(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
+ offset = GetOffset(offset, L, bits);
+ beqzalc(scratch, offset);
+ break;
+ default:
+ UNREACHABLE();
+ }
+ return true;
+}
+
+// Pre r6 we need to use a bgezal or bltzal, but they can't be used directly
+// with the slt instructions. We could use sub or add instead but we would miss
+// overflow cases, so we keep slt and add an intermediate third instruction.
+bool TurboAssembler::BranchAndLinkShortHelper(int16_t offset, Label* L,
+ Condition cond, Register rs,
+ const Operand& rt,
+ BranchDelaySlot bdslot) {
+ DCHECK(L == nullptr || offset == 0);
+ if (!is_near(L, OffsetSize::kOffset16)) return false;
+
+ Register scratch = t8;
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+
+ switch (cond) {
+ case cc_always:
+ offset = GetOffset(offset, L, OffsetSize::kOffset16);
+ bal(offset);
+ break;
+ case eq:
+ bne(rs, GetRtAsRegisterHelper(rt, scratch), 2);
+ nop();
+ offset = GetOffset(offset, L, OffsetSize::kOffset16);
+ bal(offset);
+ break;
+ case ne:
+ beq(rs, GetRtAsRegisterHelper(rt, scratch), 2);
+ nop();
+ offset = GetOffset(offset, L, OffsetSize::kOffset16);
+ bal(offset);
+ break;
+
+ // Signed comparison.
+ case greater:
+ Slt(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
+ addiu(scratch, scratch, -1);
+ offset = GetOffset(offset, L, OffsetSize::kOffset16);
+ bgezal(scratch, offset);
+ break;
+ case greater_equal:
+ Slt(scratch, rs, rt);
+ addiu(scratch, scratch, -1);
+ offset = GetOffset(offset, L, OffsetSize::kOffset16);
+ bltzal(scratch, offset);
+ break;
+ case less:
+ Slt(scratch, rs, rt);
+ addiu(scratch, scratch, -1);
+ offset = GetOffset(offset, L, OffsetSize::kOffset16);
+ bgezal(scratch, offset);
+ break;
+ case less_equal:
+ Slt(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
+ addiu(scratch, scratch, -1);
+ offset = GetOffset(offset, L, OffsetSize::kOffset16);
+ bltzal(scratch, offset);
+ break;
+
+ // Unsigned comparison.
+ case Ugreater:
+ Sltu(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
+ addiu(scratch, scratch, -1);
+ offset = GetOffset(offset, L, OffsetSize::kOffset16);
+ bgezal(scratch, offset);
+ break;
+ case Ugreater_equal:
+ Sltu(scratch, rs, rt);
+ addiu(scratch, scratch, -1);
+ offset = GetOffset(offset, L, OffsetSize::kOffset16);
+ bltzal(scratch, offset);
+ break;
+ case Uless:
+ Sltu(scratch, rs, rt);
+ addiu(scratch, scratch, -1);
+ offset = GetOffset(offset, L, OffsetSize::kOffset16);
+ bgezal(scratch, offset);
+ break;
+ case Uless_equal:
+ Sltu(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
+ addiu(scratch, scratch, -1);
+ offset = GetOffset(offset, L, OffsetSize::kOffset16);
+ bltzal(scratch, offset);
+ break;
+
+ default:
+ UNREACHABLE();
+ }
+
+ // Emit a nop in the branch delay slot if required.
+ if (bdslot == PROTECT) nop();
+
+ return true;
+}
+
+bool TurboAssembler::BranchAndLinkShortCheck(int32_t offset, Label* L,
+ Condition cond, Register rs,
+ const Operand& rt,
+ BranchDelaySlot bdslot) {
+ BRANCH_ARGS_CHECK(cond, rs, rt);
+
+ if (!L) {
+ if (kArchVariant == kMips64r6 && bdslot == PROTECT) {
+ DCHECK(is_int26(offset));
+ return BranchAndLinkShortHelperR6(offset, nullptr, cond, rs, rt);
+ } else {
+ DCHECK(is_int16(offset));
+ return BranchAndLinkShortHelper(offset, nullptr, cond, rs, rt, bdslot);
+ }
+ } else {
+ DCHECK_EQ(offset, 0);
+ if (kArchVariant == kMips64r6 && bdslot == PROTECT) {
+ return BranchAndLinkShortHelperR6(0, L, cond, rs, rt);
+ } else {
+ return BranchAndLinkShortHelper(0, L, cond, rs, rt, bdslot);
+ }
+ }
+ return false;
+}
+
+void TurboAssembler::LoadFromConstantsTable(Register destination,
+ int constant_index) {
+ DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kBuiltinsConstantsTable));
+ LoadRoot(destination, RootIndex::kBuiltinsConstantsTable);
+ Ld(destination,
+ FieldMemOperand(destination,
+ FixedArray::kHeaderSize + constant_index * kPointerSize));
+}
+
+void TurboAssembler::LoadRootRelative(Register destination, int32_t offset) {
+ Ld(destination, MemOperand(kRootRegister, offset));
+}
+
+void TurboAssembler::LoadRootRegisterOffset(Register destination,
+ intptr_t offset) {
+ if (offset == 0) {
+ Move(destination, kRootRegister);
+ } else {
+ Daddu(destination, kRootRegister, Operand(offset));
+ }
+}
+
+void TurboAssembler::Jump(Register target, Condition cond, Register rs,
+ const Operand& rt, BranchDelaySlot bd) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ if (kArchVariant == kMips64r6 && bd == PROTECT) {
+ if (cond == cc_always) {
+ jic(target, 0);
+ } else {
+ BRANCH_ARGS_CHECK(cond, rs, rt);
+ Branch(2, NegateCondition(cond), rs, rt);
+ jic(target, 0);
+ }
+ } else {
+ if (cond == cc_always) {
+ jr(target);
+ } else {
+ BRANCH_ARGS_CHECK(cond, rs, rt);
+ Branch(2, NegateCondition(cond), rs, rt);
+ jr(target);
+ }
+ // Emit a nop in the branch delay slot if required.
+ if (bd == PROTECT) nop();
+ }
+}
+
+void TurboAssembler::Jump(intptr_t target, RelocInfo::Mode rmode,
+ Condition cond, Register rs, const Operand& rt,
+ BranchDelaySlot bd) {
+ Label skip;
+ if (cond != cc_always) {
+ Branch(USE_DELAY_SLOT, &skip, NegateCondition(cond), rs, rt);
+ }
+ // The first instruction of 'li' may be placed in the delay slot.
+ // This is not an issue, t9 is expected to be clobbered anyway.
+ {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ li(t9, Operand(target, rmode));
+ Jump(t9, al, zero_reg, Operand(zero_reg), bd);
+ bind(&skip);
+ }
+}
+
+void TurboAssembler::Jump(Address target, RelocInfo::Mode rmode, Condition cond,
+ Register rs, const Operand& rt, BranchDelaySlot bd) {
+ DCHECK(!RelocInfo::IsCodeTarget(rmode));
+ Jump(static_cast<intptr_t>(target), rmode, cond, rs, rt, bd);
+}
+
+void TurboAssembler::Jump(Handle<Code> code, RelocInfo::Mode rmode,
+ Condition cond, Register rs, const Operand& rt,
+ BranchDelaySlot bd) {
+ DCHECK(RelocInfo::IsCodeTarget(rmode));
+
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ if (root_array_available_ && options().isolate_independent_code) {
+ IndirectLoadConstant(t9, code);
+ Daddu(t9, t9, Operand(Code::kHeaderSize - kHeapObjectTag));
+ Jump(t9, cond, rs, rt, bd);
+ return;
+ } else if (options().inline_offheap_trampolines) {
+ int builtin_index = Builtins::kNoBuiltinId;
+ if (isolate()->builtins()->IsBuiltinHandle(code, &builtin_index) &&
+ Builtins::IsIsolateIndependent(builtin_index)) {
+ // Inline the trampoline.
+ RecordCommentForOffHeapTrampoline(builtin_index);
+ CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
+ EmbeddedData d = EmbeddedData::FromBlob();
+ Address entry = d.InstructionStartOfBuiltin(builtin_index);
+ li(t9, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+ Jump(t9, cond, rs, rt, bd);
+ return;
+ }
+ }
+
+ Jump(static_cast<intptr_t>(code.address()), rmode, cond, rs, rt, bd);
+}
+
+void TurboAssembler::Jump(const ExternalReference& reference) {
+ li(t9, reference);
+ Jump(t9);
+}
+
+// Note: To call gcc-compiled C code on mips, you must call through t9.
+void TurboAssembler::Call(Register target, Condition cond, Register rs,
+ const Operand& rt, BranchDelaySlot bd) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ if (kArchVariant == kMips64r6 && bd == PROTECT) {
+ if (cond == cc_always) {
+ jialc(target, 0);
+ } else {
+ BRANCH_ARGS_CHECK(cond, rs, rt);
+ Branch(2, NegateCondition(cond), rs, rt);
+ jialc(target, 0);
+ }
+ } else {
+ if (cond == cc_always) {
+ jalr(target);
+ } else {
+ BRANCH_ARGS_CHECK(cond, rs, rt);
+ Branch(2, NegateCondition(cond), rs, rt);
+ jalr(target);
+ }
+ // Emit a nop in the branch delay slot if required.
+ if (bd == PROTECT) nop();
+ }
+ set_last_call_pc_(pc_);
+}
+
+void MacroAssembler::JumpIfIsInRange(Register value, unsigned lower_limit,
+ unsigned higher_limit,
+ Label* on_in_range) {
+ if (lower_limit != 0) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ Dsubu(scratch, value, Operand(lower_limit));
+ Branch(on_in_range, ls, scratch, Operand(higher_limit - lower_limit));
+ } else {
+ Branch(on_in_range, ls, value, Operand(higher_limit - lower_limit));
+ }
+}
+
+void TurboAssembler::Call(Address target, RelocInfo::Mode rmode, Condition cond,
+ Register rs, const Operand& rt, BranchDelaySlot bd) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ li(t9, Operand(static_cast<int64_t>(target), rmode), ADDRESS_LOAD);
+ Call(t9, cond, rs, rt, bd);
+}
+
+void TurboAssembler::Call(Handle<Code> code, RelocInfo::Mode rmode,
+ Condition cond, Register rs, const Operand& rt,
+ BranchDelaySlot bd) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+
+ if (root_array_available_ && options().isolate_independent_code) {
+ IndirectLoadConstant(t9, code);
+ Daddu(t9, t9, Operand(Code::kHeaderSize - kHeapObjectTag));
+ Call(t9, cond, rs, rt, bd);
+ return;
+ } else if (options().inline_offheap_trampolines) {
+ int builtin_index = Builtins::kNoBuiltinId;
+ if (isolate()->builtins()->IsBuiltinHandle(code, &builtin_index) &&
+ Builtins::IsIsolateIndependent(builtin_index)) {
+ // Inline the trampoline.
+ RecordCommentForOffHeapTrampoline(builtin_index);
+ CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
+ EmbeddedData d = EmbeddedData::FromBlob();
+ Address entry = d.InstructionStartOfBuiltin(builtin_index);
+ li(t9, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+ Call(t9, cond, rs, rt, bd);
+ return;
+ }
+ }
+
+ DCHECK(RelocInfo::IsCodeTarget(rmode));
+ DCHECK(code->IsExecutable());
+ Call(code.address(), rmode, cond, rs, rt, bd);
+}
+
+void TurboAssembler::LoadEntryFromBuiltinIndex(Register builtin_index) {
+ STATIC_ASSERT(kSystemPointerSize == 8);
+ STATIC_ASSERT(kSmiTagSize == 1);
+ STATIC_ASSERT(kSmiTag == 0);
+
+ // The builtin_index register contains the builtin index as a Smi.
+ SmiUntag(builtin_index, builtin_index);
+ Dlsa(builtin_index, kRootRegister, builtin_index, kSystemPointerSizeLog2);
+ Ld(builtin_index,
+ MemOperand(builtin_index, IsolateData::builtin_entry_table_offset()));
+}
+
+void TurboAssembler::CallBuiltinByIndex(Register builtin_index) {
+ LoadEntryFromBuiltinIndex(builtin_index);
+ Call(builtin_index);
+}
+
+void TurboAssembler::PatchAndJump(Address target) {
+ if (kArchVariant != kMips64r6) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ mov(scratch, ra);
+ bal(1); // jump to ld
+ nop(); // in the delay slot
+ ld(t9, MemOperand(ra, kInstrSize * 3)); // ra == pc_
+ jr(t9);
+ mov(ra, scratch); // in delay slot
+ DCHECK_EQ(reinterpret_cast<uint64_t>(pc_) % 8, 0);
+ *reinterpret_cast<uint64_t*>(pc_) = target; // pc_ should be align.
+ pc_ += sizeof(uint64_t);
+ } else {
+ // TODO(mips r6): Implement.
+ UNIMPLEMENTED();
+ }
+}
+
+void TurboAssembler::StoreReturnAddressAndCall(Register target) {
+ // This generates the final instruction sequence for calls to C functions
+ // once an exit frame has been constructed.
+ //
+ // Note that this assumes the caller code (i.e. the Code object currently
+ // being generated) is immovable or that the callee function cannot trigger
+ // GC, since the callee function will return to it.
+
+ // Compute the return address in lr to return to after the jump below. The pc
+ // is already at '+ 8' from the current instruction; but return is after three
+ // instructions, so add another 4 to pc to get the return address.
+
+ Assembler::BlockTrampolinePoolScope block_trampoline_pool(this);
+ static constexpr int kNumInstructionsToJump = 4;
+ Label find_ra;
+ // Adjust the value in ra to point to the correct return location, 2nd
+ // instruction past the real call into C code (the jalr(t9)), and push it.
+ // This is the return address of the exit frame.
+ if (kArchVariant >= kMips64r6) {
+ addiupc(ra, kNumInstructionsToJump + 1);
+ } else {
+ // This no-op-and-link sequence saves PC + 8 in ra register on pre-r6 MIPS
+ nal(); // nal has branch delay slot.
+ Daddu(ra, ra, kNumInstructionsToJump * kInstrSize);
+ }
+ bind(&find_ra);
+
+ // This spot was reserved in EnterExitFrame.
+ Sd(ra, MemOperand(sp));
+ // Stack space reservation moved to the branch delay slot below.
+ // Stack is still aligned.
+
+ // Call the C routine.
+ mov(t9, target); // Function pointer to t9 to conform to ABI for PIC.
+ jalr(t9);
+ // Set up sp in the delay slot.
+ daddiu(sp, sp, -kCArgsSlotsSize);
+ // Make sure the stored 'ra' points to this position.
+ DCHECK_EQ(kNumInstructionsToJump, InstructionsGeneratedSince(&find_ra));
+}
+
+void TurboAssembler::Ret(Condition cond, Register rs, const Operand& rt,
+ BranchDelaySlot bd) {
+ Jump(ra, cond, rs, rt, bd);
+}
+
+void TurboAssembler::BranchLong(Label* L, BranchDelaySlot bdslot) {
+ if (kArchVariant == kMips64r6 && bdslot == PROTECT &&
+ (!L->is_bound() || is_near_r6(L))) {
+ BranchShortHelperR6(0, L);
+ } else {
+ // Generate position independent long branch.
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ int64_t imm64 = branch_long_offset(L);
+ DCHECK(is_int32(imm64));
+ int32_t imm32 = static_cast<int32_t>(imm64);
+ or_(t8, ra, zero_reg);
+ nal(); // Read PC into ra register.
+ lui(t9, (imm32 & kHiMaskOf32) >> kLuiShift); // Branch delay slot.
+ ori(t9, t9, (imm32 & kImm16Mask));
+ daddu(t9, ra, t9);
+ if (bdslot == USE_DELAY_SLOT) {
+ or_(ra, t8, zero_reg);
+ }
+ jr(t9);
+ // Emit a or_ in the branch delay slot if it's protected.
+ if (bdslot == PROTECT) or_(ra, t8, zero_reg);
+ }
+}
+
+void TurboAssembler::BranchAndLinkLong(Label* L, BranchDelaySlot bdslot) {
+ if (kArchVariant == kMips64r6 && bdslot == PROTECT &&
+ (!L->is_bound() || is_near_r6(L))) {
+ BranchAndLinkShortHelperR6(0, L);
+ } else {
+ // Generate position independent long branch and link.
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ int64_t imm64 = branch_long_offset(L);
+ DCHECK(is_int32(imm64));
+ int32_t imm32 = static_cast<int32_t>(imm64);
+ lui(t8, (imm32 & kHiMaskOf32) >> kLuiShift);
+ nal(); // Read PC into ra register.
+ ori(t8, t8, (imm32 & kImm16Mask)); // Branch delay slot.
+ daddu(t8, ra, t8);
+ jalr(t8);
+ // Emit a nop in the branch delay slot if required.
+ if (bdslot == PROTECT) nop();
+ }
+}
+
+void TurboAssembler::DropAndRet(int drop) {
+ int32_t drop_size = drop * kSystemPointerSize;
+ DCHECK(is_int31(drop_size));
+
+ if (is_int16(drop_size)) {
+ Ret(USE_DELAY_SLOT);
+ daddiu(sp, sp, drop_size);
+ } else {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ li(scratch, drop_size);
+ Ret(USE_DELAY_SLOT);
+ daddu(sp, sp, scratch);
+ }
+}
+
+void TurboAssembler::DropAndRet(int drop, Condition cond, Register r1,
+ const Operand& r2) {
+ // Both Drop and Ret need to be conditional.
+ Label skip;
+ if (cond != cc_always) {
+ Branch(&skip, NegateCondition(cond), r1, r2);
+ }
+
+ Drop(drop);
+ Ret();
+
+ if (cond != cc_always) {
+ bind(&skip);
+ }
+}
+
+void TurboAssembler::Drop(int count, Condition cond, Register reg,
+ const Operand& op) {
+ if (count <= 0) {
+ return;
+ }
+
+ Label skip;
+
+ if (cond != al) {
+ Branch(&skip, NegateCondition(cond), reg, op);
+ }
+
+ Daddu(sp, sp, Operand(count * kPointerSize));
+
+ if (cond != al) {
+ bind(&skip);
+ }
+}
+
+void MacroAssembler::Swap(Register reg1, Register reg2, Register scratch) {
+ if (scratch == no_reg) {
+ Xor(reg1, reg1, Operand(reg2));
+ Xor(reg2, reg2, Operand(reg1));
+ Xor(reg1, reg1, Operand(reg2));
+ } else {
+ mov(scratch, reg1);
+ mov(reg1, reg2);
+ mov(reg2, scratch);
+ }
+}
+
+void TurboAssembler::Call(Label* target) { BranchAndLink(target); }
+
+void TurboAssembler::LoadAddress(Register dst, Label* target) {
+ uint64_t address = jump_address(target);
+ li(dst, address);
+}
+
+void TurboAssembler::Push(Smi smi) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ li(scratch, Operand(smi));
+ push(scratch);
+}
+
+void TurboAssembler::Push(Handle<HeapObject> handle) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ li(scratch, Operand(handle));
+ push(scratch);
+}
+
+void TurboAssembler::PushArray(Register array, Register size, Register scratch,
+ Register scratch2, PushArrayOrder order) {
+ DCHECK(!AreAliased(array, size, scratch, scratch2));
+ Label loop, entry;
+ if (order == PushArrayOrder::kReverse) {
+ mov(scratch, zero_reg);
+ jmp(&entry);
+ bind(&loop);
+ Dlsa(scratch2, array, scratch, kPointerSizeLog2);
+ Ld(scratch2, MemOperand(scratch2));
+ push(scratch2);
+ Daddu(scratch, scratch, Operand(1));
+ bind(&entry);
+ Branch(&loop, less, scratch, Operand(size));
+ } else {
+ mov(scratch, size);
+ jmp(&entry);
+ bind(&loop);
+ Dlsa(scratch2, array, scratch, kPointerSizeLog2);
+ Ld(scratch2, MemOperand(scratch2));
+ push(scratch2);
+ bind(&entry);
+ Daddu(scratch, scratch, Operand(-1));
+ Branch(&loop, greater_equal, scratch, Operand(zero_reg));
+ }
+}
+
+void MacroAssembler::MaybeDropFrames() {
+ // Check whether we need to drop frames to restart a function on the stack.
+ li(a1, ExternalReference::debug_restart_fp_address(isolate()));
+ Ld(a1, MemOperand(a1));
+ Jump(BUILTIN_CODE(isolate(), FrameDropperTrampoline), RelocInfo::CODE_TARGET,
+ ne, a1, Operand(zero_reg));
+}
+
+// ---------------------------------------------------------------------------
+// Exception handling.
+
+void MacroAssembler::PushStackHandler() {
+ // Adjust this code if not the case.
+ STATIC_ASSERT(StackHandlerConstants::kSize == 2 * kPointerSize);
+ STATIC_ASSERT(StackHandlerConstants::kNextOffset == 0 * kPointerSize);
+
+ Push(Smi::zero()); // Padding.
+
+ // Link the current handler as the next handler.
+ li(t2,
+ ExternalReference::Create(IsolateAddressId::kHandlerAddress, isolate()));
+ Ld(t1, MemOperand(t2));
+ push(t1);
+
+ // Set this new handler as the current one.
+ Sd(sp, MemOperand(t2));
+}
+
+void MacroAssembler::PopStackHandler() {
+ STATIC_ASSERT(StackHandlerConstants::kNextOffset == 0);
+ pop(a1);
+ Daddu(sp, sp,
+ Operand(
+ static_cast<int64_t>(StackHandlerConstants::kSize - kPointerSize)));
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ li(scratch,
+ ExternalReference::Create(IsolateAddressId::kHandlerAddress, isolate()));
+ Sd(a1, MemOperand(scratch));
+}
+
+void TurboAssembler::FPUCanonicalizeNaN(const DoubleRegister dst,
+ const DoubleRegister src) {
+ sub_d(dst, src, kDoubleRegZero);
+}
+
+void TurboAssembler::MovFromFloatResult(const DoubleRegister dst) {
+ if (IsMipsSoftFloatABI) {
+ if (kArchEndian == kLittle) {
+ Move(dst, v0, v1);
+ } else {
+ Move(dst, v1, v0);
+ }
+ } else {
+ Move(dst, f0); // Reg f0 is o32 ABI FP return value.
+ }
+}
+
+void TurboAssembler::MovFromFloatParameter(const DoubleRegister dst) {
+ if (IsMipsSoftFloatABI) {
+ if (kArchEndian == kLittle) {
+ Move(dst, a0, a1);
+ } else {
+ Move(dst, a1, a0);
+ }
+ } else {
+ Move(dst, f12); // Reg f12 is n64 ABI FP first argument value.
+ }
+}
+
+void TurboAssembler::MovToFloatParameter(DoubleRegister src) {
+ if (!IsMipsSoftFloatABI) {
+ Move(f12, src);
+ } else {
+ if (kArchEndian == kLittle) {
+ Move(a0, a1, src);
+ } else {
+ Move(a1, a0, src);
+ }
+ }
+}
+
+void TurboAssembler::MovToFloatResult(DoubleRegister src) {
+ if (!IsMipsSoftFloatABI) {
+ Move(f0, src);
+ } else {
+ if (kArchEndian == kLittle) {
+ Move(v0, v1, src);
+ } else {
+ Move(v1, v0, src);
+ }
+ }
+}
+
+void TurboAssembler::MovToFloatParameters(DoubleRegister src1,
+ DoubleRegister src2) {
+ if (!IsMipsSoftFloatABI) {
+ const DoubleRegister fparg2 = f13;
+ if (src2 == f12) {
+ DCHECK(src1 != fparg2);
+ Move(fparg2, src2);
+ Move(f12, src1);
+ } else {
+ Move(f12, src1);
+ Move(fparg2, src2);
+ }
+ } else {
+ if (kArchEndian == kLittle) {
+ Move(a0, a1, src1);
+ Move(a2, a3, src2);
+ } else {
+ Move(a1, a0, src1);
+ Move(a3, a2, src2);
+ }
+ }
+}
+
+// -----------------------------------------------------------------------------
+// JavaScript invokes.
+
+void TurboAssembler::PrepareForTailCall(Register callee_args_count,
+ Register caller_args_count,
+ Register scratch0, Register scratch1) {
+ // Calculate the end of destination area where we will put the arguments
+ // after we drop current frame. We add kPointerSize to count the receiver
+ // argument which is not included into formal parameters count.
+ Register dst_reg = scratch0;
+ Dlsa(dst_reg, fp, caller_args_count, kPointerSizeLog2);
+ Daddu(dst_reg, dst_reg,
+ Operand(StandardFrameConstants::kCallerSPOffset + kPointerSize));
+
+ Register src_reg = caller_args_count;
+ // Calculate the end of source area. +kPointerSize is for the receiver.
+ Dlsa(src_reg, sp, callee_args_count, kPointerSizeLog2);
+ Daddu(src_reg, src_reg, Operand(kPointerSize));
+
+ if (FLAG_debug_code) {
+ Check(lo, AbortReason::kStackAccessBelowStackPointer, src_reg,
+ Operand(dst_reg));
+ }
+
+ // Restore caller's frame pointer and return address now as they will be
+ // overwritten by the copying loop.
+ Ld(ra, MemOperand(fp, StandardFrameConstants::kCallerPCOffset));
+ Ld(fp, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
+
+ // Now copy callee arguments to the caller frame going backwards to avoid
+ // callee arguments corruption (source and destination areas could overlap).
+
+ // Both src_reg and dst_reg are pointing to the word after the one to copy,
+ // so they must be pre-decremented in the loop.
+ Register tmp_reg = scratch1;
+ Label loop, entry;
+ Branch(&entry);
+ bind(&loop);
+ Dsubu(src_reg, src_reg, Operand(kPointerSize));
+ Dsubu(dst_reg, dst_reg, Operand(kPointerSize));
+ Ld(tmp_reg, MemOperand(src_reg));
+ Sd(tmp_reg, MemOperand(dst_reg));
+ bind(&entry);
+ Branch(&loop, ne, sp, Operand(src_reg));
+
+ // Leave current frame.
+ mov(sp, dst_reg);
+}
+
+void MacroAssembler::LoadStackLimit(Register destination, StackLimitKind kind) {
+ DCHECK(root_array_available());
+ Isolate* isolate = this->isolate();
+ ExternalReference limit =
+ kind == StackLimitKind::kRealStackLimit
+ ? ExternalReference::address_of_real_jslimit(isolate)
+ : ExternalReference::address_of_jslimit(isolate);
+ DCHECK(TurboAssembler::IsAddressableThroughRootRegister(isolate, limit));
+
+ intptr_t offset =
+ TurboAssembler::RootRegisterOffsetForExternalReference(isolate, limit);
+ CHECK(is_int32(offset));
+ Ld(destination, MemOperand(kRootRegister, static_cast<int32_t>(offset)));
+}
+
+void MacroAssembler::StackOverflowCheck(Register num_args, Register scratch1,
+ Register scratch2,
+ Label* stack_overflow) {
+ // Check the stack for overflow. We are not trying to catch
+ // interruptions (e.g. debug break and preemption) here, so the "real stack
+ // limit" is checked.
+
+ LoadStackLimit(scratch1, StackLimitKind::kRealStackLimit);
+ // Make scratch1 the space we have left. The stack might already be overflowed
+ // here which will cause scratch1 to become negative.
+ dsubu(scratch1, sp, scratch1);
+ // Check if the arguments will overflow the stack.
+ dsll(scratch2, num_args, kPointerSizeLog2);
+ // Signed comparison.
+ Branch(stack_overflow, le, scratch1, Operand(scratch2));
+}
+
+void MacroAssembler::InvokePrologue(Register expected_parameter_count,
+ Register actual_parameter_count,
+ Label* done, InvokeFlag flag) {
+ Label regular_invoke;
+
+ // a0: actual arguments count
+ // a1: function (passed through to callee)
+ // a2: expected arguments count
+
+ DCHECK_EQ(actual_parameter_count, a0);
+ DCHECK_EQ(expected_parameter_count, a2);
+
+#ifdef V8_NO_ARGUMENTS_ADAPTOR
+ // If the expected parameter count is equal to the adaptor sentinel, no need
+ // to push undefined value as arguments.
+ Branch(®ular_invoke, eq, expected_parameter_count,
+ Operand(kDontAdaptArgumentsSentinel));
+
+ // If overapplication or if the actual argument count is equal to the
+ // formal parameter count, no need to push extra undefined values.
+ Dsubu(expected_parameter_count, expected_parameter_count,
+ actual_parameter_count);
+ Branch(®ular_invoke, le, expected_parameter_count, Operand(zero_reg));
+
+ Label stack_overflow;
+ StackOverflowCheck(expected_parameter_count, t0, t1, &stack_overflow);
+ // Underapplication. Move the arguments already in the stack, including the
+ // receiver and the return address.
+ {
+ Label copy;
+ Register src = a6, dest = a7;
+ mov(src, sp);
+ dsll(t0, expected_parameter_count, kSystemPointerSizeLog2);
+ Dsubu(sp, sp, Operand(t0));
+ // Update stack pointer.
+ mov(dest, sp);
+ mov(t0, actual_parameter_count);
+ bind(©);
+ Ld(t1, MemOperand(src, 0));
+ Sd(t1, MemOperand(dest, 0));
+ Dsubu(t0, t0, Operand(1));
+ Daddu(src, src, Operand(kSystemPointerSize));
+ Daddu(dest, dest, Operand(kSystemPointerSize));
+ Branch(©, ge, t0, Operand(zero_reg));
+ }
+
+ // Fill remaining expected arguments with undefined values.
+ LoadRoot(t0, RootIndex::kUndefinedValue);
+ {
+ Label loop;
+ bind(&loop);
+ Sd(t0, MemOperand(a7, 0));
+ Dsubu(expected_parameter_count, expected_parameter_count, Operand(1));
+ Daddu(a7, a7, Operand(kSystemPointerSize));
+ Branch(&loop, gt, expected_parameter_count, Operand(zero_reg));
+ }
+ b(®ular_invoke);
+ nop();
+
+ bind(&stack_overflow);
+ {
+ FrameScope frame(this,
+ has_frame() ? StackFrame::NONE : StackFrame::INTERNAL);
+ CallRuntime(Runtime::kThrowStackOverflow);
+ break_(0xCC);
+ }
+#else
+ // Check whether the expected and actual arguments count match. The registers
+ // are set up according to contract with ArgumentsAdaptorTrampoline:
+
+ Branch(®ular_invoke, eq, expected_parameter_count,
+ Operand(actual_parameter_count));
+
+ Handle<Code> adaptor = BUILTIN_CODE(isolate(), ArgumentsAdaptorTrampoline);
+ if (flag == CALL_FUNCTION) {
+ Call(adaptor);
+ Branch(done);
+ } else {
+ Jump(adaptor, RelocInfo::CODE_TARGET);
+ }
+#endif
+ bind(®ular_invoke);
+}
+
+void MacroAssembler::CheckDebugHook(Register fun, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count) {
+ Label skip_hook;
+
+ li(t0, ExternalReference::debug_hook_on_function_call_address(isolate()));
+ Lb(t0, MemOperand(t0));
+ Branch(&skip_hook, eq, t0, Operand(zero_reg));
+
+ {
+ // Load receiver to pass it later to DebugOnFunctionCall hook.
+ LoadReceiver(t0, actual_parameter_count);
+
+ FrameScope frame(this,
+ has_frame() ? StackFrame::NONE : StackFrame::INTERNAL);
+ SmiTag(expected_parameter_count);
+ Push(expected_parameter_count);
+
+ SmiTag(actual_parameter_count);
+ Push(actual_parameter_count);
+
+ if (new_target.is_valid()) {
+ Push(new_target);
+ }
+ Push(fun);
+ Push(fun);
+ Push(t0);
+ CallRuntime(Runtime::kDebugOnFunctionCall);
+ Pop(fun);
+ if (new_target.is_valid()) {
+ Pop(new_target);
+ }
+
+ Pop(actual_parameter_count);
+ SmiUntag(actual_parameter_count);
+
+ Pop(expected_parameter_count);
+ SmiUntag(expected_parameter_count);
+ }
+ bind(&skip_hook);
+}
+
+void MacroAssembler::InvokeFunctionCode(Register function, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count,
+ InvokeFlag flag) {
+ // You can't call a function without a valid frame.
+ DCHECK_IMPLIES(flag == CALL_FUNCTION, has_frame());
+ DCHECK_EQ(function, a1);
+ DCHECK_IMPLIES(new_target.is_valid(), new_target == a3);
+
+ // On function call, call into the debugger if necessary.
+ CheckDebugHook(function, new_target, expected_parameter_count,
+ actual_parameter_count);
+
+ // Clear the new.target register if not given.
+ if (!new_target.is_valid()) {
+ LoadRoot(a3, RootIndex::kUndefinedValue);
+ }
+
+ Label done;
+ InvokePrologue(expected_parameter_count, actual_parameter_count, &done, flag);
+ // We call indirectly through the code field in the function to
+ // allow recompilation to take effect without changing any of the
+ // call sites.
+ Register code = kJavaScriptCallCodeStartRegister;
+ Ld(code, FieldMemOperand(function, JSFunction::kCodeOffset));
+ if (flag == CALL_FUNCTION) {
+ Daddu(code, code, Operand(Code::kHeaderSize - kHeapObjectTag));
+ Call(code);
+ } else {
+ DCHECK(flag == JUMP_FUNCTION);
+ Daddu(code, code, Operand(Code::kHeaderSize - kHeapObjectTag));
+ Jump(code);
+ }
+
+ // Continue here if InvokePrologue does handle the invocation due to
+ // mismatched parameter counts.
+ bind(&done);
+}
+
+void MacroAssembler::InvokeFunctionWithNewTarget(
+ Register function, Register new_target, Register actual_parameter_count,
+ InvokeFlag flag) {
+ // You can't call a function without a valid frame.
+ DCHECK_IMPLIES(flag == CALL_FUNCTION, has_frame());
+
+ // Contract with called JS functions requires that function is passed in a1.
+ DCHECK_EQ(function, a1);
+ Register expected_parameter_count = a2;
+ Register temp_reg = t0;
+ Ld(temp_reg, FieldMemOperand(a1, JSFunction::kSharedFunctionInfoOffset));
+ Ld(cp, FieldMemOperand(a1, JSFunction::kContextOffset));
+ // The argument count is stored as uint16_t
+ Lhu(expected_parameter_count,
+ FieldMemOperand(temp_reg,
+ SharedFunctionInfo::kFormalParameterCountOffset));
+
+ InvokeFunctionCode(a1, new_target, expected_parameter_count,
+ actual_parameter_count, flag);
+}
+
+void MacroAssembler::InvokeFunction(Register function,
+ Register expected_parameter_count,
+ Register actual_parameter_count,
+ InvokeFlag flag) {
+ // You can't call a function without a valid frame.
+ DCHECK_IMPLIES(flag == CALL_FUNCTION, has_frame());
+
+ // Contract with called JS functions requires that function is passed in a1.
+ DCHECK_EQ(function, a1);
+
+ // Get the function and setup the context.
+ Ld(cp, FieldMemOperand(a1, JSFunction::kContextOffset));
+
+ InvokeFunctionCode(a1, no_reg, expected_parameter_count,
+ actual_parameter_count, flag);
+}
+
+// ---------------------------------------------------------------------------
+// Support functions.
+
+void MacroAssembler::GetObjectType(Register object, Register map,
+ Register type_reg) {
+ LoadMap(map, object);
+ Lhu(type_reg, FieldMemOperand(map, Map::kInstanceTypeOffset));
+}
+
+// -----------------------------------------------------------------------------
+// Runtime calls.
+
+void TurboAssembler::DaddOverflow(Register dst, Register left,
+ const Operand& right, Register overflow) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Register right_reg = no_reg;
+ Register scratch = t8;
+ if (!right.is_reg()) {
+ li(at, Operand(right));
+ right_reg = at;
+ } else {
+ right_reg = right.rm();
+ }
+
+ DCHECK(left != scratch && right_reg != scratch && dst != scratch &&
+ overflow != scratch);
+ DCHECK(overflow != left && overflow != right_reg);
+
+ if (dst == left || dst == right_reg) {
+ daddu(scratch, left, right_reg);
+ xor_(overflow, scratch, left);
+ xor_(at, scratch, right_reg);
+ and_(overflow, overflow, at);
+ mov(dst, scratch);
+ } else {
+ daddu(dst, left, right_reg);
+ xor_(overflow, dst, left);
+ xor_(at, dst, right_reg);
+ and_(overflow, overflow, at);
+ }
+}
+
+void TurboAssembler::DsubOverflow(Register dst, Register left,
+ const Operand& right, Register overflow) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Register right_reg = no_reg;
+ Register scratch = t8;
+ if (!right.is_reg()) {
+ li(at, Operand(right));
+ right_reg = at;
+ } else {
+ right_reg = right.rm();
+ }
+
+ DCHECK(left != scratch && right_reg != scratch && dst != scratch &&
+ overflow != scratch);
+ DCHECK(overflow != left && overflow != right_reg);
+
+ if (dst == left || dst == right_reg) {
+ dsubu(scratch, left, right_reg);
+ xor_(overflow, left, scratch);
+ xor_(at, left, right_reg);
+ and_(overflow, overflow, at);
+ mov(dst, scratch);
+ } else {
+ dsubu(dst, left, right_reg);
+ xor_(overflow, left, dst);
+ xor_(at, left, right_reg);
+ and_(overflow, overflow, at);
+ }
+}
+
+void TurboAssembler::MulOverflow(Register dst, Register left,
+ const Operand& right, Register overflow) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Register right_reg = no_reg;
+ Register scratch = t8;
+ if (!right.is_reg()) {
+ li(at, Operand(right));
+ right_reg = at;
+ } else {
+ right_reg = right.rm();
+ }
+
+ DCHECK(left != scratch && right_reg != scratch && dst != scratch &&
+ overflow != scratch);
+ DCHECK(overflow != left && overflow != right_reg);
+
+ if (dst == left || dst == right_reg) {
+ Mul(scratch, left, right_reg);
+ Mulh(overflow, left, right_reg);
+ mov(dst, scratch);
+ } else {
+ Mul(dst, left, right_reg);
+ Mulh(overflow, left, right_reg);
+ }
+
+ dsra32(scratch, dst, 0);
+ xor_(overflow, overflow, scratch);
+}
+
+void MacroAssembler::CallRuntime(const Runtime::Function* f, int num_arguments,
+ SaveFPRegsMode save_doubles) {
+ // All parameters are on the stack. v0 has the return value after call.
+
+ // If the expected number of arguments of the runtime function is
+ // constant, we check that the actual number of arguments match the
+ // expectation.
+ CHECK(f->nargs < 0 || f->nargs == num_arguments);
+
+ // TODO(1236192): Most runtime routines don't need the number of
+ // arguments passed in because it is constant. At some point we
+ // should remove this need and make the runtime routine entry code
+ // smarter.
+ PrepareCEntryArgs(num_arguments);
+ PrepareCEntryFunction(ExternalReference::Create(f));
+ Handle<Code> code =
+ CodeFactory::CEntry(isolate(), f->result_size, save_doubles);
+ Call(code, RelocInfo::CODE_TARGET);
+}
+
+void MacroAssembler::TailCallRuntime(Runtime::FunctionId fid) {
+ const Runtime::Function* function = Runtime::FunctionForId(fid);
+ DCHECK_EQ(1, function->result_size);
+ if (function->nargs >= 0) {
+ PrepareCEntryArgs(function->nargs);
+ }
+ JumpToExternalReference(ExternalReference::Create(fid));
+}
+
+void MacroAssembler::JumpToExternalReference(const ExternalReference& builtin,
+ BranchDelaySlot bd,
+ bool builtin_exit_frame) {
+ PrepareCEntryFunction(builtin);
+ Handle<Code> code = CodeFactory::CEntry(isolate(), 1, kDontSaveFPRegs,
+ kArgvOnStack, builtin_exit_frame);
+ Jump(code, RelocInfo::CODE_TARGET, al, zero_reg, Operand(zero_reg), bd);
+}
+
+void MacroAssembler::JumpToInstructionStream(Address entry) {
+ li(kOffHeapTrampolineRegister, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+ Jump(kOffHeapTrampolineRegister);
+}
+
+void MacroAssembler::LoadWeakValue(Register out, Register in,
+ Label* target_if_cleared) {
+ Branch(target_if_cleared, eq, in, Operand(kClearedWeakHeapObjectLower32));
+
+ And(out, in, Operand(~kWeakHeapObjectMask));
+}
+
+void MacroAssembler::IncrementCounter(StatsCounter* counter, int value,
+ Register scratch1, Register scratch2) {
+ DCHECK_GT(value, 0);
+ if (FLAG_native_code_counters && counter->Enabled()) {
+ // This operation has to be exactly 32-bit wide in case the external
+ // reference table redirects the counter to a uint32_t dummy_stats_counter_
+ // field.
+ li(scratch2, ExternalReference::Create(counter));
+ Lw(scratch1, MemOperand(scratch2));
+ Addu(scratch1, scratch1, Operand(value));
+ Sw(scratch1, MemOperand(scratch2));
+ }
+}
+
+void MacroAssembler::DecrementCounter(StatsCounter* counter, int value,
+ Register scratch1, Register scratch2) {
+ DCHECK_GT(value, 0);
+ if (FLAG_native_code_counters && counter->Enabled()) {
+ // This operation has to be exactly 32-bit wide in case the external
+ // reference table redirects the counter to a uint32_t dummy_stats_counter_
+ // field.
+ li(scratch2, ExternalReference::Create(counter));
+ Lw(scratch1, MemOperand(scratch2));
+ Subu(scratch1, scratch1, Operand(value));
+ Sw(scratch1, MemOperand(scratch2));
+ }
+}
+
+// -----------------------------------------------------------------------------
+// Debugging.
+
+void TurboAssembler::Trap() { stop(); }
+void TurboAssembler::DebugBreak() { stop(); }
+
+void TurboAssembler::Assert(Condition cc, AbortReason reason, Register rs,
+ Operand rt) {
+ if (emit_debug_code()) Check(cc, reason, rs, rt);
+}
+
+void TurboAssembler::Check(Condition cc, AbortReason reason, Register rs,
+ Operand rt) {
+ Label L;
+ Branch(&L, cc, rs, rt);
+ Abort(reason);
+ // Will not return here.
+ bind(&L);
+}
+
+void TurboAssembler::Abort(AbortReason reason) {
+ Label abort_start;
+ bind(&abort_start);
+#ifdef DEBUG
+ const char* msg = GetAbortReason(reason);
+ RecordComment("Abort message: ");
+ RecordComment(msg);
+#endif
+
+ // Avoid emitting call to builtin if requested.
+ if (trap_on_abort()) {
+ stop();
+ return;
+ }
+
+ if (should_abort_hard()) {
+ // We don't care if we constructed a frame. Just pretend we did.
+ FrameScope assume_frame(this, StackFrame::NONE);
+ PrepareCallCFunction(0, a0);
+ li(a0, Operand(static_cast<int>(reason)));
+ CallCFunction(ExternalReference::abort_with_reason(), 1);
+ return;
+ }
+
+ Move(a0, Smi::FromInt(static_cast<int>(reason)));
+
+ // Disable stub call restrictions to always allow calls to abort.
+ if (!has_frame()) {
+ // We don't actually want to generate a pile of code for this, so just
+ // claim there is a stack frame, without generating one.
+ FrameScope scope(this, StackFrame::NONE);
+ Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
+ } else {
+ Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
+ }
+ // Will not return here.
+ if (is_trampoline_pool_blocked()) {
+ // If the calling code cares about the exact number of
+ // instructions generated, we insert padding here to keep the size
+ // of the Abort macro constant.
+ // Currently in debug mode with debug_code enabled the number of
+ // generated instructions is 10, so we use this as a maximum value.
+ static const int kExpectedAbortInstructions = 10;
+ int abort_instructions = InstructionsGeneratedSince(&abort_start);
+ DCHECK_LE(abort_instructions, kExpectedAbortInstructions);
+ while (abort_instructions++ < kExpectedAbortInstructions) {
+ nop();
+ }
+ }
+}
+
+void MacroAssembler::LoadMap(Register destination, Register object) {
+ Ld(destination, FieldMemOperand(object, HeapObject::kMapOffset));
+}
+
+void MacroAssembler::LoadNativeContextSlot(int index, Register dst) {
+ LoadMap(dst, cp);
+ Ld(dst,
+ FieldMemOperand(dst, Map::kConstructorOrBackPointerOrNativeContextOffset));
+ Ld(dst, MemOperand(dst, Context::SlotOffset(index)));
+}
+
+void TurboAssembler::StubPrologue(StackFrame::Type type) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ li(scratch, Operand(StackFrame::TypeToMarker(type)));
+ PushCommonFrame(scratch);
+}
+
+void TurboAssembler::Prologue() { PushStandardFrame(a1); }
+
+void TurboAssembler::EnterFrame(StackFrame::Type type) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ int stack_offset = -3 * kPointerSize;
+ const int fp_offset = 1 * kPointerSize;
+ daddiu(sp, sp, stack_offset);
+ stack_offset = -stack_offset - kPointerSize;
+ Sd(ra, MemOperand(sp, stack_offset));
+ stack_offset -= kPointerSize;
+ Sd(fp, MemOperand(sp, stack_offset));
+ stack_offset -= kPointerSize;
+ li(t9, Operand(StackFrame::TypeToMarker(type)));
+ Sd(t9, MemOperand(sp, stack_offset));
+ // Adjust FP to point to saved FP.
+ DCHECK_EQ(stack_offset, 0);
+ Daddu(fp, sp, Operand(fp_offset));
+}
+
+void TurboAssembler::LeaveFrame(StackFrame::Type type) {
+ daddiu(sp, fp, 2 * kPointerSize);
+ Ld(ra, MemOperand(fp, 1 * kPointerSize));
+ Ld(fp, MemOperand(fp, 0 * kPointerSize));
+}
+
+void MacroAssembler::EnterExitFrame(bool save_doubles, int stack_space,
+ StackFrame::Type frame_type) {
+ DCHECK(frame_type == StackFrame::EXIT ||
+ frame_type == StackFrame::BUILTIN_EXIT);
+
+ // Set up the frame structure on the stack.
+ STATIC_ASSERT(2 * kPointerSize == ExitFrameConstants::kCallerSPDisplacement);
+ STATIC_ASSERT(1 * kPointerSize == ExitFrameConstants::kCallerPCOffset);
+ STATIC_ASSERT(0 * kPointerSize == ExitFrameConstants::kCallerFPOffset);
+
+ // This is how the stack will look:
+ // fp + 2 (==kCallerSPDisplacement) - old stack's end
+ // [fp + 1 (==kCallerPCOffset)] - saved old ra
+ // [fp + 0 (==kCallerFPOffset)] - saved old fp
+ // [fp - 1 StackFrame::EXIT Smi
+ // [fp - 2 (==kSPOffset)] - sp of the called function
+ // fp - (2 + stack_space + alignment) == sp == [fp - kSPOffset] - top of the
+ // new stack (will contain saved ra)
+
+ // Save registers and reserve room for saved entry sp.
+ daddiu(sp, sp, -2 * kPointerSize - ExitFrameConstants::kFixedFrameSizeFromFp);
+ Sd(ra, MemOperand(sp, 3 * kPointerSize));
+ Sd(fp, MemOperand(sp, 2 * kPointerSize));
+ {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ li(scratch, Operand(StackFrame::TypeToMarker(frame_type)));
+ Sd(scratch, MemOperand(sp, 1 * kPointerSize));
+ }
+ // Set up new frame pointer.
+ daddiu(fp, sp, ExitFrameConstants::kFixedFrameSizeFromFp);
+
+ if (emit_debug_code()) {
+ Sd(zero_reg, MemOperand(fp, ExitFrameConstants::kSPOffset));
+ }
+
+ {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ // Save the frame pointer and the context in top.
+ li(t8, ExternalReference::Create(IsolateAddressId::kCEntryFPAddress,
+ isolate()));
+ Sd(fp, MemOperand(t8));
+ li(t8,
+ ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
+ Sd(cp, MemOperand(t8));
+ }
+
+ const int frame_alignment = MacroAssembler::ActivationFrameAlignment();
+ if (save_doubles) {
+ // The stack is already aligned to 0 modulo 8 for stores with sdc1.
+ int kNumOfSavedRegisters = FPURegister::kNumRegisters / 2;
+ int space = kNumOfSavedRegisters * kDoubleSize;
+ Dsubu(sp, sp, Operand(space));
+ // Remember: we only need to save every 2nd double FPU value.
+ for (int i = 0; i < kNumOfSavedRegisters; i++) {
+ FPURegister reg = FPURegister::from_code(2 * i);
+ Sdc1(reg, MemOperand(sp, i * kDoubleSize));
+ }
+ }
+
+ // Reserve place for the return address, stack space and an optional slot
+ // (used by DirectCEntry to hold the return value if a struct is
+ // returned) and align the frame preparing for calling the runtime function.
+ DCHECK_GE(stack_space, 0);
+ Dsubu(sp, sp, Operand((stack_space + 2) * kPointerSize));
+ if (frame_alignment > 0) {
+ DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
+ And(sp, sp, Operand(-frame_alignment)); // Align stack.
+ }
+
+ // Set the exit frame sp value to point just before the return address
+ // location.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ daddiu(scratch, sp, kPointerSize);
+ Sd(scratch, MemOperand(fp, ExitFrameConstants::kSPOffset));
+}
+
+void MacroAssembler::LeaveExitFrame(bool save_doubles, Register argument_count,
+ bool do_return,
+ bool argument_count_is_length) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ // Optionally restore all double registers.
+ if (save_doubles) {
+ // Remember: we only need to restore every 2nd double FPU value.
+ int kNumOfSavedRegisters = FPURegister::kNumRegisters / 2;
+ Dsubu(t8, fp,
+ Operand(ExitFrameConstants::kFixedFrameSizeFromFp +
+ kNumOfSavedRegisters * kDoubleSize));
+ for (int i = 0; i < kNumOfSavedRegisters; i++) {
+ FPURegister reg = FPURegister::from_code(2 * i);
+ Ldc1(reg, MemOperand(t8, i * kDoubleSize));
+ }
+ }
+
+ // Clear top frame.
+ li(t8,
+ ExternalReference::Create(IsolateAddressId::kCEntryFPAddress, isolate()));
+ Sd(zero_reg, MemOperand(t8));
+
+ // Restore current context from top and clear it in debug mode.
+ li(t8,
+ ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
+ Ld(cp, MemOperand(t8));
+
+#ifdef DEBUG
+ li(t8,
+ ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
+ Sd(a3, MemOperand(t8));
+#endif
+
+ // Pop the arguments, restore registers, and return.
+ mov(sp, fp); // Respect ABI stack constraint.
+ Ld(fp, MemOperand(sp, ExitFrameConstants::kCallerFPOffset));
+ Ld(ra, MemOperand(sp, ExitFrameConstants::kCallerPCOffset));
+
+ if (argument_count.is_valid()) {
+ if (argument_count_is_length) {
+ daddu(sp, sp, argument_count);
+ } else {
+ Dlsa(sp, sp, argument_count, kPointerSizeLog2, t8);
+ }
+ }
+
+ if (do_return) {
+ Ret(USE_DELAY_SLOT);
+ // If returning, the instruction in the delay slot will be the addiu below.
+ }
+ daddiu(sp, sp, 2 * kPointerSize);
+}
+
+int TurboAssembler::ActivationFrameAlignment() {
+#if V8_HOST_ARCH_MIPS || V8_HOST_ARCH_MIPS64
+ // Running on the real platform. Use the alignment as mandated by the local
+ // environment.
+ // Note: This will break if we ever start generating snapshots on one Mips
+ // platform for another Mips platform with a different alignment.
+ return base::OS::ActivationFrameAlignment();
+#else // V8_HOST_ARCH_MIPS
+ // If we are using the simulator then we should always align to the expected
+ // alignment. As the simulator is used to generate snapshots we do not know
+ // if the target platform will need alignment, so this is controlled from a
+ // flag.
+ return FLAG_sim_stack_alignment;
+#endif // V8_HOST_ARCH_MIPS
+}
+
+void MacroAssembler::AssertStackIsAligned() {
+ if (emit_debug_code()) {
+ const int frame_alignment = ActivationFrameAlignment();
+ const int frame_alignment_mask = frame_alignment - 1;
+
+ if (frame_alignment > kPointerSize) {
+ Label alignment_as_expected;
+ DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
+ {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ andi(scratch, sp, frame_alignment_mask);
+ Branch(&alignment_as_expected, eq, scratch, Operand(zero_reg));
+ }
+ // Don't use Check here, as it will call Runtime_Abort re-entering here.
+ stop();
+ bind(&alignment_as_expected);
+ }
+ }
+}
+
+void TurboAssembler::SmiUntag(Register dst, const MemOperand& src) {
+ if (SmiValuesAre32Bits()) {
+ Lw(dst, MemOperand(src.rm(), SmiWordOffset(src.offset())));
+ } else {
+ DCHECK(SmiValuesAre31Bits());
+ Lw(dst, src);
+ SmiUntag(dst);
+ }
+}
+
+void TurboAssembler::JumpIfSmi(Register value, Label* smi_label,
+ Register scratch, BranchDelaySlot bd) {
+ DCHECK_EQ(0, kSmiTag);
+ andi(scratch, value, kSmiTagMask);
+ Branch(bd, smi_label, eq, scratch, Operand(zero_reg));
+}
+
+void MacroAssembler::JumpIfNotSmi(Register value, Label* not_smi_label,
+ Register scratch, BranchDelaySlot bd) {
+ DCHECK_EQ(0, kSmiTag);
+ andi(scratch, value, kSmiTagMask);
+ Branch(bd, not_smi_label, ne, scratch, Operand(zero_reg));
+}
+
+void MacroAssembler::AssertNotSmi(Register object) {
+ if (emit_debug_code()) {
+ STATIC_ASSERT(kSmiTag == 0);
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ andi(scratch, object, kSmiTagMask);
+ Check(ne, AbortReason::kOperandIsASmi, scratch, Operand(zero_reg));
+ }
+}
+
+void MacroAssembler::AssertSmi(Register object) {
+ if (emit_debug_code()) {
+ STATIC_ASSERT(kSmiTag == 0);
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ andi(scratch, object, kSmiTagMask);
+ Check(eq, AbortReason::kOperandIsASmi, scratch, Operand(zero_reg));
+ }
+}
+
+void MacroAssembler::AssertConstructor(Register object) {
+ if (emit_debug_code()) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ STATIC_ASSERT(kSmiTag == 0);
+ SmiTst(object, t8);
+ Check(ne, AbortReason::kOperandIsASmiAndNotAConstructor, t8,
+ Operand(zero_reg));
+
+ LoadMap(t8, object);
+ Lbu(t8, FieldMemOperand(t8, Map::kBitFieldOffset));
+ And(t8, t8, Operand(Map::Bits1::IsConstructorBit::kMask));
+ Check(ne, AbortReason::kOperandIsNotAConstructor, t8, Operand(zero_reg));
+ }
+}
+
+void MacroAssembler::AssertFunction(Register object) {
+ if (emit_debug_code()) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ STATIC_ASSERT(kSmiTag == 0);
+ SmiTst(object, t8);
+ Check(ne, AbortReason::kOperandIsASmiAndNotAFunction, t8,
+ Operand(zero_reg));
+ GetObjectType(object, t8, t8);
+ Check(eq, AbortReason::kOperandIsNotAFunction, t8,
+ Operand(JS_FUNCTION_TYPE));
+ }
+}
+
+void MacroAssembler::AssertBoundFunction(Register object) {
+ if (emit_debug_code()) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ STATIC_ASSERT(kSmiTag == 0);
+ SmiTst(object, t8);
+ Check(ne, AbortReason::kOperandIsASmiAndNotABoundFunction, t8,
+ Operand(zero_reg));
+ GetObjectType(object, t8, t8);
+ Check(eq, AbortReason::kOperandIsNotABoundFunction, t8,
+ Operand(JS_BOUND_FUNCTION_TYPE));
+ }
+}
+
+void MacroAssembler::AssertGeneratorObject(Register object) {
+ if (!emit_debug_code()) return;
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ STATIC_ASSERT(kSmiTag == 0);
+ SmiTst(object, t8);
+ Check(ne, AbortReason::kOperandIsASmiAndNotAGeneratorObject, t8,
+ Operand(zero_reg));
+
+ GetObjectType(object, t8, t8);
+
+ Label done;
+
+ // Check if JSGeneratorObject
+ Branch(&done, eq, t8, Operand(JS_GENERATOR_OBJECT_TYPE));
+
+ // Check if JSAsyncFunctionObject (See MacroAssembler::CompareInstanceType)
+ Branch(&done, eq, t8, Operand(JS_ASYNC_FUNCTION_OBJECT_TYPE));
+
+ // Check if JSAsyncGeneratorObject
+ Branch(&done, eq, t8, Operand(JS_ASYNC_GENERATOR_OBJECT_TYPE));
+
+ Abort(AbortReason::kOperandIsNotAGeneratorObject);
+
+ bind(&done);
+}
+
+void MacroAssembler::AssertUndefinedOrAllocationSite(Register object,
+ Register scratch) {
+ if (emit_debug_code()) {
+ Label done_checking;
+ AssertNotSmi(object);
+ LoadRoot(scratch, RootIndex::kUndefinedValue);
+ Branch(&done_checking, eq, object, Operand(scratch));
+ GetObjectType(object, scratch, scratch);
+ Assert(eq, AbortReason::kExpectedUndefinedOrCell, scratch,
+ Operand(ALLOCATION_SITE_TYPE));
+ bind(&done_checking);
+ }
+}
+
+void TurboAssembler::Float32Max(FPURegister dst, FPURegister src1,
+ FPURegister src2, Label* out_of_line) {
+ if (src1 == src2) {
+ Move_s(dst, src1);
+ return;
+ }
+
+ // Check if one of operands is NaN.
+ CompareIsNanF32(src1, src2);
+ BranchTrueF(out_of_line);
+
+ if (kArchVariant >= kMips64r6) {
+ max_s(dst, src1, src2);
+ } else {
+ Label return_left, return_right, done;
+
+ CompareF32(OLT, src1, src2);
+ BranchTrueShortF(&return_right);
+ CompareF32(OLT, src2, src1);
+ BranchTrueShortF(&return_left);
+
+ // Operands are equal, but check for +/-0.
+ {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ mfc1(t8, src1);
+ dsll32(t8, t8, 0);
+ Branch(&return_left, eq, t8, Operand(zero_reg));
+ Branch(&return_right);
+ }
+
+ bind(&return_right);
+ if (src2 != dst) {
+ Move_s(dst, src2);
+ }
+ Branch(&done);
+
+ bind(&return_left);
+ if (src1 != dst) {
+ Move_s(dst, src1);
+ }
+
+ bind(&done);
+ }
+}
+
+void TurboAssembler::Float32MaxOutOfLine(FPURegister dst, FPURegister src1,
+ FPURegister src2) {
+ add_s(dst, src1, src2);
+}
+
+void TurboAssembler::Float32Min(FPURegister dst, FPURegister src1,
+ FPURegister src2, Label* out_of_line) {
+ if (src1 == src2) {
+ Move_s(dst, src1);
+ return;
+ }
+
+ // Check if one of operands is NaN.
+ CompareIsNanF32(src1, src2);
+ BranchTrueF(out_of_line);
+
+ if (kArchVariant >= kMips64r6) {
+ min_s(dst, src1, src2);
+ } else {
+ Label return_left, return_right, done;
+
+ CompareF32(OLT, src1, src2);
+ BranchTrueShortF(&return_left);
+ CompareF32(OLT, src2, src1);
+ BranchTrueShortF(&return_right);
+
+ // Left equals right => check for -0.
+ {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ mfc1(t8, src1);
+ dsll32(t8, t8, 0);
+ Branch(&return_right, eq, t8, Operand(zero_reg));
+ Branch(&return_left);
+ }
+
+ bind(&return_right);
+ if (src2 != dst) {
+ Move_s(dst, src2);
+ }
+ Branch(&done);
+
+ bind(&return_left);
+ if (src1 != dst) {
+ Move_s(dst, src1);
+ }
+
+ bind(&done);
+ }
+}
+
+void TurboAssembler::Float32MinOutOfLine(FPURegister dst, FPURegister src1,
+ FPURegister src2) {
+ add_s(dst, src1, src2);
+}
+
+void TurboAssembler::Float64Max(FPURegister dst, FPURegister src1,
+ FPURegister src2, Label* out_of_line) {
+ if (src1 == src2) {
+ Move_d(dst, src1);
+ return;
+ }
+
+ // Check if one of operands is NaN.
+ CompareIsNanF64(src1, src2);
+ BranchTrueF(out_of_line);
+
+ if (kArchVariant >= kMips64r6) {
+ max_d(dst, src1, src2);
+ } else {
+ Label return_left, return_right, done;
+
+ CompareF64(OLT, src1, src2);
+ BranchTrueShortF(&return_right);
+ CompareF64(OLT, src2, src1);
+ BranchTrueShortF(&return_left);
+
+ // Left equals right => check for -0.
+ {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ dmfc1(t8, src1);
+ Branch(&return_left, eq, t8, Operand(zero_reg));
+ Branch(&return_right);
+ }
+
+ bind(&return_right);
+ if (src2 != dst) {
+ Move_d(dst, src2);
+ }
+ Branch(&done);
+
+ bind(&return_left);
+ if (src1 != dst) {
+ Move_d(dst, src1);
+ }
+
+ bind(&done);
+ }
+}
+
+void TurboAssembler::Float64MaxOutOfLine(FPURegister dst, FPURegister src1,
+ FPURegister src2) {
+ add_d(dst, src1, src2);
+}
+
+void TurboAssembler::Float64Min(FPURegister dst, FPURegister src1,
+ FPURegister src2, Label* out_of_line) {
+ if (src1 == src2) {
+ Move_d(dst, src1);
+ return;
+ }
+
+ // Check if one of operands is NaN.
+ CompareIsNanF64(src1, src2);
+ BranchTrueF(out_of_line);
+
+ if (kArchVariant >= kMips64r6) {
+ min_d(dst, src1, src2);
+ } else {
+ Label return_left, return_right, done;
+
+ CompareF64(OLT, src1, src2);
+ BranchTrueShortF(&return_left);
+ CompareF64(OLT, src2, src1);
+ BranchTrueShortF(&return_right);
+
+ // Left equals right => check for -0.
+ {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ dmfc1(t8, src1);
+ Branch(&return_right, eq, t8, Operand(zero_reg));
+ Branch(&return_left);
+ }
+
+ bind(&return_right);
+ if (src2 != dst) {
+ Move_d(dst, src2);
+ }
+ Branch(&done);
+
+ bind(&return_left);
+ if (src1 != dst) {
+ Move_d(dst, src1);
+ }
+
+ bind(&done);
+ }
+}
+
+void TurboAssembler::Float64MinOutOfLine(FPURegister dst, FPURegister src1,
+ FPURegister src2) {
+ add_d(dst, src1, src2);
+}
+
+static const int kRegisterPassedArguments = 8;
+
+int TurboAssembler::CalculateStackPassedWords(int num_reg_arguments,
+ int num_double_arguments) {
+ int stack_passed_words = 0;
+ num_reg_arguments += 2 * num_double_arguments;
+
+ // O32: Up to four simple arguments are passed in registers a0..a3.
+ // N64: Up to eight simple arguments are passed in registers a0..a7.
+ if (num_reg_arguments > kRegisterPassedArguments) {
+ stack_passed_words += num_reg_arguments - kRegisterPassedArguments;
+ }
+ stack_passed_words += kCArgSlotCount;
+ return stack_passed_words;
+}
+
+void TurboAssembler::PrepareCallCFunction(int num_reg_arguments,
+ int num_double_arguments,
+ Register scratch) {
+ int frame_alignment = ActivationFrameAlignment();
+
+ // n64: Up to eight simple arguments in a0..a3, a4..a7, No argument slots.
+ // O32: Up to four simple arguments are passed in registers a0..a3.
+ // Those four arguments must have reserved argument slots on the stack for
+ // mips, even though those argument slots are not normally used.
+ // Both ABIs: Remaining arguments are pushed on the stack, above (higher
+ // address than) the (O32) argument slots. (arg slot calculation handled by
+ // CalculateStackPassedWords()).
+ int stack_passed_arguments =
+ CalculateStackPassedWords(num_reg_arguments, num_double_arguments);
+ if (frame_alignment > kPointerSize) {
+ // Make stack end at alignment and make room for num_arguments - 4 words
+ // and the original value of sp.
+ mov(scratch, sp);
+ Dsubu(sp, sp, Operand((stack_passed_arguments + 1) * kPointerSize));
+ DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
+ And(sp, sp, Operand(-frame_alignment));
+ Sd(scratch, MemOperand(sp, stack_passed_arguments * kPointerSize));
+ } else {
+ Dsubu(sp, sp, Operand(stack_passed_arguments * kPointerSize));
+ }
+}
+
+void TurboAssembler::PrepareCallCFunction(int num_reg_arguments,
+ Register scratch) {
+ PrepareCallCFunction(num_reg_arguments, 0, scratch);
+}
+
+void TurboAssembler::CallCFunction(ExternalReference function,
+ int num_reg_arguments,
+ int num_double_arguments) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ li(t9, function);
+ CallCFunctionHelper(t9, num_reg_arguments, num_double_arguments);
+}
+
+void TurboAssembler::CallCFunction(Register function, int num_reg_arguments,
+ int num_double_arguments) {
+ CallCFunctionHelper(function, num_reg_arguments, num_double_arguments);
+}
+
+void TurboAssembler::CallCFunction(ExternalReference function,
+ int num_arguments) {
+ CallCFunction(function, num_arguments, 0);
+}
+
+void TurboAssembler::CallCFunction(Register function, int num_arguments) {
+ CallCFunction(function, num_arguments, 0);
+}
+
+void TurboAssembler::CallCFunctionHelper(Register function,
+ int num_reg_arguments,
+ int num_double_arguments) {
+ DCHECK_LE(num_reg_arguments + num_double_arguments, kMaxCParameters);
+ DCHECK(has_frame());
+ // Make sure that the stack is aligned before calling a C function unless
+ // running in the simulator. The simulator has its own alignment check which
+ // provides more information.
+ // The argument stots are presumed to have been set up by
+ // PrepareCallCFunction. The C function must be called via t9, for mips ABI.
+
+#if V8_HOST_ARCH_MIPS || V8_HOST_ARCH_MIPS64
+ if (emit_debug_code()) {
+ int frame_alignment = base::OS::ActivationFrameAlignment();
+ int frame_alignment_mask = frame_alignment - 1;
+ if (frame_alignment > kPointerSize) {
+ DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
+ Label alignment_as_expected;
+ {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ And(scratch, sp, Operand(frame_alignment_mask));
+ Branch(&alignment_as_expected, eq, scratch, Operand(zero_reg));
+ }
+ // Don't use Check here, as it will call Runtime_Abort possibly
+ // re-entering here.
+ stop();
+ bind(&alignment_as_expected);
+ }
+ }
+#endif // V8_HOST_ARCH_MIPS
+
+ // Just call directly. The function called cannot cause a GC, or
+ // allow preemption, so the return address in the link register
+ // stays correct.
+ {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ if (function != t9) {
+ mov(t9, function);
+ function = t9;
+ }
+
+ // Save the frame pointer and PC so that the stack layout remains iterable,
+ // even without an ExitFrame which normally exists between JS and C frames.
+ // 't' registers are caller-saved so this is safe as a scratch register.
+ Register pc_scratch = t1;
+ Register scratch = t2;
+ DCHECK(!AreAliased(pc_scratch, scratch, function));
+
+ mov(scratch, ra);
+ nal();
+ mov(pc_scratch, ra);
+ mov(ra, scratch);
+
+ // See x64 code for reasoning about how to address the isolate data fields.
+ if (root_array_available()) {
+ Sd(pc_scratch, MemOperand(kRootRegister,
+ IsolateData::fast_c_call_caller_pc_offset()));
+ Sd(fp, MemOperand(kRootRegister,
+ IsolateData::fast_c_call_caller_fp_offset()));
+ } else {
+ DCHECK_NOT_NULL(isolate());
+ li(scratch, ExternalReference::fast_c_call_caller_pc_address(isolate()));
+ Sd(pc_scratch, MemOperand(scratch));
+ li(scratch, ExternalReference::fast_c_call_caller_fp_address(isolate()));
+ Sd(fp, MemOperand(scratch));
+ }
+
+ Call(function);
+
+ // We don't unset the PC; the FP is the source of truth.
+ if (root_array_available()) {
+ Sd(zero_reg, MemOperand(kRootRegister,
+ IsolateData::fast_c_call_caller_fp_offset()));
+ } else {
+ DCHECK_NOT_NULL(isolate());
+ li(scratch, ExternalReference::fast_c_call_caller_fp_address(isolate()));
+ Sd(zero_reg, MemOperand(scratch));
+ }
+ }
+
+ int stack_passed_arguments =
+ CalculateStackPassedWords(num_reg_arguments, num_double_arguments);
+
+ if (base::OS::ActivationFrameAlignment() > kPointerSize) {
+ Ld(sp, MemOperand(sp, stack_passed_arguments * kPointerSize));
+ } else {
+ Daddu(sp, sp, Operand(stack_passed_arguments * kPointerSize));
+ }
+}
+
+#undef BRANCH_ARGS_CHECK
+
+void TurboAssembler::CheckPageFlag(Register object, Register scratch, int mask,
+ Condition cc, Label* condition_met) {
+ And(scratch, object, Operand(~kPageAlignmentMask));
+ Ld(scratch, MemOperand(scratch, BasicMemoryChunk::kFlagsOffset));
+ And(scratch, scratch, Operand(mask));
+ Branch(condition_met, cc, scratch, Operand(zero_reg));
+}
+
+Register GetRegisterThatIsNotOneOf(Register reg1, Register reg2, Register reg3,
+ Register reg4, Register reg5,
+ Register reg6) {
+ RegList regs = 0;
+ if (reg1.is_valid()) regs |= reg1.bit();
+ if (reg2.is_valid()) regs |= reg2.bit();
+ if (reg3.is_valid()) regs |= reg3.bit();
+ if (reg4.is_valid()) regs |= reg4.bit();
+ if (reg5.is_valid()) regs |= reg5.bit();
+ if (reg6.is_valid()) regs |= reg6.bit();
+
+ const RegisterConfiguration* config = RegisterConfiguration::Default();
+ for (int i = 0; i < config->num_allocatable_general_registers(); ++i) {
+ int code = config->GetAllocatableGeneralCode(i);
+ Register candidate = Register::from_code(code);
+ if (regs & candidate.bit()) continue;
+ return candidate;
+ }
+ UNREACHABLE();
+}
+
+void TurboAssembler::ComputeCodeStartAddress(Register dst) {
+ // This push on ra and the pop below together ensure that we restore the
+ // register ra, which is needed while computing the code start address.
+ push(ra);
+
+ // The nal instruction puts the address of the current instruction into
+ // the return address (ra) register, which we can use later on.
+ if (kArchVariant == kMips64r6) {
+ addiupc(ra, 1);
+ } else {
+ nal();
+ nop();
+ }
+ int pc = pc_offset();
+ li(dst, Operand(pc));
+ Dsubu(dst, ra, dst);
+
+ pop(ra); // Restore ra
+}
+
+void TurboAssembler::ResetSpeculationPoisonRegister() {
+ li(kSpeculationPoisonRegister, -1);
+}
+
+void TurboAssembler::CallForDeoptimization(Builtins::Name target, int,
+ Label* exit, DeoptimizeKind kind,
+ Label*) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ Ld(t9,
+ MemOperand(kRootRegister, IsolateData::builtin_entry_slot_offset(target)));
+ Call(t9);
+ DCHECK_EQ(SizeOfCodeGeneratedSince(exit),
+ (kind == DeoptimizeKind::kLazy)
+ ? Deoptimizer::kLazyDeoptExitSize
+ : Deoptimizer::kNonLazyDeoptExitSize);
+ USE(exit, kind);
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_MIPS64
diff --git a/src/codegen/mips64/macro-assembler-mips64.h b/src/codegen/mips64/macro-assembler-mips64.h
new file mode 100644
index 0000000..a0d5e59
--- /dev/null
+++ b/src/codegen/mips64/macro-assembler-mips64.h
@@ -0,0 +1,1274 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef INCLUDED_FROM_MACRO_ASSEMBLER_H
+#error This header must be included via macro-assembler.h
+#endif
+
+#ifndef V8_CODEGEN_MIPS64_MACRO_ASSEMBLER_MIPS64_H_
+#define V8_CODEGEN_MIPS64_MACRO_ASSEMBLER_MIPS64_H_
+
+#include "src/codegen/assembler.h"
+#include "src/codegen/mips64/assembler-mips64.h"
+#include "src/common/globals.h"
+
+namespace v8 {
+namespace internal {
+
+// Forward declarations.
+enum class AbortReason : uint8_t;
+
+// Reserved Register Usage Summary.
+//
+// Registers t8, t9, and at are reserved for use by the MacroAssembler.
+//
+// The programmer should know that the MacroAssembler may clobber these three,
+// but won't touch other registers except in special cases.
+//
+// Per the MIPS ABI, register t9 must be used for indirect function call
+// via 'jalr t9' or 'jr t9' instructions. This is relied upon by gcc when
+// trying to update gp register for position-independent-code. Whenever
+// MIPS generated code calls C code, it must be via t9 register.
+
+// Flags used for LeaveExitFrame function.
+enum LeaveExitFrameMode { EMIT_RETURN = true, NO_EMIT_RETURN = false };
+
+// Allow programmer to use Branch Delay Slot of Branches, Jumps, Calls.
+enum BranchDelaySlot { USE_DELAY_SLOT, PROTECT };
+
+// Flags used for the li macro-assembler function.
+enum LiFlags {
+ // If the constant value can be represented in just 16 bits, then
+ // optimize the li to use a single instruction, rather than lui/ori/dsll
+ // sequence. A number of other optimizations that emits less than
+ // maximum number of instructions exists.
+ OPTIMIZE_SIZE = 0,
+ // Always use 6 instructions (lui/ori/dsll sequence) for release 2 or 4
+ // instructions for release 6 (lui/ori/dahi/dati), even if the constant
+ // could be loaded with just one, so that this value is patchable later.
+ CONSTANT_SIZE = 1,
+ // For address loads only 4 instruction are required. Used to mark
+ // constant load that will be used as address without relocation
+ // information. It ensures predictable code size, so specific sites
+ // in code are patchable.
+ ADDRESS_LOAD = 2
+};
+
+enum RememberedSetAction { EMIT_REMEMBERED_SET, OMIT_REMEMBERED_SET };
+enum SmiCheck { INLINE_SMI_CHECK, OMIT_SMI_CHECK };
+enum RAStatus { kRAHasNotBeenSaved, kRAHasBeenSaved };
+
+Register GetRegisterThatIsNotOneOf(Register reg1, Register reg2 = no_reg,
+ Register reg3 = no_reg,
+ Register reg4 = no_reg,
+ Register reg5 = no_reg,
+ Register reg6 = no_reg);
+
+// -----------------------------------------------------------------------------
+// Static helper functions.
+
+#if defined(V8_TARGET_LITTLE_ENDIAN)
+#define SmiWordOffset(offset) (offset + kPointerSize / 2)
+#else
+#define SmiWordOffset(offset) offset
+#endif
+
+// Generate a MemOperand for loading a field from an object.
+inline MemOperand FieldMemOperand(Register object, int offset) {
+ return MemOperand(object, offset - kHeapObjectTag);
+}
+
+// Generate a MemOperand for storing arguments 5..N on the stack
+// when calling CallCFunction().
+// TODO(plind): Currently ONLY used for O32. Should be fixed for
+// n64, and used in RegExp code, and other places
+// with more than 8 arguments.
+inline MemOperand CFunctionArgumentOperand(int index) {
+ DCHECK_GT(index, kCArgSlotCount);
+ // Argument 5 takes the slot just past the four Arg-slots.
+ int offset = (index - 5) * kPointerSize + kCArgsSlotsSize;
+ return MemOperand(sp, offset);
+}
+
+class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase {
+ public:
+ using TurboAssemblerBase::TurboAssemblerBase;
+
+ // Activation support.
+ void EnterFrame(StackFrame::Type type);
+ void EnterFrame(StackFrame::Type type, bool load_constant_pool_pointer_reg) {
+ // Out-of-line constant pool not implemented on mips.
+ UNREACHABLE();
+ }
+ void LeaveFrame(StackFrame::Type type);
+
+ // Generates function and stub prologue code.
+ void StubPrologue(StackFrame::Type type);
+ void Prologue();
+
+ void InitializeRootRegister() {
+ ExternalReference isolate_root = ExternalReference::isolate_root(isolate());
+ li(kRootRegister, Operand(isolate_root));
+ }
+
+ // Jump unconditionally to given label.
+ // We NEED a nop in the branch delay slot, as it used by v8, for example in
+ // CodeGenerator::ProcessDeferred().
+ // Currently the branch delay slot is filled by the MacroAssembler.
+ // Use rather b(Label) for code generation.
+ void jmp(Label* L) { Branch(L); }
+
+ // -------------------------------------------------------------------------
+ // Debugging.
+
+ void Trap() override;
+ void DebugBreak() override;
+
+ // Calls Abort(msg) if the condition cc is not satisfied.
+ // Use --debug_code to enable.
+ void Assert(Condition cc, AbortReason reason, Register rs, Operand rt);
+
+ // Like Assert(), but always enabled.
+ void Check(Condition cc, AbortReason reason, Register rs, Operand rt);
+
+ // Print a message to stdout and abort execution.
+ void Abort(AbortReason msg);
+
+ // Arguments macros.
+#define COND_TYPED_ARGS Condition cond, Register r1, const Operand &r2
+#define COND_ARGS cond, r1, r2
+
+ // Cases when relocation is not needed.
+#define DECLARE_NORELOC_PROTOTYPE(Name, target_type) \
+ void Name(target_type target, BranchDelaySlot bd = PROTECT); \
+ inline void Name(BranchDelaySlot bd, target_type target) { \
+ Name(target, bd); \
+ } \
+ void Name(target_type target, COND_TYPED_ARGS, \
+ BranchDelaySlot bd = PROTECT); \
+ inline void Name(BranchDelaySlot bd, target_type target, COND_TYPED_ARGS) { \
+ Name(target, COND_ARGS, bd); \
+ }
+
+#define DECLARE_BRANCH_PROTOTYPES(Name) \
+ DECLARE_NORELOC_PROTOTYPE(Name, Label*) \
+ DECLARE_NORELOC_PROTOTYPE(Name, int32_t)
+
+ DECLARE_BRANCH_PROTOTYPES(Branch)
+ DECLARE_BRANCH_PROTOTYPES(BranchAndLink)
+ DECLARE_BRANCH_PROTOTYPES(BranchShort)
+
+#undef DECLARE_BRANCH_PROTOTYPES
+#undef COND_TYPED_ARGS
+#undef COND_ARGS
+
+ // Floating point branches
+ void CompareF32(FPUCondition cc, FPURegister cmp1, FPURegister cmp2) {
+ CompareF(S, cc, cmp1, cmp2);
+ }
+
+ void CompareIsNanF32(FPURegister cmp1, FPURegister cmp2) {
+ CompareIsNanF(S, cmp1, cmp2);
+ }
+
+ void CompareF64(FPUCondition cc, FPURegister cmp1, FPURegister cmp2) {
+ CompareF(D, cc, cmp1, cmp2);
+ }
+
+ void CompareIsNanF64(FPURegister cmp1, FPURegister cmp2) {
+ CompareIsNanF(D, cmp1, cmp2);
+ }
+
+ void BranchTrueShortF(Label* target, BranchDelaySlot bd = PROTECT);
+ void BranchFalseShortF(Label* target, BranchDelaySlot bd = PROTECT);
+
+ void BranchTrueF(Label* target, BranchDelaySlot bd = PROTECT);
+ void BranchFalseF(Label* target, BranchDelaySlot bd = PROTECT);
+
+ // MSA branches
+ void BranchMSA(Label* target, MSABranchDF df, MSABranchCondition cond,
+ MSARegister wt, BranchDelaySlot bd = PROTECT);
+
+ void Branch(Label* L, Condition cond, Register rs, RootIndex index,
+ BranchDelaySlot bdslot = PROTECT);
+
+ static int InstrCountForLi64Bit(int64_t value);
+ inline void LiLower32BitHelper(Register rd, Operand j);
+ void li_optimized(Register rd, Operand j, LiFlags mode = OPTIMIZE_SIZE);
+ // Load int32 in the rd register.
+ void li(Register rd, Operand j, LiFlags mode = OPTIMIZE_SIZE);
+ inline void li(Register rd, int64_t j, LiFlags mode = OPTIMIZE_SIZE) {
+ li(rd, Operand(j), mode);
+ }
+ // inline void li(Register rd, int32_t j, LiFlags mode = OPTIMIZE_SIZE) {
+ // li(rd, Operand(static_cast<int64_t>(j)), mode);
+ // }
+ void li(Register dst, Handle<HeapObject> value, LiFlags mode = OPTIMIZE_SIZE);
+ void li(Register dst, ExternalReference value, LiFlags mode = OPTIMIZE_SIZE);
+ void li(Register dst, const StringConstantBase* string,
+ LiFlags mode = OPTIMIZE_SIZE);
+
+ void LoadFromConstantsTable(Register destination,
+ int constant_index) override;
+ void LoadRootRegisterOffset(Register destination, intptr_t offset) override;
+ void LoadRootRelative(Register destination, int32_t offset) override;
+
+// Jump, Call, and Ret pseudo instructions implementing inter-working.
+#define COND_ARGS \
+ Condition cond = al, Register rs = zero_reg, \
+ const Operand &rt = Operand(zero_reg), \
+ BranchDelaySlot bd = PROTECT
+
+ void Jump(Register target, COND_ARGS);
+ void Jump(intptr_t target, RelocInfo::Mode rmode, COND_ARGS);
+ void Jump(Address target, RelocInfo::Mode rmode, COND_ARGS);
+ // Deffer from li, this method save target to the memory, and then load
+ // it to register use ld, it can be used in wasm jump table for concurrent
+ // patching.
+ void PatchAndJump(Address target);
+ void Jump(Handle<Code> code, RelocInfo::Mode rmode, COND_ARGS);
+ void Jump(const ExternalReference& reference) override;
+ void Call(Register target, COND_ARGS);
+ void Call(Address target, RelocInfo::Mode rmode, COND_ARGS);
+ void Call(Handle<Code> code, RelocInfo::Mode rmode = RelocInfo::CODE_TARGET,
+ COND_ARGS);
+ void Call(Label* target);
+ void LoadAddress(Register dst, Label* target);
+
+ // Load the builtin given by the Smi in |builtin_index| into the same
+ // register.
+ void LoadEntryFromBuiltinIndex(Register builtin_index);
+ void CallBuiltinByIndex(Register builtin_index) override;
+
+ void LoadCodeObjectEntry(Register destination,
+ Register code_object) override {
+ // TODO(mips): Implement.
+ UNIMPLEMENTED();
+ }
+ void CallCodeObject(Register code_object) override {
+ // TODO(mips): Implement.
+ UNIMPLEMENTED();
+ }
+ void JumpCodeObject(Register code_object) override {
+ // TODO(mips): Implement.
+ UNIMPLEMENTED();
+ }
+
+ // Generates an instruction sequence s.t. the return address points to the
+ // instruction following the call.
+ // The return address on the stack is used by frame iteration.
+ void StoreReturnAddressAndCall(Register target);
+
+ void CallForDeoptimization(Builtins::Name target, int deopt_id, Label* exit,
+ DeoptimizeKind kind,
+ Label* jump_deoptimization_entry_label);
+
+ void Ret(COND_ARGS);
+ inline void Ret(BranchDelaySlot bd, Condition cond = al,
+ Register rs = zero_reg,
+ const Operand& rt = Operand(zero_reg)) {
+ Ret(cond, rs, rt, bd);
+ }
+
+ // Emit code to discard a non-negative number of pointer-sized elements
+ // from the stack, clobbering only the sp register.
+ void Drop(int count, Condition cond = cc_always, Register reg = no_reg,
+ const Operand& op = Operand(no_reg));
+
+ // Trivial case of DropAndRet that utilizes the delay slot.
+ void DropAndRet(int drop);
+
+ void DropAndRet(int drop, Condition cond, Register reg, const Operand& op);
+
+ void Ld(Register rd, const MemOperand& rs);
+ void Sd(Register rd, const MemOperand& rs);
+
+ void push(Register src) {
+ Daddu(sp, sp, Operand(-kPointerSize));
+ Sd(src, MemOperand(sp, 0));
+ }
+ void Push(Register src) { push(src); }
+ void Push(Handle<HeapObject> handle);
+ void Push(Smi smi);
+
+ // Push two registers. Pushes leftmost register first (to highest address).
+ void Push(Register src1, Register src2) {
+ Dsubu(sp, sp, Operand(2 * kPointerSize));
+ Sd(src1, MemOperand(sp, 1 * kPointerSize));
+ Sd(src2, MemOperand(sp, 0 * kPointerSize));
+ }
+
+ // Push three registers. Pushes leftmost register first (to highest address).
+ void Push(Register src1, Register src2, Register src3) {
+ Dsubu(sp, sp, Operand(3 * kPointerSize));
+ Sd(src1, MemOperand(sp, 2 * kPointerSize));
+ Sd(src2, MemOperand(sp, 1 * kPointerSize));
+ Sd(src3, MemOperand(sp, 0 * kPointerSize));
+ }
+
+ // Push four registers. Pushes leftmost register first (to highest address).
+ void Push(Register src1, Register src2, Register src3, Register src4) {
+ Dsubu(sp, sp, Operand(4 * kPointerSize));
+ Sd(src1, MemOperand(sp, 3 * kPointerSize));
+ Sd(src2, MemOperand(sp, 2 * kPointerSize));
+ Sd(src3, MemOperand(sp, 1 * kPointerSize));
+ Sd(src4, MemOperand(sp, 0 * kPointerSize));
+ }
+
+ // Push five registers. Pushes leftmost register first (to highest address).
+ void Push(Register src1, Register src2, Register src3, Register src4,
+ Register src5) {
+ Dsubu(sp, sp, Operand(5 * kPointerSize));
+ Sd(src1, MemOperand(sp, 4 * kPointerSize));
+ Sd(src2, MemOperand(sp, 3 * kPointerSize));
+ Sd(src3, MemOperand(sp, 2 * kPointerSize));
+ Sd(src4, MemOperand(sp, 1 * kPointerSize));
+ Sd(src5, MemOperand(sp, 0 * kPointerSize));
+ }
+
+ void Push(Register src, Condition cond, Register tst1, Register tst2) {
+ // Since we don't have conditional execution we use a Branch.
+ Branch(3, cond, tst1, Operand(tst2));
+ Dsubu(sp, sp, Operand(kPointerSize));
+ Sd(src, MemOperand(sp, 0));
+ }
+
+ enum PushArrayOrder { kNormal, kReverse };
+ void PushArray(Register array, Register size, Register scratch,
+ Register scratch2, PushArrayOrder order = kNormal);
+
+ void SaveRegisters(RegList registers);
+ void RestoreRegisters(RegList registers);
+
+ void CallRecordWriteStub(Register object, Register address,
+ RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode);
+ void CallRecordWriteStub(Register object, Register address,
+ RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode, Address wasm_target);
+ void CallEphemeronKeyBarrier(Register object, Register address,
+ SaveFPRegsMode fp_mode);
+
+ // Push multiple registers on the stack.
+ // Registers are saved in numerical order, with higher numbered registers
+ // saved in higher memory addresses.
+ void MultiPush(RegList regs);
+ void MultiPushFPU(RegList regs);
+
+ // Calculate how much stack space (in bytes) are required to store caller
+ // registers excluding those specified in the arguments.
+ int RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
+ Register exclusion1 = no_reg,
+ Register exclusion2 = no_reg,
+ Register exclusion3 = no_reg) const;
+
+ // Push caller saved registers on the stack, and return the number of bytes
+ // stack pointer is adjusted.
+ int PushCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1 = no_reg,
+ Register exclusion2 = no_reg,
+ Register exclusion3 = no_reg);
+ // Restore caller saved registers from the stack, and return the number of
+ // bytes stack pointer is adjusted.
+ int PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1 = no_reg,
+ Register exclusion2 = no_reg,
+ Register exclusion3 = no_reg);
+
+ void pop(Register dst) {
+ Ld(dst, MemOperand(sp, 0));
+ Daddu(sp, sp, Operand(kPointerSize));
+ }
+ void Pop(Register dst) { pop(dst); }
+
+ // Pop two registers. Pops rightmost register first (from lower address).
+ void Pop(Register src1, Register src2) {
+ DCHECK(src1 != src2);
+ Ld(src2, MemOperand(sp, 0 * kPointerSize));
+ Ld(src1, MemOperand(sp, 1 * kPointerSize));
+ Daddu(sp, sp, 2 * kPointerSize);
+ }
+
+ // Pop three registers. Pops rightmost register first (from lower address).
+ void Pop(Register src1, Register src2, Register src3) {
+ Ld(src3, MemOperand(sp, 0 * kPointerSize));
+ Ld(src2, MemOperand(sp, 1 * kPointerSize));
+ Ld(src1, MemOperand(sp, 2 * kPointerSize));
+ Daddu(sp, sp, 3 * kPointerSize);
+ }
+
+ void Pop(uint32_t count = 1) { Daddu(sp, sp, Operand(count * kPointerSize)); }
+
+ // Pops multiple values from the stack and load them in the
+ // registers specified in regs. Pop order is the opposite as in MultiPush.
+ void MultiPop(RegList regs);
+ void MultiPopFPU(RegList regs);
+
+#define DEFINE_INSTRUCTION(instr) \
+ void instr(Register rd, Register rs, const Operand& rt); \
+ void instr(Register rd, Register rs, Register rt) { \
+ instr(rd, rs, Operand(rt)); \
+ } \
+ void instr(Register rs, Register rt, int32_t j) { instr(rs, rt, Operand(j)); }
+
+#define DEFINE_INSTRUCTION2(instr) \
+ void instr(Register rs, const Operand& rt); \
+ void instr(Register rs, Register rt) { instr(rs, Operand(rt)); } \
+ void instr(Register rs, int32_t j) { instr(rs, Operand(j)); }
+
+ DEFINE_INSTRUCTION(Addu)
+ DEFINE_INSTRUCTION(Daddu)
+ DEFINE_INSTRUCTION(Div)
+ DEFINE_INSTRUCTION(Divu)
+ DEFINE_INSTRUCTION(Ddivu)
+ DEFINE_INSTRUCTION(Mod)
+ DEFINE_INSTRUCTION(Modu)
+ DEFINE_INSTRUCTION(Ddiv)
+ DEFINE_INSTRUCTION(Subu)
+ DEFINE_INSTRUCTION(Dsubu)
+ DEFINE_INSTRUCTION(Dmod)
+ DEFINE_INSTRUCTION(Dmodu)
+ DEFINE_INSTRUCTION(Mul)
+ DEFINE_INSTRUCTION(Mulh)
+ DEFINE_INSTRUCTION(Mulhu)
+ DEFINE_INSTRUCTION(Dmul)
+ DEFINE_INSTRUCTION(Dmulh)
+ DEFINE_INSTRUCTION2(Mult)
+ DEFINE_INSTRUCTION2(Dmult)
+ DEFINE_INSTRUCTION2(Multu)
+ DEFINE_INSTRUCTION2(Dmultu)
+ DEFINE_INSTRUCTION2(Div)
+ DEFINE_INSTRUCTION2(Ddiv)
+ DEFINE_INSTRUCTION2(Divu)
+ DEFINE_INSTRUCTION2(Ddivu)
+
+ DEFINE_INSTRUCTION(And)
+ DEFINE_INSTRUCTION(Or)
+ DEFINE_INSTRUCTION(Xor)
+ DEFINE_INSTRUCTION(Nor)
+ DEFINE_INSTRUCTION2(Neg)
+
+ DEFINE_INSTRUCTION(Slt)
+ DEFINE_INSTRUCTION(Sltu)
+ DEFINE_INSTRUCTION(Sle)
+ DEFINE_INSTRUCTION(Sleu)
+ DEFINE_INSTRUCTION(Sgt)
+ DEFINE_INSTRUCTION(Sgtu)
+ DEFINE_INSTRUCTION(Sge)
+ DEFINE_INSTRUCTION(Sgeu)
+
+ // MIPS32 R2 instruction macro.
+ DEFINE_INSTRUCTION(Ror)
+ DEFINE_INSTRUCTION(Dror)
+
+#undef DEFINE_INSTRUCTION
+#undef DEFINE_INSTRUCTION2
+#undef DEFINE_INSTRUCTION3
+
+ void SmiUntag(Register dst, const MemOperand& src);
+ void SmiUntag(Register dst, Register src) {
+ if (SmiValuesAre32Bits()) {
+ dsra32(dst, src, kSmiShift - 32);
+ } else {
+ DCHECK(SmiValuesAre31Bits());
+ sra(dst, src, kSmiShift);
+ }
+ }
+
+ void SmiUntag(Register reg) { SmiUntag(reg, reg); }
+
+ // Removes current frame and its arguments from the stack preserving
+ // the arguments and a return address pushed to the stack for the next call.
+ // Both |callee_args_count| and |caller_args_count| do not include
+ // receiver. |callee_args_count| is not modified. |caller_args_count|
+ // is trashed.
+ void PrepareForTailCall(Register callee_args_count,
+ Register caller_args_count, Register scratch0,
+ Register scratch1);
+
+ int CalculateStackPassedWords(int num_reg_arguments,
+ int num_double_arguments);
+
+ // Before calling a C-function from generated code, align arguments on stack
+ // and add space for the four mips argument slots.
+ // After aligning the frame, non-register arguments must be stored on the
+ // stack, after the argument-slots using helper: CFunctionArgumentOperand().
+ // The argument count assumes all arguments are word sized.
+ // Some compilers/platforms require the stack to be aligned when calling
+ // C++ code.
+ // Needs a scratch register to do some arithmetic. This register will be
+ // trashed.
+ void PrepareCallCFunction(int num_reg_arguments, int num_double_registers,
+ Register scratch);
+ void PrepareCallCFunction(int num_reg_arguments, Register scratch);
+
+ // Arguments 1-4 are placed in registers a0 through a3 respectively.
+ // Arguments 5..n are stored to stack using following:
+ // Sw(a4, CFunctionArgumentOperand(5));
+
+ // Calls a C function and cleans up the space for arguments allocated
+ // by PrepareCallCFunction. The called function is not allowed to trigger a
+ // garbage collection, since that might move the code and invalidate the
+ // return address (unless this is somehow accounted for by the called
+ // function).
+ void CallCFunction(ExternalReference function, int num_arguments);
+ void CallCFunction(Register function, int num_arguments);
+ void CallCFunction(ExternalReference function, int num_reg_arguments,
+ int num_double_arguments);
+ void CallCFunction(Register function, int num_reg_arguments,
+ int num_double_arguments);
+ void MovFromFloatResult(DoubleRegister dst);
+ void MovFromFloatParameter(DoubleRegister dst);
+
+ // There are two ways of passing double arguments on MIPS, depending on
+ // whether soft or hard floating point ABI is used. These functions
+ // abstract parameter passing for the three different ways we call
+ // C functions from generated code.
+ void MovToFloatParameter(DoubleRegister src);
+ void MovToFloatParameters(DoubleRegister src1, DoubleRegister src2);
+ void MovToFloatResult(DoubleRegister src);
+
+ // See comments at the beginning of Builtins::Generate_CEntry.
+ inline void PrepareCEntryArgs(int num_args) { li(a0, num_args); }
+ inline void PrepareCEntryFunction(const ExternalReference& ref) {
+ li(a1, ref);
+ }
+
+ void CheckPageFlag(Register object, Register scratch, int mask, Condition cc,
+ Label* condition_met);
+#undef COND_ARGS
+
+ // Performs a truncating conversion of a floating point number as used by
+ // the JS bitwise operations. See ECMA-262 9.5: ToInt32.
+ // Exits with 'result' holding the answer.
+ void TruncateDoubleToI(Isolate* isolate, Zone* zone, Register result,
+ DoubleRegister double_input, StubCallMode stub_mode);
+
+ // Conditional move.
+ void Movz(Register rd, Register rs, Register rt);
+ void Movn(Register rd, Register rs, Register rt);
+ void Movt(Register rd, Register rs, uint16_t cc = 0);
+ void Movf(Register rd, Register rs, uint16_t cc = 0);
+
+ void LoadZeroIfFPUCondition(Register dest);
+ void LoadZeroIfNotFPUCondition(Register dest);
+
+ void LoadZeroIfConditionNotZero(Register dest, Register condition);
+ void LoadZeroIfConditionZero(Register dest, Register condition);
+ void LoadZeroOnCondition(Register rd, Register rs, const Operand& rt,
+ Condition cond);
+
+ void Clz(Register rd, Register rs);
+ void Dclz(Register rd, Register rs);
+ void Ctz(Register rd, Register rs);
+ void Dctz(Register rd, Register rs);
+ void Popcnt(Register rd, Register rs);
+ void Dpopcnt(Register rd, Register rs);
+
+ // MIPS64 R2 instruction macro.
+ void Ext(Register rt, Register rs, uint16_t pos, uint16_t size);
+ void Dext(Register rt, Register rs, uint16_t pos, uint16_t size);
+ void Ins(Register rt, Register rs, uint16_t pos, uint16_t size);
+ void Dins(Register rt, Register rs, uint16_t pos, uint16_t size);
+ void ExtractBits(Register dest, Register source, Register pos, int size,
+ bool sign_extend = false);
+ void InsertBits(Register dest, Register source, Register pos, int size);
+ void Neg_s(FPURegister fd, FPURegister fs);
+ void Neg_d(FPURegister fd, FPURegister fs);
+
+ // MIPS64 R6 instruction macros.
+ void Bovc(Register rt, Register rs, Label* L);
+ void Bnvc(Register rt, Register rs, Label* L);
+
+ // Convert single to unsigned word.
+ void Trunc_uw_s(FPURegister fd, FPURegister fs, FPURegister scratch);
+ void Trunc_uw_s(Register rd, FPURegister fs, FPURegister scratch);
+
+ // Change endianness
+ void ByteSwapSigned(Register dest, Register src, int operand_size);
+ void ByteSwapUnsigned(Register dest, Register src, int operand_size);
+
+ void Ulh(Register rd, const MemOperand& rs);
+ void Ulhu(Register rd, const MemOperand& rs);
+ void Ush(Register rd, const MemOperand& rs, Register scratch);
+
+ void Ulw(Register rd, const MemOperand& rs);
+ void Ulwu(Register rd, const MemOperand& rs);
+ void Usw(Register rd, const MemOperand& rs);
+
+ void Uld(Register rd, const MemOperand& rs);
+ void Usd(Register rd, const MemOperand& rs);
+
+ void Ulwc1(FPURegister fd, const MemOperand& rs, Register scratch);
+ void Uswc1(FPURegister fd, const MemOperand& rs, Register scratch);
+
+ void Uldc1(FPURegister fd, const MemOperand& rs, Register scratch);
+ void Usdc1(FPURegister fd, const MemOperand& rs, Register scratch);
+
+ void Lb(Register rd, const MemOperand& rs);
+ void Lbu(Register rd, const MemOperand& rs);
+ void Sb(Register rd, const MemOperand& rs);
+
+ void Lh(Register rd, const MemOperand& rs);
+ void Lhu(Register rd, const MemOperand& rs);
+ void Sh(Register rd, const MemOperand& rs);
+
+ void Lw(Register rd, const MemOperand& rs);
+ void Lwu(Register rd, const MemOperand& rs);
+ void Sw(Register rd, const MemOperand& rs);
+
+ void Lwc1(FPURegister fd, const MemOperand& src);
+ void Swc1(FPURegister fs, const MemOperand& dst);
+
+ void Ldc1(FPURegister fd, const MemOperand& src);
+ void Sdc1(FPURegister fs, const MemOperand& dst);
+
+ void Ll(Register rd, const MemOperand& rs);
+ void Sc(Register rd, const MemOperand& rs);
+
+ void Lld(Register rd, const MemOperand& rs);
+ void Scd(Register rd, const MemOperand& rs);
+
+ // Perform a floating-point min or max operation with the
+ // (IEEE-754-compatible) semantics of MIPS32's Release 6 MIN.fmt/MAX.fmt.
+ // Some cases, typically NaNs or +/-0.0, are expected to be rare and are
+ // handled in out-of-line code. The specific behaviour depends on supported
+ // instructions.
+ //
+ // These functions assume (and assert) that src1!=src2. It is permitted
+ // for the result to alias either input register.
+ void Float32Max(FPURegister dst, FPURegister src1, FPURegister src2,
+ Label* out_of_line);
+ void Float32Min(FPURegister dst, FPURegister src1, FPURegister src2,
+ Label* out_of_line);
+ void Float64Max(FPURegister dst, FPURegister src1, FPURegister src2,
+ Label* out_of_line);
+ void Float64Min(FPURegister dst, FPURegister src1, FPURegister src2,
+ Label* out_of_line);
+
+ // Generate out-of-line cases for the macros above.
+ void Float32MaxOutOfLine(FPURegister dst, FPURegister src1, FPURegister src2);
+ void Float32MinOutOfLine(FPURegister dst, FPURegister src1, FPURegister src2);
+ void Float64MaxOutOfLine(FPURegister dst, FPURegister src1, FPURegister src2);
+ void Float64MinOutOfLine(FPURegister dst, FPURegister src1, FPURegister src2);
+
+ bool IsDoubleZeroRegSet() { return has_double_zero_reg_set_; }
+
+ void mov(Register rd, Register rt) { or_(rd, rt, zero_reg); }
+
+ inline void Move(Register dst, Handle<HeapObject> handle) { li(dst, handle); }
+ inline void Move(Register dst, Smi smi) { li(dst, Operand(smi)); }
+
+ inline void Move(Register dst, Register src) {
+ if (dst != src) {
+ mov(dst, src);
+ }
+ }
+
+ inline void Move(FPURegister dst, FPURegister src) { Move_d(dst, src); }
+
+ inline void Move(Register dst_low, Register dst_high, FPURegister src) {
+ mfc1(dst_low, src);
+ mfhc1(dst_high, src);
+ }
+
+ inline void Move(Register dst, FPURegister src) { dmfc1(dst, src); }
+
+ inline void Move(FPURegister dst, Register src) { dmtc1(src, dst); }
+
+ inline void FmoveHigh(Register dst_high, FPURegister src) {
+ mfhc1(dst_high, src);
+ }
+
+ inline void FmoveHigh(FPURegister dst, Register src_high) {
+ mthc1(src_high, dst);
+ }
+
+ inline void FmoveLow(Register dst_low, FPURegister src) {
+ mfc1(dst_low, src);
+ }
+
+ void FmoveLow(FPURegister dst, Register src_low);
+
+ inline void Move(FPURegister dst, Register src_low, Register src_high) {
+ mtc1(src_low, dst);
+ mthc1(src_high, dst);
+ }
+
+ inline void Move_d(FPURegister dst, FPURegister src) {
+ if (dst != src) {
+ mov_d(dst, src);
+ }
+ }
+
+ inline void Move_s(FPURegister dst, FPURegister src) {
+ if (dst != src) {
+ mov_s(dst, src);
+ }
+ }
+
+ void Move(FPURegister dst, float imm) { Move(dst, bit_cast<uint32_t>(imm)); }
+ void Move(FPURegister dst, double imm) { Move(dst, bit_cast<uint64_t>(imm)); }
+ void Move(FPURegister dst, uint32_t src);
+ void Move(FPURegister dst, uint64_t src);
+
+ // DaddOverflow sets overflow register to a negative value if
+ // overflow occured, otherwise it is zero or positive
+ void DaddOverflow(Register dst, Register left, const Operand& right,
+ Register overflow);
+ // DsubOverflow sets overflow register to a negative value if
+ // overflow occured, otherwise it is zero or positive
+ void DsubOverflow(Register dst, Register left, const Operand& right,
+ Register overflow);
+ // MulOverflow sets overflow register to zero if no overflow occured
+ void MulOverflow(Register dst, Register left, const Operand& right,
+ Register overflow);
+
+// Number of instructions needed for calculation of switch table entry address
+#ifdef _MIPS_ARCH_MIPS64R6
+ static const int kSwitchTablePrologueSize = 6;
+#else
+ static const int kSwitchTablePrologueSize = 11;
+#endif
+
+ // GetLabelFunction must be lambda '[](size_t index) -> Label*' or a
+ // functor/function with 'Label *func(size_t index)' declaration.
+ template <typename Func>
+ void GenerateSwitchTable(Register index, size_t case_count,
+ Func GetLabelFunction);
+
+ // Load an object from the root table.
+ void LoadRoot(Register destination, RootIndex index) override;
+ void LoadRoot(Register destination, RootIndex index, Condition cond,
+ Register src1, const Operand& src2);
+
+ // If the value is a NaN, canonicalize the value else, do nothing.
+ void FPUCanonicalizeNaN(const DoubleRegister dst, const DoubleRegister src);
+
+ // ---------------------------------------------------------------------------
+ // FPU macros. These do not handle special cases like NaN or +- inf.
+
+ // Convert unsigned word to double.
+ void Cvt_d_uw(FPURegister fd, FPURegister fs);
+ void Cvt_d_uw(FPURegister fd, Register rs);
+
+ // Convert unsigned long to double.
+ void Cvt_d_ul(FPURegister fd, FPURegister fs);
+ void Cvt_d_ul(FPURegister fd, Register rs);
+
+ // Convert unsigned word to float.
+ void Cvt_s_uw(FPURegister fd, FPURegister fs);
+ void Cvt_s_uw(FPURegister fd, Register rs);
+
+ // Convert unsigned long to float.
+ void Cvt_s_ul(FPURegister fd, FPURegister fs);
+ void Cvt_s_ul(FPURegister fd, Register rs);
+
+ // Convert double to unsigned word.
+ void Trunc_uw_d(FPURegister fd, FPURegister fs, FPURegister scratch);
+ void Trunc_uw_d(Register rd, FPURegister fs, FPURegister scratch);
+
+ // Convert double to unsigned long.
+ void Trunc_ul_d(FPURegister fd, FPURegister fs, FPURegister scratch,
+ Register result = no_reg);
+ void Trunc_ul_d(Register rd, FPURegister fs, FPURegister scratch,
+ Register result = no_reg);
+
+ // Convert single to unsigned long.
+ void Trunc_ul_s(FPURegister fd, FPURegister fs, FPURegister scratch,
+ Register result = no_reg);
+ void Trunc_ul_s(Register rd, FPURegister fs, FPURegister scratch,
+ Register result = no_reg);
+
+ // Round double functions
+ void Trunc_d_d(FPURegister fd, FPURegister fs);
+ void Round_d_d(FPURegister fd, FPURegister fs);
+ void Floor_d_d(FPURegister fd, FPURegister fs);
+ void Ceil_d_d(FPURegister fd, FPURegister fs);
+
+ // Round float functions
+ void Trunc_s_s(FPURegister fd, FPURegister fs);
+ void Round_s_s(FPURegister fd, FPURegister fs);
+ void Floor_s_s(FPURegister fd, FPURegister fs);
+ void Ceil_s_s(FPURegister fd, FPURegister fs);
+
+ void MSARoundW(MSARegister dst, MSARegister src, FPURoundingMode mode);
+ void MSARoundD(MSARegister dst, MSARegister src, FPURoundingMode mode);
+
+ // Jump the register contains a smi.
+ void JumpIfSmi(Register value, Label* smi_label, Register scratch = at,
+ BranchDelaySlot bd = PROTECT);
+
+ void JumpIfEqual(Register a, int32_t b, Label* dest) {
+ li(kScratchReg, Operand(b));
+ Branch(dest, eq, a, Operand(kScratchReg));
+ }
+
+ void JumpIfLessThan(Register a, int32_t b, Label* dest) {
+ li(kScratchReg, Operand(b));
+ Branch(dest, lt, a, Operand(kScratchReg));
+ }
+
+ // Push a standard frame, consisting of ra, fp, context and JS function.
+ void PushStandardFrame(Register function_reg);
+
+ // Get the actual activation frame alignment for target environment.
+ static int ActivationFrameAlignment();
+
+ // Load Scaled Address instructions. Parameter sa (shift argument) must be
+ // between [1, 31] (inclusive). On pre-r6 architectures the scratch register
+ // may be clobbered.
+ void Lsa(Register rd, Register rs, Register rt, uint8_t sa,
+ Register scratch = at);
+ void Dlsa(Register rd, Register rs, Register rt, uint8_t sa,
+ Register scratch = at);
+
+ // Compute the start of the generated instruction stream from the current PC.
+ // This is an alternative to embedding the {CodeObject} handle as a reference.
+ void ComputeCodeStartAddress(Register dst);
+
+ void ResetSpeculationPoisonRegister();
+
+ // Control-flow integrity:
+
+ // Define a function entrypoint. This doesn't emit any code for this
+ // architecture, as control-flow integrity is not supported for it.
+ void CodeEntry() {}
+ // Define an exception handler.
+ void ExceptionHandler() {}
+ // Define an exception handler and bind a label.
+ void BindExceptionHandler(Label* label) { bind(label); }
+
+ protected:
+ inline Register GetRtAsRegisterHelper(const Operand& rt, Register scratch);
+ inline int32_t GetOffset(int32_t offset, Label* L, OffsetSize bits);
+
+ private:
+ bool has_double_zero_reg_set_ = false;
+
+ // Performs a truncating conversion of a floating point number as used by
+ // the JS bitwise operations. See ECMA-262 9.5: ToInt32. Goes to 'done' if it
+ // succeeds, otherwise falls through if result is saturated. On return
+ // 'result' either holds answer, or is clobbered on fall through.
+ void TryInlineTruncateDoubleToI(Register result, DoubleRegister input,
+ Label* done);
+
+ void CompareF(SecondaryField sizeField, FPUCondition cc, FPURegister cmp1,
+ FPURegister cmp2);
+
+ void CompareIsNanF(SecondaryField sizeField, FPURegister cmp1,
+ FPURegister cmp2);
+
+ void BranchShortMSA(MSABranchDF df, Label* target, MSABranchCondition cond,
+ MSARegister wt, BranchDelaySlot bd = PROTECT);
+
+ void CallCFunctionHelper(Register function, int num_reg_arguments,
+ int num_double_arguments);
+
+ // TODO(mips) Reorder parameters so out parameters come last.
+ bool CalculateOffset(Label* L, int32_t* offset, OffsetSize bits);
+ bool CalculateOffset(Label* L, int32_t* offset, OffsetSize bits,
+ Register* scratch, const Operand& rt);
+
+ void BranchShortHelperR6(int32_t offset, Label* L);
+ void BranchShortHelper(int16_t offset, Label* L, BranchDelaySlot bdslot);
+ bool BranchShortHelperR6(int32_t offset, Label* L, Condition cond,
+ Register rs, const Operand& rt);
+ bool BranchShortHelper(int16_t offset, Label* L, Condition cond, Register rs,
+ const Operand& rt, BranchDelaySlot bdslot);
+ bool BranchShortCheck(int32_t offset, Label* L, Condition cond, Register rs,
+ const Operand& rt, BranchDelaySlot bdslot);
+
+ void BranchAndLinkShortHelperR6(int32_t offset, Label* L);
+ void BranchAndLinkShortHelper(int16_t offset, Label* L,
+ BranchDelaySlot bdslot);
+ void BranchAndLinkShort(int32_t offset, BranchDelaySlot bdslot = PROTECT);
+ void BranchAndLinkShort(Label* L, BranchDelaySlot bdslot = PROTECT);
+ bool BranchAndLinkShortHelperR6(int32_t offset, Label* L, Condition cond,
+ Register rs, const Operand& rt);
+ bool BranchAndLinkShortHelper(int16_t offset, Label* L, Condition cond,
+ Register rs, const Operand& rt,
+ BranchDelaySlot bdslot);
+ bool BranchAndLinkShortCheck(int32_t offset, Label* L, Condition cond,
+ Register rs, const Operand& rt,
+ BranchDelaySlot bdslot);
+ void BranchLong(Label* L, BranchDelaySlot bdslot);
+ void BranchAndLinkLong(Label* L, BranchDelaySlot bdslot);
+
+ template <typename RoundFunc>
+ void RoundDouble(FPURegister dst, FPURegister src, FPURoundingMode mode,
+ RoundFunc round);
+
+ template <typename RoundFunc>
+ void RoundFloat(FPURegister dst, FPURegister src, FPURoundingMode mode,
+ RoundFunc round);
+
+ // Push a fixed frame, consisting of ra, fp.
+ void PushCommonFrame(Register marker_reg = no_reg);
+
+ void CallRecordWriteStub(Register object, Register address,
+ RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode, Handle<Code> code_target,
+ Address wasm_target);
+};
+
+// MacroAssembler implements a collection of frequently used macros.
+class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler {
+ public:
+ using TurboAssembler::TurboAssembler;
+
+ // It assumes that the arguments are located below the stack pointer.
+ // argc is the number of arguments not including the receiver.
+ // TODO(victorgomes): Remove this function once we stick with the reversed
+ // arguments order.
+ void LoadReceiver(Register dest, Register argc) {
+ Ld(dest, MemOperand(sp, 0));
+ }
+
+ void StoreReceiver(Register rec, Register argc, Register scratch) {
+ Sd(rec, MemOperand(sp, 0));
+ }
+
+ bool IsNear(Label* L, Condition cond, int rs_reg);
+
+ // Swap two registers. If the scratch register is omitted then a slightly
+ // less efficient form using xor instead of mov is emitted.
+ void Swap(Register reg1, Register reg2, Register scratch = no_reg);
+
+ void PushRoot(RootIndex index) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ LoadRoot(scratch, index);
+ Push(scratch);
+ }
+
+ // Compare the object in a register to a value and jump if they are equal.
+ void JumpIfRoot(Register with, RootIndex index, Label* if_equal) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ LoadRoot(scratch, index);
+ Branch(if_equal, eq, with, Operand(scratch));
+ }
+
+ // Compare the object in a register to a value and jump if they are not equal.
+ void JumpIfNotRoot(Register with, RootIndex index, Label* if_not_equal) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ LoadRoot(scratch, index);
+ Branch(if_not_equal, ne, with, Operand(scratch));
+ }
+
+ // Checks if value is in range [lower_limit, higher_limit] using a single
+ // comparison.
+ void JumpIfIsInRange(Register value, unsigned lower_limit,
+ unsigned higher_limit, Label* on_in_range);
+
+ // ---------------------------------------------------------------------------
+ // GC Support
+
+ // Notify the garbage collector that we wrote a pointer into an object.
+ // |object| is the object being stored into, |value| is the object being
+ // stored. value and scratch registers are clobbered by the operation.
+ // The offset is the offset from the start of the object, not the offset from
+ // the tagged HeapObject pointer. For use with FieldOperand(reg, off).
+ void RecordWriteField(
+ Register object, int offset, Register value, Register scratch,
+ RAStatus ra_status, SaveFPRegsMode save_fp,
+ RememberedSetAction remembered_set_action = EMIT_REMEMBERED_SET,
+ SmiCheck smi_check = INLINE_SMI_CHECK);
+
+ // For a given |object| notify the garbage collector that the slot |address|
+ // has been written. |value| is the object being stored. The value and
+ // address registers are clobbered by the operation.
+ void RecordWrite(
+ Register object, Register address, Register value, RAStatus ra_status,
+ SaveFPRegsMode save_fp,
+ RememberedSetAction remembered_set_action = EMIT_REMEMBERED_SET,
+ SmiCheck smi_check = INLINE_SMI_CHECK);
+
+ void Pref(int32_t hint, const MemOperand& rs);
+
+ // ---------------------------------------------------------------------------
+ // Pseudo-instructions.
+
+ void LoadWordPair(Register rd, const MemOperand& rs, Register scratch = at);
+ void StoreWordPair(Register rd, const MemOperand& rs, Register scratch = at);
+
+ // Convert double to unsigned long.
+ void Trunc_l_ud(FPURegister fd, FPURegister fs, FPURegister scratch);
+
+ void Trunc_l_d(FPURegister fd, FPURegister fs);
+ void Round_l_d(FPURegister fd, FPURegister fs);
+ void Floor_l_d(FPURegister fd, FPURegister fs);
+ void Ceil_l_d(FPURegister fd, FPURegister fs);
+
+ void Trunc_w_d(FPURegister fd, FPURegister fs);
+ void Round_w_d(FPURegister fd, FPURegister fs);
+ void Floor_w_d(FPURegister fd, FPURegister fs);
+ void Ceil_w_d(FPURegister fd, FPURegister fs);
+
+ void Madd_s(FPURegister fd, FPURegister fr, FPURegister fs, FPURegister ft,
+ FPURegister scratch);
+ void Madd_d(FPURegister fd, FPURegister fr, FPURegister fs, FPURegister ft,
+ FPURegister scratch);
+ void Msub_s(FPURegister fd, FPURegister fr, FPURegister fs, FPURegister ft,
+ FPURegister scratch);
+ void Msub_d(FPURegister fd, FPURegister fr, FPURegister fs, FPURegister ft,
+ FPURegister scratch);
+
+ void BranchShortMSA(MSABranchDF df, Label* target, MSABranchCondition cond,
+ MSARegister wt, BranchDelaySlot bd = PROTECT);
+
+ // Truncates a double using a specific rounding mode, and writes the value
+ // to the result register.
+ // The except_flag will contain any exceptions caused by the instruction.
+ // If check_inexact is kDontCheckForInexactConversion, then the inexact
+ // exception is masked.
+ void EmitFPUTruncate(
+ FPURoundingMode rounding_mode, Register result,
+ DoubleRegister double_input, Register scratch,
+ DoubleRegister double_scratch, Register except_flag,
+ CheckForInexactConversion check_inexact = kDontCheckForInexactConversion);
+
+ // Enter exit frame.
+ // argc - argument count to be dropped by LeaveExitFrame.
+ // save_doubles - saves FPU registers on stack, currently disabled.
+ // stack_space - extra stack space.
+ void EnterExitFrame(bool save_doubles, int stack_space = 0,
+ StackFrame::Type frame_type = StackFrame::EXIT);
+
+ // Leave the current exit frame.
+ void LeaveExitFrame(bool save_doubles, Register arg_count,
+ bool do_return = NO_EMIT_RETURN,
+ bool argument_count_is_length = false);
+
+ void LoadMap(Register destination, Register object);
+
+ // Make sure the stack is aligned. Only emits code in debug mode.
+ void AssertStackIsAligned();
+
+ // Load the global proxy from the current context.
+ void LoadGlobalProxy(Register dst) {
+ LoadNativeContextSlot(Context::GLOBAL_PROXY_INDEX, dst);
+ }
+
+ void LoadNativeContextSlot(int index, Register dst);
+
+ // Load the initial map from the global function. The registers
+ // function and map can be the same, function is then overwritten.
+ void LoadGlobalFunctionInitialMap(Register function, Register map,
+ Register scratch);
+
+ // -------------------------------------------------------------------------
+ // JavaScript invokes.
+
+ // Invoke the JavaScript function code by either calling or jumping.
+ void InvokeFunctionCode(Register function, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count, InvokeFlag flag);
+
+ // On function call, call into the debugger if necessary.
+ void CheckDebugHook(Register fun, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count);
+
+ // Invoke the JavaScript function in the given register. Changes the
+ // current context to the context in the function before invoking.
+ void InvokeFunctionWithNewTarget(Register function, Register new_target,
+ Register actual_parameter_count,
+ InvokeFlag flag);
+ void InvokeFunction(Register function, Register expected_parameter_count,
+ Register actual_parameter_count, InvokeFlag flag);
+
+ // Frame restart support.
+ void MaybeDropFrames();
+
+ // Exception handling.
+
+ // Push a new stack handler and link into stack handler chain.
+ void PushStackHandler();
+
+ // Unlink the stack handler on top of the stack from the stack handler chain.
+ // Must preserve the result register.
+ void PopStackHandler();
+
+ // -------------------------------------------------------------------------
+ // Support functions.
+
+ void GetObjectType(Register function, Register map, Register type_reg);
+
+ // -------------------------------------------------------------------------
+ // Runtime calls.
+
+ // Call a runtime routine.
+ void CallRuntime(const Runtime::Function* f, int num_arguments,
+ SaveFPRegsMode save_doubles = kDontSaveFPRegs);
+
+ // Convenience function: Same as above, but takes the fid instead.
+ void CallRuntime(Runtime::FunctionId fid,
+ SaveFPRegsMode save_doubles = kDontSaveFPRegs) {
+ const Runtime::Function* function = Runtime::FunctionForId(fid);
+ CallRuntime(function, function->nargs, save_doubles);
+ }
+
+ // Convenience function: Same as above, but takes the fid instead.
+ void CallRuntime(Runtime::FunctionId fid, int num_arguments,
+ SaveFPRegsMode save_doubles = kDontSaveFPRegs) {
+ CallRuntime(Runtime::FunctionForId(fid), num_arguments, save_doubles);
+ }
+
+ // Convenience function: tail call a runtime routine (jump).
+ void TailCallRuntime(Runtime::FunctionId fid);
+
+ // Jump to the builtin routine.
+ void JumpToExternalReference(const ExternalReference& builtin,
+ BranchDelaySlot bd = PROTECT,
+ bool builtin_exit_frame = false);
+
+ // Generates a trampoline to jump to the off-heap instruction stream.
+ void JumpToInstructionStream(Address entry);
+
+ // ---------------------------------------------------------------------------
+ // In-place weak references.
+ void LoadWeakValue(Register out, Register in, Label* target_if_cleared);
+
+ // -------------------------------------------------------------------------
+ // StatsCounter support.
+
+ void IncrementCounter(StatsCounter* counter, int value, Register scratch1,
+ Register scratch2);
+ void DecrementCounter(StatsCounter* counter, int value, Register scratch1,
+ Register scratch2);
+
+ // -------------------------------------------------------------------------
+ // Stack limit utilities
+
+ enum StackLimitKind { kInterruptStackLimit, kRealStackLimit };
+ void LoadStackLimit(Register destination, StackLimitKind kind);
+ void StackOverflowCheck(Register num_args, Register scratch1,
+ Register scratch2, Label* stack_overflow);
+
+ // ---------------------------------------------------------------------------
+ // Smi utilities.
+
+ void SmiTag(Register dst, Register src) {
+ STATIC_ASSERT(kSmiTag == 0);
+ if (SmiValuesAre32Bits()) {
+ dsll32(dst, src, 0);
+ } else {
+ DCHECK(SmiValuesAre31Bits());
+ Addu(dst, src, src);
+ }
+ }
+
+ void SmiTag(Register reg) { SmiTag(reg, reg); }
+
+ // Left-shifted from int32 equivalent of Smi.
+ void SmiScale(Register dst, Register src, int scale) {
+ if (SmiValuesAre32Bits()) {
+ // The int portion is upper 32-bits of 64-bit word.
+ dsra(dst, src, kSmiShift - scale);
+ } else {
+ DCHECK(SmiValuesAre31Bits());
+ DCHECK_GE(scale, kSmiTagSize);
+ sll(dst, src, scale - kSmiTagSize);
+ }
+ }
+
+ // Test if the register contains a smi.
+ inline void SmiTst(Register value, Register scratch) {
+ And(scratch, value, Operand(kSmiTagMask));
+ }
+
+ // Jump if the register contains a non-smi.
+ void JumpIfNotSmi(Register value, Label* not_smi_label, Register scratch = at,
+ BranchDelaySlot bd = PROTECT);
+
+ // Abort execution if argument is a smi, enabled via --debug-code.
+ void AssertNotSmi(Register object);
+ void AssertSmi(Register object);
+
+ // Abort execution if argument is not a Constructor, enabled via --debug-code.
+ void AssertConstructor(Register object);
+
+ // Abort execution if argument is not a JSFunction, enabled via --debug-code.
+ void AssertFunction(Register object);
+
+ // Abort execution if argument is not a JSBoundFunction,
+ // enabled via --debug-code.
+ void AssertBoundFunction(Register object);
+
+ // Abort execution if argument is not a JSGeneratorObject (or subclass),
+ // enabled via --debug-code.
+ void AssertGeneratorObject(Register object);
+
+ // Abort execution if argument is not undefined or an AllocationSite, enabled
+ // via --debug-code.
+ void AssertUndefinedOrAllocationSite(Register object, Register scratch);
+
+ template <typename Field>
+ void DecodeField(Register dst, Register src) {
+ Ext(dst, src, Field::kShift, Field::kSize);
+ }
+
+ template <typename Field>
+ void DecodeField(Register reg) {
+ DecodeField<Field>(reg, reg);
+ }
+
+ private:
+ // Helper functions for generating invokes.
+ void InvokePrologue(Register expected_parameter_count,
+ Register actual_parameter_count, Label* done,
+ InvokeFlag flag);
+
+ // Compute memory operands for safepoint stack slots.
+ static int SafepointRegisterStackIndex(int reg_code);
+
+ // Needs access to SafepointRegisterStackIndex for compiled frame
+ // traversal.
+ friend class CommonFrame;
+
+ DISALLOW_IMPLICIT_CONSTRUCTORS(MacroAssembler);
+};
+
+template <typename Func>
+void TurboAssembler::GenerateSwitchTable(Register index, size_t case_count,
+ Func GetLabelFunction) {
+ // Ensure that dd-ed labels following this instruction use 8 bytes aligned
+ // addresses.
+ BlockTrampolinePoolFor(static_cast<int>(case_count) * 2 +
+ kSwitchTablePrologueSize);
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ if (kArchVariant >= kMips64r6) {
+ // Opposite of Align(8) as we have odd number of instructions in this case.
+ if ((pc_offset() & 7) == 0) {
+ nop();
+ }
+ addiupc(scratch, 5);
+ Dlsa(scratch, scratch, index, kPointerSizeLog2);
+ Ld(scratch, MemOperand(scratch));
+ } else {
+ Label here;
+ Align(8);
+ push(ra);
+ bal(&here);
+ dsll(scratch, index, kPointerSizeLog2); // Branch delay slot.
+ bind(&here);
+ daddu(scratch, scratch, ra);
+ pop(ra);
+ Ld(scratch, MemOperand(scratch, 6 * v8::internal::kInstrSize));
+ }
+ jr(scratch);
+ nop(); // Branch delay slot nop.
+ for (size_t index = 0; index < case_count; ++index) {
+ dd(GetLabelFunction(index));
+ }
+}
+
+#define ACCESS_MASM(masm) masm->
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_MIPS64_MACRO_ASSEMBLER_MIPS64_H_
diff --git a/src/codegen/mips64/register-mips64.h b/src/codegen/mips64/register-mips64.h
new file mode 100644
index 0000000..d7b45ed
--- /dev/null
+++ b/src/codegen/mips64/register-mips64.h
@@ -0,0 +1,395 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_MIPS64_REGISTER_MIPS64_H_
+#define V8_CODEGEN_MIPS64_REGISTER_MIPS64_H_
+
+#include "src/codegen/mips64/constants-mips64.h"
+#include "src/codegen/register.h"
+#include "src/codegen/reglist.h"
+
+namespace v8 {
+namespace internal {
+
+// clang-format off
+#define GENERAL_REGISTERS(V) \
+ V(zero_reg) V(at) V(v0) V(v1) V(a0) V(a1) V(a2) V(a3) \
+ V(a4) V(a5) V(a6) V(a7) V(t0) V(t1) V(t2) V(t3) \
+ V(s0) V(s1) V(s2) V(s3) V(s4) V(s5) V(s6) V(s7) V(t8) V(t9) \
+ V(k0) V(k1) V(gp) V(sp) V(fp) V(ra)
+
+#define ALLOCATABLE_GENERAL_REGISTERS(V) \
+ V(a0) V(a1) V(a2) V(a3) \
+ V(a4) V(a5) V(a6) V(a7) V(t0) V(t1) V(t2) V(s7) \
+ V(v0) V(v1)
+
+#define DOUBLE_REGISTERS(V) \
+ V(f0) V(f1) V(f2) V(f3) V(f4) V(f5) V(f6) V(f7) \
+ V(f8) V(f9) V(f10) V(f11) V(f12) V(f13) V(f14) V(f15) \
+ V(f16) V(f17) V(f18) V(f19) V(f20) V(f21) V(f22) V(f23) \
+ V(f24) V(f25) V(f26) V(f27) V(f28) V(f29) V(f30) V(f31)
+
+#define FLOAT_REGISTERS DOUBLE_REGISTERS
+#define SIMD128_REGISTERS(V) \
+ V(w0) V(w1) V(w2) V(w3) V(w4) V(w5) V(w6) V(w7) \
+ V(w8) V(w9) V(w10) V(w11) V(w12) V(w13) V(w14) V(w15) \
+ V(w16) V(w17) V(w18) V(w19) V(w20) V(w21) V(w22) V(w23) \
+ V(w24) V(w25) V(w26) V(w27) V(w28) V(w29) V(w30) V(w31)
+
+#define ALLOCATABLE_DOUBLE_REGISTERS(V) \
+ V(f0) V(f2) V(f4) V(f6) V(f8) V(f10) V(f12) V(f14) \
+ V(f16) V(f18) V(f20) V(f22) V(f24) V(f26)
+// clang-format on
+
+// Note that the bit values must match those used in actual instruction
+// encoding.
+const int kNumRegs = 32;
+
+const RegList kJSCallerSaved = 1 << 2 | // v0
+ 1 << 3 | // v1
+ 1 << 4 | // a0
+ 1 << 5 | // a1
+ 1 << 6 | // a2
+ 1 << 7 | // a3
+ 1 << 8 | // a4
+ 1 << 9 | // a5
+ 1 << 10 | // a6
+ 1 << 11 | // a7
+ 1 << 12 | // t0
+ 1 << 13 | // t1
+ 1 << 14 | // t2
+ 1 << 15; // t3
+
+const int kNumJSCallerSaved = 14;
+
+// Callee-saved registers preserved when switching from C to JavaScript.
+const RegList kCalleeSaved = 1 << 16 | // s0
+ 1 << 17 | // s1
+ 1 << 18 | // s2
+ 1 << 19 | // s3
+ 1 << 20 | // s4
+ 1 << 21 | // s5
+ 1 << 22 | // s6 (roots in Javascript code)
+ 1 << 23 | // s7 (cp in Javascript code)
+ 1 << 30; // fp/s8
+
+const int kNumCalleeSaved = 9;
+
+const RegList kCalleeSavedFPU = 1 << 20 | // f20
+ 1 << 22 | // f22
+ 1 << 24 | // f24
+ 1 << 26 | // f26
+ 1 << 28 | // f28
+ 1 << 30; // f30
+
+const int kNumCalleeSavedFPU = 6;
+
+const RegList kCallerSavedFPU = 1 << 0 | // f0
+ 1 << 2 | // f2
+ 1 << 4 | // f4
+ 1 << 6 | // f6
+ 1 << 8 | // f8
+ 1 << 10 | // f10
+ 1 << 12 | // f12
+ 1 << 14 | // f14
+ 1 << 16 | // f16
+ 1 << 18; // f18
+
+// Number of registers for which space is reserved in safepoints. Must be a
+// multiple of 8.
+const int kNumSafepointRegisters = 24;
+
+// Define the list of registers actually saved at safepoints.
+// Note that the number of saved registers may be smaller than the reserved
+// space, i.e. kNumSafepointSavedRegisters <= kNumSafepointRegisters.
+const RegList kSafepointSavedRegisters = kJSCallerSaved | kCalleeSaved;
+const int kNumSafepointSavedRegisters = kNumJSCallerSaved + kNumCalleeSaved;
+
+const int kUndefIndex = -1;
+// Map with indexes on stack that corresponds to codes of saved registers.
+const int kSafepointRegisterStackIndexMap[kNumRegs] = {kUndefIndex, // zero_reg
+ kUndefIndex, // at
+ 0, // v0
+ 1, // v1
+ 2, // a0
+ 3, // a1
+ 4, // a2
+ 5, // a3
+ 6, // a4
+ 7, // a5
+ 8, // a6
+ 9, // a7
+ 10, // t0
+ 11, // t1
+ 12, // t2
+ 13, // t3
+ 14, // s0
+ 15, // s1
+ 16, // s2
+ 17, // s3
+ 18, // s4
+ 19, // s5
+ 20, // s6
+ 21, // s7
+ kUndefIndex, // t8
+ kUndefIndex, // t9
+ kUndefIndex, // k0
+ kUndefIndex, // k1
+ kUndefIndex, // gp
+ kUndefIndex, // sp
+ 22, // fp
+ kUndefIndex};
+
+// CPU Registers.
+//
+// 1) We would prefer to use an enum, but enum values are assignment-
+// compatible with int, which has caused code-generation bugs.
+//
+// 2) We would prefer to use a class instead of a struct but we don't like
+// the register initialization to depend on the particular initialization
+// order (which appears to be different on OS X, Linux, and Windows for the
+// installed versions of C++ we tried). Using a struct permits C-style
+// "initialization". Also, the Register objects cannot be const as this
+// forces initialization stubs in MSVC, making us dependent on initialization
+// order.
+//
+// 3) By not using an enum, we are possibly preventing the compiler from
+// doing certain constant folds, which may significantly reduce the
+// code generated for some assembly instructions (because they boil down
+// to a few constants). If this is a problem, we could change the code
+// such that we use an enum in optimized mode, and the struct in debug
+// mode. This way we get the compile-time error checking in debug mode
+// and best performance in optimized code.
+
+// -----------------------------------------------------------------------------
+// Implementation of Register and FPURegister.
+
+enum RegisterCode {
+#define REGISTER_CODE(R) kRegCode_##R,
+ GENERAL_REGISTERS(REGISTER_CODE)
+#undef REGISTER_CODE
+ kRegAfterLast
+};
+
+class Register : public RegisterBase<Register, kRegAfterLast> {
+ public:
+#if defined(V8_TARGET_LITTLE_ENDIAN)
+ static constexpr int kMantissaOffset = 0;
+ static constexpr int kExponentOffset = 4;
+#elif defined(V8_TARGET_BIG_ENDIAN)
+ static constexpr int kMantissaOffset = 4;
+ static constexpr int kExponentOffset = 0;
+#else
+#error Unknown endianness
+#endif
+
+ private:
+ friend class RegisterBase;
+ explicit constexpr Register(int code) : RegisterBase(code) {}
+};
+
+// s7: context register
+// s3: scratch register
+// s4: scratch register 2
+#define DECLARE_REGISTER(R) \
+ constexpr Register R = Register::from_code(kRegCode_##R);
+GENERAL_REGISTERS(DECLARE_REGISTER)
+#undef DECLARE_REGISTER
+
+constexpr Register no_reg = Register::no_reg();
+
+int ToNumber(Register reg);
+
+Register ToRegister(int num);
+
+constexpr bool kPadArguments = false;
+constexpr bool kSimpleFPAliasing = true;
+constexpr bool kSimdMaskRegisters = false;
+
+enum MSARegisterCode {
+#define REGISTER_CODE(R) kMsaCode_##R,
+ SIMD128_REGISTERS(REGISTER_CODE)
+#undef REGISTER_CODE
+ kMsaAfterLast
+};
+
+// MIPS SIMD (MSA) register
+class MSARegister : public RegisterBase<MSARegister, kMsaAfterLast> {
+ friend class RegisterBase;
+ explicit constexpr MSARegister(int code) : RegisterBase(code) {}
+};
+
+enum DoubleRegisterCode {
+#define REGISTER_CODE(R) kDoubleCode_##R,
+ DOUBLE_REGISTERS(REGISTER_CODE)
+#undef REGISTER_CODE
+ kDoubleAfterLast
+};
+
+// Coprocessor register.
+class FPURegister : public RegisterBase<FPURegister, kDoubleAfterLast> {
+ public:
+ // TODO(plind): Warning, inconsistent numbering here. kNumFPURegisters refers
+ // to number of 32-bit FPU regs, but kNumAllocatableRegisters refers to
+ // number of Double regs (64-bit regs, or FPU-reg-pairs).
+
+ FPURegister low() const {
+ // TODO(plind): Create DCHECK for FR=0 mode. This usage suspect for FR=1.
+ // Find low reg of a Double-reg pair, which is the reg itself.
+ DCHECK_EQ(code() % 2, 0); // Specified Double reg must be even.
+ return FPURegister::from_code(code());
+ }
+ FPURegister high() const {
+ // TODO(plind): Create DCHECK for FR=0 mode. This usage illegal in FR=1.
+ // Find high reg of a Doubel-reg pair, which is reg + 1.
+ DCHECK_EQ(code() % 2, 0); // Specified Double reg must be even.
+ return FPURegister::from_code(code() + 1);
+ }
+
+ MSARegister toW() const { return MSARegister::from_code(code()); }
+
+ private:
+ friend class RegisterBase;
+ explicit constexpr FPURegister(int code) : RegisterBase(code) {}
+};
+
+// A few double registers are reserved: one as a scratch register and one to
+// hold 0.0.
+// f28: 0.0
+// f30: scratch register.
+
+// V8 now supports the O32 ABI, and the FPU Registers are organized as 32
+// 32-bit registers, f0 through f31. When used as 'double' they are used
+// in pairs, starting with the even numbered register. So a double operation
+// on f0 really uses f0 and f1.
+// (Modern mips hardware also supports 32 64-bit registers, via setting
+// (privileged) Status Register FR bit to 1. This is used by the N32 ABI,
+// but it is not in common use. Someday we will want to support this in v8.)
+
+// For O32 ABI, Floats and Doubles refer to same set of 32 32-bit registers.
+using FloatRegister = FPURegister;
+
+using DoubleRegister = FPURegister;
+
+#define DECLARE_DOUBLE_REGISTER(R) \
+ constexpr DoubleRegister R = DoubleRegister::from_code(kDoubleCode_##R);
+DOUBLE_REGISTERS(DECLARE_DOUBLE_REGISTER)
+#undef DECLARE_DOUBLE_REGISTER
+
+constexpr DoubleRegister no_dreg = DoubleRegister::no_reg();
+
+// SIMD registers.
+using Simd128Register = MSARegister;
+
+#define DECLARE_SIMD128_REGISTER(R) \
+ constexpr Simd128Register R = Simd128Register::from_code(kMsaCode_##R);
+SIMD128_REGISTERS(DECLARE_SIMD128_REGISTER)
+#undef DECLARE_SIMD128_REGISTER
+
+const Simd128Register no_msareg = Simd128Register::no_reg();
+
+// Register aliases.
+// cp is assumed to be a callee saved register.
+constexpr Register kRootRegister = s6;
+constexpr Register cp = s7;
+constexpr Register kScratchReg = s3;
+constexpr Register kScratchReg2 = s4;
+constexpr DoubleRegister kScratchDoubleReg = f30;
+// FPU zero reg is often used to hold 0.0, but it's not hardwired to 0.0.
+constexpr DoubleRegister kDoubleRegZero = f28;
+// Used on mips64r6 for compare operations.
+// We use the last non-callee saved odd register for N64 ABI
+constexpr DoubleRegister kDoubleCompareReg = f23;
+// MSA zero and scratch regs must have the same numbers as FPU zero and scratch
+// MSA zero reg is often used to hold 0, but it's not hardwired to 0.
+constexpr Simd128Register kSimd128RegZero = w28;
+constexpr Simd128Register kSimd128ScratchReg = w30;
+
+// FPU (coprocessor 1) control registers.
+// Currently only FCSR (#31) is implemented.
+struct FPUControlRegister {
+ bool is_valid() const { return reg_code == kFCSRRegister; }
+ bool is(FPUControlRegister creg) const { return reg_code == creg.reg_code; }
+ int code() const {
+ DCHECK(is_valid());
+ return reg_code;
+ }
+ int bit() const {
+ DCHECK(is_valid());
+ return 1 << reg_code;
+ }
+ void setcode(int f) {
+ reg_code = f;
+ DCHECK(is_valid());
+ }
+ // Unfortunately we can't make this private in a struct.
+ int reg_code;
+};
+
+constexpr FPUControlRegister no_fpucreg = {kInvalidFPUControlRegister};
+constexpr FPUControlRegister FCSR = {kFCSRRegister};
+
+// MSA control registers
+struct MSAControlRegister {
+ bool is_valid() const {
+ return (reg_code == kMSAIRRegister) || (reg_code == kMSACSRRegister);
+ }
+ bool is(MSAControlRegister creg) const { return reg_code == creg.reg_code; }
+ int code() const {
+ DCHECK(is_valid());
+ return reg_code;
+ }
+ int bit() const {
+ DCHECK(is_valid());
+ return 1 << reg_code;
+ }
+ void setcode(int f) {
+ reg_code = f;
+ DCHECK(is_valid());
+ }
+ // Unfortunately we can't make this private in a struct.
+ int reg_code;
+};
+
+constexpr MSAControlRegister no_msacreg = {kInvalidMSAControlRegister};
+constexpr MSAControlRegister MSAIR = {kMSAIRRegister};
+constexpr MSAControlRegister MSACSR = {kMSACSRRegister};
+
+// Define {RegisterName} methods for the register types.
+DEFINE_REGISTER_NAMES(Register, GENERAL_REGISTERS)
+DEFINE_REGISTER_NAMES(FPURegister, DOUBLE_REGISTERS)
+DEFINE_REGISTER_NAMES(MSARegister, SIMD128_REGISTERS)
+
+// Give alias names to registers for calling conventions.
+constexpr Register kReturnRegister0 = v0;
+constexpr Register kReturnRegister1 = v1;
+constexpr Register kReturnRegister2 = a0;
+constexpr Register kJSFunctionRegister = a1;
+constexpr Register kContextRegister = s7;
+constexpr Register kAllocateSizeRegister = a0;
+constexpr Register kSpeculationPoisonRegister = a7;
+constexpr Register kInterpreterAccumulatorRegister = v0;
+constexpr Register kInterpreterBytecodeOffsetRegister = t0;
+constexpr Register kInterpreterBytecodeArrayRegister = t1;
+constexpr Register kInterpreterDispatchTableRegister = t2;
+
+constexpr Register kJavaScriptCallArgCountRegister = a0;
+constexpr Register kJavaScriptCallCodeStartRegister = a2;
+constexpr Register kJavaScriptCallTargetRegister = kJSFunctionRegister;
+constexpr Register kJavaScriptCallNewTargetRegister = a3;
+constexpr Register kJavaScriptCallExtraArg1Register = a2;
+
+constexpr Register kOffHeapTrampolineRegister = at;
+constexpr Register kRuntimeCallFunctionRegister = a1;
+constexpr Register kRuntimeCallArgCountRegister = a0;
+constexpr Register kRuntimeCallArgvRegister = a2;
+constexpr Register kWasmInstanceRegister = a0;
+constexpr Register kWasmCompileLazyFuncIndexRegister = t0;
+
+constexpr DoubleRegister kFPReturnRegister0 = f0;
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_MIPS64_REGISTER_MIPS64_H_
diff --git a/src/codegen/optimized-compilation-info.cc b/src/codegen/optimized-compilation-info.cc
new file mode 100644
index 0000000..bf45a5f
--- /dev/null
+++ b/src/codegen/optimized-compilation-info.cc
@@ -0,0 +1,254 @@
+// Copyright 2016 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/optimized-compilation-info.h"
+
+#include "src/api/api.h"
+#include "src/codegen/source-position.h"
+#include "src/debug/debug.h"
+#include "src/execution/isolate.h"
+#include "src/objects/objects-inl.h"
+#include "src/objects/shared-function-info.h"
+#include "src/tracing/trace-event.h"
+#include "src/tracing/traced-value.h"
+#include "src/wasm/function-compiler.h"
+
+namespace v8 {
+namespace internal {
+
+OptimizedCompilationInfo::OptimizedCompilationInfo(
+ Zone* zone, Isolate* isolate, Handle<SharedFunctionInfo> shared,
+ Handle<JSFunction> closure, CodeKind code_kind)
+ : code_kind_(code_kind),
+ zone_(zone),
+ optimization_id_(isolate->NextOptimizationId()) {
+ DCHECK_EQ(*shared, closure->shared());
+ DCHECK(shared->is_compiled());
+ bytecode_array_ = handle(shared->GetBytecodeArray(), isolate);
+ shared_info_ = shared;
+ closure_ = closure;
+
+ // Collect source positions for optimized code when profiling or if debugger
+ // is active, to be able to get more precise source positions at the price of
+ // more memory consumption.
+ if (isolate->NeedsDetailedOptimizedCodeLineInfo()) {
+ set_source_positions();
+ }
+
+ SetTracingFlags(shared->PassesFilter(FLAG_trace_turbo_filter));
+ ConfigureFlags();
+}
+
+OptimizedCompilationInfo::OptimizedCompilationInfo(
+ Vector<const char> debug_name, Zone* zone, CodeKind code_kind)
+ : code_kind_(code_kind),
+ zone_(zone),
+ optimization_id_(kNoOptimizationId),
+ debug_name_(debug_name) {
+ SetTracingFlags(
+ PassesFilter(debug_name, CStrVector(FLAG_trace_turbo_filter)));
+ ConfigureFlags();
+}
+
+#ifdef DEBUG
+bool OptimizedCompilationInfo::FlagSetIsValid(Flag flag) const {
+ switch (flag) {
+ case kPoisonRegisterArguments:
+ return untrusted_code_mitigations();
+ case kFunctionContextSpecializing:
+ return !IsNativeContextIndependent();
+ default:
+ return true;
+ }
+ UNREACHABLE();
+}
+
+bool OptimizedCompilationInfo::FlagGetIsValid(Flag flag) const {
+ switch (flag) {
+ case kPoisonRegisterArguments:
+ if (!GetFlag(kPoisonRegisterArguments)) return true;
+ return untrusted_code_mitigations() && called_with_code_start_register();
+ default:
+ return true;
+ }
+ UNREACHABLE();
+}
+#endif // DEBUG
+
+void OptimizedCompilationInfo::ConfigureFlags() {
+ if (FLAG_untrusted_code_mitigations) set_untrusted_code_mitigations();
+
+ switch (code_kind_) {
+ case CodeKind::TURBOFAN:
+ if (FLAG_function_context_specialization) {
+ set_function_context_specializing();
+ }
+ V8_FALLTHROUGH;
+ case CodeKind::TURBOPROP:
+ case CodeKind::NATIVE_CONTEXT_INDEPENDENT:
+ set_called_with_code_start_register();
+ set_switch_jump_table();
+ if (FLAG_turbo_splitting) set_splitting();
+ if (FLAG_untrusted_code_mitigations) set_poison_register_arguments();
+ // TODO(yangguo): Disable this in case of debugging for crbug.com/826613
+ if (FLAG_analyze_environment_liveness) set_analyze_environment_liveness();
+ break;
+ case CodeKind::BYTECODE_HANDLER:
+ set_called_with_code_start_register();
+ if (FLAG_turbo_splitting) set_splitting();
+ break;
+ case CodeKind::BUILTIN:
+ case CodeKind::FOR_TESTING:
+ if (FLAG_turbo_splitting) set_splitting();
+#if ENABLE_GDB_JIT_INTERFACE && DEBUG
+ set_source_positions();
+#endif // ENABLE_GDB_JIT_INTERFACE && DEBUG
+ break;
+ case CodeKind::WASM_FUNCTION:
+ case CodeKind::WASM_TO_CAPI_FUNCTION:
+ set_switch_jump_table();
+ break;
+ default:
+ break;
+ }
+}
+
+OptimizedCompilationInfo::~OptimizedCompilationInfo() {
+ if (disable_future_optimization() && has_shared_info()) {
+ shared_info()->DisableOptimization(bailout_reason());
+ }
+}
+
+void OptimizedCompilationInfo::ReopenHandlesInNewHandleScope(Isolate* isolate) {
+ if (!shared_info_.is_null()) {
+ shared_info_ = Handle<SharedFunctionInfo>(*shared_info_, isolate);
+ }
+ if (!bytecode_array_.is_null()) {
+ bytecode_array_ = Handle<BytecodeArray>(*bytecode_array_, isolate);
+ }
+ if (!closure_.is_null()) {
+ closure_ = Handle<JSFunction>(*closure_, isolate);
+ }
+ DCHECK(code_.is_null());
+}
+
+void OptimizedCompilationInfo::AbortOptimization(BailoutReason reason) {
+ DCHECK_NE(reason, BailoutReason::kNoReason);
+ if (bailout_reason_ == BailoutReason::kNoReason) {
+ bailout_reason_ = reason;
+ }
+ set_disable_future_optimization();
+}
+
+void OptimizedCompilationInfo::RetryOptimization(BailoutReason reason) {
+ DCHECK_NE(reason, BailoutReason::kNoReason);
+ if (disable_future_optimization()) return;
+ bailout_reason_ = reason;
+}
+
+std::unique_ptr<char[]> OptimizedCompilationInfo::GetDebugName() const {
+ if (!shared_info().is_null()) {
+ return shared_info()->DebugName().ToCString();
+ }
+ Vector<const char> name_vec = debug_name_;
+ if (name_vec.empty()) name_vec = ArrayVector("unknown");
+ std::unique_ptr<char[]> name(new char[name_vec.length() + 1]);
+ memcpy(name.get(), name_vec.begin(), name_vec.length());
+ name[name_vec.length()] = '\0';
+ return name;
+}
+
+StackFrame::Type OptimizedCompilationInfo::GetOutputStackFrameType() const {
+ switch (code_kind()) {
+ case CodeKind::FOR_TESTING:
+ case CodeKind::BYTECODE_HANDLER:
+ case CodeKind::BUILTIN:
+ return StackFrame::STUB;
+ case CodeKind::WASM_FUNCTION:
+ return StackFrame::WASM;
+ case CodeKind::WASM_TO_CAPI_FUNCTION:
+ return StackFrame::WASM_EXIT;
+ case CodeKind::JS_TO_WASM_FUNCTION:
+ return StackFrame::JS_TO_WASM;
+ case CodeKind::WASM_TO_JS_FUNCTION:
+ return StackFrame::WASM_TO_JS;
+ case CodeKind::C_WASM_ENTRY:
+ return StackFrame::C_WASM_ENTRY;
+ default:
+ UNIMPLEMENTED();
+ return StackFrame::NONE;
+ }
+}
+
+void OptimizedCompilationInfo::SetCode(Handle<Code> code) {
+ DCHECK_EQ(code->kind(), code_kind());
+ code_ = code;
+}
+
+void OptimizedCompilationInfo::SetWasmCompilationResult(
+ std::unique_ptr<wasm::WasmCompilationResult> wasm_compilation_result) {
+ wasm_compilation_result_ = std::move(wasm_compilation_result);
+}
+
+std::unique_ptr<wasm::WasmCompilationResult>
+OptimizedCompilationInfo::ReleaseWasmCompilationResult() {
+ return std::move(wasm_compilation_result_);
+}
+
+bool OptimizedCompilationInfo::has_context() const {
+ return !closure().is_null();
+}
+
+Context OptimizedCompilationInfo::context() const {
+ DCHECK(has_context());
+ return closure()->context();
+}
+
+bool OptimizedCompilationInfo::has_native_context() const {
+ return !closure().is_null() && !closure()->native_context().is_null();
+}
+
+NativeContext OptimizedCompilationInfo::native_context() const {
+ DCHECK(has_native_context());
+ return closure()->native_context();
+}
+
+bool OptimizedCompilationInfo::has_global_object() const {
+ return has_native_context();
+}
+
+JSGlobalObject OptimizedCompilationInfo::global_object() const {
+ DCHECK(has_global_object());
+ return native_context().global_object();
+}
+
+int OptimizedCompilationInfo::AddInlinedFunction(
+ Handle<SharedFunctionInfo> inlined_function,
+ Handle<BytecodeArray> inlined_bytecode, SourcePosition pos) {
+ int id = static_cast<int>(inlined_functions_.size());
+ inlined_functions_.push_back(
+ InlinedFunctionHolder(inlined_function, inlined_bytecode, pos));
+ return id;
+}
+
+void OptimizedCompilationInfo::SetTracingFlags(bool passes_filter) {
+ if (!passes_filter) return;
+ if (FLAG_trace_turbo) set_trace_turbo_json();
+ if (FLAG_trace_turbo_graph) set_trace_turbo_graph();
+ if (FLAG_trace_turbo_scheduled) set_trace_turbo_scheduled();
+ if (FLAG_trace_turbo_alloc) set_trace_turbo_allocation();
+ if (FLAG_trace_heap_broker) set_trace_heap_broker();
+}
+
+OptimizedCompilationInfo::InlinedFunctionHolder::InlinedFunctionHolder(
+ Handle<SharedFunctionInfo> inlined_shared_info,
+ Handle<BytecodeArray> inlined_bytecode, SourcePosition pos)
+ : shared_info(inlined_shared_info), bytecode_array(inlined_bytecode) {
+ position.position = pos;
+ // initialized when generating the deoptimization literals
+ position.inlined_function_id = DeoptimizationData::kNotInlinedIndex;
+}
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/optimized-compilation-info.h b/src/codegen/optimized-compilation-info.h
new file mode 100644
index 0000000..6e238d6
--- /dev/null
+++ b/src/codegen/optimized-compilation-info.h
@@ -0,0 +1,322 @@
+// Copyright 2016 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_OPTIMIZED_COMPILATION_INFO_H_
+#define V8_CODEGEN_OPTIMIZED_COMPILATION_INFO_H_
+
+#include <memory>
+
+#include "src/codegen/bailout-reason.h"
+#include "src/codegen/source-position-table.h"
+#include "src/codegen/tick-counter.h"
+#include "src/common/globals.h"
+#include "src/diagnostics/basic-block-profiler.h"
+#include "src/execution/frames.h"
+#include "src/handles/handles.h"
+#include "src/handles/persistent-handles.h"
+#include "src/objects/objects.h"
+#include "src/utils/identity-map.h"
+#include "src/utils/utils.h"
+#include "src/utils/vector.h"
+
+namespace v8 {
+
+namespace tracing {
+class TracedValue;
+} // namespace tracing
+
+namespace internal {
+
+class FunctionLiteral;
+class Isolate;
+class JavaScriptFrame;
+class JSGlobalObject;
+class Zone;
+
+namespace wasm {
+struct WasmCompilationResult;
+} // namespace wasm
+
+// OptimizedCompilationInfo encapsulates the information needed to compile
+// optimized code for a given function, and the results of the optimized
+// compilation.
+class V8_EXPORT_PRIVATE OptimizedCompilationInfo final {
+ public:
+ // Various configuration flags for a compilation, as well as some properties
+ // of the compiled code produced by a compilation.
+
+#define FLAGS(V) \
+ V(FunctionContextSpecializing, function_context_specializing, 0) \
+ V(Inlining, inlining, 1) \
+ V(DisableFutureOptimization, disable_future_optimization, 2) \
+ V(Splitting, splitting, 3) \
+ V(SourcePositions, source_positions, 4) \
+ V(BailoutOnUninitialized, bailout_on_uninitialized, 5) \
+ V(LoopPeeling, loop_peeling, 6) \
+ V(UntrustedCodeMitigations, untrusted_code_mitigations, 7) \
+ V(SwitchJumpTable, switch_jump_table, 8) \
+ V(CalledWithCodeStartRegister, called_with_code_start_register, 9) \
+ V(PoisonRegisterArguments, poison_register_arguments, 10) \
+ V(AllocationFolding, allocation_folding, 11) \
+ V(AnalyzeEnvironmentLiveness, analyze_environment_liveness, 12) \
+ V(TraceTurboJson, trace_turbo_json, 13) \
+ V(TraceTurboGraph, trace_turbo_graph, 14) \
+ V(TraceTurboScheduled, trace_turbo_scheduled, 15) \
+ V(TraceTurboAllocation, trace_turbo_allocation, 16) \
+ V(TraceHeapBroker, trace_heap_broker, 17) \
+ V(WasmRuntimeExceptionSupport, wasm_runtime_exception_support, 18) \
+ V(ConcurrentInlining, concurrent_inlining, 19)
+
+ enum Flag {
+#define DEF_ENUM(Camel, Lower, Bit) k##Camel = 1 << Bit,
+ FLAGS(DEF_ENUM)
+#undef DEF_ENUM
+ };
+
+#define DEF_GETTER(Camel, Lower, Bit) \
+ bool Lower() const { \
+ DCHECK(FlagGetIsValid(k##Camel)); \
+ return GetFlag(k##Camel); \
+ }
+ FLAGS(DEF_GETTER)
+#undef DEF_GETTER
+
+#define DEF_SETTER(Camel, Lower, Bit) \
+ void set_##Lower() { \
+ DCHECK(FlagSetIsValid(k##Camel)); \
+ SetFlag(k##Camel); \
+ }
+ FLAGS(DEF_SETTER)
+#undef DEF_SETTER
+
+#ifdef DEBUG
+ bool FlagGetIsValid(Flag flag) const;
+ bool FlagSetIsValid(Flag flag) const;
+#endif // DEBUG
+
+ // Construct a compilation info for optimized compilation.
+ OptimizedCompilationInfo(Zone* zone, Isolate* isolate,
+ Handle<SharedFunctionInfo> shared,
+ Handle<JSFunction> closure, CodeKind code_kind);
+ // Construct a compilation info for stub compilation, Wasm, and testing.
+ OptimizedCompilationInfo(Vector<const char> debug_name, Zone* zone,
+ CodeKind code_kind);
+
+ ~OptimizedCompilationInfo();
+
+ Zone* zone() { return zone_; }
+ bool is_osr() const { return !osr_offset_.IsNone(); }
+ Handle<SharedFunctionInfo> shared_info() const { return shared_info_; }
+ bool has_shared_info() const { return !shared_info().is_null(); }
+ Handle<BytecodeArray> bytecode_array() const { return bytecode_array_; }
+ bool has_bytecode_array() const { return !bytecode_array_.is_null(); }
+ Handle<JSFunction> closure() const { return closure_; }
+ Handle<Code> code() const { return code_; }
+ CodeKind code_kind() const { return code_kind_; }
+ int32_t builtin_index() const { return builtin_index_; }
+ void set_builtin_index(int32_t index) { builtin_index_ = index; }
+ BailoutId osr_offset() const { return osr_offset_; }
+ JavaScriptFrame* osr_frame() const { return osr_frame_; }
+
+ void SetPoisoningMitigationLevel(PoisoningMitigationLevel poisoning_level) {
+ poisoning_level_ = poisoning_level;
+ }
+ PoisoningMitigationLevel GetPoisoningMitigationLevel() const {
+ return poisoning_level_;
+ }
+
+ // Code getters and setters.
+
+ void SetCode(Handle<Code> code);
+
+ void SetWasmCompilationResult(std::unique_ptr<wasm::WasmCompilationResult>);
+ std::unique_ptr<wasm::WasmCompilationResult> ReleaseWasmCompilationResult();
+
+ bool has_context() const;
+ Context context() const;
+
+ bool has_native_context() const;
+ NativeContext native_context() const;
+
+ bool has_global_object() const;
+ JSGlobalObject global_object() const;
+
+ // Accessors for the different compilation modes.
+ bool IsOptimizing() const {
+ return CodeKindIsOptimizedJSFunction(code_kind());
+ }
+ bool IsNativeContextIndependent() const {
+ return code_kind() == CodeKind::NATIVE_CONTEXT_INDEPENDENT;
+ }
+ bool IsTurboprop() const { return code_kind() == CodeKind::TURBOPROP; }
+ bool IsWasm() const { return code_kind() == CodeKind::WASM_FUNCTION; }
+
+ void SetOptimizingForOsr(BailoutId osr_offset, JavaScriptFrame* osr_frame) {
+ DCHECK(IsOptimizing());
+ osr_offset_ = osr_offset;
+ osr_frame_ = osr_frame;
+ }
+
+ void set_persistent_handles(
+ std::unique_ptr<PersistentHandles> persistent_handles) {
+ DCHECK_NULL(ph_);
+ ph_ = std::move(persistent_handles);
+ DCHECK_NOT_NULL(ph_);
+ }
+
+ void set_canonical_handles(
+ std::unique_ptr<CanonicalHandlesMap> canonical_handles) {
+ DCHECK_NULL(canonical_handles_);
+ canonical_handles_ = std::move(canonical_handles);
+ DCHECK_NOT_NULL(canonical_handles_);
+ }
+
+ void ReopenHandlesInNewHandleScope(Isolate* isolate);
+
+ void AbortOptimization(BailoutReason reason);
+
+ void RetryOptimization(BailoutReason reason);
+
+ BailoutReason bailout_reason() const { return bailout_reason_; }
+
+ int optimization_id() const {
+ DCHECK(IsOptimizing());
+ return optimization_id_;
+ }
+
+ unsigned inlined_bytecode_size() const { return inlined_bytecode_size_; }
+
+ void set_inlined_bytecode_size(unsigned size) {
+ inlined_bytecode_size_ = size;
+ }
+
+ struct InlinedFunctionHolder {
+ Handle<SharedFunctionInfo> shared_info;
+ Handle<BytecodeArray> bytecode_array; // Explicit to prevent flushing.
+ InliningPosition position;
+
+ InlinedFunctionHolder(Handle<SharedFunctionInfo> inlined_shared_info,
+ Handle<BytecodeArray> inlined_bytecode,
+ SourcePosition pos);
+
+ void RegisterInlinedFunctionId(size_t inlined_function_id) {
+ position.inlined_function_id = static_cast<int>(inlined_function_id);
+ }
+ };
+
+ using InlinedFunctionList = std::vector<InlinedFunctionHolder>;
+ InlinedFunctionList& inlined_functions() { return inlined_functions_; }
+
+ // Returns the inlining id for source position tracking.
+ int AddInlinedFunction(Handle<SharedFunctionInfo> inlined_function,
+ Handle<BytecodeArray> inlined_bytecode,
+ SourcePosition pos);
+
+ std::unique_ptr<char[]> GetDebugName() const;
+
+ StackFrame::Type GetOutputStackFrameType() const;
+
+ const char* trace_turbo_filename() const {
+ return trace_turbo_filename_.get();
+ }
+
+ void set_trace_turbo_filename(std::unique_ptr<char[]> filename) {
+ trace_turbo_filename_ = std::move(filename);
+ }
+
+ TickCounter& tick_counter() { return tick_counter_; }
+
+ BasicBlockProfilerData* profiler_data() const { return profiler_data_; }
+ void set_profiler_data(BasicBlockProfilerData* profiler_data) {
+ profiler_data_ = profiler_data;
+ }
+
+ std::unique_ptr<PersistentHandles> DetachPersistentHandles() {
+ DCHECK_NOT_NULL(ph_);
+ return std::move(ph_);
+ }
+
+ std::unique_ptr<CanonicalHandlesMap> DetachCanonicalHandles() {
+ DCHECK_NOT_NULL(canonical_handles_);
+ return std::move(canonical_handles_);
+ }
+
+ private:
+ void ConfigureFlags();
+
+ void SetFlag(Flag flag) { flags_ |= flag; }
+ bool GetFlag(Flag flag) const { return (flags_ & flag) != 0; }
+
+ void SetTracingFlags(bool passes_filter);
+
+ // Compilation flags.
+ unsigned flags_ = 0;
+ PoisoningMitigationLevel poisoning_level_ =
+ PoisoningMitigationLevel::kDontPoison;
+
+ const CodeKind code_kind_;
+ int32_t builtin_index_ = -1;
+
+ // We retain a reference the bytecode array specifically to ensure it doesn't
+ // get flushed while we are optimizing the code.
+ Handle<BytecodeArray> bytecode_array_;
+ Handle<SharedFunctionInfo> shared_info_;
+ Handle<JSFunction> closure_;
+
+ // The compiled code.
+ Handle<Code> code_;
+
+ // Basic block profiling support.
+ BasicBlockProfilerData* profiler_data_ = nullptr;
+
+ // The WebAssembly compilation result, not published in the NativeModule yet.
+ std::unique_ptr<wasm::WasmCompilationResult> wasm_compilation_result_;
+
+ // Entry point when compiling for OSR, {BailoutId::None} otherwise.
+ BailoutId osr_offset_ = BailoutId::None();
+
+ // The zone from which the compilation pipeline working on this
+ // OptimizedCompilationInfo allocates.
+ Zone* const zone_;
+
+ BailoutReason bailout_reason_ = BailoutReason::kNoReason;
+
+ InlinedFunctionList inlined_functions_;
+
+ static constexpr int kNoOptimizationId = -1;
+ const int optimization_id_;
+ unsigned inlined_bytecode_size_ = 0;
+
+ // The current OSR frame for specialization or {nullptr}.
+ JavaScriptFrame* osr_frame_ = nullptr;
+
+ Vector<const char> debug_name_;
+ std::unique_ptr<char[]> trace_turbo_filename_;
+
+ TickCounter tick_counter_;
+
+ // 1) PersistentHandles created via PersistentHandlesScope inside of
+ // CompilationHandleScope
+ // 2) Owned by OptimizedCompilationInfo
+ // 3) Owned by the broker's LocalHeap when entering the LocalHeapScope.
+ // 4) Back to OptimizedCompilationInfo when exiting the LocalHeapScope.
+ //
+ // In normal execution it gets destroyed when PipelineData gets destroyed.
+ // There is a special case in GenerateCodeForTesting where the JSHeapBroker
+ // will not be retired in that same method. In this case, we need to re-attach
+ // the PersistentHandles container to the JSHeapBroker.
+ std::unique_ptr<PersistentHandles> ph_;
+
+ // Canonical handles follow the same path as described by the persistent
+ // handles above. The only difference is that is created in the
+ // CanonicalHandleScope(i.e step 1) is different).
+ std::unique_ptr<CanonicalHandlesMap> canonical_handles_;
+
+ DISALLOW_COPY_AND_ASSIGN(OptimizedCompilationInfo);
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_OPTIMIZED_COMPILATION_INFO_H_
diff --git a/src/codegen/pending-optimization-table.cc b/src/codegen/pending-optimization-table.cc
new file mode 100644
index 0000000..84e36fc
--- /dev/null
+++ b/src/codegen/pending-optimization-table.cc
@@ -0,0 +1,131 @@
+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/pending-optimization-table.h"
+
+#include "src/base/flags.h"
+#include "src/execution/isolate-inl.h"
+#include "src/heap/heap-inl.h"
+#include "src/objects/hash-table.h"
+#include "src/objects/js-objects.h"
+
+namespace v8 {
+namespace internal {
+
+enum class FunctionStatus : int {
+ kPrepareForOptimize = 1 << 0,
+ kMarkForOptimize = 1 << 1,
+ kAllowHeuristicOptimization = 1 << 2,
+};
+
+using FunctionStatusFlags = base::Flags<FunctionStatus>;
+
+void PendingOptimizationTable::PreparedForOptimization(
+ Isolate* isolate, Handle<JSFunction> function,
+ bool allow_heuristic_optimization) {
+ DCHECK(FLAG_testing_d8_test_runner);
+
+ FunctionStatusFlags status = FunctionStatus::kPrepareForOptimize;
+ if (allow_heuristic_optimization) {
+ status |= FunctionStatus::kAllowHeuristicOptimization;
+ }
+
+ Handle<ObjectHashTable> table =
+ isolate->heap()->pending_optimize_for_test_bytecode().IsUndefined()
+ ? ObjectHashTable::New(isolate, 1)
+ : handle(ObjectHashTable::cast(
+ isolate->heap()->pending_optimize_for_test_bytecode()),
+ isolate);
+ Handle<Tuple2> tuple = isolate->factory()->NewTuple2(
+ handle(function->shared().GetBytecodeArray(), isolate),
+ handle(Smi::FromInt(status), isolate), AllocationType::kYoung);
+ table =
+ ObjectHashTable::Put(table, handle(function->shared(), isolate), tuple);
+ isolate->heap()->SetPendingOptimizeForTestBytecode(*table);
+}
+
+bool PendingOptimizationTable::IsHeuristicOptimizationAllowed(
+ Isolate* isolate, JSFunction function) {
+ DCHECK(FLAG_testing_d8_test_runner);
+
+ Handle<Object> table =
+ handle(isolate->heap()->pending_optimize_for_test_bytecode(), isolate);
+ Handle<Object> entry =
+ table->IsUndefined()
+ ? handle(ReadOnlyRoots(isolate).the_hole_value(), isolate)
+ : handle(Handle<ObjectHashTable>::cast(table)->Lookup(
+ handle(function.shared(), isolate)),
+ isolate);
+ if (entry->IsTheHole()) {
+ return true;
+ }
+ DCHECK(entry->IsTuple2());
+ DCHECK(Handle<Tuple2>::cast(entry)->value2().IsSmi());
+ FunctionStatusFlags status(Smi::ToInt(Handle<Tuple2>::cast(entry)->value2()));
+ return status & FunctionStatus::kAllowHeuristicOptimization;
+}
+
+void PendingOptimizationTable::MarkedForOptimization(
+ Isolate* isolate, Handle<JSFunction> function) {
+ DCHECK(FLAG_testing_d8_test_runner);
+
+ Handle<Object> table =
+ handle(isolate->heap()->pending_optimize_for_test_bytecode(), isolate);
+ Handle<Object> entry =
+ table->IsUndefined()
+ ? handle(ReadOnlyRoots(isolate).the_hole_value(), isolate)
+ : handle(Handle<ObjectHashTable>::cast(table)->Lookup(
+ handle(function->shared(), isolate)),
+ isolate);
+ if (entry->IsTheHole()) {
+ PrintF("Error: Function ");
+ function->ShortPrint();
+ PrintF(
+ " should be prepared for optimization with "
+ "%%PrepareFunctionForOptimization before "
+ "%%OptimizeFunctionOnNextCall / %%OptimizeOSR ");
+ UNREACHABLE();
+ }
+
+ DCHECK(entry->IsTuple2());
+ DCHECK(Handle<Tuple2>::cast(entry)->value2().IsSmi());
+ FunctionStatusFlags status(Smi::ToInt(Handle<Tuple2>::cast(entry)->value2()));
+ status = status.without(FunctionStatus::kPrepareForOptimize) |
+ FunctionStatus::kMarkForOptimize;
+ Handle<Tuple2>::cast(entry)->set_value2(Smi::FromInt(status));
+ table = ObjectHashTable::Put(Handle<ObjectHashTable>::cast(table),
+ handle(function->shared(), isolate), entry);
+ isolate->heap()->SetPendingOptimizeForTestBytecode(*table);
+}
+
+void PendingOptimizationTable::FunctionWasOptimized(
+ Isolate* isolate, Handle<JSFunction> function) {
+ DCHECK(FLAG_testing_d8_test_runner);
+
+ if (isolate->heap()->pending_optimize_for_test_bytecode().IsUndefined()) {
+ return;
+ }
+
+ Handle<ObjectHashTable> table =
+ handle(ObjectHashTable::cast(
+ isolate->heap()->pending_optimize_for_test_bytecode()),
+ isolate);
+ Handle<Object> value(table->Lookup(handle(function->shared(), isolate)),
+ isolate);
+ // Remove only if we have already seen %OptimizeFunctionOnNextCall. If it is
+ // optimized for other reasons, still keep holding the bytecode since we may
+ // optimize it later.
+ if (!value->IsTheHole() &&
+ Smi::cast(Handle<Tuple2>::cast(value)->value2()).value() ==
+ static_cast<int>(FunctionStatus::kMarkForOptimize)) {
+ bool was_present;
+ table = table->Remove(isolate, table, handle(function->shared(), isolate),
+ &was_present);
+ DCHECK(was_present);
+ isolate->heap()->SetPendingOptimizeForTestBytecode(*table);
+ }
+}
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/pending-optimization-table.h b/src/codegen/pending-optimization-table.h
new file mode 100644
index 0000000..43b9397
--- /dev/null
+++ b/src/codegen/pending-optimization-table.h
@@ -0,0 +1,51 @@
+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_PENDING_OPTIMIZATION_TABLE_H_
+#define V8_CODEGEN_PENDING_OPTIMIZATION_TABLE_H_
+
+#include "src/common/globals.h"
+
+namespace v8 {
+namespace internal {
+
+// This class adds the functionality to properly test the optimized code. This
+// is only for use in tests. All these functions should only be called when
+// testing_d8_flag_for_tests is set.
+class PendingOptimizationTable {
+ public:
+ // This function should be called before we mark the function for
+ // optimization. Calling this function ensures that |function| is compiled and
+ // has a feedback vector allocated. This also holds on to the bytecode
+ // strongly in pending optimization table preventing the bytecode to be
+ // flushed.
+ static void PreparedForOptimization(Isolate* isolate,
+ Handle<JSFunction> function,
+ bool allow_heuristic_optimization);
+
+ // This function should be called when the function is marked for optimization
+ // via the intrinsics. This will update the state of the bytecode array in the
+ // pending optimization table, so that the entry can be removed once the
+ // function is optimized. If the function is already optimized it removes the
+ // entry from the table.
+ static void MarkedForOptimization(Isolate* isolate,
+ Handle<JSFunction> function);
+
+ // This function should be called once the function is optimized. If there is
+ // an entry in the pending optimization table and it is marked for removal
+ // then this function removes the entry from pending optimization table.
+ static void FunctionWasOptimized(Isolate* isolate,
+ Handle<JSFunction> function);
+
+ // This function returns whether a heuristic is allowed to trigger
+ // optimization the function. This mechanism is used in tests to prevent
+ // heuristics from interfering with manually triggered optimization.
+ static bool IsHeuristicOptimizationAllowed(Isolate* isolate,
+ JSFunction function);
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_PENDING_OPTIMIZATION_TABLE_H_
diff --git a/src/codegen/ppc/assembler-ppc-inl.h b/src/codegen/ppc/assembler-ppc-inl.h
new file mode 100644
index 0000000..f6a2f78
--- /dev/null
+++ b/src/codegen/ppc/assembler-ppc-inl.h
@@ -0,0 +1,517 @@
+// Copyright (c) 1994-2006 Sun Microsystems Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//
+// - Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// - Redistribution in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the
+// distribution.
+//
+// - Neither the name of Sun Microsystems or the names of contributors may
+// be used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+// COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+// OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// The original source code covered by the above license above has been modified
+// significantly by Google Inc.
+// Copyright 2014 the V8 project authors. All rights reserved.
+
+#ifndef V8_CODEGEN_PPC_ASSEMBLER_PPC_INL_H_
+#define V8_CODEGEN_PPC_ASSEMBLER_PPC_INL_H_
+
+#include "src/codegen/ppc/assembler-ppc.h"
+
+#include "src/codegen/assembler.h"
+#include "src/debug/debug.h"
+#include "src/objects/objects-inl.h"
+
+namespace v8 {
+namespace internal {
+
+bool CpuFeatures::SupportsOptimizer() { return true; }
+
+bool CpuFeatures::SupportsWasmSimd128() { return false; }
+
+void RelocInfo::apply(intptr_t delta) {
+ // absolute code pointer inside code object moves with the code object.
+ if (IsInternalReference(rmode_)) {
+ // Jump table entry
+ Address target = Memory<Address>(pc_);
+ Memory<Address>(pc_) = target + delta;
+ } else {
+ // mov sequence
+ DCHECK(IsInternalReferenceEncoded(rmode_));
+ Address target = Assembler::target_address_at(pc_, constant_pool_);
+ Assembler::set_target_address_at(pc_, constant_pool_, target + delta,
+ SKIP_ICACHE_FLUSH);
+ }
+}
+
+Address RelocInfo::target_internal_reference() {
+ if (IsInternalReference(rmode_)) {
+ // Jump table entry
+ return Memory<Address>(pc_);
+ } else {
+ // mov sequence
+ DCHECK(IsInternalReferenceEncoded(rmode_));
+ return Assembler::target_address_at(pc_, constant_pool_);
+ }
+}
+
+Address RelocInfo::target_internal_reference_address() {
+ DCHECK(IsInternalReference(rmode_) || IsInternalReferenceEncoded(rmode_));
+ return pc_;
+}
+
+Address RelocInfo::target_address() {
+ DCHECK(IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_) || IsWasmCall(rmode_));
+ return Assembler::target_address_at(pc_, constant_pool_);
+}
+
+Address RelocInfo::target_address_address() {
+ DCHECK(HasTargetAddressAddress());
+
+ if (FLAG_enable_embedded_constant_pool &&
+ Assembler::IsConstantPoolLoadStart(pc_)) {
+ // We return the PC for embedded constant pool since this function is used
+ // by the serializer and expects the address to reside within the code
+ // object.
+ return pc_;
+ }
+
+ // Read the address of the word containing the target_address in an
+ // instruction stream.
+ // The only architecture-independent user of this function is the serializer.
+ // The serializer uses it to find out how many raw bytes of instruction to
+ // output before the next target.
+ // For an instruction like LIS/ORI where the target bits are mixed into the
+ // instruction bits, the size of the target will be zero, indicating that the
+ // serializer should not step forward in memory after a target is resolved
+ // and written.
+ return pc_;
+}
+
+Address RelocInfo::constant_pool_entry_address() {
+ if (FLAG_enable_embedded_constant_pool) {
+ DCHECK(constant_pool_);
+ ConstantPoolEntry::Access access;
+ if (Assembler::IsConstantPoolLoadStart(pc_, &access))
+ return Assembler::target_constant_pool_address_at(
+ pc_, constant_pool_, access, ConstantPoolEntry::INTPTR);
+ }
+ UNREACHABLE();
+}
+
+void Assembler::set_target_compressed_address_at(
+ Address pc, Address constant_pool, Tagged_t target,
+ ICacheFlushMode icache_flush_mode) {
+ Assembler::set_target_address_at(
+ pc, constant_pool, static_cast<Address>(target), icache_flush_mode);
+}
+
+int RelocInfo::target_address_size() {
+ if (IsCodedSpecially()) {
+ return Assembler::kSpecialTargetSize;
+ } else {
+ return kSystemPointerSize;
+ }
+}
+
+Tagged_t Assembler::target_compressed_address_at(Address pc,
+ Address constant_pool) {
+ return static_cast<Tagged_t>(target_address_at(pc, constant_pool));
+}
+
+Handle<Object> Assembler::code_target_object_handle_at(Address pc,
+ Address constant_pool) {
+ int index =
+ static_cast<int>(target_address_at(pc, constant_pool)) & 0xFFFFFFFF;
+ return GetCodeTarget(index);
+}
+
+HeapObject RelocInfo::target_object() {
+ DCHECK(IsCodeTarget(rmode_) || IsEmbeddedObjectMode(rmode_));
+ if (IsCompressedEmbeddedObject(rmode_)) {
+ return HeapObject::cast(Object(DecompressTaggedAny(
+ host_.address(),
+ Assembler::target_compressed_address_at(pc_, constant_pool_))));
+ } else {
+ return HeapObject::cast(
+ Object(Assembler::target_address_at(pc_, constant_pool_)));
+ }
+}
+
+HeapObject RelocInfo::target_object_no_host(Isolate* isolate) {
+ if (IsCompressedEmbeddedObject(rmode_)) {
+ return HeapObject::cast(Object(DecompressTaggedAny(
+ isolate,
+ Assembler::target_compressed_address_at(pc_, constant_pool_))));
+ } else {
+ return target_object();
+ }
+}
+
+Handle<HeapObject> Assembler::compressed_embedded_object_handle_at(
+ Address pc, Address const_pool) {
+ return GetEmbeddedObject(target_compressed_address_at(pc, const_pool));
+}
+
+Handle<HeapObject> RelocInfo::target_object_handle(Assembler* origin) {
+ DCHECK(IsCodeTarget(rmode_) || IsEmbeddedObjectMode(rmode_));
+ if (IsCodeTarget(rmode_)) {
+ return Handle<HeapObject>::cast(
+ origin->code_target_object_handle_at(pc_, constant_pool_));
+ } else {
+ if (IsCompressedEmbeddedObject(rmode_)) {
+ return origin->compressed_embedded_object_handle_at(pc_, constant_pool_);
+ }
+ return Handle<HeapObject>(reinterpret_cast<Address*>(
+ Assembler::target_address_at(pc_, constant_pool_)));
+ }
+}
+
+void RelocInfo::set_target_object(Heap* heap, HeapObject target,
+ WriteBarrierMode write_barrier_mode,
+ ICacheFlushMode icache_flush_mode) {
+ DCHECK(IsCodeTarget(rmode_) || IsEmbeddedObjectMode(rmode_));
+ if (IsCompressedEmbeddedObject(rmode_)) {
+ Assembler::set_target_compressed_address_at(
+ pc_, constant_pool_, CompressTagged(target.ptr()), icache_flush_mode);
+ } else {
+ DCHECK(IsFullEmbeddedObject(rmode_));
+ Assembler::set_target_address_at(pc_, constant_pool_, target.ptr(),
+ icache_flush_mode);
+ }
+ if (write_barrier_mode == UPDATE_WRITE_BARRIER && !host().is_null() &&
+ !FLAG_disable_write_barriers) {
+ WriteBarrierForCode(host(), this, target);
+ }
+}
+
+Address RelocInfo::target_external_reference() {
+ DCHECK(rmode_ == EXTERNAL_REFERENCE);
+ return Assembler::target_address_at(pc_, constant_pool_);
+}
+
+void RelocInfo::set_target_external_reference(
+ Address target, ICacheFlushMode icache_flush_mode) {
+ DCHECK(rmode_ == RelocInfo::EXTERNAL_REFERENCE);
+ Assembler::set_target_address_at(pc_, constant_pool_, target,
+ icache_flush_mode);
+}
+
+Address RelocInfo::target_runtime_entry(Assembler* origin) {
+ DCHECK(IsRuntimeEntry(rmode_));
+ return target_address();
+}
+
+void RelocInfo::set_target_runtime_entry(Address target,
+ WriteBarrierMode write_barrier_mode,
+ ICacheFlushMode icache_flush_mode) {
+ DCHECK(IsRuntimeEntry(rmode_));
+ if (target_address() != target)
+ set_target_address(target, write_barrier_mode, icache_flush_mode);
+}
+
+Address RelocInfo::target_off_heap_target() {
+ DCHECK(IsOffHeapTarget(rmode_));
+ return Assembler::target_address_at(pc_, constant_pool_);
+}
+
+void RelocInfo::WipeOut() {
+ DCHECK(IsEmbeddedObjectMode(rmode_) || IsCodeTarget(rmode_) ||
+ IsRuntimeEntry(rmode_) || IsExternalReference(rmode_) ||
+ IsInternalReference(rmode_) || IsInternalReferenceEncoded(rmode_) ||
+ IsOffHeapTarget(rmode_));
+ if (IsInternalReference(rmode_)) {
+ // Jump table entry
+ Memory<Address>(pc_) = kNullAddress;
+ } else if (IsCompressedEmbeddedObject(rmode_)) {
+ Assembler::set_target_compressed_address_at(pc_, constant_pool_,
+ kNullAddress);
+ } else if (IsInternalReferenceEncoded(rmode_) || IsOffHeapTarget(rmode_)) {
+ // mov sequence
+ // Currently used only by deserializer, no need to flush.
+ Assembler::set_target_address_at(pc_, constant_pool_, kNullAddress,
+ SKIP_ICACHE_FLUSH);
+ } else {
+ Assembler::set_target_address_at(pc_, constant_pool_, kNullAddress);
+ }
+}
+
+Operand::Operand(Register rm) : rm_(rm), rmode_(RelocInfo::NONE) {}
+
+void Assembler::UntrackBranch() {
+ DCHECK(!trampoline_emitted_);
+ DCHECK_GT(tracked_branch_count_, 0);
+ int count = --tracked_branch_count_;
+ if (count == 0) {
+ // Reset
+ next_trampoline_check_ = kMaxInt;
+ } else {
+ next_trampoline_check_ += kTrampolineSlotsSize;
+ }
+}
+
+// Fetch the 32bit value from the FIXED_SEQUENCE lis/ori
+Address Assembler::target_address_at(Address pc, Address constant_pool) {
+ if (FLAG_enable_embedded_constant_pool && constant_pool) {
+ ConstantPoolEntry::Access access;
+ if (IsConstantPoolLoadStart(pc, &access))
+ return Memory<Address>(target_constant_pool_address_at(
+ pc, constant_pool, access, ConstantPoolEntry::INTPTR));
+ }
+
+ Instr instr1 = instr_at(pc);
+ Instr instr2 = instr_at(pc + kInstrSize);
+ // Interpret 2 instructions generated by lis/ori
+ if (IsLis(instr1) && IsOri(instr2)) {
+#if V8_TARGET_ARCH_PPC64
+ Instr instr4 = instr_at(pc + (3 * kInstrSize));
+ Instr instr5 = instr_at(pc + (4 * kInstrSize));
+ // Assemble the 64 bit value.
+ uint64_t hi = (static_cast<uint32_t>((instr1 & kImm16Mask) << 16) |
+ static_cast<uint32_t>(instr2 & kImm16Mask));
+ uint64_t lo = (static_cast<uint32_t>((instr4 & kImm16Mask) << 16) |
+ static_cast<uint32_t>(instr5 & kImm16Mask));
+ return static_cast<Address>((hi << 32) | lo);
+#else
+ // Assemble the 32 bit value.
+ return static_cast<Address>(((instr1 & kImm16Mask) << 16) |
+ (instr2 & kImm16Mask));
+#endif
+ }
+
+ UNREACHABLE();
+}
+
+#if V8_TARGET_ARCH_PPC64
+const uint32_t kLoadIntptrOpcode = LD;
+#else
+const uint32_t kLoadIntptrOpcode = LWZ;
+#endif
+
+// Constant pool load sequence detection:
+// 1) REGULAR access:
+// load <dst>, kConstantPoolRegister + <offset>
+//
+// 2) OVERFLOWED access:
+// addis <scratch>, kConstantPoolRegister, <offset_high>
+// load <dst>, <scratch> + <offset_low>
+bool Assembler::IsConstantPoolLoadStart(Address pc,
+ ConstantPoolEntry::Access* access) {
+ Instr instr = instr_at(pc);
+ uint32_t opcode = instr & kOpcodeMask;
+ if (GetRA(instr) != kConstantPoolRegister) return false;
+ bool overflowed = (opcode == ADDIS);
+#ifdef DEBUG
+ if (overflowed) {
+ opcode = instr_at(pc + kInstrSize) & kOpcodeMask;
+ }
+ DCHECK(opcode == kLoadIntptrOpcode || opcode == LFD);
+#endif
+ if (access) {
+ *access = (overflowed ? ConstantPoolEntry::OVERFLOWED
+ : ConstantPoolEntry::REGULAR);
+ }
+ return true;
+}
+
+bool Assembler::IsConstantPoolLoadEnd(Address pc,
+ ConstantPoolEntry::Access* access) {
+ Instr instr = instr_at(pc);
+ uint32_t opcode = instr & kOpcodeMask;
+ bool overflowed = false;
+ if (!(opcode == kLoadIntptrOpcode || opcode == LFD)) return false;
+ if (GetRA(instr) != kConstantPoolRegister) {
+ instr = instr_at(pc - kInstrSize);
+ opcode = instr & kOpcodeMask;
+ if ((opcode != ADDIS) || GetRA(instr) != kConstantPoolRegister) {
+ return false;
+ }
+ overflowed = true;
+ }
+ if (access) {
+ *access = (overflowed ? ConstantPoolEntry::OVERFLOWED
+ : ConstantPoolEntry::REGULAR);
+ }
+ return true;
+}
+
+int Assembler::GetConstantPoolOffset(Address pc,
+ ConstantPoolEntry::Access access,
+ ConstantPoolEntry::Type type) {
+ bool overflowed = (access == ConstantPoolEntry::OVERFLOWED);
+#ifdef DEBUG
+ ConstantPoolEntry::Access access_check =
+ static_cast<ConstantPoolEntry::Access>(-1);
+ DCHECK(IsConstantPoolLoadStart(pc, &access_check));
+ DCHECK(access_check == access);
+#endif
+ int offset;
+ if (overflowed) {
+ offset = (instr_at(pc) & kImm16Mask) << 16;
+ offset += SIGN_EXT_IMM16(instr_at(pc + kInstrSize) & kImm16Mask);
+ DCHECK(!is_int16(offset));
+ } else {
+ offset = SIGN_EXT_IMM16((instr_at(pc) & kImm16Mask));
+ }
+ return offset;
+}
+
+void Assembler::PatchConstantPoolAccessInstruction(
+ int pc_offset, int offset, ConstantPoolEntry::Access access,
+ ConstantPoolEntry::Type type) {
+ Address pc = reinterpret_cast<Address>(buffer_start_) + pc_offset;
+ bool overflowed = (access == ConstantPoolEntry::OVERFLOWED);
+ CHECK(overflowed != is_int16(offset));
+#ifdef DEBUG
+ ConstantPoolEntry::Access access_check =
+ static_cast<ConstantPoolEntry::Access>(-1);
+ DCHECK(IsConstantPoolLoadStart(pc, &access_check));
+ DCHECK(access_check == access);
+#endif
+ if (overflowed) {
+ int hi_word = static_cast<int>(offset >> 16);
+ int lo_word = static_cast<int>(offset & 0xffff);
+ if (lo_word & 0x8000) hi_word++;
+
+ Instr instr1 = instr_at(pc);
+ Instr instr2 = instr_at(pc + kInstrSize);
+ instr1 &= ~kImm16Mask;
+ instr1 |= (hi_word & kImm16Mask);
+ instr2 &= ~kImm16Mask;
+ instr2 |= (lo_word & kImm16Mask);
+ instr_at_put(pc, instr1);
+ instr_at_put(pc + kInstrSize, instr2);
+ } else {
+ Instr instr = instr_at(pc);
+ instr &= ~kImm16Mask;
+ instr |= (offset & kImm16Mask);
+ instr_at_put(pc, instr);
+ }
+}
+
+Address Assembler::target_constant_pool_address_at(
+ Address pc, Address constant_pool, ConstantPoolEntry::Access access,
+ ConstantPoolEntry::Type type) {
+ Address addr = constant_pool;
+ DCHECK(addr);
+ addr += GetConstantPoolOffset(pc, access, type);
+ return addr;
+}
+
+// This sets the branch destination (which gets loaded at the call address).
+// This is for calls and branches within generated code. The serializer
+// has already deserialized the mov instructions etc.
+// There is a FIXED_SEQUENCE assumption here
+void Assembler::deserialization_set_special_target_at(
+ Address instruction_payload, Code code, Address target) {
+ set_target_address_at(instruction_payload,
+ !code.is_null() ? code.constant_pool() : kNullAddress,
+ target);
+}
+
+int Assembler::deserialization_special_target_size(
+ Address instruction_payload) {
+ return kSpecialTargetSize;
+}
+
+void Assembler::deserialization_set_target_internal_reference_at(
+ Address pc, Address target, RelocInfo::Mode mode) {
+ if (RelocInfo::IsInternalReferenceEncoded(mode)) {
+ set_target_address_at(pc, kNullAddress, target, SKIP_ICACHE_FLUSH);
+ } else {
+ Memory<Address>(pc) = target;
+ }
+}
+
+// This code assumes the FIXED_SEQUENCE of lis/ori
+void Assembler::set_target_address_at(Address pc, Address constant_pool,
+ Address target,
+ ICacheFlushMode icache_flush_mode) {
+ if (FLAG_enable_embedded_constant_pool && constant_pool) {
+ ConstantPoolEntry::Access access;
+ if (IsConstantPoolLoadStart(pc, &access)) {
+ Memory<Address>(target_constant_pool_address_at(
+ pc, constant_pool, access, ConstantPoolEntry::INTPTR)) = target;
+ return;
+ }
+ }
+
+ Instr instr1 = instr_at(pc);
+ Instr instr2 = instr_at(pc + kInstrSize);
+ // Interpret 2 instructions generated by lis/ori
+ if (IsLis(instr1) && IsOri(instr2)) {
+#if V8_TARGET_ARCH_PPC64
+ Instr instr4 = instr_at(pc + (3 * kInstrSize));
+ Instr instr5 = instr_at(pc + (4 * kInstrSize));
+ // Needs to be fixed up when mov changes to handle 64-bit values.
+ uint32_t* p = reinterpret_cast<uint32_t*>(pc);
+ uintptr_t itarget = static_cast<uintptr_t>(target);
+
+ instr5 &= ~kImm16Mask;
+ instr5 |= itarget & kImm16Mask;
+ itarget = itarget >> 16;
+
+ instr4 &= ~kImm16Mask;
+ instr4 |= itarget & kImm16Mask;
+ itarget = itarget >> 16;
+
+ instr2 &= ~kImm16Mask;
+ instr2 |= itarget & kImm16Mask;
+ itarget = itarget >> 16;
+
+ instr1 &= ~kImm16Mask;
+ instr1 |= itarget & kImm16Mask;
+ itarget = itarget >> 16;
+
+ *p = instr1;
+ *(p + 1) = instr2;
+ *(p + 3) = instr4;
+ *(p + 4) = instr5;
+ if (icache_flush_mode != SKIP_ICACHE_FLUSH) {
+ FlushInstructionCache(p, 5 * kInstrSize);
+ }
+#else
+ uint32_t* p = reinterpret_cast<uint32_t*>(pc);
+ uint32_t itarget = static_cast<uint32_t>(target);
+ int lo_word = itarget & kImm16Mask;
+ int hi_word = itarget >> 16;
+ instr1 &= ~kImm16Mask;
+ instr1 |= hi_word;
+ instr2 &= ~kImm16Mask;
+ instr2 |= lo_word;
+
+ *p = instr1;
+ *(p + 1) = instr2;
+ if (icache_flush_mode != SKIP_ICACHE_FLUSH) {
+ FlushInstructionCache(p, 2 * kInstrSize);
+ }
+#endif
+ return;
+ }
+ UNREACHABLE();
+}
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_PPC_ASSEMBLER_PPC_INL_H_
diff --git a/src/codegen/ppc/assembler-ppc.cc b/src/codegen/ppc/assembler-ppc.cc
new file mode 100644
index 0000000..54136a9
--- /dev/null
+++ b/src/codegen/ppc/assembler-ppc.cc
@@ -0,0 +1,2015 @@
+// Copyright (c) 1994-2006 Sun Microsystems Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//
+// - Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// - Redistribution in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the
+// distribution.
+//
+// - Neither the name of Sun Microsystems or the names of contributors may
+// be used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+// COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+// OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// The original source code covered by the above license above has been
+// modified significantly by Google Inc.
+// Copyright 2014 the V8 project authors. All rights reserved.
+
+#include "src/codegen/ppc/assembler-ppc.h"
+
+#if V8_TARGET_ARCH_PPC || V8_TARGET_ARCH_PPC64
+
+#include "src/base/bits.h"
+#include "src/base/cpu.h"
+#include "src/codegen/macro-assembler.h"
+#include "src/codegen/ppc/assembler-ppc-inl.h"
+#include "src/codegen/string-constants.h"
+#include "src/deoptimizer/deoptimizer.h"
+
+namespace v8 {
+namespace internal {
+
+// Get the CPU features enabled by the build.
+static unsigned CpuFeaturesImpliedByCompiler() {
+ unsigned answer = 0;
+ return answer;
+}
+
+void CpuFeatures::ProbeImpl(bool cross_compile) {
+ supported_ |= CpuFeaturesImpliedByCompiler();
+ icache_line_size_ = 128;
+
+ // Only use statically determined features for cross compile (snapshot).
+ if (cross_compile) return;
+
+// Detect whether frim instruction is supported (POWER5+)
+// For now we will just check for processors we know do not
+// support it
+#ifndef USE_SIMULATOR
+ // Probe for additional features at runtime.
+ base::CPU cpu;
+ if (cpu.part() == base::CPU::PPC_POWER9) {
+ supported_ |= (1u << MODULO);
+ }
+#if V8_TARGET_ARCH_PPC64
+ if (cpu.part() == base::CPU::PPC_POWER8) {
+ supported_ |= (1u << FPR_GPR_MOV);
+ }
+#endif
+ if (cpu.part() == base::CPU::PPC_POWER6 ||
+ cpu.part() == base::CPU::PPC_POWER7 ||
+ cpu.part() == base::CPU::PPC_POWER8) {
+ supported_ |= (1u << LWSYNC);
+ }
+ if (cpu.part() == base::CPU::PPC_POWER7 ||
+ cpu.part() == base::CPU::PPC_POWER8) {
+ supported_ |= (1u << ISELECT);
+ supported_ |= (1u << VSX);
+ }
+#if V8_OS_LINUX
+ if (!(cpu.part() == base::CPU::PPC_G5 || cpu.part() == base::CPU::PPC_G4)) {
+ // Assume support
+ supported_ |= (1u << FPU);
+ }
+ if (cpu.icache_line_size() != base::CPU::UNKNOWN_CACHE_LINE_SIZE) {
+ icache_line_size_ = cpu.icache_line_size();
+ }
+#elif V8_OS_AIX
+ // Assume support FP support and default cache line size
+ supported_ |= (1u << FPU);
+#endif
+#else // Simulator
+ supported_ |= (1u << FPU);
+ supported_ |= (1u << LWSYNC);
+ supported_ |= (1u << ISELECT);
+ supported_ |= (1u << VSX);
+ supported_ |= (1u << MODULO);
+#if V8_TARGET_ARCH_PPC64
+ supported_ |= (1u << FPR_GPR_MOV);
+#endif
+#endif
+}
+
+void CpuFeatures::PrintTarget() {
+ const char* ppc_arch = nullptr;
+
+#if V8_TARGET_ARCH_PPC64
+ ppc_arch = "ppc64";
+#else
+ ppc_arch = "ppc";
+#endif
+
+ printf("target %s\n", ppc_arch);
+}
+
+void CpuFeatures::PrintFeatures() {
+ printf("FPU=%d\n", CpuFeatures::IsSupported(FPU));
+ printf("FPR_GPR_MOV=%d\n", CpuFeatures::IsSupported(FPR_GPR_MOV));
+ printf("LWSYNC=%d\n", CpuFeatures::IsSupported(LWSYNC));
+ printf("ISELECT=%d\n", CpuFeatures::IsSupported(ISELECT));
+ printf("VSX=%d\n", CpuFeatures::IsSupported(VSX));
+ printf("MODULO=%d\n", CpuFeatures::IsSupported(MODULO));
+}
+
+Register ToRegister(int num) {
+ DCHECK(num >= 0 && num < kNumRegisters);
+ const Register kRegisters[] = {r0, sp, r2, r3, r4, r5, r6, r7,
+ r8, r9, r10, r11, ip, r13, r14, r15,
+ r16, r17, r18, r19, r20, r21, r22, r23,
+ r24, r25, r26, r27, r28, r29, r30, fp};
+ return kRegisters[num];
+}
+
+// -----------------------------------------------------------------------------
+// Implementation of RelocInfo
+
+const int RelocInfo::kApplyMask =
+ RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE) |
+ RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE_ENCODED);
+
+bool RelocInfo::IsCodedSpecially() {
+ // The deserializer needs to know whether a pointer is specially
+ // coded. Being specially coded on PPC means that it is a lis/ori
+ // instruction sequence or is a constant pool entry, and these are
+ // always the case inside code objects.
+ return true;
+}
+
+bool RelocInfo::IsInConstantPool() {
+ if (FLAG_enable_embedded_constant_pool && constant_pool_ != kNullAddress) {
+ return Assembler::IsConstantPoolLoadStart(pc_);
+ }
+ return false;
+}
+
+uint32_t RelocInfo::wasm_call_tag() const {
+ DCHECK(rmode_ == WASM_CALL || rmode_ == WASM_STUB_CALL);
+ return static_cast<uint32_t>(
+ Assembler::target_address_at(pc_, constant_pool_));
+}
+
+// -----------------------------------------------------------------------------
+// Implementation of Operand and MemOperand
+// See assembler-ppc-inl.h for inlined constructors
+
+Operand::Operand(Handle<HeapObject> handle) {
+ rm_ = no_reg;
+ value_.immediate = static_cast<intptr_t>(handle.address());
+ rmode_ = RelocInfo::FULL_EMBEDDED_OBJECT;
+}
+
+Operand Operand::EmbeddedNumber(double value) {
+ int32_t smi;
+ if (DoubleToSmiInteger(value, &smi)) return Operand(Smi::FromInt(smi));
+ Operand result(0, RelocInfo::FULL_EMBEDDED_OBJECT);
+ result.is_heap_object_request_ = true;
+ result.value_.heap_object_request = HeapObjectRequest(value);
+ return result;
+}
+
+Operand Operand::EmbeddedStringConstant(const StringConstantBase* str) {
+ Operand result(0, RelocInfo::FULL_EMBEDDED_OBJECT);
+ result.is_heap_object_request_ = true;
+ result.value_.heap_object_request = HeapObjectRequest(str);
+ return result;
+}
+
+MemOperand::MemOperand(Register rn, int32_t offset)
+ : ra_(rn), offset_(offset), rb_(no_reg) {}
+
+MemOperand::MemOperand(Register ra, Register rb)
+ : ra_(ra), offset_(0), rb_(rb) {}
+
+void Assembler::AllocateAndInstallRequestedHeapObjects(Isolate* isolate) {
+ DCHECK_IMPLIES(isolate == nullptr, heap_object_requests_.empty());
+ for (auto& request : heap_object_requests_) {
+ Handle<HeapObject> object;
+ switch (request.kind()) {
+ case HeapObjectRequest::kHeapNumber: {
+ object = isolate->factory()->NewHeapNumber<AllocationType::kOld>(
+ request.heap_number());
+ break;
+ }
+ case HeapObjectRequest::kStringConstant: {
+ const StringConstantBase* str = request.string();
+ CHECK_NOT_NULL(str);
+ object = str->AllocateStringConstant(isolate);
+ break;
+ }
+ }
+ Address pc = reinterpret_cast<Address>(buffer_start_) + request.offset();
+ Address constant_pool = kNullAddress;
+ set_target_address_at(pc, constant_pool, object.address(),
+ SKIP_ICACHE_FLUSH);
+ }
+}
+
+// -----------------------------------------------------------------------------
+// Specific instructions, constants, and masks.
+
+Assembler::Assembler(const AssemblerOptions& options,
+ std::unique_ptr<AssemblerBuffer> buffer)
+ : AssemblerBase(options, std::move(buffer)),
+ scratch_register_list_(ip.bit()),
+ constant_pool_builder_(kLoadPtrMaxReachBits, kLoadDoubleMaxReachBits) {
+ reloc_info_writer.Reposition(buffer_start_ + buffer_->size(), pc_);
+
+ no_trampoline_pool_before_ = 0;
+ trampoline_pool_blocked_nesting_ = 0;
+ constant_pool_entry_sharing_blocked_nesting_ = 0;
+ next_trampoline_check_ = kMaxInt;
+ internal_trampoline_exception_ = false;
+ last_bound_pos_ = 0;
+ optimizable_cmpi_pos_ = -1;
+ trampoline_emitted_ = FLAG_force_long_branches;
+ tracked_branch_count_ = 0;
+ relocations_.reserve(128);
+}
+
+void Assembler::GetCode(Isolate* isolate, CodeDesc* desc,
+ SafepointTableBuilder* safepoint_table_builder,
+ int handler_table_offset) {
+ // As a crutch to avoid having to add manual Align calls wherever we use a
+ // raw workflow to create Code objects (mostly in tests), add another Align
+ // call here. It does no harm - the end of the Code object is aligned to the
+ // (larger) kCodeAlignment anyways.
+ // TODO(jgruber): Consider moving responsibility for proper alignment to
+ // metadata table builders (safepoint, handler, constant pool, code
+ // comments).
+ DataAlign(Code::kMetadataAlignment);
+
+ // Emit constant pool if necessary.
+ int constant_pool_size = EmitConstantPool();
+
+ EmitRelocations();
+
+ int code_comments_size = WriteCodeComments();
+
+ AllocateAndInstallRequestedHeapObjects(isolate);
+
+ // Set up code descriptor.
+ // TODO(jgruber): Reconsider how these offsets and sizes are maintained up to
+ // this point to make CodeDesc initialization less fiddly.
+
+ const int instruction_size = pc_offset();
+ const int code_comments_offset = instruction_size - code_comments_size;
+ const int constant_pool_offset = code_comments_offset - constant_pool_size;
+ const int handler_table_offset2 = (handler_table_offset == kNoHandlerTable)
+ ? constant_pool_offset
+ : handler_table_offset;
+ const int safepoint_table_offset =
+ (safepoint_table_builder == kNoSafepointTable)
+ ? handler_table_offset2
+ : safepoint_table_builder->GetCodeOffset();
+ const int reloc_info_offset =
+ static_cast<int>(reloc_info_writer.pos() - buffer_->start());
+ CodeDesc::Initialize(desc, this, safepoint_table_offset,
+ handler_table_offset2, constant_pool_offset,
+ code_comments_offset, reloc_info_offset);
+}
+
+void Assembler::Align(int m) {
+ DCHECK(m >= 4 && base::bits::IsPowerOfTwo(m));
+ DCHECK_EQ(pc_offset() & (kInstrSize - 1), 0);
+ while ((pc_offset() & (m - 1)) != 0) {
+ nop();
+ }
+}
+
+void Assembler::CodeTargetAlign() { Align(8); }
+
+Condition Assembler::GetCondition(Instr instr) {
+ switch (instr & kCondMask) {
+ case BT:
+ return eq;
+ case BF:
+ return ne;
+ default:
+ UNIMPLEMENTED();
+ }
+ return al;
+}
+
+bool Assembler::IsLis(Instr instr) {
+ return ((instr & kOpcodeMask) == ADDIS) && GetRA(instr) == r0;
+}
+
+bool Assembler::IsLi(Instr instr) {
+ return ((instr & kOpcodeMask) == ADDI) && GetRA(instr) == r0;
+}
+
+bool Assembler::IsAddic(Instr instr) { return (instr & kOpcodeMask) == ADDIC; }
+
+bool Assembler::IsOri(Instr instr) { return (instr & kOpcodeMask) == ORI; }
+
+bool Assembler::IsBranch(Instr instr) { return ((instr & kOpcodeMask) == BCX); }
+
+Register Assembler::GetRA(Instr instr) {
+ return Register::from_code(Instruction::RAValue(instr));
+}
+
+Register Assembler::GetRB(Instr instr) {
+ return Register::from_code(Instruction::RBValue(instr));
+}
+
+#if V8_TARGET_ARCH_PPC64
+// This code assumes a FIXED_SEQUENCE for 64bit loads (lis/ori)
+bool Assembler::Is64BitLoadIntoR12(Instr instr1, Instr instr2, Instr instr3,
+ Instr instr4, Instr instr5) {
+ // Check the instructions are indeed a five part load (into r12)
+ // 3d800000 lis r12, 0
+ // 618c0000 ori r12, r12, 0
+ // 798c07c6 rldicr r12, r12, 32, 31
+ // 658c00c3 oris r12, r12, 195
+ // 618ccd40 ori r12, r12, 52544
+ return (((instr1 >> 16) == 0x3D80) && ((instr2 >> 16) == 0x618C) &&
+ (instr3 == 0x798C07C6) && ((instr4 >> 16) == 0x658C) &&
+ ((instr5 >> 16) == 0x618C));
+}
+#else
+// This code assumes a FIXED_SEQUENCE for 32bit loads (lis/ori)
+bool Assembler::Is32BitLoadIntoR12(Instr instr1, Instr instr2) {
+ // Check the instruction is indeed a two part load (into r12)
+ // 3d802553 lis r12, 9555
+ // 618c5000 ori r12, r12, 20480
+ return (((instr1 >> 16) == 0x3D80) && ((instr2 >> 16) == 0x618C));
+}
+#endif
+
+bool Assembler::IsCmpRegister(Instr instr) {
+ return (((instr & kOpcodeMask) == EXT2) &&
+ ((EXT2 | (instr & kExt2OpcodeMask)) == CMP));
+}
+
+bool Assembler::IsRlwinm(Instr instr) {
+ return ((instr & kOpcodeMask) == RLWINMX);
+}
+
+bool Assembler::IsAndi(Instr instr) { return ((instr & kOpcodeMask) == ANDIx); }
+
+#if V8_TARGET_ARCH_PPC64
+bool Assembler::IsRldicl(Instr instr) {
+ return (((instr & kOpcodeMask) == EXT5) &&
+ ((EXT5 | (instr & kExt5OpcodeMask)) == RLDICL));
+}
+#endif
+
+bool Assembler::IsCmpImmediate(Instr instr) {
+ return ((instr & kOpcodeMask) == CMPI);
+}
+
+bool Assembler::IsCrSet(Instr instr) {
+ return (((instr & kOpcodeMask) == EXT1) &&
+ ((EXT1 | (instr & kExt1OpcodeMask)) == CREQV));
+}
+
+Register Assembler::GetCmpImmediateRegister(Instr instr) {
+ DCHECK(IsCmpImmediate(instr));
+ return GetRA(instr);
+}
+
+int Assembler::GetCmpImmediateRawImmediate(Instr instr) {
+ DCHECK(IsCmpImmediate(instr));
+ return instr & kOff16Mask;
+}
+
+// Labels refer to positions in the (to be) generated code.
+// There are bound, linked, and unused labels.
+//
+// Bound labels refer to known positions in the already
+// generated code. pos() is the position the label refers to.
+//
+// Linked labels refer to unknown positions in the code
+// to be generated; pos() is the position of the last
+// instruction using the label.
+
+// The link chain is terminated by a negative code position (must be aligned)
+const int kEndOfChain = -4;
+
+// Dummy opcodes for unbound label mov instructions or jump table entries.
+enum {
+ kUnboundMovLabelOffsetOpcode = 0 << 26,
+ kUnboundAddLabelOffsetOpcode = 1 << 26,
+ kUnboundAddLabelLongOffsetOpcode = 2 << 26,
+ kUnboundMovLabelAddrOpcode = 3 << 26,
+ kUnboundJumpTableEntryOpcode = 4 << 26
+};
+
+int Assembler::target_at(int pos) {
+ Instr instr = instr_at(pos);
+ // check which type of branch this is 16 or 26 bit offset
+ uint32_t opcode = instr & kOpcodeMask;
+ int link;
+ switch (opcode) {
+ case BX:
+ link = SIGN_EXT_IMM26(instr & kImm26Mask);
+ link &= ~(kAAMask | kLKMask); // discard AA|LK bits if present
+ break;
+ case BCX:
+ link = SIGN_EXT_IMM16((instr & kImm16Mask));
+ link &= ~(kAAMask | kLKMask); // discard AA|LK bits if present
+ break;
+ case kUnboundMovLabelOffsetOpcode:
+ case kUnboundAddLabelOffsetOpcode:
+ case kUnboundAddLabelLongOffsetOpcode:
+ case kUnboundMovLabelAddrOpcode:
+ case kUnboundJumpTableEntryOpcode:
+ link = SIGN_EXT_IMM26(instr & kImm26Mask);
+ link <<= 2;
+ break;
+ default:
+ DCHECK(false);
+ return -1;
+ }
+
+ if (link == 0) return kEndOfChain;
+ return pos + link;
+}
+
+void Assembler::target_at_put(int pos, int target_pos, bool* is_branch) {
+ Instr instr = instr_at(pos);
+ uint32_t opcode = instr & kOpcodeMask;
+
+ if (is_branch != nullptr) {
+ *is_branch = (opcode == BX || opcode == BCX);
+ }
+
+ switch (opcode) {
+ case BX: {
+ int imm26 = target_pos - pos;
+ CHECK(is_int26(imm26) && (imm26 & (kAAMask | kLKMask)) == 0);
+ if (imm26 == kInstrSize && !(instr & kLKMask)) {
+ // Branch to next instr without link.
+ instr = ORI; // nop: ori, 0,0,0
+ } else {
+ instr &= ((~kImm26Mask) | kAAMask | kLKMask);
+ instr |= (imm26 & kImm26Mask);
+ }
+ instr_at_put(pos, instr);
+ break;
+ }
+ case BCX: {
+ int imm16 = target_pos - pos;
+ CHECK(is_int16(imm16) && (imm16 & (kAAMask | kLKMask)) == 0);
+ if (imm16 == kInstrSize && !(instr & kLKMask)) {
+ // Branch to next instr without link.
+ instr = ORI; // nop: ori, 0,0,0
+ } else {
+ instr &= ((~kImm16Mask) | kAAMask | kLKMask);
+ instr |= (imm16 & kImm16Mask);
+ }
+ instr_at_put(pos, instr);
+ break;
+ }
+ case kUnboundMovLabelOffsetOpcode: {
+ // Load the position of the label relative to the generated code object
+ // pointer in a register.
+ Register dst = Register::from_code(instr_at(pos + kInstrSize));
+ int32_t offset = target_pos + (Code::kHeaderSize - kHeapObjectTag);
+ PatchingAssembler patcher(
+ options(), reinterpret_cast<byte*>(buffer_start_ + pos), 2);
+ patcher.bitwise_mov32(dst, offset);
+ break;
+ }
+ case kUnboundAddLabelLongOffsetOpcode:
+ case kUnboundAddLabelOffsetOpcode: {
+ // dst = base + position + immediate
+ Instr operands = instr_at(pos + kInstrSize);
+ Register dst = Register::from_code((operands >> 27) & 0x1F);
+ Register base = Register::from_code((operands >> 22) & 0x1F);
+ int32_t delta = (opcode == kUnboundAddLabelLongOffsetOpcode)
+ ? static_cast<int32_t>(instr_at(pos + 2 * kInstrSize))
+ : (SIGN_EXT_IMM22(operands & kImm22Mask));
+ int32_t offset = target_pos + delta;
+ PatchingAssembler patcher(
+ options(), reinterpret_cast<byte*>(buffer_start_ + pos),
+ 2 + static_cast<int32_t>(opcode == kUnboundAddLabelLongOffsetOpcode));
+ patcher.bitwise_add32(dst, base, offset);
+ if (opcode == kUnboundAddLabelLongOffsetOpcode) patcher.nop();
+ break;
+ }
+ case kUnboundMovLabelAddrOpcode: {
+ // Load the address of the label in a register.
+ Register dst = Register::from_code(instr_at(pos + kInstrSize));
+ PatchingAssembler patcher(options(),
+ reinterpret_cast<byte*>(buffer_start_ + pos),
+ kMovInstructionsNoConstantPool);
+ // Keep internal references relative until EmitRelocations.
+ patcher.bitwise_mov(dst, target_pos);
+ break;
+ }
+ case kUnboundJumpTableEntryOpcode: {
+ PatchingAssembler patcher(options(),
+ reinterpret_cast<byte*>(buffer_start_ + pos),
+ kSystemPointerSize / kInstrSize);
+ // Keep internal references relative until EmitRelocations.
+ patcher.dp(target_pos);
+ break;
+ }
+ default:
+ DCHECK(false);
+ break;
+ }
+}
+
+int Assembler::max_reach_from(int pos) {
+ Instr instr = instr_at(pos);
+ uint32_t opcode = instr & kOpcodeMask;
+
+ // check which type of branch this is 16 or 26 bit offset
+ switch (opcode) {
+ case BX:
+ return 26;
+ case BCX:
+ return 16;
+ case kUnboundMovLabelOffsetOpcode:
+ case kUnboundAddLabelOffsetOpcode:
+ case kUnboundMovLabelAddrOpcode:
+ case kUnboundJumpTableEntryOpcode:
+ return 0; // no limit on reach
+ }
+
+ DCHECK(false);
+ return 0;
+}
+
+void Assembler::bind_to(Label* L, int pos) {
+ DCHECK(0 <= pos && pos <= pc_offset()); // must have a valid binding position
+ int32_t trampoline_pos = kInvalidSlotPos;
+ bool is_branch = false;
+ while (L->is_linked()) {
+ int fixup_pos = L->pos();
+ int32_t offset = pos - fixup_pos;
+ int maxReach = max_reach_from(fixup_pos);
+ next(L); // call next before overwriting link with target at fixup_pos
+ if (maxReach && is_intn(offset, maxReach) == false) {
+ if (trampoline_pos == kInvalidSlotPos) {
+ trampoline_pos = get_trampoline_entry();
+ CHECK_NE(trampoline_pos, kInvalidSlotPos);
+ target_at_put(trampoline_pos, pos);
+ }
+ target_at_put(fixup_pos, trampoline_pos);
+ } else {
+ target_at_put(fixup_pos, pos, &is_branch);
+ }
+ }
+ L->bind_to(pos);
+
+ if (!trampoline_emitted_ && is_branch) {
+ UntrackBranch();
+ }
+
+ // Keep track of the last bound label so we don't eliminate any instructions
+ // before a bound label.
+ if (pos > last_bound_pos_) last_bound_pos_ = pos;
+}
+
+void Assembler::bind(Label* L) {
+ DCHECK(!L->is_bound()); // label can only be bound once
+ bind_to(L, pc_offset());
+}
+
+void Assembler::next(Label* L) {
+ DCHECK(L->is_linked());
+ int link = target_at(L->pos());
+ if (link == kEndOfChain) {
+ L->Unuse();
+ } else {
+ DCHECK_GE(link, 0);
+ L->link_to(link);
+ }
+}
+
+bool Assembler::is_near(Label* L, Condition cond) {
+ DCHECK(L->is_bound());
+ if (L->is_bound() == false) return false;
+
+ int maxReach = ((cond == al) ? 26 : 16);
+ int offset = L->pos() - pc_offset();
+
+ return is_intn(offset, maxReach);
+}
+
+void Assembler::a_form(Instr instr, DoubleRegister frt, DoubleRegister fra,
+ DoubleRegister frb, RCBit r) {
+ emit(instr | frt.code() * B21 | fra.code() * B16 | frb.code() * B11 | r);
+}
+
+void Assembler::d_form(Instr instr, Register rt, Register ra,
+ const intptr_t val, bool signed_disp) {
+ if (signed_disp) {
+ if (!is_int16(val)) {
+ PrintF("val = %" V8PRIdPTR ", 0x%" V8PRIxPTR "\n", val, val);
+ }
+ CHECK(is_int16(val));
+ } else {
+ if (!is_uint16(val)) {
+ PrintF("val = %" V8PRIdPTR ", 0x%" V8PRIxPTR
+ ", is_unsigned_imm16(val)=%d, kImm16Mask=0x%x\n",
+ val, val, is_uint16(val), kImm16Mask);
+ }
+ CHECK(is_uint16(val));
+ }
+ emit(instr | rt.code() * B21 | ra.code() * B16 | (kImm16Mask & val));
+}
+
+void Assembler::xo_form(Instr instr, Register rt, Register ra, Register rb,
+ OEBit o, RCBit r) {
+ emit(instr | rt.code() * B21 | ra.code() * B16 | rb.code() * B11 | o | r);
+}
+
+void Assembler::md_form(Instr instr, Register ra, Register rs, int shift,
+ int maskbit, RCBit r) {
+ int sh0_4 = shift & 0x1F;
+ int sh5 = (shift >> 5) & 0x1;
+ int m0_4 = maskbit & 0x1F;
+ int m5 = (maskbit >> 5) & 0x1;
+
+ emit(instr | rs.code() * B21 | ra.code() * B16 | sh0_4 * B11 | m0_4 * B6 |
+ m5 * B5 | sh5 * B1 | r);
+}
+
+void Assembler::mds_form(Instr instr, Register ra, Register rs, Register rb,
+ int maskbit, RCBit r) {
+ int m0_4 = maskbit & 0x1F;
+ int m5 = (maskbit >> 5) & 0x1;
+
+ emit(instr | rs.code() * B21 | ra.code() * B16 | rb.code() * B11 | m0_4 * B6 |
+ m5 * B5 | r);
+}
+
+// Returns the next free trampoline entry.
+int32_t Assembler::get_trampoline_entry() {
+ int32_t trampoline_entry = kInvalidSlotPos;
+
+ if (!internal_trampoline_exception_) {
+ trampoline_entry = trampoline_.take_slot();
+
+ if (kInvalidSlotPos == trampoline_entry) {
+ internal_trampoline_exception_ = true;
+ }
+ }
+ return trampoline_entry;
+}
+
+int Assembler::link(Label* L) {
+ int position;
+ if (L->is_bound()) {
+ position = L->pos();
+ } else {
+ if (L->is_linked()) {
+ position = L->pos(); // L's link
+ } else {
+ // was: target_pos = kEndOfChain;
+ // However, using self to mark the first reference
+ // should avoid most instances of branch offset overflow. See
+ // target_at() for where this is converted back to kEndOfChain.
+ position = pc_offset();
+ }
+ L->link_to(pc_offset());
+ }
+
+ return position;
+}
+
+// Branch instructions.
+
+void Assembler::bclr(BOfield bo, int condition_bit, LKBit lk) {
+ emit(EXT1 | bo | condition_bit * B16 | BCLRX | lk);
+}
+
+void Assembler::bcctr(BOfield bo, int condition_bit, LKBit lk) {
+ emit(EXT1 | bo | condition_bit * B16 | BCCTRX | lk);
+}
+
+// Pseudo op - branch to link register
+void Assembler::blr() { bclr(BA, 0, LeaveLK); }
+
+// Pseudo op - branch to count register -- used for "jump"
+void Assembler::bctr() { bcctr(BA, 0, LeaveLK); }
+
+void Assembler::bctrl() { bcctr(BA, 0, SetLK); }
+
+void Assembler::bc(int branch_offset, BOfield bo, int condition_bit, LKBit lk) {
+ int imm16 = branch_offset;
+ CHECK(is_int16(imm16) && (imm16 & (kAAMask | kLKMask)) == 0);
+ emit(BCX | bo | condition_bit * B16 | (imm16 & kImm16Mask) | lk);
+}
+
+void Assembler::b(int branch_offset, LKBit lk) {
+ int imm26 = branch_offset;
+ CHECK(is_int26(imm26) && (imm26 & (kAAMask | kLKMask)) == 0);
+ emit(BX | (imm26 & kImm26Mask) | lk);
+}
+
+void Assembler::xori(Register dst, Register src, const Operand& imm) {
+ d_form(XORI, src, dst, imm.immediate(), false);
+}
+
+void Assembler::xoris(Register ra, Register rs, const Operand& imm) {
+ d_form(XORIS, rs, ra, imm.immediate(), false);
+}
+
+void Assembler::rlwinm(Register ra, Register rs, int sh, int mb, int me,
+ RCBit rc) {
+ sh &= 0x1F;
+ mb &= 0x1F;
+ me &= 0x1F;
+ emit(RLWINMX | rs.code() * B21 | ra.code() * B16 | sh * B11 | mb * B6 |
+ me << 1 | rc);
+}
+
+void Assembler::rlwnm(Register ra, Register rs, Register rb, int mb, int me,
+ RCBit rc) {
+ mb &= 0x1F;
+ me &= 0x1F;
+ emit(RLWNMX | rs.code() * B21 | ra.code() * B16 | rb.code() * B11 | mb * B6 |
+ me << 1 | rc);
+}
+
+void Assembler::rlwimi(Register ra, Register rs, int sh, int mb, int me,
+ RCBit rc) {
+ sh &= 0x1F;
+ mb &= 0x1F;
+ me &= 0x1F;
+ emit(RLWIMIX | rs.code() * B21 | ra.code() * B16 | sh * B11 | mb * B6 |
+ me << 1 | rc);
+}
+
+void Assembler::slwi(Register dst, Register src, const Operand& val, RCBit rc) {
+ DCHECK((32 > val.immediate()) && (val.immediate() >= 0));
+ rlwinm(dst, src, val.immediate(), 0, 31 - val.immediate(), rc);
+}
+
+void Assembler::srwi(Register dst, Register src, const Operand& val, RCBit rc) {
+ DCHECK((32 > val.immediate()) && (val.immediate() >= 0));
+ rlwinm(dst, src, 32 - val.immediate(), val.immediate(), 31, rc);
+}
+
+void Assembler::clrrwi(Register dst, Register src, const Operand& val,
+ RCBit rc) {
+ DCHECK((32 > val.immediate()) && (val.immediate() >= 0));
+ rlwinm(dst, src, 0, 0, 31 - val.immediate(), rc);
+}
+
+void Assembler::clrlwi(Register dst, Register src, const Operand& val,
+ RCBit rc) {
+ DCHECK((32 > val.immediate()) && (val.immediate() >= 0));
+ rlwinm(dst, src, 0, val.immediate(), 31, rc);
+}
+
+void Assembler::rotlw(Register ra, Register rs, Register rb, RCBit r) {
+ rlwnm(ra, rs, rb, 0, 31, r);
+}
+
+void Assembler::rotlwi(Register ra, Register rs, int sh, RCBit r) {
+ rlwinm(ra, rs, sh, 0, 31, r);
+}
+
+void Assembler::rotrwi(Register ra, Register rs, int sh, RCBit r) {
+ rlwinm(ra, rs, 32 - sh, 0, 31, r);
+}
+
+void Assembler::subi(Register dst, Register src, const Operand& imm) {
+ addi(dst, src, Operand(-(imm.immediate())));
+}
+
+void Assembler::addc(Register dst, Register src1, Register src2, OEBit o,
+ RCBit r) {
+ xo_form(EXT2 | ADDCX, dst, src1, src2, o, r);
+}
+
+void Assembler::adde(Register dst, Register src1, Register src2, OEBit o,
+ RCBit r) {
+ xo_form(EXT2 | ADDEX, dst, src1, src2, o, r);
+}
+
+void Assembler::addze(Register dst, Register src1, OEBit o, RCBit r) {
+ // a special xo_form
+ emit(EXT2 | ADDZEX | dst.code() * B21 | src1.code() * B16 | o | r);
+}
+
+void Assembler::sub(Register dst, Register src1, Register src2, OEBit o,
+ RCBit r) {
+ xo_form(EXT2 | SUBFX, dst, src2, src1, o, r);
+}
+
+void Assembler::subc(Register dst, Register src1, Register src2, OEBit o,
+ RCBit r) {
+ xo_form(EXT2 | SUBFCX, dst, src2, src1, o, r);
+}
+
+void Assembler::sube(Register dst, Register src1, Register src2, OEBit o,
+ RCBit r) {
+ xo_form(EXT2 | SUBFEX, dst, src2, src1, o, r);
+}
+
+void Assembler::subfic(Register dst, Register src, const Operand& imm) {
+ d_form(SUBFIC, dst, src, imm.immediate(), true);
+}
+
+void Assembler::add(Register dst, Register src1, Register src2, OEBit o,
+ RCBit r) {
+ xo_form(EXT2 | ADDX, dst, src1, src2, o, r);
+}
+
+// Multiply low word
+void Assembler::mullw(Register dst, Register src1, Register src2, OEBit o,
+ RCBit r) {
+ xo_form(EXT2 | MULLW, dst, src1, src2, o, r);
+}
+
+// Multiply hi word
+void Assembler::mulhw(Register dst, Register src1, Register src2, RCBit r) {
+ xo_form(EXT2 | MULHWX, dst, src1, src2, LeaveOE, r);
+}
+
+// Multiply hi word unsigned
+void Assembler::mulhwu(Register dst, Register src1, Register src2, RCBit r) {
+ xo_form(EXT2 | MULHWUX, dst, src1, src2, LeaveOE, r);
+}
+
+// Divide word
+void Assembler::divw(Register dst, Register src1, Register src2, OEBit o,
+ RCBit r) {
+ xo_form(EXT2 | DIVW, dst, src1, src2, o, r);
+}
+
+// Divide word unsigned
+void Assembler::divwu(Register dst, Register src1, Register src2, OEBit o,
+ RCBit r) {
+ xo_form(EXT2 | DIVWU, dst, src1, src2, o, r);
+}
+
+void Assembler::addi(Register dst, Register src, const Operand& imm) {
+ DCHECK(src != r0); // use li instead to show intent
+ d_form(ADDI, dst, src, imm.immediate(), true);
+}
+
+void Assembler::addis(Register dst, Register src, const Operand& imm) {
+ DCHECK(src != r0); // use lis instead to show intent
+ d_form(ADDIS, dst, src, imm.immediate(), true);
+}
+
+void Assembler::addic(Register dst, Register src, const Operand& imm) {
+ d_form(ADDIC, dst, src, imm.immediate(), true);
+}
+
+void Assembler::andi(Register ra, Register rs, const Operand& imm) {
+ d_form(ANDIx, rs, ra, imm.immediate(), false);
+}
+
+void Assembler::andis(Register ra, Register rs, const Operand& imm) {
+ d_form(ANDISx, rs, ra, imm.immediate(), false);
+}
+
+void Assembler::ori(Register ra, Register rs, const Operand& imm) {
+ d_form(ORI, rs, ra, imm.immediate(), false);
+}
+
+void Assembler::oris(Register dst, Register src, const Operand& imm) {
+ d_form(ORIS, src, dst, imm.immediate(), false);
+}
+
+void Assembler::cmpi(Register src1, const Operand& src2, CRegister cr) {
+ intptr_t imm16 = src2.immediate();
+#if V8_TARGET_ARCH_PPC64
+ int L = 1;
+#else
+ int L = 0;
+#endif
+ DCHECK(is_int16(imm16));
+ DCHECK(cr.code() >= 0 && cr.code() <= 7);
+ imm16 &= kImm16Mask;
+ emit(CMPI | cr.code() * B23 | L * B21 | src1.code() * B16 | imm16);
+}
+
+void Assembler::cmpli(Register src1, const Operand& src2, CRegister cr) {
+ uintptr_t uimm16 = src2.immediate();
+#if V8_TARGET_ARCH_PPC64
+ int L = 1;
+#else
+ int L = 0;
+#endif
+ DCHECK(is_uint16(uimm16));
+ DCHECK(cr.code() >= 0 && cr.code() <= 7);
+ uimm16 &= kImm16Mask;
+ emit(CMPLI | cr.code() * B23 | L * B21 | src1.code() * B16 | uimm16);
+}
+
+void Assembler::cmpwi(Register src1, const Operand& src2, CRegister cr) {
+ intptr_t imm16 = src2.immediate();
+ int L = 0;
+ int pos = pc_offset();
+ DCHECK(is_int16(imm16));
+ DCHECK(cr.code() >= 0 && cr.code() <= 7);
+ imm16 &= kImm16Mask;
+
+ // For cmpwi against 0, save postition and cr for later examination
+ // of potential optimization.
+ if (imm16 == 0 && pos > 0 && last_bound_pos_ != pos) {
+ optimizable_cmpi_pos_ = pos;
+ cmpi_cr_ = cr;
+ }
+ emit(CMPI | cr.code() * B23 | L * B21 | src1.code() * B16 | imm16);
+}
+
+void Assembler::cmplwi(Register src1, const Operand& src2, CRegister cr) {
+ uintptr_t uimm16 = src2.immediate();
+ int L = 0;
+ DCHECK(is_uint16(uimm16));
+ DCHECK(cr.code() >= 0 && cr.code() <= 7);
+ uimm16 &= kImm16Mask;
+ emit(CMPLI | cr.code() * B23 | L * B21 | src1.code() * B16 | uimm16);
+}
+
+void Assembler::isel(Register rt, Register ra, Register rb, int cb) {
+ emit(EXT2 | ISEL | rt.code() * B21 | ra.code() * B16 | rb.code() * B11 |
+ cb * B6);
+}
+
+// Pseudo op - load immediate
+void Assembler::li(Register dst, const Operand& imm) {
+ d_form(ADDI, dst, r0, imm.immediate(), true);
+}
+
+void Assembler::lis(Register dst, const Operand& imm) {
+ d_form(ADDIS, dst, r0, imm.immediate(), true);
+}
+
+// Pseudo op - move register
+void Assembler::mr(Register dst, Register src) {
+ // actually or(dst, src, src)
+ orx(dst, src, src);
+}
+
+void Assembler::lbz(Register dst, const MemOperand& src) {
+ DCHECK(src.ra_ != r0);
+ d_form(LBZ, dst, src.ra(), src.offset(), true);
+}
+
+void Assembler::lhz(Register dst, const MemOperand& src) {
+ DCHECK(src.ra_ != r0);
+ d_form(LHZ, dst, src.ra(), src.offset(), true);
+}
+
+void Assembler::lwz(Register dst, const MemOperand& src) {
+ DCHECK(src.ra_ != r0);
+ d_form(LWZ, dst, src.ra(), src.offset(), true);
+}
+
+void Assembler::lwzu(Register dst, const MemOperand& src) {
+ DCHECK(src.ra_ != r0);
+ d_form(LWZU, dst, src.ra(), src.offset(), true);
+}
+
+void Assembler::lha(Register dst, const MemOperand& src) {
+ DCHECK(src.ra_ != r0);
+ d_form(LHA, dst, src.ra(), src.offset(), true);
+}
+
+void Assembler::lwa(Register dst, const MemOperand& src) {
+#if V8_TARGET_ARCH_PPC64
+ int offset = src.offset();
+ DCHECK(src.ra_ != r0);
+ CHECK(!(offset & 3) && is_int16(offset));
+ offset = kImm16Mask & offset;
+ emit(LD | dst.code() * B21 | src.ra().code() * B16 | offset | 2);
+#else
+ lwz(dst, src);
+#endif
+}
+
+void Assembler::stb(Register dst, const MemOperand& src) {
+ DCHECK(src.ra_ != r0);
+ d_form(STB, dst, src.ra(), src.offset(), true);
+}
+
+void Assembler::sth(Register dst, const MemOperand& src) {
+ DCHECK(src.ra_ != r0);
+ d_form(STH, dst, src.ra(), src.offset(), true);
+}
+
+void Assembler::stw(Register dst, const MemOperand& src) {
+ DCHECK(src.ra_ != r0);
+ d_form(STW, dst, src.ra(), src.offset(), true);
+}
+
+void Assembler::stwu(Register dst, const MemOperand& src) {
+ DCHECK(src.ra_ != r0);
+ d_form(STWU, dst, src.ra(), src.offset(), true);
+}
+
+void Assembler::neg(Register rt, Register ra, OEBit o, RCBit r) {
+ emit(EXT2 | NEGX | rt.code() * B21 | ra.code() * B16 | o | r);
+}
+
+#if V8_TARGET_ARCH_PPC64
+// 64bit specific instructions
+void Assembler::ld(Register rd, const MemOperand& src) {
+ int offset = src.offset();
+ DCHECK(src.ra_ != r0);
+ CHECK(!(offset & 3) && is_int16(offset));
+ offset = kImm16Mask & offset;
+ emit(LD | rd.code() * B21 | src.ra().code() * B16 | offset);
+}
+
+void Assembler::ldu(Register rd, const MemOperand& src) {
+ int offset = src.offset();
+ DCHECK(src.ra_ != r0);
+ CHECK(!(offset & 3) && is_int16(offset));
+ offset = kImm16Mask & offset;
+ emit(LD | rd.code() * B21 | src.ra().code() * B16 | offset | 1);
+}
+
+void Assembler::std(Register rs, const MemOperand& src) {
+ int offset = src.offset();
+ DCHECK(src.ra_ != r0);
+ CHECK(!(offset & 3) && is_int16(offset));
+ offset = kImm16Mask & offset;
+ emit(STD | rs.code() * B21 | src.ra().code() * B16 | offset);
+}
+
+void Assembler::stdu(Register rs, const MemOperand& src) {
+ int offset = src.offset();
+ DCHECK(src.ra_ != r0);
+ CHECK(!(offset & 3) && is_int16(offset));
+ offset = kImm16Mask & offset;
+ emit(STD | rs.code() * B21 | src.ra().code() * B16 | offset | 1);
+}
+
+void Assembler::rldic(Register ra, Register rs, int sh, int mb, RCBit r) {
+ md_form(EXT5 | RLDIC, ra, rs, sh, mb, r);
+}
+
+void Assembler::rldicl(Register ra, Register rs, int sh, int mb, RCBit r) {
+ md_form(EXT5 | RLDICL, ra, rs, sh, mb, r);
+}
+
+void Assembler::rldcl(Register ra, Register rs, Register rb, int mb, RCBit r) {
+ mds_form(EXT5 | RLDCL, ra, rs, rb, mb, r);
+}
+
+void Assembler::rldicr(Register ra, Register rs, int sh, int me, RCBit r) {
+ md_form(EXT5 | RLDICR, ra, rs, sh, me, r);
+}
+
+void Assembler::sldi(Register dst, Register src, const Operand& val, RCBit rc) {
+ DCHECK((64 > val.immediate()) && (val.immediate() >= 0));
+ rldicr(dst, src, val.immediate(), 63 - val.immediate(), rc);
+}
+
+void Assembler::srdi(Register dst, Register src, const Operand& val, RCBit rc) {
+ DCHECK((64 > val.immediate()) && (val.immediate() >= 0));
+ rldicl(dst, src, 64 - val.immediate(), val.immediate(), rc);
+}
+
+void Assembler::clrrdi(Register dst, Register src, const Operand& val,
+ RCBit rc) {
+ DCHECK((64 > val.immediate()) && (val.immediate() >= 0));
+ rldicr(dst, src, 0, 63 - val.immediate(), rc);
+}
+
+void Assembler::clrldi(Register dst, Register src, const Operand& val,
+ RCBit rc) {
+ DCHECK((64 > val.immediate()) && (val.immediate() >= 0));
+ rldicl(dst, src, 0, val.immediate(), rc);
+}
+
+void Assembler::rldimi(Register ra, Register rs, int sh, int mb, RCBit r) {
+ md_form(EXT5 | RLDIMI, ra, rs, sh, mb, r);
+}
+
+void Assembler::sradi(Register ra, Register rs, int sh, RCBit r) {
+ int sh0_4 = sh & 0x1F;
+ int sh5 = (sh >> 5) & 0x1;
+
+ emit(EXT2 | SRADIX | rs.code() * B21 | ra.code() * B16 | sh0_4 * B11 |
+ sh5 * B1 | r);
+}
+
+void Assembler::rotld(Register ra, Register rs, Register rb, RCBit r) {
+ rldcl(ra, rs, rb, 0, r);
+}
+
+void Assembler::rotldi(Register ra, Register rs, int sh, RCBit r) {
+ rldicl(ra, rs, sh, 0, r);
+}
+
+void Assembler::rotrdi(Register ra, Register rs, int sh, RCBit r) {
+ rldicl(ra, rs, 64 - sh, 0, r);
+}
+
+void Assembler::mulld(Register dst, Register src1, Register src2, OEBit o,
+ RCBit r) {
+ xo_form(EXT2 | MULLD, dst, src1, src2, o, r);
+}
+
+void Assembler::divd(Register dst, Register src1, Register src2, OEBit o,
+ RCBit r) {
+ xo_form(EXT2 | DIVD, dst, src1, src2, o, r);
+}
+
+void Assembler::divdu(Register dst, Register src1, Register src2, OEBit o,
+ RCBit r) {
+ xo_form(EXT2 | DIVDU, dst, src1, src2, o, r);
+}
+#endif
+
+int Assembler::instructions_required_for_mov(Register dst,
+ const Operand& src) const {
+ bool canOptimize =
+ !(src.must_output_reloc_info(this) || is_trampoline_pool_blocked());
+ if (use_constant_pool_for_mov(dst, src, canOptimize)) {
+ if (ConstantPoolAccessIsInOverflow()) {
+ return kMovInstructionsConstantPool + 1;
+ }
+ return kMovInstructionsConstantPool;
+ }
+ DCHECK(!canOptimize);
+ return kMovInstructionsNoConstantPool;
+}
+
+bool Assembler::use_constant_pool_for_mov(Register dst, const Operand& src,
+ bool canOptimize) const {
+ if (!FLAG_enable_embedded_constant_pool || !is_constant_pool_available()) {
+ // If there is no constant pool available, we must use a mov
+ // immediate sequence.
+ return false;
+ }
+ intptr_t value = src.immediate();
+#if V8_TARGET_ARCH_PPC64
+ bool allowOverflow = !((canOptimize && is_int32(value)) || dst == r0);
+#else
+ bool allowOverflow = !(canOptimize || dst == r0);
+#endif
+ if (canOptimize && is_int16(value)) {
+ // Prefer a single-instruction load-immediate.
+ return false;
+ }
+ if (!allowOverflow && ConstantPoolAccessIsInOverflow()) {
+ // Prefer non-relocatable two-instruction bitwise-mov32 over
+ // overflow sequence.
+ return false;
+ }
+
+ return true;
+}
+
+void Assembler::EnsureSpaceFor(int space_needed) {
+ if (buffer_space() <= (kGap + space_needed)) {
+ GrowBuffer(space_needed);
+ }
+}
+
+bool Operand::must_output_reloc_info(const Assembler* assembler) const {
+ if (rmode_ == RelocInfo::EXTERNAL_REFERENCE) {
+ if (assembler != nullptr && assembler->predictable_code_size()) return true;
+ return assembler->options().record_reloc_info_for_serialization;
+ } else if (RelocInfo::IsNone(rmode_)) {
+ return false;
+ }
+ return true;
+}
+
+// Primarily used for loading constants
+// This should really move to be in macro-assembler as it
+// is really a pseudo instruction
+// Some usages of this intend for a FIXED_SEQUENCE to be used
+// Todo - break this dependency so we can optimize mov() in general
+// and only use the generic version when we require a fixed sequence
+void Assembler::mov(Register dst, const Operand& src) {
+ intptr_t value;
+ if (src.IsHeapObjectRequest()) {
+ RequestHeapObject(src.heap_object_request());
+ value = 0;
+ } else {
+ value = src.immediate();
+ }
+ bool relocatable = src.must_output_reloc_info(this);
+ bool canOptimize;
+
+ canOptimize =
+ !(relocatable || (is_trampoline_pool_blocked() && !is_int16(value)));
+
+ if (!src.IsHeapObjectRequest() &&
+ use_constant_pool_for_mov(dst, src, canOptimize)) {
+ DCHECK(is_constant_pool_available());
+ if (relocatable) {
+ RecordRelocInfo(src.rmode_);
+ }
+ ConstantPoolEntry::Access access = ConstantPoolAddEntry(src.rmode_, value);
+#if V8_TARGET_ARCH_PPC64
+ if (access == ConstantPoolEntry::OVERFLOWED) {
+ addis(dst, kConstantPoolRegister, Operand::Zero());
+ ld(dst, MemOperand(dst, 0));
+ } else {
+ ld(dst, MemOperand(kConstantPoolRegister, 0));
+ }
+#else
+ if (access == ConstantPoolEntry::OVERFLOWED) {
+ addis(dst, kConstantPoolRegister, Operand::Zero());
+ lwz(dst, MemOperand(dst, 0));
+ } else {
+ lwz(dst, MemOperand(kConstantPoolRegister, 0));
+ }
+#endif
+ return;
+ }
+
+ if (canOptimize) {
+ if (is_int16(value)) {
+ li(dst, Operand(value));
+ } else {
+ uint16_t u16;
+#if V8_TARGET_ARCH_PPC64
+ if (is_int32(value)) {
+#endif
+ lis(dst, Operand(value >> 16));
+#if V8_TARGET_ARCH_PPC64
+ } else {
+ if (is_int48(value)) {
+ li(dst, Operand(value >> 32));
+ } else {
+ lis(dst, Operand(value >> 48));
+ u16 = ((value >> 32) & 0xFFFF);
+ if (u16) {
+ ori(dst, dst, Operand(u16));
+ }
+ }
+ sldi(dst, dst, Operand(32));
+ u16 = ((value >> 16) & 0xFFFF);
+ if (u16) {
+ oris(dst, dst, Operand(u16));
+ }
+ }
+#endif
+ u16 = (value & 0xFFFF);
+ if (u16) {
+ ori(dst, dst, Operand(u16));
+ }
+ }
+ return;
+ }
+
+ DCHECK(!canOptimize);
+ if (relocatable) {
+ RecordRelocInfo(src.rmode_);
+ }
+ bitwise_mov(dst, value);
+}
+
+void Assembler::bitwise_mov(Register dst, intptr_t value) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+#if V8_TARGET_ARCH_PPC64
+ int32_t hi_32 = static_cast<int32_t>(value >> 32);
+ int32_t lo_32 = static_cast<int32_t>(value);
+ int hi_word = static_cast<int>(hi_32 >> 16);
+ int lo_word = static_cast<int>(hi_32 & 0xFFFF);
+ lis(dst, Operand(SIGN_EXT_IMM16(hi_word)));
+ ori(dst, dst, Operand(lo_word));
+ sldi(dst, dst, Operand(32));
+ hi_word = static_cast<int>(((lo_32 >> 16) & 0xFFFF));
+ lo_word = static_cast<int>(lo_32 & 0xFFFF);
+ oris(dst, dst, Operand(hi_word));
+ ori(dst, dst, Operand(lo_word));
+#else
+ int hi_word = static_cast<int>(value >> 16);
+ int lo_word = static_cast<int>(value & 0xFFFF);
+ lis(dst, Operand(SIGN_EXT_IMM16(hi_word)));
+ ori(dst, dst, Operand(lo_word));
+#endif
+}
+
+void Assembler::bitwise_mov32(Register dst, int32_t value) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ int hi_word = static_cast<int>(value >> 16);
+ int lo_word = static_cast<int>(value & 0xFFFF);
+ lis(dst, Operand(SIGN_EXT_IMM16(hi_word)));
+ ori(dst, dst, Operand(lo_word));
+}
+
+void Assembler::bitwise_add32(Register dst, Register src, int32_t value) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ if (is_int16(value)) {
+ addi(dst, src, Operand(value));
+ nop();
+ } else {
+ int hi_word = static_cast<int>(value >> 16);
+ int lo_word = static_cast<int>(value & 0xFFFF);
+ if (lo_word & 0x8000) hi_word++;
+ addis(dst, src, Operand(SIGN_EXT_IMM16(hi_word)));
+ addic(dst, dst, Operand(SIGN_EXT_IMM16(lo_word)));
+ }
+}
+
+void Assembler::mov_label_offset(Register dst, Label* label) {
+ int position = link(label);
+ if (label->is_bound()) {
+ // Load the position of the label relative to the generated code object.
+ mov(dst, Operand(position + Code::kHeaderSize - kHeapObjectTag));
+ } else {
+ // Encode internal reference to unbound label. We use a dummy opcode
+ // such that it won't collide with any opcode that might appear in the
+ // label's chain. Encode the destination register in the 2nd instruction.
+ int link = position - pc_offset();
+ DCHECK_EQ(0, link & 3);
+ link >>= 2;
+ DCHECK(is_int26(link));
+
+ // When the label is bound, these instructions will be patched
+ // with a 2 instruction mov sequence that will load the
+ // destination register with the position of the label from the
+ // beginning of the code.
+ //
+ // target_at extracts the link and target_at_put patches the instructions.
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ emit(kUnboundMovLabelOffsetOpcode | (link & kImm26Mask));
+ emit(dst.code());
+ }
+}
+
+void Assembler::add_label_offset(Register dst, Register base, Label* label,
+ int delta) {
+ int position = link(label);
+ if (label->is_bound()) {
+ // dst = base + position + delta
+ position += delta;
+ bitwise_add32(dst, base, position);
+ } else {
+ // Encode internal reference to unbound label. We use a dummy opcode
+ // such that it won't collide with any opcode that might appear in the
+ // label's chain. Encode the operands in the 2nd instruction.
+ int link = position - pc_offset();
+ DCHECK_EQ(0, link & 3);
+ link >>= 2;
+ DCHECK(is_int26(link));
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+
+ emit((is_int22(delta) ? kUnboundAddLabelOffsetOpcode
+ : kUnboundAddLabelLongOffsetOpcode) |
+ (link & kImm26Mask));
+ emit(dst.code() * B27 | base.code() * B22 | (delta & kImm22Mask));
+
+ if (!is_int22(delta)) {
+ emit(delta);
+ }
+ }
+}
+
+void Assembler::mov_label_addr(Register dst, Label* label) {
+ CheckBuffer();
+ RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE_ENCODED);
+ int position = link(label);
+ if (label->is_bound()) {
+ // Keep internal references relative until EmitRelocations.
+ bitwise_mov(dst, position);
+ } else {
+ // Encode internal reference to unbound label. We use a dummy opcode
+ // such that it won't collide with any opcode that might appear in the
+ // label's chain. Encode the destination register in the 2nd instruction.
+ int link = position - pc_offset();
+ DCHECK_EQ(0, link & 3);
+ link >>= 2;
+ DCHECK(is_int26(link));
+
+ // When the label is bound, these instructions will be patched
+ // with a multi-instruction mov sequence that will load the
+ // destination register with the address of the label.
+ //
+ // target_at extracts the link and target_at_put patches the instructions.
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ emit(kUnboundMovLabelAddrOpcode | (link & kImm26Mask));
+ emit(dst.code());
+ DCHECK_GE(kMovInstructionsNoConstantPool, 2);
+ for (int i = 0; i < kMovInstructionsNoConstantPool - 2; i++) nop();
+ }
+}
+
+void Assembler::emit_label_addr(Label* label) {
+ CheckBuffer();
+ RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE);
+ int position = link(label);
+ if (label->is_bound()) {
+ // Keep internal references relative until EmitRelocations.
+ dp(position);
+ } else {
+ // Encode internal reference to unbound label. We use a dummy opcode
+ // such that it won't collide with any opcode that might appear in the
+ // label's chain.
+ int link = position - pc_offset();
+ DCHECK_EQ(0, link & 3);
+ link >>= 2;
+ DCHECK(is_int26(link));
+
+ // When the label is bound, the instruction(s) will be patched
+ // as a jump table entry containing the label address. target_at extracts
+ // the link and target_at_put patches the instruction(s).
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ emit(kUnboundJumpTableEntryOpcode | (link & kImm26Mask));
+#if V8_TARGET_ARCH_PPC64
+ nop();
+#endif
+ }
+}
+
+// Special register instructions
+void Assembler::crxor(int bt, int ba, int bb) {
+ emit(EXT1 | CRXOR | bt * B21 | ba * B16 | bb * B11);
+}
+
+void Assembler::creqv(int bt, int ba, int bb) {
+ emit(EXT1 | CREQV | bt * B21 | ba * B16 | bb * B11);
+}
+
+void Assembler::mflr(Register dst) {
+ emit(EXT2 | MFSPR | dst.code() * B21 | 256 << 11); // Ignore RC bit
+}
+
+void Assembler::mtlr(Register src) {
+ emit(EXT2 | MTSPR | src.code() * B21 | 256 << 11); // Ignore RC bit
+}
+
+void Assembler::mtctr(Register src) {
+ emit(EXT2 | MTSPR | src.code() * B21 | 288 << 11); // Ignore RC bit
+}
+
+void Assembler::mtxer(Register src) {
+ emit(EXT2 | MTSPR | src.code() * B21 | 32 << 11);
+}
+
+void Assembler::mcrfs(CRegister cr, FPSCRBit bit) {
+ DCHECK_LT(static_cast<int>(bit), 32);
+ int bf = cr.code();
+ int bfa = bit / CRWIDTH;
+ emit(EXT4 | MCRFS | bf * B23 | bfa * B18);
+}
+
+void Assembler::mfcr(Register dst) { emit(EXT2 | MFCR | dst.code() * B21); }
+
+#if V8_TARGET_ARCH_PPC64
+void Assembler::mffprd(Register dst, DoubleRegister src) {
+ emit(EXT2 | MFVSRD | src.code() * B21 | dst.code() * B16);
+}
+
+void Assembler::mffprwz(Register dst, DoubleRegister src) {
+ emit(EXT2 | MFVSRWZ | src.code() * B21 | dst.code() * B16);
+}
+
+void Assembler::mtfprd(DoubleRegister dst, Register src) {
+ emit(EXT2 | MTVSRD | dst.code() * B21 | src.code() * B16);
+}
+
+void Assembler::mtfprwz(DoubleRegister dst, Register src) {
+ emit(EXT2 | MTVSRWZ | dst.code() * B21 | src.code() * B16);
+}
+
+void Assembler::mtfprwa(DoubleRegister dst, Register src) {
+ emit(EXT2 | MTVSRWA | dst.code() * B21 | src.code() * B16);
+}
+#endif
+
+// Exception-generating instructions and debugging support.
+// Stops with a non-negative code less than kNumOfWatchedStops support
+// enabling/disabling and a counter feature. See simulator-ppc.h .
+void Assembler::stop(Condition cond, int32_t code, CRegister cr) {
+ if (cond != al) {
+ Label skip;
+ b(NegateCondition(cond), &skip, cr);
+ bkpt(0);
+ bind(&skip);
+ } else {
+ bkpt(0);
+ }
+}
+
+void Assembler::bkpt(uint32_t imm16) { emit(0x7D821008); }
+
+void Assembler::dcbf(Register ra, Register rb) {
+ emit(EXT2 | DCBF | ra.code() * B16 | rb.code() * B11);
+}
+
+void Assembler::sync() { emit(EXT2 | SYNC); }
+
+void Assembler::lwsync() { emit(EXT2 | SYNC | 1 * B21); }
+
+void Assembler::icbi(Register ra, Register rb) {
+ emit(EXT2 | ICBI | ra.code() * B16 | rb.code() * B11);
+}
+
+void Assembler::isync() { emit(EXT1 | ISYNC); }
+
+// Floating point support
+
+void Assembler::lfd(const DoubleRegister frt, const MemOperand& src) {
+ int offset = src.offset();
+ Register ra = src.ra();
+ DCHECK(ra != r0);
+ CHECK(is_int16(offset));
+ int imm16 = offset & kImm16Mask;
+ // could be x_form instruction with some casting magic
+ emit(LFD | frt.code() * B21 | ra.code() * B16 | imm16);
+}
+
+void Assembler::lfdu(const DoubleRegister frt, const MemOperand& src) {
+ int offset = src.offset();
+ Register ra = src.ra();
+ DCHECK(ra != r0);
+ CHECK(is_int16(offset));
+ int imm16 = offset & kImm16Mask;
+ // could be x_form instruction with some casting magic
+ emit(LFDU | frt.code() * B21 | ra.code() * B16 | imm16);
+}
+
+void Assembler::lfs(const DoubleRegister frt, const MemOperand& src) {
+ int offset = src.offset();
+ Register ra = src.ra();
+ CHECK(is_int16(offset));
+ DCHECK(ra != r0);
+ int imm16 = offset & kImm16Mask;
+ // could be x_form instruction with some casting magic
+ emit(LFS | frt.code() * B21 | ra.code() * B16 | imm16);
+}
+
+void Assembler::lfsu(const DoubleRegister frt, const MemOperand& src) {
+ int offset = src.offset();
+ Register ra = src.ra();
+ CHECK(is_int16(offset));
+ DCHECK(ra != r0);
+ int imm16 = offset & kImm16Mask;
+ // could be x_form instruction with some casting magic
+ emit(LFSU | frt.code() * B21 | ra.code() * B16 | imm16);
+}
+
+void Assembler::stfd(const DoubleRegister frs, const MemOperand& src) {
+ int offset = src.offset();
+ Register ra = src.ra();
+ CHECK(is_int16(offset));
+ DCHECK(ra != r0);
+ int imm16 = offset & kImm16Mask;
+ // could be x_form instruction with some casting magic
+ emit(STFD | frs.code() * B21 | ra.code() * B16 | imm16);
+}
+
+void Assembler::stfdu(const DoubleRegister frs, const MemOperand& src) {
+ int offset = src.offset();
+ Register ra = src.ra();
+ CHECK(is_int16(offset));
+ DCHECK(ra != r0);
+ int imm16 = offset & kImm16Mask;
+ // could be x_form instruction with some casting magic
+ emit(STFDU | frs.code() * B21 | ra.code() * B16 | imm16);
+}
+
+void Assembler::stfs(const DoubleRegister frs, const MemOperand& src) {
+ int offset = src.offset();
+ Register ra = src.ra();
+ CHECK(is_int16(offset));
+ DCHECK(ra != r0);
+ int imm16 = offset & kImm16Mask;
+ // could be x_form instruction with some casting magic
+ emit(STFS | frs.code() * B21 | ra.code() * B16 | imm16);
+}
+
+void Assembler::stfsu(const DoubleRegister frs, const MemOperand& src) {
+ int offset = src.offset();
+ Register ra = src.ra();
+ CHECK(is_int16(offset));
+ DCHECK(ra != r0);
+ int imm16 = offset & kImm16Mask;
+ // could be x_form instruction with some casting magic
+ emit(STFSU | frs.code() * B21 | ra.code() * B16 | imm16);
+}
+
+void Assembler::fsub(const DoubleRegister frt, const DoubleRegister fra,
+ const DoubleRegister frb, RCBit rc) {
+ a_form(EXT4 | FSUB, frt, fra, frb, rc);
+}
+
+void Assembler::fadd(const DoubleRegister frt, const DoubleRegister fra,
+ const DoubleRegister frb, RCBit rc) {
+ a_form(EXT4 | FADD, frt, fra, frb, rc);
+}
+
+void Assembler::fmul(const DoubleRegister frt, const DoubleRegister fra,
+ const DoubleRegister frc, RCBit rc) {
+ emit(EXT4 | FMUL | frt.code() * B21 | fra.code() * B16 | frc.code() * B6 |
+ rc);
+}
+
+void Assembler::fdiv(const DoubleRegister frt, const DoubleRegister fra,
+ const DoubleRegister frb, RCBit rc) {
+ a_form(EXT4 | FDIV, frt, fra, frb, rc);
+}
+
+void Assembler::fcmpu(const DoubleRegister fra, const DoubleRegister frb,
+ CRegister cr) {
+ DCHECK(cr.code() >= 0 && cr.code() <= 7);
+ emit(EXT4 | FCMPU | cr.code() * B23 | fra.code() * B16 | frb.code() * B11);
+}
+
+void Assembler::fmr(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc) {
+ emit(EXT4 | FMR | frt.code() * B21 | frb.code() * B11 | rc);
+}
+
+void Assembler::fctiwz(const DoubleRegister frt, const DoubleRegister frb) {
+ emit(EXT4 | FCTIWZ | frt.code() * B21 | frb.code() * B11);
+}
+
+void Assembler::fctiw(const DoubleRegister frt, const DoubleRegister frb) {
+ emit(EXT4 | FCTIW | frt.code() * B21 | frb.code() * B11);
+}
+
+void Assembler::fctiwuz(const DoubleRegister frt, const DoubleRegister frb) {
+ emit(EXT4 | FCTIWUZ | frt.code() * B21 | frb.code() * B11);
+}
+
+void Assembler::frin(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc) {
+ emit(EXT4 | FRIN | frt.code() * B21 | frb.code() * B11 | rc);
+}
+
+void Assembler::friz(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc) {
+ emit(EXT4 | FRIZ | frt.code() * B21 | frb.code() * B11 | rc);
+}
+
+void Assembler::frip(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc) {
+ emit(EXT4 | FRIP | frt.code() * B21 | frb.code() * B11 | rc);
+}
+
+void Assembler::frim(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc) {
+ emit(EXT4 | FRIM | frt.code() * B21 | frb.code() * B11 | rc);
+}
+
+void Assembler::frsp(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc) {
+ emit(EXT4 | FRSP | frt.code() * B21 | frb.code() * B11 | rc);
+}
+
+void Assembler::fcfid(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc) {
+ emit(EXT4 | FCFID | frt.code() * B21 | frb.code() * B11 | rc);
+}
+
+void Assembler::fcfidu(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc) {
+ emit(EXT4 | FCFIDU | frt.code() * B21 | frb.code() * B11 | rc);
+}
+
+void Assembler::fcfidus(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc) {
+ emit(EXT3 | FCFIDUS | frt.code() * B21 | frb.code() * B11 | rc);
+}
+
+void Assembler::fcfids(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc) {
+ emit(EXT3 | FCFIDS | frt.code() * B21 | frb.code() * B11 | rc);
+}
+
+void Assembler::fctid(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc) {
+ emit(EXT4 | FCTID | frt.code() * B21 | frb.code() * B11 | rc);
+}
+
+void Assembler::fctidz(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc) {
+ emit(EXT4 | FCTIDZ | frt.code() * B21 | frb.code() * B11 | rc);
+}
+
+void Assembler::fctidu(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc) {
+ emit(EXT4 | FCTIDU | frt.code() * B21 | frb.code() * B11 | rc);
+}
+
+void Assembler::fctiduz(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc) {
+ emit(EXT4 | FCTIDUZ | frt.code() * B21 | frb.code() * B11 | rc);
+}
+
+void Assembler::fsel(const DoubleRegister frt, const DoubleRegister fra,
+ const DoubleRegister frc, const DoubleRegister frb,
+ RCBit rc) {
+ emit(EXT4 | FSEL | frt.code() * B21 | fra.code() * B16 | frb.code() * B11 |
+ frc.code() * B6 | rc);
+}
+
+void Assembler::fneg(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc) {
+ emit(EXT4 | FNEG | frt.code() * B21 | frb.code() * B11 | rc);
+}
+
+void Assembler::mtfsb0(FPSCRBit bit, RCBit rc) {
+ DCHECK_LT(static_cast<int>(bit), 32);
+ int bt = bit;
+ emit(EXT4 | MTFSB0 | bt * B21 | rc);
+}
+
+void Assembler::mtfsb1(FPSCRBit bit, RCBit rc) {
+ DCHECK_LT(static_cast<int>(bit), 32);
+ int bt = bit;
+ emit(EXT4 | MTFSB1 | bt * B21 | rc);
+}
+
+void Assembler::mtfsfi(int bf, int immediate, RCBit rc) {
+ emit(EXT4 | MTFSFI | bf * B23 | immediate * B12 | rc);
+}
+
+void Assembler::mffs(const DoubleRegister frt, RCBit rc) {
+ emit(EXT4 | MFFS | frt.code() * B21 | rc);
+}
+
+void Assembler::mtfsf(const DoubleRegister frb, bool L, int FLM, bool W,
+ RCBit rc) {
+ emit(EXT4 | MTFSF | frb.code() * B11 | W * B16 | FLM * B17 | L * B25 | rc);
+}
+
+void Assembler::fsqrt(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc) {
+ emit(EXT4 | FSQRT | frt.code() * B21 | frb.code() * B11 | rc);
+}
+
+void Assembler::fabs(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc) {
+ emit(EXT4 | FABS | frt.code() * B21 | frb.code() * B11 | rc);
+}
+
+void Assembler::fmadd(const DoubleRegister frt, const DoubleRegister fra,
+ const DoubleRegister frc, const DoubleRegister frb,
+ RCBit rc) {
+ emit(EXT4 | FMADD | frt.code() * B21 | fra.code() * B16 | frb.code() * B11 |
+ frc.code() * B6 | rc);
+}
+
+void Assembler::fmsub(const DoubleRegister frt, const DoubleRegister fra,
+ const DoubleRegister frc, const DoubleRegister frb,
+ RCBit rc) {
+ emit(EXT4 | FMSUB | frt.code() * B21 | fra.code() * B16 | frb.code() * B11 |
+ frc.code() * B6 | rc);
+}
+
+// Vector instructions
+void Assembler::mfvsrd(const Register ra, const Simd128Register rs) {
+ int SX = 1;
+ emit(MFVSRD | rs.code() * B21 | ra.code() * B16 | SX);
+}
+
+void Assembler::mfvsrwz(const Register ra, const Simd128Register rs) {
+ int SX = 1;
+ emit(MFVSRWZ | rs.code() * B21 | ra.code() * B16 | SX);
+}
+
+void Assembler::mtvsrd(const Simd128Register rt, const Register ra) {
+ int TX = 1;
+ emit(MTVSRD | rt.code() * B21 | ra.code() * B16 | TX);
+}
+
+void Assembler::mtvsrdd(const Simd128Register rt, const Register ra,
+ const Register rb) {
+ int TX = 1;
+ emit(MTVSRDD | rt.code() * B21 | ra.code() * B16 | rb.code() * B11 | TX);
+}
+
+void Assembler::lxvd(const Simd128Register rt, const MemOperand& src) {
+ int TX = 1;
+ emit(LXVD | rt.code() * B21 | src.ra().code() * B16 | src.rb().code() * B11 |
+ TX);
+}
+
+void Assembler::stxvd(const Simd128Register rt, const MemOperand& dst) {
+ int SX = 1;
+ emit(STXVD | rt.code() * B21 | dst.ra().code() * B16 | dst.rb().code() * B11 |
+ SX);
+}
+
+void Assembler::xxspltib(const Simd128Register rt, const Operand& imm) {
+ int TX = 1;
+ emit(XXSPLTIB | rt.code() * B21 | imm.immediate() * B11 | TX);
+}
+
+// Pseudo instructions.
+void Assembler::nop(int type) {
+ Register reg = r0;
+ switch (type) {
+ case NON_MARKING_NOP:
+ reg = r0;
+ break;
+ case GROUP_ENDING_NOP:
+ reg = r2;
+ break;
+ case DEBUG_BREAK_NOP:
+ reg = r3;
+ break;
+ default:
+ UNIMPLEMENTED();
+ }
+
+ ori(reg, reg, Operand::Zero());
+}
+
+bool Assembler::IsNop(Instr instr, int type) {
+ int reg = 0;
+ switch (type) {
+ case NON_MARKING_NOP:
+ reg = 0;
+ break;
+ case GROUP_ENDING_NOP:
+ reg = 2;
+ break;
+ case DEBUG_BREAK_NOP:
+ reg = 3;
+ break;
+ default:
+ UNIMPLEMENTED();
+ }
+ return instr == (ORI | reg * B21 | reg * B16);
+}
+
+void Assembler::GrowBuffer(int needed) {
+ DCHECK_EQ(buffer_start_, buffer_->start());
+
+ // Compute new buffer size.
+ int old_size = buffer_->size();
+ int new_size = std::min(2 * old_size, old_size + 1 * MB);
+ int space = buffer_space() + (new_size - old_size);
+ new_size += (space < needed) ? needed - space : 0;
+
+ // Some internal data structures overflow for very large buffers,
+ // they must ensure that kMaximalBufferSize is not too large.
+ if (new_size > kMaximalBufferSize) {
+ V8::FatalProcessOutOfMemory(nullptr, "Assembler::GrowBuffer");
+ }
+
+ // Set up new buffer.
+ std::unique_ptr<AssemblerBuffer> new_buffer = buffer_->Grow(new_size);
+ DCHECK_EQ(new_size, new_buffer->size());
+ byte* new_start = new_buffer->start();
+
+ // Copy the data.
+ intptr_t pc_delta = new_start - buffer_start_;
+ intptr_t rc_delta = (new_start + new_size) - (buffer_start_ + old_size);
+ size_t reloc_size = (buffer_start_ + old_size) - reloc_info_writer.pos();
+ MemMove(new_start, buffer_start_, pc_offset());
+ MemMove(reloc_info_writer.pos() + rc_delta, reloc_info_writer.pos(),
+ reloc_size);
+
+ // Switch buffers.
+ buffer_ = std::move(new_buffer);
+ buffer_start_ = new_start;
+ pc_ += pc_delta;
+ reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta,
+ reloc_info_writer.last_pc() + pc_delta);
+
+ // None of our relocation types are pc relative pointing outside the code
+ // buffer nor pc absolute pointing inside the code buffer, so there is no need
+ // to relocate any emitted relocation entries.
+}
+
+void Assembler::db(uint8_t data) {
+ CheckBuffer();
+ *reinterpret_cast<uint8_t*>(pc_) = data;
+ pc_ += sizeof(uint8_t);
+}
+
+void Assembler::dd(uint32_t data) {
+ CheckBuffer();
+ *reinterpret_cast<uint32_t*>(pc_) = data;
+ pc_ += sizeof(uint32_t);
+}
+
+void Assembler::dq(uint64_t value) {
+ CheckBuffer();
+ *reinterpret_cast<uint64_t*>(pc_) = value;
+ pc_ += sizeof(uint64_t);
+}
+
+void Assembler::dp(uintptr_t data) {
+ CheckBuffer();
+ *reinterpret_cast<uintptr_t*>(pc_) = data;
+ pc_ += sizeof(uintptr_t);
+}
+
+void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) {
+ if (!ShouldRecordRelocInfo(rmode)) return;
+ DeferredRelocInfo rinfo(pc_offset(), rmode, data);
+ relocations_.push_back(rinfo);
+}
+
+void Assembler::EmitRelocations() {
+ EnsureSpaceFor(relocations_.size() * kMaxRelocSize);
+
+ for (std::vector<DeferredRelocInfo>::iterator it = relocations_.begin();
+ it != relocations_.end(); it++) {
+ RelocInfo::Mode rmode = it->rmode();
+ Address pc = reinterpret_cast<Address>(buffer_start_) + it->position();
+ RelocInfo rinfo(pc, rmode, it->data(), Code());
+
+ // Fix up internal references now that they are guaranteed to be bound.
+ if (RelocInfo::IsInternalReference(rmode)) {
+ // Jump table entry
+ intptr_t pos = static_cast<intptr_t>(Memory<Address>(pc));
+ Memory<Address>(pc) = reinterpret_cast<Address>(buffer_start_) + pos;
+ } else if (RelocInfo::IsInternalReferenceEncoded(rmode)) {
+ // mov sequence
+ intptr_t pos = static_cast<intptr_t>(target_address_at(pc, kNullAddress));
+ set_target_address_at(pc, 0,
+ reinterpret_cast<Address>(buffer_start_) + pos,
+ SKIP_ICACHE_FLUSH);
+ }
+
+ reloc_info_writer.Write(&rinfo);
+ }
+}
+
+void Assembler::BlockTrampolinePoolFor(int instructions) {
+ BlockTrampolinePoolBefore(pc_offset() + instructions * kInstrSize);
+}
+
+void Assembler::CheckTrampolinePool() {
+ // Some small sequences of instructions must not be broken up by the
+ // insertion of a trampoline pool; such sequences are protected by setting
+ // either trampoline_pool_blocked_nesting_ or no_trampoline_pool_before_,
+ // which are both checked here. Also, recursive calls to CheckTrampolinePool
+ // are blocked by trampoline_pool_blocked_nesting_.
+ if (trampoline_pool_blocked_nesting_ > 0) return;
+ if (pc_offset() < no_trampoline_pool_before_) {
+ next_trampoline_check_ = no_trampoline_pool_before_;
+ return;
+ }
+
+ DCHECK(!trampoline_emitted_);
+ if (tracked_branch_count_ > 0) {
+ int size = tracked_branch_count_ * kInstrSize;
+
+ // As we are only going to emit trampoline once, we need to prevent any
+ // further emission.
+ trampoline_emitted_ = true;
+ next_trampoline_check_ = kMaxInt;
+
+ // First we emit jump, then we emit trampoline pool.
+ b(size + kInstrSize, LeaveLK);
+ for (int i = size; i > 0; i -= kInstrSize) {
+ b(i, LeaveLK);
+ }
+
+ trampoline_ = Trampoline(pc_offset() - size, tracked_branch_count_);
+ }
+}
+
+PatchingAssembler::PatchingAssembler(const AssemblerOptions& options,
+ byte* address, int instructions)
+ : Assembler(options, ExternalAssemblerBuffer(
+ address, instructions * kInstrSize + kGap)) {
+ DCHECK_EQ(reloc_info_writer.pos(), buffer_start_ + buffer_->size());
+}
+
+PatchingAssembler::~PatchingAssembler() {
+ // Check that the code was patched as expected.
+ DCHECK_EQ(pc_, buffer_start_ + buffer_->size() - kGap);
+ DCHECK_EQ(reloc_info_writer.pos(), buffer_start_ + buffer_->size());
+}
+
+UseScratchRegisterScope::UseScratchRegisterScope(Assembler* assembler)
+ : assembler_(assembler),
+ old_available_(*assembler->GetScratchRegisterList()) {}
+
+UseScratchRegisterScope::~UseScratchRegisterScope() {
+ *assembler_->GetScratchRegisterList() = old_available_;
+}
+
+Register UseScratchRegisterScope::Acquire() {
+ RegList* available = assembler_->GetScratchRegisterList();
+ DCHECK_NOT_NULL(available);
+ DCHECK_NE(*available, 0);
+ int index = static_cast<int>(base::bits::CountTrailingZeros32(*available));
+ Register reg = Register::from_code(index);
+ *available &= ~reg.bit();
+ return reg;
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_PPC || V8_TARGET_ARCH_PPC64
diff --git a/src/codegen/ppc/assembler-ppc.h b/src/codegen/ppc/assembler-ppc.h
new file mode 100644
index 0000000..11497c9
--- /dev/null
+++ b/src/codegen/ppc/assembler-ppc.h
@@ -0,0 +1,1440 @@
+// Copyright (c) 1994-2006 Sun Microsystems Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//
+// - Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// - Redistribution in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the
+// distribution.
+//
+// - Neither the name of Sun Microsystems or the names of contributors may
+// be used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+// COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+// OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// The original source code covered by the above license above has been
+// modified significantly by Google Inc.
+// Copyright 2014 the V8 project authors. All rights reserved.
+
+// A light-weight PPC Assembler
+// Generates user mode instructions for the PPC architecture up
+
+#ifndef V8_CODEGEN_PPC_ASSEMBLER_PPC_H_
+#define V8_CODEGEN_PPC_ASSEMBLER_PPC_H_
+
+#include <stdio.h>
+#include <memory>
+#include <vector>
+
+#include "src/codegen/assembler.h"
+#include "src/codegen/constant-pool.h"
+#include "src/codegen/external-reference.h"
+#include "src/codegen/label.h"
+#include "src/codegen/ppc/constants-ppc.h"
+#include "src/codegen/ppc/register-ppc.h"
+#include "src/numbers/double.h"
+#include "src/objects/smi.h"
+
+namespace v8 {
+namespace internal {
+
+class SafepointTableBuilder;
+
+// -----------------------------------------------------------------------------
+// Machine instruction Operands
+
+// Class Operand represents a shifter operand in data processing instructions
+class V8_EXPORT_PRIVATE Operand {
+ public:
+ // immediate
+ V8_INLINE explicit Operand(intptr_t immediate,
+ RelocInfo::Mode rmode = RelocInfo::NONE)
+ : rmode_(rmode) {
+ value_.immediate = immediate;
+ }
+ V8_INLINE static Operand Zero() { return Operand(static_cast<intptr_t>(0)); }
+ V8_INLINE explicit Operand(const ExternalReference& f)
+ : rmode_(RelocInfo::EXTERNAL_REFERENCE) {
+ value_.immediate = static_cast<intptr_t>(f.address());
+ }
+ explicit Operand(Handle<HeapObject> handle);
+ V8_INLINE explicit Operand(Smi value) : rmode_(RelocInfo::NONE) {
+ value_.immediate = static_cast<intptr_t>(value.ptr());
+ }
+ // rm
+ V8_INLINE explicit Operand(Register rm);
+
+ static Operand EmbeddedNumber(double number); // Smi or HeapNumber.
+ static Operand EmbeddedStringConstant(const StringConstantBase* str);
+
+ // Return true if this is a register operand.
+ V8_INLINE bool is_reg() const { return rm_.is_valid(); }
+
+ bool must_output_reloc_info(const Assembler* assembler) const;
+
+ inline intptr_t immediate() const {
+ DCHECK(IsImmediate());
+ DCHECK(!IsHeapObjectRequest());
+ return value_.immediate;
+ }
+ bool IsImmediate() const { return !rm_.is_valid(); }
+
+ HeapObjectRequest heap_object_request() const {
+ DCHECK(IsHeapObjectRequest());
+ return value_.heap_object_request;
+ }
+
+ Register rm() const { return rm_; }
+
+ bool IsHeapObjectRequest() const {
+ DCHECK_IMPLIES(is_heap_object_request_, IsImmediate());
+ DCHECK_IMPLIES(is_heap_object_request_,
+ rmode_ == RelocInfo::FULL_EMBEDDED_OBJECT ||
+ rmode_ == RelocInfo::CODE_TARGET);
+ return is_heap_object_request_;
+ }
+
+ private:
+ Register rm_ = no_reg;
+ union Value {
+ Value() {}
+ HeapObjectRequest heap_object_request; // if is_heap_object_request_
+ intptr_t immediate; // otherwise
+ } value_; // valid if rm_ == no_reg
+ bool is_heap_object_request_ = false;
+
+ RelocInfo::Mode rmode_;
+
+ friend class Assembler;
+ friend class MacroAssembler;
+};
+
+// Class MemOperand represents a memory operand in load and store instructions
+// On PowerPC we have base register + 16bit signed value
+// Alternatively we can have a 16bit signed value immediate
+class V8_EXPORT_PRIVATE MemOperand {
+ public:
+ explicit MemOperand(Register rn, int32_t offset = 0);
+
+ explicit MemOperand(Register ra, Register rb);
+
+ int32_t offset() const { return offset_; }
+
+ // PowerPC - base register
+ Register ra() const { return ra_; }
+
+ Register rb() const { return rb_; }
+
+ private:
+ Register ra_; // base
+ int32_t offset_; // offset
+ Register rb_; // index
+
+ friend class Assembler;
+};
+
+class DeferredRelocInfo {
+ public:
+ DeferredRelocInfo() {}
+ DeferredRelocInfo(int position, RelocInfo::Mode rmode, intptr_t data)
+ : position_(position), rmode_(rmode), data_(data) {}
+
+ int position() const { return position_; }
+ RelocInfo::Mode rmode() const { return rmode_; }
+ intptr_t data() const { return data_; }
+
+ private:
+ int position_;
+ RelocInfo::Mode rmode_;
+ intptr_t data_;
+};
+
+class Assembler : public AssemblerBase {
+ public:
+ // Create an assembler. Instructions and relocation information are emitted
+ // into a buffer, with the instructions starting from the beginning and the
+ // relocation information starting from the end of the buffer. See CodeDesc
+ // for a detailed comment on the layout (globals.h).
+ //
+ // If the provided buffer is nullptr, the assembler allocates and grows its
+ // own buffer. Otherwise it takes ownership of the provided buffer.
+ explicit Assembler(const AssemblerOptions&,
+ std::unique_ptr<AssemblerBuffer> = {});
+
+ virtual ~Assembler() {}
+
+ // GetCode emits any pending (non-emitted) code and fills the descriptor desc.
+ static constexpr int kNoHandlerTable = 0;
+ static constexpr SafepointTableBuilder* kNoSafepointTable = nullptr;
+ void GetCode(Isolate* isolate, CodeDesc* desc,
+ SafepointTableBuilder* safepoint_table_builder,
+ int handler_table_offset);
+
+ // Convenience wrapper for code without safepoint or handler tables.
+ void GetCode(Isolate* isolate, CodeDesc* desc) {
+ GetCode(isolate, desc, kNoSafepointTable, kNoHandlerTable);
+ }
+
+ void MaybeEmitOutOfLineConstantPool() { EmitConstantPool(); }
+
+ // Label operations & relative jumps (PPUM Appendix D)
+ //
+ // Takes a branch opcode (cc) and a label (L) and generates
+ // either a backward branch or a forward branch and links it
+ // to the label fixup chain. Usage:
+ //
+ // Label L; // unbound label
+ // j(cc, &L); // forward branch to unbound label
+ // bind(&L); // bind label to the current pc
+ // j(cc, &L); // backward branch to bound label
+ // bind(&L); // illegal: a label may be bound only once
+ //
+ // Note: The same Label can be used for forward and backward branches
+ // but it may be bound only once.
+
+ void bind(Label* L); // binds an unbound label L to the current code position
+
+ // Links a label at the current pc_offset(). If already bound, returns the
+ // bound position. If already linked, returns the position of the prior link.
+ // Otherwise, returns the current pc_offset().
+ int link(Label* L);
+
+ // Determines if Label is bound and near enough so that a single
+ // branch instruction can be used to reach it.
+ bool is_near(Label* L, Condition cond);
+
+ // Returns the branch offset to the given label from the current code position
+ // Links the label to the current position if it is still unbound
+ int branch_offset(Label* L) {
+ if (L->is_unused() && !trampoline_emitted_) {
+ TrackBranch();
+ }
+ return link(L) - pc_offset();
+ }
+
+ V8_INLINE static bool IsConstantPoolLoadStart(
+ Address pc, ConstantPoolEntry::Access* access = nullptr);
+ V8_INLINE static bool IsConstantPoolLoadEnd(
+ Address pc, ConstantPoolEntry::Access* access = nullptr);
+ V8_INLINE static int GetConstantPoolOffset(Address pc,
+ ConstantPoolEntry::Access access,
+ ConstantPoolEntry::Type type);
+ V8_INLINE void PatchConstantPoolAccessInstruction(
+ int pc_offset, int offset, ConstantPoolEntry::Access access,
+ ConstantPoolEntry::Type type);
+
+ // Return the address in the constant pool of the code target address used by
+ // the branch/call instruction at pc, or the object in a mov.
+ V8_INLINE static Address target_constant_pool_address_at(
+ Address pc, Address constant_pool, ConstantPoolEntry::Access access,
+ ConstantPoolEntry::Type type);
+
+ // Read/Modify the code target address in the branch/call instruction at pc.
+ // The isolate argument is unused (and may be nullptr) when skipping flushing.
+ V8_INLINE static Address target_address_at(Address pc, Address constant_pool);
+ V8_INLINE static void set_target_address_at(
+ Address pc, Address constant_pool, Address target,
+ ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED);
+
+ // Read/Modify the code target address in the branch/call instruction at pc.
+ inline static Tagged_t target_compressed_address_at(Address pc,
+ Address constant_pool);
+ inline static void set_target_compressed_address_at(
+ Address pc, Address constant_pool, Tagged_t target,
+ ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED);
+
+ inline Handle<Object> code_target_object_handle_at(Address pc,
+ Address constant_pool);
+ inline Handle<HeapObject> compressed_embedded_object_handle_at(
+ Address pc, Address constant_pool);
+
+ // This sets the branch destination.
+ // This is for calls and branches within generated code.
+ inline static void deserialization_set_special_target_at(
+ Address instruction_payload, Code code, Address target);
+
+ // Get the size of the special target encoded at 'instruction_payload'.
+ inline static int deserialization_special_target_size(
+ Address instruction_payload);
+
+ // This sets the internal reference at the pc.
+ inline static void deserialization_set_target_internal_reference_at(
+ Address pc, Address target,
+ RelocInfo::Mode mode = RelocInfo::INTERNAL_REFERENCE);
+
+ // Here we are patching the address in the LUI/ORI instruction pair.
+ // These values are used in the serialization process and must be zero for
+ // PPC platform, as Code, Embedded Object or External-reference pointers
+ // are split across two consecutive instructions and don't exist separately
+ // in the code, so the serializer should not step forwards in memory after
+ // a target is resolved and written.
+ static constexpr int kSpecialTargetSize = 0;
+
+// Number of instructions to load an address via a mov sequence.
+#if V8_TARGET_ARCH_PPC64
+ static constexpr int kMovInstructionsConstantPool = 1;
+ static constexpr int kMovInstructionsNoConstantPool = 5;
+#if defined(V8_PPC_TAGGING_OPT)
+ static constexpr int kTaggedLoadInstructions = 1;
+#else
+ static constexpr int kTaggedLoadInstructions = 2;
+#endif
+#else
+ static constexpr int kMovInstructionsConstantPool = 1;
+ static constexpr int kMovInstructionsNoConstantPool = 2;
+ static constexpr int kTaggedLoadInstructions = 1;
+#endif
+ static constexpr int kMovInstructions = FLAG_enable_embedded_constant_pool
+ ? kMovInstructionsConstantPool
+ : kMovInstructionsNoConstantPool;
+
+ static inline int encode_crbit(const CRegister& cr, enum CRBit crbit) {
+ return ((cr.code() * CRWIDTH) + crbit);
+ }
+
+#define DECLARE_PPC_X_INSTRUCTIONS_A_FORM(name, instr_name, instr_value) \
+ inline void name(const Register rt, const Register ra, const Register rb, \
+ const RCBit rc = LeaveRC) { \
+ x_form(instr_name, rt, ra, rb, rc); \
+ }
+
+#define DECLARE_PPC_X_INSTRUCTIONS_B_FORM(name, instr_name, instr_value) \
+ inline void name(const Register ra, const Register rs, const Register rb, \
+ const RCBit rc = LeaveRC) { \
+ x_form(instr_name, rs, ra, rb, rc); \
+ }
+
+#define DECLARE_PPC_X_INSTRUCTIONS_C_FORM(name, instr_name, instr_value) \
+ inline void name(const Register dst, const Register src, \
+ const RCBit rc = LeaveRC) { \
+ x_form(instr_name, src, dst, r0, rc); \
+ }
+
+#define DECLARE_PPC_X_INSTRUCTIONS_D_FORM(name, instr_name, instr_value) \
+ template <class R> \
+ inline void name(const R rt, const Register ra, const Register rb, \
+ const RCBit rc = LeaveRC) { \
+ x_form(instr_name, rt.code(), ra.code(), rb.code(), rc); \
+ } \
+ template <class R> \
+ inline void name(const R dst, const MemOperand& src) { \
+ name(dst, src.ra(), src.rb()); \
+ }
+
+#define DECLARE_PPC_X_INSTRUCTIONS_E_FORM(name, instr_name, instr_value) \
+ inline void name(const Register dst, const Register src, const int sh, \
+ const RCBit rc = LeaveRC) { \
+ x_form(instr_name, src.code(), dst.code(), sh, rc); \
+ }
+
+#define DECLARE_PPC_X_INSTRUCTIONS_F_FORM(name, instr_name, instr_value) \
+ inline void name(const Register src1, const Register src2, \
+ const CRegister cr = cr7, const RCBit rc = LeaveRC) { \
+ x_form(instr_name, cr, src1, src2, rc); \
+ } \
+ inline void name##w(const Register src1, const Register src2, \
+ const CRegister cr = cr7, const RCBit rc = LeaveRC) { \
+ x_form(instr_name, cr.code() * B2, src1.code(), src2.code(), LeaveRC); \
+ }
+
+#define DECLARE_PPC_X_INSTRUCTIONS_EH_S_FORM(name, instr_name, instr_value) \
+ inline void name(const Register dst, const MemOperand& src) { \
+ x_form(instr_name, src.ra(), dst, src.rb(), SetEH); \
+ }
+#define DECLARE_PPC_X_INSTRUCTIONS_EH_L_FORM(name, instr_name, instr_value) \
+ inline void name(const Register dst, const MemOperand& src) { \
+ DCHECK(src.ra_ != r0); \
+ x_form(instr_name, src.ra(), dst, src.rb(), SetEH); \
+ }
+
+ inline void x_form(Instr instr, int f1, int f2, int f3, int rc) {
+ emit(instr | f1 * B21 | f2 * B16 | f3 * B11 | rc);
+ }
+ inline void x_form(Instr instr, Register rs, Register ra, Register rb,
+ RCBit rc) {
+ emit(instr | rs.code() * B21 | ra.code() * B16 | rb.code() * B11 | rc);
+ }
+ inline void x_form(Instr instr, Register ra, Register rs, Register rb,
+ EHBit eh = SetEH) {
+ emit(instr | rs.code() * B21 | ra.code() * B16 | rb.code() * B11 | eh);
+ }
+ inline void x_form(Instr instr, CRegister cr, Register s1, Register s2,
+ RCBit rc) {
+#if V8_TARGET_ARCH_PPC64
+ int L = 1;
+#else
+ int L = 0;
+#endif
+ emit(instr | cr.code() * B23 | L * B21 | s1.code() * B16 | s2.code() * B11 |
+ rc);
+ }
+
+ PPC_X_OPCODE_A_FORM_LIST(DECLARE_PPC_X_INSTRUCTIONS_A_FORM)
+ PPC_X_OPCODE_B_FORM_LIST(DECLARE_PPC_X_INSTRUCTIONS_B_FORM)
+ PPC_X_OPCODE_C_FORM_LIST(DECLARE_PPC_X_INSTRUCTIONS_C_FORM)
+ PPC_X_OPCODE_D_FORM_LIST(DECLARE_PPC_X_INSTRUCTIONS_D_FORM)
+ PPC_X_OPCODE_E_FORM_LIST(DECLARE_PPC_X_INSTRUCTIONS_E_FORM)
+ PPC_X_OPCODE_F_FORM_LIST(DECLARE_PPC_X_INSTRUCTIONS_F_FORM)
+ PPC_X_OPCODE_EH_S_FORM_LIST(DECLARE_PPC_X_INSTRUCTIONS_EH_S_FORM)
+ PPC_X_OPCODE_EH_L_FORM_LIST(DECLARE_PPC_X_INSTRUCTIONS_EH_L_FORM)
+
+ inline void notx(Register dst, Register src, RCBit rc = LeaveRC) {
+ nor(dst, src, src, rc);
+ }
+ inline void lwax(Register rt, const MemOperand& src) {
+#if V8_TARGET_ARCH_PPC64
+ Register ra = src.ra();
+ Register rb = src.rb();
+ DCHECK(ra != r0);
+ x_form(LWAX, rt, ra, rb, LeaveRC);
+#else
+ lwzx(rt, src);
+#endif
+ }
+ inline void extsw(Register rs, Register ra, RCBit rc = LeaveRC) {
+#if V8_TARGET_ARCH_PPC64
+ emit(EXT2 | EXTSW | ra.code() * B21 | rs.code() * B16 | rc);
+#else
+ // nop on 32-bit
+ DCHECK(rs == ra && rc == LeaveRC);
+#endif
+ }
+
+#undef DECLARE_PPC_X_INSTRUCTIONS_A_FORM
+#undef DECLARE_PPC_X_INSTRUCTIONS_B_FORM
+#undef DECLARE_PPC_X_INSTRUCTIONS_C_FORM
+#undef DECLARE_PPC_X_INSTRUCTIONS_D_FORM
+#undef DECLARE_PPC_X_INSTRUCTIONS_E_FORM
+#undef DECLARE_PPC_X_INSTRUCTIONS_F_FORM
+#undef DECLARE_PPC_X_INSTRUCTIONS_EH_S_FORM
+#undef DECLARE_PPC_X_INSTRUCTIONS_EH_L_FORM
+
+#define DECLARE_PPC_XX2_INSTRUCTIONS(name, instr_name, instr_value) \
+ inline void name(const Simd128Register rt, const Simd128Register rb) { \
+ xx2_form(instr_name, rt, rb); \
+ }
+
+ inline void xx2_form(Instr instr, Simd128Register t, Simd128Register b) {
+ // Using VR (high VSR) registers.
+ int BX = 1;
+ int TX = 1;
+
+ emit(instr | (t.code() & 0x1F) * B21 | (b.code() & 0x1F) * B11 | BX * B1 |
+ TX);
+ }
+
+ PPC_XX2_OPCODE_A_FORM_LIST(DECLARE_PPC_XX2_INSTRUCTIONS)
+#undef DECLARE_PPC_XX2_INSTRUCTIONS
+
+#define DECLARE_PPC_XX3_INSTRUCTIONS(name, instr_name, instr_value) \
+ inline void name(const DoubleRegister rt, const DoubleRegister ra, \
+ const DoubleRegister rb) { \
+ xx3_form(instr_name, rt, ra, rb); \
+ }
+
+ inline void xx3_form(Instr instr, DoubleRegister t, DoubleRegister a,
+ DoubleRegister b) {
+ // Using VR (high VSR) registers.
+ int AX = 1;
+ int BX = 1;
+ int TX = 1;
+
+ emit(instr | (t.code() & 0x1F) * B21 | (a.code() & 0x1F) * B16 |
+ (b.code() & 0x1F) * B11 | AX * B2 | BX * B1 | TX);
+ }
+
+ PPC_XX3_OPCODE_LIST(DECLARE_PPC_XX3_INSTRUCTIONS)
+#undef DECLARE_PPC_XX3_INSTRUCTIONS
+
+#define DECLARE_PPC_VX_INSTRUCTIONS_A_FORM(name, instr_name, instr_value) \
+ inline void name(const Simd128Register rt, const Simd128Register rb, \
+ const Operand& imm) { \
+ vx_form(instr_name, rt, rb, imm); \
+ }
+#define DECLARE_PPC_VX_INSTRUCTIONS_B_FORM(name, instr_name, instr_value) \
+ inline void name(const Simd128Register rt, const Simd128Register ra, \
+ const Simd128Register rb) { \
+ vx_form(instr_name, rt, ra, rb); \
+ }
+#define DECLARE_PPC_VX_INSTRUCTIONS_C_FORM(name, instr_name, instr_value) \
+ inline void name(const Simd128Register rt, const Simd128Register rb) { \
+ vx_form(instr_name, rt, rb); \
+ }
+
+ inline void vx_form(Instr instr, Simd128Register rt, Simd128Register rb,
+ const Operand& imm) {
+ emit(instr | rt.code() * B21 | imm.immediate() * B16 | rb.code() * B11);
+ }
+ inline void vx_form(Instr instr, Simd128Register rt, Simd128Register ra,
+ Simd128Register rb) {
+ emit(instr | rt.code() * B21 | ra.code() * B16 | rb.code() * B11);
+ }
+ inline void vx_form(Instr instr, Simd128Register rt, Simd128Register rb) {
+ emit(instr | rt.code() * B21 | rb.code() * B11);
+ }
+
+ PPC_VX_OPCODE_A_FORM_LIST(DECLARE_PPC_VX_INSTRUCTIONS_A_FORM)
+ PPC_VX_OPCODE_B_FORM_LIST(DECLARE_PPC_VX_INSTRUCTIONS_B_FORM)
+ PPC_VX_OPCODE_C_FORM_LIST(DECLARE_PPC_VX_INSTRUCTIONS_C_FORM)
+#undef DECLARE_PPC_VX_INSTRUCTIONS_A_FORM
+#undef DECLARE_PPC_VX_INSTRUCTIONS_B_FORM
+#undef DECLARE_PPC_VX_INSTRUCTIONS_C_FORM
+
+#define DECLARE_PPC_VA_INSTRUCTIONS_A_FORM(name, instr_name, instr_value) \
+ inline void name(const Simd128Register rt, const Simd128Register ra, \
+ const Simd128Register rb, const Simd128Register rc) { \
+ va_form(instr_name, rt, ra, rb, rc); \
+ }
+
+ inline void va_form(Instr instr, Simd128Register rt, Simd128Register ra,
+ Simd128Register rb, Simd128Register rc) {
+ emit(instr | rt.code() * B21 | ra.code() * B16 | rb.code() * B11 |
+ rc.code() * B6);
+ }
+
+ PPC_VA_OPCODE_A_FORM_LIST(DECLARE_PPC_VA_INSTRUCTIONS_A_FORM)
+#undef DECLARE_PPC_VA_INSTRUCTIONS_A_FORM
+
+#define DECLARE_PPC_VC_INSTRUCTIONS(name, instr_name, instr_value) \
+ inline void name(const Simd128Register rt, const Simd128Register ra, \
+ const Simd128Register rb, const RCBit rc = LeaveRC) { \
+ vc_form(instr_name, rt, ra, rb, rc); \
+ }
+
+ inline void vc_form(Instr instr, Simd128Register rt, Simd128Register ra,
+ Simd128Register rb, int rc) {
+ emit(instr | rt.code() * B21 | ra.code() * B16 | rb.code() * B11 |
+ rc * B10);
+ }
+
+ PPC_VC_OPCODE_LIST(DECLARE_PPC_VC_INSTRUCTIONS)
+#undef DECLARE_PPC_VC_INSTRUCTIONS
+
+ RegList* GetScratchRegisterList() { return &scratch_register_list_; }
+ // ---------------------------------------------------------------------------
+ // Code generation
+
+ // Insert the smallest number of nop instructions
+ // possible to align the pc offset to a multiple
+ // of m. m must be a power of 2 (>= 4).
+ void Align(int m);
+ // Insert the smallest number of zero bytes possible to align the pc offset
+ // to a mulitple of m. m must be a power of 2 (>= 2).
+ void DataAlign(int m);
+ // Aligns code to something that's optimal for a jump target for the platform.
+ void CodeTargetAlign();
+
+ // Branch instructions
+ void bclr(BOfield bo, int condition_bit, LKBit lk);
+ void blr();
+ void bc(int branch_offset, BOfield bo, int condition_bit, LKBit lk = LeaveLK);
+ void b(int branch_offset, LKBit lk);
+
+ void bcctr(BOfield bo, int condition_bit, LKBit lk);
+ void bctr();
+ void bctrl();
+
+ // Convenience branch instructions using labels
+ void b(Label* L, LKBit lk = LeaveLK) { b(branch_offset(L), lk); }
+
+ inline CRegister cmpi_optimization(CRegister cr) {
+ // Check whether the branch is preceded by an optimizable cmpi against 0.
+ // The cmpi can be deleted if it is also preceded by an instruction that
+ // sets the register used by the compare and supports a dot form.
+ unsigned int sradi_mask = kOpcodeMask | kExt2OpcodeVariant2Mask;
+ unsigned int srawi_mask = kOpcodeMask | kExt2OpcodeMask;
+ int pos = pc_offset();
+ int cmpi_pos = pc_offset() - kInstrSize;
+
+ if (cmpi_pos > 0 && optimizable_cmpi_pos_ == cmpi_pos &&
+ cmpi_cr_.code() == cr.code() && last_bound_pos_ != pos) {
+ int xpos = cmpi_pos - kInstrSize;
+ int xinstr = instr_at(xpos);
+ int cmpi_ra = (instr_at(cmpi_pos) & 0x1f0000) >> 16;
+ // ra is at the same bit position for the three cases below.
+ int ra = (xinstr & 0x1f0000) >> 16;
+ if (cmpi_ra == ra) {
+ if ((xinstr & sradi_mask) == (EXT2 | SRADIX)) {
+ cr = cr0;
+ instr_at_put(xpos, xinstr | SetRC);
+ pc_ -= kInstrSize;
+ } else if ((xinstr & srawi_mask) == (EXT2 | SRAWIX)) {
+ cr = cr0;
+ instr_at_put(xpos, xinstr | SetRC);
+ pc_ -= kInstrSize;
+ } else if ((xinstr & kOpcodeMask) == ANDIx) {
+ cr = cr0;
+ pc_ -= kInstrSize;
+ // nothing to do here since andi. records.
+ }
+ // didn't match one of the above, must keep cmpwi.
+ }
+ }
+ return cr;
+ }
+
+ void bc_short(Condition cond, Label* L, CRegister cr = cr7,
+ LKBit lk = LeaveLK) {
+ DCHECK(cond != al);
+ DCHECK(cr.code() >= 0 && cr.code() <= 7);
+
+ cr = cmpi_optimization(cr);
+
+ int b_offset = branch_offset(L);
+
+ switch (cond) {
+ case eq:
+ bc(b_offset, BT, encode_crbit(cr, CR_EQ), lk);
+ break;
+ case ne:
+ bc(b_offset, BF, encode_crbit(cr, CR_EQ), lk);
+ break;
+ case gt:
+ bc(b_offset, BT, encode_crbit(cr, CR_GT), lk);
+ break;
+ case le:
+ bc(b_offset, BF, encode_crbit(cr, CR_GT), lk);
+ break;
+ case lt:
+ bc(b_offset, BT, encode_crbit(cr, CR_LT), lk);
+ break;
+ case ge:
+ bc(b_offset, BF, encode_crbit(cr, CR_LT), lk);
+ break;
+ case unordered:
+ bc(b_offset, BT, encode_crbit(cr, CR_FU), lk);
+ break;
+ case ordered:
+ bc(b_offset, BF, encode_crbit(cr, CR_FU), lk);
+ break;
+ case overflow:
+ bc(b_offset, BT, encode_crbit(cr, CR_SO), lk);
+ break;
+ case nooverflow:
+ bc(b_offset, BF, encode_crbit(cr, CR_SO), lk);
+ break;
+ default:
+ UNIMPLEMENTED();
+ }
+ }
+
+ void bclr(Condition cond, CRegister cr = cr7, LKBit lk = LeaveLK) {
+ DCHECK(cond != al);
+ DCHECK(cr.code() >= 0 && cr.code() <= 7);
+
+ cr = cmpi_optimization(cr);
+
+ switch (cond) {
+ case eq:
+ bclr(BT, encode_crbit(cr, CR_EQ), lk);
+ break;
+ case ne:
+ bclr(BF, encode_crbit(cr, CR_EQ), lk);
+ break;
+ case gt:
+ bclr(BT, encode_crbit(cr, CR_GT), lk);
+ break;
+ case le:
+ bclr(BF, encode_crbit(cr, CR_GT), lk);
+ break;
+ case lt:
+ bclr(BT, encode_crbit(cr, CR_LT), lk);
+ break;
+ case ge:
+ bclr(BF, encode_crbit(cr, CR_LT), lk);
+ break;
+ case unordered:
+ bclr(BT, encode_crbit(cr, CR_FU), lk);
+ break;
+ case ordered:
+ bclr(BF, encode_crbit(cr, CR_FU), lk);
+ break;
+ case overflow:
+ bclr(BT, encode_crbit(cr, CR_SO), lk);
+ break;
+ case nooverflow:
+ bclr(BF, encode_crbit(cr, CR_SO), lk);
+ break;
+ default:
+ UNIMPLEMENTED();
+ }
+ }
+
+ void isel(Register rt, Register ra, Register rb, int cb);
+ void isel(Condition cond, Register rt, Register ra, Register rb,
+ CRegister cr = cr7) {
+ DCHECK(cond != al);
+ DCHECK(cr.code() >= 0 && cr.code() <= 7);
+
+ cr = cmpi_optimization(cr);
+
+ switch (cond) {
+ case eq:
+ isel(rt, ra, rb, encode_crbit(cr, CR_EQ));
+ break;
+ case ne:
+ isel(rt, rb, ra, encode_crbit(cr, CR_EQ));
+ break;
+ case gt:
+ isel(rt, ra, rb, encode_crbit(cr, CR_GT));
+ break;
+ case le:
+ isel(rt, rb, ra, encode_crbit(cr, CR_GT));
+ break;
+ case lt:
+ isel(rt, ra, rb, encode_crbit(cr, CR_LT));
+ break;
+ case ge:
+ isel(rt, rb, ra, encode_crbit(cr, CR_LT));
+ break;
+ case unordered:
+ isel(rt, ra, rb, encode_crbit(cr, CR_FU));
+ break;
+ case ordered:
+ isel(rt, rb, ra, encode_crbit(cr, CR_FU));
+ break;
+ case overflow:
+ isel(rt, ra, rb, encode_crbit(cr, CR_SO));
+ break;
+ case nooverflow:
+ isel(rt, rb, ra, encode_crbit(cr, CR_SO));
+ break;
+ default:
+ UNIMPLEMENTED();
+ }
+ }
+
+ void b(Condition cond, Label* L, CRegister cr = cr7, LKBit lk = LeaveLK) {
+ if (cond == al) {
+ b(L, lk);
+ return;
+ }
+
+ if ((L->is_bound() && is_near(L, cond)) || !is_trampoline_emitted()) {
+ bc_short(cond, L, cr, lk);
+ return;
+ }
+
+ Label skip;
+ Condition neg_cond = NegateCondition(cond);
+ bc_short(neg_cond, &skip, cr);
+ b(L, lk);
+ bind(&skip);
+ }
+
+ void bne(Label* L, CRegister cr = cr7, LKBit lk = LeaveLK) {
+ b(ne, L, cr, lk);
+ }
+ void beq(Label* L, CRegister cr = cr7, LKBit lk = LeaveLK) {
+ b(eq, L, cr, lk);
+ }
+ void blt(Label* L, CRegister cr = cr7, LKBit lk = LeaveLK) {
+ b(lt, L, cr, lk);
+ }
+ void bge(Label* L, CRegister cr = cr7, LKBit lk = LeaveLK) {
+ b(ge, L, cr, lk);
+ }
+ void ble(Label* L, CRegister cr = cr7, LKBit lk = LeaveLK) {
+ b(le, L, cr, lk);
+ }
+ void bgt(Label* L, CRegister cr = cr7, LKBit lk = LeaveLK) {
+ b(gt, L, cr, lk);
+ }
+ void bunordered(Label* L, CRegister cr = cr7, LKBit lk = LeaveLK) {
+ b(unordered, L, cr, lk);
+ }
+ void bordered(Label* L, CRegister cr = cr7, LKBit lk = LeaveLK) {
+ b(ordered, L, cr, lk);
+ }
+ void boverflow(Label* L, CRegister cr = cr0, LKBit lk = LeaveLK) {
+ b(overflow, L, cr, lk);
+ }
+ void bnooverflow(Label* L, CRegister cr = cr0, LKBit lk = LeaveLK) {
+ b(nooverflow, L, cr, lk);
+ }
+
+ // Decrement CTR; branch if CTR != 0
+ void bdnz(Label* L, LKBit lk = LeaveLK) {
+ bc(branch_offset(L), DCBNZ, 0, lk);
+ }
+
+ // Data-processing instructions
+
+ void sub(Register dst, Register src1, Register src2, OEBit s = LeaveOE,
+ RCBit r = LeaveRC);
+
+ void subc(Register dst, Register src1, Register src2, OEBit s = LeaveOE,
+ RCBit r = LeaveRC);
+ void sube(Register dst, Register src1, Register src2, OEBit s = LeaveOE,
+ RCBit r = LeaveRC);
+
+ void subfic(Register dst, Register src, const Operand& imm);
+
+ void add(Register dst, Register src1, Register src2, OEBit s = LeaveOE,
+ RCBit r = LeaveRC);
+
+ void addc(Register dst, Register src1, Register src2, OEBit o = LeaveOE,
+ RCBit r = LeaveRC);
+ void adde(Register dst, Register src1, Register src2, OEBit o = LeaveOE,
+ RCBit r = LeaveRC);
+ void addze(Register dst, Register src1, OEBit o = LeaveOE, RCBit r = LeaveRC);
+
+ void mullw(Register dst, Register src1, Register src2, OEBit o = LeaveOE,
+ RCBit r = LeaveRC);
+
+ void mulhw(Register dst, Register src1, Register src2, RCBit r = LeaveRC);
+ void mulhwu(Register dst, Register src1, Register src2, RCBit r = LeaveRC);
+
+ void divw(Register dst, Register src1, Register src2, OEBit o = LeaveOE,
+ RCBit r = LeaveRC);
+ void divwu(Register dst, Register src1, Register src2, OEBit o = LeaveOE,
+ RCBit r = LeaveRC);
+
+ void addi(Register dst, Register src, const Operand& imm);
+ void addis(Register dst, Register src, const Operand& imm);
+ void addic(Register dst, Register src, const Operand& imm);
+
+ void andi(Register ra, Register rs, const Operand& imm);
+ void andis(Register ra, Register rs, const Operand& imm);
+ void ori(Register dst, Register src, const Operand& imm);
+ void oris(Register dst, Register src, const Operand& imm);
+ void xori(Register dst, Register src, const Operand& imm);
+ void xoris(Register ra, Register rs, const Operand& imm);
+ void cmpi(Register src1, const Operand& src2, CRegister cr = cr7);
+ void cmpli(Register src1, const Operand& src2, CRegister cr = cr7);
+ void cmpwi(Register src1, const Operand& src2, CRegister cr = cr7);
+ void cmplwi(Register src1, const Operand& src2, CRegister cr = cr7);
+ void li(Register dst, const Operand& src);
+ void lis(Register dst, const Operand& imm);
+ void mr(Register dst, Register src);
+
+ void lbz(Register dst, const MemOperand& src);
+ void lhz(Register dst, const MemOperand& src);
+ void lha(Register dst, const MemOperand& src);
+ void lwz(Register dst, const MemOperand& src);
+ void lwzu(Register dst, const MemOperand& src);
+ void lwa(Register dst, const MemOperand& src);
+ void stb(Register dst, const MemOperand& src);
+ void sth(Register dst, const MemOperand& src);
+ void stw(Register dst, const MemOperand& src);
+ void stwu(Register dst, const MemOperand& src);
+ void neg(Register rt, Register ra, OEBit o = LeaveOE, RCBit c = LeaveRC);
+
+#if V8_TARGET_ARCH_PPC64
+ void ld(Register rd, const MemOperand& src);
+ void ldu(Register rd, const MemOperand& src);
+ void std(Register rs, const MemOperand& src);
+ void stdu(Register rs, const MemOperand& src);
+ void rldic(Register dst, Register src, int sh, int mb, RCBit r = LeaveRC);
+ void rldicl(Register dst, Register src, int sh, int mb, RCBit r = LeaveRC);
+ void rldcl(Register ra, Register rs, Register rb, int mb, RCBit r = LeaveRC);
+ void rldicr(Register dst, Register src, int sh, int me, RCBit r = LeaveRC);
+ void rldimi(Register dst, Register src, int sh, int mb, RCBit r = LeaveRC);
+ void sldi(Register dst, Register src, const Operand& val, RCBit rc = LeaveRC);
+ void srdi(Register dst, Register src, const Operand& val, RCBit rc = LeaveRC);
+ void clrrdi(Register dst, Register src, const Operand& val,
+ RCBit rc = LeaveRC);
+ void clrldi(Register dst, Register src, const Operand& val,
+ RCBit rc = LeaveRC);
+ void sradi(Register ra, Register rs, int sh, RCBit r = LeaveRC);
+ void rotld(Register ra, Register rs, Register rb, RCBit r = LeaveRC);
+ void rotldi(Register ra, Register rs, int sh, RCBit r = LeaveRC);
+ void rotrdi(Register ra, Register rs, int sh, RCBit r = LeaveRC);
+ void mulld(Register dst, Register src1, Register src2, OEBit o = LeaveOE,
+ RCBit r = LeaveRC);
+ void divd(Register dst, Register src1, Register src2, OEBit o = LeaveOE,
+ RCBit r = LeaveRC);
+ void divdu(Register dst, Register src1, Register src2, OEBit o = LeaveOE,
+ RCBit r = LeaveRC);
+#endif
+
+ void rlwinm(Register ra, Register rs, int sh, int mb, int me,
+ RCBit rc = LeaveRC);
+ void rlwimi(Register ra, Register rs, int sh, int mb, int me,
+ RCBit rc = LeaveRC);
+ void rlwnm(Register ra, Register rs, Register rb, int mb, int me,
+ RCBit rc = LeaveRC);
+ void slwi(Register dst, Register src, const Operand& val, RCBit rc = LeaveRC);
+ void srwi(Register dst, Register src, const Operand& val, RCBit rc = LeaveRC);
+ void clrrwi(Register dst, Register src, const Operand& val,
+ RCBit rc = LeaveRC);
+ void clrlwi(Register dst, Register src, const Operand& val,
+ RCBit rc = LeaveRC);
+ void rotlw(Register ra, Register rs, Register rb, RCBit r = LeaveRC);
+ void rotlwi(Register ra, Register rs, int sh, RCBit r = LeaveRC);
+ void rotrwi(Register ra, Register rs, int sh, RCBit r = LeaveRC);
+
+ void subi(Register dst, Register src1, const Operand& src2);
+
+ void mov(Register dst, const Operand& src);
+ void bitwise_mov(Register dst, intptr_t value);
+ void bitwise_mov32(Register dst, int32_t value);
+ void bitwise_add32(Register dst, Register src, int32_t value);
+
+ // Load the position of the label relative to the generated code object
+ // pointer in a register.
+ void mov_label_offset(Register dst, Label* label);
+
+ // dst = base + label position + delta
+ void add_label_offset(Register dst, Register base, Label* label,
+ int delta = 0);
+
+ // Load the address of the label in a register and associate with an
+ // internal reference relocation.
+ void mov_label_addr(Register dst, Label* label);
+
+ // Emit the address of the label (i.e. a jump table entry) and associate with
+ // an internal reference relocation.
+ void emit_label_addr(Label* label);
+
+ // Multiply instructions
+ void mul(Register dst, Register src1, Register src2, OEBit s = LeaveOE,
+ RCBit r = LeaveRC);
+
+ // Miscellaneous arithmetic instructions
+
+ // Special register access
+ void crxor(int bt, int ba, int bb);
+ void crclr(int bt) { crxor(bt, bt, bt); }
+ void creqv(int bt, int ba, int bb);
+ void crset(int bt) { creqv(bt, bt, bt); }
+ void mflr(Register dst);
+ void mtlr(Register src);
+ void mtctr(Register src);
+ void mtxer(Register src);
+ void mcrfs(CRegister cr, FPSCRBit bit);
+ void mfcr(Register dst);
+#if V8_TARGET_ARCH_PPC64
+ void mffprd(Register dst, DoubleRegister src);
+ void mffprwz(Register dst, DoubleRegister src);
+ void mtfprd(DoubleRegister dst, Register src);
+ void mtfprwz(DoubleRegister dst, Register src);
+ void mtfprwa(DoubleRegister dst, Register src);
+#endif
+
+ // Exception-generating instructions and debugging support
+ void stop(Condition cond = al, int32_t code = kDefaultStopCode,
+ CRegister cr = cr7);
+
+ void bkpt(uint32_t imm16); // v5 and above
+
+ void dcbf(Register ra, Register rb);
+ void sync();
+ void lwsync();
+ void icbi(Register ra, Register rb);
+ void isync();
+
+ // Support for floating point
+ void lfd(const DoubleRegister frt, const MemOperand& src);
+ void lfdu(const DoubleRegister frt, const MemOperand& src);
+ void lfs(const DoubleRegister frt, const MemOperand& src);
+ void lfsu(const DoubleRegister frt, const MemOperand& src);
+ void stfd(const DoubleRegister frs, const MemOperand& src);
+ void stfdu(const DoubleRegister frs, const MemOperand& src);
+ void stfs(const DoubleRegister frs, const MemOperand& src);
+ void stfsu(const DoubleRegister frs, const MemOperand& src);
+
+ void fadd(const DoubleRegister frt, const DoubleRegister fra,
+ const DoubleRegister frb, RCBit rc = LeaveRC);
+ void fsub(const DoubleRegister frt, const DoubleRegister fra,
+ const DoubleRegister frb, RCBit rc = LeaveRC);
+ void fdiv(const DoubleRegister frt, const DoubleRegister fra,
+ const DoubleRegister frb, RCBit rc = LeaveRC);
+ void fmul(const DoubleRegister frt, const DoubleRegister fra,
+ const DoubleRegister frc, RCBit rc = LeaveRC);
+ void fcmpu(const DoubleRegister fra, const DoubleRegister frb,
+ CRegister cr = cr7);
+ void fmr(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc = LeaveRC);
+ void fctiwz(const DoubleRegister frt, const DoubleRegister frb);
+ void fctiw(const DoubleRegister frt, const DoubleRegister frb);
+ void fctiwuz(const DoubleRegister frt, const DoubleRegister frb);
+ void frin(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc = LeaveRC);
+ void friz(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc = LeaveRC);
+ void frip(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc = LeaveRC);
+ void frim(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc = LeaveRC);
+ void frsp(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc = LeaveRC);
+ void fcfid(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc = LeaveRC);
+ void fcfidu(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc = LeaveRC);
+ void fcfidus(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc = LeaveRC);
+ void fcfids(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc = LeaveRC);
+ void fctid(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc = LeaveRC);
+ void fctidz(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc = LeaveRC);
+ void fctidu(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc = LeaveRC);
+ void fctiduz(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc = LeaveRC);
+ void fsel(const DoubleRegister frt, const DoubleRegister fra,
+ const DoubleRegister frc, const DoubleRegister frb,
+ RCBit rc = LeaveRC);
+ void fneg(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc = LeaveRC);
+ void mtfsb0(FPSCRBit bit, RCBit rc = LeaveRC);
+ void mtfsb1(FPSCRBit bit, RCBit rc = LeaveRC);
+ void mtfsfi(int bf, int immediate, RCBit rc = LeaveRC);
+ void mffs(const DoubleRegister frt, RCBit rc = LeaveRC);
+ void mtfsf(const DoubleRegister frb, bool L = 1, int FLM = 0, bool W = 0,
+ RCBit rc = LeaveRC);
+ void fsqrt(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc = LeaveRC);
+ void fabs(const DoubleRegister frt, const DoubleRegister frb,
+ RCBit rc = LeaveRC);
+ void fmadd(const DoubleRegister frt, const DoubleRegister fra,
+ const DoubleRegister frc, const DoubleRegister frb,
+ RCBit rc = LeaveRC);
+ void fmsub(const DoubleRegister frt, const DoubleRegister fra,
+ const DoubleRegister frc, const DoubleRegister frb,
+ RCBit rc = LeaveRC);
+
+ // Vector instructions
+ void mfvsrd(const Register ra, const Simd128Register r);
+ void mfvsrwz(const Register ra, const Simd128Register r);
+ void mtvsrd(const Simd128Register rt, const Register ra);
+ void mtvsrdd(const Simd128Register rt, const Register ra, const Register rb);
+ void lxvd(const Simd128Register rt, const MemOperand& src);
+ void stxvd(const Simd128Register rt, const MemOperand& src);
+ void xxspltib(const Simd128Register rt, const Operand& imm);
+
+ // Pseudo instructions
+
+ // Different nop operations are used by the code generator to detect certain
+ // states of the generated code.
+ enum NopMarkerTypes {
+ NON_MARKING_NOP = 0,
+ GROUP_ENDING_NOP,
+ DEBUG_BREAK_NOP,
+ // IC markers.
+ PROPERTY_ACCESS_INLINED,
+ PROPERTY_ACCESS_INLINED_CONTEXT,
+ PROPERTY_ACCESS_INLINED_CONTEXT_DONT_DELETE,
+ // Helper values.
+ LAST_CODE_MARKER,
+ FIRST_IC_MARKER = PROPERTY_ACCESS_INLINED
+ };
+
+ void nop(int type = 0); // 0 is the default non-marking type.
+
+ void push(Register src) {
+#if V8_TARGET_ARCH_PPC64
+ stdu(src, MemOperand(sp, -kSystemPointerSize));
+#else
+ stwu(src, MemOperand(sp, -kSystemPointerSize));
+#endif
+ }
+
+ void pop(Register dst) {
+#if V8_TARGET_ARCH_PPC64
+ ld(dst, MemOperand(sp));
+#else
+ lwz(dst, MemOperand(sp));
+#endif
+ addi(sp, sp, Operand(kSystemPointerSize));
+ }
+
+ void pop() { addi(sp, sp, Operand(kSystemPointerSize)); }
+
+ // Jump unconditionally to given label.
+ void jmp(Label* L) { b(L); }
+
+ // Check the code size generated from label to here.
+ int SizeOfCodeGeneratedSince(Label* label) {
+ return pc_offset() - label->pos();
+ }
+
+ // Check the number of instructions generated from label to here.
+ int InstructionsGeneratedSince(Label* label) {
+ return SizeOfCodeGeneratedSince(label) / kInstrSize;
+ }
+
+ // Class for scoping postponing the trampoline pool generation.
+ class BlockTrampolinePoolScope {
+ public:
+ explicit BlockTrampolinePoolScope(Assembler* assem) : assem_(assem) {
+ assem_->StartBlockTrampolinePool();
+ }
+ ~BlockTrampolinePoolScope() { assem_->EndBlockTrampolinePool(); }
+
+ private:
+ Assembler* assem_;
+
+ DISALLOW_IMPLICIT_CONSTRUCTORS(BlockTrampolinePoolScope);
+ };
+
+ // Class for scoping disabling constant pool entry merging
+ class BlockConstantPoolEntrySharingScope {
+ public:
+ explicit BlockConstantPoolEntrySharingScope(Assembler* assem)
+ : assem_(assem) {
+ assem_->StartBlockConstantPoolEntrySharing();
+ }
+ ~BlockConstantPoolEntrySharingScope() {
+ assem_->EndBlockConstantPoolEntrySharing();
+ }
+
+ private:
+ Assembler* assem_;
+
+ DISALLOW_IMPLICIT_CONSTRUCTORS(BlockConstantPoolEntrySharingScope);
+ };
+
+ // Record a deoptimization reason that can be used by a log or cpu profiler.
+ // Use --trace-deopt to enable.
+ void RecordDeoptReason(DeoptimizeReason reason, SourcePosition position,
+ int id);
+
+ // Writes a single byte or word of data in the code stream. Used
+ // for inline tables, e.g., jump-tables.
+ void db(uint8_t data);
+ void dd(uint32_t data);
+ void dq(uint64_t data);
+ void dp(uintptr_t data);
+
+ // Read/patch instructions
+ Instr instr_at(int pos) {
+ return *reinterpret_cast<Instr*>(buffer_start_ + pos);
+ }
+ void instr_at_put(int pos, Instr instr) {
+ *reinterpret_cast<Instr*>(buffer_start_ + pos) = instr;
+ }
+ static Instr instr_at(Address pc) { return *reinterpret_cast<Instr*>(pc); }
+ static void instr_at_put(Address pc, Instr instr) {
+ *reinterpret_cast<Instr*>(pc) = instr;
+ }
+ static Condition GetCondition(Instr instr);
+
+ static bool IsLis(Instr instr);
+ static bool IsLi(Instr instr);
+ static bool IsAddic(Instr instr);
+ static bool IsOri(Instr instr);
+
+ static bool IsBranch(Instr instr);
+ static Register GetRA(Instr instr);
+ static Register GetRB(Instr instr);
+#if V8_TARGET_ARCH_PPC64
+ static bool Is64BitLoadIntoR12(Instr instr1, Instr instr2, Instr instr3,
+ Instr instr4, Instr instr5);
+#else
+ static bool Is32BitLoadIntoR12(Instr instr1, Instr instr2);
+#endif
+
+ static bool IsCmpRegister(Instr instr);
+ static bool IsCmpImmediate(Instr instr);
+ static bool IsRlwinm(Instr instr);
+ static bool IsAndi(Instr instr);
+#if V8_TARGET_ARCH_PPC64
+ static bool IsRldicl(Instr instr);
+#endif
+ static bool IsCrSet(Instr instr);
+ static Register GetCmpImmediateRegister(Instr instr);
+ static int GetCmpImmediateRawImmediate(Instr instr);
+ static bool IsNop(Instr instr, int type = NON_MARKING_NOP);
+
+ // Postpone the generation of the trampoline pool for the specified number of
+ // instructions.
+ void BlockTrampolinePoolFor(int instructions);
+ void CheckTrampolinePool();
+
+ // For mov. Return the number of actual instructions required to
+ // load the operand into a register. This can be anywhere from
+ // one (constant pool small section) to five instructions (full
+ // 64-bit sequence).
+ //
+ // The value returned is only valid as long as no entries are added to the
+ // constant pool between this call and the actual instruction being emitted.
+ int instructions_required_for_mov(Register dst, const Operand& src) const;
+
+ // Decide between using the constant pool vs. a mov immediate sequence.
+ bool use_constant_pool_for_mov(Register dst, const Operand& src,
+ bool canOptimize) const;
+
+ // The code currently calls CheckBuffer() too often. This has the side
+ // effect of randomly growing the buffer in the middle of multi-instruction
+ // sequences.
+ //
+ // This function allows outside callers to check and grow the buffer
+ void EnsureSpaceFor(int space_needed);
+
+ int EmitConstantPool() { return constant_pool_builder_.Emit(this); }
+
+ bool ConstantPoolAccessIsInOverflow() const {
+ return constant_pool_builder_.NextAccess(ConstantPoolEntry::INTPTR) ==
+ ConstantPoolEntry::OVERFLOWED;
+ }
+
+ Label* ConstantPoolPosition() {
+ return constant_pool_builder_.EmittedPosition();
+ }
+
+ void EmitRelocations();
+
+ protected:
+ int buffer_space() const { return reloc_info_writer.pos() - pc_; }
+
+ // Decode instruction(s) at pos and return backchain to previous
+ // label reference or kEndOfChain.
+ int target_at(int pos);
+
+ // Patch instruction(s) at pos to target target_pos (e.g. branch)
+ void target_at_put(int pos, int target_pos, bool* is_branch = nullptr);
+
+ // Record reloc info for current pc_
+ void RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data = 0);
+ ConstantPoolEntry::Access ConstantPoolAddEntry(RelocInfo::Mode rmode,
+ intptr_t value) {
+ bool sharing_ok =
+ RelocInfo::IsNone(rmode) ||
+ (!options().record_reloc_info_for_serialization &&
+ RelocInfo::IsShareableRelocMode(rmode) &&
+ !is_constant_pool_entry_sharing_blocked() &&
+ // TODO(johnyan): make the following rmode shareable
+ !RelocInfo::IsWasmCall(rmode) && !RelocInfo::IsWasmStubCall(rmode));
+ return constant_pool_builder_.AddEntry(pc_offset(), value, sharing_ok);
+ }
+ ConstantPoolEntry::Access ConstantPoolAddEntry(Double value) {
+ return constant_pool_builder_.AddEntry(pc_offset(), value);
+ }
+
+ // Block the emission of the trampoline pool before pc_offset.
+ void BlockTrampolinePoolBefore(int pc_offset) {
+ if (no_trampoline_pool_before_ < pc_offset)
+ no_trampoline_pool_before_ = pc_offset;
+ }
+
+ void StartBlockTrampolinePool() { trampoline_pool_blocked_nesting_++; }
+ void EndBlockTrampolinePool() {
+ int count = --trampoline_pool_blocked_nesting_;
+ if (count == 0) CheckTrampolinePoolQuick();
+ }
+ bool is_trampoline_pool_blocked() const {
+ return trampoline_pool_blocked_nesting_ > 0;
+ }
+
+ void StartBlockConstantPoolEntrySharing() {
+ constant_pool_entry_sharing_blocked_nesting_++;
+ }
+ void EndBlockConstantPoolEntrySharing() {
+ constant_pool_entry_sharing_blocked_nesting_--;
+ }
+ bool is_constant_pool_entry_sharing_blocked() const {
+ return constant_pool_entry_sharing_blocked_nesting_ > 0;
+ }
+
+ bool has_exception() const { return internal_trampoline_exception_; }
+
+ bool is_trampoline_emitted() const { return trampoline_emitted_; }
+
+ // Code generation
+ // The relocation writer's position is at least kGap bytes below the end of
+ // the generated instructions. This is so that multi-instruction sequences do
+ // not have to check for overflow. The same is true for writes of large
+ // relocation info entries.
+ static constexpr int kGap = 32;
+ STATIC_ASSERT(AssemblerBase::kMinimalBufferSize >= 2 * kGap);
+
+ RelocInfoWriter reloc_info_writer;
+
+ private:
+ // Avoid overflows for displacements etc.
+ static const int kMaximalBufferSize = 512 * MB;
+
+ // Repeated checking whether the trampoline pool should be emitted is rather
+ // expensive. By default we only check again once a number of instructions
+ // has been generated.
+ int next_trampoline_check_; // pc offset of next buffer check.
+
+ // Emission of the trampoline pool may be blocked in some code sequences.
+ int trampoline_pool_blocked_nesting_; // Block emission if this is not zero.
+ int no_trampoline_pool_before_; // Block emission before this pc offset.
+
+ // Do not share constant pool entries.
+ int constant_pool_entry_sharing_blocked_nesting_;
+
+ // Relocation info generation
+ // Each relocation is encoded as a variable size value
+ static constexpr int kMaxRelocSize = RelocInfoWriter::kMaxSize;
+ std::vector<DeferredRelocInfo> relocations_;
+
+ // Scratch registers available for use by the Assembler.
+ RegList scratch_register_list_;
+
+ // The bound position, before this we cannot do instruction elimination.
+ int last_bound_pos_;
+ // Optimizable cmpi information.
+ int optimizable_cmpi_pos_;
+ CRegister cmpi_cr_ = CRegister::no_reg();
+
+ ConstantPoolBuilder constant_pool_builder_;
+
+ void CheckBuffer() {
+ if (buffer_space() <= kGap) {
+ GrowBuffer();
+ }
+ }
+
+ void GrowBuffer(int needed = 0);
+ // Code emission
+ void emit(Instr x) {
+ CheckBuffer();
+ *reinterpret_cast<Instr*>(pc_) = x;
+ pc_ += kInstrSize;
+ CheckTrampolinePoolQuick();
+ }
+ void TrackBranch() {
+ DCHECK(!trampoline_emitted_);
+ int count = tracked_branch_count_++;
+ if (count == 0) {
+ // We leave space (kMaxBlockTrampolineSectionSize)
+ // for BlockTrampolinePoolScope buffer.
+ next_trampoline_check_ =
+ pc_offset() + kMaxCondBranchReach - kMaxBlockTrampolineSectionSize;
+ } else {
+ next_trampoline_check_ -= kTrampolineSlotsSize;
+ }
+ }
+
+ inline void UntrackBranch();
+ void CheckTrampolinePoolQuick() {
+ if (pc_offset() >= next_trampoline_check_) {
+ CheckTrampolinePool();
+ }
+ }
+
+ // Instruction generation
+ void a_form(Instr instr, DoubleRegister frt, DoubleRegister fra,
+ DoubleRegister frb, RCBit r);
+ void d_form(Instr instr, Register rt, Register ra, const intptr_t val,
+ bool signed_disp);
+ void xo_form(Instr instr, Register rt, Register ra, Register rb, OEBit o,
+ RCBit r);
+ void md_form(Instr instr, Register ra, Register rs, int shift, int maskbit,
+ RCBit r);
+ void mds_form(Instr instr, Register ra, Register rs, Register rb, int maskbit,
+ RCBit r);
+
+ // Labels
+ void print(Label* L);
+ int max_reach_from(int pos);
+ void bind_to(Label* L, int pos);
+ void next(Label* L);
+
+ class Trampoline {
+ public:
+ Trampoline() {
+ next_slot_ = 0;
+ free_slot_count_ = 0;
+ }
+ Trampoline(int start, int slot_count) {
+ next_slot_ = start;
+ free_slot_count_ = slot_count;
+ }
+ int take_slot() {
+ int trampoline_slot = kInvalidSlotPos;
+ if (free_slot_count_ <= 0) {
+ // We have run out of space on trampolines.
+ // Make sure we fail in debug mode, so we become aware of each case
+ // when this happens.
+ DCHECK(0);
+ // Internal exception will be caught.
+ } else {
+ trampoline_slot = next_slot_;
+ free_slot_count_--;
+ next_slot_ += kTrampolineSlotsSize;
+ }
+ return trampoline_slot;
+ }
+
+ private:
+ int next_slot_;
+ int free_slot_count_;
+ };
+
+ int32_t get_trampoline_entry();
+ int tracked_branch_count_;
+ // If trampoline is emitted, generated code is becoming large. As
+ // this is already a slow case which can possibly break our code
+ // generation for the extreme case, we use this information to
+ // trigger different mode of branch instruction generation, where we
+ // no longer use a single branch instruction.
+ bool trampoline_emitted_;
+ static constexpr int kTrampolineSlotsSize = kInstrSize;
+ static constexpr int kMaxCondBranchReach = (1 << (16 - 1)) - 1;
+ static constexpr int kMaxBlockTrampolineSectionSize = 64 * kInstrSize;
+ static constexpr int kInvalidSlotPos = -1;
+
+ Trampoline trampoline_;
+ bool internal_trampoline_exception_;
+
+ void AllocateAndInstallRequestedHeapObjects(Isolate* isolate);
+
+ int WriteCodeComments();
+
+ friend class RegExpMacroAssemblerPPC;
+ friend class RelocInfo;
+ friend class BlockTrampolinePoolScope;
+ friend class EnsureSpace;
+ friend class UseScratchRegisterScope;
+};
+
+class EnsureSpace {
+ public:
+ explicit EnsureSpace(Assembler* assembler) { assembler->CheckBuffer(); }
+};
+
+class PatchingAssembler : public Assembler {
+ public:
+ PatchingAssembler(const AssemblerOptions& options, byte* address,
+ int instructions);
+ ~PatchingAssembler();
+};
+
+class V8_EXPORT_PRIVATE UseScratchRegisterScope {
+ public:
+ explicit UseScratchRegisterScope(Assembler* assembler);
+ ~UseScratchRegisterScope();
+
+ Register Acquire();
+
+ // Check if we have registers available to acquire.
+ bool CanAcquire() const { return *assembler_->GetScratchRegisterList() != 0; }
+
+ private:
+ friend class Assembler;
+ friend class TurboAssembler;
+
+ Assembler* assembler_;
+ RegList old_available_;
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_PPC_ASSEMBLER_PPC_H_
diff --git a/src/codegen/ppc/constants-ppc.cc b/src/codegen/ppc/constants-ppc.cc
new file mode 100644
index 0000000..ee2e19a
--- /dev/null
+++ b/src/codegen/ppc/constants-ppc.cc
@@ -0,0 +1,49 @@
+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#if V8_TARGET_ARCH_PPC || V8_TARGET_ARCH_PPC64
+
+#include "src/codegen/ppc/constants-ppc.h"
+
+namespace v8 {
+namespace internal {
+
+// These register names are defined in a way to match the native disassembler
+// formatting. See for example the command "objdump -d <binary file>".
+const char* Registers::names_[kNumRegisters] = {
+ "r0", "sp", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",
+ "r11", "ip", "r13", "r14", "r15", "r16", "r17", "r18", "r19", "r20", "r21",
+ "r22", "r23", "r24", "r25", "r26", "r27", "r28", "r29", "r30", "fp"};
+
+const char* DoubleRegisters::names_[kNumDoubleRegisters] = {
+ "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9", "d10",
+ "d11", "d12", "d13", "d14", "d15", "d16", "d17", "d18", "d19", "d20", "d21",
+ "d22", "d23", "d24", "d25", "d26", "d27", "d28", "d29", "d30", "d31"};
+
+int DoubleRegisters::Number(const char* name) {
+ for (int i = 0; i < kNumDoubleRegisters; i++) {
+ if (strcmp(names_[i], name) == 0) {
+ return i;
+ }
+ }
+
+ // No register with the requested name found.
+ return kNoRegister;
+}
+
+int Registers::Number(const char* name) {
+ // Look through the canonical names.
+ for (int i = 0; i < kNumRegisters; i++) {
+ if (strcmp(names_[i], name) == 0) {
+ return i;
+ }
+ }
+
+ // No register with the requested name found.
+ return kNoRegister;
+}
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_PPC || V8_TARGET_ARCH_PPC64
diff --git a/src/codegen/ppc/constants-ppc.h b/src/codegen/ppc/constants-ppc.h
new file mode 100644
index 0000000..f71d1be
--- /dev/null
+++ b/src/codegen/ppc/constants-ppc.h
@@ -0,0 +1,3047 @@
+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_PPC_CONSTANTS_PPC_H_
+#define V8_CODEGEN_PPC_CONSTANTS_PPC_H_
+
+#include <stdint.h>
+
+#include "src/base/logging.h"
+#include "src/base/macros.h"
+#include "src/common/globals.h"
+
+// UNIMPLEMENTED_ macro for PPC.
+#ifdef DEBUG
+#define UNIMPLEMENTED_PPC() \
+ v8::internal::PrintF("%s, \tline %d: \tfunction %s not implemented. \n", \
+ __FILE__, __LINE__, __func__)
+#else
+#define UNIMPLEMENTED_PPC()
+#endif
+
+#if (V8_HOST_ARCH_PPC || V8_HOST_ARCH_PPC64) && \
+ (V8_OS_AIX || (V8_TARGET_ARCH_PPC64 && V8_TARGET_BIG_ENDIAN && \
+ (!defined(_CALL_ELF) || _CALL_ELF == 1)))
+#define ABI_USES_FUNCTION_DESCRIPTORS 1
+#else
+#define ABI_USES_FUNCTION_DESCRIPTORS 0
+#endif
+
+#if !(V8_HOST_ARCH_PPC || V8_HOST_ARCH_PPC64) || V8_OS_AIX || \
+ V8_TARGET_ARCH_PPC64
+#define ABI_PASSES_HANDLES_IN_REGS 1
+#else
+#define ABI_PASSES_HANDLES_IN_REGS 0
+#endif
+
+#if !(V8_HOST_ARCH_PPC || V8_HOST_ARCH_PPC64) || !V8_TARGET_ARCH_PPC64 || \
+ V8_TARGET_LITTLE_ENDIAN || (defined(_CALL_ELF) && _CALL_ELF == 2)
+#define ABI_RETURNS_OBJECT_PAIRS_IN_REGS 1
+#else
+#define ABI_RETURNS_OBJECT_PAIRS_IN_REGS 0
+#endif
+
+#if !(V8_HOST_ARCH_PPC || V8_HOST_ARCH_PPC64) || \
+ (V8_TARGET_ARCH_PPC64 && \
+ (V8_TARGET_LITTLE_ENDIAN || (defined(_CALL_ELF) && _CALL_ELF == 2)))
+#define ABI_CALL_VIA_IP 1
+#else
+#define ABI_CALL_VIA_IP 0
+#endif
+
+#if !(V8_HOST_ARCH_PPC || V8_HOST_ARCH_PPC64) || V8_OS_AIX || \
+ V8_TARGET_ARCH_PPC64
+#define ABI_TOC_REGISTER 2
+#else
+#define ABI_TOC_REGISTER 13
+#endif
+namespace v8 {
+namespace internal {
+
+// TODO(sigurds): Change this value once we use relative jumps.
+constexpr size_t kMaxPCRelativeCodeRangeInMB = 0;
+
+// Used to encode a boolean value when emitting 32 bit
+// opcodes which will indicate the presence of function descriptors
+constexpr int kHasFunctionDescriptorBitShift = 9;
+constexpr int kHasFunctionDescriptorBitMask = 1
+ << kHasFunctionDescriptorBitShift;
+
+// Number of registers
+const int kNumRegisters = 32;
+
+// FP support.
+const int kNumDoubleRegisters = 32;
+
+const int kNoRegister = -1;
+
+// Used in embedded constant pool builder - max reach in bits for
+// various load instructions (one less due to unsigned)
+const int kLoadPtrMaxReachBits = 15;
+const int kLoadDoubleMaxReachBits = 15;
+
+// Actual value of root register is offset from the root array's start
+// to take advantage of negative displacement values.
+// TODO(sigurds): Choose best value.
+constexpr int kRootRegisterBias = 128;
+
+// sign-extend the least significant 16-bits of value <imm>
+#define SIGN_EXT_IMM16(imm) ((static_cast<int>(imm) << 16) >> 16)
+
+// sign-extend the least significant 22-bits of value <imm>
+#define SIGN_EXT_IMM22(imm) ((static_cast<int>(imm) << 10) >> 10)
+
+// sign-extend the least significant 26-bits of value <imm>
+#define SIGN_EXT_IMM26(imm) ((static_cast<int>(imm) << 6) >> 6)
+
+// -----------------------------------------------------------------------------
+// Conditions.
+
+// Defines constants and accessor classes to assemble, disassemble and
+// simulate PPC instructions.
+//
+// Section references in the code refer to the "PowerPC Microprocessor
+// Family: The Programmer.s Reference Guide" from 10/95
+// https://www-01.ibm.com/chips/techlib/techlib.nsf/techdocs/852569B20050FF778525699600741775/$file/prg.pdf
+//
+
+// Constants for specific fields are defined in their respective named enums.
+// General constants are in an anonymous enum in class Instr.
+enum Condition {
+ kNoCondition = -1,
+ eq = 0, // Equal.
+ ne = 1, // Not equal.
+ ge = 2, // Greater or equal.
+ lt = 3, // Less than.
+ gt = 4, // Greater than.
+ le = 5, // Less then or equal
+ unordered = 6, // Floating-point unordered
+ ordered = 7,
+ overflow = 8, // Summary overflow
+ nooverflow = 9,
+ al = 10 // Always.
+};
+
+inline Condition NegateCondition(Condition cond) {
+ DCHECK(cond != al);
+ return static_cast<Condition>(cond ^ ne);
+}
+
+// -----------------------------------------------------------------------------
+// Instructions encoding.
+
+// Instr is merely used by the Assembler to distinguish 32bit integers
+// representing instructions from usual 32 bit values.
+// Instruction objects are pointers to 32bit values, and provide methods to
+// access the various ISA fields.
+using Instr = uint32_t;
+
+#define PPC_XX3_OPCODE_LIST(V) \
+ /* VSX Scalar Add Double-Precision */ \
+ V(xsadddp, XSADDDP, 0xF0000100) \
+ /* VSX Scalar Add Single-Precision */ \
+ V(xsaddsp, XSADDSP, 0xF0000000) \
+ /* VSX Scalar Compare Ordered Double-Precision */ \
+ V(xscmpodp, XSCMPODP, 0xF0000158) \
+ /* VSX Scalar Compare Unordered Double-Precision */ \
+ V(xscmpudp, XSCMPUDP, 0xF0000118) \
+ /* VSX Scalar Copy Sign Double-Precision */ \
+ V(xscpsgndp, XSCPSGNDP, 0xF0000580) \
+ /* VSX Scalar Divide Double-Precision */ \
+ V(xsdivdp, XSDIVDP, 0xF00001C0) \
+ /* VSX Scalar Divide Single-Precision */ \
+ V(xsdivsp, XSDIVSP, 0xF00000C0) \
+ /* VSX Scalar Multiply-Add Type-A Double-Precision */ \
+ V(xsmaddadp, XSMADDADP, 0xF0000108) \
+ /* VSX Scalar Multiply-Add Type-A Single-Precision */ \
+ V(xsmaddasp, XSMADDASP, 0xF0000008) \
+ /* VSX Scalar Multiply-Add Type-M Double-Precision */ \
+ V(xsmaddmdp, XSMADDMDP, 0xF0000148) \
+ /* VSX Scalar Multiply-Add Type-M Single-Precision */ \
+ V(xsmaddmsp, XSMADDMSP, 0xF0000048) \
+ /* VSX Scalar Maximum Double-Precision */ \
+ V(xsmaxdp, XSMAXDP, 0xF0000500) \
+ /* VSX Scalar Minimum Double-Precision */ \
+ V(xsmindp, XSMINDP, 0xF0000540) \
+ /* VSX Scalar Multiply-Subtract Type-A Double-Precision */ \
+ V(xsmsubadp, XSMSUBADP, 0xF0000188) \
+ /* VSX Scalar Multiply-Subtract Type-A Single-Precision */ \
+ V(xsmsubasp, XSMSUBASP, 0xF0000088) \
+ /* VSX Scalar Multiply-Subtract Type-M Double-Precision */ \
+ V(xsmsubmdp, XSMSUBMDP, 0xF00001C8) \
+ /* VSX Scalar Multiply-Subtract Type-M Single-Precision */ \
+ V(xsmsubmsp, XSMSUBMSP, 0xF00000C8) \
+ /* VSX Scalar Multiply Double-Precision */ \
+ V(xsmuldp, XSMULDP, 0xF0000180) \
+ /* VSX Scalar Multiply Single-Precision */ \
+ V(xsmulsp, XSMULSP, 0xF0000080) \
+ /* VSX Scalar Negative Multiply-Add Type-A Double-Precision */ \
+ V(xsnmaddadp, XSNMADDADP, 0xF0000508) \
+ /* VSX Scalar Negative Multiply-Add Type-A Single-Precision */ \
+ V(xsnmaddasp, XSNMADDASP, 0xF0000408) \
+ /* VSX Scalar Negative Multiply-Add Type-M Double-Precision */ \
+ V(xsnmaddmdp, XSNMADDMDP, 0xF0000548) \
+ /* VSX Scalar Negative Multiply-Add Type-M Single-Precision */ \
+ V(xsnmaddmsp, XSNMADDMSP, 0xF0000448) \
+ /* VSX Scalar Negative Multiply-Subtract Type-A Double-Precision */ \
+ V(xsnmsubadp, XSNMSUBADP, 0xF0000588) \
+ /* VSX Scalar Negative Multiply-Subtract Type-A Single-Precision */ \
+ V(xsnmsubasp, XSNMSUBASP, 0xF0000488) \
+ /* VSX Scalar Negative Multiply-Subtract Type-M Double-Precision */ \
+ V(xsnmsubmdp, XSNMSUBMDP, 0xF00005C8) \
+ /* VSX Scalar Negative Multiply-Subtract Type-M Single-Precision */ \
+ V(xsnmsubmsp, XSNMSUBMSP, 0xF00004C8) \
+ /* VSX Scalar Reciprocal Estimate Double-Precision */ \
+ V(xsredp, XSREDP, 0xF0000168) \
+ /* VSX Scalar Subtract Double-Precision */ \
+ V(xssubdp, XSSUBDP, 0xF0000140) \
+ /* VSX Scalar Subtract Single-Precision */ \
+ V(xssubsp, XSSUBSP, 0xF0000040) \
+ /* VSX Scalar Test for software Divide Double-Precision */ \
+ V(xstdivdp, XSTDIVDP, 0xF00001E8) \
+ /* VSX Vector Add Double-Precision */ \
+ V(xvadddp, XVADDDP, 0xF0000300) \
+ /* VSX Vector Add Single-Precision */ \
+ V(xvaddsp, XVADDSP, 0xF0000200) \
+ /* VSX Vector Compare Equal To Double-Precision */ \
+ V(xvcmpeqdp, XVCMPEQDP, 0xF0000318) \
+ /* VSX Vector Compare Equal To Double-Precision & record CR6 */ \
+ V(xvcmpeqdpx, XVCMPEQDPx, 0xF0000718) \
+ /* VSX Vector Compare Equal To Single-Precision */ \
+ V(xvcmpeqsp, XVCMPEQSP, 0xF0000218) \
+ /* VSX Vector Compare Equal To Single-Precision & record CR6 */ \
+ V(xvcmpeqspx, XVCMPEQSPx, 0xF0000618) \
+ /* VSX Vector Compare Greater Than or Equal To Double-Precision */ \
+ V(xvcmpgedp, XVCMPGEDP, 0xF0000398) \
+ /* VSX Vector Compare Greater Than or Equal To Double-Precision & record */ \
+ /* CR6 */ \
+ V(xvcmpgedpx, XVCMPGEDPx, 0xF0000798) \
+ /* VSX Vector Compare Greater Than or Equal To Single-Precision */ \
+ V(xvcmpgesp, XVCMPGESP, 0xF0000298) \
+ /* VSX Vector Compare Greater Than or Equal To Single-Precision & record */ \
+ /* CR6 */ \
+ V(xvcmpgespx, XVCMPGESPx, 0xF0000698) \
+ /* VSX Vector Compare Greater Than Double-Precision */ \
+ V(xvcmpgtdp, XVCMPGTDP, 0xF0000358) \
+ /* VSX Vector Compare Greater Than Double-Precision & record CR6 */ \
+ V(xvcmpgtdpx, XVCMPGTDPx, 0xF0000758) \
+ /* VSX Vector Compare Greater Than Single-Precision */ \
+ V(xvcmpgtsp, XVCMPGTSP, 0xF0000258) \
+ /* VSX Vector Compare Greater Than Single-Precision & record CR6 */ \
+ V(xvcmpgtspx, XVCMPGTSPx, 0xF0000658) \
+ /* VSX Vector Copy Sign Double-Precision */ \
+ V(xvcpsgndp, XVCPSGNDP, 0xF0000780) \
+ /* VSX Vector Copy Sign Single-Precision */ \
+ V(xvcpsgnsp, XVCPSGNSP, 0xF0000680) \
+ /* VSX Vector Divide Double-Precision */ \
+ V(xvdivdp, XVDIVDP, 0xF00003C0) \
+ /* VSX Vector Divide Single-Precision */ \
+ V(xvdivsp, XVDIVSP, 0xF00002C0) \
+ /* VSX Vector Multiply-Add Type-A Double-Precision */ \
+ V(xvmaddadp, XVMADDADP, 0xF0000308) \
+ /* VSX Vector Multiply-Add Type-A Single-Precision */ \
+ V(xvmaddasp, XVMADDASP, 0xF0000208) \
+ /* VSX Vector Multiply-Add Type-M Double-Precision */ \
+ V(xvmaddmdp, XVMADDMDP, 0xF0000348) \
+ /* VSX Vector Multiply-Add Type-M Single-Precision */ \
+ V(xvmaddmsp, XVMADDMSP, 0xF0000248) \
+ /* VSX Vector Maximum Double-Precision */ \
+ V(xvmaxdp, XVMAXDP, 0xF0000700) \
+ /* VSX Vector Maximum Single-Precision */ \
+ V(xvmaxsp, XVMAXSP, 0xF0000600) \
+ /* VSX Vector Minimum Double-Precision */ \
+ V(xvmindp, XVMINDP, 0xF0000740) \
+ /* VSX Vector Minimum Single-Precision */ \
+ V(xvminsp, XVMINSP, 0xF0000640) \
+ /* VSX Vector Multiply-Subtract Type-A Double-Precision */ \
+ V(xvmsubadp, XVMSUBADP, 0xF0000388) \
+ /* VSX Vector Multiply-Subtract Type-A Single-Precision */ \
+ V(xvmsubasp, XVMSUBASP, 0xF0000288) \
+ /* VSX Vector Multiply-Subtract Type-M Double-Precision */ \
+ V(xvmsubmdp, XVMSUBMDP, 0xF00003C8) \
+ /* VSX Vector Multiply-Subtract Type-M Single-Precision */ \
+ V(xvmsubmsp, XVMSUBMSP, 0xF00002C8) \
+ /* VSX Vector Multiply Double-Precision */ \
+ V(xvmuldp, XVMULDP, 0xF0000380) \
+ /* VSX Vector Multiply Single-Precision */ \
+ V(xvmulsp, XVMULSP, 0xF0000280) \
+ /* VSX Vector Negative Multiply-Add Type-A Double-Precision */ \
+ V(xvnmaddadp, XVNMADDADP, 0xF0000708) \
+ /* VSX Vector Negative Multiply-Add Type-A Single-Precision */ \
+ V(xvnmaddasp, XVNMADDASP, 0xF0000608) \
+ /* VSX Vector Negative Multiply-Add Type-M Double-Precision */ \
+ V(xvnmaddmdp, XVNMADDMDP, 0xF0000748) \
+ /* VSX Vector Negative Multiply-Add Type-M Single-Precision */ \
+ V(xvnmaddmsp, XVNMADDMSP, 0xF0000648) \
+ /* VSX Vector Negative Multiply-Subtract Type-A Double-Precision */ \
+ V(xvnmsubadp, XVNMSUBADP, 0xF0000788) \
+ /* VSX Vector Negative Multiply-Subtract Type-A Single-Precision */ \
+ V(xvnmsubasp, XVNMSUBASP, 0xF0000688) \
+ /* VSX Vector Negative Multiply-Subtract Type-M Double-Precision */ \
+ V(xvnmsubmdp, XVNMSUBMDP, 0xF00007C8) \
+ /* VSX Vector Negative Multiply-Subtract Type-M Single-Precision */ \
+ V(xvnmsubmsp, XVNMSUBMSP, 0xF00006C8) \
+ /* VSX Vector Reciprocal Estimate Double-Precision */ \
+ V(xvredp, XVREDP, 0xF0000368) \
+ /* VSX Vector Subtract Double-Precision */ \
+ V(xvsubdp, XVSUBDP, 0xF0000340) \
+ /* VSX Vector Subtract Single-Precision */ \
+ V(xvsubsp, XVSUBSP, 0xF0000240) \
+ /* VSX Vector Test for software Divide Double-Precision */ \
+ V(xvtdivdp, XVTDIVDP, 0xF00003E8) \
+ /* VSX Vector Test for software Divide Single-Precision */ \
+ V(xvtdivsp, XVTDIVSP, 0xF00002E8) \
+ /* VSX Logical AND */ \
+ V(xxland, XXLAND, 0xF0000410) \
+ /* VSX Logical AND with Complement */ \
+ V(xxlandc, XXLANDC, 0xF0000450) \
+ /* VSX Logical Equivalence */ \
+ V(xxleqv, XXLEQV, 0xF00005D0) \
+ /* VSX Logical NAND */ \
+ V(xxlnand, XXLNAND, 0xF0000590) \
+ /* VSX Logical NOR */ \
+ V(xxlnor, XXLNOR, 0xF0000510) \
+ /* VSX Logical OR */ \
+ V(xxlor, XXLOR, 0xF0000490) \
+ /* VSX Logical OR with Complement */ \
+ V(xxlorc, XXLORC, 0xF0000550) \
+ /* VSX Logical XOR */ \
+ V(xxlxor, XXLXOR, 0xF00004D0) \
+ /* VSX Merge High Word */ \
+ V(xxmrghw, XXMRGHW, 0xF0000090) \
+ /* VSX Merge Low Word */ \
+ V(xxmrglw, XXMRGLW, 0xF0000190) \
+ /* VSX Permute Doubleword Immediate */ \
+ V(xxpermdi, XXPERMDI, 0xF0000050) \
+ /* VSX Shift Left Double by Word Immediate */ \
+ V(xxsldwi, XXSLDWI, 0xF0000010) \
+ /* VSX Splat Word */ \
+ V(xxspltw, XXSPLTW, 0xF0000290)
+
+#define PPC_Z23_OPCODE_LIST(V) \
+ /* Decimal Quantize */ \
+ V(dqua, DQUA, 0xEC000006) \
+ /* Decimal Quantize Immediate */ \
+ V(dquai, DQUAI, 0xEC000086) \
+ /* Decimal Quantize Immediate Quad */ \
+ V(dquaiq, DQUAIQ, 0xFC000086) \
+ /* Decimal Quantize Quad */ \
+ V(dquaq, DQUAQ, 0xFC000006) \
+ /* Decimal Floating Round To FP Integer Without Inexact */ \
+ V(drintn, DRINTN, 0xEC0001C6) \
+ /* Decimal Floating Round To FP Integer Without Inexact Quad */ \
+ V(drintnq, DRINTNQ, 0xFC0001C6) \
+ /* Decimal Floating Round To FP Integer With Inexact */ \
+ V(drintx, DRINTX, 0xEC0000C6) \
+ /* Decimal Floating Round To FP Integer With Inexact Quad */ \
+ V(drintxq, DRINTXQ, 0xFC0000C6) \
+ /* Decimal Floating Reround */ \
+ V(drrnd, DRRND, 0xEC000046) \
+ /* Decimal Floating Reround Quad */ \
+ V(drrndq, DRRNDQ, 0xFC000046)
+
+#define PPC_Z22_OPCODE_LIST(V) \
+ /* Decimal Floating Shift Coefficient Left Immediate */ \
+ V(dscli, DSCLI, 0xEC000084) \
+ /* Decimal Floating Shift Coefficient Left Immediate Quad */ \
+ V(dscliq, DSCLIQ, 0xFC000084) \
+ /* Decimal Floating Shift Coefficient Right Immediate */ \
+ V(dscri, DSCRI, 0xEC0000C4) \
+ /* Decimal Floating Shift Coefficient Right Immediate Quad */ \
+ V(dscriq, DSCRIQ, 0xFC0000C4) \
+ /* Decimal Floating Test Data Class */ \
+ V(dtstdc, DTSTDC, 0xEC000184) \
+ /* Decimal Floating Test Data Class Quad */ \
+ V(dtstdcq, DTSTDCQ, 0xFC000184) \
+ /* Decimal Floating Test Data Group */ \
+ V(dtstdg, DTSTDG, 0xEC0001C4) \
+ /* Decimal Floating Test Data Group Quad */ \
+ V(dtstdgq, DTSTDGQ, 0xFC0001C4)
+
+#define PPC_XX2_OPCODE_A_FORM_LIST(V) \
+ /* VSX Vector Absolute Value Double-Precision */ \
+ V(xvabsdp, XVABSDP, 0xF0000764) \
+ /* VSX Vector Negate Double-Precision */ \
+ V(xvnegdp, XVNEGDP, 0xF00007E4) \
+ /* VSX Vector Square Root Double-Precision */ \
+ V(xvsqrtdp, XVSQRTDP, 0xF000032C) \
+ /* VSX Vector Absolute Value Single-Precision */ \
+ V(xvabssp, XVABSSP, 0xF0000664) \
+ /* VSX Vector Negate Single-Precision */ \
+ V(xvnegsp, XVNEGSP, 0xF00006E4) \
+ /* VSX Vector Reciprocal Estimate Single-Precision */ \
+ V(xvresp, XVRESP, 0xF0000268) \
+ /* VSX Vector Reciprocal Square Root Estimate Single-Precision */ \
+ V(xvrsqrtesp, XVRSQRTESP, 0xF0000228) \
+ /* VSX Vector Square Root Single-Precision */ \
+ V(xvsqrtsp, XVSQRTSP, 0xF000022C) \
+ /* VSX Vector Convert Single-Precision to Signed Fixed-Point Word */ \
+ /* Saturate */ \
+ V(xvcvspsxws, XVCVSPSXWS, 0xF0000260) \
+ /* VSX Vector Convert Single-Precision to Unsigned Fixed-Point Word */ \
+ /* Saturate */ \
+ V(xvcvspuxws, XVCVSPUXWS, 0xF0000220) \
+ /* VSX Vector Convert Signed Fixed-Point Word to Single-Precision */ \
+ V(xvcvsxwsp, XVCVSXWSP, 0xF00002E0) \
+ /* VSX Vector Convert Unsigned Fixed-Point Word to Single-Precision */ \
+ V(xvcvuxwsp, XVCVUXWSP, 0xF00002A0) \
+ /* VSX Vector Round to Double-Precision Integer toward +Infinity */ \
+ V(xvrdpip, XVRDPIP, 0xF00003A4) \
+ /* VSX Vector Round to Double-Precision Integer toward -Infinity */ \
+ V(xvrdpim, XVRDPIM, 0xF00003E4) \
+ /* VSX Vector Round to Double-Precision Integer toward Zero */ \
+ V(xvrdpiz, XVRDPIZ, 0xF0000364) \
+ /* VSX Vector Round to Double-Precision Integer */ \
+ V(xvrdpi, XVRDPI, 0xF0000324) \
+ /* VSX Vector Round to Single-Precision Integer toward +Infinity */ \
+ V(xvrspip, XVRSPIP, 0xF00002A4) \
+ /* VSX Vector Round to Single-Precision Integer toward -Infinity */ \
+ V(xvrspim, XVRSPIM, 0xF00002E4) \
+ /* VSX Vector Round to Single-Precision Integer toward Zero */ \
+ V(xvrspiz, XVRSPIZ, 0xF0000264) \
+ /* VSX Vector Round to Single-Precision Integer */ \
+ V(xvrspi, XVRSPI, 0xF0000224)
+
+#define PPC_XX2_OPCODE_UNUSED_LIST(V) \
+ /* VSX Scalar Square Root Double-Precision */ \
+ V(xssqrtdp, XSSQRTDP, 0xF000012C) \
+ /* VSX Scalar Reciprocal Estimate Single-Precision */ \
+ V(xsresp, XSRESP, 0xF0000068) \
+ /* VSX Scalar Reciprocal Square Root Estimate Single-Precision */ \
+ V(xsrsqrtesp, XSRSQRTESP, 0xF0000028) \
+ /* VSX Scalar Square Root Single-Precision */ \
+ V(xssqrtsp, XSSQRTSP, 0xF000002C) \
+ /* Move To VSR Doubleword */ \
+ V(mtvsrd, MTVSRD, 0x7C000166) \
+ /* Move To VSR Double Doubleword */ \
+ V(mtvsrdd, MTVSRDD, 0x7C000366) \
+ /* Move To VSR Word Algebraic */ \
+ V(mtvsrwa, MTVSRWA, 0x7C0001A6) \
+ /* Move To VSR Word and Zero */ \
+ V(mtvsrwz, MTVSRWZ, 0x7C0001E6) \
+ /* VSX Scalar Absolute Value Double-Precision */ \
+ V(xsabsdp, XSABSDP, 0xF0000564) \
+ /* VSX Scalar Convert Double-Precision to Single-Precision */ \
+ V(xscvdpsp, XSCVDPSP, 0xF0000424) \
+ /* VSX Scalar Convert Double-Precision to Single-Precision format Non- */ \
+ /* signalling */ \
+ V(xscvdpspn, XSCVDPSPN, 0xF000042C) \
+ /* VSX Scalar Convert Double-Precision to Signed Fixed-Point Doubleword */ \
+ /* Saturate */ \
+ V(xscvdpsxds, XSCVDPSXDS, 0xF0000560) \
+ /* VSX Scalar Convert Double-Precision to Signed Fixed-Point Word */ \
+ /* Saturate */ \
+ V(xscvdpsxws, XSCVDPSXWS, 0xF0000160) \
+ /* VSX Scalar Convert Double-Precision to Unsigned Fixed-Point */ \
+ /* Doubleword Saturate */ \
+ V(xscvdpuxds, XSCVDPUXDS, 0xF0000520) \
+ /* VSX Scalar Convert Double-Precision to Unsigned Fixed-Point Word */ \
+ /* Saturate */ \
+ V(xscvdpuxws, XSCVDPUXWS, 0xF0000120) \
+ /* VSX Scalar Convert Single-Precision to Double-Precision (p=1) */ \
+ V(xscvspdp, XSCVSPDP, 0xF0000524) \
+ /* Scalar Convert Single-Precision to Double-Precision format Non- */ \
+ /* signalling */ \
+ V(xscvspdpn, XSCVSPDPN, 0xF000052C) \
+ /* VSX Scalar Convert Signed Fixed-Point Doubleword to Double-Precision */ \
+ V(xscvsxddp, XSCVSXDDP, 0xF00005E0) \
+ /* VSX Scalar Convert Signed Fixed-Point Doubleword to Single-Precision */ \
+ V(xscvsxdsp, XSCVSXDSP, 0xF00004E0) \
+ /* VSX Scalar Convert Unsigned Fixed-Point Doubleword to Double- */ \
+ /* Precision */ \
+ V(xscvuxddp, XSCVUXDDP, 0xF00005A0) \
+ /* VSX Scalar Convert Unsigned Fixed-Point Doubleword to Single- */ \
+ /* Precision */ \
+ V(xscvuxdsp, XSCVUXDSP, 0xF00004A0) \
+ /* VSX Scalar Negative Absolute Value Double-Precision */ \
+ V(xsnabsdp, XSNABSDP, 0xF00005A4) \
+ /* VSX Scalar Negate Double-Precision */ \
+ V(xsnegdp, XSNEGDP, 0xF00005E4) \
+ /* VSX Scalar Round to Double-Precision Integer */ \
+ V(xsrdpi, XSRDPI, 0xF0000124) \
+ /* VSX Scalar Round to Double-Precision Integer using Current rounding */ \
+ /* mode */ \
+ V(xsrdpic, XSRDPIC, 0xF00001AC) \
+ /* VSX Scalar Round to Double-Precision Integer toward -Infinity */ \
+ V(xsrdpim, XSRDPIM, 0xF00001E4) \
+ /* VSX Scalar Round to Double-Precision Integer toward +Infinity */ \
+ V(xsrdpip, XSRDPIP, 0xF00001A4) \
+ /* VSX Scalar Round to Double-Precision Integer toward Zero */ \
+ V(xsrdpiz, XSRDPIZ, 0xF0000164) \
+ /* VSX Scalar Round to Single-Precision */ \
+ V(xsrsp, XSRSP, 0xF0000464) \
+ /* VSX Scalar Reciprocal Square Root Estimate Double-Precision */ \
+ V(xsrsqrtedp, XSRSQRTEDP, 0xF0000128) \
+ /* VSX Scalar Test for software Square Root Double-Precision */ \
+ V(xstsqrtdp, XSTSQRTDP, 0xF00001A8) \
+ /* VSX Vector Convert Double-Precision to Single-Precision */ \
+ V(xvcvdpsp, XVCVDPSP, 0xF0000624) \
+ /* VSX Vector Convert Double-Precision to Signed Fixed-Point Doubleword */ \
+ /* Saturate */ \
+ V(xvcvdpsxds, XVCVDPSXDS, 0xF0000760) \
+ /* VSX Vector Convert Double-Precision to Signed Fixed-Point Word */ \
+ /* Saturate */ \
+ V(xvcvdpsxws, XVCVDPSXWS, 0xF0000360) \
+ /* VSX Vector Convert Double-Precision to Unsigned Fixed-Point */ \
+ /* Doubleword Saturate */ \
+ V(xvcvdpuxds, XVCVDPUXDS, 0xF0000720) \
+ /* VSX Vector Convert Double-Precision to Unsigned Fixed-Point Word */ \
+ /* Saturate */ \
+ V(xvcvdpuxws, XVCVDPUXWS, 0xF0000320) \
+ /* VSX Vector Convert Single-Precision to Double-Precision */ \
+ V(xvcvspdp, XVCVSPDP, 0xF0000724) \
+ /* VSX Vector Convert Single-Precision to Signed Fixed-Point Doubleword */ \
+ /* Saturate */ \
+ V(xvcvspsxds, XVCVSPSXDS, 0xF0000660) \
+ /* VSX Vector Convert Single-Precision to Unsigned Fixed-Point */ \
+ /* Doubleword Saturate */ \
+ V(xvcvspuxds, XVCVSPUXDS, 0xF0000620) \
+ /* VSX Vector Convert Signed Fixed-Point Doubleword to Double-Precision */ \
+ V(xvcvsxddp, XVCVSXDDP, 0xF00007E0) \
+ /* VSX Vector Convert Signed Fixed-Point Doubleword to Single-Precision */ \
+ V(xvcvsxdsp, XVCVSXDSP, 0xF00006E0) \
+ /* VSX Vector Convert Signed Fixed-Point Word to Double-Precision */ \
+ V(xvcvsxwdp, XVCVSXWDP, 0xF00003E0) \
+ /* VSX Vector Convert Unsigned Fixed-Point Doubleword to Double- */ \
+ /* Precision */ \
+ V(xvcvuxddp, XVCVUXDDP, 0xF00007A0) \
+ /* VSX Vector Convert Unsigned Fixed-Point Doubleword to Single- */ \
+ /* Precision */ \
+ V(xvcvuxdsp, XVCVUXDSP, 0xF00006A0) \
+ /* VSX Vector Convert Unsigned Fixed-Point Word to Double-Precision */ \
+ V(xvcvuxwdp, XVCVUXWDP, 0xF00003A0) \
+ /* VSX Vector Negative Absolute Value Double-Precision */ \
+ V(xvnabsdp, XVNABSDP, 0xF00007A4) \
+ /* VSX Vector Negative Absolute Value Single-Precision */ \
+ V(xvnabssp, XVNABSSP, 0xF00006A4) \
+ /* VSX Vector Round to Double-Precision Integer using Current rounding */ \
+ /* mode */ \
+ V(xvrdpic, XVRDPIC, 0xF00003AC) \
+ /* VSX Vector Round to Single-Precision Integer using Current rounding */ \
+ /* mode */ \
+ V(xvrspic, XVRSPIC, 0xF00002AC) \
+ /* VSX Vector Reciprocal Square Root Estimate Double-Precision */ \
+ V(xvrsqrtedp, XVRSQRTEDP, 0xF0000328) \
+ /* VSX Vector Test for software Square Root Double-Precision */ \
+ V(xvtsqrtdp, XVTSQRTDP, 0xF00003A8) \
+ /* VSX Vector Test for software Square Root Single-Precision */ \
+ V(xvtsqrtsp, XVTSQRTSP, 0xF00002A8)
+
+#define PPC_XX2_OPCODE_LIST(V) \
+ PPC_XX2_OPCODE_A_FORM_LIST(V) \
+ PPC_XX2_OPCODE_UNUSED_LIST(V)
+
+#define PPC_EVX_OPCODE_LIST(V) \
+ /* Vector Load Double Word into Double Word by External PID Indexed */ \
+ V(evlddepx, EVLDDEPX, 0x7C00063E) \
+ /* Vector Store Double of Double by External PID Indexed */ \
+ V(evstddepx, EVSTDDEPX, 0x7C00073E) \
+ /* Bit Reversed Increment */ \
+ V(brinc, BRINC, 0x1000020F) \
+ /* Vector Absolute Value */ \
+ V(evabs, EVABS, 0x10000208) \
+ /* Vector Add Immediate Word */ \
+ V(evaddiw, EVADDIW, 0x10000202) \
+ /* Vector Add Signed, Modulo, Integer to Accumulator Word */ \
+ V(evaddsmiaaw, EVADDSMIAAW, 0x100004C9) \
+ /* Vector Add Signed, Saturate, Integer to Accumulator Word */ \
+ V(evaddssiaaw, EVADDSSIAAW, 0x100004C1) \
+ /* Vector Add Unsigned, Modulo, Integer to Accumulator Word */ \
+ V(evaddumiaaw, EVADDUMIAAW, 0x100004C8) \
+ /* Vector Add Unsigned, Saturate, Integer to Accumulator Word */ \
+ V(evaddusiaaw, EVADDUSIAAW, 0x100004C0) \
+ /* Vector Add Word */ \
+ V(evaddw, EVADDW, 0x10000200) \
+ /* Vector AND */ \
+ V(evand, EVAND, 0x10000211) \
+ /* Vector AND with Complement */ \
+ V(evandc, EVANDC, 0x10000212) \
+ /* Vector Compare Equal */ \
+ V(evcmpeq, EVCMPEQ, 0x10000234) \
+ /* Vector Compare Greater Than Signed */ \
+ V(evcmpgts, EVCMPGTS, 0x10000231) \
+ /* Vector Compare Greater Than Unsigned */ \
+ V(evcmpgtu, EVCMPGTU, 0x10000230) \
+ /* Vector Compare Less Than Signed */ \
+ V(evcmplts, EVCMPLTS, 0x10000233) \
+ /* Vector Compare Less Than Unsigned */ \
+ V(evcmpltu, EVCMPLTU, 0x10000232) \
+ /* Vector Count Leading Signed Bits Word */ \
+ V(evcntlsw, EVCNTLSW, 0x1000020E) \
+ /* Vector Count Leading Zeros Word */ \
+ V(evcntlzw, EVCNTLZW, 0x1000020D) \
+ /* Vector Divide Word Signed */ \
+ V(evdivws, EVDIVWS, 0x100004C6) \
+ /* Vector Divide Word Unsigned */ \
+ V(evdivwu, EVDIVWU, 0x100004C7) \
+ /* Vector Equivalent */ \
+ V(eveqv, EVEQV, 0x10000219) \
+ /* Vector Extend Sign Byte */ \
+ V(evextsb, EVEXTSB, 0x1000020A) \
+ /* Vector Extend Sign Half Word */ \
+ V(evextsh, EVEXTSH, 0x1000020B) \
+ /* Vector Load Double Word into Double Word */ \
+ V(evldd, EVLDD, 0x10000301) \
+ /* Vector Load Double Word into Double Word Indexed */ \
+ V(evlddx, EVLDDX, 0x10000300) \
+ /* Vector Load Double into Four Half Words */ \
+ V(evldh, EVLDH, 0x10000305) \
+ /* Vector Load Double into Four Half Words Indexed */ \
+ V(evldhx, EVLDHX, 0x10000304) \
+ /* Vector Load Double into Two Words */ \
+ V(evldw, EVLDW, 0x10000303) \
+ /* Vector Load Double into Two Words Indexed */ \
+ V(evldwx, EVLDWX, 0x10000302) \
+ /* Vector Load Half Word into Half Words Even and Splat */ \
+ V(evlhhesplat, EVLHHESPLAT, 0x10000309) \
+ /* Vector Load Half Word into Half Words Even and Splat Indexed */ \
+ V(evlhhesplatx, EVLHHESPLATX, 0x10000308) \
+ /* Vector Load Half Word into Half Word Odd Signed and Splat */ \
+ V(evlhhossplat, EVLHHOSSPLAT, 0x1000030F) \
+ /* Vector Load Half Word into Half Word Odd Signed and Splat Indexed */ \
+ V(evlhhossplatx, EVLHHOSSPLATX, 0x1000030E) \
+ /* Vector Load Half Word into Half Word Odd Unsigned and Splat */ \
+ V(evlhhousplat, EVLHHOUSPLAT, 0x1000030D) \
+ /* Vector Load Half Word into Half Word Odd Unsigned and Splat Indexed */ \
+ V(evlhhousplatx, EVLHHOUSPLATX, 0x1000030C) \
+ /* Vector Load Word into Two Half Words Even */ \
+ V(evlwhe, EVLWHE, 0x10000311) \
+ /* Vector Load Word into Two Half Words Odd Signed (with sign extension) */ \
+ V(evlwhos, EVLWHOS, 0x10000317) \
+ /* Vector Load Word into Two Half Words Odd Signed Indexed (with sign */ \
+ /* extension) */ \
+ V(evlwhosx, EVLWHOSX, 0x10000316) \
+ /* Vector Load Word into Two Half Words Odd Unsigned (zero-extended) */ \
+ V(evlwhou, EVLWHOU, 0x10000315) \
+ /* Vector Load Word into Two Half Words Odd Unsigned Indexed (zero- */ \
+ /* extended) */ \
+ V(evlwhoux, EVLWHOUX, 0x10000314) \
+ /* Vector Load Word into Two Half Words and Splat */ \
+ V(evlwhsplat, EVLWHSPLAT, 0x1000031D) \
+ /* Vector Load Word into Two Half Words and Splat Indexed */ \
+ V(evlwhsplatx, EVLWHSPLATX, 0x1000031C) \
+ /* Vector Load Word into Word and Splat */ \
+ V(evlwwsplat, EVLWWSPLAT, 0x10000319) \
+ /* Vector Load Word into Word and Splat Indexed */ \
+ V(evlwwsplatx, EVLWWSPLATX, 0x10000318) \
+ /* Vector Merge High */ \
+ V(evmergehi, EVMERGEHI, 0x1000022C) \
+ /* Vector Merge High/Low */ \
+ V(evmergehilo, EVMERGEHILO, 0x1000022E) \
+ /* Vector Merge Low */ \
+ V(evmergelo, EVMERGELO, 0x1000022D) \
+ /* Vector Merge Low/High */ \
+ V(evmergelohi, EVMERGELOHI, 0x1000022F) \
+ /* Vector Multiply Half Words, Even, Guarded, Signed, Modulo, Fractional */ \
+ /* and Accumulate */ \
+ V(evmhegsmfaa, EVMHEGSMFAA, 0x1000052B) \
+ /* Vector Multiply Half Words, Even, Guarded, Signed, Modulo, Fractional */ \
+ /* and Accumulate Negative */ \
+ V(evmhegsmfan, EVMHEGSMFAN, 0x100005AB) \
+ /* Vector Multiply Half Words, Even, Guarded, Signed, Modulo, Integer */ \
+ /* and Accumulate */ \
+ V(evmhegsmiaa, EVMHEGSMIAA, 0x10000529) \
+ /* Vector Multiply Half Words, Even, Guarded, Signed, Modulo, Integer */ \
+ /* and Accumulate Negative */ \
+ V(evmhegsmian, EVMHEGSMIAN, 0x100005A9) \
+ /* Vector Multiply Half Words, Even, Guarded, Unsigned, Modulo, Integer */ \
+ /* and Accumulate */ \
+ V(evmhegumiaa, EVMHEGUMIAA, 0x10000528) \
+ /* Vector Multiply Half Words, Even, Guarded, Unsigned, Modulo, Integer */ \
+ /* and Accumulate Negative */ \
+ V(evmhegumian, EVMHEGUMIAN, 0x100005A8) \
+ /* Vector Multiply Half Words, Even, Signed, Modulo, Fractional */ \
+ V(evmhesmf, EVMHESMF, 0x1000040B) \
+ /* Vector Multiply Half Words, Even, Signed, Modulo, Fractional to */ \
+ /* Accumulator */ \
+ V(evmhesmfa, EVMHESMFA, 0x1000042B) \
+ /* Vector Multiply Half Words, Even, Signed, Modulo, Fractional and */ \
+ /* Accumulate into Words */ \
+ V(evmhesmfaaw, EVMHESMFAAW, 0x1000050B) \
+ /* Vector Multiply Half Words, Even, Signed, Modulo, Fractional and */ \
+ /* Accumulate Negative into Words */ \
+ V(evmhesmfanw, EVMHESMFANW, 0x1000058B) \
+ /* Vector Multiply Half Words, Even, Signed, Modulo, Integer */ \
+ V(evmhesmi, EVMHESMI, 0x10000409) \
+ /* Vector Multiply Half Words, Even, Signed, Modulo, Integer to */ \
+ /* Accumulator */ \
+ V(evmhesmia, EVMHESMIA, 0x10000429) \
+ /* Vector Multiply Half Words, Even, Signed, Modulo, Integer and */ \
+ /* Accumulate into Words */ \
+ V(evmhesmiaaw, EVMHESMIAAW, 0x10000509) \
+ /* Vector Multiply Half Words, Even, Signed, Modulo, Integer and */ \
+ /* Accumulate Negative into Words */ \
+ V(evmhesmianw, EVMHESMIANW, 0x10000589) \
+ /* Vector Multiply Half Words, Even, Signed, Saturate, Fractional */ \
+ V(evmhessf, EVMHESSF, 0x10000403) \
+ /* Vector Multiply Half Words, Even, Signed, Saturate, Fractional to */ \
+ /* Accumulator */ \
+ V(evmhessfa, EVMHESSFA, 0x10000423) \
+ /* Vector Multiply Half Words, Even, Signed, Saturate, Fractional and */ \
+ /* Accumulate into Words */ \
+ V(evmhessfaaw, EVMHESSFAAW, 0x10000503) \
+ /* Vector Multiply Half Words, Even, Signed, Saturate, Fractional and */ \
+ /* Accumulate Negative into Words */ \
+ V(evmhessfanw, EVMHESSFANW, 0x10000583) \
+ /* Vector Multiply Half Words, Even, Signed, Saturate, Integer and */ \
+ /* Accumulate into Words */ \
+ V(evmhessiaaw, EVMHESSIAAW, 0x10000501) \
+ /* Vector Multiply Half Words, Even, Signed, Saturate, Integer and */ \
+ /* Accumulate Negative into Words */ \
+ V(evmhessianw, EVMHESSIANW, 0x10000581) \
+ /* Vector Multiply Half Words, Even, Unsigned, Modulo, Integer */ \
+ V(evmheumi, EVMHEUMI, 0x10000408) \
+ /* Vector Multiply Half Words, Even, Unsigned, Modulo, Integer to */ \
+ /* Accumulator */ \
+ V(evmheumia, EVMHEUMIA, 0x10000428) \
+ /* Vector Multiply Half Words, Even, Unsigned, Modulo, Integer and */ \
+ /* Accumulate into Words */ \
+ V(evmheumiaaw, EVMHEUMIAAW, 0x10000508) \
+ /* Vector Multiply Half Words, Even, Unsigned, Modulo, Integer and */ \
+ /* Accumulate Negative into Words */ \
+ V(evmheumianw, EVMHEUMIANW, 0x10000588) \
+ /* Vector Multiply Half Words, Even, Unsigned, Saturate, Integer and */ \
+ /* Accumulate into Words */ \
+ V(evmheusiaaw, EVMHEUSIAAW, 0x10000500) \
+ /* Vector Multiply Half Words, Even, Unsigned, Saturate, Integer and */ \
+ /* Accumulate Negative into Words */ \
+ V(evmheusianw, EVMHEUSIANW, 0x10000580) \
+ /* Vector Multiply Half Words, Odd, Guarded, Signed, Modulo, Fractional */ \
+ /* and Accumulate */ \
+ V(evmhogsmfaa, EVMHOGSMFAA, 0x1000052F) \
+ /* Vector Multiply Half Words, Odd, Guarded, Signed, Modulo, Fractional */ \
+ /* and Accumulate Negative */ \
+ V(evmhogsmfan, EVMHOGSMFAN, 0x100005AF) \
+ /* Vector Multiply Half Words, Odd, Guarded, Signed, Modulo, Integer, */ \
+ /* and Accumulate */ \
+ V(evmhogsmiaa, EVMHOGSMIAA, 0x1000052D) \
+ /* Vector Multiply Half Words, Odd, Guarded, Signed, Modulo, Integer and */ \
+ /* Accumulate Negative */ \
+ V(evmhogsmian, EVMHOGSMIAN, 0x100005AD) \
+ /* Vector Multiply Half Words, Odd, Guarded, Unsigned, Modulo, Integer */ \
+ /* and Accumulate */ \
+ V(evmhogumiaa, EVMHOGUMIAA, 0x1000052C) \
+ /* Vector Multiply Half Words, Odd, Guarded, Unsigned, Modulo, Integer */ \
+ /* and Accumulate Negative */ \
+ V(evmhogumian, EVMHOGUMIAN, 0x100005AC) \
+ /* Vector Multiply Half Words, Odd, Signed, Modulo, Fractional */ \
+ V(evmhosmf, EVMHOSMF, 0x1000040F) \
+ /* Vector Multiply Half Words, Odd, Signed, Modulo, Fractional to */ \
+ /* Accumulator */ \
+ V(evmhosmfa, EVMHOSMFA, 0x1000042F) \
+ /* Vector Multiply Half Words, Odd, Signed, Modulo, Fractional and */ \
+ /* Accumulate into Words */ \
+ V(evmhosmfaaw, EVMHOSMFAAW, 0x1000050F) \
+ /* Vector Multiply Half Words, Odd, Signed, Modulo, Fractional and */ \
+ /* Accumulate Negative into Words */ \
+ V(evmhosmfanw, EVMHOSMFANW, 0x1000058F) \
+ /* Vector Multiply Half Words, Odd, Signed, Modulo, Integer */ \
+ V(evmhosmi, EVMHOSMI, 0x1000040D) \
+ /* Vector Multiply Half Words, Odd, Signed, Modulo, Integer to */ \
+ /* Accumulator */ \
+ V(evmhosmia, EVMHOSMIA, 0x1000042D) \
+ /* Vector Multiply Half Words, Odd, Signed, Modulo, Integer and */ \
+ /* Accumulate into Words */ \
+ V(evmhosmiaaw, EVMHOSMIAAW, 0x1000050D) \
+ /* Vector Multiply Half Words, Odd, Signed, Modulo, Integer and */ \
+ /* Accumulate Negative into Words */ \
+ V(evmhosmianw, EVMHOSMIANW, 0x1000058D) \
+ /* Vector Multiply Half Words, Odd, Signed, Saturate, Fractional */ \
+ V(evmhossf, EVMHOSSF, 0x10000407) \
+ /* Vector Multiply Half Words, Odd, Signed, Saturate, Fractional to */ \
+ /* Accumulator */ \
+ V(evmhossfa, EVMHOSSFA, 0x10000427) \
+ /* Vector Multiply Half Words, Odd, Signed, Saturate, Fractional and */ \
+ /* Accumulate into Words */ \
+ V(evmhossfaaw, EVMHOSSFAAW, 0x10000507) \
+ /* Vector Multiply Half Words, Odd, Signed, Saturate, Fractional and */ \
+ /* Accumulate Negative into Words */ \
+ V(evmhossfanw, EVMHOSSFANW, 0x10000587) \
+ /* Vector Multiply Half Words, Odd, Signed, Saturate, Integer and */ \
+ /* Accumulate into Words */ \
+ V(evmhossiaaw, EVMHOSSIAAW, 0x10000505) \
+ /* Vector Multiply Half Words, Odd, Signed, Saturate, Integer and */ \
+ /* Accumulate Negative into Words */ \
+ V(evmhossianw, EVMHOSSIANW, 0x10000585) \
+ /* Vector Multiply Half Words, Odd, Unsigned, Modulo, Integer */ \
+ V(evmhoumi, EVMHOUMI, 0x1000040C) \
+ /* Vector Multiply Half Words, Odd, Unsigned, Modulo, Integer to */ \
+ /* Accumulator */ \
+ V(evmhoumia, EVMHOUMIA, 0x1000042C) \
+ /* Vector Multiply Half Words, Odd, Unsigned, Modulo, Integer and */ \
+ /* Accumulate into Words */ \
+ V(evmhoumiaaw, EVMHOUMIAAW, 0x1000050C) \
+ /* Vector Multiply Half Words, Odd, Unsigned, Modulo, Integer and */ \
+ /* Accumulate Negative into Words */ \
+ V(evmhoumianw, EVMHOUMIANW, 0x1000058C) \
+ /* Vector Multiply Half Words, Odd, Unsigned, Saturate, Integer and */ \
+ /* Accumulate into Words */ \
+ V(evmhousiaaw, EVMHOUSIAAW, 0x10000504) \
+ /* Vector Multiply Half Words, Odd, Unsigned, Saturate, Integer and */ \
+ /* Accumulate Negative into Words */ \
+ V(evmhousianw, EVMHOUSIANW, 0x10000584) \
+ /* Initialize Accumulator */ \
+ V(evmra, EVMRA, 0x100004C4) \
+ /* Vector Multiply Word High Signed, Modulo, Fractional */ \
+ V(evmwhsmf, EVMWHSMF, 0x1000044F) \
+ /* Vector Multiply Word High Signed, Modulo, Fractional to Accumulator */ \
+ V(evmwhsmfa, EVMWHSMFA, 0x1000046F) \
+ /* Vector Multiply Word High Signed, Modulo, Integer */ \
+ V(evmwhsmi, EVMWHSMI, 0x1000044D) \
+ /* Vector Multiply Word High Signed, Modulo, Integer to Accumulator */ \
+ V(evmwhsmia, EVMWHSMIA, 0x1000046D) \
+ /* Vector Multiply Word High Signed, Saturate, Fractional */ \
+ V(evmwhssf, EVMWHSSF, 0x10000447) \
+ /* Vector Multiply Word High Signed, Saturate, Fractional to Accumulator */ \
+ V(evmwhssfa, EVMWHSSFA, 0x10000467) \
+ /* Vector Multiply Word High Unsigned, Modulo, Integer */ \
+ V(evmwhumi, EVMWHUMI, 0x1000044C) \
+ /* Vector Multiply Word High Unsigned, Modulo, Integer to Accumulator */ \
+ V(evmwhumia, EVMWHUMIA, 0x1000046C) \
+ /* Vector Multiply Word Low Signed, Modulo, Integer and Accumulate in */ \
+ /* Words */ \
+ V(evmwlsmiaaw, EVMWLSMIAAW, 0x10000549) \
+ /* Vector Multiply Word Low Signed, Modulo, Integer and Accumulate */ \
+ /* Negative in Words */ \
+ V(evmwlsmianw, EVMWLSMIANW, 0x100005C9) \
+ /* Vector Multiply Word Low Signed, Saturate, Integer and Accumulate in */ \
+ /* Words */ \
+ V(evmwlssiaaw, EVMWLSSIAAW, 0x10000541) \
+ /* Vector Multiply Word Low Signed, Saturate, Integer and Accumulate */ \
+ /* Negative in Words */ \
+ V(evmwlssianw, EVMWLSSIANW, 0x100005C1) \
+ /* Vector Multiply Word Low Unsigned, Modulo, Integer */ \
+ V(evmwlumi, EVMWLUMI, 0x10000448) \
+ /* Vector Multiply Word Low Unsigned, Modulo, Integer to Accumulator */ \
+ V(evmwlumia, EVMWLUMIA, 0x10000468) \
+ /* Vector Multiply Word Low Unsigned, Modulo, Integer and Accumulate in */ \
+ /* Words */ \
+ V(evmwlumiaaw, EVMWLUMIAAW, 0x10000548) \
+ /* Vector Multiply Word Low Unsigned, Modulo, Integer and Accumulate */ \
+ /* Negative in Words */ \
+ V(evmwlumianw, EVMWLUMIANW, 0x100005C8) \
+ /* Vector Multiply Word Low Unsigned, Saturate, Integer and Accumulate */ \
+ /* in Words */ \
+ V(evmwlusiaaw, EVMWLUSIAAW, 0x10000540) \
+ /* Vector Multiply Word Low Unsigned, Saturate, Integer and Accumulate */ \
+ /* Negative in Words */ \
+ V(evmwlusianw, EVMWLUSIANW, 0x100005C0) \
+ /* Vector Multiply Word Signed, Modulo, Fractional */ \
+ V(evmwsmf, EVMWSMF, 0x1000045B) \
+ /* Vector Multiply Word Signed, Modulo, Fractional to Accumulator */ \
+ V(evmwsmfa, EVMWSMFA, 0x1000047B) \
+ /* Vector Multiply Word Signed, Modulo, Fractional and Accumulate */ \
+ V(evmwsmfaa, EVMWSMFAA, 0x1000055B) \
+ /* Vector Multiply Word Signed, Modulo, Fractional and Accumulate */ \
+ /* Negative */ \
+ V(evmwsmfan, EVMWSMFAN, 0x100005DB) \
+ /* Vector Multiply Word Signed, Modulo, Integer */ \
+ V(evmwsmi, EVMWSMI, 0x10000459) \
+ /* Vector Multiply Word Signed, Modulo, Integer to Accumulator */ \
+ V(evmwsmia, EVMWSMIA, 0x10000479) \
+ /* Vector Multiply Word Signed, Modulo, Integer and Accumulate */ \
+ V(evmwsmiaa, EVMWSMIAA, 0x10000559) \
+ /* Vector Multiply Word Signed, Modulo, Integer and Accumulate Negative */ \
+ V(evmwsmian, EVMWSMIAN, 0x100005D9) \
+ /* Vector Multiply Word Signed, Saturate, Fractional */ \
+ V(evmwssf, EVMWSSF, 0x10000453) \
+ /* Vector Multiply Word Signed, Saturate, Fractional to Accumulator */ \
+ V(evmwssfa, EVMWSSFA, 0x10000473) \
+ /* Vector Multiply Word Signed, Saturate, Fractional and Accumulate */ \
+ V(evmwssfaa, EVMWSSFAA, 0x10000553) \
+ /* Vector Multiply Word Signed, Saturate, Fractional and Accumulate */ \
+ /* Negative */ \
+ V(evmwssfan, EVMWSSFAN, 0x100005D3) \
+ /* Vector Multiply Word Unsigned, Modulo, Integer */ \
+ V(evmwumi, EVMWUMI, 0x10000458) \
+ /* Vector Multiply Word Unsigned, Modulo, Integer to Accumulator */ \
+ V(evmwumia, EVMWUMIA, 0x10000478) \
+ /* Vector Multiply Word Unsigned, Modulo, Integer and Accumulate */ \
+ V(evmwumiaa, EVMWUMIAA, 0x10000558) \
+ /* Vector Multiply Word Unsigned, Modulo, Integer and Accumulate */ \
+ /* Negative */ \
+ V(evmwumian, EVMWUMIAN, 0x100005D8) \
+ /* Vector NAND */ \
+ V(evnand, EVNAND, 0x1000021E) \
+ /* Vector Negate */ \
+ V(evneg, EVNEG, 0x10000209) \
+ /* Vector NOR */ \
+ V(evnor, EVNOR, 0x10000218) \
+ /* Vector OR */ \
+ V(evor, EVOR, 0x10000217) \
+ /* Vector OR with Complement */ \
+ V(evorc, EVORC, 0x1000021B) \
+ /* Vector Rotate Left Word */ \
+ V(evrlw, EVRLW, 0x10000228) \
+ /* Vector Rotate Left Word Immediate */ \
+ V(evrlwi, EVRLWI, 0x1000022A) \
+ /* Vector Round Word */ \
+ V(evrndw, EVRNDW, 0x1000020C) \
+ /* Vector Shift Left Word */ \
+ V(evslw, EVSLW, 0x10000224) \
+ /* Vector Shift Left Word Immediate */ \
+ V(evslwi, EVSLWI, 0x10000226) \
+ /* Vector Splat Fractional Immediate */ \
+ V(evsplatfi, EVSPLATFI, 0x1000022B) \
+ /* Vector Splat Immediate */ \
+ V(evsplati, EVSPLATI, 0x10000229) \
+ /* Vector Shift Right Word Immediate Signed */ \
+ V(evsrwis, EVSRWIS, 0x10000223) \
+ /* Vector Shift Right Word Immediate Unsigned */ \
+ V(evsrwiu, EVSRWIU, 0x10000222) \
+ /* Vector Shift Right Word Signed */ \
+ V(evsrws, EVSRWS, 0x10000221) \
+ /* Vector Shift Right Word Unsigned */ \
+ V(evsrwu, EVSRWU, 0x10000220) \
+ /* Vector Store Double of Double */ \
+ V(evstdd, EVSTDD, 0x10000321) \
+ /* Vector Store Double of Double Indexed */ \
+ V(evstddx, EVSTDDX, 0x10000320) \
+ /* Vector Store Double of Four Half Words */ \
+ V(evstdh, EVSTDH, 0x10000325) \
+ /* Vector Store Double of Four Half Words Indexed */ \
+ V(evstdhx, EVSTDHX, 0x10000324) \
+ /* Vector Store Double of Two Words */ \
+ V(evstdw, EVSTDW, 0x10000323) \
+ /* Vector Store Double of Two Words Indexed */ \
+ V(evstdwx, EVSTDWX, 0x10000322) \
+ /* Vector Store Word of Two Half Words from Even */ \
+ V(evstwhe, EVSTWHE, 0x10000331) \
+ /* Vector Store Word of Two Half Words from Even Indexed */ \
+ V(evstwhex, EVSTWHEX, 0x10000330) \
+ /* Vector Store Word of Two Half Words from Odd */ \
+ V(evstwho, EVSTWHO, 0x10000335) \
+ /* Vector Store Word of Two Half Words from Odd Indexed */ \
+ V(evstwhox, EVSTWHOX, 0x10000334) \
+ /* Vector Store Word of Word from Even */ \
+ V(evstwwe, EVSTWWE, 0x10000339) \
+ /* Vector Store Word of Word from Even Indexed */ \
+ V(evstwwex, EVSTWWEX, 0x10000338) \
+ /* Vector Store Word of Word from Odd */ \
+ V(evstwwo, EVSTWWO, 0x1000033D) \
+ /* Vector Store Word of Word from Odd Indexed */ \
+ V(evstwwox, EVSTWWOX, 0x1000033C) \
+ /* Vector Subtract Signed, Modulo, Integer to Accumulator Word */ \
+ V(evsubfsmiaaw, EVSUBFSMIAAW, 0x100004CB) \
+ /* Vector Subtract Signed, Saturate, Integer to Accumulator Word */ \
+ V(evsubfssiaaw, EVSUBFSSIAAW, 0x100004C3) \
+ /* Vector Subtract Unsigned, Modulo, Integer to Accumulator Word */ \
+ V(evsubfumiaaw, EVSUBFUMIAAW, 0x100004CA) \
+ /* Vector Subtract Unsigned, Saturate, Integer to Accumulator Word */ \
+ V(evsubfusiaaw, EVSUBFUSIAAW, 0x100004C2) \
+ /* Vector Subtract from Word */ \
+ V(evsubfw, EVSUBFW, 0x10000204) \
+ /* Vector Subtract Immediate from Word */ \
+ V(evsubifw, EVSUBIFW, 0x10000206) \
+ /* Vector XOR */ \
+ V(evxor, EVXOR, 0x10000216) \
+ /* Floating-Point Double-Precision Absolute Value */ \
+ V(efdabs, EFDABS, 0x100002E4) \
+ /* Floating-Point Double-Precision Add */ \
+ V(efdadd, EFDADD, 0x100002E0) \
+ /* Floating-Point Double-Precision Convert from Single-Precision */ \
+ V(efdcfs, EFDCFS, 0x100002EF) \
+ /* Convert Floating-Point Double-Precision from Signed Fraction */ \
+ V(efdcfsf, EFDCFSF, 0x100002F3) \
+ /* Convert Floating-Point Double-Precision from Signed Integer */ \
+ V(efdcfsi, EFDCFSI, 0x100002F1) \
+ /* Convert Floating-Point Double-Precision from Signed Integer */ \
+ /* Doubleword */ \
+ V(efdcfsid, EFDCFSID, 0x100002E3) \
+ /* Convert Floating-Point Double-Precision from Unsigned Fraction */ \
+ V(efdcfuf, EFDCFUF, 0x100002F2) \
+ /* Convert Floating-Point Double-Precision from Unsigned Integer */ \
+ V(efdcfui, EFDCFUI, 0x100002F0) \
+ /* Convert Floating-Point Double-Precision fromUnsigned Integer */ \
+ /* Doubleword */ \
+ V(efdcfuid, EFDCFUID, 0x100002E2) \
+ /* Floating-Point Double-Precision Compare Equal */ \
+ V(efdcmpeq, EFDCMPEQ, 0x100002EE) \
+ /* Floating-Point Double-Precision Compare Greater Than */ \
+ V(efdcmpgt, EFDCMPGT, 0x100002EC) \
+ /* Floating-Point Double-Precision Compare Less Than */ \
+ V(efdcmplt, EFDCMPLT, 0x100002ED) \
+ /* Convert Floating-Point Double-Precision to Signed Fraction */ \
+ V(efdctsf, EFDCTSF, 0x100002F7) \
+ /* Convert Floating-Point Double-Precision to Signed Integer */ \
+ V(efdctsi, EFDCTSI, 0x100002F5) \
+ /* Convert Floating-Point Double-Precision to Signed Integer Doubleword */ \
+ /* with Round toward Zero */ \
+ V(efdctsidz, EFDCTSIDZ, 0x100002EB) \
+ /* Convert Floating-Point Double-Precision to Signed Integer with Round */ \
+ /* toward Zero */ \
+ V(efdctsiz, EFDCTSIZ, 0x100002FA) \
+ /* Convert Floating-Point Double-Precision to Unsigned Fraction */ \
+ V(efdctuf, EFDCTUF, 0x100002F6) \
+ /* Convert Floating-Point Double-Precision to Unsigned Integer */ \
+ V(efdctui, EFDCTUI, 0x100002F4) \
+ /* Convert Floating-Point Double-Precision to Unsigned Integer */ \
+ /* Doubleword with Round toward Zero */ \
+ V(efdctuidz, EFDCTUIDZ, 0x100002EA) \
+ /* Convert Floating-Point Double-Precision to Unsigned Integer with */ \
+ /* Round toward Zero */ \
+ V(efdctuiz, EFDCTUIZ, 0x100002F8) \
+ /* Floating-Point Double-Precision Divide */ \
+ V(efddiv, EFDDIV, 0x100002E9) \
+ /* Floating-Point Double-Precision Multiply */ \
+ V(efdmul, EFDMUL, 0x100002E8) \
+ /* Floating-Point Double-Precision Negative Absolute Value */ \
+ V(efdnabs, EFDNABS, 0x100002E5) \
+ /* Floating-Point Double-Precision Negate */ \
+ V(efdneg, EFDNEG, 0x100002E6) \
+ /* Floating-Point Double-Precision Subtract */ \
+ V(efdsub, EFDSUB, 0x100002E1) \
+ /* Floating-Point Double-Precision Test Equal */ \
+ V(efdtsteq, EFDTSTEQ, 0x100002FE) \
+ /* Floating-Point Double-Precision Test Greater Than */ \
+ V(efdtstgt, EFDTSTGT, 0x100002FC) \
+ /* Floating-Point Double-Precision Test Less Than */ \
+ V(efdtstlt, EFDTSTLT, 0x100002FD) \
+ /* Floating-Point Single-Precision Convert from Double-Precision */ \
+ V(efscfd, EFSCFD, 0x100002CF) \
+ /* Floating-Point Absolute Value */ \
+ V(efsabs, EFSABS, 0x100002C4) \
+ /* Floating-Point Add */ \
+ V(efsadd, EFSADD, 0x100002C0) \
+ /* Convert Floating-Point from Signed Fraction */ \
+ V(efscfsf, EFSCFSF, 0x100002D3) \
+ /* Convert Floating-Point from Signed Integer */ \
+ V(efscfsi, EFSCFSI, 0x100002D1) \
+ /* Convert Floating-Point from Unsigned Fraction */ \
+ V(efscfuf, EFSCFUF, 0x100002D2) \
+ /* Convert Floating-Point from Unsigned Integer */ \
+ V(efscfui, EFSCFUI, 0x100002D0) \
+ /* Floating-Point Compare Equal */ \
+ V(efscmpeq, EFSCMPEQ, 0x100002CE) \
+ /* Floating-Point Compare Greater Than */ \
+ V(efscmpgt, EFSCMPGT, 0x100002CC) \
+ /* Floating-Point Compare Less Than */ \
+ V(efscmplt, EFSCMPLT, 0x100002CD) \
+ /* Convert Floating-Point to Signed Fraction */ \
+ V(efsctsf, EFSCTSF, 0x100002D7) \
+ /* Convert Floating-Point to Signed Integer */ \
+ V(efsctsi, EFSCTSI, 0x100002D5) \
+ /* Convert Floating-Point to Signed Integer with Round toward Zero */ \
+ V(efsctsiz, EFSCTSIZ, 0x100002DA) \
+ /* Convert Floating-Point to Unsigned Fraction */ \
+ V(efsctuf, EFSCTUF, 0x100002D6) \
+ /* Convert Floating-Point to Unsigned Integer */ \
+ V(efsctui, EFSCTUI, 0x100002D4) \
+ /* Convert Floating-Point to Unsigned Integer with Round toward Zero */ \
+ V(efsctuiz, EFSCTUIZ, 0x100002D8) \
+ /* Floating-Point Divide */ \
+ V(efsdiv, EFSDIV, 0x100002C9) \
+ /* Floating-Point Multiply */ \
+ V(efsmul, EFSMUL, 0x100002C8) \
+ /* Floating-Point Negative Absolute Value */ \
+ V(efsnabs, EFSNABS, 0x100002C5) \
+ /* Floating-Point Negate */ \
+ V(efsneg, EFSNEG, 0x100002C6) \
+ /* Floating-Point Subtract */ \
+ V(efssub, EFSSUB, 0x100002C1) \
+ /* Floating-Point Test Equal */ \
+ V(efststeq, EFSTSTEQ, 0x100002DE) \
+ /* Floating-Point Test Greater Than */ \
+ V(efststgt, EFSTSTGT, 0x100002DC) \
+ /* Floating-Point Test Less Than */ \
+ V(efststlt, EFSTSTLT, 0x100002DD) \
+ /* Vector Floating-Point Absolute Value */ \
+ V(evfsabs, EVFSABS, 0x10000284) \
+ /* Vector Floating-Point Add */ \
+ V(evfsadd, EVFSADD, 0x10000280) \
+ /* Vector Convert Floating-Point from Signed Fraction */ \
+ V(evfscfsf, EVFSCFSF, 0x10000293) \
+ /* Vector Convert Floating-Point from Signed Integer */ \
+ V(evfscfsi, EVFSCFSI, 0x10000291) \
+ /* Vector Convert Floating-Point from Unsigned Fraction */ \
+ V(evfscfuf, EVFSCFUF, 0x10000292) \
+ /* Vector Convert Floating-Point from Unsigned Integer */ \
+ V(evfscfui, EVFSCFUI, 0x10000290) \
+ /* Vector Floating-Point Compare Equal */ \
+ V(evfscmpeq, EVFSCMPEQ, 0x1000028E) \
+ /* Vector Floating-Point Compare Greater Than */ \
+ V(evfscmpgt, EVFSCMPGT, 0x1000028C) \
+ /* Vector Floating-Point Compare Less Than */ \
+ V(evfscmplt, EVFSCMPLT, 0x1000028D) \
+ /* Vector Convert Floating-Point to Signed Fraction */ \
+ V(evfsctsf, EVFSCTSF, 0x10000297) \
+ /* Vector Convert Floating-Point to Signed Integer */ \
+ V(evfsctsi, EVFSCTSI, 0x10000295) \
+ /* Vector Convert Floating-Point to Signed Integer with Round toward */ \
+ /* Zero */ \
+ V(evfsctsiz, EVFSCTSIZ, 0x1000029A) \
+ /* Vector Convert Floating-Point to Unsigned Fraction */ \
+ V(evfsctuf, EVFSCTUF, 0x10000296) \
+ /* Vector Convert Floating-Point to Unsigned Integer */ \
+ V(evfsctui, EVFSCTUI, 0x10000294) \
+ /* Vector Convert Floating-Point to Unsigned Integer with Round toward */ \
+ /* Zero */ \
+ V(evfsctuiz, EVFSCTUIZ, 0x10000298) \
+ /* Vector Floating-Point Divide */ \
+ V(evfsdiv, EVFSDIV, 0x10000289) \
+ /* Vector Floating-Point Multiply */ \
+ V(evfsmul, EVFSMUL, 0x10000288) \
+ /* Vector Floating-Point Negative Absolute Value */ \
+ V(evfsnabs, EVFSNABS, 0x10000285) \
+ /* Vector Floating-Point Negate */ \
+ V(evfsneg, EVFSNEG, 0x10000286) \
+ /* Vector Floating-Point Subtract */ \
+ V(evfssub, EVFSSUB, 0x10000281) \
+ /* Vector Floating-Point Test Equal */ \
+ V(evfststeq, EVFSTSTEQ, 0x1000029E) \
+ /* Vector Floating-Point Test Greater Than */ \
+ V(evfststgt, EVFSTSTGT, 0x1000029C) \
+ /* Vector Floating-Point Test Less Than */ \
+ V(evfststlt, EVFSTSTLT, 0x1000029D)
+
+#define PPC_VC_OPCODE_LIST(V) \
+ /* Vector Compare Bounds Single-Precision */ \
+ V(vcmpbfp, VCMPBFP, 0x100003C6) \
+ /* Vector Compare Equal To Single-Precision */ \
+ V(vcmpeqfp, VCMPEQFP, 0x100000C6) \
+ /* Vector Compare Equal To Unsigned Byte */ \
+ V(vcmpequb, VCMPEQUB, 0x10000006) \
+ /* Vector Compare Equal To Unsigned Doubleword */ \
+ V(vcmpequd, VCMPEQUD, 0x100000C7) \
+ /* Vector Compare Equal To Unsigned Halfword */ \
+ V(vcmpequh, VCMPEQUH, 0x10000046) \
+ /* Vector Compare Equal To Unsigned Word */ \
+ V(vcmpequw, VCMPEQUW, 0x10000086) \
+ /* Vector Compare Greater Than or Equal To Single-Precision */ \
+ V(vcmpgefp, VCMPGEFP, 0x100001C6) \
+ /* Vector Compare Greater Than Single-Precision */ \
+ V(vcmpgtfp, VCMPGTFP, 0x100002C6) \
+ /* Vector Compare Greater Than Signed Byte */ \
+ V(vcmpgtsb, VCMPGTSB, 0x10000306) \
+ /* Vector Compare Greater Than Signed Doubleword */ \
+ V(vcmpgtsd, VCMPGTSD, 0x100003C7) \
+ /* Vector Compare Greater Than Signed Halfword */ \
+ V(vcmpgtsh, VCMPGTSH, 0x10000346) \
+ /* Vector Compare Greater Than Signed Word */ \
+ V(vcmpgtsw, VCMPGTSW, 0x10000386) \
+ /* Vector Compare Greater Than Unsigned Byte */ \
+ V(vcmpgtub, VCMPGTUB, 0x10000206) \
+ /* Vector Compare Greater Than Unsigned Doubleword */ \
+ V(vcmpgtud, VCMPGTUD, 0x100002C7) \
+ /* Vector Compare Greater Than Unsigned Halfword */ \
+ V(vcmpgtuh, VCMPGTUH, 0x10000246) \
+ /* Vector Compare Greater Than Unsigned Word */ \
+ V(vcmpgtuw, VCMPGTUW, 0x10000286)
+
+#define PPC_X_OPCODE_A_FORM_LIST(V) \
+ /* Modulo Signed Dword */ \
+ V(modsd, MODSD, 0x7C000612) \
+ /* Modulo Unsigned Dword */ \
+ V(modud, MODUD, 0x7C000212) \
+ /* Modulo Signed Word */ \
+ V(modsw, MODSW, 0x7C000616) \
+ /* Modulo Unsigned Word */ \
+ V(moduw, MODUW, 0x7C000216)
+
+#define PPC_X_OPCODE_B_FORM_LIST(V) \
+ /* XOR */ \
+ V(xor_, XORX, 0x7C000278) \
+ /* AND */ \
+ V(and_, ANDX, 0x7C000038) \
+ /* AND with Complement */ \
+ V(andc, ANDCX, 0x7C000078) \
+ /* OR */ \
+ V(orx, ORX, 0x7C000378) \
+ /* OR with Complement */ \
+ V(orc, ORC, 0x7C000338) \
+ /* NOR */ \
+ V(nor, NORX, 0x7C0000F8) \
+ /* Shift Right Word */ \
+ V(srw, SRWX, 0x7C000430) \
+ /* Shift Left Word */ \
+ V(slw, SLWX, 0x7C000030) \
+ /* Shift Right Algebraic Word */ \
+ V(sraw, SRAW, 0x7C000630) \
+ /* Shift Left Doubleword */ \
+ V(sld, SLDX, 0x7C000036) \
+ /* Shift Right Algebraic Doubleword */ \
+ V(srad, SRAD, 0x7C000634) \
+ /* Shift Right Doubleword */ \
+ V(srd, SRDX, 0x7C000436)
+
+#define PPC_X_OPCODE_C_FORM_LIST(V) \
+ /* Count Leading Zeros Word */ \
+ V(cntlzw, CNTLZWX, 0x7C000034) \
+ /* Count Leading Zeros Doubleword */ \
+ V(cntlzd, CNTLZDX, 0x7C000074) \
+ /* Population Count Byte-wise */ \
+ V(popcntb, POPCNTB, 0x7C0000F4) \
+ /* Population Count Words */ \
+ V(popcntw, POPCNTW, 0x7C0002F4) \
+ /* Population Count Doubleword */ \
+ V(popcntd, POPCNTD, 0x7C0003F4) \
+ /* Extend Sign Byte */ \
+ V(extsb, EXTSB, 0x7C000774) \
+ /* Extend Sign Halfword */ \
+ V(extsh, EXTSH, 0x7C000734)
+
+#define PPC_X_OPCODE_D_FORM_LIST(V) \
+ /* Load Halfword Byte-Reverse Indexed */ \
+ V(lhbrx, LHBRX, 0x7C00062C) \
+ /* Load Word Byte-Reverse Indexed */ \
+ V(lwbrx, LWBRX, 0x7C00042C) \
+ /* Load Doubleword Byte-Reverse Indexed */ \
+ V(ldbrx, LDBRX, 0x7C000428) \
+ /* Load Byte and Zero Indexed */ \
+ V(lbzx, LBZX, 0x7C0000AE) \
+ /* Load Byte and Zero with Update Indexed */ \
+ V(lbzux, LBZUX, 0x7C0000EE) \
+ /* Load Halfword and Zero Indexed */ \
+ V(lhzx, LHZX, 0x7C00022E) \
+ /* Load Halfword and Zero with Update Indexed */ \
+ V(lhzux, LHZUX, 0x7C00026E) \
+ /* Load Halfword Algebraic Indexed */ \
+ V(lhax, LHAX, 0x7C0002AE) \
+ /* Load Word and Zero Indexed */ \
+ V(lwzx, LWZX, 0x7C00002E) \
+ /* Load Word and Zero with Update Indexed */ \
+ V(lwzux, LWZUX, 0x7C00006E) \
+ /* Load Doubleword Indexed */ \
+ V(ldx, LDX, 0x7C00002A) \
+ /* Load Doubleword with Update Indexed */ \
+ V(ldux, LDUX, 0x7C00006A) \
+ /* Load Floating-Point Double Indexed */ \
+ V(lfdx, LFDX, 0x7C0004AE) \
+ /* Load Floating-Point Single Indexed */ \
+ V(lfsx, LFSX, 0x7C00042E) \
+ /* Load Floating-Point Double with Update Indexed */ \
+ V(lfdux, LFDUX, 0x7C0004EE) \
+ /* Load Floating-Point Single with Update Indexed */ \
+ V(lfsux, LFSUX, 0x7C00046E) \
+ /* Store Byte with Update Indexed */ \
+ V(stbux, STBUX, 0x7C0001EE) \
+ /* Store Byte Indexed */ \
+ V(stbx, STBX, 0x7C0001AE) \
+ /* Store Halfword with Update Indexed */ \
+ V(sthux, STHUX, 0x7C00036E) \
+ /* Store Halfword Indexed */ \
+ V(sthx, STHX, 0x7C00032E) \
+ /* Store Word with Update Indexed */ \
+ V(stwux, STWUX, 0x7C00016E) \
+ /* Store Word Indexed */ \
+ V(stwx, STWX, 0x7C00012E) \
+ /* Store Doubleword with Update Indexed */ \
+ V(stdux, STDUX, 0x7C00016A) \
+ /* Store Doubleword Indexed */ \
+ V(stdx, STDX, 0x7C00012A) \
+ /* Store Floating-Point Double with Update Indexed */ \
+ V(stfdux, STFDUX, 0x7C0005EE) \
+ /* Store Floating-Point Double Indexed */ \
+ V(stfdx, STFDX, 0x7C0005AE) \
+ /* Store Floating-Point Single with Update Indexed */ \
+ V(stfsux, STFSUX, 0x7C00056E) \
+ /* Store Floating-Point Single Indexed */ \
+ V(stfsx, STFSX, 0x7C00052E) \
+ /* Load Vector Indexed */ \
+ V(lvx, LVX, 0x7C0000CE) \
+ /* Store Vector Indexed */ \
+ V(stvx, STVX, 0x7C0001CE)
+
+#define PPC_X_OPCODE_E_FORM_LIST(V) \
+ /* Shift Right Algebraic Word Immediate */ \
+ V(srawi, SRAWIX, 0x7C000670)
+
+#define PPC_X_OPCODE_F_FORM_LIST(V) \
+ /* Compare */ \
+ V(cmp, CMP, 0x7C000000) \
+ /* Compare Logical */ \
+ V(cmpl, CMPL, 0x7C000040)
+
+#define PPC_X_OPCODE_EH_S_FORM_LIST(V) \
+ /* Store Byte Conditional Indexed */ \
+ V(stbcx, STBCX, 0x7C00056D) \
+ /* Store Halfword Conditional Indexed Xform */ \
+ V(sthcx, STHCX, 0x7C0005AD) \
+ /* Store Word Conditional Indexed & record CR0 */ \
+ V(stwcx, STWCX, 0x7C00012D) \
+ /* Store Doubleword Conditional Indexed & record CR0 */ \
+ V(stdcx, STDCX, 0x7C0001AD)
+
+#define PPC_X_OPCODE_EH_L_FORM_LIST(V) \
+ /* Load Byte And Reserve Indexed */ \
+ V(lbarx, LBARX, 0x7C000068) \
+ /* Load Halfword And Reserve Indexed Xform */ \
+ V(lharx, LHARX, 0x7C0000E8) \
+ /* Load Word and Reserve Indexed */ \
+ V(lwarx, LWARX, 0x7C000028) \
+ /* Load Doubleword And Reserve Indexed */ \
+ V(ldarx, LDARX, 0x7C0000A8)
+
+#define PPC_X_OPCODE_UNUSED_LIST(V) \
+ /* Bit Permute Doubleword */ \
+ V(bpermd, BPERMD, 0x7C0001F8) \
+ /* Extend Sign Word */ \
+ V(extsw, EXTSW, 0x7C0007B4) \
+ /* Load Word Algebraic with Update Indexed */ \
+ V(lwaux, LWAUX, 0x7C0002EA) \
+ /* Load Word Algebraic Indexed */ \
+ V(lwax, LWAX, 0x7C0002AA) \
+ /* Parity Doubleword */ \
+ V(prtyd, PRTYD, 0x7C000174) \
+ /* Store Doubleword Byte-Reverse Indexed */ \
+ V(stdbrx, STDBRX, 0x7C000528) \
+ /* Trap Doubleword */ \
+ V(td, TD, 0x7C000088) \
+ /* Branch Conditional to Branch Target Address Register */ \
+ V(bctar, BCTAR, 0x4C000460) \
+ /* Compare Byte */ \
+ V(cmpb, CMPB, 0x7C0003F8) \
+ /* Data Cache Block Flush */ \
+ V(dcbf, DCBF, 0x7C0000AC) \
+ /* Data Cache Block Store */ \
+ V(dcbst, DCBST, 0x7C00006C) \
+ /* Data Cache Block Touch */ \
+ V(dcbt, DCBT, 0x7C00022C) \
+ /* Data Cache Block Touch for Store */ \
+ V(dcbtst, DCBTST, 0x7C0001EC) \
+ /* Data Cache Block Zero */ \
+ V(dcbz, DCBZ, 0x7C0007EC) \
+ /* Equivalent */ \
+ V(eqv, EQV, 0x7C000238) \
+ /* Instruction Cache Block Invalidate */ \
+ V(icbi, ICBI, 0x7C0007AC) \
+ /* NAND */ \
+ V(nand, NAND, 0x7C0003B8) \
+ /* Parity Word */ \
+ V(prtyw, PRTYW, 0x7C000134) \
+ /* Store Halfword Byte-Reverse Indexed */ \
+ V(sthbrx, STHBRX, 0x7C00072C) \
+ /* Store Word Byte-Reverse Indexed */ \
+ V(stwbrx, STWBRX, 0x7C00052C) \
+ /* Synchronize */ \
+ V(sync, SYNC, 0x7C0004AC) \
+ /* Trap Word */ \
+ V(tw, TW, 0x7C000008) \
+ /* ExecuExecuted No Operation */ \
+ V(xnop, XNOP, 0x68000000) \
+ /* Convert Binary Coded Decimal To Declets */ \
+ V(cbcdtd, CBCDTD, 0x7C000274) \
+ /* Convert Declets To Binary Coded Decimal */ \
+ V(cdtbcd, CDTBCD, 0x7C000234) \
+ /* Decimal Floating Add */ \
+ V(dadd, DADD, 0xEC000004) \
+ /* Decimal Floating Add Quad */ \
+ V(daddq, DADDQ, 0xFC000004) \
+ /* Decimal Floating Convert From Fixed */ \
+ V(dcffix, DCFFIX, 0xEC000644) \
+ /* Decimal Floating Convert From Fixed Quad */ \
+ V(dcffixq, DCFFIXQ, 0xFC000644) \
+ /* Decimal Floating Compare Ordered */ \
+ V(dcmpo, DCMPO, 0xEC000104) \
+ /* Decimal Floating Compare Ordered Quad */ \
+ V(dcmpoq, DCMPOQ, 0xFC000104) \
+ /* Decimal Floating Compare Unordered */ \
+ V(dcmpu, DCMPU, 0xEC000504) \
+ /* Decimal Floating Compare Unordered Quad */ \
+ V(dcmpuq, DCMPUQ, 0xFC000504) \
+ /* Decimal Floating Convert To DFP Long */ \
+ V(dctdp, DCTDP, 0xEC000204) \
+ /* Decimal Floating Convert To Fixed */ \
+ V(dctfix, DCTFIX, 0xEC000244) \
+ /* Decimal Floating Convert To Fixed Quad */ \
+ V(dctfixq, DCTFIXQ, 0xFC000244) \
+ /* Decimal Floating Convert To DFP Extended */ \
+ V(dctqpq, DCTQPQ, 0xFC000204) \
+ /* Decimal Floating Decode DPD To BCD */ \
+ V(ddedpd, DDEDPD, 0xEC000284) \
+ /* Decimal Floating Decode DPD To BCD Quad */ \
+ V(ddedpdq, DDEDPDQ, 0xFC000284) \
+ /* Decimal Floating Divide */ \
+ V(ddiv, DDIV, 0xEC000444) \
+ /* Decimal Floating Divide Quad */ \
+ V(ddivq, DDIVQ, 0xFC000444) \
+ /* Decimal Floating Encode BCD To DPD */ \
+ V(denbcd, DENBCD, 0xEC000684) \
+ /* Decimal Floating Encode BCD To DPD Quad */ \
+ V(denbcdq, DENBCDQ, 0xFC000684) \
+ /* Decimal Floating Insert Exponent */ \
+ V(diex, DIEX, 0xEC0006C4) \
+ /* Decimal Floating Insert Exponent Quad */ \
+ V(diexq, DIEXQ, 0xFC0006C4) \
+ /* Decimal Floating Multiply */ \
+ V(dmul, DMUL, 0xEC000044) \
+ /* Decimal Floating Multiply Quad */ \
+ V(dmulq, DMULQ, 0xFC000044) \
+ /* Decimal Floating Round To DFP Long */ \
+ V(drdpq, DRDPQ, 0xFC000604) \
+ /* Decimal Floating Round To DFP Short */ \
+ V(drsp, DRSP, 0xEC000604) \
+ /* Decimal Floating Subtract */ \
+ V(dsub, DSUB, 0xEC000404) \
+ /* Decimal Floating Subtract Quad */ \
+ V(dsubq, DSUBQ, 0xFC000404) \
+ /* Decimal Floating Test Exponent */ \
+ V(dtstex, DTSTEX, 0xEC000144) \
+ /* Decimal Floating Test Exponent Quad */ \
+ V(dtstexq, DTSTEXQ, 0xFC000144) \
+ /* Decimal Floating Test Significance */ \
+ V(dtstsf, DTSTSF, 0xEC000544) \
+ /* Decimal Floating Test Significance Quad */ \
+ V(dtstsfq, DTSTSFQ, 0xFC000544) \
+ /* Decimal Floating Extract Exponent */ \
+ V(dxex, DXEX, 0xEC0002C4) \
+ /* Decimal Floating Extract Exponent Quad */ \
+ V(dxexq, DXEXQ, 0xFC0002C4) \
+ /* Decorated Storage Notify */ \
+ V(dsn, DSN, 0x7C0003C6) \
+ /* Load Byte with Decoration Indexed */ \
+ V(lbdx, LBDX, 0x7C000406) \
+ /* Load Doubleword with Decoration Indexed */ \
+ V(lddx, LDDX, 0x7C0004C6) \
+ /* Load Floating Doubleword with Decoration Indexed */ \
+ V(lfddx, LFDDX, 0x7C000646) \
+ /* Load Halfword with Decoration Indexed */ \
+ V(lhdx, LHDX, 0x7C000446) \
+ /* Load Word with Decoration Indexed */ \
+ V(lwdx, LWDX, 0x7C000486) \
+ /* Store Byte with Decoration Indexed */ \
+ V(stbdx, STBDX, 0x7C000506) \
+ /* Store Doubleword with Decoration Indexed */ \
+ V(stddx, STDDX, 0x7C0005C6) \
+ /* Store Floating Doubleword with Decoration Indexed */ \
+ V(stfddx, STFDDX, 0x7C000746) \
+ /* Store Halfword with Decoration Indexed */ \
+ V(sthdx, STHDX, 0x7C000546) \
+ /* Store Word with Decoration Indexed */ \
+ V(stwdx, STWDX, 0x7C000586) \
+ /* Data Cache Block Allocate */ \
+ V(dcba, DCBA, 0x7C0005EC) \
+ /* Data Cache Block Invalidate */ \
+ V(dcbi, DCBI, 0x7C0003AC) \
+ /* Instruction Cache Block Touch */ \
+ V(icbt, ICBT, 0x7C00002C) \
+ /* Move to Condition Register from XER */ \
+ V(mcrxr, MCRXR, 0x7C000400) \
+ /* TLB Invalidate Local Indexed */ \
+ V(tlbilx, TLBILX, 0x7C000024) \
+ /* TLB Invalidate Virtual Address Indexed */ \
+ V(tlbivax, TLBIVAX, 0x7C000624) \
+ /* TLB Read Entry */ \
+ V(tlbre, TLBRE, 0x7C000764) \
+ /* TLB Search Indexed */ \
+ V(tlbsx, TLBSX, 0x7C000724) \
+ /* TLB Write Entry */ \
+ V(tlbwe, TLBWE, 0x7C0007A4) \
+ /* Write External Enable */ \
+ V(wrtee, WRTEE, 0x7C000106) \
+ /* Write External Enable Immediate */ \
+ V(wrteei, WRTEEI, 0x7C000146) \
+ /* Data Cache Read */ \
+ V(dcread, DCREAD, 0x7C00028C) \
+ /* Instruction Cache Read */ \
+ V(icread, ICREAD, 0x7C0007CC) \
+ /* Data Cache Invalidate */ \
+ V(dci, DCI, 0x7C00038C) \
+ /* Instruction Cache Invalidate */ \
+ V(ici, ICI, 0x7C00078C) \
+ /* Move From Device Control Register User Mode Indexed */ \
+ V(mfdcrux, MFDCRUX, 0x7C000246) \
+ /* Move From Device Control Register Indexed */ \
+ V(mfdcrx, MFDCRX, 0x7C000206) \
+ /* Move To Device Control Register User Mode Indexed */ \
+ V(mtdcrux, MTDCRUX, 0x7C000346) \
+ /* Move To Device Control Register Indexed */ \
+ V(mtdcrx, MTDCRX, 0x7C000306) \
+ /* Return From Debug Interrupt */ \
+ V(rfdi, RFDI, 0x4C00004E) \
+ /* Data Cache Block Flush by External PID */ \
+ V(dcbfep, DCBFEP, 0x7C0000FE) \
+ /* Data Cache Block Store by External PID */ \
+ V(dcbstep, DCBSTEP, 0x7C00007E) \
+ /* Data Cache Block Touch by External PID */ \
+ V(dcbtep, DCBTEP, 0x7C00027E) \
+ /* Data Cache Block Touch for Store by External PID */ \
+ V(dcbtstep, DCBTSTEP, 0x7C0001FE) \
+ /* Data Cache Block Zero by External PID */ \
+ V(dcbzep, DCBZEP, 0x7C0007FE) \
+ /* Instruction Cache Block Invalidate by External PID */ \
+ V(icbiep, ICBIEP, 0x7C0007BE) \
+ /* Load Byte and Zero by External PID Indexed */ \
+ V(lbepx, LBEPX, 0x7C0000BE) \
+ /* Load Floating-Point Double by External PID Indexed */ \
+ V(lfdepx, LFDEPX, 0x7C0004BE) \
+ /* Load Halfword and Zero by External PID Indexed */ \
+ V(lhepx, LHEPX, 0x7C00023E) \
+ /* Load Vector by External PID Indexed */ \
+ V(lvepx, LVEPX, 0x7C00024E) \
+ /* Load Vector by External PID Indexed Last */ \
+ V(lvepxl, LVEPXL, 0x7C00020E) \
+ /* Load Word and Zero by External PID Indexed */ \
+ V(lwepx, LWEPX, 0x7C00003E) \
+ /* Store Byte by External PID Indexed */ \
+ V(stbepx, STBEPX, 0x7C0001BE) \
+ /* Store Floating-Point Double by External PID Indexed */ \
+ V(stfdepx, STFDEPX, 0x7C0005BE) \
+ /* Store Halfword by External PID Indexed */ \
+ V(sthepx, STHEPX, 0x7C00033E) \
+ /* Store Vector by External PID Indexed */ \
+ V(stvepx, STVEPX, 0x7C00064E) \
+ /* Store Vector by External PID Indexed Last */ \
+ V(stvepxl, STVEPXL, 0x7C00060E) \
+ /* Store Word by External PID Indexed */ \
+ V(stwepx, STWEPX, 0x7C00013E) \
+ /* Load Doubleword by External PID Indexed */ \
+ V(ldepx, LDEPX, 0x7C00003A) \
+ /* Store Doubleword by External PID Indexed */ \
+ V(stdepx, STDEPX, 0x7C00013A) \
+ /* TLB Search and Reserve Indexed */ \
+ V(tlbsrx, TLBSRX, 0x7C0006A5) \
+ /* External Control In Word Indexed */ \
+ V(eciwx, ECIWX, 0x7C00026C) \
+ /* External Control Out Word Indexed */ \
+ V(ecowx, ECOWX, 0x7C00036C) \
+ /* Data Cache Block Lock Clear */ \
+ V(dcblc, DCBLC, 0x7C00030C) \
+ /* Data Cache Block Lock Query */ \
+ V(dcblq, DCBLQ, 0x7C00034D) \
+ /* Data Cache Block Touch and Lock Set */ \
+ V(dcbtls, DCBTLS, 0x7C00014C) \
+ /* Data Cache Block Touch for Store and Lock Set */ \
+ V(dcbtstls, DCBTSTLS, 0x7C00010C) \
+ /* Instruction Cache Block Lock Clear */ \
+ V(icblc, ICBLC, 0x7C0001CC) \
+ /* Instruction Cache Block Lock Query */ \
+ V(icblq, ICBLQ, 0x7C00018D) \
+ /* Instruction Cache Block Touch and Lock Set */ \
+ V(icbtls, ICBTLS, 0x7C0003CC) \
+ /* Floating Compare Ordered */ \
+ V(fcmpo, FCMPO, 0xFC000040) \
+ /* Floating Compare Unordered */ \
+ V(fcmpu, FCMPU, 0xFC000000) \
+ /* Floating Test for software Divide */ \
+ V(ftdiv, FTDIV, 0xFC000100) \
+ /* Floating Test for software Square Root */ \
+ V(ftsqrt, FTSQRT, 0xFC000140) \
+ /* Load Floating-Point as Integer Word Algebraic Indexed */ \
+ V(lfiwax, LFIWAX, 0x7C0006AE) \
+ /* Load Floating-Point as Integer Word and Zero Indexed */ \
+ V(lfiwzx, LFIWZX, 0x7C0006EE) \
+ /* Move To Condition Register from FPSCR */ \
+ V(mcrfs, MCRFS, 0xFC000080) \
+ /* Store Floating-Point as Integer Word Indexed */ \
+ V(stfiwx, STFIWX, 0x7C0007AE) \
+ /* Load Floating-Point Double Pair Indexed */ \
+ V(lfdpx, LFDPX, 0x7C00062E) \
+ /* Store Floating-Point Double Pair Indexed */ \
+ V(stfdpx, STFDPX, 0x7C00072E) \
+ /* Floating Absolute Value */ \
+ V(fabs, FABS, 0xFC000210) \
+ /* Floating Convert From Integer Doubleword */ \
+ V(fcfid, FCFID, 0xFC00069C) \
+ /* Floating Convert From Integer Doubleword Single */ \
+ V(fcfids, FCFIDS, 0xEC00069C) \
+ /* Floating Convert From Integer Doubleword Unsigned */ \
+ V(fcfidu, FCFIDU, 0xFC00079C) \
+ /* Floating Convert From Integer Doubleword Unsigned Single */ \
+ V(fcfidus, FCFIDUS, 0xEC00079C) \
+ /* Floating Copy Sign */ \
+ V(fcpsgn, FCPSGN, 0xFC000010) \
+ /* Floating Convert To Integer Doubleword */ \
+ V(fctid, FCTID, 0xFC00065C) \
+ /* Floating Convert To Integer Doubleword Unsigned */ \
+ V(fctidu, FCTIDU, 0xFC00075C) \
+ /* Floating Convert To Integer Doubleword Unsigned with round toward */ \
+ /* Zero */ \
+ V(fctiduz, FCTIDUZ, 0xFC00075E) \
+ /* Floating Convert To Integer Doubleword with round toward Zero */ \
+ V(fctidz, FCTIDZ, 0xFC00065E) \
+ /* Floating Convert To Integer Word */ \
+ V(fctiw, FCTIW, 0xFC00001C) \
+ /* Floating Convert To Integer Word Unsigned */ \
+ V(fctiwu, FCTIWU, 0xFC00011C) \
+ /* Floating Convert To Integer Word Unsigned with round toward Zero */ \
+ V(fctiwuz, FCTIWUZ, 0xFC00011E) \
+ /* Floating Convert To Integer Word with round to Zero */ \
+ V(fctiwz, FCTIWZ, 0xFC00001E) \
+ /* Floating Move Register */ \
+ V(fmr, FMR, 0xFC000090) \
+ /* Floating Negative Absolute Value */ \
+ V(fnabs, FNABS, 0xFC000110) \
+ /* Floating Negate */ \
+ V(fneg, FNEG, 0xFC000050) \
+ /* Floating Round to Single-Precision */ \
+ V(frsp, FRSP, 0xFC000018) \
+ /* Move From FPSCR */ \
+ V(mffs, MFFS, 0xFC00048E) \
+ /* Move To FPSCR Bit 0 */ \
+ V(mtfsb0, MTFSB0, 0xFC00008C) \
+ /* Move To FPSCR Bit 1 */ \
+ V(mtfsb1, MTFSB1, 0xFC00004C) \
+ /* Move To FPSCR Field Immediate */ \
+ V(mtfsfi, MTFSFI, 0xFC00010C) \
+ /* Floating Round To Integer Minus */ \
+ V(frim, FRIM, 0xFC0003D0) \
+ /* Floating Round To Integer Nearest */ \
+ V(frin, FRIN, 0xFC000310) \
+ /* Floating Round To Integer Plus */ \
+ V(frip, FRIP, 0xFC000390) \
+ /* Floating Round To Integer toward Zero */ \
+ V(friz, FRIZ, 0xFC000350) \
+ /* Multiply Cross Halfword to Word Signed */ \
+ V(mulchw, MULCHW, 0x10000150) \
+ /* Multiply Cross Halfword to Word Unsigned */ \
+ V(mulchwu, MULCHWU, 0x10000110) \
+ /* Multiply High Halfword to Word Signed */ \
+ V(mulhhw, MULHHW, 0x10000050) \
+ /* Multiply High Halfword to Word Unsigned */ \
+ V(mulhhwu, MULHHWU, 0x10000010) \
+ /* Multiply Low Halfword to Word Signed */ \
+ V(mullhw, MULLHW, 0x10000350) \
+ /* Multiply Low Halfword to Word Unsigned */ \
+ V(mullhwu, MULLHWU, 0x10000310) \
+ /* Determine Leftmost Zero Byte DQ 56 E0000000 P 58 LSQ lq Load Quadword */ \
+ V(dlmzb, DLMZB, 0x7C00009C) \
+ /* Load Quadword And Reserve Indexed */ \
+ V(lqarx, LQARX, 0x7C000228) \
+ /* Store Quadword Conditional Indexed and record CR0 */ \
+ V(stqcx, STQCX, 0x7C00016D) \
+ /* Load String Word Immediate */ \
+ V(lswi, LSWI, 0x7C0004AA) \
+ /* Load String Word Indexed */ \
+ V(lswx, LSWX, 0x7C00042A) \
+ /* Store String Word Immediate */ \
+ V(stswi, STSWI, 0x7C0005AA) \
+ /* Store String Word Indexed */ \
+ V(stswx, STSWX, 0x7C00052A) \
+ /* Clear BHRB */ \
+ V(clrbhrb, CLRBHRB, 0x7C00035C) \
+ /* Enforce In-order Execution of I/O */ \
+ V(eieio, EIEIO, 0x7C0006AC) \
+ /* Load Byte and Zero Caching Inhibited Indexed */ \
+ V(lbzcix, LBZCIX, 0x7C0006AA) \
+ /* Load Doubleword Caching Inhibited Indexed */ \
+ V(ldcix, LDCIX, 0x7C0006EA) \
+ /* Load Halfword and Zero Caching Inhibited Indexed */ \
+ V(lhzcix, LHZCIX, 0x7C00066A) \
+ /* Load Word and Zero Caching Inhibited Indexed */ \
+ V(lwzcix, LWZCIX, 0x7C00062A) \
+ /* Move From Segment Register */ \
+ V(mfsr, MFSR, 0x7C0004A6) \
+ /* Move From Segment Register Indirect */ \
+ V(mfsrin, MFSRIN, 0x7C000526) \
+ /* Move To Machine State Register Doubleword */ \
+ V(mtmsrd, MTMSRD, 0x7C000164) \
+ /* Move To Split Little Endian */ \
+ V(mtsle, MTSLE, 0x7C000126) \
+ /* Move To Segment Register */ \
+ V(mtsr, MTSR, 0x7C0001A4) \
+ /* Move To Segment Register Indirect */ \
+ V(mtsrin, MTSRIN, 0x7C0001E4) \
+ /* SLB Find Entry ESID */ \
+ V(slbfee, SLBFEE, 0x7C0007A7) \
+ /* SLB Invalidate All */ \
+ V(slbia, SLBIA, 0x7C0003E4) \
+ /* SLB Invalidate Entry */ \
+ V(slbie, SLBIE, 0x7C000364) \
+ /* SLB Move From Entry ESID */ \
+ V(slbmfee, SLBMFEE, 0x7C000726) \
+ /* SLB Move From Entry VSID */ \
+ V(slbmfev, SLBMFEV, 0x7C0006A6) \
+ /* SLB Move To Entry */ \
+ V(slbmte, SLBMTE, 0x7C000324) \
+ /* Store Byte Caching Inhibited Indexed */ \
+ V(stbcix, STBCIX, 0x7C0007AA) \
+ /* Store Doubleword Caching Inhibited Indexed */ \
+ V(stdcix, STDCIX, 0x7C0007EA) \
+ /* Store Halfword and Zero Caching Inhibited Indexed */ \
+ V(sthcix, STHCIX, 0x7C00076A) \
+ /* Store Word and Zero Caching Inhibited Indexed */ \
+ V(stwcix, STWCIX, 0x7C00072A) \
+ /* TLB Invalidate All */ \
+ V(tlbia, TLBIA, 0x7C0002E4) \
+ /* TLB Invalidate Entry */ \
+ V(tlbie, TLBIE, 0x7C000264) \
+ /* TLB Invalidate Entry Local */ \
+ V(tlbiel, TLBIEL, 0x7C000224) \
+ /* Message Clear Privileged */ \
+ V(msgclrp, MSGCLRP, 0x7C00015C) \
+ /* Message Send Privileged */ \
+ V(msgsndp, MSGSNDP, 0x7C00011C) \
+ /* Message Clear */ \
+ V(msgclr, MSGCLR, 0x7C0001DC) \
+ /* Message Send */ \
+ V(msgsnd, MSGSND, 0x7C00019C) \
+ /* Move From Machine State Register */ \
+ V(mfmsr, MFMSR, 0x7C0000A6) \
+ /* Move To Machine State Register */ \
+ V(mtmsr, MTMSR, 0x7C000124) \
+ /* TLB Synchronize */ \
+ V(tlbsync, TLBSYNC, 0x7C00046C) \
+ /* Transaction Abort */ \
+ V(tabort, TABORT, 0x7C00071D) \
+ /* Transaction Abort Doubleword Conditional */ \
+ V(tabortdc, TABORTDC, 0x7C00065D) \
+ /* Transaction Abort Doubleword Conditional Immediate */ \
+ V(tabortdci, TABORTDCI, 0x7C0006DD) \
+ /* Transaction Abort Word Conditional */ \
+ V(tabortwc, TABORTWC, 0x7C00061D) \
+ /* Transaction Abort Word Conditional Immediate */ \
+ V(tabortwci, TABORTWCI, 0x7C00069D) \
+ /* Transaction Begin */ \
+ V(tbegin, TBEGIN, 0x7C00051D) \
+ /* Transaction Check */ \
+ V(tcheck, TCHECK, 0x7C00059C) \
+ /* Transaction End */ \
+ V(tend, TEND, 0x7C00055C) \
+ /* Transaction Recheckpoint */ \
+ V(trechkpt, TRECHKPT, 0x7C0007DD) \
+ /* Transaction Reclaim */ \
+ V(treclaim, TRECLAIM, 0x7C00075D) \
+ /* Transaction Suspend or Resume */ \
+ V(tsr, TSR, 0x7C0005DC) \
+ /* Load Vector Element Byte Indexed */ \
+ V(lvebx, LVEBX, 0x7C00000E) \
+ /* Load Vector Element Halfword Indexed */ \
+ V(lvehx, LVEHX, 0x7C00004E) \
+ /* Load Vector Element Word Indexed */ \
+ V(lvewx, LVEWX, 0x7C00008E) \
+ /* Load Vector for Shift Left */ \
+ V(lvsl, LVSL, 0x7C00000C) \
+ /* Load Vector for Shift Right */ \
+ V(lvsr, LVSR, 0x7C00004C) \
+ /* Load Vector Indexed Last */ \
+ V(lvxl, LVXL, 0x7C0002CE) \
+ /* Store Vector Element Byte Indexed */ \
+ V(stvebx, STVEBX, 0x7C00010E) \
+ /* Store Vector Element Halfword Indexed */ \
+ V(stvehx, STVEHX, 0x7C00014E) \
+ /* Store Vector Element Word Indexed */ \
+ V(stvewx, STVEWX, 0x7C00018E) \
+ /* Store Vector Indexed Last */ \
+ V(stvxl, STVXL, 0x7C0003CE) \
+ /* Floating Merge Even Word */ \
+ V(fmrgew, FMRGEW, 0xFC00078C) \
+ /* Floating Merge Odd Word */ \
+ V(fmrgow, FMRGOW, 0xFC00068C) \
+ /* Wait for Interrupt */ \
+ V(wait, WAIT, 0x7C00007C)
+
+#define PPC_X_OPCODE_LIST(V) \
+ PPC_X_OPCODE_A_FORM_LIST(V) \
+ PPC_X_OPCODE_B_FORM_LIST(V) \
+ PPC_X_OPCODE_C_FORM_LIST(V) \
+ PPC_X_OPCODE_D_FORM_LIST(V) \
+ PPC_X_OPCODE_E_FORM_LIST(V) \
+ PPC_X_OPCODE_F_FORM_LIST(V) \
+ PPC_X_OPCODE_EH_L_FORM_LIST(V) \
+ PPC_X_OPCODE_UNUSED_LIST(V)
+
+#define PPC_EVS_OPCODE_LIST(V) \
+ /* Vector Select */ \
+ V(evsel, EVSEL, 0x10000278)
+
+#define PPC_DS_OPCODE_LIST(V) \
+ /* Load Doubleword */ \
+ V(ld, LD, 0xE8000000) \
+ /* Load Doubleword with Update */ \
+ V(ldu, LDU, 0xE8000001) \
+ /* Load Word Algebraic */ \
+ V(lwa, LWA, 0xE8000002) \
+ /* Store Doubleword */ \
+ V(std, STD, 0xF8000000) \
+ /* Store Doubleword with Update */ \
+ V(stdu, STDU, 0xF8000001) \
+ /* Load Floating-Point Double Pair */ \
+ V(lfdp, LFDP, 0xE4000000) \
+ /* Store Floating-Point Double Pair */ \
+ V(stfdp, STFDP, 0xF4000000) \
+ /* Store Quadword */ \
+ V(stq, STQ, 0xF8000002)
+
+#define PPC_DQ_OPCODE_LIST(V) V(lsq, LSQ, 0xE0000000)
+
+#define PPC_D_OPCODE_LIST(V) \
+ /* Trap Doubleword Immediate */ \
+ V(tdi, TDI, 0x08000000) \
+ /* Add Immediate */ \
+ V(addi, ADDI, 0x38000000) \
+ /* Add Immediate Carrying */ \
+ V(addic, ADDIC, 0x30000000) \
+ /* Add Immediate Carrying & record CR0 */ \
+ V(addicx, ADDICx, 0x34000000) \
+ /* Add Immediate Shifted */ \
+ V(addis, ADDIS, 0x3C000000) \
+ /* AND Immediate & record CR0 */ \
+ V(andix, ANDIx, 0x70000000) \
+ /* AND Immediate Shifted & record CR0 */ \
+ V(andisx, ANDISx, 0x74000000) \
+ /* Compare Immediate */ \
+ V(cmpi, CMPI, 0x2C000000) \
+ /* Compare Logical Immediate */ \
+ V(cmpli, CMPLI, 0x28000000) \
+ /* Load Byte and Zero */ \
+ V(lbz, LBZ, 0x88000000) \
+ /* Load Byte and Zero with Update */ \
+ V(lbzu, LBZU, 0x8C000000) \
+ /* Load Halfword Algebraic */ \
+ V(lha, LHA, 0xA8000000) \
+ /* Load Halfword Algebraic with Update */ \
+ V(lhau, LHAU, 0xAC000000) \
+ /* Load Halfword and Zero */ \
+ V(lhz, LHZ, 0xA0000000) \
+ /* Load Halfword and Zero with Update */ \
+ V(lhzu, LHZU, 0xA4000000) \
+ /* Load Multiple Word */ \
+ V(lmw, LMW, 0xB8000000) \
+ /* Load Word and Zero */ \
+ V(lwz, LWZ, 0x80000000) \
+ /* Load Word and Zero with Update */ \
+ V(lwzu, LWZU, 0x84000000) \
+ /* Multiply Low Immediate */ \
+ V(mulli, MULLI, 0x1C000000) \
+ /* OR Immediate */ \
+ V(ori, ORI, 0x60000000) \
+ /* OR Immediate Shifted */ \
+ V(oris, ORIS, 0x64000000) \
+ /* Store Byte */ \
+ V(stb, STB, 0x98000000) \
+ /* Store Byte with Update */ \
+ V(stbu, STBU, 0x9C000000) \
+ /* Store Halfword */ \
+ V(sth, STH, 0xB0000000) \
+ /* Store Halfword with Update */ \
+ V(sthu, STHU, 0xB4000000) \
+ /* Store Multiple Word */ \
+ V(stmw, STMW, 0xBC000000) \
+ /* Store Word */ \
+ V(stw, STW, 0x90000000) \
+ /* Store Word with Update */ \
+ V(stwu, STWU, 0x94000000) \
+ /* Subtract From Immediate Carrying */ \
+ V(subfic, SUBFIC, 0x20000000) \
+ /* Trap Word Immediate */ \
+ V(twi, TWI, 0x0C000000) \
+ /* XOR Immediate */ \
+ V(xori, XORI, 0x68000000) \
+ /* XOR Immediate Shifted */ \
+ V(xoris, XORIS, 0x6C000000) \
+ /* Load Floating-Point Double */ \
+ V(lfd, LFD, 0xC8000000) \
+ /* Load Floating-Point Double with Update */ \
+ V(lfdu, LFDU, 0xCC000000) \
+ /* Load Floating-Point Single */ \
+ V(lfs, LFS, 0xC0000000) \
+ /* Load Floating-Point Single with Update */ \
+ V(lfsu, LFSU, 0xC4000000) \
+ /* Store Floating-Point Double */ \
+ V(stfd, STFD, 0xD8000000) \
+ /* Store Floating-Point Double with Update */ \
+ V(stfdu, STFDU, 0xDC000000) \
+ /* Store Floating-Point Single */ \
+ V(stfs, STFS, 0xD0000000) \
+ /* Store Floating-Point Single with Update */ \
+ V(stfsu, STFSU, 0xD4000000)
+
+#define PPC_XFL_OPCODE_LIST(V) \
+ /* Move To FPSCR Fields */ \
+ V(mtfsf, MTFSF, 0xFC00058E)
+
+#define PPC_XFX_OPCODE_LIST(V) \
+ /* Move From Condition Register */ \
+ V(mfcr, MFCR, 0x7C000026) \
+ /* Move From One Condition Register Field */ \
+ V(mfocrf, MFOCRF, 0x7C100026) \
+ /* Move From Special Purpose Register */ \
+ V(mfspr, MFSPR, 0x7C0002A6) \
+ /* Move To Condition Register Fields */ \
+ V(mtcrf, MTCRF, 0x7C000120) \
+ /* Move To One Condition Register Field */ \
+ V(mtocrf, MTOCRF, 0x7C100120) \
+ /* Move To Special Purpose Register */ \
+ V(mtspr, MTSPR, 0x7C0003A6) \
+ /* Debugger Notify Halt */ \
+ V(dnh, DNH, 0x4C00018C) \
+ /* Move From Device Control Register */ \
+ V(mfdcr, MFDCR, 0x7C000286) \
+ /* Move To Device Control Register */ \
+ V(mtdcr, MTDCR, 0x7C000386) \
+ /* Move from Performance Monitor Register */ \
+ V(mfpmr, MFPMR, 0x7C00029C) \
+ /* Move To Performance Monitor Register */ \
+ V(mtpmr, MTPMR, 0x7C00039C) \
+ /* Move From Branch History Rolling Buffer */ \
+ V(mfbhrbe, MFBHRBE, 0x7C00025C) \
+ /* Move From Time Base */ \
+ V(mftb, MFTB, 0x7C0002E6)
+
+#define PPC_MDS_OPCODE_LIST(V) \
+ /* Rotate Left Doubleword then Clear Left */ \
+ V(rldcl, RLDCL, 0x78000010) \
+ /* Rotate Left Doubleword then Clear Right */ \
+ V(rldcr, RLDCR, 0x78000012)
+
+#define PPC_A_OPCODE_LIST(V) \
+ /* Integer Select */ \
+ V(isel, ISEL, 0x7C00001E) \
+ /* Floating Add */ \
+ V(fadd, FADD, 0xFC00002A) \
+ /* Floating Add Single */ \
+ V(fadds, FADDS, 0xEC00002A) \
+ /* Floating Divide */ \
+ V(fdiv, FDIV, 0xFC000024) \
+ /* Floating Divide Single */ \
+ V(fdivs, FDIVS, 0xEC000024) \
+ /* Floating Multiply-Add */ \
+ V(fmadd, FMADD, 0xFC00003A) \
+ /* Floating Multiply-Add Single */ \
+ V(fmadds, FMADDS, 0xEC00003A) \
+ /* Floating Multiply-Subtract */ \
+ V(fmsub, FMSUB, 0xFC000038) \
+ /* Floating Multiply-Subtract Single */ \
+ V(fmsubs, FMSUBS, 0xEC000038) \
+ /* Floating Multiply */ \
+ V(fmul, FMUL, 0xFC000032) \
+ /* Floating Multiply Single */ \
+ V(fmuls, FMULS, 0xEC000032) \
+ /* Floating Negative Multiply-Add */ \
+ V(fnmadd, FNMADD, 0xFC00003E) \
+ /* Floating Negative Multiply-Add Single */ \
+ V(fnmadds, FNMADDS, 0xEC00003E) \
+ /* Floating Negative Multiply-Subtract */ \
+ V(fnmsub, FNMSUB, 0xFC00003C) \
+ /* Floating Negative Multiply-Subtract Single */ \
+ V(fnmsubs, FNMSUBS, 0xEC00003C) \
+ /* Floating Reciprocal Estimate Single */ \
+ V(fres, FRES, 0xEC000030) \
+ /* Floating Reciprocal Square Root Estimate */ \
+ V(frsqrte, FRSQRTE, 0xFC000034) \
+ /* Floating Select */ \
+ V(fsel, FSEL, 0xFC00002E) \
+ /* Floating Square Root */ \
+ V(fsqrt, FSQRT, 0xFC00002C) \
+ /* Floating Square Root Single */ \
+ V(fsqrts, FSQRTS, 0xEC00002C) \
+ /* Floating Subtract */ \
+ V(fsub, FSUB, 0xFC000028) \
+ /* Floating Subtract Single */ \
+ V(fsubs, FSUBS, 0xEC000028) \
+ /* Floating Reciprocal Estimate */ \
+ V(fre, FRE, 0xFC000030) \
+ /* Floating Reciprocal Square Root Estimate Single */ \
+ V(frsqrtes, FRSQRTES, 0xEC000034)
+
+#define PPC_VA_OPCODE_A_FORM_LIST(V) \
+ /* Vector Permute */ \
+ V(vperm, VPERM, 0x1000002B) \
+ /* Vector Multiply-Low-Add Unsigned Halfword Modulo */ \
+ V(vmladduhm, VMLADDUHM, 0x10000022) \
+ /* Vector Select */ \
+ V(vsel, VSEL, 0x1000002A) \
+ /* Vector Multiply-Sum Signed Halfword Modulo */ \
+ V(vmsumshm, VMSUMSHM, 0x10000028)
+
+#define PPC_VA_OPCODE_UNUSED_LIST(V) \
+ /* Vector Add Extended & write Carry Unsigned Quadword */ \
+ V(vaddecuq, VADDECUQ, 0x1000003D) \
+ /* Vector Add Extended Unsigned Quadword Modulo */ \
+ V(vaddeuqm, VADDEUQM, 0x1000003C) \
+ /* Vector Multiply-Add Single-Precision */ \
+ V(vmaddfp, VMADDFP, 0x1000002E) \
+ /* Vector Multiply-High-Add Signed Halfword Saturate */ \
+ V(vmhaddshs, VMHADDSHS, 0x10000020) \
+ /* Vector Multiply-High-Round-Add Signed Halfword Saturate */ \
+ V(vmhraddshs, VMHRADDSHS, 0x10000021) \
+ /* Vector Multiply-Sum Mixed Byte Modulo */ \
+ V(vmsummbm, VMSUMMBM, 0x10000025) \
+ /* Vector Multiply-Sum Signed Halfword Saturate */ \
+ V(vmsumshs, VMSUMSHS, 0x10000029) \
+ /* Vector Multiply-Sum Unsigned Byte Modulo */ \
+ V(vmsumubm, VMSUMUBM, 0x10000024) \
+ /* Vector Multiply-Sum Unsigned Halfword Modulo */ \
+ V(vmsumuhm, VMSUMUHM, 0x10000026) \
+ /* Vector Multiply-Sum Unsigned Halfword Saturate */ \
+ V(vmsumuhs, VMSUMUHS, 0x10000027) \
+ /* Vector Negative Multiply-Subtract Single-Precision */ \
+ V(vnmsubfp, VNMSUBFP, 0x1000002F) \
+ /* Vector Shift Left Double by Octet Immediate */ \
+ V(vsldoi, VSLDOI, 0x1000002C) \
+ /* Vector Subtract Extended & write Carry Unsigned Quadword */ \
+ V(vsubecuq, VSUBECUQ, 0x1000003F) \
+ /* Vector Subtract Extended Unsigned Quadword Modulo */ \
+ V(vsubeuqm, VSUBEUQM, 0x1000003E) \
+ /* Vector Permute and Exclusive-OR */ \
+ V(vpermxor, VPERMXOR, 0x1000002D)
+
+#define PPC_VA_OPCODE_LIST(V) \
+ PPC_VA_OPCODE_A_FORM_LIST(V) \
+ PPC_VA_OPCODE_UNUSED_LIST(V)
+
+#define PPC_XX1_OPCODE_LIST(V) \
+ /* Load VSR Scalar Doubleword Indexed */ \
+ V(lxsdx, LXSDX, 0x7C000498) \
+ /* Load VSX Scalar as Integer Word Algebraic Indexed */ \
+ V(lxsiwax, LXSIWAX, 0x7C000098) \
+ /* Load VSX Scalar as Integer Word and Zero Indexed */ \
+ V(lxsiwzx, LXSIWZX, 0x7C000018) \
+ /* Load VSX Scalar Single-Precision Indexed */ \
+ V(lxsspx, LXSSPX, 0x7C000418) \
+ /* Load VSR Vector Doubleword*2 Indexed */ \
+ V(lxvd, LXVD, 0x7C000698) \
+ /* Load VSR Vector Doubleword & Splat Indexed */ \
+ V(lxvdsx, LXVDSX, 0x7C000298) \
+ /* Load VSR Vector Word*4 Indexed */ \
+ V(lxvw, LXVW, 0x7C000618) \
+ /* Move From VSR Doubleword */ \
+ V(mfvsrd, MFVSRD, 0x7C000066) \
+ /* Move From VSR Word and Zero */ \
+ V(mfvsrwz, MFVSRWZ, 0x7C0000E6) \
+ /* Store VSR Scalar Doubleword Indexed */ \
+ V(stxsdx, STXSDX, 0x7C000598) \
+ /* Store VSX Scalar as Integer Word Indexed */ \
+ V(stxsiwx, STXSIWX, 0x7C000118) \
+ /* Store VSR Scalar Word Indexed */ \
+ V(stxsspx, STXSSPX, 0x7C000518) \
+ /* Store VSR Vector Doubleword*2 Indexed */ \
+ V(stxvd, STXVD, 0x7C000798) \
+ /* Store VSR Vector Word*4 Indexed */ \
+ V(stxvw, STXVW, 0x7C000718) \
+ /* Vector Splat Immediate Byte */ \
+ V(xxspltib, XXSPLTIB, 0xF00002D1)
+
+#define PPC_B_OPCODE_LIST(V) \
+ /* Branch Conditional */ \
+ V(bc, BCX, 0x40000000)
+
+#define PPC_XO_OPCODE_LIST(V) \
+ /* Divide Doubleword */ \
+ V(divd, DIVD, 0x7C0003D2) \
+ /* Divide Doubleword Extended */ \
+ V(divde, DIVDE, 0x7C000352) \
+ /* Divide Doubleword Extended & record OV */ \
+ V(divdeo, DIVDEO, 0x7C000752) \
+ /* Divide Doubleword Extended Unsigned */ \
+ V(divdeu, DIVDEU, 0x7C000312) \
+ /* Divide Doubleword Extended Unsigned & record OV */ \
+ V(divdeuo, DIVDEUO, 0x7C000712) \
+ /* Divide Doubleword & record OV */ \
+ V(divdo, DIVDO, 0x7C0007D2) \
+ /* Divide Doubleword Unsigned */ \
+ V(divdu, DIVDU, 0x7C000392) \
+ /* Divide Doubleword Unsigned & record OV */ \
+ V(divduo, DIVDUO, 0x7C000792) \
+ /* Multiply High Doubleword */ \
+ V(mulhd, MULHD, 0x7C000092) \
+ /* Multiply High Doubleword Unsigned */ \
+ V(mulhdu, MULHDU, 0x7C000012) \
+ /* Multiply Low Doubleword */ \
+ V(mulld, MULLD, 0x7C0001D2) \
+ /* Multiply Low Doubleword & record OV */ \
+ V(mulldo, MULLDO, 0x7C0005D2) \
+ /* Add */ \
+ V(add, ADDX, 0x7C000214) \
+ /* Add Carrying */ \
+ V(addc, ADDCX, 0x7C000014) \
+ /* Add Carrying & record OV */ \
+ V(addco, ADDCO, 0x7C000414) \
+ /* Add Extended */ \
+ V(adde, ADDEX, 0x7C000114) \
+ /* Add Extended & record OV & record OV */ \
+ V(addeo, ADDEO, 0x7C000514) \
+ /* Add to Minus One Extended */ \
+ V(addme, ADDME, 0x7C0001D4) \
+ /* Add to Minus One Extended & record OV */ \
+ V(addmeo, ADDMEO, 0x7C0005D4) \
+ /* Add & record OV */ \
+ V(addo, ADDO, 0x7C000614) \
+ /* Add to Zero Extended */ \
+ V(addze, ADDZEX, 0x7C000194) \
+ /* Add to Zero Extended & record OV */ \
+ V(addzeo, ADDZEO, 0x7C000594) \
+ /* Divide Word Format */ \
+ V(divw, DIVW, 0x7C0003D6) \
+ /* Divide Word Extended */ \
+ V(divwe, DIVWE, 0x7C000356) \
+ /* Divide Word Extended & record OV */ \
+ V(divweo, DIVWEO, 0x7C000756) \
+ /* Divide Word Extended Unsigned */ \
+ V(divweu, DIVWEU, 0x7C000316) \
+ /* Divide Word Extended Unsigned & record OV */ \
+ V(divweuo, DIVWEUO, 0x7C000716) \
+ /* Divide Word & record OV */ \
+ V(divwo, DIVWO, 0x7C0007D6) \
+ /* Divide Word Unsigned */ \
+ V(divwu, DIVWU, 0x7C000396) \
+ /* Divide Word Unsigned & record OV */ \
+ V(divwuo, DIVWUO, 0x7C000796) \
+ /* Multiply High Word */ \
+ V(mulhw, MULHWX, 0x7C000096) \
+ /* Multiply High Word Unsigned */ \
+ V(mulhwu, MULHWUX, 0x7C000016) \
+ /* Multiply Low Word */ \
+ V(mullw, MULLW, 0x7C0001D6) \
+ /* Multiply Low Word & record OV */ \
+ V(mullwo, MULLWO, 0x7C0005D6) \
+ /* Negate */ \
+ V(neg, NEGX, 0x7C0000D0) \
+ /* Negate & record OV */ \
+ V(nego, NEGO, 0x7C0004D0) \
+ /* Subtract From */ \
+ V(subf, SUBFX, 0x7C000050) \
+ /* Subtract From Carrying */ \
+ V(subfc, SUBFCX, 0x7C000010) \
+ /* Subtract From Carrying & record OV */ \
+ V(subfco, SUBFCO, 0x7C000410) \
+ /* Subtract From Extended */ \
+ V(subfe, SUBFEX, 0x7C000110) \
+ /* Subtract From Extended & record OV */ \
+ V(subfeo, SUBFEO, 0x7C000510) \
+ /* Subtract From Minus One Extended */ \
+ V(subfme, SUBFME, 0x7C0001D0) \
+ /* Subtract From Minus One Extended & record OV */ \
+ V(subfmeo, SUBFMEO, 0x7C0005D0) \
+ /* Subtract From & record OV */ \
+ V(subfo, SUBFO, 0x7C000450) \
+ /* Subtract From Zero Extended */ \
+ V(subfze, SUBFZE, 0x7C000190) \
+ /* Subtract From Zero Extended & record OV */ \
+ V(subfzeo, SUBFZEO, 0x7C000590) \
+ /* Add and Generate Sixes */ \
+ V(addg, ADDG, 0x7C000094) \
+ /* Multiply Accumulate Cross Halfword to Word Modulo Signed */ \
+ V(macchw, MACCHW, 0x10000158) \
+ /* Multiply Accumulate Cross Halfword to Word Saturate Signed */ \
+ V(macchws, MACCHWS, 0x100001D8) \
+ /* Multiply Accumulate Cross Halfword to Word Saturate Unsigned */ \
+ V(macchwsu, MACCHWSU, 0x10000198) \
+ /* Multiply Accumulate Cross Halfword to Word Modulo Unsigned */ \
+ V(macchwu, MACCHWU, 0x10000118) \
+ /* Multiply Accumulate High Halfword to Word Modulo Signed */ \
+ V(machhw, MACHHW, 0x10000058) \
+ /* Multiply Accumulate High Halfword to Word Saturate Signed */ \
+ V(machhws, MACHHWS, 0x100000D8) \
+ /* Multiply Accumulate High Halfword to Word Saturate Unsigned */ \
+ V(machhwsu, MACHHWSU, 0x10000098) \
+ /* Multiply Accumulate High Halfword to Word Modulo Unsigned */ \
+ V(machhwu, MACHHWU, 0x10000018) \
+ /* Multiply Accumulate Low Halfword to Word Modulo Signed */ \
+ V(maclhw, MACLHW, 0x10000358) \
+ /* Multiply Accumulate Low Halfword to Word Saturate Signed */ \
+ V(maclhws, MACLHWS, 0x100003D8) \
+ /* Multiply Accumulate Low Halfword to Word Saturate Unsigned */ \
+ V(maclhwsu, MACLHWSU, 0x10000398) \
+ /* Multiply Accumulate Low Halfword to Word Modulo Unsigned */ \
+ V(maclhwu, MACLHWU, 0x10000318) \
+ /* Negative Multiply Accumulate Cross Halfword to Word Modulo Signed */ \
+ V(nmacchw, NMACCHW, 0x1000015C) \
+ /* Negative Multiply Accumulate Cross Halfword to Word Saturate Signed */ \
+ V(nmacchws, NMACCHWS, 0x100001DC) \
+ /* Negative Multiply Accumulate High Halfword to Word Modulo Signed */ \
+ V(nmachhw, NMACHHW, 0x1000005C) \
+ /* Negative Multiply Accumulate High Halfword to Word Saturate Signed */ \
+ V(nmachhws, NMACHHWS, 0x100000DC) \
+ /* Negative Multiply Accumulate Low Halfword to Word Modulo Signed */ \
+ V(nmaclhw, NMACLHW, 0x1000035C) \
+ /* Negative Multiply Accumulate Low Halfword to Word Saturate Signed */ \
+ V(nmaclhws, NMACLHWS, 0x100003DC)
+
+#define PPC_XL_OPCODE_LIST(V) \
+ /* Branch Conditional to Count Register */ \
+ V(bcctr, BCCTRX, 0x4C000420) \
+ /* Branch Conditional to Link Register */ \
+ V(bclr, BCLRX, 0x4C000020) \
+ /* Condition Register AND */ \
+ V(crand, CRAND, 0x4C000202) \
+ /* Condition Register AND with Complement */ \
+ V(crandc, CRANDC, 0x4C000102) \
+ /* Condition Register Equivalent */ \
+ V(creqv, CREQV, 0x4C000242) \
+ /* Condition Register NAND */ \
+ V(crnand, CRNAND, 0x4C0001C2) \
+ /* Condition Register NOR */ \
+ V(crnor, CRNOR, 0x4C000042) \
+ /* Condition Register OR */ \
+ V(cror, CROR, 0x4C000382) \
+ /* Condition Register OR with Complement */ \
+ V(crorc, CRORC, 0x4C000342) \
+ /* Condition Register XOR */ \
+ V(crxor, CRXOR, 0x4C000182) \
+ /* Instruction Synchronize */ \
+ V(isync, ISYNC, 0x4C00012C) \
+ /* Move Condition Register Field */ \
+ V(mcrf, MCRF, 0x4C000000) \
+ /* Return From Critical Interrupt */ \
+ V(rfci, RFCI, 0x4C000066) \
+ /* Return From Interrupt */ \
+ V(rfi, RFI, 0x4C000064) \
+ /* Return From Machine Check Interrupt */ \
+ V(rfmci, RFMCI, 0x4C00004C) \
+ /* Embedded Hypervisor Privilege */ \
+ V(ehpriv, EHPRIV, 0x7C00021C) \
+ /* Return From Guest Interrupt */ \
+ V(rfgi, RFGI, 0x4C0000CC) \
+ /* Doze */ \
+ V(doze, DOZE, 0x4C000324) \
+ /* Return From Interrupt Doubleword Hypervisor */ \
+ V(hrfid, HRFID, 0x4C000224) \
+ /* Nap */ \
+ V(nap, NAP, 0x4C000364) \
+ /* Return from Event Based Branch */ \
+ V(rfebb, RFEBB, 0x4C000124) \
+ /* Return from Interrupt Doubleword */ \
+ V(rfid, RFID, 0x4C000024) \
+ /* Rip Van Winkle */ \
+ V(rvwinkle, RVWINKLE, 0x4C0003E4) \
+ /* Sleep */ \
+ V(sleep, SLEEP, 0x4C0003A4)
+
+#define PPC_XX4_OPCODE_LIST(V) \
+ /* VSX Select */ \
+ V(xxsel, XXSEL, 0xF0000030)
+
+#define PPC_I_OPCODE_LIST(V) \
+ /* Branch */ \
+ V(b, BX, 0x48000000)
+
+#define PPC_M_OPCODE_LIST(V) \
+ /* Rotate Left Word Immediate then Mask Insert */ \
+ V(rlwimi, RLWIMIX, 0x50000000) \
+ /* Rotate Left Word Immediate then AND with Mask */ \
+ V(rlwinm, RLWINMX, 0x54000000) \
+ /* Rotate Left Word then AND with Mask */ \
+ V(rlwnm, RLWNMX, 0x5C000000)
+
+#define PPC_VX_OPCODE_A_FORM_LIST(V) \
+ /* Vector Splat Byte */ \
+ V(vspltb, VSPLTB, 0x1000020C) \
+ /* Vector Splat Word */ \
+ V(vspltw, VSPLTW, 0x1000028C) \
+ /* Vector Splat Halfword */ \
+ V(vsplth, VSPLTH, 0x1000024C) \
+ /* Vector Extract Unsigned Byte */ \
+ V(vextractub, VEXTRACTUB, 0x1000020D) \
+ /* Vector Extract Unsigned Halfword */ \
+ V(vextractuh, VEXTRACTUH, 0x1000024D) \
+ /* Vector Extract Unsigned Word */ \
+ V(vextractuw, VEXTRACTUW, 0x1000028D) \
+ /* Vector Extract Doubleword */ \
+ V(vextractd, VEXTRACTD, 0x100002CD) \
+ /* Vector Insert Byte */ \
+ V(vinsertb, VINSERTB, 0x1000030D) \
+ /* Vector Insert Halfword */ \
+ V(vinserth, VINSERTH, 0x1000034D) \
+ /* Vector Insert Word */ \
+ V(vinsertw, VINSERTW, 0x1000038D) \
+ /* Vector Insert Doubleword */ \
+ V(vinsertd, VINSERTD, 0x100003CD)
+
+#define PPC_VX_OPCODE_B_FORM_LIST(V) \
+ /* Vector Logical OR */ \
+ V(vor, VOR, 0x10000484) \
+ /* Vector Logical XOR */ \
+ V(vxor, VXOR, 0x100004C4) \
+ /* Vector Logical NOR */ \
+ V(vnor, VNOR, 0x10000504) \
+ /* Vector Shift Right by Octet */ \
+ V(vsro, VSRO, 0x1000044C) \
+ /* Vector Shift Left by Octet */ \
+ V(vslo, VSLO, 0x1000040C) \
+ /* Vector Add Unsigned Doubleword Modulo */ \
+ V(vaddudm, VADDUDM, 0x100000C0) \
+ /* Vector Add Unsigned Word Modulo */ \
+ V(vadduwm, VADDUWM, 0x10000080) \
+ /* Vector Add Unsigned Halfword Modulo */ \
+ V(vadduhm, VADDUHM, 0x10000040) \
+ /* Vector Add Unsigned Byte Modulo */ \
+ V(vaddubm, VADDUBM, 0x10000000) \
+ /* Vector Add Single-Precision */ \
+ V(vaddfp, VADDFP, 0x1000000A) \
+ /* Vector Subtract Single-Precision */ \
+ V(vsubfp, VSUBFP, 0x1000004A) \
+ /* Vector Subtract Unsigned Doubleword Modulo */ \
+ V(vsubudm, VSUBUDM, 0x100004C0) \
+ /* Vector Subtract Unsigned Word Modulo */ \
+ V(vsubuwm, VSUBUWM, 0x10000480) \
+ /* Vector Subtract Unsigned Halfword Modulo */ \
+ V(vsubuhm, VSUBUHM, 0x10000440) \
+ /* Vector Subtract Unsigned Byte Modulo */ \
+ V(vsububm, VSUBUBM, 0x10000400) \
+ /* Vector Multiply Unsigned Word Modulo */ \
+ V(vmuluwm, VMULUWM, 0x10000089) \
+ /* Vector Pack Unsigned Halfword Unsigned Modulo */ \
+ V(vpkuhum, VPKUHUM, 0x1000000E) \
+ /* Vector Multiply Even Unsigned Byte */ \
+ V(vmuleub, VMULEUB, 0x10000208) \
+ /* Vector Multiply Odd Unsigned Byte */ \
+ V(vmuloub, VMULOUB, 0x10000008) \
+ /* Vector Sum across Quarter Signed Halfword Saturate */ \
+ V(vsum4shs, VSUM4SHS, 0x10000648) \
+ /* Vector Pack Unsigned Word Unsigned Saturate */ \
+ V(vpkuwus, VPKUWUS, 0x100000CE) \
+ /* Vector Sum across Half Signed Word Saturate */ \
+ V(vsum2sws, VSUM2SWS, 0x10000688) \
+ /* Vector Pack Unsigned Doubleword Unsigned Modulo */ \
+ V(vpkudum, VPKUDUM, 0x1000044E) \
+ /* Vector Maximum Signed Byte */ \
+ V(vmaxsb, VMAXSB, 0x10000102) \
+ /* Vector Maximum Unsigned Byte */ \
+ V(vmaxub, VMAXUB, 0x10000002) \
+ /* Vector Maximum Signed Doubleword */ \
+ V(vmaxsd, VMAXSD, 0x100001C2) \
+ /* Vector Maximum Unsigned Doubleword */ \
+ V(vmaxud, VMAXUD, 0x100000C2) \
+ /* Vector Maximum Signed Halfword */ \
+ V(vmaxsh, VMAXSH, 0x10000142) \
+ /* Vector Maximum Unsigned Halfword */ \
+ V(vmaxuh, VMAXUH, 0x10000042) \
+ /* Vector Maximum Signed Word */ \
+ V(vmaxsw, VMAXSW, 0x10000182) \
+ /* Vector Maximum Unsigned Word */ \
+ V(vmaxuw, VMAXUW, 0x10000082) \
+ /* Vector Minimum Signed Byte */ \
+ V(vminsb, VMINSB, 0x10000302) \
+ /* Vector Minimum Unsigned Byte */ \
+ V(vminub, VMINUB, 0x10000202) \
+ /* Vector Minimum Signed Doubleword */ \
+ V(vminsd, VMINSD, 0x100003C2) \
+ /* Vector Minimum Unsigned Doubleword */ \
+ V(vminud, VMINUD, 0x100002C2) \
+ /* Vector Minimum Signed Halfword */ \
+ V(vminsh, VMINSH, 0x10000342) \
+ /* Vector Minimum Unsigned Halfword */ \
+ V(vminuh, VMINUH, 0x10000242) \
+ /* Vector Minimum Signed Word */ \
+ V(vminsw, VMINSW, 0x10000382) \
+ /* Vector Minimum Unsigned Word */ \
+ V(vminuw, VMINUW, 0x10000282) \
+ /* Vector Shift Left Byte */ \
+ V(vslb, VSLB, 0x10000104) \
+ /* Vector Shift Left Word */ \
+ V(vslw, VSLW, 0x10000184) \
+ /* Vector Shift Left Halfword */ \
+ V(vslh, VSLH, 0x10000144) \
+ /* Vector Shift Left Doubleword */ \
+ V(vsld, VSLD, 0x100005C4) \
+ /* Vector Shift Right Byte */ \
+ V(vsrb, VSRB, 0x10000204) \
+ /* Vector Shift Right Word */ \
+ V(vsrw, VSRW, 0x10000284) \
+ /* Vector Shift Right Halfword */ \
+ V(vsrh, VSRH, 0x10000244) \
+ /* Vector Shift Right Doubleword */ \
+ V(vsrd, VSRD, 0x100006C4) \
+ /* Vector Shift Right Algebraic Byte */ \
+ V(vsrab, VSRAB, 0x10000304) \
+ /* Vector Shift Right Algebraic Word */ \
+ V(vsraw, VSRAW, 0x10000384) \
+ /* Vector Shift Right Algebraic Halfword */ \
+ V(vsrah, VSRAH, 0x10000344) \
+ /* Vector Shift Right Algebraic Doubleword */ \
+ V(vsrad, VSRAD, 0x100003C4) \
+ /* Vector Logical AND */ \
+ V(vand, VAND, 0x10000404) \
+ /* Vector Pack Signed Word Signed Saturate */ \
+ V(vpkswss, VPKSWSS, 0x100001CE) \
+ /* Vector Pack Signed Word Unsigned Saturate */ \
+ V(vpkswus, VPKSWUS, 0x1000014E) \
+ /* Vector Pack Signed Halfword Signed Saturate */ \
+ V(vpkshss, VPKSHSS, 0x1000018E) \
+ /* Vector Pack Signed Halfword Unsigned Saturate */ \
+ V(vpkshus, VPKSHUS, 0x1000010E) \
+ /* Vector Add Signed Halfword Saturate */ \
+ V(vaddshs, VADDSHS, 0x10000340) \
+ /* Vector Subtract Signed Halfword Saturate */ \
+ V(vsubshs, VSUBSHS, 0x10000740) \
+ /* Vector Add Unsigned Halfword Saturate */ \
+ V(vadduhs, VADDUHS, 0x10000240) \
+ /* Vector Subtract Unsigned Halfword Saturate */ \
+ V(vsubuhs, VSUBUHS, 0x10000640) \
+ /* Vector Add Signed Byte Saturate */ \
+ V(vaddsbs, VADDSBS, 0x10000300) \
+ /* Vector Subtract Signed Byte Saturate */ \
+ V(vsubsbs, VSUBSBS, 0x10000700) \
+ /* Vector Add Unsigned Byte Saturate */ \
+ V(vaddubs, VADDUBS, 0x10000200) \
+ /* Vector Subtract Unsigned Byte Saturate */ \
+ V(vsububs, VSUBUBS, 0x10000600) \
+ /* Vector Average Unsigned Byte */ \
+ V(vavgub, VAVGUB, 0x10000402) \
+ /* Vector Average Unsigned Halfword */ \
+ V(vavguh, VAVGUH, 0x10000442) \
+ /* Vector Logical AND with Complement */ \
+ V(vandc, VANDC, 0x10000444) \
+ /* Vector Minimum Single-Precision */ \
+ V(vminfp, VMINFP, 0x1000044A) \
+ /* Vector Maximum Single-Precision */ \
+ V(vmaxfp, VMAXFP, 0x1000040A) \
+ /* Vector Bit Permute Quadword */ \
+ V(vbpermq, VBPERMQ, 0x1000054C)
+
+#define PPC_VX_OPCODE_C_FORM_LIST(V) \
+ /* Vector Unpack Low Signed Halfword */ \
+ V(vupklsh, VUPKLSH, 0x100002CE) \
+ /* Vector Unpack High Signed Halfword */ \
+ V(vupkhsh, VUPKHSH, 0x1000024E) \
+ /* Vector Unpack Low Signed Byte */ \
+ V(vupklsb, VUPKLSB, 0x1000028E) \
+ /* Vector Unpack High Signed Byte */ \
+ V(vupkhsb, VUPKHSB, 0x1000020E)
+
+#define PPC_VX_OPCODE_UNUSED_LIST(V) \
+ /* Decimal Add Modulo */ \
+ V(bcdadd, BCDADD, 0xF0000400) \
+ /* Decimal Subtract Modulo */ \
+ V(bcdsub, BCDSUB, 0xF0000440) \
+ /* Move From Vector Status and Control Register */ \
+ V(mfvscr, MFVSCR, 0x10000604) \
+ /* Move To Vector Status and Control Register */ \
+ V(mtvscr, MTVSCR, 0x10000644) \
+ /* Vector Add & write Carry Unsigned Quadword */ \
+ V(vaddcuq, VADDCUQ, 0x10000140) \
+ /* Vector Add and Write Carry-Out Unsigned Word */ \
+ V(vaddcuw, VADDCUW, 0x10000180) \
+ /* Vector Add Signed Word Saturate */ \
+ V(vaddsws, VADDSWS, 0x10000380) \
+ /* Vector Add Unsigned Quadword Modulo */ \
+ V(vadduqm, VADDUQM, 0x10000100) \
+ /* Vector Add Unsigned Word Saturate */ \
+ V(vadduws, VADDUWS, 0x10000280) \
+ /* Vector Average Signed Byte */ \
+ V(vavgsb, VAVGSB, 0x10000502) \
+ /* Vector Average Signed Halfword */ \
+ V(vavgsh, VAVGSH, 0x10000542) \
+ /* Vector Average Signed Word */ \
+ V(vavgsw, VAVGSW, 0x10000582) \
+ /* Vector Average Unsigned Word */ \
+ V(vavguw, VAVGUW, 0x10000482) \
+ /* Vector Convert From Signed Fixed-Point Word To Single-Precision */ \
+ V(vcfsx, VCFSX, 0x1000034A) \
+ /* Vector Convert From Unsigned Fixed-Point Word To Single-Precision */ \
+ V(vcfux, VCFUX, 0x1000030A) \
+ /* Vector Count Leading Zeros Byte */ \
+ V(vclzb, VCLZB, 0x10000702) \
+ /* Vector Count Leading Zeros Doubleword */ \
+ V(vclzd, VCLZD, 0x100007C2) \
+ /* Vector Count Leading Zeros Halfword */ \
+ V(vclzh, VCLZH, 0x10000742) \
+ /* Vector Count Leading Zeros Word */ \
+ V(vclzw, VCLZW, 0x10000782) \
+ /* Vector Convert From Single-Precision To Signed Fixed-Point Word */ \
+ /* Saturate */ \
+ V(vctsxs, VCTSXS, 0x100003CA) \
+ /* Vector Convert From Single-Precision To Unsigned Fixed-Point Word */ \
+ /* Saturate */ \
+ V(vctuxs, VCTUXS, 0x1000038A) \
+ /* Vector Equivalence */ \
+ V(veqv, VEQV, 0x10000684) \
+ /* Vector 2 Raised to the Exponent Estimate Single-Precision */ \
+ V(vexptefp, VEXPTEFP, 0x1000018A) \
+ /* Vector Gather Bits by Byte by Doubleword */ \
+ V(vgbbd, VGBBD, 0x1000050C) \
+ /* Vector Log Base 2 Estimate Single-Precision */ \
+ V(vlogefp, VLOGEFP, 0x100001CA) \
+ /* Vector Merge High Byte */ \
+ V(vmrghb, VMRGHB, 0x1000000C) \
+ /* Vector Merge High Halfword */ \
+ V(vmrghh, VMRGHH, 0x1000004C) \
+ /* Vector Merge High Word */ \
+ V(vmrghw, VMRGHW, 0x1000008C) \
+ /* Vector Merge Low Byte */ \
+ V(vmrglb, VMRGLB, 0x1000010C) \
+ /* Vector Merge Low Halfword */ \
+ V(vmrglh, VMRGLH, 0x1000014C) \
+ /* Vector Merge Low Word */ \
+ V(vmrglw, VMRGLW, 0x1000018C) \
+ /* Vector Multiply Even Signed Byte */ \
+ V(vmulesb, VMULESB, 0x10000308) \
+ /* Vector Multiply Even Signed Halfword */ \
+ V(vmulesh, VMULESH, 0x10000348) \
+ /* Vector Multiply Even Signed Word */ \
+ V(vmulesw, VMULESW, 0x10000388) \
+ /* Vector Multiply Even Unsigned Halfword */ \
+ V(vmuleuh, VMULEUH, 0x10000248) \
+ /* Vector Multiply Even Unsigned Word */ \
+ V(vmuleuw, VMULEUW, 0x10000288) \
+ /* Vector Multiply Odd Signed Byte */ \
+ V(vmulosb, VMULOSB, 0x10000108) \
+ /* Vector Multiply Odd Signed Halfword */ \
+ V(vmulosh, VMULOSH, 0x10000148) \
+ /* Vector Multiply Odd Signed Word */ \
+ V(vmulosw, VMULOSW, 0x10000188) \
+ /* Vector Multiply Odd Unsigned Halfword */ \
+ V(vmulouh, VMULOUH, 0x10000048) \
+ /* Vector Multiply Odd Unsigned Word */ \
+ V(vmulouw, VMULOUW, 0x10000088) \
+ /* Vector NAND */ \
+ V(vnand, VNAND, 0x10000584) \
+ /* Vector OR with Complement */ \
+ V(vorc, VORC, 0x10000544) \
+ /* Vector Pack Pixel */ \
+ V(vpkpx, VPKPX, 0x1000030E) \
+ /* Vector Pack Signed Doubleword Signed Saturate */ \
+ V(vpksdss, VPKSDSS, 0x100005CE) \
+ /* Vector Pack Signed Doubleword Unsigned Saturate */ \
+ V(vpksdus, VPKSDUS, 0x1000054E) \
+ /* Vector Pack Unsigned Doubleword Unsigned Saturate */ \
+ V(vpkudus, VPKUDUS, 0x100004CE) \
+ /* Vector Pack Unsigned Halfword Unsigned Saturate */ \
+ V(vpkuhus, VPKUHUS, 0x1000008E) \
+ /* Vector Pack Unsigned Word Unsigned Modulo */ \
+ V(vpkuwum, VPKUWUM, 0x1000004E) \
+ /* Vector Polynomial Multiply-Sum Byte */ \
+ V(vpmsumb, VPMSUMB, 0x10000408) \
+ /* Vector Polynomial Multiply-Sum Doubleword */ \
+ V(vpmsumd, VPMSUMD, 0x100004C8) \
+ /* Vector Polynomial Multiply-Sum Halfword */ \
+ V(vpmsumh, VPMSUMH, 0x10000448) \
+ /* Vector Polynomial Multiply-Sum Word */ \
+ V(vpmsumw, VPMSUMW, 0x10000488) \
+ /* Vector Population Count Byte */ \
+ V(vpopcntb, VPOPCNTB, 0x10000703) \
+ /* Vector Population Count Doubleword */ \
+ V(vpopcntd, VPOPCNTD, 0x100007C3) \
+ /* Vector Population Count Halfword */ \
+ V(vpopcnth, VPOPCNTH, 0x10000743) \
+ /* Vector Population Count Word */ \
+ V(vpopcntw, VPOPCNTW, 0x10000783) \
+ /* Vector Reciprocal Estimate Single-Precision */ \
+ V(vrefp, VREFP, 0x1000010A) \
+ /* Vector Round to Single-Precision Integer toward -Infinity */ \
+ V(vrfim, VRFIM, 0x100002CA) \
+ /* Vector Round to Single-Precision Integer Nearest */ \
+ V(vrfin, VRFIN, 0x1000020A) \
+ /* Vector Round to Single-Precision Integer toward +Infinity */ \
+ V(vrfip, VRFIP, 0x1000028A) \
+ /* Vector Round to Single-Precision Integer toward Zero */ \
+ V(vrfiz, VRFIZ, 0x1000024A) \
+ /* Vector Rotate Left Byte */ \
+ V(vrlb, VRLB, 0x10000004) \
+ /* Vector Rotate Left Doubleword */ \
+ V(vrld, VRLD, 0x100000C4) \
+ /* Vector Rotate Left Halfword */ \
+ V(vrlh, VRLH, 0x10000044) \
+ /* Vector Rotate Left Word */ \
+ V(vrlw, VRLW, 0x10000084) \
+ /* Vector Reciprocal Square Root Estimate Single-Precision */ \
+ V(vrsqrtefp, VRSQRTEFP, 0x1000014A) \
+ /* Vector Shift Left */ \
+ V(vsl, VSL, 0x100001C4) \
+ /* Vector Splat Immediate Signed Byte */ \
+ V(vspltisb, VSPLTISB, 0x1000030C) \
+ /* Vector Splat Immediate Signed Halfword */ \
+ V(vspltish, VSPLTISH, 0x1000034C) \
+ /* Vector Splat Immediate Signed Word */ \
+ V(vspltisw, VSPLTISW, 0x1000038C) \
+ /* Vector Shift Right */ \
+ V(vsr, VSR, 0x100002C4) \
+ /* Vector Subtract & write Carry Unsigned Quadword */ \
+ V(vsubcuq, VSUBCUQ, 0x10000540) \
+ /* Vector Subtract and Write Carry-Out Unsigned Word */ \
+ V(vsubcuw, VSUBCUW, 0x10000580) \
+ /* Vector Subtract Signed Word Saturate */ \
+ V(vsubsws, VSUBSWS, 0x10000780) \
+ /* Vector Subtract Unsigned Quadword Modulo */ \
+ V(vsubuqm, VSUBUQM, 0x10000500) \
+ /* Vector Subtract Unsigned Word Saturate */ \
+ V(vsubuws, VSUBUWS, 0x10000680) \
+ /* Vector Sum across Quarter Signed Byte Saturate */ \
+ V(vsum4sbs, VSUM4SBS, 0x10000708) \
+ /* Vector Sum across Quarter Unsigned Byte Saturate */ \
+ V(vsum4bus, VSUM4BUS, 0x10000608) \
+ /* Vector Sum across Signed Word Saturate */ \
+ V(vsumsws, VSUMSWS, 0x10000788) \
+ /* Vector Unpack High Pixel */ \
+ V(vupkhpx, VUPKHPX, 0x1000034E) \
+ /* Vector Unpack High Signed Word */ \
+ V(vupkhsw, VUPKHSW, 0x1000064E) \
+ /* Vector Unpack Low Pixel */ \
+ V(vupklpx, VUPKLPX, 0x100003CE) \
+ /* Vector Unpack Low Signed Word */ \
+ V(vupklsw, VUPKLSW, 0x100006CE) \
+ /* Vector AES Cipher */ \
+ V(vcipher, VCIPHER, 0x10000508) \
+ /* Vector AES Cipher Last */ \
+ V(vcipherlast, VCIPHERLAST, 0x10000509) \
+ /* Vector AES Inverse Cipher */ \
+ V(vncipher, VNCIPHER, 0x10000548) \
+ /* Vector AES Inverse Cipher Last */ \
+ V(vncipherlast, VNCIPHERLAST, 0x10000549) \
+ /* Vector AES S-Box */ \
+ V(vsbox, VSBOX, 0x100005C8) \
+ /* Vector SHA-512 Sigma Doubleword */ \
+ V(vshasigmad, VSHASIGMAD, 0x100006C2) \
+ /* Vector SHA-256 Sigma Word */ \
+ V(vshasigmaw, VSHASIGMAW, 0x10000682) \
+ /* Vector Merge Even Word */ \
+ V(vmrgew, VMRGEW, 0x1000078C) \
+ /* Vector Merge Odd Word */ \
+ V(vmrgow, VMRGOW, 0x1000068C)
+
+#define PPC_VX_OPCODE_LIST(V) \
+ PPC_VX_OPCODE_A_FORM_LIST(V) \
+ PPC_VX_OPCODE_B_FORM_LIST(V) \
+ PPC_VX_OPCODE_C_FORM_LIST(V) \
+ PPC_VX_OPCODE_UNUSED_LIST(V)
+
+#define PPC_XS_OPCODE_LIST(V) \
+ /* Shift Right Algebraic Doubleword Immediate */ \
+ V(sradi, SRADIX, 0x7C000674)
+
+#define PPC_MD_OPCODE_LIST(V) \
+ /* Rotate Left Doubleword Immediate then Clear */ \
+ V(rldic, RLDIC, 0x78000008) \
+ /* Rotate Left Doubleword Immediate then Clear Left */ \
+ V(rldicl, RLDICL, 0x78000000) \
+ /* Rotate Left Doubleword Immediate then Clear Right */ \
+ V(rldicr, RLDICR, 0x78000004) \
+ /* Rotate Left Doubleword Immediate then Mask Insert */ \
+ V(rldimi, RLDIMI, 0x7800000C)
+
+#define PPC_SC_OPCODE_LIST(V) \
+ /* System Call */ \
+ V(sc, SC, 0x44000002)
+
+#define PPC_OPCODE_LIST(V) \
+ PPC_X_OPCODE_LIST(V) \
+ PPC_X_OPCODE_EH_S_FORM_LIST(V) \
+ PPC_XO_OPCODE_LIST(V) \
+ PPC_DS_OPCODE_LIST(V) \
+ PPC_DQ_OPCODE_LIST(V) \
+ PPC_MDS_OPCODE_LIST(V) \
+ PPC_MD_OPCODE_LIST(V) \
+ PPC_XS_OPCODE_LIST(V) \
+ PPC_D_OPCODE_LIST(V) \
+ PPC_I_OPCODE_LIST(V) \
+ PPC_B_OPCODE_LIST(V) \
+ PPC_XL_OPCODE_LIST(V) \
+ PPC_A_OPCODE_LIST(V) \
+ PPC_XFX_OPCODE_LIST(V) \
+ PPC_M_OPCODE_LIST(V) \
+ PPC_SC_OPCODE_LIST(V) \
+ PPC_Z23_OPCODE_LIST(V) \
+ PPC_Z22_OPCODE_LIST(V) \
+ PPC_EVX_OPCODE_LIST(V) \
+ PPC_XFL_OPCODE_LIST(V) \
+ PPC_EVS_OPCODE_LIST(V) \
+ PPC_VX_OPCODE_LIST(V) \
+ PPC_VA_OPCODE_LIST(V) \
+ PPC_VC_OPCODE_LIST(V) \
+ PPC_XX1_OPCODE_LIST(V) \
+ PPC_XX2_OPCODE_LIST(V) \
+ PPC_XX3_OPCODE_LIST(V) \
+ PPC_XX4_OPCODE_LIST(V)
+
+enum Opcode : uint32_t {
+#define DECLARE_INSTRUCTION(name, opcode_name, opcode_value) \
+ opcode_name = opcode_value,
+ PPC_OPCODE_LIST(DECLARE_INSTRUCTION)
+#undef DECLARE_INSTRUCTION
+ EXT0 = 0x10000000, // Extended code set 0
+ EXT1 = 0x4C000000, // Extended code set 1
+ EXT2 = 0x7C000000, // Extended code set 2
+ EXT3 = 0xEC000000, // Extended code set 3
+ EXT4 = 0xFC000000, // Extended code set 4
+ EXT5 = 0x78000000, // Extended code set 5 - 64bit only
+ EXT6 = 0xF0000000, // Extended code set 6
+};
+
+// Instruction encoding bits and masks.
+enum {
+ // Instruction encoding bit
+ B1 = 1 << 1,
+ B2 = 1 << 2,
+ B3 = 1 << 3,
+ B4 = 1 << 4,
+ B5 = 1 << 5,
+ B7 = 1 << 7,
+ B8 = 1 << 8,
+ B9 = 1 << 9,
+ B12 = 1 << 12,
+ B18 = 1 << 18,
+ B19 = 1 << 19,
+ B20 = 1 << 20,
+ B22 = 1 << 22,
+ B23 = 1 << 23,
+ B24 = 1 << 24,
+ B25 = 1 << 25,
+ B26 = 1 << 26,
+ B27 = 1 << 27,
+ B28 = 1 << 28,
+ B6 = 1 << 6,
+ B10 = 1 << 10,
+ B11 = 1 << 11,
+ B16 = 1 << 16,
+ B17 = 1 << 17,
+ B21 = 1 << 21,
+
+ // Instruction bit masks
+ kCondMask = 0x1F << 21,
+ kOff12Mask = (1 << 12) - 1,
+ kImm24Mask = (1 << 24) - 1,
+ kOff16Mask = (1 << 16) - 1,
+ kImm16Mask = (1 << 16) - 1,
+ kImm22Mask = (1 << 22) - 1,
+ kImm26Mask = (1 << 26) - 1,
+ kBOfieldMask = 0x1f << 21,
+ kOpcodeMask = 0x3f << 26,
+ kExt1OpcodeMask = 0x3ff << 1,
+ kExt2OpcodeMask = 0x3ff << 1,
+ kExt2OpcodeVariant2Mask = 0x1ff << 2,
+ kExt5OpcodeMask = 0x3 << 2,
+ kBOMask = 0x1f << 21,
+ kBIMask = 0x1F << 16,
+ kBDMask = 0x14 << 2,
+ kAAMask = 0x01 << 1,
+ kLKMask = 0x01,
+ kRCMask = 0x01,
+ kTOMask = 0x1f << 21
+};
+
+// -----------------------------------------------------------------------------
+// Addressing modes and instruction variants.
+
+// Overflow Exception
+enum OEBit {
+ SetOE = 1 << 10, // Set overflow exception
+ LeaveOE = 0 << 10 // No overflow exception
+};
+
+// Record bit
+enum RCBit { // Bit 0
+ SetRC = 1, // LT,GT,EQ,SO
+ LeaveRC = 0 // None
+};
+// Exclusive Access hint bit
+enum EHBit { // Bit 0
+ SetEH = 1, // Exclusive Access
+ LeaveEH = 0 // Atomic Update
+};
+
+// Link bit
+enum LKBit { // Bit 0
+ SetLK = 1, // Load effective address of next instruction
+ LeaveLK = 0 // No action
+};
+
+enum BOfield { // Bits 25-21
+ DCBNZF = 0 << 21, // Decrement CTR; branch if CTR != 0 and condition false
+ DCBEZF = 2 << 21, // Decrement CTR; branch if CTR == 0 and condition false
+ BF = 4 << 21, // Branch if condition false
+ DCBNZT = 8 << 21, // Decrement CTR; branch if CTR != 0 and condition true
+ DCBEZT = 10 << 21, // Decrement CTR; branch if CTR == 0 and condition true
+ BT = 12 << 21, // Branch if condition true
+ DCBNZ = 16 << 21, // Decrement CTR; branch if CTR != 0
+ DCBEZ = 18 << 21, // Decrement CTR; branch if CTR == 0
+ BA = 20 << 21 // Branch always
+};
+
+#if V8_OS_AIX
+#undef CR_LT
+#undef CR_GT
+#undef CR_EQ
+#undef CR_SO
+#endif
+
+enum CRBit { CR_LT = 0, CR_GT = 1, CR_EQ = 2, CR_SO = 3, CR_FU = 3 };
+
+#define CRWIDTH 4
+
+// These are the documented bit positions biased down by 32
+enum FPSCRBit {
+ VXSOFT = 21, // 53: Software-Defined Condition
+ VXSQRT = 22, // 54: Invalid Square Root
+ VXCVI = 23 // 55: Invalid Integer Convert
+};
+
+// -----------------------------------------------------------------------------
+// Supervisor Call (svc) specific support.
+
+// Special Software Interrupt codes when used in the presence of the PPC
+// simulator.
+// svc (formerly swi) provides a 24bit immediate value. Use bits 22:0 for
+// standard SoftwareInterrupCode. Bit 23 is reserved for the stop feature.
+enum SoftwareInterruptCodes {
+ // transition to C code
+ kCallRtRedirected = 0x10,
+ // break point
+ kBreakpoint = 0x821008, // bits23-0 of 0x7d821008 = twge r2, r2
+ // stop
+ kStopCode = 1 << 23
+};
+const uint32_t kStopCodeMask = kStopCode - 1;
+const uint32_t kMaxStopCode = kStopCode - 1;
+const int32_t kDefaultStopCode = -1;
+
+// FP rounding modes.
+enum FPRoundingMode {
+ RN = 0, // Round to Nearest.
+ RZ = 1, // Round towards zero.
+ RP = 2, // Round towards Plus Infinity.
+ RM = 3, // Round towards Minus Infinity.
+
+ // Aliases.
+ kRoundToNearest = RN,
+ kRoundToZero = RZ,
+ kRoundToPlusInf = RP,
+ kRoundToMinusInf = RM
+};
+
+const uint32_t kFPRoundingModeMask = 3;
+
+enum CheckForInexactConversion {
+ kCheckForInexactConversion,
+ kDontCheckForInexactConversion
+};
+
+// -----------------------------------------------------------------------------
+// Specific instructions, constants, and masks.
+// These constants are declared in assembler-arm.cc, as they use named registers
+// and other constants.
+
+// add(sp, sp, 4) instruction (aka Pop())
+extern const Instr kPopInstruction;
+
+// str(r, MemOperand(sp, 4, NegPreIndex), al) instruction (aka push(r))
+// register r is not encoded.
+extern const Instr kPushRegPattern;
+
+// ldr(r, MemOperand(sp, 4, PostIndex), al) instruction (aka pop(r))
+// register r is not encoded.
+extern const Instr kPopRegPattern;
+
+// use TWI to indicate redirection call for simulation mode
+const Instr rtCallRedirInstr = TWI;
+
+// -----------------------------------------------------------------------------
+// Instruction abstraction.
+
+// The class Instruction enables access to individual fields defined in the PPC
+// architecture instruction set encoding.
+// Note that the Assembler uses typedef int32_t Instr.
+//
+// Example: Test whether the instruction at ptr does set the condition code
+// bits.
+//
+// bool InstructionSetsConditionCodes(byte* ptr) {
+// Instruction* instr = Instruction::At(ptr);
+// int type = instr->TypeValue();
+// return ((type == 0) || (type == 1)) && instr->HasS();
+// }
+//
+
+constexpr uint8_t kInstrSize = 4;
+constexpr uint8_t kInstrSizeLog2 = 2;
+constexpr uint8_t kPcLoadDelta = 8;
+
+class Instruction {
+ public:
+// Helper macro to define static accessors.
+// We use the cast to char* trick to bypass the strict anti-aliasing rules.
+#define DECLARE_STATIC_TYPED_ACCESSOR(return_type, Name) \
+ static inline return_type Name(Instr instr) { \
+ char* temp = reinterpret_cast<char*>(&instr); \
+ return reinterpret_cast<Instruction*>(temp)->Name(); \
+ }
+
+#define DECLARE_STATIC_ACCESSOR(Name) DECLARE_STATIC_TYPED_ACCESSOR(int, Name)
+
+ // Get the raw instruction bits.
+ inline Instr InstructionBits() const {
+ return *reinterpret_cast<const Instr*>(this);
+ }
+
+ // Set the raw instruction bits to value.
+ inline void SetInstructionBits(Instr value) {
+ *reinterpret_cast<Instr*>(this) = value;
+ }
+
+ // Read one particular bit out of the instruction bits.
+ inline int Bit(int nr) const { return (InstructionBits() >> nr) & 1; }
+
+ // Read a bit field's value out of the instruction bits.
+ inline int Bits(int hi, int lo) const {
+ return (InstructionBits() >> lo) & ((2 << (hi - lo)) - 1);
+ }
+
+ // Read a bit field out of the instruction bits.
+ inline uint32_t BitField(int hi, int lo) const {
+ return InstructionBits() & (((2 << (hi - lo)) - 1) << lo);
+ }
+
+ // Static support.
+
+ // Read one particular bit out of the instruction bits.
+ static inline int Bit(Instr instr, int nr) { return (instr >> nr) & 1; }
+
+ // Read the value of a bit field out of the instruction bits.
+ static inline int Bits(Instr instr, int hi, int lo) {
+ return (instr >> lo) & ((2 << (hi - lo)) - 1);
+ }
+
+ // Read a bit field out of the instruction bits.
+ static inline uint32_t BitField(Instr instr, int hi, int lo) {
+ return instr & (((2 << (hi - lo)) - 1) << lo);
+ }
+
+ inline int RSValue() const { return Bits(25, 21); }
+ inline int RTValue() const { return Bits(25, 21); }
+ inline int RAValue() const { return Bits(20, 16); }
+ DECLARE_STATIC_ACCESSOR(RAValue)
+ inline int RBValue() const { return Bits(15, 11); }
+ DECLARE_STATIC_ACCESSOR(RBValue)
+ inline int RCValue() const { return Bits(10, 6); }
+ DECLARE_STATIC_ACCESSOR(RCValue)
+
+ inline int OpcodeValue() const { return static_cast<Opcode>(Bits(31, 26)); }
+ inline uint32_t OpcodeField() const {
+ return static_cast<Opcode>(BitField(31, 26));
+ }
+
+#define OPCODE_CASES(name, opcode_name, opcode_value) case opcode_name:
+
+ inline Opcode OpcodeBase() const {
+ uint32_t opcode = OpcodeField();
+ uint32_t extcode = OpcodeField();
+ switch (opcode) {
+ PPC_D_OPCODE_LIST(OPCODE_CASES)
+ PPC_I_OPCODE_LIST(OPCODE_CASES)
+ PPC_B_OPCODE_LIST(OPCODE_CASES)
+ PPC_M_OPCODE_LIST(OPCODE_CASES)
+ return static_cast<Opcode>(opcode);
+ }
+
+ opcode = extcode | BitField(10, 0);
+ switch (opcode) {
+ PPC_VX_OPCODE_LIST(OPCODE_CASES)
+ PPC_X_OPCODE_EH_S_FORM_LIST(OPCODE_CASES)
+ return static_cast<Opcode>(opcode);
+ }
+ opcode = extcode | BitField(9, 0);
+ switch (opcode) {
+ PPC_VC_OPCODE_LIST(OPCODE_CASES)
+ return static_cast<Opcode>(opcode);
+ }
+ opcode = extcode | BitField(10, 1) | BitField(20, 20);
+ switch (opcode) {
+ PPC_XFX_OPCODE_LIST(OPCODE_CASES)
+ return static_cast<Opcode>(opcode);
+ }
+ opcode = extcode | BitField(10, 1);
+ switch (opcode) {
+ PPC_X_OPCODE_LIST(OPCODE_CASES)
+ PPC_XL_OPCODE_LIST(OPCODE_CASES)
+ PPC_XFL_OPCODE_LIST(OPCODE_CASES)
+ PPC_XX1_OPCODE_LIST(OPCODE_CASES)
+ PPC_XX2_OPCODE_LIST(OPCODE_CASES)
+ PPC_EVX_OPCODE_LIST(OPCODE_CASES)
+ return static_cast<Opcode>(opcode);
+ }
+ opcode = extcode | BitField(9, 1);
+ switch (opcode) {
+ PPC_XO_OPCODE_LIST(OPCODE_CASES)
+ PPC_Z22_OPCODE_LIST(OPCODE_CASES)
+ return static_cast<Opcode>(opcode);
+ }
+ opcode = extcode | BitField(10, 2);
+ switch (opcode) {
+ PPC_XS_OPCODE_LIST(OPCODE_CASES)
+ return static_cast<Opcode>(opcode);
+ }
+ opcode = extcode | BitField(10, 3);
+ switch (opcode) {
+ PPC_EVS_OPCODE_LIST(OPCODE_CASES)
+ PPC_XX3_OPCODE_LIST(OPCODE_CASES)
+ return static_cast<Opcode>(opcode);
+ }
+ opcode = extcode | BitField(8, 1);
+ switch (opcode) {
+ PPC_Z23_OPCODE_LIST(OPCODE_CASES)
+ return static_cast<Opcode>(opcode);
+ }
+ opcode = extcode | BitField(5, 0);
+ switch (opcode) {
+ PPC_VA_OPCODE_LIST(OPCODE_CASES)
+ return static_cast<Opcode>(opcode);
+ }
+ opcode = extcode | BitField(5, 1);
+ switch (opcode) {
+ PPC_A_OPCODE_LIST(OPCODE_CASES)
+ return static_cast<Opcode>(opcode);
+ }
+ opcode = extcode | BitField(4, 1);
+ switch (opcode) {
+ PPC_MDS_OPCODE_LIST(OPCODE_CASES)
+ return static_cast<Opcode>(opcode);
+ }
+ opcode = extcode | BitField(4, 2);
+ switch (opcode) {
+ PPC_MD_OPCODE_LIST(OPCODE_CASES)
+ return static_cast<Opcode>(opcode);
+ }
+ opcode = extcode | BitField(5, 4);
+ switch (opcode) {
+ PPC_XX4_OPCODE_LIST(OPCODE_CASES)
+ return static_cast<Opcode>(opcode);
+ }
+ opcode = extcode | BitField(2, 0);
+ switch (opcode) {
+ PPC_DQ_OPCODE_LIST(OPCODE_CASES)
+ return static_cast<Opcode>(opcode);
+ }
+ opcode = extcode | BitField(1, 0);
+ switch (opcode) {
+ PPC_DS_OPCODE_LIST(OPCODE_CASES)
+ return static_cast<Opcode>(opcode);
+ }
+ opcode = extcode | BitField(1, 1);
+ switch (opcode) {
+ PPC_SC_OPCODE_LIST(OPCODE_CASES)
+ return static_cast<Opcode>(opcode);
+ }
+ UNIMPLEMENTED();
+ return static_cast<Opcode>(0);
+ }
+
+#undef OPCODE_CASES
+
+ // Fields used in Software interrupt instructions
+ inline SoftwareInterruptCodes SvcValue() const {
+ return static_cast<SoftwareInterruptCodes>(Bits(23, 0));
+ }
+
+ // Instructions are read of out a code stream. The only way to get a
+ // reference to an instruction is to convert a pointer. There is no way
+ // to allocate or create instances of class Instruction.
+ // Use the At(pc) function to create references to Instruction.
+ static Instruction* At(byte* pc) {
+ return reinterpret_cast<Instruction*>(pc);
+ }
+
+ private:
+ // We need to prevent the creation of instances of class Instruction.
+ DISALLOW_IMPLICIT_CONSTRUCTORS(Instruction);
+};
+
+// Helper functions for converting between register numbers and names.
+class Registers {
+ public:
+ // Lookup the register number for the name provided.
+ static int Number(const char* name);
+
+ private:
+ static const char* names_[kNumRegisters];
+};
+
+// Helper functions for converting between FP register numbers and names.
+class DoubleRegisters {
+ public:
+ // Lookup the register number for the name provided.
+ static int Number(const char* name);
+
+ private:
+ static const char* names_[kNumDoubleRegisters];
+};
+} // namespace internal
+} // namespace v8
+
+static constexpr int kR0DwarfCode = 0;
+static constexpr int kFpDwarfCode = 31; // frame-pointer
+static constexpr int kLrDwarfCode = 65; // return-address(lr)
+static constexpr int kSpDwarfCode = 1; // stack-pointer (sp)
+
+#endif // V8_CODEGEN_PPC_CONSTANTS_PPC_H_
diff --git a/src/codegen/ppc/cpu-ppc.cc b/src/codegen/ppc/cpu-ppc.cc
new file mode 100644
index 0000000..9559af7
--- /dev/null
+++ b/src/codegen/ppc/cpu-ppc.cc
@@ -0,0 +1,50 @@
+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// CPU specific code for ppc independent of OS goes here.
+
+#if V8_TARGET_ARCH_PPC || V8_TARGET_ARCH_PPC64
+
+#include "src/codegen/cpu-features.h"
+
+#define INSTR_AND_DATA_CACHE_COHERENCY LWSYNC
+
+namespace v8 {
+namespace internal {
+
+void CpuFeatures::FlushICache(void* buffer, size_t size) {
+#if !defined(USE_SIMULATOR)
+ if (CpuFeatures::IsSupported(INSTR_AND_DATA_CACHE_COHERENCY)) {
+ __asm__ __volatile__(
+ "sync \n"
+ "icbi 0, %0 \n"
+ "isync \n"
+ : /* no output */
+ : "r"(buffer)
+ : "memory");
+ return;
+ }
+
+ const int kCacheLineSize = CpuFeatures::icache_line_size();
+ intptr_t mask = kCacheLineSize - 1;
+ byte* start =
+ reinterpret_cast<byte*>(reinterpret_cast<intptr_t>(buffer) & ~mask);
+ byte* end = static_cast<byte*>(buffer) + size;
+ for (byte* pointer = start; pointer < end; pointer += kCacheLineSize) {
+ __asm__(
+ "dcbf 0, %0 \n"
+ "sync \n"
+ "icbi 0, %0 \n"
+ "isync \n"
+ : /* no output */
+ : "r"(pointer));
+ }
+
+#endif // !USE_SIMULATOR
+}
+} // namespace internal
+} // namespace v8
+
+#undef INSTR_AND_DATA_CACHE_COHERENCY
+#endif // V8_TARGET_ARCH_PPC || V8_TARGET_ARCH_PPC64
diff --git a/src/codegen/ppc/interface-descriptors-ppc.cc b/src/codegen/ppc/interface-descriptors-ppc.cc
new file mode 100644
index 0000000..3c2d922
--- /dev/null
+++ b/src/codegen/ppc/interface-descriptors-ppc.cc
@@ -0,0 +1,284 @@
+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#if V8_TARGET_ARCH_PPC || V8_TARGET_ARCH_PPC64
+
+#include "src/codegen/interface-descriptors.h"
+
+#include "src/execution/frames.h"
+
+namespace v8 {
+namespace internal {
+
+const Register CallInterfaceDescriptor::ContextRegister() { return cp; }
+
+void CallInterfaceDescriptor::DefaultInitializePlatformSpecific(
+ CallInterfaceDescriptorData* data, int register_parameter_count) {
+ const Register default_stub_registers[] = {r3, r4, r5, r6, r7};
+ CHECK_LE(static_cast<size_t>(register_parameter_count),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(register_parameter_count,
+ default_stub_registers);
+}
+
+void RecordWriteDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ const Register default_stub_registers[] = {r3, r4, r5, r6, r7};
+
+ data->RestrictAllocatableRegisters(default_stub_registers,
+ arraysize(default_stub_registers));
+
+ CHECK_LE(static_cast<size_t>(kParameterCount),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(kParameterCount, default_stub_registers);
+}
+
+void EphemeronKeyBarrierDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ const Register default_stub_registers[] = {r3, r4, r5, r6, r7};
+
+ data->RestrictAllocatableRegisters(default_stub_registers,
+ arraysize(default_stub_registers));
+
+ CHECK_LE(static_cast<size_t>(kParameterCount),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(kParameterCount, default_stub_registers);
+}
+
+const Register LoadDescriptor::ReceiverRegister() { return r4; }
+const Register LoadDescriptor::NameRegister() { return r5; }
+const Register LoadDescriptor::SlotRegister() { return r3; }
+
+const Register LoadWithVectorDescriptor::VectorRegister() { return r6; }
+
+const Register
+LoadWithReceiverAndVectorDescriptor::LookupStartObjectRegister() {
+ return r7;
+}
+
+const Register StoreDescriptor::ReceiverRegister() { return r4; }
+const Register StoreDescriptor::NameRegister() { return r5; }
+const Register StoreDescriptor::ValueRegister() { return r3; }
+const Register StoreDescriptor::SlotRegister() { return r7; }
+
+const Register StoreWithVectorDescriptor::VectorRegister() { return r6; }
+
+const Register StoreTransitionDescriptor::SlotRegister() { return r7; }
+const Register StoreTransitionDescriptor::VectorRegister() { return r6; }
+const Register StoreTransitionDescriptor::MapRegister() { return r8; }
+
+const Register ApiGetterDescriptor::HolderRegister() { return r3; }
+const Register ApiGetterDescriptor::CallbackRegister() { return r6; }
+
+const Register GrowArrayElementsDescriptor::ObjectRegister() { return r3; }
+const Register GrowArrayElementsDescriptor::KeyRegister() { return r6; }
+
+// static
+const Register TypeConversionDescriptor::ArgumentRegister() { return r3; }
+
+void TypeofDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {r6};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallTrampolineDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r3 : number of arguments
+ // r4 : the target to call
+ Register registers[] = {r4, r3};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r3 : number of arguments (on the stack, not including receiver)
+ // r4 : the target to call
+ // r7 : arguments list length (untagged)
+ // r5 : arguments list (FixedArray)
+ Register registers[] = {r4, r3, r7, r5};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallForwardVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r3 : number of arguments
+ // r5 : start index (to support rest parameters)
+ // r4 : the target to call
+ Register registers[] = {r4, r3, r5};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallFunctionTemplateDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r4 : function template info
+ // r5 : number of arguments (on the stack, not including receiver)
+ Register registers[] = {r4, r5};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallWithSpreadDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r3 : number of arguments (on the stack, not including receiver)
+ // r4 : the target to call
+ // r5 : the object to spread
+ Register registers[] = {r4, r3, r5};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallWithArrayLikeDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r4 : the target to call
+ // r5 : the arguments list
+ Register registers[] = {r4, r5};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r3 : number of arguments (on the stack, not including receiver)
+ // r4 : the target to call
+ // r6 : the new target
+ // r7 : arguments list length (untagged)
+ // r5 : arguments list (FixedArray)
+ Register registers[] = {r4, r6, r3, r7, r5};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructForwardVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r3 : number of arguments
+ // r6 : the new target
+ // r5 : start index (to support rest parameters)
+ // r4 : the target to call
+ Register registers[] = {r4, r6, r3, r5};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructWithSpreadDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r3 : number of arguments (on the stack, not including receiver)
+ // r4 : the target to call
+ // r6 : the new target
+ // r5 : the object to spread
+ Register registers[] = {r4, r6, r3, r5};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructWithArrayLikeDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r4 : the target to call
+ // r6 : the new target
+ // r5 : the arguments list
+ Register registers[] = {r4, r6, r5};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructStubDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r3 : number of arguments
+ // r4 : the target to call
+ // r6 : the new target
+ // r5 : allocation site or undefined
+ Register registers[] = {r4, r6, r3, r5};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void AbortDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {r4};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CompareDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {r4, r3};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void BinaryOpDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {r4, r3};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ArgumentsAdaptorDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ r4, // JSFunction
+ r6, // the new target
+ r3, // actual number of arguments
+ r5, // expected number of arguments
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ApiCallbackDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ r4, // kApiFunctionAddress
+ r5, // kArgc
+ r6, // kCallData
+ r3, // kHolder
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void InterpreterDispatchDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ kInterpreterAccumulatorRegister, kInterpreterBytecodeOffsetRegister,
+ kInterpreterBytecodeArrayRegister, kInterpreterDispatchTableRegister};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void InterpreterPushArgsThenCallDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ r3, // argument count (not including receiver)
+ r5, // address of first argument
+ r4 // the target callable to be call
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void InterpreterPushArgsThenConstructDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ r3, // argument count (not including receiver)
+ r7, // address of the first argument
+ r4, // constructor to call
+ r6, // new target
+ r5, // allocation site feedback if available, undefined otherwise
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ResumeGeneratorDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ r3, // the value to pass to the generator
+ r4 // the JSGeneratorObject to resume
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void FrameDropperTrampolineDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ r4, // loaded new FP
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void RunMicrotasksEntryDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {r3, r4};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_PPC || V8_TARGET_ARCH_PPC64
diff --git a/src/codegen/ppc/macro-assembler-ppc.cc b/src/codegen/ppc/macro-assembler-ppc.cc
new file mode 100644
index 0000000..0895580
--- /dev/null
+++ b/src/codegen/ppc/macro-assembler-ppc.cc
@@ -0,0 +1,3291 @@
+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <assert.h> // For assert
+#include <limits.h> // For LONG_MIN, LONG_MAX.
+
+#if V8_TARGET_ARCH_PPC || V8_TARGET_ARCH_PPC64
+
+#include "src/base/bits.h"
+#include "src/base/division-by-constant.h"
+#include "src/codegen/callable.h"
+#include "src/codegen/code-factory.h"
+#include "src/codegen/external-reference-table.h"
+#include "src/codegen/macro-assembler.h"
+#include "src/codegen/register-configuration.h"
+#include "src/debug/debug.h"
+#include "src/execution/frames-inl.h"
+#include "src/heap/memory-chunk.h"
+#include "src/init/bootstrapper.h"
+#include "src/logging/counters.h"
+#include "src/runtime/runtime.h"
+#include "src/snapshot/embedded/embedded-data.h"
+#include "src/snapshot/snapshot.h"
+#include "src/wasm/wasm-code-manager.h"
+
+// Satisfy cpplint check, but don't include platform-specific header. It is
+// included recursively via macro-assembler.h.
+#if 0
+#include "src/codegen/ppc/macro-assembler-ppc.h"
+#endif
+
+namespace v8 {
+namespace internal {
+
+int TurboAssembler::RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
+ Register exclusion1,
+ Register exclusion2,
+ Register exclusion3) const {
+ int bytes = 0;
+ RegList exclusions = 0;
+ if (exclusion1 != no_reg) {
+ exclusions |= exclusion1.bit();
+ if (exclusion2 != no_reg) {
+ exclusions |= exclusion2.bit();
+ if (exclusion3 != no_reg) {
+ exclusions |= exclusion3.bit();
+ }
+ }
+ }
+
+ RegList list = kJSCallerSaved & ~exclusions;
+ bytes += NumRegs(list) * kSystemPointerSize;
+
+ if (fp_mode == kSaveFPRegs) {
+ bytes += kNumCallerSavedDoubles * kDoubleSize;
+ }
+
+ return bytes;
+}
+
+int TurboAssembler::PushCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1,
+ Register exclusion2, Register exclusion3) {
+ int bytes = 0;
+ RegList exclusions = 0;
+ if (exclusion1 != no_reg) {
+ exclusions |= exclusion1.bit();
+ if (exclusion2 != no_reg) {
+ exclusions |= exclusion2.bit();
+ if (exclusion3 != no_reg) {
+ exclusions |= exclusion3.bit();
+ }
+ }
+ }
+
+ RegList list = kJSCallerSaved & ~exclusions;
+ MultiPush(list);
+ bytes += NumRegs(list) * kSystemPointerSize;
+
+ if (fp_mode == kSaveFPRegs) {
+ MultiPushDoubles(kCallerSavedDoubles);
+ bytes += kNumCallerSavedDoubles * kDoubleSize;
+ }
+
+ return bytes;
+}
+
+int TurboAssembler::PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1,
+ Register exclusion2, Register exclusion3) {
+ int bytes = 0;
+ if (fp_mode == kSaveFPRegs) {
+ MultiPopDoubles(kCallerSavedDoubles);
+ bytes += kNumCallerSavedDoubles * kDoubleSize;
+ }
+
+ RegList exclusions = 0;
+ if (exclusion1 != no_reg) {
+ exclusions |= exclusion1.bit();
+ if (exclusion2 != no_reg) {
+ exclusions |= exclusion2.bit();
+ if (exclusion3 != no_reg) {
+ exclusions |= exclusion3.bit();
+ }
+ }
+ }
+
+ RegList list = kJSCallerSaved & ~exclusions;
+ MultiPop(list);
+ bytes += NumRegs(list) * kSystemPointerSize;
+
+ return bytes;
+}
+
+void TurboAssembler::Jump(Register target) {
+ mtctr(target);
+ bctr();
+}
+
+void TurboAssembler::LoadFromConstantsTable(Register destination,
+ int constant_index) {
+ DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kBuiltinsConstantsTable));
+
+ DCHECK_NE(destination, r0);
+ LoadRoot(destination, RootIndex::kBuiltinsConstantsTable);
+ LoadTaggedPointerField(
+ destination,
+ FieldMemOperand(destination,
+ FixedArray::OffsetOfElementAt(constant_index)),
+ r0);
+}
+
+void TurboAssembler::LoadRootRelative(Register destination, int32_t offset) {
+ LoadP(destination, MemOperand(kRootRegister, offset), r0);
+}
+
+void TurboAssembler::LoadRootRegisterOffset(Register destination,
+ intptr_t offset) {
+ if (offset == 0) {
+ mr(destination, kRootRegister);
+ } else if (is_int16(offset)) {
+ addi(destination, kRootRegister, Operand(offset));
+ } else {
+ mov(destination, Operand(offset));
+ add(destination, kRootRegister, destination);
+ }
+}
+
+void TurboAssembler::Jump(intptr_t target, RelocInfo::Mode rmode,
+ Condition cond, CRegister cr) {
+ Label skip;
+
+ if (cond != al) b(NegateCondition(cond), &skip, cr);
+
+ DCHECK(rmode == RelocInfo::CODE_TARGET || rmode == RelocInfo::RUNTIME_ENTRY);
+
+ mov(ip, Operand(target, rmode));
+ mtctr(ip);
+ bctr();
+
+ bind(&skip);
+}
+
+void TurboAssembler::Jump(Address target, RelocInfo::Mode rmode, Condition cond,
+ CRegister cr) {
+ DCHECK(!RelocInfo::IsCodeTarget(rmode));
+ Jump(static_cast<intptr_t>(target), rmode, cond, cr);
+}
+
+void TurboAssembler::Jump(Handle<Code> code, RelocInfo::Mode rmode,
+ Condition cond, CRegister cr) {
+ DCHECK(RelocInfo::IsCodeTarget(rmode));
+ DCHECK_IMPLIES(options().isolate_independent_code,
+ Builtins::IsIsolateIndependentBuiltin(*code));
+
+ int builtin_index = Builtins::kNoBuiltinId;
+ bool target_is_builtin =
+ isolate()->builtins()->IsBuiltinHandle(code, &builtin_index);
+
+ if (root_array_available_ && options().isolate_independent_code) {
+ Label skip;
+ Register scratch = ip;
+ int offset = code->builtin_index() * kSystemPointerSize +
+ IsolateData::builtin_entry_table_offset();
+ LoadP(scratch, MemOperand(kRootRegister, offset), r0);
+ if (cond != al) b(NegateCondition(cond), &skip, cr);
+ Jump(scratch);
+ bind(&skip);
+ return;
+ } else if (options().inline_offheap_trampolines && target_is_builtin) {
+ // Inline the trampoline.
+ Label skip;
+ RecordCommentForOffHeapTrampoline(builtin_index);
+ EmbeddedData d = EmbeddedData::FromBlob();
+ Address entry = d.InstructionStartOfBuiltin(builtin_index);
+ // Use ip directly instead of using UseScratchRegisterScope, as we do
+ // not preserve scratch registers across calls.
+ mov(ip, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+ if (cond != al) b(NegateCondition(cond), &skip, cr);
+ Jump(ip);
+ bind(&skip);
+ return;
+ }
+ int32_t target_index = AddCodeTarget(code);
+ Jump(static_cast<intptr_t>(target_index), rmode, cond, cr);
+}
+
+void TurboAssembler::Jump(const ExternalReference& reference) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ Move(scratch, reference);
+ if (ABI_USES_FUNCTION_DESCRIPTORS) {
+ // AIX uses a function descriptor. When calling C code be
+ // aware of this descriptor and pick up values from it.
+ LoadP(ToRegister(ABI_TOC_REGISTER),
+ MemOperand(scratch, kSystemPointerSize));
+ LoadP(scratch, MemOperand(scratch, 0));
+ }
+ Jump(scratch);
+}
+
+void TurboAssembler::Call(Register target) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ // branch via link register and set LK bit for return point
+ mtctr(target);
+ bctrl();
+}
+
+void MacroAssembler::CallJSEntry(Register target) {
+ CHECK(target == r5);
+ Call(target);
+}
+
+int MacroAssembler::CallSizeNotPredictableCodeSize(Address target,
+ RelocInfo::Mode rmode,
+ Condition cond) {
+ return (2 + kMovInstructionsNoConstantPool) * kInstrSize;
+}
+
+void TurboAssembler::Call(Address target, RelocInfo::Mode rmode,
+ Condition cond) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ DCHECK(cond == al);
+
+ // This can likely be optimized to make use of bc() with 24bit relative
+ //
+ // RecordRelocInfo(x.rmode_, x.immediate);
+ // bc( BA, .... offset, LKset);
+ //
+
+ mov(ip, Operand(target, rmode));
+ mtctr(ip);
+ bctrl();
+}
+
+void TurboAssembler::Call(Handle<Code> code, RelocInfo::Mode rmode,
+ Condition cond) {
+ BlockTrampolinePoolScope block_trampoline_pool(this);
+ DCHECK(RelocInfo::IsCodeTarget(rmode));
+ DCHECK_IMPLIES(options().isolate_independent_code,
+ Builtins::IsIsolateIndependentBuiltin(*code));
+ DCHECK_IMPLIES(options().use_pc_relative_calls_and_jumps,
+ Builtins::IsIsolateIndependentBuiltin(*code));
+
+ int builtin_index = Builtins::kNoBuiltinId;
+ bool target_is_builtin =
+ isolate()->builtins()->IsBuiltinHandle(code, &builtin_index);
+
+ if (root_array_available_ && options().isolate_independent_code) {
+ Label skip;
+ int offset = code->builtin_index() * kSystemPointerSize +
+ IsolateData::builtin_entry_table_offset();
+ LoadP(ip, MemOperand(kRootRegister, offset));
+ if (cond != al) b(NegateCondition(cond), &skip);
+ Call(ip);
+ bind(&skip);
+ return;
+ } else if (options().inline_offheap_trampolines && target_is_builtin) {
+ // Inline the trampoline.
+ RecordCommentForOffHeapTrampoline(builtin_index);
+ EmbeddedData d = EmbeddedData::FromBlob();
+ Address entry = d.InstructionStartOfBuiltin(builtin_index);
+ // Use ip directly instead of using UseScratchRegisterScope, as we do
+ // not preserve scratch registers across calls.
+ mov(ip, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+ Label skip;
+ if (cond != al) b(NegateCondition(cond), &skip);
+ Call(ip);
+ bind(&skip);
+ return;
+ }
+ DCHECK(code->IsExecutable());
+ int32_t target_index = AddCodeTarget(code);
+ Call(static_cast<Address>(target_index), rmode, cond);
+}
+
+void TurboAssembler::Drop(int count) {
+ if (count > 0) {
+ Add(sp, sp, count * kSystemPointerSize, r0);
+ }
+}
+
+void TurboAssembler::Drop(Register count, Register scratch) {
+ ShiftLeftImm(scratch, count, Operand(kSystemPointerSizeLog2));
+ add(sp, sp, scratch);
+}
+
+void TurboAssembler::Call(Label* target) { b(target, SetLK); }
+
+void TurboAssembler::Push(Handle<HeapObject> handle) {
+ mov(r0, Operand(handle));
+ push(r0);
+}
+
+void TurboAssembler::Push(Smi smi) {
+ mov(r0, Operand(smi));
+ push(r0);
+}
+
+void TurboAssembler::PushArray(Register array, Register size, Register scratch,
+ Register scratch2, PushArrayOrder order) {
+ Label loop, done;
+
+ if (order == kNormal) {
+ cmpi(size, Operand::Zero());
+ beq(&done);
+ ShiftLeftImm(scratch, size, Operand(kSystemPointerSizeLog2));
+ add(scratch, array, scratch);
+ mtctr(size);
+
+ bind(&loop);
+ LoadPU(scratch2, MemOperand(scratch, -kSystemPointerSize));
+ StorePU(scratch2, MemOperand(sp, -kSystemPointerSize));
+ bdnz(&loop);
+
+ bind(&done);
+ } else {
+ cmpi(size, Operand::Zero());
+ beq(&done);
+
+ mtctr(size);
+ subi(scratch, array, Operand(kSystemPointerSize));
+
+ bind(&loop);
+ LoadPU(scratch2, MemOperand(scratch, kSystemPointerSize));
+ StorePU(scratch2, MemOperand(sp, -kSystemPointerSize));
+ bdnz(&loop);
+ bind(&done);
+ }
+}
+
+void TurboAssembler::Move(Register dst, Handle<HeapObject> value,
+ RelocInfo::Mode rmode) {
+ // TODO(jgruber,v8:8887): Also consider a root-relative load when generating
+ // non-isolate-independent code. In many cases it might be cheaper than
+ // embedding the relocatable value.
+ if (root_array_available_ && options().isolate_independent_code) {
+ IndirectLoadConstant(dst, value);
+ return;
+ } else if (RelocInfo::IsCompressedEmbeddedObject(rmode)) {
+ EmbeddedObjectIndex index = AddEmbeddedObject(value);
+ DCHECK(is_uint32(index));
+ mov(dst, Operand(static_cast<int>(index), rmode));
+ } else {
+ DCHECK(RelocInfo::IsFullEmbeddedObject(rmode));
+ mov(dst, Operand(value.address(), rmode));
+ }
+}
+
+void TurboAssembler::Move(Register dst, ExternalReference reference) {
+ // TODO(jgruber,v8:8887): Also consider a root-relative load when generating
+ // non-isolate-independent code. In many cases it might be cheaper than
+ // embedding the relocatable value.
+ if (root_array_available_ && options().isolate_independent_code) {
+ IndirectLoadExternalReference(dst, reference);
+ return;
+ }
+ mov(dst, Operand(reference));
+}
+
+void TurboAssembler::Move(Register dst, Register src, Condition cond) {
+ DCHECK(cond == al);
+ if (dst != src) {
+ mr(dst, src);
+ }
+}
+
+void TurboAssembler::Move(DoubleRegister dst, DoubleRegister src) {
+ if (dst != src) {
+ fmr(dst, src);
+ }
+}
+
+void TurboAssembler::MultiPush(RegList regs, Register location) {
+ int16_t num_to_push = base::bits::CountPopulation(regs);
+ int16_t stack_offset = num_to_push * kSystemPointerSize;
+
+ subi(location, location, Operand(stack_offset));
+ for (int16_t i = Register::kNumRegisters - 1; i >= 0; i--) {
+ if ((regs & (1 << i)) != 0) {
+ stack_offset -= kSystemPointerSize;
+ StoreP(ToRegister(i), MemOperand(location, stack_offset));
+ }
+ }
+}
+
+void TurboAssembler::MultiPop(RegList regs, Register location) {
+ int16_t stack_offset = 0;
+
+ for (int16_t i = 0; i < Register::kNumRegisters; i++) {
+ if ((regs & (1 << i)) != 0) {
+ LoadP(ToRegister(i), MemOperand(location, stack_offset));
+ stack_offset += kSystemPointerSize;
+ }
+ }
+ addi(location, location, Operand(stack_offset));
+}
+
+void TurboAssembler::MultiPushDoubles(RegList dregs, Register location) {
+ int16_t num_to_push = base::bits::CountPopulation(dregs);
+ int16_t stack_offset = num_to_push * kDoubleSize;
+
+ subi(location, location, Operand(stack_offset));
+ for (int16_t i = DoubleRegister::kNumRegisters - 1; i >= 0; i--) {
+ if ((dregs & (1 << i)) != 0) {
+ DoubleRegister dreg = DoubleRegister::from_code(i);
+ stack_offset -= kDoubleSize;
+ stfd(dreg, MemOperand(location, stack_offset));
+ }
+ }
+}
+
+void TurboAssembler::MultiPopDoubles(RegList dregs, Register location) {
+ int16_t stack_offset = 0;
+
+ for (int16_t i = 0; i < DoubleRegister::kNumRegisters; i++) {
+ if ((dregs & (1 << i)) != 0) {
+ DoubleRegister dreg = DoubleRegister::from_code(i);
+ lfd(dreg, MemOperand(location, stack_offset));
+ stack_offset += kDoubleSize;
+ }
+ }
+ addi(location, location, Operand(stack_offset));
+}
+
+void TurboAssembler::LoadRoot(Register destination, RootIndex index,
+ Condition cond) {
+ DCHECK(cond == al);
+ LoadP(destination,
+ MemOperand(kRootRegister, RootRegisterOffsetForRootIndex(index)), r0);
+}
+
+void TurboAssembler::LoadTaggedPointerField(const Register& destination,
+ const MemOperand& field_operand,
+ const Register& scratch) {
+ if (COMPRESS_POINTERS_BOOL) {
+ DecompressTaggedPointer(destination, field_operand);
+ } else {
+ LoadP(destination, field_operand, scratch);
+ }
+}
+
+void TurboAssembler::LoadAnyTaggedField(const Register& destination,
+ const MemOperand& field_operand,
+ const Register& scratch) {
+ if (COMPRESS_POINTERS_BOOL) {
+ DecompressAnyTagged(destination, field_operand);
+ } else {
+ LoadP(destination, field_operand, scratch);
+ }
+}
+
+void TurboAssembler::SmiUntag(Register dst, const MemOperand& src, RCBit rc) {
+ if (SmiValuesAre31Bits()) {
+ lwz(dst, src);
+ } else {
+ LoadP(dst, src);
+ }
+
+ SmiUntag(dst, rc);
+}
+
+void TurboAssembler::SmiUntagField(Register dst, const MemOperand& src,
+ RCBit rc) {
+ SmiUntag(dst, src, rc);
+}
+
+void TurboAssembler::StoreTaggedFieldX(const Register& value,
+ const MemOperand& dst_field_operand,
+ const Register& scratch) {
+ if (COMPRESS_POINTERS_BOOL) {
+ RecordComment("[ StoreTagged");
+ stwx(value, dst_field_operand);
+ RecordComment("]");
+ } else {
+ StorePX(value, dst_field_operand);
+ }
+}
+
+void TurboAssembler::StoreTaggedField(const Register& value,
+ const MemOperand& dst_field_operand,
+ const Register& scratch) {
+ if (COMPRESS_POINTERS_BOOL) {
+ RecordComment("[ StoreTagged");
+ StoreWord(value, dst_field_operand, scratch);
+ RecordComment("]");
+ } else {
+ StoreP(value, dst_field_operand, scratch);
+ }
+}
+
+void TurboAssembler::DecompressTaggedSigned(Register destination,
+ Register src) {
+ RecordComment("[ DecompressTaggedSigned");
+ ZeroExtWord32(destination, src);
+ RecordComment("]");
+}
+
+void TurboAssembler::DecompressTaggedSigned(Register destination,
+ MemOperand field_operand) {
+ RecordComment("[ DecompressTaggedSigned");
+ LoadWord(destination, field_operand, r0);
+ RecordComment("]");
+}
+
+void TurboAssembler::DecompressTaggedPointer(Register destination,
+ Register source) {
+ RecordComment("[ DecompressTaggedPointer");
+ ZeroExtWord32(destination, source);
+ add(destination, destination, kRootRegister);
+ RecordComment("]");
+}
+
+void TurboAssembler::DecompressTaggedPointer(Register destination,
+ MemOperand field_operand) {
+ RecordComment("[ DecompressTaggedPointer");
+ LoadWord(destination, field_operand, r0);
+ add(destination, destination, kRootRegister);
+ RecordComment("]");
+}
+
+void TurboAssembler::DecompressAnyTagged(Register destination,
+ MemOperand field_operand) {
+ RecordComment("[ DecompressAnyTagged");
+ LoadWord(destination, field_operand, r0);
+ add(destination, destination, kRootRegister);
+ RecordComment("]");
+}
+
+void TurboAssembler::DecompressAnyTagged(Register destination,
+ Register source) {
+ RecordComment("[ DecompressAnyTagged");
+ ZeroExtWord32(destination, source);
+ add(destination, destination, kRootRegister);
+ RecordComment("]");
+}
+
+void MacroAssembler::RecordWriteField(Register object, int offset,
+ Register value, Register dst,
+ LinkRegisterStatus lr_status,
+ SaveFPRegsMode save_fp,
+ RememberedSetAction remembered_set_action,
+ SmiCheck smi_check) {
+ // First, check if a write barrier is even needed. The tests below
+ // catch stores of Smis.
+ Label done;
+
+ // Skip barrier if writing a smi.
+ if (smi_check == INLINE_SMI_CHECK) {
+ JumpIfSmi(value, &done);
+ }
+
+ // Although the object register is tagged, the offset is relative to the start
+ // of the object, so so offset must be a multiple of kSystemPointerSize.
+ DCHECK(IsAligned(offset, kTaggedSize));
+
+ Add(dst, object, offset - kHeapObjectTag, r0);
+ if (emit_debug_code()) {
+ Label ok;
+ andi(r0, dst, Operand(kTaggedSize - 1));
+ beq(&ok, cr0);
+ stop();
+ bind(&ok);
+ }
+
+ RecordWrite(object, dst, value, lr_status, save_fp, remembered_set_action,
+ OMIT_SMI_CHECK);
+
+ bind(&done);
+
+ // Clobber clobbered input registers when running with the debug-code flag
+ // turned on to provoke errors.
+ if (emit_debug_code()) {
+ mov(value, Operand(bit_cast<intptr_t>(kZapValue + 4)));
+ mov(dst, Operand(bit_cast<intptr_t>(kZapValue + 8)));
+ }
+}
+
+void TurboAssembler::SaveRegisters(RegList registers) {
+ DCHECK_GT(NumRegs(registers), 0);
+ RegList regs = 0;
+ for (int i = 0; i < Register::kNumRegisters; ++i) {
+ if ((registers >> i) & 1u) {
+ regs |= Register::from_code(i).bit();
+ }
+ }
+
+ MultiPush(regs);
+}
+
+void TurboAssembler::RestoreRegisters(RegList registers) {
+ DCHECK_GT(NumRegs(registers), 0);
+ RegList regs = 0;
+ for (int i = 0; i < Register::kNumRegisters; ++i) {
+ if ((registers >> i) & 1u) {
+ regs |= Register::from_code(i).bit();
+ }
+ }
+ MultiPop(regs);
+}
+
+void TurboAssembler::CallEphemeronKeyBarrier(Register object, Register address,
+ SaveFPRegsMode fp_mode) {
+ EphemeronKeyBarrierDescriptor descriptor;
+ RegList registers = descriptor.allocatable_registers();
+
+ SaveRegisters(registers);
+
+ Register object_parameter(
+ descriptor.GetRegisterParameter(EphemeronKeyBarrierDescriptor::kObject));
+ Register slot_parameter(descriptor.GetRegisterParameter(
+ EphemeronKeyBarrierDescriptor::kSlotAddress));
+ Register fp_mode_parameter(
+ descriptor.GetRegisterParameter(EphemeronKeyBarrierDescriptor::kFPMode));
+
+ push(object);
+ push(address);
+
+ pop(slot_parameter);
+ pop(object_parameter);
+
+ Move(fp_mode_parameter, Smi::FromEnum(fp_mode));
+ Call(isolate()->builtins()->builtin_handle(Builtins::kEphemeronKeyBarrier),
+ RelocInfo::CODE_TARGET);
+ RestoreRegisters(registers);
+}
+
+void TurboAssembler::CallRecordWriteStub(
+ Register object, Register address,
+ RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode) {
+ CallRecordWriteStub(
+ object, address, remembered_set_action, fp_mode,
+ isolate()->builtins()->builtin_handle(Builtins::kRecordWrite),
+ kNullAddress);
+}
+
+void TurboAssembler::CallRecordWriteStub(
+ Register object, Register address,
+ RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode,
+ Address wasm_target) {
+ CallRecordWriteStub(object, address, remembered_set_action, fp_mode,
+ Handle<Code>::null(), wasm_target);
+}
+
+void TurboAssembler::CallRecordWriteStub(
+ Register object, Register address,
+ RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode,
+ Handle<Code> code_target, Address wasm_target) {
+ DCHECK_NE(code_target.is_null(), wasm_target == kNullAddress);
+ // TODO(albertnetymk): For now we ignore remembered_set_action and fp_mode,
+ // i.e. always emit remember set and save FP registers in RecordWriteStub. If
+ // large performance regression is observed, we should use these values to
+ // avoid unnecessary work.
+
+ RecordWriteDescriptor descriptor;
+ RegList registers = descriptor.allocatable_registers();
+
+ SaveRegisters(registers);
+
+ Register object_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kObject));
+ Register slot_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kSlot));
+ Register remembered_set_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kRememberedSet));
+ Register fp_mode_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kFPMode));
+
+ push(object);
+ push(address);
+
+ pop(slot_parameter);
+ pop(object_parameter);
+
+ Move(remembered_set_parameter, Smi::FromEnum(remembered_set_action));
+ Move(fp_mode_parameter, Smi::FromEnum(fp_mode));
+ if (code_target.is_null()) {
+ Call(wasm_target, RelocInfo::WASM_STUB_CALL);
+ } else {
+ Call(code_target, RelocInfo::CODE_TARGET);
+ }
+
+ RestoreRegisters(registers);
+}
+
+// Will clobber 4 registers: object, address, scratch, ip. The
+// register 'object' contains a heap object pointer. The heap object
+// tag is shifted away.
+void MacroAssembler::RecordWrite(Register object, Register address,
+ Register value, LinkRegisterStatus lr_status,
+ SaveFPRegsMode fp_mode,
+ RememberedSetAction remembered_set_action,
+ SmiCheck smi_check) {
+ DCHECK(object != value);
+ if (emit_debug_code()) {
+ LoadTaggedPointerField(r0, MemOperand(address));
+ cmp(r0, value);
+ Check(eq, AbortReason::kWrongAddressOrValuePassedToRecordWrite);
+ }
+
+ if ((remembered_set_action == OMIT_REMEMBERED_SET &&
+ !FLAG_incremental_marking) ||
+ FLAG_disable_write_barriers) {
+ return;
+ }
+
+ // First, check if a write barrier is even needed. The tests below
+ // catch stores of smis and stores into the young generation.
+ Label done;
+
+ if (smi_check == INLINE_SMI_CHECK) {
+ JumpIfSmi(value, &done);
+ }
+
+ CheckPageFlag(value,
+ value, // Used as scratch.
+ MemoryChunk::kPointersToHereAreInterestingMask, eq, &done);
+ CheckPageFlag(object,
+ value, // Used as scratch.
+ MemoryChunk::kPointersFromHereAreInterestingMask, eq, &done);
+
+ // Record the actual write.
+ if (lr_status == kLRHasNotBeenSaved) {
+ mflr(r0);
+ push(r0);
+ }
+ CallRecordWriteStub(object, address, remembered_set_action, fp_mode);
+ if (lr_status == kLRHasNotBeenSaved) {
+ pop(r0);
+ mtlr(r0);
+ }
+
+ bind(&done);
+
+ // Clobber clobbered registers when running with the debug-code flag
+ // turned on to provoke errors.
+ if (emit_debug_code()) {
+ mov(address, Operand(bit_cast<intptr_t>(kZapValue + 12)));
+ mov(value, Operand(bit_cast<intptr_t>(kZapValue + 16)));
+ }
+}
+
+void TurboAssembler::PushCommonFrame(Register marker_reg) {
+ int fp_delta = 0;
+ mflr(r0);
+ if (FLAG_enable_embedded_constant_pool) {
+ if (marker_reg.is_valid()) {
+ Push(r0, fp, kConstantPoolRegister, marker_reg);
+ fp_delta = 2;
+ } else {
+ Push(r0, fp, kConstantPoolRegister);
+ fp_delta = 1;
+ }
+ } else {
+ if (marker_reg.is_valid()) {
+ Push(r0, fp, marker_reg);
+ fp_delta = 1;
+ } else {
+ Push(r0, fp);
+ fp_delta = 0;
+ }
+ }
+ addi(fp, sp, Operand(fp_delta * kSystemPointerSize));
+}
+
+void TurboAssembler::PushStandardFrame(Register function_reg) {
+ int fp_delta = 0;
+ mflr(r0);
+ if (FLAG_enable_embedded_constant_pool) {
+ if (function_reg.is_valid()) {
+ Push(r0, fp, kConstantPoolRegister, cp, function_reg);
+ fp_delta = 3;
+ } else {
+ Push(r0, fp, kConstantPoolRegister, cp);
+ fp_delta = 2;
+ }
+ } else {
+ if (function_reg.is_valid()) {
+ Push(r0, fp, cp, function_reg);
+ fp_delta = 2;
+ } else {
+ Push(r0, fp, cp);
+ fp_delta = 1;
+ }
+ }
+ addi(fp, sp, Operand(fp_delta * kSystemPointerSize));
+ Push(kJavaScriptCallArgCountRegister);
+}
+
+void TurboAssembler::RestoreFrameStateForTailCall() {
+ if (FLAG_enable_embedded_constant_pool) {
+ LoadP(kConstantPoolRegister,
+ MemOperand(fp, StandardFrameConstants::kConstantPoolOffset));
+ set_constant_pool_available(false);
+ }
+ LoadP(r0, MemOperand(fp, StandardFrameConstants::kCallerPCOffset));
+ LoadP(fp, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
+ mtlr(r0);
+}
+
+void TurboAssembler::CanonicalizeNaN(const DoubleRegister dst,
+ const DoubleRegister src) {
+ // Turn potential sNaN into qNaN.
+ fsub(dst, src, kDoubleRegZero);
+}
+
+void TurboAssembler::ConvertIntToDouble(Register src, DoubleRegister dst) {
+ MovIntToDouble(dst, src, r0);
+ fcfid(dst, dst);
+}
+
+void TurboAssembler::ConvertUnsignedIntToDouble(Register src,
+ DoubleRegister dst) {
+ MovUnsignedIntToDouble(dst, src, r0);
+ fcfid(dst, dst);
+}
+
+void TurboAssembler::ConvertIntToFloat(Register src, DoubleRegister dst) {
+ MovIntToDouble(dst, src, r0);
+ fcfids(dst, dst);
+}
+
+void TurboAssembler::ConvertUnsignedIntToFloat(Register src,
+ DoubleRegister dst) {
+ MovUnsignedIntToDouble(dst, src, r0);
+ fcfids(dst, dst);
+}
+
+#if V8_TARGET_ARCH_PPC64
+void TurboAssembler::ConvertInt64ToDouble(Register src,
+ DoubleRegister double_dst) {
+ MovInt64ToDouble(double_dst, src);
+ fcfid(double_dst, double_dst);
+}
+
+void TurboAssembler::ConvertUnsignedInt64ToFloat(Register src,
+ DoubleRegister double_dst) {
+ MovInt64ToDouble(double_dst, src);
+ fcfidus(double_dst, double_dst);
+}
+
+void TurboAssembler::ConvertUnsignedInt64ToDouble(Register src,
+ DoubleRegister double_dst) {
+ MovInt64ToDouble(double_dst, src);
+ fcfidu(double_dst, double_dst);
+}
+
+void TurboAssembler::ConvertInt64ToFloat(Register src,
+ DoubleRegister double_dst) {
+ MovInt64ToDouble(double_dst, src);
+ fcfids(double_dst, double_dst);
+}
+#endif
+
+void TurboAssembler::ConvertDoubleToInt64(const DoubleRegister double_input,
+#if !V8_TARGET_ARCH_PPC64
+ const Register dst_hi,
+#endif
+ const Register dst,
+ const DoubleRegister double_dst,
+ FPRoundingMode rounding_mode) {
+ if (rounding_mode == kRoundToZero) {
+ fctidz(double_dst, double_input);
+ } else {
+ SetRoundingMode(rounding_mode);
+ fctid(double_dst, double_input);
+ ResetRoundingMode();
+ }
+
+ MovDoubleToInt64(
+#if !V8_TARGET_ARCH_PPC64
+ dst_hi,
+#endif
+ dst, double_dst);
+}
+
+#if V8_TARGET_ARCH_PPC64
+void TurboAssembler::ConvertDoubleToUnsignedInt64(
+ const DoubleRegister double_input, const Register dst,
+ const DoubleRegister double_dst, FPRoundingMode rounding_mode) {
+ if (rounding_mode == kRoundToZero) {
+ fctiduz(double_dst, double_input);
+ } else {
+ SetRoundingMode(rounding_mode);
+ fctidu(double_dst, double_input);
+ ResetRoundingMode();
+ }
+
+ MovDoubleToInt64(dst, double_dst);
+}
+#endif
+
+#if !V8_TARGET_ARCH_PPC64
+void TurboAssembler::ShiftLeftPair(Register dst_low, Register dst_high,
+ Register src_low, Register src_high,
+ Register scratch, Register shift) {
+ DCHECK(!AreAliased(dst_low, src_high));
+ DCHECK(!AreAliased(dst_high, src_low));
+ DCHECK(!AreAliased(dst_low, dst_high, shift));
+ Label less_than_32;
+ Label done;
+ cmpi(shift, Operand(32));
+ blt(&less_than_32);
+ // If shift >= 32
+ andi(scratch, shift, Operand(0x1F));
+ slw(dst_high, src_low, scratch);
+ li(dst_low, Operand::Zero());
+ b(&done);
+ bind(&less_than_32);
+ // If shift < 32
+ subfic(scratch, shift, Operand(32));
+ slw(dst_high, src_high, shift);
+ srw(scratch, src_low, scratch);
+ orx(dst_high, dst_high, scratch);
+ slw(dst_low, src_low, shift);
+ bind(&done);
+}
+
+void TurboAssembler::ShiftLeftPair(Register dst_low, Register dst_high,
+ Register src_low, Register src_high,
+ uint32_t shift) {
+ DCHECK(!AreAliased(dst_low, src_high));
+ DCHECK(!AreAliased(dst_high, src_low));
+ if (shift == 32) {
+ Move(dst_high, src_low);
+ li(dst_low, Operand::Zero());
+ } else if (shift > 32) {
+ shift &= 0x1F;
+ slwi(dst_high, src_low, Operand(shift));
+ li(dst_low, Operand::Zero());
+ } else if (shift == 0) {
+ Move(dst_low, src_low);
+ Move(dst_high, src_high);
+ } else {
+ slwi(dst_high, src_high, Operand(shift));
+ rlwimi(dst_high, src_low, shift, 32 - shift, 31);
+ slwi(dst_low, src_low, Operand(shift));
+ }
+}
+
+void TurboAssembler::ShiftRightPair(Register dst_low, Register dst_high,
+ Register src_low, Register src_high,
+ Register scratch, Register shift) {
+ DCHECK(!AreAliased(dst_low, src_high));
+ DCHECK(!AreAliased(dst_high, src_low));
+ DCHECK(!AreAliased(dst_low, dst_high, shift));
+ Label less_than_32;
+ Label done;
+ cmpi(shift, Operand(32));
+ blt(&less_than_32);
+ // If shift >= 32
+ andi(scratch, shift, Operand(0x1F));
+ srw(dst_low, src_high, scratch);
+ li(dst_high, Operand::Zero());
+ b(&done);
+ bind(&less_than_32);
+ // If shift < 32
+ subfic(scratch, shift, Operand(32));
+ srw(dst_low, src_low, shift);
+ slw(scratch, src_high, scratch);
+ orx(dst_low, dst_low, scratch);
+ srw(dst_high, src_high, shift);
+ bind(&done);
+}
+
+void TurboAssembler::ShiftRightPair(Register dst_low, Register dst_high,
+ Register src_low, Register src_high,
+ uint32_t shift) {
+ DCHECK(!AreAliased(dst_low, src_high));
+ DCHECK(!AreAliased(dst_high, src_low));
+ if (shift == 32) {
+ Move(dst_low, src_high);
+ li(dst_high, Operand::Zero());
+ } else if (shift > 32) {
+ shift &= 0x1F;
+ srwi(dst_low, src_high, Operand(shift));
+ li(dst_high, Operand::Zero());
+ } else if (shift == 0) {
+ Move(dst_low, src_low);
+ Move(dst_high, src_high);
+ } else {
+ srwi(dst_low, src_low, Operand(shift));
+ rlwimi(dst_low, src_high, 32 - shift, 0, shift - 1);
+ srwi(dst_high, src_high, Operand(shift));
+ }
+}
+
+void TurboAssembler::ShiftRightAlgPair(Register dst_low, Register dst_high,
+ Register src_low, Register src_high,
+ Register scratch, Register shift) {
+ DCHECK(!AreAliased(dst_low, src_high, shift));
+ DCHECK(!AreAliased(dst_high, src_low, shift));
+ Label less_than_32;
+ Label done;
+ cmpi(shift, Operand(32));
+ blt(&less_than_32);
+ // If shift >= 32
+ andi(scratch, shift, Operand(0x1F));
+ sraw(dst_low, src_high, scratch);
+ srawi(dst_high, src_high, 31);
+ b(&done);
+ bind(&less_than_32);
+ // If shift < 32
+ subfic(scratch, shift, Operand(32));
+ srw(dst_low, src_low, shift);
+ slw(scratch, src_high, scratch);
+ orx(dst_low, dst_low, scratch);
+ sraw(dst_high, src_high, shift);
+ bind(&done);
+}
+
+void TurboAssembler::ShiftRightAlgPair(Register dst_low, Register dst_high,
+ Register src_low, Register src_high,
+ uint32_t shift) {
+ DCHECK(!AreAliased(dst_low, src_high));
+ DCHECK(!AreAliased(dst_high, src_low));
+ if (shift == 32) {
+ Move(dst_low, src_high);
+ srawi(dst_high, src_high, 31);
+ } else if (shift > 32) {
+ shift &= 0x1F;
+ srawi(dst_low, src_high, shift);
+ srawi(dst_high, src_high, 31);
+ } else if (shift == 0) {
+ Move(dst_low, src_low);
+ Move(dst_high, src_high);
+ } else {
+ srwi(dst_low, src_low, Operand(shift));
+ rlwimi(dst_low, src_high, 32 - shift, 0, shift - 1);
+ srawi(dst_high, src_high, shift);
+ }
+}
+#endif
+
+void TurboAssembler::LoadConstantPoolPointerRegisterFromCodeTargetAddress(
+ Register code_target_address) {
+ // Builtins do not use the constant pool (see is_constant_pool_available).
+ STATIC_ASSERT(Code::kOnHeapBodyIsContiguous);
+
+ lwz(r0, MemOperand(code_target_address,
+ Code::kInstructionSizeOffset - Code::kHeaderSize));
+ lwz(kConstantPoolRegister,
+ MemOperand(code_target_address,
+ Code::kConstantPoolOffsetOffset - Code::kHeaderSize));
+ add(kConstantPoolRegister, kConstantPoolRegister, code_target_address);
+ add(kConstantPoolRegister, kConstantPoolRegister, r0);
+}
+
+void TurboAssembler::LoadPC(Register dst) {
+ b(4, SetLK);
+ mflr(dst);
+}
+
+void TurboAssembler::ComputeCodeStartAddress(Register dst) {
+ mflr(r0);
+ LoadPC(dst);
+ subi(dst, dst, Operand(pc_offset() - kInstrSize));
+ mtlr(r0);
+}
+
+void TurboAssembler::LoadConstantPoolPointerRegister() {
+ //
+ // Builtins do not use the constant pool (see is_constant_pool_available).
+ STATIC_ASSERT(Code::kOnHeapBodyIsContiguous);
+
+ LoadPC(kConstantPoolRegister);
+ int32_t delta = -pc_offset() + 4;
+ add_label_offset(kConstantPoolRegister, kConstantPoolRegister,
+ ConstantPoolPosition(), delta);
+}
+
+void TurboAssembler::StubPrologue(StackFrame::Type type) {
+ {
+ ConstantPoolUnavailableScope constant_pool_unavailable(this);
+ mov(r11, Operand(StackFrame::TypeToMarker(type)));
+ PushCommonFrame(r11);
+ }
+ if (FLAG_enable_embedded_constant_pool) {
+ LoadConstantPoolPointerRegister();
+ set_constant_pool_available(true);
+ }
+}
+
+void TurboAssembler::Prologue() {
+ PushStandardFrame(r4);
+ if (FLAG_enable_embedded_constant_pool) {
+ // base contains prologue address
+ LoadConstantPoolPointerRegister();
+ set_constant_pool_available(true);
+ }
+}
+
+void TurboAssembler::EnterFrame(StackFrame::Type type,
+ bool load_constant_pool_pointer_reg) {
+ if (FLAG_enable_embedded_constant_pool && load_constant_pool_pointer_reg) {
+ // Push type explicitly so we can leverage the constant pool.
+ // This path cannot rely on ip containing code entry.
+ PushCommonFrame();
+ LoadConstantPoolPointerRegister();
+ mov(ip, Operand(StackFrame::TypeToMarker(type)));
+ push(ip);
+ } else {
+ mov(ip, Operand(StackFrame::TypeToMarker(type)));
+ PushCommonFrame(ip);
+ }
+}
+
+int TurboAssembler::LeaveFrame(StackFrame::Type type, int stack_adjustment) {
+ ConstantPoolUnavailableScope constant_pool_unavailable(this);
+ // r3: preserved
+ // r4: preserved
+ // r5: preserved
+
+ // Drop the execution stack down to the frame pointer and restore
+ // the caller's state.
+ int frame_ends;
+ LoadP(r0, MemOperand(fp, StandardFrameConstants::kCallerPCOffset));
+ LoadP(ip, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
+ if (FLAG_enable_embedded_constant_pool) {
+ LoadP(kConstantPoolRegister,
+ MemOperand(fp, StandardFrameConstants::kConstantPoolOffset));
+ }
+ mtlr(r0);
+ frame_ends = pc_offset();
+ Add(sp, fp, StandardFrameConstants::kCallerSPOffset + stack_adjustment, r0);
+ mr(fp, ip);
+ return frame_ends;
+}
+
+// ExitFrame layout (probably wrongish.. needs updating)
+//
+// SP -> previousSP
+// LK reserved
+// sp_on_exit (for debug?)
+// oldSP->prev SP
+// LK
+// <parameters on stack>
+
+// Prior to calling EnterExitFrame, we've got a bunch of parameters
+// on the stack that we need to wrap a real frame around.. so first
+// we reserve a slot for LK and push the previous SP which is captured
+// in the fp register (r31)
+// Then - we buy a new frame
+
+void MacroAssembler::EnterExitFrame(bool save_doubles, int stack_space,
+ StackFrame::Type frame_type) {
+ DCHECK(frame_type == StackFrame::EXIT ||
+ frame_type == StackFrame::BUILTIN_EXIT);
+ // Set up the frame structure on the stack.
+ DCHECK_EQ(2 * kSystemPointerSize, ExitFrameConstants::kCallerSPDisplacement);
+ DCHECK_EQ(1 * kSystemPointerSize, ExitFrameConstants::kCallerPCOffset);
+ DCHECK_EQ(0 * kSystemPointerSize, ExitFrameConstants::kCallerFPOffset);
+ DCHECK_GT(stack_space, 0);
+
+ // This is an opportunity to build a frame to wrap
+ // all of the pushes that have happened inside of V8
+ // since we were called from C code
+
+ mov(ip, Operand(StackFrame::TypeToMarker(frame_type)));
+ PushCommonFrame(ip);
+ // Reserve room for saved entry sp.
+ subi(sp, fp, Operand(ExitFrameConstants::kFixedFrameSizeFromFp));
+
+ if (emit_debug_code()) {
+ li(r8, Operand::Zero());
+ StoreP(r8, MemOperand(fp, ExitFrameConstants::kSPOffset));
+ }
+ if (FLAG_enable_embedded_constant_pool) {
+ StoreP(kConstantPoolRegister,
+ MemOperand(fp, ExitFrameConstants::kConstantPoolOffset));
+ }
+
+ // Save the frame pointer and the context in top.
+ Move(r8, ExternalReference::Create(IsolateAddressId::kCEntryFPAddress,
+ isolate()));
+ StoreP(fp, MemOperand(r8));
+ Move(r8,
+ ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
+ StoreP(cp, MemOperand(r8));
+
+ // Optionally save all volatile double registers.
+ if (save_doubles) {
+ MultiPushDoubles(kCallerSavedDoubles);
+ // Note that d0 will be accessible at
+ // fp - ExitFrameConstants::kFrameSize -
+ // kNumCallerSavedDoubles * kDoubleSize,
+ // since the sp slot and code slot were pushed after the fp.
+ }
+
+ addi(sp, sp, Operand(-stack_space * kSystemPointerSize));
+
+ // Allocate and align the frame preparing for calling the runtime
+ // function.
+ const int frame_alignment = ActivationFrameAlignment();
+ if (frame_alignment > kSystemPointerSize) {
+ DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
+ ClearRightImm(sp, sp,
+ Operand(base::bits::WhichPowerOfTwo(frame_alignment)));
+ }
+ li(r0, Operand::Zero());
+ StorePU(r0,
+ MemOperand(sp, -kNumRequiredStackFrameSlots * kSystemPointerSize));
+
+ // Set the exit frame sp value to point just before the return address
+ // location.
+ addi(r8, sp, Operand((kStackFrameExtraParamSlot + 1) * kSystemPointerSize));
+ StoreP(r8, MemOperand(fp, ExitFrameConstants::kSPOffset));
+}
+
+int TurboAssembler::ActivationFrameAlignment() {
+#if !defined(USE_SIMULATOR)
+ // Running on the real platform. Use the alignment as mandated by the local
+ // environment.
+ // Note: This will break if we ever start generating snapshots on one PPC
+ // platform for another PPC platform with a different alignment.
+ return base::OS::ActivationFrameAlignment();
+#else // Simulated
+ // If we are using the simulator then we should always align to the expected
+ // alignment. As the simulator is used to generate snapshots we do not know
+ // if the target platform will need alignment, so this is controlled from a
+ // flag.
+ return FLAG_sim_stack_alignment;
+#endif
+}
+
+void MacroAssembler::LeaveExitFrame(bool save_doubles, Register argument_count,
+ bool argument_count_is_length) {
+ ConstantPoolUnavailableScope constant_pool_unavailable(this);
+ // Optionally restore all double registers.
+ if (save_doubles) {
+ // Calculate the stack location of the saved doubles and restore them.
+ const int kNumRegs = kNumCallerSavedDoubles;
+ const int offset =
+ (ExitFrameConstants::kFixedFrameSizeFromFp + kNumRegs * kDoubleSize);
+ addi(r6, fp, Operand(-offset));
+ MultiPopDoubles(kCallerSavedDoubles, r6);
+ }
+
+ // Clear top frame.
+ li(r6, Operand::Zero());
+ Move(ip, ExternalReference::Create(IsolateAddressId::kCEntryFPAddress,
+ isolate()));
+ StoreP(r6, MemOperand(ip));
+
+ // Restore current context from top and clear it in debug mode.
+ Move(ip,
+ ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
+ LoadP(cp, MemOperand(ip));
+
+#ifdef DEBUG
+ mov(r6, Operand(Context::kInvalidContext));
+ Move(ip,
+ ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
+ StoreP(r6, MemOperand(ip));
+#endif
+
+ // Tear down the exit frame, pop the arguments, and return.
+ LeaveFrame(StackFrame::EXIT);
+
+ if (argument_count.is_valid()) {
+ if (!argument_count_is_length) {
+ ShiftLeftImm(argument_count, argument_count,
+ Operand(kSystemPointerSizeLog2));
+ }
+ add(sp, sp, argument_count);
+ }
+}
+
+void TurboAssembler::MovFromFloatResult(const DoubleRegister dst) {
+ Move(dst, d1);
+}
+
+void TurboAssembler::MovFromFloatParameter(const DoubleRegister dst) {
+ Move(dst, d1);
+}
+
+void TurboAssembler::PrepareForTailCall(Register callee_args_count,
+ Register caller_args_count,
+ Register scratch0, Register scratch1) {
+ DCHECK(!AreAliased(callee_args_count, caller_args_count, scratch0, scratch1));
+
+ // Calculate the end of destination area where we will put the arguments
+ // after we drop current frame. We add kSystemPointerSize to count the
+ // receiver argument which is not included into formal parameters count.
+ Register dst_reg = scratch0;
+ ShiftLeftImm(dst_reg, caller_args_count, Operand(kSystemPointerSizeLog2));
+ add(dst_reg, fp, dst_reg);
+ addi(dst_reg, dst_reg,
+ Operand(StandardFrameConstants::kCallerSPOffset + kSystemPointerSize));
+
+ Register src_reg = caller_args_count;
+ // Calculate the end of source area. +kSystemPointerSize is for the receiver.
+ ShiftLeftImm(src_reg, callee_args_count, Operand(kSystemPointerSizeLog2));
+ add(src_reg, sp, src_reg);
+ addi(src_reg, src_reg, Operand(kSystemPointerSize));
+
+ if (FLAG_debug_code) {
+ cmpl(src_reg, dst_reg);
+ Check(lt, AbortReason::kStackAccessBelowStackPointer);
+ }
+
+ // Restore caller's frame pointer and return address now as they will be
+ // overwritten by the copying loop.
+ RestoreFrameStateForTailCall();
+
+ // Now copy callee arguments to the caller frame going backwards to avoid
+ // callee arguments corruption (source and destination areas could overlap).
+
+ // Both src_reg and dst_reg are pointing to the word after the one to copy,
+ // so they must be pre-decremented in the loop.
+ Register tmp_reg = scratch1;
+ Label loop;
+ addi(tmp_reg, callee_args_count, Operand(1)); // +1 for receiver
+ mtctr(tmp_reg);
+ bind(&loop);
+ LoadPU(tmp_reg, MemOperand(src_reg, -kSystemPointerSize));
+ StorePU(tmp_reg, MemOperand(dst_reg, -kSystemPointerSize));
+ bdnz(&loop);
+
+ // Leave current frame.
+ mr(sp, dst_reg);
+}
+
+void MacroAssembler::InvokePrologue(Register expected_parameter_count,
+ Register actual_parameter_count,
+ Label* done, InvokeFlag flag) {
+ Label regular_invoke;
+
+ // Check whether the expected and actual arguments count match. If not,
+ // setup registers according to contract with ArgumentsAdaptorTrampoline:
+ // r3: actual arguments count
+ // r4: function (passed through to callee)
+ // r5: expected arguments count
+
+ // The code below is made a lot easier because the calling code already sets
+ // up actual and expected registers according to the contract if values are
+ // passed in registers.
+
+ // The code below is made a lot easier because the calling code already sets
+ // up actual and expected registers according to the contract.
+ // ARM has some checks as per below, considering add them for PPC
+ // DCHECK_EQ(actual_parameter_count, r3);
+ // DCHECK_EQ(expected_parameter_count, r5);
+
+ cmp(expected_parameter_count, actual_parameter_count);
+ beq(®ular_invoke);
+
+ Handle<Code> adaptor = BUILTIN_CODE(isolate(), ArgumentsAdaptorTrampoline);
+ if (flag == CALL_FUNCTION) {
+ Call(adaptor);
+ b(done);
+ } else {
+ Jump(adaptor, RelocInfo::CODE_TARGET);
+ }
+ bind(®ular_invoke);
+}
+
+void MacroAssembler::CheckDebugHook(Register fun, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count) {
+ Label skip_hook;
+
+ ExternalReference debug_hook_active =
+ ExternalReference::debug_hook_on_function_call_address(isolate());
+ Move(r7, debug_hook_active);
+ LoadByte(r7, MemOperand(r7), r0);
+ extsb(r7, r7);
+ CmpSmiLiteral(r7, Smi::zero(), r0);
+ beq(&skip_hook);
+
+ {
+ // Load receiver to pass it later to DebugOnFunctionCall hook.
+ LoadReceiver(r7, actual_parameter_count);
+ FrameScope frame(this,
+ has_frame() ? StackFrame::NONE : StackFrame::INTERNAL);
+
+ SmiTag(expected_parameter_count);
+ Push(expected_parameter_count);
+
+ SmiTag(actual_parameter_count);
+ Push(actual_parameter_count);
+
+ if (new_target.is_valid()) {
+ Push(new_target);
+ }
+ Push(fun, fun, r7);
+ CallRuntime(Runtime::kDebugOnFunctionCall);
+ Pop(fun);
+ if (new_target.is_valid()) {
+ Pop(new_target);
+ }
+
+ Pop(actual_parameter_count);
+ SmiUntag(actual_parameter_count);
+
+ Pop(expected_parameter_count);
+ SmiUntag(expected_parameter_count);
+ }
+ bind(&skip_hook);
+}
+
+void MacroAssembler::InvokeFunctionCode(Register function, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count,
+ InvokeFlag flag) {
+ // You can't call a function without a valid frame.
+ DCHECK_IMPLIES(flag == CALL_FUNCTION, has_frame());
+ DCHECK_EQ(function, r4);
+ DCHECK_IMPLIES(new_target.is_valid(), new_target == r6);
+
+ // On function call, call into the debugger if necessary.
+ CheckDebugHook(function, new_target, expected_parameter_count,
+ actual_parameter_count);
+
+ // Clear the new.target register if not given.
+ if (!new_target.is_valid()) {
+ LoadRoot(r6, RootIndex::kUndefinedValue);
+ }
+
+ Label done;
+ InvokePrologue(expected_parameter_count, actual_parameter_count, &done, flag);
+ // We call indirectly through the code field in the function to
+ // allow recompilation to take effect without changing any of the
+ // call sites.
+ Register code = kJavaScriptCallCodeStartRegister;
+ LoadTaggedPointerField(code,
+ FieldMemOperand(function, JSFunction::kCodeOffset));
+ if (flag == CALL_FUNCTION) {
+ CallCodeObject(code);
+ } else {
+ DCHECK(flag == JUMP_FUNCTION);
+ JumpCodeObject(code);
+ }
+
+ // Continue here if InvokePrologue does handle the invocation due to
+ // mismatched parameter counts.
+ bind(&done);
+}
+
+void MacroAssembler::InvokeFunctionWithNewTarget(
+ Register fun, Register new_target, Register actual_parameter_count,
+ InvokeFlag flag) {
+ // You can't call a function without a valid frame.
+ DCHECK_IMPLIES(flag == CALL_FUNCTION, has_frame());
+
+ // Contract with called JS functions requires that function is passed in r4.
+ DCHECK_EQ(fun, r4);
+
+ Register expected_reg = r5;
+ Register temp_reg = r7;
+
+ LoadTaggedPointerField(
+ temp_reg, FieldMemOperand(r4, JSFunction::kSharedFunctionInfoOffset));
+ LoadTaggedPointerField(cp, FieldMemOperand(r4, JSFunction::kContextOffset));
+ LoadHalfWord(expected_reg,
+ FieldMemOperand(
+ temp_reg, SharedFunctionInfo::kFormalParameterCountOffset));
+
+ InvokeFunctionCode(fun, new_target, expected_reg, actual_parameter_count,
+ flag);
+}
+
+void MacroAssembler::InvokeFunction(Register function,
+ Register expected_parameter_count,
+ Register actual_parameter_count,
+ InvokeFlag flag) {
+ // You can't call a function without a valid frame.
+ DCHECK_IMPLIES(flag == CALL_FUNCTION, has_frame());
+
+ // Contract with called JS functions requires that function is passed in r4.
+ DCHECK_EQ(function, r4);
+
+ // Get the function and setup the context.
+ LoadTaggedPointerField(cp, FieldMemOperand(r4, JSFunction::kContextOffset));
+
+ InvokeFunctionCode(r4, no_reg, expected_parameter_count,
+ actual_parameter_count, flag);
+}
+
+void MacroAssembler::MaybeDropFrames() {
+ // Check whether we need to drop frames to restart a function on the stack.
+ ExternalReference restart_fp =
+ ExternalReference::debug_restart_fp_address(isolate());
+ Move(r4, restart_fp);
+ LoadP(r4, MemOperand(r4));
+ cmpi(r4, Operand::Zero());
+ Jump(BUILTIN_CODE(isolate(), FrameDropperTrampoline), RelocInfo::CODE_TARGET,
+ ne);
+}
+
+void MacroAssembler::PushStackHandler() {
+ // Adjust this code if not the case.
+ STATIC_ASSERT(StackHandlerConstants::kSize == 2 * kSystemPointerSize);
+ STATIC_ASSERT(StackHandlerConstants::kNextOffset == 0 * kSystemPointerSize);
+
+ Push(Smi::zero()); // Padding.
+
+ // Link the current handler as the next handler.
+ // Preserve r4-r8.
+ Move(r3,
+ ExternalReference::Create(IsolateAddressId::kHandlerAddress, isolate()));
+ LoadP(r0, MemOperand(r3));
+ push(r0);
+
+ // Set this new handler as the current one.
+ StoreP(sp, MemOperand(r3));
+}
+
+void MacroAssembler::PopStackHandler() {
+ STATIC_ASSERT(StackHandlerConstants::kSize == 2 * kSystemPointerSize);
+ STATIC_ASSERT(StackHandlerConstants::kNextOffset == 0);
+
+ pop(r4);
+ Move(ip,
+ ExternalReference::Create(IsolateAddressId::kHandlerAddress, isolate()));
+ StoreP(r4, MemOperand(ip));
+
+ Drop(1); // Drop padding.
+}
+
+void MacroAssembler::CompareObjectType(Register object, Register map,
+ Register type_reg, InstanceType type) {
+ const Register temp = type_reg == no_reg ? r0 : type_reg;
+
+ LoadMap(map, object);
+ CompareInstanceType(map, temp, type);
+}
+
+void MacroAssembler::CompareInstanceType(Register map, Register type_reg,
+ InstanceType type) {
+ STATIC_ASSERT(Map::kInstanceTypeOffset < 4096);
+ STATIC_ASSERT(LAST_TYPE <= 0xFFFF);
+ lhz(type_reg, FieldMemOperand(map, Map::kInstanceTypeOffset));
+ cmpi(type_reg, Operand(type));
+}
+
+void MacroAssembler::CompareRoot(Register obj, RootIndex index) {
+ DCHECK(obj != r0);
+ LoadRoot(r0, index);
+ cmp(obj, r0);
+}
+
+void TurboAssembler::AddAndCheckForOverflow(Register dst, Register left,
+ Register right,
+ Register overflow_dst,
+ Register scratch) {
+ DCHECK(dst != overflow_dst);
+ DCHECK(dst != scratch);
+ DCHECK(overflow_dst != scratch);
+ DCHECK(overflow_dst != left);
+ DCHECK(overflow_dst != right);
+
+ bool left_is_right = left == right;
+ RCBit xorRC = left_is_right ? SetRC : LeaveRC;
+
+ // C = A+B; C overflows if A/B have same sign and C has diff sign than A
+ if (dst == left) {
+ mr(scratch, left); // Preserve left.
+ add(dst, left, right); // Left is overwritten.
+ xor_(overflow_dst, dst, scratch, xorRC); // Original left.
+ if (!left_is_right) xor_(scratch, dst, right);
+ } else if (dst == right) {
+ mr(scratch, right); // Preserve right.
+ add(dst, left, right); // Right is overwritten.
+ xor_(overflow_dst, dst, left, xorRC);
+ if (!left_is_right) xor_(scratch, dst, scratch); // Original right.
+ } else {
+ add(dst, left, right);
+ xor_(overflow_dst, dst, left, xorRC);
+ if (!left_is_right) xor_(scratch, dst, right);
+ }
+ if (!left_is_right) and_(overflow_dst, scratch, overflow_dst, SetRC);
+}
+
+void TurboAssembler::AddAndCheckForOverflow(Register dst, Register left,
+ intptr_t right,
+ Register overflow_dst,
+ Register scratch) {
+ Register original_left = left;
+ DCHECK(dst != overflow_dst);
+ DCHECK(dst != scratch);
+ DCHECK(overflow_dst != scratch);
+ DCHECK(overflow_dst != left);
+
+ // C = A+B; C overflows if A/B have same sign and C has diff sign than A
+ if (dst == left) {
+ // Preserve left.
+ original_left = overflow_dst;
+ mr(original_left, left);
+ }
+ Add(dst, left, right, scratch);
+ xor_(overflow_dst, dst, original_left);
+ if (right >= 0) {
+ and_(overflow_dst, overflow_dst, dst, SetRC);
+ } else {
+ andc(overflow_dst, overflow_dst, dst, SetRC);
+ }
+}
+
+void TurboAssembler::SubAndCheckForOverflow(Register dst, Register left,
+ Register right,
+ Register overflow_dst,
+ Register scratch) {
+ DCHECK(dst != overflow_dst);
+ DCHECK(dst != scratch);
+ DCHECK(overflow_dst != scratch);
+ DCHECK(overflow_dst != left);
+ DCHECK(overflow_dst != right);
+
+ // C = A-B; C overflows if A/B have diff signs and C has diff sign than A
+ if (dst == left) {
+ mr(scratch, left); // Preserve left.
+ sub(dst, left, right); // Left is overwritten.
+ xor_(overflow_dst, dst, scratch);
+ xor_(scratch, scratch, right);
+ and_(overflow_dst, overflow_dst, scratch, SetRC);
+ } else if (dst == right) {
+ mr(scratch, right); // Preserve right.
+ sub(dst, left, right); // Right is overwritten.
+ xor_(overflow_dst, dst, left);
+ xor_(scratch, left, scratch);
+ and_(overflow_dst, overflow_dst, scratch, SetRC);
+ } else {
+ sub(dst, left, right);
+ xor_(overflow_dst, dst, left);
+ xor_(scratch, left, right);
+ and_(overflow_dst, scratch, overflow_dst, SetRC);
+ }
+}
+
+void MacroAssembler::JumpIfIsInRange(Register value, unsigned lower_limit,
+ unsigned higher_limit,
+ Label* on_in_range) {
+ Register scratch = r0;
+ if (lower_limit != 0) {
+ mov(scratch, Operand(lower_limit));
+ sub(scratch, value, scratch);
+ cmpli(scratch, Operand(higher_limit - lower_limit));
+ } else {
+ mov(scratch, Operand(higher_limit));
+ cmpl(value, scratch);
+ }
+ ble(on_in_range);
+}
+
+void TurboAssembler::TruncateDoubleToI(Isolate* isolate, Zone* zone,
+ Register result,
+ DoubleRegister double_input,
+ StubCallMode stub_mode) {
+ Label done;
+
+ TryInlineTruncateDoubleToI(result, double_input, &done);
+
+ // If we fell through then inline version didn't succeed - call stub instead.
+ mflr(r0);
+ push(r0);
+ // Put input on stack.
+ stfdu(double_input, MemOperand(sp, -kDoubleSize));
+
+ if (stub_mode == StubCallMode::kCallWasmRuntimeStub) {
+ Call(wasm::WasmCode::kDoubleToI, RelocInfo::WASM_STUB_CALL);
+ } else {
+ Call(BUILTIN_CODE(isolate, DoubleToI), RelocInfo::CODE_TARGET);
+ }
+
+ LoadP(result, MemOperand(sp));
+ addi(sp, sp, Operand(kDoubleSize));
+ pop(r0);
+ mtlr(r0);
+
+ bind(&done);
+}
+
+void TurboAssembler::TryInlineTruncateDoubleToI(Register result,
+ DoubleRegister double_input,
+ Label* done) {
+ DoubleRegister double_scratch = kScratchDoubleReg;
+#if !V8_TARGET_ARCH_PPC64
+ Register scratch = ip;
+#endif
+
+ ConvertDoubleToInt64(double_input,
+#if !V8_TARGET_ARCH_PPC64
+ scratch,
+#endif
+ result, double_scratch);
+
+// Test for overflow
+#if V8_TARGET_ARCH_PPC64
+ TestIfInt32(result, r0);
+#else
+ TestIfInt32(scratch, result, r0);
+#endif
+ beq(done);
+}
+
+void MacroAssembler::CallRuntime(const Runtime::Function* f, int num_arguments,
+ SaveFPRegsMode save_doubles) {
+ // All parameters are on the stack. r3 has the return value after call.
+
+ // If the expected number of arguments of the runtime function is
+ // constant, we check that the actual number of arguments match the
+ // expectation.
+ CHECK(f->nargs < 0 || f->nargs == num_arguments);
+
+ // TODO(1236192): Most runtime routines don't need the number of
+ // arguments passed in because it is constant. At some point we
+ // should remove this need and make the runtime routine entry code
+ // smarter.
+ mov(r3, Operand(num_arguments));
+ Move(r4, ExternalReference::Create(f));
+#if V8_TARGET_ARCH_PPC64
+ Handle<Code> code =
+ CodeFactory::CEntry(isolate(), f->result_size, save_doubles);
+#else
+ Handle<Code> code = CodeFactory::CEntry(isolate(), 1, save_doubles);
+#endif
+ Call(code, RelocInfo::CODE_TARGET);
+}
+
+void MacroAssembler::TailCallRuntime(Runtime::FunctionId fid) {
+ const Runtime::Function* function = Runtime::FunctionForId(fid);
+ DCHECK_EQ(1, function->result_size);
+ if (function->nargs >= 0) {
+ mov(r3, Operand(function->nargs));
+ }
+ JumpToExternalReference(ExternalReference::Create(fid));
+}
+
+void MacroAssembler::JumpToExternalReference(const ExternalReference& builtin,
+ bool builtin_exit_frame) {
+ Move(r4, builtin);
+ Handle<Code> code = CodeFactory::CEntry(isolate(), 1, kDontSaveFPRegs,
+ kArgvOnStack, builtin_exit_frame);
+ Jump(code, RelocInfo::CODE_TARGET);
+}
+
+void MacroAssembler::JumpToInstructionStream(Address entry) {
+ mov(kOffHeapTrampolineRegister, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+ Jump(kOffHeapTrampolineRegister);
+}
+
+void MacroAssembler::LoadWeakValue(Register out, Register in,
+ Label* target_if_cleared) {
+ cmpwi(in, Operand(kClearedWeakHeapObjectLower32));
+ beq(target_if_cleared);
+
+ mov(r0, Operand(~kWeakHeapObjectMask));
+ and_(out, in, r0);
+}
+
+void MacroAssembler::IncrementCounter(StatsCounter* counter, int value,
+ Register scratch1, Register scratch2) {
+ DCHECK_GT(value, 0);
+ if (FLAG_native_code_counters && counter->Enabled()) {
+ // This operation has to be exactly 32-bit wide in case the external
+ // reference table redirects the counter to a uint32_t dummy_stats_counter_
+ // field.
+ Move(scratch2, ExternalReference::Create(counter));
+ lwz(scratch1, MemOperand(scratch2));
+ addi(scratch1, scratch1, Operand(value));
+ stw(scratch1, MemOperand(scratch2));
+ }
+}
+
+void MacroAssembler::DecrementCounter(StatsCounter* counter, int value,
+ Register scratch1, Register scratch2) {
+ DCHECK_GT(value, 0);
+ if (FLAG_native_code_counters && counter->Enabled()) {
+ // This operation has to be exactly 32-bit wide in case the external
+ // reference table redirects the counter to a uint32_t dummy_stats_counter_
+ // field.
+ Move(scratch2, ExternalReference::Create(counter));
+ lwz(scratch1, MemOperand(scratch2));
+ subi(scratch1, scratch1, Operand(value));
+ stw(scratch1, MemOperand(scratch2));
+ }
+}
+
+void TurboAssembler::Assert(Condition cond, AbortReason reason, CRegister cr) {
+ if (emit_debug_code()) Check(cond, reason, cr);
+}
+
+void TurboAssembler::Check(Condition cond, AbortReason reason, CRegister cr) {
+ Label L;
+ b(cond, &L, cr);
+ Abort(reason);
+ // will not return here
+ bind(&L);
+}
+
+void TurboAssembler::Abort(AbortReason reason) {
+ Label abort_start;
+ bind(&abort_start);
+#ifdef DEBUG
+ const char* msg = GetAbortReason(reason);
+ RecordComment("Abort message: ");
+ RecordComment(msg);
+#endif
+
+ // Avoid emitting call to builtin if requested.
+ if (trap_on_abort()) {
+ stop();
+ return;
+ }
+
+ if (should_abort_hard()) {
+ // We don't care if we constructed a frame. Just pretend we did.
+ FrameScope assume_frame(this, StackFrame::NONE);
+ mov(r3, Operand(static_cast<int>(reason)));
+ PrepareCallCFunction(1, r4);
+ CallCFunction(ExternalReference::abort_with_reason(), 1);
+ return;
+ }
+
+ LoadSmiLiteral(r4, Smi::FromInt(static_cast<int>(reason)));
+
+ // Disable stub call restrictions to always allow calls to abort.
+ if (!has_frame_) {
+ // We don't actually want to generate a pile of code for this, so just
+ // claim there is a stack frame, without generating one.
+ FrameScope scope(this, StackFrame::NONE);
+ Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
+ } else {
+ Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
+ }
+ // will not return here
+}
+
+void MacroAssembler::LoadMap(Register destination, Register object) {
+ LoadTaggedPointerField(destination,
+ FieldMemOperand(object, HeapObject::kMapOffset));
+}
+
+void MacroAssembler::LoadNativeContextSlot(int index, Register dst) {
+ LoadMap(dst, cp);
+ LoadTaggedPointerField(
+ dst, FieldMemOperand(
+ dst, Map::kConstructorOrBackPointerOrNativeContextOffset));
+ LoadTaggedPointerField(dst, MemOperand(dst, Context::SlotOffset(index)));
+}
+
+void MacroAssembler::AssertNotSmi(Register object) {
+ if (emit_debug_code()) {
+ STATIC_ASSERT(kSmiTag == 0);
+ TestIfSmi(object, r0);
+ Check(ne, AbortReason::kOperandIsASmi, cr0);
+ }
+}
+
+void MacroAssembler::AssertSmi(Register object) {
+ if (emit_debug_code()) {
+ STATIC_ASSERT(kSmiTag == 0);
+ TestIfSmi(object, r0);
+ Check(eq, AbortReason::kOperandIsNotASmi, cr0);
+ }
+}
+
+void MacroAssembler::AssertConstructor(Register object) {
+ if (emit_debug_code()) {
+ STATIC_ASSERT(kSmiTag == 0);
+ TestIfSmi(object, r0);
+ Check(ne, AbortReason::kOperandIsASmiAndNotAConstructor, cr0);
+ push(object);
+ LoadMap(object, object);
+ lbz(object, FieldMemOperand(object, Map::kBitFieldOffset));
+ andi(object, object, Operand(Map::Bits1::IsConstructorBit::kMask));
+ pop(object);
+ Check(ne, AbortReason::kOperandIsNotAConstructor, cr0);
+ }
+}
+
+void MacroAssembler::AssertFunction(Register object) {
+ if (emit_debug_code()) {
+ STATIC_ASSERT(kSmiTag == 0);
+ TestIfSmi(object, r0);
+ Check(ne, AbortReason::kOperandIsASmiAndNotAFunction, cr0);
+ push(object);
+ CompareObjectType(object, object, object, JS_FUNCTION_TYPE);
+ pop(object);
+ Check(eq, AbortReason::kOperandIsNotAFunction);
+ }
+}
+
+void MacroAssembler::AssertBoundFunction(Register object) {
+ if (emit_debug_code()) {
+ STATIC_ASSERT(kSmiTag == 0);
+ TestIfSmi(object, r0);
+ Check(ne, AbortReason::kOperandIsASmiAndNotABoundFunction, cr0);
+ push(object);
+ CompareObjectType(object, object, object, JS_BOUND_FUNCTION_TYPE);
+ pop(object);
+ Check(eq, AbortReason::kOperandIsNotABoundFunction);
+ }
+}
+
+void MacroAssembler::AssertGeneratorObject(Register object) {
+ if (!emit_debug_code()) return;
+ TestIfSmi(object, r0);
+ Check(ne, AbortReason::kOperandIsASmiAndNotAGeneratorObject, cr0);
+
+ // Load map
+ Register map = object;
+ push(object);
+ LoadMap(map, object);
+
+ // Check if JSGeneratorObject
+ Label do_check;
+ Register instance_type = object;
+ CompareInstanceType(map, instance_type, JS_GENERATOR_OBJECT_TYPE);
+ beq(&do_check);
+
+ // Check if JSAsyncFunctionObject (See MacroAssembler::CompareInstanceType)
+ cmpi(instance_type, Operand(JS_ASYNC_FUNCTION_OBJECT_TYPE));
+ beq(&do_check);
+
+ // Check if JSAsyncGeneratorObject (See MacroAssembler::CompareInstanceType)
+ cmpi(instance_type, Operand(JS_ASYNC_GENERATOR_OBJECT_TYPE));
+
+ bind(&do_check);
+ // Restore generator object to register and perform assertion
+ pop(object);
+ Check(eq, AbortReason::kOperandIsNotAGeneratorObject);
+}
+
+void MacroAssembler::AssertUndefinedOrAllocationSite(Register object,
+ Register scratch) {
+ if (emit_debug_code()) {
+ Label done_checking;
+ AssertNotSmi(object);
+ CompareRoot(object, RootIndex::kUndefinedValue);
+ beq(&done_checking);
+ LoadMap(scratch, object);
+ CompareInstanceType(scratch, scratch, ALLOCATION_SITE_TYPE);
+ Assert(eq, AbortReason::kExpectedUndefinedOrCell);
+ bind(&done_checking);
+ }
+}
+
+static const int kRegisterPassedArguments = 8;
+
+int TurboAssembler::CalculateStackPassedWords(int num_reg_arguments,
+ int num_double_arguments) {
+ int stack_passed_words = 0;
+ if (num_double_arguments > DoubleRegister::kNumRegisters) {
+ stack_passed_words +=
+ 2 * (num_double_arguments - DoubleRegister::kNumRegisters);
+ }
+ // Up to 8 simple arguments are passed in registers r3..r10.
+ if (num_reg_arguments > kRegisterPassedArguments) {
+ stack_passed_words += num_reg_arguments - kRegisterPassedArguments;
+ }
+ return stack_passed_words;
+}
+
+void TurboAssembler::PrepareCallCFunction(int num_reg_arguments,
+ int num_double_arguments,
+ Register scratch) {
+ int frame_alignment = ActivationFrameAlignment();
+ int stack_passed_arguments =
+ CalculateStackPassedWords(num_reg_arguments, num_double_arguments);
+ int stack_space = kNumRequiredStackFrameSlots;
+
+ if (frame_alignment > kSystemPointerSize) {
+ // Make stack end at alignment and make room for stack arguments
+ // -- preserving original value of sp.
+ mr(scratch, sp);
+ addi(sp, sp, Operand(-(stack_passed_arguments + 1) * kSystemPointerSize));
+ DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
+ ClearRightImm(sp, sp,
+ Operand(base::bits::WhichPowerOfTwo(frame_alignment)));
+ StoreP(scratch,
+ MemOperand(sp, stack_passed_arguments * kSystemPointerSize));
+ } else {
+ // Make room for stack arguments
+ stack_space += stack_passed_arguments;
+ }
+
+ // Allocate frame with required slots to make ABI work.
+ li(r0, Operand::Zero());
+ StorePU(r0, MemOperand(sp, -stack_space * kSystemPointerSize));
+}
+
+void TurboAssembler::PrepareCallCFunction(int num_reg_arguments,
+ Register scratch) {
+ PrepareCallCFunction(num_reg_arguments, 0, scratch);
+}
+
+void TurboAssembler::MovToFloatParameter(DoubleRegister src) { Move(d1, src); }
+
+void TurboAssembler::MovToFloatResult(DoubleRegister src) { Move(d1, src); }
+
+void TurboAssembler::MovToFloatParameters(DoubleRegister src1,
+ DoubleRegister src2) {
+ if (src2 == d1) {
+ DCHECK(src1 != d2);
+ Move(d2, src2);
+ Move(d1, src1);
+ } else {
+ Move(d1, src1);
+ Move(d2, src2);
+ }
+}
+
+void TurboAssembler::CallCFunction(ExternalReference function,
+ int num_reg_arguments,
+ int num_double_arguments,
+ bool has_function_descriptor) {
+ Move(ip, function);
+ CallCFunctionHelper(ip, num_reg_arguments, num_double_arguments,
+ has_function_descriptor);
+}
+
+void TurboAssembler::CallCFunction(Register function, int num_reg_arguments,
+ int num_double_arguments,
+ bool has_function_descriptor) {
+ CallCFunctionHelper(function, num_reg_arguments, num_double_arguments,
+ has_function_descriptor);
+}
+
+void TurboAssembler::CallCFunction(ExternalReference function,
+ int num_arguments,
+ bool has_function_descriptor) {
+ CallCFunction(function, num_arguments, 0, has_function_descriptor);
+}
+
+void TurboAssembler::CallCFunction(Register function, int num_arguments,
+ bool has_function_descriptor) {
+ CallCFunction(function, num_arguments, 0, has_function_descriptor);
+}
+
+void TurboAssembler::CallCFunctionHelper(Register function,
+ int num_reg_arguments,
+ int num_double_arguments,
+ bool has_function_descriptor) {
+ DCHECK_LE(num_reg_arguments + num_double_arguments, kMaxCParameters);
+ DCHECK(has_frame());
+
+ // Save the frame pointer and PC so that the stack layout remains iterable,
+ // even without an ExitFrame which normally exists between JS and C frames.
+ Register addr_scratch = r7;
+ Register scratch = r8;
+ Push(scratch);
+ mflr(scratch);
+ // See x64 code for reasoning about how to address the isolate data fields.
+ if (root_array_available()) {
+ LoadPC(r0);
+ StoreP(r0, MemOperand(kRootRegister,
+ IsolateData::fast_c_call_caller_pc_offset()));
+ StoreP(fp, MemOperand(kRootRegister,
+ IsolateData::fast_c_call_caller_fp_offset()));
+ } else {
+ DCHECK_NOT_NULL(isolate());
+ Push(addr_scratch);
+
+ Move(addr_scratch,
+ ExternalReference::fast_c_call_caller_pc_address(isolate()));
+ LoadPC(r0);
+ StoreP(r0, MemOperand(addr_scratch));
+ Move(addr_scratch,
+ ExternalReference::fast_c_call_caller_fp_address(isolate()));
+ StoreP(fp, MemOperand(addr_scratch));
+ Pop(addr_scratch);
+ }
+ mtlr(scratch);
+ Pop(scratch);
+
+ // Just call directly. The function called cannot cause a GC, or
+ // allow preemption, so the return address in the link register
+ // stays correct.
+ Register dest = function;
+ if (ABI_USES_FUNCTION_DESCRIPTORS && has_function_descriptor) {
+ // AIX/PPC64BE Linux uses a function descriptor. When calling C code be
+ // aware of this descriptor and pick up values from it
+ LoadP(ToRegister(ABI_TOC_REGISTER),
+ MemOperand(function, kSystemPointerSize));
+ LoadP(ip, MemOperand(function, 0));
+ dest = ip;
+ } else if (ABI_CALL_VIA_IP) {
+ // pLinux and Simualtor, not AIX
+ Move(ip, function);
+ dest = ip;
+ }
+
+ Call(dest);
+
+ // We don't unset the PC; the FP is the source of truth.
+ Register zero_scratch = r0;
+ mov(zero_scratch, Operand::Zero());
+
+ if (root_array_available()) {
+ StoreP(
+ zero_scratch,
+ MemOperand(kRootRegister, IsolateData::fast_c_call_caller_fp_offset()));
+ } else {
+ DCHECK_NOT_NULL(isolate());
+ Push(addr_scratch);
+ Move(addr_scratch,
+ ExternalReference::fast_c_call_caller_fp_address(isolate()));
+ StoreP(zero_scratch, MemOperand(addr_scratch));
+ Pop(addr_scratch);
+ }
+
+ // Remove frame bought in PrepareCallCFunction
+ int stack_passed_arguments =
+ CalculateStackPassedWords(num_reg_arguments, num_double_arguments);
+ int stack_space = kNumRequiredStackFrameSlots + stack_passed_arguments;
+ if (ActivationFrameAlignment() > kSystemPointerSize) {
+ LoadP(sp, MemOperand(sp, stack_space * kSystemPointerSize));
+ } else {
+ addi(sp, sp, Operand(stack_space * kSystemPointerSize));
+ }
+}
+
+void TurboAssembler::CheckPageFlag(
+ Register object,
+ Register scratch, // scratch may be same register as object
+ int mask, Condition cc, Label* condition_met) {
+ DCHECK(cc == ne || cc == eq);
+ ClearRightImm(scratch, object, Operand(kPageSizeBits));
+ LoadP(scratch, MemOperand(scratch, BasicMemoryChunk::kFlagsOffset));
+
+ mov(r0, Operand(mask));
+ and_(r0, scratch, r0, SetRC);
+
+ if (cc == ne) {
+ bne(condition_met, cr0);
+ }
+ if (cc == eq) {
+ beq(condition_met, cr0);
+ }
+}
+
+void TurboAssembler::SetRoundingMode(FPRoundingMode RN) { mtfsfi(7, RN); }
+
+void TurboAssembler::ResetRoundingMode() {
+ mtfsfi(7, kRoundToNearest); // reset (default is kRoundToNearest)
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//
+// New MacroAssembler Interfaces added for PPC
+//
+////////////////////////////////////////////////////////////////////////////////
+void TurboAssembler::LoadIntLiteral(Register dst, int value) {
+ mov(dst, Operand(value));
+}
+
+void TurboAssembler::LoadSmiLiteral(Register dst, Smi smi) {
+ mov(dst, Operand(smi));
+}
+
+void TurboAssembler::LoadDoubleLiteral(DoubleRegister result, Double value,
+ Register scratch) {
+ if (FLAG_enable_embedded_constant_pool && is_constant_pool_available() &&
+ !(scratch == r0 && ConstantPoolAccessIsInOverflow())) {
+ ConstantPoolEntry::Access access = ConstantPoolAddEntry(value);
+ if (access == ConstantPoolEntry::OVERFLOWED) {
+ addis(scratch, kConstantPoolRegister, Operand::Zero());
+ lfd(result, MemOperand(scratch, 0));
+ } else {
+ lfd(result, MemOperand(kConstantPoolRegister, 0));
+ }
+ return;
+ }
+
+ // avoid gcc strict aliasing error using union cast
+ union {
+ uint64_t dval;
+#if V8_TARGET_ARCH_PPC64
+ intptr_t ival;
+#else
+ intptr_t ival[2];
+#endif
+ } litVal;
+
+ litVal.dval = value.AsUint64();
+
+#if V8_TARGET_ARCH_PPC64
+ if (CpuFeatures::IsSupported(FPR_GPR_MOV)) {
+ mov(scratch, Operand(litVal.ival));
+ mtfprd(result, scratch);
+ return;
+ }
+#endif
+
+ addi(sp, sp, Operand(-kDoubleSize));
+#if V8_TARGET_ARCH_PPC64
+ mov(scratch, Operand(litVal.ival));
+ std(scratch, MemOperand(sp));
+#else
+ LoadIntLiteral(scratch, litVal.ival[0]);
+ stw(scratch, MemOperand(sp, 0));
+ LoadIntLiteral(scratch, litVal.ival[1]);
+ stw(scratch, MemOperand(sp, 4));
+#endif
+ nop(GROUP_ENDING_NOP); // LHS/RAW optimization
+ lfd(result, MemOperand(sp, 0));
+ addi(sp, sp, Operand(kDoubleSize));
+}
+
+void TurboAssembler::MovIntToDouble(DoubleRegister dst, Register src,
+ Register scratch) {
+// sign-extend src to 64-bit
+#if V8_TARGET_ARCH_PPC64
+ if (CpuFeatures::IsSupported(FPR_GPR_MOV)) {
+ mtfprwa(dst, src);
+ return;
+ }
+#endif
+
+ DCHECK(src != scratch);
+ subi(sp, sp, Operand(kDoubleSize));
+#if V8_TARGET_ARCH_PPC64
+ extsw(scratch, src);
+ std(scratch, MemOperand(sp, 0));
+#else
+ srawi(scratch, src, 31);
+ stw(scratch, MemOperand(sp, Register::kExponentOffset));
+ stw(src, MemOperand(sp, Register::kMantissaOffset));
+#endif
+ nop(GROUP_ENDING_NOP); // LHS/RAW optimization
+ lfd(dst, MemOperand(sp, 0));
+ addi(sp, sp, Operand(kDoubleSize));
+}
+
+void TurboAssembler::MovUnsignedIntToDouble(DoubleRegister dst, Register src,
+ Register scratch) {
+// zero-extend src to 64-bit
+#if V8_TARGET_ARCH_PPC64
+ if (CpuFeatures::IsSupported(FPR_GPR_MOV)) {
+ mtfprwz(dst, src);
+ return;
+ }
+#endif
+
+ DCHECK(src != scratch);
+ subi(sp, sp, Operand(kDoubleSize));
+#if V8_TARGET_ARCH_PPC64
+ clrldi(scratch, src, Operand(32));
+ std(scratch, MemOperand(sp, 0));
+#else
+ li(scratch, Operand::Zero());
+ stw(scratch, MemOperand(sp, Register::kExponentOffset));
+ stw(src, MemOperand(sp, Register::kMantissaOffset));
+#endif
+ nop(GROUP_ENDING_NOP); // LHS/RAW optimization
+ lfd(dst, MemOperand(sp, 0));
+ addi(sp, sp, Operand(kDoubleSize));
+}
+
+void TurboAssembler::MovInt64ToDouble(DoubleRegister dst,
+#if !V8_TARGET_ARCH_PPC64
+ Register src_hi,
+#endif
+ Register src) {
+#if V8_TARGET_ARCH_PPC64
+ if (CpuFeatures::IsSupported(FPR_GPR_MOV)) {
+ mtfprd(dst, src);
+ return;
+ }
+#endif
+
+ subi(sp, sp, Operand(kDoubleSize));
+#if V8_TARGET_ARCH_PPC64
+ std(src, MemOperand(sp, 0));
+#else
+ stw(src_hi, MemOperand(sp, Register::kExponentOffset));
+ stw(src, MemOperand(sp, Register::kMantissaOffset));
+#endif
+ nop(GROUP_ENDING_NOP); // LHS/RAW optimization
+ lfd(dst, MemOperand(sp, 0));
+ addi(sp, sp, Operand(kDoubleSize));
+}
+
+#if V8_TARGET_ARCH_PPC64
+void TurboAssembler::MovInt64ComponentsToDouble(DoubleRegister dst,
+ Register src_hi,
+ Register src_lo,
+ Register scratch) {
+ if (CpuFeatures::IsSupported(FPR_GPR_MOV)) {
+ sldi(scratch, src_hi, Operand(32));
+ rldimi(scratch, src_lo, 0, 32);
+ mtfprd(dst, scratch);
+ return;
+ }
+
+ subi(sp, sp, Operand(kDoubleSize));
+ stw(src_hi, MemOperand(sp, Register::kExponentOffset));
+ stw(src_lo, MemOperand(sp, Register::kMantissaOffset));
+ nop(GROUP_ENDING_NOP); // LHS/RAW optimization
+ lfd(dst, MemOperand(sp));
+ addi(sp, sp, Operand(kDoubleSize));
+}
+#endif
+
+void TurboAssembler::InsertDoubleLow(DoubleRegister dst, Register src,
+ Register scratch) {
+#if V8_TARGET_ARCH_PPC64
+ if (CpuFeatures::IsSupported(FPR_GPR_MOV)) {
+ mffprd(scratch, dst);
+ rldimi(scratch, src, 0, 32);
+ mtfprd(dst, scratch);
+ return;
+ }
+#endif
+
+ subi(sp, sp, Operand(kDoubleSize));
+ stfd(dst, MemOperand(sp));
+ stw(src, MemOperand(sp, Register::kMantissaOffset));
+ nop(GROUP_ENDING_NOP); // LHS/RAW optimization
+ lfd(dst, MemOperand(sp));
+ addi(sp, sp, Operand(kDoubleSize));
+}
+
+void TurboAssembler::InsertDoubleHigh(DoubleRegister dst, Register src,
+ Register scratch) {
+#if V8_TARGET_ARCH_PPC64
+ if (CpuFeatures::IsSupported(FPR_GPR_MOV)) {
+ mffprd(scratch, dst);
+ rldimi(scratch, src, 32, 0);
+ mtfprd(dst, scratch);
+ return;
+ }
+#endif
+
+ subi(sp, sp, Operand(kDoubleSize));
+ stfd(dst, MemOperand(sp));
+ stw(src, MemOperand(sp, Register::kExponentOffset));
+ nop(GROUP_ENDING_NOP); // LHS/RAW optimization
+ lfd(dst, MemOperand(sp));
+ addi(sp, sp, Operand(kDoubleSize));
+}
+
+void TurboAssembler::MovDoubleLowToInt(Register dst, DoubleRegister src) {
+#if V8_TARGET_ARCH_PPC64
+ if (CpuFeatures::IsSupported(FPR_GPR_MOV)) {
+ mffprwz(dst, src);
+ return;
+ }
+#endif
+
+ subi(sp, sp, Operand(kDoubleSize));
+ stfd(src, MemOperand(sp));
+ nop(GROUP_ENDING_NOP); // LHS/RAW optimization
+ lwz(dst, MemOperand(sp, Register::kMantissaOffset));
+ addi(sp, sp, Operand(kDoubleSize));
+}
+
+void TurboAssembler::MovDoubleHighToInt(Register dst, DoubleRegister src) {
+#if V8_TARGET_ARCH_PPC64
+ if (CpuFeatures::IsSupported(FPR_GPR_MOV)) {
+ mffprd(dst, src);
+ srdi(dst, dst, Operand(32));
+ return;
+ }
+#endif
+
+ subi(sp, sp, Operand(kDoubleSize));
+ stfd(src, MemOperand(sp));
+ nop(GROUP_ENDING_NOP); // LHS/RAW optimization
+ lwz(dst, MemOperand(sp, Register::kExponentOffset));
+ addi(sp, sp, Operand(kDoubleSize));
+}
+
+void TurboAssembler::MovDoubleToInt64(
+#if !V8_TARGET_ARCH_PPC64
+ Register dst_hi,
+#endif
+ Register dst, DoubleRegister src) {
+#if V8_TARGET_ARCH_PPC64
+ if (CpuFeatures::IsSupported(FPR_GPR_MOV)) {
+ mffprd(dst, src);
+ return;
+ }
+#endif
+
+ subi(sp, sp, Operand(kDoubleSize));
+ stfd(src, MemOperand(sp));
+ nop(GROUP_ENDING_NOP); // LHS/RAW optimization
+#if V8_TARGET_ARCH_PPC64
+ ld(dst, MemOperand(sp, 0));
+#else
+ lwz(dst_hi, MemOperand(sp, Register::kExponentOffset));
+ lwz(dst, MemOperand(sp, Register::kMantissaOffset));
+#endif
+ addi(sp, sp, Operand(kDoubleSize));
+}
+
+void TurboAssembler::MovIntToFloat(DoubleRegister dst, Register src) {
+ subi(sp, sp, Operand(kFloatSize));
+ stw(src, MemOperand(sp, 0));
+ nop(GROUP_ENDING_NOP); // LHS/RAW optimization
+ lfs(dst, MemOperand(sp, 0));
+ addi(sp, sp, Operand(kFloatSize));
+}
+
+void TurboAssembler::MovFloatToInt(Register dst, DoubleRegister src) {
+ subi(sp, sp, Operand(kFloatSize));
+ stfs(src, MemOperand(sp, 0));
+ nop(GROUP_ENDING_NOP); // LHS/RAW optimization
+ lwz(dst, MemOperand(sp, 0));
+ addi(sp, sp, Operand(kFloatSize));
+}
+
+void TurboAssembler::Add(Register dst, Register src, intptr_t value,
+ Register scratch) {
+ if (is_int16(value)) {
+ addi(dst, src, Operand(value));
+ } else {
+ mov(scratch, Operand(value));
+ add(dst, src, scratch);
+ }
+}
+
+void TurboAssembler::Cmpi(Register src1, const Operand& src2, Register scratch,
+ CRegister cr) {
+ intptr_t value = src2.immediate();
+ if (is_int16(value)) {
+ cmpi(src1, src2, cr);
+ } else {
+ mov(scratch, src2);
+ cmp(src1, scratch, cr);
+ }
+}
+
+void TurboAssembler::Cmpli(Register src1, const Operand& src2, Register scratch,
+ CRegister cr) {
+ intptr_t value = src2.immediate();
+ if (is_uint16(value)) {
+ cmpli(src1, src2, cr);
+ } else {
+ mov(scratch, src2);
+ cmpl(src1, scratch, cr);
+ }
+}
+
+void TurboAssembler::Cmpwi(Register src1, const Operand& src2, Register scratch,
+ CRegister cr) {
+ intptr_t value = src2.immediate();
+ if (is_int16(value)) {
+ cmpwi(src1, src2, cr);
+ } else {
+ mov(scratch, src2);
+ cmpw(src1, scratch, cr);
+ }
+}
+
+void MacroAssembler::Cmplwi(Register src1, const Operand& src2,
+ Register scratch, CRegister cr) {
+ intptr_t value = src2.immediate();
+ if (is_uint16(value)) {
+ cmplwi(src1, src2, cr);
+ } else {
+ mov(scratch, src2);
+ cmplw(src1, scratch, cr);
+ }
+}
+
+void MacroAssembler::And(Register ra, Register rs, const Operand& rb,
+ RCBit rc) {
+ if (rb.is_reg()) {
+ and_(ra, rs, rb.rm(), rc);
+ } else {
+ if (is_uint16(rb.immediate()) && RelocInfo::IsNone(rb.rmode_) &&
+ rc == SetRC) {
+ andi(ra, rs, rb);
+ } else {
+ // mov handles the relocation.
+ DCHECK(rs != r0);
+ mov(r0, rb);
+ and_(ra, rs, r0, rc);
+ }
+ }
+}
+
+void MacroAssembler::Or(Register ra, Register rs, const Operand& rb, RCBit rc) {
+ if (rb.is_reg()) {
+ orx(ra, rs, rb.rm(), rc);
+ } else {
+ if (is_uint16(rb.immediate()) && RelocInfo::IsNone(rb.rmode_) &&
+ rc == LeaveRC) {
+ ori(ra, rs, rb);
+ } else {
+ // mov handles the relocation.
+ DCHECK(rs != r0);
+ mov(r0, rb);
+ orx(ra, rs, r0, rc);
+ }
+ }
+}
+
+void MacroAssembler::Xor(Register ra, Register rs, const Operand& rb,
+ RCBit rc) {
+ if (rb.is_reg()) {
+ xor_(ra, rs, rb.rm(), rc);
+ } else {
+ if (is_uint16(rb.immediate()) && RelocInfo::IsNone(rb.rmode_) &&
+ rc == LeaveRC) {
+ xori(ra, rs, rb);
+ } else {
+ // mov handles the relocation.
+ DCHECK(rs != r0);
+ mov(r0, rb);
+ xor_(ra, rs, r0, rc);
+ }
+ }
+}
+
+void MacroAssembler::CmpSmiLiteral(Register src1, Smi smi, Register scratch,
+ CRegister cr) {
+#if defined(V8_COMPRESS_POINTERS) || defined(V8_31BIT_SMIS_ON_64BIT_ARCH)
+ Cmpwi(src1, Operand(smi), scratch, cr);
+#else
+ LoadSmiLiteral(scratch, smi);
+ cmp(src1, scratch, cr);
+#endif
+}
+
+void MacroAssembler::CmplSmiLiteral(Register src1, Smi smi, Register scratch,
+ CRegister cr) {
+#if defined(V8_COMPRESS_POINTERS) || defined(V8_31BIT_SMIS_ON_64BIT_ARCH)
+ Cmpli(src1, Operand(smi), scratch, cr);
+#else
+ LoadSmiLiteral(scratch, smi);
+ cmpl(src1, scratch, cr);
+#endif
+}
+
+void MacroAssembler::AddSmiLiteral(Register dst, Register src, Smi smi,
+ Register scratch) {
+#if defined(V8_COMPRESS_POINTERS) || defined(V8_31BIT_SMIS_ON_64BIT_ARCH)
+ Add(dst, src, static_cast<intptr_t>(smi.ptr()), scratch);
+#else
+ LoadSmiLiteral(scratch, smi);
+ add(dst, src, scratch);
+#endif
+}
+
+void MacroAssembler::SubSmiLiteral(Register dst, Register src, Smi smi,
+ Register scratch) {
+#if defined(V8_COMPRESS_POINTERS) || defined(V8_31BIT_SMIS_ON_64BIT_ARCH)
+ Add(dst, src, -(static_cast<intptr_t>(smi.ptr())), scratch);
+#else
+ LoadSmiLiteral(scratch, smi);
+ sub(dst, src, scratch);
+#endif
+}
+
+void MacroAssembler::AndSmiLiteral(Register dst, Register src, Smi smi,
+ Register scratch, RCBit rc) {
+#if defined(V8_COMPRESS_POINTERS) || defined(V8_31BIT_SMIS_ON_64BIT_ARCH)
+ And(dst, src, Operand(smi), rc);
+#else
+ LoadSmiLiteral(scratch, smi);
+ and_(dst, src, scratch, rc);
+#endif
+}
+
+// Load a "pointer" sized value from the memory location
+void TurboAssembler::LoadP(Register dst, const MemOperand& mem,
+ Register scratch) {
+ DCHECK_EQ(mem.rb(), no_reg);
+ int offset = mem.offset();
+ int misaligned = (offset & 3);
+ int adj = (offset & 3) - 4;
+ int alignedOffset = (offset & ~3) + 4;
+
+ if (!is_int16(offset) || (misaligned && !is_int16(alignedOffset))) {
+ /* cannot use d-form */
+ mov(scratch, Operand(offset));
+ LoadPX(dst, MemOperand(mem.ra(), scratch));
+ } else {
+ if (misaligned) {
+ // adjust base to conform to offset alignment requirements
+ // Todo: enhance to use scratch if dst is unsuitable
+ DCHECK_NE(dst, r0);
+ addi(dst, mem.ra(), Operand(adj));
+ ld(dst, MemOperand(dst, alignedOffset));
+ } else {
+ ld(dst, mem);
+ }
+ }
+}
+
+void TurboAssembler::LoadPU(Register dst, const MemOperand& mem,
+ Register scratch) {
+ int offset = mem.offset();
+
+ if (!is_int16(offset)) {
+ /* cannot use d-form */
+ DCHECK(scratch != no_reg);
+ mov(scratch, Operand(offset));
+ LoadPUX(dst, MemOperand(mem.ra(), scratch));
+ } else {
+#if V8_TARGET_ARCH_PPC64
+ ldu(dst, mem);
+#else
+ lwzu(dst, mem);
+#endif
+ }
+}
+
+// Store a "pointer" sized value to the memory location
+void TurboAssembler::StoreP(Register src, const MemOperand& mem,
+ Register scratch) {
+ int offset = mem.offset();
+
+ if (!is_int16(offset)) {
+ /* cannot use d-form */
+ DCHECK(scratch != no_reg);
+ mov(scratch, Operand(offset));
+ StorePX(src, MemOperand(mem.ra(), scratch));
+ } else {
+#if V8_TARGET_ARCH_PPC64
+ int misaligned = (offset & 3);
+ if (misaligned) {
+ // adjust base to conform to offset alignment requirements
+ // a suitable scratch is required here
+ DCHECK(scratch != no_reg);
+ if (scratch == r0) {
+ LoadIntLiteral(scratch, offset);
+ stdx(src, MemOperand(mem.ra(), scratch));
+ } else {
+ addi(scratch, mem.ra(), Operand((offset & 3) - 4));
+ std(src, MemOperand(scratch, (offset & ~3) + 4));
+ }
+ } else {
+ std(src, mem);
+ }
+#else
+ stw(src, mem);
+#endif
+ }
+}
+
+void TurboAssembler::StorePU(Register src, const MemOperand& mem,
+ Register scratch) {
+ int offset = mem.offset();
+
+ if (!is_int16(offset)) {
+ /* cannot use d-form */
+ DCHECK(scratch != no_reg);
+ mov(scratch, Operand(offset));
+ StorePUX(src, MemOperand(mem.ra(), scratch));
+ } else {
+#if V8_TARGET_ARCH_PPC64
+ stdu(src, mem);
+#else
+ stwu(src, mem);
+#endif
+ }
+}
+
+void TurboAssembler::LoadWordArith(Register dst, const MemOperand& mem,
+ Register scratch) {
+ int offset = mem.offset();
+
+ if (!is_int16(offset)) {
+ DCHECK(scratch != no_reg);
+ mov(scratch, Operand(offset));
+ lwax(dst, MemOperand(mem.ra(), scratch));
+ } else {
+#if V8_TARGET_ARCH_PPC64
+ int misaligned = (offset & 3);
+ if (misaligned) {
+ // adjust base to conform to offset alignment requirements
+ // Todo: enhance to use scratch if dst is unsuitable
+ DCHECK(dst != r0);
+ addi(dst, mem.ra(), Operand((offset & 3) - 4));
+ lwa(dst, MemOperand(dst, (offset & ~3) + 4));
+ } else {
+ lwa(dst, mem);
+ }
+#else
+ lwz(dst, mem);
+#endif
+ }
+}
+
+// Variable length depending on whether offset fits into immediate field
+// MemOperand currently only supports d-form
+void TurboAssembler::LoadWord(Register dst, const MemOperand& mem,
+ Register scratch) {
+ Register base = mem.ra();
+ int offset = mem.offset();
+
+ if (!is_int16(offset)) {
+ LoadIntLiteral(scratch, offset);
+ lwzx(dst, MemOperand(base, scratch));
+ } else {
+ lwz(dst, mem);
+ }
+}
+
+// Variable length depending on whether offset fits into immediate field
+// MemOperand current only supports d-form
+void TurboAssembler::StoreWord(Register src, const MemOperand& mem,
+ Register scratch) {
+ Register base = mem.ra();
+ int offset = mem.offset();
+
+ if (!is_int16(offset)) {
+ LoadIntLiteral(scratch, offset);
+ stwx(src, MemOperand(base, scratch));
+ } else {
+ stw(src, mem);
+ }
+}
+
+void MacroAssembler::LoadHalfWordArith(Register dst, const MemOperand& mem,
+ Register scratch) {
+ int offset = mem.offset();
+
+ if (!is_int16(offset)) {
+ DCHECK(scratch != no_reg);
+ mov(scratch, Operand(offset));
+ lhax(dst, MemOperand(mem.ra(), scratch));
+ } else {
+ lha(dst, mem);
+ }
+}
+
+// Variable length depending on whether offset fits into immediate field
+// MemOperand currently only supports d-form
+void MacroAssembler::LoadHalfWord(Register dst, const MemOperand& mem,
+ Register scratch) {
+ Register base = mem.ra();
+ int offset = mem.offset();
+
+ if (!is_int16(offset)) {
+ DCHECK_NE(scratch, no_reg);
+ LoadIntLiteral(scratch, offset);
+ lhzx(dst, MemOperand(base, scratch));
+ } else {
+ lhz(dst, mem);
+ }
+}
+
+// Variable length depending on whether offset fits into immediate field
+// MemOperand current only supports d-form
+void MacroAssembler::StoreHalfWord(Register src, const MemOperand& mem,
+ Register scratch) {
+ Register base = mem.ra();
+ int offset = mem.offset();
+
+ if (!is_int16(offset)) {
+ LoadIntLiteral(scratch, offset);
+ sthx(src, MemOperand(base, scratch));
+ } else {
+ sth(src, mem);
+ }
+}
+
+// Variable length depending on whether offset fits into immediate field
+// MemOperand currently only supports d-form
+void MacroAssembler::LoadByte(Register dst, const MemOperand& mem,
+ Register scratch) {
+ Register base = mem.ra();
+ int offset = mem.offset();
+
+ if (!is_int16(offset)) {
+ LoadIntLiteral(scratch, offset);
+ lbzx(dst, MemOperand(base, scratch));
+ } else {
+ lbz(dst, mem);
+ }
+}
+
+// Variable length depending on whether offset fits into immediate field
+// MemOperand current only supports d-form
+void MacroAssembler::StoreByte(Register src, const MemOperand& mem,
+ Register scratch) {
+ Register base = mem.ra();
+ int offset = mem.offset();
+
+ if (!is_int16(offset)) {
+ LoadIntLiteral(scratch, offset);
+ stbx(src, MemOperand(base, scratch));
+ } else {
+ stb(src, mem);
+ }
+}
+
+void TurboAssembler::LoadDouble(DoubleRegister dst, const MemOperand& mem,
+ Register scratch) {
+ Register base = mem.ra();
+ int offset = mem.offset();
+
+ if (!is_int16(offset)) {
+ mov(scratch, Operand(offset));
+ lfdx(dst, MemOperand(base, scratch));
+ } else {
+ lfd(dst, mem);
+ }
+}
+
+void TurboAssembler::LoadFloat32(DoubleRegister dst, const MemOperand& mem,
+ Register scratch) {
+ Register base = mem.ra();
+ int offset = mem.offset();
+
+ if (!is_int16(offset)) {
+ mov(scratch, Operand(offset));
+ lfsx(dst, MemOperand(base, scratch));
+ } else {
+ lfs(dst, mem);
+ }
+}
+
+void MacroAssembler::LoadDoubleU(DoubleRegister dst, const MemOperand& mem,
+ Register scratch) {
+ Register base = mem.ra();
+ int offset = mem.offset();
+
+ if (!is_int16(offset)) {
+ mov(scratch, Operand(offset));
+ lfdux(dst, MemOperand(base, scratch));
+ } else {
+ lfdu(dst, mem);
+ }
+}
+
+void TurboAssembler::LoadSingle(DoubleRegister dst, const MemOperand& mem,
+ Register scratch) {
+ Register base = mem.ra();
+ int offset = mem.offset();
+
+ if (!is_int16(offset)) {
+ mov(scratch, Operand(offset));
+ lfsx(dst, MemOperand(base, scratch));
+ } else {
+ lfs(dst, mem);
+ }
+}
+
+void TurboAssembler::LoadSingleU(DoubleRegister dst, const MemOperand& mem,
+ Register scratch) {
+ Register base = mem.ra();
+ int offset = mem.offset();
+
+ if (!is_int16(offset)) {
+ mov(scratch, Operand(offset));
+ lfsux(dst, MemOperand(base, scratch));
+ } else {
+ lfsu(dst, mem);
+ }
+}
+
+void TurboAssembler::LoadSimd128(Simd128Register dst, const MemOperand& mem,
+ Register ScratchReg,
+ Simd128Register ScratchDoubleReg) {
+ // lvx needs the stack to be 16 byte aligned.
+ // We first use lxvd/stxvd to copy the content on an aligned address. lxvd
+ // itself reverses the lanes so it cannot be used as is.
+ lxvd(ScratchDoubleReg, mem);
+ mr(ScratchReg, sp);
+ ClearRightImm(
+ sp, sp,
+ Operand(base::bits::WhichPowerOfTwo(16))); // equivalent to &= -16
+ addi(sp, sp, Operand(-16));
+ stxvd(kScratchDoubleReg, MemOperand(r0, sp));
+ // Load it with correct lane ordering.
+ lvx(dst, MemOperand(r0, sp));
+ mr(sp, ScratchReg);
+}
+
+void TurboAssembler::StoreDouble(DoubleRegister src, const MemOperand& mem,
+ Register scratch) {
+ Register base = mem.ra();
+ int offset = mem.offset();
+
+ if (!is_int16(offset)) {
+ mov(scratch, Operand(offset));
+ stfdx(src, MemOperand(base, scratch));
+ } else {
+ stfd(src, mem);
+ }
+}
+
+void TurboAssembler::StoreDoubleU(DoubleRegister src, const MemOperand& mem,
+ Register scratch) {
+ Register base = mem.ra();
+ int offset = mem.offset();
+
+ if (!is_int16(offset)) {
+ mov(scratch, Operand(offset));
+ stfdux(src, MemOperand(base, scratch));
+ } else {
+ stfdu(src, mem);
+ }
+}
+
+void TurboAssembler::StoreSingle(DoubleRegister src, const MemOperand& mem,
+ Register scratch) {
+ Register base = mem.ra();
+ int offset = mem.offset();
+
+ if (!is_int16(offset)) {
+ mov(scratch, Operand(offset));
+ stfsx(src, MemOperand(base, scratch));
+ } else {
+ stfs(src, mem);
+ }
+}
+
+void TurboAssembler::StoreSingleU(DoubleRegister src, const MemOperand& mem,
+ Register scratch) {
+ Register base = mem.ra();
+ int offset = mem.offset();
+
+ if (!is_int16(offset)) {
+ mov(scratch, Operand(offset));
+ stfsux(src, MemOperand(base, scratch));
+ } else {
+ stfsu(src, mem);
+ }
+}
+
+void TurboAssembler::StoreSimd128(Simd128Register src, const MemOperand& mem,
+ Register ScratchReg,
+ Simd128Register ScratchDoubleReg) {
+ // stvx needs the stack to be 16 byte aligned.
+ // We use lxvd/stxvd to store the content on an aligned address. stxvd
+ // itself reverses the lanes so it cannot be used as is.
+ mr(ScratchReg, sp);
+ ClearRightImm(
+ sp, sp,
+ Operand(base::bits::WhichPowerOfTwo(16))); // equivalent to &= -16
+ addi(sp, sp, Operand(-16));
+ stvx(src, MemOperand(r0, sp));
+ lxvd(ScratchDoubleReg, MemOperand(r0, sp));
+ mr(sp, ScratchReg);
+ stxvd(ScratchDoubleReg, mem);
+}
+
+Register GetRegisterThatIsNotOneOf(Register reg1, Register reg2, Register reg3,
+ Register reg4, Register reg5,
+ Register reg6) {
+ RegList regs = 0;
+ if (reg1.is_valid()) regs |= reg1.bit();
+ if (reg2.is_valid()) regs |= reg2.bit();
+ if (reg3.is_valid()) regs |= reg3.bit();
+ if (reg4.is_valid()) regs |= reg4.bit();
+ if (reg5.is_valid()) regs |= reg5.bit();
+ if (reg6.is_valid()) regs |= reg6.bit();
+
+ const RegisterConfiguration* config = RegisterConfiguration::Default();
+ for (int i = 0; i < config->num_allocatable_general_registers(); ++i) {
+ int code = config->GetAllocatableGeneralCode(i);
+ Register candidate = Register::from_code(code);
+ if (regs & candidate.bit()) continue;
+ return candidate;
+ }
+ UNREACHABLE();
+}
+
+void TurboAssembler::SwapP(Register src, Register dst, Register scratch) {
+ if (src == dst) return;
+ DCHECK(!AreAliased(src, dst, scratch));
+ mr(scratch, src);
+ mr(src, dst);
+ mr(dst, scratch);
+}
+
+void TurboAssembler::SwapP(Register src, MemOperand dst, Register scratch) {
+ if (dst.ra() != r0 && dst.ra().is_valid())
+ DCHECK(!AreAliased(src, dst.ra(), scratch));
+ if (dst.rb() != r0 && dst.rb().is_valid())
+ DCHECK(!AreAliased(src, dst.rb(), scratch));
+ DCHECK(!AreAliased(src, scratch));
+ mr(scratch, src);
+ LoadP(src, dst, r0);
+ StoreP(scratch, dst, r0);
+}
+
+void TurboAssembler::SwapP(MemOperand src, MemOperand dst, Register scratch_0,
+ Register scratch_1) {
+ if (src.ra() != r0 && src.ra().is_valid())
+ DCHECK(!AreAliased(src.ra(), scratch_0, scratch_1));
+ if (src.rb() != r0 && src.rb().is_valid())
+ DCHECK(!AreAliased(src.rb(), scratch_0, scratch_1));
+ if (dst.ra() != r0 && dst.ra().is_valid())
+ DCHECK(!AreAliased(dst.ra(), scratch_0, scratch_1));
+ if (dst.rb() != r0 && dst.rb().is_valid())
+ DCHECK(!AreAliased(dst.rb(), scratch_0, scratch_1));
+ DCHECK(!AreAliased(scratch_0, scratch_1));
+ if (is_int16(src.offset()) || is_int16(dst.offset())) {
+ if (!is_int16(src.offset())) {
+ // swap operand
+ MemOperand temp = src;
+ src = dst;
+ dst = temp;
+ }
+ LoadP(scratch_1, dst, scratch_0);
+ LoadP(scratch_0, src);
+ StoreP(scratch_1, src);
+ StoreP(scratch_0, dst, scratch_1);
+ } else {
+ LoadP(scratch_1, dst, scratch_0);
+ push(scratch_1);
+ LoadP(scratch_0, src, scratch_1);
+ StoreP(scratch_0, dst, scratch_1);
+ pop(scratch_1);
+ StoreP(scratch_1, src, scratch_0);
+ }
+}
+
+void TurboAssembler::SwapFloat32(DoubleRegister src, DoubleRegister dst,
+ DoubleRegister scratch) {
+ if (src == dst) return;
+ DCHECK(!AreAliased(src, dst, scratch));
+ fmr(scratch, src);
+ fmr(src, dst);
+ fmr(dst, scratch);
+}
+
+void TurboAssembler::SwapFloat32(DoubleRegister src, MemOperand dst,
+ DoubleRegister scratch) {
+ DCHECK(!AreAliased(src, scratch));
+ fmr(scratch, src);
+ LoadSingle(src, dst, r0);
+ StoreSingle(scratch, dst, r0);
+}
+
+void TurboAssembler::SwapFloat32(MemOperand src, MemOperand dst,
+ DoubleRegister scratch_0,
+ DoubleRegister scratch_1) {
+ DCHECK(!AreAliased(scratch_0, scratch_1));
+ LoadSingle(scratch_0, src, r0);
+ LoadSingle(scratch_1, dst, r0);
+ StoreSingle(scratch_0, dst, r0);
+ StoreSingle(scratch_1, src, r0);
+}
+
+void TurboAssembler::SwapDouble(DoubleRegister src, DoubleRegister dst,
+ DoubleRegister scratch) {
+ if (src == dst) return;
+ DCHECK(!AreAliased(src, dst, scratch));
+ fmr(scratch, src);
+ fmr(src, dst);
+ fmr(dst, scratch);
+}
+
+void TurboAssembler::SwapDouble(DoubleRegister src, MemOperand dst,
+ DoubleRegister scratch) {
+ DCHECK(!AreAliased(src, scratch));
+ fmr(scratch, src);
+ LoadDouble(src, dst, r0);
+ StoreDouble(scratch, dst, r0);
+}
+
+void TurboAssembler::SwapDouble(MemOperand src, MemOperand dst,
+ DoubleRegister scratch_0,
+ DoubleRegister scratch_1) {
+ DCHECK(!AreAliased(scratch_0, scratch_1));
+ LoadDouble(scratch_0, src, r0);
+ LoadDouble(scratch_1, dst, r0);
+ StoreDouble(scratch_0, dst, r0);
+ StoreDouble(scratch_1, src, r0);
+}
+
+void TurboAssembler::SwapSimd128(Simd128Register src, Simd128Register dst,
+ Simd128Register scratch) {
+ if (src == dst) return;
+ vor(scratch, src, src);
+ vor(src, dst, dst);
+ vor(dst, scratch, scratch);
+}
+
+void TurboAssembler::SwapSimd128(Simd128Register src, MemOperand dst,
+ Simd128Register scratch) {
+ DCHECK(!AreAliased(src, scratch));
+ // push d0, to be used as scratch
+ addi(sp, sp, Operand(-kSimd128Size));
+ StoreSimd128(d0, MemOperand(r0, sp), r0, scratch);
+ mov(ip, Operand(dst.offset()));
+ LoadSimd128(d0, MemOperand(dst.ra(), ip), r0, scratch);
+ StoreSimd128(src, MemOperand(dst.ra(), ip), r0, scratch);
+ vor(src, d0, d0);
+ // restore d0
+ LoadSimd128(d0, MemOperand(r0, sp), ip, scratch);
+ addi(sp, sp, Operand(kSimd128Size));
+}
+
+void TurboAssembler::SwapSimd128(MemOperand src, MemOperand dst,
+ Simd128Register scratch) {
+ // push d0 and d1, to be used as scratch
+ addi(sp, sp, Operand(2 * -kSimd128Size));
+ StoreSimd128(d0, MemOperand(r0, sp), ip, scratch);
+ li(ip, Operand(kSimd128Size));
+ StoreSimd128(d1, MemOperand(ip, sp), r0, scratch);
+
+ mov(ip, Operand(src.offset()));
+ LoadSimd128(d0, MemOperand(src.ra(), ip), r0, scratch);
+ mov(ip, Operand(dst.offset()));
+ LoadSimd128(d1, MemOperand(dst.ra(), ip), r0, scratch);
+
+ StoreSimd128(d0, MemOperand(dst.ra(), ip), r0, scratch);
+ mov(ip, Operand(src.offset()));
+ StoreSimd128(d1, MemOperand(src.ra(), ip), r0, scratch);
+
+ // restore d0 and d1
+ LoadSimd128(d0, MemOperand(r0, sp), ip, scratch);
+ li(ip, Operand(kSimd128Size));
+ LoadSimd128(d1, MemOperand(ip, sp), r0, scratch);
+ addi(sp, sp, Operand(2 * kSimd128Size));
+}
+
+void TurboAssembler::ResetSpeculationPoisonRegister() {
+ mov(kSpeculationPoisonRegister, Operand(-1));
+}
+
+void TurboAssembler::JumpIfEqual(Register x, int32_t y, Label* dest) {
+ Cmpi(x, Operand(y), r0);
+ beq(dest);
+}
+
+void TurboAssembler::JumpIfLessThan(Register x, int32_t y, Label* dest) {
+ Cmpi(x, Operand(y), r0);
+ blt(dest);
+}
+
+void TurboAssembler::LoadEntryFromBuiltinIndex(Register builtin_index) {
+ STATIC_ASSERT(kSystemPointerSize == 8);
+ STATIC_ASSERT(kSmiTagSize == 1);
+ STATIC_ASSERT(kSmiTag == 0);
+
+ // The builtin_index register contains the builtin index as a Smi.
+ if (SmiValuesAre32Bits()) {
+ ShiftRightArithImm(builtin_index, builtin_index,
+ kSmiShift - kSystemPointerSizeLog2);
+ } else {
+ DCHECK(SmiValuesAre31Bits());
+ ShiftLeftImm(builtin_index, builtin_index,
+ Operand(kSystemPointerSizeLog2 - kSmiShift));
+ }
+ addi(builtin_index, builtin_index,
+ Operand(IsolateData::builtin_entry_table_offset()));
+ LoadPX(builtin_index, MemOperand(kRootRegister, builtin_index));
+}
+
+void TurboAssembler::CallBuiltinByIndex(Register builtin_index) {
+ LoadEntryFromBuiltinIndex(builtin_index);
+ Call(builtin_index);
+}
+
+void TurboAssembler::LoadCodeObjectEntry(Register destination,
+ Register code_object) {
+ // Code objects are called differently depending on whether we are generating
+ // builtin code (which will later be embedded into the binary) or compiling
+ // user JS code at runtime.
+ // * Builtin code runs in --jitless mode and thus must not call into on-heap
+ // Code targets. Instead, we dispatch through the builtins entry table.
+ // * Codegen at runtime does not have this restriction and we can use the
+ // shorter, branchless instruction sequence. The assumption here is that
+ // targets are usually generated code and not builtin Code objects.
+
+ if (options().isolate_independent_code) {
+ DCHECK(root_array_available());
+ Label if_code_is_off_heap, out;
+
+ Register scratch = r11;
+
+ DCHECK(!AreAliased(destination, scratch));
+ DCHECK(!AreAliased(code_object, scratch));
+
+ // Check whether the Code object is an off-heap trampoline. If so, call its
+ // (off-heap) entry point directly without going through the (on-heap)
+ // trampoline. Otherwise, just call the Code object as always.
+ LoadWordArith(scratch, FieldMemOperand(code_object, Code::kFlagsOffset));
+ mov(r0, Operand(Code::IsOffHeapTrampoline::kMask));
+ and_(r0, scratch, r0, SetRC);
+ bne(&if_code_is_off_heap, cr0);
+
+ // Not an off-heap trampoline, the entry point is at
+ // Code::raw_instruction_start().
+ addi(destination, code_object, Operand(Code::kHeaderSize - kHeapObjectTag));
+ b(&out);
+
+ // An off-heap trampoline, the entry point is loaded from the builtin entry
+ // table.
+ bind(&if_code_is_off_heap);
+ LoadWordArith(scratch,
+ FieldMemOperand(code_object, Code::kBuiltinIndexOffset));
+ ShiftLeftImm(destination, scratch, Operand(kSystemPointerSizeLog2));
+ add(destination, destination, kRootRegister);
+ LoadP(destination,
+ MemOperand(destination, IsolateData::builtin_entry_table_offset()),
+ r0);
+
+ bind(&out);
+ } else {
+ addi(destination, code_object, Operand(Code::kHeaderSize - kHeapObjectTag));
+ }
+}
+
+void TurboAssembler::CallCodeObject(Register code_object) {
+ LoadCodeObjectEntry(code_object, code_object);
+ Call(code_object);
+}
+
+void TurboAssembler::JumpCodeObject(Register code_object) {
+ LoadCodeObjectEntry(code_object, code_object);
+ Jump(code_object);
+}
+
+void TurboAssembler::StoreReturnAddressAndCall(Register target) {
+ // This generates the final instruction sequence for calls to C functions
+ // once an exit frame has been constructed.
+ //
+ // Note that this assumes the caller code (i.e. the Code object currently
+ // being generated) is immovable or that the callee function cannot trigger
+ // GC, since the callee function will return to it.
+
+ static constexpr int after_call_offset = 5 * kInstrSize;
+ Label start_call;
+ Register dest = target;
+
+ if (ABI_USES_FUNCTION_DESCRIPTORS) {
+ // AIX/PPC64BE Linux uses a function descriptor. When calling C code be
+ // aware of this descriptor and pick up values from it
+ LoadP(ToRegister(ABI_TOC_REGISTER), MemOperand(target, kSystemPointerSize));
+ LoadP(ip, MemOperand(target, 0));
+ dest = ip;
+ } else if (ABI_CALL_VIA_IP && dest != ip) {
+ Move(ip, target);
+ dest = ip;
+ }
+
+ LoadPC(r7);
+ bind(&start_call);
+ addi(r7, r7, Operand(after_call_offset));
+ StoreP(r7, MemOperand(sp, kStackFrameExtraParamSlot * kSystemPointerSize));
+ Call(dest);
+
+ DCHECK_EQ(after_call_offset - kInstrSize,
+ SizeOfCodeGeneratedSince(&start_call));
+}
+
+void TurboAssembler::CallForDeoptimization(Builtins::Name target, int,
+ Label* exit, DeoptimizeKind kind,
+ Label*) {
+ LoadP(ip, MemOperand(kRootRegister,
+ IsolateData::builtin_entry_slot_offset(target)));
+ Call(ip);
+ DCHECK_EQ(SizeOfCodeGeneratedSince(exit),
+ (kind == DeoptimizeKind::kLazy)
+ ? Deoptimizer::kLazyDeoptExitSize
+ : Deoptimizer::kNonLazyDeoptExitSize);
+ USE(exit, kind);
+}
+
+void TurboAssembler::ZeroExtByte(Register dst, Register src) {
+ clrldi(dst, src, Operand(56));
+}
+
+void TurboAssembler::ZeroExtHalfWord(Register dst, Register src) {
+ clrldi(dst, src, Operand(48));
+}
+
+void TurboAssembler::ZeroExtWord32(Register dst, Register src) {
+ clrldi(dst, src, Operand(32));
+}
+
+void TurboAssembler::Trap() { stop(); }
+void TurboAssembler::DebugBreak() { stop(); }
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_PPC || V8_TARGET_ARCH_PPC64
diff --git a/src/codegen/ppc/macro-assembler-ppc.h b/src/codegen/ppc/macro-assembler-ppc.h
new file mode 100644
index 0000000..db0d685
--- /dev/null
+++ b/src/codegen/ppc/macro-assembler-ppc.h
@@ -0,0 +1,1039 @@
+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef INCLUDED_FROM_MACRO_ASSEMBLER_H
+#error This header must be included via macro-assembler.h
+#endif
+
+#ifndef V8_CODEGEN_PPC_MACRO_ASSEMBLER_PPC_H_
+#define V8_CODEGEN_PPC_MACRO_ASSEMBLER_PPC_H_
+
+#include "src/codegen/bailout-reason.h"
+#include "src/codegen/ppc/assembler-ppc.h"
+#include "src/common/globals.h"
+#include "src/numbers/double.h"
+#include "src/objects/contexts.h"
+
+namespace v8 {
+namespace internal {
+
+// ----------------------------------------------------------------------------
+// Static helper functions
+
+// Generate a MemOperand for loading a field from an object.
+inline MemOperand FieldMemOperand(Register object, int offset) {
+ return MemOperand(object, offset - kHeapObjectTag);
+}
+
+enum RememberedSetAction { EMIT_REMEMBERED_SET, OMIT_REMEMBERED_SET };
+enum SmiCheck { INLINE_SMI_CHECK, OMIT_SMI_CHECK };
+enum LinkRegisterStatus { kLRHasNotBeenSaved, kLRHasBeenSaved };
+
+Register GetRegisterThatIsNotOneOf(Register reg1, Register reg2 = no_reg,
+ Register reg3 = no_reg,
+ Register reg4 = no_reg,
+ Register reg5 = no_reg,
+ Register reg6 = no_reg);
+
+// These exist to provide portability between 32 and 64bit
+#if V8_TARGET_ARCH_PPC64
+#define LoadPX ldx
+#define LoadPUX ldux
+#define StorePX stdx
+#define StorePUX stdux
+#define ShiftLeftImm sldi
+#define ShiftRightImm srdi
+#define ClearLeftImm clrldi
+#define ClearRightImm clrrdi
+#define ShiftRightArithImm sradi
+#define ShiftLeft_ sld
+#define ShiftRight_ srd
+#define ShiftRightArith srad
+#else
+#define LoadPX lwzx
+#define LoadPUX lwzux
+#define StorePX stwx
+#define StorePUX stwux
+#define ShiftLeftImm slwi
+#define ShiftRightImm srwi
+#define ClearLeftImm clrlwi
+#define ClearRightImm clrrwi
+#define ShiftRightArithImm srawi
+#define ShiftLeft_ slw
+#define ShiftRight_ srw
+#define ShiftRightArith sraw
+#endif
+
+class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase {
+ public:
+ using TurboAssemblerBase::TurboAssemblerBase;
+
+ // Converts the integer (untagged smi) in |src| to a double, storing
+ // the result to |dst|
+ void ConvertIntToDouble(Register src, DoubleRegister dst);
+
+ // Converts the unsigned integer (untagged smi) in |src| to
+ // a double, storing the result to |dst|
+ void ConvertUnsignedIntToDouble(Register src, DoubleRegister dst);
+
+ // Converts the integer (untagged smi) in |src| to
+ // a float, storing the result in |dst|
+ void ConvertIntToFloat(Register src, DoubleRegister dst);
+
+ // Converts the unsigned integer (untagged smi) in |src| to
+ // a float, storing the result in |dst|
+ void ConvertUnsignedIntToFloat(Register src, DoubleRegister dst);
+
+#if V8_TARGET_ARCH_PPC64
+ void ConvertInt64ToFloat(Register src, DoubleRegister double_dst);
+ void ConvertInt64ToDouble(Register src, DoubleRegister double_dst);
+ void ConvertUnsignedInt64ToFloat(Register src, DoubleRegister double_dst);
+ void ConvertUnsignedInt64ToDouble(Register src, DoubleRegister double_dst);
+#endif
+
+ // Converts the double_input to an integer. Note that, upon return,
+ // the contents of double_dst will also hold the fixed point representation.
+ void ConvertDoubleToInt64(const DoubleRegister double_input,
+#if !V8_TARGET_ARCH_PPC64
+ const Register dst_hi,
+#endif
+ const Register dst, const DoubleRegister double_dst,
+ FPRoundingMode rounding_mode = kRoundToZero);
+
+#if V8_TARGET_ARCH_PPC64
+ // Converts the double_input to an unsigned integer. Note that, upon return,
+ // the contents of double_dst will also hold the fixed point representation.
+ void ConvertDoubleToUnsignedInt64(
+ const DoubleRegister double_input, const Register dst,
+ const DoubleRegister double_dst,
+ FPRoundingMode rounding_mode = kRoundToZero);
+#endif
+
+ // Activation support.
+ void EnterFrame(StackFrame::Type type,
+ bool load_constant_pool_pointer_reg = false);
+
+ // Returns the pc offset at which the frame ends.
+ int LeaveFrame(StackFrame::Type type, int stack_adjustment = 0);
+
+ // Push a fixed frame, consisting of lr, fp, constant pool.
+ void PushCommonFrame(Register marker_reg = no_reg);
+
+ // Generates function and stub prologue code.
+ void StubPrologue(StackFrame::Type type);
+ void Prologue();
+
+ // Push a standard frame, consisting of lr, fp, constant pool,
+ // context and JS function
+ void PushStandardFrame(Register function_reg);
+
+ // Restore caller's frame pointer and return address prior to being
+ // overwritten by tail call stack preparation.
+ void RestoreFrameStateForTailCall();
+
+ // Get the actual activation frame alignment for target environment.
+ static int ActivationFrameAlignment();
+
+ void InitializeRootRegister() {
+ ExternalReference isolate_root = ExternalReference::isolate_root(isolate());
+ mov(kRootRegister, Operand(isolate_root));
+ }
+
+ // These exist to provide portability between 32 and 64bit
+ void LoadP(Register dst, const MemOperand& mem, Register scratch = no_reg);
+ void LoadPU(Register dst, const MemOperand& mem, Register scratch = no_reg);
+ void LoadWordArith(Register dst, const MemOperand& mem,
+ Register scratch = no_reg);
+ void StoreP(Register src, const MemOperand& mem, Register scratch = no_reg);
+ void StorePU(Register src, const MemOperand& mem, Register scratch = no_reg);
+
+ void LoadDouble(DoubleRegister dst, const MemOperand& mem,
+ Register scratch = no_reg);
+ void LoadFloat32(DoubleRegister dst, const MemOperand& mem,
+ Register scratch = no_reg);
+ void LoadDoubleLiteral(DoubleRegister result, Double value, Register scratch);
+ void LoadSimd128(Simd128Register dst, const MemOperand& mem,
+ Register ScratchReg, Simd128Register ScratchDoubleReg);
+
+ // load a literal signed int value <value> to GPR <dst>
+ void LoadIntLiteral(Register dst, int value);
+ // load an SMI value <value> to GPR <dst>
+ void LoadSmiLiteral(Register dst, Smi smi);
+
+ void LoadSingle(DoubleRegister dst, const MemOperand& mem,
+ Register scratch = no_reg);
+ void LoadSingleU(DoubleRegister dst, const MemOperand& mem,
+ Register scratch = no_reg);
+ void LoadPC(Register dst);
+ void ComputeCodeStartAddress(Register dst);
+
+ void StoreDouble(DoubleRegister src, const MemOperand& mem,
+ Register scratch = no_reg);
+ void StoreDoubleU(DoubleRegister src, const MemOperand& mem,
+ Register scratch = no_reg);
+
+ void StoreSingle(DoubleRegister src, const MemOperand& mem,
+ Register scratch = no_reg);
+ void StoreSingleU(DoubleRegister src, const MemOperand& mem,
+ Register scratch = no_reg);
+ void StoreSimd128(Simd128Register src, const MemOperand& mem,
+ Register ScratchReg, Simd128Register ScratchDoubleReg);
+
+ void Cmpi(Register src1, const Operand& src2, Register scratch,
+ CRegister cr = cr7);
+ void Cmpli(Register src1, const Operand& src2, Register scratch,
+ CRegister cr = cr7);
+ void Cmpwi(Register src1, const Operand& src2, Register scratch,
+ CRegister cr = cr7);
+ void CompareTagged(Register src1, Register src2, CRegister cr = cr7) {
+ if (COMPRESS_POINTERS_BOOL) {
+ cmpw(src1, src2, cr);
+ } else {
+ cmp(src1, src2, cr);
+ }
+ }
+
+ // Set new rounding mode RN to FPSCR
+ void SetRoundingMode(FPRoundingMode RN);
+
+ // reset rounding mode to default (kRoundToNearest)
+ void ResetRoundingMode();
+ void Add(Register dst, Register src, intptr_t value, Register scratch);
+
+ void Push(Register src) { push(src); }
+ // Push a handle.
+ void Push(Handle<HeapObject> handle);
+ void Push(Smi smi);
+
+ // Push two registers. Pushes leftmost register first (to highest address).
+ void Push(Register src1, Register src2) {
+ StorePU(src2, MemOperand(sp, -2 * kSystemPointerSize));
+ StoreP(src1, MemOperand(sp, kSystemPointerSize));
+ }
+
+ // Push three registers. Pushes leftmost register first (to highest address).
+ void Push(Register src1, Register src2, Register src3) {
+ StorePU(src3, MemOperand(sp, -3 * kSystemPointerSize));
+ StoreP(src2, MemOperand(sp, kSystemPointerSize));
+ StoreP(src1, MemOperand(sp, 2 * kSystemPointerSize));
+ }
+
+ // Push four registers. Pushes leftmost register first (to highest address).
+ void Push(Register src1, Register src2, Register src3, Register src4) {
+ StorePU(src4, MemOperand(sp, -4 * kSystemPointerSize));
+ StoreP(src3, MemOperand(sp, kSystemPointerSize));
+ StoreP(src2, MemOperand(sp, 2 * kSystemPointerSize));
+ StoreP(src1, MemOperand(sp, 3 * kSystemPointerSize));
+ }
+
+ // Push five registers. Pushes leftmost register first (to highest address).
+ void Push(Register src1, Register src2, Register src3, Register src4,
+ Register src5) {
+ StorePU(src5, MemOperand(sp, -5 * kSystemPointerSize));
+ StoreP(src4, MemOperand(sp, kSystemPointerSize));
+ StoreP(src3, MemOperand(sp, 2 * kSystemPointerSize));
+ StoreP(src2, MemOperand(sp, 3 * kSystemPointerSize));
+ StoreP(src1, MemOperand(sp, 4 * kSystemPointerSize));
+ }
+
+ enum PushArrayOrder { kNormal, kReverse };
+ void PushArray(Register array, Register size, Register scratch,
+ Register scratch2, PushArrayOrder order = kNormal);
+
+ void Pop(Register dst) { pop(dst); }
+
+ // Pop two registers. Pops rightmost register first (from lower address).
+ void Pop(Register src1, Register src2) {
+ LoadP(src2, MemOperand(sp, 0));
+ LoadP(src1, MemOperand(sp, kSystemPointerSize));
+ addi(sp, sp, Operand(2 * kSystemPointerSize));
+ }
+
+ // Pop three registers. Pops rightmost register first (from lower address).
+ void Pop(Register src1, Register src2, Register src3) {
+ LoadP(src3, MemOperand(sp, 0));
+ LoadP(src2, MemOperand(sp, kSystemPointerSize));
+ LoadP(src1, MemOperand(sp, 2 * kSystemPointerSize));
+ addi(sp, sp, Operand(3 * kSystemPointerSize));
+ }
+
+ // Pop four registers. Pops rightmost register first (from lower address).
+ void Pop(Register src1, Register src2, Register src3, Register src4) {
+ LoadP(src4, MemOperand(sp, 0));
+ LoadP(src3, MemOperand(sp, kSystemPointerSize));
+ LoadP(src2, MemOperand(sp, 2 * kSystemPointerSize));
+ LoadP(src1, MemOperand(sp, 3 * kSystemPointerSize));
+ addi(sp, sp, Operand(4 * kSystemPointerSize));
+ }
+
+ // Pop five registers. Pops rightmost register first (from lower address).
+ void Pop(Register src1, Register src2, Register src3, Register src4,
+ Register src5) {
+ LoadP(src5, MemOperand(sp, 0));
+ LoadP(src4, MemOperand(sp, kSystemPointerSize));
+ LoadP(src3, MemOperand(sp, 2 * kSystemPointerSize));
+ LoadP(src2, MemOperand(sp, 3 * kSystemPointerSize));
+ LoadP(src1, MemOperand(sp, 4 * kSystemPointerSize));
+ addi(sp, sp, Operand(5 * kSystemPointerSize));
+ }
+
+ void SaveRegisters(RegList registers);
+ void RestoreRegisters(RegList registers);
+
+ void CallRecordWriteStub(Register object, Register address,
+ RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode);
+ void CallRecordWriteStub(Register object, Register address,
+ RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode, Address wasm_target);
+ void CallEphemeronKeyBarrier(Register object, Register address,
+ SaveFPRegsMode fp_mode);
+
+ void MultiPush(RegList regs, Register location = sp);
+ void MultiPop(RegList regs, Register location = sp);
+
+ void MultiPushDoubles(RegList dregs, Register location = sp);
+ void MultiPopDoubles(RegList dregs, Register location = sp);
+
+ // Calculate how much stack space (in bytes) are required to store caller
+ // registers excluding those specified in the arguments.
+ int RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
+ Register exclusion1 = no_reg,
+ Register exclusion2 = no_reg,
+ Register exclusion3 = no_reg) const;
+
+ // Push caller saved registers on the stack, and return the number of bytes
+ // stack pointer is adjusted.
+ int PushCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1 = no_reg,
+ Register exclusion2 = no_reg,
+ Register exclusion3 = no_reg);
+ // Restore caller saved registers from the stack, and return the number of
+ // bytes stack pointer is adjusted.
+ int PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1 = no_reg,
+ Register exclusion2 = no_reg,
+ Register exclusion3 = no_reg);
+
+ // Load an object from the root table.
+ void LoadRoot(Register destination, RootIndex index) override {
+ LoadRoot(destination, index, al);
+ }
+ void LoadRoot(Register destination, RootIndex index, Condition cond);
+
+ void SwapP(Register src, Register dst, Register scratch);
+ void SwapP(Register src, MemOperand dst, Register scratch);
+ void SwapP(MemOperand src, MemOperand dst, Register scratch_0,
+ Register scratch_1);
+ void SwapFloat32(DoubleRegister src, DoubleRegister dst,
+ DoubleRegister scratch);
+ void SwapFloat32(DoubleRegister src, MemOperand dst, DoubleRegister scratch);
+ void SwapFloat32(MemOperand src, MemOperand dst, DoubleRegister scratch_0,
+ DoubleRegister scratch_1);
+ void SwapDouble(DoubleRegister src, DoubleRegister dst,
+ DoubleRegister scratch);
+ void SwapDouble(DoubleRegister src, MemOperand dst, DoubleRegister scratch);
+ void SwapDouble(MemOperand src, MemOperand dst, DoubleRegister scratch_0,
+ DoubleRegister scratch_1);
+ void SwapSimd128(Simd128Register src, Simd128Register dst,
+ Simd128Register scratch);
+ void SwapSimd128(Simd128Register src, MemOperand dst,
+ Simd128Register scratch);
+ void SwapSimd128(MemOperand src, MemOperand dst, Simd128Register scratch);
+
+ // Before calling a C-function from generated code, align arguments on stack.
+ // After aligning the frame, non-register arguments must be stored in
+ // sp[0], sp[4], etc., not pushed. The argument count assumes all arguments
+ // are word sized. If double arguments are used, this function assumes that
+ // all double arguments are stored before core registers; otherwise the
+ // correct alignment of the double values is not guaranteed.
+ // Some compilers/platforms require the stack to be aligned when calling
+ // C++ code.
+ // Needs a scratch register to do some arithmetic. This register will be
+ // trashed.
+ void PrepareCallCFunction(int num_reg_arguments, int num_double_registers,
+ Register scratch);
+ void PrepareCallCFunction(int num_reg_arguments, Register scratch);
+
+ void PrepareForTailCall(Register callee_args_count,
+ Register caller_args_count, Register scratch0,
+ Register scratch1);
+
+ // There are two ways of passing double arguments on ARM, depending on
+ // whether soft or hard floating point ABI is used. These functions
+ // abstract parameter passing for the three different ways we call
+ // C functions from generated code.
+ void MovToFloatParameter(DoubleRegister src);
+ void MovToFloatParameters(DoubleRegister src1, DoubleRegister src2);
+ void MovToFloatResult(DoubleRegister src);
+
+ // Calls a C function and cleans up the space for arguments allocated
+ // by PrepareCallCFunction. The called function is not allowed to trigger a
+ // garbage collection, since that might move the code and invalidate the
+ // return address (unless this is somehow accounted for by the called
+ // function).
+ void CallCFunction(ExternalReference function, int num_arguments,
+ bool has_function_descriptor = true);
+ void CallCFunction(Register function, int num_arguments,
+ bool has_function_descriptor = true);
+ void CallCFunction(ExternalReference function, int num_reg_arguments,
+ int num_double_arguments,
+ bool has_function_descriptor = true);
+ void CallCFunction(Register function, int num_reg_arguments,
+ int num_double_arguments,
+ bool has_function_descriptor = true);
+
+ void MovFromFloatParameter(DoubleRegister dst);
+ void MovFromFloatResult(DoubleRegister dst);
+
+ void Trap() override;
+ void DebugBreak() override;
+
+ // Calls Abort(msg) if the condition cond is not satisfied.
+ // Use --debug_code to enable.
+ void Assert(Condition cond, AbortReason reason, CRegister cr = cr7);
+
+ // Like Assert(), but always enabled.
+ void Check(Condition cond, AbortReason reason, CRegister cr = cr7);
+
+ // Print a message to stdout and abort execution.
+ void Abort(AbortReason reason);
+
+#if !V8_TARGET_ARCH_PPC64
+ void ShiftLeftPair(Register dst_low, Register dst_high, Register src_low,
+ Register src_high, Register scratch, Register shift);
+ void ShiftLeftPair(Register dst_low, Register dst_high, Register src_low,
+ Register src_high, uint32_t shift);
+ void ShiftRightPair(Register dst_low, Register dst_high, Register src_low,
+ Register src_high, Register scratch, Register shift);
+ void ShiftRightPair(Register dst_low, Register dst_high, Register src_low,
+ Register src_high, uint32_t shift);
+ void ShiftRightAlgPair(Register dst_low, Register dst_high, Register src_low,
+ Register src_high, Register scratch, Register shift);
+ void ShiftRightAlgPair(Register dst_low, Register dst_high, Register src_low,
+ Register src_high, uint32_t shift);
+#endif
+
+ void LoadFromConstantsTable(Register destination,
+ int constant_index) override;
+ void LoadRootRegisterOffset(Register destination, intptr_t offset) override;
+ void LoadRootRelative(Register destination, int32_t offset) override;
+
+ // Jump, Call, and Ret pseudo instructions implementing inter-working.
+ void Jump(Register target);
+ void Jump(Address target, RelocInfo::Mode rmode, Condition cond = al,
+ CRegister cr = cr7);
+ void Jump(Handle<Code> code, RelocInfo::Mode rmode, Condition cond = al,
+ CRegister cr = cr7);
+ void Jump(const ExternalReference& reference) override;
+ void Jump(intptr_t target, RelocInfo::Mode rmode, Condition cond = al,
+ CRegister cr = cr7);
+ void Call(Register target);
+ void Call(Address target, RelocInfo::Mode rmode, Condition cond = al);
+ void Call(Handle<Code> code, RelocInfo::Mode rmode = RelocInfo::CODE_TARGET,
+ Condition cond = al);
+ void Call(Label* target);
+
+ // Load the builtin given by the Smi in |builtin_index| into the same
+ // register.
+ void LoadEntryFromBuiltinIndex(Register builtin_index);
+ void LoadCodeObjectEntry(Register destination, Register code_object) override;
+ void CallCodeObject(Register code_object) override;
+ void JumpCodeObject(Register code_object) override;
+
+ void CallBuiltinByIndex(Register builtin_index) override;
+ void CallForDeoptimization(Builtins::Name target, int deopt_id, Label* exit,
+ DeoptimizeKind kind,
+ Label* jump_deoptimization_entry_label);
+
+ // Emit code to discard a non-negative number of pointer-sized elements
+ // from the stack, clobbering only the sp register.
+ void Drop(int count);
+ void Drop(Register count, Register scratch = r0);
+
+ void Ret() { blr(); }
+ void Ret(Condition cond, CRegister cr = cr7) { bclr(cond, cr); }
+ void Ret(int drop) {
+ Drop(drop);
+ blr();
+ }
+
+ // If the value is a NaN, canonicalize the value else, do nothing.
+ void CanonicalizeNaN(const DoubleRegister dst, const DoubleRegister src);
+ void CanonicalizeNaN(const DoubleRegister value) {
+ CanonicalizeNaN(value, value);
+ }
+ void CheckPageFlag(Register object, Register scratch, int mask, Condition cc,
+ Label* condition_met);
+
+ // Move values between integer and floating point registers.
+ void MovIntToDouble(DoubleRegister dst, Register src, Register scratch);
+ void MovUnsignedIntToDouble(DoubleRegister dst, Register src,
+ Register scratch);
+ void MovInt64ToDouble(DoubleRegister dst,
+#if !V8_TARGET_ARCH_PPC64
+ Register src_hi,
+#endif
+ Register src);
+#if V8_TARGET_ARCH_PPC64
+ void MovInt64ComponentsToDouble(DoubleRegister dst, Register src_hi,
+ Register src_lo, Register scratch);
+#endif
+ void InsertDoubleLow(DoubleRegister dst, Register src, Register scratch);
+ void InsertDoubleHigh(DoubleRegister dst, Register src, Register scratch);
+ void MovDoubleLowToInt(Register dst, DoubleRegister src);
+ void MovDoubleHighToInt(Register dst, DoubleRegister src);
+ void MovDoubleToInt64(
+#if !V8_TARGET_ARCH_PPC64
+ Register dst_hi,
+#endif
+ Register dst, DoubleRegister src);
+ void MovIntToFloat(DoubleRegister dst, Register src);
+ void MovFloatToInt(Register dst, DoubleRegister src);
+ // Register move. May do nothing if the registers are identical.
+ void Move(Register dst, Smi smi) { LoadSmiLiteral(dst, smi); }
+ void Move(Register dst, Handle<HeapObject> value,
+ RelocInfo::Mode rmode = RelocInfo::FULL_EMBEDDED_OBJECT);
+ void Move(Register dst, ExternalReference reference);
+ void Move(Register dst, Register src, Condition cond = al);
+ void Move(DoubleRegister dst, DoubleRegister src);
+
+ void SmiUntag(Register dst, const MemOperand& src, RCBit rc);
+ void SmiUntag(Register reg, RCBit rc = LeaveRC) { SmiUntag(reg, reg, rc); }
+
+ void SmiUntag(Register dst, Register src, RCBit rc = LeaveRC) {
+ if (COMPRESS_POINTERS_BOOL) {
+ srawi(dst, src, kSmiShift, rc);
+ } else {
+ ShiftRightArithImm(dst, src, kSmiShift, rc);
+ }
+ }
+
+ void ZeroExtByte(Register dst, Register src);
+ void ZeroExtHalfWord(Register dst, Register src);
+ void ZeroExtWord32(Register dst, Register src);
+
+ // ---------------------------------------------------------------------------
+ // Bit testing/extraction
+ //
+ // Bit numbering is such that the least significant bit is bit 0
+ // (for consistency between 32/64-bit).
+
+ // Extract consecutive bits (defined by rangeStart - rangeEnd) from src
+ // and, if !test, shift them into the least significant bits of dst.
+ inline void ExtractBitRange(Register dst, Register src, int rangeStart,
+ int rangeEnd, RCBit rc = LeaveRC,
+ bool test = false) {
+ DCHECK(rangeStart >= rangeEnd && rangeStart < kBitsPerSystemPointer);
+ int rotate = (rangeEnd == 0) ? 0 : kBitsPerSystemPointer - rangeEnd;
+ int width = rangeStart - rangeEnd + 1;
+ if (rc == SetRC && rangeStart < 16 && (rangeEnd == 0 || test)) {
+ // Prefer faster andi when applicable.
+ andi(dst, src, Operand(((1 << width) - 1) << rangeEnd));
+ } else {
+#if V8_TARGET_ARCH_PPC64
+ rldicl(dst, src, rotate, kBitsPerSystemPointer - width, rc);
+#else
+ rlwinm(dst, src, rotate, kBitsPerSystemPointer - width,
+ kBitsPerSystemPointer - 1, rc);
+#endif
+ }
+ }
+
+ inline void ExtractBit(Register dst, Register src, uint32_t bitNumber,
+ RCBit rc = LeaveRC, bool test = false) {
+ ExtractBitRange(dst, src, bitNumber, bitNumber, rc, test);
+ }
+
+ // Extract consecutive bits (defined by mask) from src and place them
+ // into the least significant bits of dst.
+ inline void ExtractBitMask(Register dst, Register src, uintptr_t mask,
+ RCBit rc = LeaveRC, bool test = false) {
+ int start = kBitsPerSystemPointer - 1;
+ int end;
+ uintptr_t bit = (1L << start);
+
+ while (bit && (mask & bit) == 0) {
+ start--;
+ bit >>= 1;
+ }
+ end = start;
+ bit >>= 1;
+
+ while (bit && (mask & bit)) {
+ end--;
+ bit >>= 1;
+ }
+
+ // 1-bits in mask must be contiguous
+ DCHECK(bit == 0 || (mask & ((bit << 1) - 1)) == 0);
+
+ ExtractBitRange(dst, src, start, end, rc, test);
+ }
+
+ // Test single bit in value.
+ inline void TestBit(Register value, int bitNumber, Register scratch = r0) {
+ ExtractBitRange(scratch, value, bitNumber, bitNumber, SetRC, true);
+ }
+
+ // Test consecutive bit range in value. Range is defined by mask.
+ inline void TestBitMask(Register value, uintptr_t mask,
+ Register scratch = r0) {
+ ExtractBitMask(scratch, value, mask, SetRC, true);
+ }
+ // Test consecutive bit range in value. Range is defined by
+ // rangeStart - rangeEnd.
+ inline void TestBitRange(Register value, int rangeStart, int rangeEnd,
+ Register scratch = r0) {
+ ExtractBitRange(scratch, value, rangeStart, rangeEnd, SetRC, true);
+ }
+
+ inline void TestIfSmi(Register value, Register scratch) {
+ TestBitRange(value, kSmiTagSize - 1, 0, scratch);
+ }
+ // Jump the register contains a smi.
+ inline void JumpIfSmi(Register value, Label* smi_label) {
+ TestIfSmi(value, r0);
+ beq(smi_label, cr0); // branch if SMI
+ }
+ void JumpIfEqual(Register x, int32_t y, Label* dest);
+ void JumpIfLessThan(Register x, int32_t y, Label* dest);
+
+#if V8_TARGET_ARCH_PPC64
+ inline void TestIfInt32(Register value, Register scratch,
+ CRegister cr = cr7) {
+ // High bits must be identical to fit into an 32-bit integer
+ extsw(scratch, value);
+ cmp(scratch, value, cr);
+ }
+#else
+ inline void TestIfInt32(Register hi_word, Register lo_word, Register scratch,
+ CRegister cr = cr7) {
+ // High bits must be identical to fit into an 32-bit integer
+ srawi(scratch, lo_word, 31);
+ cmp(scratch, hi_word, cr);
+ }
+#endif
+
+ // Overflow handling functions.
+ // Usage: call the appropriate arithmetic function and then call one of the
+ // flow control functions with the corresponding label.
+
+ // Compute dst = left + right, setting condition codes. dst may be same as
+ // either left or right (or a unique register). left and right must not be
+ // the same register.
+ void AddAndCheckForOverflow(Register dst, Register left, Register right,
+ Register overflow_dst, Register scratch = r0);
+ void AddAndCheckForOverflow(Register dst, Register left, intptr_t right,
+ Register overflow_dst, Register scratch = r0);
+
+ // Compute dst = left - right, setting condition codes. dst may be same as
+ // either left or right (or a unique register). left and right must not be
+ // the same register.
+ void SubAndCheckForOverflow(Register dst, Register left, Register right,
+ Register overflow_dst, Register scratch = r0);
+
+ // Performs a truncating conversion of a floating point number as used by
+ // the JS bitwise operations. See ECMA-262 9.5: ToInt32. Goes to 'done' if it
+ // succeeds, otherwise falls through if result is saturated. On return
+ // 'result' either holds answer, or is clobbered on fall through.
+ void TryInlineTruncateDoubleToI(Register result, DoubleRegister input,
+ Label* done);
+ void TruncateDoubleToI(Isolate* isolate, Zone* zone, Register result,
+ DoubleRegister double_input, StubCallMode stub_mode);
+
+ void LoadConstantPoolPointerRegister();
+
+ // Loads the constant pool pointer (kConstantPoolRegister).
+ void LoadConstantPoolPointerRegisterFromCodeTargetAddress(
+ Register code_target_address);
+ void AbortConstantPoolBuilding() {
+#ifdef DEBUG
+ // Avoid DCHECK(!is_linked()) failure in ~Label()
+ bind(ConstantPoolPosition());
+#endif
+ }
+
+ // Generates an instruction sequence s.t. the return address points to the
+ // instruction following the call.
+ // The return address on the stack is used by frame iteration.
+ void StoreReturnAddressAndCall(Register target);
+
+ void ResetSpeculationPoisonRegister();
+
+ // Control-flow integrity:
+
+ // Define a function entrypoint. This doesn't emit any code for this
+ // architecture, as control-flow integrity is not supported for it.
+ void CodeEntry() {}
+ // Define an exception handler.
+ void ExceptionHandler() {}
+ // Define an exception handler and bind a label.
+ void BindExceptionHandler(Label* label) { bind(label); }
+
+ // ---------------------------------------------------------------------------
+ // Pointer compression Support
+
+ // Loads a field containing a HeapObject and decompresses it if pointer
+ // compression is enabled.
+ void LoadTaggedPointerField(const Register& destination,
+ const MemOperand& field_operand,
+ const Register& scratch = no_reg);
+
+ // Loads a field containing any tagged value and decompresses it if necessary.
+ void LoadAnyTaggedField(const Register& destination,
+ const MemOperand& field_operand,
+ const Register& scratch = no_reg);
+
+ // Loads a field containing smi value and untags it.
+ void SmiUntagField(Register dst, const MemOperand& src, RCBit rc = LeaveRC);
+
+ // Compresses and stores tagged value to given on-heap location.
+ void StoreTaggedField(const Register& value,
+ const MemOperand& dst_field_operand,
+ const Register& scratch = no_reg);
+ void StoreTaggedFieldX(const Register& value,
+ const MemOperand& dst_field_operand,
+ const Register& scratch = no_reg);
+
+ void DecompressTaggedSigned(Register destination, MemOperand field_operand);
+ void DecompressTaggedSigned(Register destination, Register src);
+ void DecompressTaggedPointer(Register destination, MemOperand field_operand);
+ void DecompressTaggedPointer(Register destination, Register source);
+ void DecompressAnyTagged(Register destination, MemOperand field_operand);
+ void DecompressAnyTagged(Register destination, Register source);
+
+ void LoadWord(Register dst, const MemOperand& mem, Register scratch);
+ void StoreWord(Register src, const MemOperand& mem, Register scratch);
+
+ private:
+ static const int kSmiShift = kSmiTagSize + kSmiShiftSize;
+
+ int CalculateStackPassedWords(int num_reg_arguments,
+ int num_double_arguments);
+ void CallCFunctionHelper(Register function, int num_reg_arguments,
+ int num_double_arguments,
+ bool has_function_descriptor);
+ void CallRecordWriteStub(Register object, Register address,
+ RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode, Handle<Code> code_target,
+ Address wasm_target);
+};
+
+// MacroAssembler implements a collection of frequently used acros.
+class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler {
+ public:
+ using TurboAssembler::TurboAssembler;
+
+ // It assumes that the arguments are located below the stack pointer.
+ // argc is the number of arguments not including the receiver.
+ // TODO(victorgomes): Remove this function once we stick with the reversed
+ // arguments order.
+ void LoadReceiver(Register dest, Register argc) {
+ LoadP(dest, MemOperand(sp, 0));
+ }
+
+ void StoreReceiver(Register rec, Register argc, Register scratch) {
+ StoreP(rec, MemOperand(sp, 0));
+ }
+
+ // ---------------------------------------------------------------------------
+ // GC Support
+
+ // Notify the garbage collector that we wrote a pointer into an object.
+ // |object| is the object being stored into, |value| is the object being
+ // stored. value and scratch registers are clobbered by the operation.
+ // The offset is the offset from the start of the object, not the offset from
+ // the tagged HeapObject pointer. For use with FieldMemOperand(reg, off).
+ void RecordWriteField(
+ Register object, int offset, Register value, Register scratch,
+ LinkRegisterStatus lr_status, SaveFPRegsMode save_fp,
+ RememberedSetAction remembered_set_action = EMIT_REMEMBERED_SET,
+ SmiCheck smi_check = INLINE_SMI_CHECK);
+
+ // For a given |object| notify the garbage collector that the slot |address|
+ // has been written. |value| is the object being stored. The value and
+ // address registers are clobbered by the operation.
+ void RecordWrite(
+ Register object, Register address, Register value,
+ LinkRegisterStatus lr_status, SaveFPRegsMode save_fp,
+ RememberedSetAction remembered_set_action = EMIT_REMEMBERED_SET,
+ SmiCheck smi_check = INLINE_SMI_CHECK);
+
+ // Enter exit frame.
+ // stack_space - extra stack space, used for parameters before call to C.
+ // At least one slot (for the return address) should be provided.
+ void EnterExitFrame(bool save_doubles, int stack_space = 1,
+ StackFrame::Type frame_type = StackFrame::EXIT);
+
+ // Leave the current exit frame. Expects the return value in r0.
+ // Expect the number of values, pushed prior to the exit frame, to
+ // remove in a register (or no_reg, if there is nothing to remove).
+ void LeaveExitFrame(bool save_doubles, Register argument_count,
+ bool argument_count_is_length = false);
+
+ void LoadMap(Register destination, Register object);
+
+ // Load the global proxy from the current context.
+ void LoadGlobalProxy(Register dst) {
+ LoadNativeContextSlot(Context::GLOBAL_PROXY_INDEX, dst);
+ }
+
+ void LoadNativeContextSlot(int index, Register dst);
+
+ // ----------------------------------------------------------------
+ // new PPC macro-assembler interfaces that are slightly higher level
+ // than assembler-ppc and may generate variable length sequences
+
+ // load a literal double value <value> to FPR <result>
+
+ void LoadHalfWord(Register dst, const MemOperand& mem,
+ Register scratch = no_reg);
+ void LoadHalfWordArith(Register dst, const MemOperand& mem,
+ Register scratch = no_reg);
+ void StoreHalfWord(Register src, const MemOperand& mem, Register scratch);
+
+ void LoadByte(Register dst, const MemOperand& mem, Register scratch);
+ void StoreByte(Register src, const MemOperand& mem, Register scratch);
+
+ void LoadDoubleU(DoubleRegister dst, const MemOperand& mem,
+ Register scratch = no_reg);
+
+ void Cmplwi(Register src1, const Operand& src2, Register scratch,
+ CRegister cr = cr7);
+ void And(Register ra, Register rs, const Operand& rb, RCBit rc = LeaveRC);
+ void Or(Register ra, Register rs, const Operand& rb, RCBit rc = LeaveRC);
+ void Xor(Register ra, Register rs, const Operand& rb, RCBit rc = LeaveRC);
+
+ void AddSmiLiteral(Register dst, Register src, Smi smi, Register scratch);
+ void SubSmiLiteral(Register dst, Register src, Smi smi, Register scratch);
+ void CmpSmiLiteral(Register src1, Smi smi, Register scratch,
+ CRegister cr = cr7);
+ void CmplSmiLiteral(Register src1, Smi smi, Register scratch,
+ CRegister cr = cr7);
+ void AndSmiLiteral(Register dst, Register src, Smi smi, Register scratch,
+ RCBit rc = LeaveRC);
+
+ // ---------------------------------------------------------------------------
+ // JavaScript invokes
+
+ // Removes current frame and its arguments from the stack preserving
+ // the arguments and a return address pushed to the stack for the next call.
+ // Both |callee_args_count| and |caller_args_countg| do not include
+ // receiver. |callee_args_count| is not modified. |caller_args_count|
+ // is trashed.
+
+ // Invoke the JavaScript function code by either calling or jumping.
+ void InvokeFunctionCode(Register function, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count, InvokeFlag flag);
+
+ // On function call, call into the debugger if necessary.
+ void CheckDebugHook(Register fun, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count);
+
+ // Invoke the JavaScript function in the given register. Changes the
+ // current context to the context in the function before invoking.
+ void InvokeFunctionWithNewTarget(Register function, Register new_target,
+ Register actual_parameter_count,
+ InvokeFlag flag);
+ void InvokeFunction(Register function, Register expected_parameter_count,
+ Register actual_parameter_count, InvokeFlag flag);
+
+ // Frame restart support
+ void MaybeDropFrames();
+
+ // Exception handling
+
+ // Push a new stack handler and link into stack handler chain.
+ void PushStackHandler();
+
+ // Unlink the stack handler on top of the stack from the stack handler chain.
+ // Must preserve the result register.
+ void PopStackHandler();
+
+ // ---------------------------------------------------------------------------
+ // Support functions.
+
+ // Compare object type for heap object. heap_object contains a non-Smi
+ // whose object type should be compared with the given type. This both
+ // sets the flags and leaves the object type in the type_reg register.
+ // It leaves the map in the map register (unless the type_reg and map register
+ // are the same register). It leaves the heap object in the heap_object
+ // register unless the heap_object register is the same register as one of the
+ // other registers.
+ // Type_reg can be no_reg. In that case ip is used.
+ void CompareObjectType(Register heap_object, Register map, Register type_reg,
+ InstanceType type);
+
+ // Compare instance type in a map. map contains a valid map object whose
+ // object type should be compared with the given type. This both
+ // sets the flags and leaves the object type in the type_reg register.
+ void CompareInstanceType(Register map, Register type_reg, InstanceType type);
+
+ // Compare the object in a register to a value from the root list.
+ // Uses the ip register as scratch.
+ void CompareRoot(Register obj, RootIndex index);
+ void PushRoot(RootIndex index) {
+ LoadRoot(r0, index);
+ Push(r0);
+ }
+
+ // Compare the object in a register to a value and jump if they are equal.
+ void JumpIfRoot(Register with, RootIndex index, Label* if_equal) {
+ CompareRoot(with, index);
+ beq(if_equal);
+ }
+
+ // Compare the object in a register to a value and jump if they are not equal.
+ void JumpIfNotRoot(Register with, RootIndex index, Label* if_not_equal) {
+ CompareRoot(with, index);
+ bne(if_not_equal);
+ }
+
+ // Checks if value is in range [lower_limit, higher_limit] using a single
+ // comparison.
+ void JumpIfIsInRange(Register value, unsigned lower_limit,
+ unsigned higher_limit, Label* on_in_range);
+
+ // ---------------------------------------------------------------------------
+ // Runtime calls
+
+ static int CallSizeNotPredictableCodeSize(Address target,
+ RelocInfo::Mode rmode,
+ Condition cond = al);
+ void CallJSEntry(Register target);
+
+ // Call a runtime routine.
+ void CallRuntime(const Runtime::Function* f, int num_arguments,
+ SaveFPRegsMode save_doubles = kDontSaveFPRegs);
+ void CallRuntimeSaveDoubles(Runtime::FunctionId fid) {
+ const Runtime::Function* function = Runtime::FunctionForId(fid);
+ CallRuntime(function, function->nargs, kSaveFPRegs);
+ }
+
+ // Convenience function: Same as above, but takes the fid instead.
+ void CallRuntime(Runtime::FunctionId fid,
+ SaveFPRegsMode save_doubles = kDontSaveFPRegs) {
+ const Runtime::Function* function = Runtime::FunctionForId(fid);
+ CallRuntime(function, function->nargs, save_doubles);
+ }
+
+ // Convenience function: Same as above, but takes the fid instead.
+ void CallRuntime(Runtime::FunctionId fid, int num_arguments,
+ SaveFPRegsMode save_doubles = kDontSaveFPRegs) {
+ CallRuntime(Runtime::FunctionForId(fid), num_arguments, save_doubles);
+ }
+
+ // Convenience function: tail call a runtime routine (jump).
+ void TailCallRuntime(Runtime::FunctionId fid);
+
+ // Jump to a runtime routine.
+ void JumpToExternalReference(const ExternalReference& builtin,
+ bool builtin_exit_frame = false);
+
+ // Generates a trampoline to jump to the off-heap instruction stream.
+ void JumpToInstructionStream(Address entry);
+
+ // ---------------------------------------------------------------------------
+ // In-place weak references.
+ void LoadWeakValue(Register out, Register in, Label* target_if_cleared);
+
+ // ---------------------------------------------------------------------------
+ // StatsCounter support
+
+ void IncrementCounter(StatsCounter* counter, int value, Register scratch1,
+ Register scratch2);
+ void DecrementCounter(StatsCounter* counter, int value, Register scratch1,
+ Register scratch2);
+
+ // ---------------------------------------------------------------------------
+ // Smi utilities
+
+ // Shift left by kSmiShift
+ void SmiTag(Register reg, RCBit rc = LeaveRC) { SmiTag(reg, reg, rc); }
+ void SmiTag(Register dst, Register src, RCBit rc = LeaveRC) {
+ ShiftLeftImm(dst, src, Operand(kSmiShift), rc);
+ }
+
+ void SmiToPtrArrayOffset(Register dst, Register src) {
+#if defined(V8_COMPRESS_POINTERS) || defined(V8_31BIT_SMIS_ON_64BIT_ARCH)
+ STATIC_ASSERT(kSmiTag == 0 && kSmiShift < kSystemPointerSizeLog2);
+ ShiftLeftImm(dst, src, Operand(kSystemPointerSizeLog2 - kSmiShift));
+#else
+ STATIC_ASSERT(kSmiTag == 0 && kSmiShift > kSystemPointerSizeLog2);
+ ShiftRightArithImm(dst, src, kSmiShift - kSystemPointerSizeLog2);
+#endif
+ }
+
+ // Jump if either of the registers contain a non-smi.
+ inline void JumpIfNotSmi(Register value, Label* not_smi_label) {
+ TestIfSmi(value, r0);
+ bne(not_smi_label, cr0);
+ }
+
+ // Abort execution if argument is a smi, enabled via --debug-code.
+ void AssertNotSmi(Register object);
+ void AssertSmi(Register object);
+
+#if !defined(V8_COMPRESS_POINTERS) && !defined(V8_31BIT_SMIS_ON_64BIT_ARCH)
+ // Ensure it is permissible to read/write int value directly from
+ // upper half of the smi.
+ STATIC_ASSERT(kSmiTag == 0);
+ STATIC_ASSERT(kSmiTagSize + kSmiShiftSize == 32);
+#endif
+#if V8_TARGET_ARCH_PPC64 && V8_TARGET_LITTLE_ENDIAN
+#define SmiWordOffset(offset) (offset + kSystemPointerSize / 2)
+#else
+#define SmiWordOffset(offset) offset
+#endif
+
+ // Abort execution if argument is not a Constructor, enabled via --debug-code.
+ void AssertConstructor(Register object);
+
+ // Abort execution if argument is not a JSFunction, enabled via --debug-code.
+ void AssertFunction(Register object);
+
+ // Abort execution if argument is not a JSBoundFunction,
+ // enabled via --debug-code.
+ void AssertBoundFunction(Register object);
+
+ // Abort execution if argument is not a JSGeneratorObject (or subclass),
+ // enabled via --debug-code.
+ void AssertGeneratorObject(Register object);
+
+ // Abort execution if argument is not undefined or an AllocationSite, enabled
+ // via --debug-code.
+ void AssertUndefinedOrAllocationSite(Register object, Register scratch);
+
+ // ---------------------------------------------------------------------------
+ // Patching helpers.
+
+ template <typename Field>
+ void DecodeField(Register dst, Register src, RCBit rc = LeaveRC) {
+ ExtractBitRange(dst, src, Field::kShift + Field::kSize - 1, Field::kShift,
+ rc);
+ }
+
+ template <typename Field>
+ void DecodeField(Register reg, RCBit rc = LeaveRC) {
+ DecodeField<Field>(reg, reg, rc);
+ }
+
+ private:
+ static const int kSmiShift = kSmiTagSize + kSmiShiftSize;
+
+ // Helper functions for generating invokes.
+ void InvokePrologue(Register expected_parameter_count,
+ Register actual_parameter_count, Label* done,
+ InvokeFlag flag);
+
+ DISALLOW_IMPLICIT_CONSTRUCTORS(MacroAssembler);
+};
+
+#define ACCESS_MASM(masm) masm->
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_PPC_MACRO_ASSEMBLER_PPC_H_
diff --git a/src/codegen/ppc/register-ppc.h b/src/codegen/ppc/register-ppc.h
new file mode 100644
index 0000000..925dc35
--- /dev/null
+++ b/src/codegen/ppc/register-ppc.h
@@ -0,0 +1,318 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_PPC_REGISTER_PPC_H_
+#define V8_CODEGEN_PPC_REGISTER_PPC_H_
+
+#include "src/codegen/register.h"
+#include "src/codegen/reglist.h"
+
+namespace v8 {
+namespace internal {
+
+// clang-format off
+#define GENERAL_REGISTERS(V) \
+ V(r0) V(sp) V(r2) V(r3) V(r4) V(r5) V(r6) V(r7) \
+ V(r8) V(r9) V(r10) V(r11) V(ip) V(r13) V(r14) V(r15) \
+ V(r16) V(r17) V(r18) V(r19) V(r20) V(r21) V(r22) V(r23) \
+ V(r24) V(r25) V(r26) V(r27) V(r28) V(r29) V(r30) V(fp)
+
+#if V8_EMBEDDED_CONSTANT_POOL
+#define ALLOCATABLE_GENERAL_REGISTERS(V) \
+ V(r3) V(r4) V(r5) V(r6) V(r7) \
+ V(r8) V(r9) V(r10) V(r14) V(r15) \
+ V(r16) V(r17) V(r18) V(r19) V(r20) V(r21) V(r22) V(r23) \
+ V(r24) V(r25) V(r26) V(r27) V(r30)
+#else
+#define ALLOCATABLE_GENERAL_REGISTERS(V) \
+ V(r3) V(r4) V(r5) V(r6) V(r7) \
+ V(r8) V(r9) V(r10) V(r14) V(r15) \
+ V(r16) V(r17) V(r18) V(r19) V(r20) V(r21) V(r22) V(r23) \
+ V(r24) V(r25) V(r26) V(r27) V(r28) V(r30)
+#endif
+
+#define LOW_DOUBLE_REGISTERS(V) \
+ V(d0) V(d1) V(d2) V(d3) V(d4) V(d5) V(d6) V(d7) \
+ V(d8) V(d9) V(d10) V(d11) V(d12) V(d13) V(d14) V(d15)
+
+#define NON_LOW_DOUBLE_REGISTERS(V) \
+ V(d16) V(d17) V(d18) V(d19) V(d20) V(d21) V(d22) V(d23) \
+ V(d24) V(d25) V(d26) V(d27) V(d28) V(d29) V(d30) V(d31)
+
+#define DOUBLE_REGISTERS(V) \
+ LOW_DOUBLE_REGISTERS(V) NON_LOW_DOUBLE_REGISTERS(V)
+
+#define FLOAT_REGISTERS DOUBLE_REGISTERS
+#define SIMD128_REGISTERS DOUBLE_REGISTERS
+
+#define ALLOCATABLE_DOUBLE_REGISTERS(V) \
+ V(d1) V(d2) V(d3) V(d4) V(d5) V(d6) V(d7) \
+ V(d8) V(d9) V(d10) V(d11) V(d12) V(d15) \
+ V(d16) V(d17) V(d18) V(d19) V(d20) V(d21) V(d22) V(d23) \
+ V(d24) V(d25) V(d26) V(d27) V(d28) V(d29) V(d30) V(d31)
+
+#define C_REGISTERS(V) \
+ V(cr0) V(cr1) V(cr2) V(cr3) V(cr4) V(cr5) V(cr6) V(cr7) \
+ V(cr8) V(cr9) V(cr10) V(cr11) V(cr12) V(cr15)
+// clang-format on
+
+// Register list in load/store instructions
+// Note that the bit values must match those used in actual instruction encoding
+
+// Caller-saved/arguments registers
+const RegList kJSCallerSaved = 1 << 3 | // r3 a1
+ 1 << 4 | // r4 a2
+ 1 << 5 | // r5 a3
+ 1 << 6 | // r6 a4
+ 1 << 7 | // r7 a5
+ 1 << 8 | // r8 a6
+ 1 << 9 | // r9 a7
+ 1 << 10 | // r10 a8
+ 1 << 11;
+
+const int kNumJSCallerSaved = 9;
+
+// Return the code of the n-th caller-saved register available to JavaScript
+// e.g. JSCallerSavedReg(0) returns r0.code() == 0
+int JSCallerSavedCode(int n);
+
+// Callee-saved registers preserved when switching from C to JavaScript
+const RegList kCalleeSaved = 1 << 14 | // r14
+ 1 << 15 | // r15
+ 1 << 16 | // r16
+ 1 << 17 | // r17
+ 1 << 18 | // r18
+ 1 << 19 | // r19
+ 1 << 20 | // r20
+ 1 << 21 | // r21
+ 1 << 22 | // r22
+ 1 << 23 | // r23
+ 1 << 24 | // r24
+ 1 << 25 | // r25
+ 1 << 26 | // r26
+ 1 << 27 | // r27
+ 1 << 28 | // r28
+ 1 << 29 | // r29
+ 1 << 30 | // r20
+ 1 << 31; // r31
+
+const int kNumCalleeSaved = 18;
+
+const RegList kCallerSavedDoubles = 1 << 0 | // d0
+ 1 << 1 | // d1
+ 1 << 2 | // d2
+ 1 << 3 | // d3
+ 1 << 4 | // d4
+ 1 << 5 | // d5
+ 1 << 6 | // d6
+ 1 << 7 | // d7
+ 1 << 8 | // d8
+ 1 << 9 | // d9
+ 1 << 10 | // d10
+ 1 << 11 | // d11
+ 1 << 12 | // d12
+ 1 << 13; // d13
+
+const int kNumCallerSavedDoubles = 14;
+
+const RegList kCalleeSavedDoubles = 1 << 14 | // d14
+ 1 << 15 | // d15
+ 1 << 16 | // d16
+ 1 << 17 | // d17
+ 1 << 18 | // d18
+ 1 << 19 | // d19
+ 1 << 20 | // d20
+ 1 << 21 | // d21
+ 1 << 22 | // d22
+ 1 << 23 | // d23
+ 1 << 24 | // d24
+ 1 << 25 | // d25
+ 1 << 26 | // d26
+ 1 << 27 | // d27
+ 1 << 28 | // d28
+ 1 << 29 | // d29
+ 1 << 30 | // d30
+ 1 << 31; // d31
+
+const int kNumCalleeSavedDoubles = 18;
+
+// The following constants describe the stack frame linkage area as
+// defined by the ABI. Note that kNumRequiredStackFrameSlots must
+// satisfy alignment requirements (rounding up if required).
+#if V8_TARGET_ARCH_PPC64 && \
+ (V8_TARGET_LITTLE_ENDIAN || \
+ (defined(_CALL_ELF) && _CALL_ELF == 2)) // ELFv2 ABI
+// [0] back chain
+// [1] condition register save area
+// [2] link register save area
+// [3] TOC save area
+// [4] Parameter1 save area
+// ...
+// [11] Parameter8 save area
+// [12] Parameter9 slot (if necessary)
+// ...
+const int kNumRequiredStackFrameSlots = 12;
+const int kStackFrameLRSlot = 2;
+const int kStackFrameExtraParamSlot = 12;
+#else // AIX
+// [0] back chain
+// [1] condition register save area
+// [2] link register save area
+// [3] reserved for compiler
+// [4] reserved by binder
+// [5] TOC save area
+// [6] Parameter1 save area
+// ...
+// [13] Parameter8 save area
+// [14] Parameter9 slot (if necessary)
+// ...
+const int kNumRequiredStackFrameSlots = 14;
+const int kStackFrameLRSlot = 2;
+const int kStackFrameExtraParamSlot = 14;
+#endif
+
+enum RegisterCode {
+#define REGISTER_CODE(R) kRegCode_##R,
+ GENERAL_REGISTERS(REGISTER_CODE)
+#undef REGISTER_CODE
+ kRegAfterLast
+};
+
+class Register : public RegisterBase<Register, kRegAfterLast> {
+ public:
+#if V8_TARGET_LITTLE_ENDIAN
+ static constexpr int kMantissaOffset = 0;
+ static constexpr int kExponentOffset = 4;
+#else
+ static constexpr int kMantissaOffset = 4;
+ static constexpr int kExponentOffset = 0;
+#endif
+
+ private:
+ friend class RegisterBase;
+ explicit constexpr Register(int code) : RegisterBase(code) {}
+};
+
+ASSERT_TRIVIALLY_COPYABLE(Register);
+static_assert(sizeof(Register) == sizeof(int),
+ "Register can efficiently be passed by value");
+
+#define DEFINE_REGISTER(R) \
+ constexpr Register R = Register::from_code(kRegCode_##R);
+GENERAL_REGISTERS(DEFINE_REGISTER)
+#undef DEFINE_REGISTER
+constexpr Register no_reg = Register::no_reg();
+
+// Aliases
+constexpr Register kConstantPoolRegister = r28; // Constant pool.
+constexpr Register kRootRegister = r29; // Roots array pointer.
+constexpr Register cp = r30; // JavaScript context pointer.
+
+constexpr bool kPadArguments = false;
+constexpr bool kSimpleFPAliasing = true;
+constexpr bool kSimdMaskRegisters = false;
+
+enum DoubleRegisterCode {
+#define REGISTER_CODE(R) kDoubleCode_##R,
+ DOUBLE_REGISTERS(REGISTER_CODE)
+#undef REGISTER_CODE
+ kDoubleAfterLast
+};
+
+// Double word FP register.
+class DoubleRegister : public RegisterBase<DoubleRegister, kDoubleAfterLast> {
+ public:
+ // A few double registers are reserved: one as a scratch register and one to
+ // hold 0.0, that does not fit in the immediate field of vmov instructions.
+ // d14: 0.0
+ // d15: scratch register.
+ static constexpr int kSizeInBytes = 8;
+
+ // This function differs from kNumRegisters by returning the number of double
+ // registers supported by the current CPU, while kNumRegisters always returns
+ // 32.
+ inline static int SupportedRegisterCount();
+
+ private:
+ friend class RegisterBase;
+ explicit constexpr DoubleRegister(int code) : RegisterBase(code) {}
+};
+
+ASSERT_TRIVIALLY_COPYABLE(DoubleRegister);
+static_assert(sizeof(DoubleRegister) == sizeof(int),
+ "DoubleRegister can efficiently be passed by value");
+
+using FloatRegister = DoubleRegister;
+
+// TODO(ppc) Define SIMD registers.
+using Simd128Register = DoubleRegister;
+
+#define DEFINE_REGISTER(R) \
+ constexpr DoubleRegister R = DoubleRegister::from_code(kDoubleCode_##R);
+DOUBLE_REGISTERS(DEFINE_REGISTER)
+#undef DEFINE_REGISTER
+constexpr DoubleRegister no_dreg = DoubleRegister::no_reg();
+
+constexpr DoubleRegister kFirstCalleeSavedDoubleReg = d14;
+constexpr DoubleRegister kLastCalleeSavedDoubleReg = d31;
+constexpr DoubleRegister kDoubleRegZero = d14;
+constexpr DoubleRegister kScratchDoubleReg = d13;
+
+Register ToRegister(int num);
+
+enum CRegisterCode {
+#define REGISTER_CODE(R) kCCode_##R,
+ C_REGISTERS(REGISTER_CODE)
+#undef REGISTER_CODE
+ kCAfterLast
+};
+
+// Coprocessor register
+class CRegister : public RegisterBase<CRegister, kCAfterLast> {
+ friend class RegisterBase;
+ explicit constexpr CRegister(int code) : RegisterBase(code) {}
+};
+
+constexpr CRegister no_creg = CRegister::no_reg();
+#define DECLARE_C_REGISTER(R) \
+ constexpr CRegister R = CRegister::from_code(kCCode_##R);
+C_REGISTERS(DECLARE_C_REGISTER)
+#undef DECLARE_C_REGISTER
+
+// Define {RegisterName} methods for the register types.
+DEFINE_REGISTER_NAMES(Register, GENERAL_REGISTERS)
+DEFINE_REGISTER_NAMES(DoubleRegister, DOUBLE_REGISTERS)
+
+// Give alias names to registers for calling conventions.
+constexpr Register kReturnRegister0 = r3;
+constexpr Register kReturnRegister1 = r4;
+constexpr Register kReturnRegister2 = r5;
+constexpr Register kJSFunctionRegister = r4;
+constexpr Register kContextRegister = r30;
+constexpr Register kAllocateSizeRegister = r4;
+constexpr Register kSpeculationPoisonRegister = r14;
+constexpr Register kInterpreterAccumulatorRegister = r3;
+constexpr Register kInterpreterBytecodeOffsetRegister = r15;
+constexpr Register kInterpreterBytecodeArrayRegister = r16;
+constexpr Register kInterpreterDispatchTableRegister = r17;
+
+constexpr Register kJavaScriptCallArgCountRegister = r3;
+constexpr Register kJavaScriptCallCodeStartRegister = r5;
+constexpr Register kJavaScriptCallTargetRegister = kJSFunctionRegister;
+constexpr Register kJavaScriptCallNewTargetRegister = r6;
+constexpr Register kJavaScriptCallExtraArg1Register = r5;
+
+constexpr Register kOffHeapTrampolineRegister = ip;
+constexpr Register kRuntimeCallFunctionRegister = r4;
+constexpr Register kRuntimeCallArgCountRegister = r3;
+constexpr Register kRuntimeCallArgvRegister = r5;
+constexpr Register kWasmInstanceRegister = r10;
+constexpr Register kWasmCompileLazyFuncIndexRegister = r15;
+
+constexpr DoubleRegister kFPReturnRegister0 = d1;
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_PPC_REGISTER_PPC_H_
diff --git a/src/codegen/register-arch.h b/src/codegen/register-arch.h
new file mode 100644
index 0000000..21a7233
--- /dev/null
+++ b/src/codegen/register-arch.h
@@ -0,0 +1,31 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_REGISTER_ARCH_H_
+#define V8_CODEGEN_REGISTER_ARCH_H_
+
+#include "src/codegen/register.h"
+#include "src/codegen/reglist.h"
+
+#if V8_TARGET_ARCH_IA32
+#include "src/codegen/ia32/register-ia32.h"
+#elif V8_TARGET_ARCH_X64
+#include "src/codegen/x64/register-x64.h"
+#elif V8_TARGET_ARCH_ARM64
+#include "src/codegen/arm64/register-arm64.h"
+#elif V8_TARGET_ARCH_ARM
+#include "src/codegen/arm/register-arm.h"
+#elif V8_TARGET_ARCH_PPC || V8_TARGET_ARCH_PPC64
+#include "src/codegen/ppc/register-ppc.h"
+#elif V8_TARGET_ARCH_MIPS
+#include "src/codegen/mips/register-mips.h"
+#elif V8_TARGET_ARCH_MIPS64
+#include "src/codegen/mips64/register-mips64.h"
+#elif V8_TARGET_ARCH_S390
+#include "src/codegen/s390/register-s390.h"
+#else
+#error Unknown architecture.
+#endif
+
+#endif // V8_CODEGEN_REGISTER_ARCH_H_
diff --git a/src/codegen/register-configuration.cc b/src/codegen/register-configuration.cc
new file mode 100644
index 0000000..1c48303
--- /dev/null
+++ b/src/codegen/register-configuration.cc
@@ -0,0 +1,331 @@
+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/register-configuration.h"
+#include "src/base/lazy-instance.h"
+#include "src/codegen/cpu-features.h"
+#include "src/codegen/register-arch.h"
+#include "src/common/globals.h"
+
+namespace v8 {
+namespace internal {
+
+namespace {
+
+#define REGISTER_COUNT(R) 1 +
+static const int kMaxAllocatableGeneralRegisterCount =
+ ALLOCATABLE_GENERAL_REGISTERS(REGISTER_COUNT) 0;
+static const int kMaxAllocatableDoubleRegisterCount =
+ ALLOCATABLE_DOUBLE_REGISTERS(REGISTER_COUNT) 0;
+
+static const int kAllocatableGeneralCodes[] = {
+#define REGISTER_CODE(R) kRegCode_##R,
+ ALLOCATABLE_GENERAL_REGISTERS(REGISTER_CODE)};
+#undef REGISTER_CODE
+
+#define REGISTER_CODE(R) kDoubleCode_##R,
+static const int kAllocatableDoubleCodes[] = {
+ ALLOCATABLE_DOUBLE_REGISTERS(REGISTER_CODE)};
+#if V8_TARGET_ARCH_ARM
+static const int kAllocatableNoVFP32DoubleCodes[] = {
+ ALLOCATABLE_NO_VFP32_DOUBLE_REGISTERS(REGISTER_CODE)};
+#endif // V8_TARGET_ARCH_ARM
+#undef REGISTER_CODE
+
+STATIC_ASSERT(RegisterConfiguration::kMaxGeneralRegisters >=
+ Register::kNumRegisters);
+STATIC_ASSERT(RegisterConfiguration::kMaxFPRegisters >=
+ FloatRegister::kNumRegisters);
+STATIC_ASSERT(RegisterConfiguration::kMaxFPRegisters >=
+ DoubleRegister::kNumRegisters);
+STATIC_ASSERT(RegisterConfiguration::kMaxFPRegisters >=
+ Simd128Register::kNumRegisters);
+
+// Callers on architectures other than Arm expect this to be be constant
+// between build and runtime. Avoid adding variability on other platforms.
+static int get_num_allocatable_double_registers() {
+ return
+#if V8_TARGET_ARCH_IA32
+ kMaxAllocatableDoubleRegisterCount;
+#elif V8_TARGET_ARCH_X64
+ kMaxAllocatableDoubleRegisterCount;
+#elif V8_TARGET_ARCH_ARM
+ CpuFeatures::IsSupported(VFP32DREGS)
+ ? kMaxAllocatableDoubleRegisterCount
+ : (ALLOCATABLE_NO_VFP32_DOUBLE_REGISTERS(REGISTER_COUNT) 0);
+#elif V8_TARGET_ARCH_ARM64
+ kMaxAllocatableDoubleRegisterCount;
+#elif V8_TARGET_ARCH_MIPS
+ kMaxAllocatableDoubleRegisterCount;
+#elif V8_TARGET_ARCH_MIPS64
+ kMaxAllocatableDoubleRegisterCount;
+#elif V8_TARGET_ARCH_PPC
+ kMaxAllocatableDoubleRegisterCount;
+#elif V8_TARGET_ARCH_PPC64
+ kMaxAllocatableDoubleRegisterCount;
+#elif V8_TARGET_ARCH_S390
+ kMaxAllocatableDoubleRegisterCount;
+#else
+#error Unsupported target architecture.
+#endif
+}
+
+#undef REGISTER_COUNT
+
+// Callers on architectures other than Arm expect this to be be constant
+// between build and runtime. Avoid adding variability on other platforms.
+static const int* get_allocatable_double_codes() {
+ return
+#if V8_TARGET_ARCH_ARM
+ CpuFeatures::IsSupported(VFP32DREGS) ? kAllocatableDoubleCodes
+ : kAllocatableNoVFP32DoubleCodes;
+#else
+ kAllocatableDoubleCodes;
+#endif
+}
+
+class ArchDefaultRegisterConfiguration : public RegisterConfiguration {
+ public:
+ ArchDefaultRegisterConfiguration()
+ : RegisterConfiguration(
+ Register::kNumRegisters, DoubleRegister::kNumRegisters,
+ kMaxAllocatableGeneralRegisterCount,
+ get_num_allocatable_double_registers(), kAllocatableGeneralCodes,
+ get_allocatable_double_codes(),
+ kSimpleFPAliasing ? AliasingKind::OVERLAP : AliasingKind::COMBINE) {
+ }
+};
+
+DEFINE_LAZY_LEAKY_OBJECT_GETTER(ArchDefaultRegisterConfiguration,
+ GetDefaultRegisterConfiguration)
+
+// Allocatable registers with the masking register removed.
+class ArchDefaultPoisoningRegisterConfiguration : public RegisterConfiguration {
+ public:
+ ArchDefaultPoisoningRegisterConfiguration()
+ : RegisterConfiguration(
+ Register::kNumRegisters, DoubleRegister::kNumRegisters,
+ kMaxAllocatableGeneralRegisterCount - 1,
+ get_num_allocatable_double_registers(),
+ InitializeGeneralRegisterCodes(), get_allocatable_double_codes(),
+ kSimpleFPAliasing ? AliasingKind::OVERLAP : AliasingKind::COMBINE) {
+ }
+
+ private:
+ static const int* InitializeGeneralRegisterCodes() {
+ int filtered_index = 0;
+ for (int i = 0; i < kMaxAllocatableGeneralRegisterCount; ++i) {
+ if (kAllocatableGeneralCodes[i] != kSpeculationPoisonRegister.code()) {
+ allocatable_general_codes_[filtered_index] =
+ kAllocatableGeneralCodes[i];
+ filtered_index++;
+ }
+ }
+ DCHECK_EQ(filtered_index, kMaxAllocatableGeneralRegisterCount - 1);
+ return allocatable_general_codes_;
+ }
+
+ static int
+ allocatable_general_codes_[kMaxAllocatableGeneralRegisterCount - 1];
+};
+
+int ArchDefaultPoisoningRegisterConfiguration::allocatable_general_codes_
+ [kMaxAllocatableGeneralRegisterCount - 1];
+
+DEFINE_LAZY_LEAKY_OBJECT_GETTER(ArchDefaultPoisoningRegisterConfiguration,
+ GetDefaultPoisoningRegisterConfiguration)
+
+// RestrictedRegisterConfiguration uses the subset of allocatable general
+// registers the architecture support, which results into generating assembly
+// to use less registers. Currently, it's only used by RecordWrite code stub.
+class RestrictedRegisterConfiguration : public RegisterConfiguration {
+ public:
+ RestrictedRegisterConfiguration(
+ int num_allocatable_general_registers,
+ std::unique_ptr<int[]> allocatable_general_register_codes,
+ std::unique_ptr<char const*[]> allocatable_general_register_names)
+ : RegisterConfiguration(
+ Register::kNumRegisters, DoubleRegister::kNumRegisters,
+ num_allocatable_general_registers,
+ get_num_allocatable_double_registers(),
+ allocatable_general_register_codes.get(),
+ get_allocatable_double_codes(),
+ kSimpleFPAliasing ? AliasingKind::OVERLAP : AliasingKind::COMBINE),
+ allocatable_general_register_codes_(
+ std::move(allocatable_general_register_codes)),
+ allocatable_general_register_names_(
+ std::move(allocatable_general_register_names)) {
+ for (int i = 0; i < num_allocatable_general_registers; ++i) {
+ DCHECK(
+ IsAllocatableGeneralRegister(allocatable_general_register_codes_[i]));
+ }
+ }
+
+ bool IsAllocatableGeneralRegister(int code) {
+ for (int i = 0; i < kMaxAllocatableGeneralRegisterCount; ++i) {
+ if (code == kAllocatableGeneralCodes[i]) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private:
+ std::unique_ptr<int[]> allocatable_general_register_codes_;
+ std::unique_ptr<char const*[]> allocatable_general_register_names_;
+};
+
+} // namespace
+
+const RegisterConfiguration* RegisterConfiguration::Default() {
+ return GetDefaultRegisterConfiguration();
+}
+
+const RegisterConfiguration* RegisterConfiguration::Poisoning() {
+ return GetDefaultPoisoningRegisterConfiguration();
+}
+
+const RegisterConfiguration* RegisterConfiguration::RestrictGeneralRegisters(
+ RegList registers) {
+ int num = NumRegs(registers);
+ std::unique_ptr<int[]> codes{new int[num]};
+ std::unique_ptr<char const* []> names { new char const*[num] };
+ int counter = 0;
+ for (int i = 0; i < Default()->num_allocatable_general_registers(); ++i) {
+ auto reg = Register::from_code(Default()->GetAllocatableGeneralCode(i));
+ if (reg.bit() & registers) {
+ DCHECK(counter < num);
+ codes[counter] = reg.code();
+ names[counter] = RegisterName(Register::from_code(i));
+ counter++;
+ }
+ }
+
+ return new RestrictedRegisterConfiguration(num, std::move(codes),
+ std::move(names));
+}
+
+RegisterConfiguration::RegisterConfiguration(
+ int num_general_registers, int num_double_registers,
+ int num_allocatable_general_registers, int num_allocatable_double_registers,
+ const int* allocatable_general_codes, const int* allocatable_double_codes,
+ AliasingKind fp_aliasing_kind)
+ : num_general_registers_(num_general_registers),
+ num_float_registers_(0),
+ num_double_registers_(num_double_registers),
+ num_simd128_registers_(0),
+ num_allocatable_general_registers_(num_allocatable_general_registers),
+ num_allocatable_float_registers_(0),
+ num_allocatable_double_registers_(num_allocatable_double_registers),
+ num_allocatable_simd128_registers_(0),
+ allocatable_general_codes_mask_(0),
+ allocatable_float_codes_mask_(0),
+ allocatable_double_codes_mask_(0),
+ allocatable_simd128_codes_mask_(0),
+ allocatable_general_codes_(allocatable_general_codes),
+ allocatable_double_codes_(allocatable_double_codes),
+ fp_aliasing_kind_(fp_aliasing_kind) {
+ DCHECK_LE(num_general_registers_,
+ RegisterConfiguration::kMaxGeneralRegisters);
+ DCHECK_LE(num_double_registers_, RegisterConfiguration::kMaxFPRegisters);
+ for (int i = 0; i < num_allocatable_general_registers_; ++i) {
+ allocatable_general_codes_mask_ |= (1 << allocatable_general_codes_[i]);
+ }
+ for (int i = 0; i < num_allocatable_double_registers_; ++i) {
+ allocatable_double_codes_mask_ |= (1 << allocatable_double_codes_[i]);
+ }
+
+ if (fp_aliasing_kind_ == COMBINE) {
+ num_float_registers_ = num_double_registers_ * 2 <= kMaxFPRegisters
+ ? num_double_registers_ * 2
+ : kMaxFPRegisters;
+ num_allocatable_float_registers_ = 0;
+ for (int i = 0; i < num_allocatable_double_registers_; i++) {
+ int base_code = allocatable_double_codes_[i] * 2;
+ if (base_code >= kMaxFPRegisters) continue;
+ allocatable_float_codes_[num_allocatable_float_registers_++] = base_code;
+ allocatable_float_codes_[num_allocatable_float_registers_++] =
+ base_code + 1;
+ allocatable_float_codes_mask_ |= (0x3 << base_code);
+ }
+ num_simd128_registers_ = num_double_registers_ / 2;
+ num_allocatable_simd128_registers_ = 0;
+ int last_simd128_code = allocatable_double_codes_[0] / 2;
+ for (int i = 1; i < num_allocatable_double_registers_; i++) {
+ int next_simd128_code = allocatable_double_codes_[i] / 2;
+ // This scheme assumes allocatable_double_codes_ are strictly increasing.
+ DCHECK_GE(next_simd128_code, last_simd128_code);
+ if (last_simd128_code == next_simd128_code) {
+ allocatable_simd128_codes_[num_allocatable_simd128_registers_++] =
+ next_simd128_code;
+ allocatable_simd128_codes_mask_ |= (0x1 << next_simd128_code);
+ }
+ last_simd128_code = next_simd128_code;
+ }
+ } else {
+ DCHECK(fp_aliasing_kind_ == OVERLAP);
+ num_float_registers_ = num_simd128_registers_ = num_double_registers_;
+ num_allocatable_float_registers_ = num_allocatable_simd128_registers_ =
+ num_allocatable_double_registers_;
+ for (int i = 0; i < num_allocatable_float_registers_; ++i) {
+ allocatable_float_codes_[i] = allocatable_simd128_codes_[i] =
+ allocatable_double_codes_[i];
+ }
+ allocatable_float_codes_mask_ = allocatable_simd128_codes_mask_ =
+ allocatable_double_codes_mask_;
+ }
+}
+
+// Assert that kFloat32, kFloat64, and kSimd128 are consecutive values.
+STATIC_ASSERT(static_cast<int>(MachineRepresentation::kSimd128) ==
+ static_cast<int>(MachineRepresentation::kFloat64) + 1);
+STATIC_ASSERT(static_cast<int>(MachineRepresentation::kFloat64) ==
+ static_cast<int>(MachineRepresentation::kFloat32) + 1);
+
+int RegisterConfiguration::GetAliases(MachineRepresentation rep, int index,
+ MachineRepresentation other_rep,
+ int* alias_base_index) const {
+ DCHECK(fp_aliasing_kind_ == COMBINE);
+ DCHECK(IsFloatingPoint(rep) && IsFloatingPoint(other_rep));
+ if (rep == other_rep) {
+ *alias_base_index = index;
+ return 1;
+ }
+ int rep_int = static_cast<int>(rep);
+ int other_rep_int = static_cast<int>(other_rep);
+ if (rep_int > other_rep_int) {
+ int shift = rep_int - other_rep_int;
+ int base_index = index << shift;
+ if (base_index >= kMaxFPRegisters) {
+ // Alias indices would be out of FP register range.
+ return 0;
+ }
+ *alias_base_index = base_index;
+ return 1 << shift;
+ }
+ int shift = other_rep_int - rep_int;
+ *alias_base_index = index >> shift;
+ return 1;
+}
+
+bool RegisterConfiguration::AreAliases(MachineRepresentation rep, int index,
+ MachineRepresentation other_rep,
+ int other_index) const {
+ DCHECK(fp_aliasing_kind_ == COMBINE);
+ DCHECK(IsFloatingPoint(rep) && IsFloatingPoint(other_rep));
+ if (rep == other_rep) {
+ return index == other_index;
+ }
+ int rep_int = static_cast<int>(rep);
+ int other_rep_int = static_cast<int>(other_rep);
+ if (rep_int > other_rep_int) {
+ int shift = rep_int - other_rep_int;
+ return index == other_index >> shift;
+ }
+ int shift = other_rep_int - rep_int;
+ return index >> shift == other_index;
+}
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/register-configuration.h b/src/codegen/register-configuration.h
new file mode 100644
index 0000000..cdf9dda
--- /dev/null
+++ b/src/codegen/register-configuration.h
@@ -0,0 +1,158 @@
+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_REGISTER_CONFIGURATION_H_
+#define V8_CODEGEN_REGISTER_CONFIGURATION_H_
+
+#include "src/base/macros.h"
+#include "src/codegen/machine-type.h"
+#include "src/codegen/reglist.h"
+#include "src/common/globals.h"
+#include "src/utils/utils.h"
+
+namespace v8 {
+namespace internal {
+
+// An architecture independent representation of the sets of registers available
+// for instruction creation.
+class V8_EXPORT_PRIVATE RegisterConfiguration {
+ public:
+ enum AliasingKind {
+ // Registers alias a single register of every other size (e.g. Intel).
+ OVERLAP,
+ // Registers alias two registers of the next smaller size (e.g. ARM).
+ COMBINE
+ };
+
+ // Architecture independent maxes.
+ static constexpr int kMaxGeneralRegisters = 32;
+ static constexpr int kMaxFPRegisters = 32;
+ static constexpr int kMaxRegisters =
+ std::max(kMaxFPRegisters, kMaxGeneralRegisters);
+
+ // Default RegisterConfigurations for the target architecture.
+ static const RegisterConfiguration* Default();
+
+ // Register configuration with reserved masking register.
+ static const RegisterConfiguration* Poisoning();
+
+ static const RegisterConfiguration* RestrictGeneralRegisters(
+ RegList registers);
+
+ RegisterConfiguration(int num_general_registers, int num_double_registers,
+ int num_allocatable_general_registers,
+ int num_allocatable_double_registers,
+ const int* allocatable_general_codes,
+ const int* allocatable_double_codes,
+ AliasingKind fp_aliasing_kind);
+
+ int num_general_registers() const { return num_general_registers_; }
+ int num_float_registers() const { return num_float_registers_; }
+ int num_double_registers() const { return num_double_registers_; }
+ int num_simd128_registers() const { return num_simd128_registers_; }
+ int num_allocatable_general_registers() const {
+ return num_allocatable_general_registers_;
+ }
+ int num_allocatable_float_registers() const {
+ return num_allocatable_float_registers_;
+ }
+ // Caution: this value depends on the current cpu and may change between
+ // build and runtime. At the time of writing, the only architecture with a
+ // variable allocatable double register set is Arm.
+ int num_allocatable_double_registers() const {
+ return num_allocatable_double_registers_;
+ }
+ int num_allocatable_simd128_registers() const {
+ return num_allocatable_simd128_registers_;
+ }
+ AliasingKind fp_aliasing_kind() const { return fp_aliasing_kind_; }
+ int32_t allocatable_general_codes_mask() const {
+ return allocatable_general_codes_mask_;
+ }
+ int32_t allocatable_double_codes_mask() const {
+ return allocatable_double_codes_mask_;
+ }
+ int32_t allocatable_float_codes_mask() const {
+ return allocatable_float_codes_mask_;
+ }
+ int GetAllocatableGeneralCode(int index) const {
+ DCHECK(index >= 0 && index < num_allocatable_general_registers());
+ return allocatable_general_codes_[index];
+ }
+ bool IsAllocatableGeneralCode(int index) const {
+ return ((1 << index) & allocatable_general_codes_mask_) != 0;
+ }
+ int GetAllocatableFloatCode(int index) const {
+ DCHECK(index >= 0 && index < num_allocatable_float_registers());
+ return allocatable_float_codes_[index];
+ }
+ bool IsAllocatableFloatCode(int index) const {
+ return ((1 << index) & allocatable_float_codes_mask_) != 0;
+ }
+ int GetAllocatableDoubleCode(int index) const {
+ DCHECK(index >= 0 && index < num_allocatable_double_registers());
+ return allocatable_double_codes_[index];
+ }
+ bool IsAllocatableDoubleCode(int index) const {
+ return ((1 << index) & allocatable_double_codes_mask_) != 0;
+ }
+ int GetAllocatableSimd128Code(int index) const {
+ DCHECK(index >= 0 && index < num_allocatable_simd128_registers());
+ return allocatable_simd128_codes_[index];
+ }
+ bool IsAllocatableSimd128Code(int index) const {
+ return ((1 << index) & allocatable_simd128_codes_mask_) != 0;
+ }
+
+ const int* allocatable_general_codes() const {
+ return allocatable_general_codes_;
+ }
+ const int* allocatable_float_codes() const {
+ return allocatable_float_codes_;
+ }
+ const int* allocatable_double_codes() const {
+ return allocatable_double_codes_;
+ }
+ const int* allocatable_simd128_codes() const {
+ return allocatable_simd128_codes_;
+ }
+
+ // Aliasing calculations for floating point registers, when fp_aliasing_kind()
+ // is COMBINE. Currently only implemented for kFloat32, kFloat64, or kSimd128
+ // reps. Returns the number of aliases, and if > 0, alias_base_index is set to
+ // the index of the first alias.
+ int GetAliases(MachineRepresentation rep, int index,
+ MachineRepresentation other_rep, int* alias_base_index) const;
+ // Returns a value indicating whether two registers alias each other, when
+ // fp_aliasing_kind() is COMBINE. Currently implemented for kFloat32,
+ // kFloat64, or kSimd128 reps.
+ bool AreAliases(MachineRepresentation rep, int index,
+ MachineRepresentation other_rep, int other_index) const;
+
+ virtual ~RegisterConfiguration() = default;
+
+ private:
+ const int num_general_registers_;
+ int num_float_registers_;
+ const int num_double_registers_;
+ int num_simd128_registers_;
+ int num_allocatable_general_registers_;
+ int num_allocatable_float_registers_;
+ int num_allocatable_double_registers_;
+ int num_allocatable_simd128_registers_;
+ int32_t allocatable_general_codes_mask_;
+ int32_t allocatable_float_codes_mask_;
+ int32_t allocatable_double_codes_mask_;
+ int32_t allocatable_simd128_codes_mask_;
+ const int* allocatable_general_codes_;
+ int allocatable_float_codes_[kMaxFPRegisters];
+ const int* allocatable_double_codes_;
+ int allocatable_simd128_codes_[kMaxFPRegisters];
+ AliasingKind fp_aliasing_kind_;
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_REGISTER_CONFIGURATION_H_
diff --git a/src/codegen/register.cc b/src/codegen/register.cc
new file mode 100644
index 0000000..4ad76c6
--- /dev/null
+++ b/src/codegen/register.cc
@@ -0,0 +1,16 @@
+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/register.h"
+#include "src/codegen/register-arch.h"
+
+namespace v8 {
+namespace internal {
+
+bool ShouldPadArguments(int argument_count) {
+ return kPadArguments && (argument_count % 2 != 0);
+}
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/register.h b/src/codegen/register.h
new file mode 100644
index 0000000..2dcf0fb
--- /dev/null
+++ b/src/codegen/register.h
@@ -0,0 +1,94 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_REGISTER_H_
+#define V8_CODEGEN_REGISTER_H_
+
+#include "src/base/bounds.h"
+#include "src/codegen/reglist.h"
+
+namespace v8 {
+
+namespace internal {
+
+// Base type for CPU Registers.
+//
+// 1) We would prefer to use an enum for registers, but enum values are
+// assignment-compatible with int, which has caused code-generation bugs.
+//
+// 2) By not using an enum, we are possibly preventing the compiler from
+// doing certain constant folds, which may significantly reduce the
+// code generated for some assembly instructions (because they boil down
+// to a few constants). If this is a problem, we could change the code
+// such that we use an enum in optimized mode, and the class in debug
+// mode. This way we get the compile-time error checking in debug mode
+// and best performance in optimized code.
+template <typename SubType, int kAfterLastRegister>
+class RegisterBase {
+ public:
+ static constexpr int kCode_no_reg = -1;
+ static constexpr int kNumRegisters = kAfterLastRegister;
+
+ static constexpr SubType no_reg() { return SubType{kCode_no_reg}; }
+
+ static constexpr SubType from_code(int code) {
+ CONSTEXPR_DCHECK(base::IsInRange(code, 0, kNumRegisters - 1));
+ return SubType{code};
+ }
+
+ template <typename... Register>
+ static constexpr RegList ListOf(Register... regs) {
+ return CombineRegLists(regs.bit()...);
+ }
+
+ constexpr bool is_valid() const { return reg_code_ != kCode_no_reg; }
+
+ constexpr int code() const {
+ CONSTEXPR_DCHECK(is_valid());
+ return reg_code_;
+ }
+
+ constexpr RegList bit() const {
+ return is_valid() ? RegList{1} << code() : RegList{};
+ }
+
+ inline constexpr bool operator==(SubType other) const {
+ return reg_code_ == other.reg_code_;
+ }
+ inline constexpr bool operator!=(SubType other) const {
+ return reg_code_ != other.reg_code_;
+ }
+
+ // Used to print the name of some special registers.
+ static const char* GetSpecialRegisterName(int code) { return "UNKNOWN"; }
+
+ protected:
+ explicit constexpr RegisterBase(int code) : reg_code_(code) {}
+
+ private:
+ int reg_code_;
+};
+
+// Whether padding is needed for the given stack argument count.
+bool ShouldPadArguments(int argument_count);
+
+template <typename RegType,
+ typename = decltype(RegisterName(std::declval<RegType>()))>
+inline std::ostream& operator<<(std::ostream& os, RegType reg) {
+ return os << RegisterName(reg);
+}
+
+// Helper macros to define a {RegisterName} method based on a macro list
+// containing all names.
+#define DEFINE_REGISTER_NAMES_NAME(name) #name,
+#define DEFINE_REGISTER_NAMES(RegType, LIST) \
+ inline const char* RegisterName(RegType reg) { \
+ static constexpr const char* Names[] = {LIST(DEFINE_REGISTER_NAMES_NAME)}; \
+ STATIC_ASSERT(arraysize(Names) == RegType::kNumRegisters); \
+ return reg.is_valid() ? Names[reg.code()] : "invalid"; \
+ }
+
+} // namespace internal
+} // namespace v8
+#endif // V8_CODEGEN_REGISTER_H_
diff --git a/src/codegen/reglist.h b/src/codegen/reglist.h
new file mode 100644
index 0000000..4f1d352
--- /dev/null
+++ b/src/codegen/reglist.h
@@ -0,0 +1,45 @@
+// Copyright 2017 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_REGLIST_H_
+#define V8_CODEGEN_REGLIST_H_
+
+#include <cstdint>
+
+#include "src/base/bits.h"
+#include "src/base/template-utils.h"
+
+namespace v8 {
+namespace internal {
+
+// Register configurations.
+#if V8_TARGET_ARCH_ARM64
+using RegList = uint64_t;
+#else
+using RegList = uint32_t;
+#endif
+
+// Get the number of registers in a given register list.
+constexpr int NumRegs(RegList list) {
+ return base::bits::CountPopulation(list);
+}
+
+namespace detail {
+// Combine two RegLists by building the union of the contained registers.
+// TODO(clemensb): Replace by constexpr lambda once we have C++17.
+constexpr RegList CombineRegListsHelper(RegList list1, RegList list2) {
+ return list1 | list2;
+}
+} // namespace detail
+
+// Combine several RegLists by building the union of the contained registers.
+template <typename... RegLists>
+constexpr RegList CombineRegLists(RegLists... lists) {
+ return base::fold(detail::CombineRegListsHelper, 0, lists...);
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_REGLIST_H_
diff --git a/src/codegen/reloc-info.cc b/src/codegen/reloc-info.cc
new file mode 100644
index 0000000..7fdc2f3
--- /dev/null
+++ b/src/codegen/reloc-info.cc
@@ -0,0 +1,548 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/reloc-info.h"
+
+#include "src/codegen/assembler-inl.h"
+#include "src/codegen/code-reference.h"
+#include "src/codegen/external-reference-encoder.h"
+#include "src/deoptimizer/deoptimize-reason.h"
+#include "src/deoptimizer/deoptimizer.h"
+#include "src/heap/heap-write-barrier-inl.h"
+#include "src/objects/code-inl.h"
+#include "src/snapshot/embedded/embedded-data.h"
+
+namespace v8 {
+namespace internal {
+
+const char* const RelocInfo::kFillerCommentString = "DEOPTIMIZATION PADDING";
+
+// -----------------------------------------------------------------------------
+// Implementation of RelocInfoWriter and RelocIterator
+//
+// Relocation information is written backwards in memory, from high addresses
+// towards low addresses, byte by byte. Therefore, in the encodings listed
+// below, the first byte listed it at the highest address, and successive
+// bytes in the record are at progressively lower addresses.
+//
+// Encoding
+//
+// The most common modes are given single-byte encodings. Also, it is
+// easy to identify the type of reloc info and skip unwanted modes in
+// an iteration.
+//
+// The encoding relies on the fact that there are fewer than 14
+// different relocation modes using standard non-compact encoding.
+//
+// The first byte of a relocation record has a tag in its low 2 bits:
+// Here are the record schemes, depending on the low tag and optional higher
+// tags.
+//
+// Low tag:
+// 00: embedded_object: [6-bit pc delta] 00
+//
+// 01: code_target: [6-bit pc delta] 01
+//
+// 10: wasm_stub_call: [6-bit pc delta] 10
+//
+// 11: long_record [6 bit reloc mode] 11
+// followed by pc delta
+// followed by optional data depending on type.
+//
+// If a pc delta exceeds 6 bits, it is split into a remainder that fits into
+// 6 bits and a part that does not. The latter is encoded as a long record
+// with PC_JUMP as pseudo reloc info mode. The former is encoded as part of
+// the following record in the usual way. The long pc jump record has variable
+// length:
+// pc-jump: [PC_JUMP] 11
+// [7 bits data] 0
+// ...
+// [7 bits data] 1
+// (Bits 6..31 of pc delta, with leading zeroes
+// dropped, and last non-zero chunk tagged with 1.)
+
+const int kTagBits = 2;
+const int kTagMask = (1 << kTagBits) - 1;
+const int kLongTagBits = 6;
+
+const int kEmbeddedObjectTag = 0;
+const int kCodeTargetTag = 1;
+const int kWasmStubCallTag = 2;
+const int kDefaultTag = 3;
+
+const int kSmallPCDeltaBits = kBitsPerByte - kTagBits;
+const int kSmallPCDeltaMask = (1 << kSmallPCDeltaBits) - 1;
+const int RelocInfo::kMaxSmallPCDelta = kSmallPCDeltaMask;
+
+const int kChunkBits = 7;
+const int kChunkMask = (1 << kChunkBits) - 1;
+const int kLastChunkTagBits = 1;
+const int kLastChunkTagMask = 1;
+const int kLastChunkTag = 1;
+
+uint32_t RelocInfoWriter::WriteLongPCJump(uint32_t pc_delta) {
+ // Return if the pc_delta can fit in kSmallPCDeltaBits bits.
+ // Otherwise write a variable length PC jump for the bits that do
+ // not fit in the kSmallPCDeltaBits bits.
+ if (is_uintn(pc_delta, kSmallPCDeltaBits)) return pc_delta;
+ WriteMode(RelocInfo::PC_JUMP);
+ uint32_t pc_jump = pc_delta >> kSmallPCDeltaBits;
+ DCHECK_GT(pc_jump, 0);
+ // Write kChunkBits size chunks of the pc_jump.
+ for (; pc_jump > 0; pc_jump = pc_jump >> kChunkBits) {
+ byte b = pc_jump & kChunkMask;
+ *--pos_ = b << kLastChunkTagBits;
+ }
+ // Tag the last chunk so it can be identified.
+ *pos_ = *pos_ | kLastChunkTag;
+ // Return the remaining kSmallPCDeltaBits of the pc_delta.
+ return pc_delta & kSmallPCDeltaMask;
+}
+
+void RelocInfoWriter::WriteShortTaggedPC(uint32_t pc_delta, int tag) {
+ // Write a byte of tagged pc-delta, possibly preceded by an explicit pc-jump.
+ pc_delta = WriteLongPCJump(pc_delta);
+ *--pos_ = pc_delta << kTagBits | tag;
+}
+
+void RelocInfoWriter::WriteShortData(intptr_t data_delta) {
+ *--pos_ = static_cast<byte>(data_delta);
+}
+
+void RelocInfoWriter::WriteMode(RelocInfo::Mode rmode) {
+ STATIC_ASSERT(RelocInfo::NUMBER_OF_MODES <= (1 << kLongTagBits));
+ *--pos_ = static_cast<int>((rmode << kTagBits) | kDefaultTag);
+}
+
+void RelocInfoWriter::WriteModeAndPC(uint32_t pc_delta, RelocInfo::Mode rmode) {
+ // Write two-byte tagged pc-delta, possibly preceded by var. length pc-jump.
+ pc_delta = WriteLongPCJump(pc_delta);
+ WriteMode(rmode);
+ *--pos_ = pc_delta;
+}
+
+void RelocInfoWriter::WriteIntData(int number) {
+ for (int i = 0; i < kIntSize; i++) {
+ *--pos_ = static_cast<byte>(number);
+ // Signed right shift is arithmetic shift. Tested in test-utils.cc.
+ number = number >> kBitsPerByte;
+ }
+}
+
+void RelocInfoWriter::WriteData(intptr_t data_delta) {
+ for (int i = 0; i < kIntptrSize; i++) {
+ *--pos_ = static_cast<byte>(data_delta);
+ // Signed right shift is arithmetic shift. Tested in test-utils.cc.
+ data_delta = data_delta >> kBitsPerByte;
+ }
+}
+
+void RelocInfoWriter::Write(const RelocInfo* rinfo) {
+ RelocInfo::Mode rmode = rinfo->rmode();
+#ifdef DEBUG
+ byte* begin_pos = pos_;
+#endif
+ DCHECK(rinfo->rmode() < RelocInfo::NUMBER_OF_MODES);
+ DCHECK_GE(rinfo->pc() - reinterpret_cast<Address>(last_pc_), 0);
+ // Use unsigned delta-encoding for pc.
+ uint32_t pc_delta =
+ static_cast<uint32_t>(rinfo->pc() - reinterpret_cast<Address>(last_pc_));
+
+ // The two most common modes are given small tags, and usually fit in a byte.
+ if (rmode == RelocInfo::FULL_EMBEDDED_OBJECT) {
+ WriteShortTaggedPC(pc_delta, kEmbeddedObjectTag);
+ } else if (rmode == RelocInfo::CODE_TARGET) {
+ WriteShortTaggedPC(pc_delta, kCodeTargetTag);
+ DCHECK_LE(begin_pos - pos_, RelocInfo::kMaxCallSize);
+ } else if (rmode == RelocInfo::WASM_STUB_CALL) {
+ WriteShortTaggedPC(pc_delta, kWasmStubCallTag);
+ } else {
+ WriteModeAndPC(pc_delta, rmode);
+ if (RelocInfo::IsDeoptReason(rmode)) {
+ DCHECK_LT(rinfo->data(), 1 << kBitsPerByte);
+ WriteShortData(rinfo->data());
+ } else if (RelocInfo::IsConstPool(rmode) ||
+ RelocInfo::IsVeneerPool(rmode) || RelocInfo::IsDeoptId(rmode) ||
+ RelocInfo::IsDeoptPosition(rmode)) {
+ WriteIntData(static_cast<int>(rinfo->data()));
+ }
+ }
+ last_pc_ = reinterpret_cast<byte*>(rinfo->pc());
+#ifdef DEBUG
+ DCHECK_LE(begin_pos - pos_, kMaxSize);
+#endif
+}
+
+inline int RelocIterator::AdvanceGetTag() { return *--pos_ & kTagMask; }
+
+inline RelocInfo::Mode RelocIterator::GetMode() {
+ return static_cast<RelocInfo::Mode>((*pos_ >> kTagBits) &
+ ((1 << kLongTagBits) - 1));
+}
+
+inline void RelocIterator::ReadShortTaggedPC() {
+ rinfo_.pc_ += *pos_ >> kTagBits;
+}
+
+inline void RelocIterator::AdvanceReadPC() { rinfo_.pc_ += *--pos_; }
+
+void RelocIterator::AdvanceReadInt() {
+ int x = 0;
+ for (int i = 0; i < kIntSize; i++) {
+ x |= static_cast<int>(*--pos_) << i * kBitsPerByte;
+ }
+ rinfo_.data_ = x;
+}
+
+void RelocIterator::AdvanceReadData() {
+ intptr_t x = 0;
+ for (int i = 0; i < kIntptrSize; i++) {
+ x |= static_cast<intptr_t>(*--pos_) << i * kBitsPerByte;
+ }
+ rinfo_.data_ = x;
+}
+
+void RelocIterator::AdvanceReadLongPCJump() {
+ // Read the 32-kSmallPCDeltaBits most significant bits of the
+ // pc jump in kChunkBits bit chunks and shift them into place.
+ // Stop when the last chunk is encountered.
+ uint32_t pc_jump = 0;
+ for (int i = 0; i < kIntSize; i++) {
+ byte pc_jump_part = *--pos_;
+ pc_jump |= (pc_jump_part >> kLastChunkTagBits) << i * kChunkBits;
+ if ((pc_jump_part & kLastChunkTagMask) == 1) break;
+ }
+ // The least significant kSmallPCDeltaBits bits will be added
+ // later.
+ rinfo_.pc_ += pc_jump << kSmallPCDeltaBits;
+}
+
+inline void RelocIterator::ReadShortData() {
+ uint8_t unsigned_b = *pos_;
+ rinfo_.data_ = unsigned_b;
+}
+
+void RelocIterator::next() {
+ DCHECK(!done());
+ // Basically, do the opposite of RelocInfoWriter::Write.
+ // Reading of data is as far as possible avoided for unwanted modes,
+ // but we must always update the pc.
+ //
+ // We exit this loop by returning when we find a mode we want.
+ while (pos_ > end_) {
+ int tag = AdvanceGetTag();
+ if (tag == kEmbeddedObjectTag) {
+ ReadShortTaggedPC();
+ if (SetMode(RelocInfo::FULL_EMBEDDED_OBJECT)) return;
+ } else if (tag == kCodeTargetTag) {
+ ReadShortTaggedPC();
+ if (SetMode(RelocInfo::CODE_TARGET)) return;
+ } else if (tag == kWasmStubCallTag) {
+ ReadShortTaggedPC();
+ if (SetMode(RelocInfo::WASM_STUB_CALL)) return;
+ } else {
+ DCHECK_EQ(tag, kDefaultTag);
+ RelocInfo::Mode rmode = GetMode();
+ if (rmode == RelocInfo::PC_JUMP) {
+ AdvanceReadLongPCJump();
+ } else {
+ AdvanceReadPC();
+ if (RelocInfo::IsDeoptReason(rmode)) {
+ Advance();
+ if (SetMode(rmode)) {
+ ReadShortData();
+ return;
+ }
+ } else if (RelocInfo::IsConstPool(rmode) ||
+ RelocInfo::IsVeneerPool(rmode) ||
+ RelocInfo::IsDeoptId(rmode) ||
+ RelocInfo::IsDeoptPosition(rmode)) {
+ if (SetMode(rmode)) {
+ AdvanceReadInt();
+ return;
+ }
+ Advance(kIntSize);
+ } else if (SetMode(static_cast<RelocInfo::Mode>(rmode))) {
+ return;
+ }
+ }
+ }
+ }
+ done_ = true;
+}
+
+RelocIterator::RelocIterator(Code code, int mode_mask)
+ : RelocIterator(code, code.unchecked_relocation_info(), mode_mask) {}
+
+RelocIterator::RelocIterator(Code code, ByteArray relocation_info,
+ int mode_mask)
+ : RelocIterator(code, code.raw_instruction_start(), code.constant_pool(),
+ relocation_info.GetDataEndAddress(),
+ relocation_info.GetDataStartAddress(), mode_mask) {}
+
+RelocIterator::RelocIterator(const CodeReference code_reference, int mode_mask)
+ : RelocIterator(Code(), code_reference.instruction_start(),
+ code_reference.constant_pool(),
+ code_reference.relocation_end(),
+ code_reference.relocation_start(), mode_mask) {}
+
+RelocIterator::RelocIterator(EmbeddedData* embedded_data, Code code,
+ int mode_mask)
+ : RelocIterator(
+ code, embedded_data->InstructionStartOfBuiltin(code.builtin_index()),
+ code.constant_pool(),
+ code.relocation_start() + code.relocation_size(),
+ code.relocation_start(), mode_mask) {}
+
+RelocIterator::RelocIterator(const CodeDesc& desc, int mode_mask)
+ : RelocIterator(Code(), reinterpret_cast<Address>(desc.buffer), 0,
+ desc.buffer + desc.buffer_size,
+ desc.buffer + desc.buffer_size - desc.reloc_size,
+ mode_mask) {}
+
+RelocIterator::RelocIterator(Vector<byte> instructions,
+ Vector<const byte> reloc_info, Address const_pool,
+ int mode_mask)
+ : RelocIterator(Code(), reinterpret_cast<Address>(instructions.begin()),
+ const_pool, reloc_info.begin() + reloc_info.size(),
+ reloc_info.begin(), mode_mask) {}
+
+RelocIterator::RelocIterator(Code host, Address pc, Address constant_pool,
+ const byte* pos, const byte* end, int mode_mask)
+ : pos_(pos), end_(end), mode_mask_(mode_mask) {
+ // Relocation info is read backwards.
+ DCHECK_GE(pos_, end_);
+ rinfo_.host_ = host;
+ rinfo_.pc_ = pc;
+ rinfo_.constant_pool_ = constant_pool;
+ if (mode_mask_ == 0) pos_ = end_;
+ next();
+}
+
+// -----------------------------------------------------------------------------
+// Implementation of RelocInfo
+
+// static
+bool RelocInfo::OffHeapTargetIsCodedSpecially() {
+#if defined(V8_TARGET_ARCH_ARM) || defined(V8_TARGET_ARCH_ARM64) || \
+ defined(V8_TARGET_ARCH_X64)
+ return false;
+#elif defined(V8_TARGET_ARCH_IA32) || defined(V8_TARGET_ARCH_MIPS) || \
+ defined(V8_TARGET_ARCH_MIPS64) || defined(V8_TARGET_ARCH_PPC) || \
+ defined(V8_TARGET_ARCH_PPC64) || defined(V8_TARGET_ARCH_S390)
+ return true;
+#endif
+}
+
+Address RelocInfo::wasm_call_address() const {
+ DCHECK_EQ(rmode_, WASM_CALL);
+ return Assembler::target_address_at(pc_, constant_pool_);
+}
+
+void RelocInfo::set_wasm_call_address(Address address,
+ ICacheFlushMode icache_flush_mode) {
+ DCHECK_EQ(rmode_, WASM_CALL);
+ Assembler::set_target_address_at(pc_, constant_pool_, address,
+ icache_flush_mode);
+}
+
+Address RelocInfo::wasm_stub_call_address() const {
+ DCHECK_EQ(rmode_, WASM_STUB_CALL);
+ return Assembler::target_address_at(pc_, constant_pool_);
+}
+
+void RelocInfo::set_wasm_stub_call_address(Address address,
+ ICacheFlushMode icache_flush_mode) {
+ DCHECK_EQ(rmode_, WASM_STUB_CALL);
+ Assembler::set_target_address_at(pc_, constant_pool_, address,
+ icache_flush_mode);
+}
+
+void RelocInfo::set_target_address(Address target,
+ WriteBarrierMode write_barrier_mode,
+ ICacheFlushMode icache_flush_mode) {
+ DCHECK(IsCodeTargetMode(rmode_) || IsRuntimeEntry(rmode_) ||
+ IsWasmCall(rmode_));
+ Assembler::set_target_address_at(pc_, constant_pool_, target,
+ icache_flush_mode);
+ if (write_barrier_mode == UPDATE_WRITE_BARRIER && !host().is_null() &&
+ IsCodeTargetMode(rmode_) && !FLAG_disable_write_barriers) {
+ Code target_code = Code::GetCodeFromTargetAddress(target);
+ WriteBarrier::Marking(host(), this, target_code);
+ }
+}
+
+bool RelocInfo::HasTargetAddressAddress() const {
+ // TODO(jgruber): Investigate whether WASM_CALL is still appropriate on
+ // non-intel platforms now that wasm code is no longer on the heap.
+#if defined(V8_TARGET_ARCH_IA32) || defined(V8_TARGET_ARCH_X64)
+ static constexpr int kTargetAddressAddressModeMask =
+ ModeMask(CODE_TARGET) | ModeMask(FULL_EMBEDDED_OBJECT) |
+ ModeMask(COMPRESSED_EMBEDDED_OBJECT) | ModeMask(EXTERNAL_REFERENCE) |
+ ModeMask(OFF_HEAP_TARGET) | ModeMask(RUNTIME_ENTRY) |
+ ModeMask(WASM_CALL) | ModeMask(WASM_STUB_CALL);
+#else
+ static constexpr int kTargetAddressAddressModeMask =
+ ModeMask(CODE_TARGET) | ModeMask(RELATIVE_CODE_TARGET) |
+ ModeMask(FULL_EMBEDDED_OBJECT) | ModeMask(EXTERNAL_REFERENCE) |
+ ModeMask(OFF_HEAP_TARGET) | ModeMask(RUNTIME_ENTRY) | ModeMask(WASM_CALL);
+#endif
+ return (ModeMask(rmode_) & kTargetAddressAddressModeMask) != 0;
+}
+
+bool RelocInfo::RequiresRelocationAfterCodegen(const CodeDesc& desc) {
+ RelocIterator it(desc, RelocInfo::PostCodegenRelocationMask());
+ return !it.done();
+}
+
+bool RelocInfo::RequiresRelocation(Code code) {
+ RelocIterator it(code, RelocInfo::kApplyMask);
+ return !it.done();
+}
+
+#ifdef ENABLE_DISASSEMBLER
+const char* RelocInfo::RelocModeName(RelocInfo::Mode rmode) {
+ switch (rmode) {
+ case NONE:
+ return "no reloc";
+ case COMPRESSED_EMBEDDED_OBJECT:
+ return "compressed embedded object";
+ case FULL_EMBEDDED_OBJECT:
+ return "full embedded object";
+ case CODE_TARGET:
+ return "code target";
+ case RELATIVE_CODE_TARGET:
+ return "relative code target";
+ case RUNTIME_ENTRY:
+ return "runtime entry";
+ case EXTERNAL_REFERENCE:
+ return "external reference";
+ case INTERNAL_REFERENCE:
+ return "internal reference";
+ case INTERNAL_REFERENCE_ENCODED:
+ return "encoded internal reference";
+ case OFF_HEAP_TARGET:
+ return "off heap target";
+ case DEOPT_SCRIPT_OFFSET:
+ return "deopt script offset";
+ case DEOPT_INLINING_ID:
+ return "deopt inlining id";
+ case DEOPT_REASON:
+ return "deopt reason";
+ case DEOPT_ID:
+ return "deopt index";
+ case CONST_POOL:
+ return "constant pool";
+ case VENEER_POOL:
+ return "veneer pool";
+ case WASM_CALL:
+ return "internal wasm call";
+ case WASM_STUB_CALL:
+ return "wasm stub call";
+ case NUMBER_OF_MODES:
+ case PC_JUMP:
+ UNREACHABLE();
+ }
+ return "unknown relocation type";
+}
+
+void RelocInfo::Print(Isolate* isolate, std::ostream& os) { // NOLINT
+ os << reinterpret_cast<const void*>(pc_) << " " << RelocModeName(rmode_);
+ if (rmode_ == DEOPT_SCRIPT_OFFSET || rmode_ == DEOPT_INLINING_ID) {
+ os << " (" << data() << ")";
+ } else if (rmode_ == DEOPT_REASON) {
+ os << " ("
+ << DeoptimizeReasonToString(static_cast<DeoptimizeReason>(data_)) << ")";
+ } else if (rmode_ == FULL_EMBEDDED_OBJECT) {
+ os << " (" << Brief(target_object()) << ")";
+ } else if (rmode_ == COMPRESSED_EMBEDDED_OBJECT) {
+ os << " (" << Brief(target_object()) << " compressed)";
+ } else if (rmode_ == EXTERNAL_REFERENCE) {
+ if (isolate) {
+ ExternalReferenceEncoder ref_encoder(isolate);
+ os << " ("
+ << ref_encoder.NameOfAddress(isolate, target_external_reference())
+ << ") ";
+ }
+ os << " (" << reinterpret_cast<const void*>(target_external_reference())
+ << ")";
+ } else if (IsCodeTargetMode(rmode_)) {
+ const Address code_target = target_address();
+ Code code = Code::GetCodeFromTargetAddress(code_target);
+ DCHECK(code.IsCode());
+ os << " (" << CodeKindToString(code.kind());
+ if (Builtins::IsBuiltin(code)) {
+ os << " " << Builtins::name(code.builtin_index());
+ }
+ os << ") (" << reinterpret_cast<const void*>(target_address()) << ")";
+ } else if (IsRuntimeEntry(rmode_)) {
+ // Deoptimization bailouts are stored as runtime entries.
+ DeoptimizeKind type;
+ if (Deoptimizer::IsDeoptimizationEntry(isolate, target_address(), &type)) {
+ os << " (" << Deoptimizer::MessageFor(type, false)
+ << " deoptimization bailout)";
+ }
+ } else if (IsConstPool(rmode_)) {
+ os << " (size " << static_cast<int>(data_) << ")";
+ }
+
+ os << "\n";
+}
+#endif // ENABLE_DISASSEMBLER
+
+#ifdef VERIFY_HEAP
+void RelocInfo::Verify(Isolate* isolate) {
+ switch (rmode_) {
+ case COMPRESSED_EMBEDDED_OBJECT:
+ case FULL_EMBEDDED_OBJECT:
+ Object::VerifyPointer(isolate, target_object());
+ break;
+ case CODE_TARGET:
+ case RELATIVE_CODE_TARGET: {
+ // convert inline target address to code object
+ Address addr = target_address();
+ CHECK_NE(addr, kNullAddress);
+ // Check that we can find the right code object.
+ Code code = Code::GetCodeFromTargetAddress(addr);
+ Object found = isolate->FindCodeObject(addr);
+ CHECK(found.IsCode());
+ CHECK(code.address() == HeapObject::cast(found).address());
+ break;
+ }
+ case INTERNAL_REFERENCE:
+ case INTERNAL_REFERENCE_ENCODED: {
+ Address target = target_internal_reference();
+ Address pc = target_internal_reference_address();
+ Code code = Code::cast(isolate->FindCodeObject(pc));
+ CHECK(target >= code.InstructionStart());
+ CHECK(target <= code.InstructionEnd());
+ break;
+ }
+ case OFF_HEAP_TARGET: {
+ Address addr = target_off_heap_target();
+ CHECK_NE(addr, kNullAddress);
+ CHECK(!InstructionStream::TryLookupCode(isolate, addr).is_null());
+ break;
+ }
+ case RUNTIME_ENTRY:
+ case EXTERNAL_REFERENCE:
+ case DEOPT_SCRIPT_OFFSET:
+ case DEOPT_INLINING_ID:
+ case DEOPT_REASON:
+ case DEOPT_ID:
+ case CONST_POOL:
+ case VENEER_POOL:
+ case WASM_CALL:
+ case WASM_STUB_CALL:
+ case NONE:
+ break;
+ case NUMBER_OF_MODES:
+ case PC_JUMP:
+ UNREACHABLE();
+ }
+}
+#endif // VERIFY_HEAP
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/reloc-info.h b/src/codegen/reloc-info.h
new file mode 100644
index 0000000..e6d7dfd
--- /dev/null
+++ b/src/codegen/reloc-info.h
@@ -0,0 +1,477 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_RELOC_INFO_H_
+#define V8_CODEGEN_RELOC_INFO_H_
+
+#include "src/codegen/flush-instruction-cache.h"
+#include "src/common/globals.h"
+#include "src/objects/code.h"
+
+namespace v8 {
+namespace internal {
+
+class CodeReference;
+class EmbeddedData;
+
+// Specifies whether to perform icache flush operations on RelocInfo updates.
+// If FLUSH_ICACHE_IF_NEEDED, the icache will always be flushed if an
+// instruction was modified. If SKIP_ICACHE_FLUSH the flush will always be
+// skipped (only use this if you will flush the icache manually before it is
+// executed).
+enum ICacheFlushMode { FLUSH_ICACHE_IF_NEEDED, SKIP_ICACHE_FLUSH };
+
+// -----------------------------------------------------------------------------
+// Relocation information
+
+// Relocation information consists of the address (pc) of the datum
+// to which the relocation information applies, the relocation mode
+// (rmode), and an optional data field. The relocation mode may be
+// "descriptive" and not indicate a need for relocation, but simply
+// describe a property of the datum. Such rmodes are useful for GC
+// and nice disassembly output.
+
+class RelocInfo {
+ public:
+ // This string is used to add padding comments to the reloc info in cases
+ // where we are not sure to have enough space for patching in during
+ // lazy deoptimization. This is the case if we have indirect calls for which
+ // we do not normally record relocation info.
+ static const char* const kFillerCommentString;
+
+ // The minimum size of a comment is equal to two bytes for the extra tagged
+ // pc and kSystemPointerSize for the actual pointer to the comment.
+ static const int kMinRelocCommentSize = 2 + kSystemPointerSize;
+
+ // The maximum size for a call instruction including pc-jump.
+ static const int kMaxCallSize = 6;
+
+ // The maximum pc delta that will use the short encoding.
+ static const int kMaxSmallPCDelta;
+
+ enum Mode : int8_t {
+ // Please note the order is important (see IsRealRelocMode, IsGCRelocMode,
+ // and IsShareableRelocMode predicates below).
+
+ NONE, // Never recorded value. Most common one, hence value 0.
+
+ CODE_TARGET,
+ RELATIVE_CODE_TARGET, // LAST_CODE_TARGET_MODE
+ COMPRESSED_EMBEDDED_OBJECT,
+ FULL_EMBEDDED_OBJECT, // LAST_GCED_ENUM
+
+ WASM_CALL, // FIRST_SHAREABLE_RELOC_MODE
+ WASM_STUB_CALL,
+
+ RUNTIME_ENTRY,
+
+ EXTERNAL_REFERENCE, // The address of an external C++ function.
+ INTERNAL_REFERENCE, // An address inside the same function.
+
+ // Encoded internal reference, used only on MIPS, MIPS64 and PPC.
+ INTERNAL_REFERENCE_ENCODED,
+
+ // An off-heap instruction stream target. See http://goo.gl/Z2HUiM.
+ OFF_HEAP_TARGET,
+
+ // Marks constant and veneer pools. Only used on ARM and ARM64.
+ // They use a custom noncompact encoding.
+ CONST_POOL,
+ VENEER_POOL,
+
+ DEOPT_SCRIPT_OFFSET,
+ DEOPT_INLINING_ID, // Deoptimization source position.
+ DEOPT_REASON, // Deoptimization reason index.
+ DEOPT_ID, // Deoptimization inlining id.
+
+ // This is not an actual reloc mode, but used to encode a long pc jump that
+ // cannot be encoded as part of another record.
+ PC_JUMP,
+
+ // Pseudo-types
+ NUMBER_OF_MODES,
+
+ LAST_CODE_TARGET_MODE = RELATIVE_CODE_TARGET,
+ FIRST_REAL_RELOC_MODE = CODE_TARGET,
+ LAST_REAL_RELOC_MODE = VENEER_POOL,
+ FIRST_EMBEDDED_OBJECT_RELOC_MODE = COMPRESSED_EMBEDDED_OBJECT,
+ LAST_EMBEDDED_OBJECT_RELOC_MODE = FULL_EMBEDDED_OBJECT,
+ LAST_GCED_ENUM = LAST_EMBEDDED_OBJECT_RELOC_MODE,
+ FIRST_SHAREABLE_RELOC_MODE = WASM_CALL,
+ };
+
+ STATIC_ASSERT(NUMBER_OF_MODES <= kBitsPerInt);
+
+ RelocInfo() = default;
+
+ RelocInfo(Address pc, Mode rmode, intptr_t data, Code host,
+ Address constant_pool = kNullAddress)
+ : pc_(pc),
+ rmode_(rmode),
+ data_(data),
+ host_(host),
+ constant_pool_(constant_pool) {
+ DCHECK_IMPLIES(!COMPRESS_POINTERS_BOOL,
+ rmode != COMPRESSED_EMBEDDED_OBJECT);
+ }
+
+ static constexpr bool IsRealRelocMode(Mode mode) {
+ return mode >= FIRST_REAL_RELOC_MODE && mode <= LAST_REAL_RELOC_MODE;
+ }
+ // Is the relocation mode affected by GC?
+ static constexpr bool IsGCRelocMode(Mode mode) {
+ return mode <= LAST_GCED_ENUM;
+ }
+ static constexpr bool IsShareableRelocMode(Mode mode) {
+ return mode == RelocInfo::NONE ||
+ mode >= RelocInfo::FIRST_SHAREABLE_RELOC_MODE;
+ }
+ static constexpr bool IsCodeTarget(Mode mode) { return mode == CODE_TARGET; }
+ static constexpr bool IsCodeTargetMode(Mode mode) {
+ return mode <= LAST_CODE_TARGET_MODE;
+ }
+ static constexpr bool IsRelativeCodeTarget(Mode mode) {
+ return mode == RELATIVE_CODE_TARGET;
+ }
+ static constexpr bool IsFullEmbeddedObject(Mode mode) {
+ return mode == FULL_EMBEDDED_OBJECT;
+ }
+ static constexpr bool IsCompressedEmbeddedObject(Mode mode) {
+ return COMPRESS_POINTERS_BOOL && mode == COMPRESSED_EMBEDDED_OBJECT;
+ }
+ static constexpr bool IsEmbeddedObjectMode(Mode mode) {
+ return base::IsInRange(mode, FIRST_EMBEDDED_OBJECT_RELOC_MODE,
+ LAST_EMBEDDED_OBJECT_RELOC_MODE);
+ }
+ static constexpr bool IsRuntimeEntry(Mode mode) {
+ return mode == RUNTIME_ENTRY;
+ }
+ static constexpr bool IsWasmCall(Mode mode) { return mode == WASM_CALL; }
+ static constexpr bool IsWasmReference(Mode mode) { return mode == WASM_CALL; }
+ static constexpr bool IsWasmStubCall(Mode mode) {
+ return mode == WASM_STUB_CALL;
+ }
+ static constexpr bool IsConstPool(Mode mode) { return mode == CONST_POOL; }
+ static constexpr bool IsVeneerPool(Mode mode) { return mode == VENEER_POOL; }
+ static constexpr bool IsDeoptPosition(Mode mode) {
+ return mode == DEOPT_SCRIPT_OFFSET || mode == DEOPT_INLINING_ID;
+ }
+ static constexpr bool IsDeoptReason(Mode mode) {
+ return mode == DEOPT_REASON;
+ }
+ static constexpr bool IsDeoptId(Mode mode) { return mode == DEOPT_ID; }
+ static constexpr bool IsExternalReference(Mode mode) {
+ return mode == EXTERNAL_REFERENCE;
+ }
+ static constexpr bool IsInternalReference(Mode mode) {
+ return mode == INTERNAL_REFERENCE;
+ }
+ static constexpr bool IsInternalReferenceEncoded(Mode mode) {
+ return mode == INTERNAL_REFERENCE_ENCODED;
+ }
+ static constexpr bool IsOffHeapTarget(Mode mode) {
+ return mode == OFF_HEAP_TARGET;
+ }
+ static constexpr bool IsNone(Mode mode) { return mode == NONE; }
+
+ static bool IsOnlyForSerializer(Mode mode) {
+#ifdef V8_TARGET_ARCH_IA32
+ // On ia32, inlined off-heap trampolines must be relocated.
+ DCHECK_NE((kApplyMask & ModeMask(OFF_HEAP_TARGET)), 0);
+ DCHECK_EQ((kApplyMask & ModeMask(EXTERNAL_REFERENCE)), 0);
+ return mode == EXTERNAL_REFERENCE;
+#else
+ DCHECK_EQ((kApplyMask & ModeMask(OFF_HEAP_TARGET)), 0);
+ DCHECK_EQ((kApplyMask & ModeMask(EXTERNAL_REFERENCE)), 0);
+ return mode == EXTERNAL_REFERENCE || mode == OFF_HEAP_TARGET;
+#endif
+ }
+
+ static constexpr int ModeMask(Mode mode) { return 1 << mode; }
+
+ // Accessors
+ Address pc() const { return pc_; }
+ Mode rmode() const { return rmode_; }
+ intptr_t data() const { return data_; }
+ Code host() const { return host_; }
+ Address constant_pool() const { return constant_pool_; }
+
+ // Apply a relocation by delta bytes. When the code object is moved, PC
+ // relative addresses have to be updated as well as absolute addresses
+ // inside the code (internal references).
+ // Do not forget to flush the icache afterwards!
+ V8_INLINE void apply(intptr_t delta);
+
+ // Is the pointer this relocation info refers to coded like a plain pointer
+ // or is it strange in some way (e.g. relative or patched into a series of
+ // instructions).
+ bool IsCodedSpecially();
+
+ // The static pendant to IsCodedSpecially, just for off-heap targets. Used
+ // during deserialization, when we don't actually have a RelocInfo handy.
+ static bool OffHeapTargetIsCodedSpecially();
+
+ // If true, the pointer this relocation info refers to is an entry in the
+ // constant pool, otherwise the pointer is embedded in the instruction stream.
+ bool IsInConstantPool();
+
+ Address wasm_call_address() const;
+ Address wasm_stub_call_address() const;
+
+ uint32_t wasm_call_tag() const;
+
+ void set_wasm_call_address(
+ Address, ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED);
+ void set_wasm_stub_call_address(
+ Address, ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED);
+
+ void set_target_address(
+ Address target,
+ WriteBarrierMode write_barrier_mode = UPDATE_WRITE_BARRIER,
+ ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED);
+
+ // this relocation applies to;
+ // can only be called if IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_)
+ V8_INLINE Address target_address();
+ V8_INLINE HeapObject target_object();
+
+ // In GC operations, we don't have a host_ pointer. Retrieving a target
+ // for COMPRESSED_EMBEDDED_OBJECT mode requires an isolate.
+ V8_INLINE HeapObject target_object_no_host(Isolate* isolate);
+ V8_INLINE Handle<HeapObject> target_object_handle(Assembler* origin);
+
+ V8_INLINE void set_target_object(
+ Heap* heap, HeapObject target,
+ WriteBarrierMode write_barrier_mode = UPDATE_WRITE_BARRIER,
+ ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED);
+ V8_INLINE Address target_runtime_entry(Assembler* origin);
+ V8_INLINE void set_target_runtime_entry(
+ Address target,
+ WriteBarrierMode write_barrier_mode = UPDATE_WRITE_BARRIER,
+ ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED);
+ V8_INLINE Address target_off_heap_target();
+ V8_INLINE void set_target_external_reference(
+ Address, ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED);
+
+ // Returns the address of the constant pool entry where the target address
+ // is held. This should only be called if IsInConstantPool returns true.
+ V8_INLINE Address constant_pool_entry_address();
+
+ // Read the address of the word containing the target_address in an
+ // instruction stream. What this means exactly is architecture-independent.
+ // The only architecture-independent user of this function is the serializer.
+ // The serializer uses it to find out how many raw bytes of instruction to
+ // output before the next target. Architecture-independent code shouldn't
+ // dereference the pointer it gets back from this.
+ V8_INLINE Address target_address_address();
+ bool HasTargetAddressAddress() const;
+
+ // This indicates how much space a target takes up when deserializing a code
+ // stream. For most architectures this is just the size of a pointer. For
+ // an instruction like movw/movt where the target bits are mixed into the
+ // instruction bits the size of the target will be zero, indicating that the
+ // serializer should not step forwards in memory after a target is resolved
+ // and written. In this case the target_address_address function above
+ // should return the end of the instructions to be patched, allowing the
+ // deserializer to deserialize the instructions as raw bytes and put them in
+ // place, ready to be patched with the target.
+ V8_INLINE int target_address_size();
+
+ // Read the reference in the instruction this relocation
+ // applies to; can only be called if rmode_ is EXTERNAL_REFERENCE.
+ V8_INLINE Address target_external_reference();
+
+ // Read the reference in the instruction this relocation
+ // applies to; can only be called if rmode_ is INTERNAL_REFERENCE.
+ V8_INLINE Address target_internal_reference();
+
+ // Return the reference address this relocation applies to;
+ // can only be called if rmode_ is INTERNAL_REFERENCE.
+ V8_INLINE Address target_internal_reference_address();
+
+ // Wipe out a relocation to a fixed value, used for making snapshots
+ // reproducible.
+ V8_INLINE void WipeOut();
+
+ template <typename ObjectVisitor>
+ void Visit(ObjectVisitor* visitor) {
+ Mode mode = rmode();
+ if (IsEmbeddedObjectMode(mode)) {
+ visitor->VisitEmbeddedPointer(host(), this);
+ } else if (IsCodeTargetMode(mode)) {
+ visitor->VisitCodeTarget(host(), this);
+ } else if (IsExternalReference(mode)) {
+ visitor->VisitExternalReference(host(), this);
+ } else if (IsInternalReference(mode) || IsInternalReferenceEncoded(mode)) {
+ visitor->VisitInternalReference(host(), this);
+ } else if (IsRuntimeEntry(mode)) {
+ visitor->VisitRuntimeEntry(host(), this);
+ } else if (IsOffHeapTarget(mode)) {
+ visitor->VisitOffHeapTarget(host(), this);
+ }
+ }
+
+ // Check whether the given code contains relocation information that
+ // either is position-relative or movable by the garbage collector.
+ static bool RequiresRelocationAfterCodegen(const CodeDesc& desc);
+ static bool RequiresRelocation(Code code);
+
+#ifdef ENABLE_DISASSEMBLER
+ // Printing
+ static const char* RelocModeName(Mode rmode);
+ void Print(Isolate* isolate, std::ostream& os); // NOLINT
+#endif // ENABLE_DISASSEMBLER
+#ifdef VERIFY_HEAP
+ void Verify(Isolate* isolate);
+#endif
+
+ static const int kApplyMask; // Modes affected by apply. Depends on arch.
+
+ static constexpr int AllRealModesMask() {
+ constexpr Mode kFirstUnrealRelocMode =
+ static_cast<Mode>(RelocInfo::LAST_REAL_RELOC_MODE + 1);
+ return (ModeMask(kFirstUnrealRelocMode) - 1) &
+ ~(ModeMask(RelocInfo::FIRST_REAL_RELOC_MODE) - 1);
+ }
+
+ static int EmbeddedObjectModeMask() {
+ return ModeMask(RelocInfo::FULL_EMBEDDED_OBJECT) |
+ ModeMask(RelocInfo::COMPRESSED_EMBEDDED_OBJECT);
+ }
+
+ // In addition to modes covered by the apply mask (which is applied at GC
+ // time, among others), this covers all modes that are relocated by
+ // Code::CopyFromNoFlush after code generation.
+ static int PostCodegenRelocationMask() {
+ return ModeMask(RelocInfo::CODE_TARGET) |
+ ModeMask(RelocInfo::COMPRESSED_EMBEDDED_OBJECT) |
+ ModeMask(RelocInfo::FULL_EMBEDDED_OBJECT) |
+ ModeMask(RelocInfo::RUNTIME_ENTRY) |
+ ModeMask(RelocInfo::RELATIVE_CODE_TARGET) | kApplyMask;
+ }
+
+ private:
+ // On ARM/ARM64, note that pc_ is the address of the instruction referencing
+ // the constant pool and not the address of the constant pool entry.
+ Address pc_;
+ Mode rmode_;
+ intptr_t data_ = 0;
+ Code host_;
+ Address constant_pool_ = kNullAddress;
+ friend class RelocIterator;
+};
+
+// RelocInfoWriter serializes a stream of relocation info. It writes towards
+// lower addresses.
+class RelocInfoWriter {
+ public:
+ RelocInfoWriter() : pos_(nullptr), last_pc_(nullptr) {}
+
+ byte* pos() const { return pos_; }
+ byte* last_pc() const { return last_pc_; }
+
+ void Write(const RelocInfo* rinfo);
+
+ // Update the state of the stream after reloc info buffer
+ // and/or code is moved while the stream is active.
+ void Reposition(byte* pos, byte* pc) {
+ pos_ = pos;
+ last_pc_ = pc;
+ }
+
+ // Max size (bytes) of a written RelocInfo. Longest encoding is
+ // ExtraTag, VariableLengthPCJump, ExtraTag, pc_delta, data_delta.
+ static constexpr int kMaxSize = 1 + 4 + 1 + 1 + kSystemPointerSize;
+
+ private:
+ inline uint32_t WriteLongPCJump(uint32_t pc_delta);
+
+ inline void WriteShortTaggedPC(uint32_t pc_delta, int tag);
+ inline void WriteShortData(intptr_t data_delta);
+
+ inline void WriteMode(RelocInfo::Mode rmode);
+ inline void WriteModeAndPC(uint32_t pc_delta, RelocInfo::Mode rmode);
+ inline void WriteIntData(int data_delta);
+ inline void WriteData(intptr_t data_delta);
+
+ byte* pos_;
+ byte* last_pc_;
+
+ DISALLOW_COPY_AND_ASSIGN(RelocInfoWriter);
+};
+
+// A RelocIterator iterates over relocation information.
+// Typical use:
+//
+// for (RelocIterator it(code); !it.done(); it.next()) {
+// // do something with it.rinfo() here
+// }
+//
+// A mask can be specified to skip unwanted modes.
+class V8_EXPORT_PRIVATE RelocIterator : public Malloced {
+ public:
+ // Create a new iterator positioned at
+ // the beginning of the reloc info.
+ // Relocation information with mode k is included in the
+ // iteration iff bit k of mode_mask is set.
+ explicit RelocIterator(Code code, int mode_mask = -1);
+ explicit RelocIterator(Code code, ByteArray relocation_info, int mode_mask);
+ explicit RelocIterator(EmbeddedData* embedded_data, Code code, int mode_mask);
+ explicit RelocIterator(const CodeDesc& desc, int mode_mask = -1);
+ explicit RelocIterator(const CodeReference code_reference,
+ int mode_mask = -1);
+ explicit RelocIterator(Vector<byte> instructions,
+ Vector<const byte> reloc_info, Address const_pool,
+ int mode_mask = -1);
+ RelocIterator(RelocIterator&&) V8_NOEXCEPT = default;
+
+ // Iteration
+ bool done() const { return done_; }
+ void next();
+
+ // Return pointer valid until next next().
+ RelocInfo* rinfo() {
+ DCHECK(!done());
+ return &rinfo_;
+ }
+
+ private:
+ RelocIterator(Code host, Address pc, Address constant_pool, const byte* pos,
+ const byte* end, int mode_mask);
+
+ // Advance* moves the position before/after reading.
+ // *Read* reads from current byte(s) into rinfo_.
+ // *Get* just reads and returns info on current byte.
+ void Advance(int bytes = 1) { pos_ -= bytes; }
+ int AdvanceGetTag();
+ RelocInfo::Mode GetMode();
+
+ void AdvanceReadLongPCJump();
+
+ void ReadShortTaggedPC();
+ void ReadShortData();
+
+ void AdvanceReadPC();
+ void AdvanceReadInt();
+ void AdvanceReadData();
+
+ // If the given mode is wanted, set it in rinfo_ and return true.
+ // Else return false. Used for efficiently skipping unwanted modes.
+ bool SetMode(RelocInfo::Mode mode) {
+ return (mode_mask_ & (1 << mode)) ? (rinfo_.rmode_ = mode, true) : false;
+ }
+
+ const byte* pos_;
+ const byte* end_;
+ RelocInfo rinfo_;
+ bool done_ = false;
+ const int mode_mask_;
+
+ DISALLOW_COPY_AND_ASSIGN(RelocIterator);
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_RELOC_INFO_H_
diff --git a/src/codegen/s390/assembler-s390-inl.h b/src/codegen/s390/assembler-s390-inl.h
new file mode 100644
index 0000000..e34cfa0
--- /dev/null
+++ b/src/codegen/s390/assembler-s390-inl.h
@@ -0,0 +1,393 @@
+// Copyright (c) 1994-2006 Sun Microsystems Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//
+// - Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// - Redistribution in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the
+// distribution.
+//
+// - Neither the name of Sun Microsystems or the names of contributors may
+// be used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+// COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+// OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// The original source code covered by the above license above has been modified
+// significantly by Google Inc.
+// Copyright 2014 the V8 project authors. All rights reserved.
+
+#ifndef V8_CODEGEN_S390_ASSEMBLER_S390_INL_H_
+#define V8_CODEGEN_S390_ASSEMBLER_S390_INL_H_
+
+#include "src/codegen/s390/assembler-s390.h"
+
+#include "src/codegen/assembler.h"
+#include "src/debug/debug.h"
+#include "src/objects/objects-inl.h"
+
+namespace v8 {
+namespace internal {
+
+bool CpuFeatures::SupportsOptimizer() { return true; }
+
+bool CpuFeatures::SupportsWasmSimd128() {
+ return CpuFeatures::IsSupported(VECTOR_ENHANCE_FACILITY_1);
+}
+
+void RelocInfo::apply(intptr_t delta) {
+ // Absolute code pointer inside code object moves with the code object.
+ if (IsInternalReference(rmode_)) {
+ // Jump table entry
+ Address target = Memory<Address>(pc_);
+ Memory<Address>(pc_) = target + delta;
+ } else if (IsCodeTarget(rmode_)) {
+ SixByteInstr instr =
+ Instruction::InstructionBits(reinterpret_cast<const byte*>(pc_));
+ int32_t dis = static_cast<int32_t>(instr & 0xFFFFFFFF) * 2 // halfwords
+ - static_cast<int32_t>(delta);
+ instr >>= 32; // Clear the 4-byte displacement field.
+ instr <<= 32;
+ instr |= static_cast<uint32_t>(dis / 2);
+ Instruction::SetInstructionBits<SixByteInstr>(reinterpret_cast<byte*>(pc_),
+ instr);
+ } else {
+ // mov sequence
+ DCHECK(IsInternalReferenceEncoded(rmode_));
+ Address target = Assembler::target_address_at(pc_, constant_pool_);
+ Assembler::set_target_address_at(pc_, constant_pool_, target + delta,
+ SKIP_ICACHE_FLUSH);
+ }
+}
+
+Address RelocInfo::target_internal_reference() {
+ if (IsInternalReference(rmode_)) {
+ // Jump table entry
+ return Memory<Address>(pc_);
+ } else {
+ // mov sequence
+ DCHECK(IsInternalReferenceEncoded(rmode_));
+ return Assembler::target_address_at(pc_, constant_pool_);
+ }
+}
+
+Address RelocInfo::target_internal_reference_address() {
+ DCHECK(IsInternalReference(rmode_) || IsInternalReferenceEncoded(rmode_));
+ return pc_;
+}
+
+Address RelocInfo::target_address() {
+ DCHECK(IsRelativeCodeTarget(rmode_) || IsCodeTarget(rmode_) ||
+ IsRuntimeEntry(rmode_) || IsWasmCall(rmode_));
+ return Assembler::target_address_at(pc_, constant_pool_);
+}
+
+Address RelocInfo::target_address_address() {
+ DCHECK(HasTargetAddressAddress());
+
+ // Read the address of the word containing the target_address in an
+ // instruction stream.
+ // The only architecture-independent user of this function is the serializer.
+ // The serializer uses it to find out how many raw bytes of instruction to
+ // output before the next target.
+ // For an instruction like LIS/ORI where the target bits are mixed into the
+ // instruction bits, the size of the target will be zero, indicating that the
+ // serializer should not step forward in memory after a target is resolved
+ // and written.
+ return pc_;
+}
+
+Address RelocInfo::constant_pool_entry_address() { UNREACHABLE(); }
+
+void Assembler::set_target_compressed_address_at(
+ Address pc, Address constant_pool, Tagged_t target,
+ ICacheFlushMode icache_flush_mode) {
+ Assembler::set_target_address_at(
+ pc, constant_pool, static_cast<Address>(target), icache_flush_mode);
+}
+
+int RelocInfo::target_address_size() {
+ if (IsCodedSpecially()) {
+ return Assembler::kSpecialTargetSize;
+ } else {
+ return kSystemPointerSize;
+ }
+}
+
+Tagged_t Assembler::target_compressed_address_at(Address pc,
+ Address constant_pool) {
+ return static_cast<Tagged_t>(target_address_at(pc, constant_pool));
+}
+
+Handle<Object> Assembler::code_target_object_handle_at(Address pc) {
+ SixByteInstr instr =
+ Instruction::InstructionBits(reinterpret_cast<const byte*>(pc));
+ int index = instr & 0xFFFFFFFF;
+ return GetCodeTarget(index);
+}
+
+HeapObject RelocInfo::target_object() {
+ DCHECK(IsCodeTarget(rmode_) || IsEmbeddedObjectMode(rmode_));
+ if (IsCompressedEmbeddedObject(rmode_)) {
+ return HeapObject::cast(Object(DecompressTaggedAny(
+ host_.address(),
+ Assembler::target_compressed_address_at(pc_, constant_pool_))));
+ } else {
+ return HeapObject::cast(
+ Object(Assembler::target_address_at(pc_, constant_pool_)));
+ }
+}
+
+HeapObject RelocInfo::target_object_no_host(Isolate* isolate) {
+ if (IsCompressedEmbeddedObject(rmode_)) {
+ return HeapObject::cast(Object(DecompressTaggedAny(
+ isolate,
+ Assembler::target_compressed_address_at(pc_, constant_pool_))));
+ } else {
+ return target_object();
+ }
+}
+
+Handle<HeapObject> Assembler::compressed_embedded_object_handle_at(
+ Address pc, Address const_pool) {
+ return GetEmbeddedObject(target_compressed_address_at(pc, const_pool));
+}
+
+Handle<HeapObject> RelocInfo::target_object_handle(Assembler* origin) {
+ DCHECK(IsRelativeCodeTarget(rmode_) || IsCodeTarget(rmode_) ||
+ IsEmbeddedObjectMode(rmode_));
+ if (IsCodeTarget(rmode_) || IsRelativeCodeTarget(rmode_)) {
+ return Handle<HeapObject>::cast(origin->code_target_object_handle_at(pc_));
+ } else {
+ if (IsCompressedEmbeddedObject(rmode_)) {
+ return origin->compressed_embedded_object_handle_at(pc_, constant_pool_);
+ }
+ return Handle<HeapObject>(reinterpret_cast<Address*>(
+ Assembler::target_address_at(pc_, constant_pool_)));
+ }
+}
+
+void RelocInfo::set_target_object(Heap* heap, HeapObject target,
+ WriteBarrierMode write_barrier_mode,
+ ICacheFlushMode icache_flush_mode) {
+ DCHECK(IsCodeTarget(rmode_) || IsEmbeddedObjectMode(rmode_));
+ if (IsCompressedEmbeddedObject(rmode_)) {
+ Assembler::set_target_compressed_address_at(
+ pc_, constant_pool_, CompressTagged(target.ptr()), icache_flush_mode);
+ } else {
+ DCHECK(IsFullEmbeddedObject(rmode_));
+ Assembler::set_target_address_at(pc_, constant_pool_, target.ptr(),
+ icache_flush_mode);
+ }
+ if (write_barrier_mode == UPDATE_WRITE_BARRIER && !host().is_null() &&
+ !FLAG_disable_write_barriers) {
+ WriteBarrierForCode(host(), this, target);
+ }
+}
+
+Address RelocInfo::target_external_reference() {
+ DCHECK(rmode_ == EXTERNAL_REFERENCE);
+ return Assembler::target_address_at(pc_, constant_pool_);
+}
+
+void RelocInfo::set_target_external_reference(
+ Address target, ICacheFlushMode icache_flush_mode) {
+ DCHECK(rmode_ == RelocInfo::EXTERNAL_REFERENCE);
+ Assembler::set_target_address_at(pc_, constant_pool_, target,
+ icache_flush_mode);
+}
+
+Address RelocInfo::target_runtime_entry(Assembler* origin) {
+ DCHECK(IsRuntimeEntry(rmode_));
+ return target_address();
+}
+
+Address RelocInfo::target_off_heap_target() {
+ DCHECK(IsOffHeapTarget(rmode_));
+ return Assembler::target_address_at(pc_, constant_pool_);
+}
+
+void RelocInfo::set_target_runtime_entry(Address target,
+ WriteBarrierMode write_barrier_mode,
+ ICacheFlushMode icache_flush_mode) {
+ DCHECK(IsRuntimeEntry(rmode_));
+ if (target_address() != target)
+ set_target_address(target, write_barrier_mode, icache_flush_mode);
+}
+
+void RelocInfo::WipeOut() {
+ DCHECK(IsEmbeddedObjectMode(rmode_) || IsCodeTarget(rmode_) ||
+ IsRuntimeEntry(rmode_) || IsExternalReference(rmode_) ||
+ IsInternalReference(rmode_) || IsInternalReferenceEncoded(rmode_) ||
+ IsOffHeapTarget(rmode_));
+ if (IsInternalReference(rmode_)) {
+ // Jump table entry
+ Memory<Address>(pc_) = kNullAddress;
+ } else if (IsCompressedEmbeddedObject(rmode_)) {
+ Assembler::set_target_compressed_address_at(pc_, constant_pool_,
+ kNullAddress);
+ } else if (IsInternalReferenceEncoded(rmode_) || IsOffHeapTarget(rmode_)) {
+ // mov sequence
+ // Currently used only by deserializer, no need to flush.
+ Assembler::set_target_address_at(pc_, constant_pool_, kNullAddress,
+ SKIP_ICACHE_FLUSH);
+ } else {
+ Assembler::set_target_address_at(pc_, constant_pool_, kNullAddress);
+ }
+}
+
+// Operand constructors
+Operand::Operand(Register rm) : rm_(rm), rmode_(RelocInfo::NONE) {}
+
+// Fetch the 32bit value from the FIXED_SEQUENCE IIHF / IILF
+Address Assembler::target_address_at(Address pc, Address constant_pool) {
+ // S390 Instruction!
+ // We want to check for instructions generated by Asm::mov()
+ Opcode op1 = Instruction::S390OpcodeValue(reinterpret_cast<const byte*>(pc));
+ SixByteInstr instr_1 =
+ Instruction::InstructionBits(reinterpret_cast<const byte*>(pc));
+
+ if (BRASL == op1 || BRCL == op1) {
+ int32_t dis = static_cast<int32_t>(instr_1 & 0xFFFFFFFF) * 2;
+ return pc + dis;
+ }
+
+#if V8_TARGET_ARCH_S390X
+ int instr1_length =
+ Instruction::InstructionLength(reinterpret_cast<const byte*>(pc));
+ Opcode op2 = Instruction::S390OpcodeValue(
+ reinterpret_cast<const byte*>(pc + instr1_length));
+ SixByteInstr instr_2 = Instruction::InstructionBits(
+ reinterpret_cast<const byte*>(pc + instr1_length));
+ // IIHF for hi_32, IILF for lo_32
+ if (IIHF == op1 && IILF == op2) {
+ return static_cast<Address>(((instr_1 & 0xFFFFFFFF) << 32) |
+ ((instr_2 & 0xFFFFFFFF)));
+ }
+#else
+ // IILF loads 32-bits
+ if (IILF == op1 || CFI == op1) {
+ return static_cast<Address>((instr_1 & 0xFFFFFFFF));
+ }
+#endif
+
+ UNIMPLEMENTED();
+ return 0;
+}
+
+// This sets the branch destination (which gets loaded at the call address).
+// This is for calls and branches within generated code. The serializer
+// has already deserialized the mov instructions etc.
+// There is a FIXED_SEQUENCE assumption here
+void Assembler::deserialization_set_special_target_at(
+ Address instruction_payload, Code code, Address target) {
+ set_target_address_at(instruction_payload,
+ !code.is_null() ? code.constant_pool() : kNullAddress,
+ target);
+}
+
+int Assembler::deserialization_special_target_size(
+ Address instruction_payload) {
+ return kSpecialTargetSize;
+}
+
+void Assembler::deserialization_set_target_internal_reference_at(
+ Address pc, Address target, RelocInfo::Mode mode) {
+ if (RelocInfo::IsInternalReferenceEncoded(mode)) {
+ set_target_address_at(pc, kNullAddress, target, SKIP_ICACHE_FLUSH);
+ } else {
+ Memory<Address>(pc) = target;
+ }
+}
+
+// This code assumes the FIXED_SEQUENCE of IIHF/IILF
+void Assembler::set_target_address_at(Address pc, Address constant_pool,
+ Address target,
+ ICacheFlushMode icache_flush_mode) {
+ // Check for instructions generated by Asm::mov()
+ Opcode op1 = Instruction::S390OpcodeValue(reinterpret_cast<const byte*>(pc));
+ SixByteInstr instr_1 =
+ Instruction::InstructionBits(reinterpret_cast<const byte*>(pc));
+ bool patched = false;
+
+ if (BRASL == op1 || BRCL == op1) {
+ instr_1 >>= 32; // Zero out the lower 32-bits
+ instr_1 <<= 32;
+ int32_t halfwords = (target - pc) / 2; // number of halfwords
+ instr_1 |= static_cast<uint32_t>(halfwords);
+ Instruction::SetInstructionBits<SixByteInstr>(reinterpret_cast<byte*>(pc),
+ instr_1);
+ if (icache_flush_mode != SKIP_ICACHE_FLUSH) {
+ FlushInstructionCache(pc, 6);
+ }
+ patched = true;
+ } else {
+#if V8_TARGET_ARCH_S390X
+ int instr1_length =
+ Instruction::InstructionLength(reinterpret_cast<const byte*>(pc));
+ Opcode op2 = Instruction::S390OpcodeValue(
+ reinterpret_cast<const byte*>(pc + instr1_length));
+ SixByteInstr instr_2 = Instruction::InstructionBits(
+ reinterpret_cast<const byte*>(pc + instr1_length));
+ // IIHF for hi_32, IILF for lo_32
+ if (IIHF == op1 && IILF == op2) {
+ // IIHF
+ instr_1 >>= 32; // Zero out the lower 32-bits
+ instr_1 <<= 32;
+ instr_1 |= reinterpret_cast<uint64_t>(target) >> 32;
+
+ Instruction::SetInstructionBits<SixByteInstr>(reinterpret_cast<byte*>(pc),
+ instr_1);
+
+ // IILF
+ instr_2 >>= 32;
+ instr_2 <<= 32;
+ instr_2 |= reinterpret_cast<uint64_t>(target) & 0xFFFFFFFF;
+
+ Instruction::SetInstructionBits<SixByteInstr>(
+ reinterpret_cast<byte*>(pc + instr1_length), instr_2);
+ if (icache_flush_mode != SKIP_ICACHE_FLUSH) {
+ FlushInstructionCache(pc, 12);
+ }
+ patched = true;
+ }
+#else
+ // IILF loads 32-bits
+ if (IILF == op1 || CFI == op1) {
+ instr_1 >>= 32; // Zero out the lower 32-bits
+ instr_1 <<= 32;
+ instr_1 |= reinterpret_cast<uint32_t>(target);
+
+ Instruction::SetInstructionBits<SixByteInstr>(reinterpret_cast<byte*>(pc),
+ instr_1);
+ if (icache_flush_mode != SKIP_ICACHE_FLUSH) {
+ FlushInstructionCache(pc, 6);
+ }
+ patched = true;
+ }
+#endif
+ }
+ if (!patched) UNREACHABLE();
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_S390_ASSEMBLER_S390_INL_H_
diff --git a/src/codegen/s390/assembler-s390.cc b/src/codegen/s390/assembler-s390.cc
new file mode 100644
index 0000000..2e74f02
--- /dev/null
+++ b/src/codegen/s390/assembler-s390.cc
@@ -0,0 +1,861 @@
+// Copyright (c) 1994-2006 Sun Microsystems Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//
+// - Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// - Redistribution in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the
+// distribution.
+//
+// - Neither the name of Sun Microsystems or the names of contributors may
+// be used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+// COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+// OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// The original source code covered by the above license above has been
+// modified significantly by Google Inc.
+// Copyright 2014 the V8 project authors. All rights reserved.
+
+#include "src/codegen/s390/assembler-s390.h"
+#include <set>
+#include <string>
+
+#if V8_TARGET_ARCH_S390
+
+#if V8_HOST_ARCH_S390
+#include <elf.h> // Required for auxv checks for STFLE support
+#include <sys/auxv.h>
+#endif
+
+#include "src/base/bits.h"
+#include "src/base/cpu.h"
+#include "src/codegen/macro-assembler.h"
+#include "src/codegen/s390/assembler-s390-inl.h"
+#include "src/codegen/string-constants.h"
+#include "src/deoptimizer/deoptimizer.h"
+
+namespace v8 {
+namespace internal {
+
+// Get the CPU features enabled by the build.
+static unsigned CpuFeaturesImpliedByCompiler() {
+ unsigned answer = 0;
+ return answer;
+}
+
+static bool supportsCPUFeature(const char* feature) {
+ static std::set<std::string>& features = *new std::set<std::string>();
+ static std::set<std::string>& all_available_features =
+ *new std::set<std::string>({"iesan3", "zarch", "stfle", "msa", "ldisp",
+ "eimm", "dfp", "etf3eh", "highgprs", "te",
+ "vx"});
+ if (features.empty()) {
+#if V8_HOST_ARCH_S390
+
+#ifndef HWCAP_S390_VX
+#define HWCAP_S390_VX 2048
+#endif
+#define CHECK_AVAILABILITY_FOR(mask, value) \
+ if (f & mask) features.insert(value);
+
+ // initialize feature vector
+ uint64_t f = getauxval(AT_HWCAP);
+ CHECK_AVAILABILITY_FOR(HWCAP_S390_ESAN3, "iesan3")
+ CHECK_AVAILABILITY_FOR(HWCAP_S390_ZARCH, "zarch")
+ CHECK_AVAILABILITY_FOR(HWCAP_S390_STFLE, "stfle")
+ CHECK_AVAILABILITY_FOR(HWCAP_S390_MSA, "msa")
+ CHECK_AVAILABILITY_FOR(HWCAP_S390_LDISP, "ldisp")
+ CHECK_AVAILABILITY_FOR(HWCAP_S390_EIMM, "eimm")
+ CHECK_AVAILABILITY_FOR(HWCAP_S390_DFP, "dfp")
+ CHECK_AVAILABILITY_FOR(HWCAP_S390_ETF3EH, "etf3eh")
+ CHECK_AVAILABILITY_FOR(HWCAP_S390_HIGH_GPRS, "highgprs")
+ CHECK_AVAILABILITY_FOR(HWCAP_S390_TE, "te")
+ CHECK_AVAILABILITY_FOR(HWCAP_S390_VX, "vx")
+#else
+ // import all features
+ features.insert(all_available_features.begin(),
+ all_available_features.end());
+#endif
+ }
+ USE(all_available_features);
+ return features.find(feature) != features.end();
+}
+
+#undef CHECK_AVAILABILITY_FOR
+#undef HWCAP_S390_VX
+
+// Check whether Store Facility STFLE instruction is available on the platform.
+// Instruction returns a bit vector of the enabled hardware facilities.
+static bool supportsSTFLE() {
+#if V8_HOST_ARCH_S390
+ static bool read_tried = false;
+ static uint32_t auxv_hwcap = 0;
+
+ if (!read_tried) {
+ // Open the AUXV (auxiliary vector) pseudo-file
+ int fd = open("/proc/self/auxv", O_RDONLY);
+
+ read_tried = true;
+ if (fd != -1) {
+#if V8_TARGET_ARCH_S390X
+ static Elf64_auxv_t buffer[16];
+ Elf64_auxv_t* auxv_element;
+#else
+ static Elf32_auxv_t buffer[16];
+ Elf32_auxv_t* auxv_element;
+#endif
+ int bytes_read = 0;
+ while (bytes_read >= 0) {
+ // Read a chunk of the AUXV
+ bytes_read = read(fd, buffer, sizeof(buffer));
+ // Locate and read the platform field of AUXV if it is in the chunk
+ for (auxv_element = buffer;
+ auxv_element + sizeof(auxv_element) <= buffer + bytes_read &&
+ auxv_element->a_type != AT_NULL;
+ auxv_element++) {
+ // We are looking for HWCAP entry in AUXV to search for STFLE support
+ if (auxv_element->a_type == AT_HWCAP) {
+ /* Note: Both auxv_hwcap and buffer are static */
+ auxv_hwcap = auxv_element->a_un.a_val;
+ goto done_reading;
+ }
+ }
+ }
+ done_reading:
+ close(fd);
+ }
+ }
+
+ // Did not find result
+ if (0 == auxv_hwcap) {
+ return false;
+ }
+
+ // HWCAP_S390_STFLE is defined to be 4 in include/asm/elf.h. Currently
+ // hardcoded in case that include file does not exist.
+ const uint32_t _HWCAP_S390_STFLE = 4;
+ return (auxv_hwcap & _HWCAP_S390_STFLE);
+#else
+ // STFLE is not available on non-s390 hosts
+ return false;
+#endif
+}
+
+void CpuFeatures::ProbeImpl(bool cross_compile) {
+ supported_ |= CpuFeaturesImpliedByCompiler();
+ icache_line_size_ = 256;
+
+ // Only use statically determined features for cross compile (snapshot).
+ if (cross_compile) return;
+
+#ifdef DEBUG
+ initialized_ = true;
+#endif
+
+ static bool performSTFLE = supportsSTFLE();
+
+// Need to define host, as we are generating inlined S390 assembly to test
+// for facilities.
+#if V8_HOST_ARCH_S390
+ if (performSTFLE) {
+ // STFLE D(B) requires:
+ // GPR0 to specify # of double words to update minus 1.
+ // i.e. GPR0 = 0 for 1 doubleword
+ // D(B) to specify to memory location to store the facilities bits
+ // The facilities we are checking for are:
+ // Bit 45 - Distinct Operands for instructions like ARK, SRK, etc.
+ // As such, we require only 1 double word
+ int64_t facilities[3] = {0L};
+ int16_t reg0;
+ // LHI sets up GPR0
+ // STFLE is specified as .insn, as opcode is not recognized.
+ // We register the instructions kill r0 (LHI) and the CC (STFLE).
+ asm volatile(
+ "lhi %%r0,2\n"
+ ".insn s,0xb2b00000,%0\n"
+ : "=Q"(facilities), "=r"(reg0)
+ :
+ : "cc", "r0");
+
+ uint64_t one = static_cast<uint64_t>(1);
+ // Test for Distinct Operands Facility - Bit 45
+ if (facilities[0] & (one << (63 - 45))) {
+ supported_ |= (1u << DISTINCT_OPS);
+ }
+ // Test for General Instruction Extension Facility - Bit 34
+ if (facilities[0] & (one << (63 - 34))) {
+ supported_ |= (1u << GENERAL_INSTR_EXT);
+ }
+ // Test for Floating Point Extension Facility - Bit 37
+ if (facilities[0] & (one << (63 - 37))) {
+ supported_ |= (1u << FLOATING_POINT_EXT);
+ }
+ // Test for Vector Facility - Bit 129
+ if (facilities[2] & (one << (63 - (129 - 128))) &&
+ supportsCPUFeature("vx")) {
+ supported_ |= (1u << VECTOR_FACILITY);
+ }
+ // Test for Vector Enhancement Facility 1 - Bit 135
+ if (facilities[2] & (one << (63 - (135 - 128))) &&
+ supportsCPUFeature("vx")) {
+ supported_ |= (1u << VECTOR_ENHANCE_FACILITY_1);
+ }
+ // Test for Vector Enhancement Facility 2 - Bit 148
+ if (facilities[2] & (one << (63 - (148 - 128))) &&
+ supportsCPUFeature("vx")) {
+ supported_ |= (1u << VECTOR_ENHANCE_FACILITY_2);
+ }
+ // Test for Miscellaneous Instruction Extension Facility - Bit 58
+ if (facilities[0] & (1lu << (63 - 58))) {
+ supported_ |= (1u << MISC_INSTR_EXT2);
+ }
+ }
+#else
+ // All distinct ops instructions can be simulated
+ supported_ |= (1u << DISTINCT_OPS);
+ // RISBG can be simulated
+ supported_ |= (1u << GENERAL_INSTR_EXT);
+ supported_ |= (1u << FLOATING_POINT_EXT);
+ supported_ |= (1u << MISC_INSTR_EXT2);
+ USE(performSTFLE); // To avoid assert
+ USE(supportsCPUFeature);
+ supported_ |= (1u << VECTOR_FACILITY);
+ supported_ |= (1u << VECTOR_ENHANCE_FACILITY_1);
+#endif
+ supported_ |= (1u << FPU);
+}
+
+void CpuFeatures::PrintTarget() {
+ const char* s390_arch = nullptr;
+
+#if V8_TARGET_ARCH_S390X
+ s390_arch = "s390x";
+#else
+ s390_arch = "s390";
+#endif
+
+ PrintF("target %s\n", s390_arch);
+}
+
+void CpuFeatures::PrintFeatures() {
+ PrintF("FPU=%d\n", CpuFeatures::IsSupported(FPU));
+ PrintF("FPU_EXT=%d\n", CpuFeatures::IsSupported(FLOATING_POINT_EXT));
+ PrintF("GENERAL_INSTR=%d\n", CpuFeatures::IsSupported(GENERAL_INSTR_EXT));
+ PrintF("DISTINCT_OPS=%d\n", CpuFeatures::IsSupported(DISTINCT_OPS));
+ PrintF("VECTOR_FACILITY=%d\n", CpuFeatures::IsSupported(VECTOR_FACILITY));
+ PrintF("VECTOR_ENHANCE_FACILITY_1=%d\n",
+ CpuFeatures::IsSupported(VECTOR_ENHANCE_FACILITY_1));
+ PrintF("VECTOR_ENHANCE_FACILITY_2=%d\n",
+ CpuFeatures::IsSupported(VECTOR_ENHANCE_FACILITY_2));
+ PrintF("MISC_INSTR_EXT2=%d\n", CpuFeatures::IsSupported(MISC_INSTR_EXT2));
+}
+
+Register ToRegister(int num) {
+ DCHECK(num >= 0 && num < kNumRegisters);
+ const Register kRegisters[] = {r0, r1, r2, r3, r4, r5, r6, r7,
+ r8, r9, r10, fp, ip, r13, r14, sp};
+ return kRegisters[num];
+}
+
+// -----------------------------------------------------------------------------
+// Implementation of RelocInfo
+
+const int RelocInfo::kApplyMask =
+ RelocInfo::ModeMask(RelocInfo::CODE_TARGET) |
+ RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE);
+
+bool RelocInfo::IsCodedSpecially() {
+ // The deserializer needs to know whether a pointer is specially
+ // coded. Being specially coded on S390 means that it is an iihf/iilf
+ // instruction sequence, and that is always the case inside code
+ // objects.
+ return true;
+}
+
+bool RelocInfo::IsInConstantPool() { return false; }
+
+uint32_t RelocInfo::wasm_call_tag() const {
+ DCHECK(rmode_ == WASM_CALL || rmode_ == WASM_STUB_CALL);
+ return static_cast<uint32_t>(
+ Assembler::target_address_at(pc_, constant_pool_));
+}
+
+// -----------------------------------------------------------------------------
+// Implementation of Operand and MemOperand
+// See assembler-s390-inl.h for inlined constructors
+
+Operand::Operand(Handle<HeapObject> handle) {
+ AllowHandleDereference using_location;
+ rm_ = no_reg;
+ value_.immediate = static_cast<intptr_t>(handle.address());
+ rmode_ = RelocInfo::FULL_EMBEDDED_OBJECT;
+}
+
+Operand Operand::EmbeddedNumber(double value) {
+ int32_t smi;
+ if (DoubleToSmiInteger(value, &smi)) return Operand(Smi::FromInt(smi));
+ Operand result(0, RelocInfo::FULL_EMBEDDED_OBJECT);
+ result.is_heap_object_request_ = true;
+ result.value_.heap_object_request = HeapObjectRequest(value);
+ return result;
+}
+
+Operand Operand::EmbeddedStringConstant(const StringConstantBase* str) {
+ Operand result(0, RelocInfo::FULL_EMBEDDED_OBJECT);
+ result.is_heap_object_request_ = true;
+ result.value_.heap_object_request = HeapObjectRequest(str);
+ return result;
+}
+
+MemOperand::MemOperand(Register rn, int32_t offset)
+ : baseRegister(rn), indexRegister(r0), offset_(offset) {}
+
+MemOperand::MemOperand(Register rx, Register rb, int32_t offset)
+ : baseRegister(rb), indexRegister(rx), offset_(offset) {}
+
+void Assembler::AllocateAndInstallRequestedHeapObjects(Isolate* isolate) {
+ DCHECK_IMPLIES(isolate == nullptr, heap_object_requests_.empty());
+ for (auto& request : heap_object_requests_) {
+ Handle<HeapObject> object;
+ Address pc = reinterpret_cast<Address>(buffer_start_) + request.offset();
+ switch (request.kind()) {
+ case HeapObjectRequest::kHeapNumber: {
+ object = isolate->factory()->NewHeapNumber<AllocationType::kOld>(
+ request.heap_number());
+ set_target_address_at(pc, kNullAddress, object.address(),
+ SKIP_ICACHE_FLUSH);
+ break;
+ }
+ case HeapObjectRequest::kStringConstant: {
+ const StringConstantBase* str = request.string();
+ CHECK_NOT_NULL(str);
+ set_target_address_at(pc, kNullAddress,
+ str->AllocateStringConstant(isolate).address());
+ break;
+ }
+ }
+ }
+}
+
+// -----------------------------------------------------------------------------
+// Specific instructions, constants, and masks.
+
+Assembler::Assembler(const AssemblerOptions& options,
+ std::unique_ptr<AssemblerBuffer> buffer)
+ : AssemblerBase(options, std::move(buffer)),
+ scratch_register_list_(ip.bit()) {
+ reloc_info_writer.Reposition(buffer_start_ + buffer_->size(), pc_);
+ last_bound_pos_ = 0;
+ relocations_.reserve(128);
+}
+
+void Assembler::GetCode(Isolate* isolate, CodeDesc* desc,
+ SafepointTableBuilder* safepoint_table_builder,
+ int handler_table_offset) {
+ // As a crutch to avoid having to add manual Align calls wherever we use a
+ // raw workflow to create Code objects (mostly in tests), add another Align
+ // call here. It does no harm - the end of the Code object is aligned to the
+ // (larger) kCodeAlignment anyways.
+ // TODO(jgruber): Consider moving responsibility for proper alignment to
+ // metadata table builders (safepoint, handler, constant pool, code
+ // comments).
+ DataAlign(Code::kMetadataAlignment);
+
+ EmitRelocations();
+
+ int code_comments_size = WriteCodeComments();
+
+ AllocateAndInstallRequestedHeapObjects(isolate);
+
+ // Set up code descriptor.
+ // TODO(jgruber): Reconsider how these offsets and sizes are maintained up to
+ // this point to make CodeDesc initialization less fiddly.
+
+ static constexpr int kConstantPoolSize = 0;
+ const int instruction_size = pc_offset();
+ const int code_comments_offset = instruction_size - code_comments_size;
+ const int constant_pool_offset = code_comments_offset - kConstantPoolSize;
+ const int handler_table_offset2 = (handler_table_offset == kNoHandlerTable)
+ ? constant_pool_offset
+ : handler_table_offset;
+ const int safepoint_table_offset =
+ (safepoint_table_builder == kNoSafepointTable)
+ ? handler_table_offset2
+ : safepoint_table_builder->GetCodeOffset();
+ const int reloc_info_offset =
+ static_cast<int>(reloc_info_writer.pos() - buffer_->start());
+ CodeDesc::Initialize(desc, this, safepoint_table_offset,
+ handler_table_offset2, constant_pool_offset,
+ code_comments_offset, reloc_info_offset);
+}
+
+void Assembler::Align(int m) {
+ DCHECK(m >= 4 && base::bits::IsPowerOfTwo(m));
+ while ((pc_offset() & (m - 1)) != 0) {
+ nop(0);
+ }
+}
+
+void Assembler::CodeTargetAlign() { Align(8); }
+
+Condition Assembler::GetCondition(Instr instr) {
+ switch (instr & kCondMask) {
+ case BT:
+ return eq;
+ case BF:
+ return ne;
+ default:
+ UNIMPLEMENTED();
+ }
+ return al;
+}
+
+#if V8_TARGET_ARCH_S390X
+// This code assumes a FIXED_SEQUENCE for 64bit loads (iihf/iilf)
+bool Assembler::Is64BitLoadIntoIP(SixByteInstr instr1, SixByteInstr instr2) {
+ // Check the instructions are the iihf/iilf load into ip
+ return (((instr1 >> 32) == 0xC0C8) && ((instr2 >> 32) == 0xC0C9));
+}
+#else
+// This code assumes a FIXED_SEQUENCE for 32bit loads (iilf)
+bool Assembler::Is32BitLoadIntoIP(SixByteInstr instr) {
+ // Check the instruction is an iilf load into ip/r12.
+ return ((instr >> 32) == 0xC0C9);
+}
+#endif
+
+// Labels refer to positions in the (to be) generated code.
+// There are bound, linked, and unused labels.
+//
+// Bound labels refer to known positions in the already
+// generated code. pos() is the position the label refers to.
+//
+// Linked labels refer to unknown positions in the code
+// to be generated; pos() is the position of the last
+// instruction using the label.
+
+// The link chain is terminated by a negative code position (must be aligned)
+const int kEndOfChain = -4;
+
+// Returns the target address of the relative instructions, typically
+// of the form: pos + imm (where immediate is in # of halfwords for
+// BR* and LARL).
+int Assembler::target_at(int pos) {
+ SixByteInstr instr = instr_at(pos);
+ // check which type of branch this is 16 or 26 bit offset
+ Opcode opcode = Instruction::S390OpcodeValue(buffer_start_ + pos);
+
+ if (BRC == opcode || BRCT == opcode || BRCTG == opcode || BRXH == opcode) {
+ int16_t imm16 = SIGN_EXT_IMM16((instr & kImm16Mask));
+ imm16 <<= 1; // immediate is in # of halfwords
+ if (imm16 == 0) return kEndOfChain;
+ return pos + imm16;
+ } else if (LLILF == opcode || BRCL == opcode || LARL == opcode ||
+ BRASL == opcode) {
+ int32_t imm32 =
+ static_cast<int32_t>(instr & (static_cast<uint64_t>(0xFFFFFFFF)));
+ if (LLILF != opcode)
+ imm32 <<= 1; // BR* + LARL treat immediate in # of halfwords
+ if (imm32 == 0) return kEndOfChain;
+ return pos + imm32;
+ } else if (BRXHG == opcode) {
+ // offset is in bits 16-31 of 48 bit instruction
+ instr = instr >> 16;
+ int16_t imm16 = SIGN_EXT_IMM16((instr & kImm16Mask));
+ imm16 <<= 1; // immediate is in # of halfwords
+ if (imm16 == 0) return kEndOfChain;
+ return pos + imm16;
+ }
+
+ // Unknown condition
+ DCHECK(false);
+ return -1;
+}
+
+// Update the target address of the current relative instruction.
+void Assembler::target_at_put(int pos, int target_pos, bool* is_branch) {
+ SixByteInstr instr = instr_at(pos);
+ Opcode opcode = Instruction::S390OpcodeValue(buffer_start_ + pos);
+
+ if (is_branch != nullptr) {
+ *is_branch =
+ (opcode == BRC || opcode == BRCT || opcode == BRCTG || opcode == BRCL ||
+ opcode == BRASL || opcode == BRXH || opcode == BRXHG);
+ }
+
+ if (BRC == opcode || BRCT == opcode || BRCTG == opcode || BRXH == opcode) {
+ int16_t imm16 = target_pos - pos;
+ instr &= (~0xFFFF);
+ DCHECK(is_int16(imm16));
+ instr_at_put<FourByteInstr>(pos, instr | (imm16 >> 1));
+ return;
+ } else if (BRCL == opcode || LARL == opcode || BRASL == opcode) {
+ // Immediate is in # of halfwords
+ int32_t imm32 = target_pos - pos;
+ instr &= (~static_cast<uint64_t>(0xFFFFFFFF));
+ instr_at_put<SixByteInstr>(pos, instr | (imm32 >> 1));
+ return;
+ } else if (LLILF == opcode) {
+ DCHECK(target_pos == kEndOfChain || target_pos >= 0);
+ // Emitted label constant, not part of a branch.
+ // Make label relative to Code pointer of generated Code object.
+ int32_t imm32 = target_pos + (Code::kHeaderSize - kHeapObjectTag);
+ instr &= (~static_cast<uint64_t>(0xFFFFFFFF));
+ instr_at_put<SixByteInstr>(pos, instr | imm32);
+ return;
+ } else if (BRXHG == opcode) {
+ // Immediate is in bits 16-31 of 48 bit instruction
+ int32_t imm16 = target_pos - pos;
+ instr &= (0xFFFF0000FFFF); // clear bits 16-31
+ imm16 &= 0xFFFF; // clear high halfword
+ imm16 <<= 16;
+ // Immediate is in # of halfwords
+ instr_at_put<SixByteInstr>(pos, instr | (imm16 >> 1));
+ return;
+ }
+ DCHECK(false);
+}
+
+// Returns the maximum number of bits given instruction can address.
+int Assembler::max_reach_from(int pos) {
+ Opcode opcode = Instruction::S390OpcodeValue(buffer_start_ + pos);
+ // Check which type of instr. In theory, we can return
+ // the values below + 1, given offset is # of halfwords
+ if (BRC == opcode || BRCT == opcode || BRCTG == opcode || BRXH == opcode ||
+ BRXHG == opcode) {
+ return 16;
+ } else if (LLILF == opcode || BRCL == opcode || LARL == opcode ||
+ BRASL == opcode) {
+ return 31; // Using 31 as workaround instead of 32 as
+ // is_intn(x,32) doesn't work on 32-bit platforms.
+ // llilf: Emitted label constant, not part of
+ // a branch (regexp PushBacktrack).
+ }
+ DCHECK(false);
+ return 16;
+}
+
+void Assembler::bind_to(Label* L, int pos) {
+ DCHECK(0 <= pos && pos <= pc_offset()); // must have a valid binding position
+ bool is_branch = false;
+ while (L->is_linked()) {
+ int fixup_pos = L->pos();
+#ifdef DEBUG
+ int32_t offset = pos - fixup_pos;
+ int maxReach = max_reach_from(fixup_pos);
+#endif
+ next(L); // call next before overwriting link with target at fixup_pos
+ DCHECK(is_intn(offset, maxReach));
+ target_at_put(fixup_pos, pos, &is_branch);
+ }
+ L->bind_to(pos);
+
+ // Keep track of the last bound label so we don't eliminate any instructions
+ // before a bound label.
+ if (pos > last_bound_pos_) last_bound_pos_ = pos;
+}
+
+void Assembler::bind(Label* L) {
+ DCHECK(!L->is_bound()); // label can only be bound once
+ bind_to(L, pc_offset());
+}
+
+void Assembler::next(Label* L) {
+ DCHECK(L->is_linked());
+ int link = target_at(L->pos());
+ if (link == kEndOfChain) {
+ L->Unuse();
+ } else {
+ DCHECK_GE(link, 0);
+ L->link_to(link);
+ }
+}
+
+int Assembler::link(Label* L) {
+ int position;
+ if (L->is_bound()) {
+ position = L->pos();
+ } else {
+ if (L->is_linked()) {
+ position = L->pos(); // L's link
+ } else {
+ // was: target_pos = kEndOfChain;
+ // However, using self to mark the first reference
+ // should avoid most instances of branch offset overflow. See
+ // target_at() for where this is converted back to kEndOfChain.
+ position = pc_offset();
+ }
+ L->link_to(pc_offset());
+ }
+
+ return position;
+}
+
+void Assembler::load_label_offset(Register r1, Label* L) {
+ int target_pos;
+ int constant;
+ if (L->is_bound()) {
+ target_pos = L->pos();
+ constant = target_pos + (Code::kHeaderSize - kHeapObjectTag);
+ } else {
+ if (L->is_linked()) {
+ target_pos = L->pos(); // L's link
+ } else {
+ // was: target_pos = kEndOfChain;
+ // However, using branch to self to mark the first reference
+ // should avoid most instances of branch offset overflow. See
+ // target_at() for where this is converted back to kEndOfChain.
+ target_pos = pc_offset();
+ }
+ L->link_to(pc_offset());
+
+ constant = target_pos - pc_offset();
+ }
+ llilf(r1, Operand(constant));
+}
+
+// Pseudo op - branch on condition
+void Assembler::branchOnCond(Condition c, int branch_offset, bool is_bound) {
+ int offset_in_halfwords = branch_offset / 2;
+ if (is_bound && is_int16(offset_in_halfwords)) {
+ brc(c, Operand(offset_in_halfwords)); // short jump
+ } else {
+ brcl(c, Operand(offset_in_halfwords)); // long jump
+ }
+}
+
+// Exception-generating instructions and debugging support.
+// Stops with a non-negative code less than kNumOfWatchedStops support
+// enabling/disabling and a counter feature. See simulator-s390.h .
+void Assembler::stop(Condition cond, int32_t code, CRegister cr) {
+ if (cond != al) {
+ Label skip;
+ b(NegateCondition(cond), &skip, Label::kNear);
+ bkpt(0);
+ bind(&skip);
+ } else {
+ bkpt(0);
+ }
+}
+
+void Assembler::bkpt(uint32_t imm16) {
+ // GDB software breakpoint instruction
+ emit2bytes(0x0001);
+}
+
+// Pseudo instructions.
+void Assembler::nop(int type) {
+ switch (type) {
+ case 0:
+ lr(r0, r0);
+ break;
+ case DEBUG_BREAK_NOP:
+ // TODO(john.yan): Use a better NOP break
+ oill(r3, Operand::Zero());
+ break;
+ default:
+ UNIMPLEMENTED();
+ }
+}
+
+// -------------------------
+// Load Address Instructions
+// -------------------------
+// Load Address Relative Long
+void Assembler::larl(Register r1, Label* l) {
+ larl(r1, Operand(branch_offset(l)));
+}
+
+void Assembler::EnsureSpaceFor(int space_needed) {
+ if (buffer_space() <= (kGap + space_needed)) {
+ GrowBuffer(space_needed);
+ }
+}
+
+void Assembler::call(Handle<Code> target, RelocInfo::Mode rmode) {
+ DCHECK(RelocInfo::IsCodeTarget(rmode));
+ EnsureSpace ensure_space(this);
+
+ RecordRelocInfo(rmode);
+ int32_t target_index = AddCodeTarget(target);
+ brasl(r14, Operand(target_index));
+}
+
+void Assembler::jump(Handle<Code> target, RelocInfo::Mode rmode,
+ Condition cond) {
+ DCHECK(RelocInfo::IsRelativeCodeTarget(rmode));
+ EnsureSpace ensure_space(this);
+
+ RecordRelocInfo(rmode);
+ int32_t target_index = AddCodeTarget(target);
+ brcl(cond, Operand(target_index));
+}
+
+// end of S390instructions
+
+bool Assembler::IsNop(SixByteInstr instr, int type) {
+ DCHECK((0 == type) || (DEBUG_BREAK_NOP == type));
+ if (DEBUG_BREAK_NOP == type) {
+ return ((instr & 0xFFFFFFFF) == 0xA53B0000); // oill r3, 0
+ }
+ return ((instr & 0xFFFF) == 0x1800); // lr r0,r0
+}
+
+// dummy instruction reserved for special use.
+void Assembler::dumy(int r1, int x2, int b2, int d2) {
+#if defined(USE_SIMULATOR)
+ int op = 0xE353;
+ uint64_t code = (static_cast<uint64_t>(op & 0xFF00)) * B32 |
+ (static_cast<uint64_t>(r1) & 0xF) * B36 |
+ (static_cast<uint64_t>(x2) & 0xF) * B32 |
+ (static_cast<uint64_t>(b2) & 0xF) * B28 |
+ (static_cast<uint64_t>(d2 & 0x0FFF)) * B16 |
+ (static_cast<uint64_t>(d2 & 0x0FF000)) >> 4 |
+ (static_cast<uint64_t>(op & 0x00FF));
+ emit6bytes(code);
+#endif
+}
+
+void Assembler::GrowBuffer(int needed) {
+ DCHECK_EQ(buffer_start_, buffer_->start());
+
+ // Compute new buffer size.
+ int old_size = buffer_->size();
+ int new_size = std::min(2 * old_size, old_size + 1 * MB);
+ int space = buffer_space() + (new_size - old_size);
+ new_size += (space < needed) ? needed - space : 0;
+
+ // Some internal data structures overflow for very large buffers,
+ // they must ensure that kMaximalBufferSize is not too large.
+ if (new_size > kMaximalBufferSize) {
+ V8::FatalProcessOutOfMemory(nullptr, "Assembler::GrowBuffer");
+ }
+
+ // Set up new buffer.
+ std::unique_ptr<AssemblerBuffer> new_buffer = buffer_->Grow(new_size);
+ DCHECK_EQ(new_size, new_buffer->size());
+ byte* new_start = new_buffer->start();
+
+ // Copy the data.
+ intptr_t pc_delta = new_start - buffer_start_;
+ intptr_t rc_delta = (new_start + new_size) - (buffer_start_ + old_size);
+ size_t reloc_size = (buffer_start_ + old_size) - reloc_info_writer.pos();
+ MemMove(new_start, buffer_start_, pc_offset());
+ MemMove(reloc_info_writer.pos() + rc_delta, reloc_info_writer.pos(),
+ reloc_size);
+
+ // Switch buffers.
+ buffer_ = std::move(new_buffer);
+ buffer_start_ = new_start;
+ pc_ += pc_delta;
+ reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta,
+ reloc_info_writer.last_pc() + pc_delta);
+
+ // None of our relocation types are pc relative pointing outside the code
+ // buffer nor pc absolute pointing inside the code buffer, so there is no need
+ // to relocate any emitted relocation entries.
+}
+
+void Assembler::db(uint8_t data) {
+ CheckBuffer();
+ *reinterpret_cast<uint8_t*>(pc_) = data;
+ pc_ += sizeof(uint8_t);
+}
+
+void Assembler::dd(uint32_t data) {
+ CheckBuffer();
+ *reinterpret_cast<uint32_t*>(pc_) = data;
+ pc_ += sizeof(uint32_t);
+}
+
+void Assembler::dq(uint64_t value) {
+ CheckBuffer();
+ *reinterpret_cast<uint64_t*>(pc_) = value;
+ pc_ += sizeof(uint64_t);
+}
+
+void Assembler::dp(uintptr_t data) {
+ CheckBuffer();
+ *reinterpret_cast<uintptr_t*>(pc_) = data;
+ pc_ += sizeof(uintptr_t);
+}
+
+void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) {
+ if (!ShouldRecordRelocInfo(rmode)) return;
+ DeferredRelocInfo rinfo(pc_offset(), rmode, data);
+ relocations_.push_back(rinfo);
+}
+
+void Assembler::emit_label_addr(Label* label) {
+ CheckBuffer();
+ RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE);
+ int position = link(label);
+ DCHECK(label->is_bound());
+ // Keep internal references relative until EmitRelocations.
+ dp(position);
+}
+
+void Assembler::EmitRelocations() {
+ EnsureSpaceFor(relocations_.size() * kMaxRelocSize);
+
+ for (std::vector<DeferredRelocInfo>::iterator it = relocations_.begin();
+ it != relocations_.end(); it++) {
+ RelocInfo::Mode rmode = it->rmode();
+ Address pc = reinterpret_cast<Address>(buffer_start_) + it->position();
+ RelocInfo rinfo(pc, rmode, it->data(), Code());
+
+ // Fix up internal references now that they are guaranteed to be bound.
+ if (RelocInfo::IsInternalReference(rmode)) {
+ // Jump table entry
+ Address pos = Memory<Address>(pc);
+ Memory<Address>(pc) = reinterpret_cast<Address>(buffer_start_) + pos;
+ } else if (RelocInfo::IsInternalReferenceEncoded(rmode)) {
+ // mov sequence
+ Address pos = target_address_at(pc, 0);
+ set_target_address_at(pc, 0,
+ reinterpret_cast<Address>(buffer_start_) + pos,
+ SKIP_ICACHE_FLUSH);
+ }
+
+ reloc_info_writer.Write(&rinfo);
+ }
+}
+
+UseScratchRegisterScope::UseScratchRegisterScope(Assembler* assembler)
+ : assembler_(assembler),
+ old_available_(*assembler->GetScratchRegisterList()) {}
+
+UseScratchRegisterScope::~UseScratchRegisterScope() {
+ *assembler_->GetScratchRegisterList() = old_available_;
+}
+
+Register UseScratchRegisterScope::Acquire() {
+ RegList* available = assembler_->GetScratchRegisterList();
+ DCHECK_NOT_NULL(available);
+ DCHECK_NE(*available, 0);
+ int index = static_cast<int>(base::bits::CountTrailingZeros32(*available));
+ Register reg = Register::from_code(index);
+ *available &= ~reg.bit();
+ return reg;
+}
+} // namespace internal
+} // namespace v8
+#endif // V8_TARGET_ARCH_S390
diff --git a/src/codegen/s390/assembler-s390.h b/src/codegen/s390/assembler-s390.h
new file mode 100644
index 0000000..bf746df
--- /dev/null
+++ b/src/codegen/s390/assembler-s390.h
@@ -0,0 +1,1509 @@
+// Copyright (c) 1994-2006 Sun Microsystems Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//
+// - Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// - Redistribution in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the
+// distribution.
+//
+// - Neither the name of Sun Microsystems or the names of contributors may
+// be used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+// COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+// OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// The original source code covered by the above license above has been
+// modified significantly by Google Inc.
+// Copyright 2014 the V8 project authors. All rights reserved.
+
+// A light-weight S390 Assembler
+// Generates user mode instructions for z/Architecture
+
+#ifndef V8_CODEGEN_S390_ASSEMBLER_S390_H_
+#define V8_CODEGEN_S390_ASSEMBLER_S390_H_
+#include <stdio.h>
+#include <memory>
+#if V8_HOST_ARCH_S390
+// elf.h include is required for auxv check for STFLE facility used
+// for hardware detection, which is sensible only on s390 hosts.
+#include <elf.h>
+#endif
+
+#include <fcntl.h>
+#include <unistd.h>
+#include <vector>
+
+#include "src/codegen/assembler.h"
+#include "src/codegen/external-reference.h"
+#include "src/codegen/label.h"
+#include "src/codegen/s390/constants-s390.h"
+#include "src/codegen/s390/register-s390.h"
+#include "src/objects/smi.h"
+
+#define ABI_USES_FUNCTION_DESCRIPTORS 0
+
+#define ABI_PASSES_HANDLES_IN_REGS 1
+
+// ObjectPair is defined under runtime/runtime-util.h.
+// On 31-bit, ObjectPair == uint64_t. ABI dictates long long
+// be returned with the lower addressed half in r2
+// and the higher addressed half in r3. (Returns in Regs)
+// On 64-bit, ObjectPair is a Struct. ABI dictaes Structs be
+// returned in a storage buffer allocated by the caller,
+// with the address of this buffer passed as a hidden
+// argument in r2. (Does NOT return in Regs)
+// For x86 linux, ObjectPair is returned in registers.
+#if V8_TARGET_ARCH_S390X
+#define ABI_RETURNS_OBJECTPAIR_IN_REGS 0
+#else
+#define ABI_RETURNS_OBJECTPAIR_IN_REGS 1
+#endif
+
+#define ABI_CALL_VIA_IP 1
+
+namespace v8 {
+namespace internal {
+
+class SafepointTableBuilder;
+
+// -----------------------------------------------------------------------------
+// Machine instruction Operands
+
+// Class Operand represents a shifter operand in data processing instructions
+// defining immediate numbers and masks
+class V8_EXPORT_PRIVATE Operand {
+ public:
+ // immediate
+ V8_INLINE explicit Operand(intptr_t immediate,
+ RelocInfo::Mode rmode = RelocInfo::NONE)
+ : rmode_(rmode) {
+ value_.immediate = immediate;
+ }
+ V8_INLINE static Operand Zero() { return Operand(static_cast<intptr_t>(0)); }
+ V8_INLINE explicit Operand(const ExternalReference& f)
+ : rmode_(RelocInfo::EXTERNAL_REFERENCE) {
+ value_.immediate = static_cast<intptr_t>(f.address());
+ }
+ explicit Operand(Handle<HeapObject> handle);
+ V8_INLINE explicit Operand(Smi value) : rmode_(RelocInfo::NONE) {
+ value_.immediate = static_cast<intptr_t>(value.ptr());
+ }
+
+ // rm
+ V8_INLINE explicit Operand(Register rm);
+
+ static Operand EmbeddedNumber(double value); // Smi or HeapNumber
+ static Operand EmbeddedStringConstant(const StringConstantBase* str);
+
+ // Return true if this is a register operand.
+ V8_INLINE bool is_reg() const { return rm_.is_valid(); }
+
+ bool must_output_reloc_info(const Assembler* assembler) const;
+
+ inline intptr_t immediate() const {
+ DCHECK(!rm_.is_valid());
+ DCHECK(!is_heap_object_request());
+ return value_.immediate;
+ }
+
+ HeapObjectRequest heap_object_request() const {
+ DCHECK(is_heap_object_request());
+ return value_.heap_object_request;
+ }
+
+ inline void setBits(int n) {
+ value_.immediate =
+ (static_cast<uint32_t>(value_.immediate) << (32 - n)) >> (32 - n);
+ }
+
+ Register rm() const { return rm_; }
+
+ bool is_heap_object_request() const {
+ DCHECK_IMPLIES(is_heap_object_request_, !rm_.is_valid());
+ DCHECK_IMPLIES(is_heap_object_request_,
+ rmode_ == RelocInfo::FULL_EMBEDDED_OBJECT ||
+ rmode_ == RelocInfo::CODE_TARGET);
+ return is_heap_object_request_;
+ }
+
+ RelocInfo::Mode rmode() const { return rmode_; }
+
+ private:
+ Register rm_ = no_reg;
+ union Value {
+ Value() {}
+ HeapObjectRequest heap_object_request; // if is_heap_object_request_
+ intptr_t immediate; // otherwise
+ } value_; // valid if rm_ == no_reg
+ bool is_heap_object_request_ = false;
+
+ RelocInfo::Mode rmode_;
+
+ friend class Assembler;
+ friend class MacroAssembler;
+};
+
+using Disp = int32_t;
+
+// Class MemOperand represents a memory operand in load and store instructions
+// On S390, we have various flavours of memory operands:
+// 1) a base register + 16 bit unsigned displacement
+// 2) a base register + index register + 16 bit unsigned displacement
+// 3) a base register + index register + 20 bit signed displacement
+class V8_EXPORT_PRIVATE MemOperand {
+ public:
+ explicit MemOperand(Register rx, Disp offset = 0);
+ explicit MemOperand(Register rx, Register rb, Disp offset = 0);
+
+ int32_t offset() const { return offset_; }
+ uint32_t getDisplacement() const { return offset(); }
+
+ // Base register
+ Register rb() const {
+ DCHECK(baseRegister != no_reg);
+ return baseRegister;
+ }
+
+ Register getBaseRegister() const { return rb(); }
+
+ // Index Register
+ Register rx() const {
+ DCHECK(indexRegister != no_reg);
+ return indexRegister;
+ }
+ Register getIndexRegister() const { return rx(); }
+
+ private:
+ Register baseRegister; // base
+ Register indexRegister; // index
+ int32_t offset_; // offset
+
+ friend class Assembler;
+};
+
+class DeferredRelocInfo {
+ public:
+ DeferredRelocInfo() {}
+ DeferredRelocInfo(int position, RelocInfo::Mode rmode, intptr_t data)
+ : position_(position), rmode_(rmode), data_(data) {}
+
+ int position() const { return position_; }
+ RelocInfo::Mode rmode() const { return rmode_; }
+ intptr_t data() const { return data_; }
+
+ private:
+ int position_;
+ RelocInfo::Mode rmode_;
+ intptr_t data_;
+};
+
+class V8_EXPORT_PRIVATE Assembler : public AssemblerBase {
+ public:
+ // Create an assembler. Instructions and relocation information are emitted
+ // into a buffer, with the instructions starting from the beginning and the
+ // relocation information starting from the end of the buffer. See CodeDesc
+ // for a detailed comment on the layout (globals.h).
+ //
+ // If the provided buffer is nullptr, the assembler allocates and grows its
+ // own buffer. Otherwise it takes ownership of the provided buffer.
+ explicit Assembler(const AssemblerOptions&,
+ std::unique_ptr<AssemblerBuffer> = {});
+
+ virtual ~Assembler() {}
+
+ // GetCode emits any pending (non-emitted) code and fills the descriptor desc.
+ static constexpr int kNoHandlerTable = 0;
+ static constexpr SafepointTableBuilder* kNoSafepointTable = nullptr;
+ void GetCode(Isolate* isolate, CodeDesc* desc,
+ SafepointTableBuilder* safepoint_table_builder,
+ int handler_table_offset);
+
+ // Convenience wrapper for code without safepoint or handler tables.
+ void GetCode(Isolate* isolate, CodeDesc* desc) {
+ GetCode(isolate, desc, kNoSafepointTable, kNoHandlerTable);
+ }
+
+ // Unused on this architecture.
+ void MaybeEmitOutOfLineConstantPool() {}
+
+ // Label operations & relative jumps (PPUM Appendix D)
+ //
+ // Takes a branch opcode (cc) and a label (L) and generates
+ // either a backward branch or a forward branch and links it
+ // to the label fixup chain. Usage:
+ //
+ // Label L; // unbound label
+ // j(cc, &L); // forward branch to unbound label
+ // bind(&L); // bind label to the current pc
+ // j(cc, &L); // backward branch to bound label
+ // bind(&L); // illegal: a label may be bound only once
+ //
+ // Note: The same Label can be used for forward and backward branches
+ // but it may be bound only once.
+
+ void bind(Label* L); // binds an unbound label L to the current code position
+
+ // Links a label at the current pc_offset(). If already bound, returns the
+ // bound position. If already linked, returns the position of the prior link.
+ // Otherwise, returns the current pc_offset().
+ int link(Label* L);
+
+ // Returns the branch offset to the given label from the current code position
+ // Links the label to the current position if it is still unbound
+ int branch_offset(Label* L) { return link(L) - pc_offset(); }
+
+ void load_label_offset(Register r1, Label* L);
+
+ // Read/Modify the code target address in the branch/call instruction at pc.
+ // The isolate argument is unused (and may be nullptr) when skipping flushing.
+ V8_INLINE static Address target_address_at(Address pc, Address constant_pool);
+
+ // Read/Modify the code target address in the branch/call instruction at pc.
+ inline static Tagged_t target_compressed_address_at(Address pc,
+ Address constant_pool);
+ V8_INLINE static void set_target_address_at(
+ Address pc, Address constant_pool, Address target,
+ ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED);
+
+ inline static void set_target_compressed_address_at(
+ Address pc, Address constant_pool, Tagged_t target,
+ ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED);
+
+ inline Handle<Object> code_target_object_handle_at(Address pc);
+ inline Handle<HeapObject> compressed_embedded_object_handle_at(
+ Address pc, Address constant_pool);
+ // This sets the branch destination.
+ // This is for calls and branches within generated code.
+ inline static void deserialization_set_special_target_at(
+ Address instruction_payload, Code code, Address target);
+
+ // Get the size of the special target encoded at 'instruction_payload'.
+ inline static int deserialization_special_target_size(
+ Address instruction_payload);
+
+ // This sets the internal reference at the pc.
+ inline static void deserialization_set_target_internal_reference_at(
+ Address pc, Address target,
+ RelocInfo::Mode mode = RelocInfo::INTERNAL_REFERENCE);
+
+ // Here we are patching the address in the IIHF/IILF instruction pair.
+ // These values are used in the serialization process and must be zero for
+ // S390 platform, as Code, Embedded Object or External-reference pointers
+ // are split across two consecutive instructions and don't exist separately
+ // in the code, so the serializer should not step forwards in memory after
+ // a target is resolved and written.
+ static constexpr int kSpecialTargetSize = 0;
+// Number of bytes for instructions used to store pointer sized constant.
+#if V8_TARGET_ARCH_S390X
+ static constexpr int kBytesForPtrConstant = 12; // IIHF + IILF
+#else
+ static constexpr int kBytesForPtrConstant = 6; // IILF
+#endif
+
+ RegList* GetScratchRegisterList() { return &scratch_register_list_; }
+
+ // ---------------------------------------------------------------------------
+ // Code generation
+
+ template <class T, int size, int lo, int hi>
+ inline T getfield(T value) {
+ DCHECK(lo < hi);
+ DCHECK_GT(size, 0);
+ int mask = hi - lo;
+ int shift = size * 8 - hi;
+ uint32_t mask_value = (mask == 32) ? 0xffffffff : (1 << mask) - 1;
+ return (value & mask_value) << shift;
+ }
+
+#define DECLARE_S390_RIL_AB_INSTRUCTIONS(name, op_name, op_value) \
+ template <class R1> \
+ inline void name(R1 r1, const Operand& i2) { \
+ ril_format(op_name, r1.code(), i2.immediate()); \
+ }
+#define DECLARE_S390_RIL_C_INSTRUCTIONS(name, op_name, op_value) \
+ inline void name(Condition m1, const Operand& i2) { \
+ ril_format(op_name, m1, i2.immediate()); \
+ }
+
+ inline void ril_format(Opcode opcode, int f1, int f2) {
+ uint32_t op1 = opcode >> 4;
+ uint32_t op2 = opcode & 0xf;
+ emit6bytes(
+ getfield<uint64_t, 6, 0, 8>(op1) | getfield<uint64_t, 6, 8, 12>(f1) |
+ getfield<uint64_t, 6, 12, 16>(op2) | getfield<uint64_t, 6, 16, 48>(f2));
+ }
+ S390_RIL_A_OPCODE_LIST(DECLARE_S390_RIL_AB_INSTRUCTIONS)
+ S390_RIL_B_OPCODE_LIST(DECLARE_S390_RIL_AB_INSTRUCTIONS)
+ S390_RIL_C_OPCODE_LIST(DECLARE_S390_RIL_C_INSTRUCTIONS)
+#undef DECLARE_S390_RIL_AB_INSTRUCTIONS
+#undef DECLARE_S390_RIL_C_INSTRUCTIONS
+
+#define DECLARE_S390_RR_INSTRUCTIONS(name, op_name, op_value) \
+ inline void name(Register r1, Register r2) { \
+ rr_format(op_name, r1.code(), r2.code()); \
+ } \
+ inline void name(DoubleRegister r1, DoubleRegister r2) { \
+ rr_format(op_name, r1.code(), r2.code()); \
+ } \
+ inline void name(Condition m1, Register r2) { \
+ rr_format(op_name, m1, r2.code()); \
+ }
+
+ inline void rr_format(Opcode opcode, int f1, int f2) {
+ emit2bytes(getfield<uint16_t, 2, 0, 8>(opcode) |
+ getfield<uint16_t, 2, 8, 12>(f1) |
+ getfield<uint16_t, 2, 12, 16>(f2));
+ }
+ S390_RR_OPCODE_LIST(DECLARE_S390_RR_INSTRUCTIONS)
+#undef DECLARE_S390_RR_INSTRUCTIONS
+
+#define DECLARE_S390_RRD_INSTRUCTIONS(name, op_name, op_value) \
+ template <class R1, class R2, class R3> \
+ inline void name(R1 r1, R3 r3, R2 r2) { \
+ rrd_format(op_name, r1.code(), r3.code(), r2.code()); \
+ }
+ inline void rrd_format(Opcode opcode, int f1, int f2, int f3) {
+ emit4bytes(getfield<uint32_t, 4, 0, 16>(opcode) |
+ getfield<uint32_t, 4, 16, 20>(f1) |
+ getfield<uint32_t, 4, 24, 28>(f2) |
+ getfield<uint32_t, 4, 28, 32>(f3));
+ }
+ S390_RRD_OPCODE_LIST(DECLARE_S390_RRD_INSTRUCTIONS)
+#undef DECLARE_S390_RRD_INSTRUCTIONS
+
+#define DECLARE_S390_RRE_INSTRUCTIONS(name, op_name, op_value) \
+ template <class R1, class R2> \
+ inline void name(R1 r1, R2 r2) { \
+ rre_format(op_name, r1.code(), r2.code()); \
+ }
+ inline void rre_format(Opcode opcode, int f1, int f2) {
+ emit4bytes(getfield<uint32_t, 4, 0, 16>(opcode) |
+ getfield<uint32_t, 4, 24, 28>(f1) |
+ getfield<uint32_t, 4, 28, 32>(f2));
+ }
+ S390_RRE_OPCODE_LIST(DECLARE_S390_RRE_INSTRUCTIONS)
+ // Special format
+ void lzdr(DoubleRegister r1) { rre_format(LZDR, r1.code(), 0); }
+#undef DECLARE_S390_RRE_INSTRUCTIONS
+
+#define DECLARE_S390_RX_INSTRUCTIONS(name, op_name, op_value) \
+ template <class R1> \
+ inline void name(R1 r1, Register x2, Register b2, const Operand& d2) { \
+ rx_format(op_name, r1.code(), x2.code(), b2.code(), d2.immediate()); \
+ } \
+ template <class R1> \
+ inline void name(R1 r1, const MemOperand& opnd) { \
+ name(r1, opnd.getIndexRegister(), opnd.getBaseRegister(), \
+ Operand(opnd.getDisplacement())); \
+ }
+
+ inline void rx_format(Opcode opcode, int f1, int f2, int f3, int f4) {
+ DCHECK(is_uint8(opcode));
+ DCHECK(is_uint12(f4));
+ emit4bytes(
+ getfield<uint32_t, 4, 0, 8>(opcode) | getfield<uint32_t, 4, 8, 12>(f1) |
+ getfield<uint32_t, 4, 12, 16>(f2) | getfield<uint32_t, 4, 16, 20>(f3) |
+ getfield<uint32_t, 4, 20, 32>(f4));
+ }
+ S390_RX_A_OPCODE_LIST(DECLARE_S390_RX_INSTRUCTIONS)
+
+ void bc(Condition cond, const MemOperand& opnd) {
+ bc(cond, opnd.getIndexRegister(), opnd.getBaseRegister(),
+ Operand(opnd.getDisplacement()));
+ }
+ void bc(Condition cond, Register x2, Register b2, const Operand& d2) {
+ rx_format(BC, cond, x2.code(), b2.code(), d2.immediate());
+ }
+#undef DECLARE_S390_RX_INSTRUCTIONS
+
+#define DECLARE_S390_RXY_INSTRUCTIONS(name, op_name, op_value) \
+ template <class R1, class R2> \
+ inline void name(R1 r1, R2 r2, Register b2, const Operand& d2) { \
+ rxy_format(op_name, r1.code(), r2.code(), b2.code(), d2.immediate()); \
+ } \
+ template <class R1> \
+ inline void name(R1 r1, const MemOperand& opnd) { \
+ name(r1, opnd.getIndexRegister(), opnd.getBaseRegister(), \
+ Operand(opnd.getDisplacement())); \
+ }
+
+ inline void rxy_format(Opcode opcode, int f1, int f2, int f3, int f4) {
+ DCHECK(is_uint16(opcode));
+ DCHECK(is_int20(f4));
+ emit6bytes(getfield<uint64_t, 6, 0, 8>(opcode >> 8) |
+ getfield<uint64_t, 6, 8, 12>(f1) |
+ getfield<uint64_t, 6, 12, 16>(f2) |
+ getfield<uint64_t, 6, 16, 20>(f3) |
+ getfield<uint64_t, 6, 20, 32>(f4 & 0x0fff) |
+ getfield<uint64_t, 6, 32, 40>(f4 >> 12) |
+ getfield<uint64_t, 6, 40, 48>(opcode & 0x00ff));
+ }
+ S390_RXY_A_OPCODE_LIST(DECLARE_S390_RXY_INSTRUCTIONS)
+
+ void pfd(Condition cond, const MemOperand& opnd) {
+ pfd(cond, opnd.getIndexRegister(), opnd.getBaseRegister(),
+ Operand(opnd.getDisplacement()));
+ }
+ void pfd(Condition cond, Register x2, Register b2, const Operand& d2) {
+ rxy_format(PFD, cond, x2.code(), b2.code(), d2.immediate());
+ }
+#undef DECLARE_S390_RXY_INSTRUCTIONS
+
+ inline void rsy_format(Opcode op, int f1, int f2, int f3, int f4) {
+ DCHECK(is_int20(f4));
+ DCHECK(is_uint16(op));
+ uint64_t code =
+ (getfield<uint64_t, 6, 0, 8>(op >> 8) |
+ getfield<uint64_t, 6, 8, 12>(f1) | getfield<uint64_t, 6, 12, 16>(f2) |
+ getfield<uint64_t, 6, 16, 20>(f3) |
+ getfield<uint64_t, 6, 20, 32>(f4 & 0x0fff) |
+ getfield<uint64_t, 6, 32, 40>(f4 >> 12) |
+ getfield<uint64_t, 6, 40, 48>(op & 0xff));
+ emit6bytes(code);
+ }
+
+#define DECLARE_S390_RSY_A_INSTRUCTIONS(name, op_name, op_value) \
+ void name(Register r1, Register r3, Register b2, \
+ const Operand& d2 = Operand::Zero()) { \
+ rsy_format(op_name, r1.code(), r3.code(), b2.code(), d2.immediate()); \
+ } \
+ void name(Register r1, Register r3, Operand d2) { name(r1, r3, r0, d2); } \
+ void name(Register r1, Register r3, const MemOperand& opnd) { \
+ name(r1, r3, opnd.getBaseRegister(), Operand(opnd.getDisplacement())); \
+ }
+ S390_RSY_A_OPCODE_LIST(DECLARE_S390_RSY_A_INSTRUCTIONS)
+#undef DECLARE_S390_RSY_A_INSTRUCTIONS
+
+#define DECLARE_S390_RSY_B_INSTRUCTIONS(name, op_name, op_value) \
+ void name(Register r1, Condition m3, Register b2, const Operand& d2) { \
+ rsy_format(op_name, r1.code(), m3, b2.code(), d2.immediate()); \
+ } \
+ void name(Register r1, Condition m3, const MemOperand& opnd) { \
+ name(r1, m3, opnd.getBaseRegister(), Operand(opnd.getDisplacement())); \
+ }
+ S390_RSY_B_OPCODE_LIST(DECLARE_S390_RSY_B_INSTRUCTIONS)
+#undef DECLARE_S390_RSY_B_INSTRUCTIONS
+
+ inline void rs_format(Opcode op, int f1, int f2, int f3, const int f4) {
+ uint32_t code =
+ getfield<uint32_t, 4, 0, 8>(op) | getfield<uint32_t, 4, 8, 12>(f1) |
+ getfield<uint32_t, 4, 12, 16>(f2) | getfield<uint32_t, 4, 16, 20>(f3) |
+ getfield<uint32_t, 4, 20, 32>(f4);
+ emit4bytes(code);
+ }
+
+#define DECLARE_S390_RS_A_INSTRUCTIONS(name, op_name, op_value) \
+ void name(Register r1, Register r3, Register b2, const Operand& d2) { \
+ rs_format(op_name, r1.code(), r3.code(), b2.code(), d2.immediate()); \
+ } \
+ void name(Register r1, Register r3, const MemOperand& opnd) { \
+ name(r1, r3, opnd.getBaseRegister(), Operand(opnd.getDisplacement())); \
+ }
+ S390_RS_A_OPCODE_LIST(DECLARE_S390_RS_A_INSTRUCTIONS)
+#undef DECLARE_S390_RS_A_INSTRUCTIONS
+
+#define DECLARE_S390_RS_B_INSTRUCTIONS(name, op_name, op_value) \
+ void name(Register r1, Condition m3, Register b2, const Operand& d2) { \
+ rs_format(op_name, r1.code(), m3, b2.code(), d2.immediate()); \
+ } \
+ void name(Register r1, Condition m3, const MemOperand& opnd) { \
+ name(r1, m3, opnd.getBaseRegister(), Operand(opnd.getDisplacement())); \
+ }
+ S390_RS_B_OPCODE_LIST(DECLARE_S390_RS_B_INSTRUCTIONS)
+#undef DECLARE_S390_RS_B_INSTRUCTIONS
+
+#define DECLARE_S390_RS_SHIFT_FORMAT(name, opcode) \
+ void name(Register r1, Register r2, const Operand& opnd = Operand::Zero()) { \
+ DCHECK(r2 != r0); \
+ rs_format(opcode, r1.code(), r0.code(), r2.code(), opnd.immediate()); \
+ } \
+ void name(Register r1, const Operand& opnd) { \
+ rs_format(opcode, r1.code(), r0.code(), r0.code(), opnd.immediate()); \
+ }
+ DECLARE_S390_RS_SHIFT_FORMAT(sll, SLL)
+ DECLARE_S390_RS_SHIFT_FORMAT(srl, SRL)
+ DECLARE_S390_RS_SHIFT_FORMAT(sla, SLA)
+ DECLARE_S390_RS_SHIFT_FORMAT(sra, SRA)
+ DECLARE_S390_RS_SHIFT_FORMAT(sldl, SLDL)
+ DECLARE_S390_RS_SHIFT_FORMAT(srda, SRDA)
+ DECLARE_S390_RS_SHIFT_FORMAT(srdl, SRDL)
+#undef DECLARE_S390_RS_SHIFT_FORMAT
+
+ inline void rxe_format(Opcode op, int f1, int f2, int f3, int f4,
+ int f5 = 0) {
+ DCHECK(is_uint12(f4));
+ DCHECK(is_uint16(op));
+ uint64_t code =
+ (getfield<uint64_t, 6, 0, 8>(op >> 8) |
+ getfield<uint64_t, 6, 8, 12>(f1) | getfield<uint64_t, 6, 12, 16>(f2) |
+ getfield<uint64_t, 6, 16, 20>(f3) |
+ getfield<uint64_t, 6, 20, 32>(f4 & 0x0fff) |
+ getfield<uint64_t, 6, 32, 36>(f5) |
+ getfield<uint64_t, 6, 40, 48>(op & 0xff));
+ emit6bytes(code);
+ }
+
+#define DECLARE_S390_RXE_INSTRUCTIONS(name, op_name, op_value) \
+ void name(Register r1, Register x2, Register b2, const Operand& d2, \
+ Condition m3 = static_cast<Condition>(0)) { \
+ rxe_format(op_name, r1.code(), x2.code(), b2.code(), d2.immediate(), m3); \
+ } \
+ template <class _R1Type> \
+ void name(_R1Type r1, const MemOperand& opnd) { \
+ name(Register::from_code(r1.code()), opnd.rx(), opnd.rb(), \
+ Operand(opnd.offset())); \
+ }
+ S390_RXE_OPCODE_LIST(DECLARE_S390_RXE_INSTRUCTIONS)
+#undef DECLARE_S390_RXE_INSTRUCTIONS
+
+ inline void ri_format(Opcode opcode, int f1, int f2) {
+ uint32_t op1 = opcode >> 4;
+ uint32_t op2 = opcode & 0xf;
+ emit4bytes(
+ getfield<uint32_t, 4, 0, 8>(op1) | getfield<uint32_t, 4, 8, 12>(f1) |
+ getfield<uint32_t, 4, 12, 16>(op2) | getfield<uint32_t, 4, 16, 32>(f2));
+ }
+
+#define DECLARE_S390_RI_A_INSTRUCTIONS(name, op_name, op_value) \
+ void name(Register r, const Operand& i2) { \
+ DCHECK(is_uint12(op_name)); \
+ DCHECK(is_uint16(i2.immediate()) || is_int16(i2.immediate())); \
+ ri_format(op_name, r.code(), i2.immediate()); \
+ }
+ S390_RI_A_OPCODE_LIST(DECLARE_S390_RI_A_INSTRUCTIONS)
+#undef DECLARE_S390_RI_A_INSTRUCTIONS
+
+#define DECLARE_S390_RI_B_INSTRUCTIONS(name, op_name, op_value) \
+ void name(Register r1, const Operand& imm) { \
+ /* 2nd argument encodes # of halfwords, so divide by 2. */ \
+ int16_t numHalfwords = static_cast<int16_t>(imm.immediate()) / 2; \
+ Operand halfwordOp = Operand(numHalfwords); \
+ halfwordOp.setBits(16); \
+ ri_format(op_name, r1.code(), halfwordOp.immediate()); \
+ }
+ S390_RI_B_OPCODE_LIST(DECLARE_S390_RI_B_INSTRUCTIONS)
+#undef DECLARE_S390_RI_B_INSTRUCTIONS
+
+#define DECLARE_S390_RI_C_INSTRUCTIONS(name, op_name, op_value) \
+ void name(Condition m, const Operand& i2) { \
+ DCHECK(is_uint12(op_name)); \
+ DCHECK(is_uint4(m)); \
+ DCHECK(op_name == BRC ? is_int16(i2.immediate()) \
+ : is_uint16(i2.immediate())); \
+ ri_format(op_name, m, i2.immediate()); \
+ }
+ S390_RI_C_OPCODE_LIST(DECLARE_S390_RI_C_INSTRUCTIONS)
+#undef DECLARE_S390_RI_C_INSTRUCTIONS
+
+ inline void rrf_format(Opcode op, int f1, int f2, int f3, int f4) {
+ uint32_t code =
+ getfield<uint32_t, 4, 0, 16>(op) | getfield<uint32_t, 4, 16, 20>(f1) |
+ getfield<uint32_t, 4, 20, 24>(f2) | getfield<uint32_t, 4, 24, 28>(f3) |
+ getfield<uint32_t, 4, 28, 32>(f4);
+ emit4bytes(code);
+ }
+
+#define DECLARE_S390_RRF_A_INSTRUCTIONS(name, op_name, op_value) \
+ void name(Register r1, Condition m4, Register r2, Register r3) { \
+ rrf_format(op_name, r3.code(), m4, r1.code(), r2.code()); \
+ } \
+ void name(Register r1, Register r2, Register r3) { \
+ name(r1, Condition(0), r2, r3); \
+ }
+ S390_RRF_A_OPCODE_LIST(DECLARE_S390_RRF_A_INSTRUCTIONS)
+#undef DECLARE_S390_RRF_A_INSTRUCTIONS
+
+#define DECLARE_S390_RRF_B_INSTRUCTIONS(name, op_name, op_value) \
+ void name(Register r1, Condition m4, Register r2, Register r3) { \
+ rrf_format(op_name, r3.code(), m4, r1.code(), r2.code()); \
+ } \
+ void name(Register r1, Register r2, Register r3) { \
+ name(r1, Condition(0), r2, r3); \
+ }
+ S390_RRF_B_OPCODE_LIST(DECLARE_S390_RRF_B_INSTRUCTIONS)
+#undef DECLARE_S390_RRF_B_INSTRUCTIONS
+
+#define DECLARE_S390_RRF_C_INSTRUCTIONS(name, op_name, op_value) \
+ template <class R1, class R2> \
+ void name(Condition m3, Condition m4, R1 r1, R2 r2) { \
+ rrf_format(op_name, m3, m4, r1.code(), r2.code()); \
+ } \
+ template <class R1, class R2> \
+ void name(Condition m3, R1 r1, R2 r2) { \
+ name(m3, Condition(0), r1, r2); \
+ }
+ S390_RRF_C_OPCODE_LIST(DECLARE_S390_RRF_C_INSTRUCTIONS)
+#undef DECLARE_S390_RRF_C_INSTRUCTIONS
+
+#define DECLARE_S390_RRF_D_INSTRUCTIONS(name, op_name, op_value) \
+ template <class R1, class R2> \
+ void name(Condition m3, Condition m4, R1 r1, R2 r2) { \
+ rrf_format(op_name, m3, m4, r1.code(), r2.code()); \
+ } \
+ template <class R1, class R2> \
+ void name(Condition m3, R1 r1, R2 r2) { \
+ name(m3, Condition(0), r1, r2); \
+ }
+ S390_RRF_D_OPCODE_LIST(DECLARE_S390_RRF_D_INSTRUCTIONS)
+#undef DECLARE_S390_RRF_D_INSTRUCTIONS
+
+#define DECLARE_S390_RRF_E_INSTRUCTIONS(name, op_name, op_value) \
+ template <class M3, class M4, class R1, class R2> \
+ void name(M3 m3, M4 m4, R1 r1, R2 r2) { \
+ rrf_format(op_name, m3, m4, r1.code(), r2.code()); \
+ } \
+ template <class M3, class R1, class R2> \
+ void name(M3 m3, R1 r1, R2 r2) { \
+ name(m3, Condition(0), r1, r2); \
+ }
+ S390_RRF_E_OPCODE_LIST(DECLARE_S390_RRF_E_INSTRUCTIONS)
+#undef DECLARE_S390_RRF_E_INSTRUCTIONS
+
+ enum FIDBRA_FLAGS {
+ FIDBRA_CURRENT_ROUNDING_MODE = 0,
+ FIDBRA_ROUND_TO_NEAREST_AWAY_FROM_0 = 1,
+ // ...
+ FIDBRA_ROUND_TOWARD_0 = 5,
+ FIDBRA_ROUND_TOWARD_POS_INF = 6,
+ FIDBRA_ROUND_TOWARD_NEG_INF = 7
+ };
+
+ inline void rsi_format(Opcode op, int f1, int f2, int f3) {
+ DCHECK(is_uint8(op));
+ DCHECK(is_uint16(f3) || is_int16(f3));
+ uint32_t code =
+ getfield<uint32_t, 4, 0, 8>(op) | getfield<uint32_t, 4, 8, 12>(f1) |
+ getfield<uint32_t, 4, 12, 16>(f2) | getfield<uint32_t, 4, 16, 32>(f3);
+ emit4bytes(code);
+ }
+
+#define DECLARE_S390_RSI_INSTRUCTIONS(name, op_name, op_value) \
+ void name(Register r1, Register r3, const Operand& i2) { \
+ rsi_format(op_name, r1.code(), r3.code(), i2.immediate()); \
+ }
+ S390_RSI_OPCODE_LIST(DECLARE_S390_RSI_INSTRUCTIONS)
+#undef DECLARE_S390_RSI_INSTRUCTIONS
+
+ inline void rsl_format(Opcode op, uint16_t f1, int f2, int f3, int f4,
+ int f5) {
+ DCHECK(is_uint16(op));
+ uint64_t code =
+ getfield<uint64_t, 6, 0, 8>(op >> 8) |
+ getfield<uint64_t, 6, 8, 16>(f1) | getfield<uint64_t, 6, 16, 20>(f2) |
+ getfield<uint64_t, 6, 20, 32>(f3) | getfield<uint64_t, 6, 32, 36>(f4) |
+ getfield<uint64_t, 6, 36, 40>(f5) |
+ getfield<uint64_t, 6, 40, 48>(op & 0x00FF);
+ emit6bytes(code);
+ }
+
+#define DECLARE_S390_RSL_A_INSTRUCTIONS(name, op_name, op_value) \
+ void name(const Operand& l1, Register b1, const Operand& d1) { \
+ uint16_t L = static_cast<uint16_t>(l1.immediate() << 8); \
+ rsl_format(op_name, L, b1.code(), d1.immediate(), 0, 0); \
+ }
+ S390_RSL_A_OPCODE_LIST(DECLARE_S390_RSL_A_INSTRUCTIONS)
+#undef DECLARE_S390_RSL_A_INSTRUCTIONS
+
+#define DECLARE_S390_RSL_B_INSTRUCTIONS(name, op_name, op_value) \
+ void name(const Operand& l2, Register b2, const Operand& d2, Register r1, \
+ Condition m3) { \
+ uint16_t L = static_cast<uint16_t>(l2.immediate()); \
+ rsl_format(op_name, L, b2.code(), d2.immediate(), r1.code(), m3); \
+ }
+ S390_RSL_B_OPCODE_LIST(DECLARE_S390_RSL_B_INSTRUCTIONS)
+#undef DECLARE_S390_RSL_B_INSTRUCTIONS
+
+ inline void s_format(Opcode op, int f1, int f2) {
+ DCHECK_NE(op & 0xff00, 0);
+ DCHECK(is_uint12(f2));
+ uint32_t code = getfield<uint32_t, 4, 0, 16>(op) |
+ getfield<uint32_t, 4, 16, 20>(f1) |
+ getfield<uint32_t, 4, 20, 32>(f2);
+ emit4bytes(code);
+ }
+
+#define DECLARE_S390_S_INSTRUCTIONS(name, op_name, op_value) \
+ void name(Register b1, const Operand& d2) { \
+ Opcode op = op_name; \
+ if ((op & 0xFF00) == 0) { \
+ op = (Opcode)(op << 8); \
+ } \
+ s_format(op, b1.code(), d2.immediate()); \
+ } \
+ void name(const MemOperand& opnd) { \
+ Operand d2 = Operand(opnd.getDisplacement()); \
+ name(opnd.getBaseRegister(), d2); \
+ }
+ S390_S_OPCODE_LIST(DECLARE_S390_S_INSTRUCTIONS)
+#undef DECLARE_S390_S_INSTRUCTIONS
+
+ inline void si_format(Opcode op, int f1, int f2, int f3) {
+ uint32_t code =
+ getfield<uint32_t, 4, 0, 8>(op) | getfield<uint32_t, 4, 8, 16>(f1) |
+ getfield<uint32_t, 4, 16, 20>(f2) | getfield<uint32_t, 4, 20, 32>(f3);
+ emit4bytes(code);
+ }
+
+#define DECLARE_S390_SI_INSTRUCTIONS(name, op_name, op_value) \
+ void name(const Operand& i2, Register b1, const Operand& d1) { \
+ si_format(op_name, i2.immediate(), b1.code(), d1.immediate()); \
+ } \
+ void name(const MemOperand& opnd, const Operand& i2) { \
+ name(i2, opnd.getBaseRegister(), Operand(opnd.getDisplacement())); \
+ }
+ S390_SI_OPCODE_LIST(DECLARE_S390_SI_INSTRUCTIONS)
+#undef DECLARE_S390_SI_INSTRUCTIONS
+
+ inline void siy_format(Opcode op, int f1, int f2, int f3) {
+ DCHECK(is_uint20(f3) || is_int20(f3));
+ DCHECK(is_uint16(op));
+ DCHECK(is_uint8(f1) || is_int8(f1));
+ uint64_t code = getfield<uint64_t, 6, 0, 8>(op >> 8) |
+ getfield<uint64_t, 6, 8, 16>(f1) |
+ getfield<uint64_t, 6, 16, 20>(f2) |
+ getfield<uint64_t, 6, 20, 32>(f3) |
+ getfield<uint64_t, 6, 32, 40>(f3 >> 12) |
+ getfield<uint64_t, 6, 40, 48>(op & 0x00FF);
+ emit6bytes(code);
+ }
+
+#define DECLARE_S390_SIY_INSTRUCTIONS(name, op_name, op_value) \
+ void name(const Operand& i2, Register b1, const Operand& d1) { \
+ siy_format(op_name, i2.immediate(), b1.code(), d1.immediate()); \
+ } \
+ void name(const MemOperand& opnd, const Operand& i2) { \
+ name(i2, opnd.getBaseRegister(), Operand(opnd.getDisplacement())); \
+ }
+ S390_SIY_OPCODE_LIST(DECLARE_S390_SIY_INSTRUCTIONS)
+#undef DECLARE_S390_SIY_INSTRUCTIONS
+
+ inline void rrs_format(Opcode op, int f1, int f2, int f3, int f4, int f5) {
+ DCHECK(is_uint12(f4));
+ DCHECK(is_uint16(op));
+ uint64_t code =
+ getfield<uint64_t, 6, 0, 8>(op >> 8) |
+ getfield<uint64_t, 6, 8, 12>(f1) | getfield<uint64_t, 6, 12, 16>(f2) |
+ getfield<uint64_t, 6, 16, 20>(f3) | getfield<uint64_t, 6, 20, 32>(f4) |
+ getfield<uint64_t, 6, 32, 36>(f5) |
+ getfield<uint64_t, 6, 40, 48>(op & 0x00FF);
+ emit6bytes(code);
+ }
+
+#define DECLARE_S390_RRS_INSTRUCTIONS(name, op_name, op_value) \
+ void name(Register r1, Register r2, Register b4, const Operand& d4, \
+ Condition m3) { \
+ rrs_format(op_name, r1.code(), r2.code(), b4.code(), d4.immediate(), m3); \
+ } \
+ void name(Register r1, Register r2, Condition m3, const MemOperand& opnd) { \
+ name(r1, r2, opnd.getBaseRegister(), Operand(opnd.getDisplacement()), m3); \
+ }
+ S390_RRS_OPCODE_LIST(DECLARE_S390_RRS_INSTRUCTIONS)
+#undef DECLARE_S390_RRS_INSTRUCTIONS
+
+ inline void ris_format(Opcode op, int f1, int f2, int f3, int f4, int f5) {
+ DCHECK(is_uint12(f3));
+ DCHECK(is_uint16(op));
+ DCHECK(is_uint8(f5));
+ uint64_t code =
+ getfield<uint64_t, 6, 0, 8>(op >> 8) |
+ getfield<uint64_t, 6, 8, 12>(f1) | getfield<uint64_t, 6, 12, 16>(f2) |
+ getfield<uint64_t, 6, 16, 20>(f3) | getfield<uint64_t, 6, 20, 32>(f4) |
+ getfield<uint64_t, 6, 32, 40>(f5) |
+ getfield<uint64_t, 6, 40, 48>(op & 0x00FF);
+ emit6bytes(code);
+ }
+
+#define DECLARE_S390_RIS_INSTRUCTIONS(name, op_name, op_value) \
+ void name(Register r1, Condition m3, Register b4, const Operand& d4, \
+ const Operand& i2) { \
+ ris_format(op_name, r1.code(), m3, b4.code(), d4.immediate(), \
+ i2.immediate()); \
+ } \
+ void name(Register r1, const Operand& i2, Condition m3, \
+ const MemOperand& opnd) { \
+ name(r1, m3, opnd.getBaseRegister(), Operand(opnd.getDisplacement()), i2); \
+ }
+ S390_RIS_OPCODE_LIST(DECLARE_S390_RIS_INSTRUCTIONS)
+#undef DECLARE_S390_RIS_INSTRUCTIONS
+
+ inline void sil_format(Opcode op, int f1, int f2, int f3) {
+ DCHECK(is_uint12(f2));
+ DCHECK(is_uint16(op));
+ DCHECK(is_uint16(f3));
+ uint64_t code =
+ getfield<uint64_t, 6, 0, 16>(op) | getfield<uint64_t, 6, 16, 20>(f1) |
+ getfield<uint64_t, 6, 20, 32>(f2) | getfield<uint64_t, 6, 32, 48>(f3);
+ emit6bytes(code);
+ }
+
+#define DECLARE_S390_SIL_INSTRUCTIONS(name, op_name, op_value) \
+ void name(Register b1, const Operand& d1, const Operand& i2) { \
+ sil_format(op_name, b1.code(), d1.immediate(), i2.immediate()); \
+ } \
+ void name(const MemOperand& opnd, const Operand& i2) { \
+ name(opnd.getBaseRegister(), Operand(opnd.getDisplacement()), i2); \
+ }
+ S390_SIL_OPCODE_LIST(DECLARE_S390_SIL_INSTRUCTIONS)
+#undef DECLARE_S390_SIL_INSTRUCTIONS
+
+ inline void rie_d_format(Opcode opcode, int f1, int f2, int f3, int f4) {
+ uint32_t op1 = opcode >> 8;
+ uint32_t op2 = opcode & 0xff;
+ uint64_t code =
+ getfield<uint64_t, 6, 0, 8>(op1) | getfield<uint64_t, 6, 8, 12>(f1) |
+ getfield<uint64_t, 6, 12, 16>(f2) | getfield<uint64_t, 6, 16, 32>(f3) |
+ getfield<uint64_t, 6, 32, 40>(f4) | getfield<uint64_t, 6, 40, 48>(op2);
+ emit6bytes(code);
+ }
+
+#define DECLARE_S390_RIE_D_INSTRUCTIONS(name, op_name, op_value) \
+ void name(Register r1, Register r3, const Operand& i2) { \
+ rie_d_format(op_name, r1.code(), r3.code(), i2.immediate(), 0); \
+ }
+ S390_RIE_D_OPCODE_LIST(DECLARE_S390_RIE_D_INSTRUCTIONS)
+#undef DECLARE_S390_RIE_D_INSTRUCTIONS
+
+ inline void rie_e_format(Opcode opcode, int f1, int f2, int f3) {
+ uint32_t op1 = opcode >> 8;
+ uint32_t op2 = opcode & 0xff;
+ uint64_t code =
+ getfield<uint64_t, 6, 0, 8>(op1) | getfield<uint64_t, 6, 8, 12>(f1) |
+ getfield<uint64_t, 6, 12, 16>(f2) | getfield<uint64_t, 6, 16, 32>(f3) |
+ getfield<uint64_t, 6, 40, 48>(op2);
+ emit6bytes(code);
+ }
+
+#define DECLARE_S390_RIE_E_INSTRUCTIONS(name, op_name, op_value) \
+ void name(Register r1, Register r3, const Operand& i2) { \
+ rie_e_format(op_name, r1.code(), r3.code(), i2.immediate()); \
+ }
+ S390_RIE_E_OPCODE_LIST(DECLARE_S390_RIE_E_INSTRUCTIONS)
+#undef DECLARE_S390_RIE_E_INSTRUCTIONS
+
+ inline void rie_f_format(Opcode opcode, int f1, int f2, int f3, int f4,
+ int f5) {
+ uint32_t op1 = opcode >> 8;
+ uint32_t op2 = opcode & 0xff;
+ uint64_t code =
+ getfield<uint64_t, 6, 0, 8>(op1) | getfield<uint64_t, 6, 8, 12>(f1) |
+ getfield<uint64_t, 6, 12, 16>(f2) | getfield<uint64_t, 6, 16, 24>(f3) |
+ getfield<uint64_t, 6, 24, 32>(f4) | getfield<uint64_t, 6, 32, 40>(f5) |
+ getfield<uint64_t, 6, 40, 48>(op2);
+ emit6bytes(code);
+ }
+
+#define DECLARE_S390_RIE_F_INSTRUCTIONS(name, op_name, op_value) \
+ void name(Register dst, Register src, const Operand& startBit, \
+ const Operand& endBit, const Operand& shiftAmt) { \
+ DCHECK(is_uint8(startBit.immediate())); \
+ DCHECK(is_uint8(endBit.immediate())); \
+ DCHECK(is_uint8(shiftAmt.immediate())); \
+ rie_f_format(op_name, dst.code(), src.code(), startBit.immediate(), \
+ endBit.immediate(), shiftAmt.immediate()); \
+ }
+ S390_RIE_F_OPCODE_LIST(DECLARE_S390_RIE_F_INSTRUCTIONS)
+#undef DECLARE_S390_RIE_F_INSTRUCTIONS
+
+ inline void ss_a_format(Opcode op, int f1, int f2, int f3, int f4, int f5) {
+ DCHECK(is_uint12(f5));
+ DCHECK(is_uint12(f3));
+ DCHECK(is_uint8(f1));
+ DCHECK(is_uint8(op));
+ uint64_t code =
+ getfield<uint64_t, 6, 0, 8>(op) | getfield<uint64_t, 6, 8, 16>(f1) |
+ getfield<uint64_t, 6, 16, 20>(f2) | getfield<uint64_t, 6, 20, 32>(f3) |
+ getfield<uint64_t, 6, 32, 36>(f4) | getfield<uint64_t, 6, 36, 48>(f5);
+ emit6bytes(code);
+ }
+
+#define DECLARE_S390_SS_A_INSTRUCTIONS(name, op_name, op_value) \
+ void name(Register b1, const Operand& d1, Register b2, const Operand& d2, \
+ const Operand& length) { \
+ ss_a_format(op_name, length.immediate(), b1.code(), d1.immediate(), \
+ b2.code(), d2.immediate()); \
+ } \
+ void name(const MemOperand& opnd1, const MemOperand& opnd2, \
+ const Operand& length) { \
+ ss_a_format(op_name, length.immediate(), opnd1.getBaseRegister().code(), \
+ opnd1.getDisplacement(), opnd2.getBaseRegister().code(), \
+ opnd2.getDisplacement()); \
+ }
+ S390_SS_A_OPCODE_LIST(DECLARE_S390_SS_A_INSTRUCTIONS)
+#undef DECLARE_S390_SS_A_INSTRUCTIONS
+
+ // Helper for unconditional branch to Label with update to save register
+ void b(Register r, Label* l) {
+ int32_t halfwords = branch_offset(l) / 2;
+ brasl(r, Operand(halfwords));
+ }
+
+ // Conditional Branch Instruction - Generates either BRC / BRCL
+ void branchOnCond(Condition c, int branch_offset, bool is_bound = false);
+
+ // Helpers for conditional branch to Label
+ void b(Condition cond, Label* l, Label::Distance dist = Label::kFar) {
+ branchOnCond(cond, branch_offset(l),
+ l->is_bound() || (dist == Label::kNear));
+ }
+
+ void bc_short(Condition cond, Label* l, Label::Distance dist = Label::kFar) {
+ b(cond, l, Label::kNear);
+ }
+ // Helpers for conditional branch to Label
+ void beq(Label* l, Label::Distance dist = Label::kFar) { b(eq, l, dist); }
+ void bne(Label* l, Label::Distance dist = Label::kFar) { b(ne, l, dist); }
+ void blt(Label* l, Label::Distance dist = Label::kFar) { b(lt, l, dist); }
+ void ble(Label* l, Label::Distance dist = Label::kFar) { b(le, l, dist); }
+ void bgt(Label* l, Label::Distance dist = Label::kFar) { b(gt, l, dist); }
+ void bge(Label* l, Label::Distance dist = Label::kFar) { b(ge, l, dist); }
+ void b(Label* l, Label::Distance dist = Label::kFar) { b(al, l, dist); }
+ void jmp(Label* l, Label::Distance dist = Label::kFar) { b(al, l, dist); }
+ void bunordered(Label* l, Label::Distance dist = Label::kFar) {
+ b(unordered, l, dist);
+ }
+ void bordered(Label* l, Label::Distance dist = Label::kFar) {
+ b(ordered, l, dist);
+ }
+
+ // Helpers for conditional indirect branch off register
+ void b(Condition cond, Register r) { bcr(cond, r); }
+ void beq(Register r) { b(eq, r); }
+ void bne(Register r) { b(ne, r); }
+ void blt(Register r) { b(lt, r); }
+ void ble(Register r) { b(le, r); }
+ void bgt(Register r) { b(gt, r); }
+ void bge(Register r) { b(ge, r); }
+ void b(Register r) { b(al, r); }
+ void jmp(Register r) { b(al, r); }
+ void bunordered(Register r) { b(unordered, r); }
+ void bordered(Register r) { b(ordered, r); }
+
+ // wrappers around asm instr
+ void brxh(Register dst, Register inc, Label* L) {
+ int offset_halfwords = branch_offset(L) / 2;
+ CHECK(is_int16(offset_halfwords));
+ brxh(dst, inc, Operand(offset_halfwords));
+ }
+
+ void brxhg(Register dst, Register inc, Label* L) {
+ int offset_halfwords = branch_offset(L) / 2;
+ CHECK(is_int16(offset_halfwords));
+ brxhg(dst, inc, Operand(offset_halfwords));
+ }
+
+ template <class R1, class R2>
+ void ledbr(R1 r1, R2 r2) {
+ ledbra(Condition(0), Condition(0), r1, r2);
+ }
+
+ template <class R1, class R2>
+ void cdfbr(R1 r1, R2 r2) {
+ cdfbra(Condition(0), Condition(0), r1, r2);
+ }
+
+ template <class R1, class R2>
+ void cdgbr(R1 r1, R2 r2) {
+ cdgbra(Condition(0), Condition(0), r1, r2);
+ }
+
+ template <class R1, class R2>
+ void cegbr(R1 r1, R2 r2) {
+ cegbra(Condition(0), Condition(0), r1, r2);
+ }
+
+ template <class R1, class R2>
+ void cgebr(Condition m3, R1 r1, R2 r2) {
+ cgebra(m3, Condition(0), r1, r2);
+ }
+
+ template <class R1, class R2>
+ void cgdbr(Condition m3, R1 r1, R2 r2) {
+ cgdbra(m3, Condition(0), r1, r2);
+ }
+
+ template <class R1, class R2>
+ void cfdbr(Condition m3, R1 r1, R2 r2) {
+ cfdbra(m3, Condition(0), r1, r2);
+ }
+
+ template <class R1, class R2>
+ void cfebr(Condition m3, R1 r1, R2 r2) {
+ cfebra(m3, Condition(0), r1, r2);
+ }
+
+ // ---------------------------------------------------------------------------
+ // Code generation
+
+ // Insert the smallest number of nop instructions
+ // possible to align the pc offset to a multiple
+ // of m. m must be a power of 2 (>= 4).
+ void Align(int m);
+ // Insert the smallest number of zero bytes possible to align the pc offset
+ // to a mulitple of m. m must be a power of 2 (>= 2).
+ void DataAlign(int m);
+ // Aligns code to something that's optimal for a jump target for the platform.
+ void CodeTargetAlign();
+
+ void breakpoint(bool do_print) {
+ if (do_print) {
+ PrintF("DebugBreak is inserted to %p\n", static_cast<void*>(pc_));
+ }
+#if V8_HOST_ARCH_64_BIT
+ int64_t value = reinterpret_cast<uint64_t>(&v8::base::OS::DebugBreak);
+ int32_t hi_32 = static_cast<int64_t>(value) >> 32;
+ int32_t lo_32 = static_cast<int32_t>(value);
+
+ iihf(r1, Operand(hi_32));
+ iilf(r1, Operand(lo_32));
+#else
+ iilf(r1, Operand(reinterpret_cast<uint32_t>(&v8::base::OS::DebugBreak)));
+#endif
+ basr(r14, r1);
+ }
+
+ void call(Handle<Code> target, RelocInfo::Mode rmode);
+ void jump(Handle<Code> target, RelocInfo::Mode rmode, Condition cond);
+
+// S390 instruction generation
+#define DECLARE_VRR_A_INSTRUCTIONS(name, opcode_name, opcode_value) \
+ void name(DoubleRegister v1, DoubleRegister v2, Condition m5, Condition m4, \
+ Condition m3) { \
+ uint64_t code = (static_cast<uint64_t>(opcode_value & 0xFF00)) * B32 | \
+ (static_cast<uint64_t>(v1.code())) * B36 | \
+ (static_cast<uint64_t>(v2.code())) * B32 | \
+ (static_cast<uint64_t>(m5 & 0xF)) * B20 | \
+ (static_cast<uint64_t>(m4 & 0xF)) * B16 | \
+ (static_cast<uint64_t>(m3 & 0xF)) * B12 | \
+ (static_cast<uint64_t>(0)) * B8 | \
+ (static_cast<uint64_t>(opcode_value & 0x00FF)); \
+ emit6bytes(code); \
+ }
+ S390_VRR_A_OPCODE_LIST(DECLARE_VRR_A_INSTRUCTIONS)
+#undef DECLARE_VRR_A_INSTRUCTIONS
+
+#define DECLARE_VRR_C_INSTRUCTIONS(name, opcode_name, opcode_value) \
+ void name(DoubleRegister v1, DoubleRegister v2, DoubleRegister v3, \
+ Condition m6, Condition m5, Condition m4) { \
+ uint64_t code = (static_cast<uint64_t>(opcode_value & 0xFF00)) * B32 | \
+ (static_cast<uint64_t>(v1.code())) * B36 | \
+ (static_cast<uint64_t>(v2.code())) * B32 | \
+ (static_cast<uint64_t>(v3.code())) * B28 | \
+ (static_cast<uint64_t>(m6 & 0xF)) * B20 | \
+ (static_cast<uint64_t>(m5 & 0xF)) * B16 | \
+ (static_cast<uint64_t>(m4 & 0xF)) * B12 | \
+ (static_cast<uint64_t>(0)) * B8 | \
+ (static_cast<uint64_t>(opcode_value & 0x00FF)); \
+ emit6bytes(code); \
+ }
+ S390_VRR_C_OPCODE_LIST(DECLARE_VRR_C_INSTRUCTIONS)
+#undef DECLARE_VRR_C_INSTRUCTIONS
+
+#define DECLARE_VRR_B_INSTRUCTIONS(name, opcode_name, opcode_value) \
+ void name(DoubleRegister v1, DoubleRegister v2, DoubleRegister v3, \
+ Condition m5, Condition m4) { \
+ uint64_t code = (static_cast<uint64_t>(opcode_value & 0xFF00)) * B32 | \
+ (static_cast<uint64_t>(v1.code())) * B36 | \
+ (static_cast<uint64_t>(v2.code())) * B32 | \
+ (static_cast<uint64_t>(v3.code())) * B28 | \
+ (static_cast<uint64_t>(m5 & 0xF)) * B20 | \
+ (static_cast<uint64_t>(m4 & 0xF)) * B12 | \
+ (static_cast<uint64_t>(0)) * B8 | \
+ (static_cast<uint64_t>(opcode_value & 0x00FF)); \
+ emit6bytes(code); \
+ }
+ S390_VRR_B_OPCODE_LIST(DECLARE_VRR_B_INSTRUCTIONS)
+#undef DECLARE_VRR_B_INSTRUCTIONS
+
+#define DECLARE_VRR_E_INSTRUCTIONS(name, opcode_name, opcode_value) \
+ void name(DoubleRegister v1, DoubleRegister v2, DoubleRegister v3, \
+ DoubleRegister v4, Condition m6, Condition m5) { \
+ uint64_t code = (static_cast<uint64_t>(opcode_value & 0xFF00)) * B32 | \
+ (static_cast<uint64_t>(v1.code())) * B36 | \
+ (static_cast<uint64_t>(v2.code())) * B32 | \
+ (static_cast<uint64_t>(v3.code())) * B28 | \
+ (static_cast<uint64_t>(m6 & 0xF)) * B24 | \
+ (static_cast<uint64_t>(m5 & 0xF)) * B16 | \
+ (static_cast<uint64_t>(v4.code())) * B12 | \
+ (static_cast<uint64_t>(0)) * B8 | \
+ (static_cast<uint64_t>(opcode_value & 0x00FF)); \
+ emit6bytes(code); \
+ }
+ S390_VRR_E_OPCODE_LIST(DECLARE_VRR_E_INSTRUCTIONS)
+#undef DECLARE_VRR_E_INSTRUCTIONS
+
+#define DECLARE_VRR_F_INSTRUCTIONS(name, opcode_name, opcode_value) \
+ void name(DoubleRegister v1, Register r1, Register r2) { \
+ uint64_t code = (static_cast<uint64_t>(opcode_value & 0xFF00)) * B32 | \
+ (static_cast<uint64_t>(v1.code())) * B36 | \
+ (static_cast<uint64_t>(r1.code())) * B32 | \
+ (static_cast<uint64_t>(r2.code())) * B28 | \
+ (static_cast<uint64_t>(0)) * B8 | \
+ (static_cast<uint64_t>(opcode_value & 0x00FF)); \
+ emit6bytes(code); \
+ }
+ S390_VRR_F_OPCODE_LIST(DECLARE_VRR_F_INSTRUCTIONS)
+#undef DECLARE_VRR_E_INSTRUCTIONS
+
+#define DECLARE_VRX_INSTRUCTIONS(name, opcode_name, opcode_value) \
+ void name(DoubleRegister v1, const MemOperand& opnd, Condition m3) { \
+ uint64_t code = \
+ (static_cast<uint64_t>(opcode_value & 0xFF00)) * B32 | \
+ (static_cast<uint64_t>(v1.code())) * B36 | \
+ (static_cast<uint64_t>(opnd.getIndexRegister().code())) * B32 | \
+ (static_cast<uint64_t>(opnd.getBaseRegister().code())) * B28 | \
+ (static_cast<uint64_t>(opnd.getDisplacement())) * B16 | \
+ (static_cast<uint64_t>(m3 & 0xF)) * B12 | \
+ (static_cast<uint64_t>(0)) * B8 | \
+ (static_cast<uint64_t>(opcode_value & 0x00FF)); \
+ emit6bytes(code); \
+ }
+ S390_VRX_OPCODE_LIST(DECLARE_VRX_INSTRUCTIONS)
+#undef DECLARE_VRX_INSTRUCTIONS
+
+#define DECLARE_VRS_A_INSTRUCTIONS(name, opcode_name, opcode_value) \
+ void name(DoubleRegister v1, DoubleRegister v2, const MemOperand& opnd, \
+ Condition m4 = Condition(0)) { \
+ uint64_t code = \
+ (static_cast<uint64_t>(opcode_value & 0xFF00)) * B32 | \
+ (static_cast<uint64_t>(v1.code())) * B36 | \
+ (static_cast<uint64_t>(v2.code())) * B32 | \
+ (static_cast<uint64_t>(opnd.getBaseRegister().code())) * B28 | \
+ (static_cast<uint64_t>(opnd.getDisplacement())) * B16 | \
+ (static_cast<uint64_t>(m4 & 0xF)) * B12 | \
+ (static_cast<uint64_t>(0)) * B8 | \
+ (static_cast<uint64_t>(opcode_value & 0x00FF)); \
+ emit6bytes(code); \
+ }
+ S390_VRS_A_OPCODE_LIST(DECLARE_VRS_A_INSTRUCTIONS)
+#undef DECLARE_VRS_A_INSTRUCTIONS
+
+#define DECLARE_VRS_B_INSTRUCTIONS(name, opcode_name, opcode_value) \
+ void name(DoubleRegister v1, Register r1, const MemOperand& opnd, \
+ Condition m4 = Condition(0)) { \
+ uint64_t code = \
+ (static_cast<uint64_t>(opcode_value & 0xFF00)) * B32 | \
+ (static_cast<uint64_t>(v1.code())) * B36 | \
+ (static_cast<uint64_t>(r1.code())) * B32 | \
+ (static_cast<uint64_t>(opnd.getBaseRegister().code())) * B28 | \
+ (static_cast<uint64_t>(opnd.getDisplacement())) * B16 | \
+ (static_cast<uint64_t>(m4 & 0xF)) * B12 | \
+ (static_cast<uint64_t>(0)) * B8 | \
+ (static_cast<uint64_t>(opcode_value & 0x00FF)); \
+ emit6bytes(code); \
+ }
+ S390_VRS_B_OPCODE_LIST(DECLARE_VRS_B_INSTRUCTIONS)
+#undef DECLARE_VRS_B_INSTRUCTIONS
+
+#define DECLARE_VRS_C_INSTRUCTIONS(name, opcode_name, opcode_value) \
+ void name(Register r1, DoubleRegister v1, const MemOperand& opnd, \
+ Condition m4 = Condition(0)) { \
+ uint64_t code = \
+ (static_cast<uint64_t>(opcode_value & 0xFF00)) * B32 | \
+ (static_cast<uint64_t>(r1.code())) * B36 | \
+ (static_cast<uint64_t>(v1.code())) * B32 | \
+ (static_cast<uint64_t>(opnd.getBaseRegister().code())) * B28 | \
+ (static_cast<uint64_t>(opnd.getDisplacement())) * B16 | \
+ (static_cast<uint64_t>(m4 & 0xF)) * B12 | \
+ (static_cast<uint64_t>(0)) * B8 | \
+ (static_cast<uint64_t>(opcode_value & 0x00FF)); \
+ emit6bytes(code); \
+ }
+ S390_VRS_C_OPCODE_LIST(DECLARE_VRS_C_INSTRUCTIONS)
+#undef DECLARE_VRS_C_INSTRUCTIONS
+
+#define DECLARE_VRI_A_INSTRUCTIONS(name, opcode_name, opcode_value) \
+ void name(DoubleRegister v1, const Operand& i2, Condition m3) { \
+ uint64_t code = (static_cast<uint64_t>(opcode_value & 0xFF00)) * B32 | \
+ (static_cast<uint64_t>(v1.code())) * B36 | \
+ (static_cast<uint32_t>(i2.immediate())) * B16 | \
+ (static_cast<uint64_t>(m3 & 0xF)) * B12 | \
+ (static_cast<uint64_t>(0)) * B8 | \
+ (static_cast<uint64_t>(opcode_value & 0x00FF)); \
+ emit6bytes(code); \
+ }
+ S390_VRI_A_OPCODE_LIST(DECLARE_VRI_A_INSTRUCTIONS)
+#undef DECLARE_VRI_A_INSTRUCTIONS
+
+#define DECLARE_VRI_C_INSTRUCTIONS(name, opcode_name, opcode_value) \
+ void name(DoubleRegister v1, DoubleRegister v2, const Operand& i2, \
+ Condition m4) { \
+ uint64_t code = (static_cast<uint64_t>(opcode_value & 0xFF00)) * B32 | \
+ (static_cast<uint64_t>(v1.code())) * B36 | \
+ (static_cast<uint64_t>(v2.code())) * B32 | \
+ (static_cast<uint16_t>(i2.immediate())) * B16 | \
+ (static_cast<uint64_t>(m4 & 0xF)) * B12 | \
+ (static_cast<uint64_t>(0)) * B8 | \
+ (static_cast<uint64_t>(opcode_value & 0x00FF)); \
+ emit6bytes(code); \
+ }
+ S390_VRI_C_OPCODE_LIST(DECLARE_VRI_C_INSTRUCTIONS)
+#undef DECLARE_VRI_C_INSTRUCTIONS
+
+ // Single Element format
+ void vfa(DoubleRegister v1, DoubleRegister v2, DoubleRegister v3) {
+ vfa(v1, v2, v3, static_cast<Condition>(0), static_cast<Condition>(8),
+ static_cast<Condition>(3));
+ }
+ void vfs(DoubleRegister v1, DoubleRegister v2, DoubleRegister v3) {
+ vfs(v1, v2, v3, static_cast<Condition>(0), static_cast<Condition>(8),
+ static_cast<Condition>(3));
+ }
+ void vfm(DoubleRegister v1, DoubleRegister v2, DoubleRegister v3) {
+ vfm(v1, v2, v3, static_cast<Condition>(0), static_cast<Condition>(8),
+ static_cast<Condition>(3));
+ }
+ void vfd(DoubleRegister v1, DoubleRegister v2, DoubleRegister v3) {
+ vfd(v1, v2, v3, static_cast<Condition>(0), static_cast<Condition>(8),
+ static_cast<Condition>(3));
+ }
+
+ // Load Address Instructions
+ void larl(Register r, Label* l);
+
+ // Exception-generating instructions and debugging support
+ void stop(Condition cond = al, int32_t code = kDefaultStopCode,
+ CRegister cr = cr7);
+
+ void bkpt(uint32_t imm16); // v5 and above
+
+ // Different nop operations are used by the code generator to detect certain
+ // states of the generated code.
+ enum NopMarkerTypes {
+ NON_MARKING_NOP = 0,
+ GROUP_ENDING_NOP,
+ DEBUG_BREAK_NOP,
+ // IC markers.
+ PROPERTY_ACCESS_INLINED,
+ PROPERTY_ACCESS_INLINED_CONTEXT,
+ PROPERTY_ACCESS_INLINED_CONTEXT_DONT_DELETE,
+ // Helper values.
+ LAST_CODE_MARKER,
+ FIRST_IC_MARKER = PROPERTY_ACCESS_INLINED
+ };
+
+ void nop(int type = 0); // 0 is the default non-marking type.
+
+ void dumy(int r1, int x2, int b2, int d2);
+
+ // Check the code size generated from label to here.
+ int SizeOfCodeGeneratedSince(Label* label) {
+ return pc_offset() - label->pos();
+ }
+
+ // Record a deoptimization reason that can be used by a log or cpu profiler.
+ // Use --trace-deopt to enable.
+ void RecordDeoptReason(DeoptimizeReason reason, SourcePosition position,
+ int id);
+
+ // Writes a single byte or word of data in the code stream. Used
+ // for inline tables, e.g., jump-tables.
+ void db(uint8_t data);
+ void dd(uint32_t data);
+ void dq(uint64_t data);
+ void dp(uintptr_t data);
+
+ // Read/patch instructions
+ SixByteInstr instr_at(int pos) {
+ return Instruction::InstructionBits(buffer_start_ + pos);
+ }
+ template <typename T>
+ void instr_at_put(int pos, T instr) {
+ Instruction::SetInstructionBits<T>(buffer_start_ + pos, instr);
+ }
+
+ // Decodes instruction at pos, and returns its length
+ int32_t instr_length_at(int pos) {
+ return Instruction::InstructionLength(buffer_start_ + pos);
+ }
+
+ static SixByteInstr instr_at(byte* pc) {
+ return Instruction::InstructionBits(pc);
+ }
+
+ static Condition GetCondition(Instr instr);
+
+ static bool IsBranch(Instr instr);
+#if V8_TARGET_ARCH_S390X
+ static bool Is64BitLoadIntoIP(SixByteInstr instr1, SixByteInstr instr2);
+#else
+ static bool Is32BitLoadIntoIP(SixByteInstr instr);
+#endif
+
+ static bool IsCmpRegister(Instr instr);
+ static bool IsCmpImmediate(Instr instr);
+ static bool IsNop(SixByteInstr instr, int type = NON_MARKING_NOP);
+
+ // The code currently calls CheckBuffer() too often. This has the side
+ // effect of randomly growing the buffer in the middle of multi-instruction
+ // sequences.
+ //
+ // This function allows outside callers to check and grow the buffer
+ void EnsureSpaceFor(int space_needed);
+
+ void EmitRelocations();
+ void emit_label_addr(Label* label);
+
+ public:
+ byte* buffer_pos() const { return buffer_start_; }
+
+ protected:
+ int buffer_space() const { return reloc_info_writer.pos() - pc_; }
+
+ // Decode instruction(s) at pos and return backchain to previous
+ // label reference or kEndOfChain.
+ int target_at(int pos);
+
+ // Patch instruction(s) at pos to target target_pos (e.g. branch)
+ void target_at_put(int pos, int target_pos, bool* is_branch = nullptr);
+
+ // Record reloc info for current pc_
+ void RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data = 0);
+
+ private:
+ // Avoid overflows for displacements etc.
+ static const int kMaximalBufferSize = 512 * MB;
+
+ // Code generation
+ // The relocation writer's position is at least kGap bytes below the end of
+ // the generated instructions. This is so that multi-instruction sequences do
+ // not have to check for overflow. The same is true for writes of large
+ // relocation info entries.
+ static constexpr int kGap = 32;
+ STATIC_ASSERT(AssemblerBase::kMinimalBufferSize >= 2 * kGap);
+
+ // Relocation info generation
+ // Each relocation is encoded as a variable size value
+ static constexpr int kMaxRelocSize = RelocInfoWriter::kMaxSize;
+ RelocInfoWriter reloc_info_writer;
+ std::vector<DeferredRelocInfo> relocations_;
+
+ // Scratch registers available for use by the Assembler.
+ RegList scratch_register_list_;
+
+ // The bound position, before this we cannot do instruction elimination.
+ int last_bound_pos_;
+
+ // Code emission
+ void CheckBuffer() {
+ if (buffer_space() <= kGap) {
+ GrowBuffer();
+ }
+ }
+ void GrowBuffer(int needed = 0);
+ inline void TrackBranch();
+ inline void UntrackBranch();
+
+ // Helper to emit the binary encoding of a 2 byte instruction
+ void emit2bytes(uint16_t x) {
+ CheckBuffer();
+#if V8_TARGET_LITTLE_ENDIAN
+ // We need to emit instructions in big endian format as disassembler /
+ // simulator require the first byte of the instruction in order to decode
+ // the instruction length. Swap the bytes.
+ x = ((x & 0x00FF) << 8) | ((x & 0xFF00) >> 8);
+#endif
+ *reinterpret_cast<uint16_t*>(pc_) = x;
+ pc_ += 2;
+ }
+
+ // Helper to emit the binary encoding of a 4 byte instruction
+ void emit4bytes(uint32_t x) {
+ CheckBuffer();
+#if V8_TARGET_LITTLE_ENDIAN
+ // We need to emit instructions in big endian format as disassembler /
+ // simulator require the first byte of the instruction in order to decode
+ // the instruction length. Swap the bytes.
+ x = ((x & 0x000000FF) << 24) | ((x & 0x0000FF00) << 8) |
+ ((x & 0x00FF0000) >> 8) | ((x & 0xFF000000) >> 24);
+#endif
+ *reinterpret_cast<uint32_t*>(pc_) = x;
+ pc_ += 4;
+ }
+
+ // Helper to emit the binary encoding of a 6 byte instruction
+ void emit6bytes(uint64_t x) {
+ CheckBuffer();
+#if V8_TARGET_LITTLE_ENDIAN
+ // We need to emit instructions in big endian format as disassembler /
+ // simulator require the first byte of the instruction in order to decode
+ // the instruction length. Swap the bytes.
+ x = (static_cast<uint64_t>(x & 0xFF) << 40) |
+ (static_cast<uint64_t>((x >> 8) & 0xFF) << 32) |
+ (static_cast<uint64_t>((x >> 16) & 0xFF) << 24) |
+ (static_cast<uint64_t>((x >> 24) & 0xFF) << 16) |
+ (static_cast<uint64_t>((x >> 32) & 0xFF) << 8) |
+ (static_cast<uint64_t>((x >> 40) & 0xFF));
+ x |= (*reinterpret_cast<uint64_t*>(pc_) >> 48) << 48;
+#else
+ // We need to pad two bytes of zeros in order to get the 6-bytes
+ // stored from low address.
+ x = x << 16;
+ x |= *reinterpret_cast<uint64_t*>(pc_) & 0xFFFF;
+#endif
+ // It is safe to store 8-bytes, as CheckBuffer() guarantees we have kGap
+ // space left over.
+ *reinterpret_cast<uint64_t*>(pc_) = x;
+ pc_ += 6;
+ }
+
+ // Labels
+ void print(Label* L);
+ int max_reach_from(int pos);
+ void bind_to(Label* L, int pos);
+ void next(Label* L);
+
+ void AllocateAndInstallRequestedHeapObjects(Isolate* isolate);
+
+ int WriteCodeComments();
+
+ friend class RegExpMacroAssemblerS390;
+ friend class RelocInfo;
+ friend class EnsureSpace;
+ friend class UseScratchRegisterScope;
+};
+
+class EnsureSpace {
+ public:
+ explicit EnsureSpace(Assembler* assembler) { assembler->CheckBuffer(); }
+};
+
+class V8_EXPORT_PRIVATE UseScratchRegisterScope {
+ public:
+ explicit UseScratchRegisterScope(Assembler* assembler);
+ ~UseScratchRegisterScope();
+
+ Register Acquire();
+
+ // Check if we have registers available to acquire.
+ bool CanAcquire() const { return *assembler_->GetScratchRegisterList() != 0; }
+
+ private:
+ friend class Assembler;
+ friend class TurboAssembler;
+
+ Assembler* assembler_;
+ RegList old_available_;
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_S390_ASSEMBLER_S390_H_
diff --git a/src/codegen/s390/constants-s390.cc b/src/codegen/s390/constants-s390.cc
new file mode 100644
index 0000000..81036c2
--- /dev/null
+++ b/src/codegen/s390/constants-s390.cc
@@ -0,0 +1,309 @@
+// Copyright 2015 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#if V8_TARGET_ARCH_S390
+
+#include "src/codegen/s390/constants-s390.h"
+
+namespace v8 {
+namespace internal {
+
+Instruction::OpcodeFormatType Instruction::OpcodeFormatTable[] = {
+ // Based on Figure B-3 in z/Architecture Principles of
+ // Operation.
+ TWO_BYTE_OPCODE, // 0x00
+ TWO_BYTE_OPCODE, // 0x01
+ TWO_BYTE_DISJOINT_OPCODE, // 0x02
+ TWO_BYTE_DISJOINT_OPCODE, // 0x03
+ ONE_BYTE_OPCODE, // 0x04
+ ONE_BYTE_OPCODE, // 0x05
+ ONE_BYTE_OPCODE, // 0x06
+ ONE_BYTE_OPCODE, // 0x07
+ ONE_BYTE_OPCODE, // 0x08
+ ONE_BYTE_OPCODE, // 0x09
+ ONE_BYTE_OPCODE, // 0x0A
+ ONE_BYTE_OPCODE, // 0x0B
+ ONE_BYTE_OPCODE, // 0x0C
+ ONE_BYTE_OPCODE, // 0x0D
+ ONE_BYTE_OPCODE, // 0x0E
+ ONE_BYTE_OPCODE, // 0x0F
+ ONE_BYTE_OPCODE, // 0x10
+ ONE_BYTE_OPCODE, // 0x11
+ ONE_BYTE_OPCODE, // 0x12
+ ONE_BYTE_OPCODE, // 0x13
+ ONE_BYTE_OPCODE, // 0x14
+ ONE_BYTE_OPCODE, // 0x15
+ ONE_BYTE_OPCODE, // 0x16
+ ONE_BYTE_OPCODE, // 0x17
+ ONE_BYTE_OPCODE, // 0x18
+ ONE_BYTE_OPCODE, // 0x19
+ ONE_BYTE_OPCODE, // 0x1A
+ ONE_BYTE_OPCODE, // 0x1B
+ ONE_BYTE_OPCODE, // 0x1C
+ ONE_BYTE_OPCODE, // 0x1D
+ ONE_BYTE_OPCODE, // 0x1E
+ ONE_BYTE_OPCODE, // 0x1F
+ ONE_BYTE_OPCODE, // 0x20
+ ONE_BYTE_OPCODE, // 0x21
+ ONE_BYTE_OPCODE, // 0x22
+ ONE_BYTE_OPCODE, // 0x23
+ ONE_BYTE_OPCODE, // 0x24
+ ONE_BYTE_OPCODE, // 0x25
+ ONE_BYTE_OPCODE, // 0x26
+ ONE_BYTE_OPCODE, // 0x27
+ ONE_BYTE_OPCODE, // 0x28
+ ONE_BYTE_OPCODE, // 0x29
+ ONE_BYTE_OPCODE, // 0x2A
+ ONE_BYTE_OPCODE, // 0x2B
+ ONE_BYTE_OPCODE, // 0x2C
+ ONE_BYTE_OPCODE, // 0x2D
+ ONE_BYTE_OPCODE, // 0x2E
+ ONE_BYTE_OPCODE, // 0x2F
+ ONE_BYTE_OPCODE, // 0x30
+ ONE_BYTE_OPCODE, // 0x31
+ ONE_BYTE_OPCODE, // 0x32
+ ONE_BYTE_OPCODE, // 0x33
+ ONE_BYTE_OPCODE, // 0x34
+ ONE_BYTE_OPCODE, // 0x35
+ ONE_BYTE_OPCODE, // 0x36
+ ONE_BYTE_OPCODE, // 0x37
+ ONE_BYTE_OPCODE, // 0x38
+ ONE_BYTE_OPCODE, // 0x39
+ ONE_BYTE_OPCODE, // 0x3A
+ ONE_BYTE_OPCODE, // 0x3B
+ ONE_BYTE_OPCODE, // 0x3C
+ ONE_BYTE_OPCODE, // 0x3D
+ ONE_BYTE_OPCODE, // 0x3E
+ ONE_BYTE_OPCODE, // 0x3F
+ ONE_BYTE_OPCODE, // 0x40
+ ONE_BYTE_OPCODE, // 0x41
+ ONE_BYTE_OPCODE, // 0x42
+ ONE_BYTE_OPCODE, // 0x43
+ ONE_BYTE_OPCODE, // 0x44
+ ONE_BYTE_OPCODE, // 0x45
+ ONE_BYTE_OPCODE, // 0x46
+ ONE_BYTE_OPCODE, // 0x47
+ ONE_BYTE_OPCODE, // 0x48
+ ONE_BYTE_OPCODE, // 0x49
+ ONE_BYTE_OPCODE, // 0x4A
+ ONE_BYTE_OPCODE, // 0x4B
+ ONE_BYTE_OPCODE, // 0x4C
+ ONE_BYTE_OPCODE, // 0x4D
+ ONE_BYTE_OPCODE, // 0x4E
+ ONE_BYTE_OPCODE, // 0x4F
+ ONE_BYTE_OPCODE, // 0x50
+ ONE_BYTE_OPCODE, // 0x51
+ ONE_BYTE_OPCODE, // 0x52
+ ONE_BYTE_OPCODE, // 0x53
+ ONE_BYTE_OPCODE, // 0x54
+ ONE_BYTE_OPCODE, // 0x55
+ ONE_BYTE_OPCODE, // 0x56
+ ONE_BYTE_OPCODE, // 0x57
+ ONE_BYTE_OPCODE, // 0x58
+ ONE_BYTE_OPCODE, // 0x59
+ ONE_BYTE_OPCODE, // 0x5A
+ ONE_BYTE_OPCODE, // 0x5B
+ ONE_BYTE_OPCODE, // 0x5C
+ ONE_BYTE_OPCODE, // 0x5D
+ ONE_BYTE_OPCODE, // 0x5E
+ ONE_BYTE_OPCODE, // 0x5F
+ ONE_BYTE_OPCODE, // 0x60
+ ONE_BYTE_OPCODE, // 0x61
+ ONE_BYTE_OPCODE, // 0x62
+ ONE_BYTE_OPCODE, // 0x63
+ ONE_BYTE_OPCODE, // 0x64
+ ONE_BYTE_OPCODE, // 0x65
+ ONE_BYTE_OPCODE, // 0x66
+ ONE_BYTE_OPCODE, // 0x67
+ ONE_BYTE_OPCODE, // 0x68
+ ONE_BYTE_OPCODE, // 0x69
+ ONE_BYTE_OPCODE, // 0x6A
+ ONE_BYTE_OPCODE, // 0x6B
+ ONE_BYTE_OPCODE, // 0x6C
+ ONE_BYTE_OPCODE, // 0x6D
+ ONE_BYTE_OPCODE, // 0x6E
+ ONE_BYTE_OPCODE, // 0x6F
+ ONE_BYTE_OPCODE, // 0x70
+ ONE_BYTE_OPCODE, // 0x71
+ ONE_BYTE_OPCODE, // 0x72
+ ONE_BYTE_OPCODE, // 0x73
+ ONE_BYTE_OPCODE, // 0x74
+ ONE_BYTE_OPCODE, // 0x75
+ ONE_BYTE_OPCODE, // 0x76
+ ONE_BYTE_OPCODE, // 0x77
+ ONE_BYTE_OPCODE, // 0x78
+ ONE_BYTE_OPCODE, // 0x79
+ ONE_BYTE_OPCODE, // 0x7A
+ ONE_BYTE_OPCODE, // 0x7B
+ ONE_BYTE_OPCODE, // 0x7C
+ ONE_BYTE_OPCODE, // 0x7D
+ ONE_BYTE_OPCODE, // 0x7E
+ ONE_BYTE_OPCODE, // 0x7F
+ ONE_BYTE_OPCODE, // 0x80
+ ONE_BYTE_OPCODE, // 0x81
+ ONE_BYTE_OPCODE, // 0x82
+ ONE_BYTE_OPCODE, // 0x83
+ ONE_BYTE_OPCODE, // 0x84
+ ONE_BYTE_OPCODE, // 0x85
+ ONE_BYTE_OPCODE, // 0x86
+ ONE_BYTE_OPCODE, // 0x87
+ ONE_BYTE_OPCODE, // 0x88
+ ONE_BYTE_OPCODE, // 0x89
+ ONE_BYTE_OPCODE, // 0x8A
+ ONE_BYTE_OPCODE, // 0x8B
+ ONE_BYTE_OPCODE, // 0x8C
+ ONE_BYTE_OPCODE, // 0x8D
+ ONE_BYTE_OPCODE, // 0x8E
+ ONE_BYTE_OPCODE, // 0x8F
+ ONE_BYTE_OPCODE, // 0x90
+ ONE_BYTE_OPCODE, // 0x91
+ ONE_BYTE_OPCODE, // 0x92
+ ONE_BYTE_OPCODE, // 0x93
+ ONE_BYTE_OPCODE, // 0x94
+ ONE_BYTE_OPCODE, // 0x95
+ ONE_BYTE_OPCODE, // 0x96
+ ONE_BYTE_OPCODE, // 0x97
+ ONE_BYTE_OPCODE, // 0x98
+ ONE_BYTE_OPCODE, // 0x99
+ ONE_BYTE_OPCODE, // 0x9A
+ ONE_BYTE_OPCODE, // 0x9B
+ TWO_BYTE_DISJOINT_OPCODE, // 0x9C
+ TWO_BYTE_DISJOINT_OPCODE, // 0x9D
+ TWO_BYTE_DISJOINT_OPCODE, // 0x9E
+ TWO_BYTE_DISJOINT_OPCODE, // 0x9F
+ TWO_BYTE_DISJOINT_OPCODE, // 0xA0
+ TWO_BYTE_DISJOINT_OPCODE, // 0xA1
+ TWO_BYTE_DISJOINT_OPCODE, // 0xA2
+ TWO_BYTE_DISJOINT_OPCODE, // 0xA3
+ TWO_BYTE_DISJOINT_OPCODE, // 0xA4
+ THREE_NIBBLE_OPCODE, // 0xA5
+ TWO_BYTE_DISJOINT_OPCODE, // 0xA6
+ THREE_NIBBLE_OPCODE, // 0xA7
+ ONE_BYTE_OPCODE, // 0xA8
+ ONE_BYTE_OPCODE, // 0xA9
+ ONE_BYTE_OPCODE, // 0xAA
+ ONE_BYTE_OPCODE, // 0xAB
+ ONE_BYTE_OPCODE, // 0xAC
+ ONE_BYTE_OPCODE, // 0xAD
+ ONE_BYTE_OPCODE, // 0xAE
+ ONE_BYTE_OPCODE, // 0xAF
+ ONE_BYTE_OPCODE, // 0xB0
+ ONE_BYTE_OPCODE, // 0xB1
+ TWO_BYTE_OPCODE, // 0xB2
+ TWO_BYTE_OPCODE, // 0xB3
+ TWO_BYTE_DISJOINT_OPCODE, // 0xB4
+ TWO_BYTE_DISJOINT_OPCODE, // 0xB5
+ TWO_BYTE_DISJOINT_OPCODE, // 0xB6
+ TWO_BYTE_DISJOINT_OPCODE, // 0xB7
+ TWO_BYTE_DISJOINT_OPCODE, // 0xB8
+ TWO_BYTE_OPCODE, // 0xB9
+ ONE_BYTE_OPCODE, // 0xBA
+ ONE_BYTE_OPCODE, // 0xBB
+ ONE_BYTE_OPCODE, // 0xBC
+ ONE_BYTE_OPCODE, // 0xBD
+ ONE_BYTE_OPCODE, // 0xBE
+ ONE_BYTE_OPCODE, // 0xBF
+ THREE_NIBBLE_OPCODE, // 0xC0
+ THREE_NIBBLE_OPCODE, // 0xC1
+ THREE_NIBBLE_OPCODE, // 0xC2
+ THREE_NIBBLE_OPCODE, // 0xC3
+ THREE_NIBBLE_OPCODE, // 0xC4
+ THREE_NIBBLE_OPCODE, // 0xC5
+ THREE_NIBBLE_OPCODE, // 0xC6
+ ONE_BYTE_OPCODE, // 0xC7
+ THREE_NIBBLE_OPCODE, // 0xC8
+ THREE_NIBBLE_OPCODE, // 0xC9
+ THREE_NIBBLE_OPCODE, // 0xCA
+ THREE_NIBBLE_OPCODE, // 0xCB
+ THREE_NIBBLE_OPCODE, // 0xCC
+ TWO_BYTE_DISJOINT_OPCODE, // 0xCD
+ TWO_BYTE_DISJOINT_OPCODE, // 0xCE
+ TWO_BYTE_DISJOINT_OPCODE, // 0xCF
+ ONE_BYTE_OPCODE, // 0xD0
+ ONE_BYTE_OPCODE, // 0xD1
+ ONE_BYTE_OPCODE, // 0xD2
+ ONE_BYTE_OPCODE, // 0xD3
+ ONE_BYTE_OPCODE, // 0xD4
+ ONE_BYTE_OPCODE, // 0xD5
+ ONE_BYTE_OPCODE, // 0xD6
+ ONE_BYTE_OPCODE, // 0xD7
+ ONE_BYTE_OPCODE, // 0xD8
+ ONE_BYTE_OPCODE, // 0xD9
+ ONE_BYTE_OPCODE, // 0xDA
+ ONE_BYTE_OPCODE, // 0xDB
+ ONE_BYTE_OPCODE, // 0xDC
+ ONE_BYTE_OPCODE, // 0xDD
+ ONE_BYTE_OPCODE, // 0xDE
+ ONE_BYTE_OPCODE, // 0xDF
+ ONE_BYTE_OPCODE, // 0xE0
+ ONE_BYTE_OPCODE, // 0xE1
+ ONE_BYTE_OPCODE, // 0xE2
+ TWO_BYTE_DISJOINT_OPCODE, // 0xE3
+ TWO_BYTE_DISJOINT_OPCODE, // 0xE4
+ TWO_BYTE_OPCODE, // 0xE5
+ TWO_BYTE_DISJOINT_OPCODE, // 0xE6
+ TWO_BYTE_DISJOINT_OPCODE, // 0xE7
+ ONE_BYTE_OPCODE, // 0xE8
+ ONE_BYTE_OPCODE, // 0xE9
+ ONE_BYTE_OPCODE, // 0xEA
+ TWO_BYTE_DISJOINT_OPCODE, // 0xEB
+ TWO_BYTE_DISJOINT_OPCODE, // 0xEC
+ TWO_BYTE_DISJOINT_OPCODE, // 0xED
+ ONE_BYTE_OPCODE, // 0xEE
+ ONE_BYTE_OPCODE, // 0xEF
+ ONE_BYTE_OPCODE, // 0xF0
+ ONE_BYTE_OPCODE, // 0xF1
+ ONE_BYTE_OPCODE, // 0xF2
+ ONE_BYTE_OPCODE, // 0xF3
+ ONE_BYTE_OPCODE, // 0xF4
+ ONE_BYTE_OPCODE, // 0xF5
+ ONE_BYTE_OPCODE, // 0xF6
+ ONE_BYTE_OPCODE, // 0xF7
+ ONE_BYTE_OPCODE, // 0xF8
+ ONE_BYTE_OPCODE, // 0xF9
+ ONE_BYTE_OPCODE, // 0xFA
+ ONE_BYTE_OPCODE, // 0xFB
+ ONE_BYTE_OPCODE, // 0xFC
+ ONE_BYTE_OPCODE, // 0xFD
+ TWO_BYTE_DISJOINT_OPCODE, // 0xFE
+ TWO_BYTE_DISJOINT_OPCODE, // 0xFF
+};
+
+// These register names are defined in a way to match the native disassembler
+// formatting. See for example the command "objdump -d <binary file>".
+const char* Registers::names_[kNumRegisters] = {
+ "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7",
+ "r8", "r9", "r10", "fp", "ip", "r13", "r14", "sp"};
+
+const char* DoubleRegisters::names_[kNumDoubleRegisters] = {
+ "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7",
+ "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15"};
+
+int DoubleRegisters::Number(const char* name) {
+ for (int i = 0; i < kNumDoubleRegisters; i++) {
+ if (strcmp(names_[i], name) == 0) {
+ return i;
+ }
+ }
+
+ // No register with the requested name found.
+ return kNoRegister;
+}
+
+int Registers::Number(const char* name) {
+ // Look through the canonical names.
+ for (int i = 0; i < kNumRegisters; i++) {
+ if (strcmp(names_[i], name) == 0) {
+ return i;
+ }
+ }
+
+ // No register with the requested name found.
+ return kNoRegister;
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_S390
diff --git a/src/codegen/s390/constants-s390.h b/src/codegen/s390/constants-s390.h
new file mode 100644
index 0000000..5c52435
--- /dev/null
+++ b/src/codegen/s390/constants-s390.h
@@ -0,0 +1,2400 @@
+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_S390_CONSTANTS_S390_H_
+#define V8_CODEGEN_S390_CONSTANTS_S390_H_
+
+// Get the standard printf format macros for C99 stdint types.
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+#include <inttypes.h>
+
+#include <stdint.h>
+
+#include "src/base/logging.h"
+#include "src/base/macros.h"
+#include "src/common/globals.h"
+
+// UNIMPLEMENTED_ macro for S390.
+#ifdef DEBUG
+#define UNIMPLEMENTED_S390() \
+ v8::internal::PrintF("%s, \tline %d: \tfunction %s not implemented. \n", \
+ __FILE__, __LINE__, __func__)
+#else
+#define UNIMPLEMENTED_S390()
+#endif
+
+namespace v8 {
+namespace internal {
+
+constexpr size_t kMaxPCRelativeCodeRangeInMB = 4096;
+
+// Number of registers
+const int kNumRegisters = 16;
+
+// FP support.
+const int kNumDoubleRegisters = 16;
+
+const int kNoRegister = -1;
+
+// Actual value of root register is offset from the root array's start
+// to take advantage of negative displacement values.
+// TODO(sigurds): Choose best value.
+constexpr int kRootRegisterBias = 128;
+
+// sign-extend the least significant 16-bits of value <imm>
+#define SIGN_EXT_IMM16(imm) ((static_cast<int>(imm) << 16) >> 16)
+
+// sign-extend the least significant 26-bits of value <imm>
+#define SIGN_EXT_IMM26(imm) ((static_cast<int>(imm) << 6) >> 6)
+
+// -----------------------------------------------------------------------------
+// Conditions.
+
+// Defines constants and accessor classes to assemble, disassemble and
+// simulate z/Architecture instructions.
+//
+// Section references in the code refer to the "z/Architecture Principles
+// Of Operation" http://publibfi.boulder.ibm.com/epubs/pdf/dz9zr009.pdf
+//
+
+// Constants for specific fields are defined in their respective named enums.
+// General constants are in an anonymous enum in class Instr.
+enum Condition {
+ kNoCondition = -1,
+ eq = 0x8, // Equal.
+ ne = 0x7, // Not equal.
+ ge = 0xa, // Greater or equal.
+ lt = 0x4, // Less than.
+ gt = 0x2, // Greater than.
+ le = 0xc, // Less then or equal
+ al = 0xf, // Always.
+
+ CC_NOP = 0x0, // S390 NOP
+ CC_EQ = 0x08, // S390 condition code 0b1000
+ CC_LT = 0x04, // S390 condition code 0b0100
+ CC_LE = CC_EQ | CC_LT, // S390 condition code 0b1100
+ CC_GT = 0x02, // S390 condition code 0b0010
+ CC_GE = CC_EQ | CC_GT, // S390 condition code 0b1010
+ CC_OF = 0x01, // S390 condition code 0b0001
+ CC_NOF = 0x0E, // S390 condition code 0b1110
+ CC_ALWAYS = 0x0F, // S390 always taken branch
+ unordered = CC_OF, // Floating-point unordered
+ ordered = CC_NOF, // floating-point ordered
+ overflow = CC_OF, // Summary overflow
+ nooverflow = CC_NOF,
+
+ mask0x0 = 0, // no jumps
+ mask0x1 = 1,
+ mask0x2 = 2,
+ mask0x3 = 3,
+ mask0x4 = 4,
+ mask0x5 = 5,
+ mask0x6 = 6,
+ mask0x7 = 7,
+ mask0x8 = 8,
+ mask0x9 = 9,
+ mask0xA = 10,
+ mask0xB = 11,
+ mask0xC = 12,
+ mask0xD = 13,
+ mask0xE = 14,
+ mask0xF = 15,
+
+ // Rounding modes for floating poing facility
+ CURRENT_ROUNDING_MODE = 0,
+ ROUND_TO_NEAREST_WITH_TIES_AWAY_FROM_0 = 1,
+ ROUND_TO_PREPARE_FOR_SHORTER_PRECISION = 3,
+ ROUND_TO_NEAREST_WITH_TIES_TO_EVEN = 4,
+ ROUND_TOWARD_0 = 5,
+ ROUND_TOWARD_PLUS_INFINITE = 6,
+ ROUND_TOWARD_MINUS_INFINITE = 7
+};
+
+inline Condition NegateCondition(Condition cond) {
+ DCHECK(cond != al);
+ switch (cond) {
+ case eq:
+ return ne;
+ case ne:
+ return eq;
+ case ge:
+ return lt;
+ case gt:
+ return le;
+ case le:
+ return gt;
+ case lt:
+ return ge;
+ case lt | gt:
+ return eq;
+ case le | ge:
+ return CC_OF;
+ case CC_OF:
+ return CC_NOF;
+ default:
+ DCHECK(false);
+ }
+ return al;
+}
+
+// -----------------------------------------------------------------------------
+// Instructions encoding.
+
+// Instr is merely used by the Assembler to distinguish 32bit integers
+// representing instructions from usual 32 bit values.
+// Instruction objects are pointers to 32bit values, and provide methods to
+// access the various ISA fields.
+using Instr = int32_t;
+using TwoByteInstr = uint16_t;
+using FourByteInstr = uint32_t;
+using SixByteInstr = uint64_t;
+
+#define S390_RSY_A_OPCODE_LIST(V) \
+ V(lmg, LMG, 0xEB04) /* type = RSY_A LOAD MULTIPLE (64) */ \
+ V(srag, SRAG, 0xEB0A) /* type = RSY_A SHIFT RIGHT SINGLE (64) */ \
+ V(slag, SLAG, 0xEB0B) /* type = RSY_A SHIFT LEFT SINGLE (64) */ \
+ V(srlg, SRLG, 0xEB0C) /* type = RSY_A SHIFT RIGHT SINGLE LOGICAL (64) */ \
+ V(sllg, SLLG, 0xEB0D) /* type = RSY_A SHIFT LEFT SINGLE LOGICAL (64) */ \
+ V(tracg, TRACG, 0xEB0F) /* type = RSY_A TRACE (64) */ \
+ V(csy, CSY, 0xEB14) /* type = RSY_A COMPARE AND SWAP (32) */ \
+ V(rllg, RLLG, 0xEB1C) /* type = RSY_A ROTATE LEFT SINGLE LOGICAL (64) */ \
+ V(rll, RLL, 0xEB1D) /* type = RSY_A ROTATE LEFT SINGLE LOGICAL (32) */ \
+ V(stmg, STMG, 0xEB24) /* type = RSY_A STORE MULTIPLE (64) */ \
+ V(stctg, STCTG, 0xEB25) /* type = RSY_A STORE CONTROL (64) */ \
+ V(stmh, STMH, 0xEB26) /* type = RSY_A STORE MULTIPLE HIGH (32) */ \
+ V(lctlg, LCTLG, 0xEB2F) /* type = RSY_A LOAD CONTROL (64) */ \
+ V(csg, CSG, 0xEB30) /* type = RSY_A COMPARE AND SWAP (64) */ \
+ V(cdsy, CDSY, 0xEB31) /* type = RSY_A COMPARE DOUBLE AND SWAP (32) */ \
+ V(cdsg, CDSG, 0xEB3E) /* type = RSY_A COMPARE DOUBLE AND SWAP (64) */ \
+ V(bxhg, BXHG, 0xEB44) /* type = RSY_A BRANCH ON INDEX HIGH (64) */ \
+ V(bxleg, BXLEG, 0xEB45) /* type = RSY_A BRANCH ON INDEX LOW OR EQUAL (64) */ \
+ V(ecag, ECAG, 0xEB4C) /* type = RSY_A EXTRACT CPU ATTRIBUTE */ \
+ V(mvclu, MVCLU, 0xEB8E) /* type = RSY_A MOVE LONG UNICODE */ \
+ V(clclu, CLCLU, 0xEB8F) /* type = RSY_A COMPARE LOGICAL LONG UNICODE */ \
+ V(stmy, STMY, 0xEB90) /* type = RSY_A STORE MULTIPLE (32) */ \
+ V(lmh, LMH, 0xEB96) /* type = RSY_A LOAD MULTIPLE HIGH (32) */ \
+ V(lmy, LMY, 0xEB98) /* type = RSY_A LOAD MULTIPLE (32) */ \
+ V(lamy, LAMY, 0xEB9A) /* type = RSY_A LOAD ACCESS MULTIPLE */ \
+ V(stamy, STAMY, 0xEB9B) /* type = RSY_A STORE ACCESS MULTIPLE */ \
+ V(srak, SRAK, 0xEBDC) /* type = RSY_A SHIFT RIGHT SINGLE (32) */ \
+ V(slak, SLAK, 0xEBDD) /* type = RSY_A SHIFT LEFT SINGLE (32) */ \
+ V(srlk, SRLK, 0xEBDE) /* type = RSY_A SHIFT RIGHT SINGLE LOGICAL (32) */ \
+ V(sllk, SLLK, 0xEBDF) /* type = RSY_A SHIFT LEFT SINGLE LOGICAL (32) */ \
+ V(lang, LANG, 0xEBE4) /* type = RSY_A LOAD AND AND (64) */ \
+ V(laog, LAOG, 0xEBE6) /* type = RSY_A LOAD AND OR (64) */ \
+ V(laxg, LAXG, 0xEBE7) /* type = RSY_A LOAD AND EXCLUSIVE OR (64) */ \
+ V(laag, LAAG, 0xEBE8) /* type = RSY_A LOAD AND ADD (64) */ \
+ V(laalg, LAALG, 0xEBEA) /* type = RSY_A LOAD AND ADD LOGICAL (64) */ \
+ V(lan, LAN, 0xEBF4) /* type = RSY_A LOAD AND AND (32) */ \
+ V(lao, LAO, 0xEBF6) /* type = RSY_A LOAD AND OR (32) */ \
+ V(lax, LAX, 0xEBF7) /* type = RSY_A LOAD AND EXCLUSIVE OR (32) */ \
+ V(laa, LAA, 0xEBF8) /* type = RSY_A LOAD AND ADD (32) */ \
+ V(laal, LAAL, 0xEBFA) /* type = RSY_A LOAD AND ADD LOGICAL (32) */
+
+#define S390_RSY_B_OPCODE_LIST(V) \
+ V(clmh, CLMH, \
+ 0xEB20) /* type = RSY_B COMPARE LOGICAL CHAR. UNDER MASK (high) */ \
+ V(clmy, CLMY, \
+ 0xEB21) /* type = RSY_B COMPARE LOGICAL CHAR. UNDER MASK (low) */ \
+ V(clt, CLT, 0xEB23) /* type = RSY_B COMPARE LOGICAL AND TRAP (32) */ \
+ V(clgt, CLGT, 0xEB2B) /* type = RSY_B COMPARE LOGICAL AND TRAP (64) */ \
+ V(stcmh, STCMH, \
+ 0xEB2C) /* type = RSY_B STORE CHARACTERS UNDER MASK (high) */ \
+ V(stcmy, STCMY, 0xEB2D) /* type = RSY_B STORE CHARACTERS UNDER MASK (low) */ \
+ V(icmh, ICMH, 0xEB80) /* type = RSY_B INSERT CHARACTERS UNDER MASK (high) */ \
+ V(icmy, ICMY, 0xEB81) /* type = RSY_B INSERT CHARACTERS UNDER MASK (low) */ \
+ V(locfh, LOCFH, 0xEBE0) /* type = RSY_B LOAD HIGH ON CONDITION (32) */ \
+ V(stocfh, STOCFH, 0xEBE1) /* type = RSY_B STORE HIGH ON CONDITION */ \
+ V(locg, LOCG, 0xEBE2) /* type = RSY_B LOAD ON CONDITION (64) */ \
+ V(stocg, STOCG, 0xEBE3) /* type = RSY_B STORE ON CONDITION (64) */ \
+ V(loc, LOC, 0xEBF2) /* type = RSY_B LOAD ON CONDITION (32) */ \
+ V(stoc, STOC, 0xEBF3) /* type = RSY_B STORE ON CONDITION (32) */
+
+#define S390_RXE_OPCODE_LIST(V) \
+ V(lcbb, LCBB, 0xE727) /* type = RXE LOAD COUNT TO BLOCK BOUNDARY */ \
+ V(ldeb, LDEB, 0xED04) /* type = RXE LOAD LENGTHENED (short to long BFP) */ \
+ V(lxdb, LXDB, \
+ 0xED05) /* type = RXE LOAD LENGTHENED (long to extended BFP) */ \
+ V(lxeb, LXEB, \
+ 0xED06) /* type = RXE LOAD LENGTHENED (short to extended BFP) */ \
+ V(mxdb, MXDB, 0xED07) /* type = RXE MULTIPLY (long to extended BFP) */ \
+ V(keb, KEB, 0xED08) /* type = RXE COMPARE AND SIGNAL (short BFP) */ \
+ V(ceb, CEB, 0xED09) /* type = RXE COMPARE (short BFP) */ \
+ V(aeb, AEB, 0xED0A) /* type = RXE ADD (short BFP) */ \
+ V(seb, SEB, 0xED0B) /* type = RXE SUBTRACT (short BFP) */ \
+ V(mdeb, MDEB, 0xED0C) /* type = RXE MULTIPLY (short to long BFP) */ \
+ V(deb, DEB, 0xED0D) /* type = RXE DIVIDE (short BFP) */ \
+ V(tceb, TCEB, 0xED10) /* type = RXE TEST DATA CLASS (short BFP) */ \
+ V(tcdb, TCDB, 0xED11) /* type = RXE TEST DATA CLASS (long BFP) */ \
+ V(tcxb, TCXB, 0xED12) /* type = RXE TEST DATA CLASS (extended BFP) */ \
+ V(sqeb, SQEB, 0xED14) /* type = RXE SQUARE ROOT (short BFP) */ \
+ V(sqdb, SQDB, 0xED15) /* type = RXE SQUARE ROOT (long BFP) */ \
+ V(meeb, MEEB, 0xED17) /* type = RXE MULTIPLY (short BFP) */ \
+ V(kdb, KDB, 0xED18) /* type = RXE COMPARE AND SIGNAL (long BFP) */ \
+ V(cdb, CDB, 0xED19) /* type = RXE COMPARE (long BFP) */ \
+ V(adb, ADB, 0xED1A) /* type = RXE ADD (long BFP) */ \
+ V(sdb, SDB, 0xED1B) /* type = RXE SUBTRACT (long BFP) */ \
+ V(mdb, MDB, 0xED1C) /* type = RXE MULTIPLY (long BFP) */ \
+ V(ddb, DDB, 0xED1D) /* type = RXE DIVIDE (long BFP) */ \
+ V(lde, LDE, 0xED24) /* type = RXE LOAD LENGTHENED (short to long HFP) */ \
+ V(lxd, LXD, \
+ 0xED25) /* type = RXE LOAD LENGTHENED (long to extended HFP) */ \
+ V(lxe, LXE, \
+ 0xED26) /* type = RXE LOAD LENGTHENED (short to extended HFP) */ \
+ V(sqe, SQE, 0xED34) /* type = RXE SQUARE ROOT (short HFP) */ \
+ V(sqd, SQD, 0xED35) /* type = RXE SQUARE ROOT (long HFP) */ \
+ V(mee, MEE, 0xED37) /* type = RXE MULTIPLY (short HFP) */ \
+ V(tdcet, TDCET, 0xED50) /* type = RXE TEST DATA CLASS (short DFP) */ \
+ V(tdget, TDGET, 0xED51) /* type = RXE TEST DATA GROUP (short DFP) */ \
+ V(tdcdt, TDCDT, 0xED54) /* type = RXE TEST DATA CLASS (long DFP) */ \
+ V(tdgdt, TDGDT, 0xED55) /* type = RXE TEST DATA GROUP (long DFP) */ \
+ V(tdcxt, TDCXT, 0xED58) /* type = RXE TEST DATA CLASS (extended DFP) */ \
+ V(tdgxt, TDGXT, 0xED59) /* type = RXE TEST DATA GROUP (extended DFP) */
+
+#define S390_RRF_A_OPCODE_LIST(V) \
+ V(ipte, IPTE, 0xB221) /* type = RRF_A INVALIDATE PAGE TABLE ENTRY */ \
+ V(mdtra, MDTRA, 0xB3D0) /* type = RRF_A MULTIPLY (long DFP) */ \
+ V(ddtra, DDTRA, 0xB3D1) /* type = RRF_A DIVIDE (long DFP) */ \
+ V(adtra, ADTRA, 0xB3D2) /* type = RRF_A ADD (long DFP) */ \
+ V(sdtra, SDTRA, 0xB3D3) /* type = RRF_A SUBTRACT (long DFP) */ \
+ V(mxtra, MXTRA, 0xB3D8) /* type = RRF_A MULTIPLY (extended DFP) */ \
+ V(msrkc, MSRKC, 0xB9FD) /* type = RRF_A MULTIPLY (32)*/ \
+ V(msgrkc, MSGRKC, 0xB9ED) /* type = RRF_A MULTIPLY (64)*/ \
+ V(dxtra, DXTRA, 0xB3D9) /* type = RRF_A DIVIDE (extended DFP) */ \
+ V(axtra, AXTRA, 0xB3DA) /* type = RRF_A ADD (extended DFP) */ \
+ V(sxtra, SXTRA, 0xB3DB) /* type = RRF_A SUBTRACT (extended DFP) */ \
+ V(ahhhr, AHHHR, 0xB9C8) /* type = RRF_A ADD HIGH (32) */ \
+ V(shhhr, SHHHR, 0xB9C9) /* type = RRF_A SUBTRACT HIGH (32) */ \
+ V(alhhhr, ALHHHR, 0xB9CA) /* type = RRF_A ADD LOGICAL HIGH (32) */ \
+ V(slhhhr, SLHHHR, 0xB9CB) /* type = RRF_A SUBTRACT LOGICAL HIGH (32) */ \
+ V(ahhlr, AHHLR, 0xB9D8) /* type = RRF_A ADD HIGH (32) */ \
+ V(shhlr, SHHLR, 0xB9D9) /* type = RRF_A SUBTRACT HIGH (32) */ \
+ V(alhhlr, ALHHLR, 0xB9DA) /* type = RRF_A ADD LOGICAL HIGH (32) */ \
+ V(slhhlr, SLHHLR, 0xB9DB) /* type = RRF_A SUBTRACT LOGICAL HIGH (32) */ \
+ V(ngrk, NGRK, 0xB9E4) /* type = RRF_A AND (64) */ \
+ V(ogrk, OGRK, 0xB9E6) /* type = RRF_A OR (64) */ \
+ V(xgrk, XGRK, 0xB9E7) /* type = RRF_A EXCLUSIVE OR (64) */ \
+ V(agrk, AGRK, 0xB9E8) /* type = RRF_A ADD (64) */ \
+ V(sgrk, SGRK, 0xB9E9) /* type = RRF_A SUBTRACT (64) */ \
+ V(algrk, ALGRK, 0xB9EA) /* type = RRF_A ADD LOGICAL (64) */ \
+ V(slgrk, SLGRK, 0xB9EB) /* type = RRF_A SUBTRACT LOGICAL (64) */ \
+ V(nrk, NRK, 0xB9F4) /* type = RRF_A AND (32) */ \
+ V(ork, ORK, 0xB9F6) /* type = RRF_A OR (32) */ \
+ V(xrk, XRK, 0xB9F7) /* type = RRF_A EXCLUSIVE OR (32) */ \
+ V(ark, ARK, 0xB9F8) /* type = RRF_A ADD (32) */ \
+ V(srk, SRK, 0xB9F9) /* type = RRF_A SUBTRACT (32) */ \
+ V(alrk, ALRK, 0xB9FA) /* type = RRF_A ADD LOGICAL (32) */ \
+ V(slrk, SLRK, 0xB9FB) /* type = RRF_A SUBTRACT LOGICAL (32) */
+
+#define S390_RXF_OPCODE_LIST(V) \
+ V(maeb, MAEB, 0xED0E) /* type = RXF MULTIPLY AND ADD (short BFP) */ \
+ V(mseb, MSEB, 0xED0F) /* type = RXF MULTIPLY AND SUBTRACT (short BFP) */ \
+ V(madb, MADB, 0xED1E) /* type = RXF MULTIPLY AND ADD (long BFP) */ \
+ V(msdb, MSDB, 0xED1F) /* type = RXF MULTIPLY AND SUBTRACT (long BFP) */ \
+ V(mae, MAE, 0xED2E) /* type = RXF MULTIPLY AND ADD (short HFP) */ \
+ V(mse, MSE, 0xED2F) /* type = RXF MULTIPLY AND SUBTRACT (short HFP) */ \
+ V(mayl, MAYL, \
+ 0xED38) /* type = RXF MULTIPLY AND ADD UNNRM. (long to ext. low HFP) */ \
+ V(myl, MYL, \
+ 0xED39) /* type = RXF MULTIPLY UNNORM. (long to ext. low HFP) */ \
+ V(may, MAY, \
+ 0xED3A) /* type = RXF MULTIPLY & ADD UNNORMALIZED (long to ext. HFP) */ \
+ V(my, MY, \
+ 0xED3B) /* type = RXF MULTIPLY UNNORMALIZED (long to ext. HFP) */ \
+ V(mayh, MAYH, \
+ 0xED3C) /* type = RXF MULTIPLY AND ADD UNNRM. (long to ext. high HFP) */ \
+ V(myh, MYH, \
+ 0xED3D) /* type = RXF MULTIPLY UNNORM. (long to ext. high HFP) */ \
+ V(mad, MAD, 0xED3E) /* type = RXF MULTIPLY AND ADD (long HFP) */ \
+ V(msd, MSD, 0xED3F) /* type = RXF MULTIPLY AND SUBTRACT (long HFP) */ \
+ V(sldt, SLDT, 0xED40) /* type = RXF SHIFT SIGNIFICAND LEFT (long DFP) */ \
+ V(srdt, SRDT, 0xED41) /* type = RXF SHIFT SIGNIFICAND RIGHT (long DFP) */ \
+ V(slxt, SLXT, \
+ 0xED48) /* type = RXF SHIFT SIGNIFICAND LEFT (extended DFP) */ \
+ V(srxt, SRXT, \
+ 0xED49) /* type = RXF SHIFT SIGNIFICAND RIGHT (extended DFP) */
+
+#define S390_IE_OPCODE_LIST(V) \
+ V(niai, NIAI, 0xB2FA) /* type = IE NEXT INSTRUCTION ACCESS INTENT */
+
+#define S390_RRF_B_OPCODE_LIST(V) \
+ V(diebr, DIEBR, 0xB353) /* type = RRF_B DIVIDE TO INTEGER (short BFP) */ \
+ V(didbr, DIDBR, 0xB35B) /* type = RRF_B DIVIDE TO INTEGER (long BFP) */ \
+ V(cpsdr, CPSDR, 0xB372) /* type = RRF_B COPY SIGN (long) */ \
+ V(qadtr, QADTR, 0xB3F5) /* type = RRF_B QUANTIZE (long DFP) */ \
+ V(iedtr, IEDTR, \
+ 0xB3F6) /* type = RRF_B INSERT BIASED EXPONENT (64 to long DFP) */ \
+ V(rrdtr, RRDTR, 0xB3F7) /* type = RRF_B REROUND (long DFP) */ \
+ V(qaxtr, QAXTR, 0xB3FD) /* type = RRF_B QUANTIZE (extended DFP) */ \
+ V(iextr, IEXTR, \
+ 0xB3FE) /* type = RRF_B INSERT BIASED EXPONENT (64 to extended DFP) */ \
+ V(rrxtr, RRXTR, 0xB3FF) /* type = RRF_B REROUND (extended DFP) */ \
+ V(kmctr, KMCTR, 0xB92D) /* type = RRF_B CIPHER MESSAGE WITH COUNTER */ \
+ V(idte, IDTE, 0xB98E) /* type = RRF_B INVALIDATE DAT TABLE ENTRY */ \
+ V(crdte, CRDTE, \
+ 0xB98F) /* type = RRF_B COMPARE AND REPLACE DAT TABLE ENTRY */ \
+ V(lptea, LPTEA, 0xB9AA) /* type = RRF_B LOAD PAGE TABLE ENTRY ADDRESS */
+
+#define S390_RRF_C_OPCODE_LIST(V) \
+ V(sske, SSKE, 0xB22B) /* type = RRF_C SET STORAGE KEY EXTENDED */ \
+ V(cu21, CU21, 0xB2A6) /* type = RRF_C CONVERT UTF-16 TO UTF-8 */ \
+ V(cu12, CU12, 0xB2A7) /* type = RRF_C CONVERT UTF-8 TO UTF-16 */ \
+ V(ppa, PPA, 0xB2E8) /* type = RRF_C PERFORM PROCESSOR ASSIST */ \
+ V(cgrt, CGRT, 0xB960) /* type = RRF_C COMPARE AND TRAP (64) */ \
+ V(clgrt, CLGRT, 0xB961) /* type = RRF_C COMPARE LOGICAL AND TRAP (64) */ \
+ V(crt, CRT, 0xB972) /* type = RRF_C COMPARE AND TRAP (32) */ \
+ V(clrt, CLRT, 0xB973) /* type = RRF_C COMPARE LOGICAL AND TRAP (32) */ \
+ V(trtt, TRTT, 0xB990) /* type = RRF_C TRANSLATE TWO TO TWO */ \
+ V(trto, TRTO, 0xB991) /* type = RRF_C TRANSLATE TWO TO ONE */ \
+ V(trot, TROT, 0xB992) /* type = RRF_C TRANSLATE ONE TO TWO */ \
+ V(troo, TROO, 0xB993) /* type = RRF_C TRANSLATE ONE TO ONE */ \
+ V(cu14, CU14, 0xB9B0) /* type = RRF_C CONVERT UTF-8 TO UTF-32 */ \
+ V(cu24, CU24, 0xB9B1) /* type = RRF_C CONVERT UTF-16 TO UTF-32 */ \
+ V(trtre, TRTRE, \
+ 0xB9BD) /* type = RRF_C TRANSLATE AND TEST REVERSE EXTENDED */ \
+ V(trte, TRTE, 0xB9BF) /* type = RRF_C TRANSLATE AND TEST EXTENDED */ \
+ V(locfhr, LOCFHR, 0xB9E0) /* type = RRF_C LOAD HIGH ON CONDITION (32) */ \
+ V(locgr, LOCGR, 0xB9E2) /* type = RRF_C LOAD ON CONDITION (64) */ \
+ V(locr, LOCR, 0xB9F2) /* type = RRF_C LOAD ON CONDITION (32) */
+
+#define S390_MII_OPCODE_LIST(V) \
+ V(bprp, BPRP, 0xC5) /* type = MII BRANCH PREDICTION RELATIVE PRELOAD */
+
+#define S390_RRF_D_OPCODE_LIST(V) \
+ V(ldetr, LDETR, \
+ 0xB3D4) /* type = RRF_D LOAD LENGTHENED (short to long DFP) */ \
+ V(lxdtr, LXDTR, \
+ 0xB3DC) /* type = RRF_D LOAD LENGTHENED (long to extended DFP) */ \
+ V(csdtr, CSDTR, \
+ 0xB3E3) /* type = RRF_D CONVERT TO SIGNED PACKED (long DFP to 64) */ \
+ V(csxtr, CSXTR, \
+ 0xB3EB) /* type = RRF_D CONVERT TO SIGNED PACKED (extended DFP to 128) */
+
+#define S390_RRF_E_OPCODE_LIST(V) \
+ V(ledbra, LEDBRA, \
+ 0xB344) /* type = RRF_E LOAD ROUNDED (long to short BFP) */ \
+ V(ldxbra, LDXBRA, \
+ 0xB345) /* type = RRF_E LOAD ROUNDED (extended to long BFP) */ \
+ V(lexbra, LEXBRA, \
+ 0xB346) /* type = RRF_E LOAD ROUNDED (extended to short BFP) */ \
+ V(fixbra, FIXBRA, 0xB347) /* type = RRF_E LOAD FP INTEGER (extended BFP) */ \
+ V(tbedr, TBEDR, \
+ 0xB350) /* type = RRF_E CONVERT HFP TO BFP (long to short) */ \
+ V(tbdr, TBDR, 0xB351) /* type = RRF_E CONVERT HFP TO BFP (long) */ \
+ V(fiebra, FIEBRA, 0xB357) /* type = RRF_E LOAD FP INTEGER (short BFP) */ \
+ V(fidbra, FIDBRA, 0xB35F) /* type = RRF_E LOAD FP INTEGER (long BFP) */ \
+ V(celfbr, CELFBR, \
+ 0xB390) /* type = RRF_E CONVERT FROM LOGICAL (32 to short BFP) */ \
+ V(cdlfbr, CDLFBR, \
+ 0xB391) /* type = RRF_E CONVERT FROM LOGICAL (32 to long BFP) */ \
+ V(cxlfbr, CXLFBR, \
+ 0xB392) /* type = RRF_E CONVERT FROM LOGICAL (32 to extended BFP) */ \
+ V(cefbra, CEFBRA, \
+ 0xB394) /* type = RRF_E CONVERT FROM FIXED (32 to short BFP) */ \
+ V(cdfbra, CDFBRA, \
+ 0xB395) /* type = RRF_E CONVERT FROM FIXED (32 to long BFP) */ \
+ V(cxfbra, CXFBRA, \
+ 0xB396) /* type = RRF_E CONVERT FROM FIXED (32 to extended BFP) */ \
+ V(cfebra, CFEBRA, \
+ 0xB398) /* type = RRF_E CONVERT TO FIXED (short BFP to 32) */ \
+ V(cfdbra, CFDBRA, \
+ 0xB399) /* type = RRF_E CONVERT TO FIXED (long BFP to 32) */ \
+ V(cfxbra, CFXBRA, \
+ 0xB39A) /* type = RRF_E CONVERT TO FIXED (extended BFP to 32) */ \
+ V(clfebr, CLFEBR, \
+ 0xB39C) /* type = RRF_E CONVERT TO LOGICAL (short BFP to 32) */ \
+ V(clfdbr, CLFDBR, \
+ 0xB39D) /* type = RRF_E CONVERT TO LOGICAL (long BFP to 32) */ \
+ V(clfxbr, CLFXBR, \
+ 0xB39E) /* type = RRF_E CONVERT TO LOGICAL (extended BFP to 32) */ \
+ V(celgbr, CELGBR, \
+ 0xB3A0) /* type = RRF_E CONVERT FROM LOGICAL (64 to short BFP) */ \
+ V(cdlgbr, CDLGBR, \
+ 0xB3A1) /* type = RRF_E CONVERT FROM LOGICAL (64 to long BFP) */ \
+ V(cxlgbr, CXLGBR, \
+ 0xB3A2) /* type = RRF_E CONVERT FROM LOGICAL (64 to extended BFP) */ \
+ V(cegbra, CEGBRA, \
+ 0xB3A4) /* type = RRF_E CONVERT FROM FIXED (64 to short BFP) */ \
+ V(cdgbra, CDGBRA, \
+ 0xB3A5) /* type = RRF_E CONVERT FROM FIXED (64 to long BFP) */ \
+ V(cxgbra, CXGBRA, \
+ 0xB3A6) /* type = RRF_E CONVERT FROM FIXED (64 to extended BFP) */ \
+ V(cgebra, CGEBRA, \
+ 0xB3A8) /* type = RRF_E CONVERT TO FIXED (short BFP to 64) */ \
+ V(cgdbra, CGDBRA, \
+ 0xB3A9) /* type = RRF_E CONVERT TO FIXED (long BFP to 64) */ \
+ V(cgxbra, CGXBRA, \
+ 0xB3AA) /* type = RRF_E CONVERT TO FIXED (extended BFP to 64) */ \
+ V(clgebr, CLGEBR, \
+ 0xB3AC) /* type = RRF_E CONVERT TO LOGICAL (short BFP to 64) */ \
+ V(clgdbr, CLGDBR, \
+ 0xB3AD) /* type = RRF_E CONVERT TO LOGICAL (long BFP to 64) */ \
+ V(clgxbr, CLGXBR, \
+ 0xB3AE) /* type = RRF_E CONVERT TO LOGICAL (extended BFP to 64) */ \
+ V(cfer, CFER, 0xB3B8) /* type = RRF_E CONVERT TO FIXED (short HFP to 32) */ \
+ V(cfdr, CFDR, 0xB3B9) /* type = RRF_E CONVERT TO FIXED (long HFP to 32) */ \
+ V(cfxr, CFXR, \
+ 0xB3BA) /* type = RRF_E CONVERT TO FIXED (extended HFP to 32) */ \
+ V(cger, CGER, 0xB3C8) /* type = RRF_E CONVERT TO FIXED (short HFP to 64) */ \
+ V(cgdr, CGDR, 0xB3C9) /* type = RRF_E CONVERT TO FIXED (long HFP to 64) */ \
+ V(cgxr, CGXR, \
+ 0xB3CA) /* type = RRF_E CONVERT TO FIXED (extended HFP to 64) */ \
+ V(ledtr, LEDTR, 0xB3D5) /* type = RRF_E LOAD ROUNDED (long to short DFP) */ \
+ V(fidtr, FIDTR, 0xB3D7) /* type = RRF_E LOAD FP INTEGER (long DFP) */ \
+ V(ldxtr, LDXTR, \
+ 0xB3DD) /* type = RRF_E LOAD ROUNDED (extended to long DFP) */ \
+ V(fixtr, FIXTR, 0xB3DF) /* type = RRF_E LOAD FP INTEGER (extended DFP) */ \
+ V(cgdtra, CGDTRA, \
+ 0xB3E1) /* type = RRF_E CONVERT TO FIXED (long DFP to 64) */ \
+ V(cgxtra, CGXTRA, \
+ 0xB3E9) /* type = RRF_E CONVERT TO FIXED (extended DFP to 64) */ \
+ V(cdgtra, CDGTRA, \
+ 0xB3F1) /* type = RRF_E CONVERT FROM FIXED (64 to long DFP) */ \
+ V(cxgtra, CXGTRA, \
+ 0xB3F9) /* type = RRF_E CONVERT FROM FIXED (64 to extended DFP) */ \
+ V(cfdtr, CFDTR, 0xB941) /* type = RRF_E CONVERT TO FIXED (long DFP to 32) */ \
+ V(clgdtr, CLGDTR, \
+ 0xB942) /* type = RRF_E CONVERT TO LOGICAL (long DFP to 64) */ \
+ V(clfdtr, CLFDTR, \
+ 0xB943) /* type = RRF_E CONVERT TO LOGICAL (long DFP to 32) */ \
+ V(cfxtr, CFXTR, \
+ 0xB949) /* type = RRF_E CONVERT TO FIXED (extended DFP to 32) */ \
+ V(clgxtr, CLGXTR, \
+ 0xB94A) /* type = RRF_E CONVERT TO LOGICAL (extended DFP to 64) */ \
+ V(clfxtr, CLFXTR, \
+ 0xB94B) /* type = RRF_E CONVERT TO LOGICAL (extended DFP to 32) */ \
+ V(cdlgtr, CDLGTR, \
+ 0xB952) /* type = RRF_E CONVERT FROM LOGICAL (64 to long DFP) */ \
+ V(cdlftr, CDLFTR, \
+ 0xB953) /* type = RRF_E CONVERT FROM LOGICAL (32 to long DFP) */ \
+ V(cxlgtr, CXLGTR, \
+ 0xB95A) /* type = RRF_E CONVERT FROM LOGICAL (64 to extended DFP) */ \
+ V(cxlftr, CXLFTR, \
+ 0xB95B) /* type = RRF_E CONVERT FROM LOGICAL (32 to extended DFP) */
+
+#define S390_VRR_A_OPCODE_LIST(V) \
+ V(vpopct, VPOPCT, 0xE750) /* type = VRR_A VECTOR POPULATION COUNT */ \
+ V(vctz, VCTZ, 0xE752) /* type = VRR_A VECTOR COUNT TRAILING ZEROS */ \
+ V(vclz, VCLZ, 0xE753) /* type = VRR_A VECTOR COUNT LEADING ZEROS */ \
+ V(vlr, VLR, 0xE756) /* type = VRR_A VECTOR LOAD */ \
+ V(vistr, VISTR, 0xE75C) /* type = VRR_A VECTOR ISOLATE STRING */ \
+ V(vseg, VSEG, 0xE75F) /* type = VRR_A VECTOR SIGN EXTEND TO DOUBLEWORD */ \
+ V(vclgd, VCLGD, \
+ 0xE7C0) /* type = VRR_A VECTOR FP CONVERT TO LOGICAL 64-BIT */ \
+ V(vcdlg, VCDLG, \
+ 0xE7C1) /* type = VRR_A VECTOR FP CONVERT FROM LOGICAL 64-BIT */ \
+ V(vcgd, VCGD, 0xE7C2) /* type = VRR_A VECTOR FP CONVERT TO FIXED 64-BIT */ \
+ V(vcdg, VCDG, 0xE7C3) /* type = VRR_A VECTOR FP CONVERT FROM FIXED 64-BIT */ \
+ V(vlde, VLDE, 0xE7C4) /* type = VRR_A VECTOR FP LOAD LENGTHENED */ \
+ V(vled, VLED, 0xE7C5) /* type = VRR_A VECTOR FP LOAD ROUNDED */ \
+ V(vfi, VFI, 0xE7C7) /* type = VRR_A VECTOR LOAD FP INTEGER */ \
+ V(wfk, WFK, 0xE7CA) /* type = VRR_A VECTOR FP COMPARE AND SIGNAL SCALAR */ \
+ V(wfc, WFC, 0xE7CB) /* type = VRR_A VECTOR FP COMPARE SCALAR */ \
+ V(vfpso, VFPSO, 0xE7CC) /* type = VRR_A VECTOR FP PERFORM SIGN OPERATION */ \
+ V(vfsq, VFSQ, 0xE7CE) /* type = VRR_A VECTOR FP SQUARE ROOT */ \
+ V(vupll, VUPLL, 0xE7D4) /* type = VRR_A VECTOR UNPACK LOGICAL LOW */ \
+ V(vuplh, VUPLH, 0xE7D5) /* type = VRR_A VECTOR UNPACK LOGICAL HIGH */ \
+ V(vupl, VUPL, 0xE7D6) /* type = VRR_A VECTOR UNPACK LOW */ \
+ V(vuph, VUPH, 0xE7D7) /* type = VRR_A VECTOR UNPACK HIGH */ \
+ V(vtm, VTM, 0xE7D8) /* type = VRR_A VECTOR TEST UNDER MASK */ \
+ V(vecl, VECL, 0xE7D9) /* type = VRR_A VECTOR ELEMENT COMPARE LOGICAL */ \
+ V(vec, VEC, 0xE7DB) /* type = VRR_A VECTOR ELEMENT COMPARE */ \
+ V(vlc, VLC, 0xE7DE) /* type = VRR_A VECTOR LOAD COMPLEMENT */ \
+ V(vlp, VLP, 0xE7DF) /* type = VRR_A VECTOR LOAD POSITIVE */
+
+#define S390_VRR_B_OPCODE_LIST(V) \
+ V(vfee, VFEE, 0xE780) /* type = VRR_B VECTOR FIND ELEMENT EQUAL */ \
+ V(vfene, VFENE, 0xE781) /* type = VRR_B VECTOR FIND ELEMENT NOT EQUAL */ \
+ V(vfae, VFAE, 0xE782) /* type = VRR_B VECTOR FIND ANY ELEMENT EQUAL */ \
+ V(vpkls, VPKLS, 0xE795) /* type = VRR_B VECTOR PACK LOGICAL SATURATE */ \
+ V(vpks, VPKS, 0xE797) /* type = VRR_B VECTOR PACK SATURATE */ \
+ V(vceq, VCEQ, 0xE7F8) /* type = VRR_B VECTOR COMPARE EQUAL */ \
+ V(vchl, VCHL, 0xE7F9) /* type = VRR_B VECTOR COMPARE HIGH LOGICAL */ \
+ V(vch, VCH, 0xE7FB) /* type = VRR_B VECTOR COMPARE HIGH */
+
+#define S390_VRR_C_OPCODE_LIST(V) \
+ V(vmrl, VMRL, 0xE760) /* type = VRR_C VECTOR MERGE LOW */ \
+ V(vmrh, VMRH, 0xE761) /* type = VRR_C VECTOR MERGE HIGH */ \
+ V(vsum, VSUM, 0xE764) /* type = VRR_C VECTOR SUM ACROSS WORD */ \
+ V(vsumg, VSUMG, 0xE765) /* type = VRR_C VECTOR SUM ACROSS DOUBLEWORD */ \
+ V(vcksm, VCKSM, 0xE766) /* type = VRR_C VECTOR CHECKSUM */ \
+ V(vsumq, VSUMQ, 0xE767) /* type = VRR_C VECTOR SUM ACROSS QUADWORD */ \
+ V(vn, VN, 0xE768) /* type = VRR_C VECTOR AND */ \
+ V(vnc, VNC, 0xE769) /* type = VRR_C VECTOR AND WITH COMPLEMENT */ \
+ V(vo, VO, 0xE76A) /* type = VRR_C VECTOR OR */ \
+ V(vno, VNO, 0xE76B) /* type = VRR_C VECTOR NOR */ \
+ V(vx, VX, 0xE76D) /* type = VRR_C VECTOR EXCLUSIVE OR */ \
+ V(veslv, VESLV, 0xE770) /* type = VRR_C VECTOR ELEMENT SHIFT LEFT */ \
+ V(verllv, VERLLV, \
+ 0xE773) /* type = VRR_C VECTOR ELEMENT ROTATE LEFT LOGICAL */ \
+ V(vsl, VSL, 0xE774) /* type = VRR_C VECTOR SHIFT LEFT */ \
+ V(vslb, VSLB, 0xE775) /* type = VRR_C VECTOR SHIFT LEFT BY BYTE */ \
+ V(vesrlv, VESRLV, \
+ 0xE778) /* type = VRR_C VECTOR ELEMENT SHIFT RIGHT LOGICAL */ \
+ V(vesrav, VESRAV, \
+ 0xE77A) /* type = VRR_C VECTOR ELEMENT SHIFT RIGHT ARITHMETIC */ \
+ V(vsrl, VSRL, 0xE77C) /* type = VRR_C VECTOR SHIFT RIGHT LOGICAL */ \
+ V(vsrlb, VSRLB, \
+ 0xE77D) /* type = VRR_C VECTOR SHIFT RIGHT LOGICAL BY BYTE */ \
+ V(vsra, VSRA, 0xE77E) /* type = VRR_C VECTOR SHIFT RIGHT ARITHMETIC */ \
+ V(vsrab, VSRAB, \
+ 0xE77F) /* type = VRR_C VECTOR SHIFT RIGHT ARITHMETIC BY BYTE */ \
+ V(vpdi, VPDI, 0xE784) /* type = VRR_C VECTOR PERMUTE DOUBLEWORD IMMEDIATE */ \
+ V(vpk, VPK, 0xE794) /* type = VRR_C VECTOR PACK */ \
+ V(vmlh, VMLH, 0xE7A1) /* type = VRR_C VECTOR MULTIPLY LOGICAL HIGH */ \
+ V(vml, VML, 0xE7A2) /* type = VRR_C VECTOR MULTIPLY LOW */ \
+ V(vmh, VMH, 0xE7A3) /* type = VRR_C VECTOR MULTIPLY HIGH */ \
+ V(vmle, VMLE, 0xE7A4) /* type = VRR_C VECTOR MULTIPLY LOGICAL EVEN */ \
+ V(vmlo, VMLO, 0xE7A5) /* type = VRR_C VECTOR MULTIPLY LOGICAL ODD */ \
+ V(vme, VME, 0xE7A6) /* type = VRR_C VECTOR MULTIPLY EVEN */ \
+ V(vmo, VMO, 0xE7A7) /* type = VRR_C VECTOR MULTIPLY ODD */ \
+ V(vgfm, VGFM, 0xE7B4) /* type = VRR_C VECTOR GALOIS FIELD MULTIPLY SUM */ \
+ V(vfs, VFS, 0xE7E2) /* type = VRR_C VECTOR FP SUBTRACT */ \
+ V(vfa, VFA, 0xE7E3) /* type = VRR_C VECTOR FP ADD */ \
+ V(vfd, VFD, 0xE7E5) /* type = VRR_C VECTOR FP DIVIDE */ \
+ V(vfm, VFM, 0xE7E7) /* type = VRR_C VECTOR FP MULTIPLY */ \
+ V(vfce, VFCE, 0xE7E8) /* type = VRR_C VECTOR FP COMPARE EQUAL */ \
+ V(vfche, VFCHE, 0xE7EA) /* type = VRR_C VECTOR FP COMPARE HIGH OR EQUAL */ \
+ V(vfch, VFCH, 0xE7EB) /* type = VRR_C VECTOR FP COMPARE HIGH */ \
+ V(vfmax, VFMAX, 0xE7EF) /* type = VRR_C VECTOR FP MAXIMUM */ \
+ V(vfmin, VFMIN, 0xE7EE) /* type = VRR_C VECTOR FP MINIMUM */ \
+ V(vavgl, VAVGL, 0xE7F0) /* type = VRR_C VECTOR AVERAGE LOGICAL */ \
+ V(vacc, VACC, 0xE7F1) /* type = VRR_C VECTOR ADD COMPUTE CARRY */ \
+ V(vavg, VAVG, 0xE7F2) /* type = VRR_C VECTOR AVERAGE */ \
+ V(va, VA, 0xE7F3) /* type = VRR_C VECTOR ADD */ \
+ V(vscbi, VSCBI, \
+ 0xE7F5) /* type = VRR_C VECTOR SUBTRACT COMPUTE BORROW INDICATION */ \
+ V(vs, VS, 0xE7F7) /* type = VRR_C VECTOR SUBTRACT */ \
+ V(vmnl, VMNL, 0xE7FC) /* type = VRR_C VECTOR MINIMUM LOGICAL */ \
+ V(vmxl, VMXL, 0xE7FD) /* type = VRR_C VECTOR MAXIMUM LOGICAL */ \
+ V(vmn, VMN, 0xE7FE) /* type = VRR_C VECTOR MINIMUM */ \
+ V(vmx, VMX, 0xE7FF) /* type = VRR_C VECTOR MAXIMUM */ \
+ V(vbperm, VBPERM, 0xE785) /* type = VRR_C VECTOR BIT PERMUTE */
+
+#define S390_VRI_A_OPCODE_LIST(V) \
+ V(vleib, VLEIB, 0xE740) /* type = VRI_A VECTOR LOAD ELEMENT IMMEDIATE (8) */ \
+ V(vleih, VLEIH, \
+ 0xE741) /* type = VRI_A VECTOR LOAD ELEMENT IMMEDIATE (16) */ \
+ V(vleig, VLEIG, \
+ 0xE742) /* type = VRI_A VECTOR LOAD ELEMENT IMMEDIATE (64) */ \
+ V(vleif, VLEIF, \
+ 0xE743) /* type = VRI_A VECTOR LOAD ELEMENT IMMEDIATE (32) */ \
+ V(vgbm, VGBM, 0xE744) /* type = VRI_A VECTOR GENERATE BYTE MASK */ \
+ V(vrepi, VREPI, 0xE745) /* type = VRI_A VECTOR REPLICATE IMMEDIATE */
+
+#define S390_VRR_D_OPCODE_LIST(V) \
+ V(vstrc, VSTRC, 0xE78A) /* type = VRR_D VECTOR STRING RANGE COMPARE */ \
+ V(vmalh, VMALH, \
+ 0xE7A9) /* type = VRR_D VECTOR MULTIPLY AND ADD LOGICAL HIGH */ \
+ V(vmal, VMAL, 0xE7AA) /* type = VRR_D VECTOR MULTIPLY AND ADD LOW */ \
+ V(vmah, VMAH, 0xE7AB) /* type = VRR_D VECTOR MULTIPLY AND ADD HIGH */ \
+ V(vmale, VMALE, \
+ 0xE7AC) /* type = VRR_D VECTOR MULTIPLY AND ADD LOGICAL EVEN */ \
+ V(vmalo, VMALO, \
+ 0xE7AD) /* type = VRR_D VECTOR MULTIPLY AND ADD LOGICAL ODD */ \
+ V(vmae, VMAE, 0xE7AE) /* type = VRR_D VECTOR MULTIPLY AND ADD EVEN */ \
+ V(vmao, VMAO, 0xE7AF) /* type = VRR_D VECTOR MULTIPLY AND ADD ODD */ \
+ V(vaccc, VACCC, \
+ 0xE7B9) /* type = VRR_D VECTOR ADD WITH CARRY COMPUTE CARRY */ \
+ V(vac, VAC, 0xE7BB) /* type = VRR_D VECTOR ADD WITH CARRY */ \
+ V(vgfma, VGFMA, \
+ 0xE7BC) /* type = VRR_D VECTOR GALOIS FIELD MULTIPLY SUM AND ACCUMULATE */ \
+ V(vsbcbi, VSBCBI, 0xE7BD) /* type = VRR_D VECTOR SUBTRACT WITH BORROW */ \
+ /* COMPUTE BORROW INDICATION */ \
+ V(vsbi, VSBI, \
+ 0xE7BF) /* type = VRR_D VECTOR SUBTRACT WITH BORROW INDICATION */
+
+#define S390_VRI_B_OPCODE_LIST(V) \
+ V(vgm, VGM, 0xE746) /* type = VRI_B VECTOR GENERATE MASK */
+
+#define S390_VRR_E_OPCODE_LIST(V) \
+ V(vperm, VPERM, 0xE78C) /* type = VRR_E VECTOR PERMUTE */ \
+ V(vsel, VSEL, 0xE78D) /* type = VRR_E VECTOR SELECT */ \
+ V(vfms, VFMS, 0xE78E) /* type = VRR_E VECTOR FP MULTIPLY AND SUBTRACT */ \
+ V(vfnms, VFNMS, \
+ 0xE79E) /* type = VRR_E VECTOR FP NEGATIVE MULTIPLY AND SUBTRACT */ \
+ V(vfma, VFMA, 0xE78F) /* type = VRR_E VECTOR FP MULTIPLY AND ADD */
+
+#define S390_VRI_C_OPCODE_LIST(V) \
+ V(vrep, VREP, 0xE74D) /* type = VRI_C VECTOR REPLICATE */
+
+#define S390_VRI_D_OPCODE_LIST(V) \
+ V(verim, VERIM, \
+ 0xE772) /* type = VRI_D VECTOR ELEMENT ROTATE AND INSERT UNDER MASK */ \
+ V(vsldb, VSLDB, 0xE777) /* type = VRI_D VECTOR SHIFT LEFT DOUBLE BY BYTE */
+
+#define S390_VRR_F_OPCODE_LIST(V) \
+ V(vlvgp, VLVGP, 0xE762) /* type = VRR_F VECTOR LOAD VR FROM GRS DISJOINT */
+
+#define S390_RIS_OPCODE_LIST(V) \
+ V(cgib, CGIB, \
+ 0xECFC) /* type = RIS COMPARE IMMEDIATE AND BRANCH (64<-8) */ \
+ V(clgib, CLGIB, \
+ 0xECFD) /* type = RIS COMPARE LOGICAL IMMEDIATE AND BRANCH (64<-8) */ \
+ V(cib, CIB, 0xECFE) /* type = RIS COMPARE IMMEDIATE AND BRANCH (32<-8) */ \
+ V(clib, CLIB, \
+ 0xECFF) /* type = RIS COMPARE LOGICAL IMMEDIATE AND BRANCH (32<-8) */
+
+#define S390_VRI_E_OPCODE_LIST(V) \
+ V(vftci, VFTCI, \
+ 0xE74A) /* type = VRI_E VECTOR FP TEST DATA CLASS IMMEDIATE */
+
+#define S390_RSL_A_OPCODE_LIST(V) \
+ V(tp, TP, 0xEBC0) /* type = RSL_A TEST DECIMAL */
+
+#define S390_RSL_B_OPCODE_LIST(V) \
+ V(cpdt, CPDT, 0xEDAC) /* type = RSL_B CONVERT TO PACKED (from long DFP) */ \
+ V(cpxt, CPXT, \
+ 0xEDAD) /* type = RSL_B CONVERT TO PACKED (from extended DFP) */ \
+ V(cdpt, CDPT, 0xEDAE) /* type = RSL_B CONVERT FROM PACKED (to long DFP) */ \
+ V(cxpt, CXPT, \
+ 0xEDAF) /* type = RSL_B CONVERT FROM PACKED (to extended DFP) */ \
+ V(czdt, CZDT, 0xEDA8) /* type = RSL CONVERT TO ZONED (from long DFP) */ \
+ V(czxt, CZXT, 0xEDA9) /* type = RSL CONVERT TO ZONED (from extended DFP) */ \
+ V(cdzt, CDZT, 0xEDAA) /* type = RSL CONVERT FROM ZONED (to long DFP) */ \
+ V(cxzt, CXZT, 0xEDAB) /* type = RSL CONVERT FROM ZONED (to extended DFP) */
+
+#define S390_SI_OPCODE_LIST(V) \
+ V(tm, TM, 0x91) /* type = SI TEST UNDER MASK */ \
+ V(mvi, MVI, 0x92) /* type = SI MOVE (immediate) */ \
+ V(ni, NI, 0x94) /* type = SI AND (immediate) */ \
+ V(cli, CLI, 0x95) /* type = SI COMPARE LOGICAL (immediate) */ \
+ V(oi, OI, 0x96) /* type = SI OR (immediate) */ \
+ V(xi, XI, 0x97) /* type = SI EXCLUSIVE OR (immediate) */ \
+ V(stnsm, STNSM, 0xAC) /* type = SI STORE THEN AND SYSTEM MASK */ \
+ V(stosm, STOSM, 0xAD) /* type = SI STORE THEN OR SYSTEM MASK */ \
+ V(mc, MC, 0xAF) /* type = SI MONITOR CALL */
+
+#define S390_SIL_OPCODE_LIST(V) \
+ V(mvhhi, MVHHI, 0xE544) /* type = SIL MOVE (16<-16) */ \
+ V(mvghi, MVGHI, 0xE548) /* type = SIL MOVE (64<-16) */ \
+ V(mvhi, MVHI, 0xE54C) /* type = SIL MOVE (32<-16) */ \
+ V(chhsi, CHHSI, \
+ 0xE554) /* type = SIL COMPARE HALFWORD IMMEDIATE (16<-16) */ \
+ V(clhhsi, CLHHSI, \
+ 0xE555) /* type = SIL COMPARE LOGICAL IMMEDIATE (16<-16) */ \
+ V(cghsi, CGHSI, \
+ 0xE558) /* type = SIL COMPARE HALFWORD IMMEDIATE (64<-16) */ \
+ V(clghsi, CLGHSI, \
+ 0xE559) /* type = SIL COMPARE LOGICAL IMMEDIATE (64<-16) */ \
+ V(chsi, CHSI, 0xE55C) /* type = SIL COMPARE HALFWORD IMMEDIATE (32<-16) */ \
+ V(clfhsi, CLFHSI, \
+ 0xE55D) /* type = SIL COMPARE LOGICAL IMMEDIATE (32<-16) */ \
+ V(tbegin, TBEGIN, \
+ 0xE560) /* type = SIL TRANSACTION BEGIN (nonconstrained) */ \
+ V(tbeginc, TBEGINC, \
+ 0xE561) /* type = SIL TRANSACTION BEGIN (constrained) */
+
+#define S390_VRS_A_OPCODE_LIST(V) \
+ V(vesl, VESL, 0xE730) /* type = VRS_A VECTOR ELEMENT SHIFT LEFT */ \
+ V(verll, VERLL, \
+ 0xE733) /* type = VRS_A VECTOR ELEMENT ROTATE LEFT LOGICAL */ \
+ V(vlm, VLM, 0xE736) /* type = VRS_A VECTOR LOAD MULTIPLE */ \
+ V(vesrl, VESRL, \
+ 0xE738) /* type = VRS_A VECTOR ELEMENT SHIFT RIGHT LOGICAL */ \
+ V(vesra, VESRA, \
+ 0xE73A) /* type = VRS_A VECTOR ELEMENT SHIFT RIGHT ARITHMETIC */ \
+ V(vstm, VSTM, 0xE73E) /* type = VRS_A VECTOR STORE MULTIPLE */
+
+#define S390_RIL_A_OPCODE_LIST(V) \
+ V(lgfi, LGFI, 0xC01) /* type = RIL_A LOAD IMMEDIATE (64<-32) */ \
+ V(xihf, XIHF, 0xC06) /* type = RIL_A EXCLUSIVE OR IMMEDIATE (high) */ \
+ V(xilf, XILF, 0xC07) /* type = RIL_A EXCLUSIVE OR IMMEDIATE (low) */ \
+ V(iihf, IIHF, 0xC08) /* type = RIL_A INSERT IMMEDIATE (high) */ \
+ V(iilf, IILF, 0xC09) /* type = RIL_A INSERT IMMEDIATE (low) */ \
+ V(nihf, NIHF, 0xC0A) /* type = RIL_A AND IMMEDIATE (high) */ \
+ V(nilf, NILF, 0xC0B) /* type = RIL_A AND IMMEDIATE (low) */ \
+ V(oihf, OIHF, 0xC0C) /* type = RIL_A OR IMMEDIATE (high) */ \
+ V(oilf, OILF, 0xC0D) /* type = RIL_A OR IMMEDIATE (low) */ \
+ V(llihf, LLIHF, 0xC0E) /* type = RIL_A LOAD LOGICAL IMMEDIATE (high) */ \
+ V(llilf, LLILF, 0xC0F) /* type = RIL_A LOAD LOGICAL IMMEDIATE (low) */ \
+ V(msgfi, MSGFI, 0xC20) /* type = RIL_A MULTIPLY SINGLE IMMEDIATE (64<-32) */ \
+ V(msfi, MSFI, 0xC21) /* type = RIL_A MULTIPLY SINGLE IMMEDIATE (32) */ \
+ V(slgfi, SLGFI, \
+ 0xC24) /* type = RIL_A SUBTRACT LOGICAL IMMEDIATE (64<-32) */ \
+ V(slfi, SLFI, 0xC25) /* type = RIL_A SUBTRACT LOGICAL IMMEDIATE (32) */ \
+ V(agfi, AGFI, 0xC28) /* type = RIL_A ADD IMMEDIATE (64<-32) */ \
+ V(afi, AFI, 0xC29) /* type = RIL_A ADD IMMEDIATE (32) */ \
+ V(algfi, ALGFI, 0xC2A) /* type = RIL_A ADD LOGICAL IMMEDIATE (64<-32) */ \
+ V(alfi, ALFI, 0xC2B) /* type = RIL_A ADD LOGICAL IMMEDIATE (32) */ \
+ V(cgfi, CGFI, 0xC2C) /* type = RIL_A COMPARE IMMEDIATE (64<-32) */ \
+ V(cfi, CFI, 0xC2D) /* type = RIL_A COMPARE IMMEDIATE (32) */ \
+ V(clgfi, CLGFI, 0xC2E) /* type = RIL_A COMPARE LOGICAL IMMEDIATE (64<-32) */ \
+ V(clfi, CLFI, 0xC2F) /* type = RIL_A COMPARE LOGICAL IMMEDIATE (32) */ \
+ V(aih, AIH, 0xCC8) /* type = RIL_A ADD IMMEDIATE HIGH (32) */ \
+ V(alsih, ALSIH, \
+ 0xCCA) /* type = RIL_A ADD LOGICAL WITH SIGNED IMMEDIATE HIGH (32) */ \
+ V(alsihn, ALSIHN, \
+ 0xCCB) /* type = RIL_A ADD LOGICAL WITH SIGNED IMMEDIATE HIGH (32) */ \
+ V(cih, CIH, 0xCCD) /* type = RIL_A COMPARE IMMEDIATE HIGH (32) */ \
+ V(clih, CLIH, 0xCCF) /* type = RIL_A COMPARE LOGICAL IMMEDIATE HIGH (32) */
+
+#define S390_RIL_B_OPCODE_LIST(V) \
+ V(larl, LARL, 0xC00) /* type = RIL_B LOAD ADDRESS RELATIVE LONG */ \
+ V(brasl, BRASL, 0xC05) /* type = RIL_B BRANCH RELATIVE AND SAVE LONG */ \
+ V(llhrl, LLHRL, \
+ 0xC42) /* type = RIL_B LOAD LOGICAL HALFWORD RELATIVE LONG (32<-16) */ \
+ V(lghrl, LGHRL, \
+ 0xC44) /* type = RIL_B LOAD HALFWORD RELATIVE LONG (64<-16) */ \
+ V(lhrl, LHRL, 0xC45) /* type = RIL_B LOAD HALFWORD RELATIVE LONG (32<-16) */ \
+ V(llghrl, LLGHRL, \
+ 0xC46) /* type = RIL_B LOAD LOGICAL HALFWORD RELATIVE LONG (64<-16) */ \
+ V(sthrl, STHRL, 0xC47) /* type = RIL_B STORE HALFWORD RELATIVE LONG (16) */ \
+ V(lgrl, LGRL, 0xC48) /* type = RIL_B LOAD RELATIVE LONG (64) */ \
+ V(stgrl, STGRL, 0xC4B) /* type = RIL_B STORE RELATIVE LONG (64) */ \
+ V(lgfrl, LGFRL, 0xC4C) /* type = RIL_B LOAD RELATIVE LONG (64<-32) */ \
+ V(lrl, LRL, 0xC4D) /* type = RIL_B LOAD RELATIVE LONG (32) */ \
+ V(llgfrl, LLGFRL, \
+ 0xC4E) /* type = RIL_B LOAD LOGICAL RELATIVE LONG (64<-32) */ \
+ V(strl, STRL, 0xC4F) /* type = RIL_B STORE RELATIVE LONG (32) */ \
+ V(exrl, EXRL, 0xC60) /* type = RIL_B EXECUTE RELATIVE LONG */ \
+ V(cghrl, CGHRL, \
+ 0xC64) /* type = RIL_B COMPARE HALFWORD RELATIVE LONG (64<-16) */ \
+ V(chrl, CHRL, \
+ 0xC65) /* type = RIL_B COMPARE HALFWORD RELATIVE LONG (32<-16) */ \
+ V(clghrl, CLGHRL, \
+ 0xC66) /* type = RIL_B COMPARE LOGICAL RELATIVE LONG (64<-16) */ \
+ V(clhrl, CLHRL, \
+ 0xC67) /* type = RIL_B COMPARE LOGICAL RELATIVE LONG (32<-16) */ \
+ V(cgrl, CGRL, 0xC68) /* type = RIL_B COMPARE RELATIVE LONG (64) */ \
+ V(clgrl, CLGRL, 0xC6A) /* type = RIL_B COMPARE LOGICAL RELATIVE LONG (64) */ \
+ V(cgfrl, CGFRL, 0xC6C) /* type = RIL_B COMPARE RELATIVE LONG (64<-32) */ \
+ V(crl, CRL, 0xC6D) /* type = RIL_B COMPARE RELATIVE LONG (32) */ \
+ V(clgfrl, CLGFRL, \
+ 0xC6E) /* type = RIL_B COMPARE LOGICAL RELATIVE LONG (64<-32) */ \
+ V(clrl, CLRL, 0xC6F) /* type = RIL_B COMPARE LOGICAL RELATIVE LONG (32) */ \
+ V(brcth, BRCTH, 0xCC6) /* type = RIL_B BRANCH RELATIVE ON COUNT HIGH (32) */
+
+#define S390_VRS_B_OPCODE_LIST(V) \
+ V(vlvg, VLVG, 0xE722) /* type = VRS_B VECTOR LOAD VR ELEMENT FROM GR */ \
+ V(vll, VLL, 0xE737) /* type = VRS_B VECTOR LOAD WITH LENGTH */ \
+ V(vstl, VSTL, 0xE73F) /* type = VRS_B VECTOR STORE WITH LENGTH */
+
+#define S390_RIL_C_OPCODE_LIST(V) \
+ V(brcl, BRCL, 0xC04) /* type = RIL_C BRANCH RELATIVE ON CONDITION LONG */ \
+ V(pfdrl, PFDRL, 0xC62) /* type = RIL_C PREFETCH DATA RELATIVE LONG */
+
+#define S390_VRS_C_OPCODE_LIST(V) \
+ V(vlgv, VLGV, 0xE721) /* type = VRS_C VECTOR LOAD GR FROM VR ELEMENT */
+
+#define S390_RI_A_OPCODE_LIST(V) \
+ V(iihh, IIHH, 0xA50) /* type = RI_A INSERT IMMEDIATE (high high) */ \
+ V(iihl, IIHL, 0xA51) /* type = RI_A INSERT IMMEDIATE (high low) */ \
+ V(iilh, IILH, 0xA52) /* type = RI_A INSERT IMMEDIATE (low high) */ \
+ V(iill, IILL, 0xA53) /* type = RI_A INSERT IMMEDIATE (low low) */ \
+ V(nihh, NIHH, 0xA54) /* type = RI_A AND IMMEDIATE (high high) */ \
+ V(nihl, NIHL, 0xA55) /* type = RI_A AND IMMEDIATE (high low) */ \
+ V(nilh, NILH, 0xA56) /* type = RI_A AND IMMEDIATE (low high) */ \
+ V(nill, NILL, 0xA57) /* type = RI_A AND IMMEDIATE (low low) */ \
+ V(oihh, OIHH, 0xA58) /* type = RI_A OR IMMEDIATE (high high) */ \
+ V(oihl, OIHL, 0xA59) /* type = RI_A OR IMMEDIATE (high low) */ \
+ V(oilh, OILH, 0xA5A) /* type = RI_A OR IMMEDIATE (low high) */ \
+ V(oill, OILL, 0xA5B) /* type = RI_A OR IMMEDIATE (low low) */ \
+ V(llihh, LLIHH, 0xA5C) /* type = RI_A LOAD LOGICAL IMMEDIATE (high high) */ \
+ V(llihl, LLIHL, 0xA5D) /* type = RI_A LOAD LOGICAL IMMEDIATE (high low) */ \
+ V(llilh, LLILH, 0xA5E) /* type = RI_A LOAD LOGICAL IMMEDIATE (low high) */ \
+ V(llill, LLILL, 0xA5F) /* type = RI_A LOAD LOGICAL IMMEDIATE (low low) */ \
+ V(tmlh, TMLH, 0xA70) /* type = RI_A TEST UNDER MASK (low high) */ \
+ V(tmll, TMLL, 0xA71) /* type = RI_A TEST UNDER MASK (low low) */ \
+ V(tmhh, TMHH, 0xA72) /* type = RI_A TEST UNDER MASK (high high) */ \
+ V(tmhl, TMHL, 0xA73) /* type = RI_A TEST UNDER MASK (high low) */ \
+ V(lhi, LHI, 0xA78) /* type = RI_A LOAD HALFWORD IMMEDIATE (32)<-16 */ \
+ V(lghi, LGHI, 0xA79) /* type = RI_A LOAD HALFWORD IMMEDIATE (64<-16) */ \
+ V(ahi, AHI, 0xA7A) /* type = RI_A ADD HALFWORD IMMEDIATE (32<-16) */ \
+ V(aghi, AGHI, 0xA7B) /* type = RI_A ADD HALFWORD IMMEDIATE (64<-16) */ \
+ V(mhi, MHI, 0xA7C) /* type = RI_A MULTIPLY HALFWORD IMMEDIATE (32<-16) */ \
+ V(mghi, MGHI, 0xA7D) /* type = RI_A MULTIPLY HALFWORD IMMEDIATE (64<-16) */ \
+ V(chi, CHI, 0xA7E) /* type = RI_A COMPARE HALFWORD IMMEDIATE (32<-16) */ \
+ V(cghi, CGHI, 0xA7F) /* type = RI_A COMPARE HALFWORD IMMEDIATE (64<-16) */
+
+#define S390_RSI_OPCODE_LIST(V) \
+ V(brxh, BRXH, 0x84) /* type = RSI BRANCH RELATIVE ON INDEX HIGH (32) */ \
+ V(brxle, BRXLE, \
+ 0x85) /* type = RSI BRANCH RELATIVE ON INDEX LOW OR EQ. (32) */
+
+#define S390_RI_B_OPCODE_LIST(V) \
+ V(bras, BRAS, 0xA75) /* type = RI_B BRANCH RELATIVE AND SAVE */ \
+ V(brct, BRCT, 0xA76) /* type = RI_B BRANCH RELATIVE ON COUNT (32) */ \
+ V(brctg, BRCTG, 0xA77) /* type = RI_B BRANCH RELATIVE ON COUNT (64) */
+
+#define S390_RI_C_OPCODE_LIST(V) \
+ V(brc, BRC, 0xA74) /* type = RI_C BRANCH RELATIVE ON CONDITION */
+
+#define S390_SMI_OPCODE_LIST(V) \
+ V(bpp, BPP, 0xC7) /* type = SMI BRANCH PREDICTION PRELOAD */
+
+#define S390_RXY_A_OPCODE_LIST(V) \
+ V(ltg, LTG, 0xE302) /* type = RXY_A LOAD AND TEST (64) */ \
+ V(lrag, LRAG, 0xE303) /* type = RXY_A LOAD REAL ADDRESS (64) */ \
+ V(lg, LG, 0xE304) /* type = RXY_A LOAD (64) */ \
+ V(cvby, CVBY, 0xE306) /* type = RXY_A CONVERT TO BINARY (32) */ \
+ V(ag, AG, 0xE308) /* type = RXY_A ADD (64) */ \
+ V(sg, SG, 0xE309) /* type = RXY_A SUBTRACT (64) */ \
+ V(alg, ALG, 0xE30A) /* type = RXY_A ADD LOGICAL (64) */ \
+ V(slg, SLG, 0xE30B) /* type = RXY_A SUBTRACT LOGICAL (64) */ \
+ V(msg, MSG, 0xE30C) /* type = RXY_A MULTIPLY SINGLE (64) */ \
+ V(dsg, DSG, 0xE30D) /* type = RXY_A DIVIDE SINGLE (64) */ \
+ V(cvbg, CVBG, 0xE30E) /* type = RXY_A CONVERT TO BINARY (64) */ \
+ V(lrvg, LRVG, 0xE30F) /* type = RXY_A LOAD REVERSED (64) */ \
+ V(lt_z, LT, 0xE312) /* type = RXY_A LOAD AND TEST (32) */ \
+ V(lray, LRAY, 0xE313) /* type = RXY_A LOAD REAL ADDRESS (32) */ \
+ V(lgf, LGF, 0xE314) /* type = RXY_A LOAD (64<-32) */ \
+ V(lgh, LGH, 0xE315) /* type = RXY_A LOAD HALFWORD (64<-16) */ \
+ V(llgf, LLGF, 0xE316) /* type = RXY_A LOAD LOGICAL (64<-32) */ \
+ V(llgt, LLGT, \
+ 0xE317) /* type = RXY_A LOAD LOGICAL THIRTY ONE BITS (64<-31) */ \
+ V(agf, AGF, 0xE318) /* type = RXY_A ADD (64<-32) */ \
+ V(sgf, SGF, 0xE319) /* type = RXY_A SUBTRACT (64<-32) */ \
+ V(algf, ALGF, 0xE31A) /* type = RXY_A ADD LOGICAL (64<-32) */ \
+ V(slgf, SLGF, 0xE31B) /* type = RXY_A SUBTRACT LOGICAL (64<-32) */ \
+ V(msgf, MSGF, 0xE31C) /* type = RXY_A MULTIPLY SINGLE (64<-32) */ \
+ V(dsgf, DSGF, 0xE31D) /* type = RXY_A DIVIDE SINGLE (64<-32) */ \
+ V(lrv, LRV, 0xE31E) /* type = RXY_A LOAD REVERSED (32) */ \
+ V(lrvh, LRVH, 0xE31F) /* type = RXY_A LOAD REVERSED (16) */ \
+ V(cg, CG, 0xE320) /* type = RXY_A COMPARE (64) */ \
+ V(clg, CLG, 0xE321) /* type = RXY_A COMPARE LOGICAL (64) */ \
+ V(stg, STG, 0xE324) /* type = RXY_A STORE (64) */ \
+ V(ntstg, NTSTG, 0xE325) /* type = RXY_A NONTRANSACTIONAL STORE (64) */ \
+ V(cvdy, CVDY, 0xE326) /* type = RXY_A CONVERT TO DECIMAL (32) */ \
+ V(lzrg, LZRG, 0xE32A) /* type = RXY_A LOAD AND ZERO RIGHTMOST BYTE (64) */ \
+ V(cvdg, CVDG, 0xE32E) /* type = RXY_A CONVERT TO DECIMAL (64) */ \
+ V(strvg, STRVG, 0xE32F) /* type = RXY_A STORE REVERSED (64) */ \
+ V(cgf, CGF, 0xE330) /* type = RXY_A COMPARE (64<-32) */ \
+ V(clgf, CLGF, 0xE331) /* type = RXY_A COMPARE LOGICAL (64<-32) */ \
+ V(ltgf, LTGF, 0xE332) /* type = RXY_A LOAD AND TEST (64<-32) */ \
+ V(cgh, CGH, 0xE334) /* type = RXY_A COMPARE HALFWORD (64<-16) */ \
+ V(llzrgf, LLZRGF, \
+ 0xE33A) /* type = RXY_A LOAD LOGICAL AND ZERO RIGHTMOST BYTE (64<-32) */ \
+ V(lzrf, LZRF, 0xE33B) /* type = RXY_A LOAD AND ZERO RIGHTMOST BYTE (32) */ \
+ V(strv, STRV, 0xE33E) /* type = RXY_A STORE REVERSED (32) */ \
+ V(strvh, STRVH, 0xE33F) /* type = RXY_A STORE REVERSED (16) */ \
+ V(bctg, BCTG, 0xE346) /* type = RXY_A BRANCH ON COUNT (64) */ \
+ V(sty, STY, 0xE350) /* type = RXY_A STORE (32) */ \
+ V(msy, MSY, 0xE351) /* type = RXY_A MULTIPLY SINGLE (32) */ \
+ V(ny, NY, 0xE354) /* type = RXY_A AND (32) */ \
+ V(cly, CLY, 0xE355) /* type = RXY_A COMPARE LOGICAL (32) */ \
+ V(oy, OY, 0xE356) /* type = RXY_A OR (32) */ \
+ V(xy, XY, 0xE357) /* type = RXY_A EXCLUSIVE OR (32) */ \
+ V(ly, LY, 0xE358) /* type = RXY_A LOAD (32) */ \
+ V(cy, CY, 0xE359) /* type = RXY_A COMPARE (32) */ \
+ V(ay, AY, 0xE35A) /* type = RXY_A ADD (32) */ \
+ V(sy, SY, 0xE35B) /* type = RXY_A SUBTRACT (32) */ \
+ V(mfy, MFY, 0xE35C) /* type = RXY_A MULTIPLY (64<-32) */ \
+ V(aly, ALY, 0xE35E) /* type = RXY_A ADD LOGICAL (32) */ \
+ V(sly, SLY, 0xE35F) /* type = RXY_A SUBTRACT LOGICAL (32) */ \
+ V(sthy, STHY, 0xE370) /* type = RXY_A STORE HALFWORD (16) */ \
+ V(lay, LAY, 0xE371) /* type = RXY_A LOAD ADDRESS */ \
+ V(stcy, STCY, 0xE372) /* type = RXY_A STORE CHARACTER */ \
+ V(icy, ICY, 0xE373) /* type = RXY_A INSERT CHARACTER */ \
+ V(laey, LAEY, 0xE375) /* type = RXY_A LOAD ADDRESS EXTENDED */ \
+ V(lb, LB, 0xE376) /* type = RXY_A LOAD BYTE (32<-8) */ \
+ V(lgb, LGB, 0xE377) /* type = RXY_A LOAD BYTE (64<-8) */ \
+ V(lhy, LHY, 0xE378) /* type = RXY_A LOAD HALFWORD (32)<-16 */ \
+ V(chy, CHY, 0xE379) /* type = RXY_A COMPARE HALFWORD (32<-16) */ \
+ V(ahy, AHY, 0xE37A) /* type = RXY_A ADD HALFWORD (32<-16) */ \
+ V(shy, SHY, 0xE37B) /* type = RXY_A SUBTRACT HALFWORD (32<-16) */ \
+ V(mhy, MHY, 0xE37C) /* type = RXY_A MULTIPLY HALFWORD (32<-16) */ \
+ V(ng, NG, 0xE380) /* type = RXY_A AND (64) */ \
+ V(og, OG, 0xE381) /* type = RXY_A OR (64) */ \
+ V(xg, XG, 0xE382) /* type = RXY_A EXCLUSIVE OR (64) */ \
+ V(lgat, LGAT, 0xE385) /* type = RXY_A LOAD AND TRAP (64) */ \
+ V(mlg, MLG, 0xE386) /* type = RXY_A MULTIPLY LOGICAL (128<-64) */ \
+ V(dlg, DLG, 0xE387) /* type = RXY_A DIVIDE LOGICAL (64<-128) */ \
+ V(alcg, ALCG, 0xE388) /* type = RXY_A ADD LOGICAL WITH CARRY (64) */ \
+ V(slbg, SLBG, 0xE389) /* type = RXY_A SUBTRACT LOGICAL WITH BORROW (64) */ \
+ V(stpq, STPQ, 0xE38E) /* type = RXY_A STORE PAIR TO QUADWORD */ \
+ V(lpq, LPQ, 0xE38F) /* type = RXY_A LOAD PAIR FROM QUADWORD (64&64<-128) */ \
+ V(llgc, LLGC, 0xE390) /* type = RXY_A LOAD LOGICAL CHARACTER (64<-8) */ \
+ V(llgh, LLGH, 0xE391) /* type = RXY_A LOAD LOGICAL HALFWORD (64<-16) */ \
+ V(llc, LLC, 0xE394) /* type = RXY_A LOAD LOGICAL CHARACTER (32<-8) */ \
+ V(llh, LLH, 0xE395) /* type = RXY_A LOAD LOGICAL HALFWORD (32<-16) */ \
+ V(ml, ML, 0xE396) /* type = RXY_A MULTIPLY LOGICAL (64<-32) */ \
+ V(dl, DL, 0xE397) /* type = RXY_A DIVIDE LOGICAL (32<-64) */ \
+ V(alc, ALC, 0xE398) /* type = RXY_A ADD LOGICAL WITH CARRY (32) */ \
+ V(slb, SLB, 0xE399) /* type = RXY_A SUBTRACT LOGICAL WITH BORROW (32) */ \
+ V(llgtat, LLGTAT, \
+ 0xE39C) /* type = RXY_A LOAD LOGICAL THIRTY ONE BITS AND TRAP (64<-31) */ \
+ V(llgfat, LLGFAT, 0xE39D) /* type = RXY_A LOAD LOGICAL AND TRAP (64<-32) */ \
+ V(lat, LAT, 0xE39F) /* type = RXY_A LOAD AND TRAP (32L<-32) */ \
+ V(lbh, LBH, 0xE3C0) /* type = RXY_A LOAD BYTE HIGH (32<-8) */ \
+ V(llch, LLCH, 0xE3C2) /* type = RXY_A LOAD LOGICAL CHARACTER HIGH (32<-8) */ \
+ V(stch, STCH, 0xE3C3) /* type = RXY_A STORE CHARACTER HIGH (8) */ \
+ V(lhh, LHH, 0xE3C4) /* type = RXY_A LOAD HALFWORD HIGH (32<-16) */ \
+ V(llhh, LLHH, 0xE3C6) /* type = RXY_A LOAD LOGICAL HALFWORD HIGH (32<-16) */ \
+ V(sthh, STHH, 0xE3C7) /* type = RXY_A STORE HALFWORD HIGH (16) */ \
+ V(lfhat, LFHAT, 0xE3C8) /* type = RXY_A LOAD HIGH AND TRAP (32H<-32) */ \
+ V(lfh, LFH, 0xE3CA) /* type = RXY_A LOAD HIGH (32) */ \
+ V(stfh, STFH, 0xE3CB) /* type = RXY_A STORE HIGH (32) */ \
+ V(chf, CHF, 0xE3CD) /* type = RXY_A COMPARE HIGH (32) */ \
+ V(clhf, CLHF, 0xE3CF) /* type = RXY_A COMPARE LOGICAL HIGH (32) */ \
+ V(ley, LEY, 0xED64) /* type = RXY_A LOAD (short) */ \
+ V(ldy, LDY, 0xED65) /* type = RXY_A LOAD (long) */ \
+ V(stey, STEY, 0xED66) /* type = RXY_A STORE (short) */ \
+ V(stdy, STDY, 0xED67) /* type = RXY_A STORE (long) */ \
+ V(msc, MSC, 0xE353) /* type = RSY_A MULTIPLY SINGLE (32) */ \
+ V(msgc, MSGC, 0xE383) /* type = RSY_A MULTIPLY SINGLE (64) */
+
+#define S390_RXY_B_OPCODE_LIST(V) \
+ V(pfd, PFD, 0xE336) /* type = RXY_B PREFETCH DATA */
+
+#define S390_SIY_OPCODE_LIST(V) \
+ V(tmy, TMY, 0xEB51) /* type = SIY TEST UNDER MASK */ \
+ V(mviy, MVIY, 0xEB52) /* type = SIY MOVE (immediate) */ \
+ V(niy, NIY, 0xEB54) /* type = SIY AND (immediate) */ \
+ V(cliy, CLIY, 0xEB55) /* type = SIY COMPARE LOGICAL (immediate) */ \
+ V(oiy, OIY, 0xEB56) /* type = SIY OR (immediate) */ \
+ V(xiy, XIY, 0xEB57) /* type = SIY EXCLUSIVE OR (immediate) */ \
+ V(asi, ASI, 0xEB6A) /* type = SIY ADD IMMEDIATE (32<-8) */ \
+ V(alsi, ALSI, \
+ 0xEB6E) /* type = SIY ADD LOGICAL WITH SIGNED IMMEDIATE (32<-8) */ \
+ V(agsi, AGSI, 0xEB7A) /* type = SIY ADD IMMEDIATE (64<-8) */ \
+ V(algsi, ALGSI, \
+ 0xEB7E) /* type = SIY ADD LOGICAL WITH SIGNED IMMEDIATE (64<-8) */
+
+#define S390_SS_A_OPCODE_LIST(V) \
+ V(trtr, TRTR, 0xD0) /* type = SS_A TRANSLATE AND TEST REVERSE */ \
+ V(mvn, MVN, 0xD1) /* type = SS_A MOVE NUMERICS */ \
+ V(mvc, MVC, 0xD2) /* type = SS_A MOVE (character) */ \
+ V(mvz, MVZ, 0xD3) /* type = SS_A MOVE ZONES */ \
+ V(nc, NC, 0xD4) /* type = SS_A AND (character) */ \
+ V(clc, CLC, 0xD5) /* type = SS_A COMPARE LOGICAL (character) */ \
+ V(oc, OC, 0xD6) /* type = SS_A OR (character) */ \
+ V(xc, XC, 0xD7) /* type = SS_A EXCLUSIVE OR (character) */ \
+ V(tr, TR, 0xDC) /* type = SS_A TRANSLATE */ \
+ V(trt, TRT, 0xDD) /* type = SS_A TRANSLATE AND TEST */ \
+ V(ed, ED, 0xDE) /* type = SS_A EDIT */ \
+ V(edmk, EDMK, 0xDF) /* type = SS_A EDIT AND MARK */ \
+ V(unpku, UNPKU, 0xE2) /* type = SS_A UNPACK UNICODE */ \
+ V(mvcin, MVCIN, 0xE8) /* type = SS_A MOVE INVERSE */ \
+ V(unpka, UNPKA, 0xEA) /* type = SS_A UNPACK ASCII */
+
+#define S390_E_OPCODE_LIST(V) \
+ V(pr, PR, 0x0101) /* type = E PROGRAM RETURN */ \
+ V(upt, UPT, 0x0102) /* type = E UPDATE TREE */ \
+ V(ptff, PTFF, 0x0104) /* type = E PERFORM TIMING FACILITY FUNCTION */ \
+ V(sckpf, SCKPF, 0x0107) /* type = E SET CLOCK PROGRAMMABLE FIELD */ \
+ V(pfpo, PFPO, 0x010A) /* type = E PERFORM FLOATING-POINT OPERATION */ \
+ V(tam, TAM, 0x010B) /* type = E TEST ADDRESSING MODE */ \
+ V(sam24, SAM24, 0x010C) /* type = E SET ADDRESSING MODE (24) */ \
+ V(sam31, SAM31, 0x010D) /* type = E SET ADDRESSING MODE (31) */ \
+ V(sam64, SAM64, 0x010E) /* type = E SET ADDRESSING MODE (64) */ \
+ V(trap2, TRAP2, 0x01FF) /* type = E TRAP */
+
+#define S390_SS_B_OPCODE_LIST(V) \
+ V(mvo, MVO, 0xF1) /* type = SS_B MOVE WITH OFFSET */ \
+ V(pack, PACK, 0xF2) /* type = SS_B PACK */ \
+ V(unpk, UNPK, 0xF3) /* type = SS_B UNPACK */ \
+ V(zap, ZAP, 0xF8) /* type = SS_B ZERO AND ADD */ \
+ V(cp, CP, 0xF9) /* type = SS_B COMPARE DECIMAL */ \
+ V(ap, AP, 0xFA) /* type = SS_B ADD DECIMAL */ \
+ V(sp, SP, 0xFB) /* type = SS_B SUBTRACT DECIMAL */ \
+ V(mp, MP, 0xFC) /* type = SS_B MULTIPLY DECIMAL */ \
+ V(dp, DP, 0xFD) /* type = SS_B DIVIDE DECIMAL */
+
+#define S390_SS_C_OPCODE_LIST(V) \
+ V(srp, SRP, 0xF0) /* type = SS_C SHIFT AND ROUND DECIMAL */
+
+#define S390_SS_D_OPCODE_LIST(V) \
+ V(mvck, MVCK, 0xD9) /* type = SS_D MOVE WITH KEY */ \
+ V(mvcp, MVCP, 0xDA) /* type = SS_D MOVE TO PRIMARY */ \
+ V(mvcs, MVCS, 0xDB) /* type = SS_D MOVE TO SECONDARY */
+
+#define S390_SS_E_OPCODE_LIST(V) \
+ V(plo, PLO, 0xEE) /* type = SS_E PERFORM LOCKED OPERATION */ \
+ V(lmd, LMD, 0xEF) /* type = SS_E LOAD MULTIPLE DISJOINT (64<-32&32) */
+
+#define S390_I_OPCODE_LIST(V) \
+ V(svc, SVC, 0x0A) /* type = I SUPERVISOR CALL */
+
+#define S390_SS_F_OPCODE_LIST(V) \
+ V(pku, PKU, 0xE1) /* type = SS_F PACK UNICODE */ \
+ V(pka, PKA, 0xE9) /* type = SS_F PACK ASCII */
+
+#define S390_SSE_OPCODE_LIST(V) \
+ V(lasp, LASP, 0xE500) /* type = SSE LOAD ADDRESS SPACE PARAMETERS */ \
+ V(tprot, TPROT, 0xE501) /* type = SSE TEST PROTECTION */ \
+ V(strag, STRAG, 0xE502) /* type = SSE STORE REAL ADDRESS */ \
+ V(mvcsk, MVCSK, 0xE50E) /* type = SSE MOVE WITH SOURCE KEY */ \
+ V(mvcdk, MVCDK, 0xE50F) /* type = SSE MOVE WITH DESTINATION KEY */
+
+#define S390_SSF_OPCODE_LIST(V) \
+ V(mvcos, MVCOS, 0xC80) /* type = SSF MOVE WITH OPTIONAL SPECIFICATIONS */ \
+ V(ectg, ECTG, 0xC81) /* type = SSF EXTRACT CPU TIME */ \
+ V(csst, CSST, 0xC82) /* type = SSF COMPARE AND SWAP AND STORE */ \
+ V(lpd, LPD, 0xC84) /* type = SSF LOAD PAIR DISJOINT (32) */ \
+ V(lpdg, LPDG, 0xC85) /* type = SSF LOAD PAIR DISJOINT (64) */
+
+#define S390_RS_A_OPCODE_LIST(V) \
+ V(bxh, BXH, 0x86) /* type = RS_A BRANCH ON INDEX HIGH (32) */ \
+ V(bxle, BXLE, 0x87) /* type = RS_A BRANCH ON INDEX LOW OR EQUAL (32) */ \
+ V(srl, SRL, 0x88) /* type = RS_A SHIFT RIGHT SINGLE LOGICAL (32) */ \
+ V(sll, SLL, 0x89) /* type = RS_A SHIFT LEFT SINGLE LOGICAL (32) */ \
+ V(sra, SRA, 0x8A) /* type = RS_A SHIFT RIGHT SINGLE (32) */ \
+ V(sla, SLA, 0x8B) /* type = RS_A SHIFT LEFT SINGLE (32) */ \
+ V(srdl, SRDL, 0x8C) /* type = RS_A SHIFT RIGHT DOUBLE LOGICAL (64) */ \
+ V(sldl, SLDL, 0x8D) /* type = RS_A SHIFT LEFT DOUBLE LOGICAL (64) */ \
+ V(srda, SRDA, 0x8E) /* type = RS_A SHIFT RIGHT DOUBLE (64) */ \
+ V(slda, SLDA, 0x8F) /* type = RS_A SHIFT LEFT DOUBLE (64) */ \
+ V(stm, STM, 0x90) /* type = RS_A STORE MULTIPLE (32) */ \
+ V(lm, LM, 0x98) /* type = RS_A LOAD MULTIPLE (32) */ \
+ V(trace, TRACE, 0x99) /* type = RS_A TRACE (32) */ \
+ V(lam, LAM, 0x9A) /* type = RS_A LOAD ACCESS MULTIPLE */ \
+ V(stam, STAM, 0x9B) /* type = RS_A STORE ACCESS MULTIPLE */ \
+ V(mvcle, MVCLE, 0xA8) /* type = RS_A MOVE LONG EXTENDED */ \
+ V(clcle, CLCLE, 0xA9) /* type = RS_A COMPARE LOGICAL LONG EXTENDED */ \
+ V(sigp, SIGP, 0xAE) /* type = RS_A SIGNAL PROCESSOR */ \
+ V(stctl, STCTL, 0xB6) /* type = RS_A STORE CONTROL (32) */ \
+ V(lctl, LCTL, 0xB7) /* type = RS_A LOAD CONTROL (32) */ \
+ V(cs, CS, 0xBA) /* type = RS_A COMPARE AND SWAP (32) */ \
+ V(cds, CDS, 0xBB) /* type = RS_A COMPARE DOUBLE AND SWAP (32) */
+
+#define S390_RS_B_OPCODE_LIST(V) \
+ V(clm, CLM, 0xBD) /* type = RS_B COMPARE LOGICAL CHAR. UNDER MASK (low) */ \
+ V(stcm, STCM, 0xBE) /* type = RS_B STORE CHARACTERS UNDER MASK (low) */ \
+ V(icm, ICM, 0xBF) /* type = RS_B INSERT CHARACTERS UNDER MASK (low) */
+
+#define S390_S_OPCODE_LIST(V) \
+ V(lpsw, LPSW, 0x82) /* type = S LOAD PSW */ \
+ V(diagnose, DIAGNOSE, 0x83) /* type = S DIAGNOSE */ \
+ V(ts, TS, 0x93) /* type = S TEST AND SET */ \
+ V(stidp, STIDP, 0xB202) /* type = S STORE CPU ID */ \
+ V(sck, SCK, 0xB204) /* type = S SET CLOCK */ \
+ V(stck, STCK, 0xB205) /* type = S STORE CLOCK */ \
+ V(sckc, SCKC, 0xB206) /* type = S SET CLOCK COMPARATOR */ \
+ V(stckc, STCKC, 0xB207) /* type = S STORE CLOCK COMPARATOR */ \
+ V(spt, SPT, 0xB208) /* type = S SET CPU TIMER */ \
+ V(stpt, STPT, 0xB209) /* type = S STORE CPU TIMER */ \
+ V(spka, SPKA, 0xB20A) /* type = S SET PSW KEY FROM ADDRESS */ \
+ V(ipk, IPK, 0xB20B) /* type = S INSERT PSW KEY */ \
+ V(ptlb, PTLB, 0xB20D) /* type = S PURGE TLB */ \
+ V(spx, SPX, 0xB210) /* type = S SET PREFIX */ \
+ V(stpx, STPX, 0xB211) /* type = S STORE PREFIX */ \
+ V(stap, STAP, 0xB212) /* type = S STORE CPU ADDRESS */ \
+ V(pc, PC, 0xB218) /* type = S PROGRAM CALL */ \
+ V(sac, SAC, 0xB219) /* type = S SET ADDRESS SPACE CONTROL */ \
+ V(cfc, CFC, 0xB21A) /* type = S COMPARE AND FORM CODEWORD */ \
+ V(csch, CSCH, 0xB230) /* type = S CLEAR SUBCHANNEL */ \
+ V(hsch, HSCH, 0xB231) /* type = S HALT SUBCHANNEL */ \
+ V(msch, MSCH, 0xB232) /* type = S MODIFY SUBCHANNEL */ \
+ V(ssch, SSCH, 0xB233) /* type = S START SUBCHANNEL */ \
+ V(stsch, STSCH, 0xB234) /* type = S STORE SUBCHANNEL */ \
+ V(tsch, TSCH, 0xB235) /* type = S TEST SUBCHANNEL */ \
+ V(tpi, TPI, 0xB236) /* type = S TEST PENDING INTERRUPTION */ \
+ V(sal, SAL, 0xB237) /* type = S SET ADDRESS LIMIT */ \
+ V(rsch, RSCH, 0xB238) /* type = S RESUME SUBCHANNEL */ \
+ V(stcrw, STCRW, 0xB239) /* type = S STORE CHANNEL REPORT WORD */ \
+ V(stcps, STCPS, 0xB23A) /* type = S STORE CHANNEL PATH STATUS */ \
+ V(rchp, RCHP, 0xB23B) /* type = S RESET CHANNEL PATH */ \
+ V(schm, SCHM, 0xB23C) /* type = S SET CHANNEL MONITOR */ \
+ V(xsch, XSCH, 0xB276) /* type = S CANCEL SUBCHANNEL */ \
+ V(rp, RP_Z, 0xB277) /* type = S RESUME PROGRAM */ \
+ V(stcke, STCKE, 0xB278) /* type = S STORE CLOCK EXTENDED */ \
+ V(sacf, SACF, 0xB279) /* type = S SET ADDRESS SPACE CONTROL FAST */ \
+ V(stckf, STCKF, 0xB27C) /* type = S STORE CLOCK FAST */ \
+ V(stsi, STSI, 0xB27D) /* type = S STORE SYSTEM INFORMATION */ \
+ V(srnm, SRNM, 0xB299) /* type = S SET BFP ROUNDING MODE (2 bit) */ \
+ V(stfpc, STFPC, 0xB29C) /* type = S STORE FPC */ \
+ V(lfpc, LFPC, 0xB29D) /* type = S LOAD FPC */ \
+ V(stfle, STFLE, 0xB2B0) /* type = S STORE FACILITY LIST EXTENDED */ \
+ V(stfl, STFL, 0xB2B1) /* type = S STORE FACILITY LIST */ \
+ V(lpswe, LPSWE, 0xB2B2) /* type = S LOAD PSW EXTENDED */ \
+ V(srnmb, SRNMB, 0xB2B8) /* type = S SET BFP ROUNDING MODE (3 bit) */ \
+ V(srnmt, SRNMT, 0xB2B9) /* type = S SET DFP ROUNDING MODE */ \
+ V(lfas, LFAS, 0xB2BD) /* type = S LOAD FPC AND SIGNAL */ \
+ V(tend, TEND, 0xB2F8) /* type = S TRANSACTION END */ \
+ V(tabort, TABORT, 0xB2FC) /* type = S TRANSACTION ABORT */ \
+ V(trap4, TRAP4, 0xB2FF) /* type = S TRAP */
+
+#define S390_RX_A_OPCODE_LIST(V) \
+ V(la, LA, 0x41) /* type = RX_A LOAD ADDRESS */ \
+ V(stc, STC, 0x42) /* type = RX_A STORE CHARACTER */ \
+ V(ic_z, IC_z, 0x43) /* type = RX_A INSERT CHARACTER */ \
+ V(ex, EX, 0x44) /* type = RX_A EXECUTE */ \
+ V(bal, BAL, 0x45) /* type = RX_A BRANCH AND LINK */ \
+ V(bct, BCT, 0x46) /* type = RX_A BRANCH ON COUNT (32) */ \
+ V(lh, LH, 0x48) /* type = RX_A LOAD HALFWORD (32<-16) */ \
+ V(ch, CH, 0x49) /* type = RX_A COMPARE HALFWORD (32<-16) */ \
+ V(ah, AH, 0x4A) /* type = RX_A ADD HALFWORD (32<-16) */ \
+ V(sh, SH, 0x4B) /* type = RX_A SUBTRACT HALFWORD (32<-16) */ \
+ V(mh, MH, 0x4C) /* type = RX_A MULTIPLY HALFWORD (32<-16) */ \
+ V(bas, BAS, 0x4D) /* type = RX_A BRANCH AND SAVE */ \
+ V(cvd, CVD, 0x4E) /* type = RX_A CONVERT TO DECIMAL (32) */ \
+ V(cvb, CVB, 0x4F) /* type = RX_A CONVERT TO BINARY (32) */ \
+ V(st, ST, 0x50) /* type = RX_A STORE (32) */ \
+ V(lae, LAE, 0x51) /* type = RX_A LOAD ADDRESS EXTENDED */ \
+ V(n, N, 0x54) /* type = RX_A AND (32) */ \
+ V(cl, CL, 0x55) /* type = RX_A COMPARE LOGICAL (32) */ \
+ V(o, O, 0x56) /* type = RX_A OR (32) */ \
+ V(x, X, 0x57) /* type = RX_A EXCLUSIVE OR (32) */ \
+ V(l, L, 0x58) /* type = RX_A LOAD (32) */ \
+ V(c, C, 0x59) /* type = RX_A COMPARE (32) */ \
+ V(a, A, 0x5A) /* type = RX_A ADD (32) */ \
+ V(s, S, 0x5B) /* type = RX_A SUBTRACT (32) */ \
+ V(m, M, 0x5C) /* type = RX_A MULTIPLY (64<-32) */ \
+ V(d, D, 0x5D) /* type = RX_A DIVIDE (32<-64) */ \
+ V(al_z, AL, 0x5E) /* type = RX_A ADD LOGICAL (32) */ \
+ V(sl, SL, 0x5F) /* type = RX_A SUBTRACT LOGICAL (32) */ \
+ V(std, STD, 0x60) /* type = RX_A STORE (long) */ \
+ V(mxd, MXD, 0x67) /* type = RX_A MULTIPLY (long to extended HFP) */ \
+ V(ld, LD, 0x68) /* type = RX_A LOAD (long) */ \
+ V(cd, CD, 0x69) /* type = RX_A COMPARE (long HFP) */ \
+ V(ad, AD, 0x6A) /* type = RX_A ADD NORMALIZED (long HFP) */ \
+ V(sd, SD, 0x6B) /* type = RX_A SUBTRACT NORMALIZED (long HFP) */ \
+ V(md, MD, 0x6C) /* type = RX_A MULTIPLY (long HFP) */ \
+ V(dd, DD, 0x6D) /* type = RX_A DIVIDE (long HFP) */ \
+ V(aw, AW, 0x6E) /* type = RX_A ADD UNNORMALIZED (long HFP) */ \
+ V(sw, SW, 0x6F) /* type = RX_A SUBTRACT UNNORMALIZED (long HFP) */ \
+ V(ste, STE, 0x70) /* type = RX_A STORE (short) */ \
+ V(ms, MS, 0x71) /* type = RX_A MULTIPLY SINGLE (32) */ \
+ V(le_z, LE, 0x78) /* type = RX_A LOAD (short) */ \
+ V(ce, CE, 0x79) /* type = RX_A COMPARE (short HFP) */ \
+ V(ae, AE, 0x7A) /* type = RX_A ADD NORMALIZED (short HFP) */ \
+ V(se, SE, 0x7B) /* type = RX_A SUBTRACT NORMALIZED (short HFP) */ \
+ V(mde, MDE, 0x7C) /* type = RX_A MULTIPLY (short to long HFP) */ \
+ V(de, DE, 0x7D) /* type = RX_A DIVIDE (short HFP) */ \
+ V(au, AU, 0x7E) /* type = RX_A ADD UNNORMALIZED (short HFP) */ \
+ V(su, SU, 0x7F) /* type = RX_A SUBTRACT UNNORMALIZED (short HFP) */ \
+ V(ssm, SSM, 0x80) /* type = RX_A SET SYSTEM MASK */ \
+ V(lra, LRA, 0xB1) /* type = RX_A LOAD REAL ADDRESS (32) */ \
+ V(sth, STH, 0x40) /* type = RX_A STORE HALFWORD (16) */
+
+#define S390_RX_B_OPCODE_LIST(V) \
+ V(bc, BC, 0x47) /* type = RX_B BRANCH ON CONDITION */
+
+#define S390_RIE_A_OPCODE_LIST(V) \
+ V(cgit, CGIT, 0xEC70) /* type = RIE_A COMPARE IMMEDIATE AND TRAP (64<-16) */ \
+ V(clgit, CLGIT, \
+ 0xEC71) /* type = RIE_A COMPARE LOGICAL IMMEDIATE AND TRAP (64<-16) */ \
+ V(cit, CIT, 0xEC72) /* type = RIE_A COMPARE IMMEDIATE AND TRAP (32<-16) */ \
+ V(clfit, CLFIT, \
+ 0xEC73) /* type = RIE_A COMPARE LOGICAL IMMEDIATE AND TRAP (32<-16) */
+
+#define S390_RRD_OPCODE_LIST(V) \
+ V(maebr, MAEBR, 0xB30E) /* type = RRD MULTIPLY AND ADD (short BFP) */ \
+ V(msebr, MSEBR, 0xB30F) /* type = RRD MULTIPLY AND SUBTRACT (short BFP) */ \
+ V(madbr, MADBR, 0xB31E) /* type = RRD MULTIPLY AND ADD (long BFP) */ \
+ V(msdbr, MSDBR, 0xB31F) /* type = RRD MULTIPLY AND SUBTRACT (long BFP) */ \
+ V(maer, MAER, 0xB32E) /* type = RRD MULTIPLY AND ADD (short HFP) */ \
+ V(mser, MSER, 0xB32F) /* type = RRD MULTIPLY AND SUBTRACT (short HFP) */ \
+ V(maylr, MAYLR, \
+ 0xB338) /* type = RRD MULTIPLY AND ADD UNNRM. (long to ext. low HFP) */ \
+ V(mylr, MYLR, \
+ 0xB339) /* type = RRD MULTIPLY UNNORM. (long to ext. low HFP) */ \
+ V(mayr, MAYR, \
+ 0xB33A) /* type = RRD MULTIPLY & ADD UNNORMALIZED (long to ext. HFP) */ \
+ V(myr, MYR, \
+ 0xB33B) /* type = RRD MULTIPLY UNNORMALIZED (long to ext. HFP) */ \
+ V(mayhr, MAYHR, \
+ 0xB33C) /* type = RRD MULTIPLY AND ADD UNNRM. (long to ext. high HFP) */ \
+ V(myhr, MYHR, \
+ 0xB33D) /* type = RRD MULTIPLY UNNORM. (long to ext. high HFP) */ \
+ V(madr, MADR, 0xB33E) /* type = RRD MULTIPLY AND ADD (long HFP) */ \
+ V(msdr, MSDR, 0xB33F) /* type = RRD MULTIPLY AND SUBTRACT (long HFP) */
+
+#define S390_RIE_B_OPCODE_LIST(V) \
+ V(cgrj, CGRJ, 0xEC64) /* type = RIE_B COMPARE AND BRANCH RELATIVE (64) */ \
+ V(clgrj, CLGRJ, \
+ 0xEC65) /* type = RIE_B COMPARE LOGICAL AND BRANCH RELATIVE (64) */ \
+ V(crj, CRJ, 0xEC76) /* type = RIE_B COMPARE AND BRANCH RELATIVE (32) */ \
+ V(clrj, CLRJ, \
+ 0xEC77) /* type = RIE_B COMPARE LOGICAL AND BRANCH RELATIVE (32) */
+
+#define S390_RRE_OPCODE_LIST(V) \
+ V(ipm, IPM, 0xB222) /* type = RRE INSERT PROGRAM MASK */ \
+ V(ivsk, IVSK, 0xB223) /* type = RRE INSERT VIRTUAL STORAGE KEY */ \
+ V(iac, IAC, 0xB224) /* type = RRE INSERT ADDRESS SPACE CONTROL */ \
+ V(ssar, SSAR, 0xB225) /* type = RRE SET SECONDARY ASN */ \
+ V(epar, EPAR, 0xB226) /* type = RRE EXTRACT PRIMARY ASN */ \
+ V(esar, ESAR, 0xB227) /* type = RRE EXTRACT SECONDARY ASN */ \
+ V(pt, PT, 0xB228) /* type = RRE PROGRAM TRANSFER */ \
+ V(iske, ISKE, 0xB229) /* type = RRE INSERT STORAGE KEY EXTENDED */ \
+ V(rrbe, RRBE, 0xB22A) /* type = RRE RESET REFERENCE BIT EXTENDED */ \
+ V(tb, TB, 0xB22C) /* type = RRE TEST BLOCK */ \
+ V(dxr, DXR, 0xB22D) /* type = RRE DIVIDE (extended HFP) */ \
+ V(pgin, PGIN, 0xB22E) /* type = RRE PAGE IN */ \
+ V(pgout, PGOUT, 0xB22F) /* type = RRE PAGE OUT */ \
+ V(bakr, BAKR, 0xB240) /* type = RRE BRANCH AND STACK */ \
+ V(cksm, CKSM, 0xB241) /* type = RRE CHECKSUM */ \
+ V(sqdr, SQDR, 0xB244) /* type = RRE SQUARE ROOT (long HFP) */ \
+ V(sqer, SQER, 0xB245) /* type = RRE SQUARE ROOT (short HFP) */ \
+ V(stura, STURA, 0xB246) /* type = RRE STORE USING REAL ADDRESS (32) */ \
+ V(msta, MSTA, 0xB247) /* type = RRE MODIFY STACKED STATE */ \
+ V(palb, PALB, 0xB248) /* type = RRE PURGE ALB */ \
+ V(ereg, EREG, 0xB249) /* type = RRE EXTRACT STACKED REGISTERS (32) */ \
+ V(esta, ESTA, 0xB24A) /* type = RRE EXTRACT STACKED STATE */ \
+ V(lura, LURA, 0xB24B) /* type = RRE LOAD USING REAL ADDRESS (32) */ \
+ V(tar, TAR, 0xB24C) /* type = RRE TEST ACCESS */ \
+ V(cpya, CPYA, 0xB24D) /* type = RRE COPY ACCESS */ \
+ V(sar, SAR, 0xB24E) /* type = RRE SET ACCESS */ \
+ V(ear, EAR, 0xB24F) /* type = RRE EXTRACT ACCESS */ \
+ V(csp, CSP, 0xB250) /* type = RRE COMPARE AND SWAP AND PURGE (32) */ \
+ V(msr, MSR, 0xB252) /* type = RRE MULTIPLY SINGLE (32) */ \
+ V(mvpg, MVPG, 0xB254) /* type = RRE MOVE PAGE */ \
+ V(mvst, MVST, 0xB255) /* type = RRE MOVE STRING */ \
+ V(cuse, CUSE, 0xB257) /* type = RRE COMPARE UNTIL SUBSTRING EQUAL */ \
+ V(bsg, BSG, 0xB258) /* type = RRE BRANCH IN SUBSPACE GROUP */ \
+ V(bsa, BSA, 0xB25A) /* type = RRE BRANCH AND SET AUTHORITY */ \
+ V(clst, CLST, 0xB25D) /* type = RRE COMPARE LOGICAL STRING */ \
+ V(srst, SRST, 0xB25E) /* type = RRE SEARCH STRING */ \
+ V(cmpsc, CMPSC, 0xB263) /* type = RRE COMPRESSION CALL */ \
+ V(tre, TRE, 0xB2A5) /* type = RRE TRANSLATE EXTENDED */ \
+ V(etnd, ETND, 0xB2EC) /* type = RRE EXTRACT TRANSACTION NESTING DEPTH */ \
+ V(lpebr, LPEBR, 0xB300) /* type = RRE LOAD POSITIVE (short BFP) */ \
+ V(lnebr, LNEBR, 0xB301) /* type = RRE LOAD NEGATIVE (short BFP) */ \
+ V(ltebr, LTEBR, 0xB302) /* type = RRE LOAD AND TEST (short BFP) */ \
+ V(lcebr, LCEBR, 0xB303) /* type = RRE LOAD COMPLEMENT (short BFP) */ \
+ V(ldebr, LDEBR, \
+ 0xB304) /* type = RRE LOAD LENGTHENED (short to long BFP) */ \
+ V(lxdbr, LXDBR, \
+ 0xB305) /* type = RRE LOAD LENGTHENED (long to extended BFP) */ \
+ V(lxebr, LXEBR, \
+ 0xB306) /* type = RRE LOAD LENGTHENED (short to extended BFP) */ \
+ V(mxdbr, MXDBR, 0xB307) /* type = RRE MULTIPLY (long to extended BFP) */ \
+ V(kebr, KEBR, 0xB308) /* type = RRE COMPARE AND SIGNAL (short BFP) */ \
+ V(cebr, CEBR, 0xB309) /* type = RRE COMPARE (short BFP) */ \
+ V(aebr, AEBR, 0xB30A) /* type = RRE ADD (short BFP) */ \
+ V(sebr, SEBR, 0xB30B) /* type = RRE SUBTRACT (short BFP) */ \
+ V(mdebr, MDEBR, 0xB30C) /* type = RRE MULTIPLY (short to long BFP) */ \
+ V(debr, DEBR, 0xB30D) /* type = RRE DIVIDE (short BFP) */ \
+ V(lpdbr, LPDBR, 0xB310) /* type = RRE LOAD POSITIVE (long BFP) */ \
+ V(lndbr, LNDBR, 0xB311) /* type = RRE LOAD NEGATIVE (long BFP) */ \
+ V(ltdbr, LTDBR, 0xB312) /* type = RRE LOAD AND TEST (long BFP) */ \
+ V(lcdbr, LCDBR, 0xB313) /* type = RRE LOAD COMPLEMENT (long BFP) */ \
+ V(sqebr, SQEBR, 0xB314) /* type = RRE SQUARE ROOT (short BFP) */ \
+ V(sqdbr, SQDBR, 0xB315) /* type = RRE SQUARE ROOT (long BFP) */ \
+ V(sqxbr, SQXBR, 0xB316) /* type = RRE SQUARE ROOT (extended BFP) */ \
+ V(meebr, MEEBR, 0xB317) /* type = RRE MULTIPLY (short BFP) */ \
+ V(kdbr, KDBR, 0xB318) /* type = RRE COMPARE AND SIGNAL (long BFP) */ \
+ V(cdbr, CDBR, 0xB319) /* type = RRE COMPARE (long BFP) */ \
+ V(adbr, ADBR, 0xB31A) /* type = RRE ADD (long BFP) */ \
+ V(sdbr, SDBR, 0xB31B) /* type = RRE SUBTRACT (long BFP) */ \
+ V(mdbr, MDBR, 0xB31C) /* type = RRE MULTIPLY (long BFP) */ \
+ V(ddbr, DDBR, 0xB31D) /* type = RRE DIVIDE (long BFP) */ \
+ V(lder, LDER, 0xB324) /* type = RRE LOAD LENGTHENED (short to long HFP) */ \
+ V(lxdr, LXDR, \
+ 0xB325) /* type = RRE LOAD LENGTHENED (long to extended HFP) */ \
+ V(lxer, LXER, \
+ 0xB326) /* type = RRE LOAD LENGTHENED (short to extended HFP) */ \
+ V(sqxr, SQXR, 0xB336) /* type = RRE SQUARE ROOT (extended HFP) */ \
+ V(meer, MEER, 0xB337) /* type = RRE MULTIPLY (short HFP) */ \
+ V(lpxbr, LPXBR, 0xB340) /* type = RRE LOAD POSITIVE (extended BFP) */ \
+ V(lnxbr, LNXBR, 0xB341) /* type = RRE LOAD NEGATIVE (extended BFP) */ \
+ V(ltxbr, LTXBR, 0xB342) /* type = RRE LOAD AND TEST (extended BFP) */ \
+ V(lcxbr, LCXBR, 0xB343) /* type = RRE LOAD COMPLEMENT (extended BFP) */ \
+ V(kxbr, KXBR, 0xB348) /* type = RRE COMPARE AND SIGNAL (extended BFP) */ \
+ V(cxbr, CXBR, 0xB349) /* type = RRE COMPARE (extended BFP) */ \
+ V(axbr, AXBR, 0xB34A) /* type = RRE ADD (extended BFP) */ \
+ V(sxbr, SXBR, 0xB34B) /* type = RRE SUBTRACT (extended BFP) */ \
+ V(mxbr, MXBR, 0xB34C) /* type = RRE MULTIPLY (extended BFP) */ \
+ V(dxbr, DXBR, 0xB34D) /* type = RRE DIVIDE (extended BFP) */ \
+ V(thder, THDER, \
+ 0xB358) /* type = RRE CONVERT BFP TO HFP (short to long) */ \
+ V(thdr, THDR, 0xB359) /* type = RRE CONVERT BFP TO HFP (long) */ \
+ V(lpxr, LPXR, 0xB360) /* type = RRE LOAD POSITIVE (extended HFP) */ \
+ V(lnxr, LNXR, 0xB361) /* type = RRE LOAD NEGATIVE (extended HFP) */ \
+ V(ltxr, LTXR, 0xB362) /* type = RRE LOAD AND TEST (extended HFP) */ \
+ V(lcxr, LCXR, 0xB363) /* type = RRE LOAD COMPLEMENT (extended HFP) */ \
+ V(lxr, LXR, 0xB365) /* type = RRE LOAD (extended) */ \
+ V(lexr, LEXR, \
+ 0xB366) /* type = RRE LOAD ROUNDED (extended to short HFP) */ \
+ V(fixr, FIXR, 0xB367) /* type = RRE LOAD FP INTEGER (extended HFP) */ \
+ V(cxr, CXR, 0xB369) /* type = RRE COMPARE (extended HFP) */ \
+ V(lpdfr, LPDFR, 0xB370) /* type = RRE LOAD POSITIVE (long) */ \
+ V(lndfr, LNDFR, 0xB371) /* type = RRE LOAD NEGATIVE (long) */ \
+ V(lcdfr, LCDFR, 0xB373) /* type = RRE LOAD COMPLEMENT (long) */ \
+ V(lzer, LZER, 0xB374) /* type = RRE LOAD ZERO (short) */ \
+ V(lzdr, LZDR, 0xB375) /* type = RRE LOAD ZERO (long) */ \
+ V(lzxr, LZXR, 0xB376) /* type = RRE LOAD ZERO (extended) */ \
+ V(fier, FIER, 0xB377) /* type = RRE LOAD FP INTEGER (short HFP) */ \
+ V(fidr, FIDR, 0xB37F) /* type = RRE LOAD FP INTEGER (long HFP) */ \
+ V(sfpc, SFPC, 0xB384) /* type = RRE SET FPC */ \
+ V(sfasr, SFASR, 0xB385) /* type = RRE SET FPC AND SIGNAL */ \
+ V(efpc, EFPC, 0xB38C) /* type = RRE EXTRACT FPC */ \
+ V(cefr, CEFR, \
+ 0xB3B4) /* type = RRE CONVERT FROM FIXED (32 to short HFP) */ \
+ V(cdfr, CDFR, 0xB3B5) /* type = RRE CONVERT FROM FIXED (32 to long HFP) */ \
+ V(cxfr, CXFR, \
+ 0xB3B6) /* type = RRE CONVERT FROM FIXED (32 to extended HFP) */ \
+ V(ldgr, LDGR, 0xB3C1) /* type = RRE LOAD FPR FROM GR (64 to long) */ \
+ V(cegr, CEGR, \
+ 0xB3C4) /* type = RRE CONVERT FROM FIXED (64 to short HFP) */ \
+ V(cdgr, CDGR, 0xB3C5) /* type = RRE CONVERT FROM FIXED (64 to long HFP) */ \
+ V(cxgr, CXGR, \
+ 0xB3C6) /* type = RRE CONVERT FROM FIXED (64 to extended HFP) */ \
+ V(lgdr, LGDR, 0xB3CD) /* type = RRE LOAD GR FROM FPR (long to 64) */ \
+ V(ltdtr, LTDTR, 0xB3D6) /* type = RRE LOAD AND TEST (long DFP) */ \
+ V(ltxtr, LTXTR, 0xB3DE) /* type = RRE LOAD AND TEST (extended DFP) */ \
+ V(kdtr, KDTR, 0xB3E0) /* type = RRE COMPARE AND SIGNAL (long DFP) */ \
+ V(cudtr, CUDTR, 0xB3E2) /* type = RRE CONVERT TO UNSIGNED PACKED (long */ \
+ /* DFP to 64) CUDTR */ \
+ V(cdtr, CDTR, 0xB3E4) /* type = RRE COMPARE (long DFP) */ \
+ V(eedtr, EEDTR, \
+ 0xB3E5) /* type = RRE EXTRACT BIASED EXPONENT (long DFP to 64) */ \
+ V(esdtr, ESDTR, \
+ 0xB3E7) /* type = RRE EXTRACT SIGNIFICANCE (long DFP to 64) */ \
+ V(kxtr, KXTR, 0xB3E8) /* type = RRE COMPARE AND SIGNAL (extended DFP) */ \
+ V(cuxtr, CUXTR, \
+ 0xB3EA) /* type = RRE CONVERT TO UNSIGNED PACKED (extended DFP */ \
+ /* CUXTR to 128) */ \
+ V(cxtr, CXTR, 0xB3EC) /* type = RRE COMPARE (extended DFP) */ \
+ V(eextr, EEXTR, \
+ 0xB3ED) /* type = RRE EXTRACT BIASED EXPONENT (extended DFP to 64) */ \
+ V(esxtr, ESXTR, \
+ 0xB3EF) /* type = RRE EXTRACT SIGNIFICANCE (extended DFP to 64) */ \
+ V(cdutr, CDUTR, \
+ 0xB3F2) /* type = RRE CONVERT FROM UNSIGNED PACKED (64 to long DFP) */ \
+ V(cdstr, CDSTR, \
+ 0xB3F3) /* type = RRE CONVERT FROM SIGNED PACKED (64 to long DFP) */ \
+ V(cedtr, CEDTR, \
+ 0xB3F4) /* type = RRE COMPARE BIASED EXPONENT (long DFP) */ \
+ V(cxutr, CXUTR, \
+ 0xB3FA) /* type = RRE CONVERT FROM UNSIGNED PACKED (128 to ext. DFP) */ \
+ V(cxstr, CXSTR, 0xB3FB) /* type = RRE CONVERT FROM SIGNED PACKED (128 to*/ \
+ /* extended DFP) */ \
+ V(cextr, CEXTR, \
+ 0xB3FC) /* type = RRE COMPARE BIASED EXPONENT (extended DFP) */ \
+ V(lpgr, LPGR, 0xB900) /* type = RRE LOAD POSITIVE (64) */ \
+ V(lngr, LNGR, 0xB901) /* type = RRE LOAD NEGATIVE (64) */ \
+ V(ltgr, LTGR, 0xB902) /* type = RRE LOAD AND TEST (64) */ \
+ V(lcgr, LCGR, 0xB903) /* type = RRE LOAD COMPLEMENT (64) */ \
+ V(lgr, LGR, 0xB904) /* type = RRE LOAD (64) */ \
+ V(lurag, LURAG, 0xB905) /* type = RRE LOAD USING REAL ADDRESS (64) */ \
+ V(lgbr, LGBR, 0xB906) /* type = RRE LOAD BYTE (64<-8) */ \
+ V(lghr, LGHR, 0xB907) /* type = RRE LOAD HALFWORD (64<-16) */ \
+ V(agr, AGR, 0xB908) /* type = RRE ADD (64) */ \
+ V(sgr, SGR, 0xB909) /* type = RRE SUBTRACT (64) */ \
+ V(algr, ALGR, 0xB90A) /* type = RRE ADD LOGICAL (64) */ \
+ V(slgr, SLGR, 0xB90B) /* type = RRE SUBTRACT LOGICAL (64) */ \
+ V(msgr, MSGR, 0xB90C) /* type = RRE MULTIPLY SINGLE (64) */ \
+ V(dsgr, DSGR, 0xB90D) /* type = RRE DIVIDE SINGLE (64) */ \
+ V(eregg, EREGG, 0xB90E) /* type = RRE EXTRACT STACKED REGISTERS (64) */ \
+ V(lrvgr, LRVGR, 0xB90F) /* type = RRE LOAD REVERSED (64) */ \
+ V(lpgfr, LPGFR, 0xB910) /* type = RRE LOAD POSITIVE (64<-32) */ \
+ V(lngfr, LNGFR, 0xB911) /* type = RRE LOAD NEGATIVE (64<-32) */ \
+ V(ltgfr, LTGFR, 0xB912) /* type = RRE LOAD AND TEST (64<-32) */ \
+ V(lcgfr, LCGFR, 0xB913) /* type = RRE LOAD COMPLEMENT (64<-32) */ \
+ V(lgfr, LGFR, 0xB914) /* type = RRE LOAD (64<-32) */ \
+ V(llgfr, LLGFR, 0xB916) /* type = RRE LOAD LOGICAL (64<-32) */ \
+ V(llgtr, LLGTR, \
+ 0xB917) /* type = RRE LOAD LOGICAL THIRTY ONE BITS (64<-31) */ \
+ V(agfr, AGFR, 0xB918) /* type = RRE ADD (64<-32) */ \
+ V(sgfr, SGFR, 0xB919) /* type = RRE SUBTRACT (64<-32) */ \
+ V(algfr, ALGFR, 0xB91A) /* type = RRE ADD LOGICAL (64<-32) */ \
+ V(slgfr, SLGFR, 0xB91B) /* type = RRE SUBTRACT LOGICAL (64<-32) */ \
+ V(msgfr, MSGFR, 0xB91C) /* type = RRE MULTIPLY SINGLE (64<-32) */ \
+ V(dsgfr, DSGFR, 0xB91D) /* type = RRE DIVIDE SINGLE (64<-32) */ \
+ V(kmac, KMAC, 0xB91E) /* type = RRE COMPUTE MESSAGE AUTHENTICATION CODE */ \
+ V(lrvr, LRVR, 0xB91F) /* type = RRE LOAD REVERSED (32) */ \
+ V(cgr, CGR, 0xB920) /* type = RRE COMPARE (64) */ \
+ V(clgr, CLGR, 0xB921) /* type = RRE COMPARE LOGICAL (64) */ \
+ V(sturg, STURG, 0xB925) /* type = RRE STORE USING REAL ADDRESS (64) */ \
+ V(lbr, LBR, 0xB926) /* type = RRE LOAD BYTE (32<-8) */ \
+ V(lhr, LHR, 0xB927) /* type = RRE LOAD HALFWORD (32<-16) */ \
+ V(pckmo, PCKMO, \
+ 0xB928) /* type = RRE PERFORM CRYPTOGRAPHIC KEY MGMT. OPERATIONS */ \
+ V(kmf, KMF, 0xB92A) /* type = RRE CIPHER MESSAGE WITH CIPHER FEEDBACK */ \
+ V(kmo, KMO, 0xB92B) /* type = RRE CIPHER MESSAGE WITH OUTPUT FEEDBACK */ \
+ V(pcc, PCC, 0xB92C) /* type = RRE PERFORM CRYPTOGRAPHIC COMPUTATION */ \
+ V(km, KM, 0xB92E) /* type = RRE CIPHER MESSAGE */ \
+ V(kmc, KMC, 0xB92F) /* type = RRE CIPHER MESSAGE WITH CHAINING */ \
+ V(cgfr, CGFR, 0xB930) /* type = RRE COMPARE (64<-32) */ \
+ V(clgfr, CLGFR, 0xB931) /* type = RRE COMPARE LOGICAL (64<-32) */ \
+ V(ppno, PPNO, \
+ 0xB93C) /* type = RRE PERFORM PSEUDORANDOM NUMBER OPERATION */ \
+ V(kimd, KIMD, 0xB93E) /* type = RRE COMPUTE INTERMEDIATE MESSAGE DIGEST */ \
+ V(klmd, KLMD, 0xB93F) /* type = RRE COMPUTE LAST MESSAGE DIGEST */ \
+ V(bctgr, BCTGR, 0xB946) /* type = RRE BRANCH ON COUNT (64) */ \
+ V(cdftr, CDFTR, \
+ 0xB951) /* type = RRE CONVERT FROM FIXED (32 to long DFP) */ \
+ V(cxftr, CXFTR, \
+ 0xB959) /* type = RRE CONVERT FROM FIXED (32 to extended DFP) */ \
+ V(ngr, NGR, 0xB980) /* type = RRE AND (64) */ \
+ V(ogr, OGR, 0xB981) /* type = RRE OR (64) */ \
+ V(xgr, XGR, 0xB982) /* type = RRE EXCLUSIVE OR (64) */ \
+ V(flogr, FLOGR, 0xB983) /* type = RRE FIND LEFTMOST ONE */ \
+ V(llgcr, LLGCR, 0xB984) /* type = RRE LOAD LOGICAL CHARACTER (64<-8) */ \
+ V(llghr, LLGHR, 0xB985) /* type = RRE LOAD LOGICAL HALFWORD (64<-16) */ \
+ V(mlgr, MLGR, 0xB986) /* type = RRE MULTIPLY LOGICAL (128<-64) */ \
+ V(dlgr, DLGR, 0xB987) /* type = RRE DIVIDE LOGICAL (64<-128) */ \
+ V(alcgr, ALCGR, 0xB988) /* type = RRE ADD LOGICAL WITH CARRY (64) */ \
+ V(slbgr, SLBGR, 0xB989) /* type = RRE SUBTRACT LOGICAL WITH BORROW (64) */ \
+ V(cspg, CSPG, 0xB98A) /* type = RRE COMPARE AND SWAP AND PURGE (64) */ \
+ V(epsw, EPSW, 0xB98D) /* type = RRE EXTRACT PSW */ \
+ V(llcr, LLCR, 0xB994) /* type = RRE LOAD LOGICAL CHARACTER (32<-8) */ \
+ V(llhr, LLHR, 0xB995) /* type = RRE LOAD LOGICAL HALFWORD (32<-16) */ \
+ V(mlr, MLR, 0xB996) /* type = RRE MULTIPLY LOGICAL (64<-32) */ \
+ V(dlr, DLR, 0xB997) /* type = RRE DIVIDE LOGICAL (32<-64) */ \
+ V(alcr, ALCR, 0xB998) /* type = RRE ADD LOGICAL WITH CARRY (32) */ \
+ V(slbr, SLBR, 0xB999) /* type = RRE SUBTRACT LOGICAL WITH BORROW (32) */ \
+ V(epair, EPAIR, 0xB99A) /* type = RRE EXTRACT PRIMARY ASN AND INSTANCE */ \
+ V(esair, ESAIR, \
+ 0xB99B) /* type = RRE EXTRACT SECONDARY ASN AND INSTANCE */ \
+ V(esea, ESEA, 0xB99D) /* type = RRE EXTRACT AND SET EXTENDED AUTHORITY */ \
+ V(pti, PTI, 0xB99E) /* type = RRE PROGRAM TRANSFER WITH INSTANCE */ \
+ V(ssair, SSAIR, 0xB99F) /* type = RRE SET SECONDARY ASN WITH INSTANCE */ \
+ V(ptf, PTF, 0xB9A2) /* type = RRE PERFORM TOPOLOGY FUNCTION */ \
+ V(rrbm, RRBM, 0xB9AE) /* type = RRE RESET REFERENCE BITS MULTIPLE */ \
+ V(pfmf, PFMF, 0xB9AF) /* type = RRE PERFORM FRAME MANAGEMENT FUNCTION */ \
+ V(cu41, CU41, 0xB9B2) /* type = RRE CONVERT UTF-32 TO UTF-8 */ \
+ V(cu42, CU42, 0xB9B3) /* type = RRE CONVERT UTF-32 TO UTF-16 */ \
+ V(srstu, SRSTU, 0xB9BE) /* type = RRE SEARCH STRING UNICODE */ \
+ V(chhr, CHHR, 0xB9CD) /* type = RRE COMPARE HIGH (32) */ \
+ V(clhhr, CLHHR, 0xB9CF) /* type = RRE COMPARE LOGICAL HIGH (32) */ \
+ V(chlr, CHLR, 0xB9DD) /* type = RRE COMPARE HIGH (32) */ \
+ V(clhlr, CLHLR, 0xB9DF) /* type = RRE COMPARE LOGICAL HIGH (32) */ \
+ V(popcnt, POPCNT_Z, 0xB9E1) /* type = RRE POPULATION COUNT */
+
+#define S390_RIE_C_OPCODE_LIST(V) \
+ V(cgij, CGIJ, \
+ 0xEC7C) /* type = RIE_C COMPARE IMMEDIATE AND BRANCH RELATIVE (64<-8) */ \
+ V(clgij, CLGIJ, \
+ 0xEC7D) /* type = RIE_C COMPARE LOGICAL IMMEDIATE AND BRANCH RELATIVE */ \
+ /* (64<-8) */ \
+ V(cij, CIJ, \
+ 0xEC7E) /* type = RIE_C COMPARE IMMEDIATE AND BRANCH RELATIVE (32<-8) */ \
+ V(clij, CLIJ, 0xEC7F) /* type = RIE_C COMPARE LOGICAL IMMEDIATE AND */ \
+ /* BRANCH RELATIVE (32<-8) */
+
+#define S390_RIE_D_OPCODE_LIST(V) \
+ V(ahik, AHIK, 0xECD8) /* type = RIE_D ADD IMMEDIATE (32<-16) */ \
+ V(aghik, AGHIK, 0xECD9) /* type = RIE_D ADD IMMEDIATE (64<-16) */ \
+ V(alhsik, ALHSIK, \
+ 0xECDA) /* type = RIE_D ADD LOGICAL WITH SIGNED IMMEDIATE (32<-16) */ \
+ V(alghsik, ALGHSIK, \
+ 0xECDB) /* type = RIE_D ADD LOGICAL WITH SIGNED IMMEDIATE (64<-16) */
+
+#define S390_VRV_OPCODE_LIST(V) \
+ V(vgeg, VGEG, 0xE712) /* type = VRV VECTOR GATHER ELEMENT (64) */ \
+ V(vgef, VGEF, 0xE713) /* type = VRV VECTOR GATHER ELEMENT (32) */ \
+ V(vsceg, VSCEG, 0xE71A) /* type = VRV VECTOR SCATTER ELEMENT (64) */ \
+ V(vscef, VSCEF, 0xE71B) /* type = VRV VECTOR SCATTER ELEMENT (32) */
+
+#define S390_RIE_E_OPCODE_LIST(V) \
+ V(brxhg, BRXHG, \
+ 0xEC44) /* type = RIE_E BRANCH RELATIVE ON INDEX HIGH (64) */ \
+ V(brxlg, BRXLG, \
+ 0xEC45) /* type = RIE_E BRANCH RELATIVE ON INDEX LOW OR EQ. (64) */
+
+#define S390_RR_OPCODE_LIST(V) \
+ V(awr, AWR, 0x2E) /* type = RR ADD UNNORMALIZED (long HFP) */ \
+ V(spm, SPM, 0x04) /* type = RR SET PROGRAM MASK */ \
+ V(balr, BALR, 0x05) /* type = RR BRANCH AND LINK */ \
+ V(bctr, BCTR, 0x06) /* type = RR BRANCH ON COUNT (32) */ \
+ V(bcr, BCR, 0x07) /* type = RR BRANCH ON CONDITION */ \
+ V(bsm, BSM, 0x0B) /* type = RR BRANCH AND SET MODE */ \
+ V(bassm, BASSM, 0x0C) /* type = RR BRANCH AND SAVE AND SET MODE */ \
+ V(basr, BASR, 0x0D) /* type = RR BRANCH AND SAVE */ \
+ V(mvcl, MVCL, 0x0E) /* type = RR MOVE LONG */ \
+ V(clcl, CLCL, 0x0F) /* type = RR COMPARE LOGICAL LONG */ \
+ V(lpr, LPR, 0x10) /* type = RR LOAD POSITIVE (32) */ \
+ V(lnr, LNR, 0x11) /* type = RR LOAD NEGATIVE (32) */ \
+ V(ltr, LTR, 0x12) /* type = RR LOAD AND TEST (32) */ \
+ V(lcr, LCR, 0x13) /* type = RR LOAD COMPLEMENT (32) */ \
+ V(nr, NR, 0x14) /* type = RR AND (32) */ \
+ V(clr, CLR, 0x15) /* type = RR COMPARE LOGICAL (32) */ \
+ V(or_z, OR, 0x16) /* type = RR OR (32) */ \
+ V(xr, XR, 0x17) /* type = RR EXCLUSIVE OR (32) */ \
+ V(lr, LR, 0x18) /* type = RR LOAD (32) */ \
+ V(cr_z, CR, 0x19) /* type = RR COMPARE (32) */ \
+ V(ar, AR, 0x1A) /* type = RR ADD (32) */ \
+ V(sr, SR, 0x1B) /* type = RR SUBTRACT (32) */ \
+ V(mr_z, MR, 0x1C) /* type = RR MULTIPLY (64<-32) */ \
+ V(dr, DR, 0x1D) /* type = RR DIVIDE (32<-64) */ \
+ V(alr, ALR, 0x1E) /* type = RR ADD LOGICAL (32) */ \
+ V(slr, SLR, 0x1F) /* type = RR SUBTRACT LOGICAL (32) */ \
+ V(lpdr, LPDR, 0x20) /* type = RR LOAD POSITIVE (long HFP) */ \
+ V(lndr, LNDR, 0x21) /* type = RR LOAD NEGATIVE (long HFP) */ \
+ V(ltdr, LTDR, 0x22) /* type = RR LOAD AND TEST (long HFP) */ \
+ V(lcdr, LCDR, 0x23) /* type = RR LOAD COMPLEMENT (long HFP) */ \
+ V(hdr, HDR, 0x24) /* type = RR HALVE (long HFP) */ \
+ V(ldxr, LDXR, 0x25) /* type = RR LOAD ROUNDED (extended to long HFP) */ \
+ V(mxr, MXR, 0x26) /* type = RR MULTIPLY (extended HFP) */ \
+ V(mxdr, MXDR, 0x27) /* type = RR MULTIPLY (long to extended HFP) */ \
+ V(ldr, LDR, 0x28) /* type = RR LOAD (long) */ \
+ V(cdr, CDR, 0x29) /* type = RR COMPARE (long HFP) */ \
+ V(adr, ADR, 0x2A) /* type = RR ADD NORMALIZED (long HFP) */ \
+ V(sdr, SDR, 0x2B) /* type = RR SUBTRACT NORMALIZED (long HFP) */ \
+ V(mdr, MDR, 0x2C) /* type = RR MULTIPLY (long HFP) */ \
+ V(ddr, DDR, 0x2D) /* type = RR DIVIDE (long HFP) */ \
+ V(swr, SWR, 0x2F) /* type = RR SUBTRACT UNNORMALIZED (long HFP) */ \
+ V(lper, LPER, 0x30) /* type = RR LOAD POSITIVE (short HFP) */ \
+ V(lner, LNER, 0x31) /* type = RR LOAD NEGATIVE (short HFP) */ \
+ V(lter, LTER, 0x32) /* type = RR LOAD AND TEST (short HFP) */ \
+ V(lcer, LCER, 0x33) /* type = RR LOAD COMPLEMENT (short HFP) */ \
+ V(her_z, HER_Z, 0x34) /* type = RR HALVE (short HFP) */ \
+ V(ledr, LEDR, 0x35) /* type = RR LOAD ROUNDED (long to short HFP) */ \
+ V(axr, AXR, 0x36) /* type = RR ADD NORMALIZED (extended HFP) */ \
+ V(sxr, SXR, 0x37) /* type = RR SUBTRACT NORMALIZED (extended HFP) */ \
+ V(ler, LER, 0x38) /* type = RR LOAD (short) */ \
+ V(cer, CER, 0x39) /* type = RR COMPARE (short HFP) */ \
+ V(aer, AER, 0x3A) /* type = RR ADD NORMALIZED (short HFP) */ \
+ V(ser, SER, 0x3B) /* type = RR SUBTRACT NORMALIZED (short HFP) */ \
+ V(mder, MDER, 0x3C) /* type = RR MULTIPLY (short to long HFP) */ \
+ V(der, DER, 0x3D) /* type = RR DIVIDE (short HFP) */ \
+ V(aur, AUR, 0x3E) /* type = RR ADD UNNORMALIZED (short HFP) */ \
+ V(sur, SUR, 0x3F) /* type = RR SUBTRACT UNNORMALIZED (short HFP) */
+
+#define S390_RIE_F_OPCODE_LIST(V) \
+ V(risblg, RISBLG, \
+ 0xEC51) /* type = RIE_F ROTATE THEN INSERT SELECTED BITS LOW (64) */ \
+ V(rnsbg, RNSBG, \
+ 0xEC54) /* type = RIE_F ROTATE THEN AND SELECTED BITS (64) */ \
+ V(risbg, RISBG, \
+ 0xEC55) /* type = RIE_F ROTATE THEN INSERT SELECTED BITS (64) */ \
+ V(rosbg, ROSBG, 0xEC56) /* type = RIE_F ROTATE THEN OR SELECTED BITS (64) */ \
+ V(rxsbg, RXSBG, \
+ 0xEC57) /* type = RIE_F ROTATE THEN EXCLUSIVE OR SELECT. BITS (64) */ \
+ V(risbgn, RISBGN, \
+ 0xEC59) /* type = RIE_F ROTATE THEN INSERT SELECTED BITS (64) */ \
+ V(risbhg, RISBHG, \
+ 0xEC5D) /* type = RIE_F ROTATE THEN INSERT SELECTED BITS HIGH (64) */
+
+#define S390_VRX_OPCODE_LIST(V) \
+ V(vleb, VLEB, 0xE700) /* type = VRX VECTOR LOAD ELEMENT (8) */ \
+ V(vleh, VLEH, 0xE701) /* type = VRX VECTOR LOAD ELEMENT (16) */ \
+ V(vleg, VLEG, 0xE702) /* type = VRX VECTOR LOAD ELEMENT (64) */ \
+ V(vlef, VLEF, 0xE703) /* type = VRX VECTOR LOAD ELEMENT (32) */ \
+ V(vllez, VLLEZ, \
+ 0xE704) /* type = VRX VECTOR LOAD LOGICAL ELEMENT AND ZERO */ \
+ V(vlrep, VLREP, 0xE705) /* type = VRX VECTOR LOAD AND REPLICATE */ \
+ V(vl, VL, 0xE706) /* type = VRX VECTOR LOAD */ \
+ V(vlbb, VLBB, 0xE707) /* type = VRX VECTOR LOAD TO BLOCK BOUNDARY */ \
+ V(vsteb, VSTEB, 0xE708) /* type = VRX VECTOR STORE ELEMENT (8) */ \
+ V(vsteh, VSTEH, 0xE709) /* type = VRX VECTOR STORE ELEMENT (16) */ \
+ V(vsteg, VSTEG, 0xE70A) /* type = VRX VECTOR STORE ELEMENT (64) */ \
+ V(vstef, VSTEF, 0xE70B) /* type = VRX VECTOR STORE ELEMENT (32) */ \
+ V(vst, VST, 0xE70E) /* type = VRX VECTOR STORE */ \
+ V(vlbr, VLBR, 0xE606) /* type = VRX VECTOR LOAD BYTE REVERSED ELEMENTS */ \
+ V(vstbr, VSTBR, 0xE60E) /* type = VRX VECTOR STORE BYTE REVERSED ELEMENTS \
+ */
+
+#define S390_RIE_G_OPCODE_LIST(V) \
+ V(lochi, LOCHI, \
+ 0xEC42) /* type = RIE_G LOAD HALFWORD IMMEDIATE ON CONDITION (32<-16) */ \
+ V(locghi, LOCGHI, \
+ 0xEC46) /* type = RIE_G LOAD HALFWORD IMMEDIATE ON CONDITION (64<-16) */ \
+ V(lochhi, LOCHHI, 0xEC4E) /* type = RIE_G LOAD HALFWORD HIGH IMMEDIATE */ \
+ /* ON CONDITION (32<-16) */
+
+#define S390_RRS_OPCODE_LIST(V) \
+ V(cgrb, CGRB, 0xECE4) /* type = RRS COMPARE AND BRANCH (64) */ \
+ V(clgrb, CLGRB, 0xECE5) /* type = RRS COMPARE LOGICAL AND BRANCH (64) */ \
+ V(crb, CRB, 0xECF6) /* type = RRS COMPARE AND BRANCH (32) */ \
+ V(clrb, CLRB, 0xECF7) /* type = RRS COMPARE LOGICAL AND BRANCH (32) */
+
+#define S390_OPCODE_LIST(V) \
+ S390_RSY_A_OPCODE_LIST(V) \
+ S390_RSY_B_OPCODE_LIST(V) \
+ S390_RXE_OPCODE_LIST(V) \
+ S390_RRF_A_OPCODE_LIST(V) \
+ S390_RXF_OPCODE_LIST(V) \
+ S390_IE_OPCODE_LIST(V) \
+ S390_RRF_B_OPCODE_LIST(V) \
+ S390_RRF_C_OPCODE_LIST(V) \
+ S390_MII_OPCODE_LIST(V) \
+ S390_RRF_D_OPCODE_LIST(V) \
+ S390_RRF_E_OPCODE_LIST(V) \
+ S390_VRR_A_OPCODE_LIST(V) \
+ S390_VRR_B_OPCODE_LIST(V) \
+ S390_VRR_C_OPCODE_LIST(V) \
+ S390_VRI_A_OPCODE_LIST(V) \
+ S390_VRR_D_OPCODE_LIST(V) \
+ S390_VRI_B_OPCODE_LIST(V) \
+ S390_VRR_E_OPCODE_LIST(V) \
+ S390_VRI_C_OPCODE_LIST(V) \
+ S390_VRI_D_OPCODE_LIST(V) \
+ S390_VRR_F_OPCODE_LIST(V) \
+ S390_RIS_OPCODE_LIST(V) \
+ S390_VRI_E_OPCODE_LIST(V) \
+ S390_RSL_A_OPCODE_LIST(V) \
+ S390_RSL_B_OPCODE_LIST(V) \
+ S390_SI_OPCODE_LIST(V) \
+ S390_SIL_OPCODE_LIST(V) \
+ S390_VRS_A_OPCODE_LIST(V) \
+ S390_RIL_A_OPCODE_LIST(V) \
+ S390_RIL_B_OPCODE_LIST(V) \
+ S390_VRS_B_OPCODE_LIST(V) \
+ S390_RIL_C_OPCODE_LIST(V) \
+ S390_VRS_C_OPCODE_LIST(V) \
+ S390_RI_A_OPCODE_LIST(V) \
+ S390_RSI_OPCODE_LIST(V) \
+ S390_RI_B_OPCODE_LIST(V) \
+ S390_RI_C_OPCODE_LIST(V) \
+ S390_SMI_OPCODE_LIST(V) \
+ S390_RXY_A_OPCODE_LIST(V) \
+ S390_RXY_B_OPCODE_LIST(V) \
+ S390_SIY_OPCODE_LIST(V) \
+ S390_SS_A_OPCODE_LIST(V) \
+ S390_E_OPCODE_LIST(V) \
+ S390_SS_B_OPCODE_LIST(V) \
+ S390_SS_C_OPCODE_LIST(V) \
+ S390_SS_D_OPCODE_LIST(V) \
+ S390_SS_E_OPCODE_LIST(V) \
+ S390_I_OPCODE_LIST(V) \
+ S390_SS_F_OPCODE_LIST(V) \
+ S390_SSE_OPCODE_LIST(V) \
+ S390_SSF_OPCODE_LIST(V) \
+ S390_RS_A_OPCODE_LIST(V) \
+ S390_RS_B_OPCODE_LIST(V) \
+ S390_S_OPCODE_LIST(V) \
+ S390_RX_A_OPCODE_LIST(V) \
+ S390_RX_B_OPCODE_LIST(V) \
+ S390_RIE_A_OPCODE_LIST(V) \
+ S390_RRD_OPCODE_LIST(V) \
+ S390_RIE_B_OPCODE_LIST(V) \
+ S390_RRE_OPCODE_LIST(V) \
+ S390_RIE_C_OPCODE_LIST(V) \
+ S390_RIE_D_OPCODE_LIST(V) \
+ S390_VRV_OPCODE_LIST(V) \
+ S390_RIE_E_OPCODE_LIST(V) \
+ S390_RR_OPCODE_LIST(V) \
+ S390_RIE_F_OPCODE_LIST(V) \
+ S390_VRX_OPCODE_LIST(V) \
+ S390_RIE_G_OPCODE_LIST(V) \
+ S390_RRS_OPCODE_LIST(V)
+
+// Opcodes as defined in Appendix B-2 table
+enum Opcode {
+#define DECLARE_OPCODES(name, opcode_name, opcode_value) \
+ opcode_name = opcode_value,
+ S390_OPCODE_LIST(DECLARE_OPCODES)
+#undef DECLARE_OPCODES
+
+ BKPT = 0x0001, // GDB Software Breakpoint
+ DUMY = 0xE352 // Special dummy opcode
+};
+
+// Instruction encoding bits and masks.
+enum {
+ // Instruction encoding bit
+ B1 = 1 << 1,
+ B4 = 1 << 4,
+ B5 = 1 << 5,
+ B7 = 1 << 7,
+ B8 = 1 << 8,
+ B9 = 1 << 9,
+ B12 = 1 << 12,
+ B18 = 1 << 18,
+ B19 = 1 << 19,
+ B20 = 1 << 20,
+ B22 = 1 << 22,
+ B23 = 1 << 23,
+ B24 = 1 << 24,
+ B25 = 1 << 25,
+ B26 = 1 << 26,
+ B27 = 1 << 27,
+ B28 = 1 << 28,
+
+ B6 = 1 << 6,
+ B10 = 1 << 10,
+ B11 = 1 << 11,
+ B16 = 1 << 16,
+ B17 = 1 << 17,
+ B21 = 1 << 21,
+
+ // Instruction bit masks
+ kCondMask = 0x1F << 21,
+ kOff12Mask = (1 << 12) - 1,
+ kImm24Mask = (1 << 24) - 1,
+ kOff16Mask = (1 << 16) - 1,
+ kImm16Mask = (1 << 16) - 1,
+ kImm26Mask = (1 << 26) - 1,
+ kBOfieldMask = 0x1f << 21,
+ kOpcodeMask = 0x3f << 26,
+ kExt2OpcodeMask = 0x1f << 1,
+ kExt5OpcodeMask = 0x3 << 2,
+ kBIMask = 0x1F << 16,
+ kBDMask = 0x14 << 2,
+ kAAMask = 0x01 << 1,
+ kLKMask = 0x01,
+ kRCMask = 0x01,
+ kTOMask = 0x1f << 21
+};
+
+// S390 instructions requires bigger shifts,
+// make them macros instead of enum because of the typing issue
+#define B32 ((uint64_t)1 << 32)
+#define B36 ((uint64_t)1 << 36)
+#define B40 ((uint64_t)1 << 40)
+const FourByteInstr kFourByteBrCondMask = 0xF << 20;
+const SixByteInstr kSixByteBrCondMask = static_cast<SixByteInstr>(0xF) << 36;
+
+// -----------------------------------------------------------------------------
+// Addressing modes and instruction variants.
+
+// Overflow Exception
+enum OEBit {
+ SetOE = 1 << 10, // Set overflow exception
+ LeaveOE = 0 << 10 // No overflow exception
+};
+
+// Record bit
+enum RCBit { // Bit 0
+ SetRC = 1, // LT,GT,EQ,SO
+ LeaveRC = 0 // None
+};
+
+// Link bit
+enum LKBit { // Bit 0
+ SetLK = 1, // Load effective address of next instruction
+ LeaveLK = 0 // No action
+};
+
+enum BOfield { // Bits 25-21
+ DCBNZF = 0 << 21, // Decrement CTR; branch if CTR != 0 and condition false
+ DCBEZF = 2 << 21, // Decrement CTR; branch if CTR == 0 and condition false
+ BF = 4 << 21, // Branch if condition false
+ DCBNZT = 8 << 21, // Decrement CTR; branch if CTR != 0 and condition true
+ DCBEZT = 10 << 21, // Decrement CTR; branch if CTR == 0 and condition true
+ BT = 12 << 21, // Branch if condition true
+ DCBNZ = 16 << 21, // Decrement CTR; branch if CTR != 0
+ DCBEZ = 18 << 21, // Decrement CTR; branch if CTR == 0
+ BA = 20 << 21 // Branch always
+};
+
+#ifdef _AIX
+#undef CR_LT
+#undef CR_GT
+#undef CR_EQ
+#undef CR_SO
+#endif
+
+enum CRBit { CR_LT = 0, CR_GT = 1, CR_EQ = 2, CR_SO = 3, CR_FU = 3 };
+
+#define CRWIDTH 4
+
+// -----------------------------------------------------------------------------
+// Supervisor Call (svc) specific support.
+
+// Special Software Interrupt codes when used in the presence of the S390
+// simulator.
+// SVC provides a 24bit immediate value. Use bits 22:0 for standard
+// SoftwareInterrupCode. Bit 23 is reserved for the stop feature.
+enum SoftwareInterruptCodes {
+ // Transition to C code
+ kCallRtRedirected = 0x0010,
+ // Breakpoint
+ kBreakpoint = 0x0000,
+ // Stop
+ kStopCode = 1 << 23
+};
+const uint32_t kStopCodeMask = kStopCode - 1;
+const uint32_t kMaxStopCode = kStopCode - 1;
+const int32_t kDefaultStopCode = -1;
+
+// FP rounding modes.
+enum FPRoundingMode {
+ RN = 0, // Round to Nearest.
+ RZ = 1, // Round towards zero.
+ RP = 2, // Round towards Plus Infinity.
+ RM = 3, // Round towards Minus Infinity.
+
+ // Aliases.
+ kRoundToNearest = RN,
+ kRoundToZero = RZ,
+ kRoundToPlusInf = RP,
+ kRoundToMinusInf = RM
+};
+
+const uint32_t kFPRoundingModeMask = 3;
+
+enum CheckForInexactConversion {
+ kCheckForInexactConversion,
+ kDontCheckForInexactConversion
+};
+
+// -----------------------------------------------------------------------------
+// Specific instructions, constants, and masks.
+
+// use TRAP4 to indicate redirection call for simulation mode
+const Instr rtCallRedirInstr = TRAP4;
+
+// -----------------------------------------------------------------------------
+// Instruction abstraction.
+
+// The class Instruction enables access to individual fields defined in the
+// z/Architecture instruction set encoding.
+class Instruction {
+ public:
+ // S390 Opcode Format Types
+ // Based on the first byte of the opcode, we can determine how to extract
+ // the entire opcode of the instruction. The various favours include:
+ enum OpcodeFormatType {
+ ONE_BYTE_OPCODE, // One Byte - Bits 0 to 7
+ TWO_BYTE_OPCODE, // Two Bytes - Bits 0 to 15
+ TWO_BYTE_DISJOINT_OPCODE, // Two Bytes - Bits 0 to 7, 40 to 47
+ THREE_NIBBLE_OPCODE // Three Nibbles - Bits 0 to 7, 12 to 15
+ };
+
+ static OpcodeFormatType OpcodeFormatTable[256];
+
+ // Get the raw instruction bits.
+ template <typename T>
+ inline T InstructionBits() const {
+ return Instruction::InstructionBits<T>(reinterpret_cast<const byte*>(this));
+ }
+ inline Instr InstructionBits() const {
+ return *reinterpret_cast<const Instr*>(this);
+ }
+
+ // Set the raw instruction bits to value.
+ template <typename T>
+ inline void SetInstructionBits(T value) const {
+ Instruction::SetInstructionBits<T>(reinterpret_cast<const byte*>(this),
+ value);
+ }
+ inline void SetInstructionBits(Instr value) {
+ *reinterpret_cast<Instr*>(this) = value;
+ }
+
+ // Read one particular bit out of the instruction bits.
+ inline int Bit(int nr) const { return (InstructionBits() >> nr) & 1; }
+
+ // Read a bit field's value out of the instruction bits.
+ inline int Bits(int hi, int lo) const {
+ return (InstructionBits() >> lo) & ((2 << (hi - lo)) - 1);
+ }
+
+ // Read bits according to instruction type
+ template <typename T, typename U>
+ inline U Bits(int hi, int lo) const {
+ return (InstructionBits<T>() >> lo) & ((2 << (hi - lo)) - 1);
+ }
+
+ // Read a bit field out of the instruction bits.
+ inline int BitField(int hi, int lo) const {
+ return InstructionBits() & (((2 << (hi - lo)) - 1) << lo);
+ }
+
+ // Determine the instruction length
+ inline int InstructionLength() {
+ return Instruction::InstructionLength(reinterpret_cast<const byte*>(this));
+ }
+ // Extract the Instruction Opcode
+ inline Opcode S390OpcodeValue() {
+ return Instruction::S390OpcodeValue(reinterpret_cast<const byte*>(this));
+ }
+
+ // Static support.
+
+ // Read one particular bit out of the instruction bits.
+ static inline int Bit(Instr instr, int nr) { return (instr >> nr) & 1; }
+
+ // Read the value of a bit field out of the instruction bits.
+ static inline int Bits(Instr instr, int hi, int lo) {
+ return (instr >> lo) & ((2 << (hi - lo)) - 1);
+ }
+
+ // Read a bit field out of the instruction bits.
+ static inline int BitField(Instr instr, int hi, int lo) {
+ return instr & (((2 << (hi - lo)) - 1) << lo);
+ }
+
+ // Determine the instruction length of the given instruction
+ static inline int InstructionLength(const byte* instr) {
+ // Length can be determined by the first nibble.
+ // 0x0 to 0x3 => 2-bytes
+ // 0x4 to 0xB => 4-bytes
+ // 0xC to 0xF => 6-bytes
+ byte topNibble = (*instr >> 4) & 0xF;
+ if (topNibble <= 3)
+ return 2;
+ else if (topNibble <= 0xB)
+ return 4;
+ return 6;
+ }
+
+ // Returns the instruction bits of the given instruction
+ static inline uint64_t InstructionBits(const byte* instr) {
+ int length = InstructionLength(instr);
+ if (2 == length)
+ return static_cast<uint64_t>(InstructionBits<TwoByteInstr>(instr));
+ else if (4 == length)
+ return static_cast<uint64_t>(InstructionBits<FourByteInstr>(instr));
+ else
+ return InstructionBits<SixByteInstr>(instr);
+ }
+
+ // Extract the raw instruction bits
+ template <typename T>
+ static inline T InstructionBits(const byte* instr) {
+#if !V8_TARGET_LITTLE_ENDIAN
+ if (sizeof(T) <= 4) {
+ return *reinterpret_cast<const T*>(instr);
+ } else {
+ // We cannot read 8-byte instructon address directly, because for a
+ // six-byte instruction, the extra 2-byte address might not be
+ // allocated.
+ uint64_t fourBytes = *reinterpret_cast<const uint32_t*>(instr);
+ uint16_t twoBytes = *reinterpret_cast<const uint16_t*>(instr + 4);
+ return (fourBytes << 16 | twoBytes);
+ }
+#else
+ // Even on little endian hosts (simulation), the instructions
+ // are stored as big-endian in order to decode the opcode and
+ // instruction length.
+ T instr_bits = 0;
+
+ // 6-byte instrs are represented by uint64_t
+ uint32_t size = (sizeof(T) == 8) ? 6 : sizeof(T);
+
+ for (T i = 0; i < size; i++) {
+ instr_bits <<= 8;
+ instr_bits |= *(instr + i);
+ }
+ return instr_bits;
+#endif
+ }
+
+ // Set the Instruction Bits to value
+ template <typename T>
+ static inline void SetInstructionBits(byte* instr, T value) {
+#if V8_TARGET_LITTLE_ENDIAN
+ // The instruction bits are stored in big endian format even on little
+ // endian hosts, in order to decode instruction length and opcode.
+ // The following code will reverse the bytes so that the stores later
+ // (which are in native endianess) will effectively save the instruction
+ // in big endian.
+ if (sizeof(T) == 2) {
+ // Two Byte Instruction
+ value = ((value & 0x00FF) << 8) | ((value & 0xFF00) >> 8);
+ } else if (sizeof(T) == 4) {
+ // Four Byte Instruction
+ value = ((value & 0x000000FF) << 24) | ((value & 0x0000FF00) << 8) |
+ ((value & 0x00FF0000) >> 8) | ((value & 0xFF000000) >> 24);
+ } else if (sizeof(T) == 8) {
+ // Six Byte Instruction
+ uint64_t orig_value = static_cast<uint64_t>(value);
+ value = (static_cast<uint64_t>(orig_value & 0xFF) << 40) |
+ (static_cast<uint64_t>((orig_value >> 8) & 0xFF) << 32) |
+ (static_cast<uint64_t>((orig_value >> 16) & 0xFF) << 24) |
+ (static_cast<uint64_t>((orig_value >> 24) & 0xFF) << 16) |
+ (static_cast<uint64_t>((orig_value >> 32) & 0xFF) << 8) |
+ (static_cast<uint64_t>((orig_value >> 40) & 0xFF));
+ }
+#endif
+ if (sizeof(T) <= 4) {
+ *reinterpret_cast<T*>(instr) = value;
+ } else {
+#if V8_TARGET_LITTLE_ENDIAN
+ uint64_t orig_value = static_cast<uint64_t>(value);
+ *reinterpret_cast<uint32_t*>(instr) = static_cast<uint32_t>(value);
+ *reinterpret_cast<uint16_t*>(instr + 4) =
+ static_cast<uint16_t>((orig_value >> 32) & 0xFFFF);
+#else
+ *reinterpret_cast<uint32_t*>(instr) = static_cast<uint32_t>(value >> 16);
+ *reinterpret_cast<uint16_t*>(instr + 4) =
+ static_cast<uint16_t>(value & 0xFFFF);
+#endif
+ }
+ }
+
+ // Get Instruction Format Type
+ static OpcodeFormatType getOpcodeFormatType(const byte* instr) {
+ const byte firstByte = *instr;
+ return OpcodeFormatTable[firstByte];
+ }
+
+ // Extract the full opcode from the instruction.
+ static inline Opcode S390OpcodeValue(const byte* instr) {
+ OpcodeFormatType opcodeType = getOpcodeFormatType(instr);
+
+ // The native instructions are encoded in big-endian format
+ // even if running on little-endian host. Hence, we need
+ // to ensure we use byte* based bit-wise logic.
+ switch (opcodeType) {
+ case ONE_BYTE_OPCODE:
+ // One Byte - Bits 0 to 7
+ return static_cast<Opcode>(*instr);
+ case TWO_BYTE_OPCODE:
+ // Two Bytes - Bits 0 to 15
+ return static_cast<Opcode>((*instr << 8) | (*(instr + 1)));
+ case TWO_BYTE_DISJOINT_OPCODE:
+ // Two Bytes - Bits 0 to 7, 40 to 47
+ return static_cast<Opcode>((*instr << 8) | (*(instr + 5) & 0xFF));
+ default:
+ // case THREE_NIBBLE_OPCODE:
+ // Three Nibbles - Bits 0 to 7, 12 to 15
+ return static_cast<Opcode>((*instr << 4) | (*(instr + 1) & 0xF));
+ }
+
+ UNREACHABLE();
+ }
+
+ // Fields used in Software interrupt instructions
+ inline SoftwareInterruptCodes SvcValue() const {
+ return static_cast<SoftwareInterruptCodes>(Bits<FourByteInstr, int>(15, 0));
+ }
+
+ // Instructions are read of out a code stream. The only way to get a
+ // reference to an instruction is to convert a pointer. There is no way
+ // to allocate or create instances of class Instruction.
+ // Use the At(pc) function to create references to Instruction.
+ static Instruction* At(byte* pc) {
+ return reinterpret_cast<Instruction*>(pc);
+ }
+
+ private:
+ // We need to prevent the creation of instances of class Instruction.
+ DISALLOW_IMPLICIT_CONSTRUCTORS(Instruction);
+};
+
+#define DECLARE_FIELD_FOR_TWO_BYTE_INSTR(name, T, lo, hi) \
+ inline int name() const { \
+ return Bits<TwoByteInstr, T>(15 - (lo), 15 - (hi) + 1); \
+ }
+
+#define DECLARE_FIELD_FOR_FOUR_BYTE_INSTR(name, T, lo, hi) \
+ inline int name() const { \
+ return Bits<FourByteInstr, T>(31 - (lo), 31 - (hi) + 1); \
+ }
+
+#define DECLARE_FIELD_FOR_SIX_BYTE_INSTR(name, T, lo, hi) \
+ inline int name() const { \
+ return Bits<SixByteInstr, T>(47 - (lo), 47 - (hi) + 1); \
+ }
+
+class TwoByteInstruction : public Instruction {
+ public:
+ inline int size() const { return 2; }
+};
+
+class FourByteInstruction : public Instruction {
+ public:
+ inline int size() const { return 4; }
+};
+
+class SixByteInstruction : public Instruction {
+ public:
+ inline int size() const { return 6; }
+};
+
+// I Instruction
+class IInstruction : public TwoByteInstruction {
+ public:
+ DECLARE_FIELD_FOR_TWO_BYTE_INSTR(IValue, int, 8, 16)
+};
+
+// E Instruction
+class EInstruction : public TwoByteInstruction {};
+
+// IE Instruction
+class IEInstruction : public FourByteInstruction {
+ public:
+ DECLARE_FIELD_FOR_FOUR_BYTE_INSTR(I1Value, int, 24, 28)
+ DECLARE_FIELD_FOR_FOUR_BYTE_INSTR(I2Value, int, 28, 32)
+};
+
+// MII Instruction
+class MIIInstruction : public SixByteInstruction {
+ public:
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(M1Value, uint32_t, 8, 12)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(RI2Value, int, 12, 24)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(RI3Value, int, 24, 47)
+};
+
+// RI Instruction
+class RIInstruction : public FourByteInstruction {
+ public:
+ DECLARE_FIELD_FOR_FOUR_BYTE_INSTR(R1Value, int, 8, 12)
+ DECLARE_FIELD_FOR_FOUR_BYTE_INSTR(I2Value, int, 16, 32)
+ DECLARE_FIELD_FOR_FOUR_BYTE_INSTR(I2UnsignedValue, uint32_t, 16, 32)
+ DECLARE_FIELD_FOR_FOUR_BYTE_INSTR(M1Value, uint32_t, 8, 12)
+};
+
+// RR Instruction
+class RRInstruction : Instruction {
+ public:
+ inline int R1Value() const {
+ // the high and low parameters of Bits is the number of bits from
+ // rightmost place
+ return Bits<TwoByteInstr, int>(7, 4);
+ }
+ inline int R2Value() const { return Bits<TwoByteInstr, int>(3, 0); }
+ inline Condition M1Value() const {
+ return static_cast<Condition>(Bits<TwoByteInstr, int>(7, 4));
+ }
+
+ inline int size() const { return 2; }
+};
+
+// RRE Instruction
+class RREInstruction : Instruction {
+ public:
+ inline int R1Value() const { return Bits<FourByteInstr, int>(7, 4); }
+ inline int R2Value() const { return Bits<FourByteInstr, int>(3, 0); }
+ inline int M3Value() const { return Bits<FourByteInstr, int>(15, 12); }
+ inline int M4Value() const { return Bits<FourByteInstr, int>(19, 16); }
+ inline int size() const { return 4; }
+};
+
+// RRF Instruction
+class RRFInstruction : Instruction {
+ public:
+ inline int R1Value() const { return Bits<FourByteInstr, int>(7, 4); }
+ inline int R2Value() const { return Bits<FourByteInstr, int>(3, 0); }
+ inline int R3Value() const { return Bits<FourByteInstr, int>(15, 12); }
+ inline int M3Value() const { return Bits<FourByteInstr, int>(15, 12); }
+ inline int M4Value() const { return Bits<FourByteInstr, int>(11, 8); }
+ inline int size() const { return 4; }
+};
+
+// RRD Isntruction
+class RRDInstruction : Instruction {
+ public:
+ inline int R1Value() const { return Bits<FourByteInstr, int>(15, 12); }
+ inline int R2Value() const { return Bits<FourByteInstr, int>(3, 0); }
+ inline int R3Value() const { return Bits<FourByteInstr, int>(7, 4); }
+ inline int size() const { return 4; }
+};
+
+// RS Instruction
+class RSInstruction : Instruction {
+ public:
+ inline int R1Value() const { return Bits<FourByteInstr, int>(23, 20); }
+ inline int R3Value() const { return Bits<FourByteInstr, int>(19, 16); }
+ inline int B2Value() const { return Bits<FourByteInstr, int>(15, 12); }
+ inline unsigned int D2Value() const {
+ return Bits<FourByteInstr, unsigned int>(11, 0);
+ }
+ inline int size() const { return 4; }
+};
+
+// RSI Instruction
+class RSIInstruction : Instruction {
+ public:
+ inline int R1Value() const { return Bits<FourByteInstr, int>(23, 20); }
+ inline int R3Value() const { return Bits<FourByteInstr, int>(19, 16); }
+ inline int I2Value() const {
+ return static_cast<int32_t>(Bits<FourByteInstr, int16_t>(15, 0));
+ }
+ inline int size() const { return 4; }
+};
+
+// RSY Instruction
+class RSYInstruction : Instruction {
+ public:
+ inline int R1Value() const { return Bits<SixByteInstr, int>(39, 36); }
+ inline int R3Value() const { return Bits<SixByteInstr, int>(35, 32); }
+ inline int B2Value() const { return Bits<SixByteInstr, int>(31, 28); }
+ inline int32_t D2Value() const {
+ int32_t value = Bits<SixByteInstr, int32_t>(27, 16);
+ value += Bits<SixByteInstr, int8_t>(15, 8) << 12;
+ return value;
+ }
+ inline int size() const { return 6; }
+};
+
+// RX Instruction
+class RXInstruction : Instruction {
+ public:
+ inline int R1Value() const { return Bits<FourByteInstr, int>(23, 20); }
+ inline int X2Value() const { return Bits<FourByteInstr, int>(19, 16); }
+ inline int B2Value() const { return Bits<FourByteInstr, int>(15, 12); }
+ inline uint32_t D2Value() const {
+ return Bits<FourByteInstr, uint32_t>(11, 0);
+ }
+ inline int size() const { return 4; }
+};
+
+// RXY Instruction
+class RXYInstruction : Instruction {
+ public:
+ inline int R1Value() const { return Bits<SixByteInstr, int>(39, 36); }
+ inline int X2Value() const { return Bits<SixByteInstr, int>(35, 32); }
+ inline int B2Value() const { return Bits<SixByteInstr, int>(31, 28); }
+ inline int32_t D2Value() const {
+ int32_t value = Bits<SixByteInstr, uint32_t>(27, 16);
+ value += Bits<SixByteInstr, int8_t>(15, 8) << 12;
+ return value;
+ }
+ inline int size() const { return 6; }
+};
+
+// RIL Instruction
+class RILInstruction : Instruction {
+ public:
+ inline int R1Value() const { return Bits<SixByteInstr, int>(39, 36); }
+ inline int32_t I2Value() const { return Bits<SixByteInstr, int32_t>(31, 0); }
+ inline uint32_t I2UnsignedValue() const {
+ return Bits<SixByteInstr, uint32_t>(31, 0);
+ }
+ inline int size() const { return 6; }
+};
+
+// SI Instruction
+class SIInstruction : Instruction {
+ public:
+ inline int B1Value() const { return Bits<FourByteInstr, int>(15, 12); }
+ inline uint32_t D1Value() const {
+ return Bits<FourByteInstr, uint32_t>(11, 0);
+ }
+ inline uint8_t I2Value() const {
+ return Bits<FourByteInstr, uint8_t>(23, 16);
+ }
+ inline int size() const { return 4; }
+};
+
+// SIY Instruction
+class SIYInstruction : Instruction {
+ public:
+ inline int B1Value() const { return Bits<SixByteInstr, int>(31, 28); }
+ inline int32_t D1Value() const {
+ int32_t value = Bits<SixByteInstr, uint32_t>(27, 16);
+ value += Bits<SixByteInstr, int8_t>(15, 8) << 12;
+ return value;
+ }
+ inline uint8_t I2Value() const { return Bits<SixByteInstr, uint8_t>(39, 32); }
+ inline int size() const { return 6; }
+};
+
+// SIL Instruction
+class SILInstruction : Instruction {
+ public:
+ inline int B1Value() const { return Bits<SixByteInstr, int>(31, 28); }
+ inline int D1Value() const { return Bits<SixByteInstr, int>(27, 16); }
+ inline int I2Value() const { return Bits<SixByteInstr, int>(15, 0); }
+ inline int size() const { return 6; }
+};
+
+// SS Instruction
+class SSInstruction : Instruction {
+ public:
+ inline int B1Value() const { return Bits<SixByteInstr, int>(31, 28); }
+ inline int B2Value() const { return Bits<SixByteInstr, int>(15, 12); }
+ inline int D1Value() const { return Bits<SixByteInstr, int>(27, 16); }
+ inline int D2Value() const { return Bits<SixByteInstr, int>(11, 0); }
+ inline int Length() const { return Bits<SixByteInstr, int>(39, 32); }
+ inline int size() const { return 6; }
+};
+
+// RXE Instruction
+class RXEInstruction : Instruction {
+ public:
+ inline int R1Value() const { return Bits<SixByteInstr, int>(39, 36); }
+ inline int X2Value() const { return Bits<SixByteInstr, int>(35, 32); }
+ inline int B2Value() const { return Bits<SixByteInstr, int>(31, 28); }
+ inline int D2Value() const { return Bits<SixByteInstr, int>(27, 16); }
+ inline int size() const { return 6; }
+};
+
+// RIE Instruction
+class RIEInstruction : Instruction {
+ public:
+ inline int R1Value() const { return Bits<SixByteInstr, int>(39, 36); }
+ inline int R2Value() const { return Bits<SixByteInstr, int>(35, 32); }
+ inline int I3Value() const { return Bits<SixByteInstr, uint32_t>(31, 24); }
+ inline int I4Value() const { return Bits<SixByteInstr, uint32_t>(23, 16); }
+ inline int I5Value() const { return Bits<SixByteInstr, uint32_t>(15, 8); }
+ inline int I6Value() const {
+ return static_cast<int32_t>(Bits<SixByteInstr, int16_t>(31, 16));
+ }
+ inline int size() const { return 6; }
+};
+
+// VRR Instruction
+class VRR_A_Instruction : SixByteInstruction {
+ public:
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(R1Value, int, 8, 12)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(R2Value, int, 12, 16)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(M5Value, uint32_t, 24, 28)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(M4Value, uint32_t, 28, 32)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(M3Value, uint32_t, 32, 36)
+};
+
+class VRR_B_Instruction : SixByteInstruction {
+ public:
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(R1Value, int, 8, 12)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(R2Value, int, 12, 16)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(R3Value, int, 16, 20)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(M5Value, uint32_t, 24, 28)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(M4Value, uint32_t, 32, 36)
+};
+
+class VRR_C_Instruction : SixByteInstruction {
+ public:
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(R1Value, int, 8, 12)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(R2Value, int, 12, 16)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(R3Value, int, 16, 20)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(M6Value, uint32_t, 24, 28)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(M5Value, uint32_t, 28, 32)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(M4Value, uint32_t, 32, 36)
+};
+
+class VRR_E_Instruction : SixByteInstruction {
+ public:
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(R1Value, int, 8, 12)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(R2Value, int, 12, 16)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(R3Value, int, 16, 20)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(R4Value, int, 32, 36)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(M6Value, uint32_t, 20, 24)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(M5Value, uint32_t, 28, 32)
+};
+
+class VRR_F_Instruction : SixByteInstruction {
+ public:
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(R1Value, int, 8, 12)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(R2Value, int, 12, 16)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(R3Value, int, 16, 20)
+};
+
+class VRX_Instruction : SixByteInstruction {
+ public:
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(R1Value, int, 8, 12)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(X2Value, int, 12, 16)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(B2Value, int, 16, 20)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(D2Value, int, 20, 32)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(M3Value, uint32_t, 32, 36)
+};
+
+class VRS_Instruction : SixByteInstruction {
+ public:
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(R1Value, int, 8, 12)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(R3Value, int, 12, 16)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(B2Value, int, 16, 20)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(D2Value, int, 20, 32)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(M4Value, uint32_t, 32, 36)
+};
+
+class VRI_A_Instruction : SixByteInstruction {
+ public:
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(R1Value, int, 8, 12)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(I2Value, int, 16, 32)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(M3Value, uint32_t, 32, 36)
+};
+
+class VRI_C_Instruction : SixByteInstruction {
+ public:
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(R1Value, int, 8, 12)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(R3Value, int, 12, 16)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(I2Value, int, 16, 32)
+ DECLARE_FIELD_FOR_SIX_BYTE_INSTR(M4Value, uint32_t, 32, 36)
+};
+
+// Helper functions for converting between register numbers and names.
+class Registers {
+ public:
+ // Lookup the register number for the name provided.
+ static int Number(const char* name);
+
+ private:
+ static const char* names_[kNumRegisters];
+};
+
+// Helper functions for converting between FP register numbers and names.
+class DoubleRegisters {
+ public:
+ // Lookup the register number for the name provided.
+ static int Number(const char* name);
+
+ private:
+ static const char* names_[kNumDoubleRegisters];
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_S390_CONSTANTS_S390_H_
diff --git a/src/codegen/s390/cpu-s390.cc b/src/codegen/s390/cpu-s390.cc
new file mode 100644
index 0000000..748f402
--- /dev/null
+++ b/src/codegen/s390/cpu-s390.cc
@@ -0,0 +1,24 @@
+// Copyright 2015 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// CPU specific code for s390 independent of OS goes here.
+#if V8_TARGET_ARCH_S390
+
+#include "src/codegen/cpu-features.h"
+
+namespace v8 {
+namespace internal {
+
+void CpuFeatures::FlushICache(void* buffer, size_t size) {
+ // Given the strong memory model on z/Architecture, and the single
+ // thread nature of V8 and JavaScript, instruction cache flushing
+ // is not necessary. The architecture guarantees that if a core
+ // patches its own instruction cache, the updated instructions will be
+ // reflected automatically.
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_S390
diff --git a/src/codegen/s390/interface-descriptors-s390.cc b/src/codegen/s390/interface-descriptors-s390.cc
new file mode 100644
index 0000000..a848cdf
--- /dev/null
+++ b/src/codegen/s390/interface-descriptors-s390.cc
@@ -0,0 +1,284 @@
+// Copyright 2015 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#if V8_TARGET_ARCH_S390
+
+#include "src/codegen/interface-descriptors.h"
+
+#include "src/execution/frames.h"
+
+namespace v8 {
+namespace internal {
+
+const Register CallInterfaceDescriptor::ContextRegister() { return cp; }
+
+void CallInterfaceDescriptor::DefaultInitializePlatformSpecific(
+ CallInterfaceDescriptorData* data, int register_parameter_count) {
+ const Register default_stub_registers[] = {r2, r3, r4, r5, r6};
+ CHECK_LE(static_cast<size_t>(register_parameter_count),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(register_parameter_count,
+ default_stub_registers);
+}
+
+void RecordWriteDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ const Register default_stub_registers[] = {r2, r3, r4, r5, r6};
+
+ data->RestrictAllocatableRegisters(default_stub_registers,
+ arraysize(default_stub_registers));
+
+ CHECK_LE(static_cast<size_t>(kParameterCount),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(kParameterCount, default_stub_registers);
+}
+
+void EphemeronKeyBarrierDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ const Register default_stub_registers[] = {r2, r3, r4, r5, r6};
+
+ data->RestrictAllocatableRegisters(default_stub_registers,
+ arraysize(default_stub_registers));
+
+ CHECK_LE(static_cast<size_t>(kParameterCount),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(kParameterCount, default_stub_registers);
+}
+
+const Register LoadDescriptor::ReceiverRegister() { return r3; }
+const Register LoadDescriptor::NameRegister() { return r4; }
+const Register LoadDescriptor::SlotRegister() { return r2; }
+
+const Register LoadWithVectorDescriptor::VectorRegister() { return r5; }
+
+const Register
+LoadWithReceiverAndVectorDescriptor::LookupStartObjectRegister() {
+ return r6;
+}
+
+const Register StoreDescriptor::ReceiverRegister() { return r3; }
+const Register StoreDescriptor::NameRegister() { return r4; }
+const Register StoreDescriptor::ValueRegister() { return r2; }
+const Register StoreDescriptor::SlotRegister() { return r6; }
+
+const Register StoreWithVectorDescriptor::VectorRegister() { return r5; }
+
+const Register StoreTransitionDescriptor::SlotRegister() { return r6; }
+const Register StoreTransitionDescriptor::VectorRegister() { return r5; }
+const Register StoreTransitionDescriptor::MapRegister() { return r7; }
+
+const Register ApiGetterDescriptor::HolderRegister() { return r2; }
+const Register ApiGetterDescriptor::CallbackRegister() { return r5; }
+
+const Register GrowArrayElementsDescriptor::ObjectRegister() { return r2; }
+const Register GrowArrayElementsDescriptor::KeyRegister() { return r5; }
+
+// static
+const Register TypeConversionDescriptor::ArgumentRegister() { return r2; }
+
+void TypeofDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {r5};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallTrampolineDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r2 : number of arguments
+ // r3 : the target to call
+ Register registers[] = {r3, r2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r2 : number of arguments (on the stack, not including receiver)
+ // r3 : the target to call
+ // r6 : arguments list length (untagged)
+ // r4 : arguments list (FixedArray)
+ Register registers[] = {r3, r2, r6, r4};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallForwardVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r2 : number of arguments
+ // r4 : start index (to support rest parameters)
+ // r3 : the target to call
+ Register registers[] = {r3, r2, r4};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallFunctionTemplateDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r3 : function template info
+ // r4 : number of arguments (on the stack, not including receiver)
+ Register registers[] = {r3, r4};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallWithSpreadDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r2 : number of arguments (on the stack, not including receiver)
+ // r3 : the target to call
+ // r4 : the object to spread
+ Register registers[] = {r3, r2, r4};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallWithArrayLikeDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r3 : the target to call
+ // r4 : the arguments list
+ Register registers[] = {r3, r4};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r2 : number of arguments (on the stack, not including receiver)
+ // r3 : the target to call
+ // r5 : the new target
+ // r6 : arguments list length (untagged)
+ // r4 : arguments list (FixedArray)
+ Register registers[] = {r3, r5, r2, r6, r4};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructForwardVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r2 : number of arguments
+ // r5 : the new target
+ // r4 : start index (to support rest parameters)
+ // r3 : the target to call
+ Register registers[] = {r3, r5, r2, r4};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructWithSpreadDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r2 : number of arguments (on the stack, not including receiver)
+ // r3 : the target to call
+ // r5 : the new target
+ // r4 : the object to spread
+ Register registers[] = {r3, r5, r2, r4};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructWithArrayLikeDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r3 : the target to call
+ // r5 : the new target
+ // r4 : the arguments list
+ Register registers[] = {r3, r5, r4};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructStubDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // r2 : number of arguments
+ // r3 : the target to call
+ // r5 : the new target
+ // r4 : allocation site or undefined
+ Register registers[] = {r3, r5, r2, r4};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void AbortDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {r3};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CompareDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {r3, r2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void BinaryOpDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {r3, r2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ArgumentsAdaptorDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ r3, // JSFunction
+ r5, // the new target
+ r2, // actual number of arguments
+ r4, // expected number of arguments
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ApiCallbackDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ r3, // kApiFunctionAddress
+ r4, // kArgc
+ r5, // kCallData
+ r2, // kHolder
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void InterpreterDispatchDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ kInterpreterAccumulatorRegister, kInterpreterBytecodeOffsetRegister,
+ kInterpreterBytecodeArrayRegister, kInterpreterDispatchTableRegister};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void InterpreterPushArgsThenCallDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ r2, // argument count (not including receiver)
+ r4, // address of first argument
+ r3 // the target callable to be call
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void InterpreterPushArgsThenConstructDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ r2, // argument count (not including receiver)
+ r6, // address of the first argument
+ r3, // constructor to call
+ r5, // new target
+ r4, // allocation site feedback if available, undefined otherwise
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ResumeGeneratorDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ r2, // the value to pass to the generator
+ r3 // the JSGeneratorObject to resume
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void FrameDropperTrampolineDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ r3, // loaded new FP
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void RunMicrotasksEntryDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {r2, r3};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_S390
diff --git a/src/codegen/s390/macro-assembler-s390.cc b/src/codegen/s390/macro-assembler-s390.cc
new file mode 100644
index 0000000..4f63543
--- /dev/null
+++ b/src/codegen/s390/macro-assembler-s390.cc
@@ -0,0 +1,4553 @@
+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <assert.h> // For assert
+#include <limits.h> // For LONG_MIN, LONG_MAX.
+
+#if V8_TARGET_ARCH_S390
+
+#include "src/base/bits.h"
+#include "src/base/division-by-constant.h"
+#include "src/codegen/callable.h"
+#include "src/codegen/code-factory.h"
+#include "src/codegen/external-reference-table.h"
+#include "src/codegen/macro-assembler.h"
+#include "src/codegen/register-configuration.h"
+#include "src/debug/debug.h"
+#include "src/execution/frames-inl.h"
+#include "src/heap/memory-chunk.h"
+#include "src/init/bootstrapper.h"
+#include "src/logging/counters.h"
+#include "src/objects/smi.h"
+#include "src/runtime/runtime.h"
+#include "src/snapshot/embedded/embedded-data.h"
+#include "src/snapshot/snapshot.h"
+#include "src/wasm/wasm-code-manager.h"
+
+// Satisfy cpplint check, but don't include platform-specific header. It is
+// included recursively via macro-assembler.h.
+#if 0
+#include "src/codegen/s390/macro-assembler-s390.h"
+#endif
+
+namespace v8 {
+namespace internal {
+
+int TurboAssembler::RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
+ Register exclusion1,
+ Register exclusion2,
+ Register exclusion3) const {
+ int bytes = 0;
+ RegList exclusions = 0;
+ if (exclusion1 != no_reg) {
+ exclusions |= exclusion1.bit();
+ if (exclusion2 != no_reg) {
+ exclusions |= exclusion2.bit();
+ if (exclusion3 != no_reg) {
+ exclusions |= exclusion3.bit();
+ }
+ }
+ }
+
+ RegList list = kJSCallerSaved & ~exclusions;
+ bytes += NumRegs(list) * kSystemPointerSize;
+
+ if (fp_mode == kSaveFPRegs) {
+ bytes += NumRegs(kCallerSavedDoubles) * kDoubleSize;
+ }
+
+ return bytes;
+}
+
+int TurboAssembler::PushCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1,
+ Register exclusion2, Register exclusion3) {
+ int bytes = 0;
+ RegList exclusions = 0;
+ if (exclusion1 != no_reg) {
+ exclusions |= exclusion1.bit();
+ if (exclusion2 != no_reg) {
+ exclusions |= exclusion2.bit();
+ if (exclusion3 != no_reg) {
+ exclusions |= exclusion3.bit();
+ }
+ }
+ }
+
+ RegList list = kJSCallerSaved & ~exclusions;
+ MultiPush(list);
+ bytes += NumRegs(list) * kSystemPointerSize;
+
+ if (fp_mode == kSaveFPRegs) {
+ MultiPushDoubles(kCallerSavedDoubles);
+ bytes += NumRegs(kCallerSavedDoubles) * kDoubleSize;
+ }
+
+ return bytes;
+}
+
+int TurboAssembler::PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1,
+ Register exclusion2, Register exclusion3) {
+ int bytes = 0;
+ if (fp_mode == kSaveFPRegs) {
+ MultiPopDoubles(kCallerSavedDoubles);
+ bytes += NumRegs(kCallerSavedDoubles) * kDoubleSize;
+ }
+
+ RegList exclusions = 0;
+ if (exclusion1 != no_reg) {
+ exclusions |= exclusion1.bit();
+ if (exclusion2 != no_reg) {
+ exclusions |= exclusion2.bit();
+ if (exclusion3 != no_reg) {
+ exclusions |= exclusion3.bit();
+ }
+ }
+ }
+
+ RegList list = kJSCallerSaved & ~exclusions;
+ MultiPop(list);
+ bytes += NumRegs(list) * kSystemPointerSize;
+
+ return bytes;
+}
+
+void TurboAssembler::LoadFromConstantsTable(Register destination,
+ int constant_index) {
+ DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kBuiltinsConstantsTable));
+
+ const uint32_t offset = FixedArray::kHeaderSize +
+ constant_index * kSystemPointerSize - kHeapObjectTag;
+
+ CHECK(is_uint19(offset));
+ DCHECK_NE(destination, r0);
+ LoadRoot(destination, RootIndex::kBuiltinsConstantsTable);
+ LoadTaggedPointerField(
+ destination,
+ FieldMemOperand(destination,
+ FixedArray::OffsetOfElementAt(constant_index)),
+ r1);
+}
+
+void TurboAssembler::LoadRootRelative(Register destination, int32_t offset) {
+ LoadP(destination, MemOperand(kRootRegister, offset));
+}
+
+void TurboAssembler::LoadRootRegisterOffset(Register destination,
+ intptr_t offset) {
+ if (offset == 0) {
+ LoadRR(destination, kRootRegister);
+ } else if (is_uint12(offset)) {
+ la(destination, MemOperand(kRootRegister, offset));
+ } else {
+ DCHECK(is_int20(offset));
+ lay(destination, MemOperand(kRootRegister, offset));
+ }
+}
+
+void TurboAssembler::Jump(Register target, Condition cond) { b(cond, target); }
+
+void TurboAssembler::Jump(intptr_t target, RelocInfo::Mode rmode,
+ Condition cond) {
+ Label skip;
+
+ if (cond != al) b(NegateCondition(cond), &skip);
+
+ DCHECK(rmode == RelocInfo::CODE_TARGET || rmode == RelocInfo::RUNTIME_ENTRY);
+
+ mov(ip, Operand(target, rmode));
+ b(ip);
+
+ bind(&skip);
+}
+
+void TurboAssembler::Jump(Address target, RelocInfo::Mode rmode,
+ Condition cond) {
+ DCHECK(!RelocInfo::IsCodeTarget(rmode));
+ Jump(static_cast<intptr_t>(target), rmode, cond);
+}
+
+void TurboAssembler::Jump(Handle<Code> code, RelocInfo::Mode rmode,
+ Condition cond) {
+ DCHECK(RelocInfo::IsCodeTarget(rmode));
+ DCHECK_IMPLIES(options().isolate_independent_code,
+ Builtins::IsIsolateIndependentBuiltin(*code));
+
+ int builtin_index = Builtins::kNoBuiltinId;
+ bool target_is_builtin =
+ isolate()->builtins()->IsBuiltinHandle(code, &builtin_index);
+
+ if (options().inline_offheap_trampolines && target_is_builtin) {
+ // Inline the trampoline.
+ RecordCommentForOffHeapTrampoline(builtin_index);
+ CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
+ EmbeddedData d = EmbeddedData::FromBlob();
+ Address entry = d.InstructionStartOfBuiltin(builtin_index);
+ mov(ip, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+ b(cond, ip);
+ return;
+ }
+ jump(code, RelocInfo::RELATIVE_CODE_TARGET, cond);
+}
+
+void TurboAssembler::Jump(const ExternalReference& reference) {
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.Acquire();
+ Move(scratch, reference);
+ Jump(scratch);
+}
+
+void TurboAssembler::Call(Register target) {
+ // Branch to target via indirect branch
+ basr(r14, target);
+}
+
+void MacroAssembler::CallJSEntry(Register target) {
+ DCHECK(target == r4);
+ Call(target);
+}
+
+int MacroAssembler::CallSizeNotPredictableCodeSize(Address target,
+ RelocInfo::Mode rmode,
+ Condition cond) {
+ // S390 Assembler::move sequence is IILF / IIHF
+ int size;
+#if V8_TARGET_ARCH_S390X
+ size = 14; // IILF + IIHF + BASR
+#else
+ size = 8; // IILF + BASR
+#endif
+ return size;
+}
+
+void TurboAssembler::Call(Address target, RelocInfo::Mode rmode,
+ Condition cond) {
+ DCHECK(cond == al);
+
+ mov(ip, Operand(target, rmode));
+ basr(r14, ip);
+}
+
+void TurboAssembler::Call(Handle<Code> code, RelocInfo::Mode rmode,
+ Condition cond) {
+ DCHECK(RelocInfo::IsCodeTarget(rmode) && cond == al);
+
+ DCHECK_IMPLIES(options().isolate_independent_code,
+ Builtins::IsIsolateIndependentBuiltin(*code));
+ int builtin_index = Builtins::kNoBuiltinId;
+ bool target_is_builtin =
+ isolate()->builtins()->IsBuiltinHandle(code, &builtin_index);
+
+ if (target_is_builtin && options().inline_offheap_trampolines) {
+ // Inline the trampoline.
+ RecordCommentForOffHeapTrampoline(builtin_index);
+ CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
+ EmbeddedData d = EmbeddedData::FromBlob();
+ Address entry = d.InstructionStartOfBuiltin(builtin_index);
+ mov(ip, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+ Call(ip);
+ return;
+ }
+ DCHECK(code->IsExecutable());
+ call(code, rmode);
+}
+
+void TurboAssembler::Drop(int count) {
+ if (count > 0) {
+ int total = count * kSystemPointerSize;
+ if (is_uint12(total)) {
+ la(sp, MemOperand(sp, total));
+ } else if (is_int20(total)) {
+ lay(sp, MemOperand(sp, total));
+ } else {
+ AddP(sp, Operand(total));
+ }
+ }
+}
+
+void TurboAssembler::Drop(Register count, Register scratch) {
+ ShiftLeftP(scratch, count, Operand(kSystemPointerSizeLog2));
+ AddP(sp, sp, scratch);
+}
+
+void TurboAssembler::Call(Label* target) { b(r14, target); }
+
+void TurboAssembler::Push(Handle<HeapObject> handle) {
+ mov(r0, Operand(handle));
+ push(r0);
+}
+
+void TurboAssembler::Push(Smi smi) {
+ mov(r0, Operand(smi));
+ push(r0);
+}
+
+void TurboAssembler::Move(Register dst, Handle<HeapObject> value,
+ RelocInfo::Mode rmode) {
+ // TODO(jgruber,v8:8887): Also consider a root-relative load when generating
+ // non-isolate-independent code. In many cases it might be cheaper than
+ // embedding the relocatable value.
+ if (root_array_available_ && options().isolate_independent_code) {
+ IndirectLoadConstant(dst, value);
+ return;
+ } else if (RelocInfo::IsCompressedEmbeddedObject(rmode)) {
+ EmbeddedObjectIndex index = AddEmbeddedObject(value);
+ DCHECK(is_uint32(index));
+ mov(dst, Operand(static_cast<int>(index), rmode));
+ } else {
+ DCHECK(RelocInfo::IsFullEmbeddedObject(rmode));
+ mov(dst, Operand(value.address(), rmode));
+ }
+}
+
+void TurboAssembler::Move(Register dst, ExternalReference reference) {
+ // TODO(jgruber,v8:8887): Also consider a root-relative load when generating
+ // non-isolate-independent code. In many cases it might be cheaper than
+ // embedding the relocatable value.
+ if (root_array_available_ && options().isolate_independent_code) {
+ IndirectLoadExternalReference(dst, reference);
+ return;
+ }
+ mov(dst, Operand(reference));
+}
+
+void TurboAssembler::Move(Register dst, Register src, Condition cond) {
+ if (dst != src) {
+ if (cond == al) {
+ LoadRR(dst, src);
+ } else {
+ LoadOnConditionP(cond, dst, src);
+ }
+ }
+}
+
+void TurboAssembler::Move(DoubleRegister dst, DoubleRegister src) {
+ if (dst != src) {
+ ldr(dst, src);
+ }
+}
+
+// Wrapper around Assembler::mvc (SS-a format)
+void TurboAssembler::MoveChar(const MemOperand& opnd1, const MemOperand& opnd2,
+ const Operand& length) {
+ mvc(opnd1, opnd2, Operand(static_cast<intptr_t>(length.immediate() - 1)));
+}
+
+// Wrapper around Assembler::clc (SS-a format)
+void TurboAssembler::CompareLogicalChar(const MemOperand& opnd1,
+ const MemOperand& opnd2,
+ const Operand& length) {
+ clc(opnd1, opnd2, Operand(static_cast<intptr_t>(length.immediate() - 1)));
+}
+
+// Wrapper around Assembler::xc (SS-a format)
+void TurboAssembler::ExclusiveOrChar(const MemOperand& opnd1,
+ const MemOperand& opnd2,
+ const Operand& length) {
+ xc(opnd1, opnd2, Operand(static_cast<intptr_t>(length.immediate() - 1)));
+}
+
+// Wrapper around Assembler::risbg(n) (RIE-f)
+void TurboAssembler::RotateInsertSelectBits(Register dst, Register src,
+ const Operand& startBit,
+ const Operand& endBit,
+ const Operand& shiftAmt,
+ bool zeroBits) {
+ if (zeroBits)
+ // High tag the top bit of I4/EndBit to zero out any unselected bits
+ risbg(dst, src, startBit,
+ Operand(static_cast<intptr_t>(endBit.immediate() | 0x80)), shiftAmt);
+ else
+ risbg(dst, src, startBit, endBit, shiftAmt);
+}
+
+void TurboAssembler::BranchRelativeOnIdxHighP(Register dst, Register inc,
+ Label* L) {
+#if V8_TARGET_ARCH_S390X
+ brxhg(dst, inc, L);
+#else
+ brxh(dst, inc, L);
+#endif // V8_TARGET_ARCH_S390X
+}
+
+void TurboAssembler::PushArray(Register array, Register size, Register scratch,
+ Register scratch2, PushArrayOrder order) {
+ Label loop, done;
+
+ if (order == kNormal) {
+ ShiftLeftP(scratch, size, Operand(kSystemPointerSizeLog2));
+ lay(scratch, MemOperand(array, scratch));
+ bind(&loop);
+ CmpP(array, scratch);
+ bge(&done);
+ lay(scratch, MemOperand(scratch, -kSystemPointerSize));
+ lay(sp, MemOperand(sp, -kSystemPointerSize));
+ MoveChar(MemOperand(sp), MemOperand(scratch), Operand(kSystemPointerSize));
+ b(&loop);
+ bind(&done);
+ } else {
+ DCHECK_NE(scratch2, r0);
+ ShiftLeftP(scratch, size, Operand(kSystemPointerSizeLog2));
+ lay(scratch, MemOperand(array, scratch));
+ LoadRR(scratch2, array);
+ bind(&loop);
+ CmpP(scratch2, scratch);
+ bge(&done);
+ lay(sp, MemOperand(sp, -kSystemPointerSize));
+ MoveChar(MemOperand(sp), MemOperand(scratch2), Operand(kSystemPointerSize));
+ lay(scratch2, MemOperand(scratch2, kSystemPointerSize));
+ b(&loop);
+ bind(&done);
+ }
+}
+
+void TurboAssembler::MultiPush(RegList regs, Register location) {
+ int16_t num_to_push = base::bits::CountPopulation(regs);
+ int16_t stack_offset = num_to_push * kSystemPointerSize;
+
+ SubP(location, location, Operand(stack_offset));
+ for (int16_t i = Register::kNumRegisters - 1; i >= 0; i--) {
+ if ((regs & (1 << i)) != 0) {
+ stack_offset -= kSystemPointerSize;
+ StoreP(ToRegister(i), MemOperand(location, stack_offset));
+ }
+ }
+}
+
+void TurboAssembler::MultiPop(RegList regs, Register location) {
+ int16_t stack_offset = 0;
+
+ for (int16_t i = 0; i < Register::kNumRegisters; i++) {
+ if ((regs & (1 << i)) != 0) {
+ LoadP(ToRegister(i), MemOperand(location, stack_offset));
+ stack_offset += kSystemPointerSize;
+ }
+ }
+ AddP(location, location, Operand(stack_offset));
+}
+
+void TurboAssembler::MultiPushDoubles(RegList dregs, Register location) {
+ int16_t num_to_push = base::bits::CountPopulation(dregs);
+ int16_t stack_offset = num_to_push * kDoubleSize;
+
+ SubP(location, location, Operand(stack_offset));
+ for (int16_t i = DoubleRegister::kNumRegisters - 1; i >= 0; i--) {
+ if ((dregs & (1 << i)) != 0) {
+ DoubleRegister dreg = DoubleRegister::from_code(i);
+ stack_offset -= kDoubleSize;
+ StoreDouble(dreg, MemOperand(location, stack_offset));
+ }
+ }
+}
+
+void TurboAssembler::MultiPopDoubles(RegList dregs, Register location) {
+ int16_t stack_offset = 0;
+
+ for (int16_t i = 0; i < DoubleRegister::kNumRegisters; i++) {
+ if ((dregs & (1 << i)) != 0) {
+ DoubleRegister dreg = DoubleRegister::from_code(i);
+ LoadDouble(dreg, MemOperand(location, stack_offset));
+ stack_offset += kDoubleSize;
+ }
+ }
+ AddP(location, location, Operand(stack_offset));
+}
+
+void TurboAssembler::LoadRoot(Register destination, RootIndex index,
+ Condition) {
+ LoadP(destination,
+ MemOperand(kRootRegister, RootRegisterOffsetForRootIndex(index)), r0);
+}
+
+void TurboAssembler::LoadTaggedPointerField(const Register& destination,
+ const MemOperand& field_operand,
+ const Register& scratch) {
+ if (COMPRESS_POINTERS_BOOL) {
+ DecompressTaggedPointer(destination, field_operand);
+ } else {
+ LoadP(destination, field_operand, scratch);
+ }
+}
+
+void TurboAssembler::LoadAnyTaggedField(const Register& destination,
+ const MemOperand& field_operand,
+ const Register& scratch) {
+ if (COMPRESS_POINTERS_BOOL) {
+ DecompressAnyTagged(destination, field_operand);
+ } else {
+ LoadP(destination, field_operand, scratch);
+ }
+}
+
+void TurboAssembler::SmiUntag(Register dst, const MemOperand& src) {
+ if (SmiValuesAre31Bits()) {
+ LoadW(dst, src);
+ } else {
+ LoadP(dst, src);
+ }
+ SmiUntag(dst);
+}
+
+void TurboAssembler::SmiUntagField(Register dst, const MemOperand& src) {
+ SmiUntag(dst, src);
+}
+
+void TurboAssembler::StoreTaggedField(const Register& value,
+ const MemOperand& dst_field_operand,
+ const Register& scratch) {
+ if (COMPRESS_POINTERS_BOOL) {
+ RecordComment("[ StoreTagged");
+ StoreW(value, dst_field_operand);
+ RecordComment("]");
+ } else {
+ StoreP(value, dst_field_operand, scratch);
+ }
+}
+
+void TurboAssembler::DecompressTaggedSigned(Register destination,
+ Register src) {
+ RecordComment("[ DecompressTaggedSigned");
+ llgfr(destination, src);
+ RecordComment("]");
+}
+
+void TurboAssembler::DecompressTaggedSigned(Register destination,
+ MemOperand field_operand) {
+ RecordComment("[ DecompressTaggedSigned");
+ llgf(destination, field_operand);
+ RecordComment("]");
+}
+
+void TurboAssembler::DecompressTaggedPointer(Register destination,
+ Register source) {
+ RecordComment("[ DecompressTaggedPointer");
+ llgfr(destination, source);
+ agr(destination, kRootRegister);
+ RecordComment("]");
+}
+
+void TurboAssembler::DecompressTaggedPointer(Register destination,
+ MemOperand field_operand) {
+ RecordComment("[ DecompressTaggedPointer");
+ llgf(destination, field_operand);
+ agr(destination, kRootRegister);
+ RecordComment("]");
+}
+
+void TurboAssembler::DecompressAnyTagged(Register destination,
+ MemOperand field_operand) {
+ RecordComment("[ DecompressAnyTagged");
+ llgf(destination, field_operand);
+ agr(destination, kRootRegister);
+ RecordComment("]");
+}
+
+void TurboAssembler::DecompressAnyTagged(Register destination,
+ Register source) {
+ RecordComment("[ DecompressAnyTagged");
+ llgfr(destination, source);
+ agr(destination, kRootRegister);
+ RecordComment("]");
+}
+void MacroAssembler::RecordWriteField(Register object, int offset,
+ Register value, Register dst,
+ LinkRegisterStatus lr_status,
+ SaveFPRegsMode save_fp,
+ RememberedSetAction remembered_set_action,
+ SmiCheck smi_check) {
+ // First, check if a write barrier is even needed. The tests below
+ // catch stores of Smis.
+ Label done;
+
+ // Skip barrier if writing a smi.
+ if (smi_check == INLINE_SMI_CHECK) {
+ JumpIfSmi(value, &done);
+ }
+
+ // Although the object register is tagged, the offset is relative to the start
+ // of the object, so so offset must be a multiple of kSystemPointerSize.
+ DCHECK(IsAligned(offset, kTaggedSize));
+
+ lay(dst, MemOperand(object, offset - kHeapObjectTag));
+ if (emit_debug_code()) {
+ Label ok;
+ AndP(r0, dst, Operand(kTaggedSize - 1));
+ beq(&ok, Label::kNear);
+ stop();
+ bind(&ok);
+ }
+
+ RecordWrite(object, dst, value, lr_status, save_fp, remembered_set_action,
+ OMIT_SMI_CHECK);
+
+ bind(&done);
+
+ // Clobber clobbered input registers when running with the debug-code flag
+ // turned on to provoke errors.
+ if (emit_debug_code()) {
+ mov(value, Operand(bit_cast<intptr_t>(kZapValue + 4)));
+ mov(dst, Operand(bit_cast<intptr_t>(kZapValue + 8)));
+ }
+}
+
+void TurboAssembler::SaveRegisters(RegList registers) {
+ DCHECK_GT(NumRegs(registers), 0);
+ RegList regs = 0;
+ for (int i = 0; i < Register::kNumRegisters; ++i) {
+ if ((registers >> i) & 1u) {
+ regs |= Register::from_code(i).bit();
+ }
+ }
+ MultiPush(regs);
+}
+
+void TurboAssembler::RestoreRegisters(RegList registers) {
+ DCHECK_GT(NumRegs(registers), 0);
+ RegList regs = 0;
+ for (int i = 0; i < Register::kNumRegisters; ++i) {
+ if ((registers >> i) & 1u) {
+ regs |= Register::from_code(i).bit();
+ }
+ }
+ MultiPop(regs);
+}
+
+void TurboAssembler::CallEphemeronKeyBarrier(Register object, Register address,
+ SaveFPRegsMode fp_mode) {
+ EphemeronKeyBarrierDescriptor descriptor;
+ RegList registers = descriptor.allocatable_registers();
+
+ SaveRegisters(registers);
+
+ Register object_parameter(
+ descriptor.GetRegisterParameter(EphemeronKeyBarrierDescriptor::kObject));
+ Register slot_parameter(descriptor.GetRegisterParameter(
+ EphemeronKeyBarrierDescriptor::kSlotAddress));
+ Register fp_mode_parameter(
+ descriptor.GetRegisterParameter(EphemeronKeyBarrierDescriptor::kFPMode));
+
+ Push(object);
+ Push(address);
+
+ Pop(slot_parameter);
+ Pop(object_parameter);
+
+ Move(fp_mode_parameter, Smi::FromEnum(fp_mode));
+ Call(isolate()->builtins()->builtin_handle(Builtins::kEphemeronKeyBarrier),
+ RelocInfo::CODE_TARGET);
+ RestoreRegisters(registers);
+}
+
+void TurboAssembler::CallRecordWriteStub(
+ Register object, Register address,
+ RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode) {
+ CallRecordWriteStub(
+ object, address, remembered_set_action, fp_mode,
+ isolate()->builtins()->builtin_handle(Builtins::kRecordWrite),
+ kNullAddress);
+}
+
+void TurboAssembler::CallRecordWriteStub(
+ Register object, Register address,
+ RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode,
+ Address wasm_target) {
+ CallRecordWriteStub(object, address, remembered_set_action, fp_mode,
+ Handle<Code>::null(), wasm_target);
+}
+
+void TurboAssembler::CallRecordWriteStub(
+ Register object, Register address,
+ RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode,
+ Handle<Code> code_target, Address wasm_target) {
+ DCHECK_NE(code_target.is_null(), wasm_target == kNullAddress);
+ // TODO(albertnetymk): For now we ignore remembered_set_action and fp_mode,
+ // i.e. always emit remember set and save FP registers in RecordWriteStub. If
+ // large performance regression is observed, we should use these values to
+ // avoid unnecessary work.
+
+ RecordWriteDescriptor descriptor;
+ RegList registers = descriptor.allocatable_registers();
+
+ SaveRegisters(registers);
+ Register object_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kObject));
+ Register slot_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kSlot));
+ Register remembered_set_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kRememberedSet));
+ Register fp_mode_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kFPMode));
+
+ Push(object);
+ Push(address);
+
+ Pop(slot_parameter);
+ Pop(object_parameter);
+
+ Move(remembered_set_parameter, Smi::FromEnum(remembered_set_action));
+ Move(fp_mode_parameter, Smi::FromEnum(fp_mode));
+ if (code_target.is_null()) {
+ Call(wasm_target, RelocInfo::WASM_STUB_CALL);
+ } else {
+ Call(code_target, RelocInfo::CODE_TARGET);
+ }
+
+ RestoreRegisters(registers);
+}
+
+// Will clobber 4 registers: object, address, scratch, ip. The
+// register 'object' contains a heap object pointer. The heap object
+// tag is shifted away.
+void MacroAssembler::RecordWrite(Register object, Register address,
+ Register value, LinkRegisterStatus lr_status,
+ SaveFPRegsMode fp_mode,
+ RememberedSetAction remembered_set_action,
+ SmiCheck smi_check) {
+ DCHECK(object != value);
+ if (emit_debug_code()) {
+ LoadTaggedPointerField(r0, MemOperand(address));
+ CmpP(value, r0);
+ Check(eq, AbortReason::kWrongAddressOrValuePassedToRecordWrite);
+ }
+
+ if ((remembered_set_action == OMIT_REMEMBERED_SET &&
+ !FLAG_incremental_marking) ||
+ FLAG_disable_write_barriers) {
+ return;
+ }
+ // First, check if a write barrier is even needed. The tests below
+ // catch stores of smis and stores into the young generation.
+ Label done;
+
+ if (smi_check == INLINE_SMI_CHECK) {
+ JumpIfSmi(value, &done);
+ }
+
+ CheckPageFlag(value,
+ value, // Used as scratch.
+ MemoryChunk::kPointersToHereAreInterestingMask, eq, &done);
+ CheckPageFlag(object,
+ value, // Used as scratch.
+ MemoryChunk::kPointersFromHereAreInterestingMask, eq, &done);
+
+ // Record the actual write.
+ if (lr_status == kLRHasNotBeenSaved) {
+ push(r14);
+ }
+ CallRecordWriteStub(object, address, remembered_set_action, fp_mode);
+ if (lr_status == kLRHasNotBeenSaved) {
+ pop(r14);
+ }
+
+ bind(&done);
+
+ // Clobber clobbered registers when running with the debug-code flag
+ // turned on to provoke errors.
+ if (emit_debug_code()) {
+ mov(address, Operand(bit_cast<intptr_t>(kZapValue + 12)));
+ mov(value, Operand(bit_cast<intptr_t>(kZapValue + 16)));
+ }
+}
+
+void TurboAssembler::PushCommonFrame(Register marker_reg) {
+ int fp_delta = 0;
+ CleanseP(r14);
+ if (marker_reg.is_valid()) {
+ Push(r14, fp, marker_reg);
+ fp_delta = 1;
+ } else {
+ Push(r14, fp);
+ fp_delta = 0;
+ }
+ la(fp, MemOperand(sp, fp_delta * kSystemPointerSize));
+}
+
+void TurboAssembler::PopCommonFrame(Register marker_reg) {
+ if (marker_reg.is_valid()) {
+ Pop(r14, fp, marker_reg);
+ } else {
+ Pop(r14, fp);
+ }
+}
+
+void TurboAssembler::PushStandardFrame(Register function_reg) {
+ int fp_delta = 0;
+ CleanseP(r14);
+ if (function_reg.is_valid()) {
+ Push(r14, fp, cp, function_reg);
+ fp_delta = 2;
+ } else {
+ Push(r14, fp, cp);
+ fp_delta = 1;
+ }
+ la(fp, MemOperand(sp, fp_delta * kSystemPointerSize));
+ Push(kJavaScriptCallArgCountRegister);
+}
+
+void TurboAssembler::RestoreFrameStateForTailCall() {
+ // if (FLAG_enable_embedded_constant_pool) {
+ // LoadP(kConstantPoolRegister,
+ // MemOperand(fp, StandardFrameConstants::kConstantPoolOffset));
+ // set_constant_pool_available(false);
+ // }
+ DCHECK(!FLAG_enable_embedded_constant_pool);
+ LoadP(r14, MemOperand(fp, StandardFrameConstants::kCallerPCOffset));
+ LoadP(fp, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
+}
+
+void TurboAssembler::CanonicalizeNaN(const DoubleRegister dst,
+ const DoubleRegister src) {
+ // Turn potential sNaN into qNaN
+ if (dst != src) ldr(dst, src);
+ lzdr(kDoubleRegZero);
+ sdbr(dst, kDoubleRegZero);
+}
+
+void TurboAssembler::ConvertIntToDouble(DoubleRegister dst, Register src) {
+ cdfbr(dst, src);
+}
+
+void TurboAssembler::ConvertUnsignedIntToDouble(DoubleRegister dst,
+ Register src) {
+ if (CpuFeatures::IsSupported(FLOATING_POINT_EXT)) {
+ cdlfbr(Condition(5), Condition(0), dst, src);
+ } else {
+ // zero-extend src
+ llgfr(src, src);
+ // convert to double
+ cdgbr(dst, src);
+ }
+}
+
+void TurboAssembler::ConvertIntToFloat(DoubleRegister dst, Register src) {
+ cefbra(Condition(4), dst, src);
+}
+
+void TurboAssembler::ConvertUnsignedIntToFloat(DoubleRegister dst,
+ Register src) {
+ celfbr(Condition(4), Condition(0), dst, src);
+}
+
+void TurboAssembler::ConvertInt64ToFloat(DoubleRegister double_dst,
+ Register src) {
+ cegbr(double_dst, src);
+}
+
+void TurboAssembler::ConvertInt64ToDouble(DoubleRegister double_dst,
+ Register src) {
+ cdgbr(double_dst, src);
+}
+
+void TurboAssembler::ConvertUnsignedInt64ToFloat(DoubleRegister double_dst,
+ Register src) {
+ celgbr(Condition(0), Condition(0), double_dst, src);
+}
+
+void TurboAssembler::ConvertUnsignedInt64ToDouble(DoubleRegister double_dst,
+ Register src) {
+ cdlgbr(Condition(0), Condition(0), double_dst, src);
+}
+
+void TurboAssembler::ConvertFloat32ToInt64(const Register dst,
+ const DoubleRegister double_input,
+ FPRoundingMode rounding_mode) {
+ Condition m = Condition(0);
+ switch (rounding_mode) {
+ case kRoundToZero:
+ m = Condition(5);
+ break;
+ case kRoundToNearest:
+ UNIMPLEMENTED();
+ break;
+ case kRoundToPlusInf:
+ m = Condition(6);
+ break;
+ case kRoundToMinusInf:
+ m = Condition(7);
+ break;
+ default:
+ UNIMPLEMENTED();
+ break;
+ }
+ cgebr(m, dst, double_input);
+}
+
+void TurboAssembler::ConvertDoubleToInt64(const Register dst,
+ const DoubleRegister double_input,
+ FPRoundingMode rounding_mode) {
+ Condition m = Condition(0);
+ switch (rounding_mode) {
+ case kRoundToZero:
+ m = Condition(5);
+ break;
+ case kRoundToNearest:
+ UNIMPLEMENTED();
+ break;
+ case kRoundToPlusInf:
+ m = Condition(6);
+ break;
+ case kRoundToMinusInf:
+ m = Condition(7);
+ break;
+ default:
+ UNIMPLEMENTED();
+ break;
+ }
+ cgdbr(m, dst, double_input);
+}
+
+void TurboAssembler::ConvertDoubleToInt32(const Register dst,
+ const DoubleRegister double_input,
+ FPRoundingMode rounding_mode) {
+ Condition m = Condition(0);
+ switch (rounding_mode) {
+ case kRoundToZero:
+ m = Condition(5);
+ break;
+ case kRoundToNearest:
+ m = Condition(4);
+ break;
+ case kRoundToPlusInf:
+ m = Condition(6);
+ break;
+ case kRoundToMinusInf:
+ m = Condition(7);
+ break;
+ default:
+ UNIMPLEMENTED();
+ break;
+ }
+#ifdef V8_TARGET_ARCH_S390X
+ lghi(dst, Operand::Zero());
+#endif
+ cfdbr(m, dst, double_input);
+}
+
+void TurboAssembler::ConvertFloat32ToInt32(const Register result,
+ const DoubleRegister double_input,
+ FPRoundingMode rounding_mode) {
+ Condition m = Condition(0);
+ switch (rounding_mode) {
+ case kRoundToZero:
+ m = Condition(5);
+ break;
+ case kRoundToNearest:
+ m = Condition(4);
+ break;
+ case kRoundToPlusInf:
+ m = Condition(6);
+ break;
+ case kRoundToMinusInf:
+ m = Condition(7);
+ break;
+ default:
+ UNIMPLEMENTED();
+ break;
+ }
+#ifdef V8_TARGET_ARCH_S390X
+ lghi(result, Operand::Zero());
+#endif
+ cfebr(m, result, double_input);
+}
+
+void TurboAssembler::ConvertFloat32ToUnsignedInt32(
+ const Register result, const DoubleRegister double_input,
+ FPRoundingMode rounding_mode) {
+ Condition m = Condition(0);
+ switch (rounding_mode) {
+ case kRoundToZero:
+ m = Condition(5);
+ break;
+ case kRoundToNearest:
+ UNIMPLEMENTED();
+ break;
+ case kRoundToPlusInf:
+ m = Condition(6);
+ break;
+ case kRoundToMinusInf:
+ m = Condition(7);
+ break;
+ default:
+ UNIMPLEMENTED();
+ break;
+ }
+#ifdef V8_TARGET_ARCH_S390X
+ lghi(result, Operand::Zero());
+#endif
+ clfebr(m, Condition(0), result, double_input);
+}
+
+void TurboAssembler::ConvertFloat32ToUnsignedInt64(
+ const Register result, const DoubleRegister double_input,
+ FPRoundingMode rounding_mode) {
+ Condition m = Condition(0);
+ switch (rounding_mode) {
+ case kRoundToZero:
+ m = Condition(5);
+ break;
+ case kRoundToNearest:
+ UNIMPLEMENTED();
+ break;
+ case kRoundToPlusInf:
+ m = Condition(6);
+ break;
+ case kRoundToMinusInf:
+ m = Condition(7);
+ break;
+ default:
+ UNIMPLEMENTED();
+ break;
+ }
+ clgebr(m, Condition(0), result, double_input);
+}
+
+void TurboAssembler::ConvertDoubleToUnsignedInt64(
+ const Register dst, const DoubleRegister double_input,
+ FPRoundingMode rounding_mode) {
+ Condition m = Condition(0);
+ switch (rounding_mode) {
+ case kRoundToZero:
+ m = Condition(5);
+ break;
+ case kRoundToNearest:
+ UNIMPLEMENTED();
+ break;
+ case kRoundToPlusInf:
+ m = Condition(6);
+ break;
+ case kRoundToMinusInf:
+ m = Condition(7);
+ break;
+ default:
+ UNIMPLEMENTED();
+ break;
+ }
+ clgdbr(m, Condition(0), dst, double_input);
+}
+
+void TurboAssembler::ConvertDoubleToUnsignedInt32(
+ const Register dst, const DoubleRegister double_input,
+ FPRoundingMode rounding_mode) {
+ Condition m = Condition(0);
+ switch (rounding_mode) {
+ case kRoundToZero:
+ m = Condition(5);
+ break;
+ case kRoundToNearest:
+ UNIMPLEMENTED();
+ break;
+ case kRoundToPlusInf:
+ m = Condition(6);
+ break;
+ case kRoundToMinusInf:
+ m = Condition(7);
+ break;
+ default:
+ UNIMPLEMENTED();
+ break;
+ }
+#ifdef V8_TARGET_ARCH_S390X
+ lghi(dst, Operand::Zero());
+#endif
+ clfdbr(m, Condition(0), dst, double_input);
+}
+
+#if !V8_TARGET_ARCH_S390X
+void TurboAssembler::ShiftLeftPair(Register dst_low, Register dst_high,
+ Register src_low, Register src_high,
+ Register scratch, Register shift) {
+ LoadRR(r0, src_high);
+ LoadRR(r1, src_low);
+ sldl(r0, shift, Operand::Zero());
+ LoadRR(dst_high, r0);
+ LoadRR(dst_low, r1);
+}
+
+void TurboAssembler::ShiftLeftPair(Register dst_low, Register dst_high,
+ Register src_low, Register src_high,
+ uint32_t shift) {
+ LoadRR(r0, src_high);
+ LoadRR(r1, src_low);
+ sldl(r0, r0, Operand(shift));
+ LoadRR(dst_high, r0);
+ LoadRR(dst_low, r1);
+}
+
+void TurboAssembler::ShiftRightPair(Register dst_low, Register dst_high,
+ Register src_low, Register src_high,
+ Register scratch, Register shift) {
+ LoadRR(r0, src_high);
+ LoadRR(r1, src_low);
+ srdl(r0, shift, Operand::Zero());
+ LoadRR(dst_high, r0);
+ LoadRR(dst_low, r1);
+}
+
+void TurboAssembler::ShiftRightPair(Register dst_low, Register dst_high,
+ Register src_low, Register src_high,
+ uint32_t shift) {
+ LoadRR(r0, src_high);
+ LoadRR(r1, src_low);
+ srdl(r0, Operand(shift));
+ LoadRR(dst_high, r0);
+ LoadRR(dst_low, r1);
+}
+
+void TurboAssembler::ShiftRightArithPair(Register dst_low, Register dst_high,
+ Register src_low, Register src_high,
+ Register scratch, Register shift) {
+ LoadRR(r0, src_high);
+ LoadRR(r1, src_low);
+ srda(r0, shift, Operand::Zero());
+ LoadRR(dst_high, r0);
+ LoadRR(dst_low, r1);
+}
+
+void TurboAssembler::ShiftRightArithPair(Register dst_low, Register dst_high,
+ Register src_low, Register src_high,
+ uint32_t shift) {
+ LoadRR(r0, src_high);
+ LoadRR(r1, src_low);
+ srda(r0, r0, Operand(shift));
+ LoadRR(dst_high, r0);
+ LoadRR(dst_low, r1);
+}
+#endif
+
+void TurboAssembler::MovDoubleToInt64(Register dst, DoubleRegister src) {
+ lgdr(dst, src);
+}
+
+void TurboAssembler::MovInt64ToDouble(DoubleRegister dst, Register src) {
+ ldgr(dst, src);
+}
+
+void TurboAssembler::StubPrologue(StackFrame::Type type, Register base,
+ int prologue_offset) {
+ {
+ ConstantPoolUnavailableScope constant_pool_unavailable(this);
+ Load(r1, Operand(StackFrame::TypeToMarker(type)));
+ PushCommonFrame(r1);
+ }
+}
+
+void TurboAssembler::Prologue(Register base, int prologue_offset) {
+ DCHECK(base != no_reg);
+ PushStandardFrame(r3);
+}
+
+void TurboAssembler::EnterFrame(StackFrame::Type type,
+ bool load_constant_pool_pointer_reg) {
+ // We create a stack frame with:
+ // Return Addr <-- old sp
+ // Old FP <-- new fp
+ // CP
+ // type
+ // CodeObject <-- new sp
+
+ Load(ip, Operand(StackFrame::TypeToMarker(type)));
+ PushCommonFrame(ip);
+}
+
+int TurboAssembler::LeaveFrame(StackFrame::Type type, int stack_adjustment) {
+ // Drop the execution stack down to the frame pointer and restore
+ // the caller frame pointer, return address and constant pool pointer.
+ LoadP(r14, MemOperand(fp, StandardFrameConstants::kCallerPCOffset));
+ if (is_int20(StandardFrameConstants::kCallerSPOffset + stack_adjustment)) {
+ lay(r1, MemOperand(fp, StandardFrameConstants::kCallerSPOffset +
+ stack_adjustment));
+ } else {
+ AddP(r1, fp,
+ Operand(StandardFrameConstants::kCallerSPOffset + stack_adjustment));
+ }
+ LoadP(fp, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
+ LoadRR(sp, r1);
+ int frame_ends = pc_offset();
+ return frame_ends;
+}
+
+// ExitFrame layout (probably wrongish.. needs updating)
+//
+// SP -> previousSP
+// LK reserved
+// sp_on_exit (for debug?)
+// oldSP->prev SP
+// LK
+// <parameters on stack>
+
+// Prior to calling EnterExitFrame, we've got a bunch of parameters
+// on the stack that we need to wrap a real frame around.. so first
+// we reserve a slot for LK and push the previous SP which is captured
+// in the fp register (r11)
+// Then - we buy a new frame
+
+// r14
+// oldFP <- newFP
+// SP
+// Floats
+// gaps
+// Args
+// ABIRes <- newSP
+void MacroAssembler::EnterExitFrame(bool save_doubles, int stack_space,
+ StackFrame::Type frame_type) {
+ DCHECK(frame_type == StackFrame::EXIT ||
+ frame_type == StackFrame::BUILTIN_EXIT);
+ // Set up the frame structure on the stack.
+ DCHECK_EQ(2 * kSystemPointerSize, ExitFrameConstants::kCallerSPDisplacement);
+ DCHECK_EQ(1 * kSystemPointerSize, ExitFrameConstants::kCallerPCOffset);
+ DCHECK_EQ(0 * kSystemPointerSize, ExitFrameConstants::kCallerFPOffset);
+ DCHECK_GT(stack_space, 0);
+
+ // This is an opportunity to build a frame to wrap
+ // all of the pushes that have happened inside of V8
+ // since we were called from C code
+ CleanseP(r14);
+ Load(r1, Operand(StackFrame::TypeToMarker(frame_type)));
+ PushCommonFrame(r1);
+ // Reserve room for saved entry sp.
+ lay(sp, MemOperand(fp, -ExitFrameConstants::kFixedFrameSizeFromFp));
+
+ if (emit_debug_code()) {
+ StoreP(MemOperand(fp, ExitFrameConstants::kSPOffset), Operand::Zero(), r1);
+ }
+
+ // Save the frame pointer and the context in top.
+ Move(r1, ExternalReference::Create(IsolateAddressId::kCEntryFPAddress,
+ isolate()));
+ StoreP(fp, MemOperand(r1));
+ Move(r1,
+ ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
+ StoreP(cp, MemOperand(r1));
+
+ // Optionally save all volatile double registers.
+ if (save_doubles) {
+ MultiPushDoubles(kCallerSavedDoubles);
+ // Note that d0 will be accessible at
+ // fp - ExitFrameConstants::kFrameSize -
+ // kNumCallerSavedDoubles * kDoubleSize,
+ // since the sp slot and code slot were pushed after the fp.
+ }
+
+ lay(sp, MemOperand(sp, -stack_space * kSystemPointerSize));
+
+ // Allocate and align the frame preparing for calling the runtime
+ // function.
+ const int frame_alignment = TurboAssembler::ActivationFrameAlignment();
+ if (frame_alignment > 0) {
+ DCHECK_EQ(frame_alignment, 8);
+ ClearRightImm(sp, sp, Operand(3)); // equivalent to &= -8
+ }
+
+ lay(sp, MemOperand(sp, -kNumRequiredStackFrameSlots * kSystemPointerSize));
+ StoreP(MemOperand(sp), Operand::Zero(), r0);
+ // Set the exit frame sp value to point just before the return address
+ // location.
+ lay(r1, MemOperand(sp, kStackFrameSPSlot * kSystemPointerSize));
+ StoreP(r1, MemOperand(fp, ExitFrameConstants::kSPOffset));
+}
+
+int TurboAssembler::ActivationFrameAlignment() {
+#if !defined(USE_SIMULATOR)
+ // Running on the real platform. Use the alignment as mandated by the local
+ // environment.
+ // Note: This will break if we ever start generating snapshots on one S390
+ // platform for another S390 platform with a different alignment.
+ return base::OS::ActivationFrameAlignment();
+#else // Simulated
+ // If we are using the simulator then we should always align to the expected
+ // alignment. As the simulator is used to generate snapshots we do not know
+ // if the target platform will need alignment, so this is controlled from a
+ // flag.
+ return FLAG_sim_stack_alignment;
+#endif
+}
+
+void MacroAssembler::LeaveExitFrame(bool save_doubles, Register argument_count,
+ bool argument_count_is_length) {
+ // Optionally restore all double registers.
+ if (save_doubles) {
+ // Calculate the stack location of the saved doubles and restore them.
+ const int kNumRegs = kNumCallerSavedDoubles;
+ lay(r5, MemOperand(fp, -(ExitFrameConstants::kFixedFrameSizeFromFp +
+ kNumRegs * kDoubleSize)));
+ MultiPopDoubles(kCallerSavedDoubles, r5);
+ }
+
+ // Clear top frame.
+ Move(ip, ExternalReference::Create(IsolateAddressId::kCEntryFPAddress,
+ isolate()));
+ StoreP(MemOperand(ip), Operand(0, RelocInfo::NONE), r0);
+
+ // Restore current context from top and clear it in debug mode.
+ Move(ip,
+ ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
+ LoadP(cp, MemOperand(ip));
+
+#ifdef DEBUG
+ mov(r1, Operand(Context::kInvalidContext));
+ Move(ip,
+ ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
+ StoreP(r1, MemOperand(ip));
+#endif
+
+ // Tear down the exit frame, pop the arguments, and return.
+ LeaveFrame(StackFrame::EXIT);
+
+ if (argument_count.is_valid()) {
+ if (!argument_count_is_length) {
+ ShiftLeftP(argument_count, argument_count,
+ Operand(kSystemPointerSizeLog2));
+ }
+ la(sp, MemOperand(sp, argument_count));
+ }
+}
+
+void TurboAssembler::MovFromFloatResult(const DoubleRegister dst) {
+ Move(dst, d0);
+}
+
+void TurboAssembler::MovFromFloatParameter(const DoubleRegister dst) {
+ Move(dst, d0);
+}
+
+void TurboAssembler::PrepareForTailCall(Register callee_args_count,
+ Register caller_args_count,
+ Register scratch0, Register scratch1) {
+ DCHECK(!AreAliased(callee_args_count, caller_args_count, scratch0, scratch1));
+
+ // Calculate the end of destination area where we will put the arguments
+ // after we drop current frame. We AddP kSystemPointerSize to count the
+ // receiver argument which is not included into formal parameters count.
+ Register dst_reg = scratch0;
+ ShiftLeftP(dst_reg, caller_args_count, Operand(kSystemPointerSizeLog2));
+ AddP(dst_reg, fp, dst_reg);
+ AddP(dst_reg, dst_reg,
+ Operand(StandardFrameConstants::kCallerSPOffset + kSystemPointerSize));
+
+ Register src_reg = caller_args_count;
+ // Calculate the end of source area. +kSystemPointerSize is for the receiver.
+ ShiftLeftP(src_reg, callee_args_count, Operand(kSystemPointerSizeLog2));
+ AddP(src_reg, sp, src_reg);
+ AddP(src_reg, src_reg, Operand(kSystemPointerSize));
+
+ if (FLAG_debug_code) {
+ CmpLogicalP(src_reg, dst_reg);
+ Check(lt, AbortReason::kStackAccessBelowStackPointer);
+ }
+
+ // Restore caller's frame pointer and return address now as they will be
+ // overwritten by the copying loop.
+ RestoreFrameStateForTailCall();
+
+ // Now copy callee arguments to the caller frame going backwards to avoid
+ // callee arguments corruption (source and destination areas could overlap).
+
+ // Both src_reg and dst_reg are pointing to the word after the one to copy,
+ // so they must be pre-decremented in the loop.
+ Register tmp_reg = scratch1;
+ Label loop;
+ AddP(tmp_reg, callee_args_count, Operand(1)); // +1 for receiver
+ LoadRR(r1, tmp_reg);
+ bind(&loop);
+ LoadP(tmp_reg, MemOperand(src_reg, -kSystemPointerSize));
+ StoreP(tmp_reg, MemOperand(dst_reg, -kSystemPointerSize));
+ lay(src_reg, MemOperand(src_reg, -kSystemPointerSize));
+ lay(dst_reg, MemOperand(dst_reg, -kSystemPointerSize));
+ BranchOnCount(r1, &loop);
+
+ // Leave current frame.
+ LoadRR(sp, dst_reg);
+}
+
+void MacroAssembler::InvokePrologue(Register expected_parameter_count,
+ Register actual_parameter_count,
+ Label* done, InvokeFlag flag) {
+ Label regular_invoke;
+
+ // Check whether the expected and actual arguments count match. If not,
+ // setup registers according to contract with ArgumentsAdaptorTrampoline:
+ // r2: actual arguments count
+ // r3: function (passed through to callee)
+ // r4: expected arguments count
+
+ // The code below is made a lot easier because the calling code already sets
+ // up actual and expected registers according to the contract.
+ // ARM has some checks as per below, considering add them for S390
+ DCHECK_EQ(actual_parameter_count, r2);
+ DCHECK_EQ(expected_parameter_count, r4);
+
+ CmpP(expected_parameter_count, actual_parameter_count);
+ beq(®ular_invoke);
+
+ Handle<Code> adaptor = BUILTIN_CODE(isolate(), ArgumentsAdaptorTrampoline);
+ if (flag == CALL_FUNCTION) {
+ Call(adaptor);
+ b(done);
+ } else {
+ Jump(adaptor, RelocInfo::CODE_TARGET);
+ }
+ bind(®ular_invoke);
+}
+
+void MacroAssembler::CheckDebugHook(Register fun, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count) {
+ Label skip_hook;
+
+ ExternalReference debug_hook_active =
+ ExternalReference::debug_hook_on_function_call_address(isolate());
+ Move(r6, debug_hook_active);
+ tm(MemOperand(r6), Operand(0xFF));
+ beq(&skip_hook);
+
+ {
+ // Load receiver to pass it later to DebugOnFunctionCall hook.
+ LoadReceiver(r6, actual_parameter_count);
+ FrameScope frame(this,
+ has_frame() ? StackFrame::NONE : StackFrame::INTERNAL);
+
+ SmiTag(expected_parameter_count);
+ Push(expected_parameter_count);
+
+ SmiTag(actual_parameter_count);
+ Push(actual_parameter_count);
+
+ if (new_target.is_valid()) {
+ Push(new_target);
+ }
+ Push(fun, fun, r6);
+ CallRuntime(Runtime::kDebugOnFunctionCall);
+ Pop(fun);
+ if (new_target.is_valid()) {
+ Pop(new_target);
+ }
+
+ Pop(actual_parameter_count);
+ SmiUntag(actual_parameter_count);
+
+ Pop(expected_parameter_count);
+ SmiUntag(expected_parameter_count);
+ }
+ bind(&skip_hook);
+}
+
+void MacroAssembler::InvokeFunctionCode(Register function, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count,
+ InvokeFlag flag) {
+ // You can't call a function without a valid frame.
+ DCHECK_IMPLIES(flag == CALL_FUNCTION, has_frame());
+ DCHECK_EQ(function, r3);
+ DCHECK_IMPLIES(new_target.is_valid(), new_target == r5);
+
+ // On function call, call into the debugger if necessary.
+ CheckDebugHook(function, new_target, expected_parameter_count,
+ actual_parameter_count);
+
+ // Clear the new.target register if not given.
+ if (!new_target.is_valid()) {
+ LoadRoot(r5, RootIndex::kUndefinedValue);
+ }
+
+ Label done;
+ InvokePrologue(expected_parameter_count, actual_parameter_count, &done, flag);
+ // We call indirectly through the code field in the function to
+ // allow recompilation to take effect without changing any of the
+ // call sites.
+ Register code = kJavaScriptCallCodeStartRegister;
+ LoadTaggedPointerField(code,
+ FieldMemOperand(function, JSFunction::kCodeOffset));
+ if (flag == CALL_FUNCTION) {
+ CallCodeObject(code);
+ } else {
+ DCHECK(flag == JUMP_FUNCTION);
+ JumpCodeObject(code);
+ }
+ // Continue here if InvokePrologue does handle the invocation due to
+ // mismatched parameter counts.
+ bind(&done);
+}
+
+void MacroAssembler::InvokeFunctionWithNewTarget(
+ Register fun, Register new_target, Register actual_parameter_count,
+ InvokeFlag flag) {
+ // You can't call a function without a valid frame.
+ DCHECK_IMPLIES(flag == CALL_FUNCTION, has_frame());
+
+ // Contract with called JS functions requires that function is passed in r3.
+ DCHECK_EQ(fun, r3);
+
+ Register expected_reg = r4;
+ Register temp_reg = r6;
+ LoadTaggedPointerField(cp, FieldMemOperand(fun, JSFunction::kContextOffset));
+ LoadTaggedPointerField(
+ temp_reg, FieldMemOperand(fun, JSFunction::kSharedFunctionInfoOffset));
+ LoadLogicalHalfWordP(
+ expected_reg,
+ FieldMemOperand(temp_reg,
+ SharedFunctionInfo::kFormalParameterCountOffset));
+
+ InvokeFunctionCode(fun, new_target, expected_reg, actual_parameter_count,
+ flag);
+}
+
+void MacroAssembler::InvokeFunction(Register function,
+ Register expected_parameter_count,
+ Register actual_parameter_count,
+ InvokeFlag flag) {
+ // You can't call a function without a valid frame.
+ DCHECK_IMPLIES(flag == CALL_FUNCTION, has_frame());
+
+ // Contract with called JS functions requires that function is passed in r3.
+ DCHECK_EQ(function, r3);
+
+ // Get the function and setup the context.
+ LoadTaggedPointerField(cp,
+ FieldMemOperand(function, JSFunction::kContextOffset));
+
+ InvokeFunctionCode(r3, no_reg, expected_parameter_count,
+ actual_parameter_count, flag);
+}
+
+void MacroAssembler::MaybeDropFrames() {
+ // Check whether we need to drop frames to restart a function on the stack.
+ ExternalReference restart_fp =
+ ExternalReference::debug_restart_fp_address(isolate());
+ Move(r3, restart_fp);
+ LoadP(r3, MemOperand(r3));
+ CmpP(r3, Operand::Zero());
+ Jump(BUILTIN_CODE(isolate(), FrameDropperTrampoline), RelocInfo::CODE_TARGET,
+ ne);
+}
+
+void MacroAssembler::PushStackHandler() {
+ // Adjust this code if not the case.
+ STATIC_ASSERT(StackHandlerConstants::kSize == 2 * kSystemPointerSize);
+ STATIC_ASSERT(StackHandlerConstants::kNextOffset == 0 * kSystemPointerSize);
+
+ // Link the current handler as the next handler.
+ Move(r7,
+ ExternalReference::Create(IsolateAddressId::kHandlerAddress, isolate()));
+
+ // Buy the full stack frame for 5 slots.
+ lay(sp, MemOperand(sp, -StackHandlerConstants::kSize));
+
+ // Store padding.
+ lghi(r0, Operand::Zero());
+ StoreP(r0, MemOperand(sp)); // Padding.
+
+ // Copy the old handler into the next handler slot.
+ MoveChar(MemOperand(sp, StackHandlerConstants::kNextOffset), MemOperand(r7),
+ Operand(kSystemPointerSize));
+ // Set this new handler as the current one.
+ StoreP(sp, MemOperand(r7));
+}
+
+void MacroAssembler::PopStackHandler() {
+ STATIC_ASSERT(StackHandlerConstants::kSize == 2 * kSystemPointerSize);
+ STATIC_ASSERT(StackHandlerConstants::kNextOffset == 0);
+
+ // Pop the Next Handler into r3 and store it into Handler Address reference.
+ Pop(r3);
+ Move(ip,
+ ExternalReference::Create(IsolateAddressId::kHandlerAddress, isolate()));
+ StoreP(r3, MemOperand(ip));
+
+ Drop(1); // Drop padding.
+}
+
+void MacroAssembler::CompareObjectType(Register object, Register map,
+ Register type_reg, InstanceType type) {
+ const Register temp = type_reg == no_reg ? r0 : type_reg;
+
+ LoadMap(map, object);
+ CompareInstanceType(map, temp, type);
+}
+
+void MacroAssembler::CompareInstanceType(Register map, Register type_reg,
+ InstanceType type) {
+ STATIC_ASSERT(Map::kInstanceTypeOffset < 4096);
+ STATIC_ASSERT(LAST_TYPE <= 0xFFFF);
+ LoadHalfWordP(type_reg, FieldMemOperand(map, Map::kInstanceTypeOffset));
+ CmpP(type_reg, Operand(type));
+}
+
+void MacroAssembler::CompareRoot(Register obj, RootIndex index) {
+ int32_t offset = RootRegisterOffsetForRootIndex(index);
+#ifdef V8_TARGET_BIG_ENDIAN
+ offset += (COMPRESS_POINTERS_BOOL ? kTaggedSize : 0);
+#endif
+ CompareTagged(obj, MemOperand(kRootRegister, offset));
+}
+
+void MacroAssembler::JumpIfIsInRange(Register value, unsigned lower_limit,
+ unsigned higher_limit,
+ Label* on_in_range) {
+ if (lower_limit != 0) {
+ Register scratch = r0;
+ LoadRR(scratch, value);
+ slgfi(scratch, Operand(lower_limit));
+ CmpLogicalP(scratch, Operand(higher_limit - lower_limit));
+ } else {
+ CmpLogicalP(value, Operand(higher_limit));
+ }
+ ble(on_in_range);
+}
+
+void TurboAssembler::TruncateDoubleToI(Isolate* isolate, Zone* zone,
+ Register result,
+ DoubleRegister double_input,
+ StubCallMode stub_mode) {
+ Label done;
+
+ TryInlineTruncateDoubleToI(result, double_input, &done);
+
+ // If we fell through then inline version didn't succeed - call stub instead.
+ push(r14);
+ // Put input on stack.
+ lay(sp, MemOperand(sp, -kDoubleSize));
+ StoreDouble(double_input, MemOperand(sp));
+
+ if (stub_mode == StubCallMode::kCallWasmRuntimeStub) {
+ Call(wasm::WasmCode::kDoubleToI, RelocInfo::WASM_STUB_CALL);
+ } else {
+ Call(BUILTIN_CODE(isolate, DoubleToI), RelocInfo::CODE_TARGET);
+ }
+
+ LoadP(result, MemOperand(sp, 0));
+ la(sp, MemOperand(sp, kDoubleSize));
+ pop(r14);
+
+ bind(&done);
+}
+
+void TurboAssembler::TryInlineTruncateDoubleToI(Register result,
+ DoubleRegister double_input,
+ Label* done) {
+ ConvertDoubleToInt64(result, double_input);
+
+ // Test for overflow
+ TestIfInt32(result);
+ beq(done);
+}
+
+void MacroAssembler::CallRuntime(const Runtime::Function* f, int num_arguments,
+ SaveFPRegsMode save_doubles) {
+ // All parameters are on the stack. r2 has the return value after call.
+
+ // If the expected number of arguments of the runtime function is
+ // constant, we check that the actual number of arguments match the
+ // expectation.
+ CHECK(f->nargs < 0 || f->nargs == num_arguments);
+
+ // TODO(1236192): Most runtime routines don't need the number of
+ // arguments passed in because it is constant. At some point we
+ // should remove this need and make the runtime routine entry code
+ // smarter.
+ mov(r2, Operand(num_arguments));
+ Move(r3, ExternalReference::Create(f));
+#if V8_TARGET_ARCH_S390X
+ Handle<Code> code =
+ CodeFactory::CEntry(isolate(), f->result_size, save_doubles);
+#else
+ Handle<Code> code = CodeFactory::CEntry(isolate(), 1, save_doubles);
+#endif
+
+ Call(code, RelocInfo::CODE_TARGET);
+}
+
+void MacroAssembler::TailCallRuntime(Runtime::FunctionId fid) {
+ const Runtime::Function* function = Runtime::FunctionForId(fid);
+ DCHECK_EQ(1, function->result_size);
+ if (function->nargs >= 0) {
+ mov(r2, Operand(function->nargs));
+ }
+ JumpToExternalReference(ExternalReference::Create(fid));
+}
+
+void MacroAssembler::JumpToExternalReference(const ExternalReference& builtin,
+ bool builtin_exit_frame) {
+ Move(r3, builtin);
+ Handle<Code> code = CodeFactory::CEntry(isolate(), 1, kDontSaveFPRegs,
+ kArgvOnStack, builtin_exit_frame);
+ Jump(code, RelocInfo::CODE_TARGET);
+}
+
+void MacroAssembler::JumpToInstructionStream(Address entry) {
+ mov(kOffHeapTrampolineRegister, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+ Jump(kOffHeapTrampolineRegister);
+}
+
+void MacroAssembler::LoadWeakValue(Register out, Register in,
+ Label* target_if_cleared) {
+ Cmp32(in, Operand(kClearedWeakHeapObjectLower32));
+ beq(target_if_cleared);
+
+ AndP(out, in, Operand(~kWeakHeapObjectMask));
+}
+
+void MacroAssembler::IncrementCounter(StatsCounter* counter, int value,
+ Register scratch1, Register scratch2) {
+ DCHECK(value > 0 && is_int8(value));
+ if (FLAG_native_code_counters && counter->Enabled()) {
+ Move(scratch2, ExternalReference::Create(counter));
+ // @TODO(john.yan): can be optimized by asi()
+ LoadW(scratch1, MemOperand(scratch2));
+ AddP(scratch1, Operand(value));
+ StoreW(scratch1, MemOperand(scratch2));
+ }
+}
+
+void MacroAssembler::DecrementCounter(StatsCounter* counter, int value,
+ Register scratch1, Register scratch2) {
+ DCHECK(value > 0 && is_int8(value));
+ if (FLAG_native_code_counters && counter->Enabled()) {
+ Move(scratch2, ExternalReference::Create(counter));
+ // @TODO(john.yan): can be optimized by asi()
+ LoadW(scratch1, MemOperand(scratch2));
+ AddP(scratch1, Operand(-value));
+ StoreW(scratch1, MemOperand(scratch2));
+ }
+}
+
+void TurboAssembler::Assert(Condition cond, AbortReason reason, CRegister cr) {
+ if (emit_debug_code()) Check(cond, reason, cr);
+}
+
+void TurboAssembler::Check(Condition cond, AbortReason reason, CRegister cr) {
+ Label L;
+ b(cond, &L);
+ Abort(reason);
+ // will not return here
+ bind(&L);
+}
+
+void TurboAssembler::Abort(AbortReason reason) {
+ Label abort_start;
+ bind(&abort_start);
+#ifdef DEBUG
+ const char* msg = GetAbortReason(reason);
+ RecordComment("Abort message: ");
+ RecordComment(msg);
+#endif
+
+ // Avoid emitting call to builtin if requested.
+ if (trap_on_abort()) {
+ stop();
+ return;
+ }
+
+ if (should_abort_hard()) {
+ // We don't care if we constructed a frame. Just pretend we did.
+ FrameScope assume_frame(this, StackFrame::NONE);
+ lgfi(r2, Operand(static_cast<int>(reason)));
+ PrepareCallCFunction(1, 0, r3);
+ Move(r3, ExternalReference::abort_with_reason());
+ // Use Call directly to avoid any unneeded overhead. The function won't
+ // return anyway.
+ Call(r3);
+ return;
+ }
+
+ LoadSmiLiteral(r3, Smi::FromInt(static_cast<int>(reason)));
+
+ // Disable stub call restrictions to always allow calls to abort.
+ if (!has_frame_) {
+ // We don't actually want to generate a pile of code for this, so just
+ // claim there is a stack frame, without generating one.
+ FrameScope scope(this, StackFrame::NONE);
+ Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
+ } else {
+ Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
+ }
+ // will not return here
+}
+
+void MacroAssembler::LoadMap(Register destination, Register object) {
+ LoadTaggedPointerField(destination,
+ FieldMemOperand(object, HeapObject::kMapOffset));
+}
+
+void MacroAssembler::LoadNativeContextSlot(int index, Register dst) {
+ LoadMap(dst, cp);
+ LoadTaggedPointerField(
+ dst, FieldMemOperand(
+ dst, Map::kConstructorOrBackPointerOrNativeContextOffset));
+ LoadTaggedPointerField(dst, MemOperand(dst, Context::SlotOffset(index)));
+}
+
+void MacroAssembler::AssertNotSmi(Register object) {
+ if (emit_debug_code()) {
+ STATIC_ASSERT(kSmiTag == 0);
+ TestIfSmi(object);
+ Check(ne, AbortReason::kOperandIsASmi, cr0);
+ }
+}
+
+void MacroAssembler::AssertSmi(Register object) {
+ if (emit_debug_code()) {
+ STATIC_ASSERT(kSmiTag == 0);
+ TestIfSmi(object);
+ Check(eq, AbortReason::kOperandIsNotASmi, cr0);
+ }
+}
+
+void MacroAssembler::AssertConstructor(Register object, Register scratch) {
+ if (emit_debug_code()) {
+ STATIC_ASSERT(kSmiTag == 0);
+ TestIfSmi(object);
+ Check(ne, AbortReason::kOperandIsASmiAndNotAConstructor);
+ LoadMap(scratch, object);
+ tm(FieldMemOperand(scratch, Map::kBitFieldOffset),
+ Operand(Map::Bits1::IsConstructorBit::kMask));
+ Check(ne, AbortReason::kOperandIsNotAConstructor);
+ }
+}
+
+void MacroAssembler::AssertFunction(Register object) {
+ if (emit_debug_code()) {
+ STATIC_ASSERT(kSmiTag == 0);
+ TestIfSmi(object);
+ Check(ne, AbortReason::kOperandIsASmiAndNotAFunction, cr0);
+ push(object);
+ CompareObjectType(object, object, object, JS_FUNCTION_TYPE);
+ pop(object);
+ Check(eq, AbortReason::kOperandIsNotAFunction);
+ }
+}
+
+void MacroAssembler::AssertBoundFunction(Register object) {
+ if (emit_debug_code()) {
+ STATIC_ASSERT(kSmiTag == 0);
+ TestIfSmi(object);
+ Check(ne, AbortReason::kOperandIsASmiAndNotABoundFunction, cr0);
+ push(object);
+ CompareObjectType(object, object, object, JS_BOUND_FUNCTION_TYPE);
+ pop(object);
+ Check(eq, AbortReason::kOperandIsNotABoundFunction);
+ }
+}
+
+void MacroAssembler::AssertGeneratorObject(Register object) {
+ if (!emit_debug_code()) return;
+ TestIfSmi(object);
+ Check(ne, AbortReason::kOperandIsASmiAndNotAGeneratorObject, cr0);
+
+ // Load map
+ Register map = object;
+ push(object);
+ LoadMap(map, object);
+
+ // Check if JSGeneratorObject
+ Label do_check;
+ Register instance_type = object;
+ CompareInstanceType(map, instance_type, JS_GENERATOR_OBJECT_TYPE);
+ beq(&do_check);
+
+ // Check if JSAsyncFunctionObject (See MacroAssembler::CompareInstanceType)
+ CmpP(instance_type, Operand(JS_ASYNC_FUNCTION_OBJECT_TYPE));
+ beq(&do_check);
+
+ // Check if JSAsyncGeneratorObject (See MacroAssembler::CompareInstanceType)
+ CmpP(instance_type, Operand(JS_ASYNC_GENERATOR_OBJECT_TYPE));
+
+ bind(&do_check);
+ // Restore generator object to register and perform assertion
+ pop(object);
+ Check(eq, AbortReason::kOperandIsNotAGeneratorObject);
+}
+
+void MacroAssembler::AssertUndefinedOrAllocationSite(Register object,
+ Register scratch) {
+ if (emit_debug_code()) {
+ Label done_checking;
+ AssertNotSmi(object);
+ CompareRoot(object, RootIndex::kUndefinedValue);
+ beq(&done_checking, Label::kNear);
+ LoadMap(scratch, object);
+ CompareInstanceType(scratch, scratch, ALLOCATION_SITE_TYPE);
+ Assert(eq, AbortReason::kExpectedUndefinedOrCell);
+ bind(&done_checking);
+ }
+}
+
+static const int kRegisterPassedArguments = 5;
+
+int TurboAssembler::CalculateStackPassedWords(int num_reg_arguments,
+ int num_double_arguments) {
+ int stack_passed_words = 0;
+ if (num_double_arguments > DoubleRegister::kNumRegisters) {
+ stack_passed_words +=
+ 2 * (num_double_arguments - DoubleRegister::kNumRegisters);
+ }
+ // Up to five simple arguments are passed in registers r2..r6
+ if (num_reg_arguments > kRegisterPassedArguments) {
+ stack_passed_words += num_reg_arguments - kRegisterPassedArguments;
+ }
+ return stack_passed_words;
+}
+
+void TurboAssembler::PrepareCallCFunction(int num_reg_arguments,
+ int num_double_arguments,
+ Register scratch) {
+ int frame_alignment = ActivationFrameAlignment();
+ int stack_passed_arguments =
+ CalculateStackPassedWords(num_reg_arguments, num_double_arguments);
+ int stack_space = kNumRequiredStackFrameSlots;
+ if (frame_alignment > kSystemPointerSize) {
+ // Make stack end at alignment and make room for stack arguments
+ // -- preserving original value of sp.
+ LoadRR(scratch, sp);
+ lay(sp, MemOperand(sp, -(stack_passed_arguments + 1) * kSystemPointerSize));
+ DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
+ ClearRightImm(sp, sp,
+ Operand(base::bits::WhichPowerOfTwo(frame_alignment)));
+ StoreP(scratch,
+ MemOperand(sp, (stack_passed_arguments)*kSystemPointerSize));
+ } else {
+ stack_space += stack_passed_arguments;
+ }
+ lay(sp, MemOperand(sp, (-stack_space) * kSystemPointerSize));
+}
+
+void TurboAssembler::PrepareCallCFunction(int num_reg_arguments,
+ Register scratch) {
+ PrepareCallCFunction(num_reg_arguments, 0, scratch);
+}
+
+void TurboAssembler::MovToFloatParameter(DoubleRegister src) { Move(d0, src); }
+
+void TurboAssembler::MovToFloatResult(DoubleRegister src) { Move(d0, src); }
+
+void TurboAssembler::MovToFloatParameters(DoubleRegister src1,
+ DoubleRegister src2) {
+ if (src2 == d0) {
+ DCHECK(src1 != d2);
+ Move(d2, src2);
+ Move(d0, src1);
+ } else {
+ Move(d0, src1);
+ Move(d2, src2);
+ }
+}
+
+void TurboAssembler::CallCFunction(ExternalReference function,
+ int num_reg_arguments,
+ int num_double_arguments) {
+ Move(ip, function);
+ CallCFunctionHelper(ip, num_reg_arguments, num_double_arguments);
+}
+
+void TurboAssembler::CallCFunction(Register function, int num_reg_arguments,
+ int num_double_arguments) {
+ CallCFunctionHelper(function, num_reg_arguments, num_double_arguments);
+}
+
+void TurboAssembler::CallCFunction(ExternalReference function,
+ int num_arguments) {
+ CallCFunction(function, num_arguments, 0);
+}
+
+void TurboAssembler::CallCFunction(Register function, int num_arguments) {
+ CallCFunction(function, num_arguments, 0);
+}
+
+void TurboAssembler::CallCFunctionHelper(Register function,
+ int num_reg_arguments,
+ int num_double_arguments) {
+ DCHECK_LE(num_reg_arguments + num_double_arguments, kMaxCParameters);
+ DCHECK(has_frame());
+
+ // Save the frame pointer and PC so that the stack layout remains iterable,
+ // even without an ExitFrame which normally exists between JS and C frames.
+ Register addr_scratch = r1;
+ // See x64 code for reasoning about how to address the isolate data fields.
+ if (root_array_available()) {
+ LoadPC(r0);
+ StoreP(r0, MemOperand(kRootRegister,
+ IsolateData::fast_c_call_caller_pc_offset()));
+ StoreP(fp, MemOperand(kRootRegister,
+ IsolateData::fast_c_call_caller_fp_offset()));
+ } else {
+ DCHECK_NOT_NULL(isolate());
+
+ Move(addr_scratch,
+ ExternalReference::fast_c_call_caller_pc_address(isolate()));
+ LoadPC(r0);
+ StoreP(r0, MemOperand(addr_scratch));
+ Move(addr_scratch,
+ ExternalReference::fast_c_call_caller_fp_address(isolate()));
+ StoreP(fp, MemOperand(addr_scratch));
+ }
+
+ // Just call directly. The function called cannot cause a GC, or
+ // allow preemption, so the return address in the link register
+ // stays correct.
+ Register dest = function;
+ if (ABI_CALL_VIA_IP) {
+ Move(ip, function);
+ dest = ip;
+ }
+
+ Call(dest);
+
+ // We don't unset the PC; the FP is the source of truth.
+ Register zero_scratch = r0;
+ lghi(zero_scratch, Operand::Zero());
+
+ if (root_array_available()) {
+ StoreP(
+ zero_scratch,
+ MemOperand(kRootRegister, IsolateData::fast_c_call_caller_fp_offset()));
+ } else {
+ DCHECK_NOT_NULL(isolate());
+ Move(addr_scratch,
+ ExternalReference::fast_c_call_caller_fp_address(isolate()));
+ StoreP(zero_scratch, MemOperand(addr_scratch));
+ }
+
+ int stack_passed_arguments =
+ CalculateStackPassedWords(num_reg_arguments, num_double_arguments);
+ int stack_space = kNumRequiredStackFrameSlots + stack_passed_arguments;
+ if (ActivationFrameAlignment() > kSystemPointerSize) {
+ // Load the original stack pointer (pre-alignment) from the stack
+ LoadP(sp, MemOperand(sp, stack_space * kSystemPointerSize));
+ } else {
+ la(sp, MemOperand(sp, stack_space * kSystemPointerSize));
+ }
+}
+
+void TurboAssembler::CheckPageFlag(
+ Register object,
+ Register scratch, // scratch may be same register as object
+ int mask, Condition cc, Label* condition_met) {
+ DCHECK(cc == ne || cc == eq);
+ ClearRightImm(scratch, object, Operand(kPageSizeBits));
+
+ if (base::bits::IsPowerOfTwo(mask)) {
+ // If it's a power of two, we can use Test-Under-Mask Memory-Imm form
+ // which allows testing of a single byte in memory.
+ int32_t byte_offset = 4;
+ uint32_t shifted_mask = mask;
+ // Determine the byte offset to be tested
+ if (mask <= 0x80) {
+ byte_offset = kSystemPointerSize - 1;
+ } else if (mask < 0x8000) {
+ byte_offset = kSystemPointerSize - 2;
+ shifted_mask = mask >> 8;
+ } else if (mask < 0x800000) {
+ byte_offset = kSystemPointerSize - 3;
+ shifted_mask = mask >> 16;
+ } else {
+ byte_offset = kSystemPointerSize - 4;
+ shifted_mask = mask >> 24;
+ }
+#if V8_TARGET_LITTLE_ENDIAN
+ // Reverse the byte_offset if emulating on little endian platform
+ byte_offset = kSystemPointerSize - byte_offset - 1;
+#endif
+ tm(MemOperand(scratch, BasicMemoryChunk::kFlagsOffset + byte_offset),
+ Operand(shifted_mask));
+ } else {
+ LoadP(scratch, MemOperand(scratch, BasicMemoryChunk::kFlagsOffset));
+ AndP(r0, scratch, Operand(mask));
+ }
+ // Should be okay to remove rc
+
+ if (cc == ne) {
+ bne(condition_met);
+ }
+ if (cc == eq) {
+ beq(condition_met);
+ }
+}
+
+Register GetRegisterThatIsNotOneOf(Register reg1, Register reg2, Register reg3,
+ Register reg4, Register reg5,
+ Register reg6) {
+ RegList regs = 0;
+ if (reg1.is_valid()) regs |= reg1.bit();
+ if (reg2.is_valid()) regs |= reg2.bit();
+ if (reg3.is_valid()) regs |= reg3.bit();
+ if (reg4.is_valid()) regs |= reg4.bit();
+ if (reg5.is_valid()) regs |= reg5.bit();
+ if (reg6.is_valid()) regs |= reg6.bit();
+
+ const RegisterConfiguration* config = RegisterConfiguration::Default();
+ for (int i = 0; i < config->num_allocatable_general_registers(); ++i) {
+ int code = config->GetAllocatableGeneralCode(i);
+ Register candidate = Register::from_code(code);
+ if (regs & candidate.bit()) continue;
+ return candidate;
+ }
+ UNREACHABLE();
+}
+
+void TurboAssembler::mov(Register dst, const Operand& src) {
+#if V8_TARGET_ARCH_S390X
+ int64_t value;
+#else
+ int value;
+#endif
+ if (src.is_heap_object_request()) {
+ RequestHeapObject(src.heap_object_request());
+ value = 0;
+ } else {
+ value = src.immediate();
+ }
+
+ if (src.rmode() != RelocInfo::NONE) {
+ // some form of relocation needed
+ RecordRelocInfo(src.rmode(), value);
+ }
+
+#if V8_TARGET_ARCH_S390X
+ int32_t hi_32 = static_cast<int64_t>(value) >> 32;
+ int32_t lo_32 = static_cast<int32_t>(value);
+
+ iihf(dst, Operand(hi_32));
+ iilf(dst, Operand(lo_32));
+#else
+ iilf(dst, Operand(value));
+#endif
+}
+
+void TurboAssembler::Mul32(Register dst, const MemOperand& src1) {
+ if (is_uint12(src1.offset())) {
+ ms(dst, src1);
+ } else if (is_int20(src1.offset())) {
+ msy(dst, src1);
+ } else {
+ UNIMPLEMENTED();
+ }
+}
+
+void TurboAssembler::Mul32(Register dst, Register src1) { msr(dst, src1); }
+
+void TurboAssembler::Mul32(Register dst, const Operand& src1) {
+ msfi(dst, src1);
+}
+
+#define Generate_MulHigh32(instr) \
+ { \
+ lgfr(dst, src1); \
+ instr(dst, src2); \
+ srlg(dst, dst, Operand(32)); \
+ }
+
+void TurboAssembler::MulHigh32(Register dst, Register src1,
+ const MemOperand& src2) {
+ Generate_MulHigh32(msgf);
+}
+
+void TurboAssembler::MulHigh32(Register dst, Register src1, Register src2) {
+ if (dst == src2) {
+ std::swap(src1, src2);
+ }
+ Generate_MulHigh32(msgfr);
+}
+
+void TurboAssembler::MulHigh32(Register dst, Register src1,
+ const Operand& src2) {
+ Generate_MulHigh32(msgfi);
+}
+
+#undef Generate_MulHigh32
+
+#define Generate_MulHighU32(instr) \
+ { \
+ lr(r1, src1); \
+ instr(r0, src2); \
+ LoadlW(dst, r0); \
+ }
+
+void TurboAssembler::MulHighU32(Register dst, Register src1,
+ const MemOperand& src2) {
+ Generate_MulHighU32(ml);
+}
+
+void TurboAssembler::MulHighU32(Register dst, Register src1, Register src2) {
+ Generate_MulHighU32(mlr);
+}
+
+void TurboAssembler::MulHighU32(Register dst, Register src1,
+ const Operand& src2) {
+ USE(dst);
+ USE(src1);
+ USE(src2);
+ UNREACHABLE();
+}
+
+#undef Generate_MulHighU32
+
+#define Generate_Mul32WithOverflowIfCCUnequal(instr) \
+ { \
+ lgfr(dst, src1); \
+ instr(dst, src2); \
+ cgfr(dst, dst); \
+ }
+
+void TurboAssembler::Mul32WithOverflowIfCCUnequal(Register dst, Register src1,
+ const MemOperand& src2) {
+ Register result = dst;
+ if (src2.rx() == dst || src2.rb() == dst) dst = r0;
+ Generate_Mul32WithOverflowIfCCUnequal(msgf);
+ if (result != dst) llgfr(result, dst);
+}
+
+void TurboAssembler::Mul32WithOverflowIfCCUnequal(Register dst, Register src1,
+ Register src2) {
+ if (dst == src2) {
+ std::swap(src1, src2);
+ }
+ Generate_Mul32WithOverflowIfCCUnequal(msgfr);
+}
+
+void TurboAssembler::Mul32WithOverflowIfCCUnequal(Register dst, Register src1,
+ const Operand& src2) {
+ Generate_Mul32WithOverflowIfCCUnequal(msgfi);
+}
+
+#undef Generate_Mul32WithOverflowIfCCUnequal
+
+void TurboAssembler::Mul64(Register dst, const MemOperand& src1) {
+ if (is_int20(src1.offset())) {
+ msg(dst, src1);
+ } else {
+ UNIMPLEMENTED();
+ }
+}
+
+void TurboAssembler::Mul64(Register dst, Register src1) { msgr(dst, src1); }
+
+void TurboAssembler::Mul64(Register dst, const Operand& src1) {
+ msgfi(dst, src1);
+}
+
+void TurboAssembler::Mul(Register dst, Register src1, Register src2) {
+ if (CpuFeatures::IsSupported(MISC_INSTR_EXT2)) {
+ MulPWithCondition(dst, src1, src2);
+ } else {
+ if (dst == src2) {
+ MulP(dst, src1);
+ } else if (dst == src1) {
+ MulP(dst, src2);
+ } else {
+ Move(dst, src1);
+ MulP(dst, src2);
+ }
+ }
+}
+
+void TurboAssembler::DivP(Register dividend, Register divider) {
+ // have to make sure the src and dst are reg pairs
+ DCHECK_EQ(dividend.code() % 2, 0);
+#if V8_TARGET_ARCH_S390X
+ dsgr(dividend, divider);
+#else
+ dr(dividend, divider);
+#endif
+}
+
+#define Generate_Div32(instr) \
+ { \
+ lgfr(r1, src1); \
+ instr(r0, src2); \
+ LoadlW(dst, r1); \
+ }
+
+void TurboAssembler::Div32(Register dst, Register src1,
+ const MemOperand& src2) {
+ Generate_Div32(dsgf);
+}
+
+void TurboAssembler::Div32(Register dst, Register src1, Register src2) {
+ Generate_Div32(dsgfr);
+}
+
+#undef Generate_Div32
+
+#define Generate_DivU32(instr) \
+ { \
+ lr(r0, src1); \
+ srdl(r0, Operand(32)); \
+ instr(r0, src2); \
+ LoadlW(dst, r1); \
+ }
+
+void TurboAssembler::DivU32(Register dst, Register src1,
+ const MemOperand& src2) {
+ Generate_DivU32(dl);
+}
+
+void TurboAssembler::DivU32(Register dst, Register src1, Register src2) {
+ Generate_DivU32(dlr);
+}
+
+#undef Generate_DivU32
+
+#define Generate_Div64(instr) \
+ { \
+ lgr(r1, src1); \
+ instr(r0, src2); \
+ lgr(dst, r1); \
+ }
+
+void TurboAssembler::Div64(Register dst, Register src1,
+ const MemOperand& src2) {
+ Generate_Div64(dsg);
+}
+
+void TurboAssembler::Div64(Register dst, Register src1, Register src2) {
+ Generate_Div64(dsgr);
+}
+
+#undef Generate_Div64
+
+#define Generate_DivU64(instr) \
+ { \
+ lgr(r1, src1); \
+ lghi(r0, Operand::Zero()); \
+ instr(r0, src2); \
+ lgr(dst, r1); \
+ }
+
+void TurboAssembler::DivU64(Register dst, Register src1,
+ const MemOperand& src2) {
+ Generate_DivU64(dlg);
+}
+
+void TurboAssembler::DivU64(Register dst, Register src1, Register src2) {
+ Generate_DivU64(dlgr);
+}
+
+#undef Generate_DivU64
+
+#define Generate_Mod32(instr) \
+ { \
+ lgfr(r1, src1); \
+ instr(r0, src2); \
+ LoadlW(dst, r0); \
+ }
+
+void TurboAssembler::Mod32(Register dst, Register src1,
+ const MemOperand& src2) {
+ Generate_Mod32(dsgf);
+}
+
+void TurboAssembler::Mod32(Register dst, Register src1, Register src2) {
+ Generate_Mod32(dsgfr);
+}
+
+#undef Generate_Mod32
+
+#define Generate_ModU32(instr) \
+ { \
+ lr(r0, src1); \
+ srdl(r0, Operand(32)); \
+ instr(r0, src2); \
+ LoadlW(dst, r0); \
+ }
+
+void TurboAssembler::ModU32(Register dst, Register src1,
+ const MemOperand& src2) {
+ Generate_ModU32(dl);
+}
+
+void TurboAssembler::ModU32(Register dst, Register src1, Register src2) {
+ Generate_ModU32(dlr);
+}
+
+#undef Generate_ModU32
+
+#define Generate_Mod64(instr) \
+ { \
+ lgr(r1, src1); \
+ instr(r0, src2); \
+ lgr(dst, r0); \
+ }
+
+void TurboAssembler::Mod64(Register dst, Register src1,
+ const MemOperand& src2) {
+ Generate_Mod64(dsg);
+}
+
+void TurboAssembler::Mod64(Register dst, Register src1, Register src2) {
+ Generate_Mod64(dsgr);
+}
+
+#undef Generate_Mod64
+
+#define Generate_ModU64(instr) \
+ { \
+ lgr(r1, src1); \
+ lghi(r0, Operand::Zero()); \
+ instr(r0, src2); \
+ lgr(dst, r0); \
+ }
+
+void TurboAssembler::ModU64(Register dst, Register src1,
+ const MemOperand& src2) {
+ Generate_ModU64(dlg);
+}
+
+void TurboAssembler::ModU64(Register dst, Register src1, Register src2) {
+ Generate_ModU64(dlgr);
+}
+
+#undef Generate_ModU64
+
+void TurboAssembler::MulP(Register dst, const Operand& opnd) {
+#if V8_TARGET_ARCH_S390X
+ msgfi(dst, opnd);
+#else
+ msfi(dst, opnd);
+#endif
+}
+
+void TurboAssembler::MulP(Register dst, Register src) {
+#if V8_TARGET_ARCH_S390X
+ msgr(dst, src);
+#else
+ msr(dst, src);
+#endif
+}
+
+void TurboAssembler::MulPWithCondition(Register dst, Register src1,
+ Register src2) {
+ CHECK(CpuFeatures::IsSupported(MISC_INSTR_EXT2));
+#if V8_TARGET_ARCH_S390X
+ msgrkc(dst, src1, src2);
+#else
+ msrkc(dst, src1, src2);
+#endif
+}
+
+void TurboAssembler::MulP(Register dst, const MemOperand& opnd) {
+#if V8_TARGET_ARCH_S390X
+ if (is_uint16(opnd.offset())) {
+ ms(dst, opnd);
+ } else if (is_int20(opnd.offset())) {
+ msy(dst, opnd);
+ } else {
+ UNIMPLEMENTED();
+ }
+#else
+ if (is_int20(opnd.offset())) {
+ msg(dst, opnd);
+ } else {
+ UNIMPLEMENTED();
+ }
+#endif
+}
+
+void TurboAssembler::Sqrt(DoubleRegister result, DoubleRegister input) {
+ sqdbr(result, input);
+}
+void TurboAssembler::Sqrt(DoubleRegister result, const MemOperand& input) {
+ if (is_uint12(input.offset())) {
+ sqdb(result, input);
+ } else {
+ ldy(result, input);
+ sqdbr(result, result);
+ }
+}
+//----------------------------------------------------------------------------
+// Add Instructions
+//----------------------------------------------------------------------------
+
+// Add 32-bit (Register dst = Register dst + Immediate opnd)
+void TurboAssembler::Add32(Register dst, const Operand& opnd) {
+ if (is_int16(opnd.immediate()))
+ ahi(dst, opnd);
+ else
+ afi(dst, opnd);
+}
+
+// Add 32-bit (Register dst = Register dst + Immediate opnd)
+void TurboAssembler::Add32_RI(Register dst, const Operand& opnd) {
+ // Just a wrapper for above
+ Add32(dst, opnd);
+}
+
+// Add Pointer Size (Register dst = Register dst + Immediate opnd)
+void TurboAssembler::AddP(Register dst, const Operand& opnd) {
+#if V8_TARGET_ARCH_S390X
+ if (is_int16(opnd.immediate()))
+ aghi(dst, opnd);
+ else
+ agfi(dst, opnd);
+#else
+ Add32(dst, opnd);
+#endif
+}
+
+// Add 32-bit (Register dst = Register src + Immediate opnd)
+void TurboAssembler::Add32(Register dst, Register src, const Operand& opnd) {
+ if (dst != src) {
+ if (CpuFeatures::IsSupported(DISTINCT_OPS) && is_int16(opnd.immediate())) {
+ ahik(dst, src, opnd);
+ return;
+ }
+ lr(dst, src);
+ }
+ Add32(dst, opnd);
+}
+
+// Add 32-bit (Register dst = Register src + Immediate opnd)
+void TurboAssembler::Add32_RRI(Register dst, Register src,
+ const Operand& opnd) {
+ // Just a wrapper for above
+ Add32(dst, src, opnd);
+}
+
+// Add Pointer Size (Register dst = Register src + Immediate opnd)
+void TurboAssembler::AddP(Register dst, Register src, const Operand& opnd) {
+ if (dst != src) {
+ if (CpuFeatures::IsSupported(DISTINCT_OPS) && is_int16(opnd.immediate())) {
+ AddPImm_RRI(dst, src, opnd);
+ return;
+ }
+ LoadRR(dst, src);
+ }
+ AddP(dst, opnd);
+}
+
+// Add 32-bit (Register dst = Register dst + Register src)
+void TurboAssembler::Add32(Register dst, Register src) { ar(dst, src); }
+
+// Add Pointer Size (Register dst = Register dst + Register src)
+void TurboAssembler::AddP(Register dst, Register src) { AddRR(dst, src); }
+
+// Add Pointer Size with src extension
+// (Register dst(ptr) = Register dst (ptr) + Register src (32 | 32->64))
+// src is treated as a 32-bit signed integer, which is sign extended to
+// 64-bit if necessary.
+void TurboAssembler::AddP_ExtendSrc(Register dst, Register src) {
+#if V8_TARGET_ARCH_S390X
+ agfr(dst, src);
+#else
+ ar(dst, src);
+#endif
+}
+
+// Add 32-bit (Register dst = Register src1 + Register src2)
+void TurboAssembler::Add32(Register dst, Register src1, Register src2) {
+ if (dst != src1 && dst != src2) {
+ // We prefer to generate AR/AGR, over the non clobbering ARK/AGRK
+ // as AR is a smaller instruction
+ if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
+ ark(dst, src1, src2);
+ return;
+ } else {
+ lr(dst, src1);
+ }
+ } else if (dst == src2) {
+ src2 = src1;
+ }
+ ar(dst, src2);
+}
+
+// Add Pointer Size (Register dst = Register src1 + Register src2)
+void TurboAssembler::AddP(Register dst, Register src1, Register src2) {
+ if (dst != src1 && dst != src2) {
+ // We prefer to generate AR/AGR, over the non clobbering ARK/AGRK
+ // as AR is a smaller instruction
+ if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
+ AddP_RRR(dst, src1, src2);
+ return;
+ } else {
+ LoadRR(dst, src1);
+ }
+ } else if (dst == src2) {
+ src2 = src1;
+ }
+ AddRR(dst, src2);
+}
+
+// Add Pointer Size with src extension
+// (Register dst (ptr) = Register dst (ptr) + Register src1 (ptr) +
+// Register src2 (32 | 32->64))
+// src is treated as a 32-bit signed integer, which is sign extended to
+// 64-bit if necessary.
+void TurboAssembler::AddP_ExtendSrc(Register dst, Register src1,
+ Register src2) {
+#if V8_TARGET_ARCH_S390X
+ if (dst == src2) {
+ // The source we need to sign extend is the same as result.
+ lgfr(dst, src2);
+ agr(dst, src1);
+ } else {
+ if (dst != src1) LoadRR(dst, src1);
+ agfr(dst, src2);
+ }
+#else
+ AddP(dst, src1, src2);
+#endif
+}
+
+// Add 32-bit (Register-Memory)
+void TurboAssembler::Add32(Register dst, const MemOperand& opnd) {
+ DCHECK(is_int20(opnd.offset()));
+ if (is_uint12(opnd.offset()))
+ a(dst, opnd);
+ else
+ ay(dst, opnd);
+}
+
+// Add Pointer Size (Register-Memory)
+void TurboAssembler::AddP(Register dst, const MemOperand& opnd) {
+#if V8_TARGET_ARCH_S390X
+ DCHECK(is_int20(opnd.offset()));
+ ag(dst, opnd);
+#else
+ Add32(dst, opnd);
+#endif
+}
+
+// Add Pointer Size with src extension
+// (Register dst (ptr) = Register dst (ptr) + Mem opnd (32 | 32->64))
+// src is treated as a 32-bit signed integer, which is sign extended to
+// 64-bit if necessary.
+void TurboAssembler::AddP_ExtendSrc(Register dst, const MemOperand& opnd) {
+#if V8_TARGET_ARCH_S390X
+ DCHECK(is_int20(opnd.offset()));
+ agf(dst, opnd);
+#else
+ Add32(dst, opnd);
+#endif
+}
+
+// Add 32-bit (Memory - Immediate)
+void TurboAssembler::Add32(const MemOperand& opnd, const Operand& imm) {
+ DCHECK(is_int8(imm.immediate()));
+ DCHECK(is_int20(opnd.offset()));
+ DCHECK(CpuFeatures::IsSupported(GENERAL_INSTR_EXT));
+ asi(opnd, imm);
+}
+
+// Add Pointer-sized (Memory - Immediate)
+void TurboAssembler::AddP(const MemOperand& opnd, const Operand& imm) {
+ DCHECK(is_int8(imm.immediate()));
+ DCHECK(is_int20(opnd.offset()));
+ DCHECK(CpuFeatures::IsSupported(GENERAL_INSTR_EXT));
+#if V8_TARGET_ARCH_S390X
+ agsi(opnd, imm);
+#else
+ asi(opnd, imm);
+#endif
+}
+
+//----------------------------------------------------------------------------
+// Add Logical Instructions
+//----------------------------------------------------------------------------
+
+// Add Logical With Carry 32-bit (Register dst = Register src1 + Register src2)
+void TurboAssembler::AddLogicalWithCarry32(Register dst, Register src1,
+ Register src2) {
+ if (dst != src2 && dst != src1) {
+ lr(dst, src1);
+ alcr(dst, src2);
+ } else if (dst != src2) {
+ // dst == src1
+ DCHECK(dst == src1);
+ alcr(dst, src2);
+ } else {
+ // dst == src2
+ DCHECK(dst == src2);
+ alcr(dst, src1);
+ }
+}
+
+// Add Logical 32-bit (Register dst = Register src1 + Register src2)
+void TurboAssembler::AddLogical32(Register dst, Register src1, Register src2) {
+ if (dst != src2 && dst != src1) {
+ lr(dst, src1);
+ alr(dst, src2);
+ } else if (dst != src2) {
+ // dst == src1
+ DCHECK(dst == src1);
+ alr(dst, src2);
+ } else {
+ // dst == src2
+ DCHECK(dst == src2);
+ alr(dst, src1);
+ }
+}
+
+// Add Logical 32-bit (Register dst = Register dst + Immediate opnd)
+void TurboAssembler::AddLogical(Register dst, const Operand& imm) {
+ alfi(dst, imm);
+}
+
+// Add Logical Pointer Size (Register dst = Register dst + Immediate opnd)
+void TurboAssembler::AddLogicalP(Register dst, const Operand& imm) {
+#ifdef V8_TARGET_ARCH_S390X
+ algfi(dst, imm);
+#else
+ AddLogical(dst, imm);
+#endif
+}
+
+// Add Logical 32-bit (Register-Memory)
+void TurboAssembler::AddLogical(Register dst, const MemOperand& opnd) {
+ DCHECK(is_int20(opnd.offset()));
+ if (is_uint12(opnd.offset()))
+ al_z(dst, opnd);
+ else
+ aly(dst, opnd);
+}
+
+// Add Logical Pointer Size (Register-Memory)
+void TurboAssembler::AddLogicalP(Register dst, const MemOperand& opnd) {
+#if V8_TARGET_ARCH_S390X
+ DCHECK(is_int20(opnd.offset()));
+ alg(dst, opnd);
+#else
+ AddLogical(dst, opnd);
+#endif
+}
+
+//----------------------------------------------------------------------------
+// Subtract Instructions
+//----------------------------------------------------------------------------
+
+// Subtract Logical With Carry 32-bit (Register dst = Register src1 - Register
+// src2)
+void TurboAssembler::SubLogicalWithBorrow32(Register dst, Register src1,
+ Register src2) {
+ if (dst != src2 && dst != src1) {
+ lr(dst, src1);
+ slbr(dst, src2);
+ } else if (dst != src2) {
+ // dst == src1
+ DCHECK(dst == src1);
+ slbr(dst, src2);
+ } else {
+ // dst == src2
+ DCHECK(dst == src2);
+ lr(r0, dst);
+ SubLogicalWithBorrow32(dst, src1, r0);
+ }
+}
+
+// Subtract Logical 32-bit (Register dst = Register src1 - Register src2)
+void TurboAssembler::SubLogical32(Register dst, Register src1, Register src2) {
+ if (dst != src2 && dst != src1) {
+ lr(dst, src1);
+ slr(dst, src2);
+ } else if (dst != src2) {
+ // dst == src1
+ DCHECK(dst == src1);
+ slr(dst, src2);
+ } else {
+ // dst == src2
+ DCHECK(dst == src2);
+ lr(r0, dst);
+ SubLogical32(dst, src1, r0);
+ }
+}
+
+// Subtract 32-bit (Register dst = Register dst - Immediate opnd)
+void TurboAssembler::Sub32(Register dst, const Operand& imm) {
+ Add32(dst, Operand(-(imm.immediate())));
+}
+
+// Subtract Pointer Size (Register dst = Register dst - Immediate opnd)
+void TurboAssembler::SubP(Register dst, const Operand& imm) {
+ AddP(dst, Operand(-(imm.immediate())));
+}
+
+// Subtract 32-bit (Register dst = Register src - Immediate opnd)
+void TurboAssembler::Sub32(Register dst, Register src, const Operand& imm) {
+ Add32(dst, src, Operand(-(imm.immediate())));
+}
+
+// Subtract Pointer Sized (Register dst = Register src - Immediate opnd)
+void TurboAssembler::SubP(Register dst, Register src, const Operand& imm) {
+ AddP(dst, src, Operand(-(imm.immediate())));
+}
+
+// Subtract 32-bit (Register dst = Register dst - Register src)
+void TurboAssembler::Sub32(Register dst, Register src) { sr(dst, src); }
+
+// Subtract Pointer Size (Register dst = Register dst - Register src)
+void TurboAssembler::SubP(Register dst, Register src) { SubRR(dst, src); }
+
+// Subtract Pointer Size with src extension
+// (Register dst(ptr) = Register dst (ptr) - Register src (32 | 32->64))
+// src is treated as a 32-bit signed integer, which is sign extended to
+// 64-bit if necessary.
+void TurboAssembler::SubP_ExtendSrc(Register dst, Register src) {
+#if V8_TARGET_ARCH_S390X
+ sgfr(dst, src);
+#else
+ sr(dst, src);
+#endif
+}
+
+// Subtract 32-bit (Register = Register - Register)
+void TurboAssembler::Sub32(Register dst, Register src1, Register src2) {
+ // Use non-clobbering version if possible
+ if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
+ srk(dst, src1, src2);
+ return;
+ }
+ if (dst != src1 && dst != src2) lr(dst, src1);
+ // In scenario where we have dst = src - dst, we need to swap and negate
+ if (dst != src1 && dst == src2) {
+ Label done;
+ lcr(dst, dst); // dst = -dst
+ b(overflow, &done);
+ ar(dst, src1); // dst = dst + src
+ bind(&done);
+ } else {
+ sr(dst, src2);
+ }
+}
+
+// Subtract Pointer Sized (Register = Register - Register)
+void TurboAssembler::SubP(Register dst, Register src1, Register src2) {
+ // Use non-clobbering version if possible
+ if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
+ SubP_RRR(dst, src1, src2);
+ return;
+ }
+ if (dst != src1 && dst != src2) LoadRR(dst, src1);
+ // In scenario where we have dst = src - dst, we need to swap and negate
+ if (dst != src1 && dst == src2) {
+ Label done;
+ LoadComplementRR(dst, dst); // dst = -dst
+ b(overflow, &done);
+ AddP(dst, src1); // dst = dst + src
+ bind(&done);
+ } else {
+ SubP(dst, src2);
+ }
+}
+
+// Subtract Pointer Size with src extension
+// (Register dst(ptr) = Register dst (ptr) - Register src (32 | 32->64))
+// src is treated as a 32-bit signed integer, which is sign extended to
+// 64-bit if necessary.
+void TurboAssembler::SubP_ExtendSrc(Register dst, Register src1,
+ Register src2) {
+#if V8_TARGET_ARCH_S390X
+ if (dst != src1 && dst != src2) LoadRR(dst, src1);
+
+ // In scenario where we have dst = src - dst, we need to swap and negate
+ if (dst != src1 && dst == src2) {
+ lgfr(dst, dst); // Sign extend this operand first.
+ LoadComplementRR(dst, dst); // dst = -dst
+ AddP(dst, src1); // dst = -dst + src
+ } else {
+ sgfr(dst, src2);
+ }
+#else
+ SubP(dst, src1, src2);
+#endif
+}
+
+// Subtract 32-bit (Register-Memory)
+void TurboAssembler::Sub32(Register dst, const MemOperand& opnd) {
+ DCHECK(is_int20(opnd.offset()));
+ if (is_uint12(opnd.offset()))
+ s(dst, opnd);
+ else
+ sy(dst, opnd);
+}
+
+// Subtract Pointer Sized (Register - Memory)
+void TurboAssembler::SubP(Register dst, const MemOperand& opnd) {
+#if V8_TARGET_ARCH_S390X
+ sg(dst, opnd);
+#else
+ Sub32(dst, opnd);
+#endif
+}
+
+void TurboAssembler::MovIntToFloat(DoubleRegister dst, Register src) {
+ sllg(r0, src, Operand(32));
+ ldgr(dst, r0);
+}
+
+void TurboAssembler::MovFloatToInt(Register dst, DoubleRegister src) {
+ lgdr(dst, src);
+ srlg(dst, dst, Operand(32));
+}
+
+void TurboAssembler::SubP_ExtendSrc(Register dst, const MemOperand& opnd) {
+#if V8_TARGET_ARCH_S390X
+ DCHECK(is_int20(opnd.offset()));
+ sgf(dst, opnd);
+#else
+ Sub32(dst, opnd);
+#endif
+}
+
+// Load And Subtract 32-bit (similar to laa/lan/lao/lax)
+void TurboAssembler::LoadAndSub32(Register dst, Register src,
+ const MemOperand& opnd) {
+ lcr(dst, src);
+ laa(dst, dst, opnd);
+}
+
+void TurboAssembler::LoadAndSub64(Register dst, Register src,
+ const MemOperand& opnd) {
+ lcgr(dst, src);
+ laag(dst, dst, opnd);
+}
+
+//----------------------------------------------------------------------------
+// Subtract Logical Instructions
+//----------------------------------------------------------------------------
+
+// Subtract Logical 32-bit (Register - Memory)
+void TurboAssembler::SubLogical(Register dst, const MemOperand& opnd) {
+ DCHECK(is_int20(opnd.offset()));
+ if (is_uint12(opnd.offset()))
+ sl(dst, opnd);
+ else
+ sly(dst, opnd);
+}
+
+// Subtract Logical Pointer Sized (Register - Memory)
+void TurboAssembler::SubLogicalP(Register dst, const MemOperand& opnd) {
+ DCHECK(is_int20(opnd.offset()));
+#if V8_TARGET_ARCH_S390X
+ slgf(dst, opnd);
+#else
+ SubLogical(dst, opnd);
+#endif
+}
+
+// Subtract Logical Pointer Size with src extension
+// (Register dst (ptr) = Register dst (ptr) - Mem opnd (32 | 32->64))
+// src is treated as a 32-bit signed integer, which is sign extended to
+// 64-bit if necessary.
+void TurboAssembler::SubLogicalP_ExtendSrc(Register dst,
+ const MemOperand& opnd) {
+#if V8_TARGET_ARCH_S390X
+ DCHECK(is_int20(opnd.offset()));
+ slgf(dst, opnd);
+#else
+ SubLogical(dst, opnd);
+#endif
+}
+
+//----------------------------------------------------------------------------
+// Bitwise Operations
+//----------------------------------------------------------------------------
+
+// AND 32-bit - dst = dst & src
+void TurboAssembler::And(Register dst, Register src) { nr(dst, src); }
+
+// AND Pointer Size - dst = dst & src
+void TurboAssembler::AndP(Register dst, Register src) { AndRR(dst, src); }
+
+// Non-clobbering AND 32-bit - dst = src1 & src1
+void TurboAssembler::And(Register dst, Register src1, Register src2) {
+ if (dst != src1 && dst != src2) {
+ // We prefer to generate XR/XGR, over the non clobbering XRK/XRK
+ // as XR is a smaller instruction
+ if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
+ nrk(dst, src1, src2);
+ return;
+ } else {
+ lr(dst, src1);
+ }
+ } else if (dst == src2) {
+ src2 = src1;
+ }
+ And(dst, src2);
+}
+
+// Non-clobbering AND pointer size - dst = src1 & src1
+void TurboAssembler::AndP(Register dst, Register src1, Register src2) {
+ if (dst != src1 && dst != src2) {
+ // We prefer to generate XR/XGR, over the non clobbering XRK/XRK
+ // as XR is a smaller instruction
+ if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
+ AndP_RRR(dst, src1, src2);
+ return;
+ } else {
+ LoadRR(dst, src1);
+ }
+ } else if (dst == src2) {
+ src2 = src1;
+ }
+ AndP(dst, src2);
+}
+
+// AND 32-bit (Reg - Mem)
+void TurboAssembler::And(Register dst, const MemOperand& opnd) {
+ DCHECK(is_int20(opnd.offset()));
+ if (is_uint12(opnd.offset()))
+ n(dst, opnd);
+ else
+ ny(dst, opnd);
+}
+
+// AND Pointer Size (Reg - Mem)
+void TurboAssembler::AndP(Register dst, const MemOperand& opnd) {
+ DCHECK(is_int20(opnd.offset()));
+#if V8_TARGET_ARCH_S390X
+ ng(dst, opnd);
+#else
+ And(dst, opnd);
+#endif
+}
+
+// AND 32-bit - dst = dst & imm
+void TurboAssembler::And(Register dst, const Operand& opnd) { nilf(dst, opnd); }
+
+// AND Pointer Size - dst = dst & imm
+void TurboAssembler::AndP(Register dst, const Operand& opnd) {
+#if V8_TARGET_ARCH_S390X
+ intptr_t value = opnd.immediate();
+ if (value >> 32 != -1) {
+ // this may not work b/c condition code won't be set correctly
+ nihf(dst, Operand(value >> 32));
+ }
+ nilf(dst, Operand(value & 0xFFFFFFFF));
+#else
+ And(dst, opnd);
+#endif
+}
+
+// AND 32-bit - dst = src & imm
+void TurboAssembler::And(Register dst, Register src, const Operand& opnd) {
+ if (dst != src) lr(dst, src);
+ nilf(dst, opnd);
+}
+
+// AND Pointer Size - dst = src & imm
+void TurboAssembler::AndP(Register dst, Register src, const Operand& opnd) {
+ // Try to exploit RISBG first
+ intptr_t value = opnd.immediate();
+ if (CpuFeatures::IsSupported(GENERAL_INSTR_EXT)) {
+ intptr_t shifted_value = value;
+ int trailing_zeros = 0;
+
+ // We start checking how many trailing zeros are left at the end.
+ while ((0 != shifted_value) && (0 == (shifted_value & 1))) {
+ trailing_zeros++;
+ shifted_value >>= 1;
+ }
+
+ // If temp (value with right-most set of zeros shifted out) is 1 less
+ // than power of 2, we have consecutive bits of 1.
+ // Special case: If shift_value is zero, we cannot use RISBG, as it requires
+ // selection of at least 1 bit.
+ if ((0 != shifted_value) && base::bits::IsPowerOfTwo(shifted_value + 1)) {
+ int startBit =
+ base::bits::CountLeadingZeros64(shifted_value) - trailing_zeros;
+ int endBit = 63 - trailing_zeros;
+ // Start: startBit, End: endBit, Shift = 0, true = zero unselected bits.
+ RotateInsertSelectBits(dst, src, Operand(startBit), Operand(endBit),
+ Operand::Zero(), true);
+ return;
+ } else if (-1 == shifted_value) {
+ // A Special case in which all top bits up to MSB are 1's. In this case,
+ // we can set startBit to be 0.
+ int endBit = 63 - trailing_zeros;
+ RotateInsertSelectBits(dst, src, Operand::Zero(), Operand(endBit),
+ Operand::Zero(), true);
+ return;
+ }
+ }
+
+ // If we are &'ing zero, we can just whack the dst register and skip copy
+ if (dst != src && (0 != value)) LoadRR(dst, src);
+ AndP(dst, opnd);
+}
+
+// OR 32-bit - dst = dst & src
+void TurboAssembler::Or(Register dst, Register src) { or_z(dst, src); }
+
+// OR Pointer Size - dst = dst & src
+void TurboAssembler::OrP(Register dst, Register src) { OrRR(dst, src); }
+
+// Non-clobbering OR 32-bit - dst = src1 & src1
+void TurboAssembler::Or(Register dst, Register src1, Register src2) {
+ if (dst != src1 && dst != src2) {
+ // We prefer to generate XR/XGR, over the non clobbering XRK/XRK
+ // as XR is a smaller instruction
+ if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
+ ork(dst, src1, src2);
+ return;
+ } else {
+ lr(dst, src1);
+ }
+ } else if (dst == src2) {
+ src2 = src1;
+ }
+ Or(dst, src2);
+}
+
+// Non-clobbering OR pointer size - dst = src1 & src1
+void TurboAssembler::OrP(Register dst, Register src1, Register src2) {
+ if (dst != src1 && dst != src2) {
+ // We prefer to generate XR/XGR, over the non clobbering XRK/XRK
+ // as XR is a smaller instruction
+ if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
+ OrP_RRR(dst, src1, src2);
+ return;
+ } else {
+ LoadRR(dst, src1);
+ }
+ } else if (dst == src2) {
+ src2 = src1;
+ }
+ OrP(dst, src2);
+}
+
+// OR 32-bit (Reg - Mem)
+void TurboAssembler::Or(Register dst, const MemOperand& opnd) {
+ DCHECK(is_int20(opnd.offset()));
+ if (is_uint12(opnd.offset()))
+ o(dst, opnd);
+ else
+ oy(dst, opnd);
+}
+
+// OR Pointer Size (Reg - Mem)
+void TurboAssembler::OrP(Register dst, const MemOperand& opnd) {
+ DCHECK(is_int20(opnd.offset()));
+#if V8_TARGET_ARCH_S390X
+ og(dst, opnd);
+#else
+ Or(dst, opnd);
+#endif
+}
+
+// OR 32-bit - dst = dst & imm
+void TurboAssembler::Or(Register dst, const Operand& opnd) { oilf(dst, opnd); }
+
+// OR Pointer Size - dst = dst & imm
+void TurboAssembler::OrP(Register dst, const Operand& opnd) {
+#if V8_TARGET_ARCH_S390X
+ intptr_t value = opnd.immediate();
+ if (value >> 32 != 0) {
+ // this may not work b/c condition code won't be set correctly
+ oihf(dst, Operand(value >> 32));
+ }
+ oilf(dst, Operand(value & 0xFFFFFFFF));
+#else
+ Or(dst, opnd);
+#endif
+}
+
+// OR 32-bit - dst = src & imm
+void TurboAssembler::Or(Register dst, Register src, const Operand& opnd) {
+ if (dst != src) lr(dst, src);
+ oilf(dst, opnd);
+}
+
+// OR Pointer Size - dst = src & imm
+void TurboAssembler::OrP(Register dst, Register src, const Operand& opnd) {
+ if (dst != src) LoadRR(dst, src);
+ OrP(dst, opnd);
+}
+
+// XOR 32-bit - dst = dst & src
+void TurboAssembler::Xor(Register dst, Register src) { xr(dst, src); }
+
+// XOR Pointer Size - dst = dst & src
+void TurboAssembler::XorP(Register dst, Register src) { XorRR(dst, src); }
+
+// Non-clobbering XOR 32-bit - dst = src1 & src1
+void TurboAssembler::Xor(Register dst, Register src1, Register src2) {
+ if (dst != src1 && dst != src2) {
+ // We prefer to generate XR/XGR, over the non clobbering XRK/XRK
+ // as XR is a smaller instruction
+ if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
+ xrk(dst, src1, src2);
+ return;
+ } else {
+ lr(dst, src1);
+ }
+ } else if (dst == src2) {
+ src2 = src1;
+ }
+ Xor(dst, src2);
+}
+
+// Non-clobbering XOR pointer size - dst = src1 & src1
+void TurboAssembler::XorP(Register dst, Register src1, Register src2) {
+ if (dst != src1 && dst != src2) {
+ // We prefer to generate XR/XGR, over the non clobbering XRK/XRK
+ // as XR is a smaller instruction
+ if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
+ XorP_RRR(dst, src1, src2);
+ return;
+ } else {
+ LoadRR(dst, src1);
+ }
+ } else if (dst == src2) {
+ src2 = src1;
+ }
+ XorP(dst, src2);
+}
+
+// XOR 32-bit (Reg - Mem)
+void TurboAssembler::Xor(Register dst, const MemOperand& opnd) {
+ DCHECK(is_int20(opnd.offset()));
+ if (is_uint12(opnd.offset()))
+ x(dst, opnd);
+ else
+ xy(dst, opnd);
+}
+
+// XOR Pointer Size (Reg - Mem)
+void TurboAssembler::XorP(Register dst, const MemOperand& opnd) {
+ DCHECK(is_int20(opnd.offset()));
+#if V8_TARGET_ARCH_S390X
+ xg(dst, opnd);
+#else
+ Xor(dst, opnd);
+#endif
+}
+
+// XOR 32-bit - dst = dst & imm
+void TurboAssembler::Xor(Register dst, const Operand& opnd) { xilf(dst, opnd); }
+
+// XOR Pointer Size - dst = dst & imm
+void TurboAssembler::XorP(Register dst, const Operand& opnd) {
+#if V8_TARGET_ARCH_S390X
+ intptr_t value = opnd.immediate();
+ xihf(dst, Operand(value >> 32));
+ xilf(dst, Operand(value & 0xFFFFFFFF));
+#else
+ Xor(dst, opnd);
+#endif
+}
+
+// XOR 32-bit - dst = src & imm
+void TurboAssembler::Xor(Register dst, Register src, const Operand& opnd) {
+ if (dst != src) lr(dst, src);
+ xilf(dst, opnd);
+}
+
+// XOR Pointer Size - dst = src & imm
+void TurboAssembler::XorP(Register dst, Register src, const Operand& opnd) {
+ if (dst != src) LoadRR(dst, src);
+ XorP(dst, opnd);
+}
+
+void TurboAssembler::Not32(Register dst, Register src) {
+ if (src != no_reg && src != dst) lr(dst, src);
+ xilf(dst, Operand(0xFFFFFFFF));
+}
+
+void TurboAssembler::Not64(Register dst, Register src) {
+ if (src != no_reg && src != dst) lgr(dst, src);
+ xihf(dst, Operand(0xFFFFFFFF));
+ xilf(dst, Operand(0xFFFFFFFF));
+}
+
+void TurboAssembler::NotP(Register dst, Register src) {
+#if V8_TARGET_ARCH_S390X
+ Not64(dst, src);
+#else
+ Not32(dst, src);
+#endif
+}
+
+// works the same as mov
+void TurboAssembler::Load(Register dst, const Operand& opnd) {
+ intptr_t value = opnd.immediate();
+ if (is_int16(value)) {
+#if V8_TARGET_ARCH_S390X
+ lghi(dst, opnd);
+#else
+ lhi(dst, opnd);
+#endif
+ } else if (is_int32(value)) {
+#if V8_TARGET_ARCH_S390X
+ lgfi(dst, opnd);
+#else
+ iilf(dst, opnd);
+#endif
+ } else if (is_uint32(value)) {
+#if V8_TARGET_ARCH_S390X
+ llilf(dst, opnd);
+#else
+ iilf(dst, opnd);
+#endif
+ } else {
+ int32_t hi_32 = static_cast<int64_t>(value) >> 32;
+ int32_t lo_32 = static_cast<int32_t>(value);
+
+ iihf(dst, Operand(hi_32));
+ iilf(dst, Operand(lo_32));
+ }
+}
+
+void TurboAssembler::Load(Register dst, const MemOperand& opnd) {
+ DCHECK(is_int20(opnd.offset()));
+#if V8_TARGET_ARCH_S390X
+ lgf(dst, opnd); // 64<-32
+#else
+ if (is_uint12(opnd.offset())) {
+ l(dst, opnd);
+ } else {
+ ly(dst, opnd);
+ }
+#endif
+}
+
+void TurboAssembler::LoadPositiveP(Register result, Register input) {
+#if V8_TARGET_ARCH_S390X
+ lpgr(result, input);
+#else
+ lpr(result, input);
+#endif
+}
+
+void TurboAssembler::LoadPositive32(Register result, Register input) {
+ lpr(result, input);
+ lgfr(result, result);
+}
+
+//-----------------------------------------------------------------------------
+// Compare Helpers
+//-----------------------------------------------------------------------------
+
+// Compare 32-bit Register vs Register
+void TurboAssembler::Cmp32(Register src1, Register src2) { cr_z(src1, src2); }
+
+// Compare Pointer Sized Register vs Register
+void TurboAssembler::CmpP(Register src1, Register src2) {
+#if V8_TARGET_ARCH_S390X
+ cgr(src1, src2);
+#else
+ Cmp32(src1, src2);
+#endif
+}
+
+// Compare 32-bit Register vs Immediate
+// This helper will set up proper relocation entries if required.
+void TurboAssembler::Cmp32(Register dst, const Operand& opnd) {
+ if (opnd.rmode() == RelocInfo::NONE) {
+ intptr_t value = opnd.immediate();
+ if (is_int16(value))
+ chi(dst, opnd);
+ else
+ cfi(dst, opnd);
+ } else {
+ // Need to generate relocation record here
+ RecordRelocInfo(opnd.rmode(), opnd.immediate());
+ cfi(dst, opnd);
+ }
+}
+
+// Compare Pointer Sized Register vs Immediate
+// This helper will set up proper relocation entries if required.
+void TurboAssembler::CmpP(Register dst, const Operand& opnd) {
+#if V8_TARGET_ARCH_S390X
+ if (opnd.rmode() == RelocInfo::NONE) {
+ cgfi(dst, opnd);
+ } else {
+ mov(r0, opnd); // Need to generate 64-bit relocation
+ cgr(dst, r0);
+ }
+#else
+ Cmp32(dst, opnd);
+#endif
+}
+
+// Compare 32-bit Register vs Memory
+void TurboAssembler::Cmp32(Register dst, const MemOperand& opnd) {
+ // make sure offset is within 20 bit range
+ DCHECK(is_int20(opnd.offset()));
+ if (is_uint12(opnd.offset()))
+ c(dst, opnd);
+ else
+ cy(dst, opnd);
+}
+
+// Compare Pointer Size Register vs Memory
+void TurboAssembler::CmpP(Register dst, const MemOperand& opnd) {
+ // make sure offset is within 20 bit range
+ DCHECK(is_int20(opnd.offset()));
+#if V8_TARGET_ARCH_S390X
+ cg(dst, opnd);
+#else
+ Cmp32(dst, opnd);
+#endif
+}
+
+// Using cs or scy based on the offset
+void TurboAssembler::CmpAndSwap(Register old_val, Register new_val,
+ const MemOperand& opnd) {
+ if (is_uint12(opnd.offset())) {
+ cs(old_val, new_val, opnd);
+ } else {
+ csy(old_val, new_val, opnd);
+ }
+}
+
+void TurboAssembler::CmpAndSwap64(Register old_val, Register new_val,
+ const MemOperand& opnd) {
+ DCHECK(is_int20(opnd.offset()));
+ csg(old_val, new_val, opnd);
+}
+
+//-----------------------------------------------------------------------------
+// Compare Logical Helpers
+//-----------------------------------------------------------------------------
+
+// Compare Logical 32-bit Register vs Register
+void TurboAssembler::CmpLogical32(Register dst, Register src) { clr(dst, src); }
+
+// Compare Logical Pointer Sized Register vs Register
+void TurboAssembler::CmpLogicalP(Register dst, Register src) {
+#ifdef V8_TARGET_ARCH_S390X
+ clgr(dst, src);
+#else
+ CmpLogical32(dst, src);
+#endif
+}
+
+// Compare Logical 32-bit Register vs Immediate
+void TurboAssembler::CmpLogical32(Register dst, const Operand& opnd) {
+ clfi(dst, opnd);
+}
+
+// Compare Logical Pointer Sized Register vs Immediate
+void TurboAssembler::CmpLogicalP(Register dst, const Operand& opnd) {
+#if V8_TARGET_ARCH_S390X
+ DCHECK_EQ(static_cast<uint32_t>(opnd.immediate() >> 32), 0);
+ clgfi(dst, opnd);
+#else
+ CmpLogical32(dst, opnd);
+#endif
+}
+
+// Compare Logical 32-bit Register vs Memory
+void TurboAssembler::CmpLogical32(Register dst, const MemOperand& opnd) {
+ // make sure offset is within 20 bit range
+ DCHECK(is_int20(opnd.offset()));
+ if (is_uint12(opnd.offset()))
+ cl(dst, opnd);
+ else
+ cly(dst, opnd);
+}
+
+// Compare Logical Pointer Sized Register vs Memory
+void TurboAssembler::CmpLogicalP(Register dst, const MemOperand& opnd) {
+ // make sure offset is within 20 bit range
+ DCHECK(is_int20(opnd.offset()));
+#if V8_TARGET_ARCH_S390X
+ clg(dst, opnd);
+#else
+ CmpLogical32(dst, opnd);
+#endif
+}
+
+// Compare Logical Byte (Mem - Imm)
+void TurboAssembler::CmpLogicalByte(const MemOperand& mem, const Operand& imm) {
+ DCHECK(is_uint8(imm.immediate()));
+ if (is_uint12(mem.offset()))
+ cli(mem, imm);
+ else
+ cliy(mem, imm);
+}
+
+void TurboAssembler::Branch(Condition c, const Operand& opnd) {
+ intptr_t value = opnd.immediate();
+ if (is_int16(value))
+ brc(c, opnd);
+ else
+ brcl(c, opnd);
+}
+
+// Branch On Count. Decrement R1, and branch if R1 != 0.
+void TurboAssembler::BranchOnCount(Register r1, Label* l) {
+ int32_t offset = branch_offset(l);
+ if (is_int16(offset)) {
+#if V8_TARGET_ARCH_S390X
+ brctg(r1, Operand(offset));
+#else
+ brct(r1, Operand(offset));
+#endif
+ } else {
+ AddP(r1, Operand(-1));
+ Branch(ne, Operand(offset));
+ }
+}
+
+void TurboAssembler::LoadIntLiteral(Register dst, int value) {
+ Load(dst, Operand(value));
+}
+
+void TurboAssembler::LoadSmiLiteral(Register dst, Smi smi) {
+ intptr_t value = static_cast<intptr_t>(smi.ptr());
+#if defined(V8_COMPRESS_POINTERS) || defined(V8_31BIT_SMIS_ON_64BIT_ARCH)
+ llilf(dst, Operand(value));
+#else
+ DCHECK_EQ(value & 0xFFFFFFFF, 0);
+ // The smi value is loaded in upper 32-bits. Lower 32-bit are zeros.
+ llihf(dst, Operand(value >> 32));
+#endif
+}
+
+void TurboAssembler::LoadDoubleLiteral(DoubleRegister result, uint64_t value,
+ Register scratch) {
+ uint32_t hi_32 = value >> 32;
+ uint32_t lo_32 = static_cast<uint32_t>(value);
+
+ // Load the 64-bit value into a GPR, then transfer it to FPR via LDGR
+ if (value == 0) {
+ lzdr(result);
+ } else if (lo_32 == 0) {
+ llihf(scratch, Operand(hi_32));
+ ldgr(result, scratch);
+ } else {
+ iihf(scratch, Operand(hi_32));
+ iilf(scratch, Operand(lo_32));
+ ldgr(result, scratch);
+ }
+}
+
+void TurboAssembler::LoadDoubleLiteral(DoubleRegister result, double value,
+ Register scratch) {
+ uint64_t int_val = bit_cast<uint64_t, double>(value);
+ LoadDoubleLiteral(result, int_val, scratch);
+}
+
+void TurboAssembler::LoadFloat32Literal(DoubleRegister result, float value,
+ Register scratch) {
+ uint64_t int_val = static_cast<uint64_t>(bit_cast<uint32_t, float>(value))
+ << 32;
+ LoadDoubleLiteral(result, int_val, scratch);
+}
+
+void TurboAssembler::CmpSmiLiteral(Register src1, Smi smi, Register scratch) {
+#if defined(V8_COMPRESS_POINTERS) || defined(V8_31BIT_SMIS_ON_64BIT_ARCH)
+ // CFI takes 32-bit immediate.
+ cfi(src1, Operand(smi));
+#else
+ if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
+ cih(src1, Operand(static_cast<intptr_t>(smi.ptr()) >> 32));
+ } else {
+ LoadSmiLiteral(scratch, smi);
+ cgr(src1, scratch);
+ }
+#endif
+}
+
+// Load a "pointer" sized value from the memory location
+void TurboAssembler::LoadP(Register dst, const MemOperand& mem,
+ Register scratch) {
+ int offset = mem.offset();
+
+#if V8_TARGET_ARCH_S390X
+ MemOperand src = mem;
+ if (!is_int20(offset)) {
+ DCHECK(scratch != no_reg && scratch != r0 && mem.rx() == r0);
+ DCHECK(scratch != mem.rb());
+ LoadIntLiteral(scratch, offset);
+ src = MemOperand(mem.rb(), scratch);
+ }
+ lg(dst, src);
+#else
+ if (is_uint12(offset)) {
+ l(dst, mem);
+ } else if (is_int20(offset)) {
+ ly(dst, mem);
+ } else {
+ DCHECK(scratch != no_reg && scratch != r0 && mem.rx() == r0);
+ DCHECK(scratch != mem.rb());
+ LoadIntLiteral(scratch, offset);
+ l(dst, MemOperand(mem.rb(), scratch));
+ }
+#endif
+}
+
+// Store a "pointer" sized value to the memory location
+void TurboAssembler::StoreP(Register src, const MemOperand& mem,
+ Register scratch) {
+ if (!is_int20(mem.offset())) {
+ DCHECK(scratch != no_reg);
+ DCHECK(scratch != r0);
+ LoadIntLiteral(scratch, mem.offset());
+#if V8_TARGET_ARCH_S390X
+ stg(src, MemOperand(mem.rb(), scratch));
+#else
+ st(src, MemOperand(mem.rb(), scratch));
+#endif
+ } else {
+#if V8_TARGET_ARCH_S390X
+ stg(src, mem);
+#else
+ // StoreW will try to generate ST if offset fits, otherwise
+ // it'll generate STY.
+ StoreW(src, mem);
+#endif
+ }
+}
+
+// Store a "pointer" sized constant to the memory location
+void TurboAssembler::StoreP(const MemOperand& mem, const Operand& opnd,
+ Register scratch) {
+ // Relocations not supported
+ DCHECK_EQ(opnd.rmode(), RelocInfo::NONE);
+
+ // Try to use MVGHI/MVHI
+ if (CpuFeatures::IsSupported(GENERAL_INSTR_EXT) && is_uint12(mem.offset()) &&
+ mem.getIndexRegister() == r0 && is_int16(opnd.immediate())) {
+#if V8_TARGET_ARCH_S390X
+ mvghi(mem, opnd);
+#else
+ mvhi(mem, opnd);
+#endif
+ } else {
+ LoadImmP(scratch, opnd);
+ StoreP(scratch, mem);
+ }
+}
+
+void TurboAssembler::LoadMultipleP(Register dst1, Register dst2,
+ const MemOperand& mem) {
+#if V8_TARGET_ARCH_S390X
+ DCHECK(is_int20(mem.offset()));
+ lmg(dst1, dst2, mem);
+#else
+ if (is_uint12(mem.offset())) {
+ lm(dst1, dst2, mem);
+ } else {
+ DCHECK(is_int20(mem.offset()));
+ lmy(dst1, dst2, mem);
+ }
+#endif
+}
+
+void TurboAssembler::StoreMultipleP(Register src1, Register src2,
+ const MemOperand& mem) {
+#if V8_TARGET_ARCH_S390X
+ DCHECK(is_int20(mem.offset()));
+ stmg(src1, src2, mem);
+#else
+ if (is_uint12(mem.offset())) {
+ stm(src1, src2, mem);
+ } else {
+ DCHECK(is_int20(mem.offset()));
+ stmy(src1, src2, mem);
+ }
+#endif
+}
+
+void TurboAssembler::LoadMultipleW(Register dst1, Register dst2,
+ const MemOperand& mem) {
+ if (is_uint12(mem.offset())) {
+ lm(dst1, dst2, mem);
+ } else {
+ DCHECK(is_int20(mem.offset()));
+ lmy(dst1, dst2, mem);
+ }
+}
+
+void TurboAssembler::StoreMultipleW(Register src1, Register src2,
+ const MemOperand& mem) {
+ if (is_uint12(mem.offset())) {
+ stm(src1, src2, mem);
+ } else {
+ DCHECK(is_int20(mem.offset()));
+ stmy(src1, src2, mem);
+ }
+}
+
+// Load 32-bits and sign extend if necessary.
+void TurboAssembler::LoadW(Register dst, Register src) {
+#if V8_TARGET_ARCH_S390X
+ lgfr(dst, src);
+#else
+ if (dst != src) lr(dst, src);
+#endif
+}
+
+// Load 32-bits and sign extend if necessary.
+void TurboAssembler::LoadW(Register dst, const MemOperand& mem,
+ Register scratch) {
+ int offset = mem.offset();
+
+ if (!is_int20(offset)) {
+ DCHECK(scratch != no_reg);
+ LoadIntLiteral(scratch, offset);
+#if V8_TARGET_ARCH_S390X
+ lgf(dst, MemOperand(mem.rb(), scratch));
+#else
+ l(dst, MemOperand(mem.rb(), scratch));
+#endif
+ } else {
+#if V8_TARGET_ARCH_S390X
+ lgf(dst, mem);
+#else
+ if (is_uint12(offset)) {
+ l(dst, mem);
+ } else {
+ ly(dst, mem);
+ }
+#endif
+ }
+}
+
+// Load 32-bits and zero extend if necessary.
+void TurboAssembler::LoadlW(Register dst, Register src) {
+#if V8_TARGET_ARCH_S390X
+ llgfr(dst, src);
+#else
+ if (dst != src) lr(dst, src);
+#endif
+}
+
+// Variable length depending on whether offset fits into immediate field
+// MemOperand of RX or RXY format
+void TurboAssembler::LoadlW(Register dst, const MemOperand& mem,
+ Register scratch) {
+ Register base = mem.rb();
+ int offset = mem.offset();
+
+#if V8_TARGET_ARCH_S390X
+ if (is_int20(offset)) {
+ llgf(dst, mem);
+ } else if (scratch != no_reg) {
+ // Materialize offset into scratch register.
+ LoadIntLiteral(scratch, offset);
+ llgf(dst, MemOperand(base, scratch));
+ } else {
+ DCHECK(false);
+ }
+#else
+ bool use_RXform = false;
+ bool use_RXYform = false;
+ if (is_uint12(offset)) {
+ // RX-format supports unsigned 12-bits offset.
+ use_RXform = true;
+ } else if (is_int20(offset)) {
+ // RXY-format supports signed 20-bits offset.
+ use_RXYform = true;
+ } else if (scratch != no_reg) {
+ // Materialize offset into scratch register.
+ LoadIntLiteral(scratch, offset);
+ } else {
+ DCHECK(false);
+ }
+
+ if (use_RXform) {
+ l(dst, mem);
+ } else if (use_RXYform) {
+ ly(dst, mem);
+ } else {
+ ly(dst, MemOperand(base, scratch));
+ }
+#endif
+}
+
+void TurboAssembler::LoadLogicalHalfWordP(Register dst, const MemOperand& mem) {
+#if V8_TARGET_ARCH_S390X
+ llgh(dst, mem);
+#else
+ llh(dst, mem);
+#endif
+}
+
+void TurboAssembler::LoadLogicalHalfWordP(Register dst, Register src) {
+#if V8_TARGET_ARCH_S390X
+ llghr(dst, src);
+#else
+ llhr(dst, src);
+#endif
+}
+
+void TurboAssembler::LoadB(Register dst, const MemOperand& mem) {
+#if V8_TARGET_ARCH_S390X
+ lgb(dst, mem);
+#else
+ lb(dst, mem);
+#endif
+}
+
+void TurboAssembler::LoadB(Register dst, Register src) {
+#if V8_TARGET_ARCH_S390X
+ lgbr(dst, src);
+#else
+ lbr(dst, src);
+#endif
+}
+
+void TurboAssembler::LoadlB(Register dst, const MemOperand& mem) {
+#if V8_TARGET_ARCH_S390X
+ llgc(dst, mem);
+#else
+ llc(dst, mem);
+#endif
+}
+
+void TurboAssembler::LoadlB(Register dst, Register src) {
+#if V8_TARGET_ARCH_S390X
+ llgcr(dst, src);
+#else
+ llcr(dst, src);
+#endif
+}
+
+void TurboAssembler::LoadLogicalReversedWordP(Register dst,
+ const MemOperand& mem) {
+ lrv(dst, mem);
+ LoadlW(dst, dst);
+}
+
+void TurboAssembler::LoadLogicalReversedHalfWordP(Register dst,
+ const MemOperand& mem) {
+ lrvh(dst, mem);
+ LoadLogicalHalfWordP(dst, dst);
+}
+
+// Load And Test (Reg <- Reg)
+void TurboAssembler::LoadAndTest32(Register dst, Register src) {
+ ltr(dst, src);
+}
+
+// Load And Test
+// (Register dst(ptr) = Register src (32 | 32->64))
+// src is treated as a 32-bit signed integer, which is sign extended to
+// 64-bit if necessary.
+void TurboAssembler::LoadAndTestP_ExtendSrc(Register dst, Register src) {
+#if V8_TARGET_ARCH_S390X
+ ltgfr(dst, src);
+#else
+ ltr(dst, src);
+#endif
+}
+
+// Load And Test Pointer Sized (Reg <- Reg)
+void TurboAssembler::LoadAndTestP(Register dst, Register src) {
+#if V8_TARGET_ARCH_S390X
+ ltgr(dst, src);
+#else
+ ltr(dst, src);
+#endif
+}
+
+// Load And Test 32-bit (Reg <- Mem)
+void TurboAssembler::LoadAndTest32(Register dst, const MemOperand& mem) {
+ lt_z(dst, mem);
+}
+
+// Load And Test Pointer Sized (Reg <- Mem)
+void TurboAssembler::LoadAndTestP(Register dst, const MemOperand& mem) {
+#if V8_TARGET_ARCH_S390X
+ ltg(dst, mem);
+#else
+ lt_z(dst, mem);
+#endif
+}
+
+// Load On Condition Pointer Sized (Reg <- Reg)
+void TurboAssembler::LoadOnConditionP(Condition cond, Register dst,
+ Register src) {
+#if V8_TARGET_ARCH_S390X
+ locgr(cond, dst, src);
+#else
+ locr(cond, dst, src);
+#endif
+}
+
+// Load Double Precision (64-bit) Floating Point number from memory
+void TurboAssembler::LoadDouble(DoubleRegister dst, const MemOperand& mem) {
+ // for 32bit and 64bit we all use 64bit floating point regs
+ if (is_uint12(mem.offset())) {
+ ld(dst, mem);
+ } else {
+ ldy(dst, mem);
+ }
+}
+
+// Load Single Precision (32-bit) Floating Point number from memory
+void TurboAssembler::LoadFloat32(DoubleRegister dst, const MemOperand& mem) {
+ if (is_uint12(mem.offset())) {
+ le_z(dst, mem);
+ } else {
+ DCHECK(is_int20(mem.offset()));
+ ley(dst, mem);
+ }
+}
+
+// Load Single Precision (32-bit) Floating Point number from memory,
+// and convert to Double Precision (64-bit)
+void TurboAssembler::LoadFloat32ConvertToDouble(DoubleRegister dst,
+ const MemOperand& mem) {
+ LoadFloat32(dst, mem);
+ ldebr(dst, dst);
+}
+
+void TurboAssembler::LoadSimd128(Simd128Register dst, const MemOperand& mem,
+ Register scratch) {
+ if (is_uint12(mem.offset())) {
+ vl(dst, mem, Condition(0));
+ } else {
+ DCHECK(is_int20(mem.offset()));
+ lay(scratch, mem);
+ vl(dst, MemOperand(scratch), Condition(0));
+ }
+}
+
+// Store Double Precision (64-bit) Floating Point number to memory
+void TurboAssembler::StoreDouble(DoubleRegister dst, const MemOperand& mem) {
+ if (is_uint12(mem.offset())) {
+ std(dst, mem);
+ } else {
+ stdy(dst, mem);
+ }
+}
+
+// Store Single Precision (32-bit) Floating Point number to memory
+void TurboAssembler::StoreFloat32(DoubleRegister src, const MemOperand& mem) {
+ if (is_uint12(mem.offset())) {
+ ste(src, mem);
+ } else {
+ stey(src, mem);
+ }
+}
+
+// Convert Double precision (64-bit) to Single Precision (32-bit)
+// and store resulting Float32 to memory
+void TurboAssembler::StoreDoubleAsFloat32(DoubleRegister src,
+ const MemOperand& mem,
+ DoubleRegister scratch) {
+ ledbr(scratch, src);
+ StoreFloat32(scratch, mem);
+}
+
+void TurboAssembler::StoreSimd128(Simd128Register src, const MemOperand& mem,
+ Register scratch) {
+ if (is_uint12(mem.offset())) {
+ vst(src, mem, Condition(0));
+ } else {
+ DCHECK(is_int20(mem.offset()));
+ lay(scratch, mem);
+ vst(src, MemOperand(scratch), Condition(0));
+ }
+}
+
+void TurboAssembler::AddFloat32(DoubleRegister dst, const MemOperand& opnd,
+ DoubleRegister scratch) {
+ if (is_uint12(opnd.offset())) {
+ aeb(dst, opnd);
+ } else {
+ ley(scratch, opnd);
+ aebr(dst, scratch);
+ }
+}
+
+void TurboAssembler::AddFloat64(DoubleRegister dst, const MemOperand& opnd,
+ DoubleRegister scratch) {
+ if (is_uint12(opnd.offset())) {
+ adb(dst, opnd);
+ } else {
+ ldy(scratch, opnd);
+ adbr(dst, scratch);
+ }
+}
+
+void TurboAssembler::SubFloat32(DoubleRegister dst, const MemOperand& opnd,
+ DoubleRegister scratch) {
+ if (is_uint12(opnd.offset())) {
+ seb(dst, opnd);
+ } else {
+ ley(scratch, opnd);
+ sebr(dst, scratch);
+ }
+}
+
+void TurboAssembler::SubFloat64(DoubleRegister dst, const MemOperand& opnd,
+ DoubleRegister scratch) {
+ if (is_uint12(opnd.offset())) {
+ sdb(dst, opnd);
+ } else {
+ ldy(scratch, opnd);
+ sdbr(dst, scratch);
+ }
+}
+
+void TurboAssembler::MulFloat32(DoubleRegister dst, const MemOperand& opnd,
+ DoubleRegister scratch) {
+ if (is_uint12(opnd.offset())) {
+ meeb(dst, opnd);
+ } else {
+ ley(scratch, opnd);
+ meebr(dst, scratch);
+ }
+}
+
+void TurboAssembler::MulFloat64(DoubleRegister dst, const MemOperand& opnd,
+ DoubleRegister scratch) {
+ if (is_uint12(opnd.offset())) {
+ mdb(dst, opnd);
+ } else {
+ ldy(scratch, opnd);
+ mdbr(dst, scratch);
+ }
+}
+
+void TurboAssembler::DivFloat32(DoubleRegister dst, const MemOperand& opnd,
+ DoubleRegister scratch) {
+ if (is_uint12(opnd.offset())) {
+ deb(dst, opnd);
+ } else {
+ ley(scratch, opnd);
+ debr(dst, scratch);
+ }
+}
+
+void TurboAssembler::DivFloat64(DoubleRegister dst, const MemOperand& opnd,
+ DoubleRegister scratch) {
+ if (is_uint12(opnd.offset())) {
+ ddb(dst, opnd);
+ } else {
+ ldy(scratch, opnd);
+ ddbr(dst, scratch);
+ }
+}
+
+void TurboAssembler::LoadFloat32ToDouble(DoubleRegister dst,
+ const MemOperand& opnd,
+ DoubleRegister scratch) {
+ if (is_uint12(opnd.offset())) {
+ ldeb(dst, opnd);
+ } else {
+ ley(scratch, opnd);
+ ldebr(dst, scratch);
+ }
+}
+
+// Variable length depending on whether offset fits into immediate field
+// MemOperand of RX or RXY format
+void TurboAssembler::StoreW(Register src, const MemOperand& mem,
+ Register scratch) {
+ Register base = mem.rb();
+ int offset = mem.offset();
+
+ bool use_RXform = false;
+ bool use_RXYform = false;
+
+ if (is_uint12(offset)) {
+ // RX-format supports unsigned 12-bits offset.
+ use_RXform = true;
+ } else if (is_int20(offset)) {
+ // RXY-format supports signed 20-bits offset.
+ use_RXYform = true;
+ } else if (scratch != no_reg) {
+ // Materialize offset into scratch register.
+ LoadIntLiteral(scratch, offset);
+ } else {
+ // scratch is no_reg
+ DCHECK(false);
+ }
+
+ if (use_RXform) {
+ st(src, mem);
+ } else if (use_RXYform) {
+ sty(src, mem);
+ } else {
+ StoreW(src, MemOperand(base, scratch));
+ }
+}
+
+void TurboAssembler::LoadHalfWordP(Register dst, Register src) {
+#if V8_TARGET_ARCH_S390X
+ lghr(dst, src);
+#else
+ lhr(dst, src);
+#endif
+}
+
+// Loads 16-bits half-word value from memory and sign extends to pointer
+// sized register
+void TurboAssembler::LoadHalfWordP(Register dst, const MemOperand& mem,
+ Register scratch) {
+ Register base = mem.rb();
+ int offset = mem.offset();
+
+ if (!is_int20(offset)) {
+ DCHECK(scratch != no_reg);
+ LoadIntLiteral(scratch, offset);
+#if V8_TARGET_ARCH_S390X
+ lgh(dst, MemOperand(base, scratch));
+#else
+ lh(dst, MemOperand(base, scratch));
+#endif
+ } else {
+#if V8_TARGET_ARCH_S390X
+ lgh(dst, mem);
+#else
+ if (is_uint12(offset)) {
+ lh(dst, mem);
+ } else {
+ lhy(dst, mem);
+ }
+#endif
+ }
+}
+
+// Variable length depending on whether offset fits into immediate field
+// MemOperand current only supports d-form
+void TurboAssembler::StoreHalfWord(Register src, const MemOperand& mem,
+ Register scratch) {
+ Register base = mem.rb();
+ int offset = mem.offset();
+
+ if (is_uint12(offset)) {
+ sth(src, mem);
+ } else if (is_int20(offset)) {
+ sthy(src, mem);
+ } else {
+ DCHECK(scratch != no_reg);
+ LoadIntLiteral(scratch, offset);
+ sth(src, MemOperand(base, scratch));
+ }
+}
+
+// Variable length depending on whether offset fits into immediate field
+// MemOperand current only supports d-form
+void TurboAssembler::StoreByte(Register src, const MemOperand& mem,
+ Register scratch) {
+ Register base = mem.rb();
+ int offset = mem.offset();
+
+ if (is_uint12(offset)) {
+ stc(src, mem);
+ } else if (is_int20(offset)) {
+ stcy(src, mem);
+ } else {
+ DCHECK(scratch != no_reg);
+ LoadIntLiteral(scratch, offset);
+ stc(src, MemOperand(base, scratch));
+ }
+}
+
+// Shift left logical for 32-bit integer types.
+void TurboAssembler::ShiftLeft(Register dst, Register src, const Operand& val) {
+ if (dst == src) {
+ sll(dst, val);
+ } else if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
+ sllk(dst, src, val);
+ } else {
+ lr(dst, src);
+ sll(dst, val);
+ }
+}
+
+// Shift left logical for 32-bit integer types.
+void TurboAssembler::ShiftLeft(Register dst, Register src, Register val) {
+ if (dst == src) {
+ sll(dst, val);
+ } else if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
+ sllk(dst, src, val);
+ } else {
+ DCHECK(dst != val); // The lr/sll path clobbers val.
+ lr(dst, src);
+ sll(dst, val);
+ }
+}
+
+// Shift right logical for 32-bit integer types.
+void TurboAssembler::ShiftRight(Register dst, Register src,
+ const Operand& val) {
+ if (dst == src) {
+ srl(dst, val);
+ } else if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
+ srlk(dst, src, val);
+ } else {
+ lr(dst, src);
+ srl(dst, val);
+ }
+}
+
+// Shift right logical for 32-bit integer types.
+void TurboAssembler::ShiftRight(Register dst, Register src, Register val) {
+ if (dst == src) {
+ srl(dst, val);
+ } else if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
+ srlk(dst, src, val);
+ } else {
+ DCHECK(dst != val); // The lr/srl path clobbers val.
+ lr(dst, src);
+ srl(dst, val);
+ }
+}
+
+// Shift left arithmetic for 32-bit integer types.
+void TurboAssembler::ShiftLeftArith(Register dst, Register src,
+ const Operand& val) {
+ if (dst == src) {
+ sla(dst, val);
+ } else if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
+ slak(dst, src, val);
+ } else {
+ lr(dst, src);
+ sla(dst, val);
+ }
+}
+
+// Shift left arithmetic for 32-bit integer types.
+void TurboAssembler::ShiftLeftArith(Register dst, Register src, Register val) {
+ if (dst == src) {
+ sla(dst, val);
+ } else if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
+ slak(dst, src, val);
+ } else {
+ DCHECK(dst != val); // The lr/sla path clobbers val.
+ lr(dst, src);
+ sla(dst, val);
+ }
+}
+
+// Shift right arithmetic for 32-bit integer types.
+void TurboAssembler::ShiftRightArith(Register dst, Register src,
+ const Operand& val) {
+ if (dst == src) {
+ sra(dst, val);
+ } else if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
+ srak(dst, src, val);
+ } else {
+ lr(dst, src);
+ sra(dst, val);
+ }
+}
+
+// Shift right arithmetic for 32-bit integer types.
+void TurboAssembler::ShiftRightArith(Register dst, Register src, Register val) {
+ if (dst == src) {
+ sra(dst, val);
+ } else if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
+ srak(dst, src, val);
+ } else {
+ DCHECK(dst != val); // The lr/sra path clobbers val.
+ lr(dst, src);
+ sra(dst, val);
+ }
+}
+
+// Clear right most # of bits
+void TurboAssembler::ClearRightImm(Register dst, Register src,
+ const Operand& val) {
+ int numBitsToClear = val.immediate() % (kSystemPointerSize * 8);
+
+ // Try to use RISBG if possible
+ if (CpuFeatures::IsSupported(GENERAL_INSTR_EXT)) {
+ int endBit = 63 - numBitsToClear;
+ RotateInsertSelectBits(dst, src, Operand::Zero(), Operand(endBit),
+ Operand::Zero(), true);
+ return;
+ }
+
+ uint64_t hexMask = ~((1L << numBitsToClear) - 1);
+
+ // S390 AND instr clobbers source. Make a copy if necessary
+ if (dst != src) LoadRR(dst, src);
+
+ if (numBitsToClear <= 16) {
+ nill(dst, Operand(static_cast<uint16_t>(hexMask)));
+ } else if (numBitsToClear <= 32) {
+ nilf(dst, Operand(static_cast<uint32_t>(hexMask)));
+ } else if (numBitsToClear <= 64) {
+ nilf(dst, Operand(static_cast<intptr_t>(0)));
+ nihf(dst, Operand(hexMask >> 32));
+ }
+}
+
+void TurboAssembler::Popcnt32(Register dst, Register src) {
+ DCHECK(src != r0);
+ DCHECK(dst != r0);
+
+ popcnt(dst, src);
+ ShiftRight(r0, dst, Operand(16));
+ ar(dst, r0);
+ ShiftRight(r0, dst, Operand(8));
+ ar(dst, r0);
+ llgcr(dst, dst);
+}
+
+#ifdef V8_TARGET_ARCH_S390X
+void TurboAssembler::Popcnt64(Register dst, Register src) {
+ DCHECK(src != r0);
+ DCHECK(dst != r0);
+
+ popcnt(dst, src);
+ ShiftRightP(r0, dst, Operand(32));
+ AddP(dst, r0);
+ ShiftRightP(r0, dst, Operand(16));
+ AddP(dst, r0);
+ ShiftRightP(r0, dst, Operand(8));
+ AddP(dst, r0);
+ LoadlB(dst, dst);
+}
+#endif
+
+void TurboAssembler::SwapP(Register src, Register dst, Register scratch) {
+ if (src == dst) return;
+ DCHECK(!AreAliased(src, dst, scratch));
+ LoadRR(scratch, src);
+ LoadRR(src, dst);
+ LoadRR(dst, scratch);
+}
+
+void TurboAssembler::SwapP(Register src, MemOperand dst, Register scratch) {
+ if (dst.rx() != r0) DCHECK(!AreAliased(src, dst.rx(), scratch));
+ if (dst.rb() != r0) DCHECK(!AreAliased(src, dst.rb(), scratch));
+ DCHECK(!AreAliased(src, scratch));
+ LoadRR(scratch, src);
+ LoadP(src, dst);
+ StoreP(scratch, dst);
+}
+
+void TurboAssembler::SwapP(MemOperand src, MemOperand dst, Register scratch_0,
+ Register scratch_1) {
+ if (src.rx() != r0) DCHECK(!AreAliased(src.rx(), scratch_0, scratch_1));
+ if (src.rb() != r0) DCHECK(!AreAliased(src.rb(), scratch_0, scratch_1));
+ if (dst.rx() != r0) DCHECK(!AreAliased(dst.rx(), scratch_0, scratch_1));
+ if (dst.rb() != r0) DCHECK(!AreAliased(dst.rb(), scratch_0, scratch_1));
+ DCHECK(!AreAliased(scratch_0, scratch_1));
+ LoadP(scratch_0, src);
+ LoadP(scratch_1, dst);
+ StoreP(scratch_0, dst);
+ StoreP(scratch_1, src);
+}
+
+void TurboAssembler::SwapFloat32(DoubleRegister src, DoubleRegister dst,
+ DoubleRegister scratch) {
+ if (src == dst) return;
+ DCHECK(!AreAliased(src, dst, scratch));
+ ldr(scratch, src);
+ ldr(src, dst);
+ ldr(dst, scratch);
+}
+
+void TurboAssembler::SwapFloat32(DoubleRegister src, MemOperand dst,
+ DoubleRegister scratch) {
+ DCHECK(!AreAliased(src, scratch));
+ ldr(scratch, src);
+ LoadFloat32(src, dst);
+ StoreFloat32(scratch, dst);
+}
+
+void TurboAssembler::SwapFloat32(MemOperand src, MemOperand dst,
+ DoubleRegister scratch) {
+ // push d0, to be used as scratch
+ lay(sp, MemOperand(sp, -kDoubleSize));
+ StoreDouble(d0, MemOperand(sp));
+ LoadFloat32(scratch, src);
+ LoadFloat32(d0, dst);
+ StoreFloat32(scratch, dst);
+ StoreFloat32(d0, src);
+ // restore d0
+ LoadDouble(d0, MemOperand(sp));
+ lay(sp, MemOperand(sp, kDoubleSize));
+}
+
+void TurboAssembler::SwapDouble(DoubleRegister src, DoubleRegister dst,
+ DoubleRegister scratch) {
+ if (src == dst) return;
+ DCHECK(!AreAliased(src, dst, scratch));
+ ldr(scratch, src);
+ ldr(src, dst);
+ ldr(dst, scratch);
+}
+
+void TurboAssembler::SwapDouble(DoubleRegister src, MemOperand dst,
+ DoubleRegister scratch) {
+ DCHECK(!AreAliased(src, scratch));
+ ldr(scratch, src);
+ LoadDouble(src, dst);
+ StoreDouble(scratch, dst);
+}
+
+void TurboAssembler::SwapDouble(MemOperand src, MemOperand dst,
+ DoubleRegister scratch) {
+ // push d0, to be used as scratch
+ lay(sp, MemOperand(sp, -kDoubleSize));
+ StoreDouble(d0, MemOperand(sp));
+ LoadDouble(scratch, src);
+ LoadDouble(d0, dst);
+ StoreDouble(scratch, dst);
+ StoreDouble(d0, src);
+ // restore d0
+ LoadDouble(d0, MemOperand(sp));
+ lay(sp, MemOperand(sp, kDoubleSize));
+}
+
+void TurboAssembler::SwapSimd128(Simd128Register src, Simd128Register dst,
+ Simd128Register scratch) {
+ if (src == dst) return;
+ vlr(scratch, src, Condition(0), Condition(0), Condition(0));
+ vlr(src, dst, Condition(0), Condition(0), Condition(0));
+ vlr(dst, scratch, Condition(0), Condition(0), Condition(0));
+}
+
+void TurboAssembler::SwapSimd128(Simd128Register src, MemOperand dst,
+ Simd128Register scratch) {
+ DCHECK(!AreAliased(src, scratch));
+ vlr(scratch, src, Condition(0), Condition(0), Condition(0));
+ LoadSimd128(src, dst, ip);
+ StoreSimd128(scratch, dst, ip);
+}
+
+void TurboAssembler::SwapSimd128(MemOperand src, MemOperand dst,
+ Simd128Register scratch) {
+ // push d0, to be used as scratch
+ lay(sp, MemOperand(sp, -kSimd128Size));
+ StoreSimd128(d0, MemOperand(sp), ip);
+ LoadSimd128(scratch, src, ip);
+ LoadSimd128(d0, dst, ip);
+ StoreSimd128(scratch, dst, ip);
+ StoreSimd128(d0, src, ip);
+ // restore d0
+ LoadSimd128(d0, MemOperand(sp), ip);
+ lay(sp, MemOperand(sp, kSimd128Size));
+}
+
+void TurboAssembler::ResetSpeculationPoisonRegister() {
+ mov(kSpeculationPoisonRegister, Operand(-1));
+}
+
+void TurboAssembler::ComputeCodeStartAddress(Register dst) {
+ larl(dst, Operand(-pc_offset() / 2));
+}
+
+void TurboAssembler::LoadPC(Register dst) {
+ Label current_pc;
+ larl(dst, ¤t_pc);
+ bind(¤t_pc);
+}
+
+void TurboAssembler::JumpIfEqual(Register x, int32_t y, Label* dest) {
+ Cmp32(x, Operand(y));
+ beq(dest);
+}
+
+void TurboAssembler::JumpIfLessThan(Register x, int32_t y, Label* dest) {
+ Cmp32(x, Operand(y));
+ blt(dest);
+}
+
+void TurboAssembler::LoadEntryFromBuiltinIndex(Register builtin_index) {
+ STATIC_ASSERT(kSystemPointerSize == 8);
+ STATIC_ASSERT(kSmiTagSize == 1);
+ STATIC_ASSERT(kSmiTag == 0);
+ // The builtin_index register contains the builtin index as a Smi.
+ if (SmiValuesAre32Bits()) {
+ ShiftRightArithP(builtin_index, builtin_index,
+ Operand(kSmiShift - kSystemPointerSizeLog2));
+ } else {
+ DCHECK(SmiValuesAre31Bits());
+ ShiftLeftP(builtin_index, builtin_index,
+ Operand(kSystemPointerSizeLog2 - kSmiShift));
+ }
+ LoadP(builtin_index, MemOperand(kRootRegister, builtin_index,
+ IsolateData::builtin_entry_table_offset()));
+}
+
+void TurboAssembler::CallBuiltinByIndex(Register builtin_index) {
+ LoadEntryFromBuiltinIndex(builtin_index);
+ Call(builtin_index);
+}
+
+void TurboAssembler::LoadCodeObjectEntry(Register destination,
+ Register code_object) {
+ // Code objects are called differently depending on whether we are generating
+ // builtin code (which will later be embedded into the binary) or compiling
+ // user JS code at runtime.
+ // * Builtin code runs in --jitless mode and thus must not call into on-heap
+ // Code targets. Instead, we dispatch through the builtins entry table.
+ // * Codegen at runtime does not have this restriction and we can use the
+ // shorter, branchless instruction sequence. The assumption here is that
+ // targets are usually generated code and not builtin Code objects.
+
+ if (options().isolate_independent_code) {
+ DCHECK(root_array_available());
+ Label if_code_is_off_heap, out;
+
+ Register scratch = r1;
+
+ DCHECK(!AreAliased(destination, scratch));
+ DCHECK(!AreAliased(code_object, scratch));
+
+ // Check whether the Code object is an off-heap trampoline. If so, call its
+ // (off-heap) entry point directly without going through the (on-heap)
+ // trampoline. Otherwise, just call the Code object as always.
+ LoadW(scratch, FieldMemOperand(code_object, Code::kFlagsOffset));
+ tmlh(scratch, Operand(Code::IsOffHeapTrampoline::kMask >> 16));
+ bne(&if_code_is_off_heap);
+
+ // Not an off-heap trampoline, the entry point is at
+ // Code::raw_instruction_start().
+ AddP(destination, code_object, Operand(Code::kHeaderSize - kHeapObjectTag));
+ b(&out);
+
+ // An off-heap trampoline, the entry point is loaded from the builtin entry
+ // table.
+ bind(&if_code_is_off_heap);
+ LoadW(scratch, FieldMemOperand(code_object, Code::kBuiltinIndexOffset));
+ ShiftLeftP(destination, scratch, Operand(kSystemPointerSizeLog2));
+ AddP(destination, destination, kRootRegister);
+ LoadP(destination,
+ MemOperand(destination, IsolateData::builtin_entry_table_offset()));
+
+ bind(&out);
+ } else {
+ AddP(destination, code_object, Operand(Code::kHeaderSize - kHeapObjectTag));
+ }
+}
+
+void TurboAssembler::CallCodeObject(Register code_object) {
+ LoadCodeObjectEntry(code_object, code_object);
+ Call(code_object);
+}
+
+void TurboAssembler::JumpCodeObject(Register code_object) {
+ LoadCodeObjectEntry(code_object, code_object);
+ Jump(code_object);
+}
+
+void TurboAssembler::StoreReturnAddressAndCall(Register target) {
+ // This generates the final instruction sequence for calls to C functions
+ // once an exit frame has been constructed.
+ //
+ // Note that this assumes the caller code (i.e. the Code object currently
+ // being generated) is immovable or that the callee function cannot trigger
+ // GC, since the callee function will return to it.
+
+ Label return_label;
+ larl(r14, &return_label); // Generate the return addr of call later.
+ StoreP(r14, MemOperand(sp, kStackFrameRASlot * kSystemPointerSize));
+
+ // zLinux ABI requires caller's frame to have sufficient space for callee
+ // preserved regsiter save area.
+ b(target);
+ bind(&return_label);
+}
+
+void TurboAssembler::CallForDeoptimization(Builtins::Name target, int,
+ Label* exit, DeoptimizeKind kind,
+ Label*) {
+ LoadP(ip, MemOperand(kRootRegister,
+ IsolateData::builtin_entry_slot_offset(target)));
+ Call(ip);
+ DCHECK_EQ(SizeOfCodeGeneratedSince(exit),
+ (kind == DeoptimizeKind::kLazy)
+ ? Deoptimizer::kLazyDeoptExitSize
+ : Deoptimizer::kNonLazyDeoptExitSize);
+ USE(exit, kind);
+}
+
+void TurboAssembler::Trap() { stop(); }
+void TurboAssembler::DebugBreak() { stop(); }
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_S390
diff --git a/src/codegen/s390/macro-assembler-s390.h b/src/codegen/s390/macro-assembler-s390.h
new file mode 100644
index 0000000..f81dfb5
--- /dev/null
+++ b/src/codegen/s390/macro-assembler-s390.h
@@ -0,0 +1,1356 @@
+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef INCLUDED_FROM_MACRO_ASSEMBLER_H
+#error This header must be included via macro-assembler.h
+#endif
+
+#ifndef V8_CODEGEN_S390_MACRO_ASSEMBLER_S390_H_
+#define V8_CODEGEN_S390_MACRO_ASSEMBLER_S390_H_
+
+#include "src/codegen/bailout-reason.h"
+#include "src/codegen/s390/assembler-s390.h"
+#include "src/common/globals.h"
+#include "src/objects/contexts.h"
+
+namespace v8 {
+namespace internal {
+
+// ----------------------------------------------------------------------------
+// Static helper functions
+
+// Generate a MemOperand for loading a field from an object.
+inline MemOperand FieldMemOperand(Register object, int offset) {
+ return MemOperand(object, offset - kHeapObjectTag);
+}
+
+// Generate a MemOperand for loading a field from an object.
+inline MemOperand FieldMemOperand(Register object, Register index, int offset) {
+ return MemOperand(object, index, offset - kHeapObjectTag);
+}
+
+enum RememberedSetAction { EMIT_REMEMBERED_SET, OMIT_REMEMBERED_SET };
+enum SmiCheck { INLINE_SMI_CHECK, OMIT_SMI_CHECK };
+enum LinkRegisterStatus { kLRHasNotBeenSaved, kLRHasBeenSaved };
+
+Register GetRegisterThatIsNotOneOf(Register reg1, Register reg2 = no_reg,
+ Register reg3 = no_reg,
+ Register reg4 = no_reg,
+ Register reg5 = no_reg,
+ Register reg6 = no_reg);
+
+// These exist to provide portability between 32 and 64bit
+#if V8_TARGET_ARCH_S390X
+
+// The length of the arithmetic operation is the length
+// of the register.
+
+// Length:
+// H = halfword
+// W = word
+
+// arithmetics and bitwise
+#define AddMI agsi
+#define AddRR agr
+#define SubRR sgr
+#define AndRR ngr
+#define OrRR ogr
+#define XorRR xgr
+#define LoadComplementRR lcgr
+#define LoadNegativeRR lngr
+
+// Distinct Operands
+#define AddP_RRR agrk
+#define AddPImm_RRI aghik
+#define AddLogicalP_RRR algrk
+#define SubP_RRR sgrk
+#define SubLogicalP_RRR slgrk
+#define AndP_RRR ngrk
+#define OrP_RRR ogrk
+#define XorP_RRR xgrk
+
+// Load / Store
+#define LoadRR lgr
+#define LoadAndTestRR ltgr
+#define LoadImmP lghi
+
+// Compare
+#define CmpPH cghi
+#define CmpLogicalPW clgfi
+
+// Shifts
+#define ShiftLeftP sllg
+#define ShiftRightP srlg
+#define ShiftLeftArithP slag
+#define ShiftRightArithP srag
+#else
+
+// arithmetics and bitwise
+// Reg2Reg
+#define AddMI asi
+#define AddRR ar
+#define SubRR sr
+#define AndRR nr
+#define OrRR or_z
+#define XorRR xr
+#define LoadComplementRR lcr
+#define LoadNegativeRR lnr
+
+// Distinct Operands
+#define AddP_RRR ark
+#define AddPImm_RRI ahik
+#define AddLogicalP_RRR alrk
+#define SubP_RRR srk
+#define SubLogicalP_RRR slrk
+#define AndP_RRR nrk
+#define OrP_RRR ork
+#define XorP_RRR xrk
+
+// Load / Store
+#define LoadRR lr
+#define LoadAndTestRR ltr
+#define LoadImmP lhi
+
+// Compare
+#define CmpPH chi
+#define CmpLogicalPW clfi
+
+// Shifts
+#define ShiftLeftP ShiftLeft
+#define ShiftRightP ShiftRight
+#define ShiftLeftArithP ShiftLeftArith
+#define ShiftRightArithP ShiftRightArith
+
+#endif
+
+class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase {
+ public:
+ using TurboAssemblerBase::TurboAssemblerBase;
+
+ void LoadFromConstantsTable(Register destination,
+ int constant_index) override;
+ void LoadRootRegisterOffset(Register destination, intptr_t offset) override;
+ void LoadRootRelative(Register destination, int32_t offset) override;
+
+ // Jump, Call, and Ret pseudo instructions implementing inter-working.
+ void Jump(Register target, Condition cond = al);
+ void Jump(Address target, RelocInfo::Mode rmode, Condition cond = al);
+ void Jump(Handle<Code> code, RelocInfo::Mode rmode, Condition cond = al);
+ void Jump(const ExternalReference& reference) override;
+ // Jump the register contains a smi.
+ inline void JumpIfSmi(Register value, Label* smi_label) {
+ TestIfSmi(value);
+ beq(smi_label /*, cr0*/); // branch if SMI
+ }
+ void JumpIfEqual(Register x, int32_t y, Label* dest);
+ void JumpIfLessThan(Register x, int32_t y, Label* dest);
+
+ void Call(Register target);
+ void Call(Address target, RelocInfo::Mode rmode, Condition cond = al);
+ void Call(Handle<Code> code, RelocInfo::Mode rmode = RelocInfo::CODE_TARGET,
+ Condition cond = al);
+ void Ret() { b(r14); }
+ void Ret(Condition cond) { b(cond, r14); }
+
+ void CallForDeoptimization(Builtins::Name target, int deopt_id, Label* exit,
+ DeoptimizeKind kind,
+ Label* jump_deoptimization_entry_label);
+
+ // Emit code to discard a non-negative number of pointer-sized elements
+ // from the stack, clobbering only the sp register.
+ void Drop(int count);
+ void Drop(Register count, Register scratch = r0);
+
+ void Ret(int drop) {
+ Drop(drop);
+ Ret();
+ }
+
+ void Call(Label* target);
+
+ // Load the builtin given by the Smi in |builtin_index| into the same
+ // register.
+ void LoadEntryFromBuiltinIndex(Register builtin_index);
+ void LoadCodeObjectEntry(Register destination, Register code_object) override;
+ void CallCodeObject(Register code_object) override;
+ void JumpCodeObject(Register code_object) override;
+
+ void CallBuiltinByIndex(Register builtin_index) override;
+
+ // Register move. May do nothing if the registers are identical.
+ void Move(Register dst, Smi smi) { LoadSmiLiteral(dst, smi); }
+ void Move(Register dst, Handle<HeapObject> source,
+ RelocInfo::Mode rmode = RelocInfo::FULL_EMBEDDED_OBJECT);
+ void Move(Register dst, ExternalReference reference);
+ void Move(Register dst, Register src, Condition cond = al);
+ void Move(DoubleRegister dst, DoubleRegister src);
+
+ void MoveChar(const MemOperand& opnd1, const MemOperand& opnd2,
+ const Operand& length);
+
+ void CompareLogicalChar(const MemOperand& opnd1, const MemOperand& opnd2,
+ const Operand& length);
+
+ void ExclusiveOrChar(const MemOperand& opnd1, const MemOperand& opnd2,
+ const Operand& length);
+
+ void RotateInsertSelectBits(Register dst, Register src,
+ const Operand& startBit, const Operand& endBit,
+ const Operand& shiftAmt, bool zeroBits);
+
+ void BranchRelativeOnIdxHighP(Register dst, Register inc, Label* L);
+
+ void SaveRegisters(RegList registers);
+ void RestoreRegisters(RegList registers);
+
+ void CallRecordWriteStub(Register object, Register address,
+ RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode);
+ void CallRecordWriteStub(Register object, Register address,
+ RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode, Address wasm_target);
+ void CallEphemeronKeyBarrier(Register object, Register address,
+ SaveFPRegsMode fp_mode);
+
+ void MultiPush(RegList regs, Register location = sp);
+ void MultiPop(RegList regs, Register location = sp);
+
+ void MultiPushDoubles(RegList dregs, Register location = sp);
+ void MultiPopDoubles(RegList dregs, Register location = sp);
+
+ // Calculate how much stack space (in bytes) are required to store caller
+ // registers excluding those specified in the arguments.
+ int RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
+ Register exclusion1 = no_reg,
+ Register exclusion2 = no_reg,
+ Register exclusion3 = no_reg) const;
+
+ // Push caller saved registers on the stack, and return the number of bytes
+ // stack pointer is adjusted.
+ int PushCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1 = no_reg,
+ Register exclusion2 = no_reg,
+ Register exclusion3 = no_reg);
+ // Restore caller saved registers from the stack, and return the number of
+ // bytes stack pointer is adjusted.
+ int PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1 = no_reg,
+ Register exclusion2 = no_reg,
+ Register exclusion3 = no_reg);
+
+ // Load an object from the root table.
+ void LoadRoot(Register destination, RootIndex index) override {
+ LoadRoot(destination, index, al);
+ }
+ void LoadRoot(Register destination, RootIndex index, Condition cond);
+ //--------------------------------------------------------------------------
+ // S390 Macro Assemblers for Instructions
+ //--------------------------------------------------------------------------
+
+ // Arithmetic Operations
+
+ // Add (Register - Immediate)
+ void Add32(Register dst, const Operand& imm);
+ void Add32_RI(Register dst, const Operand& imm);
+ void AddP(Register dst, const Operand& imm);
+ void Add32(Register dst, Register src, const Operand& imm);
+ void Add32_RRI(Register dst, Register src, const Operand& imm);
+ void AddP(Register dst, Register src, const Operand& imm);
+
+ // Add (Register - Register)
+ void Add32(Register dst, Register src);
+ void AddP(Register dst, Register src);
+ void AddP_ExtendSrc(Register dst, Register src);
+ void Add32(Register dst, Register src1, Register src2);
+ void AddP(Register dst, Register src1, Register src2);
+ void AddP_ExtendSrc(Register dst, Register src1, Register src2);
+
+ // Add (Register - Mem)
+ void Add32(Register dst, const MemOperand& opnd);
+ void AddP(Register dst, const MemOperand& opnd);
+ void AddP_ExtendSrc(Register dst, const MemOperand& opnd);
+
+ // Add (Mem - Immediate)
+ void Add32(const MemOperand& opnd, const Operand& imm);
+ void AddP(const MemOperand& opnd, const Operand& imm);
+
+ // Add Logical (Register - Register)
+ void AddLogical32(Register dst, Register src1, Register src2);
+
+ // Add Logical With Carry (Register - Register)
+ void AddLogicalWithCarry32(Register dst, Register src1, Register src2);
+
+ // Add Logical (Register - Immediate)
+ void AddLogical(Register dst, const Operand& imm);
+ void AddLogicalP(Register dst, const Operand& imm);
+
+ // Add Logical (Register - Mem)
+ void AddLogical(Register dst, const MemOperand& opnd);
+ void AddLogicalP(Register dst, const MemOperand& opnd);
+
+ // Subtract (Register - Immediate)
+ void Sub32(Register dst, const Operand& imm);
+ void Sub32_RI(Register dst, const Operand& imm) { Sub32(dst, imm); }
+ void SubP(Register dst, const Operand& imm);
+ void Sub32(Register dst, Register src, const Operand& imm);
+ void Sub32_RRI(Register dst, Register src, const Operand& imm) {
+ Sub32(dst, src, imm);
+ }
+ void SubP(Register dst, Register src, const Operand& imm);
+
+ // Subtract (Register - Register)
+ void Sub32(Register dst, Register src);
+ void SubP(Register dst, Register src);
+ void SubP_ExtendSrc(Register dst, Register src);
+ void Sub32(Register dst, Register src1, Register src2);
+ void SubP(Register dst, Register src1, Register src2);
+ void SubP_ExtendSrc(Register dst, Register src1, Register src2);
+
+ // Subtract (Register - Mem)
+ void Sub32(Register dst, const MemOperand& opnd);
+ void SubP(Register dst, const MemOperand& opnd);
+ void SubP_ExtendSrc(Register dst, const MemOperand& opnd);
+ void LoadAndSub32(Register dst, Register src, const MemOperand& opnd);
+ void LoadAndSub64(Register dst, Register src, const MemOperand& opnd);
+
+ // Subtract Logical (Register - Mem)
+ void SubLogical(Register dst, const MemOperand& opnd);
+ void SubLogicalP(Register dst, const MemOperand& opnd);
+ void SubLogicalP_ExtendSrc(Register dst, const MemOperand& opnd);
+ // Subtract Logical 32-bit
+ void SubLogical32(Register dst, Register src1, Register src2);
+ // Subtract Logical With Borrow 32-bit
+ void SubLogicalWithBorrow32(Register dst, Register src1, Register src2);
+
+ // Multiply
+ void MulP(Register dst, const Operand& opnd);
+ void MulP(Register dst, Register src);
+ void MulP(Register dst, const MemOperand& opnd);
+ void Mul(Register dst, Register src1, Register src2);
+ void Mul32(Register dst, const MemOperand& src1);
+ void Mul32(Register dst, Register src1);
+ void Mul32(Register dst, const Operand& src1);
+ void MulHigh32(Register dst, Register src1, const MemOperand& src2);
+ void MulHigh32(Register dst, Register src1, Register src2);
+ void MulHigh32(Register dst, Register src1, const Operand& src2);
+ void MulHighU32(Register dst, Register src1, const MemOperand& src2);
+ void MulHighU32(Register dst, Register src1, Register src2);
+ void MulHighU32(Register dst, Register src1, const Operand& src2);
+ void Mul32WithOverflowIfCCUnequal(Register dst, Register src1,
+ const MemOperand& src2);
+ void Mul32WithOverflowIfCCUnequal(Register dst, Register src1, Register src2);
+ void Mul32WithOverflowIfCCUnequal(Register dst, Register src1,
+ const Operand& src2);
+ void Mul64(Register dst, const MemOperand& src1);
+ void Mul64(Register dst, Register src1);
+ void Mul64(Register dst, const Operand& src1);
+ void MulPWithCondition(Register dst, Register src1, Register src2);
+
+ // Divide
+ void DivP(Register dividend, Register divider);
+ void Div32(Register dst, Register src1, const MemOperand& src2);
+ void Div32(Register dst, Register src1, Register src2);
+ void DivU32(Register dst, Register src1, const MemOperand& src2);
+ void DivU32(Register dst, Register src1, Register src2);
+ void Div64(Register dst, Register src1, const MemOperand& src2);
+ void Div64(Register dst, Register src1, Register src2);
+ void DivU64(Register dst, Register src1, const MemOperand& src2);
+ void DivU64(Register dst, Register src1, Register src2);
+
+ // Mod
+ void Mod32(Register dst, Register src1, const MemOperand& src2);
+ void Mod32(Register dst, Register src1, Register src2);
+ void ModU32(Register dst, Register src1, const MemOperand& src2);
+ void ModU32(Register dst, Register src1, Register src2);
+ void Mod64(Register dst, Register src1, const MemOperand& src2);
+ void Mod64(Register dst, Register src1, Register src2);
+ void ModU64(Register dst, Register src1, const MemOperand& src2);
+ void ModU64(Register dst, Register src1, Register src2);
+
+ // Square root
+ void Sqrt(DoubleRegister result, DoubleRegister input);
+ void Sqrt(DoubleRegister result, const MemOperand& input);
+
+ // Compare
+ void Cmp32(Register src1, Register src2);
+ void CmpP(Register src1, Register src2);
+ void Cmp32(Register dst, const Operand& opnd);
+ void CmpP(Register dst, const Operand& opnd);
+ void Cmp32(Register dst, const MemOperand& opnd);
+ void CmpP(Register dst, const MemOperand& opnd);
+ void CmpAndSwap(Register old_val, Register new_val, const MemOperand& opnd);
+ void CmpAndSwap64(Register old_val, Register new_val, const MemOperand& opnd);
+
+ // Compare Logical
+ void CmpLogical32(Register src1, Register src2);
+ void CmpLogicalP(Register src1, Register src2);
+ void CmpLogical32(Register src1, const Operand& opnd);
+ void CmpLogicalP(Register src1, const Operand& opnd);
+ void CmpLogical32(Register dst, const MemOperand& opnd);
+ void CmpLogicalP(Register dst, const MemOperand& opnd);
+
+ // Compare Logical Byte (CLI/CLIY)
+ void CmpLogicalByte(const MemOperand& mem, const Operand& imm);
+
+ // Load 32bit
+ void Load(Register dst, const MemOperand& opnd);
+ void Load(Register dst, const Operand& opnd);
+ void LoadW(Register dst, const MemOperand& opnd, Register scratch = no_reg);
+ void LoadW(Register dst, Register src);
+ void LoadlW(Register dst, const MemOperand& opnd, Register scratch = no_reg);
+ void LoadlW(Register dst, Register src);
+ void LoadLogicalHalfWordP(Register dst, const MemOperand& opnd);
+ void LoadLogicalHalfWordP(Register dst, Register src);
+ void LoadB(Register dst, const MemOperand& opnd);
+ void LoadB(Register dst, Register src);
+ void LoadlB(Register dst, const MemOperand& opnd);
+ void LoadlB(Register dst, Register src);
+
+ void LoadLogicalReversedWordP(Register dst, const MemOperand& opnd);
+ void LoadLogicalReversedHalfWordP(Register dst, const MemOperand& opnd);
+
+ // Load And Test
+ void LoadAndTest32(Register dst, Register src);
+ void LoadAndTestP_ExtendSrc(Register dst, Register src);
+ void LoadAndTestP(Register dst, Register src);
+
+ void LoadAndTest32(Register dst, const MemOperand& opnd);
+ void LoadAndTestP(Register dst, const MemOperand& opnd);
+
+ // Load Floating Point
+ void LoadDouble(DoubleRegister dst, const MemOperand& opnd);
+ void LoadFloat32(DoubleRegister dst, const MemOperand& opnd);
+ void LoadFloat32ConvertToDouble(DoubleRegister dst, const MemOperand& mem);
+ void LoadSimd128(Simd128Register dst, const MemOperand& mem,
+ Register scratch);
+
+ void AddFloat32(DoubleRegister dst, const MemOperand& opnd,
+ DoubleRegister scratch);
+ void AddFloat64(DoubleRegister dst, const MemOperand& opnd,
+ DoubleRegister scratch);
+ void SubFloat32(DoubleRegister dst, const MemOperand& opnd,
+ DoubleRegister scratch);
+ void SubFloat64(DoubleRegister dst, const MemOperand& opnd,
+ DoubleRegister scratch);
+ void MulFloat32(DoubleRegister dst, const MemOperand& opnd,
+ DoubleRegister scratch);
+ void MulFloat64(DoubleRegister dst, const MemOperand& opnd,
+ DoubleRegister scratch);
+ void DivFloat32(DoubleRegister dst, const MemOperand& opnd,
+ DoubleRegister scratch);
+ void DivFloat64(DoubleRegister dst, const MemOperand& opnd,
+ DoubleRegister scratch);
+ void LoadFloat32ToDouble(DoubleRegister dst, const MemOperand& opnd,
+ DoubleRegister scratch);
+
+ // Load On Condition
+ void LoadOnConditionP(Condition cond, Register dst, Register src);
+
+ void LoadPositiveP(Register result, Register input);
+ void LoadPositive32(Register result, Register input);
+
+ // Store Floating Point
+ void StoreDouble(DoubleRegister dst, const MemOperand& opnd);
+ void StoreFloat32(DoubleRegister dst, const MemOperand& opnd);
+ void StoreDoubleAsFloat32(DoubleRegister src, const MemOperand& mem,
+ DoubleRegister scratch);
+ void StoreSimd128(Simd128Register src, const MemOperand& mem,
+ Register scratch);
+
+ void Branch(Condition c, const Operand& opnd);
+ void BranchOnCount(Register r1, Label* l);
+
+ // Shifts
+ void ShiftLeft(Register dst, Register src, Register val);
+ void ShiftLeft(Register dst, Register src, const Operand& val);
+ void ShiftRight(Register dst, Register src, Register val);
+ void ShiftRight(Register dst, Register src, const Operand& val);
+ void ShiftLeftArith(Register dst, Register src, Register shift);
+ void ShiftLeftArith(Register dst, Register src, const Operand& val);
+ void ShiftRightArith(Register dst, Register src, Register shift);
+ void ShiftRightArith(Register dst, Register src, const Operand& val);
+
+ void ClearRightImm(Register dst, Register src, const Operand& val);
+
+ // Bitwise operations
+ void And(Register dst, Register src);
+ void AndP(Register dst, Register src);
+ void And(Register dst, Register src1, Register src2);
+ void AndP(Register dst, Register src1, Register src2);
+ void And(Register dst, const MemOperand& opnd);
+ void AndP(Register dst, const MemOperand& opnd);
+ void And(Register dst, const Operand& opnd);
+ void AndP(Register dst, const Operand& opnd);
+ void And(Register dst, Register src, const Operand& opnd);
+ void AndP(Register dst, Register src, const Operand& opnd);
+ void Or(Register dst, Register src);
+ void OrP(Register dst, Register src);
+ void Or(Register dst, Register src1, Register src2);
+ void OrP(Register dst, Register src1, Register src2);
+ void Or(Register dst, const MemOperand& opnd);
+ void OrP(Register dst, const MemOperand& opnd);
+ void Or(Register dst, const Operand& opnd);
+ void OrP(Register dst, const Operand& opnd);
+ void Or(Register dst, Register src, const Operand& opnd);
+ void OrP(Register dst, Register src, const Operand& opnd);
+ void Xor(Register dst, Register src);
+ void XorP(Register dst, Register src);
+ void Xor(Register dst, Register src1, Register src2);
+ void XorP(Register dst, Register src1, Register src2);
+ void Xor(Register dst, const MemOperand& opnd);
+ void XorP(Register dst, const MemOperand& opnd);
+ void Xor(Register dst, const Operand& opnd);
+ void XorP(Register dst, const Operand& opnd);
+ void Xor(Register dst, Register src, const Operand& opnd);
+ void XorP(Register dst, Register src, const Operand& opnd);
+ void Popcnt32(Register dst, Register src);
+ void Not32(Register dst, Register src = no_reg);
+ void Not64(Register dst, Register src = no_reg);
+ void NotP(Register dst, Register src = no_reg);
+
+#ifdef V8_TARGET_ARCH_S390X
+ void Popcnt64(Register dst, Register src);
+#endif
+
+ void mov(Register dst, const Operand& src);
+
+ void CleanUInt32(Register x) {
+#ifdef V8_TARGET_ARCH_S390X
+ llgfr(x, x);
+#endif
+ }
+
+ void push(DoubleRegister src) {
+ lay(sp, MemOperand(sp, -kSystemPointerSize));
+ StoreDouble(src, MemOperand(sp));
+ }
+
+ void push(Register src) {
+ lay(sp, MemOperand(sp, -kSystemPointerSize));
+ StoreP(src, MemOperand(sp));
+ }
+
+ void pop(DoubleRegister dst) {
+ LoadDouble(dst, MemOperand(sp));
+ la(sp, MemOperand(sp, kSystemPointerSize));
+ }
+
+ void pop(Register dst) {
+ LoadP(dst, MemOperand(sp));
+ la(sp, MemOperand(sp, kSystemPointerSize));
+ }
+
+ void pop() { la(sp, MemOperand(sp, kSystemPointerSize)); }
+
+ void Push(Register src) { push(src); }
+
+ // Push a handle.
+ void Push(Handle<HeapObject> handle);
+ void Push(Smi smi);
+
+ // Push two registers. Pushes leftmost register first (to highest address).
+ void Push(Register src1, Register src2) {
+ lay(sp, MemOperand(sp, -kSystemPointerSize * 2));
+ StoreP(src1, MemOperand(sp, kSystemPointerSize));
+ StoreP(src2, MemOperand(sp, 0));
+ }
+
+ // Push three registers. Pushes leftmost register first (to highest address).
+ void Push(Register src1, Register src2, Register src3) {
+ lay(sp, MemOperand(sp, -kSystemPointerSize * 3));
+ StoreP(src1, MemOperand(sp, kSystemPointerSize * 2));
+ StoreP(src2, MemOperand(sp, kSystemPointerSize));
+ StoreP(src3, MemOperand(sp, 0));
+ }
+
+ // Push four registers. Pushes leftmost register first (to highest address).
+ void Push(Register src1, Register src2, Register src3, Register src4) {
+ lay(sp, MemOperand(sp, -kSystemPointerSize * 4));
+ StoreP(src1, MemOperand(sp, kSystemPointerSize * 3));
+ StoreP(src2, MemOperand(sp, kSystemPointerSize * 2));
+ StoreP(src3, MemOperand(sp, kSystemPointerSize));
+ StoreP(src4, MemOperand(sp, 0));
+ }
+
+ // Push five registers. Pushes leftmost register first (to highest address).
+ void Push(Register src1, Register src2, Register src3, Register src4,
+ Register src5) {
+ DCHECK(src1 != src2);
+ DCHECK(src1 != src3);
+ DCHECK(src2 != src3);
+ DCHECK(src1 != src4);
+ DCHECK(src2 != src4);
+ DCHECK(src3 != src4);
+ DCHECK(src1 != src5);
+ DCHECK(src2 != src5);
+ DCHECK(src3 != src5);
+ DCHECK(src4 != src5);
+
+ lay(sp, MemOperand(sp, -kSystemPointerSize * 5));
+ StoreP(src1, MemOperand(sp, kSystemPointerSize * 4));
+ StoreP(src2, MemOperand(sp, kSystemPointerSize * 3));
+ StoreP(src3, MemOperand(sp, kSystemPointerSize * 2));
+ StoreP(src4, MemOperand(sp, kSystemPointerSize));
+ StoreP(src5, MemOperand(sp, 0));
+ }
+
+ enum PushArrayOrder { kNormal, kReverse };
+ void PushArray(Register array, Register size, Register scratch,
+ Register scratch2, PushArrayOrder order = kNormal);
+
+ void Pop(Register dst) { pop(dst); }
+
+ // Pop two registers. Pops rightmost register first (from lower address).
+ void Pop(Register src1, Register src2) {
+ LoadP(src2, MemOperand(sp, 0));
+ LoadP(src1, MemOperand(sp, kSystemPointerSize));
+ la(sp, MemOperand(sp, 2 * kSystemPointerSize));
+ }
+
+ // Pop three registers. Pops rightmost register first (from lower address).
+ void Pop(Register src1, Register src2, Register src3) {
+ LoadP(src3, MemOperand(sp, 0));
+ LoadP(src2, MemOperand(sp, kSystemPointerSize));
+ LoadP(src1, MemOperand(sp, 2 * kSystemPointerSize));
+ la(sp, MemOperand(sp, 3 * kSystemPointerSize));
+ }
+
+ // Pop four registers. Pops rightmost register first (from lower address).
+ void Pop(Register src1, Register src2, Register src3, Register src4) {
+ LoadP(src4, MemOperand(sp, 0));
+ LoadP(src3, MemOperand(sp, kSystemPointerSize));
+ LoadP(src2, MemOperand(sp, 2 * kSystemPointerSize));
+ LoadP(src1, MemOperand(sp, 3 * kSystemPointerSize));
+ la(sp, MemOperand(sp, 4 * kSystemPointerSize));
+ }
+
+ // Pop five registers. Pops rightmost register first (from lower address).
+ void Pop(Register src1, Register src2, Register src3, Register src4,
+ Register src5) {
+ LoadP(src5, MemOperand(sp, 0));
+ LoadP(src4, MemOperand(sp, kSystemPointerSize));
+ LoadP(src3, MemOperand(sp, 2 * kSystemPointerSize));
+ LoadP(src2, MemOperand(sp, 3 * kSystemPointerSize));
+ LoadP(src1, MemOperand(sp, 4 * kSystemPointerSize));
+ la(sp, MemOperand(sp, 5 * kSystemPointerSize));
+ }
+
+ // Push a fixed frame, consisting of lr, fp, constant pool.
+ void PushCommonFrame(Register marker_reg = no_reg);
+
+ // Push a standard frame, consisting of lr, fp, constant pool,
+ // context and JS function
+ void PushStandardFrame(Register function_reg);
+
+ void PopCommonFrame(Register marker_reg = no_reg);
+
+ // Restore caller's frame pointer and return address prior to being
+ // overwritten by tail call stack preparation.
+ void RestoreFrameStateForTailCall();
+
+ void InitializeRootRegister() {
+ ExternalReference isolate_root = ExternalReference::isolate_root(isolate());
+ mov(kRootRegister, Operand(isolate_root));
+ }
+
+ // If the value is a NaN, canonicalize the value else, do nothing.
+ void CanonicalizeNaN(const DoubleRegister dst, const DoubleRegister src);
+ void CanonicalizeNaN(const DoubleRegister value) {
+ CanonicalizeNaN(value, value);
+ }
+
+ // Converts the integer (untagged smi) in |src| to a double, storing
+ // the result to |dst|
+ void ConvertIntToDouble(DoubleRegister dst, Register src);
+
+ // Converts the unsigned integer (untagged smi) in |src| to
+ // a double, storing the result to |dst|
+ void ConvertUnsignedIntToDouble(DoubleRegister dst, Register src);
+
+ // Converts the integer (untagged smi) in |src| to
+ // a float, storing the result in |dst|
+ void ConvertIntToFloat(DoubleRegister dst, Register src);
+
+ // Converts the unsigned integer (untagged smi) in |src| to
+ // a float, storing the result in |dst|
+ void ConvertUnsignedIntToFloat(DoubleRegister dst, Register src);
+
+ void ConvertInt64ToFloat(DoubleRegister double_dst, Register src);
+ void ConvertInt64ToDouble(DoubleRegister double_dst, Register src);
+ void ConvertUnsignedInt64ToFloat(DoubleRegister double_dst, Register src);
+ void ConvertUnsignedInt64ToDouble(DoubleRegister double_dst, Register src);
+
+ void MovIntToFloat(DoubleRegister dst, Register src);
+ void MovFloatToInt(Register dst, DoubleRegister src);
+ void MovDoubleToInt64(Register dst, DoubleRegister src);
+ void MovInt64ToDouble(DoubleRegister dst, Register src);
+ // Converts the double_input to an integer. Note that, upon return,
+ // the contents of double_dst will also hold the fixed point representation.
+ void ConvertFloat32ToInt64(const Register dst,
+ const DoubleRegister double_input,
+ FPRoundingMode rounding_mode = kRoundToZero);
+
+ // Converts the double_input to an integer. Note that, upon return,
+ // the contents of double_dst will also hold the fixed point representation.
+ void ConvertDoubleToInt64(const Register dst,
+ const DoubleRegister double_input,
+ FPRoundingMode rounding_mode = kRoundToZero);
+ void ConvertDoubleToInt32(const Register dst,
+ const DoubleRegister double_input,
+ FPRoundingMode rounding_mode = kRoundToZero);
+
+ void ConvertFloat32ToInt32(const Register result,
+ const DoubleRegister double_input,
+ FPRoundingMode rounding_mode);
+ void ConvertFloat32ToUnsignedInt32(
+ const Register result, const DoubleRegister double_input,
+ FPRoundingMode rounding_mode = kRoundToZero);
+ // Converts the double_input to an unsigned integer. Note that, upon return,
+ // the contents of double_dst will also hold the fixed point representation.
+ void ConvertDoubleToUnsignedInt64(
+ const Register dst, const DoubleRegister double_input,
+ FPRoundingMode rounding_mode = kRoundToZero);
+ void ConvertDoubleToUnsignedInt32(
+ const Register dst, const DoubleRegister double_input,
+ FPRoundingMode rounding_mode = kRoundToZero);
+ void ConvertFloat32ToUnsignedInt64(
+ const Register result, const DoubleRegister double_input,
+ FPRoundingMode rounding_mode = kRoundToZero);
+
+#if !V8_TARGET_ARCH_S390X
+ void ShiftLeftPair(Register dst_low, Register dst_high, Register src_low,
+ Register src_high, Register scratch, Register shift);
+ void ShiftLeftPair(Register dst_low, Register dst_high, Register src_low,
+ Register src_high, uint32_t shift);
+ void ShiftRightPair(Register dst_low, Register dst_high, Register src_low,
+ Register src_high, Register scratch, Register shift);
+ void ShiftRightPair(Register dst_low, Register dst_high, Register src_low,
+ Register src_high, uint32_t shift);
+ void ShiftRightArithPair(Register dst_low, Register dst_high,
+ Register src_low, Register src_high,
+ Register scratch, Register shift);
+ void ShiftRightArithPair(Register dst_low, Register dst_high,
+ Register src_low, Register src_high, uint32_t shift);
+#endif
+
+ // Generates function and stub prologue code.
+ void StubPrologue(StackFrame::Type type, Register base = no_reg,
+ int prologue_offset = 0);
+ void Prologue(Register base, int prologue_offset = 0);
+
+ // Get the actual activation frame alignment for target environment.
+ static int ActivationFrameAlignment();
+ // ----------------------------------------------------------------
+ // new S390 macro-assembler interfaces that are slightly higher level
+ // than assembler-s390 and may generate variable length sequences
+
+ // load a literal signed int value <value> to GPR <dst>
+ void LoadIntLiteral(Register dst, int value);
+
+ // load an SMI value <value> to GPR <dst>
+ void LoadSmiLiteral(Register dst, Smi smi);
+
+ // load a literal double value <value> to FPR <result>
+ void LoadDoubleLiteral(DoubleRegister result, double value, Register scratch);
+ void LoadDoubleLiteral(DoubleRegister result, uint64_t value,
+ Register scratch);
+
+ void LoadFloat32Literal(DoubleRegister result, float value, Register scratch);
+
+ void StoreW(Register src, const MemOperand& mem, Register scratch = no_reg);
+
+ void LoadHalfWordP(Register dst, Register src);
+
+ void LoadHalfWordP(Register dst, const MemOperand& mem,
+ Register scratch = no_reg);
+
+ void StoreHalfWord(Register src, const MemOperand& mem,
+ Register scratch = r0);
+ void StoreByte(Register src, const MemOperand& mem, Register scratch = r0);
+ void CmpSmiLiteral(Register src1, Smi smi, Register scratch);
+
+ // Set new rounding mode RN to FPSCR
+ void SetRoundingMode(FPRoundingMode RN);
+
+ // reset rounding mode to default (kRoundToNearest)
+ void ResetRoundingMode();
+
+ // These exist to provide portability between 32 and 64bit
+ void LoadP(Register dst, const MemOperand& mem, Register scratch = no_reg);
+ void StoreP(Register src, const MemOperand& mem, Register scratch = no_reg);
+ void StoreP(const MemOperand& mem, const Operand& opnd,
+ Register scratch = no_reg);
+ void LoadMultipleP(Register dst1, Register dst2, const MemOperand& mem);
+ void StoreMultipleP(Register dst1, Register dst2, const MemOperand& mem);
+ void LoadMultipleW(Register dst1, Register dst2, const MemOperand& mem);
+ void StoreMultipleW(Register dst1, Register dst2, const MemOperand& mem);
+
+ void SwapP(Register src, Register dst, Register scratch);
+ void SwapP(Register src, MemOperand dst, Register scratch);
+ void SwapP(MemOperand src, MemOperand dst, Register scratch_0,
+ Register scratch_1);
+ void SwapFloat32(DoubleRegister src, DoubleRegister dst,
+ DoubleRegister scratch);
+ void SwapFloat32(DoubleRegister src, MemOperand dst, DoubleRegister scratch);
+ void SwapFloat32(MemOperand src, MemOperand dst, DoubleRegister scratch);
+ void SwapDouble(DoubleRegister src, DoubleRegister dst,
+ DoubleRegister scratch);
+ void SwapDouble(DoubleRegister src, MemOperand dst, DoubleRegister scratch);
+ void SwapDouble(MemOperand src, MemOperand dst, DoubleRegister scratch);
+ void SwapSimd128(Simd128Register src, Simd128Register dst,
+ Simd128Register scratch);
+ void SwapSimd128(Simd128Register src, MemOperand dst,
+ Simd128Register scratch);
+ void SwapSimd128(MemOperand src, MemOperand dst, Simd128Register scratch);
+
+ // Cleanse pointer address on 31bit by zero out top bit.
+ // This is a NOP on 64-bit.
+ void CleanseP(Register src) {
+#if (V8_HOST_ARCH_S390 && !(V8_TARGET_ARCH_S390X))
+ nilh(src, Operand(0x7FFF));
+#endif
+ }
+
+ void PrepareForTailCall(Register callee_args_count,
+ Register caller_args_count, Register scratch0,
+ Register scratch1);
+
+ // ---------------------------------------------------------------------------
+ // Runtime calls
+
+ // Before calling a C-function from generated code, align arguments on stack.
+ // After aligning the frame, non-register arguments must be stored in
+ // sp[0], sp[4], etc., not pushed. The argument count assumes all arguments
+ // are word sized. If double arguments are used, this function assumes that
+ // all double arguments are stored before core registers; otherwise the
+ // correct alignment of the double values is not guaranteed.
+ // Some compilers/platforms require the stack to be aligned when calling
+ // C++ code.
+ // Needs a scratch register to do some arithmetic. This register will be
+ // trashed.
+ void PrepareCallCFunction(int num_reg_arguments, int num_double_registers,
+ Register scratch);
+ void PrepareCallCFunction(int num_reg_arguments, Register scratch);
+
+ // There are two ways of passing double arguments on ARM, depending on
+ // whether soft or hard floating point ABI is used. These functions
+ // abstract parameter passing for the three different ways we call
+ // C functions from generated code.
+ void MovToFloatParameter(DoubleRegister src);
+ void MovToFloatParameters(DoubleRegister src1, DoubleRegister src2);
+ void MovToFloatResult(DoubleRegister src);
+
+ // Calls a C function and cleans up the space for arguments allocated
+ // by PrepareCallCFunction. The called function is not allowed to trigger a
+ // garbage collection, since that might move the code and invalidate the
+ // return address (unless this is somehow accounted for by the called
+ // function).
+ void CallCFunction(ExternalReference function, int num_arguments);
+ void CallCFunction(Register function, int num_arguments);
+ void CallCFunction(ExternalReference function, int num_reg_arguments,
+ int num_double_arguments);
+ void CallCFunction(Register function, int num_reg_arguments,
+ int num_double_arguments);
+
+ void MovFromFloatParameter(DoubleRegister dst);
+ void MovFromFloatResult(DoubleRegister dst);
+
+ void Trap() override;
+ void DebugBreak() override;
+
+ // Emit code for a truncating division by a constant. The dividend register is
+ // unchanged and ip gets clobbered. Dividend and result must be different.
+ void TruncateDoubleToI(Isolate* isolate, Zone* zone, Register result,
+ DoubleRegister double_input, StubCallMode stub_mode);
+ void TryInlineTruncateDoubleToI(Register result, DoubleRegister double_input,
+ Label* done);
+
+ // ---------------------------------------------------------------------------
+ // Debugging
+
+ // Calls Abort(msg) if the condition cond is not satisfied.
+ // Use --debug_code to enable.
+ void Assert(Condition cond, AbortReason reason, CRegister cr = cr7);
+
+ // Like Assert(), but always enabled.
+ void Check(Condition cond, AbortReason reason, CRegister cr = cr7);
+
+ // Print a message to stdout and abort execution.
+ void Abort(AbortReason reason);
+
+ // ---------------------------------------------------------------------------
+ // Bit testing/extraction
+ //
+ // Bit numbering is such that the least significant bit is bit 0
+ // (for consistency between 32/64-bit).
+
+ // Extract consecutive bits (defined by rangeStart - rangeEnd) from src
+ // and place them into the least significant bits of dst.
+ inline void ExtractBitRange(Register dst, Register src, int rangeStart,
+ int rangeEnd) {
+ DCHECK(rangeStart >= rangeEnd && rangeStart < kBitsPerSystemPointer);
+
+ // Try to use RISBG if possible.
+ if (CpuFeatures::IsSupported(GENERAL_INSTR_EXT)) {
+ int shiftAmount = (64 - rangeEnd) % 64; // Convert to shift left.
+ int endBit = 63; // End is always LSB after shifting.
+ int startBit = 63 - rangeStart + rangeEnd;
+ RotateInsertSelectBits(dst, src, Operand(startBit), Operand(endBit),
+ Operand(shiftAmount), true);
+ } else {
+ if (rangeEnd > 0) // Don't need to shift if rangeEnd is zero.
+ ShiftRightP(dst, src, Operand(rangeEnd));
+ else if (dst != src) // If we didn't shift, we might need to copy
+ LoadRR(dst, src);
+ int width = rangeStart - rangeEnd + 1;
+#if V8_TARGET_ARCH_S390X
+ uint64_t mask = (static_cast<uint64_t>(1) << width) - 1;
+ nihf(dst, Operand(mask >> 32));
+ nilf(dst, Operand(mask & 0xFFFFFFFF));
+ ltgr(dst, dst);
+#else
+ uint32_t mask = (1 << width) - 1;
+ AndP(dst, Operand(mask));
+#endif
+ }
+ }
+
+ inline void ExtractBit(Register dst, Register src, uint32_t bitNumber) {
+ ExtractBitRange(dst, src, bitNumber, bitNumber);
+ }
+
+ // Extract consecutive bits (defined by mask) from src and place them
+ // into the least significant bits of dst.
+ inline void ExtractBitMask(Register dst, Register src, uintptr_t mask,
+ RCBit rc = LeaveRC) {
+ int start = kBitsPerSystemPointer - 1;
+ int end;
+ uintptr_t bit = (1L << start);
+
+ while (bit && (mask & bit) == 0) {
+ start--;
+ bit >>= 1;
+ }
+ end = start;
+ bit >>= 1;
+
+ while (bit && (mask & bit)) {
+ end--;
+ bit >>= 1;
+ }
+
+ // 1-bits in mask must be contiguous
+ DCHECK(bit == 0 || (mask & ((bit << 1) - 1)) == 0);
+
+ ExtractBitRange(dst, src, start, end);
+ }
+
+ // Test single bit in value.
+ inline void TestBit(Register value, int bitNumber, Register scratch = r0) {
+ ExtractBitRange(scratch, value, bitNumber, bitNumber);
+ }
+
+ // Test consecutive bit range in value. Range is defined by
+ // rangeStart - rangeEnd.
+ inline void TestBitRange(Register value, int rangeStart, int rangeEnd,
+ Register scratch = r0) {
+ ExtractBitRange(scratch, value, rangeStart, rangeEnd);
+ }
+
+ // Test consecutive bit range in value. Range is defined by mask.
+ inline void TestBitMask(Register value, uintptr_t mask,
+ Register scratch = r0) {
+ ExtractBitMask(scratch, value, mask, SetRC);
+ }
+ inline void TestIfSmi(Register value) { tmll(value, Operand(1)); }
+
+ inline void TestIfSmi(MemOperand value) {
+ if (is_uint12(value.offset())) {
+ tm(value, Operand(1));
+ } else if (is_int20(value.offset())) {
+ tmy(value, Operand(1));
+ } else {
+ LoadB(r0, value);
+ tmll(r0, Operand(1));
+ }
+ }
+
+ inline void TestIfInt32(Register value) {
+ // High bits must be identical to fit into an 32-bit integer
+ cgfr(value, value);
+ }
+ void SmiUntag(Register reg) { SmiUntag(reg, reg); }
+
+ void SmiUntag(Register dst, const MemOperand& src);
+ void SmiUntag(Register dst, Register src) {
+ if (SmiValuesAre31Bits()) {
+ ShiftRightArith(dst, src, Operand(kSmiShift));
+ } else {
+ ShiftRightArithP(dst, src, Operand(kSmiShift));
+ }
+ lgfr(dst, dst);
+ }
+
+ // Activation support.
+ void EnterFrame(StackFrame::Type type,
+ bool load_constant_pool_pointer_reg = false);
+ // Returns the pc offset at which the frame ends.
+ int LeaveFrame(StackFrame::Type type, int stack_adjustment = 0);
+
+ void CheckPageFlag(Register object, Register scratch, int mask, Condition cc,
+ Label* condition_met);
+
+ void ResetSpeculationPoisonRegister();
+ void ComputeCodeStartAddress(Register dst);
+ void LoadPC(Register dst);
+
+ // Control-flow integrity:
+
+ // Define a function entrypoint. This doesn't emit any code for this
+ // architecture, as control-flow integrity is not supported for it.
+ void CodeEntry() {}
+ // Define an exception handler.
+ void ExceptionHandler() {}
+ // Define an exception handler and bind a label.
+ void BindExceptionHandler(Label* label) { bind(label); }
+
+ // Generates an instruction sequence s.t. the return address points to the
+ // instruction following the call.
+ // The return address on the stack is used by frame iteration.
+ void StoreReturnAddressAndCall(Register target);
+
+ // ---------------------------------------------------------------------------
+ // Pointer compression Support
+
+ // Loads a field containing a HeapObject and decompresses it if pointer
+ // compression is enabled.
+ void LoadTaggedPointerField(const Register& destination,
+ const MemOperand& field_operand,
+ const Register& scratch = no_reg);
+
+ // Loads a field containing any tagged value and decompresses it if necessary.
+ void LoadAnyTaggedField(const Register& destination,
+ const MemOperand& field_operand,
+ const Register& scratch = no_reg);
+
+ // Loads a field containing smi value and untags it.
+ void SmiUntagField(Register dst, const MemOperand& src);
+
+ // Compresses and stores tagged value to given on-heap location.
+ void StoreTaggedField(const Register& value,
+ const MemOperand& dst_field_operand,
+ const Register& scratch = no_reg);
+
+ void DecompressTaggedSigned(Register destination, MemOperand field_operand);
+ void DecompressTaggedSigned(Register destination, Register src);
+ void DecompressTaggedPointer(Register destination, MemOperand field_operand);
+ void DecompressTaggedPointer(Register destination, Register source);
+ void DecompressAnyTagged(Register destination, MemOperand field_operand);
+ void DecompressAnyTagged(Register destination, Register source);
+
+ private:
+ static const int kSmiShift = kSmiTagSize + kSmiShiftSize;
+
+ void CallCFunctionHelper(Register function, int num_reg_arguments,
+ int num_double_arguments);
+
+ void CallRecordWriteStub(Register object, Register address,
+ RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode, Handle<Code> code_target,
+ Address wasm_target);
+
+ void Jump(intptr_t target, RelocInfo::Mode rmode, Condition cond = al);
+ int CalculateStackPassedWords(int num_reg_arguments,
+ int num_double_arguments);
+};
+
+// MacroAssembler implements a collection of frequently used macros.
+class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler {
+ public:
+ using TurboAssembler::TurboAssembler;
+
+ // It assumes that the arguments are located below the stack pointer.
+ // argc is the number of arguments not including the receiver.
+ // TODO(victorgomes): Remove this function once we stick with the reversed
+ // arguments order.
+ void LoadReceiver(Register dest, Register argc) {
+ LoadP(dest, MemOperand(sp, 0));
+ }
+
+ void StoreReceiver(Register rec, Register argc, Register scratch) {
+ StoreP(rec, MemOperand(sp, 0));
+ }
+
+ void CallRuntime(const Runtime::Function* f, int num_arguments,
+ SaveFPRegsMode save_doubles = kDontSaveFPRegs);
+ void CallRuntimeSaveDoubles(Runtime::FunctionId fid) {
+ const Runtime::Function* function = Runtime::FunctionForId(fid);
+ CallRuntime(function, function->nargs, kSaveFPRegs);
+ }
+
+ // Convenience function: Same as above, but takes the fid instead.
+ void CallRuntime(Runtime::FunctionId fid,
+ SaveFPRegsMode save_doubles = kDontSaveFPRegs) {
+ const Runtime::Function* function = Runtime::FunctionForId(fid);
+ CallRuntime(function, function->nargs, save_doubles);
+ }
+
+ // Convenience function: Same as above, but takes the fid instead.
+ void CallRuntime(Runtime::FunctionId fid, int num_arguments,
+ SaveFPRegsMode save_doubles = kDontSaveFPRegs) {
+ CallRuntime(Runtime::FunctionForId(fid), num_arguments, save_doubles);
+ }
+
+ // Convenience function: tail call a runtime routine (jump).
+ void TailCallRuntime(Runtime::FunctionId fid);
+
+ // ---------------------------------------------------------------------------
+ // Support functions.
+
+ // Compare object type for heap object. heap_object contains a non-Smi
+ // whose object type should be compared with the given type. This both
+ // sets the flags and leaves the object type in the type_reg register.
+ // It leaves the map in the map register (unless the type_reg and map register
+ // are the same register). It leaves the heap object in the heap_object
+ // register unless the heap_object register is the same register as one of the
+ // other registers.
+ // Type_reg can be no_reg. In that case ip is used.
+ void CompareObjectType(Register heap_object, Register map, Register type_reg,
+ InstanceType type);
+
+ // Compare instance type in a map. map contains a valid map object whose
+ // object type should be compared with the given type. This both
+ // sets the flags and leaves the object type in the type_reg register.
+ void CompareInstanceType(Register map, Register type_reg, InstanceType type);
+
+ // Compare the object in a register to a value from the root list.
+ // Uses the ip register as scratch.
+ void CompareRoot(Register obj, RootIndex index);
+ void PushRoot(RootIndex index) {
+ LoadRoot(r0, index);
+ Push(r0);
+ }
+
+ template <class T>
+ void CompareTagged(Register src1, T src2) {
+ if (COMPRESS_POINTERS_BOOL) {
+ Cmp32(src1, src2);
+ } else {
+ CmpP(src1, src2);
+ }
+ }
+
+ // Jump to a runtime routine.
+ void JumpToExternalReference(const ExternalReference& builtin,
+ bool builtin_exit_frame = false);
+
+ // Generates a trampoline to jump to the off-heap instruction stream.
+ void JumpToInstructionStream(Address entry);
+
+ // Compare the object in a register to a value and jump if they are equal.
+ void JumpIfRoot(Register with, RootIndex index, Label* if_equal) {
+ CompareRoot(with, index);
+ beq(if_equal);
+ }
+
+ // Compare the object in a register to a value and jump if they are not equal.
+ void JumpIfNotRoot(Register with, RootIndex index, Label* if_not_equal) {
+ CompareRoot(with, index);
+ bne(if_not_equal);
+ }
+
+ // Checks if value is in range [lower_limit, higher_limit] using a single
+ // comparison.
+ void JumpIfIsInRange(Register value, unsigned lower_limit,
+ unsigned higher_limit, Label* on_in_range);
+
+ // ---------------------------------------------------------------------------
+ // In-place weak references.
+ void LoadWeakValue(Register out, Register in, Label* target_if_cleared);
+
+ // ---------------------------------------------------------------------------
+ // StatsCounter support
+
+ void IncrementCounter(StatsCounter* counter, int value, Register scratch1,
+ Register scratch2);
+ void DecrementCounter(StatsCounter* counter, int value, Register scratch1,
+ Register scratch2);
+ // ---------------------------------------------------------------------------
+ // JavaScript invokes
+
+ // Set up call kind marking in ecx. The method takes ecx as an
+ // explicit first parameter to make the code more readable at the
+ // call sites.
+ // void SetCallKind(Register dst, CallKind kind);
+
+ // Removes current frame and its arguments from the stack preserving
+ // the arguments and a return address pushed to the stack for the next call.
+ // Both |callee_args_count| and |caller_args_count| do not include
+ // receiver. |callee_args_count| is not modified. |caller_args_count|
+ // is trashed.
+
+ // Invoke the JavaScript function code by either calling or jumping.
+ void InvokeFunctionCode(Register function, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count, InvokeFlag flag);
+
+ // On function call, call into the debugger if necessary.
+ void CheckDebugHook(Register fun, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count);
+
+ // Invoke the JavaScript function in the given register. Changes the
+ // current context to the context in the function before invoking.
+ void InvokeFunctionWithNewTarget(Register function, Register new_target,
+ Register actual_parameter_count,
+ InvokeFlag flag);
+ void InvokeFunction(Register function, Register expected_parameter_count,
+ Register actual_parameter_count, InvokeFlag flag);
+
+ // Frame restart support
+ void MaybeDropFrames();
+
+ // Exception handling
+
+ // Push a new stack handler and link into stack handler chain.
+ void PushStackHandler();
+
+ // Unlink the stack handler on top of the stack from the stack handler chain.
+ // Must preserve the result register.
+ void PopStackHandler();
+
+ // Enter exit frame.
+ // stack_space - extra stack space, used for parameters before call to C.
+ // At least one slot (for the return address) should be provided.
+ void EnterExitFrame(bool save_doubles, int stack_space = 1,
+ StackFrame::Type frame_type = StackFrame::EXIT);
+
+ // Leave the current exit frame. Expects the return value in r0.
+ // Expect the number of values, pushed prior to the exit frame, to
+ // remove in a register (or no_reg, if there is nothing to remove).
+ void LeaveExitFrame(bool save_doubles, Register argument_count,
+ bool argument_count_is_length = false);
+
+ void LoadMap(Register destination, Register object);
+
+ // Load the global proxy from the current context.
+ void LoadGlobalProxy(Register dst) {
+ LoadNativeContextSlot(Context::GLOBAL_PROXY_INDEX, dst);
+ }
+
+ void LoadNativeContextSlot(int index, Register dst);
+
+ // ---------------------------------------------------------------------------
+ // Smi utilities
+
+ // Shift left by kSmiShift
+ void SmiTag(Register reg) { SmiTag(reg, reg); }
+ void SmiTag(Register dst, Register src) {
+ ShiftLeftP(dst, src, Operand(kSmiShift));
+ }
+
+ void SmiToPtrArrayOffset(Register dst, Register src) {
+#if defined(V8_COMPRESS_POINTERS) || defined(V8_31BIT_SMIS_ON_64BIT_ARCH)
+ STATIC_ASSERT(kSmiTag == 0 && kSmiShift < kSystemPointerSizeLog2);
+ ShiftLeftP(dst, src, Operand(kSystemPointerSizeLog2 - kSmiShift));
+#else
+ STATIC_ASSERT(kSmiTag == 0 && kSmiShift > kSystemPointerSizeLog2);
+ ShiftRightArithP(dst, src, Operand(kSmiShift - kSystemPointerSizeLog2));
+#endif
+ }
+
+ // Jump if either of the registers contain a non-smi.
+ inline void JumpIfNotSmi(Register value, Label* not_smi_label) {
+ TestIfSmi(value);
+ bne(not_smi_label /*, cr0*/);
+ }
+
+ // Abort execution if argument is a smi, enabled via --debug-code.
+ void AssertNotSmi(Register object);
+ void AssertSmi(Register object);
+
+#if !defined(V8_COMPRESS_POINTERS) && !defined(V8_31BIT_SMIS_ON_64BIT_ARCH)
+ // Ensure it is permissible to read/write int value directly from
+ // upper half of the smi.
+ STATIC_ASSERT(kSmiTag == 0);
+ STATIC_ASSERT(kSmiTagSize + kSmiShiftSize == 32);
+#endif
+#if V8_TARGET_LITTLE_ENDIAN
+#define SmiWordOffset(offset) (offset + kSystemPointerSize / 2)
+#else
+#define SmiWordOffset(offset) offset
+#endif
+
+ // Abort execution if argument is not a Constructor, enabled via --debug-code.
+ void AssertConstructor(Register object, Register scratch);
+
+ // Abort execution if argument is not a JSFunction, enabled via --debug-code.
+ void AssertFunction(Register object);
+
+ // Abort execution if argument is not a JSBoundFunction,
+ // enabled via --debug-code.
+ void AssertBoundFunction(Register object);
+
+ // Abort execution if argument is not a JSGeneratorObject (or subclass),
+ // enabled via --debug-code.
+ void AssertGeneratorObject(Register object);
+
+ // Abort execution if argument is not undefined or an AllocationSite, enabled
+ // via --debug-code.
+ void AssertUndefinedOrAllocationSite(Register object, Register scratch);
+
+ template <typename Field>
+ void DecodeField(Register dst, Register src) {
+ ExtractBitRange(dst, src, Field::kShift + Field::kSize - 1, Field::kShift);
+ }
+
+ template <typename Field>
+ void DecodeField(Register reg) {
+ DecodeField<Field>(reg, reg);
+ }
+
+ // ---------------------------------------------------------------------------
+ // GC Support
+
+ void IncrementalMarkingRecordWriteHelper(Register object, Register value,
+ Register address);
+
+ void CallJSEntry(Register target);
+ static int CallSizeNotPredictableCodeSize(Address target,
+ RelocInfo::Mode rmode,
+ Condition cond = al);
+ // Notify the garbage collector that we wrote a pointer into an object.
+ // |object| is the object being stored into, |value| is the object being
+ // stored. value and scratch registers are clobbered by the operation.
+ // The offset is the offset from the start of the object, not the offset from
+ // the tagged HeapObject pointer. For use with FieldMemOperand(reg, off).
+ void RecordWriteField(
+ Register object, int offset, Register value, Register scratch,
+ LinkRegisterStatus lr_status, SaveFPRegsMode save_fp,
+ RememberedSetAction remembered_set_action = EMIT_REMEMBERED_SET,
+ SmiCheck smi_check = INLINE_SMI_CHECK);
+
+ // For a given |object| notify the garbage collector that the slot |address|
+ // has been written. |value| is the object being stored. The value and
+ // address registers are clobbered by the operation.
+ void RecordWrite(
+ Register object, Register address, Register value,
+ LinkRegisterStatus lr_status, SaveFPRegsMode save_fp,
+ RememberedSetAction remembered_set_action = EMIT_REMEMBERED_SET,
+ SmiCheck smi_check = INLINE_SMI_CHECK);
+
+ private:
+ static const int kSmiShift = kSmiTagSize + kSmiShiftSize;
+ // Helper functions for generating invokes.
+ void InvokePrologue(Register expected_parameter_count,
+ Register actual_parameter_count, Label* done,
+ InvokeFlag flag);
+
+ DISALLOW_IMPLICIT_CONSTRUCTORS(MacroAssembler);
+};
+
+#define ACCESS_MASM(masm) masm->
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_S390_MACRO_ASSEMBLER_S390_H_
diff --git a/src/codegen/s390/register-s390.h b/src/codegen/s390/register-s390.h
new file mode 100644
index 0000000..0c6da03
--- /dev/null
+++ b/src/codegen/s390/register-s390.h
@@ -0,0 +1,275 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_S390_REGISTER_S390_H_
+#define V8_CODEGEN_S390_REGISTER_S390_H_
+
+#include "src/codegen/register.h"
+#include "src/codegen/reglist.h"
+
+namespace v8 {
+namespace internal {
+
+// clang-format off
+#define GENERAL_REGISTERS(V) \
+ V(r0) V(r1) V(r2) V(r3) V(r4) V(r5) V(r6) V(r7) \
+ V(r8) V(r9) V(r10) V(fp) V(ip) V(r13) V(r14) V(sp)
+
+#define ALLOCATABLE_GENERAL_REGISTERS(V) \
+ V(r2) V(r3) V(r4) V(r5) V(r6) V(r7) \
+ V(r8) V(r9) V(r13)
+
+#define DOUBLE_REGISTERS(V) \
+ V(d0) V(d1) V(d2) V(d3) V(d4) V(d5) V(d6) V(d7) \
+ V(d8) V(d9) V(d10) V(d11) V(d12) V(d13) V(d14) V(d15)
+
+#define FLOAT_REGISTERS DOUBLE_REGISTERS
+#define SIMD128_REGISTERS DOUBLE_REGISTERS
+
+#define ALLOCATABLE_DOUBLE_REGISTERS(V) \
+ V(d1) V(d2) V(d3) V(d4) V(d5) V(d6) V(d7) \
+ V(d8) V(d9) V(d10) V(d11) V(d12) V(d15) V(d0)
+
+#define C_REGISTERS(V) \
+ V(cr0) V(cr1) V(cr2) V(cr3) V(cr4) V(cr5) V(cr6) V(cr7) \
+ V(cr8) V(cr9) V(cr10) V(cr11) V(cr12) V(cr15)
+// clang-format on
+
+// Register list in load/store instructions
+// Note that the bit values must match those used in actual instruction encoding
+
+// Caller-saved/arguments registers
+const RegList kJSCallerSaved = 1 << 1 | 1 << 2 | // r2 a1
+ 1 << 3 | // r3 a2
+ 1 << 4 | // r4 a3
+ 1 << 5; // r5 a4
+
+const int kNumJSCallerSaved = 5;
+
+// Callee-saved registers preserved when switching from C to JavaScript
+const RegList kCalleeSaved =
+ 1 << 6 | // r6 (argument passing in CEntryStub)
+ // (HandleScope logic in MacroAssembler)
+ 1 << 7 | // r7 (argument passing in CEntryStub)
+ // (HandleScope logic in MacroAssembler)
+ 1 << 8 | // r8 (argument passing in CEntryStub)
+ // (HandleScope logic in MacroAssembler)
+ 1 << 9 | // r9 (HandleScope logic in MacroAssembler)
+ 1 << 10 | // r10 (Roots register in Javascript)
+ 1 << 11 | // r11 (fp in Javascript)
+ 1 << 12 | // r12 (ip in Javascript)
+ 1 << 13; // r13 (cp in Javascript)
+// 1 << 15; // r15 (sp in Javascript)
+
+const int kNumCalleeSaved = 8;
+
+const RegList kCallerSavedDoubles = 1 << 0 | // d0
+ 1 << 1 | // d1
+ 1 << 2 | // d2
+ 1 << 3 | // d3
+ 1 << 4 | // d4
+ 1 << 5 | // d5
+ 1 << 6 | // d6
+ 1 << 7; // d7
+
+const int kNumCallerSavedDoubles = 8;
+
+const RegList kCalleeSavedDoubles = 1 << 8 | // d8
+ 1 << 9 | // d9
+ 1 << 10 | // d10
+ 1 << 11 | // d11
+ 1 << 12 | // d12
+ 1 << 13 | // d12
+ 1 << 14 | // d12
+ 1 << 15; // d13
+
+const int kNumCalleeSavedDoubles = 8;
+
+// The following constants describe the stack frame linkage area as
+// defined by the ABI.
+
+#if V8_TARGET_ARCH_S390X
+// [0] Back Chain
+// [1] Reserved for compiler use
+// [2] GPR 2
+// [3] GPR 3
+// ...
+// [15] GPR 15
+// [16] FPR 0
+// [17] FPR 2
+// [18] FPR 4
+// [19] FPR 6
+const int kNumRequiredStackFrameSlots = 20;
+const int kStackFrameRASlot = 14;
+const int kStackFrameSPSlot = 15;
+const int kStackFrameExtraParamSlot = 20;
+#else
+// [0] Back Chain
+// [1] Reserved for compiler use
+// [2] GPR 2
+// [3] GPR 3
+// ...
+// [15] GPR 15
+// [16..17] FPR 0
+// [18..19] FPR 2
+// [20..21] FPR 4
+// [22..23] FPR 6
+const int kNumRequiredStackFrameSlots = 24;
+const int kStackFrameRASlot = 14;
+const int kStackFrameSPSlot = 15;
+const int kStackFrameExtraParamSlot = 24;
+#endif
+
+// zLinux ABI requires caller frames to include sufficient space for
+// callee preserved register save area.
+#if V8_TARGET_ARCH_S390X
+const int kCalleeRegisterSaveAreaSize = 160;
+#elif V8_TARGET_ARCH_S390
+const int kCalleeRegisterSaveAreaSize = 96;
+#else
+const int kCalleeRegisterSaveAreaSize = 0;
+#endif
+
+enum RegisterCode {
+#define REGISTER_CODE(R) kRegCode_##R,
+ GENERAL_REGISTERS(REGISTER_CODE)
+#undef REGISTER_CODE
+ kRegAfterLast
+};
+
+class Register : public RegisterBase<Register, kRegAfterLast> {
+ public:
+#if V8_TARGET_LITTLE_ENDIAN
+ static constexpr int kMantissaOffset = 0;
+ static constexpr int kExponentOffset = 4;
+#else
+ static constexpr int kMantissaOffset = 4;
+ static constexpr int kExponentOffset = 0;
+#endif
+
+ private:
+ friend class RegisterBase;
+ explicit constexpr Register(int code) : RegisterBase(code) {}
+};
+
+ASSERT_TRIVIALLY_COPYABLE(Register);
+static_assert(sizeof(Register) == sizeof(int),
+ "Register can efficiently be passed by value");
+
+#define DEFINE_REGISTER(R) \
+ constexpr Register R = Register::from_code(kRegCode_##R);
+GENERAL_REGISTERS(DEFINE_REGISTER)
+#undef DEFINE_REGISTER
+constexpr Register no_reg = Register::no_reg();
+
+// Register aliases
+constexpr Register kRootRegister = r10; // Roots array pointer.
+constexpr Register cp = r13; // JavaScript context pointer.
+
+constexpr bool kPadArguments = false;
+constexpr bool kSimpleFPAliasing = true;
+constexpr bool kSimdMaskRegisters = false;
+
+enum DoubleRegisterCode {
+#define REGISTER_CODE(R) kDoubleCode_##R,
+ DOUBLE_REGISTERS(REGISTER_CODE)
+#undef REGISTER_CODE
+ kDoubleAfterLast
+};
+
+// Double word VFP register.
+class DoubleRegister : public RegisterBase<DoubleRegister, kDoubleAfterLast> {
+ public:
+ // A few double registers are reserved: one as a scratch register and one to
+ // hold 0.0, that does not fit in the immediate field of vmov instructions.
+ // d14: 0.0
+ // d15: scratch register.
+ static constexpr int kSizeInBytes = 8;
+
+ // This function differs from kNumRegisters by returning the number of double
+ // registers supported by the current CPU, while kNumRegisters always returns
+ // 32.
+ inline static int SupportedRegisterCount();
+
+ private:
+ friend class RegisterBase;
+
+ explicit constexpr DoubleRegister(int code) : RegisterBase(code) {}
+};
+
+ASSERT_TRIVIALLY_COPYABLE(DoubleRegister);
+static_assert(sizeof(DoubleRegister) == sizeof(int),
+ "DoubleRegister can efficiently be passed by value");
+
+using FloatRegister = DoubleRegister;
+
+// TODO(john.yan) Define SIMD registers.
+using Simd128Register = DoubleRegister;
+
+#define DEFINE_REGISTER(R) \
+ constexpr DoubleRegister R = DoubleRegister::from_code(kDoubleCode_##R);
+DOUBLE_REGISTERS(DEFINE_REGISTER)
+#undef DEFINE_REGISTER
+constexpr DoubleRegister no_dreg = DoubleRegister::no_reg();
+
+constexpr DoubleRegister kDoubleRegZero = d14;
+constexpr DoubleRegister kScratchDoubleReg = d13;
+
+Register ToRegister(int num);
+
+enum CRegisterCode {
+#define REGISTER_CODE(R) kCCode_##R,
+ C_REGISTERS(REGISTER_CODE)
+#undef REGISTER_CODE
+ kCAfterLast
+};
+
+// Coprocessor register
+class CRegister : public RegisterBase<CRegister, kCAfterLast> {
+ friend class RegisterBase;
+ explicit constexpr CRegister(int code) : RegisterBase(code) {}
+};
+
+constexpr CRegister no_creg = CRegister::no_reg();
+#define DECLARE_C_REGISTER(R) \
+ constexpr CRegister R = CRegister::from_code(kCCode_##R);
+C_REGISTERS(DECLARE_C_REGISTER)
+#undef DECLARE_C_REGISTER
+
+// Define {RegisterName} methods for the register types.
+DEFINE_REGISTER_NAMES(Register, GENERAL_REGISTERS)
+DEFINE_REGISTER_NAMES(DoubleRegister, DOUBLE_REGISTERS)
+
+// Give alias names to registers for calling conventions.
+constexpr Register kReturnRegister0 = r2;
+constexpr Register kReturnRegister1 = r3;
+constexpr Register kReturnRegister2 = r4;
+constexpr Register kJSFunctionRegister = r3;
+constexpr Register kContextRegister = r13;
+constexpr Register kAllocateSizeRegister = r3;
+constexpr Register kSpeculationPoisonRegister = r9;
+constexpr Register kInterpreterAccumulatorRegister = r2;
+constexpr Register kInterpreterBytecodeOffsetRegister = r6;
+constexpr Register kInterpreterBytecodeArrayRegister = r7;
+constexpr Register kInterpreterDispatchTableRegister = r8;
+
+constexpr Register kJavaScriptCallArgCountRegister = r2;
+constexpr Register kJavaScriptCallCodeStartRegister = r4;
+constexpr Register kJavaScriptCallTargetRegister = kJSFunctionRegister;
+constexpr Register kJavaScriptCallNewTargetRegister = r5;
+constexpr Register kJavaScriptCallExtraArg1Register = r4;
+
+constexpr Register kOffHeapTrampolineRegister = ip;
+constexpr Register kRuntimeCallFunctionRegister = r3;
+constexpr Register kRuntimeCallArgCountRegister = r2;
+constexpr Register kRuntimeCallArgvRegister = r4;
+constexpr Register kWasmInstanceRegister = r6;
+constexpr Register kWasmCompileLazyFuncIndexRegister = r7;
+
+constexpr DoubleRegister kFPReturnRegister0 = d0;
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_S390_REGISTER_S390_H_
diff --git a/src/codegen/safepoint-table.cc b/src/codegen/safepoint-table.cc
new file mode 100644
index 0000000..644931e
--- /dev/null
+++ b/src/codegen/safepoint-table.cc
@@ -0,0 +1,209 @@
+// Copyright 2011 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/safepoint-table.h"
+
+#include "src/codegen/assembler-inl.h"
+#include "src/codegen/macro-assembler.h"
+#include "src/deoptimizer/deoptimizer.h"
+#include "src/diagnostics/disasm.h"
+#include "src/execution/frames-inl.h"
+#include "src/utils/ostreams.h"
+#include "src/wasm/wasm-code-manager.h"
+
+namespace v8 {
+namespace internal {
+
+SafepointTable::SafepointTable(Code code)
+ : SafepointTable(code.InstructionStart(), code.SafepointTableAddress(),
+ code.stack_slots(), true) {}
+
+SafepointTable::SafepointTable(const wasm::WasmCode* code)
+ : SafepointTable(code->instruction_start(),
+ code->instruction_start() + code->safepoint_table_offset(),
+ code->stack_slots(), false) {}
+
+SafepointTable::SafepointTable(Address instruction_start,
+ Address safepoint_table_address,
+ uint32_t stack_slots, bool has_deopt)
+ : instruction_start_(instruction_start),
+ stack_slots_(stack_slots),
+ has_deopt_(has_deopt),
+ safepoint_table_address_(safepoint_table_address),
+ length_(ReadLength(safepoint_table_address)),
+ entry_size_(ReadEntrySize(safepoint_table_address)) {}
+
+unsigned SafepointTable::find_return_pc(unsigned pc_offset) {
+ for (unsigned i = 0; i < length(); i++) {
+ if (GetTrampolinePcOffset(i) == static_cast<int>(pc_offset)) {
+ return GetPcOffset(i);
+ } else if (GetPcOffset(i) == pc_offset) {
+ return pc_offset;
+ }
+ }
+ UNREACHABLE();
+}
+
+SafepointEntry SafepointTable::FindEntry(Address pc) const {
+ unsigned pc_offset = static_cast<unsigned>(pc - instruction_start_);
+ // We use kMaxUInt32 as sentinel value, so check that we don't hit that.
+ DCHECK_NE(kMaxUInt32, pc_offset);
+ unsigned len = length();
+ CHECK_GT(len, 0);
+ // If pc == kMaxUInt32, then this entry covers all call sites in the function.
+ if (len == 1 && GetPcOffset(0) == kMaxUInt32) return GetEntry(0);
+ for (unsigned i = 0; i < len; i++) {
+ // TODO(kasperl): Replace the linear search with binary search.
+ if (GetPcOffset(i) == pc_offset ||
+ (has_deopt_ &&
+ GetTrampolinePcOffset(i) == static_cast<int>(pc_offset))) {
+ return GetEntry(i);
+ }
+ }
+ UNREACHABLE();
+}
+
+void SafepointTable::PrintEntry(unsigned index,
+ std::ostream& os) const { // NOLINT
+ disasm::NameConverter converter;
+ SafepointEntry entry = GetEntry(index);
+ uint8_t* bits = entry.bits();
+
+ // Print the stack slot bits.
+ if (entry_size_ > 0) {
+ const int first = 0;
+ int last = entry_size_ - 1;
+ for (int i = first; i < last; i++) PrintBits(os, bits[i], kBitsPerByte);
+ int last_bits = stack_slots_ - ((last - first) * kBitsPerByte);
+ PrintBits(os, bits[last], last_bits);
+ }
+}
+
+void SafepointTable::PrintBits(std::ostream& os, // NOLINT
+ uint8_t byte, int digits) {
+ DCHECK(digits >= 0 && digits <= kBitsPerByte);
+ for (int i = 0; i < digits; i++) {
+ os << (((byte & (1 << i)) == 0) ? "0" : "1");
+ }
+}
+
+Safepoint SafepointTableBuilder::DefineSafepoint(
+ Assembler* assembler, Safepoint::DeoptMode deopt_mode) {
+ deoptimization_info_.push_back(
+ DeoptimizationInfo(zone_, assembler->pc_offset_for_safepoint()));
+ DeoptimizationInfo& new_info = deoptimization_info_.back();
+ return Safepoint(new_info.indexes);
+}
+
+unsigned SafepointTableBuilder::GetCodeOffset() const {
+ DCHECK(emitted_);
+ return offset_;
+}
+
+int SafepointTableBuilder::UpdateDeoptimizationInfo(int pc, int trampoline,
+ int start,
+ unsigned deopt_index) {
+ int index = start;
+ for (auto it = deoptimization_info_.Find(start);
+ it != deoptimization_info_.end(); it++, index++) {
+ if (static_cast<int>(it->pc) == pc) {
+ it->trampoline = trampoline;
+ it->deopt_index = deopt_index;
+ return index;
+ }
+ }
+ UNREACHABLE();
+}
+
+void SafepointTableBuilder::Emit(Assembler* assembler, int bits_per_entry) {
+ RemoveDuplicates();
+
+ // Make sure the safepoint table is properly aligned. Pad with nops.
+ assembler->Align(Code::kMetadataAlignment);
+ assembler->RecordComment(";;; Safepoint table.");
+ offset_ = assembler->pc_offset();
+
+ // Compute the number of bytes per safepoint entry.
+ int bytes_per_entry =
+ RoundUp(bits_per_entry, kBitsPerByte) >> kBitsPerByteLog2;
+
+ // Emit the table header.
+ STATIC_ASSERT(SafepointTable::kLengthOffset == 0 * kIntSize);
+ STATIC_ASSERT(SafepointTable::kEntrySizeOffset == 1 * kIntSize);
+ STATIC_ASSERT(SafepointTable::kHeaderSize == 2 * kIntSize);
+ int length = static_cast<int>(deoptimization_info_.size());
+ assembler->dd(length);
+ assembler->dd(bytes_per_entry);
+
+ // Emit sorted table of pc offsets together with additional info (i.e. the
+ // deoptimization index or arguments count) and trampoline offsets.
+ STATIC_ASSERT(SafepointTable::kPcOffset == 0 * kIntSize);
+ STATIC_ASSERT(SafepointTable::kEncodedInfoOffset == 1 * kIntSize);
+ STATIC_ASSERT(SafepointTable::kTrampolinePcOffset == 2 * kIntSize);
+ STATIC_ASSERT(SafepointTable::kFixedEntrySize == 3 * kIntSize);
+ for (const DeoptimizationInfo& info : deoptimization_info_) {
+ assembler->dd(info.pc);
+ assembler->dd(info.deopt_index);
+ assembler->dd(info.trampoline);
+ }
+
+ // Emit table of bitmaps.
+ ZoneVector<uint8_t> bits(bytes_per_entry, 0, zone_);
+ for (const DeoptimizationInfo& info : deoptimization_info_) {
+ ZoneChunkList<int>* indexes = info.indexes;
+ std::fill(bits.begin(), bits.end(), 0);
+
+ // Run through the indexes and build a bitmap.
+ for (int idx : *indexes) {
+ int index = bits_per_entry - 1 - idx;
+ int byte_index = index >> kBitsPerByteLog2;
+ int bit_index = index & (kBitsPerByte - 1);
+ bits[byte_index] |= (1U << bit_index);
+ }
+
+ // Emit the bitmap for the current entry.
+ for (int k = 0; k < bytes_per_entry; k++) {
+ assembler->db(bits[k]);
+ }
+ }
+ emitted_ = true;
+}
+
+void SafepointTableBuilder::RemoveDuplicates() {
+ // If the table contains more than one entry, and all entries are identical
+ // (except for the pc), replace the whole table by a single entry with pc =
+ // kMaxUInt32. This especially compacts the table for wasm code without tagged
+ // pointers and without deoptimization info.
+
+ if (deoptimization_info_.size() < 2) return;
+
+ // Check that all entries (1, size] are identical to entry 0.
+ const DeoptimizationInfo& first_info = deoptimization_info_.front();
+ for (auto it = deoptimization_info_.Find(1); it != deoptimization_info_.end();
+ it++) {
+ if (!IsIdenticalExceptForPc(first_info, *it)) return;
+ }
+
+ // If we get here, all entries were identical. Rewind the list to just one
+ // entry, and set the pc to kMaxUInt32.
+ deoptimization_info_.Rewind(1);
+ deoptimization_info_.front().pc = kMaxUInt32;
+}
+
+bool SafepointTableBuilder::IsIdenticalExceptForPc(
+ const DeoptimizationInfo& info1, const DeoptimizationInfo& info2) const {
+ if (info1.deopt_index != info2.deopt_index) return false;
+
+ ZoneChunkList<int>* indexes1 = info1.indexes;
+ ZoneChunkList<int>* indexes2 = info2.indexes;
+ if (indexes1->size() != indexes2->size()) return false;
+ if (!std::equal(indexes1->begin(), indexes1->end(), indexes2->begin())) {
+ return false;
+ }
+
+ return true;
+}
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/safepoint-table.h b/src/codegen/safepoint-table.h
new file mode 100644
index 0000000..ef6ed2f
--- /dev/null
+++ b/src/codegen/safepoint-table.h
@@ -0,0 +1,238 @@
+// Copyright 2011 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_SAFEPOINT_TABLE_H_
+#define V8_CODEGEN_SAFEPOINT_TABLE_H_
+
+#include "src/base/memory.h"
+#include "src/common/assert-scope.h"
+#include "src/utils/allocation.h"
+#include "src/utils/utils.h"
+#include "src/zone/zone-chunk-list.h"
+#include "src/zone/zone.h"
+
+namespace v8 {
+namespace internal {
+
+namespace wasm {
+class WasmCode;
+} // namespace wasm
+
+class SafepointEntry {
+ public:
+ SafepointEntry() : deopt_index_(0), bits_(nullptr), trampoline_pc_(-1) {}
+
+ SafepointEntry(unsigned deopt_index, uint8_t* bits, int trampoline_pc)
+ : deopt_index_(deopt_index), bits_(bits), trampoline_pc_(trampoline_pc) {
+ DCHECK(is_valid());
+ }
+
+ bool is_valid() const { return bits_ != nullptr; }
+
+ bool Equals(const SafepointEntry& other) const {
+ return deopt_index_ == other.deopt_index_ && bits_ == other.bits_;
+ }
+
+ void Reset() {
+ deopt_index_ = 0;
+ bits_ = nullptr;
+ }
+
+ int trampoline_pc() { return trampoline_pc_; }
+
+ static const unsigned kNoDeoptIndex = kMaxUInt32;
+
+ int deoptimization_index() const {
+ DCHECK(is_valid() && has_deoptimization_index());
+ return deopt_index_;
+ }
+
+ bool has_deoptimization_index() const {
+ DCHECK(is_valid());
+ return deopt_index_ != kNoDeoptIndex;
+ }
+
+ uint8_t* bits() {
+ DCHECK(is_valid());
+ return bits_;
+ }
+
+ private:
+ unsigned deopt_index_;
+ uint8_t* bits_;
+ // It needs to be an integer as it is -1 for eager deoptimizations.
+ int trampoline_pc_;
+};
+
+class SafepointTable {
+ public:
+ explicit SafepointTable(Code code);
+ explicit SafepointTable(const wasm::WasmCode* code);
+
+ int size() const {
+ return kHeaderSize + (length_ * (kFixedEntrySize + entry_size_));
+ }
+ unsigned length() const { return length_; }
+ unsigned entry_size() const { return entry_size_; }
+
+ unsigned GetPcOffset(unsigned index) const {
+ DCHECK(index < length_);
+ return base::Memory<uint32_t>(GetPcOffsetLocation(index));
+ }
+
+ int GetTrampolinePcOffset(unsigned index) const {
+ DCHECK(index < length_);
+ return base::Memory<int>(GetTrampolineLocation(index));
+ }
+
+ unsigned find_return_pc(unsigned pc_offset);
+
+ SafepointEntry GetEntry(unsigned index) const {
+ DCHECK(index < length_);
+ unsigned deopt_index =
+ base::Memory<uint32_t>(GetEncodedInfoLocation(index));
+ uint8_t* bits = &base::Memory<uint8_t>(entries() + (index * entry_size_));
+ int trampoline_pc =
+ has_deopt_ ? base::Memory<int>(GetTrampolineLocation(index)) : -1;
+ return SafepointEntry(deopt_index, bits, trampoline_pc);
+ }
+
+ // Returns the entry for the given pc.
+ SafepointEntry FindEntry(Address pc) const;
+
+ void PrintEntry(unsigned index, std::ostream& os) const; // NOLINT
+
+ private:
+ SafepointTable(Address instruction_start, Address safepoint_table_address,
+ uint32_t stack_slots, bool has_deopt);
+
+ static const uint8_t kNoRegisters = 0xFF;
+
+ // Layout information.
+ static const int kLengthOffset = 0;
+ static const int kEntrySizeOffset = kLengthOffset + kIntSize;
+ static const int kHeaderSize = kEntrySizeOffset + kIntSize;
+ static const int kPcOffset = 0;
+ static const int kEncodedInfoOffset = kPcOffset + kIntSize;
+ static const int kTrampolinePcOffset = kEncodedInfoOffset + kIntSize;
+ static const int kFixedEntrySize = kTrampolinePcOffset + kIntSize;
+
+ static uint32_t ReadLength(Address table) {
+ return base::Memory<uint32_t>(table + kLengthOffset);
+ }
+ static uint32_t ReadEntrySize(Address table) {
+ return base::Memory<uint32_t>(table + kEntrySizeOffset);
+ }
+ Address pc_and_deoptimization_indexes() const {
+ return safepoint_table_address_ + kHeaderSize;
+ }
+ Address entries() const {
+ return safepoint_table_address_ + kHeaderSize + (length_ * kFixedEntrySize);
+ }
+
+ Address GetPcOffsetLocation(unsigned index) const {
+ return pc_and_deoptimization_indexes() + (index * kFixedEntrySize);
+ }
+
+ Address GetEncodedInfoLocation(unsigned index) const {
+ return GetPcOffsetLocation(index) + kEncodedInfoOffset;
+ }
+
+ Address GetTrampolineLocation(unsigned index) const {
+ return GetPcOffsetLocation(index) + kTrampolinePcOffset;
+ }
+
+ static void PrintBits(std::ostream& os, uint8_t byte, int digits);
+
+ DISALLOW_HEAP_ALLOCATION(no_allocation_)
+
+ const Address instruction_start_;
+ const uint32_t stack_slots_;
+ const bool has_deopt_;
+
+ // Safepoint table layout.
+ const Address safepoint_table_address_;
+ const uint32_t length_;
+ const uint32_t entry_size_;
+
+ friend class SafepointTableBuilder;
+ friend class SafepointEntry;
+
+ DISALLOW_COPY_AND_ASSIGN(SafepointTable);
+};
+
+class Safepoint {
+ public:
+ enum DeoptMode { kNoLazyDeopt, kLazyDeopt };
+
+ static const int kNoDeoptimizationIndex = SafepointEntry::kNoDeoptIndex;
+
+ void DefinePointerSlot(int index) { indexes_->push_back(index); }
+
+ private:
+ explicit Safepoint(ZoneChunkList<int>* indexes) : indexes_(indexes) {}
+ ZoneChunkList<int>* const indexes_;
+
+ friend class SafepointTableBuilder;
+};
+
+class SafepointTableBuilder {
+ public:
+ explicit SafepointTableBuilder(Zone* zone)
+ : deoptimization_info_(zone),
+ emitted_(false),
+ zone_(zone) {}
+
+ // Get the offset of the emitted safepoint table in the code.
+ unsigned GetCodeOffset() const;
+
+ // Define a new safepoint for the current position in the body.
+ Safepoint DefineSafepoint(Assembler* assembler, Safepoint::DeoptMode mode);
+
+ // Emit the safepoint table after the body. The number of bits per
+ // entry must be enough to hold all the pointer indexes.
+ V8_EXPORT_PRIVATE void Emit(Assembler* assembler, int bits_per_entry);
+
+ // Find the Deoptimization Info with pc offset {pc} and update its
+ // trampoline field. Calling this function ensures that the safepoint
+ // table contains the trampoline PC {trampoline} that replaced the
+ // return PC {pc} on the stack.
+ int UpdateDeoptimizationInfo(int pc, int trampoline, int start,
+ unsigned deopt_index);
+
+ private:
+ struct DeoptimizationInfo {
+ unsigned pc;
+ unsigned deopt_index;
+ int trampoline;
+ ZoneChunkList<int>* indexes;
+ DeoptimizationInfo(Zone* zone, unsigned pc)
+ : pc(pc),
+ deopt_index(Safepoint::kNoDeoptimizationIndex),
+ trampoline(-1),
+ indexes(zone->New<ZoneChunkList<int>>(
+ zone, ZoneChunkList<int>::StartMode::kSmall)) {}
+ };
+
+ // Compares all fields of a {DeoptimizationInfo} except {pc} and {trampoline}.
+ bool IsIdenticalExceptForPc(const DeoptimizationInfo&,
+ const DeoptimizationInfo&) const;
+
+ // If all entries are identical, replace them by 1 entry with pc = kMaxUInt32.
+ void RemoveDuplicates();
+
+ ZoneChunkList<DeoptimizationInfo> deoptimization_info_;
+
+ unsigned offset_;
+ bool emitted_;
+
+ Zone* zone_;
+
+ DISALLOW_COPY_AND_ASSIGN(SafepointTableBuilder);
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_SAFEPOINT_TABLE_H_
diff --git a/src/codegen/signature.h b/src/codegen/signature.h
new file mode 100644
index 0000000..bba3a1b
--- /dev/null
+++ b/src/codegen/signature.h
@@ -0,0 +1,130 @@
+// Copyright 2015 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_SIGNATURE_H_
+#define V8_CODEGEN_SIGNATURE_H_
+
+#include "src/base/functional.h"
+#include "src/base/iterator.h"
+#include "src/codegen/machine-type.h"
+#include "src/zone/zone.h"
+
+namespace v8 {
+namespace internal {
+
+// Describes the inputs and outputs of a function or call.
+template <typename T>
+class Signature : public ZoneObject {
+ public:
+ constexpr Signature(size_t return_count, size_t parameter_count,
+ const T* reps)
+ : return_count_(return_count),
+ parameter_count_(parameter_count),
+ reps_(reps) {
+ DCHECK_EQ(kReturnCountOffset, offsetof(Signature, return_count_));
+ DCHECK_EQ(kParameterCountOffset, offsetof(Signature, parameter_count_));
+ DCHECK_EQ(kRepsOffset, offsetof(Signature, reps_));
+ STATIC_ASSERT(std::is_standard_layout<Signature<T>>::value);
+ }
+
+ size_t return_count() const { return return_count_; }
+ size_t parameter_count() const { return parameter_count_; }
+
+ T GetParam(size_t index) const {
+ DCHECK_LT(index, parameter_count_);
+ return reps_[return_count_ + index];
+ }
+
+ T GetReturn(size_t index = 0) const {
+ DCHECK_LT(index, return_count_);
+ return reps_[index];
+ }
+
+ // Iteration support.
+ base::iterator_range<const T*> parameters() const {
+ return {reps_ + return_count_, reps_ + return_count_ + parameter_count_};
+ }
+ base::iterator_range<const T*> returns() const {
+ return {reps_, reps_ + return_count_};
+ }
+ base::iterator_range<const T*> all() const {
+ return {reps_, reps_ + return_count_ + parameter_count_};
+ }
+
+ bool operator==(const Signature& other) const {
+ if (this == &other) return true;
+ if (parameter_count() != other.parameter_count()) return false;
+ if (return_count() != other.return_count()) return false;
+ return std::equal(all().begin(), all().end(), other.all().begin());
+ }
+ bool operator!=(const Signature& other) const { return !(*this == other); }
+
+ // For incrementally building signatures.
+ class Builder {
+ public:
+ Builder(Zone* zone, size_t return_count, size_t parameter_count)
+ : return_count_(return_count),
+ parameter_count_(parameter_count),
+ zone_(zone),
+ rcursor_(0),
+ pcursor_(0),
+ buffer_(zone->NewArray<T>(
+ static_cast<int>(return_count + parameter_count))) {}
+
+ const size_t return_count_;
+ const size_t parameter_count_;
+
+ void AddReturn(T val) {
+ DCHECK_LT(rcursor_, return_count_);
+ buffer_[rcursor_++] = val;
+ }
+
+ void AddParam(T val) {
+ DCHECK_LT(pcursor_, parameter_count_);
+ buffer_[return_count_ + pcursor_++] = val;
+ }
+
+ void AddParamAt(size_t index, T val) {
+ DCHECK_LT(index, parameter_count_);
+ buffer_[return_count_ + index] = val;
+ pcursor_ = std::max(pcursor_, index + 1);
+ }
+
+ Signature<T>* Build() {
+ DCHECK_EQ(rcursor_, return_count_);
+ DCHECK_EQ(pcursor_, parameter_count_);
+ return zone_->New<Signature<T>>(return_count_, parameter_count_, buffer_);
+ }
+
+ private:
+ Zone* zone_;
+ size_t rcursor_;
+ size_t pcursor_;
+ T* buffer_;
+ };
+
+ static constexpr size_t kReturnCountOffset = 0;
+ static constexpr size_t kParameterCountOffset =
+ kReturnCountOffset + kSizetSize;
+ static constexpr size_t kRepsOffset = kParameterCountOffset + kSizetSize;
+
+ protected:
+ size_t return_count_;
+ size_t parameter_count_;
+ const T* reps_;
+};
+
+using MachineSignature = Signature<MachineType>;
+
+template <typename T>
+size_t hash_value(const Signature<T>& sig) {
+ size_t hash = base::hash_combine(sig.parameter_count(), sig.return_count());
+ for (const T& t : sig.all()) hash = base::hash_combine(hash, t);
+ return hash;
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_SIGNATURE_H_
diff --git a/src/codegen/source-position-table.cc b/src/codegen/source-position-table.cc
new file mode 100644
index 0000000..3c8a108
--- /dev/null
+++ b/src/codegen/source-position-table.cc
@@ -0,0 +1,279 @@
+// Copyright 2016 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/source-position-table.h"
+
+#include "src/base/export-template.h"
+#include "src/heap/local-factory-inl.h"
+#include "src/objects/objects-inl.h"
+#include "src/objects/objects.h"
+
+namespace v8 {
+namespace internal {
+
+// We'll use a simple encoding scheme to record the source positions.
+// Conceptually, each position consists of:
+// - code_offset: An integer index into the BytecodeArray or code.
+// - source_position: An integer index into the source string.
+// - position type: Each position is either a statement or an expression.
+//
+// The basic idea for the encoding is to use a variable-length integer coding,
+// where each byte contains 7 bits of payload data, and 1 'more' bit that
+// determines whether additional bytes follow. Additionally:
+// - we record the difference from the previous position,
+// - we just stuff one bit for the type into the code offset,
+// - we write least-significant bits first,
+// - we use zig-zag encoding to encode both positive and negative numbers.
+
+namespace {
+
+// Each byte is encoded as MoreBit | ValueBits.
+using MoreBit = base::BitField8<bool, 7, 1>;
+using ValueBits = base::BitField8<unsigned, 0, 7>;
+
+// Helper: Add the offsets from 'other' to 'value'. Also set is_statement.
+void AddAndSetEntry(PositionTableEntry* value,
+ const PositionTableEntry& other) {
+ value->code_offset += other.code_offset;
+ value->source_position += other.source_position;
+ value->is_statement = other.is_statement;
+}
+
+// Helper: Subtract the offsets from 'other' from 'value'.
+void SubtractFromEntry(PositionTableEntry* value,
+ const PositionTableEntry& other) {
+ value->code_offset -= other.code_offset;
+ value->source_position -= other.source_position;
+}
+
+// Helper: Encode an integer.
+template <typename T>
+void EncodeInt(ZoneVector<byte>* bytes, T value) {
+ using unsigned_type = typename std::make_unsigned<T>::type;
+ // Zig-zag encoding.
+ static constexpr int kShift = sizeof(T) * kBitsPerByte - 1;
+ value = ((static_cast<unsigned_type>(value) << 1) ^ (value >> kShift));
+ DCHECK_GE(value, 0);
+ unsigned_type encoded = static_cast<unsigned_type>(value);
+ bool more;
+ do {
+ more = encoded > ValueBits::kMax;
+ byte current =
+ MoreBit::encode(more) | ValueBits::encode(encoded & ValueBits::kMask);
+ bytes->push_back(current);
+ encoded >>= ValueBits::kSize;
+ } while (more);
+}
+
+// Encode a PositionTableEntry.
+void EncodeEntry(ZoneVector<byte>* bytes, const PositionTableEntry& entry) {
+ // We only accept ascending code offsets.
+ DCHECK_GE(entry.code_offset, 0);
+ // Since code_offset is not negative, we use sign to encode is_statement.
+ EncodeInt(bytes,
+ entry.is_statement ? entry.code_offset : -entry.code_offset - 1);
+ EncodeInt(bytes, entry.source_position);
+}
+
+// Helper: Decode an integer.
+template <typename T>
+T DecodeInt(Vector<const byte> bytes, int* index) {
+ byte current;
+ int shift = 0;
+ T decoded = 0;
+ bool more;
+ do {
+ current = bytes[(*index)++];
+ decoded |= static_cast<typename std::make_unsigned<T>::type>(
+ ValueBits::decode(current))
+ << shift;
+ more = MoreBit::decode(current);
+ shift += ValueBits::kSize;
+ } while (more);
+ DCHECK_GE(decoded, 0);
+ decoded = (decoded >> 1) ^ (-(decoded & 1));
+ return decoded;
+}
+
+void DecodeEntry(Vector<const byte> bytes, int* index,
+ PositionTableEntry* entry) {
+ int tmp = DecodeInt<int>(bytes, index);
+ if (tmp >= 0) {
+ entry->is_statement = true;
+ entry->code_offset = tmp;
+ } else {
+ entry->is_statement = false;
+ entry->code_offset = -(tmp + 1);
+ }
+ entry->source_position = DecodeInt<int64_t>(bytes, index);
+}
+
+Vector<const byte> VectorFromByteArray(ByteArray byte_array) {
+ return Vector<const byte>(byte_array.GetDataStartAddress(),
+ byte_array.length());
+}
+
+#ifdef ENABLE_SLOW_DCHECKS
+void CheckTableEquals(const ZoneVector<PositionTableEntry>& raw_entries,
+ SourcePositionTableIterator* encoded) {
+ // Brute force testing: Record all positions and decode
+ // the entire table to verify they are identical.
+ auto raw = raw_entries.begin();
+ for (; !encoded->done(); encoded->Advance(), raw++) {
+ DCHECK(raw != raw_entries.end());
+ DCHECK_EQ(encoded->code_offset(), raw->code_offset);
+ DCHECK_EQ(encoded->source_position().raw(), raw->source_position);
+ DCHECK_EQ(encoded->is_statement(), raw->is_statement);
+ }
+ DCHECK(raw == raw_entries.end());
+}
+#endif
+
+} // namespace
+
+SourcePositionTableBuilder::SourcePositionTableBuilder(
+ Zone* zone, SourcePositionTableBuilder::RecordingMode mode)
+ : mode_(mode),
+ bytes_(zone),
+#ifdef ENABLE_SLOW_DCHECKS
+ raw_entries_(zone),
+#endif
+ previous_() {
+}
+
+void SourcePositionTableBuilder::AddPosition(size_t code_offset,
+ SourcePosition source_position,
+ bool is_statement) {
+ if (Omit()) return;
+ DCHECK(source_position.IsKnown());
+ int offset = static_cast<int>(code_offset);
+ AddEntry({offset, source_position.raw(), is_statement});
+}
+
+void SourcePositionTableBuilder::AddEntry(const PositionTableEntry& entry) {
+ PositionTableEntry tmp(entry);
+ SubtractFromEntry(&tmp, previous_);
+ EncodeEntry(&bytes_, tmp);
+ previous_ = entry;
+#ifdef ENABLE_SLOW_DCHECKS
+ raw_entries_.push_back(entry);
+#endif
+}
+
+template <typename LocalIsolate>
+Handle<ByteArray> SourcePositionTableBuilder::ToSourcePositionTable(
+ LocalIsolate* isolate) {
+ if (bytes_.empty()) return isolate->factory()->empty_byte_array();
+ DCHECK(!Omit());
+
+ Handle<ByteArray> table = isolate->factory()->NewByteArray(
+ static_cast<int>(bytes_.size()), AllocationType::kOld);
+ MemCopy(table->GetDataStartAddress(), bytes_.data(), bytes_.size());
+
+#ifdef ENABLE_SLOW_DCHECKS
+ // Brute force testing: Record all positions and decode
+ // the entire table to verify they are identical.
+ SourcePositionTableIterator it(
+ *table, SourcePositionTableIterator::kAll,
+ SourcePositionTableIterator::kDontSkipFunctionEntry);
+ CheckTableEquals(raw_entries_, &it);
+ // No additional source positions after creating the table.
+ mode_ = OMIT_SOURCE_POSITIONS;
+#endif
+ return table;
+}
+
+template EXPORT_TEMPLATE_DEFINE(V8_EXPORT_PRIVATE)
+ Handle<ByteArray> SourcePositionTableBuilder::ToSourcePositionTable(
+ Isolate* isolate);
+template EXPORT_TEMPLATE_DEFINE(V8_EXPORT_PRIVATE)
+ Handle<ByteArray> SourcePositionTableBuilder::ToSourcePositionTable(
+ LocalIsolate* isolate);
+
+OwnedVector<byte> SourcePositionTableBuilder::ToSourcePositionTableVector() {
+ if (bytes_.empty()) return OwnedVector<byte>();
+ DCHECK(!Omit());
+
+ OwnedVector<byte> table = OwnedVector<byte>::Of(bytes_);
+
+#ifdef ENABLE_SLOW_DCHECKS
+ // Brute force testing: Record all positions and decode
+ // the entire table to verify they are identical.
+ SourcePositionTableIterator it(
+ table.as_vector(), SourcePositionTableIterator::kAll,
+ SourcePositionTableIterator::kDontSkipFunctionEntry);
+ CheckTableEquals(raw_entries_, &it);
+ // No additional source positions after creating the table.
+ mode_ = OMIT_SOURCE_POSITIONS;
+#endif
+ return table;
+}
+
+void SourcePositionTableIterator::Initialize() {
+ Advance();
+ if (function_entry_filter_ == kSkipFunctionEntry &&
+ current_.code_offset == kFunctionEntryBytecodeOffset && !done()) {
+ Advance();
+ }
+}
+
+SourcePositionTableIterator::SourcePositionTableIterator(
+ ByteArray byte_array, IterationFilter iteration_filter,
+ FunctionEntryFilter function_entry_filter)
+ : raw_table_(VectorFromByteArray(byte_array)),
+ iteration_filter_(iteration_filter),
+ function_entry_filter_(function_entry_filter) {
+ Initialize();
+}
+
+SourcePositionTableIterator::SourcePositionTableIterator(
+ Handle<ByteArray> byte_array, IterationFilter iteration_filter,
+ FunctionEntryFilter function_entry_filter)
+ : table_(byte_array),
+ iteration_filter_(iteration_filter),
+ function_entry_filter_(function_entry_filter) {
+ Initialize();
+#ifdef DEBUG
+ // We can enable allocation because we keep the table in a handle.
+ no_gc.Release();
+#endif // DEBUG
+}
+
+SourcePositionTableIterator::SourcePositionTableIterator(
+ Vector<const byte> bytes, IterationFilter iteration_filter,
+ FunctionEntryFilter function_entry_filter)
+ : raw_table_(bytes),
+ iteration_filter_(iteration_filter),
+ function_entry_filter_(function_entry_filter) {
+ Initialize();
+#ifdef DEBUG
+ // We can enable allocation because the underlying vector does not move.
+ no_gc.Release();
+#endif // DEBUG
+}
+
+void SourcePositionTableIterator::Advance() {
+ Vector<const byte> bytes =
+ table_.is_null() ? raw_table_ : VectorFromByteArray(*table_);
+ DCHECK(!done());
+ DCHECK(index_ >= 0 && index_ <= bytes.length());
+ bool filter_satisfied = false;
+ while (!done() && !filter_satisfied) {
+ if (index_ >= bytes.length()) {
+ index_ = kDone;
+ } else {
+ PositionTableEntry tmp;
+ DecodeEntry(bytes, &index_, &tmp);
+ AddAndSetEntry(¤t_, tmp);
+ SourcePosition p = source_position();
+ filter_satisfied =
+ (iteration_filter_ == kAll) ||
+ (iteration_filter_ == kJavaScriptOnly && p.IsJavaScript()) ||
+ (iteration_filter_ == kExternalOnly && p.IsExternal());
+ }
+ }
+}
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/source-position-table.h b/src/codegen/source-position-table.h
new file mode 100644
index 0000000..a42c6a4
--- /dev/null
+++ b/src/codegen/source-position-table.h
@@ -0,0 +1,167 @@
+// Copyright 2016 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_SOURCE_POSITION_TABLE_H_
+#define V8_CODEGEN_SOURCE_POSITION_TABLE_H_
+
+#include "src/base/export-template.h"
+#include "src/codegen/source-position.h"
+#include "src/common/assert-scope.h"
+#include "src/common/checks.h"
+#include "src/common/globals.h"
+#include "src/utils/vector.h"
+#include "src/zone/zone-containers.h"
+
+namespace v8 {
+namespace internal {
+
+class ByteArray;
+template <typename T>
+class Handle;
+class Isolate;
+class Zone;
+
+struct PositionTableEntry {
+ PositionTableEntry()
+ : code_offset(kFunctionEntryBytecodeOffset),
+ source_position(0),
+ is_statement(false) {}
+ PositionTableEntry(int offset, int64_t source, bool statement)
+ : code_offset(offset), source_position(source), is_statement(statement) {}
+
+ int code_offset;
+ int64_t source_position;
+ bool is_statement;
+};
+
+class V8_EXPORT_PRIVATE SourcePositionTableBuilder {
+ public:
+ enum RecordingMode {
+ // Indicates that source positions are never to be generated. (Resulting in
+ // an empty table).
+ OMIT_SOURCE_POSITIONS,
+ // Indicates that source positions are not currently required, but may be
+ // generated later.
+ LAZY_SOURCE_POSITIONS,
+ // Indicates that source positions should be immediately generated.
+ RECORD_SOURCE_POSITIONS
+ };
+
+ explicit SourcePositionTableBuilder(
+ Zone* zone, RecordingMode mode = RECORD_SOURCE_POSITIONS);
+
+ void AddPosition(size_t code_offset, SourcePosition source_position,
+ bool is_statement);
+
+ template <typename LocalIsolate>
+ EXPORT_TEMPLATE_DECLARE(V8_EXPORT_PRIVATE)
+ Handle<ByteArray> ToSourcePositionTable(LocalIsolate* isolate);
+ OwnedVector<byte> ToSourcePositionTableVector();
+
+ inline bool Omit() const { return mode_ != RECORD_SOURCE_POSITIONS; }
+ inline bool Lazy() const { return mode_ == LAZY_SOURCE_POSITIONS; }
+
+ private:
+ void AddEntry(const PositionTableEntry& entry);
+
+ RecordingMode mode_;
+ ZoneVector<byte> bytes_;
+#ifdef ENABLE_SLOW_DCHECKS
+ ZoneVector<PositionTableEntry> raw_entries_;
+#endif
+ PositionTableEntry previous_; // Previously written entry, to compute delta.
+};
+
+class V8_EXPORT_PRIVATE SourcePositionTableIterator {
+ public:
+ // Filter that applies when advancing the iterator. If the filter isn't
+ // satisfied, we advance the iterator again.
+ enum IterationFilter { kJavaScriptOnly = 0, kExternalOnly = 1, kAll = 2 };
+ // Filter that applies only to the first entry of the source position table.
+ // If it is kSkipFunctionEntry, it will skip the FunctionEntry entry if it
+ // exists.
+ enum FunctionEntryFilter {
+ kSkipFunctionEntry = 0,
+ kDontSkipFunctionEntry = 1
+ };
+
+ // Used for saving/restoring the iterator.
+ struct IndexAndPositionState {
+ int index_;
+ PositionTableEntry position_;
+ IterationFilter iteration_filter_;
+ FunctionEntryFilter function_entry_filter_;
+ };
+
+ // We expose three flavours of the iterator, depending on the argument passed
+ // to the constructor:
+
+ // Handlified iterator allows allocation, but it needs a handle (and thus
+ // a handle scope). This is the preferred version.
+ explicit SourcePositionTableIterator(
+ Handle<ByteArray> byte_array,
+ IterationFilter iteration_filter = kJavaScriptOnly,
+ FunctionEntryFilter function_entry_filter = kSkipFunctionEntry);
+
+ // Non-handlified iterator does not need a handle scope, but it disallows
+ // allocation during its lifetime. This is useful if there is no handle
+ // scope around.
+ explicit SourcePositionTableIterator(
+ ByteArray byte_array, IterationFilter iteration_filter = kJavaScriptOnly,
+ FunctionEntryFilter function_entry_filter = kSkipFunctionEntry);
+
+ // Handle-safe iterator based on an a vector located outside the garbage
+ // collected heap, allows allocation during its lifetime.
+ explicit SourcePositionTableIterator(
+ Vector<const byte> bytes,
+ IterationFilter iteration_filter = kJavaScriptOnly,
+ FunctionEntryFilter function_entry_filter = kSkipFunctionEntry);
+
+ void Advance();
+
+ int code_offset() const {
+ DCHECK(!done());
+ return current_.code_offset;
+ }
+ SourcePosition source_position() const {
+ DCHECK(!done());
+ return SourcePosition::FromRaw(current_.source_position);
+ }
+ bool is_statement() const {
+ DCHECK(!done());
+ return current_.is_statement;
+ }
+ bool done() const { return index_ == kDone; }
+
+ IndexAndPositionState GetState() const {
+ return {index_, current_, iteration_filter_, function_entry_filter_};
+ }
+
+ void RestoreState(const IndexAndPositionState& saved_state) {
+ index_ = saved_state.index_;
+ current_ = saved_state.position_;
+ iteration_filter_ = saved_state.iteration_filter_;
+ function_entry_filter_ = saved_state.function_entry_filter_;
+ }
+
+ private:
+ // Initializes the source position interator with the first valid bytecode.
+ // Also sets the FunctionEntry SourcePosition if it exists.
+ void Initialize();
+
+ static const int kDone = -1;
+
+ Vector<const byte> raw_table_;
+ Handle<ByteArray> table_;
+ int index_ = 0;
+ PositionTableEntry current_;
+ IterationFilter iteration_filter_;
+ FunctionEntryFilter function_entry_filter_;
+ DISALLOW_HEAP_ALLOCATION(no_gc)
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_SOURCE_POSITION_TABLE_H_
diff --git a/src/codegen/source-position.cc b/src/codegen/source-position.cc
new file mode 100644
index 0000000..fa24127
--- /dev/null
+++ b/src/codegen/source-position.cc
@@ -0,0 +1,153 @@
+// Copyright 2016 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/source-position.h"
+#include "src/codegen/optimized-compilation-info.h"
+#include "src/objects/objects-inl.h"
+
+namespace v8 {
+namespace internal {
+
+std::ostream& operator<<(std::ostream& out, const SourcePositionInfo& pos) {
+ out << "<";
+ if (!pos.script.is_null() && pos.script->name().IsString()) {
+ out << String::cast(pos.script->name()).ToCString(DISALLOW_NULLS).get();
+ } else {
+ out << "unknown";
+ }
+ out << ":" << pos.line + 1 << ":" << pos.column + 1 << ">";
+ return out;
+}
+
+std::ostream& operator<<(std::ostream& out,
+ const std::vector<SourcePositionInfo>& stack) {
+ bool first = true;
+ for (const SourcePositionInfo& pos : stack) {
+ if (!first) out << " inlined at ";
+ out << pos;
+ first = false;
+ }
+ return out;
+}
+
+std::ostream& operator<<(std::ostream& out, const SourcePosition& pos) {
+ if (pos.isInlined()) {
+ out << "<inlined(" << pos.InliningId() << "):";
+ } else {
+ out << "<not inlined:";
+ }
+
+ if (pos.IsExternal()) {
+ out << pos.ExternalLine() << ", " << pos.ExternalFileId() << ">";
+ } else {
+ out << pos.ScriptOffset() << ">";
+ }
+ return out;
+}
+
+std::vector<SourcePositionInfo> SourcePosition::InliningStack(
+ OptimizedCompilationInfo* cinfo) const {
+ SourcePosition pos = *this;
+ std::vector<SourcePositionInfo> stack;
+ while (pos.isInlined()) {
+ const auto& inl = cinfo->inlined_functions()[pos.InliningId()];
+ stack.push_back(SourcePositionInfo(pos, inl.shared_info));
+ pos = inl.position.position;
+ }
+ stack.push_back(SourcePositionInfo(pos, cinfo->shared_info()));
+ return stack;
+}
+
+std::vector<SourcePositionInfo> SourcePosition::InliningStack(
+ Handle<Code> code) const {
+ Isolate* isolate = code->GetIsolate();
+ Handle<DeoptimizationData> deopt_data(
+ DeoptimizationData::cast(code->deoptimization_data()), isolate);
+ SourcePosition pos = *this;
+ std::vector<SourcePositionInfo> stack;
+ while (pos.isInlined()) {
+ InliningPosition inl =
+ deopt_data->InliningPositions().get(pos.InliningId());
+ Handle<SharedFunctionInfo> function(
+ deopt_data->GetInlinedFunction(inl.inlined_function_id), isolate);
+ stack.push_back(SourcePositionInfo(pos, function));
+ pos = inl.position;
+ }
+ Handle<SharedFunctionInfo> function(
+ SharedFunctionInfo::cast(deopt_data->SharedFunctionInfo()), isolate);
+ stack.push_back(SourcePositionInfo(pos, function));
+ return stack;
+}
+
+void SourcePosition::Print(std::ostream& out,
+ SharedFunctionInfo function) const {
+ Script::PositionInfo pos;
+ Object source_name;
+ if (function.script().IsScript()) {
+ Script script = Script::cast(function.script());
+ source_name = script.name();
+ script.GetPositionInfo(ScriptOffset(), &pos, Script::WITH_OFFSET);
+ }
+ out << "<";
+ if (source_name.IsString()) {
+ out << String::cast(source_name)
+ .ToCString(DISALLOW_NULLS, ROBUST_STRING_TRAVERSAL)
+ .get();
+ } else {
+ out << "unknown";
+ }
+ out << ":" << pos.line + 1 << ":" << pos.column + 1 << ">";
+}
+
+void SourcePosition::PrintJson(std::ostream& out) const {
+ if (IsExternal()) {
+ out << "{ \"line\" : " << ExternalLine() << ", "
+ << " \"fileId\" : " << ExternalFileId() << ", "
+ << " \"inliningId\" : " << InliningId() << "}";
+ } else {
+ out << "{ \"scriptOffset\" : " << ScriptOffset() << ", "
+ << " \"inliningId\" : " << InliningId() << "}";
+ }
+}
+
+void SourcePosition::Print(std::ostream& out, Code code) const {
+ DeoptimizationData deopt_data =
+ DeoptimizationData::cast(code.deoptimization_data());
+ if (!isInlined()) {
+ SharedFunctionInfo function(
+ SharedFunctionInfo::cast(deopt_data.SharedFunctionInfo()));
+ Print(out, function);
+ } else {
+ InliningPosition inl = deopt_data.InliningPositions().get(InliningId());
+ if (inl.inlined_function_id == -1) {
+ out << *this;
+ } else {
+ SharedFunctionInfo function =
+ deopt_data.GetInlinedFunction(inl.inlined_function_id);
+ Print(out, function);
+ }
+ out << " inlined at ";
+ inl.position.Print(out, code);
+ }
+}
+
+SourcePositionInfo::SourcePositionInfo(SourcePosition pos,
+ Handle<SharedFunctionInfo> f)
+ : position(pos),
+ shared(f),
+ script(f.is_null() || !f->script().IsScript()
+ ? Handle<Script>::null()
+ : handle(Script::cast(f->script()), f->GetIsolate())) {
+ if (!script.is_null()) {
+ Script::PositionInfo info;
+ if (Script::GetPositionInfo(script, pos.ScriptOffset(), &info,
+ Script::WITH_OFFSET)) {
+ line = info.line;
+ column = info.column;
+ }
+ }
+}
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/source-position.h b/src/codegen/source-position.h
new file mode 100644
index 0000000..0db12ae
--- /dev/null
+++ b/src/codegen/source-position.h
@@ -0,0 +1,197 @@
+// Copyright 2016 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_SOURCE_POSITION_H_
+#define V8_CODEGEN_SOURCE_POSITION_H_
+
+#include <ostream>
+
+#include "src/base/bit-field.h"
+#include "src/common/globals.h"
+#include "src/flags/flags.h"
+#include "src/handles/handles.h"
+
+namespace v8 {
+namespace internal {
+
+class Code;
+class OptimizedCompilationInfo;
+class Script;
+class SharedFunctionInfo;
+struct SourcePositionInfo;
+
+// SourcePosition stores
+// - is_external (1 bit true/false)
+//
+// - if is_external is true:
+// - external_line (20 bits, non-negative int)
+// - external_file_id (10 bits, non-negative int)
+//
+// - if is_external is false:
+// - script_offset (30 bit non-negative int or kNoSourcePosition)
+//
+// - In both cases, there is an inlining_id.
+// - inlining_id (16 bit non-negative int or kNotInlined).
+//
+// An "external" SourcePosition is one given by a file_id and a line,
+// suitable for embedding references to .cc or .tq files.
+// Otherwise, a SourcePosition contains an offset into a JavaScript
+// file.
+//
+// A defined inlining_id refers to positions in
+// OptimizedCompilationInfo::inlined_functions or
+// DeoptimizationData::InliningPositions, depending on the compilation stage.
+class SourcePosition final {
+ public:
+ explicit SourcePosition(int script_offset, int inlining_id = kNotInlined)
+ : value_(0) {
+ SetIsExternal(false);
+ SetScriptOffset(script_offset);
+ SetInliningId(inlining_id);
+ }
+
+ // External SourcePositions should use the following method to construct
+ // SourcePositions to avoid confusion.
+ static SourcePosition External(int line, int file_id) {
+ return SourcePosition(line, file_id, kNotInlined);
+ }
+
+ static SourcePosition Unknown() { return SourcePosition(kNoSourcePosition); }
+ bool IsKnown() const {
+ if (IsExternal()) return true;
+ return ScriptOffset() != kNoSourcePosition || InliningId() != kNotInlined;
+ }
+ bool isInlined() const {
+ if (IsExternal()) return false;
+ return InliningId() != kNotInlined;
+ }
+
+ bool IsExternal() const { return IsExternalField::decode(value_); }
+ bool IsJavaScript() const { return !IsExternal(); }
+
+ int ExternalLine() const {
+ DCHECK(IsExternal());
+ return ExternalLineField::decode(value_);
+ }
+
+ int ExternalFileId() const {
+ DCHECK(IsExternal());
+ return ExternalFileIdField::decode(value_);
+ }
+
+ // Assumes that the code object is optimized
+ std::vector<SourcePositionInfo> InliningStack(Handle<Code> code) const;
+ std::vector<SourcePositionInfo> InliningStack(
+ OptimizedCompilationInfo* cinfo) const;
+
+ void Print(std::ostream& out, Code code) const;
+ void PrintJson(std::ostream& out) const;
+
+ int ScriptOffset() const {
+ DCHECK(IsJavaScript());
+ return ScriptOffsetField::decode(value_) - 1;
+ }
+ int InliningId() const { return InliningIdField::decode(value_) - 1; }
+
+ void SetIsExternal(bool external) {
+ value_ = IsExternalField::update(value_, external);
+ }
+ void SetExternalLine(int line) {
+ DCHECK(IsExternal());
+ DCHECK(line <= ExternalLineField::kMax - 1);
+ value_ = ExternalLineField::update(value_, line);
+ }
+ void SetExternalFileId(int file_id) {
+ DCHECK(IsExternal());
+ DCHECK(file_id <= ExternalFileIdField::kMax - 1);
+ value_ = ExternalFileIdField::update(value_, file_id);
+ }
+
+ void SetScriptOffset(int script_offset) {
+ DCHECK(IsJavaScript());
+ DCHECK(script_offset <= ScriptOffsetField::kMax - 2);
+ DCHECK_GE(script_offset, kNoSourcePosition);
+ value_ = ScriptOffsetField::update(value_, script_offset + 1);
+ }
+ void SetInliningId(int inlining_id) {
+ DCHECK(inlining_id <= InliningIdField::kMax - 2);
+ DCHECK_GE(inlining_id, kNotInlined);
+ value_ = InliningIdField::update(value_, inlining_id + 1);
+ }
+
+ static const int kNotInlined = -1;
+ STATIC_ASSERT(kNoSourcePosition == -1);
+
+ int64_t raw() const { return static_cast<int64_t>(value_); }
+ static SourcePosition FromRaw(int64_t raw) {
+ SourcePosition position = Unknown();
+ DCHECK_GE(raw, 0);
+ position.value_ = static_cast<uint64_t>(raw);
+ return position;
+ }
+
+ private:
+ // Used by SourcePosition::External(line, file_id).
+ SourcePosition(int line, int file_id, int inlining_id) : value_(0) {
+ SetIsExternal(true);
+ SetExternalLine(line);
+ SetExternalFileId(file_id);
+ SetInliningId(inlining_id);
+ }
+
+ void Print(std::ostream& out, SharedFunctionInfo function) const;
+
+ using IsExternalField = base::BitField64<bool, 0, 1>;
+
+ // The two below are only used if IsExternal() is true.
+ using ExternalLineField = base::BitField64<int, 1, 20>;
+ using ExternalFileIdField = base::BitField64<int, 21, 10>;
+
+ // ScriptOffsetField is only used if IsExternal() is false.
+ using ScriptOffsetField = base::BitField64<int, 1, 30>;
+
+ // InliningId is in the high bits for better compression in
+ // SourcePositionTable.
+ using InliningIdField = base::BitField64<int, 31, 16>;
+
+ // Leaving the highest bit untouched to allow for signed conversion.
+ uint64_t value_;
+};
+
+inline bool operator==(const SourcePosition& lhs, const SourcePosition& rhs) {
+ return lhs.raw() == rhs.raw();
+}
+
+inline bool operator!=(const SourcePosition& lhs, const SourcePosition& rhs) {
+ return !(lhs == rhs);
+}
+
+struct InliningPosition {
+ // position of the inlined call
+ SourcePosition position = SourcePosition::Unknown();
+
+ // references position in DeoptimizationData::literals()
+ int inlined_function_id;
+};
+
+struct SourcePositionInfo {
+ SourcePositionInfo(SourcePosition pos, Handle<SharedFunctionInfo> f);
+
+ SourcePosition position;
+ Handle<SharedFunctionInfo> shared;
+ Handle<Script> script;
+ int line = -1;
+ int column = -1;
+};
+
+std::ostream& operator<<(std::ostream& out, const SourcePosition& pos);
+
+std::ostream& operator<<(std::ostream& out, const SourcePositionInfo& pos);
+std::ostream& operator<<(std::ostream& out,
+ const std::vector<SourcePositionInfo>& stack);
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_SOURCE_POSITION_H_
diff --git a/src/codegen/string-constants.cc b/src/codegen/string-constants.cc
new file mode 100644
index 0000000..92a5e97
--- /dev/null
+++ b/src/codegen/string-constants.cc
@@ -0,0 +1,188 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/string-constants.h"
+
+#include "src/base/functional.h"
+#include "src/numbers/dtoa.h"
+#include "src/objects/objects.h"
+#include "src/objects/string-inl.h"
+
+namespace v8 {
+namespace internal {
+
+Handle<String> StringConstantBase::AllocateStringConstant(
+ Isolate* isolate) const {
+ if (!flattened_.is_null()) {
+ return flattened_;
+ }
+
+ Handle<String> result;
+ switch (kind()) {
+ case StringConstantKind::kStringLiteral: {
+ result = static_cast<const StringLiteral*>(this)->str();
+ CHECK(!result.is_null());
+ break;
+ }
+ case StringConstantKind::kNumberToStringConstant: {
+ auto num_constant = static_cast<const NumberToStringConstant*>(this);
+ Handle<Object> num_obj =
+ isolate->factory()->NewNumber(num_constant->num());
+ result = isolate->factory()->NumberToString(num_obj);
+ CHECK(!result.is_null());
+ break;
+ }
+ case StringConstantKind::kStringCons: {
+ Handle<String> lhs =
+ static_cast<const StringCons*>(this)->lhs()->AllocateStringConstant(
+ isolate);
+ Handle<String> rhs =
+ static_cast<const StringCons*>(this)->rhs()->AllocateStringConstant(
+ isolate);
+ result = isolate->factory()->NewConsString(lhs, rhs).ToHandleChecked();
+ break;
+ }
+ }
+
+ // TODO(mslekova): Normally we'd want to flatten the string here
+ // but that results in OOM for too long strings.
+ Memoize(result);
+ return flattened_;
+}
+
+bool StringConstantBase::operator==(const StringConstantBase& other) const {
+ if (kind() != other.kind()) return false;
+
+ switch (kind()) {
+ case StringConstantKind::kStringLiteral: {
+ return static_cast<const StringLiteral*>(this) ==
+ static_cast<const StringLiteral*>(&other);
+ }
+ case StringConstantKind::kNumberToStringConstant: {
+ return static_cast<const NumberToStringConstant*>(this) ==
+ static_cast<const NumberToStringConstant*>(&other);
+ }
+ case StringConstantKind::kStringCons: {
+ return static_cast<const StringCons*>(this) ==
+ static_cast<const StringCons*>(&other);
+ }
+ }
+ UNREACHABLE();
+}
+
+size_t hash_value(StringConstantBase const& base) {
+ switch (base.kind()) {
+ case StringConstantKind::kStringLiteral: {
+ return hash_value(*static_cast<const StringLiteral*>(&base));
+ }
+ case StringConstantKind::kNumberToStringConstant: {
+ return hash_value(*static_cast<const NumberToStringConstant*>(&base));
+ }
+ case StringConstantKind::kStringCons: {
+ return hash_value(*static_cast<const StringCons*>(&base));
+ }
+ }
+ UNREACHABLE();
+}
+
+bool operator==(StringLiteral const& lhs, StringLiteral const& rhs) {
+ return lhs.str().address() == rhs.str().address();
+}
+
+bool operator!=(StringLiteral const& lhs, StringLiteral const& rhs) {
+ return !(lhs == rhs);
+}
+
+size_t hash_value(StringLiteral const& p) {
+ return base::hash_combine(p.str().address());
+}
+
+std::ostream& operator<<(std::ostream& os, StringLiteral const& p) {
+ return os << Brief(*p.str());
+}
+
+bool operator==(NumberToStringConstant const& lhs,
+ NumberToStringConstant const& rhs) {
+ return lhs.num() == rhs.num();
+}
+
+bool operator!=(NumberToStringConstant const& lhs,
+ NumberToStringConstant const& rhs) {
+ return !(lhs == rhs);
+}
+
+size_t hash_value(NumberToStringConstant const& p) {
+ return base::hash_combine(p.num());
+}
+
+std::ostream& operator<<(std::ostream& os, NumberToStringConstant const& p) {
+ return os << p.num();
+}
+
+bool operator==(StringCons const& lhs, StringCons const& rhs) {
+ // TODO(mslekova): Think if we can express this in a more readable manner
+ return *(lhs.lhs()) == *(rhs.lhs()) && *(lhs.rhs()) == *(rhs.rhs());
+}
+
+bool operator!=(StringCons const& lhs, StringCons const& rhs) {
+ return !(lhs == rhs);
+}
+
+size_t hash_value(StringCons const& p) {
+ return base::hash_combine(*(p.lhs()), *(p.rhs()));
+}
+
+std::ostream& operator<<(std::ostream& os, const StringConstantBase* base) {
+ os << "DelayedStringConstant: ";
+ switch (base->kind()) {
+ case StringConstantKind::kStringLiteral: {
+ os << *static_cast<const StringLiteral*>(base);
+ break;
+ }
+ case StringConstantKind::kNumberToStringConstant: {
+ os << *static_cast<const NumberToStringConstant*>(base);
+ break;
+ }
+ case StringConstantKind::kStringCons: {
+ os << *static_cast<const StringCons*>(base);
+ break;
+ }
+ }
+ return os;
+}
+
+std::ostream& operator<<(std::ostream& os, StringCons const& p) {
+ return os << p.lhs() << ", " << p.rhs();
+}
+
+size_t StringConstantBase::GetMaxStringConstantLength() const {
+ switch (kind()) {
+ case StringConstantKind::kStringLiteral: {
+ return static_cast<const StringLiteral*>(this)
+ ->GetMaxStringConstantLength();
+ }
+ case StringConstantKind::kNumberToStringConstant: {
+ return static_cast<const NumberToStringConstant*>(this)
+ ->GetMaxStringConstantLength();
+ }
+ case StringConstantKind::kStringCons: {
+ return static_cast<const StringCons*>(this)->GetMaxStringConstantLength();
+ }
+ }
+ UNREACHABLE();
+}
+
+size_t StringLiteral::GetMaxStringConstantLength() const { return length_; }
+
+size_t NumberToStringConstant::GetMaxStringConstantLength() const {
+ return kBase10MaximalLength + 1;
+}
+
+size_t StringCons::GetMaxStringConstantLength() const {
+ return lhs()->GetMaxStringConstantLength() +
+ rhs()->GetMaxStringConstantLength();
+}
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/string-constants.h b/src/codegen/string-constants.h
new file mode 100644
index 0000000..8043c60
--- /dev/null
+++ b/src/codegen/string-constants.h
@@ -0,0 +1,116 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_STRING_CONSTANTS_H_
+#define V8_CODEGEN_STRING_CONSTANTS_H_
+
+#include "src/handles/handles.h"
+#include "src/objects/string.h"
+#include "src/zone/zone.h"
+
+namespace v8 {
+namespace internal {
+
+enum class StringConstantKind {
+ kStringLiteral,
+ kNumberToStringConstant,
+ kStringCons
+};
+
+class StringConstantBase : public ZoneObject {
+ public:
+ explicit StringConstantBase(StringConstantKind kind) : kind_(kind) {}
+
+ StringConstantKind kind() const { return kind_; }
+ Handle<String> AllocateStringConstant(Isolate* isolate) const;
+
+ size_t GetMaxStringConstantLength() const;
+
+ bool operator==(const StringConstantBase& other) const;
+
+ private:
+ void Memoize(Handle<String> flattened) const { flattened_ = flattened; }
+
+ StringConstantKind kind_;
+ mutable Handle<String> flattened_ = Handle<String>::null();
+};
+
+size_t hash_value(StringConstantBase const& base);
+
+class StringLiteral final : public StringConstantBase {
+ public:
+ explicit StringLiteral(Handle<String> str, size_t length)
+ : StringConstantBase(StringConstantKind::kStringLiteral),
+ str_(str),
+ length_(length) {}
+
+ Handle<String> str() const { return str_; }
+
+ size_t GetMaxStringConstantLength() const;
+
+ private:
+ Handle<String> str_;
+ size_t length_; // We store this separately to avoid accessing the heap.
+};
+
+bool operator==(StringLiteral const& lhs, StringLiteral const& rhs);
+bool operator!=(StringLiteral const& lhs, StringLiteral const& rhs);
+
+size_t hash_value(StringLiteral const& parameters);
+
+std::ostream& operator<<(std::ostream& os, StringLiteral const& parameters);
+
+class NumberToStringConstant final : public StringConstantBase {
+ public:
+ explicit NumberToStringConstant(double num)
+ : StringConstantBase(StringConstantKind::kNumberToStringConstant),
+ num_(num) {}
+
+ double num() const { return num_; }
+
+ size_t GetMaxStringConstantLength() const;
+
+ private:
+ double num_;
+};
+
+bool operator==(NumberToStringConstant const& lhs,
+ NumberToStringConstant const& rhs);
+bool operator!=(NumberToStringConstant const& lhs,
+ NumberToStringConstant const& rhs);
+
+size_t hash_value(NumberToStringConstant const& parameters);
+
+std::ostream& operator<<(std::ostream& os,
+ NumberToStringConstant const& parameters);
+
+class StringCons final : public StringConstantBase {
+ public:
+ explicit StringCons(const StringConstantBase* lhs,
+ const StringConstantBase* rhs)
+ : StringConstantBase(StringConstantKind::kStringCons),
+ lhs_(lhs),
+ rhs_(rhs) {}
+
+ const StringConstantBase* lhs() const { return lhs_; }
+ const StringConstantBase* rhs() const { return rhs_; }
+
+ size_t GetMaxStringConstantLength() const;
+
+ private:
+ const StringConstantBase* lhs_;
+ const StringConstantBase* rhs_;
+};
+
+bool operator==(StringCons const& lhs, StringCons const& rhs);
+bool operator!=(StringCons const& lhs, StringCons const& rhs);
+
+size_t hash_value(StringCons const& parameters);
+
+std::ostream& operator<<(std::ostream& os, StringCons const& parameters);
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_STRING_CONSTANTS_H_
diff --git a/src/codegen/tick-counter.cc b/src/codegen/tick-counter.cc
new file mode 100644
index 0000000..5172201
--- /dev/null
+++ b/src/codegen/tick-counter.cc
@@ -0,0 +1,34 @@
+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/tick-counter.h"
+
+#include "src/base/logging.h"
+#include "src/base/macros.h"
+#include "src/heap/local-heap.h"
+
+namespace v8 {
+namespace internal {
+
+void TickCounter::TickAndMaybeEnterSafepoint() {
+ ++ticks_;
+ // Magical number to detect performance bugs or compiler divergence.
+ // Selected as being roughly 10x of what's needed frequently.
+ constexpr size_t kMaxTicks = 100000000;
+ USE(kMaxTicks);
+ DCHECK_LT(ticks_, kMaxTicks);
+
+ if (local_heap_) local_heap_->Safepoint();
+}
+
+void TickCounter::AttachLocalHeap(LocalHeap* local_heap) {
+ DCHECK_NULL(local_heap_);
+ local_heap_ = local_heap;
+ DCHECK_NOT_NULL(local_heap_);
+}
+
+void TickCounter::DetachLocalHeap() { local_heap_ = nullptr; }
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/tick-counter.h b/src/codegen/tick-counter.h
new file mode 100644
index 0000000..3dbb404
--- /dev/null
+++ b/src/codegen/tick-counter.h
@@ -0,0 +1,35 @@
+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_TICK_COUNTER_H_
+#define V8_CODEGEN_TICK_COUNTER_H_
+
+#include <cstddef>
+
+namespace v8 {
+namespace internal {
+
+class LocalHeap;
+
+// This method generates a tick. Also makes the current thread to enter a
+// safepoint iff it was required to do so. The tick is used as a deterministic
+// correlate of time to detect performance or divergence bugs in Turbofan.
+// TickAndMaybeEnterSafepoint() should be called frequently thoughout the
+// compilation.
+class TickCounter {
+ public:
+ void TickAndMaybeEnterSafepoint();
+ void AttachLocalHeap(LocalHeap* local_heap);
+ void DetachLocalHeap();
+ size_t CurrentTicks() const { return ticks_; }
+
+ private:
+ size_t ticks_ = 0;
+ LocalHeap* local_heap_ = nullptr;
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_TICK_COUNTER_H_
diff --git a/src/codegen/tnode.cc b/src/codegen/tnode.cc
new file mode 100644
index 0000000..07b4d15
--- /dev/null
+++ b/src/codegen/tnode.cc
@@ -0,0 +1,13 @@
+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/tnode.h"
+
+namespace v8 {
+namespace internal {
+
+constexpr MachineType MachineTypeOf<ExternalReference>::value;
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/tnode.h b/src/codegen/tnode.h
new file mode 100644
index 0000000..72f9c6e
--- /dev/null
+++ b/src/codegen/tnode.h
@@ -0,0 +1,382 @@
+// Copyright 2015 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_TNODE_H_
+#define V8_CODEGEN_TNODE_H_
+
+#include "src/codegen/machine-type.h"
+
+namespace v8 {
+namespace internal {
+
+class HeapNumber;
+class BigInt;
+class Object;
+class Smi;
+class TaggedIndex;
+
+namespace compiler {
+
+class Node;
+
+} // namespace compiler
+
+struct UntaggedT {};
+
+struct IntegralT : UntaggedT {};
+
+struct WordT : IntegralT {
+ static const MachineRepresentation kMachineRepresentation =
+ MachineType::PointerRepresentation();
+};
+
+struct RawPtrT : WordT {
+ static constexpr MachineType kMachineType = MachineType::Pointer();
+};
+
+template <class To>
+struct RawPtr : RawPtrT {};
+
+struct Word32T : IntegralT {
+ static const MachineRepresentation kMachineRepresentation =
+ MachineRepresentation::kWord32;
+};
+struct Int32T : Word32T {
+ static constexpr MachineType kMachineType = MachineType::Int32();
+};
+struct Uint32T : Word32T {
+ static constexpr MachineType kMachineType = MachineType::Uint32();
+};
+struct Int16T : Int32T {
+ static constexpr MachineType kMachineType = MachineType::Int16();
+};
+struct Uint16T : Uint32T, Int32T {
+ static constexpr MachineType kMachineType = MachineType::Uint16();
+};
+struct Int8T : Int16T {
+ static constexpr MachineType kMachineType = MachineType::Int8();
+};
+struct Uint8T : Uint16T, Int16T {
+ static constexpr MachineType kMachineType = MachineType::Uint8();
+};
+
+struct Word64T : IntegralT {
+ static const MachineRepresentation kMachineRepresentation =
+ MachineRepresentation::kWord64;
+};
+struct Int64T : Word64T {
+ static constexpr MachineType kMachineType = MachineType::Int64();
+};
+struct Uint64T : Word64T {
+ static constexpr MachineType kMachineType = MachineType::Uint64();
+};
+
+struct IntPtrT : WordT {
+ static constexpr MachineType kMachineType = MachineType::IntPtr();
+};
+struct UintPtrT : WordT {
+ static constexpr MachineType kMachineType = MachineType::UintPtr();
+};
+
+struct ExternalPointerT : UntaggedT {
+ static const MachineRepresentation kMachineRepresentation =
+ MachineType::PointerRepresentation();
+ static constexpr MachineType kMachineType = MachineType::Pointer();
+};
+
+struct Float32T : UntaggedT {
+ static const MachineRepresentation kMachineRepresentation =
+ MachineRepresentation::kFloat32;
+ static constexpr MachineType kMachineType = MachineType::Float32();
+};
+
+struct Float64T : UntaggedT {
+ static const MachineRepresentation kMachineRepresentation =
+ MachineRepresentation::kFloat64;
+ static constexpr MachineType kMachineType = MachineType::Float64();
+};
+
+#ifdef V8_COMPRESS_POINTERS
+using TaggedT = Int32T;
+#else
+using TaggedT = IntPtrT;
+#endif
+
+// Result of a comparison operation.
+struct BoolT : Word32T {};
+
+// Value type of a Turbofan node with two results.
+template <class T1, class T2>
+struct PairT {};
+
+inline constexpr MachineType CommonMachineType(MachineType type1,
+ MachineType type2) {
+ return (type1 == type2) ? type1
+ : ((type1.IsTagged() && type2.IsTagged())
+ ? MachineType::AnyTagged()
+ : MachineType::None());
+}
+
+template <class Type, class Enable = void>
+struct MachineTypeOf {
+ static constexpr MachineType value = Type::kMachineType;
+};
+
+template <class Type, class Enable>
+constexpr MachineType MachineTypeOf<Type, Enable>::value;
+
+template <>
+struct MachineTypeOf<Object> {
+ static constexpr MachineType value = MachineType::AnyTagged();
+};
+template <>
+struct MachineTypeOf<MaybeObject> {
+ static constexpr MachineType value = MachineType::AnyTagged();
+};
+template <>
+struct MachineTypeOf<Smi> {
+ static constexpr MachineType value = MachineType::TaggedSigned();
+};
+template <>
+struct MachineTypeOf<TaggedIndex> {
+ static constexpr MachineType value = MachineType::Pointer();
+};
+template <class HeapObjectSubtype>
+struct MachineTypeOf<HeapObjectSubtype,
+ typename std::enable_if<std::is_base_of<
+ HeapObject, HeapObjectSubtype>::value>::type> {
+ static constexpr MachineType value = MachineType::TaggedPointer();
+};
+template <>
+struct MachineTypeOf<ExternalReference> {
+ static constexpr MachineType value = MachineType::Pointer();
+};
+
+template <class HeapObjectSubtype>
+constexpr MachineType MachineTypeOf<
+ HeapObjectSubtype, typename std::enable_if<std::is_base_of<
+ HeapObject, HeapObjectSubtype>::value>::type>::value;
+
+template <class Type, class Enable = void>
+struct MachineRepresentationOf {
+ static const MachineRepresentation value = Type::kMachineRepresentation;
+};
+// If T defines kMachineType, then we take the machine representation from
+// there.
+template <class T>
+struct MachineRepresentationOf<T, base::void_t<decltype(T::kMachineType)>> {
+ static const MachineRepresentation value = T::kMachineType.representation();
+};
+template <class T>
+struct MachineRepresentationOf<
+ T, typename std::enable_if<std::is_base_of<Object, T>::value>::type> {
+ static const MachineRepresentation value =
+ MachineTypeOf<T>::value.representation();
+};
+template <class T>
+struct MachineRepresentationOf<
+ T, typename std::enable_if<std::is_base_of<MaybeObject, T>::value>::type> {
+ static const MachineRepresentation value =
+ MachineTypeOf<T>::value.representation();
+};
+template <>
+struct MachineRepresentationOf<ExternalReference> {
+ static const MachineRepresentation value = RawPtrT::kMachineRepresentation;
+};
+
+template <typename T>
+constexpr bool IsMachineRepresentationOf(MachineRepresentation r) {
+ return MachineRepresentationOf<T>::value == r;
+}
+
+template <class T>
+constexpr MachineRepresentation PhiMachineRepresentationOf =
+ std::is_base_of<Word32T, T>::value ? MachineRepresentation::kWord32
+ : MachineRepresentationOf<T>::value;
+
+template <class T>
+struct is_valid_type_tag {
+ static const bool value = std::is_base_of<Object, T>::value ||
+ std::is_base_of<UntaggedT, T>::value ||
+ std::is_base_of<MaybeObject, T>::value ||
+ std::is_same<ExternalReference, T>::value;
+ static const bool is_tagged = std::is_base_of<Object, T>::value ||
+ std::is_base_of<MaybeObject, T>::value;
+};
+
+template <class T1, class T2>
+struct is_valid_type_tag<PairT<T1, T2>> {
+ static const bool value =
+ is_valid_type_tag<T1>::value && is_valid_type_tag<T2>::value;
+ static const bool is_tagged = false;
+};
+
+template <class T1, class T2>
+struct UnionT;
+
+template <class T1, class T2>
+struct is_valid_type_tag<UnionT<T1, T2>> {
+ static const bool is_tagged =
+ is_valid_type_tag<T1>::is_tagged && is_valid_type_tag<T2>::is_tagged;
+ static const bool value = is_tagged;
+};
+
+template <class T1, class T2>
+struct UnionT {
+ static constexpr MachineType kMachineType =
+ CommonMachineType(MachineTypeOf<T1>::value, MachineTypeOf<T2>::value);
+ static const MachineRepresentation kMachineRepresentation =
+ kMachineType.representation();
+ static_assert(kMachineRepresentation != MachineRepresentation::kNone,
+ "no common representation");
+ static_assert(is_valid_type_tag<T1>::is_tagged &&
+ is_valid_type_tag<T2>::is_tagged,
+ "union types are only possible for tagged values");
+};
+
+using AnyTaggedT = UnionT<Object, MaybeObject>;
+using Number = UnionT<Smi, HeapNumber>;
+using Numeric = UnionT<Number, BigInt>;
+using ContextOrEmptyContext = UnionT<Context, Smi>;
+
+// A pointer to a builtin function, used by Torque's function pointers.
+using BuiltinPtr = Smi;
+
+template <class T, class U>
+struct is_subtype {
+ static const bool value =
+ std::is_base_of<U, T>::value || (std::is_same<U, MaybeObject>::value &&
+ std::is_convertible<T, Object>::value);
+};
+template <class T1, class T2, class U>
+struct is_subtype<UnionT<T1, T2>, U> {
+ static const bool value =
+ is_subtype<T1, U>::value && is_subtype<T2, U>::value;
+};
+template <class T, class U1, class U2>
+struct is_subtype<T, UnionT<U1, U2>> {
+ static const bool value =
+ is_subtype<T, U1>::value || is_subtype<T, U2>::value;
+};
+template <class T1, class T2, class U1, class U2>
+struct is_subtype<UnionT<T1, T2>, UnionT<U1, U2>> {
+ static const bool value =
+ (is_subtype<T1, U1>::value || is_subtype<T1, U2>::value) &&
+ (is_subtype<T2, U1>::value || is_subtype<T2, U2>::value);
+};
+
+template <class T, class U>
+struct types_have_common_values {
+ static const bool value = is_subtype<T, U>::value || is_subtype<U, T>::value;
+};
+template <class U>
+struct types_have_common_values<BoolT, U> {
+ static const bool value = types_have_common_values<Word32T, U>::value;
+};
+template <class U>
+struct types_have_common_values<Uint32T, U> {
+ static const bool value = types_have_common_values<Word32T, U>::value;
+};
+template <class U>
+struct types_have_common_values<Int32T, U> {
+ static const bool value = types_have_common_values<Word32T, U>::value;
+};
+template <class U>
+struct types_have_common_values<Uint64T, U> {
+ static const bool value = types_have_common_values<Word64T, U>::value;
+};
+template <class U>
+struct types_have_common_values<Int64T, U> {
+ static const bool value = types_have_common_values<Word64T, U>::value;
+};
+template <class U>
+struct types_have_common_values<IntPtrT, U> {
+ static const bool value = types_have_common_values<WordT, U>::value;
+};
+template <class U>
+struct types_have_common_values<UintPtrT, U> {
+ static const bool value = types_have_common_values<WordT, U>::value;
+};
+template <class T1, class T2, class U>
+struct types_have_common_values<UnionT<T1, T2>, U> {
+ static const bool value = types_have_common_values<T1, U>::value ||
+ types_have_common_values<T2, U>::value;
+};
+
+template <class T, class U1, class U2>
+struct types_have_common_values<T, UnionT<U1, U2>> {
+ static const bool value = types_have_common_values<T, U1>::value ||
+ types_have_common_values<T, U2>::value;
+};
+template <class T1, class T2, class U1, class U2>
+struct types_have_common_values<UnionT<T1, T2>, UnionT<U1, U2>> {
+ static const bool value = types_have_common_values<T1, U1>::value ||
+ types_have_common_values<T1, U2>::value ||
+ types_have_common_values<T2, U1>::value ||
+ types_have_common_values<T2, U2>::value;
+};
+
+// TNode<T> is an SSA value with the static type tag T, which is one of the
+// following:
+// - MaybeObject represents the type of all tagged values, including weak
+// pointers.
+// - a subclass of internal::Object represents a non-weak tagged type.
+// - a subclass of internal::UntaggedT represents an untagged type
+// - ExternalReference
+// - PairT<T1, T2> for an operation returning two values, with types T1
+// and T2
+// - UnionT<T1, T2> represents either a value of type T1 or of type T2.
+template <class T>
+class TNode {
+ public:
+ template <class U,
+ typename std::enable_if<is_subtype<U, T>::value, int>::type = 0>
+ TNode(const TNode<U>& other) : node_(other) {
+ LazyTemplateChecks();
+ }
+ TNode() : TNode(nullptr) {}
+
+ TNode operator=(TNode other) {
+ DCHECK_NOT_NULL(other.node_);
+ node_ = other.node_;
+ return *this;
+ }
+
+ bool is_null() const { return node_ == nullptr; }
+
+ operator compiler::Node*() const { return node_; }
+
+ static TNode UncheckedCast(compiler::Node* node) { return TNode(node); }
+
+ protected:
+ explicit TNode(compiler::Node* node) : node_(node) { LazyTemplateChecks(); }
+
+ private:
+ // These checks shouldn't be checked before TNode is actually used.
+ void LazyTemplateChecks() {
+ static_assert(is_valid_type_tag<T>::value, "invalid type tag");
+ }
+
+ compiler::Node* node_;
+};
+
+// SloppyTNode<T> is a variant of TNode<T> and allows implicit casts from
+// Node*. It is intended for function arguments as long as some call sites
+// still use untyped Node* arguments.
+// TODO(tebbi): Delete this class once transition is finished.
+template <class T>
+class SloppyTNode : public TNode<T> {
+ public:
+ SloppyTNode(compiler::Node* node) // NOLINT(runtime/explicit)
+ : TNode<T>(node) {}
+ template <class U, typename std::enable_if<is_subtype<U, T>::value,
+ int>::type = 0>
+ SloppyTNode(const TNode<U>& other) // NOLINT(runtime/explicit)
+ : TNode<T>(other) {}
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_TNODE_H_
diff --git a/src/codegen/turbo-assembler.cc b/src/codegen/turbo-assembler.cc
new file mode 100644
index 0000000..575529e
--- /dev/null
+++ b/src/codegen/turbo-assembler.cc
@@ -0,0 +1,126 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/turbo-assembler.h"
+
+#include "src/builtins/builtins.h"
+#include "src/builtins/constants-table-builder.h"
+#include "src/codegen/external-reference-encoder.h"
+#include "src/execution/isolate-data.h"
+#include "src/execution/isolate-inl.h"
+
+namespace v8 {
+namespace internal {
+
+TurboAssemblerBase::TurboAssemblerBase(Isolate* isolate,
+ const AssemblerOptions& options,
+ CodeObjectRequired create_code_object,
+ std::unique_ptr<AssemblerBuffer> buffer)
+ : Assembler(options, std::move(buffer)), isolate_(isolate) {
+ if (create_code_object == CodeObjectRequired::kYes) {
+ code_object_ = Handle<HeapObject>::New(
+ ReadOnlyRoots(isolate).self_reference_marker(), isolate);
+ }
+}
+
+void TurboAssemblerBase::IndirectLoadConstant(Register destination,
+ Handle<HeapObject> object) {
+ CHECK(root_array_available_);
+
+ // Before falling back to the (fairly slow) lookup from the constants table,
+ // check if any of the fast paths can be applied.
+
+ int builtin_index;
+ RootIndex root_index;
+ if (isolate()->roots_table().IsRootHandle(object, &root_index)) {
+ // Roots are loaded relative to the root register.
+ LoadRoot(destination, root_index);
+ } else if (isolate()->builtins()->IsBuiltinHandle(object, &builtin_index)) {
+ // Similar to roots, builtins may be loaded from the builtins table.
+ LoadRootRelative(destination,
+ RootRegisterOffsetForBuiltinIndex(builtin_index));
+ } else if (object.is_identical_to(code_object_) &&
+ Builtins::IsBuiltinId(maybe_builtin_index_)) {
+ // The self-reference loaded through Codevalue() may also be a builtin
+ // and thus viable for a fast load.
+ LoadRootRelative(destination,
+ RootRegisterOffsetForBuiltinIndex(maybe_builtin_index_));
+ } else {
+ CHECK(isolate()->IsGeneratingEmbeddedBuiltins());
+ // Ensure the given object is in the builtins constants table and fetch its
+ // index.
+ BuiltinsConstantsTableBuilder* builder =
+ isolate()->builtins_constants_table_builder();
+ uint32_t index = builder->AddObject(object);
+
+ // Slow load from the constants table.
+ LoadFromConstantsTable(destination, index);
+ }
+}
+
+void TurboAssemblerBase::IndirectLoadExternalReference(
+ Register destination, ExternalReference reference) {
+ CHECK(root_array_available_);
+
+ if (IsAddressableThroughRootRegister(isolate(), reference)) {
+ // Some external references can be efficiently loaded as an offset from
+ // kRootRegister.
+ intptr_t offset =
+ RootRegisterOffsetForExternalReference(isolate(), reference);
+ LoadRootRegisterOffset(destination, offset);
+ } else {
+ // Otherwise, do a memory load from the external reference table.
+ LoadRootRelative(
+ destination,
+ RootRegisterOffsetForExternalReferenceTableEntry(isolate(), reference));
+ }
+}
+
+// static
+int32_t TurboAssemblerBase::RootRegisterOffsetForRootIndex(
+ RootIndex root_index) {
+ return IsolateData::root_slot_offset(root_index);
+}
+
+// static
+int32_t TurboAssemblerBase::RootRegisterOffsetForBuiltinIndex(
+ int builtin_index) {
+ return IsolateData::builtin_slot_offset(builtin_index);
+}
+
+// static
+intptr_t TurboAssemblerBase::RootRegisterOffsetForExternalReference(
+ Isolate* isolate, const ExternalReference& reference) {
+ return static_cast<intptr_t>(reference.address() - isolate->isolate_root());
+}
+
+// static
+int32_t TurboAssemblerBase::RootRegisterOffsetForExternalReferenceTableEntry(
+ Isolate* isolate, const ExternalReference& reference) {
+ // Encode as an index into the external reference table stored on the
+ // isolate.
+ ExternalReferenceEncoder encoder(isolate);
+ ExternalReferenceEncoder::Value v = encoder.Encode(reference.address());
+ CHECK(!v.is_from_api());
+
+ return IsolateData::external_reference_table_offset() +
+ ExternalReferenceTable::OffsetOfEntry(v.index());
+}
+
+// static
+bool TurboAssemblerBase::IsAddressableThroughRootRegister(
+ Isolate* isolate, const ExternalReference& reference) {
+ Address address = reference.address();
+ return isolate->root_register_addressable_region().contains(address);
+}
+
+void TurboAssemblerBase::RecordCommentForOffHeapTrampoline(int builtin_index) {
+ if (!FLAG_code_comments) return;
+ std::ostringstream str;
+ str << "-- Inlined Trampoline to " << Builtins::name(builtin_index) << " --";
+ RecordComment(str.str().c_str());
+}
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/turbo-assembler.h b/src/codegen/turbo-assembler.h
new file mode 100644
index 0000000..26cf12d
--- /dev/null
+++ b/src/codegen/turbo-assembler.h
@@ -0,0 +1,179 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_TURBO_ASSEMBLER_H_
+#define V8_CODEGEN_TURBO_ASSEMBLER_H_
+
+#include <memory>
+
+#include "src/base/template-utils.h"
+#include "src/builtins/builtins.h"
+#include "src/codegen/assembler-arch.h"
+#include "src/roots/roots.h"
+
+namespace v8 {
+namespace internal {
+
+// Common base class for platform-specific TurboAssemblers containing
+// platform-independent bits.
+class V8_EXPORT_PRIVATE TurboAssemblerBase : public Assembler {
+ public:
+ // Constructors are declared public to inherit them in derived classes
+ // with `using` directive.
+ TurboAssemblerBase(Isolate* isolate, CodeObjectRequired create_code_object,
+ std::unique_ptr<AssemblerBuffer> buffer = {})
+ : TurboAssemblerBase(isolate, AssemblerOptions::Default(isolate),
+ create_code_object, std::move(buffer)) {}
+
+ TurboAssemblerBase(Isolate* isolate, const AssemblerOptions& options,
+ CodeObjectRequired create_code_object,
+ std::unique_ptr<AssemblerBuffer> buffer = {});
+
+ Isolate* isolate() const {
+ return isolate_;
+ }
+
+ Handle<HeapObject> CodeObject() const {
+ DCHECK(!code_object_.is_null());
+ return code_object_;
+ }
+
+ bool root_array_available() const { return root_array_available_; }
+ void set_root_array_available(bool v) { root_array_available_ = v; }
+
+ bool trap_on_abort() const { return trap_on_abort_; }
+
+ bool should_abort_hard() const { return hard_abort_; }
+ void set_abort_hard(bool v) { hard_abort_ = v; }
+
+ void set_builtin_index(int i) { maybe_builtin_index_ = i; }
+
+ void set_has_frame(bool v) { has_frame_ = v; }
+ bool has_frame() const { return has_frame_; }
+
+ virtual void Jump(const ExternalReference& reference) = 0;
+
+ // Calls the builtin given by the Smi in |builtin|. If builtins are embedded,
+ // the trampoline Code object on the heap is not used.
+ virtual void CallBuiltinByIndex(Register builtin_index) = 0;
+
+ // Calls/jumps to the given Code object. If builtins are embedded, the
+ // trampoline Code object on the heap is not used.
+ virtual void CallCodeObject(Register code_object) = 0;
+ virtual void JumpCodeObject(Register code_object) = 0;
+
+ // Loads the given Code object's entry point into the destination register.
+ virtual void LoadCodeObjectEntry(Register destination,
+ Register code_object) = 0;
+
+ // Loads the given constant or external reference without embedding its direct
+ // pointer. The produced code is isolate-independent.
+ void IndirectLoadConstant(Register destination, Handle<HeapObject> object);
+ void IndirectLoadExternalReference(Register destination,
+ ExternalReference reference);
+
+ virtual void LoadFromConstantsTable(Register destination,
+ int constant_index) = 0;
+
+ // Corresponds to: destination = kRootRegister + offset.
+ virtual void LoadRootRegisterOffset(Register destination,
+ intptr_t offset) = 0;
+
+ // Corresponds to: destination = [kRootRegister + offset].
+ virtual void LoadRootRelative(Register destination, int32_t offset) = 0;
+
+ virtual void LoadRoot(Register destination, RootIndex index) = 0;
+
+ virtual void Trap() = 0;
+ virtual void DebugBreak() = 0;
+
+ static int32_t RootRegisterOffsetForRootIndex(RootIndex root_index);
+ static int32_t RootRegisterOffsetForBuiltinIndex(int builtin_index);
+
+ // Returns the root-relative offset to reference.address().
+ static intptr_t RootRegisterOffsetForExternalReference(
+ Isolate* isolate, const ExternalReference& reference);
+
+ // Returns the root-relative offset to the external reference table entry,
+ // which itself contains reference.address().
+ static int32_t RootRegisterOffsetForExternalReferenceTableEntry(
+ Isolate* isolate, const ExternalReference& reference);
+
+ // An address is addressable through kRootRegister if it is located within
+ // isolate->root_register_addressable_region().
+ static bool IsAddressableThroughRootRegister(
+ Isolate* isolate, const ExternalReference& reference);
+
+#ifdef V8_TARGET_OS_WIN
+ // Minimum page size. We must touch memory once per page when expanding the
+ // stack, to avoid access violations.
+ static constexpr int kStackPageSize = 4 * KB;
+#endif
+
+ protected:
+ void RecordCommentForOffHeapTrampoline(int builtin_index);
+
+ Isolate* const isolate_ = nullptr;
+
+ // This handle will be patched with the code object on installation.
+ Handle<HeapObject> code_object_;
+
+ // Whether kRootRegister has been initialized.
+ bool root_array_available_ = true;
+
+ // Immediately trap instead of calling {Abort} when debug code fails.
+ bool trap_on_abort_ = FLAG_trap_on_abort;
+
+ // Emit a C call to abort instead of a runtime call.
+ bool hard_abort_ = false;
+
+ // May be set while generating builtins.
+ int maybe_builtin_index_ = Builtins::kNoBuiltinId;
+
+ bool has_frame_ = false;
+
+ DISALLOW_IMPLICIT_CONSTRUCTORS(TurboAssemblerBase);
+};
+
+// Avoids emitting calls to the {Builtins::kAbort} builtin when emitting debug
+// code during the lifetime of this scope object. For disabling debug code
+// entirely use the {DontEmitDebugCodeScope} instead.
+class HardAbortScope {
+ public:
+ explicit HardAbortScope(TurboAssemblerBase* assembler)
+ : assembler_(assembler), old_value_(assembler->should_abort_hard()) {
+ assembler_->set_abort_hard(true);
+ }
+ ~HardAbortScope() { assembler_->set_abort_hard(old_value_); }
+
+ private:
+ TurboAssemblerBase* assembler_;
+ bool old_value_;
+};
+
+#ifdef DEBUG
+struct CountIfValidRegisterFunctor {
+ template <typename RegType>
+ constexpr int operator()(int count, RegType reg) const {
+ return count + (reg.is_valid() ? 1 : 0);
+ }
+};
+
+template <typename RegType, typename... RegTypes,
+ // All arguments must be either Register or DoubleRegister.
+ typename = typename std::enable_if<
+ base::is_same<Register, RegType, RegTypes...>::value ||
+ base::is_same<DoubleRegister, RegType, RegTypes...>::value>::type>
+inline bool AreAliased(RegType first_reg, RegTypes... regs) {
+ int num_different_regs = NumRegs(RegType::ListOf(first_reg, regs...));
+ int num_given_regs =
+ base::fold(CountIfValidRegisterFunctor{}, 0, first_reg, regs...);
+ return num_different_regs < num_given_regs;
+}
+#endif
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_TURBO_ASSEMBLER_H_
diff --git a/src/codegen/unoptimized-compilation-info.cc b/src/codegen/unoptimized-compilation-info.cc
new file mode 100644
index 0000000..08cd818
--- /dev/null
+++ b/src/codegen/unoptimized-compilation-info.cc
@@ -0,0 +1,60 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/unoptimized-compilation-info.h"
+
+#include "src/ast/ast.h"
+#include "src/ast/scopes.h"
+#include "src/codegen/source-position.h"
+#include "src/debug/debug.h"
+#include "src/execution/isolate.h"
+#include "src/objects/objects-inl.h"
+#include "src/parsing/parse-info.h"
+
+namespace v8 {
+namespace internal {
+
+UnoptimizedCompilationInfo::UnoptimizedCompilationInfo(Zone* zone,
+ ParseInfo* parse_info,
+ FunctionLiteral* literal)
+ : flags_(parse_info->flags()), feedback_vector_spec_(zone) {
+ // NOTE: The parse_info passed here represents the global information gathered
+ // during parsing, but does not represent specific details of the actual
+ // function literal being compiled for this OptimizedCompilationInfo. As such,
+ // parse_info->literal() might be different from literal, and only global
+ // details of the script being parsed are relevant to this
+ // OptimizedCompilationInfo.
+ DCHECK_NOT_NULL(literal);
+ literal_ = literal;
+ source_range_map_ = parse_info->source_range_map();
+}
+
+DeclarationScope* UnoptimizedCompilationInfo::scope() const {
+ DCHECK_NOT_NULL(literal_);
+ return literal_->scope();
+}
+
+int UnoptimizedCompilationInfo::num_parameters() const {
+ return scope()->num_parameters();
+}
+
+int UnoptimizedCompilationInfo::num_parameters_including_this() const {
+ return scope()->num_parameters() + 1;
+}
+
+SourcePositionTableBuilder::RecordingMode
+UnoptimizedCompilationInfo::SourcePositionRecordingMode() const {
+ if (flags().collect_source_positions()) {
+ return SourcePositionTableBuilder::RECORD_SOURCE_POSITIONS;
+ }
+
+ // Always collect source positions for functions that cannot be lazily
+ // compiled, e.g. class member initializer functions.
+ return !literal_->AllowsLazyCompilation()
+ ? SourcePositionTableBuilder::RECORD_SOURCE_POSITIONS
+ : SourcePositionTableBuilder::LAZY_SOURCE_POSITIONS;
+}
+
+} // namespace internal
+} // namespace v8
diff --git a/src/codegen/unoptimized-compilation-info.h b/src/codegen/unoptimized-compilation-info.h
new file mode 100644
index 0000000..3cdb941
--- /dev/null
+++ b/src/codegen/unoptimized-compilation-info.h
@@ -0,0 +1,112 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_UNOPTIMIZED_COMPILATION_INFO_H_
+#define V8_CODEGEN_UNOPTIMIZED_COMPILATION_INFO_H_
+
+#include <memory>
+
+#include "src/codegen/source-position-table.h"
+#include "src/common/globals.h"
+#include "src/handles/handles.h"
+#include "src/objects/feedback-vector.h"
+#include "src/objects/objects.h"
+#include "src/parsing/parse-info.h"
+#include "src/utils/utils.h"
+
+namespace v8 {
+namespace internal {
+
+class AsmWasmData;
+class CoverageInfo;
+class DeclarationScope;
+class FunctionLiteral;
+class Isolate;
+class ParseInfo;
+class SourceRangeMap;
+class Zone;
+
+// UnoptimizedCompilationInfo encapsulates the information needed to compile
+// unoptimized code for a given function, and the results of the compilation.
+class V8_EXPORT_PRIVATE UnoptimizedCompilationInfo final {
+ public:
+ UnoptimizedCompilationInfo(Zone* zone, ParseInfo* parse_info,
+ FunctionLiteral* literal);
+
+ const UnoptimizedCompileFlags& flags() const { return flags_; }
+
+ // Accessors for the input data of the function being compiled.
+
+ FunctionLiteral* literal() const { return literal_; }
+ void set_literal(FunctionLiteral* literal) {
+ DCHECK_NOT_NULL(literal);
+ literal_ = literal;
+ }
+ void ClearLiteral() { literal_ = nullptr; }
+
+ DeclarationScope* scope() const;
+
+ int num_parameters() const;
+ int num_parameters_including_this() const;
+
+ // Accessors for optional compilation features.
+
+ SourcePositionTableBuilder::RecordingMode SourcePositionRecordingMode() const;
+
+ bool has_source_range_map() const { return source_range_map_ != nullptr; }
+ SourceRangeMap* source_range_map() const { return source_range_map_; }
+ void set_source_range_map(SourceRangeMap* source_range_map) {
+ source_range_map_ = source_range_map;
+ }
+
+ bool has_coverage_info() const { return !coverage_info_.is_null(); }
+ Handle<CoverageInfo> coverage_info() const { return coverage_info_; }
+ void set_coverage_info(Handle<CoverageInfo> coverage_info) {
+ coverage_info_ = coverage_info;
+ }
+
+ // Accessors for the output of compilation.
+
+ bool has_bytecode_array() const { return !bytecode_array_.is_null(); }
+ Handle<BytecodeArray> bytecode_array() const { return bytecode_array_; }
+ void SetBytecodeArray(Handle<BytecodeArray> bytecode_array) {
+ bytecode_array_ = bytecode_array;
+ }
+
+ bool has_asm_wasm_data() const { return !asm_wasm_data_.is_null(); }
+ Handle<AsmWasmData> asm_wasm_data() const { return asm_wasm_data_; }
+ void SetAsmWasmData(Handle<AsmWasmData> asm_wasm_data) {
+ asm_wasm_data_ = asm_wasm_data;
+ }
+
+ FeedbackVectorSpec* feedback_vector_spec() { return &feedback_vector_spec_; }
+
+ private:
+ // Compilation flags.
+ const UnoptimizedCompileFlags flags_;
+
+ // The root AST node of the function literal being compiled.
+ FunctionLiteral* literal_;
+
+ // Used when block coverage is enabled.
+ SourceRangeMap* source_range_map_;
+
+ // Encapsulates coverage information gathered by the bytecode generator.
+ // Needs to be stored on the shared function info once compilation completes.
+ Handle<CoverageInfo> coverage_info_;
+
+ // Holds the bytecode array generated by the interpreter.
+ Handle<BytecodeArray> bytecode_array_;
+
+ // Holds the asm_wasm data struct generated by the asmjs compiler.
+ Handle<AsmWasmData> asm_wasm_data_;
+
+ // Holds the feedback vector spec generated during compilation
+ FeedbackVectorSpec feedback_vector_spec_;
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_UNOPTIMIZED_COMPILATION_INFO_H_
diff --git a/src/codegen/x64/assembler-x64-inl.h b/src/codegen/x64/assembler-x64-inl.h
new file mode 100644
index 0000000..fedff2b
--- /dev/null
+++ b/src/codegen/x64/assembler-x64-inl.h
@@ -0,0 +1,427 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_X64_ASSEMBLER_X64_INL_H_
+#define V8_CODEGEN_X64_ASSEMBLER_X64_INL_H_
+
+#include "src/codegen/x64/assembler-x64.h"
+
+#include "src/base/cpu.h"
+#include "src/base/memory.h"
+#include "src/debug/debug.h"
+#include "src/objects/objects-inl.h"
+
+namespace v8 {
+namespace internal {
+
+bool CpuFeatures::SupportsOptimizer() { return true; }
+
+bool CpuFeatures::SupportsWasmSimd128() { return IsSupported(SSE4_1); }
+
+// -----------------------------------------------------------------------------
+// Implementation of Assembler
+
+void Assembler::emitl(uint32_t x) {
+ WriteUnalignedValue(reinterpret_cast<Address>(pc_), x);
+ pc_ += sizeof(uint32_t);
+}
+
+void Assembler::emitq(uint64_t x) {
+ WriteUnalignedValue(reinterpret_cast<Address>(pc_), x);
+ pc_ += sizeof(uint64_t);
+}
+
+void Assembler::emitw(uint16_t x) {
+ WriteUnalignedValue(reinterpret_cast<Address>(pc_), x);
+ pc_ += sizeof(uint16_t);
+}
+
+void Assembler::emit_runtime_entry(Address entry, RelocInfo::Mode rmode) {
+ DCHECK(RelocInfo::IsRuntimeEntry(rmode));
+ RecordRelocInfo(rmode);
+ emitl(static_cast<uint32_t>(entry - options().code_range_start));
+}
+
+void Assembler::emit(Immediate x) {
+ if (!RelocInfo::IsNone(x.rmode_)) {
+ RecordRelocInfo(x.rmode_);
+ }
+ emitl(x.value_);
+}
+
+void Assembler::emit(Immediate64 x) {
+ if (!RelocInfo::IsNone(x.rmode_)) {
+ RecordRelocInfo(x.rmode_);
+ }
+ emitq(static_cast<uint64_t>(x.value_));
+}
+
+void Assembler::emit_rex_64(Register reg, Register rm_reg) {
+ emit(0x48 | reg.high_bit() << 2 | rm_reg.high_bit());
+}
+
+void Assembler::emit_rex_64(XMMRegister reg, Register rm_reg) {
+ emit(0x48 | (reg.code() & 0x8) >> 1 | rm_reg.code() >> 3);
+}
+
+void Assembler::emit_rex_64(Register reg, XMMRegister rm_reg) {
+ emit(0x48 | (reg.code() & 0x8) >> 1 | rm_reg.code() >> 3);
+}
+
+void Assembler::emit_rex_64(XMMRegister reg, XMMRegister rm_reg) {
+ emit(0x48 | (reg.code() & 0x8) >> 1 | rm_reg.code() >> 3);
+}
+
+void Assembler::emit_rex_64(Register reg, Operand op) {
+ emit(0x48 | reg.high_bit() << 2 | op.data().rex);
+}
+
+void Assembler::emit_rex_64(XMMRegister reg, Operand op) {
+ emit(0x48 | (reg.code() & 0x8) >> 1 | op.data().rex);
+}
+
+void Assembler::emit_rex_64(Register rm_reg) {
+ DCHECK_EQ(rm_reg.code() & 0xf, rm_reg.code());
+ emit(0x48 | rm_reg.high_bit());
+}
+
+void Assembler::emit_rex_64(Operand op) { emit(0x48 | op.data().rex); }
+
+void Assembler::emit_rex_32(Register reg, Register rm_reg) {
+ emit(0x40 | reg.high_bit() << 2 | rm_reg.high_bit());
+}
+
+void Assembler::emit_rex_32(Register reg, Operand op) {
+ emit(0x40 | reg.high_bit() << 2 | op.data().rex);
+}
+
+void Assembler::emit_rex_32(Register rm_reg) { emit(0x40 | rm_reg.high_bit()); }
+
+void Assembler::emit_rex_32(Operand op) { emit(0x40 | op.data().rex); }
+
+void Assembler::emit_optional_rex_32(Register reg, Register rm_reg) {
+ byte rex_bits = reg.high_bit() << 2 | rm_reg.high_bit();
+ if (rex_bits != 0) emit(0x40 | rex_bits);
+}
+
+void Assembler::emit_optional_rex_32(Register reg, Operand op) {
+ byte rex_bits = reg.high_bit() << 2 | op.data().rex;
+ if (rex_bits != 0) emit(0x40 | rex_bits);
+}
+
+void Assembler::emit_optional_rex_32(XMMRegister reg, Operand op) {
+ byte rex_bits = (reg.code() & 0x8) >> 1 | op.data().rex;
+ if (rex_bits != 0) emit(0x40 | rex_bits);
+}
+
+void Assembler::emit_optional_rex_32(XMMRegister reg, XMMRegister base) {
+ byte rex_bits = (reg.code() & 0x8) >> 1 | (base.code() & 0x8) >> 3;
+ if (rex_bits != 0) emit(0x40 | rex_bits);
+}
+
+void Assembler::emit_optional_rex_32(XMMRegister reg, Register base) {
+ byte rex_bits = (reg.code() & 0x8) >> 1 | (base.code() & 0x8) >> 3;
+ if (rex_bits != 0) emit(0x40 | rex_bits);
+}
+
+void Assembler::emit_optional_rex_32(Register reg, XMMRegister base) {
+ byte rex_bits = (reg.code() & 0x8) >> 1 | (base.code() & 0x8) >> 3;
+ if (rex_bits != 0) emit(0x40 | rex_bits);
+}
+
+void Assembler::emit_optional_rex_32(Register rm_reg) {
+ if (rm_reg.high_bit()) emit(0x41);
+}
+
+void Assembler::emit_optional_rex_32(XMMRegister rm_reg) {
+ if (rm_reg.high_bit()) emit(0x41);
+}
+
+void Assembler::emit_optional_rex_32(Operand op) {
+ if (op.data().rex != 0) emit(0x40 | op.data().rex);
+}
+
+void Assembler::emit_optional_rex_8(Register reg) {
+ if (!reg.is_byte_register()) {
+ // Register is not one of al, bl, cl, dl. Its encoding needs REX.
+ emit_rex_32(reg);
+ }
+}
+
+void Assembler::emit_optional_rex_8(Register reg, Operand op) {
+ if (!reg.is_byte_register()) {
+ // Register is not one of al, bl, cl, dl. Its encoding needs REX.
+ emit_rex_32(reg, op);
+ } else {
+ emit_optional_rex_32(reg, op);
+ }
+}
+
+// byte 1 of 3-byte VEX
+void Assembler::emit_vex3_byte1(XMMRegister reg, XMMRegister rm,
+ LeadingOpcode m) {
+ byte rxb = static_cast<byte>(~((reg.high_bit() << 2) | rm.high_bit())) << 5;
+ emit(rxb | m);
+}
+
+// byte 1 of 3-byte VEX
+void Assembler::emit_vex3_byte1(XMMRegister reg, Operand rm, LeadingOpcode m) {
+ byte rxb = static_cast<byte>(~((reg.high_bit() << 2) | rm.data().rex)) << 5;
+ emit(rxb | m);
+}
+
+// byte 1 of 2-byte VEX
+void Assembler::emit_vex2_byte1(XMMRegister reg, XMMRegister v, VectorLength l,
+ SIMDPrefix pp) {
+ byte rv = static_cast<byte>(~((reg.high_bit() << 4) | v.code())) << 3;
+ emit(rv | l | pp);
+}
+
+// byte 2 of 3-byte VEX
+void Assembler::emit_vex3_byte2(VexW w, XMMRegister v, VectorLength l,
+ SIMDPrefix pp) {
+ emit(w | ((~v.code() & 0xf) << 3) | l | pp);
+}
+
+void Assembler::emit_vex_prefix(XMMRegister reg, XMMRegister vreg,
+ XMMRegister rm, VectorLength l, SIMDPrefix pp,
+ LeadingOpcode mm, VexW w) {
+ if (rm.high_bit() || mm != k0F || w != kW0) {
+ emit_vex3_byte0();
+ emit_vex3_byte1(reg, rm, mm);
+ emit_vex3_byte2(w, vreg, l, pp);
+ } else {
+ emit_vex2_byte0();
+ emit_vex2_byte1(reg, vreg, l, pp);
+ }
+}
+
+void Assembler::emit_vex_prefix(Register reg, Register vreg, Register rm,
+ VectorLength l, SIMDPrefix pp, LeadingOpcode mm,
+ VexW w) {
+ XMMRegister ireg = XMMRegister::from_code(reg.code());
+ XMMRegister ivreg = XMMRegister::from_code(vreg.code());
+ XMMRegister irm = XMMRegister::from_code(rm.code());
+ emit_vex_prefix(ireg, ivreg, irm, l, pp, mm, w);
+}
+
+void Assembler::emit_vex_prefix(XMMRegister reg, XMMRegister vreg, Operand rm,
+ VectorLength l, SIMDPrefix pp, LeadingOpcode mm,
+ VexW w) {
+ if (rm.data().rex || mm != k0F || w != kW0) {
+ emit_vex3_byte0();
+ emit_vex3_byte1(reg, rm, mm);
+ emit_vex3_byte2(w, vreg, l, pp);
+ } else {
+ emit_vex2_byte0();
+ emit_vex2_byte1(reg, vreg, l, pp);
+ }
+}
+
+void Assembler::emit_vex_prefix(Register reg, Register vreg, Operand rm,
+ VectorLength l, SIMDPrefix pp, LeadingOpcode mm,
+ VexW w) {
+ XMMRegister ireg = XMMRegister::from_code(reg.code());
+ XMMRegister ivreg = XMMRegister::from_code(vreg.code());
+ emit_vex_prefix(ireg, ivreg, rm, l, pp, mm, w);
+}
+
+Address Assembler::target_address_at(Address pc, Address constant_pool) {
+ return ReadUnalignedValue<int32_t>(pc) + pc + 4;
+}
+
+void Assembler::set_target_address_at(Address pc, Address constant_pool,
+ Address target,
+ ICacheFlushMode icache_flush_mode) {
+ DCHECK(is_int32(target - pc - 4));
+ WriteUnalignedValue(pc, static_cast<int32_t>(target - pc - 4));
+ if (icache_flush_mode != SKIP_ICACHE_FLUSH) {
+ FlushInstructionCache(pc, sizeof(int32_t));
+ }
+}
+
+void Assembler::deserialization_set_target_internal_reference_at(
+ Address pc, Address target, RelocInfo::Mode mode) {
+ WriteUnalignedValue(pc, target);
+}
+
+void Assembler::deserialization_set_special_target_at(
+ Address instruction_payload, Code code, Address target) {
+ set_target_address_at(instruction_payload,
+ !code.is_null() ? code.constant_pool() : kNullAddress,
+ target);
+}
+
+int Assembler::deserialization_special_target_size(
+ Address instruction_payload) {
+ return kSpecialTargetSize;
+}
+
+Handle<Code> Assembler::code_target_object_handle_at(Address pc) {
+ return GetCodeTarget(ReadUnalignedValue<int32_t>(pc));
+}
+
+Handle<HeapObject> Assembler::compressed_embedded_object_handle_at(Address pc) {
+ return GetEmbeddedObject(ReadUnalignedValue<uint32_t>(pc));
+}
+
+Address Assembler::runtime_entry_at(Address pc) {
+ return ReadUnalignedValue<int32_t>(pc) + options().code_range_start;
+}
+
+// -----------------------------------------------------------------------------
+// Implementation of RelocInfo
+
+// The modes possibly affected by apply must be in kApplyMask.
+void RelocInfo::apply(intptr_t delta) {
+ if (IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_)) {
+ WriteUnalignedValue(
+ pc_, ReadUnalignedValue<int32_t>(pc_) - static_cast<int32_t>(delta));
+ } else if (IsInternalReference(rmode_)) {
+ // Absolute code pointer inside code object moves with the code object.
+ WriteUnalignedValue(pc_, ReadUnalignedValue<Address>(pc_) + delta);
+ }
+}
+
+Address RelocInfo::target_address() {
+ DCHECK(IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_) || IsWasmCall(rmode_));
+ return Assembler::target_address_at(pc_, constant_pool_);
+}
+
+Address RelocInfo::target_address_address() {
+ DCHECK(IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_) || IsWasmCall(rmode_) ||
+ IsWasmStubCall(rmode_) || IsFullEmbeddedObject(rmode_) ||
+ IsCompressedEmbeddedObject(rmode_) || IsExternalReference(rmode_) ||
+ IsOffHeapTarget(rmode_));
+ return pc_;
+}
+
+Address RelocInfo::constant_pool_entry_address() { UNREACHABLE(); }
+
+int RelocInfo::target_address_size() {
+ if (IsCodedSpecially()) {
+ return Assembler::kSpecialTargetSize;
+ } else {
+ return IsCompressedEmbeddedObject(rmode_) ? kTaggedSize
+ : kSystemPointerSize;
+ }
+}
+
+HeapObject RelocInfo::target_object() {
+ DCHECK(IsCodeTarget(rmode_) || IsEmbeddedObjectMode(rmode_));
+ if (IsCompressedEmbeddedObject(rmode_)) {
+ CHECK(!host_.is_null());
+ Object o = static_cast<Object>(DecompressTaggedPointer(
+ host_.ptr(), ReadUnalignedValue<Tagged_t>(pc_)));
+ return HeapObject::cast(o);
+ }
+ return HeapObject::cast(Object(ReadUnalignedValue<Address>(pc_)));
+}
+
+HeapObject RelocInfo::target_object_no_host(Isolate* isolate) {
+ DCHECK(IsCodeTarget(rmode_) || IsEmbeddedObjectMode(rmode_));
+ if (IsCompressedEmbeddedObject(rmode_)) {
+ Tagged_t compressed = ReadUnalignedValue<Tagged_t>(pc_);
+ DCHECK(!HAS_SMI_TAG(compressed));
+ Object obj(DecompressTaggedPointer(isolate, compressed));
+ return HeapObject::cast(obj);
+ }
+ return HeapObject::cast(Object(ReadUnalignedValue<Address>(pc_)));
+}
+
+Handle<HeapObject> RelocInfo::target_object_handle(Assembler* origin) {
+ DCHECK(IsCodeTarget(rmode_) || IsEmbeddedObjectMode(rmode_));
+ if (IsCodeTarget(rmode_)) {
+ return origin->code_target_object_handle_at(pc_);
+ } else {
+ if (IsCompressedEmbeddedObject(rmode_)) {
+ return origin->compressed_embedded_object_handle_at(pc_);
+ }
+ return Handle<HeapObject>::cast(ReadUnalignedValue<Handle<Object>>(pc_));
+ }
+}
+
+Address RelocInfo::target_external_reference() {
+ DCHECK(rmode_ == RelocInfo::EXTERNAL_REFERENCE);
+ return ReadUnalignedValue<Address>(pc_);
+}
+
+void RelocInfo::set_target_external_reference(
+ Address target, ICacheFlushMode icache_flush_mode) {
+ DCHECK(rmode_ == RelocInfo::EXTERNAL_REFERENCE);
+ WriteUnalignedValue(pc_, target);
+ if (icache_flush_mode != SKIP_ICACHE_FLUSH) {
+ FlushInstructionCache(pc_, sizeof(Address));
+ }
+}
+
+Address RelocInfo::target_internal_reference() {
+ DCHECK(rmode_ == INTERNAL_REFERENCE);
+ return ReadUnalignedValue<Address>(pc_);
+}
+
+Address RelocInfo::target_internal_reference_address() {
+ DCHECK(rmode_ == INTERNAL_REFERENCE);
+ return pc_;
+}
+
+void RelocInfo::set_target_object(Heap* heap, HeapObject target,
+ WriteBarrierMode write_barrier_mode,
+ ICacheFlushMode icache_flush_mode) {
+ DCHECK(IsCodeTarget(rmode_) || IsEmbeddedObjectMode(rmode_));
+ if (IsCompressedEmbeddedObject(rmode_)) {
+ DCHECK(COMPRESS_POINTERS_BOOL);
+ Tagged_t tagged = CompressTagged(target.ptr());
+ WriteUnalignedValue(pc_, tagged);
+ } else {
+ WriteUnalignedValue(pc_, target.ptr());
+ }
+ if (icache_flush_mode != SKIP_ICACHE_FLUSH) {
+ FlushInstructionCache(pc_, sizeof(Address));
+ }
+ if (write_barrier_mode == UPDATE_WRITE_BARRIER && !host().is_null() &&
+ !FLAG_disable_write_barriers) {
+ WriteBarrierForCode(host(), this, target);
+ }
+}
+
+Address RelocInfo::target_runtime_entry(Assembler* origin) {
+ DCHECK(IsRuntimeEntry(rmode_));
+ return origin->runtime_entry_at(pc_);
+}
+
+void RelocInfo::set_target_runtime_entry(Address target,
+ WriteBarrierMode write_barrier_mode,
+ ICacheFlushMode icache_flush_mode) {
+ DCHECK(IsRuntimeEntry(rmode_));
+ if (target_address() != target) {
+ set_target_address(target, write_barrier_mode, icache_flush_mode);
+ }
+}
+
+Address RelocInfo::target_off_heap_target() {
+ DCHECK(IsOffHeapTarget(rmode_));
+ return ReadUnalignedValue<Address>(pc_);
+}
+
+void RelocInfo::WipeOut() {
+ if (IsFullEmbeddedObject(rmode_) || IsExternalReference(rmode_) ||
+ IsInternalReference(rmode_) || IsOffHeapTarget(rmode_)) {
+ WriteUnalignedValue(pc_, kNullAddress);
+ } else if (IsCompressedEmbeddedObject(rmode_)) {
+ Address smi_address = Smi::FromInt(0).ptr();
+ WriteUnalignedValue(pc_, CompressTagged(smi_address));
+ } else if (IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_)) {
+ // Effectively write zero into the relocation.
+ Assembler::set_target_address_at(pc_, constant_pool_,
+ pc_ + sizeof(int32_t));
+ } else {
+ UNREACHABLE();
+ }
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_X64_ASSEMBLER_X64_INL_H_
diff --git a/src/codegen/x64/assembler-x64.cc b/src/codegen/x64/assembler-x64.cc
new file mode 100644
index 0000000..5327745
--- /dev/null
+++ b/src/codegen/x64/assembler-x64.cc
@@ -0,0 +1,4259 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/x64/assembler-x64.h"
+
+#include <cstring>
+
+#if V8_TARGET_ARCH_X64
+
+#if V8_LIBC_MSVCRT
+#include <intrin.h> // _xgetbv()
+#endif
+#if V8_OS_MACOSX
+#include <sys/sysctl.h>
+#endif
+
+#include "src/base/bits.h"
+#include "src/base/cpu.h"
+#include "src/codegen/assembler-inl.h"
+#include "src/codegen/macro-assembler.h"
+#include "src/codegen/string-constants.h"
+#include "src/deoptimizer/deoptimizer.h"
+#include "src/init/v8.h"
+
+namespace v8 {
+namespace internal {
+
+// -----------------------------------------------------------------------------
+// Implementation of CpuFeatures
+
+namespace {
+
+V8_INLINE uint64_t xgetbv(unsigned int xcr) {
+#if V8_LIBC_MSVCRT
+ return _xgetbv(xcr);
+#else
+ unsigned eax, edx;
+ // Check xgetbv; this uses a .byte sequence instead of the instruction
+ // directly because older assemblers do not include support for xgetbv and
+ // there is no easy way to conditionally compile based on the assembler
+ // used.
+ __asm__ volatile(".byte 0x0F, 0x01, 0xD0" : "=a"(eax), "=d"(edx) : "c"(xcr));
+ return static_cast<uint64_t>(eax) | (static_cast<uint64_t>(edx) << 32);
+#endif
+}
+
+bool OSHasAVXSupport() {
+#if V8_OS_MACOSX
+ // Mac OS X up to 10.9 has a bug where AVX transitions were indeed being
+ // caused by ISRs, so we detect that here and disable AVX in that case.
+ char buffer[128];
+ size_t buffer_size = arraysize(buffer);
+ int ctl_name[] = {CTL_KERN, KERN_OSRELEASE};
+ if (sysctl(ctl_name, 2, buffer, &buffer_size, nullptr, 0) != 0) {
+ FATAL("V8 failed to get kernel version");
+ }
+ // The buffer now contains a string of the form XX.YY.ZZ, where
+ // XX is the major kernel version component.
+ char* period_pos = strchr(buffer, '.');
+ DCHECK_NOT_NULL(period_pos);
+ *period_pos = '\0';
+ long kernel_version_major = strtol(buffer, nullptr, 10); // NOLINT
+ if (kernel_version_major <= 13) return false;
+#endif // V8_OS_MACOSX
+ // Check whether OS claims to support AVX.
+ uint64_t feature_mask = xgetbv(0); // XCR_XFEATURE_ENABLED_MASK
+ return (feature_mask & 0x6) == 0x6;
+}
+
+} // namespace
+
+void CpuFeatures::ProbeImpl(bool cross_compile) {
+ base::CPU cpu;
+ CHECK(cpu.has_sse2()); // SSE2 support is mandatory.
+ CHECK(cpu.has_cmov()); // CMOV support is mandatory.
+
+ // Only use statically determined features for cross compile (snapshot).
+ if (cross_compile) return;
+
+ if (cpu.has_sse42() && FLAG_enable_sse4_2) supported_ |= 1u << SSE4_2;
+ if (cpu.has_sse41() && FLAG_enable_sse4_1) {
+ supported_ |= 1u << SSE4_1;
+ supported_ |= 1u << SSSE3;
+ }
+ if (cpu.has_ssse3() && FLAG_enable_ssse3) supported_ |= 1u << SSSE3;
+ if (cpu.has_sse3() && FLAG_enable_sse3) supported_ |= 1u << SSE3;
+ // SAHF is not generally available in long mode.
+ if (cpu.has_sahf() && FLAG_enable_sahf) supported_ |= 1u << SAHF;
+ if (cpu.has_avx() && FLAG_enable_avx && cpu.has_osxsave() &&
+ OSHasAVXSupport()) {
+ supported_ |= 1u << AVX;
+ }
+ if (cpu.has_fma3() && FLAG_enable_fma3 && cpu.has_osxsave() &&
+ OSHasAVXSupport()) {
+ supported_ |= 1u << FMA3;
+ }
+ if (cpu.has_bmi1() && FLAG_enable_bmi1) supported_ |= 1u << BMI1;
+ if (cpu.has_bmi2() && FLAG_enable_bmi2) supported_ |= 1u << BMI2;
+ if (cpu.has_lzcnt() && FLAG_enable_lzcnt) supported_ |= 1u << LZCNT;
+ if (cpu.has_popcnt() && FLAG_enable_popcnt) supported_ |= 1u << POPCNT;
+ if (strcmp(FLAG_mcpu, "auto") == 0) {
+ if (cpu.is_atom()) supported_ |= 1u << ATOM;
+ } else if (strcmp(FLAG_mcpu, "atom") == 0) {
+ supported_ |= 1u << ATOM;
+ }
+}
+
+void CpuFeatures::PrintTarget() {}
+void CpuFeatures::PrintFeatures() {
+ printf(
+ "SSE3=%d SSSE3=%d SSE4_1=%d SSE4_2=%d SAHF=%d AVX=%d FMA3=%d BMI1=%d "
+ "BMI2=%d "
+ "LZCNT=%d "
+ "POPCNT=%d ATOM=%d\n",
+ CpuFeatures::IsSupported(SSE3), CpuFeatures::IsSupported(SSSE3),
+ CpuFeatures::IsSupported(SSE4_1), CpuFeatures::IsSupported(SSE4_2),
+ CpuFeatures::IsSupported(SAHF), CpuFeatures::IsSupported(AVX),
+ CpuFeatures::IsSupported(FMA3), CpuFeatures::IsSupported(BMI1),
+ CpuFeatures::IsSupported(BMI2), CpuFeatures::IsSupported(LZCNT),
+ CpuFeatures::IsSupported(POPCNT), CpuFeatures::IsSupported(ATOM));
+}
+
+// -----------------------------------------------------------------------------
+// Implementation of RelocInfo
+
+uint32_t RelocInfo::wasm_call_tag() const {
+ DCHECK(rmode_ == WASM_CALL || rmode_ == WASM_STUB_CALL);
+ return ReadUnalignedValue<uint32_t>(pc_);
+}
+
+// -----------------------------------------------------------------------------
+// Implementation of Operand
+
+Operand::Operand(Operand operand, int32_t offset) {
+ DCHECK_GE(operand.data().len, 1);
+ // Operand encodes REX ModR/M [SIB] [Disp].
+ byte modrm = operand.data().buf[0];
+ DCHECK_LT(modrm, 0xC0); // Disallow mode 3 (register target).
+ bool has_sib = ((modrm & 0x07) == 0x04);
+ byte mode = modrm & 0xC0;
+ int disp_offset = has_sib ? 2 : 1;
+ int base_reg = (has_sib ? operand.data().buf[1] : modrm) & 0x07;
+ // Mode 0 with rbp/r13 as ModR/M or SIB base register always has a 32-bit
+ // displacement.
+ bool is_baseless = (mode == 0) && (base_reg == 0x05); // No base or RIP base.
+ int32_t disp_value = 0;
+ if (mode == 0x80 || is_baseless) {
+ // Mode 2 or mode 0 with rbp/r13 as base: Word displacement.
+ disp_value = ReadUnalignedValue<int32_t>(
+ reinterpret_cast<Address>(&operand.data().buf[disp_offset]));
+ } else if (mode == 0x40) {
+ // Mode 1: Byte displacement.
+ disp_value = static_cast<signed char>(operand.data().buf[disp_offset]);
+ }
+
+ // Write new operand with same registers, but with modified displacement.
+ DCHECK(offset >= 0 ? disp_value + offset > disp_value
+ : disp_value + offset < disp_value); // No overflow.
+ disp_value += offset;
+ data_.rex = operand.data().rex;
+ if (!is_int8(disp_value) || is_baseless) {
+ // Need 32 bits of displacement, mode 2 or mode 1 with register rbp/r13.
+ data_.buf[0] = (modrm & 0x3F) | (is_baseless ? 0x00 : 0x80);
+ data_.len = disp_offset + 4;
+ WriteUnalignedValue(reinterpret_cast<Address>(&data_.buf[disp_offset]),
+ disp_value);
+ } else if (disp_value != 0 || (base_reg == 0x05)) {
+ // Need 8 bits of displacement.
+ data_.buf[0] = (modrm & 0x3F) | 0x40; // Mode 1.
+ data_.len = disp_offset + 1;
+ data_.buf[disp_offset] = static_cast<byte>(disp_value);
+ } else {
+ // Need no displacement.
+ data_.buf[0] = (modrm & 0x3F); // Mode 0.
+ data_.len = disp_offset;
+ }
+ if (has_sib) {
+ data_.buf[1] = operand.data().buf[1];
+ }
+}
+
+bool Operand::AddressUsesRegister(Register reg) const {
+ int code = reg.code();
+ DCHECK_NE(data_.buf[0] & 0xC0, 0xC0); // Always a memory operand.
+ // Start with only low three bits of base register. Initial decoding
+ // doesn't distinguish on the REX.B bit.
+ int base_code = data_.buf[0] & 0x07;
+ if (base_code == rsp.code()) {
+ // SIB byte present in buf_[1].
+ // Check the index register from the SIB byte + REX.X prefix.
+ int index_code = ((data_.buf[1] >> 3) & 0x07) | ((data_.rex & 0x02) << 2);
+ // Index code (including REX.X) of 0x04 (rsp) means no index register.
+ if (index_code != rsp.code() && index_code == code) return true;
+ // Add REX.B to get the full base register code.
+ base_code = (data_.buf[1] & 0x07) | ((data_.rex & 0x01) << 3);
+ // A base register of 0x05 (rbp) with mod = 0 means no base register.
+ if (base_code == rbp.code() && ((data_.buf[0] & 0xC0) == 0)) return false;
+ return code == base_code;
+ } else {
+ // A base register with low bits of 0x05 (rbp or r13) and mod = 0 means
+ // no base register.
+ if (base_code == rbp.code() && ((data_.buf[0] & 0xC0) == 0)) return false;
+ base_code |= ((data_.rex & 0x01) << 3);
+ return code == base_code;
+ }
+}
+
+void Assembler::AllocateAndInstallRequestedHeapObjects(Isolate* isolate) {
+ DCHECK_IMPLIES(isolate == nullptr, heap_object_requests_.empty());
+ for (auto& request : heap_object_requests_) {
+ Address pc = reinterpret_cast<Address>(buffer_start_) + request.offset();
+ switch (request.kind()) {
+ case HeapObjectRequest::kHeapNumber: {
+ Handle<HeapNumber> object =
+ isolate->factory()->NewHeapNumber<AllocationType::kOld>(
+ request.heap_number());
+ WriteUnalignedValue(pc, object);
+ break;
+ }
+ case HeapObjectRequest::kStringConstant: {
+ const StringConstantBase* str = request.string();
+ CHECK_NOT_NULL(str);
+ Handle<String> allocated = str->AllocateStringConstant(isolate);
+ WriteUnalignedValue(pc, allocated);
+ break;
+ }
+ }
+ }
+}
+
+// Partial Constant Pool.
+bool ConstPool::AddSharedEntry(uint64_t data, int offset) {
+ auto existing = entries_.find(data);
+ if (existing == entries_.end()) {
+ entries_.insert(std::make_pair(data, offset + kMoveImm64Offset));
+ return false;
+ }
+
+ // Make sure this is called with strictly ascending offsets.
+ DCHECK_GT(offset + kMoveImm64Offset, existing->second);
+
+ entries_.insert(std::make_pair(data, offset + kMoveRipRelativeDispOffset));
+ return true;
+}
+
+bool ConstPool::TryRecordEntry(intptr_t data, RelocInfo::Mode mode) {
+ if (!FLAG_partial_constant_pool) return false;
+ DCHECK_WITH_MSG(
+ FLAG_text_is_readable,
+ "The partial constant pool requires a readable .text section");
+ if (!RelocInfo::IsShareableRelocMode(mode)) return false;
+
+ // Currently, partial constant pool only handles the following kinds of
+ // RelocInfo.
+ if (mode != RelocInfo::NONE && mode != RelocInfo::EXTERNAL_REFERENCE &&
+ mode != RelocInfo::OFF_HEAP_TARGET)
+ return false;
+
+ uint64_t raw_data = static_cast<uint64_t>(data);
+ int offset = assm_->pc_offset();
+ return AddSharedEntry(raw_data, offset);
+}
+
+bool ConstPool::IsMoveRipRelative(Address instr) {
+ return (ReadUnalignedValue<uint32_t>(instr) & kMoveRipRelativeMask) ==
+ kMoveRipRelativeInstr;
+}
+
+void ConstPool::Clear() { entries_.clear(); }
+
+void ConstPool::PatchEntries() {
+ for (EntryMap::iterator iter = entries_.begin(); iter != entries_.end();
+ iter = entries_.upper_bound(iter->first)) {
+ std::pair<EntryMap::iterator, EntryMap::iterator> range =
+ entries_.equal_range(iter->first);
+ int constant_entry_offset = 0;
+ for (EntryMap::iterator it = range.first; it != range.second; it++) {
+ if (it == range.first) {
+ constant_entry_offset = it->second;
+ continue;
+ }
+
+ DCHECK_GT(constant_entry_offset, 0);
+ DCHECK_LT(constant_entry_offset, it->second);
+ int32_t disp32 =
+ constant_entry_offset - (it->second + kRipRelativeDispSize);
+ Address disp_addr = assm_->addr_at(it->second);
+
+ // Check if the instruction is actually a rip-relative move.
+ DCHECK(IsMoveRipRelative(disp_addr - kMoveRipRelativeDispOffset));
+ // The displacement of the rip-relative move should be 0 before patching.
+ DCHECK(ReadUnalignedValue<uint32_t>(disp_addr) == 0);
+ WriteUnalignedValue(disp_addr, disp32);
+ }
+ }
+ Clear();
+}
+
+void Assembler::PatchConstPool() {
+ // There is nothing to do if there are no pending entries.
+ if (constpool_.IsEmpty()) {
+ return;
+ }
+ constpool_.PatchEntries();
+}
+
+bool Assembler::UseConstPoolFor(RelocInfo::Mode rmode) {
+ if (!FLAG_partial_constant_pool) return false;
+ return (rmode == RelocInfo::NONE || rmode == RelocInfo::EXTERNAL_REFERENCE ||
+ rmode == RelocInfo::OFF_HEAP_TARGET);
+}
+
+// -----------------------------------------------------------------------------
+// Implementation of Assembler.
+
+Assembler::Assembler(const AssemblerOptions& options,
+ std::unique_ptr<AssemblerBuffer> buffer)
+ : AssemblerBase(options, std::move(buffer)), constpool_(this) {
+ reloc_info_writer.Reposition(buffer_start_ + buffer_->size(), pc_);
+ if (CpuFeatures::IsSupported(SSE4_2)) {
+ EnableCpuFeature(SSE4_1);
+ }
+ if (CpuFeatures::IsSupported(SSE4_1)) {
+ EnableCpuFeature(SSSE3);
+ }
+
+#if defined(V8_OS_WIN_X64)
+ if (options.collect_win64_unwind_info) {
+ xdata_encoder_ = std::make_unique<win64_unwindinfo::XdataEncoder>(*this);
+ }
+#endif
+}
+
+void Assembler::GetCode(Isolate* isolate, CodeDesc* desc,
+ SafepointTableBuilder* safepoint_table_builder,
+ int handler_table_offset) {
+ // As a crutch to avoid having to add manual Align calls wherever we use a
+ // raw workflow to create Code objects (mostly in tests), add another Align
+ // call here. It does no harm - the end of the Code object is aligned to the
+ // (larger) kCodeAlignment anyways.
+ // TODO(jgruber): Consider moving responsibility for proper alignment to
+ // metadata table builders (safepoint, handler, constant pool, code
+ // comments).
+ DataAlign(Code::kMetadataAlignment);
+
+ PatchConstPool();
+ DCHECK(constpool_.IsEmpty());
+
+ const int code_comments_size = WriteCodeComments();
+
+ // At this point overflow() may be true, but the gap ensures
+ // that we are still not overlapping instructions and relocation info.
+ DCHECK(pc_ <= reloc_info_writer.pos()); // No overlap.
+
+ AllocateAndInstallRequestedHeapObjects(isolate);
+
+ // Set up code descriptor.
+ // TODO(jgruber): Reconsider how these offsets and sizes are maintained up to
+ // this point to make CodeDesc initialization less fiddly.
+
+ static constexpr int kConstantPoolSize = 0;
+ const int instruction_size = pc_offset();
+ const int code_comments_offset = instruction_size - code_comments_size;
+ const int constant_pool_offset = code_comments_offset - kConstantPoolSize;
+ const int handler_table_offset2 = (handler_table_offset == kNoHandlerTable)
+ ? constant_pool_offset
+ : handler_table_offset;
+ const int safepoint_table_offset =
+ (safepoint_table_builder == kNoSafepointTable)
+ ? handler_table_offset2
+ : safepoint_table_builder->GetCodeOffset();
+ const int reloc_info_offset =
+ static_cast<int>(reloc_info_writer.pos() - buffer_->start());
+ CodeDesc::Initialize(desc, this, safepoint_table_offset,
+ handler_table_offset2, constant_pool_offset,
+ code_comments_offset, reloc_info_offset);
+}
+
+void Assembler::FinalizeJumpOptimizationInfo() {
+ // Collection stage
+ auto jump_opt = jump_optimization_info();
+ if (jump_opt && jump_opt->is_collecting()) {
+ auto& bitmap = jump_opt->farjmp_bitmap();
+ int num = static_cast<int>(farjmp_positions_.size());
+ if (num && bitmap.empty()) {
+ bool can_opt = false;
+
+ bitmap.resize((num + 31) / 32, 0);
+ for (int i = 0; i < num; i++) {
+ int disp_pos = farjmp_positions_[i];
+ int disp = long_at(disp_pos);
+ if (is_int8(disp)) {
+ bitmap[i / 32] |= 1 << (i & 31);
+ can_opt = true;
+ }
+ }
+ if (can_opt) {
+ jump_opt->set_optimizable();
+ }
+ }
+ }
+}
+
+#if defined(V8_OS_WIN_X64)
+win64_unwindinfo::BuiltinUnwindInfo Assembler::GetUnwindInfo() const {
+ DCHECK(options().collect_win64_unwind_info);
+ DCHECK_NOT_NULL(xdata_encoder_);
+ return xdata_encoder_->unwinding_info();
+}
+#endif
+
+void Assembler::Align(int m) {
+ DCHECK(base::bits::IsPowerOfTwo(m));
+ int delta = (m - (pc_offset() & (m - 1))) & (m - 1);
+ Nop(delta);
+}
+
+void Assembler::CodeTargetAlign() {
+ Align(16); // Preferred alignment of jump targets on x64.
+}
+
+bool Assembler::IsNop(Address addr) {
+ byte* a = reinterpret_cast<byte*>(addr);
+ while (*a == 0x66) a++;
+ if (*a == 0x90) return true;
+ if (a[0] == 0xF && a[1] == 0x1F) return true;
+ return false;
+}
+
+void Assembler::bind_to(Label* L, int pos) {
+ DCHECK(!L->is_bound()); // Label may only be bound once.
+ DCHECK(0 <= pos && pos <= pc_offset()); // Position must be valid.
+ if (L->is_linked()) {
+ int current = L->pos();
+ int next = long_at(current);
+ while (next != current) {
+ if (current >= 4 && long_at(current - 4) == 0) {
+ // Absolute address.
+ intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_start_ + pos);
+ WriteUnalignedValue(addr_at(current - 4), imm64);
+ internal_reference_positions_.push_back(current - 4);
+ } else {
+ // Relative address, relative to point after address.
+ int imm32 = pos - (current + sizeof(int32_t));
+ long_at_put(current, imm32);
+ }
+ current = next;
+ next = long_at(next);
+ }
+ // Fix up last fixup on linked list.
+ if (current >= 4 && long_at(current - 4) == 0) {
+ // Absolute address.
+ intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_start_ + pos);
+ WriteUnalignedValue(addr_at(current - 4), imm64);
+ internal_reference_positions_.push_back(current - 4);
+ } else {
+ // Relative address, relative to point after address.
+ int imm32 = pos - (current + sizeof(int32_t));
+ long_at_put(current, imm32);
+ }
+ }
+ while (L->is_near_linked()) {
+ int fixup_pos = L->near_link_pos();
+ int offset_to_next =
+ static_cast<int>(*reinterpret_cast<int8_t*>(addr_at(fixup_pos)));
+ DCHECK_LE(offset_to_next, 0);
+ int disp = pos - (fixup_pos + sizeof(int8_t));
+ CHECK(is_int8(disp));
+ set_byte_at(fixup_pos, disp);
+ if (offset_to_next < 0) {
+ L->link_to(fixup_pos + offset_to_next, Label::kNear);
+ } else {
+ L->UnuseNear();
+ }
+ }
+
+ // Optimization stage
+ auto jump_opt = jump_optimization_info();
+ if (jump_opt && jump_opt->is_optimizing()) {
+ auto it = label_farjmp_maps_.find(L);
+ if (it != label_farjmp_maps_.end()) {
+ auto& pos_vector = it->second;
+ for (auto fixup_pos : pos_vector) {
+ int disp = pos - (fixup_pos + sizeof(int8_t));
+ CHECK(is_int8(disp));
+ set_byte_at(fixup_pos, disp);
+ }
+ label_farjmp_maps_.erase(it);
+ }
+ }
+ L->bind_to(pos);
+}
+
+void Assembler::bind(Label* L) { bind_to(L, pc_offset()); }
+
+void Assembler::record_farjmp_position(Label* L, int pos) {
+ auto& pos_vector = label_farjmp_maps_[L];
+ pos_vector.push_back(pos);
+}
+
+bool Assembler::is_optimizable_farjmp(int idx) {
+ if (predictable_code_size()) return false;
+
+ auto jump_opt = jump_optimization_info();
+ CHECK(jump_opt->is_optimizing());
+
+ auto& bitmap = jump_opt->farjmp_bitmap();
+ CHECK(idx < static_cast<int>(bitmap.size() * 32));
+ return !!(bitmap[idx / 32] & (1 << (idx & 31)));
+}
+
+void Assembler::GrowBuffer() {
+ DCHECK(buffer_overflow());
+
+ // Compute new buffer size.
+ DCHECK_EQ(buffer_start_, buffer_->start());
+ int old_size = buffer_->size();
+ int new_size = 2 * old_size;
+
+ // Some internal data structures overflow for very large buffers,
+ // they must ensure that kMaximalBufferSize is not too large.
+ if (new_size > kMaximalBufferSize) {
+ V8::FatalProcessOutOfMemory(nullptr, "Assembler::GrowBuffer");
+ }
+
+ // Set up new buffer.
+ std::unique_ptr<AssemblerBuffer> new_buffer = buffer_->Grow(new_size);
+ DCHECK_EQ(new_size, new_buffer->size());
+ byte* new_start = new_buffer->start();
+
+ // Copy the data.
+ intptr_t pc_delta = new_start - buffer_start_;
+ intptr_t rc_delta = (new_start + new_size) - (buffer_start_ + old_size);
+ size_t reloc_size = (buffer_start_ + old_size) - reloc_info_writer.pos();
+ MemMove(new_start, buffer_start_, pc_offset());
+ MemMove(rc_delta + reloc_info_writer.pos(), reloc_info_writer.pos(),
+ reloc_size);
+
+ // Switch buffers.
+ buffer_ = std::move(new_buffer);
+ buffer_start_ = new_start;
+ pc_ += pc_delta;
+ reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta,
+ reloc_info_writer.last_pc() + pc_delta);
+
+ // Relocate internal references.
+ for (auto pos : internal_reference_positions_) {
+ Address p = reinterpret_cast<Address>(buffer_start_ + pos);
+ WriteUnalignedValue(p, ReadUnalignedValue<intptr_t>(p) + pc_delta);
+ }
+
+ DCHECK(!buffer_overflow());
+}
+
+void Assembler::emit_operand(int code, Operand adr) {
+ DCHECK(is_uint3(code));
+ const unsigned length = adr.data().len;
+ DCHECK_GT(length, 0);
+
+ // Emit updated ModR/M byte containing the given register.
+ DCHECK_EQ(adr.data().buf[0] & 0x38, 0);
+ *pc_++ = adr.data().buf[0] | code << 3;
+
+ // Recognize RIP relative addressing.
+ if (adr.data().buf[0] == 5) {
+ DCHECK_EQ(9u, length);
+ Label* label = ReadUnalignedValue<Label*>(
+ reinterpret_cast<Address>(&adr.data().buf[1]));
+ if (label->is_bound()) {
+ int offset =
+ label->pos() - pc_offset() - sizeof(int32_t) + adr.data().addend;
+ DCHECK_GE(0, offset);
+ emitl(offset);
+ } else if (label->is_linked()) {
+ emitl(label->pos());
+ label->link_to(pc_offset() - sizeof(int32_t));
+ } else {
+ DCHECK(label->is_unused());
+ int32_t current = pc_offset();
+ emitl(current);
+ label->link_to(current);
+ }
+ } else {
+ // Emit the rest of the encoded operand.
+ for (unsigned i = 1; i < length; i++) *pc_++ = adr.data().buf[i];
+ }
+}
+
+// Assembler Instruction implementations.
+
+void Assembler::arithmetic_op(byte opcode, Register reg, Operand op, int size) {
+ EnsureSpace ensure_space(this);
+ emit_rex(reg, op, size);
+ emit(opcode);
+ emit_operand(reg, op);
+}
+
+void Assembler::arithmetic_op(byte opcode, Register reg, Register rm_reg,
+ int size) {
+ EnsureSpace ensure_space(this);
+ DCHECK_EQ(opcode & 0xC6, 2);
+ if (rm_reg.low_bits() == 4) { // Forces SIB byte.
+ // Swap reg and rm_reg and change opcode operand order.
+ emit_rex(rm_reg, reg, size);
+ emit(opcode ^ 0x02);
+ emit_modrm(rm_reg, reg);
+ } else {
+ emit_rex(reg, rm_reg, size);
+ emit(opcode);
+ emit_modrm(reg, rm_reg);
+ }
+}
+
+void Assembler::arithmetic_op_16(byte opcode, Register reg, Register rm_reg) {
+ EnsureSpace ensure_space(this);
+ DCHECK_EQ(opcode & 0xC6, 2);
+ if (rm_reg.low_bits() == 4) { // Forces SIB byte.
+ // Swap reg and rm_reg and change opcode operand order.
+ emit(0x66);
+ emit_optional_rex_32(rm_reg, reg);
+ emit(opcode ^ 0x02);
+ emit_modrm(rm_reg, reg);
+ } else {
+ emit(0x66);
+ emit_optional_rex_32(reg, rm_reg);
+ emit(opcode);
+ emit_modrm(reg, rm_reg);
+ }
+}
+
+void Assembler::arithmetic_op_16(byte opcode, Register reg, Operand rm_reg) {
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_optional_rex_32(reg, rm_reg);
+ emit(opcode);
+ emit_operand(reg, rm_reg);
+}
+
+void Assembler::arithmetic_op_8(byte opcode, Register reg, Operand op) {
+ EnsureSpace ensure_space(this);
+ if (!reg.is_byte_register()) {
+ emit_rex_32(reg, op);
+ } else {
+ emit_optional_rex_32(reg, op);
+ }
+ emit(opcode);
+ emit_operand(reg, op);
+}
+
+void Assembler::arithmetic_op_8(byte opcode, Register reg, Register rm_reg) {
+ EnsureSpace ensure_space(this);
+ DCHECK_EQ(opcode & 0xC6, 2);
+ if (rm_reg.low_bits() == 4) { // Forces SIB byte.
+ // Swap reg and rm_reg and change opcode operand order.
+ if (!rm_reg.is_byte_register() || !reg.is_byte_register()) {
+ // Register is not one of al, bl, cl, dl. Its encoding needs REX.
+ emit_rex_32(rm_reg, reg);
+ }
+ emit(opcode ^ 0x02);
+ emit_modrm(rm_reg, reg);
+ } else {
+ if (!reg.is_byte_register() || !rm_reg.is_byte_register()) {
+ // Register is not one of al, bl, cl, dl. Its encoding needs REX.
+ emit_rex_32(reg, rm_reg);
+ }
+ emit(opcode);
+ emit_modrm(reg, rm_reg);
+ }
+}
+
+void Assembler::immediate_arithmetic_op(byte subcode, Register dst,
+ Immediate src, int size) {
+ EnsureSpace ensure_space(this);
+ emit_rex(dst, size);
+ if (is_int8(src.value_) && RelocInfo::IsNone(src.rmode_)) {
+ emit(0x83);
+ emit_modrm(subcode, dst);
+ emit(src.value_);
+ } else if (dst == rax) {
+ emit(0x05 | (subcode << 3));
+ emit(src);
+ } else {
+ emit(0x81);
+ emit_modrm(subcode, dst);
+ emit(src);
+ }
+}
+
+void Assembler::immediate_arithmetic_op(byte subcode, Operand dst,
+ Immediate src, int size) {
+ EnsureSpace ensure_space(this);
+ emit_rex(dst, size);
+ if (is_int8(src.value_) && RelocInfo::IsNone(src.rmode_)) {
+ emit(0x83);
+ emit_operand(subcode, dst);
+ emit(src.value_);
+ } else {
+ emit(0x81);
+ emit_operand(subcode, dst);
+ emit(src);
+ }
+}
+
+void Assembler::immediate_arithmetic_op_16(byte subcode, Register dst,
+ Immediate src) {
+ EnsureSpace ensure_space(this);
+ emit(0x66); // Operand size override prefix.
+ emit_optional_rex_32(dst);
+ if (is_int8(src.value_)) {
+ emit(0x83);
+ emit_modrm(subcode, dst);
+ emit(src.value_);
+ } else if (dst == rax) {
+ emit(0x05 | (subcode << 3));
+ emitw(src.value_);
+ } else {
+ emit(0x81);
+ emit_modrm(subcode, dst);
+ emitw(src.value_);
+ }
+}
+
+void Assembler::immediate_arithmetic_op_16(byte subcode, Operand dst,
+ Immediate src) {
+ EnsureSpace ensure_space(this);
+ emit(0x66); // Operand size override prefix.
+ emit_optional_rex_32(dst);
+ if (is_int8(src.value_)) {
+ emit(0x83);
+ emit_operand(subcode, dst);
+ emit(src.value_);
+ } else {
+ emit(0x81);
+ emit_operand(subcode, dst);
+ emitw(src.value_);
+ }
+}
+
+void Assembler::immediate_arithmetic_op_8(byte subcode, Operand dst,
+ Immediate src) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(dst);
+ DCHECK(is_int8(src.value_) || is_uint8(src.value_));
+ emit(0x80);
+ emit_operand(subcode, dst);
+ emit(src.value_);
+}
+
+void Assembler::immediate_arithmetic_op_8(byte subcode, Register dst,
+ Immediate src) {
+ EnsureSpace ensure_space(this);
+ if (!dst.is_byte_register()) {
+ // Register is not one of al, bl, cl, dl. Its encoding needs REX.
+ emit_rex_32(dst);
+ }
+ DCHECK(is_int8(src.value_) || is_uint8(src.value_));
+ emit(0x80);
+ emit_modrm(subcode, dst);
+ emit(src.value_);
+}
+
+void Assembler::shift(Register dst, Immediate shift_amount, int subcode,
+ int size) {
+ EnsureSpace ensure_space(this);
+ DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_)
+ : is_uint5(shift_amount.value_));
+ if (shift_amount.value_ == 1) {
+ emit_rex(dst, size);
+ emit(0xD1);
+ emit_modrm(subcode, dst);
+ } else {
+ emit_rex(dst, size);
+ emit(0xC1);
+ emit_modrm(subcode, dst);
+ emit(shift_amount.value_);
+ }
+}
+
+void Assembler::shift(Operand dst, Immediate shift_amount, int subcode,
+ int size) {
+ EnsureSpace ensure_space(this);
+ DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_)
+ : is_uint5(shift_amount.value_));
+ if (shift_amount.value_ == 1) {
+ emit_rex(dst, size);
+ emit(0xD1);
+ emit_operand(subcode, dst);
+ } else {
+ emit_rex(dst, size);
+ emit(0xC1);
+ emit_operand(subcode, dst);
+ emit(shift_amount.value_);
+ }
+}
+
+void Assembler::shift(Register dst, int subcode, int size) {
+ EnsureSpace ensure_space(this);
+ emit_rex(dst, size);
+ emit(0xD3);
+ emit_modrm(subcode, dst);
+}
+
+void Assembler::shift(Operand dst, int subcode, int size) {
+ EnsureSpace ensure_space(this);
+ emit_rex(dst, size);
+ emit(0xD3);
+ emit_operand(subcode, dst);
+}
+
+void Assembler::bswapl(Register dst) {
+ EnsureSpace ensure_space(this);
+ emit_rex_32(dst);
+ emit(0x0F);
+ emit(0xC8 + dst.low_bits());
+}
+
+void Assembler::bswapq(Register dst) {
+ EnsureSpace ensure_space(this);
+ emit_rex_64(dst);
+ emit(0x0F);
+ emit(0xC8 + dst.low_bits());
+}
+
+void Assembler::btq(Operand dst, Register src) {
+ EnsureSpace ensure_space(this);
+ emit_rex_64(src, dst);
+ emit(0x0F);
+ emit(0xA3);
+ emit_operand(src, dst);
+}
+
+void Assembler::btsq(Operand dst, Register src) {
+ EnsureSpace ensure_space(this);
+ emit_rex_64(src, dst);
+ emit(0x0F);
+ emit(0xAB);
+ emit_operand(src, dst);
+}
+
+void Assembler::btsq(Register dst, Immediate imm8) {
+ EnsureSpace ensure_space(this);
+ emit_rex_64(dst);
+ emit(0x0F);
+ emit(0xBA);
+ emit_modrm(0x5, dst);
+ emit(imm8.value_);
+}
+
+void Assembler::btrq(Register dst, Immediate imm8) {
+ EnsureSpace ensure_space(this);
+ emit_rex_64(dst);
+ emit(0x0F);
+ emit(0xBA);
+ emit_modrm(0x6, dst);
+ emit(imm8.value_);
+}
+
+void Assembler::bsrl(Register dst, Register src) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0xBD);
+ emit_modrm(dst, src);
+}
+
+void Assembler::bsrl(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0xBD);
+ emit_operand(dst, src);
+}
+
+void Assembler::bsrq(Register dst, Register src) {
+ EnsureSpace ensure_space(this);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0xBD);
+ emit_modrm(dst, src);
+}
+
+void Assembler::bsrq(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0xBD);
+ emit_operand(dst, src);
+}
+
+void Assembler::bsfl(Register dst, Register src) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0xBC);
+ emit_modrm(dst, src);
+}
+
+void Assembler::bsfl(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0xBC);
+ emit_operand(dst, src);
+}
+
+void Assembler::bsfq(Register dst, Register src) {
+ EnsureSpace ensure_space(this);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0xBC);
+ emit_modrm(dst, src);
+}
+
+void Assembler::bsfq(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0xBC);
+ emit_operand(dst, src);
+}
+
+void Assembler::pblendw(XMMRegister dst, Operand src, uint8_t mask) {
+ sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0E);
+ emit(mask);
+}
+
+void Assembler::pblendw(XMMRegister dst, XMMRegister src, uint8_t mask) {
+ sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0E);
+ emit(mask);
+}
+
+void Assembler::palignr(XMMRegister dst, Operand src, uint8_t mask) {
+ ssse3_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0F);
+ emit(mask);
+}
+
+void Assembler::palignr(XMMRegister dst, XMMRegister src, uint8_t mask) {
+ ssse3_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0F);
+ emit(mask);
+}
+
+void Assembler::call(Label* L) {
+ EnsureSpace ensure_space(this);
+ // 1110 1000 #32-bit disp.
+ emit(0xE8);
+ if (L->is_bound()) {
+ int offset = L->pos() - pc_offset() - sizeof(int32_t);
+ DCHECK_LE(offset, 0);
+ emitl(offset);
+ } else if (L->is_linked()) {
+ emitl(L->pos());
+ L->link_to(pc_offset() - sizeof(int32_t));
+ } else {
+ DCHECK(L->is_unused());
+ int32_t current = pc_offset();
+ emitl(current);
+ L->link_to(current);
+ }
+}
+
+void Assembler::call(Address entry, RelocInfo::Mode rmode) {
+ DCHECK(RelocInfo::IsRuntimeEntry(rmode));
+ EnsureSpace ensure_space(this);
+ // 1110 1000 #32-bit disp.
+ emit(0xE8);
+ emit_runtime_entry(entry, rmode);
+}
+
+void Assembler::call(Handle<Code> target, RelocInfo::Mode rmode) {
+ DCHECK(RelocInfo::IsCodeTarget(rmode));
+ DCHECK(target->IsExecutable());
+ EnsureSpace ensure_space(this);
+ // 1110 1000 #32-bit disp.
+ emit(0xE8);
+ RecordRelocInfo(rmode);
+ int code_target_index = AddCodeTarget(target);
+ emitl(code_target_index);
+}
+
+void Assembler::near_call(intptr_t disp, RelocInfo::Mode rmode) {
+ EnsureSpace ensure_space(this);
+ emit(0xE8);
+ DCHECK(is_int32(disp));
+ RecordRelocInfo(rmode);
+ emitl(static_cast<int32_t>(disp));
+}
+
+void Assembler::near_jmp(intptr_t disp, RelocInfo::Mode rmode) {
+ EnsureSpace ensure_space(this);
+ emit(0xE9);
+ DCHECK(is_int32(disp));
+ if (!RelocInfo::IsNone(rmode)) RecordRelocInfo(rmode);
+ emitl(static_cast<int32_t>(disp));
+}
+
+void Assembler::call(Register adr) {
+ EnsureSpace ensure_space(this);
+ // Opcode: FF /2 r64.
+ emit_optional_rex_32(adr);
+ emit(0xFF);
+ emit_modrm(0x2, adr);
+}
+
+void Assembler::call(Operand op) {
+ EnsureSpace ensure_space(this);
+ // Opcode: FF /2 m64.
+ emit_optional_rex_32(op);
+ emit(0xFF);
+ emit_operand(0x2, op);
+}
+
+void Assembler::clc() {
+ EnsureSpace ensure_space(this);
+ emit(0xF8);
+}
+
+void Assembler::cld() {
+ EnsureSpace ensure_space(this);
+ emit(0xFC);
+}
+
+void Assembler::cdq() {
+ EnsureSpace ensure_space(this);
+ emit(0x99);
+}
+
+void Assembler::cmovq(Condition cc, Register dst, Register src) {
+ if (cc == always) {
+ movq(dst, src);
+ } else if (cc == never) {
+ return;
+ }
+ // No need to check CpuInfo for CMOV support, it's a required part of the
+ // 64-bit architecture.
+ DCHECK_GE(cc, 0); // Use mov for unconditional moves.
+ EnsureSpace ensure_space(this);
+ // Opcode: REX.W 0f 40 + cc /r.
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0x40 + cc);
+ emit_modrm(dst, src);
+}
+
+void Assembler::cmovq(Condition cc, Register dst, Operand src) {
+ if (cc == always) {
+ movq(dst, src);
+ } else if (cc == never) {
+ return;
+ }
+ DCHECK_GE(cc, 0);
+ EnsureSpace ensure_space(this);
+ // Opcode: REX.W 0f 40 + cc /r.
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0x40 + cc);
+ emit_operand(dst, src);
+}
+
+void Assembler::cmovl(Condition cc, Register dst, Register src) {
+ if (cc == always) {
+ movl(dst, src);
+ } else if (cc == never) {
+ return;
+ }
+ DCHECK_GE(cc, 0);
+ EnsureSpace ensure_space(this);
+ // Opcode: 0f 40 + cc /r.
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x40 + cc);
+ emit_modrm(dst, src);
+}
+
+void Assembler::cmovl(Condition cc, Register dst, Operand src) {
+ if (cc == always) {
+ movl(dst, src);
+ } else if (cc == never) {
+ return;
+ }
+ DCHECK_GE(cc, 0);
+ EnsureSpace ensure_space(this);
+ // Opcode: 0f 40 + cc /r.
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x40 + cc);
+ emit_operand(dst, src);
+}
+
+void Assembler::cmpb_al(Immediate imm8) {
+ DCHECK(is_int8(imm8.value_) || is_uint8(imm8.value_));
+ EnsureSpace ensure_space(this);
+ emit(0x3C);
+ emit(imm8.value_);
+}
+
+void Assembler::lock() {
+ EnsureSpace ensure_space(this);
+ emit(0xF0);
+}
+
+void Assembler::xaddb(Operand dst, Register src) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_8(src, dst);
+ emit(0x0F);
+ emit(0xC0);
+ emit_operand(src, dst);
+}
+
+void Assembler::xaddw(Operand dst, Register src) {
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_optional_rex_32(src, dst);
+ emit(0x0F);
+ emit(0xC1);
+ emit_operand(src, dst);
+}
+
+void Assembler::xaddl(Operand dst, Register src) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(src, dst);
+ emit(0x0F);
+ emit(0xC1);
+ emit_operand(src, dst);
+}
+
+void Assembler::xaddq(Operand dst, Register src) {
+ EnsureSpace ensure_space(this);
+ emit_rex(src, dst, kInt64Size);
+ emit(0x0F);
+ emit(0xC1);
+ emit_operand(src, dst);
+}
+
+void Assembler::cmpxchgb(Operand dst, Register src) {
+ EnsureSpace ensure_space(this);
+ if (!src.is_byte_register()) {
+ // Register is not one of al, bl, cl, dl. Its encoding needs REX.
+ emit_rex_32(src, dst);
+ } else {
+ emit_optional_rex_32(src, dst);
+ }
+ emit(0x0F);
+ emit(0xB0);
+ emit_operand(src, dst);
+}
+
+void Assembler::cmpxchgw(Operand dst, Register src) {
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_optional_rex_32(src, dst);
+ emit(0x0F);
+ emit(0xB1);
+ emit_operand(src, dst);
+}
+
+void Assembler::emit_cmpxchg(Operand dst, Register src, int size) {
+ EnsureSpace ensure_space(this);
+ emit_rex(src, dst, size);
+ emit(0x0F);
+ emit(0xB1);
+ emit_operand(src, dst);
+}
+
+void Assembler::mfence() {
+ EnsureSpace ensure_space(this);
+ emit(0x0F);
+ emit(0xAE);
+ emit(0xF0);
+}
+
+void Assembler::lfence() {
+ EnsureSpace ensure_space(this);
+ emit(0x0F);
+ emit(0xAE);
+ emit(0xE8);
+}
+
+void Assembler::cpuid() {
+ EnsureSpace ensure_space(this);
+ emit(0x0F);
+ emit(0xA2);
+}
+
+void Assembler::cqo() {
+ EnsureSpace ensure_space(this);
+ emit_rex_64();
+ emit(0x99);
+}
+
+void Assembler::emit_dec(Register dst, int size) {
+ EnsureSpace ensure_space(this);
+ emit_rex(dst, size);
+ emit(0xFF);
+ emit_modrm(0x1, dst);
+}
+
+void Assembler::emit_dec(Operand dst, int size) {
+ EnsureSpace ensure_space(this);
+ emit_rex(dst, size);
+ emit(0xFF);
+ emit_operand(1, dst);
+}
+
+void Assembler::decb(Register dst) {
+ EnsureSpace ensure_space(this);
+ if (!dst.is_byte_register()) {
+ // Register is not one of al, bl, cl, dl. Its encoding needs REX.
+ emit_rex_32(dst);
+ }
+ emit(0xFE);
+ emit_modrm(0x1, dst);
+}
+
+void Assembler::decb(Operand dst) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(dst);
+ emit(0xFE);
+ emit_operand(1, dst);
+}
+
+void Assembler::hlt() {
+ EnsureSpace ensure_space(this);
+ emit(0xF4);
+}
+
+void Assembler::emit_idiv(Register src, int size) {
+ EnsureSpace ensure_space(this);
+ emit_rex(src, size);
+ emit(0xF7);
+ emit_modrm(0x7, src);
+}
+
+void Assembler::emit_div(Register src, int size) {
+ EnsureSpace ensure_space(this);
+ emit_rex(src, size);
+ emit(0xF7);
+ emit_modrm(0x6, src);
+}
+
+void Assembler::emit_imul(Register src, int size) {
+ EnsureSpace ensure_space(this);
+ emit_rex(src, size);
+ emit(0xF7);
+ emit_modrm(0x5, src);
+}
+
+void Assembler::emit_imul(Operand src, int size) {
+ EnsureSpace ensure_space(this);
+ emit_rex(src, size);
+ emit(0xF7);
+ emit_operand(0x5, src);
+}
+
+void Assembler::emit_imul(Register dst, Register src, int size) {
+ EnsureSpace ensure_space(this);
+ emit_rex(dst, src, size);
+ emit(0x0F);
+ emit(0xAF);
+ emit_modrm(dst, src);
+}
+
+void Assembler::emit_imul(Register dst, Operand src, int size) {
+ EnsureSpace ensure_space(this);
+ emit_rex(dst, src, size);
+ emit(0x0F);
+ emit(0xAF);
+ emit_operand(dst, src);
+}
+
+void Assembler::emit_imul(Register dst, Register src, Immediate imm, int size) {
+ EnsureSpace ensure_space(this);
+ emit_rex(dst, src, size);
+ if (is_int8(imm.value_)) {
+ emit(0x6B);
+ emit_modrm(dst, src);
+ emit(imm.value_);
+ } else {
+ emit(0x69);
+ emit_modrm(dst, src);
+ emitl(imm.value_);
+ }
+}
+
+void Assembler::emit_imul(Register dst, Operand src, Immediate imm, int size) {
+ EnsureSpace ensure_space(this);
+ emit_rex(dst, src, size);
+ if (is_int8(imm.value_)) {
+ emit(0x6B);
+ emit_operand(dst, src);
+ emit(imm.value_);
+ } else {
+ emit(0x69);
+ emit_operand(dst, src);
+ emitl(imm.value_);
+ }
+}
+
+void Assembler::emit_inc(Register dst, int size) {
+ EnsureSpace ensure_space(this);
+ emit_rex(dst, size);
+ emit(0xFF);
+ emit_modrm(0x0, dst);
+}
+
+void Assembler::emit_inc(Operand dst, int size) {
+ EnsureSpace ensure_space(this);
+ emit_rex(dst, size);
+ emit(0xFF);
+ emit_operand(0, dst);
+}
+
+void Assembler::int3() {
+ EnsureSpace ensure_space(this);
+ emit(0xCC);
+}
+
+void Assembler::j(Condition cc, Label* L, Label::Distance distance) {
+ if (cc == always) {
+ jmp(L, distance);
+ return;
+ } else if (cc == never) {
+ return;
+ }
+ EnsureSpace ensure_space(this);
+ DCHECK(is_uint4(cc));
+ if (L->is_bound()) {
+ const int short_size = 2;
+ const int long_size = 6;
+ int offs = L->pos() - pc_offset();
+ DCHECK_LE(offs, 0);
+ // Determine whether we can use 1-byte offsets for backwards branches,
+ // which have a max range of 128 bytes.
+
+ // We also need to check predictable_code_size() flag here, because on x64,
+ // when the full code generator recompiles code for debugging, some places
+ // need to be padded out to a certain size. The debugger is keeping track of
+ // how often it did this so that it can adjust return addresses on the
+ // stack, but if the size of jump instructions can also change, that's not
+ // enough and the calculated offsets would be incorrect.
+ if (is_int8(offs - short_size) && !predictable_code_size()) {
+ // 0111 tttn #8-bit disp.
+ emit(0x70 | cc);
+ emit((offs - short_size) & 0xFF);
+ } else {
+ // 0000 1111 1000 tttn #32-bit disp.
+ emit(0x0F);
+ emit(0x80 | cc);
+ emitl(offs - long_size);
+ }
+ } else if (distance == Label::kNear) {
+ // 0111 tttn #8-bit disp
+ emit(0x70 | cc);
+ byte disp = 0x00;
+ if (L->is_near_linked()) {
+ int offset = L->near_link_pos() - pc_offset();
+ DCHECK(is_int8(offset));
+ disp = static_cast<byte>(offset & 0xFF);
+ }
+ L->link_to(pc_offset(), Label::kNear);
+ emit(disp);
+ } else {
+ auto jump_opt = jump_optimization_info();
+ if (V8_UNLIKELY(jump_opt)) {
+ if (jump_opt->is_optimizing() && is_optimizable_farjmp(farjmp_num_++)) {
+ // 0111 tttn #8-bit disp
+ emit(0x70 | cc);
+ record_farjmp_position(L, pc_offset());
+ emit(0);
+ return;
+ }
+ if (jump_opt->is_collecting()) {
+ farjmp_positions_.push_back(pc_offset() + 2);
+ }
+ }
+ if (L->is_linked()) {
+ // 0000 1111 1000 tttn #32-bit disp.
+ emit(0x0F);
+ emit(0x80 | cc);
+ emitl(L->pos());
+ L->link_to(pc_offset() - sizeof(int32_t));
+ } else {
+ DCHECK(L->is_unused());
+ emit(0x0F);
+ emit(0x80 | cc);
+ int32_t current = pc_offset();
+ emitl(current);
+ L->link_to(current);
+ }
+ }
+}
+
+void Assembler::j(Condition cc, Address entry, RelocInfo::Mode rmode) {
+ DCHECK(RelocInfo::IsRuntimeEntry(rmode));
+ EnsureSpace ensure_space(this);
+ DCHECK(is_uint4(cc));
+ emit(0x0F);
+ emit(0x80 | cc);
+ emit_runtime_entry(entry, rmode);
+}
+
+void Assembler::j(Condition cc, Handle<Code> target, RelocInfo::Mode rmode) {
+ if (cc == always) {
+ jmp(target, rmode);
+ return;
+ } else if (cc == never) {
+ return;
+ }
+ EnsureSpace ensure_space(this);
+ DCHECK(is_uint4(cc));
+ // 0000 1111 1000 tttn #32-bit disp.
+ emit(0x0F);
+ emit(0x80 | cc);
+ DCHECK(RelocInfo::IsCodeTarget(rmode));
+ RecordRelocInfo(rmode);
+ int code_target_index = AddCodeTarget(target);
+ emitl(code_target_index);
+}
+
+void Assembler::jmp_rel(int32_t offset) {
+ EnsureSpace ensure_space(this);
+ // The offset is encoded relative to the next instruction.
+ constexpr int32_t kShortJmpDisplacement = 1 + sizeof(int8_t);
+ constexpr int32_t kNearJmpDisplacement = 1 + sizeof(int32_t);
+ DCHECK_LE(std::numeric_limits<int32_t>::min() + kNearJmpDisplacement, offset);
+ if (is_int8(offset - kShortJmpDisplacement) && !predictable_code_size()) {
+ // 0xEB #8-bit disp.
+ emit(0xEB);
+ emit(offset - kShortJmpDisplacement);
+ } else {
+ // 0xE9 #32-bit disp.
+ emit(0xE9);
+ emitl(offset - kNearJmpDisplacement);
+ }
+}
+
+void Assembler::jmp(Label* L, Label::Distance distance) {
+ const int long_size = sizeof(int32_t);
+
+ if (L->is_bound()) {
+ int offset = L->pos() - pc_offset();
+ DCHECK_LE(offset, 0); // backward jump.
+ jmp_rel(offset);
+ return;
+ }
+
+ EnsureSpace ensure_space(this);
+ if (distance == Label::kNear) {
+ emit(0xEB);
+ byte disp = 0x00;
+ if (L->is_near_linked()) {
+ int offset = L->near_link_pos() - pc_offset();
+ DCHECK(is_int8(offset));
+ disp = static_cast<byte>(offset & 0xFF);
+ }
+ L->link_to(pc_offset(), Label::kNear);
+ emit(disp);
+ } else {
+ auto jump_opt = jump_optimization_info();
+ if (V8_UNLIKELY(jump_opt)) {
+ if (jump_opt->is_optimizing() && is_optimizable_farjmp(farjmp_num_++)) {
+ emit(0xEB);
+ record_farjmp_position(L, pc_offset());
+ emit(0);
+ return;
+ }
+ if (jump_opt->is_collecting()) {
+ farjmp_positions_.push_back(pc_offset() + 1);
+ }
+ }
+ if (L->is_linked()) {
+ // 1110 1001 #32-bit disp.
+ emit(0xE9);
+ emitl(L->pos());
+ L->link_to(pc_offset() - long_size);
+ } else {
+ // 1110 1001 #32-bit disp.
+ DCHECK(L->is_unused());
+ emit(0xE9);
+ int32_t current = pc_offset();
+ emitl(current);
+ L->link_to(current);
+ }
+ }
+}
+
+void Assembler::jmp(Handle<Code> target, RelocInfo::Mode rmode) {
+ DCHECK(RelocInfo::IsCodeTarget(rmode));
+ EnsureSpace ensure_space(this);
+ // 1110 1001 #32-bit disp.
+ emit(0xE9);
+ RecordRelocInfo(rmode);
+ int code_target_index = AddCodeTarget(target);
+ emitl(code_target_index);
+}
+
+void Assembler::jmp(Register target) {
+ EnsureSpace ensure_space(this);
+ // Opcode FF/4 r64.
+ emit_optional_rex_32(target);
+ emit(0xFF);
+ emit_modrm(0x4, target);
+}
+
+void Assembler::jmp(Operand src) {
+ EnsureSpace ensure_space(this);
+ // Opcode FF/4 m64.
+ emit_optional_rex_32(src);
+ emit(0xFF);
+ emit_operand(0x4, src);
+}
+
+void Assembler::emit_lea(Register dst, Operand src, int size) {
+ EnsureSpace ensure_space(this);
+ emit_rex(dst, src, size);
+ emit(0x8D);
+ emit_operand(dst, src);
+}
+
+void Assembler::load_rax(Address value, RelocInfo::Mode mode) {
+ EnsureSpace ensure_space(this);
+ emit(0x48); // REX.W
+ emit(0xA1);
+ emit(Immediate64(value, mode));
+}
+
+void Assembler::load_rax(ExternalReference ref) {
+ load_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE);
+}
+
+void Assembler::leave() {
+ EnsureSpace ensure_space(this);
+ emit(0xC9);
+}
+
+void Assembler::movb(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ if (!dst.is_byte_register()) {
+ // Register is not one of al, bl, cl, dl. Its encoding needs REX.
+ emit_rex_32(dst, src);
+ } else {
+ emit_optional_rex_32(dst, src);
+ }
+ emit(0x8A);
+ emit_operand(dst, src);
+}
+
+void Assembler::movb(Register dst, Immediate imm) {
+ EnsureSpace ensure_space(this);
+ if (!dst.is_byte_register()) {
+ // Register is not one of al, bl, cl, dl. Its encoding needs REX.
+ emit_rex_32(dst);
+ }
+ emit(0xB0 + dst.low_bits());
+ emit(imm.value_);
+}
+
+void Assembler::movb(Operand dst, Register src) {
+ EnsureSpace ensure_space(this);
+ if (!src.is_byte_register()) {
+ // Register is not one of al, bl, cl, dl. Its encoding needs REX.
+ emit_rex_32(src, dst);
+ } else {
+ emit_optional_rex_32(src, dst);
+ }
+ emit(0x88);
+ emit_operand(src, dst);
+}
+
+void Assembler::movb(Operand dst, Immediate imm) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(dst);
+ emit(0xC6);
+ emit_operand(0x0, dst);
+ emit(static_cast<byte>(imm.value_));
+}
+
+void Assembler::movw(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_optional_rex_32(dst, src);
+ emit(0x8B);
+ emit_operand(dst, src);
+}
+
+void Assembler::movw(Operand dst, Register src) {
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_optional_rex_32(src, dst);
+ emit(0x89);
+ emit_operand(src, dst);
+}
+
+void Assembler::movw(Operand dst, Immediate imm) {
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_optional_rex_32(dst);
+ emit(0xC7);
+ emit_operand(0x0, dst);
+ emit(static_cast<byte>(imm.value_ & 0xFF));
+ emit(static_cast<byte>(imm.value_ >> 8));
+}
+
+void Assembler::emit_mov(Register dst, Operand src, int size) {
+ EnsureSpace ensure_space(this);
+ emit_rex(dst, src, size);
+ emit(0x8B);
+ emit_operand(dst, src);
+}
+
+void Assembler::emit_mov(Register dst, Register src, int size) {
+ EnsureSpace ensure_space(this);
+ if (src.low_bits() == 4) {
+ emit_rex(src, dst, size);
+ emit(0x89);
+ emit_modrm(src, dst);
+ } else {
+ emit_rex(dst, src, size);
+ emit(0x8B);
+ emit_modrm(dst, src);
+ }
+
+#if defined(V8_OS_WIN_X64)
+ if (xdata_encoder_ && dst == rbp && src == rsp) {
+ xdata_encoder_->onMovRbpRsp();
+ }
+#endif
+}
+
+void Assembler::emit_mov(Operand dst, Register src, int size) {
+ EnsureSpace ensure_space(this);
+ emit_rex(src, dst, size);
+ emit(0x89);
+ emit_operand(src, dst);
+}
+
+void Assembler::emit_mov(Register dst, Immediate value, int size) {
+ EnsureSpace ensure_space(this);
+ emit_rex(dst, size);
+ if (size == kInt64Size) {
+ emit(0xC7);
+ emit_modrm(0x0, dst);
+ } else {
+ DCHECK_EQ(size, kInt32Size);
+ emit(0xB8 + dst.low_bits());
+ }
+ emit(value);
+}
+
+void Assembler::emit_mov(Operand dst, Immediate value, int size) {
+ EnsureSpace ensure_space(this);
+ emit_rex(dst, size);
+ emit(0xC7);
+ emit_operand(0x0, dst);
+ emit(value);
+}
+
+void Assembler::emit_mov(Register dst, Immediate64 value, int size) {
+ DCHECK_EQ(size, kInt64Size);
+ if (constpool_.TryRecordEntry(value.value_, value.rmode_)) {
+ // Emit rip-relative move with offset = 0
+ Label label;
+ emit_mov(dst, Operand(&label, 0), size);
+ bind(&label);
+ } else {
+ EnsureSpace ensure_space(this);
+ emit_rex(dst, size);
+ emit(0xB8 | dst.low_bits());
+ emit(value);
+ }
+}
+
+void Assembler::movq_imm64(Register dst, int64_t value) {
+ EnsureSpace ensure_space(this);
+ emit_rex(dst, kInt64Size);
+ emit(0xB8 | dst.low_bits());
+ emitq(static_cast<uint64_t>(value));
+}
+
+void Assembler::movq_heap_number(Register dst, double value) {
+ EnsureSpace ensure_space(this);
+ emit_rex(dst, kInt64Size);
+ emit(0xB8 | dst.low_bits());
+ RequestHeapObject(HeapObjectRequest(value));
+ emit(Immediate64(kNullAddress, RelocInfo::FULL_EMBEDDED_OBJECT));
+}
+
+void Assembler::movq_string(Register dst, const StringConstantBase* str) {
+ EnsureSpace ensure_space(this);
+ emit_rex(dst, kInt64Size);
+ emit(0xB8 | dst.low_bits());
+ RequestHeapObject(HeapObjectRequest(str));
+ emit(Immediate64(kNullAddress, RelocInfo::FULL_EMBEDDED_OBJECT));
+}
+
+// Loads the ip-relative location of the src label into the target location
+// (as a 32-bit offset sign extended to 64-bit).
+void Assembler::movl(Operand dst, Label* src) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(dst);
+ emit(0xC7);
+ emit_operand(0, dst);
+ if (src->is_bound()) {
+ int offset = src->pos() - pc_offset() - sizeof(int32_t);
+ DCHECK_LE(offset, 0);
+ emitl(offset);
+ } else if (src->is_linked()) {
+ emitl(src->pos());
+ src->link_to(pc_offset() - sizeof(int32_t));
+ } else {
+ DCHECK(src->is_unused());
+ int32_t current = pc_offset();
+ emitl(current);
+ src->link_to(current);
+ }
+}
+
+void Assembler::movsxbl(Register dst, Register src) {
+ EnsureSpace ensure_space(this);
+ if (!src.is_byte_register()) {
+ // Register is not one of al, bl, cl, dl. Its encoding needs REX.
+ emit_rex_32(dst, src);
+ } else {
+ emit_optional_rex_32(dst, src);
+ }
+ emit(0x0F);
+ emit(0xBE);
+ emit_modrm(dst, src);
+}
+
+void Assembler::movsxbl(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0xBE);
+ emit_operand(dst, src);
+}
+
+void Assembler::movsxbq(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0xBE);
+ emit_operand(dst, src);
+}
+
+void Assembler::movsxbq(Register dst, Register src) {
+ EnsureSpace ensure_space(this);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0xBE);
+ emit_modrm(dst, src);
+}
+
+void Assembler::movsxwl(Register dst, Register src) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0xBF);
+ emit_modrm(dst, src);
+}
+
+void Assembler::movsxwl(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0xBF);
+ emit_operand(dst, src);
+}
+
+void Assembler::movsxwq(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0xBF);
+ emit_operand(dst, src);
+}
+
+void Assembler::movsxwq(Register dst, Register src) {
+ EnsureSpace ensure_space(this);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0xBF);
+ emit_modrm(dst, src);
+}
+
+void Assembler::movsxlq(Register dst, Register src) {
+ EnsureSpace ensure_space(this);
+ emit_rex_64(dst, src);
+ emit(0x63);
+ emit_modrm(dst, src);
+}
+
+void Assembler::movsxlq(Register dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ emit_rex_64(dst, src);
+ emit(0x63);
+ emit_operand(dst, src);
+}
+
+void Assembler::emit_movzxb(Register dst, Operand src, int size) {
+ EnsureSpace ensure_space(this);
+ // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
+ // there is no need to make this a 64 bit operation.
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0xB6);
+ emit_operand(dst, src);
+}
+
+void Assembler::emit_movzxb(Register dst, Register src, int size) {
+ EnsureSpace ensure_space(this);
+ // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
+ // there is no need to make this a 64 bit operation.
+ if (!src.is_byte_register()) {
+ // Register is not one of al, bl, cl, dl. Its encoding needs REX.
+ emit_rex_32(dst, src);
+ } else {
+ emit_optional_rex_32(dst, src);
+ }
+ emit(0x0F);
+ emit(0xB6);
+ emit_modrm(dst, src);
+}
+
+void Assembler::emit_movzxw(Register dst, Operand src, int size) {
+ EnsureSpace ensure_space(this);
+ // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
+ // there is no need to make this a 64 bit operation.
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0xB7);
+ emit_operand(dst, src);
+}
+
+void Assembler::emit_movzxw(Register dst, Register src, int size) {
+ EnsureSpace ensure_space(this);
+ // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
+ // there is no need to make this a 64 bit operation.
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0xB7);
+ emit_modrm(dst, src);
+}
+
+void Assembler::repmovsb() {
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit(0xA4);
+}
+
+void Assembler::repmovsw() {
+ EnsureSpace ensure_space(this);
+ emit(0x66); // Operand size override.
+ emit(0xF3);
+ emit(0xA4);
+}
+
+void Assembler::emit_repmovs(int size) {
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_rex(size);
+ emit(0xA5);
+}
+
+void Assembler::repstosl() {
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit(0xAB);
+}
+
+void Assembler::repstosq() {
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_rex_64();
+ emit(0xAB);
+}
+
+void Assembler::mull(Register src) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(src);
+ emit(0xF7);
+ emit_modrm(0x4, src);
+}
+
+void Assembler::mull(Operand src) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(src);
+ emit(0xF7);
+ emit_operand(0x4, src);
+}
+
+void Assembler::mulq(Register src) {
+ EnsureSpace ensure_space(this);
+ emit_rex_64(src);
+ emit(0xF7);
+ emit_modrm(0x4, src);
+}
+
+void Assembler::negb(Register reg) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_8(reg);
+ emit(0xF6);
+ emit_modrm(0x3, reg);
+}
+
+void Assembler::negw(Register reg) {
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_optional_rex_32(reg);
+ emit(0xF7);
+ emit_modrm(0x3, reg);
+}
+
+void Assembler::negl(Register reg) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(reg);
+ emit(0xF7);
+ emit_modrm(0x3, reg);
+}
+
+void Assembler::negq(Register reg) {
+ EnsureSpace ensure_space(this);
+ emit_rex_64(reg);
+ emit(0xF7);
+ emit_modrm(0x3, reg);
+}
+
+void Assembler::negb(Operand op) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(op);
+ emit(0xF6);
+ emit_operand(0x3, op);
+}
+
+void Assembler::negw(Operand op) {
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_optional_rex_32(op);
+ emit(0xF7);
+ emit_operand(0x3, op);
+}
+
+void Assembler::negl(Operand op) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(op);
+ emit(0xF7);
+ emit_operand(0x3, op);
+}
+
+void Assembler::negq(Operand op) {
+ EnsureSpace ensure_space(this);
+ emit_rex_64(op);
+ emit(0xF7);
+ emit_operand(0x3, op);
+}
+
+void Assembler::nop() {
+ EnsureSpace ensure_space(this);
+ emit(0x90);
+}
+
+void Assembler::emit_not(Register dst, int size) {
+ EnsureSpace ensure_space(this);
+ emit_rex(dst, size);
+ emit(0xF7);
+ emit_modrm(0x2, dst);
+}
+
+void Assembler::emit_not(Operand dst, int size) {
+ EnsureSpace ensure_space(this);
+ emit_rex(dst, size);
+ emit(0xF7);
+ emit_operand(2, dst);
+}
+
+void Assembler::Nop(int n) {
+ DCHECK_LE(0, n);
+ // The recommended muti-byte sequences of NOP instructions from the Intel 64
+ // and IA-32 Architectures Software Developer's Manual.
+ //
+ // Len Assembly Byte Sequence
+ // 2 66 NOP 66 90H
+ // 3 NOP DWORD ptr [EAX] 0F 1F 00H
+ // 4 NOP DWORD ptr [EAX + 00H] 0F 1F 40 00H
+ // 5 NOP DWORD ptr [EAX + EAX*1 + 00H] 0F 1F 44 00 00H
+ // 6 66 NOP DWORD ptr [EAX + EAX*1 + 00H] 66 0F 1F 44 00 00H
+ // 7 NOP DWORD ptr [EAX + 00000000H] 0F 1F 80 00 00 00 00H
+ // 8 NOP DWORD ptr [EAX + EAX*1 + 00000000H] 0F 1F 84 00 00 00 00 00H
+ // 9 66 NOP DWORD ptr [EAX + EAX*1 + 00000000H] 66 0F 1F 84 00 00 00 00 00H
+
+ constexpr const char* kNopSequences =
+ "\x66\x90" // length 1 (@1) / 2 (@0)
+ "\x0F\x1F\x00" // length 3 (@2)
+ "\x0F\x1F\x40\x00" // length 4 (@5)
+ "\x66\x0F\x1F\x44\x00\x00" // length 5 (@10) / 6 (@9)
+ "\x0F\x1F\x80\x00\x00\x00\x00" // length 7 (@15)
+ "\x66\x0F\x1F\x84\x00\x00\x00\x00\x00"; // length 8 (@23) / 9 (@22)
+ constexpr int8_t kNopOffsets[10] = {0, 1, 0, 2, 5, 10, 9, 15, 23, 22};
+
+ do {
+ EnsureSpace ensure_space(this);
+ int nop_bytes = std::min(n, 9);
+ const char* sequence = kNopSequences + kNopOffsets[nop_bytes];
+ memcpy(pc_, sequence, nop_bytes);
+ pc_ += nop_bytes;
+ n -= nop_bytes;
+ } while (n);
+}
+
+void Assembler::popq(Register dst) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(dst);
+ emit(0x58 | dst.low_bits());
+}
+
+void Assembler::popq(Operand dst) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(dst);
+ emit(0x8F);
+ emit_operand(0, dst);
+}
+
+void Assembler::popfq() {
+ EnsureSpace ensure_space(this);
+ emit(0x9D);
+}
+
+void Assembler::pushq(Register src) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(src);
+ emit(0x50 | src.low_bits());
+
+#if defined(V8_OS_WIN_X64)
+ if (xdata_encoder_ && src == rbp) {
+ xdata_encoder_->onPushRbp();
+ }
+#endif
+}
+
+void Assembler::pushq(Operand src) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(src);
+ emit(0xFF);
+ emit_operand(6, src);
+}
+
+void Assembler::pushq(Immediate value) {
+ EnsureSpace ensure_space(this);
+ if (is_int8(value.value_)) {
+ emit(0x6A);
+ emit(value.value_); // Emit low byte of value.
+ } else {
+ emit(0x68);
+ emitl(value.value_);
+ }
+}
+
+void Assembler::pushq_imm32(int32_t imm32) {
+ EnsureSpace ensure_space(this);
+ emit(0x68);
+ emitl(imm32);
+}
+
+void Assembler::pushfq() {
+ EnsureSpace ensure_space(this);
+ emit(0x9C);
+}
+
+void Assembler::ret(int imm16) {
+ EnsureSpace ensure_space(this);
+ DCHECK(is_uint16(imm16));
+ if (imm16 == 0) {
+ emit(0xC3);
+ } else {
+ emit(0xC2);
+ emit(imm16 & 0xFF);
+ emit((imm16 >> 8) & 0xFF);
+ }
+}
+
+void Assembler::ud2() {
+ EnsureSpace ensure_space(this);
+ emit(0x0F);
+ emit(0x0B);
+}
+
+void Assembler::setcc(Condition cc, Register reg) {
+ if (cc > last_condition) {
+ movb(reg, Immediate(cc == always ? 1 : 0));
+ return;
+ }
+ EnsureSpace ensure_space(this);
+ DCHECK(is_uint4(cc));
+ if (!reg.is_byte_register()) {
+ // Register is not one of al, bl, cl, dl. Its encoding needs REX.
+ emit_rex_32(reg);
+ }
+ emit(0x0F);
+ emit(0x90 | cc);
+ emit_modrm(0x0, reg);
+}
+
+void Assembler::shld(Register dst, Register src) {
+ EnsureSpace ensure_space(this);
+ emit_rex_64(src, dst);
+ emit(0x0F);
+ emit(0xA5);
+ emit_modrm(src, dst);
+}
+
+void Assembler::shrd(Register dst, Register src) {
+ EnsureSpace ensure_space(this);
+ emit_rex_64(src, dst);
+ emit(0x0F);
+ emit(0xAD);
+ emit_modrm(src, dst);
+}
+
+void Assembler::xchgb(Register reg, Operand op) {
+ EnsureSpace ensure_space(this);
+ if (!reg.is_byte_register()) {
+ // Register is not one of al, bl, cl, dl. Its encoding needs REX.
+ emit_rex_32(reg, op);
+ } else {
+ emit_optional_rex_32(reg, op);
+ }
+ emit(0x86);
+ emit_operand(reg, op);
+}
+
+void Assembler::xchgw(Register reg, Operand op) {
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_optional_rex_32(reg, op);
+ emit(0x87);
+ emit_operand(reg, op);
+}
+
+void Assembler::emit_xchg(Register dst, Register src, int size) {
+ EnsureSpace ensure_space(this);
+ if (src == rax || dst == rax) { // Single-byte encoding
+ Register other = src == rax ? dst : src;
+ emit_rex(other, size);
+ emit(0x90 | other.low_bits());
+ } else if (dst.low_bits() == 4) {
+ emit_rex(dst, src, size);
+ emit(0x87);
+ emit_modrm(dst, src);
+ } else {
+ emit_rex(src, dst, size);
+ emit(0x87);
+ emit_modrm(src, dst);
+ }
+}
+
+void Assembler::emit_xchg(Register dst, Operand src, int size) {
+ EnsureSpace ensure_space(this);
+ emit_rex(dst, src, size);
+ emit(0x87);
+ emit_operand(dst, src);
+}
+
+void Assembler::store_rax(Address dst, RelocInfo::Mode mode) {
+ EnsureSpace ensure_space(this);
+ emit(0x48); // REX.W
+ emit(0xA3);
+ emit(Immediate64(dst, mode));
+}
+
+void Assembler::store_rax(ExternalReference ref) {
+ store_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE);
+}
+
+void Assembler::sub_sp_32(uint32_t imm) {
+ emit_rex_64();
+ emit(0x81); // using a literal 32-bit immediate.
+ emit_modrm(0x5, rsp);
+ emitl(imm);
+}
+
+void Assembler::testb(Register dst, Register src) {
+ EnsureSpace ensure_space(this);
+ emit_test(dst, src, sizeof(int8_t));
+}
+
+void Assembler::testb(Register reg, Immediate mask) {
+ DCHECK(is_int8(mask.value_) || is_uint8(mask.value_));
+ emit_test(reg, mask, sizeof(int8_t));
+}
+
+void Assembler::testb(Operand op, Immediate mask) {
+ DCHECK(is_int8(mask.value_) || is_uint8(mask.value_));
+ emit_test(op, mask, sizeof(int8_t));
+}
+
+void Assembler::testb(Operand op, Register reg) {
+ emit_test(op, reg, sizeof(int8_t));
+}
+
+void Assembler::testw(Register dst, Register src) {
+ emit_test(dst, src, sizeof(uint16_t));
+}
+
+void Assembler::testw(Register reg, Immediate mask) {
+ emit_test(reg, mask, sizeof(int16_t));
+}
+
+void Assembler::testw(Operand op, Immediate mask) {
+ emit_test(op, mask, sizeof(int16_t));
+}
+
+void Assembler::testw(Operand op, Register reg) {
+ emit_test(op, reg, sizeof(int16_t));
+}
+
+void Assembler::emit_test(Register dst, Register src, int size) {
+ EnsureSpace ensure_space(this);
+ if (src.low_bits() == 4) std::swap(dst, src);
+ if (size == sizeof(int16_t)) {
+ emit(0x66);
+ size = sizeof(int32_t);
+ }
+ bool byte_operand = size == sizeof(int8_t);
+ if (byte_operand) {
+ size = sizeof(int32_t);
+ if (!src.is_byte_register() || !dst.is_byte_register()) {
+ emit_rex_32(dst, src);
+ }
+ } else {
+ emit_rex(dst, src, size);
+ }
+ emit(byte_operand ? 0x84 : 0x85);
+ emit_modrm(dst, src);
+}
+
+void Assembler::emit_test(Register reg, Immediate mask, int size) {
+ if (is_uint8(mask.value_)) {
+ size = sizeof(int8_t);
+ } else if (is_uint16(mask.value_)) {
+ size = sizeof(int16_t);
+ }
+ EnsureSpace ensure_space(this);
+ bool half_word = size == sizeof(int16_t);
+ if (half_word) {
+ emit(0x66);
+ size = sizeof(int32_t);
+ }
+ bool byte_operand = size == sizeof(int8_t);
+ if (byte_operand) {
+ size = sizeof(int32_t);
+ if (!reg.is_byte_register()) emit_rex_32(reg);
+ } else {
+ emit_rex(reg, size);
+ }
+ if (reg == rax) {
+ emit(byte_operand ? 0xA8 : 0xA9);
+ } else {
+ emit(byte_operand ? 0xF6 : 0xF7);
+ emit_modrm(0x0, reg);
+ }
+ if (byte_operand) {
+ emit(mask.value_);
+ } else if (half_word) {
+ emitw(mask.value_);
+ } else {
+ emit(mask);
+ }
+}
+
+void Assembler::emit_test(Operand op, Immediate mask, int size) {
+ if (is_uint8(mask.value_)) {
+ size = sizeof(int8_t);
+ } else if (is_uint16(mask.value_)) {
+ size = sizeof(int16_t);
+ }
+ EnsureSpace ensure_space(this);
+ bool half_word = size == sizeof(int16_t);
+ if (half_word) {
+ emit(0x66);
+ size = sizeof(int32_t);
+ }
+ bool byte_operand = size == sizeof(int8_t);
+ if (byte_operand) {
+ size = sizeof(int32_t);
+ }
+ emit_rex(rax, op, size);
+ emit(byte_operand ? 0xF6 : 0xF7);
+ emit_operand(rax, op); // Operation code 0
+ if (byte_operand) {
+ emit(mask.value_);
+ } else if (half_word) {
+ emitw(mask.value_);
+ } else {
+ emit(mask);
+ }
+}
+
+void Assembler::emit_test(Operand op, Register reg, int size) {
+ EnsureSpace ensure_space(this);
+ if (size == sizeof(int16_t)) {
+ emit(0x66);
+ size = sizeof(int32_t);
+ }
+ bool byte_operand = size == sizeof(int8_t);
+ if (byte_operand) {
+ size = sizeof(int32_t);
+ if (!reg.is_byte_register()) {
+ // Register is not one of al, bl, cl, dl. Its encoding needs REX.
+ emit_rex_32(reg, op);
+ } else {
+ emit_optional_rex_32(reg, op);
+ }
+ } else {
+ emit_rex(reg, op, size);
+ }
+ emit(byte_operand ? 0x84 : 0x85);
+ emit_operand(reg, op);
+}
+
+// FPU instructions.
+
+void Assembler::fld(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xD9, 0xC0, i);
+}
+
+void Assembler::fld1() {
+ EnsureSpace ensure_space(this);
+ emit(0xD9);
+ emit(0xE8);
+}
+
+void Assembler::fldz() {
+ EnsureSpace ensure_space(this);
+ emit(0xD9);
+ emit(0xEE);
+}
+
+void Assembler::fldpi() {
+ EnsureSpace ensure_space(this);
+ emit(0xD9);
+ emit(0xEB);
+}
+
+void Assembler::fldln2() {
+ EnsureSpace ensure_space(this);
+ emit(0xD9);
+ emit(0xED);
+}
+
+void Assembler::fld_s(Operand adr) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(adr);
+ emit(0xD9);
+ emit_operand(0, adr);
+}
+
+void Assembler::fld_d(Operand adr) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(adr);
+ emit(0xDD);
+ emit_operand(0, adr);
+}
+
+void Assembler::fstp_s(Operand adr) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(adr);
+ emit(0xD9);
+ emit_operand(3, adr);
+}
+
+void Assembler::fstp_d(Operand adr) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(adr);
+ emit(0xDD);
+ emit_operand(3, adr);
+}
+
+void Assembler::fstp(int index) {
+ DCHECK(is_uint3(index));
+ EnsureSpace ensure_space(this);
+ emit_farith(0xDD, 0xD8, index);
+}
+
+void Assembler::fild_s(Operand adr) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(adr);
+ emit(0xDB);
+ emit_operand(0, adr);
+}
+
+void Assembler::fild_d(Operand adr) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(adr);
+ emit(0xDF);
+ emit_operand(5, adr);
+}
+
+void Assembler::fistp_s(Operand adr) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(adr);
+ emit(0xDB);
+ emit_operand(3, adr);
+}
+
+void Assembler::fisttp_s(Operand adr) {
+ DCHECK(IsEnabled(SSE3));
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(adr);
+ emit(0xDB);
+ emit_operand(1, adr);
+}
+
+void Assembler::fisttp_d(Operand adr) {
+ DCHECK(IsEnabled(SSE3));
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(adr);
+ emit(0xDD);
+ emit_operand(1, adr);
+}
+
+void Assembler::fist_s(Operand adr) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(adr);
+ emit(0xDB);
+ emit_operand(2, adr);
+}
+
+void Assembler::fistp_d(Operand adr) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(adr);
+ emit(0xDF);
+ emit_operand(7, adr);
+}
+
+void Assembler::fabs() {
+ EnsureSpace ensure_space(this);
+ emit(0xD9);
+ emit(0xE1);
+}
+
+void Assembler::fchs() {
+ EnsureSpace ensure_space(this);
+ emit(0xD9);
+ emit(0xE0);
+}
+
+void Assembler::fcos() {
+ EnsureSpace ensure_space(this);
+ emit(0xD9);
+ emit(0xFF);
+}
+
+void Assembler::fsin() {
+ EnsureSpace ensure_space(this);
+ emit(0xD9);
+ emit(0xFE);
+}
+
+void Assembler::fptan() {
+ EnsureSpace ensure_space(this);
+ emit(0xD9);
+ emit(0xF2);
+}
+
+void Assembler::fyl2x() {
+ EnsureSpace ensure_space(this);
+ emit(0xD9);
+ emit(0xF1);
+}
+
+void Assembler::f2xm1() {
+ EnsureSpace ensure_space(this);
+ emit(0xD9);
+ emit(0xF0);
+}
+
+void Assembler::fscale() {
+ EnsureSpace ensure_space(this);
+ emit(0xD9);
+ emit(0xFD);
+}
+
+void Assembler::fninit() {
+ EnsureSpace ensure_space(this);
+ emit(0xDB);
+ emit(0xE3);
+}
+
+void Assembler::fadd(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xDC, 0xC0, i);
+}
+
+void Assembler::fsub(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xDC, 0xE8, i);
+}
+
+void Assembler::fisub_s(Operand adr) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(adr);
+ emit(0xDA);
+ emit_operand(4, adr);
+}
+
+void Assembler::fmul(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xDC, 0xC8, i);
+}
+
+void Assembler::fdiv(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xDC, 0xF8, i);
+}
+
+void Assembler::faddp(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xDE, 0xC0, i);
+}
+
+void Assembler::fsubp(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xDE, 0xE8, i);
+}
+
+void Assembler::fsubrp(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xDE, 0xE0, i);
+}
+
+void Assembler::fmulp(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xDE, 0xC8, i);
+}
+
+void Assembler::fdivp(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xDE, 0xF8, i);
+}
+
+void Assembler::fprem() {
+ EnsureSpace ensure_space(this);
+ emit(0xD9);
+ emit(0xF8);
+}
+
+void Assembler::fprem1() {
+ EnsureSpace ensure_space(this);
+ emit(0xD9);
+ emit(0xF5);
+}
+
+void Assembler::fxch(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xD9, 0xC8, i);
+}
+
+void Assembler::fincstp() {
+ EnsureSpace ensure_space(this);
+ emit(0xD9);
+ emit(0xF7);
+}
+
+void Assembler::ffree(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xDD, 0xC0, i);
+}
+
+void Assembler::ftst() {
+ EnsureSpace ensure_space(this);
+ emit(0xD9);
+ emit(0xE4);
+}
+
+void Assembler::fucomp(int i) {
+ EnsureSpace ensure_space(this);
+ emit_farith(0xDD, 0xE8, i);
+}
+
+void Assembler::fucompp() {
+ EnsureSpace ensure_space(this);
+ emit(0xDA);
+ emit(0xE9);
+}
+
+void Assembler::fucomi(int i) {
+ EnsureSpace ensure_space(this);
+ emit(0xDB);
+ emit(0xE8 + i);
+}
+
+void Assembler::fucomip() {
+ EnsureSpace ensure_space(this);
+ emit(0xDF);
+ emit(0xE9);
+}
+
+void Assembler::fcompp() {
+ EnsureSpace ensure_space(this);
+ emit(0xDE);
+ emit(0xD9);
+}
+
+void Assembler::fnstsw_ax() {
+ EnsureSpace ensure_space(this);
+ emit(0xDF);
+ emit(0xE0);
+}
+
+void Assembler::fwait() {
+ EnsureSpace ensure_space(this);
+ emit(0x9B);
+}
+
+void Assembler::frndint() {
+ EnsureSpace ensure_space(this);
+ emit(0xD9);
+ emit(0xFC);
+}
+
+void Assembler::fnclex() {
+ EnsureSpace ensure_space(this);
+ emit(0xDB);
+ emit(0xE2);
+}
+
+void Assembler::sahf() {
+ // TODO(X64): Test for presence. Not all 64-bit intel CPU's have sahf
+ // in 64-bit mode. Test CpuID.
+ DCHECK(IsEnabled(SAHF));
+ EnsureSpace ensure_space(this);
+ emit(0x9E);
+}
+
+void Assembler::emit_farith(int b1, int b2, int i) {
+ DCHECK(is_uint8(b1) && is_uint8(b2)); // wrong opcode
+ DCHECK(is_uint3(i)); // illegal stack offset
+ emit(b1);
+ emit(b2 + i);
+}
+
+// SSE 2 operations.
+
+void Assembler::movd(XMMRegister dst, Register src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x6E);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movd(XMMRegister dst, Operand src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x6E);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movd(Register dst, XMMRegister src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_optional_rex_32(src, dst);
+ emit(0x0F);
+ emit(0x7E);
+ emit_sse_operand(src, dst);
+}
+
+void Assembler::movq(XMMRegister dst, Register src) {
+ // Mixing AVX and non-AVX is expensive, catch those cases
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0x6E);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movq(XMMRegister dst, Operand src) {
+ // Mixing AVX and non-AVX is expensive, catch those cases
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0x6E);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movq(Register dst, XMMRegister src) {
+ // Mixing AVX and non-AVX is expensive, catch those cases
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_rex_64(src, dst);
+ emit(0x0F);
+ emit(0x7E);
+ emit_sse_operand(src, dst);
+}
+
+void Assembler::movq(XMMRegister dst, XMMRegister src) {
+ // Mixing AVX and non-AVX is expensive, catch those cases
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ if (dst.low_bits() == 4) {
+ // Avoid unnecessary SIB byte.
+ emit(0xF3);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x7E);
+ emit_sse_operand(dst, src);
+ } else {
+ emit(0x66);
+ emit_optional_rex_32(src, dst);
+ emit(0x0F);
+ emit(0xD6);
+ emit_sse_operand(src, dst);
+ }
+}
+
+void Assembler::movdqa(Operand dst, XMMRegister src) {
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_rex_64(src, dst);
+ emit(0x0F);
+ emit(0x7F);
+ emit_sse_operand(src, dst);
+}
+
+void Assembler::movdqa(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0x6F);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movdqu(Operand dst, XMMRegister src) {
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_rex_64(src, dst);
+ emit(0x0F);
+ emit(0x7F);
+ emit_sse_operand(src, dst);
+}
+
+void Assembler::movdqu(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0x6F);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0x6F);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::pinsrw(XMMRegister dst, Register src, uint8_t imm8) {
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0xC4);
+ emit_sse_operand(dst, src);
+ emit(imm8);
+}
+
+void Assembler::pinsrw(XMMRegister dst, Operand src, uint8_t imm8) {
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0xC4);
+ emit_sse_operand(dst, src);
+ emit(imm8);
+}
+
+void Assembler::pextrq(Register dst, XMMRegister src, int8_t imm8) {
+ DCHECK(IsEnabled(SSE4_1));
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_rex_64(src, dst);
+ emit(0x0F);
+ emit(0x3A);
+ emit(0x16);
+ emit_sse_operand(src, dst);
+ emit(imm8);
+}
+
+void Assembler::pinsrq(XMMRegister dst, Register src, uint8_t imm8) {
+ DCHECK(IsEnabled(SSE4_1));
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0x3A);
+ emit(0x22);
+ emit_sse_operand(dst, src);
+ emit(imm8);
+}
+
+void Assembler::pinsrq(XMMRegister dst, Operand src, uint8_t imm8) {
+ DCHECK(IsEnabled(SSE4_1));
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0x3A);
+ emit(0x22);
+ emit_sse_operand(dst, src);
+ emit(imm8);
+}
+
+void Assembler::pinsrd(XMMRegister dst, Register src, uint8_t imm8) {
+ sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x22, imm8);
+}
+
+void Assembler::pinsrd(XMMRegister dst, Operand src, uint8_t imm8) {
+ sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x22);
+ emit(imm8);
+}
+
+void Assembler::pinsrb(XMMRegister dst, Register src, uint8_t imm8) {
+ sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x20, imm8);
+}
+
+void Assembler::pinsrb(XMMRegister dst, Operand src, uint8_t imm8) {
+ sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x20);
+ emit(imm8);
+}
+
+void Assembler::insertps(XMMRegister dst, XMMRegister src, byte imm8) {
+ DCHECK(is_uint8(imm8));
+ sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x21);
+ emit(imm8);
+}
+
+void Assembler::insertps(XMMRegister dst, Operand src, byte imm8) {
+ DCHECK(is_uint8(imm8));
+ sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x21);
+ emit(imm8);
+}
+
+void Assembler::movsd(Operand dst, XMMRegister src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0xF2); // double
+ emit_optional_rex_32(src, dst);
+ emit(0x0F);
+ emit(0x11); // store
+ emit_sse_operand(src, dst);
+}
+
+void Assembler::movsd(XMMRegister dst, XMMRegister src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0xF2); // double
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x10); // load
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movsd(XMMRegister dst, Operand src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0xF2); // double
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x10); // load
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movaps(XMMRegister dst, XMMRegister src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ if (src.low_bits() == 4) {
+ // Try to avoid an unnecessary SIB byte.
+ emit_optional_rex_32(src, dst);
+ emit(0x0F);
+ emit(0x29);
+ emit_sse_operand(src, dst);
+ } else {
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x28);
+ emit_sse_operand(dst, src);
+ }
+}
+
+void Assembler::shufps(XMMRegister dst, XMMRegister src, byte imm8) {
+ DCHECK(is_uint8(imm8));
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0xC6);
+ emit_sse_operand(dst, src);
+ emit(imm8);
+}
+
+void Assembler::movapd(XMMRegister dst, XMMRegister src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ if (src.low_bits() == 4) {
+ // Try to avoid an unnecessary SIB byte.
+ emit(0x66);
+ emit_optional_rex_32(src, dst);
+ emit(0x0F);
+ emit(0x29);
+ emit_sse_operand(src, dst);
+ } else {
+ emit(0x66);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x28);
+ emit_sse_operand(dst, src);
+ }
+}
+
+void Assembler::movupd(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x10);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movupd(Operand dst, XMMRegister src) {
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_optional_rex_32(src, dst);
+ emit(0x0F);
+ emit(0x11);
+ emit_sse_operand(src, dst);
+}
+
+void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x2E);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::ucomiss(XMMRegister dst, Operand src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x2E);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movss(XMMRegister dst, XMMRegister src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0xF3); // single
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x10); // load
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movss(XMMRegister dst, Operand src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0xF3); // single
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x10); // load
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movss(Operand src, XMMRegister dst) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0xF3); // single
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x11); // store
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movlps(XMMRegister dst, Operand src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x12);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movlps(Operand src, XMMRegister dst) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x13);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movhps(XMMRegister dst, Operand src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x16);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movhps(Operand src, XMMRegister dst) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x17);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cmpps(XMMRegister dst, XMMRegister src, int8_t cmp) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0xC2);
+ emit_sse_operand(dst, src);
+ emit(cmp);
+}
+
+void Assembler::cmpps(XMMRegister dst, Operand src, int8_t cmp) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0xC2);
+ emit_sse_operand(dst, src);
+ emit(cmp);
+}
+
+void Assembler::cmppd(XMMRegister dst, XMMRegister src, int8_t cmp) {
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0xC2);
+ emit_sse_operand(dst, src);
+ emit(cmp);
+}
+
+void Assembler::cmppd(XMMRegister dst, Operand src, int8_t cmp) {
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0xC2);
+ emit_sse_operand(dst, src);
+ emit(cmp);
+}
+
+void Assembler::cvttss2si(Register dst, Operand src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x2C);
+ emit_operand(dst, src);
+}
+
+void Assembler::cvttss2si(Register dst, XMMRegister src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x2C);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cvttsd2si(Register dst, Operand src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0xF2);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x2C);
+ emit_operand(dst, src);
+}
+
+void Assembler::cvttsd2si(Register dst, XMMRegister src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0xF2);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x2C);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cvttss2siq(Register dst, XMMRegister src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0x2C);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cvttss2siq(Register dst, Operand src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0x2C);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0xF2);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0x2C);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cvttsd2siq(Register dst, Operand src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0xF2);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0x2C);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cvttps2dq(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0x5B);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cvttps2dq(XMMRegister dst, XMMRegister src) {
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0x5B);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cvtlsi2sd(XMMRegister dst, Operand src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0xF2);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x2A);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cvtlsi2sd(XMMRegister dst, Register src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0xF2);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x2A);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cvtlsi2ss(XMMRegister dst, Operand src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x2A);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cvtlsi2ss(XMMRegister dst, Register src) {
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x2A);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cvtqsi2ss(XMMRegister dst, Operand src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0x2A);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cvtqsi2ss(XMMRegister dst, Register src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0x2A);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cvtqsi2sd(XMMRegister dst, Operand src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0xF2);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0x2A);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cvtqsi2sd(XMMRegister dst, Register src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0xF2);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0x2A);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x5A);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cvtss2sd(XMMRegister dst, Operand src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x5A);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cvtsd2si(Register dst, XMMRegister src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0xF2);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x2D);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cvtsd2siq(Register dst, XMMRegister src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0xF2);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0x2D);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::haddps(XMMRegister dst, XMMRegister src) {
+ DCHECK(IsEnabled(SSE3));
+ EnsureSpace ensure_space(this);
+ emit(0xF2);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x7C);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::haddps(XMMRegister dst, Operand src) {
+ DCHECK(IsEnabled(SSE3));
+ EnsureSpace ensure_space(this);
+ emit(0xF2);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x7C);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x2E);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::ucomisd(XMMRegister dst, Operand src) {
+ DCHECK(!IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x2E);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::cmpltsd(XMMRegister dst, XMMRegister src) {
+ EnsureSpace ensure_space(this);
+ emit(0xF2);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0xC2);
+ emit_sse_operand(dst, src);
+ emit(0x01); // LT == 1
+}
+
+void Assembler::roundss(XMMRegister dst, XMMRegister src, RoundingMode mode) {
+ DCHECK(!IsEnabled(AVX));
+ sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0A);
+ // Mask precision exception.
+ emit(static_cast<byte>(mode) | 0x8);
+}
+
+void Assembler::roundsd(XMMRegister dst, XMMRegister src, RoundingMode mode) {
+ DCHECK(!IsEnabled(AVX));
+ sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0B);
+ // Mask precision exception.
+ emit(static_cast<byte>(mode) | 0x8);
+}
+
+void Assembler::roundps(XMMRegister dst, XMMRegister src, RoundingMode mode) {
+ DCHECK(!IsEnabled(AVX));
+ sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x08);
+ // Mask precision exception.
+ emit(static_cast<byte>(mode) | 0x8);
+}
+
+void Assembler::roundpd(XMMRegister dst, XMMRegister src, RoundingMode mode) {
+ DCHECK(!IsEnabled(AVX));
+ sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x09);
+ // Mask precision exception.
+ emit(static_cast<byte>(mode) | 0x8);
+}
+
+void Assembler::movmskpd(Register dst, XMMRegister src) {
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x50);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movmskps(Register dst, XMMRegister src) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x50);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::pmovmskb(Register dst, XMMRegister src) {
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0xD7);
+ emit_sse_operand(dst, src);
+}
+
+// AVX instructions
+
+void Assembler::vmovddup(XMMRegister dst, XMMRegister src) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(dst, xmm0, src, kL128, kF2, k0F, kWIG);
+ emit(0x12);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::vmovddup(XMMRegister dst, Operand src) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(dst, xmm0, src, kL128, kF2, k0F, kWIG);
+ emit(0x12);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::vbroadcastss(XMMRegister dst, Operand src) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F38, kW0);
+ emit(0x18);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::fma_instr(byte op, XMMRegister dst, XMMRegister src1,
+ XMMRegister src2, VectorLength l, SIMDPrefix pp,
+ LeadingOpcode m, VexW w) {
+ DCHECK(IsEnabled(FMA3));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(dst, src1, src2, l, pp, m, w);
+ emit(op);
+ emit_sse_operand(dst, src2);
+}
+
+void Assembler::fma_instr(byte op, XMMRegister dst, XMMRegister src1,
+ Operand src2, VectorLength l, SIMDPrefix pp,
+ LeadingOpcode m, VexW w) {
+ DCHECK(IsEnabled(FMA3));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(dst, src1, src2, l, pp, m, w);
+ emit(op);
+ emit_sse_operand(dst, src2);
+}
+
+void Assembler::vmovd(XMMRegister dst, Register src) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ XMMRegister isrc = XMMRegister::from_code(src.code());
+ emit_vex_prefix(dst, xmm0, isrc, kL128, k66, k0F, kW0);
+ emit(0x6E);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::vmovd(XMMRegister dst, Operand src) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kW0);
+ emit(0x6E);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::vmovd(Register dst, XMMRegister src) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ XMMRegister idst = XMMRegister::from_code(dst.code());
+ emit_vex_prefix(src, xmm0, idst, kL128, k66, k0F, kW0);
+ emit(0x7E);
+ emit_sse_operand(src, dst);
+}
+
+void Assembler::vmovq(XMMRegister dst, Register src) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ XMMRegister isrc = XMMRegister::from_code(src.code());
+ emit_vex_prefix(dst, xmm0, isrc, kL128, k66, k0F, kW1);
+ emit(0x6E);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::vmovq(XMMRegister dst, Operand src) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kW1);
+ emit(0x6E);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::vmovq(Register dst, XMMRegister src) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ XMMRegister idst = XMMRegister::from_code(dst.code());
+ emit_vex_prefix(src, xmm0, idst, kL128, k66, k0F, kW1);
+ emit(0x7E);
+ emit_sse_operand(src, dst);
+}
+
+void Assembler::vmovdqu(XMMRegister dst, Operand src) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(dst, xmm0, src, kL128, kF3, k0F, kWIG);
+ emit(0x6F);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::vmovdqu(Operand dst, XMMRegister src) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(src, xmm0, dst, kL128, kF3, k0F, kWIG);
+ emit(0x7F);
+ emit_sse_operand(src, dst);
+}
+
+void Assembler::vmovlps(XMMRegister dst, XMMRegister src1, Operand src2) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG);
+ emit(0x12);
+ emit_sse_operand(dst, src2);
+}
+
+void Assembler::vmovlps(Operand dst, XMMRegister src) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(src, xmm0, dst, kL128, kNone, k0F, kWIG);
+ emit(0x13);
+ emit_sse_operand(src, dst);
+}
+
+void Assembler::vmovhps(XMMRegister dst, XMMRegister src1, Operand src2) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG);
+ emit(0x16);
+ emit_sse_operand(dst, src2);
+}
+
+void Assembler::vmovhps(Operand dst, XMMRegister src) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(src, xmm0, dst, kL128, kNone, k0F, kWIG);
+ emit(0x17);
+ emit_sse_operand(src, dst);
+}
+
+void Assembler::vinstr(byte op, XMMRegister dst, XMMRegister src1,
+ XMMRegister src2, SIMDPrefix pp, LeadingOpcode m,
+ VexW w) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(dst, src1, src2, kLIG, pp, m, w);
+ emit(op);
+ emit_sse_operand(dst, src2);
+}
+
+void Assembler::vinstr(byte op, XMMRegister dst, XMMRegister src1, Operand src2,
+ SIMDPrefix pp, LeadingOpcode m, VexW w) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(dst, src1, src2, kLIG, pp, m, w);
+ emit(op);
+ emit_sse_operand(dst, src2);
+}
+
+void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1,
+ XMMRegister src2) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG);
+ emit(op);
+ emit_sse_operand(dst, src2);
+}
+
+void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1, Operand src2) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG);
+ emit(op);
+ emit_sse_operand(dst, src2);
+}
+
+void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1,
+ XMMRegister src2, byte imm8) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG);
+ emit(op);
+ emit_sse_operand(dst, src2);
+ emit(imm8);
+}
+
+void Assembler::vpd(byte op, XMMRegister dst, XMMRegister src1,
+ XMMRegister src2) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(dst, src1, src2, kL128, k66, k0F, kWIG);
+ emit(op);
+ emit_sse_operand(dst, src2);
+}
+
+void Assembler::vpd(byte op, XMMRegister dst, XMMRegister src1, Operand src2) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(dst, src1, src2, kL128, k66, k0F, kWIG);
+ emit(op);
+ emit_sse_operand(dst, src2);
+}
+
+void Assembler::vucomiss(XMMRegister dst, XMMRegister src) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(dst, xmm0, src, kLIG, kNone, k0F, kWIG);
+ emit(0x2E);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::vucomiss(XMMRegister dst, Operand src) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(dst, xmm0, src, kLIG, kNone, k0F, kWIG);
+ emit(0x2E);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::vpmovmskb(Register dst, XMMRegister src) {
+ XMMRegister idst = XMMRegister::from_code(dst.code());
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(idst, xmm0, src, kL128, k66, k0F, kWIG);
+ emit(0xD7);
+ emit_sse_operand(idst, src);
+}
+
+void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1,
+ XMMRegister src2) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG);
+ emit(op);
+ emit_sse_operand(dst, src2);
+}
+
+void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1, Operand src2) {
+ DCHECK(IsEnabled(AVX));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG);
+ emit(op);
+ emit_sse_operand(dst, src2);
+}
+
+void Assembler::bmi1q(byte op, Register reg, Register vreg, Register rm) {
+ DCHECK(IsEnabled(BMI1));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW1);
+ emit(op);
+ emit_modrm(reg, rm);
+}
+
+void Assembler::bmi1q(byte op, Register reg, Register vreg, Operand rm) {
+ DCHECK(IsEnabled(BMI1));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW1);
+ emit(op);
+ emit_operand(reg, rm);
+}
+
+void Assembler::bmi1l(byte op, Register reg, Register vreg, Register rm) {
+ DCHECK(IsEnabled(BMI1));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW0);
+ emit(op);
+ emit_modrm(reg, rm);
+}
+
+void Assembler::bmi1l(byte op, Register reg, Register vreg, Operand rm) {
+ DCHECK(IsEnabled(BMI1));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW0);
+ emit(op);
+ emit_operand(reg, rm);
+}
+
+void Assembler::tzcntq(Register dst, Register src) {
+ DCHECK(IsEnabled(BMI1));
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0xBC);
+ emit_modrm(dst, src);
+}
+
+void Assembler::tzcntq(Register dst, Operand src) {
+ DCHECK(IsEnabled(BMI1));
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0xBC);
+ emit_operand(dst, src);
+}
+
+void Assembler::tzcntl(Register dst, Register src) {
+ DCHECK(IsEnabled(BMI1));
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0xBC);
+ emit_modrm(dst, src);
+}
+
+void Assembler::tzcntl(Register dst, Operand src) {
+ DCHECK(IsEnabled(BMI1));
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0xBC);
+ emit_operand(dst, src);
+}
+
+void Assembler::lzcntq(Register dst, Register src) {
+ DCHECK(IsEnabled(LZCNT));
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0xBD);
+ emit_modrm(dst, src);
+}
+
+void Assembler::lzcntq(Register dst, Operand src) {
+ DCHECK(IsEnabled(LZCNT));
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0xBD);
+ emit_operand(dst, src);
+}
+
+void Assembler::lzcntl(Register dst, Register src) {
+ DCHECK(IsEnabled(LZCNT));
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0xBD);
+ emit_modrm(dst, src);
+}
+
+void Assembler::lzcntl(Register dst, Operand src) {
+ DCHECK(IsEnabled(LZCNT));
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0xBD);
+ emit_operand(dst, src);
+}
+
+void Assembler::popcntq(Register dst, Register src) {
+ DCHECK(IsEnabled(POPCNT));
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0xB8);
+ emit_modrm(dst, src);
+}
+
+void Assembler::popcntq(Register dst, Operand src) {
+ DCHECK(IsEnabled(POPCNT));
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_rex_64(dst, src);
+ emit(0x0F);
+ emit(0xB8);
+ emit_operand(dst, src);
+}
+
+void Assembler::popcntl(Register dst, Register src) {
+ DCHECK(IsEnabled(POPCNT));
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0xB8);
+ emit_modrm(dst, src);
+}
+
+void Assembler::popcntl(Register dst, Operand src) {
+ DCHECK(IsEnabled(POPCNT));
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0xB8);
+ emit_operand(dst, src);
+}
+
+void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg,
+ Register rm) {
+ DCHECK(IsEnabled(BMI2));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1);
+ emit(op);
+ emit_modrm(reg, rm);
+}
+
+void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg,
+ Operand rm) {
+ DCHECK(IsEnabled(BMI2));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1);
+ emit(op);
+ emit_operand(reg, rm);
+}
+
+void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg,
+ Register rm) {
+ DCHECK(IsEnabled(BMI2));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0);
+ emit(op);
+ emit_modrm(reg, rm);
+}
+
+void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg,
+ Operand rm) {
+ DCHECK(IsEnabled(BMI2));
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0);
+ emit(op);
+ emit_operand(reg, rm);
+}
+
+void Assembler::rorxq(Register dst, Register src, byte imm8) {
+ DCHECK(IsEnabled(BMI2));
+ DCHECK(is_uint8(imm8));
+ Register vreg = Register::from_code(0); // VEX.vvvv unused
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1);
+ emit(0xF0);
+ emit_modrm(dst, src);
+ emit(imm8);
+}
+
+void Assembler::rorxq(Register dst, Operand src, byte imm8) {
+ DCHECK(IsEnabled(BMI2));
+ DCHECK(is_uint8(imm8));
+ Register vreg = Register::from_code(0); // VEX.vvvv unused
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1);
+ emit(0xF0);
+ emit_operand(dst, src);
+ emit(imm8);
+}
+
+void Assembler::rorxl(Register dst, Register src, byte imm8) {
+ DCHECK(IsEnabled(BMI2));
+ DCHECK(is_uint8(imm8));
+ Register vreg = Register::from_code(0); // VEX.vvvv unused
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0);
+ emit(0xF0);
+ emit_modrm(dst, src);
+ emit(imm8);
+}
+
+void Assembler::rorxl(Register dst, Operand src, byte imm8) {
+ DCHECK(IsEnabled(BMI2));
+ DCHECK(is_uint8(imm8));
+ Register vreg = Register::from_code(0); // VEX.vvvv unused
+ EnsureSpace ensure_space(this);
+ emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0);
+ emit(0xF0);
+ emit_operand(dst, src);
+ emit(imm8);
+}
+
+void Assembler::pause() {
+ emit(0xF3);
+ emit(0x90);
+}
+
+void Assembler::movups(XMMRegister dst, XMMRegister src) {
+ EnsureSpace ensure_space(this);
+ if (src.low_bits() == 4) {
+ // Try to avoid an unnecessary SIB byte.
+ emit_optional_rex_32(src, dst);
+ emit(0x0F);
+ emit(0x11);
+ emit_sse_operand(src, dst);
+ } else {
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x10);
+ emit_sse_operand(dst, src);
+ }
+}
+
+void Assembler::movups(XMMRegister dst, Operand src) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x10);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movups(Operand dst, XMMRegister src) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(src, dst);
+ emit(0x0F);
+ emit(0x11);
+ emit_sse_operand(src, dst);
+}
+
+void Assembler::sse_instr(XMMRegister dst, XMMRegister src, byte escape,
+ byte opcode) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(dst, src);
+ emit(escape);
+ emit(opcode);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::sse_instr(XMMRegister dst, Operand src, byte escape,
+ byte opcode) {
+ EnsureSpace ensure_space(this);
+ emit_optional_rex_32(dst, src);
+ emit(escape);
+ emit(opcode);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::sse2_instr(XMMRegister dst, XMMRegister src, byte prefix,
+ byte escape, byte opcode) {
+ EnsureSpace ensure_space(this);
+ emit(prefix);
+ emit_optional_rex_32(dst, src);
+ emit(escape);
+ emit(opcode);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::sse2_instr(XMMRegister dst, Operand src, byte prefix,
+ byte escape, byte opcode) {
+ EnsureSpace ensure_space(this);
+ emit(prefix);
+ emit_optional_rex_32(dst, src);
+ emit(escape);
+ emit(opcode);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::ssse3_instr(XMMRegister dst, XMMRegister src, byte prefix,
+ byte escape1, byte escape2, byte opcode) {
+ DCHECK(IsEnabled(SSSE3));
+ EnsureSpace ensure_space(this);
+ emit(prefix);
+ emit_optional_rex_32(dst, src);
+ emit(escape1);
+ emit(escape2);
+ emit(opcode);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::ssse3_instr(XMMRegister dst, Operand src, byte prefix,
+ byte escape1, byte escape2, byte opcode) {
+ DCHECK(IsEnabled(SSSE3));
+ EnsureSpace ensure_space(this);
+ emit(prefix);
+ emit_optional_rex_32(dst, src);
+ emit(escape1);
+ emit(escape2);
+ emit(opcode);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::sse4_instr(XMMRegister dst, Register src, byte prefix,
+ byte escape1, byte escape2, byte opcode,
+ int8_t imm8) {
+ DCHECK(is_uint8(imm8));
+ DCHECK(IsEnabled(SSE4_1));
+ EnsureSpace ensure_space(this);
+ emit(prefix);
+ emit_optional_rex_32(dst, src);
+ emit(escape1);
+ emit(escape2);
+ emit(opcode);
+ emit_sse_operand(dst, src);
+ emit(imm8);
+}
+
+void Assembler::sse4_instr(XMMRegister dst, XMMRegister src, byte prefix,
+ byte escape1, byte escape2, byte opcode) {
+ DCHECK(IsEnabled(SSE4_1));
+ EnsureSpace ensure_space(this);
+ emit(prefix);
+ emit_optional_rex_32(dst, src);
+ emit(escape1);
+ emit(escape2);
+ emit(opcode);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::sse4_instr(XMMRegister dst, Operand src, byte prefix,
+ byte escape1, byte escape2, byte opcode) {
+ DCHECK(IsEnabled(SSE4_1));
+ EnsureSpace ensure_space(this);
+ emit(prefix);
+ emit_optional_rex_32(dst, src);
+ emit(escape1);
+ emit(escape2);
+ emit(opcode);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::sse4_instr(Register dst, XMMRegister src, byte prefix,
+ byte escape1, byte escape2, byte opcode,
+ int8_t imm8) {
+ DCHECK(is_uint8(imm8));
+ DCHECK(IsEnabled(SSE4_1));
+ EnsureSpace ensure_space(this);
+ emit(prefix);
+ emit_optional_rex_32(src, dst);
+ emit(escape1);
+ emit(escape2);
+ emit(opcode);
+ emit_sse_operand(src, dst);
+ emit(imm8);
+}
+
+void Assembler::sse4_instr(Operand dst, XMMRegister src, byte prefix,
+ byte escape1, byte escape2, byte opcode,
+ int8_t imm8) {
+ DCHECK(is_uint8(imm8));
+ DCHECK(IsEnabled(SSE4_1));
+ EnsureSpace ensure_space(this);
+ emit(prefix);
+ emit_optional_rex_32(src, dst);
+ emit(escape1);
+ emit(escape2);
+ emit(opcode);
+ emit_sse_operand(src, dst);
+ emit(imm8);
+}
+
+void Assembler::sse4_2_instr(XMMRegister dst, XMMRegister src, byte prefix,
+ byte escape1, byte escape2, byte opcode) {
+ DCHECK(IsEnabled(SSE4_2));
+ EnsureSpace ensure_space(this);
+ emit(prefix);
+ emit_optional_rex_32(dst, src);
+ emit(escape1);
+ emit(escape2);
+ emit(opcode);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::sse4_2_instr(XMMRegister dst, Operand src, byte prefix,
+ byte escape1, byte escape2, byte opcode) {
+ DCHECK(IsEnabled(SSE4_2));
+ EnsureSpace ensure_space(this);
+ emit(prefix);
+ emit_optional_rex_32(dst, src);
+ emit(escape1);
+ emit(escape2);
+ emit(opcode);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::lddqu(XMMRegister dst, Operand src) {
+ DCHECK(IsEnabled(SSE3));
+ EnsureSpace ensure_space(this);
+ emit(0xF2);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0xF0);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movddup(XMMRegister dst, XMMRegister src) {
+ DCHECK(IsEnabled(SSE3));
+ EnsureSpace ensure_space(this);
+ emit(0xF2);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x12);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::movddup(XMMRegister dst, Operand src) {
+ DCHECK(IsEnabled(SSE3));
+ EnsureSpace ensure_space(this);
+ emit(0xF2);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x12);
+ emit_sse_operand(dst, src);
+}
+
+void Assembler::psrldq(XMMRegister dst, uint8_t shift) {
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_optional_rex_32(dst);
+ emit(0x0F);
+ emit(0x73);
+ emit_sse_operand(dst);
+ emit(shift);
+}
+
+void Assembler::pshufhw(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x70);
+ emit_sse_operand(dst, src);
+ emit(shuffle);
+}
+
+void Assembler::pshufhw(XMMRegister dst, Operand src, uint8_t shuffle) {
+ EnsureSpace ensure_space(this);
+ emit(0xF3);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x70);
+ emit_sse_operand(dst, src);
+ emit(shuffle);
+}
+
+void Assembler::pshuflw(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
+ EnsureSpace ensure_space(this);
+ emit(0xF2);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x70);
+ emit_sse_operand(dst, src);
+ emit(shuffle);
+}
+
+void Assembler::pshuflw(XMMRegister dst, Operand src, uint8_t shuffle) {
+ EnsureSpace ensure_space(this);
+ emit(0xF2);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x70);
+ emit_sse_operand(dst, src);
+ emit(shuffle);
+}
+
+void Assembler::pshufd(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x70);
+ emit_sse_operand(dst, src);
+ emit(shuffle);
+}
+
+void Assembler::pshufd(XMMRegister dst, Operand src, uint8_t shuffle) {
+ EnsureSpace ensure_space(this);
+ emit(0x66);
+ emit_optional_rex_32(dst, src);
+ emit(0x0F);
+ emit(0x70);
+ emit_sse_operand(dst, src);
+ emit(shuffle);
+}
+
+void Assembler::emit_sse_operand(XMMRegister reg, Operand adr) {
+ Register ireg = Register::from_code(reg.code());
+ emit_operand(ireg, adr);
+}
+
+void Assembler::emit_sse_operand(Register reg, Operand adr) {
+ emit_operand(reg, adr);
+}
+
+void Assembler::emit_sse_operand(XMMRegister dst, XMMRegister src) {
+ emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
+}
+
+void Assembler::emit_sse_operand(XMMRegister dst, Register src) {
+ emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
+}
+
+void Assembler::emit_sse_operand(Register dst, XMMRegister src) {
+ emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
+}
+
+void Assembler::emit_sse_operand(XMMRegister dst) {
+ emit(0xD8 | dst.low_bits());
+}
+
+void Assembler::db(uint8_t data) {
+ EnsureSpace ensure_space(this);
+ emit(data);
+}
+
+void Assembler::dd(uint32_t data) {
+ EnsureSpace ensure_space(this);
+ emitl(data);
+}
+
+void Assembler::dq(uint64_t data) {
+ EnsureSpace ensure_space(this);
+ emitq(data);
+}
+
+void Assembler::dq(Label* label) {
+ EnsureSpace ensure_space(this);
+ if (label->is_bound()) {
+ internal_reference_positions_.push_back(pc_offset());
+ emit(Immediate64(reinterpret_cast<Address>(buffer_start_) + label->pos(),
+ RelocInfo::INTERNAL_REFERENCE));
+ } else {
+ RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE);
+ emitl(0); // Zero for the first 32bit marks it as 64bit absolute address.
+ if (label->is_linked()) {
+ emitl(label->pos());
+ label->link_to(pc_offset() - sizeof(int32_t));
+ } else {
+ DCHECK(label->is_unused());
+ int32_t current = pc_offset();
+ emitl(current);
+ label->link_to(current);
+ }
+ }
+}
+
+// Relocation information implementations.
+
+void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) {
+ if (!ShouldRecordRelocInfo(rmode)) return;
+ RelocInfo rinfo(reinterpret_cast<Address>(pc_), rmode, data, Code());
+ reloc_info_writer.Write(&rinfo);
+}
+
+const int RelocInfo::kApplyMask =
+ RelocInfo::ModeMask(RelocInfo::CODE_TARGET) |
+ RelocInfo::ModeMask(RelocInfo::RUNTIME_ENTRY) |
+ RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE) |
+ RelocInfo::ModeMask(RelocInfo::WASM_CALL);
+
+bool RelocInfo::IsCodedSpecially() {
+ // The deserializer needs to know whether a pointer is specially coded. Being
+ // specially coded on x64 means that it is a relative 32 bit address, as used
+ // by branch instructions.
+ return (1 << rmode_) & kApplyMask;
+}
+
+bool RelocInfo::IsInConstantPool() { return false; }
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_X64
diff --git a/src/codegen/x64/assembler-x64.h b/src/codegen/x64/assembler-x64.h
new file mode 100644
index 0000000..e05eaa9
--- /dev/null
+++ b/src/codegen/x64/assembler-x64.h
@@ -0,0 +1,2372 @@
+// Copyright (c) 1994-2006 Sun Microsystems Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// - Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// - Redistribution in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// - Neither the name of Sun Microsystems or the names of contributors may
+// be used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// The original source code covered by the above license above has been
+// modified significantly by Google Inc.
+// Copyright 2012 the V8 project authors. All rights reserved.
+
+// A lightweight X64 Assembler.
+
+#ifndef V8_CODEGEN_X64_ASSEMBLER_X64_H_
+#define V8_CODEGEN_X64_ASSEMBLER_X64_H_
+
+#include <deque>
+#include <map>
+#include <memory>
+#include <vector>
+
+#include "src/codegen/assembler.h"
+#include "src/codegen/label.h"
+#include "src/codegen/x64/constants-x64.h"
+#include "src/codegen/x64/fma-instr.h"
+#include "src/codegen/x64/register-x64.h"
+#include "src/codegen/x64/sse-instr.h"
+#include "src/objects/smi.h"
+#if defined(V8_OS_WIN_X64)
+#include "src/diagnostics/unwinding-info-win64.h"
+#endif
+
+namespace v8 {
+namespace internal {
+
+class SafepointTableBuilder;
+
+// Utility functions
+
+enum Condition {
+ // any value < 0 is considered no_condition
+ no_condition = -1,
+
+ overflow = 0,
+ no_overflow = 1,
+ below = 2,
+ above_equal = 3,
+ equal = 4,
+ not_equal = 5,
+ below_equal = 6,
+ above = 7,
+ negative = 8,
+ positive = 9,
+ parity_even = 10,
+ parity_odd = 11,
+ less = 12,
+ greater_equal = 13,
+ less_equal = 14,
+ greater = 15,
+
+ // Fake conditions that are handled by the
+ // opcodes using them.
+ always = 16,
+ never = 17,
+ // aliases
+ carry = below,
+ not_carry = above_equal,
+ zero = equal,
+ not_zero = not_equal,
+ sign = negative,
+ not_sign = positive,
+ last_condition = greater
+};
+
+// Returns the equivalent of !cc.
+// Negation of the default no_condition (-1) results in a non-default
+// no_condition value (-2). As long as tests for no_condition check
+// for condition < 0, this will work as expected.
+inline Condition NegateCondition(Condition cc) {
+ return static_cast<Condition>(cc ^ 1);
+}
+
+enum RoundingMode {
+ kRoundToNearest = 0x0,
+ kRoundDown = 0x1,
+ kRoundUp = 0x2,
+ kRoundToZero = 0x3
+};
+
+// -----------------------------------------------------------------------------
+// Machine instruction Immediates
+
+class Immediate {
+ public:
+ explicit constexpr Immediate(int32_t value) : value_(value) {}
+ explicit constexpr Immediate(int32_t value, RelocInfo::Mode rmode)
+ : value_(value), rmode_(rmode) {}
+ explicit Immediate(Smi value)
+ : value_(static_cast<int32_t>(static_cast<intptr_t>(value.ptr()))) {
+ DCHECK(SmiValuesAre31Bits()); // Only available for 31-bit SMI.
+ }
+
+ private:
+ const int32_t value_;
+ const RelocInfo::Mode rmode_ = RelocInfo::NONE;
+
+ friend class Assembler;
+};
+ASSERT_TRIVIALLY_COPYABLE(Immediate);
+static_assert(sizeof(Immediate) <= kSystemPointerSize,
+ "Immediate must be small enough to pass it by value");
+
+class Immediate64 {
+ public:
+ explicit constexpr Immediate64(int64_t value) : value_(value) {}
+ explicit constexpr Immediate64(int64_t value, RelocInfo::Mode rmode)
+ : value_(value), rmode_(rmode) {}
+ explicit constexpr Immediate64(Address value, RelocInfo::Mode rmode)
+ : value_(static_cast<int64_t>(value)), rmode_(rmode) {}
+
+ private:
+ const int64_t value_;
+ const RelocInfo::Mode rmode_ = RelocInfo::NONE;
+
+ friend class Assembler;
+};
+
+// -----------------------------------------------------------------------------
+// Machine instruction Operands
+
+enum ScaleFactor : int8_t {
+ times_1 = 0,
+ times_2 = 1,
+ times_4 = 2,
+ times_8 = 3,
+ times_int_size = times_4,
+
+ times_half_system_pointer_size = times_4,
+ times_system_pointer_size = times_8,
+ times_tagged_size = (kTaggedSize == 8) ? times_8 : times_4,
+};
+
+class V8_EXPORT_PRIVATE Operand {
+ public:
+ struct Data {
+ byte rex = 0;
+ byte buf[9];
+ byte len = 1; // number of bytes of buf_ in use.
+ int8_t addend; // for rip + offset + addend.
+ };
+
+ // [base + disp/r]
+ V8_INLINE Operand(Register base, int32_t disp) {
+ if (base == rsp || base == r12) {
+ // SIB byte is needed to encode (rsp + offset) or (r12 + offset).
+ set_sib(times_1, rsp, base);
+ }
+
+ if (disp == 0 && base != rbp && base != r13) {
+ set_modrm(0, base);
+ } else if (is_int8(disp)) {
+ set_modrm(1, base);
+ set_disp8(disp);
+ } else {
+ set_modrm(2, base);
+ set_disp32(disp);
+ }
+ }
+
+ // [base + index*scale + disp/r]
+ V8_INLINE Operand(Register base, Register index, ScaleFactor scale,
+ int32_t disp) {
+ DCHECK(index != rsp);
+ set_sib(scale, index, base);
+ if (disp == 0 && base != rbp && base != r13) {
+ // This call to set_modrm doesn't overwrite the REX.B (or REX.X) bits
+ // possibly set by set_sib.
+ set_modrm(0, rsp);
+ } else if (is_int8(disp)) {
+ set_modrm(1, rsp);
+ set_disp8(disp);
+ } else {
+ set_modrm(2, rsp);
+ set_disp32(disp);
+ }
+ }
+
+ // [index*scale + disp/r]
+ V8_INLINE Operand(Register index, ScaleFactor scale, int32_t disp) {
+ DCHECK(index != rsp);
+ set_modrm(0, rsp);
+ set_sib(scale, index, rbp);
+ set_disp32(disp);
+ }
+
+ // Offset from existing memory operand.
+ // Offset is added to existing displacement as 32-bit signed values and
+ // this must not overflow.
+ Operand(Operand base, int32_t offset);
+
+ // [rip + disp/r]
+ V8_INLINE explicit Operand(Label* label, int addend = 0) {
+ data_.addend = addend;
+ DCHECK_NOT_NULL(label);
+ DCHECK(addend == 0 || (is_int8(addend) && label->is_bound()));
+ set_modrm(0, rbp);
+ set_disp64(reinterpret_cast<intptr_t>(label));
+ }
+
+ Operand(const Operand&) V8_NOEXCEPT = default;
+
+ const Data& data() const { return data_; }
+
+ // Checks whether either base or index register is the given register.
+ // Does not check the "reg" part of the Operand.
+ bool AddressUsesRegister(Register reg) const;
+
+ private:
+ V8_INLINE void set_modrm(int mod, Register rm_reg) {
+ DCHECK(is_uint2(mod));
+ data_.buf[0] = mod << 6 | rm_reg.low_bits();
+ // Set REX.B to the high bit of rm.code().
+ data_.rex |= rm_reg.high_bit();
+ }
+
+ V8_INLINE void set_sib(ScaleFactor scale, Register index, Register base) {
+ DCHECK_EQ(data_.len, 1);
+ DCHECK(is_uint2(scale));
+ // Use SIB with no index register only for base rsp or r12. Otherwise we
+ // would skip the SIB byte entirely.
+ DCHECK(index != rsp || base == rsp || base == r12);
+ data_.buf[1] = (scale << 6) | (index.low_bits() << 3) | base.low_bits();
+ data_.rex |= index.high_bit() << 1 | base.high_bit();
+ data_.len = 2;
+ }
+
+ V8_INLINE void set_disp8(int disp) {
+ DCHECK(is_int8(disp));
+ DCHECK(data_.len == 1 || data_.len == 2);
+ int8_t* p = reinterpret_cast<int8_t*>(&data_.buf[data_.len]);
+ *p = disp;
+ data_.len += sizeof(int8_t);
+ }
+
+ V8_INLINE void set_disp32(int disp) {
+ DCHECK(data_.len == 1 || data_.len == 2);
+ Address p = reinterpret_cast<Address>(&data_.buf[data_.len]);
+ WriteUnalignedValue(p, disp);
+ data_.len += sizeof(int32_t);
+ }
+
+ V8_INLINE void set_disp64(int64_t disp) {
+ DCHECK_EQ(1, data_.len);
+ Address p = reinterpret_cast<Address>(&data_.buf[data_.len]);
+ WriteUnalignedValue(p, disp);
+ data_.len += sizeof(disp);
+ }
+
+ Data data_;
+};
+ASSERT_TRIVIALLY_COPYABLE(Operand);
+static_assert(sizeof(Operand) <= 2 * kSystemPointerSize,
+ "Operand must be small enough to pass it by value");
+
+#define ASSEMBLER_INSTRUCTION_LIST(V) \
+ V(add) \
+ V(and) \
+ V(cmp) \
+ V(cmpxchg) \
+ V(dec) \
+ V(idiv) \
+ V(div) \
+ V(imul) \
+ V(inc) \
+ V(lea) \
+ V(mov) \
+ V(movzxb) \
+ V(movzxw) \
+ V(not) \
+ V(or) \
+ V(repmovs) \
+ V(sbb) \
+ V(sub) \
+ V(test) \
+ V(xchg) \
+ V(xor)
+
+// Shift instructions on operands/registers with kInt32Size and kInt64Size.
+#define SHIFT_INSTRUCTION_LIST(V) \
+ V(rol, 0x0) \
+ V(ror, 0x1) \
+ V(rcl, 0x2) \
+ V(rcr, 0x3) \
+ V(shl, 0x4) \
+ V(shr, 0x5) \
+ V(sar, 0x7)
+
+// Partial Constant Pool
+// Different from complete constant pool (like arm does), partial constant pool
+// only takes effects for shareable constants in order to reduce code size.
+// Partial constant pool does not emit constant pool entries at the end of each
+// code object. Instead, it keeps the first shareable constant inlined in the
+// instructions and uses rip-relative memory loadings for the same constants in
+// subsequent instructions. These rip-relative memory loadings will target at
+// the position of the first inlined constant. For example:
+//
+// REX.W movq r10,0x7f9f75a32c20 ; 10 bytes
+// …
+// REX.W movq r10,0x7f9f75a32c20 ; 10 bytes
+// …
+//
+// turns into
+//
+// REX.W movq r10,0x7f9f75a32c20 ; 10 bytes
+// …
+// REX.W movq r10,[rip+0xffffff96] ; 7 bytes
+// …
+
+class ConstPool {
+ public:
+ explicit ConstPool(Assembler* assm) : assm_(assm) {}
+ // Returns true when partial constant pool is valid for this entry.
+ bool TryRecordEntry(intptr_t data, RelocInfo::Mode mode);
+ bool IsEmpty() const { return entries_.empty(); }
+
+ void PatchEntries();
+ // Discard any pending pool entries.
+ void Clear();
+
+ private:
+ // Adds a shared entry to entries_. Returns true if this is not the first time
+ // we add this entry, false otherwise.
+ bool AddSharedEntry(uint64_t data, int offset);
+
+ // Check if the instruction is a rip-relative move.
+ bool IsMoveRipRelative(Address instr);
+
+ Assembler* assm_;
+
+ // Values, pc offsets of entries.
+ using EntryMap = std::multimap<uint64_t, int>;
+ EntryMap entries_;
+
+ // Number of bytes taken up by the displacement of rip-relative addressing.
+ static constexpr int kRipRelativeDispSize = 4; // 32-bit displacement.
+ // Distance between the address of the displacement in the rip-relative move
+ // instruction and the head address of the instruction.
+ static constexpr int kMoveRipRelativeDispOffset =
+ 3; // REX Opcode ModRM Displacement
+ // Distance between the address of the imm64 in the 'movq reg, imm64'
+ // instruction and the head address of the instruction.
+ static constexpr int kMoveImm64Offset = 2; // REX Opcode imm64
+ // A mask for rip-relative move instruction.
+ static constexpr uint32_t kMoveRipRelativeMask = 0x00C7FFFB;
+ // The bits for a rip-relative move instruction after mask.
+ static constexpr uint32_t kMoveRipRelativeInstr = 0x00058B48;
+};
+
+class V8_EXPORT_PRIVATE Assembler : public AssemblerBase {
+ private:
+ // We check before assembling an instruction that there is sufficient
+ // space to write an instruction and its relocation information.
+ // The relocation writer's position must be kGap bytes above the end of
+ // the generated instructions. This leaves enough space for the
+ // longest possible x64 instruction, 15 bytes, and the longest possible
+ // relocation information encoding, RelocInfoWriter::kMaxLength == 16.
+ // (There is a 15 byte limit on x64 instruction length that rules out some
+ // otherwise valid instructions.)
+ // This allows for a single, fast space check per instruction.
+ static constexpr int kGap = 32;
+ STATIC_ASSERT(AssemblerBase::kMinimalBufferSize >= 2 * kGap);
+
+ public:
+ // Create an assembler. Instructions and relocation information are emitted
+ // into a buffer, with the instructions starting from the beginning and the
+ // relocation information starting from the end of the buffer. See CodeDesc
+ // for a detailed comment on the layout (globals.h).
+ //
+ // If the provided buffer is nullptr, the assembler allocates and grows its
+ // own buffer. Otherwise it takes ownership of the provided buffer.
+ explicit Assembler(const AssemblerOptions&,
+ std::unique_ptr<AssemblerBuffer> = {});
+ ~Assembler() override = default;
+
+ // GetCode emits any pending (non-emitted) code and fills the descriptor desc.
+ static constexpr int kNoHandlerTable = 0;
+ static constexpr SafepointTableBuilder* kNoSafepointTable = nullptr;
+ void GetCode(Isolate* isolate, CodeDesc* desc,
+ SafepointTableBuilder* safepoint_table_builder,
+ int handler_table_offset);
+
+ // Convenience wrapper for code without safepoint or handler tables.
+ void GetCode(Isolate* isolate, CodeDesc* desc) {
+ GetCode(isolate, desc, kNoSafepointTable, kNoHandlerTable);
+ }
+
+ void FinalizeJumpOptimizationInfo();
+
+ // Unused on this architecture.
+ void MaybeEmitOutOfLineConstantPool() {}
+
+ // Read/Modify the code target in the relative branch/call instruction at pc.
+ // On the x64 architecture, we use relative jumps with a 32-bit displacement
+ // to jump to other Code objects in the Code space in the heap.
+ // Jumps to C functions are done indirectly through a 64-bit register holding
+ // the absolute address of the target.
+ // These functions convert between absolute Addresses of Code objects and
+ // the relative displacements stored in the code.
+ // The isolate argument is unused (and may be nullptr) when skipping flushing.
+ static inline Address target_address_at(Address pc, Address constant_pool);
+ static inline void set_target_address_at(
+ Address pc, Address constant_pool, Address target,
+ ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED);
+
+ // This sets the branch destination (which is in the instruction on x64).
+ // This is for calls and branches within generated code.
+ inline static void deserialization_set_special_target_at(
+ Address instruction_payload, Code code, Address target);
+
+ // Get the size of the special target encoded at 'instruction_payload'.
+ inline static int deserialization_special_target_size(
+ Address instruction_payload);
+
+ // This sets the internal reference at the pc.
+ inline static void deserialization_set_target_internal_reference_at(
+ Address pc, Address target,
+ RelocInfo::Mode mode = RelocInfo::INTERNAL_REFERENCE);
+
+ inline Handle<Code> code_target_object_handle_at(Address pc);
+ inline Handle<HeapObject> compressed_embedded_object_handle_at(Address pc);
+ inline Address runtime_entry_at(Address pc);
+
+ // Number of bytes taken up by the branch target in the code.
+ static constexpr int kSpecialTargetSize = 4; // 32-bit displacement.
+
+ // One byte opcode for test eax,0xXXXXXXXX.
+ static constexpr byte kTestEaxByte = 0xA9;
+ // One byte opcode for test al, 0xXX.
+ static constexpr byte kTestAlByte = 0xA8;
+ // One byte opcode for nop.
+ static constexpr byte kNopByte = 0x90;
+
+ // One byte prefix for a short conditional jump.
+ static constexpr byte kJccShortPrefix = 0x70;
+ static constexpr byte kJncShortOpcode = kJccShortPrefix | not_carry;
+ static constexpr byte kJcShortOpcode = kJccShortPrefix | carry;
+ static constexpr byte kJnzShortOpcode = kJccShortPrefix | not_zero;
+ static constexpr byte kJzShortOpcode = kJccShortPrefix | zero;
+
+ // VEX prefix encodings.
+ enum SIMDPrefix { kNone = 0x0, k66 = 0x1, kF3 = 0x2, kF2 = 0x3 };
+ enum VectorLength { kL128 = 0x0, kL256 = 0x4, kLIG = kL128, kLZ = kL128 };
+ enum VexW { kW0 = 0x0, kW1 = 0x80, kWIG = kW0 };
+ enum LeadingOpcode { k0F = 0x1, k0F38 = 0x2, k0F3A = 0x3 };
+
+ // ---------------------------------------------------------------------------
+ // Code generation
+ //
+ // Function names correspond one-to-one to x64 instruction mnemonics.
+ // Unless specified otherwise, instructions operate on 64-bit operands.
+ //
+ // If we need versions of an assembly instruction that operate on different
+ // width arguments, we add a single-letter suffix specifying the width.
+ // This is done for the following instructions: mov, cmp, inc, dec,
+ // add, sub, and test.
+ // There are no versions of these instructions without the suffix.
+ // - Instructions on 8-bit (byte) operands/registers have a trailing 'b'.
+ // - Instructions on 16-bit (word) operands/registers have a trailing 'w'.
+ // - Instructions on 32-bit (doubleword) operands/registers use 'l'.
+ // - Instructions on 64-bit (quadword) operands/registers use 'q'.
+ // - Instructions on operands/registers with pointer size use 'p'.
+
+#define DECLARE_INSTRUCTION(instruction) \
+ template <class P1> \
+ void instruction##_tagged(P1 p1) { \
+ emit_##instruction(p1, kTaggedSize); \
+ } \
+ \
+ template <class P1> \
+ void instruction##l(P1 p1) { \
+ emit_##instruction(p1, kInt32Size); \
+ } \
+ \
+ template <class P1> \
+ void instruction##q(P1 p1) { \
+ emit_##instruction(p1, kInt64Size); \
+ } \
+ \
+ template <class P1, class P2> \
+ void instruction##_tagged(P1 p1, P2 p2) { \
+ emit_##instruction(p1, p2, kTaggedSize); \
+ } \
+ \
+ template <class P1, class P2> \
+ void instruction##l(P1 p1, P2 p2) { \
+ emit_##instruction(p1, p2, kInt32Size); \
+ } \
+ \
+ template <class P1, class P2> \
+ void instruction##q(P1 p1, P2 p2) { \
+ emit_##instruction(p1, p2, kInt64Size); \
+ } \
+ \
+ template <class P1, class P2, class P3> \
+ void instruction##l(P1 p1, P2 p2, P3 p3) { \
+ emit_##instruction(p1, p2, p3, kInt32Size); \
+ } \
+ \
+ template <class P1, class P2, class P3> \
+ void instruction##q(P1 p1, P2 p2, P3 p3) { \
+ emit_##instruction(p1, p2, p3, kInt64Size); \
+ }
+ ASSEMBLER_INSTRUCTION_LIST(DECLARE_INSTRUCTION)
+#undef DECLARE_INSTRUCTION
+
+ // Insert the smallest number of nop instructions
+ // possible to align the pc offset to a multiple
+ // of m, where m must be a power of 2.
+ void Align(int m);
+ // Insert the smallest number of zero bytes possible to align the pc offset
+ // to a mulitple of m. m must be a power of 2 (>= 2).
+ void DataAlign(int m);
+ void Nop(int bytes = 1);
+ // Aligns code to something that's optimal for a jump target for the platform.
+ void CodeTargetAlign();
+
+ // Stack
+ void pushfq();
+ void popfq();
+
+ void pushq(Immediate value);
+ // Push a 32 bit integer, and guarantee that it is actually pushed as a
+ // 32 bit value, the normal push will optimize the 8 bit case.
+ void pushq_imm32(int32_t imm32);
+ void pushq(Register src);
+ void pushq(Operand src);
+
+ void popq(Register dst);
+ void popq(Operand dst);
+
+ void leave();
+
+ // Moves
+ void movb(Register dst, Operand src);
+ void movb(Register dst, Immediate imm);
+ void movb(Operand dst, Register src);
+ void movb(Operand dst, Immediate imm);
+
+ // Move the low 16 bits of a 64-bit register value to a 16-bit
+ // memory location.
+ void movw(Register dst, Operand src);
+ void movw(Operand dst, Register src);
+ void movw(Operand dst, Immediate imm);
+
+ // Move the offset of the label location relative to the current
+ // position (after the move) to the destination.
+ void movl(Operand dst, Label* src);
+
+ // Load a heap number into a register.
+ // The heap number will not be allocated and embedded into the code right
+ // away. Instead, we emit the load of a dummy object. Later, when calling
+ // Assembler::GetCode, the heap number will be allocated and the code will be
+ // patched by replacing the dummy with the actual object. The RelocInfo for
+ // the embedded object gets already recorded correctly when emitting the dummy
+ // move.
+ void movq_heap_number(Register dst, double value);
+
+ void movq_string(Register dst, const StringConstantBase* str);
+
+ // Loads a 64-bit immediate into a register, potentially using the constant
+ // pool.
+ void movq(Register dst, int64_t value) { movq(dst, Immediate64(value)); }
+ void movq(Register dst, uint64_t value) {
+ movq(dst, Immediate64(static_cast<int64_t>(value)));
+ }
+
+ // Loads a 64-bit immediate into a register without using the constant pool.
+ void movq_imm64(Register dst, int64_t value);
+
+ void movsxbl(Register dst, Register src);
+ void movsxbl(Register dst, Operand src);
+ void movsxbq(Register dst, Register src);
+ void movsxbq(Register dst, Operand src);
+ void movsxwl(Register dst, Register src);
+ void movsxwl(Register dst, Operand src);
+ void movsxwq(Register dst, Register src);
+ void movsxwq(Register dst, Operand src);
+ void movsxlq(Register dst, Register src);
+ void movsxlq(Register dst, Operand src);
+
+ // Repeated moves.
+ void repmovsb();
+ void repmovsw();
+ void repmovsl() { emit_repmovs(kInt32Size); }
+ void repmovsq() { emit_repmovs(kInt64Size); }
+
+ // Repeated store of doublewords (fill (E)CX bytes at ES:[(E)DI] with EAX).
+ void repstosl();
+ // Repeated store of quadwords (fill RCX quadwords at [RDI] with RAX).
+ void repstosq();
+
+ // Instruction to load from an immediate 64-bit pointer into RAX.
+ void load_rax(Address value, RelocInfo::Mode rmode);
+ void load_rax(ExternalReference ext);
+
+ // Conditional moves.
+ void cmovq(Condition cc, Register dst, Register src);
+ void cmovq(Condition cc, Register dst, Operand src);
+ void cmovl(Condition cc, Register dst, Register src);
+ void cmovl(Condition cc, Register dst, Operand src);
+
+ void cmpb(Register dst, Immediate src) {
+ immediate_arithmetic_op_8(0x7, dst, src);
+ }
+
+ void cmpb_al(Immediate src);
+
+ void cmpb(Register dst, Register src) { arithmetic_op_8(0x3A, dst, src); }
+
+ void cmpb(Register dst, Operand src) { arithmetic_op_8(0x3A, dst, src); }
+
+ void cmpb(Operand dst, Register src) { arithmetic_op_8(0x38, src, dst); }
+
+ void cmpb(Operand dst, Immediate src) {
+ immediate_arithmetic_op_8(0x7, dst, src);
+ }
+
+ void cmpw(Operand dst, Immediate src) {
+ immediate_arithmetic_op_16(0x7, dst, src);
+ }
+
+ void cmpw(Register dst, Immediate src) {
+ immediate_arithmetic_op_16(0x7, dst, src);
+ }
+
+ void cmpw(Register dst, Operand src) { arithmetic_op_16(0x3B, dst, src); }
+
+ void cmpw(Register dst, Register src) { arithmetic_op_16(0x3B, dst, src); }
+
+ void cmpw(Operand dst, Register src) { arithmetic_op_16(0x39, src, dst); }
+
+ void testb(Register reg, Operand op) { testb(op, reg); }
+
+ void testw(Register reg, Operand op) { testw(op, reg); }
+
+ void andb(Register dst, Immediate src) {
+ immediate_arithmetic_op_8(0x4, dst, src);
+ }
+
+ void decb(Register dst);
+ void decb(Operand dst);
+
+ // Lock prefix.
+ void lock();
+
+ void xchgb(Register reg, Operand op);
+ void xchgw(Register reg, Operand op);
+
+ void xaddb(Operand dst, Register src);
+ void xaddw(Operand dst, Register src);
+ void xaddl(Operand dst, Register src);
+ void xaddq(Operand dst, Register src);
+
+ void negb(Register reg);
+ void negw(Register reg);
+ void negl(Register reg);
+ void negq(Register reg);
+ void negb(Operand op);
+ void negw(Operand op);
+ void negl(Operand op);
+ void negq(Operand op);
+
+ void cmpxchgb(Operand dst, Register src);
+ void cmpxchgw(Operand dst, Register src);
+
+ // Sign-extends rax into rdx:rax.
+ void cqo();
+ // Sign-extends eax into edx:eax.
+ void cdq();
+
+ // Multiply eax by src, put the result in edx:eax.
+ void mull(Register src);
+ void mull(Operand src);
+ // Multiply rax by src, put the result in rdx:rax.
+ void mulq(Register src);
+
+#define DECLARE_SHIFT_INSTRUCTION(instruction, subcode) \
+ void instruction##l(Register dst, Immediate imm8) { \
+ shift(dst, imm8, subcode, kInt32Size); \
+ } \
+ \
+ void instruction##q(Register dst, Immediate imm8) { \
+ shift(dst, imm8, subcode, kInt64Size); \
+ } \
+ \
+ void instruction##l(Operand dst, Immediate imm8) { \
+ shift(dst, imm8, subcode, kInt32Size); \
+ } \
+ \
+ void instruction##q(Operand dst, Immediate imm8) { \
+ shift(dst, imm8, subcode, kInt64Size); \
+ } \
+ \
+ void instruction##l_cl(Register dst) { shift(dst, subcode, kInt32Size); } \
+ \
+ void instruction##q_cl(Register dst) { shift(dst, subcode, kInt64Size); } \
+ \
+ void instruction##l_cl(Operand dst) { shift(dst, subcode, kInt32Size); } \
+ \
+ void instruction##q_cl(Operand dst) { shift(dst, subcode, kInt64Size); }
+ SHIFT_INSTRUCTION_LIST(DECLARE_SHIFT_INSTRUCTION)
+#undef DECLARE_SHIFT_INSTRUCTION
+
+ // Shifts dst:src left by cl bits, affecting only dst.
+ void shld(Register dst, Register src);
+
+ // Shifts src:dst right by cl bits, affecting only dst.
+ void shrd(Register dst, Register src);
+
+ void store_rax(Address dst, RelocInfo::Mode mode);
+ void store_rax(ExternalReference ref);
+
+ void subb(Register dst, Immediate src) {
+ immediate_arithmetic_op_8(0x5, dst, src);
+ }
+
+ void sub_sp_32(uint32_t imm);
+
+ void testb(Register dst, Register src);
+ void testb(Register reg, Immediate mask);
+ void testb(Operand op, Immediate mask);
+ void testb(Operand op, Register reg);
+
+ void testw(Register dst, Register src);
+ void testw(Register reg, Immediate mask);
+ void testw(Operand op, Immediate mask);
+ void testw(Operand op, Register reg);
+
+ // Bit operations.
+ void bswapl(Register dst);
+ void bswapq(Register dst);
+ void btq(Operand dst, Register src);
+ void btsq(Operand dst, Register src);
+ void btsq(Register dst, Immediate imm8);
+ void btrq(Register dst, Immediate imm8);
+ void bsrq(Register dst, Register src);
+ void bsrq(Register dst, Operand src);
+ void bsrl(Register dst, Register src);
+ void bsrl(Register dst, Operand src);
+ void bsfq(Register dst, Register src);
+ void bsfq(Register dst, Operand src);
+ void bsfl(Register dst, Register src);
+ void bsfl(Register dst, Operand src);
+
+ // Miscellaneous
+ void clc();
+ void cld();
+ void cpuid();
+ void hlt();
+ void int3();
+ void nop();
+ void ret(int imm16);
+ void ud2();
+ void setcc(Condition cc, Register reg);
+
+ void pblendw(XMMRegister dst, Operand src, uint8_t mask);
+ void pblendw(XMMRegister dst, XMMRegister src, uint8_t mask);
+ void palignr(XMMRegister dst, Operand src, uint8_t mask);
+ void palignr(XMMRegister dst, XMMRegister src, uint8_t mask);
+
+ // Label operations & relative jumps (PPUM Appendix D)
+ //
+ // Takes a branch opcode (cc) and a label (L) and generates
+ // either a backward branch or a forward branch and links it
+ // to the label fixup chain. Usage:
+ //
+ // Label L; // unbound label
+ // j(cc, &L); // forward branch to unbound label
+ // bind(&L); // bind label to the current pc
+ // j(cc, &L); // backward branch to bound label
+ // bind(&L); // illegal: a label may be bound only once
+ //
+ // Note: The same Label can be used for forward and backward branches
+ // but it may be bound only once.
+
+ void bind(Label* L); // binds an unbound label L to the current code position
+
+ // Calls
+ // Call near relative 32-bit displacement, relative to next instruction.
+ void call(Label* L);
+ void call(Address entry, RelocInfo::Mode rmode);
+
+ // Explicitly emit a near call / near jump. The displacement is relative to
+ // the next instructions (which starts at {pc_offset() + kNearJmpInstrSize}).
+ static constexpr int kNearJmpInstrSize = 5;
+ void near_call(intptr_t disp, RelocInfo::Mode rmode);
+ void near_jmp(intptr_t disp, RelocInfo::Mode rmode);
+
+ void call(Handle<Code> target,
+ RelocInfo::Mode rmode = RelocInfo::CODE_TARGET);
+
+ // Call near absolute indirect, address in register
+ void call(Register adr);
+
+ // Jumps
+ // Jump short or near relative.
+ // Use a 32-bit signed displacement.
+ // Unconditional jump to L
+ void jmp(Label* L, Label::Distance distance = Label::kFar);
+ void jmp(Handle<Code> target, RelocInfo::Mode rmode);
+
+ // Jump near absolute indirect (r64)
+ void jmp(Register adr);
+ void jmp(Operand src);
+
+ // Unconditional jump relative to the current address. Low-level routine,
+ // use with caution!
+ void jmp_rel(int offset);
+
+ // Conditional jumps
+ void j(Condition cc, Label* L, Label::Distance distance = Label::kFar);
+ void j(Condition cc, Address entry, RelocInfo::Mode rmode);
+ void j(Condition cc, Handle<Code> target, RelocInfo::Mode rmode);
+
+ // Floating-point operations
+ void fld(int i);
+
+ void fld1();
+ void fldz();
+ void fldpi();
+ void fldln2();
+
+ void fld_s(Operand adr);
+ void fld_d(Operand adr);
+
+ void fstp_s(Operand adr);
+ void fstp_d(Operand adr);
+ void fstp(int index);
+
+ void fild_s(Operand adr);
+ void fild_d(Operand adr);
+
+ void fist_s(Operand adr);
+
+ void fistp_s(Operand adr);
+ void fistp_d(Operand adr);
+
+ void fisttp_s(Operand adr);
+ void fisttp_d(Operand adr);
+
+ void fabs();
+ void fchs();
+
+ void fadd(int i);
+ void fsub(int i);
+ void fmul(int i);
+ void fdiv(int i);
+
+ void fisub_s(Operand adr);
+
+ void faddp(int i = 1);
+ void fsubp(int i = 1);
+ void fsubrp(int i = 1);
+ void fmulp(int i = 1);
+ void fdivp(int i = 1);
+ void fprem();
+ void fprem1();
+
+ void fxch(int i = 1);
+ void fincstp();
+ void ffree(int i = 0);
+
+ void ftst();
+ void fucomp(int i);
+ void fucompp();
+ void fucomi(int i);
+ void fucomip();
+
+ void fcompp();
+ void fnstsw_ax();
+ void fwait();
+ void fnclex();
+
+ void fsin();
+ void fcos();
+ void fptan();
+ void fyl2x();
+ void f2xm1();
+ void fscale();
+ void fninit();
+
+ void frndint();
+
+ void sahf();
+
+ void ucomiss(XMMRegister dst, XMMRegister src);
+ void ucomiss(XMMRegister dst, Operand src);
+ void movaps(XMMRegister dst, XMMRegister src);
+
+ // Don't use this unless it's important to keep the
+ // top half of the destination register unchanged.
+ // Use movaps when moving float values and movd for integer
+ // values in xmm registers.
+ void movss(XMMRegister dst, XMMRegister src);
+
+ void movss(XMMRegister dst, Operand src);
+ void movss(Operand dst, XMMRegister src);
+
+ void movlps(XMMRegister dst, Operand src);
+ void movlps(Operand dst, XMMRegister src);
+
+ void movhps(XMMRegister dst, Operand src);
+ void movhps(Operand dst, XMMRegister src);
+
+ void shufps(XMMRegister dst, XMMRegister src, byte imm8);
+
+ void cvttss2si(Register dst, Operand src);
+ void cvttss2si(Register dst, XMMRegister src);
+ void cvtlsi2ss(XMMRegister dst, Operand src);
+ void cvtlsi2ss(XMMRegister dst, Register src);
+
+ void movmskps(Register dst, XMMRegister src);
+
+ void vinstr(byte op, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ SIMDPrefix pp, LeadingOpcode m, VexW w);
+ void vinstr(byte op, XMMRegister dst, XMMRegister src1, Operand src2,
+ SIMDPrefix pp, LeadingOpcode m, VexW w);
+
+ // SSE instructions
+ void sse_instr(XMMRegister dst, XMMRegister src, byte escape, byte opcode);
+ void sse_instr(XMMRegister dst, Operand src, byte escape, byte opcode);
+#define DECLARE_SSE_INSTRUCTION(instruction, escape, opcode) \
+ void instruction(XMMRegister dst, XMMRegister src) { \
+ sse_instr(dst, src, 0x##escape, 0x##opcode); \
+ } \
+ void instruction(XMMRegister dst, Operand src) { \
+ sse_instr(dst, src, 0x##escape, 0x##opcode); \
+ }
+
+ SSE_UNOP_INSTRUCTION_LIST(DECLARE_SSE_INSTRUCTION)
+ SSE_BINOP_INSTRUCTION_LIST(DECLARE_SSE_INSTRUCTION)
+#undef DECLARE_SSE_INSTRUCTION
+
+ // SSE instructions with prefix and SSE2 instructions
+ void sse2_instr(XMMRegister dst, XMMRegister src, byte prefix, byte escape,
+ byte opcode);
+ void sse2_instr(XMMRegister dst, Operand src, byte prefix, byte escape,
+ byte opcode);
+#define DECLARE_SSE2_INSTRUCTION(instruction, prefix, escape, opcode) \
+ void instruction(XMMRegister dst, XMMRegister src) { \
+ sse2_instr(dst, src, 0x##prefix, 0x##escape, 0x##opcode); \
+ } \
+ void instruction(XMMRegister dst, Operand src) { \
+ sse2_instr(dst, src, 0x##prefix, 0x##escape, 0x##opcode); \
+ }
+
+ // These SSE instructions have the same encoding as the SSE2 instructions.
+ SSE_INSTRUCTION_LIST_SS(DECLARE_SSE2_INSTRUCTION)
+ SSE2_INSTRUCTION_LIST(DECLARE_SSE2_INSTRUCTION)
+ SSE2_INSTRUCTION_LIST_SD(DECLARE_SSE2_INSTRUCTION)
+ SSE2_UNOP_INSTRUCTION_LIST(DECLARE_SSE2_INSTRUCTION)
+#undef DECLARE_SSE2_INSTRUCTION
+
+ void sse2_instr(XMMRegister reg, byte imm8, byte prefix, byte escape,
+ byte opcode, int extension) {
+ XMMRegister ext_reg = XMMRegister::from_code(extension);
+ sse2_instr(ext_reg, reg, prefix, escape, opcode);
+ emit(imm8);
+ }
+
+#define DECLARE_SSE2_SHIFT_IMM(instruction, prefix, escape, opcode, extension) \
+ void instruction(XMMRegister reg, byte imm8) { \
+ sse2_instr(reg, imm8, 0x##prefix, 0x##escape, 0x##opcode, 0x##extension); \
+ }
+ SSE2_INSTRUCTION_LIST_SHIFT_IMM(DECLARE_SSE2_SHIFT_IMM)
+#undef DECLARE_SSE2_SHIFT_IMM
+
+#define DECLARE_SSE2_AVX_INSTRUCTION(instruction, prefix, escape, opcode) \
+ void v##instruction(XMMRegister dst, XMMRegister src1, XMMRegister src2) { \
+ vinstr(0x##opcode, dst, src1, src2, k##prefix, k##escape, kW0); \
+ } \
+ void v##instruction(XMMRegister dst, XMMRegister src1, Operand src2) { \
+ vinstr(0x##opcode, dst, src1, src2, k##prefix, k##escape, kW0); \
+ }
+
+ SSE2_INSTRUCTION_LIST(DECLARE_SSE2_AVX_INSTRUCTION)
+#undef DECLARE_SSE2_AVX_INSTRUCTION
+
+#define DECLARE_SSE2_UNOP_AVX_INSTRUCTION(instruction, prefix, escape, opcode) \
+ void v##instruction(XMMRegister dst, XMMRegister src) { \
+ vpd(0x##opcode, dst, xmm0, src); \
+ } \
+ void v##instruction(XMMRegister dst, Operand src) { \
+ vpd(0x##opcode, dst, xmm0, src); \
+ }
+
+ SSE2_UNOP_INSTRUCTION_LIST(DECLARE_SSE2_UNOP_AVX_INSTRUCTION)
+#undef DECLARE_SSE2_UNOP_AVX_INSTRUCTION
+
+ // SSE3
+ void lddqu(XMMRegister dst, Operand src);
+ void movddup(XMMRegister dst, Operand src);
+ void movddup(XMMRegister dst, XMMRegister src);
+
+ // SSSE3
+ void ssse3_instr(XMMRegister dst, XMMRegister src, byte prefix, byte escape1,
+ byte escape2, byte opcode);
+ void ssse3_instr(XMMRegister dst, Operand src, byte prefix, byte escape1,
+ byte escape2, byte opcode);
+
+#define DECLARE_SSSE3_INSTRUCTION(instruction, prefix, escape1, escape2, \
+ opcode) \
+ void instruction(XMMRegister dst, XMMRegister src) { \
+ ssse3_instr(dst, src, 0x##prefix, 0x##escape1, 0x##escape2, 0x##opcode); \
+ } \
+ void instruction(XMMRegister dst, Operand src) { \
+ ssse3_instr(dst, src, 0x##prefix, 0x##escape1, 0x##escape2, 0x##opcode); \
+ }
+
+ SSSE3_INSTRUCTION_LIST(DECLARE_SSSE3_INSTRUCTION)
+ SSSE3_UNOP_INSTRUCTION_LIST(DECLARE_SSSE3_INSTRUCTION)
+#undef DECLARE_SSSE3_INSTRUCTION
+
+ // SSE4
+ void sse4_instr(Register dst, XMMRegister src, byte prefix, byte escape1,
+ byte escape2, byte opcode, int8_t imm8);
+ void sse4_instr(Operand dst, XMMRegister src, byte prefix, byte escape1,
+ byte escape2, byte opcode, int8_t imm8);
+ void sse4_instr(XMMRegister dst, Register src, byte prefix, byte escape1,
+ byte escape2, byte opcode, int8_t imm8);
+ void sse4_instr(XMMRegister dst, XMMRegister src, byte prefix, byte escape1,
+ byte escape2, byte opcode);
+ void sse4_instr(XMMRegister dst, Operand src, byte prefix, byte escape1,
+ byte escape2, byte opcode);
+#define DECLARE_SSE4_INSTRUCTION(instruction, prefix, escape1, escape2, \
+ opcode) \
+ void instruction(XMMRegister dst, XMMRegister src) { \
+ sse4_instr(dst, src, 0x##prefix, 0x##escape1, 0x##escape2, 0x##opcode); \
+ } \
+ void instruction(XMMRegister dst, Operand src) { \
+ sse4_instr(dst, src, 0x##prefix, 0x##escape1, 0x##escape2, 0x##opcode); \
+ }
+
+ SSE4_INSTRUCTION_LIST(DECLARE_SSE4_INSTRUCTION)
+ SSE4_UNOP_INSTRUCTION_LIST(DECLARE_SSE4_INSTRUCTION)
+ DECLARE_SSE4_INSTRUCTION(pblendvb, 66, 0F, 38, 10)
+ DECLARE_SSE4_INSTRUCTION(blendvps, 66, 0F, 38, 14)
+ DECLARE_SSE4_INSTRUCTION(blendvpd, 66, 0F, 38, 15)
+#undef DECLARE_SSE4_INSTRUCTION
+
+#define DECLARE_SSE4_EXTRACT_INSTRUCTION(instruction, prefix, escape1, \
+ escape2, opcode) \
+ void instruction(Register dst, XMMRegister src, uint8_t imm8) { \
+ sse4_instr(dst, src, 0x##prefix, 0x##escape1, 0x##escape2, 0x##opcode, \
+ imm8); \
+ } \
+ void instruction(Operand dst, XMMRegister src, uint8_t imm8) { \
+ sse4_instr(dst, src, 0x##prefix, 0x##escape1, 0x##escape2, 0x##opcode, \
+ imm8); \
+ }
+
+ SSE4_EXTRACT_INSTRUCTION_LIST(DECLARE_SSE4_EXTRACT_INSTRUCTION)
+#undef DECLARE_SSE4_EXTRACT_INSTRUCTION
+
+ // SSE4.2
+ void sse4_2_instr(XMMRegister dst, XMMRegister src, byte prefix, byte escape1,
+ byte escape2, byte opcode);
+ void sse4_2_instr(XMMRegister dst, Operand src, byte prefix, byte escape1,
+ byte escape2, byte opcode);
+#define DECLARE_SSE4_2_INSTRUCTION(instruction, prefix, escape1, escape2, \
+ opcode) \
+ void instruction(XMMRegister dst, XMMRegister src) { \
+ sse4_2_instr(dst, src, 0x##prefix, 0x##escape1, 0x##escape2, 0x##opcode); \
+ } \
+ void instruction(XMMRegister dst, Operand src) { \
+ sse4_2_instr(dst, src, 0x##prefix, 0x##escape1, 0x##escape2, 0x##opcode); \
+ }
+
+ SSE4_2_INSTRUCTION_LIST(DECLARE_SSE4_2_INSTRUCTION)
+#undef DECLARE_SSE4_2_INSTRUCTION
+
+#define DECLARE_SSE34_AVX_INSTRUCTION(instruction, prefix, escape1, escape2, \
+ opcode) \
+ void v##instruction(XMMRegister dst, XMMRegister src1, XMMRegister src2) { \
+ vinstr(0x##opcode, dst, src1, src2, k##prefix, k##escape1##escape2, kW0); \
+ } \
+ void v##instruction(XMMRegister dst, XMMRegister src1, Operand src2) { \
+ vinstr(0x##opcode, dst, src1, src2, k##prefix, k##escape1##escape2, kW0); \
+ }
+
+ SSSE3_INSTRUCTION_LIST(DECLARE_SSE34_AVX_INSTRUCTION)
+ SSE4_INSTRUCTION_LIST(DECLARE_SSE34_AVX_INSTRUCTION)
+ SSE4_2_INSTRUCTION_LIST(DECLARE_SSE34_AVX_INSTRUCTION)
+#undef DECLARE_SSE34_AVX_INSTRUCTION
+
+#define DECLARE_SSSE3_UNOP_AVX_INSTRUCTION(instruction, prefix, escape1, \
+ escape2, opcode) \
+ void v##instruction(XMMRegister dst, XMMRegister src) { \
+ vinstr(0x##opcode, dst, xmm0, src, k##prefix, k##escape1##escape2, kW0); \
+ } \
+ void v##instruction(XMMRegister dst, Operand src) { \
+ vinstr(0x##opcode, dst, xmm0, src, k##prefix, k##escape1##escape2, kW0); \
+ }
+
+ SSSE3_UNOP_INSTRUCTION_LIST(DECLARE_SSSE3_UNOP_AVX_INSTRUCTION)
+#undef DECLARE_SSSE3_UNOP_AVX_INSTRUCTION
+
+ void vpblendvb(XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ XMMRegister mask) {
+ vinstr(0x4C, dst, src1, src2, k66, k0F3A, kW0);
+ // The mask operand is encoded in bits[7:4] of the immediate byte.
+ emit(mask.code() << 4);
+ }
+
+ void vblendvps(XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ XMMRegister mask) {
+ vinstr(0x4A, dst, src1, src2, k66, k0F3A, kW0);
+ // The mask operand is encoded in bits[7:4] of the immediate byte.
+ emit(mask.code() << 4);
+ }
+
+ void vblendvpd(XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ XMMRegister mask) {
+ vinstr(0x4B, dst, src1, src2, k66, k0F3A, kW0);
+ // The mask operand is encoded in bits[7:4] of the immediate byte.
+ emit(mask.code() << 4);
+ }
+
+#define DECLARE_SSE4_PMOV_AVX_INSTRUCTION(instruction, prefix, escape1, \
+ escape2, opcode) \
+ void v##instruction(XMMRegister dst, XMMRegister src) { \
+ vinstr(0x##opcode, dst, xmm0, src, k##prefix, k##escape1##escape2, kW0); \
+ } \
+ void v##instruction(XMMRegister dst, Operand src) { \
+ vinstr(0x##opcode, dst, xmm0, src, k##prefix, k##escape1##escape2, kW0); \
+ }
+ SSE4_UNOP_INSTRUCTION_LIST(DECLARE_SSE4_PMOV_AVX_INSTRUCTION)
+#undef DECLARE_SSE4_PMOV_AVX_INSTRUCTION
+
+#define DECLARE_AVX_INSTRUCTION(instruction, prefix, escape1, escape2, opcode) \
+ void v##instruction(Register dst, XMMRegister src, uint8_t imm8) { \
+ XMMRegister idst = XMMRegister::from_code(dst.code()); \
+ vinstr(0x##opcode, src, xmm0, idst, k##prefix, k##escape1##escape2, kW0); \
+ emit(imm8); \
+ } \
+ void v##instruction(Operand dst, XMMRegister src, uint8_t imm8) { \
+ vinstr(0x##opcode, src, xmm0, dst, k##prefix, k##escape1##escape2, kW0); \
+ emit(imm8); \
+ }
+
+ SSE4_EXTRACT_INSTRUCTION_LIST(DECLARE_AVX_INSTRUCTION)
+#undef DECLARE_AVX_INSTRUCTION
+
+ void movd(XMMRegister dst, Register src);
+ void movd(XMMRegister dst, Operand src);
+ void movd(Register dst, XMMRegister src);
+ void movq(XMMRegister dst, Register src);
+ void movq(XMMRegister dst, Operand src);
+ void movq(Register dst, XMMRegister src);
+ void movq(XMMRegister dst, XMMRegister src);
+
+ // Don't use this unless it's important to keep the
+ // top half of the destination register unchanged.
+ // Use movapd when moving double values and movq for integer
+ // values in xmm registers.
+ void movsd(XMMRegister dst, XMMRegister src);
+
+ void movsd(Operand dst, XMMRegister src);
+ void movsd(XMMRegister dst, Operand src);
+
+ void movdqa(Operand dst, XMMRegister src);
+ void movdqa(XMMRegister dst, Operand src);
+
+ void movdqu(Operand dst, XMMRegister src);
+ void movdqu(XMMRegister dst, Operand src);
+ void movdqu(XMMRegister dst, XMMRegister src);
+
+ void movapd(XMMRegister dst, XMMRegister src);
+ void movupd(XMMRegister dst, Operand src);
+ void movupd(Operand dst, XMMRegister src);
+
+ void cvttsd2si(Register dst, Operand src);
+ void cvttsd2si(Register dst, XMMRegister src);
+ void cvttss2siq(Register dst, XMMRegister src);
+ void cvttss2siq(Register dst, Operand src);
+ void cvttsd2siq(Register dst, XMMRegister src);
+ void cvttsd2siq(Register dst, Operand src);
+ void cvttps2dq(XMMRegister dst, Operand src);
+ void cvttps2dq(XMMRegister dst, XMMRegister src);
+
+ void cvtlsi2sd(XMMRegister dst, Operand src);
+ void cvtlsi2sd(XMMRegister dst, Register src);
+
+ void cvtqsi2ss(XMMRegister dst, Operand src);
+ void cvtqsi2ss(XMMRegister dst, Register src);
+
+ void cvtqsi2sd(XMMRegister dst, Operand src);
+ void cvtqsi2sd(XMMRegister dst, Register src);
+
+ void cvtss2sd(XMMRegister dst, XMMRegister src);
+ void cvtss2sd(XMMRegister dst, Operand src);
+
+ void cvtsd2si(Register dst, XMMRegister src);
+ void cvtsd2siq(Register dst, XMMRegister src);
+
+ void haddps(XMMRegister dst, XMMRegister src);
+ void haddps(XMMRegister dst, Operand src);
+
+ void ucomisd(XMMRegister dst, XMMRegister src);
+ void ucomisd(XMMRegister dst, Operand src);
+ void cmpltsd(XMMRegister dst, XMMRegister src);
+
+ void movmskpd(Register dst, XMMRegister src);
+
+ void pmovmskb(Register dst, XMMRegister src);
+
+ // SSE 4.1 instruction
+ void insertps(XMMRegister dst, XMMRegister src, byte imm8);
+ void insertps(XMMRegister dst, Operand src, byte imm8);
+ void pextrq(Register dst, XMMRegister src, int8_t imm8);
+ void pinsrb(XMMRegister dst, Register src, uint8_t imm8);
+ void pinsrb(XMMRegister dst, Operand src, uint8_t imm8);
+ void pinsrw(XMMRegister dst, Register src, uint8_t imm8);
+ void pinsrw(XMMRegister dst, Operand src, uint8_t imm8);
+ void pinsrd(XMMRegister dst, Register src, uint8_t imm8);
+ void pinsrd(XMMRegister dst, Operand src, uint8_t imm8);
+ void pinsrq(XMMRegister dst, Register src, uint8_t imm8);
+ void pinsrq(XMMRegister dst, Operand src, uint8_t imm8);
+
+ void roundss(XMMRegister dst, XMMRegister src, RoundingMode mode);
+ void roundsd(XMMRegister dst, XMMRegister src, RoundingMode mode);
+ void roundps(XMMRegister dst, XMMRegister src, RoundingMode mode);
+ void roundpd(XMMRegister dst, XMMRegister src, RoundingMode mode);
+
+ void cmpps(XMMRegister dst, XMMRegister src, int8_t cmp);
+ void cmpps(XMMRegister dst, Operand src, int8_t cmp);
+ void cmppd(XMMRegister dst, XMMRegister src, int8_t cmp);
+ void cmppd(XMMRegister dst, Operand src, int8_t cmp);
+
+#define SSE_CMP_P(instr, imm8) \
+ void instr##ps(XMMRegister dst, XMMRegister src) { cmpps(dst, src, imm8); } \
+ void instr##ps(XMMRegister dst, Operand src) { cmpps(dst, src, imm8); } \
+ void instr##pd(XMMRegister dst, XMMRegister src) { cmppd(dst, src, imm8); } \
+ void instr##pd(XMMRegister dst, Operand src) { cmppd(dst, src, imm8); }
+
+ SSE_CMP_P(cmpeq, 0x0)
+ SSE_CMP_P(cmplt, 0x1)
+ SSE_CMP_P(cmple, 0x2)
+ SSE_CMP_P(cmpneq, 0x4)
+ SSE_CMP_P(cmpnlt, 0x5)
+ SSE_CMP_P(cmpnle, 0x6)
+
+#undef SSE_CMP_P
+
+ void movups(XMMRegister dst, XMMRegister src);
+ void movups(XMMRegister dst, Operand src);
+ void movups(Operand dst, XMMRegister src);
+ void psrldq(XMMRegister dst, uint8_t shift);
+ void pshufd(XMMRegister dst, XMMRegister src, uint8_t shuffle);
+ void pshufd(XMMRegister dst, Operand src, uint8_t shuffle);
+ void pshufhw(XMMRegister dst, XMMRegister src, uint8_t shuffle);
+ void pshufhw(XMMRegister dst, Operand src, uint8_t shuffle);
+ void pshuflw(XMMRegister dst, XMMRegister src, uint8_t shuffle);
+ void pshuflw(XMMRegister dst, Operand src, uint8_t shuffle);
+
+ void movlhps(XMMRegister dst, XMMRegister src) {
+ sse_instr(dst, src, 0x0F, 0x16);
+ }
+
+ // AVX instruction
+ void vmovddup(XMMRegister dst, XMMRegister src);
+ void vmovddup(XMMRegister dst, Operand src);
+ void vbroadcastss(XMMRegister dst, Operand src);
+
+ void fma_instr(byte op, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ VectorLength l, SIMDPrefix pp, LeadingOpcode m, VexW w);
+ void fma_instr(byte op, XMMRegister dst, XMMRegister src1, Operand src2,
+ VectorLength l, SIMDPrefix pp, LeadingOpcode m, VexW w);
+
+#define FMA(instr, length, prefix, escape1, escape2, extension, opcode) \
+ void instr(XMMRegister dst, XMMRegister src1, XMMRegister src2) { \
+ fma_instr(0x##opcode, dst, src1, src2, k##length, k##prefix, \
+ k##escape1##escape2, k##extension); \
+ } \
+ void instr(XMMRegister dst, XMMRegister src1, Operand src2) { \
+ fma_instr(0x##opcode, dst, src1, src2, k##length, k##prefix, \
+ k##escape1##escape2, k##extension); \
+ }
+ FMA_INSTRUCTION_LIST(FMA)
+#undef FMA
+
+ void vmovd(XMMRegister dst, Register src);
+ void vmovd(XMMRegister dst, Operand src);
+ void vmovd(Register dst, XMMRegister src);
+ void vmovq(XMMRegister dst, Register src);
+ void vmovq(XMMRegister dst, Operand src);
+ void vmovq(Register dst, XMMRegister src);
+
+ void vmovsd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vsd(0x10, dst, src1, src2);
+ }
+ void vmovsd(XMMRegister dst, Operand src) { vsd(0x10, dst, xmm0, src); }
+ void vmovsd(Operand dst, XMMRegister src) { vsd(0x11, src, xmm0, dst); }
+ void vmovdqu(XMMRegister dst, Operand src);
+ void vmovdqu(Operand dst, XMMRegister src);
+
+ void vmovlps(XMMRegister dst, XMMRegister src1, Operand src2);
+ void vmovlps(Operand dst, XMMRegister src);
+
+ void vmovhps(XMMRegister dst, XMMRegister src1, Operand src2);
+ void vmovhps(Operand dst, XMMRegister src);
+
+#define AVX_SSE_UNOP(instr, escape, opcode) \
+ void v##instr(XMMRegister dst, XMMRegister src2) { \
+ vps(0x##opcode, dst, xmm0, src2); \
+ } \
+ void v##instr(XMMRegister dst, Operand src2) { \
+ vps(0x##opcode, dst, xmm0, src2); \
+ }
+ SSE_UNOP_INSTRUCTION_LIST(AVX_SSE_UNOP)
+#undef AVX_SSE_UNOP
+
+#define AVX_SSE_BINOP(instr, escape, opcode) \
+ void v##instr(XMMRegister dst, XMMRegister src1, XMMRegister src2) { \
+ vps(0x##opcode, dst, src1, src2); \
+ } \
+ void v##instr(XMMRegister dst, XMMRegister src1, Operand src2) { \
+ vps(0x##opcode, dst, src1, src2); \
+ }
+ SSE_BINOP_INSTRUCTION_LIST(AVX_SSE_BINOP)
+#undef AVX_SSE_BINOP
+
+#define AVX_3(instr, opcode, impl) \
+ void instr(XMMRegister dst, XMMRegister src1, XMMRegister src2) { \
+ impl(opcode, dst, src1, src2); \
+ } \
+ void instr(XMMRegister dst, XMMRegister src1, Operand src2) { \
+ impl(opcode, dst, src1, src2); \
+ }
+
+ AVX_3(vhaddps, 0x7c, vsd)
+
+#define AVX_SCALAR(instr, prefix, escape, opcode) \
+ void v##instr(XMMRegister dst, XMMRegister src1, XMMRegister src2) { \
+ vinstr(0x##opcode, dst, src1, src2, k##prefix, k##escape, kWIG); \
+ } \
+ void v##instr(XMMRegister dst, XMMRegister src1, Operand src2) { \
+ vinstr(0x##opcode, dst, src1, src2, k##prefix, k##escape, kWIG); \
+ }
+ SSE_INSTRUCTION_LIST_SS(AVX_SCALAR)
+ SSE2_INSTRUCTION_LIST_SD(AVX_SCALAR)
+#undef AVX_SCALAR
+
+#undef AVX_3
+
+#define AVX_SSE2_SHIFT_IMM(instr, prefix, escape, opcode, extension) \
+ void v##instr(XMMRegister dst, XMMRegister src, byte imm8) { \
+ XMMRegister ext_reg = XMMRegister::from_code(extension); \
+ vinstr(0x##opcode, ext_reg, dst, src, k##prefix, k##escape, kWIG); \
+ emit(imm8); \
+ }
+ SSE2_INSTRUCTION_LIST_SHIFT_IMM(AVX_SSE2_SHIFT_IMM)
+#undef AVX_SSE2_SHIFT_IMM
+
+ void vmovlhps(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vinstr(0x16, dst, src1, src2, kNone, k0F, kWIG);
+ }
+ void vcvtss2sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vinstr(0x5a, dst, src1, src2, kF3, k0F, kWIG);
+ }
+ void vcvtss2sd(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vinstr(0x5a, dst, src1, src2, kF3, k0F, kWIG);
+ }
+ void vcvttps2dq(XMMRegister dst, XMMRegister src) {
+ vinstr(0x5b, dst, xmm0, src, kF3, k0F, kWIG);
+ }
+ void vcvtlsi2sd(XMMRegister dst, XMMRegister src1, Register src2) {
+ XMMRegister isrc2 = XMMRegister::from_code(src2.code());
+ vinstr(0x2a, dst, src1, isrc2, kF2, k0F, kW0);
+ }
+ void vcvtlsi2sd(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vinstr(0x2a, dst, src1, src2, kF2, k0F, kW0);
+ }
+ void vcvtlsi2ss(XMMRegister dst, XMMRegister src1, Register src2) {
+ XMMRegister isrc2 = XMMRegister::from_code(src2.code());
+ vinstr(0x2a, dst, src1, isrc2, kF3, k0F, kW0);
+ }
+ void vcvtlsi2ss(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vinstr(0x2a, dst, src1, src2, kF3, k0F, kW0);
+ }
+ void vcvtqsi2ss(XMMRegister dst, XMMRegister src1, Register src2) {
+ XMMRegister isrc2 = XMMRegister::from_code(src2.code());
+ vinstr(0x2a, dst, src1, isrc2, kF3, k0F, kW1);
+ }
+ void vcvtqsi2ss(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vinstr(0x2a, dst, src1, src2, kF3, k0F, kW1);
+ }
+ void vcvtqsi2sd(XMMRegister dst, XMMRegister src1, Register src2) {
+ XMMRegister isrc2 = XMMRegister::from_code(src2.code());
+ vinstr(0x2a, dst, src1, isrc2, kF2, k0F, kW1);
+ }
+ void vcvtqsi2sd(XMMRegister dst, XMMRegister src1, Operand src2) {
+ vinstr(0x2a, dst, src1, src2, kF2, k0F, kW1);
+ }
+ void vcvttss2si(Register dst, XMMRegister src) {
+ XMMRegister idst = XMMRegister::from_code(dst.code());
+ vinstr(0x2c, idst, xmm0, src, kF3, k0F, kW0);
+ }
+ void vcvttss2si(Register dst, Operand src) {
+ XMMRegister idst = XMMRegister::from_code(dst.code());
+ vinstr(0x2c, idst, xmm0, src, kF3, k0F, kW0);
+ }
+ void vcvttsd2si(Register dst, XMMRegister src) {
+ XMMRegister idst = XMMRegister::from_code(dst.code());
+ vinstr(0x2c, idst, xmm0, src, kF2, k0F, kW0);
+ }
+ void vcvttsd2si(Register dst, Operand src) {
+ XMMRegister idst = XMMRegister::from_code(dst.code());
+ vinstr(0x2c, idst, xmm0, src, kF2, k0F, kW0);
+ }
+ void vcvttss2siq(Register dst, XMMRegister src) {
+ XMMRegister idst = XMMRegister::from_code(dst.code());
+ vinstr(0x2c, idst, xmm0, src, kF3, k0F, kW1);
+ }
+ void vcvttss2siq(Register dst, Operand src) {
+ XMMRegister idst = XMMRegister::from_code(dst.code());
+ vinstr(0x2c, idst, xmm0, src, kF3, k0F, kW1);
+ }
+ void vcvttsd2siq(Register dst, XMMRegister src) {
+ XMMRegister idst = XMMRegister::from_code(dst.code());
+ vinstr(0x2c, idst, xmm0, src, kF2, k0F, kW1);
+ }
+ void vcvttsd2siq(Register dst, Operand src) {
+ XMMRegister idst = XMMRegister::from_code(dst.code());
+ vinstr(0x2c, idst, xmm0, src, kF2, k0F, kW1);
+ }
+ void vcvtsd2si(Register dst, XMMRegister src) {
+ XMMRegister idst = XMMRegister::from_code(dst.code());
+ vinstr(0x2d, idst, xmm0, src, kF2, k0F, kW0);
+ }
+ void vucomisd(XMMRegister dst, XMMRegister src) {
+ vinstr(0x2e, dst, xmm0, src, k66, k0F, kWIG);
+ }
+ void vucomisd(XMMRegister dst, Operand src) {
+ vinstr(0x2e, dst, xmm0, src, k66, k0F, kWIG);
+ }
+ void vroundss(XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ RoundingMode mode) {
+ vinstr(0x0a, dst, src1, src2, k66, k0F3A, kWIG);
+ emit(static_cast<byte>(mode) | 0x8); // Mask precision exception.
+ }
+ void vroundsd(XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ RoundingMode mode) {
+ vinstr(0x0b, dst, src1, src2, k66, k0F3A, kWIG);
+ emit(static_cast<byte>(mode) | 0x8); // Mask precision exception.
+ }
+ void vroundps(XMMRegister dst, XMMRegister src, RoundingMode mode) {
+ vinstr(0x08, dst, xmm0, src, k66, k0F3A, kWIG);
+ emit(static_cast<byte>(mode) | 0x8); // Mask precision exception.
+ }
+ void vroundpd(XMMRegister dst, XMMRegister src, RoundingMode mode) {
+ vinstr(0x09, dst, xmm0, src, k66, k0F3A, kWIG);
+ emit(static_cast<byte>(mode) | 0x8); // Mask precision exception.
+ }
+
+ void vsd(byte op, XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vinstr(op, dst, src1, src2, kF2, k0F, kWIG);
+ }
+ void vsd(byte op, XMMRegister dst, XMMRegister src1, Operand src2) {
+ vinstr(op, dst, src1, src2, kF2, k0F, kWIG);
+ }
+
+ void vmovss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
+ vss(0x10, dst, src1, src2);
+ }
+ void vmovss(XMMRegister dst, Operand src) { vss(0x10, dst, xmm0, src); }
+ void vmovss(Operand dst, XMMRegister src) { vss(0x11, src, xmm0, dst); }
+ void vucomiss(XMMRegister dst, XMMRegister src);
+ void vucomiss(XMMRegister dst, Operand src);
+ void vss(byte op, XMMRegister dst, XMMRegister src1, XMMRegister src2);
+ void vss(byte op, XMMRegister dst, XMMRegister src1, Operand src2);
+
+ void vshufps(XMMRegister dst, XMMRegister src1, XMMRegister src2, byte imm8) {
+ vps(0xC6, dst, src1, src2, imm8);
+ }
+
+ void vmovaps(XMMRegister dst, XMMRegister src) { vps(0x28, dst, xmm0, src); }
+ void vmovups(XMMRegister dst, XMMRegister src) { vps(0x10, dst, xmm0, src); }
+ void vmovups(XMMRegister dst, Operand src) { vps(0x10, dst, xmm0, src); }
+ void vmovups(Operand dst, XMMRegister src) { vps(0x11, src, xmm0, dst); }
+ void vmovapd(XMMRegister dst, XMMRegister src) { vpd(0x28, dst, xmm0, src); }
+ void vmovupd(XMMRegister dst, Operand src) { vpd(0x10, dst, xmm0, src); }
+ void vmovupd(Operand dst, XMMRegister src) { vpd(0x11, src, xmm0, dst); }
+ void vmovmskps(Register dst, XMMRegister src) {
+ XMMRegister idst = XMMRegister::from_code(dst.code());
+ vps(0x50, idst, xmm0, src);
+ }
+ void vmovmskpd(Register dst, XMMRegister src) {
+ XMMRegister idst = XMMRegister::from_code(dst.code());
+ vpd(0x50, idst, xmm0, src);
+ }
+ void vpmovmskb(Register dst, XMMRegister src);
+ void vcmpps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int8_t cmp) {
+ vps(0xC2, dst, src1, src2);
+ emit(cmp);
+ }
+ void vcmpps(XMMRegister dst, XMMRegister src1, Operand src2, int8_t cmp) {
+ vps(0xC2, dst, src1, src2);
+ emit(cmp);
+ }
+ void vcmppd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int8_t cmp) {
+ vpd(0xC2, dst, src1, src2);
+ emit(cmp);
+ }
+ void vcmppd(XMMRegister dst, XMMRegister src1, Operand src2, int8_t cmp) {
+ vpd(0xC2, dst, src1, src2);
+ emit(cmp);
+ }
+
+#define AVX_CMP_P(instr, imm8) \
+ void instr##ps(XMMRegister dst, XMMRegister src1, XMMRegister src2) { \
+ vcmpps(dst, src1, src2, imm8); \
+ } \
+ void instr##ps(XMMRegister dst, XMMRegister src1, Operand src2) { \
+ vcmpps(dst, src1, src2, imm8); \
+ } \
+ void instr##pd(XMMRegister dst, XMMRegister src1, XMMRegister src2) { \
+ vcmppd(dst, src1, src2, imm8); \
+ } \
+ void instr##pd(XMMRegister dst, XMMRegister src1, Operand src2) { \
+ vcmppd(dst, src1, src2, imm8); \
+ }
+
+ AVX_CMP_P(vcmpeq, 0x0)
+ AVX_CMP_P(vcmplt, 0x1)
+ AVX_CMP_P(vcmple, 0x2)
+ AVX_CMP_P(vcmpneq, 0x4)
+ AVX_CMP_P(vcmpnlt, 0x5)
+ AVX_CMP_P(vcmpnle, 0x6)
+
+#undef AVX_CMP_P
+
+ void vlddqu(XMMRegister dst, Operand src) {
+ vinstr(0xF0, dst, xmm0, src, kF2, k0F, kWIG);
+ }
+ void vinsertps(XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ byte imm8) {
+ vinstr(0x21, dst, src1, src2, k66, k0F3A, kWIG);
+ emit(imm8);
+ }
+ void vinsertps(XMMRegister dst, XMMRegister src1, Operand src2, byte imm8) {
+ vinstr(0x21, dst, src1, src2, k66, k0F3A, kWIG);
+ emit(imm8);
+ }
+ void vpextrq(Register dst, XMMRegister src, int8_t imm8) {
+ XMMRegister idst = XMMRegister::from_code(dst.code());
+ vinstr(0x16, src, xmm0, idst, k66, k0F3A, kW1);
+ emit(imm8);
+ }
+ void vpinsrb(XMMRegister dst, XMMRegister src1, Register src2, uint8_t imm8) {
+ XMMRegister isrc = XMMRegister::from_code(src2.code());
+ vinstr(0x20, dst, src1, isrc, k66, k0F3A, kW0);
+ emit(imm8);
+ }
+ void vpinsrb(XMMRegister dst, XMMRegister src1, Operand src2, uint8_t imm8) {
+ vinstr(0x20, dst, src1, src2, k66, k0F3A, kW0);
+ emit(imm8);
+ }
+ void vpinsrw(XMMRegister dst, XMMRegister src1, Register src2, uint8_t imm8) {
+ XMMRegister isrc = XMMRegister::from_code(src2.code());
+ vinstr(0xc4, dst, src1, isrc, k66, k0F, kW0);
+ emit(imm8);
+ }
+ void vpinsrw(XMMRegister dst, XMMRegister src1, Operand src2, uint8_t imm8) {
+ vinstr(0xc4, dst, src1, src2, k66, k0F, kW0);
+ emit(imm8);
+ }
+ void vpinsrd(XMMRegister dst, XMMRegister src1, Register src2, uint8_t imm8) {
+ XMMRegister isrc = XMMRegister::from_code(src2.code());
+ vinstr(0x22, dst, src1, isrc, k66, k0F3A, kW0);
+ emit(imm8);
+ }
+ void vpinsrd(XMMRegister dst, XMMRegister src1, Operand src2, uint8_t imm8) {
+ vinstr(0x22, dst, src1, src2, k66, k0F3A, kW0);
+ emit(imm8);
+ }
+ void vpinsrq(XMMRegister dst, XMMRegister src1, Register src2, uint8_t imm8) {
+ XMMRegister isrc = XMMRegister::from_code(src2.code());
+ vinstr(0x22, dst, src1, isrc, k66, k0F3A, kW1);
+ emit(imm8);
+ }
+ void vpinsrq(XMMRegister dst, XMMRegister src1, Operand src2, uint8_t imm8) {
+ vinstr(0x22, dst, src1, src2, k66, k0F3A, kW1);
+ emit(imm8);
+ }
+
+ void vpshufd(XMMRegister dst, XMMRegister src, uint8_t imm8) {
+ vinstr(0x70, dst, xmm0, src, k66, k0F, kWIG);
+ emit(imm8);
+ }
+ void vpshufd(XMMRegister dst, Operand src, uint8_t imm8) {
+ vinstr(0x70, dst, xmm0, src, k66, k0F, kWIG);
+ emit(imm8);
+ }
+ void vpshuflw(XMMRegister dst, XMMRegister src, uint8_t imm8) {
+ vinstr(0x70, dst, xmm0, src, kF2, k0F, kWIG);
+ emit(imm8);
+ }
+ void vpshuflw(XMMRegister dst, Operand src, uint8_t imm8) {
+ vinstr(0x70, dst, xmm0, src, kF2, k0F, kWIG);
+ emit(imm8);
+ }
+ void vpshufhw(XMMRegister dst, XMMRegister src, uint8_t imm8) {
+ vinstr(0x70, dst, xmm0, src, kF3, k0F, kWIG);
+ emit(imm8);
+ }
+ void vpshufhw(XMMRegister dst, Operand src, uint8_t imm8) {
+ vinstr(0x70, dst, xmm0, src, kF2, k0F, kWIG);
+ emit(imm8);
+ }
+
+ void vpblendw(XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ uint8_t mask) {
+ vinstr(0x0E, dst, src1, src2, k66, k0F3A, kWIG);
+ emit(mask);
+ }
+ void vpblendw(XMMRegister dst, XMMRegister src1, Operand src2, uint8_t mask) {
+ vinstr(0x0E, dst, src1, src2, k66, k0F3A, kWIG);
+ emit(mask);
+ }
+
+ void vpalignr(XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ uint8_t imm8) {
+ vinstr(0x0F, dst, src1, src2, k66, k0F3A, kWIG);
+ emit(imm8);
+ }
+ void vpalignr(XMMRegister dst, XMMRegister src1, Operand src2, uint8_t imm8) {
+ vinstr(0x0F, dst, src1, src2, k66, k0F3A, kWIG);
+ emit(imm8);
+ }
+
+ void vps(byte op, XMMRegister dst, XMMRegister src1, XMMRegister src2);
+ void vps(byte op, XMMRegister dst, XMMRegister src1, Operand src2);
+ void vps(byte op, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ byte imm8);
+ void vpd(byte op, XMMRegister dst, XMMRegister src1, XMMRegister src2);
+ void vpd(byte op, XMMRegister dst, XMMRegister src1, Operand src2);
+
+ // BMI instruction
+ void andnq(Register dst, Register src1, Register src2) {
+ bmi1q(0xf2, dst, src1, src2);
+ }
+ void andnq(Register dst, Register src1, Operand src2) {
+ bmi1q(0xf2, dst, src1, src2);
+ }
+ void andnl(Register dst, Register src1, Register src2) {
+ bmi1l(0xf2, dst, src1, src2);
+ }
+ void andnl(Register dst, Register src1, Operand src2) {
+ bmi1l(0xf2, dst, src1, src2);
+ }
+ void bextrq(Register dst, Register src1, Register src2) {
+ bmi1q(0xf7, dst, src2, src1);
+ }
+ void bextrq(Register dst, Operand src1, Register src2) {
+ bmi1q(0xf7, dst, src2, src1);
+ }
+ void bextrl(Register dst, Register src1, Register src2) {
+ bmi1l(0xf7, dst, src2, src1);
+ }
+ void bextrl(Register dst, Operand src1, Register src2) {
+ bmi1l(0xf7, dst, src2, src1);
+ }
+ void blsiq(Register dst, Register src) { bmi1q(0xf3, rbx, dst, src); }
+ void blsiq(Register dst, Operand src) { bmi1q(0xf3, rbx, dst, src); }
+ void blsil(Register dst, Register src) { bmi1l(0xf3, rbx, dst, src); }
+ void blsil(Register dst, Operand src) { bmi1l(0xf3, rbx, dst, src); }
+ void blsmskq(Register dst, Register src) { bmi1q(0xf3, rdx, dst, src); }
+ void blsmskq(Register dst, Operand src) { bmi1q(0xf3, rdx, dst, src); }
+ void blsmskl(Register dst, Register src) { bmi1l(0xf3, rdx, dst, src); }
+ void blsmskl(Register dst, Operand src) { bmi1l(0xf3, rdx, dst, src); }
+ void blsrq(Register dst, Register src) { bmi1q(0xf3, rcx, dst, src); }
+ void blsrq(Register dst, Operand src) { bmi1q(0xf3, rcx, dst, src); }
+ void blsrl(Register dst, Register src) { bmi1l(0xf3, rcx, dst, src); }
+ void blsrl(Register dst, Operand src) { bmi1l(0xf3, rcx, dst, src); }
+ void tzcntq(Register dst, Register src);
+ void tzcntq(Register dst, Operand src);
+ void tzcntl(Register dst, Register src);
+ void tzcntl(Register dst, Operand src);
+
+ void lzcntq(Register dst, Register src);
+ void lzcntq(Register dst, Operand src);
+ void lzcntl(Register dst, Register src);
+ void lzcntl(Register dst, Operand src);
+
+ void popcntq(Register dst, Register src);
+ void popcntq(Register dst, Operand src);
+ void popcntl(Register dst, Register src);
+ void popcntl(Register dst, Operand src);
+
+ void bzhiq(Register dst, Register src1, Register src2) {
+ bmi2q(kNone, 0xf5, dst, src2, src1);
+ }
+ void bzhiq(Register dst, Operand src1, Register src2) {
+ bmi2q(kNone, 0xf5, dst, src2, src1);
+ }
+ void bzhil(Register dst, Register src1, Register src2) {
+ bmi2l(kNone, 0xf5, dst, src2, src1);
+ }
+ void bzhil(Register dst, Operand src1, Register src2) {
+ bmi2l(kNone, 0xf5, dst, src2, src1);
+ }
+ void mulxq(Register dst1, Register dst2, Register src) {
+ bmi2q(kF2, 0xf6, dst1, dst2, src);
+ }
+ void mulxq(Register dst1, Register dst2, Operand src) {
+ bmi2q(kF2, 0xf6, dst1, dst2, src);
+ }
+ void mulxl(Register dst1, Register dst2, Register src) {
+ bmi2l(kF2, 0xf6, dst1, dst2, src);
+ }
+ void mulxl(Register dst1, Register dst2, Operand src) {
+ bmi2l(kF2, 0xf6, dst1, dst2, src);
+ }
+ void pdepq(Register dst, Register src1, Register src2) {
+ bmi2q(kF2, 0xf5, dst, src1, src2);
+ }
+ void pdepq(Register dst, Register src1, Operand src2) {
+ bmi2q(kF2, 0xf5, dst, src1, src2);
+ }
+ void pdepl(Register dst, Register src1, Register src2) {
+ bmi2l(kF2, 0xf5, dst, src1, src2);
+ }
+ void pdepl(Register dst, Register src1, Operand src2) {
+ bmi2l(kF2, 0xf5, dst, src1, src2);
+ }
+ void pextq(Register dst, Register src1, Register src2) {
+ bmi2q(kF3, 0xf5, dst, src1, src2);
+ }
+ void pextq(Register dst, Register src1, Operand src2) {
+ bmi2q(kF3, 0xf5, dst, src1, src2);
+ }
+ void pextl(Register dst, Register src1, Register src2) {
+ bmi2l(kF3, 0xf5, dst, src1, src2);
+ }
+ void pextl(Register dst, Register src1, Operand src2) {
+ bmi2l(kF3, 0xf5, dst, src1, src2);
+ }
+ void sarxq(Register dst, Register src1, Register src2) {
+ bmi2q(kF3, 0xf7, dst, src2, src1);
+ }
+ void sarxq(Register dst, Operand src1, Register src2) {
+ bmi2q(kF3, 0xf7, dst, src2, src1);
+ }
+ void sarxl(Register dst, Register src1, Register src2) {
+ bmi2l(kF3, 0xf7, dst, src2, src1);
+ }
+ void sarxl(Register dst, Operand src1, Register src2) {
+ bmi2l(kF3, 0xf7, dst, src2, src1);
+ }
+ void shlxq(Register dst, Register src1, Register src2) {
+ bmi2q(k66, 0xf7, dst, src2, src1);
+ }
+ void shlxq(Register dst, Operand src1, Register src2) {
+ bmi2q(k66, 0xf7, dst, src2, src1);
+ }
+ void shlxl(Register dst, Register src1, Register src2) {
+ bmi2l(k66, 0xf7, dst, src2, src1);
+ }
+ void shlxl(Register dst, Operand src1, Register src2) {
+ bmi2l(k66, 0xf7, dst, src2, src1);
+ }
+ void shrxq(Register dst, Register src1, Register src2) {
+ bmi2q(kF2, 0xf7, dst, src2, src1);
+ }
+ void shrxq(Register dst, Operand src1, Register src2) {
+ bmi2q(kF2, 0xf7, dst, src2, src1);
+ }
+ void shrxl(Register dst, Register src1, Register src2) {
+ bmi2l(kF2, 0xf7, dst, src2, src1);
+ }
+ void shrxl(Register dst, Operand src1, Register src2) {
+ bmi2l(kF2, 0xf7, dst, src2, src1);
+ }
+ void rorxq(Register dst, Register src, byte imm8);
+ void rorxq(Register dst, Operand src, byte imm8);
+ void rorxl(Register dst, Register src, byte imm8);
+ void rorxl(Register dst, Operand src, byte imm8);
+
+ void mfence();
+ void lfence();
+ void pause();
+
+ // Check the code size generated from label to here.
+ int SizeOfCodeGeneratedSince(Label* label) {
+ return pc_offset() - label->pos();
+ }
+
+ // Record a deoptimization reason that can be used by a log or cpu profiler.
+ // Use --trace-deopt to enable.
+ void RecordDeoptReason(DeoptimizeReason reason, SourcePosition position,
+ int id);
+
+ // Writes a single word of data in the code stream.
+ // Used for inline tables, e.g., jump-tables.
+ void db(uint8_t data);
+ void dd(uint32_t data);
+ void dq(uint64_t data);
+ void dp(uintptr_t data) { dq(data); }
+ void dq(Label* label);
+
+ // Patch entries for partial constant pool.
+ void PatchConstPool();
+
+ // Check if use partial constant pool for this rmode.
+ static bool UseConstPoolFor(RelocInfo::Mode rmode);
+
+ // Check if there is less than kGap bytes available in the buffer.
+ // If this is the case, we need to grow the buffer before emitting
+ // an instruction or relocation information.
+ inline bool buffer_overflow() const {
+ return pc_ >= reloc_info_writer.pos() - kGap;
+ }
+
+ // Get the number of bytes available in the buffer.
+ inline int available_space() const {
+ return static_cast<int>(reloc_info_writer.pos() - pc_);
+ }
+
+ static bool IsNop(Address addr);
+
+ // Avoid overflows for displacements etc.
+ static constexpr int kMaximalBufferSize = 512 * MB;
+
+ byte byte_at(int pos) { return buffer_start_[pos]; }
+ void set_byte_at(int pos, byte value) { buffer_start_[pos] = value; }
+
+#if defined(V8_OS_WIN_X64)
+ win64_unwindinfo::BuiltinUnwindInfo GetUnwindInfo() const;
+#endif
+
+ protected:
+ // Call near indirect
+ void call(Operand operand);
+
+ private:
+ Address addr_at(int pos) {
+ return reinterpret_cast<Address>(buffer_start_ + pos);
+ }
+ uint32_t long_at(int pos) {
+ return ReadUnalignedValue<uint32_t>(addr_at(pos));
+ }
+ void long_at_put(int pos, uint32_t x) {
+ WriteUnalignedValue(addr_at(pos), x);
+ }
+
+ // code emission
+ void GrowBuffer();
+
+ void emit(byte x) { *pc_++ = x; }
+ inline void emitl(uint32_t x);
+ inline void emitq(uint64_t x);
+ inline void emitw(uint16_t x);
+ inline void emit_runtime_entry(Address entry, RelocInfo::Mode rmode);
+ inline void emit(Immediate x);
+ inline void emit(Immediate64 x);
+
+ // Emits a REX prefix that encodes a 64-bit operand size and
+ // the top bit of both register codes.
+ // High bit of reg goes to REX.R, high bit of rm_reg goes to REX.B.
+ // REX.W is set.
+ inline void emit_rex_64(XMMRegister reg, Register rm_reg);
+ inline void emit_rex_64(Register reg, XMMRegister rm_reg);
+ inline void emit_rex_64(Register reg, Register rm_reg);
+ inline void emit_rex_64(XMMRegister reg, XMMRegister rm_reg);
+
+ // Emits a REX prefix that encodes a 64-bit operand size and
+ // the top bit of the destination, index, and base register codes.
+ // The high bit of reg is used for REX.R, the high bit of op's base
+ // register is used for REX.B, and the high bit of op's index register
+ // is used for REX.X. REX.W is set.
+ inline void emit_rex_64(Register reg, Operand op);
+ inline void emit_rex_64(XMMRegister reg, Operand op);
+
+ // Emits a REX prefix that encodes a 64-bit operand size and
+ // the top bit of the register code.
+ // The high bit of register is used for REX.B.
+ // REX.W is set and REX.R and REX.X are clear.
+ inline void emit_rex_64(Register rm_reg);
+
+ // Emits a REX prefix that encodes a 64-bit operand size and
+ // the top bit of the index and base register codes.
+ // The high bit of op's base register is used for REX.B, and the high
+ // bit of op's index register is used for REX.X.
+ // REX.W is set and REX.R clear.
+ inline void emit_rex_64(Operand op);
+
+ // Emit a REX prefix that only sets REX.W to choose a 64-bit operand size.
+ void emit_rex_64() { emit(0x48); }
+
+ // High bit of reg goes to REX.R, high bit of rm_reg goes to REX.B.
+ // REX.W is clear.
+ inline void emit_rex_32(Register reg, Register rm_reg);
+
+ // The high bit of reg is used for REX.R, the high bit of op's base
+ // register is used for REX.B, and the high bit of op's index register
+ // is used for REX.X. REX.W is cleared.
+ inline void emit_rex_32(Register reg, Operand op);
+
+ // High bit of rm_reg goes to REX.B.
+ // REX.W, REX.R and REX.X are clear.
+ inline void emit_rex_32(Register rm_reg);
+
+ // High bit of base goes to REX.B and high bit of index to REX.X.
+ // REX.W and REX.R are clear.
+ inline void emit_rex_32(Operand op);
+
+ // High bit of reg goes to REX.R, high bit of rm_reg goes to REX.B.
+ // REX.W is cleared. If no REX bits are set, no byte is emitted.
+ inline void emit_optional_rex_32(Register reg, Register rm_reg);
+
+ // The high bit of reg is used for REX.R, the high bit of op's base
+ // register is used for REX.B, and the high bit of op's index register
+ // is used for REX.X. REX.W is cleared. If no REX bits are set, nothing
+ // is emitted.
+ inline void emit_optional_rex_32(Register reg, Operand op);
+
+ // As for emit_optional_rex_32(Register, Register), except that
+ // the registers are XMM registers.
+ inline void emit_optional_rex_32(XMMRegister reg, XMMRegister base);
+
+ // As for emit_optional_rex_32(Register, Register), except that
+ // one of the registers is an XMM registers.
+ inline void emit_optional_rex_32(XMMRegister reg, Register base);
+
+ // As for emit_optional_rex_32(Register, Register), except that
+ // one of the registers is an XMM registers.
+ inline void emit_optional_rex_32(Register reg, XMMRegister base);
+
+ // As for emit_optional_rex_32(Register, Operand), except that
+ // the register is an XMM register.
+ inline void emit_optional_rex_32(XMMRegister reg, Operand op);
+
+ // Optionally do as emit_rex_32(Register) if the register number has
+ // the high bit set.
+ inline void emit_optional_rex_32(Register rm_reg);
+ inline void emit_optional_rex_32(XMMRegister rm_reg);
+
+ // Optionally do as emit_rex_32(Operand) if the operand register
+ // numbers have a high bit set.
+ inline void emit_optional_rex_32(Operand op);
+
+ // Calls emit_rex_32(Register) for all non-byte registers.
+ inline void emit_optional_rex_8(Register reg);
+
+ // Calls emit_rex_32(Register, Operand) for all non-byte registers, and
+ // emit_optional_rex_32(Register, Operand) for byte registers.
+ inline void emit_optional_rex_8(Register reg, Operand op);
+
+ void emit_rex(int size) {
+ if (size == kInt64Size) {
+ emit_rex_64();
+ } else {
+ DCHECK_EQ(size, kInt32Size);
+ }
+ }
+
+ template <class P1>
+ void emit_rex(P1 p1, int size) {
+ if (size == kInt64Size) {
+ emit_rex_64(p1);
+ } else {
+ DCHECK_EQ(size, kInt32Size);
+ emit_optional_rex_32(p1);
+ }
+ }
+
+ template <class P1, class P2>
+ void emit_rex(P1 p1, P2 p2, int size) {
+ if (size == kInt64Size) {
+ emit_rex_64(p1, p2);
+ } else {
+ DCHECK_EQ(size, kInt32Size);
+ emit_optional_rex_32(p1, p2);
+ }
+ }
+
+ // Emit vex prefix
+ void emit_vex2_byte0() { emit(0xc5); }
+ inline void emit_vex2_byte1(XMMRegister reg, XMMRegister v, VectorLength l,
+ SIMDPrefix pp);
+ void emit_vex3_byte0() { emit(0xc4); }
+ inline void emit_vex3_byte1(XMMRegister reg, XMMRegister rm, LeadingOpcode m);
+ inline void emit_vex3_byte1(XMMRegister reg, Operand rm, LeadingOpcode m);
+ inline void emit_vex3_byte2(VexW w, XMMRegister v, VectorLength l,
+ SIMDPrefix pp);
+ inline void emit_vex_prefix(XMMRegister reg, XMMRegister v, XMMRegister rm,
+ VectorLength l, SIMDPrefix pp, LeadingOpcode m,
+ VexW w);
+ inline void emit_vex_prefix(Register reg, Register v, Register rm,
+ VectorLength l, SIMDPrefix pp, LeadingOpcode m,
+ VexW w);
+ inline void emit_vex_prefix(XMMRegister reg, XMMRegister v, Operand rm,
+ VectorLength l, SIMDPrefix pp, LeadingOpcode m,
+ VexW w);
+ inline void emit_vex_prefix(Register reg, Register v, Operand rm,
+ VectorLength l, SIMDPrefix pp, LeadingOpcode m,
+ VexW w);
+
+ // Emit the ModR/M byte, and optionally the SIB byte and
+ // 1- or 4-byte offset for a memory operand. Also encodes
+ // the second operand of the operation, a register or operation
+ // subcode, into the reg field of the ModR/M byte.
+ void emit_operand(Register reg, Operand adr) {
+ emit_operand(reg.low_bits(), adr);
+ }
+
+ // Emit the ModR/M byte, and optionally the SIB byte and
+ // 1- or 4-byte offset for a memory operand. Also used to encode
+ // a three-bit opcode extension into the ModR/M byte.
+ void emit_operand(int rm, Operand adr);
+
+ // Emit a ModR/M byte with registers coded in the reg and rm_reg fields.
+ void emit_modrm(Register reg, Register rm_reg) {
+ emit(0xC0 | reg.low_bits() << 3 | rm_reg.low_bits());
+ }
+
+ // Emit a ModR/M byte with an operation subcode in the reg field and
+ // a register in the rm_reg field.
+ void emit_modrm(int code, Register rm_reg) {
+ DCHECK(is_uint3(code));
+ emit(0xC0 | code << 3 | rm_reg.low_bits());
+ }
+
+ // Emit the code-object-relative offset of the label's position
+ inline void emit_code_relative_offset(Label* label);
+
+ // The first argument is the reg field, the second argument is the r/m field.
+ void emit_sse_operand(XMMRegister dst, XMMRegister src);
+ void emit_sse_operand(XMMRegister reg, Operand adr);
+ void emit_sse_operand(Register reg, Operand adr);
+ void emit_sse_operand(XMMRegister dst, Register src);
+ void emit_sse_operand(Register dst, XMMRegister src);
+ void emit_sse_operand(XMMRegister dst);
+
+ // Emit machine code for one of the operations ADD, ADC, SUB, SBC,
+ // AND, OR, XOR, or CMP. The encodings of these operations are all
+ // similar, differing just in the opcode or in the reg field of the
+ // ModR/M byte.
+ void arithmetic_op_8(byte opcode, Register reg, Register rm_reg);
+ void arithmetic_op_8(byte opcode, Register reg, Operand rm_reg);
+ void arithmetic_op_16(byte opcode, Register reg, Register rm_reg);
+ void arithmetic_op_16(byte opcode, Register reg, Operand rm_reg);
+ // Operate on operands/registers with pointer size, 32-bit or 64-bit size.
+ void arithmetic_op(byte opcode, Register reg, Register rm_reg, int size);
+ void arithmetic_op(byte opcode, Register reg, Operand rm_reg, int size);
+ // Operate on a byte in memory or register.
+ void immediate_arithmetic_op_8(byte subcode, Register dst, Immediate src);
+ void immediate_arithmetic_op_8(byte subcode, Operand dst, Immediate src);
+ // Operate on a word in memory or register.
+ void immediate_arithmetic_op_16(byte subcode, Register dst, Immediate src);
+ void immediate_arithmetic_op_16(byte subcode, Operand dst, Immediate src);
+ // Operate on operands/registers with pointer size, 32-bit or 64-bit size.
+ void immediate_arithmetic_op(byte subcode, Register dst, Immediate src,
+ int size);
+ void immediate_arithmetic_op(byte subcode, Operand dst, Immediate src,
+ int size);
+
+ // Emit machine code for a shift operation.
+ void shift(Operand dst, Immediate shift_amount, int subcode, int size);
+ void shift(Register dst, Immediate shift_amount, int subcode, int size);
+ // Shift dst by cl % 64 bits.
+ void shift(Register dst, int subcode, int size);
+ void shift(Operand dst, int subcode, int size);
+
+ void emit_farith(int b1, int b2, int i);
+
+ // labels
+ // void print(Label* L);
+ void bind_to(Label* L, int pos);
+
+ // record reloc info for current pc_
+ void RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data = 0);
+
+ // Arithmetics
+ void emit_add(Register dst, Register src, int size) {
+ arithmetic_op(0x03, dst, src, size);
+ }
+
+ void emit_add(Register dst, Immediate src, int size) {
+ immediate_arithmetic_op(0x0, dst, src, size);
+ }
+
+ void emit_add(Register dst, Operand src, int size) {
+ arithmetic_op(0x03, dst, src, size);
+ }
+
+ void emit_add(Operand dst, Register src, int size) {
+ arithmetic_op(0x1, src, dst, size);
+ }
+
+ void emit_add(Operand dst, Immediate src, int size) {
+ immediate_arithmetic_op(0x0, dst, src, size);
+ }
+
+ void emit_and(Register dst, Register src, int size) {
+ arithmetic_op(0x23, dst, src, size);
+ }
+
+ void emit_and(Register dst, Operand src, int size) {
+ arithmetic_op(0x23, dst, src, size);
+ }
+
+ void emit_and(Operand dst, Register src, int size) {
+ arithmetic_op(0x21, src, dst, size);
+ }
+
+ void emit_and(Register dst, Immediate src, int size) {
+ immediate_arithmetic_op(0x4, dst, src, size);
+ }
+
+ void emit_and(Operand dst, Immediate src, int size) {
+ immediate_arithmetic_op(0x4, dst, src, size);
+ }
+
+ void emit_cmp(Register dst, Register src, int size) {
+ arithmetic_op(0x3B, dst, src, size);
+ }
+
+ void emit_cmp(Register dst, Operand src, int size) {
+ arithmetic_op(0x3B, dst, src, size);
+ }
+
+ void emit_cmp(Operand dst, Register src, int size) {
+ arithmetic_op(0x39, src, dst, size);
+ }
+
+ void emit_cmp(Register dst, Immediate src, int size) {
+ immediate_arithmetic_op(0x7, dst, src, size);
+ }
+
+ void emit_cmp(Operand dst, Immediate src, int size) {
+ immediate_arithmetic_op(0x7, dst, src, size);
+ }
+
+ // Compare {al,ax,eax,rax} with src. If equal, set ZF and write dst into
+ // src. Otherwise clear ZF and write src into {al,ax,eax,rax}. This
+ // operation is only atomic if prefixed by the lock instruction.
+ void emit_cmpxchg(Operand dst, Register src, int size);
+
+ void emit_dec(Register dst, int size);
+ void emit_dec(Operand dst, int size);
+
+ // Divide rdx:rax by src. Quotient in rax, remainder in rdx when size is 64.
+ // Divide edx:eax by lower 32 bits of src. Quotient in eax, remainder in edx
+ // when size is 32.
+ void emit_idiv(Register src, int size);
+ void emit_div(Register src, int size);
+
+ // Signed multiply instructions.
+ // rdx:rax = rax * src when size is 64 or edx:eax = eax * src when size is 32.
+ void emit_imul(Register src, int size);
+ void emit_imul(Operand src, int size);
+ void emit_imul(Register dst, Register src, int size);
+ void emit_imul(Register dst, Operand src, int size);
+ void emit_imul(Register dst, Register src, Immediate imm, int size);
+ void emit_imul(Register dst, Operand src, Immediate imm, int size);
+
+ void emit_inc(Register dst, int size);
+ void emit_inc(Operand dst, int size);
+
+ void emit_lea(Register dst, Operand src, int size);
+
+ void emit_mov(Register dst, Operand src, int size);
+ void emit_mov(Register dst, Register src, int size);
+ void emit_mov(Operand dst, Register src, int size);
+ void emit_mov(Register dst, Immediate value, int size);
+ void emit_mov(Operand dst, Immediate value, int size);
+ void emit_mov(Register dst, Immediate64 value, int size);
+
+ void emit_movzxb(Register dst, Operand src, int size);
+ void emit_movzxb(Register dst, Register src, int size);
+ void emit_movzxw(Register dst, Operand src, int size);
+ void emit_movzxw(Register dst, Register src, int size);
+
+ void emit_neg(Register dst, int size);
+ void emit_neg(Operand dst, int size);
+
+ void emit_not(Register dst, int size);
+ void emit_not(Operand dst, int size);
+
+ void emit_or(Register dst, Register src, int size) {
+ arithmetic_op(0x0B, dst, src, size);
+ }
+
+ void emit_or(Register dst, Operand src, int size) {
+ arithmetic_op(0x0B, dst, src, size);
+ }
+
+ void emit_or(Operand dst, Register src, int size) {
+ arithmetic_op(0x9, src, dst, size);
+ }
+
+ void emit_or(Register dst, Immediate src, int size) {
+ immediate_arithmetic_op(0x1, dst, src, size);
+ }
+
+ void emit_or(Operand dst, Immediate src, int size) {
+ immediate_arithmetic_op(0x1, dst, src, size);
+ }
+
+ void emit_repmovs(int size);
+
+ void emit_sbb(Register dst, Register src, int size) {
+ arithmetic_op(0x1b, dst, src, size);
+ }
+
+ void emit_sub(Register dst, Register src, int size) {
+ arithmetic_op(0x2B, dst, src, size);
+ }
+
+ void emit_sub(Register dst, Immediate src, int size) {
+ immediate_arithmetic_op(0x5, dst, src, size);
+ }
+
+ void emit_sub(Register dst, Operand src, int size) {
+ arithmetic_op(0x2B, dst, src, size);
+ }
+
+ void emit_sub(Operand dst, Register src, int size) {
+ arithmetic_op(0x29, src, dst, size);
+ }
+
+ void emit_sub(Operand dst, Immediate src, int size) {
+ immediate_arithmetic_op(0x5, dst, src, size);
+ }
+
+ void emit_test(Register dst, Register src, int size);
+ void emit_test(Register reg, Immediate mask, int size);
+ void emit_test(Operand op, Register reg, int size);
+ void emit_test(Operand op, Immediate mask, int size);
+ void emit_test(Register reg, Operand op, int size) {
+ return emit_test(op, reg, size);
+ }
+
+ void emit_xchg(Register dst, Register src, int size);
+ void emit_xchg(Register dst, Operand src, int size);
+
+ void emit_xor(Register dst, Register src, int size) {
+ if (size == kInt64Size && dst.code() == src.code()) {
+ // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
+ // there is no need to make this a 64 bit operation.
+ arithmetic_op(0x33, dst, src, kInt32Size);
+ } else {
+ arithmetic_op(0x33, dst, src, size);
+ }
+ }
+
+ void emit_xor(Register dst, Operand src, int size) {
+ arithmetic_op(0x33, dst, src, size);
+ }
+
+ void emit_xor(Register dst, Immediate src, int size) {
+ immediate_arithmetic_op(0x6, dst, src, size);
+ }
+
+ void emit_xor(Operand dst, Immediate src, int size) {
+ immediate_arithmetic_op(0x6, dst, src, size);
+ }
+
+ void emit_xor(Operand dst, Register src, int size) {
+ arithmetic_op(0x31, src, dst, size);
+ }
+
+ // Most BMI instructions are similar.
+ void bmi1q(byte op, Register reg, Register vreg, Register rm);
+ void bmi1q(byte op, Register reg, Register vreg, Operand rm);
+ void bmi1l(byte op, Register reg, Register vreg, Register rm);
+ void bmi1l(byte op, Register reg, Register vreg, Operand rm);
+ void bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg, Register rm);
+ void bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg, Operand rm);
+ void bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg, Register rm);
+ void bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg, Operand rm);
+
+ // record the position of jmp/jcc instruction
+ void record_farjmp_position(Label* L, int pos);
+
+ bool is_optimizable_farjmp(int idx);
+
+ void AllocateAndInstallRequestedHeapObjects(Isolate* isolate);
+
+ int WriteCodeComments();
+
+ friend class EnsureSpace;
+ friend class RegExpMacroAssemblerX64;
+
+ // code generation
+ RelocInfoWriter reloc_info_writer;
+
+ // Internal reference positions, required for (potential) patching in
+ // GrowBuffer(); contains only those internal references whose labels
+ // are already bound.
+ std::deque<int> internal_reference_positions_;
+
+ // Variables for this instance of assembler
+ int farjmp_num_ = 0;
+ std::deque<int> farjmp_positions_;
+ std::map<Label*, std::vector<int>> label_farjmp_maps_;
+
+ ConstPool constpool_;
+
+ friend class ConstPool;
+
+#if defined(V8_OS_WIN_X64)
+ std::unique_ptr<win64_unwindinfo::XdataEncoder> xdata_encoder_;
+#endif
+};
+
+// Helper class that ensures that there is enough space for generating
+// instructions and relocation information. The constructor makes
+// sure that there is enough space and (in debug mode) the destructor
+// checks that we did not generate too much.
+class EnsureSpace {
+ public:
+ explicit EnsureSpace(Assembler* assembler) : assembler_(assembler) {
+ if (assembler_->buffer_overflow()) assembler_->GrowBuffer();
+#ifdef DEBUG
+ space_before_ = assembler_->available_space();
+#endif
+ }
+
+#ifdef DEBUG
+ ~EnsureSpace() {
+ int bytes_generated = space_before_ - assembler_->available_space();
+ DCHECK(bytes_generated < assembler_->kGap);
+ }
+#endif
+
+ private:
+ Assembler* assembler_;
+#ifdef DEBUG
+ int space_before_;
+#endif
+};
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_X64_ASSEMBLER_X64_H_
diff --git a/src/codegen/x64/constants-x64.h b/src/codegen/x64/constants-x64.h
new file mode 100644
index 0000000..775abec
--- /dev/null
+++ b/src/codegen/x64/constants-x64.h
@@ -0,0 +1,22 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_X64_CONSTANTS_X64_H_
+#define V8_CODEGEN_X64_CONSTANTS_X64_H_
+
+#include "src/common/globals.h"
+
+namespace v8 {
+namespace internal {
+// Actual value of root register is offset from the root array's start
+// to take advantage of negative displacement values.
+// TODO(sigurds): Choose best value.
+// TODO(ishell): Choose best value for ptr-compr.
+constexpr int kRootRegisterBias = kSystemPointerSize == kTaggedSize ? 128 : 0;
+
+constexpr size_t kMaxPCRelativeCodeRangeInMB = 2048;
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_X64_CONSTANTS_X64_H_
diff --git a/src/codegen/x64/cpu-x64.cc b/src/codegen/x64/cpu-x64.cc
new file mode 100644
index 0000000..cce76d8
--- /dev/null
+++ b/src/codegen/x64/cpu-x64.cc
@@ -0,0 +1,42 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// CPU specific code for x64 independent of OS goes here.
+
+#if defined(__GNUC__) && !defined(__MINGW64__)
+#include "src/third_party/valgrind/valgrind.h"
+#endif
+
+#if V8_TARGET_ARCH_X64
+
+#include "src/codegen/cpu-features.h"
+
+namespace v8 {
+namespace internal {
+
+void CpuFeatures::FlushICache(void* start, size_t size) {
+ // No need to flush the instruction cache on Intel. On Intel instruction
+ // cache flushing is only necessary when multiple cores running the same
+ // code simultaneously. V8 (and JavaScript) is single threaded and when code
+ // is patched on an intel CPU the core performing the patching will have its
+ // own instruction cache updated automatically.
+
+ // If flushing of the instruction cache becomes necessary Windows has the
+ // API function FlushInstructionCache.
+
+ // By default, valgrind only checks the stack for writes that might need to
+ // invalidate already cached translated code. This leads to random
+ // instability when code patches or moves are sometimes unnoticed. One
+ // solution is to run valgrind with --smc-check=all, but this comes at a big
+ // performance cost. We can notify valgrind to invalidate its cache.
+#ifdef VALGRIND_DISCARD_TRANSLATIONS
+ unsigned res = VALGRIND_DISCARD_TRANSLATIONS(start, size);
+ USE(res);
+#endif
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_X64
diff --git a/src/codegen/x64/fma-instr.h b/src/codegen/x64/fma-instr.h
new file mode 100644
index 0000000..f41c91e
--- /dev/null
+++ b/src/codegen/x64/fma-instr.h
@@ -0,0 +1,38 @@
+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+#ifndef V8_CODEGEN_X64_FMA_INSTR_H_
+#define V8_CODEGEN_X64_FMA_INSTR_H_
+
+#define FMA_INSTRUCTION_LIST(V) \
+ V(vfmadd132sd, L128, 66, 0F, 38, W1, 99) \
+ V(vfmadd213sd, L128, 66, 0F, 38, W1, a9) \
+ V(vfmadd231sd, L128, 66, 0F, 38, W1, b9) \
+ V(vfmsub132sd, L128, 66, 0F, 38, W1, 9b) \
+ V(vfmsub213sd, L128, 66, 0F, 38, W1, ab) \
+ V(vfmsub231sd, L128, 66, 0F, 38, W1, bb) \
+ V(vfnmadd132sd, L128, 66, 0F, 38, W1, 9d) \
+ V(vfnmadd213sd, L128, 66, 0F, 38, W1, ad) \
+ V(vfnmadd231sd, L128, 66, 0F, 38, W1, bd) \
+ V(vfnmsub132sd, L128, 66, 0F, 38, W1, 9f) \
+ V(vfnmsub213sd, L128, 66, 0F, 38, W1, af) \
+ V(vfnmsub231sd, L128, 66, 0F, 38, W1, bf) \
+ V(vfmadd132ss, LIG, 66, 0F, 38, W0, 99) \
+ V(vfmadd213ss, LIG, 66, 0F, 38, W0, a9) \
+ V(vfmadd231ss, LIG, 66, 0F, 38, W0, b9) \
+ V(vfmsub132ss, LIG, 66, 0F, 38, W0, 9b) \
+ V(vfmsub213ss, LIG, 66, 0F, 38, W0, ab) \
+ V(vfmsub231ss, LIG, 66, 0F, 38, W0, bb) \
+ V(vfnmadd132ss, LIG, 66, 0F, 38, W0, 9d) \
+ V(vfnmadd213ss, LIG, 66, 0F, 38, W0, ad) \
+ V(vfnmadd231ss, LIG, 66, 0F, 38, W0, bd) \
+ V(vfnmsub132ss, LIG, 66, 0F, 38, W0, 9f) \
+ V(vfnmsub213ss, LIG, 66, 0F, 38, W0, af) \
+ V(vfnmsub231ss, LIG, 66, 0F, 38, W0, bf) \
+ V(vfmadd231ps, L128, 66, 0F, 38, W0, b8) \
+ V(vfnmadd231ps, L128, 66, 0F, 38, W0, bc) \
+ V(vfmadd231pd, L128, 66, 0F, 38, W1, b8) \
+ V(vfnmadd231pd, L128, 66, 0F, 38, W1, bc)
+
+#endif // V8_CODEGEN_X64_FMA_INSTR_H_
diff --git a/src/codegen/x64/interface-descriptors-x64.cc b/src/codegen/x64/interface-descriptors-x64.cc
new file mode 100644
index 0000000..e4d6b92
--- /dev/null
+++ b/src/codegen/x64/interface-descriptors-x64.cc
@@ -0,0 +1,286 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#if V8_TARGET_ARCH_X64
+
+#include "src/codegen/interface-descriptors.h"
+
+#include "src/execution/frames.h"
+
+namespace v8 {
+namespace internal {
+
+const Register CallInterfaceDescriptor::ContextRegister() { return rsi; }
+
+void CallInterfaceDescriptor::DefaultInitializePlatformSpecific(
+ CallInterfaceDescriptorData* data, int register_parameter_count) {
+ const Register default_stub_registers[] = {rax, rbx, rcx, rdx, rdi};
+ CHECK_LE(static_cast<size_t>(register_parameter_count),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(register_parameter_count,
+ default_stub_registers);
+}
+
+void RecordWriteDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ const Register default_stub_registers[] = {arg_reg_1, arg_reg_2, arg_reg_3,
+ arg_reg_4, kReturnRegister0};
+
+ data->RestrictAllocatableRegisters(default_stub_registers,
+ arraysize(default_stub_registers));
+
+ CHECK_LE(static_cast<size_t>(kParameterCount),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(kParameterCount, default_stub_registers);
+}
+
+void EphemeronKeyBarrierDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ const Register default_stub_registers[] = {arg_reg_1, arg_reg_2, arg_reg_3,
+ arg_reg_4, kReturnRegister0};
+
+ data->RestrictAllocatableRegisters(default_stub_registers,
+ arraysize(default_stub_registers));
+
+ CHECK_LE(static_cast<size_t>(kParameterCount),
+ arraysize(default_stub_registers));
+ data->InitializePlatformSpecific(kParameterCount, default_stub_registers);
+}
+
+const Register LoadDescriptor::ReceiverRegister() { return rdx; }
+const Register LoadDescriptor::NameRegister() { return rcx; }
+const Register LoadDescriptor::SlotRegister() { return rax; }
+
+const Register LoadWithVectorDescriptor::VectorRegister() { return rbx; }
+
+const Register
+LoadWithReceiverAndVectorDescriptor::LookupStartObjectRegister() {
+ return rdi;
+}
+
+const Register StoreDescriptor::ReceiverRegister() { return rdx; }
+const Register StoreDescriptor::NameRegister() { return rcx; }
+const Register StoreDescriptor::ValueRegister() { return rax; }
+const Register StoreDescriptor::SlotRegister() { return rdi; }
+
+const Register StoreWithVectorDescriptor::VectorRegister() { return rbx; }
+
+const Register StoreTransitionDescriptor::SlotRegister() { return rdi; }
+const Register StoreTransitionDescriptor::VectorRegister() { return rbx; }
+const Register StoreTransitionDescriptor::MapRegister() { return r11; }
+
+const Register ApiGetterDescriptor::HolderRegister() { return rcx; }
+const Register ApiGetterDescriptor::CallbackRegister() { return rbx; }
+
+const Register GrowArrayElementsDescriptor::ObjectRegister() { return rax; }
+const Register GrowArrayElementsDescriptor::KeyRegister() { return rbx; }
+
+void TypeofDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {rbx};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+// static
+const Register TypeConversionDescriptor::ArgumentRegister() { return rax; }
+
+void CallTrampolineDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // rax : number of arguments
+ // rdi : the target to call
+ Register registers[] = {rdi, rax};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // rax : number of arguments (on the stack, not including receiver)
+ // rdi : the target to call
+ // rcx : arguments list length (untagged)
+ // rbx : arguments list (FixedArray)
+ Register registers[] = {rdi, rax, rcx, rbx};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallForwardVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // rax : number of arguments
+ // rcx : start index (to support rest parameters)
+ // rdi : the target to call
+ Register registers[] = {rdi, rax, rcx};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallFunctionTemplateDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // rdx: the function template info
+ // rcx: number of arguments (on the stack, not including receiver)
+ Register registers[] = {rdx, rcx};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallWithSpreadDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // rax : number of arguments (on the stack, not including receiver)
+ // rdi : the target to call
+ // rbx : the object to spread
+ Register registers[] = {rdi, rax, rbx};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CallWithArrayLikeDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // rdi : the target to call
+ // rbx : the arguments list
+ Register registers[] = {rdi, rbx};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // rax : number of arguments (on the stack, not including receiver)
+ // rdi : the target to call
+ // rdx : the new target
+ // rcx : arguments list length (untagged)
+ // rbx : arguments list (FixedArray)
+ Register registers[] = {rdi, rdx, rax, rcx, rbx};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructForwardVarargsDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // rax : number of arguments
+ // rdx : the new target
+ // rcx : start index (to support rest parameters)
+ // rdi : the target to call
+ Register registers[] = {rdi, rdx, rax, rcx};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructWithSpreadDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // rax : number of arguments (on the stack, not including receiver)
+ // rdi : the target to call
+ // rdx : the new target
+ // rbx : the object to spread
+ Register registers[] = {rdi, rdx, rax, rbx};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructWithArrayLikeDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // rdi : the target to call
+ // rdx : the new target
+ // rbx : the arguments list
+ Register registers[] = {rdi, rdx, rbx};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ConstructStubDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ // rax : number of arguments
+ // rdx : the new target
+ // rdi : the target to call
+ // rbx : allocation site or undefined
+ Register registers[] = {rdi, rdx, rax, rbx};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void AbortDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {rdx};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void CompareDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {rdx, rax};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void BinaryOpDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {rdx, rax};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ArgumentsAdaptorDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ rdi, // JSFunction
+ rdx, // the new target
+ rax, // actual number of arguments
+ rbx, // expected number of arguments
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ApiCallbackDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ rdx, // api function address
+ rcx, // argument count (not including receiver)
+ rbx, // call data
+ rdi, // holder
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void InterpreterDispatchDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ kInterpreterAccumulatorRegister, kInterpreterBytecodeOffsetRegister,
+ kInterpreterBytecodeArrayRegister, kInterpreterDispatchTableRegister};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void InterpreterPushArgsThenCallDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ rax, // argument count (not including receiver)
+ rbx, // address of first argument
+ rdi // the target callable to be call
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void InterpreterPushArgsThenConstructDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ rax, // argument count (not including receiver)
+ rcx, // address of first argument
+ rdi, // constructor to call
+ rdx, // new target
+ rbx, // allocation site feedback if available, undefined otherwise
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void ResumeGeneratorDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ rax, // the value to pass to the generator
+ rdx // the JSGeneratorObject / JSAsyncGeneratorObject to resume
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void FrameDropperTrampolineDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {
+ rbx, // loaded new FP
+ };
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+void RunMicrotasksEntryDescriptor::InitializePlatformSpecific(
+ CallInterfaceDescriptorData* data) {
+ Register registers[] = {arg_reg_1, arg_reg_2};
+ data->InitializePlatformSpecific(arraysize(registers), registers);
+}
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_X64
diff --git a/src/codegen/x64/macro-assembler-x64.cc b/src/codegen/x64/macro-assembler-x64.cc
new file mode 100644
index 0000000..9f5917c
--- /dev/null
+++ b/src/codegen/x64/macro-assembler-x64.cc
@@ -0,0 +1,2979 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#if V8_TARGET_ARCH_X64
+
+#include "src/base/bits.h"
+#include "src/base/division-by-constant.h"
+#include "src/base/utils/random-number-generator.h"
+#include "src/codegen/callable.h"
+#include "src/codegen/code-factory.h"
+#include "src/codegen/external-reference-table.h"
+#include "src/codegen/macro-assembler.h"
+#include "src/codegen/register-configuration.h"
+#include "src/codegen/string-constants.h"
+#include "src/codegen/x64/assembler-x64.h"
+#include "src/common/external-pointer.h"
+#include "src/common/globals.h"
+#include "src/debug/debug.h"
+#include "src/execution/frames-inl.h"
+#include "src/heap/memory-chunk.h"
+#include "src/init/bootstrapper.h"
+#include "src/logging/counters.h"
+#include "src/objects/objects-inl.h"
+#include "src/objects/smi.h"
+#include "src/snapshot/embedded/embedded-data.h"
+#include "src/snapshot/snapshot.h"
+
+// Satisfy cpplint check, but don't include platform-specific header. It is
+// included recursively via macro-assembler.h.
+#if 0
+#include "src/codegen/x64/macro-assembler-x64.h"
+#endif
+
+namespace v8 {
+namespace internal {
+
+Operand StackArgumentsAccessor::GetArgumentOperand(int index) const {
+ DCHECK_GE(index, 0);
+ // arg[0] = rsp + kPCOnStackSize;
+ // arg[i] = arg[0] + i * kSystemPointerSize;
+ return Operand(rsp, kPCOnStackSize + index * kSystemPointerSize);
+}
+
+void MacroAssembler::Load(Register destination, ExternalReference source) {
+ if (root_array_available_ && options().enable_root_array_delta_access) {
+ intptr_t delta = RootRegisterOffsetForExternalReference(isolate(), source);
+ if (is_int32(delta)) {
+ movq(destination, Operand(kRootRegister, static_cast<int32_t>(delta)));
+ return;
+ }
+ }
+ // Safe code.
+ if (destination == rax && !options().isolate_independent_code) {
+ load_rax(source);
+ } else {
+ movq(destination, ExternalReferenceAsOperand(source));
+ }
+}
+
+void MacroAssembler::Store(ExternalReference destination, Register source) {
+ if (root_array_available_ && options().enable_root_array_delta_access) {
+ intptr_t delta =
+ RootRegisterOffsetForExternalReference(isolate(), destination);
+ if (is_int32(delta)) {
+ movq(Operand(kRootRegister, static_cast<int32_t>(delta)), source);
+ return;
+ }
+ }
+ // Safe code.
+ if (source == rax && !options().isolate_independent_code) {
+ store_rax(destination);
+ } else {
+ movq(ExternalReferenceAsOperand(destination), source);
+ }
+}
+
+void TurboAssembler::LoadFromConstantsTable(Register destination,
+ int constant_index) {
+ DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kBuiltinsConstantsTable));
+ LoadRoot(destination, RootIndex::kBuiltinsConstantsTable);
+ LoadTaggedPointerField(
+ destination,
+ FieldOperand(destination, FixedArray::OffsetOfElementAt(constant_index)));
+}
+
+void TurboAssembler::LoadRootRegisterOffset(Register destination,
+ intptr_t offset) {
+ DCHECK(is_int32(offset));
+ if (offset == 0) {
+ Move(destination, kRootRegister);
+ } else {
+ leaq(destination, Operand(kRootRegister, static_cast<int32_t>(offset)));
+ }
+}
+
+void TurboAssembler::LoadRootRelative(Register destination, int32_t offset) {
+ movq(destination, Operand(kRootRegister, offset));
+}
+
+void TurboAssembler::LoadAddress(Register destination,
+ ExternalReference source) {
+ if (root_array_available_ && options().enable_root_array_delta_access) {
+ intptr_t delta = RootRegisterOffsetForExternalReference(isolate(), source);
+ if (is_int32(delta)) {
+ leaq(destination, Operand(kRootRegister, static_cast<int32_t>(delta)));
+ return;
+ }
+ }
+ // Safe code.
+ // TODO(jgruber,v8:8887): Also consider a root-relative load when generating
+ // non-isolate-independent code. In many cases it might be cheaper than
+ // embedding the relocatable value.
+ if (root_array_available_ && options().isolate_independent_code) {
+ IndirectLoadExternalReference(destination, source);
+ return;
+ }
+ Move(destination, source);
+}
+
+Operand TurboAssembler::ExternalReferenceAsOperand(ExternalReference reference,
+ Register scratch) {
+ if (root_array_available_ && options().enable_root_array_delta_access) {
+ int64_t delta =
+ RootRegisterOffsetForExternalReference(isolate(), reference);
+ if (is_int32(delta)) {
+ return Operand(kRootRegister, static_cast<int32_t>(delta));
+ }
+ }
+ if (root_array_available_ && options().isolate_independent_code) {
+ if (IsAddressableThroughRootRegister(isolate(), reference)) {
+ // Some external references can be efficiently loaded as an offset from
+ // kRootRegister.
+ intptr_t offset =
+ RootRegisterOffsetForExternalReference(isolate(), reference);
+ CHECK(is_int32(offset));
+ return Operand(kRootRegister, static_cast<int32_t>(offset));
+ } else {
+ // Otherwise, do a memory load from the external reference table.
+ movq(scratch, Operand(kRootRegister,
+ RootRegisterOffsetForExternalReferenceTableEntry(
+ isolate(), reference)));
+ return Operand(scratch, 0);
+ }
+ }
+ Move(scratch, reference);
+ return Operand(scratch, 0);
+}
+
+void MacroAssembler::PushAddress(ExternalReference source) {
+ LoadAddress(kScratchRegister, source);
+ Push(kScratchRegister);
+}
+
+void TurboAssembler::LoadRoot(Register destination, RootIndex index) {
+ DCHECK(root_array_available_);
+ movq(destination,
+ Operand(kRootRegister, RootRegisterOffsetForRootIndex(index)));
+}
+
+void MacroAssembler::PushRoot(RootIndex index) {
+ DCHECK(root_array_available_);
+ Push(Operand(kRootRegister, RootRegisterOffsetForRootIndex(index)));
+}
+
+void TurboAssembler::CompareRoot(Register with, RootIndex index) {
+ DCHECK(root_array_available_);
+ if (base::IsInRange(index, RootIndex::kFirstStrongOrReadOnlyRoot,
+ RootIndex::kLastStrongOrReadOnlyRoot)) {
+ cmp_tagged(with,
+ Operand(kRootRegister, RootRegisterOffsetForRootIndex(index)));
+ } else {
+ // Some smi roots contain system pointer size values like stack limits.
+ cmpq(with, Operand(kRootRegister, RootRegisterOffsetForRootIndex(index)));
+ }
+}
+
+void TurboAssembler::CompareRoot(Operand with, RootIndex index) {
+ DCHECK(root_array_available_);
+ DCHECK(!with.AddressUsesRegister(kScratchRegister));
+ LoadRoot(kScratchRegister, index);
+ if (base::IsInRange(index, RootIndex::kFirstStrongOrReadOnlyRoot,
+ RootIndex::kLastStrongOrReadOnlyRoot)) {
+ cmp_tagged(with, kScratchRegister);
+ } else {
+ // Some smi roots contain system pointer size values like stack limits.
+ cmpq(with, kScratchRegister);
+ }
+}
+
+void TurboAssembler::LoadMap(Register destination, Register object) {
+ LoadTaggedPointerField(destination,
+ FieldOperand(object, HeapObject::kMapOffset));
+}
+
+void TurboAssembler::LoadTaggedPointerField(Register destination,
+ Operand field_operand) {
+ if (COMPRESS_POINTERS_BOOL) {
+ DecompressTaggedPointer(destination, field_operand);
+ } else {
+ mov_tagged(destination, field_operand);
+ }
+}
+
+void TurboAssembler::LoadAnyTaggedField(Register destination,
+ Operand field_operand) {
+ if (COMPRESS_POINTERS_BOOL) {
+ DecompressAnyTagged(destination, field_operand);
+ } else {
+ mov_tagged(destination, field_operand);
+ }
+}
+
+void TurboAssembler::PushTaggedPointerField(Operand field_operand,
+ Register scratch) {
+ if (COMPRESS_POINTERS_BOOL) {
+ DCHECK(!field_operand.AddressUsesRegister(scratch));
+ DecompressTaggedPointer(scratch, field_operand);
+ Push(scratch);
+ } else {
+ Push(field_operand);
+ }
+}
+
+void TurboAssembler::PushTaggedAnyField(Operand field_operand,
+ Register scratch) {
+ if (COMPRESS_POINTERS_BOOL) {
+ DCHECK(!field_operand.AddressUsesRegister(scratch));
+ DecompressAnyTagged(scratch, field_operand);
+ Push(scratch);
+ } else {
+ Push(field_operand);
+ }
+}
+
+void TurboAssembler::SmiUntagField(Register dst, Operand src) {
+ SmiUntag(dst, src);
+}
+
+void TurboAssembler::StoreTaggedField(Operand dst_field_operand,
+ Immediate value) {
+ if (COMPRESS_POINTERS_BOOL) {
+ movl(dst_field_operand, value);
+ } else {
+ movq(dst_field_operand, value);
+ }
+}
+
+void TurboAssembler::StoreTaggedField(Operand dst_field_operand,
+ Register value) {
+ if (COMPRESS_POINTERS_BOOL) {
+ movl(dst_field_operand, value);
+ } else {
+ movq(dst_field_operand, value);
+ }
+}
+
+void TurboAssembler::DecompressTaggedSigned(Register destination,
+ Operand field_operand) {
+ RecordComment("[ DecompressTaggedSigned");
+ movl(destination, field_operand);
+ RecordComment("]");
+}
+
+void TurboAssembler::DecompressTaggedPointer(Register destination,
+ Operand field_operand) {
+ RecordComment("[ DecompressTaggedPointer");
+ movl(destination, field_operand);
+ addq(destination, kRootRegister);
+ RecordComment("]");
+}
+
+void TurboAssembler::DecompressTaggedPointer(Register destination,
+ Register source) {
+ RecordComment("[ DecompressTaggedPointer");
+ movl(destination, source);
+ addq(destination, kRootRegister);
+ RecordComment("]");
+}
+
+void TurboAssembler::DecompressAnyTagged(Register destination,
+ Operand field_operand) {
+ RecordComment("[ DecompressAnyTagged");
+ movl(destination, field_operand);
+ addq(destination, kRootRegister);
+ RecordComment("]");
+}
+
+void MacroAssembler::RecordWriteField(Register object, int offset,
+ Register value, Register dst,
+ SaveFPRegsMode save_fp,
+ RememberedSetAction remembered_set_action,
+ SmiCheck smi_check) {
+ // First, check if a write barrier is even needed. The tests below
+ // catch stores of Smis.
+ Label done;
+
+ // Skip barrier if writing a smi.
+ if (smi_check == INLINE_SMI_CHECK) {
+ JumpIfSmi(value, &done);
+ }
+
+ // Although the object register is tagged, the offset is relative to the start
+ // of the object, so the offset must be a multiple of kTaggedSize.
+ DCHECK(IsAligned(offset, kTaggedSize));
+
+ leaq(dst, FieldOperand(object, offset));
+ if (emit_debug_code()) {
+ Label ok;
+ testb(dst, Immediate(kTaggedSize - 1));
+ j(zero, &ok, Label::kNear);
+ int3();
+ bind(&ok);
+ }
+
+ RecordWrite(object, dst, value, save_fp, remembered_set_action,
+ OMIT_SMI_CHECK);
+
+ bind(&done);
+
+ // Clobber clobbered input registers when running with the debug-code flag
+ // turned on to provoke errors.
+ if (emit_debug_code()) {
+ Move(value, kZapValue, RelocInfo::NONE);
+ Move(dst, kZapValue, RelocInfo::NONE);
+ }
+}
+
+void TurboAssembler::SaveRegisters(RegList registers) {
+ DCHECK_GT(NumRegs(registers), 0);
+ for (int i = 0; i < Register::kNumRegisters; ++i) {
+ if ((registers >> i) & 1u) {
+ pushq(Register::from_code(i));
+ }
+ }
+}
+
+void TurboAssembler::LoadExternalPointerField(Register destination,
+ Operand field_operand,
+ ExternalPointerTag tag) {
+#ifdef V8_HEAP_SANDBOX
+ LoadAddress(kScratchRegister,
+ ExternalReference::external_pointer_table_address(isolate()));
+ movq(kScratchRegister,
+ Operand(kScratchRegister, Internals::kExternalPointerTableBufferOffset));
+ movl(destination, field_operand);
+ movq(destination, Operand(kScratchRegister, destination, times_8, 0));
+ if (tag != 0) {
+ movq(kScratchRegister, Immediate64(tag));
+ xorq(destination, kScratchRegister);
+ }
+#else
+ movq(destination, field_operand);
+#endif // V8_HEAP_SANDBOX
+}
+
+void TurboAssembler::RestoreRegisters(RegList registers) {
+ DCHECK_GT(NumRegs(registers), 0);
+ for (int i = Register::kNumRegisters - 1; i >= 0; --i) {
+ if ((registers >> i) & 1u) {
+ popq(Register::from_code(i));
+ }
+ }
+}
+
+void TurboAssembler::CallEphemeronKeyBarrier(Register object, Register address,
+ SaveFPRegsMode fp_mode) {
+ EphemeronKeyBarrierDescriptor descriptor;
+ RegList registers = descriptor.allocatable_registers();
+
+ SaveRegisters(registers);
+
+ Register object_parameter(
+ descriptor.GetRegisterParameter(EphemeronKeyBarrierDescriptor::kObject));
+ Register slot_parameter(descriptor.GetRegisterParameter(
+ EphemeronKeyBarrierDescriptor::kSlotAddress));
+ Register fp_mode_parameter(
+ descriptor.GetRegisterParameter(EphemeronKeyBarrierDescriptor::kFPMode));
+
+ MovePair(slot_parameter, address, object_parameter, object);
+ Smi smi_fm = Smi::FromEnum(fp_mode);
+ Move(fp_mode_parameter, smi_fm);
+ Call(isolate()->builtins()->builtin_handle(Builtins::kEphemeronKeyBarrier),
+ RelocInfo::CODE_TARGET);
+
+ RestoreRegisters(registers);
+}
+
+void TurboAssembler::CallRecordWriteStub(
+ Register object, Register address,
+ RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode) {
+ CallRecordWriteStub(
+ object, address, remembered_set_action, fp_mode,
+ isolate()->builtins()->builtin_handle(Builtins::kRecordWrite),
+ kNullAddress);
+}
+
+void TurboAssembler::CallRecordWriteStub(
+ Register object, Register address,
+ RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode,
+ Address wasm_target) {
+ CallRecordWriteStub(object, address, remembered_set_action, fp_mode,
+ Handle<Code>::null(), wasm_target);
+}
+
+void TurboAssembler::CallRecordWriteStub(
+ Register object, Register address,
+ RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode,
+ Handle<Code> code_target, Address wasm_target) {
+ DCHECK_NE(code_target.is_null(), wasm_target == kNullAddress);
+
+ RecordWriteDescriptor descriptor;
+ RegList registers = descriptor.allocatable_registers();
+
+ SaveRegisters(registers);
+
+ Register object_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kObject));
+ Register slot_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kSlot));
+ Register remembered_set_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kRememberedSet));
+ Register fp_mode_parameter(
+ descriptor.GetRegisterParameter(RecordWriteDescriptor::kFPMode));
+
+ // Prepare argument registers for calling RecordWrite
+ // slot_parameter <= address
+ // object_parameter <= object
+ MovePair(slot_parameter, address, object_parameter, object);
+
+ Smi smi_rsa = Smi::FromEnum(remembered_set_action);
+ Smi smi_fm = Smi::FromEnum(fp_mode);
+ Move(remembered_set_parameter, smi_rsa);
+ if (smi_rsa != smi_fm) {
+ Move(fp_mode_parameter, smi_fm);
+ } else {
+ movq(fp_mode_parameter, remembered_set_parameter);
+ }
+ if (code_target.is_null()) {
+ // Use {near_call} for direct Wasm call within a module.
+ near_call(wasm_target, RelocInfo::WASM_STUB_CALL);
+ } else {
+ Call(code_target, RelocInfo::CODE_TARGET);
+ }
+
+ RestoreRegisters(registers);
+}
+
+void MacroAssembler::RecordWrite(Register object, Register address,
+ Register value, SaveFPRegsMode fp_mode,
+ RememberedSetAction remembered_set_action,
+ SmiCheck smi_check) {
+ DCHECK(object != value);
+ DCHECK(object != address);
+ DCHECK(value != address);
+ AssertNotSmi(object);
+
+ if ((remembered_set_action == OMIT_REMEMBERED_SET &&
+ !FLAG_incremental_marking) ||
+ FLAG_disable_write_barriers) {
+ return;
+ }
+
+ if (emit_debug_code()) {
+ Label ok;
+ cmp_tagged(value, Operand(address, 0));
+ j(equal, &ok, Label::kNear);
+ int3();
+ bind(&ok);
+ }
+
+ // First, check if a write barrier is even needed. The tests below
+ // catch stores of smis and stores into the young generation.
+ Label done;
+
+ if (smi_check == INLINE_SMI_CHECK) {
+ // Skip barrier if writing a smi.
+ JumpIfSmi(value, &done);
+ }
+
+ CheckPageFlag(value,
+ value, // Used as scratch.
+ MemoryChunk::kPointersToHereAreInterestingMask, zero, &done,
+ Label::kNear);
+
+ CheckPageFlag(object,
+ value, // Used as scratch.
+ MemoryChunk::kPointersFromHereAreInterestingMask, zero, &done,
+ Label::kNear);
+
+ CallRecordWriteStub(object, address, remembered_set_action, fp_mode);
+
+ bind(&done);
+
+ // Clobber clobbered registers when running with the debug-code flag
+ // turned on to provoke errors.
+ if (emit_debug_code()) {
+ Move(address, kZapValue, RelocInfo::NONE);
+ Move(value, kZapValue, RelocInfo::NONE);
+ }
+}
+
+void TurboAssembler::Assert(Condition cc, AbortReason reason) {
+ if (emit_debug_code()) Check(cc, reason);
+}
+
+void TurboAssembler::AssertUnreachable(AbortReason reason) {
+ if (emit_debug_code()) Abort(reason);
+}
+
+void TurboAssembler::Check(Condition cc, AbortReason reason) {
+ Label L;
+ j(cc, &L, Label::kNear);
+ Abort(reason);
+ // Control will not return here.
+ bind(&L);
+}
+
+void TurboAssembler::CheckStackAlignment() {
+ int frame_alignment = base::OS::ActivationFrameAlignment();
+ int frame_alignment_mask = frame_alignment - 1;
+ if (frame_alignment > kSystemPointerSize) {
+ DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
+ Label alignment_as_expected;
+ testq(rsp, Immediate(frame_alignment_mask));
+ j(zero, &alignment_as_expected, Label::kNear);
+ // Abort if stack is not aligned.
+ int3();
+ bind(&alignment_as_expected);
+ }
+}
+
+void TurboAssembler::Abort(AbortReason reason) {
+#ifdef DEBUG
+ const char* msg = GetAbortReason(reason);
+ RecordComment("Abort message: ");
+ RecordComment(msg);
+#endif
+
+ // Avoid emitting call to builtin if requested.
+ if (trap_on_abort()) {
+ int3();
+ return;
+ }
+
+ if (should_abort_hard()) {
+ // We don't care if we constructed a frame. Just pretend we did.
+ FrameScope assume_frame(this, StackFrame::NONE);
+ movl(arg_reg_1, Immediate(static_cast<int>(reason)));
+ PrepareCallCFunction(1);
+ LoadAddress(rax, ExternalReference::abort_with_reason());
+ call(rax);
+ return;
+ }
+
+ Move(rdx, Smi::FromInt(static_cast<int>(reason)));
+
+ if (!has_frame()) {
+ // We don't actually want to generate a pile of code for this, so just
+ // claim there is a stack frame, without generating one.
+ FrameScope scope(this, StackFrame::NONE);
+ Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
+ } else {
+ Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
+ }
+ // Control will not return here.
+ int3();
+}
+
+void MacroAssembler::CallRuntime(const Runtime::Function* f, int num_arguments,
+ SaveFPRegsMode save_doubles) {
+ // If the expected number of arguments of the runtime function is
+ // constant, we check that the actual number of arguments match the
+ // expectation.
+ CHECK(f->nargs < 0 || f->nargs == num_arguments);
+
+ // TODO(1236192): Most runtime routines don't need the number of
+ // arguments passed in because it is constant. At some point we
+ // should remove this need and make the runtime routine entry code
+ // smarter.
+ Set(rax, num_arguments);
+ LoadAddress(rbx, ExternalReference::Create(f));
+ Handle<Code> code =
+ CodeFactory::CEntry(isolate(), f->result_size, save_doubles);
+ Call(code, RelocInfo::CODE_TARGET);
+}
+
+void MacroAssembler::TailCallRuntime(Runtime::FunctionId fid) {
+ // ----------- S t a t e -------------
+ // -- rsp[0] : return address
+ // -- rsp[8] : argument num_arguments - 1
+ // ...
+ // -- rsp[8 * num_arguments] : argument 0 (receiver)
+ //
+ // For runtime functions with variable arguments:
+ // -- rax : number of arguments
+ // -----------------------------------
+
+ const Runtime::Function* function = Runtime::FunctionForId(fid);
+ DCHECK_EQ(1, function->result_size);
+ if (function->nargs >= 0) {
+ Set(rax, function->nargs);
+ }
+ JumpToExternalReference(ExternalReference::Create(fid));
+}
+
+void MacroAssembler::JumpToExternalReference(const ExternalReference& ext,
+ bool builtin_exit_frame) {
+ // Set the entry point and jump to the C entry runtime stub.
+ LoadAddress(rbx, ext);
+ Handle<Code> code = CodeFactory::CEntry(isolate(), 1, kDontSaveFPRegs,
+ kArgvOnStack, builtin_exit_frame);
+ Jump(code, RelocInfo::CODE_TARGET);
+}
+
+static constexpr Register saved_regs[] = {rax, rcx, rdx, rbx, rbp, rsi,
+ rdi, r8, r9, r10, r11};
+
+static constexpr int kNumberOfSavedRegs = sizeof(saved_regs) / sizeof(Register);
+
+int TurboAssembler::RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
+ Register exclusion1,
+ Register exclusion2,
+ Register exclusion3) const {
+ int bytes = 0;
+ for (int i = 0; i < kNumberOfSavedRegs; i++) {
+ Register reg = saved_regs[i];
+ if (reg != exclusion1 && reg != exclusion2 && reg != exclusion3) {
+ bytes += kSystemPointerSize;
+ }
+ }
+
+ // R12 to r15 are callee save on all platforms.
+ if (fp_mode == kSaveFPRegs) {
+ bytes += kDoubleSize * XMMRegister::kNumRegisters;
+ }
+
+ return bytes;
+}
+
+int TurboAssembler::PushCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1,
+ Register exclusion2, Register exclusion3) {
+ // We don't allow a GC during a store buffer overflow so there is no need to
+ // store the registers in any particular way, but we do have to store and
+ // restore them.
+ int bytes = 0;
+ for (int i = 0; i < kNumberOfSavedRegs; i++) {
+ Register reg = saved_regs[i];
+ if (reg != exclusion1 && reg != exclusion2 && reg != exclusion3) {
+ pushq(reg);
+ bytes += kSystemPointerSize;
+ }
+ }
+
+ // R12 to r15 are callee save on all platforms.
+ if (fp_mode == kSaveFPRegs) {
+ int delta = kDoubleSize * XMMRegister::kNumRegisters;
+ AllocateStackSpace(delta);
+ for (int i = 0; i < XMMRegister::kNumRegisters; i++) {
+ XMMRegister reg = XMMRegister::from_code(i);
+ Movsd(Operand(rsp, i * kDoubleSize), reg);
+ }
+ bytes += delta;
+ }
+
+ return bytes;
+}
+
+int TurboAssembler::PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1,
+ Register exclusion2, Register exclusion3) {
+ int bytes = 0;
+ if (fp_mode == kSaveFPRegs) {
+ for (int i = 0; i < XMMRegister::kNumRegisters; i++) {
+ XMMRegister reg = XMMRegister::from_code(i);
+ Movsd(reg, Operand(rsp, i * kDoubleSize));
+ }
+ int delta = kDoubleSize * XMMRegister::kNumRegisters;
+ addq(rsp, Immediate(kDoubleSize * XMMRegister::kNumRegisters));
+ bytes += delta;
+ }
+
+ for (int i = kNumberOfSavedRegs - 1; i >= 0; i--) {
+ Register reg = saved_regs[i];
+ if (reg != exclusion1 && reg != exclusion2 && reg != exclusion3) {
+ popq(reg);
+ bytes += kSystemPointerSize;
+ }
+ }
+
+ return bytes;
+}
+
+void TurboAssembler::Cvtss2sd(XMMRegister dst, XMMRegister src) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vcvtss2sd(dst, src, src);
+ } else {
+ cvtss2sd(dst, src);
+ }
+}
+
+void TurboAssembler::Cvtss2sd(XMMRegister dst, Operand src) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vcvtss2sd(dst, dst, src);
+ } else {
+ cvtss2sd(dst, src);
+ }
+}
+
+void TurboAssembler::Cvtsd2ss(XMMRegister dst, XMMRegister src) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vcvtsd2ss(dst, src, src);
+ } else {
+ cvtsd2ss(dst, src);
+ }
+}
+
+void TurboAssembler::Cvtsd2ss(XMMRegister dst, Operand src) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vcvtsd2ss(dst, dst, src);
+ } else {
+ cvtsd2ss(dst, src);
+ }
+}
+
+void TurboAssembler::Cvtlsi2sd(XMMRegister dst, Register src) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vcvtlsi2sd(dst, kScratchDoubleReg, src);
+ } else {
+ xorpd(dst, dst);
+ cvtlsi2sd(dst, src);
+ }
+}
+
+void TurboAssembler::Cvtlsi2sd(XMMRegister dst, Operand src) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vcvtlsi2sd(dst, kScratchDoubleReg, src);
+ } else {
+ xorpd(dst, dst);
+ cvtlsi2sd(dst, src);
+ }
+}
+
+void TurboAssembler::Cvtlsi2ss(XMMRegister dst, Register src) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vcvtlsi2ss(dst, kScratchDoubleReg, src);
+ } else {
+ xorps(dst, dst);
+ cvtlsi2ss(dst, src);
+ }
+}
+
+void TurboAssembler::Cvtlsi2ss(XMMRegister dst, Operand src) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vcvtlsi2ss(dst, kScratchDoubleReg, src);
+ } else {
+ xorps(dst, dst);
+ cvtlsi2ss(dst, src);
+ }
+}
+
+void TurboAssembler::Cvtqsi2ss(XMMRegister dst, Register src) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vcvtqsi2ss(dst, kScratchDoubleReg, src);
+ } else {
+ xorps(dst, dst);
+ cvtqsi2ss(dst, src);
+ }
+}
+
+void TurboAssembler::Cvtqsi2ss(XMMRegister dst, Operand src) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vcvtqsi2ss(dst, kScratchDoubleReg, src);
+ } else {
+ xorps(dst, dst);
+ cvtqsi2ss(dst, src);
+ }
+}
+
+void TurboAssembler::Cvtqsi2sd(XMMRegister dst, Register src) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vcvtqsi2sd(dst, kScratchDoubleReg, src);
+ } else {
+ xorpd(dst, dst);
+ cvtqsi2sd(dst, src);
+ }
+}
+
+void TurboAssembler::Cvtqsi2sd(XMMRegister dst, Operand src) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vcvtqsi2sd(dst, kScratchDoubleReg, src);
+ } else {
+ xorpd(dst, dst);
+ cvtqsi2sd(dst, src);
+ }
+}
+
+void TurboAssembler::Cvtlui2ss(XMMRegister dst, Register src) {
+ // Zero-extend the 32 bit value to 64 bit.
+ movl(kScratchRegister, src);
+ Cvtqsi2ss(dst, kScratchRegister);
+}
+
+void TurboAssembler::Cvtlui2ss(XMMRegister dst, Operand src) {
+ // Zero-extend the 32 bit value to 64 bit.
+ movl(kScratchRegister, src);
+ Cvtqsi2ss(dst, kScratchRegister);
+}
+
+void TurboAssembler::Cvtlui2sd(XMMRegister dst, Register src) {
+ // Zero-extend the 32 bit value to 64 bit.
+ movl(kScratchRegister, src);
+ Cvtqsi2sd(dst, kScratchRegister);
+}
+
+void TurboAssembler::Cvtlui2sd(XMMRegister dst, Operand src) {
+ // Zero-extend the 32 bit value to 64 bit.
+ movl(kScratchRegister, src);
+ Cvtqsi2sd(dst, kScratchRegister);
+}
+
+void TurboAssembler::Cvtqui2ss(XMMRegister dst, Register src) {
+ Label done;
+ Cvtqsi2ss(dst, src);
+ testq(src, src);
+ j(positive, &done, Label::kNear);
+
+ // Compute {src/2 | (src&1)} (retain the LSB to avoid rounding errors).
+ if (src != kScratchRegister) movq(kScratchRegister, src);
+ shrq(kScratchRegister, Immediate(1));
+ // The LSB is shifted into CF. If it is set, set the LSB in {tmp}.
+ Label msb_not_set;
+ j(not_carry, &msb_not_set, Label::kNear);
+ orq(kScratchRegister, Immediate(1));
+ bind(&msb_not_set);
+ Cvtqsi2ss(dst, kScratchRegister);
+ Addss(dst, dst);
+ bind(&done);
+}
+
+void TurboAssembler::Cvtqui2ss(XMMRegister dst, Operand src) {
+ movq(kScratchRegister, src);
+ Cvtqui2ss(dst, kScratchRegister);
+}
+
+void TurboAssembler::Cvtqui2sd(XMMRegister dst, Register src) {
+ Label done;
+ Cvtqsi2sd(dst, src);
+ testq(src, src);
+ j(positive, &done, Label::kNear);
+
+ // Compute {src/2 | (src&1)} (retain the LSB to avoid rounding errors).
+ if (src != kScratchRegister) movq(kScratchRegister, src);
+ shrq(kScratchRegister, Immediate(1));
+ // The LSB is shifted into CF. If it is set, set the LSB in {tmp}.
+ Label msb_not_set;
+ j(not_carry, &msb_not_set, Label::kNear);
+ orq(kScratchRegister, Immediate(1));
+ bind(&msb_not_set);
+ Cvtqsi2sd(dst, kScratchRegister);
+ Addsd(dst, dst);
+ bind(&done);
+}
+
+void TurboAssembler::Cvtqui2sd(XMMRegister dst, Operand src) {
+ movq(kScratchRegister, src);
+ Cvtqui2sd(dst, kScratchRegister);
+}
+
+void TurboAssembler::Cvttss2si(Register dst, XMMRegister src) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vcvttss2si(dst, src);
+ } else {
+ cvttss2si(dst, src);
+ }
+}
+
+void TurboAssembler::Cvttss2si(Register dst, Operand src) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vcvttss2si(dst, src);
+ } else {
+ cvttss2si(dst, src);
+ }
+}
+
+void TurboAssembler::Cvttsd2si(Register dst, XMMRegister src) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vcvttsd2si(dst, src);
+ } else {
+ cvttsd2si(dst, src);
+ }
+}
+
+void TurboAssembler::Cvttsd2si(Register dst, Operand src) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vcvttsd2si(dst, src);
+ } else {
+ cvttsd2si(dst, src);
+ }
+}
+
+void TurboAssembler::Cvttss2siq(Register dst, XMMRegister src) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vcvttss2siq(dst, src);
+ } else {
+ cvttss2siq(dst, src);
+ }
+}
+
+void TurboAssembler::Cvttss2siq(Register dst, Operand src) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vcvttss2siq(dst, src);
+ } else {
+ cvttss2siq(dst, src);
+ }
+}
+
+void TurboAssembler::Cvttsd2siq(Register dst, XMMRegister src) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vcvttsd2siq(dst, src);
+ } else {
+ cvttsd2siq(dst, src);
+ }
+}
+
+void TurboAssembler::Cvttsd2siq(Register dst, Operand src) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vcvttsd2siq(dst, src);
+ } else {
+ cvttsd2siq(dst, src);
+ }
+}
+
+namespace {
+template <typename OperandOrXMMRegister, bool is_double>
+void ConvertFloatToUint64(TurboAssembler* tasm, Register dst,
+ OperandOrXMMRegister src, Label* fail) {
+ Label success;
+ // There does not exist a native float-to-uint instruction, so we have to use
+ // a float-to-int, and postprocess the result.
+ if (is_double) {
+ tasm->Cvttsd2siq(dst, src);
+ } else {
+ tasm->Cvttss2siq(dst, src);
+ }
+ // If the result of the conversion is positive, we are already done.
+ tasm->testq(dst, dst);
+ tasm->j(positive, &success);
+ // The result of the first conversion was negative, which means that the
+ // input value was not within the positive int64 range. We subtract 2^63
+ // and convert it again to see if it is within the uint64 range.
+ if (is_double) {
+ tasm->Move(kScratchDoubleReg, -9223372036854775808.0);
+ tasm->Addsd(kScratchDoubleReg, src);
+ tasm->Cvttsd2siq(dst, kScratchDoubleReg);
+ } else {
+ tasm->Move(kScratchDoubleReg, -9223372036854775808.0f);
+ tasm->Addss(kScratchDoubleReg, src);
+ tasm->Cvttss2siq(dst, kScratchDoubleReg);
+ }
+ tasm->testq(dst, dst);
+ // The only possible negative value here is 0x80000000000000000, which is
+ // used on x64 to indicate an integer overflow.
+ tasm->j(negative, fail ? fail : &success);
+ // The input value is within uint64 range and the second conversion worked
+ // successfully, but we still have to undo the subtraction we did
+ // earlier.
+ tasm->Set(kScratchRegister, 0x8000000000000000);
+ tasm->orq(dst, kScratchRegister);
+ tasm->bind(&success);
+}
+} // namespace
+
+void TurboAssembler::Cvttsd2uiq(Register dst, Operand src, Label* fail) {
+ ConvertFloatToUint64<Operand, true>(this, dst, src, fail);
+}
+
+void TurboAssembler::Cvttsd2uiq(Register dst, XMMRegister src, Label* fail) {
+ ConvertFloatToUint64<XMMRegister, true>(this, dst, src, fail);
+}
+
+void TurboAssembler::Cvttss2uiq(Register dst, Operand src, Label* fail) {
+ ConvertFloatToUint64<Operand, false>(this, dst, src, fail);
+}
+
+void TurboAssembler::Cvttss2uiq(Register dst, XMMRegister src, Label* fail) {
+ ConvertFloatToUint64<XMMRegister, false>(this, dst, src, fail);
+}
+
+void TurboAssembler::Set(Register dst, int64_t x) {
+ if (x == 0) {
+ xorl(dst, dst);
+ } else if (is_uint32(x)) {
+ movl(dst, Immediate(static_cast<uint32_t>(x)));
+ } else if (is_int32(x)) {
+ movq(dst, Immediate(static_cast<int32_t>(x)));
+ } else {
+ movq(dst, x);
+ }
+}
+
+void TurboAssembler::Set(Operand dst, intptr_t x) {
+ if (is_int32(x)) {
+ movq(dst, Immediate(static_cast<int32_t>(x)));
+ } else {
+ Set(kScratchRegister, x);
+ movq(dst, kScratchRegister);
+ }
+}
+
+// ----------------------------------------------------------------------------
+// Smi tagging, untagging and tag detection.
+
+Register TurboAssembler::GetSmiConstant(Smi source) {
+ STATIC_ASSERT(kSmiTag == 0);
+ int value = source.value();
+ if (value == 0) {
+ xorl(kScratchRegister, kScratchRegister);
+ return kScratchRegister;
+ }
+ if (SmiValuesAre32Bits()) {
+ Move(kScratchRegister, source);
+ } else {
+ movl(kScratchRegister, Immediate(source));
+ }
+ return kScratchRegister;
+}
+
+void TurboAssembler::Move(Register dst, Smi source) {
+ STATIC_ASSERT(kSmiTag == 0);
+ int value = source.value();
+ if (value == 0) {
+ xorl(dst, dst);
+ } else {
+ Move(dst, source.ptr(), RelocInfo::NONE);
+ }
+}
+
+void TurboAssembler::Move(Register dst, ExternalReference ext) {
+ // TODO(jgruber,v8:8887): Also consider a root-relative load when generating
+ // non-isolate-independent code. In many cases it might be cheaper than
+ // embedding the relocatable value.
+ if (root_array_available_ && options().isolate_independent_code) {
+ IndirectLoadExternalReference(dst, ext);
+ return;
+ }
+ movq(dst, Immediate64(ext.address(), RelocInfo::EXTERNAL_REFERENCE));
+}
+
+void MacroAssembler::Cmp(Register dst, int32_t src) {
+ if (src == 0) {
+ testl(dst, dst);
+ } else {
+ cmpl(dst, Immediate(src));
+ }
+}
+
+void MacroAssembler::SmiTag(Register reg) {
+ STATIC_ASSERT(kSmiTag == 0);
+ DCHECK(SmiValuesAre32Bits() || SmiValuesAre31Bits());
+ if (COMPRESS_POINTERS_BOOL) {
+ shll(reg, Immediate(kSmiShift));
+ } else {
+ shlq(reg, Immediate(kSmiShift));
+ }
+}
+
+void MacroAssembler::SmiTag(Register dst, Register src) {
+ DCHECK(dst != src);
+ if (COMPRESS_POINTERS_BOOL) {
+ movl(dst, src);
+ } else {
+ movq(dst, src);
+ }
+ SmiTag(dst);
+}
+
+void TurboAssembler::SmiUntag(Register reg) {
+ STATIC_ASSERT(kSmiTag == 0);
+ DCHECK(SmiValuesAre32Bits() || SmiValuesAre31Bits());
+ // TODO(v8:7703): Is there a way to avoid this sign extension when pointer
+ // compression is enabled?
+ if (COMPRESS_POINTERS_BOOL) {
+ movsxlq(reg, reg);
+ }
+ sarq(reg, Immediate(kSmiShift));
+}
+
+void TurboAssembler::SmiUntag(Register dst, Register src) {
+ DCHECK(dst != src);
+ if (COMPRESS_POINTERS_BOOL) {
+ movsxlq(dst, src);
+ } else {
+ movq(dst, src);
+ }
+ // TODO(v8:7703): Call SmiUntag(reg) if we can find a way to avoid the extra
+ // mov when pointer compression is enabled.
+ STATIC_ASSERT(kSmiTag == 0);
+ DCHECK(SmiValuesAre32Bits() || SmiValuesAre31Bits());
+ sarq(dst, Immediate(kSmiShift));
+}
+
+void TurboAssembler::SmiUntag(Register dst, Operand src) {
+ if (SmiValuesAre32Bits()) {
+ movl(dst, Operand(src, kSmiShift / kBitsPerByte));
+ // Sign extend to 64-bit.
+ movsxlq(dst, dst);
+ } else {
+ DCHECK(SmiValuesAre31Bits());
+ if (COMPRESS_POINTERS_BOOL) {
+ movsxlq(dst, src);
+ } else {
+ movq(dst, src);
+ }
+ sarq(dst, Immediate(kSmiShift));
+ }
+}
+
+void MacroAssembler::SmiCompare(Register smi1, Register smi2) {
+ AssertSmi(smi1);
+ AssertSmi(smi2);
+ cmp_tagged(smi1, smi2);
+}
+
+void MacroAssembler::SmiCompare(Register dst, Smi src) {
+ AssertSmi(dst);
+ Cmp(dst, src);
+}
+
+void MacroAssembler::Cmp(Register dst, Smi src) {
+ if (src.value() == 0) {
+ test_tagged(dst, dst);
+ } else {
+ DCHECK_NE(dst, kScratchRegister);
+ Register constant_reg = GetSmiConstant(src);
+ cmp_tagged(dst, constant_reg);
+ }
+}
+
+void MacroAssembler::SmiCompare(Register dst, Operand src) {
+ AssertSmi(dst);
+ AssertSmi(src);
+ cmp_tagged(dst, src);
+}
+
+void MacroAssembler::SmiCompare(Operand dst, Register src) {
+ AssertSmi(dst);
+ AssertSmi(src);
+ cmp_tagged(dst, src);
+}
+
+void MacroAssembler::SmiCompare(Operand dst, Smi src) {
+ AssertSmi(dst);
+ if (SmiValuesAre32Bits()) {
+ cmpl(Operand(dst, kSmiShift / kBitsPerByte), Immediate(src.value()));
+ } else {
+ DCHECK(SmiValuesAre31Bits());
+ cmpl(dst, Immediate(src));
+ }
+}
+
+void MacroAssembler::Cmp(Operand dst, Smi src) {
+ // The Operand cannot use the smi register.
+ Register smi_reg = GetSmiConstant(src);
+ DCHECK(!dst.AddressUsesRegister(smi_reg));
+ cmp_tagged(dst, smi_reg);
+}
+
+Condition TurboAssembler::CheckSmi(Register src) {
+ STATIC_ASSERT(kSmiTag == 0);
+ testb(src, Immediate(kSmiTagMask));
+ return zero;
+}
+
+Condition TurboAssembler::CheckSmi(Operand src) {
+ STATIC_ASSERT(kSmiTag == 0);
+ testb(src, Immediate(kSmiTagMask));
+ return zero;
+}
+
+void TurboAssembler::JumpIfSmi(Register src, Label* on_smi,
+ Label::Distance near_jump) {
+ Condition smi = CheckSmi(src);
+ j(smi, on_smi, near_jump);
+}
+
+void MacroAssembler::JumpIfNotSmi(Register src, Label* on_not_smi,
+ Label::Distance near_jump) {
+ Condition smi = CheckSmi(src);
+ j(NegateCondition(smi), on_not_smi, near_jump);
+}
+
+void MacroAssembler::JumpIfNotSmi(Operand src, Label* on_not_smi,
+ Label::Distance near_jump) {
+ Condition smi = CheckSmi(src);
+ j(NegateCondition(smi), on_not_smi, near_jump);
+}
+
+void MacroAssembler::SmiAddConstant(Operand dst, Smi constant) {
+ if (constant.value() != 0) {
+ if (SmiValuesAre32Bits()) {
+ addl(Operand(dst, kSmiShift / kBitsPerByte), Immediate(constant.value()));
+ } else {
+ DCHECK(SmiValuesAre31Bits());
+ if (kTaggedSize == kInt64Size) {
+ // Sign-extend value after addition
+ movl(kScratchRegister, dst);
+ addl(kScratchRegister, Immediate(constant));
+ movsxlq(kScratchRegister, kScratchRegister);
+ movq(dst, kScratchRegister);
+ } else {
+ DCHECK_EQ(kTaggedSize, kInt32Size);
+ addl(dst, Immediate(constant));
+ }
+ }
+ }
+}
+
+SmiIndex MacroAssembler::SmiToIndex(Register dst, Register src, int shift) {
+ if (SmiValuesAre32Bits()) {
+ DCHECK(is_uint6(shift));
+ // There is a possible optimization if shift is in the range 60-63, but that
+ // will (and must) never happen.
+ if (dst != src) {
+ movq(dst, src);
+ }
+ if (shift < kSmiShift) {
+ sarq(dst, Immediate(kSmiShift - shift));
+ } else {
+ shlq(dst, Immediate(shift - kSmiShift));
+ }
+ return SmiIndex(dst, times_1);
+ } else {
+ DCHECK(SmiValuesAre31Bits());
+ // We have to sign extend the index register to 64-bit as the SMI might
+ // be negative.
+ movsxlq(dst, src);
+ if (shift < kSmiShift) {
+ sarq(dst, Immediate(kSmiShift - shift));
+ } else if (shift != kSmiShift) {
+ if (shift - kSmiShift <= static_cast<int>(times_8)) {
+ return SmiIndex(dst, static_cast<ScaleFactor>(shift - kSmiShift));
+ }
+ shlq(dst, Immediate(shift - kSmiShift));
+ }
+ return SmiIndex(dst, times_1);
+ }
+}
+
+void TurboAssembler::Push(Smi source) {
+ intptr_t smi = static_cast<intptr_t>(source.ptr());
+ if (is_int32(smi)) {
+ Push(Immediate(static_cast<int32_t>(smi)));
+ return;
+ }
+ int first_byte_set = base::bits::CountTrailingZeros64(smi) / 8;
+ int last_byte_set = (63 - base::bits::CountLeadingZeros64(smi)) / 8;
+ if (first_byte_set == last_byte_set) {
+ // This sequence has only 7 bytes, compared to the 12 bytes below.
+ Push(Immediate(0));
+ movb(Operand(rsp, first_byte_set),
+ Immediate(static_cast<int8_t>(smi >> (8 * first_byte_set))));
+ return;
+ }
+ Register constant = GetSmiConstant(source);
+ Push(constant);
+}
+
+// ----------------------------------------------------------------------------
+
+void TurboAssembler::Move(Register dst, Register src) {
+ if (dst != src) {
+ movq(dst, src);
+ }
+}
+
+void TurboAssembler::MovePair(Register dst0, Register src0, Register dst1,
+ Register src1) {
+ if (dst0 != src1) {
+ // Normal case: Writing to dst0 does not destroy src1.
+ Move(dst0, src0);
+ Move(dst1, src1);
+ } else if (dst1 != src0) {
+ // Only dst0 and src1 are the same register,
+ // but writing to dst1 does not destroy src0.
+ Move(dst1, src1);
+ Move(dst0, src0);
+ } else {
+ // dst0 == src1, and dst1 == src0, a swap is required:
+ // dst0 \/ src0
+ // dst1 /\ src1
+ xchgq(dst0, dst1);
+ }
+}
+
+void TurboAssembler::MoveNumber(Register dst, double value) {
+ int32_t smi;
+ if (DoubleToSmiInteger(value, &smi)) {
+ Move(dst, Smi::FromInt(smi));
+ } else {
+ movq_heap_number(dst, value);
+ }
+}
+
+void TurboAssembler::Move(XMMRegister dst, uint32_t src) {
+ if (src == 0) {
+ Xorps(dst, dst);
+ } else {
+ unsigned nlz = base::bits::CountLeadingZeros(src);
+ unsigned ntz = base::bits::CountTrailingZeros(src);
+ unsigned pop = base::bits::CountPopulation(src);
+ DCHECK_NE(0u, pop);
+ if (pop + ntz + nlz == 32) {
+ Pcmpeqd(dst, dst);
+ if (ntz) Pslld(dst, static_cast<byte>(ntz + nlz));
+ if (nlz) Psrld(dst, static_cast<byte>(nlz));
+ } else {
+ movl(kScratchRegister, Immediate(src));
+ Movd(dst, kScratchRegister);
+ }
+ }
+}
+
+void TurboAssembler::Move(XMMRegister dst, uint64_t src) {
+ if (src == 0) {
+ Xorpd(dst, dst);
+ } else {
+ unsigned nlz = base::bits::CountLeadingZeros(src);
+ unsigned ntz = base::bits::CountTrailingZeros(src);
+ unsigned pop = base::bits::CountPopulation(src);
+ DCHECK_NE(0u, pop);
+ if (pop + ntz + nlz == 64) {
+ Pcmpeqd(dst, dst);
+ if (ntz) Psllq(dst, static_cast<byte>(ntz + nlz));
+ if (nlz) Psrlq(dst, static_cast<byte>(nlz));
+ } else {
+ uint32_t lower = static_cast<uint32_t>(src);
+ uint32_t upper = static_cast<uint32_t>(src >> 32);
+ if (upper == 0) {
+ Move(dst, lower);
+ } else {
+ movq(kScratchRegister, src);
+ Movq(dst, kScratchRegister);
+ }
+ }
+ }
+}
+
+void TurboAssembler::Move(XMMRegister dst, uint64_t high, uint64_t low) {
+ Move(dst, low);
+ movq(kScratchRegister, high);
+ Pinsrq(dst, kScratchRegister, uint8_t{1});
+}
+
+// ----------------------------------------------------------------------------
+
+void MacroAssembler::Absps(XMMRegister dst) {
+ Andps(dst, ExternalReferenceAsOperand(
+ ExternalReference::address_of_float_abs_constant()));
+}
+
+void MacroAssembler::Negps(XMMRegister dst) {
+ Xorps(dst, ExternalReferenceAsOperand(
+ ExternalReference::address_of_float_neg_constant()));
+}
+
+void MacroAssembler::Abspd(XMMRegister dst) {
+ Andps(dst, ExternalReferenceAsOperand(
+ ExternalReference::address_of_double_abs_constant()));
+}
+
+void MacroAssembler::Negpd(XMMRegister dst) {
+ Xorps(dst, ExternalReferenceAsOperand(
+ ExternalReference::address_of_double_neg_constant()));
+}
+
+void MacroAssembler::Cmp(Register dst, Handle<Object> source) {
+ if (source->IsSmi()) {
+ Cmp(dst, Smi::cast(*source));
+ } else {
+ Move(kScratchRegister, Handle<HeapObject>::cast(source));
+ cmp_tagged(dst, kScratchRegister);
+ }
+}
+
+void MacroAssembler::Cmp(Operand dst, Handle<Object> source) {
+ if (source->IsSmi()) {
+ Cmp(dst, Smi::cast(*source));
+ } else {
+ Move(kScratchRegister, Handle<HeapObject>::cast(source));
+ cmp_tagged(dst, kScratchRegister);
+ }
+}
+
+void MacroAssembler::JumpIfIsInRange(Register value, unsigned lower_limit,
+ unsigned higher_limit, Label* on_in_range,
+ Label::Distance near_jump) {
+ if (lower_limit != 0) {
+ leal(kScratchRegister, Operand(value, 0u - lower_limit));
+ cmpl(kScratchRegister, Immediate(higher_limit - lower_limit));
+ } else {
+ cmpl(value, Immediate(higher_limit));
+ }
+ j(below_equal, on_in_range, near_jump);
+}
+
+void TurboAssembler::Push(Handle<HeapObject> source) {
+ Move(kScratchRegister, source);
+ Push(kScratchRegister);
+}
+
+void TurboAssembler::PushArray(Register array, Register size, Register scratch,
+ PushArrayOrder order) {
+ DCHECK(!AreAliased(array, size, scratch));
+ Register counter = scratch;
+ Label loop, entry;
+ if (order == PushArrayOrder::kReverse) {
+ Set(counter, 0);
+ jmp(&entry);
+ bind(&loop);
+ Push(Operand(array, counter, times_system_pointer_size, 0));
+ incq(counter);
+ bind(&entry);
+ cmpq(counter, size);
+ j(less, &loop, Label::kNear);
+ } else {
+ movq(counter, size);
+ jmp(&entry);
+ bind(&loop);
+ Push(Operand(array, counter, times_system_pointer_size, 0));
+ bind(&entry);
+ decq(counter);
+ j(greater_equal, &loop, Label::kNear);
+ }
+}
+
+void TurboAssembler::Move(Register result, Handle<HeapObject> object,
+ RelocInfo::Mode rmode) {
+ // TODO(jgruber,v8:8887): Also consider a root-relative load when generating
+ // non-isolate-independent code. In many cases it might be cheaper than
+ // embedding the relocatable value.
+ if (root_array_available_ && options().isolate_independent_code) {
+ // TODO(v8:9706): Fix-it! This load will always uncompress the value
+ // even when we are loading a compressed embedded object.
+ IndirectLoadConstant(result, object);
+ } else if (RelocInfo::IsCompressedEmbeddedObject(rmode)) {
+ EmbeddedObjectIndex index = AddEmbeddedObject(object);
+ DCHECK(is_uint32(index));
+ movl(result, Immediate(static_cast<int>(index), rmode));
+ } else {
+ DCHECK(RelocInfo::IsFullEmbeddedObject(rmode));
+ movq(result, Immediate64(object.address(), rmode));
+ }
+}
+
+void TurboAssembler::Move(Operand dst, Handle<HeapObject> object,
+ RelocInfo::Mode rmode) {
+ Move(kScratchRegister, object, rmode);
+ movq(dst, kScratchRegister);
+}
+
+void TurboAssembler::MoveStringConstant(Register result,
+ const StringConstantBase* string,
+ RelocInfo::Mode rmode) {
+ movq_string(result, string);
+}
+
+void MacroAssembler::Drop(int stack_elements) {
+ if (stack_elements > 0) {
+ addq(rsp, Immediate(stack_elements * kSystemPointerSize));
+ }
+}
+
+void MacroAssembler::DropUnderReturnAddress(int stack_elements,
+ Register scratch) {
+ DCHECK_GT(stack_elements, 0);
+ if (stack_elements == 1) {
+ popq(MemOperand(rsp, 0));
+ return;
+ }
+
+ PopReturnAddressTo(scratch);
+ Drop(stack_elements);
+ PushReturnAddressFrom(scratch);
+}
+
+void TurboAssembler::Push(Register src) { pushq(src); }
+
+void TurboAssembler::Push(Operand src) { pushq(src); }
+
+void MacroAssembler::PushQuad(Operand src) { pushq(src); }
+
+void TurboAssembler::Push(Immediate value) { pushq(value); }
+
+void MacroAssembler::PushImm32(int32_t imm32) { pushq_imm32(imm32); }
+
+void MacroAssembler::Pop(Register dst) { popq(dst); }
+
+void MacroAssembler::Pop(Operand dst) { popq(dst); }
+
+void MacroAssembler::PopQuad(Operand dst) { popq(dst); }
+
+void TurboAssembler::Jump(const ExternalReference& reference) {
+ DCHECK(root_array_available());
+ jmp(Operand(kRootRegister, RootRegisterOffsetForExternalReferenceTableEntry(
+ isolate(), reference)));
+}
+
+void TurboAssembler::Jump(Operand op) { jmp(op); }
+
+void TurboAssembler::Jump(Address destination, RelocInfo::Mode rmode) {
+ Move(kScratchRegister, destination, rmode);
+ jmp(kScratchRegister);
+}
+
+void TurboAssembler::Jump(Handle<Code> code_object, RelocInfo::Mode rmode,
+ Condition cc) {
+ DCHECK_IMPLIES(options().isolate_independent_code,
+ Builtins::IsIsolateIndependentBuiltin(*code_object));
+ if (options().inline_offheap_trampolines) {
+ int builtin_index = Builtins::kNoBuiltinId;
+ if (isolate()->builtins()->IsBuiltinHandle(code_object, &builtin_index)) {
+ Label skip;
+ if (cc != always) {
+ if (cc == never) return;
+ j(NegateCondition(cc), &skip, Label::kNear);
+ }
+ // Inline the trampoline.
+ RecordCommentForOffHeapTrampoline(builtin_index);
+ CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
+ EmbeddedData d = EmbeddedData::FromBlob();
+ Address entry = d.InstructionStartOfBuiltin(builtin_index);
+ Move(kScratchRegister, entry, RelocInfo::OFF_HEAP_TARGET);
+ jmp(kScratchRegister);
+ bind(&skip);
+ return;
+ }
+ }
+ j(cc, code_object, rmode);
+}
+
+void MacroAssembler::JumpToInstructionStream(Address entry) {
+ Move(kOffHeapTrampolineRegister, entry, RelocInfo::OFF_HEAP_TARGET);
+ jmp(kOffHeapTrampolineRegister);
+}
+
+void TurboAssembler::Call(ExternalReference ext) {
+ LoadAddress(kScratchRegister, ext);
+ call(kScratchRegister);
+}
+
+void TurboAssembler::Call(Operand op) {
+ if (!CpuFeatures::IsSupported(ATOM)) {
+ call(op);
+ } else {
+ movq(kScratchRegister, op);
+ call(kScratchRegister);
+ }
+}
+
+void TurboAssembler::Call(Address destination, RelocInfo::Mode rmode) {
+ Move(kScratchRegister, destination, rmode);
+ call(kScratchRegister);
+}
+
+void TurboAssembler::Call(Handle<Code> code_object, RelocInfo::Mode rmode) {
+ DCHECK_IMPLIES(options().isolate_independent_code,
+ Builtins::IsIsolateIndependentBuiltin(*code_object));
+ if (options().inline_offheap_trampolines) {
+ int builtin_index = Builtins::kNoBuiltinId;
+ if (isolate()->builtins()->IsBuiltinHandle(code_object, &builtin_index)) {
+ // Inline the trampoline.
+ CallBuiltin(builtin_index);
+ return;
+ }
+ }
+ DCHECK(RelocInfo::IsCodeTarget(rmode));
+ call(code_object, rmode);
+}
+
+Operand TurboAssembler::EntryFromBuiltinIndexAsOperand(
+ Builtins::Name builtin_index) {
+ DCHECK(root_array_available());
+ return Operand(kRootRegister,
+ IsolateData::builtin_entry_slot_offset(builtin_index));
+}
+
+Operand TurboAssembler::EntryFromBuiltinIndexAsOperand(Register builtin_index) {
+ if (SmiValuesAre32Bits()) {
+ // The builtin_index register contains the builtin index as a Smi.
+ SmiUntag(builtin_index);
+ return Operand(kRootRegister, builtin_index, times_system_pointer_size,
+ IsolateData::builtin_entry_table_offset());
+ } else {
+ DCHECK(SmiValuesAre31Bits());
+
+ // The builtin_index register contains the builtin index as a Smi.
+ // Untagging is folded into the indexing operand below (we use
+ // times_half_system_pointer_size since smis are already shifted by one).
+ return Operand(kRootRegister, builtin_index, times_half_system_pointer_size,
+ IsolateData::builtin_entry_table_offset());
+ }
+}
+
+void TurboAssembler::CallBuiltinByIndex(Register builtin_index) {
+ Call(EntryFromBuiltinIndexAsOperand(builtin_index));
+}
+
+void TurboAssembler::CallBuiltin(int builtin_index) {
+ DCHECK(Builtins::IsBuiltinId(builtin_index));
+ RecordCommentForOffHeapTrampoline(builtin_index);
+ CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
+ EmbeddedData d = EmbeddedData::FromBlob();
+ Address entry = d.InstructionStartOfBuiltin(builtin_index);
+ Move(kScratchRegister, entry, RelocInfo::OFF_HEAP_TARGET);
+ call(kScratchRegister);
+}
+
+void TurboAssembler::LoadCodeObjectEntry(Register destination,
+ Register code_object) {
+ // Code objects are called differently depending on whether we are generating
+ // builtin code (which will later be embedded into the binary) or compiling
+ // user JS code at runtime.
+ // * Builtin code runs in --jitless mode and thus must not call into on-heap
+ // Code targets. Instead, we dispatch through the builtins entry table.
+ // * Codegen at runtime does not have this restriction and we can use the
+ // shorter, branchless instruction sequence. The assumption here is that
+ // targets are usually generated code and not builtin Code objects.
+
+ if (options().isolate_independent_code) {
+ DCHECK(root_array_available());
+ Label if_code_is_off_heap, out;
+
+ // Check whether the Code object is an off-heap trampoline. If so, call its
+ // (off-heap) entry point directly without going through the (on-heap)
+ // trampoline. Otherwise, just call the Code object as always.
+ testl(FieldOperand(code_object, Code::kFlagsOffset),
+ Immediate(Code::IsOffHeapTrampoline::kMask));
+ j(not_equal, &if_code_is_off_heap);
+
+ // Not an off-heap trampoline, the entry point is at
+ // Code::raw_instruction_start().
+ Move(destination, code_object);
+ addq(destination, Immediate(Code::kHeaderSize - kHeapObjectTag));
+ jmp(&out);
+
+ // An off-heap trampoline, the entry point is loaded from the builtin entry
+ // table.
+ bind(&if_code_is_off_heap);
+ movl(destination, FieldOperand(code_object, Code::kBuiltinIndexOffset));
+ movq(destination,
+ Operand(kRootRegister, destination, times_system_pointer_size,
+ IsolateData::builtin_entry_table_offset()));
+
+ bind(&out);
+ } else {
+ Move(destination, code_object);
+ addq(destination, Immediate(Code::kHeaderSize - kHeapObjectTag));
+ }
+}
+
+void TurboAssembler::CallCodeObject(Register code_object) {
+ LoadCodeObjectEntry(code_object, code_object);
+ call(code_object);
+}
+
+void TurboAssembler::JumpCodeObject(Register code_object) {
+ LoadCodeObjectEntry(code_object, code_object);
+ jmp(code_object);
+}
+
+void TurboAssembler::RetpolineCall(Register reg) {
+ Label setup_return, setup_target, inner_indirect_branch, capture_spec;
+
+ jmp(&setup_return); // Jump past the entire retpoline below.
+
+ bind(&inner_indirect_branch);
+ call(&setup_target);
+
+ bind(&capture_spec);
+ pause();
+ jmp(&capture_spec);
+
+ bind(&setup_target);
+ movq(Operand(rsp, 0), reg);
+ ret(0);
+
+ bind(&setup_return);
+ call(&inner_indirect_branch); // Callee will return after this instruction.
+}
+
+void TurboAssembler::RetpolineCall(Address destination, RelocInfo::Mode rmode) {
+ Move(kScratchRegister, destination, rmode);
+ RetpolineCall(kScratchRegister);
+}
+
+void TurboAssembler::RetpolineJump(Register reg) {
+ Label setup_target, capture_spec;
+
+ call(&setup_target);
+
+ bind(&capture_spec);
+ pause();
+ jmp(&capture_spec);
+
+ bind(&setup_target);
+ movq(Operand(rsp, 0), reg);
+ ret(0);
+}
+
+void TurboAssembler::Shufps(XMMRegister dst, XMMRegister src, byte imm8) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope avx_scope(this, AVX);
+ vshufps(dst, src, src, imm8);
+ } else {
+ if (dst != src) {
+ movss(dst, src);
+ }
+ shufps(dst, src, static_cast<byte>(0));
+ }
+}
+
+void TurboAssembler::Pextrd(Register dst, XMMRegister src, uint8_t imm8) {
+ if (imm8 == 0) {
+ Movd(dst, src);
+ return;
+ }
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vpextrd(dst, src, imm8);
+ return;
+ } else if (CpuFeatures::IsSupported(SSE4_1)) {
+ CpuFeatureScope sse_scope(this, SSE4_1);
+ pextrd(dst, src, imm8);
+ return;
+ }
+ DCHECK_EQ(1, imm8);
+ movq(dst, src);
+ shrq(dst, Immediate(32));
+}
+
+namespace {
+
+template <typename Src>
+using AvxFn = void (Assembler::*)(XMMRegister, XMMRegister, Src, uint8_t);
+template <typename Src>
+using NoAvxFn = void (Assembler::*)(XMMRegister, Src, uint8_t);
+
+template <typename Src>
+void PinsrHelper(Assembler* assm, AvxFn<Src> avx, NoAvxFn<Src> noavx,
+ XMMRegister dst, XMMRegister src1, Src src2, uint8_t imm8,
+ base::Optional<CpuFeature> feature = base::nullopt) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(assm, AVX);
+ (assm->*avx)(dst, src1, src2, imm8);
+ return;
+ }
+
+ if (dst != src1) {
+ assm->movdqu(dst, src1);
+ }
+ if (feature.has_value()) {
+ DCHECK(CpuFeatures::IsSupported(*feature));
+ CpuFeatureScope scope(assm, *feature);
+ (assm->*noavx)(dst, src2, imm8);
+ } else {
+ (assm->*noavx)(dst, src2, imm8);
+ }
+}
+} // namespace
+
+void TurboAssembler::Pinsrb(XMMRegister dst, XMMRegister src1, Register src2,
+ uint8_t imm8) {
+ PinsrHelper(this, &Assembler::vpinsrb, &Assembler::pinsrb, dst, src1, src2,
+ imm8, base::Optional<CpuFeature>(SSE4_1));
+}
+
+void TurboAssembler::Pinsrb(XMMRegister dst, XMMRegister src1, Operand src2,
+ uint8_t imm8) {
+ PinsrHelper(this, &Assembler::vpinsrb, &Assembler::pinsrb, dst, src1, src2,
+ imm8, base::Optional<CpuFeature>(SSE4_1));
+}
+
+void TurboAssembler::Pinsrw(XMMRegister dst, XMMRegister src1, Register src2,
+ uint8_t imm8) {
+ PinsrHelper(this, &Assembler::vpinsrw, &Assembler::pinsrw, dst, src1, src2,
+ imm8);
+}
+
+void TurboAssembler::Pinsrw(XMMRegister dst, XMMRegister src1, Operand src2,
+ uint8_t imm8) {
+ PinsrHelper(this, &Assembler::vpinsrw, &Assembler::pinsrw, dst, src1, src2,
+ imm8);
+}
+
+void TurboAssembler::Pinsrd(XMMRegister dst, XMMRegister src1, Register src2,
+ uint8_t imm8) {
+ // Need a fall back when SSE4_1 is unavailable. Pinsrb and Pinsrq are used
+ // only by Wasm SIMD, which requires SSE4_1 already.
+ if (CpuFeatures::IsSupported(SSE4_1)) {
+ PinsrHelper(this, &Assembler::vpinsrd, &Assembler::pinsrd, dst, src1, src2,
+ imm8, base::Optional<CpuFeature>(SSE4_1));
+ return;
+ }
+
+ Movd(kScratchDoubleReg, src2);
+ if (imm8 == 1) {
+ punpckldq(dst, kScratchDoubleReg);
+ } else {
+ DCHECK_EQ(0, imm8);
+ Movss(dst, kScratchDoubleReg);
+ }
+}
+
+void TurboAssembler::Pinsrd(XMMRegister dst, XMMRegister src1, Operand src2,
+ uint8_t imm8) {
+ // Need a fall back when SSE4_1 is unavailable. Pinsrb and Pinsrq are used
+ // only by Wasm SIMD, which requires SSE4_1 already.
+ if (CpuFeatures::IsSupported(SSE4_1)) {
+ PinsrHelper(this, &Assembler::vpinsrd, &Assembler::pinsrd, dst, src1, src2,
+ imm8, base::Optional<CpuFeature>(SSE4_1));
+ return;
+ }
+
+ Movd(kScratchDoubleReg, src2);
+ if (imm8 == 1) {
+ punpckldq(dst, kScratchDoubleReg);
+ } else {
+ DCHECK_EQ(0, imm8);
+ Movss(dst, kScratchDoubleReg);
+ }
+}
+
+void TurboAssembler::Pinsrd(XMMRegister dst, Register src2, uint8_t imm8) {
+ Pinsrd(dst, dst, src2, imm8);
+}
+
+void TurboAssembler::Pinsrd(XMMRegister dst, Operand src2, uint8_t imm8) {
+ Pinsrd(dst, dst, src2, imm8);
+}
+
+void TurboAssembler::Pinsrq(XMMRegister dst, XMMRegister src1, Register src2,
+ uint8_t imm8) {
+ PinsrHelper(this, &Assembler::vpinsrq, &Assembler::pinsrq, dst, src1, src2,
+ imm8, base::Optional<CpuFeature>(SSE4_1));
+}
+
+void TurboAssembler::Pinsrq(XMMRegister dst, XMMRegister src1, Operand src2,
+ uint8_t imm8) {
+ PinsrHelper(this, &Assembler::vpinsrq, &Assembler::pinsrq, dst, src1, src2,
+ imm8, base::Optional<CpuFeature>(SSE4_1));
+}
+
+void TurboAssembler::Psllq(XMMRegister dst, byte imm8) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vpsllq(dst, dst, imm8);
+ } else {
+ DCHECK(!IsEnabled(AVX));
+ psllq(dst, imm8);
+ }
+}
+
+void TurboAssembler::Psrlq(XMMRegister dst, byte imm8) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vpsrlq(dst, dst, imm8);
+ } else {
+ DCHECK(!IsEnabled(AVX));
+ psrlq(dst, imm8);
+ }
+}
+
+void TurboAssembler::Pslld(XMMRegister dst, byte imm8) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vpslld(dst, dst, imm8);
+ } else {
+ DCHECK(!IsEnabled(AVX));
+ pslld(dst, imm8);
+ }
+}
+
+void TurboAssembler::Pblendvb(XMMRegister dst, XMMRegister src1,
+ XMMRegister src2, XMMRegister mask) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope avx_scope(this, AVX);
+ vpblendvb(dst, src1, src2, mask);
+ } else {
+ DCHECK_EQ(dst, src1);
+ DCHECK_EQ(xmm0, mask);
+ pblendvb(dst, src2);
+ }
+}
+
+void TurboAssembler::Blendvps(XMMRegister dst, XMMRegister src1,
+ XMMRegister src2, XMMRegister mask) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope avx_scope(this, AVX);
+ vblendvps(dst, src1, src2, mask);
+ } else {
+ DCHECK_EQ(dst, src1);
+ DCHECK_EQ(xmm0, mask);
+ blendvps(dst, src2);
+ }
+}
+
+void TurboAssembler::Blendvpd(XMMRegister dst, XMMRegister src1,
+ XMMRegister src2, XMMRegister mask) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope avx_scope(this, AVX);
+ vblendvpd(dst, src1, src2, mask);
+ } else {
+ DCHECK_EQ(dst, src1);
+ DCHECK_EQ(xmm0, mask);
+ blendvpd(dst, src2);
+ }
+}
+
+void TurboAssembler::Pshufb(XMMRegister dst, XMMRegister src,
+ XMMRegister mask) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope avx_scope(this, AVX);
+ vpshufb(dst, src, mask);
+ } else {
+ // Make sure these are different so that we won't overwrite mask.
+ DCHECK_NE(dst, mask);
+ if (dst != src) {
+ movapd(dst, src);
+ }
+ CpuFeatureScope sse_scope(this, SSSE3);
+ pshufb(dst, mask);
+ }
+}
+
+void TurboAssembler::Psrld(XMMRegister dst, byte imm8) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(this, AVX);
+ vpsrld(dst, dst, imm8);
+ } else {
+ DCHECK(!IsEnabled(AVX));
+ psrld(dst, imm8);
+ }
+}
+
+void TurboAssembler::Lzcntl(Register dst, Register src) {
+ if (CpuFeatures::IsSupported(LZCNT)) {
+ CpuFeatureScope scope(this, LZCNT);
+ lzcntl(dst, src);
+ return;
+ }
+ Label not_zero_src;
+ bsrl(dst, src);
+ j(not_zero, ¬_zero_src, Label::kNear);
+ movl(dst, Immediate(63)); // 63^31 == 32
+ bind(¬_zero_src);
+ xorl(dst, Immediate(31)); // for x in [0..31], 31^x == 31 - x
+}
+
+void TurboAssembler::Lzcntl(Register dst, Operand src) {
+ if (CpuFeatures::IsSupported(LZCNT)) {
+ CpuFeatureScope scope(this, LZCNT);
+ lzcntl(dst, src);
+ return;
+ }
+ Label not_zero_src;
+ bsrl(dst, src);
+ j(not_zero, ¬_zero_src, Label::kNear);
+ movl(dst, Immediate(63)); // 63^31 == 32
+ bind(¬_zero_src);
+ xorl(dst, Immediate(31)); // for x in [0..31], 31^x == 31 - x
+}
+
+void TurboAssembler::Lzcntq(Register dst, Register src) {
+ if (CpuFeatures::IsSupported(LZCNT)) {
+ CpuFeatureScope scope(this, LZCNT);
+ lzcntq(dst, src);
+ return;
+ }
+ Label not_zero_src;
+ bsrq(dst, src);
+ j(not_zero, ¬_zero_src, Label::kNear);
+ movl(dst, Immediate(127)); // 127^63 == 64
+ bind(¬_zero_src);
+ xorl(dst, Immediate(63)); // for x in [0..63], 63^x == 63 - x
+}
+
+void TurboAssembler::Lzcntq(Register dst, Operand src) {
+ if (CpuFeatures::IsSupported(LZCNT)) {
+ CpuFeatureScope scope(this, LZCNT);
+ lzcntq(dst, src);
+ return;
+ }
+ Label not_zero_src;
+ bsrq(dst, src);
+ j(not_zero, ¬_zero_src, Label::kNear);
+ movl(dst, Immediate(127)); // 127^63 == 64
+ bind(¬_zero_src);
+ xorl(dst, Immediate(63)); // for x in [0..63], 63^x == 63 - x
+}
+
+void TurboAssembler::Tzcntq(Register dst, Register src) {
+ if (CpuFeatures::IsSupported(BMI1)) {
+ CpuFeatureScope scope(this, BMI1);
+ tzcntq(dst, src);
+ return;
+ }
+ Label not_zero_src;
+ bsfq(dst, src);
+ j(not_zero, ¬_zero_src, Label::kNear);
+ // Define the result of tzcnt(0) separately, because bsf(0) is undefined.
+ movl(dst, Immediate(64));
+ bind(¬_zero_src);
+}
+
+void TurboAssembler::Tzcntq(Register dst, Operand src) {
+ if (CpuFeatures::IsSupported(BMI1)) {
+ CpuFeatureScope scope(this, BMI1);
+ tzcntq(dst, src);
+ return;
+ }
+ Label not_zero_src;
+ bsfq(dst, src);
+ j(not_zero, ¬_zero_src, Label::kNear);
+ // Define the result of tzcnt(0) separately, because bsf(0) is undefined.
+ movl(dst, Immediate(64));
+ bind(¬_zero_src);
+}
+
+void TurboAssembler::Tzcntl(Register dst, Register src) {
+ if (CpuFeatures::IsSupported(BMI1)) {
+ CpuFeatureScope scope(this, BMI1);
+ tzcntl(dst, src);
+ return;
+ }
+ Label not_zero_src;
+ bsfl(dst, src);
+ j(not_zero, ¬_zero_src, Label::kNear);
+ movl(dst, Immediate(32)); // The result of tzcnt is 32 if src = 0.
+ bind(¬_zero_src);
+}
+
+void TurboAssembler::Tzcntl(Register dst, Operand src) {
+ if (CpuFeatures::IsSupported(BMI1)) {
+ CpuFeatureScope scope(this, BMI1);
+ tzcntl(dst, src);
+ return;
+ }
+ Label not_zero_src;
+ bsfl(dst, src);
+ j(not_zero, ¬_zero_src, Label::kNear);
+ movl(dst, Immediate(32)); // The result of tzcnt is 32 if src = 0.
+ bind(¬_zero_src);
+}
+
+void TurboAssembler::Popcntl(Register dst, Register src) {
+ if (CpuFeatures::IsSupported(POPCNT)) {
+ CpuFeatureScope scope(this, POPCNT);
+ popcntl(dst, src);
+ return;
+ }
+ UNREACHABLE();
+}
+
+void TurboAssembler::Popcntl(Register dst, Operand src) {
+ if (CpuFeatures::IsSupported(POPCNT)) {
+ CpuFeatureScope scope(this, POPCNT);
+ popcntl(dst, src);
+ return;
+ }
+ UNREACHABLE();
+}
+
+void TurboAssembler::Popcntq(Register dst, Register src) {
+ if (CpuFeatures::IsSupported(POPCNT)) {
+ CpuFeatureScope scope(this, POPCNT);
+ popcntq(dst, src);
+ return;
+ }
+ UNREACHABLE();
+}
+
+void TurboAssembler::Popcntq(Register dst, Operand src) {
+ if (CpuFeatures::IsSupported(POPCNT)) {
+ CpuFeatureScope scope(this, POPCNT);
+ popcntq(dst, src);
+ return;
+ }
+ UNREACHABLE();
+}
+
+// Order general registers are pushed by Pushad:
+// rax, rcx, rdx, rbx, rsi, rdi, r8, r9, r11, r14, r15.
+const int
+ MacroAssembler::kSafepointPushRegisterIndices[Register::kNumRegisters] = {
+ 0, 1, 2, 3, -1, -1, 4, 5, 6, 7, -1, 8, 9, -1, 10, 11};
+
+void MacroAssembler::PushStackHandler() {
+ // Adjust this code if not the case.
+ STATIC_ASSERT(StackHandlerConstants::kSize == 2 * kSystemPointerSize);
+ STATIC_ASSERT(StackHandlerConstants::kNextOffset == 0);
+
+ Push(Immediate(0)); // Padding.
+
+ // Link the current handler as the next handler.
+ ExternalReference handler_address =
+ ExternalReference::Create(IsolateAddressId::kHandlerAddress, isolate());
+ Push(ExternalReferenceAsOperand(handler_address));
+
+ // Set this new handler as the current one.
+ movq(ExternalReferenceAsOperand(handler_address), rsp);
+}
+
+void MacroAssembler::PopStackHandler() {
+ STATIC_ASSERT(StackHandlerConstants::kNextOffset == 0);
+ ExternalReference handler_address =
+ ExternalReference::Create(IsolateAddressId::kHandlerAddress, isolate());
+ Pop(ExternalReferenceAsOperand(handler_address));
+ addq(rsp, Immediate(StackHandlerConstants::kSize - kSystemPointerSize));
+}
+
+void TurboAssembler::Ret() { ret(0); }
+
+void TurboAssembler::Ret(int bytes_dropped, Register scratch) {
+ if (is_uint16(bytes_dropped)) {
+ ret(bytes_dropped);
+ } else {
+ PopReturnAddressTo(scratch);
+ addq(rsp, Immediate(bytes_dropped));
+ PushReturnAddressFrom(scratch);
+ ret(0);
+ }
+}
+
+void MacroAssembler::CmpObjectType(Register heap_object, InstanceType type,
+ Register map) {
+ LoadMap(map, heap_object);
+ CmpInstanceType(map, type);
+}
+
+void MacroAssembler::CmpInstanceType(Register map, InstanceType type) {
+ cmpw(FieldOperand(map, Map::kInstanceTypeOffset), Immediate(type));
+}
+
+void MacroAssembler::AssertNotSmi(Register object) {
+ if (emit_debug_code()) {
+ Condition is_smi = CheckSmi(object);
+ Check(NegateCondition(is_smi), AbortReason::kOperandIsASmi);
+ }
+}
+
+void MacroAssembler::AssertSmi(Register object) {
+ if (emit_debug_code()) {
+ Condition is_smi = CheckSmi(object);
+ Check(is_smi, AbortReason::kOperandIsNotASmi);
+ }
+}
+
+void MacroAssembler::AssertSmi(Operand object) {
+ if (emit_debug_code()) {
+ Condition is_smi = CheckSmi(object);
+ Check(is_smi, AbortReason::kOperandIsNotASmi);
+ }
+}
+
+void TurboAssembler::AssertZeroExtended(Register int32_register) {
+ if (emit_debug_code()) {
+ DCHECK_NE(int32_register, kScratchRegister);
+ movq(kScratchRegister, int64_t{0x0000000100000000});
+ cmpq(kScratchRegister, int32_register);
+ Check(above, AbortReason::k32BitValueInRegisterIsNotZeroExtended);
+ }
+}
+
+void MacroAssembler::AssertConstructor(Register object) {
+ if (emit_debug_code()) {
+ testb(object, Immediate(kSmiTagMask));
+ Check(not_equal, AbortReason::kOperandIsASmiAndNotAConstructor);
+ Push(object);
+ LoadMap(object, object);
+ testb(FieldOperand(object, Map::kBitFieldOffset),
+ Immediate(Map::Bits1::IsConstructorBit::kMask));
+ Pop(object);
+ Check(not_zero, AbortReason::kOperandIsNotAConstructor);
+ }
+}
+
+void MacroAssembler::AssertFunction(Register object) {
+ if (emit_debug_code()) {
+ testb(object, Immediate(kSmiTagMask));
+ Check(not_equal, AbortReason::kOperandIsASmiAndNotAFunction);
+ Push(object);
+ CmpObjectType(object, JS_FUNCTION_TYPE, object);
+ Pop(object);
+ Check(equal, AbortReason::kOperandIsNotAFunction);
+ }
+}
+
+void MacroAssembler::AssertBoundFunction(Register object) {
+ if (emit_debug_code()) {
+ testb(object, Immediate(kSmiTagMask));
+ Check(not_equal, AbortReason::kOperandIsASmiAndNotABoundFunction);
+ Push(object);
+ CmpObjectType(object, JS_BOUND_FUNCTION_TYPE, object);
+ Pop(object);
+ Check(equal, AbortReason::kOperandIsNotABoundFunction);
+ }
+}
+
+void MacroAssembler::AssertGeneratorObject(Register object) {
+ if (!emit_debug_code()) return;
+ testb(object, Immediate(kSmiTagMask));
+ Check(not_equal, AbortReason::kOperandIsASmiAndNotAGeneratorObject);
+
+ // Load map
+ Register map = object;
+ Push(object);
+ LoadMap(map, object);
+
+ Label do_check;
+ // Check if JSGeneratorObject
+ CmpInstanceType(map, JS_GENERATOR_OBJECT_TYPE);
+ j(equal, &do_check);
+
+ // Check if JSAsyncFunctionObject
+ CmpInstanceType(map, JS_ASYNC_FUNCTION_OBJECT_TYPE);
+ j(equal, &do_check);
+
+ // Check if JSAsyncGeneratorObject
+ CmpInstanceType(map, JS_ASYNC_GENERATOR_OBJECT_TYPE);
+
+ bind(&do_check);
+ // Restore generator object to register and perform assertion
+ Pop(object);
+ Check(equal, AbortReason::kOperandIsNotAGeneratorObject);
+}
+
+void MacroAssembler::AssertUndefinedOrAllocationSite(Register object) {
+ if (emit_debug_code()) {
+ Label done_checking;
+ AssertNotSmi(object);
+ Cmp(object, isolate()->factory()->undefined_value());
+ j(equal, &done_checking);
+ Cmp(FieldOperand(object, 0), isolate()->factory()->allocation_site_map());
+ Assert(equal, AbortReason::kExpectedUndefinedOrCell);
+ bind(&done_checking);
+ }
+}
+
+void MacroAssembler::LoadWeakValue(Register in_out, Label* target_if_cleared) {
+ cmpl(in_out, Immediate(kClearedWeakHeapObjectLower32));
+ j(equal, target_if_cleared);
+
+ andq(in_out, Immediate(~static_cast<int32_t>(kWeakHeapObjectMask)));
+}
+
+void MacroAssembler::IncrementCounter(StatsCounter* counter, int value) {
+ DCHECK_GT(value, 0);
+ if (FLAG_native_code_counters && counter->Enabled()) {
+ Operand counter_operand =
+ ExternalReferenceAsOperand(ExternalReference::Create(counter));
+ // This operation has to be exactly 32-bit wide in case the external
+ // reference table redirects the counter to a uint32_t dummy_stats_counter_
+ // field.
+ if (value == 1) {
+ incl(counter_operand);
+ } else {
+ addl(counter_operand, Immediate(value));
+ }
+ }
+}
+
+void MacroAssembler::DecrementCounter(StatsCounter* counter, int value) {
+ DCHECK_GT(value, 0);
+ if (FLAG_native_code_counters && counter->Enabled()) {
+ Operand counter_operand =
+ ExternalReferenceAsOperand(ExternalReference::Create(counter));
+ // This operation has to be exactly 32-bit wide in case the external
+ // reference table redirects the counter to a uint32_t dummy_stats_counter_
+ // field.
+ if (value == 1) {
+ decl(counter_operand);
+ } else {
+ subl(counter_operand, Immediate(value));
+ }
+ }
+}
+
+void MacroAssembler::MaybeDropFrames() {
+ // Check whether we need to drop frames to restart a function on the stack.
+ ExternalReference restart_fp =
+ ExternalReference::debug_restart_fp_address(isolate());
+ Load(rbx, restart_fp);
+ testq(rbx, rbx);
+
+ Label dont_drop;
+ j(zero, &dont_drop, Label::kNear);
+ Jump(BUILTIN_CODE(isolate(), FrameDropperTrampoline), RelocInfo::CODE_TARGET);
+
+ bind(&dont_drop);
+}
+
+void TurboAssembler::PrepareForTailCall(Register callee_args_count,
+ Register caller_args_count,
+ Register scratch0, Register scratch1) {
+ DCHECK(!AreAliased(callee_args_count, caller_args_count, scratch0, scratch1));
+
+ // Calculate the destination address where we will put the return address
+ // after we drop current frame.
+ Register new_sp_reg = scratch0;
+ subq(caller_args_count, callee_args_count);
+ leaq(new_sp_reg, Operand(rbp, caller_args_count, times_system_pointer_size,
+ StandardFrameConstants::kCallerPCOffset));
+
+ if (FLAG_debug_code) {
+ cmpq(rsp, new_sp_reg);
+ Check(below, AbortReason::kStackAccessBelowStackPointer);
+ }
+
+ // Copy return address from caller's frame to current frame's return address
+ // to avoid its trashing and let the following loop copy it to the right
+ // place.
+ Register tmp_reg = scratch1;
+ movq(tmp_reg, Operand(rbp, StandardFrameConstants::kCallerPCOffset));
+ movq(Operand(rsp, 0), tmp_reg);
+
+ // Restore caller's frame pointer now as it could be overwritten by
+ // the copying loop.
+ movq(rbp, Operand(rbp, StandardFrameConstants::kCallerFPOffset));
+
+ // +2 here is to copy both receiver and return address.
+ Register count_reg = caller_args_count;
+ leaq(count_reg, Operand(callee_args_count, 2));
+
+ // Now copy callee arguments to the caller frame going backwards to avoid
+ // callee arguments corruption (source and destination areas could overlap).
+ Label loop, entry;
+ jmp(&entry, Label::kNear);
+ bind(&loop);
+ decq(count_reg);
+ movq(tmp_reg, Operand(rsp, count_reg, times_system_pointer_size, 0));
+ movq(Operand(new_sp_reg, count_reg, times_system_pointer_size, 0), tmp_reg);
+ bind(&entry);
+ cmpq(count_reg, Immediate(0));
+ j(not_equal, &loop, Label::kNear);
+
+ // Leave current frame.
+ movq(rsp, new_sp_reg);
+}
+
+void MacroAssembler::InvokeFunction(Register function, Register new_target,
+ Register actual_parameter_count,
+ InvokeFlag flag) {
+ LoadTaggedPointerField(
+ rbx, FieldOperand(function, JSFunction::kSharedFunctionInfoOffset));
+ movzxwq(rbx,
+ FieldOperand(rbx, SharedFunctionInfo::kFormalParameterCountOffset));
+
+ InvokeFunction(function, new_target, rbx, actual_parameter_count, flag);
+}
+
+void MacroAssembler::InvokeFunction(Register function, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count,
+ InvokeFlag flag) {
+ DCHECK(function == rdi);
+ LoadTaggedPointerField(rsi,
+ FieldOperand(function, JSFunction::kContextOffset));
+ InvokeFunctionCode(rdi, new_target, expected_parameter_count,
+ actual_parameter_count, flag);
+}
+
+void MacroAssembler::InvokeFunctionCode(Register function, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count,
+ InvokeFlag flag) {
+ // You can't call a function without a valid frame.
+ DCHECK_IMPLIES(flag == CALL_FUNCTION, has_frame());
+ DCHECK_EQ(function, rdi);
+ DCHECK_IMPLIES(new_target.is_valid(), new_target == rdx);
+
+ // On function call, call into the debugger if necessary.
+ Label debug_hook, continue_after_hook;
+ {
+ ExternalReference debug_hook_active =
+ ExternalReference::debug_hook_on_function_call_address(isolate());
+ Operand debug_hook_active_operand =
+ ExternalReferenceAsOperand(debug_hook_active);
+ cmpb(debug_hook_active_operand, Immediate(0));
+ j(not_equal, &debug_hook);
+ }
+ bind(&continue_after_hook);
+
+ // Clear the new.target register if not given.
+ if (!new_target.is_valid()) {
+ LoadRoot(rdx, RootIndex::kUndefinedValue);
+ }
+
+ Label done;
+ InvokePrologue(expected_parameter_count, actual_parameter_count, &done, flag);
+ // We call indirectly through the code field in the function to
+ // allow recompilation to take effect without changing any of the
+ // call sites.
+ static_assert(kJavaScriptCallCodeStartRegister == rcx, "ABI mismatch");
+ LoadTaggedPointerField(rcx, FieldOperand(function, JSFunction::kCodeOffset));
+ if (flag == CALL_FUNCTION) {
+ CallCodeObject(rcx);
+ } else {
+ DCHECK(flag == JUMP_FUNCTION);
+ JumpCodeObject(rcx);
+ }
+ jmp(&done, Label::kNear);
+
+ // Deferred debug hook.
+ bind(&debug_hook);
+ CallDebugOnFunctionCall(function, new_target, expected_parameter_count,
+ actual_parameter_count);
+ jmp(&continue_after_hook);
+
+ bind(&done);
+}
+
+Operand MacroAssembler::StackLimitAsOperand(StackLimitKind kind) {
+ DCHECK(root_array_available());
+ Isolate* isolate = this->isolate();
+ ExternalReference limit =
+ kind == StackLimitKind::kRealStackLimit
+ ? ExternalReference::address_of_real_jslimit(isolate)
+ : ExternalReference::address_of_jslimit(isolate);
+ DCHECK(TurboAssembler::IsAddressableThroughRootRegister(isolate, limit));
+
+ intptr_t offset =
+ TurboAssembler::RootRegisterOffsetForExternalReference(isolate, limit);
+ CHECK(is_int32(offset));
+ return Operand(kRootRegister, static_cast<int32_t>(offset));
+}
+
+void MacroAssembler::StackOverflowCheck(
+ Register num_args, Register scratch, Label* stack_overflow,
+ Label::Distance stack_overflow_distance) {
+ DCHECK_NE(num_args, scratch);
+ // Check the stack for overflow. We are not trying to catch
+ // interruptions (e.g. debug break and preemption) here, so the "real stack
+ // limit" is checked.
+ movq(kScratchRegister, StackLimitAsOperand(StackLimitKind::kRealStackLimit));
+ movq(scratch, rsp);
+ // Make scratch the space we have left. The stack might already be overflowed
+ // here which will cause scratch to become negative.
+ subq(scratch, kScratchRegister);
+ // TODO(victorgomes): Use ia32 approach with leaq, since it requires less
+ // instructions.
+ sarq(scratch, Immediate(kSystemPointerSizeLog2));
+ // Check if the arguments will overflow the stack.
+ cmpq(scratch, num_args);
+ // Signed comparison.
+ // TODO(victorgomes): Save some bytes in the builtins that use stack checks
+ // by jumping to a builtin that throws the exception.
+ j(less_equal, stack_overflow, stack_overflow_distance);
+}
+
+void MacroAssembler::InvokePrologue(Register expected_parameter_count,
+ Register actual_parameter_count,
+ Label* done, InvokeFlag flag) {
+ if (expected_parameter_count != actual_parameter_count) {
+ Label regular_invoke;
+#ifdef V8_NO_ARGUMENTS_ADAPTOR
+ // If the expected parameter count is equal to the adaptor sentinel, no need
+ // to push undefined value as arguments.
+ cmpl(expected_parameter_count, Immediate(kDontAdaptArgumentsSentinel));
+ j(equal, ®ular_invoke, Label::kFar);
+
+ // If overapplication or if the actual argument count is equal to the
+ // formal parameter count, no need to push extra undefined values.
+ subq(expected_parameter_count, actual_parameter_count);
+ j(less_equal, ®ular_invoke, Label::kFar);
+
+ Label stack_overflow;
+ StackOverflowCheck(expected_parameter_count, rcx, &stack_overflow);
+
+ // Underapplication. Move the arguments already in the stack, including the
+ // receiver and the return address.
+ {
+ Label copy, check;
+ Register src = r8, dest = rsp, num = r9, current = r11;
+ movq(src, rsp);
+ leaq(kScratchRegister,
+ Operand(expected_parameter_count, times_system_pointer_size, 0));
+ AllocateStackSpace(kScratchRegister);
+ // Extra words are the receiver and the return address (if a jump).
+ int extra_words = flag == CALL_FUNCTION ? 1 : 2;
+ leaq(num, Operand(rax, extra_words)); // Number of words to copy.
+ Set(current, 0);
+ // Fall-through to the loop body because there are non-zero words to copy.
+ bind(©);
+ movq(kScratchRegister,
+ Operand(src, current, times_system_pointer_size, 0));
+ movq(Operand(dest, current, times_system_pointer_size, 0),
+ kScratchRegister);
+ incq(current);
+ bind(&check);
+ cmpq(current, num);
+ j(less, ©);
+ leaq(r8, Operand(rsp, num, times_system_pointer_size, 0));
+ }
+ // Fill remaining expected arguments with undefined values.
+ LoadRoot(kScratchRegister, RootIndex::kUndefinedValue);
+ {
+ Label loop;
+ bind(&loop);
+ decq(expected_parameter_count);
+ movq(Operand(r8, expected_parameter_count, times_system_pointer_size, 0),
+ kScratchRegister);
+ j(greater, &loop, Label::kNear);
+ }
+ jmp(®ular_invoke);
+
+ bind(&stack_overflow);
+ {
+ FrameScope frame(this,
+ has_frame() ? StackFrame::NONE : StackFrame::INTERNAL);
+ CallRuntime(Runtime::kThrowStackOverflow);
+ int3(); // This should be unreachable.
+ }
+#else
+ // Both expected and actual are in (different) registers. This
+ // is the case when we invoke functions using call and apply.
+ cmpq(expected_parameter_count, actual_parameter_count);
+ j(equal, ®ular_invoke, Label::kNear);
+ DCHECK_EQ(actual_parameter_count, rax);
+ DCHECK_EQ(expected_parameter_count, rbx);
+ Handle<Code> adaptor = BUILTIN_CODE(isolate(), ArgumentsAdaptorTrampoline);
+ if (flag == CALL_FUNCTION) {
+ Call(adaptor, RelocInfo::CODE_TARGET);
+ jmp(done, Label::kNear);
+ } else {
+ Jump(adaptor, RelocInfo::CODE_TARGET);
+ }
+#endif
+
+ bind(®ular_invoke);
+ } else {
+ Move(rax, actual_parameter_count);
+ }
+}
+
+void MacroAssembler::CallDebugOnFunctionCall(Register fun, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count) {
+ FrameScope frame(this, has_frame() ? StackFrame::NONE : StackFrame::INTERNAL);
+
+ SmiTag(expected_parameter_count);
+ Push(expected_parameter_count);
+
+ SmiTag(actual_parameter_count);
+ Push(actual_parameter_count);
+ SmiUntag(actual_parameter_count);
+
+ if (new_target.is_valid()) {
+ Push(new_target);
+ }
+ Push(fun);
+ Push(fun);
+ // Arguments are located 2 words below the base pointer.
+ Operand receiver_op = Operand(rbp, kSystemPointerSize * 2);
+ Push(receiver_op);
+ CallRuntime(Runtime::kDebugOnFunctionCall);
+ Pop(fun);
+ if (new_target.is_valid()) {
+ Pop(new_target);
+ }
+ Pop(actual_parameter_count);
+ SmiUntag(actual_parameter_count);
+ Pop(expected_parameter_count);
+ SmiUntag(expected_parameter_count);
+}
+
+void TurboAssembler::StubPrologue(StackFrame::Type type) {
+ pushq(rbp); // Caller's frame pointer.
+ movq(rbp, rsp);
+ Push(Immediate(StackFrame::TypeToMarker(type)));
+}
+
+void TurboAssembler::Prologue() {
+ pushq(rbp); // Caller's frame pointer.
+ movq(rbp, rsp);
+ Push(kContextRegister); // Callee's context.
+ Push(kJSFunctionRegister); // Callee's JS function.
+ Push(kJavaScriptCallArgCountRegister); // Actual argument count.
+}
+
+void TurboAssembler::EnterFrame(StackFrame::Type type) {
+ pushq(rbp);
+ movq(rbp, rsp);
+ Push(Immediate(StackFrame::TypeToMarker(type)));
+}
+
+void TurboAssembler::LeaveFrame(StackFrame::Type type) {
+ if (emit_debug_code()) {
+ cmpq(Operand(rbp, CommonFrameConstants::kContextOrFrameTypeOffset),
+ Immediate(StackFrame::TypeToMarker(type)));
+ Check(equal, AbortReason::kStackFrameTypesMustMatch);
+ }
+ movq(rsp, rbp);
+ popq(rbp);
+}
+
+#ifdef V8_TARGET_OS_WIN
+void TurboAssembler::AllocateStackSpace(Register bytes_scratch) {
+ // In windows, we cannot increment the stack size by more than one page
+ // (minimum page size is 4KB) without accessing at least one byte on the
+ // page. Check this:
+ // https://msdn.microsoft.com/en-us/library/aa227153(v=vs.60).aspx.
+ Label check_offset;
+ Label touch_next_page;
+ jmp(&check_offset);
+ bind(&touch_next_page);
+ subq(rsp, Immediate(kStackPageSize));
+ // Just to touch the page, before we increment further.
+ movb(Operand(rsp, 0), Immediate(0));
+ subq(bytes_scratch, Immediate(kStackPageSize));
+
+ bind(&check_offset);
+ cmpq(bytes_scratch, Immediate(kStackPageSize));
+ j(greater, &touch_next_page);
+
+ subq(rsp, bytes_scratch);
+}
+
+void TurboAssembler::AllocateStackSpace(int bytes) {
+ while (bytes > kStackPageSize) {
+ subq(rsp, Immediate(kStackPageSize));
+ movb(Operand(rsp, 0), Immediate(0));
+ bytes -= kStackPageSize;
+ }
+ subq(rsp, Immediate(bytes));
+}
+#endif
+
+void MacroAssembler::EnterExitFramePrologue(bool save_rax,
+ StackFrame::Type frame_type) {
+ DCHECK(frame_type == StackFrame::EXIT ||
+ frame_type == StackFrame::BUILTIN_EXIT);
+
+ // Set up the frame structure on the stack.
+ // All constants are relative to the frame pointer of the exit frame.
+ DCHECK_EQ(kFPOnStackSize + kPCOnStackSize,
+ ExitFrameConstants::kCallerSPDisplacement);
+ DCHECK_EQ(kFPOnStackSize, ExitFrameConstants::kCallerPCOffset);
+ DCHECK_EQ(0 * kSystemPointerSize, ExitFrameConstants::kCallerFPOffset);
+ pushq(rbp);
+ movq(rbp, rsp);
+
+ // Reserve room for entry stack pointer.
+ Push(Immediate(StackFrame::TypeToMarker(frame_type)));
+ DCHECK_EQ(-2 * kSystemPointerSize, ExitFrameConstants::kSPOffset);
+ Push(Immediate(0)); // Saved entry sp, patched before call.
+
+ // Save the frame pointer and the context in top.
+ if (save_rax) {
+ movq(r14, rax); // Backup rax in callee-save register.
+ }
+
+ Store(
+ ExternalReference::Create(IsolateAddressId::kCEntryFPAddress, isolate()),
+ rbp);
+ Store(ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()),
+ rsi);
+ Store(
+ ExternalReference::Create(IsolateAddressId::kCFunctionAddress, isolate()),
+ rbx);
+}
+
+void MacroAssembler::EnterExitFrameEpilogue(int arg_stack_space,
+ bool save_doubles) {
+#ifdef V8_TARGET_OS_WIN
+ const int kShadowSpace = 4;
+ arg_stack_space += kShadowSpace;
+#endif
+ // Optionally save all XMM registers.
+ if (save_doubles) {
+ int space = XMMRegister::kNumRegisters * kDoubleSize +
+ arg_stack_space * kSystemPointerSize;
+ AllocateStackSpace(space);
+ int offset = -ExitFrameConstants::kFixedFrameSizeFromFp;
+ const RegisterConfiguration* config = RegisterConfiguration::Default();
+ for (int i = 0; i < config->num_allocatable_double_registers(); ++i) {
+ DoubleRegister reg =
+ DoubleRegister::from_code(config->GetAllocatableDoubleCode(i));
+ Movsd(Operand(rbp, offset - ((i + 1) * kDoubleSize)), reg);
+ }
+ } else if (arg_stack_space > 0) {
+ AllocateStackSpace(arg_stack_space * kSystemPointerSize);
+ }
+
+ // Get the required frame alignment for the OS.
+ const int kFrameAlignment = base::OS::ActivationFrameAlignment();
+ if (kFrameAlignment > 0) {
+ DCHECK(base::bits::IsPowerOfTwo(kFrameAlignment));
+ DCHECK(is_int8(kFrameAlignment));
+ andq(rsp, Immediate(-kFrameAlignment));
+ }
+
+ // Patch the saved entry sp.
+ movq(Operand(rbp, ExitFrameConstants::kSPOffset), rsp);
+}
+
+void MacroAssembler::EnterExitFrame(int arg_stack_space, bool save_doubles,
+ StackFrame::Type frame_type) {
+ EnterExitFramePrologue(true, frame_type);
+
+ // Set up argv in callee-saved register r15. It is reused in LeaveExitFrame,
+ // so it must be retained across the C-call.
+ int offset = StandardFrameConstants::kCallerSPOffset - kSystemPointerSize;
+ leaq(r15, Operand(rbp, r14, times_system_pointer_size, offset));
+
+ EnterExitFrameEpilogue(arg_stack_space, save_doubles);
+}
+
+void MacroAssembler::EnterApiExitFrame(int arg_stack_space) {
+ EnterExitFramePrologue(false, StackFrame::EXIT);
+ EnterExitFrameEpilogue(arg_stack_space, false);
+}
+
+void MacroAssembler::LeaveExitFrame(bool save_doubles, bool pop_arguments) {
+ // Registers:
+ // r15 : argv
+ if (save_doubles) {
+ int offset = -ExitFrameConstants::kFixedFrameSizeFromFp;
+ const RegisterConfiguration* config = RegisterConfiguration::Default();
+ for (int i = 0; i < config->num_allocatable_double_registers(); ++i) {
+ DoubleRegister reg =
+ DoubleRegister::from_code(config->GetAllocatableDoubleCode(i));
+ Movsd(reg, Operand(rbp, offset - ((i + 1) * kDoubleSize)));
+ }
+ }
+
+ if (pop_arguments) {
+ // Get the return address from the stack and restore the frame pointer.
+ movq(rcx, Operand(rbp, kFPOnStackSize));
+ movq(rbp, Operand(rbp, 0 * kSystemPointerSize));
+
+ // Drop everything up to and including the arguments and the receiver
+ // from the caller stack.
+ leaq(rsp, Operand(r15, 1 * kSystemPointerSize));
+
+ PushReturnAddressFrom(rcx);
+ } else {
+ // Otherwise just leave the exit frame.
+ leave();
+ }
+
+ LeaveExitFrameEpilogue();
+}
+
+void MacroAssembler::LeaveApiExitFrame() {
+ movq(rsp, rbp);
+ popq(rbp);
+
+ LeaveExitFrameEpilogue();
+}
+
+void MacroAssembler::LeaveExitFrameEpilogue() {
+ // Restore current context from top and clear it in debug mode.
+ ExternalReference context_address =
+ ExternalReference::Create(IsolateAddressId::kContextAddress, isolate());
+ Operand context_operand = ExternalReferenceAsOperand(context_address);
+ movq(rsi, context_operand);
+#ifdef DEBUG
+ movq(context_operand, Immediate(Context::kInvalidContext));
+#endif
+
+ // Clear the top frame.
+ ExternalReference c_entry_fp_address =
+ ExternalReference::Create(IsolateAddressId::kCEntryFPAddress, isolate());
+ Operand c_entry_fp_operand = ExternalReferenceAsOperand(c_entry_fp_address);
+ movq(c_entry_fp_operand, Immediate(0));
+}
+
+#ifdef V8_TARGET_OS_WIN
+static const int kRegisterPassedArguments = 4;
+#else
+static const int kRegisterPassedArguments = 6;
+#endif
+
+void MacroAssembler::LoadNativeContextSlot(int index, Register dst) {
+ // Load native context.
+ LoadMap(dst, rsi);
+ LoadTaggedPointerField(
+ dst,
+ FieldOperand(dst, Map::kConstructorOrBackPointerOrNativeContextOffset));
+ // Load value from native context.
+ LoadTaggedPointerField(dst, Operand(dst, Context::SlotOffset(index)));
+}
+
+int TurboAssembler::ArgumentStackSlotsForCFunctionCall(int num_arguments) {
+ // On Windows 64 stack slots are reserved by the caller for all arguments
+ // including the ones passed in registers, and space is always allocated for
+ // the four register arguments even if the function takes fewer than four
+ // arguments.
+ // On AMD64 ABI (Linux/Mac) the first six arguments are passed in registers
+ // and the caller does not reserve stack slots for them.
+ DCHECK_GE(num_arguments, 0);
+#ifdef V8_TARGET_OS_WIN
+ const int kMinimumStackSlots = kRegisterPassedArguments;
+ if (num_arguments < kMinimumStackSlots) return kMinimumStackSlots;
+ return num_arguments;
+#else
+ if (num_arguments < kRegisterPassedArguments) return 0;
+ return num_arguments - kRegisterPassedArguments;
+#endif
+}
+
+void TurboAssembler::PrepareCallCFunction(int num_arguments) {
+ int frame_alignment = base::OS::ActivationFrameAlignment();
+ DCHECK_NE(frame_alignment, 0);
+ DCHECK_GE(num_arguments, 0);
+
+ // Make stack end at alignment and allocate space for arguments and old rsp.
+ movq(kScratchRegister, rsp);
+ DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
+ int argument_slots_on_stack =
+ ArgumentStackSlotsForCFunctionCall(num_arguments);
+ AllocateStackSpace((argument_slots_on_stack + 1) * kSystemPointerSize);
+ andq(rsp, Immediate(-frame_alignment));
+ movq(Operand(rsp, argument_slots_on_stack * kSystemPointerSize),
+ kScratchRegister);
+}
+
+void TurboAssembler::CallCFunction(ExternalReference function,
+ int num_arguments) {
+ LoadAddress(rax, function);
+ CallCFunction(rax, num_arguments);
+}
+
+void TurboAssembler::CallCFunction(Register function, int num_arguments) {
+ DCHECK_LE(num_arguments, kMaxCParameters);
+ DCHECK(has_frame());
+ // Check stack alignment.
+ if (emit_debug_code()) {
+ CheckStackAlignment();
+ }
+
+ // Save the frame pointer and PC so that the stack layout remains iterable,
+ // even without an ExitFrame which normally exists between JS and C frames.
+ Label get_pc;
+ DCHECK(!AreAliased(kScratchRegister, function));
+ leaq(kScratchRegister, Operand(&get_pc, 0));
+ bind(&get_pc);
+
+ // Addressing the following external references is tricky because we need
+ // this to work in three situations:
+ // 1. In wasm compilation, the isolate is nullptr and thus no
+ // ExternalReference can be created, but we can construct the address
+ // directly using the root register and a static offset.
+ // 2. In normal JIT (and builtin) compilation, the external reference is
+ // usually addressed through the root register, so we can use the direct
+ // offset directly in most cases.
+ // 3. In regexp compilation, the external reference is embedded into the reloc
+ // info.
+ // The solution here is to use root register offsets wherever possible in
+ // which case we can construct it directly. When falling back to external
+ // references we need to ensure that the scratch register does not get
+ // accidentally overwritten. If we run into more such cases in the future, we
+ // should implement a more general solution.
+ if (root_array_available()) {
+ movq(Operand(kRootRegister, IsolateData::fast_c_call_caller_pc_offset()),
+ kScratchRegister);
+ movq(Operand(kRootRegister, IsolateData::fast_c_call_caller_fp_offset()),
+ rbp);
+ } else {
+ DCHECK_NOT_NULL(isolate());
+ // Use alternative scratch register in order not to overwrite
+ // kScratchRegister.
+ Register scratch = r12;
+ pushq(scratch);
+
+ movq(ExternalReferenceAsOperand(
+ ExternalReference::fast_c_call_caller_pc_address(isolate()),
+ scratch),
+ kScratchRegister);
+ movq(ExternalReferenceAsOperand(
+ ExternalReference::fast_c_call_caller_fp_address(isolate())),
+ rbp);
+
+ popq(scratch);
+ }
+
+ call(function);
+
+ // We don't unset the PC; the FP is the source of truth.
+ if (root_array_available()) {
+ movq(Operand(kRootRegister, IsolateData::fast_c_call_caller_fp_offset()),
+ Immediate(0));
+ } else {
+ DCHECK_NOT_NULL(isolate());
+ movq(ExternalReferenceAsOperand(
+ ExternalReference::fast_c_call_caller_fp_address(isolate())),
+ Immediate(0));
+ }
+
+ DCHECK_NE(base::OS::ActivationFrameAlignment(), 0);
+ DCHECK_GE(num_arguments, 0);
+ int argument_slots_on_stack =
+ ArgumentStackSlotsForCFunctionCall(num_arguments);
+ movq(rsp, Operand(rsp, argument_slots_on_stack * kSystemPointerSize));
+}
+
+void TurboAssembler::CheckPageFlag(Register object, Register scratch, int mask,
+ Condition cc, Label* condition_met,
+ Label::Distance condition_met_distance) {
+ DCHECK(cc == zero || cc == not_zero);
+ if (scratch == object) {
+ andq(scratch, Immediate(~kPageAlignmentMask));
+ } else {
+ movq(scratch, Immediate(~kPageAlignmentMask));
+ andq(scratch, object);
+ }
+ if (mask < (1 << kBitsPerByte)) {
+ testb(Operand(scratch, BasicMemoryChunk::kFlagsOffset),
+ Immediate(static_cast<uint8_t>(mask)));
+ } else {
+ testl(Operand(scratch, BasicMemoryChunk::kFlagsOffset), Immediate(mask));
+ }
+ j(cc, condition_met, condition_met_distance);
+}
+
+void TurboAssembler::ComputeCodeStartAddress(Register dst) {
+ Label current;
+ bind(¤t);
+ int pc = pc_offset();
+ // Load effective address to get the address of the current instruction.
+ leaq(dst, Operand(¤t, -pc));
+}
+
+void TurboAssembler::ResetSpeculationPoisonRegister() {
+ // TODO(tebbi): Perhaps, we want to put an lfence here.
+ Set(kSpeculationPoisonRegister, -1);
+}
+
+void TurboAssembler::CallForDeoptimization(Builtins::Name target, int,
+ Label* exit, DeoptimizeKind kind,
+ Label*) {
+ // Note: Assembler::call is used here on purpose to guarantee fixed-size
+ // exits even on Atom CPUs; see TurboAssembler::Call for Atom-specific
+ // performance tuning which emits a different instruction sequence.
+ call(EntryFromBuiltinIndexAsOperand(target));
+ DCHECK_EQ(SizeOfCodeGeneratedSince(exit),
+ (kind == DeoptimizeKind::kLazy)
+ ? Deoptimizer::kLazyDeoptExitSize
+ : Deoptimizer::kNonLazyDeoptExitSize);
+ USE(exit, kind);
+}
+
+void TurboAssembler::Trap() { int3(); }
+void TurboAssembler::DebugBreak() { int3(); }
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_TARGET_ARCH_X64
diff --git a/src/codegen/x64/macro-assembler-x64.h b/src/codegen/x64/macro-assembler-x64.h
new file mode 100644
index 0000000..9fc4d94
--- /dev/null
+++ b/src/codegen/x64/macro-assembler-x64.h
@@ -0,0 +1,1119 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef INCLUDED_FROM_MACRO_ASSEMBLER_H
+#error This header must be included via macro-assembler.h
+#endif
+
+#ifndef V8_CODEGEN_X64_MACRO_ASSEMBLER_X64_H_
+#define V8_CODEGEN_X64_MACRO_ASSEMBLER_X64_H_
+
+#include "src/base/flags.h"
+#include "src/codegen/bailout-reason.h"
+#include "src/codegen/x64/assembler-x64.h"
+#include "src/common/globals.h"
+#include "src/objects/contexts.h"
+
+namespace v8 {
+namespace internal {
+
+// Convenience for platform-independent signatures.
+using MemOperand = Operand;
+
+class StringConstantBase;
+
+enum RememberedSetAction { EMIT_REMEMBERED_SET, OMIT_REMEMBERED_SET };
+enum SmiCheck { INLINE_SMI_CHECK, OMIT_SMI_CHECK };
+
+struct SmiIndex {
+ SmiIndex(Register index_register, ScaleFactor scale)
+ : reg(index_register), scale(scale) {}
+ Register reg;
+ ScaleFactor scale;
+};
+
+// TODO(victorgomes): Move definition to macro-assembler.h, once all other
+// platforms are updated.
+enum class StackLimitKind { kInterruptStackLimit, kRealStackLimit };
+
+// Convenient class to access arguments below the stack pointer.
+class StackArgumentsAccessor {
+ public:
+ // argc = the number of arguments not including the receiver.
+ explicit StackArgumentsAccessor(Register argc) : argc_(argc) {
+ DCHECK_NE(argc_, no_reg);
+ }
+
+ // Argument 0 is the receiver (despite argc not including the receiver).
+ Operand operator[](int index) const { return GetArgumentOperand(index); }
+
+ Operand GetArgumentOperand(int index) const;
+ Operand GetReceiverOperand() const { return GetArgumentOperand(0); }
+
+ private:
+ const Register argc_;
+
+ DISALLOW_IMPLICIT_CONSTRUCTORS(StackArgumentsAccessor);
+};
+
+class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase {
+ public:
+ using TurboAssemblerBase::TurboAssemblerBase;
+
+ template <typename Dst, typename... Args>
+ struct AvxHelper {
+ Assembler* assm;
+ base::Optional<CpuFeature> feature = base::nullopt;
+ // Call a method where the AVX version expects the dst argument to be
+ // duplicated.
+ template <void (Assembler::*avx)(Dst, Dst, Args...),
+ void (Assembler::*no_avx)(Dst, Args...)>
+ void emit(Dst dst, Args... args) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(assm, AVX);
+ (assm->*avx)(dst, dst, args...);
+ } else if (feature.has_value()) {
+ DCHECK(CpuFeatures::IsSupported(*feature));
+ CpuFeatureScope scope(assm, *feature);
+ (assm->*no_avx)(dst, args...);
+ } else {
+ (assm->*no_avx)(dst, args...);
+ }
+ }
+
+ // Call a method where the AVX version expects no duplicated dst argument.
+ template <void (Assembler::*avx)(Dst, Args...),
+ void (Assembler::*no_avx)(Dst, Args...)>
+ void emit(Dst dst, Args... args) {
+ if (CpuFeatures::IsSupported(AVX)) {
+ CpuFeatureScope scope(assm, AVX);
+ (assm->*avx)(dst, args...);
+ } else if (feature.has_value()) {
+ DCHECK(CpuFeatures::IsSupported(*feature));
+ CpuFeatureScope scope(assm, *feature);
+ (assm->*no_avx)(dst, args...);
+ } else {
+ (assm->*no_avx)(dst, args...);
+ }
+ }
+ };
+
+#define AVX_OP(macro_name, name) \
+ template <typename Dst, typename... Args> \
+ void macro_name(Dst dst, Args... args) { \
+ AvxHelper<Dst, Args...>{this} \
+ .template emit<&Assembler::v##name, &Assembler::name>(dst, args...); \
+ }
+
+#define AVX_OP_SSE3(macro_name, name) \
+ template <typename Dst, typename... Args> \
+ void macro_name(Dst dst, Args... args) { \
+ AvxHelper<Dst, Args...>{this, base::Optional<CpuFeature>(SSE3)} \
+ .template emit<&Assembler::v##name, &Assembler::name>(dst, args...); \
+ }
+
+#define AVX_OP_SSSE3(macro_name, name) \
+ template <typename Dst, typename... Args> \
+ void macro_name(Dst dst, Args... args) { \
+ AvxHelper<Dst, Args...>{this, base::Optional<CpuFeature>(SSSE3)} \
+ .template emit<&Assembler::v##name, &Assembler::name>(dst, args...); \
+ }
+
+#define AVX_OP_SSE4_1(macro_name, name) \
+ template <typename Dst, typename... Args> \
+ void macro_name(Dst dst, Args... args) { \
+ AvxHelper<Dst, Args...>{this, base::Optional<CpuFeature>(SSE4_1)} \
+ .template emit<&Assembler::v##name, &Assembler::name>(dst, args...); \
+ }
+#define AVX_OP_SSE4_2(macro_name, name) \
+ template <typename Dst, typename... Args> \
+ void macro_name(Dst dst, Args... args) { \
+ AvxHelper<Dst, Args...>{this, base::Optional<CpuFeature>(SSE4_2)} \
+ .template emit<&Assembler::v##name, &Assembler::name>(dst, args...); \
+ }
+ AVX_OP(Subsd, subsd)
+ AVX_OP(Divss, divss)
+ AVX_OP(Divsd, divsd)
+ AVX_OP(Orps, orps)
+ AVX_OP(Xorps, xorps)
+ AVX_OP(Xorpd, xorpd)
+ AVX_OP(Movd, movd)
+ AVX_OP(Movq, movq)
+ AVX_OP(Movaps, movaps)
+ AVX_OP(Movapd, movapd)
+ AVX_OP(Movups, movups)
+ AVX_OP(Movmskps, movmskps)
+ AVX_OP(Movmskpd, movmskpd)
+ AVX_OP(Pmovmskb, pmovmskb)
+ AVX_OP(Movss, movss)
+ AVX_OP(Movsd, movsd)
+ AVX_OP(Movdqu, movdqu)
+ AVX_OP(Movlps, movlps)
+ AVX_OP(Movhps, movhps)
+ AVX_OP(Pcmpeqb, pcmpeqb)
+ AVX_OP(Pcmpeqw, pcmpeqw)
+ AVX_OP(Pcmpeqd, pcmpeqd)
+ AVX_OP(Pcmpgtb, pcmpgtb)
+ AVX_OP(Pcmpgtw, pcmpgtw)
+ AVX_OP(Pmaxsw, pmaxsw)
+ AVX_OP(Pmaxub, pmaxub)
+ AVX_OP(Pminsw, pminsw)
+ AVX_OP(Pminub, pminub)
+ AVX_OP(Addss, addss)
+ AVX_OP(Addsd, addsd)
+ AVX_OP(Mulsd, mulsd)
+ AVX_OP(Andps, andps)
+ AVX_OP(Andnps, andnps)
+ AVX_OP(Andpd, andpd)
+ AVX_OP(Andnpd, andnpd)
+ AVX_OP(Orpd, orpd)
+ AVX_OP(Cmpeqps, cmpeqps)
+ AVX_OP(Cmpltps, cmpltps)
+ AVX_OP(Cmpleps, cmpleps)
+ AVX_OP(Cmpneqps, cmpneqps)
+ AVX_OP(Cmpnltps, cmpnltps)
+ AVX_OP(Cmpnleps, cmpnleps)
+ AVX_OP(Cmpeqpd, cmpeqpd)
+ AVX_OP(Cmpltpd, cmpltpd)
+ AVX_OP(Cmplepd, cmplepd)
+ AVX_OP(Cmpneqpd, cmpneqpd)
+ AVX_OP(Cmpnltpd, cmpnltpd)
+ AVX_OP(Cmpnlepd, cmpnlepd)
+ AVX_OP(Sqrtss, sqrtss)
+ AVX_OP(Sqrtsd, sqrtsd)
+ AVX_OP(Sqrtps, sqrtps)
+ AVX_OP(Sqrtpd, sqrtpd)
+ AVX_OP(Cvttps2dq, cvttps2dq)
+ AVX_OP(Ucomiss, ucomiss)
+ AVX_OP(Ucomisd, ucomisd)
+ AVX_OP(Pand, pand)
+ AVX_OP(Por, por)
+ AVX_OP(Pxor, pxor)
+ AVX_OP(Psubb, psubb)
+ AVX_OP(Psubw, psubw)
+ AVX_OP(Psubd, psubd)
+ AVX_OP(Psubq, psubq)
+ AVX_OP(Psubsb, psubsb)
+ AVX_OP(Psubsw, psubsw)
+ AVX_OP(Psubusb, psubusb)
+ AVX_OP(Psubusw, psubusw)
+ AVX_OP(Pslld, pslld)
+ AVX_OP(Pavgb, pavgb)
+ AVX_OP(Pavgw, pavgw)
+ AVX_OP(Psraw, psraw)
+ AVX_OP(Psrad, psrad)
+ AVX_OP(Psllw, psllw)
+ AVX_OP(Psllq, psllq)
+ AVX_OP(Psrlw, psrlw)
+ AVX_OP(Psrld, psrld)
+ AVX_OP(Psrlq, psrlq)
+ AVX_OP(Pmaddwd, pmaddwd)
+ AVX_OP(Paddb, paddb)
+ AVX_OP(Paddw, paddw)
+ AVX_OP(Paddd, paddd)
+ AVX_OP(Paddq, paddq)
+ AVX_OP(Paddsb, paddsb)
+ AVX_OP(Paddsw, paddsw)
+ AVX_OP(Paddusb, paddusb)
+ AVX_OP(Paddusw, paddusw)
+ AVX_OP(Pcmpgtd, pcmpgtd)
+ AVX_OP(Pmullw, pmullw)
+ AVX_OP(Pmuludq, pmuludq)
+ AVX_OP(Addpd, addpd)
+ AVX_OP(Subpd, subpd)
+ AVX_OP(Mulpd, mulpd)
+ AVX_OP(Minps, minps)
+ AVX_OP(Minpd, minpd)
+ AVX_OP(Divpd, divpd)
+ AVX_OP(Maxps, maxps)
+ AVX_OP(Maxpd, maxpd)
+ AVX_OP(Cvtdq2ps, cvtdq2ps)
+ AVX_OP(Rcpps, rcpps)
+ AVX_OP(Rsqrtps, rsqrtps)
+ AVX_OP(Addps, addps)
+ AVX_OP(Subps, subps)
+ AVX_OP(Mulps, mulps)
+ AVX_OP(Divps, divps)
+ AVX_OP(Pshuflw, pshuflw)
+ AVX_OP(Pshufhw, pshufhw)
+ AVX_OP(Packsswb, packsswb)
+ AVX_OP(Packuswb, packuswb)
+ AVX_OP(Packssdw, packssdw)
+ AVX_OP(Punpcklbw, punpcklbw)
+ AVX_OP(Punpcklwd, punpcklwd)
+ AVX_OP(Punpckldq, punpckldq)
+ AVX_OP(Punpckhbw, punpckhbw)
+ AVX_OP(Punpckhwd, punpckhwd)
+ AVX_OP(Punpckhdq, punpckhdq)
+ AVX_OP(Punpcklqdq, punpcklqdq)
+ AVX_OP(Punpckhqdq, punpckhqdq)
+ AVX_OP(Pshufd, pshufd)
+ AVX_OP(Cmpps, cmpps)
+ AVX_OP(Cmppd, cmppd)
+ AVX_OP(Movlhps, movlhps)
+ AVX_OP_SSE3(Haddps, haddps)
+ AVX_OP_SSE3(Movddup, movddup)
+ AVX_OP_SSSE3(Phaddd, phaddd)
+ AVX_OP_SSSE3(Phaddw, phaddw)
+ AVX_OP_SSSE3(Pshufb, pshufb)
+ AVX_OP_SSSE3(Psignb, psignb)
+ AVX_OP_SSSE3(Psignw, psignw)
+ AVX_OP_SSSE3(Psignd, psignd)
+ AVX_OP_SSSE3(Palignr, palignr)
+ AVX_OP_SSSE3(Pabsb, pabsb)
+ AVX_OP_SSSE3(Pabsw, pabsw)
+ AVX_OP_SSSE3(Pabsd, pabsd)
+ AVX_OP_SSE4_1(Pcmpeqq, pcmpeqq)
+ AVX_OP_SSE4_1(Packusdw, packusdw)
+ AVX_OP_SSE4_1(Pminsb, pminsb)
+ AVX_OP_SSE4_1(Pminsd, pminsd)
+ AVX_OP_SSE4_1(Pminuw, pminuw)
+ AVX_OP_SSE4_1(Pminud, pminud)
+ AVX_OP_SSE4_1(Pmaxsb, pmaxsb)
+ AVX_OP_SSE4_1(Pmaxsd, pmaxsd)
+ AVX_OP_SSE4_1(Pmaxuw, pmaxuw)
+ AVX_OP_SSE4_1(Pmaxud, pmaxud)
+ AVX_OP_SSE4_1(Pmulld, pmulld)
+ AVX_OP_SSE4_1(Extractps, extractps)
+ AVX_OP_SSE4_1(Insertps, insertps)
+ AVX_OP_SSE4_1(Pinsrq, pinsrq)
+ AVX_OP_SSE4_1(Pblendw, pblendw)
+ AVX_OP_SSE4_1(Ptest, ptest)
+ AVX_OP_SSE4_1(Pmovsxbw, pmovsxbw)
+ AVX_OP_SSE4_1(Pmovsxwd, pmovsxwd)
+ AVX_OP_SSE4_1(Pmovsxdq, pmovsxdq)
+ AVX_OP_SSE4_1(Pmovzxbw, pmovzxbw)
+ AVX_OP_SSE4_1(Pmovzxwd, pmovzxwd)
+ AVX_OP_SSE4_1(Pmovzxdq, pmovzxdq)
+ AVX_OP_SSE4_1(Pextrb, pextrb)
+ AVX_OP_SSE4_1(Pextrw, pextrw)
+ AVX_OP_SSE4_1(Pextrq, pextrq)
+ AVX_OP_SSE4_1(Roundps, roundps)
+ AVX_OP_SSE4_1(Roundpd, roundpd)
+ AVX_OP_SSE4_1(Roundss, roundss)
+ AVX_OP_SSE4_1(Roundsd, roundsd)
+ AVX_OP_SSE4_2(Pcmpgtq, pcmpgtq)
+
+#undef AVX_OP
+
+ void PushReturnAddressFrom(Register src) { pushq(src); }
+ void PopReturnAddressTo(Register dst) { popq(dst); }
+
+ void Ret();
+
+ // Return and drop arguments from stack, where the number of arguments
+ // may be bigger than 2^16 - 1. Requires a scratch register.
+ void Ret(int bytes_dropped, Register scratch);
+
+ // Load a register with a long value as efficiently as possible.
+ void Set(Register dst, int64_t x);
+ void Set(Operand dst, intptr_t x);
+
+ // Operations on roots in the root-array.
+ void LoadRoot(Register destination, RootIndex index) override;
+ void LoadRoot(Operand destination, RootIndex index) {
+ LoadRoot(kScratchRegister, index);
+ movq(destination, kScratchRegister);
+ }
+
+ void Push(Register src);
+ void Push(Operand src);
+ void Push(Immediate value);
+ void Push(Smi smi);
+ void Push(Handle<HeapObject> source);
+
+ enum class PushArrayOrder { kNormal, kReverse };
+ // `array` points to the first element (the lowest address).
+ // `array` and `size` are not modified.
+ void PushArray(Register array, Register size, Register scratch,
+ PushArrayOrder order = PushArrayOrder::kNormal);
+
+ // Before calling a C-function from generated code, align arguments on stack.
+ // After aligning the frame, arguments must be stored in rsp[0], rsp[8],
+ // etc., not pushed. The argument count assumes all arguments are word sized.
+ // The number of slots reserved for arguments depends on platform. On Windows
+ // stack slots are reserved for the arguments passed in registers. On other
+ // platforms stack slots are only reserved for the arguments actually passed
+ // on the stack.
+ void PrepareCallCFunction(int num_arguments);
+
+ // Calls a C function and cleans up the space for arguments allocated
+ // by PrepareCallCFunction. The called function is not allowed to trigger a
+ // garbage collection, since that might move the code and invalidate the
+ // return address (unless this is somehow accounted for by the called
+ // function).
+ void CallCFunction(ExternalReference function, int num_arguments);
+ void CallCFunction(Register function, int num_arguments);
+
+ // Calculate the number of stack slots to reserve for arguments when calling a
+ // C function.
+ int ArgumentStackSlotsForCFunctionCall(int num_arguments);
+
+ void CheckPageFlag(Register object, Register scratch, int mask, Condition cc,
+ Label* condition_met,
+ Label::Distance condition_met_distance = Label::kFar);
+
+ void Cvtss2sd(XMMRegister dst, XMMRegister src);
+ void Cvtss2sd(XMMRegister dst, Operand src);
+ void Cvtsd2ss(XMMRegister dst, XMMRegister src);
+ void Cvtsd2ss(XMMRegister dst, Operand src);
+ void Cvttsd2si(Register dst, XMMRegister src);
+ void Cvttsd2si(Register dst, Operand src);
+ void Cvttsd2siq(Register dst, XMMRegister src);
+ void Cvttsd2siq(Register dst, Operand src);
+ void Cvttss2si(Register dst, XMMRegister src);
+ void Cvttss2si(Register dst, Operand src);
+ void Cvttss2siq(Register dst, XMMRegister src);
+ void Cvttss2siq(Register dst, Operand src);
+ void Cvtlui2ss(XMMRegister dst, Register src);
+ void Cvtlui2ss(XMMRegister dst, Operand src);
+ void Cvtlui2sd(XMMRegister dst, Register src);
+ void Cvtlui2sd(XMMRegister dst, Operand src);
+ void Cvtqui2ss(XMMRegister dst, Register src);
+ void Cvtqui2ss(XMMRegister dst, Operand src);
+ void Cvtqui2sd(XMMRegister dst, Register src);
+ void Cvtqui2sd(XMMRegister dst, Operand src);
+ void Cvttsd2uiq(Register dst, Operand src, Label* fail = nullptr);
+ void Cvttsd2uiq(Register dst, XMMRegister src, Label* fail = nullptr);
+ void Cvttss2uiq(Register dst, Operand src, Label* fail = nullptr);
+ void Cvttss2uiq(Register dst, XMMRegister src, Label* fail = nullptr);
+
+ // cvtsi2sd and cvtsi2ss instructions only write to the low 64/32-bit of dst
+ // register, which hinders register renaming and makes dependence chains
+ // longer. So we use xorpd to clear the dst register before cvtsi2sd for
+ // non-AVX and a scratch XMM register as first src for AVX to solve this
+ // issue.
+ void Cvtqsi2ss(XMMRegister dst, Register src);
+ void Cvtqsi2ss(XMMRegister dst, Operand src);
+ void Cvtqsi2sd(XMMRegister dst, Register src);
+ void Cvtqsi2sd(XMMRegister dst, Operand src);
+ void Cvtlsi2ss(XMMRegister dst, Register src);
+ void Cvtlsi2ss(XMMRegister dst, Operand src);
+ void Cvtlsi2sd(XMMRegister dst, Register src);
+ void Cvtlsi2sd(XMMRegister dst, Operand src);
+
+ void Lzcntq(Register dst, Register src);
+ void Lzcntq(Register dst, Operand src);
+ void Lzcntl(Register dst, Register src);
+ void Lzcntl(Register dst, Operand src);
+ void Tzcntq(Register dst, Register src);
+ void Tzcntq(Register dst, Operand src);
+ void Tzcntl(Register dst, Register src);
+ void Tzcntl(Register dst, Operand src);
+ void Popcntl(Register dst, Register src);
+ void Popcntl(Register dst, Operand src);
+ void Popcntq(Register dst, Register src);
+ void Popcntq(Register dst, Operand src);
+
+ // Is the value a tagged smi.
+ Condition CheckSmi(Register src);
+ Condition CheckSmi(Operand src);
+
+ // Jump to label if the value is a tagged smi.
+ void JumpIfSmi(Register src, Label* on_smi,
+ Label::Distance near_jump = Label::kFar);
+
+ void JumpIfEqual(Register a, int32_t b, Label* dest) {
+ cmpl(a, Immediate(b));
+ j(equal, dest);
+ }
+
+ void JumpIfLessThan(Register a, int32_t b, Label* dest) {
+ cmpl(a, Immediate(b));
+ j(less, dest);
+ }
+
+ void LoadMap(Register destination, Register object);
+
+ void Move(Register dst, Smi source);
+
+ void Move(Operand dst, Smi source) {
+ Register constant = GetSmiConstant(source);
+ movq(dst, constant);
+ }
+
+ void Move(Register dst, ExternalReference ext);
+
+ void Move(XMMRegister dst, uint32_t src);
+ void Move(XMMRegister dst, uint64_t src);
+ void Move(XMMRegister dst, float src) { Move(dst, bit_cast<uint32_t>(src)); }
+ void Move(XMMRegister dst, double src) { Move(dst, bit_cast<uint64_t>(src)); }
+ void Move(XMMRegister dst, uint64_t high, uint64_t low);
+
+ // Move if the registers are not identical.
+ void Move(Register target, Register source);
+
+ void Move(Register dst, Handle<HeapObject> source,
+ RelocInfo::Mode rmode = RelocInfo::FULL_EMBEDDED_OBJECT);
+ void Move(Operand dst, Handle<HeapObject> source,
+ RelocInfo::Mode rmode = RelocInfo::FULL_EMBEDDED_OBJECT);
+
+ // Loads a pointer into a register with a relocation mode.
+ void Move(Register dst, Address ptr, RelocInfo::Mode rmode) {
+ // This method must not be used with heap object references. The stored
+ // address is not GC safe. Use the handle version instead.
+ DCHECK(rmode == RelocInfo::NONE || rmode > RelocInfo::LAST_GCED_ENUM);
+ movq(dst, Immediate64(ptr, rmode));
+ }
+
+ // Move src0 to dst0 and src1 to dst1, handling possible overlaps.
+ void MovePair(Register dst0, Register src0, Register dst1, Register src1);
+
+ void MoveStringConstant(
+ Register result, const StringConstantBase* string,
+ RelocInfo::Mode rmode = RelocInfo::FULL_EMBEDDED_OBJECT);
+
+ // Convert smi to word-size sign-extended value.
+ void SmiUntag(Register reg);
+ // Requires dst != src
+ void SmiUntag(Register dst, Register src);
+ void SmiUntag(Register dst, Operand src);
+
+ // Loads the address of the external reference into the destination
+ // register.
+ void LoadAddress(Register destination, ExternalReference source);
+
+ void LoadFromConstantsTable(Register destination,
+ int constant_index) override;
+ void LoadRootRegisterOffset(Register destination, intptr_t offset) override;
+ void LoadRootRelative(Register destination, int32_t offset) override;
+
+ // Operand pointing to an external reference.
+ // May emit code to set up the scratch register. The operand is
+ // only guaranteed to be correct as long as the scratch register
+ // isn't changed.
+ // If the operand is used more than once, use a scratch register
+ // that is guaranteed not to be clobbered.
+ Operand ExternalReferenceAsOperand(ExternalReference reference,
+ Register scratch = kScratchRegister);
+
+ void Call(Register reg) { call(reg); }
+ void Call(Operand op);
+ void Call(Handle<Code> code_object, RelocInfo::Mode rmode);
+ void Call(Address destination, RelocInfo::Mode rmode);
+ void Call(ExternalReference ext);
+ void Call(Label* target) { call(target); }
+
+ Operand EntryFromBuiltinIndexAsOperand(Builtins::Name builtin_index);
+ Operand EntryFromBuiltinIndexAsOperand(Register builtin_index);
+ void CallBuiltinByIndex(Register builtin_index) override;
+ void CallBuiltin(int builtin_index);
+
+ void LoadCodeObjectEntry(Register destination, Register code_object) override;
+ void CallCodeObject(Register code_object) override;
+ void JumpCodeObject(Register code_object) override;
+
+ void RetpolineCall(Register reg);
+ void RetpolineCall(Address destination, RelocInfo::Mode rmode);
+
+ void Jump(Address destination, RelocInfo::Mode rmode);
+ void Jump(const ExternalReference& reference) override;
+ void Jump(Operand op);
+ void Jump(Handle<Code> code_object, RelocInfo::Mode rmode,
+ Condition cc = always);
+
+ void RetpolineJump(Register reg);
+
+ void CallForDeoptimization(Builtins::Name target, int deopt_id, Label* exit,
+ DeoptimizeKind kind,
+ Label* jump_deoptimization_entry_label);
+
+ void Trap() override;
+ void DebugBreak() override;
+
+ // Shufps that will mov src into dst if AVX is not supported.
+ void Shufps(XMMRegister dst, XMMRegister src, byte imm8);
+
+ // Non-SSE2 instructions.
+ void Pextrd(Register dst, XMMRegister src, uint8_t imm8);
+
+ void Pinsrb(XMMRegister dst, XMMRegister src1, Register src2, uint8_t imm8);
+ void Pinsrb(XMMRegister dst, XMMRegister src1, Operand src2, uint8_t imm8);
+ void Pinsrw(XMMRegister dst, XMMRegister src1, Register src2, uint8_t imm8);
+ void Pinsrw(XMMRegister dst, XMMRegister src1, Operand src2, uint8_t imm8);
+ void Pinsrd(XMMRegister dst, XMMRegister src1, Register src2, uint8_t imm8);
+ void Pinsrd(XMMRegister dst, XMMRegister src1, Operand src2, uint8_t imm8);
+ void Pinsrd(XMMRegister dst, Register src2, uint8_t imm8);
+ void Pinsrd(XMMRegister dst, Operand src2, uint8_t imm8);
+ void Pinsrq(XMMRegister dst, XMMRegister src1, Register src2, uint8_t imm8);
+ void Pinsrq(XMMRegister dst, XMMRegister src1, Operand src2, uint8_t imm8);
+
+ void Psllq(XMMRegister dst, int imm8) { Psllq(dst, static_cast<byte>(imm8)); }
+ void Psllq(XMMRegister dst, byte imm8);
+ void Psrlq(XMMRegister dst, int imm8) { Psrlq(dst, static_cast<byte>(imm8)); }
+ void Psrlq(XMMRegister dst, byte imm8);
+ void Pslld(XMMRegister dst, byte imm8);
+ void Psrld(XMMRegister dst, byte imm8);
+
+ void Pblendvb(XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ XMMRegister mask);
+ void Blendvps(XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ XMMRegister mask);
+ void Blendvpd(XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ XMMRegister mask);
+
+ // Supports both SSE and AVX. Move src1 to dst if they are not equal on SSE.
+ void Pshufb(XMMRegister dst, XMMRegister src1, XMMRegister src2);
+
+ void CompareRoot(Register with, RootIndex index);
+ void CompareRoot(Operand with, RootIndex index);
+
+ // Generates function and stub prologue code.
+ void StubPrologue(StackFrame::Type type);
+ void Prologue();
+
+ // Calls Abort(msg) if the condition cc is not satisfied.
+ // Use --debug_code to enable.
+ void Assert(Condition cc, AbortReason reason);
+
+ // Like Assert(), but without condition.
+ // Use --debug_code to enable.
+ void AssertUnreachable(AbortReason reason);
+
+ // Abort execution if a 64 bit register containing a 32 bit payload does not
+ // have zeros in the top 32 bits, enabled via --debug-code.
+ void AssertZeroExtended(Register reg);
+
+ // Like Assert(), but always enabled.
+ void Check(Condition cc, AbortReason reason);
+
+ // Print a message to stdout and abort execution.
+ void Abort(AbortReason msg);
+
+ // Check that the stack is aligned.
+ void CheckStackAlignment();
+
+ // Activation support.
+ void EnterFrame(StackFrame::Type type);
+ void EnterFrame(StackFrame::Type type, bool load_constant_pool_pointer_reg) {
+ // Out-of-line constant pool not implemented on x64.
+ UNREACHABLE();
+ }
+ void LeaveFrame(StackFrame::Type type);
+
+// Allocate stack space of given size (i.e. decrement {rsp} by the value
+// stored in the given register, or by a constant). If you need to perform a
+// stack check, do it before calling this function because this function may
+// write into the newly allocated space. It may also overwrite the given
+// register's value, in the version that takes a register.
+#ifdef V8_TARGET_OS_WIN
+ void AllocateStackSpace(Register bytes_scratch);
+ void AllocateStackSpace(int bytes);
+#else
+ void AllocateStackSpace(Register bytes) { subq(rsp, bytes); }
+ void AllocateStackSpace(int bytes) { subq(rsp, Immediate(bytes)); }
+#endif
+
+ // Removes current frame and its arguments from the stack preserving the
+ // arguments and a return address pushed to the stack for the next call. Both
+ // |callee_args_count| and |caller_args_count| do not include receiver.
+ // |callee_args_count| is not modified. |caller_args_count| is trashed.
+ void PrepareForTailCall(Register callee_args_count,
+ Register caller_args_count, Register scratch0,
+ Register scratch1);
+
+ void InitializeRootRegister() {
+ ExternalReference isolate_root = ExternalReference::isolate_root(isolate());
+ Move(kRootRegister, isolate_root);
+ }
+
+ void SaveRegisters(RegList registers);
+ void RestoreRegisters(RegList registers);
+
+ void CallRecordWriteStub(Register object, Register address,
+ RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode);
+ void CallRecordWriteStub(Register object, Register address,
+ RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode, Address wasm_target);
+ void CallEphemeronKeyBarrier(Register object, Register address,
+ SaveFPRegsMode fp_mode);
+
+ void MoveNumber(Register dst, double value);
+ void MoveNonSmi(Register dst, double value);
+
+ // Calculate how much stack space (in bytes) are required to store caller
+ // registers excluding those specified in the arguments.
+ int RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
+ Register exclusion1 = no_reg,
+ Register exclusion2 = no_reg,
+ Register exclusion3 = no_reg) const;
+
+ // PushCallerSaved and PopCallerSaved do not arrange the registers in any
+ // particular order so they are not useful for calls that can cause a GC.
+ // The caller can exclude up to 3 registers that do not need to be saved and
+ // restored.
+
+ // Push caller saved registers on the stack, and return the number of bytes
+ // stack pointer is adjusted.
+ int PushCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1 = no_reg,
+ Register exclusion2 = no_reg,
+ Register exclusion3 = no_reg);
+ // Restore caller saved registers from the stack, and return the number of
+ // bytes stack pointer is adjusted.
+ int PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1 = no_reg,
+ Register exclusion2 = no_reg,
+ Register exclusion3 = no_reg);
+
+ // Compute the start of the generated instruction stream from the current PC.
+ // This is an alternative to embedding the {CodeObject} handle as a reference.
+ void ComputeCodeStartAddress(Register dst);
+
+ void ResetSpeculationPoisonRegister();
+
+ // Control-flow integrity:
+
+ // Define a function entrypoint. This doesn't emit any code for this
+ // architecture, as control-flow integrity is not supported for it.
+ void CodeEntry() {}
+ // Define an exception handler.
+ void ExceptionHandler() {}
+ // Define an exception handler and bind a label.
+ void BindExceptionHandler(Label* label) { bind(label); }
+
+ // ---------------------------------------------------------------------------
+ // Pointer compression support
+
+ // Loads a field containing a HeapObject and decompresses it if pointer
+ // compression is enabled.
+ void LoadTaggedPointerField(Register destination, Operand field_operand);
+
+ // Loads a field containing any tagged value and decompresses it if necessary.
+ void LoadAnyTaggedField(Register destination, Operand field_operand);
+
+ // Loads a field containing a HeapObject, decompresses it if necessary and
+ // pushes full pointer to the stack. When pointer compression is enabled,
+ // uses |scratch| to decompress the value.
+ void PushTaggedPointerField(Operand field_operand, Register scratch);
+
+ // Loads a field containing any tagged value, decompresses it if necessary and
+ // pushes the full pointer to the stack. When pointer compression is enabled,
+ // uses |scratch| to decompress the value.
+ void PushTaggedAnyField(Operand field_operand, Register scratch);
+
+ // Loads a field containing smi value and untags it.
+ void SmiUntagField(Register dst, Operand src);
+
+ // Compresses tagged value if necessary and stores it to given on-heap
+ // location.
+ void StoreTaggedField(Operand dst_field_operand, Immediate immediate);
+ void StoreTaggedField(Operand dst_field_operand, Register value);
+
+ // The following macros work even when pointer compression is not enabled.
+ void DecompressTaggedSigned(Register destination, Operand field_operand);
+ void DecompressTaggedPointer(Register destination, Operand field_operand);
+ void DecompressTaggedPointer(Register destination, Register source);
+ void DecompressAnyTagged(Register destination, Operand field_operand);
+
+ // ---------------------------------------------------------------------------
+ // V8 Heap sandbox support
+
+ // Loads a field containing off-heap pointer and does necessary decoding
+ // if V8 heap sandbox is enabled.
+ void LoadExternalPointerField(Register destination, Operand field_operand,
+ ExternalPointerTag tag);
+
+ protected:
+ static const int kSmiShift = kSmiTagSize + kSmiShiftSize;
+
+ // Returns a register holding the smi value. The register MUST NOT be
+ // modified. It may be the "smi 1 constant" register.
+ Register GetSmiConstant(Smi value);
+
+ void CallRecordWriteStub(Register object, Register address,
+ RememberedSetAction remembered_set_action,
+ SaveFPRegsMode fp_mode, Handle<Code> code_target,
+ Address wasm_target);
+};
+
+// MacroAssembler implements a collection of frequently used macros.
+class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler {
+ public:
+ using TurboAssembler::TurboAssembler;
+
+ // Loads and stores the value of an external reference.
+ // Special case code for load and store to take advantage of
+ // load_rax/store_rax if possible/necessary.
+ // For other operations, just use:
+ // Operand operand = ExternalReferenceAsOperand(extref);
+ // operation(operand, ..);
+ void Load(Register destination, ExternalReference source);
+ void Store(ExternalReference destination, Register source);
+
+ // Pushes the address of the external reference onto the stack.
+ void PushAddress(ExternalReference source);
+
+ // Operations on roots in the root-array.
+ // Load a root value where the index (or part of it) is variable.
+ // The variable_offset register is added to the fixed_offset value
+ // to get the index into the root-array.
+ void PushRoot(RootIndex index);
+
+ // Compare the object in a register to a value and jump if they are equal.
+ void JumpIfRoot(Register with, RootIndex index, Label* if_equal,
+ Label::Distance if_equal_distance = Label::kFar) {
+ CompareRoot(with, index);
+ j(equal, if_equal, if_equal_distance);
+ }
+ void JumpIfRoot(Operand with, RootIndex index, Label* if_equal,
+ Label::Distance if_equal_distance = Label::kFar) {
+ CompareRoot(with, index);
+ j(equal, if_equal, if_equal_distance);
+ }
+
+ // Compare the object in a register to a value and jump if they are not equal.
+ void JumpIfNotRoot(Register with, RootIndex index, Label* if_not_equal,
+ Label::Distance if_not_equal_distance = Label::kFar) {
+ CompareRoot(with, index);
+ j(not_equal, if_not_equal, if_not_equal_distance);
+ }
+ void JumpIfNotRoot(Operand with, RootIndex index, Label* if_not_equal,
+ Label::Distance if_not_equal_distance = Label::kFar) {
+ CompareRoot(with, index);
+ j(not_equal, if_not_equal, if_not_equal_distance);
+ }
+
+ // ---------------------------------------------------------------------------
+ // GC Support
+
+ // Notify the garbage collector that we wrote a pointer into an object.
+ // |object| is the object being stored into, |value| is the object being
+ // stored. value and scratch registers are clobbered by the operation.
+ // The offset is the offset from the start of the object, not the offset from
+ // the tagged HeapObject pointer. For use with FieldOperand(reg, off).
+ void RecordWriteField(
+ Register object, int offset, Register value, Register scratch,
+ SaveFPRegsMode save_fp,
+ RememberedSetAction remembered_set_action = EMIT_REMEMBERED_SET,
+ SmiCheck smi_check = INLINE_SMI_CHECK);
+
+ // For page containing |object| mark region covering |address|
+ // dirty. |object| is the object being stored into, |value| is the
+ // object being stored. The address and value registers are clobbered by the
+ // operation. RecordWrite filters out smis so it does not update
+ // the write barrier if the value is a smi.
+ void RecordWrite(
+ Register object, Register address, Register value, SaveFPRegsMode save_fp,
+ RememberedSetAction remembered_set_action = EMIT_REMEMBERED_SET,
+ SmiCheck smi_check = INLINE_SMI_CHECK);
+
+ // Frame restart support.
+ void MaybeDropFrames();
+
+ // Enter specific kind of exit frame; either in normal or
+ // debug mode. Expects the number of arguments in register rax and
+ // sets up the number of arguments in register rdi and the pointer
+ // to the first argument in register rsi.
+ //
+ // Allocates arg_stack_space * kSystemPointerSize memory (not GCed) on the
+ // stack accessible via StackSpaceOperand.
+ void EnterExitFrame(int arg_stack_space = 0, bool save_doubles = false,
+ StackFrame::Type frame_type = StackFrame::EXIT);
+
+ // Enter specific kind of exit frame. Allocates
+ // (arg_stack_space * kSystemPointerSize) memory (not GCed) on the stack
+ // accessible via StackSpaceOperand.
+ void EnterApiExitFrame(int arg_stack_space);
+
+ // Leave the current exit frame. Expects/provides the return value in
+ // register rax:rdx (untouched) and the pointer to the first
+ // argument in register rsi (if pop_arguments == true).
+ void LeaveExitFrame(bool save_doubles = false, bool pop_arguments = true);
+
+ // Leave the current exit frame. Expects/provides the return value in
+ // register rax (untouched).
+ void LeaveApiExitFrame();
+
+ // ---------------------------------------------------------------------------
+ // JavaScript invokes
+
+ // Invoke the JavaScript function code by either calling or jumping.
+ void InvokeFunctionCode(Register function, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count, InvokeFlag flag);
+
+ // On function call, call into the debugger.
+ void CallDebugOnFunctionCall(Register fun, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count);
+
+ // Invoke the JavaScript function in the given register. Changes the
+ // current context to the context in the function before invoking.
+ void InvokeFunction(Register function, Register new_target,
+ Register actual_parameter_count, InvokeFlag flag);
+
+ void InvokeFunction(Register function, Register new_target,
+ Register expected_parameter_count,
+ Register actual_parameter_count, InvokeFlag flag);
+
+ // ---------------------------------------------------------------------------
+ // Conversions between tagged smi values and non-tagged integer values.
+
+ // Tag an word-size value. The result must be known to be a valid smi value.
+ void SmiTag(Register reg);
+ // Requires dst != src
+ void SmiTag(Register dst, Register src);
+
+ // Simple comparison of smis. Both sides must be known smis to use these,
+ // otherwise use Cmp.
+ void SmiCompare(Register smi1, Register smi2);
+ void SmiCompare(Register dst, Smi src);
+ void SmiCompare(Register dst, Operand src);
+ void SmiCompare(Operand dst, Register src);
+ void SmiCompare(Operand dst, Smi src);
+
+ // Functions performing a check on a known or potential smi. Returns
+ // a condition that is satisfied if the check is successful.
+
+ // Test-and-jump functions. Typically combines a check function
+ // above with a conditional jump.
+
+ // Jump to label if the value is not a tagged smi.
+ void JumpIfNotSmi(Register src, Label* on_not_smi,
+ Label::Distance near_jump = Label::kFar);
+
+ // Jump to label if the value is not a tagged smi.
+ void JumpIfNotSmi(Operand src, Label* on_not_smi,
+ Label::Distance near_jump = Label::kFar);
+
+ // Operations on tagged smi values.
+
+ // Smis represent a subset of integers. The subset is always equivalent to
+ // a two's complement interpretation of a fixed number of bits.
+
+ // Add an integer constant to a tagged smi, giving a tagged smi as result.
+ // No overflow testing on the result is done.
+ void SmiAddConstant(Operand dst, Smi constant);
+
+ // Specialized operations
+
+ // Converts, if necessary, a smi to a combination of number and
+ // multiplier to be used as a scaled index.
+ // The src register contains a *positive* smi value. The shift is the
+ // power of two to multiply the index value by (e.g. to index by
+ // smi-value * kSystemPointerSize, pass the smi and kSystemPointerSizeLog2).
+ // The returned index register may be either src or dst, depending
+ // on what is most efficient. If src and dst are different registers,
+ // src is always unchanged.
+ SmiIndex SmiToIndex(Register dst, Register src, int shift);
+
+ // ---------------------------------------------------------------------------
+ // Macro instructions.
+
+ void Cmp(Register dst, Handle<Object> source);
+ void Cmp(Operand dst, Handle<Object> source);
+ void Cmp(Register dst, Smi src);
+ void Cmp(Operand dst, Smi src);
+ void Cmp(Register dst, int32_t src);
+
+ // Checks if value is in range [lower_limit, higher_limit] using a single
+ // comparison.
+ void JumpIfIsInRange(Register value, unsigned lower_limit,
+ unsigned higher_limit, Label* on_in_range,
+ Label::Distance near_jump = Label::kFar);
+
+ // Emit code to discard a non-negative number of pointer-sized elements
+ // from the stack, clobbering only the rsp register.
+ void Drop(int stack_elements);
+ // Emit code to discard a positive number of pointer-sized elements
+ // from the stack under the return address which remains on the top,
+ // clobbering the rsp register.
+ void DropUnderReturnAddress(int stack_elements,
+ Register scratch = kScratchRegister);
+
+ void PushQuad(Operand src);
+ void PushImm32(int32_t imm32);
+ void Pop(Register dst);
+ void Pop(Operand dst);
+ void PopQuad(Operand dst);
+
+ // ---------------------------------------------------------------------------
+ // SIMD macros.
+ void Absps(XMMRegister dst);
+ void Negps(XMMRegister dst);
+ void Abspd(XMMRegister dst);
+ void Negpd(XMMRegister dst);
+ // Generates a trampoline to jump to the off-heap instruction stream.
+ void JumpToInstructionStream(Address entry);
+
+ // Compare object type for heap object.
+ // Always use unsigned comparisons: above and below, not less and greater.
+ // Incoming register is heap_object and outgoing register is map.
+ // They may be the same register, and may be kScratchRegister.
+ void CmpObjectType(Register heap_object, InstanceType type, Register map);
+
+ // Compare instance type for map.
+ // Always use unsigned comparisons: above and below, not less and greater.
+ void CmpInstanceType(Register map, InstanceType type);
+
+ template <typename Field>
+ void DecodeField(Register reg) {
+ static const int shift = Field::kShift;
+ static const int mask = Field::kMask >> Field::kShift;
+ if (shift != 0) {
+ shrq(reg, Immediate(shift));
+ }
+ andq(reg, Immediate(mask));
+ }
+
+ // Abort execution if argument is a smi, enabled via --debug-code.
+ void AssertNotSmi(Register object);
+
+ // Abort execution if argument is not a smi, enabled via --debug-code.
+ void AssertSmi(Register object);
+ void AssertSmi(Operand object);
+
+ // Abort execution if argument is not a Constructor, enabled via --debug-code.
+ void AssertConstructor(Register object);
+
+ // Abort execution if argument is not a JSFunction, enabled via --debug-code.
+ void AssertFunction(Register object);
+
+ // Abort execution if argument is not a JSBoundFunction,
+ // enabled via --debug-code.
+ void AssertBoundFunction(Register object);
+
+ // Abort execution if argument is not a JSGeneratorObject (or subclass),
+ // enabled via --debug-code.
+ void AssertGeneratorObject(Register object);
+
+ // Abort execution if argument is not undefined or an AllocationSite, enabled
+ // via --debug-code.
+ void AssertUndefinedOrAllocationSite(Register object);
+
+ // ---------------------------------------------------------------------------
+ // Exception handling
+
+ // Push a new stack handler and link it into stack handler chain.
+ void PushStackHandler();
+
+ // Unlink the stack handler on top of the stack from the stack handler chain.
+ void PopStackHandler();
+
+ // ---------------------------------------------------------------------------
+ // Support functions.
+
+ // Load the global proxy from the current context.
+ void LoadGlobalProxy(Register dst) {
+ LoadNativeContextSlot(Context::GLOBAL_PROXY_INDEX, dst);
+ }
+
+ // Load the native context slot with the current index.
+ void LoadNativeContextSlot(int index, Register dst);
+
+ // ---------------------------------------------------------------------------
+ // Runtime calls
+
+ // Call a runtime routine.
+ void CallRuntime(const Runtime::Function* f, int num_arguments,
+ SaveFPRegsMode save_doubles = kDontSaveFPRegs);
+
+ // Convenience function: Same as above, but takes the fid instead.
+ void CallRuntime(Runtime::FunctionId fid,
+ SaveFPRegsMode save_doubles = kDontSaveFPRegs) {
+ const Runtime::Function* function = Runtime::FunctionForId(fid);
+ CallRuntime(function, function->nargs, save_doubles);
+ }
+
+ // Convenience function: Same as above, but takes the fid instead.
+ void CallRuntime(Runtime::FunctionId fid, int num_arguments,
+ SaveFPRegsMode save_doubles = kDontSaveFPRegs) {
+ CallRuntime(Runtime::FunctionForId(fid), num_arguments, save_doubles);
+ }
+
+ // Convenience function: tail call a runtime routine (jump)
+ void TailCallRuntime(Runtime::FunctionId fid);
+
+ // Jump to a runtime routines
+ void JumpToExternalReference(const ExternalReference& ext,
+ bool builtin_exit_frame = false);
+
+ // ---------------------------------------------------------------------------
+ // StatsCounter support
+ void IncrementCounter(StatsCounter* counter, int value);
+ void DecrementCounter(StatsCounter* counter, int value);
+
+ // ---------------------------------------------------------------------------
+ // Stack limit utilities
+ Operand StackLimitAsOperand(StackLimitKind kind);
+ void StackOverflowCheck(
+ Register num_args, Register scratch, Label* stack_overflow,
+ Label::Distance stack_overflow_distance = Label::kFar);
+
+ // ---------------------------------------------------------------------------
+ // In-place weak references.
+ void LoadWeakValue(Register in_out, Label* target_if_cleared);
+
+ // ---------------------------------------------------------------------------
+ // Debugging
+
+ static int SafepointRegisterStackIndex(Register reg) {
+ return SafepointRegisterStackIndex(reg.code());
+ }
+
+ private:
+ // Order general registers are pushed by Pushad.
+ // rax, rcx, rdx, rbx, rsi, rdi, r8, r9, r11, r12, r14, r15.
+ static const int kSafepointPushRegisterIndices[Register::kNumRegisters];
+ static const int kNumSafepointSavedRegisters = 12;
+
+ // Helper functions for generating invokes.
+ void InvokePrologue(Register expected_parameter_count,
+ Register actual_parameter_count, Label* done,
+ InvokeFlag flag);
+
+ void EnterExitFramePrologue(bool save_rax, StackFrame::Type frame_type);
+
+ // Allocates arg_stack_space * kSystemPointerSize memory (not GCed) on the
+ // stack accessible via StackSpaceOperand.
+ void EnterExitFrameEpilogue(int arg_stack_space, bool save_doubles);
+
+ void LeaveExitFrameEpilogue();
+
+ // Compute memory operands for safepoint stack slots.
+ static int SafepointRegisterStackIndex(int reg_code) {
+ return kNumSafepointRegisters - kSafepointPushRegisterIndices[reg_code] - 1;
+ }
+
+ // Needs access to SafepointRegisterStackIndex for compiled frame
+ // traversal.
+ friend class CommonFrame;
+
+ DISALLOW_IMPLICIT_CONSTRUCTORS(MacroAssembler);
+};
+
+// -----------------------------------------------------------------------------
+// Static helper functions.
+
+// Generate an Operand for loading a field from an object.
+inline Operand FieldOperand(Register object, int offset) {
+ return Operand(object, offset - kHeapObjectTag);
+}
+
+// Generate an Operand for loading an indexed field from an object.
+inline Operand FieldOperand(Register object, Register index, ScaleFactor scale,
+ int offset) {
+ return Operand(object, index, scale, offset - kHeapObjectTag);
+}
+
+// Provides access to exit frame stack space (not GCed).
+inline Operand StackSpaceOperand(int index) {
+#ifdef V8_TARGET_OS_WIN
+ const int kShaddowSpace = 4;
+ return Operand(rsp, (index + kShaddowSpace) * kSystemPointerSize);
+#else
+ return Operand(rsp, index * kSystemPointerSize);
+#endif
+}
+
+inline Operand StackOperandForReturnAddress(int32_t disp) {
+ return Operand(rsp, disp);
+}
+
+#define ACCESS_MASM(masm) masm->
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_X64_MACRO_ASSEMBLER_X64_H_
diff --git a/src/codegen/x64/register-x64.h b/src/codegen/x64/register-x64.h
new file mode 100644
index 0000000..7d5aaab
--- /dev/null
+++ b/src/codegen/x64/register-x64.h
@@ -0,0 +1,225 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_X64_REGISTER_X64_H_
+#define V8_CODEGEN_X64_REGISTER_X64_H_
+
+#include "src/codegen/register.h"
+#include "src/codegen/reglist.h"
+
+namespace v8 {
+namespace internal {
+
+#define GENERAL_REGISTERS(V) \
+ V(rax) \
+ V(rcx) \
+ V(rdx) \
+ V(rbx) \
+ V(rsp) \
+ V(rbp) \
+ V(rsi) \
+ V(rdi) \
+ V(r8) \
+ V(r9) \
+ V(r10) \
+ V(r11) \
+ V(r12) \
+ V(r13) \
+ V(r14) \
+ V(r15)
+
+#define ALLOCATABLE_GENERAL_REGISTERS(V) \
+ V(rax) \
+ V(rbx) \
+ V(rdx) \
+ V(rcx) \
+ V(rsi) \
+ V(rdi) \
+ V(r8) \
+ V(r9) \
+ V(r11) \
+ V(r12) \
+ V(r14) \
+ V(r15)
+
+enum RegisterCode {
+#define REGISTER_CODE(R) kRegCode_##R,
+ GENERAL_REGISTERS(REGISTER_CODE)
+#undef REGISTER_CODE
+ kRegAfterLast
+};
+
+class Register : public RegisterBase<Register, kRegAfterLast> {
+ public:
+ bool is_byte_register() const { return code() <= 3; }
+ // Return the high bit of the register code as a 0 or 1. Used often
+ // when constructing the REX prefix byte.
+ int high_bit() const { return code() >> 3; }
+ // Return the 3 low bits of the register code. Used when encoding registers
+ // in modR/M, SIB, and opcode bytes.
+ int low_bits() const { return code() & 0x7; }
+
+ private:
+ friend class RegisterBase<Register, kRegAfterLast>;
+ explicit constexpr Register(int code) : RegisterBase(code) {}
+};
+
+ASSERT_TRIVIALLY_COPYABLE(Register);
+static_assert(sizeof(Register) == sizeof(int),
+ "Register can efficiently be passed by value");
+
+#define DECLARE_REGISTER(R) \
+ constexpr Register R = Register::from_code(kRegCode_##R);
+GENERAL_REGISTERS(DECLARE_REGISTER)
+#undef DECLARE_REGISTER
+constexpr Register no_reg = Register::no_reg();
+
+constexpr int kNumRegs = 16;
+
+constexpr RegList kJSCallerSaved =
+ Register::ListOf(rax, rcx, rdx,
+ rbx, // used as a caller-saved register in JavaScript code
+ rdi); // callee function
+
+constexpr int kNumJSCallerSaved = 5;
+
+// Number of registers for which space is reserved in safepoints.
+constexpr int kNumSafepointRegisters = 16;
+
+#ifdef V8_TARGET_OS_WIN
+// Windows calling convention
+constexpr Register arg_reg_1 = rcx;
+constexpr Register arg_reg_2 = rdx;
+constexpr Register arg_reg_3 = r8;
+constexpr Register arg_reg_4 = r9;
+#else
+// AMD64 calling convention
+constexpr Register arg_reg_1 = rdi;
+constexpr Register arg_reg_2 = rsi;
+constexpr Register arg_reg_3 = rdx;
+constexpr Register arg_reg_4 = rcx;
+#endif // V8_TARGET_OS_WIN
+
+#define DOUBLE_REGISTERS(V) \
+ V(xmm0) \
+ V(xmm1) \
+ V(xmm2) \
+ V(xmm3) \
+ V(xmm4) \
+ V(xmm5) \
+ V(xmm6) \
+ V(xmm7) \
+ V(xmm8) \
+ V(xmm9) \
+ V(xmm10) \
+ V(xmm11) \
+ V(xmm12) \
+ V(xmm13) \
+ V(xmm14) \
+ V(xmm15)
+
+#define FLOAT_REGISTERS DOUBLE_REGISTERS
+#define SIMD128_REGISTERS DOUBLE_REGISTERS
+
+#define ALLOCATABLE_DOUBLE_REGISTERS(V) \
+ V(xmm0) \
+ V(xmm1) \
+ V(xmm2) \
+ V(xmm3) \
+ V(xmm4) \
+ V(xmm5) \
+ V(xmm6) \
+ V(xmm7) \
+ V(xmm8) \
+ V(xmm9) \
+ V(xmm10) \
+ V(xmm11) \
+ V(xmm12) \
+ V(xmm13) \
+ V(xmm14)
+
+constexpr bool kPadArguments = false;
+constexpr bool kSimpleFPAliasing = true;
+constexpr bool kSimdMaskRegisters = false;
+
+enum DoubleRegisterCode {
+#define REGISTER_CODE(R) kDoubleCode_##R,
+ DOUBLE_REGISTERS(REGISTER_CODE)
+#undef REGISTER_CODE
+ kDoubleAfterLast
+};
+
+class XMMRegister : public RegisterBase<XMMRegister, kDoubleAfterLast> {
+ public:
+ // Return the high bit of the register code as a 0 or 1. Used often
+ // when constructing the REX prefix byte.
+ int high_bit() const { return code() >> 3; }
+ // Return the 3 low bits of the register code. Used when encoding registers
+ // in modR/M, SIB, and opcode bytes.
+ int low_bits() const { return code() & 0x7; }
+
+ private:
+ friend class RegisterBase<XMMRegister, kDoubleAfterLast>;
+ explicit constexpr XMMRegister(int code) : RegisterBase(code) {}
+};
+
+ASSERT_TRIVIALLY_COPYABLE(XMMRegister);
+static_assert(sizeof(XMMRegister) == sizeof(int),
+ "XMMRegister can efficiently be passed by value");
+
+using FloatRegister = XMMRegister;
+
+using DoubleRegister = XMMRegister;
+
+using Simd128Register = XMMRegister;
+
+#define DECLARE_REGISTER(R) \
+ constexpr DoubleRegister R = DoubleRegister::from_code(kDoubleCode_##R);
+DOUBLE_REGISTERS(DECLARE_REGISTER)
+#undef DECLARE_REGISTER
+constexpr DoubleRegister no_dreg = DoubleRegister::no_reg();
+
+// Define {RegisterName} methods for the register types.
+DEFINE_REGISTER_NAMES(Register, GENERAL_REGISTERS)
+DEFINE_REGISTER_NAMES(XMMRegister, DOUBLE_REGISTERS)
+
+// Give alias names to registers for calling conventions.
+constexpr Register kReturnRegister0 = rax;
+constexpr Register kReturnRegister1 = rdx;
+constexpr Register kReturnRegister2 = r8;
+constexpr Register kJSFunctionRegister = rdi;
+constexpr Register kContextRegister = rsi;
+constexpr Register kAllocateSizeRegister = rdx;
+constexpr Register kSpeculationPoisonRegister = r12;
+constexpr Register kInterpreterAccumulatorRegister = rax;
+constexpr Register kInterpreterBytecodeOffsetRegister = r9;
+constexpr Register kInterpreterBytecodeArrayRegister = r14;
+constexpr Register kInterpreterDispatchTableRegister = r15;
+
+constexpr Register kJavaScriptCallArgCountRegister = rax;
+constexpr Register kJavaScriptCallCodeStartRegister = rcx;
+constexpr Register kJavaScriptCallTargetRegister = kJSFunctionRegister;
+constexpr Register kJavaScriptCallNewTargetRegister = rdx;
+constexpr Register kJavaScriptCallExtraArg1Register = rbx;
+
+constexpr Register kRuntimeCallFunctionRegister = rbx;
+constexpr Register kRuntimeCallArgCountRegister = rax;
+constexpr Register kRuntimeCallArgvRegister = r15;
+constexpr Register kWasmInstanceRegister = rsi;
+
+// Default scratch register used by MacroAssembler (and other code that needs
+// a spare register). The register isn't callee save, and not used by the
+// function calling convention.
+constexpr Register kScratchRegister = r10;
+constexpr XMMRegister kScratchDoubleReg = xmm15;
+constexpr Register kRootRegister = r13; // callee save
+
+constexpr Register kOffHeapTrampolineRegister = kScratchRegister;
+
+constexpr DoubleRegister kFPReturnRegister0 = xmm0;
+
+} // namespace internal
+} // namespace v8
+
+#endif // V8_CODEGEN_X64_REGISTER_X64_H_
diff --git a/src/codegen/x64/sse-instr.h b/src/codegen/x64/sse-instr.h
new file mode 100644
index 0000000..52107ed
--- /dev/null
+++ b/src/codegen/x64/sse-instr.h
@@ -0,0 +1,176 @@
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_X64_SSE_INSTR_H_
+#define V8_CODEGEN_X64_SSE_INSTR_H_
+
+// SSE instructions whose AVX version has two operands.
+#define SSE_UNOP_INSTRUCTION_LIST(V) \
+ V(sqrtps, 0F, 51) \
+ V(rsqrtps, 0F, 52) \
+ V(rcpps, 0F, 53) \
+ V(cvtdq2ps, 0F, 5B)
+
+// SSE instructions whose AVX version has three operands.
+#define SSE_BINOP_INSTRUCTION_LIST(V) \
+ V(andps, 0F, 54) \
+ V(andnps, 0F, 55) \
+ V(orps, 0F, 56) \
+ V(xorps, 0F, 57) \
+ V(addps, 0F, 58) \
+ V(mulps, 0F, 59) \
+ V(subps, 0F, 5C) \
+ V(minps, 0F, 5D) \
+ V(divps, 0F, 5E) \
+ V(maxps, 0F, 5F)
+
+// Instructions dealing with scalar single-precision values.
+#define SSE_INSTRUCTION_LIST_SS(V) \
+ V(sqrtss, F3, 0F, 51) \
+ V(addss, F3, 0F, 58) \
+ V(mulss, F3, 0F, 59) \
+ V(subss, F3, 0F, 5C) \
+ V(minss, F3, 0F, 5D) \
+ V(divss, F3, 0F, 5E) \
+ V(maxss, F3, 0F, 5F)
+
+#define SSE2_INSTRUCTION_LIST(V) \
+ V(andpd, 66, 0F, 54) \
+ V(andnpd, 66, 0F, 55) \
+ V(orpd, 66, 0F, 56) \
+ V(xorpd, 66, 0F, 57) \
+ V(addpd, 66, 0F, 58) \
+ V(mulpd, 66, 0F, 59) \
+ V(subpd, 66, 0F, 5C) \
+ V(minpd, 66, 0F, 5D) \
+ V(maxpd, 66, 0F, 5F) \
+ V(divpd, 66, 0F, 5E) \
+ V(punpcklbw, 66, 0F, 60) \
+ V(punpcklwd, 66, 0F, 61) \
+ V(punpckldq, 66, 0F, 62) \
+ V(packsswb, 66, 0F, 63) \
+ V(packuswb, 66, 0F, 67) \
+ V(punpckhbw, 66, 0F, 68) \
+ V(punpckhwd, 66, 0F, 69) \
+ V(punpckhdq, 66, 0F, 6A) \
+ V(packssdw, 66, 0F, 6B) \
+ V(punpcklqdq, 66, 0F, 6C) \
+ V(punpckhqdq, 66, 0F, 6D) \
+ V(pmaddwd, 66, 0F, F5) \
+ V(paddb, 66, 0F, FC) \
+ V(paddw, 66, 0F, FD) \
+ V(paddd, 66, 0F, FE) \
+ V(paddq, 66, 0F, D4) \
+ V(paddsb, 66, 0F, EC) \
+ V(paddsw, 66, 0F, ED) \
+ V(paddusb, 66, 0F, DC) \
+ V(paddusw, 66, 0F, DD) \
+ V(pcmpeqb, 66, 0F, 74) \
+ V(pcmpeqw, 66, 0F, 75) \
+ V(pcmpeqd, 66, 0F, 76) \
+ V(pcmpgtb, 66, 0F, 64) \
+ V(pcmpgtw, 66, 0F, 65) \
+ V(pcmpgtd, 66, 0F, 66) \
+ V(pmaxsw, 66, 0F, EE) \
+ V(pmaxub, 66, 0F, DE) \
+ V(pminsw, 66, 0F, EA) \
+ V(pminub, 66, 0F, DA) \
+ V(pmullw, 66, 0F, D5) \
+ V(pmuludq, 66, 0F, F4) \
+ V(psllw, 66, 0F, F1) \
+ V(pslld, 66, 0F, F2) \
+ V(psllq, 66, 0F, F3) \
+ V(pavgb, 66, 0F, E0) \
+ V(psraw, 66, 0F, E1) \
+ V(psrad, 66, 0F, E2) \
+ V(pavgw, 66, 0F, E3) \
+ V(psrlw, 66, 0F, D1) \
+ V(psrld, 66, 0F, D2) \
+ V(psrlq, 66, 0F, D3) \
+ V(psubb, 66, 0F, F8) \
+ V(psubw, 66, 0F, F9) \
+ V(psubd, 66, 0F, FA) \
+ V(psubq, 66, 0F, FB) \
+ V(psubsb, 66, 0F, E8) \
+ V(psubsw, 66, 0F, E9) \
+ V(psubusb, 66, 0F, D8) \
+ V(psubusw, 66, 0F, D9) \
+ V(pand, 66, 0F, DB) \
+ V(por, 66, 0F, EB) \
+ V(pxor, 66, 0F, EF)
+
+// SSE2 instructions whose AVX version has two operands.
+#define SSE2_UNOP_INSTRUCTION_LIST(V) \
+ V(sqrtpd, 66, 0F, 51) \
+ V(cvtps2dq, 66, 0F, 5B)
+
+// SSE2 shift instructions with an immediate operand. The last element is the
+// extension to the opcode.
+#define SSE2_INSTRUCTION_LIST_SHIFT_IMM(V) \
+ V(psrlw, 66, 0F, 71, 2) \
+ V(psrld, 66, 0F, 72, 2) \
+ V(psrlq, 66, 0F, 73, 2) \
+ V(psraw, 66, 0F, 71, 4) \
+ V(psrad, 66, 0F, 72, 4) \
+ V(psllw, 66, 0F, 71, 6) \
+ V(pslld, 66, 0F, 72, 6) \
+ V(psllq, 66, 0F, 73, 6)
+
+// Instructions dealing with scalar double-precision values.
+#define SSE2_INSTRUCTION_LIST_SD(V) \
+ V(sqrtsd, F2, 0F, 51) \
+ V(addsd, F2, 0F, 58) \
+ V(mulsd, F2, 0F, 59) \
+ V(cvtsd2ss, F2, 0F, 5A) \
+ V(subsd, F2, 0F, 5C) \
+ V(minsd, F2, 0F, 5D) \
+ V(divsd, F2, 0F, 5E) \
+ V(maxsd, F2, 0F, 5F)
+
+#define SSSE3_INSTRUCTION_LIST(V) \
+ V(phaddd, 66, 0F, 38, 02) \
+ V(phaddw, 66, 0F, 38, 01) \
+ V(pshufb, 66, 0F, 38, 00) \
+ V(psignb, 66, 0F, 38, 08) \
+ V(psignw, 66, 0F, 38, 09) \
+ V(psignd, 66, 0F, 38, 0A)
+
+// SSSE3 instructions whose AVX version has two operands.
+#define SSSE3_UNOP_INSTRUCTION_LIST(V) \
+ V(pabsb, 66, 0F, 38, 1C) \
+ V(pabsw, 66, 0F, 38, 1D) \
+ V(pabsd, 66, 0F, 38, 1E)
+
+#define SSE4_INSTRUCTION_LIST(V) \
+ V(pcmpeqq, 66, 0F, 38, 29) \
+ V(packusdw, 66, 0F, 38, 2B) \
+ V(pminsb, 66, 0F, 38, 38) \
+ V(pminsd, 66, 0F, 38, 39) \
+ V(pminuw, 66, 0F, 38, 3A) \
+ V(pminud, 66, 0F, 38, 3B) \
+ V(pmaxsb, 66, 0F, 38, 3C) \
+ V(pmaxsd, 66, 0F, 38, 3D) \
+ V(pmaxuw, 66, 0F, 38, 3E) \
+ V(pmaxud, 66, 0F, 38, 3F) \
+ V(pmulld, 66, 0F, 38, 40)
+
+// SSE instructions whose AVX version has two operands.
+#define SSE4_UNOP_INSTRUCTION_LIST(V) \
+ V(ptest, 66, 0F, 38, 17) \
+ V(pmovsxbw, 66, 0F, 38, 20) \
+ V(pmovsxwd, 66, 0F, 38, 23) \
+ V(pmovsxdq, 66, 0F, 38, 25) \
+ V(pmovzxbw, 66, 0F, 38, 30) \
+ V(pmovzxwd, 66, 0F, 38, 33) \
+ V(pmovzxdq, 66, 0F, 38, 35)
+
+#define SSE4_EXTRACT_INSTRUCTION_LIST(V) \
+ V(extractps, 66, 0F, 3A, 17) \
+ V(pextrb, 66, 0F, 3A, 14) \
+ V(pextrw, 66, 0F, 3A, 15) \
+ V(pextrd, 66, 0F, 3A, 16)
+
+#define SSE4_2_INSTRUCTION_LIST(V) V(pcmpgtq, 66, 0F, 38, 37)
+
+#endif // V8_CODEGEN_X64_SSE_INSTR_H_