Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2015 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | #include "intrinsics_x86.h" |
| 18 | |
Andreas Gampe | 21030dd | 2015-05-07 14:46:15 -0700 | [diff] [blame] | 19 | #include <limits> |
| 20 | |
Mark Mendell | fb8d279 | 2015-03-31 22:16:59 -0400 | [diff] [blame] | 21 | #include "arch/x86/instruction_set_features_x86.h" |
Mathieu Chartier | e401d14 | 2015-04-22 13:56:20 -0700 | [diff] [blame] | 22 | #include "art_method.h" |
Mark Mendell | d589767 | 2015-08-12 21:16:41 -0400 | [diff] [blame] | 23 | #include "base/bit_utils.h" |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 24 | #include "code_generator_x86.h" |
| 25 | #include "entrypoints/quick/quick_entrypoints.h" |
Andreas Gampe | 09659c2 | 2017-09-18 18:23:32 -0700 | [diff] [blame] | 26 | #include "heap_poisoning.h" |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 27 | #include "intrinsics.h" |
Andreas Gampe | 85b62f2 | 2015-09-09 13:15:38 -0700 | [diff] [blame] | 28 | #include "intrinsics_utils.h" |
Andreas Gampe | c6ea7d0 | 2017-02-01 16:46:28 -0800 | [diff] [blame] | 29 | #include "lock_word.h" |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 30 | #include "mirror/array-inl.h" |
Andreas Gampe | c15a2f4 | 2017-04-21 12:09:39 -0700 | [diff] [blame] | 31 | #include "mirror/object_array-inl.h" |
Andreas Gampe | c6ea7d0 | 2017-02-01 16:46:28 -0800 | [diff] [blame] | 32 | #include "mirror/reference.h" |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 33 | #include "mirror/string.h" |
Andreas Gampe | c6ea7d0 | 2017-02-01 16:46:28 -0800 | [diff] [blame] | 34 | #include "scoped_thread_state_change-inl.h" |
Andreas Gampe | b486a98 | 2017-06-01 13:45:54 -0700 | [diff] [blame] | 35 | #include "thread-current-inl.h" |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 36 | #include "utils/x86/assembler_x86.h" |
| 37 | #include "utils/x86/constants_x86.h" |
| 38 | |
| 39 | namespace art { |
| 40 | |
| 41 | namespace x86 { |
| 42 | |
| 43 | static constexpr int kDoubleNaNHigh = 0x7FF80000; |
| 44 | static constexpr int kDoubleNaNLow = 0x00000000; |
Mark P Mendell | 2f10a5f | 2016-01-25 14:47:50 +0000 | [diff] [blame] | 45 | static constexpr int64_t kDoubleNaN = INT64_C(0x7FF8000000000000); |
| 46 | static constexpr int32_t kFloatNaN = INT32_C(0x7FC00000); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 47 | |
Mark Mendell | fb8d279 | 2015-03-31 22:16:59 -0400 | [diff] [blame] | 48 | IntrinsicLocationsBuilderX86::IntrinsicLocationsBuilderX86(CodeGeneratorX86* codegen) |
Mark P Mendell | 2f10a5f | 2016-01-25 14:47:50 +0000 | [diff] [blame] | 49 | : arena_(codegen->GetGraph()->GetArena()), |
| 50 | codegen_(codegen) { |
Mark Mendell | fb8d279 | 2015-03-31 22:16:59 -0400 | [diff] [blame] | 51 | } |
| 52 | |
| 53 | |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 54 | X86Assembler* IntrinsicCodeGeneratorX86::GetAssembler() { |
Roland Levillain | b488b78 | 2015-10-22 11:38:49 +0100 | [diff] [blame] | 55 | return down_cast<X86Assembler*>(codegen_->GetAssembler()); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 56 | } |
| 57 | |
| 58 | ArenaAllocator* IntrinsicCodeGeneratorX86::GetAllocator() { |
| 59 | return codegen_->GetGraph()->GetArena(); |
| 60 | } |
| 61 | |
| 62 | bool IntrinsicLocationsBuilderX86::TryDispatch(HInvoke* invoke) { |
| 63 | Dispatch(invoke); |
| 64 | LocationSummary* res = invoke->GetLocations(); |
Roland Levillain | 0d5a281 | 2015-11-13 10:07:31 +0000 | [diff] [blame] | 65 | if (res == nullptr) { |
| 66 | return false; |
| 67 | } |
Roland Levillain | 0d5a281 | 2015-11-13 10:07:31 +0000 | [diff] [blame] | 68 | return res->Intrinsified(); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 69 | } |
| 70 | |
Roland Levillain | ec525fc | 2015-04-28 15:50:20 +0100 | [diff] [blame] | 71 | static void MoveArguments(HInvoke* invoke, CodeGeneratorX86* codegen) { |
Roland Levillain | 2d27c8e | 2015-04-28 15:48:45 +0100 | [diff] [blame] | 72 | InvokeDexCallingConventionVisitorX86 calling_convention_visitor; |
Roland Levillain | ec525fc | 2015-04-28 15:50:20 +0100 | [diff] [blame] | 73 | IntrinsicVisitor::MoveArguments(invoke, codegen, &calling_convention_visitor); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 74 | } |
| 75 | |
Andreas Gampe | 85b62f2 | 2015-09-09 13:15:38 -0700 | [diff] [blame] | 76 | using IntrinsicSlowPathX86 = IntrinsicSlowPath<InvokeDexCallingConventionVisitorX86>; |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 77 | |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 78 | // NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. |
| 79 | #define __ down_cast<X86Assembler*>(codegen->GetAssembler())-> // NOLINT |
| 80 | |
| 81 | // Slow path implementing the SystemArrayCopy intrinsic copy loop with read barriers. |
| 82 | class ReadBarrierSystemArrayCopySlowPathX86 : public SlowPathCode { |
| 83 | public: |
| 84 | explicit ReadBarrierSystemArrayCopySlowPathX86(HInstruction* instruction) |
| 85 | : SlowPathCode(instruction) { |
| 86 | DCHECK(kEmitCompilerReadBarrier); |
| 87 | DCHECK(kUseBakerReadBarrier); |
| 88 | } |
| 89 | |
| 90 | void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { |
| 91 | CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen); |
| 92 | LocationSummary* locations = instruction_->GetLocations(); |
| 93 | DCHECK(locations->CanCall()); |
| 94 | DCHECK(instruction_->IsInvokeStaticOrDirect()) |
| 95 | << "Unexpected instruction in read barrier arraycopy slow path: " |
| 96 | << instruction_->DebugName(); |
| 97 | DCHECK(instruction_->GetLocations()->Intrinsified()); |
| 98 | DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kSystemArrayCopy); |
| 99 | |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 100 | int32_t element_size = DataType::Size(DataType::Type::kReference); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 101 | uint32_t offset = mirror::Array::DataOffset(element_size).Uint32Value(); |
| 102 | |
| 103 | Register src = locations->InAt(0).AsRegister<Register>(); |
| 104 | Location src_pos = locations->InAt(1); |
| 105 | Register dest = locations->InAt(2).AsRegister<Register>(); |
| 106 | Location dest_pos = locations->InAt(3); |
| 107 | Location length = locations->InAt(4); |
| 108 | Location temp1_loc = locations->GetTemp(0); |
| 109 | Register temp1 = temp1_loc.AsRegister<Register>(); |
| 110 | Register temp2 = locations->GetTemp(1).AsRegister<Register>(); |
| 111 | Register temp3 = locations->GetTemp(2).AsRegister<Register>(); |
| 112 | |
| 113 | __ Bind(GetEntryLabel()); |
| 114 | // In this code path, registers `temp1`, `temp2`, and `temp3` |
| 115 | // (resp.) are not used for the base source address, the base |
| 116 | // destination address, and the end source address (resp.), as in |
| 117 | // other SystemArrayCopy intrinsic code paths. Instead they are |
| 118 | // (resp.) used for: |
| 119 | // - the loop index (`i`); |
| 120 | // - the source index (`src_index`) and the loaded (source) |
| 121 | // reference (`value`); and |
| 122 | // - the destination index (`dest_index`). |
| 123 | |
| 124 | // i = 0 |
| 125 | __ xorl(temp1, temp1); |
| 126 | NearLabel loop; |
| 127 | __ Bind(&loop); |
| 128 | // value = src_array[i + src_pos] |
| 129 | if (src_pos.IsConstant()) { |
| 130 | int32_t constant = src_pos.GetConstant()->AsIntConstant()->GetValue(); |
| 131 | int32_t adjusted_offset = offset + constant * element_size; |
| 132 | __ movl(temp2, Address(src, temp1, ScaleFactor::TIMES_4, adjusted_offset)); |
| 133 | } else { |
| 134 | __ leal(temp2, Address(src_pos.AsRegister<Register>(), temp1, ScaleFactor::TIMES_1, 0)); |
| 135 | __ movl(temp2, Address(src, temp2, ScaleFactor::TIMES_4, offset)); |
| 136 | } |
| 137 | __ MaybeUnpoisonHeapReference(temp2); |
| 138 | // TODO: Inline the mark bit check before calling the runtime? |
| 139 | // value = ReadBarrier::Mark(value) |
| 140 | // No need to save live registers; it's taken care of by the |
| 141 | // entrypoint. Also, there is no need to update the stack mask, |
| 142 | // as this runtime call will not trigger a garbage collection. |
| 143 | // (See ReadBarrierMarkSlowPathX86::EmitNativeCode for more |
| 144 | // explanations.) |
| 145 | DCHECK_NE(temp2, ESP); |
| 146 | DCHECK(0 <= temp2 && temp2 < kNumberOfCpuRegisters) << temp2; |
Roland Levillain | 97c4646 | 2017-05-11 14:04:03 +0100 | [diff] [blame] | 147 | int32_t entry_point_offset = Thread::ReadBarrierMarkEntryPointsOffset<kX86PointerSize>(temp2); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 148 | // This runtime call does not require a stack map. |
| 149 | x86_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); |
| 150 | __ MaybePoisonHeapReference(temp2); |
| 151 | // dest_array[i + dest_pos] = value |
| 152 | if (dest_pos.IsConstant()) { |
| 153 | int32_t constant = dest_pos.GetConstant()->AsIntConstant()->GetValue(); |
| 154 | int32_t adjusted_offset = offset + constant * element_size; |
| 155 | __ movl(Address(dest, temp1, ScaleFactor::TIMES_4, adjusted_offset), temp2); |
| 156 | } else { |
| 157 | __ leal(temp3, Address(dest_pos.AsRegister<Register>(), temp1, ScaleFactor::TIMES_1, 0)); |
| 158 | __ movl(Address(dest, temp3, ScaleFactor::TIMES_4, offset), temp2); |
| 159 | } |
| 160 | // ++i |
| 161 | __ addl(temp1, Immediate(1)); |
| 162 | // if (i != length) goto loop |
| 163 | x86_codegen->GenerateIntCompare(temp1_loc, length); |
| 164 | __ j(kNotEqual, &loop); |
| 165 | __ jmp(GetExitLabel()); |
| 166 | } |
| 167 | |
| 168 | const char* GetDescription() const OVERRIDE { return "ReadBarrierSystemArrayCopySlowPathX86"; } |
| 169 | |
| 170 | private: |
| 171 | DISALLOW_COPY_AND_ASSIGN(ReadBarrierSystemArrayCopySlowPathX86); |
| 172 | }; |
| 173 | |
| 174 | #undef __ |
| 175 | |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 176 | #define __ assembler-> |
| 177 | |
| 178 | static void CreateFPToIntLocations(ArenaAllocator* arena, HInvoke* invoke, bool is64bit) { |
| 179 | LocationSummary* locations = new (arena) LocationSummary(invoke, |
| 180 | LocationSummary::kNoCall, |
| 181 | kIntrinsified); |
| 182 | locations->SetInAt(0, Location::RequiresFpuRegister()); |
| 183 | locations->SetOut(Location::RequiresRegister()); |
| 184 | if (is64bit) { |
| 185 | locations->AddTemp(Location::RequiresFpuRegister()); |
| 186 | } |
| 187 | } |
| 188 | |
| 189 | static void CreateIntToFPLocations(ArenaAllocator* arena, HInvoke* invoke, bool is64bit) { |
| 190 | LocationSummary* locations = new (arena) LocationSummary(invoke, |
| 191 | LocationSummary::kNoCall, |
| 192 | kIntrinsified); |
| 193 | locations->SetInAt(0, Location::RequiresRegister()); |
| 194 | locations->SetOut(Location::RequiresFpuRegister()); |
| 195 | if (is64bit) { |
| 196 | locations->AddTemp(Location::RequiresFpuRegister()); |
| 197 | locations->AddTemp(Location::RequiresFpuRegister()); |
| 198 | } |
| 199 | } |
| 200 | |
| 201 | static void MoveFPToInt(LocationSummary* locations, bool is64bit, X86Assembler* assembler) { |
| 202 | Location input = locations->InAt(0); |
| 203 | Location output = locations->Out(); |
| 204 | if (is64bit) { |
| 205 | // Need to use the temporary. |
| 206 | XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); |
| 207 | __ movsd(temp, input.AsFpuRegister<XmmRegister>()); |
| 208 | __ movd(output.AsRegisterPairLow<Register>(), temp); |
| 209 | __ psrlq(temp, Immediate(32)); |
| 210 | __ movd(output.AsRegisterPairHigh<Register>(), temp); |
| 211 | } else { |
| 212 | __ movd(output.AsRegister<Register>(), input.AsFpuRegister<XmmRegister>()); |
| 213 | } |
| 214 | } |
| 215 | |
| 216 | static void MoveIntToFP(LocationSummary* locations, bool is64bit, X86Assembler* assembler) { |
| 217 | Location input = locations->InAt(0); |
| 218 | Location output = locations->Out(); |
| 219 | if (is64bit) { |
| 220 | // Need to use the temporary. |
| 221 | XmmRegister temp1 = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); |
| 222 | XmmRegister temp2 = locations->GetTemp(1).AsFpuRegister<XmmRegister>(); |
| 223 | __ movd(temp1, input.AsRegisterPairLow<Register>()); |
| 224 | __ movd(temp2, input.AsRegisterPairHigh<Register>()); |
| 225 | __ punpckldq(temp1, temp2); |
| 226 | __ movsd(output.AsFpuRegister<XmmRegister>(), temp1); |
| 227 | } else { |
| 228 | __ movd(output.AsFpuRegister<XmmRegister>(), input.AsRegister<Register>()); |
| 229 | } |
| 230 | } |
| 231 | |
| 232 | void IntrinsicLocationsBuilderX86::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) { |
Roland Levillain | bf84a3d | 2015-12-04 14:33:02 +0000 | [diff] [blame] | 233 | CreateFPToIntLocations(arena_, invoke, /* is64bit */ true); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 234 | } |
| 235 | void IntrinsicLocationsBuilderX86::VisitDoubleLongBitsToDouble(HInvoke* invoke) { |
Roland Levillain | bf84a3d | 2015-12-04 14:33:02 +0000 | [diff] [blame] | 236 | CreateIntToFPLocations(arena_, invoke, /* is64bit */ true); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 237 | } |
| 238 | |
| 239 | void IntrinsicCodeGeneratorX86::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) { |
Roland Levillain | bf84a3d | 2015-12-04 14:33:02 +0000 | [diff] [blame] | 240 | MoveFPToInt(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 241 | } |
| 242 | void IntrinsicCodeGeneratorX86::VisitDoubleLongBitsToDouble(HInvoke* invoke) { |
Roland Levillain | bf84a3d | 2015-12-04 14:33:02 +0000 | [diff] [blame] | 243 | MoveIntToFP(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 244 | } |
| 245 | |
| 246 | void IntrinsicLocationsBuilderX86::VisitFloatFloatToRawIntBits(HInvoke* invoke) { |
Roland Levillain | bf84a3d | 2015-12-04 14:33:02 +0000 | [diff] [blame] | 247 | CreateFPToIntLocations(arena_, invoke, /* is64bit */ false); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 248 | } |
| 249 | void IntrinsicLocationsBuilderX86::VisitFloatIntBitsToFloat(HInvoke* invoke) { |
Roland Levillain | bf84a3d | 2015-12-04 14:33:02 +0000 | [diff] [blame] | 250 | CreateIntToFPLocations(arena_, invoke, /* is64bit */ false); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 251 | } |
| 252 | |
| 253 | void IntrinsicCodeGeneratorX86::VisitFloatFloatToRawIntBits(HInvoke* invoke) { |
Roland Levillain | bf84a3d | 2015-12-04 14:33:02 +0000 | [diff] [blame] | 254 | MoveFPToInt(invoke->GetLocations(), /* is64bit */ false, GetAssembler()); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 255 | } |
| 256 | void IntrinsicCodeGeneratorX86::VisitFloatIntBitsToFloat(HInvoke* invoke) { |
Roland Levillain | bf84a3d | 2015-12-04 14:33:02 +0000 | [diff] [blame] | 257 | MoveIntToFP(invoke->GetLocations(), /* is64bit */ false, GetAssembler()); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 258 | } |
| 259 | |
| 260 | static void CreateIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { |
| 261 | LocationSummary* locations = new (arena) LocationSummary(invoke, |
| 262 | LocationSummary::kNoCall, |
| 263 | kIntrinsified); |
| 264 | locations->SetInAt(0, Location::RequiresRegister()); |
| 265 | locations->SetOut(Location::SameAsFirstInput()); |
| 266 | } |
| 267 | |
| 268 | static void CreateLongToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { |
| 269 | LocationSummary* locations = new (arena) LocationSummary(invoke, |
| 270 | LocationSummary::kNoCall, |
| 271 | kIntrinsified); |
| 272 | locations->SetInAt(0, Location::RequiresRegister()); |
| 273 | locations->SetOut(Location::RequiresRegister()); |
| 274 | } |
| 275 | |
| 276 | static void CreateLongToLongLocations(ArenaAllocator* arena, HInvoke* invoke) { |
| 277 | LocationSummary* locations = new (arena) LocationSummary(invoke, |
| 278 | LocationSummary::kNoCall, |
| 279 | kIntrinsified); |
| 280 | locations->SetInAt(0, Location::RequiresRegister()); |
| 281 | locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); |
| 282 | } |
| 283 | |
| 284 | static void GenReverseBytes(LocationSummary* locations, |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 285 | DataType::Type size, |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 286 | X86Assembler* assembler) { |
| 287 | Register out = locations->Out().AsRegister<Register>(); |
| 288 | |
| 289 | switch (size) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 290 | case DataType::Type::kInt16: |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 291 | // TODO: Can be done with an xchg of 8b registers. This is straight from Quick. |
| 292 | __ bswapl(out); |
| 293 | __ sarl(out, Immediate(16)); |
| 294 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 295 | case DataType::Type::kInt32: |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 296 | __ bswapl(out); |
| 297 | break; |
| 298 | default: |
| 299 | LOG(FATAL) << "Unexpected size for reverse-bytes: " << size; |
| 300 | UNREACHABLE(); |
| 301 | } |
| 302 | } |
| 303 | |
| 304 | void IntrinsicLocationsBuilderX86::VisitIntegerReverseBytes(HInvoke* invoke) { |
| 305 | CreateIntToIntLocations(arena_, invoke); |
| 306 | } |
| 307 | |
| 308 | void IntrinsicCodeGeneratorX86::VisitIntegerReverseBytes(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 309 | GenReverseBytes(invoke->GetLocations(), DataType::Type::kInt32, GetAssembler()); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 310 | } |
| 311 | |
Mark Mendell | 58d25fd | 2015-04-03 14:52:31 -0400 | [diff] [blame] | 312 | void IntrinsicLocationsBuilderX86::VisitLongReverseBytes(HInvoke* invoke) { |
| 313 | CreateLongToLongLocations(arena_, invoke); |
| 314 | } |
| 315 | |
| 316 | void IntrinsicCodeGeneratorX86::VisitLongReverseBytes(HInvoke* invoke) { |
| 317 | LocationSummary* locations = invoke->GetLocations(); |
| 318 | Location input = locations->InAt(0); |
| 319 | Register input_lo = input.AsRegisterPairLow<Register>(); |
| 320 | Register input_hi = input.AsRegisterPairHigh<Register>(); |
| 321 | Location output = locations->Out(); |
| 322 | Register output_lo = output.AsRegisterPairLow<Register>(); |
| 323 | Register output_hi = output.AsRegisterPairHigh<Register>(); |
| 324 | |
| 325 | X86Assembler* assembler = GetAssembler(); |
| 326 | // Assign the inputs to the outputs, mixing low/high. |
| 327 | __ movl(output_lo, input_hi); |
| 328 | __ movl(output_hi, input_lo); |
| 329 | __ bswapl(output_lo); |
| 330 | __ bswapl(output_hi); |
| 331 | } |
| 332 | |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 333 | void IntrinsicLocationsBuilderX86::VisitShortReverseBytes(HInvoke* invoke) { |
| 334 | CreateIntToIntLocations(arena_, invoke); |
| 335 | } |
| 336 | |
| 337 | void IntrinsicCodeGeneratorX86::VisitShortReverseBytes(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 338 | GenReverseBytes(invoke->GetLocations(), DataType::Type::kInt16, GetAssembler()); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 339 | } |
| 340 | |
| 341 | |
| 342 | // TODO: Consider Quick's way of doing Double abs through integer operations, as the immediate we |
| 343 | // need is 64b. |
| 344 | |
| 345 | static void CreateFloatToFloat(ArenaAllocator* arena, HInvoke* invoke) { |
| 346 | // TODO: Enable memory operations when the assembler supports them. |
| 347 | LocationSummary* locations = new (arena) LocationSummary(invoke, |
| 348 | LocationSummary::kNoCall, |
| 349 | kIntrinsified); |
| 350 | locations->SetInAt(0, Location::RequiresFpuRegister()); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 351 | locations->SetOut(Location::SameAsFirstInput()); |
Mark P Mendell | 2f10a5f | 2016-01-25 14:47:50 +0000 | [diff] [blame] | 352 | HInvokeStaticOrDirect* static_or_direct = invoke->AsInvokeStaticOrDirect(); |
| 353 | DCHECK(static_or_direct != nullptr); |
Nicolas Geoffray | 9779307 | 2016-02-16 15:33:54 +0000 | [diff] [blame] | 354 | if (static_or_direct->HasSpecialInput() && |
| 355 | invoke->InputAt(static_or_direct->GetSpecialInputIndex())->IsX86ComputeBaseMethodAddress()) { |
Mark P Mendell | 2f10a5f | 2016-01-25 14:47:50 +0000 | [diff] [blame] | 356 | // We need addressibility for the constant area. |
| 357 | locations->SetInAt(1, Location::RequiresRegister()); |
| 358 | // We need a temporary to hold the constant. |
| 359 | locations->AddTemp(Location::RequiresFpuRegister()); |
| 360 | } |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 361 | } |
| 362 | |
Nicolas Geoffray | 133719e | 2017-01-22 15:44:39 +0000 | [diff] [blame] | 363 | static void MathAbsFP(HInvoke* invoke, |
Mark P Mendell | 2f10a5f | 2016-01-25 14:47:50 +0000 | [diff] [blame] | 364 | bool is64bit, |
| 365 | X86Assembler* assembler, |
| 366 | CodeGeneratorX86* codegen) { |
Nicolas Geoffray | 133719e | 2017-01-22 15:44:39 +0000 | [diff] [blame] | 367 | LocationSummary* locations = invoke->GetLocations(); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 368 | Location output = locations->Out(); |
| 369 | |
Mark P Mendell | 2f10a5f | 2016-01-25 14:47:50 +0000 | [diff] [blame] | 370 | DCHECK(output.IsFpuRegister()); |
Nicolas Geoffray | 9779307 | 2016-02-16 15:33:54 +0000 | [diff] [blame] | 371 | if (locations->GetInputCount() == 2 && locations->InAt(1).IsValid()) { |
Nicolas Geoffray | 133719e | 2017-01-22 15:44:39 +0000 | [diff] [blame] | 372 | HX86ComputeBaseMethodAddress* method_address = |
| 373 | invoke->InputAt(1)->AsX86ComputeBaseMethodAddress(); |
Mark P Mendell | 2f10a5f | 2016-01-25 14:47:50 +0000 | [diff] [blame] | 374 | DCHECK(locations->InAt(1).IsRegister()); |
| 375 | // We also have a constant area pointer. |
| 376 | Register constant_area = locations->InAt(1).AsRegister<Register>(); |
| 377 | XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); |
| 378 | if (is64bit) { |
Nicolas Geoffray | 133719e | 2017-01-22 15:44:39 +0000 | [diff] [blame] | 379 | __ movsd(temp, codegen->LiteralInt64Address( |
| 380 | INT64_C(0x7FFFFFFFFFFFFFFF), method_address, constant_area)); |
Mark P Mendell | 2f10a5f | 2016-01-25 14:47:50 +0000 | [diff] [blame] | 381 | __ andpd(output.AsFpuRegister<XmmRegister>(), temp); |
| 382 | } else { |
Nicolas Geoffray | 133719e | 2017-01-22 15:44:39 +0000 | [diff] [blame] | 383 | __ movss(temp, codegen->LiteralInt32Address( |
| 384 | INT32_C(0x7FFFFFFF), method_address, constant_area)); |
Mark P Mendell | 2f10a5f | 2016-01-25 14:47:50 +0000 | [diff] [blame] | 385 | __ andps(output.AsFpuRegister<XmmRegister>(), temp); |
| 386 | } |
| 387 | } else { |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 388 | // Create the right constant on an aligned stack. |
| 389 | if (is64bit) { |
| 390 | __ subl(ESP, Immediate(8)); |
| 391 | __ pushl(Immediate(0x7FFFFFFF)); |
| 392 | __ pushl(Immediate(0xFFFFFFFF)); |
| 393 | __ andpd(output.AsFpuRegister<XmmRegister>(), Address(ESP, 0)); |
| 394 | } else { |
| 395 | __ subl(ESP, Immediate(12)); |
| 396 | __ pushl(Immediate(0x7FFFFFFF)); |
| 397 | __ andps(output.AsFpuRegister<XmmRegister>(), Address(ESP, 0)); |
| 398 | } |
| 399 | __ addl(ESP, Immediate(16)); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 400 | } |
| 401 | } |
| 402 | |
| 403 | void IntrinsicLocationsBuilderX86::VisitMathAbsDouble(HInvoke* invoke) { |
| 404 | CreateFloatToFloat(arena_, invoke); |
| 405 | } |
| 406 | |
| 407 | void IntrinsicCodeGeneratorX86::VisitMathAbsDouble(HInvoke* invoke) { |
Nicolas Geoffray | 133719e | 2017-01-22 15:44:39 +0000 | [diff] [blame] | 408 | MathAbsFP(invoke, /* is64bit */ true, GetAssembler(), codegen_); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 409 | } |
| 410 | |
| 411 | void IntrinsicLocationsBuilderX86::VisitMathAbsFloat(HInvoke* invoke) { |
| 412 | CreateFloatToFloat(arena_, invoke); |
| 413 | } |
| 414 | |
| 415 | void IntrinsicCodeGeneratorX86::VisitMathAbsFloat(HInvoke* invoke) { |
Nicolas Geoffray | 133719e | 2017-01-22 15:44:39 +0000 | [diff] [blame] | 416 | MathAbsFP(invoke, /* is64bit */ false, GetAssembler(), codegen_); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 417 | } |
| 418 | |
| 419 | static void CreateAbsIntLocation(ArenaAllocator* arena, HInvoke* invoke) { |
| 420 | LocationSummary* locations = new (arena) LocationSummary(invoke, |
| 421 | LocationSummary::kNoCall, |
| 422 | kIntrinsified); |
| 423 | locations->SetInAt(0, Location::RegisterLocation(EAX)); |
| 424 | locations->SetOut(Location::SameAsFirstInput()); |
| 425 | locations->AddTemp(Location::RegisterLocation(EDX)); |
| 426 | } |
| 427 | |
| 428 | static void GenAbsInteger(LocationSummary* locations, X86Assembler* assembler) { |
| 429 | Location output = locations->Out(); |
| 430 | Register out = output.AsRegister<Register>(); |
| 431 | DCHECK_EQ(out, EAX); |
| 432 | Register temp = locations->GetTemp(0).AsRegister<Register>(); |
| 433 | DCHECK_EQ(temp, EDX); |
| 434 | |
| 435 | // Sign extend EAX into EDX. |
| 436 | __ cdq(); |
| 437 | |
| 438 | // XOR EAX with sign. |
| 439 | __ xorl(EAX, EDX); |
| 440 | |
| 441 | // Subtract out sign to correct. |
| 442 | __ subl(EAX, EDX); |
| 443 | |
| 444 | // The result is in EAX. |
| 445 | } |
| 446 | |
| 447 | static void CreateAbsLongLocation(ArenaAllocator* arena, HInvoke* invoke) { |
| 448 | LocationSummary* locations = new (arena) LocationSummary(invoke, |
| 449 | LocationSummary::kNoCall, |
| 450 | kIntrinsified); |
| 451 | locations->SetInAt(0, Location::RequiresRegister()); |
| 452 | locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); |
| 453 | locations->AddTemp(Location::RequiresRegister()); |
| 454 | } |
| 455 | |
| 456 | static void GenAbsLong(LocationSummary* locations, X86Assembler* assembler) { |
| 457 | Location input = locations->InAt(0); |
| 458 | Register input_lo = input.AsRegisterPairLow<Register>(); |
| 459 | Register input_hi = input.AsRegisterPairHigh<Register>(); |
| 460 | Location output = locations->Out(); |
| 461 | Register output_lo = output.AsRegisterPairLow<Register>(); |
| 462 | Register output_hi = output.AsRegisterPairHigh<Register>(); |
| 463 | Register temp = locations->GetTemp(0).AsRegister<Register>(); |
| 464 | |
| 465 | // Compute the sign into the temporary. |
| 466 | __ movl(temp, input_hi); |
| 467 | __ sarl(temp, Immediate(31)); |
| 468 | |
| 469 | // Store the sign into the output. |
| 470 | __ movl(output_lo, temp); |
| 471 | __ movl(output_hi, temp); |
| 472 | |
| 473 | // XOR the input to the output. |
| 474 | __ xorl(output_lo, input_lo); |
| 475 | __ xorl(output_hi, input_hi); |
| 476 | |
| 477 | // Subtract the sign. |
| 478 | __ subl(output_lo, temp); |
| 479 | __ sbbl(output_hi, temp); |
| 480 | } |
| 481 | |
| 482 | void IntrinsicLocationsBuilderX86::VisitMathAbsInt(HInvoke* invoke) { |
| 483 | CreateAbsIntLocation(arena_, invoke); |
| 484 | } |
| 485 | |
| 486 | void IntrinsicCodeGeneratorX86::VisitMathAbsInt(HInvoke* invoke) { |
| 487 | GenAbsInteger(invoke->GetLocations(), GetAssembler()); |
| 488 | } |
| 489 | |
| 490 | void IntrinsicLocationsBuilderX86::VisitMathAbsLong(HInvoke* invoke) { |
| 491 | CreateAbsLongLocation(arena_, invoke); |
| 492 | } |
| 493 | |
| 494 | void IntrinsicCodeGeneratorX86::VisitMathAbsLong(HInvoke* invoke) { |
| 495 | GenAbsLong(invoke->GetLocations(), GetAssembler()); |
| 496 | } |
| 497 | |
Nicolas Geoffray | 133719e | 2017-01-22 15:44:39 +0000 | [diff] [blame] | 498 | static void GenMinMaxFP(HInvoke* invoke, |
Mark P Mendell | 2f10a5f | 2016-01-25 14:47:50 +0000 | [diff] [blame] | 499 | bool is_min, |
| 500 | bool is_double, |
| 501 | X86Assembler* assembler, |
| 502 | CodeGeneratorX86* codegen) { |
Nicolas Geoffray | 133719e | 2017-01-22 15:44:39 +0000 | [diff] [blame] | 503 | LocationSummary* locations = invoke->GetLocations(); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 504 | Location op1_loc = locations->InAt(0); |
| 505 | Location op2_loc = locations->InAt(1); |
| 506 | Location out_loc = locations->Out(); |
| 507 | XmmRegister out = out_loc.AsFpuRegister<XmmRegister>(); |
| 508 | |
| 509 | // Shortcut for same input locations. |
| 510 | if (op1_loc.Equals(op2_loc)) { |
| 511 | DCHECK(out_loc.Equals(op1_loc)); |
| 512 | return; |
| 513 | } |
| 514 | |
| 515 | // (out := op1) |
| 516 | // out <=? op2 |
| 517 | // if Nan jmp Nan_label |
| 518 | // if out is min jmp done |
| 519 | // if op2 is min jmp op2_label |
| 520 | // handle -0/+0 |
| 521 | // jmp done |
| 522 | // Nan_label: |
| 523 | // out := NaN |
| 524 | // op2_label: |
| 525 | // out := op2 |
| 526 | // done: |
| 527 | // |
| 528 | // This removes one jmp, but needs to copy one input (op1) to out. |
| 529 | // |
| 530 | // TODO: This is straight from Quick (except literal pool). Make NaN an out-of-line slowpath? |
| 531 | |
| 532 | XmmRegister op2 = op2_loc.AsFpuRegister<XmmRegister>(); |
| 533 | |
Mark Mendell | 0c9497d | 2015-08-21 09:30:05 -0400 | [diff] [blame] | 534 | NearLabel nan, done, op2_label; |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 535 | if (is_double) { |
| 536 | __ ucomisd(out, op2); |
| 537 | } else { |
| 538 | __ ucomiss(out, op2); |
| 539 | } |
| 540 | |
| 541 | __ j(Condition::kParityEven, &nan); |
| 542 | |
| 543 | __ j(is_min ? Condition::kAbove : Condition::kBelow, &op2_label); |
| 544 | __ j(is_min ? Condition::kBelow : Condition::kAbove, &done); |
| 545 | |
| 546 | // Handle 0.0/-0.0. |
| 547 | if (is_min) { |
| 548 | if (is_double) { |
| 549 | __ orpd(out, op2); |
| 550 | } else { |
| 551 | __ orps(out, op2); |
| 552 | } |
| 553 | } else { |
| 554 | if (is_double) { |
| 555 | __ andpd(out, op2); |
| 556 | } else { |
| 557 | __ andps(out, op2); |
| 558 | } |
| 559 | } |
| 560 | __ jmp(&done); |
| 561 | |
| 562 | // NaN handling. |
| 563 | __ Bind(&nan); |
Mark P Mendell | 2f10a5f | 2016-01-25 14:47:50 +0000 | [diff] [blame] | 564 | // Do we have a constant area pointer? |
Nicolas Geoffray | 9779307 | 2016-02-16 15:33:54 +0000 | [diff] [blame] | 565 | if (locations->GetInputCount() == 3 && locations->InAt(2).IsValid()) { |
Nicolas Geoffray | 133719e | 2017-01-22 15:44:39 +0000 | [diff] [blame] | 566 | HX86ComputeBaseMethodAddress* method_address = |
| 567 | invoke->InputAt(2)->AsX86ComputeBaseMethodAddress(); |
Mark P Mendell | 2f10a5f | 2016-01-25 14:47:50 +0000 | [diff] [blame] | 568 | DCHECK(locations->InAt(2).IsRegister()); |
| 569 | Register constant_area = locations->InAt(2).AsRegister<Register>(); |
| 570 | if (is_double) { |
Nicolas Geoffray | 133719e | 2017-01-22 15:44:39 +0000 | [diff] [blame] | 571 | __ movsd(out, codegen->LiteralInt64Address(kDoubleNaN, method_address, constant_area)); |
Mark P Mendell | 2f10a5f | 2016-01-25 14:47:50 +0000 | [diff] [blame] | 572 | } else { |
Nicolas Geoffray | 133719e | 2017-01-22 15:44:39 +0000 | [diff] [blame] | 573 | __ movss(out, codegen->LiteralInt32Address(kFloatNaN, method_address, constant_area)); |
Mark P Mendell | 2f10a5f | 2016-01-25 14:47:50 +0000 | [diff] [blame] | 574 | } |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 575 | } else { |
Mark P Mendell | 2f10a5f | 2016-01-25 14:47:50 +0000 | [diff] [blame] | 576 | if (is_double) { |
| 577 | __ pushl(Immediate(kDoubleNaNHigh)); |
| 578 | __ pushl(Immediate(kDoubleNaNLow)); |
| 579 | __ movsd(out, Address(ESP, 0)); |
| 580 | __ addl(ESP, Immediate(8)); |
| 581 | } else { |
| 582 | __ pushl(Immediate(kFloatNaN)); |
| 583 | __ movss(out, Address(ESP, 0)); |
| 584 | __ addl(ESP, Immediate(4)); |
| 585 | } |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 586 | } |
| 587 | __ jmp(&done); |
| 588 | |
| 589 | // out := op2; |
| 590 | __ Bind(&op2_label); |
| 591 | if (is_double) { |
| 592 | __ movsd(out, op2); |
| 593 | } else { |
| 594 | __ movss(out, op2); |
| 595 | } |
| 596 | |
| 597 | // Done. |
| 598 | __ Bind(&done); |
| 599 | } |
| 600 | |
| 601 | static void CreateFPFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) { |
| 602 | LocationSummary* locations = new (arena) LocationSummary(invoke, |
| 603 | LocationSummary::kNoCall, |
| 604 | kIntrinsified); |
| 605 | locations->SetInAt(0, Location::RequiresFpuRegister()); |
| 606 | locations->SetInAt(1, Location::RequiresFpuRegister()); |
| 607 | // The following is sub-optimal, but all we can do for now. It would be fine to also accept |
| 608 | // the second input to be the output (we can simply swap inputs). |
| 609 | locations->SetOut(Location::SameAsFirstInput()); |
Mark P Mendell | 2f10a5f | 2016-01-25 14:47:50 +0000 | [diff] [blame] | 610 | HInvokeStaticOrDirect* static_or_direct = invoke->AsInvokeStaticOrDirect(); |
| 611 | DCHECK(static_or_direct != nullptr); |
Nicolas Geoffray | 9779307 | 2016-02-16 15:33:54 +0000 | [diff] [blame] | 612 | if (static_or_direct->HasSpecialInput() && |
| 613 | invoke->InputAt(static_or_direct->GetSpecialInputIndex())->IsX86ComputeBaseMethodAddress()) { |
Mark P Mendell | 2f10a5f | 2016-01-25 14:47:50 +0000 | [diff] [blame] | 614 | locations->SetInAt(2, Location::RequiresRegister()); |
| 615 | } |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 616 | } |
| 617 | |
| 618 | void IntrinsicLocationsBuilderX86::VisitMathMinDoubleDouble(HInvoke* invoke) { |
| 619 | CreateFPFPToFPLocations(arena_, invoke); |
| 620 | } |
| 621 | |
| 622 | void IntrinsicCodeGeneratorX86::VisitMathMinDoubleDouble(HInvoke* invoke) { |
Nicolas Geoffray | 133719e | 2017-01-22 15:44:39 +0000 | [diff] [blame] | 623 | GenMinMaxFP(invoke, |
Mark P Mendell | 2f10a5f | 2016-01-25 14:47:50 +0000 | [diff] [blame] | 624 | /* is_min */ true, |
| 625 | /* is_double */ true, |
| 626 | GetAssembler(), |
| 627 | codegen_); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 628 | } |
| 629 | |
| 630 | void IntrinsicLocationsBuilderX86::VisitMathMinFloatFloat(HInvoke* invoke) { |
| 631 | CreateFPFPToFPLocations(arena_, invoke); |
| 632 | } |
| 633 | |
| 634 | void IntrinsicCodeGeneratorX86::VisitMathMinFloatFloat(HInvoke* invoke) { |
Nicolas Geoffray | 133719e | 2017-01-22 15:44:39 +0000 | [diff] [blame] | 635 | GenMinMaxFP(invoke, |
Mark P Mendell | 2f10a5f | 2016-01-25 14:47:50 +0000 | [diff] [blame] | 636 | /* is_min */ true, |
| 637 | /* is_double */ false, |
| 638 | GetAssembler(), |
| 639 | codegen_); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 640 | } |
| 641 | |
| 642 | void IntrinsicLocationsBuilderX86::VisitMathMaxDoubleDouble(HInvoke* invoke) { |
| 643 | CreateFPFPToFPLocations(arena_, invoke); |
| 644 | } |
| 645 | |
| 646 | void IntrinsicCodeGeneratorX86::VisitMathMaxDoubleDouble(HInvoke* invoke) { |
Nicolas Geoffray | 133719e | 2017-01-22 15:44:39 +0000 | [diff] [blame] | 647 | GenMinMaxFP(invoke, |
Mark P Mendell | 2f10a5f | 2016-01-25 14:47:50 +0000 | [diff] [blame] | 648 | /* is_min */ false, |
| 649 | /* is_double */ true, |
| 650 | GetAssembler(), |
| 651 | codegen_); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 652 | } |
| 653 | |
| 654 | void IntrinsicLocationsBuilderX86::VisitMathMaxFloatFloat(HInvoke* invoke) { |
| 655 | CreateFPFPToFPLocations(arena_, invoke); |
| 656 | } |
| 657 | |
| 658 | void IntrinsicCodeGeneratorX86::VisitMathMaxFloatFloat(HInvoke* invoke) { |
Nicolas Geoffray | 133719e | 2017-01-22 15:44:39 +0000 | [diff] [blame] | 659 | GenMinMaxFP(invoke, |
Mark P Mendell | 2f10a5f | 2016-01-25 14:47:50 +0000 | [diff] [blame] | 660 | /* is_min */ false, |
| 661 | /* is_double */ false, |
| 662 | GetAssembler(), |
| 663 | codegen_); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 664 | } |
| 665 | |
| 666 | static void GenMinMax(LocationSummary* locations, bool is_min, bool is_long, |
| 667 | X86Assembler* assembler) { |
| 668 | Location op1_loc = locations->InAt(0); |
| 669 | Location op2_loc = locations->InAt(1); |
| 670 | |
| 671 | // Shortcut for same input locations. |
| 672 | if (op1_loc.Equals(op2_loc)) { |
| 673 | // Can return immediately, as op1_loc == out_loc. |
| 674 | // Note: if we ever support separate registers, e.g., output into memory, we need to check for |
| 675 | // a copy here. |
| 676 | DCHECK(locations->Out().Equals(op1_loc)); |
| 677 | return; |
| 678 | } |
| 679 | |
| 680 | if (is_long) { |
| 681 | // Need to perform a subtract to get the sign right. |
| 682 | // op1 is already in the same location as the output. |
| 683 | Location output = locations->Out(); |
| 684 | Register output_lo = output.AsRegisterPairLow<Register>(); |
| 685 | Register output_hi = output.AsRegisterPairHigh<Register>(); |
| 686 | |
| 687 | Register op2_lo = op2_loc.AsRegisterPairLow<Register>(); |
| 688 | Register op2_hi = op2_loc.AsRegisterPairHigh<Register>(); |
| 689 | |
| 690 | // Spare register to compute the subtraction to set condition code. |
| 691 | Register temp = locations->GetTemp(0).AsRegister<Register>(); |
| 692 | |
| 693 | // Subtract off op2_low. |
| 694 | __ movl(temp, output_lo); |
| 695 | __ subl(temp, op2_lo); |
| 696 | |
| 697 | // Now use the same tempo and the borrow to finish the subtraction of op2_hi. |
| 698 | __ movl(temp, output_hi); |
| 699 | __ sbbl(temp, op2_hi); |
| 700 | |
| 701 | // Now the condition code is correct. |
| 702 | Condition cond = is_min ? Condition::kGreaterEqual : Condition::kLess; |
| 703 | __ cmovl(cond, output_lo, op2_lo); |
| 704 | __ cmovl(cond, output_hi, op2_hi); |
| 705 | } else { |
| 706 | Register out = locations->Out().AsRegister<Register>(); |
| 707 | Register op2 = op2_loc.AsRegister<Register>(); |
| 708 | |
| 709 | // (out := op1) |
| 710 | // out <=? op2 |
| 711 | // if out is min jmp done |
| 712 | // out := op2 |
| 713 | // done: |
| 714 | |
| 715 | __ cmpl(out, op2); |
| 716 | Condition cond = is_min ? Condition::kGreater : Condition::kLess; |
| 717 | __ cmovl(cond, out, op2); |
| 718 | } |
| 719 | } |
| 720 | |
| 721 | static void CreateIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { |
| 722 | LocationSummary* locations = new (arena) LocationSummary(invoke, |
| 723 | LocationSummary::kNoCall, |
| 724 | kIntrinsified); |
| 725 | locations->SetInAt(0, Location::RequiresRegister()); |
| 726 | locations->SetInAt(1, Location::RequiresRegister()); |
| 727 | locations->SetOut(Location::SameAsFirstInput()); |
| 728 | } |
| 729 | |
| 730 | static void CreateLongLongToLongLocations(ArenaAllocator* arena, HInvoke* invoke) { |
| 731 | LocationSummary* locations = new (arena) LocationSummary(invoke, |
| 732 | LocationSummary::kNoCall, |
| 733 | kIntrinsified); |
| 734 | locations->SetInAt(0, Location::RequiresRegister()); |
| 735 | locations->SetInAt(1, Location::RequiresRegister()); |
| 736 | locations->SetOut(Location::SameAsFirstInput()); |
| 737 | // Register to use to perform a long subtract to set cc. |
| 738 | locations->AddTemp(Location::RequiresRegister()); |
| 739 | } |
| 740 | |
| 741 | void IntrinsicLocationsBuilderX86::VisitMathMinIntInt(HInvoke* invoke) { |
| 742 | CreateIntIntToIntLocations(arena_, invoke); |
| 743 | } |
| 744 | |
| 745 | void IntrinsicCodeGeneratorX86::VisitMathMinIntInt(HInvoke* invoke) { |
Roland Levillain | bf84a3d | 2015-12-04 14:33:02 +0000 | [diff] [blame] | 746 | GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ false, GetAssembler()); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 747 | } |
| 748 | |
| 749 | void IntrinsicLocationsBuilderX86::VisitMathMinLongLong(HInvoke* invoke) { |
| 750 | CreateLongLongToLongLocations(arena_, invoke); |
| 751 | } |
| 752 | |
| 753 | void IntrinsicCodeGeneratorX86::VisitMathMinLongLong(HInvoke* invoke) { |
Roland Levillain | bf84a3d | 2015-12-04 14:33:02 +0000 | [diff] [blame] | 754 | GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ true, GetAssembler()); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 755 | } |
| 756 | |
| 757 | void IntrinsicLocationsBuilderX86::VisitMathMaxIntInt(HInvoke* invoke) { |
| 758 | CreateIntIntToIntLocations(arena_, invoke); |
| 759 | } |
| 760 | |
| 761 | void IntrinsicCodeGeneratorX86::VisitMathMaxIntInt(HInvoke* invoke) { |
Roland Levillain | bf84a3d | 2015-12-04 14:33:02 +0000 | [diff] [blame] | 762 | GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ false, GetAssembler()); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 763 | } |
| 764 | |
| 765 | void IntrinsicLocationsBuilderX86::VisitMathMaxLongLong(HInvoke* invoke) { |
| 766 | CreateLongLongToLongLocations(arena_, invoke); |
| 767 | } |
| 768 | |
| 769 | void IntrinsicCodeGeneratorX86::VisitMathMaxLongLong(HInvoke* invoke) { |
Roland Levillain | bf84a3d | 2015-12-04 14:33:02 +0000 | [diff] [blame] | 770 | GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ true, GetAssembler()); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 771 | } |
| 772 | |
| 773 | static void CreateFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) { |
| 774 | LocationSummary* locations = new (arena) LocationSummary(invoke, |
| 775 | LocationSummary::kNoCall, |
| 776 | kIntrinsified); |
| 777 | locations->SetInAt(0, Location::RequiresFpuRegister()); |
| 778 | locations->SetOut(Location::RequiresFpuRegister()); |
| 779 | } |
| 780 | |
| 781 | void IntrinsicLocationsBuilderX86::VisitMathSqrt(HInvoke* invoke) { |
| 782 | CreateFPToFPLocations(arena_, invoke); |
| 783 | } |
| 784 | |
| 785 | void IntrinsicCodeGeneratorX86::VisitMathSqrt(HInvoke* invoke) { |
| 786 | LocationSummary* locations = invoke->GetLocations(); |
| 787 | XmmRegister in = locations->InAt(0).AsFpuRegister<XmmRegister>(); |
| 788 | XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>(); |
| 789 | |
| 790 | GetAssembler()->sqrtsd(out, in); |
| 791 | } |
| 792 | |
Mark Mendell | fb8d279 | 2015-03-31 22:16:59 -0400 | [diff] [blame] | 793 | static void InvokeOutOfLineIntrinsic(CodeGeneratorX86* codegen, HInvoke* invoke) { |
Roland Levillain | ec525fc | 2015-04-28 15:50:20 +0100 | [diff] [blame] | 794 | MoveArguments(invoke, codegen); |
Mark Mendell | fb8d279 | 2015-03-31 22:16:59 -0400 | [diff] [blame] | 795 | |
| 796 | DCHECK(invoke->IsInvokeStaticOrDirect()); |
Nicolas Geoffray | 94015b9 | 2015-06-04 18:21:04 +0100 | [diff] [blame] | 797 | codegen->GenerateStaticOrDirectCall(invoke->AsInvokeStaticOrDirect(), |
| 798 | Location::RegisterLocation(EAX)); |
Mark Mendell | fb8d279 | 2015-03-31 22:16:59 -0400 | [diff] [blame] | 799 | |
| 800 | // Copy the result back to the expected output. |
| 801 | Location out = invoke->GetLocations()->Out(); |
| 802 | if (out.IsValid()) { |
| 803 | DCHECK(out.IsRegister()); |
Andreas Gampe | 85b62f2 | 2015-09-09 13:15:38 -0700 | [diff] [blame] | 804 | codegen->MoveFromReturnRegister(out, invoke->GetType()); |
Mark Mendell | fb8d279 | 2015-03-31 22:16:59 -0400 | [diff] [blame] | 805 | } |
| 806 | } |
| 807 | |
| 808 | static void CreateSSE41FPToFPLocations(ArenaAllocator* arena, |
| 809 | HInvoke* invoke, |
| 810 | CodeGeneratorX86* codegen) { |
| 811 | // Do we have instruction support? |
| 812 | if (codegen->GetInstructionSetFeatures().HasSSE4_1()) { |
| 813 | CreateFPToFPLocations(arena, invoke); |
| 814 | return; |
| 815 | } |
| 816 | |
| 817 | // We have to fall back to a call to the intrinsic. |
| 818 | LocationSummary* locations = new (arena) LocationSummary(invoke, |
Serban Constantinescu | 54ff482 | 2016-07-07 18:03:19 +0100 | [diff] [blame] | 819 | LocationSummary::kCallOnMainOnly); |
Mark Mendell | fb8d279 | 2015-03-31 22:16:59 -0400 | [diff] [blame] | 820 | InvokeRuntimeCallingConvention calling_convention; |
| 821 | locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetFpuRegisterAt(0))); |
| 822 | locations->SetOut(Location::FpuRegisterLocation(XMM0)); |
| 823 | // Needs to be EAX for the invoke. |
| 824 | locations->AddTemp(Location::RegisterLocation(EAX)); |
| 825 | } |
| 826 | |
| 827 | static void GenSSE41FPToFPIntrinsic(CodeGeneratorX86* codegen, |
| 828 | HInvoke* invoke, |
| 829 | X86Assembler* assembler, |
| 830 | int round_mode) { |
| 831 | LocationSummary* locations = invoke->GetLocations(); |
| 832 | if (locations->WillCall()) { |
| 833 | InvokeOutOfLineIntrinsic(codegen, invoke); |
| 834 | } else { |
| 835 | XmmRegister in = locations->InAt(0).AsFpuRegister<XmmRegister>(); |
| 836 | XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>(); |
| 837 | __ roundsd(out, in, Immediate(round_mode)); |
| 838 | } |
| 839 | } |
| 840 | |
| 841 | void IntrinsicLocationsBuilderX86::VisitMathCeil(HInvoke* invoke) { |
| 842 | CreateSSE41FPToFPLocations(arena_, invoke, codegen_); |
| 843 | } |
| 844 | |
| 845 | void IntrinsicCodeGeneratorX86::VisitMathCeil(HInvoke* invoke) { |
| 846 | GenSSE41FPToFPIntrinsic(codegen_, invoke, GetAssembler(), 2); |
| 847 | } |
| 848 | |
| 849 | void IntrinsicLocationsBuilderX86::VisitMathFloor(HInvoke* invoke) { |
| 850 | CreateSSE41FPToFPLocations(arena_, invoke, codegen_); |
| 851 | } |
| 852 | |
| 853 | void IntrinsicCodeGeneratorX86::VisitMathFloor(HInvoke* invoke) { |
| 854 | GenSSE41FPToFPIntrinsic(codegen_, invoke, GetAssembler(), 1); |
| 855 | } |
| 856 | |
| 857 | void IntrinsicLocationsBuilderX86::VisitMathRint(HInvoke* invoke) { |
| 858 | CreateSSE41FPToFPLocations(arena_, invoke, codegen_); |
| 859 | } |
| 860 | |
| 861 | void IntrinsicCodeGeneratorX86::VisitMathRint(HInvoke* invoke) { |
| 862 | GenSSE41FPToFPIntrinsic(codegen_, invoke, GetAssembler(), 0); |
| 863 | } |
| 864 | |
Mark Mendell | fb8d279 | 2015-03-31 22:16:59 -0400 | [diff] [blame] | 865 | void IntrinsicLocationsBuilderX86::VisitMathRoundFloat(HInvoke* invoke) { |
| 866 | // Do we have instruction support? |
| 867 | if (codegen_->GetInstructionSetFeatures().HasSSE4_1()) { |
Aart Bik | 2c9f495 | 2016-08-01 16:52:27 -0700 | [diff] [blame] | 868 | HInvokeStaticOrDirect* static_or_direct = invoke->AsInvokeStaticOrDirect(); |
| 869 | DCHECK(static_or_direct != nullptr); |
Mark Mendell | fb8d279 | 2015-03-31 22:16:59 -0400 | [diff] [blame] | 870 | LocationSummary* locations = new (arena_) LocationSummary(invoke, |
| 871 | LocationSummary::kNoCall, |
| 872 | kIntrinsified); |
| 873 | locations->SetInAt(0, Location::RequiresFpuRegister()); |
Aart Bik | 2c9f495 | 2016-08-01 16:52:27 -0700 | [diff] [blame] | 874 | if (static_or_direct->HasSpecialInput() && |
| 875 | invoke->InputAt( |
| 876 | static_or_direct->GetSpecialInputIndex())->IsX86ComputeBaseMethodAddress()) { |
| 877 | locations->SetInAt(1, Location::RequiresRegister()); |
| 878 | } |
Nicolas Geoffray | d9b9240 | 2015-04-21 10:02:22 +0100 | [diff] [blame] | 879 | locations->SetOut(Location::RequiresRegister()); |
Mark Mendell | fb8d279 | 2015-03-31 22:16:59 -0400 | [diff] [blame] | 880 | locations->AddTemp(Location::RequiresFpuRegister()); |
| 881 | locations->AddTemp(Location::RequiresFpuRegister()); |
| 882 | return; |
| 883 | } |
| 884 | |
| 885 | // We have to fall back to a call to the intrinsic. |
| 886 | LocationSummary* locations = new (arena_) LocationSummary(invoke, |
Aart Bik | 2c9f495 | 2016-08-01 16:52:27 -0700 | [diff] [blame] | 887 | LocationSummary::kCallOnMainOnly); |
Mark Mendell | fb8d279 | 2015-03-31 22:16:59 -0400 | [diff] [blame] | 888 | InvokeRuntimeCallingConvention calling_convention; |
| 889 | locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetFpuRegisterAt(0))); |
| 890 | locations->SetOut(Location::RegisterLocation(EAX)); |
| 891 | // Needs to be EAX for the invoke. |
| 892 | locations->AddTemp(Location::RegisterLocation(EAX)); |
| 893 | } |
| 894 | |
| 895 | void IntrinsicCodeGeneratorX86::VisitMathRoundFloat(HInvoke* invoke) { |
| 896 | LocationSummary* locations = invoke->GetLocations(); |
Aart Bik | 2c9f495 | 2016-08-01 16:52:27 -0700 | [diff] [blame] | 897 | if (locations->WillCall()) { // TODO: can we reach this? |
Mark Mendell | fb8d279 | 2015-03-31 22:16:59 -0400 | [diff] [blame] | 898 | InvokeOutOfLineIntrinsic(codegen_, invoke); |
| 899 | return; |
| 900 | } |
| 901 | |
Mark Mendell | fb8d279 | 2015-03-31 22:16:59 -0400 | [diff] [blame] | 902 | XmmRegister in = locations->InAt(0).AsFpuRegister<XmmRegister>(); |
Aart Bik | 2c9f495 | 2016-08-01 16:52:27 -0700 | [diff] [blame] | 903 | XmmRegister t1 = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); |
| 904 | XmmRegister t2 = locations->GetTemp(1).AsFpuRegister<XmmRegister>(); |
Mark Mendell | fb8d279 | 2015-03-31 22:16:59 -0400 | [diff] [blame] | 905 | Register out = locations->Out().AsRegister<Register>(); |
Aart Bik | 2c9f495 | 2016-08-01 16:52:27 -0700 | [diff] [blame] | 906 | NearLabel skip_incr, done; |
Mark Mendell | fb8d279 | 2015-03-31 22:16:59 -0400 | [diff] [blame] | 907 | X86Assembler* assembler = GetAssembler(); |
| 908 | |
Aart Bik | 2c9f495 | 2016-08-01 16:52:27 -0700 | [diff] [blame] | 909 | // Since no direct x86 rounding instruction matches the required semantics, |
| 910 | // this intrinsic is implemented as follows: |
| 911 | // result = floor(in); |
| 912 | // if (in - result >= 0.5f) |
| 913 | // result = result + 1.0f; |
| 914 | __ movss(t2, in); |
| 915 | __ roundss(t1, in, Immediate(1)); |
| 916 | __ subss(t2, t1); |
Aart Bik | 0cf8d9c | 2016-08-10 14:05:54 -0700 | [diff] [blame] | 917 | if (locations->GetInputCount() == 2 && locations->InAt(1).IsValid()) { |
| 918 | // Direct constant area available. |
Nicolas Geoffray | 133719e | 2017-01-22 15:44:39 +0000 | [diff] [blame] | 919 | HX86ComputeBaseMethodAddress* method_address = |
| 920 | invoke->InputAt(1)->AsX86ComputeBaseMethodAddress(); |
Aart Bik | 0cf8d9c | 2016-08-10 14:05:54 -0700 | [diff] [blame] | 921 | Register constant_area = locations->InAt(1).AsRegister<Register>(); |
Nicolas Geoffray | 133719e | 2017-01-22 15:44:39 +0000 | [diff] [blame] | 922 | __ comiss(t2, codegen_->LiteralInt32Address(bit_cast<int32_t, float>(0.5f), |
| 923 | method_address, |
| 924 | constant_area)); |
Aart Bik | 0cf8d9c | 2016-08-10 14:05:54 -0700 | [diff] [blame] | 925 | __ j(kBelow, &skip_incr); |
Nicolas Geoffray | 133719e | 2017-01-22 15:44:39 +0000 | [diff] [blame] | 926 | __ addss(t1, codegen_->LiteralInt32Address(bit_cast<int32_t, float>(1.0f), |
| 927 | method_address, |
| 928 | constant_area)); |
Aart Bik | 0cf8d9c | 2016-08-10 14:05:54 -0700 | [diff] [blame] | 929 | __ Bind(&skip_incr); |
| 930 | } else { |
| 931 | // No constant area: go through stack. |
| 932 | __ pushl(Immediate(bit_cast<int32_t, float>(0.5f))); |
| 933 | __ pushl(Immediate(bit_cast<int32_t, float>(1.0f))); |
| 934 | __ comiss(t2, Address(ESP, 4)); |
| 935 | __ j(kBelow, &skip_incr); |
| 936 | __ addss(t1, Address(ESP, 0)); |
| 937 | __ Bind(&skip_incr); |
| 938 | __ addl(ESP, Immediate(8)); |
| 939 | } |
Mark Mendell | fb8d279 | 2015-03-31 22:16:59 -0400 | [diff] [blame] | 940 | |
Aart Bik | 2c9f495 | 2016-08-01 16:52:27 -0700 | [diff] [blame] | 941 | // Final conversion to an integer. Unfortunately this also does not have a |
| 942 | // direct x86 instruction, since NaN should map to 0 and large positive |
| 943 | // values need to be clipped to the extreme value. |
Mark Mendell | fb8d279 | 2015-03-31 22:16:59 -0400 | [diff] [blame] | 944 | __ movl(out, Immediate(kPrimIntMax)); |
Aart Bik | 2c9f495 | 2016-08-01 16:52:27 -0700 | [diff] [blame] | 945 | __ cvtsi2ss(t2, out); |
| 946 | __ comiss(t1, t2); |
| 947 | __ j(kAboveEqual, &done); // clipped to max (already in out), does not jump on unordered |
| 948 | __ movl(out, Immediate(0)); // does not change flags |
| 949 | __ j(kUnordered, &done); // NaN mapped to 0 (just moved in out) |
| 950 | __ cvttss2si(out, t1); |
Mark Mendell | fb8d279 | 2015-03-31 22:16:59 -0400 | [diff] [blame] | 951 | __ Bind(&done); |
| 952 | } |
| 953 | |
Mark Mendell | a4f1220 | 2015-08-06 15:23:34 -0400 | [diff] [blame] | 954 | static void CreateFPToFPCallLocations(ArenaAllocator* arena, |
| 955 | HInvoke* invoke) { |
| 956 | LocationSummary* locations = new (arena) LocationSummary(invoke, |
Serban Constantinescu | 54ff482 | 2016-07-07 18:03:19 +0100 | [diff] [blame] | 957 | LocationSummary::kCallOnMainOnly, |
Mark Mendell | a4f1220 | 2015-08-06 15:23:34 -0400 | [diff] [blame] | 958 | kIntrinsified); |
| 959 | InvokeRuntimeCallingConvention calling_convention; |
| 960 | locations->SetInAt(0, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); |
| 961 | locations->SetOut(Location::FpuRegisterLocation(XMM0)); |
| 962 | } |
| 963 | |
| 964 | static void GenFPToFPCall(HInvoke* invoke, CodeGeneratorX86* codegen, QuickEntrypointEnum entry) { |
| 965 | LocationSummary* locations = invoke->GetLocations(); |
| 966 | DCHECK(locations->WillCall()); |
| 967 | DCHECK(invoke->IsInvokeStaticOrDirect()); |
| 968 | X86Assembler* assembler = codegen->GetAssembler(); |
| 969 | |
| 970 | // We need some place to pass the parameters. |
| 971 | __ subl(ESP, Immediate(16)); |
| 972 | __ cfi().AdjustCFAOffset(16); |
| 973 | |
| 974 | // Pass the parameters at the bottom of the stack. |
| 975 | __ movsd(Address(ESP, 0), XMM0); |
| 976 | |
| 977 | // If we have a second parameter, pass it next. |
| 978 | if (invoke->GetNumberOfArguments() == 2) { |
| 979 | __ movsd(Address(ESP, 8), XMM1); |
| 980 | } |
| 981 | |
| 982 | // Now do the actual call. |
Serban Constantinescu | ba45db0 | 2016-07-12 22:53:02 +0100 | [diff] [blame] | 983 | codegen->InvokeRuntime(entry, invoke, invoke->GetDexPc()); |
Mark Mendell | a4f1220 | 2015-08-06 15:23:34 -0400 | [diff] [blame] | 984 | |
| 985 | // Extract the return value from the FP stack. |
| 986 | __ fstpl(Address(ESP, 0)); |
| 987 | __ movsd(XMM0, Address(ESP, 0)); |
| 988 | |
| 989 | // And clean up the stack. |
| 990 | __ addl(ESP, Immediate(16)); |
| 991 | __ cfi().AdjustCFAOffset(-16); |
Mark Mendell | a4f1220 | 2015-08-06 15:23:34 -0400 | [diff] [blame] | 992 | } |
| 993 | |
| 994 | void IntrinsicLocationsBuilderX86::VisitMathCos(HInvoke* invoke) { |
| 995 | CreateFPToFPCallLocations(arena_, invoke); |
| 996 | } |
| 997 | |
| 998 | void IntrinsicCodeGeneratorX86::VisitMathCos(HInvoke* invoke) { |
| 999 | GenFPToFPCall(invoke, codegen_, kQuickCos); |
| 1000 | } |
| 1001 | |
| 1002 | void IntrinsicLocationsBuilderX86::VisitMathSin(HInvoke* invoke) { |
| 1003 | CreateFPToFPCallLocations(arena_, invoke); |
| 1004 | } |
| 1005 | |
| 1006 | void IntrinsicCodeGeneratorX86::VisitMathSin(HInvoke* invoke) { |
| 1007 | GenFPToFPCall(invoke, codegen_, kQuickSin); |
| 1008 | } |
| 1009 | |
| 1010 | void IntrinsicLocationsBuilderX86::VisitMathAcos(HInvoke* invoke) { |
| 1011 | CreateFPToFPCallLocations(arena_, invoke); |
| 1012 | } |
| 1013 | |
| 1014 | void IntrinsicCodeGeneratorX86::VisitMathAcos(HInvoke* invoke) { |
| 1015 | GenFPToFPCall(invoke, codegen_, kQuickAcos); |
| 1016 | } |
| 1017 | |
| 1018 | void IntrinsicLocationsBuilderX86::VisitMathAsin(HInvoke* invoke) { |
| 1019 | CreateFPToFPCallLocations(arena_, invoke); |
| 1020 | } |
| 1021 | |
| 1022 | void IntrinsicCodeGeneratorX86::VisitMathAsin(HInvoke* invoke) { |
| 1023 | GenFPToFPCall(invoke, codegen_, kQuickAsin); |
| 1024 | } |
| 1025 | |
| 1026 | void IntrinsicLocationsBuilderX86::VisitMathAtan(HInvoke* invoke) { |
| 1027 | CreateFPToFPCallLocations(arena_, invoke); |
| 1028 | } |
| 1029 | |
| 1030 | void IntrinsicCodeGeneratorX86::VisitMathAtan(HInvoke* invoke) { |
| 1031 | GenFPToFPCall(invoke, codegen_, kQuickAtan); |
| 1032 | } |
| 1033 | |
| 1034 | void IntrinsicLocationsBuilderX86::VisitMathCbrt(HInvoke* invoke) { |
| 1035 | CreateFPToFPCallLocations(arena_, invoke); |
| 1036 | } |
| 1037 | |
| 1038 | void IntrinsicCodeGeneratorX86::VisitMathCbrt(HInvoke* invoke) { |
| 1039 | GenFPToFPCall(invoke, codegen_, kQuickCbrt); |
| 1040 | } |
| 1041 | |
| 1042 | void IntrinsicLocationsBuilderX86::VisitMathCosh(HInvoke* invoke) { |
| 1043 | CreateFPToFPCallLocations(arena_, invoke); |
| 1044 | } |
| 1045 | |
| 1046 | void IntrinsicCodeGeneratorX86::VisitMathCosh(HInvoke* invoke) { |
| 1047 | GenFPToFPCall(invoke, codegen_, kQuickCosh); |
| 1048 | } |
| 1049 | |
| 1050 | void IntrinsicLocationsBuilderX86::VisitMathExp(HInvoke* invoke) { |
| 1051 | CreateFPToFPCallLocations(arena_, invoke); |
| 1052 | } |
| 1053 | |
| 1054 | void IntrinsicCodeGeneratorX86::VisitMathExp(HInvoke* invoke) { |
| 1055 | GenFPToFPCall(invoke, codegen_, kQuickExp); |
| 1056 | } |
| 1057 | |
| 1058 | void IntrinsicLocationsBuilderX86::VisitMathExpm1(HInvoke* invoke) { |
| 1059 | CreateFPToFPCallLocations(arena_, invoke); |
| 1060 | } |
| 1061 | |
| 1062 | void IntrinsicCodeGeneratorX86::VisitMathExpm1(HInvoke* invoke) { |
| 1063 | GenFPToFPCall(invoke, codegen_, kQuickExpm1); |
| 1064 | } |
| 1065 | |
| 1066 | void IntrinsicLocationsBuilderX86::VisitMathLog(HInvoke* invoke) { |
| 1067 | CreateFPToFPCallLocations(arena_, invoke); |
| 1068 | } |
| 1069 | |
| 1070 | void IntrinsicCodeGeneratorX86::VisitMathLog(HInvoke* invoke) { |
| 1071 | GenFPToFPCall(invoke, codegen_, kQuickLog); |
| 1072 | } |
| 1073 | |
| 1074 | void IntrinsicLocationsBuilderX86::VisitMathLog10(HInvoke* invoke) { |
| 1075 | CreateFPToFPCallLocations(arena_, invoke); |
| 1076 | } |
| 1077 | |
| 1078 | void IntrinsicCodeGeneratorX86::VisitMathLog10(HInvoke* invoke) { |
| 1079 | GenFPToFPCall(invoke, codegen_, kQuickLog10); |
| 1080 | } |
| 1081 | |
| 1082 | void IntrinsicLocationsBuilderX86::VisitMathSinh(HInvoke* invoke) { |
| 1083 | CreateFPToFPCallLocations(arena_, invoke); |
| 1084 | } |
| 1085 | |
| 1086 | void IntrinsicCodeGeneratorX86::VisitMathSinh(HInvoke* invoke) { |
| 1087 | GenFPToFPCall(invoke, codegen_, kQuickSinh); |
| 1088 | } |
| 1089 | |
| 1090 | void IntrinsicLocationsBuilderX86::VisitMathTan(HInvoke* invoke) { |
| 1091 | CreateFPToFPCallLocations(arena_, invoke); |
| 1092 | } |
| 1093 | |
| 1094 | void IntrinsicCodeGeneratorX86::VisitMathTan(HInvoke* invoke) { |
| 1095 | GenFPToFPCall(invoke, codegen_, kQuickTan); |
| 1096 | } |
| 1097 | |
| 1098 | void IntrinsicLocationsBuilderX86::VisitMathTanh(HInvoke* invoke) { |
| 1099 | CreateFPToFPCallLocations(arena_, invoke); |
| 1100 | } |
| 1101 | |
| 1102 | void IntrinsicCodeGeneratorX86::VisitMathTanh(HInvoke* invoke) { |
| 1103 | GenFPToFPCall(invoke, codegen_, kQuickTanh); |
| 1104 | } |
| 1105 | |
| 1106 | static void CreateFPFPToFPCallLocations(ArenaAllocator* arena, |
| 1107 | HInvoke* invoke) { |
| 1108 | LocationSummary* locations = new (arena) LocationSummary(invoke, |
Serban Constantinescu | 54ff482 | 2016-07-07 18:03:19 +0100 | [diff] [blame] | 1109 | LocationSummary::kCallOnMainOnly, |
Mark Mendell | a4f1220 | 2015-08-06 15:23:34 -0400 | [diff] [blame] | 1110 | kIntrinsified); |
| 1111 | InvokeRuntimeCallingConvention calling_convention; |
| 1112 | locations->SetInAt(0, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); |
| 1113 | locations->SetInAt(1, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(1))); |
| 1114 | locations->SetOut(Location::FpuRegisterLocation(XMM0)); |
| 1115 | } |
| 1116 | |
| 1117 | void IntrinsicLocationsBuilderX86::VisitMathAtan2(HInvoke* invoke) { |
| 1118 | CreateFPFPToFPCallLocations(arena_, invoke); |
| 1119 | } |
| 1120 | |
| 1121 | void IntrinsicCodeGeneratorX86::VisitMathAtan2(HInvoke* invoke) { |
| 1122 | GenFPToFPCall(invoke, codegen_, kQuickAtan2); |
| 1123 | } |
| 1124 | |
| 1125 | void IntrinsicLocationsBuilderX86::VisitMathHypot(HInvoke* invoke) { |
| 1126 | CreateFPFPToFPCallLocations(arena_, invoke); |
| 1127 | } |
| 1128 | |
| 1129 | void IntrinsicCodeGeneratorX86::VisitMathHypot(HInvoke* invoke) { |
| 1130 | GenFPToFPCall(invoke, codegen_, kQuickHypot); |
| 1131 | } |
| 1132 | |
| 1133 | void IntrinsicLocationsBuilderX86::VisitMathNextAfter(HInvoke* invoke) { |
| 1134 | CreateFPFPToFPCallLocations(arena_, invoke); |
| 1135 | } |
| 1136 | |
| 1137 | void IntrinsicCodeGeneratorX86::VisitMathNextAfter(HInvoke* invoke) { |
| 1138 | GenFPToFPCall(invoke, codegen_, kQuickNextAfter); |
| 1139 | } |
| 1140 | |
Mark Mendell | 6bc53a9 | 2015-07-01 14:26:52 -0400 | [diff] [blame] | 1141 | void IntrinsicLocationsBuilderX86::VisitSystemArrayCopyChar(HInvoke* invoke) { |
| 1142 | // We need at least two of the positions or length to be an integer constant, |
| 1143 | // or else we won't have enough free registers. |
| 1144 | HIntConstant* src_pos = invoke->InputAt(1)->AsIntConstant(); |
| 1145 | HIntConstant* dest_pos = invoke->InputAt(3)->AsIntConstant(); |
| 1146 | HIntConstant* length = invoke->InputAt(4)->AsIntConstant(); |
| 1147 | |
| 1148 | int num_constants = |
| 1149 | ((src_pos != nullptr) ? 1 : 0) |
| 1150 | + ((dest_pos != nullptr) ? 1 : 0) |
| 1151 | + ((length != nullptr) ? 1 : 0); |
| 1152 | |
| 1153 | if (num_constants < 2) { |
| 1154 | // Not enough free registers. |
| 1155 | return; |
| 1156 | } |
| 1157 | |
| 1158 | // As long as we are checking, we might as well check to see if the src and dest |
| 1159 | // positions are >= 0. |
| 1160 | if ((src_pos != nullptr && src_pos->GetValue() < 0) || |
| 1161 | (dest_pos != nullptr && dest_pos->GetValue() < 0)) { |
| 1162 | // We will have to fail anyways. |
| 1163 | return; |
| 1164 | } |
| 1165 | |
| 1166 | // And since we are already checking, check the length too. |
| 1167 | if (length != nullptr) { |
| 1168 | int32_t len = length->GetValue(); |
| 1169 | if (len < 0) { |
| 1170 | // Just call as normal. |
| 1171 | return; |
| 1172 | } |
| 1173 | } |
| 1174 | |
| 1175 | // Okay, it is safe to generate inline code. |
| 1176 | LocationSummary* locations = |
| 1177 | new (arena_) LocationSummary(invoke, LocationSummary::kCallOnSlowPath, kIntrinsified); |
| 1178 | // arraycopy(Object src, int srcPos, Object dest, int destPos, int length). |
| 1179 | locations->SetInAt(0, Location::RequiresRegister()); |
| 1180 | locations->SetInAt(1, Location::RegisterOrConstant(invoke->InputAt(1))); |
| 1181 | locations->SetInAt(2, Location::RequiresRegister()); |
| 1182 | locations->SetInAt(3, Location::RegisterOrConstant(invoke->InputAt(3))); |
| 1183 | locations->SetInAt(4, Location::RegisterOrConstant(invoke->InputAt(4))); |
| 1184 | |
| 1185 | // And we need some temporaries. We will use REP MOVSW, so we need fixed registers. |
| 1186 | locations->AddTemp(Location::RegisterLocation(ESI)); |
| 1187 | locations->AddTemp(Location::RegisterLocation(EDI)); |
| 1188 | locations->AddTemp(Location::RegisterLocation(ECX)); |
| 1189 | } |
| 1190 | |
| 1191 | static void CheckPosition(X86Assembler* assembler, |
| 1192 | Location pos, |
| 1193 | Register input, |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 1194 | Location length, |
Andreas Gampe | 85b62f2 | 2015-09-09 13:15:38 -0700 | [diff] [blame] | 1195 | SlowPathCode* slow_path, |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 1196 | Register temp, |
| 1197 | bool length_is_input_length = false) { |
| 1198 | // Where is the length in the Array? |
Mark Mendell | 6bc53a9 | 2015-07-01 14:26:52 -0400 | [diff] [blame] | 1199 | const uint32_t length_offset = mirror::Array::LengthOffset().Uint32Value(); |
| 1200 | |
| 1201 | if (pos.IsConstant()) { |
| 1202 | int32_t pos_const = pos.GetConstant()->AsIntConstant()->GetValue(); |
| 1203 | if (pos_const == 0) { |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 1204 | if (!length_is_input_length) { |
| 1205 | // Check that length(input) >= length. |
| 1206 | if (length.IsConstant()) { |
| 1207 | __ cmpl(Address(input, length_offset), |
| 1208 | Immediate(length.GetConstant()->AsIntConstant()->GetValue())); |
| 1209 | } else { |
| 1210 | __ cmpl(Address(input, length_offset), length.AsRegister<Register>()); |
| 1211 | } |
| 1212 | __ j(kLess, slow_path->GetEntryLabel()); |
| 1213 | } |
Mark Mendell | 6bc53a9 | 2015-07-01 14:26:52 -0400 | [diff] [blame] | 1214 | } else { |
| 1215 | // Check that length(input) >= pos. |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 1216 | __ movl(temp, Address(input, length_offset)); |
| 1217 | __ subl(temp, Immediate(pos_const)); |
Mark Mendell | 6bc53a9 | 2015-07-01 14:26:52 -0400 | [diff] [blame] | 1218 | __ j(kLess, slow_path->GetEntryLabel()); |
| 1219 | |
| 1220 | // Check that (length(input) - pos) >= length. |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 1221 | if (length.IsConstant()) { |
| 1222 | __ cmpl(temp, Immediate(length.GetConstant()->AsIntConstant()->GetValue())); |
| 1223 | } else { |
| 1224 | __ cmpl(temp, length.AsRegister<Register>()); |
| 1225 | } |
Mark Mendell | 6bc53a9 | 2015-07-01 14:26:52 -0400 | [diff] [blame] | 1226 | __ j(kLess, slow_path->GetEntryLabel()); |
| 1227 | } |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 1228 | } else if (length_is_input_length) { |
| 1229 | // The only way the copy can succeed is if pos is zero. |
| 1230 | Register pos_reg = pos.AsRegister<Register>(); |
| 1231 | __ testl(pos_reg, pos_reg); |
| 1232 | __ j(kNotEqual, slow_path->GetEntryLabel()); |
Mark Mendell | 6bc53a9 | 2015-07-01 14:26:52 -0400 | [diff] [blame] | 1233 | } else { |
| 1234 | // Check that pos >= 0. |
| 1235 | Register pos_reg = pos.AsRegister<Register>(); |
| 1236 | __ testl(pos_reg, pos_reg); |
| 1237 | __ j(kLess, slow_path->GetEntryLabel()); |
| 1238 | |
| 1239 | // Check that pos <= length(input). |
| 1240 | __ cmpl(Address(input, length_offset), pos_reg); |
| 1241 | __ j(kLess, slow_path->GetEntryLabel()); |
| 1242 | |
| 1243 | // Check that (length(input) - pos) >= length. |
| 1244 | __ movl(temp, Address(input, length_offset)); |
| 1245 | __ subl(temp, pos_reg); |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 1246 | if (length.IsConstant()) { |
| 1247 | __ cmpl(temp, Immediate(length.GetConstant()->AsIntConstant()->GetValue())); |
| 1248 | } else { |
| 1249 | __ cmpl(temp, length.AsRegister<Register>()); |
| 1250 | } |
Mark Mendell | 6bc53a9 | 2015-07-01 14:26:52 -0400 | [diff] [blame] | 1251 | __ j(kLess, slow_path->GetEntryLabel()); |
| 1252 | } |
| 1253 | } |
| 1254 | |
| 1255 | void IntrinsicCodeGeneratorX86::VisitSystemArrayCopyChar(HInvoke* invoke) { |
| 1256 | X86Assembler* assembler = GetAssembler(); |
| 1257 | LocationSummary* locations = invoke->GetLocations(); |
| 1258 | |
| 1259 | Register src = locations->InAt(0).AsRegister<Register>(); |
| 1260 | Location srcPos = locations->InAt(1); |
| 1261 | Register dest = locations->InAt(2).AsRegister<Register>(); |
| 1262 | Location destPos = locations->InAt(3); |
| 1263 | Location length = locations->InAt(4); |
| 1264 | |
| 1265 | // Temporaries that we need for MOVSW. |
| 1266 | Register src_base = locations->GetTemp(0).AsRegister<Register>(); |
| 1267 | DCHECK_EQ(src_base, ESI); |
| 1268 | Register dest_base = locations->GetTemp(1).AsRegister<Register>(); |
| 1269 | DCHECK_EQ(dest_base, EDI); |
| 1270 | Register count = locations->GetTemp(2).AsRegister<Register>(); |
| 1271 | DCHECK_EQ(count, ECX); |
| 1272 | |
Andreas Gampe | 85b62f2 | 2015-09-09 13:15:38 -0700 | [diff] [blame] | 1273 | SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathX86(invoke); |
Mark Mendell | 6bc53a9 | 2015-07-01 14:26:52 -0400 | [diff] [blame] | 1274 | codegen_->AddSlowPath(slow_path); |
| 1275 | |
| 1276 | // Bail out if the source and destination are the same (to handle overlap). |
| 1277 | __ cmpl(src, dest); |
| 1278 | __ j(kEqual, slow_path->GetEntryLabel()); |
| 1279 | |
| 1280 | // Bail out if the source is null. |
| 1281 | __ testl(src, src); |
| 1282 | __ j(kEqual, slow_path->GetEntryLabel()); |
| 1283 | |
| 1284 | // Bail out if the destination is null. |
| 1285 | __ testl(dest, dest); |
| 1286 | __ j(kEqual, slow_path->GetEntryLabel()); |
| 1287 | |
| 1288 | // If the length is negative, bail out. |
| 1289 | // We have already checked in the LocationsBuilder for the constant case. |
| 1290 | if (!length.IsConstant()) { |
| 1291 | __ cmpl(length.AsRegister<Register>(), length.AsRegister<Register>()); |
| 1292 | __ j(kLess, slow_path->GetEntryLabel()); |
| 1293 | } |
| 1294 | |
| 1295 | // We need the count in ECX. |
| 1296 | if (length.IsConstant()) { |
| 1297 | __ movl(count, Immediate(length.GetConstant()->AsIntConstant()->GetValue())); |
| 1298 | } else { |
| 1299 | __ movl(count, length.AsRegister<Register>()); |
| 1300 | } |
| 1301 | |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 1302 | // Validity checks: source. Use src_base as a temporary register. |
| 1303 | CheckPosition(assembler, srcPos, src, Location::RegisterLocation(count), slow_path, src_base); |
Mark Mendell | 6bc53a9 | 2015-07-01 14:26:52 -0400 | [diff] [blame] | 1304 | |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 1305 | // Validity checks: dest. Use src_base as a temporary register. |
| 1306 | CheckPosition(assembler, destPos, dest, Location::RegisterLocation(count), slow_path, src_base); |
Mark Mendell | 6bc53a9 | 2015-07-01 14:26:52 -0400 | [diff] [blame] | 1307 | |
| 1308 | // Okay, everything checks out. Finally time to do the copy. |
| 1309 | // Check assumption that sizeof(Char) is 2 (used in scaling below). |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1310 | const size_t char_size = DataType::Size(DataType::Type::kUint16); |
Mark Mendell | 6bc53a9 | 2015-07-01 14:26:52 -0400 | [diff] [blame] | 1311 | DCHECK_EQ(char_size, 2u); |
| 1312 | |
| 1313 | const uint32_t data_offset = mirror::Array::DataOffset(char_size).Uint32Value(); |
| 1314 | |
| 1315 | if (srcPos.IsConstant()) { |
| 1316 | int32_t srcPos_const = srcPos.GetConstant()->AsIntConstant()->GetValue(); |
| 1317 | __ leal(src_base, Address(src, char_size * srcPos_const + data_offset)); |
| 1318 | } else { |
| 1319 | __ leal(src_base, Address(src, srcPos.AsRegister<Register>(), |
| 1320 | ScaleFactor::TIMES_2, data_offset)); |
| 1321 | } |
| 1322 | if (destPos.IsConstant()) { |
| 1323 | int32_t destPos_const = destPos.GetConstant()->AsIntConstant()->GetValue(); |
| 1324 | |
| 1325 | __ leal(dest_base, Address(dest, char_size * destPos_const + data_offset)); |
| 1326 | } else { |
| 1327 | __ leal(dest_base, Address(dest, destPos.AsRegister<Register>(), |
| 1328 | ScaleFactor::TIMES_2, data_offset)); |
| 1329 | } |
| 1330 | |
| 1331 | // Do the move. |
| 1332 | __ rep_movsw(); |
| 1333 | |
| 1334 | __ Bind(slow_path->GetExitLabel()); |
| 1335 | } |
| 1336 | |
Nicolas Geoffray | d75948a | 2015-03-27 09:53:16 +0000 | [diff] [blame] | 1337 | void IntrinsicLocationsBuilderX86::VisitStringCompareTo(HInvoke* invoke) { |
| 1338 | // The inputs plus one temp. |
| 1339 | LocationSummary* locations = new (arena_) LocationSummary(invoke, |
Serban Constantinescu | 806f012 | 2016-03-09 11:10:16 +0000 | [diff] [blame] | 1340 | LocationSummary::kCallOnMainAndSlowPath, |
Nicolas Geoffray | d75948a | 2015-03-27 09:53:16 +0000 | [diff] [blame] | 1341 | kIntrinsified); |
| 1342 | InvokeRuntimeCallingConvention calling_convention; |
| 1343 | locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); |
| 1344 | locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); |
| 1345 | locations->SetOut(Location::RegisterLocation(EAX)); |
Nicolas Geoffray | d75948a | 2015-03-27 09:53:16 +0000 | [diff] [blame] | 1346 | } |
| 1347 | |
| 1348 | void IntrinsicCodeGeneratorX86::VisitStringCompareTo(HInvoke* invoke) { |
| 1349 | X86Assembler* assembler = GetAssembler(); |
| 1350 | LocationSummary* locations = invoke->GetLocations(); |
| 1351 | |
Nicolas Geoffray | 512e04d | 2015-03-27 17:21:24 +0000 | [diff] [blame] | 1352 | // Note that the null check must have been done earlier. |
Calin Juravle | 641547a | 2015-04-21 22:08:51 +0100 | [diff] [blame] | 1353 | DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); |
Nicolas Geoffray | d75948a | 2015-03-27 09:53:16 +0000 | [diff] [blame] | 1354 | |
| 1355 | Register argument = locations->InAt(1).AsRegister<Register>(); |
| 1356 | __ testl(argument, argument); |
Andreas Gampe | 85b62f2 | 2015-09-09 13:15:38 -0700 | [diff] [blame] | 1357 | SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathX86(invoke); |
Nicolas Geoffray | d75948a | 2015-03-27 09:53:16 +0000 | [diff] [blame] | 1358 | codegen_->AddSlowPath(slow_path); |
| 1359 | __ j(kEqual, slow_path->GetEntryLabel()); |
| 1360 | |
Serban Constantinescu | ba45db0 | 2016-07-12 22:53:02 +0100 | [diff] [blame] | 1361 | codegen_->InvokeRuntime(kQuickStringCompareTo, invoke, invoke->GetDexPc(), slow_path); |
Nicolas Geoffray | d75948a | 2015-03-27 09:53:16 +0000 | [diff] [blame] | 1362 | __ Bind(slow_path->GetExitLabel()); |
| 1363 | } |
| 1364 | |
Agi Csaki | d7138c8 | 2015-08-13 17:46:44 -0700 | [diff] [blame] | 1365 | void IntrinsicLocationsBuilderX86::VisitStringEquals(HInvoke* invoke) { |
| 1366 | LocationSummary* locations = new (arena_) LocationSummary(invoke, |
| 1367 | LocationSummary::kNoCall, |
| 1368 | kIntrinsified); |
| 1369 | locations->SetInAt(0, Location::RequiresRegister()); |
| 1370 | locations->SetInAt(1, Location::RequiresRegister()); |
| 1371 | |
| 1372 | // Request temporary registers, ECX and EDI needed for repe_cmpsl instruction. |
| 1373 | locations->AddTemp(Location::RegisterLocation(ECX)); |
| 1374 | locations->AddTemp(Location::RegisterLocation(EDI)); |
| 1375 | |
| 1376 | // Set output, ESI needed for repe_cmpsl instruction anyways. |
| 1377 | locations->SetOut(Location::RegisterLocation(ESI), Location::kOutputOverlap); |
| 1378 | } |
| 1379 | |
| 1380 | void IntrinsicCodeGeneratorX86::VisitStringEquals(HInvoke* invoke) { |
| 1381 | X86Assembler* assembler = GetAssembler(); |
| 1382 | LocationSummary* locations = invoke->GetLocations(); |
| 1383 | |
| 1384 | Register str = locations->InAt(0).AsRegister<Register>(); |
| 1385 | Register arg = locations->InAt(1).AsRegister<Register>(); |
| 1386 | Register ecx = locations->GetTemp(0).AsRegister<Register>(); |
| 1387 | Register edi = locations->GetTemp(1).AsRegister<Register>(); |
| 1388 | Register esi = locations->Out().AsRegister<Register>(); |
| 1389 | |
Mark Mendell | 0c9497d | 2015-08-21 09:30:05 -0400 | [diff] [blame] | 1390 | NearLabel end, return_true, return_false; |
Agi Csaki | d7138c8 | 2015-08-13 17:46:44 -0700 | [diff] [blame] | 1391 | |
| 1392 | // Get offsets of count, value, and class fields within a string object. |
| 1393 | const uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); |
| 1394 | const uint32_t value_offset = mirror::String::ValueOffset().Uint32Value(); |
| 1395 | const uint32_t class_offset = mirror::Object::ClassOffset().Uint32Value(); |
| 1396 | |
| 1397 | // Note that the null check must have been done earlier. |
| 1398 | DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); |
| 1399 | |
Nicolas Geoffray | a83a54d | 2015-10-02 17:30:26 +0100 | [diff] [blame] | 1400 | StringEqualsOptimizations optimizations(invoke); |
| 1401 | if (!optimizations.GetArgumentNotNull()) { |
| 1402 | // Check if input is null, return false if it is. |
| 1403 | __ testl(arg, arg); |
| 1404 | __ j(kEqual, &return_false); |
| 1405 | } |
Agi Csaki | d7138c8 | 2015-08-13 17:46:44 -0700 | [diff] [blame] | 1406 | |
Nicolas Geoffray | a83a54d | 2015-10-02 17:30:26 +0100 | [diff] [blame] | 1407 | if (!optimizations.GetArgumentIsString()) { |
Vladimir Marko | 53b5200 | 2016-05-24 19:30:45 +0100 | [diff] [blame] | 1408 | // Instanceof check for the argument by comparing class fields. |
| 1409 | // All string objects must have the same type since String cannot be subclassed. |
| 1410 | // Receiver must be a string object, so its class field is equal to all strings' class fields. |
| 1411 | // If the argument is a string object, its class field must be equal to receiver's class field. |
Nicolas Geoffray | a83a54d | 2015-10-02 17:30:26 +0100 | [diff] [blame] | 1412 | __ movl(ecx, Address(str, class_offset)); |
| 1413 | __ cmpl(ecx, Address(arg, class_offset)); |
| 1414 | __ j(kNotEqual, &return_false); |
| 1415 | } |
Agi Csaki | d7138c8 | 2015-08-13 17:46:44 -0700 | [diff] [blame] | 1416 | |
| 1417 | // Reference equality check, return true if same reference. |
| 1418 | __ cmpl(str, arg); |
| 1419 | __ j(kEqual, &return_true); |
| 1420 | |
jessicahandojo | 4877b79 | 2016-09-08 19:49:13 -0700 | [diff] [blame] | 1421 | // Load length and compression flag of receiver string. |
Agi Csaki | d7138c8 | 2015-08-13 17:46:44 -0700 | [diff] [blame] | 1422 | __ movl(ecx, Address(str, count_offset)); |
jessicahandojo | 4877b79 | 2016-09-08 19:49:13 -0700 | [diff] [blame] | 1423 | // Check if lengths and compression flags are equal, return false if they're not. |
| 1424 | // Two identical strings will always have same compression style since |
| 1425 | // compression style is decided on alloc. |
Agi Csaki | d7138c8 | 2015-08-13 17:46:44 -0700 | [diff] [blame] | 1426 | __ cmpl(ecx, Address(arg, count_offset)); |
| 1427 | __ j(kNotEqual, &return_false); |
Vladimir Marko | fdaf0f4 | 2016-10-13 19:29:53 +0100 | [diff] [blame] | 1428 | // Return true if strings are empty. Even with string compression `count == 0` means empty. |
| 1429 | static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, |
| 1430 | "Expecting 0=compressed, 1=uncompressed"); |
| 1431 | __ jecxz(&return_true); |
Agi Csaki | d7138c8 | 2015-08-13 17:46:44 -0700 | [diff] [blame] | 1432 | |
jessicahandojo | 4877b79 | 2016-09-08 19:49:13 -0700 | [diff] [blame] | 1433 | if (mirror::kUseStringCompression) { |
| 1434 | NearLabel string_uncompressed; |
Vladimir Marko | fdaf0f4 | 2016-10-13 19:29:53 +0100 | [diff] [blame] | 1435 | // Extract length and differentiate between both compressed or both uncompressed. |
| 1436 | // Different compression style is cut above. |
| 1437 | __ shrl(ecx, Immediate(1)); |
| 1438 | __ j(kCarrySet, &string_uncompressed); |
jessicahandojo | 4877b79 | 2016-09-08 19:49:13 -0700 | [diff] [blame] | 1439 | // Divide string length by 2, rounding up, and continue as if uncompressed. |
Vladimir Marko | fdaf0f4 | 2016-10-13 19:29:53 +0100 | [diff] [blame] | 1440 | __ addl(ecx, Immediate(1)); |
jessicahandojo | 4877b79 | 2016-09-08 19:49:13 -0700 | [diff] [blame] | 1441 | __ shrl(ecx, Immediate(1)); |
| 1442 | __ Bind(&string_uncompressed); |
| 1443 | } |
Agi Csaki | d7138c8 | 2015-08-13 17:46:44 -0700 | [diff] [blame] | 1444 | // Load starting addresses of string values into ESI/EDI as required for repe_cmpsl instruction. |
| 1445 | __ leal(esi, Address(str, value_offset)); |
| 1446 | __ leal(edi, Address(arg, value_offset)); |
| 1447 | |
jessicahandojo | 4877b79 | 2016-09-08 19:49:13 -0700 | [diff] [blame] | 1448 | // Divide string length by 2 to compare characters 2 at a time and adjust for lengths not |
| 1449 | // divisible by 2. |
Agi Csaki | d7138c8 | 2015-08-13 17:46:44 -0700 | [diff] [blame] | 1450 | __ addl(ecx, Immediate(1)); |
| 1451 | __ shrl(ecx, Immediate(1)); |
| 1452 | |
jessicahandojo | 4877b79 | 2016-09-08 19:49:13 -0700 | [diff] [blame] | 1453 | // Assertions that must hold in order to compare strings 2 characters (uncompressed) |
| 1454 | // or 4 characters (compressed) at a time. |
Agi Csaki | d7138c8 | 2015-08-13 17:46:44 -0700 | [diff] [blame] | 1455 | DCHECK_ALIGNED(value_offset, 4); |
| 1456 | static_assert(IsAligned<4>(kObjectAlignment), "String of odd length is not zero padded"); |
| 1457 | |
| 1458 | // Loop to compare strings two characters at a time starting at the beginning of the string. |
| 1459 | __ repe_cmpsl(); |
| 1460 | // If strings are not equal, zero flag will be cleared. |
| 1461 | __ j(kNotEqual, &return_false); |
| 1462 | |
| 1463 | // Return true and exit the function. |
| 1464 | // If loop does not result in returning false, we return true. |
| 1465 | __ Bind(&return_true); |
| 1466 | __ movl(esi, Immediate(1)); |
| 1467 | __ jmp(&end); |
| 1468 | |
| 1469 | // Return false and exit the function. |
| 1470 | __ Bind(&return_false); |
| 1471 | __ xorl(esi, esi); |
| 1472 | __ Bind(&end); |
| 1473 | } |
| 1474 | |
Andreas Gampe | 21030dd | 2015-05-07 14:46:15 -0700 | [diff] [blame] | 1475 | static void CreateStringIndexOfLocations(HInvoke* invoke, |
| 1476 | ArenaAllocator* allocator, |
| 1477 | bool start_at_zero) { |
| 1478 | LocationSummary* locations = new (allocator) LocationSummary(invoke, |
| 1479 | LocationSummary::kCallOnSlowPath, |
| 1480 | kIntrinsified); |
| 1481 | // The data needs to be in EDI for scasw. So request that the string is there, anyways. |
| 1482 | locations->SetInAt(0, Location::RegisterLocation(EDI)); |
| 1483 | // If we look for a constant char, we'll still have to copy it into EAX. So just request the |
| 1484 | // allocator to do that, anyways. We can still do the constant check by checking the parameter |
| 1485 | // of the instruction explicitly. |
| 1486 | // Note: This works as we don't clobber EAX anywhere. |
| 1487 | locations->SetInAt(1, Location::RegisterLocation(EAX)); |
| 1488 | if (!start_at_zero) { |
| 1489 | locations->SetInAt(2, Location::RequiresRegister()); // The starting index. |
| 1490 | } |
| 1491 | // As we clobber EDI during execution anyways, also use it as the output. |
| 1492 | locations->SetOut(Location::SameAsFirstInput()); |
| 1493 | |
| 1494 | // repne scasw uses ECX as the counter. |
| 1495 | locations->AddTemp(Location::RegisterLocation(ECX)); |
| 1496 | // Need another temporary to be able to compute the result. |
| 1497 | locations->AddTemp(Location::RequiresRegister()); |
jessicahandojo | 4877b79 | 2016-09-08 19:49:13 -0700 | [diff] [blame] | 1498 | if (mirror::kUseStringCompression) { |
| 1499 | // Need another temporary to be able to save unflagged string length. |
| 1500 | locations->AddTemp(Location::RequiresRegister()); |
| 1501 | } |
Andreas Gampe | 21030dd | 2015-05-07 14:46:15 -0700 | [diff] [blame] | 1502 | } |
| 1503 | |
| 1504 | static void GenerateStringIndexOf(HInvoke* invoke, |
| 1505 | X86Assembler* assembler, |
| 1506 | CodeGeneratorX86* codegen, |
| 1507 | ArenaAllocator* allocator, |
| 1508 | bool start_at_zero) { |
| 1509 | LocationSummary* locations = invoke->GetLocations(); |
| 1510 | |
| 1511 | // Note that the null check must have been done earlier. |
| 1512 | DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); |
| 1513 | |
| 1514 | Register string_obj = locations->InAt(0).AsRegister<Register>(); |
| 1515 | Register search_value = locations->InAt(1).AsRegister<Register>(); |
| 1516 | Register counter = locations->GetTemp(0).AsRegister<Register>(); |
| 1517 | Register string_length = locations->GetTemp(1).AsRegister<Register>(); |
| 1518 | Register out = locations->Out().AsRegister<Register>(); |
jessicahandojo | 4877b79 | 2016-09-08 19:49:13 -0700 | [diff] [blame] | 1519 | // Only used when string compression feature is on. |
| 1520 | Register string_length_flagged; |
Andreas Gampe | 21030dd | 2015-05-07 14:46:15 -0700 | [diff] [blame] | 1521 | |
| 1522 | // Check our assumptions for registers. |
| 1523 | DCHECK_EQ(string_obj, EDI); |
| 1524 | DCHECK_EQ(search_value, EAX); |
| 1525 | DCHECK_EQ(counter, ECX); |
| 1526 | DCHECK_EQ(out, EDI); |
| 1527 | |
| 1528 | // Check for code points > 0xFFFF. Either a slow-path check when we don't know statically, |
Vladimir Marko | fb6c90a | 2016-05-06 15:52:12 +0100 | [diff] [blame] | 1529 | // or directly dispatch for a large constant, or omit slow-path for a small constant or a char. |
Andreas Gampe | 85b62f2 | 2015-09-09 13:15:38 -0700 | [diff] [blame] | 1530 | SlowPathCode* slow_path = nullptr; |
Vladimir Marko | fb6c90a | 2016-05-06 15:52:12 +0100 | [diff] [blame] | 1531 | HInstruction* code_point = invoke->InputAt(1); |
| 1532 | if (code_point->IsIntConstant()) { |
Vladimir Marko | da05108 | 2016-05-17 16:10:20 +0100 | [diff] [blame] | 1533 | if (static_cast<uint32_t>(code_point->AsIntConstant()->GetValue()) > |
Andreas Gampe | 21030dd | 2015-05-07 14:46:15 -0700 | [diff] [blame] | 1534 | std::numeric_limits<uint16_t>::max()) { |
| 1535 | // Always needs the slow-path. We could directly dispatch to it, but this case should be |
| 1536 | // rare, so for simplicity just put the full slow-path down and branch unconditionally. |
| 1537 | slow_path = new (allocator) IntrinsicSlowPathX86(invoke); |
| 1538 | codegen->AddSlowPath(slow_path); |
| 1539 | __ jmp(slow_path->GetEntryLabel()); |
| 1540 | __ Bind(slow_path->GetExitLabel()); |
| 1541 | return; |
| 1542 | } |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1543 | } else if (code_point->GetType() != DataType::Type::kUint16) { |
Andreas Gampe | 21030dd | 2015-05-07 14:46:15 -0700 | [diff] [blame] | 1544 | __ cmpl(search_value, Immediate(std::numeric_limits<uint16_t>::max())); |
| 1545 | slow_path = new (allocator) IntrinsicSlowPathX86(invoke); |
| 1546 | codegen->AddSlowPath(slow_path); |
| 1547 | __ j(kAbove, slow_path->GetEntryLabel()); |
| 1548 | } |
| 1549 | |
| 1550 | // From here down, we know that we are looking for a char that fits in 16 bits. |
| 1551 | // Location of reference to data array within the String object. |
| 1552 | int32_t value_offset = mirror::String::ValueOffset().Int32Value(); |
| 1553 | // Location of count within the String object. |
| 1554 | int32_t count_offset = mirror::String::CountOffset().Int32Value(); |
| 1555 | |
Vladimir Marko | fdaf0f4 | 2016-10-13 19:29:53 +0100 | [diff] [blame] | 1556 | // Load the count field of the string containing the length and compression flag. |
Andreas Gampe | 21030dd | 2015-05-07 14:46:15 -0700 | [diff] [blame] | 1557 | __ movl(string_length, Address(string_obj, count_offset)); |
| 1558 | |
Vladimir Marko | fdaf0f4 | 2016-10-13 19:29:53 +0100 | [diff] [blame] | 1559 | // Do a zero-length check. Even with string compression `count == 0` means empty. |
| 1560 | static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, |
| 1561 | "Expecting 0=compressed, 1=uncompressed"); |
Andreas Gampe | 21030dd | 2015-05-07 14:46:15 -0700 | [diff] [blame] | 1562 | // TODO: Support jecxz. |
Mark Mendell | 0c9497d | 2015-08-21 09:30:05 -0400 | [diff] [blame] | 1563 | NearLabel not_found_label; |
Andreas Gampe | 21030dd | 2015-05-07 14:46:15 -0700 | [diff] [blame] | 1564 | __ testl(string_length, string_length); |
| 1565 | __ j(kEqual, ¬_found_label); |
| 1566 | |
Vladimir Marko | fdaf0f4 | 2016-10-13 19:29:53 +0100 | [diff] [blame] | 1567 | if (mirror::kUseStringCompression) { |
| 1568 | string_length_flagged = locations->GetTemp(2).AsRegister<Register>(); |
| 1569 | __ movl(string_length_flagged, string_length); |
| 1570 | // Extract the length and shift out the least significant bit used as compression flag. |
| 1571 | __ shrl(string_length, Immediate(1)); |
| 1572 | } |
| 1573 | |
Andreas Gampe | 21030dd | 2015-05-07 14:46:15 -0700 | [diff] [blame] | 1574 | if (start_at_zero) { |
| 1575 | // Number of chars to scan is the same as the string length. |
| 1576 | __ movl(counter, string_length); |
| 1577 | |
| 1578 | // Move to the start of the string. |
| 1579 | __ addl(string_obj, Immediate(value_offset)); |
| 1580 | } else { |
| 1581 | Register start_index = locations->InAt(2).AsRegister<Register>(); |
| 1582 | |
| 1583 | // Do a start_index check. |
| 1584 | __ cmpl(start_index, string_length); |
| 1585 | __ j(kGreaterEqual, ¬_found_label); |
| 1586 | |
| 1587 | // Ensure we have a start index >= 0; |
| 1588 | __ xorl(counter, counter); |
| 1589 | __ cmpl(start_index, Immediate(0)); |
| 1590 | __ cmovl(kGreater, counter, start_index); |
| 1591 | |
jessicahandojo | 4877b79 | 2016-09-08 19:49:13 -0700 | [diff] [blame] | 1592 | if (mirror::kUseStringCompression) { |
| 1593 | NearLabel modify_counter, offset_uncompressed_label; |
Vladimir Marko | fdaf0f4 | 2016-10-13 19:29:53 +0100 | [diff] [blame] | 1594 | __ testl(string_length_flagged, Immediate(1)); |
| 1595 | __ j(kNotZero, &offset_uncompressed_label); |
jessicahandojo | 4877b79 | 2016-09-08 19:49:13 -0700 | [diff] [blame] | 1596 | // Move to the start of the string: string_obj + value_offset + start_index. |
| 1597 | __ leal(string_obj, Address(string_obj, counter, ScaleFactor::TIMES_1, value_offset)); |
| 1598 | __ jmp(&modify_counter); |
Andreas Gampe | 21030dd | 2015-05-07 14:46:15 -0700 | [diff] [blame] | 1599 | |
jessicahandojo | 4877b79 | 2016-09-08 19:49:13 -0700 | [diff] [blame] | 1600 | // Move to the start of the string: string_obj + value_offset + 2 * start_index. |
| 1601 | __ Bind(&offset_uncompressed_label); |
| 1602 | __ leal(string_obj, Address(string_obj, counter, ScaleFactor::TIMES_2, value_offset)); |
| 1603 | |
| 1604 | // Now update ecx (the repne scasw work counter). We have string.length - start_index left to |
| 1605 | // compare. |
| 1606 | __ Bind(&modify_counter); |
| 1607 | } else { |
| 1608 | __ leal(string_obj, Address(string_obj, counter, ScaleFactor::TIMES_2, value_offset)); |
| 1609 | } |
Andreas Gampe | 21030dd | 2015-05-07 14:46:15 -0700 | [diff] [blame] | 1610 | __ negl(counter); |
| 1611 | __ leal(counter, Address(string_length, counter, ScaleFactor::TIMES_1, 0)); |
| 1612 | } |
| 1613 | |
jessicahandojo | 4877b79 | 2016-09-08 19:49:13 -0700 | [diff] [blame] | 1614 | if (mirror::kUseStringCompression) { |
| 1615 | NearLabel uncompressed_string_comparison; |
| 1616 | NearLabel comparison_done; |
Vladimir Marko | fdaf0f4 | 2016-10-13 19:29:53 +0100 | [diff] [blame] | 1617 | __ testl(string_length_flagged, Immediate(1)); |
| 1618 | __ j(kNotZero, &uncompressed_string_comparison); |
Andreas Gampe | 21030dd | 2015-05-07 14:46:15 -0700 | [diff] [blame] | 1619 | |
jessicahandojo | 4877b79 | 2016-09-08 19:49:13 -0700 | [diff] [blame] | 1620 | // Check if EAX (search_value) is ASCII. |
| 1621 | __ cmpl(search_value, Immediate(127)); |
| 1622 | __ j(kGreater, ¬_found_label); |
| 1623 | // Comparing byte-per-byte. |
| 1624 | __ repne_scasb(); |
| 1625 | __ jmp(&comparison_done); |
| 1626 | |
| 1627 | // Everything is set up for repne scasw: |
| 1628 | // * Comparison address in EDI. |
| 1629 | // * Counter in ECX. |
| 1630 | __ Bind(&uncompressed_string_comparison); |
| 1631 | __ repne_scasw(); |
| 1632 | __ Bind(&comparison_done); |
| 1633 | } else { |
| 1634 | __ repne_scasw(); |
| 1635 | } |
Andreas Gampe | 21030dd | 2015-05-07 14:46:15 -0700 | [diff] [blame] | 1636 | // Did we find a match? |
| 1637 | __ j(kNotEqual, ¬_found_label); |
| 1638 | |
| 1639 | // Yes, we matched. Compute the index of the result. |
| 1640 | __ subl(string_length, counter); |
| 1641 | __ leal(out, Address(string_length, -1)); |
| 1642 | |
Mark Mendell | 0c9497d | 2015-08-21 09:30:05 -0400 | [diff] [blame] | 1643 | NearLabel done; |
Andreas Gampe | 21030dd | 2015-05-07 14:46:15 -0700 | [diff] [blame] | 1644 | __ jmp(&done); |
| 1645 | |
| 1646 | // Failed to match; return -1. |
| 1647 | __ Bind(¬_found_label); |
| 1648 | __ movl(out, Immediate(-1)); |
| 1649 | |
| 1650 | // And join up at the end. |
| 1651 | __ Bind(&done); |
| 1652 | if (slow_path != nullptr) { |
| 1653 | __ Bind(slow_path->GetExitLabel()); |
| 1654 | } |
| 1655 | } |
| 1656 | |
| 1657 | void IntrinsicLocationsBuilderX86::VisitStringIndexOf(HInvoke* invoke) { |
Roland Levillain | bf84a3d | 2015-12-04 14:33:02 +0000 | [diff] [blame] | 1658 | CreateStringIndexOfLocations(invoke, arena_, /* start_at_zero */ true); |
Andreas Gampe | 21030dd | 2015-05-07 14:46:15 -0700 | [diff] [blame] | 1659 | } |
| 1660 | |
| 1661 | void IntrinsicCodeGeneratorX86::VisitStringIndexOf(HInvoke* invoke) { |
Roland Levillain | bf84a3d | 2015-12-04 14:33:02 +0000 | [diff] [blame] | 1662 | GenerateStringIndexOf(invoke, GetAssembler(), codegen_, GetAllocator(), /* start_at_zero */ true); |
Andreas Gampe | 21030dd | 2015-05-07 14:46:15 -0700 | [diff] [blame] | 1663 | } |
| 1664 | |
| 1665 | void IntrinsicLocationsBuilderX86::VisitStringIndexOfAfter(HInvoke* invoke) { |
Roland Levillain | bf84a3d | 2015-12-04 14:33:02 +0000 | [diff] [blame] | 1666 | CreateStringIndexOfLocations(invoke, arena_, /* start_at_zero */ false); |
Andreas Gampe | 21030dd | 2015-05-07 14:46:15 -0700 | [diff] [blame] | 1667 | } |
| 1668 | |
| 1669 | void IntrinsicCodeGeneratorX86::VisitStringIndexOfAfter(HInvoke* invoke) { |
Roland Levillain | bf84a3d | 2015-12-04 14:33:02 +0000 | [diff] [blame] | 1670 | GenerateStringIndexOf( |
| 1671 | invoke, GetAssembler(), codegen_, GetAllocator(), /* start_at_zero */ false); |
Andreas Gampe | 21030dd | 2015-05-07 14:46:15 -0700 | [diff] [blame] | 1672 | } |
| 1673 | |
Jeff Hao | 848f70a | 2014-01-15 13:49:50 -0800 | [diff] [blame] | 1674 | void IntrinsicLocationsBuilderX86::VisitStringNewStringFromBytes(HInvoke* invoke) { |
| 1675 | LocationSummary* locations = new (arena_) LocationSummary(invoke, |
Serban Constantinescu | 806f012 | 2016-03-09 11:10:16 +0000 | [diff] [blame] | 1676 | LocationSummary::kCallOnMainAndSlowPath, |
Jeff Hao | 848f70a | 2014-01-15 13:49:50 -0800 | [diff] [blame] | 1677 | kIntrinsified); |
| 1678 | InvokeRuntimeCallingConvention calling_convention; |
| 1679 | locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); |
| 1680 | locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); |
| 1681 | locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); |
| 1682 | locations->SetInAt(3, Location::RegisterLocation(calling_convention.GetRegisterAt(3))); |
| 1683 | locations->SetOut(Location::RegisterLocation(EAX)); |
Jeff Hao | 848f70a | 2014-01-15 13:49:50 -0800 | [diff] [blame] | 1684 | } |
| 1685 | |
| 1686 | void IntrinsicCodeGeneratorX86::VisitStringNewStringFromBytes(HInvoke* invoke) { |
| 1687 | X86Assembler* assembler = GetAssembler(); |
| 1688 | LocationSummary* locations = invoke->GetLocations(); |
| 1689 | |
| 1690 | Register byte_array = locations->InAt(0).AsRegister<Register>(); |
| 1691 | __ testl(byte_array, byte_array); |
Andreas Gampe | 85b62f2 | 2015-09-09 13:15:38 -0700 | [diff] [blame] | 1692 | SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathX86(invoke); |
Jeff Hao | 848f70a | 2014-01-15 13:49:50 -0800 | [diff] [blame] | 1693 | codegen_->AddSlowPath(slow_path); |
| 1694 | __ j(kEqual, slow_path->GetEntryLabel()); |
| 1695 | |
Serban Constantinescu | ba45db0 | 2016-07-12 22:53:02 +0100 | [diff] [blame] | 1696 | codegen_->InvokeRuntime(kQuickAllocStringFromBytes, invoke, invoke->GetDexPc()); |
Roland Levillain | f969a20 | 2016-03-09 16:14:00 +0000 | [diff] [blame] | 1697 | CheckEntrypointTypes<kQuickAllocStringFromBytes, void*, void*, int32_t, int32_t, int32_t>(); |
Jeff Hao | 848f70a | 2014-01-15 13:49:50 -0800 | [diff] [blame] | 1698 | __ Bind(slow_path->GetExitLabel()); |
| 1699 | } |
| 1700 | |
| 1701 | void IntrinsicLocationsBuilderX86::VisitStringNewStringFromChars(HInvoke* invoke) { |
| 1702 | LocationSummary* locations = new (arena_) LocationSummary(invoke, |
Serban Constantinescu | 54ff482 | 2016-07-07 18:03:19 +0100 | [diff] [blame] | 1703 | LocationSummary::kCallOnMainOnly, |
Jeff Hao | 848f70a | 2014-01-15 13:49:50 -0800 | [diff] [blame] | 1704 | kIntrinsified); |
| 1705 | InvokeRuntimeCallingConvention calling_convention; |
| 1706 | locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); |
| 1707 | locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); |
| 1708 | locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); |
| 1709 | locations->SetOut(Location::RegisterLocation(EAX)); |
| 1710 | } |
| 1711 | |
| 1712 | void IntrinsicCodeGeneratorX86::VisitStringNewStringFromChars(HInvoke* invoke) { |
Roland Levillain | cc3839c | 2016-02-29 16:23:48 +0000 | [diff] [blame] | 1713 | // No need to emit code checking whether `locations->InAt(2)` is a null |
| 1714 | // pointer, as callers of the native method |
| 1715 | // |
| 1716 | // java.lang.StringFactory.newStringFromChars(int offset, int charCount, char[] data) |
| 1717 | // |
| 1718 | // all include a null check on `data` before calling that method. |
Serban Constantinescu | ba45db0 | 2016-07-12 22:53:02 +0100 | [diff] [blame] | 1719 | codegen_->InvokeRuntime(kQuickAllocStringFromChars, invoke, invoke->GetDexPc()); |
Roland Levillain | f969a20 | 2016-03-09 16:14:00 +0000 | [diff] [blame] | 1720 | CheckEntrypointTypes<kQuickAllocStringFromChars, void*, int32_t, int32_t, void*>(); |
Jeff Hao | 848f70a | 2014-01-15 13:49:50 -0800 | [diff] [blame] | 1721 | } |
| 1722 | |
| 1723 | void IntrinsicLocationsBuilderX86::VisitStringNewStringFromString(HInvoke* invoke) { |
| 1724 | LocationSummary* locations = new (arena_) LocationSummary(invoke, |
Serban Constantinescu | 806f012 | 2016-03-09 11:10:16 +0000 | [diff] [blame] | 1725 | LocationSummary::kCallOnMainAndSlowPath, |
Jeff Hao | 848f70a | 2014-01-15 13:49:50 -0800 | [diff] [blame] | 1726 | kIntrinsified); |
| 1727 | InvokeRuntimeCallingConvention calling_convention; |
| 1728 | locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); |
| 1729 | locations->SetOut(Location::RegisterLocation(EAX)); |
Jeff Hao | 848f70a | 2014-01-15 13:49:50 -0800 | [diff] [blame] | 1730 | } |
| 1731 | |
| 1732 | void IntrinsicCodeGeneratorX86::VisitStringNewStringFromString(HInvoke* invoke) { |
| 1733 | X86Assembler* assembler = GetAssembler(); |
| 1734 | LocationSummary* locations = invoke->GetLocations(); |
| 1735 | |
| 1736 | Register string_to_copy = locations->InAt(0).AsRegister<Register>(); |
| 1737 | __ testl(string_to_copy, string_to_copy); |
Andreas Gampe | 85b62f2 | 2015-09-09 13:15:38 -0700 | [diff] [blame] | 1738 | SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathX86(invoke); |
Jeff Hao | 848f70a | 2014-01-15 13:49:50 -0800 | [diff] [blame] | 1739 | codegen_->AddSlowPath(slow_path); |
| 1740 | __ j(kEqual, slow_path->GetEntryLabel()); |
| 1741 | |
Serban Constantinescu | ba45db0 | 2016-07-12 22:53:02 +0100 | [diff] [blame] | 1742 | codegen_->InvokeRuntime(kQuickAllocStringFromString, invoke, invoke->GetDexPc()); |
Roland Levillain | f969a20 | 2016-03-09 16:14:00 +0000 | [diff] [blame] | 1743 | CheckEntrypointTypes<kQuickAllocStringFromString, void*, void*>(); |
Jeff Hao | 848f70a | 2014-01-15 13:49:50 -0800 | [diff] [blame] | 1744 | __ Bind(slow_path->GetExitLabel()); |
| 1745 | } |
| 1746 | |
Mark Mendell | 8f8926a | 2015-08-17 11:39:06 -0400 | [diff] [blame] | 1747 | void IntrinsicLocationsBuilderX86::VisitStringGetCharsNoCheck(HInvoke* invoke) { |
| 1748 | // public void getChars(int srcBegin, int srcEnd, char[] dst, int dstBegin); |
| 1749 | LocationSummary* locations = new (arena_) LocationSummary(invoke, |
| 1750 | LocationSummary::kNoCall, |
| 1751 | kIntrinsified); |
| 1752 | locations->SetInAt(0, Location::RequiresRegister()); |
| 1753 | locations->SetInAt(1, Location::RegisterOrConstant(invoke->InputAt(1))); |
| 1754 | // Place srcEnd in ECX to save a move below. |
| 1755 | locations->SetInAt(2, Location::RegisterLocation(ECX)); |
| 1756 | locations->SetInAt(3, Location::RequiresRegister()); |
| 1757 | locations->SetInAt(4, Location::RequiresRegister()); |
| 1758 | |
| 1759 | // And we need some temporaries. We will use REP MOVSW, so we need fixed registers. |
| 1760 | // We don't have enough registers to also grab ECX, so handle below. |
| 1761 | locations->AddTemp(Location::RegisterLocation(ESI)); |
| 1762 | locations->AddTemp(Location::RegisterLocation(EDI)); |
| 1763 | } |
| 1764 | |
| 1765 | void IntrinsicCodeGeneratorX86::VisitStringGetCharsNoCheck(HInvoke* invoke) { |
| 1766 | X86Assembler* assembler = GetAssembler(); |
| 1767 | LocationSummary* locations = invoke->GetLocations(); |
| 1768 | |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1769 | size_t char_component_size = DataType::Size(DataType::Type::kUint16); |
Mark Mendell | 8f8926a | 2015-08-17 11:39:06 -0400 | [diff] [blame] | 1770 | // Location of data in char array buffer. |
| 1771 | const uint32_t data_offset = mirror::Array::DataOffset(char_component_size).Uint32Value(); |
| 1772 | // Location of char array data in string. |
| 1773 | const uint32_t value_offset = mirror::String::ValueOffset().Uint32Value(); |
| 1774 | |
| 1775 | // public void getChars(int srcBegin, int srcEnd, char[] dst, int dstBegin); |
| 1776 | Register obj = locations->InAt(0).AsRegister<Register>(); |
| 1777 | Location srcBegin = locations->InAt(1); |
| 1778 | int srcBegin_value = |
| 1779 | srcBegin.IsConstant() ? srcBegin.GetConstant()->AsIntConstant()->GetValue() : 0; |
| 1780 | Register srcEnd = locations->InAt(2).AsRegister<Register>(); |
| 1781 | Register dst = locations->InAt(3).AsRegister<Register>(); |
| 1782 | Register dstBegin = locations->InAt(4).AsRegister<Register>(); |
| 1783 | |
| 1784 | // Check assumption that sizeof(Char) is 2 (used in scaling below). |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1785 | const size_t char_size = DataType::Size(DataType::Type::kUint16); |
Mark Mendell | 8f8926a | 2015-08-17 11:39:06 -0400 | [diff] [blame] | 1786 | DCHECK_EQ(char_size, 2u); |
| 1787 | |
Mark Mendell | 8f8926a | 2015-08-17 11:39:06 -0400 | [diff] [blame] | 1788 | // Compute the number of chars (words) to move. |
jessicahandojo | 4877b79 | 2016-09-08 19:49:13 -0700 | [diff] [blame] | 1789 | // Save ECX, since we don't know if it will be used later. |
Mark Mendell | 8f8926a | 2015-08-17 11:39:06 -0400 | [diff] [blame] | 1790 | __ pushl(ECX); |
| 1791 | int stack_adjust = kX86WordSize; |
| 1792 | __ cfi().AdjustCFAOffset(stack_adjust); |
| 1793 | DCHECK_EQ(srcEnd, ECX); |
| 1794 | if (srcBegin.IsConstant()) { |
jessicahandojo | 4877b79 | 2016-09-08 19:49:13 -0700 | [diff] [blame] | 1795 | __ subl(ECX, Immediate(srcBegin_value)); |
Mark Mendell | 8f8926a | 2015-08-17 11:39:06 -0400 | [diff] [blame] | 1796 | } else { |
| 1797 | DCHECK(srcBegin.IsRegister()); |
| 1798 | __ subl(ECX, srcBegin.AsRegister<Register>()); |
| 1799 | } |
| 1800 | |
jessicahandojo | 4877b79 | 2016-09-08 19:49:13 -0700 | [diff] [blame] | 1801 | NearLabel done; |
| 1802 | if (mirror::kUseStringCompression) { |
| 1803 | // Location of count in string |
| 1804 | const uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1805 | const size_t c_char_size = DataType::Size(DataType::Type::kInt8); |
jessicahandojo | 4877b79 | 2016-09-08 19:49:13 -0700 | [diff] [blame] | 1806 | DCHECK_EQ(c_char_size, 1u); |
| 1807 | __ pushl(EAX); |
| 1808 | __ cfi().AdjustCFAOffset(stack_adjust); |
| 1809 | |
| 1810 | NearLabel copy_loop, copy_uncompressed; |
Vladimir Marko | fdaf0f4 | 2016-10-13 19:29:53 +0100 | [diff] [blame] | 1811 | __ testl(Address(obj, count_offset), Immediate(1)); |
| 1812 | static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, |
| 1813 | "Expecting 0=compressed, 1=uncompressed"); |
| 1814 | __ j(kNotZero, ©_uncompressed); |
jessicahandojo | 4877b79 | 2016-09-08 19:49:13 -0700 | [diff] [blame] | 1815 | // Compute the address of the source string by adding the number of chars from |
| 1816 | // the source beginning to the value offset of a string. |
| 1817 | __ leal(ESI, CodeGeneratorX86::ArrayAddress(obj, srcBegin, TIMES_1, value_offset)); |
| 1818 | |
| 1819 | // Start the loop to copy String's value to Array of Char. |
| 1820 | __ leal(EDI, Address(dst, dstBegin, ScaleFactor::TIMES_2, data_offset)); |
| 1821 | __ Bind(©_loop); |
| 1822 | __ jecxz(&done); |
| 1823 | // Use EAX temporary (convert byte from ESI to word). |
| 1824 | // TODO: Use LODSB/STOSW (not supported by X86Assembler) with AH initialized to 0. |
| 1825 | __ movzxb(EAX, Address(ESI, 0)); |
| 1826 | __ movw(Address(EDI, 0), EAX); |
| 1827 | __ leal(EDI, Address(EDI, char_size)); |
| 1828 | __ leal(ESI, Address(ESI, c_char_size)); |
| 1829 | // TODO: Add support for LOOP to X86Assembler. |
| 1830 | __ subl(ECX, Immediate(1)); |
| 1831 | __ jmp(©_loop); |
| 1832 | __ Bind(©_uncompressed); |
| 1833 | } |
| 1834 | |
| 1835 | // Do the copy for uncompressed string. |
| 1836 | // Compute the address of the destination buffer. |
| 1837 | __ leal(EDI, Address(dst, dstBegin, ScaleFactor::TIMES_2, data_offset)); |
| 1838 | __ leal(ESI, CodeGeneratorX86::ArrayAddress(obj, srcBegin, TIMES_2, value_offset)); |
Mark Mendell | 8f8926a | 2015-08-17 11:39:06 -0400 | [diff] [blame] | 1839 | __ rep_movsw(); |
| 1840 | |
jessicahandojo | 4877b79 | 2016-09-08 19:49:13 -0700 | [diff] [blame] | 1841 | __ Bind(&done); |
| 1842 | if (mirror::kUseStringCompression) { |
| 1843 | // Restore EAX. |
| 1844 | __ popl(EAX); |
| 1845 | __ cfi().AdjustCFAOffset(-stack_adjust); |
| 1846 | } |
| 1847 | // Restore ECX. |
Mark Mendell | 8f8926a | 2015-08-17 11:39:06 -0400 | [diff] [blame] | 1848 | __ popl(ECX); |
| 1849 | __ cfi().AdjustCFAOffset(-stack_adjust); |
| 1850 | } |
| 1851 | |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1852 | static void GenPeek(LocationSummary* locations, DataType::Type size, X86Assembler* assembler) { |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 1853 | Register address = locations->InAt(0).AsRegisterPairLow<Register>(); |
| 1854 | Location out_loc = locations->Out(); |
| 1855 | // x86 allows unaligned access. We do not have to check the input or use specific instructions |
| 1856 | // to avoid a SIGBUS. |
| 1857 | switch (size) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1858 | case DataType::Type::kInt8: |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 1859 | __ movsxb(out_loc.AsRegister<Register>(), Address(address, 0)); |
| 1860 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1861 | case DataType::Type::kInt16: |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 1862 | __ movsxw(out_loc.AsRegister<Register>(), Address(address, 0)); |
| 1863 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1864 | case DataType::Type::kInt32: |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 1865 | __ movl(out_loc.AsRegister<Register>(), Address(address, 0)); |
| 1866 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1867 | case DataType::Type::kInt64: |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 1868 | __ movl(out_loc.AsRegisterPairLow<Register>(), Address(address, 0)); |
| 1869 | __ movl(out_loc.AsRegisterPairHigh<Register>(), Address(address, 4)); |
| 1870 | break; |
| 1871 | default: |
| 1872 | LOG(FATAL) << "Type not recognized for peek: " << size; |
| 1873 | UNREACHABLE(); |
| 1874 | } |
| 1875 | } |
| 1876 | |
| 1877 | void IntrinsicLocationsBuilderX86::VisitMemoryPeekByte(HInvoke* invoke) { |
| 1878 | CreateLongToIntLocations(arena_, invoke); |
| 1879 | } |
| 1880 | |
| 1881 | void IntrinsicCodeGeneratorX86::VisitMemoryPeekByte(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1882 | GenPeek(invoke->GetLocations(), DataType::Type::kInt8, GetAssembler()); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 1883 | } |
| 1884 | |
| 1885 | void IntrinsicLocationsBuilderX86::VisitMemoryPeekIntNative(HInvoke* invoke) { |
| 1886 | CreateLongToIntLocations(arena_, invoke); |
| 1887 | } |
| 1888 | |
| 1889 | void IntrinsicCodeGeneratorX86::VisitMemoryPeekIntNative(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1890 | GenPeek(invoke->GetLocations(), DataType::Type::kInt32, GetAssembler()); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 1891 | } |
| 1892 | |
| 1893 | void IntrinsicLocationsBuilderX86::VisitMemoryPeekLongNative(HInvoke* invoke) { |
| 1894 | CreateLongToLongLocations(arena_, invoke); |
| 1895 | } |
| 1896 | |
| 1897 | void IntrinsicCodeGeneratorX86::VisitMemoryPeekLongNative(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1898 | GenPeek(invoke->GetLocations(), DataType::Type::kInt64, GetAssembler()); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 1899 | } |
| 1900 | |
| 1901 | void IntrinsicLocationsBuilderX86::VisitMemoryPeekShortNative(HInvoke* invoke) { |
| 1902 | CreateLongToIntLocations(arena_, invoke); |
| 1903 | } |
| 1904 | |
| 1905 | void IntrinsicCodeGeneratorX86::VisitMemoryPeekShortNative(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1906 | GenPeek(invoke->GetLocations(), DataType::Type::kInt16, GetAssembler()); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 1907 | } |
| 1908 | |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1909 | static void CreateLongIntToVoidLocations(ArenaAllocator* arena, DataType::Type size, |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 1910 | HInvoke* invoke) { |
| 1911 | LocationSummary* locations = new (arena) LocationSummary(invoke, |
| 1912 | LocationSummary::kNoCall, |
| 1913 | kIntrinsified); |
| 1914 | locations->SetInAt(0, Location::RequiresRegister()); |
Roland Levillain | 4c0eb42 | 2015-04-24 16:43:49 +0100 | [diff] [blame] | 1915 | HInstruction* value = invoke->InputAt(1); |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1916 | if (size == DataType::Type::kInt8) { |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 1917 | locations->SetInAt(1, Location::ByteRegisterOrConstant(EDX, value)); |
| 1918 | } else { |
| 1919 | locations->SetInAt(1, Location::RegisterOrConstant(value)); |
| 1920 | } |
| 1921 | } |
| 1922 | |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1923 | static void GenPoke(LocationSummary* locations, DataType::Type size, X86Assembler* assembler) { |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 1924 | Register address = locations->InAt(0).AsRegisterPairLow<Register>(); |
| 1925 | Location value_loc = locations->InAt(1); |
| 1926 | // x86 allows unaligned access. We do not have to check the input or use specific instructions |
| 1927 | // to avoid a SIGBUS. |
| 1928 | switch (size) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1929 | case DataType::Type::kInt8: |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 1930 | if (value_loc.IsConstant()) { |
| 1931 | __ movb(Address(address, 0), |
| 1932 | Immediate(value_loc.GetConstant()->AsIntConstant()->GetValue())); |
| 1933 | } else { |
| 1934 | __ movb(Address(address, 0), value_loc.AsRegister<ByteRegister>()); |
| 1935 | } |
| 1936 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1937 | case DataType::Type::kInt16: |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 1938 | if (value_loc.IsConstant()) { |
| 1939 | __ movw(Address(address, 0), |
| 1940 | Immediate(value_loc.GetConstant()->AsIntConstant()->GetValue())); |
| 1941 | } else { |
| 1942 | __ movw(Address(address, 0), value_loc.AsRegister<Register>()); |
| 1943 | } |
| 1944 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1945 | case DataType::Type::kInt32: |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 1946 | if (value_loc.IsConstant()) { |
| 1947 | __ movl(Address(address, 0), |
| 1948 | Immediate(value_loc.GetConstant()->AsIntConstant()->GetValue())); |
| 1949 | } else { |
| 1950 | __ movl(Address(address, 0), value_loc.AsRegister<Register>()); |
| 1951 | } |
| 1952 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1953 | case DataType::Type::kInt64: |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 1954 | if (value_loc.IsConstant()) { |
| 1955 | int64_t value = value_loc.GetConstant()->AsLongConstant()->GetValue(); |
| 1956 | __ movl(Address(address, 0), Immediate(Low32Bits(value))); |
| 1957 | __ movl(Address(address, 4), Immediate(High32Bits(value))); |
| 1958 | } else { |
| 1959 | __ movl(Address(address, 0), value_loc.AsRegisterPairLow<Register>()); |
| 1960 | __ movl(Address(address, 4), value_loc.AsRegisterPairHigh<Register>()); |
| 1961 | } |
| 1962 | break; |
| 1963 | default: |
| 1964 | LOG(FATAL) << "Type not recognized for poke: " << size; |
| 1965 | UNREACHABLE(); |
| 1966 | } |
| 1967 | } |
| 1968 | |
| 1969 | void IntrinsicLocationsBuilderX86::VisitMemoryPokeByte(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1970 | CreateLongIntToVoidLocations(arena_, DataType::Type::kInt8, invoke); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 1971 | } |
| 1972 | |
| 1973 | void IntrinsicCodeGeneratorX86::VisitMemoryPokeByte(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1974 | GenPoke(invoke->GetLocations(), DataType::Type::kInt8, GetAssembler()); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 1975 | } |
| 1976 | |
| 1977 | void IntrinsicLocationsBuilderX86::VisitMemoryPokeIntNative(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1978 | CreateLongIntToVoidLocations(arena_, DataType::Type::kInt32, invoke); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 1979 | } |
| 1980 | |
| 1981 | void IntrinsicCodeGeneratorX86::VisitMemoryPokeIntNative(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1982 | GenPoke(invoke->GetLocations(), DataType::Type::kInt32, GetAssembler()); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 1983 | } |
| 1984 | |
| 1985 | void IntrinsicLocationsBuilderX86::VisitMemoryPokeLongNative(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1986 | CreateLongIntToVoidLocations(arena_, DataType::Type::kInt64, invoke); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 1987 | } |
| 1988 | |
| 1989 | void IntrinsicCodeGeneratorX86::VisitMemoryPokeLongNative(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1990 | GenPoke(invoke->GetLocations(), DataType::Type::kInt64, GetAssembler()); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 1991 | } |
| 1992 | |
| 1993 | void IntrinsicLocationsBuilderX86::VisitMemoryPokeShortNative(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1994 | CreateLongIntToVoidLocations(arena_, DataType::Type::kInt16, invoke); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 1995 | } |
| 1996 | |
| 1997 | void IntrinsicCodeGeneratorX86::VisitMemoryPokeShortNative(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1998 | GenPoke(invoke->GetLocations(), DataType::Type::kInt16, GetAssembler()); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 1999 | } |
| 2000 | |
| 2001 | void IntrinsicLocationsBuilderX86::VisitThreadCurrentThread(HInvoke* invoke) { |
| 2002 | LocationSummary* locations = new (arena_) LocationSummary(invoke, |
| 2003 | LocationSummary::kNoCall, |
| 2004 | kIntrinsified); |
| 2005 | locations->SetOut(Location::RequiresRegister()); |
| 2006 | } |
| 2007 | |
| 2008 | void IntrinsicCodeGeneratorX86::VisitThreadCurrentThread(HInvoke* invoke) { |
| 2009 | Register out = invoke->GetLocations()->Out().AsRegister<Register>(); |
Andreas Gampe | 542451c | 2016-07-26 09:02:02 -0700 | [diff] [blame] | 2010 | GetAssembler()->fs()->movl(out, Address::Absolute(Thread::PeerOffset<kX86PointerSize>())); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2011 | } |
| 2012 | |
Roland Levillain | 0d5a281 | 2015-11-13 10:07:31 +0000 | [diff] [blame] | 2013 | static void GenUnsafeGet(HInvoke* invoke, |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2014 | DataType::Type type, |
Roland Levillain | 0d5a281 | 2015-11-13 10:07:31 +0000 | [diff] [blame] | 2015 | bool is_volatile, |
| 2016 | CodeGeneratorX86* codegen) { |
| 2017 | X86Assembler* assembler = down_cast<X86Assembler*>(codegen->GetAssembler()); |
| 2018 | LocationSummary* locations = invoke->GetLocations(); |
| 2019 | Location base_loc = locations->InAt(1); |
| 2020 | Register base = base_loc.AsRegister<Register>(); |
| 2021 | Location offset_loc = locations->InAt(2); |
| 2022 | Register offset = offset_loc.AsRegisterPairLow<Register>(); |
| 2023 | Location output_loc = locations->Out(); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2024 | |
| 2025 | switch (type) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2026 | case DataType::Type::kInt32: { |
Roland Levillain | 0d5a281 | 2015-11-13 10:07:31 +0000 | [diff] [blame] | 2027 | Register output = output_loc.AsRegister<Register>(); |
| 2028 | __ movl(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); |
Roland Levillain | 7c1559a | 2015-12-15 10:55:36 +0000 | [diff] [blame] | 2029 | break; |
| 2030 | } |
| 2031 | |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2032 | case DataType::Type::kReference: { |
Roland Levillain | 7c1559a | 2015-12-15 10:55:36 +0000 | [diff] [blame] | 2033 | Register output = output_loc.AsRegister<Register>(); |
| 2034 | if (kEmitCompilerReadBarrier) { |
| 2035 | if (kUseBakerReadBarrier) { |
Sang, Chunlei | 0fcd2b8 | 2016-04-05 17:12:59 +0800 | [diff] [blame] | 2036 | Address src(base, offset, ScaleFactor::TIMES_1, 0); |
| 2037 | codegen->GenerateReferenceLoadWithBakerReadBarrier( |
Vladimir Marko | 953437b | 2016-08-24 08:30:46 +0000 | [diff] [blame] | 2038 | invoke, output_loc, base, src, /* needs_null_check */ false); |
Roland Levillain | 7c1559a | 2015-12-15 10:55:36 +0000 | [diff] [blame] | 2039 | } else { |
| 2040 | __ movl(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); |
| 2041 | codegen->GenerateReadBarrierSlow( |
| 2042 | invoke, output_loc, output_loc, base_loc, 0U, offset_loc); |
| 2043 | } |
| 2044 | } else { |
| 2045 | __ movl(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); |
| 2046 | __ MaybeUnpoisonHeapReference(output); |
Roland Levillain | 4d02711 | 2015-07-01 15:41:14 +0100 | [diff] [blame] | 2047 | } |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2048 | break; |
Roland Levillain | 4d02711 | 2015-07-01 15:41:14 +0100 | [diff] [blame] | 2049 | } |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2050 | |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2051 | case DataType::Type::kInt64: { |
Roland Levillain | 0d5a281 | 2015-11-13 10:07:31 +0000 | [diff] [blame] | 2052 | Register output_lo = output_loc.AsRegisterPairLow<Register>(); |
| 2053 | Register output_hi = output_loc.AsRegisterPairHigh<Register>(); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2054 | if (is_volatile) { |
| 2055 | // Need to use a XMM to read atomically. |
| 2056 | XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); |
| 2057 | __ movsd(temp, Address(base, offset, ScaleFactor::TIMES_1, 0)); |
| 2058 | __ movd(output_lo, temp); |
| 2059 | __ psrlq(temp, Immediate(32)); |
| 2060 | __ movd(output_hi, temp); |
| 2061 | } else { |
| 2062 | __ movl(output_lo, Address(base, offset, ScaleFactor::TIMES_1, 0)); |
| 2063 | __ movl(output_hi, Address(base, offset, ScaleFactor::TIMES_1, 4)); |
| 2064 | } |
| 2065 | } |
| 2066 | break; |
| 2067 | |
| 2068 | default: |
| 2069 | LOG(FATAL) << "Unsupported op size " << type; |
| 2070 | UNREACHABLE(); |
| 2071 | } |
| 2072 | } |
| 2073 | |
Roland Levillain | 7c1559a | 2015-12-15 10:55:36 +0000 | [diff] [blame] | 2074 | static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, |
| 2075 | HInvoke* invoke, |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2076 | DataType::Type type, |
Roland Levillain | 7c1559a | 2015-12-15 10:55:36 +0000 | [diff] [blame] | 2077 | bool is_volatile) { |
Roland Levillain | 0d5a281 | 2015-11-13 10:07:31 +0000 | [diff] [blame] | 2078 | bool can_call = kEmitCompilerReadBarrier && |
| 2079 | (invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObject || |
| 2080 | invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2081 | LocationSummary* locations = new (arena) LocationSummary(invoke, |
Roland Levillain | a1aa3b1 | 2016-10-26 13:03:38 +0100 | [diff] [blame] | 2082 | (can_call |
| 2083 | ? LocationSummary::kCallOnSlowPath |
| 2084 | : LocationSummary::kNoCall), |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2085 | kIntrinsified); |
Vladimir Marko | 70e9746 | 2016-08-09 11:04:26 +0100 | [diff] [blame] | 2086 | if (can_call && kUseBakerReadBarrier) { |
Vladimir Marko | 804b03f | 2016-09-14 16:26:36 +0100 | [diff] [blame] | 2087 | locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. |
Vladimir Marko | 70e9746 | 2016-08-09 11:04:26 +0100 | [diff] [blame] | 2088 | } |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2089 | locations->SetInAt(0, Location::NoLocation()); // Unused receiver. |
| 2090 | locations->SetInAt(1, Location::RequiresRegister()); |
| 2091 | locations->SetInAt(2, Location::RequiresRegister()); |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2092 | if (type == DataType::Type::kInt64) { |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2093 | if (is_volatile) { |
| 2094 | // Need to use XMM to read volatile. |
| 2095 | locations->AddTemp(Location::RequiresFpuRegister()); |
Roland Levillain | 3d31242 | 2016-06-23 13:53:42 +0100 | [diff] [blame] | 2096 | locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2097 | } else { |
| 2098 | locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); |
| 2099 | } |
| 2100 | } else { |
Roland Levillain | 3d31242 | 2016-06-23 13:53:42 +0100 | [diff] [blame] | 2101 | locations->SetOut(Location::RequiresRegister(), |
Roland Levillain | a1aa3b1 | 2016-10-26 13:03:38 +0100 | [diff] [blame] | 2102 | (can_call ? Location::kOutputOverlap : Location::kNoOutputOverlap)); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2103 | } |
| 2104 | } |
| 2105 | |
| 2106 | void IntrinsicLocationsBuilderX86::VisitUnsafeGet(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2107 | CreateIntIntIntToIntLocations(arena_, invoke, DataType::Type::kInt32, /* is_volatile */ false); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2108 | } |
| 2109 | void IntrinsicLocationsBuilderX86::VisitUnsafeGetVolatile(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2110 | CreateIntIntIntToIntLocations(arena_, invoke, DataType::Type::kInt32, /* is_volatile */ true); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2111 | } |
| 2112 | void IntrinsicLocationsBuilderX86::VisitUnsafeGetLong(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2113 | CreateIntIntIntToIntLocations(arena_, invoke, DataType::Type::kInt64, /* is_volatile */ false); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2114 | } |
| 2115 | void IntrinsicLocationsBuilderX86::VisitUnsafeGetLongVolatile(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2116 | CreateIntIntIntToIntLocations(arena_, invoke, DataType::Type::kInt64, /* is_volatile */ true); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2117 | } |
| 2118 | void IntrinsicLocationsBuilderX86::VisitUnsafeGetObject(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2119 | CreateIntIntIntToIntLocations( |
| 2120 | arena_, invoke, DataType::Type::kReference, /* is_volatile */ false); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2121 | } |
| 2122 | void IntrinsicLocationsBuilderX86::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2123 | CreateIntIntIntToIntLocations(arena_, invoke, DataType::Type::kReference, /* is_volatile */ true); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2124 | } |
| 2125 | |
| 2126 | |
| 2127 | void IntrinsicCodeGeneratorX86::VisitUnsafeGet(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2128 | GenUnsafeGet(invoke, DataType::Type::kInt32, /* is_volatile */ false, codegen_); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2129 | } |
| 2130 | void IntrinsicCodeGeneratorX86::VisitUnsafeGetVolatile(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2131 | GenUnsafeGet(invoke, DataType::Type::kInt32, /* is_volatile */ true, codegen_); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2132 | } |
| 2133 | void IntrinsicCodeGeneratorX86::VisitUnsafeGetLong(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2134 | GenUnsafeGet(invoke, DataType::Type::kInt64, /* is_volatile */ false, codegen_); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2135 | } |
| 2136 | void IntrinsicCodeGeneratorX86::VisitUnsafeGetLongVolatile(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2137 | GenUnsafeGet(invoke, DataType::Type::kInt64, /* is_volatile */ true, codegen_); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2138 | } |
| 2139 | void IntrinsicCodeGeneratorX86::VisitUnsafeGetObject(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2140 | GenUnsafeGet(invoke, DataType::Type::kReference, /* is_volatile */ false, codegen_); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2141 | } |
| 2142 | void IntrinsicCodeGeneratorX86::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2143 | GenUnsafeGet(invoke, DataType::Type::kReference, /* is_volatile */ true, codegen_); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2144 | } |
| 2145 | |
| 2146 | |
| 2147 | static void CreateIntIntIntIntToVoidPlusTempsLocations(ArenaAllocator* arena, |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2148 | DataType::Type type, |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2149 | HInvoke* invoke, |
| 2150 | bool is_volatile) { |
| 2151 | LocationSummary* locations = new (arena) LocationSummary(invoke, |
| 2152 | LocationSummary::kNoCall, |
| 2153 | kIntrinsified); |
| 2154 | locations->SetInAt(0, Location::NoLocation()); // Unused receiver. |
| 2155 | locations->SetInAt(1, Location::RequiresRegister()); |
| 2156 | locations->SetInAt(2, Location::RequiresRegister()); |
| 2157 | locations->SetInAt(3, Location::RequiresRegister()); |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2158 | if (type == DataType::Type::kReference) { |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2159 | // Need temp registers for card-marking. |
Roland Levillain | 4d02711 | 2015-07-01 15:41:14 +0100 | [diff] [blame] | 2160 | locations->AddTemp(Location::RequiresRegister()); // Possibly used for reference poisoning too. |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2161 | // Ensure the value is in a byte register. |
| 2162 | locations->AddTemp(Location::RegisterLocation(ECX)); |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2163 | } else if (type == DataType::Type::kInt64 && is_volatile) { |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2164 | locations->AddTemp(Location::RequiresFpuRegister()); |
| 2165 | locations->AddTemp(Location::RequiresFpuRegister()); |
| 2166 | } |
| 2167 | } |
| 2168 | |
| 2169 | void IntrinsicLocationsBuilderX86::VisitUnsafePut(HInvoke* invoke) { |
Roland Levillain | bf84a3d | 2015-12-04 14:33:02 +0000 | [diff] [blame] | 2170 | CreateIntIntIntIntToVoidPlusTempsLocations( |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2171 | arena_, DataType::Type::kInt32, invoke, /* is_volatile */ false); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2172 | } |
| 2173 | void IntrinsicLocationsBuilderX86::VisitUnsafePutOrdered(HInvoke* invoke) { |
Roland Levillain | bf84a3d | 2015-12-04 14:33:02 +0000 | [diff] [blame] | 2174 | CreateIntIntIntIntToVoidPlusTempsLocations( |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2175 | arena_, DataType::Type::kInt32, invoke, /* is_volatile */ false); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2176 | } |
| 2177 | void IntrinsicLocationsBuilderX86::VisitUnsafePutVolatile(HInvoke* invoke) { |
Roland Levillain | bf84a3d | 2015-12-04 14:33:02 +0000 | [diff] [blame] | 2178 | CreateIntIntIntIntToVoidPlusTempsLocations( |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2179 | arena_, DataType::Type::kInt32, invoke, /* is_volatile */ true); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2180 | } |
| 2181 | void IntrinsicLocationsBuilderX86::VisitUnsafePutObject(HInvoke* invoke) { |
Roland Levillain | bf84a3d | 2015-12-04 14:33:02 +0000 | [diff] [blame] | 2182 | CreateIntIntIntIntToVoidPlusTempsLocations( |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2183 | arena_, DataType::Type::kReference, invoke, /* is_volatile */ false); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2184 | } |
| 2185 | void IntrinsicLocationsBuilderX86::VisitUnsafePutObjectOrdered(HInvoke* invoke) { |
Roland Levillain | bf84a3d | 2015-12-04 14:33:02 +0000 | [diff] [blame] | 2186 | CreateIntIntIntIntToVoidPlusTempsLocations( |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2187 | arena_, DataType::Type::kReference, invoke, /* is_volatile */ false); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2188 | } |
| 2189 | void IntrinsicLocationsBuilderX86::VisitUnsafePutObjectVolatile(HInvoke* invoke) { |
Roland Levillain | bf84a3d | 2015-12-04 14:33:02 +0000 | [diff] [blame] | 2190 | CreateIntIntIntIntToVoidPlusTempsLocations( |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2191 | arena_, DataType::Type::kReference, invoke, /* is_volatile */ true); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2192 | } |
| 2193 | void IntrinsicLocationsBuilderX86::VisitUnsafePutLong(HInvoke* invoke) { |
Roland Levillain | bf84a3d | 2015-12-04 14:33:02 +0000 | [diff] [blame] | 2194 | CreateIntIntIntIntToVoidPlusTempsLocations( |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2195 | arena_, DataType::Type::kInt64, invoke, /* is_volatile */ false); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2196 | } |
| 2197 | void IntrinsicLocationsBuilderX86::VisitUnsafePutLongOrdered(HInvoke* invoke) { |
Roland Levillain | bf84a3d | 2015-12-04 14:33:02 +0000 | [diff] [blame] | 2198 | CreateIntIntIntIntToVoidPlusTempsLocations( |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2199 | arena_, DataType::Type::kInt64, invoke, /* is_volatile */ false); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2200 | } |
| 2201 | void IntrinsicLocationsBuilderX86::VisitUnsafePutLongVolatile(HInvoke* invoke) { |
Roland Levillain | bf84a3d | 2015-12-04 14:33:02 +0000 | [diff] [blame] | 2202 | CreateIntIntIntIntToVoidPlusTempsLocations( |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2203 | arena_, DataType::Type::kInt64, invoke, /* is_volatile */ true); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2204 | } |
| 2205 | |
| 2206 | // We don't care for ordered: it requires an AnyStore barrier, which is already given by the x86 |
| 2207 | // memory model. |
| 2208 | static void GenUnsafePut(LocationSummary* locations, |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2209 | DataType::Type type, |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2210 | bool is_volatile, |
| 2211 | CodeGeneratorX86* codegen) { |
Roland Levillain | b488b78 | 2015-10-22 11:38:49 +0100 | [diff] [blame] | 2212 | X86Assembler* assembler = down_cast<X86Assembler*>(codegen->GetAssembler()); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2213 | Register base = locations->InAt(1).AsRegister<Register>(); |
| 2214 | Register offset = locations->InAt(2).AsRegisterPairLow<Register>(); |
| 2215 | Location value_loc = locations->InAt(3); |
| 2216 | |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2217 | if (type == DataType::Type::kInt64) { |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2218 | Register value_lo = value_loc.AsRegisterPairLow<Register>(); |
| 2219 | Register value_hi = value_loc.AsRegisterPairHigh<Register>(); |
| 2220 | if (is_volatile) { |
| 2221 | XmmRegister temp1 = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); |
| 2222 | XmmRegister temp2 = locations->GetTemp(1).AsFpuRegister<XmmRegister>(); |
| 2223 | __ movd(temp1, value_lo); |
| 2224 | __ movd(temp2, value_hi); |
| 2225 | __ punpckldq(temp1, temp2); |
| 2226 | __ movsd(Address(base, offset, ScaleFactor::TIMES_1, 0), temp1); |
| 2227 | } else { |
| 2228 | __ movl(Address(base, offset, ScaleFactor::TIMES_1, 0), value_lo); |
| 2229 | __ movl(Address(base, offset, ScaleFactor::TIMES_1, 4), value_hi); |
| 2230 | } |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2231 | } else if (kPoisonHeapReferences && type == DataType::Type::kReference) { |
Roland Levillain | 4d02711 | 2015-07-01 15:41:14 +0100 | [diff] [blame] | 2232 | Register temp = locations->GetTemp(0).AsRegister<Register>(); |
| 2233 | __ movl(temp, value_loc.AsRegister<Register>()); |
| 2234 | __ PoisonHeapReference(temp); |
| 2235 | __ movl(Address(base, offset, ScaleFactor::TIMES_1, 0), temp); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2236 | } else { |
| 2237 | __ movl(Address(base, offset, ScaleFactor::TIMES_1, 0), value_loc.AsRegister<Register>()); |
| 2238 | } |
| 2239 | |
| 2240 | if (is_volatile) { |
Mark P Mendell | 17077d8 | 2015-12-16 19:15:59 +0000 | [diff] [blame] | 2241 | codegen->MemoryFence(); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2242 | } |
| 2243 | |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2244 | if (type == DataType::Type::kReference) { |
Nicolas Geoffray | 07276db | 2015-05-18 14:22:09 +0100 | [diff] [blame] | 2245 | bool value_can_be_null = true; // TODO: Worth finding out this information? |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2246 | codegen->MarkGCCard(locations->GetTemp(0).AsRegister<Register>(), |
| 2247 | locations->GetTemp(1).AsRegister<Register>(), |
| 2248 | base, |
Nicolas Geoffray | 07276db | 2015-05-18 14:22:09 +0100 | [diff] [blame] | 2249 | value_loc.AsRegister<Register>(), |
| 2250 | value_can_be_null); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2251 | } |
| 2252 | } |
| 2253 | |
| 2254 | void IntrinsicCodeGeneratorX86::VisitUnsafePut(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2255 | GenUnsafePut(invoke->GetLocations(), DataType::Type::kInt32, /* is_volatile */ false, codegen_); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2256 | } |
| 2257 | void IntrinsicCodeGeneratorX86::VisitUnsafePutOrdered(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2258 | GenUnsafePut(invoke->GetLocations(), DataType::Type::kInt32, /* is_volatile */ false, codegen_); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2259 | } |
| 2260 | void IntrinsicCodeGeneratorX86::VisitUnsafePutVolatile(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2261 | GenUnsafePut(invoke->GetLocations(), DataType::Type::kInt32, /* is_volatile */ true, codegen_); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2262 | } |
| 2263 | void IntrinsicCodeGeneratorX86::VisitUnsafePutObject(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2264 | GenUnsafePut( |
| 2265 | invoke->GetLocations(), DataType::Type::kReference, /* is_volatile */ false, codegen_); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2266 | } |
| 2267 | void IntrinsicCodeGeneratorX86::VisitUnsafePutObjectOrdered(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2268 | GenUnsafePut( |
| 2269 | invoke->GetLocations(), DataType::Type::kReference, /* is_volatile */ false, codegen_); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2270 | } |
| 2271 | void IntrinsicCodeGeneratorX86::VisitUnsafePutObjectVolatile(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2272 | GenUnsafePut( |
| 2273 | invoke->GetLocations(), DataType::Type::kReference, /* is_volatile */ true, codegen_); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2274 | } |
| 2275 | void IntrinsicCodeGeneratorX86::VisitUnsafePutLong(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2276 | GenUnsafePut(invoke->GetLocations(), DataType::Type::kInt64, /* is_volatile */ false, codegen_); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2277 | } |
| 2278 | void IntrinsicCodeGeneratorX86::VisitUnsafePutLongOrdered(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2279 | GenUnsafePut(invoke->GetLocations(), DataType::Type::kInt64, /* is_volatile */ false, codegen_); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2280 | } |
| 2281 | void IntrinsicCodeGeneratorX86::VisitUnsafePutLongVolatile(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2282 | GenUnsafePut(invoke->GetLocations(), DataType::Type::kInt64, /* is_volatile */ true, codegen_); |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 2283 | } |
| 2284 | |
Roland Levillain | a1aa3b1 | 2016-10-26 13:03:38 +0100 | [diff] [blame] | 2285 | static void CreateIntIntIntIntIntToInt(ArenaAllocator* arena, |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2286 | DataType::Type type, |
Mark Mendell | 58d25fd | 2015-04-03 14:52:31 -0400 | [diff] [blame] | 2287 | HInvoke* invoke) { |
Roland Levillain | a1aa3b1 | 2016-10-26 13:03:38 +0100 | [diff] [blame] | 2288 | bool can_call = kEmitCompilerReadBarrier && |
| 2289 | kUseBakerReadBarrier && |
| 2290 | (invoke->GetIntrinsic() == Intrinsics::kUnsafeCASObject); |
Mark Mendell | 58d25fd | 2015-04-03 14:52:31 -0400 | [diff] [blame] | 2291 | LocationSummary* locations = new (arena) LocationSummary(invoke, |
Roland Levillain | a1aa3b1 | 2016-10-26 13:03:38 +0100 | [diff] [blame] | 2292 | (can_call |
| 2293 | ? LocationSummary::kCallOnSlowPath |
| 2294 | : LocationSummary::kNoCall), |
Mark Mendell | 58d25fd | 2015-04-03 14:52:31 -0400 | [diff] [blame] | 2295 | kIntrinsified); |
| 2296 | locations->SetInAt(0, Location::NoLocation()); // Unused receiver. |
| 2297 | locations->SetInAt(1, Location::RequiresRegister()); |
| 2298 | // Offset is a long, but in 32 bit mode, we only need the low word. |
| 2299 | // Can we update the invoke here to remove a TypeConvert to Long? |
| 2300 | locations->SetInAt(2, Location::RequiresRegister()); |
| 2301 | // Expected value must be in EAX or EDX:EAX. |
| 2302 | // For long, new value must be in ECX:EBX. |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2303 | if (type == DataType::Type::kInt64) { |
Mark Mendell | 58d25fd | 2015-04-03 14:52:31 -0400 | [diff] [blame] | 2304 | locations->SetInAt(3, Location::RegisterPairLocation(EAX, EDX)); |
| 2305 | locations->SetInAt(4, Location::RegisterPairLocation(EBX, ECX)); |
| 2306 | } else { |
| 2307 | locations->SetInAt(3, Location::RegisterLocation(EAX)); |
| 2308 | locations->SetInAt(4, Location::RequiresRegister()); |
| 2309 | } |
| 2310 | |
| 2311 | // Force a byte register for the output. |
| 2312 | locations->SetOut(Location::RegisterLocation(EAX)); |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2313 | if (type == DataType::Type::kReference) { |
Roland Levillain | a1aa3b1 | 2016-10-26 13:03:38 +0100 | [diff] [blame] | 2314 | // Need temporary registers for card-marking, and possibly for |
| 2315 | // (Baker) read barrier. |
Roland Levillain | b488b78 | 2015-10-22 11:38:49 +0100 | [diff] [blame] | 2316 | locations->AddTemp(Location::RequiresRegister()); // Possibly used for reference poisoning too. |
Mark Mendell | 58d25fd | 2015-04-03 14:52:31 -0400 | [diff] [blame] | 2317 | // Need a byte register for marking. |
| 2318 | locations->AddTemp(Location::RegisterLocation(ECX)); |
| 2319 | } |
| 2320 | } |
| 2321 | |
| 2322 | void IntrinsicLocationsBuilderX86::VisitUnsafeCASInt(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2323 | CreateIntIntIntIntIntToInt(arena_, DataType::Type::kInt32, invoke); |
Mark Mendell | 58d25fd | 2015-04-03 14:52:31 -0400 | [diff] [blame] | 2324 | } |
| 2325 | |
| 2326 | void IntrinsicLocationsBuilderX86::VisitUnsafeCASLong(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2327 | CreateIntIntIntIntIntToInt(arena_, DataType::Type::kInt64, invoke); |
Mark Mendell | 58d25fd | 2015-04-03 14:52:31 -0400 | [diff] [blame] | 2328 | } |
| 2329 | |
| 2330 | void IntrinsicLocationsBuilderX86::VisitUnsafeCASObject(HInvoke* invoke) { |
Roland Levillain | a1aa3b1 | 2016-10-26 13:03:38 +0100 | [diff] [blame] | 2331 | // The only read barrier implementation supporting the |
| 2332 | // UnsafeCASObject intrinsic is the Baker-style read barriers. |
| 2333 | if (kEmitCompilerReadBarrier && !kUseBakerReadBarrier) { |
Roland Levillain | 391b866 | 2015-12-18 11:43:38 +0000 | [diff] [blame] | 2334 | return; |
| 2335 | } |
| 2336 | |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2337 | CreateIntIntIntIntIntToInt(arena_, DataType::Type::kReference, invoke); |
Mark Mendell | 58d25fd | 2015-04-03 14:52:31 -0400 | [diff] [blame] | 2338 | } |
| 2339 | |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2340 | static void GenCAS(DataType::Type type, HInvoke* invoke, CodeGeneratorX86* codegen) { |
Roland Levillain | b488b78 | 2015-10-22 11:38:49 +0100 | [diff] [blame] | 2341 | X86Assembler* assembler = down_cast<X86Assembler*>(codegen->GetAssembler()); |
Mark Mendell | 58d25fd | 2015-04-03 14:52:31 -0400 | [diff] [blame] | 2342 | LocationSummary* locations = invoke->GetLocations(); |
| 2343 | |
| 2344 | Register base = locations->InAt(1).AsRegister<Register>(); |
| 2345 | Register offset = locations->InAt(2).AsRegisterPairLow<Register>(); |
| 2346 | Location out = locations->Out(); |
| 2347 | DCHECK_EQ(out.AsRegister<Register>(), EAX); |
| 2348 | |
Roland Levillain | a1aa3b1 | 2016-10-26 13:03:38 +0100 | [diff] [blame] | 2349 | // The address of the field within the holding object. |
| 2350 | Address field_addr(base, offset, ScaleFactor::TIMES_1, 0); |
| 2351 | |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2352 | if (type == DataType::Type::kReference) { |
Roland Levillain | a1aa3b1 | 2016-10-26 13:03:38 +0100 | [diff] [blame] | 2353 | // The only read barrier implementation supporting the |
| 2354 | // UnsafeCASObject intrinsic is the Baker-style read barriers. |
| 2355 | DCHECK(!kEmitCompilerReadBarrier || kUseBakerReadBarrier); |
| 2356 | |
| 2357 | Location temp1_loc = locations->GetTemp(0); |
| 2358 | Register temp1 = temp1_loc.AsRegister<Register>(); |
| 2359 | Register temp2 = locations->GetTemp(1).AsRegister<Register>(); |
| 2360 | |
Roland Levillain | 4d02711 | 2015-07-01 15:41:14 +0100 | [diff] [blame] | 2361 | Register expected = locations->InAt(3).AsRegister<Register>(); |
Roland Levillain | b488b78 | 2015-10-22 11:38:49 +0100 | [diff] [blame] | 2362 | // Ensure `expected` is in EAX (required by the CMPXCHG instruction). |
Roland Levillain | 4d02711 | 2015-07-01 15:41:14 +0100 | [diff] [blame] | 2363 | DCHECK_EQ(expected, EAX); |
Mark Mendell | 58d25fd | 2015-04-03 14:52:31 -0400 | [diff] [blame] | 2364 | Register value = locations->InAt(4).AsRegister<Register>(); |
Roland Levillain | 4d02711 | 2015-07-01 15:41:14 +0100 | [diff] [blame] | 2365 | |
Roland Levillain | b488b78 | 2015-10-22 11:38:49 +0100 | [diff] [blame] | 2366 | // Mark card for object assuming new value is stored. |
| 2367 | bool value_can_be_null = true; // TODO: Worth finding out this information? |
Roland Levillain | a1aa3b1 | 2016-10-26 13:03:38 +0100 | [diff] [blame] | 2368 | codegen->MarkGCCard(temp1, temp2, base, value, value_can_be_null); |
| 2369 | |
| 2370 | if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { |
| 2371 | // Need to make sure the reference stored in the field is a to-space |
| 2372 | // one before attempting the CAS or the CAS could fail incorrectly. |
| 2373 | codegen->GenerateReferenceLoadWithBakerReadBarrier( |
| 2374 | invoke, |
| 2375 | temp1_loc, // Unused, used only as a "temporary" within the read barrier. |
| 2376 | base, |
| 2377 | field_addr, |
| 2378 | /* needs_null_check */ false, |
| 2379 | /* always_update_field */ true, |
| 2380 | &temp2); |
| 2381 | } |
Roland Levillain | b488b78 | 2015-10-22 11:38:49 +0100 | [diff] [blame] | 2382 | |
| 2383 | bool base_equals_value = (base == value); |
| 2384 | if (kPoisonHeapReferences) { |
| 2385 | if (base_equals_value) { |
| 2386 | // If `base` and `value` are the same register location, move |
| 2387 | // `value` to a temporary register. This way, poisoning |
| 2388 | // `value` won't invalidate `base`. |
Roland Levillain | a1aa3b1 | 2016-10-26 13:03:38 +0100 | [diff] [blame] | 2389 | value = temp1; |
Roland Levillain | b488b78 | 2015-10-22 11:38:49 +0100 | [diff] [blame] | 2390 | __ movl(value, base); |
Roland Levillain | 4d02711 | 2015-07-01 15:41:14 +0100 | [diff] [blame] | 2391 | } |
Roland Levillain | b488b78 | 2015-10-22 11:38:49 +0100 | [diff] [blame] | 2392 | |
| 2393 | // Check that the register allocator did not assign the location |
| 2394 | // of `expected` (EAX) to `value` nor to `base`, so that heap |
| 2395 | // poisoning (when enabled) works as intended below. |
| 2396 | // - If `value` were equal to `expected`, both references would |
| 2397 | // be poisoned twice, meaning they would not be poisoned at |
| 2398 | // all, as heap poisoning uses address negation. |
| 2399 | // - If `base` were equal to `expected`, poisoning `expected` |
| 2400 | // would invalidate `base`. |
| 2401 | DCHECK_NE(value, expected); |
| 2402 | DCHECK_NE(base, expected); |
| 2403 | |
| 2404 | __ PoisonHeapReference(expected); |
| 2405 | __ PoisonHeapReference(value); |
Mark Mendell | 58d25fd | 2015-04-03 14:52:31 -0400 | [diff] [blame] | 2406 | } |
| 2407 | |
Roland Levillain | a1aa3b1 | 2016-10-26 13:03:38 +0100 | [diff] [blame] | 2408 | __ LockCmpxchgl(field_addr, value); |
Mark Mendell | 58d25fd | 2015-04-03 14:52:31 -0400 | [diff] [blame] | 2409 | |
Roland Levillain | 0d5a281 | 2015-11-13 10:07:31 +0000 | [diff] [blame] | 2410 | // LOCK CMPXCHG has full barrier semantics, and we don't need |
Roland Levillain | b488b78 | 2015-10-22 11:38:49 +0100 | [diff] [blame] | 2411 | // scheduling barriers at this time. |
Mark Mendell | 58d25fd | 2015-04-03 14:52:31 -0400 | [diff] [blame] | 2412 | |
Roland Levillain | a1aa3b1 | 2016-10-26 13:03:38 +0100 | [diff] [blame] | 2413 | // Convert ZF into the Boolean result. |
Roland Levillain | b488b78 | 2015-10-22 11:38:49 +0100 | [diff] [blame] | 2414 | __ setb(kZero, out.AsRegister<Register>()); |
| 2415 | __ movzxb(out.AsRegister<Register>(), out.AsRegister<ByteRegister>()); |
Roland Levillain | 4d02711 | 2015-07-01 15:41:14 +0100 | [diff] [blame] | 2416 | |
Roland Levillain | 391b866 | 2015-12-18 11:43:38 +0000 | [diff] [blame] | 2417 | // If heap poisoning is enabled, we need to unpoison the values |
| 2418 | // that were poisoned earlier. |
Roland Levillain | b488b78 | 2015-10-22 11:38:49 +0100 | [diff] [blame] | 2419 | if (kPoisonHeapReferences) { |
| 2420 | if (base_equals_value) { |
| 2421 | // `value` has been moved to a temporary register, no need to |
| 2422 | // unpoison it. |
| 2423 | } else { |
| 2424 | // Ensure `value` is different from `out`, so that unpoisoning |
| 2425 | // the former does not invalidate the latter. |
| 2426 | DCHECK_NE(value, out.AsRegister<Register>()); |
| 2427 | __ UnpoisonHeapReference(value); |
| 2428 | } |
| 2429 | // Do not unpoison the reference contained in register |
| 2430 | // `expected`, as it is the same as register `out` (EAX). |
| 2431 | } |
| 2432 | } else { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2433 | if (type == DataType::Type::kInt32) { |
Roland Levillain | b488b78 | 2015-10-22 11:38:49 +0100 | [diff] [blame] | 2434 | // Ensure the expected value is in EAX (required by the CMPXCHG |
| 2435 | // instruction). |
| 2436 | DCHECK_EQ(locations->InAt(3).AsRegister<Register>(), EAX); |
Roland Levillain | a1aa3b1 | 2016-10-26 13:03:38 +0100 | [diff] [blame] | 2437 | __ LockCmpxchgl(field_addr, locations->InAt(4).AsRegister<Register>()); |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2438 | } else if (type == DataType::Type::kInt64) { |
Roland Levillain | b488b78 | 2015-10-22 11:38:49 +0100 | [diff] [blame] | 2439 | // Ensure the expected value is in EAX:EDX and that the new |
| 2440 | // value is in EBX:ECX (required by the CMPXCHG8B instruction). |
| 2441 | DCHECK_EQ(locations->InAt(3).AsRegisterPairLow<Register>(), EAX); |
| 2442 | DCHECK_EQ(locations->InAt(3).AsRegisterPairHigh<Register>(), EDX); |
| 2443 | DCHECK_EQ(locations->InAt(4).AsRegisterPairLow<Register>(), EBX); |
| 2444 | DCHECK_EQ(locations->InAt(4).AsRegisterPairHigh<Register>(), ECX); |
Roland Levillain | a1aa3b1 | 2016-10-26 13:03:38 +0100 | [diff] [blame] | 2445 | __ LockCmpxchg8b(field_addr); |
Roland Levillain | b488b78 | 2015-10-22 11:38:49 +0100 | [diff] [blame] | 2446 | } else { |
| 2447 | LOG(FATAL) << "Unexpected CAS type " << type; |
| 2448 | } |
| 2449 | |
Roland Levillain | 0d5a281 | 2015-11-13 10:07:31 +0000 | [diff] [blame] | 2450 | // LOCK CMPXCHG/LOCK CMPXCHG8B have full barrier semantics, and we |
| 2451 | // don't need scheduling barriers at this time. |
Roland Levillain | b488b78 | 2015-10-22 11:38:49 +0100 | [diff] [blame] | 2452 | |
Roland Levillain | a1aa3b1 | 2016-10-26 13:03:38 +0100 | [diff] [blame] | 2453 | // Convert ZF into the Boolean result. |
Roland Levillain | b488b78 | 2015-10-22 11:38:49 +0100 | [diff] [blame] | 2454 | __ setb(kZero, out.AsRegister<Register>()); |
| 2455 | __ movzxb(out.AsRegister<Register>(), out.AsRegister<ByteRegister>()); |
Roland Levillain | 4d02711 | 2015-07-01 15:41:14 +0100 | [diff] [blame] | 2456 | } |
Mark Mendell | 58d25fd | 2015-04-03 14:52:31 -0400 | [diff] [blame] | 2457 | } |
| 2458 | |
| 2459 | void IntrinsicCodeGeneratorX86::VisitUnsafeCASInt(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2460 | GenCAS(DataType::Type::kInt32, invoke, codegen_); |
Mark Mendell | 58d25fd | 2015-04-03 14:52:31 -0400 | [diff] [blame] | 2461 | } |
| 2462 | |
| 2463 | void IntrinsicCodeGeneratorX86::VisitUnsafeCASLong(HInvoke* invoke) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2464 | GenCAS(DataType::Type::kInt64, invoke, codegen_); |
Mark Mendell | 58d25fd | 2015-04-03 14:52:31 -0400 | [diff] [blame] | 2465 | } |
| 2466 | |
| 2467 | void IntrinsicCodeGeneratorX86::VisitUnsafeCASObject(HInvoke* invoke) { |
Roland Levillain | a1aa3b1 | 2016-10-26 13:03:38 +0100 | [diff] [blame] | 2468 | // The only read barrier implementation supporting the |
| 2469 | // UnsafeCASObject intrinsic is the Baker-style read barriers. |
| 2470 | DCHECK(!kEmitCompilerReadBarrier || kUseBakerReadBarrier); |
Roland Levillain | 3d31242 | 2016-06-23 13:53:42 +0100 | [diff] [blame] | 2471 | |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2472 | GenCAS(DataType::Type::kReference, invoke, codegen_); |
Mark Mendell | 58d25fd | 2015-04-03 14:52:31 -0400 | [diff] [blame] | 2473 | } |
| 2474 | |
| 2475 | void IntrinsicLocationsBuilderX86::VisitIntegerReverse(HInvoke* invoke) { |
| 2476 | LocationSummary* locations = new (arena_) LocationSummary(invoke, |
| 2477 | LocationSummary::kNoCall, |
| 2478 | kIntrinsified); |
| 2479 | locations->SetInAt(0, Location::RequiresRegister()); |
| 2480 | locations->SetOut(Location::SameAsFirstInput()); |
| 2481 | locations->AddTemp(Location::RequiresRegister()); |
| 2482 | } |
| 2483 | |
| 2484 | static void SwapBits(Register reg, Register temp, int32_t shift, int32_t mask, |
| 2485 | X86Assembler* assembler) { |
| 2486 | Immediate imm_shift(shift); |
| 2487 | Immediate imm_mask(mask); |
| 2488 | __ movl(temp, reg); |
| 2489 | __ shrl(reg, imm_shift); |
| 2490 | __ andl(temp, imm_mask); |
| 2491 | __ andl(reg, imm_mask); |
| 2492 | __ shll(temp, imm_shift); |
| 2493 | __ orl(reg, temp); |
| 2494 | } |
| 2495 | |
| 2496 | void IntrinsicCodeGeneratorX86::VisitIntegerReverse(HInvoke* invoke) { |
Aart Bik | a19616e | 2016-02-01 18:57:58 -0800 | [diff] [blame] | 2497 | X86Assembler* assembler = GetAssembler(); |
Mark Mendell | 58d25fd | 2015-04-03 14:52:31 -0400 | [diff] [blame] | 2498 | LocationSummary* locations = invoke->GetLocations(); |
| 2499 | |
| 2500 | Register reg = locations->InAt(0).AsRegister<Register>(); |
| 2501 | Register temp = locations->GetTemp(0).AsRegister<Register>(); |
| 2502 | |
| 2503 | /* |
| 2504 | * Use one bswap instruction to reverse byte order first and then use 3 rounds of |
| 2505 | * swapping bits to reverse bits in a number x. Using bswap to save instructions |
| 2506 | * compared to generic luni implementation which has 5 rounds of swapping bits. |
| 2507 | * x = bswap x |
| 2508 | * x = (x & 0x55555555) << 1 | (x >> 1) & 0x55555555; |
| 2509 | * x = (x & 0x33333333) << 2 | (x >> 2) & 0x33333333; |
| 2510 | * x = (x & 0x0F0F0F0F) << 4 | (x >> 4) & 0x0F0F0F0F; |
| 2511 | */ |
| 2512 | __ bswapl(reg); |
| 2513 | SwapBits(reg, temp, 1, 0x55555555, assembler); |
| 2514 | SwapBits(reg, temp, 2, 0x33333333, assembler); |
| 2515 | SwapBits(reg, temp, 4, 0x0f0f0f0f, assembler); |
| 2516 | } |
| 2517 | |
| 2518 | void IntrinsicLocationsBuilderX86::VisitLongReverse(HInvoke* invoke) { |
| 2519 | LocationSummary* locations = new (arena_) LocationSummary(invoke, |
| 2520 | LocationSummary::kNoCall, |
| 2521 | kIntrinsified); |
| 2522 | locations->SetInAt(0, Location::RequiresRegister()); |
| 2523 | locations->SetOut(Location::SameAsFirstInput()); |
| 2524 | locations->AddTemp(Location::RequiresRegister()); |
| 2525 | } |
| 2526 | |
| 2527 | void IntrinsicCodeGeneratorX86::VisitLongReverse(HInvoke* invoke) { |
Aart Bik | a19616e | 2016-02-01 18:57:58 -0800 | [diff] [blame] | 2528 | X86Assembler* assembler = GetAssembler(); |
Mark Mendell | 58d25fd | 2015-04-03 14:52:31 -0400 | [diff] [blame] | 2529 | LocationSummary* locations = invoke->GetLocations(); |
| 2530 | |
| 2531 | Register reg_low = locations->InAt(0).AsRegisterPairLow<Register>(); |
| 2532 | Register reg_high = locations->InAt(0).AsRegisterPairHigh<Register>(); |
| 2533 | Register temp = locations->GetTemp(0).AsRegister<Register>(); |
| 2534 | |
| 2535 | // We want to swap high/low, then bswap each one, and then do the same |
| 2536 | // as a 32 bit reverse. |
| 2537 | // Exchange high and low. |
| 2538 | __ movl(temp, reg_low); |
| 2539 | __ movl(reg_low, reg_high); |
| 2540 | __ movl(reg_high, temp); |
| 2541 | |
| 2542 | // bit-reverse low |
| 2543 | __ bswapl(reg_low); |
| 2544 | SwapBits(reg_low, temp, 1, 0x55555555, assembler); |
| 2545 | SwapBits(reg_low, temp, 2, 0x33333333, assembler); |
| 2546 | SwapBits(reg_low, temp, 4, 0x0f0f0f0f, assembler); |
| 2547 | |
| 2548 | // bit-reverse high |
| 2549 | __ bswapl(reg_high); |
| 2550 | SwapBits(reg_high, temp, 1, 0x55555555, assembler); |
| 2551 | SwapBits(reg_high, temp, 2, 0x33333333, assembler); |
| 2552 | SwapBits(reg_high, temp, 4, 0x0f0f0f0f, assembler); |
| 2553 | } |
| 2554 | |
Aart Bik | c39dac1 | 2016-01-21 08:59:48 -0800 | [diff] [blame] | 2555 | static void CreateBitCountLocations( |
| 2556 | ArenaAllocator* arena, CodeGeneratorX86* codegen, HInvoke* invoke, bool is_long) { |
| 2557 | if (!codegen->GetInstructionSetFeatures().HasPopCnt()) { |
| 2558 | // Do nothing if there is no popcnt support. This results in generating |
| 2559 | // a call for the intrinsic rather than direct code. |
| 2560 | return; |
| 2561 | } |
| 2562 | LocationSummary* locations = new (arena) LocationSummary(invoke, |
| 2563 | LocationSummary::kNoCall, |
| 2564 | kIntrinsified); |
| 2565 | if (is_long) { |
Aart Bik | c39dac1 | 2016-01-21 08:59:48 -0800 | [diff] [blame] | 2566 | locations->AddTemp(Location::RequiresRegister()); |
Aart Bik | c39dac1 | 2016-01-21 08:59:48 -0800 | [diff] [blame] | 2567 | } |
Aart Bik | 2a94607 | 2016-01-21 12:49:00 -0800 | [diff] [blame] | 2568 | locations->SetInAt(0, Location::Any()); |
Aart Bik | c39dac1 | 2016-01-21 08:59:48 -0800 | [diff] [blame] | 2569 | locations->SetOut(Location::RequiresRegister()); |
| 2570 | } |
| 2571 | |
Aart Bik | a19616e | 2016-02-01 18:57:58 -0800 | [diff] [blame] | 2572 | static void GenBitCount(X86Assembler* assembler, |
| 2573 | CodeGeneratorX86* codegen, |
| 2574 | HInvoke* invoke, bool is_long) { |
Aart Bik | c39dac1 | 2016-01-21 08:59:48 -0800 | [diff] [blame] | 2575 | LocationSummary* locations = invoke->GetLocations(); |
| 2576 | Location src = locations->InAt(0); |
| 2577 | Register out = locations->Out().AsRegister<Register>(); |
| 2578 | |
| 2579 | if (invoke->InputAt(0)->IsConstant()) { |
| 2580 | // Evaluate this at compile time. |
| 2581 | int64_t value = Int64FromConstant(invoke->InputAt(0)->AsConstant()); |
Roland Levillain | fa3912e | 2016-04-01 18:21:55 +0100 | [diff] [blame] | 2582 | int32_t result = is_long |
Aart Bik | c39dac1 | 2016-01-21 08:59:48 -0800 | [diff] [blame] | 2583 | ? POPCOUNT(static_cast<uint64_t>(value)) |
| 2584 | : POPCOUNT(static_cast<uint32_t>(value)); |
Roland Levillain | fa3912e | 2016-04-01 18:21:55 +0100 | [diff] [blame] | 2585 | codegen->Load32BitValue(out, result); |
Aart Bik | c39dac1 | 2016-01-21 08:59:48 -0800 | [diff] [blame] | 2586 | return; |
| 2587 | } |
| 2588 | |
| 2589 | // Handle the non-constant cases. |
| 2590 | if (!is_long) { |
| 2591 | if (src.IsRegister()) { |
| 2592 | __ popcntl(out, src.AsRegister<Register>()); |
| 2593 | } else { |
| 2594 | DCHECK(src.IsStackSlot()); |
| 2595 | __ popcntl(out, Address(ESP, src.GetStackIndex())); |
| 2596 | } |
Aart Bik | 2a94607 | 2016-01-21 12:49:00 -0800 | [diff] [blame] | 2597 | } else { |
| 2598 | // The 64-bit case needs to worry about two parts. |
| 2599 | Register temp = locations->GetTemp(0).AsRegister<Register>(); |
| 2600 | if (src.IsRegisterPair()) { |
| 2601 | __ popcntl(temp, src.AsRegisterPairLow<Register>()); |
| 2602 | __ popcntl(out, src.AsRegisterPairHigh<Register>()); |
| 2603 | } else { |
| 2604 | DCHECK(src.IsDoubleStackSlot()); |
| 2605 | __ popcntl(temp, Address(ESP, src.GetStackIndex())); |
| 2606 | __ popcntl(out, Address(ESP, src.GetHighStackIndex(kX86WordSize))); |
| 2607 | } |
| 2608 | __ addl(out, temp); |
Aart Bik | c39dac1 | 2016-01-21 08:59:48 -0800 | [diff] [blame] | 2609 | } |
Aart Bik | c39dac1 | 2016-01-21 08:59:48 -0800 | [diff] [blame] | 2610 | } |
| 2611 | |
| 2612 | void IntrinsicLocationsBuilderX86::VisitIntegerBitCount(HInvoke* invoke) { |
| 2613 | CreateBitCountLocations(arena_, codegen_, invoke, /* is_long */ false); |
| 2614 | } |
| 2615 | |
| 2616 | void IntrinsicCodeGeneratorX86::VisitIntegerBitCount(HInvoke* invoke) { |
Aart Bik | a19616e | 2016-02-01 18:57:58 -0800 | [diff] [blame] | 2617 | GenBitCount(GetAssembler(), codegen_, invoke, /* is_long */ false); |
Aart Bik | c39dac1 | 2016-01-21 08:59:48 -0800 | [diff] [blame] | 2618 | } |
| 2619 | |
| 2620 | void IntrinsicLocationsBuilderX86::VisitLongBitCount(HInvoke* invoke) { |
| 2621 | CreateBitCountLocations(arena_, codegen_, invoke, /* is_long */ true); |
| 2622 | } |
| 2623 | |
| 2624 | void IntrinsicCodeGeneratorX86::VisitLongBitCount(HInvoke* invoke) { |
Aart Bik | a19616e | 2016-02-01 18:57:58 -0800 | [diff] [blame] | 2625 | GenBitCount(GetAssembler(), codegen_, invoke, /* is_long */ true); |
Aart Bik | c39dac1 | 2016-01-21 08:59:48 -0800 | [diff] [blame] | 2626 | } |
| 2627 | |
Mark Mendell | d589767 | 2015-08-12 21:16:41 -0400 | [diff] [blame] | 2628 | static void CreateLeadingZeroLocations(ArenaAllocator* arena, HInvoke* invoke, bool is_long) { |
| 2629 | LocationSummary* locations = new (arena) LocationSummary(invoke, |
| 2630 | LocationSummary::kNoCall, |
| 2631 | kIntrinsified); |
| 2632 | if (is_long) { |
| 2633 | locations->SetInAt(0, Location::RequiresRegister()); |
| 2634 | } else { |
| 2635 | locations->SetInAt(0, Location::Any()); |
| 2636 | } |
| 2637 | locations->SetOut(Location::RequiresRegister()); |
| 2638 | } |
| 2639 | |
Aart Bik | a19616e | 2016-02-01 18:57:58 -0800 | [diff] [blame] | 2640 | static void GenLeadingZeros(X86Assembler* assembler, |
| 2641 | CodeGeneratorX86* codegen, |
| 2642 | HInvoke* invoke, bool is_long) { |
Mark Mendell | d589767 | 2015-08-12 21:16:41 -0400 | [diff] [blame] | 2643 | LocationSummary* locations = invoke->GetLocations(); |
| 2644 | Location src = locations->InAt(0); |
| 2645 | Register out = locations->Out().AsRegister<Register>(); |
| 2646 | |
| 2647 | if (invoke->InputAt(0)->IsConstant()) { |
| 2648 | // Evaluate this at compile time. |
| 2649 | int64_t value = Int64FromConstant(invoke->InputAt(0)->AsConstant()); |
| 2650 | if (value == 0) { |
| 2651 | value = is_long ? 64 : 32; |
| 2652 | } else { |
| 2653 | value = is_long ? CLZ(static_cast<uint64_t>(value)) : CLZ(static_cast<uint32_t>(value)); |
| 2654 | } |
Aart Bik | a19616e | 2016-02-01 18:57:58 -0800 | [diff] [blame] | 2655 | codegen->Load32BitValue(out, value); |
Mark Mendell | d589767 | 2015-08-12 21:16:41 -0400 | [diff] [blame] | 2656 | return; |
| 2657 | } |
| 2658 | |
| 2659 | // Handle the non-constant cases. |
| 2660 | if (!is_long) { |
| 2661 | if (src.IsRegister()) { |
| 2662 | __ bsrl(out, src.AsRegister<Register>()); |
| 2663 | } else { |
| 2664 | DCHECK(src.IsStackSlot()); |
| 2665 | __ bsrl(out, Address(ESP, src.GetStackIndex())); |
| 2666 | } |
| 2667 | |
| 2668 | // BSR sets ZF if the input was zero, and the output is undefined. |
Mark Mendell | 0c9497d | 2015-08-21 09:30:05 -0400 | [diff] [blame] | 2669 | NearLabel all_zeroes, done; |
Mark Mendell | d589767 | 2015-08-12 21:16:41 -0400 | [diff] [blame] | 2670 | __ j(kEqual, &all_zeroes); |
| 2671 | |
| 2672 | // Correct the result from BSR to get the final CLZ result. |
| 2673 | __ xorl(out, Immediate(31)); |
| 2674 | __ jmp(&done); |
| 2675 | |
| 2676 | // Fix the zero case with the expected result. |
| 2677 | __ Bind(&all_zeroes); |
| 2678 | __ movl(out, Immediate(32)); |
| 2679 | |
| 2680 | __ Bind(&done); |
| 2681 | return; |
| 2682 | } |
| 2683 | |
| 2684 | // 64 bit case needs to worry about both parts of the register. |
| 2685 | DCHECK(src.IsRegisterPair()); |
| 2686 | Register src_lo = src.AsRegisterPairLow<Register>(); |
| 2687 | Register src_hi = src.AsRegisterPairHigh<Register>(); |
Mark Mendell | 0c9497d | 2015-08-21 09:30:05 -0400 | [diff] [blame] | 2688 | NearLabel handle_low, done, all_zeroes; |
Mark Mendell | d589767 | 2015-08-12 21:16:41 -0400 | [diff] [blame] | 2689 | |
| 2690 | // Is the high word zero? |
| 2691 | __ testl(src_hi, src_hi); |
| 2692 | __ j(kEqual, &handle_low); |
| 2693 | |
| 2694 | // High word is not zero. We know that the BSR result is defined in this case. |
| 2695 | __ bsrl(out, src_hi); |
| 2696 | |
| 2697 | // Correct the result from BSR to get the final CLZ result. |
| 2698 | __ xorl(out, Immediate(31)); |
| 2699 | __ jmp(&done); |
| 2700 | |
| 2701 | // High word was zero. We have to compute the low word count and add 32. |
| 2702 | __ Bind(&handle_low); |
| 2703 | __ bsrl(out, src_lo); |
| 2704 | __ j(kEqual, &all_zeroes); |
| 2705 | |
| 2706 | // We had a valid result. Use an XOR to both correct the result and add 32. |
| 2707 | __ xorl(out, Immediate(63)); |
| 2708 | __ jmp(&done); |
| 2709 | |
| 2710 | // All zero case. |
| 2711 | __ Bind(&all_zeroes); |
| 2712 | __ movl(out, Immediate(64)); |
| 2713 | |
| 2714 | __ Bind(&done); |
| 2715 | } |
| 2716 | |
| 2717 | void IntrinsicLocationsBuilderX86::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) { |
| 2718 | CreateLeadingZeroLocations(arena_, invoke, /* is_long */ false); |
| 2719 | } |
| 2720 | |
| 2721 | void IntrinsicCodeGeneratorX86::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) { |
Aart Bik | a19616e | 2016-02-01 18:57:58 -0800 | [diff] [blame] | 2722 | GenLeadingZeros(GetAssembler(), codegen_, invoke, /* is_long */ false); |
Mark Mendell | d589767 | 2015-08-12 21:16:41 -0400 | [diff] [blame] | 2723 | } |
| 2724 | |
| 2725 | void IntrinsicLocationsBuilderX86::VisitLongNumberOfLeadingZeros(HInvoke* invoke) { |
| 2726 | CreateLeadingZeroLocations(arena_, invoke, /* is_long */ true); |
| 2727 | } |
| 2728 | |
| 2729 | void IntrinsicCodeGeneratorX86::VisitLongNumberOfLeadingZeros(HInvoke* invoke) { |
Aart Bik | a19616e | 2016-02-01 18:57:58 -0800 | [diff] [blame] | 2730 | GenLeadingZeros(GetAssembler(), codegen_, invoke, /* is_long */ true); |
Mark Mendell | d589767 | 2015-08-12 21:16:41 -0400 | [diff] [blame] | 2731 | } |
| 2732 | |
Mark Mendell | 2d55479 | 2015-09-15 21:45:18 -0400 | [diff] [blame] | 2733 | static void CreateTrailingZeroLocations(ArenaAllocator* arena, HInvoke* invoke, bool is_long) { |
| 2734 | LocationSummary* locations = new (arena) LocationSummary(invoke, |
| 2735 | LocationSummary::kNoCall, |
| 2736 | kIntrinsified); |
| 2737 | if (is_long) { |
| 2738 | locations->SetInAt(0, Location::RequiresRegister()); |
| 2739 | } else { |
| 2740 | locations->SetInAt(0, Location::Any()); |
| 2741 | } |
| 2742 | locations->SetOut(Location::RequiresRegister()); |
| 2743 | } |
| 2744 | |
Aart Bik | a19616e | 2016-02-01 18:57:58 -0800 | [diff] [blame] | 2745 | static void GenTrailingZeros(X86Assembler* assembler, |
| 2746 | CodeGeneratorX86* codegen, |
| 2747 | HInvoke* invoke, bool is_long) { |
Mark Mendell | 2d55479 | 2015-09-15 21:45:18 -0400 | [diff] [blame] | 2748 | LocationSummary* locations = invoke->GetLocations(); |
| 2749 | Location src = locations->InAt(0); |
| 2750 | Register out = locations->Out().AsRegister<Register>(); |
| 2751 | |
| 2752 | if (invoke->InputAt(0)->IsConstant()) { |
| 2753 | // Evaluate this at compile time. |
| 2754 | int64_t value = Int64FromConstant(invoke->InputAt(0)->AsConstant()); |
| 2755 | if (value == 0) { |
| 2756 | value = is_long ? 64 : 32; |
| 2757 | } else { |
| 2758 | value = is_long ? CTZ(static_cast<uint64_t>(value)) : CTZ(static_cast<uint32_t>(value)); |
| 2759 | } |
Aart Bik | a19616e | 2016-02-01 18:57:58 -0800 | [diff] [blame] | 2760 | codegen->Load32BitValue(out, value); |
Mark Mendell | 2d55479 | 2015-09-15 21:45:18 -0400 | [diff] [blame] | 2761 | return; |
| 2762 | } |
| 2763 | |
| 2764 | // Handle the non-constant cases. |
| 2765 | if (!is_long) { |
| 2766 | if (src.IsRegister()) { |
| 2767 | __ bsfl(out, src.AsRegister<Register>()); |
| 2768 | } else { |
| 2769 | DCHECK(src.IsStackSlot()); |
| 2770 | __ bsfl(out, Address(ESP, src.GetStackIndex())); |
| 2771 | } |
| 2772 | |
| 2773 | // BSF sets ZF if the input was zero, and the output is undefined. |
| 2774 | NearLabel done; |
| 2775 | __ j(kNotEqual, &done); |
| 2776 | |
| 2777 | // Fix the zero case with the expected result. |
| 2778 | __ movl(out, Immediate(32)); |
| 2779 | |
| 2780 | __ Bind(&done); |
| 2781 | return; |
| 2782 | } |
| 2783 | |
| 2784 | // 64 bit case needs to worry about both parts of the register. |
| 2785 | DCHECK(src.IsRegisterPair()); |
| 2786 | Register src_lo = src.AsRegisterPairLow<Register>(); |
| 2787 | Register src_hi = src.AsRegisterPairHigh<Register>(); |
| 2788 | NearLabel done, all_zeroes; |
| 2789 | |
| 2790 | // If the low word is zero, then ZF will be set. If not, we have the answer. |
| 2791 | __ bsfl(out, src_lo); |
| 2792 | __ j(kNotEqual, &done); |
| 2793 | |
| 2794 | // Low word was zero. We have to compute the high word count and add 32. |
| 2795 | __ bsfl(out, src_hi); |
| 2796 | __ j(kEqual, &all_zeroes); |
| 2797 | |
| 2798 | // We had a valid result. Add 32 to account for the low word being zero. |
| 2799 | __ addl(out, Immediate(32)); |
| 2800 | __ jmp(&done); |
| 2801 | |
| 2802 | // All zero case. |
| 2803 | __ Bind(&all_zeroes); |
| 2804 | __ movl(out, Immediate(64)); |
| 2805 | |
| 2806 | __ Bind(&done); |
| 2807 | } |
| 2808 | |
| 2809 | void IntrinsicLocationsBuilderX86::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) { |
| 2810 | CreateTrailingZeroLocations(arena_, invoke, /* is_long */ false); |
| 2811 | } |
| 2812 | |
| 2813 | void IntrinsicCodeGeneratorX86::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) { |
Aart Bik | a19616e | 2016-02-01 18:57:58 -0800 | [diff] [blame] | 2814 | GenTrailingZeros(GetAssembler(), codegen_, invoke, /* is_long */ false); |
Mark Mendell | 2d55479 | 2015-09-15 21:45:18 -0400 | [diff] [blame] | 2815 | } |
| 2816 | |
| 2817 | void IntrinsicLocationsBuilderX86::VisitLongNumberOfTrailingZeros(HInvoke* invoke) { |
| 2818 | CreateTrailingZeroLocations(arena_, invoke, /* is_long */ true); |
| 2819 | } |
| 2820 | |
| 2821 | void IntrinsicCodeGeneratorX86::VisitLongNumberOfTrailingZeros(HInvoke* invoke) { |
Aart Bik | a19616e | 2016-02-01 18:57:58 -0800 | [diff] [blame] | 2822 | GenTrailingZeros(GetAssembler(), codegen_, invoke, /* is_long */ true); |
Mark Mendell | 2d55479 | 2015-09-15 21:45:18 -0400 | [diff] [blame] | 2823 | } |
| 2824 | |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 2825 | static bool IsSameInput(HInstruction* instruction, size_t input0, size_t input1) { |
| 2826 | return instruction->InputAt(input0) == instruction->InputAt(input1); |
| 2827 | } |
| 2828 | |
Roland Levillain | 9cc0ea8 | 2017-03-16 11:25:59 +0000 | [diff] [blame] | 2829 | // Compute base address for the System.arraycopy intrinsic in `base`. |
| 2830 | static void GenSystemArrayCopyBaseAddress(X86Assembler* assembler, |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2831 | DataType::Type type, |
Roland Levillain | 9cc0ea8 | 2017-03-16 11:25:59 +0000 | [diff] [blame] | 2832 | const Register& array, |
| 2833 | const Location& pos, |
| 2834 | const Register& base) { |
| 2835 | // This routine is only used by the SystemArrayCopy intrinsic at the |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2836 | // moment. We can allow DataType::Type::kReference as `type` to implement |
Roland Levillain | 9cc0ea8 | 2017-03-16 11:25:59 +0000 | [diff] [blame] | 2837 | // the SystemArrayCopyChar intrinsic. |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2838 | DCHECK_EQ(type, DataType::Type::kReference); |
| 2839 | const int32_t element_size = DataType::Size(type); |
| 2840 | const ScaleFactor scale_factor = static_cast<ScaleFactor>(DataType::SizeShift(type)); |
Roland Levillain | 9cc0ea8 | 2017-03-16 11:25:59 +0000 | [diff] [blame] | 2841 | const uint32_t data_offset = mirror::Array::DataOffset(element_size).Uint32Value(); |
| 2842 | |
| 2843 | if (pos.IsConstant()) { |
| 2844 | int32_t constant = pos.GetConstant()->AsIntConstant()->GetValue(); |
| 2845 | __ leal(base, Address(array, element_size * constant + data_offset)); |
| 2846 | } else { |
| 2847 | __ leal(base, Address(array, pos.AsRegister<Register>(), scale_factor, data_offset)); |
| 2848 | } |
| 2849 | } |
| 2850 | |
| 2851 | // Compute end source address for the System.arraycopy intrinsic in `end`. |
| 2852 | static void GenSystemArrayCopyEndAddress(X86Assembler* assembler, |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2853 | DataType::Type type, |
Roland Levillain | 9cc0ea8 | 2017-03-16 11:25:59 +0000 | [diff] [blame] | 2854 | const Location& copy_length, |
| 2855 | const Register& base, |
| 2856 | const Register& end) { |
| 2857 | // This routine is only used by the SystemArrayCopy intrinsic at the |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2858 | // moment. We can allow DataType::Type::kReference as `type` to implement |
Roland Levillain | 9cc0ea8 | 2017-03-16 11:25:59 +0000 | [diff] [blame] | 2859 | // the SystemArrayCopyChar intrinsic. |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 2860 | DCHECK_EQ(type, DataType::Type::kReference); |
| 2861 | const int32_t element_size = DataType::Size(type); |
| 2862 | const ScaleFactor scale_factor = static_cast<ScaleFactor>(DataType::SizeShift(type)); |
Roland Levillain | 9cc0ea8 | 2017-03-16 11:25:59 +0000 | [diff] [blame] | 2863 | |
| 2864 | if (copy_length.IsConstant()) { |
| 2865 | int32_t constant = copy_length.GetConstant()->AsIntConstant()->GetValue(); |
| 2866 | __ leal(end, Address(base, element_size * constant)); |
| 2867 | } else { |
| 2868 | __ leal(end, Address(base, copy_length.AsRegister<Register>(), scale_factor, 0)); |
| 2869 | } |
| 2870 | } |
| 2871 | |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 2872 | void IntrinsicLocationsBuilderX86::VisitSystemArrayCopy(HInvoke* invoke) { |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 2873 | // The only read barrier implementation supporting the |
| 2874 | // SystemArrayCopy intrinsic is the Baker-style read barriers. |
| 2875 | if (kEmitCompilerReadBarrier && !kUseBakerReadBarrier) { |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 2876 | return; |
| 2877 | } |
| 2878 | |
| 2879 | CodeGenerator::CreateSystemArrayCopyLocationSummary(invoke); |
| 2880 | if (invoke->GetLocations() != nullptr) { |
| 2881 | // Need a byte register for marking. |
| 2882 | invoke->GetLocations()->SetTempAt(1, Location::RegisterLocation(ECX)); |
| 2883 | |
| 2884 | static constexpr size_t kSrc = 0; |
| 2885 | static constexpr size_t kSrcPos = 1; |
| 2886 | static constexpr size_t kDest = 2; |
| 2887 | static constexpr size_t kDestPos = 3; |
| 2888 | static constexpr size_t kLength = 4; |
| 2889 | |
| 2890 | if (!invoke->InputAt(kSrcPos)->IsIntConstant() && |
| 2891 | !invoke->InputAt(kDestPos)->IsIntConstant() && |
| 2892 | !invoke->InputAt(kLength)->IsIntConstant()) { |
| 2893 | if (!IsSameInput(invoke, kSrcPos, kDestPos) && |
| 2894 | !IsSameInput(invoke, kSrcPos, kLength) && |
| 2895 | !IsSameInput(invoke, kDestPos, kLength) && |
| 2896 | !IsSameInput(invoke, kSrc, kDest)) { |
| 2897 | // Not enough registers, make the length also take a stack slot. |
| 2898 | invoke->GetLocations()->SetInAt(kLength, Location::Any()); |
| 2899 | } |
| 2900 | } |
| 2901 | } |
| 2902 | } |
| 2903 | |
| 2904 | void IntrinsicCodeGeneratorX86::VisitSystemArrayCopy(HInvoke* invoke) { |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 2905 | // The only read barrier implementation supporting the |
| 2906 | // SystemArrayCopy intrinsic is the Baker-style read barriers. |
| 2907 | DCHECK(!kEmitCompilerReadBarrier || kUseBakerReadBarrier); |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 2908 | |
| 2909 | X86Assembler* assembler = GetAssembler(); |
| 2910 | LocationSummary* locations = invoke->GetLocations(); |
| 2911 | |
| 2912 | uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); |
| 2913 | uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); |
| 2914 | uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); |
| 2915 | uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 2916 | uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 2917 | |
| 2918 | Register src = locations->InAt(0).AsRegister<Register>(); |
| 2919 | Location src_pos = locations->InAt(1); |
| 2920 | Register dest = locations->InAt(2).AsRegister<Register>(); |
| 2921 | Location dest_pos = locations->InAt(3); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 2922 | Location length_arg = locations->InAt(4); |
| 2923 | Location length = length_arg; |
| 2924 | Location temp1_loc = locations->GetTemp(0); |
| 2925 | Register temp1 = temp1_loc.AsRegister<Register>(); |
| 2926 | Location temp2_loc = locations->GetTemp(1); |
| 2927 | Register temp2 = temp2_loc.AsRegister<Register>(); |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 2928 | |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 2929 | SlowPathCode* intrinsic_slow_path = new (GetAllocator()) IntrinsicSlowPathX86(invoke); |
| 2930 | codegen_->AddSlowPath(intrinsic_slow_path); |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 2931 | |
| 2932 | NearLabel conditions_on_positions_validated; |
| 2933 | SystemArrayCopyOptimizations optimizations(invoke); |
| 2934 | |
| 2935 | // If source and destination are the same, we go to slow path if we need to do |
| 2936 | // forward copying. |
| 2937 | if (src_pos.IsConstant()) { |
| 2938 | int32_t src_pos_constant = src_pos.GetConstant()->AsIntConstant()->GetValue(); |
| 2939 | if (dest_pos.IsConstant()) { |
| 2940 | int32_t dest_pos_constant = dest_pos.GetConstant()->AsIntConstant()->GetValue(); |
| 2941 | if (optimizations.GetDestinationIsSource()) { |
| 2942 | // Checked when building locations. |
| 2943 | DCHECK_GE(src_pos_constant, dest_pos_constant); |
| 2944 | } else if (src_pos_constant < dest_pos_constant) { |
| 2945 | __ cmpl(src, dest); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 2946 | __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 2947 | } |
| 2948 | } else { |
| 2949 | if (!optimizations.GetDestinationIsSource()) { |
| 2950 | __ cmpl(src, dest); |
| 2951 | __ j(kNotEqual, &conditions_on_positions_validated); |
| 2952 | } |
| 2953 | __ cmpl(dest_pos.AsRegister<Register>(), Immediate(src_pos_constant)); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 2954 | __ j(kGreater, intrinsic_slow_path->GetEntryLabel()); |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 2955 | } |
| 2956 | } else { |
| 2957 | if (!optimizations.GetDestinationIsSource()) { |
| 2958 | __ cmpl(src, dest); |
| 2959 | __ j(kNotEqual, &conditions_on_positions_validated); |
| 2960 | } |
| 2961 | if (dest_pos.IsConstant()) { |
| 2962 | int32_t dest_pos_constant = dest_pos.GetConstant()->AsIntConstant()->GetValue(); |
| 2963 | __ cmpl(src_pos.AsRegister<Register>(), Immediate(dest_pos_constant)); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 2964 | __ j(kLess, intrinsic_slow_path->GetEntryLabel()); |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 2965 | } else { |
| 2966 | __ cmpl(src_pos.AsRegister<Register>(), dest_pos.AsRegister<Register>()); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 2967 | __ j(kLess, intrinsic_slow_path->GetEntryLabel()); |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 2968 | } |
| 2969 | } |
| 2970 | |
| 2971 | __ Bind(&conditions_on_positions_validated); |
| 2972 | |
| 2973 | if (!optimizations.GetSourceIsNotNull()) { |
| 2974 | // Bail out if the source is null. |
| 2975 | __ testl(src, src); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 2976 | __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 2977 | } |
| 2978 | |
| 2979 | if (!optimizations.GetDestinationIsNotNull() && !optimizations.GetDestinationIsSource()) { |
| 2980 | // Bail out if the destination is null. |
| 2981 | __ testl(dest, dest); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 2982 | __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 2983 | } |
| 2984 | |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 2985 | Location temp3_loc = locations->GetTemp(2); |
| 2986 | Register temp3 = temp3_loc.AsRegister<Register>(); |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 2987 | if (length.IsStackSlot()) { |
| 2988 | __ movl(temp3, Address(ESP, length.GetStackIndex())); |
| 2989 | length = Location::RegisterLocation(temp3); |
| 2990 | } |
| 2991 | |
| 2992 | // If the length is negative, bail out. |
| 2993 | // We have already checked in the LocationsBuilder for the constant case. |
| 2994 | if (!length.IsConstant() && |
| 2995 | !optimizations.GetCountIsSourceLength() && |
| 2996 | !optimizations.GetCountIsDestinationLength()) { |
| 2997 | __ testl(length.AsRegister<Register>(), length.AsRegister<Register>()); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 2998 | __ j(kLess, intrinsic_slow_path->GetEntryLabel()); |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 2999 | } |
| 3000 | |
| 3001 | // Validity checks: source. |
| 3002 | CheckPosition(assembler, |
| 3003 | src_pos, |
| 3004 | src, |
| 3005 | length, |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3006 | intrinsic_slow_path, |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 3007 | temp1, |
| 3008 | optimizations.GetCountIsSourceLength()); |
| 3009 | |
| 3010 | // Validity checks: dest. |
| 3011 | CheckPosition(assembler, |
| 3012 | dest_pos, |
| 3013 | dest, |
| 3014 | length, |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3015 | intrinsic_slow_path, |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 3016 | temp1, |
| 3017 | optimizations.GetCountIsDestinationLength()); |
| 3018 | |
| 3019 | if (!optimizations.GetDoesNotNeedTypeCheck()) { |
| 3020 | // Check whether all elements of the source array are assignable to the component |
| 3021 | // type of the destination array. We do two checks: the classes are the same, |
| 3022 | // or the destination is Object[]. If none of these checks succeed, we go to the |
| 3023 | // slow path. |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3024 | |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 3025 | if (!optimizations.GetSourceIsNonPrimitiveArray()) { |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3026 | if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { |
| 3027 | // /* HeapReference<Class> */ temp1 = src->klass_ |
| 3028 | codegen_->GenerateFieldLoadWithBakerReadBarrier( |
Vladimir Marko | 953437b | 2016-08-24 08:30:46 +0000 | [diff] [blame] | 3029 | invoke, temp1_loc, src, class_offset, /* needs_null_check */ false); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3030 | // Bail out if the source is not a non primitive array. |
| 3031 | // /* HeapReference<Class> */ temp1 = temp1->component_type_ |
| 3032 | codegen_->GenerateFieldLoadWithBakerReadBarrier( |
Vladimir Marko | 953437b | 2016-08-24 08:30:46 +0000 | [diff] [blame] | 3033 | invoke, temp1_loc, temp1, component_offset, /* needs_null_check */ false); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3034 | __ testl(temp1, temp1); |
| 3035 | __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); |
| 3036 | // If heap poisoning is enabled, `temp1` has been unpoisoned |
| 3037 | // by the the previous call to GenerateFieldLoadWithBakerReadBarrier. |
| 3038 | } else { |
| 3039 | // /* HeapReference<Class> */ temp1 = src->klass_ |
| 3040 | __ movl(temp1, Address(src, class_offset)); |
| 3041 | __ MaybeUnpoisonHeapReference(temp1); |
| 3042 | // Bail out if the source is not a non primitive array. |
| 3043 | // /* HeapReference<Class> */ temp1 = temp1->component_type_ |
| 3044 | __ movl(temp1, Address(temp1, component_offset)); |
| 3045 | __ testl(temp1, temp1); |
| 3046 | __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); |
| 3047 | __ MaybeUnpoisonHeapReference(temp1); |
| 3048 | } |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 3049 | __ cmpw(Address(temp1, primitive_offset), Immediate(Primitive::kPrimNot)); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3050 | __ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 3051 | } |
| 3052 | |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3053 | if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { |
| 3054 | if (length.Equals(Location::RegisterLocation(temp3))) { |
| 3055 | // When Baker read barriers are enabled, register `temp3`, |
| 3056 | // which in the present case contains the `length` parameter, |
| 3057 | // will be overwritten below. Make the `length` location |
| 3058 | // reference the original stack location; it will be moved |
| 3059 | // back to `temp3` later if necessary. |
| 3060 | DCHECK(length_arg.IsStackSlot()); |
| 3061 | length = length_arg; |
| 3062 | } |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 3063 | |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3064 | // /* HeapReference<Class> */ temp1 = dest->klass_ |
| 3065 | codegen_->GenerateFieldLoadWithBakerReadBarrier( |
Vladimir Marko | 953437b | 2016-08-24 08:30:46 +0000 | [diff] [blame] | 3066 | invoke, temp1_loc, dest, class_offset, /* needs_null_check */ false); |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 3067 | |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3068 | if (!optimizations.GetDestinationIsNonPrimitiveArray()) { |
| 3069 | // Bail out if the destination is not a non primitive array. |
| 3070 | // |
| 3071 | // Register `temp1` is not trashed by the read barrier emitted |
| 3072 | // by GenerateFieldLoadWithBakerReadBarrier below, as that |
| 3073 | // method produces a call to a ReadBarrierMarkRegX entry point, |
| 3074 | // which saves all potentially live registers, including |
| 3075 | // temporaries such a `temp1`. |
| 3076 | // /* HeapReference<Class> */ temp2 = temp1->component_type_ |
| 3077 | codegen_->GenerateFieldLoadWithBakerReadBarrier( |
Vladimir Marko | 953437b | 2016-08-24 08:30:46 +0000 | [diff] [blame] | 3078 | invoke, temp2_loc, temp1, component_offset, /* needs_null_check */ false); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3079 | __ testl(temp2, temp2); |
| 3080 | __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); |
| 3081 | // If heap poisoning is enabled, `temp2` has been unpoisoned |
| 3082 | // by the the previous call to GenerateFieldLoadWithBakerReadBarrier. |
| 3083 | __ cmpw(Address(temp2, primitive_offset), Immediate(Primitive::kPrimNot)); |
| 3084 | __ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); |
| 3085 | } |
| 3086 | |
| 3087 | // For the same reason given earlier, `temp1` is not trashed by the |
| 3088 | // read barrier emitted by GenerateFieldLoadWithBakerReadBarrier below. |
| 3089 | // /* HeapReference<Class> */ temp2 = src->klass_ |
| 3090 | codegen_->GenerateFieldLoadWithBakerReadBarrier( |
Vladimir Marko | 953437b | 2016-08-24 08:30:46 +0000 | [diff] [blame] | 3091 | invoke, temp2_loc, src, class_offset, /* needs_null_check */ false); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3092 | // Note: if heap poisoning is on, we are comparing two unpoisoned references here. |
| 3093 | __ cmpl(temp1, temp2); |
| 3094 | |
| 3095 | if (optimizations.GetDestinationIsTypedObjectArray()) { |
| 3096 | NearLabel do_copy; |
| 3097 | __ j(kEqual, &do_copy); |
| 3098 | // /* HeapReference<Class> */ temp1 = temp1->component_type_ |
| 3099 | codegen_->GenerateFieldLoadWithBakerReadBarrier( |
Vladimir Marko | 953437b | 2016-08-24 08:30:46 +0000 | [diff] [blame] | 3100 | invoke, temp1_loc, temp1, component_offset, /* needs_null_check */ false); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3101 | // We do not need to emit a read barrier for the following |
| 3102 | // heap reference load, as `temp1` is only used in a |
| 3103 | // comparison with null below, and this reference is not |
| 3104 | // kept afterwards. |
| 3105 | __ cmpl(Address(temp1, super_offset), Immediate(0)); |
| 3106 | __ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); |
| 3107 | __ Bind(&do_copy); |
| 3108 | } else { |
| 3109 | __ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); |
| 3110 | } |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 3111 | } else { |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3112 | // Non read barrier code. |
| 3113 | |
| 3114 | // /* HeapReference<Class> */ temp1 = dest->klass_ |
| 3115 | __ movl(temp1, Address(dest, class_offset)); |
| 3116 | if (!optimizations.GetDestinationIsNonPrimitiveArray()) { |
| 3117 | __ MaybeUnpoisonHeapReference(temp1); |
| 3118 | // Bail out if the destination is not a non primitive array. |
| 3119 | // /* HeapReference<Class> */ temp2 = temp1->component_type_ |
| 3120 | __ movl(temp2, Address(temp1, component_offset)); |
| 3121 | __ testl(temp2, temp2); |
| 3122 | __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); |
| 3123 | __ MaybeUnpoisonHeapReference(temp2); |
| 3124 | __ cmpw(Address(temp2, primitive_offset), Immediate(Primitive::kPrimNot)); |
| 3125 | __ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); |
| 3126 | // Re-poison the heap reference to make the compare instruction below |
| 3127 | // compare two poisoned references. |
| 3128 | __ PoisonHeapReference(temp1); |
| 3129 | } |
| 3130 | |
| 3131 | // Note: if heap poisoning is on, we are comparing two poisoned references here. |
| 3132 | __ cmpl(temp1, Address(src, class_offset)); |
| 3133 | |
| 3134 | if (optimizations.GetDestinationIsTypedObjectArray()) { |
| 3135 | NearLabel do_copy; |
| 3136 | __ j(kEqual, &do_copy); |
| 3137 | __ MaybeUnpoisonHeapReference(temp1); |
| 3138 | // /* HeapReference<Class> */ temp1 = temp1->component_type_ |
| 3139 | __ movl(temp1, Address(temp1, component_offset)); |
| 3140 | __ MaybeUnpoisonHeapReference(temp1); |
| 3141 | __ cmpl(Address(temp1, super_offset), Immediate(0)); |
| 3142 | __ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); |
| 3143 | __ Bind(&do_copy); |
| 3144 | } else { |
| 3145 | __ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); |
| 3146 | } |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 3147 | } |
| 3148 | } else if (!optimizations.GetSourceIsNonPrimitiveArray()) { |
| 3149 | DCHECK(optimizations.GetDestinationIsNonPrimitiveArray()); |
| 3150 | // Bail out if the source is not a non primitive array. |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3151 | if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { |
| 3152 | // /* HeapReference<Class> */ temp1 = src->klass_ |
| 3153 | codegen_->GenerateFieldLoadWithBakerReadBarrier( |
Vladimir Marko | 953437b | 2016-08-24 08:30:46 +0000 | [diff] [blame] | 3154 | invoke, temp1_loc, src, class_offset, /* needs_null_check */ false); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3155 | // /* HeapReference<Class> */ temp1 = temp1->component_type_ |
| 3156 | codegen_->GenerateFieldLoadWithBakerReadBarrier( |
Vladimir Marko | 953437b | 2016-08-24 08:30:46 +0000 | [diff] [blame] | 3157 | invoke, temp1_loc, temp1, component_offset, /* needs_null_check */ false); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3158 | __ testl(temp1, temp1); |
| 3159 | __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); |
| 3160 | // If heap poisoning is enabled, `temp1` has been unpoisoned |
| 3161 | // by the the previous call to GenerateFieldLoadWithBakerReadBarrier. |
| 3162 | } else { |
| 3163 | // /* HeapReference<Class> */ temp1 = src->klass_ |
| 3164 | __ movl(temp1, Address(src, class_offset)); |
| 3165 | __ MaybeUnpoisonHeapReference(temp1); |
| 3166 | // /* HeapReference<Class> */ temp1 = temp1->component_type_ |
| 3167 | __ movl(temp1, Address(temp1, component_offset)); |
| 3168 | __ testl(temp1, temp1); |
| 3169 | __ j(kEqual, intrinsic_slow_path->GetEntryLabel()); |
| 3170 | __ MaybeUnpoisonHeapReference(temp1); |
| 3171 | } |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 3172 | __ cmpw(Address(temp1, primitive_offset), Immediate(Primitive::kPrimNot)); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3173 | __ j(kNotEqual, intrinsic_slow_path->GetEntryLabel()); |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 3174 | } |
| 3175 | |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 3176 | const DataType::Type type = DataType::Type::kReference; |
| 3177 | const int32_t element_size = DataType::Size(type); |
Roland Levillain | 9cc0ea8 | 2017-03-16 11:25:59 +0000 | [diff] [blame] | 3178 | |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3179 | // Compute the base source address in `temp1`. |
Roland Levillain | 9cc0ea8 | 2017-03-16 11:25:59 +0000 | [diff] [blame] | 3180 | GenSystemArrayCopyBaseAddress(GetAssembler(), type, src, src_pos, temp1); |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 3181 | |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3182 | if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { |
| 3183 | // If it is needed (in the case of the fast-path loop), the base |
| 3184 | // destination address is computed later, as `temp2` is used for |
| 3185 | // intermediate computations. |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 3186 | |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3187 | // Compute the end source address in `temp3`. |
Roland Levillain | 9cc0ea8 | 2017-03-16 11:25:59 +0000 | [diff] [blame] | 3188 | if (length.IsStackSlot()) { |
| 3189 | // Location `length` is again pointing at a stack slot, as |
| 3190 | // register `temp3` (which was containing the length parameter |
| 3191 | // earlier) has been overwritten; restore it now |
| 3192 | DCHECK(length.Equals(length_arg)); |
| 3193 | __ movl(temp3, Address(ESP, length.GetStackIndex())); |
| 3194 | length = Location::RegisterLocation(temp3); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3195 | } |
Roland Levillain | 9cc0ea8 | 2017-03-16 11:25:59 +0000 | [diff] [blame] | 3196 | GenSystemArrayCopyEndAddress(GetAssembler(), type, length, temp1, temp3); |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 3197 | |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3198 | // SystemArrayCopy implementation for Baker read barriers (see |
| 3199 | // also CodeGeneratorX86::GenerateReferenceLoadWithBakerReadBarrier): |
| 3200 | // |
| 3201 | // if (src_ptr != end_ptr) { |
| 3202 | // uint32_t rb_state = Lockword(src->monitor_).ReadBarrierState(); |
| 3203 | // lfence; // Load fence or artificial data dependency to prevent load-load reordering |
Hiroshi Yamauchi | 12b58b2 | 2016-11-01 11:55:29 -0700 | [diff] [blame] | 3204 | // bool is_gray = (rb_state == ReadBarrier::GrayState()); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3205 | // if (is_gray) { |
| 3206 | // // Slow-path copy. |
| 3207 | // for (size_t i = 0; i != length; ++i) { |
| 3208 | // dest_array[dest_pos + i] = |
| 3209 | // MaybePoison(ReadBarrier::Mark(MaybeUnpoison(src_array[src_pos + i]))); |
| 3210 | // } |
| 3211 | // } else { |
| 3212 | // // Fast-path copy. |
| 3213 | // do { |
| 3214 | // *dest_ptr++ = *src_ptr++; |
| 3215 | // } while (src_ptr != end_ptr) |
| 3216 | // } |
| 3217 | // } |
| 3218 | |
| 3219 | NearLabel loop, done; |
| 3220 | |
| 3221 | // Don't enter copy loop if `length == 0`. |
| 3222 | __ cmpl(temp1, temp3); |
| 3223 | __ j(kEqual, &done); |
| 3224 | |
Vladimir Marko | 953437b | 2016-08-24 08:30:46 +0000 | [diff] [blame] | 3225 | // Given the numeric representation, it's enough to check the low bit of the rb_state. |
Hiroshi Yamauchi | 12b58b2 | 2016-11-01 11:55:29 -0700 | [diff] [blame] | 3226 | static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); |
| 3227 | static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); |
Vladimir Marko | 953437b | 2016-08-24 08:30:46 +0000 | [diff] [blame] | 3228 | constexpr uint32_t gray_byte_position = LockWord::kReadBarrierStateShift / kBitsPerByte; |
| 3229 | constexpr uint32_t gray_bit_position = LockWord::kReadBarrierStateShift % kBitsPerByte; |
| 3230 | constexpr int32_t test_value = static_cast<int8_t>(1 << gray_bit_position); |
| 3231 | |
Hiroshi Yamauchi | 12b58b2 | 2016-11-01 11:55:29 -0700 | [diff] [blame] | 3232 | // if (rb_state == ReadBarrier::GrayState()) |
Vladimir Marko | 953437b | 2016-08-24 08:30:46 +0000 | [diff] [blame] | 3233 | // goto slow_path; |
| 3234 | // At this point, just do the "if" and make sure that flags are preserved until the branch. |
| 3235 | __ testb(Address(src, monitor_offset + gray_byte_position), Immediate(test_value)); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3236 | |
| 3237 | // Load fence to prevent load-load reordering. |
| 3238 | // Note that this is a no-op, thanks to the x86 memory model. |
| 3239 | codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); |
| 3240 | |
| 3241 | // Slow path used to copy array when `src` is gray. |
| 3242 | SlowPathCode* read_barrier_slow_path = |
| 3243 | new (GetAllocator()) ReadBarrierSystemArrayCopySlowPathX86(invoke); |
| 3244 | codegen_->AddSlowPath(read_barrier_slow_path); |
| 3245 | |
Vladimir Marko | 953437b | 2016-08-24 08:30:46 +0000 | [diff] [blame] | 3246 | // We have done the "if" of the gray bit check above, now branch based on the flags. |
| 3247 | __ j(kNotZero, read_barrier_slow_path->GetEntryLabel()); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3248 | |
| 3249 | // Fast-path copy. |
Roland Levillain | 9cc0ea8 | 2017-03-16 11:25:59 +0000 | [diff] [blame] | 3250 | // Compute the base destination address in `temp2`. |
| 3251 | GenSystemArrayCopyBaseAddress(GetAssembler(), type, dest, dest_pos, temp2); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3252 | // Iterate over the arrays and do a raw copy of the objects. We don't need to |
| 3253 | // poison/unpoison. |
| 3254 | __ Bind(&loop); |
| 3255 | __ pushl(Address(temp1, 0)); |
| 3256 | __ cfi().AdjustCFAOffset(4); |
| 3257 | __ popl(Address(temp2, 0)); |
| 3258 | __ cfi().AdjustCFAOffset(-4); |
| 3259 | __ addl(temp1, Immediate(element_size)); |
| 3260 | __ addl(temp2, Immediate(element_size)); |
| 3261 | __ cmpl(temp1, temp3); |
| 3262 | __ j(kNotEqual, &loop); |
| 3263 | |
| 3264 | __ Bind(read_barrier_slow_path->GetExitLabel()); |
| 3265 | __ Bind(&done); |
| 3266 | } else { |
| 3267 | // Non read barrier code. |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3268 | // Compute the base destination address in `temp2`. |
Roland Levillain | 9cc0ea8 | 2017-03-16 11:25:59 +0000 | [diff] [blame] | 3269 | GenSystemArrayCopyBaseAddress(GetAssembler(), type, dest, dest_pos, temp2); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3270 | // Compute the end source address in `temp3`. |
Roland Levillain | 9cc0ea8 | 2017-03-16 11:25:59 +0000 | [diff] [blame] | 3271 | GenSystemArrayCopyEndAddress(GetAssembler(), type, length, temp1, temp3); |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3272 | // Iterate over the arrays and do a raw copy of the objects. We don't need to |
| 3273 | // poison/unpoison. |
| 3274 | NearLabel loop, done; |
| 3275 | __ cmpl(temp1, temp3); |
| 3276 | __ j(kEqual, &done); |
| 3277 | __ Bind(&loop); |
| 3278 | __ pushl(Address(temp1, 0)); |
| 3279 | __ cfi().AdjustCFAOffset(4); |
| 3280 | __ popl(Address(temp2, 0)); |
| 3281 | __ cfi().AdjustCFAOffset(-4); |
| 3282 | __ addl(temp1, Immediate(element_size)); |
| 3283 | __ addl(temp2, Immediate(element_size)); |
| 3284 | __ cmpl(temp1, temp3); |
| 3285 | __ j(kNotEqual, &loop); |
| 3286 | __ Bind(&done); |
| 3287 | } |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 3288 | |
| 3289 | // We only need one card marking on the destination array. |
Roland Levillain | 9cc0ea8 | 2017-03-16 11:25:59 +0000 | [diff] [blame] | 3290 | codegen_->MarkGCCard(temp1, temp2, dest, Register(kNoRegister), /* value_can_be_null */ false); |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 3291 | |
Roland Levillain | 0b671c0 | 2016-08-19 12:02:34 +0100 | [diff] [blame] | 3292 | __ Bind(intrinsic_slow_path->GetExitLabel()); |
Nicolas Geoffray | fea1abd | 2016-07-06 12:09:12 +0100 | [diff] [blame] | 3293 | } |
| 3294 | |
Nicolas Geoffray | 331605a | 2017-03-01 11:01:41 +0000 | [diff] [blame] | 3295 | void IntrinsicLocationsBuilderX86::VisitIntegerValueOf(HInvoke* invoke) { |
| 3296 | InvokeRuntimeCallingConvention calling_convention; |
| 3297 | IntrinsicVisitor::ComputeIntegerValueOfLocations( |
| 3298 | invoke, |
| 3299 | codegen_, |
| 3300 | Location::RegisterLocation(EAX), |
| 3301 | Location::RegisterLocation(calling_convention.GetRegisterAt(0))); |
| 3302 | } |
| 3303 | |
| 3304 | void IntrinsicCodeGeneratorX86::VisitIntegerValueOf(HInvoke* invoke) { |
| 3305 | IntrinsicVisitor::IntegerValueOfInfo info = IntrinsicVisitor::ComputeIntegerValueOfInfo(); |
| 3306 | LocationSummary* locations = invoke->GetLocations(); |
| 3307 | X86Assembler* assembler = GetAssembler(); |
| 3308 | |
| 3309 | Register out = locations->Out().AsRegister<Register>(); |
| 3310 | InvokeRuntimeCallingConvention calling_convention; |
| 3311 | if (invoke->InputAt(0)->IsConstant()) { |
| 3312 | int32_t value = invoke->InputAt(0)->AsIntConstant()->GetValue(); |
| 3313 | if (value >= info.low && value <= info.high) { |
| 3314 | // Just embed the j.l.Integer in the code. |
| 3315 | ScopedObjectAccess soa(Thread::Current()); |
| 3316 | mirror::Object* boxed = info.cache->Get(value + (-info.low)); |
| 3317 | DCHECK(boxed != nullptr && Runtime::Current()->GetHeap()->ObjectIsInBootImageSpace(boxed)); |
| 3318 | uint32_t address = dchecked_integral_cast<uint32_t>(reinterpret_cast<uintptr_t>(boxed)); |
| 3319 | __ movl(out, Immediate(address)); |
| 3320 | } else { |
| 3321 | // Allocate and initialize a new j.l.Integer. |
| 3322 | // TODO: If we JIT, we could allocate the j.l.Integer now, and store it in the |
| 3323 | // JIT object table. |
| 3324 | uint32_t address = dchecked_integral_cast<uint32_t>(reinterpret_cast<uintptr_t>(info.integer)); |
| 3325 | __ movl(calling_convention.GetRegisterAt(0), Immediate(address)); |
| 3326 | codegen_->InvokeRuntime(kQuickAllocObjectInitialized, invoke, invoke->GetDexPc()); |
| 3327 | CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>(); |
| 3328 | __ movl(Address(out, info.value_offset), Immediate(value)); |
| 3329 | } |
| 3330 | } else { |
| 3331 | Register in = locations->InAt(0).AsRegister<Register>(); |
| 3332 | // Check bounds of our cache. |
| 3333 | __ leal(out, Address(in, -info.low)); |
| 3334 | __ cmpl(out, Immediate(info.high - info.low + 1)); |
| 3335 | NearLabel allocate, done; |
| 3336 | __ j(kAboveEqual, &allocate); |
| 3337 | // If the value is within the bounds, load the j.l.Integer directly from the array. |
| 3338 | uint32_t data_offset = mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value(); |
| 3339 | uint32_t address = dchecked_integral_cast<uint32_t>(reinterpret_cast<uintptr_t>(info.cache)); |
| 3340 | __ movl(out, Address(out, TIMES_4, data_offset + address)); |
| 3341 | __ MaybeUnpoisonHeapReference(out); |
| 3342 | __ jmp(&done); |
| 3343 | __ Bind(&allocate); |
| 3344 | // Otherwise allocate and initialize a new j.l.Integer. |
| 3345 | address = dchecked_integral_cast<uint32_t>(reinterpret_cast<uintptr_t>(info.integer)); |
| 3346 | __ movl(calling_convention.GetRegisterAt(0), Immediate(address)); |
| 3347 | codegen_->InvokeRuntime(kQuickAllocObjectInitialized, invoke, invoke->GetDexPc()); |
| 3348 | CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>(); |
| 3349 | __ movl(Address(out, info.value_offset), in); |
| 3350 | __ Bind(&done); |
| 3351 | } |
| 3352 | } |
| 3353 | |
Nicolas Geoffray | 365719c | 2017-03-08 13:11:50 +0000 | [diff] [blame] | 3354 | void IntrinsicLocationsBuilderX86::VisitThreadInterrupted(HInvoke* invoke) { |
| 3355 | LocationSummary* locations = new (arena_) LocationSummary(invoke, |
| 3356 | LocationSummary::kNoCall, |
| 3357 | kIntrinsified); |
| 3358 | locations->SetOut(Location::RequiresRegister()); |
| 3359 | } |
| 3360 | |
| 3361 | void IntrinsicCodeGeneratorX86::VisitThreadInterrupted(HInvoke* invoke) { |
| 3362 | X86Assembler* assembler = GetAssembler(); |
| 3363 | Register out = invoke->GetLocations()->Out().AsRegister<Register>(); |
| 3364 | Address address = Address::Absolute(Thread::InterruptedOffset<kX86PointerSize>().Int32Value()); |
| 3365 | NearLabel done; |
| 3366 | __ fs()->movl(out, address); |
| 3367 | __ testl(out, out); |
| 3368 | __ j(kEqual, &done); |
| 3369 | __ fs()->movl(address, Immediate(0)); |
| 3370 | codegen_->MemoryFence(); |
| 3371 | __ Bind(&done); |
| 3372 | } |
| 3373 | |
| 3374 | |
Aart Bik | 2f9fcc9 | 2016-03-01 15:16:54 -0800 | [diff] [blame] | 3375 | UNIMPLEMENTED_INTRINSIC(X86, MathRoundDouble) |
Vladimir Marko | 4ee8e29 | 2017-06-02 15:39:30 +0000 | [diff] [blame] | 3376 | UNIMPLEMENTED_INTRINSIC(X86, ReferenceGetReferent) |
Aart Bik | 2f9fcc9 | 2016-03-01 15:16:54 -0800 | [diff] [blame] | 3377 | UNIMPLEMENTED_INTRINSIC(X86, FloatIsInfinite) |
| 3378 | UNIMPLEMENTED_INTRINSIC(X86, DoubleIsInfinite) |
| 3379 | UNIMPLEMENTED_INTRINSIC(X86, IntegerHighestOneBit) |
| 3380 | UNIMPLEMENTED_INTRINSIC(X86, LongHighestOneBit) |
| 3381 | UNIMPLEMENTED_INTRINSIC(X86, IntegerLowestOneBit) |
| 3382 | UNIMPLEMENTED_INTRINSIC(X86, LongLowestOneBit) |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 3383 | |
Aart Bik | ff7d89c | 2016-11-07 08:49:28 -0800 | [diff] [blame] | 3384 | UNIMPLEMENTED_INTRINSIC(X86, StringStringIndexOf); |
| 3385 | UNIMPLEMENTED_INTRINSIC(X86, StringStringIndexOfAfter); |
Aart Bik | 71bf7b4 | 2016-11-16 10:17:46 -0800 | [diff] [blame] | 3386 | UNIMPLEMENTED_INTRINSIC(X86, StringBufferAppend); |
| 3387 | UNIMPLEMENTED_INTRINSIC(X86, StringBufferLength); |
| 3388 | UNIMPLEMENTED_INTRINSIC(X86, StringBufferToString); |
| 3389 | UNIMPLEMENTED_INTRINSIC(X86, StringBuilderAppend); |
| 3390 | UNIMPLEMENTED_INTRINSIC(X86, StringBuilderLength); |
| 3391 | UNIMPLEMENTED_INTRINSIC(X86, StringBuilderToString); |
Aart Bik | ff7d89c | 2016-11-07 08:49:28 -0800 | [diff] [blame] | 3392 | |
Aart Bik | 0e54c01 | 2016-03-04 12:08:31 -0800 | [diff] [blame] | 3393 | // 1.8. |
| 3394 | UNIMPLEMENTED_INTRINSIC(X86, UnsafeGetAndAddInt) |
| 3395 | UNIMPLEMENTED_INTRINSIC(X86, UnsafeGetAndAddLong) |
| 3396 | UNIMPLEMENTED_INTRINSIC(X86, UnsafeGetAndSetInt) |
| 3397 | UNIMPLEMENTED_INTRINSIC(X86, UnsafeGetAndSetLong) |
| 3398 | UNIMPLEMENTED_INTRINSIC(X86, UnsafeGetAndSetObject) |
Aart Bik | 0e54c01 | 2016-03-04 12:08:31 -0800 | [diff] [blame] | 3399 | |
Aart Bik | 2f9fcc9 | 2016-03-01 15:16:54 -0800 | [diff] [blame] | 3400 | UNREACHABLE_INTRINSICS(X86) |
Roland Levillain | 4d02711 | 2015-07-01 15:41:14 +0100 | [diff] [blame] | 3401 | |
| 3402 | #undef __ |
| 3403 | |
Mark Mendell | 09ed1a3 | 2015-03-25 08:30:06 -0400 | [diff] [blame] | 3404 | } // namespace x86 |
| 3405 | } // namespace art |