| /* |
| * Copyright (C) 2017 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| #include "code_generator_x86_64.h" |
| |
| #include "mirror/array-inl.h" |
| #include "mirror/string.h" |
| |
| namespace art { |
| namespace x86_64 { |
| |
| // NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. |
| #define __ down_cast<X86_64Assembler*>(GetAssembler())-> // NOLINT |
| |
| void LocationsBuilderX86_64::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { |
| LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); |
| HInstruction* input = instruction->InputAt(0); |
| bool is_zero = IsZeroBitPattern(input); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kBool: |
| case DataType::Type::kInt8: |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| case DataType::Type::kInt32: |
| case DataType::Type::kInt64: |
| locations->SetInAt(0, is_zero ? Location::ConstantLocation(input->AsConstant()) |
| : Location::RequiresRegister()); |
| locations->SetOut(Location::RequiresFpuRegister()); |
| break; |
| case DataType::Type::kFloat32: |
| case DataType::Type::kFloat64: |
| locations->SetInAt(0, is_zero ? Location::ConstantLocation(input->AsConstant()) |
| : Location::RequiresFpuRegister()); |
| locations->SetOut(is_zero ? Location::RequiresFpuRegister() |
| : Location::SameAsFirstInput()); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| void InstructionCodeGeneratorX86_64::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { |
| LocationSummary* locations = instruction->GetLocations(); |
| XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| |
| // Shorthand for any type of zero. |
| if (IsZeroBitPattern(instruction->InputAt(0))) { |
| __ xorps(dst, dst); |
| return; |
| } |
| |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kBool: |
| case DataType::Type::kInt8: |
| DCHECK_EQ(16u, instruction->GetVectorLength()); |
| __ movd(dst, locations->InAt(0).AsRegister<CpuRegister>(), /*64-bit*/ false); |
| __ punpcklbw(dst, dst); |
| __ punpcklwd(dst, dst); |
| __ pshufd(dst, dst, Immediate(0)); |
| break; |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| DCHECK_EQ(8u, instruction->GetVectorLength()); |
| __ movd(dst, locations->InAt(0).AsRegister<CpuRegister>(), /*64-bit*/ false); |
| __ punpcklwd(dst, dst); |
| __ pshufd(dst, dst, Immediate(0)); |
| break; |
| case DataType::Type::kInt32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| __ movd(dst, locations->InAt(0).AsRegister<CpuRegister>(), /*64-bit*/ false); |
| __ pshufd(dst, dst, Immediate(0)); |
| break; |
| case DataType::Type::kInt64: |
| DCHECK_EQ(2u, instruction->GetVectorLength()); |
| __ movd(dst, locations->InAt(0).AsRegister<CpuRegister>(), /*64-bit*/ true); |
| __ punpcklqdq(dst, dst); |
| break; |
| case DataType::Type::kFloat32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| DCHECK(locations->InAt(0).Equals(locations->Out())); |
| __ shufps(dst, dst, Immediate(0)); |
| break; |
| case DataType::Type::kFloat64: |
| DCHECK_EQ(2u, instruction->GetVectorLength()); |
| DCHECK(locations->InAt(0).Equals(locations->Out())); |
| __ shufpd(dst, dst, Immediate(0)); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| void LocationsBuilderX86_64::VisitVecExtractScalar(HVecExtractScalar* instruction) { |
| LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kBool: |
| case DataType::Type::kInt8: |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| case DataType::Type::kInt32: |
| case DataType::Type::kInt64: |
| locations->SetInAt(0, Location::RequiresFpuRegister()); |
| locations->SetOut(Location::RequiresRegister()); |
| break; |
| case DataType::Type::kFloat32: |
| case DataType::Type::kFloat64: |
| locations->SetInAt(0, Location::RequiresFpuRegister()); |
| locations->SetOut(Location::SameAsFirstInput()); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| void InstructionCodeGeneratorX86_64::VisitVecExtractScalar(HVecExtractScalar* instruction) { |
| LocationSummary* locations = instruction->GetLocations(); |
| XmmRegister src = locations->InAt(0).AsFpuRegister<XmmRegister>(); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kBool: |
| case DataType::Type::kInt8: |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: // TODO: up to here, and? |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| case DataType::Type::kInt32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| __ movd(locations->Out().AsRegister<CpuRegister>(), src, /*64-bit*/ false); |
| break; |
| case DataType::Type::kInt64: |
| DCHECK_EQ(2u, instruction->GetVectorLength()); |
| __ movd(locations->Out().AsRegister<CpuRegister>(), src, /*64-bit*/ true); |
| break; |
| case DataType::Type::kFloat32: |
| case DataType::Type::kFloat64: |
| DCHECK_LE(2u, instruction->GetVectorLength()); |
| DCHECK_LE(instruction->GetVectorLength(), 4u); |
| DCHECK(locations->InAt(0).Equals(locations->Out())); // no code required |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| // Helper to set up locations for vector unary operations. |
| static void CreateVecUnOpLocations(ArenaAllocator* arena, HVecUnaryOperation* instruction) { |
| LocationSummary* locations = new (arena) LocationSummary(instruction); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kBool: |
| case DataType::Type::kInt8: |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| case DataType::Type::kInt32: |
| case DataType::Type::kInt64: |
| case DataType::Type::kFloat32: |
| case DataType::Type::kFloat64: |
| locations->SetInAt(0, Location::RequiresFpuRegister()); |
| locations->SetOut(Location::RequiresFpuRegister()); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| void LocationsBuilderX86_64::VisitVecReduce(HVecReduce* instruction) { |
| CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); |
| // Long reduction or min/max require a temporary. |
| if (instruction->GetPackedType() == DataType::Type::kInt64 || |
| instruction->GetKind() == HVecReduce::kMin || |
| instruction->GetKind() == HVecReduce::kMax) { |
| instruction->GetLocations()->AddTemp(Location::RequiresFpuRegister()); |
| } |
| } |
| |
| void InstructionCodeGeneratorX86_64::VisitVecReduce(HVecReduce* instruction) { |
| LocationSummary* locations = instruction->GetLocations(); |
| XmmRegister src = locations->InAt(0).AsFpuRegister<XmmRegister>(); |
| XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kInt32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| switch (instruction->GetKind()) { |
| case HVecReduce::kSum: |
| __ movaps(dst, src); |
| __ phaddd(dst, dst); |
| __ phaddd(dst, dst); |
| break; |
| case HVecReduce::kMin: { |
| XmmRegister tmp = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); |
| __ movaps(tmp, src); |
| __ movaps(dst, src); |
| __ psrldq(tmp, Immediate(8)); |
| __ pminsd(dst, tmp); |
| __ psrldq(tmp, Immediate(4)); |
| __ pminsd(dst, tmp); |
| break; |
| } |
| case HVecReduce::kMax: { |
| XmmRegister tmp = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); |
| __ movaps(tmp, src); |
| __ movaps(dst, src); |
| __ psrldq(tmp, Immediate(8)); |
| __ pmaxsd(dst, tmp); |
| __ psrldq(tmp, Immediate(4)); |
| __ pmaxsd(dst, tmp); |
| break; |
| } |
| } |
| break; |
| case DataType::Type::kInt64: { |
| DCHECK_EQ(2u, instruction->GetVectorLength()); |
| XmmRegister tmp = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); |
| switch (instruction->GetKind()) { |
| case HVecReduce::kSum: |
| __ movaps(tmp, src); |
| __ movaps(dst, src); |
| __ punpckhqdq(tmp, tmp); |
| __ paddq(dst, tmp); |
| break; |
| case HVecReduce::kMin: |
| case HVecReduce::kMax: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| } |
| break; |
| } |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| void LocationsBuilderX86_64::VisitVecCnv(HVecCnv* instruction) { |
| CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); |
| } |
| |
| void InstructionCodeGeneratorX86_64::VisitVecCnv(HVecCnv* instruction) { |
| LocationSummary* locations = instruction->GetLocations(); |
| XmmRegister src = locations->InAt(0).AsFpuRegister<XmmRegister>(); |
| XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| DataType::Type from = instruction->GetInputType(); |
| DataType::Type to = instruction->GetResultType(); |
| if (from == DataType::Type::kInt32 && to == DataType::Type::kFloat32) { |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| __ cvtdq2ps(dst, src); |
| } else { |
| LOG(FATAL) << "Unsupported SIMD type"; |
| } |
| } |
| |
| void LocationsBuilderX86_64::VisitVecNeg(HVecNeg* instruction) { |
| CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); |
| } |
| |
| void InstructionCodeGeneratorX86_64::VisitVecNeg(HVecNeg* instruction) { |
| LocationSummary* locations = instruction->GetLocations(); |
| XmmRegister src = locations->InAt(0).AsFpuRegister<XmmRegister>(); |
| XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kInt8: |
| DCHECK_EQ(16u, instruction->GetVectorLength()); |
| __ pxor(dst, dst); |
| __ psubb(dst, src); |
| break; |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| DCHECK_EQ(8u, instruction->GetVectorLength()); |
| __ pxor(dst, dst); |
| __ psubw(dst, src); |
| break; |
| case DataType::Type::kInt32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| __ pxor(dst, dst); |
| __ psubd(dst, src); |
| break; |
| case DataType::Type::kInt64: |
| DCHECK_EQ(2u, instruction->GetVectorLength()); |
| __ pxor(dst, dst); |
| __ psubq(dst, src); |
| break; |
| case DataType::Type::kFloat32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| __ xorps(dst, dst); |
| __ subps(dst, src); |
| break; |
| case DataType::Type::kFloat64: |
| DCHECK_EQ(2u, instruction->GetVectorLength()); |
| __ xorpd(dst, dst); |
| __ subpd(dst, src); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| void LocationsBuilderX86_64::VisitVecAbs(HVecAbs* instruction) { |
| CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); |
| // Integral-abs requires a temporary for the comparison. |
| if (instruction->GetPackedType() == DataType::Type::kInt32) { |
| instruction->GetLocations()->AddTemp(Location::RequiresFpuRegister()); |
| } |
| } |
| |
| void InstructionCodeGeneratorX86_64::VisitVecAbs(HVecAbs* instruction) { |
| LocationSummary* locations = instruction->GetLocations(); |
| XmmRegister src = locations->InAt(0).AsFpuRegister<XmmRegister>(); |
| XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kInt32: { |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| XmmRegister tmp = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); |
| __ movaps(dst, src); |
| __ pxor(tmp, tmp); |
| __ pcmpgtd(tmp, dst); |
| __ pxor(dst, tmp); |
| __ psubd(dst, tmp); |
| break; |
| } |
| case DataType::Type::kFloat32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| __ pcmpeqb(dst, dst); // all ones |
| __ psrld(dst, Immediate(1)); |
| __ andps(dst, src); |
| break; |
| case DataType::Type::kFloat64: |
| DCHECK_EQ(2u, instruction->GetVectorLength()); |
| __ pcmpeqb(dst, dst); // all ones |
| __ psrlq(dst, Immediate(1)); |
| __ andpd(dst, src); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| void LocationsBuilderX86_64::VisitVecNot(HVecNot* instruction) { |
| CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); |
| // Boolean-not requires a temporary to construct the 16 x one. |
| if (instruction->GetPackedType() == DataType::Type::kBool) { |
| instruction->GetLocations()->AddTemp(Location::RequiresFpuRegister()); |
| } |
| } |
| |
| void InstructionCodeGeneratorX86_64::VisitVecNot(HVecNot* instruction) { |
| LocationSummary* locations = instruction->GetLocations(); |
| XmmRegister src = locations->InAt(0).AsFpuRegister<XmmRegister>(); |
| XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kBool: { // special case boolean-not |
| DCHECK_EQ(16u, instruction->GetVectorLength()); |
| XmmRegister tmp = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); |
| __ pxor(dst, dst); |
| __ pcmpeqb(tmp, tmp); // all ones |
| __ psubb(dst, tmp); // 16 x one |
| __ pxor(dst, src); |
| break; |
| } |
| case DataType::Type::kInt8: |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| case DataType::Type::kInt32: |
| case DataType::Type::kInt64: |
| DCHECK_LE(2u, instruction->GetVectorLength()); |
| DCHECK_LE(instruction->GetVectorLength(), 16u); |
| __ pcmpeqb(dst, dst); // all ones |
| __ pxor(dst, src); |
| break; |
| case DataType::Type::kFloat32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| __ pcmpeqb(dst, dst); // all ones |
| __ xorps(dst, src); |
| break; |
| case DataType::Type::kFloat64: |
| DCHECK_EQ(2u, instruction->GetVectorLength()); |
| __ pcmpeqb(dst, dst); // all ones |
| __ xorpd(dst, src); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| // Helper to set up locations for vector binary operations. |
| static void CreateVecBinOpLocations(ArenaAllocator* arena, HVecBinaryOperation* instruction) { |
| LocationSummary* locations = new (arena) LocationSummary(instruction); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kBool: |
| case DataType::Type::kInt8: |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| case DataType::Type::kInt32: |
| case DataType::Type::kInt64: |
| case DataType::Type::kFloat32: |
| case DataType::Type::kFloat64: |
| locations->SetInAt(0, Location::RequiresFpuRegister()); |
| locations->SetInAt(1, Location::RequiresFpuRegister()); |
| locations->SetOut(Location::SameAsFirstInput()); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| void LocationsBuilderX86_64::VisitVecAdd(HVecAdd* instruction) { |
| CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); |
| } |
| |
| void InstructionCodeGeneratorX86_64::VisitVecAdd(HVecAdd* instruction) { |
| LocationSummary* locations = instruction->GetLocations(); |
| DCHECK(locations->InAt(0).Equals(locations->Out())); |
| XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); |
| XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kInt8: |
| DCHECK_EQ(16u, instruction->GetVectorLength()); |
| __ paddb(dst, src); |
| break; |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| DCHECK_EQ(8u, instruction->GetVectorLength()); |
| __ paddw(dst, src); |
| break; |
| case DataType::Type::kInt32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| __ paddd(dst, src); |
| break; |
| case DataType::Type::kInt64: |
| DCHECK_EQ(2u, instruction->GetVectorLength()); |
| __ paddq(dst, src); |
| break; |
| case DataType::Type::kFloat32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| __ addps(dst, src); |
| break; |
| case DataType::Type::kFloat64: |
| DCHECK_EQ(2u, instruction->GetVectorLength()); |
| __ addpd(dst, src); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| void LocationsBuilderX86_64::VisitVecHalvingAdd(HVecHalvingAdd* instruction) { |
| CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); |
| } |
| |
| void InstructionCodeGeneratorX86_64::VisitVecHalvingAdd(HVecHalvingAdd* instruction) { |
| LocationSummary* locations = instruction->GetLocations(); |
| DCHECK(locations->InAt(0).Equals(locations->Out())); |
| XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); |
| XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| |
| DCHECK(instruction->IsRounded()); |
| DCHECK(instruction->IsUnsigned()); |
| |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kInt8: |
| DCHECK_EQ(16u, instruction->GetVectorLength()); |
| __ pavgb(dst, src); |
| return; |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| DCHECK_EQ(8u, instruction->GetVectorLength()); |
| __ pavgw(dst, src); |
| return; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| void LocationsBuilderX86_64::VisitVecSub(HVecSub* instruction) { |
| CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); |
| } |
| |
| void InstructionCodeGeneratorX86_64::VisitVecSub(HVecSub* instruction) { |
| LocationSummary* locations = instruction->GetLocations(); |
| DCHECK(locations->InAt(0).Equals(locations->Out())); |
| XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); |
| XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kInt8: |
| DCHECK_EQ(16u, instruction->GetVectorLength()); |
| __ psubb(dst, src); |
| break; |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| DCHECK_EQ(8u, instruction->GetVectorLength()); |
| __ psubw(dst, src); |
| break; |
| case DataType::Type::kInt32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| __ psubd(dst, src); |
| break; |
| case DataType::Type::kInt64: |
| DCHECK_EQ(2u, instruction->GetVectorLength()); |
| __ psubq(dst, src); |
| break; |
| case DataType::Type::kFloat32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| __ subps(dst, src); |
| break; |
| case DataType::Type::kFloat64: |
| DCHECK_EQ(2u, instruction->GetVectorLength()); |
| __ subpd(dst, src); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| void LocationsBuilderX86_64::VisitVecMul(HVecMul* instruction) { |
| CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); |
| } |
| |
| void InstructionCodeGeneratorX86_64::VisitVecMul(HVecMul* instruction) { |
| LocationSummary* locations = instruction->GetLocations(); |
| DCHECK(locations->InAt(0).Equals(locations->Out())); |
| XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); |
| XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| DCHECK_EQ(8u, instruction->GetVectorLength()); |
| __ pmullw(dst, src); |
| break; |
| case DataType::Type::kInt32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| __ pmulld(dst, src); |
| break; |
| case DataType::Type::kFloat32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| __ mulps(dst, src); |
| break; |
| case DataType::Type::kFloat64: |
| DCHECK_EQ(2u, instruction->GetVectorLength()); |
| __ mulpd(dst, src); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| void LocationsBuilderX86_64::VisitVecDiv(HVecDiv* instruction) { |
| CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); |
| } |
| |
| void InstructionCodeGeneratorX86_64::VisitVecDiv(HVecDiv* instruction) { |
| LocationSummary* locations = instruction->GetLocations(); |
| DCHECK(locations->InAt(0).Equals(locations->Out())); |
| XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); |
| XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kFloat32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| __ divps(dst, src); |
| break; |
| case DataType::Type::kFloat64: |
| DCHECK_EQ(2u, instruction->GetVectorLength()); |
| __ divpd(dst, src); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| void LocationsBuilderX86_64::VisitVecMin(HVecMin* instruction) { |
| CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); |
| } |
| |
| void InstructionCodeGeneratorX86_64::VisitVecMin(HVecMin* instruction) { |
| LocationSummary* locations = instruction->GetLocations(); |
| DCHECK(locations->InAt(0).Equals(locations->Out())); |
| XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); |
| XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kInt8: |
| DCHECK_EQ(16u, instruction->GetVectorLength()); |
| if (instruction->IsUnsigned()) { |
| __ pminub(dst, src); |
| } else { |
| __ pminsb(dst, src); |
| } |
| break; |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| DCHECK_EQ(8u, instruction->GetVectorLength()); |
| if (instruction->IsUnsigned()) { |
| __ pminuw(dst, src); |
| } else { |
| __ pminsw(dst, src); |
| } |
| break; |
| case DataType::Type::kInt32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| if (instruction->IsUnsigned()) { |
| __ pminud(dst, src); |
| } else { |
| __ pminsd(dst, src); |
| } |
| break; |
| // Next cases are sloppy wrt 0.0 vs -0.0. |
| case DataType::Type::kFloat32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| DCHECK(!instruction->IsUnsigned()); |
| __ minps(dst, src); |
| break; |
| case DataType::Type::kFloat64: |
| DCHECK_EQ(2u, instruction->GetVectorLength()); |
| DCHECK(!instruction->IsUnsigned()); |
| __ minpd(dst, src); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| void LocationsBuilderX86_64::VisitVecMax(HVecMax* instruction) { |
| CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); |
| } |
| |
| void InstructionCodeGeneratorX86_64::VisitVecMax(HVecMax* instruction) { |
| LocationSummary* locations = instruction->GetLocations(); |
| DCHECK(locations->InAt(0).Equals(locations->Out())); |
| XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); |
| XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kInt8: |
| DCHECK_EQ(16u, instruction->GetVectorLength()); |
| if (instruction->IsUnsigned()) { |
| __ pmaxub(dst, src); |
| } else { |
| __ pmaxsb(dst, src); |
| } |
| break; |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| DCHECK_EQ(8u, instruction->GetVectorLength()); |
| if (instruction->IsUnsigned()) { |
| __ pmaxuw(dst, src); |
| } else { |
| __ pmaxsw(dst, src); |
| } |
| break; |
| case DataType::Type::kInt32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| if (instruction->IsUnsigned()) { |
| __ pmaxud(dst, src); |
| } else { |
| __ pmaxsd(dst, src); |
| } |
| break; |
| // Next cases are sloppy wrt 0.0 vs -0.0. |
| case DataType::Type::kFloat32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| DCHECK(!instruction->IsUnsigned()); |
| __ maxps(dst, src); |
| break; |
| case DataType::Type::kFloat64: |
| DCHECK_EQ(2u, instruction->GetVectorLength()); |
| DCHECK(!instruction->IsUnsigned()); |
| __ maxpd(dst, src); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| void LocationsBuilderX86_64::VisitVecAnd(HVecAnd* instruction) { |
| CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); |
| } |
| |
| void InstructionCodeGeneratorX86_64::VisitVecAnd(HVecAnd* instruction) { |
| LocationSummary* locations = instruction->GetLocations(); |
| DCHECK(locations->InAt(0).Equals(locations->Out())); |
| XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); |
| XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kBool: |
| case DataType::Type::kInt8: |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| case DataType::Type::kInt32: |
| case DataType::Type::kInt64: |
| DCHECK_LE(2u, instruction->GetVectorLength()); |
| DCHECK_LE(instruction->GetVectorLength(), 16u); |
| __ pand(dst, src); |
| break; |
| case DataType::Type::kFloat32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| __ andps(dst, src); |
| break; |
| case DataType::Type::kFloat64: |
| DCHECK_EQ(2u, instruction->GetVectorLength()); |
| __ andpd(dst, src); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| void LocationsBuilderX86_64::VisitVecAndNot(HVecAndNot* instruction) { |
| CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); |
| } |
| |
| void InstructionCodeGeneratorX86_64::VisitVecAndNot(HVecAndNot* instruction) { |
| LocationSummary* locations = instruction->GetLocations(); |
| DCHECK(locations->InAt(0).Equals(locations->Out())); |
| XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); |
| XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kBool: |
| case DataType::Type::kInt8: |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| case DataType::Type::kInt32: |
| case DataType::Type::kInt64: |
| DCHECK_LE(2u, instruction->GetVectorLength()); |
| DCHECK_LE(instruction->GetVectorLength(), 16u); |
| __ pandn(dst, src); |
| break; |
| case DataType::Type::kFloat32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| __ andnps(dst, src); |
| break; |
| case DataType::Type::kFloat64: |
| DCHECK_EQ(2u, instruction->GetVectorLength()); |
| __ andnpd(dst, src); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| void LocationsBuilderX86_64::VisitVecOr(HVecOr* instruction) { |
| CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); |
| } |
| |
| void InstructionCodeGeneratorX86_64::VisitVecOr(HVecOr* instruction) { |
| LocationSummary* locations = instruction->GetLocations(); |
| DCHECK(locations->InAt(0).Equals(locations->Out())); |
| XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); |
| XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kBool: |
| case DataType::Type::kInt8: |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| case DataType::Type::kInt32: |
| case DataType::Type::kInt64: |
| DCHECK_LE(2u, instruction->GetVectorLength()); |
| DCHECK_LE(instruction->GetVectorLength(), 16u); |
| __ por(dst, src); |
| break; |
| case DataType::Type::kFloat32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| __ orps(dst, src); |
| break; |
| case DataType::Type::kFloat64: |
| DCHECK_EQ(2u, instruction->GetVectorLength()); |
| __ orpd(dst, src); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| void LocationsBuilderX86_64::VisitVecXor(HVecXor* instruction) { |
| CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); |
| } |
| |
| void InstructionCodeGeneratorX86_64::VisitVecXor(HVecXor* instruction) { |
| LocationSummary* locations = instruction->GetLocations(); |
| DCHECK(locations->InAt(0).Equals(locations->Out())); |
| XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); |
| XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kBool: |
| case DataType::Type::kInt8: |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| case DataType::Type::kInt32: |
| case DataType::Type::kInt64: |
| DCHECK_LE(2u, instruction->GetVectorLength()); |
| DCHECK_LE(instruction->GetVectorLength(), 16u); |
| __ pxor(dst, src); |
| break; |
| case DataType::Type::kFloat32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| __ xorps(dst, src); |
| break; |
| case DataType::Type::kFloat64: |
| DCHECK_EQ(2u, instruction->GetVectorLength()); |
| __ xorpd(dst, src); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| // Helper to set up locations for vector shift operations. |
| static void CreateVecShiftLocations(ArenaAllocator* arena, HVecBinaryOperation* instruction) { |
| LocationSummary* locations = new (arena) LocationSummary(instruction); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| case DataType::Type::kInt32: |
| case DataType::Type::kInt64: |
| locations->SetInAt(0, Location::RequiresFpuRegister()); |
| locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant())); |
| locations->SetOut(Location::SameAsFirstInput()); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| void LocationsBuilderX86_64::VisitVecShl(HVecShl* instruction) { |
| CreateVecShiftLocations(GetGraph()->GetArena(), instruction); |
| } |
| |
| void InstructionCodeGeneratorX86_64::VisitVecShl(HVecShl* instruction) { |
| LocationSummary* locations = instruction->GetLocations(); |
| DCHECK(locations->InAt(0).Equals(locations->Out())); |
| int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); |
| XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| DCHECK_EQ(8u, instruction->GetVectorLength()); |
| __ psllw(dst, Immediate(static_cast<int8_t>(value))); |
| break; |
| case DataType::Type::kInt32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| __ pslld(dst, Immediate(static_cast<int8_t>(value))); |
| break; |
| case DataType::Type::kInt64: |
| DCHECK_EQ(2u, instruction->GetVectorLength()); |
| __ psllq(dst, Immediate(static_cast<int8_t>(value))); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| void LocationsBuilderX86_64::VisitVecShr(HVecShr* instruction) { |
| CreateVecShiftLocations(GetGraph()->GetArena(), instruction); |
| } |
| |
| void InstructionCodeGeneratorX86_64::VisitVecShr(HVecShr* instruction) { |
| LocationSummary* locations = instruction->GetLocations(); |
| DCHECK(locations->InAt(0).Equals(locations->Out())); |
| int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); |
| XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| DCHECK_EQ(8u, instruction->GetVectorLength()); |
| __ psraw(dst, Immediate(static_cast<int8_t>(value))); |
| break; |
| case DataType::Type::kInt32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| __ psrad(dst, Immediate(static_cast<int8_t>(value))); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| void LocationsBuilderX86_64::VisitVecUShr(HVecUShr* instruction) { |
| CreateVecShiftLocations(GetGraph()->GetArena(), instruction); |
| } |
| |
| void InstructionCodeGeneratorX86_64::VisitVecUShr(HVecUShr* instruction) { |
| LocationSummary* locations = instruction->GetLocations(); |
| DCHECK(locations->InAt(0).Equals(locations->Out())); |
| int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); |
| XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| DCHECK_EQ(8u, instruction->GetVectorLength()); |
| __ psrlw(dst, Immediate(static_cast<int8_t>(value))); |
| break; |
| case DataType::Type::kInt32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| __ psrld(dst, Immediate(static_cast<int8_t>(value))); |
| break; |
| case DataType::Type::kInt64: |
| DCHECK_EQ(2u, instruction->GetVectorLength()); |
| __ psrlq(dst, Immediate(static_cast<int8_t>(value))); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| void LocationsBuilderX86_64::VisitVecSetScalars(HVecSetScalars* instruction) { |
| LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); |
| |
| DCHECK_EQ(1u, instruction->InputCount()); // only one input currently implemented |
| |
| HInstruction* input = instruction->InputAt(0); |
| bool is_zero = IsZeroBitPattern(input); |
| |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kBool: |
| case DataType::Type::kInt8: |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| case DataType::Type::kInt32: |
| case DataType::Type::kInt64: |
| locations->SetInAt(0, is_zero ? Location::ConstantLocation(input->AsConstant()) |
| : Location::RequiresRegister()); |
| locations->SetOut(Location::RequiresFpuRegister()); |
| break; |
| case DataType::Type::kFloat32: |
| case DataType::Type::kFloat64: |
| locations->SetInAt(0, is_zero ? Location::ConstantLocation(input->AsConstant()) |
| : Location::RequiresFpuRegister()); |
| locations->SetOut(Location::RequiresFpuRegister()); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| void InstructionCodeGeneratorX86_64::VisitVecSetScalars(HVecSetScalars* instruction) { |
| LocationSummary* locations = instruction->GetLocations(); |
| XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| |
| DCHECK_EQ(1u, instruction->InputCount()); // only one input currently implemented |
| |
| // Zero out all other elements first. |
| __ xorps(dst, dst); |
| |
| // Shorthand for any type of zero. |
| if (IsZeroBitPattern(instruction->InputAt(0))) { |
| return; |
| } |
| |
| // Set required elements. |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kBool: |
| case DataType::Type::kInt8: |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: // TODO: up to here, and? |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| case DataType::Type::kInt32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| __ movd(dst, locations->InAt(0).AsRegister<CpuRegister>()); |
| break; |
| case DataType::Type::kInt64: |
| DCHECK_EQ(2u, instruction->GetVectorLength()); |
| __ movd(dst, locations->InAt(0).AsRegister<CpuRegister>()); // is 64-bit |
| break; |
| case DataType::Type::kFloat32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| __ movss(dst, locations->InAt(0).AsFpuRegister<XmmRegister>()); |
| break; |
| case DataType::Type::kFloat64: |
| DCHECK_EQ(2u, instruction->GetVectorLength()); |
| __ movsd(dst, locations->InAt(0).AsFpuRegister<XmmRegister>()); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| // Helper to set up locations for vector accumulations. |
| static void CreateVecAccumLocations(ArenaAllocator* arena, HVecOperation* instruction) { |
| LocationSummary* locations = new (arena) LocationSummary(instruction); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kInt8: |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| case DataType::Type::kInt32: |
| case DataType::Type::kInt64: |
| locations->SetInAt(0, Location::RequiresFpuRegister()); |
| locations->SetInAt(1, Location::RequiresFpuRegister()); |
| locations->SetInAt(2, Location::RequiresFpuRegister()); |
| locations->SetOut(Location::SameAsFirstInput()); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| void LocationsBuilderX86_64::VisitVecMultiplyAccumulate(HVecMultiplyAccumulate* instruction) { |
| CreateVecAccumLocations(GetGraph()->GetArena(), instruction); |
| } |
| |
| void InstructionCodeGeneratorX86_64::VisitVecMultiplyAccumulate(HVecMultiplyAccumulate* instruction) { |
| // TODO: pmaddwd? |
| LOG(FATAL) << "No SIMD for " << instruction->GetId(); |
| } |
| |
| void LocationsBuilderX86_64::VisitVecSADAccumulate(HVecSADAccumulate* instruction) { |
| CreateVecAccumLocations(GetGraph()->GetArena(), instruction); |
| } |
| |
| void InstructionCodeGeneratorX86_64::VisitVecSADAccumulate(HVecSADAccumulate* instruction) { |
| // TODO: psadbw for unsigned? |
| LOG(FATAL) << "No SIMD for " << instruction->GetId(); |
| } |
| |
| // Helper to set up locations for vector memory operations. |
| static void CreateVecMemLocations(ArenaAllocator* arena, |
| HVecMemoryOperation* instruction, |
| bool is_load) { |
| LocationSummary* locations = new (arena) LocationSummary(instruction); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kBool: |
| case DataType::Type::kInt8: |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| case DataType::Type::kInt32: |
| case DataType::Type::kInt64: |
| case DataType::Type::kFloat32: |
| case DataType::Type::kFloat64: |
| locations->SetInAt(0, Location::RequiresRegister()); |
| locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); |
| if (is_load) { |
| locations->SetOut(Location::RequiresFpuRegister()); |
| } else { |
| locations->SetInAt(2, Location::RequiresFpuRegister()); |
| } |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| // Helper to construct address for vector memory operations. |
| static Address VecAddress(LocationSummary* locations, size_t size, bool is_string_char_at) { |
| Location base = locations->InAt(0); |
| Location index = locations->InAt(1); |
| ScaleFactor scale = TIMES_1; |
| switch (size) { |
| case 2: scale = TIMES_2; break; |
| case 4: scale = TIMES_4; break; |
| case 8: scale = TIMES_8; break; |
| default: break; |
| } |
| // Incorporate the string or array offset in the address computation. |
| uint32_t offset = is_string_char_at |
| ? mirror::String::ValueOffset().Uint32Value() |
| : mirror::Array::DataOffset(size).Uint32Value(); |
| return CodeGeneratorX86_64::ArrayAddress(base.AsRegister<CpuRegister>(), index, scale, offset); |
| } |
| |
| void LocationsBuilderX86_64::VisitVecLoad(HVecLoad* instruction) { |
| CreateVecMemLocations(GetGraph()->GetArena(), instruction, /*is_load*/ true); |
| // String load requires a temporary for the compressed load. |
| if (mirror::kUseStringCompression && instruction->IsStringCharAt()) { |
| instruction->GetLocations()->AddTemp(Location::RequiresFpuRegister()); |
| } |
| } |
| |
| void InstructionCodeGeneratorX86_64::VisitVecLoad(HVecLoad* instruction) { |
| LocationSummary* locations = instruction->GetLocations(); |
| size_t size = DataType::Size(instruction->GetPackedType()); |
| Address address = VecAddress(locations, size, instruction->IsStringCharAt()); |
| XmmRegister reg = locations->Out().AsFpuRegister<XmmRegister>(); |
| bool is_aligned16 = instruction->GetAlignment().IsAlignedAt(16); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kUint16: |
| DCHECK_EQ(8u, instruction->GetVectorLength()); |
| // Special handling of compressed/uncompressed string load. |
| if (mirror::kUseStringCompression && instruction->IsStringCharAt()) { |
| NearLabel done, not_compressed; |
| XmmRegister tmp = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); |
| // Test compression bit. |
| static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, |
| "Expecting 0=compressed, 1=uncompressed"); |
| uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); |
| __ testb(Address(locations->InAt(0).AsRegister<CpuRegister>(), count_offset), Immediate(1)); |
| __ j(kNotZero, ¬_compressed); |
| // Zero extend 8 compressed bytes into 8 chars. |
| __ movsd(reg, VecAddress(locations, 1, instruction->IsStringCharAt())); |
| __ pxor(tmp, tmp); |
| __ punpcklbw(reg, tmp); |
| __ jmp(&done); |
| // Load 8 direct uncompressed chars. |
| __ Bind(¬_compressed); |
| is_aligned16 ? __ movdqa(reg, address) : __ movdqu(reg, address); |
| __ Bind(&done); |
| return; |
| } |
| FALLTHROUGH_INTENDED; |
| case DataType::Type::kBool: |
| case DataType::Type::kInt8: |
| case DataType::Type::kInt16: |
| case DataType::Type::kInt32: |
| case DataType::Type::kInt64: |
| DCHECK_LE(2u, instruction->GetVectorLength()); |
| DCHECK_LE(instruction->GetVectorLength(), 16u); |
| is_aligned16 ? __ movdqa(reg, address) : __ movdqu(reg, address); |
| break; |
| case DataType::Type::kFloat32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| is_aligned16 ? __ movaps(reg, address) : __ movups(reg, address); |
| break; |
| case DataType::Type::kFloat64: |
| DCHECK_EQ(2u, instruction->GetVectorLength()); |
| is_aligned16 ? __ movapd(reg, address) : __ movupd(reg, address); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| void LocationsBuilderX86_64::VisitVecStore(HVecStore* instruction) { |
| CreateVecMemLocations(GetGraph()->GetArena(), instruction, /*is_load*/ false); |
| } |
| |
| void InstructionCodeGeneratorX86_64::VisitVecStore(HVecStore* instruction) { |
| LocationSummary* locations = instruction->GetLocations(); |
| size_t size = DataType::Size(instruction->GetPackedType()); |
| Address address = VecAddress(locations, size, /*is_string_char_at*/ false); |
| XmmRegister reg = locations->InAt(2).AsFpuRegister<XmmRegister>(); |
| bool is_aligned16 = instruction->GetAlignment().IsAlignedAt(16); |
| switch (instruction->GetPackedType()) { |
| case DataType::Type::kBool: |
| case DataType::Type::kInt8: |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| case DataType::Type::kInt32: |
| case DataType::Type::kInt64: |
| DCHECK_LE(2u, instruction->GetVectorLength()); |
| DCHECK_LE(instruction->GetVectorLength(), 16u); |
| is_aligned16 ? __ movdqa(address, reg) : __ movdqu(address, reg); |
| break; |
| case DataType::Type::kFloat32: |
| DCHECK_EQ(4u, instruction->GetVectorLength()); |
| is_aligned16 ? __ movaps(address, reg) : __ movups(address, reg); |
| break; |
| case DataType::Type::kFloat64: |
| DCHECK_EQ(2u, instruction->GetVectorLength()); |
| is_aligned16 ? __ movapd(address, reg) : __ movupd(address, reg); |
| break; |
| default: |
| LOG(FATAL) << "Unsupported SIMD type"; |
| UNREACHABLE(); |
| } |
| } |
| |
| #undef __ |
| |
| } // namespace x86_64 |
| } // namespace art |