Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2017 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | #include "code_generator_x86.h" |
Andreas Gampe | 895f922 | 2017-07-05 09:53:32 -0700 | [diff] [blame] | 18 | |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 19 | #include "mirror/array-inl.h" |
Andreas Gampe | 895f922 | 2017-07-05 09:53:32 -0700 | [diff] [blame] | 20 | #include "mirror/string.h" |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 21 | |
| 22 | namespace art { |
| 23 | namespace x86 { |
| 24 | |
| 25 | // NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. |
| 26 | #define __ down_cast<X86Assembler*>(GetAssembler())-> // NOLINT |
| 27 | |
| 28 | void LocationsBuilderX86::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 29 | LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 30 | HInstruction* input = instruction->InputAt(0); |
| 31 | bool is_zero = IsZeroBitPattern(input); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 32 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 33 | case DataType::Type::kInt64: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 34 | // Long needs extra temporary to load from the register pair. |
| 35 | if (!is_zero) { |
| 36 | locations->AddTemp(Location::RequiresFpuRegister()); |
| 37 | } |
| 38 | FALLTHROUGH_INTENDED; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 39 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 40 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 41 | case DataType::Type::kInt8: |
| 42 | case DataType::Type::kUint16: |
| 43 | case DataType::Type::kInt16: |
| 44 | case DataType::Type::kInt32: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 45 | locations->SetInAt(0, is_zero ? Location::ConstantLocation(input->AsConstant()) |
| 46 | : Location::RequiresRegister()); |
| 47 | locations->SetOut(Location::RequiresFpuRegister()); |
| 48 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 49 | case DataType::Type::kFloat32: |
| 50 | case DataType::Type::kFloat64: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 51 | locations->SetInAt(0, is_zero ? Location::ConstantLocation(input->AsConstant()) |
| 52 | : Location::RequiresFpuRegister()); |
| 53 | locations->SetOut(is_zero ? Location::RequiresFpuRegister() |
| 54 | : Location::SameAsFirstInput()); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 55 | break; |
| 56 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 57 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 58 | UNREACHABLE(); |
| 59 | } |
| 60 | } |
| 61 | |
| 62 | void InstructionCodeGeneratorX86::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { |
| 63 | LocationSummary* locations = instruction->GetLocations(); |
| 64 | XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| 65 | |
| 66 | // Shorthand for any type of zero. |
| 67 | if (IsZeroBitPattern(instruction->InputAt(0))) { |
| 68 | __ xorps(dst, dst); |
| 69 | return; |
| 70 | } |
| 71 | |
| 72 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 73 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 74 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 75 | case DataType::Type::kInt8: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 76 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 77 | __ movd(dst, locations->InAt(0).AsRegister<Register>()); |
| 78 | __ punpcklbw(dst, dst); |
| 79 | __ punpcklwd(dst, dst); |
| 80 | __ pshufd(dst, dst, Immediate(0)); |
| 81 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 82 | case DataType::Type::kUint16: |
| 83 | case DataType::Type::kInt16: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 84 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 85 | __ movd(dst, locations->InAt(0).AsRegister<Register>()); |
| 86 | __ punpcklwd(dst, dst); |
| 87 | __ pshufd(dst, dst, Immediate(0)); |
| 88 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 89 | case DataType::Type::kInt32: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 90 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 91 | __ movd(dst, locations->InAt(0).AsRegister<Register>()); |
| 92 | __ pshufd(dst, dst, Immediate(0)); |
| 93 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 94 | case DataType::Type::kInt64: { |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 95 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
Aart Bik | b67f7e2 | 2018-01-18 13:29:19 -0800 | [diff] [blame] | 96 | XmmRegister tmp = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 97 | __ movd(dst, locations->InAt(0).AsRegisterPairLow<Register>()); |
| 98 | __ movd(tmp, locations->InAt(0).AsRegisterPairHigh<Register>()); |
| 99 | __ punpckldq(dst, tmp); |
| 100 | __ punpcklqdq(dst, dst); |
| 101 | break; |
| 102 | } |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 103 | case DataType::Type::kFloat32: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 104 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
Aart Bik | b67f7e2 | 2018-01-18 13:29:19 -0800 | [diff] [blame] | 105 | DCHECK(locations->InAt(0).Equals(locations->Out())); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 106 | __ shufps(dst, dst, Immediate(0)); |
| 107 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 108 | case DataType::Type::kFloat64: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 109 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
Aart Bik | b67f7e2 | 2018-01-18 13:29:19 -0800 | [diff] [blame] | 110 | DCHECK(locations->InAt(0).Equals(locations->Out())); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 111 | __ shufpd(dst, dst, Immediate(0)); |
| 112 | break; |
| 113 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 114 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 115 | UNREACHABLE(); |
| 116 | } |
| 117 | } |
| 118 | |
| 119 | void LocationsBuilderX86::VisitVecExtractScalar(HVecExtractScalar* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 120 | LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 121 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 122 | case DataType::Type::kInt64: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 123 | // Long needs extra temporary to store into the register pair. |
Aart Bik | a57b4ee | 2017-08-30 21:21:41 +0000 | [diff] [blame] | 124 | locations->AddTemp(Location::RequiresFpuRegister()); |
Aart Bik | 9879d0e | 2017-08-15 10:51:25 -0700 | [diff] [blame] | 125 | FALLTHROUGH_INTENDED; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 126 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 127 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 128 | case DataType::Type::kInt8: |
| 129 | case DataType::Type::kUint16: |
| 130 | case DataType::Type::kInt16: |
| 131 | case DataType::Type::kInt32: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 132 | locations->SetInAt(0, Location::RequiresFpuRegister()); |
| 133 | locations->SetOut(Location::RequiresRegister()); |
Aart Bik | 9879d0e | 2017-08-15 10:51:25 -0700 | [diff] [blame] | 134 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 135 | case DataType::Type::kFloat32: |
| 136 | case DataType::Type::kFloat64: |
Aart Bik | a57b4ee | 2017-08-30 21:21:41 +0000 | [diff] [blame] | 137 | locations->SetInAt(0, Location::RequiresFpuRegister()); |
Aart Bik | 9879d0e | 2017-08-15 10:51:25 -0700 | [diff] [blame] | 138 | locations->SetOut(Location::SameAsFirstInput()); |
| 139 | break; |
| 140 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 141 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | 9879d0e | 2017-08-15 10:51:25 -0700 | [diff] [blame] | 142 | UNREACHABLE(); |
| 143 | } |
| 144 | } |
| 145 | |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 146 | void InstructionCodeGeneratorX86::VisitVecExtractScalar(HVecExtractScalar* instruction) { |
Aart Bik | 9879d0e | 2017-08-15 10:51:25 -0700 | [diff] [blame] | 147 | LocationSummary* locations = instruction->GetLocations(); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 148 | XmmRegister src = locations->InAt(0).AsFpuRegister<XmmRegister>(); |
Aart Bik | 9879d0e | 2017-08-15 10:51:25 -0700 | [diff] [blame] | 149 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 150 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 151 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 152 | case DataType::Type::kInt8: |
| 153 | case DataType::Type::kUint16: |
| 154 | case DataType::Type::kInt16: // TODO: up to here, and? |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 155 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 156 | UNREACHABLE(); |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 157 | case DataType::Type::kInt32: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 158 | DCHECK_LE(4u, instruction->GetVectorLength()); |
| 159 | DCHECK_LE(instruction->GetVectorLength(), 16u); |
| 160 | __ movd(locations->Out().AsRegister<Register>(), src); |
Aart Bik | 9879d0e | 2017-08-15 10:51:25 -0700 | [diff] [blame] | 161 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 162 | case DataType::Type::kInt64: { |
Aart Bik | 9879d0e | 2017-08-15 10:51:25 -0700 | [diff] [blame] | 163 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
Aart Bik | b67f7e2 | 2018-01-18 13:29:19 -0800 | [diff] [blame] | 164 | XmmRegister tmp = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 165 | __ movd(locations->Out().AsRegisterPairLow<Register>(), src); |
| 166 | __ pshufd(tmp, src, Immediate(1)); |
| 167 | __ movd(locations->Out().AsRegisterPairHigh<Register>(), tmp); |
Aart Bik | 9879d0e | 2017-08-15 10:51:25 -0700 | [diff] [blame] | 168 | break; |
| 169 | } |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 170 | case DataType::Type::kFloat32: |
| 171 | case DataType::Type::kFloat64: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 172 | DCHECK_LE(2u, instruction->GetVectorLength()); |
| 173 | DCHECK_LE(instruction->GetVectorLength(), 4u); |
| 174 | DCHECK(locations->InAt(0).Equals(locations->Out())); // no code required |
Aart Bik | 9879d0e | 2017-08-15 10:51:25 -0700 | [diff] [blame] | 175 | break; |
| 176 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 177 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | 9879d0e | 2017-08-15 10:51:25 -0700 | [diff] [blame] | 178 | UNREACHABLE(); |
| 179 | } |
| 180 | } |
| 181 | |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 182 | // Helper to set up locations for vector unary operations. |
Vladimir Marko | e764d2e | 2017-10-05 14:35:55 +0100 | [diff] [blame] | 183 | static void CreateVecUnOpLocations(ArenaAllocator* allocator, HVecUnaryOperation* instruction) { |
| 184 | LocationSummary* locations = new (allocator) LocationSummary(instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 185 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 186 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 187 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 188 | case DataType::Type::kInt8: |
| 189 | case DataType::Type::kUint16: |
| 190 | case DataType::Type::kInt16: |
| 191 | case DataType::Type::kInt32: |
| 192 | case DataType::Type::kInt64: |
| 193 | case DataType::Type::kFloat32: |
| 194 | case DataType::Type::kFloat64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 195 | locations->SetInAt(0, Location::RequiresFpuRegister()); |
| 196 | locations->SetOut(Location::RequiresFpuRegister()); |
| 197 | break; |
| 198 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 199 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 200 | UNREACHABLE(); |
| 201 | } |
| 202 | } |
| 203 | |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 204 | void LocationsBuilderX86::VisitVecReduce(HVecReduce* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 205 | CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 206 | // Long reduction or min/max require a temporary. |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 207 | if (instruction->GetPackedType() == DataType::Type::kInt64 || |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 208 | instruction->GetKind() == HVecReduce::kMin || |
| 209 | instruction->GetKind() == HVecReduce::kMax) { |
| 210 | instruction->GetLocations()->AddTemp(Location::RequiresFpuRegister()); |
| 211 | } |
| 212 | } |
| 213 | |
| 214 | void InstructionCodeGeneratorX86::VisitVecReduce(HVecReduce* instruction) { |
| 215 | LocationSummary* locations = instruction->GetLocations(); |
| 216 | XmmRegister src = locations->InAt(0).AsFpuRegister<XmmRegister>(); |
| 217 | XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| 218 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 219 | case DataType::Type::kInt32: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 220 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 221 | switch (instruction->GetKind()) { |
| 222 | case HVecReduce::kSum: |
| 223 | __ movaps(dst, src); |
| 224 | __ phaddd(dst, dst); |
| 225 | __ phaddd(dst, dst); |
| 226 | break; |
| 227 | case HVecReduce::kMin: { |
| 228 | XmmRegister tmp = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); |
| 229 | __ movaps(tmp, src); |
| 230 | __ movaps(dst, src); |
| 231 | __ psrldq(tmp, Immediate(8)); |
| 232 | __ pminsd(dst, tmp); |
| 233 | __ psrldq(tmp, Immediate(4)); |
| 234 | __ pminsd(dst, tmp); |
| 235 | break; |
| 236 | } |
| 237 | case HVecReduce::kMax: { |
| 238 | XmmRegister tmp = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); |
| 239 | __ movaps(tmp, src); |
| 240 | __ movaps(dst, src); |
| 241 | __ psrldq(tmp, Immediate(8)); |
| 242 | __ pmaxsd(dst, tmp); |
| 243 | __ psrldq(tmp, Immediate(4)); |
| 244 | __ pmaxsd(dst, tmp); |
| 245 | break; |
| 246 | } |
| 247 | } |
| 248 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 249 | case DataType::Type::kInt64: { |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 250 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 251 | XmmRegister tmp = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); |
| 252 | switch (instruction->GetKind()) { |
| 253 | case HVecReduce::kSum: |
| 254 | __ movaps(tmp, src); |
| 255 | __ movaps(dst, src); |
| 256 | __ punpckhqdq(tmp, tmp); |
| 257 | __ paddq(dst, tmp); |
| 258 | break; |
| 259 | case HVecReduce::kMin: |
| 260 | case HVecReduce::kMax: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 261 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 262 | } |
| 263 | break; |
| 264 | } |
| 265 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 266 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 267 | UNREACHABLE(); |
| 268 | } |
| 269 | } |
| 270 | |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 271 | void LocationsBuilderX86::VisitVecCnv(HVecCnv* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 272 | CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 273 | } |
| 274 | |
| 275 | void InstructionCodeGeneratorX86::VisitVecCnv(HVecCnv* instruction) { |
| 276 | LocationSummary* locations = instruction->GetLocations(); |
| 277 | XmmRegister src = locations->InAt(0).AsFpuRegister<XmmRegister>(); |
| 278 | XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 279 | DataType::Type from = instruction->GetInputType(); |
| 280 | DataType::Type to = instruction->GetResultType(); |
| 281 | if (from == DataType::Type::kInt32 && to == DataType::Type::kFloat32) { |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 282 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 283 | __ cvtdq2ps(dst, src); |
| 284 | } else { |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 285 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 286 | } |
| 287 | } |
| 288 | |
| 289 | void LocationsBuilderX86::VisitVecNeg(HVecNeg* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 290 | CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 291 | } |
| 292 | |
| 293 | void InstructionCodeGeneratorX86::VisitVecNeg(HVecNeg* instruction) { |
| 294 | LocationSummary* locations = instruction->GetLocations(); |
| 295 | XmmRegister src = locations->InAt(0).AsFpuRegister<XmmRegister>(); |
| 296 | XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| 297 | switch (instruction->GetPackedType()) { |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 298 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 299 | case DataType::Type::kInt8: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 300 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 301 | __ pxor(dst, dst); |
| 302 | __ psubb(dst, src); |
| 303 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 304 | case DataType::Type::kUint16: |
| 305 | case DataType::Type::kInt16: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 306 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 307 | __ pxor(dst, dst); |
| 308 | __ psubw(dst, src); |
| 309 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 310 | case DataType::Type::kInt32: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 311 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 312 | __ pxor(dst, dst); |
| 313 | __ psubd(dst, src); |
| 314 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 315 | case DataType::Type::kInt64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 316 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 317 | __ pxor(dst, dst); |
| 318 | __ psubq(dst, src); |
| 319 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 320 | case DataType::Type::kFloat32: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 321 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 322 | __ xorps(dst, dst); |
| 323 | __ subps(dst, src); |
| 324 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 325 | case DataType::Type::kFloat64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 326 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 327 | __ xorpd(dst, dst); |
| 328 | __ subpd(dst, src); |
| 329 | break; |
| 330 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 331 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 332 | UNREACHABLE(); |
| 333 | } |
| 334 | } |
| 335 | |
Aart Bik | 6daebeb | 2017-04-03 14:35:41 -0700 | [diff] [blame] | 336 | void LocationsBuilderX86::VisitVecAbs(HVecAbs* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 337 | CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | 472821b | 2017-04-27 17:23:51 -0700 | [diff] [blame] | 338 | // Integral-abs requires a temporary for the comparison. |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 339 | if (instruction->GetPackedType() == DataType::Type::kInt32) { |
Aart Bik | 6daebeb | 2017-04-03 14:35:41 -0700 | [diff] [blame] | 340 | instruction->GetLocations()->AddTemp(Location::RequiresFpuRegister()); |
| 341 | } |
| 342 | } |
| 343 | |
| 344 | void InstructionCodeGeneratorX86::VisitVecAbs(HVecAbs* instruction) { |
| 345 | LocationSummary* locations = instruction->GetLocations(); |
| 346 | XmmRegister src = locations->InAt(0).AsFpuRegister<XmmRegister>(); |
| 347 | XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| 348 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 349 | case DataType::Type::kInt32: { |
Aart Bik | 6daebeb | 2017-04-03 14:35:41 -0700 | [diff] [blame] | 350 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 351 | XmmRegister tmp = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); |
| 352 | __ movaps(dst, src); |
| 353 | __ pxor(tmp, tmp); |
| 354 | __ pcmpgtd(tmp, dst); |
| 355 | __ pxor(dst, tmp); |
| 356 | __ psubd(dst, tmp); |
| 357 | break; |
| 358 | } |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 359 | case DataType::Type::kFloat32: |
Aart Bik | 6daebeb | 2017-04-03 14:35:41 -0700 | [diff] [blame] | 360 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 361 | __ pcmpeqb(dst, dst); // all ones |
| 362 | __ psrld(dst, Immediate(1)); |
| 363 | __ andps(dst, src); |
| 364 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 365 | case DataType::Type::kFloat64: |
Aart Bik | 6daebeb | 2017-04-03 14:35:41 -0700 | [diff] [blame] | 366 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 367 | __ pcmpeqb(dst, dst); // all ones |
| 368 | __ psrlq(dst, Immediate(1)); |
| 369 | __ andpd(dst, src); |
| 370 | break; |
| 371 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 372 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | 6daebeb | 2017-04-03 14:35:41 -0700 | [diff] [blame] | 373 | UNREACHABLE(); |
| 374 | } |
| 375 | } |
| 376 | |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 377 | void LocationsBuilderX86::VisitVecNot(HVecNot* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 378 | CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 379 | // Boolean-not requires a temporary to construct the 16 x one. |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 380 | if (instruction->GetPackedType() == DataType::Type::kBool) { |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 381 | instruction->GetLocations()->AddTemp(Location::RequiresFpuRegister()); |
| 382 | } |
| 383 | } |
| 384 | |
| 385 | void InstructionCodeGeneratorX86::VisitVecNot(HVecNot* instruction) { |
| 386 | LocationSummary* locations = instruction->GetLocations(); |
| 387 | XmmRegister src = locations->InAt(0).AsFpuRegister<XmmRegister>(); |
| 388 | XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| 389 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 390 | case DataType::Type::kBool: { // special case boolean-not |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 391 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 392 | XmmRegister tmp = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); |
| 393 | __ pxor(dst, dst); |
| 394 | __ pcmpeqb(tmp, tmp); // all ones |
| 395 | __ psubb(dst, tmp); // 16 x one |
| 396 | __ pxor(dst, src); |
| 397 | break; |
| 398 | } |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 399 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 400 | case DataType::Type::kInt8: |
| 401 | case DataType::Type::kUint16: |
| 402 | case DataType::Type::kInt16: |
| 403 | case DataType::Type::kInt32: |
| 404 | case DataType::Type::kInt64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 405 | DCHECK_LE(2u, instruction->GetVectorLength()); |
| 406 | DCHECK_LE(instruction->GetVectorLength(), 16u); |
| 407 | __ pcmpeqb(dst, dst); // all ones |
| 408 | __ pxor(dst, src); |
| 409 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 410 | case DataType::Type::kFloat32: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 411 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 412 | __ pcmpeqb(dst, dst); // all ones |
| 413 | __ xorps(dst, src); |
| 414 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 415 | case DataType::Type::kFloat64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 416 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 417 | __ pcmpeqb(dst, dst); // all ones |
| 418 | __ xorpd(dst, src); |
| 419 | break; |
| 420 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 421 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 422 | UNREACHABLE(); |
| 423 | } |
| 424 | } |
| 425 | |
| 426 | // Helper to set up locations for vector binary operations. |
Vladimir Marko | e764d2e | 2017-10-05 14:35:55 +0100 | [diff] [blame] | 427 | static void CreateVecBinOpLocations(ArenaAllocator* allocator, HVecBinaryOperation* instruction) { |
| 428 | LocationSummary* locations = new (allocator) LocationSummary(instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 429 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 430 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 431 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 432 | case DataType::Type::kInt8: |
| 433 | case DataType::Type::kUint16: |
| 434 | case DataType::Type::kInt16: |
| 435 | case DataType::Type::kInt32: |
| 436 | case DataType::Type::kInt64: |
| 437 | case DataType::Type::kFloat32: |
| 438 | case DataType::Type::kFloat64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 439 | locations->SetInAt(0, Location::RequiresFpuRegister()); |
| 440 | locations->SetInAt(1, Location::RequiresFpuRegister()); |
| 441 | locations->SetOut(Location::SameAsFirstInput()); |
| 442 | break; |
| 443 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 444 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 445 | UNREACHABLE(); |
| 446 | } |
| 447 | } |
| 448 | |
| 449 | void LocationsBuilderX86::VisitVecAdd(HVecAdd* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 450 | CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 451 | } |
| 452 | |
| 453 | void InstructionCodeGeneratorX86::VisitVecAdd(HVecAdd* instruction) { |
| 454 | LocationSummary* locations = instruction->GetLocations(); |
| 455 | DCHECK(locations->InAt(0).Equals(locations->Out())); |
| 456 | XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); |
| 457 | XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| 458 | switch (instruction->GetPackedType()) { |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 459 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 460 | case DataType::Type::kInt8: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 461 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 462 | __ paddb(dst, src); |
| 463 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 464 | case DataType::Type::kUint16: |
| 465 | case DataType::Type::kInt16: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 466 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 467 | __ paddw(dst, src); |
| 468 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 469 | case DataType::Type::kInt32: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 470 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 471 | __ paddd(dst, src); |
| 472 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 473 | case DataType::Type::kInt64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 474 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 475 | __ paddq(dst, src); |
| 476 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 477 | case DataType::Type::kFloat32: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 478 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 479 | __ addps(dst, src); |
| 480 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 481 | case DataType::Type::kFloat64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 482 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 483 | __ addpd(dst, src); |
| 484 | break; |
| 485 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 486 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
| 487 | UNREACHABLE(); |
| 488 | } |
| 489 | } |
| 490 | |
| 491 | void LocationsBuilderX86::VisitVecSaturationAdd(HVecSaturationAdd* instruction) { |
| 492 | CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); |
| 493 | } |
| 494 | |
| 495 | void InstructionCodeGeneratorX86::VisitVecSaturationAdd(HVecSaturationAdd* instruction) { |
| 496 | LocationSummary* locations = instruction->GetLocations(); |
| 497 | DCHECK(locations->InAt(0).Equals(locations->Out())); |
| 498 | XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); |
| 499 | XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| 500 | switch (instruction->GetPackedType()) { |
| 501 | case DataType::Type::kUint8: |
| 502 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 503 | __ paddusb(dst, src); |
| 504 | break; |
| 505 | case DataType::Type::kInt8: |
| 506 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 507 | __ paddsb(dst, src); |
| 508 | break; |
| 509 | case DataType::Type::kUint16: |
| 510 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 511 | __ paddusw(dst, src); |
| 512 | break; |
| 513 | case DataType::Type::kInt16: |
| 514 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 515 | __ paddsw(dst, src); |
| 516 | break; |
| 517 | default: |
| 518 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 519 | UNREACHABLE(); |
| 520 | } |
| 521 | } |
| 522 | |
Aart Bik | f3e61ee | 2017-04-12 17:09:20 -0700 | [diff] [blame] | 523 | void LocationsBuilderX86::VisitVecHalvingAdd(HVecHalvingAdd* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 524 | CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f3e61ee | 2017-04-12 17:09:20 -0700 | [diff] [blame] | 525 | } |
| 526 | |
| 527 | void InstructionCodeGeneratorX86::VisitVecHalvingAdd(HVecHalvingAdd* instruction) { |
| 528 | LocationSummary* locations = instruction->GetLocations(); |
| 529 | DCHECK(locations->InAt(0).Equals(locations->Out())); |
| 530 | XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); |
| 531 | XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| 532 | |
| 533 | DCHECK(instruction->IsRounded()); |
Aart Bik | f3e61ee | 2017-04-12 17:09:20 -0700 | [diff] [blame] | 534 | |
| 535 | switch (instruction->GetPackedType()) { |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 536 | case DataType::Type::kUint8: |
Aart Bik | f3e61ee | 2017-04-12 17:09:20 -0700 | [diff] [blame] | 537 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 538 | __ pavgb(dst, src); |
| 539 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 540 | case DataType::Type::kUint16: |
Aart Bik | f3e61ee | 2017-04-12 17:09:20 -0700 | [diff] [blame] | 541 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 542 | __ pavgw(dst, src); |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 543 | break; |
Aart Bik | f3e61ee | 2017-04-12 17:09:20 -0700 | [diff] [blame] | 544 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 545 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f3e61ee | 2017-04-12 17:09:20 -0700 | [diff] [blame] | 546 | UNREACHABLE(); |
| 547 | } |
| 548 | } |
| 549 | |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 550 | void LocationsBuilderX86::VisitVecSub(HVecSub* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 551 | CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 552 | } |
| 553 | |
| 554 | void InstructionCodeGeneratorX86::VisitVecSub(HVecSub* instruction) { |
| 555 | LocationSummary* locations = instruction->GetLocations(); |
| 556 | DCHECK(locations->InAt(0).Equals(locations->Out())); |
| 557 | XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); |
| 558 | XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| 559 | switch (instruction->GetPackedType()) { |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 560 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 561 | case DataType::Type::kInt8: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 562 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 563 | __ psubb(dst, src); |
| 564 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 565 | case DataType::Type::kUint16: |
| 566 | case DataType::Type::kInt16: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 567 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 568 | __ psubw(dst, src); |
| 569 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 570 | case DataType::Type::kInt32: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 571 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 572 | __ psubd(dst, src); |
| 573 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 574 | case DataType::Type::kInt64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 575 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 576 | __ psubq(dst, src); |
| 577 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 578 | case DataType::Type::kFloat32: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 579 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 580 | __ subps(dst, src); |
| 581 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 582 | case DataType::Type::kFloat64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 583 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 584 | __ subpd(dst, src); |
| 585 | break; |
| 586 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 587 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
| 588 | UNREACHABLE(); |
| 589 | } |
| 590 | } |
| 591 | |
| 592 | void LocationsBuilderX86::VisitVecSaturationSub(HVecSaturationSub* instruction) { |
| 593 | CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); |
| 594 | } |
| 595 | |
| 596 | void InstructionCodeGeneratorX86::VisitVecSaturationSub(HVecSaturationSub* instruction) { |
| 597 | LocationSummary* locations = instruction->GetLocations(); |
| 598 | DCHECK(locations->InAt(0).Equals(locations->Out())); |
| 599 | XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); |
| 600 | XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| 601 | switch (instruction->GetPackedType()) { |
| 602 | case DataType::Type::kUint8: |
| 603 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 604 | __ psubusb(dst, src); |
| 605 | break; |
| 606 | case DataType::Type::kInt8: |
| 607 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 608 | __ psubsb(dst, src); |
| 609 | break; |
| 610 | case DataType::Type::kUint16: |
| 611 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 612 | __ psubusw(dst, src); |
| 613 | break; |
| 614 | case DataType::Type::kInt16: |
| 615 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 616 | __ psubsw(dst, src); |
| 617 | break; |
| 618 | default: |
| 619 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 620 | UNREACHABLE(); |
| 621 | } |
| 622 | } |
| 623 | |
| 624 | void LocationsBuilderX86::VisitVecMul(HVecMul* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 625 | CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 626 | } |
| 627 | |
| 628 | void InstructionCodeGeneratorX86::VisitVecMul(HVecMul* instruction) { |
| 629 | LocationSummary* locations = instruction->GetLocations(); |
| 630 | DCHECK(locations->InAt(0).Equals(locations->Out())); |
| 631 | XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); |
| 632 | XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| 633 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 634 | case DataType::Type::kUint16: |
| 635 | case DataType::Type::kInt16: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 636 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 637 | __ pmullw(dst, src); |
| 638 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 639 | case DataType::Type::kInt32: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 640 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 641 | __ pmulld(dst, src); |
| 642 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 643 | case DataType::Type::kFloat32: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 644 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 645 | __ mulps(dst, src); |
| 646 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 647 | case DataType::Type::kFloat64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 648 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 649 | __ mulpd(dst, src); |
| 650 | break; |
| 651 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 652 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 653 | UNREACHABLE(); |
| 654 | } |
| 655 | } |
| 656 | |
| 657 | void LocationsBuilderX86::VisitVecDiv(HVecDiv* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 658 | CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 659 | } |
| 660 | |
| 661 | void InstructionCodeGeneratorX86::VisitVecDiv(HVecDiv* instruction) { |
| 662 | LocationSummary* locations = instruction->GetLocations(); |
| 663 | DCHECK(locations->InAt(0).Equals(locations->Out())); |
| 664 | XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); |
| 665 | XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| 666 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 667 | case DataType::Type::kFloat32: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 668 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 669 | __ divps(dst, src); |
| 670 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 671 | case DataType::Type::kFloat64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 672 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 673 | __ divpd(dst, src); |
| 674 | break; |
| 675 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 676 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 677 | UNREACHABLE(); |
| 678 | } |
| 679 | } |
| 680 | |
Aart Bik | f3e61ee | 2017-04-12 17:09:20 -0700 | [diff] [blame] | 681 | void LocationsBuilderX86::VisitVecMin(HVecMin* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 682 | CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f3e61ee | 2017-04-12 17:09:20 -0700 | [diff] [blame] | 683 | } |
| 684 | |
| 685 | void InstructionCodeGeneratorX86::VisitVecMin(HVecMin* instruction) { |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 686 | LocationSummary* locations = instruction->GetLocations(); |
| 687 | DCHECK(locations->InAt(0).Equals(locations->Out())); |
| 688 | XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); |
| 689 | XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| 690 | switch (instruction->GetPackedType()) { |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 691 | case DataType::Type::kUint8: |
| 692 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 693 | __ pminub(dst, src); |
| 694 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 695 | case DataType::Type::kInt8: |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 696 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 697 | __ pminsb(dst, src); |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 698 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 699 | case DataType::Type::kUint16: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 700 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 701 | __ pminuw(dst, src); |
| 702 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 703 | case DataType::Type::kInt16: |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 704 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 705 | __ pminsw(dst, src); |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 706 | break; |
Aart Bik | 66c158e | 2018-01-31 12:55:04 -0800 | [diff] [blame] | 707 | case DataType::Type::kUint32: |
| 708 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 709 | __ pminud(dst, src); |
| 710 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 711 | case DataType::Type::kInt32: |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 712 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
Aart Bik | 66c158e | 2018-01-31 12:55:04 -0800 | [diff] [blame] | 713 | __ pminsd(dst, src); |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 714 | break; |
| 715 | // Next cases are sloppy wrt 0.0 vs -0.0. |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 716 | case DataType::Type::kFloat32: |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 717 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 718 | __ minps(dst, src); |
| 719 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 720 | case DataType::Type::kFloat64: |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 721 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 722 | __ minpd(dst, src); |
| 723 | break; |
| 724 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 725 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 726 | UNREACHABLE(); |
| 727 | } |
Aart Bik | f3e61ee | 2017-04-12 17:09:20 -0700 | [diff] [blame] | 728 | } |
| 729 | |
| 730 | void LocationsBuilderX86::VisitVecMax(HVecMax* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 731 | CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f3e61ee | 2017-04-12 17:09:20 -0700 | [diff] [blame] | 732 | } |
| 733 | |
| 734 | void InstructionCodeGeneratorX86::VisitVecMax(HVecMax* instruction) { |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 735 | LocationSummary* locations = instruction->GetLocations(); |
| 736 | DCHECK(locations->InAt(0).Equals(locations->Out())); |
| 737 | XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); |
| 738 | XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| 739 | switch (instruction->GetPackedType()) { |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 740 | case DataType::Type::kUint8: |
| 741 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 742 | __ pmaxub(dst, src); |
| 743 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 744 | case DataType::Type::kInt8: |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 745 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 746 | __ pmaxsb(dst, src); |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 747 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 748 | case DataType::Type::kUint16: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 749 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 750 | __ pmaxuw(dst, src); |
| 751 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 752 | case DataType::Type::kInt16: |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 753 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 754 | __ pmaxsw(dst, src); |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 755 | break; |
Aart Bik | 66c158e | 2018-01-31 12:55:04 -0800 | [diff] [blame] | 756 | case DataType::Type::kUint32: |
| 757 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 758 | __ pmaxud(dst, src); |
| 759 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 760 | case DataType::Type::kInt32: |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 761 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
Aart Bik | 66c158e | 2018-01-31 12:55:04 -0800 | [diff] [blame] | 762 | __ pmaxsd(dst, src); |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 763 | break; |
| 764 | // Next cases are sloppy wrt 0.0 vs -0.0. |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 765 | case DataType::Type::kFloat32: |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 766 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 767 | __ maxps(dst, src); |
| 768 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 769 | case DataType::Type::kFloat64: |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 770 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 771 | __ maxpd(dst, src); |
| 772 | break; |
| 773 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 774 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 775 | UNREACHABLE(); |
| 776 | } |
Aart Bik | f3e61ee | 2017-04-12 17:09:20 -0700 | [diff] [blame] | 777 | } |
| 778 | |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 779 | void LocationsBuilderX86::VisitVecAnd(HVecAnd* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 780 | CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 781 | } |
| 782 | |
| 783 | void InstructionCodeGeneratorX86::VisitVecAnd(HVecAnd* instruction) { |
| 784 | LocationSummary* locations = instruction->GetLocations(); |
| 785 | DCHECK(locations->InAt(0).Equals(locations->Out())); |
| 786 | XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); |
| 787 | XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| 788 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 789 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 790 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 791 | case DataType::Type::kInt8: |
| 792 | case DataType::Type::kUint16: |
| 793 | case DataType::Type::kInt16: |
| 794 | case DataType::Type::kInt32: |
| 795 | case DataType::Type::kInt64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 796 | DCHECK_LE(2u, instruction->GetVectorLength()); |
| 797 | DCHECK_LE(instruction->GetVectorLength(), 16u); |
| 798 | __ pand(dst, src); |
| 799 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 800 | case DataType::Type::kFloat32: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 801 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 802 | __ andps(dst, src); |
| 803 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 804 | case DataType::Type::kFloat64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 805 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 806 | __ andpd(dst, src); |
| 807 | break; |
| 808 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 809 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 810 | UNREACHABLE(); |
| 811 | } |
| 812 | } |
| 813 | |
| 814 | void LocationsBuilderX86::VisitVecAndNot(HVecAndNot* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 815 | CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 816 | } |
| 817 | |
| 818 | void InstructionCodeGeneratorX86::VisitVecAndNot(HVecAndNot* instruction) { |
| 819 | LocationSummary* locations = instruction->GetLocations(); |
| 820 | DCHECK(locations->InAt(0).Equals(locations->Out())); |
| 821 | XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); |
| 822 | XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| 823 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 824 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 825 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 826 | case DataType::Type::kInt8: |
| 827 | case DataType::Type::kUint16: |
| 828 | case DataType::Type::kInt16: |
| 829 | case DataType::Type::kInt32: |
| 830 | case DataType::Type::kInt64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 831 | DCHECK_LE(2u, instruction->GetVectorLength()); |
| 832 | DCHECK_LE(instruction->GetVectorLength(), 16u); |
| 833 | __ pandn(dst, src); |
| 834 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 835 | case DataType::Type::kFloat32: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 836 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 837 | __ andnps(dst, src); |
| 838 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 839 | case DataType::Type::kFloat64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 840 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 841 | __ andnpd(dst, src); |
| 842 | break; |
| 843 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 844 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 845 | UNREACHABLE(); |
| 846 | } |
| 847 | } |
| 848 | |
| 849 | void LocationsBuilderX86::VisitVecOr(HVecOr* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 850 | CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 851 | } |
| 852 | |
| 853 | void InstructionCodeGeneratorX86::VisitVecOr(HVecOr* instruction) { |
| 854 | LocationSummary* locations = instruction->GetLocations(); |
| 855 | DCHECK(locations->InAt(0).Equals(locations->Out())); |
| 856 | XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); |
| 857 | XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| 858 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 859 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 860 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 861 | case DataType::Type::kInt8: |
| 862 | case DataType::Type::kUint16: |
| 863 | case DataType::Type::kInt16: |
| 864 | case DataType::Type::kInt32: |
| 865 | case DataType::Type::kInt64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 866 | DCHECK_LE(2u, instruction->GetVectorLength()); |
| 867 | DCHECK_LE(instruction->GetVectorLength(), 16u); |
| 868 | __ por(dst, src); |
| 869 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 870 | case DataType::Type::kFloat32: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 871 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 872 | __ orps(dst, src); |
| 873 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 874 | case DataType::Type::kFloat64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 875 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 876 | __ orpd(dst, src); |
| 877 | break; |
| 878 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 879 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 880 | UNREACHABLE(); |
| 881 | } |
| 882 | } |
| 883 | |
| 884 | void LocationsBuilderX86::VisitVecXor(HVecXor* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 885 | CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 886 | } |
| 887 | |
| 888 | void InstructionCodeGeneratorX86::VisitVecXor(HVecXor* instruction) { |
| 889 | LocationSummary* locations = instruction->GetLocations(); |
| 890 | DCHECK(locations->InAt(0).Equals(locations->Out())); |
| 891 | XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); |
| 892 | XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| 893 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 894 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 895 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 896 | case DataType::Type::kInt8: |
| 897 | case DataType::Type::kUint16: |
| 898 | case DataType::Type::kInt16: |
| 899 | case DataType::Type::kInt32: |
| 900 | case DataType::Type::kInt64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 901 | DCHECK_LE(2u, instruction->GetVectorLength()); |
| 902 | DCHECK_LE(instruction->GetVectorLength(), 16u); |
| 903 | __ pxor(dst, src); |
| 904 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 905 | case DataType::Type::kFloat32: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 906 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 907 | __ xorps(dst, src); |
| 908 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 909 | case DataType::Type::kFloat64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 910 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 911 | __ xorpd(dst, src); |
| 912 | break; |
| 913 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 914 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 915 | UNREACHABLE(); |
| 916 | } |
| 917 | } |
| 918 | |
| 919 | // Helper to set up locations for vector shift operations. |
Vladimir Marko | e764d2e | 2017-10-05 14:35:55 +0100 | [diff] [blame] | 920 | static void CreateVecShiftLocations(ArenaAllocator* allocator, HVecBinaryOperation* instruction) { |
| 921 | LocationSummary* locations = new (allocator) LocationSummary(instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 922 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 923 | case DataType::Type::kUint16: |
| 924 | case DataType::Type::kInt16: |
| 925 | case DataType::Type::kInt32: |
| 926 | case DataType::Type::kInt64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 927 | locations->SetInAt(0, Location::RequiresFpuRegister()); |
| 928 | locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant())); |
| 929 | locations->SetOut(Location::SameAsFirstInput()); |
| 930 | break; |
| 931 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 932 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 933 | UNREACHABLE(); |
| 934 | } |
| 935 | } |
| 936 | |
| 937 | void LocationsBuilderX86::VisitVecShl(HVecShl* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 938 | CreateVecShiftLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 939 | } |
| 940 | |
| 941 | void InstructionCodeGeneratorX86::VisitVecShl(HVecShl* instruction) { |
| 942 | LocationSummary* locations = instruction->GetLocations(); |
| 943 | DCHECK(locations->InAt(0).Equals(locations->Out())); |
| 944 | int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); |
| 945 | XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| 946 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 947 | case DataType::Type::kUint16: |
| 948 | case DataType::Type::kInt16: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 949 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 950 | __ psllw(dst, Immediate(static_cast<uint8_t>(value))); |
| 951 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 952 | case DataType::Type::kInt32: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 953 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 954 | __ pslld(dst, Immediate(static_cast<uint8_t>(value))); |
| 955 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 956 | case DataType::Type::kInt64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 957 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 958 | __ psllq(dst, Immediate(static_cast<uint8_t>(value))); |
| 959 | break; |
| 960 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 961 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 962 | UNREACHABLE(); |
| 963 | } |
| 964 | } |
| 965 | |
| 966 | void LocationsBuilderX86::VisitVecShr(HVecShr* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 967 | CreateVecShiftLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 968 | } |
| 969 | |
| 970 | void InstructionCodeGeneratorX86::VisitVecShr(HVecShr* instruction) { |
| 971 | LocationSummary* locations = instruction->GetLocations(); |
| 972 | DCHECK(locations->InAt(0).Equals(locations->Out())); |
| 973 | int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); |
| 974 | XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| 975 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 976 | case DataType::Type::kUint16: |
| 977 | case DataType::Type::kInt16: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 978 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 979 | __ psraw(dst, Immediate(static_cast<uint8_t>(value))); |
| 980 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 981 | case DataType::Type::kInt32: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 982 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 983 | __ psrad(dst, Immediate(static_cast<uint8_t>(value))); |
| 984 | break; |
| 985 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 986 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 987 | UNREACHABLE(); |
| 988 | } |
| 989 | } |
| 990 | |
| 991 | void LocationsBuilderX86::VisitVecUShr(HVecUShr* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 992 | CreateVecShiftLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 993 | } |
| 994 | |
| 995 | void InstructionCodeGeneratorX86::VisitVecUShr(HVecUShr* instruction) { |
| 996 | LocationSummary* locations = instruction->GetLocations(); |
| 997 | DCHECK(locations->InAt(0).Equals(locations->Out())); |
| 998 | int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); |
| 999 | XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| 1000 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1001 | case DataType::Type::kUint16: |
| 1002 | case DataType::Type::kInt16: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1003 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 1004 | __ psrlw(dst, Immediate(static_cast<uint8_t>(value))); |
| 1005 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1006 | case DataType::Type::kInt32: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1007 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 1008 | __ psrld(dst, Immediate(static_cast<uint8_t>(value))); |
| 1009 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1010 | case DataType::Type::kInt64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1011 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 1012 | __ psrlq(dst, Immediate(static_cast<uint8_t>(value))); |
| 1013 | break; |
| 1014 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 1015 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1016 | UNREACHABLE(); |
| 1017 | } |
| 1018 | } |
| 1019 | |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 1020 | void LocationsBuilderX86::VisitVecSetScalars(HVecSetScalars* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 1021 | LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 1022 | |
| 1023 | DCHECK_EQ(1u, instruction->InputCount()); // only one input currently implemented |
| 1024 | |
| 1025 | HInstruction* input = instruction->InputAt(0); |
| 1026 | bool is_zero = IsZeroBitPattern(input); |
| 1027 | |
| 1028 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1029 | case DataType::Type::kInt64: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 1030 | // Long needs extra temporary to load from register pairs. |
| 1031 | if (!is_zero) { |
| 1032 | locations->AddTemp(Location::RequiresFpuRegister()); |
| 1033 | } |
| 1034 | FALLTHROUGH_INTENDED; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1035 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 1036 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1037 | case DataType::Type::kInt8: |
| 1038 | case DataType::Type::kUint16: |
| 1039 | case DataType::Type::kInt16: |
| 1040 | case DataType::Type::kInt32: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 1041 | locations->SetInAt(0, is_zero ? Location::ConstantLocation(input->AsConstant()) |
| 1042 | : Location::RequiresRegister()); |
| 1043 | locations->SetOut(Location::RequiresFpuRegister()); |
| 1044 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1045 | case DataType::Type::kFloat32: |
| 1046 | case DataType::Type::kFloat64: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 1047 | locations->SetInAt(0, is_zero ? Location::ConstantLocation(input->AsConstant()) |
| 1048 | : Location::RequiresFpuRegister()); |
| 1049 | locations->SetOut(Location::RequiresFpuRegister()); |
| 1050 | break; |
| 1051 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 1052 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 1053 | UNREACHABLE(); |
| 1054 | } |
| 1055 | } |
| 1056 | |
| 1057 | void InstructionCodeGeneratorX86::VisitVecSetScalars(HVecSetScalars* instruction) { |
| 1058 | LocationSummary* locations = instruction->GetLocations(); |
| 1059 | XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); |
| 1060 | |
| 1061 | DCHECK_EQ(1u, instruction->InputCount()); // only one input currently implemented |
| 1062 | |
| 1063 | // Zero out all other elements first. |
| 1064 | __ xorps(dst, dst); |
| 1065 | |
| 1066 | // Shorthand for any type of zero. |
| 1067 | if (IsZeroBitPattern(instruction->InputAt(0))) { |
| 1068 | return; |
| 1069 | } |
| 1070 | |
| 1071 | // Set required elements. |
| 1072 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1073 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 1074 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1075 | case DataType::Type::kInt8: |
| 1076 | case DataType::Type::kUint16: |
| 1077 | case DataType::Type::kInt16: // TODO: up to here, and? |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 1078 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 1079 | UNREACHABLE(); |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1080 | case DataType::Type::kInt32: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 1081 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 1082 | __ movd(dst, locations->InAt(0).AsRegister<Register>()); |
| 1083 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1084 | case DataType::Type::kInt64: { |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 1085 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
Aart Bik | b67f7e2 | 2018-01-18 13:29:19 -0800 | [diff] [blame] | 1086 | XmmRegister tmp = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 1087 | __ xorps(tmp, tmp); |
| 1088 | __ movd(dst, locations->InAt(0).AsRegisterPairLow<Register>()); |
| 1089 | __ movd(tmp, locations->InAt(0).AsRegisterPairHigh<Register>()); |
| 1090 | __ punpckldq(dst, tmp); |
| 1091 | break; |
| 1092 | } |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1093 | case DataType::Type::kFloat32: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 1094 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 1095 | __ movss(dst, locations->InAt(1).AsFpuRegister<XmmRegister>()); |
| 1096 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1097 | case DataType::Type::kFloat64: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 1098 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 1099 | __ movsd(dst, locations->InAt(1).AsFpuRegister<XmmRegister>()); |
| 1100 | break; |
| 1101 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 1102 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 1103 | UNREACHABLE(); |
| 1104 | } |
| 1105 | } |
| 1106 | |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1107 | // Helper to set up locations for vector accumulations. |
Vladimir Marko | e764d2e | 2017-10-05 14:35:55 +0100 | [diff] [blame] | 1108 | static void CreateVecAccumLocations(ArenaAllocator* allocator, HVecOperation* instruction) { |
| 1109 | LocationSummary* locations = new (allocator) LocationSummary(instruction); |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1110 | switch (instruction->GetPackedType()) { |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 1111 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1112 | case DataType::Type::kInt8: |
| 1113 | case DataType::Type::kUint16: |
| 1114 | case DataType::Type::kInt16: |
| 1115 | case DataType::Type::kInt32: |
| 1116 | case DataType::Type::kInt64: |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1117 | locations->SetInAt(0, Location::RequiresFpuRegister()); |
| 1118 | locations->SetInAt(1, Location::RequiresFpuRegister()); |
| 1119 | locations->SetInAt(2, Location::RequiresFpuRegister()); |
| 1120 | locations->SetOut(Location::SameAsFirstInput()); |
| 1121 | break; |
| 1122 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 1123 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1124 | UNREACHABLE(); |
| 1125 | } |
Artem Serov | f34dd20 | 2017-04-10 17:41:46 +0100 | [diff] [blame] | 1126 | } |
| 1127 | |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1128 | void LocationsBuilderX86::VisitVecMultiplyAccumulate(HVecMultiplyAccumulate* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 1129 | CreateVecAccumLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1130 | } |
| 1131 | |
| 1132 | void InstructionCodeGeneratorX86::VisitVecMultiplyAccumulate(HVecMultiplyAccumulate* instruction) { |
| 1133 | // TODO: pmaddwd? |
| 1134 | LOG(FATAL) << "No SIMD for " << instruction->GetId(); |
| 1135 | } |
| 1136 | |
| 1137 | void LocationsBuilderX86::VisitVecSADAccumulate(HVecSADAccumulate* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 1138 | CreateVecAccumLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1139 | } |
| 1140 | |
| 1141 | void InstructionCodeGeneratorX86::VisitVecSADAccumulate(HVecSADAccumulate* instruction) { |
| 1142 | // TODO: psadbw for unsigned? |
| 1143 | LOG(FATAL) << "No SIMD for " << instruction->GetId(); |
Artem Serov | f34dd20 | 2017-04-10 17:41:46 +0100 | [diff] [blame] | 1144 | } |
| 1145 | |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1146 | // Helper to set up locations for vector memory operations. |
Vladimir Marko | e764d2e | 2017-10-05 14:35:55 +0100 | [diff] [blame] | 1147 | static void CreateVecMemLocations(ArenaAllocator* allocator, |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1148 | HVecMemoryOperation* instruction, |
| 1149 | bool is_load) { |
Vladimir Marko | e764d2e | 2017-10-05 14:35:55 +0100 | [diff] [blame] | 1150 | LocationSummary* locations = new (allocator) LocationSummary(instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1151 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1152 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 1153 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1154 | case DataType::Type::kInt8: |
| 1155 | case DataType::Type::kUint16: |
| 1156 | case DataType::Type::kInt16: |
| 1157 | case DataType::Type::kInt32: |
| 1158 | case DataType::Type::kInt64: |
| 1159 | case DataType::Type::kFloat32: |
| 1160 | case DataType::Type::kFloat64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1161 | locations->SetInAt(0, Location::RequiresRegister()); |
| 1162 | locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); |
| 1163 | if (is_load) { |
| 1164 | locations->SetOut(Location::RequiresFpuRegister()); |
| 1165 | } else { |
| 1166 | locations->SetInAt(2, Location::RequiresFpuRegister()); |
| 1167 | } |
| 1168 | break; |
| 1169 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 1170 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1171 | UNREACHABLE(); |
| 1172 | } |
| 1173 | } |
| 1174 | |
Aart Bik | 472821b | 2017-04-27 17:23:51 -0700 | [diff] [blame] | 1175 | // Helper to construct address for vector memory operations. |
| 1176 | static Address VecAddress(LocationSummary* locations, size_t size, bool is_string_char_at) { |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1177 | Location base = locations->InAt(0); |
| 1178 | Location index = locations->InAt(1); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1179 | ScaleFactor scale = TIMES_1; |
| 1180 | switch (size) { |
| 1181 | case 2: scale = TIMES_2; break; |
| 1182 | case 4: scale = TIMES_4; break; |
| 1183 | case 8: scale = TIMES_8; break; |
| 1184 | default: break; |
| 1185 | } |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 1186 | // Incorporate the string or array offset in the address computation. |
Aart Bik | 472821b | 2017-04-27 17:23:51 -0700 | [diff] [blame] | 1187 | uint32_t offset = is_string_char_at |
| 1188 | ? mirror::String::ValueOffset().Uint32Value() |
| 1189 | : mirror::Array::DataOffset(size).Uint32Value(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1190 | return CodeGeneratorX86::ArrayAddress(base.AsRegister<Register>(), index, scale, offset); |
| 1191 | } |
| 1192 | |
| 1193 | void LocationsBuilderX86::VisitVecLoad(HVecLoad* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 1194 | CreateVecMemLocations(GetGraph()->GetAllocator(), instruction, /*is_load*/ true); |
Aart Bik | 472821b | 2017-04-27 17:23:51 -0700 | [diff] [blame] | 1195 | // String load requires a temporary for the compressed load. |
| 1196 | if (mirror::kUseStringCompression && instruction->IsStringCharAt()) { |
| 1197 | instruction->GetLocations()->AddTemp(Location::RequiresFpuRegister()); |
| 1198 | } |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1199 | } |
| 1200 | |
| 1201 | void InstructionCodeGeneratorX86::VisitVecLoad(HVecLoad* instruction) { |
Aart Bik | 472821b | 2017-04-27 17:23:51 -0700 | [diff] [blame] | 1202 | LocationSummary* locations = instruction->GetLocations(); |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1203 | size_t size = DataType::Size(instruction->GetPackedType()); |
Aart Bik | 472821b | 2017-04-27 17:23:51 -0700 | [diff] [blame] | 1204 | Address address = VecAddress(locations, size, instruction->IsStringCharAt()); |
| 1205 | XmmRegister reg = locations->Out().AsFpuRegister<XmmRegister>(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1206 | bool is_aligned16 = instruction->GetAlignment().IsAlignedAt(16); |
| 1207 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1208 | case DataType::Type::kUint16: |
Aart Bik | 472821b | 2017-04-27 17:23:51 -0700 | [diff] [blame] | 1209 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 1210 | // Special handling of compressed/uncompressed string load. |
| 1211 | if (mirror::kUseStringCompression && instruction->IsStringCharAt()) { |
| 1212 | NearLabel done, not_compressed; |
| 1213 | XmmRegister tmp = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); |
| 1214 | // Test compression bit. |
| 1215 | static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, |
| 1216 | "Expecting 0=compressed, 1=uncompressed"); |
| 1217 | uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); |
| 1218 | __ testb(Address(locations->InAt(0).AsRegister<Register>(), count_offset), Immediate(1)); |
| 1219 | __ j(kNotZero, ¬_compressed); |
| 1220 | // Zero extend 8 compressed bytes into 8 chars. |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 1221 | __ movsd(reg, VecAddress(locations, 1, instruction->IsStringCharAt())); |
Aart Bik | 472821b | 2017-04-27 17:23:51 -0700 | [diff] [blame] | 1222 | __ pxor(tmp, tmp); |
| 1223 | __ punpcklbw(reg, tmp); |
| 1224 | __ jmp(&done); |
| 1225 | // Load 4 direct uncompressed chars. |
| 1226 | __ Bind(¬_compressed); |
| 1227 | is_aligned16 ? __ movdqa(reg, address) : __ movdqu(reg, address); |
| 1228 | __ Bind(&done); |
| 1229 | return; |
| 1230 | } |
| 1231 | FALLTHROUGH_INTENDED; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1232 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 1233 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1234 | case DataType::Type::kInt8: |
| 1235 | case DataType::Type::kInt16: |
| 1236 | case DataType::Type::kInt32: |
| 1237 | case DataType::Type::kInt64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1238 | DCHECK_LE(2u, instruction->GetVectorLength()); |
| 1239 | DCHECK_LE(instruction->GetVectorLength(), 16u); |
| 1240 | is_aligned16 ? __ movdqa(reg, address) : __ movdqu(reg, address); |
| 1241 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1242 | case DataType::Type::kFloat32: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1243 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 1244 | is_aligned16 ? __ movaps(reg, address) : __ movups(reg, address); |
| 1245 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1246 | case DataType::Type::kFloat64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1247 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 1248 | is_aligned16 ? __ movapd(reg, address) : __ movupd(reg, address); |
| 1249 | break; |
| 1250 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 1251 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1252 | UNREACHABLE(); |
| 1253 | } |
| 1254 | } |
| 1255 | |
| 1256 | void LocationsBuilderX86::VisitVecStore(HVecStore* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 1257 | CreateVecMemLocations(GetGraph()->GetAllocator(), instruction, /*is_load*/ false); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1258 | } |
| 1259 | |
| 1260 | void InstructionCodeGeneratorX86::VisitVecStore(HVecStore* instruction) { |
Aart Bik | 472821b | 2017-04-27 17:23:51 -0700 | [diff] [blame] | 1261 | LocationSummary* locations = instruction->GetLocations(); |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1262 | size_t size = DataType::Size(instruction->GetPackedType()); |
Aart Bik | 472821b | 2017-04-27 17:23:51 -0700 | [diff] [blame] | 1263 | Address address = VecAddress(locations, size, /*is_string_char_at*/ false); |
| 1264 | XmmRegister reg = locations->InAt(2).AsFpuRegister<XmmRegister>(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1265 | bool is_aligned16 = instruction->GetAlignment().IsAlignedAt(16); |
| 1266 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1267 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 1268 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1269 | case DataType::Type::kInt8: |
| 1270 | case DataType::Type::kUint16: |
| 1271 | case DataType::Type::kInt16: |
| 1272 | case DataType::Type::kInt32: |
| 1273 | case DataType::Type::kInt64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1274 | DCHECK_LE(2u, instruction->GetVectorLength()); |
| 1275 | DCHECK_LE(instruction->GetVectorLength(), 16u); |
| 1276 | is_aligned16 ? __ movdqa(address, reg) : __ movdqu(address, reg); |
| 1277 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1278 | case DataType::Type::kFloat32: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1279 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 1280 | is_aligned16 ? __ movaps(address, reg) : __ movups(address, reg); |
| 1281 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1282 | case DataType::Type::kFloat64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1283 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 1284 | is_aligned16 ? __ movapd(address, reg) : __ movupd(address, reg); |
| 1285 | break; |
| 1286 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 1287 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1288 | UNREACHABLE(); |
| 1289 | } |
| 1290 | } |
| 1291 | |
| 1292 | #undef __ |
| 1293 | |
| 1294 | } // namespace x86 |
| 1295 | } // namespace art |