Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2017 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | #include "code_generator_arm64.h" |
Andreas Gampe | 895f922 | 2017-07-05 09:53:32 -0700 | [diff] [blame] | 18 | |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 19 | #include "mirror/array-inl.h" |
Andreas Gampe | 895f922 | 2017-07-05 09:53:32 -0700 | [diff] [blame] | 20 | #include "mirror/string.h" |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 21 | |
| 22 | using namespace vixl::aarch64; // NOLINT(build/namespaces) |
| 23 | |
| 24 | namespace art { |
| 25 | namespace arm64 { |
| 26 | |
Artem Serov | 8dfe746 | 2017-06-01 14:28:48 +0100 | [diff] [blame] | 27 | using helpers::ARM64EncodableConstantOrRegister; |
| 28 | using helpers::Arm64CanEncodeConstantAsImmediate; |
Aart Bik | 472821b | 2017-04-27 17:23:51 -0700 | [diff] [blame] | 29 | using helpers::DRegisterFrom; |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 30 | using helpers::HeapOperand; |
| 31 | using helpers::InputRegisterAt; |
| 32 | using helpers::Int64ConstantFrom; |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 33 | using helpers::OutputRegister; |
| 34 | using helpers::VRegisterFrom; |
Nicolas Geoffray | 982334c | 2017-09-02 12:54:16 +0000 | [diff] [blame] | 35 | using helpers::WRegisterFrom; |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 36 | using helpers::XRegisterFrom; |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 37 | |
| 38 | #define __ GetVIXLAssembler()-> |
| 39 | |
| 40 | void LocationsBuilderARM64::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 41 | LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction); |
Artem Serov | 8dfe746 | 2017-06-01 14:28:48 +0100 | [diff] [blame] | 42 | HInstruction* input = instruction->InputAt(0); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 43 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 44 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 45 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 46 | case DataType::Type::kInt8: |
| 47 | case DataType::Type::kUint16: |
| 48 | case DataType::Type::kInt16: |
| 49 | case DataType::Type::kInt32: |
| 50 | case DataType::Type::kInt64: |
Artem Serov | 8dfe746 | 2017-06-01 14:28:48 +0100 | [diff] [blame] | 51 | locations->SetInAt(0, ARM64EncodableConstantOrRegister(input, instruction)); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 52 | locations->SetOut(Location::RequiresFpuRegister()); |
| 53 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 54 | case DataType::Type::kFloat32: |
| 55 | case DataType::Type::kFloat64: |
Artem Serov | 8dfe746 | 2017-06-01 14:28:48 +0100 | [diff] [blame] | 56 | if (input->IsConstant() && |
| 57 | Arm64CanEncodeConstantAsImmediate(input->AsConstant(), instruction)) { |
| 58 | locations->SetInAt(0, Location::ConstantLocation(input->AsConstant())); |
| 59 | locations->SetOut(Location::RequiresFpuRegister()); |
| 60 | } else { |
| 61 | locations->SetInAt(0, Location::RequiresFpuRegister()); |
| 62 | locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); |
| 63 | } |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 64 | break; |
| 65 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 66 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 67 | UNREACHABLE(); |
| 68 | } |
| 69 | } |
| 70 | |
| 71 | void InstructionCodeGeneratorARM64::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { |
| 72 | LocationSummary* locations = instruction->GetLocations(); |
Artem Serov | 8dfe746 | 2017-06-01 14:28:48 +0100 | [diff] [blame] | 73 | Location src_loc = locations->InAt(0); |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 74 | VRegister dst = VRegisterFrom(locations->Out()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 75 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 76 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 77 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 78 | case DataType::Type::kInt8: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 79 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
Artem Serov | 8dfe746 | 2017-06-01 14:28:48 +0100 | [diff] [blame] | 80 | if (src_loc.IsConstant()) { |
| 81 | __ Movi(dst.V16B(), Int64ConstantFrom(src_loc)); |
| 82 | } else { |
| 83 | __ Dup(dst.V16B(), InputRegisterAt(instruction, 0)); |
| 84 | } |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 85 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 86 | case DataType::Type::kUint16: |
| 87 | case DataType::Type::kInt16: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 88 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
Artem Serov | 8dfe746 | 2017-06-01 14:28:48 +0100 | [diff] [blame] | 89 | if (src_loc.IsConstant()) { |
| 90 | __ Movi(dst.V8H(), Int64ConstantFrom(src_loc)); |
| 91 | } else { |
| 92 | __ Dup(dst.V8H(), InputRegisterAt(instruction, 0)); |
| 93 | } |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 94 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 95 | case DataType::Type::kInt32: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 96 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
Artem Serov | 8dfe746 | 2017-06-01 14:28:48 +0100 | [diff] [blame] | 97 | if (src_loc.IsConstant()) { |
| 98 | __ Movi(dst.V4S(), Int64ConstantFrom(src_loc)); |
| 99 | } else { |
| 100 | __ Dup(dst.V4S(), InputRegisterAt(instruction, 0)); |
| 101 | } |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 102 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 103 | case DataType::Type::kInt64: |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 104 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
Artem Serov | 8dfe746 | 2017-06-01 14:28:48 +0100 | [diff] [blame] | 105 | if (src_loc.IsConstant()) { |
| 106 | __ Movi(dst.V2D(), Int64ConstantFrom(src_loc)); |
| 107 | } else { |
| 108 | __ Dup(dst.V2D(), XRegisterFrom(src_loc)); |
| 109 | } |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 110 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 111 | case DataType::Type::kFloat32: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 112 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
Artem Serov | 8dfe746 | 2017-06-01 14:28:48 +0100 | [diff] [blame] | 113 | if (src_loc.IsConstant()) { |
| 114 | __ Fmov(dst.V4S(), src_loc.GetConstant()->AsFloatConstant()->GetValue()); |
| 115 | } else { |
| 116 | __ Dup(dst.V4S(), VRegisterFrom(src_loc).V4S(), 0); |
| 117 | } |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 118 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 119 | case DataType::Type::kFloat64: |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 120 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
Artem Serov | 8dfe746 | 2017-06-01 14:28:48 +0100 | [diff] [blame] | 121 | if (src_loc.IsConstant()) { |
| 122 | __ Fmov(dst.V2D(), src_loc.GetConstant()->AsDoubleConstant()->GetValue()); |
| 123 | } else { |
| 124 | __ Dup(dst.V2D(), VRegisterFrom(src_loc).V2D(), 0); |
| 125 | } |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 126 | break; |
| 127 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 128 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 129 | UNREACHABLE(); |
| 130 | } |
| 131 | } |
| 132 | |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 133 | void LocationsBuilderARM64::VisitVecExtractScalar(HVecExtractScalar* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 134 | LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 135 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 136 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 137 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 138 | case DataType::Type::kInt8: |
| 139 | case DataType::Type::kUint16: |
| 140 | case DataType::Type::kInt16: |
| 141 | case DataType::Type::kInt32: |
| 142 | case DataType::Type::kInt64: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 143 | locations->SetInAt(0, Location::RequiresFpuRegister()); |
| 144 | locations->SetOut(Location::RequiresRegister()); |
| 145 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 146 | case DataType::Type::kFloat32: |
| 147 | case DataType::Type::kFloat64: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 148 | locations->SetInAt(0, Location::RequiresFpuRegister()); |
| 149 | locations->SetOut(Location::SameAsFirstInput()); |
| 150 | break; |
| 151 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 152 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 153 | UNREACHABLE(); |
| 154 | } |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 155 | } |
| 156 | |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 157 | void InstructionCodeGeneratorARM64::VisitVecExtractScalar(HVecExtractScalar* instruction) { |
| 158 | LocationSummary* locations = instruction->GetLocations(); |
| 159 | VRegister src = VRegisterFrom(locations->InAt(0)); |
| 160 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 161 | case DataType::Type::kInt32: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 162 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 163 | __ Umov(OutputRegister(instruction), src.V4S(), 0); |
| 164 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 165 | case DataType::Type::kInt64: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 166 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 167 | __ Umov(OutputRegister(instruction), src.V2D(), 0); |
| 168 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 169 | case DataType::Type::kFloat32: |
| 170 | case DataType::Type::kFloat64: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 171 | DCHECK_LE(2u, instruction->GetVectorLength()); |
| 172 | DCHECK_LE(instruction->GetVectorLength(), 4u); |
| 173 | DCHECK(locations->InAt(0).Equals(locations->Out())); // no code required |
| 174 | break; |
| 175 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 176 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 177 | UNREACHABLE(); |
| 178 | } |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 179 | } |
| 180 | |
| 181 | // Helper to set up locations for vector unary operations. |
Vladimir Marko | e764d2e | 2017-10-05 14:35:55 +0100 | [diff] [blame] | 182 | static void CreateVecUnOpLocations(ArenaAllocator* allocator, HVecUnaryOperation* instruction) { |
| 183 | LocationSummary* locations = new (allocator) LocationSummary(instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 184 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 185 | case DataType::Type::kBool: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 186 | locations->SetInAt(0, Location::RequiresFpuRegister()); |
| 187 | locations->SetOut(Location::RequiresFpuRegister(), |
| 188 | instruction->IsVecNot() ? Location::kOutputOverlap |
| 189 | : Location::kNoOutputOverlap); |
| 190 | break; |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 191 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 192 | case DataType::Type::kInt8: |
| 193 | case DataType::Type::kUint16: |
| 194 | case DataType::Type::kInt16: |
| 195 | case DataType::Type::kInt32: |
| 196 | case DataType::Type::kInt64: |
| 197 | case DataType::Type::kFloat32: |
| 198 | case DataType::Type::kFloat64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 199 | locations->SetInAt(0, Location::RequiresFpuRegister()); |
| 200 | locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); |
| 201 | break; |
| 202 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 203 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 204 | UNREACHABLE(); |
| 205 | } |
| 206 | } |
| 207 | |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 208 | void LocationsBuilderARM64::VisitVecReduce(HVecReduce* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 209 | CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 210 | } |
| 211 | |
| 212 | void InstructionCodeGeneratorARM64::VisitVecReduce(HVecReduce* instruction) { |
| 213 | LocationSummary* locations = instruction->GetLocations(); |
| 214 | VRegister src = VRegisterFrom(locations->InAt(0)); |
| 215 | VRegister dst = DRegisterFrom(locations->Out()); |
| 216 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 217 | case DataType::Type::kInt32: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 218 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 219 | switch (instruction->GetKind()) { |
| 220 | case HVecReduce::kSum: |
| 221 | __ Addv(dst.S(), src.V4S()); |
| 222 | break; |
| 223 | case HVecReduce::kMin: |
| 224 | __ Sminv(dst.S(), src.V4S()); |
| 225 | break; |
| 226 | case HVecReduce::kMax: |
| 227 | __ Smaxv(dst.S(), src.V4S()); |
| 228 | break; |
| 229 | } |
| 230 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 231 | case DataType::Type::kInt64: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 232 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 233 | switch (instruction->GetKind()) { |
| 234 | case HVecReduce::kSum: |
| 235 | __ Addp(dst.D(), src.V2D()); |
| 236 | break; |
| 237 | default: |
| 238 | LOG(FATAL) << "Unsupported SIMD min/max"; |
| 239 | UNREACHABLE(); |
| 240 | } |
| 241 | break; |
| 242 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 243 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 244 | UNREACHABLE(); |
| 245 | } |
| 246 | } |
| 247 | |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 248 | void LocationsBuilderARM64::VisitVecCnv(HVecCnv* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 249 | CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 250 | } |
| 251 | |
| 252 | void InstructionCodeGeneratorARM64::VisitVecCnv(HVecCnv* instruction) { |
| 253 | LocationSummary* locations = instruction->GetLocations(); |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 254 | VRegister src = VRegisterFrom(locations->InAt(0)); |
| 255 | VRegister dst = VRegisterFrom(locations->Out()); |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 256 | DataType::Type from = instruction->GetInputType(); |
| 257 | DataType::Type to = instruction->GetResultType(); |
| 258 | if (from == DataType::Type::kInt32 && to == DataType::Type::kFloat32) { |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 259 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 260 | __ Scvtf(dst.V4S(), src.V4S()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 261 | } else { |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 262 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 263 | } |
| 264 | } |
| 265 | |
| 266 | void LocationsBuilderARM64::VisitVecNeg(HVecNeg* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 267 | CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 268 | } |
| 269 | |
| 270 | void InstructionCodeGeneratorARM64::VisitVecNeg(HVecNeg* instruction) { |
| 271 | LocationSummary* locations = instruction->GetLocations(); |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 272 | VRegister src = VRegisterFrom(locations->InAt(0)); |
| 273 | VRegister dst = VRegisterFrom(locations->Out()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 274 | switch (instruction->GetPackedType()) { |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 275 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 276 | case DataType::Type::kInt8: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 277 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 278 | __ Neg(dst.V16B(), src.V16B()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 279 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 280 | case DataType::Type::kUint16: |
| 281 | case DataType::Type::kInt16: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 282 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 283 | __ Neg(dst.V8H(), src.V8H()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 284 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 285 | case DataType::Type::kInt32: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 286 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 287 | __ Neg(dst.V4S(), src.V4S()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 288 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 289 | case DataType::Type::kInt64: |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 290 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 291 | __ Neg(dst.V2D(), src.V2D()); |
| 292 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 293 | case DataType::Type::kFloat32: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 294 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 295 | __ Fneg(dst.V4S(), src.V4S()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 296 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 297 | case DataType::Type::kFloat64: |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 298 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 299 | __ Fneg(dst.V2D(), src.V2D()); |
| 300 | break; |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 301 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 302 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 303 | UNREACHABLE(); |
| 304 | } |
| 305 | } |
| 306 | |
Aart Bik | 6daebeb | 2017-04-03 14:35:41 -0700 | [diff] [blame] | 307 | void LocationsBuilderARM64::VisitVecAbs(HVecAbs* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 308 | CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | 6daebeb | 2017-04-03 14:35:41 -0700 | [diff] [blame] | 309 | } |
| 310 | |
| 311 | void InstructionCodeGeneratorARM64::VisitVecAbs(HVecAbs* instruction) { |
| 312 | LocationSummary* locations = instruction->GetLocations(); |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 313 | VRegister src = VRegisterFrom(locations->InAt(0)); |
| 314 | VRegister dst = VRegisterFrom(locations->Out()); |
Aart Bik | 6daebeb | 2017-04-03 14:35:41 -0700 | [diff] [blame] | 315 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 316 | case DataType::Type::kInt8: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 317 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 318 | __ Abs(dst.V16B(), src.V16B()); |
Aart Bik | 6daebeb | 2017-04-03 14:35:41 -0700 | [diff] [blame] | 319 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 320 | case DataType::Type::kInt16: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 321 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 322 | __ Abs(dst.V8H(), src.V8H()); |
Aart Bik | 6daebeb | 2017-04-03 14:35:41 -0700 | [diff] [blame] | 323 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 324 | case DataType::Type::kInt32: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 325 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 326 | __ Abs(dst.V4S(), src.V4S()); |
Aart Bik | 6daebeb | 2017-04-03 14:35:41 -0700 | [diff] [blame] | 327 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 328 | case DataType::Type::kInt64: |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 329 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 330 | __ Abs(dst.V2D(), src.V2D()); |
| 331 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 332 | case DataType::Type::kFloat32: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 333 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 334 | __ Fabs(dst.V4S(), src.V4S()); |
Aart Bik | 6daebeb | 2017-04-03 14:35:41 -0700 | [diff] [blame] | 335 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 336 | case DataType::Type::kFloat64: |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 337 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 338 | __ Fabs(dst.V2D(), src.V2D()); |
| 339 | break; |
Aart Bik | 6daebeb | 2017-04-03 14:35:41 -0700 | [diff] [blame] | 340 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 341 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 342 | UNREACHABLE(); |
Aart Bik | 6daebeb | 2017-04-03 14:35:41 -0700 | [diff] [blame] | 343 | } |
| 344 | } |
| 345 | |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 346 | void LocationsBuilderARM64::VisitVecNot(HVecNot* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 347 | CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 348 | } |
| 349 | |
| 350 | void InstructionCodeGeneratorARM64::VisitVecNot(HVecNot* instruction) { |
| 351 | LocationSummary* locations = instruction->GetLocations(); |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 352 | VRegister src = VRegisterFrom(locations->InAt(0)); |
| 353 | VRegister dst = VRegisterFrom(locations->Out()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 354 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 355 | case DataType::Type::kBool: // special case boolean-not |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 356 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 357 | __ Movi(dst.V16B(), 1); |
| 358 | __ Eor(dst.V16B(), dst.V16B(), src.V16B()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 359 | break; |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 360 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 361 | case DataType::Type::kInt8: |
| 362 | case DataType::Type::kUint16: |
| 363 | case DataType::Type::kInt16: |
| 364 | case DataType::Type::kInt32: |
| 365 | case DataType::Type::kInt64: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 366 | __ Not(dst.V16B(), src.V16B()); // lanes do not matter |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 367 | break; |
| 368 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 369 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 370 | UNREACHABLE(); |
| 371 | } |
| 372 | } |
| 373 | |
| 374 | // Helper to set up locations for vector binary operations. |
Vladimir Marko | e764d2e | 2017-10-05 14:35:55 +0100 | [diff] [blame] | 375 | static void CreateVecBinOpLocations(ArenaAllocator* allocator, HVecBinaryOperation* instruction) { |
| 376 | LocationSummary* locations = new (allocator) LocationSummary(instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 377 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 378 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 379 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 380 | case DataType::Type::kInt8: |
| 381 | case DataType::Type::kUint16: |
| 382 | case DataType::Type::kInt16: |
| 383 | case DataType::Type::kInt32: |
| 384 | case DataType::Type::kInt64: |
| 385 | case DataType::Type::kFloat32: |
| 386 | case DataType::Type::kFloat64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 387 | locations->SetInAt(0, Location::RequiresFpuRegister()); |
| 388 | locations->SetInAt(1, Location::RequiresFpuRegister()); |
| 389 | locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); |
| 390 | break; |
| 391 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 392 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 393 | UNREACHABLE(); |
| 394 | } |
| 395 | } |
| 396 | |
| 397 | void LocationsBuilderARM64::VisitVecAdd(HVecAdd* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 398 | CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 399 | } |
| 400 | |
| 401 | void InstructionCodeGeneratorARM64::VisitVecAdd(HVecAdd* instruction) { |
| 402 | LocationSummary* locations = instruction->GetLocations(); |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 403 | VRegister lhs = VRegisterFrom(locations->InAt(0)); |
| 404 | VRegister rhs = VRegisterFrom(locations->InAt(1)); |
| 405 | VRegister dst = VRegisterFrom(locations->Out()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 406 | switch (instruction->GetPackedType()) { |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 407 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 408 | case DataType::Type::kInt8: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 409 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 410 | __ Add(dst.V16B(), lhs.V16B(), rhs.V16B()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 411 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 412 | case DataType::Type::kUint16: |
| 413 | case DataType::Type::kInt16: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 414 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 415 | __ Add(dst.V8H(), lhs.V8H(), rhs.V8H()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 416 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 417 | case DataType::Type::kInt32: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 418 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 419 | __ Add(dst.V4S(), lhs.V4S(), rhs.V4S()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 420 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 421 | case DataType::Type::kInt64: |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 422 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 423 | __ Add(dst.V2D(), lhs.V2D(), rhs.V2D()); |
| 424 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 425 | case DataType::Type::kFloat32: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 426 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 427 | __ Fadd(dst.V4S(), lhs.V4S(), rhs.V4S()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 428 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 429 | case DataType::Type::kFloat64: |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 430 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 431 | __ Fadd(dst.V2D(), lhs.V2D(), rhs.V2D()); |
| 432 | break; |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 433 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 434 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
| 435 | UNREACHABLE(); |
| 436 | } |
| 437 | } |
| 438 | |
| 439 | void LocationsBuilderARM64::VisitVecSaturationAdd(HVecSaturationAdd* instruction) { |
| 440 | CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); |
| 441 | } |
| 442 | |
| 443 | void InstructionCodeGeneratorARM64::VisitVecSaturationAdd(HVecSaturationAdd* instruction) { |
| 444 | LocationSummary* locations = instruction->GetLocations(); |
| 445 | VRegister lhs = VRegisterFrom(locations->InAt(0)); |
| 446 | VRegister rhs = VRegisterFrom(locations->InAt(1)); |
| 447 | VRegister dst = VRegisterFrom(locations->Out()); |
| 448 | switch (instruction->GetPackedType()) { |
| 449 | case DataType::Type::kUint8: |
| 450 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 451 | __ Uqadd(dst.V16B(), lhs.V16B(), rhs.V16B()); |
| 452 | break; |
| 453 | case DataType::Type::kInt8: |
| 454 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 455 | __ Sqadd(dst.V16B(), lhs.V16B(), rhs.V16B()); |
| 456 | break; |
| 457 | case DataType::Type::kUint16: |
| 458 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 459 | __ Uqadd(dst.V8H(), lhs.V8H(), rhs.V8H()); |
| 460 | break; |
| 461 | case DataType::Type::kInt16: |
| 462 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 463 | __ Sqadd(dst.V8H(), lhs.V8H(), rhs.V8H()); |
| 464 | break; |
| 465 | default: |
| 466 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 467 | UNREACHABLE(); |
| 468 | } |
| 469 | } |
| 470 | |
Aart Bik | f3e61ee | 2017-04-12 17:09:20 -0700 | [diff] [blame] | 471 | void LocationsBuilderARM64::VisitVecHalvingAdd(HVecHalvingAdd* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 472 | CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f3e61ee | 2017-04-12 17:09:20 -0700 | [diff] [blame] | 473 | } |
| 474 | |
| 475 | void InstructionCodeGeneratorARM64::VisitVecHalvingAdd(HVecHalvingAdd* instruction) { |
| 476 | LocationSummary* locations = instruction->GetLocations(); |
| 477 | VRegister lhs = VRegisterFrom(locations->InAt(0)); |
| 478 | VRegister rhs = VRegisterFrom(locations->InAt(1)); |
| 479 | VRegister dst = VRegisterFrom(locations->Out()); |
| 480 | switch (instruction->GetPackedType()) { |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 481 | case DataType::Type::kUint8: |
| 482 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 483 | instruction->IsRounded() |
| 484 | ? __ Urhadd(dst.V16B(), lhs.V16B(), rhs.V16B()) |
| 485 | : __ Uhadd(dst.V16B(), lhs.V16B(), rhs.V16B()); |
| 486 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 487 | case DataType::Type::kInt8: |
Aart Bik | f3e61ee | 2017-04-12 17:09:20 -0700 | [diff] [blame] | 488 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 489 | instruction->IsRounded() |
| 490 | ? __ Srhadd(dst.V16B(), lhs.V16B(), rhs.V16B()) |
| 491 | : __ Shadd(dst.V16B(), lhs.V16B(), rhs.V16B()); |
Aart Bik | f3e61ee | 2017-04-12 17:09:20 -0700 | [diff] [blame] | 492 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 493 | case DataType::Type::kUint16: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 494 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 495 | instruction->IsRounded() |
| 496 | ? __ Urhadd(dst.V8H(), lhs.V8H(), rhs.V8H()) |
| 497 | : __ Uhadd(dst.V8H(), lhs.V8H(), rhs.V8H()); |
| 498 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 499 | case DataType::Type::kInt16: |
Aart Bik | f3e61ee | 2017-04-12 17:09:20 -0700 | [diff] [blame] | 500 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 501 | instruction->IsRounded() |
| 502 | ? __ Srhadd(dst.V8H(), lhs.V8H(), rhs.V8H()) |
| 503 | : __ Shadd(dst.V8H(), lhs.V8H(), rhs.V8H()); |
Aart Bik | f3e61ee | 2017-04-12 17:09:20 -0700 | [diff] [blame] | 504 | break; |
| 505 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 506 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f3e61ee | 2017-04-12 17:09:20 -0700 | [diff] [blame] | 507 | UNREACHABLE(); |
| 508 | } |
| 509 | } |
| 510 | |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 511 | void LocationsBuilderARM64::VisitVecSub(HVecSub* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 512 | CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 513 | } |
| 514 | |
| 515 | void InstructionCodeGeneratorARM64::VisitVecSub(HVecSub* instruction) { |
| 516 | LocationSummary* locations = instruction->GetLocations(); |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 517 | VRegister lhs = VRegisterFrom(locations->InAt(0)); |
| 518 | VRegister rhs = VRegisterFrom(locations->InAt(1)); |
| 519 | VRegister dst = VRegisterFrom(locations->Out()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 520 | switch (instruction->GetPackedType()) { |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 521 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 522 | case DataType::Type::kInt8: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 523 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 524 | __ Sub(dst.V16B(), lhs.V16B(), rhs.V16B()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 525 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 526 | case DataType::Type::kUint16: |
| 527 | case DataType::Type::kInt16: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 528 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 529 | __ Sub(dst.V8H(), lhs.V8H(), rhs.V8H()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 530 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 531 | case DataType::Type::kInt32: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 532 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 533 | __ Sub(dst.V4S(), lhs.V4S(), rhs.V4S()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 534 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 535 | case DataType::Type::kInt64: |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 536 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 537 | __ Sub(dst.V2D(), lhs.V2D(), rhs.V2D()); |
| 538 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 539 | case DataType::Type::kFloat32: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 540 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 541 | __ Fsub(dst.V4S(), lhs.V4S(), rhs.V4S()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 542 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 543 | case DataType::Type::kFloat64: |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 544 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 545 | __ Fsub(dst.V2D(), lhs.V2D(), rhs.V2D()); |
| 546 | break; |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 547 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 548 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
| 549 | UNREACHABLE(); |
| 550 | } |
| 551 | } |
| 552 | |
| 553 | void LocationsBuilderARM64::VisitVecSaturationSub(HVecSaturationSub* instruction) { |
| 554 | CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); |
| 555 | } |
| 556 | |
| 557 | void InstructionCodeGeneratorARM64::VisitVecSaturationSub(HVecSaturationSub* instruction) { |
| 558 | LocationSummary* locations = instruction->GetLocations(); |
| 559 | VRegister lhs = VRegisterFrom(locations->InAt(0)); |
| 560 | VRegister rhs = VRegisterFrom(locations->InAt(1)); |
| 561 | VRegister dst = VRegisterFrom(locations->Out()); |
| 562 | switch (instruction->GetPackedType()) { |
| 563 | case DataType::Type::kUint8: |
| 564 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 565 | __ Uqsub(dst.V16B(), lhs.V16B(), rhs.V16B()); |
| 566 | break; |
| 567 | case DataType::Type::kInt8: |
| 568 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 569 | __ Sqsub(dst.V16B(), lhs.V16B(), rhs.V16B()); |
| 570 | break; |
| 571 | case DataType::Type::kUint16: |
| 572 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 573 | __ Uqsub(dst.V8H(), lhs.V8H(), rhs.V8H()); |
| 574 | break; |
| 575 | case DataType::Type::kInt16: |
| 576 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 577 | __ Sqsub(dst.V8H(), lhs.V8H(), rhs.V8H()); |
| 578 | break; |
| 579 | default: |
| 580 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 581 | UNREACHABLE(); |
| 582 | } |
| 583 | } |
| 584 | |
| 585 | void LocationsBuilderARM64::VisitVecMul(HVecMul* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 586 | CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 587 | } |
| 588 | |
| 589 | void InstructionCodeGeneratorARM64::VisitVecMul(HVecMul* instruction) { |
| 590 | LocationSummary* locations = instruction->GetLocations(); |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 591 | VRegister lhs = VRegisterFrom(locations->InAt(0)); |
| 592 | VRegister rhs = VRegisterFrom(locations->InAt(1)); |
| 593 | VRegister dst = VRegisterFrom(locations->Out()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 594 | switch (instruction->GetPackedType()) { |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 595 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 596 | case DataType::Type::kInt8: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 597 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 598 | __ Mul(dst.V16B(), lhs.V16B(), rhs.V16B()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 599 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 600 | case DataType::Type::kUint16: |
| 601 | case DataType::Type::kInt16: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 602 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 603 | __ Mul(dst.V8H(), lhs.V8H(), rhs.V8H()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 604 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 605 | case DataType::Type::kInt32: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 606 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 607 | __ Mul(dst.V4S(), lhs.V4S(), rhs.V4S()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 608 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 609 | case DataType::Type::kFloat32: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 610 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 611 | __ Fmul(dst.V4S(), lhs.V4S(), rhs.V4S()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 612 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 613 | case DataType::Type::kFloat64: |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 614 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 615 | __ Fmul(dst.V2D(), lhs.V2D(), rhs.V2D()); |
| 616 | break; |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 617 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 618 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 619 | UNREACHABLE(); |
| 620 | } |
| 621 | } |
| 622 | |
| 623 | void LocationsBuilderARM64::VisitVecDiv(HVecDiv* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 624 | CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 625 | } |
| 626 | |
| 627 | void InstructionCodeGeneratorARM64::VisitVecDiv(HVecDiv* instruction) { |
| 628 | LocationSummary* locations = instruction->GetLocations(); |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 629 | VRegister lhs = VRegisterFrom(locations->InAt(0)); |
| 630 | VRegister rhs = VRegisterFrom(locations->InAt(1)); |
| 631 | VRegister dst = VRegisterFrom(locations->Out()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 632 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 633 | case DataType::Type::kFloat32: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 634 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 635 | __ Fdiv(dst.V4S(), lhs.V4S(), rhs.V4S()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 636 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 637 | case DataType::Type::kFloat64: |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 638 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 639 | __ Fdiv(dst.V2D(), lhs.V2D(), rhs.V2D()); |
| 640 | break; |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 641 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 642 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 643 | UNREACHABLE(); |
| 644 | } |
| 645 | } |
| 646 | |
Aart Bik | f3e61ee | 2017-04-12 17:09:20 -0700 | [diff] [blame] | 647 | void LocationsBuilderARM64::VisitVecMin(HVecMin* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 648 | CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f3e61ee | 2017-04-12 17:09:20 -0700 | [diff] [blame] | 649 | } |
| 650 | |
| 651 | void InstructionCodeGeneratorARM64::VisitVecMin(HVecMin* instruction) { |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 652 | LocationSummary* locations = instruction->GetLocations(); |
| 653 | VRegister lhs = VRegisterFrom(locations->InAt(0)); |
| 654 | VRegister rhs = VRegisterFrom(locations->InAt(1)); |
| 655 | VRegister dst = VRegisterFrom(locations->Out()); |
| 656 | switch (instruction->GetPackedType()) { |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 657 | case DataType::Type::kUint8: |
| 658 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 659 | __ Umin(dst.V16B(), lhs.V16B(), rhs.V16B()); |
| 660 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 661 | case DataType::Type::kInt8: |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 662 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 663 | __ Smin(dst.V16B(), lhs.V16B(), rhs.V16B()); |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 664 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 665 | case DataType::Type::kUint16: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 666 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 667 | __ Umin(dst.V8H(), lhs.V8H(), rhs.V8H()); |
| 668 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 669 | case DataType::Type::kInt16: |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 670 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 671 | __ Smin(dst.V8H(), lhs.V8H(), rhs.V8H()); |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 672 | break; |
Aart Bik | 66c158e | 2018-01-31 12:55:04 -0800 | [diff] [blame] | 673 | case DataType::Type::kUint32: |
| 674 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 675 | __ Umin(dst.V4S(), lhs.V4S(), rhs.V4S()); |
| 676 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 677 | case DataType::Type::kInt32: |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 678 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
Aart Bik | 66c158e | 2018-01-31 12:55:04 -0800 | [diff] [blame] | 679 | __ Smin(dst.V4S(), lhs.V4S(), rhs.V4S()); |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 680 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 681 | case DataType::Type::kFloat32: |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 682 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 683 | __ Fmin(dst.V4S(), lhs.V4S(), rhs.V4S()); |
| 684 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 685 | case DataType::Type::kFloat64: |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 686 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 687 | __ Fmin(dst.V2D(), lhs.V2D(), rhs.V2D()); |
| 688 | break; |
| 689 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 690 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 691 | UNREACHABLE(); |
| 692 | } |
Aart Bik | f3e61ee | 2017-04-12 17:09:20 -0700 | [diff] [blame] | 693 | } |
| 694 | |
| 695 | void LocationsBuilderARM64::VisitVecMax(HVecMax* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 696 | CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f3e61ee | 2017-04-12 17:09:20 -0700 | [diff] [blame] | 697 | } |
| 698 | |
| 699 | void InstructionCodeGeneratorARM64::VisitVecMax(HVecMax* instruction) { |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 700 | LocationSummary* locations = instruction->GetLocations(); |
| 701 | VRegister lhs = VRegisterFrom(locations->InAt(0)); |
| 702 | VRegister rhs = VRegisterFrom(locations->InAt(1)); |
| 703 | VRegister dst = VRegisterFrom(locations->Out()); |
| 704 | switch (instruction->GetPackedType()) { |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 705 | case DataType::Type::kUint8: |
| 706 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 707 | __ Umax(dst.V16B(), lhs.V16B(), rhs.V16B()); |
| 708 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 709 | case DataType::Type::kInt8: |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 710 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 711 | __ Smax(dst.V16B(), lhs.V16B(), rhs.V16B()); |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 712 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 713 | case DataType::Type::kUint16: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 714 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 715 | __ Umax(dst.V8H(), lhs.V8H(), rhs.V8H()); |
| 716 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 717 | case DataType::Type::kInt16: |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 718 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 719 | __ Smax(dst.V8H(), lhs.V8H(), rhs.V8H()); |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 720 | break; |
Aart Bik | 66c158e | 2018-01-31 12:55:04 -0800 | [diff] [blame] | 721 | case DataType::Type::kUint32: |
| 722 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 723 | __ Umax(dst.V4S(), lhs.V4S(), rhs.V4S()); |
| 724 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 725 | case DataType::Type::kInt32: |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 726 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
Aart Bik | 66c158e | 2018-01-31 12:55:04 -0800 | [diff] [blame] | 727 | __ Smax(dst.V4S(), lhs.V4S(), rhs.V4S()); |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 728 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 729 | case DataType::Type::kFloat32: |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 730 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 731 | __ Fmax(dst.V4S(), lhs.V4S(), rhs.V4S()); |
| 732 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 733 | case DataType::Type::kFloat64: |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 734 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 735 | __ Fmax(dst.V2D(), lhs.V2D(), rhs.V2D()); |
| 736 | break; |
| 737 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 738 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | c8e93c7 | 2017-05-10 10:49:22 -0700 | [diff] [blame] | 739 | UNREACHABLE(); |
| 740 | } |
Aart Bik | f3e61ee | 2017-04-12 17:09:20 -0700 | [diff] [blame] | 741 | } |
| 742 | |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 743 | void LocationsBuilderARM64::VisitVecAnd(HVecAnd* instruction) { |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 744 | // TODO: Allow constants supported by BIC (vector, immediate). |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 745 | CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 746 | } |
| 747 | |
| 748 | void InstructionCodeGeneratorARM64::VisitVecAnd(HVecAnd* instruction) { |
| 749 | LocationSummary* locations = instruction->GetLocations(); |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 750 | VRegister lhs = VRegisterFrom(locations->InAt(0)); |
| 751 | VRegister rhs = VRegisterFrom(locations->InAt(1)); |
| 752 | VRegister dst = VRegisterFrom(locations->Out()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 753 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 754 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 755 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 756 | case DataType::Type::kInt8: |
| 757 | case DataType::Type::kUint16: |
| 758 | case DataType::Type::kInt16: |
| 759 | case DataType::Type::kInt32: |
| 760 | case DataType::Type::kInt64: |
| 761 | case DataType::Type::kFloat32: |
| 762 | case DataType::Type::kFloat64: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 763 | __ And(dst.V16B(), lhs.V16B(), rhs.V16B()); // lanes do not matter |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 764 | break; |
| 765 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 766 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 767 | UNREACHABLE(); |
| 768 | } |
| 769 | } |
| 770 | |
| 771 | void LocationsBuilderARM64::VisitVecAndNot(HVecAndNot* instruction) { |
| 772 | LOG(FATAL) << "Unsupported SIMD instruction " << instruction->GetId(); |
| 773 | } |
| 774 | |
| 775 | void InstructionCodeGeneratorARM64::VisitVecAndNot(HVecAndNot* instruction) { |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 776 | // TODO: Use BIC (vector, register). |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 777 | LOG(FATAL) << "Unsupported SIMD instruction " << instruction->GetId(); |
| 778 | } |
| 779 | |
| 780 | void LocationsBuilderARM64::VisitVecOr(HVecOr* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 781 | CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 782 | } |
| 783 | |
| 784 | void InstructionCodeGeneratorARM64::VisitVecOr(HVecOr* instruction) { |
| 785 | LocationSummary* locations = instruction->GetLocations(); |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 786 | VRegister lhs = VRegisterFrom(locations->InAt(0)); |
| 787 | VRegister rhs = VRegisterFrom(locations->InAt(1)); |
| 788 | VRegister dst = VRegisterFrom(locations->Out()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 789 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 790 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 791 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 792 | case DataType::Type::kInt8: |
| 793 | case DataType::Type::kUint16: |
| 794 | case DataType::Type::kInt16: |
| 795 | case DataType::Type::kInt32: |
| 796 | case DataType::Type::kInt64: |
| 797 | case DataType::Type::kFloat32: |
| 798 | case DataType::Type::kFloat64: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 799 | __ Orr(dst.V16B(), lhs.V16B(), rhs.V16B()); // lanes do not matter |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 800 | break; |
| 801 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 802 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 803 | UNREACHABLE(); |
| 804 | } |
| 805 | } |
| 806 | |
| 807 | void LocationsBuilderARM64::VisitVecXor(HVecXor* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 808 | CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 809 | } |
| 810 | |
| 811 | void InstructionCodeGeneratorARM64::VisitVecXor(HVecXor* instruction) { |
| 812 | LocationSummary* locations = instruction->GetLocations(); |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 813 | VRegister lhs = VRegisterFrom(locations->InAt(0)); |
| 814 | VRegister rhs = VRegisterFrom(locations->InAt(1)); |
| 815 | VRegister dst = VRegisterFrom(locations->Out()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 816 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 817 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 818 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 819 | case DataType::Type::kInt8: |
| 820 | case DataType::Type::kUint16: |
| 821 | case DataType::Type::kInt16: |
| 822 | case DataType::Type::kInt32: |
| 823 | case DataType::Type::kInt64: |
| 824 | case DataType::Type::kFloat32: |
| 825 | case DataType::Type::kFloat64: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 826 | __ Eor(dst.V16B(), lhs.V16B(), rhs.V16B()); // lanes do not matter |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 827 | break; |
| 828 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 829 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 830 | UNREACHABLE(); |
| 831 | } |
| 832 | } |
| 833 | |
| 834 | // Helper to set up locations for vector shift operations. |
Vladimir Marko | e764d2e | 2017-10-05 14:35:55 +0100 | [diff] [blame] | 835 | static void CreateVecShiftLocations(ArenaAllocator* allocator, HVecBinaryOperation* instruction) { |
| 836 | LocationSummary* locations = new (allocator) LocationSummary(instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 837 | switch (instruction->GetPackedType()) { |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 838 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 839 | case DataType::Type::kInt8: |
| 840 | case DataType::Type::kUint16: |
| 841 | case DataType::Type::kInt16: |
| 842 | case DataType::Type::kInt32: |
| 843 | case DataType::Type::kInt64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 844 | locations->SetInAt(0, Location::RequiresFpuRegister()); |
| 845 | locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant())); |
| 846 | locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); |
| 847 | break; |
| 848 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 849 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 850 | UNREACHABLE(); |
| 851 | } |
| 852 | } |
| 853 | |
| 854 | void LocationsBuilderARM64::VisitVecShl(HVecShl* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 855 | CreateVecShiftLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 856 | } |
| 857 | |
| 858 | void InstructionCodeGeneratorARM64::VisitVecShl(HVecShl* instruction) { |
| 859 | LocationSummary* locations = instruction->GetLocations(); |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 860 | VRegister lhs = VRegisterFrom(locations->InAt(0)); |
| 861 | VRegister dst = VRegisterFrom(locations->Out()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 862 | int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); |
| 863 | switch (instruction->GetPackedType()) { |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 864 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 865 | case DataType::Type::kInt8: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 866 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 867 | __ Shl(dst.V16B(), lhs.V16B(), value); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 868 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 869 | case DataType::Type::kUint16: |
| 870 | case DataType::Type::kInt16: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 871 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 872 | __ Shl(dst.V8H(), lhs.V8H(), value); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 873 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 874 | case DataType::Type::kInt32: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 875 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 876 | __ Shl(dst.V4S(), lhs.V4S(), value); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 877 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 878 | case DataType::Type::kInt64: |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 879 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 880 | __ Shl(dst.V2D(), lhs.V2D(), value); |
| 881 | break; |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 882 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 883 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 884 | UNREACHABLE(); |
| 885 | } |
| 886 | } |
| 887 | |
| 888 | void LocationsBuilderARM64::VisitVecShr(HVecShr* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 889 | CreateVecShiftLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 890 | } |
| 891 | |
| 892 | void InstructionCodeGeneratorARM64::VisitVecShr(HVecShr* instruction) { |
| 893 | LocationSummary* locations = instruction->GetLocations(); |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 894 | VRegister lhs = VRegisterFrom(locations->InAt(0)); |
| 895 | VRegister dst = VRegisterFrom(locations->Out()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 896 | int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); |
| 897 | switch (instruction->GetPackedType()) { |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 898 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 899 | case DataType::Type::kInt8: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 900 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 901 | __ Sshr(dst.V16B(), lhs.V16B(), value); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 902 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 903 | case DataType::Type::kUint16: |
| 904 | case DataType::Type::kInt16: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 905 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 906 | __ Sshr(dst.V8H(), lhs.V8H(), value); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 907 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 908 | case DataType::Type::kInt32: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 909 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 910 | __ Sshr(dst.V4S(), lhs.V4S(), value); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 911 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 912 | case DataType::Type::kInt64: |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 913 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 914 | __ Sshr(dst.V2D(), lhs.V2D(), value); |
| 915 | break; |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 916 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 917 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 918 | UNREACHABLE(); |
| 919 | } |
| 920 | } |
| 921 | |
| 922 | void LocationsBuilderARM64::VisitVecUShr(HVecUShr* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 923 | CreateVecShiftLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 924 | } |
| 925 | |
| 926 | void InstructionCodeGeneratorARM64::VisitVecUShr(HVecUShr* instruction) { |
| 927 | LocationSummary* locations = instruction->GetLocations(); |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 928 | VRegister lhs = VRegisterFrom(locations->InAt(0)); |
| 929 | VRegister dst = VRegisterFrom(locations->Out()); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 930 | int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); |
| 931 | switch (instruction->GetPackedType()) { |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 932 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 933 | case DataType::Type::kInt8: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 934 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 935 | __ Ushr(dst.V16B(), lhs.V16B(), value); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 936 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 937 | case DataType::Type::kUint16: |
| 938 | case DataType::Type::kInt16: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 939 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 940 | __ Ushr(dst.V8H(), lhs.V8H(), value); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 941 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 942 | case DataType::Type::kInt32: |
Artem Serov | d4bccf1 | 2017-04-03 18:47:32 +0100 | [diff] [blame] | 943 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 944 | __ Ushr(dst.V4S(), lhs.V4S(), value); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 945 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 946 | case DataType::Type::kInt64: |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 947 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 948 | __ Ushr(dst.V2D(), lhs.V2D(), value); |
| 949 | break; |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 950 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 951 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 952 | UNREACHABLE(); |
| 953 | } |
| 954 | } |
| 955 | |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 956 | void LocationsBuilderARM64::VisitVecSetScalars(HVecSetScalars* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 957 | LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 958 | |
| 959 | DCHECK_EQ(1u, instruction->InputCount()); // only one input currently implemented |
| 960 | |
| 961 | HInstruction* input = instruction->InputAt(0); |
| 962 | bool is_zero = IsZeroBitPattern(input); |
| 963 | |
| 964 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 965 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 966 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 967 | case DataType::Type::kInt8: |
| 968 | case DataType::Type::kUint16: |
| 969 | case DataType::Type::kInt16: |
| 970 | case DataType::Type::kInt32: |
| 971 | case DataType::Type::kInt64: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 972 | locations->SetInAt(0, is_zero ? Location::ConstantLocation(input->AsConstant()) |
| 973 | : Location::RequiresRegister()); |
| 974 | locations->SetOut(Location::RequiresFpuRegister()); |
| 975 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 976 | case DataType::Type::kFloat32: |
| 977 | case DataType::Type::kFloat64: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 978 | locations->SetInAt(0, is_zero ? Location::ConstantLocation(input->AsConstant()) |
| 979 | : Location::RequiresFpuRegister()); |
| 980 | locations->SetOut(Location::RequiresFpuRegister()); |
| 981 | break; |
| 982 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 983 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 984 | UNREACHABLE(); |
| 985 | } |
| 986 | } |
| 987 | |
| 988 | void InstructionCodeGeneratorARM64::VisitVecSetScalars(HVecSetScalars* instruction) { |
| 989 | LocationSummary* locations = instruction->GetLocations(); |
| 990 | VRegister dst = VRegisterFrom(locations->Out()); |
| 991 | |
| 992 | DCHECK_EQ(1u, instruction->InputCount()); // only one input currently implemented |
| 993 | |
| 994 | // Zero out all other elements first. |
| 995 | __ Movi(dst.V16B(), 0); |
| 996 | |
| 997 | // Shorthand for any type of zero. |
| 998 | if (IsZeroBitPattern(instruction->InputAt(0))) { |
| 999 | return; |
| 1000 | } |
| 1001 | |
| 1002 | // Set required elements. |
| 1003 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1004 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 1005 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1006 | case DataType::Type::kInt8: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 1007 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 1008 | __ Mov(dst.V16B(), 0, InputRegisterAt(instruction, 0)); |
| 1009 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1010 | case DataType::Type::kUint16: |
| 1011 | case DataType::Type::kInt16: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 1012 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 1013 | __ Mov(dst.V8H(), 0, InputRegisterAt(instruction, 0)); |
| 1014 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1015 | case DataType::Type::kInt32: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 1016 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 1017 | __ Mov(dst.V4S(), 0, InputRegisterAt(instruction, 0)); |
| 1018 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1019 | case DataType::Type::kInt64: |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 1020 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 1021 | __ Mov(dst.V2D(), 0, InputRegisterAt(instruction, 0)); |
| 1022 | break; |
| 1023 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 1024 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | 0148de4 | 2017-09-05 09:25:01 -0700 | [diff] [blame] | 1025 | UNREACHABLE(); |
| 1026 | } |
| 1027 | } |
| 1028 | |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1029 | // Helper to set up locations for vector accumulations. |
Vladimir Marko | e764d2e | 2017-10-05 14:35:55 +0100 | [diff] [blame] | 1030 | static void CreateVecAccumLocations(ArenaAllocator* allocator, HVecOperation* instruction) { |
| 1031 | LocationSummary* locations = new (allocator) LocationSummary(instruction); |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1032 | switch (instruction->GetPackedType()) { |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 1033 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1034 | case DataType::Type::kInt8: |
| 1035 | case DataType::Type::kUint16: |
| 1036 | case DataType::Type::kInt16: |
| 1037 | case DataType::Type::kInt32: |
| 1038 | case DataType::Type::kInt64: |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1039 | locations->SetInAt(0, Location::RequiresFpuRegister()); |
| 1040 | locations->SetInAt(1, Location::RequiresFpuRegister()); |
| 1041 | locations->SetInAt(2, Location::RequiresFpuRegister()); |
Artem Serov | f34dd20 | 2017-04-10 17:41:46 +0100 | [diff] [blame] | 1042 | locations->SetOut(Location::SameAsFirstInput()); |
| 1043 | break; |
| 1044 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 1045 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Artem Serov | f34dd20 | 2017-04-10 17:41:46 +0100 | [diff] [blame] | 1046 | UNREACHABLE(); |
| 1047 | } |
| 1048 | } |
| 1049 | |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1050 | void LocationsBuilderARM64::VisitVecMultiplyAccumulate(HVecMultiplyAccumulate* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 1051 | CreateVecAccumLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1052 | } |
| 1053 | |
Artem Serov | f34dd20 | 2017-04-10 17:41:46 +0100 | [diff] [blame] | 1054 | // Some early revisions of the Cortex-A53 have an erratum (835769) whereby it is possible for a |
| 1055 | // 64-bit scalar multiply-accumulate instruction in AArch64 state to generate an incorrect result. |
| 1056 | // However vector MultiplyAccumulate instruction is not affected. |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1057 | void InstructionCodeGeneratorARM64::VisitVecMultiplyAccumulate(HVecMultiplyAccumulate* instruction) { |
| 1058 | LocationSummary* locations = instruction->GetLocations(); |
| 1059 | VRegister acc = VRegisterFrom(locations->InAt(0)); |
| 1060 | VRegister left = VRegisterFrom(locations->InAt(1)); |
| 1061 | VRegister right = VRegisterFrom(locations->InAt(2)); |
| 1062 | |
| 1063 | DCHECK(locations->InAt(0).Equals(locations->Out())); |
| 1064 | |
| 1065 | switch (instruction->GetPackedType()) { |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 1066 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1067 | case DataType::Type::kInt8: |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1068 | DCHECK_EQ(16u, instruction->GetVectorLength()); |
| 1069 | if (instruction->GetOpKind() == HInstruction::kAdd) { |
Artem Serov | f34dd20 | 2017-04-10 17:41:46 +0100 | [diff] [blame] | 1070 | __ Mla(acc.V16B(), left.V16B(), right.V16B()); |
| 1071 | } else { |
| 1072 | __ Mls(acc.V16B(), left.V16B(), right.V16B()); |
| 1073 | } |
| 1074 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1075 | case DataType::Type::kUint16: |
| 1076 | case DataType::Type::kInt16: |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1077 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 1078 | if (instruction->GetOpKind() == HInstruction::kAdd) { |
Artem Serov | f34dd20 | 2017-04-10 17:41:46 +0100 | [diff] [blame] | 1079 | __ Mla(acc.V8H(), left.V8H(), right.V8H()); |
| 1080 | } else { |
| 1081 | __ Mls(acc.V8H(), left.V8H(), right.V8H()); |
| 1082 | } |
| 1083 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1084 | case DataType::Type::kInt32: |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1085 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 1086 | if (instruction->GetOpKind() == HInstruction::kAdd) { |
Artem Serov | f34dd20 | 2017-04-10 17:41:46 +0100 | [diff] [blame] | 1087 | __ Mla(acc.V4S(), left.V4S(), right.V4S()); |
| 1088 | } else { |
| 1089 | __ Mls(acc.V4S(), left.V4S(), right.V4S()); |
| 1090 | } |
| 1091 | break; |
| 1092 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 1093 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1094 | UNREACHABLE(); |
| 1095 | } |
| 1096 | } |
| 1097 | |
| 1098 | void LocationsBuilderARM64::VisitVecSADAccumulate(HVecSADAccumulate* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 1099 | CreateVecAccumLocations(GetGraph()->GetAllocator(), instruction); |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1100 | // Some conversions require temporary registers. |
| 1101 | LocationSummary* locations = instruction->GetLocations(); |
| 1102 | HVecOperation* a = instruction->InputAt(1)->AsVecOperation(); |
| 1103 | HVecOperation* b = instruction->InputAt(2)->AsVecOperation(); |
Vladimir Marko | 61b9228 | 2017-10-11 13:23:17 +0100 | [diff] [blame] | 1104 | DCHECK_EQ(HVecOperation::ToSignedType(a->GetPackedType()), |
| 1105 | HVecOperation::ToSignedType(b->GetPackedType())); |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1106 | switch (a->GetPackedType()) { |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 1107 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1108 | case DataType::Type::kInt8: |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1109 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1110 | case DataType::Type::kInt64: |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1111 | locations->AddTemp(Location::RequiresFpuRegister()); |
| 1112 | locations->AddTemp(Location::RequiresFpuRegister()); |
| 1113 | FALLTHROUGH_INTENDED; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1114 | case DataType::Type::kInt32: |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1115 | locations->AddTemp(Location::RequiresFpuRegister()); |
| 1116 | locations->AddTemp(Location::RequiresFpuRegister()); |
| 1117 | break; |
| 1118 | default: |
| 1119 | break; |
| 1120 | } |
| 1121 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1122 | case DataType::Type::kUint16: |
| 1123 | case DataType::Type::kInt16: |
| 1124 | if (instruction->GetPackedType() == DataType::Type::kInt64) { |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1125 | locations->AddTemp(Location::RequiresFpuRegister()); |
| 1126 | locations->AddTemp(Location::RequiresFpuRegister()); |
| 1127 | } |
| 1128 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1129 | case DataType::Type::kInt32: |
| 1130 | case DataType::Type::kInt64: |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1131 | if (instruction->GetPackedType() == a->GetPackedType()) { |
| 1132 | locations->AddTemp(Location::RequiresFpuRegister()); |
| 1133 | } |
| 1134 | break; |
| 1135 | default: |
| 1136 | break; |
| 1137 | } |
| 1138 | } |
| 1139 | |
| 1140 | void InstructionCodeGeneratorARM64::VisitVecSADAccumulate(HVecSADAccumulate* instruction) { |
| 1141 | LocationSummary* locations = instruction->GetLocations(); |
| 1142 | VRegister acc = VRegisterFrom(locations->InAt(0)); |
| 1143 | VRegister left = VRegisterFrom(locations->InAt(1)); |
| 1144 | VRegister right = VRegisterFrom(locations->InAt(2)); |
| 1145 | |
| 1146 | DCHECK(locations->InAt(0).Equals(locations->Out())); |
| 1147 | |
| 1148 | // Handle all feasible acc_T += sad(a_S, b_S) type combinations (T x S). |
| 1149 | HVecOperation* a = instruction->InputAt(1)->AsVecOperation(); |
| 1150 | HVecOperation* b = instruction->InputAt(2)->AsVecOperation(); |
Vladimir Marko | 61b9228 | 2017-10-11 13:23:17 +0100 | [diff] [blame] | 1151 | DCHECK_EQ(HVecOperation::ToSignedType(a->GetPackedType()), |
| 1152 | HVecOperation::ToSignedType(b->GetPackedType())); |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1153 | switch (a->GetPackedType()) { |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 1154 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1155 | case DataType::Type::kInt8: |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1156 | DCHECK_EQ(16u, a->GetVectorLength()); |
| 1157 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1158 | case DataType::Type::kInt16: |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1159 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 1160 | __ Sabal(acc.V8H(), left.V8B(), right.V8B()); |
| 1161 | __ Sabal2(acc.V8H(), left.V16B(), right.V16B()); |
| 1162 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1163 | case DataType::Type::kInt32: { |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1164 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 1165 | VRegister tmp1 = VRegisterFrom(locations->GetTemp(0)); |
| 1166 | VRegister tmp2 = VRegisterFrom(locations->GetTemp(1)); |
| 1167 | __ Sxtl(tmp1.V8H(), left.V8B()); |
| 1168 | __ Sxtl(tmp2.V8H(), right.V8B()); |
| 1169 | __ Sabal(acc.V4S(), tmp1.V4H(), tmp2.V4H()); |
| 1170 | __ Sabal2(acc.V4S(), tmp1.V8H(), tmp2.V8H()); |
| 1171 | __ Sxtl2(tmp1.V8H(), left.V16B()); |
| 1172 | __ Sxtl2(tmp2.V8H(), right.V16B()); |
| 1173 | __ Sabal(acc.V4S(), tmp1.V4H(), tmp2.V4H()); |
| 1174 | __ Sabal2(acc.V4S(), tmp1.V8H(), tmp2.V8H()); |
| 1175 | break; |
| 1176 | } |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1177 | case DataType::Type::kInt64: { |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1178 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 1179 | VRegister tmp1 = VRegisterFrom(locations->GetTemp(0)); |
| 1180 | VRegister tmp2 = VRegisterFrom(locations->GetTemp(1)); |
| 1181 | VRegister tmp3 = VRegisterFrom(locations->GetTemp(2)); |
| 1182 | VRegister tmp4 = VRegisterFrom(locations->GetTemp(3)); |
| 1183 | __ Sxtl(tmp1.V8H(), left.V8B()); |
| 1184 | __ Sxtl(tmp2.V8H(), right.V8B()); |
| 1185 | __ Sxtl(tmp3.V4S(), tmp1.V4H()); |
| 1186 | __ Sxtl(tmp4.V4S(), tmp2.V4H()); |
| 1187 | __ Sabal(acc.V2D(), tmp3.V2S(), tmp4.V2S()); |
| 1188 | __ Sabal2(acc.V2D(), tmp3.V4S(), tmp4.V4S()); |
| 1189 | __ Sxtl2(tmp3.V4S(), tmp1.V8H()); |
| 1190 | __ Sxtl2(tmp4.V4S(), tmp2.V8H()); |
| 1191 | __ Sabal(acc.V2D(), tmp3.V2S(), tmp4.V2S()); |
| 1192 | __ Sabal2(acc.V2D(), tmp3.V4S(), tmp4.V4S()); |
| 1193 | __ Sxtl2(tmp1.V8H(), left.V16B()); |
| 1194 | __ Sxtl2(tmp2.V8H(), right.V16B()); |
| 1195 | __ Sxtl(tmp3.V4S(), tmp1.V4H()); |
| 1196 | __ Sxtl(tmp4.V4S(), tmp2.V4H()); |
| 1197 | __ Sabal(acc.V2D(), tmp3.V2S(), tmp4.V2S()); |
| 1198 | __ Sabal2(acc.V2D(), tmp3.V4S(), tmp4.V4S()); |
| 1199 | __ Sxtl2(tmp3.V4S(), tmp1.V8H()); |
| 1200 | __ Sxtl2(tmp4.V4S(), tmp2.V8H()); |
| 1201 | __ Sabal(acc.V2D(), tmp3.V2S(), tmp4.V2S()); |
| 1202 | __ Sabal2(acc.V2D(), tmp3.V4S(), tmp4.V4S()); |
| 1203 | break; |
| 1204 | } |
| 1205 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 1206 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1207 | UNREACHABLE(); |
| 1208 | } |
| 1209 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1210 | case DataType::Type::kUint16: |
| 1211 | case DataType::Type::kInt16: |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1212 | DCHECK_EQ(8u, a->GetVectorLength()); |
| 1213 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1214 | case DataType::Type::kInt32: |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1215 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 1216 | __ Sabal(acc.V4S(), left.V4H(), right.V4H()); |
| 1217 | __ Sabal2(acc.V4S(), left.V8H(), right.V8H()); |
| 1218 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1219 | case DataType::Type::kInt64: { |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1220 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 1221 | VRegister tmp1 = VRegisterFrom(locations->GetTemp(0)); |
| 1222 | VRegister tmp2 = VRegisterFrom(locations->GetTemp(1)); |
| 1223 | __ Sxtl(tmp1.V4S(), left.V4H()); |
| 1224 | __ Sxtl(tmp2.V4S(), right.V4H()); |
| 1225 | __ Sabal(acc.V2D(), tmp1.V2S(), tmp2.V2S()); |
| 1226 | __ Sabal2(acc.V2D(), tmp1.V4S(), tmp2.V4S()); |
| 1227 | __ Sxtl2(tmp1.V4S(), left.V8H()); |
| 1228 | __ Sxtl2(tmp2.V4S(), right.V8H()); |
| 1229 | __ Sabal(acc.V2D(), tmp1.V2S(), tmp2.V2S()); |
| 1230 | __ Sabal2(acc.V2D(), tmp1.V4S(), tmp2.V4S()); |
| 1231 | break; |
| 1232 | } |
| 1233 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 1234 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1235 | UNREACHABLE(); |
| 1236 | } |
| 1237 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1238 | case DataType::Type::kInt32: |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1239 | DCHECK_EQ(4u, a->GetVectorLength()); |
| 1240 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1241 | case DataType::Type::kInt32: { |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1242 | DCHECK_EQ(4u, instruction->GetVectorLength()); |
| 1243 | VRegister tmp = VRegisterFrom(locations->GetTemp(0)); |
| 1244 | __ Sub(tmp.V4S(), left.V4S(), right.V4S()); |
| 1245 | __ Abs(tmp.V4S(), tmp.V4S()); |
| 1246 | __ Add(acc.V4S(), acc.V4S(), tmp.V4S()); |
| 1247 | break; |
| 1248 | } |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1249 | case DataType::Type::kInt64: |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1250 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 1251 | __ Sabal(acc.V2D(), left.V2S(), right.V2S()); |
| 1252 | __ Sabal2(acc.V2D(), left.V4S(), right.V4S()); |
| 1253 | break; |
| 1254 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 1255 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1256 | UNREACHABLE(); |
| 1257 | } |
| 1258 | break; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1259 | case DataType::Type::kInt64: |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1260 | DCHECK_EQ(2u, a->GetVectorLength()); |
| 1261 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1262 | case DataType::Type::kInt64: { |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1263 | DCHECK_EQ(2u, instruction->GetVectorLength()); |
| 1264 | VRegister tmp = VRegisterFrom(locations->GetTemp(0)); |
| 1265 | __ Sub(tmp.V2D(), left.V2D(), right.V2D()); |
| 1266 | __ Abs(tmp.V2D(), tmp.V2D()); |
| 1267 | __ Add(acc.V2D(), acc.V2D(), tmp.V2D()); |
| 1268 | break; |
| 1269 | } |
| 1270 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 1271 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | dbbac8f | 2017-09-01 13:06:08 -0700 | [diff] [blame] | 1272 | UNREACHABLE(); |
| 1273 | } |
| 1274 | break; |
| 1275 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 1276 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Artem Serov | f34dd20 | 2017-04-10 17:41:46 +0100 | [diff] [blame] | 1277 | } |
| 1278 | } |
| 1279 | |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1280 | // Helper to set up locations for vector memory operations. |
Vladimir Marko | e764d2e | 2017-10-05 14:35:55 +0100 | [diff] [blame] | 1281 | static void CreateVecMemLocations(ArenaAllocator* allocator, |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1282 | HVecMemoryOperation* instruction, |
| 1283 | bool is_load) { |
Vladimir Marko | e764d2e | 2017-10-05 14:35:55 +0100 | [diff] [blame] | 1284 | LocationSummary* locations = new (allocator) LocationSummary(instruction); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1285 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1286 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 1287 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1288 | case DataType::Type::kInt8: |
| 1289 | case DataType::Type::kUint16: |
| 1290 | case DataType::Type::kInt16: |
| 1291 | case DataType::Type::kInt32: |
| 1292 | case DataType::Type::kInt64: |
| 1293 | case DataType::Type::kFloat32: |
| 1294 | case DataType::Type::kFloat64: |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1295 | locations->SetInAt(0, Location::RequiresRegister()); |
| 1296 | locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); |
| 1297 | if (is_load) { |
| 1298 | locations->SetOut(Location::RequiresFpuRegister()); |
| 1299 | } else { |
| 1300 | locations->SetInAt(2, Location::RequiresFpuRegister()); |
| 1301 | } |
| 1302 | break; |
| 1303 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 1304 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1305 | UNREACHABLE(); |
| 1306 | } |
| 1307 | } |
| 1308 | |
Aart Bik | 472821b | 2017-04-27 17:23:51 -0700 | [diff] [blame] | 1309 | // Helper to set up locations for vector memory operations. Returns the memory operand and, |
| 1310 | // if used, sets the output parameter scratch to a temporary register used in this operand, |
| 1311 | // so that the client can release it right after the memory operand use. |
| 1312 | MemOperand InstructionCodeGeneratorARM64::VecAddress( |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1313 | HVecMemoryOperation* instruction, |
Aart Bik | 472821b | 2017-04-27 17:23:51 -0700 | [diff] [blame] | 1314 | UseScratchRegisterScope* temps_scope, |
| 1315 | size_t size, |
| 1316 | bool is_string_char_at, |
| 1317 | /*out*/ Register* scratch) { |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1318 | LocationSummary* locations = instruction->GetLocations(); |
| 1319 | Register base = InputRegisterAt(instruction, 0); |
Artem Serov | e1811ed | 2017-04-27 16:50:47 +0100 | [diff] [blame] | 1320 | |
| 1321 | if (instruction->InputAt(1)->IsIntermediateAddressIndex()) { |
| 1322 | DCHECK(!is_string_char_at); |
| 1323 | return MemOperand(base.X(), InputRegisterAt(instruction, 1).X()); |
| 1324 | } |
| 1325 | |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1326 | Location index = locations->InAt(1); |
Aart Bik | 472821b | 2017-04-27 17:23:51 -0700 | [diff] [blame] | 1327 | uint32_t offset = is_string_char_at |
| 1328 | ? mirror::String::ValueOffset().Uint32Value() |
| 1329 | : mirror::Array::DataOffset(size).Uint32Value(); |
| 1330 | size_t shift = ComponentSizeShiftWidth(size); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1331 | |
Artem Serov | 0225b77 | 2017-04-19 15:43:53 +0100 | [diff] [blame] | 1332 | // HIntermediateAddress optimization is only applied for scalar ArrayGet and ArraySet. |
| 1333 | DCHECK(!instruction->InputAt(0)->IsIntermediateAddress()); |
| 1334 | |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1335 | if (index.IsConstant()) { |
| 1336 | offset += Int64ConstantFrom(index) << shift; |
Artem Serov | 0225b77 | 2017-04-19 15:43:53 +0100 | [diff] [blame] | 1337 | return HeapOperand(base, offset); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1338 | } else { |
Aart Bik | 472821b | 2017-04-27 17:23:51 -0700 | [diff] [blame] | 1339 | *scratch = temps_scope->AcquireSameSizeAs(base); |
| 1340 | __ Add(*scratch, base, Operand(WRegisterFrom(index), LSL, shift)); |
| 1341 | return HeapOperand(*scratch, offset); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1342 | } |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1343 | } |
| 1344 | |
| 1345 | void LocationsBuilderARM64::VisitVecLoad(HVecLoad* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 1346 | CreateVecMemLocations(GetGraph()->GetAllocator(), instruction, /*is_load*/ true); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1347 | } |
| 1348 | |
| 1349 | void InstructionCodeGeneratorARM64::VisitVecLoad(HVecLoad* instruction) { |
Aart Bik | 472821b | 2017-04-27 17:23:51 -0700 | [diff] [blame] | 1350 | LocationSummary* locations = instruction->GetLocations(); |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1351 | size_t size = DataType::Size(instruction->GetPackedType()); |
Aart Bik | 472821b | 2017-04-27 17:23:51 -0700 | [diff] [blame] | 1352 | VRegister reg = VRegisterFrom(locations->Out()); |
Artem Serov | 0225b77 | 2017-04-19 15:43:53 +0100 | [diff] [blame] | 1353 | UseScratchRegisterScope temps(GetVIXLAssembler()); |
Aart Bik | 472821b | 2017-04-27 17:23:51 -0700 | [diff] [blame] | 1354 | Register scratch; |
Artem Serov | 0225b77 | 2017-04-19 15:43:53 +0100 | [diff] [blame] | 1355 | |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1356 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1357 | case DataType::Type::kUint16: |
Aart Bik | 472821b | 2017-04-27 17:23:51 -0700 | [diff] [blame] | 1358 | DCHECK_EQ(8u, instruction->GetVectorLength()); |
| 1359 | // Special handling of compressed/uncompressed string load. |
| 1360 | if (mirror::kUseStringCompression && instruction->IsStringCharAt()) { |
| 1361 | vixl::aarch64::Label uncompressed_load, done; |
| 1362 | // Test compression bit. |
| 1363 | static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, |
| 1364 | "Expecting 0=compressed, 1=uncompressed"); |
| 1365 | uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); |
| 1366 | Register length = temps.AcquireW(); |
| 1367 | __ Ldr(length, HeapOperand(InputRegisterAt(instruction, 0), count_offset)); |
| 1368 | __ Tbnz(length.W(), 0, &uncompressed_load); |
| 1369 | temps.Release(length); // no longer needed |
| 1370 | // Zero extend 8 compressed bytes into 8 chars. |
| 1371 | __ Ldr(DRegisterFrom(locations->Out()).V8B(), |
| 1372 | VecAddress(instruction, &temps, 1, /*is_string_char_at*/ true, &scratch)); |
| 1373 | __ Uxtl(reg.V8H(), reg.V8B()); |
| 1374 | __ B(&done); |
| 1375 | if (scratch.IsValid()) { |
| 1376 | temps.Release(scratch); // if used, no longer needed |
| 1377 | } |
| 1378 | // Load 8 direct uncompressed chars. |
| 1379 | __ Bind(&uncompressed_load); |
| 1380 | __ Ldr(reg, VecAddress(instruction, &temps, size, /*is_string_char_at*/ true, &scratch)); |
| 1381 | __ Bind(&done); |
| 1382 | return; |
| 1383 | } |
| 1384 | FALLTHROUGH_INTENDED; |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1385 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 1386 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1387 | case DataType::Type::kInt8: |
| 1388 | case DataType::Type::kInt16: |
| 1389 | case DataType::Type::kInt32: |
| 1390 | case DataType::Type::kFloat32: |
| 1391 | case DataType::Type::kInt64: |
| 1392 | case DataType::Type::kFloat64: |
Artem Serov | 0225b77 | 2017-04-19 15:43:53 +0100 | [diff] [blame] | 1393 | DCHECK_LE(2u, instruction->GetVectorLength()); |
| 1394 | DCHECK_LE(instruction->GetVectorLength(), 16u); |
Aart Bik | 472821b | 2017-04-27 17:23:51 -0700 | [diff] [blame] | 1395 | __ Ldr(reg, VecAddress(instruction, &temps, size, instruction->IsStringCharAt(), &scratch)); |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 1396 | break; |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1397 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 1398 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1399 | UNREACHABLE(); |
| 1400 | } |
| 1401 | } |
| 1402 | |
| 1403 | void LocationsBuilderARM64::VisitVecStore(HVecStore* instruction) { |
Vladimir Marko | ca6fff8 | 2017-10-03 14:49:14 +0100 | [diff] [blame] | 1404 | CreateVecMemLocations(GetGraph()->GetAllocator(), instruction, /*is_load*/ false); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1405 | } |
| 1406 | |
| 1407 | void InstructionCodeGeneratorARM64::VisitVecStore(HVecStore* instruction) { |
Aart Bik | 472821b | 2017-04-27 17:23:51 -0700 | [diff] [blame] | 1408 | LocationSummary* locations = instruction->GetLocations(); |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1409 | size_t size = DataType::Size(instruction->GetPackedType()); |
Aart Bik | 472821b | 2017-04-27 17:23:51 -0700 | [diff] [blame] | 1410 | VRegister reg = VRegisterFrom(locations->InAt(2)); |
Artem Serov | 0225b77 | 2017-04-19 15:43:53 +0100 | [diff] [blame] | 1411 | UseScratchRegisterScope temps(GetVIXLAssembler()); |
Aart Bik | 472821b | 2017-04-27 17:23:51 -0700 | [diff] [blame] | 1412 | Register scratch; |
Artem Serov | 0225b77 | 2017-04-19 15:43:53 +0100 | [diff] [blame] | 1413 | |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1414 | switch (instruction->GetPackedType()) { |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1415 | case DataType::Type::kBool: |
Vladimir Marko | d5d2f2c | 2017-09-26 12:37:26 +0100 | [diff] [blame] | 1416 | case DataType::Type::kUint8: |
Vladimir Marko | 0ebe0d8 | 2017-09-21 22:50:39 +0100 | [diff] [blame] | 1417 | case DataType::Type::kInt8: |
| 1418 | case DataType::Type::kUint16: |
| 1419 | case DataType::Type::kInt16: |
| 1420 | case DataType::Type::kInt32: |
| 1421 | case DataType::Type::kFloat32: |
| 1422 | case DataType::Type::kInt64: |
| 1423 | case DataType::Type::kFloat64: |
Artem Serov | 0225b77 | 2017-04-19 15:43:53 +0100 | [diff] [blame] | 1424 | DCHECK_LE(2u, instruction->GetVectorLength()); |
| 1425 | DCHECK_LE(instruction->GetVectorLength(), 16u); |
Aart Bik | 472821b | 2017-04-27 17:23:51 -0700 | [diff] [blame] | 1426 | __ Str(reg, VecAddress(instruction, &temps, size, /*is_string_char_at*/ false, &scratch)); |
Artem Serov | b31f91f | 2017-04-05 11:31:19 +0100 | [diff] [blame] | 1427 | break; |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1428 | default: |
Aart Bik | 29aa082 | 2018-03-08 11:28:00 -0800 | [diff] [blame] | 1429 | LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); |
Aart Bik | f8f5a16 | 2017-02-06 15:35:29 -0800 | [diff] [blame] | 1430 | UNREACHABLE(); |
| 1431 | } |
| 1432 | } |
| 1433 | |
| 1434 | #undef __ |
| 1435 | |
| 1436 | } // namespace arm64 |
| 1437 | } // namespace art |