blob: df58e7dbef0406ec4061f1bcf2dad20953c71c2d [file] [log] [blame]
Tom Stellardca166212017-01-30 21:56:46 +00001//===- AMDGPULegalizerInfo.cpp -----------------------------------*- C++ -*-==//
2//
Chandler Carruth2946cd72019-01-19 08:50:56 +00003// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Tom Stellardca166212017-01-30 21:56:46 +00006//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the Machinelegalizer class for
10/// AMDGPU.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
David Blaikie36a0f222018-03-23 23:58:31 +000014#include "AMDGPU.h"
Craig Topper2fa14362018-03-29 17:21:10 +000015#include "AMDGPULegalizerInfo.h"
Matt Arsenault85803362018-03-17 15:17:41 +000016#include "AMDGPUTargetMachine.h"
Matt Arsenaulta8b43392019-02-08 02:40:47 +000017#include "SIMachineFunctionInfo.h"
Matt Arsenault6ce1b4f2019-07-10 16:31:19 +000018#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
Matt Arsenaulta8b43392019-02-08 02:40:47 +000019#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
David Blaikieb3bde2e2017-11-17 01:07:10 +000020#include "llvm/CodeGen/TargetOpcodes.h"
Craig Topper2fa14362018-03-29 17:21:10 +000021#include "llvm/CodeGen/ValueTypes.h"
Tom Stellardca166212017-01-30 21:56:46 +000022#include "llvm/IR/DerivedTypes.h"
Chandler Carruth6bda14b2017-06-06 11:49:48 +000023#include "llvm/IR/Type.h"
Tom Stellardca166212017-01-30 21:56:46 +000024#include "llvm/Support/Debug.h"
25
Matt Arsenaulte2c86cc2019-07-01 18:45:36 +000026#define DEBUG_TYPE "amdgpu-legalinfo"
27
Tom Stellardca166212017-01-30 21:56:46 +000028using namespace llvm;
Daniel Sanders9ade5592018-01-29 17:37:29 +000029using namespace LegalizeActions;
Matt Arsenault990f5072019-01-25 00:51:00 +000030using namespace LegalizeMutations;
Matt Arsenault7ac79ed2019-01-20 19:45:18 +000031using namespace LegalityPredicates;
Tom Stellardca166212017-01-30 21:56:46 +000032
Matt Arsenaultd9141892019-02-07 19:10:15 +000033
34static LegalityPredicate isMultiple32(unsigned TypeIdx,
35 unsigned MaxSize = 512) {
36 return [=](const LegalityQuery &Query) {
37 const LLT Ty = Query.Types[TypeIdx];
38 const LLT EltTy = Ty.getScalarType();
39 return Ty.getSizeInBits() <= MaxSize && EltTy.getSizeInBits() % 32 == 0;
40 };
41}
42
Matt Arsenault18ec3822019-02-11 22:00:39 +000043static LegalityPredicate isSmallOddVector(unsigned TypeIdx) {
44 return [=](const LegalityQuery &Query) {
45 const LLT Ty = Query.Types[TypeIdx];
46 return Ty.isVector() &&
47 Ty.getNumElements() % 2 != 0 &&
48 Ty.getElementType().getSizeInBits() < 32;
49 };
50}
51
52static LegalizeMutation oneMoreElement(unsigned TypeIdx) {
53 return [=](const LegalityQuery &Query) {
54 const LLT Ty = Query.Types[TypeIdx];
55 const LLT EltTy = Ty.getElementType();
56 return std::make_pair(TypeIdx, LLT::vector(Ty.getNumElements() + 1, EltTy));
57 };
58}
59
Matt Arsenault26b7e852019-02-19 16:30:19 +000060static LegalizeMutation fewerEltsToSize64Vector(unsigned TypeIdx) {
61 return [=](const LegalityQuery &Query) {
62 const LLT Ty = Query.Types[TypeIdx];
63 const LLT EltTy = Ty.getElementType();
64 unsigned Size = Ty.getSizeInBits();
65 unsigned Pieces = (Size + 63) / 64;
66 unsigned NewNumElts = (Ty.getNumElements() + 1) / Pieces;
67 return std::make_pair(TypeIdx, LLT::scalarOrVector(NewNumElts, EltTy));
68 };
69}
70
71static LegalityPredicate vectorWiderThan(unsigned TypeIdx, unsigned Size) {
72 return [=](const LegalityQuery &Query) {
73 const LLT QueryTy = Query.Types[TypeIdx];
74 return QueryTy.isVector() && QueryTy.getSizeInBits() > Size;
75 };
76}
77
Matt Arsenaultb4c95b32019-02-19 17:03:09 +000078static LegalityPredicate numElementsNotEven(unsigned TypeIdx) {
79 return [=](const LegalityQuery &Query) {
80 const LLT QueryTy = Query.Types[TypeIdx];
81 return QueryTy.isVector() && QueryTy.getNumElements() % 2 != 0;
82 };
83}
Matt Arsenault18ec3822019-02-11 22:00:39 +000084
Matt Arsenault4dd57552019-07-09 14:17:31 +000085// Any combination of 32 or 64-bit elements up to 512 bits, and multiples of
86// v2s16.
87static LegalityPredicate isRegisterType(unsigned TypeIdx) {
88 return [=](const LegalityQuery &Query) {
89 const LLT Ty = Query.Types[TypeIdx];
90 if (Ty.isVector()) {
91 const int EltSize = Ty.getElementType().getSizeInBits();
92 return EltSize == 32 || EltSize == 64 ||
Matt Arsenault3f1a3452019-07-09 22:48:04 +000093 (EltSize == 16 && Ty.getNumElements() % 2 == 0) ||
94 EltSize == 128 || EltSize == 256;
Matt Arsenault4dd57552019-07-09 14:17:31 +000095 }
96
97 return Ty.getSizeInBits() % 32 == 0 && Ty.getSizeInBits() <= 512;
98 };
99}
100
Matt Arsenault9e8e8c62019-07-01 18:49:01 +0000101AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
102 const GCNTargetMachine &TM)
103 : ST(ST_) {
Tom Stellardca166212017-01-30 21:56:46 +0000104 using namespace TargetOpcode;
105
Matt Arsenault85803362018-03-17 15:17:41 +0000106 auto GetAddrSpacePtr = [&TM](unsigned AS) {
107 return LLT::pointer(AS, TM.getPointerSizeInBits(AS));
108 };
109
110 const LLT S1 = LLT::scalar(1);
Matt Arsenault888aa5d2019-02-03 00:07:33 +0000111 const LLT S8 = LLT::scalar(8);
Matt Arsenault45991592019-01-18 21:33:50 +0000112 const LLT S16 = LLT::scalar(16);
Tom Stellardca166212017-01-30 21:56:46 +0000113 const LLT S32 = LLT::scalar(32);
114 const LLT S64 = LLT::scalar(64);
Matt Arsenaultca676342019-01-25 02:36:32 +0000115 const LLT S128 = LLT::scalar(128);
Matt Arsenaultff6a9a22019-01-20 18:40:36 +0000116 const LLT S256 = LLT::scalar(256);
Tom Stellardeebbfc22018-06-30 04:09:44 +0000117 const LLT S512 = LLT::scalar(512);
Matt Arsenault85803362018-03-17 15:17:41 +0000118
Matt Arsenaultbee2ad72018-12-21 03:03:11 +0000119 const LLT V2S16 = LLT::vector(2, 16);
Matt Arsenaulta1515d22019-01-08 01:30:02 +0000120 const LLT V4S16 = LLT::vector(4, 16);
Matt Arsenaultbee2ad72018-12-21 03:03:11 +0000121
122 const LLT V2S32 = LLT::vector(2, 32);
123 const LLT V3S32 = LLT::vector(3, 32);
124 const LLT V4S32 = LLT::vector(4, 32);
125 const LLT V5S32 = LLT::vector(5, 32);
126 const LLT V6S32 = LLT::vector(6, 32);
127 const LLT V7S32 = LLT::vector(7, 32);
128 const LLT V8S32 = LLT::vector(8, 32);
129 const LLT V9S32 = LLT::vector(9, 32);
130 const LLT V10S32 = LLT::vector(10, 32);
131 const LLT V11S32 = LLT::vector(11, 32);
132 const LLT V12S32 = LLT::vector(12, 32);
133 const LLT V13S32 = LLT::vector(13, 32);
134 const LLT V14S32 = LLT::vector(14, 32);
135 const LLT V15S32 = LLT::vector(15, 32);
136 const LLT V16S32 = LLT::vector(16, 32);
137
138 const LLT V2S64 = LLT::vector(2, 64);
139 const LLT V3S64 = LLT::vector(3, 64);
140 const LLT V4S64 = LLT::vector(4, 64);
141 const LLT V5S64 = LLT::vector(5, 64);
142 const LLT V6S64 = LLT::vector(6, 64);
143 const LLT V7S64 = LLT::vector(7, 64);
144 const LLT V8S64 = LLT::vector(8, 64);
145
146 std::initializer_list<LLT> AllS32Vectors =
147 {V2S32, V3S32, V4S32, V5S32, V6S32, V7S32, V8S32,
148 V9S32, V10S32, V11S32, V12S32, V13S32, V14S32, V15S32, V16S32};
149 std::initializer_list<LLT> AllS64Vectors =
150 {V2S64, V3S64, V4S64, V5S64, V6S64, V7S64, V8S64};
151
Matt Arsenault85803362018-03-17 15:17:41 +0000152 const LLT GlobalPtr = GetAddrSpacePtr(AMDGPUAS::GLOBAL_ADDRESS);
153 const LLT ConstantPtr = GetAddrSpacePtr(AMDGPUAS::CONSTANT_ADDRESS);
Matt Arsenault685d1e82018-03-17 15:17:45 +0000154 const LLT LocalPtr = GetAddrSpacePtr(AMDGPUAS::LOCAL_ADDRESS);
Matt Arsenault0da63502018-08-31 05:49:54 +0000155 const LLT FlatPtr = GetAddrSpacePtr(AMDGPUAS::FLAT_ADDRESS);
156 const LLT PrivatePtr = GetAddrSpacePtr(AMDGPUAS::PRIVATE_ADDRESS);
Matt Arsenault85803362018-03-17 15:17:41 +0000157
Matt Arsenault934e5342018-12-13 20:34:15 +0000158 const LLT CodePtr = FlatPtr;
159
Matt Arsenault9e5e8682019-02-14 22:24:28 +0000160 const std::initializer_list<LLT> AddrSpaces64 = {
161 GlobalPtr, ConstantPtr, FlatPtr
162 };
163
164 const std::initializer_list<LLT> AddrSpaces32 = {
165 LocalPtr, PrivatePtr
Matt Arsenault685d1e82018-03-17 15:17:45 +0000166 };
Tom Stellardca166212017-01-30 21:56:46 +0000167
Matt Arsenault40d1faf2019-07-01 17:35:53 +0000168 const std::initializer_list<LLT> FPTypesBase = {
169 S32, S64
170 };
171
172 const std::initializer_list<LLT> FPTypes16 = {
173 S32, S64, S16
174 };
175
Matt Arsenault6ce1b4f2019-07-10 16:31:19 +0000176 const std::initializer_list<LLT> FPTypesPK16 = {
177 S32, S64, S16, V2S16
178 };
179
Matt Arsenaultadc40ba2019-01-08 01:22:47 +0000180 setAction({G_BRCOND, S1}, Legal);
181
Matt Arsenault2e0ee472019-02-21 15:48:13 +0000182 // TODO: All multiples of 32, vectors of pointers, all v2s16 pairs, more
183 // elements for v3s16
184 getActionDefinitionsBuilder(G_PHI)
185 .legalFor({S32, S64, V2S16, V4S16, S1, S128, S256})
186 .legalFor(AllS32Vectors)
187 .legalFor(AllS64Vectors)
188 .legalFor(AddrSpaces64)
189 .legalFor(AddrSpaces32)
190 .clampScalar(0, S32, S256)
191 .widenScalarToNextPow2(0, 32)
Matt Arsenaultd3093c22019-02-28 00:16:32 +0000192 .clampMaxNumElements(0, S32, 16)
Matt Arsenault72bcf152019-02-28 00:01:05 +0000193 .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
Matt Arsenault2e0ee472019-02-21 15:48:13 +0000194 .legalIf(isPointer(0));
195
Matt Arsenaultef59cb62019-07-01 18:18:55 +0000196 if (ST.has16BitInsts()) {
197 getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL})
198 .legalFor({S32, S16})
199 .clampScalar(0, S16, S32)
200 .scalarize(0);
201 } else {
202 getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL})
203 .legalFor({S32})
204 .clampScalar(0, S32, S32)
205 .scalarize(0);
206 }
Matt Arsenault2e0ee472019-02-21 15:48:13 +0000207
Matt Arsenaultef59cb62019-07-01 18:18:55 +0000208 getActionDefinitionsBuilder({G_UMULH, G_SMULH})
Matt Arsenault5d622fb2019-01-25 03:23:04 +0000209 .legalFor({S32})
Matt Arsenault211e89d2019-01-27 00:52:51 +0000210 .clampScalar(0, S32, S32)
Matt Arsenault5d622fb2019-01-25 03:23:04 +0000211 .scalarize(0);
Matt Arsenault43398832018-12-20 01:35:49 +0000212
Matt Arsenault26a6c742019-01-26 23:47:07 +0000213 // Report legal for any types we can handle anywhere. For the cases only legal
214 // on the SALU, RegBankSelect will be able to re-legalize.
Matt Arsenault43398832018-12-20 01:35:49 +0000215 getActionDefinitionsBuilder({G_AND, G_OR, G_XOR})
Matt Arsenault26a6c742019-01-26 23:47:07 +0000216 .legalFor({S32, S1, S64, V2S32, V2S16, V4S16})
217 .clampScalar(0, S32, S64)
Matt Arsenault26b7e852019-02-19 16:30:19 +0000218 .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
219 .fewerElementsIf(vectorWiderThan(0, 32), fewerEltsToSize64Vector(0))
Matt Arsenaultf4bfe4c2019-02-25 21:32:48 +0000220 .widenScalarToNextPow2(0)
Matt Arsenault26a6c742019-01-26 23:47:07 +0000221 .scalarize(0);
Tom Stellardee6e6452017-06-12 20:54:56 +0000222
Matt Arsenault68c668a2019-01-08 01:09:09 +0000223 getActionDefinitionsBuilder({G_UADDO, G_SADDO, G_USUBO, G_SSUBO,
224 G_UADDE, G_SADDE, G_USUBE, G_SSUBE})
Matt Arsenault4d475942019-01-26 23:44:51 +0000225 .legalFor({{S32, S1}})
226 .clampScalar(0, S32, S32);
Matt Arsenault2cc15b62019-01-08 01:03:58 +0000227
Matt Arsenault7ac79ed2019-01-20 19:45:18 +0000228 getActionDefinitionsBuilder(G_BITCAST)
229 .legalForCartesianProduct({S32, V2S16})
230 .legalForCartesianProduct({S64, V2S32, V4S16})
231 .legalForCartesianProduct({V2S64, V4S32})
232 // Don't worry about the size constraint.
233 .legalIf(all(isPointer(0), isPointer(1)));
Tom Stellardff63ee02017-06-19 13:15:45 +0000234
Matt Arsenault00ccd132019-02-12 14:54:55 +0000235 if (ST.has16BitInsts()) {
236 getActionDefinitionsBuilder(G_FCONSTANT)
237 .legalFor({S32, S64, S16})
238 .clampScalar(0, S16, S64);
239 } else {
240 getActionDefinitionsBuilder(G_FCONSTANT)
241 .legalFor({S32, S64})
242 .clampScalar(0, S32, S64);
243 }
Tom Stellardeebbfc22018-06-30 04:09:44 +0000244
Matt Arsenaultb3feccd2018-06-25 15:42:12 +0000245 getActionDefinitionsBuilder(G_IMPLICIT_DEF)
Matt Arsenaultd9141892019-02-07 19:10:15 +0000246 .legalFor({S1, S32, S64, V2S32, V4S32, V2S16, V4S16, GlobalPtr,
247 ConstantPtr, LocalPtr, FlatPtr, PrivatePtr})
Matt Arsenault18ec3822019-02-11 22:00:39 +0000248 .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
Matt Arsenaultd9141892019-02-07 19:10:15 +0000249 .clampScalarOrElt(0, S32, S512)
Matt Arsenault0f2debb2019-02-08 14:46:27 +0000250 .legalIf(isMultiple32(0))
Matt Arsenault82b10392019-02-25 20:46:06 +0000251 .widenScalarToNextPow2(0, 32)
252 .clampMaxNumElements(0, S32, 16);
Matt Arsenaultb3feccd2018-06-25 15:42:12 +0000253
Matt Arsenaultabdc4f22018-03-17 15:17:48 +0000254
Tom Stellarde0424122017-06-03 01:13:33 +0000255 // FIXME: i1 operands to intrinsics should always be legal, but other i1
256 // values may not be legal. We need to figure out how to distinguish
257 // between these two scenarios.
Matt Arsenault45991592019-01-18 21:33:50 +0000258 getActionDefinitionsBuilder(G_CONSTANT)
Matt Arsenault2065c942019-02-02 23:33:49 +0000259 .legalFor({S1, S32, S64, GlobalPtr,
260 LocalPtr, ConstantPtr, PrivatePtr, FlatPtr })
Matt Arsenault45991592019-01-18 21:33:50 +0000261 .clampScalar(0, S32, S64)
Matt Arsenault2065c942019-02-02 23:33:49 +0000262 .widenScalarToNextPow2(0)
263 .legalIf(isPointer(0));
Matt Arsenault06cbb272018-03-01 19:16:52 +0000264
Matt Arsenaultc94e26c2018-12-18 09:46:13 +0000265 setAction({G_FRAME_INDEX, PrivatePtr}, Legal);
266
Matt Arsenault93fdec72019-02-07 18:03:11 +0000267 auto &FPOpActions = getActionDefinitionsBuilder(
Matt Arsenault9dba67f2019-02-11 17:05:20 +0000268 { G_FADD, G_FMUL, G_FNEG, G_FABS, G_FMA, G_FCANONICALIZE})
Matt Arsenault93fdec72019-02-07 18:03:11 +0000269 .legalFor({S32, S64});
270
271 if (ST.has16BitInsts()) {
272 if (ST.hasVOP3PInsts())
273 FPOpActions.legalFor({S16, V2S16});
274 else
275 FPOpActions.legalFor({S16});
276 }
277
Matt Arsenault6ce1b4f2019-07-10 16:31:19 +0000278 auto &MinNumMaxNum = getActionDefinitionsBuilder({
279 G_FMINNUM, G_FMAXNUM, G_FMINNUM_IEEE, G_FMAXNUM_IEEE});
280
281 if (ST.hasVOP3PInsts()) {
282 MinNumMaxNum.customFor(FPTypesPK16)
283 .clampMaxNumElements(0, S16, 2)
284 .clampScalar(0, S16, S64)
285 .scalarize(0);
286 } else if (ST.has16BitInsts()) {
287 MinNumMaxNum.customFor(FPTypes16)
288 .clampScalar(0, S16, S64)
289 .scalarize(0);
290 } else {
291 MinNumMaxNum.customFor(FPTypesBase)
292 .clampScalar(0, S32, S64)
293 .scalarize(0);
294 }
295
296 // TODO: Implement
297 getActionDefinitionsBuilder({G_FMINIMUM, G_FMAXIMUM}).lower();
298
Matt Arsenault93fdec72019-02-07 18:03:11 +0000299 if (ST.hasVOP3PInsts())
300 FPOpActions.clampMaxNumElements(0, S16, 2);
301 FPOpActions
302 .scalarize(0)
303 .clampScalar(0, ST.has16BitInsts() ? S16 : S32, S64);
Tom Stellardd0c6cf22017-10-27 23:57:41 +0000304
Matt Arsenaultc0f75692019-02-07 18:14:39 +0000305 if (ST.has16BitInsts()) {
306 getActionDefinitionsBuilder(G_FSQRT)
307 .legalFor({S32, S64, S16})
308 .scalarize(0)
309 .clampScalar(0, S16, S64);
310 } else {
311 getActionDefinitionsBuilder(G_FSQRT)
312 .legalFor({S32, S64})
313 .scalarize(0)
314 .clampScalar(0, S32, S64);
315 }
316
Matt Arsenaultdff33c32018-12-20 00:37:02 +0000317 getActionDefinitionsBuilder(G_FPTRUNC)
Matt Arsenaulte6cebd02019-01-25 04:37:33 +0000318 .legalFor({{S32, S64}, {S16, S32}})
319 .scalarize(0);
Matt Arsenaultdff33c32018-12-20 00:37:02 +0000320
Matt Arsenault24563ef2019-01-20 18:34:24 +0000321 getActionDefinitionsBuilder(G_FPEXT)
322 .legalFor({{S64, S32}, {S32, S16}})
Matt Arsenaultca676342019-01-25 02:36:32 +0000323 .lowerFor({{S64, S16}}) // FIXME: Implement
324 .scalarize(0);
Matt Arsenault24563ef2019-01-20 18:34:24 +0000325
Matt Arsenaultb1843e12019-07-09 23:34:29 +0000326 // TODO: Verify V_BFI_B32 is generated from expanded bit ops.
327 getActionDefinitionsBuilder(G_FCOPYSIGN).lower();
Matt Arsenault1448f562019-05-17 12:19:52 +0000328
Matt Arsenault745fd9f2019-01-20 19:10:31 +0000329 getActionDefinitionsBuilder(G_FSUB)
Matt Arsenaultaebb2ee2019-01-22 20:14:29 +0000330 // Use actual fsub instruction
331 .legalFor({S32})
332 // Must use fadd + fneg
333 .lowerFor({S64, S16, V2S16})
Matt Arsenault990f5072019-01-25 00:51:00 +0000334 .scalarize(0)
Matt Arsenaultaebb2ee2019-01-22 20:14:29 +0000335 .clampScalar(0, S32, S64);
Matt Arsenaulte01e7c82018-12-18 09:19:03 +0000336
Matt Arsenault24563ef2019-01-20 18:34:24 +0000337 getActionDefinitionsBuilder({G_SEXT, G_ZEXT, G_ANYEXT})
Matt Arsenault46ffe682019-01-20 19:28:20 +0000338 .legalFor({{S64, S32}, {S32, S16}, {S64, S16},
Matt Arsenaultca676342019-01-25 02:36:32 +0000339 {S32, S1}, {S64, S1}, {S16, S1},
340 // FIXME: Hack
Matt Arsenaultf4bfe4c2019-02-25 21:32:48 +0000341 {S64, LLT::scalar(33)},
Matt Arsenault888aa5d2019-02-03 00:07:33 +0000342 {S32, S8}, {S128, S32}, {S128, S64}, {S32, LLT::scalar(24)}})
Matt Arsenaultca676342019-01-25 02:36:32 +0000343 .scalarize(0);
Matt Arsenaultf38f4832018-12-13 08:23:51 +0000344
Matt Arsenaultfb671642019-01-22 00:20:17 +0000345 getActionDefinitionsBuilder({G_SITOFP, G_UITOFP})
Matt Arsenaulte6cebd02019-01-25 04:37:33 +0000346 .legalFor({{S32, S32}, {S64, S32}})
Matt Arsenault02b5ca82019-05-17 23:05:13 +0000347 .lowerFor({{S32, S64}})
Matt Arsenault2f292202019-05-17 23:05:18 +0000348 .customFor({{S64, S64}})
Matt Arsenaulte6cebd02019-01-25 04:37:33 +0000349 .scalarize(0);
Matt Arsenaultdd022ce2018-03-01 19:04:25 +0000350
Matt Arsenaultfb671642019-01-22 00:20:17 +0000351 getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI})
Matt Arsenaulte6cebd02019-01-25 04:37:33 +0000352 .legalFor({{S32, S32}, {S32, S64}})
353 .scalarize(0);
Tom Stellard33445762018-02-07 04:47:59 +0000354
Matt Arsenault6aebcd52019-05-17 12:20:01 +0000355 getActionDefinitionsBuilder(G_INTRINSIC_ROUND)
Matt Arsenault2e5f9002019-01-27 00:12:21 +0000356 .legalFor({S32, S64})
357 .scalarize(0);
Matt Arsenaultf4c21c52018-12-21 03:14:45 +0000358
Matt Arsenault6aafc5e2019-05-17 12:19:57 +0000359 if (ST.getGeneration() >= AMDGPUSubtarget::SEA_ISLANDS) {
Matt Arsenaulta510b572019-05-17 12:20:05 +0000360 getActionDefinitionsBuilder({G_INTRINSIC_TRUNC, G_FCEIL, G_FRINT})
Matt Arsenault6aafc5e2019-05-17 12:19:57 +0000361 .legalFor({S32, S64})
362 .clampScalar(0, S32, S64)
363 .scalarize(0);
364 } else {
Matt Arsenaulta510b572019-05-17 12:20:05 +0000365 getActionDefinitionsBuilder({G_INTRINSIC_TRUNC, G_FCEIL, G_FRINT})
Matt Arsenault6aafc5e2019-05-17 12:19:57 +0000366 .legalFor({S32})
367 .customFor({S64})
368 .clampScalar(0, S32, S64)
369 .scalarize(0);
370 }
Tom Stellardca166212017-01-30 21:56:46 +0000371
Matt Arsenault9e5e8682019-02-14 22:24:28 +0000372 getActionDefinitionsBuilder(G_GEP)
373 .legalForCartesianProduct(AddrSpaces64, {S64})
374 .legalForCartesianProduct(AddrSpaces32, {S32})
375 .scalarize(0);
Matt Arsenault3b9a82f2019-01-25 04:54:00 +0000376
Matt Arsenault934e5342018-12-13 20:34:15 +0000377 setAction({G_BLOCK_ADDR, CodePtr}, Legal);
378
Matt Arsenault8b8eee52019-07-09 14:10:43 +0000379 auto &CmpBuilder =
380 getActionDefinitionsBuilder(G_ICMP)
Matt Arsenault58f9d3d2019-02-02 23:35:15 +0000381 .legalForCartesianProduct(
382 {S1}, {S32, S64, GlobalPtr, LocalPtr, ConstantPtr, PrivatePtr, FlatPtr})
Matt Arsenault8b8eee52019-07-09 14:10:43 +0000383 .legalFor({{S1, S32}, {S1, S64}});
384 if (ST.has16BitInsts()) {
385 CmpBuilder.legalFor({{S1, S16}});
386 }
387
388 CmpBuilder
Matt Arsenault58f9d3d2019-02-02 23:35:15 +0000389 .widenScalarToNextPow2(1)
390 .clampScalar(1, S32, S64)
391 .scalarize(0)
392 .legalIf(all(typeIs(0, S1), isPointer(1)));
393
394 getActionDefinitionsBuilder(G_FCMP)
Matt Arsenault40d1faf2019-07-01 17:35:53 +0000395 .legalForCartesianProduct({S1}, ST.has16BitInsts() ? FPTypes16 : FPTypesBase)
Matt Arsenault1b1e6852019-01-25 02:59:34 +0000396 .widenScalarToNextPow2(1)
397 .clampScalar(1, S32, S64)
Matt Arsenaultded2f822019-01-26 23:54:53 +0000398 .scalarize(0);
Matt Arsenault1b1e6852019-01-25 02:59:34 +0000399
Matt Arsenault95fd95c2019-01-25 04:03:38 +0000400 // FIXME: fexp, flog2, flog10 needs to be custom lowered.
401 getActionDefinitionsBuilder({G_FPOW, G_FEXP, G_FEXP2,
402 G_FLOG, G_FLOG2, G_FLOG10})
403 .legalFor({S32})
404 .scalarize(0);
Tom Stellard8cd60a52017-06-06 14:16:50 +0000405
Matt Arsenaultd5684f72019-01-31 02:09:57 +0000406 // The 64-bit versions produce 32-bit results, but only on the SALU.
407 getActionDefinitionsBuilder({G_CTLZ, G_CTLZ_ZERO_UNDEF,
408 G_CTTZ, G_CTTZ_ZERO_UNDEF,
409 G_CTPOP})
410 .legalFor({{S32, S32}, {S32, S64}})
411 .clampScalar(0, S32, S32)
Matt Arsenault75e30c42019-02-20 16:42:52 +0000412 .clampScalar(1, S32, S64)
Matt Arsenaultb10fa8d2019-02-21 15:22:20 +0000413 .scalarize(0)
414 .widenScalarToNextPow2(0, 32)
415 .widenScalarToNextPow2(1, 32);
Matt Arsenaultd5684f72019-01-31 02:09:57 +0000416
Matt Arsenaultd1bfc8d2019-01-31 02:34:03 +0000417 // TODO: Expand for > s32
418 getActionDefinitionsBuilder(G_BSWAP)
419 .legalFor({S32})
420 .clampScalar(0, S32, S32)
421 .scalarize(0);
Matt Arsenaultd5684f72019-01-31 02:09:57 +0000422
Matt Arsenault0f3ba442019-05-23 17:58:48 +0000423 if (ST.has16BitInsts()) {
424 if (ST.hasVOP3PInsts()) {
425 getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
426 .legalFor({S32, S16, V2S16})
427 .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
428 .clampMaxNumElements(0, S16, 2)
429 .clampScalar(0, S16, S32)
430 .widenScalarToNextPow2(0)
431 .scalarize(0);
432 } else {
433 getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
434 .legalFor({S32, S16})
435 .widenScalarToNextPow2(0)
436 .clampScalar(0, S16, S32)
437 .scalarize(0);
438 }
439 } else {
440 getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
441 .legalFor({S32})
442 .clampScalar(0, S32, S32)
443 .widenScalarToNextPow2(0)
444 .scalarize(0);
445 }
Matt Arsenaultf38f4832018-12-13 08:23:51 +0000446
Matt Arsenaultcbaada62019-02-02 23:29:55 +0000447 auto smallerThan = [](unsigned TypeIdx0, unsigned TypeIdx1) {
448 return [=](const LegalityQuery &Query) {
449 return Query.Types[TypeIdx0].getSizeInBits() <
450 Query.Types[TypeIdx1].getSizeInBits();
451 };
452 };
453
454 auto greaterThan = [](unsigned TypeIdx0, unsigned TypeIdx1) {
455 return [=](const LegalityQuery &Query) {
456 return Query.Types[TypeIdx0].getSizeInBits() >
457 Query.Types[TypeIdx1].getSizeInBits();
458 };
459 };
460
Tom Stellard7c650782018-10-05 04:34:09 +0000461 getActionDefinitionsBuilder(G_INTTOPTR)
Matt Arsenaultcbaada62019-02-02 23:29:55 +0000462 // List the common cases
Matt Arsenault9e5e8682019-02-14 22:24:28 +0000463 .legalForCartesianProduct(AddrSpaces64, {S64})
464 .legalForCartesianProduct(AddrSpaces32, {S32})
Matt Arsenaultcbaada62019-02-02 23:29:55 +0000465 .scalarize(0)
466 // Accept any address space as long as the size matches
467 .legalIf(sameSize(0, 1))
468 .widenScalarIf(smallerThan(1, 0),
469 [](const LegalityQuery &Query) {
470 return std::make_pair(1, LLT::scalar(Query.Types[0].getSizeInBits()));
471 })
472 .narrowScalarIf(greaterThan(1, 0),
473 [](const LegalityQuery &Query) {
474 return std::make_pair(1, LLT::scalar(Query.Types[0].getSizeInBits()));
475 });
Matt Arsenault85803362018-03-17 15:17:41 +0000476
Matt Arsenaultf38f4832018-12-13 08:23:51 +0000477 getActionDefinitionsBuilder(G_PTRTOINT)
Matt Arsenaultcbaada62019-02-02 23:29:55 +0000478 // List the common cases
Matt Arsenault9e5e8682019-02-14 22:24:28 +0000479 .legalForCartesianProduct(AddrSpaces64, {S64})
480 .legalForCartesianProduct(AddrSpaces32, {S32})
Matt Arsenaultcbaada62019-02-02 23:29:55 +0000481 .scalarize(0)
482 // Accept any address space as long as the size matches
483 .legalIf(sameSize(0, 1))
484 .widenScalarIf(smallerThan(0, 1),
485 [](const LegalityQuery &Query) {
486 return std::make_pair(0, LLT::scalar(Query.Types[1].getSizeInBits()));
487 })
488 .narrowScalarIf(
489 greaterThan(0, 1),
490 [](const LegalityQuery &Query) {
491 return std::make_pair(0, LLT::scalar(Query.Types[1].getSizeInBits()));
492 });
Matt Arsenaultf38f4832018-12-13 08:23:51 +0000493
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000494 if (ST.hasFlatAddressSpace()) {
495 getActionDefinitionsBuilder(G_ADDRSPACE_CAST)
496 .scalarize(0)
497 .custom();
498 }
499
Matt Arsenault85803362018-03-17 15:17:41 +0000500 getActionDefinitionsBuilder({G_LOAD, G_STORE})
Matt Arsenault18619af2019-01-29 18:13:02 +0000501 .narrowScalarIf([](const LegalityQuery &Query) {
502 unsigned Size = Query.Types[0].getSizeInBits();
503 unsigned MemSize = Query.MMODescrs[0].SizeInBits;
504 return (Size > 32 && MemSize < Size);
505 },
506 [](const LegalityQuery &Query) {
507 return std::make_pair(0, LLT::scalar(32));
508 })
Matt Arsenault9e8e8c62019-07-01 18:49:01 +0000509 .fewerElementsIf([=](const LegalityQuery &Query) {
Matt Arsenault045bc9a2019-01-30 02:35:38 +0000510 unsigned MemSize = Query.MMODescrs[0].SizeInBits;
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000511 return (MemSize == 96) &&
512 Query.Types[0].isVector() &&
Matt Arsenaulte4c2e9b2019-06-19 23:54:58 +0000513 !ST.hasDwordx3LoadStores();
Matt Arsenault045bc9a2019-01-30 02:35:38 +0000514 },
515 [=](const LegalityQuery &Query) {
516 return std::make_pair(0, V2S32);
517 })
Matt Arsenault9e8e8c62019-07-01 18:49:01 +0000518 .legalIf([=](const LegalityQuery &Query) {
Matt Arsenault85803362018-03-17 15:17:41 +0000519 const LLT &Ty0 = Query.Types[0];
520
Matt Arsenault18619af2019-01-29 18:13:02 +0000521 unsigned Size = Ty0.getSizeInBits();
522 unsigned MemSize = Query.MMODescrs[0].SizeInBits;
Matt Arsenaulteb2603c2019-02-02 23:39:13 +0000523 if (Size < 32 || (Size > 32 && MemSize < Size))
Matt Arsenault18619af2019-01-29 18:13:02 +0000524 return false;
525
526 if (Ty0.isVector() && Size != MemSize)
527 return false;
528
Matt Arsenault85803362018-03-17 15:17:41 +0000529 // TODO: Decompose private loads into 4-byte components.
530 // TODO: Illegal flat loads on SI
Matt Arsenault18619af2019-01-29 18:13:02 +0000531 switch (MemSize) {
532 case 8:
533 case 16:
534 return Size == 32;
Matt Arsenault85803362018-03-17 15:17:41 +0000535 case 32:
536 case 64:
537 case 128:
538 return true;
539
540 case 96:
Matt Arsenaulte4c2e9b2019-06-19 23:54:58 +0000541 return ST.hasDwordx3LoadStores();
Matt Arsenault85803362018-03-17 15:17:41 +0000542
543 case 256:
544 case 512:
Tom Stellardd0ba79f2019-07-10 00:22:41 +0000545 // TODO: Possibly support loads of i256 and i512 . This will require
546 // adding i256 and i512 types to MVT in order for to be able to use
547 // TableGen.
548 // TODO: Add support for other vector types, this will require
549 // defining more value mappings for the new types.
550 return Ty0.isVector() && (Ty0.getScalarType().getSizeInBits() == 32 ||
551 Ty0.getScalarType().getSizeInBits() == 64);
552
Matt Arsenault85803362018-03-17 15:17:41 +0000553 default:
554 return false;
555 }
Matt Arsenault18619af2019-01-29 18:13:02 +0000556 })
557 .clampScalar(0, S32, S64);
Matt Arsenault85803362018-03-17 15:17:41 +0000558
559
Matt Arsenault530d05e2019-02-14 22:41:09 +0000560 // FIXME: Handle alignment requirements.
Matt Arsenault6614f852019-01-22 19:02:10 +0000561 auto &ExtLoads = getActionDefinitionsBuilder({G_SEXTLOAD, G_ZEXTLOAD})
Matt Arsenault530d05e2019-02-14 22:41:09 +0000562 .legalForTypesWithMemDesc({
563 {S32, GlobalPtr, 8, 8},
564 {S32, GlobalPtr, 16, 8},
565 {S32, LocalPtr, 8, 8},
566 {S32, LocalPtr, 16, 8},
567 {S32, PrivatePtr, 8, 8},
568 {S32, PrivatePtr, 16, 8}});
Matt Arsenault6614f852019-01-22 19:02:10 +0000569 if (ST.hasFlatAddressSpace()) {
Matt Arsenault530d05e2019-02-14 22:41:09 +0000570 ExtLoads.legalForTypesWithMemDesc({{S32, FlatPtr, 8, 8},
571 {S32, FlatPtr, 16, 8}});
Matt Arsenault6614f852019-01-22 19:02:10 +0000572 }
573
574 ExtLoads.clampScalar(0, S32, S32)
575 .widenScalarToNextPow2(0)
576 .unsupportedIfMemSizeNotPow2()
577 .lower();
578
Matt Arsenault36d40922018-12-20 00:33:49 +0000579 auto &Atomics = getActionDefinitionsBuilder(
580 {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB,
581 G_ATOMICRMW_AND, G_ATOMICRMW_OR, G_ATOMICRMW_XOR,
582 G_ATOMICRMW_MAX, G_ATOMICRMW_MIN, G_ATOMICRMW_UMAX,
583 G_ATOMICRMW_UMIN, G_ATOMIC_CMPXCHG})
584 .legalFor({{S32, GlobalPtr}, {S32, LocalPtr},
585 {S64, GlobalPtr}, {S64, LocalPtr}});
586 if (ST.hasFlatAddressSpace()) {
587 Atomics.legalFor({{S32, FlatPtr}, {S64, FlatPtr}});
588 }
Tom Stellardca166212017-01-30 21:56:46 +0000589
Matt Arsenault96e47012019-01-18 21:42:55 +0000590 // TODO: Pointer types, any 32-bit or 64-bit vector
591 getActionDefinitionsBuilder(G_SELECT)
Matt Arsenaultfdf36722019-07-01 15:42:47 +0000592 .legalForCartesianProduct({S32, S64, S16, V2S32, V2S16, V4S16,
Matt Arsenault10547232019-02-04 14:04:52 +0000593 GlobalPtr, LocalPtr, FlatPtr, PrivatePtr,
594 LLT::vector(2, LocalPtr), LLT::vector(2, PrivatePtr)}, {S1})
Matt Arsenaultfdf36722019-07-01 15:42:47 +0000595 .clampScalar(0, S16, S64)
Matt Arsenaultb4c95b32019-02-19 17:03:09 +0000596 .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
597 .fewerElementsIf(numElementsNotEven(0), scalarize(0))
Matt Arsenaultdc6c7852019-01-30 04:19:31 +0000598 .scalarize(1)
Matt Arsenault2491f822019-02-02 23:31:50 +0000599 .clampMaxNumElements(0, S32, 2)
600 .clampMaxNumElements(0, LocalPtr, 2)
601 .clampMaxNumElements(0, PrivatePtr, 2)
Matt Arsenaultb4c95b32019-02-19 17:03:09 +0000602 .scalarize(0)
Matt Arsenault4ed6cca2019-04-05 14:03:04 +0000603 .widenScalarToNextPow2(0)
Matt Arsenault2491f822019-02-02 23:31:50 +0000604 .legalIf(all(isPointer(0), typeIs(1, S1)));
Tom Stellard2860a422017-06-07 13:54:51 +0000605
Matt Arsenault4c5e8f512019-01-22 22:00:19 +0000606 // TODO: Only the low 4/5/6 bits of the shift amount are observed, so we can
607 // be more flexible with the shift amount type.
608 auto &Shifts = getActionDefinitionsBuilder({G_SHL, G_LSHR, G_ASHR})
609 .legalFor({{S32, S32}, {S64, S32}});
Matt Arsenaultf6cab162019-01-30 03:36:25 +0000610 if (ST.has16BitInsts()) {
Matt Arsenaultc83b8232019-02-07 17:38:00 +0000611 if (ST.hasVOP3PInsts()) {
612 Shifts.legalFor({{S16, S32}, {S16, S16}, {V2S16, V2S16}})
613 .clampMaxNumElements(0, S16, 2);
614 } else
615 Shifts.legalFor({{S16, S32}, {S16, S16}});
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +0000616
617 Shifts.clampScalar(1, S16, S32);
Matt Arsenaultf6cab162019-01-30 03:36:25 +0000618 Shifts.clampScalar(0, S16, S64);
Matt Arsenaultb0a22702019-02-08 15:06:24 +0000619 Shifts.widenScalarToNextPow2(0, 16);
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +0000620 } else {
621 // Make sure we legalize the shift amount type first, as the general
622 // expansion for the shifted type will produce much worse code if it hasn't
623 // been truncated already.
624 Shifts.clampScalar(1, S32, S32);
Matt Arsenault4c5e8f512019-01-22 22:00:19 +0000625 Shifts.clampScalar(0, S32, S64);
Matt Arsenaultb0a22702019-02-08 15:06:24 +0000626 Shifts.widenScalarToNextPow2(0, 32);
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +0000627 }
628 Shifts.scalarize(0);
Tom Stellardca166212017-01-30 21:56:46 +0000629
Matt Arsenault7b9ed892018-03-12 13:35:53 +0000630 for (unsigned Op : {G_EXTRACT_VECTOR_ELT, G_INSERT_VECTOR_ELT}) {
Matt Arsenault63786292019-01-22 20:38:15 +0000631 unsigned VecTypeIdx = Op == G_EXTRACT_VECTOR_ELT ? 1 : 0;
632 unsigned EltTypeIdx = Op == G_EXTRACT_VECTOR_ELT ? 0 : 1;
633 unsigned IdxTypeIdx = 2;
634
Matt Arsenault7b9ed892018-03-12 13:35:53 +0000635 getActionDefinitionsBuilder(Op)
636 .legalIf([=](const LegalityQuery &Query) {
Matt Arsenault63786292019-01-22 20:38:15 +0000637 const LLT &VecTy = Query.Types[VecTypeIdx];
638 const LLT &IdxTy = Query.Types[IdxTypeIdx];
Matt Arsenault7b9ed892018-03-12 13:35:53 +0000639 return VecTy.getSizeInBits() % 32 == 0 &&
640 VecTy.getSizeInBits() <= 512 &&
641 IdxTy.getSizeInBits() == 32;
Matt Arsenault63786292019-01-22 20:38:15 +0000642 })
643 .clampScalar(EltTypeIdx, S32, S64)
644 .clampScalar(VecTypeIdx, S32, S64)
645 .clampScalar(IdxTypeIdx, S32, S32);
Matt Arsenault7b9ed892018-03-12 13:35:53 +0000646 }
647
Matt Arsenault63786292019-01-22 20:38:15 +0000648 getActionDefinitionsBuilder(G_EXTRACT_VECTOR_ELT)
649 .unsupportedIf([=](const LegalityQuery &Query) {
650 const LLT &EltTy = Query.Types[1].getElementType();
651 return Query.Types[0] != EltTy;
652 });
653
Matt Arsenaultc4d07552019-02-20 16:11:22 +0000654 for (unsigned Op : {G_EXTRACT, G_INSERT}) {
655 unsigned BigTyIdx = Op == G_EXTRACT ? 1 : 0;
656 unsigned LitTyIdx = Op == G_EXTRACT ? 0 : 1;
657
658 // FIXME: Doesn't handle extract of illegal sizes.
659 getActionDefinitionsBuilder(Op)
Matt Arsenault91be65b2019-02-07 17:25:51 +0000660 .legalIf([=](const LegalityQuery &Query) {
Matt Arsenaultc4d07552019-02-20 16:11:22 +0000661 const LLT BigTy = Query.Types[BigTyIdx];
662 const LLT LitTy = Query.Types[LitTyIdx];
663 return (BigTy.getSizeInBits() % 32 == 0) &&
664 (LitTy.getSizeInBits() % 16 == 0);
665 })
Matt Arsenault91be65b2019-02-07 17:25:51 +0000666 .widenScalarIf(
Matt Arsenaultc4d07552019-02-20 16:11:22 +0000667 [=](const LegalityQuery &Query) {
668 const LLT BigTy = Query.Types[BigTyIdx];
669 return (BigTy.getScalarSizeInBits() < 16);
670 },
671 LegalizeMutations::widenScalarOrEltToNextPow2(BigTyIdx, 16))
672 .widenScalarIf(
673 [=](const LegalityQuery &Query) {
674 const LLT LitTy = Query.Types[LitTyIdx];
675 return (LitTy.getScalarSizeInBits() < 16);
676 },
677 LegalizeMutations::widenScalarOrEltToNextPow2(LitTyIdx, 16))
Matt Arsenault2b6f76f2019-04-22 15:22:46 +0000678 .moreElementsIf(isSmallOddVector(BigTyIdx), oneMoreElement(BigTyIdx))
679 .widenScalarToNextPow2(BigTyIdx, 32);
680
Matt Arsenaultc4d07552019-02-20 16:11:22 +0000681 }
Matt Arsenault71272e62018-03-05 16:25:15 +0000682
Amara Emerson5ec14602018-12-10 18:44:58 +0000683 getActionDefinitionsBuilder(G_BUILD_VECTOR)
Matt Arsenaultaebb2ee2019-01-22 20:14:29 +0000684 .legalForCartesianProduct(AllS32Vectors, {S32})
685 .legalForCartesianProduct(AllS64Vectors, {S64})
686 .clampNumElements(0, V16S32, V16S32)
687 .clampNumElements(0, V2S64, V8S64)
688 .minScalarSameAs(1, 0)
Matt Arsenault3f1a3452019-07-09 22:48:04 +0000689 .legalIf(isRegisterType(0))
690 .minScalarOrElt(0, S32);
Matt Arsenaultbee2ad72018-12-21 03:03:11 +0000691
Matt Arsenaulta1515d22019-01-08 01:30:02 +0000692 getActionDefinitionsBuilder(G_CONCAT_VECTORS)
Matt Arsenault4dd57552019-07-09 14:17:31 +0000693 .legalIf(isRegisterType(0));
Matt Arsenaulta1515d22019-01-08 01:30:02 +0000694
Matt Arsenault503afda2018-03-12 13:35:43 +0000695 // Merge/Unmerge
696 for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
697 unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
698 unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
699
Matt Arsenaultff6a9a22019-01-20 18:40:36 +0000700 auto notValidElt = [=](const LegalityQuery &Query, unsigned TypeIdx) {
701 const LLT &Ty = Query.Types[TypeIdx];
702 if (Ty.isVector()) {
703 const LLT &EltTy = Ty.getElementType();
704 if (EltTy.getSizeInBits() < 8 || EltTy.getSizeInBits() > 64)
705 return true;
706 if (!isPowerOf2_32(EltTy.getSizeInBits()))
707 return true;
708 }
709 return false;
710 };
711
Matt Arsenault503afda2018-03-12 13:35:43 +0000712 getActionDefinitionsBuilder(Op)
Matt Arsenaultd8d193d2019-01-29 23:17:35 +0000713 .widenScalarToNextPow2(LitTyIdx, /*Min*/ 16)
714 // Clamp the little scalar to s8-s256 and make it a power of 2. It's not
715 // worth considering the multiples of 64 since 2*192 and 2*384 are not
716 // valid.
717 .clampScalar(LitTyIdx, S16, S256)
718 .widenScalarToNextPow2(LitTyIdx, /*Min*/ 32)
719
Matt Arsenaultff6a9a22019-01-20 18:40:36 +0000720 // Break up vectors with weird elements into scalars
721 .fewerElementsIf(
722 [=](const LegalityQuery &Query) { return notValidElt(Query, 0); },
Matt Arsenault990f5072019-01-25 00:51:00 +0000723 scalarize(0))
Matt Arsenaultff6a9a22019-01-20 18:40:36 +0000724 .fewerElementsIf(
725 [=](const LegalityQuery &Query) { return notValidElt(Query, 1); },
Matt Arsenault990f5072019-01-25 00:51:00 +0000726 scalarize(1))
Matt Arsenaultff6a9a22019-01-20 18:40:36 +0000727 .clampScalar(BigTyIdx, S32, S512)
728 .widenScalarIf(
729 [=](const LegalityQuery &Query) {
730 const LLT &Ty = Query.Types[BigTyIdx];
731 return !isPowerOf2_32(Ty.getSizeInBits()) &&
732 Ty.getSizeInBits() % 16 != 0;
733 },
734 [=](const LegalityQuery &Query) {
735 // Pick the next power of 2, or a multiple of 64 over 128.
736 // Whichever is smaller.
737 const LLT &Ty = Query.Types[BigTyIdx];
738 unsigned NewSizeInBits = 1 << Log2_32_Ceil(Ty.getSizeInBits() + 1);
739 if (NewSizeInBits >= 256) {
740 unsigned RoundedTo = alignTo<64>(Ty.getSizeInBits() + 1);
741 if (RoundedTo < NewSizeInBits)
742 NewSizeInBits = RoundedTo;
743 }
744 return std::make_pair(BigTyIdx, LLT::scalar(NewSizeInBits));
745 })
Matt Arsenault503afda2018-03-12 13:35:43 +0000746 .legalIf([=](const LegalityQuery &Query) {
747 const LLT &BigTy = Query.Types[BigTyIdx];
748 const LLT &LitTy = Query.Types[LitTyIdx];
Matt Arsenaultff6a9a22019-01-20 18:40:36 +0000749
750 if (BigTy.isVector() && BigTy.getSizeInBits() < 32)
751 return false;
752 if (LitTy.isVector() && LitTy.getSizeInBits() < 32)
753 return false;
754
755 return BigTy.getSizeInBits() % 16 == 0 &&
756 LitTy.getSizeInBits() % 16 == 0 &&
Matt Arsenault503afda2018-03-12 13:35:43 +0000757 BigTy.getSizeInBits() <= 512;
758 })
759 // Any vectors left are the wrong size. Scalarize them.
Matt Arsenault990f5072019-01-25 00:51:00 +0000760 .scalarize(0)
761 .scalarize(1);
Matt Arsenault503afda2018-03-12 13:35:43 +0000762 }
763
Tom Stellardca166212017-01-30 21:56:46 +0000764 computeTables();
Roman Tereshin76c29c62018-05-31 16:16:48 +0000765 verify(*ST.getInstrInfo());
Tom Stellardca166212017-01-30 21:56:46 +0000766}
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000767
768bool AMDGPULegalizerInfo::legalizeCustom(MachineInstr &MI,
769 MachineRegisterInfo &MRI,
770 MachineIRBuilder &MIRBuilder,
771 GISelChangeObserver &Observer) const {
772 switch (MI.getOpcode()) {
773 case TargetOpcode::G_ADDRSPACE_CAST:
774 return legalizeAddrSpaceCast(MI, MRI, MIRBuilder);
Matt Arsenault6aafc5e2019-05-17 12:19:57 +0000775 case TargetOpcode::G_FRINT:
776 return legalizeFrint(MI, MRI, MIRBuilder);
Matt Arsenaulta510b572019-05-17 12:20:05 +0000777 case TargetOpcode::G_FCEIL:
778 return legalizeFceil(MI, MRI, MIRBuilder);
Matt Arsenault6aebcd52019-05-17 12:20:01 +0000779 case TargetOpcode::G_INTRINSIC_TRUNC:
780 return legalizeIntrinsicTrunc(MI, MRI, MIRBuilder);
Matt Arsenault2f292202019-05-17 23:05:18 +0000781 case TargetOpcode::G_SITOFP:
782 return legalizeITOFP(MI, MRI, MIRBuilder, true);
783 case TargetOpcode::G_UITOFP:
784 return legalizeITOFP(MI, MRI, MIRBuilder, false);
Matt Arsenault6ce1b4f2019-07-10 16:31:19 +0000785 case TargetOpcode::G_FMINNUM:
786 case TargetOpcode::G_FMAXNUM:
787 case TargetOpcode::G_FMINNUM_IEEE:
788 case TargetOpcode::G_FMAXNUM_IEEE:
789 return legalizeMinNumMaxNum(MI, MRI, MIRBuilder);
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000790 default:
791 return false;
792 }
793
794 llvm_unreachable("expected switch to return");
795}
796
Matt Arsenault1178dc32019-06-28 01:16:46 +0000797Register AMDGPULegalizerInfo::getSegmentAperture(
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000798 unsigned AS,
799 MachineRegisterInfo &MRI,
800 MachineIRBuilder &MIRBuilder) const {
801 MachineFunction &MF = MIRBuilder.getMF();
802 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
803 const LLT S32 = LLT::scalar(32);
804
805 if (ST.hasApertureRegs()) {
806 // FIXME: Use inline constants (src_{shared, private}_base) instead of
807 // getreg.
808 unsigned Offset = AS == AMDGPUAS::LOCAL_ADDRESS ?
809 AMDGPU::Hwreg::OFFSET_SRC_SHARED_BASE :
810 AMDGPU::Hwreg::OFFSET_SRC_PRIVATE_BASE;
811 unsigned WidthM1 = AS == AMDGPUAS::LOCAL_ADDRESS ?
812 AMDGPU::Hwreg::WIDTH_M1_SRC_SHARED_BASE :
813 AMDGPU::Hwreg::WIDTH_M1_SRC_PRIVATE_BASE;
814 unsigned Encoding =
815 AMDGPU::Hwreg::ID_MEM_BASES << AMDGPU::Hwreg::ID_SHIFT_ |
816 Offset << AMDGPU::Hwreg::OFFSET_SHIFT_ |
817 WidthM1 << AMDGPU::Hwreg::WIDTH_M1_SHIFT_;
818
Matt Arsenault1178dc32019-06-28 01:16:46 +0000819 Register ApertureReg = MRI.createGenericVirtualRegister(S32);
820 Register GetReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000821
822 MIRBuilder.buildInstr(AMDGPU::S_GETREG_B32)
823 .addDef(GetReg)
824 .addImm(Encoding);
825 MRI.setType(GetReg, S32);
826
Amara Emerson946b1242019-04-15 05:04:20 +0000827 auto ShiftAmt = MIRBuilder.buildConstant(S32, WidthM1 + 1);
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000828 MIRBuilder.buildInstr(TargetOpcode::G_SHL)
829 .addDef(ApertureReg)
830 .addUse(GetReg)
Amara Emerson946b1242019-04-15 05:04:20 +0000831 .addUse(ShiftAmt.getReg(0));
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000832
833 return ApertureReg;
834 }
835
Matt Arsenault1178dc32019-06-28 01:16:46 +0000836 Register QueuePtr = MRI.createGenericVirtualRegister(
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000837 LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
838
839 // FIXME: Placeholder until we can track the input registers.
840 MIRBuilder.buildConstant(QueuePtr, 0xdeadbeef);
841
842 // Offset into amd_queue_t for group_segment_aperture_base_hi /
843 // private_segment_aperture_base_hi.
844 uint32_t StructOffset = (AS == AMDGPUAS::LOCAL_ADDRESS) ? 0x40 : 0x44;
845
846 // FIXME: Don't use undef
847 Value *V = UndefValue::get(PointerType::get(
848 Type::getInt8Ty(MF.getFunction().getContext()),
849 AMDGPUAS::CONSTANT_ADDRESS));
850
851 MachinePointerInfo PtrInfo(V, StructOffset);
852 MachineMemOperand *MMO = MF.getMachineMemOperand(
853 PtrInfo,
854 MachineMemOperand::MOLoad |
855 MachineMemOperand::MODereferenceable |
856 MachineMemOperand::MOInvariant,
857 4,
858 MinAlign(64, StructOffset));
859
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000860 Register LoadResult = MRI.createGenericVirtualRegister(S32);
861 Register LoadAddr;
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000862
863 MIRBuilder.materializeGEP(LoadAddr, QueuePtr, LLT::scalar(64), StructOffset);
864 MIRBuilder.buildLoad(LoadResult, LoadAddr, *MMO);
865 return LoadResult;
866}
867
868bool AMDGPULegalizerInfo::legalizeAddrSpaceCast(
869 MachineInstr &MI, MachineRegisterInfo &MRI,
870 MachineIRBuilder &MIRBuilder) const {
871 MachineFunction &MF = MIRBuilder.getMF();
872
873 MIRBuilder.setInstr(MI);
874
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000875 Register Dst = MI.getOperand(0).getReg();
876 Register Src = MI.getOperand(1).getReg();
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000877
878 LLT DstTy = MRI.getType(Dst);
879 LLT SrcTy = MRI.getType(Src);
880 unsigned DestAS = DstTy.getAddressSpace();
881 unsigned SrcAS = SrcTy.getAddressSpace();
882
883 // TODO: Avoid reloading from the queue ptr for each cast, or at least each
884 // vector element.
885 assert(!DstTy.isVector());
886
887 const AMDGPUTargetMachine &TM
888 = static_cast<const AMDGPUTargetMachine &>(MF.getTarget());
889
890 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
891 if (ST.getTargetLowering()->isNoopAddrSpaceCast(SrcAS, DestAS)) {
Matt Arsenaultdc88a2c2019-02-08 14:16:11 +0000892 MI.setDesc(MIRBuilder.getTII().get(TargetOpcode::G_BITCAST));
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000893 return true;
894 }
895
896 if (SrcAS == AMDGPUAS::FLAT_ADDRESS) {
897 assert(DestAS == AMDGPUAS::LOCAL_ADDRESS ||
898 DestAS == AMDGPUAS::PRIVATE_ADDRESS);
899 unsigned NullVal = TM.getNullPointerValue(DestAS);
900
Amara Emerson946b1242019-04-15 05:04:20 +0000901 auto SegmentNull = MIRBuilder.buildConstant(DstTy, NullVal);
902 auto FlatNull = MIRBuilder.buildConstant(SrcTy, 0);
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000903
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000904 Register PtrLo32 = MRI.createGenericVirtualRegister(DstTy);
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000905
906 // Extract low 32-bits of the pointer.
907 MIRBuilder.buildExtract(PtrLo32, Src, 0);
908
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000909 Register CmpRes = MRI.createGenericVirtualRegister(LLT::scalar(1));
Amara Emerson946b1242019-04-15 05:04:20 +0000910 MIRBuilder.buildICmp(CmpInst::ICMP_NE, CmpRes, Src, FlatNull.getReg(0));
911 MIRBuilder.buildSelect(Dst, CmpRes, PtrLo32, SegmentNull.getReg(0));
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000912
913 MI.eraseFromParent();
914 return true;
915 }
916
917 assert(SrcAS == AMDGPUAS::LOCAL_ADDRESS ||
918 SrcAS == AMDGPUAS::PRIVATE_ADDRESS);
919
Amara Emerson946b1242019-04-15 05:04:20 +0000920 auto SegmentNull =
921 MIRBuilder.buildConstant(SrcTy, TM.getNullPointerValue(SrcAS));
922 auto FlatNull =
923 MIRBuilder.buildConstant(DstTy, TM.getNullPointerValue(DestAS));
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000924
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000925 Register ApertureReg = getSegmentAperture(DestAS, MRI, MIRBuilder);
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000926
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000927 Register CmpRes = MRI.createGenericVirtualRegister(LLT::scalar(1));
Amara Emerson946b1242019-04-15 05:04:20 +0000928 MIRBuilder.buildICmp(CmpInst::ICMP_NE, CmpRes, Src, SegmentNull.getReg(0));
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000929
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000930 Register BuildPtr = MRI.createGenericVirtualRegister(DstTy);
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000931
932 // Coerce the type of the low half of the result so we can use merge_values.
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000933 Register SrcAsInt = MRI.createGenericVirtualRegister(LLT::scalar(32));
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000934 MIRBuilder.buildInstr(TargetOpcode::G_PTRTOINT)
935 .addDef(SrcAsInt)
936 .addUse(Src);
937
938 // TODO: Should we allow mismatched types but matching sizes in merges to
939 // avoid the ptrtoint?
940 MIRBuilder.buildMerge(BuildPtr, {SrcAsInt, ApertureReg});
Amara Emerson946b1242019-04-15 05:04:20 +0000941 MIRBuilder.buildSelect(Dst, CmpRes, BuildPtr, FlatNull.getReg(0));
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000942
943 MI.eraseFromParent();
944 return true;
945}
Matt Arsenault6aafc5e2019-05-17 12:19:57 +0000946
947bool AMDGPULegalizerInfo::legalizeFrint(
948 MachineInstr &MI, MachineRegisterInfo &MRI,
949 MachineIRBuilder &MIRBuilder) const {
950 MIRBuilder.setInstr(MI);
951
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000952 Register Src = MI.getOperand(1).getReg();
Matt Arsenault6aafc5e2019-05-17 12:19:57 +0000953 LLT Ty = MRI.getType(Src);
954 assert(Ty.isScalar() && Ty.getSizeInBits() == 64);
955
956 APFloat C1Val(APFloat::IEEEdouble(), "0x1.0p+52");
957 APFloat C2Val(APFloat::IEEEdouble(), "0x1.fffffffffffffp+51");
958
959 auto C1 = MIRBuilder.buildFConstant(Ty, C1Val);
960 auto CopySign = MIRBuilder.buildFCopysign(Ty, C1, Src);
961
962 // TODO: Should this propagate fast-math-flags?
963 auto Tmp1 = MIRBuilder.buildFAdd(Ty, Src, CopySign);
964 auto Tmp2 = MIRBuilder.buildFSub(Ty, Tmp1, CopySign);
965
966 auto C2 = MIRBuilder.buildFConstant(Ty, C2Val);
967 auto Fabs = MIRBuilder.buildFAbs(Ty, Src);
968
969 auto Cond = MIRBuilder.buildFCmp(CmpInst::FCMP_OGT, LLT::scalar(1), Fabs, C2);
970 MIRBuilder.buildSelect(MI.getOperand(0).getReg(), Cond, Src, Tmp2);
971 return true;
972}
Matt Arsenault6aebcd52019-05-17 12:20:01 +0000973
Matt Arsenaulta510b572019-05-17 12:20:05 +0000974bool AMDGPULegalizerInfo::legalizeFceil(
975 MachineInstr &MI, MachineRegisterInfo &MRI,
976 MachineIRBuilder &B) const {
977 B.setInstr(MI);
978
Matt Arsenault1a02d302019-05-17 12:59:27 +0000979 const LLT S1 = LLT::scalar(1);
980 const LLT S64 = LLT::scalar(64);
981
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000982 Register Src = MI.getOperand(1).getReg();
Matt Arsenault1a02d302019-05-17 12:59:27 +0000983 assert(MRI.getType(Src) == S64);
Matt Arsenaulta510b572019-05-17 12:20:05 +0000984
985 // result = trunc(src)
986 // if (src > 0.0 && src != result)
987 // result += 1.0
988
Matt Arsenaulta510b572019-05-17 12:20:05 +0000989 auto Trunc = B.buildInstr(TargetOpcode::G_INTRINSIC_TRUNC, {S64}, {Src});
990
Matt Arsenaulta510b572019-05-17 12:20:05 +0000991 const auto Zero = B.buildFConstant(S64, 0.0);
992 const auto One = B.buildFConstant(S64, 1.0);
993 auto Lt0 = B.buildFCmp(CmpInst::FCMP_OGT, S1, Src, Zero);
994 auto NeTrunc = B.buildFCmp(CmpInst::FCMP_ONE, S1, Src, Trunc);
995 auto And = B.buildAnd(S1, Lt0, NeTrunc);
996 auto Add = B.buildSelect(S64, And, One, Zero);
997
998 // TODO: Should this propagate fast-math-flags?
999 B.buildFAdd(MI.getOperand(0).getReg(), Trunc, Add);
1000 return true;
1001}
1002
Matt Arsenault6aebcd52019-05-17 12:20:01 +00001003static MachineInstrBuilder extractF64Exponent(unsigned Hi,
1004 MachineIRBuilder &B) {
1005 const unsigned FractBits = 52;
1006 const unsigned ExpBits = 11;
1007 LLT S32 = LLT::scalar(32);
1008
1009 auto Const0 = B.buildConstant(S32, FractBits - 32);
1010 auto Const1 = B.buildConstant(S32, ExpBits);
1011
1012 auto ExpPart = B.buildIntrinsic(Intrinsic::amdgcn_ubfe, {S32}, false)
1013 .addUse(Const0.getReg(0))
1014 .addUse(Const1.getReg(0));
1015
1016 return B.buildSub(S32, ExpPart, B.buildConstant(S32, 1023));
1017}
1018
1019bool AMDGPULegalizerInfo::legalizeIntrinsicTrunc(
1020 MachineInstr &MI, MachineRegisterInfo &MRI,
1021 MachineIRBuilder &B) const {
1022 B.setInstr(MI);
1023
Matt Arsenault1a02d302019-05-17 12:59:27 +00001024 const LLT S1 = LLT::scalar(1);
1025 const LLT S32 = LLT::scalar(32);
1026 const LLT S64 = LLT::scalar(64);
Matt Arsenault6aebcd52019-05-17 12:20:01 +00001027
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001028 Register Src = MI.getOperand(1).getReg();
Matt Arsenault1a02d302019-05-17 12:59:27 +00001029 assert(MRI.getType(Src) == S64);
Matt Arsenault6aebcd52019-05-17 12:20:01 +00001030
1031 // TODO: Should this use extract since the low half is unused?
1032 auto Unmerge = B.buildUnmerge({S32, S32}, Src);
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001033 Register Hi = Unmerge.getReg(1);
Matt Arsenault6aebcd52019-05-17 12:20:01 +00001034
1035 // Extract the upper half, since this is where we will find the sign and
1036 // exponent.
1037 auto Exp = extractF64Exponent(Hi, B);
1038
1039 const unsigned FractBits = 52;
1040
1041 // Extract the sign bit.
1042 const auto SignBitMask = B.buildConstant(S32, UINT32_C(1) << 31);
1043 auto SignBit = B.buildAnd(S32, Hi, SignBitMask);
1044
1045 const auto FractMask = B.buildConstant(S64, (UINT64_C(1) << FractBits) - 1);
1046
1047 const auto Zero32 = B.buildConstant(S32, 0);
1048
1049 // Extend back to 64-bits.
1050 auto SignBit64 = B.buildMerge(S64, {Zero32.getReg(0), SignBit.getReg(0)});
1051
1052 auto Shr = B.buildAShr(S64, FractMask, Exp);
1053 auto Not = B.buildNot(S64, Shr);
1054 auto Tmp0 = B.buildAnd(S64, Src, Not);
1055 auto FiftyOne = B.buildConstant(S32, FractBits - 1);
1056
1057 auto ExpLt0 = B.buildICmp(CmpInst::ICMP_SLT, S1, Exp, Zero32);
1058 auto ExpGt51 = B.buildICmp(CmpInst::ICMP_SGT, S1, Exp, FiftyOne);
1059
1060 auto Tmp1 = B.buildSelect(S64, ExpLt0, SignBit64, Tmp0);
1061 B.buildSelect(MI.getOperand(0).getReg(), ExpGt51, Src, Tmp1);
1062 return true;
1063}
Matt Arsenault2f292202019-05-17 23:05:18 +00001064
1065bool AMDGPULegalizerInfo::legalizeITOFP(
1066 MachineInstr &MI, MachineRegisterInfo &MRI,
1067 MachineIRBuilder &B, bool Signed) const {
1068 B.setInstr(MI);
1069
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001070 Register Dst = MI.getOperand(0).getReg();
1071 Register Src = MI.getOperand(1).getReg();
Matt Arsenault2f292202019-05-17 23:05:18 +00001072
1073 const LLT S64 = LLT::scalar(64);
1074 const LLT S32 = LLT::scalar(32);
1075
1076 assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S64);
1077
1078 auto Unmerge = B.buildUnmerge({S32, S32}, Src);
1079
1080 auto CvtHi = Signed ?
1081 B.buildSITOFP(S64, Unmerge.getReg(1)) :
1082 B.buildUITOFP(S64, Unmerge.getReg(1));
1083
1084 auto CvtLo = B.buildUITOFP(S64, Unmerge.getReg(0));
1085
1086 auto ThirtyTwo = B.buildConstant(S32, 32);
1087 auto LdExp = B.buildIntrinsic(Intrinsic::amdgcn_ldexp, {S64}, false)
1088 .addUse(CvtHi.getReg(0))
1089 .addUse(ThirtyTwo.getReg(0));
1090
1091 // TODO: Should this propagate fast-math-flags?
1092 B.buildFAdd(Dst, LdExp, CvtLo);
1093 MI.eraseFromParent();
1094 return true;
1095}
Matt Arsenaulte15770a2019-07-01 18:40:23 +00001096
Matt Arsenault6ce1b4f2019-07-10 16:31:19 +00001097bool AMDGPULegalizerInfo::legalizeMinNumMaxNum(
1098 MachineInstr &MI, MachineRegisterInfo &MRI,
1099 MachineIRBuilder &B) const {
1100 MachineFunction &MF = B.getMF();
1101 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1102
1103 const bool IsIEEEOp = MI.getOpcode() == AMDGPU::G_FMINNUM_IEEE ||
1104 MI.getOpcode() == AMDGPU::G_FMAXNUM_IEEE;
1105
1106 // With ieee_mode disabled, the instructions have the correct behavior
1107 // already for G_FMINNUM/G_FMAXNUM
1108 if (!MFI->getMode().IEEE)
1109 return !IsIEEEOp;
1110
1111 if (IsIEEEOp)
1112 return true;
1113
1114 MachineIRBuilder HelperBuilder(MI);
1115 GISelObserverWrapper DummyObserver;
1116 LegalizerHelper Helper(MF, DummyObserver, HelperBuilder);
1117 HelperBuilder.setMBB(*MI.getParent());
1118 return Helper.lowerFMinNumMaxNum(MI) == LegalizerHelper::Legalized;
1119}
1120
Matt Arsenaulte15770a2019-07-01 18:40:23 +00001121// Return the use branch instruction, otherwise null if the usage is invalid.
1122static MachineInstr *verifyCFIntrinsic(MachineInstr &MI,
1123 MachineRegisterInfo &MRI) {
1124 Register CondDef = MI.getOperand(0).getReg();
1125 if (!MRI.hasOneNonDBGUse(CondDef))
1126 return nullptr;
1127
1128 MachineInstr &UseMI = *MRI.use_instr_nodbg_begin(CondDef);
1129 return UseMI.getParent() == MI.getParent() &&
1130 UseMI.getOpcode() == AMDGPU::G_BRCOND ? &UseMI : nullptr;
1131}
1132
Matt Arsenaulte2c86cc2019-07-01 18:45:36 +00001133Register AMDGPULegalizerInfo::getLiveInRegister(MachineRegisterInfo &MRI,
1134 Register Reg, LLT Ty) const {
1135 Register LiveIn = MRI.getLiveInVirtReg(Reg);
1136 if (LiveIn)
1137 return LiveIn;
1138
1139 Register NewReg = MRI.createGenericVirtualRegister(Ty);
1140 MRI.addLiveIn(Reg, NewReg);
1141 return NewReg;
1142}
1143
1144bool AMDGPULegalizerInfo::loadInputValue(Register DstReg, MachineIRBuilder &B,
1145 const ArgDescriptor *Arg) const {
1146 if (!Arg->isRegister())
1147 return false; // TODO: Handle these
1148
1149 assert(Arg->getRegister() != 0);
1150 assert(Arg->getRegister().isPhysical());
1151
1152 MachineRegisterInfo &MRI = *B.getMRI();
1153
1154 LLT Ty = MRI.getType(DstReg);
1155 Register LiveIn = getLiveInRegister(MRI, Arg->getRegister(), Ty);
1156
1157 if (Arg->isMasked()) {
1158 // TODO: Should we try to emit this once in the entry block?
1159 const LLT S32 = LLT::scalar(32);
1160 const unsigned Mask = Arg->getMask();
1161 const unsigned Shift = countTrailingZeros<unsigned>(Mask);
1162
1163 auto ShiftAmt = B.buildConstant(S32, Shift);
1164 auto LShr = B.buildLShr(S32, LiveIn, ShiftAmt);
1165 B.buildAnd(DstReg, LShr, B.buildConstant(S32, Mask >> Shift));
1166 } else
1167 B.buildCopy(DstReg, LiveIn);
1168
1169 // Insert the argument copy if it doens't already exist.
1170 // FIXME: It seems EmitLiveInCopies isn't called anywhere?
1171 if (!MRI.getVRegDef(LiveIn)) {
1172 MachineBasicBlock &EntryMBB = B.getMF().front();
1173 EntryMBB.addLiveIn(Arg->getRegister());
1174 B.setInsertPt(EntryMBB, EntryMBB.begin());
1175 B.buildCopy(LiveIn, Arg->getRegister());
1176 }
1177
1178 return true;
1179}
1180
1181bool AMDGPULegalizerInfo::legalizePreloadedArgIntrin(
1182 MachineInstr &MI,
1183 MachineRegisterInfo &MRI,
1184 MachineIRBuilder &B,
1185 AMDGPUFunctionArgInfo::PreloadedValue ArgType) const {
1186 B.setInstr(MI);
1187
1188 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
1189
1190 const ArgDescriptor *Arg;
1191 const TargetRegisterClass *RC;
1192 std::tie(Arg, RC) = MFI->getPreloadedValue(ArgType);
1193 if (!Arg) {
1194 LLVM_DEBUG(dbgs() << "Required arg register missing\n");
1195 return false;
1196 }
1197
1198 if (loadInputValue(MI.getOperand(0).getReg(), B, Arg)) {
1199 MI.eraseFromParent();
1200 return true;
1201 }
1202
1203 return false;
1204}
1205
Matt Arsenault9e8e8c62019-07-01 18:49:01 +00001206bool AMDGPULegalizerInfo::legalizeImplicitArgPtr(MachineInstr &MI,
1207 MachineRegisterInfo &MRI,
1208 MachineIRBuilder &B) const {
1209 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
1210 if (!MFI->isEntryFunction()) {
1211 return legalizePreloadedArgIntrin(MI, MRI, B,
1212 AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR);
1213 }
1214
1215 B.setInstr(MI);
1216
1217 uint64_t Offset =
1218 ST.getTargetLowering()->getImplicitParameterOffset(
1219 B.getMF(), AMDGPUTargetLowering::FIRST_IMPLICIT);
1220 Register DstReg = MI.getOperand(0).getReg();
1221 LLT DstTy = MRI.getType(DstReg);
1222 LLT IdxTy = LLT::scalar(DstTy.getSizeInBits());
1223
1224 const ArgDescriptor *Arg;
1225 const TargetRegisterClass *RC;
1226 std::tie(Arg, RC)
1227 = MFI->getPreloadedValue(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
1228 if (!Arg)
1229 return false;
1230
1231 Register KernargPtrReg = MRI.createGenericVirtualRegister(DstTy);
1232 if (!loadInputValue(KernargPtrReg, B, Arg))
1233 return false;
1234
1235 B.buildGEP(DstReg, KernargPtrReg, B.buildConstant(IdxTy, Offset).getReg(0));
1236 MI.eraseFromParent();
1237 return true;
1238}
1239
Matt Arsenaulte15770a2019-07-01 18:40:23 +00001240bool AMDGPULegalizerInfo::legalizeIntrinsic(MachineInstr &MI,
1241 MachineRegisterInfo &MRI,
1242 MachineIRBuilder &B) const {
1243 // Replace the use G_BRCOND with the exec manipulate and branch pseudos.
1244 switch (MI.getOperand(MI.getNumExplicitDefs()).getIntrinsicID()) {
1245 case Intrinsic::amdgcn_if: {
1246 if (MachineInstr *BrCond = verifyCFIntrinsic(MI, MRI)) {
1247 const SIRegisterInfo *TRI
1248 = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
1249
1250 B.setInstr(*BrCond);
1251 Register Def = MI.getOperand(1).getReg();
1252 Register Use = MI.getOperand(3).getReg();
1253 B.buildInstr(AMDGPU::SI_IF)
1254 .addDef(Def)
1255 .addUse(Use)
1256 .addMBB(BrCond->getOperand(1).getMBB());
1257
1258 MRI.setRegClass(Def, TRI->getWaveMaskRegClass());
1259 MRI.setRegClass(Use, TRI->getWaveMaskRegClass());
1260 MI.eraseFromParent();
1261 BrCond->eraseFromParent();
1262 return true;
1263 }
1264
1265 return false;
1266 }
1267 case Intrinsic::amdgcn_loop: {
1268 if (MachineInstr *BrCond = verifyCFIntrinsic(MI, MRI)) {
1269 const SIRegisterInfo *TRI
1270 = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
1271
1272 B.setInstr(*BrCond);
1273 Register Reg = MI.getOperand(2).getReg();
1274 B.buildInstr(AMDGPU::SI_LOOP)
1275 .addUse(Reg)
1276 .addMBB(BrCond->getOperand(1).getMBB());
1277 MI.eraseFromParent();
1278 BrCond->eraseFromParent();
1279 MRI.setRegClass(Reg, TRI->getWaveMaskRegClass());
1280 return true;
1281 }
1282
1283 return false;
1284 }
Matt Arsenault9e8e8c62019-07-01 18:49:01 +00001285 case Intrinsic::amdgcn_kernarg_segment_ptr:
1286 return legalizePreloadedArgIntrin(
1287 MI, MRI, B, AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
1288 case Intrinsic::amdgcn_implicitarg_ptr:
1289 return legalizeImplicitArgPtr(MI, MRI, B);
Matt Arsenaulte2c86cc2019-07-01 18:45:36 +00001290 case Intrinsic::amdgcn_workitem_id_x:
1291 return legalizePreloadedArgIntrin(MI, MRI, B,
1292 AMDGPUFunctionArgInfo::WORKITEM_ID_X);
1293 case Intrinsic::amdgcn_workitem_id_y:
1294 return legalizePreloadedArgIntrin(MI, MRI, B,
1295 AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
1296 case Intrinsic::amdgcn_workitem_id_z:
1297 return legalizePreloadedArgIntrin(MI, MRI, B,
1298 AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
Matt Arsenault756d8192019-07-01 18:47:22 +00001299 case Intrinsic::amdgcn_workgroup_id_x:
1300 return legalizePreloadedArgIntrin(MI, MRI, B,
1301 AMDGPUFunctionArgInfo::WORKGROUP_ID_X);
1302 case Intrinsic::amdgcn_workgroup_id_y:
1303 return legalizePreloadedArgIntrin(MI, MRI, B,
1304 AMDGPUFunctionArgInfo::WORKGROUP_ID_Y);
1305 case Intrinsic::amdgcn_workgroup_id_z:
1306 return legalizePreloadedArgIntrin(MI, MRI, B,
1307 AMDGPUFunctionArgInfo::WORKGROUP_ID_Z);
Matt Arsenaultbae36362019-07-01 18:50:50 +00001308 case Intrinsic::amdgcn_dispatch_ptr:
1309 return legalizePreloadedArgIntrin(MI, MRI, B,
1310 AMDGPUFunctionArgInfo::DISPATCH_PTR);
1311 case Intrinsic::amdgcn_queue_ptr:
1312 return legalizePreloadedArgIntrin(MI, MRI, B,
1313 AMDGPUFunctionArgInfo::QUEUE_PTR);
1314 case Intrinsic::amdgcn_implicit_buffer_ptr:
1315 return legalizePreloadedArgIntrin(
1316 MI, MRI, B, AMDGPUFunctionArgInfo::IMPLICIT_BUFFER_PTR);
1317 case Intrinsic::amdgcn_dispatch_id:
1318 return legalizePreloadedArgIntrin(MI, MRI, B,
1319 AMDGPUFunctionArgInfo::DISPATCH_ID);
Matt Arsenaulte15770a2019-07-01 18:40:23 +00001320 default:
1321 return true;
1322 }
1323
1324 return true;
1325}