blob: f33fe2e128dc0d4b9f55a53f2519ba47149808dd [file] [log] [blame]
Tom Stellardca166212017-01-30 21:56:46 +00001//===- AMDGPULegalizerInfo.cpp -----------------------------------*- C++ -*-==//
2//
Chandler Carruth2946cd72019-01-19 08:50:56 +00003// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Tom Stellardca166212017-01-30 21:56:46 +00006//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the Machinelegalizer class for
10/// AMDGPU.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
David Blaikie36a0f222018-03-23 23:58:31 +000014#include "AMDGPU.h"
Craig Topper2fa14362018-03-29 17:21:10 +000015#include "AMDGPULegalizerInfo.h"
Matt Arsenault85803362018-03-17 15:17:41 +000016#include "AMDGPUTargetMachine.h"
Matt Arsenaulta8b43392019-02-08 02:40:47 +000017#include "SIMachineFunctionInfo.h"
18
19#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
David Blaikieb3bde2e2017-11-17 01:07:10 +000020#include "llvm/CodeGen/TargetOpcodes.h"
Craig Topper2fa14362018-03-29 17:21:10 +000021#include "llvm/CodeGen/ValueTypes.h"
Tom Stellardca166212017-01-30 21:56:46 +000022#include "llvm/IR/DerivedTypes.h"
Chandler Carruth6bda14b2017-06-06 11:49:48 +000023#include "llvm/IR/Type.h"
Tom Stellardca166212017-01-30 21:56:46 +000024#include "llvm/Support/Debug.h"
25
Matt Arsenaulte2c86cc2019-07-01 18:45:36 +000026#define DEBUG_TYPE "amdgpu-legalinfo"
27
Tom Stellardca166212017-01-30 21:56:46 +000028using namespace llvm;
Daniel Sanders9ade5592018-01-29 17:37:29 +000029using namespace LegalizeActions;
Matt Arsenault990f5072019-01-25 00:51:00 +000030using namespace LegalizeMutations;
Matt Arsenault7ac79ed2019-01-20 19:45:18 +000031using namespace LegalityPredicates;
Tom Stellardca166212017-01-30 21:56:46 +000032
Matt Arsenaultd9141892019-02-07 19:10:15 +000033
34static LegalityPredicate isMultiple32(unsigned TypeIdx,
35 unsigned MaxSize = 512) {
36 return [=](const LegalityQuery &Query) {
37 const LLT Ty = Query.Types[TypeIdx];
38 const LLT EltTy = Ty.getScalarType();
39 return Ty.getSizeInBits() <= MaxSize && EltTy.getSizeInBits() % 32 == 0;
40 };
41}
42
Matt Arsenault18ec3822019-02-11 22:00:39 +000043static LegalityPredicate isSmallOddVector(unsigned TypeIdx) {
44 return [=](const LegalityQuery &Query) {
45 const LLT Ty = Query.Types[TypeIdx];
46 return Ty.isVector() &&
47 Ty.getNumElements() % 2 != 0 &&
48 Ty.getElementType().getSizeInBits() < 32;
49 };
50}
51
52static LegalizeMutation oneMoreElement(unsigned TypeIdx) {
53 return [=](const LegalityQuery &Query) {
54 const LLT Ty = Query.Types[TypeIdx];
55 const LLT EltTy = Ty.getElementType();
56 return std::make_pair(TypeIdx, LLT::vector(Ty.getNumElements() + 1, EltTy));
57 };
58}
59
Matt Arsenault26b7e852019-02-19 16:30:19 +000060static LegalizeMutation fewerEltsToSize64Vector(unsigned TypeIdx) {
61 return [=](const LegalityQuery &Query) {
62 const LLT Ty = Query.Types[TypeIdx];
63 const LLT EltTy = Ty.getElementType();
64 unsigned Size = Ty.getSizeInBits();
65 unsigned Pieces = (Size + 63) / 64;
66 unsigned NewNumElts = (Ty.getNumElements() + 1) / Pieces;
67 return std::make_pair(TypeIdx, LLT::scalarOrVector(NewNumElts, EltTy));
68 };
69}
70
71static LegalityPredicate vectorWiderThan(unsigned TypeIdx, unsigned Size) {
72 return [=](const LegalityQuery &Query) {
73 const LLT QueryTy = Query.Types[TypeIdx];
74 return QueryTy.isVector() && QueryTy.getSizeInBits() > Size;
75 };
76}
77
Matt Arsenaultb4c95b32019-02-19 17:03:09 +000078static LegalityPredicate numElementsNotEven(unsigned TypeIdx) {
79 return [=](const LegalityQuery &Query) {
80 const LLT QueryTy = Query.Types[TypeIdx];
81 return QueryTy.isVector() && QueryTy.getNumElements() % 2 != 0;
82 };
83}
Matt Arsenault18ec3822019-02-11 22:00:39 +000084
Matt Arsenault4dd57552019-07-09 14:17:31 +000085// Any combination of 32 or 64-bit elements up to 512 bits, and multiples of
86// v2s16.
87static LegalityPredicate isRegisterType(unsigned TypeIdx) {
88 return [=](const LegalityQuery &Query) {
89 const LLT Ty = Query.Types[TypeIdx];
90 if (Ty.isVector()) {
91 const int EltSize = Ty.getElementType().getSizeInBits();
92 return EltSize == 32 || EltSize == 64 ||
Matt Arsenault3f1a3452019-07-09 22:48:04 +000093 (EltSize == 16 && Ty.getNumElements() % 2 == 0) ||
94 EltSize == 128 || EltSize == 256;
Matt Arsenault4dd57552019-07-09 14:17:31 +000095 }
96
97 return Ty.getSizeInBits() % 32 == 0 && Ty.getSizeInBits() <= 512;
98 };
99}
100
Matt Arsenault9e8e8c62019-07-01 18:49:01 +0000101AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
102 const GCNTargetMachine &TM)
103 : ST(ST_) {
Tom Stellardca166212017-01-30 21:56:46 +0000104 using namespace TargetOpcode;
105
Matt Arsenault85803362018-03-17 15:17:41 +0000106 auto GetAddrSpacePtr = [&TM](unsigned AS) {
107 return LLT::pointer(AS, TM.getPointerSizeInBits(AS));
108 };
109
110 const LLT S1 = LLT::scalar(1);
Matt Arsenault888aa5d2019-02-03 00:07:33 +0000111 const LLT S8 = LLT::scalar(8);
Matt Arsenault45991592019-01-18 21:33:50 +0000112 const LLT S16 = LLT::scalar(16);
Tom Stellardca166212017-01-30 21:56:46 +0000113 const LLT S32 = LLT::scalar(32);
114 const LLT S64 = LLT::scalar(64);
Matt Arsenaultca676342019-01-25 02:36:32 +0000115 const LLT S128 = LLT::scalar(128);
Matt Arsenaultff6a9a22019-01-20 18:40:36 +0000116 const LLT S256 = LLT::scalar(256);
Tom Stellardeebbfc22018-06-30 04:09:44 +0000117 const LLT S512 = LLT::scalar(512);
Matt Arsenault85803362018-03-17 15:17:41 +0000118
Matt Arsenaultbee2ad72018-12-21 03:03:11 +0000119 const LLT V2S16 = LLT::vector(2, 16);
Matt Arsenaulta1515d22019-01-08 01:30:02 +0000120 const LLT V4S16 = LLT::vector(4, 16);
Matt Arsenaultbee2ad72018-12-21 03:03:11 +0000121
122 const LLT V2S32 = LLT::vector(2, 32);
123 const LLT V3S32 = LLT::vector(3, 32);
124 const LLT V4S32 = LLT::vector(4, 32);
125 const LLT V5S32 = LLT::vector(5, 32);
126 const LLT V6S32 = LLT::vector(6, 32);
127 const LLT V7S32 = LLT::vector(7, 32);
128 const LLT V8S32 = LLT::vector(8, 32);
129 const LLT V9S32 = LLT::vector(9, 32);
130 const LLT V10S32 = LLT::vector(10, 32);
131 const LLT V11S32 = LLT::vector(11, 32);
132 const LLT V12S32 = LLT::vector(12, 32);
133 const LLT V13S32 = LLT::vector(13, 32);
134 const LLT V14S32 = LLT::vector(14, 32);
135 const LLT V15S32 = LLT::vector(15, 32);
136 const LLT V16S32 = LLT::vector(16, 32);
137
138 const LLT V2S64 = LLT::vector(2, 64);
139 const LLT V3S64 = LLT::vector(3, 64);
140 const LLT V4S64 = LLT::vector(4, 64);
141 const LLT V5S64 = LLT::vector(5, 64);
142 const LLT V6S64 = LLT::vector(6, 64);
143 const LLT V7S64 = LLT::vector(7, 64);
144 const LLT V8S64 = LLT::vector(8, 64);
145
146 std::initializer_list<LLT> AllS32Vectors =
147 {V2S32, V3S32, V4S32, V5S32, V6S32, V7S32, V8S32,
148 V9S32, V10S32, V11S32, V12S32, V13S32, V14S32, V15S32, V16S32};
149 std::initializer_list<LLT> AllS64Vectors =
150 {V2S64, V3S64, V4S64, V5S64, V6S64, V7S64, V8S64};
151
Matt Arsenault85803362018-03-17 15:17:41 +0000152 const LLT GlobalPtr = GetAddrSpacePtr(AMDGPUAS::GLOBAL_ADDRESS);
153 const LLT ConstantPtr = GetAddrSpacePtr(AMDGPUAS::CONSTANT_ADDRESS);
Matt Arsenault685d1e82018-03-17 15:17:45 +0000154 const LLT LocalPtr = GetAddrSpacePtr(AMDGPUAS::LOCAL_ADDRESS);
Matt Arsenault0da63502018-08-31 05:49:54 +0000155 const LLT FlatPtr = GetAddrSpacePtr(AMDGPUAS::FLAT_ADDRESS);
156 const LLT PrivatePtr = GetAddrSpacePtr(AMDGPUAS::PRIVATE_ADDRESS);
Matt Arsenault85803362018-03-17 15:17:41 +0000157
Matt Arsenault934e5342018-12-13 20:34:15 +0000158 const LLT CodePtr = FlatPtr;
159
Matt Arsenault9e5e8682019-02-14 22:24:28 +0000160 const std::initializer_list<LLT> AddrSpaces64 = {
161 GlobalPtr, ConstantPtr, FlatPtr
162 };
163
164 const std::initializer_list<LLT> AddrSpaces32 = {
165 LocalPtr, PrivatePtr
Matt Arsenault685d1e82018-03-17 15:17:45 +0000166 };
Tom Stellardca166212017-01-30 21:56:46 +0000167
Matt Arsenault40d1faf2019-07-01 17:35:53 +0000168 const std::initializer_list<LLT> FPTypesBase = {
169 S32, S64
170 };
171
172 const std::initializer_list<LLT> FPTypes16 = {
173 S32, S64, S16
174 };
175
Matt Arsenaultadc40ba2019-01-08 01:22:47 +0000176 setAction({G_BRCOND, S1}, Legal);
177
Matt Arsenault2e0ee472019-02-21 15:48:13 +0000178 // TODO: All multiples of 32, vectors of pointers, all v2s16 pairs, more
179 // elements for v3s16
180 getActionDefinitionsBuilder(G_PHI)
181 .legalFor({S32, S64, V2S16, V4S16, S1, S128, S256})
182 .legalFor(AllS32Vectors)
183 .legalFor(AllS64Vectors)
184 .legalFor(AddrSpaces64)
185 .legalFor(AddrSpaces32)
186 .clampScalar(0, S32, S256)
187 .widenScalarToNextPow2(0, 32)
Matt Arsenaultd3093c22019-02-28 00:16:32 +0000188 .clampMaxNumElements(0, S32, 16)
Matt Arsenault72bcf152019-02-28 00:01:05 +0000189 .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
Matt Arsenault2e0ee472019-02-21 15:48:13 +0000190 .legalIf(isPointer(0));
191
Matt Arsenaultef59cb62019-07-01 18:18:55 +0000192 if (ST.has16BitInsts()) {
193 getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL})
194 .legalFor({S32, S16})
195 .clampScalar(0, S16, S32)
196 .scalarize(0);
197 } else {
198 getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL})
199 .legalFor({S32})
200 .clampScalar(0, S32, S32)
201 .scalarize(0);
202 }
Matt Arsenault2e0ee472019-02-21 15:48:13 +0000203
Matt Arsenaultef59cb62019-07-01 18:18:55 +0000204 getActionDefinitionsBuilder({G_UMULH, G_SMULH})
Matt Arsenault5d622fb2019-01-25 03:23:04 +0000205 .legalFor({S32})
Matt Arsenault211e89d2019-01-27 00:52:51 +0000206 .clampScalar(0, S32, S32)
Matt Arsenault5d622fb2019-01-25 03:23:04 +0000207 .scalarize(0);
Matt Arsenault43398832018-12-20 01:35:49 +0000208
Matt Arsenault26a6c742019-01-26 23:47:07 +0000209 // Report legal for any types we can handle anywhere. For the cases only legal
210 // on the SALU, RegBankSelect will be able to re-legalize.
Matt Arsenault43398832018-12-20 01:35:49 +0000211 getActionDefinitionsBuilder({G_AND, G_OR, G_XOR})
Matt Arsenault26a6c742019-01-26 23:47:07 +0000212 .legalFor({S32, S1, S64, V2S32, V2S16, V4S16})
213 .clampScalar(0, S32, S64)
Matt Arsenault26b7e852019-02-19 16:30:19 +0000214 .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
215 .fewerElementsIf(vectorWiderThan(0, 32), fewerEltsToSize64Vector(0))
Matt Arsenaultf4bfe4c2019-02-25 21:32:48 +0000216 .widenScalarToNextPow2(0)
Matt Arsenault26a6c742019-01-26 23:47:07 +0000217 .scalarize(0);
Tom Stellardee6e6452017-06-12 20:54:56 +0000218
Matt Arsenault68c668a2019-01-08 01:09:09 +0000219 getActionDefinitionsBuilder({G_UADDO, G_SADDO, G_USUBO, G_SSUBO,
220 G_UADDE, G_SADDE, G_USUBE, G_SSUBE})
Matt Arsenault4d475942019-01-26 23:44:51 +0000221 .legalFor({{S32, S1}})
222 .clampScalar(0, S32, S32);
Matt Arsenault2cc15b62019-01-08 01:03:58 +0000223
Matt Arsenault7ac79ed2019-01-20 19:45:18 +0000224 getActionDefinitionsBuilder(G_BITCAST)
225 .legalForCartesianProduct({S32, V2S16})
226 .legalForCartesianProduct({S64, V2S32, V4S16})
227 .legalForCartesianProduct({V2S64, V4S32})
228 // Don't worry about the size constraint.
229 .legalIf(all(isPointer(0), isPointer(1)));
Tom Stellardff63ee02017-06-19 13:15:45 +0000230
Matt Arsenault00ccd132019-02-12 14:54:55 +0000231 if (ST.has16BitInsts()) {
232 getActionDefinitionsBuilder(G_FCONSTANT)
233 .legalFor({S32, S64, S16})
234 .clampScalar(0, S16, S64);
235 } else {
236 getActionDefinitionsBuilder(G_FCONSTANT)
237 .legalFor({S32, S64})
238 .clampScalar(0, S32, S64);
239 }
Tom Stellardeebbfc22018-06-30 04:09:44 +0000240
Matt Arsenaultb3feccd2018-06-25 15:42:12 +0000241 getActionDefinitionsBuilder(G_IMPLICIT_DEF)
Matt Arsenaultd9141892019-02-07 19:10:15 +0000242 .legalFor({S1, S32, S64, V2S32, V4S32, V2S16, V4S16, GlobalPtr,
243 ConstantPtr, LocalPtr, FlatPtr, PrivatePtr})
Matt Arsenault18ec3822019-02-11 22:00:39 +0000244 .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
Matt Arsenaultd9141892019-02-07 19:10:15 +0000245 .clampScalarOrElt(0, S32, S512)
Matt Arsenault0f2debb2019-02-08 14:46:27 +0000246 .legalIf(isMultiple32(0))
Matt Arsenault82b10392019-02-25 20:46:06 +0000247 .widenScalarToNextPow2(0, 32)
248 .clampMaxNumElements(0, S32, 16);
Matt Arsenaultb3feccd2018-06-25 15:42:12 +0000249
Matt Arsenaultabdc4f22018-03-17 15:17:48 +0000250
Tom Stellarde0424122017-06-03 01:13:33 +0000251 // FIXME: i1 operands to intrinsics should always be legal, but other i1
252 // values may not be legal. We need to figure out how to distinguish
253 // between these two scenarios.
Matt Arsenault45991592019-01-18 21:33:50 +0000254 getActionDefinitionsBuilder(G_CONSTANT)
Matt Arsenault2065c942019-02-02 23:33:49 +0000255 .legalFor({S1, S32, S64, GlobalPtr,
256 LocalPtr, ConstantPtr, PrivatePtr, FlatPtr })
Matt Arsenault45991592019-01-18 21:33:50 +0000257 .clampScalar(0, S32, S64)
Matt Arsenault2065c942019-02-02 23:33:49 +0000258 .widenScalarToNextPow2(0)
259 .legalIf(isPointer(0));
Matt Arsenault06cbb272018-03-01 19:16:52 +0000260
Matt Arsenaultc94e26c2018-12-18 09:46:13 +0000261 setAction({G_FRAME_INDEX, PrivatePtr}, Legal);
262
Matt Arsenault93fdec72019-02-07 18:03:11 +0000263 auto &FPOpActions = getActionDefinitionsBuilder(
Matt Arsenault9dba67f2019-02-11 17:05:20 +0000264 { G_FADD, G_FMUL, G_FNEG, G_FABS, G_FMA, G_FCANONICALIZE})
Matt Arsenault93fdec72019-02-07 18:03:11 +0000265 .legalFor({S32, S64});
266
267 if (ST.has16BitInsts()) {
268 if (ST.hasVOP3PInsts())
269 FPOpActions.legalFor({S16, V2S16});
270 else
271 FPOpActions.legalFor({S16});
272 }
273
274 if (ST.hasVOP3PInsts())
275 FPOpActions.clampMaxNumElements(0, S16, 2);
276 FPOpActions
277 .scalarize(0)
278 .clampScalar(0, ST.has16BitInsts() ? S16 : S32, S64);
Tom Stellardd0c6cf22017-10-27 23:57:41 +0000279
Matt Arsenaultc0f75692019-02-07 18:14:39 +0000280 if (ST.has16BitInsts()) {
281 getActionDefinitionsBuilder(G_FSQRT)
282 .legalFor({S32, S64, S16})
283 .scalarize(0)
284 .clampScalar(0, S16, S64);
285 } else {
286 getActionDefinitionsBuilder(G_FSQRT)
287 .legalFor({S32, S64})
288 .scalarize(0)
289 .clampScalar(0, S32, S64);
290 }
291
Matt Arsenaultdff33c32018-12-20 00:37:02 +0000292 getActionDefinitionsBuilder(G_FPTRUNC)
Matt Arsenaulte6cebd02019-01-25 04:37:33 +0000293 .legalFor({{S32, S64}, {S16, S32}})
294 .scalarize(0);
Matt Arsenaultdff33c32018-12-20 00:37:02 +0000295
Matt Arsenault24563ef2019-01-20 18:34:24 +0000296 getActionDefinitionsBuilder(G_FPEXT)
297 .legalFor({{S64, S32}, {S32, S16}})
Matt Arsenaultca676342019-01-25 02:36:32 +0000298 .lowerFor({{S64, S16}}) // FIXME: Implement
299 .scalarize(0);
Matt Arsenault24563ef2019-01-20 18:34:24 +0000300
Matt Arsenaultb1843e12019-07-09 23:34:29 +0000301 // TODO: Verify V_BFI_B32 is generated from expanded bit ops.
302 getActionDefinitionsBuilder(G_FCOPYSIGN).lower();
Matt Arsenault1448f562019-05-17 12:19:52 +0000303
Matt Arsenault745fd9f2019-01-20 19:10:31 +0000304 getActionDefinitionsBuilder(G_FSUB)
Matt Arsenaultaebb2ee2019-01-22 20:14:29 +0000305 // Use actual fsub instruction
306 .legalFor({S32})
307 // Must use fadd + fneg
308 .lowerFor({S64, S16, V2S16})
Matt Arsenault990f5072019-01-25 00:51:00 +0000309 .scalarize(0)
Matt Arsenaultaebb2ee2019-01-22 20:14:29 +0000310 .clampScalar(0, S32, S64);
Matt Arsenaulte01e7c82018-12-18 09:19:03 +0000311
Matt Arsenault24563ef2019-01-20 18:34:24 +0000312 getActionDefinitionsBuilder({G_SEXT, G_ZEXT, G_ANYEXT})
Matt Arsenault46ffe682019-01-20 19:28:20 +0000313 .legalFor({{S64, S32}, {S32, S16}, {S64, S16},
Matt Arsenaultca676342019-01-25 02:36:32 +0000314 {S32, S1}, {S64, S1}, {S16, S1},
315 // FIXME: Hack
Matt Arsenaultf4bfe4c2019-02-25 21:32:48 +0000316 {S64, LLT::scalar(33)},
Matt Arsenault888aa5d2019-02-03 00:07:33 +0000317 {S32, S8}, {S128, S32}, {S128, S64}, {S32, LLT::scalar(24)}})
Matt Arsenaultca676342019-01-25 02:36:32 +0000318 .scalarize(0);
Matt Arsenaultf38f4832018-12-13 08:23:51 +0000319
Matt Arsenaultfb671642019-01-22 00:20:17 +0000320 getActionDefinitionsBuilder({G_SITOFP, G_UITOFP})
Matt Arsenaulte6cebd02019-01-25 04:37:33 +0000321 .legalFor({{S32, S32}, {S64, S32}})
Matt Arsenault02b5ca82019-05-17 23:05:13 +0000322 .lowerFor({{S32, S64}})
Matt Arsenault2f292202019-05-17 23:05:18 +0000323 .customFor({{S64, S64}})
Matt Arsenaulte6cebd02019-01-25 04:37:33 +0000324 .scalarize(0);
Matt Arsenaultdd022ce2018-03-01 19:04:25 +0000325
Matt Arsenaultfb671642019-01-22 00:20:17 +0000326 getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI})
Matt Arsenaulte6cebd02019-01-25 04:37:33 +0000327 .legalFor({{S32, S32}, {S32, S64}})
328 .scalarize(0);
Tom Stellard33445762018-02-07 04:47:59 +0000329
Matt Arsenault6aebcd52019-05-17 12:20:01 +0000330 getActionDefinitionsBuilder(G_INTRINSIC_ROUND)
Matt Arsenault2e5f9002019-01-27 00:12:21 +0000331 .legalFor({S32, S64})
332 .scalarize(0);
Matt Arsenaultf4c21c52018-12-21 03:14:45 +0000333
Matt Arsenault6aafc5e2019-05-17 12:19:57 +0000334 if (ST.getGeneration() >= AMDGPUSubtarget::SEA_ISLANDS) {
Matt Arsenaulta510b572019-05-17 12:20:05 +0000335 getActionDefinitionsBuilder({G_INTRINSIC_TRUNC, G_FCEIL, G_FRINT})
Matt Arsenault6aafc5e2019-05-17 12:19:57 +0000336 .legalFor({S32, S64})
337 .clampScalar(0, S32, S64)
338 .scalarize(0);
339 } else {
Matt Arsenaulta510b572019-05-17 12:20:05 +0000340 getActionDefinitionsBuilder({G_INTRINSIC_TRUNC, G_FCEIL, G_FRINT})
Matt Arsenault6aafc5e2019-05-17 12:19:57 +0000341 .legalFor({S32})
342 .customFor({S64})
343 .clampScalar(0, S32, S64)
344 .scalarize(0);
345 }
Tom Stellardca166212017-01-30 21:56:46 +0000346
Matt Arsenault9e5e8682019-02-14 22:24:28 +0000347 getActionDefinitionsBuilder(G_GEP)
348 .legalForCartesianProduct(AddrSpaces64, {S64})
349 .legalForCartesianProduct(AddrSpaces32, {S32})
350 .scalarize(0);
Matt Arsenault3b9a82f2019-01-25 04:54:00 +0000351
Matt Arsenault934e5342018-12-13 20:34:15 +0000352 setAction({G_BLOCK_ADDR, CodePtr}, Legal);
353
Matt Arsenault8b8eee52019-07-09 14:10:43 +0000354 auto &CmpBuilder =
355 getActionDefinitionsBuilder(G_ICMP)
Matt Arsenault58f9d3d2019-02-02 23:35:15 +0000356 .legalForCartesianProduct(
357 {S1}, {S32, S64, GlobalPtr, LocalPtr, ConstantPtr, PrivatePtr, FlatPtr})
Matt Arsenault8b8eee52019-07-09 14:10:43 +0000358 .legalFor({{S1, S32}, {S1, S64}});
359 if (ST.has16BitInsts()) {
360 CmpBuilder.legalFor({{S1, S16}});
361 }
362
363 CmpBuilder
Matt Arsenault58f9d3d2019-02-02 23:35:15 +0000364 .widenScalarToNextPow2(1)
365 .clampScalar(1, S32, S64)
366 .scalarize(0)
367 .legalIf(all(typeIs(0, S1), isPointer(1)));
368
369 getActionDefinitionsBuilder(G_FCMP)
Matt Arsenault40d1faf2019-07-01 17:35:53 +0000370 .legalForCartesianProduct({S1}, ST.has16BitInsts() ? FPTypes16 : FPTypesBase)
Matt Arsenault1b1e6852019-01-25 02:59:34 +0000371 .widenScalarToNextPow2(1)
372 .clampScalar(1, S32, S64)
Matt Arsenaultded2f822019-01-26 23:54:53 +0000373 .scalarize(0);
Matt Arsenault1b1e6852019-01-25 02:59:34 +0000374
Matt Arsenault95fd95c2019-01-25 04:03:38 +0000375 // FIXME: fexp, flog2, flog10 needs to be custom lowered.
376 getActionDefinitionsBuilder({G_FPOW, G_FEXP, G_FEXP2,
377 G_FLOG, G_FLOG2, G_FLOG10})
378 .legalFor({S32})
379 .scalarize(0);
Tom Stellard8cd60a52017-06-06 14:16:50 +0000380
Matt Arsenaultd5684f72019-01-31 02:09:57 +0000381 // The 64-bit versions produce 32-bit results, but only on the SALU.
382 getActionDefinitionsBuilder({G_CTLZ, G_CTLZ_ZERO_UNDEF,
383 G_CTTZ, G_CTTZ_ZERO_UNDEF,
384 G_CTPOP})
385 .legalFor({{S32, S32}, {S32, S64}})
386 .clampScalar(0, S32, S32)
Matt Arsenault75e30c42019-02-20 16:42:52 +0000387 .clampScalar(1, S32, S64)
Matt Arsenaultb10fa8d2019-02-21 15:22:20 +0000388 .scalarize(0)
389 .widenScalarToNextPow2(0, 32)
390 .widenScalarToNextPow2(1, 32);
Matt Arsenaultd5684f72019-01-31 02:09:57 +0000391
Matt Arsenaultd1bfc8d2019-01-31 02:34:03 +0000392 // TODO: Expand for > s32
393 getActionDefinitionsBuilder(G_BSWAP)
394 .legalFor({S32})
395 .clampScalar(0, S32, S32)
396 .scalarize(0);
Matt Arsenaultd5684f72019-01-31 02:09:57 +0000397
Matt Arsenault0f3ba442019-05-23 17:58:48 +0000398 if (ST.has16BitInsts()) {
399 if (ST.hasVOP3PInsts()) {
400 getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
401 .legalFor({S32, S16, V2S16})
402 .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
403 .clampMaxNumElements(0, S16, 2)
404 .clampScalar(0, S16, S32)
405 .widenScalarToNextPow2(0)
406 .scalarize(0);
407 } else {
408 getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
409 .legalFor({S32, S16})
410 .widenScalarToNextPow2(0)
411 .clampScalar(0, S16, S32)
412 .scalarize(0);
413 }
414 } else {
415 getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
416 .legalFor({S32})
417 .clampScalar(0, S32, S32)
418 .widenScalarToNextPow2(0)
419 .scalarize(0);
420 }
Matt Arsenaultf38f4832018-12-13 08:23:51 +0000421
Matt Arsenaultcbaada62019-02-02 23:29:55 +0000422 auto smallerThan = [](unsigned TypeIdx0, unsigned TypeIdx1) {
423 return [=](const LegalityQuery &Query) {
424 return Query.Types[TypeIdx0].getSizeInBits() <
425 Query.Types[TypeIdx1].getSizeInBits();
426 };
427 };
428
429 auto greaterThan = [](unsigned TypeIdx0, unsigned TypeIdx1) {
430 return [=](const LegalityQuery &Query) {
431 return Query.Types[TypeIdx0].getSizeInBits() >
432 Query.Types[TypeIdx1].getSizeInBits();
433 };
434 };
435
Tom Stellard7c650782018-10-05 04:34:09 +0000436 getActionDefinitionsBuilder(G_INTTOPTR)
Matt Arsenaultcbaada62019-02-02 23:29:55 +0000437 // List the common cases
Matt Arsenault9e5e8682019-02-14 22:24:28 +0000438 .legalForCartesianProduct(AddrSpaces64, {S64})
439 .legalForCartesianProduct(AddrSpaces32, {S32})
Matt Arsenaultcbaada62019-02-02 23:29:55 +0000440 .scalarize(0)
441 // Accept any address space as long as the size matches
442 .legalIf(sameSize(0, 1))
443 .widenScalarIf(smallerThan(1, 0),
444 [](const LegalityQuery &Query) {
445 return std::make_pair(1, LLT::scalar(Query.Types[0].getSizeInBits()));
446 })
447 .narrowScalarIf(greaterThan(1, 0),
448 [](const LegalityQuery &Query) {
449 return std::make_pair(1, LLT::scalar(Query.Types[0].getSizeInBits()));
450 });
Matt Arsenault85803362018-03-17 15:17:41 +0000451
Matt Arsenaultf38f4832018-12-13 08:23:51 +0000452 getActionDefinitionsBuilder(G_PTRTOINT)
Matt Arsenaultcbaada62019-02-02 23:29:55 +0000453 // List the common cases
Matt Arsenault9e5e8682019-02-14 22:24:28 +0000454 .legalForCartesianProduct(AddrSpaces64, {S64})
455 .legalForCartesianProduct(AddrSpaces32, {S32})
Matt Arsenaultcbaada62019-02-02 23:29:55 +0000456 .scalarize(0)
457 // Accept any address space as long as the size matches
458 .legalIf(sameSize(0, 1))
459 .widenScalarIf(smallerThan(0, 1),
460 [](const LegalityQuery &Query) {
461 return std::make_pair(0, LLT::scalar(Query.Types[1].getSizeInBits()));
462 })
463 .narrowScalarIf(
464 greaterThan(0, 1),
465 [](const LegalityQuery &Query) {
466 return std::make_pair(0, LLT::scalar(Query.Types[1].getSizeInBits()));
467 });
Matt Arsenaultf38f4832018-12-13 08:23:51 +0000468
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000469 if (ST.hasFlatAddressSpace()) {
470 getActionDefinitionsBuilder(G_ADDRSPACE_CAST)
471 .scalarize(0)
472 .custom();
473 }
474
Matt Arsenault85803362018-03-17 15:17:41 +0000475 getActionDefinitionsBuilder({G_LOAD, G_STORE})
Matt Arsenault18619af2019-01-29 18:13:02 +0000476 .narrowScalarIf([](const LegalityQuery &Query) {
477 unsigned Size = Query.Types[0].getSizeInBits();
478 unsigned MemSize = Query.MMODescrs[0].SizeInBits;
479 return (Size > 32 && MemSize < Size);
480 },
481 [](const LegalityQuery &Query) {
482 return std::make_pair(0, LLT::scalar(32));
483 })
Matt Arsenault9e8e8c62019-07-01 18:49:01 +0000484 .fewerElementsIf([=](const LegalityQuery &Query) {
Matt Arsenault045bc9a2019-01-30 02:35:38 +0000485 unsigned MemSize = Query.MMODescrs[0].SizeInBits;
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000486 return (MemSize == 96) &&
487 Query.Types[0].isVector() &&
Matt Arsenaulte4c2e9b2019-06-19 23:54:58 +0000488 !ST.hasDwordx3LoadStores();
Matt Arsenault045bc9a2019-01-30 02:35:38 +0000489 },
490 [=](const LegalityQuery &Query) {
491 return std::make_pair(0, V2S32);
492 })
Matt Arsenault9e8e8c62019-07-01 18:49:01 +0000493 .legalIf([=](const LegalityQuery &Query) {
Matt Arsenault85803362018-03-17 15:17:41 +0000494 const LLT &Ty0 = Query.Types[0];
495
Matt Arsenault18619af2019-01-29 18:13:02 +0000496 unsigned Size = Ty0.getSizeInBits();
497 unsigned MemSize = Query.MMODescrs[0].SizeInBits;
Matt Arsenaulteb2603c2019-02-02 23:39:13 +0000498 if (Size < 32 || (Size > 32 && MemSize < Size))
Matt Arsenault18619af2019-01-29 18:13:02 +0000499 return false;
500
501 if (Ty0.isVector() && Size != MemSize)
502 return false;
503
Matt Arsenault85803362018-03-17 15:17:41 +0000504 // TODO: Decompose private loads into 4-byte components.
505 // TODO: Illegal flat loads on SI
Matt Arsenault18619af2019-01-29 18:13:02 +0000506 switch (MemSize) {
507 case 8:
508 case 16:
509 return Size == 32;
Matt Arsenault85803362018-03-17 15:17:41 +0000510 case 32:
511 case 64:
512 case 128:
513 return true;
514
515 case 96:
Matt Arsenaulte4c2e9b2019-06-19 23:54:58 +0000516 return ST.hasDwordx3LoadStores();
Matt Arsenault85803362018-03-17 15:17:41 +0000517
518 case 256:
519 case 512:
520 // TODO: constant loads
521 default:
522 return false;
523 }
Matt Arsenault18619af2019-01-29 18:13:02 +0000524 })
525 .clampScalar(0, S32, S64);
Matt Arsenault85803362018-03-17 15:17:41 +0000526
527
Matt Arsenault530d05e2019-02-14 22:41:09 +0000528 // FIXME: Handle alignment requirements.
Matt Arsenault6614f852019-01-22 19:02:10 +0000529 auto &ExtLoads = getActionDefinitionsBuilder({G_SEXTLOAD, G_ZEXTLOAD})
Matt Arsenault530d05e2019-02-14 22:41:09 +0000530 .legalForTypesWithMemDesc({
531 {S32, GlobalPtr, 8, 8},
532 {S32, GlobalPtr, 16, 8},
533 {S32, LocalPtr, 8, 8},
534 {S32, LocalPtr, 16, 8},
535 {S32, PrivatePtr, 8, 8},
536 {S32, PrivatePtr, 16, 8}});
Matt Arsenault6614f852019-01-22 19:02:10 +0000537 if (ST.hasFlatAddressSpace()) {
Matt Arsenault530d05e2019-02-14 22:41:09 +0000538 ExtLoads.legalForTypesWithMemDesc({{S32, FlatPtr, 8, 8},
539 {S32, FlatPtr, 16, 8}});
Matt Arsenault6614f852019-01-22 19:02:10 +0000540 }
541
542 ExtLoads.clampScalar(0, S32, S32)
543 .widenScalarToNextPow2(0)
544 .unsupportedIfMemSizeNotPow2()
545 .lower();
546
Matt Arsenault36d40922018-12-20 00:33:49 +0000547 auto &Atomics = getActionDefinitionsBuilder(
548 {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB,
549 G_ATOMICRMW_AND, G_ATOMICRMW_OR, G_ATOMICRMW_XOR,
550 G_ATOMICRMW_MAX, G_ATOMICRMW_MIN, G_ATOMICRMW_UMAX,
551 G_ATOMICRMW_UMIN, G_ATOMIC_CMPXCHG})
552 .legalFor({{S32, GlobalPtr}, {S32, LocalPtr},
553 {S64, GlobalPtr}, {S64, LocalPtr}});
554 if (ST.hasFlatAddressSpace()) {
555 Atomics.legalFor({{S32, FlatPtr}, {S64, FlatPtr}});
556 }
Tom Stellardca166212017-01-30 21:56:46 +0000557
Matt Arsenault96e47012019-01-18 21:42:55 +0000558 // TODO: Pointer types, any 32-bit or 64-bit vector
559 getActionDefinitionsBuilder(G_SELECT)
Matt Arsenaultfdf36722019-07-01 15:42:47 +0000560 .legalForCartesianProduct({S32, S64, S16, V2S32, V2S16, V4S16,
Matt Arsenault10547232019-02-04 14:04:52 +0000561 GlobalPtr, LocalPtr, FlatPtr, PrivatePtr,
562 LLT::vector(2, LocalPtr), LLT::vector(2, PrivatePtr)}, {S1})
Matt Arsenaultfdf36722019-07-01 15:42:47 +0000563 .clampScalar(0, S16, S64)
Matt Arsenaultb4c95b32019-02-19 17:03:09 +0000564 .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
565 .fewerElementsIf(numElementsNotEven(0), scalarize(0))
Matt Arsenaultdc6c7852019-01-30 04:19:31 +0000566 .scalarize(1)
Matt Arsenault2491f822019-02-02 23:31:50 +0000567 .clampMaxNumElements(0, S32, 2)
568 .clampMaxNumElements(0, LocalPtr, 2)
569 .clampMaxNumElements(0, PrivatePtr, 2)
Matt Arsenaultb4c95b32019-02-19 17:03:09 +0000570 .scalarize(0)
Matt Arsenault4ed6cca2019-04-05 14:03:04 +0000571 .widenScalarToNextPow2(0)
Matt Arsenault2491f822019-02-02 23:31:50 +0000572 .legalIf(all(isPointer(0), typeIs(1, S1)));
Tom Stellard2860a422017-06-07 13:54:51 +0000573
Matt Arsenault4c5e8f512019-01-22 22:00:19 +0000574 // TODO: Only the low 4/5/6 bits of the shift amount are observed, so we can
575 // be more flexible with the shift amount type.
576 auto &Shifts = getActionDefinitionsBuilder({G_SHL, G_LSHR, G_ASHR})
577 .legalFor({{S32, S32}, {S64, S32}});
Matt Arsenaultf6cab162019-01-30 03:36:25 +0000578 if (ST.has16BitInsts()) {
Matt Arsenaultc83b8232019-02-07 17:38:00 +0000579 if (ST.hasVOP3PInsts()) {
580 Shifts.legalFor({{S16, S32}, {S16, S16}, {V2S16, V2S16}})
581 .clampMaxNumElements(0, S16, 2);
582 } else
583 Shifts.legalFor({{S16, S32}, {S16, S16}});
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +0000584
585 Shifts.clampScalar(1, S16, S32);
Matt Arsenaultf6cab162019-01-30 03:36:25 +0000586 Shifts.clampScalar(0, S16, S64);
Matt Arsenaultb0a22702019-02-08 15:06:24 +0000587 Shifts.widenScalarToNextPow2(0, 16);
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +0000588 } else {
589 // Make sure we legalize the shift amount type first, as the general
590 // expansion for the shifted type will produce much worse code if it hasn't
591 // been truncated already.
592 Shifts.clampScalar(1, S32, S32);
Matt Arsenault4c5e8f512019-01-22 22:00:19 +0000593 Shifts.clampScalar(0, S32, S64);
Matt Arsenaultb0a22702019-02-08 15:06:24 +0000594 Shifts.widenScalarToNextPow2(0, 32);
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +0000595 }
596 Shifts.scalarize(0);
Tom Stellardca166212017-01-30 21:56:46 +0000597
Matt Arsenault7b9ed892018-03-12 13:35:53 +0000598 for (unsigned Op : {G_EXTRACT_VECTOR_ELT, G_INSERT_VECTOR_ELT}) {
Matt Arsenault63786292019-01-22 20:38:15 +0000599 unsigned VecTypeIdx = Op == G_EXTRACT_VECTOR_ELT ? 1 : 0;
600 unsigned EltTypeIdx = Op == G_EXTRACT_VECTOR_ELT ? 0 : 1;
601 unsigned IdxTypeIdx = 2;
602
Matt Arsenault7b9ed892018-03-12 13:35:53 +0000603 getActionDefinitionsBuilder(Op)
604 .legalIf([=](const LegalityQuery &Query) {
Matt Arsenault63786292019-01-22 20:38:15 +0000605 const LLT &VecTy = Query.Types[VecTypeIdx];
606 const LLT &IdxTy = Query.Types[IdxTypeIdx];
Matt Arsenault7b9ed892018-03-12 13:35:53 +0000607 return VecTy.getSizeInBits() % 32 == 0 &&
608 VecTy.getSizeInBits() <= 512 &&
609 IdxTy.getSizeInBits() == 32;
Matt Arsenault63786292019-01-22 20:38:15 +0000610 })
611 .clampScalar(EltTypeIdx, S32, S64)
612 .clampScalar(VecTypeIdx, S32, S64)
613 .clampScalar(IdxTypeIdx, S32, S32);
Matt Arsenault7b9ed892018-03-12 13:35:53 +0000614 }
615
Matt Arsenault63786292019-01-22 20:38:15 +0000616 getActionDefinitionsBuilder(G_EXTRACT_VECTOR_ELT)
617 .unsupportedIf([=](const LegalityQuery &Query) {
618 const LLT &EltTy = Query.Types[1].getElementType();
619 return Query.Types[0] != EltTy;
620 });
621
Matt Arsenaultc4d07552019-02-20 16:11:22 +0000622 for (unsigned Op : {G_EXTRACT, G_INSERT}) {
623 unsigned BigTyIdx = Op == G_EXTRACT ? 1 : 0;
624 unsigned LitTyIdx = Op == G_EXTRACT ? 0 : 1;
625
626 // FIXME: Doesn't handle extract of illegal sizes.
627 getActionDefinitionsBuilder(Op)
Matt Arsenault91be65b2019-02-07 17:25:51 +0000628 .legalIf([=](const LegalityQuery &Query) {
Matt Arsenaultc4d07552019-02-20 16:11:22 +0000629 const LLT BigTy = Query.Types[BigTyIdx];
630 const LLT LitTy = Query.Types[LitTyIdx];
631 return (BigTy.getSizeInBits() % 32 == 0) &&
632 (LitTy.getSizeInBits() % 16 == 0);
633 })
Matt Arsenault91be65b2019-02-07 17:25:51 +0000634 .widenScalarIf(
Matt Arsenaultc4d07552019-02-20 16:11:22 +0000635 [=](const LegalityQuery &Query) {
636 const LLT BigTy = Query.Types[BigTyIdx];
637 return (BigTy.getScalarSizeInBits() < 16);
638 },
639 LegalizeMutations::widenScalarOrEltToNextPow2(BigTyIdx, 16))
640 .widenScalarIf(
641 [=](const LegalityQuery &Query) {
642 const LLT LitTy = Query.Types[LitTyIdx];
643 return (LitTy.getScalarSizeInBits() < 16);
644 },
645 LegalizeMutations::widenScalarOrEltToNextPow2(LitTyIdx, 16))
Matt Arsenault2b6f76f2019-04-22 15:22:46 +0000646 .moreElementsIf(isSmallOddVector(BigTyIdx), oneMoreElement(BigTyIdx))
647 .widenScalarToNextPow2(BigTyIdx, 32);
648
Matt Arsenaultc4d07552019-02-20 16:11:22 +0000649 }
Matt Arsenault71272e62018-03-05 16:25:15 +0000650
Amara Emerson5ec14602018-12-10 18:44:58 +0000651 getActionDefinitionsBuilder(G_BUILD_VECTOR)
Matt Arsenaultaebb2ee2019-01-22 20:14:29 +0000652 .legalForCartesianProduct(AllS32Vectors, {S32})
653 .legalForCartesianProduct(AllS64Vectors, {S64})
654 .clampNumElements(0, V16S32, V16S32)
655 .clampNumElements(0, V2S64, V8S64)
656 .minScalarSameAs(1, 0)
Matt Arsenault3f1a3452019-07-09 22:48:04 +0000657 .legalIf(isRegisterType(0))
658 .minScalarOrElt(0, S32);
Matt Arsenaultbee2ad72018-12-21 03:03:11 +0000659
Matt Arsenaulta1515d22019-01-08 01:30:02 +0000660 getActionDefinitionsBuilder(G_CONCAT_VECTORS)
Matt Arsenault4dd57552019-07-09 14:17:31 +0000661 .legalIf(isRegisterType(0));
Matt Arsenaulta1515d22019-01-08 01:30:02 +0000662
Matt Arsenault503afda2018-03-12 13:35:43 +0000663 // Merge/Unmerge
664 for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
665 unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
666 unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
667
Matt Arsenaultff6a9a22019-01-20 18:40:36 +0000668 auto notValidElt = [=](const LegalityQuery &Query, unsigned TypeIdx) {
669 const LLT &Ty = Query.Types[TypeIdx];
670 if (Ty.isVector()) {
671 const LLT &EltTy = Ty.getElementType();
672 if (EltTy.getSizeInBits() < 8 || EltTy.getSizeInBits() > 64)
673 return true;
674 if (!isPowerOf2_32(EltTy.getSizeInBits()))
675 return true;
676 }
677 return false;
678 };
679
Matt Arsenault503afda2018-03-12 13:35:43 +0000680 getActionDefinitionsBuilder(Op)
Matt Arsenaultd8d193d2019-01-29 23:17:35 +0000681 .widenScalarToNextPow2(LitTyIdx, /*Min*/ 16)
682 // Clamp the little scalar to s8-s256 and make it a power of 2. It's not
683 // worth considering the multiples of 64 since 2*192 and 2*384 are not
684 // valid.
685 .clampScalar(LitTyIdx, S16, S256)
686 .widenScalarToNextPow2(LitTyIdx, /*Min*/ 32)
687
Matt Arsenaultff6a9a22019-01-20 18:40:36 +0000688 // Break up vectors with weird elements into scalars
689 .fewerElementsIf(
690 [=](const LegalityQuery &Query) { return notValidElt(Query, 0); },
Matt Arsenault990f5072019-01-25 00:51:00 +0000691 scalarize(0))
Matt Arsenaultff6a9a22019-01-20 18:40:36 +0000692 .fewerElementsIf(
693 [=](const LegalityQuery &Query) { return notValidElt(Query, 1); },
Matt Arsenault990f5072019-01-25 00:51:00 +0000694 scalarize(1))
Matt Arsenaultff6a9a22019-01-20 18:40:36 +0000695 .clampScalar(BigTyIdx, S32, S512)
696 .widenScalarIf(
697 [=](const LegalityQuery &Query) {
698 const LLT &Ty = Query.Types[BigTyIdx];
699 return !isPowerOf2_32(Ty.getSizeInBits()) &&
700 Ty.getSizeInBits() % 16 != 0;
701 },
702 [=](const LegalityQuery &Query) {
703 // Pick the next power of 2, or a multiple of 64 over 128.
704 // Whichever is smaller.
705 const LLT &Ty = Query.Types[BigTyIdx];
706 unsigned NewSizeInBits = 1 << Log2_32_Ceil(Ty.getSizeInBits() + 1);
707 if (NewSizeInBits >= 256) {
708 unsigned RoundedTo = alignTo<64>(Ty.getSizeInBits() + 1);
709 if (RoundedTo < NewSizeInBits)
710 NewSizeInBits = RoundedTo;
711 }
712 return std::make_pair(BigTyIdx, LLT::scalar(NewSizeInBits));
713 })
Matt Arsenault503afda2018-03-12 13:35:43 +0000714 .legalIf([=](const LegalityQuery &Query) {
715 const LLT &BigTy = Query.Types[BigTyIdx];
716 const LLT &LitTy = Query.Types[LitTyIdx];
Matt Arsenaultff6a9a22019-01-20 18:40:36 +0000717
718 if (BigTy.isVector() && BigTy.getSizeInBits() < 32)
719 return false;
720 if (LitTy.isVector() && LitTy.getSizeInBits() < 32)
721 return false;
722
723 return BigTy.getSizeInBits() % 16 == 0 &&
724 LitTy.getSizeInBits() % 16 == 0 &&
Matt Arsenault503afda2018-03-12 13:35:43 +0000725 BigTy.getSizeInBits() <= 512;
726 })
727 // Any vectors left are the wrong size. Scalarize them.
Matt Arsenault990f5072019-01-25 00:51:00 +0000728 .scalarize(0)
729 .scalarize(1);
Matt Arsenault503afda2018-03-12 13:35:43 +0000730 }
731
Tom Stellardca166212017-01-30 21:56:46 +0000732 computeTables();
Roman Tereshin76c29c62018-05-31 16:16:48 +0000733 verify(*ST.getInstrInfo());
Tom Stellardca166212017-01-30 21:56:46 +0000734}
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000735
736bool AMDGPULegalizerInfo::legalizeCustom(MachineInstr &MI,
737 MachineRegisterInfo &MRI,
738 MachineIRBuilder &MIRBuilder,
739 GISelChangeObserver &Observer) const {
740 switch (MI.getOpcode()) {
741 case TargetOpcode::G_ADDRSPACE_CAST:
742 return legalizeAddrSpaceCast(MI, MRI, MIRBuilder);
Matt Arsenault6aafc5e2019-05-17 12:19:57 +0000743 case TargetOpcode::G_FRINT:
744 return legalizeFrint(MI, MRI, MIRBuilder);
Matt Arsenaulta510b572019-05-17 12:20:05 +0000745 case TargetOpcode::G_FCEIL:
746 return legalizeFceil(MI, MRI, MIRBuilder);
Matt Arsenault6aebcd52019-05-17 12:20:01 +0000747 case TargetOpcode::G_INTRINSIC_TRUNC:
748 return legalizeIntrinsicTrunc(MI, MRI, MIRBuilder);
Matt Arsenault2f292202019-05-17 23:05:18 +0000749 case TargetOpcode::G_SITOFP:
750 return legalizeITOFP(MI, MRI, MIRBuilder, true);
751 case TargetOpcode::G_UITOFP:
752 return legalizeITOFP(MI, MRI, MIRBuilder, false);
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000753 default:
754 return false;
755 }
756
757 llvm_unreachable("expected switch to return");
758}
759
Matt Arsenault1178dc32019-06-28 01:16:46 +0000760Register AMDGPULegalizerInfo::getSegmentAperture(
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000761 unsigned AS,
762 MachineRegisterInfo &MRI,
763 MachineIRBuilder &MIRBuilder) const {
764 MachineFunction &MF = MIRBuilder.getMF();
765 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
766 const LLT S32 = LLT::scalar(32);
767
768 if (ST.hasApertureRegs()) {
769 // FIXME: Use inline constants (src_{shared, private}_base) instead of
770 // getreg.
771 unsigned Offset = AS == AMDGPUAS::LOCAL_ADDRESS ?
772 AMDGPU::Hwreg::OFFSET_SRC_SHARED_BASE :
773 AMDGPU::Hwreg::OFFSET_SRC_PRIVATE_BASE;
774 unsigned WidthM1 = AS == AMDGPUAS::LOCAL_ADDRESS ?
775 AMDGPU::Hwreg::WIDTH_M1_SRC_SHARED_BASE :
776 AMDGPU::Hwreg::WIDTH_M1_SRC_PRIVATE_BASE;
777 unsigned Encoding =
778 AMDGPU::Hwreg::ID_MEM_BASES << AMDGPU::Hwreg::ID_SHIFT_ |
779 Offset << AMDGPU::Hwreg::OFFSET_SHIFT_ |
780 WidthM1 << AMDGPU::Hwreg::WIDTH_M1_SHIFT_;
781
Matt Arsenault1178dc32019-06-28 01:16:46 +0000782 Register ApertureReg = MRI.createGenericVirtualRegister(S32);
783 Register GetReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000784
785 MIRBuilder.buildInstr(AMDGPU::S_GETREG_B32)
786 .addDef(GetReg)
787 .addImm(Encoding);
788 MRI.setType(GetReg, S32);
789
Amara Emerson946b1242019-04-15 05:04:20 +0000790 auto ShiftAmt = MIRBuilder.buildConstant(S32, WidthM1 + 1);
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000791 MIRBuilder.buildInstr(TargetOpcode::G_SHL)
792 .addDef(ApertureReg)
793 .addUse(GetReg)
Amara Emerson946b1242019-04-15 05:04:20 +0000794 .addUse(ShiftAmt.getReg(0));
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000795
796 return ApertureReg;
797 }
798
Matt Arsenault1178dc32019-06-28 01:16:46 +0000799 Register QueuePtr = MRI.createGenericVirtualRegister(
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000800 LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
801
802 // FIXME: Placeholder until we can track the input registers.
803 MIRBuilder.buildConstant(QueuePtr, 0xdeadbeef);
804
805 // Offset into amd_queue_t for group_segment_aperture_base_hi /
806 // private_segment_aperture_base_hi.
807 uint32_t StructOffset = (AS == AMDGPUAS::LOCAL_ADDRESS) ? 0x40 : 0x44;
808
809 // FIXME: Don't use undef
810 Value *V = UndefValue::get(PointerType::get(
811 Type::getInt8Ty(MF.getFunction().getContext()),
812 AMDGPUAS::CONSTANT_ADDRESS));
813
814 MachinePointerInfo PtrInfo(V, StructOffset);
815 MachineMemOperand *MMO = MF.getMachineMemOperand(
816 PtrInfo,
817 MachineMemOperand::MOLoad |
818 MachineMemOperand::MODereferenceable |
819 MachineMemOperand::MOInvariant,
820 4,
821 MinAlign(64, StructOffset));
822
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000823 Register LoadResult = MRI.createGenericVirtualRegister(S32);
824 Register LoadAddr;
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000825
826 MIRBuilder.materializeGEP(LoadAddr, QueuePtr, LLT::scalar(64), StructOffset);
827 MIRBuilder.buildLoad(LoadResult, LoadAddr, *MMO);
828 return LoadResult;
829}
830
831bool AMDGPULegalizerInfo::legalizeAddrSpaceCast(
832 MachineInstr &MI, MachineRegisterInfo &MRI,
833 MachineIRBuilder &MIRBuilder) const {
834 MachineFunction &MF = MIRBuilder.getMF();
835
836 MIRBuilder.setInstr(MI);
837
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000838 Register Dst = MI.getOperand(0).getReg();
839 Register Src = MI.getOperand(1).getReg();
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000840
841 LLT DstTy = MRI.getType(Dst);
842 LLT SrcTy = MRI.getType(Src);
843 unsigned DestAS = DstTy.getAddressSpace();
844 unsigned SrcAS = SrcTy.getAddressSpace();
845
846 // TODO: Avoid reloading from the queue ptr for each cast, or at least each
847 // vector element.
848 assert(!DstTy.isVector());
849
850 const AMDGPUTargetMachine &TM
851 = static_cast<const AMDGPUTargetMachine &>(MF.getTarget());
852
853 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
854 if (ST.getTargetLowering()->isNoopAddrSpaceCast(SrcAS, DestAS)) {
Matt Arsenaultdc88a2c2019-02-08 14:16:11 +0000855 MI.setDesc(MIRBuilder.getTII().get(TargetOpcode::G_BITCAST));
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000856 return true;
857 }
858
859 if (SrcAS == AMDGPUAS::FLAT_ADDRESS) {
860 assert(DestAS == AMDGPUAS::LOCAL_ADDRESS ||
861 DestAS == AMDGPUAS::PRIVATE_ADDRESS);
862 unsigned NullVal = TM.getNullPointerValue(DestAS);
863
Amara Emerson946b1242019-04-15 05:04:20 +0000864 auto SegmentNull = MIRBuilder.buildConstant(DstTy, NullVal);
865 auto FlatNull = MIRBuilder.buildConstant(SrcTy, 0);
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000866
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000867 Register PtrLo32 = MRI.createGenericVirtualRegister(DstTy);
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000868
869 // Extract low 32-bits of the pointer.
870 MIRBuilder.buildExtract(PtrLo32, Src, 0);
871
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000872 Register CmpRes = MRI.createGenericVirtualRegister(LLT::scalar(1));
Amara Emerson946b1242019-04-15 05:04:20 +0000873 MIRBuilder.buildICmp(CmpInst::ICMP_NE, CmpRes, Src, FlatNull.getReg(0));
874 MIRBuilder.buildSelect(Dst, CmpRes, PtrLo32, SegmentNull.getReg(0));
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000875
876 MI.eraseFromParent();
877 return true;
878 }
879
880 assert(SrcAS == AMDGPUAS::LOCAL_ADDRESS ||
881 SrcAS == AMDGPUAS::PRIVATE_ADDRESS);
882
Amara Emerson946b1242019-04-15 05:04:20 +0000883 auto SegmentNull =
884 MIRBuilder.buildConstant(SrcTy, TM.getNullPointerValue(SrcAS));
885 auto FlatNull =
886 MIRBuilder.buildConstant(DstTy, TM.getNullPointerValue(DestAS));
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000887
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000888 Register ApertureReg = getSegmentAperture(DestAS, MRI, MIRBuilder);
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000889
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000890 Register CmpRes = MRI.createGenericVirtualRegister(LLT::scalar(1));
Amara Emerson946b1242019-04-15 05:04:20 +0000891 MIRBuilder.buildICmp(CmpInst::ICMP_NE, CmpRes, Src, SegmentNull.getReg(0));
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000892
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000893 Register BuildPtr = MRI.createGenericVirtualRegister(DstTy);
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000894
895 // Coerce the type of the low half of the result so we can use merge_values.
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000896 Register SrcAsInt = MRI.createGenericVirtualRegister(LLT::scalar(32));
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000897 MIRBuilder.buildInstr(TargetOpcode::G_PTRTOINT)
898 .addDef(SrcAsInt)
899 .addUse(Src);
900
901 // TODO: Should we allow mismatched types but matching sizes in merges to
902 // avoid the ptrtoint?
903 MIRBuilder.buildMerge(BuildPtr, {SrcAsInt, ApertureReg});
Amara Emerson946b1242019-04-15 05:04:20 +0000904 MIRBuilder.buildSelect(Dst, CmpRes, BuildPtr, FlatNull.getReg(0));
Matt Arsenaulta8b43392019-02-08 02:40:47 +0000905
906 MI.eraseFromParent();
907 return true;
908}
Matt Arsenault6aafc5e2019-05-17 12:19:57 +0000909
910bool AMDGPULegalizerInfo::legalizeFrint(
911 MachineInstr &MI, MachineRegisterInfo &MRI,
912 MachineIRBuilder &MIRBuilder) const {
913 MIRBuilder.setInstr(MI);
914
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000915 Register Src = MI.getOperand(1).getReg();
Matt Arsenault6aafc5e2019-05-17 12:19:57 +0000916 LLT Ty = MRI.getType(Src);
917 assert(Ty.isScalar() && Ty.getSizeInBits() == 64);
918
919 APFloat C1Val(APFloat::IEEEdouble(), "0x1.0p+52");
920 APFloat C2Val(APFloat::IEEEdouble(), "0x1.fffffffffffffp+51");
921
922 auto C1 = MIRBuilder.buildFConstant(Ty, C1Val);
923 auto CopySign = MIRBuilder.buildFCopysign(Ty, C1, Src);
924
925 // TODO: Should this propagate fast-math-flags?
926 auto Tmp1 = MIRBuilder.buildFAdd(Ty, Src, CopySign);
927 auto Tmp2 = MIRBuilder.buildFSub(Ty, Tmp1, CopySign);
928
929 auto C2 = MIRBuilder.buildFConstant(Ty, C2Val);
930 auto Fabs = MIRBuilder.buildFAbs(Ty, Src);
931
932 auto Cond = MIRBuilder.buildFCmp(CmpInst::FCMP_OGT, LLT::scalar(1), Fabs, C2);
933 MIRBuilder.buildSelect(MI.getOperand(0).getReg(), Cond, Src, Tmp2);
934 return true;
935}
Matt Arsenault6aebcd52019-05-17 12:20:01 +0000936
Matt Arsenaulta510b572019-05-17 12:20:05 +0000937bool AMDGPULegalizerInfo::legalizeFceil(
938 MachineInstr &MI, MachineRegisterInfo &MRI,
939 MachineIRBuilder &B) const {
940 B.setInstr(MI);
941
Matt Arsenault1a02d302019-05-17 12:59:27 +0000942 const LLT S1 = LLT::scalar(1);
943 const LLT S64 = LLT::scalar(64);
944
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000945 Register Src = MI.getOperand(1).getReg();
Matt Arsenault1a02d302019-05-17 12:59:27 +0000946 assert(MRI.getType(Src) == S64);
Matt Arsenaulta510b572019-05-17 12:20:05 +0000947
948 // result = trunc(src)
949 // if (src > 0.0 && src != result)
950 // result += 1.0
951
Matt Arsenaulta510b572019-05-17 12:20:05 +0000952 auto Trunc = B.buildInstr(TargetOpcode::G_INTRINSIC_TRUNC, {S64}, {Src});
953
Matt Arsenaulta510b572019-05-17 12:20:05 +0000954 const auto Zero = B.buildFConstant(S64, 0.0);
955 const auto One = B.buildFConstant(S64, 1.0);
956 auto Lt0 = B.buildFCmp(CmpInst::FCMP_OGT, S1, Src, Zero);
957 auto NeTrunc = B.buildFCmp(CmpInst::FCMP_ONE, S1, Src, Trunc);
958 auto And = B.buildAnd(S1, Lt0, NeTrunc);
959 auto Add = B.buildSelect(S64, And, One, Zero);
960
961 // TODO: Should this propagate fast-math-flags?
962 B.buildFAdd(MI.getOperand(0).getReg(), Trunc, Add);
963 return true;
964}
965
Matt Arsenault6aebcd52019-05-17 12:20:01 +0000966static MachineInstrBuilder extractF64Exponent(unsigned Hi,
967 MachineIRBuilder &B) {
968 const unsigned FractBits = 52;
969 const unsigned ExpBits = 11;
970 LLT S32 = LLT::scalar(32);
971
972 auto Const0 = B.buildConstant(S32, FractBits - 32);
973 auto Const1 = B.buildConstant(S32, ExpBits);
974
975 auto ExpPart = B.buildIntrinsic(Intrinsic::amdgcn_ubfe, {S32}, false)
976 .addUse(Const0.getReg(0))
977 .addUse(Const1.getReg(0));
978
979 return B.buildSub(S32, ExpPart, B.buildConstant(S32, 1023));
980}
981
982bool AMDGPULegalizerInfo::legalizeIntrinsicTrunc(
983 MachineInstr &MI, MachineRegisterInfo &MRI,
984 MachineIRBuilder &B) const {
985 B.setInstr(MI);
986
Matt Arsenault1a02d302019-05-17 12:59:27 +0000987 const LLT S1 = LLT::scalar(1);
988 const LLT S32 = LLT::scalar(32);
989 const LLT S64 = LLT::scalar(64);
Matt Arsenault6aebcd52019-05-17 12:20:01 +0000990
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000991 Register Src = MI.getOperand(1).getReg();
Matt Arsenault1a02d302019-05-17 12:59:27 +0000992 assert(MRI.getType(Src) == S64);
Matt Arsenault6aebcd52019-05-17 12:20:01 +0000993
994 // TODO: Should this use extract since the low half is unused?
995 auto Unmerge = B.buildUnmerge({S32, S32}, Src);
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000996 Register Hi = Unmerge.getReg(1);
Matt Arsenault6aebcd52019-05-17 12:20:01 +0000997
998 // Extract the upper half, since this is where we will find the sign and
999 // exponent.
1000 auto Exp = extractF64Exponent(Hi, B);
1001
1002 const unsigned FractBits = 52;
1003
1004 // Extract the sign bit.
1005 const auto SignBitMask = B.buildConstant(S32, UINT32_C(1) << 31);
1006 auto SignBit = B.buildAnd(S32, Hi, SignBitMask);
1007
1008 const auto FractMask = B.buildConstant(S64, (UINT64_C(1) << FractBits) - 1);
1009
1010 const auto Zero32 = B.buildConstant(S32, 0);
1011
1012 // Extend back to 64-bits.
1013 auto SignBit64 = B.buildMerge(S64, {Zero32.getReg(0), SignBit.getReg(0)});
1014
1015 auto Shr = B.buildAShr(S64, FractMask, Exp);
1016 auto Not = B.buildNot(S64, Shr);
1017 auto Tmp0 = B.buildAnd(S64, Src, Not);
1018 auto FiftyOne = B.buildConstant(S32, FractBits - 1);
1019
1020 auto ExpLt0 = B.buildICmp(CmpInst::ICMP_SLT, S1, Exp, Zero32);
1021 auto ExpGt51 = B.buildICmp(CmpInst::ICMP_SGT, S1, Exp, FiftyOne);
1022
1023 auto Tmp1 = B.buildSelect(S64, ExpLt0, SignBit64, Tmp0);
1024 B.buildSelect(MI.getOperand(0).getReg(), ExpGt51, Src, Tmp1);
1025 return true;
1026}
Matt Arsenault2f292202019-05-17 23:05:18 +00001027
1028bool AMDGPULegalizerInfo::legalizeITOFP(
1029 MachineInstr &MI, MachineRegisterInfo &MRI,
1030 MachineIRBuilder &B, bool Signed) const {
1031 B.setInstr(MI);
1032
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001033 Register Dst = MI.getOperand(0).getReg();
1034 Register Src = MI.getOperand(1).getReg();
Matt Arsenault2f292202019-05-17 23:05:18 +00001035
1036 const LLT S64 = LLT::scalar(64);
1037 const LLT S32 = LLT::scalar(32);
1038
1039 assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S64);
1040
1041 auto Unmerge = B.buildUnmerge({S32, S32}, Src);
1042
1043 auto CvtHi = Signed ?
1044 B.buildSITOFP(S64, Unmerge.getReg(1)) :
1045 B.buildUITOFP(S64, Unmerge.getReg(1));
1046
1047 auto CvtLo = B.buildUITOFP(S64, Unmerge.getReg(0));
1048
1049 auto ThirtyTwo = B.buildConstant(S32, 32);
1050 auto LdExp = B.buildIntrinsic(Intrinsic::amdgcn_ldexp, {S64}, false)
1051 .addUse(CvtHi.getReg(0))
1052 .addUse(ThirtyTwo.getReg(0));
1053
1054 // TODO: Should this propagate fast-math-flags?
1055 B.buildFAdd(Dst, LdExp, CvtLo);
1056 MI.eraseFromParent();
1057 return true;
1058}
Matt Arsenaulte15770a2019-07-01 18:40:23 +00001059
1060// Return the use branch instruction, otherwise null if the usage is invalid.
1061static MachineInstr *verifyCFIntrinsic(MachineInstr &MI,
1062 MachineRegisterInfo &MRI) {
1063 Register CondDef = MI.getOperand(0).getReg();
1064 if (!MRI.hasOneNonDBGUse(CondDef))
1065 return nullptr;
1066
1067 MachineInstr &UseMI = *MRI.use_instr_nodbg_begin(CondDef);
1068 return UseMI.getParent() == MI.getParent() &&
1069 UseMI.getOpcode() == AMDGPU::G_BRCOND ? &UseMI : nullptr;
1070}
1071
Matt Arsenaulte2c86cc2019-07-01 18:45:36 +00001072Register AMDGPULegalizerInfo::getLiveInRegister(MachineRegisterInfo &MRI,
1073 Register Reg, LLT Ty) const {
1074 Register LiveIn = MRI.getLiveInVirtReg(Reg);
1075 if (LiveIn)
1076 return LiveIn;
1077
1078 Register NewReg = MRI.createGenericVirtualRegister(Ty);
1079 MRI.addLiveIn(Reg, NewReg);
1080 return NewReg;
1081}
1082
1083bool AMDGPULegalizerInfo::loadInputValue(Register DstReg, MachineIRBuilder &B,
1084 const ArgDescriptor *Arg) const {
1085 if (!Arg->isRegister())
1086 return false; // TODO: Handle these
1087
1088 assert(Arg->getRegister() != 0);
1089 assert(Arg->getRegister().isPhysical());
1090
1091 MachineRegisterInfo &MRI = *B.getMRI();
1092
1093 LLT Ty = MRI.getType(DstReg);
1094 Register LiveIn = getLiveInRegister(MRI, Arg->getRegister(), Ty);
1095
1096 if (Arg->isMasked()) {
1097 // TODO: Should we try to emit this once in the entry block?
1098 const LLT S32 = LLT::scalar(32);
1099 const unsigned Mask = Arg->getMask();
1100 const unsigned Shift = countTrailingZeros<unsigned>(Mask);
1101
1102 auto ShiftAmt = B.buildConstant(S32, Shift);
1103 auto LShr = B.buildLShr(S32, LiveIn, ShiftAmt);
1104 B.buildAnd(DstReg, LShr, B.buildConstant(S32, Mask >> Shift));
1105 } else
1106 B.buildCopy(DstReg, LiveIn);
1107
1108 // Insert the argument copy if it doens't already exist.
1109 // FIXME: It seems EmitLiveInCopies isn't called anywhere?
1110 if (!MRI.getVRegDef(LiveIn)) {
1111 MachineBasicBlock &EntryMBB = B.getMF().front();
1112 EntryMBB.addLiveIn(Arg->getRegister());
1113 B.setInsertPt(EntryMBB, EntryMBB.begin());
1114 B.buildCopy(LiveIn, Arg->getRegister());
1115 }
1116
1117 return true;
1118}
1119
1120bool AMDGPULegalizerInfo::legalizePreloadedArgIntrin(
1121 MachineInstr &MI,
1122 MachineRegisterInfo &MRI,
1123 MachineIRBuilder &B,
1124 AMDGPUFunctionArgInfo::PreloadedValue ArgType) const {
1125 B.setInstr(MI);
1126
1127 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
1128
1129 const ArgDescriptor *Arg;
1130 const TargetRegisterClass *RC;
1131 std::tie(Arg, RC) = MFI->getPreloadedValue(ArgType);
1132 if (!Arg) {
1133 LLVM_DEBUG(dbgs() << "Required arg register missing\n");
1134 return false;
1135 }
1136
1137 if (loadInputValue(MI.getOperand(0).getReg(), B, Arg)) {
1138 MI.eraseFromParent();
1139 return true;
1140 }
1141
1142 return false;
1143}
1144
Matt Arsenault9e8e8c62019-07-01 18:49:01 +00001145bool AMDGPULegalizerInfo::legalizeImplicitArgPtr(MachineInstr &MI,
1146 MachineRegisterInfo &MRI,
1147 MachineIRBuilder &B) const {
1148 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
1149 if (!MFI->isEntryFunction()) {
1150 return legalizePreloadedArgIntrin(MI, MRI, B,
1151 AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR);
1152 }
1153
1154 B.setInstr(MI);
1155
1156 uint64_t Offset =
1157 ST.getTargetLowering()->getImplicitParameterOffset(
1158 B.getMF(), AMDGPUTargetLowering::FIRST_IMPLICIT);
1159 Register DstReg = MI.getOperand(0).getReg();
1160 LLT DstTy = MRI.getType(DstReg);
1161 LLT IdxTy = LLT::scalar(DstTy.getSizeInBits());
1162
1163 const ArgDescriptor *Arg;
1164 const TargetRegisterClass *RC;
1165 std::tie(Arg, RC)
1166 = MFI->getPreloadedValue(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
1167 if (!Arg)
1168 return false;
1169
1170 Register KernargPtrReg = MRI.createGenericVirtualRegister(DstTy);
1171 if (!loadInputValue(KernargPtrReg, B, Arg))
1172 return false;
1173
1174 B.buildGEP(DstReg, KernargPtrReg, B.buildConstant(IdxTy, Offset).getReg(0));
1175 MI.eraseFromParent();
1176 return true;
1177}
1178
Matt Arsenaulte15770a2019-07-01 18:40:23 +00001179bool AMDGPULegalizerInfo::legalizeIntrinsic(MachineInstr &MI,
1180 MachineRegisterInfo &MRI,
1181 MachineIRBuilder &B) const {
1182 // Replace the use G_BRCOND with the exec manipulate and branch pseudos.
1183 switch (MI.getOperand(MI.getNumExplicitDefs()).getIntrinsicID()) {
1184 case Intrinsic::amdgcn_if: {
1185 if (MachineInstr *BrCond = verifyCFIntrinsic(MI, MRI)) {
1186 const SIRegisterInfo *TRI
1187 = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
1188
1189 B.setInstr(*BrCond);
1190 Register Def = MI.getOperand(1).getReg();
1191 Register Use = MI.getOperand(3).getReg();
1192 B.buildInstr(AMDGPU::SI_IF)
1193 .addDef(Def)
1194 .addUse(Use)
1195 .addMBB(BrCond->getOperand(1).getMBB());
1196
1197 MRI.setRegClass(Def, TRI->getWaveMaskRegClass());
1198 MRI.setRegClass(Use, TRI->getWaveMaskRegClass());
1199 MI.eraseFromParent();
1200 BrCond->eraseFromParent();
1201 return true;
1202 }
1203
1204 return false;
1205 }
1206 case Intrinsic::amdgcn_loop: {
1207 if (MachineInstr *BrCond = verifyCFIntrinsic(MI, MRI)) {
1208 const SIRegisterInfo *TRI
1209 = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
1210
1211 B.setInstr(*BrCond);
1212 Register Reg = MI.getOperand(2).getReg();
1213 B.buildInstr(AMDGPU::SI_LOOP)
1214 .addUse(Reg)
1215 .addMBB(BrCond->getOperand(1).getMBB());
1216 MI.eraseFromParent();
1217 BrCond->eraseFromParent();
1218 MRI.setRegClass(Reg, TRI->getWaveMaskRegClass());
1219 return true;
1220 }
1221
1222 return false;
1223 }
Matt Arsenault9e8e8c62019-07-01 18:49:01 +00001224 case Intrinsic::amdgcn_kernarg_segment_ptr:
1225 return legalizePreloadedArgIntrin(
1226 MI, MRI, B, AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
1227 case Intrinsic::amdgcn_implicitarg_ptr:
1228 return legalizeImplicitArgPtr(MI, MRI, B);
Matt Arsenaulte2c86cc2019-07-01 18:45:36 +00001229 case Intrinsic::amdgcn_workitem_id_x:
1230 return legalizePreloadedArgIntrin(MI, MRI, B,
1231 AMDGPUFunctionArgInfo::WORKITEM_ID_X);
1232 case Intrinsic::amdgcn_workitem_id_y:
1233 return legalizePreloadedArgIntrin(MI, MRI, B,
1234 AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
1235 case Intrinsic::amdgcn_workitem_id_z:
1236 return legalizePreloadedArgIntrin(MI, MRI, B,
1237 AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
Matt Arsenault756d8192019-07-01 18:47:22 +00001238 case Intrinsic::amdgcn_workgroup_id_x:
1239 return legalizePreloadedArgIntrin(MI, MRI, B,
1240 AMDGPUFunctionArgInfo::WORKGROUP_ID_X);
1241 case Intrinsic::amdgcn_workgroup_id_y:
1242 return legalizePreloadedArgIntrin(MI, MRI, B,
1243 AMDGPUFunctionArgInfo::WORKGROUP_ID_Y);
1244 case Intrinsic::amdgcn_workgroup_id_z:
1245 return legalizePreloadedArgIntrin(MI, MRI, B,
1246 AMDGPUFunctionArgInfo::WORKGROUP_ID_Z);
Matt Arsenaultbae36362019-07-01 18:50:50 +00001247 case Intrinsic::amdgcn_dispatch_ptr:
1248 return legalizePreloadedArgIntrin(MI, MRI, B,
1249 AMDGPUFunctionArgInfo::DISPATCH_PTR);
1250 case Intrinsic::amdgcn_queue_ptr:
1251 return legalizePreloadedArgIntrin(MI, MRI, B,
1252 AMDGPUFunctionArgInfo::QUEUE_PTR);
1253 case Intrinsic::amdgcn_implicit_buffer_ptr:
1254 return legalizePreloadedArgIntrin(
1255 MI, MRI, B, AMDGPUFunctionArgInfo::IMPLICIT_BUFFER_PTR);
1256 case Intrinsic::amdgcn_dispatch_id:
1257 return legalizePreloadedArgIntrin(MI, MRI, B,
1258 AMDGPUFunctionArgInfo::DISPATCH_ID);
Matt Arsenaulte15770a2019-07-01 18:40:23 +00001259 default:
1260 return true;
1261 }
1262
1263 return true;
1264}