blob: b57cc00a71ffacd8ce30000de41cbf035337c07f [file] [log] [blame]
Tom Stellard75aadc22012-12-11 21:25:42 +00001//===-- AMDGPUISelLowering.cpp - AMDGPU Common DAG lowering functions -----===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10/// \file
11/// \brief This is the parent TargetLowering class for hardware code gen
12/// targets.
13//
14//===----------------------------------------------------------------------===//
15
16#include "AMDGPUISelLowering.h"
Tom Stellarded882c22013-06-03 17:40:11 +000017#include "AMDGPU.h"
Tom Stellardca166212017-01-30 21:56:46 +000018#include "AMDGPUCallLowering.h"
Tom Stellard81d871d2013-11-13 23:36:50 +000019#include "AMDGPUFrameLowering.h"
Matt Arsenaultc791f392014-06-23 18:00:31 +000020#include "AMDGPUIntrinsicInfo.h"
Christian Konig2c8f6d52013-03-07 09:03:52 +000021#include "AMDGPURegisterInfo.h"
Christian Konig2c8f6d52013-03-07 09:03:52 +000022#include "AMDGPUSubtarget.h"
Tom Stellardacfeebf2013-07-23 01:48:05 +000023#include "R600MachineFunctionInfo.h"
Tom Stellarded882c22013-06-03 17:40:11 +000024#include "SIMachineFunctionInfo.h"
Christian Konig2c8f6d52013-03-07 09:03:52 +000025#include "llvm/CodeGen/CallingConvLower.h"
Tom Stellard75aadc22012-12-11 21:25:42 +000026#include "llvm/CodeGen/MachineFunction.h"
27#include "llvm/CodeGen/MachineRegisterInfo.h"
28#include "llvm/CodeGen/SelectionDAG.h"
29#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
Tom Stellardc026e8b2013-06-28 15:47:08 +000030#include "llvm/IR/DataLayout.h"
Oliver Stannard7e7d9832016-02-02 13:52:43 +000031#include "llvm/IR/DiagnosticInfo.h"
Matt Arsenault6e3a4512016-01-18 22:01:13 +000032#include "SIInstrInfo.h"
Tom Stellard75aadc22012-12-11 21:25:42 +000033using namespace llvm;
Matt Arsenault16353872014-04-22 16:42:00 +000034
Matt Arsenaulte935f052016-06-18 05:15:53 +000035static bool allocateKernArg(unsigned ValNo, MVT ValVT, MVT LocVT,
36 CCValAssign::LocInfo LocInfo,
37 ISD::ArgFlagsTy ArgFlags, CCState &State) {
38 MachineFunction &MF = State.getMachineFunction();
39 AMDGPUMachineFunction *MFI = MF.getInfo<AMDGPUMachineFunction>();
Tom Stellardaf775432013-10-23 00:44:32 +000040
Tom Stellardbbeb45a2016-09-16 21:53:00 +000041 uint64_t Offset = MFI->allocateKernArg(LocVT.getStoreSize(),
Matt Arsenaulte935f052016-06-18 05:15:53 +000042 ArgFlags.getOrigAlign());
43 State.addLoc(CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT, LocInfo));
Tom Stellardaf775432013-10-23 00:44:32 +000044 return true;
45}
Tom Stellard75aadc22012-12-11 21:25:42 +000046
Christian Konig2c8f6d52013-03-07 09:03:52 +000047#include "AMDGPUGenCallingConv.inc"
48
Matt Arsenaultc9df7942014-06-11 03:29:54 +000049// Find a larger type to do a load / store of a vector with.
50EVT AMDGPUTargetLowering::getEquivalentMemType(LLVMContext &Ctx, EVT VT) {
51 unsigned StoreSize = VT.getStoreSizeInBits();
52 if (StoreSize <= 32)
53 return EVT::getIntegerVT(Ctx, StoreSize);
54
55 assert(StoreSize % 32 == 0 && "Store size not a multiple of 32");
56 return EVT::getVectorVT(Ctx, MVT::i32, StoreSize / 32);
57}
58
Matt Arsenault43e92fe2016-06-24 06:30:11 +000059AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
Eric Christopher7792e322015-01-30 23:24:40 +000060 const AMDGPUSubtarget &STI)
61 : TargetLowering(TM), Subtarget(&STI) {
Tom Stellard75aadc22012-12-11 21:25:42 +000062 // Lower floating point store/load to integer store/load to reduce the number
63 // of patterns in tablegen.
Tom Stellard75aadc22012-12-11 21:25:42 +000064 setOperationAction(ISD::LOAD, MVT::f32, Promote);
65 AddPromotedToType(ISD::LOAD, MVT::f32, MVT::i32);
66
Tom Stellardadf732c2013-07-18 21:43:48 +000067 setOperationAction(ISD::LOAD, MVT::v2f32, Promote);
68 AddPromotedToType(ISD::LOAD, MVT::v2f32, MVT::v2i32);
69
Tom Stellard75aadc22012-12-11 21:25:42 +000070 setOperationAction(ISD::LOAD, MVT::v4f32, Promote);
71 AddPromotedToType(ISD::LOAD, MVT::v4f32, MVT::v4i32);
72
Tom Stellardaf775432013-10-23 00:44:32 +000073 setOperationAction(ISD::LOAD, MVT::v8f32, Promote);
74 AddPromotedToType(ISD::LOAD, MVT::v8f32, MVT::v8i32);
75
76 setOperationAction(ISD::LOAD, MVT::v16f32, Promote);
77 AddPromotedToType(ISD::LOAD, MVT::v16f32, MVT::v16i32);
78
Matt Arsenault71e66762016-05-21 02:27:49 +000079 setOperationAction(ISD::LOAD, MVT::i64, Promote);
80 AddPromotedToType(ISD::LOAD, MVT::i64, MVT::v2i32);
81
82 setOperationAction(ISD::LOAD, MVT::v2i64, Promote);
83 AddPromotedToType(ISD::LOAD, MVT::v2i64, MVT::v4i32);
84
Tom Stellard7512c082013-07-12 18:14:56 +000085 setOperationAction(ISD::LOAD, MVT::f64, Promote);
Matt Arsenault71e66762016-05-21 02:27:49 +000086 AddPromotedToType(ISD::LOAD, MVT::f64, MVT::v2i32);
Tom Stellard7512c082013-07-12 18:14:56 +000087
Matt Arsenaulte8a076a2014-05-08 18:01:56 +000088 setOperationAction(ISD::LOAD, MVT::v2f64, Promote);
Matt Arsenault71e66762016-05-21 02:27:49 +000089 AddPromotedToType(ISD::LOAD, MVT::v2f64, MVT::v4i32);
Tom Stellard0344cdf2013-08-01 15:23:42 +000090
Matt Arsenaultbd223422015-01-14 01:35:17 +000091 // There are no 64-bit extloads. These should be done as a 32-bit extload and
92 // an extension to 64-bit.
93 for (MVT VT : MVT::integer_valuetypes()) {
94 setLoadExtAction(ISD::EXTLOAD, MVT::i64, VT, Expand);
95 setLoadExtAction(ISD::SEXTLOAD, MVT::i64, VT, Expand);
96 setLoadExtAction(ISD::ZEXTLOAD, MVT::i64, VT, Expand);
97 }
98
Matt Arsenault71e66762016-05-21 02:27:49 +000099 for (MVT VT : MVT::integer_valuetypes()) {
100 if (VT == MVT::i64)
101 continue;
102
103 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
104 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i8, Legal);
105 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i16, Legal);
106 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i32, Expand);
107
108 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote);
109 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i8, Legal);
110 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i16, Legal);
111 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i32, Expand);
112
113 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i1, Promote);
114 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i8, Legal);
115 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i16, Legal);
116 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i32, Expand);
117 }
118
Ahmed Bougacha2b6917b2015-01-08 00:51:32 +0000119 for (MVT VT : MVT::integer_vector_valuetypes()) {
120 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i8, Expand);
121 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i8, Expand);
122 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v2i8, Expand);
123 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i8, Expand);
124 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i8, Expand);
125 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v4i8, Expand);
126 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i16, Expand);
127 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i16, Expand);
128 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v2i16, Expand);
129 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i16, Expand);
130 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i16, Expand);
131 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v4i16, Expand);
132 }
Tom Stellardb03edec2013-08-16 01:12:16 +0000133
Matt Arsenault71e66762016-05-21 02:27:49 +0000134 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
135 setLoadExtAction(ISD::EXTLOAD, MVT::v2f32, MVT::v2f16, Expand);
136 setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Expand);
137 setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, MVT::v8f16, Expand);
138
139 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
140 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f32, Expand);
141 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f32, Expand);
142 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f32, Expand);
143
144 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
145 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Expand);
146 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Expand);
147 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f16, Expand);
148
149 setOperationAction(ISD::STORE, MVT::f32, Promote);
150 AddPromotedToType(ISD::STORE, MVT::f32, MVT::i32);
151
152 setOperationAction(ISD::STORE, MVT::v2f32, Promote);
153 AddPromotedToType(ISD::STORE, MVT::v2f32, MVT::v2i32);
154
155 setOperationAction(ISD::STORE, MVT::v4f32, Promote);
156 AddPromotedToType(ISD::STORE, MVT::v4f32, MVT::v4i32);
157
158 setOperationAction(ISD::STORE, MVT::v8f32, Promote);
159 AddPromotedToType(ISD::STORE, MVT::v8f32, MVT::v8i32);
160
161 setOperationAction(ISD::STORE, MVT::v16f32, Promote);
162 AddPromotedToType(ISD::STORE, MVT::v16f32, MVT::v16i32);
163
164 setOperationAction(ISD::STORE, MVT::i64, Promote);
165 AddPromotedToType(ISD::STORE, MVT::i64, MVT::v2i32);
166
167 setOperationAction(ISD::STORE, MVT::v2i64, Promote);
168 AddPromotedToType(ISD::STORE, MVT::v2i64, MVT::v4i32);
169
170 setOperationAction(ISD::STORE, MVT::f64, Promote);
171 AddPromotedToType(ISD::STORE, MVT::f64, MVT::v2i32);
172
173 setOperationAction(ISD::STORE, MVT::v2f64, Promote);
174 AddPromotedToType(ISD::STORE, MVT::v2f64, MVT::v4i32);
175
Matt Arsenault71e66762016-05-21 02:27:49 +0000176 setTruncStoreAction(MVT::i64, MVT::i1, Expand);
177 setTruncStoreAction(MVT::i64, MVT::i8, Expand);
178 setTruncStoreAction(MVT::i64, MVT::i16, Expand);
179 setTruncStoreAction(MVT::i64, MVT::i32, Expand);
180
181 setTruncStoreAction(MVT::v2i64, MVT::v2i1, Expand);
182 setTruncStoreAction(MVT::v2i64, MVT::v2i8, Expand);
183 setTruncStoreAction(MVT::v2i64, MVT::v2i16, Expand);
184 setTruncStoreAction(MVT::v2i64, MVT::v2i32, Expand);
185
186 setTruncStoreAction(MVT::f32, MVT::f16, Expand);
187 setTruncStoreAction(MVT::v2f32, MVT::v2f16, Expand);
188 setTruncStoreAction(MVT::v4f32, MVT::v4f16, Expand);
189 setTruncStoreAction(MVT::v8f32, MVT::v8f16, Expand);
190
191 setTruncStoreAction(MVT::f64, MVT::f16, Expand);
192 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
193
194 setTruncStoreAction(MVT::v2f64, MVT::v2f32, Expand);
195 setTruncStoreAction(MVT::v2f64, MVT::v2f16, Expand);
196
197 setTruncStoreAction(MVT::v4f64, MVT::v4f32, Expand);
198 setTruncStoreAction(MVT::v4f64, MVT::v4f16, Expand);
199
200 setTruncStoreAction(MVT::v8f64, MVT::v8f32, Expand);
201 setTruncStoreAction(MVT::v8f64, MVT::v8f16, Expand);
202
203
204 setOperationAction(ISD::Constant, MVT::i32, Legal);
205 setOperationAction(ISD::Constant, MVT::i64, Legal);
206 setOperationAction(ISD::ConstantFP, MVT::f32, Legal);
207 setOperationAction(ISD::ConstantFP, MVT::f64, Legal);
208
209 setOperationAction(ISD::BR_JT, MVT::Other, Expand);
210 setOperationAction(ISD::BRIND, MVT::Other, Expand);
211
212 // This is totally unsupported, just custom lower to produce an error.
213 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom);
214
215 // We need to custom lower some of the intrinsics
216 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
217 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
218
219 // Library functions. These default to Expand, but we have instructions
220 // for them.
221 setOperationAction(ISD::FCEIL, MVT::f32, Legal);
222 setOperationAction(ISD::FEXP2, MVT::f32, Legal);
223 setOperationAction(ISD::FPOW, MVT::f32, Legal);
224 setOperationAction(ISD::FLOG2, MVT::f32, Legal);
225 setOperationAction(ISD::FABS, MVT::f32, Legal);
226 setOperationAction(ISD::FFLOOR, MVT::f32, Legal);
227 setOperationAction(ISD::FRINT, MVT::f32, Legal);
228 setOperationAction(ISD::FTRUNC, MVT::f32, Legal);
229 setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
230 setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
231
232 setOperationAction(ISD::FROUND, MVT::f32, Custom);
233 setOperationAction(ISD::FROUND, MVT::f64, Custom);
234
235 setOperationAction(ISD::FNEARBYINT, MVT::f32, Custom);
236 setOperationAction(ISD::FNEARBYINT, MVT::f64, Custom);
237
238 setOperationAction(ISD::FREM, MVT::f32, Custom);
239 setOperationAction(ISD::FREM, MVT::f64, Custom);
240
241 // v_mad_f32 does not support denormals according to some sources.
242 if (!Subtarget->hasFP32Denormals())
243 setOperationAction(ISD::FMAD, MVT::f32, Legal);
244
245 // Expand to fneg + fadd.
246 setOperationAction(ISD::FSUB, MVT::f64, Expand);
247
248 setOperationAction(ISD::CONCAT_VECTORS, MVT::v4i32, Custom);
249 setOperationAction(ISD::CONCAT_VECTORS, MVT::v4f32, Custom);
250 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i32, Custom);
251 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8f32, Custom);
252 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v2f32, Custom);
253 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v2i32, Custom);
254 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v4f32, Custom);
255 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v4i32, Custom);
256 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8f32, Custom);
257 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8i32, Custom);
Tom Stellardaeb45642014-02-04 17:18:43 +0000258
Matt Arsenaulte8208ec2014-06-18 17:05:26 +0000259 if (Subtarget->getGeneration() < AMDGPUSubtarget::SEA_ISLANDS) {
Matt Arsenault46010932014-06-18 17:05:30 +0000260 setOperationAction(ISD::FCEIL, MVT::f64, Custom);
261 setOperationAction(ISD::FTRUNC, MVT::f64, Custom);
Matt Arsenaulte8208ec2014-06-18 17:05:26 +0000262 setOperationAction(ISD::FRINT, MVT::f64, Custom);
Matt Arsenault46010932014-06-18 17:05:30 +0000263 setOperationAction(ISD::FFLOOR, MVT::f64, Custom);
Matt Arsenaulte8208ec2014-06-18 17:05:26 +0000264 }
265
Matt Arsenault6e439652014-06-10 19:00:20 +0000266 if (!Subtarget->hasBFI()) {
267 // fcopysign can be done in a single instruction with BFI.
268 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand);
269 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
270 }
271
Tim Northoverf861de32014-07-18 08:43:24 +0000272 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
Tom Stellard94c21bc2016-11-01 16:31:48 +0000273 setOperationAction(ISD::FP_TO_FP16, MVT::f64, Custom);
Matt Arsenault86e02ce2017-03-15 19:04:26 +0000274 setOperationAction(ISD::FP_TO_FP16, MVT::f32, Custom);
Tim Northoverf861de32014-07-18 08:43:24 +0000275
Matt Arsenaultfd8c24e2014-06-13 17:20:53 +0000276 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 };
277 for (MVT VT : ScalarIntVTs) {
Matt Arsenault71e66762016-05-21 02:27:49 +0000278 // These should use [SU]DIVREM, so set them to expand
Jan Vesely4a33bc62014-08-12 17:31:17 +0000279 setOperationAction(ISD::SDIV, VT, Expand);
Matt Arsenault71e66762016-05-21 02:27:49 +0000280 setOperationAction(ISD::UDIV, VT, Expand);
281 setOperationAction(ISD::SREM, VT, Expand);
282 setOperationAction(ISD::UREM, VT, Expand);
Matt Arsenault717c1d02014-06-15 21:08:58 +0000283
Matt Arsenaultfd8c24e2014-06-13 17:20:53 +0000284 // GPU does not have divrem function for signed or unsigned.
Jan Vesely109efdf2014-06-22 21:43:00 +0000285 setOperationAction(ISD::SDIVREM, VT, Custom);
Matt Arsenault717c1d02014-06-15 21:08:58 +0000286 setOperationAction(ISD::UDIVREM, VT, Custom);
Matt Arsenaultfd8c24e2014-06-13 17:20:53 +0000287
288 // GPU does not have [S|U]MUL_LOHI functions as a single instruction.
289 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
290 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
291
292 setOperationAction(ISD::BSWAP, VT, Expand);
293 setOperationAction(ISD::CTTZ, VT, Expand);
294 setOperationAction(ISD::CTLZ, VT, Expand);
295 }
296
Matt Arsenault60425062014-06-10 19:18:28 +0000297 if (!Subtarget->hasBCNT(32))
298 setOperationAction(ISD::CTPOP, MVT::i32, Expand);
299
300 if (!Subtarget->hasBCNT(64))
301 setOperationAction(ISD::CTPOP, MVT::i64, Expand);
302
Matt Arsenault717c1d02014-06-15 21:08:58 +0000303 // The hardware supports 32-bit ROTR, but not ROTL.
304 setOperationAction(ISD::ROTL, MVT::i32, Expand);
305 setOperationAction(ISD::ROTL, MVT::i64, Expand);
306 setOperationAction(ISD::ROTR, MVT::i64, Expand);
307
308 setOperationAction(ISD::MUL, MVT::i64, Expand);
309 setOperationAction(ISD::MULHU, MVT::i64, Expand);
310 setOperationAction(ISD::MULHS, MVT::i64, Expand);
Matt Arsenault717c1d02014-06-15 21:08:58 +0000311 setOperationAction(ISD::UDIV, MVT::i32, Expand);
312 setOperationAction(ISD::UREM, MVT::i32, Expand);
313 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
Matt Arsenaultf7c95e32014-10-03 23:54:41 +0000314 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
Matt Arsenaultc9961752014-10-03 23:54:56 +0000315 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
316 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
Matt Arsenault717c1d02014-06-15 21:08:58 +0000317 setOperationAction(ISD::SELECT_CC, MVT::i64, Expand);
Matt Arsenaultb5b51102014-06-10 19:18:21 +0000318
Matt Arsenault5881f4e2015-06-09 00:52:37 +0000319 setOperationAction(ISD::SMIN, MVT::i32, Legal);
320 setOperationAction(ISD::UMIN, MVT::i32, Legal);
321 setOperationAction(ISD::SMAX, MVT::i32, Legal);
322 setOperationAction(ISD::UMAX, MVT::i32, Legal);
323
Matt Arsenaultde5fbe92016-01-11 17:02:00 +0000324 if (Subtarget->hasFFBH())
325 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Custom);
Jan Vesely6ddb8dd2014-07-15 15:51:09 +0000326
Craig Topper33772c52016-04-28 03:34:31 +0000327 if (Subtarget->hasFFBL())
328 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Legal);
Matt Arsenault5ca3c722016-01-11 16:37:46 +0000329
Matt Arsenaultf058d672016-01-11 16:50:29 +0000330 setOperationAction(ISD::CTLZ, MVT::i64, Custom);
331 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Custom);
332
Matt Arsenault59b8b772016-03-01 04:58:17 +0000333 // We only really have 32-bit BFE instructions (and 16-bit on VI).
334 //
335 // On SI+ there are 64-bit BFEs, but they are scalar only and there isn't any
336 // effort to match them now. We want this to be false for i64 cases when the
337 // extraction isn't restricted to the upper or lower half. Ideally we would
338 // have some pass reduce 64-bit extracts to 32-bit if possible. Extracts that
339 // span the midpoint are probably relatively rare, so don't worry about them
340 // for now.
341 if (Subtarget->hasBFE())
342 setHasExtractBitsInsn(true);
343
Matt Arsenaultfd8c24e2014-06-13 17:20:53 +0000344 static const MVT::SimpleValueType VectorIntTypes[] = {
Tom Stellardf6d80232013-08-21 22:14:17 +0000345 MVT::v2i32, MVT::v4i32
Aaron Watry0a794a462013-06-25 13:55:57 +0000346 };
Aaron Watry0a794a462013-06-25 13:55:57 +0000347
Matt Arsenaultfd8c24e2014-06-13 17:20:53 +0000348 for (MVT VT : VectorIntTypes) {
Matt Arsenaultb5b51102014-06-10 19:18:21 +0000349 // Expand the following operations for the current type by default.
Aaron Watry0a794a462013-06-25 13:55:57 +0000350 setOperationAction(ISD::ADD, VT, Expand);
351 setOperationAction(ISD::AND, VT, Expand);
Tom Stellardaa313d02013-07-30 14:31:03 +0000352 setOperationAction(ISD::FP_TO_SINT, VT, Expand);
353 setOperationAction(ISD::FP_TO_UINT, VT, Expand);
Aaron Watry0a794a462013-06-25 13:55:57 +0000354 setOperationAction(ISD::MUL, VT, Expand);
Valery Pykhtin8a89d362016-11-01 10:26:48 +0000355 setOperationAction(ISD::MULHU, VT, Expand);
356 setOperationAction(ISD::MULHS, VT, Expand);
Aaron Watry0a794a462013-06-25 13:55:57 +0000357 setOperationAction(ISD::OR, VT, Expand);
358 setOperationAction(ISD::SHL, VT, Expand);
Aaron Watry0a794a462013-06-25 13:55:57 +0000359 setOperationAction(ISD::SRA, VT, Expand);
Matt Arsenault825fb0b2014-06-13 04:00:30 +0000360 setOperationAction(ISD::SRL, VT, Expand);
361 setOperationAction(ISD::ROTL, VT, Expand);
362 setOperationAction(ISD::ROTR, VT, Expand);
Aaron Watry0a794a462013-06-25 13:55:57 +0000363 setOperationAction(ISD::SUB, VT, Expand);
Matt Arsenault825fb0b2014-06-13 04:00:30 +0000364 setOperationAction(ISD::SINT_TO_FP, VT, Expand);
Tom Stellardaa313d02013-07-30 14:31:03 +0000365 setOperationAction(ISD::UINT_TO_FP, VT, Expand);
Jan Vesely109efdf2014-06-22 21:43:00 +0000366 setOperationAction(ISD::SDIV, VT, Expand);
Matt Arsenault717c1d02014-06-15 21:08:58 +0000367 setOperationAction(ISD::UDIV, VT, Expand);
Matt Arsenaultfd8c24e2014-06-13 17:20:53 +0000368 setOperationAction(ISD::SREM, VT, Expand);
Aaron Watry0a794a462013-06-25 13:55:57 +0000369 setOperationAction(ISD::UREM, VT, Expand);
Matt Arsenaultfd8c24e2014-06-13 17:20:53 +0000370 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
371 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
Jan Vesely109efdf2014-06-22 21:43:00 +0000372 setOperationAction(ISD::SDIVREM, VT, Custom);
Artyom Skrobov63471332015-10-15 09:18:47 +0000373 setOperationAction(ISD::UDIVREM, VT, Expand);
Matt Arsenaultc4d3d3a2014-06-23 18:00:49 +0000374 setOperationAction(ISD::ADDC, VT, Expand);
375 setOperationAction(ISD::SUBC, VT, Expand);
376 setOperationAction(ISD::ADDE, VT, Expand);
377 setOperationAction(ISD::SUBE, VT, Expand);
Matt Arsenault9fe669c2014-03-06 17:34:03 +0000378 setOperationAction(ISD::SELECT, VT, Expand);
Tom Stellard67ae4762013-07-18 21:43:35 +0000379 setOperationAction(ISD::VSELECT, VT, Expand);
Matt Arsenaulte54e1c32014-06-23 18:00:44 +0000380 setOperationAction(ISD::SELECT_CC, VT, Expand);
Aaron Watry0a794a462013-06-25 13:55:57 +0000381 setOperationAction(ISD::XOR, VT, Expand);
Matt Arsenault13ccc8f2014-06-09 16:20:25 +0000382 setOperationAction(ISD::BSWAP, VT, Expand);
Matt Arsenaultb5b51102014-06-10 19:18:21 +0000383 setOperationAction(ISD::CTPOP, VT, Expand);
384 setOperationAction(ISD::CTTZ, VT, Expand);
385 setOperationAction(ISD::CTLZ, VT, Expand);
Matt Arsenaulte54e1c32014-06-23 18:00:44 +0000386 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Expand);
Aaron Watry0a794a462013-06-25 13:55:57 +0000387 }
Tom Stellarda92ff872013-08-16 23:51:24 +0000388
Matt Arsenaultfd8c24e2014-06-13 17:20:53 +0000389 static const MVT::SimpleValueType FloatVectorTypes[] = {
Tom Stellardf6d80232013-08-21 22:14:17 +0000390 MVT::v2f32, MVT::v4f32
Tom Stellarda92ff872013-08-16 23:51:24 +0000391 };
Tom Stellarda92ff872013-08-16 23:51:24 +0000392
Matt Arsenaultfd8c24e2014-06-13 17:20:53 +0000393 for (MVT VT : FloatVectorTypes) {
Tom Stellard175e7a82013-11-27 21:23:39 +0000394 setOperationAction(ISD::FABS, VT, Expand);
Matt Arsenault7c936902014-10-21 23:01:01 +0000395 setOperationAction(ISD::FMINNUM, VT, Expand);
396 setOperationAction(ISD::FMAXNUM, VT, Expand);
Tom Stellarda92ff872013-08-16 23:51:24 +0000397 setOperationAction(ISD::FADD, VT, Expand);
Jan Vesely85f0dbc2014-06-18 17:57:29 +0000398 setOperationAction(ISD::FCEIL, VT, Expand);
Tom Stellard3dbf1f82014-05-02 15:41:47 +0000399 setOperationAction(ISD::FCOS, VT, Expand);
Tom Stellarda92ff872013-08-16 23:51:24 +0000400 setOperationAction(ISD::FDIV, VT, Expand);
Tom Stellard5222a882014-06-20 17:06:05 +0000401 setOperationAction(ISD::FEXP2, VT, Expand);
Tom Stellarda79e9f02014-06-20 17:06:07 +0000402 setOperationAction(ISD::FLOG2, VT, Expand);
Matt Arsenault16e31332014-09-10 21:44:27 +0000403 setOperationAction(ISD::FREM, VT, Expand);
Tom Stellardbfebd1f2014-02-04 17:18:37 +0000404 setOperationAction(ISD::FPOW, VT, Expand);
Tom Stellardad3aff22013-08-16 23:51:29 +0000405 setOperationAction(ISD::FFLOOR, VT, Expand);
Tom Stellardeddfa692013-12-20 05:11:55 +0000406 setOperationAction(ISD::FTRUNC, VT, Expand);
Tom Stellarda92ff872013-08-16 23:51:24 +0000407 setOperationAction(ISD::FMUL, VT, Expand);
Matt Arsenaultc6f8fdb2014-06-26 01:28:05 +0000408 setOperationAction(ISD::FMA, VT, Expand);
Tom Stellardb249b752013-08-16 23:51:33 +0000409 setOperationAction(ISD::FRINT, VT, Expand);
Matt Arsenault692bd5e2014-06-18 22:03:45 +0000410 setOperationAction(ISD::FNEARBYINT, VT, Expand);
Tom Stellarde118b8b2013-10-29 16:37:20 +0000411 setOperationAction(ISD::FSQRT, VT, Expand);
Tom Stellard3dbf1f82014-05-02 15:41:47 +0000412 setOperationAction(ISD::FSIN, VT, Expand);
Tom Stellarda92ff872013-08-16 23:51:24 +0000413 setOperationAction(ISD::FSUB, VT, Expand);
Matt Arsenault616a8e42014-06-01 07:38:21 +0000414 setOperationAction(ISD::FNEG, VT, Expand);
Matt Arsenault616a8e42014-06-01 07:38:21 +0000415 setOperationAction(ISD::VSELECT, VT, Expand);
Matt Arsenaulte54e1c32014-06-23 18:00:44 +0000416 setOperationAction(ISD::SELECT_CC, VT, Expand);
Matt Arsenault6e439652014-06-10 19:00:20 +0000417 setOperationAction(ISD::FCOPYSIGN, VT, Expand);
Matt Arsenaulte54e1c32014-06-23 18:00:44 +0000418 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Expand);
Tom Stellarda92ff872013-08-16 23:51:24 +0000419 }
Matt Arsenaultfae02982014-03-17 18:58:11 +0000420
Matt Arsenault1cc49912016-05-25 17:34:58 +0000421 // This causes using an unrolled select operation rather than expansion with
422 // bit operations. This is in general better, but the alternative using BFI
423 // instructions may be better if the select sources are SGPRs.
424 setOperationAction(ISD::SELECT, MVT::v2f32, Promote);
425 AddPromotedToType(ISD::SELECT, MVT::v2f32, MVT::v2i32);
426
427 setOperationAction(ISD::SELECT, MVT::v4f32, Promote);
428 AddPromotedToType(ISD::SELECT, MVT::v4f32, MVT::v4i32);
429
Matt Arsenault38d8ed22016-12-09 17:49:14 +0000430 // There are no libcalls of any kind.
431 for (int I = 0; I < RTLIB::UNKNOWN_LIBCALL; ++I)
432 setLibcallName(static_cast<RTLIB::Libcall>(I), nullptr);
433
Matt Arsenaultfcdddf92014-11-26 21:23:15 +0000434 setBooleanContents(ZeroOrNegativeOneBooleanContent);
435 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
436
Matt Arsenaultfd8c24e2014-06-13 17:20:53 +0000437 setSchedulingPreference(Sched::RegPressure);
438 setJumpIsExpensive(true);
Matt Arsenault88716832017-01-10 19:08:15 +0000439
440 // FIXME: This is only partially true. If we have to do vector compares, any
441 // SGPR pair can be a condition register. If we have a uniform condition, we
442 // are better off doing SALU operations, where there is only one SCC. For now,
443 // we don't have a way of knowing during instruction selection if a condition
444 // will be uniform and we always use vector compares. Assume we are using
445 // vector compares until that is fixed.
Stanislav Mekhanoshin0ee250e2016-11-28 18:58:49 +0000446 setHasMultipleConditionRegisters(true);
Matt Arsenaultfd8c24e2014-06-13 17:20:53 +0000447
Matt Arsenault996a0ef2014-08-09 03:46:58 +0000448 // SI at least has hardware support for floating point exceptions, but no way
449 // of using or handling them is implemented. They are also optional in OpenCL
450 // (Section 7.3)
Matt Arsenaultf639c322016-01-28 20:53:42 +0000451 setHasFloatingPointExceptions(Subtarget->hasFPExceptions());
Matt Arsenault996a0ef2014-08-09 03:46:58 +0000452
Matt Arsenaultd5f91fd2014-06-23 18:00:52 +0000453 PredictableSelectIsExpensive = false;
454
Nirav Dave93f9d5c2017-02-02 18:24:55 +0000455 // We want to find all load dependencies for long chains of stores to enable
456 // merging into very wide vectors. The problem is with vectors with > 4
457 // elements. MergeConsecutiveStores will attempt to merge these because x8/x16
458 // vectors are a legal type, even though we have to split the loads
459 // usually. When we can more precisely specify load legality per address
460 // space, we should be able to make FindBetterChain/MergeConsecutiveStores
461 // smarter so that they can figure out what to do in 2 iterations without all
462 // N > 4 stores on the same chain.
463 GatherAllAliasesMaxDepth = 16;
464
Matt Arsenault0699ef32017-02-09 22:00:42 +0000465 // memcpy/memmove/memset are expanded in the IR, so we shouldn't need to worry
466 // about these during lowering.
467 MaxStoresPerMemcpy = 0xffffffff;
468 MaxStoresPerMemmove = 0xffffffff;
469 MaxStoresPerMemset = 0xffffffff;
Matt Arsenault71e66762016-05-21 02:27:49 +0000470
471 setTargetDAGCombine(ISD::BITCAST);
Matt Arsenault71e66762016-05-21 02:27:49 +0000472 setTargetDAGCombine(ISD::SHL);
473 setTargetDAGCombine(ISD::SRA);
474 setTargetDAGCombine(ISD::SRL);
475 setTargetDAGCombine(ISD::MUL);
Matt Arsenault2712d4a2016-08-27 01:32:27 +0000476 setTargetDAGCombine(ISD::MULHU);
477 setTargetDAGCombine(ISD::MULHS);
Matt Arsenault71e66762016-05-21 02:27:49 +0000478 setTargetDAGCombine(ISD::SELECT);
479 setTargetDAGCombine(ISD::SELECT_CC);
480 setTargetDAGCombine(ISD::STORE);
481 setTargetDAGCombine(ISD::FADD);
482 setTargetDAGCombine(ISD::FSUB);
Matt Arsenault2529fba2017-01-12 00:09:34 +0000483 setTargetDAGCombine(ISD::FNEG);
Matt Arsenault9dba9bd2017-02-02 02:27:04 +0000484 setTargetDAGCombine(ISD::FABS);
Tom Stellard75aadc22012-12-11 21:25:42 +0000485}
486
Tom Stellard28d06de2013-08-05 22:22:07 +0000487//===----------------------------------------------------------------------===//
488// Target Information
489//===----------------------------------------------------------------------===//
490
Matt Arsenaulta8fcfad2017-02-02 23:21:23 +0000491LLVM_READNONE
Matt Arsenault45337df2017-01-12 18:58:15 +0000492static bool fnegFoldsIntoOp(unsigned Opc) {
493 switch (Opc) {
494 case ISD::FADD:
495 case ISD::FSUB:
496 case ISD::FMUL:
497 case ISD::FMA:
498 case ISD::FMAD:
Matt Arsenault2511c032017-02-03 00:23:15 +0000499 case ISD::FMINNUM:
500 case ISD::FMAXNUM:
Matt Arsenault45337df2017-01-12 18:58:15 +0000501 case ISD::FSIN:
Matt Arsenault53f0cc22017-01-26 01:25:36 +0000502 case ISD::FTRUNC:
503 case ISD::FRINT:
504 case ISD::FNEARBYINT:
Matt Arsenault45337df2017-01-12 18:58:15 +0000505 case AMDGPUISD::RCP:
506 case AMDGPUISD::RCP_LEGACY:
507 case AMDGPUISD::SIN_HW:
508 case AMDGPUISD::FMUL_LEGACY:
Matt Arsenaulte1b59532017-02-03 00:51:50 +0000509 case AMDGPUISD::FMIN_LEGACY:
510 case AMDGPUISD::FMAX_LEGACY:
Matt Arsenault45337df2017-01-12 18:58:15 +0000511 return true;
512 default:
513 return false;
514 }
515}
516
Matt Arsenaulta8fcfad2017-02-02 23:21:23 +0000517/// \p returns true if the operation will definitely need to use a 64-bit
518/// encoding, and thus will use a VOP3 encoding regardless of the source
519/// modifiers.
520LLVM_READONLY
521static bool opMustUseVOP3Encoding(const SDNode *N, MVT VT) {
522 return N->getNumOperands() > 2 || VT == MVT::f64;
523}
524
525// Most FP instructions support source modifiers, but this could be refined
526// slightly.
527LLVM_READONLY
528static bool hasSourceMods(const SDNode *N) {
529 if (isa<MemSDNode>(N))
530 return false;
531
532 switch (N->getOpcode()) {
533 case ISD::CopyToReg:
534 case ISD::SELECT:
535 case ISD::FDIV:
536 case ISD::FREM:
537 case ISD::INLINEASM:
538 case AMDGPUISD::INTERP_P1:
539 case AMDGPUISD::INTERP_P2:
540 case AMDGPUISD::DIV_SCALE:
541 return false;
542 default:
543 return true;
544 }
545}
546
547static bool allUsesHaveSourceMods(const SDNode *N, unsigned CostThreshold = 4) {
548 // Some users (such as 3-operand FMA/MAD) must use a VOP3 encoding, and thus
549 // it is truly free to use a source modifier in all cases. If there are
550 // multiple users but for each one will necessitate using VOP3, there will be
551 // a code size increase. Try to avoid increasing code size unless we know it
552 // will save on the instruction count.
553 unsigned NumMayIncreaseSize = 0;
554 MVT VT = N->getValueType(0).getScalarType().getSimpleVT();
555
556 // XXX - Should this limit number of uses to check?
557 for (const SDNode *U : N->uses()) {
558 if (!hasSourceMods(U))
559 return false;
560
561 if (!opMustUseVOP3Encoding(U, VT)) {
562 if (++NumMayIncreaseSize > CostThreshold)
563 return false;
564 }
565 }
566
567 return true;
568}
569
Mehdi Amini44ede332015-07-09 02:09:04 +0000570MVT AMDGPUTargetLowering::getVectorIdxTy(const DataLayout &) const {
Tom Stellard28d06de2013-08-05 22:22:07 +0000571 return MVT::i32;
572}
573
Matt Arsenaultd5f91fd2014-06-23 18:00:52 +0000574bool AMDGPUTargetLowering::isSelectSupported(SelectSupportKind SelType) const {
575 return true;
576}
577
Matt Arsenault14d46452014-06-15 20:23:38 +0000578// The backend supports 32 and 64 bit floating point immediates.
579// FIXME: Why are we reporting vectors of FP immediates as legal?
580bool AMDGPUTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const {
581 EVT ScalarVT = VT.getScalarType();
Matt Arsenault4e55c1e2016-12-22 03:05:30 +0000582 return (ScalarVT == MVT::f32 || ScalarVT == MVT::f64 ||
583 (ScalarVT == MVT::f16 && Subtarget->has16BitInsts()));
Matt Arsenault14d46452014-06-15 20:23:38 +0000584}
585
586// We don't want to shrink f64 / f32 constants.
587bool AMDGPUTargetLowering::ShouldShrinkFPConstant(EVT VT) const {
588 EVT ScalarVT = VT.getScalarType();
589 return (ScalarVT != MVT::f32 && ScalarVT != MVT::f64);
590}
591
Matt Arsenault810cb622014-12-12 00:00:24 +0000592bool AMDGPUTargetLowering::shouldReduceLoadWidth(SDNode *N,
593 ISD::LoadExtType,
594 EVT NewVT) const {
595
596 unsigned NewSize = NewVT.getStoreSizeInBits();
597
598 // If we are reducing to a 32-bit load, this is always better.
599 if (NewSize == 32)
600 return true;
601
602 EVT OldVT = N->getValueType(0);
603 unsigned OldSize = OldVT.getStoreSizeInBits();
604
605 // Don't produce extloads from sub 32-bit types. SI doesn't have scalar
606 // extloads, so doing one requires using a buffer_load. In cases where we
607 // still couldn't use a scalar load, using the wider load shouldn't really
608 // hurt anything.
609
610 // If the old size already had to be an extload, there's no harm in continuing
611 // to reduce the width.
612 return (OldSize < 32);
613}
614
Matt Arsenaultc5559bb2013-11-15 04:42:23 +0000615bool AMDGPUTargetLowering::isLoadBitCastBeneficial(EVT LoadTy,
616 EVT CastTy) const {
Matt Arsenaultc5559bb2013-11-15 04:42:23 +0000617
Matt Arsenault327bb5a2016-07-01 22:47:50 +0000618 assert(LoadTy.getSizeInBits() == CastTy.getSizeInBits());
Matt Arsenaultc5559bb2013-11-15 04:42:23 +0000619
Matt Arsenault327bb5a2016-07-01 22:47:50 +0000620 if (LoadTy.getScalarType() == MVT::i32)
621 return false;
622
623 unsigned LScalarSize = LoadTy.getScalarSizeInBits();
624 unsigned CastScalarSize = CastTy.getScalarSizeInBits();
625
626 return (LScalarSize < CastScalarSize) ||
627 (CastScalarSize >= 32);
Matt Arsenaultc5559bb2013-11-15 04:42:23 +0000628}
Tom Stellard28d06de2013-08-05 22:22:07 +0000629
Matt Arsenaultb56d8432015-01-13 19:46:48 +0000630// SI+ has instructions for cttz / ctlz for 32-bit values. This is probably also
631// profitable with the expansion for 64-bit since it's generally good to
632// speculate things.
633// FIXME: These should really have the size as a parameter.
634bool AMDGPUTargetLowering::isCheapToSpeculateCttz() const {
635 return true;
636}
637
638bool AMDGPUTargetLowering::isCheapToSpeculateCtlz() const {
639 return true;
640}
641
Tom Stellard75aadc22012-12-11 21:25:42 +0000642//===---------------------------------------------------------------------===//
Tom Stellardc54731a2013-07-23 23:55:03 +0000643// Target Properties
644//===---------------------------------------------------------------------===//
645
646bool AMDGPUTargetLowering::isFAbsFree(EVT VT) const {
647 assert(VT.isFloatingPoint());
Matt Arsenaulteb522e62017-02-27 22:15:25 +0000648
649 // Packed operations do not have a fabs modifier.
650 return VT == MVT::f32 || VT == MVT::f64 ||
651 (Subtarget->has16BitInsts() && VT == MVT::f16);
Tom Stellardc54731a2013-07-23 23:55:03 +0000652}
653
654bool AMDGPUTargetLowering::isFNegFree(EVT VT) const {
Matt Arsenaulteb522e62017-02-27 22:15:25 +0000655 assert(VT.isFloatingPoint());
656 return VT == MVT::f32 || VT == MVT::f64 ||
657 (Subtarget->has16BitInsts() && VT == MVT::f16) ||
658 (Subtarget->hasVOP3PInsts() && VT == MVT::v2f16);
Tom Stellardc54731a2013-07-23 23:55:03 +0000659}
660
Matt Arsenault65ad1602015-05-24 00:51:27 +0000661bool AMDGPUTargetLowering:: storeOfVectorConstantIsCheap(EVT MemVT,
662 unsigned NumElem,
663 unsigned AS) const {
664 return true;
665}
666
Matt Arsenault61dc2352015-10-12 23:59:50 +0000667bool AMDGPUTargetLowering::aggressivelyPreferBuildVectorSources(EVT VecVT) const {
668 // There are few operations which truly have vector input operands. Any vector
669 // operation is going to involve operations on each component, and a
670 // build_vector will be a copy per element, so it always makes sense to use a
671 // build_vector input in place of the extracted element to avoid a copy into a
672 // super register.
673 //
674 // We should probably only do this if all users are extracts only, but this
675 // should be the common case.
676 return true;
677}
678
Benjamin Kramer53f9df42014-02-12 10:17:54 +0000679bool AMDGPUTargetLowering::isTruncateFree(EVT Source, EVT Dest) const {
Matt Arsenault0cdcd962014-02-10 19:57:42 +0000680 // Truncate is just accessing a subregister.
Tom Stellard115a6152016-11-10 16:02:37 +0000681
682 unsigned SrcSize = Source.getSizeInBits();
683 unsigned DestSize = Dest.getSizeInBits();
684
685 return DestSize < SrcSize && DestSize % 32 == 0 ;
Benjamin Kramer53f9df42014-02-12 10:17:54 +0000686}
687
688bool AMDGPUTargetLowering::isTruncateFree(Type *Source, Type *Dest) const {
689 // Truncate is just accessing a subregister.
Tom Stellard115a6152016-11-10 16:02:37 +0000690
691 unsigned SrcSize = Source->getScalarSizeInBits();
692 unsigned DestSize = Dest->getScalarSizeInBits();
693
694 if (DestSize== 16 && Subtarget->has16BitInsts())
695 return SrcSize >= 32;
696
697 return DestSize < SrcSize && DestSize % 32 == 0;
Matt Arsenault0cdcd962014-02-10 19:57:42 +0000698}
699
Matt Arsenaultb517c812014-03-27 17:23:31 +0000700bool AMDGPUTargetLowering::isZExtFree(Type *Src, Type *Dest) const {
Mehdi Aminia749f2a2015-07-09 02:09:52 +0000701 unsigned SrcSize = Src->getScalarSizeInBits();
702 unsigned DestSize = Dest->getScalarSizeInBits();
Matt Arsenaultb517c812014-03-27 17:23:31 +0000703
Tom Stellard115a6152016-11-10 16:02:37 +0000704 if (SrcSize == 16 && Subtarget->has16BitInsts())
705 return DestSize >= 32;
706
Matt Arsenaultb517c812014-03-27 17:23:31 +0000707 return SrcSize == 32 && DestSize == 64;
708}
709
710bool AMDGPUTargetLowering::isZExtFree(EVT Src, EVT Dest) const {
711 // Any register load of a 64-bit value really requires 2 32-bit moves. For all
712 // practical purposes, the extra mov 0 to load a 64-bit is free. As used,
713 // this will enable reducing 64-bit operations the 32-bit, which is always
714 // good.
Tom Stellard115a6152016-11-10 16:02:37 +0000715
716 if (Src == MVT::i16)
717 return Dest == MVT::i32 ||Dest == MVT::i64 ;
718
Matt Arsenaultb517c812014-03-27 17:23:31 +0000719 return Src == MVT::i32 && Dest == MVT::i64;
720}
721
Aaron Ballman3c81e462014-06-26 13:45:47 +0000722bool AMDGPUTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
723 return isZExtFree(Val.getValueType(), VT2);
724}
725
Matt Arsenaulta7f1e0c2014-03-24 19:43:31 +0000726bool AMDGPUTargetLowering::isNarrowingProfitable(EVT SrcVT, EVT DestVT) const {
727 // There aren't really 64-bit registers, but pairs of 32-bit ones and only a
728 // limited number of native 64-bit operations. Shrinking an operation to fit
729 // in a single 32-bit register should always be helpful. As currently used,
730 // this is much less general than the name suggests, and is only used in
731 // places trying to reduce the sizes of loads. Shrinking loads to < 32-bits is
732 // not profitable, and may actually be harmful.
733 return SrcVT.getSizeInBits() > 32 && DestVT.getSizeInBits() == 32;
734}
735
Tom Stellardc54731a2013-07-23 23:55:03 +0000736//===---------------------------------------------------------------------===//
Tom Stellard75aadc22012-12-11 21:25:42 +0000737// TargetLowering Callbacks
738//===---------------------------------------------------------------------===//
739
Tom Stellardca166212017-01-30 21:56:46 +0000740CCAssignFn *AMDGPUCallLowering::CCAssignFnForCall(CallingConv::ID CC,
741 bool IsVarArg) const {
742 return CC_AMDGPU;
743}
744
Tom Stellardbbeb45a2016-09-16 21:53:00 +0000745/// The SelectionDAGBuilder will automatically promote function arguments
746/// with illegal types. However, this does not work for the AMDGPU targets
747/// since the function arguments are stored in memory as these illegal types.
748/// In order to handle this properly we need to get the original types sizes
749/// from the LLVM IR Function and fixup the ISD:InputArg values before
750/// passing them to AnalyzeFormalArguments()
Christian Konig2c8f6d52013-03-07 09:03:52 +0000751
Tom Stellardbbeb45a2016-09-16 21:53:00 +0000752/// When the SelectionDAGBuilder computes the Ins, it takes care of splitting
753/// input values across multiple registers. Each item in the Ins array
754/// represents a single value that will be stored in regsters. Ins[x].VT is
755/// the value type of the value that will be stored in the register, so
756/// whatever SDNode we lower the argument to needs to be this type.
757///
758/// In order to correctly lower the arguments we need to know the size of each
759/// argument. Since Ins[x].VT gives us the size of the register that will
760/// hold the value, we need to look at Ins[x].ArgVT to see the 'real' type
761/// for the orignal function argument so that we can deduce the correct memory
762/// type to use for Ins[x]. In most cases the correct memory type will be
763/// Ins[x].ArgVT. However, this will not always be the case. If, for example,
764/// we have a kernel argument of type v8i8, this argument will be split into
765/// 8 parts and each part will be represented by its own item in the Ins array.
766/// For each part the Ins[x].ArgVT will be the v8i8, which is the full type of
767/// the argument before it was split. From this, we deduce that the memory type
768/// for each individual part is i8. We pass the memory type as LocVT to the
769/// calling convention analysis function and the register type (Ins[x].VT) as
770/// the ValVT.
771void AMDGPUTargetLowering::analyzeFormalArgumentsCompute(CCState &State,
772 const SmallVectorImpl<ISD::InputArg> &Ins) const {
773 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
774 const ISD::InputArg &In = Ins[i];
775 EVT MemVT;
776
777 unsigned NumRegs = getNumRegisters(State.getContext(), In.ArgVT);
778
Tom Stellard7998db62016-09-16 22:20:24 +0000779 if (!Subtarget->isAmdHsaOS() &&
780 (In.ArgVT == MVT::i16 || In.ArgVT == MVT::i8 || In.ArgVT == MVT::f16)) {
Tom Stellardbbeb45a2016-09-16 21:53:00 +0000781 // The ABI says the caller will extend these values to 32-bits.
782 MemVT = In.ArgVT.isInteger() ? MVT::i32 : MVT::f32;
783 } else if (NumRegs == 1) {
784 // This argument is not split, so the IR type is the memory type.
785 assert(!In.Flags.isSplit());
786 if (In.ArgVT.isExtended()) {
787 // We have an extended type, like i24, so we should just use the register type
788 MemVT = In.VT;
789 } else {
790 MemVT = In.ArgVT;
791 }
792 } else if (In.ArgVT.isVector() && In.VT.isVector() &&
793 In.ArgVT.getScalarType() == In.VT.getScalarType()) {
794 assert(In.ArgVT.getVectorNumElements() > In.VT.getVectorNumElements());
795 // We have a vector value which has been split into a vector with
796 // the same scalar type, but fewer elements. This should handle
797 // all the floating-point vector types.
798 MemVT = In.VT;
799 } else if (In.ArgVT.isVector() &&
800 In.ArgVT.getVectorNumElements() == NumRegs) {
801 // This arg has been split so that each element is stored in a separate
802 // register.
803 MemVT = In.ArgVT.getScalarType();
804 } else if (In.ArgVT.isExtended()) {
805 // We have an extended type, like i65.
806 MemVT = In.VT;
807 } else {
808 unsigned MemoryBits = In.ArgVT.getStoreSizeInBits() / NumRegs;
809 assert(In.ArgVT.getStoreSizeInBits() % NumRegs == 0);
810 if (In.VT.isInteger()) {
811 MemVT = EVT::getIntegerVT(State.getContext(), MemoryBits);
812 } else if (In.VT.isVector()) {
813 assert(!In.VT.getScalarType().isFloatingPoint());
814 unsigned NumElements = In.VT.getVectorNumElements();
815 assert(MemoryBits % NumElements == 0);
816 // This vector type has been split into another vector type with
817 // a different elements size.
818 EVT ScalarVT = EVT::getIntegerVT(State.getContext(),
819 MemoryBits / NumElements);
820 MemVT = EVT::getVectorVT(State.getContext(), ScalarVT, NumElements);
821 } else {
822 llvm_unreachable("cannot deduce memory type.");
823 }
824 }
825
826 // Convert one element vectors to scalar.
827 if (MemVT.isVector() && MemVT.getVectorNumElements() == 1)
828 MemVT = MemVT.getScalarType();
829
830 if (MemVT.isExtended()) {
831 // This should really only happen if we have vec3 arguments
832 assert(MemVT.isVector() && MemVT.getVectorNumElements() == 3);
833 MemVT = MemVT.getPow2VectorType(State.getContext());
834 }
835
836 assert(MemVT.isSimple());
837 allocateKernArg(i, In.VT, MemVT.getSimpleVT(), CCValAssign::Full, In.Flags,
838 State);
839 }
840}
841
842void AMDGPUTargetLowering::AnalyzeFormalArguments(CCState &State,
843 const SmallVectorImpl<ISD::InputArg> &Ins) const {
Christian Konig2c8f6d52013-03-07 09:03:52 +0000844 State.AnalyzeFormalArguments(Ins, CC_AMDGPU);
Tom Stellard75aadc22012-12-11 21:25:42 +0000845}
846
Marek Olsak8a0f3352016-01-13 17:23:04 +0000847void AMDGPUTargetLowering::AnalyzeReturn(CCState &State,
848 const SmallVectorImpl<ISD::OutputArg> &Outs) const {
849
850 State.AnalyzeReturn(Outs, RetCC_SI);
851}
852
Benjamin Kramerbdc49562016-06-12 15:39:02 +0000853SDValue
854AMDGPUTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
855 bool isVarArg,
856 const SmallVectorImpl<ISD::OutputArg> &Outs,
857 const SmallVectorImpl<SDValue> &OutVals,
858 const SDLoc &DL, SelectionDAG &DAG) const {
Matt Arsenault9babdf42016-06-22 20:15:28 +0000859 return DAG.getNode(AMDGPUISD::ENDPGM, DL, MVT::Other, Chain);
Tom Stellard75aadc22012-12-11 21:25:42 +0000860}
861
862//===---------------------------------------------------------------------===//
863// Target specific lowering
864//===---------------------------------------------------------------------===//
865
Matt Arsenault16353872014-04-22 16:42:00 +0000866SDValue AMDGPUTargetLowering::LowerCall(CallLoweringInfo &CLI,
867 SmallVectorImpl<SDValue> &InVals) const {
868 SDValue Callee = CLI.Callee;
869 SelectionDAG &DAG = CLI.DAG;
870
871 const Function &Fn = *DAG.getMachineFunction().getFunction();
872
873 StringRef FuncName("<unknown>");
874
Matt Arsenaultde1c34102014-04-25 22:22:01 +0000875 if (const ExternalSymbolSDNode *G = dyn_cast<ExternalSymbolSDNode>(Callee))
876 FuncName = G->getSymbol();
877 else if (const GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee))
Matt Arsenault16353872014-04-22 16:42:00 +0000878 FuncName = G->getGlobal()->getName();
879
Oliver Stannard7e7d9832016-02-02 13:52:43 +0000880 DiagnosticInfoUnsupported NoCalls(
881 Fn, "unsupported call to function " + FuncName, CLI.DL.getDebugLoc());
Matt Arsenault16353872014-04-22 16:42:00 +0000882 DAG.getContext()->diagnose(NoCalls);
Matt Arsenault9430b912016-05-18 16:10:11 +0000883
Matt Arsenault0b386362016-12-15 20:50:12 +0000884 if (!CLI.IsTailCall) {
885 for (unsigned I = 0, E = CLI.Ins.size(); I != E; ++I)
886 InVals.push_back(DAG.getUNDEF(CLI.Ins[I].VT));
887 }
Matt Arsenault9430b912016-05-18 16:10:11 +0000888
889 return DAG.getEntryNode();
Matt Arsenault16353872014-04-22 16:42:00 +0000890}
891
Matt Arsenault19c54882015-08-26 18:37:13 +0000892SDValue AMDGPUTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
893 SelectionDAG &DAG) const {
894 const Function &Fn = *DAG.getMachineFunction().getFunction();
895
Oliver Stannard7e7d9832016-02-02 13:52:43 +0000896 DiagnosticInfoUnsupported NoDynamicAlloca(Fn, "unsupported dynamic alloca",
897 SDLoc(Op).getDebugLoc());
Matt Arsenault19c54882015-08-26 18:37:13 +0000898 DAG.getContext()->diagnose(NoDynamicAlloca);
Diana Picuse440f992016-06-23 09:19:16 +0000899 auto Ops = {DAG.getConstant(0, SDLoc(), Op.getValueType()), Op.getOperand(0)};
900 return DAG.getMergeValues(Ops, SDLoc());
Matt Arsenault19c54882015-08-26 18:37:13 +0000901}
902
Matt Arsenault14d46452014-06-15 20:23:38 +0000903SDValue AMDGPUTargetLowering::LowerOperation(SDValue Op,
904 SelectionDAG &DAG) const {
Tom Stellard75aadc22012-12-11 21:25:42 +0000905 switch (Op.getOpcode()) {
906 default:
Matthias Braun8c209aa2017-01-28 02:02:38 +0000907 Op->print(errs(), &DAG);
Matt Arsenaulteaa3a7e2013-12-10 21:37:42 +0000908 llvm_unreachable("Custom lowering code for this"
909 "instruction is not implemented yet!");
Tom Stellard75aadc22012-12-11 21:25:42 +0000910 break;
Tom Stellard75aadc22012-12-11 21:25:42 +0000911 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG);
Tom Stellardd86003e2013-08-14 23:25:00 +0000912 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
913 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG);
Tom Stellard75aadc22012-12-11 21:25:42 +0000914 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
915 case ISD::UDIVREM: return LowerUDIVREM(Op, DAG);
Jan Vesely109efdf2014-06-22 21:43:00 +0000916 case ISD::SDIVREM: return LowerSDIVREM(Op, DAG);
Matt Arsenault16e31332014-09-10 21:44:27 +0000917 case ISD::FREM: return LowerFREM(Op, DAG);
Matt Arsenault46010932014-06-18 17:05:30 +0000918 case ISD::FCEIL: return LowerFCEIL(Op, DAG);
919 case ISD::FTRUNC: return LowerFTRUNC(Op, DAG);
Matt Arsenaulte8208ec2014-06-18 17:05:26 +0000920 case ISD::FRINT: return LowerFRINT(Op, DAG);
Matt Arsenault692bd5e2014-06-18 22:03:45 +0000921 case ISD::FNEARBYINT: return LowerFNEARBYINT(Op, DAG);
Matt Arsenaultb0055482015-01-21 18:18:25 +0000922 case ISD::FROUND: return LowerFROUND(Op, DAG);
Matt Arsenault46010932014-06-18 17:05:30 +0000923 case ISD::FFLOOR: return LowerFFLOOR(Op, DAG);
Matt Arsenaultf7c95e32014-10-03 23:54:41 +0000924 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG);
Tom Stellardc947d8c2013-10-30 17:22:05 +0000925 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG);
Tom Stellard94c21bc2016-11-01 16:31:48 +0000926 case ISD::FP_TO_FP16: return LowerFP_TO_FP16(Op, DAG);
Matt Arsenaultc9961752014-10-03 23:54:56 +0000927 case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG);
928 case ISD::FP_TO_UINT: return LowerFP_TO_UINT(Op, DAG);
Matt Arsenaultf058d672016-01-11 16:50:29 +0000929 case ISD::CTLZ:
930 case ISD::CTLZ_ZERO_UNDEF:
931 return LowerCTLZ(Op, DAG);
Matt Arsenault19c54882015-08-26 18:37:13 +0000932 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
Tom Stellard75aadc22012-12-11 21:25:42 +0000933 }
934 return Op;
935}
936
Matt Arsenaultd125d742014-03-27 17:23:24 +0000937void AMDGPUTargetLowering::ReplaceNodeResults(SDNode *N,
938 SmallVectorImpl<SDValue> &Results,
939 SelectionDAG &DAG) const {
940 switch (N->getOpcode()) {
941 case ISD::SIGN_EXTEND_INREG:
942 // Different parts of legalization seem to interpret which type of
943 // sign_extend_inreg is the one to check for custom lowering. The extended
944 // from type is what really matters, but some places check for custom
945 // lowering of the result type. This results in trying to use
946 // ReplaceNodeResults to sext_in_reg to an illegal type, so we'll just do
947 // nothing here and let the illegal result integer be handled normally.
948 return;
Matt Arsenaultd125d742014-03-27 17:23:24 +0000949 default:
950 return;
951 }
952}
953
Matt Arsenaultcc8d3b82014-11-13 19:56:13 +0000954static bool hasDefinedInitializer(const GlobalValue *GV) {
955 const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV);
956 if (!GVar || !GVar->hasInitializer())
957 return false;
958
Matt Arsenault8226fc42016-03-02 23:00:21 +0000959 return !isa<UndefValue>(GVar->getInitializer());
Matt Arsenaultcc8d3b82014-11-13 19:56:13 +0000960}
961
Tom Stellardc026e8b2013-06-28 15:47:08 +0000962SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunction* MFI,
963 SDValue Op,
964 SelectionDAG &DAG) const {
965
Mehdi Amini44ede332015-07-09 02:09:04 +0000966 const DataLayout &DL = DAG.getDataLayout();
Tom Stellardc026e8b2013-06-28 15:47:08 +0000967 GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Op);
Tom Stellardc026e8b2013-06-28 15:47:08 +0000968 const GlobalValue *GV = G->getGlobal();
Tom Stellardc026e8b2013-06-28 15:47:08 +0000969
Tom Stellard04c0e982014-01-22 19:24:21 +0000970 switch (G->getAddressSpace()) {
Tom Stellard04c0e982014-01-22 19:24:21 +0000971 case AMDGPUAS::LOCAL_ADDRESS: {
972 // XXX: What does the value of G->getOffset() mean?
973 assert(G->getOffset() == 0 &&
974 "Do not know what to do with an non-zero offset");
Tom Stellardc026e8b2013-06-28 15:47:08 +0000975
Matt Arsenaultcc8d3b82014-11-13 19:56:13 +0000976 // TODO: We could emit code to handle the initialization somewhere.
977 if (hasDefinedInitializer(GV))
978 break;
979
Matt Arsenault52ef4012016-07-26 16:45:58 +0000980 unsigned Offset = MFI->allocateLDSGlobal(DL, *GV);
981 return DAG.getConstant(Offset, SDLoc(Op), Op.getValueType());
Tom Stellard04c0e982014-01-22 19:24:21 +0000982 }
Tom Stellard04c0e982014-01-22 19:24:21 +0000983 }
Matt Arsenaultcc8d3b82014-11-13 19:56:13 +0000984
985 const Function &Fn = *DAG.getMachineFunction().getFunction();
Oliver Stannard7e7d9832016-02-02 13:52:43 +0000986 DiagnosticInfoUnsupported BadInit(
987 Fn, "unsupported initializer for address space", SDLoc(Op).getDebugLoc());
Matt Arsenaultcc8d3b82014-11-13 19:56:13 +0000988 DAG.getContext()->diagnose(BadInit);
989 return SDValue();
Tom Stellardc026e8b2013-06-28 15:47:08 +0000990}
991
Tom Stellardd86003e2013-08-14 23:25:00 +0000992SDValue AMDGPUTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
993 SelectionDAG &DAG) const {
994 SmallVector<SDValue, 8> Args;
Tom Stellardd86003e2013-08-14 23:25:00 +0000995
Tom Stellardff5cf0e2015-04-23 22:59:24 +0000996 for (const SDUse &U : Op->ops())
997 DAG.ExtractVectorElements(U.get(), Args);
Tom Stellardd86003e2013-08-14 23:25:00 +0000998
Ahmed Bougacha128f8732016-04-26 21:15:30 +0000999 return DAG.getBuildVector(Op.getValueType(), SDLoc(Op), Args);
Tom Stellardd86003e2013-08-14 23:25:00 +00001000}
1001
1002SDValue AMDGPUTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
1003 SelectionDAG &DAG) const {
1004
1005 SmallVector<SDValue, 8> Args;
Tom Stellardd86003e2013-08-14 23:25:00 +00001006 unsigned Start = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
Matt Arsenault9ec3cf22014-04-11 17:47:30 +00001007 EVT VT = Op.getValueType();
1008 DAG.ExtractVectorElements(Op.getOperand(0), Args, Start,
1009 VT.getVectorNumElements());
Tom Stellardd86003e2013-08-14 23:25:00 +00001010
Ahmed Bougacha128f8732016-04-26 21:15:30 +00001011 return DAG.getBuildVector(Op.getValueType(), SDLoc(Op), Args);
Tom Stellardd86003e2013-08-14 23:25:00 +00001012}
1013
Tom Stellard75aadc22012-12-11 21:25:42 +00001014SDValue AMDGPUTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
1015 SelectionDAG &DAG) const {
1016 unsigned IntrinsicID = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
Andrew Trickef9de2a2013-05-25 02:42:55 +00001017 SDLoc DL(Op);
Tom Stellard75aadc22012-12-11 21:25:42 +00001018 EVT VT = Op.getValueType();
1019
1020 switch (IntrinsicID) {
Matt Arsenault2fdf2a12017-02-21 23:35:48 +00001021 default: return Op;
Matt Arsenault2fdf2a12017-02-21 23:35:48 +00001022 case AMDGPUIntrinsic::AMDGPU_bfe_i32:
1023 return DAG.getNode(AMDGPUISD::BFE_I32, DL, VT,
1024 Op.getOperand(1),
1025 Op.getOperand(2),
1026 Op.getOperand(3));
Matt Arsenault4c537172014-03-31 18:21:18 +00001027
Matt Arsenault2fdf2a12017-02-21 23:35:48 +00001028 case AMDGPUIntrinsic::AMDGPU_bfe_u32:
1029 return DAG.getNode(AMDGPUISD::BFE_U32, DL, VT,
1030 Op.getOperand(1),
1031 Op.getOperand(2),
1032 Op.getOperand(3));
Tom Stellard75aadc22012-12-11 21:25:42 +00001033 }
1034}
1035
Tom Stellard75aadc22012-12-11 21:25:42 +00001036/// \brief Generate Min/Max node
Matt Arsenaultda7a6562017-02-01 00:42:40 +00001037SDValue AMDGPUTargetLowering::combineFMinMaxLegacy(const SDLoc &DL, EVT VT,
Benjamin Kramerbdc49562016-06-12 15:39:02 +00001038 SDValue LHS, SDValue RHS,
1039 SDValue True, SDValue False,
Matt Arsenault1e3a4eb2014-12-12 02:30:37 +00001040 SDValue CC,
1041 DAGCombinerInfo &DCI) const {
Matt Arsenaultda59f3d2014-11-13 23:03:09 +00001042 if (!(LHS == True && RHS == False) && !(LHS == False && RHS == True))
1043 return SDValue();
Tom Stellard75aadc22012-12-11 21:25:42 +00001044
Matt Arsenault1e3a4eb2014-12-12 02:30:37 +00001045 SelectionDAG &DAG = DCI.DAG;
Tom Stellard75aadc22012-12-11 21:25:42 +00001046 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get();
1047 switch (CCOpcode) {
1048 case ISD::SETOEQ:
1049 case ISD::SETONE:
1050 case ISD::SETUNE:
1051 case ISD::SETNE:
1052 case ISD::SETUEQ:
1053 case ISD::SETEQ:
1054 case ISD::SETFALSE:
1055 case ISD::SETFALSE2:
1056 case ISD::SETTRUE:
1057 case ISD::SETTRUE2:
1058 case ISD::SETUO:
1059 case ISD::SETO:
Matt Arsenaultda59f3d2014-11-13 23:03:09 +00001060 break;
Tom Stellard75aadc22012-12-11 21:25:42 +00001061 case ISD::SETULE:
Matt Arsenault1e3a4eb2014-12-12 02:30:37 +00001062 case ISD::SETULT: {
Matt Arsenault1e3a4eb2014-12-12 02:30:37 +00001063 if (LHS == True)
1064 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, RHS, LHS);
1065 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, LHS, RHS);
1066 }
Tom Stellard75aadc22012-12-11 21:25:42 +00001067 case ISD::SETOLE:
1068 case ISD::SETOLT:
1069 case ISD::SETLE:
1070 case ISD::SETLT: {
Matt Arsenault1e3a4eb2014-12-12 02:30:37 +00001071 // Ordered. Assume ordered for undefined.
1072
1073 // Only do this after legalization to avoid interfering with other combines
1074 // which might occur.
1075 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG &&
1076 !DCI.isCalledByLegalizer())
1077 return SDValue();
Marek Olsakbe047802014-12-07 12:19:03 +00001078
Matt Arsenault36094d72014-11-15 05:02:57 +00001079 // We need to permute the operands to get the correct NaN behavior. The
1080 // selected operand is the second one based on the failing compare with NaN,
1081 // so permute it based on the compare type the hardware uses.
1082 if (LHS == True)
Matt Arsenault1e3a4eb2014-12-12 02:30:37 +00001083 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, LHS, RHS);
1084 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, RHS, LHS);
Tom Stellard75aadc22012-12-11 21:25:42 +00001085 }
Tom Stellard75aadc22012-12-11 21:25:42 +00001086 case ISD::SETUGE:
Matt Arsenault1e3a4eb2014-12-12 02:30:37 +00001087 case ISD::SETUGT: {
Matt Arsenault36094d72014-11-15 05:02:57 +00001088 if (LHS == True)
1089 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, RHS, LHS);
1090 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, LHS, RHS);
Tom Stellard75aadc22012-12-11 21:25:42 +00001091 }
Matt Arsenault1e3a4eb2014-12-12 02:30:37 +00001092 case ISD::SETGT:
1093 case ISD::SETGE:
1094 case ISD::SETOGE:
1095 case ISD::SETOGT: {
1096 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG &&
1097 !DCI.isCalledByLegalizer())
1098 return SDValue();
1099
1100 if (LHS == True)
1101 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, LHS, RHS);
1102 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, RHS, LHS);
1103 }
Tom Stellard75aadc22012-12-11 21:25:42 +00001104 case ISD::SETCC_INVALID:
Matt Arsenaulteaa3a7e2013-12-10 21:37:42 +00001105 llvm_unreachable("Invalid setcc condcode!");
Tom Stellard75aadc22012-12-11 21:25:42 +00001106 }
Tom Stellardafa8b532014-05-09 16:42:16 +00001107 return SDValue();
Tom Stellard75aadc22012-12-11 21:25:42 +00001108}
1109
Matt Arsenault6e3a4512016-01-18 22:01:13 +00001110std::pair<SDValue, SDValue>
1111AMDGPUTargetLowering::split64BitValue(SDValue Op, SelectionDAG &DAG) const {
1112 SDLoc SL(Op);
1113
1114 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op);
1115
1116 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
1117 const SDValue One = DAG.getConstant(1, SL, MVT::i32);
1118
1119 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero);
1120 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One);
1121
1122 return std::make_pair(Lo, Hi);
1123}
1124
Matt Arsenault33e3ece2016-01-18 22:09:04 +00001125SDValue AMDGPUTargetLowering::getLoHalf64(SDValue Op, SelectionDAG &DAG) const {
1126 SDLoc SL(Op);
1127
1128 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op);
1129 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
1130 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero);
1131}
1132
1133SDValue AMDGPUTargetLowering::getHiHalf64(SDValue Op, SelectionDAG &DAG) const {
1134 SDLoc SL(Op);
1135
1136 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op);
1137 const SDValue One = DAG.getConstant(1, SL, MVT::i32);
1138 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One);
1139}
1140
Matt Arsenault83e60582014-07-24 17:10:35 +00001141SDValue AMDGPUTargetLowering::SplitVectorLoad(const SDValue Op,
1142 SelectionDAG &DAG) const {
Matt Arsenault9c499c32016-04-14 23:31:26 +00001143 LoadSDNode *Load = cast<LoadSDNode>(Op);
Matt Arsenault83e60582014-07-24 17:10:35 +00001144 EVT VT = Op.getValueType();
1145
Matt Arsenault9c499c32016-04-14 23:31:26 +00001146
Matt Arsenault83e60582014-07-24 17:10:35 +00001147 // If this is a 2 element vector, we really want to scalarize and not create
1148 // weird 1 element vectors.
1149 if (VT.getVectorNumElements() == 2)
Matt Arsenault9c499c32016-04-14 23:31:26 +00001150 return scalarizeVectorLoad(Load, DAG);
Matt Arsenault83e60582014-07-24 17:10:35 +00001151
Matt Arsenault83e60582014-07-24 17:10:35 +00001152 SDValue BasePtr = Load->getBasePtr();
1153 EVT PtrVT = BasePtr.getValueType();
1154 EVT MemVT = Load->getMemoryVT();
1155 SDLoc SL(Op);
Matt Arsenault52a52a52015-12-14 16:59:40 +00001156
1157 const MachinePointerInfo &SrcValue = Load->getMemOperand()->getPointerInfo();
Matt Arsenault83e60582014-07-24 17:10:35 +00001158
1159 EVT LoVT, HiVT;
1160 EVT LoMemVT, HiMemVT;
1161 SDValue Lo, Hi;
1162
1163 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
1164 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemVT);
1165 std::tie(Lo, Hi) = DAG.SplitVector(Op, SL, LoVT, HiVT);
Matt Arsenault52a52a52015-12-14 16:59:40 +00001166
1167 unsigned Size = LoMemVT.getStoreSize();
1168 unsigned BaseAlign = Load->getAlignment();
1169 unsigned HiAlign = MinAlign(BaseAlign, Size);
1170
Justin Lebar9c375812016-07-15 18:27:10 +00001171 SDValue LoLoad = DAG.getExtLoad(Load->getExtensionType(), SL, LoVT,
1172 Load->getChain(), BasePtr, SrcValue, LoMemVT,
1173 BaseAlign, Load->getMemOperand()->getFlags());
Matt Arsenault83e60582014-07-24 17:10:35 +00001174 SDValue HiPtr = DAG.getNode(ISD::ADD, SL, PtrVT, BasePtr,
Matt Arsenault52a52a52015-12-14 16:59:40 +00001175 DAG.getConstant(Size, SL, PtrVT));
Justin Lebar9c375812016-07-15 18:27:10 +00001176 SDValue HiLoad =
1177 DAG.getExtLoad(Load->getExtensionType(), SL, HiVT, Load->getChain(),
1178 HiPtr, SrcValue.getWithOffset(LoMemVT.getStoreSize()),
1179 HiMemVT, HiAlign, Load->getMemOperand()->getFlags());
Matt Arsenault83e60582014-07-24 17:10:35 +00001180
1181 SDValue Ops[] = {
1182 DAG.getNode(ISD::CONCAT_VECTORS, SL, VT, LoLoad, HiLoad),
1183 DAG.getNode(ISD::TokenFactor, SL, MVT::Other,
1184 LoLoad.getValue(1), HiLoad.getValue(1))
1185 };
1186
1187 return DAG.getMergeValues(Ops, SL);
1188}
1189
Matt Arsenault83e60582014-07-24 17:10:35 +00001190SDValue AMDGPUTargetLowering::SplitVectorStore(SDValue Op,
1191 SelectionDAG &DAG) const {
1192 StoreSDNode *Store = cast<StoreSDNode>(Op);
1193 SDValue Val = Store->getValue();
1194 EVT VT = Val.getValueType();
1195
1196 // If this is a 2 element vector, we really want to scalarize and not create
1197 // weird 1 element vectors.
1198 if (VT.getVectorNumElements() == 2)
Matt Arsenault9c499c32016-04-14 23:31:26 +00001199 return scalarizeVectorStore(Store, DAG);
Matt Arsenault83e60582014-07-24 17:10:35 +00001200
1201 EVT MemVT = Store->getMemoryVT();
1202 SDValue Chain = Store->getChain();
1203 SDValue BasePtr = Store->getBasePtr();
1204 SDLoc SL(Op);
1205
1206 EVT LoVT, HiVT;
1207 EVT LoMemVT, HiMemVT;
1208 SDValue Lo, Hi;
1209
1210 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
1211 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemVT);
1212 std::tie(Lo, Hi) = DAG.SplitVector(Val, SL, LoVT, HiVT);
1213
1214 EVT PtrVT = BasePtr.getValueType();
1215 SDValue HiPtr = DAG.getNode(ISD::ADD, SL, PtrVT, BasePtr,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001216 DAG.getConstant(LoMemVT.getStoreSize(), SL,
1217 PtrVT));
Matt Arsenault83e60582014-07-24 17:10:35 +00001218
Matt Arsenault52a52a52015-12-14 16:59:40 +00001219 const MachinePointerInfo &SrcValue = Store->getMemOperand()->getPointerInfo();
1220 unsigned BaseAlign = Store->getAlignment();
1221 unsigned Size = LoMemVT.getStoreSize();
1222 unsigned HiAlign = MinAlign(BaseAlign, Size);
1223
Justin Lebar9c375812016-07-15 18:27:10 +00001224 SDValue LoStore =
1225 DAG.getTruncStore(Chain, SL, Lo, BasePtr, SrcValue, LoMemVT, BaseAlign,
1226 Store->getMemOperand()->getFlags());
1227 SDValue HiStore =
1228 DAG.getTruncStore(Chain, SL, Hi, HiPtr, SrcValue.getWithOffset(Size),
1229 HiMemVT, HiAlign, Store->getMemOperand()->getFlags());
Matt Arsenault83e60582014-07-24 17:10:35 +00001230
1231 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoStore, HiStore);
1232}
1233
Matt Arsenault0daeb632014-07-24 06:59:20 +00001234// This is a shortcut for integer division because we have fast i32<->f32
1235// conversions, and fast f32 reciprocal instructions. The fractional part of a
Matt Arsenault81a70952016-05-21 01:53:33 +00001236// float is enough to accurately represent up to a 24-bit signed integer.
Matt Arsenault4e3d3832016-05-19 21:09:58 +00001237SDValue AMDGPUTargetLowering::LowerDIVREM24(SDValue Op, SelectionDAG &DAG,
1238 bool Sign) const {
Matt Arsenault1578aa72014-06-15 20:08:02 +00001239 SDLoc DL(Op);
Matt Arsenault0daeb632014-07-24 06:59:20 +00001240 EVT VT = Op.getValueType();
Matt Arsenault1578aa72014-06-15 20:08:02 +00001241 SDValue LHS = Op.getOperand(0);
1242 SDValue RHS = Op.getOperand(1);
Matt Arsenault0daeb632014-07-24 06:59:20 +00001243 MVT IntVT = MVT::i32;
1244 MVT FltVT = MVT::f32;
1245
Matt Arsenault81a70952016-05-21 01:53:33 +00001246 unsigned LHSSignBits = DAG.ComputeNumSignBits(LHS);
1247 if (LHSSignBits < 9)
1248 return SDValue();
1249
1250 unsigned RHSSignBits = DAG.ComputeNumSignBits(RHS);
1251 if (RHSSignBits < 9)
1252 return SDValue();
Jan Veselye5ca27d2014-08-12 17:31:20 +00001253
Matt Arsenault4e3d3832016-05-19 21:09:58 +00001254 unsigned BitSize = VT.getSizeInBits();
Matt Arsenault81a70952016-05-21 01:53:33 +00001255 unsigned SignBits = std::min(LHSSignBits, RHSSignBits);
1256 unsigned DivBits = BitSize - SignBits;
1257 if (Sign)
1258 ++DivBits;
1259
1260 ISD::NodeType ToFp = Sign ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
1261 ISD::NodeType ToInt = Sign ? ISD::FP_TO_SINT : ISD::FP_TO_UINT;
Matt Arsenault0daeb632014-07-24 06:59:20 +00001262
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001263 SDValue jq = DAG.getConstant(1, DL, IntVT);
Matt Arsenault1578aa72014-06-15 20:08:02 +00001264
Matt Arsenault4e3d3832016-05-19 21:09:58 +00001265 if (Sign) {
Jan Veselye5ca27d2014-08-12 17:31:20 +00001266 // char|short jq = ia ^ ib;
1267 jq = DAG.getNode(ISD::XOR, DL, VT, LHS, RHS);
Matt Arsenault1578aa72014-06-15 20:08:02 +00001268
Jan Veselye5ca27d2014-08-12 17:31:20 +00001269 // jq = jq >> (bitsize - 2)
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001270 jq = DAG.getNode(ISD::SRA, DL, VT, jq,
1271 DAG.getConstant(BitSize - 2, DL, VT));
Matt Arsenault1578aa72014-06-15 20:08:02 +00001272
Jan Veselye5ca27d2014-08-12 17:31:20 +00001273 // jq = jq | 0x1
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001274 jq = DAG.getNode(ISD::OR, DL, VT, jq, DAG.getConstant(1, DL, VT));
Jan Veselye5ca27d2014-08-12 17:31:20 +00001275 }
Matt Arsenault1578aa72014-06-15 20:08:02 +00001276
1277 // int ia = (int)LHS;
Matt Arsenault4e3d3832016-05-19 21:09:58 +00001278 SDValue ia = LHS;
Matt Arsenault1578aa72014-06-15 20:08:02 +00001279
1280 // int ib, (int)RHS;
Matt Arsenault4e3d3832016-05-19 21:09:58 +00001281 SDValue ib = RHS;
Matt Arsenault1578aa72014-06-15 20:08:02 +00001282
1283 // float fa = (float)ia;
Jan Veselye5ca27d2014-08-12 17:31:20 +00001284 SDValue fa = DAG.getNode(ToFp, DL, FltVT, ia);
Matt Arsenault1578aa72014-06-15 20:08:02 +00001285
1286 // float fb = (float)ib;
Jan Veselye5ca27d2014-08-12 17:31:20 +00001287 SDValue fb = DAG.getNode(ToFp, DL, FltVT, ib);
Matt Arsenault1578aa72014-06-15 20:08:02 +00001288
Matt Arsenault0daeb632014-07-24 06:59:20 +00001289 SDValue fq = DAG.getNode(ISD::FMUL, DL, FltVT,
1290 fa, DAG.getNode(AMDGPUISD::RCP, DL, FltVT, fb));
Matt Arsenault1578aa72014-06-15 20:08:02 +00001291
1292 // fq = trunc(fq);
Matt Arsenault0daeb632014-07-24 06:59:20 +00001293 fq = DAG.getNode(ISD::FTRUNC, DL, FltVT, fq);
Matt Arsenault1578aa72014-06-15 20:08:02 +00001294
1295 // float fqneg = -fq;
Matt Arsenault0daeb632014-07-24 06:59:20 +00001296 SDValue fqneg = DAG.getNode(ISD::FNEG, DL, FltVT, fq);
Matt Arsenault1578aa72014-06-15 20:08:02 +00001297
1298 // float fr = mad(fqneg, fb, fa);
Matt Arsenaultd8ed2072017-03-08 00:48:46 +00001299 unsigned OpCode = Subtarget->hasFP32Denormals() ?
1300 (unsigned)AMDGPUISD::FMAD_FTZ :
Wei Ding4d3d4ca2017-02-24 23:00:29 +00001301 (unsigned)ISD::FMAD;
1302 SDValue fr = DAG.getNode(OpCode, DL, FltVT, fqneg, fb, fa);
Matt Arsenault1578aa72014-06-15 20:08:02 +00001303
1304 // int iq = (int)fq;
Jan Veselye5ca27d2014-08-12 17:31:20 +00001305 SDValue iq = DAG.getNode(ToInt, DL, IntVT, fq);
Matt Arsenault1578aa72014-06-15 20:08:02 +00001306
1307 // fr = fabs(fr);
Matt Arsenault0daeb632014-07-24 06:59:20 +00001308 fr = DAG.getNode(ISD::FABS, DL, FltVT, fr);
Matt Arsenault1578aa72014-06-15 20:08:02 +00001309
1310 // fb = fabs(fb);
Matt Arsenault0daeb632014-07-24 06:59:20 +00001311 fb = DAG.getNode(ISD::FABS, DL, FltVT, fb);
1312
Mehdi Amini44ede332015-07-09 02:09:04 +00001313 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
Matt Arsenault1578aa72014-06-15 20:08:02 +00001314
1315 // int cv = fr >= fb;
Matt Arsenault0daeb632014-07-24 06:59:20 +00001316 SDValue cv = DAG.getSetCC(DL, SetCCVT, fr, fb, ISD::SETOGE);
1317
Matt Arsenault1578aa72014-06-15 20:08:02 +00001318 // jq = (cv ? jq : 0);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001319 jq = DAG.getNode(ISD::SELECT, DL, VT, cv, jq, DAG.getConstant(0, DL, VT));
Matt Arsenault0daeb632014-07-24 06:59:20 +00001320
Jan Veselye5ca27d2014-08-12 17:31:20 +00001321 // dst = iq + jq;
Jan Vesely4a33bc62014-08-12 17:31:17 +00001322 SDValue Div = DAG.getNode(ISD::ADD, DL, VT, iq, jq);
1323
Jan Veselye5ca27d2014-08-12 17:31:20 +00001324 // Rem needs compensation, it's easier to recompute it
Jan Vesely4a33bc62014-08-12 17:31:17 +00001325 SDValue Rem = DAG.getNode(ISD::MUL, DL, VT, Div, RHS);
1326 Rem = DAG.getNode(ISD::SUB, DL, VT, LHS, Rem);
1327
Matt Arsenault81a70952016-05-21 01:53:33 +00001328 // Truncate to number of bits this divide really is.
1329 if (Sign) {
1330 SDValue InRegSize
1331 = DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), DivBits));
1332 Div = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Div, InRegSize);
1333 Rem = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Rem, InRegSize);
1334 } else {
1335 SDValue TruncMask = DAG.getConstant((UINT64_C(1) << DivBits) - 1, DL, VT);
1336 Div = DAG.getNode(ISD::AND, DL, VT, Div, TruncMask);
1337 Rem = DAG.getNode(ISD::AND, DL, VT, Rem, TruncMask);
1338 }
1339
Matt Arsenault4e3d3832016-05-19 21:09:58 +00001340 return DAG.getMergeValues({ Div, Rem }, DL);
Matt Arsenault1578aa72014-06-15 20:08:02 +00001341}
1342
Tom Stellardbf69d762014-11-15 01:07:53 +00001343void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
1344 SelectionDAG &DAG,
1345 SmallVectorImpl<SDValue> &Results) const {
1346 assert(Op.getValueType() == MVT::i64);
1347
1348 SDLoc DL(Op);
1349 EVT VT = Op.getValueType();
1350 EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext());
1351
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001352 SDValue one = DAG.getConstant(1, DL, HalfVT);
1353 SDValue zero = DAG.getConstant(0, DL, HalfVT);
Tom Stellardbf69d762014-11-15 01:07:53 +00001354
1355 //HiLo split
1356 SDValue LHS = Op.getOperand(0);
1357 SDValue LHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, zero);
1358 SDValue LHS_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, one);
1359
1360 SDValue RHS = Op.getOperand(1);
1361 SDValue RHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, zero);
1362 SDValue RHS_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, one);
1363
Jan Vesely5f715d32015-01-22 23:42:43 +00001364 if (VT == MVT::i64 &&
1365 DAG.MaskedValueIsZero(RHS, APInt::getHighBitsSet(64, 32)) &&
1366 DAG.MaskedValueIsZero(LHS, APInt::getHighBitsSet(64, 32))) {
1367
1368 SDValue Res = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(HalfVT, HalfVT),
1369 LHS_Lo, RHS_Lo);
1370
Ahmed Bougacha128f8732016-04-26 21:15:30 +00001371 SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(0), zero});
1372 SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(1), zero});
Matt Arsenaultd275fca2016-03-01 05:06:05 +00001373
1374 Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV));
1375 Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM));
Jan Vesely5f715d32015-01-22 23:42:43 +00001376 return;
1377 }
1378
Tom Stellardbf69d762014-11-15 01:07:53 +00001379 // Get Speculative values
1380 SDValue DIV_Part = DAG.getNode(ISD::UDIV, DL, HalfVT, LHS_Hi, RHS_Lo);
1381 SDValue REM_Part = DAG.getNode(ISD::UREM, DL, HalfVT, LHS_Hi, RHS_Lo);
1382
Tom Stellardbf69d762014-11-15 01:07:53 +00001383 SDValue REM_Lo = DAG.getSelectCC(DL, RHS_Hi, zero, REM_Part, LHS_Hi, ISD::SETEQ);
Ahmed Bougacha128f8732016-04-26 21:15:30 +00001384 SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {REM_Lo, zero});
Matt Arsenaultd275fca2016-03-01 05:06:05 +00001385 REM = DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM);
Tom Stellardbf69d762014-11-15 01:07:53 +00001386
1387 SDValue DIV_Hi = DAG.getSelectCC(DL, RHS_Hi, zero, DIV_Part, zero, ISD::SETEQ);
1388 SDValue DIV_Lo = zero;
1389
1390 const unsigned halfBitWidth = HalfVT.getSizeInBits();
1391
1392 for (unsigned i = 0; i < halfBitWidth; ++i) {
Jan Veselyf7987ca2015-01-22 23:42:39 +00001393 const unsigned bitPos = halfBitWidth - i - 1;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001394 SDValue POS = DAG.getConstant(bitPos, DL, HalfVT);
Jan Veselyf7987ca2015-01-22 23:42:39 +00001395 // Get value of high bit
Jan Vesely811ef522015-04-12 23:45:01 +00001396 SDValue HBit = DAG.getNode(ISD::SRL, DL, HalfVT, LHS_Lo, POS);
1397 HBit = DAG.getNode(ISD::AND, DL, HalfVT, HBit, one);
Jan Veselyf7987ca2015-01-22 23:42:39 +00001398 HBit = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, HBit);
Tom Stellardbf69d762014-11-15 01:07:53 +00001399
Jan Veselyf7987ca2015-01-22 23:42:39 +00001400 // Shift
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001401 REM = DAG.getNode(ISD::SHL, DL, VT, REM, DAG.getConstant(1, DL, VT));
Jan Veselyf7987ca2015-01-22 23:42:39 +00001402 // Add LHS high bit
1403 REM = DAG.getNode(ISD::OR, DL, VT, REM, HBit);
Tom Stellardbf69d762014-11-15 01:07:53 +00001404
Aaron Ballmanef0fe1e2016-03-30 21:30:00 +00001405 SDValue BIT = DAG.getConstant(1ULL << bitPos, DL, HalfVT);
Tom Stellard83171b32014-11-15 01:07:57 +00001406 SDValue realBIT = DAG.getSelectCC(DL, REM, RHS, BIT, zero, ISD::SETUGE);
Tom Stellardbf69d762014-11-15 01:07:53 +00001407
1408 DIV_Lo = DAG.getNode(ISD::OR, DL, HalfVT, DIV_Lo, realBIT);
1409
1410 // Update REM
Tom Stellardbf69d762014-11-15 01:07:53 +00001411 SDValue REM_sub = DAG.getNode(ISD::SUB, DL, VT, REM, RHS);
Tom Stellard83171b32014-11-15 01:07:57 +00001412 REM = DAG.getSelectCC(DL, REM, RHS, REM_sub, REM, ISD::SETUGE);
Tom Stellardbf69d762014-11-15 01:07:53 +00001413 }
1414
Ahmed Bougacha128f8732016-04-26 21:15:30 +00001415 SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {DIV_Lo, DIV_Hi});
Matt Arsenaultd275fca2016-03-01 05:06:05 +00001416 DIV = DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV);
Tom Stellardbf69d762014-11-15 01:07:53 +00001417 Results.push_back(DIV);
1418 Results.push_back(REM);
1419}
1420
Tom Stellard75aadc22012-12-11 21:25:42 +00001421SDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op,
Matt Arsenault46013d92014-05-11 21:24:41 +00001422 SelectionDAG &DAG) const {
Andrew Trickef9de2a2013-05-25 02:42:55 +00001423 SDLoc DL(Op);
Tom Stellard75aadc22012-12-11 21:25:42 +00001424 EVT VT = Op.getValueType();
1425
Tom Stellardbf69d762014-11-15 01:07:53 +00001426 if (VT == MVT::i64) {
1427 SmallVector<SDValue, 2> Results;
1428 LowerUDIVREM64(Op, DAG, Results);
1429 return DAG.getMergeValues(Results, DL);
1430 }
1431
Matt Arsenault81a70952016-05-21 01:53:33 +00001432 if (VT == MVT::i32) {
1433 if (SDValue Res = LowerDIVREM24(Op, DAG, false))
1434 return Res;
1435 }
1436
Tom Stellard75aadc22012-12-11 21:25:42 +00001437 SDValue Num = Op.getOperand(0);
1438 SDValue Den = Op.getOperand(1);
1439
Tom Stellard75aadc22012-12-11 21:25:42 +00001440 // RCP = URECIP(Den) = 2^32 / Den + e
1441 // e is rounding error.
1442 SDValue RCP = DAG.getNode(AMDGPUISD::URECIP, DL, VT, Den);
1443
Tom Stellard4349b192014-09-22 15:35:30 +00001444 // RCP_LO = mul(RCP, Den) */
1445 SDValue RCP_LO = DAG.getNode(ISD::MUL, DL, VT, RCP, Den);
Tom Stellard75aadc22012-12-11 21:25:42 +00001446
1447 // RCP_HI = mulhu (RCP, Den) */
1448 SDValue RCP_HI = DAG.getNode(ISD::MULHU, DL, VT, RCP, Den);
1449
1450 // NEG_RCP_LO = -RCP_LO
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001451 SDValue NEG_RCP_LO = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
Tom Stellard75aadc22012-12-11 21:25:42 +00001452 RCP_LO);
1453
1454 // ABS_RCP_LO = (RCP_HI == 0 ? NEG_RCP_LO : RCP_LO)
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001455 SDValue ABS_RCP_LO = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, DL, VT),
Tom Stellard75aadc22012-12-11 21:25:42 +00001456 NEG_RCP_LO, RCP_LO,
1457 ISD::SETEQ);
1458 // Calculate the rounding error from the URECIP instruction
1459 // E = mulhu(ABS_RCP_LO, RCP)
1460 SDValue E = DAG.getNode(ISD::MULHU, DL, VT, ABS_RCP_LO, RCP);
1461
1462 // RCP_A_E = RCP + E
1463 SDValue RCP_A_E = DAG.getNode(ISD::ADD, DL, VT, RCP, E);
1464
1465 // RCP_S_E = RCP - E
1466 SDValue RCP_S_E = DAG.getNode(ISD::SUB, DL, VT, RCP, E);
1467
1468 // Tmp0 = (RCP_HI == 0 ? RCP_A_E : RCP_SUB_E)
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001469 SDValue Tmp0 = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, DL, VT),
Tom Stellard75aadc22012-12-11 21:25:42 +00001470 RCP_A_E, RCP_S_E,
1471 ISD::SETEQ);
1472 // Quotient = mulhu(Tmp0, Num)
1473 SDValue Quotient = DAG.getNode(ISD::MULHU, DL, VT, Tmp0, Num);
1474
1475 // Num_S_Remainder = Quotient * Den
Tom Stellard4349b192014-09-22 15:35:30 +00001476 SDValue Num_S_Remainder = DAG.getNode(ISD::MUL, DL, VT, Quotient, Den);
Tom Stellard75aadc22012-12-11 21:25:42 +00001477
1478 // Remainder = Num - Num_S_Remainder
1479 SDValue Remainder = DAG.getNode(ISD::SUB, DL, VT, Num, Num_S_Remainder);
1480
1481 // Remainder_GE_Den = (Remainder >= Den ? -1 : 0)
1482 SDValue Remainder_GE_Den = DAG.getSelectCC(DL, Remainder, Den,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001483 DAG.getConstant(-1, DL, VT),
1484 DAG.getConstant(0, DL, VT),
Vincent Lejeune4f3751f2013-11-06 17:36:04 +00001485 ISD::SETUGE);
1486 // Remainder_GE_Zero = (Num >= Num_S_Remainder ? -1 : 0)
1487 SDValue Remainder_GE_Zero = DAG.getSelectCC(DL, Num,
1488 Num_S_Remainder,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001489 DAG.getConstant(-1, DL, VT),
1490 DAG.getConstant(0, DL, VT),
Vincent Lejeune4f3751f2013-11-06 17:36:04 +00001491 ISD::SETUGE);
Tom Stellard75aadc22012-12-11 21:25:42 +00001492 // Tmp1 = Remainder_GE_Den & Remainder_GE_Zero
1493 SDValue Tmp1 = DAG.getNode(ISD::AND, DL, VT, Remainder_GE_Den,
1494 Remainder_GE_Zero);
1495
1496 // Calculate Division result:
1497
1498 // Quotient_A_One = Quotient + 1
1499 SDValue Quotient_A_One = DAG.getNode(ISD::ADD, DL, VT, Quotient,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001500 DAG.getConstant(1, DL, VT));
Tom Stellard75aadc22012-12-11 21:25:42 +00001501
1502 // Quotient_S_One = Quotient - 1
1503 SDValue Quotient_S_One = DAG.getNode(ISD::SUB, DL, VT, Quotient,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001504 DAG.getConstant(1, DL, VT));
Tom Stellard75aadc22012-12-11 21:25:42 +00001505
1506 // Div = (Tmp1 == 0 ? Quotient : Quotient_A_One)
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001507 SDValue Div = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, DL, VT),
Tom Stellard75aadc22012-12-11 21:25:42 +00001508 Quotient, Quotient_A_One, ISD::SETEQ);
1509
1510 // Div = (Remainder_GE_Zero == 0 ? Quotient_S_One : Div)
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001511 Div = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, DL, VT),
Tom Stellard75aadc22012-12-11 21:25:42 +00001512 Quotient_S_One, Div, ISD::SETEQ);
1513
1514 // Calculate Rem result:
1515
1516 // Remainder_S_Den = Remainder - Den
1517 SDValue Remainder_S_Den = DAG.getNode(ISD::SUB, DL, VT, Remainder, Den);
1518
1519 // Remainder_A_Den = Remainder + Den
1520 SDValue Remainder_A_Den = DAG.getNode(ISD::ADD, DL, VT, Remainder, Den);
1521
1522 // Rem = (Tmp1 == 0 ? Remainder : Remainder_S_Den)
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001523 SDValue Rem = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, DL, VT),
Tom Stellard75aadc22012-12-11 21:25:42 +00001524 Remainder, Remainder_S_Den, ISD::SETEQ);
1525
1526 // Rem = (Remainder_GE_Zero == 0 ? Remainder_A_Den : Rem)
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001527 Rem = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, DL, VT),
Tom Stellard75aadc22012-12-11 21:25:42 +00001528 Remainder_A_Den, Rem, ISD::SETEQ);
Matt Arsenault7939acd2014-04-07 16:44:24 +00001529 SDValue Ops[2] = {
1530 Div,
1531 Rem
1532 };
Craig Topper64941d92014-04-27 19:20:57 +00001533 return DAG.getMergeValues(Ops, DL);
Tom Stellard75aadc22012-12-11 21:25:42 +00001534}
1535
Jan Vesely109efdf2014-06-22 21:43:00 +00001536SDValue AMDGPUTargetLowering::LowerSDIVREM(SDValue Op,
1537 SelectionDAG &DAG) const {
1538 SDLoc DL(Op);
1539 EVT VT = Op.getValueType();
1540
Jan Vesely109efdf2014-06-22 21:43:00 +00001541 SDValue LHS = Op.getOperand(0);
1542 SDValue RHS = Op.getOperand(1);
1543
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001544 SDValue Zero = DAG.getConstant(0, DL, VT);
1545 SDValue NegOne = DAG.getConstant(-1, DL, VT);
Jan Vesely4a33bc62014-08-12 17:31:17 +00001546
Matt Arsenault81a70952016-05-21 01:53:33 +00001547 if (VT == MVT::i32) {
1548 if (SDValue Res = LowerDIVREM24(Op, DAG, true))
1549 return Res;
Jan Vesely5f715d32015-01-22 23:42:43 +00001550 }
Matt Arsenault81a70952016-05-21 01:53:33 +00001551
Jan Vesely5f715d32015-01-22 23:42:43 +00001552 if (VT == MVT::i64 &&
1553 DAG.ComputeNumSignBits(LHS) > 32 &&
1554 DAG.ComputeNumSignBits(RHS) > 32) {
1555 EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext());
1556
1557 //HiLo split
1558 SDValue LHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, Zero);
1559 SDValue RHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, Zero);
1560 SDValue DIVREM = DAG.getNode(ISD::SDIVREM, DL, DAG.getVTList(HalfVT, HalfVT),
1561 LHS_Lo, RHS_Lo);
1562 SDValue Res[2] = {
1563 DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(0)),
1564 DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(1))
1565 };
1566 return DAG.getMergeValues(Res, DL);
1567 }
1568
Jan Vesely109efdf2014-06-22 21:43:00 +00001569 SDValue LHSign = DAG.getSelectCC(DL, LHS, Zero, NegOne, Zero, ISD::SETLT);
1570 SDValue RHSign = DAG.getSelectCC(DL, RHS, Zero, NegOne, Zero, ISD::SETLT);
1571 SDValue DSign = DAG.getNode(ISD::XOR, DL, VT, LHSign, RHSign);
1572 SDValue RSign = LHSign; // Remainder sign is the same as LHS
1573
1574 LHS = DAG.getNode(ISD::ADD, DL, VT, LHS, LHSign);
1575 RHS = DAG.getNode(ISD::ADD, DL, VT, RHS, RHSign);
1576
1577 LHS = DAG.getNode(ISD::XOR, DL, VT, LHS, LHSign);
1578 RHS = DAG.getNode(ISD::XOR, DL, VT, RHS, RHSign);
1579
1580 SDValue Div = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(VT, VT), LHS, RHS);
1581 SDValue Rem = Div.getValue(1);
1582
1583 Div = DAG.getNode(ISD::XOR, DL, VT, Div, DSign);
1584 Rem = DAG.getNode(ISD::XOR, DL, VT, Rem, RSign);
1585
1586 Div = DAG.getNode(ISD::SUB, DL, VT, Div, DSign);
1587 Rem = DAG.getNode(ISD::SUB, DL, VT, Rem, RSign);
1588
1589 SDValue Res[2] = {
1590 Div,
1591 Rem
1592 };
1593 return DAG.getMergeValues(Res, DL);
1594}
1595
Matt Arsenault16e31332014-09-10 21:44:27 +00001596// (frem x, y) -> (fsub x, (fmul (ftrunc (fdiv x, y)), y))
1597SDValue AMDGPUTargetLowering::LowerFREM(SDValue Op, SelectionDAG &DAG) const {
1598 SDLoc SL(Op);
1599 EVT VT = Op.getValueType();
1600 SDValue X = Op.getOperand(0);
1601 SDValue Y = Op.getOperand(1);
1602
Sanjay Patela2607012015-09-16 16:31:21 +00001603 // TODO: Should this propagate fast-math-flags?
1604
Matt Arsenault16e31332014-09-10 21:44:27 +00001605 SDValue Div = DAG.getNode(ISD::FDIV, SL, VT, X, Y);
1606 SDValue Floor = DAG.getNode(ISD::FTRUNC, SL, VT, Div);
1607 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, Floor, Y);
1608
1609 return DAG.getNode(ISD::FSUB, SL, VT, X, Mul);
1610}
1611
Matt Arsenault46010932014-06-18 17:05:30 +00001612SDValue AMDGPUTargetLowering::LowerFCEIL(SDValue Op, SelectionDAG &DAG) const {
1613 SDLoc SL(Op);
1614 SDValue Src = Op.getOperand(0);
1615
1616 // result = trunc(src)
1617 // if (src > 0.0 && src != result)
1618 // result += 1.0
1619
1620 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
1621
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001622 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64);
1623 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64);
Matt Arsenault46010932014-06-18 17:05:30 +00001624
Mehdi Amini44ede332015-07-09 02:09:04 +00001625 EVT SetCCVT =
1626 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64);
Matt Arsenault46010932014-06-18 17:05:30 +00001627
1628 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOGT);
1629 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE);
1630 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc);
1631
1632 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, One, Zero);
Sanjay Patela2607012015-09-16 16:31:21 +00001633 // TODO: Should this propagate fast-math-flags?
Matt Arsenault46010932014-06-18 17:05:30 +00001634 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add);
1635}
1636
Benjamin Kramerbdc49562016-06-12 15:39:02 +00001637static SDValue extractF64Exponent(SDValue Hi, const SDLoc &SL,
1638 SelectionDAG &DAG) {
Matt Arsenaultb0055482015-01-21 18:18:25 +00001639 const unsigned FractBits = 52;
1640 const unsigned ExpBits = 11;
1641
1642 SDValue ExpPart = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32,
1643 Hi,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001644 DAG.getConstant(FractBits - 32, SL, MVT::i32),
1645 DAG.getConstant(ExpBits, SL, MVT::i32));
Matt Arsenaultb0055482015-01-21 18:18:25 +00001646 SDValue Exp = DAG.getNode(ISD::SUB, SL, MVT::i32, ExpPart,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001647 DAG.getConstant(1023, SL, MVT::i32));
Matt Arsenaultb0055482015-01-21 18:18:25 +00001648
1649 return Exp;
1650}
1651
Matt Arsenault46010932014-06-18 17:05:30 +00001652SDValue AMDGPUTargetLowering::LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const {
1653 SDLoc SL(Op);
1654 SDValue Src = Op.getOperand(0);
1655
1656 assert(Op.getValueType() == MVT::f64);
1657
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001658 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
1659 const SDValue One = DAG.getConstant(1, SL, MVT::i32);
Matt Arsenault46010932014-06-18 17:05:30 +00001660
1661 SDValue VecSrc = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src);
1662
1663 // Extract the upper half, since this is where we will find the sign and
1664 // exponent.
1665 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, VecSrc, One);
1666
Matt Arsenaultb0055482015-01-21 18:18:25 +00001667 SDValue Exp = extractF64Exponent(Hi, SL, DAG);
Matt Arsenault46010932014-06-18 17:05:30 +00001668
Matt Arsenaultb0055482015-01-21 18:18:25 +00001669 const unsigned FractBits = 52;
Matt Arsenault46010932014-06-18 17:05:30 +00001670
1671 // Extract the sign bit.
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001672 const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, SL, MVT::i32);
Matt Arsenault46010932014-06-18 17:05:30 +00001673 SDValue SignBit = DAG.getNode(ISD::AND, SL, MVT::i32, Hi, SignBitMask);
1674
1675 // Extend back to to 64-bits.
Ahmed Bougacha128f8732016-04-26 21:15:30 +00001676 SDValue SignBit64 = DAG.getBuildVector(MVT::v2i32, SL, {Zero, SignBit});
Matt Arsenault46010932014-06-18 17:05:30 +00001677 SignBit64 = DAG.getNode(ISD::BITCAST, SL, MVT::i64, SignBit64);
1678
1679 SDValue BcInt = DAG.getNode(ISD::BITCAST, SL, MVT::i64, Src);
Matt Arsenault2b0fa432014-06-18 22:11:03 +00001680 const SDValue FractMask
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001681 = DAG.getConstant((UINT64_C(1) << FractBits) - 1, SL, MVT::i64);
Matt Arsenault46010932014-06-18 17:05:30 +00001682
1683 SDValue Shr = DAG.getNode(ISD::SRA, SL, MVT::i64, FractMask, Exp);
1684 SDValue Not = DAG.getNOT(SL, Shr, MVT::i64);
1685 SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, BcInt, Not);
1686
Mehdi Amini44ede332015-07-09 02:09:04 +00001687 EVT SetCCVT =
1688 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i32);
Matt Arsenault46010932014-06-18 17:05:30 +00001689
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001690 const SDValue FiftyOne = DAG.getConstant(FractBits - 1, SL, MVT::i32);
Matt Arsenault46010932014-06-18 17:05:30 +00001691
1692 SDValue ExpLt0 = DAG.getSetCC(SL, SetCCVT, Exp, Zero, ISD::SETLT);
1693 SDValue ExpGt51 = DAG.getSetCC(SL, SetCCVT, Exp, FiftyOne, ISD::SETGT);
1694
1695 SDValue Tmp1 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpLt0, SignBit64, Tmp0);
1696 SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpGt51, BcInt, Tmp1);
1697
1698 return DAG.getNode(ISD::BITCAST, SL, MVT::f64, Tmp2);
1699}
1700
Matt Arsenaulte8208ec2014-06-18 17:05:26 +00001701SDValue AMDGPUTargetLowering::LowerFRINT(SDValue Op, SelectionDAG &DAG) const {
1702 SDLoc SL(Op);
1703 SDValue Src = Op.getOperand(0);
1704
1705 assert(Op.getValueType() == MVT::f64);
1706
Stephan Bergmann17c7f702016-12-14 11:57:17 +00001707 APFloat C1Val(APFloat::IEEEdouble(), "0x1.0p+52");
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001708 SDValue C1 = DAG.getConstantFP(C1Val, SL, MVT::f64);
Matt Arsenaulte8208ec2014-06-18 17:05:26 +00001709 SDValue CopySign = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, C1, Src);
1710
Sanjay Patela2607012015-09-16 16:31:21 +00001711 // TODO: Should this propagate fast-math-flags?
1712
Matt Arsenaulte8208ec2014-06-18 17:05:26 +00001713 SDValue Tmp1 = DAG.getNode(ISD::FADD, SL, MVT::f64, Src, CopySign);
1714 SDValue Tmp2 = DAG.getNode(ISD::FSUB, SL, MVT::f64, Tmp1, CopySign);
1715
1716 SDValue Fabs = DAG.getNode(ISD::FABS, SL, MVT::f64, Src);
Matt Arsenaultd22626f2014-06-18 17:45:58 +00001717
Stephan Bergmann17c7f702016-12-14 11:57:17 +00001718 APFloat C2Val(APFloat::IEEEdouble(), "0x1.fffffffffffffp+51");
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001719 SDValue C2 = DAG.getConstantFP(C2Val, SL, MVT::f64);
Matt Arsenaulte8208ec2014-06-18 17:05:26 +00001720
Mehdi Amini44ede332015-07-09 02:09:04 +00001721 EVT SetCCVT =
1722 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64);
Matt Arsenaulte8208ec2014-06-18 17:05:26 +00001723 SDValue Cond = DAG.getSetCC(SL, SetCCVT, Fabs, C2, ISD::SETOGT);
1724
1725 return DAG.getSelect(SL, MVT::f64, Cond, Src, Tmp2);
1726}
1727
Matt Arsenault692bd5e2014-06-18 22:03:45 +00001728SDValue AMDGPUTargetLowering::LowerFNEARBYINT(SDValue Op, SelectionDAG &DAG) const {
1729 // FNEARBYINT and FRINT are the same, except in their handling of FP
1730 // exceptions. Those aren't really meaningful for us, and OpenCL only has
1731 // rint, so just treat them as equivalent.
1732 return DAG.getNode(ISD::FRINT, SDLoc(Op), Op.getValueType(), Op.getOperand(0));
1733}
1734
Matt Arsenaultb0055482015-01-21 18:18:25 +00001735// XXX - May require not supporting f32 denormals?
Matt Arsenaultb5d23272017-03-24 20:04:18 +00001736
1737// Don't handle v2f16. The extra instructions to scalarize and repack around the
1738// compare and vselect end up producing worse code than scalarizing the whole
1739// operation.
1740SDValue AMDGPUTargetLowering::LowerFROUND32_16(SDValue Op, SelectionDAG &DAG) const {
Matt Arsenaultb0055482015-01-21 18:18:25 +00001741 SDLoc SL(Op);
1742 SDValue X = Op.getOperand(0);
Matt Arsenaultb5d23272017-03-24 20:04:18 +00001743 EVT VT = Op.getValueType();
Matt Arsenaultb0055482015-01-21 18:18:25 +00001744
Matt Arsenaultb5d23272017-03-24 20:04:18 +00001745 SDValue T = DAG.getNode(ISD::FTRUNC, SL, VT, X);
Matt Arsenaultb0055482015-01-21 18:18:25 +00001746
Sanjay Patela2607012015-09-16 16:31:21 +00001747 // TODO: Should this propagate fast-math-flags?
1748
Matt Arsenaultb5d23272017-03-24 20:04:18 +00001749 SDValue Diff = DAG.getNode(ISD::FSUB, SL, VT, X, T);
Matt Arsenaultb0055482015-01-21 18:18:25 +00001750
Matt Arsenaultb5d23272017-03-24 20:04:18 +00001751 SDValue AbsDiff = DAG.getNode(ISD::FABS, SL, VT, Diff);
Matt Arsenaultb0055482015-01-21 18:18:25 +00001752
Matt Arsenaultb5d23272017-03-24 20:04:18 +00001753 const SDValue Zero = DAG.getConstantFP(0.0, SL, VT);
1754 const SDValue One = DAG.getConstantFP(1.0, SL, VT);
1755 const SDValue Half = DAG.getConstantFP(0.5, SL, VT);
Matt Arsenaultb0055482015-01-21 18:18:25 +00001756
Matt Arsenaultb5d23272017-03-24 20:04:18 +00001757 SDValue SignOne = DAG.getNode(ISD::FCOPYSIGN, SL, VT, One, X);
Matt Arsenaultb0055482015-01-21 18:18:25 +00001758
Mehdi Amini44ede332015-07-09 02:09:04 +00001759 EVT SetCCVT =
Matt Arsenaultb5d23272017-03-24 20:04:18 +00001760 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
Matt Arsenaultb0055482015-01-21 18:18:25 +00001761
1762 SDValue Cmp = DAG.getSetCC(SL, SetCCVT, AbsDiff, Half, ISD::SETOGE);
1763
Matt Arsenaultb5d23272017-03-24 20:04:18 +00001764 SDValue Sel = DAG.getNode(ISD::SELECT, SL, VT, Cmp, SignOne, Zero);
Matt Arsenaultb0055482015-01-21 18:18:25 +00001765
Matt Arsenaultb5d23272017-03-24 20:04:18 +00001766 return DAG.getNode(ISD::FADD, SL, VT, T, Sel);
Matt Arsenaultb0055482015-01-21 18:18:25 +00001767}
1768
1769SDValue AMDGPUTargetLowering::LowerFROUND64(SDValue Op, SelectionDAG &DAG) const {
1770 SDLoc SL(Op);
1771 SDValue X = Op.getOperand(0);
1772
1773 SDValue L = DAG.getNode(ISD::BITCAST, SL, MVT::i64, X);
1774
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001775 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
1776 const SDValue One = DAG.getConstant(1, SL, MVT::i32);
1777 const SDValue NegOne = DAG.getConstant(-1, SL, MVT::i32);
1778 const SDValue FiftyOne = DAG.getConstant(51, SL, MVT::i32);
Mehdi Amini44ede332015-07-09 02:09:04 +00001779 EVT SetCCVT =
1780 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i32);
Matt Arsenaultb0055482015-01-21 18:18:25 +00001781
1782 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, X);
1783
1784 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BC, One);
1785
1786 SDValue Exp = extractF64Exponent(Hi, SL, DAG);
1787
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001788 const SDValue Mask = DAG.getConstant(INT64_C(0x000fffffffffffff), SL,
1789 MVT::i64);
Matt Arsenaultb0055482015-01-21 18:18:25 +00001790
1791 SDValue M = DAG.getNode(ISD::SRA, SL, MVT::i64, Mask, Exp);
1792 SDValue D = DAG.getNode(ISD::SRA, SL, MVT::i64,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001793 DAG.getConstant(INT64_C(0x0008000000000000), SL,
1794 MVT::i64),
Matt Arsenaultb0055482015-01-21 18:18:25 +00001795 Exp);
1796
1797 SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, L, M);
1798 SDValue Tmp1 = DAG.getSetCC(SL, SetCCVT,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001799 DAG.getConstant(0, SL, MVT::i64), Tmp0,
Matt Arsenaultb0055482015-01-21 18:18:25 +00001800 ISD::SETNE);
1801
1802 SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, Tmp1,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001803 D, DAG.getConstant(0, SL, MVT::i64));
Matt Arsenaultb0055482015-01-21 18:18:25 +00001804 SDValue K = DAG.getNode(ISD::ADD, SL, MVT::i64, L, Tmp2);
1805
1806 K = DAG.getNode(ISD::AND, SL, MVT::i64, K, DAG.getNOT(SL, M, MVT::i64));
1807 K = DAG.getNode(ISD::BITCAST, SL, MVT::f64, K);
1808
1809 SDValue ExpLt0 = DAG.getSetCC(SL, SetCCVT, Exp, Zero, ISD::SETLT);
1810 SDValue ExpGt51 = DAG.getSetCC(SL, SetCCVT, Exp, FiftyOne, ISD::SETGT);
1811 SDValue ExpEqNegOne = DAG.getSetCC(SL, SetCCVT, NegOne, Exp, ISD::SETEQ);
1812
1813 SDValue Mag = DAG.getNode(ISD::SELECT, SL, MVT::f64,
1814 ExpEqNegOne,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001815 DAG.getConstantFP(1.0, SL, MVT::f64),
1816 DAG.getConstantFP(0.0, SL, MVT::f64));
Matt Arsenaultb0055482015-01-21 18:18:25 +00001817
1818 SDValue S = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, Mag, X);
1819
1820 K = DAG.getNode(ISD::SELECT, SL, MVT::f64, ExpLt0, S, K);
1821 K = DAG.getNode(ISD::SELECT, SL, MVT::f64, ExpGt51, X, K);
1822
1823 return K;
1824}
1825
1826SDValue AMDGPUTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const {
1827 EVT VT = Op.getValueType();
1828
Matt Arsenaultb5d23272017-03-24 20:04:18 +00001829 if (VT == MVT::f32 || VT == MVT::f16)
1830 return LowerFROUND32_16(Op, DAG);
Matt Arsenaultb0055482015-01-21 18:18:25 +00001831
1832 if (VT == MVT::f64)
1833 return LowerFROUND64(Op, DAG);
1834
1835 llvm_unreachable("unhandled type");
1836}
1837
Matt Arsenault46010932014-06-18 17:05:30 +00001838SDValue AMDGPUTargetLowering::LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const {
1839 SDLoc SL(Op);
1840 SDValue Src = Op.getOperand(0);
1841
1842 // result = trunc(src);
1843 // if (src < 0.0 && src != result)
1844 // result += -1.0.
1845
1846 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
1847
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001848 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64);
1849 const SDValue NegOne = DAG.getConstantFP(-1.0, SL, MVT::f64);
Matt Arsenault46010932014-06-18 17:05:30 +00001850
Mehdi Amini44ede332015-07-09 02:09:04 +00001851 EVT SetCCVT =
1852 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64);
Matt Arsenault46010932014-06-18 17:05:30 +00001853
1854 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOLT);
1855 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE);
1856 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc);
1857
1858 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, NegOne, Zero);
Sanjay Patela2607012015-09-16 16:31:21 +00001859 // TODO: Should this propagate fast-math-flags?
Matt Arsenault46010932014-06-18 17:05:30 +00001860 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add);
1861}
1862
Matt Arsenaultf058d672016-01-11 16:50:29 +00001863SDValue AMDGPUTargetLowering::LowerCTLZ(SDValue Op, SelectionDAG &DAG) const {
1864 SDLoc SL(Op);
1865 SDValue Src = Op.getOperand(0);
Matt Arsenaultf058d672016-01-11 16:50:29 +00001866 bool ZeroUndef = Op.getOpcode() == ISD::CTLZ_ZERO_UNDEF;
Matt Arsenaultde5fbe92016-01-11 17:02:00 +00001867
1868 if (ZeroUndef && Src.getValueType() == MVT::i32)
1869 return DAG.getNode(AMDGPUISD::FFBH_U32, SL, MVT::i32, Src);
1870
Matt Arsenaultf058d672016-01-11 16:50:29 +00001871 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src);
1872
1873 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
1874 const SDValue One = DAG.getConstant(1, SL, MVT::i32);
1875
1876 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero);
1877 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One);
1878
1879 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(),
1880 *DAG.getContext(), MVT::i32);
1881
1882 SDValue Hi0 = DAG.getSetCC(SL, SetCCVT, Hi, Zero, ISD::SETEQ);
1883
1884 SDValue CtlzLo = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SL, MVT::i32, Lo);
1885 SDValue CtlzHi = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SL, MVT::i32, Hi);
1886
1887 const SDValue Bits32 = DAG.getConstant(32, SL, MVT::i32);
1888 SDValue Add = DAG.getNode(ISD::ADD, SL, MVT::i32, CtlzLo, Bits32);
1889
1890 // ctlz(x) = hi_32(x) == 0 ? ctlz(lo_32(x)) + 32 : ctlz(hi_32(x))
1891 SDValue NewCtlz = DAG.getNode(ISD::SELECT, SL, MVT::i32, Hi0, Add, CtlzHi);
1892
1893 if (!ZeroUndef) {
1894 // Test if the full 64-bit input is zero.
1895
1896 // FIXME: DAG combines turn what should be an s_and_b64 into a v_or_b32,
1897 // which we probably don't want.
1898 SDValue Lo0 = DAG.getSetCC(SL, SetCCVT, Lo, Zero, ISD::SETEQ);
1899 SDValue SrcIsZero = DAG.getNode(ISD::AND, SL, SetCCVT, Lo0, Hi0);
1900
1901 // TODO: If i64 setcc is half rate, it can result in 1 fewer instruction
1902 // with the same cycles, otherwise it is slower.
1903 // SDValue SrcIsZero = DAG.getSetCC(SL, SetCCVT, Src,
1904 // DAG.getConstant(0, SL, MVT::i64), ISD::SETEQ);
1905
1906 const SDValue Bits32 = DAG.getConstant(64, SL, MVT::i32);
1907
1908 // The instruction returns -1 for 0 input, but the defined intrinsic
1909 // behavior is to return the number of bits.
1910 NewCtlz = DAG.getNode(ISD::SELECT, SL, MVT::i32,
1911 SrcIsZero, Bits32, NewCtlz);
1912 }
1913
1914 return DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i64, NewCtlz);
1915}
1916
Matt Arsenault5e0bdb82016-01-11 22:01:48 +00001917SDValue AMDGPUTargetLowering::LowerINT_TO_FP32(SDValue Op, SelectionDAG &DAG,
1918 bool Signed) const {
1919 // Unsigned
1920 // cul2f(ulong u)
1921 //{
1922 // uint lz = clz(u);
1923 // uint e = (u != 0) ? 127U + 63U - lz : 0;
1924 // u = (u << lz) & 0x7fffffffffffffffUL;
1925 // ulong t = u & 0xffffffffffUL;
1926 // uint v = (e << 23) | (uint)(u >> 40);
1927 // uint r = t > 0x8000000000UL ? 1U : (t == 0x8000000000UL ? v & 1U : 0U);
1928 // return as_float(v + r);
1929 //}
1930 // Signed
1931 // cl2f(long l)
1932 //{
1933 // long s = l >> 63;
1934 // float r = cul2f((l + s) ^ s);
1935 // return s ? -r : r;
1936 //}
1937
1938 SDLoc SL(Op);
1939 SDValue Src = Op.getOperand(0);
1940 SDValue L = Src;
1941
1942 SDValue S;
1943 if (Signed) {
1944 const SDValue SignBit = DAG.getConstant(63, SL, MVT::i64);
1945 S = DAG.getNode(ISD::SRA, SL, MVT::i64, L, SignBit);
1946
1947 SDValue LPlusS = DAG.getNode(ISD::ADD, SL, MVT::i64, L, S);
1948 L = DAG.getNode(ISD::XOR, SL, MVT::i64, LPlusS, S);
1949 }
1950
1951 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(),
1952 *DAG.getContext(), MVT::f32);
1953
1954
1955 SDValue ZeroI32 = DAG.getConstant(0, SL, MVT::i32);
1956 SDValue ZeroI64 = DAG.getConstant(0, SL, MVT::i64);
1957 SDValue LZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SL, MVT::i64, L);
1958 LZ = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LZ);
1959
1960 SDValue K = DAG.getConstant(127U + 63U, SL, MVT::i32);
1961 SDValue E = DAG.getSelect(SL, MVT::i32,
1962 DAG.getSetCC(SL, SetCCVT, L, ZeroI64, ISD::SETNE),
1963 DAG.getNode(ISD::SUB, SL, MVT::i32, K, LZ),
1964 ZeroI32);
1965
1966 SDValue U = DAG.getNode(ISD::AND, SL, MVT::i64,
1967 DAG.getNode(ISD::SHL, SL, MVT::i64, L, LZ),
1968 DAG.getConstant((-1ULL) >> 1, SL, MVT::i64));
1969
1970 SDValue T = DAG.getNode(ISD::AND, SL, MVT::i64, U,
1971 DAG.getConstant(0xffffffffffULL, SL, MVT::i64));
1972
1973 SDValue UShl = DAG.getNode(ISD::SRL, SL, MVT::i64,
1974 U, DAG.getConstant(40, SL, MVT::i64));
1975
1976 SDValue V = DAG.getNode(ISD::OR, SL, MVT::i32,
1977 DAG.getNode(ISD::SHL, SL, MVT::i32, E, DAG.getConstant(23, SL, MVT::i32)),
1978 DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, UShl));
1979
1980 SDValue C = DAG.getConstant(0x8000000000ULL, SL, MVT::i64);
1981 SDValue RCmp = DAG.getSetCC(SL, SetCCVT, T, C, ISD::SETUGT);
1982 SDValue TCmp = DAG.getSetCC(SL, SetCCVT, T, C, ISD::SETEQ);
1983
1984 SDValue One = DAG.getConstant(1, SL, MVT::i32);
1985
1986 SDValue VTrunc1 = DAG.getNode(ISD::AND, SL, MVT::i32, V, One);
1987
1988 SDValue R = DAG.getSelect(SL, MVT::i32,
1989 RCmp,
1990 One,
1991 DAG.getSelect(SL, MVT::i32, TCmp, VTrunc1, ZeroI32));
1992 R = DAG.getNode(ISD::ADD, SL, MVT::i32, V, R);
1993 R = DAG.getNode(ISD::BITCAST, SL, MVT::f32, R);
1994
1995 if (!Signed)
1996 return R;
1997
1998 SDValue RNeg = DAG.getNode(ISD::FNEG, SL, MVT::f32, R);
1999 return DAG.getSelect(SL, MVT::f32, DAG.getSExtOrTrunc(S, SL, SetCCVT), RNeg, R);
2000}
2001
Matt Arsenaultf7c95e32014-10-03 23:54:41 +00002002SDValue AMDGPUTargetLowering::LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG,
2003 bool Signed) const {
2004 SDLoc SL(Op);
2005 SDValue Src = Op.getOperand(0);
2006
2007 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src);
2008
2009 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BC,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00002010 DAG.getConstant(0, SL, MVT::i32));
Matt Arsenaultf7c95e32014-10-03 23:54:41 +00002011 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BC,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00002012 DAG.getConstant(1, SL, MVT::i32));
Matt Arsenaultf7c95e32014-10-03 23:54:41 +00002013
2014 SDValue CvtHi = DAG.getNode(Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP,
2015 SL, MVT::f64, Hi);
2016
2017 SDValue CvtLo = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f64, Lo);
2018
2019 SDValue LdExp = DAG.getNode(AMDGPUISD::LDEXP, SL, MVT::f64, CvtHi,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00002020 DAG.getConstant(32, SL, MVT::i32));
Sanjay Patela2607012015-09-16 16:31:21 +00002021 // TODO: Should this propagate fast-math-flags?
Matt Arsenaultf7c95e32014-10-03 23:54:41 +00002022 return DAG.getNode(ISD::FADD, SL, MVT::f64, LdExp, CvtLo);
2023}
2024
Tom Stellardc947d8c2013-10-30 17:22:05 +00002025SDValue AMDGPUTargetLowering::LowerUINT_TO_FP(SDValue Op,
2026 SelectionDAG &DAG) const {
Matt Arsenault5e0bdb82016-01-11 22:01:48 +00002027 assert(Op.getOperand(0).getValueType() == MVT::i64 &&
2028 "operation should be legal");
Tom Stellardc947d8c2013-10-30 17:22:05 +00002029
Konstantin Zhuravlyovf86e4b72016-11-13 07:01:11 +00002030 // TODO: Factor out code common with LowerSINT_TO_FP.
2031
Matt Arsenaultf7c95e32014-10-03 23:54:41 +00002032 EVT DestVT = Op.getValueType();
Konstantin Zhuravlyovf86e4b72016-11-13 07:01:11 +00002033 if (Subtarget->has16BitInsts() && DestVT == MVT::f16) {
2034 SDLoc DL(Op);
2035 SDValue Src = Op.getOperand(0);
2036
2037 SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src);
2038 SDValue FPRoundFlag = DAG.getIntPtrConstant(0, SDLoc(Op));
2039 SDValue FPRound =
2040 DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag);
2041
2042 return FPRound;
2043 }
Matt Arsenaultf7c95e32014-10-03 23:54:41 +00002044
Matt Arsenault5e0bdb82016-01-11 22:01:48 +00002045 if (DestVT == MVT::f32)
2046 return LowerINT_TO_FP32(Op, DAG, false);
Matt Arsenaultf7c95e32014-10-03 23:54:41 +00002047
Matt Arsenaultedc7dcb2016-07-28 00:32:05 +00002048 assert(DestVT == MVT::f64);
2049 return LowerINT_TO_FP64(Op, DAG, false);
Tom Stellardc947d8c2013-10-30 17:22:05 +00002050}
Tom Stellardfbab8272013-08-16 01:12:11 +00002051
Matt Arsenaultf7c95e32014-10-03 23:54:41 +00002052SDValue AMDGPUTargetLowering::LowerSINT_TO_FP(SDValue Op,
2053 SelectionDAG &DAG) const {
Matt Arsenault5e0bdb82016-01-11 22:01:48 +00002054 assert(Op.getOperand(0).getValueType() == MVT::i64 &&
2055 "operation should be legal");
2056
Konstantin Zhuravlyovf86e4b72016-11-13 07:01:11 +00002057 // TODO: Factor out code common with LowerUINT_TO_FP.
2058
Matt Arsenault5e0bdb82016-01-11 22:01:48 +00002059 EVT DestVT = Op.getValueType();
Konstantin Zhuravlyovf86e4b72016-11-13 07:01:11 +00002060 if (Subtarget->has16BitInsts() && DestVT == MVT::f16) {
2061 SDLoc DL(Op);
2062 SDValue Src = Op.getOperand(0);
2063
2064 SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src);
2065 SDValue FPRoundFlag = DAG.getIntPtrConstant(0, SDLoc(Op));
2066 SDValue FPRound =
2067 DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag);
2068
2069 return FPRound;
2070 }
2071
Matt Arsenault5e0bdb82016-01-11 22:01:48 +00002072 if (DestVT == MVT::f32)
2073 return LowerINT_TO_FP32(Op, DAG, true);
2074
Matt Arsenaultedc7dcb2016-07-28 00:32:05 +00002075 assert(DestVT == MVT::f64);
2076 return LowerINT_TO_FP64(Op, DAG, true);
Matt Arsenaultf7c95e32014-10-03 23:54:41 +00002077}
2078
Matt Arsenaultc9961752014-10-03 23:54:56 +00002079SDValue AMDGPUTargetLowering::LowerFP64_TO_INT(SDValue Op, SelectionDAG &DAG,
2080 bool Signed) const {
2081 SDLoc SL(Op);
2082
2083 SDValue Src = Op.getOperand(0);
2084
2085 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
2086
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00002087 SDValue K0 = DAG.getConstantFP(BitsToDouble(UINT64_C(0x3df0000000000000)), SL,
2088 MVT::f64);
2089 SDValue K1 = DAG.getConstantFP(BitsToDouble(UINT64_C(0xc1f0000000000000)), SL,
2090 MVT::f64);
Sanjay Patela2607012015-09-16 16:31:21 +00002091 // TODO: Should this propagate fast-math-flags?
Matt Arsenaultc9961752014-10-03 23:54:56 +00002092 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f64, Trunc, K0);
2093
2094 SDValue FloorMul = DAG.getNode(ISD::FFLOOR, SL, MVT::f64, Mul);
2095
2096
2097 SDValue Fma = DAG.getNode(ISD::FMA, SL, MVT::f64, FloorMul, K1, Trunc);
2098
2099 SDValue Hi = DAG.getNode(Signed ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, SL,
2100 MVT::i32, FloorMul);
2101 SDValue Lo = DAG.getNode(ISD::FP_TO_UINT, SL, MVT::i32, Fma);
2102
Ahmed Bougacha128f8732016-04-26 21:15:30 +00002103 SDValue Result = DAG.getBuildVector(MVT::v2i32, SL, {Lo, Hi});
Matt Arsenaultc9961752014-10-03 23:54:56 +00002104
2105 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Result);
2106}
2107
Tom Stellard94c21bc2016-11-01 16:31:48 +00002108SDValue AMDGPUTargetLowering::LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) const {
Matt Arsenault86e02ce2017-03-15 19:04:26 +00002109 SDLoc DL(Op);
2110 SDValue N0 = Op.getOperand(0);
2111
2112 // Convert to target node to get known bits
2113 if (N0.getValueType() == MVT::f32)
2114 return DAG.getNode(AMDGPUISD::FP_TO_FP16, DL, Op.getValueType(), N0);
Tom Stellard94c21bc2016-11-01 16:31:48 +00002115
2116 if (getTargetMachine().Options.UnsafeFPMath) {
2117 // There is a generic expand for FP_TO_FP16 with unsafe fast math.
2118 return SDValue();
2119 }
2120
Matt Arsenault86e02ce2017-03-15 19:04:26 +00002121 assert(N0.getSimpleValueType() == MVT::f64);
Tom Stellard94c21bc2016-11-01 16:31:48 +00002122
2123 // f64 -> f16 conversion using round-to-nearest-even rounding mode.
2124 const unsigned ExpMask = 0x7ff;
2125 const unsigned ExpBiasf64 = 1023;
2126 const unsigned ExpBiasf16 = 15;
2127 SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
2128 SDValue One = DAG.getConstant(1, DL, MVT::i32);
2129 SDValue U = DAG.getNode(ISD::BITCAST, DL, MVT::i64, N0);
2130 SDValue UH = DAG.getNode(ISD::SRL, DL, MVT::i64, U,
2131 DAG.getConstant(32, DL, MVT::i64));
2132 UH = DAG.getZExtOrTrunc(UH, DL, MVT::i32);
2133 U = DAG.getZExtOrTrunc(U, DL, MVT::i32);
2134 SDValue E = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
2135 DAG.getConstant(20, DL, MVT::i64));
2136 E = DAG.getNode(ISD::AND, DL, MVT::i32, E,
2137 DAG.getConstant(ExpMask, DL, MVT::i32));
2138 // Subtract the fp64 exponent bias (1023) to get the real exponent and
2139 // add the f16 bias (15) to get the biased exponent for the f16 format.
2140 E = DAG.getNode(ISD::ADD, DL, MVT::i32, E,
2141 DAG.getConstant(-ExpBiasf64 + ExpBiasf16, DL, MVT::i32));
2142
2143 SDValue M = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
2144 DAG.getConstant(8, DL, MVT::i32));
2145 M = DAG.getNode(ISD::AND, DL, MVT::i32, M,
2146 DAG.getConstant(0xffe, DL, MVT::i32));
2147
2148 SDValue MaskedSig = DAG.getNode(ISD::AND, DL, MVT::i32, UH,
2149 DAG.getConstant(0x1ff, DL, MVT::i32));
2150 MaskedSig = DAG.getNode(ISD::OR, DL, MVT::i32, MaskedSig, U);
2151
2152 SDValue Lo40Set = DAG.getSelectCC(DL, MaskedSig, Zero, Zero, One, ISD::SETEQ);
2153 M = DAG.getNode(ISD::OR, DL, MVT::i32, M, Lo40Set);
2154
2155 // (M != 0 ? 0x0200 : 0) | 0x7c00;
2156 SDValue I = DAG.getNode(ISD::OR, DL, MVT::i32,
2157 DAG.getSelectCC(DL, M, Zero, DAG.getConstant(0x0200, DL, MVT::i32),
2158 Zero, ISD::SETNE), DAG.getConstant(0x7c00, DL, MVT::i32));
2159
2160 // N = M | (E << 12);
2161 SDValue N = DAG.getNode(ISD::OR, DL, MVT::i32, M,
2162 DAG.getNode(ISD::SHL, DL, MVT::i32, E,
2163 DAG.getConstant(12, DL, MVT::i32)));
2164
2165 // B = clamp(1-E, 0, 13);
2166 SDValue OneSubExp = DAG.getNode(ISD::SUB, DL, MVT::i32,
2167 One, E);
2168 SDValue B = DAG.getNode(ISD::SMAX, DL, MVT::i32, OneSubExp, Zero);
2169 B = DAG.getNode(ISD::SMIN, DL, MVT::i32, B,
2170 DAG.getConstant(13, DL, MVT::i32));
2171
2172 SDValue SigSetHigh = DAG.getNode(ISD::OR, DL, MVT::i32, M,
2173 DAG.getConstant(0x1000, DL, MVT::i32));
2174
2175 SDValue D = DAG.getNode(ISD::SRL, DL, MVT::i32, SigSetHigh, B);
2176 SDValue D0 = DAG.getNode(ISD::SHL, DL, MVT::i32, D, B);
2177 SDValue D1 = DAG.getSelectCC(DL, D0, SigSetHigh, One, Zero, ISD::SETNE);
2178 D = DAG.getNode(ISD::OR, DL, MVT::i32, D, D1);
2179
2180 SDValue V = DAG.getSelectCC(DL, E, One, D, N, ISD::SETLT);
2181 SDValue VLow3 = DAG.getNode(ISD::AND, DL, MVT::i32, V,
2182 DAG.getConstant(0x7, DL, MVT::i32));
2183 V = DAG.getNode(ISD::SRL, DL, MVT::i32, V,
2184 DAG.getConstant(2, DL, MVT::i32));
2185 SDValue V0 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(3, DL, MVT::i32),
2186 One, Zero, ISD::SETEQ);
2187 SDValue V1 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(5, DL, MVT::i32),
2188 One, Zero, ISD::SETGT);
2189 V1 = DAG.getNode(ISD::OR, DL, MVT::i32, V0, V1);
2190 V = DAG.getNode(ISD::ADD, DL, MVT::i32, V, V1);
2191
2192 V = DAG.getSelectCC(DL, E, DAG.getConstant(30, DL, MVT::i32),
2193 DAG.getConstant(0x7c00, DL, MVT::i32), V, ISD::SETGT);
2194 V = DAG.getSelectCC(DL, E, DAG.getConstant(1039, DL, MVT::i32),
2195 I, V, ISD::SETEQ);
2196
2197 // Extract the sign bit.
2198 SDValue Sign = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
2199 DAG.getConstant(16, DL, MVT::i32));
2200 Sign = DAG.getNode(ISD::AND, DL, MVT::i32, Sign,
2201 DAG.getConstant(0x8000, DL, MVT::i32));
2202
2203 V = DAG.getNode(ISD::OR, DL, MVT::i32, Sign, V);
2204 return DAG.getZExtOrTrunc(V, DL, Op.getValueType());
2205}
2206
Matt Arsenaultc9961752014-10-03 23:54:56 +00002207SDValue AMDGPUTargetLowering::LowerFP_TO_SINT(SDValue Op,
2208 SelectionDAG &DAG) const {
2209 SDValue Src = Op.getOperand(0);
2210
Konstantin Zhuravlyovf86e4b72016-11-13 07:01:11 +00002211 // TODO: Factor out code common with LowerFP_TO_UINT.
2212
2213 EVT SrcVT = Src.getValueType();
2214 if (Subtarget->has16BitInsts() && SrcVT == MVT::f16) {
2215 SDLoc DL(Op);
2216
2217 SDValue FPExtend = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src);
2218 SDValue FpToInt32 =
2219 DAG.getNode(Op.getOpcode(), DL, MVT::i64, FPExtend);
2220
2221 return FpToInt32;
2222 }
2223
Matt Arsenaultc9961752014-10-03 23:54:56 +00002224 if (Op.getValueType() == MVT::i64 && Src.getValueType() == MVT::f64)
2225 return LowerFP64_TO_INT(Op, DAG, true);
2226
2227 return SDValue();
2228}
2229
2230SDValue AMDGPUTargetLowering::LowerFP_TO_UINT(SDValue Op,
2231 SelectionDAG &DAG) const {
2232 SDValue Src = Op.getOperand(0);
2233
Konstantin Zhuravlyovf86e4b72016-11-13 07:01:11 +00002234 // TODO: Factor out code common with LowerFP_TO_SINT.
2235
2236 EVT SrcVT = Src.getValueType();
2237 if (Subtarget->has16BitInsts() && SrcVT == MVT::f16) {
2238 SDLoc DL(Op);
2239
2240 SDValue FPExtend = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src);
2241 SDValue FpToInt32 =
2242 DAG.getNode(Op.getOpcode(), DL, MVT::i64, FPExtend);
2243
2244 return FpToInt32;
2245 }
2246
Matt Arsenaultc9961752014-10-03 23:54:56 +00002247 if (Op.getValueType() == MVT::i64 && Src.getValueType() == MVT::f64)
2248 return LowerFP64_TO_INT(Op, DAG, false);
2249
2250 return SDValue();
2251}
2252
Matt Arsenaultfae02982014-03-17 18:58:11 +00002253SDValue AMDGPUTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
2254 SelectionDAG &DAG) const {
2255 EVT ExtraVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2256 MVT VT = Op.getSimpleValueType();
2257 MVT ScalarVT = VT.getScalarType();
2258
Matt Arsenaultedc7dcb2016-07-28 00:32:05 +00002259 assert(VT.isVector());
Matt Arsenaultfae02982014-03-17 18:58:11 +00002260
2261 SDValue Src = Op.getOperand(0);
Matt Arsenault5dbd5db2014-04-22 03:49:30 +00002262 SDLoc DL(Op);
Matt Arsenaultfae02982014-03-17 18:58:11 +00002263
Matt Arsenault5dbd5db2014-04-22 03:49:30 +00002264 // TODO: Don't scalarize on Evergreen?
2265 unsigned NElts = VT.getVectorNumElements();
2266 SmallVector<SDValue, 8> Args;
2267 DAG.ExtractVectorElements(Src, Args, 0, NElts);
Matt Arsenaultfae02982014-03-17 18:58:11 +00002268
Matt Arsenault5dbd5db2014-04-22 03:49:30 +00002269 SDValue VTOp = DAG.getValueType(ExtraVT.getScalarType());
2270 for (unsigned I = 0; I < NElts; ++I)
2271 Args[I] = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, ScalarVT, Args[I], VTOp);
Matt Arsenaultfae02982014-03-17 18:58:11 +00002272
Ahmed Bougacha128f8732016-04-26 21:15:30 +00002273 return DAG.getBuildVector(VT, DL, Args);
Matt Arsenaultfae02982014-03-17 18:58:11 +00002274}
2275
Tom Stellard75aadc22012-12-11 21:25:42 +00002276//===----------------------------------------------------------------------===//
Tom Stellard50122a52014-04-07 19:45:41 +00002277// Custom DAG optimizations
2278//===----------------------------------------------------------------------===//
2279
2280static bool isU24(SDValue Op, SelectionDAG &DAG) {
2281 APInt KnownZero, KnownOne;
2282 EVT VT = Op.getValueType();
Jay Foada0653a32014-05-14 21:14:37 +00002283 DAG.computeKnownBits(Op, KnownZero, KnownOne);
Tom Stellard50122a52014-04-07 19:45:41 +00002284
2285 return (VT.getSizeInBits() - KnownZero.countLeadingOnes()) <= 24;
2286}
2287
2288static bool isI24(SDValue Op, SelectionDAG &DAG) {
2289 EVT VT = Op.getValueType();
2290
2291 // In order for this to be a signed 24-bit value, bit 23, must
2292 // be a sign bit.
2293 return VT.getSizeInBits() >= 24 && // Types less than 24-bit should be treated
2294 // as unsigned 24-bit values.
2295 (VT.getSizeInBits() - DAG.ComputeNumSignBits(Op)) < 24;
2296}
2297
Tom Stellard09c2bd62016-10-14 19:14:29 +00002298static bool simplifyI24(SDNode *Node24, unsigned OpIdx,
2299 TargetLowering::DAGCombinerInfo &DCI) {
Tom Stellard50122a52014-04-07 19:45:41 +00002300
2301 SelectionDAG &DAG = DCI.DAG;
Tom Stellard09c2bd62016-10-14 19:14:29 +00002302 SDValue Op = Node24->getOperand(OpIdx);
Tom Stellard50122a52014-04-07 19:45:41 +00002303 EVT VT = Op.getValueType();
2304
2305 APInt Demanded = APInt::getLowBitsSet(VT.getSizeInBits(), 24);
2306 APInt KnownZero, KnownOne;
2307 TargetLowering::TargetLoweringOpt TLO(DAG, true, true);
Tom Stellard09c2bd62016-10-14 19:14:29 +00002308 if (TLO.SimplifyDemandedBits(Node24, OpIdx, Demanded, DCI))
Matt Arsenault2712d4a2016-08-27 01:32:27 +00002309 return true;
Matt Arsenault2712d4a2016-08-27 01:32:27 +00002310
2311 return false;
Tom Stellard50122a52014-04-07 19:45:41 +00002312}
2313
Matt Arsenault5565f65e2014-05-22 18:09:07 +00002314template <typename IntTy>
Benjamin Kramerbdc49562016-06-12 15:39:02 +00002315static SDValue constantFoldBFE(SelectionDAG &DAG, IntTy Src0, uint32_t Offset,
2316 uint32_t Width, const SDLoc &DL) {
Matt Arsenault5565f65e2014-05-22 18:09:07 +00002317 if (Width + Offset < 32) {
Matt Arsenault46cbc432014-09-19 00:42:06 +00002318 uint32_t Shl = static_cast<uint32_t>(Src0) << (32 - Offset - Width);
2319 IntTy Result = static_cast<IntTy>(Shl) >> (32 - Width);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00002320 return DAG.getConstant(Result, DL, MVT::i32);
Matt Arsenault5565f65e2014-05-22 18:09:07 +00002321 }
2322
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00002323 return DAG.getConstant(Src0 >> Offset, DL, MVT::i32);
Matt Arsenault5565f65e2014-05-22 18:09:07 +00002324}
2325
Matt Arsenault327bb5a2016-07-01 22:47:50 +00002326static bool hasVolatileUser(SDNode *Val) {
2327 for (SDNode *U : Val->uses()) {
2328 if (MemSDNode *M = dyn_cast<MemSDNode>(U)) {
2329 if (M->isVolatile())
2330 return true;
2331 }
2332 }
2333
2334 return false;
2335}
2336
Matt Arsenault8af47a02016-07-01 22:55:55 +00002337bool AMDGPUTargetLowering::shouldCombineMemoryType(EVT VT) const {
Matt Arsenault327bb5a2016-07-01 22:47:50 +00002338 // i32 vectors are the canonical memory type.
2339 if (VT.getScalarType() == MVT::i32 || isTypeLegal(VT))
2340 return false;
2341
Matt Arsenault327bb5a2016-07-01 22:47:50 +00002342 if (!VT.isByteSized())
2343 return false;
2344
2345 unsigned Size = VT.getStoreSize();
2346
2347 if ((Size == 1 || Size == 2 || Size == 4) && !VT.isVector())
2348 return false;
2349
2350 if (Size == 3 || (Size > 4 && (Size % 4 != 0)))
2351 return false;
2352
Matt Arsenaultca3976f2014-07-15 02:06:31 +00002353 return true;
2354}
2355
Matt Arsenault327bb5a2016-07-01 22:47:50 +00002356// Replace load of an illegal type with a store of a bitcast to a friendlier
2357// type.
2358SDValue AMDGPUTargetLowering::performLoadCombine(SDNode *N,
2359 DAGCombinerInfo &DCI) const {
2360 if (!DCI.isBeforeLegalize())
2361 return SDValue();
2362
2363 LoadSDNode *LN = cast<LoadSDNode>(N);
2364 if (LN->isVolatile() || !ISD::isNormalLoad(LN) || hasVolatileUser(LN))
2365 return SDValue();
2366
Matt Arsenault327bb5a2016-07-01 22:47:50 +00002367 SDLoc SL(N);
2368 SelectionDAG &DAG = DCI.DAG;
2369 EVT VT = LN->getMemoryVT();
Matt Arsenault8af47a02016-07-01 22:55:55 +00002370
2371 unsigned Size = VT.getStoreSize();
2372 unsigned Align = LN->getAlignment();
2373 if (Align < Size && isTypeLegal(VT)) {
2374 bool IsFast;
2375 unsigned AS = LN->getAddressSpace();
2376
2377 // Expand unaligned loads earlier than legalization. Due to visitation order
2378 // problems during legalization, the emitted instructions to pack and unpack
2379 // the bytes again are not eliminated in the case of an unaligned copy.
2380 if (!allowsMisalignedMemoryAccesses(VT, AS, Align, &IsFast)) {
Matt Arsenaultb50eb8d2016-08-31 21:52:27 +00002381 if (VT.isVector())
2382 return scalarizeVectorLoad(LN, DAG);
2383
Matt Arsenault8af47a02016-07-01 22:55:55 +00002384 SDValue Ops[2];
2385 std::tie(Ops[0], Ops[1]) = expandUnalignedLoad(LN, DAG);
2386 return DAG.getMergeValues(Ops, SDLoc(N));
2387 }
2388
2389 if (!IsFast)
2390 return SDValue();
2391 }
2392
2393 if (!shouldCombineMemoryType(VT))
2394 return SDValue();
2395
Matt Arsenault327bb5a2016-07-01 22:47:50 +00002396 EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT);
2397
2398 SDValue NewLoad
2399 = DAG.getLoad(NewVT, SL, LN->getChain(),
2400 LN->getBasePtr(), LN->getMemOperand());
2401
2402 SDValue BC = DAG.getNode(ISD::BITCAST, SL, VT, NewLoad);
2403 DCI.CombineTo(N, BC, NewLoad.getValue(1));
2404 return SDValue(N, 0);
2405}
2406
2407// Replace store of an illegal type with a store of a bitcast to a friendlier
2408// type.
Matt Arsenaultca3976f2014-07-15 02:06:31 +00002409SDValue AMDGPUTargetLowering::performStoreCombine(SDNode *N,
2410 DAGCombinerInfo &DCI) const {
2411 if (!DCI.isBeforeLegalize())
2412 return SDValue();
2413
2414 StoreSDNode *SN = cast<StoreSDNode>(N);
Matt Arsenault327bb5a2016-07-01 22:47:50 +00002415 if (SN->isVolatile() || !ISD::isNormalStore(SN))
Matt Arsenaultca3976f2014-07-15 02:06:31 +00002416 return SDValue();
2417
Matt Arsenault327bb5a2016-07-01 22:47:50 +00002418 EVT VT = SN->getMemoryVT();
Matt Arsenault8af47a02016-07-01 22:55:55 +00002419 unsigned Size = VT.getStoreSize();
Matt Arsenaultca3976f2014-07-15 02:06:31 +00002420
2421 SDLoc SL(N);
2422 SelectionDAG &DAG = DCI.DAG;
Matt Arsenault8af47a02016-07-01 22:55:55 +00002423 unsigned Align = SN->getAlignment();
2424 if (Align < Size && isTypeLegal(VT)) {
2425 bool IsFast;
2426 unsigned AS = SN->getAddressSpace();
2427
2428 // Expand unaligned stores earlier than legalization. Due to visitation
2429 // order problems during legalization, the emitted instructions to pack and
2430 // unpack the bytes again are not eliminated in the case of an unaligned
2431 // copy.
Matt Arsenaultb50eb8d2016-08-31 21:52:27 +00002432 if (!allowsMisalignedMemoryAccesses(VT, AS, Align, &IsFast)) {
2433 if (VT.isVector())
2434 return scalarizeVectorStore(SN, DAG);
2435
Matt Arsenault8af47a02016-07-01 22:55:55 +00002436 return expandUnalignedStore(SN, DAG);
Matt Arsenaultb50eb8d2016-08-31 21:52:27 +00002437 }
Matt Arsenault8af47a02016-07-01 22:55:55 +00002438
2439 if (!IsFast)
2440 return SDValue();
2441 }
2442
2443 if (!shouldCombineMemoryType(VT))
2444 return SDValue();
2445
Matt Arsenault327bb5a2016-07-01 22:47:50 +00002446 EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT);
Matt Arsenault8af47a02016-07-01 22:55:55 +00002447 SDValue Val = SN->getValue();
2448
2449 //DCI.AddToWorklist(Val.getNode());
Matt Arsenaultca3976f2014-07-15 02:06:31 +00002450
Matt Arsenault327bb5a2016-07-01 22:47:50 +00002451 bool OtherUses = !Val.hasOneUse();
2452 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NewVT, Val);
2453 if (OtherUses) {
2454 SDValue CastBack = DAG.getNode(ISD::BITCAST, SL, VT, CastVal);
2455 DAG.ReplaceAllUsesOfValueWith(Val, CastBack);
2456 }
Matt Arsenaultca3976f2014-07-15 02:06:31 +00002457
Matt Arsenault327bb5a2016-07-01 22:47:50 +00002458 return DAG.getStore(SN->getChain(), SL, CastVal,
Matt Arsenaultca3976f2014-07-15 02:06:31 +00002459 SN->getBasePtr(), SN->getMemOperand());
2460}
2461
Matt Arsenault2fdf2a12017-02-21 23:35:48 +00002462SDValue AMDGPUTargetLowering::performClampCombine(SDNode *N,
2463 DAGCombinerInfo &DCI) const {
2464 ConstantFPSDNode *CSrc = dyn_cast<ConstantFPSDNode>(N->getOperand(0));
2465 if (!CSrc)
2466 return SDValue();
2467
2468 const APFloat &F = CSrc->getValueAPF();
2469 APFloat Zero = APFloat::getZero(F.getSemantics());
2470 APFloat::cmpResult Cmp0 = F.compare(Zero);
2471 if (Cmp0 == APFloat::cmpLessThan ||
2472 (Cmp0 == APFloat::cmpUnordered && Subtarget->enableDX10Clamp())) {
2473 return DCI.DAG.getConstantFP(Zero, SDLoc(N), N->getValueType(0));
2474 }
2475
2476 APFloat One(F.getSemantics(), "1.0");
2477 APFloat::cmpResult Cmp1 = F.compare(One);
2478 if (Cmp1 == APFloat::cmpGreaterThan)
2479 return DCI.DAG.getConstantFP(One, SDLoc(N), N->getValueType(0));
2480
2481 return SDValue(CSrc, 0);
2482}
2483
Matt Arsenaultfa5f7672016-09-14 15:19:03 +00002484/// Split the 64-bit value \p LHS into two 32-bit components, and perform the
2485/// binary operation \p Opc to it with the corresponding constant operands.
2486SDValue AMDGPUTargetLowering::splitBinaryBitConstantOpImpl(
2487 DAGCombinerInfo &DCI, const SDLoc &SL,
2488 unsigned Opc, SDValue LHS,
2489 uint32_t ValLo, uint32_t ValHi) const {
Matt Arsenault6e3a4512016-01-18 22:01:13 +00002490 SelectionDAG &DAG = DCI.DAG;
Matt Arsenault6e3a4512016-01-18 22:01:13 +00002491 SDValue Lo, Hi;
Matt Arsenaultfa5f7672016-09-14 15:19:03 +00002492 std::tie(Lo, Hi) = split64BitValue(LHS, DAG);
Matt Arsenault6e3a4512016-01-18 22:01:13 +00002493
Matt Arsenaultfa5f7672016-09-14 15:19:03 +00002494 SDValue LoRHS = DAG.getConstant(ValLo, SL, MVT::i32);
2495 SDValue HiRHS = DAG.getConstant(ValHi, SL, MVT::i32);
Matt Arsenault6e3a4512016-01-18 22:01:13 +00002496
Matt Arsenaultfa5f7672016-09-14 15:19:03 +00002497 SDValue LoAnd = DAG.getNode(Opc, SL, MVT::i32, Lo, LoRHS);
2498 SDValue HiAnd = DAG.getNode(Opc, SL, MVT::i32, Hi, HiRHS);
Matt Arsenault6e3a4512016-01-18 22:01:13 +00002499
Matt Arsenaultefa3fe12016-04-22 22:48:38 +00002500 // Re-visit the ands. It's possible we eliminated one of them and it could
2501 // simplify the vector.
2502 DCI.AddToWorklist(Lo.getNode());
2503 DCI.AddToWorklist(Hi.getNode());
2504
Ahmed Bougacha128f8732016-04-26 21:15:30 +00002505 SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {LoAnd, HiAnd});
Matt Arsenault6e3a4512016-01-18 22:01:13 +00002506 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec);
2507}
2508
Matt Arsenault24692112015-07-14 18:20:33 +00002509SDValue AMDGPUTargetLowering::performShlCombine(SDNode *N,
2510 DAGCombinerInfo &DCI) const {
2511 if (N->getValueType(0) != MVT::i64)
2512 return SDValue();
2513
Matt Arsenault3cbbc102016-01-18 21:55:14 +00002514 // i64 (shl x, C) -> (build_pair 0, (shl x, C -32))
Matt Arsenault24692112015-07-14 18:20:33 +00002515
Matt Arsenault3cbbc102016-01-18 21:55:14 +00002516 // On some subtargets, 64-bit shift is a quarter rate instruction. In the
2517 // common case, splitting this into a move and a 32-bit shift is faster and
2518 // the same code size.
Matt Arsenault24692112015-07-14 18:20:33 +00002519 const ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N->getOperand(1));
Matt Arsenault3cbbc102016-01-18 21:55:14 +00002520 if (!RHS)
2521 return SDValue();
2522
2523 unsigned RHSVal = RHS->getZExtValue();
2524 if (RHSVal < 32)
Matt Arsenault24692112015-07-14 18:20:33 +00002525 return SDValue();
2526
2527 SDValue LHS = N->getOperand(0);
2528
2529 SDLoc SL(N);
2530 SelectionDAG &DAG = DCI.DAG;
2531
Matt Arsenault3cbbc102016-01-18 21:55:14 +00002532 SDValue ShiftAmt = DAG.getConstant(RHSVal - 32, SL, MVT::i32);
2533
Matt Arsenault24692112015-07-14 18:20:33 +00002534 SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LHS);
Matt Arsenault3cbbc102016-01-18 21:55:14 +00002535 SDValue NewShift = DAG.getNode(ISD::SHL, SL, MVT::i32, Lo, ShiftAmt);
Matt Arsenault24692112015-07-14 18:20:33 +00002536
2537 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
Matt Arsenault80edab92016-01-18 21:43:36 +00002538
Ahmed Bougacha128f8732016-04-26 21:15:30 +00002539 SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {Zero, NewShift});
Matt Arsenault3cbbc102016-01-18 21:55:14 +00002540 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec);
Matt Arsenault24692112015-07-14 18:20:33 +00002541}
2542
Matt Arsenault33e3ece2016-01-18 22:09:04 +00002543SDValue AMDGPUTargetLowering::performSraCombine(SDNode *N,
2544 DAGCombinerInfo &DCI) const {
2545 if (N->getValueType(0) != MVT::i64)
2546 return SDValue();
2547
2548 const ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N->getOperand(1));
2549 if (!RHS)
2550 return SDValue();
2551
2552 SelectionDAG &DAG = DCI.DAG;
2553 SDLoc SL(N);
2554 unsigned RHSVal = RHS->getZExtValue();
2555
2556 // (sra i64:x, 32) -> build_pair x, (sra hi_32(x), 31)
2557 if (RHSVal == 32) {
2558 SDValue Hi = getHiHalf64(N->getOperand(0), DAG);
2559 SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi,
2560 DAG.getConstant(31, SL, MVT::i32));
2561
Ahmed Bougacha128f8732016-04-26 21:15:30 +00002562 SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {Hi, NewShift});
Matt Arsenault33e3ece2016-01-18 22:09:04 +00002563 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec);
2564 }
2565
2566 // (sra i64:x, 63) -> build_pair (sra hi_32(x), 31), (sra hi_32(x), 31)
2567 if (RHSVal == 63) {
2568 SDValue Hi = getHiHalf64(N->getOperand(0), DAG);
2569 SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi,
2570 DAG.getConstant(31, SL, MVT::i32));
Ahmed Bougacha128f8732016-04-26 21:15:30 +00002571 SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, NewShift});
Matt Arsenault33e3ece2016-01-18 22:09:04 +00002572 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec);
2573 }
2574
2575 return SDValue();
2576}
2577
Matt Arsenault80edab92016-01-18 21:43:36 +00002578SDValue AMDGPUTargetLowering::performSrlCombine(SDNode *N,
2579 DAGCombinerInfo &DCI) const {
2580 if (N->getValueType(0) != MVT::i64)
2581 return SDValue();
2582
2583 const ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N->getOperand(1));
2584 if (!RHS)
2585 return SDValue();
2586
2587 unsigned ShiftAmt = RHS->getZExtValue();
2588 if (ShiftAmt < 32)
2589 return SDValue();
2590
2591 // srl i64:x, C for C >= 32
2592 // =>
2593 // build_pair (srl hi_32(x), C - 32), 0
2594
2595 SelectionDAG &DAG = DCI.DAG;
2596 SDLoc SL(N);
2597
2598 SDValue One = DAG.getConstant(1, SL, MVT::i32);
2599 SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
2600
2601 SDValue VecOp = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, N->getOperand(0));
2602 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32,
2603 VecOp, One);
2604
2605 SDValue NewConst = DAG.getConstant(ShiftAmt - 32, SL, MVT::i32);
2606 SDValue NewShift = DAG.getNode(ISD::SRL, SL, MVT::i32, Hi, NewConst);
2607
Ahmed Bougacha128f8732016-04-26 21:15:30 +00002608 SDValue BuildPair = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, Zero});
Matt Arsenault80edab92016-01-18 21:43:36 +00002609
2610 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildPair);
2611}
2612
Matt Arsenault2712d4a2016-08-27 01:32:27 +00002613// We need to specifically handle i64 mul here to avoid unnecessary conversion
2614// instructions. If we only match on the legalized i64 mul expansion,
2615// SimplifyDemandedBits will be unable to remove them because there will be
2616// multiple uses due to the separate mul + mulh[su].
2617static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL,
2618 SDValue N0, SDValue N1, unsigned Size, bool Signed) {
2619 if (Size <= 32) {
2620 unsigned MulOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
2621 return DAG.getNode(MulOpc, SL, MVT::i32, N0, N1);
2622 }
2623
2624 // Because we want to eliminate extension instructions before the
2625 // operation, we need to create a single user here (i.e. not the separate
2626 // mul_lo + mul_hi) so that SimplifyDemandedBits will deal with it.
2627
2628 unsigned MulOpc = Signed ? AMDGPUISD::MUL_LOHI_I24 : AMDGPUISD::MUL_LOHI_U24;
2629
2630 SDValue Mul = DAG.getNode(MulOpc, SL,
2631 DAG.getVTList(MVT::i32, MVT::i32), N0, N1);
2632
2633 return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64,
2634 Mul.getValue(0), Mul.getValue(1));
2635}
2636
Matt Arsenaultd0e0f0a2014-06-30 17:55:48 +00002637SDValue AMDGPUTargetLowering::performMulCombine(SDNode *N,
2638 DAGCombinerInfo &DCI) const {
2639 EVT VT = N->getValueType(0);
2640
Matt Arsenault2712d4a2016-08-27 01:32:27 +00002641 unsigned Size = VT.getSizeInBits();
2642 if (VT.isVector() || Size > 64)
Matt Arsenaultd0e0f0a2014-06-30 17:55:48 +00002643 return SDValue();
2644
Tom Stellard115a6152016-11-10 16:02:37 +00002645 // There are i16 integer mul/mad.
2646 if (Subtarget->has16BitInsts() && VT.getScalarType().bitsLE(MVT::i16))
2647 return SDValue();
2648
Matt Arsenaultd0e0f0a2014-06-30 17:55:48 +00002649 SelectionDAG &DAG = DCI.DAG;
2650 SDLoc DL(N);
2651
2652 SDValue N0 = N->getOperand(0);
2653 SDValue N1 = N->getOperand(1);
2654 SDValue Mul;
2655
2656 if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) {
2657 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32);
2658 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32);
Matt Arsenault2712d4a2016-08-27 01:32:27 +00002659 Mul = getMul24(DAG, DL, N0, N1, Size, false);
Matt Arsenaultd0e0f0a2014-06-30 17:55:48 +00002660 } else if (Subtarget->hasMulI24() && isI24(N0, DAG) && isI24(N1, DAG)) {
2661 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32);
2662 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32);
Matt Arsenault2712d4a2016-08-27 01:32:27 +00002663 Mul = getMul24(DAG, DL, N0, N1, Size, true);
Matt Arsenaultd0e0f0a2014-06-30 17:55:48 +00002664 } else {
2665 return SDValue();
2666 }
2667
2668 // We need to use sext even for MUL_U24, because MUL_U24 is used
2669 // for signed multiply of 8 and 16-bit types.
2670 return DAG.getSExtOrTrunc(Mul, DL, VT);
2671}
2672
Matt Arsenault2712d4a2016-08-27 01:32:27 +00002673SDValue AMDGPUTargetLowering::performMulhsCombine(SDNode *N,
2674 DAGCombinerInfo &DCI) const {
2675 EVT VT = N->getValueType(0);
2676
2677 if (!Subtarget->hasMulI24() || VT.isVector())
2678 return SDValue();
2679
2680 SelectionDAG &DAG = DCI.DAG;
2681 SDLoc DL(N);
2682
2683 SDValue N0 = N->getOperand(0);
2684 SDValue N1 = N->getOperand(1);
2685
2686 if (!isI24(N0, DAG) || !isI24(N1, DAG))
2687 return SDValue();
2688
2689 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32);
2690 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32);
2691
2692 SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_I24, DL, MVT::i32, N0, N1);
2693 DCI.AddToWorklist(Mulhi.getNode());
2694 return DAG.getSExtOrTrunc(Mulhi, DL, VT);
2695}
2696
2697SDValue AMDGPUTargetLowering::performMulhuCombine(SDNode *N,
2698 DAGCombinerInfo &DCI) const {
2699 EVT VT = N->getValueType(0);
2700
2701 if (!Subtarget->hasMulU24() || VT.isVector() || VT.getSizeInBits() > 32)
2702 return SDValue();
2703
2704 SelectionDAG &DAG = DCI.DAG;
2705 SDLoc DL(N);
2706
2707 SDValue N0 = N->getOperand(0);
2708 SDValue N1 = N->getOperand(1);
2709
2710 if (!isU24(N0, DAG) || !isU24(N1, DAG))
2711 return SDValue();
2712
2713 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32);
2714 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32);
2715
2716 SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_U24, DL, MVT::i32, N0, N1);
2717 DCI.AddToWorklist(Mulhi.getNode());
2718 return DAG.getZExtOrTrunc(Mulhi, DL, VT);
2719}
2720
2721SDValue AMDGPUTargetLowering::performMulLoHi24Combine(
2722 SDNode *N, DAGCombinerInfo &DCI) const {
2723 SelectionDAG &DAG = DCI.DAG;
2724
Tom Stellard09c2bd62016-10-14 19:14:29 +00002725 // Simplify demanded bits before splitting into multiple users.
2726 if (simplifyI24(N, 0, DCI) || simplifyI24(N, 1, DCI))
2727 return SDValue();
2728
Matt Arsenault2712d4a2016-08-27 01:32:27 +00002729 SDValue N0 = N->getOperand(0);
2730 SDValue N1 = N->getOperand(1);
2731
Matt Arsenault2712d4a2016-08-27 01:32:27 +00002732 bool Signed = (N->getOpcode() == AMDGPUISD::MUL_LOHI_I24);
2733
2734 unsigned MulLoOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
2735 unsigned MulHiOpc = Signed ? AMDGPUISD::MULHI_I24 : AMDGPUISD::MULHI_U24;
2736
2737 SDLoc SL(N);
2738
2739 SDValue MulLo = DAG.getNode(MulLoOpc, SL, MVT::i32, N0, N1);
2740 SDValue MulHi = DAG.getNode(MulHiOpc, SL, MVT::i32, N0, N1);
2741 return DAG.getMergeValues({ MulLo, MulHi }, SL);
2742}
2743
Matt Arsenaultde5fbe92016-01-11 17:02:00 +00002744static bool isNegativeOne(SDValue Val) {
2745 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val))
2746 return C->isAllOnesValue();
2747 return false;
2748}
2749
2750static bool isCtlzOpc(unsigned Opc) {
2751 return Opc == ISD::CTLZ || Opc == ISD::CTLZ_ZERO_UNDEF;
2752}
2753
Konstantin Zhuravlyovd971a112016-11-01 17:49:33 +00002754SDValue AMDGPUTargetLowering::getFFBH_U32(SelectionDAG &DAG,
2755 SDValue Op,
2756 const SDLoc &DL) const {
Matt Arsenault5319b0a2016-01-11 17:02:06 +00002757 EVT VT = Op.getValueType();
Konstantin Zhuravlyovd971a112016-11-01 17:49:33 +00002758 EVT LegalVT = getTypeToTransformTo(*DAG.getContext(), VT);
2759 if (LegalVT != MVT::i32 && (Subtarget->has16BitInsts() &&
2760 LegalVT != MVT::i16))
Matt Arsenault5319b0a2016-01-11 17:02:06 +00002761 return SDValue();
2762
2763 if (VT != MVT::i32)
Konstantin Zhuravlyovd971a112016-11-01 17:49:33 +00002764 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Op);
Matt Arsenault5319b0a2016-01-11 17:02:06 +00002765
Konstantin Zhuravlyovd971a112016-11-01 17:49:33 +00002766 SDValue FFBH = DAG.getNode(AMDGPUISD::FFBH_U32, DL, MVT::i32, Op);
Matt Arsenault5319b0a2016-01-11 17:02:06 +00002767 if (VT != MVT::i32)
Konstantin Zhuravlyovd971a112016-11-01 17:49:33 +00002768 FFBH = DAG.getNode(ISD::TRUNCATE, DL, VT, FFBH);
Matt Arsenault5319b0a2016-01-11 17:02:06 +00002769
2770 return FFBH;
2771}
2772
Matt Arsenaultde5fbe92016-01-11 17:02:00 +00002773// The native instructions return -1 on 0 input. Optimize out a select that
2774// produces -1 on 0.
2775//
2776// TODO: If zero is not undef, we could also do this if the output is compared
2777// against the bitwidth.
2778//
2779// TODO: Should probably combine against FFBH_U32 instead of ctlz directly.
Benjamin Kramerbdc49562016-06-12 15:39:02 +00002780SDValue AMDGPUTargetLowering::performCtlzCombine(const SDLoc &SL, SDValue Cond,
2781 SDValue LHS, SDValue RHS,
Matt Arsenaultde5fbe92016-01-11 17:02:00 +00002782 DAGCombinerInfo &DCI) const {
2783 ConstantSDNode *CmpRhs = dyn_cast<ConstantSDNode>(Cond.getOperand(1));
2784 if (!CmpRhs || !CmpRhs->isNullValue())
2785 return SDValue();
2786
2787 SelectionDAG &DAG = DCI.DAG;
2788 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
2789 SDValue CmpLHS = Cond.getOperand(0);
2790
2791 // select (setcc x, 0, eq), -1, (ctlz_zero_undef x) -> ffbh_u32 x
2792 if (CCOpcode == ISD::SETEQ &&
2793 isCtlzOpc(RHS.getOpcode()) &&
2794 RHS.getOperand(0) == CmpLHS &&
2795 isNegativeOne(LHS)) {
Konstantin Zhuravlyovd971a112016-11-01 17:49:33 +00002796 return getFFBH_U32(DAG, CmpLHS, SL);
Matt Arsenaultde5fbe92016-01-11 17:02:00 +00002797 }
2798
2799 // select (setcc x, 0, ne), (ctlz_zero_undef x), -1 -> ffbh_u32 x
2800 if (CCOpcode == ISD::SETNE &&
2801 isCtlzOpc(LHS.getOpcode()) &&
2802 LHS.getOperand(0) == CmpLHS &&
2803 isNegativeOne(RHS)) {
Konstantin Zhuravlyovd971a112016-11-01 17:49:33 +00002804 return getFFBH_U32(DAG, CmpLHS, SL);
Matt Arsenaultde5fbe92016-01-11 17:02:00 +00002805 }
2806
2807 return SDValue();
2808}
2809
Matt Arsenault2a04ff92017-01-11 23:57:38 +00002810static SDValue distributeOpThroughSelect(TargetLowering::DAGCombinerInfo &DCI,
2811 unsigned Op,
2812 const SDLoc &SL,
2813 SDValue Cond,
2814 SDValue N1,
2815 SDValue N2) {
2816 SelectionDAG &DAG = DCI.DAG;
2817 EVT VT = N1.getValueType();
2818
2819 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT, Cond,
2820 N1.getOperand(0), N2.getOperand(0));
2821 DCI.AddToWorklist(NewSelect.getNode());
2822 return DAG.getNode(Op, SL, VT, NewSelect);
2823}
2824
2825// Pull a free FP operation out of a select so it may fold into uses.
2826//
2827// select c, (fneg x), (fneg y) -> fneg (select c, x, y)
2828// select c, (fneg x), k -> fneg (select c, x, (fneg k))
2829//
2830// select c, (fabs x), (fabs y) -> fabs (select c, x, y)
2831// select c, (fabs x), +k -> fabs (select c, x, k)
2832static SDValue foldFreeOpFromSelect(TargetLowering::DAGCombinerInfo &DCI,
2833 SDValue N) {
2834 SelectionDAG &DAG = DCI.DAG;
2835 SDValue Cond = N.getOperand(0);
2836 SDValue LHS = N.getOperand(1);
2837 SDValue RHS = N.getOperand(2);
2838
2839 EVT VT = N.getValueType();
2840 if ((LHS.getOpcode() == ISD::FABS && RHS.getOpcode() == ISD::FABS) ||
2841 (LHS.getOpcode() == ISD::FNEG && RHS.getOpcode() == ISD::FNEG)) {
2842 return distributeOpThroughSelect(DCI, LHS.getOpcode(),
2843 SDLoc(N), Cond, LHS, RHS);
2844 }
2845
2846 bool Inv = false;
2847 if (RHS.getOpcode() == ISD::FABS || RHS.getOpcode() == ISD::FNEG) {
2848 std::swap(LHS, RHS);
2849 Inv = true;
2850 }
2851
2852 // TODO: Support vector constants.
2853 ConstantFPSDNode *CRHS = dyn_cast<ConstantFPSDNode>(RHS);
2854 if ((LHS.getOpcode() == ISD::FNEG || LHS.getOpcode() == ISD::FABS) && CRHS) {
2855 SDLoc SL(N);
2856 // If one side is an fneg/fabs and the other is a constant, we can push the
2857 // fneg/fabs down. If it's an fabs, the constant needs to be non-negative.
2858 SDValue NewLHS = LHS.getOperand(0);
2859 SDValue NewRHS = RHS;
2860
Matt Arsenault45337df2017-01-12 18:58:15 +00002861 // Careful: if the neg can be folded up, don't try to pull it back down.
2862 bool ShouldFoldNeg = true;
Matt Arsenault2a04ff92017-01-11 23:57:38 +00002863
Matt Arsenault45337df2017-01-12 18:58:15 +00002864 if (NewLHS.hasOneUse()) {
2865 unsigned Opc = NewLHS.getOpcode();
2866 if (LHS.getOpcode() == ISD::FNEG && fnegFoldsIntoOp(Opc))
2867 ShouldFoldNeg = false;
2868 if (LHS.getOpcode() == ISD::FABS && Opc == ISD::FMUL)
2869 ShouldFoldNeg = false;
2870 }
Matt Arsenault2a04ff92017-01-11 23:57:38 +00002871
Matt Arsenault45337df2017-01-12 18:58:15 +00002872 if (ShouldFoldNeg) {
2873 if (LHS.getOpcode() == ISD::FNEG)
2874 NewRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
2875 else if (CRHS->isNegative())
2876 return SDValue();
Matt Arsenault2a04ff92017-01-11 23:57:38 +00002877
Matt Arsenault45337df2017-01-12 18:58:15 +00002878 if (Inv)
2879 std::swap(NewLHS, NewRHS);
2880
2881 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT,
2882 Cond, NewLHS, NewRHS);
2883 DCI.AddToWorklist(NewSelect.getNode());
2884 return DAG.getNode(LHS.getOpcode(), SL, VT, NewSelect);
2885 }
Matt Arsenault2a04ff92017-01-11 23:57:38 +00002886 }
2887
2888 return SDValue();
2889}
2890
2891
Matt Arsenaultde5fbe92016-01-11 17:02:00 +00002892SDValue AMDGPUTargetLowering::performSelectCombine(SDNode *N,
2893 DAGCombinerInfo &DCI) const {
Matt Arsenault2a04ff92017-01-11 23:57:38 +00002894 if (SDValue Folded = foldFreeOpFromSelect(DCI, SDValue(N, 0)))
2895 return Folded;
2896
Matt Arsenaultde5fbe92016-01-11 17:02:00 +00002897 SDValue Cond = N->getOperand(0);
2898 if (Cond.getOpcode() != ISD::SETCC)
2899 return SDValue();
2900
2901 EVT VT = N->getValueType(0);
2902 SDValue LHS = Cond.getOperand(0);
2903 SDValue RHS = Cond.getOperand(1);
2904 SDValue CC = Cond.getOperand(2);
2905
2906 SDValue True = N->getOperand(1);
2907 SDValue False = N->getOperand(2);
2908
Matt Arsenault0b26e472016-12-22 21:40:08 +00002909 if (Cond.hasOneUse()) { // TODO: Look for multiple select uses.
2910 SelectionDAG &DAG = DCI.DAG;
2911 if ((DAG.isConstantValueOfAnyType(True) ||
2912 DAG.isConstantValueOfAnyType(True)) &&
2913 (!DAG.isConstantValueOfAnyType(False) &&
2914 !DAG.isConstantValueOfAnyType(False))) {
2915 // Swap cmp + select pair to move constant to false input.
2916 // This will allow using VOPC cndmasks more often.
2917 // select (setcc x, y), k, x -> select (setcc y, x) x, x
2918
2919 SDLoc SL(N);
2920 ISD::CondCode NewCC = getSetCCInverse(cast<CondCodeSDNode>(CC)->get(),
2921 LHS.getValueType().isInteger());
2922
2923 SDValue NewCond = DAG.getSetCC(SL, Cond.getValueType(), LHS, RHS, NewCC);
2924 return DAG.getNode(ISD::SELECT, SL, VT, NewCond, False, True);
2925 }
Matt Arsenault0b26e472016-12-22 21:40:08 +00002926
Matt Arsenaultda7a6562017-02-01 00:42:40 +00002927 if (VT == MVT::f32 && Subtarget->hasFminFmaxLegacy()) {
2928 SDValue MinMax
2929 = combineFMinMaxLegacy(SDLoc(N), VT, LHS, RHS, True, False, CC, DCI);
2930 // Revisit this node so we can catch min3/max3/med3 patterns.
2931 //DCI.AddToWorklist(MinMax.getNode());
2932 return MinMax;
2933 }
Matt Arsenault5b39b342016-01-28 20:53:48 +00002934 }
Matt Arsenaultde5fbe92016-01-11 17:02:00 +00002935
2936 // There's no reason to not do this if the condition has other uses.
Matt Arsenault5319b0a2016-01-11 17:02:06 +00002937 return performCtlzCombine(SDLoc(N), Cond, True, False, DCI);
Matt Arsenaultde5fbe92016-01-11 17:02:00 +00002938}
2939
Matt Arsenault2511c032017-02-03 00:23:15 +00002940static bool isConstantFPZero(SDValue N) {
2941 if (const ConstantFPSDNode *C = isConstOrConstSplatFP(N))
2942 return C->isZero() && !C->isNegative();
2943 return false;
2944}
2945
Matt Arsenaulte1b59532017-02-03 00:51:50 +00002946static unsigned inverseMinMax(unsigned Opc) {
2947 switch (Opc) {
2948 case ISD::FMAXNUM:
2949 return ISD::FMINNUM;
2950 case ISD::FMINNUM:
2951 return ISD::FMAXNUM;
2952 case AMDGPUISD::FMAX_LEGACY:
2953 return AMDGPUISD::FMIN_LEGACY;
2954 case AMDGPUISD::FMIN_LEGACY:
2955 return AMDGPUISD::FMAX_LEGACY;
2956 default:
2957 llvm_unreachable("invalid min/max opcode");
2958 }
2959}
2960
Matt Arsenault2529fba2017-01-12 00:09:34 +00002961SDValue AMDGPUTargetLowering::performFNegCombine(SDNode *N,
2962 DAGCombinerInfo &DCI) const {
2963 SelectionDAG &DAG = DCI.DAG;
2964 SDValue N0 = N->getOperand(0);
2965 EVT VT = N->getValueType(0);
2966
2967 unsigned Opc = N0.getOpcode();
2968
2969 // If the input has multiple uses and we can either fold the negate down, or
2970 // the other uses cannot, give up. This both prevents unprofitable
2971 // transformations and infinite loops: we won't repeatedly try to fold around
2972 // a negate that has no 'good' form.
Matt Arsenaulta8fcfad2017-02-02 23:21:23 +00002973 if (N0.hasOneUse()) {
2974 // This may be able to fold into the source, but at a code size cost. Don't
2975 // fold if the fold into the user is free.
2976 if (allUsesHaveSourceMods(N, 0))
2977 return SDValue();
2978 } else {
2979 if (fnegFoldsIntoOp(Opc) &&
2980 (allUsesHaveSourceMods(N) || !allUsesHaveSourceMods(N0.getNode())))
2981 return SDValue();
2982 }
Matt Arsenault2529fba2017-01-12 00:09:34 +00002983
2984 SDLoc SL(N);
2985 switch (Opc) {
2986 case ISD::FADD: {
Matt Arsenault3e6f9b52017-01-19 06:35:27 +00002987 if (!mayIgnoreSignedZero(N0))
2988 return SDValue();
2989
Matt Arsenault2529fba2017-01-12 00:09:34 +00002990 // (fneg (fadd x, y)) -> (fadd (fneg x), (fneg y))
2991 SDValue LHS = N0.getOperand(0);
2992 SDValue RHS = N0.getOperand(1);
2993
2994 if (LHS.getOpcode() != ISD::FNEG)
2995 LHS = DAG.getNode(ISD::FNEG, SL, VT, LHS);
2996 else
2997 LHS = LHS.getOperand(0);
2998
2999 if (RHS.getOpcode() != ISD::FNEG)
3000 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
3001 else
3002 RHS = RHS.getOperand(0);
3003
Matt Arsenault7b49ad72017-01-23 19:08:34 +00003004 SDValue Res = DAG.getNode(ISD::FADD, SL, VT, LHS, RHS, N0->getFlags());
Matt Arsenault2529fba2017-01-12 00:09:34 +00003005 if (!N0.hasOneUse())
3006 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
3007 return Res;
3008 }
Matt Arsenaulta8c325e2017-01-12 18:26:30 +00003009 case ISD::FMUL:
3010 case AMDGPUISD::FMUL_LEGACY: {
Matt Arsenault4103a812017-01-12 00:23:20 +00003011 // (fneg (fmul x, y)) -> (fmul x, (fneg y))
Matt Arsenaulta8c325e2017-01-12 18:26:30 +00003012 // (fneg (fmul_legacy x, y)) -> (fmul_legacy x, (fneg y))
Matt Arsenault4103a812017-01-12 00:23:20 +00003013 SDValue LHS = N0.getOperand(0);
3014 SDValue RHS = N0.getOperand(1);
3015
3016 if (LHS.getOpcode() == ISD::FNEG)
3017 LHS = LHS.getOperand(0);
3018 else if (RHS.getOpcode() == ISD::FNEG)
3019 RHS = RHS.getOperand(0);
3020 else
3021 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
3022
Matt Arsenault7b49ad72017-01-23 19:08:34 +00003023 SDValue Res = DAG.getNode(Opc, SL, VT, LHS, RHS, N0->getFlags());
Matt Arsenault4103a812017-01-12 00:23:20 +00003024 if (!N0.hasOneUse())
3025 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
3026 return Res;
3027 }
Matt Arsenault63f95372017-01-12 00:32:16 +00003028 case ISD::FMA:
3029 case ISD::FMAD: {
Matt Arsenault3e6f9b52017-01-19 06:35:27 +00003030 if (!mayIgnoreSignedZero(N0))
3031 return SDValue();
3032
Matt Arsenault63f95372017-01-12 00:32:16 +00003033 // (fneg (fma x, y, z)) -> (fma x, (fneg y), (fneg z))
3034 SDValue LHS = N0.getOperand(0);
3035 SDValue MHS = N0.getOperand(1);
3036 SDValue RHS = N0.getOperand(2);
3037
3038 if (LHS.getOpcode() == ISD::FNEG)
3039 LHS = LHS.getOperand(0);
3040 else if (MHS.getOpcode() == ISD::FNEG)
3041 MHS = MHS.getOperand(0);
3042 else
3043 MHS = DAG.getNode(ISD::FNEG, SL, VT, MHS);
3044
3045 if (RHS.getOpcode() != ISD::FNEG)
3046 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
3047 else
3048 RHS = RHS.getOperand(0);
3049
3050 SDValue Res = DAG.getNode(Opc, SL, VT, LHS, MHS, RHS);
3051 if (!N0.hasOneUse())
3052 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
3053 return Res;
3054 }
Matt Arsenault2511c032017-02-03 00:23:15 +00003055 case ISD::FMAXNUM:
Matt Arsenaulte1b59532017-02-03 00:51:50 +00003056 case ISD::FMINNUM:
3057 case AMDGPUISD::FMAX_LEGACY:
3058 case AMDGPUISD::FMIN_LEGACY: {
Matt Arsenault2511c032017-02-03 00:23:15 +00003059 // fneg (fmaxnum x, y) -> fminnum (fneg x), (fneg y)
3060 // fneg (fminnum x, y) -> fmaxnum (fneg x), (fneg y)
Matt Arsenaulte1b59532017-02-03 00:51:50 +00003061 // fneg (fmax_legacy x, y) -> fmin_legacy (fneg x), (fneg y)
3062 // fneg (fmin_legacy x, y) -> fmax_legacy (fneg x), (fneg y)
3063
Matt Arsenault2511c032017-02-03 00:23:15 +00003064 SDValue LHS = N0.getOperand(0);
3065 SDValue RHS = N0.getOperand(1);
3066
3067 // 0 doesn't have a negated inline immediate.
3068 // TODO: Shouldn't fold 1/2pi either, and should be generalized to other
3069 // operations.
3070 if (isConstantFPZero(RHS))
3071 return SDValue();
3072
3073 SDValue NegLHS = DAG.getNode(ISD::FNEG, SL, VT, LHS);
3074 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
Matt Arsenaulte1b59532017-02-03 00:51:50 +00003075 unsigned Opposite = inverseMinMax(Opc);
Matt Arsenault2511c032017-02-03 00:23:15 +00003076
3077 SDValue Res = DAG.getNode(Opposite, SL, VT, NegLHS, NegRHS, N0->getFlags());
3078 if (!N0.hasOneUse())
3079 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
3080 return Res;
3081 }
Matt Arsenaultff7e5aa2017-01-12 17:46:35 +00003082 case ISD::FP_EXTEND:
Matt Arsenault53f0cc22017-01-26 01:25:36 +00003083 case ISD::FTRUNC:
3084 case ISD::FRINT:
3085 case ISD::FNEARBYINT: // XXX - Should fround be handled?
3086 case ISD::FSIN:
Matt Arsenaultff7e5aa2017-01-12 17:46:35 +00003087 case AMDGPUISD::RCP:
Matt Arsenault31c039e2017-01-12 18:48:09 +00003088 case AMDGPUISD::RCP_LEGACY:
Matt Arsenault31c039e2017-01-12 18:48:09 +00003089 case AMDGPUISD::SIN_HW: {
Matt Arsenault98d2bf102017-01-12 17:46:28 +00003090 SDValue CvtSrc = N0.getOperand(0);
3091 if (CvtSrc.getOpcode() == ISD::FNEG) {
3092 // (fneg (fp_extend (fneg x))) -> (fp_extend x)
Matt Arsenaultff7e5aa2017-01-12 17:46:35 +00003093 // (fneg (rcp (fneg x))) -> (rcp x)
Matt Arsenault4242d482017-01-12 17:46:33 +00003094 return DAG.getNode(Opc, SL, VT, CvtSrc.getOperand(0));
Matt Arsenault98d2bf102017-01-12 17:46:28 +00003095 }
3096
3097 if (!N0.hasOneUse())
3098 return SDValue();
3099
3100 // (fneg (fp_extend x)) -> (fp_extend (fneg x))
Matt Arsenaultff7e5aa2017-01-12 17:46:35 +00003101 // (fneg (rcp x)) -> (rcp (fneg x))
Matt Arsenault98d2bf102017-01-12 17:46:28 +00003102 SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc);
Matt Arsenault7b49ad72017-01-23 19:08:34 +00003103 return DAG.getNode(Opc, SL, VT, Neg, N0->getFlags());
Matt Arsenault4242d482017-01-12 17:46:33 +00003104 }
3105 case ISD::FP_ROUND: {
3106 SDValue CvtSrc = N0.getOperand(0);
3107
3108 if (CvtSrc.getOpcode() == ISD::FNEG) {
3109 // (fneg (fp_round (fneg x))) -> (fp_round x)
3110 return DAG.getNode(ISD::FP_ROUND, SL, VT,
3111 CvtSrc.getOperand(0), N0.getOperand(1));
3112 }
3113
3114 if (!N0.hasOneUse())
3115 return SDValue();
3116
3117 // (fneg (fp_round x)) -> (fp_round (fneg x))
3118 SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc);
3119 return DAG.getNode(ISD::FP_ROUND, SL, VT, Neg, N0.getOperand(1));
Matt Arsenault98d2bf102017-01-12 17:46:28 +00003120 }
Matt Arsenault9dba9bd2017-02-02 02:27:04 +00003121 case ISD::FP16_TO_FP: {
3122 // v_cvt_f32_f16 supports source modifiers on pre-VI targets without legal
3123 // f16, but legalization of f16 fneg ends up pulling it out of the source.
3124 // Put the fneg back as a legal source operation that can be matched later.
3125 SDLoc SL(N);
3126
3127 SDValue Src = N0.getOperand(0);
3128 EVT SrcVT = Src.getValueType();
3129
3130 // fneg (fp16_to_fp x) -> fp16_to_fp (xor x, 0x8000)
3131 SDValue IntFNeg = DAG.getNode(ISD::XOR, SL, SrcVT, Src,
3132 DAG.getConstant(0x8000, SL, SrcVT));
3133 return DAG.getNode(ISD::FP16_TO_FP, SL, N->getValueType(0), IntFNeg);
3134 }
3135 default:
3136 return SDValue();
3137 }
3138}
3139
3140SDValue AMDGPUTargetLowering::performFAbsCombine(SDNode *N,
3141 DAGCombinerInfo &DCI) const {
3142 SelectionDAG &DAG = DCI.DAG;
3143 SDValue N0 = N->getOperand(0);
3144
3145 if (!N0.hasOneUse())
3146 return SDValue();
3147
3148 switch (N0.getOpcode()) {
3149 case ISD::FP16_TO_FP: {
3150 assert(!Subtarget->has16BitInsts() && "should only see if f16 is illegal");
3151 SDLoc SL(N);
3152 SDValue Src = N0.getOperand(0);
3153 EVT SrcVT = Src.getValueType();
3154
3155 // fabs (fp16_to_fp x) -> fp16_to_fp (and x, 0x7fff)
3156 SDValue IntFAbs = DAG.getNode(ISD::AND, SL, SrcVT, Src,
3157 DAG.getConstant(0x7fff, SL, SrcVT));
3158 return DAG.getNode(ISD::FP16_TO_FP, SL, N->getValueType(0), IntFAbs);
3159 }
Matt Arsenault2529fba2017-01-12 00:09:34 +00003160 default:
3161 return SDValue();
3162 }
3163}
3164
Tom Stellard50122a52014-04-07 19:45:41 +00003165SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
Matt Arsenaultca3976f2014-07-15 02:06:31 +00003166 DAGCombinerInfo &DCI) const {
Tom Stellard50122a52014-04-07 19:45:41 +00003167 SelectionDAG &DAG = DCI.DAG;
3168 SDLoc DL(N);
3169
3170 switch(N->getOpcode()) {
Matt Arsenault24e33d12015-07-03 23:33:38 +00003171 default:
3172 break;
Matt Arsenault79003342016-04-14 21:58:07 +00003173 case ISD::BITCAST: {
3174 EVT DestVT = N->getValueType(0);
Matt Arsenaultd99ef112016-09-17 15:44:16 +00003175
3176 // Push casts through vector builds. This helps avoid emitting a large
3177 // number of copies when materializing floating point vector constants.
3178 //
3179 // vNt1 bitcast (vNt0 (build_vector t0:x, t0:y)) =>
3180 // vnt1 = build_vector (t1 (bitcast t0:x)), (t1 (bitcast t0:y))
3181 if (DestVT.isVector()) {
3182 SDValue Src = N->getOperand(0);
3183 if (Src.getOpcode() == ISD::BUILD_VECTOR) {
3184 EVT SrcVT = Src.getValueType();
3185 unsigned NElts = DestVT.getVectorNumElements();
3186
3187 if (SrcVT.getVectorNumElements() == NElts) {
3188 EVT DestEltVT = DestVT.getVectorElementType();
3189
3190 SmallVector<SDValue, 8> CastedElts;
3191 SDLoc SL(N);
3192 for (unsigned I = 0, E = SrcVT.getVectorNumElements(); I != E; ++I) {
3193 SDValue Elt = Src.getOperand(I);
3194 CastedElts.push_back(DAG.getNode(ISD::BITCAST, DL, DestEltVT, Elt));
3195 }
3196
3197 return DAG.getBuildVector(DestVT, SL, CastedElts);
3198 }
3199 }
3200 }
3201
Matt Arsenault79003342016-04-14 21:58:07 +00003202 if (DestVT.getSizeInBits() != 64 && !DestVT.isVector())
3203 break;
3204
3205 // Fold bitcasts of constants.
3206 //
3207 // v2i32 (bitcast i64:k) -> build_vector lo_32(k), hi_32(k)
3208 // TODO: Generalize and move to DAGCombiner
3209 SDValue Src = N->getOperand(0);
3210 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Src)) {
3211 assert(Src.getValueType() == MVT::i64);
3212 SDLoc SL(N);
3213 uint64_t CVal = C->getZExtValue();
3214 return DAG.getNode(ISD::BUILD_VECTOR, SL, DestVT,
3215 DAG.getConstant(Lo_32(CVal), SL, MVT::i32),
3216 DAG.getConstant(Hi_32(CVal), SL, MVT::i32));
3217 }
3218
3219 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Src)) {
3220 const APInt &Val = C->getValueAPF().bitcastToAPInt();
3221 SDLoc SL(N);
3222 uint64_t CVal = Val.getZExtValue();
3223 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32,
3224 DAG.getConstant(Lo_32(CVal), SL, MVT::i32),
3225 DAG.getConstant(Hi_32(CVal), SL, MVT::i32));
3226
3227 return DAG.getNode(ISD::BITCAST, SL, DestVT, Vec);
3228 }
3229
3230 break;
3231 }
Matt Arsenault24692112015-07-14 18:20:33 +00003232 case ISD::SHL: {
3233 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
3234 break;
3235
3236 return performShlCombine(N, DCI);
3237 }
Matt Arsenault80edab92016-01-18 21:43:36 +00003238 case ISD::SRL: {
3239 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
3240 break;
3241
3242 return performSrlCombine(N, DCI);
3243 }
Matt Arsenault33e3ece2016-01-18 22:09:04 +00003244 case ISD::SRA: {
3245 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
3246 break;
3247
3248 return performSraCombine(N, DCI);
3249 }
Matt Arsenault24e33d12015-07-03 23:33:38 +00003250 case ISD::MUL:
3251 return performMulCombine(N, DCI);
Matt Arsenault2712d4a2016-08-27 01:32:27 +00003252 case ISD::MULHS:
3253 return performMulhsCombine(N, DCI);
3254 case ISD::MULHU:
3255 return performMulhuCombine(N, DCI);
Matt Arsenault24e33d12015-07-03 23:33:38 +00003256 case AMDGPUISD::MUL_I24:
Matt Arsenault2712d4a2016-08-27 01:32:27 +00003257 case AMDGPUISD::MUL_U24:
3258 case AMDGPUISD::MULHI_I24:
3259 case AMDGPUISD::MULHI_U24: {
Tom Stellard6c7dd982016-10-21 20:25:11 +00003260 // If the first call to simplify is successfull, then N may end up being
3261 // deleted, so we shouldn't call simplifyI24 again.
3262 simplifyI24(N, 0, DCI) || simplifyI24(N, 1, DCI);
Matt Arsenault24e33d12015-07-03 23:33:38 +00003263 return SDValue();
3264 }
Matt Arsenault2712d4a2016-08-27 01:32:27 +00003265 case AMDGPUISD::MUL_LOHI_I24:
3266 case AMDGPUISD::MUL_LOHI_U24:
3267 return performMulLoHi24Combine(N, DCI);
Matt Arsenaultde5fbe92016-01-11 17:02:00 +00003268 case ISD::SELECT:
3269 return performSelectCombine(N, DCI);
Matt Arsenault2529fba2017-01-12 00:09:34 +00003270 case ISD::FNEG:
3271 return performFNegCombine(N, DCI);
Matt Arsenault9dba9bd2017-02-02 02:27:04 +00003272 case ISD::FABS:
3273 return performFAbsCombine(N, DCI);
Matt Arsenault5565f65e2014-05-22 18:09:07 +00003274 case AMDGPUISD::BFE_I32:
3275 case AMDGPUISD::BFE_U32: {
3276 assert(!N->getValueType(0).isVector() &&
3277 "Vector handling of BFE not implemented");
3278 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2));
3279 if (!Width)
3280 break;
3281
3282 uint32_t WidthVal = Width->getZExtValue() & 0x1f;
3283 if (WidthVal == 0)
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003284 return DAG.getConstant(0, DL, MVT::i32);
Matt Arsenault5565f65e2014-05-22 18:09:07 +00003285
3286 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
3287 if (!Offset)
3288 break;
3289
3290 SDValue BitsFrom = N->getOperand(0);
3291 uint32_t OffsetVal = Offset->getZExtValue() & 0x1f;
3292
3293 bool Signed = N->getOpcode() == AMDGPUISD::BFE_I32;
3294
3295 if (OffsetVal == 0) {
3296 // This is already sign / zero extended, so try to fold away extra BFEs.
3297 unsigned SignBits = Signed ? (32 - WidthVal + 1) : (32 - WidthVal);
3298
3299 unsigned OpSignBits = DAG.ComputeNumSignBits(BitsFrom);
3300 if (OpSignBits >= SignBits)
3301 return BitsFrom;
Matt Arsenault05e96f42014-05-22 18:09:12 +00003302
3303 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), WidthVal);
3304 if (Signed) {
3305 // This is a sign_extend_inreg. Replace it to take advantage of existing
3306 // DAG Combines. If not eliminated, we will match back to BFE during
3307 // selection.
3308
3309 // TODO: The sext_inreg of extended types ends, although we can could
3310 // handle them in a single BFE.
3311 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, BitsFrom,
3312 DAG.getValueType(SmallVT));
3313 }
3314
3315 return DAG.getZeroExtendInReg(BitsFrom, DL, SmallVT);
Matt Arsenault5565f65e2014-05-22 18:09:07 +00003316 }
3317
Matt Arsenaultf1794202014-10-15 05:07:00 +00003318 if (ConstantSDNode *CVal = dyn_cast<ConstantSDNode>(BitsFrom)) {
Matt Arsenault5565f65e2014-05-22 18:09:07 +00003319 if (Signed) {
3320 return constantFoldBFE<int32_t>(DAG,
Matt Arsenault46cbc432014-09-19 00:42:06 +00003321 CVal->getSExtValue(),
Matt Arsenault5565f65e2014-05-22 18:09:07 +00003322 OffsetVal,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003323 WidthVal,
3324 DL);
Matt Arsenault5565f65e2014-05-22 18:09:07 +00003325 }
3326
3327 return constantFoldBFE<uint32_t>(DAG,
Matt Arsenault6462f942014-09-18 15:52:26 +00003328 CVal->getZExtValue(),
Matt Arsenault5565f65e2014-05-22 18:09:07 +00003329 OffsetVal,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003330 WidthVal,
3331 DL);
Matt Arsenault5565f65e2014-05-22 18:09:07 +00003332 }
3333
Matt Arsenault05e96f42014-05-22 18:09:12 +00003334 if ((OffsetVal + WidthVal) >= 32) {
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003335 SDValue ShiftVal = DAG.getConstant(OffsetVal, DL, MVT::i32);
Matt Arsenault05e96f42014-05-22 18:09:12 +00003336 return DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, MVT::i32,
3337 BitsFrom, ShiftVal);
3338 }
3339
Matt Arsenault7b68fdf2014-10-15 17:58:34 +00003340 if (BitsFrom.hasOneUse()) {
Matt Arsenault6de7af42014-10-15 23:37:42 +00003341 APInt Demanded = APInt::getBitsSet(32,
3342 OffsetVal,
3343 OffsetVal + WidthVal);
3344
Matt Arsenault7b68fdf2014-10-15 17:58:34 +00003345 APInt KnownZero, KnownOne;
3346 TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
3347 !DCI.isBeforeLegalizeOps());
3348 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3349 if (TLO.ShrinkDemandedConstant(BitsFrom, Demanded) ||
3350 TLI.SimplifyDemandedBits(BitsFrom, Demanded,
3351 KnownZero, KnownOne, TLO)) {
3352 DCI.CommitTargetLoweringOpt(TLO);
3353 }
Matt Arsenault5565f65e2014-05-22 18:09:07 +00003354 }
3355
3356 break;
3357 }
Matt Arsenault327bb5a2016-07-01 22:47:50 +00003358 case ISD::LOAD:
3359 return performLoadCombine(N, DCI);
Matt Arsenaultca3976f2014-07-15 02:06:31 +00003360 case ISD::STORE:
3361 return performStoreCombine(N, DCI);
Matt Arsenault2fdf2a12017-02-21 23:35:48 +00003362 case AMDGPUISD::CLAMP:
3363 return performClampCombine(N, DCI);
Matt Arsenaultd8ed2072017-03-08 00:48:46 +00003364 case AMDGPUISD::RCP: {
3365 if (const auto *CFP = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) {
3366 // XXX - Should this flush denormals?
3367 const APFloat &Val = CFP->getValueAPF();
3368 APFloat One(Val.getSemantics(), "1.0");
3369 return DAG.getConstantFP(One / Val, SDLoc(N), N->getValueType(0));
3370 }
3371
3372 break;
3373 }
Tom Stellard50122a52014-04-07 19:45:41 +00003374 }
3375 return SDValue();
3376}
3377
3378//===----------------------------------------------------------------------===//
Tom Stellard75aadc22012-12-11 21:25:42 +00003379// Helper functions
3380//===----------------------------------------------------------------------===//
3381
Tom Stellard75aadc22012-12-11 21:25:42 +00003382SDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG,
3383 const TargetRegisterClass *RC,
3384 unsigned Reg, EVT VT) const {
3385 MachineFunction &MF = DAG.getMachineFunction();
3386 MachineRegisterInfo &MRI = MF.getRegInfo();
3387 unsigned VirtualRegister;
3388 if (!MRI.isLiveIn(Reg)) {
3389 VirtualRegister = MRI.createVirtualRegister(RC);
3390 MRI.addLiveIn(Reg, VirtualRegister);
3391 } else {
3392 VirtualRegister = MRI.getLiveInVirtReg(Reg);
3393 }
3394 return DAG.getRegister(VirtualRegister, VT);
3395}
3396
Tom Stellarddcb9f092015-07-09 21:20:37 +00003397uint32_t AMDGPUTargetLowering::getImplicitParameterOffset(
3398 const AMDGPUMachineFunction *MFI, const ImplicitParameter Param) const {
Tom Stellardb2869eb2016-09-09 19:28:00 +00003399 unsigned Alignment = Subtarget->getAlignmentForImplicitArgPtr();
3400 uint64_t ArgOffset = alignTo(MFI->getABIArgOffset(), Alignment);
Tom Stellarddcb9f092015-07-09 21:20:37 +00003401 switch (Param) {
3402 case GRID_DIM:
3403 return ArgOffset;
3404 case GRID_OFFSET:
3405 return ArgOffset + 4;
3406 }
3407 llvm_unreachable("unexpected implicit parameter type");
3408}
3409
Tom Stellard75aadc22012-12-11 21:25:42 +00003410#define NODE_NAME_CASE(node) case AMDGPUISD::node: return #node;
3411
3412const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
Matthias Braund04893f2015-05-07 21:33:59 +00003413 switch ((AMDGPUISD::NodeType)Opcode) {
3414 case AMDGPUISD::FIRST_NUMBER: break;
Tom Stellard75aadc22012-12-11 21:25:42 +00003415 // AMDIL DAG nodes
Tom Stellard75aadc22012-12-11 21:25:42 +00003416 NODE_NAME_CASE(UMUL);
Tom Stellard75aadc22012-12-11 21:25:42 +00003417 NODE_NAME_CASE(BRANCH_COND);
3418
3419 // AMDGPU DAG nodes
Matt Arsenaultc5b641a2017-03-17 20:41:45 +00003420 NODE_NAME_CASE(IF)
3421 NODE_NAME_CASE(ELSE)
3422 NODE_NAME_CASE(LOOP)
Matt Arsenault5b20fbb2017-03-21 22:18:10 +00003423 NODE_NAME_CASE(CALL)
3424 NODE_NAME_CASE(RET_FLAG)
3425 NODE_NAME_CASE(RETURN_TO_EPILOG)
Matt Arsenault9babdf42016-06-22 20:15:28 +00003426 NODE_NAME_CASE(ENDPGM)
Tom Stellard75aadc22012-12-11 21:25:42 +00003427 NODE_NAME_CASE(DWORDADDR)
3428 NODE_NAME_CASE(FRACT)
Wei Ding07e03712016-07-28 16:42:13 +00003429 NODE_NAME_CASE(SETCC)
Tom Stellard8485fa02016-12-07 02:42:15 +00003430 NODE_NAME_CASE(SETREG)
3431 NODE_NAME_CASE(FMA_W_CHAIN)
3432 NODE_NAME_CASE(FMUL_W_CHAIN)
Matt Arsenault5d47d4a2014-06-12 21:15:44 +00003433 NODE_NAME_CASE(CLAMP)
Matthias Braund04893f2015-05-07 21:33:59 +00003434 NODE_NAME_CASE(COS_HW)
3435 NODE_NAME_CASE(SIN_HW)
Matt Arsenaultda59f3d2014-11-13 23:03:09 +00003436 NODE_NAME_CASE(FMAX_LEGACY)
Matt Arsenaultda59f3d2014-11-13 23:03:09 +00003437 NODE_NAME_CASE(FMIN_LEGACY)
Matt Arsenaultcc3c2b32014-11-14 20:08:52 +00003438 NODE_NAME_CASE(FMAX3)
3439 NODE_NAME_CASE(SMAX3)
3440 NODE_NAME_CASE(UMAX3)
3441 NODE_NAME_CASE(FMIN3)
3442 NODE_NAME_CASE(SMIN3)
3443 NODE_NAME_CASE(UMIN3)
Matt Arsenaultf639c322016-01-28 20:53:42 +00003444 NODE_NAME_CASE(FMED3)
3445 NODE_NAME_CASE(SMED3)
3446 NODE_NAME_CASE(UMED3)
Matt Arsenaulta0050b02014-06-19 01:19:19 +00003447 NODE_NAME_CASE(URECIP)
3448 NODE_NAME_CASE(DIV_SCALE)
3449 NODE_NAME_CASE(DIV_FMAS)
3450 NODE_NAME_CASE(DIV_FIXUP)
Wei Ding4d3d4ca2017-02-24 23:00:29 +00003451 NODE_NAME_CASE(FMAD_FTZ)
Matt Arsenaulta0050b02014-06-19 01:19:19 +00003452 NODE_NAME_CASE(TRIG_PREOP)
3453 NODE_NAME_CASE(RCP)
3454 NODE_NAME_CASE(RSQ)
Matt Arsenault32fc5272016-07-26 16:45:45 +00003455 NODE_NAME_CASE(RCP_LEGACY)
Matt Arsenault257d48d2014-06-24 22:13:39 +00003456 NODE_NAME_CASE(RSQ_LEGACY)
Matt Arsenault32fc5272016-07-26 16:45:45 +00003457 NODE_NAME_CASE(FMUL_LEGACY)
Matt Arsenault79963e82016-02-13 01:03:00 +00003458 NODE_NAME_CASE(RSQ_CLAMP)
Matt Arsenault2e7cc482014-08-15 17:30:25 +00003459 NODE_NAME_CASE(LDEXP)
Matt Arsenault4831ce52015-01-06 23:00:37 +00003460 NODE_NAME_CASE(FP_CLASS)
Matt Arsenaulta0050b02014-06-19 01:19:19 +00003461 NODE_NAME_CASE(DOT4)
Matthias Braund04893f2015-05-07 21:33:59 +00003462 NODE_NAME_CASE(CARRY)
3463 NODE_NAME_CASE(BORROW)
Matt Arsenaultfae02982014-03-17 18:58:11 +00003464 NODE_NAME_CASE(BFE_U32)
3465 NODE_NAME_CASE(BFE_I32)
Matt Arsenaultb3458362014-03-31 18:21:13 +00003466 NODE_NAME_CASE(BFI)
3467 NODE_NAME_CASE(BFM)
Matt Arsenaultde5fbe92016-01-11 17:02:00 +00003468 NODE_NAME_CASE(FFBH_U32)
Matt Arsenaultb51dcb92016-07-18 18:40:51 +00003469 NODE_NAME_CASE(FFBH_I32)
Tom Stellard50122a52014-04-07 19:45:41 +00003470 NODE_NAME_CASE(MUL_U24)
3471 NODE_NAME_CASE(MUL_I24)
Matt Arsenault2712d4a2016-08-27 01:32:27 +00003472 NODE_NAME_CASE(MULHI_U24)
3473 NODE_NAME_CASE(MULHI_I24)
3474 NODE_NAME_CASE(MUL_LOHI_U24)
3475 NODE_NAME_CASE(MUL_LOHI_I24)
Matt Arsenaulteb260202014-05-22 18:00:15 +00003476 NODE_NAME_CASE(MAD_U24)
3477 NODE_NAME_CASE(MAD_I24)
Matthias Braund04893f2015-05-07 21:33:59 +00003478 NODE_NAME_CASE(TEXTURE_FETCH)
Tom Stellard75aadc22012-12-11 21:25:42 +00003479 NODE_NAME_CASE(EXPORT)
Matt Arsenault7bee6ac2016-12-05 20:23:10 +00003480 NODE_NAME_CASE(EXPORT_DONE)
3481 NODE_NAME_CASE(R600_EXPORT)
Tom Stellardff62c352013-01-23 02:09:03 +00003482 NODE_NAME_CASE(CONST_ADDRESS)
Tom Stellardf3b2a1e2013-02-06 17:32:29 +00003483 NODE_NAME_CASE(REGISTER_LOAD)
3484 NODE_NAME_CASE(REGISTER_STORE)
Tom Stellard9fa17912013-08-14 23:24:45 +00003485 NODE_NAME_CASE(LOAD_INPUT)
3486 NODE_NAME_CASE(SAMPLE)
3487 NODE_NAME_CASE(SAMPLEB)
3488 NODE_NAME_CASE(SAMPLED)
3489 NODE_NAME_CASE(SAMPLEL)
Matt Arsenault364a6742014-06-11 17:50:44 +00003490 NODE_NAME_CASE(CVT_F32_UBYTE0)
3491 NODE_NAME_CASE(CVT_F32_UBYTE1)
3492 NODE_NAME_CASE(CVT_F32_UBYTE2)
3493 NODE_NAME_CASE(CVT_F32_UBYTE3)
Matt Arsenault1f17c662017-02-22 00:27:34 +00003494 NODE_NAME_CASE(CVT_PKRTZ_F16_F32)
Matt Arsenault86e02ce2017-03-15 19:04:26 +00003495 NODE_NAME_CASE(FP_TO_FP16)
Tom Stellard880a80a2014-06-17 16:53:14 +00003496 NODE_NAME_CASE(BUILD_VERTICAL_VECTOR)
Tom Stellard067c8152014-07-21 14:01:14 +00003497 NODE_NAME_CASE(CONST_DATA_PTR)
Tom Stellardbf3e6e52016-06-14 20:29:59 +00003498 NODE_NAME_CASE(PC_ADD_REL_OFFSET)
Matt Arsenault03006fd2016-07-19 16:27:56 +00003499 NODE_NAME_CASE(KILL)
Jan Veselyf1705042017-01-20 21:24:26 +00003500 NODE_NAME_CASE(DUMMY_CHAIN)
Matthias Braund04893f2015-05-07 21:33:59 +00003501 case AMDGPUISD::FIRST_MEM_OPCODE_NUMBER: break;
Tom Stellardfc92e772015-05-12 14:18:14 +00003502 NODE_NAME_CASE(SENDMSG)
Jan Veselyd48445d2017-01-04 18:06:55 +00003503 NODE_NAME_CASE(SENDMSGHALT)
Tom Stellard2a9d9472015-05-12 15:00:46 +00003504 NODE_NAME_CASE(INTERP_MOV)
3505 NODE_NAME_CASE(INTERP_P1)
3506 NODE_NAME_CASE(INTERP_P2)
Tom Stellardd3ee8c12013-08-16 01:12:06 +00003507 NODE_NAME_CASE(STORE_MSKOR)
Matt Arsenaultdfaf4262016-04-25 19:27:09 +00003508 NODE_NAME_CASE(LOAD_CONSTANT)
Tom Stellardafcf12f2013-09-12 02:55:14 +00003509 NODE_NAME_CASE(TBUFFER_STORE_FORMAT)
Tom Stellard354a43c2016-04-01 18:27:37 +00003510 NODE_NAME_CASE(ATOMIC_CMP_SWAP)
Matt Arsenaulta9dbdca2016-04-12 14:05:04 +00003511 NODE_NAME_CASE(ATOMIC_INC)
3512 NODE_NAME_CASE(ATOMIC_DEC)
Tom Stellard6f9ef142016-12-20 17:19:44 +00003513 NODE_NAME_CASE(BUFFER_LOAD)
3514 NODE_NAME_CASE(BUFFER_LOAD_FORMAT)
Matthias Braund04893f2015-05-07 21:33:59 +00003515 case AMDGPUISD::LAST_AMDGPU_ISD_NUMBER: break;
Tom Stellard75aadc22012-12-11 21:25:42 +00003516 }
Matthias Braund04893f2015-05-07 21:33:59 +00003517 return nullptr;
Tom Stellard75aadc22012-12-11 21:25:42 +00003518}
Matt Arsenault0c274fe2014-03-25 18:18:27 +00003519
Evandro Menezes21f9ce12016-11-10 23:31:06 +00003520SDValue AMDGPUTargetLowering::getSqrtEstimate(SDValue Operand,
3521 SelectionDAG &DAG, int Enabled,
3522 int &RefinementSteps,
3523 bool &UseOneConstNR,
3524 bool Reciprocal) const {
Matt Arsenaulte93d06a2015-01-13 20:53:18 +00003525 EVT VT = Operand.getValueType();
3526
3527 if (VT == MVT::f32) {
3528 RefinementSteps = 0;
3529 return DAG.getNode(AMDGPUISD::RSQ, SDLoc(Operand), VT, Operand);
3530 }
3531
3532 // TODO: There is also f64 rsq instruction, but the documentation is less
3533 // clear on its precision.
3534
3535 return SDValue();
3536}
3537
Matt Arsenaultbf0db912015-01-13 20:53:23 +00003538SDValue AMDGPUTargetLowering::getRecipEstimate(SDValue Operand,
Sanjay Patel0051efc2016-10-20 16:55:45 +00003539 SelectionDAG &DAG, int Enabled,
3540 int &RefinementSteps) const {
Matt Arsenaultbf0db912015-01-13 20:53:23 +00003541 EVT VT = Operand.getValueType();
3542
3543 if (VT == MVT::f32) {
3544 // Reciprocal, < 1 ulp error.
3545 //
3546 // This reciprocal approximation converges to < 0.5 ulp error with one
3547 // newton rhapson performed with two fused multiple adds (FMAs).
3548
3549 RefinementSteps = 0;
3550 return DAG.getNode(AMDGPUISD::RCP, SDLoc(Operand), VT, Operand);
3551 }
3552
3553 // TODO: There is also f64 rcp instruction, but the documentation is less
3554 // clear on its precision.
3555
3556 return SDValue();
3557}
3558
Jay Foada0653a32014-05-14 21:14:37 +00003559void AMDGPUTargetLowering::computeKnownBitsForTargetNode(
Matt Arsenault0c274fe2014-03-25 18:18:27 +00003560 const SDValue Op,
3561 APInt &KnownZero,
3562 APInt &KnownOne,
3563 const SelectionDAG &DAG,
3564 unsigned Depth) const {
Matt Arsenault378bf9c2014-03-31 19:35:33 +00003565
Matt Arsenault0c274fe2014-03-25 18:18:27 +00003566 KnownZero = KnownOne = APInt(KnownOne.getBitWidth(), 0); // Don't know anything.
Matt Arsenaultaf6df9d2014-05-22 18:09:00 +00003567
3568 APInt KnownZero2;
3569 APInt KnownOne2;
Matt Arsenault378bf9c2014-03-31 19:35:33 +00003570 unsigned Opc = Op.getOpcode();
Matt Arsenaultaf6df9d2014-05-22 18:09:00 +00003571
Matt Arsenault378bf9c2014-03-31 19:35:33 +00003572 switch (Opc) {
Matt Arsenaultaf6df9d2014-05-22 18:09:00 +00003573 default:
3574 break;
Jan Vesely808fff52015-04-30 17:15:56 +00003575 case AMDGPUISD::CARRY:
3576 case AMDGPUISD::BORROW: {
3577 KnownZero = APInt::getHighBitsSet(32, 31);
3578 break;
3579 }
3580
Matt Arsenaultaf6df9d2014-05-22 18:09:00 +00003581 case AMDGPUISD::BFE_I32:
3582 case AMDGPUISD::BFE_U32: {
3583 ConstantSDNode *CWidth = dyn_cast<ConstantSDNode>(Op.getOperand(2));
3584 if (!CWidth)
3585 return;
3586
3587 unsigned BitWidth = 32;
3588 uint32_t Width = CWidth->getZExtValue() & 0x1f;
Matt Arsenaultaf6df9d2014-05-22 18:09:00 +00003589
Matt Arsenaulta3fe7c62014-10-16 20:07:40 +00003590 if (Opc == AMDGPUISD::BFE_U32)
Matt Arsenaultaf6df9d2014-05-22 18:09:00 +00003591 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - Width);
3592
Matt Arsenault378bf9c2014-03-31 19:35:33 +00003593 break;
3594 }
Matt Arsenault86e02ce2017-03-15 19:04:26 +00003595 case AMDGPUISD::FP_TO_FP16: {
3596 unsigned BitWidth = KnownZero.getBitWidth();
3597
3598 // High bits are zero.
3599 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 16);
3600 break;
3601 }
Matt Arsenaultaf6df9d2014-05-22 18:09:00 +00003602 }
Matt Arsenault0c274fe2014-03-25 18:18:27 +00003603}
Matt Arsenaultbf8694d2014-05-22 18:09:03 +00003604
3605unsigned AMDGPUTargetLowering::ComputeNumSignBitsForTargetNode(
3606 SDValue Op,
3607 const SelectionDAG &DAG,
3608 unsigned Depth) const {
3609 switch (Op.getOpcode()) {
3610 case AMDGPUISD::BFE_I32: {
3611 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Op.getOperand(2));
3612 if (!Width)
3613 return 1;
3614
3615 unsigned SignBits = 32 - Width->getZExtValue() + 1;
Artyom Skrobov314ee042015-11-25 19:41:11 +00003616 if (!isNullConstant(Op.getOperand(1)))
Matt Arsenaultbf8694d2014-05-22 18:09:03 +00003617 return SignBits;
3618
3619 // TODO: Could probably figure something out with non-0 offsets.
3620 unsigned Op0SignBits = DAG.ComputeNumSignBits(Op.getOperand(0), Depth + 1);
3621 return std::max(SignBits, Op0SignBits);
3622 }
3623
Matt Arsenault5565f65e2014-05-22 18:09:07 +00003624 case AMDGPUISD::BFE_U32: {
3625 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Op.getOperand(2));
3626 return Width ? 32 - (Width->getZExtValue() & 0x1f) : 1;
3627 }
3628
Jan Vesely808fff52015-04-30 17:15:56 +00003629 case AMDGPUISD::CARRY:
3630 case AMDGPUISD::BORROW:
3631 return 31;
3632
Matt Arsenaultbf8694d2014-05-22 18:09:03 +00003633 default:
3634 return 1;
3635 }
3636}