blob: 52a500c0d5e3b5f69d1046803c814c83c17a5f7c [file] [log] [blame]
Tom Stellard75aadc22012-12-11 21:25:42 +00001//===-- AMDGPUISelLowering.cpp - AMDGPU Common DAG lowering functions -----===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10/// \file
11/// \brief This is the parent TargetLowering class for hardware code gen
12/// targets.
13//
14//===----------------------------------------------------------------------===//
15
16#include "AMDGPUISelLowering.h"
Tom Stellarded882c22013-06-03 17:40:11 +000017#include "AMDGPU.h"
Tom Stellard81d871d2013-11-13 23:36:50 +000018#include "AMDGPUFrameLowering.h"
Christian Konig2c8f6d52013-03-07 09:03:52 +000019#include "AMDGPURegisterInfo.h"
Christian Konig2c8f6d52013-03-07 09:03:52 +000020#include "AMDGPUSubtarget.h"
Benjamin Kramerd78bb462013-05-23 17:10:37 +000021#include "AMDILIntrinsicInfo.h"
Tom Stellardacfeebf2013-07-23 01:48:05 +000022#include "R600MachineFunctionInfo.h"
Tom Stellarded882c22013-06-03 17:40:11 +000023#include "SIMachineFunctionInfo.h"
Tom Stellard04c0e982014-01-22 19:24:21 +000024#include "llvm/Analysis/ValueTracking.h"
Christian Konig2c8f6d52013-03-07 09:03:52 +000025#include "llvm/CodeGen/CallingConvLower.h"
Tom Stellard75aadc22012-12-11 21:25:42 +000026#include "llvm/CodeGen/MachineFunction.h"
27#include "llvm/CodeGen/MachineRegisterInfo.h"
28#include "llvm/CodeGen/SelectionDAG.h"
29#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
Tom Stellardc026e8b2013-06-28 15:47:08 +000030#include "llvm/IR/DataLayout.h"
Matt Arsenault16353872014-04-22 16:42:00 +000031#include "llvm/IR/DiagnosticInfo.h"
32#include "llvm/IR/DiagnosticPrinter.h"
Tom Stellard75aadc22012-12-11 21:25:42 +000033
34using namespace llvm;
Matt Arsenault16353872014-04-22 16:42:00 +000035
36namespace {
37
38/// Diagnostic information for unimplemented or unsupported feature reporting.
39class DiagnosticInfoUnsupported : public DiagnosticInfo {
40private:
41 const Twine &Description;
42 const Function &Fn;
43
44 static int KindID;
45
46 static int getKindID() {
47 if (KindID == 0)
48 KindID = llvm::getNextAvailablePluginDiagnosticKind();
49 return KindID;
50 }
51
52public:
53 DiagnosticInfoUnsupported(const Function &Fn, const Twine &Desc,
54 DiagnosticSeverity Severity = DS_Error)
55 : DiagnosticInfo(getKindID(), Severity),
56 Description(Desc),
57 Fn(Fn) { }
58
59 const Function &getFunction() const { return Fn; }
60 const Twine &getDescription() const { return Description; }
61
62 void print(DiagnosticPrinter &DP) const override {
63 DP << "unsupported " << getDescription() << " in " << Fn.getName();
64 }
65
66 static bool classof(const DiagnosticInfo *DI) {
67 return DI->getKind() == getKindID();
68 }
69};
70
71int DiagnosticInfoUnsupported::KindID = 0;
72}
73
74
Tom Stellardaf775432013-10-23 00:44:32 +000075static bool allocateStack(unsigned ValNo, MVT ValVT, MVT LocVT,
76 CCValAssign::LocInfo LocInfo,
77 ISD::ArgFlagsTy ArgFlags, CCState &State) {
Matt Arsenault52226f92013-12-14 18:21:59 +000078 unsigned Offset = State.AllocateStack(ValVT.getStoreSize(),
79 ArgFlags.getOrigAlign());
80 State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo));
Tom Stellardaf775432013-10-23 00:44:32 +000081
82 return true;
83}
Tom Stellard75aadc22012-12-11 21:25:42 +000084
Christian Konig2c8f6d52013-03-07 09:03:52 +000085#include "AMDGPUGenCallingConv.inc"
86
Tom Stellard75aadc22012-12-11 21:25:42 +000087AMDGPUTargetLowering::AMDGPUTargetLowering(TargetMachine &TM) :
88 TargetLowering(TM, new TargetLoweringObjectFileELF()) {
89
Matt Arsenault41e2f2b2014-02-24 21:01:28 +000090 Subtarget = &TM.getSubtarget<AMDGPUSubtarget>();
91
Tom Stellard75aadc22012-12-11 21:25:42 +000092 // Initialize target lowering borrowed from AMDIL
93 InitAMDILLowering();
94
95 // We need to custom lower some of the intrinsics
96 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
97
98 // Library functions. These default to Expand, but we have instructions
99 // for them.
100 setOperationAction(ISD::FCEIL, MVT::f32, Legal);
101 setOperationAction(ISD::FEXP2, MVT::f32, Legal);
102 setOperationAction(ISD::FPOW, MVT::f32, Legal);
103 setOperationAction(ISD::FLOG2, MVT::f32, Legal);
104 setOperationAction(ISD::FABS, MVT::f32, Legal);
105 setOperationAction(ISD::FFLOOR, MVT::f32, Legal);
106 setOperationAction(ISD::FRINT, MVT::f32, Legal);
Tom Stellard4d566b22013-11-27 21:23:20 +0000107 setOperationAction(ISD::FROUND, MVT::f32, Legal);
Tom Stellardeddfa692013-12-20 05:11:55 +0000108 setOperationAction(ISD::FTRUNC, MVT::f32, Legal);
Tom Stellard75aadc22012-12-11 21:25:42 +0000109
Tom Stellard5643c4a2013-05-20 15:02:19 +0000110 // The hardware supports ROTR, but not ROTL
111 setOperationAction(ISD::ROTL, MVT::i32, Expand);
112
Tom Stellard75aadc22012-12-11 21:25:42 +0000113 // Lower floating point store/load to integer store/load to reduce the number
114 // of patterns in tablegen.
115 setOperationAction(ISD::STORE, MVT::f32, Promote);
116 AddPromotedToType(ISD::STORE, MVT::f32, MVT::i32);
117
Tom Stellarded2f6142013-07-18 21:43:42 +0000118 setOperationAction(ISD::STORE, MVT::v2f32, Promote);
119 AddPromotedToType(ISD::STORE, MVT::v2f32, MVT::v2i32);
120
Tom Stellard75aadc22012-12-11 21:25:42 +0000121 setOperationAction(ISD::STORE, MVT::v4f32, Promote);
122 AddPromotedToType(ISD::STORE, MVT::v4f32, MVT::v4i32);
123
Tom Stellardaf775432013-10-23 00:44:32 +0000124 setOperationAction(ISD::STORE, MVT::v8f32, Promote);
125 AddPromotedToType(ISD::STORE, MVT::v8f32, MVT::v8i32);
126
127 setOperationAction(ISD::STORE, MVT::v16f32, Promote);
128 AddPromotedToType(ISD::STORE, MVT::v16f32, MVT::v16i32);
129
Tom Stellard7512c082013-07-12 18:14:56 +0000130 setOperationAction(ISD::STORE, MVT::f64, Promote);
131 AddPromotedToType(ISD::STORE, MVT::f64, MVT::i64);
132
Tom Stellard2ffc3302013-08-26 15:05:44 +0000133 // Custom lowering of vector stores is required for local address space
134 // stores.
135 setOperationAction(ISD::STORE, MVT::v4i32, Custom);
136 // XXX: Native v2i32 local address space stores are possible, but not
137 // currently implemented.
138 setOperationAction(ISD::STORE, MVT::v2i32, Custom);
139
Tom Stellardfbab8272013-08-16 01:12:11 +0000140 setTruncStoreAction(MVT::v2i32, MVT::v2i16, Custom);
141 setTruncStoreAction(MVT::v2i32, MVT::v2i8, Custom);
142 setTruncStoreAction(MVT::v4i32, MVT::v4i8, Custom);
Matt Arsenaulte389dd52014-03-12 18:45:52 +0000143
Tom Stellardfbab8272013-08-16 01:12:11 +0000144 // XXX: This can be change to Custom, once ExpandVectorStores can
145 // handle 64-bit stores.
146 setTruncStoreAction(MVT::v4i32, MVT::v4i16, Expand);
147
Matt Arsenaulte389dd52014-03-12 18:45:52 +0000148 setTruncStoreAction(MVT::i64, MVT::i1, Expand);
149 setTruncStoreAction(MVT::v2i64, MVT::v2i1, Expand);
150 setTruncStoreAction(MVT::v4i64, MVT::v4i1, Expand);
151
152
Tom Stellard75aadc22012-12-11 21:25:42 +0000153 setOperationAction(ISD::LOAD, MVT::f32, Promote);
154 AddPromotedToType(ISD::LOAD, MVT::f32, MVT::i32);
155
Tom Stellardadf732c2013-07-18 21:43:48 +0000156 setOperationAction(ISD::LOAD, MVT::v2f32, Promote);
157 AddPromotedToType(ISD::LOAD, MVT::v2f32, MVT::v2i32);
158
Tom Stellard75aadc22012-12-11 21:25:42 +0000159 setOperationAction(ISD::LOAD, MVT::v4f32, Promote);
160 AddPromotedToType(ISD::LOAD, MVT::v4f32, MVT::v4i32);
161
Tom Stellardaf775432013-10-23 00:44:32 +0000162 setOperationAction(ISD::LOAD, MVT::v8f32, Promote);
163 AddPromotedToType(ISD::LOAD, MVT::v8f32, MVT::v8i32);
164
165 setOperationAction(ISD::LOAD, MVT::v16f32, Promote);
166 AddPromotedToType(ISD::LOAD, MVT::v16f32, MVT::v16i32);
167
Tom Stellard7512c082013-07-12 18:14:56 +0000168 setOperationAction(ISD::LOAD, MVT::f64, Promote);
169 AddPromotedToType(ISD::LOAD, MVT::f64, MVT::i64);
170
Tom Stellardd86003e2013-08-14 23:25:00 +0000171 setOperationAction(ISD::CONCAT_VECTORS, MVT::v4i32, Custom);
172 setOperationAction(ISD::CONCAT_VECTORS, MVT::v4f32, Custom);
Tom Stellard967bf582014-02-13 23:34:15 +0000173 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i32, Custom);
174 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8f32, Custom);
Tom Stellardd86003e2013-08-14 23:25:00 +0000175 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v2f32, Custom);
Tom Stellard967bf582014-02-13 23:34:15 +0000176 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v2i32, Custom);
177 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v4f32, Custom);
178 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v4i32, Custom);
179 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8f32, Custom);
180 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8i32, Custom);
Tom Stellard0344cdf2013-08-01 15:23:42 +0000181
Tom Stellardb03edec2013-08-16 01:12:16 +0000182 setLoadExtAction(ISD::EXTLOAD, MVT::v2i8, Expand);
183 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i8, Expand);
184 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i8, Expand);
185 setLoadExtAction(ISD::EXTLOAD, MVT::v4i8, Expand);
186 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i8, Expand);
187 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i8, Expand);
188 setLoadExtAction(ISD::EXTLOAD, MVT::v2i16, Expand);
189 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i16, Expand);
190 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i16, Expand);
191 setLoadExtAction(ISD::EXTLOAD, MVT::v4i16, Expand);
192 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i16, Expand);
193 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i16, Expand);
194
Tom Stellardaeb45642014-02-04 17:18:43 +0000195 setOperationAction(ISD::BR_CC, MVT::i1, Expand);
196
Tom Stellardbeed74a2013-07-23 01:47:46 +0000197 setOperationAction(ISD::FNEG, MVT::v2f32, Expand);
198 setOperationAction(ISD::FNEG, MVT::v4f32, Expand);
199
Tom Stellardc947d8c2013-10-30 17:22:05 +0000200 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
201
Christian Konig70a50322013-03-27 09:12:51 +0000202 setOperationAction(ISD::MUL, MVT::i64, Expand);
203
Tom Stellard75aadc22012-12-11 21:25:42 +0000204 setOperationAction(ISD::UDIV, MVT::i32, Expand);
205 setOperationAction(ISD::UDIVREM, MVT::i32, Custom);
Tom Stellard5f337882014-04-29 23:12:43 +0000206 setOperationAction(ISD::UDIVREM, MVT::i64, Custom);
Tom Stellard75aadc22012-12-11 21:25:42 +0000207 setOperationAction(ISD::UREM, MVT::i32, Expand);
Tom Stellard67ae4762013-07-18 21:43:35 +0000208 setOperationAction(ISD::VSELECT, MVT::v2f32, Expand);
209 setOperationAction(ISD::VSELECT, MVT::v4f32, Expand);
Aaron Watry0a794a462013-06-25 13:55:57 +0000210
Tom Stellardf6d80232013-08-21 22:14:17 +0000211 static const MVT::SimpleValueType IntTypes[] = {
212 MVT::v2i32, MVT::v4i32
Aaron Watry0a794a462013-06-25 13:55:57 +0000213 };
Tom Stellarda92ff872013-08-16 23:51:24 +0000214 const size_t NumIntTypes = array_lengthof(IntTypes);
Aaron Watry0a794a462013-06-25 13:55:57 +0000215
Tom Stellarda92ff872013-08-16 23:51:24 +0000216 for (unsigned int x = 0; x < NumIntTypes; ++x) {
Tom Stellardf6d80232013-08-21 22:14:17 +0000217 MVT::SimpleValueType VT = IntTypes[x];
Aaron Watry0a794a462013-06-25 13:55:57 +0000218 //Expand the following operations for the current type by default
219 setOperationAction(ISD::ADD, VT, Expand);
220 setOperationAction(ISD::AND, VT, Expand);
Tom Stellardaa313d02013-07-30 14:31:03 +0000221 setOperationAction(ISD::FP_TO_SINT, VT, Expand);
222 setOperationAction(ISD::FP_TO_UINT, VT, Expand);
Aaron Watry0a794a462013-06-25 13:55:57 +0000223 setOperationAction(ISD::MUL, VT, Expand);
224 setOperationAction(ISD::OR, VT, Expand);
225 setOperationAction(ISD::SHL, VT, Expand);
Tom Stellardaa313d02013-07-30 14:31:03 +0000226 setOperationAction(ISD::SINT_TO_FP, VT, Expand);
Aaron Watry0a794a462013-06-25 13:55:57 +0000227 setOperationAction(ISD::SRL, VT, Expand);
228 setOperationAction(ISD::SRA, VT, Expand);
229 setOperationAction(ISD::SUB, VT, Expand);
230 setOperationAction(ISD::UDIV, VT, Expand);
Tom Stellardaa313d02013-07-30 14:31:03 +0000231 setOperationAction(ISD::UINT_TO_FP, VT, Expand);
Aaron Watry0a794a462013-06-25 13:55:57 +0000232 setOperationAction(ISD::UREM, VT, Expand);
Matt Arsenault9fe669c2014-03-06 17:34:03 +0000233 setOperationAction(ISD::SELECT, VT, Expand);
Tom Stellard67ae4762013-07-18 21:43:35 +0000234 setOperationAction(ISD::VSELECT, VT, Expand);
Aaron Watry0a794a462013-06-25 13:55:57 +0000235 setOperationAction(ISD::XOR, VT, Expand);
236 }
Tom Stellarda92ff872013-08-16 23:51:24 +0000237
Tom Stellardf6d80232013-08-21 22:14:17 +0000238 static const MVT::SimpleValueType FloatTypes[] = {
239 MVT::v2f32, MVT::v4f32
Tom Stellarda92ff872013-08-16 23:51:24 +0000240 };
241 const size_t NumFloatTypes = array_lengthof(FloatTypes);
242
243 for (unsigned int x = 0; x < NumFloatTypes; ++x) {
Tom Stellardf6d80232013-08-21 22:14:17 +0000244 MVT::SimpleValueType VT = FloatTypes[x];
Tom Stellard175e7a82013-11-27 21:23:39 +0000245 setOperationAction(ISD::FABS, VT, Expand);
Tom Stellarda92ff872013-08-16 23:51:24 +0000246 setOperationAction(ISD::FADD, VT, Expand);
247 setOperationAction(ISD::FDIV, VT, Expand);
Tom Stellardbfebd1f2014-02-04 17:18:37 +0000248 setOperationAction(ISD::FPOW, VT, Expand);
Tom Stellardad3aff22013-08-16 23:51:29 +0000249 setOperationAction(ISD::FFLOOR, VT, Expand);
Tom Stellardeddfa692013-12-20 05:11:55 +0000250 setOperationAction(ISD::FTRUNC, VT, Expand);
Tom Stellarda92ff872013-08-16 23:51:24 +0000251 setOperationAction(ISD::FMUL, VT, Expand);
Tom Stellardb249b752013-08-16 23:51:33 +0000252 setOperationAction(ISD::FRINT, VT, Expand);
Tom Stellarde118b8b2013-10-29 16:37:20 +0000253 setOperationAction(ISD::FSQRT, VT, Expand);
Tom Stellarda92ff872013-08-16 23:51:24 +0000254 setOperationAction(ISD::FSUB, VT, Expand);
Matt Arsenault9fe669c2014-03-06 17:34:03 +0000255 setOperationAction(ISD::SELECT, VT, Expand);
Tom Stellarda92ff872013-08-16 23:51:24 +0000256 }
Matt Arsenaultfae02982014-03-17 18:58:11 +0000257
Tom Stellard50122a52014-04-07 19:45:41 +0000258 setTargetDAGCombine(ISD::MUL);
Tom Stellard75aadc22012-12-11 21:25:42 +0000259}
260
Tom Stellard28d06de2013-08-05 22:22:07 +0000261//===----------------------------------------------------------------------===//
262// Target Information
263//===----------------------------------------------------------------------===//
264
265MVT AMDGPUTargetLowering::getVectorIdxTy() const {
266 return MVT::i32;
267}
268
Matt Arsenaultc5559bb2013-11-15 04:42:23 +0000269bool AMDGPUTargetLowering::isLoadBitCastBeneficial(EVT LoadTy,
270 EVT CastTy) const {
271 if (LoadTy.getSizeInBits() != CastTy.getSizeInBits())
272 return true;
273
274 unsigned LScalarSize = LoadTy.getScalarType().getSizeInBits();
275 unsigned CastScalarSize = CastTy.getScalarType().getSizeInBits();
276
277 return ((LScalarSize <= CastScalarSize) ||
278 (CastScalarSize >= 32) ||
279 (LScalarSize < 32));
280}
Tom Stellard28d06de2013-08-05 22:22:07 +0000281
Tom Stellard75aadc22012-12-11 21:25:42 +0000282//===---------------------------------------------------------------------===//
Tom Stellardc54731a2013-07-23 23:55:03 +0000283// Target Properties
284//===---------------------------------------------------------------------===//
285
286bool AMDGPUTargetLowering::isFAbsFree(EVT VT) const {
287 assert(VT.isFloatingPoint());
288 return VT == MVT::f32;
289}
290
291bool AMDGPUTargetLowering::isFNegFree(EVT VT) const {
292 assert(VT.isFloatingPoint());
293 return VT == MVT::f32;
294}
295
Benjamin Kramer53f9df42014-02-12 10:17:54 +0000296bool AMDGPUTargetLowering::isTruncateFree(EVT Source, EVT Dest) const {
Matt Arsenault0cdcd962014-02-10 19:57:42 +0000297 // Truncate is just accessing a subregister.
Benjamin Kramer53f9df42014-02-12 10:17:54 +0000298 return Dest.bitsLT(Source) && (Dest.getSizeInBits() % 32 == 0);
299}
300
301bool AMDGPUTargetLowering::isTruncateFree(Type *Source, Type *Dest) const {
302 // Truncate is just accessing a subregister.
303 return Dest->getPrimitiveSizeInBits() < Source->getPrimitiveSizeInBits() &&
304 (Dest->getPrimitiveSizeInBits() % 32 == 0);
Matt Arsenault0cdcd962014-02-10 19:57:42 +0000305}
306
Matt Arsenaultb517c812014-03-27 17:23:31 +0000307bool AMDGPUTargetLowering::isZExtFree(Type *Src, Type *Dest) const {
308 const DataLayout *DL = getDataLayout();
309 unsigned SrcSize = DL->getTypeSizeInBits(Src->getScalarType());
310 unsigned DestSize = DL->getTypeSizeInBits(Dest->getScalarType());
311
312 return SrcSize == 32 && DestSize == 64;
313}
314
315bool AMDGPUTargetLowering::isZExtFree(EVT Src, EVT Dest) const {
316 // Any register load of a 64-bit value really requires 2 32-bit moves. For all
317 // practical purposes, the extra mov 0 to load a 64-bit is free. As used,
318 // this will enable reducing 64-bit operations the 32-bit, which is always
319 // good.
320 return Src == MVT::i32 && Dest == MVT::i64;
321}
322
Matt Arsenaulta7f1e0c2014-03-24 19:43:31 +0000323bool AMDGPUTargetLowering::isNarrowingProfitable(EVT SrcVT, EVT DestVT) const {
324 // There aren't really 64-bit registers, but pairs of 32-bit ones and only a
325 // limited number of native 64-bit operations. Shrinking an operation to fit
326 // in a single 32-bit register should always be helpful. As currently used,
327 // this is much less general than the name suggests, and is only used in
328 // places trying to reduce the sizes of loads. Shrinking loads to < 32-bits is
329 // not profitable, and may actually be harmful.
330 return SrcVT.getSizeInBits() > 32 && DestVT.getSizeInBits() == 32;
331}
332
Tom Stellardc54731a2013-07-23 23:55:03 +0000333//===---------------------------------------------------------------------===//
Tom Stellard75aadc22012-12-11 21:25:42 +0000334// TargetLowering Callbacks
335//===---------------------------------------------------------------------===//
336
Christian Konig2c8f6d52013-03-07 09:03:52 +0000337void AMDGPUTargetLowering::AnalyzeFormalArguments(CCState &State,
338 const SmallVectorImpl<ISD::InputArg> &Ins) const {
339
340 State.AnalyzeFormalArguments(Ins, CC_AMDGPU);
Tom Stellard75aadc22012-12-11 21:25:42 +0000341}
342
343SDValue AMDGPUTargetLowering::LowerReturn(
344 SDValue Chain,
345 CallingConv::ID CallConv,
346 bool isVarArg,
347 const SmallVectorImpl<ISD::OutputArg> &Outs,
348 const SmallVectorImpl<SDValue> &OutVals,
Andrew Trickef9de2a2013-05-25 02:42:55 +0000349 SDLoc DL, SelectionDAG &DAG) const {
Tom Stellard75aadc22012-12-11 21:25:42 +0000350 return DAG.getNode(AMDGPUISD::RET_FLAG, DL, MVT::Other, Chain);
351}
352
353//===---------------------------------------------------------------------===//
354// Target specific lowering
355//===---------------------------------------------------------------------===//
356
Matt Arsenault16353872014-04-22 16:42:00 +0000357SDValue AMDGPUTargetLowering::LowerCall(CallLoweringInfo &CLI,
358 SmallVectorImpl<SDValue> &InVals) const {
359 SDValue Callee = CLI.Callee;
360 SelectionDAG &DAG = CLI.DAG;
361
362 const Function &Fn = *DAG.getMachineFunction().getFunction();
363
364 StringRef FuncName("<unknown>");
365
Matt Arsenaultde1c34102014-04-25 22:22:01 +0000366 if (const ExternalSymbolSDNode *G = dyn_cast<ExternalSymbolSDNode>(Callee))
367 FuncName = G->getSymbol();
368 else if (const GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee))
Matt Arsenault16353872014-04-22 16:42:00 +0000369 FuncName = G->getGlobal()->getName();
370
371 DiagnosticInfoUnsupported NoCalls(Fn, "call to function " + FuncName);
372 DAG.getContext()->diagnose(NoCalls);
373 return SDValue();
374}
375
Tom Stellard75aadc22012-12-11 21:25:42 +0000376SDValue AMDGPUTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG)
377 const {
378 switch (Op.getOpcode()) {
379 default:
380 Op.getNode()->dump();
Matt Arsenaulteaa3a7e2013-12-10 21:37:42 +0000381 llvm_unreachable("Custom lowering code for this"
382 "instruction is not implemented yet!");
Tom Stellard75aadc22012-12-11 21:25:42 +0000383 break;
384 // AMDIL DAG lowering
385 case ISD::SDIV: return LowerSDIV(Op, DAG);
386 case ISD::SREM: return LowerSREM(Op, DAG);
387 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG);
388 case ISD::BRCOND: return LowerBRCOND(Op, DAG);
389 // AMDGPU DAG lowering
Tom Stellardd86003e2013-08-14 23:25:00 +0000390 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
391 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG);
Tom Stellard81d871d2013-11-13 23:36:50 +0000392 case ISD::FrameIndex: return LowerFrameIndex(Op, DAG);
Tom Stellard75aadc22012-12-11 21:25:42 +0000393 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
394 case ISD::UDIVREM: return LowerUDIVREM(Op, DAG);
Tom Stellardc947d8c2013-10-30 17:22:05 +0000395 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG);
Tom Stellard75aadc22012-12-11 21:25:42 +0000396 }
397 return Op;
398}
399
Matt Arsenaultd125d742014-03-27 17:23:24 +0000400void AMDGPUTargetLowering::ReplaceNodeResults(SDNode *N,
401 SmallVectorImpl<SDValue> &Results,
402 SelectionDAG &DAG) const {
403 switch (N->getOpcode()) {
404 case ISD::SIGN_EXTEND_INREG:
405 // Different parts of legalization seem to interpret which type of
406 // sign_extend_inreg is the one to check for custom lowering. The extended
407 // from type is what really matters, but some places check for custom
408 // lowering of the result type. This results in trying to use
409 // ReplaceNodeResults to sext_in_reg to an illegal type, so we'll just do
410 // nothing here and let the illegal result integer be handled normally.
411 return;
Tom Stellard5f337882014-04-29 23:12:43 +0000412 case ISD::UDIV: {
413 SDValue Op = SDValue(N, 0);
414 SDLoc DL(Op);
415 EVT VT = Op.getValueType();
416 SDValue UDIVREM = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(VT, VT),
417 N->getOperand(0), N->getOperand(1));
418 Results.push_back(UDIVREM);
419 break;
420 }
421 case ISD::UREM: {
422 SDValue Op = SDValue(N, 0);
423 SDLoc DL(Op);
424 EVT VT = Op.getValueType();
425 SDValue UDIVREM = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(VT, VT),
426 N->getOperand(0), N->getOperand(1));
427 Results.push_back(UDIVREM.getValue(1));
428 break;
429 }
Tom Stellardbcd318f2014-04-29 23:12:45 +0000430 case ISD::UDIVREM: {
431 SDValue Op = SDValue(N, 0);
432 SDLoc DL(Op);
433 EVT VT = Op.getValueType();
434 EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext());
435
Tom Stellard676f5712014-04-29 23:12:46 +0000436 SDValue one = DAG.getConstant(1, HalfVT);
437 SDValue zero = DAG.getConstant(0, HalfVT);
438
Tom Stellardbcd318f2014-04-29 23:12:45 +0000439 //HiLo split
Tom Stellard676f5712014-04-29 23:12:46 +0000440 SDValue LHS = N->getOperand(0);
441 SDValue LHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, zero);
442 SDValue LHS_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, one);
Tom Stellardbcd318f2014-04-29 23:12:45 +0000443
444 SDValue RHS = N->getOperand(1);
Tom Stellard676f5712014-04-29 23:12:46 +0000445 SDValue RHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, zero);
446 SDValue RHS_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, one);
Tom Stellardbcd318f2014-04-29 23:12:45 +0000447
Tom Stellard676f5712014-04-29 23:12:46 +0000448 // Get Speculative values
449 SDValue DIV_Part = DAG.getNode(ISD::UDIV, DL, HalfVT, LHS_Hi, RHS_Lo);
450 SDValue REM_Part = DAG.getNode(ISD::UREM, DL, HalfVT, LHS_Hi, RHS_Lo);
Tom Stellardbcd318f2014-04-29 23:12:45 +0000451
Tom Stellard676f5712014-04-29 23:12:46 +0000452 SDValue REM_Hi = zero;
453 SDValue REM_Lo = DAG.getSelectCC(DL, RHS_Hi, zero, REM_Part, LHS_Hi, ISD::SETEQ);
454
455 SDValue DIV_Hi = DAG.getSelectCC(DL, RHS_Hi, zero, DIV_Part, zero, ISD::SETEQ);
456 SDValue DIV_Lo = zero;
457
Tom Stellardbcd318f2014-04-29 23:12:45 +0000458 const unsigned halfBitWidth = HalfVT.getSizeInBits();
459
Tom Stellard676f5712014-04-29 23:12:46 +0000460 for (unsigned i = 0; i < halfBitWidth; ++i) {
461 SDValue POS = DAG.getConstant(halfBitWidth - i - 1, HalfVT);
Tom Stellardbcd318f2014-04-29 23:12:45 +0000462 // Get Value of high bit
Tom Stellard676f5712014-04-29 23:12:46 +0000463 SDValue HBit;
464 if (halfBitWidth == 32 && Subtarget->hasBFE()) {
465 HBit = DAG.getNode(AMDGPUISD::BFE_U32, DL, HalfVT, LHS_Lo, POS, one);
466 } else {
467 HBit = DAG.getNode(ISD::SRL, DL, HalfVT, LHS_Lo, POS);
468 HBit = DAG.getNode(ISD::AND, DL, HalfVT, HBit, one);
469 }
Tom Stellardbcd318f2014-04-29 23:12:45 +0000470
Tom Stellard676f5712014-04-29 23:12:46 +0000471 SDValue Carry = DAG.getNode(ISD::SRL, DL, HalfVT, REM_Lo,
472 DAG.getConstant(halfBitWidth - 1, HalfVT));
473 REM_Hi = DAG.getNode(ISD::SHL, DL, HalfVT, REM_Hi, one);
474 REM_Hi = DAG.getNode(ISD::OR, DL, HalfVT, REM_Hi, Carry);
Tom Stellardbcd318f2014-04-29 23:12:45 +0000475
Tom Stellard676f5712014-04-29 23:12:46 +0000476 REM_Lo = DAG.getNode(ISD::SHL, DL, HalfVT, REM_Lo, one);
477 REM_Lo = DAG.getNode(ISD::OR, DL, HalfVT, REM_Lo, HBit);
Tom Stellardbcd318f2014-04-29 23:12:45 +0000478
Tom Stellard676f5712014-04-29 23:12:46 +0000479
480 SDValue REM = DAG.getNode(ISD::BUILD_PAIR, DL, VT, REM_Lo, REM_Hi);
481
482 SDValue BIT = DAG.getConstant(1 << (halfBitWidth - i - 1), HalfVT);
483 SDValue realBIT = DAG.getSelectCC(DL, REM, RHS, BIT, zero, ISD::SETGE);
484
485 DIV_Lo = DAG.getNode(ISD::OR, DL, HalfVT, DIV_Lo, realBIT);
Tom Stellardbcd318f2014-04-29 23:12:45 +0000486
487 // Update REM
Tom Stellard676f5712014-04-29 23:12:46 +0000488
Tom Stellardbcd318f2014-04-29 23:12:45 +0000489 SDValue REM_sub = DAG.getNode(ISD::SUB, DL, VT, REM, RHS);
490
491 REM = DAG.getSelectCC(DL, REM, RHS, REM_sub, REM, ISD::SETGE);
Tom Stellard676f5712014-04-29 23:12:46 +0000492 REM_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, REM, zero);
493 REM_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, REM, one);
Tom Stellardbcd318f2014-04-29 23:12:45 +0000494 }
495
Tom Stellard676f5712014-04-29 23:12:46 +0000496 SDValue REM = DAG.getNode(ISD::BUILD_PAIR, DL, VT, REM_Lo, REM_Hi);
497 SDValue DIV = DAG.getNode(ISD::BUILD_PAIR, DL, VT, DIV_Lo, DIV_Hi);
Tom Stellardbcd318f2014-04-29 23:12:45 +0000498 Results.push_back(DIV);
499 Results.push_back(REM);
500 break;
501 }
Matt Arsenaultd125d742014-03-27 17:23:24 +0000502 default:
503 return;
504 }
505}
506
Tom Stellard04c0e982014-01-22 19:24:21 +0000507SDValue AMDGPUTargetLowering::LowerConstantInitializer(const Constant* Init,
508 const GlobalValue *GV,
509 const SDValue &InitPtr,
510 SDValue Chain,
511 SelectionDAG &DAG) const {
512 const DataLayout *TD = getTargetMachine().getDataLayout();
513 SDLoc DL(InitPtr);
514 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Init)) {
515 EVT VT = EVT::getEVT(CI->getType());
516 PointerType *PtrTy = PointerType::get(CI->getType(), 0);
517 return DAG.getStore(Chain, DL, DAG.getConstant(*CI, VT), InitPtr,
518 MachinePointerInfo(UndefValue::get(PtrTy)), false, false,
519 TD->getPrefTypeAlignment(CI->getType()));
520 } else if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Init)) {
521 EVT VT = EVT::getEVT(CFP->getType());
522 PointerType *PtrTy = PointerType::get(CFP->getType(), 0);
523 return DAG.getStore(Chain, DL, DAG.getConstantFP(*CFP, VT), InitPtr,
524 MachinePointerInfo(UndefValue::get(PtrTy)), false, false,
525 TD->getPrefTypeAlignment(CFP->getType()));
526 } else if (Init->getType()->isAggregateType()) {
527 EVT PtrVT = InitPtr.getValueType();
528 unsigned NumElements = Init->getType()->getArrayNumElements();
529 SmallVector<SDValue, 8> Chains;
530 for (unsigned i = 0; i < NumElements; ++i) {
531 SDValue Offset = DAG.getConstant(i * TD->getTypeAllocSize(
532 Init->getType()->getArrayElementType()), PtrVT);
533 SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, InitPtr, Offset);
534 Chains.push_back(LowerConstantInitializer(Init->getAggregateElement(i),
535 GV, Ptr, Chain, DAG));
536 }
Craig Topper48d114b2014-04-26 18:35:24 +0000537 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
Tom Stellard04c0e982014-01-22 19:24:21 +0000538 } else {
539 Init->dump();
540 llvm_unreachable("Unhandled constant initializer");
541 }
542}
543
Tom Stellardc026e8b2013-06-28 15:47:08 +0000544SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunction* MFI,
545 SDValue Op,
546 SelectionDAG &DAG) const {
547
548 const DataLayout *TD = getTargetMachine().getDataLayout();
549 GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Op);
Tom Stellardc026e8b2013-06-28 15:47:08 +0000550 const GlobalValue *GV = G->getGlobal();
Tom Stellardc026e8b2013-06-28 15:47:08 +0000551
Tom Stellard04c0e982014-01-22 19:24:21 +0000552 switch (G->getAddressSpace()) {
553 default: llvm_unreachable("Global Address lowering not implemented for this "
554 "address space");
555 case AMDGPUAS::LOCAL_ADDRESS: {
556 // XXX: What does the value of G->getOffset() mean?
557 assert(G->getOffset() == 0 &&
558 "Do not know what to do with an non-zero offset");
Tom Stellardc026e8b2013-06-28 15:47:08 +0000559
Tom Stellard04c0e982014-01-22 19:24:21 +0000560 unsigned Offset;
561 if (MFI->LocalMemoryObjects.count(GV) == 0) {
562 uint64_t Size = TD->getTypeAllocSize(GV->getType()->getElementType());
563 Offset = MFI->LDSSize;
564 MFI->LocalMemoryObjects[GV] = Offset;
565 // XXX: Account for alignment?
566 MFI->LDSSize += Size;
567 } else {
568 Offset = MFI->LocalMemoryObjects[GV];
569 }
570
571 return DAG.getConstant(Offset, getPointerTy(G->getAddressSpace()));
572 }
573 case AMDGPUAS::CONSTANT_ADDRESS: {
574 MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo();
575 Type *EltType = GV->getType()->getElementType();
576 unsigned Size = TD->getTypeAllocSize(EltType);
577 unsigned Alignment = TD->getPrefTypeAlignment(EltType);
578
579 const GlobalVariable *Var = dyn_cast<GlobalVariable>(GV);
580 const Constant *Init = Var->getInitializer();
581 int FI = FrameInfo->CreateStackObject(Size, Alignment, false);
582 SDValue InitPtr = DAG.getFrameIndex(FI,
583 getPointerTy(AMDGPUAS::PRIVATE_ADDRESS));
584 SmallVector<SDNode*, 8> WorkList;
585
586 for (SDNode::use_iterator I = DAG.getEntryNode()->use_begin(),
587 E = DAG.getEntryNode()->use_end(); I != E; ++I) {
588 if (I->getOpcode() != AMDGPUISD::REGISTER_LOAD && I->getOpcode() != ISD::LOAD)
589 continue;
590 WorkList.push_back(*I);
591 }
592 SDValue Chain = LowerConstantInitializer(Init, GV, InitPtr, DAG.getEntryNode(), DAG);
593 for (SmallVector<SDNode*, 8>::iterator I = WorkList.begin(),
594 E = WorkList.end(); I != E; ++I) {
595 SmallVector<SDValue, 8> Ops;
596 Ops.push_back(Chain);
597 for (unsigned i = 1; i < (*I)->getNumOperands(); ++i) {
598 Ops.push_back((*I)->getOperand(i));
599 }
Craig Topper8c0b4d02014-04-28 05:57:50 +0000600 DAG.UpdateNodeOperands(*I, Ops);
Tom Stellard04c0e982014-01-22 19:24:21 +0000601 }
602 return DAG.getZExtOrTrunc(InitPtr, SDLoc(Op),
603 getPointerTy(AMDGPUAS::CONSTANT_ADDRESS));
604 }
605 }
Tom Stellardc026e8b2013-06-28 15:47:08 +0000606}
607
Tom Stellardd86003e2013-08-14 23:25:00 +0000608SDValue AMDGPUTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
609 SelectionDAG &DAG) const {
610 SmallVector<SDValue, 8> Args;
611 SDValue A = Op.getOperand(0);
612 SDValue B = Op.getOperand(1);
613
Matt Arsenault9ec3cf22014-04-11 17:47:30 +0000614 DAG.ExtractVectorElements(A, Args);
615 DAG.ExtractVectorElements(B, Args);
Tom Stellardd86003e2013-08-14 23:25:00 +0000616
Craig Topper48d114b2014-04-26 18:35:24 +0000617 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Op), Op.getValueType(), Args);
Tom Stellardd86003e2013-08-14 23:25:00 +0000618}
619
620SDValue AMDGPUTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
621 SelectionDAG &DAG) const {
622
623 SmallVector<SDValue, 8> Args;
Tom Stellardd86003e2013-08-14 23:25:00 +0000624 unsigned Start = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
Matt Arsenault9ec3cf22014-04-11 17:47:30 +0000625 EVT VT = Op.getValueType();
626 DAG.ExtractVectorElements(Op.getOperand(0), Args, Start,
627 VT.getVectorNumElements());
Tom Stellardd86003e2013-08-14 23:25:00 +0000628
Craig Topper48d114b2014-04-26 18:35:24 +0000629 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Op), Op.getValueType(), Args);
Tom Stellardd86003e2013-08-14 23:25:00 +0000630}
631
Tom Stellard81d871d2013-11-13 23:36:50 +0000632SDValue AMDGPUTargetLowering::LowerFrameIndex(SDValue Op,
633 SelectionDAG &DAG) const {
634
635 MachineFunction &MF = DAG.getMachineFunction();
636 const AMDGPUFrameLowering *TFL =
637 static_cast<const AMDGPUFrameLowering*>(getTargetMachine().getFrameLowering());
638
639 FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Op);
640 assert(FIN);
641
642 unsigned FrameIndex = FIN->getIndex();
643 unsigned Offset = TFL->getFrameIndexOffset(MF, FrameIndex);
644 return DAG.getConstant(Offset * 4 * TFL->getStackWidth(MF),
645 Op.getValueType());
646}
Tom Stellardd86003e2013-08-14 23:25:00 +0000647
Tom Stellard75aadc22012-12-11 21:25:42 +0000648SDValue AMDGPUTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
649 SelectionDAG &DAG) const {
650 unsigned IntrinsicID = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
Andrew Trickef9de2a2013-05-25 02:42:55 +0000651 SDLoc DL(Op);
Tom Stellard75aadc22012-12-11 21:25:42 +0000652 EVT VT = Op.getValueType();
653
654 switch (IntrinsicID) {
655 default: return Op;
656 case AMDGPUIntrinsic::AMDIL_abs:
657 return LowerIntrinsicIABS(Op, DAG);
658 case AMDGPUIntrinsic::AMDIL_exp:
659 return DAG.getNode(ISD::FEXP2, DL, VT, Op.getOperand(1));
660 case AMDGPUIntrinsic::AMDGPU_lrp:
661 return LowerIntrinsicLRP(Op, DAG);
662 case AMDGPUIntrinsic::AMDIL_fraction:
663 return DAG.getNode(AMDGPUISD::FRACT, DL, VT, Op.getOperand(1));
Tom Stellard75aadc22012-12-11 21:25:42 +0000664 case AMDGPUIntrinsic::AMDIL_max:
665 return DAG.getNode(AMDGPUISD::FMAX, DL, VT, Op.getOperand(1),
666 Op.getOperand(2));
667 case AMDGPUIntrinsic::AMDGPU_imax:
668 return DAG.getNode(AMDGPUISD::SMAX, DL, VT, Op.getOperand(1),
669 Op.getOperand(2));
670 case AMDGPUIntrinsic::AMDGPU_umax:
671 return DAG.getNode(AMDGPUISD::UMAX, DL, VT, Op.getOperand(1),
672 Op.getOperand(2));
673 case AMDGPUIntrinsic::AMDIL_min:
674 return DAG.getNode(AMDGPUISD::FMIN, DL, VT, Op.getOperand(1),
675 Op.getOperand(2));
676 case AMDGPUIntrinsic::AMDGPU_imin:
677 return DAG.getNode(AMDGPUISD::SMIN, DL, VT, Op.getOperand(1),
678 Op.getOperand(2));
679 case AMDGPUIntrinsic::AMDGPU_umin:
680 return DAG.getNode(AMDGPUISD::UMIN, DL, VT, Op.getOperand(1),
681 Op.getOperand(2));
Matt Arsenault4c537172014-03-31 18:21:18 +0000682
683 case AMDGPUIntrinsic::AMDGPU_bfe_i32:
684 return DAG.getNode(AMDGPUISD::BFE_I32, DL, VT,
685 Op.getOperand(1),
686 Op.getOperand(2),
687 Op.getOperand(3));
688
689 case AMDGPUIntrinsic::AMDGPU_bfe_u32:
690 return DAG.getNode(AMDGPUISD::BFE_U32, DL, VT,
691 Op.getOperand(1),
692 Op.getOperand(2),
693 Op.getOperand(3));
694
695 case AMDGPUIntrinsic::AMDGPU_bfi:
696 return DAG.getNode(AMDGPUISD::BFI, DL, VT,
697 Op.getOperand(1),
698 Op.getOperand(2),
699 Op.getOperand(3));
700
701 case AMDGPUIntrinsic::AMDGPU_bfm:
702 return DAG.getNode(AMDGPUISD::BFM, DL, VT,
703 Op.getOperand(1),
704 Op.getOperand(2));
705
Tom Stellard75aadc22012-12-11 21:25:42 +0000706 case AMDGPUIntrinsic::AMDIL_round_nearest:
707 return DAG.getNode(ISD::FRINT, DL, VT, Op.getOperand(1));
708 }
709}
710
711///IABS(a) = SMAX(sub(0, a), a)
712SDValue AMDGPUTargetLowering::LowerIntrinsicIABS(SDValue Op,
713 SelectionDAG &DAG) const {
714
Andrew Trickef9de2a2013-05-25 02:42:55 +0000715 SDLoc DL(Op);
Tom Stellard75aadc22012-12-11 21:25:42 +0000716 EVT VT = Op.getValueType();
717 SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, VT),
718 Op.getOperand(1));
719
720 return DAG.getNode(AMDGPUISD::SMAX, DL, VT, Neg, Op.getOperand(1));
721}
722
723/// Linear Interpolation
724/// LRP(a, b, c) = muladd(a, b, (1 - a) * c)
725SDValue AMDGPUTargetLowering::LowerIntrinsicLRP(SDValue Op,
726 SelectionDAG &DAG) const {
Andrew Trickef9de2a2013-05-25 02:42:55 +0000727 SDLoc DL(Op);
Tom Stellard75aadc22012-12-11 21:25:42 +0000728 EVT VT = Op.getValueType();
729 SDValue OneSubA = DAG.getNode(ISD::FSUB, DL, VT,
730 DAG.getConstantFP(1.0f, MVT::f32),
731 Op.getOperand(1));
732 SDValue OneSubAC = DAG.getNode(ISD::FMUL, DL, VT, OneSubA,
733 Op.getOperand(3));
Vincent Lejeune1ce13f52013-02-18 14:11:28 +0000734 return DAG.getNode(ISD::FADD, DL, VT,
735 DAG.getNode(ISD::FMUL, DL, VT, Op.getOperand(1), Op.getOperand(2)),
736 OneSubAC);
Tom Stellard75aadc22012-12-11 21:25:42 +0000737}
738
739/// \brief Generate Min/Max node
740SDValue AMDGPUTargetLowering::LowerMinMax(SDValue Op,
741 SelectionDAG &DAG) const {
Andrew Trickef9de2a2013-05-25 02:42:55 +0000742 SDLoc DL(Op);
Tom Stellard75aadc22012-12-11 21:25:42 +0000743 EVT VT = Op.getValueType();
744
745 SDValue LHS = Op.getOperand(0);
746 SDValue RHS = Op.getOperand(1);
747 SDValue True = Op.getOperand(2);
748 SDValue False = Op.getOperand(3);
749 SDValue CC = Op.getOperand(4);
750
751 if (VT != MVT::f32 ||
752 !((LHS == True && RHS == False) || (LHS == False && RHS == True))) {
753 return SDValue();
754 }
755
756 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get();
757 switch (CCOpcode) {
758 case ISD::SETOEQ:
759 case ISD::SETONE:
760 case ISD::SETUNE:
761 case ISD::SETNE:
762 case ISD::SETUEQ:
763 case ISD::SETEQ:
764 case ISD::SETFALSE:
765 case ISD::SETFALSE2:
766 case ISD::SETTRUE:
767 case ISD::SETTRUE2:
768 case ISD::SETUO:
769 case ISD::SETO:
Matt Arsenaulteaa3a7e2013-12-10 21:37:42 +0000770 llvm_unreachable("Operation should already be optimised!");
Tom Stellard75aadc22012-12-11 21:25:42 +0000771 case ISD::SETULE:
772 case ISD::SETULT:
773 case ISD::SETOLE:
774 case ISD::SETOLT:
775 case ISD::SETLE:
776 case ISD::SETLT: {
777 if (LHS == True)
778 return DAG.getNode(AMDGPUISD::FMIN, DL, VT, LHS, RHS);
779 else
780 return DAG.getNode(AMDGPUISD::FMAX, DL, VT, LHS, RHS);
781 }
782 case ISD::SETGT:
783 case ISD::SETGE:
784 case ISD::SETUGE:
785 case ISD::SETOGE:
786 case ISD::SETUGT:
787 case ISD::SETOGT: {
788 if (LHS == True)
789 return DAG.getNode(AMDGPUISD::FMAX, DL, VT, LHS, RHS);
790 else
791 return DAG.getNode(AMDGPUISD::FMIN, DL, VT, LHS, RHS);
792 }
793 case ISD::SETCC_INVALID:
Matt Arsenaulteaa3a7e2013-12-10 21:37:42 +0000794 llvm_unreachable("Invalid setcc condcode!");
Tom Stellard75aadc22012-12-11 21:25:42 +0000795 }
796 return Op;
797}
798
Tom Stellard35bb18c2013-08-26 15:06:04 +0000799SDValue AMDGPUTargetLowering::SplitVectorLoad(const SDValue &Op,
800 SelectionDAG &DAG) const {
801 LoadSDNode *Load = dyn_cast<LoadSDNode>(Op);
802 EVT MemEltVT = Load->getMemoryVT().getVectorElementType();
803 EVT EltVT = Op.getValueType().getVectorElementType();
804 EVT PtrVT = Load->getBasePtr().getValueType();
805 unsigned NumElts = Load->getMemoryVT().getVectorNumElements();
806 SmallVector<SDValue, 8> Loads;
807 SDLoc SL(Op);
808
809 for (unsigned i = 0, e = NumElts; i != e; ++i) {
810 SDValue Ptr = DAG.getNode(ISD::ADD, SL, PtrVT, Load->getBasePtr(),
811 DAG.getConstant(i * (MemEltVT.getSizeInBits() / 8), PtrVT));
812 Loads.push_back(DAG.getExtLoad(Load->getExtensionType(), SL, EltVT,
813 Load->getChain(), Ptr,
814 MachinePointerInfo(Load->getMemOperand()->getValue()),
815 MemEltVT, Load->isVolatile(), Load->isNonTemporal(),
816 Load->getAlignment()));
817 }
Craig Topper48d114b2014-04-26 18:35:24 +0000818 return DAG.getNode(ISD::BUILD_VECTOR, SL, Op.getValueType(), Loads);
Tom Stellard35bb18c2013-08-26 15:06:04 +0000819}
820
Tom Stellard2ffc3302013-08-26 15:05:44 +0000821SDValue AMDGPUTargetLowering::MergeVectorStore(const SDValue &Op,
822 SelectionDAG &DAG) const {
823 StoreSDNode *Store = dyn_cast<StoreSDNode>(Op);
824 EVT MemVT = Store->getMemoryVT();
825 unsigned MemBits = MemVT.getSizeInBits();
Tom Stellard75aadc22012-12-11 21:25:42 +0000826
Matt Arsenaultca6dcfc2014-03-05 21:47:22 +0000827 // Byte stores are really expensive, so if possible, try to pack 32-bit vector
828 // truncating store into an i32 store.
829 // XXX: We could also handle optimize other vector bitwidths.
Tom Stellard2ffc3302013-08-26 15:05:44 +0000830 if (!MemVT.isVector() || MemBits > 32) {
831 return SDValue();
832 }
833
834 SDLoc DL(Op);
Matt Arsenaulta3c8cde2014-04-22 04:11:14 +0000835 SDValue Value = Store->getValue();
Tom Stellard2ffc3302013-08-26 15:05:44 +0000836 EVT VT = Value.getValueType();
Matt Arsenaulta3c8cde2014-04-22 04:11:14 +0000837 EVT ElemVT = VT.getVectorElementType();
838 SDValue Ptr = Store->getBasePtr();
Tom Stellard2ffc3302013-08-26 15:05:44 +0000839 EVT MemEltVT = MemVT.getVectorElementType();
840 unsigned MemEltBits = MemEltVT.getSizeInBits();
841 unsigned MemNumElements = MemVT.getVectorNumElements();
Matt Arsenaulta3c8cde2014-04-22 04:11:14 +0000842 unsigned PackedSize = MemVT.getStoreSizeInBits();
843 SDValue Mask = DAG.getConstant((1 << MemEltBits) - 1, MVT::i32);
844
845 assert(Value.getValueType().getScalarSizeInBits() >= 32);
Matt Arsenault02117142014-03-11 01:38:53 +0000846
Tom Stellard2ffc3302013-08-26 15:05:44 +0000847 SDValue PackedValue;
848 for (unsigned i = 0; i < MemNumElements; ++i) {
Tom Stellard2ffc3302013-08-26 15:05:44 +0000849 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ElemVT, Value,
850 DAG.getConstant(i, MVT::i32));
Matt Arsenaulta3c8cde2014-04-22 04:11:14 +0000851 Elt = DAG.getZExtOrTrunc(Elt, DL, MVT::i32);
852 Elt = DAG.getNode(ISD::AND, DL, MVT::i32, Elt, Mask); // getZeroExtendInReg
853
854 SDValue Shift = DAG.getConstant(MemEltBits * i, MVT::i32);
855 Elt = DAG.getNode(ISD::SHL, DL, MVT::i32, Elt, Shift);
856
Tom Stellard2ffc3302013-08-26 15:05:44 +0000857 if (i == 0) {
858 PackedValue = Elt;
859 } else {
Matt Arsenaulta3c8cde2014-04-22 04:11:14 +0000860 PackedValue = DAG.getNode(ISD::OR, DL, MVT::i32, PackedValue, Elt);
Tom Stellard2ffc3302013-08-26 15:05:44 +0000861 }
862 }
Matt Arsenaulta3c8cde2014-04-22 04:11:14 +0000863
864 if (PackedSize < 32) {
865 EVT PackedVT = EVT::getIntegerVT(*DAG.getContext(), PackedSize);
866 return DAG.getTruncStore(Store->getChain(), DL, PackedValue, Ptr,
867 Store->getMemOperand()->getPointerInfo(),
868 PackedVT,
869 Store->isNonTemporal(), Store->isVolatile(),
870 Store->getAlignment());
871 }
872
Tom Stellard2ffc3302013-08-26 15:05:44 +0000873 return DAG.getStore(Store->getChain(), DL, PackedValue, Ptr,
Matt Arsenaulta3c8cde2014-04-22 04:11:14 +0000874 Store->getMemOperand()->getPointerInfo(),
Tom Stellard2ffc3302013-08-26 15:05:44 +0000875 Store->isVolatile(), Store->isNonTemporal(),
876 Store->getAlignment());
877}
878
879SDValue AMDGPUTargetLowering::SplitVectorStore(SDValue Op,
880 SelectionDAG &DAG) const {
881 StoreSDNode *Store = cast<StoreSDNode>(Op);
882 EVT MemEltVT = Store->getMemoryVT().getVectorElementType();
883 EVT EltVT = Store->getValue().getValueType().getVectorElementType();
884 EVT PtrVT = Store->getBasePtr().getValueType();
885 unsigned NumElts = Store->getMemoryVT().getVectorNumElements();
886 SDLoc SL(Op);
887
888 SmallVector<SDValue, 8> Chains;
889
890 for (unsigned i = 0, e = NumElts; i != e; ++i) {
891 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
892 Store->getValue(), DAG.getConstant(i, MVT::i32));
893 SDValue Ptr = DAG.getNode(ISD::ADD, SL, PtrVT,
894 Store->getBasePtr(),
895 DAG.getConstant(i * (MemEltVT.getSizeInBits() / 8),
896 PtrVT));
Tom Stellardf3d166a2013-08-26 15:05:49 +0000897 Chains.push_back(DAG.getTruncStore(Store->getChain(), SL, Val, Ptr,
Tom Stellard2ffc3302013-08-26 15:05:44 +0000898 MachinePointerInfo(Store->getMemOperand()->getValue()),
Tom Stellardf3d166a2013-08-26 15:05:49 +0000899 MemEltVT, Store->isVolatile(), Store->isNonTemporal(),
Tom Stellard2ffc3302013-08-26 15:05:44 +0000900 Store->getAlignment()));
901 }
Craig Topper48d114b2014-04-26 18:35:24 +0000902 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Chains);
Tom Stellard2ffc3302013-08-26 15:05:44 +0000903}
904
Tom Stellarde9373602014-01-22 19:24:14 +0000905SDValue AMDGPUTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
906 SDLoc DL(Op);
907 LoadSDNode *Load = cast<LoadSDNode>(Op);
908 ISD::LoadExtType ExtType = Load->getExtensionType();
Matt Arsenaultf9a995d2014-03-06 17:34:12 +0000909 EVT VT = Op.getValueType();
910 EVT MemVT = Load->getMemoryVT();
911
912 if (ExtType != ISD::NON_EXTLOAD && !VT.isVector() && VT.getSizeInBits() > 32) {
913 // We can do the extload to 32-bits, and then need to separately extend to
914 // 64-bits.
915
916 SDValue ExtLoad32 = DAG.getExtLoad(ExtType, DL, MVT::i32,
917 Load->getChain(),
918 Load->getBasePtr(),
919 MemVT,
920 Load->getMemOperand());
921 return DAG.getNode(ISD::getExtForLoadExtType(ExtType), DL, VT, ExtLoad32);
922 }
Tom Stellarde9373602014-01-22 19:24:14 +0000923
Matt Arsenault470acd82014-04-15 22:28:39 +0000924 if (ExtType == ISD::NON_EXTLOAD && VT.getSizeInBits() < 32) {
925 assert(VT == MVT::i1 && "Only i1 non-extloads expected");
926 // FIXME: Copied from PPC
927 // First, load into 32 bits, then truncate to 1 bit.
928
929 SDValue Chain = Load->getChain();
930 SDValue BasePtr = Load->getBasePtr();
931 MachineMemOperand *MMO = Load->getMemOperand();
932
933 SDValue NewLD = DAG.getExtLoad(ISD::EXTLOAD, DL, MVT::i32, Chain,
934 BasePtr, MVT::i8, MMO);
935 return DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD);
936 }
937
Tom Stellard04c0e982014-01-22 19:24:21 +0000938 // Lower loads constant address space global variable loads
939 if (Load->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS &&
Nick Lewyckyaad475b2014-04-15 07:22:52 +0000940 isa<GlobalVariable>(
941 GetUnderlyingObject(Load->getMemOperand()->getValue()))) {
Tom Stellard04c0e982014-01-22 19:24:21 +0000942
943 SDValue Ptr = DAG.getZExtOrTrunc(Load->getBasePtr(), DL,
944 getPointerTy(AMDGPUAS::PRIVATE_ADDRESS));
945 Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, Ptr,
946 DAG.getConstant(2, MVT::i32));
947 return DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, Op.getValueType(),
948 Load->getChain(), Ptr,
949 DAG.getTargetConstant(0, MVT::i32), Op.getOperand(2));
950 }
951
Tom Stellarde9373602014-01-22 19:24:14 +0000952 if (Load->getAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS ||
953 ExtType == ISD::NON_EXTLOAD || Load->getMemoryVT().bitsGE(MVT::i32))
954 return SDValue();
955
956
Tom Stellarde9373602014-01-22 19:24:14 +0000957 SDValue Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, Load->getBasePtr(),
958 DAG.getConstant(2, MVT::i32));
959 SDValue Ret = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, Op.getValueType(),
960 Load->getChain(), Ptr,
961 DAG.getTargetConstant(0, MVT::i32),
962 Op.getOperand(2));
963 SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32,
964 Load->getBasePtr(),
965 DAG.getConstant(0x3, MVT::i32));
966 SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx,
967 DAG.getConstant(3, MVT::i32));
Matt Arsenault74891cd2014-03-15 00:08:22 +0000968
Tom Stellarde9373602014-01-22 19:24:14 +0000969 Ret = DAG.getNode(ISD::SRL, DL, MVT::i32, Ret, ShiftAmt);
Matt Arsenault74891cd2014-03-15 00:08:22 +0000970
971 EVT MemEltVT = MemVT.getScalarType();
Tom Stellarde9373602014-01-22 19:24:14 +0000972 if (ExtType == ISD::SEXTLOAD) {
Matt Arsenault74891cd2014-03-15 00:08:22 +0000973 SDValue MemEltVTNode = DAG.getValueType(MemEltVT);
974 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, Ret, MemEltVTNode);
Tom Stellarde9373602014-01-22 19:24:14 +0000975 }
976
Matt Arsenault74891cd2014-03-15 00:08:22 +0000977 return DAG.getZeroExtendInReg(Ret, DL, MemEltVT);
Tom Stellarde9373602014-01-22 19:24:14 +0000978}
979
Tom Stellard2ffc3302013-08-26 15:05:44 +0000980SDValue AMDGPUTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
Tom Stellarde9373602014-01-22 19:24:14 +0000981 SDLoc DL(Op);
Tom Stellard2ffc3302013-08-26 15:05:44 +0000982 SDValue Result = AMDGPUTargetLowering::MergeVectorStore(Op, DAG);
983 if (Result.getNode()) {
984 return Result;
985 }
986
987 StoreSDNode *Store = cast<StoreSDNode>(Op);
Tom Stellarde9373602014-01-22 19:24:14 +0000988 SDValue Chain = Store->getChain();
Tom Stellard81d871d2013-11-13 23:36:50 +0000989 if ((Store->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
990 Store->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS) &&
Tom Stellard2ffc3302013-08-26 15:05:44 +0000991 Store->getValue().getValueType().isVector()) {
992 return SplitVectorStore(Op, DAG);
993 }
Tom Stellarde9373602014-01-22 19:24:14 +0000994
Matt Arsenault74891cd2014-03-15 00:08:22 +0000995 EVT MemVT = Store->getMemoryVT();
Tom Stellarde9373602014-01-22 19:24:14 +0000996 if (Store->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS &&
Matt Arsenault74891cd2014-03-15 00:08:22 +0000997 MemVT.bitsLT(MVT::i32)) {
Tom Stellarde9373602014-01-22 19:24:14 +0000998 unsigned Mask = 0;
999 if (Store->getMemoryVT() == MVT::i8) {
1000 Mask = 0xff;
1001 } else if (Store->getMemoryVT() == MVT::i16) {
1002 Mask = 0xffff;
1003 }
Matt Arsenaultea330fb2014-03-15 00:08:26 +00001004 SDValue BasePtr = Store->getBasePtr();
1005 SDValue Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, BasePtr,
Tom Stellarde9373602014-01-22 19:24:14 +00001006 DAG.getConstant(2, MVT::i32));
1007 SDValue Dst = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, MVT::i32,
1008 Chain, Ptr, DAG.getTargetConstant(0, MVT::i32));
Matt Arsenaultea330fb2014-03-15 00:08:26 +00001009
1010 SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32, BasePtr,
Tom Stellarde9373602014-01-22 19:24:14 +00001011 DAG.getConstant(0x3, MVT::i32));
Matt Arsenaultea330fb2014-03-15 00:08:26 +00001012
Tom Stellarde9373602014-01-22 19:24:14 +00001013 SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx,
1014 DAG.getConstant(3, MVT::i32));
Matt Arsenaultea330fb2014-03-15 00:08:26 +00001015
Tom Stellarde9373602014-01-22 19:24:14 +00001016 SDValue SExtValue = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i32,
1017 Store->getValue());
Matt Arsenault74891cd2014-03-15 00:08:22 +00001018
1019 SDValue MaskedValue = DAG.getZeroExtendInReg(SExtValue, DL, MemVT);
1020
Tom Stellarde9373602014-01-22 19:24:14 +00001021 SDValue ShiftedValue = DAG.getNode(ISD::SHL, DL, MVT::i32,
1022 MaskedValue, ShiftAmt);
Matt Arsenault74891cd2014-03-15 00:08:22 +00001023
Tom Stellarde9373602014-01-22 19:24:14 +00001024 SDValue DstMask = DAG.getNode(ISD::SHL, DL, MVT::i32, DAG.getConstant(Mask, MVT::i32),
1025 ShiftAmt);
1026 DstMask = DAG.getNode(ISD::XOR, DL, MVT::i32, DstMask,
1027 DAG.getConstant(0xffffffff, MVT::i32));
1028 Dst = DAG.getNode(ISD::AND, DL, MVT::i32, Dst, DstMask);
1029
1030 SDValue Value = DAG.getNode(ISD::OR, DL, MVT::i32, Dst, ShiftedValue);
1031 return DAG.getNode(AMDGPUISD::REGISTER_STORE, DL, MVT::Other,
1032 Chain, Value, Ptr, DAG.getTargetConstant(0, MVT::i32));
1033 }
Tom Stellard2ffc3302013-08-26 15:05:44 +00001034 return SDValue();
1035}
Tom Stellard75aadc22012-12-11 21:25:42 +00001036
1037SDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op,
1038 SelectionDAG &DAG) const {
Andrew Trickef9de2a2013-05-25 02:42:55 +00001039 SDLoc DL(Op);
Tom Stellard75aadc22012-12-11 21:25:42 +00001040 EVT VT = Op.getValueType();
1041
1042 SDValue Num = Op.getOperand(0);
1043 SDValue Den = Op.getOperand(1);
1044
Tom Stellard75aadc22012-12-11 21:25:42 +00001045 // RCP = URECIP(Den) = 2^32 / Den + e
1046 // e is rounding error.
1047 SDValue RCP = DAG.getNode(AMDGPUISD::URECIP, DL, VT, Den);
1048
1049 // RCP_LO = umulo(RCP, Den) */
1050 SDValue RCP_LO = DAG.getNode(ISD::UMULO, DL, VT, RCP, Den);
1051
1052 // RCP_HI = mulhu (RCP, Den) */
1053 SDValue RCP_HI = DAG.getNode(ISD::MULHU, DL, VT, RCP, Den);
1054
1055 // NEG_RCP_LO = -RCP_LO
1056 SDValue NEG_RCP_LO = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, VT),
1057 RCP_LO);
1058
1059 // ABS_RCP_LO = (RCP_HI == 0 ? NEG_RCP_LO : RCP_LO)
1060 SDValue ABS_RCP_LO = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, VT),
1061 NEG_RCP_LO, RCP_LO,
1062 ISD::SETEQ);
1063 // Calculate the rounding error from the URECIP instruction
1064 // E = mulhu(ABS_RCP_LO, RCP)
1065 SDValue E = DAG.getNode(ISD::MULHU, DL, VT, ABS_RCP_LO, RCP);
1066
1067 // RCP_A_E = RCP + E
1068 SDValue RCP_A_E = DAG.getNode(ISD::ADD, DL, VT, RCP, E);
1069
1070 // RCP_S_E = RCP - E
1071 SDValue RCP_S_E = DAG.getNode(ISD::SUB, DL, VT, RCP, E);
1072
1073 // Tmp0 = (RCP_HI == 0 ? RCP_A_E : RCP_SUB_E)
1074 SDValue Tmp0 = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, VT),
1075 RCP_A_E, RCP_S_E,
1076 ISD::SETEQ);
1077 // Quotient = mulhu(Tmp0, Num)
1078 SDValue Quotient = DAG.getNode(ISD::MULHU, DL, VT, Tmp0, Num);
1079
1080 // Num_S_Remainder = Quotient * Den
1081 SDValue Num_S_Remainder = DAG.getNode(ISD::UMULO, DL, VT, Quotient, Den);
1082
1083 // Remainder = Num - Num_S_Remainder
1084 SDValue Remainder = DAG.getNode(ISD::SUB, DL, VT, Num, Num_S_Remainder);
1085
1086 // Remainder_GE_Den = (Remainder >= Den ? -1 : 0)
1087 SDValue Remainder_GE_Den = DAG.getSelectCC(DL, Remainder, Den,
1088 DAG.getConstant(-1, VT),
1089 DAG.getConstant(0, VT),
Vincent Lejeune4f3751f2013-11-06 17:36:04 +00001090 ISD::SETUGE);
1091 // Remainder_GE_Zero = (Num >= Num_S_Remainder ? -1 : 0)
1092 SDValue Remainder_GE_Zero = DAG.getSelectCC(DL, Num,
1093 Num_S_Remainder,
Tom Stellard75aadc22012-12-11 21:25:42 +00001094 DAG.getConstant(-1, VT),
1095 DAG.getConstant(0, VT),
Vincent Lejeune4f3751f2013-11-06 17:36:04 +00001096 ISD::SETUGE);
Tom Stellard75aadc22012-12-11 21:25:42 +00001097 // Tmp1 = Remainder_GE_Den & Remainder_GE_Zero
1098 SDValue Tmp1 = DAG.getNode(ISD::AND, DL, VT, Remainder_GE_Den,
1099 Remainder_GE_Zero);
1100
1101 // Calculate Division result:
1102
1103 // Quotient_A_One = Quotient + 1
1104 SDValue Quotient_A_One = DAG.getNode(ISD::ADD, DL, VT, Quotient,
1105 DAG.getConstant(1, VT));
1106
1107 // Quotient_S_One = Quotient - 1
1108 SDValue Quotient_S_One = DAG.getNode(ISD::SUB, DL, VT, Quotient,
1109 DAG.getConstant(1, VT));
1110
1111 // Div = (Tmp1 == 0 ? Quotient : Quotient_A_One)
1112 SDValue Div = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, VT),
1113 Quotient, Quotient_A_One, ISD::SETEQ);
1114
1115 // Div = (Remainder_GE_Zero == 0 ? Quotient_S_One : Div)
1116 Div = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, VT),
1117 Quotient_S_One, Div, ISD::SETEQ);
1118
1119 // Calculate Rem result:
1120
1121 // Remainder_S_Den = Remainder - Den
1122 SDValue Remainder_S_Den = DAG.getNode(ISD::SUB, DL, VT, Remainder, Den);
1123
1124 // Remainder_A_Den = Remainder + Den
1125 SDValue Remainder_A_Den = DAG.getNode(ISD::ADD, DL, VT, Remainder, Den);
1126
1127 // Rem = (Tmp1 == 0 ? Remainder : Remainder_S_Den)
1128 SDValue Rem = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, VT),
1129 Remainder, Remainder_S_Den, ISD::SETEQ);
1130
1131 // Rem = (Remainder_GE_Zero == 0 ? Remainder_A_Den : Rem)
1132 Rem = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, VT),
1133 Remainder_A_Den, Rem, ISD::SETEQ);
Matt Arsenault7939acd2014-04-07 16:44:24 +00001134 SDValue Ops[2] = {
1135 Div,
1136 Rem
1137 };
Craig Topper64941d92014-04-27 19:20:57 +00001138 return DAG.getMergeValues(Ops, DL);
Tom Stellard75aadc22012-12-11 21:25:42 +00001139}
1140
Tom Stellardc947d8c2013-10-30 17:22:05 +00001141SDValue AMDGPUTargetLowering::LowerUINT_TO_FP(SDValue Op,
1142 SelectionDAG &DAG) const {
1143 SDValue S0 = Op.getOperand(0);
1144 SDLoc DL(Op);
1145 if (Op.getValueType() != MVT::f32 || S0.getValueType() != MVT::i64)
1146 return SDValue();
1147
1148 // f32 uint_to_fp i64
1149 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, S0,
1150 DAG.getConstant(0, MVT::i32));
1151 SDValue FloatLo = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, Lo);
1152 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, S0,
1153 DAG.getConstant(1, MVT::i32));
1154 SDValue FloatHi = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, Hi);
1155 FloatHi = DAG.getNode(ISD::FMUL, DL, MVT::f32, FloatHi,
1156 DAG.getConstantFP(4294967296.0f, MVT::f32)); // 2^32
1157 return DAG.getNode(ISD::FADD, DL, MVT::f32, FloatLo, FloatHi);
1158
1159}
Tom Stellardfbab8272013-08-16 01:12:11 +00001160
Matt Arsenaultfae02982014-03-17 18:58:11 +00001161SDValue AMDGPUTargetLowering::ExpandSIGN_EXTEND_INREG(SDValue Op,
1162 unsigned BitsDiff,
1163 SelectionDAG &DAG) const {
1164 MVT VT = Op.getSimpleValueType();
1165 SDLoc DL(Op);
1166 SDValue Shift = DAG.getConstant(BitsDiff, VT);
1167 // Shift left by 'Shift' bits.
1168 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, Op.getOperand(0), Shift);
1169 // Signed shift Right by 'Shift' bits.
1170 return DAG.getNode(ISD::SRA, DL, VT, Shl, Shift);
1171}
1172
1173SDValue AMDGPUTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
1174 SelectionDAG &DAG) const {
1175 EVT ExtraVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1176 MVT VT = Op.getSimpleValueType();
1177 MVT ScalarVT = VT.getScalarType();
1178
Matt Arsenault5dbd5db2014-04-22 03:49:30 +00001179 if (!VT.isVector())
1180 return SDValue();
Matt Arsenaultfae02982014-03-17 18:58:11 +00001181
1182 SDValue Src = Op.getOperand(0);
Matt Arsenault5dbd5db2014-04-22 03:49:30 +00001183 SDLoc DL(Op);
Matt Arsenaultfae02982014-03-17 18:58:11 +00001184
Matt Arsenault5dbd5db2014-04-22 03:49:30 +00001185 // TODO: Don't scalarize on Evergreen?
1186 unsigned NElts = VT.getVectorNumElements();
1187 SmallVector<SDValue, 8> Args;
1188 DAG.ExtractVectorElements(Src, Args, 0, NElts);
Matt Arsenaultfae02982014-03-17 18:58:11 +00001189
Matt Arsenault5dbd5db2014-04-22 03:49:30 +00001190 SDValue VTOp = DAG.getValueType(ExtraVT.getScalarType());
1191 for (unsigned I = 0; I < NElts; ++I)
1192 Args[I] = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, ScalarVT, Args[I], VTOp);
Matt Arsenaultfae02982014-03-17 18:58:11 +00001193
Craig Topper48d114b2014-04-26 18:35:24 +00001194 return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Args);
Matt Arsenaultfae02982014-03-17 18:58:11 +00001195}
1196
Tom Stellard75aadc22012-12-11 21:25:42 +00001197//===----------------------------------------------------------------------===//
Tom Stellard50122a52014-04-07 19:45:41 +00001198// Custom DAG optimizations
1199//===----------------------------------------------------------------------===//
1200
1201static bool isU24(SDValue Op, SelectionDAG &DAG) {
1202 APInt KnownZero, KnownOne;
1203 EVT VT = Op.getValueType();
1204 DAG.ComputeMaskedBits(Op, KnownZero, KnownOne);
1205
1206 return (VT.getSizeInBits() - KnownZero.countLeadingOnes()) <= 24;
1207}
1208
1209static bool isI24(SDValue Op, SelectionDAG &DAG) {
1210 EVT VT = Op.getValueType();
1211
1212 // In order for this to be a signed 24-bit value, bit 23, must
1213 // be a sign bit.
1214 return VT.getSizeInBits() >= 24 && // Types less than 24-bit should be treated
1215 // as unsigned 24-bit values.
1216 (VT.getSizeInBits() - DAG.ComputeNumSignBits(Op)) < 24;
1217}
1218
1219static void simplifyI24(SDValue Op, TargetLowering::DAGCombinerInfo &DCI) {
1220
1221 SelectionDAG &DAG = DCI.DAG;
1222 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1223 EVT VT = Op.getValueType();
1224
1225 APInt Demanded = APInt::getLowBitsSet(VT.getSizeInBits(), 24);
1226 APInt KnownZero, KnownOne;
1227 TargetLowering::TargetLoweringOpt TLO(DAG, true, true);
1228 if (TLI.SimplifyDemandedBits(Op, Demanded, KnownZero, KnownOne, TLO))
1229 DCI.CommitTargetLoweringOpt(TLO);
1230}
1231
1232SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
1233 DAGCombinerInfo &DCI) const {
1234 SelectionDAG &DAG = DCI.DAG;
1235 SDLoc DL(N);
1236
1237 switch(N->getOpcode()) {
1238 default: break;
1239 case ISD::MUL: {
1240 EVT VT = N->getValueType(0);
1241 SDValue N0 = N->getOperand(0);
1242 SDValue N1 = N->getOperand(1);
1243 SDValue Mul;
1244
1245 // FIXME: Add support for 24-bit multiply with 64-bit output on SI.
1246 if (VT.isVector() || VT.getSizeInBits() > 32)
1247 break;
1248
1249 if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) {
1250 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32);
1251 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32);
1252 Mul = DAG.getNode(AMDGPUISD::MUL_U24, DL, MVT::i32, N0, N1);
1253 } else if (Subtarget->hasMulI24() && isI24(N0, DAG) && isI24(N1, DAG)) {
1254 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32);
1255 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32);
1256 Mul = DAG.getNode(AMDGPUISD::MUL_I24, DL, MVT::i32, N0, N1);
1257 } else {
1258 break;
1259 }
1260
Tom Stellardaeeea8a2014-04-17 21:00:13 +00001261 // We need to use sext even for MUL_U24, because MUL_U24 is used
1262 // for signed multiply of 8 and 16-bit types.
Tom Stellard50122a52014-04-07 19:45:41 +00001263 SDValue Reg = DAG.getSExtOrTrunc(Mul, DL, VT);
1264
1265 return Reg;
1266 }
1267 case AMDGPUISD::MUL_I24:
1268 case AMDGPUISD::MUL_U24: {
1269 SDValue N0 = N->getOperand(0);
1270 SDValue N1 = N->getOperand(1);
1271 simplifyI24(N0, DCI);
1272 simplifyI24(N1, DCI);
1273 return SDValue();
1274 }
1275 }
1276 return SDValue();
1277}
1278
1279//===----------------------------------------------------------------------===//
Tom Stellard75aadc22012-12-11 21:25:42 +00001280// Helper functions
1281//===----------------------------------------------------------------------===//
1282
Tom Stellardaf775432013-10-23 00:44:32 +00001283void AMDGPUTargetLowering::getOriginalFunctionArgs(
1284 SelectionDAG &DAG,
1285 const Function *F,
1286 const SmallVectorImpl<ISD::InputArg> &Ins,
1287 SmallVectorImpl<ISD::InputArg> &OrigIns) const {
1288
1289 for (unsigned i = 0, e = Ins.size(); i < e; ++i) {
1290 if (Ins[i].ArgVT == Ins[i].VT) {
1291 OrigIns.push_back(Ins[i]);
1292 continue;
1293 }
1294
1295 EVT VT;
1296 if (Ins[i].ArgVT.isVector() && !Ins[i].VT.isVector()) {
1297 // Vector has been split into scalars.
1298 VT = Ins[i].ArgVT.getVectorElementType();
1299 } else if (Ins[i].VT.isVector() && Ins[i].ArgVT.isVector() &&
1300 Ins[i].ArgVT.getVectorElementType() !=
1301 Ins[i].VT.getVectorElementType()) {
1302 // Vector elements have been promoted
1303 VT = Ins[i].ArgVT;
1304 } else {
1305 // Vector has been spilt into smaller vectors.
1306 VT = Ins[i].VT;
1307 }
1308
1309 ISD::InputArg Arg(Ins[i].Flags, VT, VT, Ins[i].Used,
1310 Ins[i].OrigArgIndex, Ins[i].PartOffset);
1311 OrigIns.push_back(Arg);
1312 }
1313}
1314
Tom Stellard75aadc22012-12-11 21:25:42 +00001315bool AMDGPUTargetLowering::isHWTrueValue(SDValue Op) const {
1316 if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Op)) {
1317 return CFP->isExactlyValue(1.0);
1318 }
1319 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
1320 return C->isAllOnesValue();
1321 }
1322 return false;
1323}
1324
1325bool AMDGPUTargetLowering::isHWFalseValue(SDValue Op) const {
1326 if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Op)) {
1327 return CFP->getValueAPF().isZero();
1328 }
1329 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
1330 return C->isNullValue();
1331 }
1332 return false;
1333}
1334
1335SDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG,
1336 const TargetRegisterClass *RC,
1337 unsigned Reg, EVT VT) const {
1338 MachineFunction &MF = DAG.getMachineFunction();
1339 MachineRegisterInfo &MRI = MF.getRegInfo();
1340 unsigned VirtualRegister;
1341 if (!MRI.isLiveIn(Reg)) {
1342 VirtualRegister = MRI.createVirtualRegister(RC);
1343 MRI.addLiveIn(Reg, VirtualRegister);
1344 } else {
1345 VirtualRegister = MRI.getLiveInVirtReg(Reg);
1346 }
1347 return DAG.getRegister(VirtualRegister, VT);
1348}
1349
1350#define NODE_NAME_CASE(node) case AMDGPUISD::node: return #node;
1351
1352const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
1353 switch (Opcode) {
Craig Topper062a2ba2014-04-25 05:30:21 +00001354 default: return nullptr;
Tom Stellard75aadc22012-12-11 21:25:42 +00001355 // AMDIL DAG nodes
Tom Stellard75aadc22012-12-11 21:25:42 +00001356 NODE_NAME_CASE(CALL);
1357 NODE_NAME_CASE(UMUL);
1358 NODE_NAME_CASE(DIV_INF);
1359 NODE_NAME_CASE(RET_FLAG);
1360 NODE_NAME_CASE(BRANCH_COND);
1361
1362 // AMDGPU DAG nodes
1363 NODE_NAME_CASE(DWORDADDR)
1364 NODE_NAME_CASE(FRACT)
1365 NODE_NAME_CASE(FMAX)
1366 NODE_NAME_CASE(SMAX)
1367 NODE_NAME_CASE(UMAX)
1368 NODE_NAME_CASE(FMIN)
1369 NODE_NAME_CASE(SMIN)
1370 NODE_NAME_CASE(UMIN)
Matt Arsenaultfae02982014-03-17 18:58:11 +00001371 NODE_NAME_CASE(BFE_U32)
1372 NODE_NAME_CASE(BFE_I32)
Matt Arsenaultb3458362014-03-31 18:21:13 +00001373 NODE_NAME_CASE(BFI)
1374 NODE_NAME_CASE(BFM)
Tom Stellard50122a52014-04-07 19:45:41 +00001375 NODE_NAME_CASE(MUL_U24)
1376 NODE_NAME_CASE(MUL_I24)
Tom Stellard75aadc22012-12-11 21:25:42 +00001377 NODE_NAME_CASE(URECIP)
Matt Arsenault21a3faa2014-02-24 21:01:21 +00001378 NODE_NAME_CASE(DOT4)
Tom Stellard75aadc22012-12-11 21:25:42 +00001379 NODE_NAME_CASE(EXPORT)
Tom Stellardff62c352013-01-23 02:09:03 +00001380 NODE_NAME_CASE(CONST_ADDRESS)
Tom Stellardf3b2a1e2013-02-06 17:32:29 +00001381 NODE_NAME_CASE(REGISTER_LOAD)
1382 NODE_NAME_CASE(REGISTER_STORE)
Tom Stellard9fa17912013-08-14 23:24:45 +00001383 NODE_NAME_CASE(LOAD_CONSTANT)
1384 NODE_NAME_CASE(LOAD_INPUT)
1385 NODE_NAME_CASE(SAMPLE)
1386 NODE_NAME_CASE(SAMPLEB)
1387 NODE_NAME_CASE(SAMPLED)
1388 NODE_NAME_CASE(SAMPLEL)
Tom Stellardd3ee8c12013-08-16 01:12:06 +00001389 NODE_NAME_CASE(STORE_MSKOR)
Tom Stellardafcf12f2013-09-12 02:55:14 +00001390 NODE_NAME_CASE(TBUFFER_STORE_FORMAT)
Tom Stellard75aadc22012-12-11 21:25:42 +00001391 }
1392}
Matt Arsenault0c274fe2014-03-25 18:18:27 +00001393
Matt Arsenault378bf9c2014-03-31 19:35:33 +00001394static void computeMaskedBitsForMinMax(const SDValue Op0,
1395 const SDValue Op1,
1396 APInt &KnownZero,
1397 APInt &KnownOne,
1398 const SelectionDAG &DAG,
1399 unsigned Depth) {
1400 APInt Op0Zero, Op0One;
1401 APInt Op1Zero, Op1One;
1402 DAG.ComputeMaskedBits(Op0, Op0Zero, Op0One, Depth);
1403 DAG.ComputeMaskedBits(Op1, Op1Zero, Op1One, Depth);
1404
1405 KnownZero = Op0Zero & Op1Zero;
1406 KnownOne = Op0One & Op1One;
1407}
1408
Matt Arsenault0c274fe2014-03-25 18:18:27 +00001409void AMDGPUTargetLowering::computeMaskedBitsForTargetNode(
1410 const SDValue Op,
1411 APInt &KnownZero,
1412 APInt &KnownOne,
1413 const SelectionDAG &DAG,
1414 unsigned Depth) const {
Matt Arsenault378bf9c2014-03-31 19:35:33 +00001415
Matt Arsenault0c274fe2014-03-25 18:18:27 +00001416 KnownZero = KnownOne = APInt(KnownOne.getBitWidth(), 0); // Don't know anything.
Matt Arsenault378bf9c2014-03-31 19:35:33 +00001417 unsigned Opc = Op.getOpcode();
1418 switch (Opc) {
1419 case ISD::INTRINSIC_WO_CHAIN: {
1420 // FIXME: The intrinsic should just use the node.
1421 switch (cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue()) {
1422 case AMDGPUIntrinsic::AMDGPU_imax:
1423 case AMDGPUIntrinsic::AMDGPU_umax:
1424 case AMDGPUIntrinsic::AMDGPU_imin:
1425 case AMDGPUIntrinsic::AMDGPU_umin:
1426 computeMaskedBitsForMinMax(Op.getOperand(1), Op.getOperand(2),
1427 KnownZero, KnownOne, DAG, Depth);
1428 break;
1429 default:
1430 break;
1431 }
1432
1433 break;
1434 }
1435 case AMDGPUISD::SMAX:
1436 case AMDGPUISD::UMAX:
1437 case AMDGPUISD::SMIN:
1438 case AMDGPUISD::UMIN:
1439 computeMaskedBitsForMinMax(Op.getOperand(0), Op.getOperand(1),
1440 KnownZero, KnownOne, DAG, Depth);
1441 break;
1442 default:
1443 break;
1444 }
Matt Arsenault0c274fe2014-03-25 18:18:27 +00001445}