blob: 90cd4ffaf0bf42ff61cf409e4e0138f6da215acd [file] [log] [blame]
Matt Arsenault7836f892016-01-20 21:22:21 +00001//===-- AMDGPUISelDAGToDAG.cpp - A dag to dag inst selector for AMDGPU ----===//
Tom Stellard75aadc22012-12-11 21:25:42 +00002//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//==-----------------------------------------------------------------------===//
9//
10/// \file
11/// \brief Defines an instruction selector for the AMDGPU target.
12//
13//===----------------------------------------------------------------------===//
Matt Arsenault592d0682015-12-01 23:04:05 +000014
Eugene Zelenko2bc2f332016-12-09 22:06:55 +000015#include "AMDGPU.h"
Matt Arsenault7016f132017-08-03 22:30:46 +000016#include "AMDGPUArgumentUsageInfo.h"
Chandler Carruth6bda14b2017-06-06 11:49:48 +000017#include "AMDGPUISelLowering.h" // For AMDGPUISD
Tom Stellard75aadc22012-12-11 21:25:42 +000018#include "AMDGPUInstrInfo.h"
Eugene Zelenko2bc2f332016-12-09 22:06:55 +000019#include "AMDGPURegisterInfo.h"
Tom Stellard2e59a452014-06-13 01:32:00 +000020#include "AMDGPUSubtarget.h"
Matt Arsenaultcc852232017-10-10 20:22:07 +000021#include "AMDGPUTargetMachine.h"
Eugene Zelenko2bc2f332016-12-09 22:06:55 +000022#include "SIDefines.h"
Christian Konigf82901a2013-02-26 17:52:23 +000023#include "SIISelLowering.h"
Chandler Carruth6bda14b2017-06-06 11:49:48 +000024#include "SIInstrInfo.h"
Tom Stellardb02094e2014-07-21 15:45:01 +000025#include "SIMachineFunctionInfo.h"
Chandler Carruth6bda14b2017-06-06 11:49:48 +000026#include "SIRegisterInfo.h"
Eugene Zelenko2bc2f332016-12-09 22:06:55 +000027#include "llvm/ADT/APInt.h"
28#include "llvm/ADT/SmallVector.h"
29#include "llvm/ADT/StringRef.h"
Alexander Timofeev2e5eece2018-03-05 15:12:21 +000030#include "llvm/Analysis/DivergenceAnalysis.h"
Jan Veselyf97de002016-05-13 20:39:29 +000031#include "llvm/Analysis/ValueTracking.h"
Tom Stellard58ac7442014-04-29 23:12:48 +000032#include "llvm/CodeGen/FunctionLoweringInfo.h"
Eugene Zelenko2bc2f332016-12-09 22:06:55 +000033#include "llvm/CodeGen/ISDOpcodes.h"
34#include "llvm/CodeGen/MachineFunction.h"
35#include "llvm/CodeGen/MachineRegisterInfo.h"
36#include "llvm/CodeGen/MachineValueType.h"
Benjamin Kramerd78bb462013-05-23 17:10:37 +000037#include "llvm/CodeGen/SelectionDAG.h"
Tom Stellard75aadc22012-12-11 21:25:42 +000038#include "llvm/CodeGen/SelectionDAGISel.h"
Eugene Zelenko2bc2f332016-12-09 22:06:55 +000039#include "llvm/CodeGen/SelectionDAGNodes.h"
40#include "llvm/CodeGen/ValueTypes.h"
41#include "llvm/IR/BasicBlock.h"
42#include "llvm/IR/Instruction.h"
43#include "llvm/MC/MCInstrDesc.h"
44#include "llvm/Support/Casting.h"
45#include "llvm/Support/CodeGen.h"
46#include "llvm/Support/ErrorHandling.h"
47#include "llvm/Support/MathExtras.h"
48#include <cassert>
49#include <cstdint>
50#include <new>
51#include <vector>
Tom Stellard75aadc22012-12-11 21:25:42 +000052
53using namespace llvm;
54
Matt Arsenaultd2759212016-02-13 01:24:08 +000055namespace llvm {
Eugene Zelenko2bc2f332016-12-09 22:06:55 +000056
Matt Arsenaultd2759212016-02-13 01:24:08 +000057class R600InstrInfo;
Eugene Zelenko2bc2f332016-12-09 22:06:55 +000058
59} // end namespace llvm
Matt Arsenaultd2759212016-02-13 01:24:08 +000060
Tom Stellard75aadc22012-12-11 21:25:42 +000061//===----------------------------------------------------------------------===//
62// Instruction Selector Implementation
63//===----------------------------------------------------------------------===//
64
65namespace {
Tom Stellardbc4497b2016-02-12 23:45:29 +000066
Tom Stellard75aadc22012-12-11 21:25:42 +000067/// AMDGPU specific code to select AMDGPU machine instructions for
68/// SelectionDAG operations.
69class AMDGPUDAGToDAGISel : public SelectionDAGISel {
70 // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can
71 // make the right decision when generating code for different targets.
Eric Christopher7792e322015-01-30 23:24:40 +000072 const AMDGPUSubtarget *Subtarget;
Yaxun Liu1a14bfa2017-03-27 14:04:01 +000073 AMDGPUAS AMDGPUASI;
Matt Arsenaultcc852232017-10-10 20:22:07 +000074 bool EnableLateStructurizeCFG;
NAKAMURA Takumia9cb5382015-09-22 11:14:39 +000075
Tom Stellard75aadc22012-12-11 21:25:42 +000076public:
Matt Arsenault7016f132017-08-03 22:30:46 +000077 explicit AMDGPUDAGToDAGISel(TargetMachine *TM = nullptr,
78 CodeGenOpt::Level OptLevel = CodeGenOpt::Default)
79 : SelectionDAGISel(*TM, OptLevel) {
80 AMDGPUASI = AMDGPU::getAMDGPUAS(*TM);
Matt Arsenaultcc852232017-10-10 20:22:07 +000081 EnableLateStructurizeCFG = AMDGPUTargetMachine::EnableLateStructurizeCFG;
Yaxun Liu1a14bfa2017-03-27 14:04:01 +000082 }
Eugene Zelenko2bc2f332016-12-09 22:06:55 +000083 ~AMDGPUDAGToDAGISel() override = default;
Konstantin Zhuravlyov60a83732016-10-03 18:47:26 +000084
Matt Arsenault7016f132017-08-03 22:30:46 +000085 void getAnalysisUsage(AnalysisUsage &AU) const override {
86 AU.addRequired<AMDGPUArgumentUsageInfo>();
Alexander Timofeev2e5eece2018-03-05 15:12:21 +000087 AU.addRequired<DivergenceAnalysis>();
Matt Arsenault7016f132017-08-03 22:30:46 +000088 SelectionDAGISel::getAnalysisUsage(AU);
89 }
90
Eric Christopher7792e322015-01-30 23:24:40 +000091 bool runOnMachineFunction(MachineFunction &MF) override;
Justin Bogner95927c02016-05-12 21:03:32 +000092 void Select(SDNode *N) override;
Mehdi Amini117296c2016-10-01 02:56:57 +000093 StringRef getPassName() const override;
Craig Topper5656db42014-04-29 07:57:24 +000094 void PostprocessISelDAG() override;
Tom Stellard75aadc22012-12-11 21:25:42 +000095
Tom Stellard20287692017-08-08 04:57:55 +000096protected:
97 void SelectBuildVector(SDNode *N, unsigned RegClassID);
98
Tom Stellard75aadc22012-12-11 21:25:42 +000099private:
Matt Arsenault156d3ae2017-05-17 21:02:58 +0000100 std::pair<SDValue, SDValue> foldFrameIndex(SDValue N) const;
Matt Arsenaultf84e5d92017-01-31 03:07:46 +0000101 bool isNoNanSrc(SDValue N) const;
Matt Arsenaultfe267752016-07-28 00:32:02 +0000102 bool isInlineImmediate(const SDNode *N) const;
Vincent Lejeunec6896792013-06-04 23:17:15 +0000103 bool FoldOperand(SDValue &Src, SDValue &Sel, SDValue &Neg, SDValue &Abs,
Tom Stellard84021442013-07-23 01:48:24 +0000104 const R600InstrInfo *TII);
Tom Stellard365366f2013-01-23 02:09:06 +0000105 bool FoldOperands(unsigned, const R600InstrInfo *, std::vector<SDValue> &);
Vincent Lejeunec6896792013-06-04 23:17:15 +0000106 bool FoldDotOperands(unsigned, const R600InstrInfo *, std::vector<SDValue> &);
Tom Stellard75aadc22012-12-11 21:25:42 +0000107
Jan Vesely43b7b5b2016-04-07 19:23:11 +0000108 bool isConstantLoad(const MemSDNode *N, int cbID) const;
Tom Stellardbc4497b2016-02-12 23:45:29 +0000109 bool isUniformBr(const SDNode *N) const;
110
Tom Stellard381a94a2015-05-12 15:00:49 +0000111 SDNode *glueCopyToM0(SDNode *N) const;
112
Tom Stellarddf94dc32013-08-14 23:24:24 +0000113 const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const;
Tom Stellard365366f2013-01-23 02:09:06 +0000114 bool SelectGlobalValueConstantOffset(SDValue Addr, SDValue& IntPtr);
Matt Arsenault209a7b92014-04-18 07:40:20 +0000115 bool SelectGlobalValueVariableOffset(SDValue Addr, SDValue &BaseReg,
116 SDValue& Offset);
Tom Stellard20287692017-08-08 04:57:55 +0000117 virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset);
118 virtual bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset);
Tom Stellard85e8b6d2014-08-22 18:49:33 +0000119 bool isDSOffsetLegal(const SDValue &Base, unsigned Offset,
120 unsigned OffsetBits) const;
121 bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const;
Tom Stellardf3fc5552014-08-22 18:49:35 +0000122 bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
123 SDValue &Offset1) const;
Changpeng Fangb41574a2015-12-22 20:55:23 +0000124 bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
Tom Stellard155bbb72014-08-11 22:18:17 +0000125 SDValue &SOffset, SDValue &Offset, SDValue &Offen,
126 SDValue &Idxen, SDValue &Addr64, SDValue &GLC, SDValue &SLC,
127 SDValue &TFE) const;
128 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
Tom Stellard1f9939f2015-02-27 14:59:41 +0000129 SDValue &SOffset, SDValue &Offset, SDValue &GLC,
130 SDValue &SLC, SDValue &TFE) const;
Tom Stellard7980fc82014-09-25 18:30:26 +0000131 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
Tom Stellardc53861a2015-02-11 00:34:32 +0000132 SDValue &VAddr, SDValue &SOffset, SDValue &Offset,
Tom Stellard7980fc82014-09-25 18:30:26 +0000133 SDValue &SLC) const;
Matt Arsenaultb81495d2017-09-20 05:01:53 +0000134 bool SelectMUBUFScratchOffen(SDNode *Parent,
Matt Arsenault156d3ae2017-05-17 21:02:58 +0000135 SDValue Addr, SDValue &RSrc, SDValue &VAddr,
Matt Arsenault0774ea22017-04-24 19:40:59 +0000136 SDValue &SOffset, SDValue &ImmOffset) const;
Matt Arsenaultb81495d2017-09-20 05:01:53 +0000137 bool SelectMUBUFScratchOffset(SDNode *Parent,
Matt Arsenault156d3ae2017-05-17 21:02:58 +0000138 SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
Matt Arsenault0774ea22017-04-24 19:40:59 +0000139 SDValue &Offset) const;
140
Tom Stellard155bbb72014-08-11 22:18:17 +0000141 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &SOffset,
142 SDValue &Offset, SDValue &GLC, SDValue &SLC,
Tom Stellardb02094e2014-07-21 15:45:01 +0000143 SDValue &TFE) const;
Tom Stellard7980fc82014-09-25 18:30:26 +0000144 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
Matt Arsenault88701812016-06-09 23:42:48 +0000145 SDValue &Offset, SDValue &SLC) const;
Jan Vesely43b7b5b2016-04-07 19:23:11 +0000146 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
147 SDValue &Offset) const;
Nicolai Haehnlea6092592016-06-15 07:13:05 +0000148 bool SelectMUBUFConstant(SDValue Constant,
Nicolai Haehnle3003ba02016-03-18 16:24:20 +0000149 SDValue &SOffset,
150 SDValue &ImmOffset) const;
151 bool SelectMUBUFIntrinsicOffset(SDValue Offset, SDValue &SOffset,
152 SDValue &ImmOffset) const;
153 bool SelectMUBUFIntrinsicVOffset(SDValue Offset, SDValue &SOffset,
154 SDValue &ImmOffset, SDValue &VOffset) const;
Matt Arsenault7757c592016-06-09 23:42:54 +0000155
Matt Arsenaultdb7c6a82017-06-12 16:53:51 +0000156 bool SelectFlatAtomic(SDValue Addr, SDValue &VAddr,
157 SDValue &Offset, SDValue &SLC) const;
Matt Arsenault4e309b02017-07-29 01:03:53 +0000158 bool SelectFlatAtomicSigned(SDValue Addr, SDValue &VAddr,
159 SDValue &Offset, SDValue &SLC) const;
160
161 template <bool IsSigned>
Matt Arsenaultdb7c6a82017-06-12 16:53:51 +0000162 bool SelectFlatOffset(SDValue Addr, SDValue &VAddr,
163 SDValue &Offset, SDValue &SLC) const;
Matt Arsenault7757c592016-06-09 23:42:54 +0000164
Tom Stellarddee26a22015-08-06 19:28:30 +0000165 bool SelectSMRDOffset(SDValue ByteOffsetNode, SDValue &Offset,
166 bool &Imm) const;
Matt Arsenault923712b2018-02-09 16:57:57 +0000167 SDValue Expand32BitAddress(SDValue Addr) const;
Tom Stellarddee26a22015-08-06 19:28:30 +0000168 bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue &Offset,
169 bool &Imm) const;
170 bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
Marek Olsak8973a0a2017-05-24 14:53:50 +0000171 bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
Tom Stellarddee26a22015-08-06 19:28:30 +0000172 bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
173 bool SelectSMRDBufferImm(SDValue Addr, SDValue &Offset) const;
Marek Olsak8973a0a2017-05-24 14:53:50 +0000174 bool SelectSMRDBufferImm32(SDValue Addr, SDValue &Offset) const;
Nicolai Haehnle7968c342016-07-12 08:12:16 +0000175 bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const;
Matt Arsenaultf84e5d92017-01-31 03:07:46 +0000176
177 bool SelectVOP3Mods_NNaN(SDValue In, SDValue &Src, SDValue &SrcMods) const;
Matt Arsenaultd7e23032017-09-07 18:05:07 +0000178 bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods) const;
Tom Stellardb4a313a2014-08-01 00:32:39 +0000179 bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
Matt Arsenaultdf58e822017-04-25 21:17:38 +0000180 bool SelectVOP3NoMods(SDValue In, SDValue &Src) const;
Tom Stellardb4a313a2014-08-01 00:32:39 +0000181 bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
182 SDValue &Clamp, SDValue &Omod) const;
Tom Stellarddb5a11f2015-07-13 15:47:57 +0000183 bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
184 SDValue &Clamp, SDValue &Omod) const;
Tom Stellard75aadc22012-12-11 21:25:42 +0000185
Matt Arsenault4831ce52015-01-06 23:00:37 +0000186 bool SelectVOP3Mods0Clamp0OMod(SDValue In, SDValue &Src, SDValue &SrcMods,
187 SDValue &Clamp,
188 SDValue &Omod) const;
Matt Arsenault1cffa4c2014-11-13 19:49:04 +0000189
Dmitry Preobrazhenskyc512d442017-03-27 15:57:17 +0000190 bool SelectVOP3OMods(SDValue In, SDValue &Src,
191 SDValue &Clamp, SDValue &Omod) const;
192
Matt Arsenaulteb522e62017-02-27 22:15:25 +0000193 bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
194 bool SelectVOP3PMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
195 SDValue &Clamp) const;
196
Dmitry Preobrazhenskyabf28392017-07-21 13:54:11 +0000197 bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods) const;
198 bool SelectVOP3OpSel0(SDValue In, SDValue &Src, SDValue &SrcMods,
199 SDValue &Clamp) const;
200
201 bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
202 bool SelectVOP3OpSelMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
203 SDValue &Clamp) const;
Matt Arsenaultd7e23032017-09-07 18:05:07 +0000204 bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, unsigned &Mods) const;
Matt Arsenault76935122017-09-20 20:28:39 +0000205 bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
Dmitry Preobrazhenskyabf28392017-07-21 13:54:11 +0000206
Matt Arsenaulte1cd4822017-11-13 00:22:09 +0000207 bool SelectHi16Elt(SDValue In, SDValue &Src) const;
208
Justin Bogner95927c02016-05-12 21:03:32 +0000209 void SelectADD_SUB_I64(SDNode *N);
Matt Arsenaultee3f0ac2017-01-30 18:11:38 +0000210 void SelectUADDO_USUBO(SDNode *N);
Justin Bogner95927c02016-05-12 21:03:32 +0000211 void SelectDIV_SCALE(SDNode *N);
Matt Arsenault4f6318f2017-11-06 17:04:37 +0000212 void SelectMAD_64_32(SDNode *N);
Tom Stellard8485fa02016-12-07 02:42:15 +0000213 void SelectFMA_W_CHAIN(SDNode *N);
214 void SelectFMUL_W_CHAIN(SDNode *N);
Matt Arsenault9fa3f932014-06-23 18:00:34 +0000215
Benjamin Kramerbdc49562016-06-12 15:39:02 +0000216 SDNode *getS_BFE(unsigned Opcode, const SDLoc &DL, SDValue Val,
Marek Olsak9b728682015-03-24 13:40:27 +0000217 uint32_t Offset, uint32_t Width);
Justin Bogner95927c02016-05-12 21:03:32 +0000218 void SelectS_BFEFromShifts(SDNode *N);
219 void SelectS_BFE(SDNode *N);
Matt Arsenault7b1dc2c2016-09-17 02:02:19 +0000220 bool isCBranchSCC(const SDNode *N) const;
Justin Bogner95927c02016-05-12 21:03:32 +0000221 void SelectBRCOND(SDNode *N);
Matt Arsenaultd7e23032017-09-07 18:05:07 +0000222 void SelectFMAD(SDNode *N);
Matt Arsenault88701812016-06-09 23:42:48 +0000223 void SelectATOMIC_CMP_SWAP(SDNode *N);
Marek Olsak9b728682015-03-24 13:40:27 +0000224
Tom Stellard20287692017-08-08 04:57:55 +0000225protected:
Tom Stellard75aadc22012-12-11 21:25:42 +0000226 // Include the pieces autogenerated from the target description.
227#include "AMDGPUGenDAGISel.inc"
228};
Eugene Zelenko2bc2f332016-12-09 22:06:55 +0000229
Tom Stellard20287692017-08-08 04:57:55 +0000230class R600DAGToDAGISel : public AMDGPUDAGToDAGISel {
231public:
232 explicit R600DAGToDAGISel(TargetMachine *TM, CodeGenOpt::Level OptLevel) :
233 AMDGPUDAGToDAGISel(TM, OptLevel) {}
234
235 void Select(SDNode *N) override;
236
237 bool SelectADDRIndirect(SDValue Addr, SDValue &Base,
238 SDValue &Offset) override;
239 bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
240 SDValue &Offset) override;
241};
242
Tom Stellard75aadc22012-12-11 21:25:42 +0000243} // end anonymous namespace
244
Matt Arsenault7016f132017-08-03 22:30:46 +0000245INITIALIZE_PASS_BEGIN(AMDGPUDAGToDAGISel, "isel",
246 "AMDGPU DAG->DAG Pattern Instruction Selection", false, false)
247INITIALIZE_PASS_DEPENDENCY(AMDGPUArgumentUsageInfo)
248INITIALIZE_PASS_END(AMDGPUDAGToDAGISel, "isel",
249 "AMDGPU DAG->DAG Pattern Instruction Selection", false, false)
250
Tom Stellard75aadc22012-12-11 21:25:42 +0000251/// \brief This pass converts a legalized DAG into a AMDGPU-specific
252// DAG, ready for instruction scheduling.
Matt Arsenault7016f132017-08-03 22:30:46 +0000253FunctionPass *llvm::createAMDGPUISelDag(TargetMachine *TM,
Konstantin Zhuravlyov60a83732016-10-03 18:47:26 +0000254 CodeGenOpt::Level OptLevel) {
255 return new AMDGPUDAGToDAGISel(TM, OptLevel);
Tom Stellard75aadc22012-12-11 21:25:42 +0000256}
257
Tom Stellard20287692017-08-08 04:57:55 +0000258/// \brief This pass converts a legalized DAG into a R600-specific
259// DAG, ready for instruction scheduling.
260FunctionPass *llvm::createR600ISelDag(TargetMachine *TM,
261 CodeGenOpt::Level OptLevel) {
262 return new R600DAGToDAGISel(TM, OptLevel);
263}
264
Eric Christopher7792e322015-01-30 23:24:40 +0000265bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
Matt Arsenault43e92fe2016-06-24 06:30:11 +0000266 Subtarget = &MF.getSubtarget<AMDGPUSubtarget>();
Eric Christopher7792e322015-01-30 23:24:40 +0000267 return SelectionDAGISel::runOnMachineFunction(MF);
Tom Stellard75aadc22012-12-11 21:25:42 +0000268}
269
Matt Arsenaultf84e5d92017-01-31 03:07:46 +0000270bool AMDGPUDAGToDAGISel::isNoNanSrc(SDValue N) const {
271 if (TM.Options.NoNaNsFPMath)
272 return true;
273
274 // TODO: Move into isKnownNeverNaN
Amara Emersond28f0cd42017-05-01 15:17:51 +0000275 if (N->getFlags().isDefined())
276 return N->getFlags().hasNoNaNs();
Matt Arsenaultf84e5d92017-01-31 03:07:46 +0000277
278 return CurDAG->isKnownNeverNaN(N);
279}
280
Matt Arsenaultfe267752016-07-28 00:32:02 +0000281bool AMDGPUDAGToDAGISel::isInlineImmediate(const SDNode *N) const {
282 const SIInstrInfo *TII
283 = static_cast<const SISubtarget *>(Subtarget)->getInstrInfo();
284
285 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N))
286 return TII->isInlineConstant(C->getAPIntValue());
287
288 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N))
289 return TII->isInlineConstant(C->getValueAPF().bitcastToAPInt());
290
291 return false;
Tom Stellard7ed0b522014-04-03 20:19:27 +0000292}
293
Tom Stellarddf94dc32013-08-14 23:24:24 +0000294/// \brief Determine the register class for \p OpNo
295/// \returns The register class of the virtual register that will be used for
296/// the given operand number \OpNo or NULL if the register class cannot be
297/// determined.
298const TargetRegisterClass *AMDGPUDAGToDAGISel::getOperandRegClass(SDNode *N,
299 unsigned OpNo) const {
Matt Arsenaultc507cdb2016-11-01 23:22:17 +0000300 if (!N->isMachineOpcode()) {
301 if (N->getOpcode() == ISD::CopyToReg) {
302 unsigned Reg = cast<RegisterSDNode>(N->getOperand(1))->getReg();
303 if (TargetRegisterInfo::isVirtualRegister(Reg)) {
304 MachineRegisterInfo &MRI = CurDAG->getMachineFunction().getRegInfo();
305 return MRI.getRegClass(Reg);
306 }
307
308 const SIRegisterInfo *TRI
309 = static_cast<const SISubtarget *>(Subtarget)->getRegisterInfo();
310 return TRI->getPhysRegClass(Reg);
311 }
312
Matt Arsenault209a7b92014-04-18 07:40:20 +0000313 return nullptr;
Matt Arsenaultc507cdb2016-11-01 23:22:17 +0000314 }
Matt Arsenault209a7b92014-04-18 07:40:20 +0000315
Tom Stellarddf94dc32013-08-14 23:24:24 +0000316 switch (N->getMachineOpcode()) {
317 default: {
Eric Christopherd9134482014-08-04 21:25:23 +0000318 const MCInstrDesc &Desc =
Eric Christopher7792e322015-01-30 23:24:40 +0000319 Subtarget->getInstrInfo()->get(N->getMachineOpcode());
Alexey Samsonov3186eb32013-08-15 07:11:34 +0000320 unsigned OpIdx = Desc.getNumDefs() + OpNo;
321 if (OpIdx >= Desc.getNumOperands())
Matt Arsenault209a7b92014-04-18 07:40:20 +0000322 return nullptr;
Alexey Samsonov3186eb32013-08-15 07:11:34 +0000323 int RegClass = Desc.OpInfo[OpIdx].RegClass;
Matt Arsenault209a7b92014-04-18 07:40:20 +0000324 if (RegClass == -1)
325 return nullptr;
326
Eric Christopher7792e322015-01-30 23:24:40 +0000327 return Subtarget->getRegisterInfo()->getRegClass(RegClass);
Tom Stellarddf94dc32013-08-14 23:24:24 +0000328 }
329 case AMDGPU::REG_SEQUENCE: {
Matt Arsenault209a7b92014-04-18 07:40:20 +0000330 unsigned RCID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
Eric Christopherd9134482014-08-04 21:25:23 +0000331 const TargetRegisterClass *SuperRC =
Eric Christopher7792e322015-01-30 23:24:40 +0000332 Subtarget->getRegisterInfo()->getRegClass(RCID);
Matt Arsenault209a7b92014-04-18 07:40:20 +0000333
334 SDValue SubRegOp = N->getOperand(OpNo + 1);
335 unsigned SubRegIdx = cast<ConstantSDNode>(SubRegOp)->getZExtValue();
Eric Christopher7792e322015-01-30 23:24:40 +0000336 return Subtarget->getRegisterInfo()->getSubClassWithSubReg(SuperRC,
337 SubRegIdx);
Tom Stellarddf94dc32013-08-14 23:24:24 +0000338 }
339 }
340}
341
Tom Stellard381a94a2015-05-12 15:00:49 +0000342SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N) const {
Matt Arsenault3f71c0e2017-11-29 00:55:57 +0000343 if (cast<MemSDNode>(N)->getAddressSpace() != AMDGPUASI.LOCAL_ADDRESS ||
344 !Subtarget->ldsRequiresM0Init())
Tom Stellard381a94a2015-05-12 15:00:49 +0000345 return N;
346
347 const SITargetLowering& Lowering =
348 *static_cast<const SITargetLowering*>(getTargetLowering());
349
350 // Write max value to m0 before each load operation
351
352 SDValue M0 = Lowering.copyToM0(*CurDAG, CurDAG->getEntryNode(), SDLoc(N),
353 CurDAG->getTargetConstant(-1, SDLoc(N), MVT::i32));
354
355 SDValue Glue = M0.getValue(1);
356
357 SmallVector <SDValue, 8> Ops;
358 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
359 Ops.push_back(N->getOperand(i));
360 }
361 Ops.push_back(Glue);
Matt Arsenaulte6667de2017-12-04 22:18:22 +0000362 return CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), Ops);
Tom Stellard381a94a2015-05-12 15:00:49 +0000363}
364
Matt Arsenault61cb6fa2015-11-11 00:01:36 +0000365static unsigned selectSGPRVectorRegClassID(unsigned NumVectorElts) {
Matt Arsenaultf1aebbf2015-11-02 23:30:48 +0000366 switch (NumVectorElts) {
367 case 1:
Marek Olsak79c05872016-11-25 17:37:09 +0000368 return AMDGPU::SReg_32_XM0RegClassID;
Matt Arsenaultf1aebbf2015-11-02 23:30:48 +0000369 case 2:
370 return AMDGPU::SReg_64RegClassID;
371 case 4:
372 return AMDGPU::SReg_128RegClassID;
373 case 8:
374 return AMDGPU::SReg_256RegClassID;
375 case 16:
376 return AMDGPU::SReg_512RegClassID;
377 }
378
379 llvm_unreachable("invalid vector size");
380}
381
Matt Arsenaulteb522e62017-02-27 22:15:25 +0000382static bool getConstantValue(SDValue N, uint32_t &Out) {
383 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
384 Out = C->getAPIntValue().getZExtValue();
385 return true;
386 }
387
388 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) {
389 Out = C->getValueAPF().bitcastToAPInt().getZExtValue();
390 return true;
391 }
392
393 return false;
394}
395
Tom Stellard20287692017-08-08 04:57:55 +0000396void AMDGPUDAGToDAGISel::SelectBuildVector(SDNode *N, unsigned RegClassID) {
Tom Stellard20287692017-08-08 04:57:55 +0000397 EVT VT = N->getValueType(0);
398 unsigned NumVectorElts = VT.getVectorNumElements();
399 EVT EltVT = VT.getVectorElementType();
400 const AMDGPURegisterInfo *TRI = Subtarget->getRegisterInfo();
401 SDLoc DL(N);
402 SDValue RegClass = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
403
404 if (NumVectorElts == 1) {
405 CurDAG->SelectNodeTo(N, AMDGPU::COPY_TO_REGCLASS, EltVT, N->getOperand(0),
406 RegClass);
407 return;
408 }
409
410 assert(NumVectorElts <= 16 && "Vectors with more than 16 elements not "
411 "supported yet");
412 // 16 = Max Num Vector Elements
413 // 2 = 2 REG_SEQUENCE operands per element (value, subreg index)
414 // 1 = Vector Register Class
415 SmallVector<SDValue, 16 * 2 + 1> RegSeqArgs(NumVectorElts * 2 + 1);
416
417 RegSeqArgs[0] = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
418 bool IsRegSeq = true;
419 unsigned NOps = N->getNumOperands();
420 for (unsigned i = 0; i < NOps; i++) {
421 // XXX: Why is this here?
422 if (isa<RegisterSDNode>(N->getOperand(i))) {
423 IsRegSeq = false;
424 break;
425 }
426 RegSeqArgs[1 + (2 * i)] = N->getOperand(i);
427 RegSeqArgs[1 + (2 * i) + 1] =
428 CurDAG->getTargetConstant(TRI->getSubRegFromChannel(i), DL,
429 MVT::i32);
430 }
431 if (NOps != NumVectorElts) {
432 // Fill in the missing undef elements if this was a scalar_to_vector.
Tom Stellard03aa3ae2017-08-08 05:52:00 +0000433 assert(N->getOpcode() == ISD::SCALAR_TO_VECTOR && NOps < NumVectorElts);
Tom Stellard20287692017-08-08 04:57:55 +0000434 MachineSDNode *ImpDef = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
435 DL, EltVT);
436 for (unsigned i = NOps; i < NumVectorElts; ++i) {
437 RegSeqArgs[1 + (2 * i)] = SDValue(ImpDef, 0);
438 RegSeqArgs[1 + (2 * i) + 1] =
439 CurDAG->getTargetConstant(TRI->getSubRegFromChannel(i), DL, MVT::i32);
440 }
441 }
442
443 if (!IsRegSeq)
444 SelectCode(N);
445 CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, N->getVTList(), RegSeqArgs);
446}
447
Justin Bogner95927c02016-05-12 21:03:32 +0000448void AMDGPUDAGToDAGISel::Select(SDNode *N) {
Tom Stellard75aadc22012-12-11 21:25:42 +0000449 unsigned int Opc = N->getOpcode();
450 if (N->isMachineOpcode()) {
Tim Northover31d093c2013-09-22 08:21:56 +0000451 N->setNodeId(-1);
Justin Bogner95927c02016-05-12 21:03:32 +0000452 return; // Already selected.
Tom Stellard75aadc22012-12-11 21:25:42 +0000453 }
Matt Arsenault78b86702014-04-18 05:19:26 +0000454
Matt Arsenaulta9dbdca2016-04-12 14:05:04 +0000455 if (isa<AtomicSDNode>(N) ||
Daniil Fukalovd5fca552018-01-17 14:05:05 +0000456 (Opc == AMDGPUISD::ATOMIC_INC || Opc == AMDGPUISD::ATOMIC_DEC ||
457 Opc == AMDGPUISD::ATOMIC_LOAD_FADD ||
458 Opc == AMDGPUISD::ATOMIC_LOAD_FMIN ||
459 Opc == AMDGPUISD::ATOMIC_LOAD_FMAX))
Tom Stellard381a94a2015-05-12 15:00:49 +0000460 N = glueCopyToM0(N);
461
Tom Stellard75aadc22012-12-11 21:25:42 +0000462 switch (Opc) {
Matt Arsenault84445dd2017-11-30 22:51:26 +0000463 default:
464 break;
Tom Stellard1f15bff2014-02-25 21:36:18 +0000465 // We are selecting i64 ADD here instead of custom lower it during
466 // DAG legalization, so we can fold some i64 ADDs used for address
467 // calculation into the LOAD and STORE instructions.
Nicolai Haehnle67624af2016-10-14 10:30:00 +0000468 case ISD::ADDC:
469 case ISD::ADDE:
Nicolai Haehnle67624af2016-10-14 10:30:00 +0000470 case ISD::SUBC:
471 case ISD::SUBE: {
Tom Stellard20287692017-08-08 04:57:55 +0000472 if (N->getValueType(0) != MVT::i64)
Tom Stellard1f15bff2014-02-25 21:36:18 +0000473 break;
474
Justin Bogner95927c02016-05-12 21:03:32 +0000475 SelectADD_SUB_I64(N);
476 return;
Tom Stellard1f15bff2014-02-25 21:36:18 +0000477 }
Matt Arsenaultee3f0ac2017-01-30 18:11:38 +0000478 case ISD::UADDO:
479 case ISD::USUBO: {
480 SelectUADDO_USUBO(N);
481 return;
482 }
Tom Stellard8485fa02016-12-07 02:42:15 +0000483 case AMDGPUISD::FMUL_W_CHAIN: {
484 SelectFMUL_W_CHAIN(N);
485 return;
486 }
487 case AMDGPUISD::FMA_W_CHAIN: {
488 SelectFMA_W_CHAIN(N);
489 return;
490 }
491
Matt Arsenault064c2062014-06-11 17:40:32 +0000492 case ISD::SCALAR_TO_VECTOR:
Vincent Lejeune3b6f20e2013-03-05 15:04:49 +0000493 case ISD::BUILD_VECTOR: {
Tom Stellard8e5da412013-08-14 23:24:32 +0000494 EVT VT = N->getValueType(0);
495 unsigned NumVectorElts = VT.getVectorNumElements();
Matt Arsenaulteb522e62017-02-27 22:15:25 +0000496
497 if (VT == MVT::v2i16 || VT == MVT::v2f16) {
498 if (Opc == ISD::BUILD_VECTOR) {
499 uint32_t LHSVal, RHSVal;
500 if (getConstantValue(N->getOperand(0), LHSVal) &&
501 getConstantValue(N->getOperand(1), RHSVal)) {
502 uint32_t K = LHSVal | (RHSVal << 16);
503 CurDAG->SelectNodeTo(N, AMDGPU::S_MOV_B32, VT,
504 CurDAG->getTargetConstant(K, SDLoc(N), MVT::i32));
505 return;
506 }
507 }
508
509 break;
510 }
511
Tom Stellard03aa3ae2017-08-08 05:52:00 +0000512 assert(VT.getVectorElementType().bitsEq(MVT::i32));
Tom Stellard20287692017-08-08 04:57:55 +0000513 unsigned RegClassID = selectSGPRVectorRegClassID(NumVectorElts);
514 SelectBuildVector(N, RegClassID);
Justin Bogner95927c02016-05-12 21:03:32 +0000515 return;
Vincent Lejeune3b6f20e2013-03-05 15:04:49 +0000516 }
Tom Stellard754f80f2013-04-05 23:31:51 +0000517 case ISD::BUILD_PAIR: {
518 SDValue RC, SubReg0, SubReg1;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000519 SDLoc DL(N);
Tom Stellard754f80f2013-04-05 23:31:51 +0000520 if (N->getValueType(0) == MVT::i128) {
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000521 RC = CurDAG->getTargetConstant(AMDGPU::SReg_128RegClassID, DL, MVT::i32);
522 SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32);
523 SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32);
Tom Stellard754f80f2013-04-05 23:31:51 +0000524 } else if (N->getValueType(0) == MVT::i64) {
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000525 RC = CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32);
526 SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
527 SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
Tom Stellard754f80f2013-04-05 23:31:51 +0000528 } else {
529 llvm_unreachable("Unhandled value type for BUILD_PAIR");
530 }
531 const SDValue Ops[] = { RC, N->getOperand(0), SubReg0,
532 N->getOperand(1), SubReg1 };
Justin Bogner95927c02016-05-12 21:03:32 +0000533 ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL,
534 N->getValueType(0), Ops));
535 return;
Tom Stellard754f80f2013-04-05 23:31:51 +0000536 }
Tom Stellard7ed0b522014-04-03 20:19:27 +0000537
538 case ISD::Constant:
539 case ISD::ConstantFP: {
Tom Stellard20287692017-08-08 04:57:55 +0000540 if (N->getValueType(0).getSizeInBits() != 64 || isInlineImmediate(N))
Tom Stellard7ed0b522014-04-03 20:19:27 +0000541 break;
542
543 uint64_t Imm;
544 if (ConstantFPSDNode *FP = dyn_cast<ConstantFPSDNode>(N))
545 Imm = FP->getValueAPF().bitcastToAPInt().getZExtValue();
546 else {
Tom Stellard3cbe0142014-04-07 19:31:13 +0000547 ConstantSDNode *C = cast<ConstantSDNode>(N);
Tom Stellard7ed0b522014-04-03 20:19:27 +0000548 Imm = C->getZExtValue();
549 }
550
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000551 SDLoc DL(N);
552 SDNode *Lo = CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
553 CurDAG->getConstant(Imm & 0xFFFFFFFF, DL,
554 MVT::i32));
555 SDNode *Hi = CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
556 CurDAG->getConstant(Imm >> 32, DL, MVT::i32));
Tom Stellard7ed0b522014-04-03 20:19:27 +0000557 const SDValue Ops[] = {
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000558 CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
559 SDValue(Lo, 0), CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
560 SDValue(Hi, 0), CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32)
Tom Stellard7ed0b522014-04-03 20:19:27 +0000561 };
562
Justin Bogner95927c02016-05-12 21:03:32 +0000563 ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL,
564 N->getValueType(0), Ops));
565 return;
Tom Stellard7ed0b522014-04-03 20:19:27 +0000566 }
Matt Arsenault4bf43d42015-09-25 17:27:08 +0000567 case ISD::LOAD:
Tom Stellard096b8c12015-02-04 20:49:49 +0000568 case ISD::STORE: {
Tom Stellard381a94a2015-05-12 15:00:49 +0000569 N = glueCopyToM0(N);
Tom Stellard096b8c12015-02-04 20:49:49 +0000570 break;
571 }
Matt Arsenault78b86702014-04-18 05:19:26 +0000572
573 case AMDGPUISD::BFE_I32:
574 case AMDGPUISD::BFE_U32: {
Matt Arsenault78b86702014-04-18 05:19:26 +0000575 // There is a scalar version available, but unlike the vector version which
576 // has a separate operand for the offset and width, the scalar version packs
577 // the width and offset into a single operand. Try to move to the scalar
578 // version if the offsets are constant, so that we can try to keep extended
579 // loads of kernel arguments in SGPRs.
580
581 // TODO: Technically we could try to pattern match scalar bitshifts of
582 // dynamic values, but it's probably not useful.
583 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
584 if (!Offset)
585 break;
586
587 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2));
588 if (!Width)
589 break;
590
591 bool Signed = Opc == AMDGPUISD::BFE_I32;
592
Matt Arsenault78b86702014-04-18 05:19:26 +0000593 uint32_t OffsetVal = Offset->getZExtValue();
594 uint32_t WidthVal = Width->getZExtValue();
595
Justin Bogner95927c02016-05-12 21:03:32 +0000596 ReplaceNode(N, getS_BFE(Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32,
597 SDLoc(N), N->getOperand(0), OffsetVal, WidthVal));
598 return;
Matt Arsenault78b86702014-04-18 05:19:26 +0000599 }
Matt Arsenaultf2b0aeb2014-06-23 18:28:28 +0000600 case AMDGPUISD::DIV_SCALE: {
Justin Bogner95927c02016-05-12 21:03:32 +0000601 SelectDIV_SCALE(N);
602 return;
Matt Arsenaultf2b0aeb2014-06-23 18:28:28 +0000603 }
Matt Arsenault4f6318f2017-11-06 17:04:37 +0000604 case AMDGPUISD::MAD_I64_I32:
605 case AMDGPUISD::MAD_U64_U32: {
606 SelectMAD_64_32(N);
607 return;
608 }
Tom Stellard3457a842014-10-09 19:06:00 +0000609 case ISD::CopyToReg: {
610 const SITargetLowering& Lowering =
611 *static_cast<const SITargetLowering*>(getTargetLowering());
Matt Arsenault0d0d6c22017-04-12 21:58:23 +0000612 N = Lowering.legalizeTargetIndependentNode(N, *CurDAG);
Tom Stellard3457a842014-10-09 19:06:00 +0000613 break;
614 }
Marek Olsak9b728682015-03-24 13:40:27 +0000615 case ISD::AND:
616 case ISD::SRL:
617 case ISD::SRA:
Matt Arsenault7e8de012016-04-22 22:59:16 +0000618 case ISD::SIGN_EXTEND_INREG:
Tom Stellard20287692017-08-08 04:57:55 +0000619 if (N->getValueType(0) != MVT::i32)
Marek Olsak9b728682015-03-24 13:40:27 +0000620 break;
621
Justin Bogner95927c02016-05-12 21:03:32 +0000622 SelectS_BFE(N);
623 return;
Tom Stellardbc4497b2016-02-12 23:45:29 +0000624 case ISD::BRCOND:
Justin Bogner95927c02016-05-12 21:03:32 +0000625 SelectBRCOND(N);
626 return;
Matt Arsenaultd7e23032017-09-07 18:05:07 +0000627 case ISD::FMAD:
628 SelectFMAD(N);
629 return;
Matt Arsenault88701812016-06-09 23:42:48 +0000630 case AMDGPUISD::ATOMIC_CMP_SWAP:
631 SelectATOMIC_CMP_SWAP(N);
632 return;
Tom Stellard75aadc22012-12-11 21:25:42 +0000633 }
Tom Stellard3457a842014-10-09 19:06:00 +0000634
Justin Bogner95927c02016-05-12 21:03:32 +0000635 SelectCode(N);
Tom Stellard365366f2013-01-23 02:09:06 +0000636}
637
Jan Vesely43b7b5b2016-04-07 19:23:11 +0000638bool AMDGPUDAGToDAGISel::isConstantLoad(const MemSDNode *N, int CbId) const {
639 if (!N->readMem())
640 return false;
Matt Arsenault209a7b92014-04-18 07:40:20 +0000641 if (CbId == -1)
Matt Arsenault923712b2018-02-09 16:57:57 +0000642 return N->getAddressSpace() == AMDGPUASI.CONSTANT_ADDRESS ||
643 N->getAddressSpace() == AMDGPUASI.CONSTANT_ADDRESS_32BIT;
Matt Arsenault209a7b92014-04-18 07:40:20 +0000644
Yaxun Liu1a14bfa2017-03-27 14:04:01 +0000645 return N->getAddressSpace() == AMDGPUASI.CONSTANT_BUFFER_0 + CbId;
Matt Arsenault3f981402014-09-15 15:41:53 +0000646}
647
Tom Stellardbc4497b2016-02-12 23:45:29 +0000648bool AMDGPUDAGToDAGISel::isUniformBr(const SDNode *N) const {
649 const BasicBlock *BB = FuncInfo->MBB->getBasicBlock();
Nicolai Haehnle05b127d2016-04-14 17:42:35 +0000650 const Instruction *Term = BB->getTerminator();
651 return Term->getMetadata("amdgpu.uniform") ||
652 Term->getMetadata("structurizecfg.uniform");
Tom Stellardbc4497b2016-02-12 23:45:29 +0000653}
654
Mehdi Amini117296c2016-10-01 02:56:57 +0000655StringRef AMDGPUDAGToDAGISel::getPassName() const {
Tom Stellard75aadc22012-12-11 21:25:42 +0000656 return "AMDGPU DAG->DAG Pattern Instruction Selection";
657}
658
Tom Stellard41fc7852013-07-23 01:48:42 +0000659//===----------------------------------------------------------------------===//
660// Complex Patterns
661//===----------------------------------------------------------------------===//
Tom Stellard75aadc22012-12-11 21:25:42 +0000662
Tom Stellard365366f2013-01-23 02:09:06 +0000663bool AMDGPUDAGToDAGISel::SelectGlobalValueConstantOffset(SDValue Addr,
Matt Arsenault209a7b92014-04-18 07:40:20 +0000664 SDValue& IntPtr) {
Tom Stellard365366f2013-01-23 02:09:06 +0000665 if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Addr)) {
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000666 IntPtr = CurDAG->getIntPtrConstant(Cst->getZExtValue() / 4, SDLoc(Addr),
667 true);
Tom Stellard365366f2013-01-23 02:09:06 +0000668 return true;
669 }
670 return false;
671}
672
673bool AMDGPUDAGToDAGISel::SelectGlobalValueVariableOffset(SDValue Addr,
674 SDValue& BaseReg, SDValue &Offset) {
Matt Arsenault209a7b92014-04-18 07:40:20 +0000675 if (!isa<ConstantSDNode>(Addr)) {
Tom Stellard365366f2013-01-23 02:09:06 +0000676 BaseReg = Addr;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000677 Offset = CurDAG->getIntPtrConstant(0, SDLoc(Addr), true);
Tom Stellard365366f2013-01-23 02:09:06 +0000678 return true;
679 }
680 return false;
681}
682
Tom Stellard75aadc22012-12-11 21:25:42 +0000683bool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
Tom Stellard20287692017-08-08 04:57:55 +0000684 SDValue &Offset) {
685 return false;
Tom Stellard75aadc22012-12-11 21:25:42 +0000686}
687
Tom Stellardf3b2a1e2013-02-06 17:32:29 +0000688bool AMDGPUDAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base,
689 SDValue &Offset) {
690 ConstantSDNode *C;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000691 SDLoc DL(Addr);
Tom Stellardf3b2a1e2013-02-06 17:32:29 +0000692
693 if ((C = dyn_cast<ConstantSDNode>(Addr))) {
694 Base = CurDAG->getRegister(AMDGPU::INDIRECT_BASE_ADDR, MVT::i32);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000695 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
Jan Vesely06200bd2017-01-06 21:00:46 +0000696 } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) &&
697 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) {
698 Base = CurDAG->getRegister(AMDGPU::INDIRECT_BASE_ADDR, MVT::i32);
699 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
Tom Stellardf3b2a1e2013-02-06 17:32:29 +0000700 } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) &&
701 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) {
702 Base = Addr.getOperand(0);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000703 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
Tom Stellardf3b2a1e2013-02-06 17:32:29 +0000704 } else {
705 Base = Addr;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000706 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
Tom Stellardf3b2a1e2013-02-06 17:32:29 +0000707 }
708
709 return true;
710}
Christian Konigd910b7d2013-02-26 17:52:16 +0000711
Matt Arsenault84445dd2017-11-30 22:51:26 +0000712// FIXME: Should only handle addcarry/subcarry
Justin Bogner95927c02016-05-12 21:03:32 +0000713void AMDGPUDAGToDAGISel::SelectADD_SUB_I64(SDNode *N) {
Matt Arsenault9fa3f932014-06-23 18:00:34 +0000714 SDLoc DL(N);
715 SDValue LHS = N->getOperand(0);
716 SDValue RHS = N->getOperand(1);
717
Nicolai Haehnle67624af2016-10-14 10:30:00 +0000718 unsigned Opcode = N->getOpcode();
719 bool ConsumeCarry = (Opcode == ISD::ADDE || Opcode == ISD::SUBE);
720 bool ProduceCarry =
721 ConsumeCarry || Opcode == ISD::ADDC || Opcode == ISD::SUBC;
Matt Arsenault84445dd2017-11-30 22:51:26 +0000722 bool IsAdd = Opcode == ISD::ADD || Opcode == ISD::ADDC || Opcode == ISD::ADDE;
Matt Arsenaultb8b51532014-06-23 18:00:38 +0000723
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000724 SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
725 SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
Matt Arsenault9fa3f932014-06-23 18:00:34 +0000726
727 SDNode *Lo0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
728 DL, MVT::i32, LHS, Sub0);
729 SDNode *Hi0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
730 DL, MVT::i32, LHS, Sub1);
731
732 SDNode *Lo1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
733 DL, MVT::i32, RHS, Sub0);
734 SDNode *Hi1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
735 DL, MVT::i32, RHS, Sub1);
736
737 SDVTList VTList = CurDAG->getVTList(MVT::i32, MVT::Glue);
Matt Arsenault9fa3f932014-06-23 18:00:34 +0000738
Tom Stellard80942a12014-09-05 14:07:59 +0000739 unsigned Opc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
Matt Arsenaultb8b51532014-06-23 18:00:38 +0000740 unsigned CarryOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
741
Nicolai Haehnle67624af2016-10-14 10:30:00 +0000742 SDNode *AddLo;
743 if (!ConsumeCarry) {
744 SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0) };
745 AddLo = CurDAG->getMachineNode(Opc, DL, VTList, Args);
746 } else {
747 SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0), N->getOperand(2) };
748 AddLo = CurDAG->getMachineNode(CarryOpc, DL, VTList, Args);
749 }
750 SDValue AddHiArgs[] = {
751 SDValue(Hi0, 0),
752 SDValue(Hi1, 0),
753 SDValue(AddLo, 1)
754 };
755 SDNode *AddHi = CurDAG->getMachineNode(CarryOpc, DL, VTList, AddHiArgs);
Matt Arsenault9fa3f932014-06-23 18:00:34 +0000756
Nicolai Haehnle67624af2016-10-14 10:30:00 +0000757 SDValue RegSequenceArgs[] = {
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000758 CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
Matt Arsenault9fa3f932014-06-23 18:00:34 +0000759 SDValue(AddLo,0),
760 Sub0,
761 SDValue(AddHi,0),
762 Sub1,
763 };
Nicolai Haehnle67624af2016-10-14 10:30:00 +0000764 SDNode *RegSequence = CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL,
765 MVT::i64, RegSequenceArgs);
766
767 if (ProduceCarry) {
768 // Replace the carry-use
Nirav Dave5f0ab712018-03-17 19:24:54 +0000769 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 1), SDValue(AddHi, 1));
Nicolai Haehnle67624af2016-10-14 10:30:00 +0000770 }
771
772 // Replace the remaining uses.
Nirav Dave5f0ab712018-03-17 19:24:54 +0000773 CurDAG->ReplaceAllUsesWith(N, RegSequence);
774 CurDAG->RemoveDeadNode(N);
Matt Arsenault9fa3f932014-06-23 18:00:34 +0000775}
776
Matt Arsenaultee3f0ac2017-01-30 18:11:38 +0000777void AMDGPUDAGToDAGISel::SelectUADDO_USUBO(SDNode *N) {
778 // The name of the opcodes are misleading. v_add_i32/v_sub_i32 have unsigned
779 // carry out despite the _i32 name. These were renamed in VI to _U32.
780 // FIXME: We should probably rename the opcodes here.
781 unsigned Opc = N->getOpcode() == ISD::UADDO ?
782 AMDGPU::V_ADD_I32_e64 : AMDGPU::V_SUB_I32_e64;
783
784 CurDAG->SelectNodeTo(N, Opc, N->getVTList(),
785 { N->getOperand(0), N->getOperand(1) });
786}
787
Tom Stellard8485fa02016-12-07 02:42:15 +0000788void AMDGPUDAGToDAGISel::SelectFMA_W_CHAIN(SDNode *N) {
789 SDLoc SL(N);
790 // src0_modifiers, src0, src1_modifiers, src1, src2_modifiers, src2, clamp, omod
791 SDValue Ops[10];
792
793 SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[6], Ops[7]);
794 SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]);
795 SelectVOP3Mods(N->getOperand(3), Ops[5], Ops[4]);
796 Ops[8] = N->getOperand(0);
797 Ops[9] = N->getOperand(4);
798
799 CurDAG->SelectNodeTo(N, AMDGPU::V_FMA_F32, N->getVTList(), Ops);
800}
801
802void AMDGPUDAGToDAGISel::SelectFMUL_W_CHAIN(SDNode *N) {
803 SDLoc SL(N);
NAKAMURA Takumi6f43bd42017-10-18 13:31:28 +0000804 // src0_modifiers, src0, src1_modifiers, src1, clamp, omod
Tom Stellard8485fa02016-12-07 02:42:15 +0000805 SDValue Ops[8];
806
807 SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[4], Ops[5]);
808 SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]);
809 Ops[6] = N->getOperand(0);
810 Ops[7] = N->getOperand(3);
811
812 CurDAG->SelectNodeTo(N, AMDGPU::V_MUL_F32_e64, N->getVTList(), Ops);
813}
814
Matt Arsenault044f1d12015-02-14 04:24:28 +0000815// We need to handle this here because tablegen doesn't support matching
816// instructions with multiple outputs.
Justin Bogner95927c02016-05-12 21:03:32 +0000817void AMDGPUDAGToDAGISel::SelectDIV_SCALE(SDNode *N) {
Matt Arsenaultf2b0aeb2014-06-23 18:28:28 +0000818 SDLoc SL(N);
819 EVT VT = N->getValueType(0);
820
821 assert(VT == MVT::f32 || VT == MVT::f64);
822
823 unsigned Opc
824 = (VT == MVT::f64) ? AMDGPU::V_DIV_SCALE_F64 : AMDGPU::V_DIV_SCALE_F32;
825
Matt Arsenault3b99f122017-01-19 06:04:12 +0000826 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2) };
827 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
Matt Arsenaultf2b0aeb2014-06-23 18:28:28 +0000828}
829
Matt Arsenault4f6318f2017-11-06 17:04:37 +0000830// We need to handle this here because tablegen doesn't support matching
831// instructions with multiple outputs.
832void AMDGPUDAGToDAGISel::SelectMAD_64_32(SDNode *N) {
833 SDLoc SL(N);
834 bool Signed = N->getOpcode() == AMDGPUISD::MAD_I64_I32;
835 unsigned Opc = Signed ? AMDGPU::V_MAD_I64_I32 : AMDGPU::V_MAD_U64_U32;
836
837 SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1);
838 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
839 Clamp };
840 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
841}
842
Tom Stellard85e8b6d2014-08-22 18:49:33 +0000843bool AMDGPUDAGToDAGISel::isDSOffsetLegal(const SDValue &Base, unsigned Offset,
844 unsigned OffsetBits) const {
Tom Stellard85e8b6d2014-08-22 18:49:33 +0000845 if ((OffsetBits == 16 && !isUInt<16>(Offset)) ||
846 (OffsetBits == 8 && !isUInt<8>(Offset)))
847 return false;
848
Matt Arsenault706f9302015-07-06 16:01:58 +0000849 if (Subtarget->getGeneration() >= AMDGPUSubtarget::SEA_ISLANDS ||
850 Subtarget->unsafeDSOffsetFoldingEnabled())
Tom Stellard85e8b6d2014-08-22 18:49:33 +0000851 return true;
852
853 // On Southern Islands instruction with a negative base value and an offset
854 // don't seem to work.
855 return CurDAG->SignBitIsZero(Base);
856}
857
858bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(SDValue Addr, SDValue &Base,
859 SDValue &Offset) const {
Tom Stellard92b24f32016-04-29 14:34:26 +0000860 SDLoc DL(Addr);
Tom Stellard85e8b6d2014-08-22 18:49:33 +0000861 if (CurDAG->isBaseWithConstantOffset(Addr)) {
862 SDValue N0 = Addr.getOperand(0);
863 SDValue N1 = Addr.getOperand(1);
864 ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
865 if (isDSOffsetLegal(N0, C1->getSExtValue(), 16)) {
866 // (add n0, c0)
867 Base = N0;
Tom Stellard92b24f32016-04-29 14:34:26 +0000868 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
Tom Stellard85e8b6d2014-08-22 18:49:33 +0000869 return true;
870 }
Matt Arsenault966a94f2015-09-08 19:34:22 +0000871 } else if (Addr.getOpcode() == ISD::SUB) {
872 // sub C, x -> add (sub 0, x), C
873 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) {
874 int64_t ByteOffset = C->getSExtValue();
875 if (isUInt<16>(ByteOffset)) {
Matt Arsenault966a94f2015-09-08 19:34:22 +0000876 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
Tom Stellard85e8b6d2014-08-22 18:49:33 +0000877
Matt Arsenault966a94f2015-09-08 19:34:22 +0000878 // XXX - This is kind of hacky. Create a dummy sub node so we can check
879 // the known bits in isDSOffsetLegal. We need to emit the selected node
880 // here, so this is thrown away.
881 SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32,
882 Zero, Addr.getOperand(1));
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000883
Matt Arsenault966a94f2015-09-08 19:34:22 +0000884 if (isDSOffsetLegal(Sub, ByteOffset, 16)) {
Matt Arsenault84445dd2017-11-30 22:51:26 +0000885 // FIXME: Select to VOP3 version for with-carry.
886 unsigned SubOp = Subtarget->hasAddNoCarry() ?
887 AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_I32_e32;
888
Matt Arsenault966a94f2015-09-08 19:34:22 +0000889 MachineSDNode *MachineSub
Matt Arsenault84445dd2017-11-30 22:51:26 +0000890 = CurDAG->getMachineNode(SubOp, DL, MVT::i32,
Matt Arsenault966a94f2015-09-08 19:34:22 +0000891 Zero, Addr.getOperand(1));
892
893 Base = SDValue(MachineSub, 0);
Tom Stellard26a2ab72016-06-10 00:01:04 +0000894 Offset = CurDAG->getTargetConstant(ByteOffset, DL, MVT::i16);
Matt Arsenault966a94f2015-09-08 19:34:22 +0000895 return true;
896 }
897 }
898 }
899 } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
900 // If we have a constant address, prefer to put the constant into the
901 // offset. This can save moves to load the constant address since multiple
902 // operations can share the zero base address register, and enables merging
903 // into read2 / write2 instructions.
904
905 SDLoc DL(Addr);
906
Matt Arsenaulte775f5f2014-10-14 17:21:19 +0000907 if (isUInt<16>(CAddr->getZExtValue())) {
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000908 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
Tom Stellardc8d79202014-10-15 21:08:59 +0000909 MachineSDNode *MovZero = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000910 DL, MVT::i32, Zero);
Tom Stellardc8d79202014-10-15 21:08:59 +0000911 Base = SDValue(MovZero, 0);
Tom Stellard26a2ab72016-06-10 00:01:04 +0000912 Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16);
Matt Arsenaulte775f5f2014-10-14 17:21:19 +0000913 return true;
914 }
915 }
916
Tom Stellard85e8b6d2014-08-22 18:49:33 +0000917 // default case
918 Base = Addr;
Matt Arsenault966a94f2015-09-08 19:34:22 +0000919 Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i16);
Tom Stellard85e8b6d2014-08-22 18:49:33 +0000920 return true;
921}
922
Matt Arsenault966a94f2015-09-08 19:34:22 +0000923// TODO: If offset is too big, put low 16-bit into offset.
Tom Stellardf3fc5552014-08-22 18:49:35 +0000924bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base,
925 SDValue &Offset0,
926 SDValue &Offset1) const {
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000927 SDLoc DL(Addr);
928
Tom Stellardf3fc5552014-08-22 18:49:35 +0000929 if (CurDAG->isBaseWithConstantOffset(Addr)) {
930 SDValue N0 = Addr.getOperand(0);
931 SDValue N1 = Addr.getOperand(1);
932 ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
933 unsigned DWordOffset0 = C1->getZExtValue() / 4;
934 unsigned DWordOffset1 = DWordOffset0 + 1;
935 // (add n0, c0)
936 if (isDSOffsetLegal(N0, DWordOffset1, 8)) {
937 Base = N0;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000938 Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8);
939 Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8);
Tom Stellardf3fc5552014-08-22 18:49:35 +0000940 return true;
941 }
Matt Arsenault966a94f2015-09-08 19:34:22 +0000942 } else if (Addr.getOpcode() == ISD::SUB) {
943 // sub C, x -> add (sub 0, x), C
944 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) {
945 unsigned DWordOffset0 = C->getZExtValue() / 4;
946 unsigned DWordOffset1 = DWordOffset0 + 1;
Tom Stellardf3fc5552014-08-22 18:49:35 +0000947
Matt Arsenault966a94f2015-09-08 19:34:22 +0000948 if (isUInt<8>(DWordOffset0)) {
949 SDLoc DL(Addr);
950 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
951
952 // XXX - This is kind of hacky. Create a dummy sub node so we can check
953 // the known bits in isDSOffsetLegal. We need to emit the selected node
954 // here, so this is thrown away.
955 SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32,
956 Zero, Addr.getOperand(1));
957
958 if (isDSOffsetLegal(Sub, DWordOffset1, 8)) {
Matt Arsenault84445dd2017-11-30 22:51:26 +0000959 unsigned SubOp = Subtarget->hasAddNoCarry() ?
960 AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_I32_e32;
961
Matt Arsenault966a94f2015-09-08 19:34:22 +0000962 MachineSDNode *MachineSub
Matt Arsenault84445dd2017-11-30 22:51:26 +0000963 = CurDAG->getMachineNode(SubOp, DL, MVT::i32,
Matt Arsenault966a94f2015-09-08 19:34:22 +0000964 Zero, Addr.getOperand(1));
965
966 Base = SDValue(MachineSub, 0);
967 Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8);
968 Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8);
969 return true;
970 }
971 }
972 }
973 } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
Matt Arsenault1a74aff2014-10-15 18:06:43 +0000974 unsigned DWordOffset0 = CAddr->getZExtValue() / 4;
975 unsigned DWordOffset1 = DWordOffset0 + 1;
976 assert(4 * DWordOffset0 == CAddr->getZExtValue());
977
978 if (isUInt<8>(DWordOffset0) && isUInt<8>(DWordOffset1)) {
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000979 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
Matt Arsenault1a74aff2014-10-15 18:06:43 +0000980 MachineSDNode *MovZero
981 = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000982 DL, MVT::i32, Zero);
Matt Arsenault1a74aff2014-10-15 18:06:43 +0000983 Base = SDValue(MovZero, 0);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000984 Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8);
985 Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8);
Matt Arsenault1a74aff2014-10-15 18:06:43 +0000986 return true;
987 }
988 }
989
Tom Stellardf3fc5552014-08-22 18:49:35 +0000990 // default case
Matt Arsenault0efdd062016-09-09 22:29:28 +0000991
992 // FIXME: This is broken on SI where we still need to check if the base
993 // pointer is positive here.
Tom Stellardf3fc5552014-08-22 18:49:35 +0000994 Base = Addr;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000995 Offset0 = CurDAG->getTargetConstant(0, DL, MVT::i8);
996 Offset1 = CurDAG->getTargetConstant(1, DL, MVT::i8);
Tom Stellardf3fc5552014-08-22 18:49:35 +0000997 return true;
998}
999
Changpeng Fangb41574a2015-12-22 20:55:23 +00001000bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr,
Tom Stellard155bbb72014-08-11 22:18:17 +00001001 SDValue &VAddr, SDValue &SOffset,
1002 SDValue &Offset, SDValue &Offen,
1003 SDValue &Idxen, SDValue &Addr64,
1004 SDValue &GLC, SDValue &SLC,
1005 SDValue &TFE) const {
Changpeng Fangb41574a2015-12-22 20:55:23 +00001006 // Subtarget prefers to use flat instruction
1007 if (Subtarget->useFlatForGlobal())
1008 return false;
1009
Tom Stellardb02c2682014-06-24 23:33:07 +00001010 SDLoc DL(Addr);
1011
Jan Vesely43b7b5b2016-04-07 19:23:11 +00001012 if (!GLC.getNode())
1013 GLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
1014 if (!SLC.getNode())
1015 SLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001016 TFE = CurDAG->getTargetConstant(0, DL, MVT::i1);
Tom Stellard155bbb72014-08-11 22:18:17 +00001017
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001018 Idxen = CurDAG->getTargetConstant(0, DL, MVT::i1);
1019 Offen = CurDAG->getTargetConstant(0, DL, MVT::i1);
1020 Addr64 = CurDAG->getTargetConstant(0, DL, MVT::i1);
1021 SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
Tom Stellard155bbb72014-08-11 22:18:17 +00001022
Tom Stellardb02c2682014-06-24 23:33:07 +00001023 if (CurDAG->isBaseWithConstantOffset(Addr)) {
1024 SDValue N0 = Addr.getOperand(0);
1025 SDValue N1 = Addr.getOperand(1);
1026 ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1027
Tom Stellard94b72312015-02-11 00:34:35 +00001028 if (N0.getOpcode() == ISD::ADD) {
1029 // (add (add N2, N3), C1) -> addr64
1030 SDValue N2 = N0.getOperand(0);
1031 SDValue N3 = N0.getOperand(1);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001032 Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
Tom Stellard94b72312015-02-11 00:34:35 +00001033 Ptr = N2;
1034 VAddr = N3;
1035 } else {
Tom Stellard155bbb72014-08-11 22:18:17 +00001036 // (add N0, C1) -> offset
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001037 VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32);
Tom Stellard155bbb72014-08-11 22:18:17 +00001038 Ptr = N0;
Tom Stellard94b72312015-02-11 00:34:35 +00001039 }
1040
Marek Olsakffadcb72017-11-09 01:52:17 +00001041 if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue())) {
Matt Arsenault88701812016-06-09 23:42:48 +00001042 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
1043 return true;
1044 }
1045
1046 if (isUInt<32>(C1->getZExtValue())) {
Tom Stellard94b72312015-02-11 00:34:35 +00001047 // Illegal offset, store it in soffset.
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001048 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
Tom Stellard94b72312015-02-11 00:34:35 +00001049 SOffset = SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001050 CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32)),
1051 0);
Changpeng Fangb41574a2015-12-22 20:55:23 +00001052 return true;
Tom Stellardb02c2682014-06-24 23:33:07 +00001053 }
1054 }
Tom Stellard94b72312015-02-11 00:34:35 +00001055
Tom Stellardb02c2682014-06-24 23:33:07 +00001056 if (Addr.getOpcode() == ISD::ADD) {
Tom Stellard155bbb72014-08-11 22:18:17 +00001057 // (add N0, N1) -> addr64
Tom Stellardb02c2682014-06-24 23:33:07 +00001058 SDValue N0 = Addr.getOperand(0);
1059 SDValue N1 = Addr.getOperand(1);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001060 Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
Tom Stellard155bbb72014-08-11 22:18:17 +00001061 Ptr = N0;
1062 VAddr = N1;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001063 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
Changpeng Fangb41574a2015-12-22 20:55:23 +00001064 return true;
Tom Stellardb02c2682014-06-24 23:33:07 +00001065 }
1066
Tom Stellard155bbb72014-08-11 22:18:17 +00001067 // default case -> offset
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001068 VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32);
Tom Stellard155bbb72014-08-11 22:18:17 +00001069 Ptr = Addr;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001070 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
Changpeng Fangb41574a2015-12-22 20:55:23 +00001071
1072 return true;
Tom Stellard155bbb72014-08-11 22:18:17 +00001073}
1074
1075bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
Tom Stellardc53861a2015-02-11 00:34:32 +00001076 SDValue &VAddr, SDValue &SOffset,
Tom Stellard1f9939f2015-02-27 14:59:41 +00001077 SDValue &Offset, SDValue &GLC,
1078 SDValue &SLC, SDValue &TFE) const {
1079 SDValue Ptr, Offen, Idxen, Addr64;
Tom Stellard155bbb72014-08-11 22:18:17 +00001080
Tom Stellard70580f82015-07-20 14:28:41 +00001081 // addr64 bit was removed for volcanic islands.
1082 if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
1083 return false;
1084
Changpeng Fangb41574a2015-12-22 20:55:23 +00001085 if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
1086 GLC, SLC, TFE))
1087 return false;
Tom Stellard155bbb72014-08-11 22:18:17 +00001088
1089 ConstantSDNode *C = cast<ConstantSDNode>(Addr64);
1090 if (C->getSExtValue()) {
1091 SDLoc DL(Addr);
Matt Arsenault485defe2014-11-05 19:01:17 +00001092
1093 const SITargetLowering& Lowering =
1094 *static_cast<const SITargetLowering*>(getTargetLowering());
1095
1096 SRsrc = SDValue(Lowering.wrapAddr64Rsrc(*CurDAG, DL, Ptr), 0);
Tom Stellard155bbb72014-08-11 22:18:17 +00001097 return true;
1098 }
Matt Arsenault485defe2014-11-05 19:01:17 +00001099
Tom Stellard155bbb72014-08-11 22:18:17 +00001100 return false;
1101}
1102
Tom Stellard7980fc82014-09-25 18:30:26 +00001103bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
Tom Stellardc53861a2015-02-11 00:34:32 +00001104 SDValue &VAddr, SDValue &SOffset,
NAKAMURA Takumi0a7d0ad2015-09-22 11:15:07 +00001105 SDValue &Offset,
1106 SDValue &SLC) const {
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001107 SLC = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i1);
Tom Stellard1f9939f2015-02-27 14:59:41 +00001108 SDValue GLC, TFE;
Tom Stellard7980fc82014-09-25 18:30:26 +00001109
Tom Stellard1f9939f2015-02-27 14:59:41 +00001110 return SelectMUBUFAddr64(Addr, SRsrc, VAddr, SOffset, Offset, GLC, SLC, TFE);
Tom Stellard7980fc82014-09-25 18:30:26 +00001111}
1112
Matt Arsenault156d3ae2017-05-17 21:02:58 +00001113static bool isStackPtrRelative(const MachinePointerInfo &PtrInfo) {
1114 auto PSV = PtrInfo.V.dyn_cast<const PseudoSourceValue *>();
1115 return PSV && PSV->isStack();
Matt Arsenaultac0fc842016-09-17 16:09:55 +00001116}
1117
Matt Arsenault156d3ae2017-05-17 21:02:58 +00001118std::pair<SDValue, SDValue> AMDGPUDAGToDAGISel::foldFrameIndex(SDValue N) const {
1119 const MachineFunction &MF = CurDAG->getMachineFunction();
1120 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1121
1122 if (auto FI = dyn_cast<FrameIndexSDNode>(N)) {
1123 SDValue TFI = CurDAG->getTargetFrameIndex(FI->getIndex(),
1124 FI->getValueType(0));
1125
1126 // If we can resolve this to a frame index access, this is relative to the
1127 // frame pointer SGPR.
1128 return std::make_pair(TFI, CurDAG->getRegister(Info->getFrameOffsetReg(),
1129 MVT::i32));
1130 }
1131
1132 // If we don't know this private access is a local stack object, it needs to
1133 // be relative to the entry point's scratch wave offset register.
1134 return std::make_pair(N, CurDAG->getRegister(Info->getScratchWaveOffsetReg(),
1135 MVT::i32));
1136}
1137
Matt Arsenaultb81495d2017-09-20 05:01:53 +00001138bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffen(SDNode *Parent,
Matt Arsenault156d3ae2017-05-17 21:02:58 +00001139 SDValue Addr, SDValue &Rsrc,
Matt Arsenault0774ea22017-04-24 19:40:59 +00001140 SDValue &VAddr, SDValue &SOffset,
1141 SDValue &ImmOffset) const {
Tom Stellardb02094e2014-07-21 15:45:01 +00001142
1143 SDLoc DL(Addr);
1144 MachineFunction &MF = CurDAG->getMachineFunction();
Matt Arsenault0e3d3892015-11-30 21:15:53 +00001145 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
Tom Stellardb02094e2014-07-21 15:45:01 +00001146
Matt Arsenault0e3d3892015-11-30 21:15:53 +00001147 Rsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
Tom Stellardb02094e2014-07-21 15:45:01 +00001148
Matt Arsenault0774ea22017-04-24 19:40:59 +00001149 if (ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1150 unsigned Imm = CAddr->getZExtValue();
Matt Arsenault0774ea22017-04-24 19:40:59 +00001151
1152 SDValue HighBits = CurDAG->getTargetConstant(Imm & ~4095, DL, MVT::i32);
1153 MachineSDNode *MovHighBits = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
1154 DL, MVT::i32, HighBits);
1155 VAddr = SDValue(MovHighBits, 0);
Matt Arsenault156d3ae2017-05-17 21:02:58 +00001156
1157 // In a call sequence, stores to the argument stack area are relative to the
1158 // stack pointer.
Matt Arsenaultb81495d2017-09-20 05:01:53 +00001159 const MachinePointerInfo &PtrInfo = cast<MemSDNode>(Parent)->getPointerInfo();
Matt Arsenault156d3ae2017-05-17 21:02:58 +00001160 unsigned SOffsetReg = isStackPtrRelative(PtrInfo) ?
1161 Info->getStackPtrOffsetReg() : Info->getScratchWaveOffsetReg();
1162
1163 SOffset = CurDAG->getRegister(SOffsetReg, MVT::i32);
Matt Arsenault0774ea22017-04-24 19:40:59 +00001164 ImmOffset = CurDAG->getTargetConstant(Imm & 4095, DL, MVT::i16);
1165 return true;
1166 }
1167
Tom Stellardb02094e2014-07-21 15:45:01 +00001168 if (CurDAG->isBaseWithConstantOffset(Addr)) {
Matt Arsenault0774ea22017-04-24 19:40:59 +00001169 // (add n0, c1)
1170
Tom Stellard78655fc2015-07-16 19:40:09 +00001171 SDValue N0 = Addr.getOperand(0);
Tom Stellardb02094e2014-07-21 15:45:01 +00001172 SDValue N1 = Addr.getOperand(1);
Matt Arsenaultcd099612016-02-24 04:55:29 +00001173
Matt Arsenaultcaf0ed42017-11-30 00:52:40 +00001174 // Offsets in vaddr must be positive if range checking is enabled.
Matt Arsenault45b98182017-11-15 00:45:43 +00001175 //
Matt Arsenaultcaf0ed42017-11-30 00:52:40 +00001176 // The total computation of vaddr + soffset + offset must not overflow. If
1177 // vaddr is negative, even if offset is 0 the sgpr offset add will end up
Matt Arsenault45b98182017-11-15 00:45:43 +00001178 // overflowing.
Matt Arsenaultcaf0ed42017-11-30 00:52:40 +00001179 //
1180 // Prior to gfx9, MUBUF instructions with the vaddr offset enabled would
1181 // always perform a range check. If a negative vaddr base index was used,
1182 // this would fail the range check. The overall address computation would
1183 // compute a valid address, but this doesn't happen due to the range
1184 // check. For out-of-bounds MUBUF loads, a 0 is returned.
1185 //
1186 // Therefore it should be safe to fold any VGPR offset on gfx9 into the
1187 // MUBUF vaddr, but not on older subtargets which can only do this if the
1188 // sign bit is known 0.
Matt Arsenaultcd099612016-02-24 04:55:29 +00001189 ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
Matt Arsenault45b98182017-11-15 00:45:43 +00001190 if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue()) &&
Matt Arsenaultcaf0ed42017-11-30 00:52:40 +00001191 (!Subtarget->privateMemoryResourceIsRangeChecked() ||
1192 CurDAG->SignBitIsZero(N0))) {
Matt Arsenault156d3ae2017-05-17 21:02:58 +00001193 std::tie(VAddr, SOffset) = foldFrameIndex(N0);
Matt Arsenaultcd099612016-02-24 04:55:29 +00001194 ImmOffset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
1195 return true;
Tom Stellardb02094e2014-07-21 15:45:01 +00001196 }
1197 }
1198
Tom Stellardb02094e2014-07-21 15:45:01 +00001199 // (node)
Matt Arsenault156d3ae2017-05-17 21:02:58 +00001200 std::tie(VAddr, SOffset) = foldFrameIndex(Addr);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001201 ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i16);
Tom Stellardb02094e2014-07-21 15:45:01 +00001202 return true;
1203}
1204
Matt Arsenaultb81495d2017-09-20 05:01:53 +00001205bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffset(SDNode *Parent,
Matt Arsenault156d3ae2017-05-17 21:02:58 +00001206 SDValue Addr,
Matt Arsenault0774ea22017-04-24 19:40:59 +00001207 SDValue &SRsrc,
1208 SDValue &SOffset,
1209 SDValue &Offset) const {
1210 ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr);
Marek Olsakffadcb72017-11-09 01:52:17 +00001211 if (!CAddr || !SIInstrInfo::isLegalMUBUFImmOffset(CAddr->getZExtValue()))
Matt Arsenault0774ea22017-04-24 19:40:59 +00001212 return false;
1213
1214 SDLoc DL(Addr);
1215 MachineFunction &MF = CurDAG->getMachineFunction();
1216 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1217
1218 SRsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
Matt Arsenault156d3ae2017-05-17 21:02:58 +00001219
Matt Arsenaultb81495d2017-09-20 05:01:53 +00001220 const MachinePointerInfo &PtrInfo = cast<MemSDNode>(Parent)->getPointerInfo();
Matt Arsenault156d3ae2017-05-17 21:02:58 +00001221 unsigned SOffsetReg = isStackPtrRelative(PtrInfo) ?
1222 Info->getStackPtrOffsetReg() : Info->getScratchWaveOffsetReg();
1223
1224 // FIXME: Get from MachinePointerInfo? We should only be using the frame
1225 // offset if we know this is in a call sequence.
1226 SOffset = CurDAG->getRegister(SOffsetReg, MVT::i32);
1227
Matt Arsenault0774ea22017-04-24 19:40:59 +00001228 Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16);
1229 return true;
1230}
1231
Tom Stellard155bbb72014-08-11 22:18:17 +00001232bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1233 SDValue &SOffset, SDValue &Offset,
1234 SDValue &GLC, SDValue &SLC,
1235 SDValue &TFE) const {
1236 SDValue Ptr, VAddr, Offen, Idxen, Addr64;
Tom Stellard794c8c02014-12-02 17:05:41 +00001237 const SIInstrInfo *TII =
Eric Christopher7792e322015-01-30 23:24:40 +00001238 static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo());
Tom Stellardb02094e2014-07-21 15:45:01 +00001239
Changpeng Fangb41574a2015-12-22 20:55:23 +00001240 if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
1241 GLC, SLC, TFE))
1242 return false;
Tom Stellardb02094e2014-07-21 15:45:01 +00001243
Tom Stellard155bbb72014-08-11 22:18:17 +00001244 if (!cast<ConstantSDNode>(Offen)->getSExtValue() &&
1245 !cast<ConstantSDNode>(Idxen)->getSExtValue() &&
1246 !cast<ConstantSDNode>(Addr64)->getSExtValue()) {
Tom Stellard794c8c02014-12-02 17:05:41 +00001247 uint64_t Rsrc = TII->getDefaultRsrcDataFormat() |
Tom Stellard155bbb72014-08-11 22:18:17 +00001248 APInt::getAllOnesValue(32).getZExtValue(); // Size
1249 SDLoc DL(Addr);
Matt Arsenaultf3cd4512014-11-05 19:01:19 +00001250
1251 const SITargetLowering& Lowering =
1252 *static_cast<const SITargetLowering*>(getTargetLowering());
1253
1254 SRsrc = SDValue(Lowering.buildRSRC(*CurDAG, DL, Ptr, 0, Rsrc), 0);
Tom Stellard155bbb72014-08-11 22:18:17 +00001255 return true;
1256 }
1257 return false;
Tom Stellardb02094e2014-07-21 15:45:01 +00001258}
1259
Tom Stellard7980fc82014-09-25 18:30:26 +00001260bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
Jan Vesely43b7b5b2016-04-07 19:23:11 +00001261 SDValue &Soffset, SDValue &Offset
1262 ) const {
1263 SDValue GLC, SLC, TFE;
1264
1265 return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE);
1266}
1267bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
Tom Stellard7980fc82014-09-25 18:30:26 +00001268 SDValue &Soffset, SDValue &Offset,
Matt Arsenault88701812016-06-09 23:42:48 +00001269 SDValue &SLC) const {
1270 SDValue GLC, TFE;
Tom Stellard7980fc82014-09-25 18:30:26 +00001271
1272 return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE);
1273}
1274
Nicolai Haehnlea6092592016-06-15 07:13:05 +00001275bool AMDGPUDAGToDAGISel::SelectMUBUFConstant(SDValue Constant,
Nicolai Haehnle3003ba02016-03-18 16:24:20 +00001276 SDValue &SOffset,
1277 SDValue &ImmOffset) const {
1278 SDLoc DL(Constant);
Nicolai Haehnle312b64f2017-10-10 12:22:23 +00001279 const uint32_t Align = 4;
1280 const uint32_t MaxImm = alignDown(4095, Align);
Nicolai Haehnle3003ba02016-03-18 16:24:20 +00001281 uint32_t Imm = cast<ConstantSDNode>(Constant)->getZExtValue();
1282 uint32_t Overflow = 0;
1283
Nicolai Haehnle312b64f2017-10-10 12:22:23 +00001284 if (Imm > MaxImm) {
1285 if (Imm <= MaxImm + 64) {
1286 // Use an SOffset inline constant for 4..64
1287 Overflow = Imm - MaxImm;
1288 Imm = MaxImm;
Nicolai Haehnle3003ba02016-03-18 16:24:20 +00001289 } else {
1290 // Try to keep the same value in SOffset for adjacent loads, so that
1291 // the corresponding register contents can be re-used.
1292 //
Nicolai Haehnle312b64f2017-10-10 12:22:23 +00001293 // Load values with all low-bits (except for alignment bits) set into
1294 // SOffset, so that a larger range of values can be covered using
1295 // s_movk_i32.
1296 //
1297 // Atomic operations fail to work correctly when individual address
1298 // components are unaligned, even if their sum is aligned.
1299 uint32_t High = (Imm + Align) & ~4095;
1300 uint32_t Low = (Imm + Align) & 4095;
Nicolai Haehnle3003ba02016-03-18 16:24:20 +00001301 Imm = Low;
Nicolai Haehnle312b64f2017-10-10 12:22:23 +00001302 Overflow = High - Align;
Nicolai Haehnle3003ba02016-03-18 16:24:20 +00001303 }
1304 }
1305
Nicolai Haehnlea6092592016-06-15 07:13:05 +00001306 // There is a hardware bug in SI and CI which prevents address clamping in
1307 // MUBUF instructions from working correctly with SOffsets. The immediate
1308 // offset is unaffected.
1309 if (Overflow > 0 &&
1310 Subtarget->getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS)
1311 return false;
1312
Nicolai Haehnle3003ba02016-03-18 16:24:20 +00001313 ImmOffset = CurDAG->getTargetConstant(Imm, DL, MVT::i16);
1314
1315 if (Overflow <= 64)
1316 SOffset = CurDAG->getTargetConstant(Overflow, DL, MVT::i32);
1317 else
1318 SOffset = SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
1319 CurDAG->getTargetConstant(Overflow, DL, MVT::i32)),
1320 0);
Nicolai Haehnlea6092592016-06-15 07:13:05 +00001321
1322 return true;
Nicolai Haehnle3003ba02016-03-18 16:24:20 +00001323}
1324
1325bool AMDGPUDAGToDAGISel::SelectMUBUFIntrinsicOffset(SDValue Offset,
1326 SDValue &SOffset,
1327 SDValue &ImmOffset) const {
1328 SDLoc DL(Offset);
1329
1330 if (!isa<ConstantSDNode>(Offset))
1331 return false;
1332
Nicolai Haehnlea6092592016-06-15 07:13:05 +00001333 return SelectMUBUFConstant(Offset, SOffset, ImmOffset);
Nicolai Haehnle3003ba02016-03-18 16:24:20 +00001334}
1335
1336bool AMDGPUDAGToDAGISel::SelectMUBUFIntrinsicVOffset(SDValue Offset,
1337 SDValue &SOffset,
1338 SDValue &ImmOffset,
1339 SDValue &VOffset) const {
1340 SDLoc DL(Offset);
1341
1342 // Don't generate an unnecessary voffset for constant offsets.
Nicolai Haehnlea6092592016-06-15 07:13:05 +00001343 if (isa<ConstantSDNode>(Offset)) {
1344 SDValue Tmp1, Tmp2;
1345
1346 // When necessary, use a voffset in <= CI anyway to work around a hardware
1347 // bug.
1348 if (Subtarget->getGeneration() > AMDGPUSubtarget::SEA_ISLANDS ||
1349 SelectMUBUFConstant(Offset, Tmp1, Tmp2))
1350 return false;
1351 }
Nicolai Haehnle3003ba02016-03-18 16:24:20 +00001352
1353 if (CurDAG->isBaseWithConstantOffset(Offset)) {
1354 SDValue N0 = Offset.getOperand(0);
1355 SDValue N1 = Offset.getOperand(1);
Nicolai Haehnlea6092592016-06-15 07:13:05 +00001356 if (cast<ConstantSDNode>(N1)->getSExtValue() >= 0 &&
1357 SelectMUBUFConstant(N1, SOffset, ImmOffset)) {
1358 VOffset = N0;
1359 return true;
1360 }
Nicolai Haehnle3003ba02016-03-18 16:24:20 +00001361 }
1362
Nicolai Haehnlea6092592016-06-15 07:13:05 +00001363 SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1364 ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1365 VOffset = Offset;
1366
Nicolai Haehnle3003ba02016-03-18 16:24:20 +00001367 return true;
1368}
1369
Matt Arsenault4e309b02017-07-29 01:03:53 +00001370template <bool IsSigned>
Matt Arsenaultdb7c6a82017-06-12 16:53:51 +00001371bool AMDGPUDAGToDAGISel::SelectFlatOffset(SDValue Addr,
1372 SDValue &VAddr,
1373 SDValue &Offset,
1374 SDValue &SLC) const {
1375 int64_t OffsetVal = 0;
1376
1377 if (Subtarget->hasFlatInstOffsets() &&
1378 CurDAG->isBaseWithConstantOffset(Addr)) {
1379 SDValue N0 = Addr.getOperand(0);
1380 SDValue N1 = Addr.getOperand(1);
Matt Arsenault4e309b02017-07-29 01:03:53 +00001381 int64_t COffsetVal = cast<ConstantSDNode>(N1)->getSExtValue();
1382
1383 if ((IsSigned && isInt<13>(COffsetVal)) ||
1384 (!IsSigned && isUInt<12>(COffsetVal))) {
Matt Arsenaultdb7c6a82017-06-12 16:53:51 +00001385 Addr = N0;
1386 OffsetVal = COffsetVal;
1387 }
1388 }
1389
Matt Arsenault7757c592016-06-09 23:42:54 +00001390 VAddr = Addr;
Matt Arsenaultdb7c6a82017-06-12 16:53:51 +00001391 Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i16);
Matt Arsenault47ccafe2017-05-11 17:38:33 +00001392 SLC = CurDAG->getTargetConstant(0, SDLoc(), MVT::i1);
Matt Arsenaultdb7c6a82017-06-12 16:53:51 +00001393
Matt Arsenault7757c592016-06-09 23:42:54 +00001394 return true;
1395}
1396
Matt Arsenaultdb7c6a82017-06-12 16:53:51 +00001397bool AMDGPUDAGToDAGISel::SelectFlatAtomic(SDValue Addr,
1398 SDValue &VAddr,
1399 SDValue &Offset,
1400 SDValue &SLC) const {
Matt Arsenault4e309b02017-07-29 01:03:53 +00001401 return SelectFlatOffset<false>(Addr, VAddr, Offset, SLC);
1402}
1403
1404bool AMDGPUDAGToDAGISel::SelectFlatAtomicSigned(SDValue Addr,
1405 SDValue &VAddr,
1406 SDValue &Offset,
1407 SDValue &SLC) const {
1408 return SelectFlatOffset<true>(Addr, VAddr, Offset, SLC);
Matt Arsenaultdb7c6a82017-06-12 16:53:51 +00001409}
1410
Tom Stellarddee26a22015-08-06 19:28:30 +00001411bool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDValue ByteOffsetNode,
1412 SDValue &Offset, bool &Imm) const {
1413
1414 // FIXME: Handle non-constant offsets.
1415 ConstantSDNode *C = dyn_cast<ConstantSDNode>(ByteOffsetNode);
1416 if (!C)
1417 return false;
1418
1419 SDLoc SL(ByteOffsetNode);
Marek Olsak8973a0a2017-05-24 14:53:50 +00001420 AMDGPUSubtarget::Generation Gen = Subtarget->getGeneration();
Tom Stellarddee26a22015-08-06 19:28:30 +00001421 int64_t ByteOffset = C->getSExtValue();
Tom Stellard08efb7e2017-01-27 18:41:14 +00001422 int64_t EncodedOffset = AMDGPU::getSMRDEncodedOffset(*Subtarget, ByteOffset);
Tom Stellarddee26a22015-08-06 19:28:30 +00001423
Tom Stellard08efb7e2017-01-27 18:41:14 +00001424 if (AMDGPU::isLegalSMRDImmOffset(*Subtarget, ByteOffset)) {
Tom Stellarddee26a22015-08-06 19:28:30 +00001425 Offset = CurDAG->getTargetConstant(EncodedOffset, SL, MVT::i32);
1426 Imm = true;
1427 return true;
1428 }
1429
Tom Stellard217361c2015-08-06 19:28:38 +00001430 if (!isUInt<32>(EncodedOffset) || !isUInt<32>(ByteOffset))
1431 return false;
1432
Marek Olsak8973a0a2017-05-24 14:53:50 +00001433 if (Gen == AMDGPUSubtarget::SEA_ISLANDS && isUInt<32>(EncodedOffset)) {
1434 // 32-bit Immediates are supported on Sea Islands.
Tom Stellard217361c2015-08-06 19:28:38 +00001435 Offset = CurDAG->getTargetConstant(EncodedOffset, SL, MVT::i32);
1436 } else {
Tom Stellarddee26a22015-08-06 19:28:30 +00001437 SDValue C32Bit = CurDAG->getTargetConstant(ByteOffset, SL, MVT::i32);
1438 Offset = SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32,
1439 C32Bit), 0);
Tom Stellarddee26a22015-08-06 19:28:30 +00001440 }
Tom Stellard217361c2015-08-06 19:28:38 +00001441 Imm = false;
1442 return true;
Tom Stellarddee26a22015-08-06 19:28:30 +00001443}
1444
Matt Arsenault923712b2018-02-09 16:57:57 +00001445SDValue AMDGPUDAGToDAGISel::Expand32BitAddress(SDValue Addr) const {
1446 if (Addr.getValueType() != MVT::i32)
1447 return Addr;
1448
1449 // Zero-extend a 32-bit address.
1450 SDLoc SL(Addr);
1451
1452 const MachineFunction &MF = CurDAG->getMachineFunction();
1453 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1454 unsigned AddrHiVal = Info->get32BitAddressHighBits();
1455 SDValue AddrHi = CurDAG->getTargetConstant(AddrHiVal, SL, MVT::i32);
1456
1457 const SDValue Ops[] = {
1458 CurDAG->getTargetConstant(AMDGPU::SReg_64_XEXECRegClassID, SL, MVT::i32),
1459 Addr,
1460 CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32),
1461 SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, AddrHi),
1462 0),
1463 CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32),
1464 };
1465
1466 return SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, SL, MVT::i64,
1467 Ops), 0);
1468}
1469
Tom Stellarddee26a22015-08-06 19:28:30 +00001470bool AMDGPUDAGToDAGISel::SelectSMRD(SDValue Addr, SDValue &SBase,
1471 SDValue &Offset, bool &Imm) const {
Tom Stellarddee26a22015-08-06 19:28:30 +00001472 SDLoc SL(Addr);
Matt Arsenault923712b2018-02-09 16:57:57 +00001473
Tom Stellarddee26a22015-08-06 19:28:30 +00001474 if (CurDAG->isBaseWithConstantOffset(Addr)) {
1475 SDValue N0 = Addr.getOperand(0);
1476 SDValue N1 = Addr.getOperand(1);
1477
1478 if (SelectSMRDOffset(N1, Offset, Imm)) {
Matt Arsenault923712b2018-02-09 16:57:57 +00001479 SBase = Expand32BitAddress(N0);
Tom Stellarddee26a22015-08-06 19:28:30 +00001480 return true;
1481 }
1482 }
Matt Arsenault923712b2018-02-09 16:57:57 +00001483 SBase = Expand32BitAddress(Addr);
Tom Stellarddee26a22015-08-06 19:28:30 +00001484 Offset = CurDAG->getTargetConstant(0, SL, MVT::i32);
1485 Imm = true;
1486 return true;
1487}
1488
1489bool AMDGPUDAGToDAGISel::SelectSMRDImm(SDValue Addr, SDValue &SBase,
1490 SDValue &Offset) const {
1491 bool Imm;
Marek Olsak8973a0a2017-05-24 14:53:50 +00001492 return SelectSMRD(Addr, SBase, Offset, Imm) && Imm;
1493}
Tom Stellarddee26a22015-08-06 19:28:30 +00001494
Marek Olsak8973a0a2017-05-24 14:53:50 +00001495bool AMDGPUDAGToDAGISel::SelectSMRDImm32(SDValue Addr, SDValue &SBase,
1496 SDValue &Offset) const {
1497
1498 if (Subtarget->getGeneration() != AMDGPUSubtarget::SEA_ISLANDS)
1499 return false;
1500
1501 bool Imm;
Tom Stellard217361c2015-08-06 19:28:38 +00001502 if (!SelectSMRD(Addr, SBase, Offset, Imm))
1503 return false;
1504
Marek Olsak8973a0a2017-05-24 14:53:50 +00001505 return !Imm && isa<ConstantSDNode>(Offset);
Tom Stellard217361c2015-08-06 19:28:38 +00001506}
1507
Tom Stellarddee26a22015-08-06 19:28:30 +00001508bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(SDValue Addr, SDValue &SBase,
1509 SDValue &Offset) const {
1510 bool Imm;
Tom Stellard217361c2015-08-06 19:28:38 +00001511 return SelectSMRD(Addr, SBase, Offset, Imm) && !Imm &&
1512 !isa<ConstantSDNode>(Offset);
Tom Stellarddee26a22015-08-06 19:28:30 +00001513}
1514
1515bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm(SDValue Addr,
1516 SDValue &Offset) const {
1517 bool Imm;
Marek Olsak8973a0a2017-05-24 14:53:50 +00001518 return SelectSMRDOffset(Addr, Offset, Imm) && Imm;
1519}
Tom Stellarddee26a22015-08-06 19:28:30 +00001520
Marek Olsak8973a0a2017-05-24 14:53:50 +00001521bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(SDValue Addr,
1522 SDValue &Offset) const {
1523 if (Subtarget->getGeneration() != AMDGPUSubtarget::SEA_ISLANDS)
1524 return false;
1525
1526 bool Imm;
Tom Stellard217361c2015-08-06 19:28:38 +00001527 if (!SelectSMRDOffset(Addr, Offset, Imm))
1528 return false;
1529
Marek Olsak8973a0a2017-05-24 14:53:50 +00001530 return !Imm && isa<ConstantSDNode>(Offset);
Tom Stellard217361c2015-08-06 19:28:38 +00001531}
1532
Nicolai Haehnle7968c342016-07-12 08:12:16 +00001533bool AMDGPUDAGToDAGISel::SelectMOVRELOffset(SDValue Index,
1534 SDValue &Base,
1535 SDValue &Offset) const {
Matt Arsenault1322b6f2016-07-09 01:13:56 +00001536 SDLoc DL(Index);
1537
1538 if (CurDAG->isBaseWithConstantOffset(Index)) {
1539 SDValue N0 = Index.getOperand(0);
1540 SDValue N1 = Index.getOperand(1);
1541 ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1542
1543 // (add n0, c0)
1544 Base = N0;
1545 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32);
1546 return true;
1547 }
1548
Nicolai Haehnle7968c342016-07-12 08:12:16 +00001549 if (isa<ConstantSDNode>(Index))
1550 return false;
Matt Arsenault1322b6f2016-07-09 01:13:56 +00001551
1552 Base = Index;
1553 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1554 return true;
1555}
1556
Benjamin Kramerbdc49562016-06-12 15:39:02 +00001557SDNode *AMDGPUDAGToDAGISel::getS_BFE(unsigned Opcode, const SDLoc &DL,
1558 SDValue Val, uint32_t Offset,
1559 uint32_t Width) {
Marek Olsak9b728682015-03-24 13:40:27 +00001560 // Transformation function, pack the offset and width of a BFE into
1561 // the format expected by the S_BFE_I32 / S_BFE_U32. In the second
1562 // source, bits [5:0] contain the offset and bits [22:16] the width.
1563 uint32_t PackedVal = Offset | (Width << 16);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001564 SDValue PackedConst = CurDAG->getTargetConstant(PackedVal, DL, MVT::i32);
Marek Olsak9b728682015-03-24 13:40:27 +00001565
1566 return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, PackedConst);
1567}
1568
Justin Bogner95927c02016-05-12 21:03:32 +00001569void AMDGPUDAGToDAGISel::SelectS_BFEFromShifts(SDNode *N) {
Marek Olsak9b728682015-03-24 13:40:27 +00001570 // "(a << b) srl c)" ---> "BFE_U32 a, (c-b), (32-c)
1571 // "(a << b) sra c)" ---> "BFE_I32 a, (c-b), (32-c)
1572 // Predicate: 0 < b <= c < 32
1573
1574 const SDValue &Shl = N->getOperand(0);
1575 ConstantSDNode *B = dyn_cast<ConstantSDNode>(Shl->getOperand(1));
1576 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
1577
1578 if (B && C) {
1579 uint32_t BVal = B->getZExtValue();
1580 uint32_t CVal = C->getZExtValue();
1581
1582 if (0 < BVal && BVal <= CVal && CVal < 32) {
1583 bool Signed = N->getOpcode() == ISD::SRA;
1584 unsigned Opcode = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
1585
Justin Bogner95927c02016-05-12 21:03:32 +00001586 ReplaceNode(N, getS_BFE(Opcode, SDLoc(N), Shl.getOperand(0), CVal - BVal,
1587 32 - CVal));
1588 return;
Marek Olsak9b728682015-03-24 13:40:27 +00001589 }
1590 }
Justin Bogner95927c02016-05-12 21:03:32 +00001591 SelectCode(N);
Marek Olsak9b728682015-03-24 13:40:27 +00001592}
1593
Justin Bogner95927c02016-05-12 21:03:32 +00001594void AMDGPUDAGToDAGISel::SelectS_BFE(SDNode *N) {
Marek Olsak9b728682015-03-24 13:40:27 +00001595 switch (N->getOpcode()) {
1596 case ISD::AND:
1597 if (N->getOperand(0).getOpcode() == ISD::SRL) {
1598 // "(a srl b) & mask" ---> "BFE_U32 a, b, popcount(mask)"
1599 // Predicate: isMask(mask)
1600 const SDValue &Srl = N->getOperand(0);
1601 ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(Srl.getOperand(1));
1602 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
1603
1604 if (Shift && Mask) {
1605 uint32_t ShiftVal = Shift->getZExtValue();
1606 uint32_t MaskVal = Mask->getZExtValue();
1607
1608 if (isMask_32(MaskVal)) {
1609 uint32_t WidthVal = countPopulation(MaskVal);
1610
Justin Bogner95927c02016-05-12 21:03:32 +00001611 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N),
1612 Srl.getOperand(0), ShiftVal, WidthVal));
1613 return;
Marek Olsak9b728682015-03-24 13:40:27 +00001614 }
1615 }
1616 }
1617 break;
1618 case ISD::SRL:
1619 if (N->getOperand(0).getOpcode() == ISD::AND) {
1620 // "(a & mask) srl b)" ---> "BFE_U32 a, b, popcount(mask >> b)"
1621 // Predicate: isMask(mask >> b)
1622 const SDValue &And = N->getOperand(0);
1623 ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(N->getOperand(1));
1624 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(And->getOperand(1));
1625
1626 if (Shift && Mask) {
1627 uint32_t ShiftVal = Shift->getZExtValue();
1628 uint32_t MaskVal = Mask->getZExtValue() >> ShiftVal;
1629
1630 if (isMask_32(MaskVal)) {
1631 uint32_t WidthVal = countPopulation(MaskVal);
1632
Justin Bogner95927c02016-05-12 21:03:32 +00001633 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N),
1634 And.getOperand(0), ShiftVal, WidthVal));
1635 return;
Marek Olsak9b728682015-03-24 13:40:27 +00001636 }
1637 }
Justin Bogner95927c02016-05-12 21:03:32 +00001638 } else if (N->getOperand(0).getOpcode() == ISD::SHL) {
1639 SelectS_BFEFromShifts(N);
1640 return;
1641 }
Marek Olsak9b728682015-03-24 13:40:27 +00001642 break;
1643 case ISD::SRA:
Justin Bogner95927c02016-05-12 21:03:32 +00001644 if (N->getOperand(0).getOpcode() == ISD::SHL) {
1645 SelectS_BFEFromShifts(N);
1646 return;
1647 }
Marek Olsak9b728682015-03-24 13:40:27 +00001648 break;
Matt Arsenault7e8de012016-04-22 22:59:16 +00001649
1650 case ISD::SIGN_EXTEND_INREG: {
1651 // sext_inreg (srl x, 16), i8 -> bfe_i32 x, 16, 8
1652 SDValue Src = N->getOperand(0);
1653 if (Src.getOpcode() != ISD::SRL)
1654 break;
1655
1656 const ConstantSDNode *Amt = dyn_cast<ConstantSDNode>(Src.getOperand(1));
1657 if (!Amt)
1658 break;
1659
1660 unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits();
Justin Bogner95927c02016-05-12 21:03:32 +00001661 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_I32, SDLoc(N), Src.getOperand(0),
1662 Amt->getZExtValue(), Width));
1663 return;
Matt Arsenault7e8de012016-04-22 22:59:16 +00001664 }
Marek Olsak9b728682015-03-24 13:40:27 +00001665 }
1666
Justin Bogner95927c02016-05-12 21:03:32 +00001667 SelectCode(N);
Marek Olsak9b728682015-03-24 13:40:27 +00001668}
1669
Matt Arsenault7b1dc2c2016-09-17 02:02:19 +00001670bool AMDGPUDAGToDAGISel::isCBranchSCC(const SDNode *N) const {
1671 assert(N->getOpcode() == ISD::BRCOND);
1672 if (!N->hasOneUse())
1673 return false;
1674
1675 SDValue Cond = N->getOperand(1);
1676 if (Cond.getOpcode() == ISD::CopyToReg)
1677 Cond = Cond.getOperand(2);
1678
1679 if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse())
1680 return false;
1681
1682 MVT VT = Cond.getOperand(0).getSimpleValueType();
1683 if (VT == MVT::i32)
1684 return true;
1685
1686 if (VT == MVT::i64) {
1687 auto ST = static_cast<const SISubtarget *>(Subtarget);
1688
1689 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
1690 return (CC == ISD::SETEQ || CC == ISD::SETNE) && ST->hasScalarCompareEq64();
1691 }
1692
1693 return false;
1694}
1695
Justin Bogner95927c02016-05-12 21:03:32 +00001696void AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) {
Tom Stellardbc4497b2016-02-12 23:45:29 +00001697 SDValue Cond = N->getOperand(1);
1698
Matt Arsenault327188a2016-12-15 21:57:11 +00001699 if (Cond.isUndef()) {
1700 CurDAG->SelectNodeTo(N, AMDGPU::SI_BR_UNDEF, MVT::Other,
1701 N->getOperand(2), N->getOperand(0));
1702 return;
1703 }
1704
Matt Arsenaultd674e0a2017-10-10 20:34:49 +00001705 bool UseSCCBr = isCBranchSCC(N) && isUniformBr(N);
1706 unsigned BrOp = UseSCCBr ? AMDGPU::S_CBRANCH_SCC1 : AMDGPU::S_CBRANCH_VCCNZ;
1707 unsigned CondReg = UseSCCBr ? AMDGPU::SCC : AMDGPU::VCC;
Tom Stellardbc4497b2016-02-12 23:45:29 +00001708 SDLoc SL(N);
1709
Tim Renouf6eaad1e2018-01-09 21:34:43 +00001710 if (!UseSCCBr) {
1711 // This is the case that we are selecting to S_CBRANCH_VCCNZ. We have not
1712 // analyzed what generates the vcc value, so we do not know whether vcc
1713 // bits for disabled lanes are 0. Thus we need to mask out bits for
1714 // disabled lanes.
1715 //
1716 // For the case that we select S_CBRANCH_SCC1 and it gets
1717 // changed to S_CBRANCH_VCCNZ in SIFixSGPRCopies, SIFixSGPRCopies calls
1718 // SIInstrInfo::moveToVALU which inserts the S_AND).
1719 //
1720 // We could add an analysis of what generates the vcc value here and omit
1721 // the S_AND when is unnecessary. But it would be better to add a separate
1722 // pass after SIFixSGPRCopies to do the unnecessary S_AND removal, so it
1723 // catches both cases.
1724 Cond = SDValue(CurDAG->getMachineNode(AMDGPU::S_AND_B64, SL, MVT::i1,
1725 CurDAG->getRegister(AMDGPU::EXEC, MVT::i1),
1726 Cond),
1727 0);
1728 }
1729
Matt Arsenaultd674e0a2017-10-10 20:34:49 +00001730 SDValue VCC = CurDAG->getCopyToReg(N->getOperand(0), SL, CondReg, Cond);
1731 CurDAG->SelectNodeTo(N, BrOp, MVT::Other,
Justin Bogner95927c02016-05-12 21:03:32 +00001732 N->getOperand(2), // Basic Block
Matt Arsenaultf530e8b2016-11-07 19:09:33 +00001733 VCC.getValue(0));
Tom Stellardbc4497b2016-02-12 23:45:29 +00001734}
1735
Matt Arsenaultd7e23032017-09-07 18:05:07 +00001736void AMDGPUDAGToDAGISel::SelectFMAD(SDNode *N) {
1737 MVT VT = N->getSimpleValueType(0);
1738 if (VT != MVT::f32 || !Subtarget->hasMadMixInsts()) {
1739 SelectCode(N);
1740 return;
1741 }
1742
1743 SDValue Src0 = N->getOperand(0);
1744 SDValue Src1 = N->getOperand(1);
1745 SDValue Src2 = N->getOperand(2);
1746 unsigned Src0Mods, Src1Mods, Src2Mods;
1747
1748 // Avoid using v_mad_mix_f32 unless there is actually an operand using the
1749 // conversion from f16.
1750 bool Sel0 = SelectVOP3PMadMixModsImpl(Src0, Src0, Src0Mods);
1751 bool Sel1 = SelectVOP3PMadMixModsImpl(Src1, Src1, Src1Mods);
1752 bool Sel2 = SelectVOP3PMadMixModsImpl(Src2, Src2, Src2Mods);
1753
1754 assert(!Subtarget->hasFP32Denormals() &&
1755 "fmad selected with denormals enabled");
1756 // TODO: We can select this with f32 denormals enabled if all the sources are
1757 // converted from f16 (in which case fmad isn't legal).
1758
1759 if (Sel0 || Sel1 || Sel2) {
1760 // For dummy operands.
1761 SDValue Zero = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32);
1762 SDValue Ops[] = {
1763 CurDAG->getTargetConstant(Src0Mods, SDLoc(), MVT::i32), Src0,
1764 CurDAG->getTargetConstant(Src1Mods, SDLoc(), MVT::i32), Src1,
1765 CurDAG->getTargetConstant(Src2Mods, SDLoc(), MVT::i32), Src2,
1766 CurDAG->getTargetConstant(0, SDLoc(), MVT::i1),
1767 Zero, Zero
1768 };
1769
1770 CurDAG->SelectNodeTo(N, AMDGPU::V_MAD_MIX_F32, MVT::f32, Ops);
1771 } else {
1772 SelectCode(N);
1773 }
1774}
1775
Matt Arsenault88701812016-06-09 23:42:48 +00001776// This is here because there isn't a way to use the generated sub0_sub1 as the
1777// subreg index to EXTRACT_SUBREG in tablegen.
1778void AMDGPUDAGToDAGISel::SelectATOMIC_CMP_SWAP(SDNode *N) {
1779 MemSDNode *Mem = cast<MemSDNode>(N);
1780 unsigned AS = Mem->getAddressSpace();
Yaxun Liu1a14bfa2017-03-27 14:04:01 +00001781 if (AS == AMDGPUASI.FLAT_ADDRESS) {
Matt Arsenault7757c592016-06-09 23:42:54 +00001782 SelectCode(N);
1783 return;
1784 }
Matt Arsenault88701812016-06-09 23:42:48 +00001785
1786 MVT VT = N->getSimpleValueType(0);
1787 bool Is32 = (VT == MVT::i32);
1788 SDLoc SL(N);
1789
1790 MachineSDNode *CmpSwap = nullptr;
1791 if (Subtarget->hasAddr64()) {
Vitaly Buka74503982017-10-15 05:35:02 +00001792 SDValue SRsrc, VAddr, SOffset, Offset, SLC;
Matt Arsenault88701812016-06-09 23:42:48 +00001793
1794 if (SelectMUBUFAddr64(Mem->getBasePtr(), SRsrc, VAddr, SOffset, Offset, SLC)) {
Matt Arsenaulte5456ce2017-07-20 21:06:04 +00001795 unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_ADDR64_RTN :
1796 AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_ADDR64_RTN;
Matt Arsenault88701812016-06-09 23:42:48 +00001797 SDValue CmpVal = Mem->getOperand(2);
1798
1799 // XXX - Do we care about glue operands?
1800
1801 SDValue Ops[] = {
1802 CmpVal, VAddr, SRsrc, SOffset, Offset, SLC, Mem->getChain()
1803 };
1804
1805 CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops);
1806 }
1807 }
1808
1809 if (!CmpSwap) {
1810 SDValue SRsrc, SOffset, Offset, SLC;
1811 if (SelectMUBUFOffset(Mem->getBasePtr(), SRsrc, SOffset, Offset, SLC)) {
Matt Arsenaulte5456ce2017-07-20 21:06:04 +00001812 unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN :
1813 AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_OFFSET_RTN;
Matt Arsenault88701812016-06-09 23:42:48 +00001814
1815 SDValue CmpVal = Mem->getOperand(2);
1816 SDValue Ops[] = {
1817 CmpVal, SRsrc, SOffset, Offset, SLC, Mem->getChain()
1818 };
1819
1820 CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops);
1821 }
1822 }
1823
1824 if (!CmpSwap) {
1825 SelectCode(N);
1826 return;
1827 }
1828
1829 MachineSDNode::mmo_iterator MMOs = MF->allocateMemRefsArray(1);
1830 *MMOs = Mem->getMemOperand();
1831 CmpSwap->setMemRefs(MMOs, MMOs + 1);
1832
1833 unsigned SubReg = Is32 ? AMDGPU::sub0 : AMDGPU::sub0_sub1;
1834 SDValue Extract
1835 = CurDAG->getTargetExtractSubreg(SubReg, SL, VT, SDValue(CmpSwap, 0));
1836
1837 ReplaceUses(SDValue(N, 0), Extract);
1838 ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 1));
1839 CurDAG->RemoveDeadNode(N);
1840}
1841
Matt Arsenaultd7e23032017-09-07 18:05:07 +00001842bool AMDGPUDAGToDAGISel::SelectVOP3ModsImpl(SDValue In, SDValue &Src,
1843 unsigned &Mods) const {
1844 Mods = 0;
Tom Stellardb4a313a2014-08-01 00:32:39 +00001845 Src = In;
1846
1847 if (Src.getOpcode() == ISD::FNEG) {
1848 Mods |= SISrcMods::NEG;
1849 Src = Src.getOperand(0);
1850 }
1851
1852 if (Src.getOpcode() == ISD::FABS) {
1853 Mods |= SISrcMods::ABS;
1854 Src = Src.getOperand(0);
1855 }
1856
Tom Stellardb4a313a2014-08-01 00:32:39 +00001857 return true;
1858}
1859
Matt Arsenaultd7e23032017-09-07 18:05:07 +00001860bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src,
1861 SDValue &SrcMods) const {
1862 unsigned Mods;
1863 if (SelectVOP3ModsImpl(In, Src, Mods)) {
1864 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
1865 return true;
1866 }
1867
1868 return false;
1869}
1870
Matt Arsenaultf84e5d92017-01-31 03:07:46 +00001871bool AMDGPUDAGToDAGISel::SelectVOP3Mods_NNaN(SDValue In, SDValue &Src,
1872 SDValue &SrcMods) const {
1873 SelectVOP3Mods(In, Src, SrcMods);
1874 return isNoNanSrc(Src);
1875}
1876
Matt Arsenaultdf58e822017-04-25 21:17:38 +00001877bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const {
1878 if (In.getOpcode() == ISD::FABS || In.getOpcode() == ISD::FNEG)
1879 return false;
1880
1881 Src = In;
1882 return true;
Tom Stellarddb5a11f2015-07-13 15:47:57 +00001883}
1884
Tom Stellardb4a313a2014-08-01 00:32:39 +00001885bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(SDValue In, SDValue &Src,
1886 SDValue &SrcMods, SDValue &Clamp,
1887 SDValue &Omod) const {
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001888 SDLoc DL(In);
Matt Arsenaultdf58e822017-04-25 21:17:38 +00001889 Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
1890 Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
Tom Stellardb4a313a2014-08-01 00:32:39 +00001891
1892 return SelectVOP3Mods(In, Src, SrcMods);
1893}
1894
Matt Arsenault4831ce52015-01-06 23:00:37 +00001895bool AMDGPUDAGToDAGISel::SelectVOP3Mods0Clamp0OMod(SDValue In, SDValue &Src,
1896 SDValue &SrcMods,
1897 SDValue &Clamp,
1898 SDValue &Omod) const {
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001899 Clamp = Omod = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32);
Matt Arsenault4831ce52015-01-06 23:00:37 +00001900 return SelectVOP3Mods(In, Src, SrcMods);
1901}
1902
Dmitry Preobrazhenskyc512d442017-03-27 15:57:17 +00001903bool AMDGPUDAGToDAGISel::SelectVOP3OMods(SDValue In, SDValue &Src,
1904 SDValue &Clamp, SDValue &Omod) const {
1905 Src = In;
1906
1907 SDLoc DL(In);
Matt Arsenaultdf58e822017-04-25 21:17:38 +00001908 Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
1909 Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
Dmitry Preobrazhenskyc512d442017-03-27 15:57:17 +00001910
1911 return true;
1912}
1913
Matt Arsenault98f29462017-05-17 20:30:58 +00001914static SDValue stripBitcast(SDValue Val) {
1915 return Val.getOpcode() == ISD::BITCAST ? Val.getOperand(0) : Val;
1916}
1917
1918// Figure out if this is really an extract of the high 16-bits of a dword.
1919static bool isExtractHiElt(SDValue In, SDValue &Out) {
1920 In = stripBitcast(In);
1921 if (In.getOpcode() != ISD::TRUNCATE)
1922 return false;
1923
1924 SDValue Srl = In.getOperand(0);
1925 if (Srl.getOpcode() == ISD::SRL) {
1926 if (ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(Srl.getOperand(1))) {
1927 if (ShiftAmt->getZExtValue() == 16) {
1928 Out = stripBitcast(Srl.getOperand(0));
1929 return true;
1930 }
1931 }
1932 }
1933
1934 return false;
1935}
1936
1937// Look through operations that obscure just looking at the low 16-bits of the
1938// same register.
1939static SDValue stripExtractLoElt(SDValue In) {
1940 if (In.getOpcode() == ISD::TRUNCATE) {
1941 SDValue Src = In.getOperand(0);
1942 if (Src.getValueType().getSizeInBits() == 32)
1943 return stripBitcast(Src);
1944 }
1945
1946 return In;
1947}
1948
Matt Arsenaulteb522e62017-02-27 22:15:25 +00001949bool AMDGPUDAGToDAGISel::SelectVOP3PMods(SDValue In, SDValue &Src,
1950 SDValue &SrcMods) const {
1951 unsigned Mods = 0;
1952 Src = In;
1953
Matt Arsenaulteb522e62017-02-27 22:15:25 +00001954 if (Src.getOpcode() == ISD::FNEG) {
Matt Arsenault786eeea2017-05-17 20:00:00 +00001955 Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
Matt Arsenaulteb522e62017-02-27 22:15:25 +00001956 Src = Src.getOperand(0);
1957 }
1958
Matt Arsenault786eeea2017-05-17 20:00:00 +00001959 if (Src.getOpcode() == ISD::BUILD_VECTOR) {
1960 unsigned VecMods = Mods;
1961
Matt Arsenault98f29462017-05-17 20:30:58 +00001962 SDValue Lo = stripBitcast(Src.getOperand(0));
1963 SDValue Hi = stripBitcast(Src.getOperand(1));
Matt Arsenault786eeea2017-05-17 20:00:00 +00001964
1965 if (Lo.getOpcode() == ISD::FNEG) {
Matt Arsenault98f29462017-05-17 20:30:58 +00001966 Lo = stripBitcast(Lo.getOperand(0));
Matt Arsenault786eeea2017-05-17 20:00:00 +00001967 Mods ^= SISrcMods::NEG;
1968 }
1969
1970 if (Hi.getOpcode() == ISD::FNEG) {
Matt Arsenault98f29462017-05-17 20:30:58 +00001971 Hi = stripBitcast(Hi.getOperand(0));
Matt Arsenault786eeea2017-05-17 20:00:00 +00001972 Mods ^= SISrcMods::NEG_HI;
1973 }
1974
Matt Arsenault98f29462017-05-17 20:30:58 +00001975 if (isExtractHiElt(Lo, Lo))
1976 Mods |= SISrcMods::OP_SEL_0;
1977
1978 if (isExtractHiElt(Hi, Hi))
1979 Mods |= SISrcMods::OP_SEL_1;
1980
1981 Lo = stripExtractLoElt(Lo);
1982 Hi = stripExtractLoElt(Hi);
1983
Matt Arsenault786eeea2017-05-17 20:00:00 +00001984 if (Lo == Hi && !isInlineImmediate(Lo.getNode())) {
1985 // Really a scalar input. Just select from the low half of the register to
1986 // avoid packing.
1987
1988 Src = Lo;
1989 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
1990 return true;
1991 }
1992
1993 Mods = VecMods;
1994 }
1995
Matt Arsenaulteb522e62017-02-27 22:15:25 +00001996 // Packed instructions do not have abs modifiers.
Matt Arsenaulteb522e62017-02-27 22:15:25 +00001997 Mods |= SISrcMods::OP_SEL_1;
1998
1999 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2000 return true;
2001}
2002
2003bool AMDGPUDAGToDAGISel::SelectVOP3PMods0(SDValue In, SDValue &Src,
2004 SDValue &SrcMods,
2005 SDValue &Clamp) const {
2006 SDLoc SL(In);
2007
2008 // FIXME: Handle clamp and op_sel
2009 Clamp = CurDAG->getTargetConstant(0, SL, MVT::i32);
2010
2011 return SelectVOP3PMods(In, Src, SrcMods);
2012}
2013
Dmitry Preobrazhenskyabf28392017-07-21 13:54:11 +00002014bool AMDGPUDAGToDAGISel::SelectVOP3OpSel(SDValue In, SDValue &Src,
2015 SDValue &SrcMods) const {
2016 Src = In;
2017 // FIXME: Handle op_sel
2018 SrcMods = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32);
2019 return true;
2020}
2021
2022bool AMDGPUDAGToDAGISel::SelectVOP3OpSel0(SDValue In, SDValue &Src,
2023 SDValue &SrcMods,
2024 SDValue &Clamp) const {
2025 SDLoc SL(In);
2026
2027 // FIXME: Handle clamp
2028 Clamp = CurDAG->getTargetConstant(0, SL, MVT::i32);
2029
2030 return SelectVOP3OpSel(In, Src, SrcMods);
2031}
2032
2033bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods(SDValue In, SDValue &Src,
2034 SDValue &SrcMods) const {
2035 // FIXME: Handle op_sel
2036 return SelectVOP3Mods(In, Src, SrcMods);
2037}
2038
2039bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods0(SDValue In, SDValue &Src,
2040 SDValue &SrcMods,
2041 SDValue &Clamp) const {
2042 SDLoc SL(In);
2043
2044 // FIXME: Handle clamp
2045 Clamp = CurDAG->getTargetConstant(0, SL, MVT::i32);
2046
2047 return SelectVOP3OpSelMods(In, Src, SrcMods);
2048}
2049
Matt Arsenaultd7e23032017-09-07 18:05:07 +00002050// The return value is not whether the match is possible (which it always is),
2051// but whether or not it a conversion is really used.
2052bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src,
2053 unsigned &Mods) const {
2054 Mods = 0;
2055 SelectVOP3ModsImpl(In, Src, Mods);
2056
2057 if (Src.getOpcode() == ISD::FP_EXTEND) {
2058 Src = Src.getOperand(0);
2059 assert(Src.getValueType() == MVT::f16);
2060 Src = stripBitcast(Src);
2061
Matt Arsenault550c66d2017-10-13 20:45:49 +00002062 // Be careful about folding modifiers if we already have an abs. fneg is
2063 // applied last, so we don't want to apply an earlier fneg.
2064 if ((Mods & SISrcMods::ABS) == 0) {
2065 unsigned ModsTmp;
2066 SelectVOP3ModsImpl(Src, Src, ModsTmp);
2067
2068 if ((ModsTmp & SISrcMods::NEG) != 0)
2069 Mods ^= SISrcMods::NEG;
2070
2071 if ((ModsTmp & SISrcMods::ABS) != 0)
2072 Mods |= SISrcMods::ABS;
2073 }
2074
Matt Arsenaultd7e23032017-09-07 18:05:07 +00002075 // op_sel/op_sel_hi decide the source type and source.
2076 // If the source's op_sel_hi is set, it indicates to do a conversion from fp16.
2077 // If the sources's op_sel is set, it picks the high half of the source
2078 // register.
2079
2080 Mods |= SISrcMods::OP_SEL_1;
Matt Arsenault550c66d2017-10-13 20:45:49 +00002081 if (isExtractHiElt(Src, Src)) {
Matt Arsenaultd7e23032017-09-07 18:05:07 +00002082 Mods |= SISrcMods::OP_SEL_0;
2083
Matt Arsenault550c66d2017-10-13 20:45:49 +00002084 // TODO: Should we try to look for neg/abs here?
2085 }
2086
Matt Arsenaultd7e23032017-09-07 18:05:07 +00002087 return true;
2088 }
2089
2090 return false;
2091}
2092
Matt Arsenault76935122017-09-20 20:28:39 +00002093bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixMods(SDValue In, SDValue &Src,
2094 SDValue &SrcMods) const {
2095 unsigned Mods = 0;
2096 SelectVOP3PMadMixModsImpl(In, Src, Mods);
2097 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2098 return true;
2099}
2100
Matt Arsenaulte1cd4822017-11-13 00:22:09 +00002101// TODO: Can we identify things like v_mad_mixhi_f16?
2102bool AMDGPUDAGToDAGISel::SelectHi16Elt(SDValue In, SDValue &Src) const {
2103 if (In.isUndef()) {
2104 Src = In;
2105 return true;
2106 }
2107
2108 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(In)) {
2109 SDLoc SL(In);
2110 SDValue K = CurDAG->getTargetConstant(C->getZExtValue() << 16, SL, MVT::i32);
2111 MachineSDNode *MovK = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
2112 SL, MVT::i32, K);
2113 Src = SDValue(MovK, 0);
2114 return true;
2115 }
2116
2117 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(In)) {
2118 SDLoc SL(In);
2119 SDValue K = CurDAG->getTargetConstant(
2120 C->getValueAPF().bitcastToAPInt().getZExtValue() << 16, SL, MVT::i32);
2121 MachineSDNode *MovK = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
2122 SL, MVT::i32, K);
2123 Src = SDValue(MovK, 0);
2124 return true;
2125 }
2126
2127 return isExtractHiElt(In, Src);
2128}
2129
Christian Konigd910b7d2013-02-26 17:52:16 +00002130void AMDGPUDAGToDAGISel::PostprocessISelDAG() {
Bill Wendlinga3cd3502013-06-19 21:36:55 +00002131 const AMDGPUTargetLowering& Lowering =
Matt Arsenault209a7b92014-04-18 07:40:20 +00002132 *static_cast<const AMDGPUTargetLowering*>(getTargetLowering());
Vincent Lejeuneab3baf82013-09-12 23:44:44 +00002133 bool IsModified = false;
2134 do {
2135 IsModified = false;
Matt Arsenault68f05052017-12-04 22:18:27 +00002136
Vincent Lejeuneab3baf82013-09-12 23:44:44 +00002137 // Go over all selected nodes and try to fold them a bit more
Matt Arsenault68f05052017-12-04 22:18:27 +00002138 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_begin();
2139 while (Position != CurDAG->allnodes_end()) {
2140 SDNode *Node = &*Position++;
2141 MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(Node);
Vincent Lejeuneab3baf82013-09-12 23:44:44 +00002142 if (!MachineNode)
2143 continue;
Christian Konigd910b7d2013-02-26 17:52:16 +00002144
Vincent Lejeuneab3baf82013-09-12 23:44:44 +00002145 SDNode *ResNode = Lowering.PostISelFolding(MachineNode, *CurDAG);
Matt Arsenault68f05052017-12-04 22:18:27 +00002146 if (ResNode != Node) {
2147 if (ResNode)
2148 ReplaceUses(Node, ResNode);
Vincent Lejeuneab3baf82013-09-12 23:44:44 +00002149 IsModified = true;
2150 }
Tom Stellard2183b702013-06-03 17:39:46 +00002151 }
Vincent Lejeuneab3baf82013-09-12 23:44:44 +00002152 CurDAG->RemoveDeadNodes();
2153 } while (IsModified);
Christian Konigd910b7d2013-02-26 17:52:16 +00002154}
Tom Stellard20287692017-08-08 04:57:55 +00002155
2156void R600DAGToDAGISel::Select(SDNode *N) {
2157 unsigned int Opc = N->getOpcode();
2158 if (N->isMachineOpcode()) {
2159 N->setNodeId(-1);
2160 return; // Already selected.
2161 }
2162
2163 switch (Opc) {
2164 default: break;
2165 case AMDGPUISD::BUILD_VERTICAL_VECTOR:
2166 case ISD::SCALAR_TO_VECTOR:
2167 case ISD::BUILD_VECTOR: {
2168 EVT VT = N->getValueType(0);
2169 unsigned NumVectorElts = VT.getVectorNumElements();
2170 unsigned RegClassID;
2171 // BUILD_VECTOR was lowered into an IMPLICIT_DEF + 4 INSERT_SUBREG
2172 // that adds a 128 bits reg copy when going through TwoAddressInstructions
2173 // pass. We want to avoid 128 bits copies as much as possible because they
2174 // can't be bundled by our scheduler.
2175 switch(NumVectorElts) {
2176 case 2: RegClassID = AMDGPU::R600_Reg64RegClassID; break;
2177 case 4:
2178 if (Opc == AMDGPUISD::BUILD_VERTICAL_VECTOR)
2179 RegClassID = AMDGPU::R600_Reg128VerticalRegClassID;
2180 else
2181 RegClassID = AMDGPU::R600_Reg128RegClassID;
2182 break;
2183 default: llvm_unreachable("Do not know how to lower this BUILD_VECTOR");
2184 }
2185 SelectBuildVector(N, RegClassID);
2186 return;
2187 }
2188 }
2189
2190 SelectCode(N);
2191}
2192
2193bool R600DAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base,
2194 SDValue &Offset) {
2195 ConstantSDNode *C;
2196 SDLoc DL(Addr);
2197
2198 if ((C = dyn_cast<ConstantSDNode>(Addr))) {
2199 Base = CurDAG->getRegister(AMDGPU::INDIRECT_BASE_ADDR, MVT::i32);
2200 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
2201 } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) &&
2202 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) {
2203 Base = CurDAG->getRegister(AMDGPU::INDIRECT_BASE_ADDR, MVT::i32);
2204 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
2205 } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) &&
2206 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) {
2207 Base = Addr.getOperand(0);
2208 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
2209 } else {
2210 Base = Addr;
2211 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
2212 }
2213
2214 return true;
2215}
2216
2217bool R600DAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
2218 SDValue &Offset) {
2219 ConstantSDNode *IMMOffset;
2220
2221 if (Addr.getOpcode() == ISD::ADD
2222 && (IMMOffset = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))
2223 && isInt<16>(IMMOffset->getZExtValue())) {
2224
2225 Base = Addr.getOperand(0);
2226 Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr),
2227 MVT::i32);
2228 return true;
2229 // If the pointer address is constant, we can move it to the offset field.
2230 } else if ((IMMOffset = dyn_cast<ConstantSDNode>(Addr))
2231 && isInt<16>(IMMOffset->getZExtValue())) {
2232 Base = CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
2233 SDLoc(CurDAG->getEntryNode()),
2234 AMDGPU::ZERO, MVT::i32);
2235 Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr),
2236 MVT::i32);
2237 return true;
2238 }
2239
2240 // Default case, no offset
2241 Base = Addr;
2242 Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
2243 return true;
2244}