AMDGPU: Select MIMG instructions manually in SITargetLowering
Summary:
Having TableGen patterns for image intrinsics is hitting limitations:
for D16 we already have to manually pre-lower the packing of data
values, and we will have to do the same for A16 eventually.
Since there is already some custom C++ code anyway, it is arguably easier
to just do everything in C++, now that we can use the beefed-up generic
tables backend of TableGen to provide all the required metadata and map
intrinsics to corresponding opcodes. With this approach, all image
intrinsic lowering happens in SITargetLowering::lowerImage. That code is
dense due to all the cases that it handles, but it should still be easier
to follow than what we had before, by virtue of it all being done in a
single location, and by virtue of not relying on the TableGen pattern
magic that very few people really understand.
This means that we will have MachineSDNodes with MIMG instructions
during DAG combining, but that seems alright: previously we had
intrinsic nodes instead, but those are similarly opaque to the generic
CodeGen infrastructure, and the final pattern matching just did a 1:1
translation to machine instructions anyway. If anything, the fact that
we now merge the address words into a vector before DAG combine should
be an advantage.
Change-Id: I417f26bd88f54ce9781c1668acc01f3f99774de6
Reviewers: arsenm, rampitec, rtaylor, tstellar
Subscribers: kzhuravl, wdng, yaxunl, dstuttard, tpr, t-tye, llvm-commits
Differential Revision: https://reviews.llvm.org/D48017
llvm-svn: 335228
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index d71f391..e17cef8 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -4516,6 +4516,245 @@
return DAG.getUNDEF(VT);
}
+static SDValue getBuildDwordsVector(SelectionDAG &DAG, SDLoc DL,
+ ArrayRef<SDValue> Elts) {
+ assert(!Elts.empty());
+ MVT Type;
+ unsigned NumElts;
+
+ if (Elts.size() == 1) {
+ Type = MVT::f32;
+ NumElts = 1;
+ } else if (Elts.size() == 2) {
+ Type = MVT::v2f32;
+ NumElts = 2;
+ } else if (Elts.size() <= 4) {
+ Type = MVT::v4f32;
+ NumElts = 4;
+ } else if (Elts.size() <= 8) {
+ Type = MVT::v8f32;
+ NumElts = 8;
+ } else {
+ assert(Elts.size() <= 16);
+ Type = MVT::v16f32;
+ NumElts = 16;
+ }
+
+ SmallVector<SDValue, 16> VecElts(NumElts);
+ for (unsigned i = 0; i < Elts.size(); ++i) {
+ SDValue Elt = Elts[i];
+ if (Elt.getValueType() != MVT::f32)
+ Elt = DAG.getBitcast(MVT::f32, Elt);
+ VecElts[i] = Elt;
+ }
+ for (unsigned i = Elts.size(); i < NumElts; ++i)
+ VecElts[i] = DAG.getUNDEF(MVT::f32);
+
+ if (NumElts == 1)
+ return VecElts[0];
+ return DAG.getBuildVector(Type, DL, VecElts);
+}
+
+static bool parseCachePolicy(SDValue CachePolicy, SelectionDAG &DAG,
+ SDValue *GLC, SDValue *SLC) {
+ auto CachePolicyConst = dyn_cast<ConstantSDNode>(CachePolicy.getNode());
+ if (!CachePolicyConst)
+ return false;
+
+ uint64_t Value = CachePolicyConst->getZExtValue();
+ SDLoc DL(CachePolicy);
+ if (GLC) {
+ *GLC = DAG.getTargetConstant((Value & 0x1) ? 1 : 0, DL, MVT::i32);
+ Value &= ~(uint64_t)0x1;
+ }
+ if (SLC) {
+ *SLC = DAG.getTargetConstant((Value & 0x2) ? 1 : 0, DL, MVT::i32);
+ Value &= ~(uint64_t)0x2;
+ }
+
+ return Value == 0;
+}
+
+SDValue SITargetLowering::lowerImage(SDValue Op,
+ const AMDGPU::ImageDimIntrinsicInfo *Intr,
+ SelectionDAG &DAG) const {
+ SDLoc DL(Op);
+ MachineFunction &MF = DAG.getMachineFunction();
+ const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
+ AMDGPU::getMIMGBaseOpcodeInfo(Intr->BaseOpcode);
+ const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfo(Intr->Dim);
+
+ SmallVector<EVT, 2> ResultTypes(Op->value_begin(), Op->value_end());
+ bool IsD16 = false;
+ SDValue VData;
+ int NumVDataDwords;
+ unsigned AddrIdx; // Index of first address argument
+ unsigned DMask;
+
+ if (BaseOpcode->Atomic) {
+ VData = Op.getOperand(2);
+
+ bool Is64Bit = VData.getValueType() == MVT::i64;
+ if (BaseOpcode->AtomicX2) {
+ SDValue VData2 = Op.getOperand(3);
+ VData = DAG.getBuildVector(Is64Bit ? MVT::v2i64 : MVT::v2i32, DL,
+ {VData, VData2});
+ if (Is64Bit)
+ VData = DAG.getBitcast(MVT::v4i32, VData);
+
+ ResultTypes[0] = Is64Bit ? MVT::v2i64 : MVT::v2i32;
+ DMask = Is64Bit ? 0xf : 0x3;
+ NumVDataDwords = Is64Bit ? 4 : 2;
+ AddrIdx = 4;
+ } else {
+ DMask = Is64Bit ? 0x3 : 0x1;
+ NumVDataDwords = Is64Bit ? 2 : 1;
+ AddrIdx = 3;
+ }
+ } else {
+ unsigned DMaskIdx;
+
+ if (BaseOpcode->Store) {
+ VData = Op.getOperand(2);
+
+ MVT StoreVT = VData.getSimpleValueType();
+ if (StoreVT.getScalarType() == MVT::f16) {
+ if (Subtarget->getGeneration() < SISubtarget::VOLCANIC_ISLANDS ||
+ !BaseOpcode->HasD16)
+ return Op; // D16 is unsupported for this instruction
+
+ IsD16 = true;
+ VData = handleD16VData(VData, DAG);
+ }
+
+ NumVDataDwords = (VData.getValueType().getSizeInBits() + 31) / 32;
+ DMaskIdx = 3;
+ } else {
+ MVT LoadVT = Op.getSimpleValueType();
+ if (LoadVT.getScalarType() == MVT::f16) {
+ if (Subtarget->getGeneration() < SISubtarget::VOLCANIC_ISLANDS ||
+ !BaseOpcode->HasD16)
+ return Op; // D16 is unsupported for this instruction
+
+ IsD16 = true;
+ if (LoadVT.isVector() && Subtarget->hasUnpackedD16VMem())
+ ResultTypes[0] = (LoadVT == MVT::v2f16) ? MVT::v2i32 : MVT::v4i32;
+ }
+
+ NumVDataDwords = (ResultTypes[0].getSizeInBits() + 31) / 32;
+ DMaskIdx = isa<MemSDNode>(Op) ? 2 : 1;
+ }
+
+ auto DMaskConst = dyn_cast<ConstantSDNode>(Op.getOperand(DMaskIdx));
+ if (!DMaskConst)
+ return Op;
+
+ AddrIdx = DMaskIdx + 1;
+ DMask = DMaskConst->getZExtValue();
+ if (!DMask && !BaseOpcode->Store) {
+ // Eliminate no-op loads. Stores with dmask == 0 are *not* no-op: they
+ // store the channels' default values.
+ SDValue Undef = DAG.getUNDEF(Op.getValueType());
+ if (isa<MemSDNode>(Op))
+ return DAG.getMergeValues({Undef, Op.getOperand(0)}, DL);
+ return Undef;
+ }
+ }
+
+ unsigned NumVAddrs = BaseOpcode->NumExtraArgs +
+ (BaseOpcode->Gradients ? DimInfo->NumGradients : 0) +
+ (BaseOpcode->Coordinates ? DimInfo->NumCoords : 0) +
+ (BaseOpcode->LodOrClampOrMip ? 1 : 0);
+ SmallVector<SDValue, 4> VAddrs;
+ for (unsigned i = 0; i < NumVAddrs; ++i)
+ VAddrs.push_back(Op.getOperand(AddrIdx + i));
+ SDValue VAddr = getBuildDwordsVector(DAG, DL, VAddrs);
+
+ SDValue True = DAG.getTargetConstant(1, DL, MVT::i1);
+ SDValue False = DAG.getTargetConstant(0, DL, MVT::i1);
+ unsigned CtrlIdx; // Index of texfailctrl argument
+ SDValue Unorm;
+ if (!BaseOpcode->Sampler) {
+ Unorm = True;
+ CtrlIdx = AddrIdx + NumVAddrs + 1;
+ } else {
+ auto UnormConst =
+ dyn_cast<ConstantSDNode>(Op.getOperand(AddrIdx + NumVAddrs + 2));
+ if (!UnormConst)
+ return Op;
+
+ Unorm = UnormConst->getZExtValue() ? True : False;
+ CtrlIdx = AddrIdx + NumVAddrs + 3;
+ }
+
+ SDValue TexFail = Op.getOperand(CtrlIdx);
+ auto TexFailConst = dyn_cast<ConstantSDNode>(TexFail.getNode());
+ if (!TexFailConst || TexFailConst->getZExtValue() != 0)
+ return Op;
+
+ SDValue GLC;
+ SDValue SLC;
+ if (BaseOpcode->Atomic) {
+ GLC = True; // TODO no-return optimization
+ if (!parseCachePolicy(Op.getOperand(CtrlIdx + 1), DAG, nullptr, &SLC))
+ return Op;
+ } else {
+ if (!parseCachePolicy(Op.getOperand(CtrlIdx + 1), DAG, &GLC, &SLC))
+ return Op;
+ }
+
+ SmallVector<SDValue, 14> Ops;
+ if (BaseOpcode->Store || BaseOpcode->Atomic)
+ Ops.push_back(VData); // vdata
+ Ops.push_back(VAddr);
+ Ops.push_back(Op.getOperand(AddrIdx + NumVAddrs)); // rsrc
+ if (BaseOpcode->Sampler)
+ Ops.push_back(Op.getOperand(AddrIdx + NumVAddrs + 1)); // sampler
+ Ops.push_back(DAG.getTargetConstant(DMask, DL, MVT::i32));
+ Ops.push_back(Unorm);
+ Ops.push_back(GLC);
+ Ops.push_back(SLC);
+ Ops.push_back(False); // r128
+ Ops.push_back(False); // tfe
+ Ops.push_back(False); // lwe
+ Ops.push_back(DimInfo->DA ? True : False);
+ if (BaseOpcode->HasD16)
+ Ops.push_back(IsD16 ? True : False);
+ if (isa<MemSDNode>(Op))
+ Ops.push_back(Op.getOperand(0)); // chain
+
+ int NumVAddrDwords = VAddr.getValueType().getSizeInBits() / 32;
+ int Opcode = -1;
+
+ if (Subtarget->getGeneration() >= SISubtarget::VOLCANIC_ISLANDS)
+ Opcode = AMDGPU::getMIMGOpcode(Intr->BaseOpcode, AMDGPU::MIMGEncGfx8,
+ NumVDataDwords, NumVAddrDwords);
+ if (Opcode == -1)
+ Opcode = AMDGPU::getMIMGOpcode(Intr->BaseOpcode, AMDGPU::MIMGEncGfx6,
+ NumVDataDwords, NumVAddrDwords);
+ assert(Opcode != -1);
+
+ MachineSDNode *NewNode = DAG.getMachineNode(Opcode, DL, ResultTypes, Ops);
+ if (auto MemOp = dyn_cast<MemSDNode>(Op)) {
+ MachineInstr::mmo_iterator MemRefs = MF.allocateMemRefsArray(1);
+ *MemRefs = MemOp->getMemOperand();
+ NewNode->setMemRefs(MemRefs, MemRefs + 1);
+ }
+
+ if (BaseOpcode->AtomicX2) {
+ SmallVector<SDValue, 1> Elt;
+ DAG.ExtractVectorElements(SDValue(NewNode, 0), Elt, 0, 1);
+ return DAG.getMergeValues({Elt[0], SDValue(NewNode, 1)}, DL);
+ } else if (IsD16 && !BaseOpcode->Store) {
+ MVT LoadVT = Op.getSimpleValueType();
+ SDValue Adjusted = adjustLoadValueTypeImpl(
+ SDValue(NewNode, 0), LoadVT, DL, DAG, Subtarget->hasUnpackedD16VMem());
+ return DAG.getMergeValues({Adjusted, SDValue(NewNode, 1)}, DL);
+ }
+
+ return SDValue(NewNode, 0);
+}
+
SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
SelectionDAG &DAG) const {
MachineFunction &MF = DAG.getMachineFunction();
@@ -4853,6 +5092,10 @@
return SDValue();
}
default:
+ if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
+ AMDGPU::getImageDimIntrinsicInfo(IntrinsicID))
+ return lowerImage(Op, ImageDimIntr, DAG);
+
return Op;
}
}
@@ -5134,15 +5377,9 @@
return SDValue();
}
default:
- if (Subtarget->hasUnpackedD16VMem() &&
- Op.getValueType().isVector() &&
- Op.getValueType().getScalarSizeInBits() == 16) {
- if (const AMDGPU::D16ImageDimIntrinsic *D16ImageDimIntr =
- AMDGPU::lookupD16ImageDimIntrinsic(IntrID)) {
- return adjustLoadValueType(D16ImageDimIntr->D16HelperIntr,
- cast<MemSDNode>(Op), DAG, true);
- }
- }
+ if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
+ AMDGPU::getImageDimIntrinsicInfo(IntrID))
+ return lowerImage(Op, ImageDimIntr, DAG);
return SDValue();
}
@@ -5392,25 +5629,9 @@
return SDValue();
}
default: {
- const AMDGPU::D16ImageDimIntrinsic *D16ImageDimIntr =
- AMDGPU::lookupD16ImageDimIntrinsic(IntrinsicID);
- if (D16ImageDimIntr) {
- SDValue VData = Op.getOperand(2);
- EVT StoreVT = VData.getValueType();
- if (Subtarget->hasUnpackedD16VMem() &&
- StoreVT.isVector() &&
- StoreVT.getScalarSizeInBits() == 16) {
- SmallVector<SDValue, 12> Ops(Op.getNode()->op_values());
-
- Ops[1] = DAG.getConstant(D16ImageDimIntr->D16HelperIntr, DL, MVT::i32);
- Ops[2] = handleD16VData(VData, DAG);
-
- MemSDNode *M = cast<MemSDNode>(Op);
- return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, Op->getVTList(),
- Ops, M->getMemoryVT(),
- M->getMemOperand());
- }
- }
+ if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
+ AMDGPU::getImageDimIntrinsicInfo(IntrinsicID))
+ return lowerImage(Op, ImageDimIntr, DAG);
return Op;
}