AMDGPU: Select mulhi 24-bit instructions
llvm-svn: 279902
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index b628d2a..923494e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -467,6 +467,8 @@
setTargetDAGCombine(ISD::SRA);
setTargetDAGCombine(ISD::SRL);
setTargetDAGCombine(ISD::MUL);
+ setTargetDAGCombine(ISD::MULHU);
+ setTargetDAGCombine(ISD::MULHS);
setTargetDAGCombine(ISD::SELECT);
setTargetDAGCombine(ISD::SELECT_CC);
setTargetDAGCombine(ISD::STORE);
@@ -1985,7 +1987,7 @@
(VT.getSizeInBits() - DAG.ComputeNumSignBits(Op)) < 24;
}
-static void simplifyI24(SDValue Op, TargetLowering::DAGCombinerInfo &DCI) {
+static bool simplifyI24(SDValue Op, TargetLowering::DAGCombinerInfo &DCI) {
SelectionDAG &DAG = DCI.DAG;
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
@@ -1994,8 +1996,12 @@
APInt Demanded = APInt::getLowBitsSet(VT.getSizeInBits(), 24);
APInt KnownZero, KnownOne;
TargetLowering::TargetLoweringOpt TLO(DAG, true, true);
- if (TLI.SimplifyDemandedBits(Op, Demanded, KnownZero, KnownOne, TLO))
+ if (TLI.SimplifyDemandedBits(Op, Demanded, KnownZero, KnownOne, TLO)) {
DCI.CommitTargetLoweringOpt(TLO);
+ return true;
+ }
+
+ return false;
}
template <typename IntTy>
@@ -2285,11 +2291,36 @@
return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildPair);
}
+// We need to specifically handle i64 mul here to avoid unnecessary conversion
+// instructions. If we only match on the legalized i64 mul expansion,
+// SimplifyDemandedBits will be unable to remove them because there will be
+// multiple uses due to the separate mul + mulh[su].
+static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL,
+ SDValue N0, SDValue N1, unsigned Size, bool Signed) {
+ if (Size <= 32) {
+ unsigned MulOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
+ return DAG.getNode(MulOpc, SL, MVT::i32, N0, N1);
+ }
+
+ // Because we want to eliminate extension instructions before the
+ // operation, we need to create a single user here (i.e. not the separate
+ // mul_lo + mul_hi) so that SimplifyDemandedBits will deal with it.
+
+ unsigned MulOpc = Signed ? AMDGPUISD::MUL_LOHI_I24 : AMDGPUISD::MUL_LOHI_U24;
+
+ SDValue Mul = DAG.getNode(MulOpc, SL,
+ DAG.getVTList(MVT::i32, MVT::i32), N0, N1);
+
+ return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64,
+ Mul.getValue(0), Mul.getValue(1));
+}
+
SDValue AMDGPUTargetLowering::performMulCombine(SDNode *N,
DAGCombinerInfo &DCI) const {
EVT VT = N->getValueType(0);
- if (VT.isVector() || VT.getSizeInBits() > 32)
+ unsigned Size = VT.getSizeInBits();
+ if (VT.isVector() || Size > 64)
return SDValue();
SelectionDAG &DAG = DCI.DAG;
@@ -2302,11 +2333,11 @@
if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) {
N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32);
N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32);
- Mul = DAG.getNode(AMDGPUISD::MUL_U24, DL, MVT::i32, N0, N1);
+ Mul = getMul24(DAG, DL, N0, N1, Size, false);
} else if (Subtarget->hasMulI24() && isI24(N0, DAG) && isI24(N1, DAG)) {
N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32);
N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32);
- Mul = DAG.getNode(AMDGPUISD::MUL_I24, DL, MVT::i32, N0, N1);
+ Mul = getMul24(DAG, DL, N0, N1, Size, true);
} else {
return SDValue();
}
@@ -2316,6 +2347,77 @@
return DAG.getSExtOrTrunc(Mul, DL, VT);
}
+SDValue AMDGPUTargetLowering::performMulhsCombine(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ EVT VT = N->getValueType(0);
+
+ if (!Subtarget->hasMulI24() || VT.isVector())
+ return SDValue();
+
+ SelectionDAG &DAG = DCI.DAG;
+ SDLoc DL(N);
+
+ SDValue N0 = N->getOperand(0);
+ SDValue N1 = N->getOperand(1);
+
+ if (!isI24(N0, DAG) || !isI24(N1, DAG))
+ return SDValue();
+
+ N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32);
+ N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32);
+
+ SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_I24, DL, MVT::i32, N0, N1);
+ DCI.AddToWorklist(Mulhi.getNode());
+ return DAG.getSExtOrTrunc(Mulhi, DL, VT);
+}
+
+SDValue AMDGPUTargetLowering::performMulhuCombine(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ EVT VT = N->getValueType(0);
+
+ if (!Subtarget->hasMulU24() || VT.isVector() || VT.getSizeInBits() > 32)
+ return SDValue();
+
+ SelectionDAG &DAG = DCI.DAG;
+ SDLoc DL(N);
+
+ SDValue N0 = N->getOperand(0);
+ SDValue N1 = N->getOperand(1);
+
+ if (!isU24(N0, DAG) || !isU24(N1, DAG))
+ return SDValue();
+
+ N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32);
+ N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32);
+
+ SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_U24, DL, MVT::i32, N0, N1);
+ DCI.AddToWorklist(Mulhi.getNode());
+ return DAG.getZExtOrTrunc(Mulhi, DL, VT);
+}
+
+SDValue AMDGPUTargetLowering::performMulLoHi24Combine(
+ SDNode *N, DAGCombinerInfo &DCI) const {
+ SelectionDAG &DAG = DCI.DAG;
+
+ SDValue N0 = N->getOperand(0);
+ SDValue N1 = N->getOperand(1);
+
+ // Simplify demanded bits before splitting into multiple users.
+ if (simplifyI24(N0, DCI) || simplifyI24(N1, DCI))
+ return SDValue();
+
+ bool Signed = (N->getOpcode() == AMDGPUISD::MUL_LOHI_I24);
+
+ unsigned MulLoOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
+ unsigned MulHiOpc = Signed ? AMDGPUISD::MULHI_I24 : AMDGPUISD::MULHI_U24;
+
+ SDLoc SL(N);
+
+ SDValue MulLo = DAG.getNode(MulLoOpc, SL, MVT::i32, N0, N1);
+ SDValue MulHi = DAG.getNode(MulHiOpc, SL, MVT::i32, N0, N1);
+ return DAG.getMergeValues({ MulLo, MulHi }, SL);
+}
+
static bool isNegativeOne(SDValue Val) {
if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val))
return C->isAllOnesValue();
@@ -2476,14 +2578,23 @@
}
case ISD::MUL:
return performMulCombine(N, DCI);
+ case ISD::MULHS:
+ return performMulhsCombine(N, DCI);
+ case ISD::MULHU:
+ return performMulhuCombine(N, DCI);
case AMDGPUISD::MUL_I24:
- case AMDGPUISD::MUL_U24: {
+ case AMDGPUISD::MUL_U24:
+ case AMDGPUISD::MULHI_I24:
+ case AMDGPUISD::MULHI_U24: {
SDValue N0 = N->getOperand(0);
SDValue N1 = N->getOperand(1);
simplifyI24(N0, DCI);
simplifyI24(N1, DCI);
return SDValue();
}
+ case AMDGPUISD::MUL_LOHI_I24:
+ case AMDGPUISD::MUL_LOHI_U24:
+ return performMulLoHi24Combine(N, DCI);
case ISD::SELECT:
return performSelectCombine(N, DCI);
case AMDGPUISD::BFE_I32:
@@ -2695,6 +2806,10 @@
NODE_NAME_CASE(FFBH_I32)
NODE_NAME_CASE(MUL_U24)
NODE_NAME_CASE(MUL_I24)
+ NODE_NAME_CASE(MULHI_U24)
+ NODE_NAME_CASE(MULHI_I24)
+ NODE_NAME_CASE(MUL_LOHI_U24)
+ NODE_NAME_CASE(MUL_LOHI_I24)
NODE_NAME_CASE(MAD_U24)
NODE_NAME_CASE(MAD_I24)
NODE_NAME_CASE(TEXTURE_FETCH)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
index 20775df..1aa2d1a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -70,6 +70,9 @@
SDValue performSraCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performSrlCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performMulCombine(SDNode *N, DAGCombinerInfo &DCI) const;
+ SDValue performMulhsCombine(SDNode *N, DAGCombinerInfo &DCI) const;
+ SDValue performMulhuCombine(SDNode *N, DAGCombinerInfo &DCI) const;
+ SDValue performMulLoHi24Combine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performCtlzCombine(const SDLoc &SL, SDValue Cond, SDValue LHS,
SDValue RHS, DAGCombinerInfo &DCI) const;
SDValue performSelectCombine(SDNode *N, DAGCombinerInfo &DCI) const;
@@ -226,9 +229,9 @@
DWORDADDR,
FRACT,
CLAMP,
- // This is SETCC with the full mask result which is used for a compare with a
+ // This is SETCC with the full mask result which is used for a compare with a
// result bit per item in the wavefront.
- SETCC,
+ SETCC,
// SIN_HW, COS_HW - f32 for SI, 1 ULP max error, valid from -100 pi to 100 pi.
// Denormals handled on some parts.
@@ -272,8 +275,12 @@
FFBH_I32,
MUL_U24,
MUL_I24,
+ MULHI_U24,
+ MULHI_I24,
MAD_U24,
MAD_I24,
+ MUL_LOHI_I24,
+ MUL_LOHI_U24,
TEXTURE_FETCH,
EXPORT,
CONST_ADDRESS,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.td b/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.td
index 1547d22..4bccd81 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.td
@@ -216,13 +216,20 @@
def AMDGPUffbh_u32 : SDNode<"AMDGPUISD::FFBH_U32", SDTIntUnaryOp>;
def AMDGPUffbh_i32 : SDNode<"AMDGPUISD::FFBH_I32", SDTIntUnaryOp>;
-// Signed and unsigned 24-bit mulitply. The highest 8-bits are ignore when
-// performing the mulitply. The result is a 32-bit value.
+// Signed and unsigned 24-bit multiply. The highest 8-bits are ignore
+// when performing the mulitply. The result is a 32-bit value.
def AMDGPUmul_u24 : SDNode<"AMDGPUISD::MUL_U24", SDTIntBinOp,
- [SDNPCommutative]
+ [SDNPCommutative, SDNPAssociative]
>;
def AMDGPUmul_i24 : SDNode<"AMDGPUISD::MUL_I24", SDTIntBinOp,
- [SDNPCommutative]
+ [SDNPCommutative, SDNPAssociative]
+>;
+
+def AMDGPUmulhi_u24 : SDNode<"AMDGPUISD::MULHI_U24", SDTIntBinOp,
+ [SDNPCommutative, SDNPAssociative]
+>;
+def AMDGPUmulhi_i24 : SDNode<"AMDGPUISD::MULHI_I24", SDTIntBinOp,
+ [SDNPCommutative, SDNPAssociative]
>;
def AMDGPUmad_u24 : SDNode<"AMDGPUISD::MAD_U24", AMDGPUDTIntTernaryOp,
diff --git a/llvm/lib/Target/AMDGPU/CaymanInstructions.td b/llvm/lib/Target/AMDGPU/CaymanInstructions.td
index a38a3bb..6b8e85a 100644
--- a/llvm/lib/Target/AMDGPU/CaymanInstructions.td
+++ b/llvm/lib/Target/AMDGPU/CaymanInstructions.td
@@ -37,6 +37,9 @@
def MULHI_INT_cm : MULHI_INT_Common<0x90>;
def MULLO_UINT_cm : MULLO_UINT_Common<0x91>;
def MULHI_UINT_cm : MULHI_UINT_Common<0x92>;
+def MULHI_INT_cm24 : MULHI_INT24_Common<0x5c>;
+def MULHI_UINT_cm24 : MULHI_UINT24_Common<0xb2>;
+
def RECIPSQRT_CLAMPED_cm : RECIPSQRT_CLAMPED_Common<0x87>;
def EXP_IEEE_cm : EXP_IEEE_Common<0x81>;
def LOG_IEEE_cm : LOG_IEEE_Common<0x83>;
diff --git a/llvm/lib/Target/AMDGPU/EvergreenInstructions.td b/llvm/lib/Target/AMDGPU/EvergreenInstructions.td
index 62de72a..4112ad1 100644
--- a/llvm/lib/Target/AMDGPU/EvergreenInstructions.td
+++ b/llvm/lib/Target/AMDGPU/EvergreenInstructions.td
@@ -72,6 +72,8 @@
def MULHI_INT_eg : MULHI_INT_Common<0x90>;
def MULLO_UINT_eg : MULLO_UINT_Common<0x91>;
def MULHI_UINT_eg : MULHI_UINT_Common<0x92>;
+def MULHI_UINT24_eg : MULHI_UINT24_Common<0xb2>;
+
def RECIP_UINT_eg : RECIP_UINT_Common<0x94>;
def RECIPSQRT_CLAMPED_eg : RECIPSQRT_CLAMPED_Common<0x87>;
def EXP_IEEE_eg : EXP_IEEE_Common<0x81>;
diff --git a/llvm/lib/Target/AMDGPU/R600Instructions.td b/llvm/lib/Target/AMDGPU/R600Instructions.td
index 40424ca..cf457bd 100644
--- a/llvm/lib/Target/AMDGPU/R600Instructions.td
+++ b/llvm/lib/Target/AMDGPU/R600Instructions.td
@@ -1073,18 +1073,27 @@
class LSHR_Common <bits<11> inst> : R600_2OP_Helper <inst, "LSHR", srl>;
class ASHR_Common <bits<11> inst> : R600_2OP_Helper <inst, "ASHR", sra>;
class MULHI_INT_Common <bits<11> inst> : R600_2OP_Helper <
- inst, "MULHI_INT", mulhs
-> {
+ inst, "MULHI_INT", mulhs> {
let Itinerary = TransALU;
}
+
+class MULHI_INT24_Common <bits<11> inst> : R600_2OP_Helper <
+ inst, "MULHI_INT24", AMDGPUmulhi_i24> {
+ let Itinerary = VecALU;
+}
+
class MULHI_UINT_Common <bits<11> inst> : R600_2OP_Helper <
- inst, "MULHI", mulhu
-> {
+ inst, "MULHI", mulhu> {
let Itinerary = TransALU;
}
+
+class MULHI_UINT24_Common <bits<11> inst> : R600_2OP_Helper <
+ inst, "MULHI_UINT24", AMDGPUmulhi_u24> {
+ let Itinerary = VecALU;
+}
+
class MULLO_INT_Common <bits<11> inst> : R600_2OP_Helper <
- inst, "MULLO_INT", mul
-> {
+ inst, "MULLO_INT", mul> {
let Itinerary = TransALU;
}
class MULLO_UINT_Common <bits<11> inst> : R600_2OP <inst, "MULLO_UINT", []> {
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index e02e2ae..70fdc5d 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -1362,7 +1362,7 @@
>;
defm V_MUL_HI_I32_I24 : VOP2Inst <vop2<0xa,0x7>, "v_mul_hi_i32_i24",
- VOP_I32_I32_I32
+ VOP_I32_I32_I32, AMDGPUmulhi_i24
>;
defm V_MUL_U32_U24 : VOP2Inst <vop2<0xb, 0x8>, "v_mul_u32_u24",
@@ -1370,7 +1370,7 @@
>;
defm V_MUL_HI_U32_U24 : VOP2Inst <vop2<0xc,0x9>, "v_mul_hi_u32_u24",
- VOP_I32_I32_I32
+ VOP_I32_I32_I32, AMDGPUmulhi_u24
>;
defm V_MIN_F32 : VOP2Inst <vop2<0xf, 0xa>, "v_min_f32", VOP_F32_F32_F32,