[FPEnv] Constrained FCmp intrinsics
This adds support for constrained floating-point comparison intrinsics.
Specifically, we add:
declare <ty2>
@llvm.experimental.constrained.fcmp(<type> <op1>, <type> <op2>,
metadata <condition code>,
metadata <exception behavior>)
declare <ty2>
@llvm.experimental.constrained.fcmps(<type> <op1>, <type> <op2>,
metadata <condition code>,
metadata <exception behavior>)
The first variant implements an IEEE "quiet" comparison (i.e. we only
get an invalid FP exception if either argument is a SNaN), while the
second variant implements an IEEE "signaling" comparison (i.e. we get
an invalid FP exception if either argument is any NaN).
The condition code is implemented as a metadata string. The same set
of predicates as for the fcmp instruction is supported (except for the
"true" and "false" predicates).
These new intrinsics are mapped by SelectionDAG codegen onto two new
ISD opcodes, ISD::STRICT_FSETCC and ISD::STRICT_FSETCCS, again
representing quiet vs. signaling comparison operations. Otherwise
those nodes look like SETCC nodes, with an additional chain argument
and result as usual for strict FP nodes. The patch includes support
for the common legalization operations for those nodes.
The patch also includes full SystemZ back-end support for the new
ISD nodes, mapping them to all available SystemZ instruction to
fully implement strict semantics (scalar and vector).
Differential Revision: https://reviews.llvm.org/D69281
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 70cb20e..b3860dd 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -1036,11 +1036,17 @@
Node->getOperand(2).getValueType());
break;
case ISD::SELECT_CC:
+ case ISD::STRICT_FSETCC:
+ case ISD::STRICT_FSETCCS:
case ISD::SETCC:
case ISD::BR_CC: {
unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 :
+ Node->getOpcode() == ISD::STRICT_FSETCC ? 3 :
+ Node->getOpcode() == ISD::STRICT_FSETCCS ? 3 :
Node->getOpcode() == ISD::SETCC ? 2 : 1;
- unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0;
+ unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 :
+ Node->getOpcode() == ISD::STRICT_FSETCC ? 1 :
+ Node->getOpcode() == ISD::STRICT_FSETCCS ? 1 : 0;
MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
ISD::CondCode CCCode =
cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index dd08264..9e4e5ad 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -75,6 +75,8 @@
case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break;
case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break;
case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
+ case ISD::STRICT_FSETCC:
+ case ISD::STRICT_FSETCCS:
case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
case ISD::SMIN:
case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break;
@@ -817,7 +819,8 @@
}
SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
- EVT InVT = N->getOperand(0).getValueType();
+ unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
+ EVT InVT = N->getOperand(OpNo).getValueType();
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
EVT SVT = getSetCCResultType(InVT);
@@ -836,12 +839,22 @@
}
SDLoc dl(N);
- assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() &&
+ assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() &&
"Vector compare must return a vector result!");
// Get the SETCC result using the canonical SETCC type.
- SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
- N->getOperand(1), N->getOperand(2));
+ SDValue SetCC;
+ if (N->isStrictFPOpcode()) {
+ EVT VTs[] = {SVT, MVT::Other};
+ SDValue Opers[] = {N->getOperand(0), N->getOperand(1),
+ N->getOperand(2), N->getOperand(3)};
+ SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers);
+ // Legalize the chain result - switch anything that used the old chain to
+ // use the new one.
+ ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1));
+ } else
+ SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
+ N->getOperand(1), N->getOperand(2));
// Convert to the expected type.
return DAG.getSExtOrTrunc(SetCC, dl, NVT);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 2770d51..1ef6292 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -1323,7 +1323,14 @@
unsigned NumElems = VT.getVectorNumElements();
unsigned NumOpers = Op.getNumOperands();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
- EVT ValueVTs[] = {EltVT, MVT::Other};
+
+ EVT TmpEltVT = EltVT;
+ if (Op->getOpcode() == ISD::STRICT_FSETCC ||
+ Op->getOpcode() == ISD::STRICT_FSETCCS)
+ TmpEltVT = TLI.getSetCCResultType(DAG.getDataLayout(),
+ *DAG.getContext(), TmpEltVT);
+
+ EVT ValueVTs[] = {TmpEltVT, MVT::Other};
SDValue Chain = Op.getOperand(0);
SDLoc dl(Op);
@@ -1350,9 +1357,18 @@
}
SDValue ScalarOp = DAG.getNode(Op->getOpcode(), dl, ValueVTs, Opers);
+ SDValue ScalarResult = ScalarOp.getValue(0);
+ SDValue ScalarChain = ScalarOp.getValue(1);
- OpValues.push_back(ScalarOp.getValue(0));
- OpChains.push_back(ScalarOp.getValue(1));
+ if (Op->getOpcode() == ISD::STRICT_FSETCC ||
+ Op->getOpcode() == ISD::STRICT_FSETCCS)
+ ScalarResult = DAG.getSelect(dl, EltVT, ScalarResult,
+ DAG.getConstant(APInt::getAllOnesValue
+ (EltVT.getSizeInBits()), dl, EltVT),
+ DAG.getConstant(0, dl, EltVT));
+
+ OpValues.push_back(ScalarResult);
+ OpChains.push_back(ScalarChain);
}
SDValue Result = DAG.getBuildVector(VT, dl, OpValues);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 9403b344e..4090ee5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -3758,6 +3758,17 @@
WidenVT, N->getOperand(0));
}
+// Return true is this is a SETCC node or a strict version of it.
+static inline bool isSETCCOp(unsigned Opcode) {
+ switch (Opcode) {
+ case ISD::SETCC:
+ case ISD::STRICT_FSETCC:
+ case ISD::STRICT_FSETCCS:
+ return true;
+ }
+ return false;
+}
+
// Return true if this is a node that could have two SETCCs as operands.
static inline bool isLogicalMaskOp(unsigned Opcode) {
switch (Opcode) {
@@ -3769,6 +3780,13 @@
return false;
}
+// If N is a SETCC or a strict variant of it, return the type
+// of the compare operands.
+static inline EVT getSETCCOperandType(SDValue N) {
+ unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
+ return N->getOperand(OpNo).getValueType();
+}
+
// This is used just for the assert in convertMask(). Check that this either
// a SETCC or a previously handled SETCC by convertMask().
#ifndef NDEBUG
@@ -3791,7 +3809,7 @@
return isSETCCorConvertedSETCC(N.getOperand(0)) &&
isSETCCorConvertedSETCC(N.getOperand(1));
- return (N.getOpcode() == ISD::SETCC ||
+ return (isSETCCOp(N.getOpcode()) ||
ISD::isBuildVectorOfConstantSDNodes(N.getNode()));
}
#endif
@@ -3806,10 +3824,17 @@
assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
// Make a new Mask node, with a legal result VT.
+ SDValue Mask;
SmallVector<SDValue, 4> Ops;
for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
Ops.push_back(InMask->getOperand(i));
- SDValue Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops);
+ if (InMask->isStrictFPOpcode()) {
+ Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask),
+ { MaskVT, MVT::Other }, Ops);
+ ReplaceValueWith(InMask.getValue(1), Mask.getValue(1));
+ }
+ else
+ Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops);
// If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
// extend or truncate is needed.
@@ -3862,7 +3887,7 @@
if (N->getOpcode() != ISD::VSELECT)
return SDValue();
- if (Cond->getOpcode() != ISD::SETCC && !isLogicalMaskOp(Cond->getOpcode()))
+ if (!isSETCCOp(Cond->getOpcode()) && !isLogicalMaskOp(Cond->getOpcode()))
return SDValue();
// If this is a splitted VSELECT that was previously already handled, do
@@ -3885,8 +3910,8 @@
return SDValue();
// If there is support for an i1 vector mask, don't touch.
- if (Cond.getOpcode() == ISD::SETCC) {
- EVT SetCCOpVT = Cond->getOperand(0).getValueType();
+ if (isSETCCOp(Cond.getOpcode())) {
+ EVT SetCCOpVT = getSETCCOperandType(Cond);
while (TLI.getTypeAction(Ctx, SetCCOpVT) != TargetLowering::TypeLegal)
SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT);
EVT SetCCResVT = getSetCCResultType(SetCCOpVT);
@@ -3917,17 +3942,17 @@
ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
SDValue Mask;
- if (Cond->getOpcode() == ISD::SETCC) {
- EVT MaskVT = getSetCCResultType(Cond.getOperand(0).getValueType());
+ if (isSETCCOp(Cond->getOpcode())) {
+ EVT MaskVT = getSetCCResultType(getSETCCOperandType(Cond));
Mask = convertMask(Cond, MaskVT, ToMaskVT);
} else if (isLogicalMaskOp(Cond->getOpcode()) &&
- Cond->getOperand(0).getOpcode() == ISD::SETCC &&
- Cond->getOperand(1).getOpcode() == ISD::SETCC) {
+ isSETCCOp(Cond->getOperand(0).getOpcode()) &&
+ isSETCCOp(Cond->getOperand(1).getOpcode())) {
// Cond is (AND/OR/XOR (SETCC, SETCC))
SDValue SETCC0 = Cond->getOperand(0);
SDValue SETCC1 = Cond->getOperand(1);
- EVT VT0 = getSetCCResultType(SETCC0.getOperand(0).getValueType());
- EVT VT1 = getSetCCResultType(SETCC1.getOperand(0).getValueType());
+ EVT VT0 = getSetCCResultType(getSETCCOperandType(SETCC0));
+ EVT VT1 = getSetCCResultType(getSETCCOperandType(SETCC1));
unsigned ScalarBits0 = VT0.getScalarSizeInBits();
unsigned ScalarBits1 = VT1.getScalarSizeInBits();
unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index c1c599c..9ca51e7 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -2798,12 +2798,16 @@
Known.Zero.setBitsFrom(1);
break;
case ISD::SETCC:
+ case ISD::STRICT_FSETCC:
+ case ISD::STRICT_FSETCCS: {
+ unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
// If we know the result of a setcc has the top bits zero, use this info.
- if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
+ if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) ==
TargetLowering::ZeroOrOneBooleanContent &&
BitWidth > 1)
Known.Zero.setBitsFrom(1);
break;
+ }
case ISD::SHL:
if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) {
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
@@ -3662,11 +3666,15 @@
return VTBits;
break;
case ISD::SETCC:
+ case ISD::STRICT_FSETCC:
+ case ISD::STRICT_FSETCCS: {
+ unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
// If setcc returns 0/-1, all bits are sign bits.
- if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
+ if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) ==
TargetLowering::ZeroOrNegativeOneBooleanContent)
return VTBits;
break;
+ }
case ISD::ROTL:
case ISD::ROTR:
if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
@@ -7833,6 +7841,8 @@
llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");
#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break;
+#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
+ case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break;
#include "llvm/IR/ConstrainedOps.def"
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 9952d4d..de4690d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6915,9 +6915,21 @@
#include "llvm/IR/ConstrainedOps.def"
}
- if (Opcode == ISD::STRICT_FP_ROUND)
+ // A few strict DAG nodes carry additional operands that are not
+ // set up by the default code above.
+ switch (Opcode) {
+ default: break;
+ case ISD::STRICT_FP_ROUND:
Opers.push_back(
DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())));
+ break;
+ case ISD::STRICT_FSETCC:
+ case ISD::STRICT_FSETCCS: {
+ auto *FPCmp = dyn_cast<ConstrainedFPCmpIntrinsic>(&FPI);
+ Opers.push_back(DAG.getCondCode(getFCmpCondCode(FPCmp->getPredicate())));
+ break;
+ }
+ }
SDVTList VTs = DAG.getVTList(ValueVTs);
SDValue Result = DAG.getNode(Opcode, sdl, VTs, Opers);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index f863d98..8312a9a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -270,6 +270,8 @@
case ISD::STRICT_FPOWI: return "strict_fpowi";
case ISD::SETCC: return "setcc";
case ISD::SETCCCARRY: return "setcccarry";
+ case ISD::STRICT_FSETCC: return "strict_fsetcc";
+ case ISD::STRICT_FSETCCS: return "strict_fsetccs";
case ISD::SELECT: return "select";
case ISD::VSELECT: return "vselect";
case ISD::SELECT_CC: return "select_cc";
diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp
index 12b1bf1..b23742b 100644
--- a/llvm/lib/IR/IntrinsicInst.cpp
+++ b/llvm/lib/IR/IntrinsicInst.cpp
@@ -121,6 +121,30 @@
return StrToExceptionBehavior(cast<MDString>(MD)->getString());
}
+FCmpInst::Predicate
+ConstrainedFPCmpIntrinsic::getPredicate() const {
+ Metadata *MD =
+ cast<MetadataAsValue>(getArgOperand(2))->getMetadata();
+ if (!MD || !isa<MDString>(MD))
+ return FCmpInst::BAD_FCMP_PREDICATE;
+ return StringSwitch<FCmpInst::Predicate>(cast<MDString>(MD)->getString())
+ .Case("oeq", FCmpInst::FCMP_OEQ)
+ .Case("ogt", FCmpInst::FCMP_OGT)
+ .Case("oge", FCmpInst::FCMP_OGE)
+ .Case("olt", FCmpInst::FCMP_OLT)
+ .Case("ole", FCmpInst::FCMP_OLE)
+ .Case("one", FCmpInst::FCMP_ONE)
+ .Case("ord", FCmpInst::FCMP_ORD)
+ .Case("uno", FCmpInst::FCMP_UNO)
+ .Case("ueq", FCmpInst::FCMP_UEQ)
+ .Case("ugt", FCmpInst::FCMP_UGT)
+ .Case("uge", FCmpInst::FCMP_UGE)
+ .Case("ult", FCmpInst::FCMP_ULT)
+ .Case("ule", FCmpInst::FCMP_ULE)
+ .Case("une", FCmpInst::FCMP_UNE)
+ .Default(FCmpInst::BAD_FCMP_PREDICATE);
+}
+
bool ConstrainedFPIntrinsic::isUnaryOp() const {
switch (getIntrinsicID()) {
default:
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index b32b289..876865e 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -4740,6 +4740,9 @@
llvm_unreachable("Invalid constrained FP intrinsic!");
}
NumOperands += (1 + HasRoundingMD);
+ // Compare intrinsics carry an extra predicate metadata operand.
+ if (isa<ConstrainedFPCmpIntrinsic>(FPI))
+ NumOperands += 1;
Assert((FPI.getNumArgOperands() == NumOperands),
"invalid arguments for constrained FP intrinsic", &FPI);
@@ -4762,6 +4765,14 @@
break;
}
+ case Intrinsic::experimental_constrained_fcmp:
+ case Intrinsic::experimental_constrained_fcmps: {
+ auto Pred = dyn_cast<ConstrainedFPCmpIntrinsic>(&FPI)->getPredicate();
+ Assert(CmpInst::isFPPredicate(Pred),
+ "invalid predicate for constrained FP comparison intrinsic", &FPI);
+ break;
+ }
+
case Intrinsic::experimental_constrained_fptosi:
case Intrinsic::experimental_constrained_fptoui: {
Value *Operand = FPI.getArgOperand(0);
diff --git a/llvm/lib/Target/SystemZ/SystemZElimCompare.cpp b/llvm/lib/Target/SystemZ/SystemZElimCompare.cpp
index 90f664a..8b12808 100644
--- a/llvm/lib/Target/SystemZ/SystemZElimCompare.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZElimCompare.cpp
@@ -295,6 +295,11 @@
MIB.setMemRefs(MI.memoperands());
MI.eraseFromParent();
+ // Mark instruction as raising an FP exception if applicable. We already
+ // verified earlier that this move is valid.
+ if (Compare.mayRaiseFPException())
+ MIB.setMIFlag(MachineInstr::MIFlag::FPExcept);
+
return true;
}
@@ -312,6 +317,18 @@
const MCInstrDesc &Desc = TII->get(Opcode);
unsigned MIFlags = Desc.TSFlags;
+ // If Compare may raise an FP exception, we can only eliminate it
+ // if MI itself would have already raised the exception.
+ if (Compare.mayRaiseFPException()) {
+ // If the caller will change MI to use ConvOpc, only test whether
+ // ConvOpc is suitable; it is on the caller to set the MI flag.
+ if (ConvOpc && !Desc.mayRaiseFPException())
+ return false;
+ // If the caller will not change MI, we test the MI flag here.
+ if (!ConvOpc && !MI.mayRaiseFPException())
+ return false;
+ }
+
// See which compare-style condition codes are available.
unsigned ReusableCCMask = SystemZII::getCompareZeroCCMask(MIFlags);
@@ -454,6 +471,12 @@
CCRefs |= getRegReferences(MI, SystemZ::CC);
if (CCRefs.Use && CCRefs.Def)
break;
+ // Eliminating a Compare that may raise an FP exception will move
+ // raising the exception to some earlier MI. We cannot do this if
+ // there is anything in between that might change exception flags.
+ if (Compare.mayRaiseFPException() &&
+ (MI.isCall() || MI.hasUnmodeledSideEffects()))
+ break;
}
// Also do a forward search to handle cases where an instruction after the
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 7994176..0e58b52 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -32,12 +32,16 @@
namespace {
// Represents information about a comparison.
struct Comparison {
- Comparison(SDValue Op0In, SDValue Op1In)
- : Op0(Op0In), Op1(Op1In), Opcode(0), ICmpType(0), CCValid(0), CCMask(0) {}
+ Comparison(SDValue Op0In, SDValue Op1In, SDValue ChainIn)
+ : Op0(Op0In), Op1(Op1In), Chain(ChainIn),
+ Opcode(0), ICmpType(0), CCValid(0), CCMask(0) {}
// The operands to the comparison.
SDValue Op0, Op1;
+ // Chain if this is a strict floating-point comparison.
+ SDValue Chain;
+
// The opcode that should be used to compare Op0 and Op1.
unsigned Opcode;
@@ -132,6 +136,8 @@
if (isTypeLegal(VT)) {
// Lower SET_CC into an IPM-based sequence.
setOperationAction(ISD::SETCC, VT, Custom);
+ setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
+ setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
// Expand SELECT(C, A, B) into SELECT_CC(X, 0, A, B, NE).
setOperationAction(ISD::SELECT, VT, Expand);
@@ -373,6 +379,9 @@
// Map SETCCs onto one of VCE, VCH or VCHL, swapping the operands
// and inverting the result as necessary.
setOperationAction(ISD::SETCC, VT, Custom);
+ setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
+ if (Subtarget.hasVectorEnhancements1())
+ setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
}
}
@@ -2168,6 +2177,10 @@
// negation to set CC, so avoiding separate LOAD AND TEST and
// LOAD (NEGATIVE/COMPLEMENT) instructions.
static void adjustForFNeg(Comparison &C) {
+ // This optimization is invalid for strict comparisons, since FNEG
+ // does not raise any exceptions.
+ if (C.Chain)
+ return;
auto *C1 = dyn_cast<ConstantFPSDNode>(C.Op1);
if (C1 && C1->isZero()) {
for (auto I = C.Op0->use_begin(), E = C.Op0->use_end(); I != E; ++I) {
@@ -2455,7 +2468,7 @@
static Comparison getIntrinsicCmp(SelectionDAG &DAG, unsigned Opcode,
SDValue Call, unsigned CCValid, uint64_t CC,
ISD::CondCode Cond) {
- Comparison C(Call, SDValue());
+ Comparison C(Call, SDValue(), SDValue());
C.Opcode = Opcode;
C.CCValid = CCValid;
if (Cond == ISD::SETEQ)
@@ -2486,8 +2499,11 @@
// Decide how to implement a comparison of type Cond between CmpOp0 with CmpOp1.
static Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1,
- ISD::CondCode Cond, const SDLoc &DL) {
+ ISD::CondCode Cond, const SDLoc &DL,
+ SDValue Chain = SDValue(),
+ bool IsSignaling = false) {
if (CmpOp1.getOpcode() == ISD::Constant) {
+ assert(!Chain);
uint64_t Constant = cast<ConstantSDNode>(CmpOp1)->getZExtValue();
unsigned Opcode, CCValid;
if (CmpOp0.getOpcode() == ISD::INTRINSIC_W_CHAIN &&
@@ -2499,13 +2515,19 @@
isIntrinsicWithCC(CmpOp0, Opcode, CCValid))
return getIntrinsicCmp(DAG, Opcode, CmpOp0, CCValid, Constant, Cond);
}
- Comparison C(CmpOp0, CmpOp1);
+ Comparison C(CmpOp0, CmpOp1, Chain);
C.CCMask = CCMaskForCondCode(Cond);
if (C.Op0.getValueType().isFloatingPoint()) {
C.CCValid = SystemZ::CCMASK_FCMP;
- C.Opcode = SystemZISD::FCMP;
+ if (!C.Chain)
+ C.Opcode = SystemZISD::FCMP;
+ else if (!IsSignaling)
+ C.Opcode = SystemZISD::STRICT_FCMP;
+ else
+ C.Opcode = SystemZISD::STRICT_FCMPS;
adjustForFNeg(C);
} else {
+ assert(!C.Chain);
C.CCValid = SystemZ::CCMASK_ICMP;
C.Opcode = SystemZISD::ICMP;
// Choose the type of comparison. Equality and inequality tests can
@@ -2563,6 +2585,10 @@
return DAG.getNode(SystemZISD::TM, DL, MVT::i32, C.Op0, C.Op1,
DAG.getTargetConstant(RegisterOnly, DL, MVT::i32));
}
+ if (C.Chain) {
+ SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
+ return DAG.getNode(C.Opcode, DL, VTs, C.Chain, C.Op0, C.Op1);
+ }
return DAG.getNode(C.Opcode, DL, MVT::i32, C.Op0, C.Op1);
}
@@ -2607,24 +2633,51 @@
}
// Return the SystemISD vector comparison operation for CC, or 0 if it cannot
-// be done directly. IsFP is true if CC is for a floating-point rather than
-// integer comparison.
-static unsigned getVectorComparison(ISD::CondCode CC, bool IsFP) {
+// be done directly. Mode is CmpMode::Int for integer comparisons, CmpMode::FP
+// for regular floating-point comparisons, CmpMode::StrictFP for strict (quiet)
+// floating-point comparisons, and CmpMode::SignalingFP for strict signaling
+// floating-point comparisons.
+enum class CmpMode { Int, FP, StrictFP, SignalingFP };
+static unsigned getVectorComparison(ISD::CondCode CC, CmpMode Mode) {
switch (CC) {
case ISD::SETOEQ:
case ISD::SETEQ:
- return IsFP ? SystemZISD::VFCMPE : SystemZISD::VICMPE;
+ switch (Mode) {
+ case CmpMode::Int: return SystemZISD::VICMPE;
+ case CmpMode::FP: return SystemZISD::VFCMPE;
+ case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPE;
+ case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPES;
+ default: llvm_unreachable("Bad mode");
+ }
case ISD::SETOGE:
case ISD::SETGE:
- return IsFP ? SystemZISD::VFCMPHE : static_cast<SystemZISD::NodeType>(0);
+ switch (Mode) {
+ case CmpMode::Int: return 0;
+ case CmpMode::FP: return SystemZISD::VFCMPHE;
+ case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPHE;
+ case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPHES;
+ default: llvm_unreachable("Bad mode");
+ }
case ISD::SETOGT:
case ISD::SETGT:
- return IsFP ? SystemZISD::VFCMPH : SystemZISD::VICMPH;
+ switch (Mode) {
+ case CmpMode::Int: return SystemZISD::VICMPH;
+ case CmpMode::FP: return SystemZISD::VFCMPH;
+ case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPH;
+ case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPHS;
+ default: llvm_unreachable("Bad mode");
+ }
case ISD::SETUGT:
- return IsFP ? static_cast<SystemZISD::NodeType>(0) : SystemZISD::VICMPHL;
+ switch (Mode) {
+ case CmpMode::Int: return SystemZISD::VICMPHL;
+ case CmpMode::FP: return 0;
+ case CmpMode::StrictFP: return 0;
+ case CmpMode::SignalingFP: return 0;
+ default: llvm_unreachable("Bad mode");
+ }
default:
return 0;
@@ -2633,17 +2686,16 @@
// Return the SystemZISD vector comparison operation for CC or its inverse,
// or 0 if neither can be done directly. Indicate in Invert whether the
-// result is for the inverse of CC. IsFP is true if CC is for a
-// floating-point rather than integer comparison.
-static unsigned getVectorComparisonOrInvert(ISD::CondCode CC, bool IsFP,
+// result is for the inverse of CC. Mode is as above.
+static unsigned getVectorComparisonOrInvert(ISD::CondCode CC, CmpMode Mode,
bool &Invert) {
- if (unsigned Opcode = getVectorComparison(CC, IsFP)) {
+ if (unsigned Opcode = getVectorComparison(CC, Mode)) {
Invert = false;
return Opcode;
}
- CC = ISD::getSetCCInverse(CC, !IsFP);
- if (unsigned Opcode = getVectorComparison(CC, IsFP)) {
+ CC = ISD::getSetCCInverse(CC, Mode == CmpMode::Int);
+ if (unsigned Opcode = getVectorComparison(CC, Mode)) {
Invert = true;
return Opcode;
}
@@ -2652,44 +2704,73 @@
}
// Return a v2f64 that contains the extended form of elements Start and Start+1
-// of v4f32 value Op.
+// of v4f32 value Op. If Chain is nonnull, return the strict form.
static SDValue expandV4F32ToV2F64(SelectionDAG &DAG, int Start, const SDLoc &DL,
- SDValue Op) {
+ SDValue Op, SDValue Chain) {
int Mask[] = { Start, -1, Start + 1, -1 };
Op = DAG.getVectorShuffle(MVT::v4f32, DL, Op, DAG.getUNDEF(MVT::v4f32), Mask);
+ if (Chain) {
+ SDVTList VTs = DAG.getVTList(MVT::v2f64, MVT::Other);
+ return DAG.getNode(SystemZISD::STRICT_VEXTEND, DL, VTs, Chain, Op);
+ }
return DAG.getNode(SystemZISD::VEXTEND, DL, MVT::v2f64, Op);
}
// Build a comparison of vectors CmpOp0 and CmpOp1 using opcode Opcode,
-// producing a result of type VT.
+// producing a result of type VT. If Chain is nonnull, return the strict form.
SDValue SystemZTargetLowering::getVectorCmp(SelectionDAG &DAG, unsigned Opcode,
const SDLoc &DL, EVT VT,
SDValue CmpOp0,
- SDValue CmpOp1) const {
+ SDValue CmpOp1,
+ SDValue Chain) const {
// There is no hardware support for v4f32 (unless we have the vector
// enhancements facility 1), so extend the vector into two v2f64s
// and compare those.
if (CmpOp0.getValueType() == MVT::v4f32 &&
!Subtarget.hasVectorEnhancements1()) {
- SDValue H0 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp0);
- SDValue L0 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp0);
- SDValue H1 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp1);
- SDValue L1 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp1);
+ SDValue H0 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp0, Chain);
+ SDValue L0 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp0, Chain);
+ SDValue H1 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp1, Chain);
+ SDValue L1 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp1, Chain);
+ if (Chain) {
+ SDVTList VTs = DAG.getVTList(MVT::v2i64, MVT::Other);
+ SDValue HRes = DAG.getNode(Opcode, DL, VTs, Chain, H0, H1);
+ SDValue LRes = DAG.getNode(Opcode, DL, VTs, Chain, L0, L1);
+ SDValue Res = DAG.getNode(SystemZISD::PACK, DL, VT, HRes, LRes);
+ SDValue Chains[6] = { H0.getValue(1), L0.getValue(1),
+ H1.getValue(1), L1.getValue(1),
+ HRes.getValue(1), LRes.getValue(1) };
+ SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
+ SDValue Ops[2] = { Res, NewChain };
+ return DAG.getMergeValues(Ops, DL);
+ }
SDValue HRes = DAG.getNode(Opcode, DL, MVT::v2i64, H0, H1);
SDValue LRes = DAG.getNode(Opcode, DL, MVT::v2i64, L0, L1);
return DAG.getNode(SystemZISD::PACK, DL, VT, HRes, LRes);
}
+ if (Chain) {
+ SDVTList VTs = DAG.getVTList(VT, MVT::Other);
+ return DAG.getNode(Opcode, DL, VTs, Chain, CmpOp0, CmpOp1);
+ }
return DAG.getNode(Opcode, DL, VT, CmpOp0, CmpOp1);
}
// Lower a vector comparison of type CC between CmpOp0 and CmpOp1, producing
-// an integer mask of type VT.
+// an integer mask of type VT. If Chain is nonnull, we have a strict
+// floating-point comparison. If in addition IsSignaling is true, we have
+// a strict signaling floating-point comparison.
SDValue SystemZTargetLowering::lowerVectorSETCC(SelectionDAG &DAG,
const SDLoc &DL, EVT VT,
ISD::CondCode CC,
SDValue CmpOp0,
- SDValue CmpOp1) const {
+ SDValue CmpOp1,
+ SDValue Chain,
+ bool IsSignaling) const {
bool IsFP = CmpOp0.getValueType().isFloatingPoint();
+ assert (!Chain || IsFP);
+ assert (!IsSignaling || Chain);
+ CmpMode Mode = IsSignaling ? CmpMode::SignalingFP :
+ Chain ? CmpMode::StrictFP : IsFP ? CmpMode::FP : CmpMode::Int;
bool Invert = false;
SDValue Cmp;
switch (CC) {
@@ -2699,9 +2780,14 @@
LLVM_FALLTHROUGH;
case ISD::SETO: {
assert(IsFP && "Unexpected integer comparison");
- SDValue LT = getVectorCmp(DAG, SystemZISD::VFCMPH, DL, VT, CmpOp1, CmpOp0);
- SDValue GE = getVectorCmp(DAG, SystemZISD::VFCMPHE, DL, VT, CmpOp0, CmpOp1);
+ SDValue LT = getVectorCmp(DAG, getVectorComparison(ISD::SETOGT, Mode),
+ DL, VT, CmpOp1, CmpOp0, Chain);
+ SDValue GE = getVectorCmp(DAG, getVectorComparison(ISD::SETOGE, Mode),
+ DL, VT, CmpOp0, CmpOp1, Chain);
Cmp = DAG.getNode(ISD::OR, DL, VT, LT, GE);
+ if (Chain)
+ Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
+ LT.getValue(1), GE.getValue(1));
break;
}
@@ -2711,9 +2797,14 @@
LLVM_FALLTHROUGH;
case ISD::SETONE: {
assert(IsFP && "Unexpected integer comparison");
- SDValue LT = getVectorCmp(DAG, SystemZISD::VFCMPH, DL, VT, CmpOp1, CmpOp0);
- SDValue GT = getVectorCmp(DAG, SystemZISD::VFCMPH, DL, VT, CmpOp0, CmpOp1);
+ SDValue LT = getVectorCmp(DAG, getVectorComparison(ISD::SETOGT, Mode),
+ DL, VT, CmpOp1, CmpOp0, Chain);
+ SDValue GT = getVectorCmp(DAG, getVectorComparison(ISD::SETOGT, Mode),
+ DL, VT, CmpOp0, CmpOp1, Chain);
Cmp = DAG.getNode(ISD::OR, DL, VT, LT, GT);
+ if (Chain)
+ Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
+ LT.getValue(1), GT.getValue(1));
break;
}
@@ -2721,15 +2812,17 @@
// matter whether we try the inversion or the swap first, since
// there are no cases where both work.
default:
- if (unsigned Opcode = getVectorComparisonOrInvert(CC, IsFP, Invert))
- Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp0, CmpOp1);
+ if (unsigned Opcode = getVectorComparisonOrInvert(CC, Mode, Invert))
+ Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp0, CmpOp1, Chain);
else {
CC = ISD::getSetCCSwappedOperands(CC);
- if (unsigned Opcode = getVectorComparisonOrInvert(CC, IsFP, Invert))
- Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp1, CmpOp0);
+ if (unsigned Opcode = getVectorComparisonOrInvert(CC, Mode, Invert))
+ Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp1, CmpOp0, Chain);
else
llvm_unreachable("Unhandled comparison");
}
+ if (Chain)
+ Chain = Cmp.getValue(1);
break;
}
if (Invert) {
@@ -2737,6 +2830,10 @@
DAG.getSplatBuildVector(VT, DL, DAG.getConstant(-1, DL, MVT::i64));
Cmp = DAG.getNode(ISD::XOR, DL, VT, Cmp, Mask);
}
+ if (Chain && Chain.getNode() != Cmp.getNode()) {
+ SDValue Ops[2] = { Cmp, Chain };
+ Cmp = DAG.getMergeValues(Ops, DL);
+ }
return Cmp;
}
@@ -2755,6 +2852,29 @@
return emitSETCC(DAG, DL, CCReg, C.CCValid, C.CCMask);
}
+SDValue SystemZTargetLowering::lowerSTRICT_FSETCC(SDValue Op,
+ SelectionDAG &DAG,
+ bool IsSignaling) const {
+ SDValue Chain = Op.getOperand(0);
+ SDValue CmpOp0 = Op.getOperand(1);
+ SDValue CmpOp1 = Op.getOperand(2);
+ ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(3))->get();
+ SDLoc DL(Op);
+ EVT VT = Op.getNode()->getValueType(0);
+ if (VT.isVector()) {
+ SDValue Res = lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1,
+ Chain, IsSignaling);
+ return Res.getValue(Op.getResNo());
+ }
+
+ Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL, Chain, IsSignaling));
+ SDValue CCReg = emitCmp(DAG, DL, C);
+ CCReg->setFlags(Op->getFlags());
+ SDValue Result = emitSETCC(DAG, DL, CCReg, C.CCValid, C.CCMask);
+ SDValue Ops[2] = { Result, CCReg.getValue(1) };
+ return DAG.getMergeValues(Ops, DL);
+}
+
SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
SDValue CmpOp0 = Op.getOperand(2);
@@ -4966,6 +5086,10 @@
return lowerSELECT_CC(Op, DAG);
case ISD::SETCC:
return lowerSETCC(Op, DAG);
+ case ISD::STRICT_FSETCC:
+ return lowerSTRICT_FSETCC(Op, DAG, false);
+ case ISD::STRICT_FSETCCS:
+ return lowerSTRICT_FSETCC(Op, DAG, true);
case ISD::GlobalAddress:
return lowerGlobalAddress(cast<GlobalAddressSDNode>(Op), DAG);
case ISD::GlobalTLSAddress:
@@ -5171,6 +5295,8 @@
OPCODE(IABS);
OPCODE(ICMP);
OPCODE(FCMP);
+ OPCODE(STRICT_FCMP);
+ OPCODE(STRICT_FCMPS);
OPCODE(TM);
OPCODE(BR_CCMASK);
OPCODE(SELECT_CCMASK);
@@ -5233,13 +5359,20 @@
OPCODE(VICMPHS);
OPCODE(VICMPHLS);
OPCODE(VFCMPE);
+ OPCODE(STRICT_VFCMPE);
+ OPCODE(STRICT_VFCMPES);
OPCODE(VFCMPH);
+ OPCODE(STRICT_VFCMPH);
+ OPCODE(STRICT_VFCMPHS);
OPCODE(VFCMPHE);
+ OPCODE(STRICT_VFCMPHE);
+ OPCODE(STRICT_VFCMPHES);
OPCODE(VFCMPES);
OPCODE(VFCMPHS);
OPCODE(VFCMPHES);
OPCODE(VFTCI);
OPCODE(VEXTEND);
+ OPCODE(STRICT_VEXTEND);
OPCODE(VROUND);
OPCODE(VTM);
OPCODE(VFAE_CC);
@@ -7554,7 +7687,8 @@
// Replace pseudo with a normal load-and-test that models the def as
// well.
BuildMI(*MBB, MI, DL, TII->get(Opcode), DstReg)
- .addReg(SrcReg);
+ .addReg(SrcReg)
+ .setMIFlags(MI.getFlags());
MI.eraseFromParent();
return MBB;
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
index f774b8a..e49c47e 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
@@ -58,7 +58,8 @@
ICMP,
// Floating-point comparisons. The two operands are the values to compare.
- FCMP,
+ // Regular and strict (quiet and signaling) versions.
+ FCMP, STRICT_FCMP, STRICT_FCMPS,
// Test under mask. The first operand is ANDed with the second operand
// and the condition codes are set on the result. The third operand is
@@ -248,9 +249,10 @@
// Compare floating-point vector operands 0 and 1 to produce the usual 0/-1
// vector result. VFCMPE is for "ordered and equal", VFCMPH for "ordered and
// greater than" and VFCMPHE for "ordered and greater than or equal to".
- VFCMPE,
- VFCMPH,
- VFCMPHE,
+ // Regular and strict (quiet and signaling) versions.
+ VFCMPE, STRICT_VFCMPE, STRICT_VFCMPES,
+ VFCMPH, STRICT_VFCMPH, STRICT_VFCMPHS,
+ VFCMPHE, STRICT_VFCMPHE, STRICT_VFCMPHES,
// Likewise, but also set the condition codes on the result.
VFCMPES,
@@ -261,8 +263,8 @@
VFTCI,
// Extend the even f32 elements of vector operand 0 to produce a vector
- // of f64 elements.
- VEXTEND,
+ // of f64 elements. Regular and strict versions.
+ VEXTEND, STRICT_VEXTEND,
// Round the f64 elements of vector operand 0 to f32s and store them in the
// even elements of the result.
@@ -531,11 +533,15 @@
// Implement LowerOperation for individual opcodes.
SDValue getVectorCmp(SelectionDAG &DAG, unsigned Opcode,
const SDLoc &DL, EVT VT,
- SDValue CmpOp0, SDValue CmpOp1) const;
+ SDValue CmpOp0, SDValue CmpOp1, SDValue Chain) const;
SDValue lowerVectorSETCC(SelectionDAG &DAG, const SDLoc &DL,
EVT VT, ISD::CondCode CC,
- SDValue CmpOp0, SDValue CmpOp1) const;
+ SDValue CmpOp0, SDValue CmpOp1,
+ SDValue Chain = SDValue(),
+ bool IsSignaling = false) const;
SDValue lowerSETCC(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerSTRICT_FSETCC(SDValue Op, SelectionDAG &DAG,
+ bool IsSignaling) const;
SDValue lowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerGlobalAddress(GlobalAddressSDNode *Node,
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrFP.td b/llvm/lib/Target/SystemZ/SystemZInstrFP.td
index 3a185e5..2aea5f9 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrFP.td
+++ b/llvm/lib/Target/SystemZ/SystemZInstrFP.td
@@ -537,19 +537,19 @@
//===----------------------------------------------------------------------===//
let Uses = [FPC], mayRaiseFPException = 1, Defs = [CC], CCValues = 0xF in {
- def CEBR : CompareRRE<"cebr", 0xB309, z_fcmp, FP32, FP32>;
- def CDBR : CompareRRE<"cdbr", 0xB319, z_fcmp, FP64, FP64>;
- def CXBR : CompareRRE<"cxbr", 0xB349, z_fcmp, FP128, FP128>;
+ def CEBR : CompareRRE<"cebr", 0xB309, z_any_fcmp, FP32, FP32>;
+ def CDBR : CompareRRE<"cdbr", 0xB319, z_any_fcmp, FP64, FP64>;
+ def CXBR : CompareRRE<"cxbr", 0xB349, z_any_fcmp, FP128, FP128>;
- def CEB : CompareRXE<"ceb", 0xED09, z_fcmp, FP32, load, 4>;
- def CDB : CompareRXE<"cdb", 0xED19, z_fcmp, FP64, load, 8>;
+ def CEB : CompareRXE<"ceb", 0xED09, z_any_fcmp, FP32, load, 4>;
+ def CDB : CompareRXE<"cdb", 0xED19, z_any_fcmp, FP64, load, 8>;
- def KEBR : CompareRRE<"kebr", 0xB308, null_frag, FP32, FP32>;
- def KDBR : CompareRRE<"kdbr", 0xB318, null_frag, FP64, FP64>;
- def KXBR : CompareRRE<"kxbr", 0xB348, null_frag, FP128, FP128>;
+ def KEBR : CompareRRE<"kebr", 0xB308, z_strict_fcmps, FP32, FP32>;
+ def KDBR : CompareRRE<"kdbr", 0xB318, z_strict_fcmps, FP64, FP64>;
+ def KXBR : CompareRRE<"kxbr", 0xB348, z_strict_fcmps, FP128, FP128>;
- def KEB : CompareRXE<"keb", 0xED08, null_frag, FP32, load, 4>;
- def KDB : CompareRXE<"kdb", 0xED18, null_frag, FP64, load, 8>;
+ def KEB : CompareRXE<"keb", 0xED08, z_strict_fcmps, FP32, load, 4>;
+ def KDB : CompareRXE<"kdb", 0xED18, z_strict_fcmps, FP64, load, 8>;
}
// Test Data Class.
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrVector.td b/llvm/lib/Target/SystemZ/SystemZInstrVector.td
index 02364bb..fe242dd 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrVector.td
+++ b/llvm/lib/Target/SystemZ/SystemZInstrVector.td
@@ -1134,7 +1134,7 @@
// Load lengthened.
let Uses = [FPC], mayRaiseFPException = 1 in {
def VLDE : UnaryVRRaFloatGeneric<"vlde", 0xE7C4>;
- def VLDEB : UnaryVRRa<"vldeb", 0xE7C4, z_vextend, v128db, v128sb, 2, 0>;
+ def VLDEB : UnaryVRRa<"vldeb", 0xE7C4, z_any_vextend, v128db, v128sb, 2, 0>;
def WLDEB : UnaryVRRa<"wldeb", 0xE7C4, any_fpextend, v64db, v32sb, 2, 8>;
}
let Predicates = [FeatureVectorEnhancements1] in {
@@ -1364,32 +1364,32 @@
// Compare scalar.
let Uses = [FPC], mayRaiseFPException = 1, Defs = [CC] in {
def WFC : CompareVRRaFloatGeneric<"wfc", 0xE7CB>;
- def WFCDB : CompareVRRa<"wfcdb", 0xE7CB, z_fcmp, v64db, 3>;
+ def WFCDB : CompareVRRa<"wfcdb", 0xE7CB, z_any_fcmp, v64db, 3>;
let Predicates = [FeatureVectorEnhancements1] in {
- def WFCSB : CompareVRRa<"wfcsb", 0xE7CB, z_fcmp, v32sb, 2>;
- def WFCXB : CompareVRRa<"wfcxb", 0xE7CB, z_fcmp, v128xb, 4>;
+ def WFCSB : CompareVRRa<"wfcsb", 0xE7CB, z_any_fcmp, v32sb, 2>;
+ def WFCXB : CompareVRRa<"wfcxb", 0xE7CB, z_any_fcmp, v128xb, 4>;
}
}
// Compare and signal scalar.
let Uses = [FPC], mayRaiseFPException = 1, Defs = [CC] in {
def WFK : CompareVRRaFloatGeneric<"wfk", 0xE7CA>;
- def WFKDB : CompareVRRa<"wfkdb", 0xE7CA, null_frag, v64db, 3>;
+ def WFKDB : CompareVRRa<"wfkdb", 0xE7CA, z_strict_fcmps, v64db, 3>;
let Predicates = [FeatureVectorEnhancements1] in {
- def WFKSB : CompareVRRa<"wfksb", 0xE7CA, null_frag, v32sb, 2>;
- def WFKXB : CompareVRRa<"wfkxb", 0xE7CA, null_frag, v128xb, 4>;
+ def WFKSB : CompareVRRa<"wfksb", 0xE7CA, z_strict_fcmps, v32sb, 2>;
+ def WFKXB : CompareVRRa<"wfkxb", 0xE7CA, z_strict_fcmps, v128xb, 4>;
}
}
// Compare equal.
let Uses = [FPC], mayRaiseFPException = 1 in {
def VFCE : BinaryVRRcSPairFloatGeneric<"vfce", 0xE7E8>;
- defm VFCEDB : BinaryVRRcSPair<"vfcedb", 0xE7E8, z_vfcmpe, z_vfcmpes,
+ defm VFCEDB : BinaryVRRcSPair<"vfcedb", 0xE7E8, z_any_vfcmpe, z_vfcmpes,
v128g, v128db, 3, 0>;
defm WFCEDB : BinaryVRRcSPair<"wfcedb", 0xE7E8, null_frag, null_frag,
v64g, v64db, 3, 8>;
let Predicates = [FeatureVectorEnhancements1] in {
- defm VFCESB : BinaryVRRcSPair<"vfcesb", 0xE7E8, z_vfcmpe, z_vfcmpes,
+ defm VFCESB : BinaryVRRcSPair<"vfcesb", 0xE7E8, z_any_vfcmpe, z_vfcmpes,
v128f, v128sb, 2, 0>;
defm WFCESB : BinaryVRRcSPair<"wfcesb", 0xE7E8, null_frag, null_frag,
v32f, v32sb, 2, 8>;
@@ -1401,11 +1401,11 @@
// Compare and signal equal.
let Uses = [FPC], mayRaiseFPException = 1,
Predicates = [FeatureVectorEnhancements1] in {
- defm VFKEDB : BinaryVRRcSPair<"vfkedb", 0xE7E8, null_frag, null_frag,
+ defm VFKEDB : BinaryVRRcSPair<"vfkedb", 0xE7E8, z_strict_vfcmpes, null_frag,
v128g, v128db, 3, 4>;
defm WFKEDB : BinaryVRRcSPair<"wfkedb", 0xE7E8, null_frag, null_frag,
v64g, v64db, 3, 12>;
- defm VFKESB : BinaryVRRcSPair<"vfkesb", 0xE7E8, null_frag, null_frag,
+ defm VFKESB : BinaryVRRcSPair<"vfkesb", 0xE7E8, z_strict_vfcmpes, null_frag,
v128f, v128sb, 2, 4>;
defm WFKESB : BinaryVRRcSPair<"wfkesb", 0xE7E8, null_frag, null_frag,
v32f, v32sb, 2, 12>;
@@ -1416,12 +1416,12 @@
// Compare high.
let Uses = [FPC], mayRaiseFPException = 1 in {
def VFCH : BinaryVRRcSPairFloatGeneric<"vfch", 0xE7EB>;
- defm VFCHDB : BinaryVRRcSPair<"vfchdb", 0xE7EB, z_vfcmph, z_vfcmphs,
+ defm VFCHDB : BinaryVRRcSPair<"vfchdb", 0xE7EB, z_any_vfcmph, z_vfcmphs,
v128g, v128db, 3, 0>;
defm WFCHDB : BinaryVRRcSPair<"wfchdb", 0xE7EB, null_frag, null_frag,
v64g, v64db, 3, 8>;
let Predicates = [FeatureVectorEnhancements1] in {
- defm VFCHSB : BinaryVRRcSPair<"vfchsb", 0xE7EB, z_vfcmph, z_vfcmphs,
+ defm VFCHSB : BinaryVRRcSPair<"vfchsb", 0xE7EB, z_any_vfcmph, z_vfcmphs,
v128f, v128sb, 2, 0>;
defm WFCHSB : BinaryVRRcSPair<"wfchsb", 0xE7EB, null_frag, null_frag,
v32f, v32sb, 2, 8>;
@@ -1433,11 +1433,11 @@
// Compare and signal high.
let Uses = [FPC], mayRaiseFPException = 1,
Predicates = [FeatureVectorEnhancements1] in {
- defm VFKHDB : BinaryVRRcSPair<"vfkhdb", 0xE7EB, null_frag, null_frag,
+ defm VFKHDB : BinaryVRRcSPair<"vfkhdb", 0xE7EB, z_strict_vfcmphs, null_frag,
v128g, v128db, 3, 4>;
defm WFKHDB : BinaryVRRcSPair<"wfkhdb", 0xE7EB, null_frag, null_frag,
v64g, v64db, 3, 12>;
- defm VFKHSB : BinaryVRRcSPair<"vfkhsb", 0xE7EB, null_frag, null_frag,
+ defm VFKHSB : BinaryVRRcSPair<"vfkhsb", 0xE7EB, z_strict_vfcmphs, null_frag,
v128f, v128sb, 2, 4>;
defm WFKHSB : BinaryVRRcSPair<"wfkhsb", 0xE7EB, null_frag, null_frag,
v32f, v32sb, 2, 12>;
@@ -1448,12 +1448,12 @@
// Compare high or equal.
let Uses = [FPC], mayRaiseFPException = 1 in {
def VFCHE : BinaryVRRcSPairFloatGeneric<"vfche", 0xE7EA>;
- defm VFCHEDB : BinaryVRRcSPair<"vfchedb", 0xE7EA, z_vfcmphe, z_vfcmphes,
+ defm VFCHEDB : BinaryVRRcSPair<"vfchedb", 0xE7EA, z_any_vfcmphe, z_vfcmphes,
v128g, v128db, 3, 0>;
defm WFCHEDB : BinaryVRRcSPair<"wfchedb", 0xE7EA, null_frag, null_frag,
v64g, v64db, 3, 8>;
let Predicates = [FeatureVectorEnhancements1] in {
- defm VFCHESB : BinaryVRRcSPair<"vfchesb", 0xE7EA, z_vfcmphe, z_vfcmphes,
+ defm VFCHESB : BinaryVRRcSPair<"vfchesb", 0xE7EA, z_any_vfcmphe, z_vfcmphes,
v128f, v128sb, 2, 0>;
defm WFCHESB : BinaryVRRcSPair<"wfchesb", 0xE7EA, null_frag, null_frag,
v32f, v32sb, 2, 8>;
@@ -1465,11 +1465,11 @@
// Compare and signal high or equal.
let Uses = [FPC], mayRaiseFPException = 1,
Predicates = [FeatureVectorEnhancements1] in {
- defm VFKHEDB : BinaryVRRcSPair<"vfkhedb", 0xE7EA, null_frag, null_frag,
+ defm VFKHEDB : BinaryVRRcSPair<"vfkhedb", 0xE7EA, z_strict_vfcmphes, null_frag,
v128g, v128db, 3, 4>;
defm WFKHEDB : BinaryVRRcSPair<"wfkhedb", 0xE7EA, null_frag, null_frag,
v64g, v64db, 3, 12>;
- defm VFKHESB : BinaryVRRcSPair<"vfkhesb", 0xE7EA, null_frag, null_frag,
+ defm VFKHESB : BinaryVRRcSPair<"vfkhesb", 0xE7EA, z_strict_vfcmphes, null_frag,
v128f, v128sb, 2, 4>;
defm WFKHESB : BinaryVRRcSPair<"wfkhesb", 0xE7EA, null_frag, null_frag,
v32f, v32sb, 2, 12>;
diff --git a/llvm/lib/Target/SystemZ/SystemZOperators.td b/llvm/lib/Target/SystemZ/SystemZOperators.td
index 6fe383e..0beefc4 100644
--- a/llvm/lib/Target/SystemZ/SystemZOperators.td
+++ b/llvm/lib/Target/SystemZ/SystemZOperators.td
@@ -258,6 +258,10 @@
def z_iabs : SDNode<"SystemZISD::IABS", SDTIntUnaryOp, []>;
def z_icmp : SDNode<"SystemZISD::ICMP", SDT_ZICmp>;
def z_fcmp : SDNode<"SystemZISD::FCMP", SDT_ZCmp>;
+def z_strict_fcmp : SDNode<"SystemZISD::STRICT_FCMP", SDT_ZCmp,
+ [SDNPHasChain]>;
+def z_strict_fcmps : SDNode<"SystemZISD::STRICT_FCMPS", SDT_ZCmp,
+ [SDNPHasChain]>;
def z_tm : SDNode<"SystemZISD::TM", SDT_ZICmp>;
def z_br_ccmask_1 : SDNode<"SystemZISD::BR_CCMASK", SDT_ZBRCCMask,
[SDNPHasChain]>;
@@ -328,12 +332,26 @@
def z_vicmphs : SDNode<"SystemZISD::VICMPHS", SDT_ZVecBinaryCC>;
def z_vicmphls : SDNode<"SystemZISD::VICMPHLS", SDT_ZVecBinaryCC>;
def z_vfcmpe : SDNode<"SystemZISD::VFCMPE", SDT_ZVecBinaryConv>;
+def z_strict_vfcmpe : SDNode<"SystemZISD::STRICT_VFCMPE",
+ SDT_ZVecBinaryConv, [SDNPHasChain]>;
+def z_strict_vfcmpes : SDNode<"SystemZISD::STRICT_VFCMPES",
+ SDT_ZVecBinaryConv, [SDNPHasChain]>;
def z_vfcmph : SDNode<"SystemZISD::VFCMPH", SDT_ZVecBinaryConv>;
+def z_strict_vfcmph : SDNode<"SystemZISD::STRICT_VFCMPH",
+ SDT_ZVecBinaryConv, [SDNPHasChain]>;
+def z_strict_vfcmphs : SDNode<"SystemZISD::STRICT_VFCMPHS",
+ SDT_ZVecBinaryConv, [SDNPHasChain]>;
def z_vfcmphe : SDNode<"SystemZISD::VFCMPHE", SDT_ZVecBinaryConv>;
+def z_strict_vfcmphe : SDNode<"SystemZISD::STRICT_VFCMPHE",
+ SDT_ZVecBinaryConv, [SDNPHasChain]>;
+def z_strict_vfcmphes : SDNode<"SystemZISD::STRICT_VFCMPHES",
+ SDT_ZVecBinaryConv, [SDNPHasChain]>;
def z_vfcmpes : SDNode<"SystemZISD::VFCMPES", SDT_ZVecBinaryConvCC>;
def z_vfcmphs : SDNode<"SystemZISD::VFCMPHS", SDT_ZVecBinaryConvCC>;
def z_vfcmphes : SDNode<"SystemZISD::VFCMPHES", SDT_ZVecBinaryConvCC>;
def z_vextend : SDNode<"SystemZISD::VEXTEND", SDT_ZVecUnaryConv>;
+def z_strict_vextend : SDNode<"SystemZISD::STRICT_VEXTEND",
+ SDT_ZVecUnaryConv, [SDNPHasChain]>;
def z_vround : SDNode<"SystemZISD::VROUND", SDT_ZVecUnaryConv>;
def z_vtm : SDNode<"SystemZISD::VTM", SDT_ZCmp>;
def z_vfae_cc : SDNode<"SystemZISD::VFAE_CC", SDT_ZVecTernaryIntCC>;
@@ -707,6 +725,23 @@
// Floating-point negative absolute.
def fnabs : PatFrag<(ops node:$ptr), (fneg (fabs node:$ptr))>;
+// Strict floating-point fragments.
+def z_any_fcmp : PatFrags<(ops node:$lhs, node:$rhs),
+ [(z_strict_fcmp node:$lhs, node:$rhs),
+ (z_fcmp node:$lhs, node:$rhs)]>;
+def z_any_vfcmpe : PatFrags<(ops node:$lhs, node:$rhs),
+ [(z_strict_vfcmpe node:$lhs, node:$rhs),
+ (z_vfcmpe node:$lhs, node:$rhs)]>;
+def z_any_vfcmph : PatFrags<(ops node:$lhs, node:$rhs),
+ [(z_strict_vfcmph node:$lhs, node:$rhs),
+ (z_vfcmph node:$lhs, node:$rhs)]>;
+def z_any_vfcmphe : PatFrags<(ops node:$lhs, node:$rhs),
+ [(z_strict_vfcmphe node:$lhs, node:$rhs),
+ (z_vfcmphe node:$lhs, node:$rhs)]>;
+def z_any_vextend : PatFrags<(ops node:$src),
+ [(z_strict_vextend node:$src),
+ (z_vextend node:$src)]>;
+
// Create a unary operator that loads from memory and then performs
// the given operation on it.
class loadu<SDPatternOperator operator, SDPatternOperator load = load>
diff --git a/llvm/lib/Target/SystemZ/SystemZPatterns.td b/llvm/lib/Target/SystemZ/SystemZPatterns.td
index 65300fb..501a694 100644
--- a/llvm/lib/Target/SystemZ/SystemZPatterns.td
+++ b/llvm/lib/Target/SystemZ/SystemZPatterns.td
@@ -148,9 +148,9 @@
// registers in CLS against zero. The instruction has separate R1 and R2
// operands, but they must be the same when the instruction is used like this.
multiclass CompareZeroFP<Instruction insn, RegisterOperand cls> {
- def : Pat<(z_fcmp cls:$reg, (fpimm0)), (insn cls:$reg, cls:$reg)>;
+ def : Pat<(z_any_fcmp cls:$reg, (fpimm0)), (insn cls:$reg, cls:$reg)>;
// The sign of the zero makes no difference.
- def : Pat<(z_fcmp cls:$reg, (fpimmneg0)), (insn cls:$reg, cls:$reg)>;
+ def : Pat<(z_any_fcmp cls:$reg, (fpimmneg0)), (insn cls:$reg, cls:$reg)>;
}
// Use INSN for performing binary operation OPERATION of type VT
diff --git a/llvm/lib/Target/SystemZ/SystemZShortenInst.cpp b/llvm/lib/Target/SystemZ/SystemZShortenInst.cpp
index 2aca22c..1b9c8ea 100644
--- a/llvm/lib/Target/SystemZ/SystemZShortenInst.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZShortenInst.cpp
@@ -313,6 +313,14 @@
Changed |= shortenOn01(MI, SystemZ::CEBR);
break;
+ case SystemZ::WFKDB:
+ Changed |= shortenOn01(MI, SystemZ::KDBR);
+ break;
+
+ case SystemZ::WFKSB:
+ Changed |= shortenOn01(MI, SystemZ::KEBR);
+ break;
+
case SystemZ::VL32:
// For z13 we prefer LDE over LE to avoid partial register dependencies.
Changed |= shortenOn0(MI, SystemZ::LDE32);