Add intrinsics for constrained floating point operations
This commit introduces a set of experimental intrinsics intended to prevent
optimizations that make assumptions about the rounding mode and floating point
exception behavior. These intrinsics will later be extended to specify
flush-to-zero behavior. More work is also required to model instruction
dependencies in machine code and to generate these instructions from clang
(when required by pragmas and/or command line options that are not currently
supported).
Differential Revision: https://reviews.llvm.org/D27028
llvm-svn: 293226
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 2e65a2c..b654086 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -5336,6 +5336,13 @@
getValue(I.getArgOperand(1)),
getValue(I.getArgOperand(2))));
return nullptr;
+ case Intrinsic::experimental_constrained_fadd:
+ case Intrinsic::experimental_constrained_fsub:
+ case Intrinsic::experimental_constrained_fmul:
+ case Intrinsic::experimental_constrained_fdiv:
+ case Intrinsic::experimental_constrained_frem:
+ visitConstrainedFPIntrinsic(I, Intrinsic);
+ return nullptr;
case Intrinsic::fmuladd: {
EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
@@ -5784,6 +5791,46 @@
}
}
+void SelectionDAGBuilder::visitConstrainedFPIntrinsic(const CallInst &I,
+ unsigned Intrinsic) {
+ SDLoc sdl = getCurSDLoc();
+ unsigned Opcode;
+ switch (Intrinsic) {
+ default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
+ case Intrinsic::experimental_constrained_fadd:
+ Opcode = ISD::STRICT_FADD;
+ break;
+ case Intrinsic::experimental_constrained_fsub:
+ Opcode = ISD::STRICT_FSUB;
+ break;
+ case Intrinsic::experimental_constrained_fmul:
+ Opcode = ISD::STRICT_FMUL;
+ break;
+ case Intrinsic::experimental_constrained_fdiv:
+ Opcode = ISD::STRICT_FDIV;
+ break;
+ case Intrinsic::experimental_constrained_frem:
+ Opcode = ISD::STRICT_FREM;
+ break;
+ }
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+ SDValue Chain = getRoot();
+ SDValue Ops[3] = { Chain, getValue(I.getArgOperand(0)),
+ getValue(I.getArgOperand(1)) };
+ SmallVector<EVT, 4> ValueVTs;
+ ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
+ ValueVTs.push_back(MVT::Other); // Out chain
+
+ SDVTList VTs = DAG.getVTList(ValueVTs);
+ SDValue Result = DAG.getNode(Opcode, sdl, VTs, Ops);
+
+ assert(Result.getNode()->getNumValues() == 2);
+ SDValue OutChain = Result.getValue(1);
+ DAG.setRoot(OutChain);
+ SDValue FPResult = Result.getValue(0);
+ setValue(&I, FPResult);
+}
+
std::pair<SDValue, SDValue>
SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
const BasicBlock *EHPadBB) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index 0a0fbc7..5ecc753 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -901,6 +901,7 @@
void visitInlineAsm(ImmutableCallSite CS);
const char *visitIntrinsicCall(const CallInst &I, unsigned Intrinsic);
void visitTargetIntrinsic(const CallInst &I, unsigned Intrinsic);
+ void visitConstrainedFPIntrinsic(const CallInst &I, unsigned Intrinsic);
void visitVAStart(const CallInst &I);
void visitVAArg(const VAArgInst &I);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 6d717b4..99dd906 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -925,6 +925,50 @@
};
} // end anonymous namespace
+static bool isStrictFPOp(SDNode *Node, unsigned &NewOpc) {
+ unsigned OrigOpc = Node->getOpcode();
+ switch (OrigOpc) {
+ case ISD::STRICT_FADD: NewOpc = ISD::FADD; return true;
+ case ISD::STRICT_FSUB: NewOpc = ISD::FSUB; return true;
+ case ISD::STRICT_FMUL: NewOpc = ISD::FMUL; return true;
+ case ISD::STRICT_FDIV: NewOpc = ISD::FDIV; return true;
+ case ISD::STRICT_FREM: NewOpc = ISD::FREM; return true;
+ default: return false;
+ }
+}
+
+SDNode* SelectionDAGISel::MutateStrictFPToFP(SDNode *Node, unsigned NewOpc) {
+ assert(((Node->getOpcode() == ISD::STRICT_FADD && NewOpc == ISD::FADD) ||
+ (Node->getOpcode() == ISD::STRICT_FSUB && NewOpc == ISD::FSUB) ||
+ (Node->getOpcode() == ISD::STRICT_FMUL && NewOpc == ISD::FMUL) ||
+ (Node->getOpcode() == ISD::STRICT_FDIV && NewOpc == ISD::FDIV) ||
+ (Node->getOpcode() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) &&
+ "Unexpected StrictFP opcode!");
+
+ // We're taking this node out of the chain, so we need to re-link things.
+ SDValue InputChain = Node->getOperand(0);
+ SDValue OutputChain = SDValue(Node, 1);
+ CurDAG->ReplaceAllUsesOfValueWith(OutputChain, InputChain);
+
+ SDVTList VTs = CurDAG->getVTList(Node->getOperand(1).getValueType());
+ SDValue Ops[2] = { Node->getOperand(1), Node->getOperand(2) };
+ SDNode *Res = CurDAG->MorphNodeTo(Node, NewOpc, VTs, Ops);
+
+ // MorphNodeTo can operate in two ways: if an existing node with the
+ // specified operands exists, it can just return it. Otherwise, it
+ // updates the node in place to have the requested operands.
+ if (Res == Node) {
+ // If we updated the node in place, reset the node ID. To the isel,
+ // this should be just like a newly allocated machine node.
+ Res->setNodeId(-1);
+ } else {
+ CurDAG->ReplaceAllUsesWith(Node, Res);
+ CurDAG->RemoveDeadNode(Node);
+ }
+
+ return Res;
+}
+
void SelectionDAGISel::DoInstructionSelection() {
DEBUG(dbgs() << "===== Instruction selection begins: BB#"
<< FuncInfo->MBB->getNumber()
@@ -960,7 +1004,23 @@
if (Node->use_empty())
continue;
+ // When we are using non-default rounding modes or FP exception behavior
+ // FP operations are represented by StrictFP pseudo-operations. They
+ // need to be simplified here so that the target-specific instruction
+ // selectors know how to handle them.
+ //
+ // If the current node is a strict FP pseudo-op, the isStrictFPOp()
+ // function will provide the corresponding normal FP opcode to which the
+ // node should be mutated.
+ unsigned NormalFPOpc = ISD::UNDEF;
+ bool IsStrictFPOp = isStrictFPOp(Node, NormalFPOpc);
+ if (IsStrictFPOp)
+ Node = MutateStrictFPToFP(Node, NormalFPOpc);
+
Select(Node);
+
+ // FIXME: Add code here to attach an implicit def and use of
+ // target-specific FP environment registers.
}
CurDAG->setRoot(Dummy.getValue());
diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp
index 2402506..c9814a9 100644
--- a/llvm/lib/IR/IntrinsicInst.cpp
+++ b/llvm/lib/IR/IntrinsicInst.cpp
@@ -21,6 +21,7 @@
//
//===----------------------------------------------------------------------===//
+#include "llvm/ADT/StringSwitch.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/GlobalVariable.h"
@@ -93,3 +94,34 @@
LLVMContext &Context = M->getContext();
return ConstantInt::get(Type::getInt64Ty(Context), 1);
}
+
+ConstrainedFPIntrinsic::RoundingMode
+ConstrainedFPIntrinsic::getRoundingMode() const {
+ Metadata *MD = dyn_cast<MetadataAsValue>(getOperand(2))->getMetadata();
+ if (!MD || !isa<MDString>(MD))
+ return rmInvalid;
+ StringRef RoundingArg = cast<MDString>(MD)->getString();
+
+ // For dynamic rounding mode, we use round to nearest but we will set the
+ // 'exact' SDNodeFlag so that the value will not be rounded.
+ return StringSwitch<RoundingMode>(RoundingArg)
+ .Case("round.dynamic", rmDynamic)
+ .Case("round.tonearest", rmToNearest)
+ .Case("round.downward", rmDownward)
+ .Case("round.upward", rmUpward)
+ .Case("round.towardzero", rmTowardZero)
+ .Default(rmInvalid);
+}
+
+ConstrainedFPIntrinsic::ExceptionBehavior
+ConstrainedFPIntrinsic::getExceptionBehavior() const {
+ Metadata *MD = dyn_cast<MetadataAsValue>(getOperand(3))->getMetadata();
+ if (!MD || !isa<MDString>(MD))
+ return ebInvalid;
+ StringRef ExceptionArg = cast<MDString>(MD)->getString();
+ return StringSwitch<ExceptionBehavior>(ExceptionArg)
+ .Case("fpexcept.ignore", ebIgnore)
+ .Case("fpexcept.maytrap", ebMayTrap)
+ .Case("fpexcept.strict", ebStrict)
+ .Default(ebInvalid);
+}
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 5855059..41593af 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -457,6 +457,7 @@
void visitUserOp1(Instruction &I);
void visitUserOp2(Instruction &I) { visitUserOp1(I); }
void visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS);
+ void visitConstrainedFPIntrinsic(ConstrainedFPIntrinsic &FPI);
template <class DbgIntrinsicTy>
void visitDbgIntrinsic(StringRef Kind, DbgIntrinsicTy &DII);
void visitAtomicCmpXchgInst(AtomicCmpXchgInst &CXI);
@@ -3929,6 +3930,14 @@
"constant int",
CS);
break;
+ case Intrinsic::experimental_constrained_fadd:
+ case Intrinsic::experimental_constrained_fsub:
+ case Intrinsic::experimental_constrained_fmul:
+ case Intrinsic::experimental_constrained_fdiv:
+ case Intrinsic::experimental_constrained_frem:
+ visitConstrainedFPIntrinsic(
+ cast<ConstrainedFPIntrinsic>(*CS.getInstruction()));
+ break;
case Intrinsic::dbg_declare: // llvm.dbg.declare
Assert(isa<MetadataAsValue>(CS.getArgOperand(0)),
"invalid llvm.dbg.declare intrinsic call 1", CS);
@@ -4294,6 +4303,15 @@
return nullptr;
}
+void Verifier::visitConstrainedFPIntrinsic(ConstrainedFPIntrinsic &FPI) {
+ Assert(isa<MetadataAsValue>(FPI.getOperand(2)),
+ "invalid rounding mode argument", &FPI);
+ Assert(FPI.getRoundingMode() != ConstrainedFPIntrinsic::rmInvalid,
+ "invalid rounding mode argument", &FPI);
+ Assert(FPI.getExceptionBehavior() != ConstrainedFPIntrinsic::ebInvalid,
+ "invalid exception behavior argument", &FPI);
+}
+
template <class DbgIntrinsicTy>
void Verifier::visitDbgIntrinsic(StringRef Kind, DbgIntrinsicTy &DII) {
auto *MD = cast<MetadataAsValue>(DII.getArgOperand(0))->getMetadata();