Add "first class" lowering for deopt operand bundles
Summary:
After this change, deopt operand bundles can be lowered directly by
SelectionDAG into STATEPOINT instructions (which are then lowered to a
call or sequence of nop, with an associated __llvm_stackmaps entry0.
This obviates the need to round-trip deoptimization state through
gc.statepoint via RewriteStatepointsForGC.
Reviewers: reames, atrick, majnemer, JosephTremoulet, pgavlin
Subscribers: sanjoy, mcrosier, majnemer, llvm-commits
Differential Revision: http://reviews.llvm.org/D18257
llvm-svn: 264015
diff --git a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
index e36f99f..8c738ff 100644
--- a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
@@ -1351,6 +1351,12 @@
return false;
}
+ // FastISel does not handle any operand bundles except OB_funclet.
+ if (ImmutableCallSite CS = ImmutableCallSite(I))
+ for (unsigned i = 0, e = CS.getNumOperandBundles(); i != e; ++i)
+ if (CS.getOperandBundleAt(i).getTagID() != LLVMContext::OB_funclet)
+ return false;
+
DbgLoc = I->getDebugLoc();
SavedInsertPt = FuncInfo.InsertPt;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 3e4111e..d01dd98 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -2127,6 +2127,15 @@
MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
const BasicBlock *EHPadBB = I.getSuccessor(1);
+#ifndef NDEBUG
+ // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
+ // have to do anything here to lower funclet bundles.
+ for (unsigned i = 0, e = I.getNumOperandBundles(); i != e; ++i)
+ assert((I.getOperandBundleAt(i).isDeoptOperandBundle() ||
+ I.getOperandBundleAt(i).isFuncletOperandBundle()) &&
+ "Cannot lower invokes with arbitrary operand bundles yet!");
+#endif
+
const Value *Callee(I.getCalledValue());
const Function *Fn = dyn_cast<Function>(Callee);
if (isa<InlineAsm>(Callee))
@@ -2146,8 +2155,15 @@
LowerStatepoint(ImmutableStatepoint(&I), EHPadBB);
break;
}
- } else
+ } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {
+ // Currently we do not lower any intrinsic calls with deopt operand bundles.
+ // Eventually we will support lowering the @llvm.experimental.deoptimize
+ // intrinsic, and right now there are no plans to support other intrinsics
+ // with deopt state.
+ LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
+ } else {
LowerCallTo(&I, getValue(Callee), false, EHPadBB);
+ }
// If the value of the invoke is used outside of its defining block, make it
// available as a virtual register.
@@ -6100,9 +6116,22 @@
RenameFn,
DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()));
- // Check if we can potentially perform a tail call. More detailed checking is
- // be done within LowerCallTo, after more information about the call is known.
- LowerCallTo(&I, Callee, I.isTailCall());
+#ifndef NDEBUG
+ // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
+ // have to do anything here to lower funclet bundles.
+ for (unsigned i = 0, e = I.getNumOperandBundles(); i != e; ++i)
+ assert((I.getOperandBundleAt(i).isDeoptOperandBundle() ||
+ I.getOperandBundleAt(i).isFuncletOperandBundle()) &&
+ "Cannot lower calls with arbitrary operand bundles!");
+#endif
+
+ if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
+ LowerCallSiteWithDeoptBundle(&I, Callee, nullptr);
+ else
+ // Check if we can potentially perform a tail call. More detailed checking
+ // is be done within LowerCallTo, after more information about the call is
+ // known.
+ LowerCallTo(&I, Callee, I.isTailCall());
}
namespace {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index fc12be0..5e27a1c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -772,12 +772,16 @@
};
/// Lower \p SLI into a STATEPOINT instruction.
- SDValue LowerAsStatepoint(StatepointLoweringInfo &SLI);
+ SDValue LowerAsSTATEPOINT(StatepointLoweringInfo &SLI);
// This function is responsible for the whole statepoint lowering process.
// It uniformly handles invoke and call statepoints.
void LowerStatepoint(ImmutableStatepoint Statepoint,
const BasicBlock *EHPadBB = nullptr);
+
+ void LowerCallSiteWithDeoptBundle(ImmutableCallSite CS, SDValue Callee,
+ const BasicBlock *EHPadBB);
+
private:
// Terminator instructions.
void visitRet(const ReturnInst &I);
diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
index 5dbc772..fb0cba1 100644
--- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
@@ -441,25 +441,30 @@
// Lower the deopt and gc arguments for this statepoint. Layout will be:
// deopt argument length, deopt arguments.., gc arguments...
#ifndef NDEBUG
- // Check that each of the gc pointer and bases we've gotten out of the
- // safepoint is something the strategy thinks might be a pointer (or vector
- // of pointers) into the GC heap. This is basically just here to help catch
- // errors during statepoint insertion. TODO: This should actually be in the
- // Verifier, but we can't get to the GCStrategy from there (yet).
- GCStrategy &S = Builder.GFI->getStrategy();
- for (const Value *V : SI.Bases) {
- auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
- if (Opt.hasValue()) {
- assert(Opt.getValue() &&
- "non gc managed base pointer found in statepoint");
+ if (auto *GFI = Builder.GFI) {
+ // Check that each of the gc pointer and bases we've gotten out of the
+ // safepoint is something the strategy thinks might be a pointer (or vector
+ // of pointers) into the GC heap. This is basically just here to help catch
+ // errors during statepoint insertion. TODO: This should actually be in the
+ // Verifier, but we can't get to the GCStrategy from there (yet).
+ GCStrategy &S = GFI->getStrategy();
+ for (const Value *V : SI.Bases) {
+ auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
+ if (Opt.hasValue()) {
+ assert(Opt.getValue() &&
+ "non gc managed base pointer found in statepoint");
+ }
}
- }
- for (const Value *V : SI.Ptrs) {
- auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
- if (Opt.hasValue()) {
- assert(Opt.getValue() &&
- "non gc managed derived pointer found in statepoint");
+ for (const Value *V : SI.Ptrs) {
+ auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
+ if (Opt.hasValue()) {
+ assert(Opt.getValue() &&
+ "non gc managed derived pointer found in statepoint");
+ }
}
+ } else {
+ assert(SI.Bases.empty() && "No gc specified, so cannot relocate pointers!");
+ assert(SI.Ptrs.empty() && "No gc specified, so cannot relocate pointers!");
}
#endif
@@ -550,7 +555,7 @@
}
}
-SDValue SelectionDAGBuilder::LowerAsStatepoint(
+SDValue SelectionDAGBuilder::LowerAsSTATEPOINT(
SelectionDAGBuilder::StatepointLoweringInfo &SI) {
// The basic scheme here is that information about both the original call and
// the safepoint is encoded in the CallInst. We create a temporary call and
@@ -794,7 +799,7 @@
SI.NumPatchBytes = ISP.getNumPatchBytes();
SI.EHPadBB = EHPadBB;
- SDValue ReturnValue = LowerAsStatepoint(SI);
+ SDValue ReturnValue = LowerAsSTATEPOINT(SI);
// Export the result value if needed
const Instruction *GCResult = ISP.getGCResult();
@@ -830,6 +835,37 @@
}
}
+void SelectionDAGBuilder::LowerCallSiteWithDeoptBundle(
+ ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB) {
+ assert(CS.getNumOperandBundles() == 1 &&
+ "Only deopt operand bundles can be lowered!");
+
+ StatepointLoweringInfo SI(DAG);
+ unsigned ArgBeginIndex = CS.arg_begin() - CS.getInstruction()->op_begin();
+ populateCallLoweringInfo(SI.CLI, CS, ArgBeginIndex, CS.getNumArgOperands(),
+ Callee, CS.getType(), false);
+
+ auto DeoptBundle = CS.getOperandBundleAt(0);
+ assert(DeoptBundle.getTagID() == LLVMContext::OB_deopt && "Should be!");
+
+ unsigned DefaultID = StatepointDirectives::DeoptBundleStatepointID;
+
+ auto SD = parseStatepointDirectivesFromAttrs(CS.getAttributes());
+ SI.ID = SD.StatepointID.getValueOr(DefaultID);
+ SI.NumPatchBytes = SD.NumPatchBytes.getValueOr(0);
+
+ SI.DeoptState =
+ ArrayRef<const Use>(DeoptBundle.Inputs.begin(), DeoptBundle.Inputs.end());
+ SI.StatepointFlags = static_cast<uint64_t>(StatepointFlags::None);
+ SI.EHPadBB = EHPadBB;
+
+ if (SDValue ReturnVal = LowerAsSTATEPOINT(SI)) {
+ const Instruction *Inst = CS.getInstruction();
+ ReturnVal = lowerRangeToAssertZExt(DAG, *Inst, ReturnVal);
+ setValue(Inst, ReturnVal);
+ }
+}
+
void SelectionDAGBuilder::visitGCResult(const CallInst &CI) {
// The result value of the gc_result is simply the result of the actual
// call. We've already emitted this, so just grab the value.