Change the INLINEASM_BR MachineInstr to be a non-terminating instruction.
Before this instruction supported output values, it fit fairly
naturally as a terminator. However, being a terminator while also
supporting outputs causes some trouble, as the physreg->vreg COPY
operations cannot be in the same block.
Modeling it as a non-terminator allows it to be handled the same way
as invoke is handled already.
Most of the changes here were created by auditing all the existing
users of MachineBasicBlock::isEHPad() and
MachineBasicBlock::hasEHPadSuccessor(), and adding calls to
isInlineAsmBrIndirectTarget or mayHaveInlineAsmBr, as appropriate.
Reviewed By: nickdesaulniers, void
Differential Revision: https://reviews.llvm.org/D79794
diff --git a/llvm/lib/CodeGen/BranchFolding.cpp b/llvm/lib/CodeGen/BranchFolding.cpp
index 022689c..c6d5aa3 100644
--- a/llvm/lib/CodeGen/BranchFolding.cpp
+++ b/llvm/lib/CodeGen/BranchFolding.cpp
@@ -1083,8 +1083,9 @@
if (!UniquePreds.insert(PBB).second)
continue;
- // Skip blocks which may jump to a landing pad. Can't tail merge these.
- if (PBB->hasEHPadSuccessor())
+ // Skip blocks which may jump to a landing pad or jump from an asm blob.
+ // Can't tail merge these.
+ if (PBB->hasEHPadSuccessor() || PBB->mayHaveInlineAsmBr())
continue;
// After block placement, only consider predecessors that belong to the
@@ -1665,13 +1666,15 @@
if (!MBB->isEHPad()) {
// Check all the predecessors of this block. If one of them has no fall
- // throughs, move this block right after it.
+ // throughs, and analyzeBranch thinks it _could_ fallthrough to this
+ // block, move this block right after it.
for (MachineBasicBlock *PredBB : MBB->predecessors()) {
// Analyze the branch at the end of the pred.
MachineBasicBlock *PredTBB = nullptr, *PredFBB = nullptr;
SmallVector<MachineOperand, 4> PredCond;
if (PredBB != MBB && !PredBB->canFallThrough() &&
!TII->analyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true) &&
+ (PredTBB == MBB || PredFBB == MBB) &&
(!CurFallsThru || !CurTBB || !CurFBB) &&
(!CurFallsThru || MBB->getNumber() >= PredBB->getNumber())) {
// If the current block doesn't fall through, just move it.
@@ -1697,21 +1700,24 @@
}
if (!CurFallsThru) {
- // Check all successors to see if we can move this block before it.
- for (MachineBasicBlock *SuccBB : MBB->successors()) {
- // Analyze the branch at the end of the block before the succ.
- MachineFunction::iterator SuccPrev = --SuccBB->getIterator();
+ // Check analyzable branch-successors to see if we can move this block
+ // before one.
+ if (!CurUnAnalyzable) {
+ for (MachineBasicBlock *SuccBB : {CurFBB, CurTBB}) {
+ if (!SuccBB)
+ continue;
+ // Analyze the branch at the end of the block before the succ.
+ MachineFunction::iterator SuccPrev = --SuccBB->getIterator();
- // If this block doesn't already fall-through to that successor, and if
- // the succ doesn't already have a block that can fall through into it,
- // and if the successor isn't an EH destination, we can arrange for the
- // fallthrough to happen.
- if (SuccBB != MBB && &*SuccPrev != MBB &&
- !SuccPrev->canFallThrough() && !CurUnAnalyzable &&
- !SuccBB->isEHPad()) {
- MBB->moveBefore(SuccBB);
- MadeChange = true;
- goto ReoptimizeBlock;
+ // If this block doesn't already fall-through to that successor, and
+ // if the succ doesn't already have a block that can fall through into
+ // it, we can arrange for the fallthrough to happen.
+ if (SuccBB != MBB && &*SuccPrev != MBB &&
+ !SuccPrev->canFallThrough()) {
+ MBB->moveBefore(SuccBB);
+ MadeChange = true;
+ goto ReoptimizeBlock;
+ }
}
}
diff --git a/llvm/lib/CodeGen/MachineBasicBlock.cpp b/llvm/lib/CodeGen/MachineBasicBlock.cpp
index e848741..2d4b604 100644
--- a/llvm/lib/CodeGen/MachineBasicBlock.cpp
+++ b/llvm/lib/CodeGen/MachineBasicBlock.cpp
@@ -277,8 +277,16 @@
}
#endif
+bool MachineBasicBlock::mayHaveInlineAsmBr() const {
+ for (const MachineBasicBlock *Succ : successors()) {
+ if (Succ->isInlineAsmBrIndirectTarget())
+ return true;
+ }
+ return false;
+}
+
bool MachineBasicBlock::isLegalToHoistInto() const {
- if (isReturnBlock() || hasEHPadSuccessor())
+ if (isReturnBlock() || hasEHPadSuccessor() || mayHaveInlineAsmBr())
return false;
return true;
}
@@ -1132,7 +1140,7 @@
// Splitting the critical edge to a callbr's indirect block isn't advised.
// Don't do it in this generic function.
- if (isInlineAsmBrIndirectTarget(Succ))
+ if (Succ->isInlineAsmBrIndirectTarget())
return false;
const MachineFunction *MF = getParent();
diff --git a/llvm/lib/CodeGen/MachineSink.cpp b/llvm/lib/CodeGen/MachineSink.cpp
index 415e38d..1d253a6 100644
--- a/llvm/lib/CodeGen/MachineSink.cpp
+++ b/llvm/lib/CodeGen/MachineSink.cpp
@@ -734,6 +734,13 @@
if (SuccToSinkTo && SuccToSinkTo->isEHPad())
return nullptr;
+ // It ought to be okay to sink instructions into an INLINEASM_BR target, but
+ // only if we make sure that MI occurs _before_ an INLINEASM_BR instruction in
+ // the source block (which this code does not yet do). So for now, forbid
+ // doing so.
+ if (SuccToSinkTo && SuccToSinkTo->isInlineAsmBrIndirectTarget())
+ return nullptr;
+
return SuccToSinkTo;
}
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index d19e295..cfc6e38 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -584,7 +584,6 @@
// it is an entry block or landing pad.
for (const auto &LI : MBB->liveins()) {
if (isAllocatable(LI.PhysReg) && !MBB->isEHPad() &&
- !MBB->isInlineAsmBrDefaultTarget() &&
MBB->getIterator() != MBB->getParent()->begin()) {
report("MBB has allocatable live-in, but isn't entry or landing-pad.", MBB);
report_context(LI.PhysReg);
@@ -730,7 +729,7 @@
continue;
// Also accept successors which are for exception-handling or might be
// inlineasm_br targets.
- if (SuccMBB->isEHPad() || MBB->isInlineAsmBrIndirectTarget(SuccMBB))
+ if (SuccMBB->isEHPad() || SuccMBB->isInlineAsmBrIndirectTarget())
continue;
report("MBB has unexpected successors which are not branch targets, "
"fallthrough, EHPads, or inlineasm_br targets.",
diff --git a/llvm/lib/CodeGen/PHIEliminationUtils.cpp b/llvm/lib/CodeGen/PHIEliminationUtils.cpp
index 3a2cdaf..bae96eb 100644
--- a/llvm/lib/CodeGen/PHIEliminationUtils.cpp
+++ b/llvm/lib/CodeGen/PHIEliminationUtils.cpp
@@ -26,8 +26,9 @@
// Usually, we just want to insert the copy before the first terminator
// instruction. However, for the edge going to a landing pad, we must insert
- // the copy before the call/invoke instruction.
- if (!SuccMBB->isEHPad())
+ // the copy before the call/invoke instruction. Similarly for an INLINEASM_BR
+ // going to an indirect target.
+ if (!SuccMBB->isEHPad() && !SuccMBB->isInlineAsmBrIndirectTarget())
return MBB->getFirstTerminator();
// Discover any defs/uses in this basic block.
diff --git a/llvm/lib/CodeGen/RegisterCoalescer.cpp b/llvm/lib/CodeGen/RegisterCoalescer.cpp
index 27b6436..17160a9 100644
--- a/llvm/lib/CodeGen/RegisterCoalescer.cpp
+++ b/llvm/lib/CodeGen/RegisterCoalescer.cpp
@@ -1064,7 +1064,9 @@
return false;
MachineBasicBlock &MBB = *CopyMI.getParent();
- if (MBB.isEHPad())
+ // If this block is the target of an invoke/inlineasm_br, moving the copy into
+ // the predecessor is tricker, and we don't handle it.
+ if (MBB.isEHPad() || MBB.isInlineAsmBrIndirectTarget())
return false;
if (MBB.pred_size() != 2)
diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
index 731cd23..ce20d50 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
@@ -1033,57 +1033,6 @@
}
}
- // Split after an INLINEASM_BR block with outputs. This allows us to keep the
- // copy to/from register instructions from being between two terminator
- // instructions, which causes the machine instruction verifier agita.
- auto TI = llvm::find_if(*BB, [](const MachineInstr &MI){
- return MI.getOpcode() == TargetOpcode::INLINEASM_BR;
- });
- auto SplicePt = TI != BB->end() ? std::next(TI) : BB->end();
- if (TI != BB->end() && SplicePt != BB->end() &&
- TI->getOpcode() == TargetOpcode::INLINEASM_BR &&
- SplicePt->getOpcode() == TargetOpcode::COPY) {
- MachineBasicBlock *FallThrough = BB->getFallThrough();
- if (!FallThrough)
- for (const MachineOperand &MO : BB->back().operands())
- if (MO.isMBB()) {
- FallThrough = MO.getMBB();
- break;
- }
- assert(FallThrough && "Cannot find default dest block for callbr!");
-
- MachineBasicBlock *CopyBB = MF.CreateMachineBasicBlock(BB->getBasicBlock());
- MachineFunction::iterator BBI(*BB);
- MF.insert(++BBI, CopyBB);
-
- CopyBB->splice(CopyBB->begin(), BB, SplicePt, BB->end());
- CopyBB->setInlineAsmBrDefaultTarget();
-
- CopyBB->addSuccessor(FallThrough, BranchProbability::getOne());
- BB->removeSuccessor(FallThrough);
- BB->addSuccessor(CopyBB, BranchProbability::getOne());
-
- // Mark all physical registers defined in the original block as being live
- // on entry to the copy block.
- for (const auto &MI : *CopyBB)
- for (const MachineOperand &MO : MI.operands())
- if (MO.isReg()) {
- Register reg = MO.getReg();
- if (Register::isPhysicalRegister(reg)) {
- CopyBB->addLiveIn(reg);
- break;
- }
- }
-
- CopyBB->normalizeSuccProbs();
- BB->normalizeSuccProbs();
-
- BB->transferInlineAsmBrIndirectTargets(CopyBB);
-
- InsertPos = CopyBB->end();
- return CopyBB;
- }
-
InsertPos = Emitter.getInsertPos();
return Emitter.getBlock();
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 476d7a7..c5d897a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -2885,14 +2885,13 @@
// Retrieve successors.
MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];
- Return->setInlineAsmBrDefaultTarget();
// Update successor info.
addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
MachineBasicBlock *Target = FuncInfo.MBBMap[I.getIndirectDest(i)];
addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
- CallBrMBB->addInlineAsmBrIndirectTarget(Target);
+ Target->setIsInlineAsmBrIndirectTarget();
}
CallBrMBB->normalizeSuccProbs();
@@ -2965,16 +2964,6 @@
for (unsigned i = 0, e = SL->BitTestCases.size(); i != e; ++i)
if (SL->BitTestCases[i].Parent == First)
SL->BitTestCases[i].Parent = Last;
-
- // SelectionDAGISel::FinishBasicBlock will add PHI operands for the
- // successors of the fallthrough block. Here, we add PHI operands for the
- // successors of the INLINEASM_BR block itself.
- if (First->getFirstTerminator()->getOpcode() == TargetOpcode::INLINEASM_BR)
- for (std::pair<MachineInstr *, unsigned> &pair : FuncInfo.PHINodesToUpdate)
- if (First->isSuccessor(pair.first->getParent()))
- MachineInstrBuilder(*First->getParent(), pair.first)
- .addReg(pair.second)
- .addMBB(First);
}
void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
@@ -7845,7 +7834,6 @@
}
};
-using SDISelAsmOperandInfoVector = SmallVector<SDISelAsmOperandInfo, 16>;
} // end anonymous namespace
@@ -8091,7 +8079,7 @@
const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
/// ConstraintOperands - Information about all of the constraints.
- SDISelAsmOperandInfoVector ConstraintOperands;
+ SmallVector<SDISelAsmOperandInfo, 16> ConstraintOperands;
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints(
diff --git a/llvm/lib/CodeGen/ShrinkWrap.cpp b/llvm/lib/CodeGen/ShrinkWrap.cpp
index 85dd4f5..ce43fb1 100644
--- a/llvm/lib/CodeGen/ShrinkWrap.cpp
+++ b/llvm/lib/CodeGen/ShrinkWrap.cpp
@@ -494,17 +494,15 @@
"EH Funclets are not supported yet.",
MBB.front().getDebugLoc(), &MBB);
- if (MBB.isEHPad()) {
- // Push the prologue and epilogue outside of
- // the region that may throw by making sure
- // that all the landing pads are at least at the
- // boundary of the save and restore points.
- // The problem with exceptions is that the throw
- // is not properly modeled and in particular, a
- // basic block can jump out from the middle.
+ if (MBB.isEHPad() || MBB.isInlineAsmBrIndirectTarget()) {
+ // Push the prologue and epilogue outside of the region that may throw (or
+ // jump out via inlineasm_br), by making sure that all the landing pads
+ // are at least at the boundary of the save and restore points. The
+ // problem is that a basic block can jump out from the middle in these
+ // cases, which we do not handle.
updateSaveRestorePoints(MBB, RS.get());
if (!ArePointsInteresting()) {
- LLVM_DEBUG(dbgs() << "EHPad prevents shrink-wrapping\n");
+ LLVM_DEBUG(dbgs() << "EHPad/inlineasm_br prevents shrink-wrapping\n");
return false;
}
continue;
diff --git a/llvm/lib/CodeGen/SplitKit.cpp b/llvm/lib/CodeGen/SplitKit.cpp
index 5a80908..8dec620 100644
--- a/llvm/lib/CodeGen/SplitKit.cpp
+++ b/llvm/lib/CodeGen/SplitKit.cpp
@@ -80,10 +80,15 @@
std::pair<SlotIndex, SlotIndex> &LIP = LastInsertPoint[Num];
SlotIndex MBBEnd = LIS.getMBBEndIdx(&MBB);
- SmallVector<const MachineBasicBlock *, 1> EHPadSuccessors;
- for (const MachineBasicBlock *SMBB : MBB.successors())
- if (SMBB->isEHPad())
- EHPadSuccessors.push_back(SMBB);
+ SmallVector<const MachineBasicBlock *, 1> ExceptionalSuccessors;
+ bool EHPadSuccessor = false;
+ for (const MachineBasicBlock *SMBB : MBB.successors()) {
+ if (SMBB->isEHPad()) {
+ ExceptionalSuccessors.push_back(SMBB);
+ EHPadSuccessor = true;
+ } else if (SMBB->isInlineAsmBrIndirectTarget())
+ ExceptionalSuccessors.push_back(SMBB);
+ }
// Compute insert points on the first call. The pair is independent of the
// current live interval.
@@ -94,15 +99,17 @@
else
LIP.first = LIS.getInstructionIndex(*FirstTerm);
- // If there is a landing pad successor, also find the call instruction.
- if (EHPadSuccessors.empty())
+ // If there is a landing pad or inlineasm_br successor, also find the
+ // instruction. If there is no such instruction, we don't need to do
+ // anything special. We assume there cannot be multiple instructions that
+ // are Calls with EHPad successors or INLINEASM_BR in a block. Further, we
+ // assume that if there are any, they will be after any other call
+ // instructions in the block.
+ if (ExceptionalSuccessors.empty())
return LIP.first;
- // There may not be a call instruction (?) in which case we ignore LPad.
- LIP.second = LIP.first;
- for (MachineBasicBlock::const_iterator I = MBB.end(), E = MBB.begin();
- I != E;) {
- --I;
- if (I->isCall()) {
+ for (auto I = MBB.rbegin(), E = MBB.rend(); I != E; ++I) {
+ if ((EHPadSuccessor && I->isCall()) ||
+ I->getOpcode() == TargetOpcode::INLINEASM_BR) {
LIP.second = LIS.getInstructionIndex(*I);
break;
}
@@ -114,7 +121,7 @@
if (!LIP.second)
return LIP.first;
- if (none_of(EHPadSuccessors, [&](const MachineBasicBlock *EHPad) {
+ if (none_of(ExceptionalSuccessors, [&](const MachineBasicBlock *EHPad) {
return LIS.isLiveInToMBB(CurLI, EHPad);
}))
return LIP.first;
diff --git a/llvm/lib/CodeGen/SplitKit.h b/llvm/lib/CodeGen/SplitKit.h
index 71ecf90..3ab5f25 100644
--- a/llvm/lib/CodeGen/SplitKit.h
+++ b/llvm/lib/CodeGen/SplitKit.h
@@ -54,7 +54,7 @@
/// Last legal insert point in each basic block in the current function.
/// The first entry is the first terminator, the second entry is the
/// last valid point to insert a split or spill for a variable that is
- /// live into a landing pad successor.
+ /// live into a landing pad or inlineasm_br successor.
SmallVector<std::pair<SlotIndex, SlotIndex>, 8> LastInsertPoint;
SlotIndex computeLastInsertPoint(const LiveInterval &CurLI,
diff --git a/llvm/lib/CodeGen/TailDuplicator.cpp b/llvm/lib/CodeGen/TailDuplicator.cpp
index 36b1809..bd55418 100644
--- a/llvm/lib/CodeGen/TailDuplicator.cpp
+++ b/llvm/lib/CodeGen/TailDuplicator.cpp
@@ -716,7 +716,7 @@
TailBB->pred_end());
bool Changed = false;
for (MachineBasicBlock *PredBB : Preds) {
- if (PredBB->hasEHPadSuccessor())
+ if (PredBB->hasEHPadSuccessor() || PredBB->mayHaveInlineAsmBr())
continue;
if (bothUsedInPHI(*PredBB, Succs))
diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp
index 9dbd612..24f3f96 100644
--- a/llvm/lib/CodeGen/TargetInstrInfo.cpp
+++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp
@@ -999,6 +999,10 @@
if (MI.isTerminator() || MI.isPosition())
return true;
+ // INLINEASM_BR can jump to another block
+ if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
+ return true;
+
// Don't attempt to schedule around any instruction that defines
// a stack-oriented pointer, as it's unlikely to be profitable. This
// saves compile time, because it doesn't require every single