[LiveRange] Reset the VNIs when splitting subranges
When splitting a subrange we end up with two different subranges covering
two different, non overlapping, lanes.
As part of this splitting the VNIs of the original live-range need
to be dispatched to the subranges according to which lanes they are
actually defining.
Prior to this patch we were assuming that all values were defining
all lanes. This was wrong as demonstrated by llvm.org/PR40835.
Differential Revision: https://reviews.llvm.org/D59731
llvm-svn: 357032
diff --git a/llvm/lib/CodeGen/LiveInterval.cpp b/llvm/lib/CodeGen/LiveInterval.cpp
index 821680e..3dc0303 100644
--- a/llvm/lib/CodeGen/LiveInterval.cpp
+++ b/llvm/lib/CodeGen/LiveInterval.cpp
@@ -879,8 +879,53 @@
SubRanges = nullptr;
}
-void LiveInterval::refineSubRanges(BumpPtrAllocator &Allocator,
- LaneBitmask LaneMask, std::function<void(LiveInterval::SubRange&)> Apply) {
+/// For each VNI in \p SR, check whether or not that value defines part
+/// of the mask describe by \p LaneMask and if not, remove that value
+/// from \p SR.
+static void stripValuesNotDefiningMask(unsigned Reg, LiveInterval::SubRange &SR,
+ LaneBitmask LaneMask,
+ const SlotIndexes &Indexes,
+ const TargetRegisterInfo &TRI) {
+ // Phys reg should not be tracked at subreg level.
+ // Same for noreg (Reg == 0).
+ if (!TargetRegisterInfo::isVirtualRegister(Reg) || !Reg)
+ return;
+ // Remove the values that don't define those lanes.
+ SmallVector<VNInfo *, 8> ToBeRemoved;
+ for (VNInfo *VNI : SR.valnos) {
+ if (VNI->isUnused())
+ continue;
+ // PHI definitions don't have MI attached, so there is nothing
+ // we can use to strip the VNI.
+ if (VNI->isPHIDef())
+ continue;
+ const MachineInstr *MI = Indexes.getInstructionFromIndex(VNI->def);
+ assert(MI && "Cannot find the definition of a value");
+ bool hasDef = false;
+ for (ConstMIBundleOperands MOI(*MI); MOI.isValid(); ++MOI) {
+ if (!MOI->isReg() || !MOI->isDef())
+ continue;
+ if (MOI->getReg() != Reg)
+ continue;
+ if ((TRI.getSubRegIndexLaneMask(MOI->getSubReg()) & LaneMask).none())
+ continue;
+ hasDef = true;
+ break;
+ }
+
+ if (!hasDef)
+ ToBeRemoved.push_back(VNI);
+ }
+ for (VNInfo *VNI : ToBeRemoved)
+ SR.removeValNo(VNI);
+
+ assert(!SR.empty() && "At least one value should be defined by this mask");
+}
+
+void LiveInterval::refineSubRanges(
+ BumpPtrAllocator &Allocator, LaneBitmask LaneMask,
+ std::function<void(LiveInterval::SubRange &)> Apply,
+ const SlotIndexes &Indexes, const TargetRegisterInfo &TRI) {
LaneBitmask ToApply = LaneMask;
for (SubRange &SR : subranges()) {
LaneBitmask SRMask = SR.LaneMask;
@@ -898,6 +943,10 @@
SR.LaneMask = SRMask & ~Matching;
// Create a new subrange for the matching part
MatchingRange = createSubRangeFrom(Allocator, Matching, SR);
+ // Now that the subrange is split in half, make sure we
+ // only keep in the subranges the VNIs that touch the related half.
+ stripValuesNotDefiningMask(reg, *MatchingRange, Matching, Indexes, TRI);
+ stripValuesNotDefiningMask(reg, SR, SR.LaneMask, Indexes, TRI);
}
Apply(*MatchingRange);
ToApply &= ~Matching;
diff --git a/llvm/lib/CodeGen/LiveRangeCalc.cpp b/llvm/lib/CodeGen/LiveRangeCalc.cpp
index 0020221..d670f28 100644
--- a/llvm/lib/CodeGen/LiveRangeCalc.cpp
+++ b/llvm/lib/CodeGen/LiveRangeCalc.cpp
@@ -95,10 +95,11 @@
}
LI.refineSubRanges(*Alloc, SubMask,
- [&MO, this](LiveInterval::SubRange &SR) {
- if (MO.isDef())
- createDeadDef(*Indexes, *Alloc, SR, MO);
- });
+ [&MO, this](LiveInterval::SubRange &SR) {
+ if (MO.isDef())
+ createDeadDef(*Indexes, *Alloc, SR, MO);
+ },
+ *Indexes, TRI);
}
// Create the def in the main liverange. We do not have to do this if
diff --git a/llvm/lib/CodeGen/RegisterCoalescer.cpp b/llvm/lib/CodeGen/RegisterCoalescer.cpp
index 4f10777..40b7e8b 100644
--- a/llvm/lib/CodeGen/RegisterCoalescer.cpp
+++ b/llvm/lib/CodeGen/RegisterCoalescer.cpp
@@ -924,23 +924,25 @@
}
SlotIndex AIdx = CopyIdx.getRegSlot(true);
LaneBitmask MaskA;
+ const SlotIndexes &Indexes = *LIS->getSlotIndexes();
for (LiveInterval::SubRange &SA : IntA.subranges()) {
VNInfo *ASubValNo = SA.getVNInfoAt(AIdx);
assert(ASubValNo != nullptr);
MaskA |= SA.LaneMask;
- IntB.refineSubRanges(Allocator, SA.LaneMask,
- [&Allocator,&SA,CopyIdx,ASubValNo,&ShrinkB]
- (LiveInterval::SubRange &SR) {
- VNInfo *BSubValNo = SR.empty()
- ? SR.getNextValue(CopyIdx, Allocator)
- : SR.getVNInfoAt(CopyIdx);
- assert(BSubValNo != nullptr);
- auto P = addSegmentsWithValNo(SR, BSubValNo, SA, ASubValNo);
- ShrinkB |= P.second;
- if (P.first)
- BSubValNo->def = ASubValNo->def;
- });
+ IntB.refineSubRanges(
+ Allocator, SA.LaneMask,
+ [&Allocator, &SA, CopyIdx, ASubValNo,
+ &ShrinkB](LiveInterval::SubRange &SR) {
+ VNInfo *BSubValNo = SR.empty() ? SR.getNextValue(CopyIdx, Allocator)
+ : SR.getVNInfoAt(CopyIdx);
+ assert(BSubValNo != nullptr);
+ auto P = addSegmentsWithValNo(SR, BSubValNo, SA, ASubValNo);
+ ShrinkB |= P.second;
+ if (P.first)
+ BSubValNo->def = ASubValNo->def;
+ },
+ Indexes, *TRI);
}
// Go over all subranges of IntB that have not been covered by IntA,
// and delete the segments starting at CopyIdx. This can happen if
@@ -3262,16 +3264,18 @@
LaneBitmask LaneMask,
CoalescerPair &CP) {
BumpPtrAllocator &Allocator = LIS->getVNInfoAllocator();
- LI.refineSubRanges(Allocator, LaneMask,
- [this,&Allocator,&ToMerge,&CP](LiveInterval::SubRange &SR) {
- if (SR.empty()) {
- SR.assign(ToMerge, Allocator);
- } else {
- // joinSubRegRange() destroys the merged range, so we need a copy.
- LiveRange RangeCopy(ToMerge, Allocator);
- joinSubRegRanges(SR, RangeCopy, SR.LaneMask, CP);
- }
- });
+ LI.refineSubRanges(
+ Allocator, LaneMask,
+ [this, &Allocator, &ToMerge, &CP](LiveInterval::SubRange &SR) {
+ if (SR.empty()) {
+ SR.assign(ToMerge, Allocator);
+ } else {
+ // joinSubRegRange() destroys the merged range, so we need a copy.
+ LiveRange RangeCopy(ToMerge, Allocator);
+ joinSubRegRanges(SR, RangeCopy, SR.LaneMask, CP);
+ }
+ },
+ *LIS->getSlotIndexes(), *TRI);
}
bool RegisterCoalescer::isHighCostLiveInterval(LiveInterval &LI) {
diff --git a/llvm/lib/CodeGen/SplitKit.cpp b/llvm/lib/CodeGen/SplitKit.cpp
index 0bdbdce3..5c944fe 100644
--- a/llvm/lib/CodeGen/SplitKit.cpp
+++ b/llvm/lib/CodeGen/SplitKit.cpp
@@ -520,17 +520,18 @@
.addReg(FromReg, 0, SubIdx);
BumpPtrAllocator &Allocator = LIS.getVNInfoAllocator();
+ SlotIndexes &Indexes = *LIS.getSlotIndexes();
if (FirstCopy) {
- SlotIndexes &Indexes = *LIS.getSlotIndexes();
Def = Indexes.insertMachineInstrInMaps(*CopyMI, Late).getRegSlot();
} else {
CopyMI->bundleWithPred();
}
LaneBitmask LaneMask = TRI.getSubRegIndexLaneMask(SubIdx);
DestLI.refineSubRanges(Allocator, LaneMask,
- [Def, &Allocator](LiveInterval::SubRange& SR) {
- SR.createDeadDef(Def, Allocator);
- });
+ [Def, &Allocator](LiveInterval::SubRange &SR) {
+ SR.createDeadDef(Def, Allocator);
+ },
+ Indexes, TRI);
return Def;
}