[CodeGen] Split out the notions of MI invariance and MI dereferenceability.
Summary:
An IR load can be invariant, dereferenceable, neither, or both. But
currently, MI's notion of invariance is IR-invariant &&
IR-dereferenceable.
This patch splits up the notions of invariance and dereferenceability at
the MI level. It's NFC, so adds some probably-unnecessary
"is-dereferenceable" checks, which we can remove later if desired.
Reviewers: chandlerc, tstellarAMD
Subscribers: jholewinski, arsenm, nemanjai, llvm-commits
Differential Revision: https://reviews.llvm.org/D23371
llvm-svn: 281151
diff --git a/llvm/lib/CodeGen/MIRParser/MILexer.cpp b/llvm/lib/CodeGen/MIRParser/MILexer.cpp
index 0850ae7..d94d6a5 100644
--- a/llvm/lib/CodeGen/MIRParser/MILexer.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MILexer.cpp
@@ -222,6 +222,7 @@
.Case("target-flags", MIToken::kw_target_flags)
.Case("volatile", MIToken::kw_volatile)
.Case("non-temporal", MIToken::kw_non_temporal)
+ .Case("dereferenceable", MIToken::kw_dereferenceable)
.Case("invariant", MIToken::kw_invariant)
.Case("align", MIToken::kw_align)
.Case("stack", MIToken::kw_stack)
diff --git a/llvm/lib/CodeGen/MIRParser/MILexer.h b/llvm/lib/CodeGen/MIRParser/MILexer.h
index c0d9561..6911700 100644
--- a/llvm/lib/CodeGen/MIRParser/MILexer.h
+++ b/llvm/lib/CodeGen/MIRParser/MILexer.h
@@ -54,6 +54,7 @@
kw_implicit_define,
kw_def,
kw_dead,
+ kw_dereferenceable,
kw_killed,
kw_undef,
kw_internal,
@@ -166,7 +167,7 @@
bool isMemoryOperandFlag() const {
return Kind == kw_volatile || Kind == kw_non_temporal ||
- Kind == kw_invariant;
+ Kind == kw_dereferenceable || Kind == kw_invariant;
}
bool is(TokenKind K) const { return Kind == K; }
diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
index 4dc964e..696ba48 100644
--- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
@@ -1766,6 +1766,9 @@
case MIToken::kw_non_temporal:
Flags |= MachineMemOperand::MONonTemporal;
break;
+ case MIToken::kw_dereferenceable:
+ Flags |= MachineMemOperand::MODereferenceable;
+ break;
case MIToken::kw_invariant:
Flags |= MachineMemOperand::MOInvariant;
break;
diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp
index b981f59..e293a07 100644
--- a/llvm/lib/CodeGen/MIRPrinter.cpp
+++ b/llvm/lib/CodeGen/MIRPrinter.cpp
@@ -892,6 +892,8 @@
OS << "volatile ";
if (Op.isNonTemporal())
OS << "non-temporal ";
+ if (Op.isDereferenceable())
+ OS << "dereferenceable ";
if (Op.isInvariant())
OS << "invariant ";
if (Op.isLoad())
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 90ebf2c..fb9bd8f 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -650,10 +650,10 @@
OS << ")";
}
- // Print nontemporal info.
if (isNonTemporal())
OS << "(nontemporal)";
-
+ if (isDereferenceable())
+ OS << "(dereferenceable)";
if (isInvariant())
OS << "(invariant)";
}
@@ -1580,7 +1580,8 @@
for (MachineMemOperand *MMO : memoperands()) {
if (MMO->isVolatile()) return false;
if (MMO->isStore()) return false;
- if (MMO->isInvariant()) continue;
+ if (MMO->isInvariant() && MMO->isDereferenceable())
+ continue;
// A load from a constant PseudoSourceValue is invariant.
if (const PseudoSourceValue *PSV = MMO->getPseudoValue())
diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp
index dc61eba..45e9d50 100644
--- a/llvm/lib/CodeGen/MachinePipeliner.cpp
+++ b/llvm/lib/CodeGen/MachinePipeliner.cpp
@@ -3082,7 +3082,8 @@
MachineInstr::mmo_iterator NewMemRefs = MF.allocateMemRefsArray(NumRefs);
unsigned Refs = 0;
for (MachineMemOperand *MMO : NewMI.memoperands()) {
- if (MMO->isVolatile() || MMO->isInvariant() || (!MMO->getValue())) {
+ if (MMO->isVolatile() || (MMO->isInvariant() && MMO->isDereferenceable()) ||
+ (!MMO->getValue())) {
NewMemRefs[Refs++] = MMO;
continue;
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 249abb9..8783bb2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -14280,6 +14280,8 @@
MachineMemOperand::Flags MMOFlags = LLD->getMemOperand()->getFlags();
if (!RLD->isInvariant())
MMOFlags &= ~MachineMemOperand::MOInvariant;
+ if (!RLD->isDereferenceable())
+ MMOFlags &= ~MachineMemOperand::MODereferenceable;
if (LLD->getExtensionType() == ISD::NON_EXTLOAD) {
// FIXME: Discards pointer and AA info.
Load = DAG.getLoad(TheSelect->getValueType(0), SDLoc(TheSelect),
diff --git a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
index 1d961af..d25ea4e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
@@ -2183,6 +2183,8 @@
bool IsNonTemporal = I->getMetadata(LLVMContext::MD_nontemporal) != nullptr;
bool IsInvariant = I->getMetadata(LLVMContext::MD_invariant_load) != nullptr;
+ bool IsDereferenceable =
+ I->getMetadata(LLVMContext::MD_dereferenceable) != nullptr;
const MDNode *Ranges = I->getMetadata(LLVMContext::MD_range);
AAMDNodes AAInfo;
@@ -2197,6 +2199,8 @@
Flags |= MachineMemOperand::MOVolatile;
if (IsNonTemporal)
Flags |= MachineMemOperand::MONonTemporal;
+ if (IsDereferenceable)
+ Flags |= MachineMemOperand::MODereferenceable;
if (IsInvariant)
Flags |= MachineMemOperand::MOInvariant;
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 31ebf7b..3716365 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -632,7 +632,8 @@
SDLoc dl(N);
auto MMOFlags =
- L->getMemOperand()->getFlags() & ~MachineMemOperand::MOInvariant;
+ L->getMemOperand()->getFlags() &
+ ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
SDValue NewL;
if (L->getExtensionType() == ISD::NON_EXTLOAD) {
NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl,
@@ -2085,7 +2086,8 @@
// Load the value as an integer value with the same number of bits.
EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
auto MMOFlags =
- L->getMemOperand()->getFlags() & ~MachineMemOperand::MOInvariant;
+ L->getMemOperand()->getFlags() &
+ ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
SDValue newL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), IVT,
SDLoc(N), L->getChain(), L->getBasePtr(),
L->getOffset(), L->getPointerInfo(), IVT,
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 991b19b..f050eb5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -5167,9 +5167,10 @@
ISD::MemIndexedMode AM) {
LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
- // Don't propagate the invariant flag.
+ // Don't propagate the invariant or dereferenceable flags.
auto MMOFlags =
- LD->getMemOperand()->getFlags() & ~MachineMemOperand::MOInvariant;
+ LD->getMemOperand()->getFlags() &
+ ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
LD->getChain(), Base, Offset, LD->getPointerInfo(),
LD->getMemoryVT(), LD->getAlignment(), MMOFlags,
@@ -6716,6 +6717,7 @@
: SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
MemSDNodeBits.IsVolatile = MMO->isVolatile();
MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal();
+ MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable();
MemSDNodeBits.IsInvariant = MMO->isInvariant();
// We check here that the size of the memory operand fits within the size of
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 5e9bc0d..e67915b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -2010,7 +2010,8 @@
if (Global) {
MachinePointerInfo MPInfo(Global);
MachineInstr::mmo_iterator MemRefs = MF.allocateMemRefsArray(1);
- auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant;
+ auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
+ MachineMemOperand::MODereferenceable;
*MemRefs = MF.getMachineMemOperand(MPInfo, Flags, PtrTy.getSizeInBits() / 8,
DAG.getEVTAlignment(PtrTy));
Node->setMemRefs(MemRefs, MemRefs + 1);
@@ -3473,17 +3474,8 @@
bool isVolatile = I.isVolatile();
bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
-
- // The IR notion of invariant_load only guarantees that all *non-faulting*
- // invariant loads result in the same value. The MI notion of invariant load
- // guarantees that the load can be legally moved to any location within its
- // containing function. The MI notion of invariant_load is stronger than the
- // IR notion of invariant_load -- an MI invariant_load is an IR invariant_load
- // with a guarantee that the location being loaded from is dereferenceable
- // throughout the function's lifetime.
-
- bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr &&
- isDereferenceablePointer(SV, DAG.getDataLayout());
+ bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr;
+ bool isDereferenceable = isDereferenceablePointer(SV, DAG.getDataLayout());
unsigned Alignment = I.getAlignment();
AAMDNodes AAInfo;
@@ -3551,6 +3543,8 @@
MMOFlags |= MachineMemOperand::MONonTemporal;
if (isInvariant)
MMOFlags |= MachineMemOperand::MOInvariant;
+ if (isDereferenceable)
+ MMOFlags |= MachineMemOperand::MODereferenceable;
SDValue L = DAG.getLoad(ValueVTs[i], dl, Root, A,
MachinePointerInfo(SV, Offsets[i]), Alignment,