Add an "original alignment" field to load and store nodes. This enables the
DAG Combiner to disambiguate chains for loads and stores of types which are
broken up by the Legalizer into smaller pieces.
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@81813 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h
index 9ab0415..064fc87 100644
--- a/include/llvm/CodeGen/SelectionDAG.h
+++ b/include/llvm/CodeGen/SelectionDAG.h
@@ -544,7 +544,7 @@
///
SDValue getLoad(EVT VT, DebugLoc dl, SDValue Chain, SDValue Ptr,
const Value *SV, int SVOffset, bool isVolatile=false,
- unsigned Alignment=0);
+ unsigned Alignment=0, unsigned OrigAlignment=0);
SDValue getExtLoad(ISD::LoadExtType ExtType, DebugLoc dl, EVT VT,
SDValue Chain, SDValue Ptr, const Value *SV,
int SVOffset, EVT EVT, bool isVolatile=false,
@@ -552,16 +552,16 @@
SDValue getIndexedLoad(SDValue OrigLoad, DebugLoc dl, SDValue Base,
SDValue Offset, ISD::MemIndexedMode AM);
SDValue getLoad(ISD::MemIndexedMode AM, DebugLoc dl, ISD::LoadExtType ExtType,
- EVT VT, SDValue Chain,
- SDValue Ptr, SDValue Offset,
- const Value *SV, int SVOffset, EVT EVT,
- bool isVolatile=false, unsigned Alignment=0);
+ EVT VT, SDValue Chain, SDValue Ptr, SDValue Offset,
+ const Value *SV, int SVOffset, EVT EVT,
+ bool isVolatile=false, unsigned Alignment=0,
+ unsigned OrigAlignment=0);
/// getStore - Helper function to build ISD::STORE nodes.
///
SDValue getStore(SDValue Chain, DebugLoc dl, SDValue Val, SDValue Ptr,
const Value *SV, int SVOffset, bool isVolatile=false,
- unsigned Alignment=0);
+ unsigned Alignment=0, unsigned OrigAlignment=0);
SDValue getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val, SDValue Ptr,
const Value *SV, int SVOffset, EVT TVT,
bool isVolatile=false, unsigned Alignment=0);
diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h
index c42ff38..a3ce5d4 100644
--- a/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -1519,17 +1519,22 @@
//! SVOffset - Memory location offset. Note that base is defined in MemSDNode
int SVOffset;
+ //! OrigAlign - The original alignment of this MemSDNode in the case where
+ // this node was created by legalize from a MemSDNode with known alignment.
+ unsigned OrigAlign;
+
public:
MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, EVT MemoryVT,
const Value *srcValue, int SVOff,
- unsigned alignment, bool isvolatile);
+ unsigned alignment, bool isvolatile, unsigned oalign);
MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, const SDValue *Ops,
unsigned NumOps, EVT MemoryVT, const Value *srcValue, int SVOff,
- unsigned alignment, bool isvolatile);
+ unsigned alignment, bool isvolatile, unsigned oalign);
/// Returns alignment and volatility of the memory access
unsigned getAlignment() const { return (1u << (SubclassData >> 6)) >> 1; }
+ unsigned getOriginalAlignment() const { return OrigAlign; }
bool isVolatile() const { return (SubclassData >> 5) & 1; }
/// getRawSubclassData - Return the SubclassData value, which contains an
@@ -1600,14 +1605,14 @@
SDValue Cmp, SDValue Swp, const Value* SrcVal,
unsigned Align=0)
: MemSDNode(Opc, dl, VTL, MemVT, SrcVal, /*SVOffset=*/0,
- Align, /*isVolatile=*/true) {
+ Align, /*isVolatile=*/true, /* OrigAlign=*/0) {
InitOperands(Ops, Chain, Ptr, Cmp, Swp);
}
AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT,
SDValue Chain, SDValue Ptr,
SDValue Val, const Value* SrcVal, unsigned Align=0)
: MemSDNode(Opc, dl, VTL, MemVT, SrcVal, /*SVOffset=*/0,
- Align, /*isVolatile=*/true) {
+ Align, /*isVolatile=*/true, /* OrigAlign=*/0) {
InitOperands(Ops, Chain, Ptr, Val);
}
@@ -1648,7 +1653,8 @@
const SDValue *Ops, unsigned NumOps,
EVT MemoryVT, const Value *srcValue, int SVO,
unsigned Align, bool Vol, bool ReadMem, bool WriteMem)
- : MemSDNode(Opc, dl, VTs, Ops, NumOps, MemoryVT, srcValue, SVO, Align, Vol),
+ : MemSDNode(Opc, dl, VTs, Ops, NumOps, MemoryVT, srcValue, SVO, Align, Vol,
+ /* OrigAlign=*/0),
ReadMem(ReadMem), WriteMem(WriteMem) {
}
@@ -2263,8 +2269,9 @@
public:
LSBaseSDNode(ISD::NodeType NodeTy, DebugLoc dl, SDValue *Operands,
unsigned numOperands, SDVTList VTs, ISD::MemIndexedMode AM,
- EVT VT, const Value *SV, int SVO, unsigned Align, bool Vol)
- : MemSDNode(NodeTy, dl, VTs, VT, SV, SVO, Align, Vol) {
+ EVT VT, const Value *SV, int SVO, unsigned Align, bool Vol,
+ unsigned OAlign)
+ : MemSDNode(NodeTy, dl, VTs, VT, SV, SVO, Align, Vol, OAlign) {
assert(Align != 0 && "Loads and stores should have non-zero aligment");
SubclassData |= AM << 2;
assert(getAddressingMode() == AM && "MemIndexedMode encoding error!");
@@ -2302,9 +2309,10 @@
friend class SelectionDAG;
LoadSDNode(SDValue *ChainPtrOff, DebugLoc dl, SDVTList VTs,
ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT LVT,
- const Value *SV, int O=0, unsigned Align=0, bool Vol=false)
+ const Value *SV, int O=0, unsigned Align=0, bool Vol=false,
+ unsigned OAlign=0)
: LSBaseSDNode(ISD::LOAD, dl, ChainPtrOff, 3,
- VTs, AM, LVT, SV, O, Align, Vol) {
+ VTs, AM, LVT, SV, O, Align, Vol, OAlign) {
SubclassData |= (unsigned short)ETy;
assert(getExtensionType() == ETy && "LoadExtType encoding error!");
}
@@ -2331,9 +2339,10 @@
friend class SelectionDAG;
StoreSDNode(SDValue *ChainValuePtrOff, DebugLoc dl, SDVTList VTs,
ISD::MemIndexedMode AM, bool isTrunc, EVT SVT,
- const Value *SV, int O=0, unsigned Align=0, bool Vol=false)
+ const Value *SV, int O=0, unsigned Align=0, bool Vol=false,
+ unsigned OAlign=0)
: LSBaseSDNode(ISD::STORE, dl, ChainValuePtrOff, 4,
- VTs, AM, SVT, SV, O, Align, Vol) {
+ VTs, AM, SVT, SV, O, Align, Vol, OAlign) {
SubclassData |= (unsigned short)isTrunc;
assert(isTruncatingStore() == isTrunc && "isTrunc encoding error!");
}
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 0a523fa..014e62e 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -3646,9 +3646,12 @@
ISD::LoadExtType ExtType, EVT VT, SDValue Chain,
SDValue Ptr, SDValue Offset,
const Value *SV, int SVOffset, EVT EVT,
- bool isVolatile, unsigned Alignment) {
+ bool isVolatile, unsigned Alignment,
+ unsigned OrigAlignment) {
if (Alignment == 0) // Ensure that codegen never sees alignment 0
Alignment = getEVTAlignment(VT);
+ if (OrigAlignment == 0)
+ OrigAlignment = Alignment;
if (VT == EVT) {
ExtType = ISD::NON_EXTLOAD;
@@ -3679,12 +3682,13 @@
AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3);
ID.AddInteger(EVT.getRawBits());
ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, isVolatile, Alignment));
+ ID.AddInteger(OrigAlignment);
void *IP = 0;
if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
return SDValue(E, 0);
SDNode *N = NodeAllocator.Allocate<LoadSDNode>();
new (N) LoadSDNode(Ops, dl, VTs, AM, ExtType, EVT, SV, SVOffset,
- Alignment, isVolatile);
+ Alignment, isVolatile, OrigAlignment);
CSEMap.InsertNode(N, IP);
AllNodes.push_back(N);
return SDValue(N, 0);
@@ -3693,10 +3697,11 @@
SDValue SelectionDAG::getLoad(EVT VT, DebugLoc dl,
SDValue Chain, SDValue Ptr,
const Value *SV, int SVOffset,
- bool isVolatile, unsigned Alignment) {
+ bool isVolatile, unsigned Alignment,
+ unsigned OrigAlignment) {
SDValue Undef = getUNDEF(Ptr.getValueType());
return getLoad(ISD::UNINDEXED, dl, ISD::NON_EXTLOAD, VT, Chain, Ptr, Undef,
- SV, SVOffset, VT, isVolatile, Alignment);
+ SV, SVOffset, VT, isVolatile, Alignment, OrigAlignment);
}
SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, DebugLoc dl, EVT VT,
@@ -3723,11 +3728,14 @@
SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val,
SDValue Ptr, const Value *SV, int SVOffset,
- bool isVolatile, unsigned Alignment) {
+ bool isVolatile, unsigned Alignment,
+ unsigned OrigAlignment) {
EVT VT = Val.getValueType();
if (Alignment == 0) // Ensure that codegen never sees alignment 0
Alignment = getEVTAlignment(VT);
+ if (OrigAlignment == 0)
+ OrigAlignment = Alignment;
SDVTList VTs = getVTList(MVT::Other);
SDValue Undef = getUNDEF(Ptr.getValueType());
@@ -3737,12 +3745,13 @@
ID.AddInteger(VT.getRawBits());
ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED,
isVolatile, Alignment));
+ ID.AddInteger(OrigAlignment);
void *IP = 0;
if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
return SDValue(E, 0);
SDNode *N = NodeAllocator.Allocate<StoreSDNode>();
new (N) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, false,
- VT, SV, SVOffset, Alignment, isVolatile);
+ VT, SV, SVOffset, Alignment, isVolatile, OrigAlignment);
CSEMap.InsertNode(N, IP);
AllNodes.push_back(N);
return SDValue(N, 0);
@@ -4968,9 +4977,10 @@
}
MemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, EVT memvt,
- const Value *srcValue, int SVO,
- unsigned alignment, bool vol)
- : SDNode(Opc, dl, VTs), MemoryVT(memvt), SrcValue(srcValue), SVOffset(SVO) {
+ const Value *srcValue, int SVO, unsigned alignment,
+ bool vol, unsigned origAlign)
+ : SDNode(Opc, dl, VTs), MemoryVT(memvt), SrcValue(srcValue), SVOffset(SVO),
+ OrigAlign(origAlign) {
SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, vol, alignment);
assert(isPowerOf2_32(alignment) && "Alignment is not a power of 2!");
assert(getAlignment() == alignment && "Alignment representation error!");
@@ -4978,11 +4988,11 @@
}
MemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs,
- const SDValue *Ops,
- unsigned NumOps, EVT memvt, const Value *srcValue,
- int SVO, unsigned alignment, bool vol)
+ const SDValue *Ops, unsigned NumOps, EVT memvt,
+ const Value *srcValue, int SVO, unsigned alignment,
+ bool vol, unsigned origAlign)
: SDNode(Opc, dl, VTs, Ops, NumOps),
- MemoryVT(memvt), SrcValue(srcValue), SVOffset(SVO) {
+ MemoryVT(memvt), SrcValue(srcValue), SVOffset(SVO), OrigAlign(origAlign) {
SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, vol, alignment);
assert(isPowerOf2_32(alignment) && "Alignment is not a power of 2!");
assert(getAlignment() == alignment && "Alignment representation error!");
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuild.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuild.cpp
index ad59912..8b9a5a5 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGBuild.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuild.cpp
@@ -2812,11 +2812,10 @@
EVT PtrVT = Ptr.getValueType();
for (unsigned i = 0; i != NumValues; ++i) {
SDValue L = DAG.getLoad(ValueVTs[i], getCurDebugLoc(), Root,
- DAG.getNode(ISD::ADD, getCurDebugLoc(),
- PtrVT, Ptr,
- DAG.getConstant(Offsets[i], PtrVT)),
- SV, Offsets[i],
- isVolatile, Alignment);
+ DAG.getNode(ISD::ADD, getCurDebugLoc(),
+ PtrVT, Ptr,
+ DAG.getConstant(Offsets[i], PtrVT)),
+ SV, Offsets[i], isVolatile, Alignment, Alignment);
Values[i] = L;
Chains[i] = L.getValue(1);
}
@@ -2866,7 +2865,7 @@
PtrVT, Ptr,
DAG.getConstant(Offsets[i], PtrVT)),
PtrV, Offsets[i],
- isVolatile, Alignment);
+ isVolatile, Alignment, Alignment);
DAG.setRoot(DAG.getNode(ISD::TokenFactor, getCurDebugLoc(),
MVT::Other, &Chains[0], NumValues));