[MachineMemOperand] Move synchronization scope and atomic orderings from SDNode to MachineMemOperand, and remove redundant getAtomic* member functions from SelectionDAG.
Differential Revision: https://reviews.llvm.org/D24577
llvm-svn: 284312
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index 5fb8be7..8cb5b19 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -306,9 +306,12 @@
MachineMemOperand *MachineFunction::getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s,
- unsigned base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges) {
+ unsigned base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
+ SynchronizationScope SynchScope, AtomicOrdering Ordering,
+ AtomicOrdering FailureOrdering) {
return new (Allocator)
- MachineMemOperand(PtrInfo, f, s, base_alignment, AAInfo, Ranges);
+ MachineMemOperand(PtrInfo, f, s, base_alignment, AAInfo, Ranges,
+ SynchScope, Ordering, FailureOrdering);
}
MachineMemOperand *
@@ -318,13 +321,15 @@
return new (Allocator)
MachineMemOperand(MachinePointerInfo(MMO->getValue(),
MMO->getOffset()+Offset),
- MMO->getFlags(), Size,
- MMO->getBaseAlignment());
+ MMO->getFlags(), Size, MMO->getBaseAlignment(),
+ AAMDNodes(), nullptr, MMO->getSynchScope(),
+ MMO->getOrdering(), MMO->getFailureOrdering());
return new (Allocator)
MachineMemOperand(MachinePointerInfo(MMO->getPseudoValue(),
MMO->getOffset()+Offset),
- MMO->getFlags(), Size,
- MMO->getBaseAlignment());
+ MMO->getFlags(), Size, MMO->getBaseAlignment(),
+ AAMDNodes(), nullptr, MMO->getSynchScope(),
+ MMO->getOrdering(), MMO->getFailureOrdering());
}
MachineInstr::mmo_iterator
@@ -355,7 +360,9 @@
getMachineMemOperand((*I)->getPointerInfo(),
(*I)->getFlags() & ~MachineMemOperand::MOStore,
(*I)->getSize(), (*I)->getBaseAlignment(),
- (*I)->getAAInfo());
+ (*I)->getAAInfo(), nullptr,
+ (*I)->getSynchScope(), (*I)->getOrdering(),
+ (*I)->getFailureOrdering());
Result[Index] = JustLoad;
}
++Index;
@@ -387,7 +394,9 @@
getMachineMemOperand((*I)->getPointerInfo(),
(*I)->getFlags() & ~MachineMemOperand::MOLoad,
(*I)->getSize(), (*I)->getBaseAlignment(),
- (*I)->getAAInfo());
+ (*I)->getAAInfo(), nullptr,
+ (*I)->getSynchScope(), (*I)->getOrdering(),
+ (*I)->getFailureOrdering());
Result[Index] = JustStore;
}
++Index;
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 473be32..dca0c93 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -537,7 +537,10 @@
MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
uint64_t s, unsigned int a,
const AAMDNodes &AAInfo,
- const MDNode *Ranges)
+ const MDNode *Ranges,
+ SynchronizationScope SynchScope,
+ AtomicOrdering Ordering,
+ AtomicOrdering FailureOrdering)
: PtrInfo(ptrinfo), Size(s), FlagVals(f), BaseAlignLog2(Log2_32(a) + 1),
AAInfo(AAInfo), Ranges(Ranges) {
assert((PtrInfo.V.isNull() || PtrInfo.V.is<const PseudoSourceValue*>() ||
@@ -545,6 +548,13 @@
"invalid pointer value");
assert(getBaseAlignment() == a && "Alignment is not a power of 2!");
assert((isLoad() || isStore()) && "Not a load/store!");
+
+ AtomicInfo.SynchScope = static_cast<unsigned>(SynchScope);
+ assert(getSynchScope() == SynchScope && "Value truncated");
+ AtomicInfo.Ordering = static_cast<unsigned>(Ordering);
+ assert(getOrdering() == Ordering && "Value truncated");
+ AtomicInfo.FailureOrdering = static_cast<unsigned>(FailureOrdering);
+ assert(getFailureOrdering() == FailureOrdering && "Value truncated");
}
/// Profile - Gather unique data for the object.
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index a0c8097..66d8008 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -2835,10 +2835,7 @@
SDValue Swap = DAG.getAtomicCmpSwap(
ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
Node->getOperand(0), Node->getOperand(1), Zero, Zero,
- cast<AtomicSDNode>(Node)->getMemOperand(),
- cast<AtomicSDNode>(Node)->getOrdering(),
- cast<AtomicSDNode>(Node)->getOrdering(),
- cast<AtomicSDNode>(Node)->getSynchScope());
+ cast<AtomicSDNode>(Node)->getMemOperand());
Results.push_back(Swap.getValue(0));
Results.push_back(Swap.getValue(1));
break;
@@ -2849,9 +2846,7 @@
cast<AtomicSDNode>(Node)->getMemoryVT(),
Node->getOperand(0),
Node->getOperand(1), Node->getOperand(2),
- cast<AtomicSDNode>(Node)->getMemOperand(),
- cast<AtomicSDNode>(Node)->getOrdering(),
- cast<AtomicSDNode>(Node)->getSynchScope());
+ cast<AtomicSDNode>(Node)->getMemOperand());
Results.push_back(Swap.getValue(1));
break;
}
@@ -2863,10 +2858,7 @@
SDValue Res = DAG.getAtomicCmpSwap(
ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
- Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand(),
- cast<AtomicSDNode>(Node)->getSuccessOrdering(),
- cast<AtomicSDNode>(Node)->getFailureOrdering(),
- cast<AtomicSDNode>(Node)->getSynchScope());
+ Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
SDValue ExtRes = Res;
SDValue LHS = Res;
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index d081db5..41ca0b4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -188,8 +188,7 @@
SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
N->getMemoryVT(), ResVT,
N->getChain(), N->getBasePtr(),
- N->getMemOperand(), N->getOrdering(),
- N->getSynchScope());
+ N->getMemOperand());
// Legalize the chain result - switch anything that used the old chain to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
@@ -201,8 +200,7 @@
SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
N->getMemoryVT(),
N->getChain(), N->getBasePtr(),
- Op2, N->getMemOperand(), N->getOrdering(),
- N->getSynchScope());
+ Op2, N->getMemOperand());
// Legalize the chain result - switch anything that used the old chain to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
@@ -225,8 +223,7 @@
SDValue Res = DAG.getAtomicCmpSwap(
ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs,
N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
- N->getMemOperand(), N->getSuccessOrdering(), N->getFailureOrdering(),
- N->getSynchScope());
+ N->getMemOperand());
ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
return Res.getValue(1);
@@ -238,8 +235,7 @@
DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
SDValue Res = DAG.getAtomicCmpSwap(
N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
- N->getBasePtr(), Op2, Op3, N->getMemOperand(), N->getSuccessOrdering(),
- N->getFailureOrdering(), N->getSynchScope());
+ N->getBasePtr(), Op2, Op3, N->getMemOperand());
// Update the use to N with the newly created Res.
for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
ReplaceValueWith(SDValue(N, i), Res.getValue(i));
@@ -997,8 +993,7 @@
SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
SDValue Op2 = GetPromotedInteger(N->getOperand(2));
return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
- N->getChain(), N->getBasePtr(), Op2, N->getMemOperand(),
- N->getOrdering(), N->getSynchScope());
+ N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
}
SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
@@ -1368,8 +1363,7 @@
SDValue Tmp = DAG.getAtomicCmpSwap(
ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
- AN->getMemOperand(), AN->getSuccessOrdering(), AN->getFailureOrdering(),
- AN->getSynchScope());
+ AN->getMemOperand());
// Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
// success simply by comparing the loaded value against the ingoing
@@ -2733,10 +2727,7 @@
SDValue Swap = DAG.getAtomicCmpSwap(
ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
- N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand(),
- cast<AtomicSDNode>(N)->getOrdering(),
- cast<AtomicSDNode>(N)->getOrdering(),
- cast<AtomicSDNode>(N)->getSynchScope());
+ N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
@@ -3224,9 +3215,7 @@
cast<AtomicSDNode>(N)->getMemoryVT(),
N->getOperand(0),
N->getOperand(1), N->getOperand(2),
- cast<AtomicSDNode>(N)->getMemOperand(),
- cast<AtomicSDNode>(N)->getOrdering(),
- cast<AtomicSDNode>(N)->getSynchScope());
+ cast<AtomicSDNode>(N)->getMemOperand());
return Swap.getValue(1);
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 3671422..19fae7f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -4831,10 +4831,7 @@
SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
SDVTList VTList, ArrayRef<SDValue> Ops,
- MachineMemOperand *MMO,
- AtomicOrdering SuccessOrdering,
- AtomicOrdering FailureOrdering,
- SynchronizationScope SynchScope) {
+ MachineMemOperand *MMO) {
FoldingSetNodeID ID;
ID.AddInteger(MemVT.getRawBits());
AddNodeIDNode(ID, Opcode, VTList, Ops);
@@ -4846,8 +4843,7 @@
}
auto *N = newSDNode<AtomicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
- VTList, MemVT, MMO, SuccessOrdering,
- FailureOrdering, SynchScope);
+ VTList, MemVT, MMO);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP);
@@ -4855,14 +4851,6 @@
return SDValue(N, 0);
}
-SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
- SDVTList VTList, ArrayRef<SDValue> Ops,
- MachineMemOperand *MMO, AtomicOrdering Ordering,
- SynchronizationScope SynchScope) {
- return getAtomic(Opcode, dl, MemVT, VTList, Ops, MMO, Ordering,
- Ordering, SynchScope);
-}
-
SDValue SelectionDAG::getAtomicCmpSwap(
unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain,
SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo,
@@ -4882,26 +4870,23 @@
auto Flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad |
MachineMemOperand::MOStore;
MachineMemOperand *MMO =
- MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment);
+ MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment,
+ AAMDNodes(), nullptr, SynchScope, SuccessOrdering,
+ FailureOrdering);
- return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO,
- SuccessOrdering, FailureOrdering, SynchScope);
+ return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO);
}
SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
EVT MemVT, SDVTList VTs, SDValue Chain,
SDValue Ptr, SDValue Cmp, SDValue Swp,
- MachineMemOperand *MMO,
- AtomicOrdering SuccessOrdering,
- AtomicOrdering FailureOrdering,
- SynchronizationScope SynchScope) {
+ MachineMemOperand *MMO) {
assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
- return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO,
- SuccessOrdering, FailureOrdering, SynchScope);
+ return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
}
SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
@@ -4927,16 +4912,15 @@
MachineMemOperand *MMO =
MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
- MemVT.getStoreSize(), Alignment);
+ MemVT.getStoreSize(), Alignment, AAMDNodes(),
+ nullptr, SynchScope, Ordering);
- return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO,
- Ordering, SynchScope);
+ return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO);
}
SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
SDValue Chain, SDValue Ptr, SDValue Val,
- MachineMemOperand *MMO, AtomicOrdering Ordering,
- SynchronizationScope SynchScope) {
+ MachineMemOperand *MMO) {
assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
Opcode == ISD::ATOMIC_LOAD_SUB ||
Opcode == ISD::ATOMIC_LOAD_AND ||
@@ -4956,18 +4940,17 @@
SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
getVTList(VT, MVT::Other);
SDValue Ops[] = {Chain, Ptr, Val};
- return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
+ return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
}
SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
EVT VT, SDValue Chain, SDValue Ptr,
- MachineMemOperand *MMO, AtomicOrdering Ordering,
- SynchronizationScope SynchScope) {
+ MachineMemOperand *MMO) {
assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
SDVTList VTs = getVTList(VT, MVT::Other);
SDValue Ops[] = {Chain, Ptr};
- return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
+ return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
}
/// getMergeValues - Create a MERGE_VALUES node from the given operands.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 1f4dfc4..bad8370 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3973,13 +3973,13 @@
MachineMemOperand::MOLoad,
VT.getStoreSize(),
I.getAlignment() ? I.getAlignment() :
- DAG.getEVTAlignment(VT));
+ DAG.getEVTAlignment(VT),
+ AAMDNodes(), nullptr, Scope, Order);
InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
SDValue L =
DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
- getValue(I.getPointerOperand()), MMO,
- Order, Scope);
+ getValue(I.getPointerOperand()), MMO);
SDValue OutChain = L.getValue(1);