Improve MachineMemOperand handling.
- Allocate MachineMemOperands and MachineMemOperand lists in MachineFunctions.
This eliminates MachineInstr's std::list member and allows the data to be
created by isel and live for the remainder of codegen, avoiding a lot of
copying and unnecessary translation. This also shrinks MemSDNode.
- Delete MemOperandSDNode. Introduce MachineSDNode which has dedicated
fields for MachineMemOperands.
- Change MemSDNode to have a MachineMemOperand member instead of its own
fields with the same information. This introduces some redundancy, but
it's more consistent with what MachineInstr will eventually want.
- Ignore alignment when searching for redundant loads for CSE, but remember
the greatest alignment.
Target-specific code which previously used MemOperandSDNodes with generic
SDNodes now use MemIntrinsicSDNodes, with opcodes in a designated range
so that the SelectionDAG framework knows that MachineMemOperand information
is available.
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@82794 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/lib/CodeGen/LiveIntervalAnalysis.cpp b/lib/CodeGen/LiveIntervalAnalysis.cpp
index 825cf39..e1b23fd 100644
--- a/lib/CodeGen/LiveIntervalAnalysis.cpp
+++ b/lib/CodeGen/LiveIntervalAnalysis.cpp
@@ -25,6 +25,7 @@
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/Passes.h"
#include "llvm/CodeGen/PseudoSourceValue.h"
@@ -1443,12 +1444,12 @@
// If the instruction accesses memory and the memory could be non-constant,
// assume the instruction is not rematerializable.
- for (std::list<MachineMemOperand>::const_iterator
- I = MI->memoperands_begin(), E = MI->memoperands_end(); I != E; ++I){
- const MachineMemOperand &MMO = *I;
- if (MMO.isVolatile() || MMO.isStore())
+ for (MachineInstr::mmo_iterator I = MI->memoperands_begin(),
+ E = MI->memoperands_end(); I != E; ++I){
+ const MachineMemOperand *MMO = *I;
+ if (MMO->isVolatile() || MMO->isStore())
return false;
- const Value *V = MMO.getValue();
+ const Value *V = MMO->getValue();
if (!V)
return false;
if (const PseudoSourceValue *PSV = dyn_cast<PseudoSourceValue>(V)) {
diff --git a/lib/CodeGen/MachineFunction.cpp b/lib/CodeGen/MachineFunction.cpp
index d0773ff..3a47265 100644
--- a/lib/CodeGen/MachineFunction.cpp
+++ b/lib/CodeGen/MachineFunction.cpp
@@ -190,11 +190,6 @@
///
void
MachineFunction::DeleteMachineInstr(MachineInstr *MI) {
- // Clear the instructions memoperands. This must be done manually because
- // the instruction's parent pointer is now null, so it can't properly
- // deallocate them on its own.
- MI->clearMemOperands(*this);
-
MI->~MachineInstr();
InstructionRecycler.Deallocate(Allocator, MI);
}
@@ -217,6 +212,29 @@
BasicBlockRecycler.Deallocate(Allocator, MBB);
}
+MachineMemOperand *
+MachineFunction::getMachineMemOperand(const Value *v, unsigned f,
+ int64_t o, uint64_t s,
+ unsigned base_alignment) {
+ return new (Allocator.Allocate<MachineMemOperand>())
+ MachineMemOperand(v, f, o, s, base_alignment);
+}
+
+MachineMemOperand *
+MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
+ int64_t Offset, uint64_t Size) {
+ return new (Allocator.Allocate<MachineMemOperand>())
+ MachineMemOperand(MMO->getValue(), MMO->getFlags(),
+ int64_t(uint64_t(MMO->getOffset()) +
+ uint64_t(Offset)),
+ Size, MMO->getBaseAlignment());
+}
+
+MachineInstr::mmo_iterator
+MachineFunction::allocateMemRefsArray(unsigned long Num) {
+ return Allocator.Allocate<MachineMemOperand *>(Num);
+}
+
void MachineFunction::dump() const {
print(errs());
}
diff --git a/lib/CodeGen/MachineInstr.cpp b/lib/CodeGen/MachineInstr.cpp
index baab120..2d06aa4 100644
--- a/lib/CodeGen/MachineInstr.cpp
+++ b/lib/CodeGen/MachineInstr.cpp
@@ -17,6 +17,7 @@
#include "llvm/Value.h"
#include "llvm/Assembly/Writer.h"
#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/PseudoSourceValue.h"
#include "llvm/Target/TargetMachine.h"
@@ -298,40 +299,56 @@
ID.AddInteger(Flags);
}
-raw_ostream &llvm::operator<<(raw_ostream &OS, const MachineMemOperand &MRO) {
- assert((MRO.isLoad() || MRO.isStore()) &&
+void MachineMemOperand::refineAlignment(const MachineMemOperand *MMO) {
+ // The Value and Offset may differ due to CSE. But the flags and size
+ // should be the same.
+ assert(MMO->getFlags() == getFlags() && "Flags mismatch!");
+ assert(MMO->getSize() == getSize() && "Size mismatch!");
+
+ if (MMO->getBaseAlignment() >= getBaseAlignment()) {
+ // Update the alignment value.
+ Flags = (Flags & 7) | ((Log2_32(MMO->getBaseAlignment()) + 1) << 3);
+ // Also update the base and offset, because the new alignment may
+ // not be applicable with the old ones.
+ V = MMO->getValue();
+ Offset = MMO->getOffset();
+ }
+}
+
+raw_ostream &llvm::operator<<(raw_ostream &OS, const MachineMemOperand &MMO) {
+ assert((MMO.isLoad() || MMO.isStore()) &&
"SV has to be a load, store or both.");
- if (MRO.isVolatile())
+ if (MMO.isVolatile())
OS << "Volatile ";
- if (MRO.isLoad())
+ if (MMO.isLoad())
OS << "LD";
- if (MRO.isStore())
+ if (MMO.isStore())
OS << "ST";
- OS << MRO.getSize();
+ OS << MMO.getSize();
// Print the address information.
OS << "[";
- if (!MRO.getValue())
+ if (!MMO.getValue())
OS << "<unknown>";
else
- WriteAsOperand(OS, MRO.getValue(), /*PrintType=*/false);
+ WriteAsOperand(OS, MMO.getValue(), /*PrintType=*/false);
// If the alignment of the memory reference itself differs from the alignment
// of the base pointer, print the base alignment explicitly, next to the base
// pointer.
- if (MRO.getBaseAlignment() != MRO.getAlignment())
- OS << "(align=" << MRO.getBaseAlignment() << ")";
+ if (MMO.getBaseAlignment() != MMO.getAlignment())
+ OS << "(align=" << MMO.getBaseAlignment() << ")";
- if (MRO.getOffset() != 0)
- OS << "+" << MRO.getOffset();
+ if (MMO.getOffset() != 0)
+ OS << "+" << MMO.getOffset();
OS << "]";
// Print the alignment of the reference.
- if (MRO.getBaseAlignment() != MRO.getAlignment() ||
- MRO.getBaseAlignment() != MRO.getSize())
- OS << "(align=" << MRO.getAlignment() << ")";
+ if (MMO.getBaseAlignment() != MMO.getAlignment() ||
+ MMO.getBaseAlignment() != MMO.getSize())
+ OS << "(align=" << MMO.getAlignment() << ")";
return OS;
}
@@ -343,7 +360,8 @@
/// MachineInstr ctor - This constructor creates a dummy MachineInstr with
/// TID NULL and no operands.
MachineInstr::MachineInstr()
- : TID(0), NumImplicitOps(0), Parent(0), debugLoc(DebugLoc::getUnknownLoc()) {
+ : TID(0), NumImplicitOps(0), MemRefs(0), MemRefsEnd(0),
+ Parent(0), debugLoc(DebugLoc::getUnknownLoc()) {
// Make sure that we get added to a machine basicblock
LeakDetector::addGarbageObject(this);
}
@@ -362,7 +380,7 @@
/// TargetInstrDesc or the numOperands if it is not zero. (for
/// instructions with variable number of operands).
MachineInstr::MachineInstr(const TargetInstrDesc &tid, bool NoImp)
- : TID(&tid), NumImplicitOps(0), Parent(0),
+ : TID(&tid), NumImplicitOps(0), MemRefs(0), MemRefsEnd(0), Parent(0),
debugLoc(DebugLoc::getUnknownLoc()) {
if (!NoImp && TID->getImplicitDefs())
for (const unsigned *ImpDefs = TID->getImplicitDefs(); *ImpDefs; ++ImpDefs)
@@ -380,7 +398,8 @@
/// MachineInstr ctor - As above, but with a DebugLoc.
MachineInstr::MachineInstr(const TargetInstrDesc &tid, const DebugLoc dl,
bool NoImp)
- : TID(&tid), NumImplicitOps(0), Parent(0), debugLoc(dl) {
+ : TID(&tid), NumImplicitOps(0), MemRefs(0), MemRefsEnd(0),
+ Parent(0), debugLoc(dl) {
if (!NoImp && TID->getImplicitDefs())
for (const unsigned *ImpDefs = TID->getImplicitDefs(); *ImpDefs; ++ImpDefs)
NumImplicitOps++;
@@ -399,7 +418,7 @@
/// basic block.
///
MachineInstr::MachineInstr(MachineBasicBlock *MBB, const TargetInstrDesc &tid)
- : TID(&tid), NumImplicitOps(0), Parent(0),
+ : TID(&tid), NumImplicitOps(0), MemRefs(0), MemRefsEnd(0), Parent(0),
debugLoc(DebugLoc::getUnknownLoc()) {
assert(MBB && "Cannot use inserting ctor with null basic block!");
if (TID->ImplicitDefs)
@@ -419,7 +438,8 @@
///
MachineInstr::MachineInstr(MachineBasicBlock *MBB, const DebugLoc dl,
const TargetInstrDesc &tid)
- : TID(&tid), NumImplicitOps(0), Parent(0), debugLoc(dl) {
+ : TID(&tid), NumImplicitOps(0), MemRefs(0), MemRefsEnd(0),
+ Parent(0), debugLoc(dl) {
assert(MBB && "Cannot use inserting ctor with null basic block!");
if (TID->ImplicitDefs)
for (const unsigned *ImpDefs = TID->getImplicitDefs(); *ImpDefs; ++ImpDefs)
@@ -437,8 +457,9 @@
/// MachineInstr ctor - Copies MachineInstr arg exactly
///
MachineInstr::MachineInstr(MachineFunction &MF, const MachineInstr &MI)
- : TID(&MI.getDesc()), NumImplicitOps(0), Parent(0),
- debugLoc(MI.getDebugLoc()) {
+ : TID(&MI.getDesc()), NumImplicitOps(0),
+ MemRefs(MI.MemRefs), MemRefsEnd(MI.MemRefsEnd),
+ Parent(0), debugLoc(MI.getDebugLoc()) {
Operands.reserve(MI.getNumOperands());
// Add operands
@@ -446,11 +467,6 @@
addOperand(MI.getOperand(i));
NumImplicitOps = MI.NumImplicitOps;
- // Add memory operands.
- for (std::list<MachineMemOperand>::const_iterator i = MI.memoperands_begin(),
- j = MI.memoperands_end(); i != j; ++i)
- addMemOperand(MF, *i);
-
// Set parent to null.
Parent = 0;
@@ -459,8 +475,6 @@
MachineInstr::~MachineInstr() {
LeakDetector::removeGarbageObject(this);
- assert(MemOperands.empty() &&
- "MachineInstr being deleted with live memoperands!");
#ifndef NDEBUG
for (unsigned i = 0, e = Operands.size(); i != e; ++i) {
assert(Operands[i].ParentMI == this && "ParentMI mismatch!");
@@ -621,18 +635,24 @@
}
}
-/// addMemOperand - Add a MachineMemOperand to the machine instruction,
-/// referencing arbitrary storage.
+/// addMemOperand - Add a MachineMemOperand to the machine instruction.
+/// This function should be used only occasionally. The setMemRefs function
+/// is the primary method for setting up a MachineInstr's MemRefs list.
void MachineInstr::addMemOperand(MachineFunction &MF,
- const MachineMemOperand &MO) {
- MemOperands.push_back(MO);
-}
+ MachineMemOperand *MO) {
+ mmo_iterator OldMemRefs = MemRefs;
+ mmo_iterator OldMemRefsEnd = MemRefsEnd;
-/// clearMemOperands - Erase all of this MachineInstr's MachineMemOperands.
-void MachineInstr::clearMemOperands(MachineFunction &MF) {
- MemOperands.clear();
-}
+ size_t NewNum = (MemRefsEnd - MemRefs) + 1;
+ mmo_iterator NewMemRefs = MF.allocateMemRefsArray(NewNum);
+ mmo_iterator NewMemRefsEnd = NewMemRefs + NewNum;
+ std::copy(OldMemRefs, OldMemRefsEnd, NewMemRefs);
+ NewMemRefs[NewNum - 1] = MO;
+
+ MemRefs = NewMemRefs;
+ MemRefsEnd = NewMemRefsEnd;
+}
/// removeFromParent - This method unlinks 'this' from the containing basic
/// block, and returns it, but does not delete it.
@@ -972,9 +992,8 @@
return true;
// Check the memory reference information for volatile references.
- for (std::list<MachineMemOperand>::const_iterator I = memoperands_begin(),
- E = memoperands_end(); I != E; ++I)
- if (I->isVolatile())
+ for (mmo_iterator I = memoperands_begin(), E = memoperands_end(); I != E; ++I)
+ if ((*I)->isVolatile())
return true;
return false;
@@ -1004,9 +1023,9 @@
if (!memoperands_empty()) {
OS << ", Mem:";
- for (std::list<MachineMemOperand>::const_iterator i = memoperands_begin(),
- e = memoperands_end(); i != e; ++i) {
- OS << *i;
+ for (mmo_iterator i = memoperands_begin(), e = memoperands_end();
+ i != e; ++i) {
+ OS << **i;
if (next(i) != e)
OS << " ";
}
diff --git a/lib/CodeGen/ScheduleDAGEmit.cpp b/lib/CodeGen/ScheduleDAGEmit.cpp
index 770f5bb..0d15c02 100644
--- a/lib/CodeGen/ScheduleDAGEmit.cpp
+++ b/lib/CodeGen/ScheduleDAGEmit.cpp
@@ -28,10 +28,6 @@
#include "llvm/Support/MathExtras.h"
using namespace llvm;
-void ScheduleDAG::AddMemOperand(MachineInstr *MI, const MachineMemOperand &MO) {
- MI->addMemOperand(MF, MO);
-}
-
void ScheduleDAG::EmitNoop() {
TII->insertNoop(*BB, InsertPos);
}
diff --git a/lib/CodeGen/ScheduleDAGInstrs.cpp b/lib/CodeGen/ScheduleDAGInstrs.cpp
index b065d60..b55e606 100644
--- a/lib/CodeGen/ScheduleDAGInstrs.cpp
+++ b/lib/CodeGen/ScheduleDAGInstrs.cpp
@@ -17,6 +17,7 @@
#include "llvm/Operator.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/PseudoSourceValue.h"
#include "llvm/Target/TargetMachine.h"
@@ -96,11 +97,11 @@
/// object, return the Value for that object. Otherwise return null.
static const Value *getUnderlyingObjectForInstr(const MachineInstr *MI) {
if (!MI->hasOneMemOperand() ||
- !MI->memoperands_begin()->getValue() ||
- MI->memoperands_begin()->isVolatile())
+ !(*MI->memoperands_begin())->getValue() ||
+ (*MI->memoperands_begin())->isVolatile())
return 0;
- const Value *V = MI->memoperands_begin()->getValue();
+ const Value *V = (*MI->memoperands_begin())->getValue();
if (!V)
return 0;
@@ -335,10 +336,10 @@
if (!ChainTID.isCall() &&
!ChainTID.hasUnmodeledSideEffects() &&
ChainMI->hasOneMemOperand() &&
- !ChainMI->memoperands_begin()->isVolatile() &&
- ChainMI->memoperands_begin()->getValue())
+ !(*ChainMI->memoperands_begin())->isVolatile() &&
+ (*ChainMI->memoperands_begin())->getValue())
// We know that the Chain accesses one specific memory location.
- ChainMMO = &*ChainMI->memoperands_begin();
+ ChainMMO = *ChainMI->memoperands_begin();
else
// Unknown memory accesses. Assume the worst.
ChainMMO = 0;
diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
index 6c6e96a..3e2101a 100644
--- a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
+++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
@@ -263,19 +263,10 @@
}
/// CountOperands - The inputs to target nodes have any actual inputs first,
-/// followed by special operands that describe memory references, then an
-/// optional chain operand, then an optional flag operand. Compute the number
-/// of actual operands that will go into the resulting MachineInstr.
+/// followed by an optional chain operand, then an optional flag operand.
+/// Compute the number of actual operands that will go into the resulting
+/// MachineInstr.
unsigned ScheduleDAGSDNodes::CountOperands(SDNode *Node) {
- unsigned N = ComputeMemOperandsEnd(Node);
- while (N && isa<MemOperandSDNode>(Node->getOperand(N - 1).getNode()))
- --N; // Ignore MEMOPERAND nodes
- return N;
-}
-
-/// ComputeMemOperandsEnd - Find the index one past the last MemOperandSDNode
-/// operand
-unsigned ScheduleDAGSDNodes::ComputeMemOperandsEnd(SDNode *Node) {
unsigned N = Node->getNumOperands();
while (N && Node->getOperand(N - 1).getValueType() == MVT::Flag)
--N;
@@ -284,7 +275,6 @@
return N;
}
-
void ScheduleDAGSDNodes::dumpNode(const SUnit *SU) const {
if (!SU->getNode()) {
errs() << "PHYS REG COPY\n";
diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h
index 51041ef..0a6816a 100644
--- a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h
+++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h
@@ -58,7 +58,6 @@
if (isa<ConstantPoolSDNode>(Node)) return true;
if (isa<JumpTableSDNode>(Node)) return true;
if (isa<ExternalSymbolSDNode>(Node)) return true;
- if (isa<MemOperandSDNode>(Node)) return true;
if (Node->getOpcode() == ISD::EntryToken) return true;
return false;
}
@@ -99,15 +98,11 @@
static unsigned CountResults(SDNode *Node);
/// CountOperands - The inputs to target nodes have any actual inputs first,
- /// followed by special operands that describe memory references, then an
- /// optional chain operand, then flag operands. Compute the number of
- /// actual operands that will go into the resulting MachineInstr.
+ /// followed by an optional chain operand, then flag operands. Compute
+ /// the number of actual operands that will go into the resulting
+ /// MachineInstr.
static unsigned CountOperands(SDNode *Node);
- /// ComputeMemOperandsEnd - Find the index one past the last
- /// MemOperandSDNode operand
- static unsigned ComputeMemOperandsEnd(SDNode *Node);
-
/// EmitNode - Generate machine code for an node and needed dependencies.
/// VRBaseMap contains, for each already emitted node, the first virtual
/// register number for the results of the node.
diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodesEmit.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodesEmit.cpp
index c404873..c9e40ff 100644
--- a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodesEmit.cpp
+++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodesEmit.cpp
@@ -497,7 +497,6 @@
const TargetInstrDesc &II = TII->get(Opc);
unsigned NumResults = CountResults(Node);
unsigned NodeOperands = CountOperands(Node);
- unsigned MemOperandsEnd = ComputeMemOperandsEnd(Node);
bool HasPhysRegOuts = (NumResults > II.getNumDefs()) &&
II.getImplicitDefs() != 0;
#ifndef NDEBUG
@@ -525,9 +524,9 @@
AddOperand(MI, Node->getOperand(i), i-NumSkip+II.getNumDefs(), &II,
VRBaseMap);
- // Emit all of the memory operands of this instruction
- for (unsigned i = NodeOperands; i != MemOperandsEnd; ++i)
- AddMemOperand(MI,cast<MemOperandSDNode>(Node->getOperand(i+NumSkip))->MO);
+ // Transfer all of the memory reference descriptions of this instruction.
+ MI->setMemRefs(cast<MachineSDNode>(Node)->memoperands_begin(),
+ cast<MachineSDNode>(Node)->memoperands_end());
if (II.usesCustomDAGSchedInsertionHook()) {
// Insert this instruction into the basic block using a target
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 2b2e911..f6fed21 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -402,11 +402,6 @@
case ISD::SRCVALUE:
ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
break;
- case ISD::MEMOPERAND: {
- const MachineMemOperand &MO = cast<MemOperandSDNode>(N)->MO;
- MO.Profile(ID);
- break;
- }
case ISD::FrameIndex:
case ISD::TargetFrameIndex:
ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
@@ -481,20 +476,18 @@
}
/// encodeMemSDNodeFlags - Generic routine for computing a value for use in
-/// the CSE map that carries alignment, volatility, indexing mode, and
+/// the CSE map that carries volatility, indexing mode, and
/// extension/truncation information.
///
static inline unsigned
-encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM,
- bool isVolatile, unsigned Alignment) {
+encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile) {
assert((ConvType & 3) == ConvType &&
"ConvType may not require more than 2 bits!");
assert((AM & 7) == AM &&
"AM may not require more than 3 bits!");
return ConvType |
(AM << 2) |
- (isVolatile << 5) |
- ((Log2_32(Alignment) + 1) << 6);
+ (isVolatile << 5);
}
//===----------------------------------------------------------------------===//
@@ -1330,28 +1323,6 @@
return SDValue(N, 0);
}
-SDValue SelectionDAG::getMemOperand(const MachineMemOperand &MO) {
-#ifndef NDEBUG
- const Value *v = MO.getValue();
- assert((!v || isa<PointerType>(v->getType())) &&
- "SrcValue is not a pointer?");
-#endif
-
- FoldingSetNodeID ID;
- AddNodeIDNode(ID, ISD::MEMOPERAND, getVTList(MVT::Other), 0, 0);
- MO.Profile(ID);
-
- void *IP = 0;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
- return SDValue(E, 0);
-
- SDNode *N = NodeAllocator.Allocate<MemOperandSDNode>();
- new (N) MemOperandSDNode(MO);
- CSEMap.InsertNode(N, IP);
- AllNodes.push_back(N);
- return SDValue(N, 0);
-}
-
/// getShiftAmountOperand - Return the specified value casted to
/// the target's desired shift amount type.
SDValue SelectionDAG::getShiftAmountOperand(SDValue Op) {
@@ -3523,25 +3494,49 @@
SDValue Ptr, SDValue Cmp,
SDValue Swp, const Value* PtrVal,
unsigned Alignment) {
+ if (Alignment == 0) // Ensure that codegen never sees alignment 0
+ Alignment = getEVTAlignment(MemVT);
+
+ // Check if the memory reference references a frame index
+ if (!PtrVal)
+ if (const FrameIndexSDNode *FI =
+ dyn_cast<const FrameIndexSDNode>(Ptr.getNode()))
+ PtrVal = PseudoSourceValue::getFixedStack(FI->getIndex());
+
+ MachineFunction &MF = getMachineFunction();
+ unsigned Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
+
+ // For now, atomics are considered to be volatile always.
+ Flags |= MachineMemOperand::MOVolatile;
+
+ MachineMemOperand *MMO =
+ MF.getMachineMemOperand(PtrVal, Flags, 0,
+ MemVT.getStoreSize(), Alignment);
+
+ return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO);
+}
+
+SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
+ SDValue Chain,
+ SDValue Ptr, SDValue Cmp,
+ SDValue Swp, MachineMemOperand *MMO) {
assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op");
assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
EVT VT = Cmp.getValueType();
- if (Alignment == 0) // Ensure that codegen never sees alignment 0
- Alignment = getEVTAlignment(MemVT);
-
SDVTList VTs = getVTList(VT, MVT::Other);
FoldingSetNodeID ID;
ID.AddInteger(MemVT.getRawBits());
SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
AddNodeIDNode(ID, Opcode, VTs, Ops, 4);
void* IP = 0;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
+ cast<AtomicSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0);
+ }
SDNode* N = NodeAllocator.Allocate<AtomicSDNode>();
- new (N) AtomicSDNode(Opcode, dl, VTs, MemVT,
- Chain, Ptr, Cmp, Swp, PtrVal, Alignment);
+ new (N) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, Ptr, Cmp, Swp, MMO);
CSEMap.InsertNode(N, IP);
AllNodes.push_back(N);
return SDValue(N, 0);
@@ -3552,6 +3547,32 @@
SDValue Ptr, SDValue Val,
const Value* PtrVal,
unsigned Alignment) {
+ if (Alignment == 0) // Ensure that codegen never sees alignment 0
+ Alignment = getEVTAlignment(MemVT);
+
+ // Check if the memory reference references a frame index
+ if (!PtrVal)
+ if (const FrameIndexSDNode *FI =
+ dyn_cast<const FrameIndexSDNode>(Ptr.getNode()))
+ PtrVal = PseudoSourceValue::getFixedStack(FI->getIndex());
+
+ MachineFunction &MF = getMachineFunction();
+ unsigned Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
+
+ // For now, atomics are considered to be volatile always.
+ Flags |= MachineMemOperand::MOVolatile;
+
+ MachineMemOperand *MMO =
+ MF.getMachineMemOperand(PtrVal, Flags, 0,
+ MemVT.getStoreSize(), Alignment);
+
+ return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO);
+}
+
+SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
+ SDValue Chain,
+ SDValue Ptr, SDValue Val,
+ MachineMemOperand *MMO) {
assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
Opcode == ISD::ATOMIC_LOAD_SUB ||
Opcode == ISD::ATOMIC_LOAD_AND ||
@@ -3567,20 +3588,18 @@
EVT VT = Val.getValueType();
- if (Alignment == 0) // Ensure that codegen never sees alignment 0
- Alignment = getEVTAlignment(MemVT);
-
SDVTList VTs = getVTList(VT, MVT::Other);
FoldingSetNodeID ID;
ID.AddInteger(MemVT.getRawBits());
SDValue Ops[] = {Chain, Ptr, Val};
AddNodeIDNode(ID, Opcode, VTs, Ops, 3);
void* IP = 0;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
+ cast<AtomicSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0);
+ }
SDNode* N = NodeAllocator.Allocate<AtomicSDNode>();
- new (N) AtomicSDNode(Opcode, dl, VTs, MemVT,
- Chain, Ptr, Val, PtrVal, Alignment);
+ new (N) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, Ptr, Val, MMO);
CSEMap.InsertNode(N, IP);
AllNodes.push_back(N);
return SDValue(N, 0);
@@ -3619,23 +3638,51 @@
EVT MemVT, const Value *srcValue, int SVOff,
unsigned Align, bool Vol,
bool ReadMem, bool WriteMem) {
+ if (Align == 0) // Ensure that codegen never sees alignment 0
+ Align = getEVTAlignment(MemVT);
+
+ MachineFunction &MF = getMachineFunction();
+ unsigned Flags = 0;
+ if (WriteMem)
+ Flags |= MachineMemOperand::MOStore;
+ if (ReadMem)
+ Flags |= MachineMemOperand::MOLoad;
+ if (Vol)
+ Flags |= MachineMemOperand::MOVolatile;
+ MachineMemOperand *MMO =
+ MF.getMachineMemOperand(srcValue, Flags, SVOff,
+ MemVT.getStoreSize(), Align);
+
+ return getMemIntrinsicNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO);
+}
+
+SDValue
+SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList,
+ const SDValue *Ops, unsigned NumOps,
+ EVT MemVT, MachineMemOperand *MMO) {
+ assert((Opcode == ISD::INTRINSIC_VOID ||
+ Opcode == ISD::INTRINSIC_W_CHAIN ||
+ (Opcode <= INT_MAX &&
+ (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
+ "Opcode is not a memory-accessing opcode!");
+
// Memoize the node unless it returns a flag.
MemIntrinsicSDNode *N;
if (VTList.VTs[VTList.NumVTs-1] != MVT::Flag) {
FoldingSetNodeID ID;
AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps);
void *IP = 0;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
+ cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0);
+ }
N = NodeAllocator.Allocate<MemIntrinsicSDNode>();
- new (N) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, MemVT,
- srcValue, SVOff, Align, Vol, ReadMem, WriteMem);
+ new (N) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO);
CSEMap.InsertNode(N, IP);
} else {
N = NodeAllocator.Allocate<MemIntrinsicSDNode>();
- new (N) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, MemVT,
- srcValue, SVOff, Align, Vol, ReadMem, WriteMem);
+ new (N) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO);
}
AllNodes.push_back(N);
return SDValue(N, 0);
@@ -3650,6 +3697,27 @@
if (Alignment == 0) // Ensure that codegen never sees alignment 0
Alignment = getEVTAlignment(VT);
+ // Check if the memory reference references a frame index
+ if (!SV)
+ if (const FrameIndexSDNode *FI =
+ dyn_cast<const FrameIndexSDNode>(Ptr.getNode()))
+ SV = PseudoSourceValue::getFixedStack(FI->getIndex());
+
+ MachineFunction &MF = getMachineFunction();
+ unsigned Flags = MachineMemOperand::MOLoad;
+ if (isVolatile)
+ Flags |= MachineMemOperand::MOVolatile;
+ MachineMemOperand *MMO =
+ MF.getMachineMemOperand(SV, Flags, SVOffset,
+ MemVT.getStoreSize(), Alignment);
+ return getLoad(AM, dl, ExtType, VT, Chain, Ptr, Offset, MemVT, MMO);
+}
+
+SDValue
+SelectionDAG::getLoad(ISD::MemIndexedMode AM, DebugLoc dl,
+ ISD::LoadExtType ExtType, EVT VT, SDValue Chain,
+ SDValue Ptr, SDValue Offset, EVT MemVT,
+ MachineMemOperand *MMO) {
if (VT == MemVT) {
ExtType = ISD::NON_EXTLOAD;
} else if (ExtType == ISD::NON_EXTLOAD) {
@@ -3678,13 +3746,14 @@
FoldingSetNodeID ID;
AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3);
ID.AddInteger(MemVT.getRawBits());
- ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, isVolatile, Alignment));
+ ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile()));
void *IP = 0;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
+ cast<LoadSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0);
+ }
SDNode *N = NodeAllocator.Allocate<LoadSDNode>();
- new (N) LoadSDNode(Ops, dl, VTs, AM, ExtType, MemVT, SV, SVOffset,
- Alignment, isVolatile);
+ new (N) LoadSDNode(Ops, dl, VTs, AM, ExtType, MemVT, MMO);
CSEMap.InsertNode(N, IP);
AllNodes.push_back(N);
return SDValue(N, 0);
@@ -3724,25 +3793,43 @@
SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val,
SDValue Ptr, const Value *SV, int SVOffset,
bool isVolatile, unsigned Alignment) {
- EVT VT = Val.getValueType();
-
if (Alignment == 0) // Ensure that codegen never sees alignment 0
- Alignment = getEVTAlignment(VT);
+ Alignment = getEVTAlignment(Val.getValueType());
+ // Check if the memory reference references a frame index
+ if (!SV)
+ if (const FrameIndexSDNode *FI =
+ dyn_cast<const FrameIndexSDNode>(Ptr.getNode()))
+ SV = PseudoSourceValue::getFixedStack(FI->getIndex());
+
+ MachineFunction &MF = getMachineFunction();
+ unsigned Flags = MachineMemOperand::MOStore;
+ if (isVolatile)
+ Flags |= MachineMemOperand::MOVolatile;
+ MachineMemOperand *MMO =
+ MF.getMachineMemOperand(SV, Flags, SVOffset,
+ Val.getValueType().getStoreSize(), Alignment);
+
+ return getStore(Chain, dl, Val, Ptr, MMO);
+}
+
+SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val,
+ SDValue Ptr, MachineMemOperand *MMO) {
+ EVT VT = Val.getValueType();
SDVTList VTs = getVTList(MVT::Other);
SDValue Undef = getUNDEF(Ptr.getValueType());
SDValue Ops[] = { Chain, Val, Ptr, Undef };
FoldingSetNodeID ID;
AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4);
ID.AddInteger(VT.getRawBits());
- ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED,
- isVolatile, Alignment));
+ ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile()));
void *IP = 0;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
+ cast<StoreSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0);
+ }
SDNode *N = NodeAllocator.Allocate<StoreSDNode>();
- new (N) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, false,
- VT, SV, SVOffset, Alignment, isVolatile);
+ new (N) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, false, VT, MMO);
CSEMap.InsertNode(N, IP);
AllNodes.push_back(N);
return SDValue(N, 0);
@@ -3752,17 +3839,37 @@
SDValue Ptr, const Value *SV,
int SVOffset, EVT SVT,
bool isVolatile, unsigned Alignment) {
+ if (Alignment == 0) // Ensure that codegen never sees alignment 0
+ Alignment = getEVTAlignment(SVT);
+
+ // Check if the memory reference references a frame index
+ if (!SV)
+ if (const FrameIndexSDNode *FI =
+ dyn_cast<const FrameIndexSDNode>(Ptr.getNode()))
+ SV = PseudoSourceValue::getFixedStack(FI->getIndex());
+
+ MachineFunction &MF = getMachineFunction();
+ unsigned Flags = MachineMemOperand::MOStore;
+ if (isVolatile)
+ Flags |= MachineMemOperand::MOVolatile;
+ MachineMemOperand *MMO =
+ MF.getMachineMemOperand(SV, Flags, SVOffset, SVT.getStoreSize(), Alignment);
+
+ return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
+}
+
+SDValue SelectionDAG::getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val,
+ SDValue Ptr, EVT SVT,
+ MachineMemOperand *MMO) {
EVT VT = Val.getValueType();
if (VT == SVT)
- return getStore(Chain, dl, Val, Ptr, SV, SVOffset, isVolatile, Alignment);
+ return getStore(Chain, dl, Val, Ptr, MMO);
assert(VT.bitsGT(SVT) && "Not a truncation?");
assert(VT.isInteger() == SVT.isInteger() &&
"Can't do FP-INT conversion!");
- if (Alignment == 0) // Ensure that codegen never sees alignment 0
- Alignment = getEVTAlignment(VT);
SDVTList VTs = getVTList(MVT::Other);
SDValue Undef = getUNDEF(Ptr.getValueType());
@@ -3770,14 +3877,14 @@
FoldingSetNodeID ID;
AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4);
ID.AddInteger(SVT.getRawBits());
- ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED,
- isVolatile, Alignment));
+ ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile()));
void *IP = 0;
- if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
+ cast<StoreSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0);
+ }
SDNode *N = NodeAllocator.Allocate<StoreSDNode>();
- new (N) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, true,
- SVT, SV, SVOffset, Alignment, isVolatile);
+ new (N) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, true, SVT, MMO);
CSEMap.InsertNode(N, IP);
AllNodes.push_back(N);
return SDValue(N, 0);
@@ -3801,8 +3908,7 @@
SDNode *N = NodeAllocator.Allocate<StoreSDNode>();
new (N) StoreSDNode(Ops, dl, VTs, AM,
ST->isTruncatingStore(), ST->getMemoryVT(),
- ST->getSrcValue(), ST->getSrcValueOffset(),
- ST->getAlignment(), ST->isVolatile());
+ ST->getMemOperand());
CSEMap.InsertNode(N, IP);
AllNodes.push_back(N);
return SDValue(N, 0);
@@ -4454,29 +4560,35 @@
DeadNodeSet.insert(Used);
}
- // If NumOps is larger than the # of operands we currently have, reallocate
- // the operand list.
- if (NumOps > N->NumOperands) {
- if (N->OperandsNeedDelete)
- delete[] N->OperandList;
-
- if (N->isMachineOpcode()) {
- // We're creating a final node that will live unmorphed for the
- // remainder of the current SelectionDAG iteration, so we can allocate
- // the operands directly out of a pool with no recycling metadata.
- N->OperandList = OperandAllocator.Allocate<SDUse>(NumOps);
- N->OperandsNeedDelete = false;
- } else {
- N->OperandList = new SDUse[NumOps];
+ if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) {
+ // Initialize the memory references information.
+ MN->setMemRefs(0, 0);
+ // If NumOps is larger than the # of operands we can have in a
+ // MachineSDNode, reallocate the operand list.
+ if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) {
+ if (MN->OperandsNeedDelete)
+ delete[] MN->OperandList;
+ if (NumOps > array_lengthof(MN->LocalOperands))
+ // We're creating a final node that will live unmorphed for the
+ // remainder of the current SelectionDAG iteration, so we can allocate
+ // the operands directly out of a pool with no recycling metadata.
+ MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
+ Ops, NumOps);
+ else
+ MN->InitOperands(MN->LocalOperands, Ops, NumOps);
+ MN->OperandsNeedDelete = false;
+ } else
+ MN->InitOperands(MN->OperandList, Ops, NumOps);
+ } else {
+ // If NumOps is larger than the # of operands we currently have, reallocate
+ // the operand list.
+ if (NumOps > N->NumOperands) {
+ if (N->OperandsNeedDelete)
+ delete[] N->OperandList;
+ N->InitOperands(new SDUse[NumOps], Ops, NumOps);
N->OperandsNeedDelete = true;
- }
- }
-
- // Assign the new operands.
- N->NumOperands = NumOps;
- for (unsigned i = 0, e = NumOps; i != e; ++i) {
- N->OperandList[i].setUser(N);
- N->OperandList[i].setInitial(Ops[i]);
+ } else
+ MN->InitOperands(MN->OperandList, Ops, NumOps);
}
// Delete any nodes that are still dead after adding the uses for the
@@ -4501,41 +4613,49 @@
/// node of the specified opcode and operands, it returns that node instead of
/// the current one.
SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT) {
- return getNode(~Opcode, dl, VT).getNode();
+ SDVTList VTs = getVTList(VT);
+ return getMachineNode(Opcode, dl, VTs, 0, 0);
}
SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT,
SDValue Op1) {
- return getNode(~Opcode, dl, VT, Op1).getNode();
+ SDVTList VTs = getVTList(VT);
+ SDValue Ops[] = { Op1 };
+ return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops));
}
SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT,
SDValue Op1, SDValue Op2) {
- return getNode(~Opcode, dl, VT, Op1, Op2).getNode();
+ SDVTList VTs = getVTList(VT);
+ SDValue Ops[] = { Op1, Op2 };
+ return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops));
}
SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT,
SDValue Op1, SDValue Op2,
SDValue Op3) {
- return getNode(~Opcode, dl, VT, Op1, Op2, Op3).getNode();
+ SDVTList VTs = getVTList(VT);
+ SDValue Ops[] = { Op1, Op2, Op3 };
+ return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops));
}
SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT,
const SDValue *Ops, unsigned NumOps) {
- return getNode(~Opcode, dl, VT, Ops, NumOps).getNode();
+ SDVTList VTs = getVTList(VT);
+ return getMachineNode(Opcode, dl, VTs, Ops, NumOps);
}
SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl,
EVT VT1, EVT VT2) {
SDVTList VTs = getVTList(VT1, VT2);
- SDValue Op;
- return getNode(~Opcode, dl, VTs, &Op, 0).getNode();
+ return getMachineNode(Opcode, dl, VTs, 0, 0);
}
SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT1,
EVT VT2, SDValue Op1) {
SDVTList VTs = getVTList(VT1, VT2);
- return getNode(~Opcode, dl, VTs, &Op1, 1).getNode();
+ SDValue Ops[] = { Op1 };
+ return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops));
}
SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT1,
@@ -4543,7 +4663,7 @@
SDValue Op2) {
SDVTList VTs = getVTList(VT1, VT2);
SDValue Ops[] = { Op1, Op2 };
- return getNode(~Opcode, dl, VTs, Ops, 2).getNode();
+ return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops));
}
SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT1,
@@ -4551,14 +4671,14 @@
SDValue Op2, SDValue Op3) {
SDVTList VTs = getVTList(VT1, VT2);
SDValue Ops[] = { Op1, Op2, Op3 };
- return getNode(~Opcode, dl, VTs, Ops, 3).getNode();
+ return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops));
}
SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl,
EVT VT1, EVT VT2,
const SDValue *Ops, unsigned NumOps) {
SDVTList VTs = getVTList(VT1, VT2);
- return getNode(~Opcode, dl, VTs, Ops, NumOps).getNode();
+ return getMachineNode(Opcode, dl, VTs, Ops, NumOps);
}
SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl,
@@ -4566,7 +4686,7 @@
SDValue Op1, SDValue Op2) {
SDVTList VTs = getVTList(VT1, VT2, VT3);
SDValue Ops[] = { Op1, Op2 };
- return getNode(~Opcode, dl, VTs, Ops, 2).getNode();
+ return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops));
}
SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl,
@@ -4575,27 +4695,67 @@
SDValue Op3) {
SDVTList VTs = getVTList(VT1, VT2, VT3);
SDValue Ops[] = { Op1, Op2, Op3 };
- return getNode(~Opcode, dl, VTs, Ops, 3).getNode();
+ return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops));
}
SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl,
EVT VT1, EVT VT2, EVT VT3,
const SDValue *Ops, unsigned NumOps) {
SDVTList VTs = getVTList(VT1, VT2, VT3);
- return getNode(~Opcode, dl, VTs, Ops, NumOps).getNode();
+ return getMachineNode(Opcode, dl, VTs, Ops, NumOps);
}
SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT1,
EVT VT2, EVT VT3, EVT VT4,
const SDValue *Ops, unsigned NumOps) {
SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
- return getNode(~Opcode, dl, VTs, Ops, NumOps).getNode();
+ return getMachineNode(Opcode, dl, VTs, Ops, NumOps);
}
SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl,
const std::vector<EVT> &ResultTys,
const SDValue *Ops, unsigned NumOps) {
- return getNode(~Opcode, dl, ResultTys, Ops, NumOps).getNode();
+ SDVTList VTs = getVTList(&ResultTys[0], ResultTys.size());
+ return getMachineNode(Opcode, dl, VTs, Ops, NumOps);
+}
+
+SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc DL, SDVTList VTs,
+ const SDValue *Ops, unsigned NumOps) {
+ bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Flag;
+ MachineSDNode *N;
+ void *IP;
+
+ if (DoCSE) {
+ FoldingSetNodeID ID;
+ AddNodeIDNode(ID, ~Opcode, VTs, Ops, NumOps);
+ IP = 0;
+ if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+ return E;
+ }
+
+ // Allocate a new MachineSDNode.
+ N = NodeAllocator.Allocate<MachineSDNode>();
+ new (N) MachineSDNode(~Opcode, DL, VTs);
+
+ // Initialize the operands list.
+ if (NumOps > array_lengthof(N->LocalOperands))
+ // We're creating a final node that will live unmorphed for the
+ // remainder of the current SelectionDAG iteration, so we can allocate
+ // the operands directly out of a pool with no recycling metadata.
+ N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
+ Ops, NumOps);
+ else
+ N->InitOperands(N->LocalOperands, Ops, NumOps);
+ N->OperandsNeedDelete = false;
+
+ if (DoCSE)
+ CSEMap.InsertNode(N, IP);
+
+ AllNodes.push_back(N);
+#ifndef NDEBUG
+ VerifyNode(N);
+#endif
+ return N;
}
/// getTargetExtractSubreg - A convenience function for creating
@@ -4968,57 +5128,21 @@
}
MemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, EVT memvt,
- const Value *srcValue, int SVO, unsigned alignment,
- bool vol)
- : SDNode(Opc, dl, VTs), MemoryVT(memvt), SrcValue(srcValue), SVOffset(SVO) {
- SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, vol, alignment);
- assert(isPowerOf2_32(alignment) && "Alignment is not a power of 2!");
- assert(getOriginalAlignment() == alignment && "Alignment encoding error!");
- assert(isVolatile() == vol && "Volatile encoding error!");
+ MachineMemOperand *mmo)
+ : SDNode(Opc, dl, VTs), MemoryVT(memvt), MMO(mmo) {
+ SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile());
+ assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
+ assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!");
}
MemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs,
const SDValue *Ops, unsigned NumOps, EVT memvt,
- const Value *srcValue, int SVO, unsigned alignment,
- bool vol)
+ MachineMemOperand *mmo)
: SDNode(Opc, dl, VTs, Ops, NumOps),
- MemoryVT(memvt), SrcValue(srcValue), SVOffset(SVO) {
- SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, vol, alignment);
- assert(isPowerOf2_32(alignment) && "Alignment is not a power of 2!");
- assert(getOriginalAlignment() == alignment && "Alignment encoding error!");
- assert(isVolatile() == vol && "Volatile encoding error!");
-}
-
-/// getMemOperand - Return a MachineMemOperand object describing the memory
-/// reference performed by this memory reference.
-MachineMemOperand MemSDNode::getMemOperand() const {
- int Flags = 0;
- if (isa<LoadSDNode>(this))
- Flags = MachineMemOperand::MOLoad;
- else if (isa<StoreSDNode>(this))
- Flags = MachineMemOperand::MOStore;
- else if (isa<AtomicSDNode>(this)) {
- Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
- }
- else {
- const MemIntrinsicSDNode* MemIntrinNode = dyn_cast<MemIntrinsicSDNode>(this);
- assert(MemIntrinNode && "Unknown MemSDNode opcode!");
- if (MemIntrinNode->readMem()) Flags |= MachineMemOperand::MOLoad;
- if (MemIntrinNode->writeMem()) Flags |= MachineMemOperand::MOStore;
- }
-
- int Size = (getMemoryVT().getSizeInBits() + 7) >> 3;
- if (isVolatile()) Flags |= MachineMemOperand::MOVolatile;
-
- // Check if the memory reference references a frame index
- const FrameIndexSDNode *FI =
- dyn_cast<const FrameIndexSDNode>(getBasePtr().getNode());
- if (!getSrcValue() && FI)
- return MachineMemOperand(PseudoSourceValue::getFixedStack(FI->getIndex()),
- Flags, 0, Size, getOriginalAlignment());
- else
- return MachineMemOperand(getSrcValue(), Flags, getSrcValueOffset(),
- Size, getOriginalAlignment());
+ MemoryVT(memvt), MMO(mmo) {
+ SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile());
+ assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
+ assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!");
}
/// Profile - Gather unique data for the node.
@@ -5221,7 +5345,6 @@
case ISD::PCMARKER: return "PCMarker";
case ISD::READCYCLECOUNTER: return "ReadCycleCounter";
case ISD::SRCVALUE: return "SrcValue";
- case ISD::MEMOPERAND: return "MemOperand";
case ISD::EntryToken: return "EntryToken";
case ISD::TokenFactor: return "TokenFactor";
case ISD::AssertSext: return "AssertSext";
@@ -5500,8 +5623,20 @@
}
void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const {
- if (!isTargetOpcode() && getOpcode() == ISD::VECTOR_SHUFFLE) {
- const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(this);
+ if (const MachineSDNode *MN = dyn_cast<MachineSDNode>(this)) {
+ if (!MN->memoperands_empty()) {
+ OS << "<";
+ OS << "Mem:";
+ for (MachineSDNode::mmo_iterator i = MN->memoperands_begin(),
+ e = MN->memoperands_end(); i != e; ++i) {
+ OS << **i;
+ if (next(i) != e)
+ OS << " ";
+ }
+ OS << ">";
+ }
+ } else if (const ShuffleVectorSDNode *SVN =
+ dyn_cast<ShuffleVectorSDNode>(this)) {
OS << "<";
for (unsigned i = 0, e = ValueList[0].getVectorNumElements(); i != e; ++i) {
int Idx = SVN->getMaskElt(i);
@@ -5512,9 +5647,7 @@
OS << Idx;
}
OS << ">";
- }
-
- if (const ConstantSDNode *CSDN = dyn_cast<ConstantSDNode>(this)) {
+ } else if (const ConstantSDNode *CSDN = dyn_cast<ConstantSDNode>(this)) {
OS << '<' << CSDN->getAPIntValue() << '>';
} else if (const ConstantFPSDNode *CSDN = dyn_cast<ConstantFPSDNode>(this)) {
if (&CSDN->getValueAPF().getSemantics()==&APFloat::IEEEsingle)
@@ -5579,68 +5712,40 @@
OS << "<" << M->getValue() << ">";
else
OS << "<null>";
- } else if (const MemOperandSDNode *M = dyn_cast<MemOperandSDNode>(this)) {
- OS << ": " << M->MO;
} else if (const VTSDNode *N = dyn_cast<VTSDNode>(this)) {
OS << ":" << N->getVT().getEVTString();
}
else if (const LoadSDNode *LD = dyn_cast<LoadSDNode>(this)) {
- const Value *SrcValue = LD->getSrcValue();
- int SrcOffset = LD->getSrcValueOffset();
- OS << " <";
- if (SrcValue)
- OS << SrcValue;
- else
- OS << "null";
- OS << ":" << SrcOffset << ">";
+ OS << " <" << *LD->getMemOperand();
bool doExt = true;
switch (LD->getExtensionType()) {
default: doExt = false; break;
- case ISD::EXTLOAD: OS << " <anyext "; break;
- case ISD::SEXTLOAD: OS << " <sext "; break;
- case ISD::ZEXTLOAD: OS << " <zext "; break;
+ case ISD::EXTLOAD: OS << ", anyext"; break;
+ case ISD::SEXTLOAD: OS << ", sext"; break;
+ case ISD::ZEXTLOAD: OS << ", zext"; break;
}
if (doExt)
- OS << LD->getMemoryVT().getEVTString() << ">";
+ OS << " from " << LD->getMemoryVT().getEVTString();
const char *AM = getIndexedModeName(LD->getAddressingMode());
if (*AM)
- OS << " " << AM;
- if (LD->isVolatile())
- OS << " <volatile>";
- OS << " alignment=" << LD->getAlignment();
+ OS << ", " << AM;
+
+ OS << ">";
} else if (const StoreSDNode *ST = dyn_cast<StoreSDNode>(this)) {
- const Value *SrcValue = ST->getSrcValue();
- int SrcOffset = ST->getSrcValueOffset();
- OS << " <";
- if (SrcValue)
- OS << SrcValue;
- else
- OS << "null";
- OS << ":" << SrcOffset << ">";
+ OS << " <" << *ST->getMemOperand();
if (ST->isTruncatingStore())
- OS << " <trunc " << ST->getMemoryVT().getEVTString() << ">";
+ OS << ", trunc to " << ST->getMemoryVT().getEVTString();
const char *AM = getIndexedModeName(ST->getAddressingMode());
if (*AM)
- OS << " " << AM;
- if (ST->isVolatile())
- OS << " <volatile>";
- OS << " alignment=" << ST->getAlignment();
- } else if (const AtomicSDNode* AT = dyn_cast<AtomicSDNode>(this)) {
- const Value *SrcValue = AT->getSrcValue();
- int SrcOffset = AT->getSrcValueOffset();
- OS << " <";
- if (SrcValue)
- OS << SrcValue;
- else
- OS << "null";
- OS << ":" << SrcOffset << ">";
- if (AT->isVolatile())
- OS << " <volatile>";
- OS << " alignment=" << AT->getAlignment();
+ OS << ", " << AM;
+
+ OS << ">";
+ } else if (const MemSDNode* M = dyn_cast<MemSDNode>(this)) {
+ OS << " <" << *M->getMemOperand() << ">";
}
}
diff --git a/lib/CodeGen/StackSlotColoring.cpp b/lib/CodeGen/StackSlotColoring.cpp
index 7d88dad..fad0808 100644
--- a/lib/CodeGen/StackSlotColoring.cpp
+++ b/lib/CodeGen/StackSlotColoring.cpp
@@ -18,6 +18,7 @@
#include "llvm/CodeGen/LiveStackAnalysis.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/PseudoSourceValue.h"
#include "llvm/Support/CommandLine.h"
@@ -451,6 +452,7 @@
/// to old frame index with new one.
void StackSlotColoring::RewriteInstruction(MachineInstr *MI, int OldFI,
int NewFI, MachineFunction &MF) {
+ // Update the operands.
for (unsigned i = 0, ee = MI->getNumOperands(); i != ee; ++i) {
MachineOperand &MO = MI->getOperand(i);
if (!MO.isFI())
@@ -461,22 +463,15 @@
MO.setIndex(NewFI);
}
- // Update the MachineMemOperand for the new memory location.
- // FIXME: We need a better method of managing these too.
- SmallVector<MachineMemOperand, 2> MMOs(MI->memoperands_begin(),
- MI->memoperands_end());
- MI->clearMemOperands(MF);
+ // Update the memory references. This changes the MachineMemOperands
+ // directly. They may be in use by multiple instructions, however all
+ // instructions using OldFI are being rewritten to use NewFI.
const Value *OldSV = PseudoSourceValue::getFixedStack(OldFI);
- for (unsigned i = 0, ee = MMOs.size(); i != ee; ++i) {
- if (MMOs[i].getValue() != OldSV)
- MI->addMemOperand(MF, MMOs[i]);
- else {
- MachineMemOperand MMO(PseudoSourceValue::getFixedStack(NewFI),
- MMOs[i].getFlags(), MMOs[i].getOffset(),
- MMOs[i].getSize(), MMOs[i].getBaseAlignment());
- MI->addMemOperand(MF, MMO);
- }
- }
+ const Value *NewSV = PseudoSourceValue::getFixedStack(NewFI);
+ for (MachineInstr::mmo_iterator I = MI->memoperands_begin(),
+ E = MI->memoperands_end(); I != E; ++I)
+ if ((*I)->getValue() == OldSV)
+ (*I)->setValue(NewSV);
}
/// PropagateBackward - Traverse backward and look for the definition of
diff --git a/lib/CodeGen/TargetInstrInfoImpl.cpp b/lib/CodeGen/TargetInstrInfoImpl.cpp
index f184cb7..ab67cd2 100644
--- a/lib/CodeGen/TargetInstrInfoImpl.cpp
+++ b/lib/CodeGen/TargetInstrInfoImpl.cpp
@@ -17,6 +17,7 @@
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/PseudoSourceValue.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
@@ -203,11 +204,11 @@
"Folded a use to a non-load!");
const MachineFrameInfo &MFI = *MF.getFrameInfo();
assert(MFI.getObjectOffset(FrameIndex) != -1);
- MachineMemOperand MMO(PseudoSourceValue::getFixedStack(FrameIndex),
- Flags,
- /*Offset=*/0,
- MFI.getObjectSize(FrameIndex),
- MFI.getObjectAlignment(FrameIndex));
+ MachineMemOperand *MMO =
+ MF.getMachineMemOperand(PseudoSourceValue::getFixedStack(FrameIndex),
+ Flags, /*Offset=*/0,
+ MFI.getObjectSize(FrameIndex),
+ MFI.getObjectAlignment(FrameIndex));
NewMI->addMemOperand(MF, MMO);
return NewMI;
@@ -232,9 +233,8 @@
if (!NewMI) return 0;
// Copy the memoperands from the load to the folded instruction.
- for (std::list<MachineMemOperand>::iterator I = LoadMI->memoperands_begin(),
- E = LoadMI->memoperands_end(); I != E; ++I)
- NewMI->addMemOperand(MF, *I);
+ NewMI->setMemRefs(LoadMI->memoperands_begin(),
+ LoadMI->memoperands_end());
return NewMI;
}