Fix the SVOffset values for loads and stores produced by
memcpy/memset expansion. It was a bug for the SVOffset value
to be used in the actual address calculations.
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@50359 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h
index 0d0e4d3..e17e32e 100644
--- a/include/llvm/CodeGen/SelectionDAG.h
+++ b/include/llvm/CodeGen/SelectionDAG.h
@@ -326,17 +326,17 @@
SDOperand getMemcpy(SDOperand Chain, SDOperand Dst, SDOperand Src,
SDOperand Size, unsigned Align,
bool AlwaysInline,
- const Value *DstSV, uint64_t DstOff,
- const Value *SrcSV, uint64_t SrcOff);
+ const Value *DstSV, uint64_t DstSVOff,
+ const Value *SrcSV, uint64_t SrcSVOff);
SDOperand getMemmove(SDOperand Chain, SDOperand Dst, SDOperand Src,
SDOperand Size, unsigned Align,
- const Value *DstSV, uint64_t DstOff,
- const Value *SrcSV, uint64_t SrcOff);
+ const Value *DstSV, uint64_t DstOSVff,
+ const Value *SrcSV, uint64_t SrcSVOff);
SDOperand getMemset(SDOperand Chain, SDOperand Dst, SDOperand Src,
SDOperand Size, unsigned Align,
- const Value *DstSV, uint64_t DstOff);
+ const Value *DstSV, uint64_t DstSVOff);
/// getSetCC - Helper function to make it easier to build SetCC's if you just
/// have an ISD::CondCode instead of an SDOperand.
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index bfc3506..dfd422a 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -2590,8 +2590,8 @@
SDOperand Src, uint64_t Size,
unsigned Align,
bool AlwaysInline,
- const Value *DstSV, uint64_t DstOff,
- const Value *SrcSV, uint64_t SrcOff) {
+ const Value *DstSV, uint64_t DstSVOff,
+ const Value *SrcSV, uint64_t SrcSVOff){
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
// Expand memcpy to a series of store ops if the size operand falls below
@@ -2610,6 +2610,7 @@
GlobalAddressSDNode *G = NULL;
std::string Str;
bool CopyFromStr = false;
+ uint64_t SrcOff = 0, DstOff = 0;
if (Src.getOpcode() == ISD::GlobalAddress)
G = cast<GlobalAddressSDNode>(Src);
@@ -2640,15 +2641,15 @@
Store =
DAG.getStore(Chain, Value,
getMemBasePlusOffset(Dst, DstOff, DAG),
- DstSV, DstOff);
+ DstSV, DstSVOff + DstOff);
} else {
Value = DAG.getLoad(VT, Chain,
getMemBasePlusOffset(Src, SrcOff, DAG),
- SrcSV, SrcOff, false, Align);
+ SrcSV, SrcSVOff + SrcOff, false, Align);
Store =
DAG.getStore(Chain, Value,
getMemBasePlusOffset(Dst, DstOff, DAG),
- DstSV, DstOff, false, Align);
+ DstSV, DstSVOff + DstOff, false, Align);
}
OutChains.push_back(Store);
SrcOff += VTSize;
@@ -2663,7 +2664,7 @@
SDOperand Chain, SDOperand Dst,
SDOperand Src, uint64_t Size,
unsigned Align,
- const Value *DstSV, uint64_t DstOff) {
+ const Value *DstSV, uint64_t DstSVOff) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
// Expand memset to a series of load/store ops if the size operand
@@ -2674,6 +2675,7 @@
return SDOperand();
SmallVector<SDOperand, 8> OutChains;
+ uint64_t DstOff = 0;
unsigned NumMemOps = MemOps.size();
for (unsigned i = 0; i < NumMemOps; i++) {
@@ -2682,7 +2684,7 @@
SDOperand Value = getMemsetValue(Src, VT, DAG);
SDOperand Store = DAG.getStore(Chain, Value,
getMemBasePlusOffset(Dst, DstOff, DAG),
- DstSV, DstOff);
+ DstSV, DstSVOff + DstOff);
OutChains.push_back(Store);
DstOff += VTSize;
}
@@ -2694,8 +2696,8 @@
SDOperand SelectionDAG::getMemcpy(SDOperand Chain, SDOperand Dst,
SDOperand Src, SDOperand Size,
unsigned Align, bool AlwaysInline,
- const Value *DstSV, uint64_t DstOff,
- const Value *SrcSV, uint64_t SrcOff) {
+ const Value *DstSV, uint64_t DstSVOff,
+ const Value *SrcSV, uint64_t SrcSVOff) {
// Check to see if we should lower the memcpy to loads and stores first.
// For cases within the target-specified limits, this is the best choice.
@@ -2707,7 +2709,7 @@
SDOperand Result =
getMemcpyLoadsAndStores(*this, Chain, Dst, Src, ConstantSize->getValue(),
- Align, false, DstSV, DstOff, SrcSV, SrcOff);
+ Align, false, DstSV, DstSVOff, SrcSV, SrcSVOff);
if (Result.Val)
return Result;
}
@@ -2717,7 +2719,7 @@
SDOperand Result =
TLI.EmitTargetCodeForMemcpy(*this, Chain, Dst, Src, Size, Align,
AlwaysInline,
- DstSV, DstOff, SrcSV, SrcOff);
+ DstSV, DstSVOff, SrcSV, SrcSVOff);
if (Result.Val)
return Result;
@@ -2727,7 +2729,7 @@
assert(ConstantSize && "AlwaysInline requires a constant size!");
return getMemcpyLoadsAndStores(*this, Chain, Dst, Src,
ConstantSize->getValue(), Align, true,
- DstSV, DstOff, SrcSV, SrcOff);
+ DstSV, DstSVOff, SrcSV, SrcSVOff);
}
// Emit a library call.
@@ -2748,8 +2750,8 @@
SDOperand SelectionDAG::getMemmove(SDOperand Chain, SDOperand Dst,
SDOperand Src, SDOperand Size,
unsigned Align,
- const Value *DstSV, uint64_t DstOff,
- const Value *SrcSV, uint64_t SrcOff) {
+ const Value *DstSV, uint64_t DstSVOff,
+ const Value *SrcSV, uint64_t SrcSVOff) {
// TODO: Optimize small memmove cases with simple loads and stores,
// ensuring that all loads precede all stores. This can cause severe
@@ -2759,7 +2761,7 @@
// code. If the target chooses to do this, this is the next best.
SDOperand Result =
TLI.EmitTargetCodeForMemmove(*this, Chain, Dst, Src, Size, Align,
- DstSV, DstOff, SrcSV, SrcOff);
+ DstSV, DstSVOff, SrcSV, SrcSVOff);
if (Result.Val)
return Result;
@@ -2781,7 +2783,7 @@
SDOperand SelectionDAG::getMemset(SDOperand Chain, SDOperand Dst,
SDOperand Src, SDOperand Size,
unsigned Align,
- const Value *DstSV, uint64_t DstOff) {
+ const Value *DstSV, uint64_t DstSVOff) {
// Check to see if we should lower the memset to stores first.
// For cases within the target-specified limits, this is the best choice.
@@ -2793,7 +2795,7 @@
SDOperand Result =
getMemsetStores(*this, Chain, Dst, Src, ConstantSize->getValue(), Align,
- DstSV, DstOff);
+ DstSV, DstSVOff);
if (Result.Val)
return Result;
}
@@ -2802,7 +2804,7 @@
// code. If the target chooses to do this, this is the next best.
SDOperand Result =
TLI.EmitTargetCodeForMemset(*this, Chain, Dst, Src, Size, Align,
- DstSV, DstOff);
+ DstSV, DstSVOff);
if (Result.Val)
return Result;
diff --git a/lib/Target/ARM/ARMISelLowering.cpp b/lib/Target/ARM/ARMISelLowering.cpp
index 6a581f7..dc76b7b 100644
--- a/lib/Target/ARM/ARMISelLowering.cpp
+++ b/lib/Target/ARM/ARMISelLowering.cpp
@@ -1247,8 +1247,8 @@
SDOperand Dst, SDOperand Src,
SDOperand Size, unsigned Align,
bool AlwaysInline,
- const Value *DstSV, uint64_t DstOff,
- const Value *SrcSV, uint64_t SrcOff){
+ const Value *DstSV, uint64_t DstSVOff,
+ const Value *SrcSV, uint64_t SrcSVOff){
// Do repeated 4-byte loads and stores. To be improved.
// This requires 4-byte alignment.
if ((Align & 3) != 0)
@@ -1271,6 +1271,7 @@
const unsigned MAX_LOADS_IN_LDM = 6;
SDOperand TFOps[MAX_LOADS_IN_LDM];
SDOperand Loads[MAX_LOADS_IN_LDM];
+ uint64_t SrcOff = 0, DstOff = 0;
// Emit up to MAX_LOADS_IN_LDM loads, then a TokenFactor barrier, then the
// same number of stores. The loads and stores will get combined into
@@ -1281,7 +1282,7 @@
Loads[i] = DAG.getLoad(VT, Chain,
DAG.getNode(ISD::ADD, MVT::i32, Src,
DAG.getConstant(SrcOff, MVT::i32)),
- SrcSV, SrcOff);
+ SrcSV, SrcSVOff + SrcOff);
TFOps[i] = Loads[i].getValue(1);
SrcOff += VTSize;
}
@@ -1292,7 +1293,7 @@
TFOps[i] = DAG.getStore(Chain, Loads[i],
DAG.getNode(ISD::ADD, MVT::i32, Dst,
DAG.getConstant(DstOff, MVT::i32)),
- DstSV, DstOff);
+ DstSV, DstSVOff + DstOff);
DstOff += VTSize;
}
Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, &TFOps[0], i);
@@ -1318,7 +1319,7 @@
Loads[i] = DAG.getLoad(VT, Chain,
DAG.getNode(ISD::ADD, MVT::i32, Src,
DAG.getConstant(SrcOff, MVT::i32)),
- SrcSV, SrcOff);
+ SrcSV, SrcSVOff + SrcOff);
TFOps[i] = Loads[i].getValue(1);
++i;
SrcOff += VTSize;
@@ -1340,7 +1341,7 @@
TFOps[i] = DAG.getStore(Chain, Loads[i],
DAG.getNode(ISD::ADD, MVT::i32, Dst,
DAG.getConstant(DstOff, MVT::i32)),
- DstSV, DstOff);
+ DstSV, DstSVOff + DstOff);
++i;
DstOff += VTSize;
BytesLeft -= VTSize;
diff --git a/lib/Target/ARM/ARMISelLowering.h b/lib/Target/ARM/ARMISelLowering.h
index 13f5c08..ce6d5fe 100644
--- a/lib/Target/ARM/ARMISelLowering.h
+++ b/lib/Target/ARM/ARMISelLowering.h
@@ -149,8 +149,8 @@
SDOperand Dst, SDOperand Src,
SDOperand Size, unsigned Align,
bool AlwaysInline,
- const Value *DstSV, uint64_t DstOff,
- const Value *SrcSV, uint64_t SrcOff);
+ const Value *DstSV, uint64_t DstSVOff,
+ const Value *SrcSV, uint64_t SrcSVOff);
};
}
diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp
index c0ba4d1..799ba00 100644
--- a/lib/Target/X86/X86ISelLowering.cpp
+++ b/lib/Target/X86/X86ISelLowering.cpp
@@ -4763,7 +4763,7 @@
SDOperand Chain,
SDOperand Dst, SDOperand Src,
SDOperand Size, unsigned Align,
- const Value *DstSV, uint64_t DstOff) {
+ const Value *DstSV, uint64_t DstSVOff) {
ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
/// If not DWORD aligned or size is more than the threshold, call the library.
@@ -4890,7 +4890,7 @@
DAG.getConstant(Offset, AddrVT)),
Src,
DAG.getConstant(BytesLeft, SizeVT),
- Align, DstSV, 0);
+ Align, DstSV, DstSVOff + Offset);
}
// TODO: Use a Tokenfactor, as in memcpy, instead of a single chain.
@@ -4903,8 +4903,8 @@
SDOperand Dst, SDOperand Src,
SDOperand Size, unsigned Align,
bool AlwaysInline,
- const Value *DstSV, uint64_t DstOff,
- const Value *SrcSV, uint64_t SrcOff){
+ const Value *DstSV, uint64_t DstSVOff,
+ const Value *SrcSV, uint64_t SrcSVOff){
// This requires the copy size to be a constant, preferrably
// within a subtarget-specific limit.
@@ -4964,7 +4964,8 @@
DAG.getConstant(Offset, SrcVT)),
DAG.getConstant(BytesLeft, SizeVT),
Align, AlwaysInline,
- DstSV, 0, SrcSV, 0));
+ DstSV, DstSVOff + Offset,
+ SrcSV, SrcSVOff + Offset));
}
return DAG.getNode(ISD::TokenFactor, MVT::Other, &Results[0], Results.size());
diff --git a/lib/Target/X86/X86ISelLowering.h b/lib/Target/X86/X86ISelLowering.h
index 6b02f33..987cf73 100644
--- a/lib/Target/X86/X86ISelLowering.h
+++ b/lib/Target/X86/X86ISelLowering.h
@@ -538,14 +538,14 @@
SDOperand Chain,
SDOperand Dst, SDOperand Src,
SDOperand Size, unsigned Align,
- const Value *DstSV, uint64_t DstOff);
+ const Value *DstSV, uint64_t DstSVOff);
SDOperand EmitTargetCodeForMemcpy(SelectionDAG &DAG,
SDOperand Chain,
SDOperand Dst, SDOperand Src,
SDOperand Size, unsigned Align,
bool AlwaysInline,
- const Value *DstSV, uint64_t DstOff,
- const Value *SrcSV, uint64_t SrcOff);
+ const Value *DstSV, uint64_t DstSVOff,
+ const Value *SrcSV, uint64_t SrcSVOff);
};
}