[SVE][CodeGen] Scalable vector MVT size queries
* Implements scalable size queries for MVTs, split out from D53137.
* Contains a fix for FindMemType to avoid using scalable vector type
to contain non-scalable types.
* Explicit casts for several places where implicit integer sign
changes or promotion from 32 to 64 bits caused problems.
* CodeGenDAGPatterns will treat scalable and non-scalable vector types
as different.
Reviewers: greened, cameron.mcinally, sdesmalen, rovka
Reviewed By: rovka
Differential Revision: https://reviews.llvm.org/D66871
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 6f68313..9780b69 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -220,11 +220,13 @@
ForCodeSize = DAG.getMachineFunction().getFunction().hasOptSize();
MaximumLegalStoreInBits = 0;
+ // We use the minimum store size here, since that's all we can guarantee
+ // for the scalable vector types.
for (MVT VT : MVT::all_valuetypes())
if (EVT(VT).isSimple() && VT != MVT::Other &&
TLI.isTypeLegal(EVT(VT)) &&
- VT.getSizeInBits() >= MaximumLegalStoreInBits)
- MaximumLegalStoreInBits = VT.getSizeInBits();
+ VT.getSizeInBits().getKnownMinSize() >= MaximumLegalStoreInBits)
+ MaximumLegalStoreInBits = VT.getSizeInBits().getKnownMinSize();
}
void ConsiderForPruning(SDNode *N) {
@@ -13969,8 +13971,8 @@
// the stored value). With Offset=n (for n > 0) the loaded value starts at the
// n:th least significant byte of the stored value.
if (DAG.getDataLayout().isBigEndian())
- Offset = (STMemType.getStoreSizeInBits() -
- LDMemType.getStoreSizeInBits()) / 8 - Offset;
+ Offset = ((int64_t)STMemType.getStoreSizeInBits() -
+ (int64_t)LDMemType.getStoreSizeInBits()) / 8 - Offset;
// Check that the stored value cover all bits that are loaded.
bool STCoversLD =
@@ -15127,7 +15129,7 @@
// The latest Node in the DAG.
SDLoc DL(StoreNodes[0].MemNode);
- int64_t ElementSizeBits = MemVT.getStoreSizeInBits();
+ TypeSize ElementSizeBits = MemVT.getStoreSizeInBits();
unsigned SizeInBits = NumStores * ElementSizeBits;
unsigned NumMemElts = MemVT.isVector() ? MemVT.getVectorNumElements() : 1;
@@ -15512,7 +15514,7 @@
Attribute::NoImplicitFloat);
// This function cannot currently deal with non-byte-sized memory sizes.
- if (ElementSizeBytes * 8 != MemVT.getSizeInBits())
+ if (ElementSizeBytes * 8 != (int64_t)MemVT.getSizeInBits())
return false;
if (!MemVT.isSimple())
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 6c47c5b..70c0951 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -23,6 +23,7 @@
#include "llvm/IR/DataLayout.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/TypeSize.h"
using namespace llvm;
#define DEBUG_TYPE "legalize-types"
@@ -4680,7 +4681,8 @@
unsigned Width, EVT WidenVT,
unsigned Align = 0, unsigned WidenEx = 0) {
EVT WidenEltVT = WidenVT.getVectorElementType();
- unsigned WidenWidth = WidenVT.getSizeInBits();
+ const bool Scalable = WidenVT.isScalableVector();
+ unsigned WidenWidth = WidenVT.getSizeInBits().getKnownMinSize();
unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
unsigned AlignInBits = Align*8;
@@ -4691,23 +4693,27 @@
// See if there is larger legal integer than the element type to load/store.
unsigned VT;
- for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE;
- VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) {
- EVT MemVT((MVT::SimpleValueType) VT);
- unsigned MemVTWidth = MemVT.getSizeInBits();
- if (MemVT.getSizeInBits() <= WidenEltWidth)
- break;
- auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
- if ((Action == TargetLowering::TypeLegal ||
- Action == TargetLowering::TypePromoteInteger) &&
- (WidenWidth % MemVTWidth) == 0 &&
- isPowerOf2_32(WidenWidth / MemVTWidth) &&
- (MemVTWidth <= Width ||
- (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
- if (MemVTWidth == WidenWidth)
- return MemVT;
- RetVT = MemVT;
- break;
+ // Don't bother looking for an integer type if the vector is scalable, skip
+ // to vector types.
+ if (!Scalable) {
+ for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE;
+ VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) {
+ EVT MemVT((MVT::SimpleValueType) VT);
+ unsigned MemVTWidth = MemVT.getSizeInBits();
+ if (MemVT.getSizeInBits() <= WidenEltWidth)
+ break;
+ auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
+ if ((Action == TargetLowering::TypeLegal ||
+ Action == TargetLowering::TypePromoteInteger) &&
+ (WidenWidth % MemVTWidth) == 0 &&
+ isPowerOf2_32(WidenWidth / MemVTWidth) &&
+ (MemVTWidth <= Width ||
+ (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
+ if (MemVTWidth == WidenWidth)
+ return MemVT;
+ RetVT = MemVT;
+ break;
+ }
}
}
@@ -4716,7 +4722,10 @@
for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE;
VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) {
EVT MemVT = (MVT::SimpleValueType) VT;
- unsigned MemVTWidth = MemVT.getSizeInBits();
+ // Skip vector MVTs which don't match the scalable property of WidenVT.
+ if (Scalable != MemVT.isScalableVector())
+ continue;
+ unsigned MemVTWidth = MemVT.getSizeInBits().getKnownMinSize();
auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
if ((Action == TargetLowering::TypeLegal ||
Action == TargetLowering::TypePromoteInteger) &&
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 1b02f96..abd0465 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -8842,7 +8842,9 @@
// We check here that the size of the memory operand fits within the size of
// the MMO. This is because the MMO might indicate only a possible address
// range instead of specifying the affected memory addresses precisely.
- assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
+ // TODO: Make MachineMemOperands aware of scalable vectors.
+ assert(memvt.getStoreSize().getKnownMinSize() <= MMO->getSize() &&
+ "Size mismatch!");
}
/// Profile - Gather unique data for the node.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 4bb5d1e..54a3142 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -4304,7 +4304,10 @@
MachineMemOperand *MMO =
DAG.getMachineFunction().
getMachineMemOperand(MachinePointerInfo(PtrOperand),
- MachineMemOperand::MOStore, VT.getStoreSize(),
+ MachineMemOperand::MOStore,
+ // TODO: Make MachineMemOperands aware of scalable
+ // vectors.
+ VT.getStoreSize().getKnownMinSize(),
Alignment, AAInfo);
SDValue StoreNode = DAG.getMaskedStore(getRoot(), sdl, Src0, Ptr, Mask, VT,
MMO, false /* Truncating */,
@@ -4408,7 +4411,10 @@
const Value *MemOpBasePtr = UniformBase ? BasePtr : nullptr;
MachineMemOperand *MMO = DAG.getMachineFunction().
getMachineMemOperand(MachinePointerInfo(MemOpBasePtr),
- MachineMemOperand::MOStore, VT.getStoreSize(),
+ MachineMemOperand::MOStore,
+ // TODO: Make MachineMemOperands aware of scalable
+ // vectors.
+ VT.getStoreSize().getKnownMinSize(),
Alignment, AAInfo);
if (!UniformBase) {
Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
@@ -4477,7 +4483,10 @@
MachineMemOperand *MMO =
DAG.getMachineFunction().
getMachineMemOperand(MachinePointerInfo(PtrOperand),
- MachineMemOperand::MOLoad, VT.getStoreSize(),
+ MachineMemOperand::MOLoad,
+ // TODO: Make MachineMemOperands aware of scalable
+ // vectors.
+ VT.getStoreSize().getKnownMinSize(),
Alignment, AAInfo, Ranges);
SDValue Load = DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Mask, Src0, VT, MMO,
@@ -4528,7 +4537,10 @@
MachineMemOperand *MMO =
DAG.getMachineFunction().
getMachineMemOperand(MachinePointerInfo(UniformBase ? BasePtr : nullptr),
- MachineMemOperand::MOLoad, VT.getStoreSize(),
+ MachineMemOperand::MOLoad,
+ // TODO: Make MachineMemOperands aware of scalable
+ // vectors.
+ VT.getStoreSize().getKnownMinSize(),
Alignment, AAInfo, Ranges);
if (!UniformBase) {
@@ -9248,9 +9260,11 @@
for (unsigned j = 0; j != NumParts; ++j) {
// if it isn't first piece, alignment must be 1
+ // For scalable vectors the scalable part is currently handled
+ // by individual targets, so we just use the known minimum size here.
ISD::OutputArg MyFlags(Flags, Parts[j].getValueType(), VT,
- i < CLI.NumFixedArgs,
- i, j*Parts[j].getValueType().getStoreSize());
+ i < CLI.NumFixedArgs, i,
+ j*Parts[j].getValueType().getStoreSize().getKnownMinSize());
if (NumParts > 1 && j == 0)
MyFlags.Flags.setSplit();
else if (j != 0) {
@@ -9719,8 +9733,11 @@
unsigned NumRegs = TLI->getNumRegistersForCallingConv(
*CurDAG->getContext(), F.getCallingConv(), VT);
for (unsigned i = 0; i != NumRegs; ++i) {
+ // For scalable vectors, use the minimum size; individual targets
+ // are responsible for handling scalable vector arguments and
+ // return values.
ISD::InputArg MyFlags(Flags, RegisterVT, VT, isArgValueUsed,
- ArgNo, PartBase+i*RegisterVT.getStoreSize());
+ ArgNo, PartBase+i*RegisterVT.getStoreSize().getKnownMinSize());
if (NumRegs > 1 && i == 0)
MyFlags.Flags.setSplit();
// if it isn't first piece, alignment must be 1
@@ -9733,7 +9750,7 @@
}
if (NeedsRegBlock && Value == NumValues - 1)
Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
- PartBase += VT.getStoreSize();
+ PartBase += VT.getStoreSize().getKnownMinSize();
}
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
index fad98b6..c628f37 100644
--- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
@@ -384,7 +384,8 @@
// can consider allowing spills of smaller values to larger slots
// (i.e. change the '==' in the assert below to a '>=').
MachineFrameInfo &MFI = Builder.DAG.getMachineFunction().getFrameInfo();
- assert((MFI.getObjectSize(Index) * 8) == Incoming.getValueSizeInBits() &&
+ assert((MFI.getObjectSize(Index) * 8) ==
+ (int64_t)Incoming.getValueSizeInBits() &&
"Bad spill: stack slot does not match!");
// Note: Using the alignment of the spill slot (rather than the abi or