Expandload and Compressstore intrinsics
2 new intrinsics covering AVX-512 compress/expand functionality.
This implementation includes syntax, DAG builder, operation lowering and tests.
Does not include: handling of illegal data types, codegen prepare pass and the cost model.
llvm-svn: 285876
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index e1da385..4e55a22 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -5583,7 +5583,7 @@
Alignment, MST->getAAInfo(), MST->getRanges());
Lo = DAG.getMaskedStore(Chain, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO,
- MST->isTruncatingStore());
+ MST->isTruncatingStore(), MST->isCompressingStore());
unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
@@ -5596,7 +5596,7 @@
MST->getRanges());
Hi = DAG.getMaskedStore(Chain, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,
- MST->isTruncatingStore());
+ MST->isTruncatingStore(), MST->isCompressingStore());
AddToWorklist(Lo.getNode());
AddToWorklist(Hi.getNode());
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 0669536..0f3cca0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -1212,7 +1212,7 @@
return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask,
N->getMemoryVT(), N->getMemOperand(),
- TruncateStore);
+ TruncateStore, N->isCompressingStore());
}
SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 96683a2..eea5dcc 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3667,16 +3667,39 @@
DAG.setRoot(StoreNode);
}
-void SelectionDAGBuilder::visitMaskedStore(const CallInst &I) {
+void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
+ bool IsCompressing) {
SDLoc sdl = getCurSDLoc();
- // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
- Value *PtrOperand = I.getArgOperand(1);
+ auto getMaskedStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
+ unsigned& Alignment) {
+ // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
+ Src0 = I.getArgOperand(0);
+ Ptr = I.getArgOperand(1);
+ Alignment = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
+ Mask = I.getArgOperand(3);
+ };
+ auto getCompressingStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
+ unsigned& Alignment) {
+ // llvm.masked.compressstore.*(Src0, Ptr, Mask)
+ Src0 = I.getArgOperand(0);
+ Ptr = I.getArgOperand(1);
+ Mask = I.getArgOperand(2);
+ Alignment = 0;
+ };
+
+ Value *PtrOperand, *MaskOperand, *Src0Operand;
+ unsigned Alignment;
+ if (IsCompressing)
+ getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
+ else
+ getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
+
SDValue Ptr = getValue(PtrOperand);
- SDValue Src0 = getValue(I.getArgOperand(0));
- SDValue Mask = getValue(I.getArgOperand(3));
+ SDValue Src0 = getValue(Src0Operand);
+ SDValue Mask = getValue(MaskOperand);
+
EVT VT = Src0.getValueType();
- unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
if (!Alignment)
Alignment = DAG.getEVTAlignment(VT);
@@ -3689,7 +3712,8 @@
MachineMemOperand::MOStore, VT.getStoreSize(),
Alignment, AAInfo);
SDValue StoreNode = DAG.getMaskedStore(getRoot(), sdl, Src0, Ptr, Mask, VT,
- MMO, false);
+ MMO, false /* Truncating */,
+ IsCompressing);
DAG.setRoot(StoreNode);
setValue(&I, StoreNode);
}
@@ -3710,7 +3734,7 @@
// extract the spalt value and use it as a uniform base.
// In all other cases the function returns 'false'.
//
-static bool getUniformBase(const Value *& Ptr, SDValue& Base, SDValue& Index,
+static bool getUniformBase(const Value* &Ptr, SDValue& Base, SDValue& Index,
SelectionDAGBuilder* SDB) {
SelectionDAG& DAG = SDB->DAG;
@@ -3790,18 +3814,38 @@
setValue(&I, Scatter);
}
-void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I) {
+void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
SDLoc sdl = getCurSDLoc();
- // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
- Value *PtrOperand = I.getArgOperand(0);
- SDValue Ptr = getValue(PtrOperand);
- SDValue Src0 = getValue(I.getArgOperand(3));
- SDValue Mask = getValue(I.getArgOperand(2));
+ auto getMaskedLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
+ unsigned& Alignment) {
+ // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
+ Ptr = I.getArgOperand(0);
+ Alignment = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
+ Mask = I.getArgOperand(2);
+ Src0 = I.getArgOperand(3);
+ };
+ auto getExpandingLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
+ unsigned& Alignment) {
+ // @llvm.masked.expandload.*(Ptr, Mask, Src0)
+ Ptr = I.getArgOperand(0);
+ Alignment = 0;
+ Mask = I.getArgOperand(1);
+ Src0 = I.getArgOperand(2);
+ };
- const TargetLowering &TLI = DAG.getTargetLoweringInfo();
- EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
- unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
+ Value *PtrOperand, *MaskOperand, *Src0Operand;
+ unsigned Alignment;
+ if (IsExpanding)
+ getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
+ else
+ getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
+
+ SDValue Ptr = getValue(PtrOperand);
+ SDValue Src0 = getValue(Src0Operand);
+ SDValue Mask = getValue(MaskOperand);
+
+ EVT VT = Src0.getValueType();
if (!Alignment)
Alignment = DAG.getEVTAlignment(VT);
@@ -3821,7 +3865,7 @@
Alignment, AAInfo, Ranges);
SDValue Load = DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Mask, Src0, VT, MMO,
- ISD::NON_EXTLOAD, false);
+ ISD::NON_EXTLOAD, IsExpanding);
if (AddToChain) {
SDValue OutChain = Load.getValue(1);
DAG.setRoot(OutChain);
@@ -5054,6 +5098,12 @@
case Intrinsic::masked_store:
visitMaskedStore(I);
return nullptr;
+ case Intrinsic::masked_expandload:
+ visitMaskedLoad(I, true /* IsExpanding */);
+ return nullptr;
+ case Intrinsic::masked_compressstore:
+ visitMaskedStore(I, true /* IsCompressing */);
+ return nullptr;
case Intrinsic::x86_mmx_pslli_w:
case Intrinsic::x86_mmx_pslli_d:
case Intrinsic::x86_mmx_pslli_q:
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index 122d092..abde8a8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -874,8 +874,8 @@
void visitAlloca(const AllocaInst &I);
void visitLoad(const LoadInst &I);
void visitStore(const StoreInst &I);
- void visitMaskedLoad(const CallInst &I);
- void visitMaskedStore(const CallInst &I);
+ void visitMaskedLoad(const CallInst &I, bool IsExpanding = false);
+ void visitMaskedStore(const CallInst &I, bool IsCompressing = false);
void visitMaskedGather(const CallInst &I);
void visitMaskedScatter(const CallInst &I);
void visitAtomicCmpXchg(const AtomicCmpXchgInst &I);