[Clang][SVE] Parse builtin type string for scalable vectors
This patch adds 'q' to mean 'scalable vector' in the builtin
type string, and for SVE will return the matching builtin
type as defined in the C/C++ language extensions for SVE.
This patch also adds some scaffolding to generate the arm_sve.h
header file, and some builtin definitions (+CodeGen) to be able
to implement some simple masked load intrinsics that use the
ACLE types, such as:
svint8_t test_svld1_s8(svbool_t pg, const int8_t *base) {
return svld1_s8(pg, base);
}
Reviewers: efriedma, rjmccall, rovka, rsandifo-arm, rengolin
Reviewed By: efriedma
Tags: #clang
Differential Revision: https://reviews.llvm.org/D75298
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 4368809..77d5e59 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -2100,16 +2100,16 @@
// Because the length is only known at runtime, we use a dummy value
// of 0 for the static length. The alignment values are those defined
// by the Procedure Call Standard for the Arm Architecture.
-#define SVE_VECTOR_TYPE(Name, Id, SingletonId, ElKind, ElBits, IsSigned, IsFP)\
- case BuiltinType::Id: \
- Width = 0; \
- Align = 128; \
- break;
-#define SVE_PREDICATE_TYPE(Name, Id, SingletonId, ElKind) \
- case BuiltinType::Id: \
- Width = 0; \
- Align = 16; \
- break;
+#define SVE_VECTOR_TYPE(Name, Id, SingletonId, NumEls, ElBits, IsSigned, IsFP) \
+ case BuiltinType::Id: \
+ Width = 0; \
+ Align = 128; \
+ break;
+#define SVE_PREDICATE_TYPE(Name, Id, SingletonId, NumEls) \
+ case BuiltinType::Id: \
+ Width = 0; \
+ Align = 16; \
+ break;
#include "clang/Basic/AArch64SVEACLETypes.def"
}
break;
@@ -3584,6 +3584,28 @@
return QualType(newType, 0);
}
+/// getScalableVectorType - Return the unique reference to a scalable vector
+/// type of the specified element type and size. VectorType must be a built-in
+/// type.
+QualType ASTContext::getScalableVectorType(QualType EltTy,
+ unsigned NumElts) const {
+ if (Target->hasAArch64SVETypes()) {
+ uint64_t EltTySize = getTypeSize(EltTy);
+#define SVE_VECTOR_TYPE(Name, Id, SingletonId, NumEls, ElBits, IsSigned, IsFP) \
+ if (!EltTy->isBooleanType() && \
+ ((EltTy->hasIntegerRepresentation() && \
+ EltTy->hasSignedIntegerRepresentation() == IsSigned) || \
+ (EltTy->hasFloatingRepresentation() && IsFP)) && \
+ EltTySize == ElBits && NumElts == NumEls) \
+ return SingletonId;
+#define SVE_PREDICATE_TYPE(Name, Id, SingletonId, NumEls) \
+ if (EltTy->isBooleanType() && NumElts == NumEls) \
+ return SingletonId;
+#include "clang/Basic/AArch64SVEACLETypes.def"
+ }
+ return QualType();
+}
+
/// getVectorType - Return the unique reference to a vector type of
/// the specified element type and size. VectorType must be a built-in type.
QualType ASTContext::getVectorType(QualType vecType, unsigned NumElts,
@@ -9699,6 +9721,19 @@
else
Type = Context.getLValueReferenceType(Type);
break;
+ case 'q': {
+ char *End;
+ unsigned NumElements = strtoul(Str, &End, 10);
+ assert(End != Str && "Missing vector size");
+ Str = End;
+
+ QualType ElementType = DecodeTypeFromStr(Str, Context, Error,
+ RequiresICE, false);
+ assert(!RequiresICE && "Can't require vector ICE");
+
+ Type = Context.getScalableVectorType(ElementType, NumElements);
+ break;
+ }
case 'V': {
char *End;
unsigned NumElements = strtoul(Str, &End, 10);
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 436084e..d333aef 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -7384,6 +7384,58 @@
return Op;
}
+// Reinterpret the input predicate so that it can be used to correctly isolate
+// the elements of the specified datatype.
+Value *CodeGenFunction::EmitSVEPredicateCast(Value *Pred,
+ llvm::VectorType *VTy) {
+ llvm::VectorType *RTy = llvm::VectorType::get(
+ IntegerType::get(getLLVMContext(), 1), VTy->getElementCount());
+ if (Pred->getType() == RTy)
+ return Pred;
+
+ unsigned IntID;
+ llvm::Type *IntrinsicTy;
+ switch (VTy->getNumElements()) {
+ default:
+ llvm_unreachable("unsupported element count!");
+ case 2:
+ case 4:
+ case 8:
+ IntID = Intrinsic::aarch64_sve_convert_from_svbool;
+ IntrinsicTy = RTy;
+ break;
+ case 16:
+ IntID = Intrinsic::aarch64_sve_convert_to_svbool;
+ IntrinsicTy = Pred->getType();
+ break;
+ }
+
+ Function *F = CGM.getIntrinsic(IntID, IntrinsicTy);
+ Value *C = Builder.CreateCall(F, Pred);
+ assert(C->getType() == RTy && "Unexpected return type!");
+ return C;
+}
+
+Value *CodeGenFunction::EmitSVEMaskedLoad(llvm::Type *ReturnTy,
+ SmallVectorImpl<Value *> &Ops) {
+ llvm::PointerType *PTy = cast<llvm::PointerType>(Ops[1]->getType());
+ llvm::Type *MemEltTy = PTy->getPointerElementType();
+
+ // The vector type that is returned may be different from the
+ // eventual type loaded from memory.
+ auto VectorTy = cast<llvm::VectorType>(ReturnTy);
+ auto MemoryTy =
+ llvm::VectorType::get(MemEltTy, VectorTy->getVectorElementCount());
+
+ Value *Offset = Builder.getInt32(0);
+ Value *Predicate = EmitSVEPredicateCast(Ops[0], MemoryTy);
+ Value *BasePtr = Builder.CreateBitCast(Ops[1], MemoryTy->getPointerTo());
+ BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Offset);
+
+ Value *Splat0 = Constant::getNullValue(MemoryTy);
+ return Builder.CreateMaskedLoad(BasePtr, Align(1), Predicate, Splat0);
+}
+
Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
const CallExpr *E,
llvm::Triple::ArchType Arch) {
@@ -7420,6 +7472,27 @@
return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID));
}
+ switch (BuiltinID) {
+ case AArch64::BI__builtin_sve_svld1_u8:
+ case AArch64::BI__builtin_sve_svld1_u16:
+ case AArch64::BI__builtin_sve_svld1_u32:
+ case AArch64::BI__builtin_sve_svld1_u64:
+ case AArch64::BI__builtin_sve_svld1_s8:
+ case AArch64::BI__builtin_sve_svld1_s16:
+ case AArch64::BI__builtin_sve_svld1_s32:
+ case AArch64::BI__builtin_sve_svld1_s64:
+ case AArch64::BI__builtin_sve_svld1_f16:
+ case AArch64::BI__builtin_sve_svld1_f32:
+ case AArch64::BI__builtin_sve_svld1_f64: {
+ llvm::SmallVector<Value *, 4> Ops = {EmitScalarExpr(E->getArg(0)),
+ EmitScalarExpr(E->getArg(1))};
+ llvm::Type *Ty = ConvertType(E->getType());
+ return EmitSVEMaskedLoad(Ty, Ops);
+ }
+ default:
+ break;
+ }
+
if (BuiltinID == AArch64::BI__builtin_arm_prefetch) {
Value *Address = EmitScalarExpr(E->getArg(0));
Value *RW = EmitScalarExpr(E->getArg(1));
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index d6c2afc..3393b1b 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -495,13 +495,15 @@
// Scan function arguments for vector width.
for (llvm::Argument &A : CurFn->args())
if (auto *VT = dyn_cast<llvm::VectorType>(A.getType()))
- LargestVectorWidth = std::max((uint64_t)LargestVectorWidth,
- VT->getPrimitiveSizeInBits().getFixedSize());
+ LargestVectorWidth =
+ std::max((uint64_t)LargestVectorWidth,
+ VT->getPrimitiveSizeInBits().getKnownMinSize());
// Update vector width based on return type.
if (auto *VT = dyn_cast<llvm::VectorType>(CurFn->getReturnType()))
- LargestVectorWidth = std::max((uint64_t)LargestVectorWidth,
- VT->getPrimitiveSizeInBits().getFixedSize());
+ LargestVectorWidth =
+ std::max((uint64_t)LargestVectorWidth,
+ VT->getPrimitiveSizeInBits().getKnownMinSize());
// Add the required-vector-width attribute. This contains the max width from:
// 1. min-vector-width attribute used in the source program.
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index f82e489..e470bec 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -3900,6 +3900,11 @@
llvm::Value *EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt,
llvm::Type *Ty, bool usgn, const char *name);
llvm::Value *vectorWrapScalar16(llvm::Value *Op);
+
+ llvm::Value *EmitSVEPredicateCast(llvm::Value *Pred, llvm::VectorType *VTy);
+ llvm::Value *EmitSVEMaskedLoad(llvm::Type *ReturnTy,
+ SmallVectorImpl<llvm::Value *> &Ops);
+
llvm::Value *EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E,
llvm::Triple::ArchType Arch);
llvm::Value *EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
diff --git a/clang/lib/Headers/CMakeLists.txt b/clang/lib/Headers/CMakeLists.txt
index 9237843..28d43cb 100644
--- a/clang/lib/Headers/CMakeLists.txt
+++ b/clang/lib/Headers/CMakeLists.txt
@@ -184,6 +184,8 @@
clang_generate_header(-gen-arm-neon arm_neon.td arm_neon.h)
# Generate arm_fp16.h
clang_generate_header(-gen-arm-fp16 arm_fp16.td arm_fp16.h)
+# Generate arm_sve.h
+clang_generate_header(-gen-arm-sve-header arm_sve.td arm_sve.h)
# Generate arm_mve.h
clang_generate_header(-gen-arm-mve-header arm_mve.td arm_mve.h)
# Generate arm_cde.h
diff --git a/clang/lib/Headers/module.modulemap b/clang/lib/Headers/module.modulemap
index 7954a77..6894672 100644
--- a/clang/lib/Headers/module.modulemap
+++ b/clang/lib/Headers/module.modulemap
@@ -27,6 +27,12 @@
header "arm_fp16.h"
export *
}
+
+ explicit module sve {
+ requires sve
+ header "arm_sve.h"
+ export *
+ }
}
explicit module intel {