[AArch64] Implement Vector Funtion ABI name mangling.
Summary:
The name mangling scheme is defined in section 3.5 of the "Vector function application binary interface specification for AArch64" [1].
[1] https://developer.arm.com/products/software-development-tools/hpc/arm-compiler-for-hpc/vector-function-abi
Reviewers: rengolin, ABataev
Reviewed By: ABataev
Subscribers: sdesmalen, javed.absar, kristof.beyls, jdoerfert, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D60583
llvm-svn: 358490
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 4f63133..da4e355 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -9648,6 +9648,307 @@
}
}
+// This are the Functions that are needed to mangle the name of the
+// vector functions generated by the compiler, according to the rules
+// defined in the "Vector Function ABI specifications for AArch64",
+// available at
+// https://developer.arm.com/products/software-development-tools/hpc/arm-compiler-for-hpc/vector-function-abi.
+
+/// Maps To Vector (MTV), as defined in 3.1.1 of the AAVFABI.
+///
+/// TODO: Need to implement the behavior for reference marked with a
+/// var or no linear modifiers (1.b in the section). For this, we
+/// need to extend ParamKindTy to support the linear modifiers.
+static bool getAArch64MTV(QualType QT, ParamKindTy Kind) {
+ QT = QT.getCanonicalType();
+
+ if (QT->isVoidType())
+ return false;
+
+ if (Kind == ParamKindTy::Uniform)
+ return false;
+
+ if (Kind == ParamKindTy::Linear)
+ return false;
+
+ // TODO: Handle linear references with modifiers
+
+ if (Kind == ParamKindTy::LinearWithVarStride)
+ return false;
+
+ return true;
+}
+
+/// Pass By Value (PBV), as defined in 3.1.2 of the AAVFABI.
+static bool getAArch64PBV(QualType QT, ASTContext &C) {
+ QT = QT.getCanonicalType();
+ unsigned Size = C.getTypeSize(QT);
+
+ // Only scalars and complex within 16 bytes wide set PVB to true.
+ if (Size != 8 && Size != 16 && Size != 32 && Size != 64 && Size != 128)
+ return false;
+
+ if (QT->isFloatingType())
+ return true;
+
+ if (QT->isIntegerType())
+ return true;
+
+ if (QT->isPointerType())
+ return true;
+
+ // TODO: Add support for complex types (section 3.1.2, item 2).
+
+ return false;
+}
+
+/// Computes the lane size (LS) of a return type or of an input parameter,
+/// as defined by `LS(P)` in 3.2.1 of the AAVFABI.
+/// TODO: Add support for references, section 3.2.1, item 1.
+static unsigned getAArch64LS(QualType QT, ParamKindTy Kind, ASTContext &C) {
+ if (getAArch64MTV(QT, Kind) && QT.getCanonicalType()->isPointerType()) {
+ QualType PTy = QT.getCanonicalType()->getPointeeType();
+ if (getAArch64PBV(PTy, C))
+ return C.getTypeSize(PTy);
+ }
+ if (getAArch64PBV(QT, C))
+ return C.getTypeSize(QT);
+
+ return C.getTypeSize(C.getUIntPtrType());
+}
+
+// Get Narrowest Data Size (NDS) and Widest Data Size (WDS) from the
+// signature of the scalar function, as defined in 3.2.2 of the
+// AAVFABI.
+static std::tuple<unsigned, unsigned, bool>
+getNDSWDS(const FunctionDecl *FD, ArrayRef<ParamAttrTy> ParamAttrs) {
+ QualType RetType = FD->getReturnType().getCanonicalType();
+
+ ASTContext &C = FD->getASTContext();
+
+ bool OutputBecomesInput = false;
+
+ llvm::SmallVector<unsigned, 8> Sizes;
+ if (!RetType->isVoidType()) {
+ Sizes.push_back(getAArch64LS(RetType, ParamKindTy::Vector, C));
+ if (!getAArch64PBV(RetType, C) && getAArch64MTV(RetType, {}))
+ OutputBecomesInput = true;
+ }
+ for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) {
+ QualType QT = FD->getParamDecl(I)->getType().getCanonicalType();
+ Sizes.push_back(getAArch64LS(QT, ParamAttrs[I].Kind, C));
+ }
+
+ assert(!Sizes.empty() && "Unable to determine NDS and WDS.");
+ // The LS of a function parameter / return value can only be a power
+ // of 2, starting from 8 bits, up to 128.
+ assert(std::all_of(Sizes.begin(), Sizes.end(),
+ [](unsigned Size) {
+ return Size == 8 || Size == 16 || Size == 32 ||
+ Size == 64 || Size == 128;
+ }) &&
+ "Invalid size");
+
+ return std::make_tuple(*std::min_element(std::begin(Sizes), std::end(Sizes)),
+ *std::max_element(std::begin(Sizes), std::end(Sizes)),
+ OutputBecomesInput);
+}
+
+/// Mangle the parameter part of the vector function name according to
+/// their OpenMP classification. The mangling function is defined in
+/// section 3.5 of the AAVFABI.
+static std::string mangleVectorParameters(ArrayRef<ParamAttrTy> ParamAttrs) {
+ SmallString<256> Buffer;
+ llvm::raw_svector_ostream Out(Buffer);
+ for (const auto &ParamAttr : ParamAttrs) {
+ switch (ParamAttr.Kind) {
+ case LinearWithVarStride:
+ Out << "ls" << ParamAttr.StrideOrArg;
+ break;
+ case Linear:
+ Out << 'l';
+ // Don't print the step value if it is not present or if it is
+ // equal to 1.
+ if (!!ParamAttr.StrideOrArg && ParamAttr.StrideOrArg != 1)
+ Out << ParamAttr.StrideOrArg;
+ break;
+ case Uniform:
+ Out << 'u';
+ break;
+ case Vector:
+ Out << 'v';
+ break;
+ }
+
+ if (!!ParamAttr.Alignment)
+ Out << 'a' << ParamAttr.Alignment;
+ }
+
+ return Out.str();
+}
+
+// Function used to add the attribute. The parameter `VLEN` is
+// templated to allow the use of "x" when targeting scalable functions
+// for SVE.
+template <typename T>
+static void addAArch64VectorName(T VLEN, StringRef LMask, StringRef Prefix,
+ char ISA, StringRef ParSeq,
+ StringRef MangledName, bool OutputBecomesInput,
+ llvm::Function *Fn) {
+ SmallString<256> Buffer;
+ llvm::raw_svector_ostream Out(Buffer);
+ Out << Prefix << ISA << LMask << VLEN;
+ if (OutputBecomesInput)
+ Out << "v";
+ Out << ParSeq << "_" << MangledName;
+ Fn->addFnAttr(Out.str());
+}
+
+// Helper function to generate the Advanced SIMD names depending on
+// the value of the NDS when simdlen is not present.
+static void addAArch64AdvSIMDNDSNames(unsigned NDS, StringRef Mask,
+ StringRef Prefix, char ISA,
+ StringRef ParSeq, StringRef MangledName,
+ bool OutputBecomesInput,
+ llvm::Function *Fn) {
+ switch (NDS) {
+ case 8:
+ addAArch64VectorName(8, Mask, Prefix, ISA, ParSeq, MangledName,
+ OutputBecomesInput, Fn);
+ addAArch64VectorName(16, Mask, Prefix, ISA, ParSeq, MangledName,
+ OutputBecomesInput, Fn);
+ break;
+ case 16:
+ addAArch64VectorName(4, Mask, Prefix, ISA, ParSeq, MangledName,
+ OutputBecomesInput, Fn);
+ addAArch64VectorName(8, Mask, Prefix, ISA, ParSeq, MangledName,
+ OutputBecomesInput, Fn);
+ break;
+ case 32:
+ addAArch64VectorName(2, Mask, Prefix, ISA, ParSeq, MangledName,
+ OutputBecomesInput, Fn);
+ addAArch64VectorName(4, Mask, Prefix, ISA, ParSeq, MangledName,
+ OutputBecomesInput, Fn);
+ break;
+ case 64:
+ case 128:
+ addAArch64VectorName(2, Mask, Prefix, ISA, ParSeq, MangledName,
+ OutputBecomesInput, Fn);
+ break;
+ default:
+ llvm_unreachable("Scalar type is too wide.");
+ }
+}
+
+/// Emit vector function attributes for AArch64, as defined in the AAVFABI.
+static void emitAArch64DeclareSimdFunction(
+ CodeGenModule &CGM, const FunctionDecl *FD, unsigned UserVLEN,
+ ArrayRef<ParamAttrTy> ParamAttrs,
+ OMPDeclareSimdDeclAttr::BranchStateTy State, StringRef MangledName,
+ char ISA, unsigned VecRegSize, llvm::Function *Fn, SourceLocation SLoc) {
+
+ // Get basic data for building the vector signature.
+ const auto Data = getNDSWDS(FD, ParamAttrs);
+ const unsigned NDS = std::get<0>(Data);
+ const unsigned WDS = std::get<1>(Data);
+ const bool OutputBecomesInput = std::get<2>(Data);
+
+ // Check the values provided via `simdlen` by the user.
+ // 1. A `simdlen(1)` doesn't produce vector signatures,
+ if (UserVLEN == 1) {
+ unsigned DiagID = CGM.getDiags().getCustomDiagID(
+ DiagnosticsEngine::Warning,
+ "The clause simdlen(1) has no effect when targeting aarch64.");
+ CGM.getDiags().Report(SLoc, DiagID);
+ return;
+ }
+
+ // 2. Section 3.3.1, item 1: user input must be a power of 2 for
+ // Advanced SIMD output.
+ if (ISA == 'n' && UserVLEN && !llvm::isPowerOf2_32(UserVLEN)) {
+ unsigned DiagID = CGM.getDiags().getCustomDiagID(
+ DiagnosticsEngine::Warning, "The value specified in simdlen must be a "
+ "power of 2 when targeting Advanced SIMD.");
+ CGM.getDiags().Report(SLoc, DiagID);
+ return;
+ }
+
+ // 3. Section 3.4.1. SVE fixed lengh must obey the architectural
+ // limits.
+ if (ISA == 's' && UserVLEN != 0) {
+ if ((UserVLEN * WDS > 2048) || (UserVLEN * WDS % 128 != 0)) {
+ unsigned DiagID = CGM.getDiags().getCustomDiagID(
+ DiagnosticsEngine::Warning, "The clause simdlen must fit the %0-bit "
+ "lanes in the architectural constraints "
+ "for SVE (min is 128-bit, max is "
+ "2048-bit, by steps of 128-bit)");
+ CGM.getDiags().Report(SLoc, DiagID) << WDS;
+ return;
+ }
+ }
+
+ // Sort out parameter sequence.
+ const std::string ParSeq = mangleVectorParameters(ParamAttrs);
+ StringRef Prefix = "_ZGV";
+ // Generate simdlen from user input (if any).
+ if (UserVLEN) {
+ if (ISA == 's') {
+ // SVE generates only a masked function.
+ addAArch64VectorName(UserVLEN, "M", Prefix, ISA, ParSeq, MangledName,
+ OutputBecomesInput, Fn);
+ } else {
+ assert(ISA == 'n' && "Expected ISA either 's' or 'n'.");
+ // Advanced SIMD generates one or two functions, depending on
+ // the `[not]inbranch` clause.
+ switch (State) {
+ case OMPDeclareSimdDeclAttr::BS_Undefined:
+ addAArch64VectorName(UserVLEN, "N", Prefix, ISA, ParSeq, MangledName,
+ OutputBecomesInput, Fn);
+ addAArch64VectorName(UserVLEN, "M", Prefix, ISA, ParSeq, MangledName,
+ OutputBecomesInput, Fn);
+ break;
+ case OMPDeclareSimdDeclAttr::BS_Notinbranch:
+ addAArch64VectorName(UserVLEN, "N", Prefix, ISA, ParSeq, MangledName,
+ OutputBecomesInput, Fn);
+ break;
+ case OMPDeclareSimdDeclAttr::BS_Inbranch:
+ addAArch64VectorName(UserVLEN, "M", Prefix, ISA, ParSeq, MangledName,
+ OutputBecomesInput, Fn);
+ break;
+ }
+ }
+ } else {
+ // If no user simdlen is provided, follow the AAVFABI rules for
+ // generating the vector length.
+ if (ISA == 's') {
+ // SVE, section 3.4.1, item 1.
+ addAArch64VectorName("x", "M", Prefix, ISA, ParSeq, MangledName,
+ OutputBecomesInput, Fn);
+ } else {
+ assert(ISA == 'n' && "Expected ISA either 's' or 'n'.");
+ // Advanced SIMD, Section 3.3.1 of the AAVFABI, generates one or
+ // two vector names depending on the use of the clause
+ // `[not]inbranch`.
+ switch (State) {
+ case OMPDeclareSimdDeclAttr::BS_Undefined:
+ addAArch64AdvSIMDNDSNames(NDS, "N", Prefix, ISA, ParSeq, MangledName,
+ OutputBecomesInput, Fn);
+ addAArch64AdvSIMDNDSNames(NDS, "M", Prefix, ISA, ParSeq, MangledName,
+ OutputBecomesInput, Fn);
+ break;
+ case OMPDeclareSimdDeclAttr::BS_Notinbranch:
+ addAArch64AdvSIMDNDSNames(NDS, "N", Prefix, ISA, ParSeq, MangledName,
+ OutputBecomesInput, Fn);
+ break;
+ case OMPDeclareSimdDeclAttr::BS_Inbranch:
+ addAArch64AdvSIMDNDSNames(NDS, "M", Prefix, ISA, ParSeq, MangledName,
+ OutputBecomesInput, Fn);
+ break;
+ }
+ }
+ }
+}
+
void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD,
llvm::Function *Fn) {
ASTContext &C = CGM.getContext();
@@ -9734,12 +10035,26 @@
++MI;
}
llvm::APSInt VLENVal;
- if (const Expr *VLEN = Attr->getSimdlen())
- VLENVal = VLEN->EvaluateKnownConstInt(C);
+ SourceLocation ExprLoc;
+ const Expr *VLENExpr = Attr->getSimdlen();
+ if (VLENExpr) {
+ VLENVal = VLENExpr->EvaluateKnownConstInt(C);
+ ExprLoc = VLENExpr->getExprLoc();
+ }
OMPDeclareSimdDeclAttr::BranchStateTy State = Attr->getBranchState();
if (CGM.getTriple().getArch() == llvm::Triple::x86 ||
- CGM.getTriple().getArch() == llvm::Triple::x86_64)
+ CGM.getTriple().getArch() == llvm::Triple::x86_64) {
emitX86DeclareSimdFunction(FD, Fn, VLENVal, ParamAttrs, State);
+ } else if (CGM.getTriple().getArch() == llvm::Triple::aarch64) {
+ unsigned VLEN = VLENVal.getExtValue();
+ StringRef MangledName = Fn->getName();
+ if (CGM.getTarget().hasFeature("sve"))
+ emitAArch64DeclareSimdFunction(CGM, FD, VLEN, ParamAttrs, State,
+ MangledName, 's', 128, Fn, ExprLoc);
+ if (CGM.getTarget().hasFeature("neon"))
+ emitAArch64DeclareSimdFunction(CGM, FD, VLEN, ParamAttrs, State,
+ MangledName, 'n', 128, Fn, ExprLoc);
+ }
}
FD = FD->getPreviousDecl();
}