[hip][cuda] Fix the extended lambda name mangling issue.
Summary:
- HIP/CUDA host side needs to use device kernel symbol name to match the
device side binaries. Without a consistent naming between host- and
device-side compilations, it's risky that wrong device binaries are
executed. Consistent naming is usually not an issue until unnamed
types are used, especially the lambda. In this patch, the consistent
name mangling is addressed for the extended lambdas, i.e. the lambdas
annotated with `__device__`.
- In [Itanium C++ ABI][1], the mangling of the lambda is generally
unspecified unless, in certain cases, ODR rule is required to ensure
consisent naming cross TUs. The extended lambda is such a case as its
name may be part of a device kernel function, e.g., the extended
lambda is used as a template argument and etc. Thus, we need to force
ODR for extended lambdas as they are referenced in both device- and
host-side TUs. Furthermore, if a extended lambda is nested in other
(extended or not) lambdas, those lambdas are required to follow ODR
naming as well. This patch revises the current lambda mangle numbering
to force ODR from an extended lambda to all its parent lambdas.
- On the other side, the aforementioned ODR naming should not change
those lambdas' original linkages, i.e., we cannot replace the original
`internal` with `linkonce_odr`; otherwise, we may violate ODR in
general. This patch introduces a new field `HasKnownInternalLinkage`
in lambda data to decouple the current linkage calculation based on
mangling number assigned.
[1]: https://itanium-cxx-abi.github.io/cxx-abi/abi.html
Reviewers: tra, rsmith, yaxunl, martong, shafik
Subscribers: cfe-commits
Tags: #clang
Differential Revision: https://reviews.llvm.org/D68818
llvm-svn: 375309
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index a549cf9..54acca7 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -2694,7 +2694,8 @@
ExpectedDecl CDeclOrErr = import(DCXX->getLambdaContextDecl());
if (!CDeclOrErr)
return CDeclOrErr.takeError();
- D2CXX->setLambdaMangling(DCXX->getLambdaManglingNumber(), *CDeclOrErr);
+ D2CXX->setLambdaMangling(DCXX->getLambdaManglingNumber(), *CDeclOrErr,
+ DCXX->hasKnownLambdaInternalLinkage());
} else if (DCXX->isInjectedClassName()) {
// We have to be careful to do a similar dance to the one in
// Sema::ActOnStartCXXMemberDeclarations
diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index 9ebf1c3..80235d8 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -1385,7 +1385,8 @@
case Decl::CXXRecord: {
const auto *Record = cast<CXXRecordDecl>(D);
if (Record->isLambda()) {
- if (!Record->getLambdaManglingNumber()) {
+ if (Record->hasKnownLambdaInternalLinkage() ||
+ !Record->getLambdaManglingNumber()) {
// This lambda has no mangling number, so it's internal.
return getInternalLinkageFor(D);
}
@@ -1402,7 +1403,8 @@
// };
const CXXRecordDecl *OuterMostLambda =
getOutermostEnclosingLambda(Record);
- if (!OuterMostLambda->getLambdaManglingNumber())
+ if (OuterMostLambda->hasKnownLambdaInternalLinkage() ||
+ !OuterMostLambda->getLambdaManglingNumber())
return getInternalLinkageFor(D);
return getLVForClosure(
diff --git a/clang/lib/Sema/SemaLambda.cpp b/clang/lib/Sema/SemaLambda.cpp
index df4f0dd..749b0f2 100644
--- a/clang/lib/Sema/SemaLambda.cpp
+++ b/clang/lib/Sema/SemaLambda.cpp
@@ -335,7 +335,7 @@
case StaticDataMember:
// -- the initializers of nonspecialized static members of template classes
if (!IsInNonspecializedTemplate)
- return std::make_tuple(nullptr, nullptr);
+ return std::make_tuple(nullptr, ManglingContextDecl);
// Fall through to get the current context.
LLVM_FALLTHROUGH;
@@ -356,14 +356,15 @@
llvm_unreachable("unexpected context");
}
-CXXMethodDecl *Sema::startLambdaDefinition(
- CXXRecordDecl *Class, SourceRange IntroducerRange,
- TypeSourceInfo *MethodTypeInfo, SourceLocation EndLoc,
- ArrayRef<ParmVarDecl *> Params, ConstexprSpecKind ConstexprKind,
- Optional<std::pair<unsigned, Decl *>> Mangling) {
+CXXMethodDecl *Sema::startLambdaDefinition(CXXRecordDecl *Class,
+ SourceRange IntroducerRange,
+ TypeSourceInfo *MethodTypeInfo,
+ SourceLocation EndLoc,
+ ArrayRef<ParmVarDecl *> Params,
+ ConstexprSpecKind ConstexprKind) {
QualType MethodType = MethodTypeInfo->getType();
TemplateParameterList *TemplateParams =
- getGenericLambdaTemplateParameterList(getCurLambda(), *this);
+ getGenericLambdaTemplateParameterList(getCurLambda(), *this);
// If a lambda appears in a dependent context or is a generic lambda (has
// template parameters) and has an 'auto' return type, deduce it to a
// dependent type.
@@ -425,20 +426,55 @@
P->setOwningFunction(Method);
}
+ return Method;
+}
+
+void Sema::handleLambdaNumbering(
+ CXXRecordDecl *Class, CXXMethodDecl *Method,
+ Optional<std::tuple<unsigned, bool, Decl *>> Mangling) {
if (Mangling) {
- Class->setLambdaMangling(Mangling->first, Mangling->second);
- } else {
- MangleNumberingContext *MCtx;
+ unsigned ManglingNumber;
+ bool HasKnownInternalLinkage;
Decl *ManglingContextDecl;
- std::tie(MCtx, ManglingContextDecl) =
- getCurrentMangleNumberContext(Class->getDeclContext());
- if (MCtx) {
- unsigned ManglingNumber = MCtx->getManglingNumber(Method);
- Class->setLambdaMangling(ManglingNumber, ManglingContextDecl);
- }
+ std::tie(ManglingNumber, HasKnownInternalLinkage, ManglingContextDecl) =
+ Mangling.getValue();
+ Class->setLambdaMangling(ManglingNumber, ManglingContextDecl,
+ HasKnownInternalLinkage);
+ return;
}
- return Method;
+ auto getMangleNumberingContext =
+ [this](CXXRecordDecl *Class, Decl *ManglingContextDecl) -> MangleNumberingContext * {
+ // Get mangle numbering context if there's any extra decl context.
+ if (ManglingContextDecl)
+ return &Context.getManglingNumberContext(
+ ASTContext::NeedExtraManglingDecl, ManglingContextDecl);
+ // Otherwise, from that lambda's decl context.
+ auto DC = Class->getDeclContext();
+ while (auto *CD = dyn_cast<CapturedDecl>(DC))
+ DC = CD->getParent();
+ return &Context.getManglingNumberContext(DC);
+ };
+
+ MangleNumberingContext *MCtx;
+ Decl *ManglingContextDecl;
+ std::tie(MCtx, ManglingContextDecl) =
+ getCurrentMangleNumberContext(Class->getDeclContext());
+ bool HasKnownInternalLinkage = false;
+ if (!MCtx && getLangOpts().CUDA) {
+ // Force lambda numbering in CUDA/HIP as we need to name lambdas following
+ // ODR. Both device- and host-compilation need to have a consistent naming
+ // on kernel functions. As lambdas are potential part of these `__global__`
+ // function names, they needs numbering following ODR.
+ MCtx = getMangleNumberingContext(Class, ManglingContextDecl);
+ assert(MCtx && "Retrieving mangle numbering context failed!");
+ HasKnownInternalLinkage = true;
+ }
+ if (MCtx) {
+ unsigned ManglingNumber = MCtx->getManglingNumber(Method);
+ Class->setLambdaMangling(ManglingNumber, ManglingContextDecl,
+ HasKnownInternalLinkage);
+ }
}
void Sema::buildLambdaScope(LambdaScopeInfo *LSI,
@@ -951,6 +987,9 @@
if (getLangOpts().CUDA)
CUDASetLambdaAttrs(Method);
+ // Number the lambda for linkage purposes if necessary.
+ handleLambdaNumbering(Class, Method);
+
// Introduce the function call operator as the current declaration context.
PushDeclContext(CurScope, Method);
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 89a7b8c..4b3a670 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -11497,17 +11497,18 @@
E->getCaptureDefault());
getDerived().transformedLocalDecl(OldClass, {Class});
- Optional<std::pair<unsigned, Decl*>> Mangling;
+ Optional<std::tuple<unsigned, bool, Decl *>> Mangling;
if (getDerived().ReplacingOriginal())
- Mangling = std::make_pair(OldClass->getLambdaManglingNumber(),
- OldClass->getLambdaContextDecl());
+ Mangling = std::make_tuple(OldClass->getLambdaManglingNumber(),
+ OldClass->hasKnownLambdaInternalLinkage(),
+ OldClass->getLambdaContextDecl());
// Build the call operator.
CXXMethodDecl *NewCallOperator = getSema().startLambdaDefinition(
Class, E->getIntroducerRange(), NewCallOpTSI,
E->getCallOperator()->getEndLoc(),
NewCallOpTSI->getTypeLoc().castAs<FunctionProtoTypeLoc>().getParams(),
- E->getCallOperator()->getConstexprKind(), Mangling);
+ E->getCallOperator()->getConstexprKind());
LSI->CallOperator = NewCallOperator;
@@ -11527,6 +11528,9 @@
getDerived().transformAttrs(E->getCallOperator(), NewCallOperator);
getDerived().transformedLocalDecl(E->getCallOperator(), {NewCallOperator});
+ // Number the lambda for linkage purposes if necessary.
+ getSema().handleLambdaNumbering(Class, NewCallOperator, Mangling);
+
// Introduce the context of the call operator.
Sema::ContextRAII SavedContext(getSema(), NewCallOperator,
/*NewThisContext*/false);
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index 65d6252..9aa8c77 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -1690,6 +1690,7 @@
Lambda.CaptureDefault = Record.readInt();
Lambda.NumCaptures = Record.readInt();
Lambda.NumExplicitCaptures = Record.readInt();
+ Lambda.HasKnownInternalLinkage = Record.readInt();
Lambda.ManglingNumber = Record.readInt();
Lambda.ContextDecl = ReadDeclID();
Lambda.Captures = (Capture *)Reader.getContext().Allocate(
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index c6b05c9..28affed 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -6224,6 +6224,7 @@
Record->push_back(Lambda.CaptureDefault);
Record->push_back(Lambda.NumCaptures);
Record->push_back(Lambda.NumExplicitCaptures);
+ Record->push_back(Lambda.HasKnownInternalLinkage);
Record->push_back(Lambda.ManglingNumber);
AddDeclRef(D->getLambdaContextDecl());
AddTypeSourceInfo(Lambda.MethodTyInfo);