CodeGen: Fix address space of global variable
Certain targets (e.g. amdgcn) require global variable to stay in global or constant address
space. In C or C++ global variables are emitted in the default (generic) address space.
This patch introduces virtual functions TargetCodeGenInfo::getGlobalVarAddressSpace
and TargetInfo::getConstantAddressSpace to handle this in a general approach.
It only affects IR generated for amdgcn target.
Differential Revision: https://reviews.llvm.org/D33842
llvm-svn: 307470
diff --git a/clang/lib/CodeGen/CGBlocks.cpp b/clang/lib/CodeGen/CGBlocks.cpp
index f2de42b..1810489 100644
--- a/clang/lib/CodeGen/CGBlocks.cpp
+++ b/clang/lib/CodeGen/CGBlocks.cpp
@@ -736,9 +736,9 @@
llvm::Constant *isa =
(!CGM.getContext().getLangOpts().OpenCL)
? CGM.getNSConcreteStackBlock()
- : CGM.getNullPointer(cast<llvm::PointerType>(
- CGM.getNSConcreteStackBlock()->getType()),
- QualType(getContext().VoidPtrTy));
+ : CGM.getNullPointer(VoidPtrPtrTy,
+ CGM.getContext().getPointerType(
+ QualType(CGM.getContext().VoidPtrTy)));
isa = llvm::ConstantExpr::getBitCast(isa, VoidPtrTy);
// Build the block descriptor.
@@ -1141,12 +1141,11 @@
auto fields = builder.beginStruct();
// isa
- fields.add(
- (!CGM.getContext().getLangOpts().OpenCL)
- ? CGM.getNSConcreteGlobalBlock()
- : CGM.getNullPointer(cast<llvm::PointerType>(
- CGM.getNSConcreteGlobalBlock()->getType()),
- QualType(CGM.getContext().VoidPtrTy)));
+ fields.add((!CGM.getContext().getLangOpts().OpenCL)
+ ? CGM.getNSConcreteGlobalBlock()
+ : CGM.getNullPointer(CGM.VoidPtrPtrTy,
+ CGM.getContext().getPointerType(QualType(
+ CGM.getContext().VoidPtrTy))));
// __flags
BlockFlags flags = BLOCK_IS_GLOBAL | BLOCK_HAS_SIGNATURE;
diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp
index 4b656ea..2351786 100644
--- a/clang/lib/CodeGen/CGDecl.cpp
+++ b/clang/lib/CodeGen/CGDecl.cpp
@@ -221,8 +221,8 @@
Name = getStaticDeclName(*this, D);
llvm::Type *LTy = getTypes().ConvertTypeForMem(Ty);
- unsigned AddrSpace =
- GetGlobalVarAddressSpace(&D, getContext().getTargetAddressSpace(Ty));
+ unsigned AS = GetGlobalVarAddressSpace(&D);
+ unsigned TargetAS = getContext().getTargetAddressSpace(AS);
// Local address space cannot have an initializer.
llvm::Constant *Init = nullptr;
@@ -231,12 +231,9 @@
else
Init = llvm::UndefValue::get(LTy);
- llvm::GlobalVariable *GV =
- new llvm::GlobalVariable(getModule(), LTy,
- Ty.isConstant(getContext()), Linkage,
- Init, Name, nullptr,
- llvm::GlobalVariable::NotThreadLocal,
- AddrSpace);
+ llvm::GlobalVariable *GV = new llvm::GlobalVariable(
+ getModule(), LTy, Ty.isConstant(getContext()), Linkage, Init, Name,
+ nullptr, llvm::GlobalVariable::NotThreadLocal, TargetAS);
GV->setAlignment(getContext().getDeclAlign(&D).getQuantity());
setGlobalVisibility(GV, &D);
@@ -254,11 +251,12 @@
}
// Make sure the result is of the correct type.
- unsigned ExpectedAddrSpace = getContext().getTargetAddressSpace(Ty);
+ unsigned ExpectedAS = Ty.getAddressSpace();
llvm::Constant *Addr = GV;
- if (AddrSpace != ExpectedAddrSpace) {
- llvm::PointerType *PTy = llvm::PointerType::get(LTy, ExpectedAddrSpace);
- Addr = llvm::ConstantExpr::getAddrSpaceCast(GV, PTy);
+ if (AS != ExpectedAS) {
+ Addr = getTargetCodeGenInfo().performAddrSpaceCast(
+ *this, GV, AS, ExpectedAS,
+ LTy->getPointerTo(getContext().getTargetAddressSpace(ExpectedAS)));
}
setStaticLocalDeclAddress(&D, Addr);
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 2ee1c96..9f40ee5 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -338,9 +338,10 @@
}
}
-static Address
-createReferenceTemporary(CodeGenFunction &CGF,
- const MaterializeTemporaryExpr *M, const Expr *Inner) {
+static Address createReferenceTemporary(CodeGenFunction &CGF,
+ const MaterializeTemporaryExpr *M,
+ const Expr *Inner) {
+ auto &TCG = CGF.getTargetHooks();
switch (M->getStorageDuration()) {
case SD_FullExpression:
case SD_Automatic: {
@@ -353,13 +354,24 @@
(Ty->isArrayType() || Ty->isRecordType()) &&
CGF.CGM.isTypeConstant(Ty, true))
if (llvm::Constant *Init = CGF.CGM.EmitConstantExpr(Inner, Ty, &CGF)) {
- auto *GV = new llvm::GlobalVariable(
- CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
- llvm::GlobalValue::PrivateLinkage, Init, ".ref.tmp");
- CharUnits alignment = CGF.getContext().getTypeAlignInChars(Ty);
- GV->setAlignment(alignment.getQuantity());
- // FIXME: Should we put the new global into a COMDAT?
- return Address(GV, alignment);
+ if (auto AddrSpace = CGF.getTarget().getConstantAddressSpace()) {
+ auto AS = AddrSpace.getValue();
+ auto *GV = new llvm::GlobalVariable(
+ CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
+ llvm::GlobalValue::PrivateLinkage, Init, ".ref.tmp", nullptr,
+ llvm::GlobalValue::NotThreadLocal,
+ CGF.getContext().getTargetAddressSpace(AS));
+ CharUnits alignment = CGF.getContext().getTypeAlignInChars(Ty);
+ GV->setAlignment(alignment.getQuantity());
+ llvm::Constant *C = GV;
+ if (AS != LangAS::Default)
+ C = TCG.performAddrSpaceCast(
+ CGF.CGM, GV, AS, LangAS::Default,
+ GV->getValueType()->getPointerTo(
+ CGF.getContext().getTargetAddressSpace(LangAS::Default)));
+ // FIXME: Should we put the new global into a COMDAT?
+ return Address(C, alignment);
+ }
}
return CGF.CreateMemTemp(Ty, "ref.tmp");
}
@@ -440,9 +452,11 @@
// Create and initialize the reference temporary.
Address Object = createReferenceTemporary(*this, M, E);
- if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object.getPointer())) {
+ if (auto *Var = dyn_cast<llvm::GlobalVariable>(
+ Object.getPointer()->stripPointerCasts())) {
Object = Address(llvm::ConstantExpr::getBitCast(
- Var, ConvertTypeForMem(E->getType())->getPointerTo()),
+ cast<llvm::Constant>(Object.getPointer()),
+ ConvertTypeForMem(E->getType())->getPointerTo()),
Object.getAlignment());
// If the temporary is a global and has a constant initializer or is a
// constant temporary that we promoted to a global, we may have already
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 6785111b..85aa5d4 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -1479,6 +1479,9 @@
const TargetInfo &getTarget() const { return Target; }
llvm::LLVMContext &getLLVMContext() { return CGM.getLLVMContext(); }
+ const TargetCodeGenInfo &getTargetHooks() const {
+ return CGM.getTargetCodeGenInfo();
+ }
//===--------------------------------------------------------------------===//
// Cleanups
@@ -3820,10 +3823,6 @@
private:
QualType getVarArgType(const Expr *Arg);
- const TargetCodeGenInfo &getTargetHooks() const {
- return CGM.getTargetCodeGenInfo();
- }
-
void EmitDeclMetadata();
BlockByrefHelpers *buildByrefHelpers(llvm::StructType &byrefType,
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 10838b7..e1866ea 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -2367,11 +2367,13 @@
return llvm::ConstantExpr::getBitCast(Entry, Ty);
}
- unsigned AddrSpace = GetGlobalVarAddressSpace(D, Ty->getAddressSpace());
+ auto AddrSpace = GetGlobalVarAddressSpace(D);
+ auto TargetAddrSpace = getContext().getTargetAddressSpace(AddrSpace);
+
auto *GV = new llvm::GlobalVariable(
getModule(), Ty->getElementType(), false,
llvm::GlobalValue::ExternalLinkage, nullptr, MangledName, nullptr,
- llvm::GlobalVariable::NotThreadLocal, AddrSpace);
+ llvm::GlobalVariable::NotThreadLocal, TargetAddrSpace);
// If we already created a global with the same mangled name (but different
// type) before, take its name and remove it from its parent.
@@ -2428,8 +2430,14 @@
GV->setSection(".cp.rodata");
}
- if (AddrSpace != Ty->getAddressSpace())
- return llvm::ConstantExpr::getAddrSpaceCast(GV, Ty);
+ auto ExpectedAS =
+ D ? D->getType().getAddressSpace()
+ : (LangOpts.OpenCL ? LangAS::opencl_global : LangAS::Default);
+ auto ExpectedTargetAS = getContext().getTargetAddressSpace(ExpectedAS);
+ assert(ExpectedTargetAS == Ty->getPointerAddressSpace());
+ if (AddrSpace != ExpectedAS)
+ return getTargetCodeGenInfo().performAddrSpaceCast(*this, GV, AddrSpace,
+ ExpectedAS, Ty);
return GV;
}
@@ -2563,18 +2571,27 @@
getDataLayout().getTypeStoreSizeInBits(Ty));
}
-unsigned CodeGenModule::GetGlobalVarAddressSpace(const VarDecl *D,
- unsigned AddrSpace) {
- if (D && LangOpts.CUDA && LangOpts.CUDAIsDevice) {
- if (D->hasAttr<CUDAConstantAttr>())
- AddrSpace = getContext().getTargetAddressSpace(LangAS::cuda_constant);
- else if (D->hasAttr<CUDASharedAttr>())
- AddrSpace = getContext().getTargetAddressSpace(LangAS::cuda_shared);
- else
- AddrSpace = getContext().getTargetAddressSpace(LangAS::cuda_device);
+unsigned CodeGenModule::GetGlobalVarAddressSpace(const VarDecl *D) {
+ unsigned AddrSpace;
+ if (LangOpts.OpenCL) {
+ AddrSpace = D ? D->getType().getAddressSpace() : LangAS::opencl_global;
+ assert(AddrSpace == LangAS::opencl_global ||
+ AddrSpace == LangAS::opencl_constant ||
+ AddrSpace == LangAS::opencl_local ||
+ AddrSpace >= LangAS::FirstTargetAddressSpace);
+ return AddrSpace;
}
- return AddrSpace;
+ if (LangOpts.CUDA && LangOpts.CUDAIsDevice) {
+ if (D && D->hasAttr<CUDAConstantAttr>())
+ return LangAS::cuda_constant;
+ else if (D && D->hasAttr<CUDASharedAttr>())
+ return LangAS::cuda_shared;
+ else
+ return LangAS::cuda_device;
+ }
+
+ return getTargetCodeGenInfo().getGlobalVarAddressSpace(*this, D);
}
template<typename SomeDecl>
@@ -2727,10 +2744,9 @@
// "extern int x[];") and then a definition of a different type (e.g.
// "int x[10];"). This also happens when an initializer has a different type
// from the type of the global (this happens with unions).
- if (!GV ||
- GV->getType()->getElementType() != InitType ||
+ if (!GV || GV->getType()->getElementType() != InitType ||
GV->getType()->getAddressSpace() !=
- GetGlobalVarAddressSpace(D, getContext().getTargetAddressSpace(ASTTy))) {
+ getContext().getTargetAddressSpace(GetGlobalVarAddressSpace(D))) {
// Move the old entry aside so that we'll create a new one.
Entry->setName(StringRef());
@@ -3739,20 +3755,26 @@
Linkage = llvm::GlobalVariable::InternalLinkage;
}
}
- unsigned AddrSpace = GetGlobalVarAddressSpace(
- VD, getContext().getTargetAddressSpace(MaterializedType));
+ unsigned AddrSpace =
+ VD ? GetGlobalVarAddressSpace(VD) : MaterializedType.getAddressSpace();
+ auto TargetAS = getContext().getTargetAddressSpace(AddrSpace);
auto *GV = new llvm::GlobalVariable(
getModule(), Type, Constant, Linkage, InitialValue, Name.c_str(),
- /*InsertBefore=*/nullptr, llvm::GlobalVariable::NotThreadLocal,
- AddrSpace);
+ /*InsertBefore=*/nullptr, llvm::GlobalVariable::NotThreadLocal, TargetAS);
setGlobalVisibility(GV, VD);
GV->setAlignment(Align.getQuantity());
if (supportsCOMDAT() && GV->isWeakForLinker())
GV->setComdat(TheModule.getOrInsertComdat(GV->getName()));
if (VD->getTLSKind())
setTLSMode(GV, *VD);
- MaterializedGlobalTemporaryMap[E] = GV;
- return ConstantAddress(GV, Align);
+ llvm::Constant *CV = GV;
+ if (AddrSpace != LangAS::Default)
+ CV = getTargetCodeGenInfo().performAddrSpaceCast(
+ *this, GV, AddrSpace, LangAS::Default,
+ Type->getPointerTo(
+ getContext().getTargetAddressSpace(LangAS::Default)));
+ MaterializedGlobalTemporaryMap[E] = CV;
+ return ConstantAddress(CV, Align);
}
/// EmitObjCPropertyImplementations - Emit information for synthesized
diff --git a/clang/lib/CodeGen/CodeGenModule.h b/clang/lib/CodeGen/CodeGenModule.h
index c5f1a2b..b162e72 100644
--- a/clang/lib/CodeGen/CodeGenModule.h
+++ b/clang/lib/CodeGen/CodeGenModule.h
@@ -710,11 +710,15 @@
SourceLocation Loc = SourceLocation(),
bool TLS = false);
- /// Return the address space of the underlying global variable for D, as
+ /// Return the AST address space of the underlying global variable for D, as
/// determined by its declaration. Normally this is the same as the address
/// space of D's type, but in CUDA, address spaces are associated with
- /// declarations, not types.
- unsigned GetGlobalVarAddressSpace(const VarDecl *D, unsigned AddrSpace);
+ /// declarations, not types. If D is nullptr, return the default address
+ /// space for global variable.
+ ///
+ /// For languages without explicit address spaces, if D has default address
+ /// space, target-specific global or constant address space may be returned.
+ unsigned GetGlobalVarAddressSpace(const VarDecl *D);
/// Return the llvm::Constant for the address of the given global variable.
/// If Ty is non-null and if the global doesn't exist, then it will be created
diff --git a/clang/lib/CodeGen/TargetInfo.cpp b/clang/lib/CodeGen/TargetInfo.cpp
index 8d00e05..faec994 100644
--- a/clang/lib/CodeGen/TargetInfo.cpp
+++ b/clang/lib/CodeGen/TargetInfo.cpp
@@ -416,14 +416,33 @@
return llvm::ConstantPointerNull::get(T);
}
+unsigned TargetCodeGenInfo::getGlobalVarAddressSpace(CodeGenModule &CGM,
+ const VarDecl *D) const {
+ assert(!CGM.getLangOpts().OpenCL &&
+ !(CGM.getLangOpts().CUDA && CGM.getLangOpts().CUDAIsDevice) &&
+ "Address space agnostic languages only");
+ return D ? D->getType().getAddressSpace() : LangAS::Default;
+}
+
llvm::Value *TargetCodeGenInfo::performAddrSpaceCast(
CodeGen::CodeGenFunction &CGF, llvm::Value *Src, unsigned SrcAddr,
unsigned DestAddr, llvm::Type *DestTy, bool isNonNull) const {
// Since target may map different address spaces in AST to the same address
// space, an address space conversion may end up as a bitcast.
+ if (auto *C = dyn_cast<llvm::Constant>(Src))
+ return performAddrSpaceCast(CGF.CGM, C, SrcAddr, DestAddr, DestTy);
return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Src, DestTy);
}
+llvm::Constant *
+TargetCodeGenInfo::performAddrSpaceCast(CodeGenModule &CGM, llvm::Constant *Src,
+ unsigned SrcAddr, unsigned DestAddr,
+ llvm::Type *DestTy) const {
+ // Since target may map different address spaces in AST to the same address
+ // space, an address space conversion may end up as a bitcast.
+ return llvm::ConstantExpr::getPointerCast(Src, DestTy);
+}
+
static bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays);
/// isEmptyField - Return true iff a the field is "empty", that is it
@@ -7325,6 +7344,8 @@
return LangAS::FirstTargetAddressSpace +
getABIInfo().getDataLayout().getAllocaAddrSpace();
}
+ unsigned getGlobalVarAddressSpace(CodeGenModule &CGM,
+ const VarDecl *D) const override;
};
}
@@ -7408,6 +7429,31 @@
llvm::ConstantPointerNull::get(NPT), PT);
}
+unsigned
+AMDGPUTargetCodeGenInfo::getGlobalVarAddressSpace(CodeGenModule &CGM,
+ const VarDecl *D) const {
+ assert(!CGM.getLangOpts().OpenCL &&
+ !(CGM.getLangOpts().CUDA && CGM.getLangOpts().CUDAIsDevice) &&
+ "Address space agnostic languages only");
+ unsigned DefaultGlobalAS =
+ LangAS::FirstTargetAddressSpace +
+ CGM.getContext().getTargetAddressSpace(LangAS::opencl_global);
+ if (!D)
+ return DefaultGlobalAS;
+
+ unsigned AddrSpace = D->getType().getAddressSpace();
+ assert(AddrSpace == LangAS::Default ||
+ AddrSpace >= LangAS::FirstTargetAddressSpace);
+ if (AddrSpace != LangAS::Default)
+ return AddrSpace;
+
+ if (CGM.isTypeConstant(D->getType(), false)) {
+ if (auto ConstAS = CGM.getTarget().getConstantAddressSpace())
+ return ConstAS.getValue();
+ }
+ return DefaultGlobalAS;
+}
+
//===----------------------------------------------------------------------===//
// SPARC v8 ABI Implementation.
// Based on the SPARC Compliance Definition version 2.4.1.
diff --git a/clang/lib/CodeGen/TargetInfo.h b/clang/lib/CodeGen/TargetInfo.h
index 247d01d..952ef96 100644
--- a/clang/lib/CodeGen/TargetInfo.h
+++ b/clang/lib/CodeGen/TargetInfo.h
@@ -229,6 +229,13 @@
virtual llvm::Constant *getNullPointer(const CodeGen::CodeGenModule &CGM,
llvm::PointerType *T, QualType QT) const;
+ /// Get target favored AST address space of a global variable for languages
+ /// other than OpenCL and CUDA.
+ /// If \p D is nullptr, returns the default target favored address space
+ /// for global variable.
+ virtual unsigned getGlobalVarAddressSpace(CodeGenModule &CGM,
+ const VarDecl *D) const;
+
/// Get the AST address space for alloca.
virtual unsigned getASTAllocaAddressSpace() const { return LangAS::Default; }
@@ -243,6 +250,15 @@
unsigned DestAddr,
llvm::Type *DestTy,
bool IsNonNull = false) const;
+
+ /// Perform address space cast of a constant expression of pointer type.
+ /// \param V is the LLVM constant to be casted to another address space.
+ /// \param SrcAddr is the language address space of \p V.
+ /// \param DestAddr is the targeted language address space.
+ /// \param DestTy is the destination LLVM pointer type.
+ virtual llvm::Constant *
+ performAddrSpaceCast(CodeGenModule &CGM, llvm::Constant *V, unsigned SrcAddr,
+ unsigned DestAddr, llvm::Type *DestTy) const;
};
} // namespace CodeGen