Update aosp/master clang for rebase to r256229
http://b/26987366
Change-Id: I5d349c9843ea5c24d6e455956f8a446393b6873d
diff --git a/lib/CodeGen/ABIInfo.h b/lib/CodeGen/ABIInfo.h
index cc8652e..a65f270 100644
--- a/lib/CodeGen/ABIInfo.h
+++ b/lib/CodeGen/ABIInfo.h
@@ -25,6 +25,8 @@
class TargetInfo;
namespace CodeGen {
+ class ABIArgInfo;
+ class Address;
class CGCXXABI;
class CGFunctionInfo;
class CodeGenFunction;
@@ -79,8 +81,15 @@
// the ABI information any lower than CodeGen. Of course, for
// VAArg handling it has to be at this level; there is no way to
// abstract this out.
- virtual llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGen::CodeGenFunction &CGF) const = 0;
+ virtual CodeGen::Address EmitVAArg(CodeGen::CodeGenFunction &CGF,
+ CodeGen::Address VAListAddr,
+ QualType Ty) const = 0;
+
+ /// Emit the target dependent code to load a value of
+ /// \arg Ty from the \c __builtin_ms_va_list pointed to by \arg VAListAddr.
+ virtual CodeGen::Address EmitMSVAArg(CodeGen::CodeGenFunction &CGF,
+ CodeGen::Address VAListAddr,
+ QualType Ty) const;
virtual bool isHomogeneousAggregateBaseType(QualType Ty) const;
@@ -92,6 +101,15 @@
bool isHomogeneousAggregate(QualType Ty, const Type *&Base,
uint64_t &Members) const;
+ /// A convenience method to return an indirect ABIArgInfo with an
+ /// expected alignment equal to the ABI alignment of the given type.
+ CodeGen::ABIArgInfo
+ getNaturalAlignIndirect(QualType Ty, bool ByRef = true,
+ bool Realign = false,
+ llvm::Type *Padding = nullptr) const;
+
+ CodeGen::ABIArgInfo
+ getNaturalAlignIndirectInReg(QualType Ty, bool Realign = false) const;
};
} // end namespace clang
diff --git a/lib/CodeGen/Address.h b/lib/CodeGen/Address.h
new file mode 100644
index 0000000..9d145fa
--- /dev/null
+++ b/lib/CodeGen/Address.h
@@ -0,0 +1,126 @@
+//===-- Address.h - An aligned address -------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This class provides a simple wrapper for a pair of a pointer and an
+// alignment.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_CODEGEN_ADDRESS_H
+#define LLVM_CLANG_LIB_CODEGEN_ADDRESS_H
+
+#include "llvm/IR/Constants.h"
+#include "clang/AST/CharUnits.h"
+
+namespace clang {
+namespace CodeGen {
+
+/// An aligned address.
+class Address {
+ llvm::Value *Pointer;
+ CharUnits Alignment;
+public:
+ Address(llvm::Value *pointer, CharUnits alignment)
+ : Pointer(pointer), Alignment(alignment) {
+ assert((!alignment.isZero() || pointer == nullptr) &&
+ "creating valid address with invalid alignment");
+ }
+
+ static Address invalid() { return Address(nullptr, CharUnits()); }
+ bool isValid() const { return Pointer != nullptr; }
+
+ llvm::Value *getPointer() const {
+ assert(isValid());
+ return Pointer;
+ }
+
+ /// Return the type of the pointer value.
+ llvm::PointerType *getType() const {
+ return llvm::cast<llvm::PointerType>(getPointer()->getType());
+ }
+
+ /// Return the type of the values stored in this address.
+ ///
+ /// When IR pointer types lose their element type, we should simply
+ /// store it in Address instead for the convenience of writing code.
+ llvm::Type *getElementType() const {
+ return getType()->getElementType();
+ }
+
+ /// Return the address space that this address resides in.
+ unsigned getAddressSpace() const {
+ return getType()->getAddressSpace();
+ }
+
+ /// Return the IR name of the pointer value.
+ llvm::StringRef getName() const {
+ return getPointer()->getName();
+ }
+
+ /// Return the alignment of this pointer.
+ CharUnits getAlignment() const {
+ assert(isValid());
+ return Alignment;
+ }
+};
+
+/// A specialization of Address that requires the address to be an
+/// LLVM Constant.
+class ConstantAddress : public Address {
+public:
+ ConstantAddress(llvm::Constant *pointer, CharUnits alignment)
+ : Address(pointer, alignment) {}
+
+ static ConstantAddress invalid() {
+ return ConstantAddress(nullptr, CharUnits());
+ }
+
+ llvm::Constant *getPointer() const {
+ return llvm::cast<llvm::Constant>(Address::getPointer());
+ }
+
+ ConstantAddress getBitCast(llvm::Type *ty) const {
+ return ConstantAddress(llvm::ConstantExpr::getBitCast(getPointer(), ty),
+ getAlignment());
+ }
+
+ ConstantAddress getElementBitCast(llvm::Type *ty) const {
+ return getBitCast(ty->getPointerTo(getAddressSpace()));
+ }
+
+ static bool isaImpl(Address addr) {
+ return llvm::isa<llvm::Constant>(addr.getPointer());
+ }
+ static ConstantAddress castImpl(Address addr) {
+ return ConstantAddress(llvm::cast<llvm::Constant>(addr.getPointer()),
+ addr.getAlignment());
+ }
+};
+
+}
+}
+
+namespace llvm {
+ // Present a minimal LLVM-like casting interface.
+ template <class U> inline U cast(clang::CodeGen::Address addr) {
+ return U::castImpl(addr);
+ }
+ template <class U> inline bool isa(clang::CodeGen::Address addr) {
+ return U::isaImpl(addr);
+ }
+}
+
+namespace clang {
+ // Make our custom isa and cast available in namespace clang, to mirror
+ // what we do for LLVM's versions in Basic/LLVM.h.
+ using llvm::isa;
+ using llvm::cast;
+}
+
+#endif
diff --git a/lib/CodeGen/Android.mk b/lib/CodeGen/Android.mk
index 0341a3c..00219c7 100644
--- a/lib/CodeGen/Android.mk
+++ b/lib/CodeGen/Android.mk
@@ -61,6 +61,7 @@
ItaniumCXXABI.cpp \
MicrosoftCXXABI.cpp \
ModuleBuilder.cpp \
+ ObjectFilePCHContainerOperations.cpp \
SanitizerMetadata.cpp \
TargetInfo.cpp
@@ -77,6 +78,7 @@
include $(CLANG_HOST_BUILD_MK)
include $(CLANG_VERSION_INC_MK)
include $(CLANG_TBLGEN_RULES_MK)
+include $(LLVM_GEN_ATTRIBUTES_MK)
include $(LLVM_GEN_INTRINSICS_MK)
include $(BUILD_HOST_STATIC_LIBRARY)
@@ -93,5 +95,6 @@
include $(CLANG_DEVICE_BUILD_MK)
include $(CLANG_VERSION_INC_MK)
include $(CLANG_TBLGEN_RULES_MK)
+include $(LLVM_GEN_ATTRIBUTES_MK)
include $(LLVM_GEN_INTRINSICS_MK)
include $(BUILD_STATIC_LIBRARY)
diff --git a/lib/CodeGen/BackendUtil.cpp b/lib/CodeGen/BackendUtil.cpp
index 30e9ebf..82297e7 100644
--- a/lib/CodeGen/BackendUtil.cpp
+++ b/lib/CodeGen/BackendUtil.cpp
@@ -14,6 +14,7 @@
#include "clang/Frontend/CodeGenOptions.h"
#include "clang/Frontend/FrontendDiagnostic.h"
#include "clang/Frontend/Utils.h"
+#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
@@ -21,6 +22,7 @@
#include "llvm/CodeGen/RegAllocRegistry.h"
#include "llvm/CodeGen/SchedulerRegistry.h"
#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/FunctionInfo.h"
#include "llvm/IR/IRPrintingPasses.h"
#include "llvm/IR/LegacyPassManager.h"
#include "llvm/IR/Module.h"
@@ -52,6 +54,7 @@
const clang::TargetOptions &TargetOpts;
const LangOptions &LangOpts;
Module *TheModule;
+ std::unique_ptr<FunctionInfoIndex> FunctionIndex;
Timer CodeGenerationTime;
@@ -112,15 +115,14 @@
bool AddEmitPasses(BackendAction Action, raw_pwrite_stream &OS);
public:
- EmitAssemblyHelper(DiagnosticsEngine &_Diags,
- const CodeGenOptions &CGOpts,
+ EmitAssemblyHelper(DiagnosticsEngine &_Diags, const CodeGenOptions &CGOpts,
const clang::TargetOptions &TOpts,
- const LangOptions &LOpts,
- Module *M)
- : Diags(_Diags), CodeGenOpts(CGOpts), TargetOpts(TOpts), LangOpts(LOpts),
- TheModule(M), CodeGenerationTime("Code Generation Time"),
- CodeGenPasses(nullptr), PerModulePasses(nullptr),
- PerFunctionPasses(nullptr) {}
+ const LangOptions &LOpts, Module *M,
+ std::unique_ptr<FunctionInfoIndex> Index)
+ : Diags(_Diags), CodeGenOpts(CGOpts), TargetOpts(TOpts), LangOpts(LOpts),
+ TheModule(M), FunctionIndex(std::move(Index)),
+ CodeGenerationTime("Code Generation Time"), CodeGenPasses(nullptr),
+ PerModulePasses(nullptr), PerFunctionPasses(nullptr) {}
~EmitAssemblyHelper() {
delete CodeGenPasses;
@@ -166,14 +168,6 @@
PM.add(createObjCARCOptPass());
}
-static void addSampleProfileLoaderPass(const PassManagerBuilder &Builder,
- legacy::PassManagerBase &PM) {
- const PassManagerBuilderWrapper &BuilderWrapper =
- static_cast<const PassManagerBuilderWrapper &>(Builder);
- const CodeGenOptions &CGOpts = BuilderWrapper.getCGOpts();
- PM.add(createSampleProfileLoaderPass(CGOpts.SampleProfileFile));
-}
-
static void addAddDiscriminatorsPass(const PassManagerBuilder &Builder,
legacy::PassManagerBase &PM) {
PM.add(createAddDiscriminatorsPass());
@@ -201,8 +195,20 @@
static void addAddressSanitizerPasses(const PassManagerBuilder &Builder,
legacy::PassManagerBase &PM) {
- PM.add(createAddressSanitizerFunctionPass());
- PM.add(createAddressSanitizerModulePass());
+ const PassManagerBuilderWrapper &BuilderWrapper =
+ static_cast<const PassManagerBuilderWrapper&>(Builder);
+ const CodeGenOptions &CGOpts = BuilderWrapper.getCGOpts();
+ bool Recover = CGOpts.SanitizeRecover.has(SanitizerKind::Address);
+ PM.add(createAddressSanitizerFunctionPass(/*CompileKernel*/false, Recover));
+ PM.add(createAddressSanitizerModulePass(/*CompileKernel*/false, Recover));
+}
+
+static void addKernelAddressSanitizerPasses(const PassManagerBuilder &Builder,
+ legacy::PassManagerBase &PM) {
+ PM.add(createAddressSanitizerFunctionPass(/*CompileKernel*/true,
+ /*Recover*/true));
+ PM.add(createAddressSanitizerModulePass(/*CompileKernel*/true,
+ /*Recover*/true));
}
static void addMemorySanitizerPass(const PassManagerBuilder &Builder,
@@ -266,6 +272,9 @@
}
void EmitAssemblyHelper::CreatePasses() {
+ if (CodeGenOpts.DisableLLVMPasses)
+ return;
+
unsigned OptLevel = CodeGenOpts.OptimizationLevel;
CodeGenOptions::InliningMethod Inlining = CodeGenOpts.getInlining();
@@ -277,6 +286,29 @@
}
PassManagerBuilderWrapper PMBuilder(CodeGenOpts, LangOpts);
+
+ // Figure out TargetLibraryInfo.
+ Triple TargetTriple(TheModule->getTargetTriple());
+ PMBuilder.LibraryInfo = createTLII(TargetTriple, CodeGenOpts);
+
+ switch (Inlining) {
+ case CodeGenOptions::NoInlining:
+ break;
+ case CodeGenOptions::NormalInlining: {
+ PMBuilder.Inliner =
+ createFunctionInliningPass(OptLevel, CodeGenOpts.OptimizeSize);
+ break;
+ }
+ case CodeGenOptions::OnlyAlwaysInlining:
+ // Respect always_inline.
+ if (OptLevel == 0)
+ // Do not insert lifetime intrinsics at -O0.
+ PMBuilder.Inliner = createAlwaysInlinerPass(false);
+ else
+ PMBuilder.Inliner = createAlwaysInlinerPass();
+ break;
+ }
+
PMBuilder.OptLevel = OptLevel;
PMBuilder.SizeLevel = CodeGenOpts.OptimizeSize;
PMBuilder.BBVectorize = CodeGenOpts.VectorizeBB;
@@ -286,15 +318,23 @@
PMBuilder.DisableUnitAtATime = !CodeGenOpts.UnitAtATime;
PMBuilder.DisableUnrollLoops = !CodeGenOpts.UnrollLoops;
PMBuilder.MergeFunctions = CodeGenOpts.MergeFunctions;
+ PMBuilder.PrepareForLTO = CodeGenOpts.PrepareForLTO;
PMBuilder.RerollLoops = CodeGenOpts.RerollLoops;
+ legacy::PassManager *MPM = getPerModulePasses();
+
+ // If we are performing a ThinLTO importing compile, invoke the LTO
+ // pipeline and pass down the in-memory function index.
+ if (!CodeGenOpts.ThinLTOIndexFile.empty()) {
+ assert(FunctionIndex && "Expected non-empty function index");
+ PMBuilder.FunctionIndex = FunctionIndex.get();
+ PMBuilder.populateLTOPassManager(*MPM);
+ return;
+ }
+
PMBuilder.addExtension(PassManagerBuilder::EP_EarlyAsPossible,
addAddDiscriminatorsPass);
- if (!CodeGenOpts.SampleProfileFile.empty())
- PMBuilder.addExtension(PassManagerBuilder::EP_EarlyAsPossible,
- addSampleProfileLoaderPass);
-
// In ObjC ARC mode, add the main ARC optimization passes.
if (LangOpts.ObjCAutoRefCount) {
PMBuilder.addExtension(PassManagerBuilder::EP_EarlyAsPossible,
@@ -328,6 +368,13 @@
addAddressSanitizerPasses);
}
+ if (LangOpts.Sanitize.has(SanitizerKind::KernelAddress)) {
+ PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast,
+ addKernelAddressSanitizerPasses);
+ PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
+ addKernelAddressSanitizerPasses);
+ }
+
if (LangOpts.Sanitize.has(SanitizerKind::Memory)) {
PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast,
addMemorySanitizerPass);
@@ -349,27 +396,6 @@
addDataFlowSanitizerPass);
}
- // Figure out TargetLibraryInfo.
- Triple TargetTriple(TheModule->getTargetTriple());
- PMBuilder.LibraryInfo = createTLII(TargetTriple, CodeGenOpts);
-
- switch (Inlining) {
- case CodeGenOptions::NoInlining: break;
- case CodeGenOptions::NormalInlining: {
- PMBuilder.Inliner =
- createFunctionInliningPass(OptLevel, CodeGenOpts.OptimizeSize);
- break;
- }
- case CodeGenOptions::OnlyAlwaysInlining:
- // Respect always_inline.
- if (OptLevel == 0)
- // Do not insert lifetime intrinsics at -O0.
- PMBuilder.Inliner = createAlwaysInlinerPass(false);
- else
- PMBuilder.Inliner = createAlwaysInlinerPass();
- break;
- }
-
// Set up the per-function pass manager.
legacy::FunctionPassManager *FPM = getPerFunctionPasses();
if (CodeGenOpts.VerifyModule)
@@ -377,7 +403,6 @@
PMBuilder.populateFunctionPassManager(*FPM);
// Set up the per-module pass manager.
- legacy::PassManager *MPM = getPerModulePasses();
if (!CodeGenOpts.RewriteMapFiles.empty())
addSymbolRewriterPass(CodeGenOpts, MPM);
@@ -406,6 +431,9 @@
MPM->add(createInstrProfilingPass(Options));
}
+ if (!CodeGenOpts.SampleProfileFile.empty())
+ MPM->add(createSampleProfileLoaderPass(CodeGenOpts.SampleProfileFile));
+
PMBuilder.populateModulePassManager(*MPM);
}
@@ -441,22 +469,16 @@
BackendArgs.push_back("-limit-float-precision");
BackendArgs.push_back(CodeGenOpts.LimitFloatPrecision.c_str());
}
- if (llvm::TimePassesIsEnabled)
- BackendArgs.push_back("-time-passes");
- for (unsigned i = 0, e = CodeGenOpts.BackendOptions.size(); i != e; ++i)
- BackendArgs.push_back(CodeGenOpts.BackendOptions[i].c_str());
+ for (const std::string &BackendOption : CodeGenOpts.BackendOptions)
+ BackendArgs.push_back(BackendOption.c_str());
BackendArgs.push_back(nullptr);
llvm::cl::ParseCommandLineOptions(BackendArgs.size() - 1,
BackendArgs.data());
- std::string FeaturesStr;
- if (!TargetOpts.Features.empty()) {
- SubtargetFeatures Features;
- for (const std::string &Feature : TargetOpts.Features)
- Features.AddFeature(Feature);
- FeaturesStr = Features.getString();
- }
+ std::string FeaturesStr =
+ llvm::join(TargetOpts.Features.begin(), TargetOpts.Features.end(), ",");
+ // Keep this synced with the equivalent code in tools/driver/cc1as_main.cpp.
llvm::Reloc::Model RM = llvm::Reloc::Default;
if (CodeGenOpts.RelocationModel == "static") {
RM = llvm::Reloc::Static;
@@ -485,24 +507,16 @@
.Case("posix", llvm::ThreadModel::POSIX)
.Case("single", llvm::ThreadModel::Single);
- if (CodeGenOpts.DisableIntegratedAS)
- Options.DisableIntegratedAS = true;
-
- if (CodeGenOpts.CompressDebugSections)
- Options.CompressDebugSections = true;
-
- if (CodeGenOpts.UseInitArray)
- Options.UseInitArray = true;
-
// Set float ABI type.
- if (CodeGenOpts.FloatABI == "soft" || CodeGenOpts.FloatABI == "softfp")
- Options.FloatABIType = llvm::FloatABI::Soft;
- else if (CodeGenOpts.FloatABI == "hard")
- Options.FloatABIType = llvm::FloatABI::Hard;
- else {
- assert(CodeGenOpts.FloatABI.empty() && "Invalid float abi!");
- Options.FloatABIType = llvm::FloatABI::Default;
- }
+ assert((CodeGenOpts.FloatABI == "soft" || CodeGenOpts.FloatABI == "softfp" ||
+ CodeGenOpts.FloatABI == "hard" || CodeGenOpts.FloatABI.empty()) &&
+ "Invalid Floating Point ABI!");
+ Options.FloatABIType =
+ llvm::StringSwitch<llvm::FloatABI::ABIType>(CodeGenOpts.FloatABI)
+ .Case("soft", llvm::FloatABI::Soft)
+ .Case("softfp", llvm::FloatABI::Soft)
+ .Case("hard", llvm::FloatABI::Hard)
+ .Default(llvm::FloatABI::Default);
// Set FP fusion mode.
switch (CodeGenOpts.getFPContractMode()) {
@@ -517,22 +531,48 @@
break;
}
+ Options.UseInitArray = CodeGenOpts.UseInitArray;
+ Options.DisableIntegratedAS = CodeGenOpts.DisableIntegratedAS;
+ Options.CompressDebugSections = CodeGenOpts.CompressDebugSections;
+
+ // Set EABI version.
+ Options.EABIVersion = llvm::StringSwitch<llvm::EABI>(CodeGenOpts.EABIVersion)
+ .Case("4", llvm::EABI::EABI4)
+ .Case("5", llvm::EABI::EABI5)
+ .Case("gnu", llvm::EABI::GNU)
+ .Default(llvm::EABI::Default);
+
Options.LessPreciseFPMADOption = CodeGenOpts.LessPreciseFPMAD;
Options.NoInfsFPMath = CodeGenOpts.NoInfsFPMath;
Options.NoNaNsFPMath = CodeGenOpts.NoNaNsFPMath;
Options.NoZerosInBSS = CodeGenOpts.NoZeroInitializedInBSS;
Options.UnsafeFPMath = CodeGenOpts.UnsafeFPMath;
Options.StackAlignmentOverride = CodeGenOpts.StackAlignment;
- Options.TrapFuncName = CodeGenOpts.TrapFuncName;
Options.PositionIndependentExecutable = LangOpts.PIELevel != 0;
Options.FunctionSections = CodeGenOpts.FunctionSections;
Options.DataSections = CodeGenOpts.DataSections;
Options.UniqueSectionNames = CodeGenOpts.UniqueSectionNames;
+ Options.EmulatedTLS = CodeGenOpts.EmulatedTLS;
+ switch (CodeGenOpts.getDebuggerTuning()) {
+ case CodeGenOptions::DebuggerKindGDB:
+ Options.DebuggerTuning = llvm::DebuggerKind::GDB;
+ break;
+ case CodeGenOptions::DebuggerKindLLDB:
+ Options.DebuggerTuning = llvm::DebuggerKind::LLDB;
+ break;
+ case CodeGenOptions::DebuggerKindSCE:
+ Options.DebuggerTuning = llvm::DebuggerKind::SCE;
+ break;
+ default:
+ break;
+ }
Options.MCOptions.MCRelaxAll = CodeGenOpts.RelaxAll;
Options.MCOptions.MCSaveTempLabels = CodeGenOpts.SaveTempLabels;
Options.MCOptions.MCUseDwarfDirectory = !CodeGenOpts.NoDwarfDirectoryAsm;
Options.MCOptions.MCNoExecStack = CodeGenOpts.NoExecStack;
+ Options.MCOptions.MCIncrementalLinkerCompatible =
+ CodeGenOpts.IncrementalLinkerCompatible;
Options.MCOptions.MCFatalWarnings = CodeGenOpts.FatalWarnings;
Options.MCOptions.AsmVerbose = CodeGenOpts.AsmVerbose;
Options.MCOptions.ABIName = TargetOpts.ABI;
@@ -591,7 +631,10 @@
if (!TM)
TM.reset(CreateTargetMachine(UsesCodeGen));
- if (UsesCodeGen && !TM) return;
+ if (UsesCodeGen && !TM)
+ return;
+ if (TM)
+ TheModule->setDataLayout(TM->createDataLayout());
CreatePasses();
switch (Action) {
@@ -599,8 +642,8 @@
break;
case Backend_EmitBC:
- getPerModulePasses()->add(
- createBitcodeWriterPass(*OS, CodeGenOpts.EmitLLVMUseLists));
+ getPerModulePasses()->add(createBitcodeWriterPass(
+ *OS, CodeGenOpts.EmitLLVMUseLists, CodeGenOpts.EmitFunctionSummary));
break;
case Backend_EmitLL:
@@ -645,16 +688,17 @@
const clang::TargetOptions &TOpts,
const LangOptions &LOpts, StringRef TDesc,
Module *M, BackendAction Action,
- raw_pwrite_stream *OS) {
- EmitAssemblyHelper AsmHelper(Diags, CGOpts, TOpts, LOpts, M);
+ raw_pwrite_stream *OS,
+ std::unique_ptr<FunctionInfoIndex> Index) {
+ EmitAssemblyHelper AsmHelper(Diags, CGOpts, TOpts, LOpts, M,
+ std::move(Index));
AsmHelper.EmitAssembly(Action, OS);
// If an optional clang TargetInfo description string was passed in, use it to
// verify the LLVM TargetMachine's DataLayout.
if (AsmHelper.TM && !TDesc.empty()) {
- std::string DLDesc =
- AsmHelper.TM->getDataLayout()->getStringRepresentation();
+ std::string DLDesc = M->getDataLayout().getStringRepresentation();
if (DLDesc != TDesc) {
unsigned DiagID = Diags.getCustomDiagID(
DiagnosticsEngine::Error, "backend data layout '%0' does not match "
diff --git a/lib/CodeGen/CGAtomic.cpp b/lib/CodeGen/CGAtomic.cpp
index da82249..24de30b 100644
--- a/lib/CodeGen/CGAtomic.cpp
+++ b/lib/CodeGen/CGAtomic.cpp
@@ -80,7 +80,7 @@
AtomicSizeInBits = C.toBits(
C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
.RoundUpToAlignment(lvalue.getAlignment()));
- auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldAddr());
+ auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
auto OffsetInChars =
(C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
lvalue.getAlignment();
@@ -93,8 +93,10 @@
BFI = OrigBFI;
BFI.Offset = Offset;
BFI.StorageSize = AtomicSizeInBits;
- LVal = LValue::MakeBitfield(Addr, BFI, lvalue.getType(),
- lvalue.getAlignment());
+ BFI.StorageOffset += OffsetInChars;
+ LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()),
+ BFI, lvalue.getType(),
+ lvalue.getAlignmentSource());
LVal.setTBAAInfo(lvalue.getTBAAInfo());
AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
if (AtomicTy.isNull()) {
@@ -117,10 +119,8 @@
ValueTy = lvalue.getType();
ValueSizeInBits = C.getTypeSize(ValueTy);
AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
- lvalue.getType(), lvalue.getExtVectorAddr()
- ->getType()
- ->getPointerElementType()
- ->getVectorNumElements());
+ lvalue.getType(), lvalue.getExtVectorAddress()
+ .getElementType()->getVectorNumElements());
AtomicSizeInBits = C.getTypeSize(AtomicTy);
AtomicAlign = ValueAlign = lvalue.getAlignment();
LVal = lvalue;
@@ -138,15 +138,22 @@
TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
bool shouldUseLibcall() const { return UseLibcall; }
const LValue &getAtomicLValue() const { return LVal; }
- llvm::Value *getAtomicAddress() const {
+ llvm::Value *getAtomicPointer() const {
if (LVal.isSimple())
- return LVal.getAddress();
+ return LVal.getPointer();
else if (LVal.isBitField())
- return LVal.getBitFieldAddr();
+ return LVal.getBitFieldPointer();
else if (LVal.isVectorElt())
- return LVal.getVectorAddr();
+ return LVal.getVectorPointer();
assert(LVal.isExtVectorElt());
- return LVal.getExtVectorAddr();
+ return LVal.getExtVectorPointer();
+ }
+ Address getAtomicAddress() const {
+ return Address(getAtomicPointer(), getAtomicAlignment());
+ }
+
+ Address getAtomicAddressAsAtomicIntPointer() const {
+ return emitCastToAtomicIntPointer(getAtomicAddress());
}
/// Is the atomic size larger than the underlying value type?
@@ -166,13 +173,18 @@
return CGF.CGM.getSize(size);
}
- /// Cast the given pointer to an integer pointer suitable for
- /// atomic operations.
- llvm::Value *emitCastToAtomicIntPointer(llvm::Value *addr) const;
+ /// Cast the given pointer to an integer pointer suitable for atomic
+ /// operations if the source.
+ Address emitCastToAtomicIntPointer(Address Addr) const;
+
+ /// If Addr is compatible with the iN that will be used for an atomic
+ /// operation, bitcast it. Otherwise, create a temporary that is suitable
+ /// and copy the value across.
+ Address convertToAtomicIntPointer(Address Addr) const;
/// Turn an atomic-layout object into an r-value.
- RValue convertTempToRValue(llvm::Value *addr, AggValueSlot resultSlot,
- SourceLocation loc, bool AsValue) const;
+ RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
+ SourceLocation loc, bool AsValue) const;
/// \brief Converts a rvalue to integer value.
llvm::Value *convertRValueToInt(RValue RVal) const;
@@ -187,12 +199,12 @@
/// Project an l-value down to the value field.
LValue projectValue() const {
assert(LVal.isSimple());
- llvm::Value *addr = getAtomicAddress();
+ Address addr = getAtomicAddress();
if (hasPadding())
- addr = CGF.Builder.CreateStructGEP(nullptr, addr, 0);
+ addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits());
- return LValue::MakeAddr(addr, getValueType(), LVal.getAlignment(),
- CGF.getContext(), LVal.getTBAAInfo());
+ return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
+ LVal.getAlignmentSource(), LVal.getTBAAInfo());
}
/// \brief Emits atomic load.
@@ -227,18 +239,18 @@
bool IsVolatile);
/// Materialize an atomic r-value in atomic-layout memory.
- llvm::Value *materializeRValue(RValue rvalue) const;
+ Address materializeRValue(RValue rvalue) const;
/// \brief Translates LLVM atomic ordering to GNU atomic ordering for
/// libcalls.
static AtomicExpr::AtomicOrderingKind
translateAtomicOrdering(const llvm::AtomicOrdering AO);
+ /// \brief Creates temp alloca for intermediate operations on atomic value.
+ Address CreateTempAlloca() const;
private:
bool requiresMemSetZero(llvm::Type *type) const;
- /// \brief Creates temp alloca for intermediate operations on atomic value.
- llvm::Value *CreateTempAlloca() const;
/// \brief Emits atomic load as a libcall.
void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
@@ -293,16 +305,16 @@
llvm_unreachable("Unhandled AtomicOrdering");
}
-llvm::Value *AtomicInfo::CreateTempAlloca() const {
- auto *TempAlloca = CGF.CreateMemTemp(
+Address AtomicInfo::CreateTempAlloca() const {
+ Address TempAlloca = CGF.CreateMemTemp(
(LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
: AtomicTy,
+ getAtomicAlignment(),
"atomic-temp");
- TempAlloca->setAlignment(getAtomicAlignment().getQuantity());
// Cast to pointer to value type for bitfields.
if (LVal.isBitField())
return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- TempAlloca, getAtomicAddress()->getType());
+ TempAlloca, getAtomicAddress().getType());
return TempAlloca;
}
@@ -350,7 +362,7 @@
bool AtomicInfo::emitMemSetZeroIfNecessary() const {
assert(LVal.isSimple());
- llvm::Value *addr = LVal.getAddress();
+ llvm::Value *addr = LVal.getPointer();
if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
return false;
@@ -362,19 +374,17 @@
}
static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
- llvm::Value *Dest, llvm::Value *Ptr,
- llvm::Value *Val1, llvm::Value *Val2,
- uint64_t Size, unsigned Align,
+ Address Dest, Address Ptr,
+ Address Val1, Address Val2,
+ uint64_t Size,
llvm::AtomicOrdering SuccessOrder,
llvm::AtomicOrdering FailureOrder) {
// Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
- llvm::LoadInst *Expected = CGF.Builder.CreateLoad(Val1);
- Expected->setAlignment(Align);
- llvm::LoadInst *Desired = CGF.Builder.CreateLoad(Val2);
- Desired->setAlignment(Align);
+ llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
+ llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
- Ptr, Expected, Desired, SuccessOrder, FailureOrder);
+ Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder);
Pair->setVolatile(E->isVolatile());
Pair->setWeak(IsWeak);
@@ -399,26 +409,24 @@
CGF.Builder.SetInsertPoint(StoreExpectedBB);
// Update the memory at Expected with Old's value.
- llvm::StoreInst *StoreExpected = CGF.Builder.CreateStore(Old, Val1);
- StoreExpected->setAlignment(Align);
+ CGF.Builder.CreateStore(Old, Val1);
// Finally, branch to the exit point.
CGF.Builder.CreateBr(ContinueBB);
CGF.Builder.SetInsertPoint(ContinueBB);
// Update the memory at Dest with Cmp's value.
CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
- return;
}
/// Given an ordering required on success, emit all possible cmpxchg
/// instructions to cope with the provided (but possibly only dynamically known)
/// FailureOrder.
static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
- bool IsWeak, llvm::Value *Dest,
- llvm::Value *Ptr, llvm::Value *Val1,
- llvm::Value *Val2,
+ bool IsWeak, Address Dest,
+ Address Ptr, Address Val1,
+ Address Val2,
llvm::Value *FailureOrderVal,
- uint64_t Size, unsigned Align,
+ uint64_t Size,
llvm::AtomicOrdering SuccessOrder) {
llvm::AtomicOrdering FailureOrder;
if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
@@ -439,7 +447,7 @@
FailureOrder =
llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder);
}
- emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, Align,
+ emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size,
SuccessOrder, FailureOrder);
return;
}
@@ -464,13 +472,13 @@
// doesn't fold to a constant for the ordering.
CGF.Builder.SetInsertPoint(MonotonicBB);
emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
- Size, Align, SuccessOrder, llvm::Monotonic);
+ Size, SuccessOrder, llvm::Monotonic);
CGF.Builder.CreateBr(ContBB);
if (AcquireBB) {
CGF.Builder.SetInsertPoint(AcquireBB);
emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
- Size, Align, SuccessOrder, llvm::Acquire);
+ Size, SuccessOrder, llvm::Acquire);
CGF.Builder.CreateBr(ContBB);
SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume),
AcquireBB);
@@ -480,7 +488,7 @@
if (SeqCstBB) {
CGF.Builder.SetInsertPoint(SeqCstBB);
emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
- Size, Align, SuccessOrder, llvm::SequentiallyConsistent);
+ Size, SuccessOrder, llvm::SequentiallyConsistent);
CGF.Builder.CreateBr(ContBB);
SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst),
SeqCstBB);
@@ -489,11 +497,10 @@
CGF.Builder.SetInsertPoint(ContBB);
}
-static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest,
- llvm::Value *Ptr, llvm::Value *Val1, llvm::Value *Val2,
+static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
+ Address Ptr, Address Val1, Address Val2,
llvm::Value *IsWeak, llvm::Value *FailureOrder,
- uint64_t Size, unsigned Align,
- llvm::AtomicOrdering Order) {
+ uint64_t Size, llvm::AtomicOrdering Order) {
llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
@@ -503,17 +510,17 @@
case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
- FailureOrder, Size, Align, Order);
+ FailureOrder, Size, Order);
return;
case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
- FailureOrder, Size, Align, Order);
+ FailureOrder, Size, Order);
return;
case AtomicExpr::AO__atomic_compare_exchange:
case AtomicExpr::AO__atomic_compare_exchange_n: {
if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
- Val1, Val2, FailureOrder, Size, Align, Order);
+ Val1, Val2, FailureOrder, Size, Order);
} else {
// Create all the relevant BB's
llvm::BasicBlock *StrongBB =
@@ -527,12 +534,12 @@
CGF.Builder.SetInsertPoint(StrongBB);
emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
- FailureOrder, Size, Align, Order);
+ FailureOrder, Size, Order);
CGF.Builder.CreateBr(ContBB);
CGF.Builder.SetInsertPoint(WeakBB);
emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
- FailureOrder, Size, Align, Order);
+ FailureOrder, Size, Order);
CGF.Builder.CreateBr(ContBB);
CGF.Builder.SetInsertPoint(ContBB);
@@ -544,22 +551,17 @@
case AtomicExpr::AO__atomic_load: {
llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
Load->setAtomic(Order);
- Load->setAlignment(Size);
Load->setVolatile(E->isVolatile());
- llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Load, Dest);
- StoreDest->setAlignment(Align);
+ CGF.Builder.CreateStore(Load, Dest);
return;
}
case AtomicExpr::AO__c11_atomic_store:
case AtomicExpr::AO__atomic_store:
case AtomicExpr::AO__atomic_store_n: {
- assert(!Dest && "Store does not return a value");
- llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
- LoadVal1->setAlignment(Align);
+ llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
Store->setAtomic(Order);
- Store->setAlignment(Size);
Store->setVolatile(E->isVolatile());
return;
}
@@ -611,17 +613,16 @@
break;
case AtomicExpr::AO__atomic_nand_fetch:
- PostOp = llvm::Instruction::And;
- // Fall through.
+ PostOp = llvm::Instruction::And; // the NOT is special cased below
+ // Fall through.
case AtomicExpr::AO__atomic_fetch_nand:
Op = llvm::AtomicRMWInst::Nand;
break;
}
- llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
- LoadVal1->setAlignment(Align);
+ llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
llvm::AtomicRMWInst *RMWI =
- CGF.Builder.CreateAtomicRMW(Op, Ptr, LoadVal1, Order);
+ CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order);
RMWI->setVolatile(E->isVolatile());
// For __atomic_*_fetch operations, perform the operation again to
@@ -631,15 +632,14 @@
Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
Result = CGF.Builder.CreateNot(Result);
- llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Result, Dest);
- StoreDest->setAlignment(Align);
+ CGF.Builder.CreateStore(Result, Dest);
}
// This function emits any expression (scalar, complex, or aggregate)
// into a temporary alloca.
-static llvm::Value *
+static Address
EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
- llvm::Value *DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
+ Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
/*Init*/ true);
return DeclPtr;
@@ -651,14 +651,15 @@
SourceLocation Loc, CharUnits SizeInChars) {
if (UseOptimizedLibcall) {
// Load value and pass it to the function directly.
- unsigned Align = CGF.getContext().getTypeAlignInChars(ValTy).getQuantity();
+ CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
ValTy =
CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
SizeInBits)->getPointerTo();
- Val = CGF.EmitLoadOfScalar(CGF.Builder.CreateBitCast(Val, IPtrTy), false,
- Align, CGF.getContext().getPointerType(ValTy),
+ Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align);
+ Val = CGF.EmitLoadOfScalar(Ptr, false,
+ CGF.getContext().getPointerType(ValTy),
Loc);
// Coerce the value into an appropriately sized integer type.
Args.add(RValue::get(Val), ValTy);
@@ -669,27 +670,27 @@
}
}
-RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) {
+RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
QualType MemTy = AtomicTy;
if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
MemTy = AT->getValueType();
- CharUnits sizeChars = getContext().getTypeSizeInChars(AtomicTy);
+ CharUnits sizeChars, alignChars;
+ std::tie(sizeChars, alignChars) = getContext().getTypeInfoInChars(AtomicTy);
uint64_t Size = sizeChars.getQuantity();
- CharUnits alignChars = getContext().getTypeAlignInChars(AtomicTy);
- unsigned Align = alignChars.getQuantity();
- unsigned MaxInlineWidthInBits =
- getTarget().getMaxAtomicInlineWidth();
- bool UseLibcall = (Size != Align ||
+ unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
+ bool UseLibcall = (sizeChars != alignChars ||
getContext().toBits(sizeChars) > MaxInlineWidthInBits);
- llvm::Value *IsWeak = nullptr, *OrderFail = nullptr, *Val1 = nullptr,
- *Val2 = nullptr;
- llvm::Value *Ptr = EmitScalarExpr(E->getPtr());
+ llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
+
+ Address Val1 = Address::invalid();
+ Address Val2 = Address::invalid();
+ Address Dest = Address::invalid();
+ Address Ptr(EmitScalarExpr(E->getPtr()), alignChars);
if (E->getOp() == AtomicExpr::AO__c11_atomic_init) {
- assert(!Dest && "Init does not return a value");
- LValue lvalue = LValue::MakeAddr(Ptr, AtomicTy, alignChars, getContext());
+ LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
EmitAtomicInit(E->getVal1(), lvalue);
return RValue::get(nullptr);
}
@@ -698,32 +699,32 @@
switch (E->getOp()) {
case AtomicExpr::AO__c11_atomic_init:
- llvm_unreachable("Already handled!");
+ llvm_unreachable("Already handled above with EmitAtomicInit!");
case AtomicExpr::AO__c11_atomic_load:
case AtomicExpr::AO__atomic_load_n:
break;
case AtomicExpr::AO__atomic_load:
- Dest = EmitScalarExpr(E->getVal1());
+ Dest = EmitPointerWithAlignment(E->getVal1());
break;
case AtomicExpr::AO__atomic_store:
- Val1 = EmitScalarExpr(E->getVal1());
+ Val1 = EmitPointerWithAlignment(E->getVal1());
break;
case AtomicExpr::AO__atomic_exchange:
- Val1 = EmitScalarExpr(E->getVal1());
- Dest = EmitScalarExpr(E->getVal2());
+ Val1 = EmitPointerWithAlignment(E->getVal1());
+ Dest = EmitPointerWithAlignment(E->getVal2());
break;
case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
case AtomicExpr::AO__atomic_compare_exchange_n:
case AtomicExpr::AO__atomic_compare_exchange:
- Val1 = EmitScalarExpr(E->getVal1());
+ Val1 = EmitPointerWithAlignment(E->getVal1());
if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
- Val2 = EmitScalarExpr(E->getVal2());
+ Val2 = EmitPointerWithAlignment(E->getVal2());
else
Val2 = EmitValToTemp(*this, E->getVal2());
OrderFail = EmitScalarExpr(E->getOrderFail());
@@ -743,8 +744,9 @@
CharUnits PointeeIncAmt =
getContext().getTypeSizeInChars(MemTy->getPointeeType());
Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
- Val1 = CreateMemTemp(Val1Ty, ".atomictmp");
- EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Val1, Val1Ty));
+ auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
+ Val1 = Temp;
+ EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
break;
}
// Fall through.
@@ -773,31 +775,63 @@
QualType RValTy = E->getType().getUnqualifiedType();
- auto GetDest = [&] {
- if (!RValTy->isVoidType() && !Dest) {
- Dest = CreateMemTemp(RValTy, ".atomicdst");
- }
- return Dest;
- };
+ // The inlined atomics only function on iN types, where N is a power of 2. We
+ // need to make sure (via temporaries if necessary) that all incoming values
+ // are compatible.
+ LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
+ AtomicInfo Atomics(*this, AtomicVal);
+
+ Ptr = Atomics.emitCastToAtomicIntPointer(Ptr);
+ if (Val1.isValid()) Val1 = Atomics.convertToAtomicIntPointer(Val1);
+ if (Val2.isValid()) Val2 = Atomics.convertToAtomicIntPointer(Val2);
+ if (Dest.isValid())
+ Dest = Atomics.emitCastToAtomicIntPointer(Dest);
+ else if (E->isCmpXChg())
+ Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
+ else if (!RValTy->isVoidType())
+ Dest = Atomics.emitCastToAtomicIntPointer(Atomics.CreateTempAlloca());
// Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
if (UseLibcall) {
bool UseOptimizedLibcall = false;
switch (E->getOp()) {
+ case AtomicExpr::AO__c11_atomic_init:
+ llvm_unreachable("Already handled above with EmitAtomicInit!");
+
case AtomicExpr::AO__c11_atomic_fetch_add:
case AtomicExpr::AO__atomic_fetch_add:
case AtomicExpr::AO__c11_atomic_fetch_and:
case AtomicExpr::AO__atomic_fetch_and:
case AtomicExpr::AO__c11_atomic_fetch_or:
case AtomicExpr::AO__atomic_fetch_or:
+ case AtomicExpr::AO__atomic_fetch_nand:
case AtomicExpr::AO__c11_atomic_fetch_sub:
case AtomicExpr::AO__atomic_fetch_sub:
case AtomicExpr::AO__c11_atomic_fetch_xor:
case AtomicExpr::AO__atomic_fetch_xor:
+ case AtomicExpr::AO__atomic_add_fetch:
+ case AtomicExpr::AO__atomic_and_fetch:
+ case AtomicExpr::AO__atomic_nand_fetch:
+ case AtomicExpr::AO__atomic_or_fetch:
+ case AtomicExpr::AO__atomic_sub_fetch:
+ case AtomicExpr::AO__atomic_xor_fetch:
// For these, only library calls for certain sizes exist.
UseOptimizedLibcall = true;
break;
- default:
+
+ case AtomicExpr::AO__c11_atomic_load:
+ case AtomicExpr::AO__c11_atomic_store:
+ case AtomicExpr::AO__c11_atomic_exchange:
+ case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
+ case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
+ case AtomicExpr::AO__atomic_load_n:
+ case AtomicExpr::AO__atomic_load:
+ case AtomicExpr::AO__atomic_store_n:
+ case AtomicExpr::AO__atomic_store:
+ case AtomicExpr::AO__atomic_exchange_n:
+ case AtomicExpr::AO__atomic_exchange:
+ case AtomicExpr::AO__atomic_compare_exchange_n:
+ case AtomicExpr::AO__atomic_compare_exchange:
// Only use optimized library calls for sizes for which they exist.
if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
UseOptimizedLibcall = true;
@@ -811,14 +845,19 @@
getContext().getSizeType());
}
// Atomic address is the first or second parameter
- Args.add(RValue::get(EmitCastToVoidPtr(Ptr)), getContext().VoidPtrTy);
+ Args.add(RValue::get(EmitCastToVoidPtr(Ptr.getPointer())),
+ getContext().VoidPtrTy);
std::string LibCallName;
QualType LoweredMemTy =
MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
QualType RetTy;
bool HaveRetTy = false;
+ llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
switch (E->getOp()) {
+ case AtomicExpr::AO__c11_atomic_init:
+ llvm_unreachable("Already handled!");
+
// There is only one libcall for compare an exchange, because there is no
// optimisation benefit possible from a libcall version of a weak compare
// and exchange.
@@ -833,9 +872,10 @@
LibCallName = "__atomic_compare_exchange";
RetTy = getContext().BoolTy;
HaveRetTy = true;
- Args.add(RValue::get(EmitCastToVoidPtr(Val1)), getContext().VoidPtrTy);
- AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2, MemTy,
- E->getExprLoc(), sizeChars);
+ Args.add(RValue::get(EmitCastToVoidPtr(Val1.getPointer())),
+ getContext().VoidPtrTy);
+ AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
+ MemTy, E->getExprLoc(), sizeChars);
Args.add(RValue::get(Order), getContext().IntTy);
Order = OrderFail;
break;
@@ -846,8 +886,8 @@
case AtomicExpr::AO__atomic_exchange_n:
case AtomicExpr::AO__atomic_exchange:
LibCallName = "__atomic_exchange";
- AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
- E->getExprLoc(), sizeChars);
+ AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
+ MemTy, E->getExprLoc(), sizeChars);
break;
// void __atomic_store(size_t size, void *mem, void *val, int order)
// void __atomic_store_N(T *mem, T val, int order)
@@ -857,8 +897,8 @@
LibCallName = "__atomic_store";
RetTy = getContext().VoidTy;
HaveRetTy = true;
- AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
- E->getExprLoc(), sizeChars);
+ AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
+ MemTy, E->getExprLoc(), sizeChars);
break;
// void __atomic_load(size_t size, void *mem, void *return, int order)
// T __atomic_load_N(T *mem, int order)
@@ -867,42 +907,71 @@
case AtomicExpr::AO__atomic_load_n:
LibCallName = "__atomic_load";
break;
+ // T __atomic_add_fetch_N(T *mem, T val, int order)
// T __atomic_fetch_add_N(T *mem, T val, int order)
+ case AtomicExpr::AO__atomic_add_fetch:
+ PostOp = llvm::Instruction::Add;
+ // Fall through.
case AtomicExpr::AO__c11_atomic_fetch_add:
case AtomicExpr::AO__atomic_fetch_add:
LibCallName = "__atomic_fetch_add";
- AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy,
- E->getExprLoc(), sizeChars);
+ AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
+ LoweredMemTy, E->getExprLoc(), sizeChars);
break;
+ // T __atomic_and_fetch_N(T *mem, T val, int order)
// T __atomic_fetch_and_N(T *mem, T val, int order)
+ case AtomicExpr::AO__atomic_and_fetch:
+ PostOp = llvm::Instruction::And;
+ // Fall through.
case AtomicExpr::AO__c11_atomic_fetch_and:
case AtomicExpr::AO__atomic_fetch_and:
LibCallName = "__atomic_fetch_and";
- AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
- E->getExprLoc(), sizeChars);
+ AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
+ MemTy, E->getExprLoc(), sizeChars);
break;
+ // T __atomic_or_fetch_N(T *mem, T val, int order)
// T __atomic_fetch_or_N(T *mem, T val, int order)
+ case AtomicExpr::AO__atomic_or_fetch:
+ PostOp = llvm::Instruction::Or;
+ // Fall through.
case AtomicExpr::AO__c11_atomic_fetch_or:
case AtomicExpr::AO__atomic_fetch_or:
LibCallName = "__atomic_fetch_or";
- AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
- E->getExprLoc(), sizeChars);
+ AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
+ MemTy, E->getExprLoc(), sizeChars);
break;
+ // T __atomic_sub_fetch_N(T *mem, T val, int order)
// T __atomic_fetch_sub_N(T *mem, T val, int order)
+ case AtomicExpr::AO__atomic_sub_fetch:
+ PostOp = llvm::Instruction::Sub;
+ // Fall through.
case AtomicExpr::AO__c11_atomic_fetch_sub:
case AtomicExpr::AO__atomic_fetch_sub:
LibCallName = "__atomic_fetch_sub";
- AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy,
- E->getExprLoc(), sizeChars);
+ AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
+ LoweredMemTy, E->getExprLoc(), sizeChars);
break;
+ // T __atomic_xor_fetch_N(T *mem, T val, int order)
// T __atomic_fetch_xor_N(T *mem, T val, int order)
+ case AtomicExpr::AO__atomic_xor_fetch:
+ PostOp = llvm::Instruction::Xor;
+ // Fall through.
case AtomicExpr::AO__c11_atomic_fetch_xor:
case AtomicExpr::AO__atomic_fetch_xor:
LibCallName = "__atomic_fetch_xor";
- AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
- E->getExprLoc(), sizeChars);
+ AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
+ MemTy, E->getExprLoc(), sizeChars);
break;
- default: return EmitUnsupportedRValue(E, "atomic library call");
+ // T __atomic_nand_fetch_N(T *mem, T val, int order)
+ // T __atomic_fetch_nand_N(T *mem, T val, int order)
+ case AtomicExpr::AO__atomic_nand_fetch:
+ PostOp = llvm::Instruction::And; // the NOT is special cased below
+ // Fall through.
+ case AtomicExpr::AO__atomic_fetch_nand:
+ LibCallName = "__atomic_fetch_nand";
+ AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
+ MemTy, E->getExprLoc(), sizeChars);
+ break;
}
// Optimized functions have the size in their name.
@@ -918,30 +987,46 @@
} else {
// Value is returned through parameter before the order.
RetTy = getContext().VoidTy;
- Args.add(RValue::get(EmitCastToVoidPtr(Dest)), getContext().VoidPtrTy);
+ Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())),
+ getContext().VoidPtrTy);
}
}
// order is always the last parameter
Args.add(RValue::get(Order),
getContext().IntTy);
+ // PostOp is only needed for the atomic_*_fetch operations, and
+ // thus is only needed for and implemented in the
+ // UseOptimizedLibcall codepath.
+ assert(UseOptimizedLibcall || !PostOp);
+
RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
// The value is returned directly from the libcall.
- if (HaveRetTy && !RetTy->isVoidType())
+ if (E->isCmpXChg())
return Res;
- // The value is returned via an explicit out param.
- if (RetTy->isVoidType())
- return RValue::get(nullptr);
- // The value is returned directly for optimized libcalls but the caller is
- // expected an out-param.
- if (UseOptimizedLibcall) {
+
+ // The value is returned directly for optimized libcalls but the expr
+ // provided an out-param.
+ if (UseOptimizedLibcall && Res.getScalarVal()) {
llvm::Value *ResVal = Res.getScalarVal();
- llvm::StoreInst *StoreDest = Builder.CreateStore(
+ if (PostOp) {
+ llvm::Value *LoadVal1 = Args[1].RV.getScalarVal();
+ ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
+ }
+ if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
+ ResVal = Builder.CreateNot(ResVal);
+
+ Builder.CreateStore(
ResVal,
- Builder.CreateBitCast(GetDest(), ResVal->getType()->getPointerTo()));
- StoreDest->setAlignment(Align);
+ Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo()));
}
- return convertTempToRValue(Dest, RValTy, E->getExprLoc());
+
+ if (RValTy->isVoidType())
+ return RValue::get(nullptr);
+
+ return convertTempToRValue(
+ Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
+ RValTy, E->getExprLoc());
}
bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
@@ -951,45 +1036,35 @@
E->getOp() == AtomicExpr::AO__atomic_load ||
E->getOp() == AtomicExpr::AO__atomic_load_n;
- llvm::Type *ITy =
- llvm::IntegerType::get(getLLVMContext(), Size * 8);
- llvm::Value *OrigDest = GetDest();
- Ptr = Builder.CreateBitCast(
- Ptr, ITy->getPointerTo(Ptr->getType()->getPointerAddressSpace()));
- if (Val1) Val1 = Builder.CreateBitCast(Val1, ITy->getPointerTo());
- if (Val2) Val2 = Builder.CreateBitCast(Val2, ITy->getPointerTo());
- if (Dest && !E->isCmpXChg())
- Dest = Builder.CreateBitCast(Dest, ITy->getPointerTo());
-
if (isa<llvm::ConstantInt>(Order)) {
int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
switch (ord) {
case AtomicExpr::AO_ABI_memory_order_relaxed:
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
- Size, Align, llvm::Monotonic);
+ Size, llvm::Monotonic);
break;
case AtomicExpr::AO_ABI_memory_order_consume:
case AtomicExpr::AO_ABI_memory_order_acquire:
if (IsStore)
break; // Avoid crashing on code with undefined behavior
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
- Size, Align, llvm::Acquire);
+ Size, llvm::Acquire);
break;
case AtomicExpr::AO_ABI_memory_order_release:
if (IsLoad)
break; // Avoid crashing on code with undefined behavior
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
- Size, Align, llvm::Release);
+ Size, llvm::Release);
break;
case AtomicExpr::AO_ABI_memory_order_acq_rel:
if (IsLoad || IsStore)
break; // Avoid crashing on code with undefined behavior
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
- Size, Align, llvm::AcquireRelease);
+ Size, llvm::AcquireRelease);
break;
case AtomicExpr::AO_ABI_memory_order_seq_cst:
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
- Size, Align, llvm::SequentiallyConsistent);
+ Size, llvm::SequentiallyConsistent);
break;
default: // invalid order
// We should not ever get here normally, but it's hard to
@@ -998,7 +1073,10 @@
}
if (RValTy->isVoidType())
return RValue::get(nullptr);
- return convertTempToRValue(OrigDest, RValTy, E->getExprLoc());
+
+ return convertTempToRValue(
+ Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
+ RValTy, E->getExprLoc());
}
// Long case, when Order isn't obviously constant.
@@ -1027,12 +1105,12 @@
// Emit all the different atomics
Builder.SetInsertPoint(MonotonicBB);
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
- Size, Align, llvm::Monotonic);
+ Size, llvm::Monotonic);
Builder.CreateBr(ContBB);
if (!IsStore) {
Builder.SetInsertPoint(AcquireBB);
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
- Size, Align, llvm::Acquire);
+ Size, llvm::Acquire);
Builder.CreateBr(ContBB);
SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume),
AcquireBB);
@@ -1042,7 +1120,7 @@
if (!IsLoad) {
Builder.SetInsertPoint(ReleaseBB);
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
- Size, Align, llvm::Release);
+ Size, llvm::Release);
Builder.CreateBr(ContBB);
SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_release),
ReleaseBB);
@@ -1050,14 +1128,14 @@
if (!IsLoad && !IsStore) {
Builder.SetInsertPoint(AcqRelBB);
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
- Size, Align, llvm::AcquireRelease);
+ Size, llvm::AcquireRelease);
Builder.CreateBr(ContBB);
SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acq_rel),
AcqRelBB);
}
Builder.SetInsertPoint(SeqCstBB);
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
- Size, Align, llvm::SequentiallyConsistent);
+ Size, llvm::SequentiallyConsistent);
Builder.CreateBr(ContBB);
SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst),
SeqCstBB);
@@ -1066,47 +1144,65 @@
Builder.SetInsertPoint(ContBB);
if (RValTy->isVoidType())
return RValue::get(nullptr);
- return convertTempToRValue(OrigDest, RValTy, E->getExprLoc());
+
+ assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
+ return convertTempToRValue(
+ Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
+ RValTy, E->getExprLoc());
}
-llvm::Value *AtomicInfo::emitCastToAtomicIntPointer(llvm::Value *addr) const {
+Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const {
unsigned addrspace =
- cast<llvm::PointerType>(addr->getType())->getAddressSpace();
+ cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace();
llvm::IntegerType *ty =
llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
}
-RValue AtomicInfo::convertTempToRValue(llvm::Value *addr,
- AggValueSlot resultSlot,
- SourceLocation loc, bool AsValue) const {
+Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
+ llvm::Type *Ty = Addr.getElementType();
+ uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
+ if (SourceSizeInBits != AtomicSizeInBits) {
+ Address Tmp = CreateTempAlloca();
+ CGF.Builder.CreateMemCpy(Tmp, Addr,
+ std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
+ Addr = Tmp;
+ }
+
+ return emitCastToAtomicIntPointer(Addr);
+}
+
+RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
+ AggValueSlot resultSlot,
+ SourceLocation loc,
+ bool asValue) const {
if (LVal.isSimple()) {
if (EvaluationKind == TEK_Aggregate)
return resultSlot.asRValue();
// Drill into the padding structure if we have one.
if (hasPadding())
- addr = CGF.Builder.CreateStructGEP(nullptr, addr, 0);
+ addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits());
// Otherwise, just convert the temporary to an r-value using the
// normal conversion routine.
return CGF.convertTempToRValue(addr, getValueType(), loc);
}
- if (!AsValue)
+ if (!asValue)
// Get RValue from temp memory as atomic for non-simple lvalues
- return RValue::get(
- CGF.Builder.CreateAlignedLoad(addr, AtomicAlign.getQuantity()));
+ return RValue::get(CGF.Builder.CreateLoad(addr));
if (LVal.isBitField())
- return CGF.EmitLoadOfBitfieldLValue(LValue::MakeBitfield(
- addr, LVal.getBitFieldInfo(), LVal.getType(), LVal.getAlignment()));
+ return CGF.EmitLoadOfBitfieldLValue(
+ LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
+ LVal.getAlignmentSource()));
if (LVal.isVectorElt())
- return CGF.EmitLoadOfLValue(LValue::MakeVectorElt(addr, LVal.getVectorIdx(),
- LVal.getType(),
- LVal.getAlignment()),
- loc);
+ return CGF.EmitLoadOfLValue(
+ LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
+ LVal.getAlignmentSource()), loc);
assert(LVal.isExtVectorElt());
return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
- addr, LVal.getExtVectorElts(), LVal.getType(), LVal.getAlignment()));
+ addr, LVal.getExtVectorElts(), LVal.getType(),
+ LVal.getAlignmentSource()));
}
RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
@@ -1122,7 +1218,7 @@
!AsValue)) {
auto *ValTy = AsValue
? CGF.ConvertTypeForMem(ValueTy)
- : getAtomicAddress()->getType()->getPointerElementType();
+ : getAtomicAddress().getType()->getPointerElementType();
if (ValTy->isIntegerTy()) {
assert(IntVal->getType() == ValTy && "Different integer types.");
return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
@@ -1134,25 +1230,22 @@
// Create a temporary. This needs to be big enough to hold the
// atomic integer.
- llvm::Value *Temp;
+ Address Temp = Address::invalid();
bool TempIsVolatile = false;
- CharUnits TempAlignment;
if (AsValue && getEvaluationKind() == TEK_Aggregate) {
assert(!ResultSlot.isIgnored());
- Temp = ResultSlot.getAddr();
- TempAlignment = getValueAlignment();
+ Temp = ResultSlot.getAddress();
TempIsVolatile = ResultSlot.isVolatile();
} else {
Temp = CreateTempAlloca();
- TempAlignment = getAtomicAlignment();
}
// Slam the integer into the temporary.
- llvm::Value *CastTemp = emitCastToAtomicIntPointer(Temp);
- CGF.Builder.CreateAlignedStore(IntVal, CastTemp, TempAlignment.getQuantity())
+ Address CastTemp = emitCastToAtomicIntPointer(Temp);
+ CGF.Builder.CreateStore(IntVal, CastTemp)
->setVolatile(TempIsVolatile);
- return convertTempToRValue(Temp, ResultSlot, Loc, AsValue);
+ return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
}
void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
@@ -1160,7 +1253,7 @@
// void __atomic_load(size_t size, void *mem, void *return, int order);
CallArgList Args;
Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
- Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicAddress())),
+ Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
CGF.getContext().VoidPtrTy);
Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
CGF.getContext().VoidPtrTy);
@@ -1173,16 +1266,15 @@
llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
bool IsVolatile) {
// Okay, we're doing this natively.
- llvm::Value *Addr = emitCastToAtomicIntPointer(getAtomicAddress());
+ Address Addr = getAtomicAddressAsAtomicIntPointer();
llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
Load->setAtomic(AO);
// Other decoration.
- Load->setAlignment(getAtomicAlignment().getQuantity());
if (IsVolatile)
Load->setVolatile(true);
if (LVal.getTBAAInfo())
- CGF.CGM.DecorateInstruction(Load, LVal.getTBAAInfo());
+ CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
return Load;
}
@@ -1190,11 +1282,12 @@
/// we are operating under /volatile:ms *and* the LValue itself is volatile and
/// performing such an operation can be performed without a libcall.
bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
+ if (!CGM.getCodeGenOpts().MSVolatile) return false;
AtomicInfo AI(*this, LV);
bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
// An atomic is inline if we don't need to use a libcall.
bool AtomicIsInline = !AI.shouldUseLibcall();
- return CGM.getCodeGenOpts().MSVolatile && IsVolatile && AtomicIsInline;
+ return IsVolatile && AtomicIsInline;
}
/// An type is a candidate for having its loads and stores be made atomic if
@@ -1226,18 +1319,18 @@
bool IsVolatile) {
// Check whether we should use a library call.
if (shouldUseLibcall()) {
- llvm::Value *TempAddr;
+ Address TempAddr = Address::invalid();
if (LVal.isSimple() && !ResultSlot.isIgnored()) {
assert(getEvaluationKind() == TEK_Aggregate);
- TempAddr = ResultSlot.getAddr();
+ TempAddr = ResultSlot.getAddress();
} else
TempAddr = CreateTempAlloca();
- EmitAtomicLoadLibcall(TempAddr, AO, IsVolatile);
+ EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
// Okay, turn that back into the original value or whole atomic (for
// non-simple lvalues) type.
- return convertTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
+ return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
}
// Okay, we're doing this natively.
@@ -1245,7 +1338,7 @@
// If we're ignoring an aggregate return, don't do anything.
if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
- return RValue::getAggregate(nullptr, false);
+ return RValue::getAggregate(Address::invalid(), false);
// Okay, turn that back into the original value or atomic (for non-simple
// lvalues) type.
@@ -1271,11 +1364,10 @@
// any padding. Just do an aggregate copy of that type.
if (rvalue.isAggregate()) {
CGF.EmitAggregateCopy(getAtomicAddress(),
- rvalue.getAggregateAddr(),
+ rvalue.getAggregateAddress(),
getAtomicType(),
(rvalue.isVolatileQualified()
- || LVal.isVolatileQualified()),
- LVal.getAlignment());
+ || LVal.isVolatileQualified()));
return;
}
@@ -1298,15 +1390,14 @@
/// Materialize an r-value into memory for the purposes of storing it
/// to an atomic type.
-llvm::Value *AtomicInfo::materializeRValue(RValue rvalue) const {
+Address AtomicInfo::materializeRValue(RValue rvalue) const {
// Aggregate r-values are already in memory, and EmitAtomicStore
// requires them to be values of the atomic type.
if (rvalue.isAggregate())
- return rvalue.getAggregateAddr();
+ return rvalue.getAggregateAddress();
// Otherwise, make a temporary and materialize into it.
- LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType(),
- getAtomicAlignment());
+ LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
AtomicInfo Atomics(CGF, TempLV);
Atomics.emitCopyIntoMemory(rvalue);
return TempLV.getAddress();
@@ -1331,20 +1422,20 @@
}
// Otherwise, we need to go through memory.
// Put the r-value in memory.
- llvm::Value *Addr = materializeRValue(RVal);
+ Address Addr = materializeRValue(RVal);
// Cast the temporary to the atomic int type and pull a value out.
Addr = emitCastToAtomicIntPointer(Addr);
- return CGF.Builder.CreateAlignedLoad(Addr,
- getAtomicAlignment().getQuantity());
+ return CGF.Builder.CreateLoad(Addr);
}
std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
// Do the atomic store.
- auto *Addr = emitCastToAtomicIntPointer(getAtomicAddress());
- auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr, ExpectedVal, DesiredVal,
+ Address Addr = getAtomicAddressAsAtomicIntPointer();
+ auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
+ ExpectedVal, DesiredVal,
Success, Failure);
// Other decoration.
Inst->setVolatile(LVal.isVolatileQualified());
@@ -1365,7 +1456,7 @@
// void *desired, int success, int failure);
CallArgList Args;
Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
- Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicAddress())),
+ Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
CGF.getContext().VoidPtrTy);
Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
CGF.getContext().VoidPtrTy);
@@ -1393,13 +1484,14 @@
// Check whether we should use a library call.
if (shouldUseLibcall()) {
// Produce a source address.
- auto *ExpectedAddr = materializeRValue(Expected);
- auto *DesiredAddr = materializeRValue(Desired);
- auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr,
+ Address ExpectedAddr = materializeRValue(Expected);
+ Address DesiredAddr = materializeRValue(Desired);
+ auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
+ DesiredAddr.getPointer(),
Success, Failure);
return std::make_pair(
- convertTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
- SourceLocation(), /*AsValue=*/false),
+ convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
+ SourceLocation(), /*AsValue=*/false),
Res);
}
@@ -1418,42 +1510,41 @@
static void
EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
const llvm::function_ref<RValue(RValue)> &UpdateOp,
- llvm::Value *DesiredAddr) {
- llvm::Value *Ptr = nullptr;
- LValue UpdateLVal;
+ Address DesiredAddr) {
RValue UpRVal;
LValue AtomicLVal = Atomics.getAtomicLValue();
LValue DesiredLVal;
if (AtomicLVal.isSimple()) {
UpRVal = OldRVal;
- DesiredLVal =
- LValue::MakeAddr(DesiredAddr, AtomicLVal.getType(),
- AtomicLVal.getAlignment(), CGF.CGM.getContext());
+ DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
} else {
// Build new lvalue for temp address
- Ptr = Atomics.materializeRValue(OldRVal);
+ Address Ptr = Atomics.materializeRValue(OldRVal);
+ LValue UpdateLVal;
if (AtomicLVal.isBitField()) {
UpdateLVal =
LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
- AtomicLVal.getType(), AtomicLVal.getAlignment());
+ AtomicLVal.getType(),
+ AtomicLVal.getAlignmentSource());
DesiredLVal =
LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
- AtomicLVal.getType(), AtomicLVal.getAlignment());
+ AtomicLVal.getType(),
+ AtomicLVal.getAlignmentSource());
} else if (AtomicLVal.isVectorElt()) {
UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
AtomicLVal.getType(),
- AtomicLVal.getAlignment());
+ AtomicLVal.getAlignmentSource());
DesiredLVal = LValue::MakeVectorElt(
DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
- AtomicLVal.getAlignment());
+ AtomicLVal.getAlignmentSource());
} else {
assert(AtomicLVal.isExtVectorElt());
UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
AtomicLVal.getType(),
- AtomicLVal.getAlignment());
+ AtomicLVal.getAlignmentSource());
DesiredLVal = LValue::MakeExtVectorElt(
DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
- AtomicLVal.getAlignment());
+ AtomicLVal.getAlignmentSource());
}
UpdateLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
@@ -1475,26 +1566,26 @@
bool IsVolatile) {
auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
- llvm::Value *ExpectedAddr = CreateTempAlloca();
+ Address ExpectedAddr = CreateTempAlloca();
- EmitAtomicLoadLibcall(ExpectedAddr, AO, IsVolatile);
+ EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
auto *ContBB = CGF.createBasicBlock("atomic_cont");
auto *ExitBB = CGF.createBasicBlock("atomic_exit");
CGF.EmitBlock(ContBB);
- auto *DesiredAddr = CreateTempAlloca();
+ Address DesiredAddr = CreateTempAlloca();
if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
- requiresMemSetZero(
- getAtomicAddress()->getType()->getPointerElementType())) {
- auto *OldVal = CGF.Builder.CreateAlignedLoad(
- ExpectedAddr, getAtomicAlignment().getQuantity());
- CGF.Builder.CreateAlignedStore(OldVal, DesiredAddr,
- getAtomicAlignment().getQuantity());
+ requiresMemSetZero(getAtomicAddress().getElementType())) {
+ auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
+ CGF.Builder.CreateStore(OldVal, DesiredAddr);
}
- auto OldRVal = convertTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
- SourceLocation(), /*AsValue=*/false);
+ auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
+ AggValueSlot::ignored(),
+ SourceLocation(), /*AsValue=*/false);
EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
auto *Res =
- EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr, AO, Failure);
+ EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
+ DesiredAddr.getPointer(),
+ AO, Failure);
CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
}
@@ -1514,19 +1605,16 @@
llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
/*NumReservedValues=*/2);
PHI->addIncoming(OldVal, CurBB);
- auto *NewAtomicAddr = CreateTempAlloca();
- auto *NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
+ Address NewAtomicAddr = CreateTempAlloca();
+ Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
- requiresMemSetZero(
- getAtomicAddress()->getType()->getPointerElementType())) {
- CGF.Builder.CreateAlignedStore(PHI, NewAtomicIntAddr,
- getAtomicAlignment().getQuantity());
+ requiresMemSetZero(getAtomicAddress().getElementType())) {
+ CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
}
auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
SourceLocation(), /*AsValue=*/false);
EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
- auto *DesiredVal = CGF.Builder.CreateAlignedLoad(
- NewAtomicIntAddr, getAtomicAlignment().getQuantity());
+ auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
// Try to write new value using cmpxchg operation
auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
@@ -1535,23 +1623,25 @@
}
static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
- RValue UpdateRVal, llvm::Value *DesiredAddr) {
+ RValue UpdateRVal, Address DesiredAddr) {
LValue AtomicLVal = Atomics.getAtomicLValue();
LValue DesiredLVal;
// Build new lvalue for temp address
if (AtomicLVal.isBitField()) {
DesiredLVal =
LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
- AtomicLVal.getType(), AtomicLVal.getAlignment());
+ AtomicLVal.getType(),
+ AtomicLVal.getAlignmentSource());
} else if (AtomicLVal.isVectorElt()) {
DesiredLVal =
LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
- AtomicLVal.getType(), AtomicLVal.getAlignment());
+ AtomicLVal.getType(),
+ AtomicLVal.getAlignmentSource());
} else {
assert(AtomicLVal.isExtVectorElt());
DesiredLVal = LValue::MakeExtVectorElt(
DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
- AtomicLVal.getAlignment());
+ AtomicLVal.getAlignmentSource());
}
DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
// Store new value in the corresponding memory area
@@ -1563,24 +1653,23 @@
RValue UpdateRVal, bool IsVolatile) {
auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
- llvm::Value *ExpectedAddr = CreateTempAlloca();
+ Address ExpectedAddr = CreateTempAlloca();
- EmitAtomicLoadLibcall(ExpectedAddr, AO, IsVolatile);
+ EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
auto *ContBB = CGF.createBasicBlock("atomic_cont");
auto *ExitBB = CGF.createBasicBlock("atomic_exit");
CGF.EmitBlock(ContBB);
- auto *DesiredAddr = CreateTempAlloca();
+ Address DesiredAddr = CreateTempAlloca();
if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
- requiresMemSetZero(
- getAtomicAddress()->getType()->getPointerElementType())) {
- auto *OldVal = CGF.Builder.CreateAlignedLoad(
- ExpectedAddr, getAtomicAlignment().getQuantity());
- CGF.Builder.CreateAlignedStore(OldVal, DesiredAddr,
- getAtomicAlignment().getQuantity());
+ requiresMemSetZero(getAtomicAddress().getElementType())) {
+ auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
+ CGF.Builder.CreateStore(OldVal, DesiredAddr);
}
EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
auto *Res =
- EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr, AO, Failure);
+ EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
+ DesiredAddr.getPointer(),
+ AO, Failure);
CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
}
@@ -1599,17 +1688,14 @@
llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
/*NumReservedValues=*/2);
PHI->addIncoming(OldVal, CurBB);
- auto *NewAtomicAddr = CreateTempAlloca();
- auto *NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
+ Address NewAtomicAddr = CreateTempAlloca();
+ Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
- requiresMemSetZero(
- getAtomicAddress()->getType()->getPointerElementType())) {
- CGF.Builder.CreateAlignedStore(PHI, NewAtomicIntAddr,
- getAtomicAlignment().getQuantity());
+ requiresMemSetZero(getAtomicAddress().getElementType())) {
+ CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
}
EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
- auto *DesiredVal = CGF.Builder.CreateAlignedLoad(
- NewAtomicIntAddr, getAtomicAlignment().getQuantity());
+ auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
// Try to write new value using cmpxchg operation
auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
@@ -1660,8 +1746,8 @@
// If this is an aggregate r-value, it should agree in type except
// maybe for address-space qualification.
assert(!rvalue.isAggregate() ||
- rvalue.getAggregateAddr()->getType()->getPointerElementType()
- == dest.getAddress()->getType()->getPointerElementType());
+ rvalue.getAggregateAddress().getElementType()
+ == dest.getAddress().getElementType());
AtomicInfo atomics(*this, dest);
LValue LVal = atomics.getAtomicLValue();
@@ -1676,15 +1762,16 @@
// Check whether we should use a library call.
if (atomics.shouldUseLibcall()) {
// Produce a source address.
- llvm::Value *srcAddr = atomics.materializeRValue(rvalue);
+ Address srcAddr = atomics.materializeRValue(rvalue);
// void __atomic_store(size_t size, void *mem, void *val, int order)
CallArgList args;
args.add(RValue::get(atomics.getAtomicSizeValue()),
getContext().getSizeType());
- args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicAddress())),
+ args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())),
getContext().VoidPtrTy);
- args.add(RValue::get(EmitCastToVoidPtr(srcAddr)), getContext().VoidPtrTy);
+ args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())),
+ getContext().VoidPtrTy);
args.add(RValue::get(llvm::ConstantInt::get(
IntTy, AtomicInfo::translateAtomicOrdering(AO))),
getContext().IntTy);
@@ -1696,10 +1783,10 @@
llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
// Do the atomic store.
- llvm::Value *addr =
+ Address addr =
atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
intValue = Builder.CreateIntCast(
- intValue, addr->getType()->getPointerElementType(), /*isSigned=*/false);
+ intValue, addr.getElementType(), /*isSigned=*/false);
llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
// Initializations don't need to be atomic.
@@ -1707,11 +1794,10 @@
store->setAtomic(AO);
// Other decoration.
- store->setAlignment(dest.getAlignment().getQuantity());
if (IsVolatile)
store->setVolatile(true);
if (dest.getTBAAInfo())
- CGM.DecorateInstruction(store, dest.getTBAAInfo());
+ CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());
return;
}
@@ -1728,11 +1814,11 @@
// If this is an aggregate r-value, it should agree in type except
// maybe for address-space qualification.
assert(!Expected.isAggregate() ||
- Expected.getAggregateAddr()->getType()->getPointerElementType() ==
- Obj.getAddress()->getType()->getPointerElementType());
+ Expected.getAggregateAddress().getElementType() ==
+ Obj.getAddress().getElementType());
assert(!Desired.isAggregate() ||
- Desired.getAggregateAddr()->getType()->getPointerElementType() ==
- Obj.getAddress()->getType()->getPointerElementType());
+ Desired.getAggregateAddress().getElementType() ==
+ Obj.getAddress().getElementType());
AtomicInfo Atomics(*this, Obj);
return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
diff --git a/lib/CodeGen/CGBlocks.cpp b/lib/CodeGen/CGBlocks.cpp
index 3fd344c..ba2941e 100644
--- a/lib/CodeGen/CGBlocks.cpp
+++ b/lib/CodeGen/CGBlocks.cpp
@@ -1,4 +1,4 @@
-//===--- CGBlocks.cpp - Emit LLVM Code for declarations -------------------===//
+//===--- CGBlocks.cpp - Emit LLVM Code for declarations ---------*- C++ -*-===//
//
// The LLVM Compiler Infrastructure
//
@@ -30,7 +30,7 @@
CGBlockInfo::CGBlockInfo(const BlockDecl *block, StringRef name)
: Name(name), CXXThisIndex(0), CanBeGlobal(false), NeedsCopyDispose(false),
HasCXXObject(false), UsesStret(false), HasCapturedVariableLayout(false),
- StructureType(nullptr), Block(block),
+ LocalAddress(Address::invalid()), StructureType(nullptr), Block(block),
DominatingIP(nullptr) {
// Skip asm prefix, if any. 'name' is usually taken directly from
@@ -40,7 +40,7 @@
}
// Anchor the vtable to this translation unit.
-CodeGenModule::ByrefHelpers::~ByrefHelpers() {}
+BlockByrefHelpers::~BlockByrefHelpers() {}
/// Build the given block as a global block.
static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM,
@@ -78,7 +78,7 @@
ASTContext &C = CGM.getContext();
llvm::Type *ulong = CGM.getTypes().ConvertType(C.UnsignedLongTy);
- llvm::Type *i8p = NULL;
+ llvm::Type *i8p = nullptr;
if (CGM.getLangOpts().OpenCL)
i8p =
llvm::Type::getInt8PtrTy(
@@ -111,7 +111,7 @@
std::string typeAtEncoding =
CGM.getContext().getObjCEncodingForBlock(blockInfo.getBlockExpr());
elements.push_back(llvm::ConstantExpr::getBitCast(
- CGM.GetAddrOfConstantCString(typeAtEncoding), i8p));
+ CGM.GetAddrOfConstantCString(typeAtEncoding).getPointer(), i8p));
// GC layout.
if (C.getLangOpts().ObjC1) {
@@ -203,46 +203,36 @@
Capture(capture), Type(type) {}
/// Tell the block info that this chunk has the given field index.
- void setIndex(CGBlockInfo &info, unsigned index) {
- if (!Capture)
+ void setIndex(CGBlockInfo &info, unsigned index, CharUnits offset) {
+ if (!Capture) {
info.CXXThisIndex = index;
- else
- info.Captures[Capture->getVariable()]
- = CGBlockInfo::Capture::makeIndex(index);
+ info.CXXThisOffset = offset;
+ } else {
+ info.Captures.insert({Capture->getVariable(),
+ CGBlockInfo::Capture::makeIndex(index, offset)});
+ }
}
};
/// Order by 1) all __strong together 2) next, all byfref together 3) next,
/// all __weak together. Preserve descending alignment in all situations.
bool operator<(const BlockLayoutChunk &left, const BlockLayoutChunk &right) {
- CharUnits LeftValue, RightValue;
- bool LeftByref = left.Capture ? left.Capture->isByRef() : false;
- bool RightByref = right.Capture ? right.Capture->isByRef() : false;
-
- if (left.Lifetime == Qualifiers::OCL_Strong &&
- left.Alignment >= right.Alignment)
- LeftValue = CharUnits::fromQuantity(64);
- else if (LeftByref && left.Alignment >= right.Alignment)
- LeftValue = CharUnits::fromQuantity(32);
- else if (left.Lifetime == Qualifiers::OCL_Weak &&
- left.Alignment >= right.Alignment)
- LeftValue = CharUnits::fromQuantity(16);
- else
- LeftValue = left.Alignment;
- if (right.Lifetime == Qualifiers::OCL_Strong &&
- right.Alignment >= left.Alignment)
- RightValue = CharUnits::fromQuantity(64);
- else if (RightByref && right.Alignment >= left.Alignment)
- RightValue = CharUnits::fromQuantity(32);
- else if (right.Lifetime == Qualifiers::OCL_Weak &&
- right.Alignment >= left.Alignment)
- RightValue = CharUnits::fromQuantity(16);
- else
- RightValue = right.Alignment;
-
- return LeftValue > RightValue;
+ if (left.Alignment != right.Alignment)
+ return left.Alignment > right.Alignment;
+
+ auto getPrefOrder = [](const BlockLayoutChunk &chunk) {
+ if (chunk.Capture && chunk.Capture->isByRef())
+ return 1;
+ if (chunk.Lifetime == Qualifiers::OCL_Strong)
+ return 0;
+ if (chunk.Lifetime == Qualifiers::OCL_Weak)
+ return 2;
+ return 3;
+ };
+
+ return getPrefOrder(left) < getPrefOrder(right);
}
-}
+} // end anonymous namespace
/// Determines if the given type is safe for constant capture in C++.
static bool isSafeForCXXConstantCapture(QualType type) {
@@ -302,31 +292,20 @@
static void initializeForBlockHeader(CodeGenModule &CGM, CGBlockInfo &info,
SmallVectorImpl<llvm::Type*> &elementTypes) {
- ASTContext &C = CGM.getContext();
+ // The header is basically 'struct { void *; int; int; void *; void *; }'.
+ // Assert that that struct is packed.
+ assert(CGM.getIntSize() <= CGM.getPointerSize());
+ assert(CGM.getIntAlign() <= CGM.getPointerAlign());
+ assert((2 * CGM.getIntSize()).isMultipleOf(CGM.getPointerAlign()));
- // The header is basically a 'struct { void *; int; int; void *; void *; }'.
- CharUnits ptrSize, ptrAlign, intSize, intAlign;
- std::tie(ptrSize, ptrAlign) = C.getTypeInfoInChars(C.VoidPtrTy);
- std::tie(intSize, intAlign) = C.getTypeInfoInChars(C.IntTy);
-
- // Are there crazy embedded platforms where this isn't true?
- assert(intSize <= ptrSize && "layout assumptions horribly violated");
-
- CharUnits headerSize = ptrSize;
- if (2 * intSize < ptrAlign) headerSize += ptrSize;
- else headerSize += 2 * intSize;
- headerSize += 2 * ptrSize;
-
- info.BlockAlign = ptrAlign;
- info.BlockSize = headerSize;
+ info.BlockAlign = CGM.getPointerAlign();
+ info.BlockSize = 3 * CGM.getPointerSize() + 2 * CGM.getIntSize();
assert(elementTypes.empty());
- llvm::Type *i8p = CGM.getTypes().ConvertType(C.VoidPtrTy);
- llvm::Type *intTy = CGM.getTypes().ConvertType(C.IntTy);
- elementTypes.push_back(i8p);
- elementTypes.push_back(intTy);
- elementTypes.push_back(intTy);
- elementTypes.push_back(i8p);
+ elementTypes.push_back(CGM.VoidPtrTy);
+ elementTypes.push_back(CGM.IntTy);
+ elementTypes.push_back(CGM.IntTy);
+ elementTypes.push_back(CGM.VoidPtrTy);
elementTypes.push_back(CGM.getBlockDescriptorType());
assert(elementTypes.size() == BlockHeaderSize);
@@ -365,6 +344,8 @@
"Can't capture 'this' outside a method");
QualType thisType = cast<CXXMethodDecl>(CGF->CurFuncDecl)->getThisType(C);
+ // Theoretically, this could be in a different address space, so
+ // don't assume standard pointer size/align.
llvm::Type *llvmType = CGM.getTypes().ConvertType(thisType);
std::pair<CharUnits,CharUnits> tinfo
= CGM.getContext().getTypeInfoInChars(thisType);
@@ -384,15 +365,12 @@
info.NeedsCopyDispose = true;
// Just use void* instead of a pointer to the byref type.
- QualType byRefPtrTy = C.VoidPtrTy;
+ CharUnits align = CGM.getPointerAlign();
+ maxFieldAlign = std::max(maxFieldAlign, align);
- llvm::Type *llvmType = CGM.getTypes().ConvertType(byRefPtrTy);
- std::pair<CharUnits,CharUnits> tinfo
- = CGM.getContext().getTypeInfoInChars(byRefPtrTy);
- maxFieldAlign = std::max(maxFieldAlign, tinfo.second);
-
- layout.push_back(BlockLayoutChunk(tinfo.second, tinfo.first,
- Qualifiers::OCL_None, &CI, llvmType));
+ layout.push_back(BlockLayoutChunk(align, CGM.getPointerSize(),
+ Qualifiers::OCL_None, &CI,
+ CGM.VoidPtrTy));
continue;
}
@@ -421,9 +399,15 @@
// Block pointers require copy/dispose. So do Objective-C pointers.
} else if (variable->getType()->isObjCRetainableType()) {
- info.NeedsCopyDispose = true;
- // used for mrr below.
- lifetime = Qualifiers::OCL_Strong;
+ // But honor the inert __unsafe_unretained qualifier, which doesn't
+ // actually make it into the type system.
+ if (variable->getType()->isObjCInertUnsafeUnretainedType()) {
+ lifetime = Qualifiers::OCL_ExplicitNone;
+ } else {
+ info.NeedsCopyDispose = true;
+ // used for mrr below.
+ lifetime = Qualifiers::OCL_Strong;
+ }
// So do types that require non-trivial copy construction.
} else if (CI.hasCopyExpr()) {
@@ -504,18 +488,13 @@
for (; li != le; ++li) {
assert(endAlign >= li->Alignment);
- li->setIndex(info, elementTypes.size());
+ li->setIndex(info, elementTypes.size(), blockSize);
elementTypes.push_back(li->Type);
blockSize += li->Size;
endAlign = getLowBit(blockSize);
// ...until we get to the alignment of the maximum field.
if (endAlign >= maxFieldAlign) {
- if (li == first) {
- // No user field was appended. So, a gap was added.
- // Save total gap size for use in block layout bit map.
- info.BlockHeaderForcedGapSize = li->Size;
- }
break;
}
}
@@ -532,6 +511,12 @@
CharUnits newBlockSize = blockSize.RoundUpToAlignment(maxFieldAlign);
CharUnits padding = newBlockSize - blockSize;
+ // If we haven't yet added any fields, remember that there was an
+ // initial gap; this need to go into the block layout bit map.
+ if (blockSize == info.BlockHeaderForcedGapOffset) {
+ info.BlockHeaderForcedGapSize = padding;
+ }
+
elementTypes.push_back(llvm::ArrayType::get(CGM.Int8Ty,
padding.getQuantity()));
blockSize = newBlockSize;
@@ -556,7 +541,7 @@
endAlign = getLowBit(blockSize);
}
assert(endAlign >= li->Alignment);
- li->setIndex(info, elementTypes.size());
+ li->setIndex(info, elementTypes.size(), blockSize);
elementTypes.push_back(li->Type);
blockSize += li->Size;
endAlign = getLowBit(blockSize);
@@ -586,9 +571,8 @@
if (blockInfo.CanBeGlobal) return;
// Make the allocation for the block.
- blockInfo.Address =
- CGF.CreateTempAlloca(blockInfo.StructureType, "block");
- blockInfo.Address->setAlignment(blockInfo.BlockAlign.getQuantity());
+ blockInfo.LocalAddress = CGF.CreateTempAlloca(blockInfo.StructureType,
+ blockInfo.BlockAlign, "block");
// If there are cleanups to emit, enter them (but inactive).
if (!blockInfo.NeedsCopyDispose) return;
@@ -621,12 +605,13 @@
}
// GEP down to the address.
- llvm::Value *addr = CGF.Builder.CreateStructGEP(
- blockInfo.StructureType, blockInfo.Address, capture.getIndex());
+ Address addr = CGF.Builder.CreateStructGEP(blockInfo.LocalAddress,
+ capture.getIndex(),
+ capture.getOffset());
// We can use that GEP as the dominating IP.
if (!blockInfo.DominatingIP)
- blockInfo.DominatingIP = cast<llvm::Instruction>(addr);
+ blockInfo.DominatingIP = cast<llvm::Instruction>(addr.getPointer());
CleanupKind cleanupKind = InactiveNormalCleanup;
bool useArrayEHCleanup = CGF.needsEHCleanup(dtorKind);
@@ -721,9 +706,8 @@
// Build the block descriptor.
llvm::Constant *descriptor = buildBlockDescriptor(CGM, blockInfo);
- llvm::Type *blockTy = blockInfo.StructureType;
- llvm::AllocaInst *blockAddr = blockInfo.Address;
- assert(blockAddr && "block has no address!");
+ Address blockAddr = blockInfo.LocalAddress;
+ assert(blockAddr.isValid() && "block has no address!");
// Compute the initial on-stack block flags.
BlockFlags flags = BLOCK_HAS_SIGNATURE;
@@ -732,27 +716,44 @@
if (blockInfo.HasCXXObject) flags |= BLOCK_HAS_CXX_OBJ;
if (blockInfo.UsesStret) flags |= BLOCK_USE_STRET;
- // Initialize the block literal.
- Builder.CreateStore(
- isa, Builder.CreateStructGEP(blockTy, blockAddr, 0, "block.isa"));
- Builder.CreateStore(
- llvm::ConstantInt::get(IntTy, flags.getBitMask()),
- Builder.CreateStructGEP(blockTy, blockAddr, 1, "block.flags"));
- Builder.CreateStore(
- llvm::ConstantInt::get(IntTy, 0),
- Builder.CreateStructGEP(blockTy, blockAddr, 2, "block.reserved"));
- Builder.CreateStore(
- blockFn, Builder.CreateStructGEP(blockTy, blockAddr, 3, "block.invoke"));
- Builder.CreateStore(descriptor, Builder.CreateStructGEP(blockTy, blockAddr, 4,
- "block.descriptor"));
+ auto projectField =
+ [&](unsigned index, CharUnits offset, const Twine &name) -> Address {
+ return Builder.CreateStructGEP(blockAddr, index, offset, name);
+ };
+ auto storeField =
+ [&](llvm::Value *value, unsigned index, CharUnits offset,
+ const Twine &name) {
+ Builder.CreateStore(value, projectField(index, offset, name));
+ };
+
+ // Initialize the block header.
+ {
+ // We assume all the header fields are densely packed.
+ unsigned index = 0;
+ CharUnits offset;
+ auto addHeaderField =
+ [&](llvm::Value *value, CharUnits size, const Twine &name) {
+ storeField(value, index, offset, name);
+ offset += size;
+ index++;
+ };
+
+ addHeaderField(isa, getPointerSize(), "block.isa");
+ addHeaderField(llvm::ConstantInt::get(IntTy, flags.getBitMask()),
+ getIntSize(), "block.flags");
+ addHeaderField(llvm::ConstantInt::get(IntTy, 0),
+ getIntSize(), "block.reserved");
+ addHeaderField(blockFn, getPointerSize(), "block.invoke");
+ addHeaderField(descriptor, getPointerSize(), "block.descriptor");
+ }
// Finally, capture all the values into the block.
const BlockDecl *blockDecl = blockInfo.getBlockDecl();
// First, 'this'.
if (blockDecl->capturesCXXThis()) {
- llvm::Value *addr = Builder.CreateStructGEP(
- blockTy, blockAddr, blockInfo.CXXThisIndex, "block.captured-this.addr");
+ Address addr = projectField(blockInfo.CXXThisIndex, blockInfo.CXXThisOffset,
+ "block.captured-this.addr");
Builder.CreateStore(LoadCXXThis(), addr);
}
@@ -765,35 +766,37 @@
if (capture.isConstant()) continue;
QualType type = variable->getType();
- CharUnits align = getContext().getDeclAlign(variable);
// This will be a [[type]]*, except that a byref entry will just be
// an i8**.
- llvm::Value *blockField = Builder.CreateStructGEP(
- blockTy, blockAddr, capture.getIndex(), "block.captured");
+ Address blockField =
+ projectField(capture.getIndex(), capture.getOffset(), "block.captured");
// Compute the address of the thing we're going to move into the
// block literal.
- llvm::Value *src;
+ Address src = Address::invalid();
if (BlockInfo && CI.isNested()) {
// We need to use the capture from the enclosing block.
const CGBlockInfo::Capture &enclosingCapture =
BlockInfo->getCapture(variable);
// This is a [[type]]*, except that a byref entry wil just be an i8**.
- src = Builder.CreateStructGEP(BlockInfo->StructureType, LoadBlockStruct(),
+ src = Builder.CreateStructGEP(LoadBlockStruct(),
enclosingCapture.getIndex(),
+ enclosingCapture.getOffset(),
"block.capture.addr");
} else if (blockDecl->isConversionFromLambda()) {
// The lambda capture in a lambda's conversion-to-block-pointer is
// special; we'll simply emit it directly.
- src = nullptr;
+ src = Address::invalid();
} else {
// Just look it up in the locals map, which will give us back a
// [[type]]*. If that doesn't work, do the more elaborate DRE
// emission.
- src = LocalDeclMap.lookup(variable);
- if (!src) {
+ auto it = LocalDeclMap.find(variable);
+ if (it != LocalDeclMap.end()) {
+ src = it->second;
+ } else {
DeclRefExpr declRef(
const_cast<VarDecl *>(variable),
/*RefersToEnclosingVariableOrCapture*/ CI.isNested(), type,
@@ -808,14 +811,14 @@
// live a shorter life than the stack byref anyway.
if (CI.isByRef()) {
// Get a void* that points to the byref struct.
+ llvm::Value *byrefPointer;
if (CI.isNested())
- src = Builder.CreateAlignedLoad(src, align.getQuantity(),
- "byref.capture");
+ byrefPointer = Builder.CreateLoad(src, "byref.capture");
else
- src = Builder.CreateBitCast(src, VoidPtrTy);
+ byrefPointer = Builder.CreateBitCast(src.getPointer(), VoidPtrTy);
// Write that void* into the capture field.
- Builder.CreateAlignedStore(src, blockField, align.getQuantity());
+ Builder.CreateStore(byrefPointer, blockField);
// If we have a copy constructor, evaluate that into the block field.
} else if (const Expr *copyExpr = CI.getCopyExpr()) {
@@ -823,7 +826,7 @@
// If we have a lambda conversion, emit the expression
// directly into the block instead.
AggValueSlot Slot =
- AggValueSlot::forAddr(blockField, align, Qualifiers(),
+ AggValueSlot::forAddr(blockField, Qualifiers(),
AggValueSlot::IsDestructed,
AggValueSlot::DoesNotNeedGCBarriers,
AggValueSlot::IsNotAliased);
@@ -834,9 +837,8 @@
// If it's a reference variable, copy the reference into the block field.
} else if (type->isReferenceType()) {
- llvm::Value *ref =
- Builder.CreateAlignedLoad(src, align.getQuantity(), "ref.val");
- Builder.CreateAlignedStore(ref, blockField, align.getQuantity());
+ llvm::Value *ref = Builder.CreateLoad(src, "ref.val");
+ Builder.CreateStore(ref, blockField);
// If this is an ARC __strong block-pointer variable, don't do a
// block copy.
@@ -848,13 +850,11 @@
} else if (type.getObjCLifetime() == Qualifiers::OCL_Strong &&
type->isBlockPointerType()) {
// Load the block and do a simple retain.
- LValue srcLV = MakeAddrLValue(src, type, align);
- llvm::Value *value = EmitLoadOfScalar(srcLV, SourceLocation());
+ llvm::Value *value = Builder.CreateLoad(src, "block.captured_block");
value = EmitARCRetainNonBlock(value);
// Do a primitive store to the block field.
- LValue destLV = MakeAddrLValue(blockField, type, align);
- EmitStoreOfScalar(value, destLV, /*init*/ true);
+ Builder.CreateStore(value, blockField);
// Otherwise, fake up a POD copy into the block field.
} else {
@@ -876,7 +876,7 @@
// attributed to a reasonable location - otherwise it may be attributed to
// locations of subexpressions in the initialization.
EmitExprAsInit(&l2r, &blockFieldPseudoVar,
- MakeAddrLValue(blockField, type, align),
+ MakeAddrLValue(blockField, type, AlignmentSource::Decl),
/*captured by init*/ false);
}
@@ -891,7 +891,7 @@
// Cast to the converted block-pointer type, which happens (somewhat
// unfortunately) to be a pointer to function type.
llvm::Value *result =
- Builder.CreateBitCast(blockAddr,
+ Builder.CreateBitCast(blockAddr.getPointer(),
ConvertType(blockInfo.getBlockExpr()->getType()));
return result;
@@ -949,7 +949,6 @@
return GenericBlockLiteralType;
}
-
RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E,
ReturnValueSlot ReturnValue) {
const BlockPointerType *BPT =
@@ -966,8 +965,8 @@
Builder.CreateBitCast(Callee, BlockLiteralTy, "block.literal");
// Get the function pointer from the literal.
- llvm::Value *FuncPtr = Builder.CreateStructGEP(
- CGM.getGenericBlockLiteralType(), BlockLiteral, 3);
+ llvm::Value *FuncPtr =
+ Builder.CreateStructGEP(CGM.getGenericBlockLiteralType(), BlockLiteral, 3);
BlockLiteral = Builder.CreateBitCast(BlockLiteral, VoidPtrTy);
@@ -978,11 +977,10 @@
QualType FnType = BPT->getPointeeType();
// And the rest of the arguments.
- EmitCallArgs(Args, FnType->getAs<FunctionProtoType>(),
- E->arg_begin(), E->arg_end());
+ EmitCallArgs(Args, FnType->getAs<FunctionProtoType>(), E->arguments());
// Load the function.
- llvm::Value *Func = Builder.CreateLoad(FuncPtr);
+ llvm::Value *Func = Builder.CreateAlignedLoad(FuncPtr, getPointerAlign());
const FunctionType *FuncTy = FnType->castAs<FunctionType>();
const CGFunctionInfo &FnInfo =
@@ -998,41 +996,35 @@
return EmitCall(FnInfo, Func, ReturnValue, Args);
}
-llvm::Value *CodeGenFunction::GetAddrOfBlockDecl(const VarDecl *variable,
- bool isByRef) {
+Address CodeGenFunction::GetAddrOfBlockDecl(const VarDecl *variable,
+ bool isByRef) {
assert(BlockInfo && "evaluating block ref without block information?");
const CGBlockInfo::Capture &capture = BlockInfo->getCapture(variable);
// Handle constant captures.
- if (capture.isConstant()) return LocalDeclMap[variable];
+ if (capture.isConstant()) return LocalDeclMap.find(variable)->second;
- llvm::Value *addr =
- Builder.CreateStructGEP(BlockInfo->StructureType, LoadBlockStruct(),
- capture.getIndex(), "block.capture.addr");
+ Address addr =
+ Builder.CreateStructGEP(LoadBlockStruct(), capture.getIndex(),
+ capture.getOffset(), "block.capture.addr");
if (isByRef) {
// addr should be a void** right now. Load, then cast the result
// to byref*.
- addr = Builder.CreateLoad(addr);
- auto *byrefType = BuildByRefType(variable);
- llvm::PointerType *byrefPointerType = llvm::PointerType::get(byrefType, 0);
- addr = Builder.CreateBitCast(addr, byrefPointerType,
- "byref.addr");
+ auto &byrefInfo = getBlockByrefInfo(variable);
+ addr = Address(Builder.CreateLoad(addr), byrefInfo.ByrefAlignment);
- // Follow the forwarding pointer.
- addr = Builder.CreateStructGEP(byrefType, addr, 1, "byref.forwarding");
- addr = Builder.CreateLoad(addr, "byref.addr.forwarded");
+ auto byrefPointerType = llvm::PointerType::get(byrefInfo.Type, 0);
+ addr = Builder.CreateBitCast(addr, byrefPointerType, "byref.addr");
- // Cast back to byref* and GEP over to the actual object.
- addr = Builder.CreateBitCast(addr, byrefPointerType);
- addr = Builder.CreateStructGEP(byrefType, addr,
- getByRefValueLLVMField(variable).second,
- variable->getNameAsString());
+ addr = emitBlockByrefAddress(addr, byrefInfo, /*follow*/ true,
+ variable->getName());
}
- if (variable->getType()->isReferenceType())
- addr = Builder.CreateLoad(addr, "ref.tmp");
+ if (auto refType = variable->getType()->getAs<ReferenceType>()) {
+ addr = EmitLoadOfReference(addr, refType);
+ }
return addr;
}
@@ -1049,7 +1041,7 @@
// Using that metadata, generate the actual block function.
llvm::Constant *blockFn;
{
- llvm::DenseMap<const Decl*, llvm::Value*> LocalDeclMap;
+ CodeGenFunction::DeclMapTy LocalDeclMap;
blockFn = CodeGenFunction(*this).GenerateBlockFunction(GlobalDecl(),
blockInfo,
LocalDeclMap,
@@ -1103,6 +1095,44 @@
return llvm::ConstantExpr::getBitCast(literal, requiredType);
}
+void CodeGenFunction::setBlockContextParameter(const ImplicitParamDecl *D,
+ unsigned argNum,
+ llvm::Value *arg) {
+ assert(BlockInfo && "not emitting prologue of block invocation function?!");
+
+ llvm::Value *localAddr = nullptr;
+ if (CGM.getCodeGenOpts().OptimizationLevel == 0) {
+ // Allocate a stack slot to let the debug info survive the RA.
+ Address alloc = CreateMemTemp(D->getType(), D->getName() + ".addr");
+ Builder.CreateStore(arg, alloc);
+ localAddr = Builder.CreateLoad(alloc);
+ }
+
+ if (CGDebugInfo *DI = getDebugInfo()) {
+ if (CGM.getCodeGenOpts().getDebugInfo()
+ >= CodeGenOptions::LimitedDebugInfo) {
+ DI->setLocation(D->getLocation());
+ DI->EmitDeclareOfBlockLiteralArgVariable(*BlockInfo, arg, argNum,
+ localAddr, Builder);
+ }
+ }
+
+ SourceLocation StartLoc = BlockInfo->getBlockExpr()->getBody()->getLocStart();
+ ApplyDebugLocation Scope(*this, StartLoc);
+
+ // Instead of messing around with LocalDeclMap, just set the value
+ // directly as BlockPointer.
+ BlockPointer = Builder.CreateBitCast(arg,
+ BlockInfo->StructureType->getPointerTo(),
+ "block");
+}
+
+Address CodeGenFunction::LoadBlockStruct() {
+ assert(BlockInfo && "not in a block invocation function!");
+ assert(BlockPointer && "no block pointer set!");
+ return Address(BlockPointer, BlockInfo->BlockAlign);
+}
+
llvm::Function *
CodeGenFunction::GenerateBlockFunction(GlobalDecl GD,
const CGBlockInfo &blockInfo,
@@ -1122,7 +1152,7 @@
for (DeclMapTy::const_iterator i = ldm.begin(), e = ldm.end(); i != e; ++i) {
const auto *var = dyn_cast<VarDecl>(i->first);
if (var && !var->hasLocalStorage())
- LocalDeclMap[var] = i->second;
+ setAddrOfLocalVar(var, i->second);
}
// Begin building the function declaration.
@@ -1163,35 +1193,28 @@
blockInfo.getBlockExpr()->getBody()->getLocStart());
// Okay. Undo some of what StartFunction did.
-
- // Pull the 'self' reference out of the local decl map.
- llvm::Value *blockAddr = LocalDeclMap[&selfDecl];
- LocalDeclMap.erase(&selfDecl);
- BlockPointer = Builder.CreateBitCast(blockAddr,
- blockInfo.StructureType->getPointerTo(),
- "block");
+
// At -O0 we generate an explicit alloca for the BlockPointer, so the RA
// won't delete the dbg.declare intrinsics for captured variables.
llvm::Value *BlockPointerDbgLoc = BlockPointer;
if (CGM.getCodeGenOpts().OptimizationLevel == 0) {
// Allocate a stack slot for it, so we can point the debugger to it
- llvm::AllocaInst *Alloca = CreateTempAlloca(BlockPointer->getType(),
- "block.addr");
- unsigned Align = getContext().getDeclAlign(&selfDecl).getQuantity();
- Alloca->setAlignment(Align);
+ Address Alloca = CreateTempAlloca(BlockPointer->getType(),
+ getPointerAlign(),
+ "block.addr");
// Set the DebugLocation to empty, so the store is recognized as a
// frame setup instruction by llvm::DwarfDebug::beginFunction().
auto NL = ApplyDebugLocation::CreateEmpty(*this);
- Builder.CreateAlignedStore(BlockPointer, Alloca, Align);
- BlockPointerDbgLoc = Alloca;
+ Builder.CreateStore(BlockPointer, Alloca);
+ BlockPointerDbgLoc = Alloca.getPointer();
}
// If we have a C++ 'this' reference, go ahead and force it into
// existence now.
if (blockDecl->capturesCXXThis()) {
- llvm::Value *addr =
- Builder.CreateStructGEP(blockInfo.StructureType, BlockPointer,
- blockInfo.CXXThisIndex, "block.captured-this");
+ Address addr =
+ Builder.CreateStructGEP(LoadBlockStruct(), blockInfo.CXXThisIndex,
+ blockInfo.CXXThisOffset, "block.captured-this");
CXXThisValue = Builder.CreateLoad(addr, "this");
}
@@ -1201,15 +1224,13 @@
const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable);
if (!capture.isConstant()) continue;
- unsigned align = getContext().getDeclAlign(variable).getQuantity();
+ CharUnits align = getContext().getDeclAlign(variable);
+ Address alloca =
+ CreateMemTemp(variable->getType(), align, "block.captured-const");
- llvm::AllocaInst *alloca =
- CreateMemTemp(variable->getType(), "block.captured-const");
- alloca->setAlignment(align);
+ Builder.CreateStore(capture.getConstant(), alloca);
- Builder.CreateAlignedStore(capture.getConstant(), alloca, align);
-
- LocalDeclMap[variable] = alloca;
+ setAddrOfLocalVar(variable, alloca);
}
// Save a spot to insert the debug information for all the DeclRefExprs.
@@ -1220,7 +1241,7 @@
if (IsLambdaConversionToBlock)
EmitLambdaBlockInvokeBody();
else {
- PGO.assignRegionCounters(blockDecl, fn);
+ PGO.assignRegionCounters(GlobalDecl(blockDecl), fn);
incrementProfileCounter(blockDecl->getBody());
EmitStmt(blockDecl->getBody());
}
@@ -1243,15 +1264,15 @@
>= CodeGenOptions::LimitedDebugInfo) {
const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable);
if (capture.isConstant()) {
- DI->EmitDeclareOfAutoVariable(variable, LocalDeclMap[variable],
+ auto addr = LocalDeclMap.find(variable)->second;
+ DI->EmitDeclareOfAutoVariable(variable, addr.getPointer(),
Builder);
continue;
}
- DI->EmitDeclareOfBlockDeclRefVariable(variable, BlockPointerDbgLoc,
- Builder, blockInfo,
- entry_ptr == entry->end()
- ? nullptr : entry_ptr);
+ DI->EmitDeclareOfBlockDeclRefVariable(
+ variable, BlockPointerDbgLoc, Builder, blockInfo,
+ entry_ptr == entry->end() ? nullptr : &*entry_ptr);
}
}
// Recover location if it was changed in the above loop.
@@ -1288,7 +1309,6 @@
}
*/
-
/// Generate the copy-helper function for a block closure object:
/// static void block_copy_helper(block_t *dst, block_t *src);
/// The runtime will have previously initialized 'dst' by doing a
@@ -1330,18 +1350,21 @@
nullptr, SC_Static,
false,
false);
+
+ CGM.SetInternalFunctionAttributes(nullptr, Fn, FI);
+
auto NL = ApplyDebugLocation::CreateEmpty(*this);
StartFunction(FD, C.VoidTy, Fn, FI, args);
// Create a scope with an artificial location for the body of this function.
auto AL = ApplyDebugLocation::CreateArtificial(*this);
llvm::Type *structPtrTy = blockInfo.StructureType->getPointerTo();
- llvm::Value *src = GetAddrOfLocalVar(&srcDecl);
- src = Builder.CreateLoad(src);
+ Address src = GetAddrOfLocalVar(&srcDecl);
+ src = Address(Builder.CreateLoad(src), blockInfo.BlockAlign);
src = Builder.CreateBitCast(src, structPtrTy, "block.source");
- llvm::Value *dst = GetAddrOfLocalVar(&dstDecl);
- dst = Builder.CreateLoad(dst);
+ Address dst = GetAddrOfLocalVar(&dstDecl);
+ dst = Address(Builder.CreateLoad(dst), blockInfo.BlockAlign);
dst = Builder.CreateBitCast(dst, structPtrTy, "block.dest");
const BlockDecl *blockDecl = blockInfo.getBlockDecl();
@@ -1375,40 +1398,38 @@
flags = BLOCK_FIELD_IS_BLOCK;
// Special rules for ARC captures:
- if (getLangOpts().ObjCAutoRefCount) {
- Qualifiers qs = type.getQualifiers();
+ Qualifiers qs = type.getQualifiers();
- // We need to register __weak direct captures with the runtime.
- if (qs.getObjCLifetime() == Qualifiers::OCL_Weak) {
- useARCWeakCopy = true;
+ // We need to register __weak direct captures with the runtime.
+ if (qs.getObjCLifetime() == Qualifiers::OCL_Weak) {
+ useARCWeakCopy = true;
- // We need to retain the copied value for __strong direct captures.
- } else if (qs.getObjCLifetime() == Qualifiers::OCL_Strong) {
- // If it's a block pointer, we have to copy the block and
- // assign that to the destination pointer, so we might as
- // well use _Block_object_assign. Otherwise we can avoid that.
- if (!isBlockPointer)
- useARCStrongCopy = true;
-
- // Otherwise the memcpy is fine.
- } else {
- continue;
- }
+ // We need to retain the copied value for __strong direct captures.
+ } else if (qs.getObjCLifetime() == Qualifiers::OCL_Strong) {
+ // If it's a block pointer, we have to copy the block and
+ // assign that to the destination pointer, so we might as
+ // well use _Block_object_assign. Otherwise we can avoid that.
+ if (!isBlockPointer)
+ useARCStrongCopy = true;
// Non-ARC captures of retainable pointers are strong and
// therefore require a call to _Block_object_assign.
- } else {
+ } else if (!qs.getObjCLifetime() && !getLangOpts().ObjCAutoRefCount) {
// fall through
+
+ // Otherwise the memcpy is fine.
+ } else {
+ continue;
}
+
+ // For all other types, the memcpy is fine.
} else {
continue;
}
unsigned index = capture.getIndex();
- llvm::Value *srcField =
- Builder.CreateStructGEP(blockInfo.StructureType, src, index);
- llvm::Value *dstField =
- Builder.CreateStructGEP(blockInfo.StructureType, dst, index);
+ Address srcField = Builder.CreateStructGEP(src, index, capture.getOffset());
+ Address dstField = Builder.CreateStructGEP(dst, index, capture.getOffset());
// If there's an explicit copy expression, we do that.
if (copyExpr) {
@@ -1435,11 +1456,12 @@
// We don't need this anymore, so kill it. It's not quite
// worth the annoyance to avoid creating it in the first place.
- cast<llvm::Instruction>(dstField)->eraseFromParent();
+ cast<llvm::Instruction>(dstField.getPointer())->eraseFromParent();
}
} else {
srcValue = Builder.CreateBitCast(srcValue, VoidPtrTy);
- llvm::Value *dstAddr = Builder.CreateBitCast(dstField, VoidPtrTy);
+ llvm::Value *dstAddr =
+ Builder.CreateBitCast(dstField.getPointer(), VoidPtrTy);
llvm::Value *args[] = {
dstAddr, srcValue, llvm::ConstantInt::get(Int32Ty, flags.getBitMask())
};
@@ -1502,6 +1524,9 @@
SourceLocation(), II, C.VoidTy,
nullptr, SC_Static,
false, false);
+
+ CGM.SetInternalFunctionAttributes(nullptr, Fn, FI);
+
// Create a scope with an artificial location for the body of this function.
auto NL = ApplyDebugLocation::CreateEmpty(*this);
StartFunction(FD, C.VoidTy, Fn, FI, args);
@@ -1509,8 +1534,8 @@
llvm::Type *structPtrTy = blockInfo.StructureType->getPointerTo();
- llvm::Value *src = GetAddrOfLocalVar(&srcDecl);
- src = Builder.CreateLoad(src);
+ Address src = GetAddrOfLocalVar(&srcDecl);
+ src = Address(Builder.CreateLoad(src), blockInfo.BlockAlign);
src = Builder.CreateBitCast(src, structPtrTy, "block");
const BlockDecl *blockDecl = blockInfo.getBlockDecl();
@@ -1544,29 +1569,31 @@
flags = BLOCK_FIELD_IS_BLOCK;
// Special rules for ARC captures.
- if (getLangOpts().ObjCAutoRefCount) {
- Qualifiers qs = type.getQualifiers();
+ Qualifiers qs = type.getQualifiers();
- // Don't generate special dispose logic for a captured object
- // unless it's __strong or __weak.
- if (!qs.hasStrongOrWeakObjCLifetime())
- continue;
+ // Use objc_storeStrong for __strong direct captures; the
+ // dynamic tools really like it when we do this.
+ if (qs.getObjCLifetime() == Qualifiers::OCL_Strong) {
+ useARCStrongDestroy = true;
- // Support __weak direct captures.
- if (qs.getObjCLifetime() == Qualifiers::OCL_Weak)
- useARCWeakDestroy = true;
+ // Support __weak direct captures.
+ } else if (qs.getObjCLifetime() == Qualifiers::OCL_Weak) {
+ useARCWeakDestroy = true;
- // Tools really want us to use objc_storeStrong here.
- else
- useARCStrongDestroy = true;
+ // Non-ARC captures are strong, and we need to use _Block_object_dispose.
+ } else if (!qs.hasObjCLifetime() && !getLangOpts().ObjCAutoRefCount) {
+ // fall through
+
+ // Otherwise, we have nothing to do.
+ } else {
+ continue;
}
} else {
continue;
}
- unsigned index = capture.getIndex();
- llvm::Value *srcField =
- Builder.CreateStructGEP(blockInfo.StructureType, src, index);
+ Address srcField =
+ Builder.CreateStructGEP(src, capture.getIndex(), capture.getOffset());
// If there's an explicit copy expression, we do that.
if (dtor) {
@@ -1600,15 +1627,15 @@
namespace {
/// Emits the copy/dispose helper functions for a __block object of id type.
-class ObjectByrefHelpers : public CodeGenModule::ByrefHelpers {
+class ObjectByrefHelpers final : public BlockByrefHelpers {
BlockFieldFlags Flags;
public:
ObjectByrefHelpers(CharUnits alignment, BlockFieldFlags flags)
- : ByrefHelpers(alignment), Flags(flags) {}
+ : BlockByrefHelpers(alignment), Flags(flags) {}
- void emitCopy(CodeGenFunction &CGF, llvm::Value *destField,
- llvm::Value *srcField) override {
+ void emitCopy(CodeGenFunction &CGF, Address destField,
+ Address srcField) override {
destField = CGF.Builder.CreateBitCast(destField, CGF.VoidPtrTy);
srcField = CGF.Builder.CreateBitCast(srcField, CGF.VoidPtrPtrTy);
@@ -1619,11 +1646,11 @@
llvm::Value *flagsVal = llvm::ConstantInt::get(CGF.Int32Ty, flags);
llvm::Value *fn = CGF.CGM.getBlockObjectAssign();
- llvm::Value *args[] = { destField, srcValue, flagsVal };
+ llvm::Value *args[] = { destField.getPointer(), srcValue, flagsVal };
CGF.EmitNounwindRuntimeCall(fn, args);
}
- void emitDispose(CodeGenFunction &CGF, llvm::Value *field) override {
+ void emitDispose(CodeGenFunction &CGF, Address field) override {
field = CGF.Builder.CreateBitCast(field, CGF.Int8PtrTy->getPointerTo(0));
llvm::Value *value = CGF.Builder.CreateLoad(field);
@@ -1636,16 +1663,16 @@
};
/// Emits the copy/dispose helpers for an ARC __block __weak variable.
-class ARCWeakByrefHelpers : public CodeGenModule::ByrefHelpers {
+class ARCWeakByrefHelpers final : public BlockByrefHelpers {
public:
- ARCWeakByrefHelpers(CharUnits alignment) : ByrefHelpers(alignment) {}
+ ARCWeakByrefHelpers(CharUnits alignment) : BlockByrefHelpers(alignment) {}
- void emitCopy(CodeGenFunction &CGF, llvm::Value *destField,
- llvm::Value *srcField) override {
+ void emitCopy(CodeGenFunction &CGF, Address destField,
+ Address srcField) override {
CGF.EmitARCMoveWeak(destField, srcField);
}
- void emitDispose(CodeGenFunction &CGF, llvm::Value *field) override {
+ void emitDispose(CodeGenFunction &CGF, Address field) override {
CGF.EmitARCDestroyWeak(field);
}
@@ -1657,36 +1684,31 @@
/// Emits the copy/dispose helpers for an ARC __block __strong variable
/// that's not of block-pointer type.
-class ARCStrongByrefHelpers : public CodeGenModule::ByrefHelpers {
+class ARCStrongByrefHelpers final : public BlockByrefHelpers {
public:
- ARCStrongByrefHelpers(CharUnits alignment) : ByrefHelpers(alignment) {}
+ ARCStrongByrefHelpers(CharUnits alignment) : BlockByrefHelpers(alignment) {}
- void emitCopy(CodeGenFunction &CGF, llvm::Value *destField,
- llvm::Value *srcField) override {
+ void emitCopy(CodeGenFunction &CGF, Address destField,
+ Address srcField) override {
// Do a "move" by copying the value and then zeroing out the old
// variable.
- llvm::LoadInst *value = CGF.Builder.CreateLoad(srcField);
- value->setAlignment(Alignment.getQuantity());
+ llvm::Value *value = CGF.Builder.CreateLoad(srcField);
llvm::Value *null =
llvm::ConstantPointerNull::get(cast<llvm::PointerType>(value->getType()));
if (CGF.CGM.getCodeGenOpts().OptimizationLevel == 0) {
- llvm::StoreInst *store = CGF.Builder.CreateStore(null, destField);
- store->setAlignment(Alignment.getQuantity());
+ CGF.Builder.CreateStore(null, destField);
CGF.EmitARCStoreStrongCall(destField, value, /*ignored*/ true);
CGF.EmitARCStoreStrongCall(srcField, null, /*ignored*/ true);
return;
}
- llvm::StoreInst *store = CGF.Builder.CreateStore(value, destField);
- store->setAlignment(Alignment.getQuantity());
-
- store = CGF.Builder.CreateStore(null, srcField);
- store->setAlignment(Alignment.getQuantity());
+ CGF.Builder.CreateStore(value, destField);
+ CGF.Builder.CreateStore(null, srcField);
}
- void emitDispose(CodeGenFunction &CGF, llvm::Value *field) override {
+ void emitDispose(CodeGenFunction &CGF, Address field) override {
CGF.EmitARCDestroyStrong(field, ARCImpreciseLifetime);
}
@@ -1698,25 +1720,22 @@
/// Emits the copy/dispose helpers for an ARC __block __strong
/// variable that's of block-pointer type.
-class ARCStrongBlockByrefHelpers : public CodeGenModule::ByrefHelpers {
+class ARCStrongBlockByrefHelpers final : public BlockByrefHelpers {
public:
- ARCStrongBlockByrefHelpers(CharUnits alignment) : ByrefHelpers(alignment) {}
+ ARCStrongBlockByrefHelpers(CharUnits alignment)
+ : BlockByrefHelpers(alignment) {}
- void emitCopy(CodeGenFunction &CGF, llvm::Value *destField,
- llvm::Value *srcField) override {
+ void emitCopy(CodeGenFunction &CGF, Address destField,
+ Address srcField) override {
// Do the copy with objc_retainBlock; that's all that
// _Block_object_assign would do anyway, and we'd have to pass the
// right arguments to make sure it doesn't get no-op'ed.
- llvm::LoadInst *oldValue = CGF.Builder.CreateLoad(srcField);
- oldValue->setAlignment(Alignment.getQuantity());
-
+ llvm::Value *oldValue = CGF.Builder.CreateLoad(srcField);
llvm::Value *copy = CGF.EmitARCRetainBlock(oldValue, /*mandatory*/ true);
-
- llvm::StoreInst *store = CGF.Builder.CreateStore(copy, destField);
- store->setAlignment(Alignment.getQuantity());
+ CGF.Builder.CreateStore(copy, destField);
}
- void emitDispose(CodeGenFunction &CGF, llvm::Value *field) override {
+ void emitDispose(CodeGenFunction &CGF, Address field) override {
CGF.EmitARCDestroyStrong(field, ARCImpreciseLifetime);
}
@@ -1728,23 +1747,23 @@
/// Emits the copy/dispose helpers for a __block variable with a
/// nontrivial copy constructor or destructor.
-class CXXByrefHelpers : public CodeGenModule::ByrefHelpers {
+class CXXByrefHelpers final : public BlockByrefHelpers {
QualType VarType;
const Expr *CopyExpr;
public:
CXXByrefHelpers(CharUnits alignment, QualType type,
const Expr *copyExpr)
- : ByrefHelpers(alignment), VarType(type), CopyExpr(copyExpr) {}
+ : BlockByrefHelpers(alignment), VarType(type), CopyExpr(copyExpr) {}
bool needsCopy() const override { return CopyExpr != nullptr; }
- void emitCopy(CodeGenFunction &CGF, llvm::Value *destField,
- llvm::Value *srcField) override {
+ void emitCopy(CodeGenFunction &CGF, Address destField,
+ Address srcField) override {
if (!CopyExpr) return;
CGF.EmitSynthesizedCXXCopyCtor(destField, srcField, CopyExpr);
}
- void emitDispose(CodeGenFunction &CGF, llvm::Value *field) override {
+ void emitDispose(CodeGenFunction &CGF, Address field) override {
EHScopeStack::stable_iterator cleanupDepth = CGF.EHStack.stable_begin();
CGF.PushDestructorCleanup(VarType, field);
CGF.PopCleanupBlocks(cleanupDepth);
@@ -1757,10 +1776,8 @@
} // end anonymous namespace
static llvm::Constant *
-generateByrefCopyHelper(CodeGenFunction &CGF,
- llvm::StructType &byrefType,
- unsigned valueFieldIndex,
- CodeGenModule::ByrefHelpers &byrefInfo) {
+generateByrefCopyHelper(CodeGenFunction &CGF, const BlockByrefInfo &byrefInfo,
+ BlockByrefHelpers &generator) {
ASTContext &Context = CGF.getContext();
QualType R = Context.VoidTy;
@@ -1777,8 +1794,7 @@
const CGFunctionInfo &FI = CGF.CGM.getTypes().arrangeFreeFunctionDeclaration(
R, args, FunctionType::ExtInfo(), /*variadic=*/false);
- CodeGenTypes &Types = CGF.CGM.getTypes();
- llvm::FunctionType *LTy = Types.GetFunctionType(FI);
+ llvm::FunctionType *LTy = CGF.CGM.getTypes().GetFunctionType(FI);
// FIXME: We'd like to put these into a mergable by content, with
// internal linkage.
@@ -1796,26 +1812,30 @@
SC_Static,
false, false);
+ CGF.CGM.SetInternalFunctionAttributes(nullptr, Fn, FI);
+
CGF.StartFunction(FD, R, Fn, FI, args);
- if (byrefInfo.needsCopy()) {
- llvm::Type *byrefPtrType = byrefType.getPointerTo(0);
+ if (generator.needsCopy()) {
+ llvm::Type *byrefPtrType = byrefInfo.Type->getPointerTo(0);
// dst->x
- llvm::Value *destField = CGF.GetAddrOfLocalVar(&dst);
- destField = CGF.Builder.CreateLoad(destField);
+ Address destField = CGF.GetAddrOfLocalVar(&dst);
+ destField = Address(CGF.Builder.CreateLoad(destField),
+ byrefInfo.ByrefAlignment);
destField = CGF.Builder.CreateBitCast(destField, byrefPtrType);
- destField = CGF.Builder.CreateStructGEP(&byrefType, destField,
- valueFieldIndex, "x");
+ destField = CGF.emitBlockByrefAddress(destField, byrefInfo, false,
+ "dest-object");
// src->x
- llvm::Value *srcField = CGF.GetAddrOfLocalVar(&src);
- srcField = CGF.Builder.CreateLoad(srcField);
+ Address srcField = CGF.GetAddrOfLocalVar(&src);
+ srcField = Address(CGF.Builder.CreateLoad(srcField),
+ byrefInfo.ByrefAlignment);
srcField = CGF.Builder.CreateBitCast(srcField, byrefPtrType);
- srcField =
- CGF.Builder.CreateStructGEP(&byrefType, srcField, valueFieldIndex, "x");
+ srcField = CGF.emitBlockByrefAddress(srcField, byrefInfo, false,
+ "src-object");
- byrefInfo.emitCopy(CGF, destField, srcField);
+ generator.emitCopy(CGF, destField, srcField);
}
CGF.FinishFunction();
@@ -1825,19 +1845,17 @@
/// Build the copy helper for a __block variable.
static llvm::Constant *buildByrefCopyHelper(CodeGenModule &CGM,
- llvm::StructType &byrefType,
- unsigned byrefValueIndex,
- CodeGenModule::ByrefHelpers &info) {
+ const BlockByrefInfo &byrefInfo,
+ BlockByrefHelpers &generator) {
CodeGenFunction CGF(CGM);
- return generateByrefCopyHelper(CGF, byrefType, byrefValueIndex, info);
+ return generateByrefCopyHelper(CGF, byrefInfo, generator);
}
/// Generate code for a __block variable's dispose helper.
static llvm::Constant *
generateByrefDisposeHelper(CodeGenFunction &CGF,
- llvm::StructType &byrefType,
- unsigned byrefValueIndex,
- CodeGenModule::ByrefHelpers &byrefInfo) {
+ const BlockByrefInfo &byrefInfo,
+ BlockByrefHelpers &generator) {
ASTContext &Context = CGF.getContext();
QualType R = Context.VoidTy;
@@ -1849,8 +1867,7 @@
const CGFunctionInfo &FI = CGF.CGM.getTypes().arrangeFreeFunctionDeclaration(
R, args, FunctionType::ExtInfo(), /*variadic=*/false);
- CodeGenTypes &Types = CGF.CGM.getTypes();
- llvm::FunctionType *LTy = Types.GetFunctionType(FI);
+ llvm::FunctionType *LTy = CGF.CGM.getTypes().GetFunctionType(FI);
// FIXME: We'd like to put these into a mergable by content, with
// internal linkage.
@@ -1868,15 +1885,19 @@
SourceLocation(), II, R, nullptr,
SC_Static,
false, false);
+
+ CGF.CGM.SetInternalFunctionAttributes(nullptr, Fn, FI);
+
CGF.StartFunction(FD, R, Fn, FI, args);
- if (byrefInfo.needsDispose()) {
- llvm::Value *V = CGF.GetAddrOfLocalVar(&src);
- V = CGF.Builder.CreateLoad(V);
- V = CGF.Builder.CreateBitCast(V, byrefType.getPointerTo(0));
- V = CGF.Builder.CreateStructGEP(&byrefType, V, byrefValueIndex, "x");
+ if (generator.needsDispose()) {
+ Address addr = CGF.GetAddrOfLocalVar(&src);
+ addr = Address(CGF.Builder.CreateLoad(addr), byrefInfo.ByrefAlignment);
+ auto byrefPtrType = byrefInfo.Type->getPointerTo(0);
+ addr = CGF.Builder.CreateBitCast(addr, byrefPtrType);
+ addr = CGF.emitBlockByrefAddress(addr, byrefInfo, false, "object");
- byrefInfo.emitDispose(CGF, V);
+ generator.emitDispose(CGF, addr);
}
CGF.FinishFunction();
@@ -1886,38 +1907,29 @@
/// Build the dispose helper for a __block variable.
static llvm::Constant *buildByrefDisposeHelper(CodeGenModule &CGM,
- llvm::StructType &byrefType,
- unsigned byrefValueIndex,
- CodeGenModule::ByrefHelpers &info) {
+ const BlockByrefInfo &byrefInfo,
+ BlockByrefHelpers &generator) {
CodeGenFunction CGF(CGM);
- return generateByrefDisposeHelper(CGF, byrefType, byrefValueIndex, info);
+ return generateByrefDisposeHelper(CGF, byrefInfo, generator);
}
/// Lazily build the copy and dispose helpers for a __block variable
/// with the given information.
-template <class T> static T *buildByrefHelpers(CodeGenModule &CGM,
- llvm::StructType &byrefTy,
- unsigned byrefValueIndex,
- T &byrefInfo) {
- // Increase the field's alignment to be at least pointer alignment,
- // since the layout of the byref struct will guarantee at least that.
- byrefInfo.Alignment = std::max(byrefInfo.Alignment,
- CharUnits::fromQuantity(CGM.PointerAlignInBytes));
-
+template <class T>
+static T *buildByrefHelpers(CodeGenModule &CGM, const BlockByrefInfo &byrefInfo,
+ T &&generator) {
llvm::FoldingSetNodeID id;
- byrefInfo.Profile(id);
+ generator.Profile(id);
void *insertPos;
- CodeGenModule::ByrefHelpers *node
+ BlockByrefHelpers *node
= CGM.ByrefHelpersCache.FindNodeOrInsertPos(id, insertPos);
if (node) return static_cast<T*>(node);
- byrefInfo.CopyHelper =
- buildByrefCopyHelper(CGM, byrefTy, byrefValueIndex, byrefInfo);
- byrefInfo.DisposeHelper =
- buildByrefDisposeHelper(CGM, byrefTy, byrefValueIndex,byrefInfo);
+ generator.CopyHelper = buildByrefCopyHelper(CGM, byrefInfo, generator);
+ generator.DisposeHelper = buildByrefDisposeHelper(CGM, byrefInfo, generator);
- T *copy = new (CGM.getContext()) T(byrefInfo);
+ T *copy = new (CGM.getContext()) T(std::move(generator));
CGM.ByrefHelpersCache.InsertNode(copy, insertPos);
return copy;
}
@@ -1925,20 +1937,25 @@
/// Build the copy and dispose helpers for the given __block variable
/// emission. Places the helpers in the global cache. Returns null
/// if no helpers are required.
-CodeGenModule::ByrefHelpers *
+BlockByrefHelpers *
CodeGenFunction::buildByrefHelpers(llvm::StructType &byrefType,
const AutoVarEmission &emission) {
const VarDecl &var = *emission.Variable;
QualType type = var.getType();
- unsigned byrefValueIndex = getByRefValueLLVMField(&var).second;
+ auto &byrefInfo = getBlockByrefInfo(&var);
+
+ // The alignment we care about for the purposes of uniquing byref
+ // helpers is the alignment of the actual byref value field.
+ CharUnits valueAlignment =
+ byrefInfo.ByrefAlignment.alignmentAtOffset(byrefInfo.FieldOffset);
if (const CXXRecordDecl *record = type->getAsCXXRecordDecl()) {
const Expr *copyExpr = CGM.getContext().getBlockVarCopyInits(&var);
if (!copyExpr && record->hasTrivialDestructor()) return nullptr;
- CXXByrefHelpers byrefInfo(emission.Alignment, type, copyExpr);
- return ::buildByrefHelpers(CGM, byrefType, byrefValueIndex, byrefInfo);
+ return ::buildByrefHelpers(
+ CGM, byrefInfo, CXXByrefHelpers(valueAlignment, type, copyExpr));
}
// Otherwise, if we don't have a retainable type, there's nothing to do.
@@ -1949,8 +1966,6 @@
// If we have lifetime, that dominates.
if (Qualifiers::ObjCLifetime lifetime = qs.getObjCLifetime()) {
- assert(getLangOpts().ObjCAutoRefCount);
-
switch (lifetime) {
case Qualifiers::OCL_None: llvm_unreachable("impossible");
@@ -1961,24 +1976,23 @@
// Tell the runtime that this is ARC __weak, called by the
// byref routines.
- case Qualifiers::OCL_Weak: {
- ARCWeakByrefHelpers byrefInfo(emission.Alignment);
- return ::buildByrefHelpers(CGM, byrefType, byrefValueIndex, byrefInfo);
- }
+ case Qualifiers::OCL_Weak:
+ return ::buildByrefHelpers(CGM, byrefInfo,
+ ARCWeakByrefHelpers(valueAlignment));
// ARC __strong __block variables need to be retained.
case Qualifiers::OCL_Strong:
// Block pointers need to be copied, and there's no direct
// transfer possible.
if (type->isBlockPointerType()) {
- ARCStrongBlockByrefHelpers byrefInfo(emission.Alignment);
- return ::buildByrefHelpers(CGM, byrefType, byrefValueIndex, byrefInfo);
+ return ::buildByrefHelpers(CGM, byrefInfo,
+ ARCStrongBlockByrefHelpers(valueAlignment));
// Otherwise, we transfer ownership of the retain from the stack
// to the heap.
} else {
- ARCStrongByrefHelpers byrefInfo(emission.Alignment);
- return ::buildByrefHelpers(CGM, byrefType, byrefValueIndex, byrefInfo);
+ return ::buildByrefHelpers(CGM, byrefInfo,
+ ARCStrongByrefHelpers(valueAlignment));
}
}
llvm_unreachable("fell out of lifetime switch!");
@@ -1997,28 +2011,33 @@
if (type.isObjCGCWeak())
flags |= BLOCK_FIELD_IS_WEAK;
- ObjectByrefHelpers byrefInfo(emission.Alignment, flags);
- return ::buildByrefHelpers(CGM, byrefType, byrefValueIndex, byrefInfo);
+ return ::buildByrefHelpers(CGM, byrefInfo,
+ ObjectByrefHelpers(valueAlignment, flags));
}
-std::pair<llvm::Type *, unsigned>
-CodeGenFunction::getByRefValueLLVMField(const ValueDecl *VD) const {
- assert(ByRefValueInfo.count(VD) && "Did not find value!");
-
- return ByRefValueInfo.find(VD)->second;
+Address CodeGenFunction::emitBlockByrefAddress(Address baseAddr,
+ const VarDecl *var,
+ bool followForward) {
+ auto &info = getBlockByrefInfo(var);
+ return emitBlockByrefAddress(baseAddr, info, followForward, var->getName());
}
-llvm::Value *CodeGenFunction::BuildBlockByrefAddress(llvm::Value *BaseAddr,
- const VarDecl *V) {
- auto P = getByRefValueLLVMField(V);
- llvm::Value *Loc =
- Builder.CreateStructGEP(P.first, BaseAddr, 1, "forwarding");
- Loc = Builder.CreateLoad(Loc);
- Loc = Builder.CreateStructGEP(P.first, Loc, P.second, V->getNameAsString());
- return Loc;
+Address CodeGenFunction::emitBlockByrefAddress(Address baseAddr,
+ const BlockByrefInfo &info,
+ bool followForward,
+ const llvm::Twine &name) {
+ // Chase the forwarding address if requested.
+ if (followForward) {
+ Address forwardingAddr =
+ Builder.CreateStructGEP(baseAddr, 1, getPointerSize(), "forwarding");
+ baseAddr = Address(Builder.CreateLoad(forwardingAddr), info.ByrefAlignment);
+ }
+
+ return Builder.CreateStructGEP(baseAddr, info.FieldIndex,
+ info.FieldOffset, name);
}
-/// BuildByRefType - This routine changes a __block variable declared as T x
+/// BuildByrefInfo - This routine changes a __block variable declared as T x
/// into:
///
/// struct {
@@ -2033,108 +2052,116 @@
/// T x;
/// } x
///
-llvm::Type *CodeGenFunction::BuildByRefType(const VarDecl *D) {
- std::pair<llvm::Type *, unsigned> &Info = ByRefValueInfo[D];
- if (Info.first)
- return Info.first;
-
- QualType Ty = D->getType();
+const BlockByrefInfo &CodeGenFunction::getBlockByrefInfo(const VarDecl *D) {
+ auto it = BlockByrefInfos.find(D);
+ if (it != BlockByrefInfos.end())
+ return it->second;
- SmallVector<llvm::Type *, 8> types;
-
- llvm::StructType *ByRefType =
+ llvm::StructType *byrefType =
llvm::StructType::create(getLLVMContext(),
"struct.__block_byref_" + D->getNameAsString());
+ QualType Ty = D->getType();
+
+ CharUnits size;
+ SmallVector<llvm::Type *, 8> types;
+
// void *__isa;
types.push_back(Int8PtrTy);
+ size += getPointerSize();
// void *__forwarding;
- types.push_back(llvm::PointerType::getUnqual(ByRefType));
+ types.push_back(llvm::PointerType::getUnqual(byrefType));
+ size += getPointerSize();
// int32_t __flags;
types.push_back(Int32Ty);
+ size += CharUnits::fromQuantity(4);
// int32_t __size;
types.push_back(Int32Ty);
+ size += CharUnits::fromQuantity(4);
+
// Note that this must match *exactly* the logic in buildByrefHelpers.
- bool HasCopyAndDispose = getContext().BlockRequiresCopying(Ty, D);
- if (HasCopyAndDispose) {
+ bool hasCopyAndDispose = getContext().BlockRequiresCopying(Ty, D);
+ if (hasCopyAndDispose) {
/// void *__copy_helper;
types.push_back(Int8PtrTy);
+ size += getPointerSize();
/// void *__destroy_helper;
types.push_back(Int8PtrTy);
+ size += getPointerSize();
}
+
bool HasByrefExtendedLayout = false;
Qualifiers::ObjCLifetime Lifetime;
if (getContext().getByrefLifetime(Ty, Lifetime, HasByrefExtendedLayout) &&
- HasByrefExtendedLayout)
+ HasByrefExtendedLayout) {
/// void *__byref_variable_layout;
types.push_back(Int8PtrTy);
-
- bool Packed = false;
- CharUnits Align = getContext().getDeclAlign(D);
- if (Align >
- getContext().toCharUnitsFromBits(getTarget().getPointerAlign(0))) {
- // We have to insert padding.
-
- // The struct above has 2 32-bit integers.
- unsigned CurrentOffsetInBytes = 4 * 2;
-
- // And either 2, 3, 4 or 5 pointers.
- unsigned noPointers = 2;
- if (HasCopyAndDispose)
- noPointers += 2;
- if (HasByrefExtendedLayout)
- noPointers += 1;
-
- CurrentOffsetInBytes += noPointers * CGM.getDataLayout().getTypeAllocSize(Int8PtrTy);
-
- // Align the offset.
- unsigned AlignedOffsetInBytes =
- llvm::RoundUpToAlignment(CurrentOffsetInBytes, Align.getQuantity());
-
- unsigned NumPaddingBytes = AlignedOffsetInBytes - CurrentOffsetInBytes;
- if (NumPaddingBytes > 0) {
- llvm::Type *Ty = Int8Ty;
- // FIXME: We need a sema error for alignment larger than the minimum of
- // the maximal stack alignment and the alignment of malloc on the system.
- if (NumPaddingBytes > 1)
- Ty = llvm::ArrayType::get(Ty, NumPaddingBytes);
-
- types.push_back(Ty);
-
- // We want a packed struct.
- Packed = true;
- }
+ size += CharUnits::fromQuantity(PointerSizeInBytes);
}
// T x;
- types.push_back(ConvertTypeForMem(Ty));
-
- ByRefType->setBody(types, Packed);
-
- Info.first = ByRefType;
-
- Info.second = types.size() - 1;
-
- return Info.first;
+ llvm::Type *varTy = ConvertTypeForMem(Ty);
+
+ bool packed = false;
+ CharUnits varAlign = getContext().getDeclAlign(D);
+ CharUnits varOffset = size.RoundUpToAlignment(varAlign);
+
+ // We may have to insert padding.
+ if (varOffset != size) {
+ llvm::Type *paddingTy =
+ llvm::ArrayType::get(Int8Ty, (varOffset - size).getQuantity());
+
+ types.push_back(paddingTy);
+ size = varOffset;
+
+ // Conversely, we might have to prevent LLVM from inserting padding.
+ } else if (CGM.getDataLayout().getABITypeAlignment(varTy)
+ > varAlign.getQuantity()) {
+ packed = true;
+ }
+ types.push_back(varTy);
+
+ byrefType->setBody(types, packed);
+
+ BlockByrefInfo info;
+ info.Type = byrefType;
+ info.FieldIndex = types.size() - 1;
+ info.FieldOffset = varOffset;
+ info.ByrefAlignment = std::max(varAlign, getPointerAlign());
+
+ auto pair = BlockByrefInfos.insert({D, info});
+ assert(pair.second && "info was inserted recursively?");
+ return pair.first->second;
}
/// Initialize the structural components of a __block variable, i.e.
/// everything but the actual object.
void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) {
// Find the address of the local.
- llvm::Value *addr = emission.Address;
+ Address addr = emission.Addr;
// That's an alloca of the byref structure type.
llvm::StructType *byrefType = cast<llvm::StructType>(
- cast<llvm::PointerType>(addr->getType())->getElementType());
+ cast<llvm::PointerType>(addr.getPointer()->getType())->getElementType());
+
+ unsigned nextHeaderIndex = 0;
+ CharUnits nextHeaderOffset;
+ auto storeHeaderField = [&](llvm::Value *value, CharUnits fieldSize,
+ const Twine &name) {
+ auto fieldAddr = Builder.CreateStructGEP(addr, nextHeaderIndex,
+ nextHeaderOffset, name);
+ Builder.CreateStore(value, fieldAddr);
+
+ nextHeaderIndex++;
+ nextHeaderOffset += fieldSize;
+ };
// Build the byref helpers if necessary. This is null if we don't need any.
- CodeGenModule::ByrefHelpers *helpers =
- buildByrefHelpers(*byrefType, emission);
+ BlockByrefHelpers *helpers = buildByrefHelpers(*byrefType, emission);
const VarDecl &D = *emission.Variable;
QualType type = D.getType();
@@ -2143,7 +2170,7 @@
Qualifiers::ObjCLifetime ByrefLifetime;
bool ByRefHasLifetime =
getContext().getByrefLifetime(type, ByrefLifetime, HasByrefExtendedLayout);
-
+
llvm::Value *V;
// Initialize the 'isa', which is just 0 or 1.
@@ -2151,12 +2178,10 @@
if (type.isObjCGCWeak())
isa = 1;
V = Builder.CreateIntToPtr(Builder.getInt32(isa), Int8PtrTy, "isa");
- Builder.CreateStore(V,
- Builder.CreateStructGEP(nullptr, addr, 0, "byref.isa"));
+ storeHeaderField(V, getPointerSize(), "byref.isa");
// Store the address of the variable into its own forwarding pointer.
- Builder.CreateStore(
- addr, Builder.CreateStructGEP(nullptr, addr, 1, "byref.forwarding"));
+ storeHeaderField(addr.getPointer(), getPointerSize(), "byref.forwarding");
// Blocks ABI:
// c) the flags field is set to either 0 if no helper functions are
@@ -2202,31 +2227,23 @@
printf("\n");
}
}
-
- Builder.CreateStore(llvm::ConstantInt::get(IntTy, flags.getBitMask()),
- Builder.CreateStructGEP(nullptr, addr, 2, "byref.flags"));
+ storeHeaderField(llvm::ConstantInt::get(IntTy, flags.getBitMask()),
+ getIntSize(), "byref.flags");
CharUnits byrefSize = CGM.GetTargetTypeStoreSize(byrefType);
V = llvm::ConstantInt::get(IntTy, byrefSize.getQuantity());
- Builder.CreateStore(V,
- Builder.CreateStructGEP(nullptr, addr, 3, "byref.size"));
+ storeHeaderField(V, getIntSize(), "byref.size");
if (helpers) {
- llvm::Value *copy_helper = Builder.CreateStructGEP(nullptr, addr, 4);
- Builder.CreateStore(helpers->CopyHelper, copy_helper);
-
- llvm::Value *destroy_helper = Builder.CreateStructGEP(nullptr, addr, 5);
- Builder.CreateStore(helpers->DisposeHelper, destroy_helper);
+ storeHeaderField(helpers->CopyHelper, getPointerSize(),
+ "byref.copyHelper");
+ storeHeaderField(helpers->DisposeHelper, getPointerSize(),
+ "byref.disposeHelper");
}
+
if (ByRefHasLifetime && HasByrefExtendedLayout) {
- llvm::Constant* ByrefLayoutInfo = CGM.getObjCRuntime().BuildByrefLayout(CGM, type);
- llvm::Value *ByrefInfoAddr =
- Builder.CreateStructGEP(nullptr, addr, helpers ? 6 : 4, "byref.layout");
- // cast destination to pointer to source type.
- llvm::Type *DesTy = ByrefLayoutInfo->getType();
- DesTy = DesTy->getPointerTo();
- llvm::Value *BC = Builder.CreatePointerCast(ByrefInfoAddr, DesTy);
- Builder.CreateStore(ByrefLayoutInfo, BC);
+ auto layoutInfo = CGM.getObjCRuntime().BuildByrefLayout(CGM, type);
+ storeHeaderField(layoutInfo, getPointerSize(), "byref.layout");
}
}
@@ -2240,7 +2257,8 @@
}
namespace {
- struct CallBlockRelease : EHScopeStack::Cleanup {
+ /// Release a __block variable.
+ struct CallBlockRelease final : EHScopeStack::Cleanup {
llvm::Value *Addr;
CallBlockRelease(llvm::Value *Addr) : Addr(Addr) {}
@@ -2249,7 +2267,7 @@
CGF.BuildBlockRelease(Addr, BLOCK_FIELD_IS_BYREF);
}
};
-}
+} // end anonymous namespace
/// Enter a cleanup to destroy a __block variable. Note that this
/// cleanup should be a no-op if the variable hasn't left the stack
@@ -2260,7 +2278,8 @@
if (CGM.getLangOpts().getGC() == LangOptions::GCOnly)
return;
- EHStack.pushCleanup<CallBlockRelease>(NormalAndEHCleanup, emission.Address);
+ EHStack.pushCleanup<CallBlockRelease>(NormalAndEHCleanup,
+ emission.Addr.getPointer());
}
/// Adjust the declaration of something from the blocks API.
diff --git a/lib/CodeGen/CGBlocks.h b/lib/CodeGen/CGBlocks.h
index c4eed0d..1edabef 100644
--- a/lib/CodeGen/CGBlocks.h
+++ b/lib/CodeGen/CGBlocks.h
@@ -140,6 +140,15 @@
return BlockFieldFlags(l) | BlockFieldFlags(r);
}
+/// Information about the layout of a __block variable.
+class BlockByrefInfo {
+public:
+ llvm::StructType *Type;
+ unsigned FieldIndex;
+ CharUnits ByrefAlignment;
+ CharUnits FieldOffset;
+};
+
/// CGBlockInfo - Information to generate a block literal.
class CGBlockInfo {
public:
@@ -152,14 +161,19 @@
class Capture {
uintptr_t Data;
EHScopeStack::stable_iterator Cleanup;
+ CharUnits::QuantityType Offset;
public:
bool isIndex() const { return (Data & 1) != 0; }
bool isConstant() const { return !isIndex(); }
- unsigned getIndex() const { assert(isIndex()); return Data >> 1; }
- llvm::Value *getConstant() const {
- assert(isConstant());
- return reinterpret_cast<llvm::Value*>(Data);
+
+ unsigned getIndex() const {
+ assert(isIndex());
+ return Data >> 1;
+ }
+ CharUnits getOffset() const {
+ assert(isIndex());
+ return CharUnits::fromQuantity(Offset);
}
EHScopeStack::stable_iterator getCleanup() const {
assert(isIndex());
@@ -170,9 +184,15 @@
Cleanup = cleanup;
}
- static Capture makeIndex(unsigned index) {
+ llvm::Value *getConstant() const {
+ assert(isConstant());
+ return reinterpret_cast<llvm::Value*>(Data);
+ }
+
+ static Capture makeIndex(unsigned index, CharUnits offset) {
Capture v;
v.Data = (index << 1) | 1;
+ v.Offset = offset.getQuantity();
return v;
}
@@ -205,12 +225,13 @@
/// The mapping of allocated indexes within the block.
llvm::DenseMap<const VarDecl*, Capture> Captures;
- llvm::AllocaInst *Address;
+ Address LocalAddress;
llvm::StructType *StructureType;
const BlockDecl *Block;
const BlockExpr *BlockExpression;
CharUnits BlockSize;
CharUnits BlockAlign;
+ CharUnits CXXThisOffset;
// Offset of the gap caused by block header having a smaller
// alignment than the alignment of the block descriptor. This
diff --git a/lib/CodeGen/CGBuilder.h b/lib/CodeGen/CGBuilder.h
index 6610659..489f341 100644
--- a/lib/CodeGen/CGBuilder.h
+++ b/lib/CodeGen/CGBuilder.h
@@ -11,6 +11,8 @@
#define LLVM_CLANG_LIB_CODEGEN_CGBUILDER_H
#include "llvm/IR/IRBuilder.h"
+#include "Address.h"
+#include "CodeGenTypeCache.h"
namespace clang {
namespace CodeGen {
@@ -22,9 +24,9 @@
/// instructions.
template <bool PreserveNames>
class CGBuilderInserter
- : protected llvm::IRBuilderDefaultInserter<PreserveNames> {
+ : protected llvm::IRBuilderDefaultInserter<PreserveNames> {
public:
- CGBuilderInserter() : CGF(nullptr) {}
+ CGBuilderInserter() = default;
explicit CGBuilderInserter(CodeGenFunction *CGF) : CGF(CGF) {}
protected:
@@ -33,9 +35,7 @@
llvm::BasicBlock *BB,
llvm::BasicBlock::iterator InsertPt) const;
private:
- void operator=(const CGBuilderInserter &) = delete;
-
- CodeGenFunction *CGF;
+ CodeGenFunction *CGF = nullptr;
};
// Don't preserve names on values in an optimized build.
@@ -44,9 +44,260 @@
#else
#define PreserveNames true
#endif
+
typedef CGBuilderInserter<PreserveNames> CGBuilderInserterTy;
+
typedef llvm::IRBuilder<PreserveNames, llvm::ConstantFolder,
- CGBuilderInserterTy> CGBuilderTy;
+ CGBuilderInserterTy> CGBuilderBaseTy;
+
+class CGBuilderTy : public CGBuilderBaseTy {
+ /// Storing a reference to the type cache here makes it a lot easier
+ /// to build natural-feeling, target-specific IR.
+ const CodeGenTypeCache &TypeCache;
+public:
+ CGBuilderTy(const CodeGenTypeCache &TypeCache, llvm::LLVMContext &C)
+ : CGBuilderBaseTy(C), TypeCache(TypeCache) {}
+ CGBuilderTy(const CodeGenTypeCache &TypeCache,
+ llvm::LLVMContext &C, const llvm::ConstantFolder &F,
+ const CGBuilderInserterTy &Inserter)
+ : CGBuilderBaseTy(C, F, Inserter), TypeCache(TypeCache) {}
+ CGBuilderTy(const CodeGenTypeCache &TypeCache, llvm::Instruction *I)
+ : CGBuilderBaseTy(I), TypeCache(TypeCache) {}
+ CGBuilderTy(const CodeGenTypeCache &TypeCache, llvm::BasicBlock *BB)
+ : CGBuilderBaseTy(BB), TypeCache(TypeCache) {}
+
+ llvm::ConstantInt *getSize(CharUnits N) {
+ return llvm::ConstantInt::get(TypeCache.SizeTy, N.getQuantity());
+ }
+ llvm::ConstantInt *getSize(uint64_t N) {
+ return llvm::ConstantInt::get(TypeCache.SizeTy, N);
+ }
+
+ // Note that we intentionally hide the CreateLoad APIs that don't
+ // take an alignment.
+ llvm::LoadInst *CreateLoad(Address Addr, const llvm::Twine &Name = "") {
+ return CreateAlignedLoad(Addr.getPointer(),
+ Addr.getAlignment().getQuantity(),
+ Name);
+ }
+ llvm::LoadInst *CreateLoad(Address Addr, const char *Name) {
+ // This overload is required to prevent string literals from
+ // ending up in the IsVolatile overload.
+ return CreateAlignedLoad(Addr.getPointer(),
+ Addr.getAlignment().getQuantity(),
+ Name);
+ }
+ llvm::LoadInst *CreateLoad(Address Addr, bool IsVolatile,
+ const llvm::Twine &Name = "") {
+ return CreateAlignedLoad(Addr.getPointer(),
+ Addr.getAlignment().getQuantity(),
+ IsVolatile,
+ Name);
+ }
+
+ using CGBuilderBaseTy::CreateAlignedLoad;
+ llvm::LoadInst *CreateAlignedLoad(llvm::Value *Addr, CharUnits Align,
+ const llvm::Twine &Name = "") {
+ return CreateAlignedLoad(Addr, Align.getQuantity(), Name);
+ }
+ llvm::LoadInst *CreateAlignedLoad(llvm::Value *Addr, CharUnits Align,
+ const char *Name) {
+ return CreateAlignedLoad(Addr, Align.getQuantity(), Name);
+ }
+ llvm::LoadInst *CreateAlignedLoad(llvm::Type *Ty, llvm::Value *Addr,
+ CharUnits Align,
+ const llvm::Twine &Name = "") {
+ assert(Addr->getType()->getPointerElementType() == Ty);
+ return CreateAlignedLoad(Addr, Align.getQuantity(), Name);
+ }
+ llvm::LoadInst *CreateAlignedLoad(llvm::Value *Addr, CharUnits Align,
+ bool IsVolatile,
+ const llvm::Twine &Name = "") {
+ return CreateAlignedLoad(Addr, Align.getQuantity(), IsVolatile, Name);
+ }
+
+ // Note that we intentionally hide the CreateStore APIs that don't
+ // take an alignment.
+ llvm::StoreInst *CreateStore(llvm::Value *Val, Address Addr,
+ bool IsVolatile = false) {
+ return CreateAlignedStore(Val, Addr.getPointer(),
+ Addr.getAlignment().getQuantity(), IsVolatile);
+ }
+
+ using CGBuilderBaseTy::CreateAlignedStore;
+ llvm::StoreInst *CreateAlignedStore(llvm::Value *Val, llvm::Value *Addr,
+ CharUnits Align, bool IsVolatile = false) {
+ return CreateAlignedStore(Val, Addr, Align.getQuantity(), IsVolatile);
+ }
+
+ // FIXME: these "default-aligned" APIs should be removed,
+ // but I don't feel like fixing all the builtin code right now.
+ llvm::LoadInst *CreateDefaultAlignedLoad(llvm::Value *Addr,
+ const llvm::Twine &Name = "") {
+ return CGBuilderBaseTy::CreateLoad(Addr, false, Name);
+ }
+ llvm::LoadInst *CreateDefaultAlignedLoad(llvm::Value *Addr,
+ const char *Name) {
+ return CGBuilderBaseTy::CreateLoad(Addr, false, Name);
+ }
+ llvm::LoadInst *CreateDefaultAlignedLoad(llvm::Value *Addr, bool IsVolatile,
+ const llvm::Twine &Name = "") {
+ return CGBuilderBaseTy::CreateLoad(Addr, IsVolatile, Name);
+ }
+
+ llvm::StoreInst *CreateDefaultAlignedStore(llvm::Value *Val,
+ llvm::Value *Addr,
+ bool IsVolatile = false) {
+ return CGBuilderBaseTy::CreateStore(Val, Addr, IsVolatile);
+ }
+
+ /// Emit a load from an i1 flag variable.
+ llvm::LoadInst *CreateFlagLoad(llvm::Value *Addr,
+ const llvm::Twine &Name = "") {
+ assert(Addr->getType()->getPointerElementType() == getInt1Ty());
+ return CreateAlignedLoad(getInt1Ty(), Addr, CharUnits::One(), Name);
+ }
+
+ /// Emit a store to an i1 flag variable.
+ llvm::StoreInst *CreateFlagStore(bool Value, llvm::Value *Addr) {
+ assert(Addr->getType()->getPointerElementType() == getInt1Ty());
+ return CreateAlignedStore(getInt1(Value), Addr, CharUnits::One());
+ }
+
+ using CGBuilderBaseTy::CreateBitCast;
+ Address CreateBitCast(Address Addr, llvm::Type *Ty,
+ const llvm::Twine &Name = "") {
+ return Address(CreateBitCast(Addr.getPointer(), Ty, Name),
+ Addr.getAlignment());
+ }
+
+ /// Cast the element type of the given address to a different type,
+ /// preserving information like the alignment and address space.
+ Address CreateElementBitCast(Address Addr, llvm::Type *Ty,
+ const llvm::Twine &Name = "") {
+ auto PtrTy = Ty->getPointerTo(Addr.getAddressSpace());
+ return CreateBitCast(Addr, PtrTy, Name);
+ }
+
+ using CGBuilderBaseTy::CreatePointerBitCastOrAddrSpaceCast;
+ Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty,
+ const llvm::Twine &Name = "") {
+ llvm::Value *Ptr =
+ CreatePointerBitCastOrAddrSpaceCast(Addr.getPointer(), Ty, Name);
+ return Address(Ptr, Addr.getAlignment());
+ }
+
+ using CGBuilderBaseTy::CreateStructGEP;
+ Address CreateStructGEP(Address Addr, unsigned Index, CharUnits Offset,
+ const llvm::Twine &Name = "") {
+ return Address(CreateStructGEP(Addr.getElementType(),
+ Addr.getPointer(), Index, Name),
+ Addr.getAlignment().alignmentAtOffset(Offset));
+ }
+
+ /// Given
+ /// %addr = [n x T]* ...
+ /// produce
+ /// %name = getelementptr inbounds %addr, i64 0, i64 index
+ /// where i64 is actually the target word size.
+ ///
+ /// This API assumes that drilling into an array like this is always
+ /// an inbounds operation.
+ ///
+ /// \param EltSize - the size of the type T in bytes
+ Address CreateConstArrayGEP(Address Addr, uint64_t Index, CharUnits EltSize,
+ const llvm::Twine &Name = "") {
+ return Address(CreateInBoundsGEP(Addr.getPointer(),
+ {getSize(CharUnits::Zero()),
+ getSize(Index)},
+ Name),
+ Addr.getAlignment().alignmentAtOffset(Index * EltSize));
+ }
+
+ /// Given
+ /// %addr = T* ...
+ /// produce
+ /// %name = getelementptr inbounds %addr, i64 index
+ /// where i64 is actually the target word size.
+ ///
+ /// \param EltSize - the size of the type T in bytes
+ Address CreateConstInBoundsGEP(Address Addr, uint64_t Index,
+ CharUnits EltSize,
+ const llvm::Twine &Name = "") {
+ return Address(CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(),
+ getSize(Index), Name),
+ Addr.getAlignment().alignmentAtOffset(Index * EltSize));
+ }
+
+ /// Given
+ /// %addr = T* ...
+ /// produce
+ /// %name = getelementptr inbounds %addr, i64 index
+ /// where i64 is actually the target word size.
+ ///
+ /// \param EltSize - the size of the type T in bytes
+ Address CreateConstGEP(Address Addr, uint64_t Index, CharUnits EltSize,
+ const llvm::Twine &Name = "") {
+ return Address(CreateGEP(Addr.getElementType(), Addr.getPointer(),
+ getSize(Index), Name),
+ Addr.getAlignment().alignmentAtOffset(Index * EltSize));
+ }
+
+ /// Given a pointer to i8, adjust it by a given constant offset.
+ Address CreateConstInBoundsByteGEP(Address Addr, CharUnits Offset,
+ const llvm::Twine &Name = "") {
+ assert(Addr.getElementType() == TypeCache.Int8Ty);
+ return Address(CreateInBoundsGEP(Addr.getPointer(), getSize(Offset), Name),
+ Addr.getAlignment().alignmentAtOffset(Offset));
+ }
+ Address CreateConstByteGEP(Address Addr, CharUnits Offset,
+ const llvm::Twine &Name = "") {
+ assert(Addr.getElementType() == TypeCache.Int8Ty);
+ return Address(CreateGEP(Addr.getPointer(), getSize(Offset), Name),
+ Addr.getAlignment().alignmentAtOffset(Offset));
+ }
+
+ llvm::Value *CreateConstInBoundsByteGEP(llvm::Value *Ptr, CharUnits Offset,
+ const llvm::Twine &Name = "") {
+ assert(Ptr->getType()->getPointerElementType() == TypeCache.Int8Ty);
+ return CreateInBoundsGEP(Ptr, getSize(Offset), Name);
+ }
+ llvm::Value *CreateConstByteGEP(llvm::Value *Ptr, CharUnits Offset,
+ const llvm::Twine &Name = "") {
+ assert(Ptr->getType()->getPointerElementType() == TypeCache.Int8Ty);
+ return CreateGEP(Ptr, getSize(Offset), Name);
+ }
+
+ using CGBuilderBaseTy::CreateMemCpy;
+ llvm::CallInst *CreateMemCpy(Address Dest, Address Src, llvm::Value *Size,
+ bool IsVolatile = false) {
+ auto Align = std::min(Dest.getAlignment(), Src.getAlignment());
+ return CreateMemCpy(Dest.getPointer(), Src.getPointer(), Size,
+ Align.getQuantity(), IsVolatile);
+ }
+ llvm::CallInst *CreateMemCpy(Address Dest, Address Src, uint64_t Size,
+ bool IsVolatile = false) {
+ auto Align = std::min(Dest.getAlignment(), Src.getAlignment());
+ return CreateMemCpy(Dest.getPointer(), Src.getPointer(), Size,
+ Align.getQuantity(), IsVolatile);
+ }
+
+ using CGBuilderBaseTy::CreateMemMove;
+ llvm::CallInst *CreateMemMove(Address Dest, Address Src, llvm::Value *Size,
+ bool IsVolatile = false) {
+ auto Align = std::min(Dest.getAlignment(), Src.getAlignment());
+ return CreateMemMove(Dest.getPointer(), Src.getPointer(), Size,
+ Align.getQuantity(), IsVolatile);
+ }
+
+ using CGBuilderBaseTy::CreateMemSet;
+ llvm::CallInst *CreateMemSet(Address Dest, llvm::Value *Value,
+ llvm::Value *Size, bool IsVolatile = false) {
+ return CreateMemSet(Dest.getPointer(), Value, Size,
+ Dest.getAlignment().getQuantity(), IsVolatile);
+ }
+};
+
#undef PreserveNames
} // end namespace CodeGen
diff --git a/lib/CodeGen/CGBuiltin.cpp b/lib/CodeGen/CGBuiltin.cpp
index 2b9631d..787ac53 100644
--- a/lib/CodeGen/CGBuiltin.cpp
+++ b/lib/CodeGen/CGBuiltin.cpp
@@ -48,7 +48,7 @@
if (FD->hasAttr<AsmLabelAttr>())
Name = getMangledName(D);
else
- Name = Context.BuiltinInfo.GetName(BuiltinID) + 10;
+ Name = Context.BuiltinInfo.getName(BuiltinID) + 10;
llvm::FunctionType *Ty =
cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType()));
@@ -82,9 +82,9 @@
/// Utility to insert an atomic instruction based on Instrinsic::ID
/// and the expression node.
-static RValue EmitBinaryAtomic(CodeGenFunction &CGF,
- llvm::AtomicRMWInst::BinOp Kind,
- const CallExpr *E) {
+static Value *MakeBinaryAtomicValue(CodeGenFunction &CGF,
+ llvm::AtomicRMWInst::BinOp Kind,
+ const CallExpr *E) {
QualType T = E->getType();
assert(E->getArg(0)->getType()->isPointerType());
assert(CGF.getContext().hasSameUnqualifiedType(T,
@@ -108,8 +108,35 @@
llvm::Value *Result =
CGF.Builder.CreateAtomicRMW(Kind, Args[0], Args[1],
llvm::SequentiallyConsistent);
- Result = EmitFromInt(CGF, Result, T, ValueType);
- return RValue::get(Result);
+ return EmitFromInt(CGF, Result, T, ValueType);
+}
+
+static Value *EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E) {
+ Value *Val = CGF.EmitScalarExpr(E->getArg(0));
+ Value *Address = CGF.EmitScalarExpr(E->getArg(1));
+
+ // Convert the type of the pointer to a pointer to the stored type.
+ Val = CGF.EmitToMemory(Val, E->getArg(0)->getType());
+ Value *BC = CGF.Builder.CreateBitCast(
+ Address, llvm::PointerType::getUnqual(Val->getType()), "cast");
+ LValue LV = CGF.MakeNaturalAlignAddrLValue(BC, E->getArg(0)->getType());
+ LV.setNontemporal(true);
+ CGF.EmitStoreOfScalar(Val, LV, false);
+ return nullptr;
+}
+
+static Value *EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E) {
+ Value *Address = CGF.EmitScalarExpr(E->getArg(0));
+
+ LValue LV = CGF.MakeNaturalAlignAddrLValue(Address, E->getType());
+ LV.setNontemporal(true);
+ return CGF.EmitLoadOfScalar(LV, E->getExprLoc());
+}
+
+static RValue EmitBinaryAtomic(CodeGenFunction &CGF,
+ llvm::AtomicRMWInst::BinOp Kind,
+ const CallExpr *E) {
+ return RValue::get(MakeBinaryAtomicValue(CGF, Kind, E));
}
/// Utility to insert an atomic instruction based Instrinsic::ID and
@@ -151,6 +178,47 @@
return RValue::get(Result);
}
+/// @brief Utility to insert an atomic cmpxchg instruction.
+///
+/// @param CGF The current codegen function.
+/// @param E Builtin call expression to convert to cmpxchg.
+/// arg0 - address to operate on
+/// arg1 - value to compare with
+/// arg2 - new value
+/// @param ReturnBool Specifies whether to return success flag of
+/// cmpxchg result or the old value.
+///
+/// @returns result of cmpxchg, according to ReturnBool
+static Value *MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E,
+ bool ReturnBool) {
+ QualType T = ReturnBool ? E->getArg(1)->getType() : E->getType();
+ llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
+ unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
+
+ llvm::IntegerType *IntType = llvm::IntegerType::get(
+ CGF.getLLVMContext(), CGF.getContext().getTypeSize(T));
+ llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
+
+ Value *Args[3];
+ Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
+ Args[1] = CGF.EmitScalarExpr(E->getArg(1));
+ llvm::Type *ValueType = Args[1]->getType();
+ Args[1] = EmitToInt(CGF, Args[1], T, IntType);
+ Args[2] = EmitToInt(CGF, CGF.EmitScalarExpr(E->getArg(2)), T, IntType);
+
+ Value *Pair = CGF.Builder.CreateAtomicCmpXchg(Args[0], Args[1], Args[2],
+ llvm::SequentiallyConsistent,
+ llvm::SequentiallyConsistent);
+ if (ReturnBool)
+ // Extract boolean success flag and zext it to int.
+ return CGF.Builder.CreateZExt(CGF.Builder.CreateExtractValue(Pair, 1),
+ CGF.ConvertType(E->getType()));
+ else
+ // Extract old value and emit it using the same type as compare value.
+ return EmitFromInt(CGF, CGF.Builder.CreateExtractValue(Pair, 0), T,
+ ValueType);
+}
+
/// EmitFAbs - Emit a call to @llvm.fabs().
static Value *EmitFAbs(CodeGenFunction &CGF, Value *V) {
Value *F = CGF.CGM.getIntrinsic(Intrinsic::fabs, V->getType());
@@ -169,10 +237,20 @@
llvm::Type *IntTy = llvm::IntegerType::get(C, Width);
V = CGF.Builder.CreateBitCast(V, IntTy);
if (Ty->isPPC_FP128Ty()) {
- // The higher-order double comes first, and so we need to truncate the
- // pair to extract the overall sign. The order of the pair is the same
- // in both little- and big-Endian modes.
+ // We want the sign bit of the higher-order double. The bitcast we just
+ // did works as if the double-double was stored to memory and then
+ // read as an i128. The "store" will put the higher-order double in the
+ // lower address in both little- and big-Endian modes, but the "load"
+ // will treat those bits as a different part of the i128: the low bits in
+ // little-Endian, the high bits in big-Endian. Therefore, on big-Endian
+ // we need to shift the high bits down to the low before truncating.
Width >>= 1;
+ if (CGF.getTarget().isBigEndian()) {
+ Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
+ V = CGF.Builder.CreateLShr(V, ShiftCst);
+ }
+ // We are truncating value in order to extract the higher-order
+ // double, which we will be using to extract the sign from.
IntTy = llvm::IntegerType::get(C, Width);
V = CGF.Builder.CreateTrunc(V, IntTy);
}
@@ -210,6 +288,125 @@
return CGF.Builder.CreateExtractValue(Tmp, 0);
}
+namespace {
+ struct WidthAndSignedness {
+ unsigned Width;
+ bool Signed;
+ };
+}
+
+static WidthAndSignedness
+getIntegerWidthAndSignedness(const clang::ASTContext &context,
+ const clang::QualType Type) {
+ assert(Type->isIntegerType() && "Given type is not an integer.");
+ unsigned Width = Type->isBooleanType() ? 1 : context.getTypeInfo(Type).Width;
+ bool Signed = Type->isSignedIntegerType();
+ return {Width, Signed};
+}
+
+// Given one or more integer types, this function produces an integer type that
+// encompasses them: any value in one of the given types could be expressed in
+// the encompassing type.
+static struct WidthAndSignedness
+EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> Types) {
+ assert(Types.size() > 0 && "Empty list of types.");
+
+ // If any of the given types is signed, we must return a signed type.
+ bool Signed = false;
+ for (const auto &Type : Types) {
+ Signed |= Type.Signed;
+ }
+
+ // The encompassing type must have a width greater than or equal to the width
+ // of the specified types. Aditionally, if the encompassing type is signed,
+ // its width must be strictly greater than the width of any unsigned types
+ // given.
+ unsigned Width = 0;
+ for (const auto &Type : Types) {
+ unsigned MinWidth = Type.Width + (Signed && !Type.Signed);
+ if (Width < MinWidth) {
+ Width = MinWidth;
+ }
+ }
+
+ return {Width, Signed};
+}
+
+Value *CodeGenFunction::EmitVAStartEnd(Value *ArgValue, bool IsStart) {
+ llvm::Type *DestType = Int8PtrTy;
+ if (ArgValue->getType() != DestType)
+ ArgValue =
+ Builder.CreateBitCast(ArgValue, DestType, ArgValue->getName().data());
+
+ Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
+ return Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue);
+}
+
+/// Checks if using the result of __builtin_object_size(p, @p From) in place of
+/// __builtin_object_size(p, @p To) is correct
+static bool areBOSTypesCompatible(int From, int To) {
+ // Note: Our __builtin_object_size implementation currently treats Type=0 and
+ // Type=2 identically. Encoding this implementation detail here may make
+ // improving __builtin_object_size difficult in the future, so it's omitted.
+ return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
+}
+
+static llvm::Value *
+getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType) {
+ return ConstantInt::get(ResType, (Type & 2) ? 0 : -1, /*isSigned=*/true);
+}
+
+llvm::Value *
+CodeGenFunction::evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
+ llvm::IntegerType *ResType) {
+ uint64_t ObjectSize;
+ if (!E->tryEvaluateObjectSize(ObjectSize, getContext(), Type))
+ return emitBuiltinObjectSize(E, Type, ResType);
+ return ConstantInt::get(ResType, ObjectSize, /*isSigned=*/true);
+}
+
+/// Returns a Value corresponding to the size of the given expression.
+/// This Value may be either of the following:
+/// - A llvm::Argument (if E is a param with the pass_object_size attribute on
+/// it)
+/// - A call to the @llvm.objectsize intrinsic
+llvm::Value *
+CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type,
+ llvm::IntegerType *ResType) {
+ // We need to reference an argument if the pointer is a parameter with the
+ // pass_object_size attribute.
+ if (auto *D = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts())) {
+ auto *Param = dyn_cast<ParmVarDecl>(D->getDecl());
+ auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();
+ if (Param != nullptr && PS != nullptr &&
+ areBOSTypesCompatible(PS->getType(), Type)) {
+ auto Iter = SizeArguments.find(Param);
+ assert(Iter != SizeArguments.end());
+
+ const ImplicitParamDecl *D = Iter->second;
+ auto DIter = LocalDeclMap.find(D);
+ assert(DIter != LocalDeclMap.end());
+
+ return EmitLoadOfScalar(DIter->second, /*volatile=*/false,
+ getContext().getSizeType(), E->getLocStart());
+ }
+ }
+
+ // LLVM can't handle Type=3 appropriately, and __builtin_object_size shouldn't
+ // evaluate E for side-effects. In either case, we shouldn't lower to
+ // @llvm.objectsize.
+ if (Type == 3 || E->HasSideEffects(getContext()))
+ return getDefaultBuiltinObjectSizeResult(Type, ResType);
+
+ // LLVM only supports 0 and 2, make sure that we pass along that
+ // as a boolean.
+ auto *CI = ConstantInt::get(Builder.getInt1Ty(), (Type & 2) >> 1);
+ // FIXME: Get right address space.
+ llvm::Type *Tys[] = {ResType, Builder.getInt8PtrTy(0)};
+ Value *F = CGM.getIntrinsic(Intrinsic::objectsize, Tys);
+ return Builder.CreateCall(F, {EmitScalarExpr(E), CI});
+}
+
RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
unsigned BuiltinID, const CallExpr *E,
ReturnValueSlot ReturnValue) {
@@ -233,22 +430,15 @@
case Builtin::BI__builtin_stdarg_start:
case Builtin::BI__builtin_va_start:
case Builtin::BI__va_start:
- case Builtin::BI__builtin_va_end: {
- Value *ArgValue = (BuiltinID == Builtin::BI__va_start)
- ? EmitScalarExpr(E->getArg(0))
- : EmitVAListRef(E->getArg(0));
- llvm::Type *DestType = Int8PtrTy;
- if (ArgValue->getType() != DestType)
- ArgValue = Builder.CreateBitCast(ArgValue, DestType,
- ArgValue->getName().data());
-
- Intrinsic::ID inst = (BuiltinID == Builtin::BI__builtin_va_end) ?
- Intrinsic::vaend : Intrinsic::vastart;
- return RValue::get(Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue));
- }
+ case Builtin::BI__builtin_va_end:
+ return RValue::get(
+ EmitVAStartEnd(BuiltinID == Builtin::BI__va_start
+ ? EmitScalarExpr(E->getArg(0))
+ : EmitVAListRef(E->getArg(0)).getPointer(),
+ BuiltinID != Builtin::BI__builtin_va_end));
case Builtin::BI__builtin_va_copy: {
- Value *DstPtr = EmitVAListRef(E->getArg(0));
- Value *SrcPtr = EmitVAListRef(E->getArg(1));
+ Value *DstPtr = EmitVAListRef(E->getArg(0)).getPointer();
+ Value *SrcPtr = EmitVAListRef(E->getArg(1)).getPointer();
llvm::Type *Type = Int8PtrTy;
@@ -409,6 +599,12 @@
"cast");
return RValue::get(Result);
}
+ case Builtin::BI__builtin_unpredictable: {
+ // Always return the argument of __builtin_unpredictable. LLVM does not
+ // handle this builtin. Metadata for this builtin should be added directly
+ // to instructions such as branches or switches that use it.
+ return RValue::get(EmitScalarExpr(E->getArg(0)));
+ }
case Builtin::BI__builtin_expect: {
Value *ArgValue = EmitScalarExpr(E->getArg(0));
llvm::Type *ArgType = ArgValue->getType();
@@ -455,26 +651,13 @@
return RValue::get(Builder.CreateCall(F, ArgValue));
}
case Builtin::BI__builtin_object_size: {
- // We rely on constant folding to deal with expressions with side effects.
- assert(!E->getArg(0)->HasSideEffects(getContext()) &&
- "should have been constant folded");
+ unsigned Type =
+ E->getArg(1)->EvaluateKnownConstInt(getContext()).getZExtValue();
+ auto *ResType = cast<llvm::IntegerType>(ConvertType(E->getType()));
- // We pass this builtin onto the optimizer so that it can
- // figure out the object size in more complex cases.
- llvm::Type *ResType = ConvertType(E->getType());
-
- // LLVM only supports 0 and 2, make sure that we pass along that
- // as a boolean.
- Value *Ty = EmitScalarExpr(E->getArg(1));
- ConstantInt *CI = dyn_cast<ConstantInt>(Ty);
- assert(CI);
- uint64_t val = CI->getZExtValue();
- CI = ConstantInt::get(Builder.getInt1Ty(), (val & 0x2) >> 1);
- // FIXME: Get right address space.
- llvm::Type *Tys[] = { ResType, Builder.getInt8PtrTy(0) };
- Value *F = CGM.getIntrinsic(Intrinsic::objectsize, Tys);
- return RValue::get(
- Builder.CreateCall(F, {EmitScalarExpr(E->getArg(0)), CI}));
+ // We pass this builtin onto the optimizer so that it can figure out the
+ // object size in more complex cases.
+ return RValue::get(emitBuiltinObjectSize(E->getArg(0), Type, ResType));
}
case Builtin::BI__builtin_prefetch: {
Value *Locality, *RW, *Address = EmitScalarExpr(E->getArg(0));
@@ -489,7 +672,7 @@
}
case Builtin::BI__builtin_readcyclecounter: {
Value *F = CGM.getIntrinsic(Intrinsic::readcyclecounter);
- return RValue::get(Builder.CreateCall(F, {}));
+ return RValue::get(Builder.CreateCall(F));
}
case Builtin::BI__builtin___clear_cache: {
Value *Begin = EmitScalarExpr(E->getArg(0));
@@ -497,14 +680,10 @@
Value *F = CGM.getIntrinsic(Intrinsic::clear_cache);
return RValue::get(Builder.CreateCall(F, {Begin, End}));
}
- case Builtin::BI__builtin_trap: {
- Value *F = CGM.getIntrinsic(Intrinsic::trap);
- return RValue::get(Builder.CreateCall(F, {}));
- }
- case Builtin::BI__debugbreak: {
- Value *F = CGM.getIntrinsic(Intrinsic::debugtrap);
- return RValue::get(Builder.CreateCall(F, {}));
- }
+ case Builtin::BI__builtin_trap:
+ return RValue::get(EmitTrapCall(Intrinsic::trap));
+ case Builtin::BI__debugbreak:
+ return RValue::get(EmitTrapCall(Intrinsic::debugtrap));
case Builtin::BI__builtin_unreachable: {
if (SanOpts.has(SanitizerKind::Unreachable)) {
SanitizerScope SanScope(this);
@@ -695,29 +874,24 @@
}
case Builtin::BIbzero:
case Builtin::BI__builtin_bzero: {
- std::pair<llvm::Value*, unsigned> Dest =
- EmitPointerWithAlignment(E->getArg(0));
+ Address Dest = EmitPointerWithAlignment(E->getArg(0));
Value *SizeVal = EmitScalarExpr(E->getArg(1));
- EmitNonNullArgCheck(RValue::get(Dest.first), E->getArg(0)->getType(),
+ EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
E->getArg(0)->getExprLoc(), FD, 0);
- Builder.CreateMemSet(Dest.first, Builder.getInt8(0), SizeVal,
- Dest.second, false);
- return RValue::get(Dest.first);
+ Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal, false);
+ return RValue::get(Dest.getPointer());
}
case Builtin::BImemcpy:
case Builtin::BI__builtin_memcpy: {
- std::pair<llvm::Value*, unsigned> Dest =
- EmitPointerWithAlignment(E->getArg(0));
- std::pair<llvm::Value*, unsigned> Src =
- EmitPointerWithAlignment(E->getArg(1));
+ Address Dest = EmitPointerWithAlignment(E->getArg(0));
+ Address Src = EmitPointerWithAlignment(E->getArg(1));
Value *SizeVal = EmitScalarExpr(E->getArg(2));
- unsigned Align = std::min(Dest.second, Src.second);
- EmitNonNullArgCheck(RValue::get(Dest.first), E->getArg(0)->getType(),
+ EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
E->getArg(0)->getExprLoc(), FD, 0);
- EmitNonNullArgCheck(RValue::get(Src.first), E->getArg(1)->getType(),
+ EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(),
E->getArg(1)->getExprLoc(), FD, 1);
- Builder.CreateMemCpy(Dest.first, Src.first, SizeVal, Align, false);
- return RValue::get(Dest.first);
+ Builder.CreateMemCpy(Dest, Src, SizeVal, false);
+ return RValue::get(Dest.getPointer());
}
case Builtin::BI__builtin___memcpy_chk: {
@@ -728,23 +902,20 @@
break;
if (Size.ugt(DstSize))
break;
- std::pair<llvm::Value*, unsigned> Dest =
- EmitPointerWithAlignment(E->getArg(0));
- std::pair<llvm::Value*, unsigned> Src =
- EmitPointerWithAlignment(E->getArg(1));
+ Address Dest = EmitPointerWithAlignment(E->getArg(0));
+ Address Src = EmitPointerWithAlignment(E->getArg(1));
Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
- unsigned Align = std::min(Dest.second, Src.second);
- Builder.CreateMemCpy(Dest.first, Src.first, SizeVal, Align, false);
- return RValue::get(Dest.first);
+ Builder.CreateMemCpy(Dest, Src, SizeVal, false);
+ return RValue::get(Dest.getPointer());
}
case Builtin::BI__builtin_objc_memmove_collectable: {
- Value *Address = EmitScalarExpr(E->getArg(0));
- Value *SrcAddr = EmitScalarExpr(E->getArg(1));
+ Address DestAddr = EmitPointerWithAlignment(E->getArg(0));
+ Address SrcAddr = EmitPointerWithAlignment(E->getArg(1));
Value *SizeVal = EmitScalarExpr(E->getArg(2));
CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this,
- Address, SrcAddr, SizeVal);
- return RValue::get(Address);
+ DestAddr, SrcAddr, SizeVal);
+ return RValue::get(DestAddr.getPointer());
}
case Builtin::BI__builtin___memmove_chk: {
@@ -755,42 +926,35 @@
break;
if (Size.ugt(DstSize))
break;
- std::pair<llvm::Value*, unsigned> Dest =
- EmitPointerWithAlignment(E->getArg(0));
- std::pair<llvm::Value*, unsigned> Src =
- EmitPointerWithAlignment(E->getArg(1));
+ Address Dest = EmitPointerWithAlignment(E->getArg(0));
+ Address Src = EmitPointerWithAlignment(E->getArg(1));
Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
- unsigned Align = std::min(Dest.second, Src.second);
- Builder.CreateMemMove(Dest.first, Src.first, SizeVal, Align, false);
- return RValue::get(Dest.first);
+ Builder.CreateMemMove(Dest, Src, SizeVal, false);
+ return RValue::get(Dest.getPointer());
}
case Builtin::BImemmove:
case Builtin::BI__builtin_memmove: {
- std::pair<llvm::Value*, unsigned> Dest =
- EmitPointerWithAlignment(E->getArg(0));
- std::pair<llvm::Value*, unsigned> Src =
- EmitPointerWithAlignment(E->getArg(1));
+ Address Dest = EmitPointerWithAlignment(E->getArg(0));
+ Address Src = EmitPointerWithAlignment(E->getArg(1));
Value *SizeVal = EmitScalarExpr(E->getArg(2));
- unsigned Align = std::min(Dest.second, Src.second);
- EmitNonNullArgCheck(RValue::get(Dest.first), E->getArg(0)->getType(),
+ EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
E->getArg(0)->getExprLoc(), FD, 0);
- EmitNonNullArgCheck(RValue::get(Src.first), E->getArg(1)->getType(),
+ EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(),
E->getArg(1)->getExprLoc(), FD, 1);
- Builder.CreateMemMove(Dest.first, Src.first, SizeVal, Align, false);
- return RValue::get(Dest.first);
+ Builder.CreateMemMove(Dest, Src, SizeVal, false);
+ return RValue::get(Dest.getPointer());
}
case Builtin::BImemset:
case Builtin::BI__builtin_memset: {
- std::pair<llvm::Value*, unsigned> Dest =
- EmitPointerWithAlignment(E->getArg(0));
+ Address Dest = EmitPointerWithAlignment(E->getArg(0));
Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
Builder.getInt8Ty());
Value *SizeVal = EmitScalarExpr(E->getArg(2));
- EmitNonNullArgCheck(RValue::get(Dest.first), E->getArg(0)->getType(),
+ EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
E->getArg(0)->getExprLoc(), FD, 0);
- Builder.CreateMemSet(Dest.first, ByteVal, SizeVal, Dest.second, false);
- return RValue::get(Dest.first);
+ Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
+ return RValue::get(Dest.getPointer());
}
case Builtin::BI__builtin___memset_chk: {
// fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
@@ -800,13 +964,12 @@
break;
if (Size.ugt(DstSize))
break;
- std::pair<llvm::Value*, unsigned> Dest =
- EmitPointerWithAlignment(E->getArg(0));
+ Address Dest = EmitPointerWithAlignment(E->getArg(0));
Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
Builder.getInt8Ty());
Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
- Builder.CreateMemSet(Dest.first, ByteVal, SizeVal, Dest.second, false);
- return RValue::get(Dest.first);
+ Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
+ return RValue::get(Dest.getPointer());
}
case Builtin::BI__builtin_dwarf_cfa: {
// The offset in bytes from the first argument to the CFA.
@@ -824,14 +987,14 @@
llvm::ConstantInt::get(Int32Ty, Offset)));
}
case Builtin::BI__builtin_return_address: {
- Value *Depth = EmitScalarExpr(E->getArg(0));
- Depth = Builder.CreateIntCast(Depth, Int32Ty, false);
+ Value *Depth =
+ CGM.EmitConstantExpr(E->getArg(0), getContext().UnsignedIntTy, this);
Value *F = CGM.getIntrinsic(Intrinsic::returnaddress);
return RValue::get(Builder.CreateCall(F, Depth));
}
case Builtin::BI__builtin_frame_address: {
- Value *Depth = EmitScalarExpr(E->getArg(0));
- Depth = Builder.CreateIntCast(Depth, Int32Ty, false);
+ Value *Depth =
+ CGM.EmitConstantExpr(E->getArg(0), getContext().UnsignedIntTy, this);
Value *F = CGM.getIntrinsic(Intrinsic::frameaddress);
return RValue::get(Builder.CreateCall(F, Depth));
}
@@ -881,7 +1044,7 @@
}
case Builtin::BI__builtin_unwind_init: {
Value *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init);
- return RValue::get(Builder.CreateCall(F, {}));
+ return RValue::get(Builder.CreateCall(F));
}
case Builtin::BI__builtin_extend_pointer: {
// Extends a pointer to the size of an _Unwind_Word, which is
@@ -910,7 +1073,7 @@
}
case Builtin::BI__builtin_setjmp: {
// Buffer is a void**.
- Value *Buf = EmitScalarExpr(E->getArg(0));
+ Address Buf = EmitPointerWithAlignment(E->getArg(0));
// Store the frame pointer to the setjmp buffer.
Value *FrameAddr =
@@ -920,15 +1083,15 @@
// Store the stack pointer to the setjmp buffer.
Value *StackAddr =
- Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave), {});
- Value *StackSaveSlot =
- Builder.CreateGEP(Buf, ConstantInt::get(Int32Ty, 2));
+ Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave));
+ Address StackSaveSlot =
+ Builder.CreateConstInBoundsGEP(Buf, 2, getPointerSize());
Builder.CreateStore(StackAddr, StackSaveSlot);
// Call LLVM's EH setjmp, which is lightweight.
Value *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp);
Buf = Builder.CreateBitCast(Buf, Int8PtrTy);
- return RValue::get(Builder.CreateCall(F, Buf));
+ return RValue::get(Builder.CreateCall(F, Buf.getPointer()));
}
case Builtin::BI__builtin_longjmp: {
Value *Buf = EmitScalarExpr(E->getArg(0));
@@ -1057,58 +1220,15 @@
case Builtin::BI__sync_val_compare_and_swap_2:
case Builtin::BI__sync_val_compare_and_swap_4:
case Builtin::BI__sync_val_compare_and_swap_8:
- case Builtin::BI__sync_val_compare_and_swap_16: {
- QualType T = E->getType();
- llvm::Value *DestPtr = EmitScalarExpr(E->getArg(0));
- unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
-
- llvm::IntegerType *IntType =
- llvm::IntegerType::get(getLLVMContext(),
- getContext().getTypeSize(T));
- llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
-
- Value *Args[3];
- Args[0] = Builder.CreateBitCast(DestPtr, IntPtrType);
- Args[1] = EmitScalarExpr(E->getArg(1));
- llvm::Type *ValueType = Args[1]->getType();
- Args[1] = EmitToInt(*this, Args[1], T, IntType);
- Args[2] = EmitToInt(*this, EmitScalarExpr(E->getArg(2)), T, IntType);
-
- Value *Result = Builder.CreateAtomicCmpXchg(Args[0], Args[1], Args[2],
- llvm::SequentiallyConsistent,
- llvm::SequentiallyConsistent);
- Result = Builder.CreateExtractValue(Result, 0);
- Result = EmitFromInt(*this, Result, T, ValueType);
- return RValue::get(Result);
- }
+ case Builtin::BI__sync_val_compare_and_swap_16:
+ return RValue::get(MakeAtomicCmpXchgValue(*this, E, false));
case Builtin::BI__sync_bool_compare_and_swap_1:
case Builtin::BI__sync_bool_compare_and_swap_2:
case Builtin::BI__sync_bool_compare_and_swap_4:
case Builtin::BI__sync_bool_compare_and_swap_8:
- case Builtin::BI__sync_bool_compare_and_swap_16: {
- QualType T = E->getArg(1)->getType();
- llvm::Value *DestPtr = EmitScalarExpr(E->getArg(0));
- unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
-
- llvm::IntegerType *IntType =
- llvm::IntegerType::get(getLLVMContext(),
- getContext().getTypeSize(T));
- llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
-
- Value *Args[3];
- Args[0] = Builder.CreateBitCast(DestPtr, IntPtrType);
- Args[1] = EmitToInt(*this, EmitScalarExpr(E->getArg(1)), T, IntType);
- Args[2] = EmitToInt(*this, EmitScalarExpr(E->getArg(2)), T, IntType);
-
- Value *Pair = Builder.CreateAtomicCmpXchg(Args[0], Args[1], Args[2],
- llvm::SequentiallyConsistent,
- llvm::SequentiallyConsistent);
- Value *Result = Builder.CreateExtractValue(Pair, 1);
- // zext bool to int.
- Result = Builder.CreateZExt(Result, ConvertType(E->getType()));
- return RValue::get(Result);
- }
+ case Builtin::BI__sync_bool_compare_and_swap_16:
+ return RValue::get(MakeAtomicCmpXchgValue(*this, E, true));
case Builtin::BI__sync_swap_1:
case Builtin::BI__sync_swap_2:
@@ -1136,8 +1256,8 @@
StoreSize.getQuantity() * 8);
Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo());
llvm::StoreInst *Store =
- Builder.CreateStore(llvm::Constant::getNullValue(ITy), Ptr);
- Store->setAlignment(StoreSize.getQuantity());
+ Builder.CreateAlignedStore(llvm::Constant::getNullValue(ITy), Ptr,
+ StoreSize);
Store->setAtomic(llvm::Release);
return RValue::get(nullptr);
}
@@ -1154,6 +1274,10 @@
return RValue::get(nullptr);
}
+ case Builtin::BI__builtin_nontemporal_load:
+ return RValue::get(EmitNontemporalLoad(*this, E));
+ case Builtin::BI__builtin_nontemporal_store:
+ return RValue::get(EmitNontemporalStore(*this, E));
case Builtin::BI__c11_atomic_is_lock_free:
case Builtin::BI__atomic_is_lock_free: {
// Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the
@@ -1271,15 +1395,14 @@
bool Volatile =
PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
- Value *Ptr = EmitScalarExpr(E->getArg(0));
- unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace();
+ Address Ptr = EmitPointerWithAlignment(E->getArg(0));
+ unsigned AddrSpace = Ptr.getPointer()->getType()->getPointerAddressSpace();
Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
Value *NewVal = Builder.getInt8(0);
Value *Order = EmitScalarExpr(E->getArg(1));
if (isa<llvm::ConstantInt>(Order)) {
int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile);
- Store->setAlignment(1);
switch (ord) {
case 0: // memory_order_relaxed
default: // invalid order
@@ -1312,7 +1435,6 @@
for (unsigned i = 0; i < 3; ++i) {
Builder.SetInsertPoint(BBs[i]);
StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile);
- Store->setAlignment(1);
Store->setOrdering(Orders[i]);
Builder.CreateBr(ContBB);
}
@@ -1494,8 +1616,7 @@
llvm::Value *X = EmitScalarExpr(E->getArg(0));
llvm::Value *Y = EmitScalarExpr(E->getArg(1));
llvm::Value *Carryin = EmitScalarExpr(E->getArg(2));
- std::pair<llvm::Value*, unsigned> CarryOutPtr =
- EmitPointerWithAlignment(E->getArg(3));
+ Address CarryOutPtr = EmitPointerWithAlignment(E->getArg(3));
// Decide if we are lowering to a uadd.with.overflow or usub.with.overflow.
llvm::Intrinsic::ID IntrinsicId;
@@ -1526,11 +1647,91 @@
Sum1, Carryin, Carry2);
llvm::Value *CarryOut = Builder.CreateZExt(Builder.CreateOr(Carry1, Carry2),
X->getType());
- llvm::StoreInst *CarryOutStore = Builder.CreateStore(CarryOut,
- CarryOutPtr.first);
- CarryOutStore->setAlignment(CarryOutPtr.second);
+ Builder.CreateStore(CarryOut, CarryOutPtr);
return RValue::get(Sum2);
}
+
+ case Builtin::BI__builtin_add_overflow:
+ case Builtin::BI__builtin_sub_overflow:
+ case Builtin::BI__builtin_mul_overflow: {
+ const clang::Expr *LeftArg = E->getArg(0);
+ const clang::Expr *RightArg = E->getArg(1);
+ const clang::Expr *ResultArg = E->getArg(2);
+
+ clang::QualType ResultQTy =
+ ResultArg->getType()->castAs<PointerType>()->getPointeeType();
+
+ WidthAndSignedness LeftInfo =
+ getIntegerWidthAndSignedness(CGM.getContext(), LeftArg->getType());
+ WidthAndSignedness RightInfo =
+ getIntegerWidthAndSignedness(CGM.getContext(), RightArg->getType());
+ WidthAndSignedness ResultInfo =
+ getIntegerWidthAndSignedness(CGM.getContext(), ResultQTy);
+ WidthAndSignedness EncompassingInfo =
+ EncompassingIntegerType({LeftInfo, RightInfo, ResultInfo});
+
+ llvm::Type *EncompassingLLVMTy =
+ llvm::IntegerType::get(CGM.getLLVMContext(), EncompassingInfo.Width);
+
+ llvm::Type *ResultLLVMTy = CGM.getTypes().ConvertType(ResultQTy);
+
+ llvm::Intrinsic::ID IntrinsicId;
+ switch (BuiltinID) {
+ default:
+ llvm_unreachable("Unknown overflow builtin id.");
+ case Builtin::BI__builtin_add_overflow:
+ IntrinsicId = EncompassingInfo.Signed
+ ? llvm::Intrinsic::sadd_with_overflow
+ : llvm::Intrinsic::uadd_with_overflow;
+ break;
+ case Builtin::BI__builtin_sub_overflow:
+ IntrinsicId = EncompassingInfo.Signed
+ ? llvm::Intrinsic::ssub_with_overflow
+ : llvm::Intrinsic::usub_with_overflow;
+ break;
+ case Builtin::BI__builtin_mul_overflow:
+ IntrinsicId = EncompassingInfo.Signed
+ ? llvm::Intrinsic::smul_with_overflow
+ : llvm::Intrinsic::umul_with_overflow;
+ break;
+ }
+
+ llvm::Value *Left = EmitScalarExpr(LeftArg);
+ llvm::Value *Right = EmitScalarExpr(RightArg);
+ Address ResultPtr = EmitPointerWithAlignment(ResultArg);
+
+ // Extend each operand to the encompassing type.
+ Left = Builder.CreateIntCast(Left, EncompassingLLVMTy, LeftInfo.Signed);
+ Right = Builder.CreateIntCast(Right, EncompassingLLVMTy, RightInfo.Signed);
+
+ // Perform the operation on the extended values.
+ llvm::Value *Overflow, *Result;
+ Result = EmitOverflowIntrinsic(*this, IntrinsicId, Left, Right, Overflow);
+
+ if (EncompassingInfo.Width > ResultInfo.Width) {
+ // The encompassing type is wider than the result type, so we need to
+ // truncate it.
+ llvm::Value *ResultTrunc = Builder.CreateTrunc(Result, ResultLLVMTy);
+
+ // To see if the truncation caused an overflow, we will extend
+ // the result and then compare it to the original result.
+ llvm::Value *ResultTruncExt = Builder.CreateIntCast(
+ ResultTrunc, EncompassingLLVMTy, ResultInfo.Signed);
+ llvm::Value *TruncationOverflow =
+ Builder.CreateICmpNE(Result, ResultTruncExt);
+
+ Overflow = Builder.CreateOr(Overflow, TruncationOverflow);
+ Result = ResultTrunc;
+ }
+
+ // Finally, store the result using the pointer.
+ bool isVolatile =
+ ResultArg->getType()->getPointeeType().isVolatileQualified();
+ Builder.CreateStore(EmitToMemory(Result, ResultQTy), ResultPtr, isVolatile);
+
+ return RValue::get(Overflow);
+ }
+
case Builtin::BI__builtin_uadd_overflow:
case Builtin::BI__builtin_uaddl_overflow:
case Builtin::BI__builtin_uaddll_overflow:
@@ -1555,13 +1756,12 @@
// Scalarize our inputs.
llvm::Value *X = EmitScalarExpr(E->getArg(0));
llvm::Value *Y = EmitScalarExpr(E->getArg(1));
- std::pair<llvm::Value *, unsigned> SumOutPtr =
- EmitPointerWithAlignment(E->getArg(2));
+ Address SumOutPtr = EmitPointerWithAlignment(E->getArg(2));
// Decide which of the overflow intrinsics we are lowering to:
llvm::Intrinsic::ID IntrinsicId;
switch (BuiltinID) {
- default: llvm_unreachable("Unknown security overflow builtin id.");
+ default: llvm_unreachable("Unknown overflow builtin id.");
case Builtin::BI__builtin_uadd_overflow:
case Builtin::BI__builtin_uaddl_overflow:
case Builtin::BI__builtin_uaddll_overflow:
@@ -1597,13 +1797,12 @@
llvm::Value *Carry;
llvm::Value *Sum = EmitOverflowIntrinsic(*this, IntrinsicId, X, Y, Carry);
- llvm::StoreInst *SumOutStore = Builder.CreateStore(Sum, SumOutPtr.first);
- SumOutStore->setAlignment(SumOutPtr.second);
+ Builder.CreateStore(Sum, SumOutPtr);
return RValue::get(Carry);
}
case Builtin::BI__builtin_addressof:
- return RValue::get(EmitLValue(E->getArg(0)).getAddress());
+ return RValue::get(EmitLValue(E->getArg(0)).getPointer());
case Builtin::BI__builtin_operator_new:
return EmitBuiltinNewDeleteCall(FD->getType()->castAs<FunctionProtoType>(),
E->getArg(0), false);
@@ -1778,8 +1977,15 @@
if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID))
return emitLibraryCall(*this, FD, E, EmitScalarExpr(E->getCallee()));
+ // Check that a call to a target specific builtin has the correct target
+ // features.
+ // This is down here to avoid non-target specific builtins, however, if
+ // generic builtins start to require generic target features then we
+ // can move this up to the beginning of the function.
+ checkTargetFeatures(E, FD);
+
// See if we have a target specific intrinsic.
- const char *Name = getContext().BuiltinInfo.GetName(BuiltinID);
+ const char *Name = getContext().BuiltinInfo.getName(BuiltinID);
Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
if (const char *Prefix =
llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch())) {
@@ -1857,34 +2063,54 @@
return GetUndefRValue(E->getType());
}
-Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID,
- const CallExpr *E) {
- switch (getTarget().getTriple().getArch()) {
+static Value *EmitTargetArchBuiltinExpr(CodeGenFunction *CGF,
+ unsigned BuiltinID, const CallExpr *E,
+ llvm::Triple::ArchType Arch) {
+ switch (Arch) {
case llvm::Triple::arm:
case llvm::Triple::armeb:
case llvm::Triple::thumb:
case llvm::Triple::thumbeb:
- return EmitARMBuiltinExpr(BuiltinID, E);
+ return CGF->EmitARMBuiltinExpr(BuiltinID, E);
case llvm::Triple::aarch64:
case llvm::Triple::aarch64_be:
- return EmitAArch64BuiltinExpr(BuiltinID, E);
+ return CGF->EmitAArch64BuiltinExpr(BuiltinID, E);
case llvm::Triple::x86:
case llvm::Triple::x86_64:
- return EmitX86BuiltinExpr(BuiltinID, E);
+ return CGF->EmitX86BuiltinExpr(BuiltinID, E);
case llvm::Triple::ppc:
case llvm::Triple::ppc64:
case llvm::Triple::ppc64le:
- return EmitPPCBuiltinExpr(BuiltinID, E);
+ return CGF->EmitPPCBuiltinExpr(BuiltinID, E);
case llvm::Triple::r600:
case llvm::Triple::amdgcn:
- return EmitR600BuiltinExpr(BuiltinID, E);
+ return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E);
case llvm::Triple::systemz:
- return EmitSystemZBuiltinExpr(BuiltinID, E);
+ return CGF->EmitSystemZBuiltinExpr(BuiltinID, E);
+ case llvm::Triple::nvptx:
+ case llvm::Triple::nvptx64:
+ return CGF->EmitNVPTXBuiltinExpr(BuiltinID, E);
+ case llvm::Triple::wasm32:
+ case llvm::Triple::wasm64:
+ return CGF->EmitWebAssemblyBuiltinExpr(BuiltinID, E);
default:
return nullptr;
}
}
+Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID,
+ const CallExpr *E) {
+ if (getContext().BuiltinInfo.isAuxBuiltinID(BuiltinID)) {
+ assert(getContext().getAuxTargetInfo() && "Missing aux target info");
+ return EmitTargetArchBuiltinExpr(
+ this, getContext().BuiltinInfo.getAuxBuiltinID(BuiltinID), E,
+ getContext().getAuxTargetInfo()->getTriple().getArch());
+ }
+
+ return EmitTargetArchBuiltinExpr(this, BuiltinID, E,
+ getTarget().getTriple().getArch());
+}
+
static llvm::VectorType *GetNeonType(CodeGenFunction *CGF,
NeonTypeFlags TypeFlags,
bool V1Ty=false) {
@@ -1915,6 +2141,19 @@
llvm_unreachable("Unknown vector element type!");
}
+static llvm::VectorType *GetFloatNeonType(CodeGenFunction *CGF,
+ NeonTypeFlags IntTypeFlags) {
+ int IsQuad = IntTypeFlags.isQuad();
+ switch (IntTypeFlags.getEltType()) {
+ case NeonTypeFlags::Int32:
+ return llvm::VectorType::get(CGF->FloatTy, (2 << IsQuad));
+ case NeonTypeFlags::Int64:
+ return llvm::VectorType::get(CGF->DoubleTy, (1 << IsQuad));
+ default:
+ llvm_unreachable("Type can't be converted to floating-point!");
+ }
+}
+
Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C) {
unsigned nElts = cast<llvm::VectorType>(V->getType())->getNumElements();
Value* SV = llvm::ConstantVector::getSplat(nElts, C);
@@ -1938,10 +2177,7 @@
Value *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty,
bool neg) {
int SV = cast<ConstantInt>(V)->getSExtValue();
-
- llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
- llvm::Constant *C = ConstantInt::get(VTy->getElementType(), neg ? -SV : SV);
- return llvm::ConstantVector::getSplat(VTy->getNumElements(), C);
+ return ConstantInt::get(Ty, neg ? -SV : SV);
}
// \brief Right-shift a vector by a constant.
@@ -1960,8 +2196,7 @@
if (ShiftAmt == EltSize) {
if (usgn) {
// Right-shifting an unsigned value by its size yields 0.
- llvm::Constant *Zero = ConstantInt::get(VTy->getElementType(), 0);
- return llvm::ConstantVector::getSplat(VTy->getNumElements(), Zero);
+ return llvm::ConstantAggregateZero::get(VTy);
} else {
// Right-shifting a signed value by its size is equivalent
// to a shift of size-1.
@@ -1977,61 +2212,6 @@
return Builder.CreateAShr(Vec, Shift, name);
}
-/// GetPointeeAlignment - Given an expression with a pointer type, find the
-/// alignment of the type referenced by the pointer. Skip over implicit
-/// casts.
-std::pair<llvm::Value*, unsigned>
-CodeGenFunction::EmitPointerWithAlignment(const Expr *Addr) {
- assert(Addr->getType()->isPointerType());
- Addr = Addr->IgnoreParens();
- if (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(Addr)) {
- if ((ICE->getCastKind() == CK_BitCast || ICE->getCastKind() == CK_NoOp) &&
- ICE->getSubExpr()->getType()->isPointerType()) {
- std::pair<llvm::Value*, unsigned> Ptr =
- EmitPointerWithAlignment(ICE->getSubExpr());
- Ptr.first = Builder.CreateBitCast(Ptr.first,
- ConvertType(Addr->getType()));
- return Ptr;
- } else if (ICE->getCastKind() == CK_ArrayToPointerDecay) {
- LValue LV = EmitLValue(ICE->getSubExpr());
- unsigned Align = LV.getAlignment().getQuantity();
- if (!Align) {
- // FIXME: Once LValues are fixed to always set alignment,
- // zap this code.
- QualType PtTy = ICE->getSubExpr()->getType();
- if (!PtTy->isIncompleteType())
- Align = getContext().getTypeAlignInChars(PtTy).getQuantity();
- else
- Align = 1;
- }
- return std::make_pair(LV.getAddress(), Align);
- }
- }
- if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(Addr)) {
- if (UO->getOpcode() == UO_AddrOf) {
- LValue LV = EmitLValue(UO->getSubExpr());
- unsigned Align = LV.getAlignment().getQuantity();
- if (!Align) {
- // FIXME: Once LValues are fixed to always set alignment,
- // zap this code.
- QualType PtTy = UO->getSubExpr()->getType();
- if (!PtTy->isIncompleteType())
- Align = getContext().getTypeAlignInChars(PtTy).getQuantity();
- else
- Align = 1;
- }
- return std::make_pair(LV.getAddress(), Align);
- }
- }
-
- unsigned Align = 1;
- QualType PtTy = Addr->getType()->getPointeeType();
- if (!PtTy->isIncompleteType())
- Align = getContext().getTypeAlignInChars(PtTy).getQuantity();
-
- return std::make_pair(EmitScalarExpr(Addr), Align);
-}
-
enum {
AddRetType = (1 << 0),
Add1ArgType = (1 << 1),
@@ -2054,31 +2234,36 @@
AddRetType | VectorizeRetType | Add1ArgType | InventFloatType
};
- struct NeonIntrinsicInfo {
+namespace {
+struct NeonIntrinsicInfo {
+ const char *NameHint;
unsigned BuiltinID;
unsigned LLVMIntrinsic;
unsigned AltLLVMIntrinsic;
- const char *NameHint;
unsigned TypeModifier;
bool operator<(unsigned RHSBuiltinID) const {
return BuiltinID < RHSBuiltinID;
}
+ bool operator<(const NeonIntrinsicInfo &TE) const {
+ return BuiltinID < TE.BuiltinID;
+ }
};
+} // end anonymous namespace
#define NEONMAP0(NameBase) \
- { NEON::BI__builtin_neon_ ## NameBase, 0, 0, #NameBase, 0 }
+ { #NameBase, NEON::BI__builtin_neon_ ## NameBase, 0, 0, 0 }
#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
- { NEON:: BI__builtin_neon_ ## NameBase, \
- Intrinsic::LLVMIntrinsic, 0, #NameBase, TypeModifier }
+ { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
+ Intrinsic::LLVMIntrinsic, 0, TypeModifier }
#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \
- { NEON:: BI__builtin_neon_ ## NameBase, \
+ { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \
- #NameBase, TypeModifier }
+ TypeModifier }
-static NeonIntrinsicInfo ARMSIMDIntrinsicMap [] = {
+static const NeonIntrinsicInfo ARMSIMDIntrinsicMap [] = {
NEONMAP2(vabd_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts),
NEONMAP2(vabdq_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts),
NEONMAP1(vabs_v, arm_neon_vabs, 0),
@@ -2104,7 +2289,7 @@
NEONMAP1(vclzq_v, ctlz, Add1ArgType),
NEONMAP1(vcnt_v, ctpop, Add1ArgType),
NEONMAP1(vcntq_v, ctpop, Add1ArgType),
- NEONMAP1(vcvt_f16_v, arm_neon_vcvtfp2hf, 0),
+ NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0),
NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0),
NEONMAP0(vcvt_f32_v),
NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
@@ -2295,7 +2480,7 @@
NEONMAP0(vzipq_v)
};
-static NeonIntrinsicInfo AArch64SIMDIntrinsicMap[] = {
+static const NeonIntrinsicInfo AArch64SIMDIntrinsicMap[] = {
NEONMAP1(vabs_v, aarch64_neon_abs, 0),
NEONMAP1(vabsq_v, aarch64_neon_abs, 0),
NEONMAP0(vaddhn_v),
@@ -2317,7 +2502,7 @@
NEONMAP1(vclzq_v, ctlz, Add1ArgType),
NEONMAP1(vcnt_v, ctpop, Add1ArgType),
NEONMAP1(vcntq_v, ctpop, Add1ArgType),
- NEONMAP1(vcvt_f16_v, aarch64_neon_vcvtfp2hf, 0),
+ NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0),
NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0),
NEONMAP0(vcvt_f32_v),
NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
@@ -2410,7 +2595,7 @@
NEONMAP0(vtstq_v),
};
-static NeonIntrinsicInfo AArch64SISDIntrinsicMap[] = {
+static const NeonIntrinsicInfo AArch64SISDIntrinsicMap[] = {
NEONMAP1(vabdd_f64, aarch64_sisd_fabd, Add1ArgType),
NEONMAP1(vabds_f32, aarch64_sisd_fabd, Add1ArgType),
NEONMAP1(vabsd_s64, aarch64_neon_abs, Add1ArgType),
@@ -2621,9 +2806,7 @@
#ifndef NDEBUG
if (!MapProvenSorted) {
- // FIXME: use std::is_sorted once C++11 is allowed
- for (unsigned i = 0; i < IntrinsicMap.size() - 1; ++i)
- assert(IntrinsicMap[i].BuiltinID <= IntrinsicMap[i + 1].BuiltinID);
+ assert(std::is_sorted(std::begin(IntrinsicMap), std::end(IntrinsicMap)));
MapProvenSorted = true;
}
#endif
@@ -2742,7 +2925,7 @@
Value *CodeGenFunction::EmitCommonNeonBuiltinExpr(
unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic,
const char *NameHint, unsigned Modifier, const CallExpr *E,
- SmallVectorImpl<llvm::Value *> &Ops, llvm::Value *Align) {
+ SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1) {
// Get the last argument, which specifies the vector type.
llvm::APSInt NeonTypeConst;
const Expr *Arg = E->getArg(E->getNumArgs() - 1);
@@ -2759,6 +2942,10 @@
if (!Ty)
return nullptr;
+ auto getAlignmentValue32 = [&](Address addr) -> Value* {
+ return Builder.getInt32(addr.getAlignment().getQuantity());
+ };
+
unsigned Int = LLVMIntrinsic;
if ((Modifier & UnsignedAlts) && !Usgn)
Int = AltLLVMIntrinsic;
@@ -2780,9 +2967,8 @@
Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn");
// %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
- Constant *ShiftAmt = ConstantInt::get(SrcTy->getElementType(),
- SrcTy->getScalarSizeInBits() / 2);
- ShiftAmt = ConstantVector::getSplat(VTy->getNumElements(), ShiftAmt);
+ Constant *ShiftAmt =
+ ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn");
// %res = trunc <4 x i32> %high to <4 x i16>
@@ -2820,13 +3006,7 @@
case NEON::BI__builtin_neon_vcvt_n_f64_v:
case NEON::BI__builtin_neon_vcvtq_n_f32_v:
case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
- bool Double =
- (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64);
- llvm::Type *FloatTy =
- GetNeonType(this, NeonTypeFlags(Double ? NeonTypeFlags::Float64
- : NeonTypeFlags::Float32,
- false, Quad));
- llvm::Type *Tys[2] = { FloatTy, Ty };
+ llvm::Type *Tys[2] = { GetFloatNeonType(this, Type), Ty };
Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
Function *F = CGM.getIntrinsic(Int, Tys);
return EmitNeonCall(F, Ops, "vcvt_n");
@@ -2839,13 +3019,7 @@
case NEON::BI__builtin_neon_vcvtq_n_u32_v:
case NEON::BI__builtin_neon_vcvtq_n_s64_v:
case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
- bool Double =
- (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64);
- llvm::Type *FloatTy =
- GetNeonType(this, NeonTypeFlags(Double ? NeonTypeFlags::Float64
- : NeonTypeFlags::Float32,
- false, Quad));
- llvm::Type *Tys[2] = { Ty, FloatTy };
+ llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
return EmitNeonCall(F, Ops, "vcvt_n");
}
@@ -2857,13 +3031,7 @@
case NEON::BI__builtin_neon_vcvtq_u32_v:
case NEON::BI__builtin_neon_vcvtq_s64_v:
case NEON::BI__builtin_neon_vcvtq_u64_v: {
- bool Double =
- (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64);
- llvm::Type *FloatTy =
- GetNeonType(this, NeonTypeFlags(Double ? NeonTypeFlags::Float64
- : NeonTypeFlags::Float32,
- false, Quad));
- Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy);
+ Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type));
return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt")
: Builder.CreateFPToSI(Ops[0], Ty, "vcvt");
}
@@ -2899,13 +3067,7 @@
case NEON::BI__builtin_neon_vcvtmq_s64_v:
case NEON::BI__builtin_neon_vcvtmq_u32_v:
case NEON::BI__builtin_neon_vcvtmq_u64_v: {
- bool Double =
- (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64);
- llvm::Type *InTy =
- GetNeonType(this,
- NeonTypeFlags(Double ? NeonTypeFlags::Float64
- : NeonTypeFlags::Float32, false, Quad));
- llvm::Type *Tys[2] = { Ty, InTy };
+ llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint);
}
case NEON::BI__builtin_neon_vext_v:
@@ -2931,28 +3093,31 @@
return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
}
case NEON::BI__builtin_neon_vld1_v:
- case NEON::BI__builtin_neon_vld1q_v:
- Ops.push_back(Align);
- return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vld1");
+ case NEON::BI__builtin_neon_vld1q_v: {
+ llvm::Type *Tys[] = {Ty, Int8PtrTy};
+ Ops.push_back(getAlignmentValue32(PtrOp0));
+ return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vld1");
+ }
case NEON::BI__builtin_neon_vld2_v:
case NEON::BI__builtin_neon_vld2q_v:
case NEON::BI__builtin_neon_vld3_v:
case NEON::BI__builtin_neon_vld3q_v:
case NEON::BI__builtin_neon_vld4_v:
case NEON::BI__builtin_neon_vld4q_v: {
- Function *F = CGM.getIntrinsic(LLVMIntrinsic, Ty);
+ llvm::Type *Tys[] = {Ty, Int8PtrTy};
+ Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
+ Value *Align = getAlignmentValue32(PtrOp1);
Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint);
Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
- return Builder.CreateStore(Ops[1], Ops[0]);
+ return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
}
case NEON::BI__builtin_neon_vld1_dup_v:
case NEON::BI__builtin_neon_vld1q_dup_v: {
Value *V = UndefValue::get(Ty);
Ty = llvm::PointerType::getUnqual(VTy->getElementType());
- Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
- LoadInst *Ld = Builder.CreateLoad(Ops[0]);
- Ld->setAlignment(cast<ConstantInt>(Align)->getZExtValue());
+ PtrOp0 = Builder.CreateBitCast(PtrOp0, Ty);
+ LoadInst *Ld = Builder.CreateLoad(PtrOp0);
llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
Ops[0] = Builder.CreateInsertElement(V, Ld, CI);
return EmitNeonSplat(Ops[0], CI);
@@ -2963,14 +3128,15 @@
case NEON::BI__builtin_neon_vld3q_lane_v:
case NEON::BI__builtin_neon_vld4_lane_v:
case NEON::BI__builtin_neon_vld4q_lane_v: {
- Function *F = CGM.getIntrinsic(LLVMIntrinsic, Ty);
+ llvm::Type *Tys[] = {Ty, Int8PtrTy};
+ Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
for (unsigned I = 2; I < Ops.size() - 1; ++I)
Ops[I] = Builder.CreateBitCast(Ops[I], Ty);
- Ops.push_back(Align);
+ Ops.push_back(getAlignmentValue32(PtrOp1));
Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), NameHint);
Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
- return Builder.CreateStore(Ops[1], Ops[0]);
+ return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
}
case NEON::BI__builtin_neon_vmovl_v: {
llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
@@ -3017,14 +3183,10 @@
case NEON::BI__builtin_neon_vqdmlal_v:
case NEON::BI__builtin_neon_vqdmlsl_v: {
SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end());
- Value *Mul = EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty),
- MulOps, "vqdmlal");
-
- SmallVector<Value *, 2> AccumOps;
- AccumOps.push_back(Ops[0]);
- AccumOps.push_back(Mul);
- return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty),
- AccumOps, NameHint);
+ Ops[1] =
+ EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), MulOps, "vqdmlal");
+ Ops.resize(2);
+ return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), Ops, NameHint);
}
case NEON::BI__builtin_neon_vqshl_n_v:
case NEON::BI__builtin_neon_vqshlq_n_v:
@@ -3086,9 +3248,11 @@
case NEON::BI__builtin_neon_vst3_lane_v:
case NEON::BI__builtin_neon_vst3q_lane_v:
case NEON::BI__builtin_neon_vst4_lane_v:
- case NEON::BI__builtin_neon_vst4q_lane_v:
- Ops.push_back(Align);
- return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "");
+ case NEON::BI__builtin_neon_vst4q_lane_v: {
+ llvm::Type *Tys[] = {Int8PtrTy, Ty};
+ Ops.push_back(getAlignmentValue32(PtrOp0));
+ return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "");
+ }
case NEON::BI__builtin_neon_vsubhn_v: {
llvm::VectorType *SrcTy =
llvm::VectorType::getExtendedElementVectorType(VTy);
@@ -3099,9 +3263,8 @@
Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn");
// %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
- Constant *ShiftAmt = ConstantInt::get(SrcTy->getElementType(),
- SrcTy->getScalarSizeInBits() / 2);
- ShiftAmt = ConstantVector::getSplat(VTy->getNumElements(), ShiftAmt);
+ Constant *ShiftAmt =
+ ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn");
// %res = trunc <4 x i32> %high to <4 x i16>
@@ -3123,7 +3286,7 @@
Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
SV = llvm::ConstantVector::get(Indices);
SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn");
- SV = Builder.CreateStore(SV, Addr);
+ SV = Builder.CreateDefaultAlignedStore(SV, Addr);
}
return SV;
}
@@ -3151,7 +3314,7 @@
Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
SV = llvm::ConstantVector::get(Indices);
SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp");
- SV = Builder.CreateStore(SV, Addr);
+ SV = Builder.CreateDefaultAlignedStore(SV, Addr);
}
return SV;
}
@@ -3171,7 +3334,7 @@
Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
SV = llvm::ConstantVector::get(Indices);
SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip");
- SV = Builder.CreateStore(SV, Addr);
+ SV = Builder.CreateDefaultAlignedStore(SV, Addr);
}
return SV;
}
@@ -3250,33 +3413,37 @@
}
Value *CodeGenFunction::GetValueForARMHint(unsigned BuiltinID) {
+ unsigned Value;
switch (BuiltinID) {
default:
return nullptr;
case ARM::BI__builtin_arm_nop:
- return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint),
- llvm::ConstantInt::get(Int32Ty, 0));
+ Value = 0;
+ break;
case ARM::BI__builtin_arm_yield:
case ARM::BI__yield:
- return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint),
- llvm::ConstantInt::get(Int32Ty, 1));
+ Value = 1;
+ break;
case ARM::BI__builtin_arm_wfe:
case ARM::BI__wfe:
- return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint),
- llvm::ConstantInt::get(Int32Ty, 2));
+ Value = 2;
+ break;
case ARM::BI__builtin_arm_wfi:
case ARM::BI__wfi:
- return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint),
- llvm::ConstantInt::get(Int32Ty, 3));
+ Value = 3;
+ break;
case ARM::BI__builtin_arm_sev:
case ARM::BI__sev:
- return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint),
- llvm::ConstantInt::get(Int32Ty, 4));
+ Value = 4;
+ break;
case ARM::BI__builtin_arm_sevl:
case ARM::BI__sevl:
- return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint),
- llvm::ConstantInt::get(Int32Ty, 5));
+ Value = 5;
+ break;
}
+
+ return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint),
+ llvm::ConstantInt::get(Int32Ty, Value));
}
// Generates the IR for the read/write special register builtin,
@@ -3339,6 +3506,42 @@
return Builder.CreateCall(F, { Metadata, ArgValue });
}
+/// Return true if BuiltinID is an overloaded Neon intrinsic with an extra
+/// argument that specifies the vector type.
+static bool HasExtraNeonArgument(unsigned BuiltinID) {
+ switch (BuiltinID) {
+ default: break;
+ case NEON::BI__builtin_neon_vget_lane_i8:
+ case NEON::BI__builtin_neon_vget_lane_i16:
+ case NEON::BI__builtin_neon_vget_lane_i32:
+ case NEON::BI__builtin_neon_vget_lane_i64:
+ case NEON::BI__builtin_neon_vget_lane_f32:
+ case NEON::BI__builtin_neon_vgetq_lane_i8:
+ case NEON::BI__builtin_neon_vgetq_lane_i16:
+ case NEON::BI__builtin_neon_vgetq_lane_i32:
+ case NEON::BI__builtin_neon_vgetq_lane_i64:
+ case NEON::BI__builtin_neon_vgetq_lane_f32:
+ case NEON::BI__builtin_neon_vset_lane_i8:
+ case NEON::BI__builtin_neon_vset_lane_i16:
+ case NEON::BI__builtin_neon_vset_lane_i32:
+ case NEON::BI__builtin_neon_vset_lane_i64:
+ case NEON::BI__builtin_neon_vset_lane_f32:
+ case NEON::BI__builtin_neon_vsetq_lane_i8:
+ case NEON::BI__builtin_neon_vsetq_lane_i16:
+ case NEON::BI__builtin_neon_vsetq_lane_i32:
+ case NEON::BI__builtin_neon_vsetq_lane_i64:
+ case NEON::BI__builtin_neon_vsetq_lane_f32:
+ case NEON::BI__builtin_neon_vsha1h_u32:
+ case NEON::BI__builtin_neon_vsha1cq_u32:
+ case NEON::BI__builtin_neon_vsha1pq_u32:
+ case NEON::BI__builtin_neon_vsha1mq_u32:
+ case ARM::BI_MoveToCoprocessor:
+ case ARM::BI_MoveToCoprocessor2:
+ return false;
+ }
+ return true;
+}
+
Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
const CallExpr *E) {
if (auto Hint = GetValueForARMHint(BuiltinID))
@@ -3361,7 +3564,7 @@
: InlineAsm::get(FTy, ".inst 0x" + utohexstr(ZExtValue), "",
/*SideEffects=*/true);
- return Builder.CreateCall(Emit, {});
+ return Builder.CreateCall(Emit);
}
if (BuiltinID == ARM::BI__builtin_arm_dbg) {
@@ -3390,9 +3593,9 @@
if (BuiltinID == ARM::BI__clear_cache) {
assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments");
const FunctionDecl *FD = E->getDirectCallee();
- SmallVector<Value*, 2> Ops;
+ Value *Ops[2];
for (unsigned i = 0; i < 2; i++)
- Ops.push_back(EmitScalarExpr(E->getArg(i)));
+ Ops[i] = EmitScalarExpr(E->getArg(i));
llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType());
llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
StringRef Name = FD->getName();
@@ -3466,11 +3669,11 @@
: Intrinsic::arm_strexd);
llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, nullptr);
- Value *Tmp = CreateMemTemp(E->getArg(0)->getType());
+ Address Tmp = CreateMemTemp(E->getArg(0)->getType());
Value *Val = EmitScalarExpr(E->getArg(0));
Builder.CreateStore(Val, Tmp);
- Value *LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy));
+ Address LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy));
Val = Builder.CreateLoad(LdPtr);
Value *Arg0 = Builder.CreateExtractValue(Val, 0);
@@ -3505,7 +3708,7 @@
if (BuiltinID == ARM::BI__builtin_arm_clrex) {
Function *F = CGM.getIntrinsic(Intrinsic::arm_clrex);
- return Builder.CreateCall(F, {});
+ return Builder.CreateCall(F);
}
// CRC32
@@ -3589,9 +3792,16 @@
getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
assert(Error == ASTContext::GE_None && "Should not codegen an error");
+ auto getAlignmentValue32 = [&](Address addr) -> Value* {
+ return Builder.getInt32(addr.getAlignment().getQuantity());
+ };
+
+ Address PtrOp0 = Address::invalid();
+ Address PtrOp1 = Address::invalid();
SmallVector<Value*, 4> Ops;
- llvm::Value *Align = nullptr;
- for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) {
+ bool HasExtraArg = HasExtraNeonArgument(BuiltinID);
+ unsigned NumArgs = E->getNumArgs() - (HasExtraArg ? 1 : 0);
+ for (unsigned i = 0, e = NumArgs; i != e; i++) {
if (i == 0) {
switch (BuiltinID) {
case NEON::BI__builtin_neon_vld1_v:
@@ -3618,10 +3828,8 @@
case NEON::BI__builtin_neon_vst4q_lane_v:
// Get the alignment for the argument in addition to the value;
// we'll use it later.
- std::pair<llvm::Value*, unsigned> Src =
- EmitPointerWithAlignment(E->getArg(0));
- Ops.push_back(Src.first);
- Align = Builder.getInt32(Src.second);
+ PtrOp0 = EmitPointerWithAlignment(E->getArg(0));
+ Ops.push_back(PtrOp0.getPointer());
continue;
}
}
@@ -3644,10 +3852,8 @@
case NEON::BI__builtin_neon_vld4_dup_v:
// Get the alignment for the argument in addition to the value;
// we'll use it later.
- std::pair<llvm::Value*, unsigned> Src =
- EmitPointerWithAlignment(E->getArg(1));
- Ops.push_back(Src.first);
- Align = Builder.getInt32(Src.second);
+ PtrOp1 = EmitPointerWithAlignment(E->getArg(1));
+ Ops.push_back(PtrOp1.getPointer());
continue;
}
}
@@ -3666,8 +3872,7 @@
switch (BuiltinID) {
default: break;
- // vget_lane and vset_lane are not overloaded and do not have an extra
- // argument that specifies the vector type.
+
case NEON::BI__builtin_neon_vget_lane_i8:
case NEON::BI__builtin_neon_vget_lane_i16:
case NEON::BI__builtin_neon_vget_lane_i32:
@@ -3678,8 +3883,8 @@
case NEON::BI__builtin_neon_vgetq_lane_i32:
case NEON::BI__builtin_neon_vgetq_lane_i64:
case NEON::BI__builtin_neon_vgetq_lane_f32:
- return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
- "vget_lane");
+ return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane");
+
case NEON::BI__builtin_neon_vset_lane_i8:
case NEON::BI__builtin_neon_vset_lane_i16:
case NEON::BI__builtin_neon_vset_lane_i32:
@@ -3690,29 +3895,34 @@
case NEON::BI__builtin_neon_vsetq_lane_i32:
case NEON::BI__builtin_neon_vsetq_lane_i64:
case NEON::BI__builtin_neon_vsetq_lane_f32:
- Ops.push_back(EmitScalarExpr(E->getArg(2)));
return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
- // Non-polymorphic crypto instructions also not overloaded
case NEON::BI__builtin_neon_vsha1h_u32:
- Ops.push_back(EmitScalarExpr(E->getArg(0)));
return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops,
"vsha1h");
case NEON::BI__builtin_neon_vsha1cq_u32:
- Ops.push_back(EmitScalarExpr(E->getArg(2)));
return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops,
"vsha1h");
case NEON::BI__builtin_neon_vsha1pq_u32:
- Ops.push_back(EmitScalarExpr(E->getArg(2)));
return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops,
"vsha1h");
case NEON::BI__builtin_neon_vsha1mq_u32:
- Ops.push_back(EmitScalarExpr(E->getArg(2)));
return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops,
"vsha1h");
+
+ // The ARM _MoveToCoprocessor builtins put the input register value as
+ // the first argument, but the LLVM intrinsic expects it as the third one.
+ case ARM::BI_MoveToCoprocessor:
+ case ARM::BI_MoveToCoprocessor2: {
+ Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI_MoveToCoprocessor ?
+ Intrinsic::arm_mcr : Intrinsic::arm_mcr2);
+ return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0],
+ Ops[3], Ops[4], Ops[5]});
+ }
}
// Get the last argument, which specifies the vector type.
+ assert(HasExtraArg);
llvm::APSInt Result;
const Expr *Arg = E->getArg(E->getNumArgs()-1);
if (!Arg->isIntegerConstantExpr(Result, getContext()))
@@ -3754,7 +3964,7 @@
if (Builtin)
return EmitCommonNeonBuiltinExpr(
Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
- Builtin->NameHint, Builtin->TypeModifier, E, Ops, Align);
+ Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1);
unsigned Int;
switch (BuiltinID) {
@@ -3765,27 +3975,25 @@
if (VTy->getElementType()->isIntegerTy(64)) {
// Extract the other lane.
Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
- int Lane = cast<ConstantInt>(Ops[2])->getZExtValue();
+ uint32_t Lane = cast<ConstantInt>(Ops[2])->getZExtValue();
Value *SV = llvm::ConstantVector::get(ConstantInt::get(Int32Ty, 1-Lane));
Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
// Load the value as a one-element vector.
Ty = llvm::VectorType::get(VTy->getElementType(), 1);
- Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Ty);
+ llvm::Type *Tys[] = {Ty, Int8PtrTy};
+ Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Tys);
+ Value *Align = getAlignmentValue32(PtrOp0);
Value *Ld = Builder.CreateCall(F, {Ops[0], Align});
// Combine them.
- SmallVector<Constant*, 2> Indices;
- Indices.push_back(ConstantInt::get(Int32Ty, 1-Lane));
- Indices.push_back(ConstantInt::get(Int32Ty, Lane));
- SV = llvm::ConstantVector::get(Indices);
+ uint32_t Indices[] = {1 - Lane, Lane};
+ SV = llvm::ConstantDataVector::get(getLLVMContext(), Indices);
return Builder.CreateShuffleVector(Ops[1], Ld, SV, "vld1q_lane");
}
// fall through
case NEON::BI__builtin_neon_vld1_lane_v: {
Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
- Ty = llvm::PointerType::getUnqual(VTy->getElementType());
- Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
- LoadInst *Ld = Builder.CreateLoad(Ops[0]);
- Ld->setAlignment(cast<ConstantInt>(Align)->getZExtValue());
+ PtrOp0 = Builder.CreateElementBitCast(PtrOp0, VTy->getElementType());
+ Value *Ld = Builder.CreateLoad(PtrOp0);
return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane");
}
case NEON::BI__builtin_neon_vld2_dup_v:
@@ -3805,11 +4013,13 @@
break;
default: llvm_unreachable("unknown vld_dup intrinsic?");
}
- Function *F = CGM.getIntrinsic(Int, Ty);
+ llvm::Type *Tys[] = {Ty, Int8PtrTy};
+ Function *F = CGM.getIntrinsic(Int, Tys);
+ llvm::Value *Align = getAlignmentValue32(PtrOp1);
Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, "vld_dup");
Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
- return Builder.CreateStore(Ops[1], Ops[0]);
+ return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
}
switch (BuiltinID) {
case NEON::BI__builtin_neon_vld2_dup_v:
@@ -3823,7 +4033,8 @@
break;
default: llvm_unreachable("unknown vld_dup intrinsic?");
}
- Function *F = CGM.getIntrinsic(Int, Ty);
+ llvm::Type *Tys[] = {Ty, Int8PtrTy};
+ Function *F = CGM.getIntrinsic(Int, Tys);
llvm::StructType *STy = cast<llvm::StructType>(F->getReturnType());
SmallVector<Value*, 6> Args;
@@ -3832,7 +4043,7 @@
llvm::Constant *CI = ConstantInt::get(Int32Ty, 0);
Args.push_back(CI);
- Args.push_back(Align);
+ Args.push_back(getAlignmentValue32(PtrOp1));
Ops[1] = Builder.CreateCall(F, Args, "vld_dup");
// splat lane 0 to all elts in each vector of the result.
@@ -3845,7 +4056,7 @@
}
Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
- return Builder.CreateStore(Ops[1], Ops[0]);
+ return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
}
case NEON::BI__builtin_neon_vqrshrn_n_v:
Int =
@@ -3897,18 +4108,17 @@
Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2]));
Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
- Ops[2] = Align;
+ Ops[2] = getAlignmentValue32(PtrOp0);
+ llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()};
return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1,
- Ops[1]->getType()), Ops);
+ Tys), Ops);
}
// fall through
case NEON::BI__builtin_neon_vst1_lane_v: {
Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
- StoreInst *St = Builder.CreateStore(Ops[1],
- Builder.CreateBitCast(Ops[0], Ty));
- St->setAlignment(cast<ConstantInt>(Align)->getZExtValue());
+ auto St = Builder.CreateStore(Ops[1], Builder.CreateBitCast(PtrOp0, Ty));
return St;
}
case NEON::BI__builtin_neon_vtbl1_v:
@@ -3985,52 +4195,41 @@
// Determine the type of this overloaded NEON intrinsic.
NeonTypeFlags Type(Result.getZExtValue());
- llvm::VectorType *VTy = GetNeonType(&CGF, Type);
- llvm::Type *Ty = VTy;
+ llvm::VectorType *Ty = GetNeonType(&CGF, Type);
if (!Ty)
return nullptr;
- unsigned nElts = VTy->getNumElements();
-
CodeGen::CGBuilderTy &Builder = CGF.Builder;
// AArch64 scalar builtins are not overloaded, they do not have an extra
// argument that specifies the vector type, need to handle each case.
- SmallVector<Value *, 2> TblOps;
switch (BuiltinID) {
case NEON::BI__builtin_neon_vtbl1_v: {
- TblOps.push_back(Ops[0]);
- return packTBLDVectorList(CGF, TblOps, nullptr, Ops[1], Ty,
- Intrinsic::aarch64_neon_tbl1, "vtbl1");
+ return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 1), nullptr,
+ Ops[1], Ty, Intrinsic::aarch64_neon_tbl1,
+ "vtbl1");
}
case NEON::BI__builtin_neon_vtbl2_v: {
- TblOps.push_back(Ops[0]);
- TblOps.push_back(Ops[1]);
- return packTBLDVectorList(CGF, TblOps, nullptr, Ops[2], Ty,
- Intrinsic::aarch64_neon_tbl1, "vtbl1");
+ return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 2), nullptr,
+ Ops[2], Ty, Intrinsic::aarch64_neon_tbl1,
+ "vtbl1");
}
case NEON::BI__builtin_neon_vtbl3_v: {
- TblOps.push_back(Ops[0]);
- TblOps.push_back(Ops[1]);
- TblOps.push_back(Ops[2]);
- return packTBLDVectorList(CGF, TblOps, nullptr, Ops[3], Ty,
- Intrinsic::aarch64_neon_tbl2, "vtbl2");
+ return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 3), nullptr,
+ Ops[3], Ty, Intrinsic::aarch64_neon_tbl2,
+ "vtbl2");
}
case NEON::BI__builtin_neon_vtbl4_v: {
- TblOps.push_back(Ops[0]);
- TblOps.push_back(Ops[1]);
- TblOps.push_back(Ops[2]);
- TblOps.push_back(Ops[3]);
- return packTBLDVectorList(CGF, TblOps, nullptr, Ops[4], Ty,
- Intrinsic::aarch64_neon_tbl2, "vtbl2");
+ return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 4), nullptr,
+ Ops[4], Ty, Intrinsic::aarch64_neon_tbl2,
+ "vtbl2");
}
case NEON::BI__builtin_neon_vtbx1_v: {
- TblOps.push_back(Ops[1]);
- Value *TblRes = packTBLDVectorList(CGF, TblOps, nullptr, Ops[2], Ty,
- Intrinsic::aarch64_neon_tbl1, "vtbl1");
+ Value *TblRes =
+ packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 1), nullptr, Ops[2],
+ Ty, Intrinsic::aarch64_neon_tbl1, "vtbl1");
- llvm::Constant *Eight = ConstantInt::get(VTy->getElementType(), 8);
- Value* EightV = llvm::ConstantVector::getSplat(nElts, Eight);
+ llvm::Constant *EightV = ConstantInt::get(Ty, 8);
Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV);
CmpRes = Builder.CreateSExt(CmpRes, Ty);
@@ -4039,20 +4238,16 @@
return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx");
}
case NEON::BI__builtin_neon_vtbx2_v: {
- TblOps.push_back(Ops[1]);
- TblOps.push_back(Ops[2]);
- return packTBLDVectorList(CGF, TblOps, Ops[0], Ops[3], Ty,
- Intrinsic::aarch64_neon_tbx1, "vtbx1");
+ return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 2), Ops[0],
+ Ops[3], Ty, Intrinsic::aarch64_neon_tbx1,
+ "vtbx1");
}
case NEON::BI__builtin_neon_vtbx3_v: {
- TblOps.push_back(Ops[1]);
- TblOps.push_back(Ops[2]);
- TblOps.push_back(Ops[3]);
- Value *TblRes = packTBLDVectorList(CGF, TblOps, nullptr, Ops[4], Ty,
- Intrinsic::aarch64_neon_tbl2, "vtbl2");
+ Value *TblRes =
+ packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 3), nullptr, Ops[4],
+ Ty, Intrinsic::aarch64_neon_tbl2, "vtbl2");
- llvm::Constant *TwentyFour = ConstantInt::get(VTy->getElementType(), 24);
- Value* TwentyFourV = llvm::ConstantVector::getSplat(nElts, TwentyFour);
+ llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24);
Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4],
TwentyFourV);
CmpRes = Builder.CreateSExt(CmpRes, Ty);
@@ -4062,12 +4257,9 @@
return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx");
}
case NEON::BI__builtin_neon_vtbx4_v: {
- TblOps.push_back(Ops[1]);
- TblOps.push_back(Ops[2]);
- TblOps.push_back(Ops[3]);
- TblOps.push_back(Ops[4]);
- return packTBLDVectorList(CGF, TblOps, Ops[0], Ops[5], Ty,
- Intrinsic::aarch64_neon_tbx2, "vtbx2");
+ return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 4), Ops[0],
+ Ops[5], Ty, Intrinsic::aarch64_neon_tbx2,
+ "vtbx2");
}
case NEON::BI__builtin_neon_vqtbl1_v:
case NEON::BI__builtin_neon_vqtbl1q_v:
@@ -4112,15 +4304,6 @@
return Op;
}
-Value *CodeGenFunction::vectorWrapScalar8(Value *Op) {
- llvm::Type *VTy = llvm::VectorType::get(Int8Ty, 8);
- Op = Builder.CreateBitCast(Op, Int8Ty);
- Value *V = UndefValue::get(VTy);
- llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
- Op = Builder.CreateInsertElement(V, Op, CI);
- return Op;
-}
-
Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
const CallExpr *E) {
unsigned HintID = static_cast<unsigned>(-1);
@@ -4192,9 +4375,9 @@
if (BuiltinID == AArch64::BI__clear_cache) {
assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments");
const FunctionDecl *FD = E->getDirectCallee();
- SmallVector<Value*, 2> Ops;
+ Value *Ops[2];
for (unsigned i = 0; i < 2; i++)
- Ops.push_back(EmitScalarExpr(E->getArg(i)));
+ Ops[i] = EmitScalarExpr(E->getArg(i));
llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType());
llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
StringRef Name = FD->getName();
@@ -4253,14 +4436,11 @@
: Intrinsic::aarch64_stxp);
llvm::Type *STy = llvm::StructType::get(Int64Ty, Int64Ty, nullptr);
- Value *One = llvm::ConstantInt::get(Int32Ty, 1);
- Value *Tmp = Builder.CreateAlloca(ConvertType(E->getArg(0)->getType()),
- One);
- Value *Val = EmitScalarExpr(E->getArg(0));
- Builder.CreateStore(Val, Tmp);
+ Address Tmp = CreateMemTemp(E->getArg(0)->getType());
+ EmitAnyExprToMem(E->getArg(0), Tmp, Qualifiers(), /*init*/ true);
- Value *LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy));
- Val = Builder.CreateLoad(LdPtr);
+ Tmp = Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(STy));
+ llvm::Value *Val = Builder.CreateLoad(Tmp);
Value *Arg0 = Builder.CreateExtractValue(Val, 0);
Value *Arg1 = Builder.CreateExtractValue(Val, 1);
@@ -4295,7 +4475,12 @@
if (BuiltinID == AArch64::BI__builtin_arm_clrex) {
Function *F = CGM.getIntrinsic(Intrinsic::aarch64_clrex);
- return Builder.CreateCall(F, {});
+ return Builder.CreateCall(F);
+ }
+
+ if (BuiltinID == AArch64::BI__builtin_thread_pointer) {
+ Function *F = CGM.getIntrinsic(Intrinsic::aarch64_thread_pointer);
+ return Builder.CreateCall(F);
}
// CRC32
@@ -4409,12 +4594,12 @@
case NEON::BI__builtin_neon_vldrq_p128: {
llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128);
Value *Ptr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), Int128PTy);
- return Builder.CreateLoad(Ptr);
+ return Builder.CreateDefaultAlignedLoad(Ptr);
}
case NEON::BI__builtin_neon_vstrq_p128: {
llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128);
Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy);
- return Builder.CreateStore(EmitScalarExpr(E->getArg(1)), Ptr);
+ return Builder.CreateDefaultAlignedStore(EmitScalarExpr(E->getArg(1)), Ptr);
}
case NEON::BI__builtin_neon_vcvts_u32_f32:
case NEON::BI__builtin_neon_vcvtd_u64_f64:
@@ -4447,8 +4632,7 @@
return Builder.CreateSIToFP(Ops[0], FTy);
}
case NEON::BI__builtin_neon_vpaddd_s64: {
- llvm::Type *Ty =
- llvm::VectorType::get(llvm::Type::getInt64Ty(getLLVMContext()), 2);
+ llvm::Type *Ty = llvm::VectorType::get(Int64Ty, 2);
Value *Vec = EmitScalarExpr(E->getArg(0));
// The vector is v2f64, so make sure it's bitcast to that.
Vec = Builder.CreateBitCast(Vec, Ty, "v2i64");
@@ -4461,7 +4645,7 @@
}
case NEON::BI__builtin_neon_vpaddd_f64: {
llvm::Type *Ty =
- llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 2);
+ llvm::VectorType::get(DoubleTy, 2);
Value *Vec = EmitScalarExpr(E->getArg(0));
// The vector is v2f64, so make sure it's bitcast to that.
Vec = Builder.CreateBitCast(Vec, Ty, "v2f64");
@@ -4474,7 +4658,7 @@
}
case NEON::BI__builtin_neon_vpadds_f32: {
llvm::Type *Ty =
- llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 2);
+ llvm::VectorType::get(FloatTy, 2);
Value *Vec = EmitScalarExpr(E->getArg(0));
// The vector is v2f32, so make sure it's bitcast to that.
Vec = Builder.CreateBitCast(Vec, Ty, "v2f32");
@@ -4522,12 +4706,11 @@
ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT, "vcltz");
case NEON::BI__builtin_neon_vceqzd_u64: {
- llvm::Type *Ty = llvm::Type::getInt64Ty(getLLVMContext());
Ops.push_back(EmitScalarExpr(E->getArg(0)));
- Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
- Ops[0] = Builder.CreateICmp(llvm::ICmpInst::ICMP_EQ, Ops[0],
- llvm::Constant::getNullValue(Ty));
- return Builder.CreateSExt(Ops[0], Ty, "vceqzd");
+ Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
+ Ops[0] =
+ Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(Int64Ty));
+ return Builder.CreateSExt(Ops[0], Int64Ty, "vceqzd");
}
case NEON::BI__builtin_neon_vceqd_f64:
case NEON::BI__builtin_neon_vcled_f64:
@@ -4601,14 +4784,13 @@
}
case NEON::BI__builtin_neon_vtstd_s64:
case NEON::BI__builtin_neon_vtstd_u64: {
- llvm::Type *Ty = llvm::Type::getInt64Ty(getLLVMContext());
Ops.push_back(EmitScalarExpr(E->getArg(1)));
- Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
- Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
+ Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
+ Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
- llvm::Constant::getNullValue(Ty));
- return Builder.CreateSExt(Ops[0], Ty, "vtstd");
+ llvm::Constant::getNullValue(Int64Ty));
+ return Builder.CreateSExt(Ops[0], Int64Ty, "vtstd");
}
case NEON::BI__builtin_neon_vset_lane_i8:
case NEON::BI__builtin_neon_vset_lane_i16:
@@ -4631,89 +4813,80 @@
case NEON::BI__builtin_neon_vsetq_lane_f64:
// The vector type needs a cast for the v2f64 variant.
Ops[1] = Builder.CreateBitCast(Ops[1],
- llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 2));
+ llvm::VectorType::get(DoubleTy, 2));
Ops.push_back(EmitScalarExpr(E->getArg(2)));
return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
case NEON::BI__builtin_neon_vget_lane_i8:
case NEON::BI__builtin_neon_vdupb_lane_i8:
- Ops[0] = Builder.CreateBitCast(Ops[0],
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8));
+ Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 8));
return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
"vget_lane");
case NEON::BI__builtin_neon_vgetq_lane_i8:
case NEON::BI__builtin_neon_vdupb_laneq_i8:
- Ops[0] = Builder.CreateBitCast(Ops[0],
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16));
+ Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 16));
return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
"vgetq_lane");
case NEON::BI__builtin_neon_vget_lane_i16:
case NEON::BI__builtin_neon_vduph_lane_i16:
- Ops[0] = Builder.CreateBitCast(Ops[0],
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4));
+ Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 4));
return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
"vget_lane");
case NEON::BI__builtin_neon_vgetq_lane_i16:
case NEON::BI__builtin_neon_vduph_laneq_i16:
- Ops[0] = Builder.CreateBitCast(Ops[0],
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8));
+ Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 8));
return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
"vgetq_lane");
case NEON::BI__builtin_neon_vget_lane_i32:
case NEON::BI__builtin_neon_vdups_lane_i32:
- Ops[0] = Builder.CreateBitCast(
- Ops[0],
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 32), 2));
+ Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 2));
return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
"vget_lane");
case NEON::BI__builtin_neon_vdups_lane_f32:
Ops[0] = Builder.CreateBitCast(Ops[0],
- llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 2));
+ llvm::VectorType::get(FloatTy, 2));
return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
"vdups_lane");
case NEON::BI__builtin_neon_vgetq_lane_i32:
case NEON::BI__builtin_neon_vdups_laneq_i32:
- Ops[0] = Builder.CreateBitCast(Ops[0],
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 32), 4));
+ Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 4));
return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
"vgetq_lane");
case NEON::BI__builtin_neon_vget_lane_i64:
case NEON::BI__builtin_neon_vdupd_lane_i64:
- Ops[0] = Builder.CreateBitCast(Ops[0],
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 64), 1));
+ Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 1));
return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
"vget_lane");
case NEON::BI__builtin_neon_vdupd_lane_f64:
Ops[0] = Builder.CreateBitCast(Ops[0],
- llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 1));
+ llvm::VectorType::get(DoubleTy, 1));
return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
"vdupd_lane");
case NEON::BI__builtin_neon_vgetq_lane_i64:
case NEON::BI__builtin_neon_vdupd_laneq_i64:
- Ops[0] = Builder.CreateBitCast(Ops[0],
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 64), 2));
+ Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2));
return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
"vgetq_lane");
case NEON::BI__builtin_neon_vget_lane_f32:
Ops[0] = Builder.CreateBitCast(Ops[0],
- llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 2));
+ llvm::VectorType::get(FloatTy, 2));
return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
"vget_lane");
case NEON::BI__builtin_neon_vget_lane_f64:
Ops[0] = Builder.CreateBitCast(Ops[0],
- llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 1));
+ llvm::VectorType::get(DoubleTy, 1));
return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
"vget_lane");
case NEON::BI__builtin_neon_vgetq_lane_f32:
case NEON::BI__builtin_neon_vdups_laneq_f32:
Ops[0] = Builder.CreateBitCast(Ops[0],
- llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 4));
+ llvm::VectorType::get(FloatTy, 4));
return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
"vgetq_lane");
case NEON::BI__builtin_neon_vgetq_lane_f64:
case NEON::BI__builtin_neon_vdupd_laneq_f64:
Ops[0] = Builder.CreateBitCast(Ops[0],
- llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 2));
+ llvm::VectorType::get(DoubleTy, 2));
return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
"vgetq_lane");
case NEON::BI__builtin_neon_vaddd_s64:
@@ -4886,7 +5059,8 @@
if (Builtin)
return EmitCommonNeonBuiltinExpr(
Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
- Builtin->NameHint, Builtin->TypeModifier, E, Ops, nullptr);
+ Builtin->NameHint, Builtin->TypeModifier, E, Ops,
+ /*never use addresses*/ Address::invalid(), Address::invalid());
if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops))
return V;
@@ -5052,15 +5226,13 @@
Int = Intrinsic::aarch64_neon_fmaxnm;
return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm");
case NEON::BI__builtin_neon_vrecpss_f32: {
- llvm::Type *f32Type = llvm::Type::getFloatTy(getLLVMContext());
Ops.push_back(EmitScalarExpr(E->getArg(1)));
- return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, f32Type),
+ return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, FloatTy),
Ops, "vrecps");
}
case NEON::BI__builtin_neon_vrecpsd_f64: {
- llvm::Type *f64Type = llvm::Type::getDoubleTy(getLLVMContext());
Ops.push_back(EmitScalarExpr(E->getArg(1)));
- return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, f64Type),
+ return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, DoubleTy),
Ops, "vrecps");
}
case NEON::BI__builtin_neon_vqshrun_n_v:
@@ -5163,13 +5335,7 @@
case NEON::BI__builtin_neon_vcvtq_u32_v:
case NEON::BI__builtin_neon_vcvtq_s64_v:
case NEON::BI__builtin_neon_vcvtq_u64_v: {
- bool Double =
- (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64);
- llvm::Type *InTy =
- GetNeonType(this,
- NeonTypeFlags(Double ? NeonTypeFlags::Float64
- : NeonTypeFlags::Float32, false, quad));
- Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
+ Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type));
if (usgn)
return Builder.CreateFPToUI(Ops[0], Ty);
return Builder.CreateFPToSI(Ops[0], Ty);
@@ -5183,13 +5349,7 @@
case NEON::BI__builtin_neon_vcvta_u64_v:
case NEON::BI__builtin_neon_vcvtaq_u64_v: {
Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
- bool Double =
- (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64);
- llvm::Type *InTy =
- GetNeonType(this,
- NeonTypeFlags(Double ? NeonTypeFlags::Float64
- : NeonTypeFlags::Float32, false, quad));
- llvm::Type *Tys[2] = { Ty, InTy };
+ llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta");
}
case NEON::BI__builtin_neon_vcvtm_s32_v:
@@ -5201,13 +5361,7 @@
case NEON::BI__builtin_neon_vcvtm_u64_v:
case NEON::BI__builtin_neon_vcvtmq_u64_v: {
Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
- bool Double =
- (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64);
- llvm::Type *InTy =
- GetNeonType(this,
- NeonTypeFlags(Double ? NeonTypeFlags::Float64
- : NeonTypeFlags::Float32, false, quad));
- llvm::Type *Tys[2] = { Ty, InTy };
+ llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm");
}
case NEON::BI__builtin_neon_vcvtn_s32_v:
@@ -5219,13 +5373,7 @@
case NEON::BI__builtin_neon_vcvtn_u64_v:
case NEON::BI__builtin_neon_vcvtnq_u64_v: {
Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
- bool Double =
- (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64);
- llvm::Type *InTy =
- GetNeonType(this,
- NeonTypeFlags(Double ? NeonTypeFlags::Float64
- : NeonTypeFlags::Float32, false, quad));
- llvm::Type *Tys[2] = { Ty, InTy };
+ llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn");
}
case NEON::BI__builtin_neon_vcvtp_s32_v:
@@ -5237,13 +5385,7 @@
case NEON::BI__builtin_neon_vcvtp_u64_v:
case NEON::BI__builtin_neon_vcvtpq_u64_v: {
Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
- bool Double =
- (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64);
- llvm::Type *InTy =
- GetNeonType(this,
- NeonTypeFlags(Double ? NeonTypeFlags::Float64
- : NeonTypeFlags::Float32, false, quad));
- llvm::Type *Tys[2] = { Ty, InTy };
+ llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp");
}
case NEON::BI__builtin_neon_vmulx_v:
@@ -5294,232 +5436,192 @@
// FALLTHROUGH
case NEON::BI__builtin_neon_vaddv_s8: {
Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int8Ty, 8);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
- return Builder.CreateTrunc(Ops[0],
- llvm::IntegerType::get(getLLVMContext(), 8));
+ return Builder.CreateTrunc(Ops[0], Int8Ty);
}
case NEON::BI__builtin_neon_vaddv_u16:
usgn = true;
// FALLTHROUGH
case NEON::BI__builtin_neon_vaddv_s16: {
Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int16Ty, 4);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
- return Builder.CreateTrunc(Ops[0],
- llvm::IntegerType::get(getLLVMContext(), 16));
+ return Builder.CreateTrunc(Ops[0], Int16Ty);
}
case NEON::BI__builtin_neon_vaddvq_u8:
usgn = true;
// FALLTHROUGH
case NEON::BI__builtin_neon_vaddvq_s8: {
Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int8Ty, 16);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
- return Builder.CreateTrunc(Ops[0],
- llvm::IntegerType::get(getLLVMContext(), 8));
+ return Builder.CreateTrunc(Ops[0], Int8Ty);
}
case NEON::BI__builtin_neon_vaddvq_u16:
usgn = true;
// FALLTHROUGH
case NEON::BI__builtin_neon_vaddvq_s16: {
Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int16Ty, 8);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
- return Builder.CreateTrunc(Ops[0],
- llvm::IntegerType::get(getLLVMContext(), 16));
+ return Builder.CreateTrunc(Ops[0], Int16Ty);
}
case NEON::BI__builtin_neon_vmaxv_u8: {
Int = Intrinsic::aarch64_neon_umaxv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int8Ty, 8);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
- return Builder.CreateTrunc(Ops[0],
- llvm::IntegerType::get(getLLVMContext(), 8));
+ return Builder.CreateTrunc(Ops[0], Int8Ty);
}
case NEON::BI__builtin_neon_vmaxv_u16: {
Int = Intrinsic::aarch64_neon_umaxv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int16Ty, 4);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
- return Builder.CreateTrunc(Ops[0],
- llvm::IntegerType::get(getLLVMContext(), 16));
+ return Builder.CreateTrunc(Ops[0], Int16Ty);
}
case NEON::BI__builtin_neon_vmaxvq_u8: {
Int = Intrinsic::aarch64_neon_umaxv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int8Ty, 16);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
- return Builder.CreateTrunc(Ops[0],
- llvm::IntegerType::get(getLLVMContext(), 8));
+ return Builder.CreateTrunc(Ops[0], Int8Ty);
}
case NEON::BI__builtin_neon_vmaxvq_u16: {
Int = Intrinsic::aarch64_neon_umaxv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int16Ty, 8);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
- return Builder.CreateTrunc(Ops[0],
- llvm::IntegerType::get(getLLVMContext(), 16));
+ return Builder.CreateTrunc(Ops[0], Int16Ty);
}
case NEON::BI__builtin_neon_vmaxv_s8: {
Int = Intrinsic::aarch64_neon_smaxv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int8Ty, 8);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
- return Builder.CreateTrunc(Ops[0],
- llvm::IntegerType::get(getLLVMContext(), 8));
+ return Builder.CreateTrunc(Ops[0], Int8Ty);
}
case NEON::BI__builtin_neon_vmaxv_s16: {
Int = Intrinsic::aarch64_neon_smaxv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int16Ty, 4);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
- return Builder.CreateTrunc(Ops[0],
- llvm::IntegerType::get(getLLVMContext(), 16));
+ return Builder.CreateTrunc(Ops[0], Int16Ty);
}
case NEON::BI__builtin_neon_vmaxvq_s8: {
Int = Intrinsic::aarch64_neon_smaxv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int8Ty, 16);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
- return Builder.CreateTrunc(Ops[0],
- llvm::IntegerType::get(getLLVMContext(), 8));
+ return Builder.CreateTrunc(Ops[0], Int8Ty);
}
case NEON::BI__builtin_neon_vmaxvq_s16: {
Int = Intrinsic::aarch64_neon_smaxv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int16Ty, 8);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
- return Builder.CreateTrunc(Ops[0],
- llvm::IntegerType::get(getLLVMContext(), 16));
+ return Builder.CreateTrunc(Ops[0], Int16Ty);
}
case NEON::BI__builtin_neon_vminv_u8: {
Int = Intrinsic::aarch64_neon_uminv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int8Ty, 8);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
- return Builder.CreateTrunc(Ops[0],
- llvm::IntegerType::get(getLLVMContext(), 8));
+ return Builder.CreateTrunc(Ops[0], Int8Ty);
}
case NEON::BI__builtin_neon_vminv_u16: {
Int = Intrinsic::aarch64_neon_uminv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int16Ty, 4);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
- return Builder.CreateTrunc(Ops[0],
- llvm::IntegerType::get(getLLVMContext(), 16));
+ return Builder.CreateTrunc(Ops[0], Int16Ty);
}
case NEON::BI__builtin_neon_vminvq_u8: {
Int = Intrinsic::aarch64_neon_uminv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int8Ty, 16);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
- return Builder.CreateTrunc(Ops[0],
- llvm::IntegerType::get(getLLVMContext(), 8));
+ return Builder.CreateTrunc(Ops[0], Int8Ty);
}
case NEON::BI__builtin_neon_vminvq_u16: {
Int = Intrinsic::aarch64_neon_uminv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int16Ty, 8);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
- return Builder.CreateTrunc(Ops[0],
- llvm::IntegerType::get(getLLVMContext(), 16));
+ return Builder.CreateTrunc(Ops[0], Int16Ty);
}
case NEON::BI__builtin_neon_vminv_s8: {
Int = Intrinsic::aarch64_neon_sminv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int8Ty, 8);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
- return Builder.CreateTrunc(Ops[0],
- llvm::IntegerType::get(getLLVMContext(), 8));
+ return Builder.CreateTrunc(Ops[0], Int8Ty);
}
case NEON::BI__builtin_neon_vminv_s16: {
Int = Intrinsic::aarch64_neon_sminv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int16Ty, 4);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
- return Builder.CreateTrunc(Ops[0],
- llvm::IntegerType::get(getLLVMContext(), 16));
+ return Builder.CreateTrunc(Ops[0], Int16Ty);
}
case NEON::BI__builtin_neon_vminvq_s8: {
Int = Intrinsic::aarch64_neon_sminv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int8Ty, 16);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
- return Builder.CreateTrunc(Ops[0],
- llvm::IntegerType::get(getLLVMContext(), 8));
+ return Builder.CreateTrunc(Ops[0], Int8Ty);
}
case NEON::BI__builtin_neon_vminvq_s16: {
Int = Intrinsic::aarch64_neon_sminv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int16Ty, 8);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
- return Builder.CreateTrunc(Ops[0],
- llvm::IntegerType::get(getLLVMContext(), 16));
+ return Builder.CreateTrunc(Ops[0], Int16Ty);
}
case NEON::BI__builtin_neon_vmul_n_f64: {
Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
@@ -5528,80 +5630,68 @@
}
case NEON::BI__builtin_neon_vaddlv_u8: {
Int = Intrinsic::aarch64_neon_uaddlv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int8Ty, 8);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
- return Builder.CreateTrunc(Ops[0],
- llvm::IntegerType::get(getLLVMContext(), 16));
+ return Builder.CreateTrunc(Ops[0], Int16Ty);
}
case NEON::BI__builtin_neon_vaddlv_u16: {
Int = Intrinsic::aarch64_neon_uaddlv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int16Ty, 4);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
}
case NEON::BI__builtin_neon_vaddlvq_u8: {
Int = Intrinsic::aarch64_neon_uaddlv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int8Ty, 16);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
- return Builder.CreateTrunc(Ops[0],
- llvm::IntegerType::get(getLLVMContext(), 16));
+ return Builder.CreateTrunc(Ops[0], Int16Ty);
}
case NEON::BI__builtin_neon_vaddlvq_u16: {
Int = Intrinsic::aarch64_neon_uaddlv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int16Ty, 8);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
}
case NEON::BI__builtin_neon_vaddlv_s8: {
Int = Intrinsic::aarch64_neon_saddlv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int8Ty, 8);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
- return Builder.CreateTrunc(Ops[0],
- llvm::IntegerType::get(getLLVMContext(), 16));
+ return Builder.CreateTrunc(Ops[0], Int16Ty);
}
case NEON::BI__builtin_neon_vaddlv_s16: {
Int = Intrinsic::aarch64_neon_saddlv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int16Ty, 4);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
}
case NEON::BI__builtin_neon_vaddlvq_s8: {
Int = Intrinsic::aarch64_neon_saddlv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int8Ty, 16);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
- return Builder.CreateTrunc(Ops[0],
- llvm::IntegerType::get(getLLVMContext(), 16));
+ return Builder.CreateTrunc(Ops[0], Int16Ty);
}
case NEON::BI__builtin_neon_vaddlvq_s16: {
Int = Intrinsic::aarch64_neon_saddlv;
- Ty = llvm::IntegerType::get(getLLVMContext(), 32);
- VTy =
- llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8);
+ Ty = Int32Ty;
+ VTy = llvm::VectorType::get(Int16Ty, 8);
llvm::Type *Tys[2] = { Ty, VTy };
Ops.push_back(EmitScalarExpr(E->getArg(0)));
return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
@@ -5664,7 +5754,7 @@
Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN");
Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
- return Builder.CreateStore(Ops[1], Ops[0]);
+ return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
}
case NEON::BI__builtin_neon_vst1_x2_v:
case NEON::BI__builtin_neon_vst1q_x2_v:
@@ -5689,32 +5779,31 @@
Int = Intrinsic::aarch64_neon_st1x4;
break;
}
- SmallVector<Value *, 4> IntOps(Ops.begin()+1, Ops.end());
- IntOps.push_back(Ops[0]);
- return EmitNeonCall(CGM.getIntrinsic(Int, Tys), IntOps, "");
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
+ return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "");
}
case NEON::BI__builtin_neon_vld1_v:
case NEON::BI__builtin_neon_vld1q_v:
Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy));
- return Builder.CreateLoad(Ops[0]);
+ return Builder.CreateDefaultAlignedLoad(Ops[0]);
case NEON::BI__builtin_neon_vst1_v:
case NEON::BI__builtin_neon_vst1q_v:
Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy));
Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
- return Builder.CreateStore(Ops[1], Ops[0]);
+ return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
case NEON::BI__builtin_neon_vld1_lane_v:
case NEON::BI__builtin_neon_vld1q_lane_v:
Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
Ty = llvm::PointerType::getUnqual(VTy->getElementType());
Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
- Ops[0] = Builder.CreateLoad(Ops[0]);
+ Ops[0] = Builder.CreateDefaultAlignedLoad(Ops[0]);
return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane");
case NEON::BI__builtin_neon_vld1_dup_v:
case NEON::BI__builtin_neon_vld1q_dup_v: {
Value *V = UndefValue::get(Ty);
Ty = llvm::PointerType::getUnqual(VTy->getElementType());
Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
- Ops[0] = Builder.CreateLoad(Ops[0]);
+ Ops[0] = Builder.CreateDefaultAlignedLoad(Ops[0]);
llvm::Constant *CI = ConstantInt::get(Int32Ty, 0);
Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI);
return EmitNeonSplat(Ops[0], CI);
@@ -5724,7 +5813,8 @@
Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
- return Builder.CreateStore(Ops[1], Builder.CreateBitCast(Ops[0], Ty));
+ return Builder.CreateDefaultAlignedStore(Ops[1],
+ Builder.CreateBitCast(Ops[0], Ty));
case NEON::BI__builtin_neon_vld2_v:
case NEON::BI__builtin_neon_vld2q_v: {
llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
@@ -5734,7 +5824,7 @@
Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
Ops[0] = Builder.CreateBitCast(Ops[0],
llvm::PointerType::getUnqual(Ops[1]->getType()));
- return Builder.CreateStore(Ops[1], Ops[0]);
+ return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
}
case NEON::BI__builtin_neon_vld3_v:
case NEON::BI__builtin_neon_vld3q_v: {
@@ -5745,7 +5835,7 @@
Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
Ops[0] = Builder.CreateBitCast(Ops[0],
llvm::PointerType::getUnqual(Ops[1]->getType()));
- return Builder.CreateStore(Ops[1], Ops[0]);
+ return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
}
case NEON::BI__builtin_neon_vld4_v:
case NEON::BI__builtin_neon_vld4q_v: {
@@ -5756,7 +5846,7 @@
Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
Ops[0] = Builder.CreateBitCast(Ops[0],
llvm::PointerType::getUnqual(Ops[1]->getType()));
- return Builder.CreateStore(Ops[1], Ops[0]);
+ return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
}
case NEON::BI__builtin_neon_vld2_dup_v:
case NEON::BI__builtin_neon_vld2q_dup_v: {
@@ -5768,7 +5858,7 @@
Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
Ops[0] = Builder.CreateBitCast(Ops[0],
llvm::PointerType::getUnqual(Ops[1]->getType()));
- return Builder.CreateStore(Ops[1], Ops[0]);
+ return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
}
case NEON::BI__builtin_neon_vld3_dup_v:
case NEON::BI__builtin_neon_vld3q_dup_v: {
@@ -5780,7 +5870,7 @@
Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
Ops[0] = Builder.CreateBitCast(Ops[0],
llvm::PointerType::getUnqual(Ops[1]->getType()));
- return Builder.CreateStore(Ops[1], Ops[0]);
+ return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
}
case NEON::BI__builtin_neon_vld4_dup_v:
case NEON::BI__builtin_neon_vld4q_dup_v: {
@@ -5792,7 +5882,7 @@
Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
Ops[0] = Builder.CreateBitCast(Ops[0],
llvm::PointerType::getUnqual(Ops[1]->getType()));
- return Builder.CreateStore(Ops[1], Ops[0]);
+ return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
}
case NEON::BI__builtin_neon_vld2_lane_v:
case NEON::BI__builtin_neon_vld2q_lane_v: {
@@ -5802,12 +5892,11 @@
Ops.erase(Ops.begin()+1);
Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
- Ops[3] = Builder.CreateZExt(Ops[3],
- llvm::IntegerType::get(getLLVMContext(), 64));
+ Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld2_lane");
Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
- return Builder.CreateStore(Ops[1], Ops[0]);
+ return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
}
case NEON::BI__builtin_neon_vld3_lane_v:
case NEON::BI__builtin_neon_vld3q_lane_v: {
@@ -5818,12 +5907,11 @@
Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
- Ops[4] = Builder.CreateZExt(Ops[4],
- llvm::IntegerType::get(getLLVMContext(), 64));
+ Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty);
Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane");
Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
- return Builder.CreateStore(Ops[1], Ops[0]);
+ return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
}
case NEON::BI__builtin_neon_vld4_lane_v:
case NEON::BI__builtin_neon_vld4q_lane_v: {
@@ -5835,12 +5923,11 @@
Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
- Ops[5] = Builder.CreateZExt(Ops[5],
- llvm::IntegerType::get(getLLVMContext(), 64));
+ Ops[5] = Builder.CreateZExt(Ops[5], Int64Ty);
Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld4_lane");
Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
- return Builder.CreateStore(Ops[1], Ops[0]);
+ return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
}
case NEON::BI__builtin_neon_vst2_v:
case NEON::BI__builtin_neon_vst2q_v: {
@@ -5854,8 +5941,7 @@
case NEON::BI__builtin_neon_vst2q_lane_v: {
Ops.push_back(Ops[0]);
Ops.erase(Ops.begin());
- Ops[2] = Builder.CreateZExt(Ops[2],
- llvm::IntegerType::get(getLLVMContext(), 64));
+ Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty);
llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2lane, Tys),
Ops, "");
@@ -5872,8 +5958,7 @@
case NEON::BI__builtin_neon_vst3q_lane_v: {
Ops.push_back(Ops[0]);
Ops.erase(Ops.begin());
- Ops[3] = Builder.CreateZExt(Ops[3],
- llvm::IntegerType::get(getLLVMContext(), 64));
+ Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3lane, Tys),
Ops, "");
@@ -5890,8 +5975,7 @@
case NEON::BI__builtin_neon_vst4q_lane_v: {
Ops.push_back(Ops[0]);
Ops.erase(Ops.begin());
- Ops[4] = Builder.CreateZExt(Ops[4],
- llvm::IntegerType::get(getLLVMContext(), 64));
+ Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty);
llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4lane, Tys),
Ops, "");
@@ -5912,7 +5996,7 @@
Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
SV = llvm::ConstantVector::get(Indices);
SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn");
- SV = Builder.CreateStore(SV, Addr);
+ SV = Builder.CreateDefaultAlignedStore(SV, Addr);
}
return SV;
}
@@ -5931,7 +6015,7 @@
Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
SV = llvm::ConstantVector::get(Indices);
SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp");
- SV = Builder.CreateStore(SV, Addr);
+ SV = Builder.CreateDefaultAlignedStore(SV, Addr);
}
return SV;
}
@@ -5951,7 +6035,7 @@
Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
SV = llvm::ConstantVector::get(Indices);
SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip");
- SV = Builder.CreateStore(SV, Addr);
+ SV = Builder.CreateDefaultAlignedStore(SV, Addr);
}
return SV;
}
@@ -6028,6 +6112,31 @@
Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
const CallExpr *E) {
+ if (BuiltinID == X86::BI__builtin_ms_va_start ||
+ BuiltinID == X86::BI__builtin_ms_va_end)
+ return EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).getPointer(),
+ BuiltinID == X86::BI__builtin_ms_va_start);
+ if (BuiltinID == X86::BI__builtin_ms_va_copy) {
+ // Lower this manually. We can't reliably determine whether or not any
+ // given va_copy() is for a Win64 va_list from the calling convention
+ // alone, because it's legal to do this from a System V ABI function.
+ // With opaque pointer types, we won't have enough information in LLVM
+ // IR to determine this from the argument types, either. Best to do it
+ // now, while we have enough information.
+ Address DestAddr = EmitMSVAListRef(E->getArg(0));
+ Address SrcAddr = EmitMSVAListRef(E->getArg(1));
+
+ llvm::Type *BPP = Int8PtrPtrTy;
+
+ DestAddr = Address(Builder.CreateBitCast(DestAddr.getPointer(), BPP, "cp"),
+ DestAddr.getAlignment());
+ SrcAddr = Address(Builder.CreateBitCast(SrcAddr.getPointer(), BPP, "ap"),
+ SrcAddr.getAlignment());
+
+ Value *ArgPtr = Builder.CreateLoad(SrcAddr, "ap.val");
+ return Builder.CreateStore(ArgPtr, DestAddr);
+ }
+
SmallVector<Value*, 4> Ops;
// Find out if any arguments are required to be integer constant expressions.
@@ -6053,14 +6162,96 @@
switch (BuiltinID) {
default: return nullptr;
+ case X86::BI__builtin_cpu_supports: {
+ const Expr *FeatureExpr = E->getArg(0)->IgnoreParenCasts();
+ StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString();
+
+ // TODO: When/if this becomes more than x86 specific then use a TargetInfo
+ // based mapping.
+ // Processor features and mapping to processor feature value.
+ enum X86Features {
+ CMOV = 0,
+ MMX,
+ POPCNT,
+ SSE,
+ SSE2,
+ SSE3,
+ SSSE3,
+ SSE4_1,
+ SSE4_2,
+ AVX,
+ AVX2,
+ SSE4_A,
+ FMA4,
+ XOP,
+ FMA,
+ AVX512F,
+ BMI,
+ BMI2,
+ MAX
+ };
+
+ X86Features Feature = StringSwitch<X86Features>(FeatureStr)
+ .Case("cmov", X86Features::CMOV)
+ .Case("mmx", X86Features::MMX)
+ .Case("popcnt", X86Features::POPCNT)
+ .Case("sse", X86Features::SSE)
+ .Case("sse2", X86Features::SSE2)
+ .Case("sse3", X86Features::SSE3)
+ .Case("sse4.1", X86Features::SSE4_1)
+ .Case("sse4.2", X86Features::SSE4_2)
+ .Case("avx", X86Features::AVX)
+ .Case("avx2", X86Features::AVX2)
+ .Case("sse4a", X86Features::SSE4_A)
+ .Case("fma4", X86Features::FMA4)
+ .Case("xop", X86Features::XOP)
+ .Case("fma", X86Features::FMA)
+ .Case("avx512f", X86Features::AVX512F)
+ .Case("bmi", X86Features::BMI)
+ .Case("bmi2", X86Features::BMI2)
+ .Default(X86Features::MAX);
+ assert(Feature != X86Features::MAX && "Invalid feature!");
+
+ // Matching the struct layout from the compiler-rt/libgcc structure that is
+ // filled in:
+ // unsigned int __cpu_vendor;
+ // unsigned int __cpu_type;
+ // unsigned int __cpu_subtype;
+ // unsigned int __cpu_features[1];
+ llvm::Type *STy = llvm::StructType::get(
+ Int32Ty, Int32Ty, Int32Ty, llvm::ArrayType::get(Int32Ty, 1), nullptr);
+
+ // Grab the global __cpu_model.
+ llvm::Constant *CpuModel = CGM.CreateRuntimeVariable(STy, "__cpu_model");
+
+ // Grab the first (0th) element from the field __cpu_features off of the
+ // global in the struct STy.
+ Value *Idxs[] = {
+ ConstantInt::get(Int32Ty, 0),
+ ConstantInt::get(Int32Ty, 3),
+ ConstantInt::get(Int32Ty, 0)
+ };
+ Value *CpuFeatures = Builder.CreateGEP(STy, CpuModel, Idxs);
+ Value *Features = Builder.CreateAlignedLoad(CpuFeatures,
+ CharUnits::fromQuantity(4));
+
+ // Check the value of the bit corresponding to the feature requested.
+ Value *Bitset = Builder.CreateAnd(
+ Features, llvm::ConstantInt::get(Int32Ty, 1 << Feature));
+ return Builder.CreateICmpNE(Bitset, llvm::ConstantInt::get(Int32Ty, 0));
+ }
case X86::BI_mm_prefetch: {
- Value *Address = EmitScalarExpr(E->getArg(0));
+ Value *Address = Ops[0];
Value *RW = ConstantInt::get(Int32Ty, 0);
- Value *Locality = EmitScalarExpr(E->getArg(1));
+ Value *Locality = Ops[1];
Value *Data = ConstantInt::get(Int32Ty, 1);
Value *F = CGM.getIntrinsic(Intrinsic::prefetch);
return Builder.CreateCall(F, {Address, RW, Locality, Data});
}
+ case X86::BI__builtin_ia32_undef128:
+ case X86::BI__builtin_ia32_undef256:
+ case X86::BI__builtin_ia32_undef512:
+ return UndefValue::get(ConvertType(E->getType()));
case X86::BI__builtin_ia32_vec_init_v8qi:
case X86::BI__builtin_ia32_vec_init_v4hi:
case X86::BI__builtin_ia32_vec_init_v2si:
@@ -6070,17 +6261,57 @@
return Builder.CreateExtractElement(Ops[0],
llvm::ConstantInt::get(Ops[1]->getType(), 0));
case X86::BI__builtin_ia32_ldmxcsr: {
- Value *Tmp = CreateMemTemp(E->getArg(0)->getType());
+ Address Tmp = CreateMemTemp(E->getArg(0)->getType());
Builder.CreateStore(Ops[0], Tmp);
return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr),
- Builder.CreateBitCast(Tmp, Int8PtrTy));
+ Builder.CreateBitCast(Tmp.getPointer(), Int8PtrTy));
}
case X86::BI__builtin_ia32_stmxcsr: {
- Value *Tmp = CreateMemTemp(E->getType());
+ Address Tmp = CreateMemTemp(E->getType());
Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr),
- Builder.CreateBitCast(Tmp, Int8PtrTy));
+ Builder.CreateBitCast(Tmp.getPointer(), Int8PtrTy));
return Builder.CreateLoad(Tmp, "stmxcsr");
}
+ case X86::BI__builtin_ia32_xsave:
+ case X86::BI__builtin_ia32_xsave64:
+ case X86::BI__builtin_ia32_xrstor:
+ case X86::BI__builtin_ia32_xrstor64:
+ case X86::BI__builtin_ia32_xsaveopt:
+ case X86::BI__builtin_ia32_xsaveopt64:
+ case X86::BI__builtin_ia32_xrstors:
+ case X86::BI__builtin_ia32_xrstors64:
+ case X86::BI__builtin_ia32_xsavec:
+ case X86::BI__builtin_ia32_xsavec64:
+ case X86::BI__builtin_ia32_xsaves:
+ case X86::BI__builtin_ia32_xsaves64: {
+ Intrinsic::ID ID;
+#define INTRINSIC_X86_XSAVE_ID(NAME) \
+ case X86::BI__builtin_ia32_##NAME: \
+ ID = Intrinsic::x86_##NAME; \
+ break
+ switch (BuiltinID) {
+ default: llvm_unreachable("Unsupported intrinsic!");
+ INTRINSIC_X86_XSAVE_ID(xsave);
+ INTRINSIC_X86_XSAVE_ID(xsave64);
+ INTRINSIC_X86_XSAVE_ID(xrstor);
+ INTRINSIC_X86_XSAVE_ID(xrstor64);
+ INTRINSIC_X86_XSAVE_ID(xsaveopt);
+ INTRINSIC_X86_XSAVE_ID(xsaveopt64);
+ INTRINSIC_X86_XSAVE_ID(xrstors);
+ INTRINSIC_X86_XSAVE_ID(xrstors64);
+ INTRINSIC_X86_XSAVE_ID(xsavec);
+ INTRINSIC_X86_XSAVE_ID(xsavec64);
+ INTRINSIC_X86_XSAVE_ID(xsaves);
+ INTRINSIC_X86_XSAVE_ID(xsaves64);
+ }
+#undef INTRINSIC_X86_XSAVE_ID
+ Value *Mhi = Builder.CreateTrunc(
+ Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, 32)), Int32Ty);
+ Value *Mlo = Builder.CreateTrunc(Ops[1], Int32Ty);
+ Ops[1] = Mhi;
+ Ops.push_back(Mlo);
+ return Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
+ }
case X86::BI__builtin_ia32_storehps:
case X86::BI__builtin_ia32_storelps: {
llvm::Type *PtrTy = llvm::PointerType::getUnqual(Int64Ty);
@@ -6096,7 +6327,7 @@
// cast pointer to i64 & store
Ops[0] = Builder.CreateBitCast(Ops[0], PtrTy);
- return Builder.CreateStore(Ops[1], Ops[0]);
+ return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
}
case X86::BI__builtin_ia32_palignr128:
case X86::BI__builtin_ia32_palignr256: {
@@ -6117,21 +6348,23 @@
// but less than two lanes, convert to shifting in zeroes.
if (ShiftVal > NumLaneElts) {
ShiftVal -= NumLaneElts;
+ Ops[1] = Ops[0];
Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType());
}
- SmallVector<llvm::Constant*, 32> Indices;
+ uint32_t Indices[32];
// 256-bit palignr operates on 128-bit lanes so we need to handle that
for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
for (unsigned i = 0; i != NumLaneElts; ++i) {
unsigned Idx = ShiftVal + i;
if (Idx >= NumLaneElts)
Idx += NumElts - NumLaneElts; // End of lane, switch operand.
- Indices.push_back(llvm::ConstantInt::get(Int32Ty, Idx + l));
+ Indices[l + i] = Idx + l;
}
}
- Value* SV = llvm::ConstantVector::get(Indices);
+ Value *SV = llvm::ConstantDataVector::get(getLLVMContext(),
+ makeArrayRef(Indices, NumElts));
return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr");
}
case X86::BI__builtin_ia32_pslldqi256: {
@@ -6142,13 +6375,13 @@
if (shiftVal >= 16)
return llvm::Constant::getNullValue(ConvertType(E->getType()));
- SmallVector<llvm::Constant*, 32> Indices;
+ uint32_t Indices[32];
// 256-bit pslldq operates on 128-bit lanes so we need to handle that
for (unsigned l = 0; l != 32; l += 16) {
for (unsigned i = 0; i != 16; ++i) {
unsigned Idx = 32 + i - shiftVal;
if (Idx < 32) Idx -= 16; // end of lane, switch operand.
- Indices.push_back(llvm::ConstantInt::get(Int32Ty, Idx + l));
+ Indices[l + i] = Idx + l;
}
}
@@ -6156,7 +6389,7 @@
Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast");
Value *Zero = llvm::Constant::getNullValue(VecTy);
- Value *SV = llvm::ConstantVector::get(Indices);
+ Value *SV = llvm::ConstantDataVector::get(getLLVMContext(), Indices);
SV = Builder.CreateShuffleVector(Zero, Ops[0], SV, "pslldq");
llvm::Type *ResultType = ConvertType(E->getType());
return Builder.CreateBitCast(SV, ResultType, "cast");
@@ -6169,13 +6402,13 @@
if (shiftVal >= 16)
return llvm::Constant::getNullValue(ConvertType(E->getType()));
- SmallVector<llvm::Constant*, 32> Indices;
+ uint32_t Indices[32];
// 256-bit psrldq operates on 128-bit lanes so we need to handle that
for (unsigned l = 0; l != 32; l += 16) {
for (unsigned i = 0; i != 16; ++i) {
unsigned Idx = i + shiftVal;
if (Idx >= 16) Idx += 16; // end of lane, switch operand.
- Indices.push_back(llvm::ConstantInt::get(Int32Ty, Idx + l));
+ Indices[l + i] = Idx + l;
}
}
@@ -6183,7 +6416,7 @@
Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast");
Value *Zero = llvm::Constant::getNullValue(VecTy);
- Value *SV = llvm::ConstantVector::get(Indices);
+ Value *SV = llvm::ConstantDataVector::get(getLLVMContext(), Indices);
SV = Builder.CreateShuffleVector(Ops[0], Zero, SV, "psrldq");
llvm::Type *ResultType = ConvertType(E->getType());
return Builder.CreateBitCast(SV, ResultType, "cast");
@@ -6203,7 +6436,7 @@
Value *BC = Builder.CreateBitCast(Ops[0],
llvm::PointerType::getUnqual(Ops[1]->getType()),
"cast");
- StoreInst *SI = Builder.CreateStore(Ops[1], BC);
+ StoreInst *SI = Builder.CreateDefaultAlignedStore(Ops[1], BC);
SI->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node);
// If the operand is an integer, we can't assume alignment. Otherwise,
@@ -6254,8 +6487,9 @@
break;
}
- Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID), {});
- Builder.CreateStore(Builder.CreateExtractValue(Call, 0), Ops[0]);
+ Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID));
+ Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 0),
+ Ops[0]);
return Builder.CreateExtractValue(Call, 1);
}
// SSE comparison intrisics
@@ -6422,6 +6656,11 @@
switch (BuiltinID) {
default: return nullptr;
+ // __builtin_ppc_get_timebase is GCC 4.8+'s PowerPC-specific name for what we
+ // call __builtin_readcyclecounter.
+ case PPC::BI__builtin_ppc_get_timebase:
+ return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::readcyclecounter));
+
// vec_ld, vec_lvsl, vec_lvsr
case PPC::BI__builtin_altivec_lvx:
case PPC::BI__builtin_altivec_lvxl:
@@ -6512,6 +6751,103 @@
llvm::Function *F = CGM.getIntrinsic(ID);
return Builder.CreateCall(F, Ops, "");
}
+ // Square root
+ case PPC::BI__builtin_vsx_xvsqrtsp:
+ case PPC::BI__builtin_vsx_xvsqrtdp: {
+ llvm::Type *ResultType = ConvertType(E->getType());
+ Value *X = EmitScalarExpr(E->getArg(0));
+ ID = Intrinsic::sqrt;
+ llvm::Function *F = CGM.getIntrinsic(ID, ResultType);
+ return Builder.CreateCall(F, X);
+ }
+ // Count leading zeros
+ case PPC::BI__builtin_altivec_vclzb:
+ case PPC::BI__builtin_altivec_vclzh:
+ case PPC::BI__builtin_altivec_vclzw:
+ case PPC::BI__builtin_altivec_vclzd: {
+ llvm::Type *ResultType = ConvertType(E->getType());
+ Value *X = EmitScalarExpr(E->getArg(0));
+ Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false);
+ Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType);
+ return Builder.CreateCall(F, {X, Undef});
+ }
+ // Copy sign
+ case PPC::BI__builtin_vsx_xvcpsgnsp:
+ case PPC::BI__builtin_vsx_xvcpsgndp: {
+ llvm::Type *ResultType = ConvertType(E->getType());
+ Value *X = EmitScalarExpr(E->getArg(0));
+ Value *Y = EmitScalarExpr(E->getArg(1));
+ ID = Intrinsic::copysign;
+ llvm::Function *F = CGM.getIntrinsic(ID, ResultType);
+ return Builder.CreateCall(F, {X, Y});
+ }
+ // Rounding/truncation
+ case PPC::BI__builtin_vsx_xvrspip:
+ case PPC::BI__builtin_vsx_xvrdpip:
+ case PPC::BI__builtin_vsx_xvrdpim:
+ case PPC::BI__builtin_vsx_xvrspim:
+ case PPC::BI__builtin_vsx_xvrdpi:
+ case PPC::BI__builtin_vsx_xvrspi:
+ case PPC::BI__builtin_vsx_xvrdpic:
+ case PPC::BI__builtin_vsx_xvrspic:
+ case PPC::BI__builtin_vsx_xvrdpiz:
+ case PPC::BI__builtin_vsx_xvrspiz: {
+ llvm::Type *ResultType = ConvertType(E->getType());
+ Value *X = EmitScalarExpr(E->getArg(0));
+ if (BuiltinID == PPC::BI__builtin_vsx_xvrdpim ||
+ BuiltinID == PPC::BI__builtin_vsx_xvrspim)
+ ID = Intrinsic::floor;
+ else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpi ||
+ BuiltinID == PPC::BI__builtin_vsx_xvrspi)
+ ID = Intrinsic::round;
+ else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpic ||
+ BuiltinID == PPC::BI__builtin_vsx_xvrspic)
+ ID = Intrinsic::nearbyint;
+ else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpip ||
+ BuiltinID == PPC::BI__builtin_vsx_xvrspip)
+ ID = Intrinsic::ceil;
+ else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpiz ||
+ BuiltinID == PPC::BI__builtin_vsx_xvrspiz)
+ ID = Intrinsic::trunc;
+ llvm::Function *F = CGM.getIntrinsic(ID, ResultType);
+ return Builder.CreateCall(F, X);
+ }
+ // FMA variations
+ case PPC::BI__builtin_vsx_xvmaddadp:
+ case PPC::BI__builtin_vsx_xvmaddasp:
+ case PPC::BI__builtin_vsx_xvnmaddadp:
+ case PPC::BI__builtin_vsx_xvnmaddasp:
+ case PPC::BI__builtin_vsx_xvmsubadp:
+ case PPC::BI__builtin_vsx_xvmsubasp:
+ case PPC::BI__builtin_vsx_xvnmsubadp:
+ case PPC::BI__builtin_vsx_xvnmsubasp: {
+ llvm::Type *ResultType = ConvertType(E->getType());
+ Value *X = EmitScalarExpr(E->getArg(0));
+ Value *Y = EmitScalarExpr(E->getArg(1));
+ Value *Z = EmitScalarExpr(E->getArg(2));
+ Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
+ llvm::Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType);
+ switch (BuiltinID) {
+ case PPC::BI__builtin_vsx_xvmaddadp:
+ case PPC::BI__builtin_vsx_xvmaddasp:
+ return Builder.CreateCall(F, {X, Y, Z});
+ case PPC::BI__builtin_vsx_xvnmaddadp:
+ case PPC::BI__builtin_vsx_xvnmaddasp:
+ return Builder.CreateFSub(Zero,
+ Builder.CreateCall(F, {X, Y, Z}), "sub");
+ case PPC::BI__builtin_vsx_xvmsubadp:
+ case PPC::BI__builtin_vsx_xvmsubasp:
+ return Builder.CreateCall(F,
+ {X, Y, Builder.CreateFSub(Zero, Z, "sub")});
+ case PPC::BI__builtin_vsx_xvnmsubadp:
+ case PPC::BI__builtin_vsx_xvnmsubasp:
+ Value *FsubRes =
+ Builder.CreateCall(F, {X, Y, Builder.CreateFSub(Zero, Z, "sub")});
+ return Builder.CreateFSub(Zero, FsubRes, "sub");
+ }
+ llvm_unreachable("Unknown FMA operation");
+ return nullptr; // Suppress no-return warning
+ }
}
}
@@ -6548,16 +6884,15 @@
return CGF.Builder.CreateCall(F, {Src0, Src1});
}
-Value *CodeGenFunction::EmitR600BuiltinExpr(unsigned BuiltinID,
- const CallExpr *E) {
+Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
+ const CallExpr *E) {
switch (BuiltinID) {
- case R600::BI__builtin_amdgpu_div_scale:
- case R600::BI__builtin_amdgpu_div_scalef: {
+ case AMDGPU::BI__builtin_amdgpu_div_scale:
+ case AMDGPU::BI__builtin_amdgpu_div_scalef: {
// Translate from the intrinsics's struct return to the builtin's out
// argument.
- std::pair<llvm::Value *, unsigned> FlagOutPtr
- = EmitPointerWithAlignment(E->getArg(3));
+ Address FlagOutPtr = EmitPointerWithAlignment(E->getArg(3));
llvm::Value *X = EmitScalarExpr(E->getArg(0));
llvm::Value *Y = EmitScalarExpr(E->getArg(1));
@@ -6572,15 +6907,14 @@
llvm::Value *Flag = Builder.CreateExtractValue(Tmp, 1);
llvm::Type *RealFlagType
- = FlagOutPtr.first->getType()->getPointerElementType();
+ = FlagOutPtr.getPointer()->getType()->getPointerElementType();
llvm::Value *FlagExt = Builder.CreateZExt(Flag, RealFlagType);
- llvm::StoreInst *FlagStore = Builder.CreateStore(FlagExt, FlagOutPtr.first);
- FlagStore->setAlignment(FlagOutPtr.second);
+ Builder.CreateStore(FlagExt, FlagOutPtr);
return Result;
}
- case R600::BI__builtin_amdgpu_div_fmas:
- case R600::BI__builtin_amdgpu_div_fmasf: {
+ case AMDGPU::BI__builtin_amdgpu_div_fmas:
+ case AMDGPU::BI__builtin_amdgpu_div_fmasf: {
llvm::Value *Src0 = EmitScalarExpr(E->getArg(0));
llvm::Value *Src1 = EmitScalarExpr(E->getArg(1));
llvm::Value *Src2 = EmitScalarExpr(E->getArg(2));
@@ -6591,26 +6925,26 @@
llvm::Value *Src3ToBool = Builder.CreateIsNotNull(Src3);
return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool});
}
- case R600::BI__builtin_amdgpu_div_fixup:
- case R600::BI__builtin_amdgpu_div_fixupf:
+ case AMDGPU::BI__builtin_amdgpu_div_fixup:
+ case AMDGPU::BI__builtin_amdgpu_div_fixupf:
return emitTernaryFPBuiltin(*this, E, Intrinsic::AMDGPU_div_fixup);
- case R600::BI__builtin_amdgpu_trig_preop:
- case R600::BI__builtin_amdgpu_trig_preopf:
+ case AMDGPU::BI__builtin_amdgpu_trig_preop:
+ case AMDGPU::BI__builtin_amdgpu_trig_preopf:
return emitFPIntBuiltin(*this, E, Intrinsic::AMDGPU_trig_preop);
- case R600::BI__builtin_amdgpu_rcp:
- case R600::BI__builtin_amdgpu_rcpf:
+ case AMDGPU::BI__builtin_amdgpu_rcp:
+ case AMDGPU::BI__builtin_amdgpu_rcpf:
return emitUnaryFPBuiltin(*this, E, Intrinsic::AMDGPU_rcp);
- case R600::BI__builtin_amdgpu_rsq:
- case R600::BI__builtin_amdgpu_rsqf:
+ case AMDGPU::BI__builtin_amdgpu_rsq:
+ case AMDGPU::BI__builtin_amdgpu_rsqf:
return emitUnaryFPBuiltin(*this, E, Intrinsic::AMDGPU_rsq);
- case R600::BI__builtin_amdgpu_rsq_clamped:
- case R600::BI__builtin_amdgpu_rsq_clampedf:
+ case AMDGPU::BI__builtin_amdgpu_rsq_clamped:
+ case AMDGPU::BI__builtin_amdgpu_rsq_clampedf:
return emitUnaryFPBuiltin(*this, E, Intrinsic::AMDGPU_rsq_clamped);
- case R600::BI__builtin_amdgpu_ldexp:
- case R600::BI__builtin_amdgpu_ldexpf:
+ case AMDGPU::BI__builtin_amdgpu_ldexp:
+ case AMDGPU::BI__builtin_amdgpu_ldexpf:
return emitFPIntBuiltin(*this, E, Intrinsic::AMDGPU_ldexp);
- case R600::BI__builtin_amdgpu_class:
- case R600::BI__builtin_amdgpu_classf:
+ case AMDGPU::BI__builtin_amdgpu_class:
+ case AMDGPU::BI__builtin_amdgpu_classf:
return emitFPIntBuiltin(*this, E, Intrinsic::AMDGPU_class);
default:
return nullptr;
@@ -6627,7 +6961,7 @@
SmallVector<Value *, 8> Args(NumArgs);
for (unsigned I = 0; I < NumArgs; ++I)
Args[I] = CGF.EmitScalarExpr(E->getArg(I));
- Value *CCPtr = CGF.EmitScalarExpr(E->getArg(NumArgs));
+ Address CCPtr = CGF.EmitPointerWithAlignment(E->getArg(NumArgs));
Value *F = CGF.CGM.getIntrinsic(IntrinsicID);
Value *Call = CGF.Builder.CreateCall(F, Args);
Value *CC = CGF.Builder.CreateExtractValue(Call, 1);
@@ -6859,3 +7193,97 @@
return nullptr;
}
}
+
+Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
+ const CallExpr *E) {
+ switch (BuiltinID) {
+ case NVPTX::BI__nvvm_atom_add_gen_i:
+ case NVPTX::BI__nvvm_atom_add_gen_l:
+ case NVPTX::BI__nvvm_atom_add_gen_ll:
+ return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Add, E);
+
+ case NVPTX::BI__nvvm_atom_sub_gen_i:
+ case NVPTX::BI__nvvm_atom_sub_gen_l:
+ case NVPTX::BI__nvvm_atom_sub_gen_ll:
+ return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Sub, E);
+
+ case NVPTX::BI__nvvm_atom_and_gen_i:
+ case NVPTX::BI__nvvm_atom_and_gen_l:
+ case NVPTX::BI__nvvm_atom_and_gen_ll:
+ return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::And, E);
+
+ case NVPTX::BI__nvvm_atom_or_gen_i:
+ case NVPTX::BI__nvvm_atom_or_gen_l:
+ case NVPTX::BI__nvvm_atom_or_gen_ll:
+ return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Or, E);
+
+ case NVPTX::BI__nvvm_atom_xor_gen_i:
+ case NVPTX::BI__nvvm_atom_xor_gen_l:
+ case NVPTX::BI__nvvm_atom_xor_gen_ll:
+ return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Xor, E);
+
+ case NVPTX::BI__nvvm_atom_xchg_gen_i:
+ case NVPTX::BI__nvvm_atom_xchg_gen_l:
+ case NVPTX::BI__nvvm_atom_xchg_gen_ll:
+ return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Xchg, E);
+
+ case NVPTX::BI__nvvm_atom_max_gen_i:
+ case NVPTX::BI__nvvm_atom_max_gen_l:
+ case NVPTX::BI__nvvm_atom_max_gen_ll:
+ return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Max, E);
+
+ case NVPTX::BI__nvvm_atom_max_gen_ui:
+ case NVPTX::BI__nvvm_atom_max_gen_ul:
+ case NVPTX::BI__nvvm_atom_max_gen_ull:
+ return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::UMax, E);
+
+ case NVPTX::BI__nvvm_atom_min_gen_i:
+ case NVPTX::BI__nvvm_atom_min_gen_l:
+ case NVPTX::BI__nvvm_atom_min_gen_ll:
+ return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Min, E);
+
+ case NVPTX::BI__nvvm_atom_min_gen_ui:
+ case NVPTX::BI__nvvm_atom_min_gen_ul:
+ case NVPTX::BI__nvvm_atom_min_gen_ull:
+ return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::UMin, E);
+
+ case NVPTX::BI__nvvm_atom_cas_gen_i:
+ case NVPTX::BI__nvvm_atom_cas_gen_l:
+ case NVPTX::BI__nvvm_atom_cas_gen_ll:
+ // __nvvm_atom_cas_gen_* should return the old value rather than the
+ // success flag.
+ return MakeAtomicCmpXchgValue(*this, E, /*ReturnBool=*/false);
+
+ case NVPTX::BI__nvvm_atom_add_gen_f: {
+ Value *Ptr = EmitScalarExpr(E->getArg(0));
+ Value *Val = EmitScalarExpr(E->getArg(1));
+ // atomicrmw only deals with integer arguments so we need to use
+ // LLVM's nvvm_atomic_load_add_f32 intrinsic for that.
+ Value *FnALAF32 =
+ CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_add_f32, Ptr->getType());
+ return Builder.CreateCall(FnALAF32, {Ptr, Val});
+ }
+
+ default:
+ return nullptr;
+ }
+}
+
+Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
+ const CallExpr *E) {
+ switch (BuiltinID) {
+ case WebAssembly::BI__builtin_wasm_memory_size: {
+ llvm::Type *ResultType = ConvertType(E->getType());
+ Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_memory_size, ResultType);
+ return Builder.CreateCall(Callee);
+ }
+ case WebAssembly::BI__builtin_wasm_grow_memory: {
+ Value *X = EmitScalarExpr(E->getArg(0));
+ Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_grow_memory, X->getType());
+ return Builder.CreateCall(Callee, X);
+ }
+
+ default:
+ return nullptr;
+ }
+}
diff --git a/lib/CodeGen/CGCUDANV.cpp b/lib/CodeGen/CGCUDANV.cpp
index 67d0ab7..045e19b 100644
--- a/lib/CodeGen/CGCUDANV.cpp
+++ b/lib/CodeGen/CGCUDANV.cpp
@@ -57,9 +57,9 @@
unsigned Alignment = 0) {
llvm::Constant *Zeros[] = {llvm::ConstantInt::get(SizeTy, 0),
llvm::ConstantInt::get(SizeTy, 0)};
- auto *ConstStr = CGM.GetAddrOfConstantCString(Str, Name.c_str());
- return llvm::ConstantExpr::getGetElementPtr(ConstStr->getValueType(),
- ConstStr, Zeros);
+ auto ConstStr = CGM.GetAddrOfConstantCString(Str, Name.c_str());
+ return llvm::ConstantExpr::getGetElementPtr(ConstStr.getElementType(),
+ ConstStr.getPointer(), Zeros);
}
void emitDeviceStubBody(CodeGenFunction &CGF, FunctionArgList &Args);
@@ -121,7 +121,7 @@
std::vector<llvm::Type *> ArgTypes;
for (FunctionArgList::const_iterator I = Args.begin(), E = Args.end();
I != E; ++I) {
- llvm::Value *V = CGF.GetAddrOfLocalVar(*I);
+ llvm::Value *V = CGF.GetAddrOfLocalVar(*I).getPointer();
ArgValues.push_back(V);
assert(isa<llvm::PointerType>(V->getType()) && "Arg type not PointerType");
ArgTypes.push_back(cast<llvm::PointerType>(V->getType())->getElementType());
@@ -173,7 +173,7 @@
llvm::GlobalValue::InternalLinkage, "__cuda_register_kernels", &TheModule);
llvm::BasicBlock *EntryBB =
llvm::BasicBlock::Create(Context, "entry", RegisterKernelsFunc);
- CGBuilderTy Builder(Context);
+ CGBuilderTy Builder(CGM, Context);
Builder.SetInsertPoint(EntryBB);
// void __cudaRegisterFunction(void **, const char *, char *, const char *,
@@ -230,7 +230,7 @@
llvm::GlobalValue::InternalLinkage, "__cuda_module_ctor", &TheModule);
llvm::BasicBlock *CtorEntryBB =
llvm::BasicBlock::Create(Context, "entry", ModuleCtorFunc);
- CGBuilderTy CtorBuilder(Context);
+ CGBuilderTy CtorBuilder(CGM, Context);
CtorBuilder.SetInsertPoint(CtorEntryBB);
@@ -267,7 +267,8 @@
llvm::GlobalVariable *GpuBinaryHandle = new llvm::GlobalVariable(
TheModule, VoidPtrPtrTy, false, llvm::GlobalValue::InternalLinkage,
llvm::ConstantPointerNull::get(VoidPtrPtrTy), "__cuda_gpubin_handle");
- CtorBuilder.CreateStore(RegisterFatbinCall, GpuBinaryHandle, false);
+ CtorBuilder.CreateAlignedStore(RegisterFatbinCall, GpuBinaryHandle,
+ CGM.getPointerAlign());
// Call __cuda_register_kernels(GpuBinaryHandle);
CtorBuilder.CreateCall(RegisterKernelsFunc, RegisterFatbinCall);
@@ -300,12 +301,13 @@
llvm::GlobalValue::InternalLinkage, "__cuda_module_dtor", &TheModule);
llvm::BasicBlock *DtorEntryBB =
llvm::BasicBlock::Create(Context, "entry", ModuleDtorFunc);
- CGBuilderTy DtorBuilder(Context);
+ CGBuilderTy DtorBuilder(CGM, Context);
DtorBuilder.SetInsertPoint(DtorEntryBB);
for (llvm::GlobalVariable *GpuBinaryHandle : GpuBinaryHandles) {
- DtorBuilder.CreateCall(UnregisterFatbinFunc,
- DtorBuilder.CreateLoad(GpuBinaryHandle, false));
+ auto HandleValue =
+ DtorBuilder.CreateAlignedLoad(GpuBinaryHandle, CGM.getPointerAlign());
+ DtorBuilder.CreateCall(UnregisterFatbinFunc, HandleValue);
}
DtorBuilder.CreateRetVoid();
diff --git a/lib/CodeGen/CGCXX.cpp b/lib/CodeGen/CGCXX.cpp
index 7d7ed78..6847df9 100644
--- a/lib/CodeGen/CGCXX.cpp
+++ b/lib/CodeGen/CGCXX.cpp
@@ -28,6 +28,7 @@
using namespace clang;
using namespace CodeGen;
+
/// Try to emit a base destructor as an alias to its primary
/// base-class destructor.
bool CodeGenModule::TryEmitBaseDestructorAsAlias(const CXXDestructorDecl *D) {
@@ -39,6 +40,12 @@
if (getCodeGenOpts().OptimizationLevel == 0)
return true;
+ // If sanitizing memory to check for use-after-dtor, do not emit as
+ // an alias, unless this class owns no members.
+ if (getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
+ !D->getParent()->field_empty())
+ return true;
+
// If the destructor doesn't have a trivial body, we have to emit it
// separately.
if (!D->hasTrivialBody())
@@ -124,11 +131,6 @@
if (!llvm::GlobalAlias::isValidLinkage(Linkage))
return true;
- // Don't create a weak alias for a dllexport'd symbol.
- if (AliasDecl.getDecl()->hasAttr<DLLExportAttr>() &&
- llvm::GlobalValue::isWeakForLinker(Linkage))
- return true;
-
llvm::GlobalValue::LinkageTypes TargetLinkage =
getFunctionLinkage(TargetDecl);
@@ -141,8 +143,8 @@
return false;
// Derive the type for the alias.
- llvm::PointerType *AliasType
- = getTypes().GetFunctionType(AliasDecl)->getPointerTo();
+ llvm::Type *AliasValueType = getTypes().GetFunctionType(AliasDecl);
+ llvm::PointerType *AliasType = AliasValueType->getPointerTo();
// Find the referent. Some aliases might require a bitcast, in
// which case the caller is responsible for ensuring the soundness
@@ -166,6 +168,16 @@
return false;
}
+ // If we have a weak, non-discardable alias (weak, weak_odr), like an extern
+ // template instantiation or a dllexported class, avoid forming it on COFF.
+ // A COFF weak external alias cannot satisfy a normal undefined symbol
+ // reference from another TU. The other TU must also mark the referenced
+ // symbol as weak, which we cannot rely on.
+ if (llvm::GlobalValue::isWeakForLinker(Linkage) &&
+ getTriple().isOSBinFormatCOFF()) {
+ return true;
+ }
+
if (!InEveryTU) {
// If we don't have a definition for the destructor yet, don't
// emit. We can't emit aliases to declarations; that's just not
@@ -182,8 +194,8 @@
return true;
// Create the alias with no name.
- auto *Alias =
- llvm::GlobalAlias::create(AliasType, Linkage, "", Aliasee, &getModule());
+ auto *Alias = llvm::GlobalAlias::create(AliasValueType, 0, Linkage, "",
+ Aliasee, &getModule());
// Switch any previous uses to the alias.
if (Entry) {
@@ -207,7 +219,8 @@
const CGFunctionInfo &FnInfo =
getTypes().arrangeCXXStructorDeclaration(MD, Type);
auto *Fn = cast<llvm::Function>(
- getAddrOfCXXStructor(MD, Type, &FnInfo, nullptr, true));
+ getAddrOfCXXStructor(MD, Type, &FnInfo, /*FnType=*/nullptr,
+ /*DontDefer=*/true, /*IsForDefinition=*/true));
GlobalDecl GD;
if (const auto *DD = dyn_cast<CXXDestructorDecl>(MD)) {
@@ -226,9 +239,9 @@
return Fn;
}
-llvm::GlobalValue *CodeGenModule::getAddrOfCXXStructor(
+llvm::Constant *CodeGenModule::getAddrOfCXXStructor(
const CXXMethodDecl *MD, StructorType Type, const CGFunctionInfo *FnInfo,
- llvm::FunctionType *FnType, bool DontDefer) {
+ llvm::FunctionType *FnType, bool DontDefer, bool IsForDefinition) {
GlobalDecl GD;
if (auto *CD = dyn_cast<CXXConstructorDecl>(MD)) {
GD = GlobalDecl(CD, toCXXCtorType(Type));
@@ -236,19 +249,15 @@
GD = GlobalDecl(cast<CXXDestructorDecl>(MD), toCXXDtorType(Type));
}
- StringRef Name = getMangledName(GD);
- if (llvm::GlobalValue *Existing = GetGlobalValue(Name))
- return Existing;
-
if (!FnType) {
if (!FnInfo)
FnInfo = &getTypes().arrangeCXXStructorDeclaration(MD, Type);
FnType = getTypes().GetFunctionType(*FnInfo);
}
- return cast<llvm::Function>(GetOrCreateLLVMFunction(Name, FnType, GD,
- /*ForVTable=*/false,
- DontDefer));
+ return GetOrCreateLLVMFunction(
+ getMangledName(GD), FnType, GD, /*ForVTable=*/false, DontDefer,
+ /*isThunk=*/false, /*ExtraAttrs=*/llvm::AttributeSet(), IsForDefinition);
}
static llvm::Value *BuildAppleKextVirtualCall(CodeGenFunction &CGF,
@@ -270,7 +279,7 @@
VTableIndex += AddressPoint;
llvm::Value *VFuncPtr =
CGF.Builder.CreateConstInBoundsGEP1_64(VTable, VTableIndex, "vfnkxt");
- return CGF.Builder.CreateLoad(VFuncPtr);
+ return CGF.Builder.CreateAlignedLoad(VFuncPtr, CGF.PointerAlignInBytes);
}
/// BuildAppleKextVirtualCall - This routine is to support gcc's kext ABI making
diff --git a/lib/CodeGen/CGCXXABI.cpp b/lib/CodeGen/CGCXXABI.cpp
index cb7e6df..e4da447 100644
--- a/lib/CodeGen/CGCXXABI.cpp
+++ b/lib/CodeGen/CGCXXABI.cpp
@@ -13,6 +13,7 @@
//===----------------------------------------------------------------------===//
#include "CGCXXABI.h"
+#include "CGCleanup.h"
using namespace clang;
using namespace CodeGen;
@@ -73,25 +74,28 @@
}
llvm::Value *CGCXXABI::EmitLoadOfMemberFunctionPointer(
- CodeGenFunction &CGF, const Expr *E, llvm::Value *&This,
+ CodeGenFunction &CGF, const Expr *E, Address This,
+ llvm::Value *&ThisPtrForCall,
llvm::Value *MemPtr, const MemberPointerType *MPT) {
ErrorUnsupportedABI(CGF, "calls through member pointers");
+ ThisPtrForCall = This.getPointer();
const FunctionProtoType *FPT =
MPT->getPointeeType()->getAs<FunctionProtoType>();
const CXXRecordDecl *RD =
cast<CXXRecordDecl>(MPT->getClass()->getAs<RecordType>()->getDecl());
llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(
- CGM.getTypes().arrangeCXXMethodType(RD, FPT));
+ CGM.getTypes().arrangeCXXMethodType(RD, FPT, /*FD=*/nullptr));
return llvm::Constant::getNullValue(FTy->getPointerTo());
}
llvm::Value *
CGCXXABI::EmitMemberDataPointerAddress(CodeGenFunction &CGF, const Expr *E,
- llvm::Value *Base, llvm::Value *MemPtr,
+ Address Base, llvm::Value *MemPtr,
const MemberPointerType *MPT) {
ErrorUnsupportedABI(CGF, "loads of member pointers");
- llvm::Type *Ty = CGF.ConvertType(MPT->getPointeeType())->getPointerTo();
+ llvm::Type *Ty = CGF.ConvertType(MPT->getPointeeType())
+ ->getPointerTo(Base.getAddressSpace());
return llvm::Constant::getNullValue(Ty);
}
@@ -130,10 +134,9 @@
return GetBogusMemberPointer(QualType(MPT, 0));
}
-llvm::Constant *CGCXXABI::EmitMemberPointer(const CXXMethodDecl *MD) {
- return GetBogusMemberPointer(
- CGM.getContext().getMemberPointerType(MD->getType(),
- MD->getParent()->getTypeForDecl()));
+llvm::Constant *CGCXXABI::EmitMemberFunctionPointer(const CXXMethodDecl *MD) {
+ return GetBogusMemberPointer(CGM.getContext().getMemberPointerType(
+ MD->getType(), MD->getParent()->getTypeForDecl()));
}
llvm::Constant *CGCXXABI::EmitMemberDataPointer(const MemberPointerType *MPT,
@@ -160,13 +163,24 @@
&CGM.getContext().Idents.get("this"),
MD->getThisType(CGM.getContext()));
params.push_back(ThisDecl);
- getThisDecl(CGF) = ThisDecl;
+ CGF.CXXABIThisDecl = ThisDecl;
+
+ // Compute the presumed alignment of 'this', which basically comes
+ // down to whether we know it's a complete object or not.
+ auto &Layout = CGF.getContext().getASTRecordLayout(MD->getParent());
+ if (MD->getParent()->getNumVBases() == 0 || // avoid vcall in common case
+ MD->getParent()->hasAttr<FinalAttr>() ||
+ !isThisCompleteObject(CGF.CurGD)) {
+ CGF.CXXABIThisAlignment = Layout.getAlignment();
+ } else {
+ CGF.CXXABIThisAlignment = Layout.getNonVirtualAlignment();
+ }
}
void CGCXXABI::EmitThisParam(CodeGenFunction &CGF) {
/// Initialize the 'this' slot.
assert(getThisDecl(CGF) && "no 'this' variable for function");
- getThisValue(CGF)
+ CGF.CXXABIThisValue
= CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(getThisDecl(CGF)),
"this");
}
@@ -187,14 +201,14 @@
return CharUnits::Zero();
}
-llvm::Value *CGCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
- llvm::Value *NewPtr,
- llvm::Value *NumElements,
- const CXXNewExpr *expr,
- QualType ElementType) {
+Address CGCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
+ Address NewPtr,
+ llvm::Value *NumElements,
+ const CXXNewExpr *expr,
+ QualType ElementType) {
// Should never be called.
ErrorUnsupportedABI(CGF, "array cookie initialization");
- return nullptr;
+ return Address::invalid();
}
bool CGCXXABI::requiresArrayCookie(const CXXDeleteExpr *expr,
@@ -216,31 +230,30 @@
return expr->getAllocatedType().isDestructedType();
}
-void CGCXXABI::ReadArrayCookie(CodeGenFunction &CGF, llvm::Value *ptr,
+void CGCXXABI::ReadArrayCookie(CodeGenFunction &CGF, Address ptr,
const CXXDeleteExpr *expr, QualType eltTy,
llvm::Value *&numElements,
llvm::Value *&allocPtr, CharUnits &cookieSize) {
// Derive a char* in the same address space as the pointer.
- unsigned AS = ptr->getType()->getPointerAddressSpace();
- llvm::Type *charPtrTy = CGF.Int8Ty->getPointerTo(AS);
- ptr = CGF.Builder.CreateBitCast(ptr, charPtrTy);
+ ptr = CGF.Builder.CreateElementBitCast(ptr, CGF.Int8Ty);
// If we don't need an array cookie, bail out early.
if (!requiresArrayCookie(expr, eltTy)) {
- allocPtr = ptr;
+ allocPtr = ptr.getPointer();
numElements = nullptr;
cookieSize = CharUnits::Zero();
return;
}
cookieSize = getArrayCookieSizeImpl(eltTy);
- allocPtr = CGF.Builder.CreateConstInBoundsGEP1_64(ptr,
- -cookieSize.getQuantity());
- numElements = readArrayCookieImpl(CGF, allocPtr, cookieSize);
+ Address allocAddr =
+ CGF.Builder.CreateConstInBoundsByteGEP(ptr, -cookieSize);
+ allocPtr = allocAddr.getPointer();
+ numElements = readArrayCookieImpl(CGF, allocAddr, cookieSize);
}
llvm::Value *CGCXXABI::readArrayCookieImpl(CodeGenFunction &CGF,
- llvm::Value *ptr,
+ Address ptr,
CharUnits cookieSize) {
ErrorUnsupportedABI(CGF, "reading a new[] cookie");
return llvm::ConstantInt::get(CGF.SizeTy, 0);
@@ -309,3 +322,11 @@
// Just call std::terminate and ignore the violating exception.
return CGF.EmitNounwindRuntimeCall(CGF.CGM.getTerminateFn());
}
+
+CatchTypeInfo CGCXXABI::getCatchAllTypeInfo() {
+ return CatchTypeInfo{nullptr, 0};
+}
+
+std::vector<CharUnits> CGCXXABI::getVBPtrOffsets(const CXXRecordDecl *RD) {
+ return std::vector<CharUnits>();
+}
diff --git a/lib/CodeGen/CGCXXABI.h b/lib/CodeGen/CGCXXABI.h
index 2c73921..3f240b1 100644
--- a/lib/CodeGen/CGCXXABI.h
+++ b/lib/CodeGen/CGCXXABI.h
@@ -37,6 +37,7 @@
namespace CodeGen {
class CodeGenFunction;
class CodeGenModule;
+struct CatchTypeInfo;
/// \brief Implements C++ ABI-specific code generation functions.
class CGCXXABI {
@@ -48,12 +49,15 @@
: CGM(CGM), MangleCtx(CGM.getContext().createMangleContext()) {}
protected:
- ImplicitParamDecl *&getThisDecl(CodeGenFunction &CGF) {
+ ImplicitParamDecl *getThisDecl(CodeGenFunction &CGF) {
return CGF.CXXABIThisDecl;
}
- llvm::Value *&getThisValue(CodeGenFunction &CGF) {
+ llvm::Value *getThisValue(CodeGenFunction &CGF) {
return CGF.CXXABIThisValue;
}
+ Address getThisAddress(CodeGenFunction &CGF) {
+ return Address(CGF.CXXABIThisValue, CGF.CXXABIThisAlignment);
+ }
/// Issue a diagnostic about unsupported features in the ABI.
void ErrorUnsupportedABI(CodeGenFunction &CGF, StringRef S);
@@ -77,6 +81,12 @@
virtual bool requiresArrayCookie(const CXXDeleteExpr *E, QualType eltType);
virtual bool requiresArrayCookie(const CXXNewExpr *E);
+ /// Determine whether there's something special about the rules of
+ /// the ABI tell us that 'this' is a complete object within the
+ /// given function. Obvious common logic like being defined on a
+ /// final class will have been taken care of by the caller.
+ virtual bool isThisCompleteObject(GlobalDecl GD) const = 0;
+
public:
virtual ~CGCXXABI();
@@ -135,13 +145,14 @@
/// pointer. Apply the this-adjustment and set 'This' to the
/// adjusted value.
virtual llvm::Value *EmitLoadOfMemberFunctionPointer(
- CodeGenFunction &CGF, const Expr *E, llvm::Value *&This,
- llvm::Value *MemPtr, const MemberPointerType *MPT);
+ CodeGenFunction &CGF, const Expr *E, Address This,
+ llvm::Value *&ThisPtrForCall, llvm::Value *MemPtr,
+ const MemberPointerType *MPT);
/// Calculate an l-value from an object and a data member pointer.
virtual llvm::Value *
EmitMemberDataPointerAddress(CodeGenFunction &CGF, const Expr *E,
- llvm::Value *Base, llvm::Value *MemPtr,
+ Address Base, llvm::Value *MemPtr,
const MemberPointerType *MPT);
/// Perform a derived-to-base, base-to-derived, or bitcast member
@@ -164,15 +175,11 @@
return true;
}
- virtual bool isTypeInfoCalculable(QualType Ty) const {
- return !Ty->isIncompleteType();
- }
-
/// Create a null member pointer of the given type.
virtual llvm::Constant *EmitNullMemberPointer(const MemberPointerType *MPT);
/// Create a member pointer for the given method.
- virtual llvm::Constant *EmitMemberPointer(const CXXMethodDecl *MD);
+ virtual llvm::Constant *EmitMemberFunctionPointer(const CXXMethodDecl *MD);
/// Create a member pointer for the given field.
virtual llvm::Constant *EmitMemberDataPointer(const MemberPointerType *MPT,
@@ -212,12 +219,17 @@
public:
virtual void emitVirtualObjectDelete(CodeGenFunction &CGF,
const CXXDeleteExpr *DE,
- llvm::Value *Ptr, QualType ElementType,
+ Address Ptr, QualType ElementType,
const CXXDestructorDecl *Dtor) = 0;
virtual void emitRethrow(CodeGenFunction &CGF, bool isNoReturn) = 0;
virtual void emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) = 0;
virtual llvm::GlobalVariable *getThrowInfo(QualType T) { return nullptr; }
+ /// \brief Determine whether it's possible to emit a vtable for \p RD, even
+ /// though we do not know that the vtable has been marked as used by semantic
+ /// analysis.
+ virtual bool canSpeculativelyEmitVTable(const CXXRecordDecl *RD) const = 0;
+
virtual void emitBeginCatch(CodeGenFunction &CGF, const CXXCatchStmt *C) = 0;
virtual llvm::CallInst *
@@ -225,33 +237,34 @@
llvm::Value *Exn);
virtual llvm::Constant *getAddrOfRTTIDescriptor(QualType Ty) = 0;
- virtual llvm::Constant *
+ virtual CatchTypeInfo
getAddrOfCXXCatchHandlerType(QualType Ty, QualType CatchHandlerType) = 0;
+ virtual CatchTypeInfo getCatchAllTypeInfo();
virtual bool shouldTypeidBeNullChecked(bool IsDeref,
QualType SrcRecordTy) = 0;
virtual void EmitBadTypeidCall(CodeGenFunction &CGF) = 0;
virtual llvm::Value *EmitTypeid(CodeGenFunction &CGF, QualType SrcRecordTy,
- llvm::Value *ThisPtr,
+ Address ThisPtr,
llvm::Type *StdTypeInfoPtrTy) = 0;
virtual bool shouldDynamicCastCallBeNullChecked(bool SrcIsPtr,
QualType SrcRecordTy) = 0;
virtual llvm::Value *
- EmitDynamicCastCall(CodeGenFunction &CGF, llvm::Value *Value,
+ EmitDynamicCastCall(CodeGenFunction &CGF, Address Value,
QualType SrcRecordTy, QualType DestTy,
QualType DestRecordTy, llvm::BasicBlock *CastEnd) = 0;
virtual llvm::Value *EmitDynamicCastToVoid(CodeGenFunction &CGF,
- llvm::Value *Value,
+ Address Value,
QualType SrcRecordTy,
QualType DestTy) = 0;
virtual bool EmitBadCastCall(CodeGenFunction &CGF) = 0;
virtual llvm::Value *GetVirtualBaseClassOffset(CodeGenFunction &CGF,
- llvm::Value *This,
+ Address This,
const CXXRecordDecl *ClassDecl,
const CXXRecordDecl *BaseClassDecl) = 0;
@@ -294,10 +307,9 @@
/// Perform ABI-specific "this" argument adjustment required prior to
/// a call of a virtual function.
/// The "VirtualCall" argument is true iff the call itself is virtual.
- virtual llvm::Value *
+ virtual Address
adjustThisArgumentForVirtualFunctionCall(CodeGenFunction &CGF, GlobalDecl GD,
- llvm::Value *This,
- bool VirtualCall) {
+ Address This, bool VirtualCall) {
return This;
}
@@ -337,19 +349,31 @@
virtual void EmitDestructorCall(CodeGenFunction &CGF,
const CXXDestructorDecl *DD, CXXDtorType Type,
bool ForVirtualBase, bool Delegating,
- llvm::Value *This) = 0;
+ Address This) = 0;
/// Emits the VTable definitions required for the given record type.
virtual void emitVTableDefinitions(CodeGenVTables &CGVT,
const CXXRecordDecl *RD) = 0;
+ /// Checks if ABI requires extra virtual offset for vtable field.
+ virtual bool
+ isVirtualOffsetNeededForVTableField(CodeGenFunction &CGF,
+ CodeGenFunction::VPtr Vptr) = 0;
+
+ /// Checks if ABI requires to initilize vptrs for given dynamic class.
+ virtual bool doStructorsInitializeVPtrs(const CXXRecordDecl *VTableClass) = 0;
+
+ /// Get the address point of the vtable for the given base subobject.
+ virtual llvm::Constant *
+ getVTableAddressPoint(BaseSubobject Base,
+ const CXXRecordDecl *VTableClass) = 0;
+
/// Get the address point of the vtable for the given base subobject while
- /// building a constructor or a destructor. On return, NeedsVirtualOffset
- /// tells if a virtual base adjustment is needed in order to get the offset
- /// of the base subobject.
- virtual llvm::Value *getVTableAddressPointInStructor(
- CodeGenFunction &CGF, const CXXRecordDecl *RD, BaseSubobject Base,
- const CXXRecordDecl *NearestVBase, bool &NeedsVirtualOffset) = 0;
+ /// building a constructor or a destructor.
+ virtual llvm::Value *
+ getVTableAddressPointInStructor(CodeGenFunction &CGF, const CXXRecordDecl *RD,
+ BaseSubobject Base,
+ const CXXRecordDecl *NearestVBase) = 0;
/// Get the address point of the vtable for the given base subobject while
/// building a constexpr.
@@ -365,13 +389,14 @@
/// Build a virtual function pointer in the ABI-specific way.
virtual llvm::Value *getVirtualFunctionPointer(CodeGenFunction &CGF,
GlobalDecl GD,
- llvm::Value *This,
- llvm::Type *Ty) = 0;
+ Address This,
+ llvm::Type *Ty,
+ SourceLocation Loc) = 0;
/// Emit the ABI-specific virtual destructor call.
virtual llvm::Value *
EmitVirtualDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *Dtor,
- CXXDtorType DtorType, llvm::Value *This,
+ CXXDtorType DtorType, Address This,
const CXXMemberCallExpr *CE) = 0;
virtual void adjustCallArgsForDestructorThunk(CodeGenFunction &CGF,
@@ -387,11 +412,11 @@
GlobalDecl GD, bool ReturnAdjustment) = 0;
virtual llvm::Value *performThisAdjustment(CodeGenFunction &CGF,
- llvm::Value *This,
+ Address This,
const ThisAdjustment &TA) = 0;
virtual llvm::Value *performReturnAdjustment(CodeGenFunction &CGF,
- llvm::Value *Ret,
+ Address Ret,
const ReturnAdjustment &RA) = 0;
virtual void EmitReturnFromThunk(CodeGenFunction &CGF,
@@ -400,6 +425,9 @@
virtual size_t getSrcArgforCopyCtor(const CXXConstructorDecl *,
FunctionArgList &Args) const = 0;
+ /// Gets the offsets of all the virtual base pointers in a given class.
+ virtual std::vector<CharUnits> getVBPtrOffsets(const CXXRecordDecl *RD);
+
/// Gets the pure virtual member call function.
virtual StringRef GetPureVirtualCallName() = 0;
@@ -428,11 +456,11 @@
/// always a size_t
/// \param ElementType - the base element allocated type,
/// i.e. the allocated type after stripping all array types
- virtual llvm::Value *InitializeArrayCookie(CodeGenFunction &CGF,
- llvm::Value *NewPtr,
- llvm::Value *NumElements,
- const CXXNewExpr *expr,
- QualType ElementType);
+ virtual Address InitializeArrayCookie(CodeGenFunction &CGF,
+ Address NewPtr,
+ llvm::Value *NumElements,
+ const CXXNewExpr *expr,
+ QualType ElementType);
/// Reads the array cookie associated with the given pointer,
/// if it has one.
@@ -447,7 +475,7 @@
/// function
/// \param CookieSize - an out parameter which will be initialized
/// with the size of the cookie, or zero if there is no cookie
- virtual void ReadArrayCookie(CodeGenFunction &CGF, llvm::Value *Ptr,
+ virtual void ReadArrayCookie(CodeGenFunction &CGF, Address Ptr,
const CXXDeleteExpr *expr,
QualType ElementType, llvm::Value *&NumElements,
llvm::Value *&AllocPtr, CharUnits &CookieSize);
@@ -470,8 +498,7 @@
/// Other parameters are as above.
///
/// \return a size_t
- virtual llvm::Value *readArrayCookieImpl(CodeGenFunction &IGF,
- llvm::Value *ptr,
+ virtual llvm::Value *readArrayCookieImpl(CodeGenFunction &IGF, Address ptr,
CharUnits cookieSize);
public:
@@ -511,11 +538,9 @@
/// thread_local variables, a list of functions to perform the
/// initialization.
virtual void EmitThreadLocalInitFuncs(
- CodeGenModule &CGM,
- ArrayRef<std::pair<const VarDecl *, llvm::GlobalVariable *>>
- CXXThreadLocals,
+ CodeGenModule &CGM, ArrayRef<const VarDecl *> CXXThreadLocals,
ArrayRef<llvm::Function *> CXXThreadLocalInits,
- ArrayRef<llvm::GlobalVariable *> CXXThreadLocalInitVars) = 0;
+ ArrayRef<const VarDecl *> CXXThreadLocalInitVars) = 0;
// Determine if references to thread_local global variables can be made
// directly or require access through a thread wrapper function.
diff --git a/lib/CodeGen/CGCall.cpp b/lib/CodeGen/CGCall.cpp
index 6903073..49b5df0 100644
--- a/lib/CodeGen/CGCall.cpp
+++ b/lib/CodeGen/CGCall.cpp
@@ -15,12 +15,14 @@
#include "CGCall.h"
#include "ABIInfo.h"
#include "CGCXXABI.h"
+#include "CGCleanup.h"
#include "CodeGenFunction.h"
#include "CodeGenModule.h"
#include "TargetInfo.h"
#include "clang/AST/Decl.h"
#include "clang/AST/DeclCXX.h"
#include "clang/AST/DeclObjC.h"
+#include "clang/Basic/TargetBuiltins.h"
#include "clang/Basic/TargetInfo.h"
#include "clang/CodeGen/CGFunctionInfo.h"
#include "clang/Frontend/CodeGenOptions.h"
@@ -32,7 +34,6 @@
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/Transforms/Utils/Local.h"
-#include <sstream>
using namespace clang;
using namespace CodeGen;
@@ -92,15 +93,41 @@
FTNP->getExtInfo(), RequiredArgs(0));
}
+/// Adds the formal paramaters in FPT to the given prefix. If any parameter in
+/// FPT has pass_object_size attrs, then we'll add parameters for those, too.
+static void appendParameterTypes(const CodeGenTypes &CGT,
+ SmallVectorImpl<CanQualType> &prefix,
+ const CanQual<FunctionProtoType> &FPT,
+ const FunctionDecl *FD) {
+ // Fast path: unknown target.
+ if (FD == nullptr) {
+ prefix.append(FPT->param_type_begin(), FPT->param_type_end());
+ return;
+ }
+
+ // In the vast majority cases, we'll have precisely FPT->getNumParams()
+ // parameters; the only thing that can change this is the presence of
+ // pass_object_size. So, we preallocate for the common case.
+ prefix.reserve(prefix.size() + FPT->getNumParams());
+
+ assert(FD->getNumParams() == FPT->getNumParams());
+ for (unsigned I = 0, E = FPT->getNumParams(); I != E; ++I) {
+ prefix.push_back(FPT->getParamType(I));
+ if (FD->getParamDecl(I)->hasAttr<PassObjectSizeAttr>())
+ prefix.push_back(CGT.getContext().getSizeType());
+ }
+}
+
/// Arrange the LLVM function layout for a value of the given function
/// type, on top of any implicit parameters already stored.
static const CGFunctionInfo &
arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,
SmallVectorImpl<CanQualType> &prefix,
- CanQual<FunctionProtoType> FTP) {
+ CanQual<FunctionProtoType> FTP,
+ const FunctionDecl *FD) {
RequiredArgs required = RequiredArgs::forPrototypePlus(FTP, prefix.size());
// FIXME: Kill copy.
- prefix.append(FTP->param_type_begin(), FTP->param_type_end());
+ appendParameterTypes(CGT, prefix, FTP, FD);
CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
return CGT.arrangeLLVMFunctionInfo(resultType, instanceMethod,
/*chainCall=*/false, prefix,
@@ -110,10 +137,11 @@
/// Arrange the argument and result information for a value of the
/// given freestanding function type.
const CGFunctionInfo &
-CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) {
+CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP,
+ const FunctionDecl *FD) {
SmallVector<CanQualType, 16> argTypes;
return ::arrangeLLVMFunctionInfo(*this, /*instanceMethod=*/false, argTypes,
- FTP);
+ FTP, FD);
}
static CallingConv getCallingConventionForDecl(const Decl *D, bool IsWindows) {
@@ -156,7 +184,8 @@
/// constructor or destructor.
const CGFunctionInfo &
CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD,
- const FunctionProtoType *FTP) {
+ const FunctionProtoType *FTP,
+ const CXXMethodDecl *MD) {
SmallVector<CanQualType, 16> argTypes;
// Add the 'this' pointer.
@@ -167,7 +196,7 @@
return ::arrangeLLVMFunctionInfo(
*this, true, argTypes,
- FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
+ FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>(), MD);
}
/// Arrange the argument and result information for a declaration or
@@ -184,10 +213,10 @@
if (MD->isInstance()) {
// The abstract case is perfectly fine.
const CXXRecordDecl *ThisType = TheCXXABI.getThisArgumentTypeForMethod(MD);
- return arrangeCXXMethodType(ThisType, prototype.getTypePtr());
+ return arrangeCXXMethodType(ThisType, prototype.getTypePtr(), MD);
}
- return arrangeFreeFunctionType(prototype);
+ return arrangeFreeFunctionType(prototype, MD);
}
const CGFunctionInfo &
@@ -208,7 +237,7 @@
CanQual<FunctionProtoType> FTP = GetFormalType(MD);
// Add the formal parameters.
- argTypes.append(FTP->param_type_begin(), FTP->param_type_end());
+ appendParameterTypes(*this, argTypes, FTP, MD);
TheCXXABI.buildStructorSignature(MD, Type, argTypes);
@@ -274,7 +303,7 @@
}
assert(isa<FunctionProtoType>(FTy));
- return arrangeFreeFunctionType(FTy.getAs<FunctionProtoType>());
+ return arrangeFreeFunctionType(FTy.getAs<FunctionProtoType>(), FD);
}
/// Arrange the argument and result information for the declaration or
@@ -554,6 +583,7 @@
FI->HasRegParm = info.getHasRegParm();
FI->RegParm = info.getRegParm();
FI->ArgStruct = nullptr;
+ FI->ArgStructAlign = 0;
FI->NumArgs = argTypes.size();
FI->getArgsBuffer()[0].type = resultType;
for (unsigned i = 0, e = argTypes.size(); i != e; ++i)
@@ -727,6 +757,21 @@
}
}
+static void forConstantArrayExpansion(CodeGenFunction &CGF,
+ ConstantArrayExpansion *CAE,
+ Address BaseAddr,
+ llvm::function_ref<void(Address)> Fn) {
+ CharUnits EltSize = CGF.getContext().getTypeSizeInChars(CAE->EltTy);
+ CharUnits EltAlign =
+ BaseAddr.getAlignment().alignmentOfArrayElement(EltSize);
+
+ for (int i = 0, n = CAE->NumElts; i < n; i++) {
+ llvm::Value *EltAddr =
+ CGF.Builder.CreateConstGEP2_32(nullptr, BaseAddr.getPointer(), 0, i);
+ Fn(Address(EltAddr, EltAlign));
+ }
+}
+
void CodeGenFunction::ExpandTypeFromArgs(
QualType Ty, LValue LV, SmallVectorImpl<llvm::Argument *>::iterator &AI) {
assert(LV.isSimple() &&
@@ -734,17 +779,16 @@
auto Exp = getTypeExpansion(Ty, getContext());
if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
- for (int i = 0, n = CAExp->NumElts; i < n; i++) {
- llvm::Value *EltAddr =
- Builder.CreateConstGEP2_32(nullptr, LV.getAddress(), 0, i);
+ forConstantArrayExpansion(*this, CAExp, LV.getAddress(),
+ [&](Address EltAddr) {
LValue LV = MakeAddrLValue(EltAddr, CAExp->EltTy);
ExpandTypeFromArgs(CAExp->EltTy, LV, AI);
- }
+ });
} else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
- llvm::Value *This = LV.getAddress();
+ Address This = LV.getAddress();
for (const CXXBaseSpecifier *BS : RExp->Bases) {
// Perform a single step derived-to-base conversion.
- llvm::Value *Base =
+ Address Base =
GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
/*NullCheckValue=*/false, SourceLocation());
LValue SubLV = MakeAddrLValue(Base, BS->getType());
@@ -757,15 +801,10 @@
LValue SubLV = EmitLValueForField(LV, FD);
ExpandTypeFromArgs(FD->getType(), SubLV, AI);
}
- } else if (auto CExp = dyn_cast<ComplexExpansion>(Exp.get())) {
- llvm::Value *RealAddr =
- Builder.CreateStructGEP(nullptr, LV.getAddress(), 0, "real");
- EmitStoreThroughLValue(RValue::get(*AI++),
- MakeAddrLValue(RealAddr, CExp->EltTy));
- llvm::Value *ImagAddr =
- Builder.CreateStructGEP(nullptr, LV.getAddress(), 1, "imag");
- EmitStoreThroughLValue(RValue::get(*AI++),
- MakeAddrLValue(ImagAddr, CExp->EltTy));
+ } else if (isa<ComplexExpansion>(Exp.get())) {
+ auto realValue = *AI++;
+ auto imagValue = *AI++;
+ EmitStoreOfComplex(ComplexPairTy(realValue, imagValue), LV, /*init*/ true);
} else {
assert(isa<NoExpansion>(Exp.get()));
EmitStoreThroughLValue(RValue::get(*AI++), LV);
@@ -777,18 +816,17 @@
SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) {
auto Exp = getTypeExpansion(Ty, getContext());
if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
- llvm::Value *Addr = RV.getAggregateAddr();
- for (int i = 0, n = CAExp->NumElts; i < n; i++) {
- llvm::Value *EltAddr = Builder.CreateConstGEP2_32(nullptr, Addr, 0, i);
+ forConstantArrayExpansion(*this, CAExp, RV.getAggregateAddress(),
+ [&](Address EltAddr) {
RValue EltRV =
convertTempToRValue(EltAddr, CAExp->EltTy, SourceLocation());
ExpandTypeToArgs(CAExp->EltTy, EltRV, IRFuncTy, IRCallArgs, IRCallArgPos);
- }
+ });
} else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
- llvm::Value *This = RV.getAggregateAddr();
+ Address This = RV.getAggregateAddress();
for (const CXXBaseSpecifier *BS : RExp->Bases) {
// Perform a single step derived-to-base conversion.
- llvm::Value *Base =
+ Address Base =
GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
/*NullCheckValue=*/false, SourceLocation());
RValue BaseRV = RValue::getAggregate(Base);
@@ -823,12 +861,22 @@
}
}
+/// Create a temporary allocation for the purposes of coercion.
+static Address CreateTempAllocaForCoercion(CodeGenFunction &CGF, llvm::Type *Ty,
+ CharUnits MinAlign) {
+ // Don't use an alignment that's worse than what LLVM would prefer.
+ auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlignment(Ty);
+ CharUnits Align = std::max(MinAlign, CharUnits::fromQuantity(PrefAlign));
+
+ return CGF.CreateTempAlloca(Ty, Align);
+}
+
/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
/// accessing some number of bytes out of it, try to gep into the struct to get
/// at its inner goodness. Dive as deep as possible without entering an element
/// with an in-memory size smaller than DstSize.
-static llvm::Value *
-EnterStructPointerForCoercedAccess(llvm::Value *SrcPtr,
+static Address
+EnterStructPointerForCoercedAccess(Address SrcPtr,
llvm::StructType *SrcSTy,
uint64_t DstSize, CodeGenFunction &CGF) {
// We can't dive into a zero-element struct.
@@ -847,11 +895,10 @@
return SrcPtr;
// GEP into the first element.
- SrcPtr = CGF.Builder.CreateConstGEP2_32(SrcSTy, SrcPtr, 0, 0, "coerce.dive");
+ SrcPtr = CGF.Builder.CreateStructGEP(SrcPtr, 0, CharUnits(), "coerce.dive");
// If the first element is a struct, recurse.
- llvm::Type *SrcTy =
- cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
+ llvm::Type *SrcTy = SrcPtr.getElementType();
if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy))
return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
@@ -913,26 +960,25 @@
/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
-/// a pointer to an object of type \arg Ty.
+/// a pointer to an object of type \arg Ty, known to be aligned to
+/// \arg SrcAlign bytes.
///
/// This safely handles the case when the src type is smaller than the
/// destination type; in this situation the values of bits which not
/// present in the src are undefined.
-static llvm::Value *CreateCoercedLoad(llvm::Value *SrcPtr,
- llvm::Type *Ty,
+static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty,
CodeGenFunction &CGF) {
- llvm::Type *SrcTy =
- cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
+ llvm::Type *SrcTy = Src.getElementType();
// If SrcTy and Ty are the same, just do a load.
if (SrcTy == Ty)
- return CGF.Builder.CreateLoad(SrcPtr);
+ return CGF.Builder.CreateLoad(Src);
uint64_t DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);
if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
- SrcPtr = EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
- SrcTy = cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
+ Src = EnterStructPointerForCoercedAccess(Src, SrcSTy, DstSize, CGF);
+ SrcTy = Src.getType()->getElementType();
}
uint64_t SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
@@ -941,7 +987,7 @@
// extension or truncation to the desired type.
if ((isa<llvm::IntegerType>(Ty) || isa<llvm::PointerType>(Ty)) &&
(isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy))) {
- llvm::LoadInst *Load = CGF.Builder.CreateLoad(SrcPtr);
+ llvm::Value *Load = CGF.Builder.CreateLoad(Src);
return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF);
}
@@ -953,24 +999,17 @@
//
// FIXME: Assert that we aren't truncating non-padding bits when have access
// to that information.
- llvm::Value *Casted =
- CGF.Builder.CreateBitCast(SrcPtr, llvm::PointerType::getUnqual(Ty));
- llvm::LoadInst *Load = CGF.Builder.CreateLoad(Casted);
- // FIXME: Use better alignment / avoid requiring aligned load.
- Load->setAlignment(1);
- return Load;
+ Src = CGF.Builder.CreateBitCast(Src, llvm::PointerType::getUnqual(Ty));
+ return CGF.Builder.CreateLoad(Src);
}
- // Otherwise do coercion through memory. This is stupid, but
- // simple.
- llvm::Value *Tmp = CGF.CreateTempAlloca(Ty);
- llvm::Type *I8PtrTy = CGF.Builder.getInt8PtrTy();
- llvm::Value *Casted = CGF.Builder.CreateBitCast(Tmp, I8PtrTy);
- llvm::Value *SrcCasted = CGF.Builder.CreateBitCast(SrcPtr, I8PtrTy);
- // FIXME: Use better alignment.
+ // Otherwise do coercion through memory. This is stupid, but simple.
+ Address Tmp = CreateTempAllocaForCoercion(CGF, Ty, Src.getAlignment());
+ Address Casted = CGF.Builder.CreateBitCast(Tmp, CGF.Int8PtrTy);
+ Address SrcCasted = CGF.Builder.CreateBitCast(Src, CGF.Int8PtrTy);
CGF.Builder.CreateMemCpy(Casted, SrcCasted,
llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize),
- 1, false);
+ false);
return CGF.Builder.CreateLoad(Tmp);
}
@@ -979,48 +1018,46 @@
// fast-isel.
// FIXME: Do we need to recurse here?
static void BuildAggStore(CodeGenFunction &CGF, llvm::Value *Val,
- llvm::Value *DestPtr, bool DestIsVolatile,
- bool LowAlignment) {
+ Address Dest, bool DestIsVolatile) {
// Prefer scalar stores to first-class aggregate stores.
if (llvm::StructType *STy =
dyn_cast<llvm::StructType>(Val->getType())) {
+ const llvm::StructLayout *Layout =
+ CGF.CGM.getDataLayout().getStructLayout(STy);
+
for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
- llvm::Value *EltPtr = CGF.Builder.CreateConstGEP2_32(STy, DestPtr, 0, i);
+ auto EltOffset = CharUnits::fromQuantity(Layout->getElementOffset(i));
+ Address EltPtr = CGF.Builder.CreateStructGEP(Dest, i, EltOffset);
llvm::Value *Elt = CGF.Builder.CreateExtractValue(Val, i);
- llvm::StoreInst *SI = CGF.Builder.CreateStore(Elt, EltPtr,
- DestIsVolatile);
- if (LowAlignment)
- SI->setAlignment(1);
+ CGF.Builder.CreateStore(Elt, EltPtr, DestIsVolatile);
}
} else {
- llvm::StoreInst *SI = CGF.Builder.CreateStore(Val, DestPtr, DestIsVolatile);
- if (LowAlignment)
- SI->setAlignment(1);
+ CGF.Builder.CreateStore(Val, Dest, DestIsVolatile);
}
}
/// CreateCoercedStore - Create a store to \arg DstPtr from \arg Src,
-/// where the source and destination may have different types.
+/// where the source and destination may have different types. The
+/// destination is known to be aligned to \arg DstAlign bytes.
///
/// This safely handles the case when the src type is larger than the
/// destination type; the upper bits of the src will be lost.
static void CreateCoercedStore(llvm::Value *Src,
- llvm::Value *DstPtr,
+ Address Dst,
bool DstIsVolatile,
CodeGenFunction &CGF) {
llvm::Type *SrcTy = Src->getType();
- llvm::Type *DstTy =
- cast<llvm::PointerType>(DstPtr->getType())->getElementType();
+ llvm::Type *DstTy = Dst.getType()->getElementType();
if (SrcTy == DstTy) {
- CGF.Builder.CreateStore(Src, DstPtr, DstIsVolatile);
+ CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
return;
}
uint64_t SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
if (llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
- DstPtr = EnterStructPointerForCoercedAccess(DstPtr, DstSTy, SrcSize, CGF);
- DstTy = cast<llvm::PointerType>(DstPtr->getType())->getElementType();
+ Dst = EnterStructPointerForCoercedAccess(Dst, DstSTy, SrcSize, CGF);
+ DstTy = Dst.getType()->getElementType();
}
// If the source and destination are integer or pointer types, just do an
@@ -1028,7 +1065,7 @@
if ((isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy)) &&
(isa<llvm::IntegerType>(DstTy) || isa<llvm::PointerType>(DstTy))) {
Src = CoerceIntOrPtrToIntOrPtr(Src, DstTy, CGF);
- CGF.Builder.CreateStore(Src, DstPtr, DstIsVolatile);
+ CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
return;
}
@@ -1036,10 +1073,8 @@
// If store is legal, just bitcast the src pointer.
if (SrcSize <= DstSize) {
- llvm::Value *Casted =
- CGF.Builder.CreateBitCast(DstPtr, llvm::PointerType::getUnqual(SrcTy));
- // FIXME: Use better alignment / avoid requiring aligned store.
- BuildAggStore(CGF, Src, Casted, DstIsVolatile, true);
+ Dst = CGF.Builder.CreateBitCast(Dst, llvm::PointerType::getUnqual(SrcTy));
+ BuildAggStore(CGF, Src, Dst, DstIsVolatile);
} else {
// Otherwise do coercion through memory. This is stupid, but
// simple.
@@ -1050,18 +1085,27 @@
//
// FIXME: Assert that we aren't truncating non-padding bits when have access
// to that information.
- llvm::Value *Tmp = CGF.CreateTempAlloca(SrcTy);
+ Address Tmp = CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment());
CGF.Builder.CreateStore(Src, Tmp);
- llvm::Type *I8PtrTy = CGF.Builder.getInt8PtrTy();
- llvm::Value *Casted = CGF.Builder.CreateBitCast(Tmp, I8PtrTy);
- llvm::Value *DstCasted = CGF.Builder.CreateBitCast(DstPtr, I8PtrTy);
- // FIXME: Use better alignment.
+ Address Casted = CGF.Builder.CreateBitCast(Tmp, CGF.Int8PtrTy);
+ Address DstCasted = CGF.Builder.CreateBitCast(Dst, CGF.Int8PtrTy);
CGF.Builder.CreateMemCpy(DstCasted, Casted,
llvm::ConstantInt::get(CGF.IntPtrTy, DstSize),
- 1, false);
+ false);
}
}
+static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr,
+ const ABIArgInfo &info) {
+ if (unsigned offset = info.getDirectOffset()) {
+ addr = CGF.Builder.CreateElementBitCast(addr, CGF.Int8Ty);
+ addr = CGF.Builder.CreateConstInBoundsByteGEP(addr,
+ CharUnits::fromQuantity(offset));
+ addr = CGF.Builder.CreateElementBitCast(addr, info.getCoerceToType());
+ }
+ return addr;
+}
+
namespace {
/// Encapsulates information about the way function arguments from
@@ -1275,12 +1319,7 @@
}
break;
- case ABIArgInfo::Indirect: {
- assert(!retAI.getIndirectAlign() && "Align unused on indirect return.");
- resultType = llvm::Type::getVoidTy(getLLVMContext());
- break;
- }
-
+ case ABIArgInfo::Indirect:
case ABIArgInfo::Ignore:
resultType = llvm::Type::getVoidTy(getLLVMContext());
break;
@@ -1381,8 +1420,19 @@
return GetFunctionType(*Info);
}
+static void AddAttributesFromFunctionProtoType(ASTContext &Ctx,
+ llvm::AttrBuilder &FuncAttrs,
+ const FunctionProtoType *FPT) {
+ if (!FPT)
+ return;
+
+ if (!isUnresolvedExceptionSpec(FPT->getExceptionSpecType()) &&
+ FPT->isNothrow(Ctx))
+ FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
+}
+
void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
- const Decl *TargetDecl,
+ CGCalleeInfo CalleeInfo,
AttributeListType &PAL,
unsigned &CallingConv,
bool AttrOnCallSite) {
@@ -1395,6 +1445,13 @@
if (FI.isNoReturn())
FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
+ // If we have information about the function prototype, we can learn
+ // attributes form there.
+ AddAttributesFromFunctionProtoType(getContext(), FuncAttrs,
+ CalleeInfo.getCalleeFunctionProtoType());
+
+ const Decl *TargetDecl = CalleeInfo.getCalleeDecl();
+
// FIXME: handle sseregparm someday...
if (TargetDecl) {
if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
@@ -1407,9 +1464,8 @@
FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate);
if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
- const FunctionProtoType *FPT = Fn->getType()->getAs<FunctionProtoType>();
- if (FPT && FPT->isNothrow(getContext()))
- FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
+ AddAttributesFromFunctionProtoType(
+ getContext(), FuncAttrs, Fn->getType()->getAs<FunctionProtoType>());
// Don't use [[noreturn]] or _Noreturn for a call to a virtual function.
// These attributes are not inherited by overloads.
const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Fn);
@@ -1417,13 +1473,16 @@
FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
}
- // 'const' and 'pure' attribute functions are also nounwind.
+ // 'const', 'pure' and 'noalias' attributed functions are also nounwind.
if (TargetDecl->hasAttr<ConstAttr>()) {
FuncAttrs.addAttribute(llvm::Attribute::ReadNone);
FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
} else if (TargetDecl->hasAttr<PureAttr>()) {
FuncAttrs.addAttribute(llvm::Attribute::ReadOnly);
FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
+ } else if (TargetDecl->hasAttr<NoAliasAttr>()) {
+ FuncAttrs.addAttribute(llvm::Attribute::ArgMemOnly);
+ FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
}
if (TargetDecl->hasAttr<RestrictAttr>())
RetAttrs.addAttribute(llvm::Attribute::NoAlias);
@@ -1453,6 +1512,8 @@
// Attributes that should go on the call site only.
if (!CodeGenOpts.SimplifyLibCalls)
FuncAttrs.addAttribute(llvm::Attribute::NoBuiltin);
+ if (!CodeGenOpts.TrapFuncName.empty())
+ FuncAttrs.addAttribute("trap-func-name", CodeGenOpts.TrapFuncName);
} else {
// Attributes that should go on the function, but not the call site.
if (!CodeGenOpts.DisableFPElim) {
@@ -1465,8 +1526,12 @@
FuncAttrs.addAttribute("no-frame-pointer-elim-non-leaf");
}
+ bool DisableTailCalls =
+ CodeGenOpts.DisableTailCalls ||
+ (TargetDecl && TargetDecl->hasAttr<DisableTailCallsAttr>());
FuncAttrs.addAttribute("disable-tail-calls",
- llvm::toStringRef(CodeGenOpts.DisableTailCalls));
+ llvm::toStringRef(DisableTailCalls));
+
FuncAttrs.addAttribute("less-precise-fpmad",
llvm::toStringRef(CodeGenOpts.LessPreciseFPMAD));
FuncAttrs.addAttribute("no-infs-fp-math",
@@ -1480,65 +1545,53 @@
FuncAttrs.addAttribute("stack-protector-buffer-size",
llvm::utostr(CodeGenOpts.SSPBufferSize));
- if (!CodeGenOpts.StackRealignment)
- FuncAttrs.addAttribute("no-realign-stack");
+ if (CodeGenOpts.StackRealignment)
+ FuncAttrs.addAttribute("stackrealign");
// Add target-cpu and target-features attributes to functions. If
// we have a decl for the function and it has a target attribute then
// parse that and add it to the feature set.
StringRef TargetCPU = getTarget().getTargetOpts().CPU;
-
- // TODO: Features gets us the features on the command line including
- // feature dependencies. For canonicalization purposes we might want to
- // avoid putting features in the target-features set if we know it'll be
- // one of the default features in the backend, e.g. corei7-avx and +avx or
- // figure out non-explicit dependencies.
- std::vector<std::string> Features(getTarget().getTargetOpts().Features);
-
- // TODO: The target attribute complicates this further by allowing multiple
- // additional features to be tacked on to the feature string for a
- // particular function. For now we simply append to the set of features and
- // let backend resolution fix them up.
const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl);
- if (FD) {
- if (const TargetAttr *TD = FD->getAttr<TargetAttr>()) {
- StringRef FeaturesStr = TD->getFeatures();
- SmallVector<StringRef, 1> AttrFeatures;
- FeaturesStr.split(AttrFeatures, ",");
+ if (FD && FD->hasAttr<TargetAttr>()) {
+ llvm::StringMap<bool> FeatureMap;
+ getFunctionFeatureMap(FeatureMap, FD);
- // Grab the various features and prepend a "+" to turn on the feature to
- // the backend and add them to our existing set of Features.
- for (auto &Feature : AttrFeatures) {
- // While we're here iterating check for a different target cpu.
- if (Feature.startswith("arch="))
- TargetCPU = Feature.split("=").second;
- else if (Feature.startswith("tune="))
- // We don't support cpu tuning this way currently.
- ;
- else if (Feature.startswith("fpmath="))
- // TODO: Support the fpmath option this way. It will require checking
- // overall feature validity for the function with the rest of the
- // attributes on the function.
- ;
- else if (Feature.startswith("mno-"))
- Features.push_back("-" + Feature.split("-").second.str());
- else
- Features.push_back("+" + Feature.str());
- }
+ // Produce the canonical string for this set of features.
+ std::vector<std::string> Features;
+ for (llvm::StringMap<bool>::const_iterator it = FeatureMap.begin(),
+ ie = FeatureMap.end();
+ it != ie; ++it)
+ Features.push_back((it->second ? "+" : "-") + it->first().str());
+
+ // Now add the target-cpu and target-features to the function.
+ // While we populated the feature map above, we still need to
+ // get and parse the target attribute so we can get the cpu for
+ // the function.
+ const auto *TD = FD->getAttr<TargetAttr>();
+ TargetAttr::ParsedTargetAttr ParsedAttr = TD->parse();
+ if (ParsedAttr.second != "")
+ TargetCPU = ParsedAttr.second;
+ if (TargetCPU != "")
+ FuncAttrs.addAttribute("target-cpu", TargetCPU);
+ if (!Features.empty()) {
+ std::sort(Features.begin(), Features.end());
+ FuncAttrs.addAttribute(
+ "target-features",
+ llvm::join(Features.begin(), Features.end(), ","));
}
- }
-
- // Now add the target-cpu and target-features to the function.
- if (TargetCPU != "")
- FuncAttrs.addAttribute("target-cpu", TargetCPU);
- if (!Features.empty()) {
- std::stringstream TargetFeatures;
- std::copy(Features.begin(), Features.end(),
- std::ostream_iterator<std::string>(TargetFeatures, ","));
-
- // The drop_back gets rid of the trailing space.
- FuncAttrs.addAttribute("target-features",
- StringRef(TargetFeatures.str()).drop_back(1));
+ } else {
+ // Otherwise just add the existing target cpu and target features to the
+ // function.
+ std::vector<std::string> &Features = getTarget().getTargetOpts().Features;
+ if (TargetCPU != "")
+ FuncAttrs.addAttribute("target-cpu", TargetCPU);
+ if (!Features.empty()) {
+ std::sort(Features.begin(), Features.end());
+ FuncAttrs.addAttribute(
+ "target-features",
+ llvm::join(Features.begin(), Features.end(), ","));
+ }
}
}
@@ -1642,20 +1695,37 @@
Attrs.addAttribute(llvm::Attribute::InReg);
break;
- case ABIArgInfo::Indirect:
+ case ABIArgInfo::Indirect: {
if (AI.getInReg())
Attrs.addAttribute(llvm::Attribute::InReg);
if (AI.getIndirectByVal())
Attrs.addAttribute(llvm::Attribute::ByVal);
- Attrs.addAlignmentAttr(AI.getIndirectAlign());
+ CharUnits Align = AI.getIndirectAlign();
+
+ // In a byval argument, it is important that the required
+ // alignment of the type is honored, as LLVM might be creating a
+ // *new* stack object, and needs to know what alignment to give
+ // it. (Sometimes it can deduce a sensible alignment on its own,
+ // but not if clang decides it must emit a packed struct, or the
+ // user specifies increased alignment requirements.)
+ //
+ // This is different from indirect *not* byval, where the object
+ // exists already, and the align attribute is purely
+ // informative.
+ assert(!Align.isZero());
+
+ // For now, only add this when we have a byval argument.
+ // TODO: be less lazy about updating test cases.
+ if (AI.getIndirectByVal())
+ Attrs.addAlignmentAttr(Align.getQuantity());
// byval disables readnone and readonly.
FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
.removeAttribute(llvm::Attribute::ReadNone);
break;
-
+ }
case ABIArgInfo::Ignore:
case ABIArgInfo::Expand:
continue;
@@ -1775,10 +1845,14 @@
// If we're using inalloca, all the memory arguments are GEPs off of the last
// parameter, which is a pointer to the complete memory area.
- llvm::Value *ArgStruct = nullptr;
+ Address ArgStruct = Address::invalid();
+ const llvm::StructLayout *ArgStructLayout = nullptr;
if (IRFunctionArgs.hasInallocaArg()) {
- ArgStruct = FnArgs[IRFunctionArgs.getInallocaArgNo()];
- assert(ArgStruct->getType() == FI.getArgStruct()->getPointerTo());
+ ArgStructLayout = CGM.getDataLayout().getStructLayout(FI.getArgStruct());
+ ArgStruct = Address(FnArgs[IRFunctionArgs.getInallocaArgNo()],
+ FI.getArgStructAlignment());
+
+ assert(ArgStruct.getType() == FI.getArgStruct()->getPointerTo());
}
// Name the struct return parameter.
@@ -1792,9 +1866,7 @@
// Track if we received the parameter as a pointer (indirect, byval, or
// inalloca). If already have a pointer, EmitParmDecl doesn't need to copy it
// into a local alloca for us.
- enum ValOrPointer { HaveValue = 0, HavePointer = 1 };
- typedef llvm::PointerIntPair<llvm::Value *, 1> ValueAndIsPtr;
- SmallVector<ValueAndIsPtr, 16> ArgVals;
+ SmallVector<ParamValue, 16> ArgVals;
ArgVals.reserve(Args.size());
// Create a pointer value for every parameter declaration. This usually
@@ -1820,49 +1892,47 @@
switch (ArgI.getKind()) {
case ABIArgInfo::InAlloca: {
assert(NumIRArgs == 0);
- llvm::Value *V =
- Builder.CreateStructGEP(FI.getArgStruct(), ArgStruct,
- ArgI.getInAllocaFieldIndex(), Arg->getName());
- ArgVals.push_back(ValueAndIsPtr(V, HavePointer));
+ auto FieldIndex = ArgI.getInAllocaFieldIndex();
+ CharUnits FieldOffset =
+ CharUnits::fromQuantity(ArgStructLayout->getElementOffset(FieldIndex));
+ Address V = Builder.CreateStructGEP(ArgStruct, FieldIndex, FieldOffset,
+ Arg->getName());
+ ArgVals.push_back(ParamValue::forIndirect(V));
break;
}
case ABIArgInfo::Indirect: {
assert(NumIRArgs == 1);
- llvm::Value *V = FnArgs[FirstIRArg];
+ Address ParamAddr = Address(FnArgs[FirstIRArg], ArgI.getIndirectAlign());
if (!hasScalarEvaluationKind(Ty)) {
// Aggregates and complex variables are accessed by reference. All we
- // need to do is realign the value, if requested
+ // need to do is realign the value, if requested.
+ Address V = ParamAddr;
if (ArgI.getIndirectRealign()) {
- llvm::Value *AlignedTemp = CreateMemTemp(Ty, "coerce");
+ Address AlignedTemp = CreateMemTemp(Ty, "coerce");
// Copy from the incoming argument pointer to the temporary with the
// appropriate alignment.
//
// FIXME: We should have a common utility for generating an aggregate
// copy.
- llvm::Type *I8PtrTy = Builder.getInt8PtrTy();
CharUnits Size = getContext().getTypeSizeInChars(Ty);
- llvm::Value *Dst = Builder.CreateBitCast(AlignedTemp, I8PtrTy);
- llvm::Value *Src = Builder.CreateBitCast(V, I8PtrTy);
- Builder.CreateMemCpy(Dst,
- Src,
- llvm::ConstantInt::get(IntPtrTy,
- Size.getQuantity()),
- ArgI.getIndirectAlign(),
- false);
+ auto SizeVal = llvm::ConstantInt::get(IntPtrTy, Size.getQuantity());
+ Address Dst = Builder.CreateBitCast(AlignedTemp, Int8PtrTy);
+ Address Src = Builder.CreateBitCast(ParamAddr, Int8PtrTy);
+ Builder.CreateMemCpy(Dst, Src, SizeVal, false);
V = AlignedTemp;
}
- ArgVals.push_back(ValueAndIsPtr(V, HavePointer));
+ ArgVals.push_back(ParamValue::forIndirect(V));
} else {
// Load scalar value from indirect argument.
- V = EmitLoadOfScalar(V, false, ArgI.getIndirectAlign(), Ty,
- Arg->getLocStart());
+ llvm::Value *V =
+ EmitLoadOfScalar(ParamAddr, false, Ty, Arg->getLocStart());
if (isPromoted)
V = emitArgumentDemotion(*this, Arg, V);
- ArgVals.push_back(ValueAndIsPtr(V, HaveValue));
+ ArgVals.push_back(ParamValue::forDirect(V));
}
break;
}
@@ -1967,68 +2037,49 @@
if (V->getType() != LTy)
V = Builder.CreateBitCast(V, LTy);
- ArgVals.push_back(ValueAndIsPtr(V, HaveValue));
+ ArgVals.push_back(ParamValue::forDirect(V));
break;
}
- llvm::AllocaInst *Alloca = CreateMemTemp(Ty, Arg->getName());
+ Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg),
+ Arg->getName());
- // The alignment we need to use is the max of the requested alignment for
- // the argument plus the alignment required by our access code below.
- unsigned AlignmentToUse =
- CGM.getDataLayout().getABITypeAlignment(ArgI.getCoerceToType());
- AlignmentToUse = std::max(AlignmentToUse,
- (unsigned)getContext().getDeclAlign(Arg).getQuantity());
-
- Alloca->setAlignment(AlignmentToUse);
- llvm::Value *V = Alloca;
- llvm::Value *Ptr = V; // Pointer to store into.
-
- // If the value is offset in memory, apply the offset now.
- if (unsigned Offs = ArgI.getDirectOffset()) {
- Ptr = Builder.CreateBitCast(Ptr, Builder.getInt8PtrTy());
- Ptr = Builder.CreateConstGEP1_32(Builder.getInt8Ty(), Ptr, Offs);
- Ptr = Builder.CreateBitCast(Ptr,
- llvm::PointerType::getUnqual(ArgI.getCoerceToType()));
- }
+ // Pointer to store into.
+ Address Ptr = emitAddressAtOffset(*this, Alloca, ArgI);
// Fast-isel and the optimizer generally like scalar values better than
// FCAs, so we flatten them if this is safe to do for this argument.
llvm::StructType *STy = dyn_cast<llvm::StructType>(ArgI.getCoerceToType());
if (ArgI.isDirect() && ArgI.getCanBeFlattened() && STy &&
STy->getNumElements() > 1) {
+ auto SrcLayout = CGM.getDataLayout().getStructLayout(STy);
uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(STy);
- llvm::Type *DstTy =
- cast<llvm::PointerType>(Ptr->getType())->getElementType();
+ llvm::Type *DstTy = Ptr.getElementType();
uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(DstTy);
+ Address AddrToStoreInto = Address::invalid();
if (SrcSize <= DstSize) {
- Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(STy));
-
- assert(STy->getNumElements() == NumIRArgs);
- for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
- auto AI = FnArgs[FirstIRArg + i];
- AI->setName(Arg->getName() + ".coerce" + Twine(i));
- llvm::Value *EltPtr = Builder.CreateConstGEP2_32(STy, Ptr, 0, i);
- Builder.CreateStore(AI, EltPtr);
- }
+ AddrToStoreInto =
+ Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(STy));
} else {
- llvm::AllocaInst *TempAlloca =
- CreateTempAlloca(ArgI.getCoerceToType(), "coerce");
- TempAlloca->setAlignment(AlignmentToUse);
- llvm::Value *TempV = TempAlloca;
-
- assert(STy->getNumElements() == NumIRArgs);
- for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
- auto AI = FnArgs[FirstIRArg + i];
- AI->setName(Arg->getName() + ".coerce" + Twine(i));
- llvm::Value *EltPtr =
- Builder.CreateConstGEP2_32(ArgI.getCoerceToType(), TempV, 0, i);
- Builder.CreateStore(AI, EltPtr);
- }
-
- Builder.CreateMemCpy(Ptr, TempV, DstSize, AlignmentToUse);
+ AddrToStoreInto =
+ CreateTempAlloca(STy, Alloca.getAlignment(), "coerce");
}
+
+ assert(STy->getNumElements() == NumIRArgs);
+ for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
+ auto AI = FnArgs[FirstIRArg + i];
+ AI->setName(Arg->getName() + ".coerce" + Twine(i));
+ auto Offset = CharUnits::fromQuantity(SrcLayout->getElementOffset(i));
+ Address EltPtr =
+ Builder.CreateStructGEP(AddrToStoreInto, i, Offset);
+ Builder.CreateStore(AI, EltPtr);
+ }
+
+ if (SrcSize > DstSize) {
+ Builder.CreateMemCpy(Ptr, AddrToStoreInto, DstSize);
+ }
+
} else {
// Simple case, just do a coerced store of the argument into the alloca.
assert(NumIRArgs == 1);
@@ -2037,15 +2088,15 @@
CreateCoercedStore(AI, Ptr, /*DestIsVolatile=*/false, *this);
}
-
// Match to what EmitParmDecl is expecting for this type.
if (CodeGenFunction::hasScalarEvaluationKind(Ty)) {
- V = EmitLoadOfScalar(V, false, AlignmentToUse, Ty, Arg->getLocStart());
+ llvm::Value *V =
+ EmitLoadOfScalar(Alloca, false, Ty, Arg->getLocStart());
if (isPromoted)
V = emitArgumentDemotion(*this, Arg, V);
- ArgVals.push_back(ValueAndIsPtr(V, HaveValue));
+ ArgVals.push_back(ParamValue::forDirect(V));
} else {
- ArgVals.push_back(ValueAndIsPtr(V, HavePointer));
+ ArgVals.push_back(ParamValue::forIndirect(Alloca));
}
break;
}
@@ -2054,11 +2105,9 @@
// If this structure was expanded into multiple arguments then
// we need to create a temporary and reconstruct it from the
// arguments.
- llvm::AllocaInst *Alloca = CreateMemTemp(Ty);
- CharUnits Align = getContext().getDeclAlign(Arg);
- Alloca->setAlignment(Align.getQuantity());
- LValue LV = MakeAddrLValue(Alloca, Ty, Align);
- ArgVals.push_back(ValueAndIsPtr(Alloca, HavePointer));
+ Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));
+ LValue LV = MakeAddrLValue(Alloca, Ty);
+ ArgVals.push_back(ParamValue::forIndirect(Alloca));
auto FnArgIter = FnArgs.begin() + FirstIRArg;
ExpandTypeFromArgs(Ty, LV, FnArgIter);
@@ -2074,10 +2123,10 @@
assert(NumIRArgs == 0);
// Initialize the local variable appropriately.
if (!hasScalarEvaluationKind(Ty)) {
- ArgVals.push_back(ValueAndIsPtr(CreateMemTemp(Ty), HavePointer));
+ ArgVals.push_back(ParamValue::forIndirect(CreateMemTemp(Ty)));
} else {
llvm::Value *U = llvm::UndefValue::get(ConvertType(Arg->getType()));
- ArgVals.push_back(ValueAndIsPtr(U, HaveValue));
+ ArgVals.push_back(ParamValue::forDirect(U));
}
break;
}
@@ -2085,12 +2134,10 @@
if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
for (int I = Args.size() - 1; I >= 0; --I)
- EmitParmDecl(*Args[I], ArgVals[I].getPointer(), ArgVals[I].getInt(),
- I + 1);
+ EmitParmDecl(*Args[I], ArgVals[I], I + 1);
} else {
for (unsigned I = 0, E = Args.size(); I != E; ++I)
- EmitParmDecl(*Args[I], ArgVals[I].getPointer(), ArgVals[I].getInt(),
- I + 1);
+ EmitParmDecl(*Args[I], ArgVals[I], I + 1);
}
}
@@ -2143,9 +2190,9 @@
bool doRetainAutorelease;
- if (call->getCalledValue() == CGF.CGM.getARCEntrypoints().objc_retain) {
+ if (call->getCalledValue() == CGF.CGM.getObjCEntrypoints().objc_retain) {
doRetainAutorelease = true;
- } else if (call->getCalledValue() == CGF.CGM.getARCEntrypoints()
+ } else if (call->getCalledValue() == CGF.CGM.getObjCEntrypoints()
.objc_retainAutoreleasedReturnValue) {
doRetainAutorelease = false;
@@ -2154,7 +2201,7 @@
// for that call. If we can't find it, we can't do this
// optimization. But it should always be the immediately previous
// instruction, unless we needed bitcasts around the call.
- if (CGF.CGM.getARCEntrypoints().retainAutoreleasedReturnValueMarker) {
+ if (CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker) {
llvm::Instruction *prev = call->getPrevNode();
assert(prev);
if (isa<llvm::BitCastInst>(prev)) {
@@ -2163,7 +2210,7 @@
}
assert(isa<llvm::CallInst>(prev));
assert(cast<llvm::CallInst>(prev)->getCalledValue() ==
- CGF.CGM.getARCEntrypoints().retainAutoreleasedReturnValueMarker);
+ CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker);
insnsToKill.push_back(prev);
}
} else {
@@ -2208,7 +2255,7 @@
llvm::CallInst *retainCall =
dyn_cast<llvm::CallInst>(result->stripPointerCasts());
if (!retainCall ||
- retainCall->getCalledValue() != CGF.CGM.getARCEntrypoints().objc_retain)
+ retainCall->getCalledValue() != CGF.CGM.getObjCEntrypoints().objc_retain)
return nullptr;
// Look for an ordinary load of 'self'.
@@ -2216,7 +2263,7 @@
llvm::LoadInst *load =
dyn_cast<llvm::LoadInst>(retainedValue->stripPointerCasts());
if (!load || load->isAtomic() || load->isVolatile() ||
- load->getPointerOperand() != CGF.GetAddrOfLocalVar(self))
+ load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getPointer())
return nullptr;
// Okay! Burn it all down. This relies for correctness on the
@@ -2253,11 +2300,23 @@
/// Heuristically search for a dominating store to the return-value slot.
static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
+ // Check if a User is a store which pointerOperand is the ReturnValue.
+ // We are looking for stores to the ReturnValue, not for stores of the
+ // ReturnValue to some other location.
+ auto GetStoreIfValid = [&CGF](llvm::User *U) -> llvm::StoreInst * {
+ auto *SI = dyn_cast<llvm::StoreInst>(U);
+ if (!SI || SI->getPointerOperand() != CGF.ReturnValue.getPointer())
+ return nullptr;
+ // These aren't actually possible for non-coerced returns, and we
+ // only care about non-coerced returns on this code path.
+ assert(!SI->isAtomic() && !SI->isVolatile());
+ return SI;
+ };
// If there are multiple uses of the return-value slot, just check
// for something immediately preceding the IP. Sometimes this can
// happen with how we generate implicit-returns; it can also happen
// with noreturn cleanups.
- if (!CGF.ReturnValue->hasOneUse()) {
+ if (!CGF.ReturnValue.getPointer()->hasOneUse()) {
llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
if (IP->empty()) return nullptr;
llvm::Instruction *I = &IP->back();
@@ -2281,21 +2340,13 @@
break;
}
- llvm::StoreInst *store = dyn_cast<llvm::StoreInst>(I);
- if (!store) return nullptr;
- if (store->getPointerOperand() != CGF.ReturnValue) return nullptr;
- assert(!store->isAtomic() && !store->isVolatile()); // see below
- return store;
+ return GetStoreIfValid(I);
}
llvm::StoreInst *store =
- dyn_cast<llvm::StoreInst>(CGF.ReturnValue->user_back());
+ GetStoreIfValid(CGF.ReturnValue.getPointer()->user_back());
if (!store) return nullptr;
- // These aren't actually possible for non-coerced returns, and we
- // only care about non-coerced returns on this code path.
- assert(!store->isAtomic() && !store->isVolatile());
-
// Now do a first-and-dirty dominance check: just walk up the
// single-predecessors chain from the current insertion point.
llvm::BasicBlock *StoreBB = store->getParent();
@@ -2320,7 +2371,7 @@
}
// Functions with no result always return void.
- if (!ReturnValue) {
+ if (!ReturnValue.isValid()) {
Builder.CreateRetVoid();
return;
}
@@ -2338,10 +2389,10 @@
if (RetAI.getInAllocaSRet()) {
llvm::Function::arg_iterator EI = CurFn->arg_end();
--EI;
- llvm::Value *ArgStruct = EI;
+ llvm::Value *ArgStruct = &*EI;
llvm::Value *SRet = Builder.CreateStructGEP(
nullptr, ArgStruct, RetAI.getInAllocaFieldIndex());
- RV = Builder.CreateLoad(SRet, "sret");
+ RV = Builder.CreateAlignedLoad(SRet, getPointerAlign(), "sret");
}
break;
@@ -2352,9 +2403,8 @@
switch (getEvaluationKind(RetTy)) {
case TEK_Complex: {
ComplexPairTy RT =
- EmitLoadOfComplex(MakeNaturalAlignAddrLValue(ReturnValue, RetTy),
- EndLoc);
- EmitStoreOfComplex(RT, MakeNaturalAlignAddrLValue(AI, RetTy),
+ EmitLoadOfComplex(MakeAddrLValue(ReturnValue, RetTy), EndLoc);
+ EmitStoreOfComplex(RT, MakeNaturalAlignAddrLValue(&*AI, RetTy),
/*isInit*/ true);
break;
}
@@ -2363,7 +2413,7 @@
break;
case TEK_Scalar:
EmitStoreOfScalar(Builder.CreateLoad(ReturnValue),
- MakeNaturalAlignAddrLValue(AI, RetTy),
+ MakeNaturalAlignAddrLValue(&*AI, RetTy),
/*isInit*/ true);
break;
}
@@ -2391,9 +2441,12 @@
SI->eraseFromParent();
// If that was the only use of the return value, nuke it as well now.
- if (ReturnValue->use_empty() && isa<llvm::AllocaInst>(ReturnValue)) {
- cast<llvm::AllocaInst>(ReturnValue)->eraseFromParent();
- ReturnValue = nullptr;
+ auto returnValueInst = ReturnValue.getPointer();
+ if (returnValueInst->use_empty()) {
+ if (auto alloca = dyn_cast<llvm::AllocaInst>(returnValueInst)) {
+ alloca->eraseFromParent();
+ ReturnValue = Address::invalid();
+ }
}
// Otherwise, we have to do a simple load.
@@ -2401,14 +2454,8 @@
RV = Builder.CreateLoad(ReturnValue);
}
} else {
- llvm::Value *V = ReturnValue;
// If the value is offset in memory, apply the offset now.
- if (unsigned Offs = RetAI.getDirectOffset()) {
- V = Builder.CreateBitCast(V, Builder.getInt8PtrTy());
- V = Builder.CreateConstGEP1_32(Builder.getInt8Ty(), V, Offs);
- V = Builder.CreateBitCast(V,
- llvm::PointerType::getUnqual(RetAI.getCoerceToType()));
- }
+ Address V = emitAddressAtOffset(*this, ReturnValue, RetAI);
RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this);
}
@@ -2433,8 +2480,8 @@
llvm::Instruction *Ret;
if (RV) {
- if (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute)) {
- if (auto RetNNAttr = CurGD.getDecl()->getAttr<ReturnsNonNullAttr>()) {
+ if (CurCodeDecl && SanOpts.has(SanitizerKind::ReturnsNonnullAttribute)) {
+ if (auto RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>()) {
SanitizerScope SanScope(this);
llvm::Value *Cond = Builder.CreateICmpNE(
RV, llvm::Constant::getNullValue(RV->getType()));
@@ -2460,14 +2507,20 @@
return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
}
-static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF, QualType Ty) {
+static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF,
+ QualType Ty) {
// FIXME: Generate IR in one pass, rather than going back and fixing up these
// placeholders.
llvm::Type *IRTy = CGF.ConvertTypeForMem(Ty);
llvm::Value *Placeholder =
- llvm::UndefValue::get(IRTy->getPointerTo()->getPointerTo());
- Placeholder = CGF.Builder.CreateLoad(Placeholder);
- return AggValueSlot::forAddr(Placeholder, CharUnits::Zero(),
+ llvm::UndefValue::get(IRTy->getPointerTo()->getPointerTo());
+ Placeholder = CGF.Builder.CreateDefaultAlignedLoad(Placeholder);
+
+ // FIXME: When we generate this IR in one pass, we shouldn't need
+ // this win32-specific alignment hack.
+ CharUnits Align = CharUnits::fromQuantity(4);
+
+ return AggValueSlot::forAddr(Address(Placeholder, Align),
Ty.getQualifiers(),
AggValueSlot::IsNotDestructed,
AggValueSlot::DoesNotNeedGCBarriers,
@@ -2480,7 +2533,7 @@
// StartFunction converted the ABI-lowered parameter(s) into a
// local alloca. We need to turn that into an r-value suitable
// for EmitCall.
- llvm::Value *local = GetAddrOfLocalVar(param);
+ Address local = GetAddrOfLocalVar(param);
QualType type = param->getType();
@@ -2515,20 +2568,21 @@
static void emitWriteback(CodeGenFunction &CGF,
const CallArgList::Writeback &writeback) {
const LValue &srcLV = writeback.Source;
- llvm::Value *srcAddr = srcLV.getAddress();
- assert(!isProvablyNull(srcAddr) &&
+ Address srcAddr = srcLV.getAddress();
+ assert(!isProvablyNull(srcAddr.getPointer()) &&
"shouldn't have writeback for provably null argument");
llvm::BasicBlock *contBB = nullptr;
// If the argument wasn't provably non-null, we need to null check
// before doing the store.
- bool provablyNonNull = isProvablyNonNull(srcAddr);
+ bool provablyNonNull = isProvablyNonNull(srcAddr.getPointer());
if (!provablyNonNull) {
llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback");
contBB = CGF.createBasicBlock("icr.done");
- llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull");
+ llvm::Value *isNull =
+ CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
CGF.Builder.CreateCondBr(isNull, contBB, writebackBB);
CGF.EmitBlock(writebackBB);
}
@@ -2537,9 +2591,8 @@
llvm::Value *value = CGF.Builder.CreateLoad(writeback.Temporary);
// Cast it back, in case we're writing an id to a Foo* or something.
- value = CGF.Builder.CreateBitCast(value,
- cast<llvm::PointerType>(srcAddr->getType())->getElementType(),
- "icr.writeback-cast");
+ value = CGF.Builder.CreateBitCast(value, srcAddr.getElementType(),
+ "icr.writeback-cast");
// Perform the writeback.
@@ -2589,10 +2642,9 @@
ArrayRef<CallArgList::CallArgCleanup> Cleanups =
CallArgs.getCleanupsToDeactivate();
// Iterate in reverse to increase the likelihood of popping the cleanup.
- for (ArrayRef<CallArgList::CallArgCleanup>::reverse_iterator
- I = Cleanups.rbegin(), E = Cleanups.rend(); I != E; ++I) {
- CGF.DeactivateCleanupBlock(I->Cleanup, I->IsActiveIP);
- I->IsActiveIP->eraseFromParent();
+ for (const auto &I : llvm::reverse(Cleanups)) {
+ CGF.DeactivateCleanupBlock(I.Cleanup, I.IsActiveIP);
+ I.IsActiveIP->eraseFromParent();
}
}
@@ -2604,25 +2656,27 @@
}
/// Emit an argument that's being passed call-by-writeback. That is,
-/// we are passing the address of
+/// we are passing the address of an __autoreleased temporary; it
+/// might be copy-initialized with the current value of the given
+/// address, but it will definitely be copied out of after the call.
static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
const ObjCIndirectCopyRestoreExpr *CRE) {
LValue srcLV;
// Make an optimistic effort to emit the address as an l-value.
- // This can fail if the the argument expression is more complicated.
+ // This can fail if the argument expression is more complicated.
if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(CRE->getSubExpr())) {
srcLV = CGF.EmitLValue(lvExpr);
// Otherwise, just emit it as a scalar.
} else {
- llvm::Value *srcAddr = CGF.EmitScalarExpr(CRE->getSubExpr());
+ Address srcAddr = CGF.EmitPointerWithAlignment(CRE->getSubExpr());
QualType srcAddrType =
CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
- srcLV = CGF.MakeNaturalAlignAddrLValue(srcAddr, srcAddrType);
+ srcLV = CGF.MakeAddrLValue(srcAddr, srcAddrType);
}
- llvm::Value *srcAddr = srcLV.getAddress();
+ Address srcAddr = srcLV.getAddress();
// The dest and src types don't necessarily match in LLVM terms
// because of the crazy ObjC compatibility rules.
@@ -2631,15 +2685,16 @@
cast<llvm::PointerType>(CGF.ConvertType(CRE->getType()));
// If the address is a constant null, just pass the appropriate null.
- if (isProvablyNull(srcAddr)) {
+ if (isProvablyNull(srcAddr.getPointer())) {
args.add(RValue::get(llvm::ConstantPointerNull::get(destType)),
CRE->getType());
return;
}
// Create the temporary.
- llvm::Value *temp = CGF.CreateTempAlloca(destType->getElementType(),
- "icr.temp");
+ Address temp = CGF.CreateTempAlloca(destType->getElementType(),
+ CGF.getPointerAlign(),
+ "icr.temp");
// Loading an l-value can introduce a cleanup if the l-value is __weak,
// and that cleanup will be conditional if we can't prove that the l-value
// isn't null, so we need to register a dominating point so that the cleanups
@@ -2661,15 +2716,16 @@
// If the address is *not* known to be non-null, we need to switch.
llvm::Value *finalArgument;
- bool provablyNonNull = isProvablyNonNull(srcAddr);
+ bool provablyNonNull = isProvablyNonNull(srcAddr.getPointer());
if (provablyNonNull) {
- finalArgument = temp;
+ finalArgument = temp.getPointer();
} else {
- llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull");
+ llvm::Value *isNull =
+ CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
finalArgument = CGF.Builder.CreateSelect(isNull,
llvm::ConstantPointerNull::get(destType),
- temp, "icr.argument");
+ temp.getPointer(), "icr.argument");
// If we need to copy, then the load has to be conditional, which
// means we need control flow.
@@ -2736,24 +2792,12 @@
// Save the stack.
llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stacksave);
StackBase = CGF.Builder.CreateCall(F, {}, "inalloca.save");
-
- // Control gets really tied up in landing pads, so we have to spill the
- // stacksave to an alloca to avoid violating SSA form.
- // TODO: This is dead if we never emit the cleanup. We should create the
- // alloca and store lazily on the first cleanup emission.
- StackBaseMem = CGF.CreateTempAlloca(CGF.Int8PtrTy, "inalloca.spmem");
- CGF.Builder.CreateStore(StackBase, StackBaseMem);
- CGF.pushStackRestore(EHCleanup, StackBaseMem);
- StackCleanup = CGF.EHStack.getInnermostEHScope();
- assert(StackCleanup.isValid());
}
void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const {
if (StackBase) {
- CGF.DeactivateCleanupBlock(StackCleanup, StackBase);
+ // Restore the stack after the call.
llvm::Value *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stackrestore);
- // We could load StackBase from StackBaseMem, but in the non-exceptional
- // case we can skip it.
CGF.Builder.CreateCall(F, StackBase);
}
}
@@ -2783,12 +2827,26 @@
"nonnull_arg", StaticData, None);
}
-void CodeGenFunction::EmitCallArgs(CallArgList &Args,
- ArrayRef<QualType> ArgTypes,
- CallExpr::const_arg_iterator ArgBeg,
- CallExpr::const_arg_iterator ArgEnd,
- const FunctionDecl *CalleeDecl,
- unsigned ParamsToSkip) {
+void CodeGenFunction::EmitCallArgs(
+ CallArgList &Args, ArrayRef<QualType> ArgTypes,
+ llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
+ const FunctionDecl *CalleeDecl, unsigned ParamsToSkip) {
+ assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));
+
+ auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg) {
+ if (CalleeDecl == nullptr || I >= CalleeDecl->getNumParams())
+ return;
+ auto *PS = CalleeDecl->getParamDecl(I)->getAttr<PassObjectSizeAttr>();
+ if (PS == nullptr)
+ return;
+
+ const auto &Context = getContext();
+ auto SizeTy = Context.getSizeType();
+ auto T = Builder.getIntNTy(Context.getTypeSize(SizeTy));
+ llvm::Value *V = evaluateOrEmitBuiltinObjectSize(Arg, PS->getType(), T);
+ Args.add(RValue::get(V), SizeTy);
+ };
+
// We *have* to evaluate arguments from right to left in the MS C++ ABI,
// because arguments are destroyed left to right in the callee.
if (CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
@@ -2805,10 +2863,11 @@
// Evaluate each argument.
size_t CallArgsStart = Args.size();
for (int I = ArgTypes.size() - 1; I >= 0; --I) {
- CallExpr::const_arg_iterator Arg = ArgBeg + I;
+ CallExpr::const_arg_iterator Arg = ArgRange.begin() + I;
EmitCallArg(Args, *Arg, ArgTypes[I]);
- EmitNonNullArgCheck(Args.back().RV, ArgTypes[I], Arg->getExprLoc(),
+ EmitNonNullArgCheck(Args.back().RV, ArgTypes[I], (*Arg)->getExprLoc(),
CalleeDecl, ParamsToSkip + I);
+ MaybeEmitImplicitObjectSize(I, *Arg);
}
// Un-reverse the arguments we just evaluated so they match up with the LLVM
@@ -2818,21 +2877,22 @@
}
for (unsigned I = 0, E = ArgTypes.size(); I != E; ++I) {
- CallExpr::const_arg_iterator Arg = ArgBeg + I;
- assert(Arg != ArgEnd);
+ CallExpr::const_arg_iterator Arg = ArgRange.begin() + I;
+ assert(Arg != ArgRange.end());
EmitCallArg(Args, *Arg, ArgTypes[I]);
- EmitNonNullArgCheck(Args.back().RV, ArgTypes[I], Arg->getExprLoc(),
+ EmitNonNullArgCheck(Args.back().RV, ArgTypes[I], (*Arg)->getExprLoc(),
CalleeDecl, ParamsToSkip + I);
+ MaybeEmitImplicitObjectSize(I, *Arg);
}
}
namespace {
-struct DestroyUnpassedArg : EHScopeStack::Cleanup {
- DestroyUnpassedArg(llvm::Value *Addr, QualType Ty)
+struct DestroyUnpassedArg final : EHScopeStack::Cleanup {
+ DestroyUnpassedArg(Address Addr, QualType Ty)
: Addr(Addr), Ty(Ty) {}
- llvm::Value *Addr;
+ Address Addr;
QualType Ty;
void Emit(CodeGenFunction &CGF, Flags flags) override {
@@ -2843,8 +2903,6 @@
}
};
-}
-
struct DisableDebugLocationUpdates {
CodeGenFunction &CGF;
bool disabledDebugInfo;
@@ -2858,6 +2916,8 @@
}
};
+} // end anonymous namespace
+
void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
QualType type) {
DisableDebugLocationUpdates Dis(*this, E);
@@ -2906,7 +2966,8 @@
// Create a no-op GEP between the placeholder and the cleanup so we can
// RAUW it successfully. It also serves as a marker of the first
// instruction where the cleanup is active.
- pushFullExprCleanup<DestroyUnpassedArg>(EHCleanup, Slot.getAddr(), type);
+ pushFullExprCleanup<DestroyUnpassedArg>(EHCleanup, Slot.getAddress(),
+ type);
// This unreachable is a temporary marker which will be removed later.
llvm::Instruction *IsActive = Builder.CreateUnreachable();
args.addArgCleanupDeactivation(EHStack.getInnermostEHScope(), IsActive);
@@ -2923,9 +2984,8 @@
} else {
// We can't represent a misaligned lvalue in the CallArgList, so copy
// to an aligned temporary now.
- llvm::Value *tmp = CreateMemTemp(type);
- EmitAggregateCopy(tmp, L.getAddress(), type, L.isVolatile(),
- L.getAlignment());
+ Address tmp = CreateMemTemp(type);
+ EmitAggregateCopy(tmp, L.getAddress(), type, L.isVolatile());
args.add(RValue::getAggregate(tmp), type);
}
return;
@@ -2998,19 +3058,41 @@
return call;
}
+// Calls which may throw must have operand bundles indicating which funclet
+// they are nested within.
+static void
+getBundlesForFunclet(llvm::Value *Callee,
+ llvm::Instruction *CurrentFuncletPad,
+ SmallVectorImpl<llvm::OperandBundleDef> &BundleList) {
+ // There is no need for a funclet operand bundle if we aren't inside a funclet.
+ if (!CurrentFuncletPad)
+ return;
+
+ // Skip intrinsics which cannot throw.
+ auto *CalleeFn = dyn_cast<llvm::Function>(Callee->stripPointerCasts());
+ if (CalleeFn && CalleeFn->isIntrinsic() && CalleeFn->doesNotThrow())
+ return;
+
+ BundleList.emplace_back("funclet", CurrentFuncletPad);
+}
+
/// Emits a call or invoke to the given noreturn runtime function.
void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(llvm::Value *callee,
ArrayRef<llvm::Value*> args) {
+ SmallVector<llvm::OperandBundleDef, 1> BundleList;
+ getBundlesForFunclet(callee, CurrentFuncletPad, BundleList);
+
if (getInvokeDest()) {
llvm::InvokeInst *invoke =
Builder.CreateInvoke(callee,
getUnreachableBlock(),
getInvokeDest(),
- args);
+ args,
+ BundleList);
invoke->setDoesNotReturn();
invoke->setCallingConv(getRuntimeCC());
} else {
- llvm::CallInst *call = Builder.CreateCall(callee, args);
+ llvm::CallInst *call = Builder.CreateCall(callee, args, BundleList);
call->setDoesNotReturn();
call->setCallingConv(getRuntimeCC());
Builder.CreateUnreachable();
@@ -3035,12 +3117,6 @@
return callSite;
}
-llvm::CallSite
-CodeGenFunction::EmitCallOrInvoke(llvm::Value *Callee,
- const Twine &Name) {
- return EmitCallOrInvoke(Callee, None, Name);
-}
-
/// Emits a call or invoke instruction to the given function, depending
/// on the current state of the EH stack.
llvm::CallSite
@@ -3085,7 +3161,7 @@
llvm::Value *Callee,
ReturnValueSlot ReturnValue,
const CallArgList &CallArgs,
- const Decl *TargetDecl,
+ CGCalleeInfo CalleeInfo,
llvm::Instruction **callOrInvoke) {
// FIXME: We no longer need the types from CallArgs; lift up and simplify.
@@ -3100,8 +3176,10 @@
// If we're using inalloca, insert the allocation after the stack save.
// FIXME: Do this earlier rather than hacking it in here!
- llvm::AllocaInst *ArgMemory = nullptr;
+ Address ArgMemory = Address::invalid();
+ const llvm::StructLayout *ArgMemoryLayout = nullptr;
if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
+ ArgMemoryLayout = CGM.getDataLayout().getStructLayout(ArgStruct);
llvm::Instruction *IP = CallArgs.getStackBase();
llvm::AllocaInst *AI;
if (IP) {
@@ -3110,36 +3188,44 @@
} else {
AI = CreateTempAlloca(ArgStruct, "argmem");
}
+ auto Align = CallInfo.getArgStructAlignment();
+ AI->setAlignment(Align.getQuantity());
AI->setUsedWithInAlloca(true);
assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
- ArgMemory = AI;
+ ArgMemory = Address(AI, Align);
}
+ // Helper function to drill into the inalloca allocation.
+ auto createInAllocaStructGEP = [&](unsigned FieldIndex) -> Address {
+ auto FieldOffset =
+ CharUnits::fromQuantity(ArgMemoryLayout->getElementOffset(FieldIndex));
+ return Builder.CreateStructGEP(ArgMemory, FieldIndex, FieldOffset);
+ };
+
ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);
SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs());
// If the call returns a temporary with struct return, create a temporary
// alloca to hold the result, unless one is given to us.
- llvm::Value *SRetPtr = nullptr;
+ Address SRetPtr = Address::invalid();
size_t UnusedReturnSize = 0;
if (RetAI.isIndirect() || RetAI.isInAlloca()) {
- SRetPtr = ReturnValue.getValue();
- if (!SRetPtr) {
+ if (!ReturnValue.isNull()) {
+ SRetPtr = ReturnValue.getValue();
+ } else {
SRetPtr = CreateMemTemp(RetTy);
if (HaveInsertPoint() && ReturnValue.isUnused()) {
uint64_t size =
CGM.getDataLayout().getTypeAllocSize(ConvertTypeForMem(RetTy));
- if (EmitLifetimeStart(size, SRetPtr))
+ if (EmitLifetimeStart(size, SRetPtr.getPointer()))
UnusedReturnSize = size;
}
}
if (IRFunctionArgs.hasSRetArg()) {
- IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr;
+ IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr.getPointer();
} else {
- llvm::Value *Addr =
- Builder.CreateStructGEP(ArgMemory->getAllocatedType(), ArgMemory,
- RetAI.getInAllocaFieldIndex());
- Builder.CreateStore(SRetPtr, Addr);
+ Address Addr = createInAllocaStructGEP(RetAI.getInAllocaFieldIndex());
+ Builder.CreateStore(SRetPtr.getPointer(), Addr);
}
}
@@ -3152,8 +3238,6 @@
const ABIArgInfo &ArgInfo = info_it->info;
RValue RV = I->RV;
- CharUnits TypeAlign = getContext().getTypeAlignInChars(I->Ty);
-
// Insert a padding argument to ensure proper alignment.
if (IRFunctionArgs.hasPaddingArg(ArgNo))
IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
@@ -3169,27 +3253,23 @@
if (RV.isAggregate()) {
// Replace the placeholder with the appropriate argument slot GEP.
llvm::Instruction *Placeholder =
- cast<llvm::Instruction>(RV.getAggregateAddr());
+ cast<llvm::Instruction>(RV.getAggregatePointer());
CGBuilderTy::InsertPoint IP = Builder.saveIP();
Builder.SetInsertPoint(Placeholder);
- llvm::Value *Addr =
- Builder.CreateStructGEP(ArgMemory->getAllocatedType(), ArgMemory,
- ArgInfo.getInAllocaFieldIndex());
+ Address Addr = createInAllocaStructGEP(ArgInfo.getInAllocaFieldIndex());
Builder.restoreIP(IP);
- deferPlaceholderReplacement(Placeholder, Addr);
+ deferPlaceholderReplacement(Placeholder, Addr.getPointer());
} else {
// Store the RValue into the argument struct.
- llvm::Value *Addr =
- Builder.CreateStructGEP(ArgMemory->getAllocatedType(), ArgMemory,
- ArgInfo.getInAllocaFieldIndex());
- unsigned AS = Addr->getType()->getPointerAddressSpace();
+ Address Addr = createInAllocaStructGEP(ArgInfo.getInAllocaFieldIndex());
+ unsigned AS = Addr.getType()->getPointerAddressSpace();
llvm::Type *MemType = ConvertTypeForMem(I->Ty)->getPointerTo(AS);
// There are some cases where a trivial bitcast is not avoidable. The
// definition of a type later in a translation unit may change it's type
// from {}* to (%struct.foo*)*.
- if (Addr->getType() != MemType)
+ if (Addr.getType() != MemType)
Addr = Builder.CreateBitCast(Addr, MemType);
- LValue argLV = MakeAddrLValue(Addr, I->Ty, TypeAlign);
+ LValue argLV = MakeAddrLValue(Addr, I->Ty);
EmitInitStoreOfNonAggregate(*this, RV, argLV);
}
break;
@@ -3199,12 +3279,10 @@
assert(NumIRArgs == 1);
if (RV.isScalar() || RV.isComplex()) {
// Make a temporary alloca to pass the argument.
- llvm::AllocaInst *AI = CreateMemTemp(I->Ty);
- if (ArgInfo.getIndirectAlign() > AI->getAlignment())
- AI->setAlignment(ArgInfo.getIndirectAlign());
- IRCallArgs[FirstIRArg] = AI;
+ Address Addr = CreateMemTemp(I->Ty, ArgInfo.getIndirectAlign());
+ IRCallArgs[FirstIRArg] = Addr.getPointer();
- LValue argLV = MakeAddrLValue(AI, I->Ty, TypeAlign);
+ LValue argLV = MakeAddrLValue(Addr, I->Ty);
EmitInitStoreOfNonAggregate(*this, RV, argLV);
} else {
// We want to avoid creating an unnecessary temporary+copy here;
@@ -3215,27 +3293,27 @@
// we cannot force it to be sufficiently aligned.
// 3. If the argument is byval, but RV is located in an address space
// different than that of the argument (0).
- llvm::Value *Addr = RV.getAggregateAddr();
- unsigned Align = ArgInfo.getIndirectAlign();
+ Address Addr = RV.getAggregateAddress();
+ CharUnits Align = ArgInfo.getIndirectAlign();
const llvm::DataLayout *TD = &CGM.getDataLayout();
- const unsigned RVAddrSpace = Addr->getType()->getPointerAddressSpace();
+ const unsigned RVAddrSpace = Addr.getType()->getAddressSpace();
const unsigned ArgAddrSpace =
(FirstIRArg < IRFuncTy->getNumParams()
? IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace()
: 0);
if ((!ArgInfo.getIndirectByVal() && I->NeedsCopy) ||
- (ArgInfo.getIndirectByVal() && TypeAlign.getQuantity() < Align &&
- llvm::getOrEnforceKnownAlignment(Addr, Align, *TD) < Align) ||
+ (ArgInfo.getIndirectByVal() && Addr.getAlignment() < Align &&
+ llvm::getOrEnforceKnownAlignment(Addr.getPointer(),
+ Align.getQuantity(), *TD)
+ < Align.getQuantity()) ||
(ArgInfo.getIndirectByVal() && (RVAddrSpace != ArgAddrSpace))) {
// Create an aligned temporary, and copy to it.
- llvm::AllocaInst *AI = CreateMemTemp(I->Ty);
- if (Align > AI->getAlignment())
- AI->setAlignment(Align);
- IRCallArgs[FirstIRArg] = AI;
+ Address AI = CreateMemTemp(I->Ty, ArgInfo.getIndirectAlign());
+ IRCallArgs[FirstIRArg] = AI.getPointer();
EmitAggregateCopy(AI, Addr, I->Ty, RV.isVolatileQualified());
} else {
// Skip the extra memcpy call.
- IRCallArgs[FirstIRArg] = Addr;
+ IRCallArgs[FirstIRArg] = Addr.getPointer();
}
}
break;
@@ -3255,7 +3333,7 @@
if (RV.isScalar())
V = RV.getScalarVal();
else
- V = Builder.CreateLoad(RV.getAggregateAddr());
+ V = Builder.CreateLoad(RV.getAggregateAddress());
// We might have to widen integers, but we should never truncate.
if (ArgInfo.getCoerceToType() != V->getType() &&
@@ -3272,30 +3350,24 @@
}
// FIXME: Avoid the conversion through memory if possible.
- llvm::Value *SrcPtr;
+ Address Src = Address::invalid();
if (RV.isScalar() || RV.isComplex()) {
- SrcPtr = CreateMemTemp(I->Ty, "coerce");
- LValue SrcLV = MakeAddrLValue(SrcPtr, I->Ty, TypeAlign);
+ Src = CreateMemTemp(I->Ty, "coerce");
+ LValue SrcLV = MakeAddrLValue(Src, I->Ty);
EmitInitStoreOfNonAggregate(*this, RV, SrcLV);
- } else
- SrcPtr = RV.getAggregateAddr();
+ } else {
+ Src = RV.getAggregateAddress();
+ }
// If the value is offset in memory, apply the offset now.
- if (unsigned Offs = ArgInfo.getDirectOffset()) {
- SrcPtr = Builder.CreateBitCast(SrcPtr, Builder.getInt8PtrTy());
- SrcPtr = Builder.CreateConstGEP1_32(Builder.getInt8Ty(), SrcPtr, Offs);
- SrcPtr = Builder.CreateBitCast(SrcPtr,
- llvm::PointerType::getUnqual(ArgInfo.getCoerceToType()));
-
- }
+ Src = emitAddressAtOffset(*this, Src, ArgInfo);
// Fast-isel and the optimizer generally like scalar values better than
// FCAs, so we flatten them if this is safe to do for this argument.
llvm::StructType *STy =
dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType());
if (STy && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
- llvm::Type *SrcTy =
- cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
+ llvm::Type *SrcTy = Src.getType()->getElementType();
uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(SrcTy);
uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(STy);
@@ -3304,28 +3376,28 @@
// of the destination type to allow loading all of it. The bits past
// the source value are left undef.
if (SrcSize < DstSize) {
- llvm::AllocaInst *TempAlloca
- = CreateTempAlloca(STy, SrcPtr->getName() + ".coerce");
- Builder.CreateMemCpy(TempAlloca, SrcPtr, SrcSize, 0);
- SrcPtr = TempAlloca;
+ Address TempAlloca
+ = CreateTempAlloca(STy, Src.getAlignment(),
+ Src.getName() + ".coerce");
+ Builder.CreateMemCpy(TempAlloca, Src, SrcSize);
+ Src = TempAlloca;
} else {
- SrcPtr = Builder.CreateBitCast(SrcPtr,
- llvm::PointerType::getUnqual(STy));
+ Src = Builder.CreateBitCast(Src, llvm::PointerType::getUnqual(STy));
}
+ auto SrcLayout = CGM.getDataLayout().getStructLayout(STy);
assert(NumIRArgs == STy->getNumElements());
for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
- llvm::Value *EltPtr = Builder.CreateConstGEP2_32(STy, SrcPtr, 0, i);
- llvm::LoadInst *LI = Builder.CreateLoad(EltPtr);
- // We don't know what we're loading from.
- LI->setAlignment(1);
+ auto Offset = CharUnits::fromQuantity(SrcLayout->getElementOffset(i));
+ Address EltPtr = Builder.CreateStructGEP(Src, i, Offset);
+ llvm::Value *LI = Builder.CreateLoad(EltPtr);
IRCallArgs[FirstIRArg + i] = LI;
}
} else {
// In the simple case, just pass the coerced loaded value.
assert(NumIRArgs == 1);
IRCallArgs[FirstIRArg] =
- CreateCoercedLoad(SrcPtr, ArgInfo.getCoerceToType(), *this);
+ CreateCoercedLoad(Src, ArgInfo.getCoerceToType(), *this);
}
break;
@@ -3339,8 +3411,8 @@
}
}
- if (ArgMemory) {
- llvm::Value *Arg = ArgMemory;
+ if (ArgMemory.isValid()) {
+ llvm::Value *Arg = ArgMemory.getPointer();
if (CallInfo.isVariadic()) {
// When passing non-POD arguments by value to variadic functions, we will
// end up with a variadic prototype and an inalloca call site. In such
@@ -3418,23 +3490,37 @@
unsigned CallingConv;
CodeGen::AttributeListType AttributeList;
- CGM.ConstructAttributeList(CallInfo, TargetDecl, AttributeList,
- CallingConv, true);
+ CGM.ConstructAttributeList(CallInfo, CalleeInfo, AttributeList, CallingConv,
+ true);
llvm::AttributeSet Attrs = llvm::AttributeSet::get(getLLVMContext(),
AttributeList);
- llvm::BasicBlock *InvokeDest = nullptr;
- if (!Attrs.hasAttribute(llvm::AttributeSet::FunctionIndex,
- llvm::Attribute::NoUnwind) ||
- currentFunctionUsesSEHTry())
- InvokeDest = getInvokeDest();
+ bool CannotThrow;
+ if (currentFunctionUsesSEHTry()) {
+ // SEH cares about asynchronous exceptions, everything can "throw."
+ CannotThrow = false;
+ } else if (isCleanupPadScope() &&
+ EHPersonality::get(*this).isMSVCXXPersonality()) {
+ // The MSVC++ personality will implicitly terminate the program if an
+ // exception is thrown. An unwind edge cannot be reached.
+ CannotThrow = true;
+ } else {
+ // Otherwise, nowunind callsites will never throw.
+ CannotThrow = Attrs.hasAttribute(llvm::AttributeSet::FunctionIndex,
+ llvm::Attribute::NoUnwind);
+ }
+ llvm::BasicBlock *InvokeDest = CannotThrow ? nullptr : getInvokeDest();
+
+ SmallVector<llvm::OperandBundleDef, 1> BundleList;
+ getBundlesForFunclet(Callee, CurrentFuncletPad, BundleList);
llvm::CallSite CS;
if (!InvokeDest) {
- CS = Builder.CreateCall(Callee, IRCallArgs);
+ CS = Builder.CreateCall(Callee, IRCallArgs, BundleList);
} else {
llvm::BasicBlock *Cont = createBasicBlock("invoke.cont");
- CS = Builder.CreateInvoke(Callee, Cont, InvokeDest, IRCallArgs);
+ CS = Builder.CreateInvoke(Callee, Cont, InvokeDest, IRCallArgs,
+ BundleList);
EmitBlock(Cont);
}
if (callOrInvoke)
@@ -3466,7 +3552,7 @@
if (CS.doesNotReturn()) {
if (UnusedReturnSize)
EmitLifetimeEnd(llvm::ConstantInt::get(Int64Ty, UnusedReturnSize),
- SRetPtr);
+ SRetPtr.getPointer());
Builder.CreateUnreachable();
Builder.ClearInsertionPoint();
@@ -3493,6 +3579,12 @@
// lexical order, so deactivate it and run it manually here.
CallArgs.freeArgumentMemory(*this);
+ if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(CI)) {
+ const Decl *TargetDecl = CalleeInfo.getCalleeDecl();
+ if (TargetDecl && TargetDecl->hasAttr<NotTailCalledAttr>())
+ Call->setTailCallKind(llvm::CallInst::TCK_NoTail);
+ }
+
RValue Ret = [&] {
switch (RetAI.getKind()) {
case ABIArgInfo::InAlloca:
@@ -3500,7 +3592,7 @@
RValue ret = convertTempToRValue(SRetPtr, RetTy, SourceLocation());
if (UnusedReturnSize)
EmitLifetimeEnd(llvm::ConstantInt::get(Int64Ty, UnusedReturnSize),
- SRetPtr);
+ SRetPtr.getPointer());
return ret;
}
@@ -3520,14 +3612,14 @@
return RValue::getComplex(std::make_pair(Real, Imag));
}
case TEK_Aggregate: {
- llvm::Value *DestPtr = ReturnValue.getValue();
+ Address DestPtr = ReturnValue.getValue();
bool DestIsVolatile = ReturnValue.isVolatile();
- if (!DestPtr) {
+ if (!DestPtr.isValid()) {
DestPtr = CreateMemTemp(RetTy, "agg.tmp");
DestIsVolatile = false;
}
- BuildAggStore(*this, CI, DestPtr, DestIsVolatile, false);
+ BuildAggStore(*this, CI, DestPtr, DestIsVolatile);
return RValue::getAggregate(DestPtr);
}
case TEK_Scalar: {
@@ -3542,23 +3634,16 @@
llvm_unreachable("bad evaluation kind");
}
- llvm::Value *DestPtr = ReturnValue.getValue();
+ Address DestPtr = ReturnValue.getValue();
bool DestIsVolatile = ReturnValue.isVolatile();
- if (!DestPtr) {
+ if (!DestPtr.isValid()) {
DestPtr = CreateMemTemp(RetTy, "coerce");
DestIsVolatile = false;
}
// If the value is offset in memory, apply the offset now.
- llvm::Value *StorePtr = DestPtr;
- if (unsigned Offs = RetAI.getDirectOffset()) {
- StorePtr = Builder.CreateBitCast(StorePtr, Builder.getInt8PtrTy());
- StorePtr =
- Builder.CreateConstGEP1_32(Builder.getInt8Ty(), StorePtr, Offs);
- StorePtr = Builder.CreateBitCast(StorePtr,
- llvm::PointerType::getUnqual(RetAI.getCoerceToType()));
- }
+ Address StorePtr = emitAddressAtOffset(*this, DestPtr, RetAI);
CreateCoercedStore(CI, StorePtr, DestIsVolatile, *this);
return convertTempToRValue(DestPtr, RetTy, SourceLocation());
@@ -3571,6 +3656,8 @@
llvm_unreachable("Unhandled ABIArgInfo::Kind");
} ();
+ const Decl *TargetDecl = CalleeInfo.getCalleeDecl();
+
if (Ret.isScalar() && TargetDecl) {
if (const auto *AA = TargetDecl->getAttr<AssumeAlignedAttr>()) {
llvm::Value *OffsetValue = nullptr;
@@ -3589,6 +3676,12 @@
/* VarArg handling */
-llvm::Value *CodeGenFunction::EmitVAArg(llvm::Value *VAListAddr, QualType Ty) {
- return CGM.getTypes().getABIInfo().EmitVAArg(VAListAddr, Ty, *this);
+Address CodeGenFunction::EmitVAArg(VAArgExpr *VE, Address &VAListAddr) {
+ VAListAddr = VE->isMicrosoftABI()
+ ? EmitMSVAListRef(VE->getSubExpr())
+ : EmitVAListRef(VE->getSubExpr());
+ QualType Ty = VE->getType();
+ if (VE->isMicrosoftABI())
+ return CGM.getTypes().getABIInfo().EmitMSVAArg(*this, VAListAddr, Ty);
+ return CGM.getTypes().getABIInfo().EmitVAArg(*this, VAListAddr, Ty);
}
diff --git a/lib/CodeGen/CGCall.h b/lib/CodeGen/CGCall.h
index 7a4708e..2ebd09b 100644
--- a/lib/CodeGen/CGCall.h
+++ b/lib/CodeGen/CGCall.h
@@ -56,7 +56,7 @@
class CallArgList :
public SmallVector<CallArg, 16> {
public:
- CallArgList() : StackBase(nullptr), StackBaseMem(nullptr) {}
+ CallArgList() : StackBase(nullptr) {}
struct Writeback {
/// The original argument. Note that the argument l-value
@@ -64,7 +64,7 @@
LValue Source;
/// The temporary alloca.
- llvm::Value *Temporary;
+ Address Temporary;
/// A value to "use" after the writeback, or null.
llvm::Value *ToUse;
@@ -88,12 +88,9 @@
other.Writebacks.begin(), other.Writebacks.end());
}
- void addWriteback(LValue srcLV, llvm::Value *temporary,
+ void addWriteback(LValue srcLV, Address temporary,
llvm::Value *toUse) {
- Writeback writeback;
- writeback.Source = srcLV;
- writeback.Temporary = temporary;
- writeback.ToUse = toUse;
+ Writeback writeback = { srcLV, temporary, toUse };
Writebacks.push_back(writeback);
}
@@ -137,9 +134,6 @@
/// The stacksave call. It dominates all of the argument evaluation.
llvm::CallInst *StackBase;
- /// The alloca holding the stackbase. We need it to maintain SSA form.
- llvm::AllocaInst *StackBaseMem;
-
/// The iterator pointing to the stack restore cleanup. We manually run and
/// deactivate this cleanup after the call in the unexceptional case because
/// it doesn't run in the normal order.
@@ -156,6 +150,7 @@
/// function can be stored, and whether the address is volatile or not.
class ReturnValueSlot {
llvm::PointerIntPair<llvm::Value *, 2, unsigned int> Value;
+ CharUnits Alignment;
// Return value slot flags
enum Flags {
@@ -165,14 +160,15 @@
public:
ReturnValueSlot() {}
- ReturnValueSlot(llvm::Value *Value, bool IsVolatile, bool IsUnused = false)
- : Value(Value,
- (IsVolatile ? IS_VOLATILE : 0) | (IsUnused ? IS_UNUSED : 0)) {}
+ ReturnValueSlot(Address Addr, bool IsVolatile, bool IsUnused = false)
+ : Value(Addr.isValid() ? Addr.getPointer() : nullptr,
+ (IsVolatile ? IS_VOLATILE : 0) | (IsUnused ? IS_UNUSED : 0)),
+ Alignment(Addr.isValid() ? Addr.getAlignment() : CharUnits::Zero()) {}
- bool isNull() const { return !getValue(); }
+ bool isNull() const { return !getValue().isValid(); }
bool isVolatile() const { return Value.getInt() & IS_VOLATILE; }
- llvm::Value *getValue() const { return Value.getPointer(); }
+ Address getValue() const { return Address(Value.getPointer(), Alignment); }
bool isUnused() const { return Value.getInt() & IS_UNUSED; }
};
diff --git a/lib/CodeGen/CGClass.cpp b/lib/CodeGen/CGClass.cpp
index 1320cd3..2e566de 100644
--- a/lib/CodeGen/CGClass.cpp
+++ b/lib/CodeGen/CGClass.cpp
@@ -1,4 +1,4 @@
-//===--- CGClass.cpp - Emit LLVM Code for C++ classes ---------------------===//
+//===--- CGClass.cpp - Emit LLVM Code for C++ classes -----------*- C++ -*-===//
//
// The LLVM Compiler Infrastructure
//
@@ -25,17 +25,130 @@
#include "clang/CodeGen/CGFunctionInfo.h"
#include "clang/Frontend/CodeGenOptions.h"
#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/Metadata.h"
using namespace clang;
using namespace CodeGen;
-static CharUnits
-ComputeNonVirtualBaseClassOffset(ASTContext &Context,
- const CXXRecordDecl *DerivedClass,
- CastExpr::path_const_iterator Start,
- CastExpr::path_const_iterator End) {
+/// Return the best known alignment for an unknown pointer to a
+/// particular class.
+CharUnits CodeGenModule::getClassPointerAlignment(const CXXRecordDecl *RD) {
+ if (!RD->isCompleteDefinition())
+ return CharUnits::One(); // Hopefully won't be used anywhere.
+
+ auto &layout = getContext().getASTRecordLayout(RD);
+
+ // If the class is final, then we know that the pointer points to an
+ // object of that type and can use the full alignment.
+ if (RD->hasAttr<FinalAttr>()) {
+ return layout.getAlignment();
+
+ // Otherwise, we have to assume it could be a subclass.
+ } else {
+ return layout.getNonVirtualAlignment();
+ }
+}
+
+/// Return the best known alignment for a pointer to a virtual base,
+/// given the alignment of a pointer to the derived class.
+CharUnits CodeGenModule::getVBaseAlignment(CharUnits actualDerivedAlign,
+ const CXXRecordDecl *derivedClass,
+ const CXXRecordDecl *vbaseClass) {
+ // The basic idea here is that an underaligned derived pointer might
+ // indicate an underaligned base pointer.
+
+ assert(vbaseClass->isCompleteDefinition());
+ auto &baseLayout = getContext().getASTRecordLayout(vbaseClass);
+ CharUnits expectedVBaseAlign = baseLayout.getNonVirtualAlignment();
+
+ return getDynamicOffsetAlignment(actualDerivedAlign, derivedClass,
+ expectedVBaseAlign);
+}
+
+CharUnits
+CodeGenModule::getDynamicOffsetAlignment(CharUnits actualBaseAlign,
+ const CXXRecordDecl *baseDecl,
+ CharUnits expectedTargetAlign) {
+ // If the base is an incomplete type (which is, alas, possible with
+ // member pointers), be pessimistic.
+ if (!baseDecl->isCompleteDefinition())
+ return std::min(actualBaseAlign, expectedTargetAlign);
+
+ auto &baseLayout = getContext().getASTRecordLayout(baseDecl);
+ CharUnits expectedBaseAlign = baseLayout.getNonVirtualAlignment();
+
+ // If the class is properly aligned, assume the target offset is, too.
+ //
+ // This actually isn't necessarily the right thing to do --- if the
+ // class is a complete object, but it's only properly aligned for a
+ // base subobject, then the alignments of things relative to it are
+ // probably off as well. (Note that this requires the alignment of
+ // the target to be greater than the NV alignment of the derived
+ // class.)
+ //
+ // However, our approach to this kind of under-alignment can only
+ // ever be best effort; after all, we're never going to propagate
+ // alignments through variables or parameters. Note, in particular,
+ // that constructing a polymorphic type in an address that's less
+ // than pointer-aligned will generally trap in the constructor,
+ // unless we someday add some sort of attribute to change the
+ // assumed alignment of 'this'. So our goal here is pretty much
+ // just to allow the user to explicitly say that a pointer is
+ // under-aligned and then safely access its fields and v-tables.
+ if (actualBaseAlign >= expectedBaseAlign) {
+ return expectedTargetAlign;
+ }
+
+ // Otherwise, we might be offset by an arbitrary multiple of the
+ // actual alignment. The correct adjustment is to take the min of
+ // the two alignments.
+ return std::min(actualBaseAlign, expectedTargetAlign);
+}
+
+Address CodeGenFunction::LoadCXXThisAddress() {
+ assert(CurFuncDecl && "loading 'this' without a func declaration?");
+ assert(isa<CXXMethodDecl>(CurFuncDecl));
+
+ // Lazily compute CXXThisAlignment.
+ if (CXXThisAlignment.isZero()) {
+ // Just use the best known alignment for the parent.
+ // TODO: if we're currently emitting a complete-object ctor/dtor,
+ // we can always use the complete-object alignment.
+ auto RD = cast<CXXMethodDecl>(CurFuncDecl)->getParent();
+ CXXThisAlignment = CGM.getClassPointerAlignment(RD);
+ }
+
+ return Address(LoadCXXThis(), CXXThisAlignment);
+}
+
+/// Emit the address of a field using a member data pointer.
+///
+/// \param E Only used for emergency diagnostics
+Address
+CodeGenFunction::EmitCXXMemberDataPointerAddress(const Expr *E, Address base,
+ llvm::Value *memberPtr,
+ const MemberPointerType *memberPtrType,
+ AlignmentSource *alignSource) {
+ // Ask the ABI to compute the actual address.
+ llvm::Value *ptr =
+ CGM.getCXXABI().EmitMemberDataPointerAddress(*this, E, base,
+ memberPtr, memberPtrType);
+
+ QualType memberType = memberPtrType->getPointeeType();
+ CharUnits memberAlign = getNaturalTypeAlignment(memberType, alignSource);
+ memberAlign =
+ CGM.getDynamicOffsetAlignment(base.getAlignment(),
+ memberPtrType->getClass()->getAsCXXRecordDecl(),
+ memberAlign);
+ return Address(ptr, memberAlign);
+}
+
+CharUnits CodeGenModule::computeNonVirtualBaseClassOffset(
+ const CXXRecordDecl *DerivedClass, CastExpr::path_const_iterator Start,
+ CastExpr::path_const_iterator End) {
CharUnits Offset = CharUnits::Zero();
+ const ASTContext &Context = getContext();
const CXXRecordDecl *RD = DerivedClass;
for (CastExpr::path_const_iterator I = Start; I != End; ++I) {
@@ -64,8 +177,7 @@
assert(PathBegin != PathEnd && "Base path should not be empty!");
CharUnits Offset =
- ComputeNonVirtualBaseClassOffset(getContext(), ClassDecl,
- PathBegin, PathEnd);
+ computeNonVirtualBaseClassOffset(ClassDecl, PathBegin, PathEnd);
if (Offset.isZero())
return nullptr;
@@ -80,15 +192,13 @@
/// when the type is known to be complete (e.g. in complete destructors).
///
/// The object pointed to by 'This' is assumed to be non-null.
-llvm::Value *
-CodeGenFunction::GetAddressOfDirectBaseInCompleteClass(llvm::Value *This,
+Address
+CodeGenFunction::GetAddressOfDirectBaseInCompleteClass(Address This,
const CXXRecordDecl *Derived,
const CXXRecordDecl *Base,
bool BaseIsVirtual) {
// 'this' must be a pointer (in some address space) to Derived.
- assert(This->getType()->isPointerTy() &&
- cast<llvm::PointerType>(This->getType())->getElementType()
- == ConvertType(Derived));
+ assert(This.getElementType() == ConvertType(Derived));
// Compute the offset of the virtual base.
CharUnits Offset;
@@ -100,20 +210,22 @@
// Shift and cast down to the base type.
// TODO: for complete types, this should be possible with a GEP.
- llvm::Value *V = This;
- if (Offset.isPositive()) {
- V = Builder.CreateBitCast(V, Int8PtrTy);
- V = Builder.CreateConstInBoundsGEP1_64(V, Offset.getQuantity());
+ Address V = This;
+ if (!Offset.isZero()) {
+ V = Builder.CreateElementBitCast(V, Int8Ty);
+ V = Builder.CreateConstInBoundsByteGEP(V, Offset);
}
- V = Builder.CreateBitCast(V, ConvertType(Base)->getPointerTo());
+ V = Builder.CreateElementBitCast(V, ConvertType(Base));
return V;
}
-static llvm::Value *
-ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, llvm::Value *ptr,
+static Address
+ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, Address addr,
CharUnits nonVirtualOffset,
- llvm::Value *virtualOffset) {
+ llvm::Value *virtualOffset,
+ const CXXRecordDecl *derivedClass,
+ const CXXRecordDecl *nearestVBase) {
// Assert that we have something to do.
assert(!nonVirtualOffset.isZero() || virtualOffset != nullptr);
@@ -130,13 +242,27 @@
}
// Apply the base offset.
+ llvm::Value *ptr = addr.getPointer();
ptr = CGF.Builder.CreateBitCast(ptr, CGF.Int8PtrTy);
ptr = CGF.Builder.CreateInBoundsGEP(ptr, baseOffset, "add.ptr");
- return ptr;
+
+ // If we have a virtual component, the alignment of the result will
+ // be relative only to the known alignment of that vbase.
+ CharUnits alignment;
+ if (virtualOffset) {
+ assert(nearestVBase && "virtual offset without vbase?");
+ alignment = CGF.CGM.getVBaseAlignment(addr.getAlignment(),
+ derivedClass, nearestVBase);
+ } else {
+ alignment = addr.getAlignment();
+ }
+ alignment = alignment.alignmentAtOffset(nonVirtualOffset);
+
+ return Address(ptr, alignment);
}
-llvm::Value *CodeGenFunction::GetAddressOfBaseClass(
- llvm::Value *Value, const CXXRecordDecl *Derived,
+Address CodeGenFunction::GetAddressOfBaseClass(
+ Address Value, const CXXRecordDecl *Derived,
CastExpr::path_const_iterator PathBegin,
CastExpr::path_const_iterator PathEnd, bool NullCheckValue,
SourceLocation Loc) {
@@ -158,9 +284,8 @@
// Compute the static offset of the ultimate destination within its
// allocating subobject (the virtual base, if there is one, or else
// the "complete" object that we see).
- CharUnits NonVirtualOffset =
- ComputeNonVirtualBaseClassOffset(getContext(), VBase ? VBase : Derived,
- Start, PathEnd);
+ CharUnits NonVirtualOffset = CGM.computeNonVirtualBaseClassOffset(
+ VBase ? VBase : Derived, Start, PathEnd);
// If there's a virtual step, we can sometimes "devirtualize" it.
// For now, that's limited to when the derived type is final.
@@ -177,14 +302,14 @@
ConvertType((PathEnd[-1])->getType())->getPointerTo();
QualType DerivedTy = getContext().getRecordType(Derived);
- CharUnits DerivedAlign = getContext().getTypeAlignInChars(DerivedTy);
+ CharUnits DerivedAlign = CGM.getClassPointerAlignment(Derived);
// If the static offset is zero and we don't have a virtual step,
// just do a bitcast; null checks are unnecessary.
if (NonVirtualOffset.isZero() && !VBase) {
if (sanitizePerformTypeCheck()) {
- EmitTypeCheck(TCK_Upcast, Loc, Value, DerivedTy, DerivedAlign,
- !NullCheckValue);
+ EmitTypeCheck(TCK_Upcast, Loc, Value.getPointer(),
+ DerivedTy, DerivedAlign, !NullCheckValue);
}
return Builder.CreateBitCast(Value, BasePtrTy);
}
@@ -199,14 +324,14 @@
llvm::BasicBlock *notNullBB = createBasicBlock("cast.notnull");
endBB = createBasicBlock("cast.end");
- llvm::Value *isNull = Builder.CreateIsNull(Value);
+ llvm::Value *isNull = Builder.CreateIsNull(Value.getPointer());
Builder.CreateCondBr(isNull, endBB, notNullBB);
EmitBlock(notNullBB);
}
if (sanitizePerformTypeCheck()) {
- EmitTypeCheck(VBase ? TCK_UpcastToVirtualBase : TCK_Upcast, Loc, Value,
- DerivedTy, DerivedAlign, true);
+ EmitTypeCheck(VBase ? TCK_UpcastToVirtualBase : TCK_Upcast, Loc,
+ Value.getPointer(), DerivedTy, DerivedAlign, true);
}
// Compute the virtual offset.
@@ -217,9 +342,8 @@
}
// Apply both offsets.
- Value = ApplyNonVirtualAndVirtualOffset(*this, Value,
- NonVirtualOffset,
- VirtualOffset);
+ Value = ApplyNonVirtualAndVirtualOffset(*this, Value, NonVirtualOffset,
+ VirtualOffset, Derived, VBase);
// Cast to the destination type.
Value = Builder.CreateBitCast(Value, BasePtrTy);
@@ -231,16 +355,16 @@
EmitBlock(endBB);
llvm::PHINode *PHI = Builder.CreatePHI(BasePtrTy, 2, "cast.result");
- PHI->addIncoming(Value, notNullBB);
+ PHI->addIncoming(Value.getPointer(), notNullBB);
PHI->addIncoming(llvm::Constant::getNullValue(BasePtrTy), origBB);
- Value = PHI;
+ Value = Address(PHI, Value.getAlignment());
}
return Value;
}
-llvm::Value *
-CodeGenFunction::GetAddressOfDerivedClass(llvm::Value *Value,
+Address
+CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr,
const CXXRecordDecl *Derived,
CastExpr::path_const_iterator PathBegin,
CastExpr::path_const_iterator PathEnd,
@@ -256,7 +380,7 @@
if (!NonVirtualOffset) {
// No offset, we can just cast back.
- return Builder.CreateBitCast(Value, DerivedPtrTy);
+ return Builder.CreateBitCast(BaseAddr, DerivedPtrTy);
}
llvm::BasicBlock *CastNull = nullptr;
@@ -268,19 +392,20 @@
CastNotNull = createBasicBlock("cast.notnull");
CastEnd = createBasicBlock("cast.end");
- llvm::Value *IsNull = Builder.CreateIsNull(Value);
+ llvm::Value *IsNull = Builder.CreateIsNull(BaseAddr.getPointer());
Builder.CreateCondBr(IsNull, CastNull, CastNotNull);
EmitBlock(CastNotNull);
}
// Apply the offset.
- Value = Builder.CreateBitCast(Value, Int8PtrTy);
+ llvm::Value *Value = Builder.CreateBitCast(BaseAddr.getPointer(), Int8PtrTy);
Value = Builder.CreateGEP(Value, Builder.CreateNeg(NonVirtualOffset),
"sub.ptr");
// Just cast.
Value = Builder.CreateBitCast(Value, DerivedPtrTy);
+ // Produce a PHI if we had a null-check.
if (NullCheckValue) {
Builder.CreateBr(CastEnd);
EmitBlock(CastNull);
@@ -289,12 +414,11 @@
llvm::PHINode *PHI = Builder.CreatePHI(Value->getType(), 2);
PHI->addIncoming(Value, CastNotNull);
- PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()),
- CastNull);
+ PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()), CastNull);
Value = PHI;
}
- return Value;
+ return Address(Value, CGM.getClassPointerAlignment(Derived));
}
llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD,
@@ -348,7 +472,7 @@
namespace {
/// Call the destructor for a direct base class.
- struct CallBaseDtor : EHScopeStack::Cleanup {
+ struct CallBaseDtor final : EHScopeStack::Cleanup {
const CXXRecordDecl *BaseClass;
bool BaseIsVirtual;
CallBaseDtor(const CXXRecordDecl *Base, bool BaseIsVirtual)
@@ -359,8 +483,8 @@
cast<CXXMethodDecl>(CGF.CurCodeDecl)->getParent();
const CXXDestructorDecl *D = BaseClass->getDestructor();
- llvm::Value *Addr =
- CGF.GetAddressOfDirectBaseInCompleteClass(CGF.LoadCXXThis(),
+ Address Addr =
+ CGF.GetAddressOfDirectBaseInCompleteClass(CGF.LoadCXXThisAddress(),
DerivedClass, BaseClass,
BaseIsVirtual);
CGF.EmitCXXDestructorCall(D, Dtor_Base, BaseIsVirtual,
@@ -384,7 +508,7 @@
// external code might potentially access the vtable.
void VisitCXXThisExpr(const CXXThisExpr *E) { UsesThis = true; }
};
-}
+} // end anonymous namespace
static bool BaseInitializerUsesThis(ASTContext &C, const Expr *Init) {
DynamicThisUseChecker Checker(C);
@@ -399,7 +523,7 @@
assert(BaseInit->isBaseInitializer() &&
"Must have base initializer!");
- llvm::Value *ThisPtr = CGF.LoadCXXThis();
+ Address ThisPtr = CGF.LoadCXXThisAddress();
const Type *BaseType = BaseInit->getBaseClass();
CXXRecordDecl *BaseClassDecl =
@@ -419,13 +543,12 @@
// We can pretend to be a complete class because it only matters for
// virtual bases, and we only do virtual bases for complete ctors.
- llvm::Value *V =
+ Address V =
CGF.GetAddressOfDirectBaseInCompleteClass(ThisPtr, ClassDecl,
BaseClassDecl,
isBaseVirtual);
- CharUnits Alignment = CGF.getContext().getTypeAlignInChars(BaseType);
AggValueSlot AggSlot =
- AggValueSlot::forAddr(V, Alignment, Qualifiers(),
+ AggValueSlot::forAddr(V, Qualifiers(),
AggValueSlot::IsDestructed,
AggValueSlot::DoesNotNeedGCBarriers,
AggValueSlot::IsNotAliased);
@@ -441,17 +564,17 @@
static void EmitAggMemberInitializer(CodeGenFunction &CGF,
LValue LHS,
Expr *Init,
- llvm::Value *ArrayIndexVar,
+ Address ArrayIndexVar,
QualType T,
ArrayRef<VarDecl *> ArrayIndexes,
unsigned Index) {
if (Index == ArrayIndexes.size()) {
LValue LV = LHS;
- if (ArrayIndexVar) {
+ if (ArrayIndexVar.isValid()) {
// If we have an array index variable, load it and use it as an offset.
// Then, increment the value.
- llvm::Value *Dest = LHS.getAddress();
+ llvm::Value *Dest = LHS.getPointer();
llvm::Value *ArrayIndex = CGF.Builder.CreateLoad(ArrayIndexVar);
Dest = CGF.Builder.CreateInBoundsGEP(Dest, ArrayIndex, "destaddress");
llvm::Value *Next = llvm::ConstantInt::get(ArrayIndex->getType(), 1);
@@ -459,9 +582,9 @@
CGF.Builder.CreateStore(Next, ArrayIndexVar);
// Update the LValue.
- LV.setAddress(Dest);
- CharUnits Align = CGF.getContext().getTypeAlignInChars(T);
- LV.setAlignment(std::min(Align, LV.getAlignment()));
+ CharUnits EltSize = CGF.getContext().getTypeSizeInChars(T);
+ CharUnits Align = LV.getAlignment().alignmentOfArrayElement(EltSize);
+ LV.setAddress(Address(Dest, Align));
}
switch (CGF.getEvaluationKind(T)) {
@@ -488,14 +611,11 @@
const ConstantArrayType *Array = CGF.getContext().getAsConstantArrayType(T);
assert(Array && "Array initialization without the array type?");
- llvm::Value *IndexVar
- = CGF.GetAddrOfLocalVar(ArrayIndexes[Index]);
- assert(IndexVar && "Array index variable not loaded");
+ Address IndexVar = CGF.GetAddrOfLocalVar(ArrayIndexes[Index]);
// Initialize this index variable to zero.
llvm::Value* Zero
- = llvm::Constant::getNullValue(
- CGF.ConvertType(CGF.getContext().getSizeType()));
+ = llvm::Constant::getNullValue(IndexVar.getElementType());
CGF.Builder.CreateStore(Zero, IndexVar);
// Start the loop with a block that tests the condition.
@@ -557,6 +677,20 @@
return false;
}
+static void EmitLValueForAnyFieldInitialization(CodeGenFunction &CGF,
+ CXXCtorInitializer *MemberInit,
+ LValue &LHS) {
+ FieldDecl *Field = MemberInit->getAnyMember();
+ if (MemberInit->isIndirectMemberInitializer()) {
+ // If we are initializing an anonymous union field, drill down to the field.
+ IndirectFieldDecl *IndirectField = MemberInit->getIndirectMember();
+ for (const auto *I : IndirectField->chain())
+ LHS = CGF.EmitLValueForFieldInitialization(LHS, cast<FieldDecl>(I));
+ } else {
+ LHS = CGF.EmitLValueForFieldInitialization(LHS, Field);
+ }
+}
+
static void EmitMemberInitializer(CodeGenFunction &CGF,
const CXXRecordDecl *ClassDecl,
CXXCtorInitializer *MemberInit,
@@ -575,16 +709,7 @@
QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
LValue LHS = CGF.MakeNaturalAlignAddrLValue(ThisPtr, RecordTy);
- if (MemberInit->isIndirectMemberInitializer()) {
- // If we are initializing an anonymous union field, drill down to
- // the field.
- IndirectFieldDecl *IndirectField = MemberInit->getIndirectMember();
- for (const auto *I : IndirectField->chain())
- LHS = CGF.EmitLValueForFieldInitialization(LHS, cast<FieldDecl>(I));
- FieldType = MemberInit->getIndirectMember()->getAnonField()->getType();
- } else {
- LHS = CGF.EmitLValueForFieldInitialization(LHS, Field);
- }
+ EmitLValueForAnyFieldInitialization(CGF, MemberInit, LHS);
// Special case: if we are in a copy or move constructor, and we are copying
// an array of PODs or classes with trivial copy constructors, ignore the
@@ -609,6 +734,11 @@
// Copy the aggregate.
CGF.EmitAggregateCopy(LHS.getAddress(), Src.getAddress(), FieldType,
LHS.isVolatileQualified());
+ // Ensure that we destroy the objects if an exception is thrown later in
+ // the constructor.
+ QualType::DestructionKind dtorKind = FieldType.isDestructedType();
+ if (CGF.needsEHCleanup(dtorKind))
+ CGF.pushEHDestroy(dtorKind, LHS.getAddress(), FieldType);
return;
}
}
@@ -619,9 +749,8 @@
CGF.EmitInitializerForField(Field, LHS, MemberInit->getInit(), ArrayIndexes);
}
-void CodeGenFunction::EmitInitializerForField(
- FieldDecl *Field, LValue LHS, Expr *Init,
- ArrayRef<VarDecl *> ArrayIndexes) {
+void CodeGenFunction::EmitInitializerForField(FieldDecl *Field, LValue LHS,
+ Expr *Init, ArrayRef<VarDecl *> ArrayIndexes) {
QualType FieldType = Field->getType();
switch (getEvaluationKind(FieldType)) {
case TEK_Scalar:
@@ -636,26 +765,23 @@
EmitComplexExprIntoLValue(Init, LHS, /*isInit*/ true);
break;
case TEK_Aggregate: {
- llvm::Value *ArrayIndexVar = nullptr;
+ Address ArrayIndexVar = Address::invalid();
if (ArrayIndexes.size()) {
- llvm::Type *SizeTy = ConvertType(getContext().getSizeType());
-
// The LHS is a pointer to the first object we'll be constructing, as
// a flat array.
QualType BaseElementTy = getContext().getBaseElementType(FieldType);
llvm::Type *BasePtr = ConvertType(BaseElementTy);
BasePtr = llvm::PointerType::getUnqual(BasePtr);
- llvm::Value *BaseAddrPtr = Builder.CreateBitCast(LHS.getAddress(),
- BasePtr);
+ Address BaseAddrPtr = Builder.CreateBitCast(LHS.getAddress(), BasePtr);
LHS = MakeAddrLValue(BaseAddrPtr, BaseElementTy);
// Create an array index that will be used to walk over all of the
// objects we're constructing.
- ArrayIndexVar = CreateTempAlloca(SizeTy, "object.index");
- llvm::Value *Zero = llvm::Constant::getNullValue(SizeTy);
+ ArrayIndexVar = CreateMemTemp(getContext().getSizeType(), "object.index");
+ llvm::Value *Zero =
+ llvm::Constant::getNullValue(ArrayIndexVar.getElementType());
Builder.CreateStore(Zero, ArrayIndexVar);
-
// Emit the block variables for the array indices, if any.
for (unsigned I = 0, N = ArrayIndexes.size(); I != N; ++I)
EmitAutoVarDecl(*ArrayIndexes[I]);
@@ -804,7 +930,7 @@
return;
}
- const FunctionDecl *Definition = 0;
+ const FunctionDecl *Definition = nullptr;
Stmt *Body = Ctor->getBody(Definition);
assert(Definition == Ctor && "emitting wrong constructor body");
@@ -861,7 +987,7 @@
SanitizerSet OldSanOpts;
};
}
-
+
namespace {
class FieldMemcpyizer {
public:
@@ -909,44 +1035,30 @@
return;
}
- CharUnits Alignment;
-
uint64_t FirstByteOffset;
if (FirstField->isBitField()) {
const CGRecordLayout &RL =
CGF.getTypes().getCGRecordLayout(FirstField->getParent());
const CGBitFieldInfo &BFInfo = RL.getBitFieldInfo(FirstField);
- Alignment = CharUnits::fromQuantity(BFInfo.StorageAlignment);
// FirstFieldOffset is not appropriate for bitfields,
- // it won't tell us what the storage offset should be and thus might not
- // be properly aligned.
- //
- // Instead calculate the storage offset using the offset of the field in
- // the struct type.
- const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
- FirstByteOffset =
- DL.getStructLayout(RL.getLLVMType())
- ->getElementOffsetInBits(RL.getLLVMFieldNo(FirstField));
+ // we need to use the storage offset instead.
+ FirstByteOffset = CGF.getContext().toBits(BFInfo.StorageOffset);
} else {
- Alignment = CGF.getContext().getDeclAlign(FirstField);
FirstByteOffset = FirstFieldOffset;
}
- assert((CGF.getContext().toCharUnitsFromBits(FirstByteOffset) %
- Alignment) == 0 && "Bad field alignment.");
-
CharUnits MemcpySize = getMemcpySize(FirstByteOffset);
QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
- llvm::Value *ThisPtr = CGF.LoadCXXThis();
- LValue DestLV = CGF.MakeNaturalAlignAddrLValue(ThisPtr, RecordTy);
+ Address ThisPtr = CGF.LoadCXXThisAddress();
+ LValue DestLV = CGF.MakeAddrLValue(ThisPtr, RecordTy);
LValue Dest = CGF.EmitLValueForFieldInitialization(DestLV, FirstField);
llvm::Value *SrcPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(SrcRec));
LValue SrcLV = CGF.MakeNaturalAlignAddrLValue(SrcPtr, RecordTy);
LValue Src = CGF.EmitLValueForFieldInitialization(SrcLV, FirstField);
- emitMemcpyIR(Dest.isBitField() ? Dest.getBitFieldAddr() : Dest.getAddress(),
- Src.isBitField() ? Src.getBitFieldAddr() : Src.getAddress(),
- MemcpySize, Alignment);
+ emitMemcpyIR(Dest.isBitField() ? Dest.getBitFieldAddress() : Dest.getAddress(),
+ Src.isBitField() ? Src.getBitFieldAddress() : Src.getAddress(),
+ MemcpySize);
reset();
}
@@ -960,20 +1072,18 @@
private:
- void emitMemcpyIR(llvm::Value *DestPtr, llvm::Value *SrcPtr,
- CharUnits Size, CharUnits Alignment) {
- llvm::PointerType *DPT = cast<llvm::PointerType>(DestPtr->getType());
+ void emitMemcpyIR(Address DestPtr, Address SrcPtr, CharUnits Size) {
+ llvm::PointerType *DPT = DestPtr.getType();
llvm::Type *DBP =
llvm::Type::getInt8PtrTy(CGF.getLLVMContext(), DPT->getAddressSpace());
DestPtr = CGF.Builder.CreateBitCast(DestPtr, DBP);
- llvm::PointerType *SPT = cast<llvm::PointerType>(SrcPtr->getType());
+ llvm::PointerType *SPT = SrcPtr.getType();
llvm::Type *SBP =
llvm::Type::getInt8PtrTy(CGF.getLLVMContext(), SPT->getAddressSpace());
SrcPtr = CGF.Builder.CreateBitCast(SrcPtr, SBP);
- CGF.Builder.CreateMemCpy(DestPtr, SrcPtr, Size.getQuantity(),
- Alignment.getQuantity());
+ CGF.Builder.CreateMemCpy(DestPtr, SrcPtr, Size.getQuantity());
}
void addInitialField(FieldDecl *F) {
@@ -1081,6 +1191,7 @@
CopyingValueRepresentation CVR(CGF);
EmitMemberInitializer(CGF, ConstructorDecl->getParent(),
AggregatedInits[0], ConstructorDecl, Args);
+ AggregatedInits.clear();
}
reset();
return;
@@ -1092,15 +1203,19 @@
}
void pushEHDestructors() {
- llvm::Value *ThisPtr = CGF.LoadCXXThis();
+ Address ThisPtr = CGF.LoadCXXThisAddress();
QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
- LValue LHS = CGF.MakeNaturalAlignAddrLValue(ThisPtr, RecordTy);
+ LValue LHS = CGF.MakeAddrLValue(ThisPtr, RecordTy);
for (unsigned i = 0; i < AggregatedInits.size(); ++i) {
- QualType FieldType = AggregatedInits[i]->getMember()->getType();
+ CXXCtorInitializer *MemberInit = AggregatedInits[i];
+ QualType FieldType = MemberInit->getAnyMember()->getType();
QualType::DestructionKind dtorKind = FieldType.isDestructedType();
- if (CGF.needsEHCleanup(dtorKind))
- CGF.pushEHDestroy(dtorKind, LHS.getAddress(), FieldType);
+ if (!CGF.needsEHCleanup(dtorKind))
+ continue;
+ LValue FieldLHS = LHS;
+ EmitLValueForAnyFieldInitialization(CGF, MemberInit, FieldLHS);
+ CGF.pushEHDestroy(dtorKind, FieldLHS.getAddress(), FieldType);
}
}
@@ -1227,7 +1342,13 @@
emitAggregatedStmts();
}
};
+} // end anonymous namespace
+static bool isInitializerOfDynamicClass(const CXXCtorInitializer *BaseInit) {
+ const Type *BaseType = BaseInit->getBaseClass();
+ const auto *BaseClassDecl =
+ cast<CXXRecordDecl>(BaseType->getAs<RecordType>()->getDecl());
+ return BaseClassDecl->isDynamicClass();
}
/// EmitCtorPrologue - This routine generates necessary code to initialize
@@ -1253,8 +1374,13 @@
assert(BaseCtorContinueBB);
}
+ llvm::Value *const OldThis = CXXThisValue;
// Virtual base initializers first.
for (; B != E && (*B)->isBaseInitializer() && (*B)->isBaseVirtual(); B++) {
+ if (CGM.getCodeGenOpts().StrictVTablePointers &&
+ CGM.getCodeGenOpts().OptimizationLevel > 0 &&
+ isInitializerOfDynamicClass(*B))
+ CXXThisValue = Builder.CreateInvariantGroupBarrier(LoadCXXThis());
EmitBaseInitializer(*this, ClassDecl, *B, CtorType);
}
@@ -1267,13 +1393,20 @@
// Then, non-virtual base initializers.
for (; B != E && (*B)->isBaseInitializer(); B++) {
assert(!(*B)->isBaseVirtual());
+
+ if (CGM.getCodeGenOpts().StrictVTablePointers &&
+ CGM.getCodeGenOpts().OptimizationLevel > 0 &&
+ isInitializerOfDynamicClass(*B))
+ CXXThisValue = Builder.CreateInvariantGroupBarrier(LoadCXXThis());
EmitBaseInitializer(*this, ClassDecl, *B, CtorType);
}
+ CXXThisValue = OldThis;
+
InitializeVTablePointers(ClassDecl);
// And finally, initialize class members.
- FieldConstructionScope FCS(*this, CXXThisValue);
+ FieldConstructionScope FCS(*this, LoadCXXThisAddress());
ConstructorMemcpyizer CM(*this, CD, Args);
for (; B != E; B++) {
CXXCtorInitializer *Member = (*B);
@@ -1333,7 +1466,7 @@
static bool
FieldHasTrivialDestructorBody(ASTContext &Context,
- const FieldDecl *Field)
+ const FieldDecl *Field)
{
QualType FieldBaseElementType = Context.getBaseElementType(Field->getType());
@@ -1342,20 +1475,28 @@
return true;
CXXRecordDecl *FieldClassDecl = cast<CXXRecordDecl>(RT->getDecl());
+
+ // The destructor for an implicit anonymous union member is never invoked.
+ if (FieldClassDecl->isUnion() && FieldClassDecl->isAnonymousStructOrUnion())
+ return false;
+
return HasTrivialDestructorBody(Context, FieldClassDecl, FieldClassDecl);
}
/// CanSkipVTablePointerInitialization - Check whether we need to initialize
/// any vtable pointers before calling this destructor.
-static bool CanSkipVTablePointerInitialization(ASTContext &Context,
+static bool CanSkipVTablePointerInitialization(CodeGenFunction &CGF,
const CXXDestructorDecl *Dtor) {
+ const CXXRecordDecl *ClassDecl = Dtor->getParent();
+ if (!ClassDecl->isDynamicClass())
+ return true;
+
if (!Dtor->hasTrivialBody())
return false;
// Check the fields.
- const CXXRecordDecl *ClassDecl = Dtor->getParent();
for (const auto *Field : ClassDecl->fields())
- if (!FieldHasTrivialDestructorBody(Context, Field))
+ if (!FieldHasTrivialDestructorBody(CGF.getContext(), Field))
return false;
return true;
@@ -1377,7 +1518,7 @@
if (DtorType == Dtor_Deleting) {
EnterDtorCleanups(Dtor, Dtor_Deleting);
EmitCXXDestructorCall(Dtor, Dtor_Complete, /*ForVirtualBase=*/false,
- /*Delegating=*/false, LoadCXXThis());
+ /*Delegating=*/false, LoadCXXThisAddress());
PopCleanupBlock();
return;
}
@@ -1412,7 +1553,7 @@
if (!isTryBody) {
EmitCXXDestructorCall(Dtor, Dtor_Base, /*ForVirtualBase=*/false,
- /*Delegating=*/false, LoadCXXThis());
+ /*Delegating=*/false, LoadCXXThisAddress());
break;
}
// Fallthrough: act like we're in the base variant.
@@ -1424,8 +1565,14 @@
EnterDtorCleanups(Dtor, Dtor_Base);
// Initialize the vtable pointers before entering the body.
- if (!CanSkipVTablePointerInitialization(getContext(), Dtor))
- InitializeVTablePointers(Dtor->getParent());
+ if (!CanSkipVTablePointerInitialization(*this, Dtor)) {
+ // Insert the llvm.invariant.group.barrier intrinsic before initializing
+ // the vptrs to cancel any previous assumptions we might have made.
+ if (CGM.getCodeGenOpts().StrictVTablePointers &&
+ CGM.getCodeGenOpts().OptimizationLevel > 0)
+ CXXThisValue = Builder.CreateInvariantGroupBarrier(LoadCXXThis());
+ InitializeVTablePointers(Dtor->getParent());
+ }
if (isTryBody)
EmitStmt(cast<CXXTryStmt>(Body)->getTryBlock());
@@ -1439,6 +1586,7 @@
// the caller's body.
if (getLangOpts().AppleKext)
CurFn->addFnAttr(llvm::Attribute::AlwaysInline);
+
break;
}
@@ -1467,7 +1615,7 @@
namespace {
/// Call the operator delete associated with the current destructor.
- struct CallDtorDelete : EHScopeStack::Cleanup {
+ struct CallDtorDelete final : EHScopeStack::Cleanup {
CallDtorDelete() {}
void Emit(CodeGenFunction &CGF, Flags flags) override {
@@ -1478,11 +1626,11 @@
}
};
- struct CallDtorDeleteConditional : EHScopeStack::Cleanup {
+ struct CallDtorDeleteConditional final : EHScopeStack::Cleanup {
llvm::Value *ShouldDeleteCondition;
public:
CallDtorDeleteConditional(llvm::Value *ShouldDeleteCondition)
- : ShouldDeleteCondition(ShouldDeleteCondition) {
+ : ShouldDeleteCondition(ShouldDeleteCondition) {
assert(ShouldDeleteCondition != nullptr);
}
@@ -1504,7 +1652,7 @@
}
};
- class DestroyField : public EHScopeStack::Cleanup {
+ class DestroyField final : public EHScopeStack::Cleanup {
const FieldDecl *field;
CodeGenFunction::Destroyer *destroyer;
bool useEHCleanupForArray;
@@ -1512,12 +1660,12 @@
public:
DestroyField(const FieldDecl *field, CodeGenFunction::Destroyer *destroyer,
bool useEHCleanupForArray)
- : field(field), destroyer(destroyer),
- useEHCleanupForArray(useEHCleanupForArray) {}
+ : field(field), destroyer(destroyer),
+ useEHCleanupForArray(useEHCleanupForArray) {}
void Emit(CodeGenFunction &CGF, Flags flags) override {
// Find the address of the field.
- llvm::Value *thisValue = CGF.LoadCXXThis();
+ Address thisValue = CGF.LoadCXXThisAddress();
QualType RecordTy = CGF.getContext().getTagDeclType(field->getParent());
LValue ThisLV = CGF.MakeAddrLValue(thisValue, RecordTy);
LValue LV = CGF.EmitLValueForField(ThisLV, field);
@@ -1527,7 +1675,133 @@
flags.isForNormalCleanup() && useEHCleanupForArray);
}
};
-}
+
+ static void EmitSanitizerDtorCallback(CodeGenFunction &CGF, llvm::Value *Ptr,
+ CharUnits::QuantityType PoisonSize) {
+ // Pass in void pointer and size of region as arguments to runtime
+ // function
+ llvm::Value *Args[] = {CGF.Builder.CreateBitCast(Ptr, CGF.VoidPtrTy),
+ llvm::ConstantInt::get(CGF.SizeTy, PoisonSize)};
+
+ llvm::Type *ArgTypes[] = {CGF.VoidPtrTy, CGF.SizeTy};
+
+ llvm::FunctionType *FnType =
+ llvm::FunctionType::get(CGF.VoidTy, ArgTypes, false);
+ llvm::Value *Fn =
+ CGF.CGM.CreateRuntimeFunction(FnType, "__sanitizer_dtor_callback");
+ CGF.EmitNounwindRuntimeCall(Fn, Args);
+ }
+
+ class SanitizeDtorMembers final : public EHScopeStack::Cleanup {
+ const CXXDestructorDecl *Dtor;
+
+ public:
+ SanitizeDtorMembers(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {}
+
+ // Generate function call for handling object poisoning.
+ // Disables tail call elimination, to prevent the current stack frame
+ // from disappearing from the stack trace.
+ void Emit(CodeGenFunction &CGF, Flags flags) override {
+ const ASTRecordLayout &Layout =
+ CGF.getContext().getASTRecordLayout(Dtor->getParent());
+
+ // Nothing to poison.
+ if (Layout.getFieldCount() == 0)
+ return;
+
+ // Prevent the current stack frame from disappearing from the stack trace.
+ CGF.CurFn->addFnAttr("disable-tail-calls", "true");
+
+ // Construct pointer to region to begin poisoning, and calculate poison
+ // size, so that only members declared in this class are poisoned.
+ ASTContext &Context = CGF.getContext();
+ unsigned fieldIndex = 0;
+ int startIndex = -1;
+ // RecordDecl::field_iterator Field;
+ for (const FieldDecl *Field : Dtor->getParent()->fields()) {
+ // Poison field if it is trivial
+ if (FieldHasTrivialDestructorBody(Context, Field)) {
+ // Start sanitizing at this field
+ if (startIndex < 0)
+ startIndex = fieldIndex;
+
+ // Currently on the last field, and it must be poisoned with the
+ // current block.
+ if (fieldIndex == Layout.getFieldCount() - 1) {
+ PoisonMembers(CGF, startIndex, Layout.getFieldCount());
+ }
+ } else if (startIndex >= 0) {
+ // No longer within a block of memory to poison, so poison the block
+ PoisonMembers(CGF, startIndex, fieldIndex);
+ // Re-set the start index
+ startIndex = -1;
+ }
+ fieldIndex += 1;
+ }
+ }
+
+ private:
+ /// \param layoutStartOffset index of the ASTRecordLayout field to
+ /// start poisoning (inclusive)
+ /// \param layoutEndOffset index of the ASTRecordLayout field to
+ /// end poisoning (exclusive)
+ void PoisonMembers(CodeGenFunction &CGF, unsigned layoutStartOffset,
+ unsigned layoutEndOffset) {
+ ASTContext &Context = CGF.getContext();
+ const ASTRecordLayout &Layout =
+ Context.getASTRecordLayout(Dtor->getParent());
+
+ llvm::ConstantInt *OffsetSizePtr = llvm::ConstantInt::get(
+ CGF.SizeTy,
+ Context.toCharUnitsFromBits(Layout.getFieldOffset(layoutStartOffset))
+ .getQuantity());
+
+ llvm::Value *OffsetPtr = CGF.Builder.CreateGEP(
+ CGF.Builder.CreateBitCast(CGF.LoadCXXThis(), CGF.Int8PtrTy),
+ OffsetSizePtr);
+
+ CharUnits::QuantityType PoisonSize;
+ if (layoutEndOffset >= Layout.getFieldCount()) {
+ PoisonSize = Layout.getNonVirtualSize().getQuantity() -
+ Context.toCharUnitsFromBits(
+ Layout.getFieldOffset(layoutStartOffset))
+ .getQuantity();
+ } else {
+ PoisonSize = Context.toCharUnitsFromBits(
+ Layout.getFieldOffset(layoutEndOffset) -
+ Layout.getFieldOffset(layoutStartOffset))
+ .getQuantity();
+ }
+
+ if (PoisonSize == 0)
+ return;
+
+ EmitSanitizerDtorCallback(CGF, OffsetPtr, PoisonSize);
+ }
+ };
+
+ class SanitizeDtorVTable final : public EHScopeStack::Cleanup {
+ const CXXDestructorDecl *Dtor;
+
+ public:
+ SanitizeDtorVTable(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {}
+
+ // Generate function call for handling vtable pointer poisoning.
+ void Emit(CodeGenFunction &CGF, Flags flags) override {
+ assert(Dtor->getParent()->isDynamicClass());
+ (void)Dtor;
+ ASTContext &Context = CGF.getContext();
+ // Poison vtable and vtable ptr if they exist for this class.
+ llvm::Value *VTablePtr = CGF.LoadCXXThis();
+
+ CharUnits::QuantityType PoisonSize =
+ Context.toCharUnitsFromBits(CGF.PointerWidthInBits).getQuantity();
+ // Pass in void pointer and size of region as arguments to runtime
+ // function
+ EmitSanitizerDtorCallback(CGF, VTablePtr, PoisonSize);
+ }
+ };
+} // end anonymous namespace
/// \brief Emit all code that comes at the end of class's
/// destructor. This is to call destructors on members and base classes
@@ -1561,6 +1835,12 @@
// The complete-destructor phase just destructs all the virtual bases.
if (DtorType == Dtor_Complete) {
+ // Poison the vtable pointer such that access after the base
+ // and member destructors are invoked is invalid.
+ if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
+ SanOpts.has(SanitizerKind::Memory) && ClassDecl->getNumVBases() &&
+ ClassDecl->isPolymorphic())
+ EHStack.pushCleanup<SanitizeDtorVTable>(NormalAndEHCleanup, DD);
// We push them in the forward order so that they'll be popped in
// the reverse order.
@@ -1581,6 +1861,12 @@
}
assert(DtorType == Dtor_Base);
+ // Poison the vtable pointer if it has no virtual bases, but inherits
+ // virtual functions.
+ if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
+ SanOpts.has(SanitizerKind::Memory) && !ClassDecl->getNumVBases() &&
+ ClassDecl->isPolymorphic())
+ EHStack.pushCleanup<SanitizeDtorVTable>(NormalAndEHCleanup, DD);
// Destroy non-virtual bases.
for (const auto &Base : ClassDecl->bases()) {
@@ -1599,6 +1885,12 @@
/*BaseIsVirtual*/ false);
}
+ // Poison fields such that access after their destructors are
+ // invoked, and before the base class destructor runs, is invalid.
+ if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
+ SanOpts.has(SanitizerKind::Memory))
+ EHStack.pushCleanup<SanitizeDtorMembers>(NormalAndEHCleanup, DD);
+
// Destroy direct fields.
for (const auto *Field : ClassDecl->fields()) {
QualType type = Field->getType();
@@ -1626,7 +1918,7 @@
/// zero-initialized before it is constructed
void CodeGenFunction::EmitCXXAggrConstructorCall(
const CXXConstructorDecl *ctor, const ConstantArrayType *arrayType,
- llvm::Value *arrayBegin, const CXXConstructExpr *E, bool zeroInitialize) {
+ Address arrayBegin, const CXXConstructExpr *E, bool zeroInitialize) {
QualType elementType;
llvm::Value *numElements =
emitArrayLength(arrayType, elementType, arrayBegin);
@@ -1640,15 +1932,14 @@
/// \param ctor the constructor to call for each element
/// \param numElements the number of elements in the array;
/// may be zero
-/// \param arrayBegin a T*, where T is the type constructed by ctor
+/// \param arrayBase a T*, where T is the type constructed by ctor
/// \param zeroInitialize true if each element should be
/// zero-initialized before it is constructed
void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor,
llvm::Value *numElements,
- llvm::Value *arrayBegin,
+ Address arrayBase,
const CXXConstructExpr *E,
bool zeroInitialize) {
-
// It's legal for numElements to be zero. This can happen both
// dynamically, because x can be zero in 'new A[x]', and statically,
// because of GCC extensions that permit zero-length arrays. There
@@ -1672,6 +1963,7 @@
}
// Find the end of the array.
+ llvm::Value *arrayBegin = arrayBase.getPointer();
llvm::Value *arrayEnd = Builder.CreateInBoundsGEP(arrayBegin, numElements,
"arrayctor.end");
@@ -1685,11 +1977,21 @@
// Inside the loop body, emit the constructor call on the array element.
+ // The alignment of the base, adjusted by the size of a single element,
+ // provides a conservative estimate of the alignment of every element.
+ // (This assumes we never start tracking offsetted alignments.)
+ //
+ // Note that these are complete objects and so we don't need to
+ // use the non-virtual size or alignment.
QualType type = getContext().getTypeDeclType(ctor->getParent());
+ CharUnits eltAlignment =
+ arrayBase.getAlignment()
+ .alignmentOfArrayElement(getContext().getTypeSizeInChars(type));
+ Address curAddr = Address(cur, eltAlignment);
// Zero initialize the storage, if requested.
if (zeroInitialize)
- EmitNullInitialization(cur, type);
+ EmitNullInitialization(curAddr, type);
// C++ [class.temporary]p4:
// There are two contexts in which temporaries are destroyed at a different
@@ -1707,11 +2009,12 @@
if (getLangOpts().Exceptions &&
!ctor->getParent()->hasTrivialDestructor()) {
Destroyer *destroyer = destroyCXXObject;
- pushRegularPartialArrayCleanup(arrayBegin, cur, type, *destroyer);
+ pushRegularPartialArrayCleanup(arrayBegin, cur, type, eltAlignment,
+ *destroyer);
}
EmitCXXConstructorCall(ctor, Ctor_Complete, /*ForVirtualBase=*/false,
- /*Delegating=*/false, cur, E);
+ /*Delegating=*/false, curAddr, E);
}
// Go to the next element.
@@ -1732,7 +2035,7 @@
}
void CodeGenFunction::destroyCXXObject(CodeGenFunction &CGF,
- llvm::Value *addr,
+ Address addr,
QualType type) {
const RecordType *rtype = type->castAs<RecordType>();
const CXXRecordDecl *record = cast<CXXRecordDecl>(rtype->getDecl());
@@ -1745,14 +2048,16 @@
void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
CXXCtorType Type,
bool ForVirtualBase,
- bool Delegating, llvm::Value *This,
+ bool Delegating, Address This,
const CXXConstructExpr *E) {
+ const CXXRecordDecl *ClassDecl = D->getParent();
+
// C++11 [class.mfct.non-static]p2:
// If a non-static member function of a class X is called for an object that
// is not of type X, or of a type derived from X, the behavior is undefined.
// FIXME: Provide a source location here.
- EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, SourceLocation(), This,
- getContext().getRecordType(D->getParent()));
+ EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, SourceLocation(),
+ This.getPointer(), getContext().getRecordType(ClassDecl));
if (D->isTrivial() && D->isDefaultConstructor()) {
assert(E->getNumArgs() == 0 && "trivial default ctor with args");
@@ -1767,8 +2072,8 @@
const Expr *Arg = E->getArg(0);
QualType SrcTy = Arg->getType();
- llvm::Value *Src = EmitLValue(Arg).getAddress();
- QualType DestTy = getContext().getTypeDeclType(D->getParent());
+ Address Src = EmitLValue(Arg).getAddress();
+ QualType DestTy = getContext().getTypeDeclType(ClassDecl);
EmitAggregateCopyCtor(This, Src, DestTy, SrcTy);
return;
}
@@ -1776,11 +2081,11 @@
CallArgList Args;
// Push the this ptr.
- Args.add(RValue::get(This), D->getThisType(getContext()));
+ Args.add(RValue::get(This.getPointer()), D->getThisType(getContext()));
// Add the rest of the user-supplied arguments.
const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
- EmitCallArgs(Args, FPT, E->arg_begin(), E->arg_end(), E->getConstructor());
+ EmitCallArgs(Args, FPT, E->arguments(), E->getConstructor());
// Insert any ABI-specific implicit constructor arguments.
unsigned ExtraArgs = CGM.getCXXABI().addImplicitConstructorArgs(
@@ -1791,19 +2096,64 @@
const CGFunctionInfo &Info =
CGM.getTypes().arrangeCXXConstructorCall(Args, D, Type, ExtraArgs);
EmitCall(Info, Callee, ReturnValueSlot(), Args, D);
+
+ // Generate vtable assumptions if we're constructing a complete object
+ // with a vtable. We don't do this for base subobjects for two reasons:
+ // first, it's incorrect for classes with virtual bases, and second, we're
+ // about to overwrite the vptrs anyway.
+ // We also have to make sure if we can refer to vtable:
+ // - Otherwise we can refer to vtable if it's safe to speculatively emit.
+ // FIXME: If vtable is used by ctor/dtor, or if vtable is external and we are
+ // sure that definition of vtable is not hidden,
+ // then we are always safe to refer to it.
+ // FIXME: It looks like InstCombine is very inefficient on dealing with
+ // assumes. Make assumption loads require -fstrict-vtable-pointers temporarily.
+ if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
+ ClassDecl->isDynamicClass() && Type != Ctor_Base &&
+ CGM.getCXXABI().canSpeculativelyEmitVTable(ClassDecl) &&
+ CGM.getCodeGenOpts().StrictVTablePointers)
+ EmitVTableAssumptionLoads(ClassDecl, This);
+}
+
+void CodeGenFunction::EmitVTableAssumptionLoad(const VPtr &Vptr, Address This) {
+ llvm::Value *VTableGlobal =
+ CGM.getCXXABI().getVTableAddressPoint(Vptr.Base, Vptr.VTableClass);
+ if (!VTableGlobal)
+ return;
+
+ // We can just use the base offset in the complete class.
+ CharUnits NonVirtualOffset = Vptr.Base.getBaseOffset();
+
+ if (!NonVirtualOffset.isZero())
+ This =
+ ApplyNonVirtualAndVirtualOffset(*this, This, NonVirtualOffset, nullptr,
+ Vptr.VTableClass, Vptr.NearestVBase);
+
+ llvm::Value *VPtrValue =
+ GetVTablePtr(This, VTableGlobal->getType(), Vptr.VTableClass);
+ llvm::Value *Cmp =
+ Builder.CreateICmpEQ(VPtrValue, VTableGlobal, "cmp.vtables");
+ Builder.CreateAssumption(Cmp);
+}
+
+void CodeGenFunction::EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl,
+ Address This) {
+ if (CGM.getCXXABI().doStructorsInitializeVPtrs(ClassDecl))
+ for (const VPtr &Vptr : getVTablePointers(ClassDecl))
+ EmitVTableAssumptionLoad(Vptr, This);
}
void
CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D,
- llvm::Value *This, llvm::Value *Src,
- const CXXConstructExpr *E) {
+ Address This, Address Src,
+ const CXXConstructExpr *E) {
if (isMemcpyEquivalentSpecialMember(D)) {
assert(E->getNumArgs() == 1 && "unexpected argcount for trivial ctor");
assert(D->isCopyOrMoveConstructor() &&
"trivial 1-arg ctor not a copy/move ctor");
EmitAggregateCopyCtor(This, Src,
getContext().getTypeDeclType(D->getParent()),
- E->arg_begin()->getType());
+ (*E->arg_begin())->getType());
return;
}
llvm::Value *Callee = CGM.getAddrOfCXXStructor(D, StructorType::Complete);
@@ -1815,16 +2165,16 @@
CallArgList Args;
// Push the this ptr.
- Args.add(RValue::get(This), D->getThisType(getContext()));
+ Args.add(RValue::get(This.getPointer()), D->getThisType(getContext()));
// Push the src ptr.
QualType QT = *(FPT->param_type_begin());
llvm::Type *t = CGM.getTypes().ConvertType(QT);
Src = Builder.CreateBitCast(Src, t);
- Args.add(RValue::get(Src), QT);
+ Args.add(RValue::get(Src.getPointer()), QT);
// Skip over first argument (Src).
- EmitCallArgs(Args, FPT, E->arg_begin() + 1, E->arg_end(), E->getConstructor(),
+ EmitCallArgs(Args, FPT, drop_begin(E->arguments(), 1), E->getConstructor(),
/*ParamsToSkip*/ 1);
EmitCall(CGM.getTypes().arrangeCXXMethodCall(Args, FPT, RequiredArgs::All),
@@ -1874,12 +2224,12 @@
}
namespace {
- struct CallDelegatingCtorDtor : EHScopeStack::Cleanup {
+ struct CallDelegatingCtorDtor final : EHScopeStack::Cleanup {
const CXXDestructorDecl *Dtor;
- llvm::Value *Addr;
+ Address Addr;
CXXDtorType Type;
- CallDelegatingCtorDtor(const CXXDestructorDecl *D, llvm::Value *Addr,
+ CallDelegatingCtorDtor(const CXXDestructorDecl *D, Address Addr,
CXXDtorType Type)
: Dtor(D), Addr(Addr), Type(Type) {}
@@ -1888,19 +2238,17 @@
/*Delegating=*/true, Addr);
}
};
-}
+} // end anonymous namespace
void
CodeGenFunction::EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor,
const FunctionArgList &Args) {
assert(Ctor->isDelegatingConstructor());
- llvm::Value *ThisPtr = LoadCXXThis();
+ Address ThisPtr = LoadCXXThisAddress();
- QualType Ty = getContext().getTagDeclType(Ctor->getParent());
- CharUnits Alignment = getContext().getTypeAlignInChars(Ty);
AggValueSlot AggSlot =
- AggValueSlot::forAddr(ThisPtr, Alignment, Qualifiers(),
+ AggValueSlot::forAddr(ThisPtr, Qualifiers(),
AggValueSlot::IsDestructed,
AggValueSlot::DoesNotNeedGCBarriers,
AggValueSlot::IsNotAliased);
@@ -1922,17 +2270,17 @@
CXXDtorType Type,
bool ForVirtualBase,
bool Delegating,
- llvm::Value *This) {
+ Address This) {
CGM.getCXXABI().EmitDestructorCall(*this, DD, Type, ForVirtualBase,
Delegating, This);
}
namespace {
- struct CallLocalDtor : EHScopeStack::Cleanup {
+ struct CallLocalDtor final : EHScopeStack::Cleanup {
const CXXDestructorDecl *Dtor;
- llvm::Value *Addr;
+ Address Addr;
- CallLocalDtor(const CXXDestructorDecl *D, llvm::Value *Addr)
+ CallLocalDtor(const CXXDestructorDecl *D, Address Addr)
: Dtor(D), Addr(Addr) {}
void Emit(CodeGenFunction &CGF, Flags flags) override {
@@ -1944,11 +2292,11 @@
}
void CodeGenFunction::PushDestructorCleanup(const CXXDestructorDecl *D,
- llvm::Value *Addr) {
+ Address Addr) {
EHStack.pushCleanup<CallLocalDtor>(NormalAndEHCleanup, D, Addr);
}
-void CodeGenFunction::PushDestructorCleanup(QualType T, llvm::Value *Addr) {
+void CodeGenFunction::PushDestructorCleanup(QualType T, Address Addr) {
CXXRecordDecl *ClassDecl = T->getAsCXXRecordDecl();
if (!ClassDecl) return;
if (ClassDecl->hasTrivialDestructor()) return;
@@ -1958,24 +2306,12 @@
PushDestructorCleanup(D, Addr);
}
-void
-CodeGenFunction::InitializeVTablePointer(BaseSubobject Base,
- const CXXRecordDecl *NearestVBase,
- CharUnits OffsetFromNearestVBase,
- const CXXRecordDecl *VTableClass) {
- const CXXRecordDecl *RD = Base.getBase();
-
- // Don't initialize the vtable pointer if the class is marked with the
- // 'novtable' attribute.
- if ((RD == VTableClass || RD == NearestVBase) &&
- VTableClass->hasAttr<MSNoVTableAttr>())
- return;
-
+void CodeGenFunction::InitializeVTablePointer(const VPtr &Vptr) {
// Compute the address point.
- bool NeedsVirtualOffset;
llvm::Value *VTableAddressPoint =
CGM.getCXXABI().getVTableAddressPointInStructor(
- *this, VTableClass, Base, NearestVBase, NeedsVirtualOffset);
+ *this, Vptr.VTableClass, Vptr.Base, Vptr.NearestVBase);
+
if (!VTableAddressPoint)
return;
@@ -1983,26 +2319,25 @@
llvm::Value *VirtualOffset = nullptr;
CharUnits NonVirtualOffset = CharUnits::Zero();
- if (NeedsVirtualOffset) {
+ if (CGM.getCXXABI().isVirtualOffsetNeededForVTableField(*this, Vptr)) {
// We need to use the virtual base offset offset because the virtual base
// might have a different offset in the most derived class.
- VirtualOffset = CGM.getCXXABI().GetVirtualBaseClassOffset(*this,
- LoadCXXThis(),
- VTableClass,
- NearestVBase);
- NonVirtualOffset = OffsetFromNearestVBase;
+
+ VirtualOffset = CGM.getCXXABI().GetVirtualBaseClassOffset(
+ *this, LoadCXXThisAddress(), Vptr.VTableClass, Vptr.NearestVBase);
+ NonVirtualOffset = Vptr.OffsetFromNearestVBase;
} else {
// We can just use the base offset in the complete class.
- NonVirtualOffset = Base.getBaseOffset();
+ NonVirtualOffset = Vptr.Base.getBaseOffset();
}
// Apply the offsets.
- llvm::Value *VTableField = LoadCXXThis();
+ Address VTableField = LoadCXXThisAddress();
if (!NonVirtualOffset.isZero() || VirtualOffset)
- VTableField = ApplyNonVirtualAndVirtualOffset(*this, VTableField,
- NonVirtualOffset,
- VirtualOffset);
+ VTableField = ApplyNonVirtualAndVirtualOffset(
+ *this, VTableField, NonVirtualOffset, VirtualOffset, Vptr.VTableClass,
+ Vptr.NearestVBase);
// Finally, store the address point. Use the same LLVM types as the field to
// support optimization.
@@ -2012,23 +2347,39 @@
->getPointerTo();
VTableField = Builder.CreateBitCast(VTableField, VTablePtrTy->getPointerTo());
VTableAddressPoint = Builder.CreateBitCast(VTableAddressPoint, VTablePtrTy);
+
llvm::StoreInst *Store = Builder.CreateStore(VTableAddressPoint, VTableField);
- CGM.DecorateInstruction(Store, CGM.getTBAAInfoForVTablePtr());
+ CGM.DecorateInstructionWithTBAA(Store, CGM.getTBAAInfoForVTablePtr());
+ if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
+ CGM.getCodeGenOpts().StrictVTablePointers)
+ CGM.DecorateInstructionWithInvariantGroup(Store, Vptr.VTableClass);
}
-void
-CodeGenFunction::InitializeVTablePointers(BaseSubobject Base,
- const CXXRecordDecl *NearestVBase,
- CharUnits OffsetFromNearestVBase,
- bool BaseIsNonVirtualPrimaryBase,
- const CXXRecordDecl *VTableClass,
- VisitedVirtualBasesSetTy& VBases) {
+CodeGenFunction::VPtrsVector
+CodeGenFunction::getVTablePointers(const CXXRecordDecl *VTableClass) {
+ CodeGenFunction::VPtrsVector VPtrsResult;
+ VisitedVirtualBasesSetTy VBases;
+ getVTablePointers(BaseSubobject(VTableClass, CharUnits::Zero()),
+ /*NearestVBase=*/nullptr,
+ /*OffsetFromNearestVBase=*/CharUnits::Zero(),
+ /*BaseIsNonVirtualPrimaryBase=*/false, VTableClass, VBases,
+ VPtrsResult);
+ return VPtrsResult;
+}
+
+void CodeGenFunction::getVTablePointers(BaseSubobject Base,
+ const CXXRecordDecl *NearestVBase,
+ CharUnits OffsetFromNearestVBase,
+ bool BaseIsNonVirtualPrimaryBase,
+ const CXXRecordDecl *VTableClass,
+ VisitedVirtualBasesSetTy &VBases,
+ VPtrsVector &Vptrs) {
// If this base is a non-virtual primary base the address point has already
// been set.
if (!BaseIsNonVirtualPrimaryBase) {
// Initialize the vtable pointer for this base.
- InitializeVTablePointer(Base, NearestVBase, OffsetFromNearestVBase,
- VTableClass);
+ VPtr Vptr = {Base, NearestVBase, OffsetFromNearestVBase, VTableClass};
+ Vptrs.push_back(Vptr);
}
const CXXRecordDecl *RD = Base.getBase();
@@ -2066,11 +2417,10 @@
BaseDeclIsNonVirtualPrimaryBase = Layout.getPrimaryBase() == BaseDecl;
}
- InitializeVTablePointers(BaseSubobject(BaseDecl, BaseOffset),
- I.isVirtual() ? BaseDecl : NearestVBase,
- BaseOffsetFromNearestVBase,
- BaseDeclIsNonVirtualPrimaryBase,
- VTableClass, VBases);
+ getVTablePointers(
+ BaseSubobject(BaseDecl, BaseOffset),
+ I.isVirtual() ? BaseDecl : NearestVBase, BaseOffsetFromNearestVBase,
+ BaseDeclIsNonVirtualPrimaryBase, VTableClass, VBases, Vptrs);
}
}
@@ -2080,21 +2430,25 @@
return;
// Initialize the vtable pointers for this class and all of its bases.
- VisitedVirtualBasesSetTy VBases;
- InitializeVTablePointers(BaseSubobject(RD, CharUnits::Zero()),
- /*NearestVBase=*/nullptr,
- /*OffsetFromNearestVBase=*/CharUnits::Zero(),
- /*BaseIsNonVirtualPrimaryBase=*/false, RD, VBases);
+ if (CGM.getCXXABI().doStructorsInitializeVPtrs(RD))
+ for (const VPtr &Vptr : getVTablePointers(RD))
+ InitializeVTablePointer(Vptr);
if (RD->getNumVBases())
CGM.getCXXABI().initializeHiddenVirtualInheritanceMembers(*this, RD);
}
-llvm::Value *CodeGenFunction::GetVTablePtr(llvm::Value *This,
- llvm::Type *Ty) {
- llvm::Value *VTablePtrSrc = Builder.CreateBitCast(This, Ty->getPointerTo());
+llvm::Value *CodeGenFunction::GetVTablePtr(Address This,
+ llvm::Type *VTableTy,
+ const CXXRecordDecl *RD) {
+ Address VTablePtrSrc = Builder.CreateElementBitCast(This, VTableTy);
llvm::Instruction *VTable = Builder.CreateLoad(VTablePtrSrc, "vtable");
- CGM.DecorateInstruction(VTable, CGM.getTBAAInfoForVTablePtr());
+ CGM.DecorateInstructionWithTBAA(VTable, CGM.getTBAAInfoForVTablePtr());
+
+ if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
+ CGM.getCodeGenOpts().StrictVTablePointers)
+ CGM.DecorateInstructionWithInvariantGroup(VTable, RD);
+
return VTable;
}
@@ -2134,17 +2488,21 @@
}
void CodeGenFunction::EmitVTablePtrCheckForCall(const CXXMethodDecl *MD,
- llvm::Value *VTable) {
+ llvm::Value *VTable,
+ CFITypeCheckKind TCK,
+ SourceLocation Loc) {
const CXXRecordDecl *ClassDecl = MD->getParent();
if (!SanOpts.has(SanitizerKind::CFICastStrict))
ClassDecl = LeastDerivedClassWithSameLayout(ClassDecl);
- EmitVTablePtrCheck(ClassDecl, VTable);
+ EmitVTablePtrCheck(ClassDecl, VTable, TCK, Loc);
}
void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T,
llvm::Value *Derived,
- bool MayBeNull) {
+ bool MayBeNull,
+ CFITypeCheckKind TCK,
+ SourceLocation Loc) {
if (!getLangOpts().CPlusPlus)
return;
@@ -2157,19 +2515,10 @@
if (!ClassDecl->isCompleteDefinition() || !ClassDecl->isDynamicClass())
return;
- SmallString<64> MangledName;
- llvm::raw_svector_ostream Out(MangledName);
- CGM.getCXXABI().getMangleContext().mangleCXXRTTI(T.getUnqualifiedType(),
- Out);
-
- // Blacklist based on the mangled type.
- if (CGM.getContext().getSanitizerBlacklist().isBlacklistedType(Out.str()))
- return;
-
if (!SanOpts.has(SanitizerKind::CFICastStrict))
ClassDecl = LeastDerivedClassWithSameLayout(ClassDecl);
- llvm::BasicBlock *ContBlock = 0;
+ llvm::BasicBlock *ContBlock = nullptr;
if (MayBeNull) {
llvm::Value *DerivedNotNull =
@@ -2183,8 +2532,10 @@
EmitBlock(CheckBlock);
}
- llvm::Value *VTable = GetVTablePtr(Derived, Int8PtrTy);
- EmitVTablePtrCheck(ClassDecl, VTable);
+ llvm::Value *VTable =
+ GetVTablePtr(Address(Derived, getPointerAlign()), Int8PtrTy, ClassDecl);
+
+ EmitVTablePtrCheck(ClassDecl, VTable, TCK, Loc);
if (MayBeNull) {
Builder.CreateBr(ContBlock);
@@ -2193,32 +2544,53 @@
}
void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD,
- llvm::Value *VTable) {
- // FIXME: Add blacklisting scheme.
- if (RD->isInStdNamespace())
+ llvm::Value *VTable,
+ CFITypeCheckKind TCK,
+ SourceLocation Loc) {
+ if (CGM.IsCFIBlacklistedRecord(RD))
return;
- std::string OutName;
- llvm::raw_string_ostream Out(OutName);
- CGM.getCXXABI().getMangleContext().mangleCXXVTableBitSet(RD, Out);
+ SanitizerScope SanScope(this);
- llvm::Value *BitSetName = llvm::MetadataAsValue::get(
- getLLVMContext(), llvm::MDString::get(getLLVMContext(), Out.str()));
+ llvm::Metadata *MD =
+ CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
+ llvm::Value *BitSetName = llvm::MetadataAsValue::get(getLLVMContext(), MD);
- llvm::Value *BitSetTest = Builder.CreateCall(
- CGM.getIntrinsic(llvm::Intrinsic::bitset_test),
- {Builder.CreateBitCast(VTable, CGM.Int8PtrTy), BitSetName});
+ llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy);
+ llvm::Value *BitSetTest =
+ Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::bitset_test),
+ {CastedVTable, BitSetName});
- llvm::BasicBlock *ContBlock = createBasicBlock("vtable.check.cont");
- llvm::BasicBlock *TrapBlock = createBasicBlock("vtable.check.trap");
+ if (CGM.getCodeGenOpts().SanitizeCfiCrossDso) {
+ if (auto TypeId = CGM.CreateCfiIdForTypeMetadata(MD)) {
+ EmitCfiSlowPathCheck(BitSetTest, TypeId, CastedVTable);
+ return;
+ }
+ }
- Builder.CreateCondBr(BitSetTest, ContBlock, TrapBlock);
+ SanitizerMask M;
+ switch (TCK) {
+ case CFITCK_VCall:
+ M = SanitizerKind::CFIVCall;
+ break;
+ case CFITCK_NVCall:
+ M = SanitizerKind::CFINVCall;
+ break;
+ case CFITCK_DerivedCast:
+ M = SanitizerKind::CFIDerivedCast;
+ break;
+ case CFITCK_UnrelatedCast:
+ M = SanitizerKind::CFIUnrelatedCast;
+ break;
+ }
- EmitBlock(TrapBlock);
- Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::trap), {});
- Builder.CreateUnreachable();
-
- EmitBlock(ContBlock);
+ llvm::Constant *StaticData[] = {
+ EmitCheckSourceLocation(Loc),
+ EmitCheckTypeDescriptor(QualType(RD->getTypeForDecl(), 0)),
+ llvm::ConstantInt::get(Int8Ty, TCK),
+ };
+ EmitCheck(std::make_pair(BitSetTest, M), "cfi_bad_type", StaticData,
+ CastedVTable);
}
// FIXME: Ideally Expr::IgnoreParenNoopCasts should do this, but it doesn't do
@@ -2355,8 +2727,8 @@
CallArgList CallArgs;
QualType ThisType = getContext().getPointerType(getContext().getRecordType(Lambda));
- llvm::Value *ThisPtr = GetAddrOfBlockDecl(variable, false);
- CallArgs.add(RValue::get(ThisPtr), ThisType);
+ Address ThisPtr = GetAddrOfBlockDecl(variable, false);
+ CallArgs.add(RValue::get(ThisPtr.getPointer()), ThisType);
// Add the rest of the parameters.
for (auto param : BD->params())
diff --git a/lib/CodeGen/CGCleanup.cpp b/lib/CodeGen/CGCleanup.cpp
index d97e405..245d6a7 100644
--- a/lib/CodeGen/CGCleanup.cpp
+++ b/lib/CodeGen/CGCleanup.cpp
@@ -19,6 +19,7 @@
#include "CGCleanup.h"
#include "CodeGenFunction.h"
+#include "llvm/Support/SaveAndRestore.h"
using namespace clang;
using namespace CodeGen;
@@ -27,7 +28,7 @@
if (rv.isScalar())
return DominatingLLVMValue::needsSaving(rv.getScalarVal());
if (rv.isAggregate())
- return DominatingLLVMValue::needsSaving(rv.getAggregateAddr());
+ return DominatingLLVMValue::needsSaving(rv.getAggregatePointer());
return true;
}
@@ -41,9 +42,10 @@
return saved_type(V, ScalarLiteral);
// Everything else needs an alloca.
- llvm::Value *addr = CGF.CreateTempAlloca(V->getType(), "saved-rvalue");
+ Address addr =
+ CGF.CreateDefaultAlignTempAlloca(V->getType(), "saved-rvalue");
CGF.Builder.CreateStore(V, addr);
- return saved_type(addr, ScalarAddress);
+ return saved_type(addr.getPointer(), ScalarAddress);
}
if (rv.isComplex()) {
@@ -51,42 +53,56 @@
llvm::Type *ComplexTy =
llvm::StructType::get(V.first->getType(), V.second->getType(),
(void*) nullptr);
- llvm::Value *addr = CGF.CreateTempAlloca(ComplexTy, "saved-complex");
+ Address addr = CGF.CreateDefaultAlignTempAlloca(ComplexTy, "saved-complex");
CGF.Builder.CreateStore(V.first,
- CGF.Builder.CreateStructGEP(ComplexTy, addr, 0));
+ CGF.Builder.CreateStructGEP(addr, 0, CharUnits()));
+ CharUnits offset = CharUnits::fromQuantity(
+ CGF.CGM.getDataLayout().getTypeAllocSize(V.first->getType()));
CGF.Builder.CreateStore(V.second,
- CGF.Builder.CreateStructGEP(ComplexTy, addr, 1));
- return saved_type(addr, ComplexAddress);
+ CGF.Builder.CreateStructGEP(addr, 1, offset));
+ return saved_type(addr.getPointer(), ComplexAddress);
}
assert(rv.isAggregate());
- llvm::Value *V = rv.getAggregateAddr(); // TODO: volatile?
- if (!DominatingLLVMValue::needsSaving(V))
- return saved_type(V, AggregateLiteral);
+ Address V = rv.getAggregateAddress(); // TODO: volatile?
+ if (!DominatingLLVMValue::needsSaving(V.getPointer()))
+ return saved_type(V.getPointer(), AggregateLiteral,
+ V.getAlignment().getQuantity());
- llvm::Value *addr = CGF.CreateTempAlloca(V->getType(), "saved-rvalue");
- CGF.Builder.CreateStore(V, addr);
- return saved_type(addr, AggregateAddress);
+ Address addr =
+ CGF.CreateTempAlloca(V.getType(), CGF.getPointerAlign(), "saved-rvalue");
+ CGF.Builder.CreateStore(V.getPointer(), addr);
+ return saved_type(addr.getPointer(), AggregateAddress,
+ V.getAlignment().getQuantity());
}
/// Given a saved r-value produced by SaveRValue, perform the code
/// necessary to restore it to usability at the current insertion
/// point.
RValue DominatingValue<RValue>::saved_type::restore(CodeGenFunction &CGF) {
+ auto getSavingAddress = [&](llvm::Value *value) {
+ auto alignment = cast<llvm::AllocaInst>(value)->getAlignment();
+ return Address(value, CharUnits::fromQuantity(alignment));
+ };
switch (K) {
case ScalarLiteral:
return RValue::get(Value);
case ScalarAddress:
- return RValue::get(CGF.Builder.CreateLoad(Value));
+ return RValue::get(CGF.Builder.CreateLoad(getSavingAddress(Value)));
case AggregateLiteral:
- return RValue::getAggregate(Value);
- case AggregateAddress:
- return RValue::getAggregate(CGF.Builder.CreateLoad(Value));
+ return RValue::getAggregate(Address(Value, CharUnits::fromQuantity(Align)));
+ case AggregateAddress: {
+ auto addr = CGF.Builder.CreateLoad(getSavingAddress(Value));
+ return RValue::getAggregate(Address(addr, CharUnits::fromQuantity(Align)));
+ }
case ComplexAddress: {
- llvm::Value *real =
- CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(nullptr, Value, 0));
- llvm::Value *imag =
- CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(nullptr, Value, 1));
+ Address address = getSavingAddress(Value);
+ llvm::Value *real = CGF.Builder.CreateLoad(
+ CGF.Builder.CreateStructGEP(address, 0, CharUnits()));
+ CharUnits offset = CharUnits::fromQuantity(
+ CGF.CGM.getDataLayout().getTypeAllocSize(real->getType()));
+ llvm::Value *imag = CGF.Builder.CreateLoad(
+ CGF.Builder.CreateStructGEP(address, 1, offset));
return RValue::getComplex(real, imag);
}
}
@@ -96,6 +112,7 @@
/// Push an entry of the given size onto this protected-scope stack.
char *EHScopeStack::allocate(size_t Size) {
+ Size = llvm::RoundUpToAlignment(Size, ScopeStackAlignment);
if (!StartOfBuffer) {
unsigned Capacity = 1024;
while (Capacity < Size) Capacity *= 2;
@@ -125,6 +142,10 @@
return StartOfData;
}
+void EHScopeStack::deallocate(size_t Size) {
+ StartOfData += llvm::RoundUpToAlignment(Size, ScopeStackAlignment);
+}
+
bool EHScopeStack::containsOnlyLifetimeMarkers(
EHScopeStack::stable_iterator Old) const {
for (EHScopeStack::iterator it = begin(); stabilize(it) != Old; it++) {
@@ -147,26 +168,8 @@
return stable_end();
}
-EHScopeStack::stable_iterator EHScopeStack::getInnermostActiveEHScope() const {
- for (stable_iterator si = getInnermostEHScope(), se = stable_end();
- si != se; ) {
- // Skip over inactive cleanups.
- EHCleanupScope *cleanup = dyn_cast<EHCleanupScope>(&*find(si));
- if (cleanup && !cleanup->isActive()) {
- si = cleanup->getEnclosingEHScope();
- continue;
- }
-
- // All other scopes are always active.
- return si;
- }
-
- return stable_end();
-}
-
void *EHScopeStack::pushCleanup(CleanupKind Kind, size_t Size) {
- assert(((Size % sizeof(void*)) == 0) && "cleanup type is misaligned");
char *Buffer = allocate(EHCleanupScope::getSizeForCleanupSize(Size));
bool IsNormalCleanup = Kind & NormalCleanup;
bool IsEHCleanup = Kind & EHCleanup;
@@ -194,7 +197,7 @@
EHCleanupScope &Cleanup = cast<EHCleanupScope>(*begin());
InnermostNormalCleanup = Cleanup.getEnclosingNormalCleanup();
InnermostEHScope = Cleanup.getEnclosingEHScope();
- StartOfData += Cleanup.getAllocatedSize();
+ deallocate(Cleanup.getAllocatedSize());
// Destroy the cleanup.
Cleanup.Destroy();
@@ -224,7 +227,7 @@
assert(!empty() && "popping exception stack when not empty");
EHFilterScope &filter = cast<EHFilterScope>(*begin());
- StartOfData += EHFilterScope::getSizeForNumFilters(filter.getNumFilters());
+ deallocate(EHFilterScope::getSizeForNumFilters(filter.getNumFilters()));
InnermostEHScope = filter.getEnclosingEHScope();
}
@@ -264,8 +267,8 @@
void CodeGenFunction::initFullExprCleanup() {
// Create a variable to decide whether the cleanup needs to be run.
- llvm::AllocaInst *active
- = CreateTempAlloca(Builder.getInt1Ty(), "cleanup.cond");
+ Address active = CreateTempAlloca(Builder.getInt1Ty(), CharUnits::One(),
+ "cleanup.cond");
// Initialize it to false at a site that's guaranteed to be run
// before each evaluation.
@@ -276,7 +279,7 @@
// Set that as the active flag in the cleanup.
EHCleanupScope &cleanup = cast<EHCleanupScope>(*EHStack.begin());
- assert(!cleanup.getActiveFlag() && "cleanup already has active flag?");
+ assert(!cleanup.hasActiveFlag() && "cleanup already has active flag?");
cleanup.setActiveFlag(active);
if (cleanup.isNormalCleanup()) cleanup.setTestFlagInNormalCleanup();
@@ -285,6 +288,19 @@
void EHScopeStack::Cleanup::anchor() {}
+static void createStoreInstBefore(llvm::Value *value, Address addr,
+ llvm::Instruction *beforeInst) {
+ auto store = new llvm::StoreInst(value, addr.getPointer(), beforeInst);
+ store->setAlignment(addr.getAlignment().getQuantity());
+}
+
+static llvm::LoadInst *createLoadInstBefore(Address addr, const Twine &name,
+ llvm::Instruction *beforeInst) {
+ auto load = new llvm::LoadInst(addr.getPointer(), name, beforeInst);
+ load->setAlignment(addr.getAlignment().getQuantity());
+ return load;
+}
+
/// All the branch fixups on the EH stack have propagated out past the
/// outermost normal cleanup; resolve them all by adding cases to the
/// given switch instruction.
@@ -307,9 +323,9 @@
// i.e. where there's an unresolved fixup inside a single cleanup
// entry which we're currently popping.
if (Fixup.OptimisticBranchBlock == nullptr) {
- new llvm::StoreInst(CGF.Builder.getInt32(Fixup.DestinationIndex),
- CGF.getNormalCleanupDestSlot(),
- Fixup.InitialBranch);
+ createStoreInstBefore(CGF.Builder.getInt32(Fixup.DestinationIndex),
+ CGF.getNormalCleanupDestSlot(),
+ Fixup.InitialBranch);
Fixup.InitialBranch->setSuccessor(0, CleanupEntry);
}
@@ -335,8 +351,8 @@
if (llvm::BranchInst *Br = dyn_cast<llvm::BranchInst>(Term)) {
assert(Br->isUnconditional());
- llvm::LoadInst *Load =
- new llvm::LoadInst(CGF.getNormalCleanupDestSlot(), "cleanup.dest", Term);
+ auto Load = createLoadInstBefore(CGF.getNormalCleanupDestSlot(),
+ "cleanup.dest", Term);
llvm::SwitchInst *Switch =
llvm::SwitchInst::Create(Load, Br->getSuccessor(0), 4, Block);
Br->eraseFromParent();
@@ -481,20 +497,11 @@
static void EmitCleanup(CodeGenFunction &CGF,
EHScopeStack::Cleanup *Fn,
EHScopeStack::Cleanup::Flags flags,
- llvm::Value *ActiveFlag) {
- // Itanium EH cleanups occur within a terminate scope. Microsoft SEH doesn't
- // have this behavior, and the Microsoft C++ runtime will call terminate for
- // us if the cleanup throws.
- bool PushedTerminate = false;
- if (flags.isForEHCleanup() && !CGF.getTarget().getCXXABI().isMicrosoft()) {
- CGF.EHStack.pushTerminate();
- PushedTerminate = true;
- }
-
+ Address ActiveFlag) {
// If there's an active flag, load it and skip the cleanup if it's
// false.
llvm::BasicBlock *ContBB = nullptr;
- if (ActiveFlag) {
+ if (ActiveFlag.isValid()) {
ContBB = CGF.createBasicBlock("cleanup.done");
llvm::BasicBlock *CleanupBB = CGF.createBasicBlock("cleanup.action");
llvm::Value *IsActive
@@ -508,12 +515,8 @@
assert(CGF.HaveInsertPoint() && "cleanup ended with no insertion point?");
// Emit the continuation block if there was an active flag.
- if (ActiveFlag)
+ if (ActiveFlag.isValid())
CGF.EmitBlock(ContBB);
-
- // Leave the terminate scope.
- if (PushedTerminate)
- CGF.EHStack.popTerminate();
}
static void ForwardPrebranchedFallthrough(llvm::BasicBlock *Exit,
@@ -588,10 +591,12 @@
// Remember activation information.
bool IsActive = Scope.isActive();
- llvm::Value *NormalActiveFlag =
- Scope.shouldTestFlagInNormalCleanup() ? Scope.getActiveFlag() : nullptr;
- llvm::Value *EHActiveFlag =
- Scope.shouldTestFlagInEHCleanup() ? Scope.getActiveFlag() : nullptr;
+ Address NormalActiveFlag =
+ Scope.shouldTestFlagInNormalCleanup() ? Scope.getActiveFlag()
+ : Address::invalid();
+ Address EHActiveFlag =
+ Scope.shouldTestFlagInEHCleanup() ? Scope.getActiveFlag()
+ : Address::invalid();
// Check whether we need an EH cleanup. This is only true if we've
// generated a lazy EH cleanup block.
@@ -674,13 +679,10 @@
// Copy the cleanup emission data out. Note that SmallVector
// guarantees maximal alignment for its buffer regardless of its
// type parameter.
- SmallVector<char, 8*sizeof(void*)> CleanupBuffer;
- CleanupBuffer.reserve(Scope.getCleanupSize());
- memcpy(CleanupBuffer.data(),
- Scope.getCleanupBuffer(), Scope.getCleanupSize());
- CleanupBuffer.set_size(Scope.getCleanupSize());
- EHScopeStack::Cleanup *Fn =
- reinterpret_cast<EHScopeStack::Cleanup*>(CleanupBuffer.data());
+ auto *CleanupSource = reinterpret_cast<char *>(Scope.getCleanupBuffer());
+ SmallVector<char, 8 * sizeof(void *)> CleanupBuffer(
+ CleanupSource, CleanupSource + Scope.getCleanupSize());
+ auto *Fn = reinterpret_cast<EHScopeStack::Cleanup *>(CleanupBuffer.data());
EHScopeStack::Cleanup::Flags cleanupFlags;
if (Scope.isNormalCleanup())
@@ -761,7 +763,7 @@
// Clean up the possibly dead store to the cleanup dest slot.
llvm::Instruction *NormalCleanupDestSlot =
- cast<llvm::Instruction>(getNormalCleanupDestSlot());
+ cast<llvm::Instruction>(getNormalCleanupDestSlot().getPointer());
if (NormalCleanupDestSlot->hasOneUse()) {
NormalCleanupDestSlot->user_back()->eraseFromParent();
NormalCleanupDestSlot->eraseFromParent();
@@ -787,7 +789,8 @@
const unsigned SwitchCapacity = 10;
llvm::LoadInst *Load =
- new llvm::LoadInst(getNormalCleanupDestSlot(), "cleanup.dest");
+ createLoadInstBefore(getNormalCleanupDestSlot(), "cleanup.dest",
+ nullptr);
llvm::SwitchInst *Switch =
llvm::SwitchInst::Create(Load, Default, SwitchCapacity);
@@ -833,9 +836,9 @@
BranchFixup &Fixup = EHStack.getBranchFixup(I);
if (!Fixup.Destination) continue;
if (!Fixup.OptimisticBranchBlock) {
- new llvm::StoreInst(Builder.getInt32(Fixup.DestinationIndex),
- getNormalCleanupDestSlot(),
- Fixup.InitialBranch);
+ createStoreInstBefore(Builder.getInt32(Fixup.DestinationIndex),
+ getNormalCleanupDestSlot(),
+ Fixup.InitialBranch);
Fixup.InitialBranch->setSuccessor(0, NormalEntry);
}
Fixup.OptimisticBranchBlock = NormalExit;
@@ -893,15 +896,40 @@
EmitBlock(EHEntry);
+ llvm::BasicBlock *NextAction = getEHDispatchBlock(EHParent);
+
+ // Push a terminate scope or cleanupendpad scope around the potentially
+ // throwing cleanups. For funclet EH personalities, the cleanupendpad models
+ // program termination when cleanups throw.
+ bool PushedTerminate = false;
+ SaveAndRestore<llvm::Instruction *> RestoreCurrentFuncletPad(
+ CurrentFuncletPad);
+ llvm::CleanupPadInst *CPI = nullptr;
+ if (!EHPersonality::get(*this).usesFuncletPads()) {
+ EHStack.pushTerminate();
+ PushedTerminate = true;
+ } else {
+ llvm::Value *ParentPad = CurrentFuncletPad;
+ if (!ParentPad)
+ ParentPad = llvm::ConstantTokenNone::get(CGM.getLLVMContext());
+ CurrentFuncletPad = CPI = Builder.CreateCleanupPad(ParentPad);
+ }
+
// We only actually emit the cleanup code if the cleanup is either
// active or was used before it was deactivated.
- if (EHActiveFlag || IsActive) {
-
+ if (EHActiveFlag.isValid() || IsActive) {
cleanupFlags.setIsForEHCleanup();
EmitCleanup(*this, Fn, cleanupFlags, EHActiveFlag);
}
- Builder.CreateBr(getEHDispatchBlock(EHParent));
+ if (CPI)
+ Builder.CreateCleanupRet(CPI, NextAction);
+ else
+ Builder.CreateBr(NextAction);
+
+ // Leave the terminate scope.
+ if (PushedTerminate)
+ EHStack.popTerminate();
Builder.restoreIP(SavedIP);
@@ -977,7 +1005,7 @@
// Store the index at the start.
llvm::ConstantInt *Index = Builder.getInt32(Dest.getDestIndex());
- new llvm::StoreInst(Index, getNormalCleanupDestSlot(), BI);
+ createStoreInstBefore(Index, getNormalCleanupDestSlot(), BI);
// Adjust BI to point to the first cleanup block.
{
@@ -1096,23 +1124,24 @@
// If it hasn't yet been used as either, we're done.
if (!needFlag) return;
- llvm::AllocaInst *var = Scope.getActiveFlag();
- if (!var) {
- var = CGF.CreateTempAlloca(CGF.Builder.getInt1Ty(), "cleanup.isactive");
+ Address var = Scope.getActiveFlag();
+ if (!var.isValid()) {
+ var = CGF.CreateTempAlloca(CGF.Builder.getInt1Ty(), CharUnits::One(),
+ "cleanup.isactive");
Scope.setActiveFlag(var);
assert(dominatingIP && "no existing variable and no dominating IP!");
// Initialize to true or false depending on whether it was
// active up to this point.
- llvm::Value *value = CGF.Builder.getInt1(kind == ForDeactivation);
+ llvm::Constant *value = CGF.Builder.getInt1(kind == ForDeactivation);
// If we're in a conditional block, ignore the dominating IP and
// use the outermost conditional branch.
if (CGF.isInConditionalBranch()) {
CGF.setBeforeOutermostConditional(value, var);
} else {
- new llvm::StoreInst(value, var, dominatingIP);
+ createStoreInstBefore(value, var, dominatingIP);
}
}
@@ -1154,17 +1183,17 @@
Scope.setActive(false);
}
-llvm::Value *CodeGenFunction::getNormalCleanupDestSlot() {
+Address CodeGenFunction::getNormalCleanupDestSlot() {
if (!NormalCleanupDest)
NormalCleanupDest =
CreateTempAlloca(Builder.getInt32Ty(), "cleanup.dest.slot");
- return NormalCleanupDest;
+ return Address(NormalCleanupDest, CharUnits::fromQuantity(4));
}
/// Emits all the code to cause the given temporary to be cleaned up.
void CodeGenFunction::EmitCXXTemporary(const CXXTemporary *Temporary,
QualType TempType,
- llvm::Value *Ptr) {
+ Address Ptr) {
pushDestroy(NormalAndEHCleanup, Ptr, TempType, destroyCXXObject,
/*useEHCleanup*/ true);
}
diff --git a/lib/CodeGen/CGCleanup.h b/lib/CodeGen/CGCleanup.h
index 81c6412..909f00b 100644
--- a/lib/CodeGen/CGCleanup.h
+++ b/lib/CodeGen/CGCleanup.h
@@ -15,6 +15,8 @@
#define LLVM_CLANG_LIB_CODEGEN_CGCLEANUP_H
#include "EHScopeStack.h"
+
+#include "Address.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
@@ -26,7 +28,17 @@
}
namespace clang {
+class FunctionDecl;
namespace CodeGen {
+class CodeGenModule;
+class CodeGenFunction;
+
+/// The MS C++ ABI needs a pointer to RTTI data plus some flags to describe the
+/// type of a catch handler, so we use this wrapper.
+struct CatchTypeInfo {
+ llvm::Constant *RTTI;
+ unsigned Flags;
+};
/// A protected scope for zero-cost EH handling.
class EHScope {
@@ -37,9 +49,9 @@
class CommonBitFields {
friend class EHScope;
- unsigned Kind : 2;
+ unsigned Kind : 3;
};
- enum { NumCommonBits = 2 };
+ enum { NumCommonBits = 3 };
protected:
class CatchBitFields {
@@ -78,7 +90,7 @@
/// The number of fixups required by enclosing scopes (not including
/// this one). If this is the top cleanup scope, all the fixups
/// from this index onwards belong to this scope.
- unsigned FixupDepth : 32 - 18 - NumCommonBits; // currently 13
+ unsigned FixupDepth : 32 - 18 - NumCommonBits; // currently 12
};
class FilterBitFields {
@@ -96,7 +108,7 @@
};
public:
- enum Kind { Cleanup, Catch, Terminate, Filter };
+ enum Kind { Cleanup, Catch, Terminate, Filter, PadEnd };
EHScope(Kind kind, EHScopeStack::stable_iterator enclosingEHScope)
: CachedLandingPad(nullptr), CachedEHDispatchBlock(nullptr),
@@ -148,12 +160,12 @@
struct Handler {
/// A type info value, or null (C++ null, not an LLVM null pointer)
/// for a catch-all.
- llvm::Constant *Type;
+ CatchTypeInfo Type;
/// The catch handler for this type.
llvm::BasicBlock *Block;
- bool isCatchAll() const { return Type == nullptr; }
+ bool isCatchAll() const { return Type.RTTI == nullptr; }
};
private:
@@ -183,11 +195,17 @@
}
void setCatchAllHandler(unsigned I, llvm::BasicBlock *Block) {
- setHandler(I, /*catchall*/ nullptr, Block);
+ setHandler(I, CatchTypeInfo{nullptr, 0}, Block);
}
void setHandler(unsigned I, llvm::Constant *Type, llvm::BasicBlock *Block) {
assert(I < getNumHandlers());
+ getHandlers()[I].Type = CatchTypeInfo{Type, 0};
+ getHandlers()[I].Block = Block;
+ }
+
+ void setHandler(unsigned I, CatchTypeInfo Type, llvm::BasicBlock *Block) {
+ assert(I < getNumHandlers());
getHandlers()[I].Type = Type;
getHandlers()[I].Block = Block;
}
@@ -216,7 +234,7 @@
};
/// A cleanup scope which generates the cleanup blocks lazily.
-class EHCleanupScope : public EHScope {
+class LLVM_ALIGNAS(/*alignof(uint64_t)*/ 8) EHCleanupScope : public EHScope {
/// The nearest normal cleanup scope enclosing this one.
EHScopeStack::stable_iterator EnclosingNormal;
@@ -302,8 +320,14 @@
bool isLifetimeMarker() const { return CleanupBits.IsLifetimeMarker; }
void setLifetimeMarker() { CleanupBits.IsLifetimeMarker = true; }
- llvm::AllocaInst *getActiveFlag() const { return ActiveFlag; }
- void setActiveFlag(llvm::AllocaInst *Var) { ActiveFlag = Var; }
+ bool hasActiveFlag() const { return ActiveFlag != nullptr; }
+ Address getActiveFlag() const {
+ return Address(ActiveFlag, CharUnits::One());
+ }
+ void setActiveFlag(Address Var) {
+ assert(Var.getAlignment().isOne());
+ ActiveFlag = cast<llvm::AllocaInst>(Var.getPointer());
+ }
void setTestFlagInNormalCleanup() {
CleanupBits.TestFlagInNormalCleanup = true;
@@ -396,6 +420,15 @@
return (Scope->getKind() == Cleanup);
}
};
+// NOTE: there's a bunch of different data classes tacked on after an
+// EHCleanupScope. It is asserted (in EHScopeStack::pushCleanup*) that
+// they don't require greater alignment than ScopeStackAlignment. So,
+// EHCleanupScope ought to have alignment equal to that -- not more
+// (would be misaligned by the stack allocator), and not less (would
+// break the appended classes).
+static_assert(llvm::AlignOf<EHCleanupScope>::Alignment ==
+ EHScopeStack::ScopeStackAlignment,
+ "EHCleanupScope expected alignment");
/// An exceptions scope which filters exceptions thrown through it.
/// Only exceptions matching the filter types will be permitted to be
@@ -454,6 +487,17 @@
}
};
+class EHPadEndScope : public EHScope {
+public:
+ EHPadEndScope(EHScopeStack::stable_iterator enclosingEHScope)
+ : EHScope(PadEnd, enclosingEHScope) {}
+ static size_t getSize() { return sizeof(EHPadEndScope); }
+
+ static bool classof(const EHScope *scope) {
+ return scope->getKind() == PadEnd;
+ }
+};
+
/// A non-stable pointer into the scope stack.
class EHScopeStack::iterator {
char *Ptr;
@@ -472,27 +516,31 @@
EHScope &operator*() const { return *get(); }
iterator &operator++() {
+ size_t Size;
switch (get()->getKind()) {
case EHScope::Catch:
- Ptr += EHCatchScope::getSizeForNumHandlers(
- static_cast<const EHCatchScope*>(get())->getNumHandlers());
+ Size = EHCatchScope::getSizeForNumHandlers(
+ static_cast<const EHCatchScope *>(get())->getNumHandlers());
break;
case EHScope::Filter:
- Ptr += EHFilterScope::getSizeForNumFilters(
- static_cast<const EHFilterScope*>(get())->getNumFilters());
+ Size = EHFilterScope::getSizeForNumFilters(
+ static_cast<const EHFilterScope *>(get())->getNumFilters());
break;
case EHScope::Cleanup:
- Ptr += static_cast<const EHCleanupScope*>(get())
- ->getAllocatedSize();
+ Size = static_cast<const EHCleanupScope *>(get())->getAllocatedSize();
break;
case EHScope::Terminate:
- Ptr += EHTerminateScope::getSize();
+ Size = EHTerminateScope::getSize();
+ break;
+
+ case EHScope::PadEnd:
+ Size = EHPadEndScope::getSize();
break;
}
-
+ Ptr += llvm::RoundUpToAlignment(Size, ScopeStackAlignment);
return *this;
}
@@ -528,7 +576,7 @@
EHCatchScope &scope = cast<EHCatchScope>(*begin());
InnermostEHScope = scope.getEnclosingEHScope();
- StartOfData += EHCatchScope::getSizeForNumHandlers(scope.getNumHandlers());
+ deallocate(EHCatchScope::getSizeForNumHandlers(scope.getNumHandlers()));
}
inline void EHScopeStack::popTerminate() {
@@ -536,7 +584,7 @@
EHTerminateScope &scope = cast<EHTerminateScope>(*begin());
InnermostEHScope = scope.getEnclosingEHScope();
- StartOfData += EHTerminateScope::getSize();
+ deallocate(EHTerminateScope::getSize());
}
inline EHScopeStack::iterator EHScopeStack::find(stable_iterator sp) const {
@@ -551,6 +599,43 @@
return stable_iterator(EndOfBuffer - ir.Ptr);
}
+/// The exceptions personality for a function.
+struct EHPersonality {
+ const char *PersonalityFn;
+
+ // If this is non-null, this personality requires a non-standard
+ // function for rethrowing an exception after a catchall cleanup.
+ // This function must have prototype void(void*).
+ const char *CatchallRethrowFn;
+
+ static const EHPersonality &get(CodeGenModule &CGM, const FunctionDecl *FD);
+ static const EHPersonality &get(CodeGenFunction &CGF);
+
+ static const EHPersonality GNU_C;
+ static const EHPersonality GNU_C_SJLJ;
+ static const EHPersonality GNU_C_SEH;
+ static const EHPersonality GNU_ObjC;
+ static const EHPersonality GNUstep_ObjC;
+ static const EHPersonality GNU_ObjCXX;
+ static const EHPersonality NeXT_ObjC;
+ static const EHPersonality GNU_CPlusPlus;
+ static const EHPersonality GNU_CPlusPlus_SJLJ;
+ static const EHPersonality GNU_CPlusPlus_SEH;
+ static const EHPersonality MSVC_except_handler;
+ static const EHPersonality MSVC_C_specific_handler;
+ static const EHPersonality MSVC_CxxFrameHandler3;
+
+ /// Does this personality use landingpads or the family of pad instructions
+ /// designed to form funclets?
+ bool usesFuncletPads() const { return isMSVCPersonality(); }
+
+ bool isMSVCPersonality() const {
+ return this == &MSVC_except_handler || this == &MSVC_C_specific_handler ||
+ this == &MSVC_CxxFrameHandler3;
+ }
+
+ bool isMSVCXXPersonality() const { return this == &MSVC_CxxFrameHandler3; }
+};
}
}
diff --git a/lib/CodeGen/CGDebugInfo.cpp b/lib/CodeGen/CGDebugInfo.cpp
index 48458db..78e3978 100644
--- a/lib/CodeGen/CGDebugInfo.cpp
+++ b/lib/CodeGen/CGDebugInfo.cpp
@@ -27,6 +27,9 @@
#include "clang/Basic/SourceManager.h"
#include "clang/Basic/Version.h"
#include "clang/Frontend/CodeGenOptions.h"
+#include "clang/Lex/HeaderSearchOptions.h"
+#include "clang/Lex/ModuleMap.h"
+#include "clang/Lex/PreprocessorOptions.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/IR/Constants.h"
@@ -35,7 +38,6 @@
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Module.h"
-#include "llvm/Support/Dwarf.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
using namespace clang;
@@ -43,7 +45,10 @@
CGDebugInfo::CGDebugInfo(CodeGenModule &CGM)
: CGM(CGM), DebugKind(CGM.getCodeGenOpts().getDebugInfo()),
+ DebugTypeExtRefs(CGM.getCodeGenOpts().DebugTypeExtRefs),
DBuilder(CGM.getModule()) {
+ for (const auto &KV : CGM.getCodeGenOpts().DebugPrefixMap)
+ DebugPrefixMap[KV.first] = KV.second;
CreateCompileUnit();
}
@@ -54,58 +59,65 @@
ApplyDebugLocation::ApplyDebugLocation(CodeGenFunction &CGF,
SourceLocation TemporaryLocation)
- : CGF(CGF) {
+ : CGF(&CGF) {
init(TemporaryLocation);
}
ApplyDebugLocation::ApplyDebugLocation(CodeGenFunction &CGF,
bool DefaultToEmpty,
SourceLocation TemporaryLocation)
- : CGF(CGF) {
+ : CGF(&CGF) {
init(TemporaryLocation, DefaultToEmpty);
}
void ApplyDebugLocation::init(SourceLocation TemporaryLocation,
bool DefaultToEmpty) {
- if (auto *DI = CGF.getDebugInfo()) {
- OriginalLocation = CGF.Builder.getCurrentDebugLocation();
- if (TemporaryLocation.isInvalid()) {
- if (DefaultToEmpty)
- CGF.Builder.SetCurrentDebugLocation(llvm::DebugLoc());
- else {
- // Construct a location that has a valid scope, but no line info.
- assert(!DI->LexicalBlockStack.empty());
- CGF.Builder.SetCurrentDebugLocation(
- llvm::DebugLoc::get(0, 0, DI->LexicalBlockStack.back()));
- }
- } else
- DI->EmitLocation(CGF.Builder, TemporaryLocation);
+ auto *DI = CGF->getDebugInfo();
+ if (!DI) {
+ CGF = nullptr;
+ return;
}
+
+ OriginalLocation = CGF->Builder.getCurrentDebugLocation();
+ if (TemporaryLocation.isValid()) {
+ DI->EmitLocation(CGF->Builder, TemporaryLocation);
+ return;
+ }
+
+ if (DefaultToEmpty) {
+ CGF->Builder.SetCurrentDebugLocation(llvm::DebugLoc());
+ return;
+ }
+
+ // Construct a location that has a valid scope, but no line info.
+ assert(!DI->LexicalBlockStack.empty());
+ CGF->Builder.SetCurrentDebugLocation(
+ llvm::DebugLoc::get(0, 0, DI->LexicalBlockStack.back()));
}
ApplyDebugLocation::ApplyDebugLocation(CodeGenFunction &CGF, const Expr *E)
- : CGF(CGF) {
+ : CGF(&CGF) {
init(E->getExprLoc());
}
ApplyDebugLocation::ApplyDebugLocation(CodeGenFunction &CGF, llvm::DebugLoc Loc)
- : CGF(CGF) {
- if (CGF.getDebugInfo()) {
- OriginalLocation = CGF.Builder.getCurrentDebugLocation();
- if (Loc)
- CGF.Builder.SetCurrentDebugLocation(std::move(Loc));
+ : CGF(&CGF) {
+ if (!CGF.getDebugInfo()) {
+ this->CGF = nullptr;
+ return;
}
+ OriginalLocation = CGF.Builder.getCurrentDebugLocation();
+ if (Loc)
+ CGF.Builder.SetCurrentDebugLocation(std::move(Loc));
}
ApplyDebugLocation::~ApplyDebugLocation() {
// Query CGF so the location isn't overwritten when location updates are
// temporarily disabled (for C++ default function arguments)
- if (CGF.getDebugInfo())
- CGF.Builder.SetCurrentDebugLocation(std::move(OriginalLocation));
+ if (CGF)
+ CGF->Builder.SetCurrentDebugLocation(std::move(OriginalLocation));
}
-/// ArtificialLocation - An RAII object that temporarily switches to
-/// an artificial debug location that has a valid scope, but no line
void CGDebugInfo::setLocation(SourceLocation Loc) {
// If the new location isn't valid return.
if (Loc.isInvalid())
@@ -138,10 +150,16 @@
}
}
-/// getContextDescriptor - Get context info for the decl.
-llvm::DIScope *CGDebugInfo::getContextDescriptor(const Decl *Context) {
+llvm::DIScope *CGDebugInfo::getDeclContextDescriptor(const Decl *D) {
+ llvm::DIScope *Mod = getParentModuleOrNull(D);
+ return getContextDescriptor(cast<Decl>(D->getDeclContext()),
+ Mod ? Mod : TheCU);
+}
+
+llvm::DIScope *CGDebugInfo::getContextDescriptor(const Decl *Context,
+ llvm::DIScope *Default) {
if (!Context)
- return TheCU;
+ return Default;
auto I = RegionMap.find(Context);
if (I != RegionMap.end()) {
@@ -157,33 +175,39 @@
if (!RDecl->isDependentType())
return getOrCreateType(CGM.getContext().getTypeDeclType(RDecl),
getOrCreateMainFile());
- return TheCU;
+ return Default;
}
-/// getFunctionName - Get function name for the given FunctionDecl. If the
-/// name is constructed on demand (e.g. C++ destructor) then the name
-/// is stored on the side.
StringRef CGDebugInfo::getFunctionName(const FunctionDecl *FD) {
assert(FD && "Invalid FunctionDecl!");
IdentifierInfo *FII = FD->getIdentifier();
FunctionTemplateSpecializationInfo *Info =
FD->getTemplateSpecializationInfo();
- if (!Info && FII)
+
+ if (!Info && FII && !CGM.getCodeGenOpts().EmitCodeView)
return FII->getName();
// Otherwise construct human readable name for debug info.
SmallString<128> NS;
llvm::raw_svector_ostream OS(NS);
- FD->printName(OS);
+ PrintingPolicy Policy(CGM.getLangOpts());
- // Add any template specialization args.
- if (Info) {
- const TemplateArgumentList *TArgs = Info->TemplateArguments;
- const TemplateArgument *Args = TArgs->data();
- unsigned NumArgs = TArgs->size();
- PrintingPolicy Policy(CGM.getLangOpts());
- TemplateSpecializationType::PrintTemplateArgumentList(OS, Args, NumArgs,
- Policy);
+ if (CGM.getCodeGenOpts().EmitCodeView) {
+ // Print a fully qualified name like MSVC would.
+ Policy.MSVCFormatting = true;
+ FD->printQualifiedName(OS, Policy);
+ } else {
+ // Print the unqualified name with some template arguments. This is what
+ // DWARF-based debuggers expect.
+ FD->printName(OS);
+ // Add any template specialization args.
+ if (Info) {
+ const TemplateArgumentList *TArgs = Info->TemplateArguments;
+ const TemplateArgument *Args = TArgs->data();
+ unsigned NumArgs = TArgs->size();
+ TemplateSpecializationType::PrintTemplateArgumentList(OS, Args, NumArgs,
+ Policy);
+ }
}
// Copy this name on the side and use its reference.
@@ -201,6 +225,13 @@
} else if (const ObjCInterfaceDecl *OID =
dyn_cast<const ObjCInterfaceDecl>(DC)) {
OS << OID->getName();
+ } else if (const ObjCCategoryDecl *OC = dyn_cast<ObjCCategoryDecl>(DC)) {
+ if (OC->IsClassExtension()) {
+ OS << OC->getClassInterface()->getName();
+ } else {
+ OS << ((const NamedDecl *)OC)->getIdentifier()->getNameStart() << '('
+ << OC->getIdentifier()->getNameStart() << ')';
+ }
} else if (const ObjCCategoryImplDecl *OCD =
dyn_cast<const ObjCCategoryImplDecl>(DC)) {
OS << ((const NamedDecl *)OCD)->getIdentifier()->getNameStart() << '('
@@ -218,13 +249,10 @@
return internString(OS.str());
}
-/// getSelectorName - Return selector name. This is used for debugging
-/// info.
StringRef CGDebugInfo::getSelectorName(Selector S) {
return internString(S.getAsString());
}
-/// getClassName - Get class name including template argument list.
StringRef CGDebugInfo::getClassName(const RecordDecl *RD) {
// quick optimization to avoid having to intern strings that are already
// stored reliably elsewhere
@@ -242,18 +270,19 @@
return internString(Name);
}
-/// getOrCreateFile - Get the file debug info descriptor for the input location.
llvm::DIFile *CGDebugInfo::getOrCreateFile(SourceLocation Loc) {
if (!Loc.isValid())
// If Location is not valid then use main input file.
- return DBuilder.createFile(TheCU->getFilename(), TheCU->getDirectory());
+ return DBuilder.createFile(remapDIPath(TheCU->getFilename()),
+ remapDIPath(TheCU->getDirectory()));
SourceManager &SM = CGM.getContext().getSourceManager();
PresumedLoc PLoc = SM.getPresumedLoc(Loc);
if (PLoc.isInvalid() || StringRef(PLoc.getFilename()).empty())
// If the location is not valid then use main input file.
- return DBuilder.createFile(TheCU->getFilename(), TheCU->getDirectory());
+ return DBuilder.createFile(remapDIPath(TheCU->getFilename()),
+ remapDIPath(TheCU->getDirectory()));
// Cache the results.
const char *fname = PLoc.getFilename();
@@ -265,20 +294,25 @@
return cast<llvm::DIFile>(V);
}
- llvm::DIFile *F =
- DBuilder.createFile(PLoc.getFilename(), getCurrentDirname());
+ llvm::DIFile *F = DBuilder.createFile(remapDIPath(PLoc.getFilename()),
+ remapDIPath(getCurrentDirname()));
DIFileCache[fname].reset(F);
return F;
}
-/// getOrCreateMainFile - Get the file info for main compile unit.
llvm::DIFile *CGDebugInfo::getOrCreateMainFile() {
- return DBuilder.createFile(TheCU->getFilename(), TheCU->getDirectory());
+ return DBuilder.createFile(remapDIPath(TheCU->getFilename()),
+ remapDIPath(TheCU->getDirectory()));
}
-/// getLineNumber - Get line number for the location. If location is invalid
-/// then use current location.
+std::string CGDebugInfo::remapDIPath(StringRef Path) const {
+ for (const auto &Entry : DebugPrefixMap)
+ if (Path.startswith(Entry.first))
+ return (Twine(Entry.second) + Path.substr(Entry.first.size())).str();
+ return Path.str();
+}
+
unsigned CGDebugInfo::getLineNumber(SourceLocation Loc) {
if (Loc.isInvalid() && CurLoc.isInvalid())
return 0;
@@ -287,7 +321,6 @@
return PLoc.isValid() ? PLoc.getLine() : 0;
}
-/// getColumnNumber - Get column number for the location.
unsigned CGDebugInfo::getColumnNumber(SourceLocation Loc, bool Force) {
// We may not want column information at all.
if (!Force && !CGM.getCodeGenOpts().DebugColumnInfo)
@@ -312,7 +345,6 @@
return CWDName = internString(CWD);
}
-/// CreateCompileUnit - Create new compile unit.
void CGDebugInfo::CreateCompileUnit() {
// Should we be asking the SourceManager for the main file name, instead of
@@ -334,7 +366,7 @@
// file to determine the real absolute path for the file.
std::string MainFileDir;
if (const FileEntry *MainFile = SM.getFileEntryForID(SM.getMainFileID())) {
- MainFileDir = MainFile->getDir()->getName();
+ MainFileDir = remapDIPath(MainFile->getDir()->getName());
if (MainFileDir != ".") {
llvm::SmallString<1024> MainFileDirSS(MainFileDir);
llvm::sys::path::append(MainFileDirSS, MainFileName);
@@ -342,13 +374,6 @@
}
}
- // Save filename string.
- StringRef Filename = internString(MainFileName);
-
- // Save split dwarf file string.
- std::string SplitDwarfFile = CGM.getCodeGenOpts().SplitDwarfFile;
- StringRef SplitDwarfFilename = internString(SplitDwarfFile);
-
llvm::dwarf::SourceLanguage LangTag;
const LangOptions &LO = CGM.getLangOpts();
if (LO.CPlusPlus) {
@@ -374,17 +399,15 @@
// Create new compile unit.
// FIXME - Eliminate TheCU.
TheCU = DBuilder.createCompileUnit(
- LangTag, Filename, getCurrentDirname(), Producer, LO.Optimize,
- CGM.getCodeGenOpts().DwarfDebugFlags, RuntimeVers, SplitDwarfFilename,
+ LangTag, remapDIPath(MainFileName), remapDIPath(getCurrentDirname()),
+ Producer, LO.Optimize, CGM.getCodeGenOpts().DwarfDebugFlags, RuntimeVers,
+ CGM.getCodeGenOpts().SplitDwarfFile,
DebugKind <= CodeGenOptions::DebugLineTablesOnly
? llvm::DIBuilder::LineTablesOnly
: llvm::DIBuilder::FullDebug,
- 0 /* DWOid */,
- DebugKind != CodeGenOptions::LocTrackingOnly);
+ 0 /* DWOid */, DebugKind != CodeGenOptions::LocTrackingOnly);
}
-/// CreateType - Get the Basic type from the cache or create a new
-/// one if necessary.
llvm::DIType *CGDebugInfo::CreateType(const BuiltinType *BT) {
llvm::dwarf::TypeKind Encoding;
StringRef BTName;
@@ -453,6 +476,24 @@
case BuiltinType::OCLImage2dArray:
return getOrCreateStructPtrType("opencl_image2d_array_t",
OCLImage2dArrayDITy);
+ case BuiltinType::OCLImage2dDepth:
+ return getOrCreateStructPtrType("opencl_image2d_depth_t",
+ OCLImage2dDepthDITy);
+ case BuiltinType::OCLImage2dArrayDepth:
+ return getOrCreateStructPtrType("opencl_image2d_array_depth_t",
+ OCLImage2dArrayDepthDITy);
+ case BuiltinType::OCLImage2dMSAA:
+ return getOrCreateStructPtrType("opencl_image2d_msaa_t",
+ OCLImage2dMSAADITy);
+ case BuiltinType::OCLImage2dArrayMSAA:
+ return getOrCreateStructPtrType("opencl_image2d_array_msaa_t",
+ OCLImage2dArrayMSAADITy);
+ case BuiltinType::OCLImage2dMSAADepth:
+ return getOrCreateStructPtrType("opencl_image2d_msaa_depth_t",
+ OCLImage2dMSAADepthDITy);
+ case BuiltinType::OCLImage2dArrayMSAADepth:
+ return getOrCreateStructPtrType("opencl_image2d_array_msaa_depth_t",
+ OCLImage2dArrayMSAADepthDITy);
case BuiltinType::OCLImage3d:
return getOrCreateStructPtrType("opencl_image3d_t", OCLImage3dDITy);
case BuiltinType::OCLSampler:
@@ -461,6 +502,14 @@
CGM.getContext().getTypeAlign(BT), llvm::dwarf::DW_ATE_unsigned);
case BuiltinType::OCLEvent:
return getOrCreateStructPtrType("opencl_event_t", OCLEventDITy);
+ case BuiltinType::OCLClkEvent:
+ return getOrCreateStructPtrType("opencl_clk_event_t", OCLClkEventDITy);
+ case BuiltinType::OCLQueue:
+ return getOrCreateStructPtrType("opencl_queue_t", OCLQueueDITy);
+ case BuiltinType::OCLNDRange:
+ return getOrCreateStructPtrType("opencl_ndrange_t", OCLNDRangeDITy);
+ case BuiltinType::OCLReserveID:
+ return getOrCreateStructPtrType("opencl_reserve_id_t", OCLReserveIDDITy);
case BuiltinType::UChar:
case BuiltinType::Char_U:
@@ -535,8 +584,6 @@
return DBuilder.createBasicType("complex", Size, Align, Encoding);
}
-/// CreateCVRType - Get the qualified type from the cache or create
-/// a new one if necessary.
llvm::DIType *CGDebugInfo::CreateQualifiedType(QualType Ty,
llvm::DIFile *Unit) {
QualifierCollector Qc;
@@ -590,18 +637,29 @@
Ty->getPointeeType(), Unit);
}
+/// \return whether a C++ mangling exists for the type defined by TD.
+static bool hasCXXMangling(const TagDecl *TD, llvm::DICompileUnit *TheCU) {
+ switch (TheCU->getSourceLanguage()) {
+ case llvm::dwarf::DW_LANG_C_plus_plus:
+ return true;
+ case llvm::dwarf::DW_LANG_ObjC_plus_plus:
+ return isa<CXXRecordDecl>(TD) || isa<EnumDecl>(TD);
+ default:
+ return false;
+ }
+}
+
/// In C++ mode, types have linkage, so we can rely on the ODR and
/// on their mangled names, if they're external.
static SmallString<256> getUniqueTagTypeName(const TagType *Ty,
CodeGenModule &CGM,
llvm::DICompileUnit *TheCU) {
SmallString<256> FullName;
- // FIXME: ODR should apply to ObjC++ exactly the same wasy it does to C++.
- // For now, only apply ODR with C++.
const TagDecl *TD = Ty->getDecl();
- if (TheCU->getSourceLanguage() != llvm::dwarf::DW_LANG_C_plus_plus ||
- !TD->isExternallyVisible())
+
+ if (!hasCXXMangling(TD, TheCU) || !TD->isExternallyVisible())
return FullName;
+
// Microsoft Mangler does not have support for mangleCXXRTTIName yet.
if (CGM.getTarget().getCXXABI().isMicrosoft())
return FullName;
@@ -610,10 +668,10 @@
// a unique string for a type?
llvm::raw_svector_ostream Out(FullName);
CGM.getCXXABI().getMangleContext().mangleCXXRTTIName(QualType(Ty, 0), Out);
- Out.flush();
return FullName;
}
+/// \return the approproate DWARF tag for a composite type.
static llvm::dwarf::Tag getTagForRecord(const RecordDecl *RD) {
llvm::dwarf::Tag Tag;
if (RD->isStruct() || RD->isInterface())
@@ -629,7 +687,6 @@
return Tag;
}
-// Creates a forward declaration for a RecordDecl in the given context.
llvm::DICompositeType *
CGDebugInfo::getOrCreateRecordFwdDecl(const RecordType *Ty,
llvm::DIScope *Ctx) {
@@ -664,10 +721,6 @@
const Type *Ty,
QualType PointeeTy,
llvm::DIFile *Unit) {
- if (Tag == llvm::dwarf::DW_TAG_reference_type ||
- Tag == llvm::dwarf::DW_TAG_rvalue_reference_type)
- return DBuilder.createReferenceType(Tag, getOrCreateType(PointeeTy, Unit));
-
// Bit size, align and offset of the type.
// Size is always the size of a pointer. We can't use getTypeSize here
// because that does not return the correct value for references.
@@ -675,8 +728,13 @@
uint64_t Size = CGM.getTarget().getPointerWidth(AS);
uint64_t Align = CGM.getContext().getTypeAlign(Ty);
- return DBuilder.createPointerType(getOrCreateType(PointeeTy, Unit), Size,
- Align);
+ if (Tag == llvm::dwarf::DW_TAG_reference_type ||
+ Tag == llvm::dwarf::DW_TAG_rvalue_reference_type)
+ return DBuilder.createReferenceType(Tag, getOrCreateType(PointeeTy, Unit),
+ Size, Align);
+ else
+ return DBuilder.createPointerType(getOrCreateType(PointeeTy, Unit), Size,
+ Align);
}
llvm::DIType *CGDebugInfo::getOrCreateStructPtrType(StringRef Name,
@@ -692,9 +750,6 @@
llvm::DIType *CGDebugInfo::CreateType(const BlockPointerType *Ty,
llvm::DIFile *Unit) {
- if (BlockLiteralGeneric)
- return BlockLiteralGeneric;
-
SmallVector<llvm::Metadata *, 8> EltTys;
QualType FType;
uint64_t FieldSize, FieldOffset;
@@ -710,10 +765,10 @@
EltTys.clear();
unsigned Flags = llvm::DINode::FlagAppleBlock;
- unsigned LineNo = getLineNumber(CurLoc);
+ unsigned LineNo = 0;
auto *EltTy =
- DBuilder.createStructType(Unit, "__block_descriptor", Unit, LineNo,
+ DBuilder.createStructType(Unit, "__block_descriptor", nullptr, LineNo,
FieldOffset, 0, Flags, nullptr, Elements);
// Bit size, align and offset of the type.
@@ -733,19 +788,22 @@
FType = CGM.getContext().getPointerType(CGM.getContext().VoidTy);
FieldSize = CGM.getContext().getTypeSize(Ty);
FieldAlign = CGM.getContext().getTypeAlign(Ty);
- EltTys.push_back(DBuilder.createMemberType(Unit, "__descriptor", Unit, LineNo,
+ EltTys.push_back(DBuilder.createMemberType(Unit, "__descriptor", nullptr, LineNo,
FieldSize, FieldAlign, FieldOffset,
0, DescTy));
FieldOffset += FieldSize;
Elements = DBuilder.getOrCreateArray(EltTys);
+ // The __block_literal_generic structs are marked with a special
+ // DW_AT_APPLE_BLOCK attribute and are an implementation detail only
+ // the debugger needs to know about. To allow type uniquing, emit
+ // them without a name or a location.
EltTy =
- DBuilder.createStructType(Unit, "__block_literal_generic", Unit, LineNo,
+ DBuilder.createStructType(Unit, "", nullptr, LineNo,
FieldOffset, 0, Flags, nullptr, Elements);
- BlockLiteralGeneric = DBuilder.createPointerType(EltTy, Size);
- return BlockLiteralGeneric;
+ return DBuilder.createPointerType(EltTy, Size);
}
llvm::DIType *CGDebugInfo::CreateType(const TemplateSpecializationType *Ty,
@@ -766,9 +824,9 @@
Ty->getTemplateName().getAsTemplateDecl())->getTemplatedDecl();
SourceLocation Loc = AliasDecl->getLocation();
- return DBuilder.createTypedef(
- Src, internString(OS.str()), getOrCreateFile(Loc), getLineNumber(Loc),
- getContextDescriptor(cast<Decl>(AliasDecl->getDeclContext())));
+ return DBuilder.createTypedef(Src, OS.str(), getOrCreateFile(Loc),
+ getLineNumber(Loc),
+ getDeclContextDescriptor(AliasDecl));
}
llvm::DIType *CGDebugInfo::CreateType(const TypedefType *Ty,
@@ -781,7 +839,7 @@
return DBuilder.createTypedef(
getOrCreateType(Ty->getDecl()->getUnderlyingType(), Unit),
Ty->getDecl()->getName(), getOrCreateFile(Loc), getLineNumber(Loc),
- getContextDescriptor(cast<Decl>(Ty->getDecl()->getDeclContext())));
+ getDeclContextDescriptor(Ty->getDecl()));
}
llvm::DIType *CGDebugInfo::CreateType(const FunctionType *Ty,
@@ -803,7 +861,7 @@
}
llvm::DITypeRefArray EltTypeArray = DBuilder.getOrCreateTypeArray(EltTys);
- return DBuilder.createSubroutineType(Unit, EltTypeArray);
+ return DBuilder.createSubroutineType(EltTypeArray);
}
/// Convert an AccessSpecifier into the corresponding DINode flag.
@@ -858,7 +916,6 @@
AlignInBits, offsetInBits, flags, debugType);
}
-/// CollectRecordLambdaFields - Helper for CollectRecordFields.
void CGDebugInfo::CollectRecordLambdaFields(
const CXXRecordDecl *CXXDecl, SmallVectorImpl<llvm::Metadata *> &elements,
llvm::DIType *RecordTy) {
@@ -903,7 +960,6 @@
}
}
-/// Helper for CollectRecordFields.
llvm::DIDerivedType *
CGDebugInfo::CreateRecordStaticField(const VarDecl *Var, llvm::DIType *RecordTy,
const RecordDecl *RD) {
@@ -933,7 +989,6 @@
return GV;
}
-/// CollectRecordNormalField - Helper for CollectRecordFields.
void CGDebugInfo::CollectRecordNormalField(
const FieldDecl *field, uint64_t OffsetInBits, llvm::DIFile *tunit,
SmallVectorImpl<llvm::Metadata *> &elements, llvm::DIType *RecordTy,
@@ -958,8 +1013,6 @@
elements.push_back(fieldType);
}
-/// CollectRecordFields - A helper function to collect debug info for
-/// record fields. This is used while creating debug info entry for a Record.
void CGDebugInfo::CollectRecordFields(
const RecordDecl *record, llvm::DIFile *tunit,
SmallVectorImpl<llvm::Metadata *> &elements,
@@ -983,7 +1036,7 @@
if (MI != StaticDataMemberCache.end()) {
assert(MI->second &&
"Static data member declaration should still exist");
- elements.push_back(cast<llvm::DIDerivedTypeBase>(MI->second));
+ elements.push_back(MI->second);
} else {
auto Field = CreateRecordStaticField(V, RecordTy, record);
elements.push_back(Field);
@@ -998,9 +1051,6 @@
}
}
-/// getOrCreateMethodType - CXXMethodDecl's type is a FunctionType. This
-/// function type is not updated to include implicit "this" pointer. Use this
-/// routine to get a method type which includes "this" pointer.
llvm::DISubroutineType *
CGDebugInfo::getOrCreateMethodType(const CXXMethodDecl *Method,
llvm::DIFile *Unit) {
@@ -1062,7 +1112,7 @@
if (Func->getExtProtoInfo().RefQualifier == RQ_RValue)
Flags |= llvm::DINode::FlagRValueReference;
- return DBuilder.createSubroutineType(Unit, EltTypeArray, Flags);
+ return DBuilder.createSubroutineType(EltTypeArray, Flags);
}
/// isFunctionLocalClass - Return true if CXXRecordDecl is defined
@@ -1075,8 +1125,6 @@
return false;
}
-/// CreateCXXMemberFunction - A helper function to create a subprogram for
-/// a single member function GlobalDecl.
llvm::DISubprogram *CGDebugInfo::CreateCXXMemberFunction(
const CXXMethodDecl *Method, llvm::DIFile *Unit, llvm::DIType *RecordTy) {
bool IsCtorOrDtor =
@@ -1145,16 +1193,13 @@
RecordTy, MethodName, MethodLinkageName, MethodDefUnit, MethodLine,
MethodTy, /*isLocalToUnit=*/false,
/* isDefinition=*/false, Virtuality, VIndex, ContainingType, Flags,
- CGM.getLangOpts().Optimize, nullptr, TParamsArray.get());
+ CGM.getLangOpts().Optimize, TParamsArray.get());
SPCache[Method->getCanonicalDecl()].reset(SP);
return SP;
}
-/// CollectCXXMemberFunctions - A helper function to collect debug info for
-/// C++ member functions. This is used while creating debug info entry for
-/// a Record.
void CGDebugInfo::CollectCXXMemberFunctions(
const CXXRecordDecl *RD, llvm::DIFile *Unit,
SmallVectorImpl<llvm::Metadata *> &EltTys, llvm::DIType *RecordTy) {
@@ -1168,12 +1213,13 @@
// the member being added to type units by LLVM, while still allowing it
// to be emitted into the type declaration/reference inside the compile
// unit.
+ // Ditto 'nodebug' methods, for consistency with CodeGenFunction.cpp.
// FIXME: Handle Using(Shadow?)Decls here to create
// DW_TAG_imported_declarations inside the class for base decls brought into
// derived classes. GDB doesn't seem to notice/leverage these when I tried
// it, so I'm not rushing to fix this. (GCC seems to produce them, if
// referenced)
- if (!Method || Method->isImplicit())
+ if (!Method || Method->isImplicit() || Method->hasAttr<NoDebugAttr>())
continue;
if (Method->getType()->getAs<FunctionProtoType>()->getContainedAutoType())
@@ -1192,9 +1238,6 @@
}
}
-/// CollectCXXBases - A helper function to collect debug info for
-/// C++ base classes. This is used while creating debug info entry for
-/// a Record.
void CGDebugInfo::CollectCXXBases(const CXXRecordDecl *RD, llvm::DIFile *Unit,
SmallVectorImpl<llvm::Metadata *> &EltTys,
llvm::DIType *RecordTy) {
@@ -1232,7 +1275,6 @@
}
}
-/// CollectTemplateParams - A helper function to collect template parameters.
llvm::DINodeArray
CGDebugInfo::CollectTemplateParams(const TemplateParameterList *TPList,
ArrayRef<TemplateArgument> TAList,
@@ -1268,7 +1310,7 @@
// Member function pointers have special support for building them, though
// this is currently unsupported in LLVM CodeGen.
else if ((MD = dyn_cast<CXXMethodDecl>(D)) && MD->isInstance())
- V = CGM.getCXXABI().EmitMemberPointer(MD);
+ V = CGM.getCXXABI().EmitMemberFunctionPointer(MD);
else if (const auto *FD = dyn_cast<FunctionDecl>(D))
V = CGM.GetAddrOfFunction(FD);
// Member data pointers have special handling too to compute the fixed
@@ -1337,8 +1379,6 @@
return DBuilder.getOrCreateArray(TemplateParams);
}
-/// CollectFunctionTemplateParams - A helper function to collect debug
-/// info for function template parameters.
llvm::DINodeArray
CGDebugInfo::CollectFunctionTemplateParams(const FunctionDecl *FD,
llvm::DIFile *Unit) {
@@ -1353,8 +1393,6 @@
return llvm::DINodeArray();
}
-/// CollectCXXTemplateParams - A helper function to collect debug info for
-/// template parameters.
llvm::DINodeArray CGDebugInfo::CollectCXXTemplateParams(
const ClassTemplateSpecializationDecl *TSpecial, llvm::DIFile *Unit) {
// Always get the full list of parameters, not just the ones from
@@ -1365,7 +1403,6 @@
return CollectTemplateParams(TPList, TAList.asArray(), Unit);
}
-/// getOrCreateVTablePtrType - Return debug info descriptor for vtable.
llvm::DIType *CGDebugInfo::getOrCreateVTablePtrType(llvm::DIFile *Unit) {
if (VTablePtrType)
return VTablePtrType;
@@ -1375,7 +1412,7 @@
/* Function type */
llvm::Metadata *STy = getOrCreateType(Context.IntTy, Unit);
llvm::DITypeRefArray SElements = DBuilder.getOrCreateTypeArray(STy);
- llvm::DIType *SubTy = DBuilder.createSubroutineType(Unit, SElements);
+ llvm::DIType *SubTy = DBuilder.createSubroutineType(SElements);
unsigned Size = Context.getTypeSize(Context.VoidPtrTy);
llvm::DIType *vtbl_ptr_type =
DBuilder.createPointerType(SubTy, Size, 0, "__vtbl_ptr_type");
@@ -1383,14 +1420,11 @@
return VTablePtrType;
}
-/// getVTableName - Get vtable name for the given Class.
StringRef CGDebugInfo::getVTableName(const CXXRecordDecl *RD) {
// Copy the gdb compatible name on the side and use its reference.
return internString("_vptr$", RD->getNameAsString());
}
-/// CollectVTableInfo - If the C++ class has vtable info then insert appropriate
-/// debug info entry in EltTys vector.
void CGDebugInfo::CollectVTableInfo(const CXXRecordDecl *RD, llvm::DIFile *Unit,
SmallVectorImpl<llvm::Metadata *> &EltTys) {
const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD);
@@ -1410,7 +1444,6 @@
EltTys.push_back(VPTR);
}
-/// getOrCreateRecordType - Emit record type's standalone debug info.
llvm::DIType *CGDebugInfo::getOrCreateRecordType(QualType RTy,
SourceLocation Loc) {
assert(DebugKind >= CodeGenOptions::LimitedDebugInfo);
@@ -1418,12 +1451,23 @@
return T;
}
-/// getOrCreateInterfaceType - Emit an objective c interface type standalone
-/// debug info.
llvm::DIType *CGDebugInfo::getOrCreateInterfaceType(QualType D,
SourceLocation Loc) {
+ return getOrCreateStandaloneType(D, Loc);
+}
+
+llvm::DIType *CGDebugInfo::getOrCreateStandaloneType(QualType D,
+ SourceLocation Loc) {
assert(DebugKind >= CodeGenOptions::LimitedDebugInfo);
+ assert(!D.isNull() && "null type");
llvm::DIType *T = getOrCreateType(D, getOrCreateFile(Loc));
+ assert(T && "could not create debug info for type");
+
+ // Composite types with UIDs were already retained by DIBuilder
+ // because they are only referenced by name in the IR.
+ if (auto *CTy = dyn_cast<llvm::DICompositeType>(T))
+ if (!CTy->getIdentifier().empty())
+ return T;
RetainedTypes.push_back(D.getAsOpaquePtr());
return T;
}
@@ -1455,6 +1499,9 @@
if (CXXDecl->isDynamicClass())
return;
+ if (DebugTypeExtRefs && RD->isFromASTFile())
+ return;
+
QualType Ty = CGM.getContext().getRecordType(RD);
llvm::DIType *T = getTypeOrNull(Ty);
if (T && T->isForwardDecl())
@@ -1485,8 +1532,13 @@
}
static bool shouldOmitDefinition(CodeGenOptions::DebugInfoKind DebugKind,
+ bool DebugTypeExtRefs,
const RecordDecl *RD,
const LangOptions &LangOpts) {
+ // Does the type exist in an imported clang module?
+ if (DebugTypeExtRefs && RD->isFromASTFile() && RD->getDefinition())
+ return true;
+
if (DebugKind > CodeGenOptions::LimitedDebugInfo)
return false;
@@ -1517,14 +1569,13 @@
return false;
}
-/// CreateType - get structure or union type.
llvm::DIType *CGDebugInfo::CreateType(const RecordType *Ty) {
RecordDecl *RD = Ty->getDecl();
llvm::DIType *T = cast_or_null<llvm::DIType>(getTypeOrNull(QualType(Ty, 0)));
- if (T || shouldOmitDefinition(DebugKind, RD, CGM.getLangOpts())) {
+ if (T || shouldOmitDefinition(DebugKind, DebugTypeExtRefs, RD,
+ CGM.getLangOpts())) {
if (!T)
- T = getOrCreateRecordFwdDecl(
- Ty, getContextDescriptor(cast<Decl>(RD->getDeclContext())));
+ T = getOrCreateRecordFwdDecl(Ty, getDeclContextDescriptor(RD));
return T;
}
@@ -1543,9 +1594,7 @@
// its members. Finally, we create a descriptor for the complete type (which
// may refer to the forward decl if the struct is recursive) and replace all
// uses of the forward declaration with the final definition.
-
- auto *FwdDecl =
- cast<llvm::DICompositeType>(getOrCreateLimitedType(Ty, DefUnit));
+ llvm::DICompositeType *FwdDecl = getOrCreateLimitedType(Ty, DefUnit);
const RecordDecl *D = RD->getDefinition();
if (!D || !D->isCompleteDefinition())
@@ -1591,7 +1640,6 @@
return FwdDecl;
}
-/// CreateType - get objective-c object type.
llvm::DIType *CGDebugInfo::CreateType(const ObjCObjectType *Ty,
llvm::DIFile *Unit) {
// Ignore protocols.
@@ -1622,13 +1670,18 @@
Setter->getDeclName().getObjCSelector().getNameForSlot(0);
}
-/// CreateType - get objective-c interface type.
llvm::DIType *CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
llvm::DIFile *Unit) {
ObjCInterfaceDecl *ID = Ty->getDecl();
if (!ID)
return nullptr;
+ // Return a forward declaration if this type was imported from a clang module.
+ if (DebugTypeExtRefs && ID->isFromASTFile() && ID->getDefinition())
+ return DBuilder.createForwardDecl(llvm::dwarf::DW_TAG_structure_type,
+ ID->getName(),
+ getDeclContextDescriptor(ID), Unit, 0);
+
// Get overall information about the record type for the debug info.
llvm::DIFile *DefUnit = getOrCreateFile(ID->getLocation());
unsigned Line = getLineNumber(ID->getLocation());
@@ -1639,9 +1692,10 @@
// debug type since we won't be able to lay out the entire type.
ObjCInterfaceDecl *Def = ID->getDefinition();
if (!Def || !Def->getImplementation()) {
+ llvm::DIScope *Mod = getParentModuleOrNull(ID);
llvm::DIType *FwdDecl = DBuilder.createReplaceableCompositeType(
- llvm::dwarf::DW_TAG_structure_type, ID->getName(), TheCU, DefUnit, Line,
- RuntimeLang);
+ llvm::dwarf::DW_TAG_structure_type, ID->getName(), Mod ? Mod : TheCU,
+ DefUnit, Line, RuntimeLang);
ObjCInterfaceCache.push_back(ObjCInterfaceCacheEntry(Ty, FwdDecl, Unit));
return FwdDecl;
}
@@ -1649,6 +1703,61 @@
return CreateTypeDefinition(Ty, Unit);
}
+llvm::DIModule *
+CGDebugInfo::getOrCreateModuleRef(ExternalASTSource::ASTSourceDescriptor Mod,
+ bool CreateSkeletonCU) {
+ // Use the Module pointer as the key into the cache. This is a
+ // nullptr if the "Module" is a PCH, which is safe because we don't
+ // support chained PCH debug info, so there can only be a single PCH.
+ const Module *M = Mod.getModuleOrNull();
+ auto ModRef = ModuleCache.find(M);
+ if (ModRef != ModuleCache.end())
+ return cast<llvm::DIModule>(ModRef->second);
+
+ // Macro definitions that were defined with "-D" on the command line.
+ SmallString<128> ConfigMacros;
+ {
+ llvm::raw_svector_ostream OS(ConfigMacros);
+ const auto &PPOpts = CGM.getPreprocessorOpts();
+ unsigned I = 0;
+ // Translate the macro definitions back into a commmand line.
+ for (auto &M : PPOpts.Macros) {
+ if (++I > 1)
+ OS << " ";
+ const std::string &Macro = M.first;
+ bool Undef = M.second;
+ OS << "\"-" << (Undef ? 'U' : 'D');
+ for (char c : Macro)
+ switch (c) {
+ case '\\' : OS << "\\\\"; break;
+ case '"' : OS << "\\\""; break;
+ default: OS << c;
+ }
+ OS << '\"';
+ }
+ }
+
+ bool IsRootModule = M ? !M->Parent : true;
+ if (CreateSkeletonCU && IsRootModule) {
+ llvm::DIBuilder DIB(CGM.getModule());
+ DIB.createCompileUnit(TheCU->getSourceLanguage(), Mod.getModuleName(),
+ Mod.getPath(), TheCU->getProducer(), true,
+ StringRef(), 0, Mod.getASTFile(),
+ llvm::DIBuilder::FullDebug, Mod.getSignature());
+ DIB.finalize();
+ }
+ llvm::DIModule *Parent =
+ IsRootModule ? nullptr
+ : getOrCreateModuleRef(
+ ExternalASTSource::ASTSourceDescriptor(*M->Parent),
+ CreateSkeletonCU);
+ llvm::DIModule *DIMod =
+ DBuilder.createModule(Parent, Mod.getModuleName(), ConfigMacros,
+ Mod.getPath(), CGM.getHeaderSearchOpts().Sysroot);
+ ModuleCache[M].reset(DIMod);
+ return DIMod;
+}
+
llvm::DIType *CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty,
llvm::DIFile *Unit) {
ObjCInterfaceDecl *ID = Ty->getDecl();
@@ -1664,9 +1773,10 @@
if (ID->getImplementation())
Flags |= llvm::DINode::FlagObjcClassComplete;
+ llvm::DIScope *Mod = getParentModuleOrNull(ID);
llvm::DICompositeType *RealDecl = DBuilder.createStructType(
- Unit, ID->getName(), DefUnit, Line, Size, Align, Flags, nullptr,
- llvm::DINodeArray(), RuntimeLang);
+ Mod ? Mod : Unit, ID->getName(), DefUnit, Line, Size, Align, Flags,
+ nullptr, llvm::DINodeArray(), RuntimeLang);
QualType QTy(Ty, 0);
TypeCache[QTy.getAsOpaquePtr()].reset(RealDecl);
@@ -1690,7 +1800,7 @@
}
// Create entries for all of the properties.
- for (const auto *PD : ID->properties()) {
+ auto AddProperty = [&](const ObjCPropertyDecl *PD) {
SourceLocation Loc = PD->getLocation();
llvm::DIFile *PUnit = getOrCreateFile(Loc);
unsigned PLine = getLineNumber(Loc);
@@ -1704,6 +1814,21 @@
: getSelectorName(PD->getSetterName()),
PD->getPropertyAttributes(), getOrCreateType(PD->getType(), PUnit));
EltTys.push_back(PropertyNode);
+ };
+ {
+ llvm::SmallPtrSet<const IdentifierInfo*, 16> PropertySet;
+ for (const ObjCCategoryDecl *ClassExt : ID->known_extensions())
+ for (auto *PD : ClassExt->properties()) {
+ PropertySet.insert(PD->getIdentifier());
+ AddProperty(PD);
+ }
+ for (const auto *PD : ID->properties()) {
+ // Don't emit duplicate metadata for properties that were already in a
+ // class extension.
+ if (!PropertySet.insert(PD->getIdentifier()).second)
+ continue;
+ AddProperty(PD);
+ }
}
const ASTRecordLayout &RL = CGM.getContext().getASTObjCInterfaceLayout(ID);
@@ -1878,9 +2003,8 @@
llvm::DIType *CGDebugInfo::CreateType(const MemberPointerType *Ty,
llvm::DIFile *U) {
- uint64_t Size = CGM.getCXXABI().isTypeInfoCalculable(QualType(Ty, 0))
- ? CGM.getContext().getTypeSize(Ty)
- : 0;
+ uint64_t Size =
+ !Ty->isIncompleteType() ? CGM.getContext().getTypeSize(Ty) : 0;
llvm::DIType *ClassType = getOrCreateType(QualType(Ty->getClass(), 0), U);
if (Ty->isMemberDataPointerType())
return DBuilder.createMemberPointerType(
@@ -1901,9 +2025,9 @@
return getOrCreateType(Ty->getValueType(), U);
}
-/// CreateEnumType - get enumeration type.
llvm::DIType *CGDebugInfo::CreateEnumType(const EnumType *Ty) {
const EnumDecl *ED = Ty->getDecl();
+
uint64_t Size = 0;
uint64_t Align = 0;
if (!ED->getTypeForDecl()->isIncompleteType()) {
@@ -1913,11 +2037,13 @@
SmallString<256> FullName = getUniqueTagTypeName(Ty, CGM, TheCU);
+ bool isImportedFromModule =
+ DebugTypeExtRefs && ED->isFromASTFile() && ED->getDefinition();
+
// If this is just a forward declaration, construct an appropriately
// marked node and just return it.
- if (!ED->getDefinition()) {
- llvm::DIScope *EDContext =
- getContextDescriptor(cast<Decl>(ED->getDeclContext()));
+ if (isImportedFromModule || !ED->getDefinition()) {
+ llvm::DIScope *EDContext = getDeclContextDescriptor(ED);
llvm::DIFile *DefUnit = getOrCreateFile(ED->getLocation());
unsigned Line = getLineNumber(ED->getLocation());
StringRef EDName = ED->getName();
@@ -1957,8 +2083,7 @@
llvm::DIFile *DefUnit = getOrCreateFile(ED->getLocation());
unsigned Line = getLineNumber(ED->getLocation());
- llvm::DIScope *EnumContext =
- getContextDescriptor(cast<Decl>(ED->getDeclContext()));
+ llvm::DIScope *EnumContext = getDeclContextDescriptor(ED);
llvm::DIType *ClassTy =
ED->isFixed() ? getOrCreateType(ED->getIntegerType(), DefUnit) : nullptr;
return DBuilder.createEnumerationType(EnumContext, ED->getName(), DefUnit,
@@ -2021,8 +2146,6 @@
} while (true);
}
-/// getType - Get the type from the cache or return null type if it doesn't
-/// exist.
llvm::DIType *CGDebugInfo::getTypeOrNull(QualType Ty) {
// Unwrap the type as needed for debug information.
@@ -2049,8 +2172,6 @@
RetainedTypes.push_back(CGM.getContext().getRecordType(&SD).getAsOpaquePtr());
}
-/// getOrCreateType - Get the type from the cache or create a new
-/// one if necessary.
llvm::DIType *CGDebugInfo::getOrCreateType(QualType Ty, llvm::DIFile *Unit) {
if (Ty.isNull())
return nullptr;
@@ -2061,9 +2182,8 @@
if (auto *T = getTypeOrNull(Ty))
return T;
- // Otherwise create the type.
llvm::DIType *Res = CreateTypeNode(Ty, Unit);
- void *TyPtr = Ty.getAsOpaquePtr();
+ void* TyPtr = Ty.getAsOpaquePtr();
// And update the type cache.
TypeCache[TyPtr].reset(Res);
@@ -2071,33 +2191,38 @@
return Res;
}
-/// Currently the checksum of an interface includes the number of
-/// ivars and property accessors.
-unsigned CGDebugInfo::Checksum(const ObjCInterfaceDecl *ID) {
- // The assumption is that the number of ivars can only increase
- // monotonically, so it is safe to just use their current number as
- // a checksum.
- unsigned Sum = 0;
- for (const ObjCIvarDecl *Ivar = ID->all_declared_ivar_begin();
- Ivar != nullptr; Ivar = Ivar->getNextIvar())
- ++Sum;
-
- return Sum;
-}
-
-ObjCInterfaceDecl *CGDebugInfo::getObjCInterfaceDecl(QualType Ty) {
- switch (Ty->getTypeClass()) {
- case Type::ObjCObjectPointer:
- return getObjCInterfaceDecl(
- cast<ObjCObjectPointerType>(Ty)->getPointeeType());
- case Type::ObjCInterface:
- return cast<ObjCInterfaceType>(Ty)->getDecl();
- default:
+llvm::DIModule *CGDebugInfo::getParentModuleOrNull(const Decl *D) {
+ // A forward declaration inside a module header does not belong to the module.
+ if (isa<RecordDecl>(D) && !cast<RecordDecl>(D)->getDefinition())
return nullptr;
+ if (DebugTypeExtRefs && D->isFromASTFile()) {
+ // Record a reference to an imported clang module or precompiled header.
+ auto *Reader = CGM.getContext().getExternalSource();
+ auto Idx = D->getOwningModuleID();
+ auto Info = Reader->getSourceDescriptor(Idx);
+ if (Info)
+ return getOrCreateModuleRef(*Info, /*SkeletonCU=*/true);
+ } else if (ClangModuleMap) {
+ // We are building a clang module or a precompiled header.
+ //
+ // TODO: When D is a CXXRecordDecl or a C++ Enum, the ODR applies
+ // and it wouldn't be necessary to specify the parent scope
+ // because the type is already unique by definition (it would look
+ // like the output of -fno-standalone-debug). On the other hand,
+ // the parent scope helps a consumer to quickly locate the object
+ // file where the type's definition is located, so it might be
+ // best to make this behavior a command line or debugger tuning
+ // option.
+ FullSourceLoc Loc(D->getLocation(), CGM.getContext().getSourceManager());
+ if (Module *M = ClangModuleMap->inferModuleFromLocation(Loc)) {
+ auto Info = ExternalASTSource::ASTSourceDescriptor(*M);
+ return getOrCreateModuleRef(Info, /*SkeletonCU=*/false);
+ }
}
+
+ return nullptr;
}
-/// CreateTypeNode - Create a new debug type node.
llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit) {
// Handle qualifiers, which recursively handles what they refer to.
if (Ty.hasLocalQualifiers())
@@ -2178,13 +2303,11 @@
llvm_unreachable("type should have been unwrapped!");
}
-/// getOrCreateLimitedType - Get the type from the cache or create a new
-/// limited type if necessary.
-llvm::DIType *CGDebugInfo::getOrCreateLimitedType(const RecordType *Ty,
- llvm::DIFile *Unit) {
+llvm::DICompositeType *CGDebugInfo::getOrCreateLimitedType(const RecordType *Ty,
+ llvm::DIFile *Unit) {
QualType QTy(Ty, 0);
- auto *T = cast_or_null<llvm::DICompositeTypeBase>(getTypeOrNull(QTy));
+ auto *T = cast_or_null<llvm::DICompositeType>(getTypeOrNull(QTy));
// We may have cached a forward decl when we could have created
// a non-forward decl. Go ahead and create a non-forward decl
@@ -2214,8 +2337,7 @@
unsigned Line = getLineNumber(RD->getLocation());
StringRef RDName = getClassName(RD);
- llvm::DIScope *RDContext =
- getContextDescriptor(cast<Decl>(RD->getDeclContext()));
+ llvm::DIScope *RDContext = getDeclContextDescriptor(RD);
// If we ended up creating the type during the context chain construction,
// just return that.
@@ -2273,7 +2395,6 @@
DBuilder.replaceVTableHolder(RealDecl, ContainingType);
}
-/// CreateMemberType - Create new member and increase Offset by FType's size.
llvm::DIType *CGDebugInfo::CreateMemberType(llvm::DIFile *Unit, QualType FType,
StringRef Name, uint64_t *Offset) {
llvm::DIType *FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
@@ -2312,8 +2433,10 @@
dyn_cast_or_null<NamespaceDecl>(FD->getDeclContext()))
FDContext = getOrCreateNameSpace(NSDecl);
else if (const RecordDecl *RDecl =
- dyn_cast_or_null<RecordDecl>(FD->getDeclContext()))
- FDContext = getContextDescriptor(cast<Decl>(RDecl));
+ dyn_cast_or_null<RecordDecl>(FD->getDeclContext())) {
+ llvm::DIScope *Mod = getParentModuleOrNull(RDecl);
+ FDContext = getContextDescriptor(RDecl, Mod ? Mod : TheCU);
+ }
// Collect template parameters.
TParamsArray = CollectFunctionTemplateParams(FD, Unit);
}
@@ -2361,7 +2484,9 @@
// outside the class by putting it in the global scope.
if (DC->isRecord())
DC = CGM.getContext().getTranslationUnitDecl();
- VDContext = getContextDescriptor(dyn_cast<Decl>(DC));
+
+ llvm::DIScope *Mod = getParentModuleOrNull(VD);
+ VDContext = getContextDescriptor(cast<Decl>(DC), Mod ? Mod : TheCU);
}
llvm::DISubprogram *
@@ -2386,7 +2511,7 @@
llvm::DISubprogram *SP = DBuilder.createTempFunctionFwdDecl(
DContext, Name, LinkageName, Unit, Line,
getOrCreateFunctionType(FD, FnType, Unit), !FD->isExternallyVisible(),
- false /*declaration*/, 0, Flags, CGM.getLangOpts().Optimize, nullptr,
+ /* isDefinition = */ false, 0, Flags, CGM.getLangOpts().Optimize,
TParamsArray.get(), getFunctionDeclaration(FD));
const FunctionDecl *CanonDecl = cast<FunctionDecl>(FD->getCanonicalDecl());
FwdDeclReplaceMap.emplace_back(std::piecewise_construct,
@@ -2438,8 +2563,6 @@
return nullptr;
}
-/// getFunctionDeclaration - Return debug info descriptor to describe method
-/// declaration for the given method definition.
llvm::DISubprogram *CGDebugInfo::getFunctionDeclaration(const Decl *D) {
if (!D || DebugKind <= CodeGenOptions::DebugLineTablesOnly)
return nullptr;
@@ -2449,7 +2572,7 @@
return nullptr;
// Setup context.
- auto *S = getContextDescriptor(cast<Decl>(D->getDeclContext()));
+ auto *S = getDeclContextDescriptor(D);
auto MI = SPCache.find(FD->getCanonicalDecl());
if (MI == SPCache.end()) {
@@ -2484,8 +2607,7 @@
if (!D || DebugKind <= CodeGenOptions::DebugLineTablesOnly)
// Create fake but valid subroutine type. Otherwise -verify would fail, and
// subprogram DIE will miss DW_AT_decl_file and DW_AT_decl_line fields.
- return DBuilder.createSubroutineType(F,
- DBuilder.getOrCreateTypeArray(None));
+ return DBuilder.createSubroutineType(DBuilder.getOrCreateTypeArray(None));
if (const CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(D))
return getOrCreateMethodType(Method, F);
@@ -2503,11 +2625,17 @@
Elts.push_back(getOrCreateType(ResultTy, F));
// "self" pointer is always first argument.
- QualType SelfDeclTy = OMethod->getSelfDecl()->getType();
- Elts.push_back(CreateSelfType(SelfDeclTy, getOrCreateType(SelfDeclTy, F)));
+ QualType SelfDeclTy;
+ if (auto *SelfDecl = OMethod->getSelfDecl())
+ SelfDeclTy = SelfDecl->getType();
+ else if (auto *FPT = dyn_cast<FunctionProtoType>(FnType))
+ if (FPT->getNumParams() > 1)
+ SelfDeclTy = FPT->getParamType(0);
+ if (!SelfDeclTy.isNull())
+ Elts.push_back(CreateSelfType(SelfDeclTy, getOrCreateType(SelfDeclTy, F)));
// "_cmd" pointer is always second argument.
Elts.push_back(DBuilder.createArtificialType(
- getOrCreateType(OMethod->getCmdDecl()->getType(), F)));
+ getOrCreateType(CGM.getContext().getObjCSelType(), F)));
// Get rest of the arguments.
for (const auto *PI : OMethod->params())
Elts.push_back(getOrCreateType(PI->getType(), F));
@@ -2516,7 +2644,7 @@
Elts.push_back(DBuilder.createUnspecifiedParameter());
llvm::DITypeRefArray EltTypeArray = DBuilder.getOrCreateTypeArray(Elts);
- return DBuilder.createSubroutineType(F, EltTypeArray);
+ return DBuilder.createSubroutineType(EltTypeArray);
}
// Handle variadic function types; they need an additional
@@ -2530,13 +2658,12 @@
EltTys.push_back(getOrCreateType(FPT->getParamType(i), F));
EltTys.push_back(DBuilder.createUnspecifiedParameter());
llvm::DITypeRefArray EltTypeArray = DBuilder.getOrCreateTypeArray(EltTys);
- return DBuilder.createSubroutineType(F, EltTypeArray);
+ return DBuilder.createSubroutineType(EltTypeArray);
}
return cast<llvm::DISubroutineType>(getOrCreateType(FnType, F));
}
-/// EmitFunctionStart - Constructs the debug code for entering a function.
void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, SourceLocation Loc,
SourceLocation ScopeLoc, QualType FnType,
llvm::Function *Fn, CGBuilderTy &Builder) {
@@ -2597,8 +2724,9 @@
llvm::DISubprogram *SP = DBuilder.createFunction(
FDContext, Name, LinkageName, Unit, LineNo,
getOrCreateFunctionType(D, FnType, Unit), Fn->hasInternalLinkage(),
- true /*definition*/, ScopeLine, Flags, CGM.getLangOpts().Optimize, Fn,
+ true /*definition*/, ScopeLine, Flags, CGM.getLangOpts().Optimize,
TParamsArray.get(), getFunctionDeclaration(D));
+ Fn->setSubprogram(SP);
// We might get here with a VarDecl in the case we're generating
// code for the initialization of globals. Do not record these decls
// as they will overwrite the actual VarDecl Decl in the cache.
@@ -2612,9 +2740,48 @@
RegionMap[D].reset(SP);
}
-/// EmitLocation - Emit metadata to indicate a change in line/column
-/// information in the source file. If the location is invalid, the
-/// previous location will be reused.
+void CGDebugInfo::EmitFunctionDecl(GlobalDecl GD, SourceLocation Loc,
+ QualType FnType) {
+ StringRef Name;
+ StringRef LinkageName;
+
+ const Decl *D = GD.getDecl();
+ if (!D)
+ return;
+
+ unsigned Flags = 0;
+ llvm::DIFile *Unit = getOrCreateFile(Loc);
+ llvm::DIScope *FDContext = getDeclContextDescriptor(D);
+ llvm::DINodeArray TParamsArray;
+ if (isa<FunctionDecl>(D)) {
+ // If there is a DISubprogram for this function available then use it.
+ collectFunctionDeclProps(GD, Unit, Name, LinkageName, FDContext,
+ TParamsArray, Flags);
+ } else if (const ObjCMethodDecl *OMD = dyn_cast<ObjCMethodDecl>(D)) {
+ Name = getObjCMethodName(OMD);
+ Flags |= llvm::DINode::FlagPrototyped;
+ } else {
+ llvm_unreachable("not a function or ObjC method");
+ }
+ if (!Name.empty() && Name[0] == '\01')
+ Name = Name.substr(1);
+
+ if (D->isImplicit()) {
+ Flags |= llvm::DINode::FlagArtificial;
+ // Artificial functions without a location should not silently reuse CurLoc.
+ if (Loc.isInvalid())
+ CurLoc = SourceLocation();
+ }
+ unsigned LineNo = getLineNumber(Loc);
+ unsigned ScopeLine = 0;
+
+ DBuilder.createFunction(FDContext, Name, LinkageName, Unit, LineNo,
+ getOrCreateFunctionType(D, FnType, Unit),
+ false /*internalLinkage*/, true /*definition*/,
+ ScopeLine, Flags, CGM.getLangOpts().Optimize,
+ TParamsArray.get(), getFunctionDeclaration(D));
+}
+
void CGDebugInfo::EmitLocation(CGBuilderTy &Builder, SourceLocation Loc) {
// Update our current location
setLocation(Loc);
@@ -2627,8 +2794,6 @@
getLineNumber(CurLoc), getColumnNumber(CurLoc), Scope));
}
-/// CreateLexicalBlock - Creates a new lexical block node and pushes it on
-/// the stack.
void CGDebugInfo::CreateLexicalBlock(SourceLocation Loc) {
llvm::MDNode *Back = nullptr;
if (!LexicalBlockStack.empty())
@@ -2638,8 +2803,6 @@
getColumnNumber(CurLoc)));
}
-/// EmitLexicalBlockStart - Constructs the debug code for entering a declarative
-/// region - beginning of a DW_TAG_lexical_block.
void CGDebugInfo::EmitLexicalBlockStart(CGBuilderTy &Builder,
SourceLocation Loc) {
// Set our current location.
@@ -2656,8 +2819,6 @@
CreateLexicalBlock(Loc);
}
-/// EmitLexicalBlockEnd - Constructs the debug code for exiting a declarative
-/// region - end of a DW_TAG_lexical_block.
void CGDebugInfo::EmitLexicalBlockEnd(CGBuilderTy &Builder,
SourceLocation Loc) {
assert(!LexicalBlockStack.empty() && "Region stack mismatch, stack empty!");
@@ -2671,7 +2832,6 @@
LexicalBlockStack.pop_back();
}
-/// EmitFunctionEnd - Constructs the debug code for exiting a function.
void CGDebugInfo::EmitFunctionEnd(CGBuilderTy &Builder) {
assert(!LexicalBlockStack.empty() && "Region stack mismatch, stack empty!");
unsigned RCount = FnBeginRegionCount.back();
@@ -2686,8 +2846,6 @@
FnBeginRegionCount.pop_back();
}
-// EmitTypeForVarWithBlocksAttr - Build up structure info for the byref.
-// See BuildByRefType.
llvm::DIType *CGDebugInfo::EmitTypeForVarWithBlocksAttr(const VarDecl *VD,
uint64_t *XOffset) {
@@ -2761,9 +2919,8 @@
nullptr, Elements);
}
-/// EmitDeclare - Emit local variable declaration debug info.
-void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::dwarf::Tag Tag,
- llvm::Value *Storage, unsigned ArgNo,
+void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::Value *Storage,
+ llvm::Optional<unsigned> ArgNo,
CGBuilderTy &Builder) {
assert(DebugKind >= CodeGenOptions::LimitedDebugInfo);
assert(!LexicalBlockStack.empty() && "Region stack mismatch, stack empty!");
@@ -2802,7 +2959,7 @@
// FIXME: There has to be a better way to do this, but for static
// functions there won't be an implicit param at arg1 and
// otherwise it is 'self' or 'this'.
- if (isa<ImplicitParamDecl>(VD) && ArgNo == 1)
+ if (isa<ImplicitParamDecl>(VD) && ArgNo && *ArgNo == 1)
Flags |= llvm::DINode::FlagObjectPointer;
if (llvm::Argument *Arg = dyn_cast<llvm::Argument>(Storage))
if (Arg->getType()->isPointerTy() && !Arg->hasByValAttr() &&
@@ -2827,8 +2984,11 @@
Expr.push_back(offset.getQuantity());
// Create the descriptor for the variable.
- auto *D = DBuilder.createLocalVariable(Tag, Scope, VD->getName(), Unit,
- Line, Ty, ArgNo);
+ auto *D = ArgNo
+ ? DBuilder.createParameterVariable(Scope, VD->getName(),
+ *ArgNo, Unit, Line, Ty)
+ : DBuilder.createAutoVariable(Scope, VD->getName(), Unit,
+ Line, Ty);
// Insert an llvm.dbg.declare into the current block.
DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(Expr),
@@ -2858,10 +3018,9 @@
continue;
// Use VarDecl's Tag, Scope and Line number.
- auto *D = DBuilder.createLocalVariable(
- Tag, Scope, FieldName, Unit, Line, FieldTy,
- CGM.getLangOpts().Optimize, Flags | llvm::DINode::FlagArtificial,
- ArgNo);
+ auto *D = DBuilder.createAutoVariable(
+ Scope, FieldName, Unit, Line, FieldTy, CGM.getLangOpts().Optimize,
+ Flags | llvm::DINode::FlagArtificial);
// Insert an llvm.dbg.declare into the current block.
DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(Expr),
@@ -2873,8 +3032,12 @@
// Create the descriptor for the variable.
auto *D =
- DBuilder.createLocalVariable(Tag, Scope, Name, Unit, Line, Ty,
- CGM.getLangOpts().Optimize, Flags, ArgNo);
+ ArgNo
+ ? DBuilder.createParameterVariable(Scope, Name, *ArgNo, Unit, Line,
+ Ty, CGM.getLangOpts().Optimize,
+ Flags)
+ : DBuilder.createAutoVariable(Scope, Name, Unit, Line, Ty,
+ CGM.getLangOpts().Optimize, Flags);
// Insert an llvm.dbg.declare into the current block.
DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(Expr),
@@ -2886,15 +3049,9 @@
llvm::Value *Storage,
CGBuilderTy &Builder) {
assert(DebugKind >= CodeGenOptions::LimitedDebugInfo);
- EmitDeclare(VD, llvm::dwarf::DW_TAG_auto_variable, Storage, 0, Builder);
+ EmitDeclare(VD, Storage, llvm::None, Builder);
}
-/// Look up the completed type for a self pointer in the TypeCache and
-/// create a copy of it with the ObjectPointer and Artificial flags
-/// set. If the type is not cached, a new one is created. This should
-/// never happen though, since creating a type for the implicit self
-/// argument implies that we already parsed the interface definition
-/// and the ivar declarations in the implementation.
llvm::DIType *CGDebugInfo::CreateSelfType(const QualType &QualTy,
llvm::DIType *Ty) {
llvm::DIType *CachedTy = getTypeOrNull(QualTy);
@@ -2957,8 +3114,7 @@
}
// Create the descriptor for the variable.
- auto *D = DBuilder.createLocalVariable(
- llvm::dwarf::DW_TAG_auto_variable,
+ auto *D = DBuilder.createAutoVariable(
cast<llvm::DILocalScope>(LexicalBlockStack.back()), VD->getName(), Unit,
Line, Ty);
@@ -2972,13 +3128,11 @@
Builder.GetInsertBlock());
}
-/// EmitDeclareOfArgVariable - Emit call to llvm.dbg.declare for an argument
-/// variable declaration.
void CGDebugInfo::EmitDeclareOfArgVariable(const VarDecl *VD, llvm::Value *AI,
unsigned ArgNo,
CGBuilderTy &Builder) {
assert(DebugKind >= CodeGenOptions::LimitedDebugInfo);
- EmitDeclare(VD, llvm::dwarf::DW_TAG_arg_variable, AI, ArgNo, Builder);
+ EmitDeclare(VD, AI, ArgNo, Builder);
}
namespace {
@@ -3007,7 +3161,7 @@
unsigned column = getColumnNumber(loc);
// Build the debug-info type for the block literal.
- getContextDescriptor(cast<Decl>(blockDecl->getDeclContext()));
+ getDeclContextDescriptor(blockDecl);
const llvm::StructLayout *blockLayout =
CGM.getDataLayout().getStructLayout(block.StructureType);
@@ -3120,9 +3274,9 @@
auto *scope = cast<llvm::DILocalScope>(LexicalBlockStack.back());
// Create the descriptor for the parameter.
- auto *debugVar = DBuilder.createLocalVariable(
- llvm::dwarf::DW_TAG_arg_variable, scope, Arg->getName(), tunit, line,
- type, CGM.getLangOpts().Optimize, flags, ArgNo);
+ auto *debugVar = DBuilder.createParameterVariable(
+ scope, Arg->getName(), ArgNo, tunit, line, type,
+ CGM.getLangOpts().Optimize, flags);
if (LocalAddr) {
// Insert an llvm.dbg.value into the current block.
@@ -3137,8 +3291,6 @@
Builder.GetInsertBlock());
}
-/// If D is an out-of-class definition of a static data member of a class, find
-/// its corresponding in-class declaration.
llvm::DIDerivedType *
CGDebugInfo::getOrCreateStaticDataMemberDeclarationOrNull(const VarDecl *D) {
if (!D->isStaticDataMember())
@@ -3147,20 +3299,16 @@
auto MI = StaticDataMemberCache.find(D->getCanonicalDecl());
if (MI != StaticDataMemberCache.end()) {
assert(MI->second && "Static data member declaration should still exist");
- return cast<llvm::DIDerivedType>(MI->second);
+ return MI->second;
}
// If the member wasn't found in the cache, lazily construct and add it to the
// type (used when a limited form of the type is emitted).
auto DC = D->getDeclContext();
- auto *Ctxt =
- cast<llvm::DICompositeType>(getContextDescriptor(cast<Decl>(DC)));
+ auto *Ctxt = cast<llvm::DICompositeType>(getDeclContextDescriptor(D));
return CreateRecordStaticField(D, Ctxt, cast<RecordDecl>(DC));
}
-/// Recursively collect all of the member fields of a global anonymous decl and
-/// create static variables for them. The first time this is called it needs
-/// to be on a union and then from there we can have additional unnamed fields.
llvm::DIGlobalVariable *CGDebugInfo::CollectAnonRecordDecls(
const RecordDecl *RD, llvm::DIFile *Unit, unsigned LineNo,
StringRef LinkageName, llvm::GlobalVariable *Var, llvm::DIScope *DContext) {
@@ -3186,7 +3334,6 @@
return GV;
}
-/// EmitGlobalVariable - Emit information about a global variable.
void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var,
const VarDecl *D) {
assert(DebugKind >= CodeGenOptions::LimitedDebugInfo);
@@ -3206,7 +3353,7 @@
// variable for each member of the anonymous union so that it's possible
// to find the name of any field in the union.
if (T->isUnionType() && DeclName.empty()) {
- const RecordDecl *RD = cast<RecordType>(T)->getDecl();
+ const RecordDecl *RD = T->castAs<RecordType>()->getDecl();
assert(RD->isAnonymousStructOrUnion() &&
"unnamed non-anonymous struct or union?");
GV = CollectAnonRecordDecls(RD, Unit, LineNo, LinkageName, Var, DContext);
@@ -3219,7 +3366,6 @@
DeclCache[D->getCanonicalDecl()].reset(static_cast<llvm::Metadata *>(GV));
}
-/// EmitGlobalVariable - Emit global variable's debug info.
void CGDebugInfo::EmitGlobalVariable(const ValueDecl *VD,
llvm::Constant *Init) {
assert(DebugKind >= CodeGenOptions::LimitedDebugInfo);
@@ -3244,15 +3390,14 @@
auto *VarD = cast<VarDecl>(VD);
if (VarD->isStaticDataMember()) {
auto *RD = cast<RecordDecl>(VarD->getDeclContext());
- getContextDescriptor(RD);
+ getDeclContextDescriptor(VarD);
// Ensure that the type is retained even though it's otherwise unreferenced.
RetainedTypes.push_back(
CGM.getContext().getRecordType(RD).getAsOpaquePtr());
return;
}
- llvm::DIScope *DContext =
- getContextDescriptor(dyn_cast<Decl>(VD->getDeclContext()));
+ llvm::DIScope *DContext = getDeclContextDescriptor(VD);
auto &GV = DeclCache[VD];
if (GV)
@@ -3265,16 +3410,21 @@
llvm::DIScope *CGDebugInfo::getCurrentContextDescriptor(const Decl *D) {
if (!LexicalBlockStack.empty())
return LexicalBlockStack.back();
- return getContextDescriptor(D);
+ llvm::DIScope *Mod = getParentModuleOrNull(D);
+ return getContextDescriptor(D, Mod ? Mod : TheCU);
}
void CGDebugInfo::EmitUsingDirective(const UsingDirectiveDecl &UD) {
if (CGM.getCodeGenOpts().getDebugInfo() < CodeGenOptions::LimitedDebugInfo)
return;
- DBuilder.createImportedModule(
- getCurrentContextDescriptor(cast<Decl>(UD.getDeclContext())),
- getOrCreateNameSpace(UD.getNominatedNamespace()),
- getLineNumber(UD.getLocation()));
+ const NamespaceDecl *NSDecl = UD.getNominatedNamespace();
+ if (!NSDecl->isAnonymousNamespace() ||
+ CGM.getCodeGenOpts().DebugExplicitImport) {
+ DBuilder.createImportedModule(
+ getCurrentContextDescriptor(cast<Decl>(UD.getDeclContext())),
+ getOrCreateNameSpace(NSDecl),
+ getLineNumber(UD.getLocation()));
+ }
}
void CGDebugInfo::EmitUsingDecl(const UsingDecl &UD) {
@@ -3292,6 +3442,16 @@
getLineNumber(USD.getLocation()));
}
+void CGDebugInfo::EmitImportDecl(const ImportDecl &ID) {
+ if (Module *M = ID.getImportedModule()) {
+ auto Info = ExternalASTSource::ASTSourceDescriptor(*M);
+ DBuilder.createImportedDeclaration(
+ getCurrentContextDescriptor(cast<Decl>(ID.getDeclContext())),
+ getOrCreateModuleRef(Info, DebugTypeExtRefs),
+ getLineNumber(ID.getLocation()));
+ }
+}
+
llvm::DIImportedEntity *
CGDebugInfo::EmitNamespaceAlias(const NamespaceAliasDecl &NA) {
if (CGM.getCodeGenOpts().getDebugInfo() < CodeGenOptions::LimitedDebugInfo)
@@ -3316,8 +3476,6 @@
return R;
}
-/// getOrCreateNamesSpace - Return namespace descriptor for the given
-/// namespace decl.
llvm::DINamespace *
CGDebugInfo::getOrCreateNameSpace(const NamespaceDecl *NSDecl) {
NSDecl = NSDecl->getCanonicalDecl();
@@ -3327,14 +3485,19 @@
unsigned LineNo = getLineNumber(NSDecl->getLocation());
llvm::DIFile *FileD = getOrCreateFile(NSDecl->getLocation());
- llvm::DIScope *Context =
- getContextDescriptor(dyn_cast<Decl>(NSDecl->getDeclContext()));
+ llvm::DIScope *Context = getDeclContextDescriptor(NSDecl);
llvm::DINamespace *NS =
DBuilder.createNameSpace(Context, NSDecl->getName(), FileD, LineNo);
NameSpaceCache[NSDecl].reset(NS);
return NS;
}
+void CGDebugInfo::setDwoId(uint64_t Signature) {
+ assert(TheCU && "no main compile unit");
+ TheCU->setDWOId(Signature);
+}
+
+
void CGDebugInfo::finalize() {
// Creating types might create further types - invalidating the current
// element and the size(), so don't cache/reference them.
@@ -3378,9 +3541,9 @@
// We keep our own list of retained types, because we need to look
// up the final type in the type cache.
- for (std::vector<void *>::const_iterator RI = RetainedTypes.begin(),
- RE = RetainedTypes.end(); RI != RE; ++RI)
- DBuilder.retainType(cast<llvm::DIType>(TypeCache[*RI]));
+ for (auto &RT : RetainedTypes)
+ if (auto MD = TypeCache[RT])
+ DBuilder.retainType(cast<llvm::DIType>(MD));
DBuilder.finalize();
}
diff --git a/lib/CodeGen/CGDebugInfo.h b/lib/CodeGen/CGDebugInfo.h
index 8509e07..57d5c80 100644
--- a/lib/CodeGen/CGDebugInfo.h
+++ b/lib/CodeGen/CGDebugInfo.h
@@ -20,39 +20,43 @@
#include "clang/Basic/SourceLocation.h"
#include "clang/Frontend/CodeGenOptions.h"
#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/Optional.h"
#include "llvm/IR/DIBuilder.h"
#include "llvm/IR/DebugInfo.h"
#include "llvm/IR/ValueHandle.h"
#include "llvm/Support/Allocator.h"
namespace llvm {
- class MDNode;
+class MDNode;
}
namespace clang {
- class CXXMethodDecl;
- class VarDecl;
- class ObjCInterfaceDecl;
- class ObjCIvarDecl;
- class ClassTemplateSpecializationDecl;
- class GlobalDecl;
- class UsingDecl;
+class CXXMethodDecl;
+class ClassTemplateSpecializationDecl;
+class GlobalDecl;
+class ModuleMap;
+class ObjCInterfaceDecl;
+class ObjCIvarDecl;
+class UsingDecl;
+class VarDecl;
namespace CodeGen {
- class CodeGenModule;
- class CodeGenFunction;
- class CGBlockInfo;
+class CodeGenModule;
+class CodeGenFunction;
+class CGBlockInfo;
-/// \brief This class gathers all debug information during compilation
-/// and is responsible for emitting to llvm globals or pass directly to
-/// the backend.
+/// This class gathers all debug information during compilation and is
+/// responsible for emitting to llvm globals or pass directly to the
+/// backend.
class CGDebugInfo {
friend class ApplyDebugLocation;
friend class SaveAndRestoreLocation;
CodeGenModule &CGM;
const CodeGenOptions::DebugInfoKind DebugKind;
+ bool DebugTypeExtRefs;
llvm::DIBuilder DBuilder;
llvm::DICompileUnit *TheCU = nullptr;
+ ModuleMap *ClangModuleMap = nullptr;
SourceLocation CurLoc;
llvm::DIType *VTablePtrType = nullptr;
llvm::DIType *ClassTy = nullptr;
@@ -63,13 +67,24 @@
llvm::DIType *OCLImage1dBufferDITy = nullptr;
llvm::DIType *OCLImage2dDITy = nullptr;
llvm::DIType *OCLImage2dArrayDITy = nullptr;
+ llvm::DIType *OCLImage2dDepthDITy = nullptr;
+ llvm::DIType *OCLImage2dArrayDepthDITy = nullptr;
+ llvm::DIType *OCLImage2dMSAADITy = nullptr;
+ llvm::DIType *OCLImage2dArrayMSAADITy = nullptr;
+ llvm::DIType *OCLImage2dMSAADepthDITy = nullptr;
+ llvm::DIType *OCLImage2dArrayMSAADepthDITy = nullptr;
llvm::DIType *OCLImage3dDITy = nullptr;
llvm::DIType *OCLEventDITy = nullptr;
- llvm::DIType *BlockLiteralGeneric = nullptr;
+ llvm::DIType *OCLClkEventDITy = nullptr;
+ llvm::DIType *OCLQueueDITy = nullptr;
+ llvm::DIType *OCLNDRangeDITy = nullptr;
+ llvm::DIType *OCLReserveIDDITy = nullptr;
- /// \brief Cache of previously constructed Types.
+ /// Cache of previously constructed Types.
llvm::DenseMap<const void *, llvm::TrackingMDRef> TypeCache;
+ llvm::SmallDenseMap<llvm::StringRef, llvm::StringRef> DebugPrefixMap;
+
struct ObjCInterfaceCacheEntry {
const ObjCInterfaceType *Type;
llvm::DIType *Decl;
@@ -79,47 +94,52 @@
: Type(Type), Decl(Decl), Unit(Unit) {}
};
- /// \brief Cache of previously constructed interfaces
- /// which may change.
+ /// Cache of previously constructed interfaces which may change.
llvm::SmallVector<ObjCInterfaceCacheEntry, 32> ObjCInterfaceCache;
- /// \brief list of interfaces we want to keep even if orphaned.
+ /// Cache of references to clang modules and precompiled headers.
+ llvm::DenseMap<const Module *, llvm::TrackingMDRef> ModuleCache;
+
+ /// List of interfaces we want to keep even if orphaned.
std::vector<void *> RetainedTypes;
- /// \brief Cache of forward declared types to RAUW at the end of
+ /// Cache of forward declared types to RAUW at the end of
/// compilation.
std::vector<std::pair<const TagType *, llvm::TrackingMDRef>> ReplaceMap;
- /// \brief Cache of replaceable forward declarartions (functions and
+ /// Cache of replaceable forward declarartions (functions and
/// variables) to RAUW at the end of compilation.
std::vector<std::pair<const DeclaratorDecl *, llvm::TrackingMDRef>>
FwdDeclReplaceMap;
- // LexicalBlockStack - Keep track of our current nested lexical block.
+ /// Keep track of our current nested lexical block.
std::vector<llvm::TypedTrackingMDRef<llvm::DIScope>> LexicalBlockStack;
llvm::DenseMap<const Decl *, llvm::TrackingMDRef> RegionMap;
- // FnBeginRegionCount - Keep track of LexicalBlockStack counter at the
- // beginning of a function. This is used to pop unbalanced regions at
- // the end of a function.
+ /// Keep track of LexicalBlockStack counter at the beginning of a
+ /// function. This is used to pop unbalanced regions at the end of a
+ /// function.
std::vector<unsigned> FnBeginRegionCount;
- /// \brief This is a storage for names that are
- /// constructed on demand. For example, C++ destructors, C++ operators etc..
+ /// This is a storage for names that are constructed on demand. For
+ /// example, C++ destructors, C++ operators etc..
llvm::BumpPtrAllocator DebugInfoNames;
StringRef CWDName;
llvm::DenseMap<const char *, llvm::TrackingMDRef> DIFileCache;
llvm::DenseMap<const FunctionDecl *, llvm::TrackingMDRef> SPCache;
- /// \brief Cache declarations relevant to DW_TAG_imported_declarations (C++
+ /// Cache declarations relevant to DW_TAG_imported_declarations (C++
/// using declarations) that aren't covered by other more specific caches.
llvm::DenseMap<const Decl *, llvm::TrackingMDRef> DeclCache;
llvm::DenseMap<const NamespaceDecl *, llvm::TrackingMDRef> NameSpaceCache;
llvm::DenseMap<const NamespaceAliasDecl *, llvm::TrackingMDRef>
NamespaceAliasCache;
- llvm::DenseMap<const Decl *, llvm::TrackingMDRef> StaticDataMemberCache;
+ llvm::DenseMap<const Decl *, llvm::TypedTrackingMDRef<llvm::DIDerivedType>>
+ StaticDataMemberCache;
/// Helper functions for getOrCreateType.
- unsigned Checksum(const ObjCInterfaceDecl *InterfaceDecl);
+ /// @{
+ /// Currently the checksum of an interface includes the number of
+ /// ivars and property accessors.
llvm::DIType *CreateType(const BuiltinType *Ty);
llvm::DIType *CreateType(const ComplexType *Ty);
llvm::DIType *CreateQualifiedType(QualType Ty, llvm::DIFile *Fg);
@@ -130,14 +150,17 @@
llvm::DIType *CreateType(const PointerType *Ty, llvm::DIFile *F);
llvm::DIType *CreateType(const BlockPointerType *Ty, llvm::DIFile *F);
llvm::DIType *CreateType(const FunctionType *Ty, llvm::DIFile *F);
+ /// Get structure or union type.
llvm::DIType *CreateType(const RecordType *Tyg);
llvm::DIType *CreateTypeDefinition(const RecordType *Ty);
llvm::DICompositeType *CreateLimitedType(const RecordType *Ty);
void CollectContainingType(const CXXRecordDecl *RD,
llvm::DICompositeType *CT);
+ /// Get Objective-C interface type.
llvm::DIType *CreateType(const ObjCInterfaceType *Ty, llvm::DIFile *F);
llvm::DIType *CreateTypeDefinition(const ObjCInterfaceType *Ty,
llvm::DIFile *F);
+ /// Get Objective-C object type.
llvm::DIType *CreateType(const ObjCObjectType *Ty, llvm::DIFile *F);
llvm::DIType *CreateType(const VectorType *Ty, llvm::DIFile *F);
llvm::DIType *CreateType(const ArrayType *Ty, llvm::DIFile *F);
@@ -145,10 +168,25 @@
llvm::DIType *CreateType(const RValueReferenceType *Ty, llvm::DIFile *Unit);
llvm::DIType *CreateType(const MemberPointerType *Ty, llvm::DIFile *F);
llvm::DIType *CreateType(const AtomicType *Ty, llvm::DIFile *F);
+ /// Get enumeration type.
llvm::DIType *CreateEnumType(const EnumType *Ty);
llvm::DIType *CreateTypeDefinition(const EnumType *Ty);
+ /// Look up the completed type for a self pointer in the TypeCache and
+ /// create a copy of it with the ObjectPointer and Artificial flags
+ /// set. If the type is not cached, a new one is created. This should
+ /// never happen though, since creating a type for the implicit self
+ /// argument implies that we already parsed the interface definition
+ /// and the ivar declarations in the implementation.
llvm::DIType *CreateSelfType(const QualType &QualTy, llvm::DIType *Ty);
+ /// @}
+
+ /// Get the type from the cache or return null type if it doesn't
+ /// exist.
llvm::DIType *getTypeOrNull(const QualType);
+ /// Return the debug type for a C++ method.
+ /// \arg CXXMethodDecl is of FunctionType. This function type is
+ /// not updated to include implicit \c this pointer. Use this routine
+ /// to get a method type which includes \c this pointer.
llvm::DISubroutineType *getOrCreateMethodType(const CXXMethodDecl *Method,
llvm::DIFile *F);
llvm::DISubroutineType *
@@ -156,32 +194,45 @@
llvm::DIFile *Unit);
llvm::DISubroutineType *
getOrCreateFunctionType(const Decl *D, QualType FnType, llvm::DIFile *F);
+ /// \return debug info descriptor for vtable.
llvm::DIType *getOrCreateVTablePtrType(llvm::DIFile *F);
+ /// \return namespace descriptor for the given namespace decl.
llvm::DINamespace *getOrCreateNameSpace(const NamespaceDecl *N);
- llvm::DIType *getOrCreateTypeDeclaration(QualType PointeeTy, llvm::DIFile *F);
llvm::DIType *CreatePointerLikeType(llvm::dwarf::Tag Tag, const Type *Ty,
QualType PointeeTy, llvm::DIFile *F);
-
- llvm::Value *getCachedInterfaceTypeOrNull(const QualType Ty);
llvm::DIType *getOrCreateStructPtrType(StringRef Name, llvm::DIType *&Cache);
+ /// A helper function to create a subprogram for a single member
+ /// function GlobalDecl.
llvm::DISubprogram *CreateCXXMemberFunction(const CXXMethodDecl *Method,
llvm::DIFile *F,
llvm::DIType *RecordTy);
+ /// A helper function to collect debug info for C++ member
+ /// functions. This is used while creating debug info entry for a
+ /// Record.
void CollectCXXMemberFunctions(const CXXRecordDecl *Decl, llvm::DIFile *F,
SmallVectorImpl<llvm::Metadata *> &E,
llvm::DIType *T);
+ /// A helper function to collect debug info for C++ base
+ /// classes. This is used while creating debug info entry for a
+ /// Record.
void CollectCXXBases(const CXXRecordDecl *Decl, llvm::DIFile *F,
SmallVectorImpl<llvm::Metadata *> &EltTys,
llvm::DIType *RecordTy);
+ /// A helper function to collect template parameters.
llvm::DINodeArray CollectTemplateParams(const TemplateParameterList *TPList,
ArrayRef<TemplateArgument> TAList,
llvm::DIFile *Unit);
+ /// A helper function to collect debug info for function template
+ /// parameters.
llvm::DINodeArray CollectFunctionTemplateParams(const FunctionDecl *FD,
llvm::DIFile *Unit);
+
+ /// A helper function to collect debug info for template
+ /// parameters.
llvm::DINodeArray
CollectCXXTemplateParams(const ClassTemplateSpecializationDecl *TS,
llvm::DIFile *F);
@@ -192,7 +243,8 @@
llvm::DIFile *tunit, llvm::DIScope *scope,
const RecordDecl *RD = nullptr);
- // Helpers for collecting fields of a record.
+ /// Helpers for collecting fields of a record.
+ /// @{
void CollectRecordLambdaFields(const CXXRecordDecl *CXXDecl,
SmallVectorImpl<llvm::Metadata *> &E,
llvm::DIType *RecordTy);
@@ -207,11 +259,13 @@
SmallVectorImpl<llvm::Metadata *> &E,
llvm::DICompositeType *RecordTy);
+ /// If the C++ class has vtable info then insert appropriate debug
+ /// info entry in EltTys vector.
void CollectVTableInfo(const CXXRecordDecl *Decl, llvm::DIFile *F,
SmallVectorImpl<llvm::Metadata *> &EltTys);
+ /// @}
- // CreateLexicalBlock - Create a new lexical block node and push it on
- // the stack.
+ /// Create a new lexical block node and push it on the stack.
void CreateLexicalBlock(SourceLocation Loc);
public:
@@ -220,85 +274,100 @@
void finalize();
- /// \brief Update the current source location. If \arg loc is
- /// invalid it is ignored.
+ /// Set the main CU's DwoId field to \p Signature.
+ void setDwoId(uint64_t Signature);
+
+ /// When generating debug information for a clang module or
+ /// precompiled header, this module map will be used to determine
+ /// the module of origin of each Decl.
+ void setModuleMap(ModuleMap &MMap) { ClangModuleMap = &MMap; }
+
+ /// Update the current source location. If \arg loc is invalid it is
+ /// ignored.
void setLocation(SourceLocation Loc);
- /// \brief Emit metadata to indicate a change in line/column
- /// information in the source file.
+ /// Emit metadata to indicate a change in line/column information in
+ /// the source file. If the location is invalid, the previous
+ /// location will be reused.
void EmitLocation(CGBuilderTy &Builder, SourceLocation Loc);
- /// \brief Emit a call to llvm.dbg.function.start to indicate
+ /// Emit a call to llvm.dbg.function.start to indicate
/// start of a new function.
/// \param Loc The location of the function header.
/// \param ScopeLoc The location of the function body.
- void EmitFunctionStart(GlobalDecl GD,
- SourceLocation Loc, SourceLocation ScopeLoc,
- QualType FnType, llvm::Function *Fn,
- CGBuilderTy &Builder);
+ void EmitFunctionStart(GlobalDecl GD, SourceLocation Loc,
+ SourceLocation ScopeLoc, QualType FnType,
+ llvm::Function *Fn, CGBuilderTy &Builder);
- /// \brief Constructs the debug code for exiting a function.
+ /// Emit debug info for a function declaration.
+ void EmitFunctionDecl(GlobalDecl GD, SourceLocation Loc, QualType FnType);
+
+ /// Constructs the debug code for exiting a function.
void EmitFunctionEnd(CGBuilderTy &Builder);
- /// \brief Emit metadata to indicate the beginning of a
- /// new lexical block and push the block onto the stack.
+ /// Emit metadata to indicate the beginning of a new lexical block
+ /// and push the block onto the stack.
void EmitLexicalBlockStart(CGBuilderTy &Builder, SourceLocation Loc);
- /// \brief Emit metadata to indicate the end of a new lexical
- /// block and pop the current block.
+ /// Emit metadata to indicate the end of a new lexical block and pop
+ /// the current block.
void EmitLexicalBlockEnd(CGBuilderTy &Builder, SourceLocation Loc);
- /// \brief Emit call to llvm.dbg.declare for an automatic
- /// variable declaration.
+ /// Emit call to \c llvm.dbg.declare for an automatic variable
+ /// declaration.
void EmitDeclareOfAutoVariable(const VarDecl *Decl, llvm::Value *AI,
CGBuilderTy &Builder);
- /// \brief Emit call to llvm.dbg.declare for an
- /// imported variable declaration in a block.
+ /// Emit call to \c llvm.dbg.declare for an imported variable
+ /// declaration in a block.
void EmitDeclareOfBlockDeclRefVariable(const VarDecl *variable,
llvm::Value *storage,
CGBuilderTy &Builder,
const CGBlockInfo &blockInfo,
- llvm::Instruction *InsertPoint = 0);
+ llvm::Instruction *InsertPoint = nullptr);
- /// \brief Emit call to llvm.dbg.declare for an argument
- /// variable declaration.
+ /// Emit call to \c llvm.dbg.declare for an argument variable
+ /// declaration.
void EmitDeclareOfArgVariable(const VarDecl *Decl, llvm::Value *AI,
unsigned ArgNo, CGBuilderTy &Builder);
- /// \brief Emit call to
- /// llvm.dbg.declare for the block-literal argument to a block
- /// invocation function.
+ /// Emit call to \c llvm.dbg.declare for the block-literal argument
+ /// to a block invocation function.
void EmitDeclareOfBlockLiteralArgVariable(const CGBlockInfo &block,
llvm::Value *Arg, unsigned ArgNo,
llvm::Value *LocalAddr,
CGBuilderTy &Builder);
- /// \brief Emit information about a global variable.
+ /// Emit information about a global variable.
void EmitGlobalVariable(llvm::GlobalVariable *GV, const VarDecl *Decl);
- /// \brief Emit global variable's debug info.
+ /// Emit global variable's debug info.
void EmitGlobalVariable(const ValueDecl *VD, llvm::Constant *Init);
- /// \brief Emit C++ using directive.
+ /// Emit C++ using directive.
void EmitUsingDirective(const UsingDirectiveDecl &UD);
- /// \brief Emit the type explicitly casted to.
+ /// Emit the type explicitly casted to.
void EmitExplicitCastType(QualType Ty);
- /// \brief Emit C++ using declaration.
+ /// Emit C++ using declaration.
void EmitUsingDecl(const UsingDecl &UD);
- /// \brief Emit C++ namespace alias.
+ /// Emit an @import declaration.
+ void EmitImportDecl(const ImportDecl &ID);
+
+ /// Emit C++ namespace alias.
llvm::DIImportedEntity *EmitNamespaceAlias(const NamespaceAliasDecl &NA);
- /// \brief Emit record type's standalone debug info.
+ /// Emit record type's standalone debug info.
llvm::DIType *getOrCreateRecordType(QualType Ty, SourceLocation L);
- /// \brief Emit an objective c interface type standalone
- /// debug info.
+ /// Emit an Objective-C interface type standalone debug info.
llvm::DIType *getOrCreateInterfaceType(QualType Ty, SourceLocation Loc);
+ /// Emit standalone debug info for a type.
+ llvm::DIType *getOrCreateStandaloneType(QualType Ty, SourceLocation Loc);
+
void completeType(const EnumDecl *ED);
void completeType(const RecordDecl *RD);
void completeRequiredType(const RecordDecl *RD);
@@ -307,116 +376,130 @@
void completeTemplateDefinition(const ClassTemplateSpecializationDecl &SD);
private:
- /// \brief Emit call to llvm.dbg.declare for a variable declaration.
- /// Tag accepts custom types DW_TAG_arg_variable and DW_TAG_auto_variable,
- /// otherwise would be of type llvm::dwarf::Tag.
- void EmitDeclare(const VarDecl *decl, llvm::dwarf::Tag Tag, llvm::Value *AI,
- unsigned ArgNo, CGBuilderTy &Builder);
+ /// Emit call to llvm.dbg.declare for a variable declaration.
+ void EmitDeclare(const VarDecl *decl, llvm::Value *AI,
+ llvm::Optional<unsigned> ArgNo, CGBuilderTy &Builder);
- // EmitTypeForVarWithBlocksAttr - Build up structure info for the byref.
- // See BuildByRefType.
+ /// Build up structure info for the byref. See \a BuildByRefType.
llvm::DIType *EmitTypeForVarWithBlocksAttr(const VarDecl *VD,
uint64_t *OffSet);
- /// \brief Get context info for the decl.
- llvm::DIScope *getContextDescriptor(const Decl *Decl);
+ /// Get context info for the DeclContext of \p Decl.
+ llvm::DIScope *getDeclContextDescriptor(const Decl *D);
+ /// Get context info for a given DeclContext \p Decl.
+ llvm::DIScope *getContextDescriptor(const Decl *Context,
+ llvm::DIScope *Default);
llvm::DIScope *getCurrentContextDescriptor(const Decl *Decl);
- /// \brief Create a forward decl for a RecordType in a given context.
+ /// Create a forward decl for a RecordType in a given context.
llvm::DICompositeType *getOrCreateRecordFwdDecl(const RecordType *,
llvm::DIScope *);
- /// \brief Return current directory name.
+ /// Return current directory name.
StringRef getCurrentDirname();
- /// \brief Create new compile unit.
+ /// Create new compile unit.
void CreateCompileUnit();
- /// \brief Get the file debug info descriptor for the input
- /// location.
+ /// Remap a given path with the current debug prefix map
+ std::string remapDIPath(StringRef) const;
+
+ /// Get the file debug info descriptor for the input location.
llvm::DIFile *getOrCreateFile(SourceLocation Loc);
- /// \brief Get the file info for main compile unit.
+ /// Get the file info for main compile unit.
llvm::DIFile *getOrCreateMainFile();
- /// \brief Get the type from the cache or create a new type if
- /// necessary.
+ /// Get the type from the cache or create a new type if necessary.
llvm::DIType *getOrCreateType(QualType Ty, llvm::DIFile *Fg);
- /// \brief Get the type from the cache or create a new
- /// partial type if necessary.
- llvm::DIType *getOrCreateLimitedType(const RecordType *Ty, llvm::DIFile *F);
+ /// Get a reference to a clang module. If \p CreateSkeletonCU is true,
+ /// this also creates a split dwarf skeleton compile unit.
+ llvm::DIModule *
+ getOrCreateModuleRef(ExternalASTSource::ASTSourceDescriptor Mod,
+ bool CreateSkeletonCU);
- /// \brief Create type metadata for a source language type.
+ /// DebugTypeExtRefs: If \p D originated in a clang module, return it.
+ llvm::DIModule *getParentModuleOrNull(const Decl *D);
+
+ /// Get the type from the cache or create a new partial type if
+ /// necessary.
+ llvm::DICompositeType *getOrCreateLimitedType(const RecordType *Ty,
+ llvm::DIFile *F);
+
+ /// Create type metadata for a source language type.
llvm::DIType *CreateTypeNode(QualType Ty, llvm::DIFile *Fg);
- /// \brief return the underlying ObjCInterfaceDecl
- /// if Ty is an ObjCInterface or a pointer to one.
- ObjCInterfaceDecl* getObjCInterfaceDecl(QualType Ty);
-
- /// \brief Create new member and increase Offset by FType's size.
+ /// Create new member and increase Offset by FType's size.
llvm::DIType *CreateMemberType(llvm::DIFile *Unit, QualType FType,
StringRef Name, uint64_t *Offset);
- /// \brief Retrieve the DIDescriptor, if any, for the canonical form of this
+ /// Retrieve the DIDescriptor, if any, for the canonical form of this
/// declaration.
llvm::DINode *getDeclarationOrDefinition(const Decl *D);
- /// \brief Return debug info descriptor to describe method
+ /// \return debug info descriptor to describe method
/// declaration for the given method definition.
llvm::DISubprogram *getFunctionDeclaration(const Decl *D);
- /// Return debug info descriptor to describe in-class static data member
- /// declaration for the given out-of-class definition.
+ /// \return debug info descriptor to describe in-class static data
+ /// member declaration for the given out-of-class definition. If D
+ /// is an out-of-class definition of a static data member of a
+ /// class, find its corresponding in-class declaration.
llvm::DIDerivedType *
getOrCreateStaticDataMemberDeclarationOrNull(const VarDecl *D);
- /// \brief Create a subprogram describing the forward
- /// decalration represented in the given FunctionDecl.
+ /// Create a subprogram describing the forward declaration
+ /// represented in the given FunctionDecl.
llvm::DISubprogram *getFunctionForwardDeclaration(const FunctionDecl *FD);
- /// \brief Create a global variable describing the forward decalration
+ /// Create a global variable describing the forward decalration
/// represented in the given VarDecl.
llvm::DIGlobalVariable *
getGlobalVariableForwardDeclaration(const VarDecl *VD);
- /// Return a global variable that represents one of the collection of
- /// global variables created for an anonmyous union.
+ /// \brief Return a global variable that represents one of the
+ /// collection of global variables created for an anonmyous union.
+ ///
+ /// Recursively collect all of the member fields of a global
+ /// anonymous decl and create static variables for them. The first
+ /// time this is called it needs to be on a union and then from
+ /// there we can have additional unnamed fields.
llvm::DIGlobalVariable *
CollectAnonRecordDecls(const RecordDecl *RD, llvm::DIFile *Unit,
unsigned LineNo, StringRef LinkageName,
llvm::GlobalVariable *Var, llvm::DIScope *DContext);
- /// \brief Get function name for the given FunctionDecl. If the
- /// name is constructed on demand (e.g. C++ destructor) then the name
- /// is stored on the side.
+ /// Get function name for the given FunctionDecl. If the name is
+ /// constructed on demand (e.g., C++ destructor) then the name is
+ /// stored on the side.
StringRef getFunctionName(const FunctionDecl *FD);
- /// \brief Returns the unmangled name of an Objective-C method.
+ /// Returns the unmangled name of an Objective-C method.
/// This is the display name for the debugging info.
StringRef getObjCMethodName(const ObjCMethodDecl *FD);
- /// \brief Return selector name. This is used for debugging
+ /// Return selector name. This is used for debugging
/// info.
StringRef getSelectorName(Selector S);
- /// \brief Get class name including template argument list.
+ /// Get class name including template argument list.
StringRef getClassName(const RecordDecl *RD);
- /// \brief Get vtable name for the given Class.
+ /// Get the vtable name for the given class.
StringRef getVTableName(const CXXRecordDecl *Decl);
- /// \brief Get line number for the location. If location is invalid
+ /// Get line number for the location. If location is invalid
/// then use current location.
unsigned getLineNumber(SourceLocation Loc);
- /// \brief Get column number for the location. If location is
+ /// Get column number for the location. If location is
/// invalid then use current location.
/// \param Force Assume DebugColumnInfo option is true.
- unsigned getColumnNumber(SourceLocation Loc, bool Force=false);
+ unsigned getColumnNumber(SourceLocation Loc, bool Force = false);
- /// \brief Collect various properties of a FunctionDecl.
+ /// Collect various properties of a FunctionDecl.
/// \param GD A GlobalDecl whose getDecl() must return a FunctionDecl.
void collectFunctionDeclProps(GlobalDecl GD, llvm::DIFile *Unit,
StringRef &Name, StringRef &LinkageName,
@@ -424,23 +507,25 @@
llvm::DINodeArray &TParamsArray,
unsigned &Flags);
- /// \brief Collect various properties of a VarDecl.
+ /// Collect various properties of a VarDecl.
void collectVarDeclProps(const VarDecl *VD, llvm::DIFile *&Unit,
unsigned &LineNo, QualType &T, StringRef &Name,
StringRef &LinkageName, llvm::DIScope *&VDContext);
- /// \brief Allocate a copy of \p A using the DebugInfoNames allocator
+ /// Allocate a copy of \p A using the DebugInfoNames allocator
/// and return a reference to it. If multiple arguments are given the strings
/// are concatenated.
StringRef internString(StringRef A, StringRef B = StringRef()) {
char *Data = DebugInfoNames.Allocate<char>(A.size() + B.size());
- std::memcpy(Data, A.data(), A.size());
- std::memcpy(Data + A.size(), B.data(), B.size());
+ if (!A.empty())
+ std::memcpy(Data, A.data(), A.size());
+ if (!B.empty())
+ std::memcpy(Data + A.size(), B.data(), B.size());
return StringRef(Data, A.size() + B.size());
}
};
-/// \brief A scoped helper to set the current debug location to the specified
+/// A scoped helper to set the current debug location to the specified
/// location or preferred location of the specified Expr.
class ApplyDebugLocation {
private:
@@ -449,56 +534,54 @@
SourceLocation TemporaryLocation);
llvm::DebugLoc OriginalLocation;
- CodeGenFunction &CGF;
-public:
+ CodeGenFunction *CGF;
- /// \brief Set the location to the (valid) TemporaryLocation.
+public:
+ /// Set the location to the (valid) TemporaryLocation.
ApplyDebugLocation(CodeGenFunction &CGF, SourceLocation TemporaryLocation);
ApplyDebugLocation(CodeGenFunction &CGF, const Expr *E);
ApplyDebugLocation(CodeGenFunction &CGF, llvm::DebugLoc Loc);
+ ApplyDebugLocation(ApplyDebugLocation &&Other) : CGF(Other.CGF) {
+ Other.CGF = nullptr;
+ }
~ApplyDebugLocation();
- /// \brief Apply TemporaryLocation if it is valid. Otherwise switch to an
- /// artificial debug location that has a valid scope, but no line information.
+ /// \brief Apply TemporaryLocation if it is valid. Otherwise switch
+ /// to an artificial debug location that has a valid scope, but no
+ /// line information.
///
- /// Artificial locations are useful when emitting compiler-generated helper
- /// functions that have no source location associated with them. The DWARF
- /// specification allows the compiler to use the special line number 0 to
- /// indicate code that can not be attributed to any source location. Note that
- /// passing an empty SourceLocation to CGDebugInfo::setLocation() will result
- /// in the last valid location being reused.
+ /// Artificial locations are useful when emitting compiler-generated
+ /// helper functions that have no source location associated with
+ /// them. The DWARF specification allows the compiler to use the
+ /// special line number 0 to indicate code that can not be
+ /// attributed to any source location. Note that passing an empty
+ /// SourceLocation to CGDebugInfo::setLocation() will result in the
+ /// last valid location being reused.
static ApplyDebugLocation CreateArtificial(CodeGenFunction &CGF) {
return ApplyDebugLocation(CGF, false, SourceLocation());
}
- /// \brief Apply TemporaryLocation if it is valid. Otherwise switch to an
- /// artificial debug location that has a valid scope, but no line information.
- static ApplyDebugLocation CreateDefaultArtificial(CodeGenFunction &CGF,
- SourceLocation TemporaryLocation) {
+ /// \brief Apply TemporaryLocation if it is valid. Otherwise switch
+ /// to an artificial debug location that has a valid scope, but no
+ /// line information.
+ static ApplyDebugLocation
+ CreateDefaultArtificial(CodeGenFunction &CGF,
+ SourceLocation TemporaryLocation) {
return ApplyDebugLocation(CGF, false, TemporaryLocation);
}
- /// \brief Set the IRBuilder to not attach debug locations. Note that passing
- /// an empty SourceLocation to CGDebugInfo::setLocation() will result in the
- /// last valid location being reused. Note that all instructions that do not
- /// have a location at the beginning of a function are counted towards to
- /// funciton prologue.
+ /// Set the IRBuilder to not attach debug locations. Note that
+ /// passing an empty SourceLocation to \a CGDebugInfo::setLocation()
+ /// will result in the last valid location being reused. Note that
+ /// all instructions that do not have a location at the beginning of
+ /// a function are counted towards to function prologue.
static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF) {
return ApplyDebugLocation(CGF, true, SourceLocation());
}
- /// \brief Apply TemporaryLocation if it is valid. Otherwise set the IRBuilder
- /// to not attach debug locations.
- static ApplyDebugLocation CreateDefaultEmpty(CodeGenFunction &CGF,
- SourceLocation TemporaryLocation) {
- return ApplyDebugLocation(CGF, true, TemporaryLocation);
- }
-
};
-
} // namespace CodeGen
} // namespace clang
-
-#endif
+#endif // LLVM_CLANG_LIB_CODEGEN_CGDEBUGINFO_H
diff --git a/lib/CodeGen/CGDecl.cpp b/lib/CodeGen/CGDecl.cpp
index 07dbce4..b78e80d 100644
--- a/lib/CodeGen/CGDecl.cpp
+++ b/lib/CodeGen/CGDecl.cpp
@@ -12,6 +12,7 @@
//===----------------------------------------------------------------------===//
#include "CodeGenFunction.h"
+#include "CGBlocks.h"
#include "CGCleanup.h"
#include "CGDebugInfo.h"
#include "CGOpenCLRuntime.h"
@@ -34,6 +35,7 @@
void CodeGenFunction::EmitDecl(const Decl &D) {
switch (D.getKind()) {
+ case Decl::BuiltinTemplate:
case Decl::TranslationUnit:
case Decl::ExternCContext:
case Decl::Namespace:
@@ -79,6 +81,7 @@
case Decl::Captured:
case Decl::ClassScopeFunctionSpecialization:
case Decl::UsingShadow:
+ case Decl::ObjCTypeParam:
llvm_unreachable("Declaration should not be in declstmts!");
case Decl::Function: // void X();
case Decl::Record: // struct/union/class X;
@@ -141,7 +144,7 @@
// Don't emit it now, allow it to be emitted lazily on its first use.
return;
- if (D.getStorageClass() == SC_OpenCLWorkGroupLocal)
+ if (D.getType().getAddressSpace() == LangAS::opencl_local)
return CGM.getOpenCLRuntime().EmitWorkGroupLocalVarDecl(*this, D);
assert(D.hasLocalStorage());
@@ -310,6 +313,7 @@
OldGV->getThreadLocalMode(),
CGM.getContext().getTargetAddressSpace(D.getType()));
GV->setVisibility(OldGV->getVisibility());
+ GV->setComdat(OldGV->getComdat());
// Steal the name of the old global
GV->takeName(OldGV);
@@ -338,17 +342,15 @@
void CodeGenFunction::EmitStaticVarDecl(const VarDecl &D,
llvm::GlobalValue::LinkageTypes Linkage) {
- llvm::Value *&DMEntry = LocalDeclMap[&D];
- assert(!DMEntry && "Decl already exists in localdeclmap!");
-
// Check to see if we already have a global variable for this
// declaration. This can happen when double-emitting function
// bodies, e.g. with complete and base constructors.
llvm::Constant *addr = CGM.getOrCreateStaticVarDecl(D, Linkage);
+ CharUnits alignment = getContext().getDeclAlign(&D);
// Store into LocalDeclMap before generating initializer to handle
// circular references.
- DMEntry = addr;
+ setAddrOfLocalVar(&D, Address(addr, alignment));
// We can't have a VLA here, but we can have a pointer to a VLA,
// even though that doesn't really make any sense.
@@ -365,7 +367,7 @@
if (D.getInit())
var = AddInitializerToStaticVarDecl(D, var);
- var->setAlignment(getContext().getDeclAlign(&D).getQuantity());
+ var->setAlignment(alignment.getQuantity());
if (D.hasAttr<AnnotateAttr>())
CGM.AddGlobalAnnotations(&D, var);
@@ -383,7 +385,8 @@
// RAUW's the GV uses of this constant will be invalid.
llvm::Constant *castedAddr =
llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(var, expectedType);
- DMEntry = castedAddr;
+ if (var != castedAddr)
+ LocalDeclMap.find(&D)->second = Address(castedAddr, alignment);
CGM.setStaticLocalDeclAddress(&D, castedAddr);
CGM.getSanitizerMetadata()->reportGlobalToASan(var, D);
@@ -398,14 +401,14 @@
}
namespace {
- struct DestroyObject : EHScopeStack::Cleanup {
- DestroyObject(llvm::Value *addr, QualType type,
+ struct DestroyObject final : EHScopeStack::Cleanup {
+ DestroyObject(Address addr, QualType type,
CodeGenFunction::Destroyer *destroyer,
bool useEHCleanupForArray)
: addr(addr), type(type), destroyer(destroyer),
useEHCleanupForArray(useEHCleanupForArray) {}
- llvm::Value *addr;
+ Address addr;
QualType type;
CodeGenFunction::Destroyer *destroyer;
bool useEHCleanupForArray;
@@ -419,15 +422,15 @@
}
};
- struct DestroyNRVOVariable : EHScopeStack::Cleanup {
- DestroyNRVOVariable(llvm::Value *addr,
+ struct DestroyNRVOVariable final : EHScopeStack::Cleanup {
+ DestroyNRVOVariable(Address addr,
const CXXDestructorDecl *Dtor,
llvm::Value *NRVOFlag)
: Dtor(Dtor), NRVOFlag(NRVOFlag), Loc(addr) {}
const CXXDestructorDecl *Dtor;
llvm::Value *NRVOFlag;
- llvm::Value *Loc;
+ Address Loc;
void Emit(CodeGenFunction &CGF, Flags flags) override {
// Along the exceptions path we always execute the dtor.
@@ -438,7 +441,8 @@
// If we exited via NRVO, we skip the destructor call.
llvm::BasicBlock *RunDtorBB = CGF.createBasicBlock("nrvo.unused");
SkipDtorBB = CGF.createBasicBlock("nrvo.skipdtor");
- llvm::Value *DidNRVO = CGF.Builder.CreateLoad(NRVOFlag, "nrvo.val");
+ llvm::Value *DidNRVO =
+ CGF.Builder.CreateFlagLoad(NRVOFlag, "nrvo.val");
CGF.Builder.CreateCondBr(DidNRVO, SkipDtorBB, RunDtorBB);
CGF.EmitBlock(RunDtorBB);
}
@@ -452,9 +456,9 @@
}
};
- struct CallStackRestore : EHScopeStack::Cleanup {
- llvm::Value *Stack;
- CallStackRestore(llvm::Value *Stack) : Stack(Stack) {}
+ struct CallStackRestore final : EHScopeStack::Cleanup {
+ Address Stack;
+ CallStackRestore(Address Stack) : Stack(Stack) {}
void Emit(CodeGenFunction &CGF, Flags flags) override {
llvm::Value *V = CGF.Builder.CreateLoad(Stack);
llvm::Value *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stackrestore);
@@ -462,7 +466,7 @@
}
};
- struct ExtendGCLifetime : EHScopeStack::Cleanup {
+ struct ExtendGCLifetime final : EHScopeStack::Cleanup {
const VarDecl &Var;
ExtendGCLifetime(const VarDecl *var) : Var(*var) {}
@@ -477,7 +481,7 @@
}
};
- struct CallCleanupFunction : EHScopeStack::Cleanup {
+ struct CallCleanupFunction final : EHScopeStack::Cleanup {
llvm::Constant *CleanupFn;
const CGFunctionInfo &FnInfo;
const VarDecl &Var;
@@ -491,7 +495,7 @@
Var.getType(), VK_LValue, SourceLocation());
// Compute the address of the local variable, in case it's a byref
// or something.
- llvm::Value *Addr = CGF.EmitDeclRefLValue(&DRE).getAddress();
+ llvm::Value *Addr = CGF.EmitDeclRefLValue(&DRE).getPointer();
// In some cases, the type of the function argument will be different from
// the type of the pointer. An example of this is
@@ -511,12 +515,12 @@
};
/// A cleanup to call @llvm.lifetime.end.
- class CallLifetimeEnd : public EHScopeStack::Cleanup {
+ class CallLifetimeEnd final : public EHScopeStack::Cleanup {
llvm::Value *Addr;
llvm::Value *Size;
public:
- CallLifetimeEnd(llvm::Value *addr, llvm::Value *size)
- : Addr(addr), Size(size) {}
+ CallLifetimeEnd(Address addr, llvm::Value *size)
+ : Addr(addr.getPointer()), Size(size) {}
void Emit(CodeGenFunction &CGF, Flags flags) override {
CGF.EmitLifetimeEnd(Size, Addr);
@@ -527,7 +531,7 @@
/// EmitAutoVarWithLifetime - Does the setup required for an automatic
/// variable with lifetime.
static void EmitAutoVarWithLifetime(CodeGenFunction &CGF, const VarDecl &var,
- llvm::Value *addr,
+ Address addr,
Qualifiers::ObjCLifetime lifetime) {
switch (lifetime) {
case Qualifiers::OCL_None:
@@ -579,9 +583,9 @@
}
}
- for (Stmt::const_child_range children = s->children(); children; ++children)
- // children might be null; as in missing decl or conditional of an if-stmt.
- if ((*children) && isAccessedBy(var, *children))
+ for (const Stmt *SubStmt : s->children())
+ // SubStmt might be null; as in missing decl or conditional of an if-stmt.
+ if (SubStmt && isAccessedBy(var, SubStmt))
return true;
return false;
@@ -594,10 +598,61 @@
return isAccessedBy(*var, e);
}
+static bool tryEmitARCCopyWeakInit(CodeGenFunction &CGF,
+ const LValue &destLV, const Expr *init) {
+ bool needsCast = false;
+
+ while (auto castExpr = dyn_cast<CastExpr>(init->IgnoreParens())) {
+ switch (castExpr->getCastKind()) {
+ // Look through casts that don't require representation changes.
+ case CK_NoOp:
+ case CK_BitCast:
+ case CK_BlockPointerToObjCPointerCast:
+ needsCast = true;
+ break;
+
+ // If we find an l-value to r-value cast from a __weak variable,
+ // emit this operation as a copy or move.
+ case CK_LValueToRValue: {
+ const Expr *srcExpr = castExpr->getSubExpr();
+ if (srcExpr->getType().getObjCLifetime() != Qualifiers::OCL_Weak)
+ return false;
+
+ // Emit the source l-value.
+ LValue srcLV = CGF.EmitLValue(srcExpr);
+
+ // Handle a formal type change to avoid asserting.
+ auto srcAddr = srcLV.getAddress();
+ if (needsCast) {
+ srcAddr = CGF.Builder.CreateElementBitCast(srcAddr,
+ destLV.getAddress().getElementType());
+ }
+
+ // If it was an l-value, use objc_copyWeak.
+ if (srcExpr->getValueKind() == VK_LValue) {
+ CGF.EmitARCCopyWeak(destLV.getAddress(), srcAddr);
+ } else {
+ assert(srcExpr->getValueKind() == VK_XValue);
+ CGF.EmitARCMoveWeak(destLV.getAddress(), srcAddr);
+ }
+ return true;
+ }
+
+ // Stop at anything else.
+ default:
+ return false;
+ }
+
+ init = castExpr->getSubExpr();
+ continue;
+ }
+ return false;
+}
+
static void drillIntoBlockVariable(CodeGenFunction &CGF,
LValue &lvalue,
const VarDecl *var) {
- lvalue.setAddress(CGF.BuildBlockByrefAddress(lvalue.getAddress(), var));
+ lvalue.setAddress(CGF.emitBlockByrefAddress(lvalue.getAddress(), var));
}
void CodeGenFunction::EmitScalarInit(const Expr *init, const ValueDecl *D,
@@ -635,15 +690,12 @@
if (capturedByInit) {
// We can use a simple GEP for this because it can't have been
// moved yet.
- tempLV.setAddress(Builder.CreateStructGEP(
- nullptr, tempLV.getAddress(),
- getByRefValueLLVMField(cast<VarDecl>(D)).second));
+ tempLV.setAddress(emitBlockByrefAddress(tempLV.getAddress(),
+ cast<VarDecl>(D),
+ /*follow*/ false));
}
- llvm::PointerType *ty
- = cast<llvm::PointerType>(tempLV.getAddress()->getType());
- ty = cast<llvm::PointerType>(ty->getElementType());
-
+ auto ty = cast<llvm::PointerType>(tempLV.getAddress().getElementType());
llvm::Value *zero = llvm::ConstantPointerNull::get(ty);
// If __weak, we want to use a barrier under certain conditions.
@@ -673,6 +725,12 @@
}
case Qualifiers::OCL_Weak: {
+ // If it's not accessed by the initializer, try to emit the
+ // initialization with a copy or move.
+ if (!accessedByInit && tryEmitARCCopyWeakInit(*this, lvalue, init)) {
+ return;
+ }
+
// No way to optimize a producing initializer into this. It's not
// worth optimizing for, because the value will immediately
// disappear in the common case.
@@ -787,7 +845,7 @@
if (isa<llvm::ConstantInt>(Init) || isa<llvm::ConstantFP>(Init) ||
isa<llvm::ConstantVector>(Init) || isa<llvm::BlockAddress>(Init) ||
isa<llvm::ConstantExpr>(Init)) {
- Builder.CreateStore(Init, Loc, isVolatile);
+ Builder.CreateDefaultAlignedStore(Init, Loc, isVolatile);
return;
}
@@ -890,13 +948,12 @@
emission.IsByRef = isByRef;
CharUnits alignment = getContext().getDeclAlign(&D);
- emission.Alignment = alignment;
// If the type is variably-modified, emit all the VLA sizes for it.
if (Ty->isVariablyModifiedType())
EmitVariablyModifiedType(Ty);
- llvm::Value *DeclPtr;
+ Address address = Address::invalid();
if (Ty->isConstantSizeType()) {
bool NRVO = getLangOpts().ElideConstructors &&
D.isNRVOVariable();
@@ -922,7 +979,8 @@
CGM.isTypeConstant(Ty, true)) {
EmitStaticVarDecl(D, llvm::GlobalValue::InternalLinkage);
- emission.Address = nullptr; // signal this condition to later callbacks
+ // Signal this condition to later callbacks.
+ emission.Addr = Address::invalid();
assert(emission.wasEmittedAsGlobal());
return emission;
}
@@ -933,13 +991,12 @@
// A normal fixed sized variable becomes an alloca in the entry block,
// unless it's an NRVO variable.
- llvm::Type *LTy = ConvertTypeForMem(Ty);
if (NRVO) {
// The named return value optimization: allocate this variable in the
// return slot, so that we can elide the copy when returning this
// variable (C++0x [class.copy]p34).
- DeclPtr = ReturnValue;
+ address = ReturnValue;
if (const RecordType *RecordTy = Ty->getAs<RecordType>()) {
if (!cast<CXXRecordDecl>(RecordTy->getDecl())->hasTrivialDestructor()) {
@@ -947,34 +1004,46 @@
// to this variable. Set it to zero to indicate that NRVO was not
// applied.
llvm::Value *Zero = Builder.getFalse();
- llvm::Value *NRVOFlag = CreateTempAlloca(Zero->getType(), "nrvo");
+ Address NRVOFlag =
+ CreateTempAlloca(Zero->getType(), CharUnits::One(), "nrvo");
EnsureInsertPoint();
Builder.CreateStore(Zero, NRVOFlag);
// Record the NRVO flag for this variable.
- NRVOFlags[&D] = NRVOFlag;
- emission.NRVOFlag = NRVOFlag;
+ NRVOFlags[&D] = NRVOFlag.getPointer();
+ emission.NRVOFlag = NRVOFlag.getPointer();
}
}
} else {
- if (isByRef)
- LTy = BuildByRefType(&D);
+ CharUnits allocaAlignment;
+ llvm::Type *allocaTy;
+ if (isByRef) {
+ auto &byrefInfo = getBlockByrefInfo(&D);
+ allocaTy = byrefInfo.Type;
+ allocaAlignment = byrefInfo.ByrefAlignment;
+ } else {
+ allocaTy = ConvertTypeForMem(Ty);
+ allocaAlignment = alignment;
+ }
- llvm::AllocaInst *Alloc = CreateTempAlloca(LTy);
- Alloc->setName(D.getName());
+ // Create the alloca. Note that we set the name separately from
+ // building the instruction so that it's there even in no-asserts
+ // builds.
+ address = CreateTempAlloca(allocaTy, allocaAlignment);
+ address.getPointer()->setName(D.getName());
- CharUnits allocaAlignment = alignment;
- if (isByRef)
- allocaAlignment = std::max(allocaAlignment,
- getContext().toCharUnitsFromBits(getTarget().getPointerAlign(0)));
- Alloc->setAlignment(allocaAlignment.getQuantity());
- DeclPtr = Alloc;
+ // Don't emit lifetime markers for MSVC catch parameters. The lifetime of
+ // the catch parameter starts in the catchpad instruction, and we can't
+ // insert code in those basic blocks.
+ bool IsMSCatchParam =
+ D.isExceptionVariable() && getTarget().getCXXABI().isMicrosoft();
// Emit a lifetime intrinsic if meaningful. There's no point
// in doing this if we don't have a valid insertion point (?).
- uint64_t size = CGM.getDataLayout().getTypeAllocSize(LTy);
- if (HaveInsertPoint()) {
- emission.SizeForLifetimeMarkers = EmitLifetimeStart(size, Alloc);
+ if (HaveInsertPoint() && !IsMSCatchParam) {
+ uint64_t size = CGM.getDataLayout().getTypeAllocSize(allocaTy);
+ emission.SizeForLifetimeMarkers =
+ EmitLifetimeStart(size, address.getPointer());
} else {
assert(!emission.useLifetimeMarkers());
}
@@ -984,11 +1053,11 @@
if (!DidCallStackSave) {
// Save the stack.
- llvm::Value *Stack = CreateTempAlloca(Int8PtrTy, "saved_stack");
+ Address Stack =
+ CreateTempAlloca(Int8PtrTy, getPointerAlign(), "saved_stack");
llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::stacksave);
- llvm::Value *V = Builder.CreateCall(F, {});
-
+ llvm::Value *V = Builder.CreateCall(F);
Builder.CreateStore(V, Stack);
DidCallStackSave = true;
@@ -1008,13 +1077,11 @@
llvm::AllocaInst *vla = Builder.CreateAlloca(llvmTy, elementCount, "vla");
vla->setAlignment(alignment.getQuantity());
- DeclPtr = vla;
+ address = Address(vla, alignment);
}
- llvm::Value *&DMEntry = LocalDeclMap[&D];
- assert(!DMEntry && "Decl already exists in localdeclmap!");
- DMEntry = DeclPtr;
- emission.Address = DeclPtr;
+ setAddrOfLocalVar(&D, address);
+ emission.Addr = address;
// Emit debug info for local var declaration.
if (HaveInsertPoint())
@@ -1022,12 +1089,12 @@
if (CGM.getCodeGenOpts().getDebugInfo()
>= CodeGenOptions::LimitedDebugInfo) {
DI->setLocation(D.getLocation());
- DI->EmitDeclareOfAutoVariable(&D, DeclPtr, Builder);
+ DI->EmitDeclareOfAutoVariable(&D, address.getPointer(), Builder);
}
}
if (D.hasAttr<AnnotateAttr>())
- EmitVarAnnotations(&D, emission.Address);
+ EmitVarAnnotations(&D, address.getPointer());
return emission;
}
@@ -1074,8 +1141,8 @@
return false;
}
- for (Stmt::const_child_range children = e->children(); children; ++children)
- if (isCapturedBy(var, cast<Expr>(*children)))
+ for (const Stmt *SubStmt : e->children())
+ if (isCapturedBy(var, cast<Expr>(SubStmt)))
return true;
return false;
@@ -1123,15 +1190,13 @@
if (isTrivialInitializer(Init))
return;
- CharUnits alignment = emission.Alignment;
-
// Check whether this is a byref variable that's potentially
// captured and moved by its own initializer. If so, we'll need to
// emit the initializer first, then copy into the variable.
bool capturedByInit = emission.IsByRef && isCapturedBy(D, Init);
- llvm::Value *Loc =
- capturedByInit ? emission.Address : emission.getObjectAddress(*this);
+ Address Loc =
+ capturedByInit ? emission.Addr : emission.getObjectAddress(*this);
llvm::Constant *constant = nullptr;
if (emission.IsConstantAggregate || D.isConstexpr()) {
@@ -1140,14 +1205,14 @@
}
if (!constant) {
- LValue lv = MakeAddrLValue(Loc, type, alignment);
+ LValue lv = MakeAddrLValue(Loc, type);
lv.setNonGC(true);
return EmitExprAsInit(Init, &D, lv, capturedByInit);
}
if (!emission.IsConstantAggregate) {
// For simple scalar/complex initialization, store the value directly.
- LValue lv = MakeAddrLValue(Loc, type, alignment);
+ LValue lv = MakeAddrLValue(Loc, type);
lv.setNonGC(true);
return EmitStoreThroughLValue(RValue::get(constant), lv, true);
}
@@ -1161,7 +1226,7 @@
getContext().getTypeSizeInChars(type).getQuantity());
llvm::Type *BP = Int8PtrTy;
- if (Loc->getType() != BP)
+ if (Loc.getType() != BP)
Loc = Builder.CreateBitCast(Loc, BP);
// If the initializer is all or mostly zeros, codegen with memset then do
@@ -1169,11 +1234,12 @@
if (shouldUseMemSetPlusStoresToInitialize(constant,
CGM.getDataLayout().getTypeAllocSize(constant->getType()))) {
Builder.CreateMemSet(Loc, llvm::ConstantInt::get(Int8Ty, 0), SizeVal,
- alignment.getQuantity(), isVolatile);
+ isVolatile);
// Zero and undef don't require a stores.
if (!constant->isNullValue() && !isa<llvm::UndefValue>(constant)) {
Loc = Builder.CreateBitCast(Loc, constant->getType()->getPointerTo());
- emitStoresForInitAfterMemset(constant, Loc, isVolatile, Builder);
+ emitStoresForInitAfterMemset(constant, Loc.getPointer(),
+ isVolatile, Builder);
}
} else {
// Otherwise, create a temporary global with the initializer then
@@ -1183,15 +1249,14 @@
new llvm::GlobalVariable(CGM.getModule(), constant->getType(), true,
llvm::GlobalValue::PrivateLinkage,
constant, Name);
- GV->setAlignment(alignment.getQuantity());
+ GV->setAlignment(Loc.getAlignment().getQuantity());
GV->setUnnamedAddr(true);
- llvm::Value *SrcPtr = GV;
- if (SrcPtr->getType() != BP)
+ Address SrcPtr = Address(GV, Loc.getAlignment());
+ if (SrcPtr.getType() != BP)
SrcPtr = Builder.CreateBitCast(SrcPtr, BP);
- Builder.CreateMemCpy(Loc, SrcPtr, SizeVal, alignment.getQuantity(),
- isVolatile);
+ Builder.CreateMemCpy(Loc, SrcPtr, SizeVal, isVolatile);
}
}
@@ -1252,7 +1317,7 @@
// Note that for __block variables, we want to destroy the
// original stack object, not the possibly forwarded object.
- llvm::Value *addr = emission.getObjectAddress(*this);
+ Address addr = emission.getObjectAddress(*this);
const VarDecl *var = emission.Variable;
QualType type = var->getType();
@@ -1270,8 +1335,8 @@
if (emission.NRVOFlag) {
assert(!type->isArrayType());
CXXDestructorDecl *dtor = type->getAsCXXRecordDecl()->getDestructor();
- EHStack.pushCleanup<DestroyNRVOVariable>(cleanupKind, addr, dtor,
- emission.NRVOFlag);
+ EHStack.pushCleanup<DestroyNRVOVariable>(cleanupKind, addr,
+ dtor, emission.NRVOFlag);
return;
}
break;
@@ -1368,7 +1433,7 @@
/// pushEHDestroy - Push the standard destructor for the given type as
/// an EH-only cleanup.
void CodeGenFunction::pushEHDestroy(QualType::DestructionKind dtorKind,
- llvm::Value *addr, QualType type) {
+ Address addr, QualType type) {
assert(dtorKind && "cannot push destructor for trivial type");
assert(needsEHCleanup(dtorKind));
@@ -1378,7 +1443,7 @@
/// pushDestroy - Push the standard destructor for the given type as
/// at least a normal cleanup.
void CodeGenFunction::pushDestroy(QualType::DestructionKind dtorKind,
- llvm::Value *addr, QualType type) {
+ Address addr, QualType type) {
assert(dtorKind && "cannot push destructor for trivial type");
CleanupKind cleanupKind = getCleanupKind(dtorKind);
@@ -1386,19 +1451,19 @@
cleanupKind & EHCleanup);
}
-void CodeGenFunction::pushDestroy(CleanupKind cleanupKind, llvm::Value *addr,
+void CodeGenFunction::pushDestroy(CleanupKind cleanupKind, Address addr,
QualType type, Destroyer *destroyer,
bool useEHCleanupForArray) {
pushFullExprCleanup<DestroyObject>(cleanupKind, addr, type,
destroyer, useEHCleanupForArray);
}
-void CodeGenFunction::pushStackRestore(CleanupKind Kind, llvm::Value *SPMem) {
+void CodeGenFunction::pushStackRestore(CleanupKind Kind, Address SPMem) {
EHStack.pushCleanup<CallStackRestore>(Kind, SPMem);
}
void CodeGenFunction::pushLifetimeExtendedDestroy(
- CleanupKind cleanupKind, llvm::Value *addr, QualType type,
+ CleanupKind cleanupKind, Address addr, QualType type,
Destroyer *destroyer, bool useEHCleanupForArray) {
assert(!isInConditionalBranch() &&
"performing lifetime extension from within conditional");
@@ -1428,15 +1493,18 @@
/// \param useEHCleanupForArray - whether an EH cleanup should be
/// used when destroying array elements, in case one of the
/// destructions throws an exception
-void CodeGenFunction::emitDestroy(llvm::Value *addr, QualType type,
+void CodeGenFunction::emitDestroy(Address addr, QualType type,
Destroyer *destroyer,
bool useEHCleanupForArray) {
const ArrayType *arrayType = getContext().getAsArrayType(type);
if (!arrayType)
return destroyer(*this, addr, type);
- llvm::Value *begin = addr;
- llvm::Value *length = emitArrayLength(arrayType, type, begin);
+ llvm::Value *length = emitArrayLength(arrayType, type, addr);
+
+ CharUnits elementAlign =
+ addr.getAlignment()
+ .alignmentOfArrayElement(getContext().getTypeSizeInChars(type));
// Normally we have to check whether the array is zero-length.
bool checkZeroLength = true;
@@ -1448,8 +1516,9 @@
checkZeroLength = false;
}
+ llvm::Value *begin = addr.getPointer();
llvm::Value *end = Builder.CreateInBoundsGEP(begin, length);
- emitArrayDestroy(begin, end, type, destroyer,
+ emitArrayDestroy(begin, end, type, elementAlign, destroyer,
checkZeroLength, useEHCleanupForArray);
}
@@ -1458,18 +1527,19 @@
///
/// \param begin - a type* denoting the first element of the array
/// \param end - a type* denoting one past the end of the array
-/// \param type - the element type of the array
+/// \param elementType - the element type of the array
/// \param destroyer - the function to call to destroy elements
/// \param useEHCleanup - whether to push an EH cleanup to destroy
/// the remaining elements in case the destruction of a single
/// element throws
void CodeGenFunction::emitArrayDestroy(llvm::Value *begin,
llvm::Value *end,
- QualType type,
+ QualType elementType,
+ CharUnits elementAlign,
Destroyer *destroyer,
bool checkZeroLength,
bool useEHCleanup) {
- assert(!type->isArrayType());
+ assert(!elementType->isArrayType());
// The basic structure here is a do-while loop, because we don't
// need to check for the zero-element case.
@@ -1495,10 +1565,11 @@
"arraydestroy.element");
if (useEHCleanup)
- pushRegularPartialArrayCleanup(begin, element, type, destroyer);
+ pushRegularPartialArrayCleanup(begin, element, elementType, elementAlign,
+ destroyer);
// Perform the actual destruction there.
- destroyer(*this, element, type);
+ destroyer(*this, Address(element, elementAlign), elementType);
if (useEHCleanup)
PopCleanupBlock();
@@ -1516,7 +1587,7 @@
/// emitArrayDestroy, the element type here may still be an array type.
static void emitPartialArrayDestroy(CodeGenFunction &CGF,
llvm::Value *begin, llvm::Value *end,
- QualType type,
+ QualType type, CharUnits elementAlign,
CodeGenFunction::Destroyer *destroyer) {
// If the element type is itself an array, drill down.
unsigned arrayDepth = 0;
@@ -1528,9 +1599,9 @@
}
if (arrayDepth) {
- llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, arrayDepth+1);
+ llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0);
- SmallVector<llvm::Value*,4> gepIndices(arrayDepth, zero);
+ SmallVector<llvm::Value*,4> gepIndices(arrayDepth+1, zero);
begin = CGF.Builder.CreateInBoundsGEP(begin, gepIndices, "pad.arraybegin");
end = CGF.Builder.CreateInBoundsGEP(end, gepIndices, "pad.arrayend");
}
@@ -1538,7 +1609,7 @@
// Destroy the array. We don't ever need an EH cleanup because we
// assume that we're in an EH cleanup ourselves, so a throwing
// destructor causes an immediate terminate.
- CGF.emitArrayDestroy(begin, end, type, destroyer,
+ CGF.emitArrayDestroy(begin, end, type, elementAlign, destroyer,
/*checkZeroLength*/ true, /*useEHCleanup*/ false);
}
@@ -1546,44 +1617,49 @@
/// RegularPartialArrayDestroy - a cleanup which performs a partial
/// array destroy where the end pointer is regularly determined and
/// does not need to be loaded from a local.
- class RegularPartialArrayDestroy : public EHScopeStack::Cleanup {
+ class RegularPartialArrayDestroy final : public EHScopeStack::Cleanup {
llvm::Value *ArrayBegin;
llvm::Value *ArrayEnd;
QualType ElementType;
CodeGenFunction::Destroyer *Destroyer;
+ CharUnits ElementAlign;
public:
RegularPartialArrayDestroy(llvm::Value *arrayBegin, llvm::Value *arrayEnd,
- QualType elementType,
+ QualType elementType, CharUnits elementAlign,
CodeGenFunction::Destroyer *destroyer)
: ArrayBegin(arrayBegin), ArrayEnd(arrayEnd),
- ElementType(elementType), Destroyer(destroyer) {}
+ ElementType(elementType), Destroyer(destroyer),
+ ElementAlign(elementAlign) {}
void Emit(CodeGenFunction &CGF, Flags flags) override {
emitPartialArrayDestroy(CGF, ArrayBegin, ArrayEnd,
- ElementType, Destroyer);
+ ElementType, ElementAlign, Destroyer);
}
};
/// IrregularPartialArrayDestroy - a cleanup which performs a
/// partial array destroy where the end pointer is irregularly
/// determined and must be loaded from a local.
- class IrregularPartialArrayDestroy : public EHScopeStack::Cleanup {
+ class IrregularPartialArrayDestroy final : public EHScopeStack::Cleanup {
llvm::Value *ArrayBegin;
- llvm::Value *ArrayEndPointer;
+ Address ArrayEndPointer;
QualType ElementType;
CodeGenFunction::Destroyer *Destroyer;
+ CharUnits ElementAlign;
public:
IrregularPartialArrayDestroy(llvm::Value *arrayBegin,
- llvm::Value *arrayEndPointer,
+ Address arrayEndPointer,
QualType elementType,
+ CharUnits elementAlign,
CodeGenFunction::Destroyer *destroyer)
: ArrayBegin(arrayBegin), ArrayEndPointer(arrayEndPointer),
- ElementType(elementType), Destroyer(destroyer) {}
+ ElementType(elementType), Destroyer(destroyer),
+ ElementAlign(elementAlign) {}
void Emit(CodeGenFunction &CGF, Flags flags) override {
llvm::Value *arrayEnd = CGF.Builder.CreateLoad(ArrayEndPointer);
emitPartialArrayDestroy(CGF, ArrayBegin, arrayEnd,
- ElementType, Destroyer);
+ ElementType, ElementAlign, Destroyer);
}
};
}
@@ -1595,12 +1671,14 @@
/// \param elementType - the immediate element type of the array;
/// possibly still an array type
void CodeGenFunction::pushIrregularPartialArrayCleanup(llvm::Value *arrayBegin,
- llvm::Value *arrayEndPointer,
+ Address arrayEndPointer,
QualType elementType,
+ CharUnits elementAlign,
Destroyer *destroyer) {
pushFullExprCleanup<IrregularPartialArrayDestroy>(EHCleanup,
arrayBegin, arrayEndPointer,
- elementType, destroyer);
+ elementType, elementAlign,
+ destroyer);
}
/// pushRegularPartialArrayCleanup - Push an EH cleanup to destroy
@@ -1612,10 +1690,12 @@
void CodeGenFunction::pushRegularPartialArrayCleanup(llvm::Value *arrayBegin,
llvm::Value *arrayEnd,
QualType elementType,
+ CharUnits elementAlign,
Destroyer *destroyer) {
pushFullExprCleanup<RegularPartialArrayDestroy>(EHCleanup,
arrayBegin, arrayEnd,
- elementType, destroyer);
+ elementType, elementAlign,
+ destroyer);
}
/// Lazily declare the @llvm.lifetime.start intrinsic.
@@ -1639,7 +1719,7 @@
/// function. This is used to balance out the incoming +1 of a
/// ns_consumed argument when we can't reasonably do that just by
/// not doing the initial retain for a __block argument.
- struct ConsumeARCParameter : EHScopeStack::Cleanup {
+ struct ConsumeARCParameter final : EHScopeStack::Cleanup {
ConsumeARCParameter(llvm::Value *param,
ARCPreciseLifetime_t precise)
: Param(param), Precise(precise) {}
@@ -1655,56 +1735,38 @@
/// Emit an alloca (or GlobalValue depending on target)
/// for the specified parameter and set up LocalDeclMap.
-void CodeGenFunction::EmitParmDecl(const VarDecl &D, llvm::Value *Arg,
- bool ArgIsPointer, unsigned ArgNo) {
+void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg,
+ unsigned ArgNo) {
// FIXME: Why isn't ImplicitParamDecl a ParmVarDecl?
assert((isa<ParmVarDecl>(D) || isa<ImplicitParamDecl>(D)) &&
"Invalid argument to EmitParmDecl");
- Arg->setName(D.getName());
+ Arg.getAnyValue()->setName(D.getName());
QualType Ty = D.getType();
// Use better IR generation for certain implicit parameters.
- if (isa<ImplicitParamDecl>(D)) {
+ if (auto IPD = dyn_cast<ImplicitParamDecl>(&D)) {
// The only implicit argument a block has is its literal.
+ // We assume this is always passed directly.
if (BlockInfo) {
- LocalDeclMap[&D] = Arg;
- llvm::Value *LocalAddr = nullptr;
- if (CGM.getCodeGenOpts().OptimizationLevel == 0) {
- // Allocate a stack slot to let the debug info survive the RA.
- llvm::AllocaInst *Alloc = CreateTempAlloca(ConvertTypeForMem(Ty),
- D.getName() + ".addr");
- Alloc->setAlignment(getContext().getDeclAlign(&D).getQuantity());
- LValue lv = MakeAddrLValue(Alloc, Ty, getContext().getDeclAlign(&D));
- EmitStoreOfScalar(Arg, lv, /* isInitialization */ true);
- LocalAddr = Builder.CreateLoad(Alloc);
- }
-
- if (CGDebugInfo *DI = getDebugInfo()) {
- if (CGM.getCodeGenOpts().getDebugInfo()
- >= CodeGenOptions::LimitedDebugInfo) {
- DI->setLocation(D.getLocation());
- DI->EmitDeclareOfBlockLiteralArgVariable(*BlockInfo, Arg, ArgNo,
- LocalAddr, Builder);
- }
- }
-
+ setBlockContextParameter(IPD, ArgNo, Arg.getDirectValue());
return;
}
}
- llvm::Value *DeclPtr;
+ Address DeclPtr = Address::invalid();
bool DoStore = false;
bool IsScalar = hasScalarEvaluationKind(Ty);
- CharUnits Align = getContext().getDeclAlign(&D);
// If we already have a pointer to the argument, reuse the input pointer.
- if (ArgIsPointer) {
+ if (Arg.isIndirect()) {
+ DeclPtr = Arg.getIndirectAddress();
// If we have a prettier pointer type at this point, bitcast to that.
- unsigned AS = cast<llvm::PointerType>(Arg->getType())->getAddressSpace();
+ unsigned AS = DeclPtr.getType()->getAddressSpace();
llvm::Type *IRTy = ConvertTypeForMem(Ty)->getPointerTo(AS);
- DeclPtr = Arg->getType() == IRTy ? Arg : Builder.CreateBitCast(Arg, IRTy,
- D.getName());
+ if (DeclPtr.getType() != IRTy)
+ DeclPtr = Builder.CreateBitCast(DeclPtr, IRTy, D.getName());
+
// Push a destructor cleanup for this parameter if the ABI requires it.
// Don't push a cleanup in a thunk for a method that will also emit a
// cleanup.
@@ -1716,14 +1778,14 @@
}
} else {
// Otherwise, create a temporary to hold the value.
- llvm::AllocaInst *Alloc = CreateTempAlloca(ConvertTypeForMem(Ty),
- D.getName() + ".addr");
- Alloc->setAlignment(Align.getQuantity());
- DeclPtr = Alloc;
+ DeclPtr = CreateMemTemp(Ty, getContext().getDeclAlign(&D),
+ D.getName() + ".addr");
DoStore = true;
}
- LValue lv = MakeAddrLValue(DeclPtr, Ty, Align);
+ llvm::Value *ArgVal = (DoStore ? Arg.getDirectValue() : nullptr);
+
+ LValue lv = MakeAddrLValue(DeclPtr, Ty);
if (IsScalar) {
Qualifiers qs = Ty.getQualifiers();
if (Qualifiers::ObjCLifetime lt = qs.getObjCLifetime()) {
@@ -1753,26 +1815,26 @@
// objc_storeStrong attempts to release its old value.
llvm::Value *Null = CGM.EmitNullConstant(D.getType());
EmitStoreOfScalar(Null, lv, /* isInitialization */ true);
- EmitARCStoreStrongCall(lv.getAddress(), Arg, true);
+ EmitARCStoreStrongCall(lv.getAddress(), ArgVal, true);
DoStore = false;
}
else
// Don't use objc_retainBlock for block pointers, because we
// don't want to Block_copy something just because we got it
// as a parameter.
- Arg = EmitARCRetainNonBlock(Arg);
+ ArgVal = EmitARCRetainNonBlock(ArgVal);
}
} else {
// Push the cleanup for a consumed parameter.
if (isConsumed) {
ARCPreciseLifetime_t precise = (D.hasAttr<ObjCPreciseLifetimeAttr>()
? ARCPreciseLifetime : ARCImpreciseLifetime);
- EHStack.pushCleanup<ConsumeARCParameter>(getARCCleanupKind(), Arg,
+ EHStack.pushCleanup<ConsumeARCParameter>(getARCCleanupKind(), ArgVal,
precise);
}
if (lt == Qualifiers::OCL_Weak) {
- EmitARCInitWeak(DeclPtr, Arg);
+ EmitARCInitWeak(DeclPtr, ArgVal);
DoStore = false; // The weak init is a store, no need to do two.
}
}
@@ -1784,20 +1846,18 @@
// Store the initial value into the alloca.
if (DoStore)
- EmitStoreOfScalar(Arg, lv, /* isInitialization */ true);
+ EmitStoreOfScalar(ArgVal, lv, /* isInitialization */ true);
- llvm::Value *&DMEntry = LocalDeclMap[&D];
- assert(!DMEntry && "Decl already exists in localdeclmap!");
- DMEntry = DeclPtr;
+ setAddrOfLocalVar(&D, DeclPtr);
// Emit debug info for param declaration.
if (CGDebugInfo *DI = getDebugInfo()) {
if (CGM.getCodeGenOpts().getDebugInfo()
>= CodeGenOptions::LimitedDebugInfo) {
- DI->EmitDeclareOfArgVariable(&D, DeclPtr, ArgNo, Builder);
+ DI->EmitDeclareOfArgVariable(&D, DeclPtr.getPointer(), ArgNo, Builder);
}
}
if (D.hasAttr<AnnotateAttr>())
- EmitVarAnnotations(&D, DeclPtr);
+ EmitVarAnnotations(&D, DeclPtr.getPointer());
}
diff --git a/lib/CodeGen/CGDeclCXX.cpp b/lib/CodeGen/CGDeclCXX.cpp
index 50a4996..adba731 100644
--- a/lib/CodeGen/CGDeclCXX.cpp
+++ b/lib/CodeGen/CGDeclCXX.cpp
@@ -24,16 +24,13 @@
using namespace CodeGen;
static void EmitDeclInit(CodeGenFunction &CGF, const VarDecl &D,
- llvm::Constant *DeclPtr) {
+ ConstantAddress DeclPtr) {
assert(D.hasGlobalStorage() && "VarDecl must have global storage!");
assert(!D.getType()->isReferenceType() &&
"Should not call EmitDeclInit on a reference!");
- ASTContext &Context = CGF.getContext();
-
- CharUnits alignment = Context.getDeclAlign(&D);
QualType type = D.getType();
- LValue lv = CGF.MakeAddrLValue(DeclPtr, type, alignment);
+ LValue lv = CGF.MakeAddrLValue(DeclPtr, type);
const Expr *Init = D.getInit();
switch (CGF.getEvaluationKind(type)) {
@@ -64,7 +61,7 @@
/// Emit code to cause the destruction of the given variable with
/// static storage duration.
static void EmitDeclDestroy(CodeGenFunction &CGF, const VarDecl &D,
- llvm::Constant *addr) {
+ ConstantAddress addr) {
CodeGenModule &CGM = CGF.CGM;
// FIXME: __attribute__((cleanup)) ?
@@ -99,7 +96,7 @@
function = CGM.getAddrOfCXXStructor(dtor, StructorType::Complete);
argument = llvm::ConstantExpr::getBitCast(
- addr, CGF.getTypes().ConvertType(type)->getPointerTo());
+ addr.getPointer(), CGF.getTypes().ConvertType(type)->getPointerTo());
// Otherwise, the standard logic requires a helper function.
} else {
@@ -162,25 +159,26 @@
DeclPtr = llvm::ConstantExpr::getAddrSpaceCast(DeclPtr, PTy);
}
+ ConstantAddress DeclAddr(DeclPtr, getContext().getDeclAlign(&D));
+
if (!T->isReferenceType()) {
if (getLangOpts().OpenMP && D.hasAttr<OMPThreadPrivateDeclAttr>())
(void)CGM.getOpenMPRuntime().emitThreadPrivateVarDefinition(
- &D, DeclPtr, D.getAttr<OMPThreadPrivateDeclAttr>()->getLocation(),
+ &D, DeclAddr, D.getAttr<OMPThreadPrivateDeclAttr>()->getLocation(),
PerformInit, this);
if (PerformInit)
- EmitDeclInit(*this, D, DeclPtr);
+ EmitDeclInit(*this, D, DeclAddr);
if (CGM.isTypeConstant(D.getType(), true))
EmitDeclInvariant(*this, D, DeclPtr);
else
- EmitDeclDestroy(*this, D, DeclPtr);
+ EmitDeclDestroy(*this, D, DeclAddr);
return;
}
assert(PerformInit && "cannot have constant initializer which needs "
"destruction for reference");
- unsigned Alignment = getContext().getDeclAlign(&D).getQuantity();
RValue RV = EmitReferenceBindingToExpr(Init);
- EmitStoreOfScalar(RV.getScalarVal(), DeclPtr, false, Alignment, T);
+ EmitStoreOfScalar(RV.getScalarVal(), DeclAddr, false, T);
}
/// Create a stub function, suitable for being passed to atexit,
@@ -195,13 +193,15 @@
llvm::raw_svector_ostream Out(FnName);
CGM.getCXXABI().getMangleContext().mangleDynamicAtExitDestructor(&VD, Out);
}
+
+ const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
llvm::Function *fn = CGM.CreateGlobalInitOrDestructFunction(ty, FnName.str(),
+ FI,
VD.getLocation());
CodeGenFunction CGF(CGM);
- CGF.StartFunction(&VD, CGM.getContext().VoidTy, fn,
- CGM.getTypes().arrangeNullaryFunction(), FunctionArgList());
+ CGF.StartFunction(&VD, CGM.getContext().VoidTy, fn, FI, FunctionArgList());
llvm::CallInst *call = CGF.Builder.CreateCall(dtor, addr);
@@ -249,7 +249,8 @@
}
llvm::Function *CodeGenModule::CreateGlobalInitOrDestructFunction(
- llvm::FunctionType *FTy, const Twine &Name, SourceLocation Loc, bool TLS) {
+ llvm::FunctionType *FTy, const Twine &Name, const CGFunctionInfo &FI,
+ SourceLocation Loc, bool TLS) {
llvm::Function *Fn =
llvm::Function::Create(FTy, llvm::GlobalValue::InternalLinkage,
Name, &getModule());
@@ -259,7 +260,7 @@
Fn->setSection(Section);
}
- SetLLVMFunctionAttributes(nullptr, getTypes().arrangeNullaryFunction(), Fn);
+ SetInternalFunctionAttributes(nullptr, Fn, FI);
Fn->setCallingConv(getRuntimeCC());
@@ -267,7 +268,8 @@
Fn->setDoesNotThrow();
if (!isInSanitizerBlacklist(Fn, Loc)) {
- if (getLangOpts().Sanitize.has(SanitizerKind::Address))
+ if (getLangOpts().Sanitize.hasOneOf(SanitizerKind::Address |
+ SanitizerKind::KernelAddress))
Fn->addFnAttr(llvm::Attribute::SanitizeAddress);
if (getLangOpts().Sanitize.has(SanitizerKind::Thread))
Fn->addFnAttr(llvm::Attribute::SanitizeThread);
@@ -316,7 +318,9 @@
// Create a variable initialization function.
llvm::Function *Fn =
- CreateGlobalInitOrDestructFunction(FTy, FnName.str(), D->getLocation());
+ CreateGlobalInitOrDestructFunction(FTy, FnName.str(),
+ getTypes().arrangeNullaryFunction(),
+ D->getLocation());
auto *ISA = D->getAttr<InitSegAttr>();
CodeGenFunction(*this).GenerateCXXGlobalVarDeclInitFunc(Fn, D, Addr,
@@ -333,7 +337,7 @@
// FIXME: We only need to register one __cxa_thread_atexit function for the
// entire TU.
CXXThreadLocalInits.push_back(Fn);
- CXXThreadLocalInitVars.push_back(Addr);
+ CXXThreadLocalInitVars.push_back(D);
} else if (PerformInit && ISA) {
EmitPointerToInitFunc(D, Addr, Fn, ISA);
} else if (auto *IPA = D->getAttr<InitPriorityAttr>()) {
@@ -391,7 +395,7 @@
return;
llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false);
-
+ const CGFunctionInfo &FI = getTypes().arrangeNullaryFunction();
// Create our global initialization function.
if (!PrioritizedCXXGlobalInits.empty()) {
@@ -415,7 +419,7 @@
// Priority is always <= 65535 (enforced by sema).
PrioritySuffix = std::string(6-PrioritySuffix.size(), '0')+PrioritySuffix;
llvm::Function *Fn = CreateGlobalInitOrDestructFunction(
- FTy, "_GLOBAL__I_" + PrioritySuffix);
+ FTy, "_GLOBAL__I_" + PrioritySuffix, FI);
for (; I < PrioE; ++I)
LocalCXXGlobalInits.push_back(I->second);
@@ -423,6 +427,7 @@
CodeGenFunction(*this).GenerateCXXGlobalInitFunc(Fn, LocalCXXGlobalInits);
AddGlobalCtor(Fn, Priority);
}
+ PrioritizedCXXGlobalInits.clear();
}
SmallString<128> FileName;
@@ -444,13 +449,12 @@
}
llvm::Function *Fn = CreateGlobalInitOrDestructFunction(
- FTy, llvm::Twine("_GLOBAL__sub_I_", FileName));
+ FTy, llvm::Twine("_GLOBAL__sub_I_", FileName), FI);
CodeGenFunction(*this).GenerateCXXGlobalInitFunc(Fn, CXXGlobalInits);
AddGlobalCtor(Fn);
CXXGlobalInits.clear();
- PrioritizedCXXGlobalInits.clear();
}
void CodeGenModule::EmitCXXGlobalDtorFunc() {
@@ -460,7 +464,9 @@
llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false);
// Create our global destructor function.
- llvm::Function *Fn = CreateGlobalInitOrDestructFunction(FTy, "_GLOBAL__D_a");
+ const CGFunctionInfo &FI = getTypes().arrangeNullaryFunction();
+ llvm::Function *Fn =
+ CreateGlobalInitOrDestructFunction(FTy, "_GLOBAL__D_a", FI);
CodeGenFunction(*this).GenerateCXXGlobalDtorsFunc(Fn, CXXGlobalDtors);
AddGlobalDtor(Fn);
@@ -497,7 +503,7 @@
void
CodeGenFunction::GenerateCXXGlobalInitFunc(llvm::Function *Fn,
ArrayRef<llvm::Function *> Decls,
- llvm::GlobalVariable *Guard) {
+ Address Guard) {
{
auto NL = ApplyDebugLocation::CreateEmpty(*this);
StartFunction(GlobalDecl(), getContext().VoidTy, Fn,
@@ -506,20 +512,20 @@
auto AL = ApplyDebugLocation::CreateArtificial(*this);
llvm::BasicBlock *ExitBlock = nullptr;
- if (Guard) {
+ if (Guard.isValid()) {
// If we have a guard variable, check whether we've already performed
// these initializations. This happens for TLS initialization functions.
llvm::Value *GuardVal = Builder.CreateLoad(Guard);
llvm::Value *Uninit = Builder.CreateIsNull(GuardVal,
"guard.uninitialized");
- // Mark as initialized before initializing anything else. If the
- // initializers use previously-initialized thread_local vars, that's
- // probably supposed to be OK, but the standard doesn't say.
- Builder.CreateStore(llvm::ConstantInt::get(GuardVal->getType(),1), Guard);
llvm::BasicBlock *InitBlock = createBasicBlock("init");
ExitBlock = createBasicBlock("exit");
Builder.CreateCondBr(Uninit, InitBlock, ExitBlock);
EmitBlock(InitBlock);
+ // Mark as initialized before initializing anything else. If the
+ // initializers use previously-initialized thread_local vars, that's
+ // probably supposed to be OK, but the standard doesn't say.
+ Builder.CreateStore(llvm::ConstantInt::get(GuardVal->getType(),1), Guard);
}
RunCleanupsScope Scope(*this);
@@ -571,9 +577,10 @@
}
/// generateDestroyHelper - Generates a helper function which, when
-/// invoked, destroys the given object.
+/// invoked, destroys the given object. The address of the object
+/// should be in global memory.
llvm::Function *CodeGenFunction::generateDestroyHelper(
- llvm::Constant *addr, QualType type, Destroyer *destroyer,
+ Address addr, QualType type, Destroyer *destroyer,
bool useEHCleanupForArray, const VarDecl *VD) {
FunctionArgList args;
ImplicitParamDecl dst(getContext(), nullptr, SourceLocation(), nullptr,
@@ -584,7 +591,7 @@
getContext().VoidTy, args, FunctionType::ExtInfo(), /*variadic=*/false);
llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
llvm::Function *fn = CGM.CreateGlobalInitOrDestructFunction(
- FTy, "__cxx_global_array_dtor", VD->getLocation());
+ FTy, "__cxx_global_array_dtor", FI, VD->getLocation());
CurEHLocation = VD->getLocStart();
diff --git a/lib/CodeGen/CGException.cpp b/lib/CodeGen/CGException.cpp
index d9a3f0b..fce2e75 100644
--- a/lib/CodeGen/CGException.cpp
+++ b/lib/CodeGen/CGException.cpp
@@ -1,4 +1,4 @@
-//===--- CGException.cpp - Emit LLVM Code for C++ exceptions --------------===//
+//===--- CGException.cpp - Emit LLVM Code for C++ exceptions ----*- C++ -*-===//
//
// The LLVM Compiler Infrastructure
//
@@ -20,6 +20,7 @@
#include "clang/AST/StmtCXX.h"
#include "clang/AST/StmtObjC.h"
#include "clang/AST/StmtVisitor.h"
+#include "clang/Basic/TargetBuiltins.h"
#include "llvm/IR/CallSite.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicInst.h"
@@ -80,38 +81,6 @@
return CGM.CreateRuntimeFunction(FTy, Name);
}
-namespace {
- /// The exceptions personality for a function.
- struct EHPersonality {
- const char *PersonalityFn;
-
- // If this is non-null, this personality requires a non-standard
- // function for rethrowing an exception after a catchall cleanup.
- // This function must have prototype void(void*).
- const char *CatchallRethrowFn;
-
- static const EHPersonality &get(CodeGenModule &CGM,
- const FunctionDecl *FD);
- static const EHPersonality &get(CodeGenFunction &CGF) {
- return get(CGF.CGM, dyn_cast_or_null<FunctionDecl>(CGF.CurCodeDecl));
- }
-
- static const EHPersonality GNU_C;
- static const EHPersonality GNU_C_SJLJ;
- static const EHPersonality GNU_C_SEH;
- static const EHPersonality GNU_ObjC;
- static const EHPersonality GNUstep_ObjC;
- static const EHPersonality GNU_ObjCXX;
- static const EHPersonality NeXT_ObjC;
- static const EHPersonality GNU_CPlusPlus;
- static const EHPersonality GNU_CPlusPlus_SJLJ;
- static const EHPersonality GNU_CPlusPlus_SEH;
- static const EHPersonality MSVC_except_handler;
- static const EHPersonality MSVC_C_specific_handler;
- static const EHPersonality MSVC_CxxFrameHandler3;
- };
-}
-
const EHPersonality EHPersonality::GNU_C = { "__gcc_personality_v0", nullptr };
const EHPersonality
EHPersonality::GNU_C_SJLJ = { "__gcc_personality_sj0", nullptr };
@@ -160,6 +129,7 @@
return getCPersonality(T, L);
case ObjCRuntime::MacOSX:
case ObjCRuntime::iOS:
+ case ObjCRuntime::WatchOS:
return EHPersonality::NeXT_ObjC;
case ObjCRuntime::GNUstep:
if (L.ObjCRuntime.getVersion() >= VersionTuple(1, 7))
@@ -191,6 +161,7 @@
// function on targets using (backend-driven) SJLJ EH.
case ObjCRuntime::MacOSX:
case ObjCRuntime::iOS:
+ case ObjCRuntime::WatchOS:
return EHPersonality::NeXT_ObjC;
// In the fragile ABI, just use C++ exception handling and hope
@@ -220,14 +191,16 @@
const llvm::Triple &T = CGM.getTarget().getTriple();
const LangOptions &L = CGM.getLangOpts();
+ // Functions using SEH get an SEH personality.
+ if (FD && FD->usesSEHTry())
+ return getSEHPersonalityMSVC(T);
+
// Try to pick a personality function that is compatible with MSVC if we're
// not compiling Obj-C. Obj-C users better have an Obj-C runtime that supports
// the GCC-style personality function.
if (T.isWindowsMSVCEnvironment() && !L.ObjC1) {
if (L.SjLjExceptions)
return EHPersonality::GNU_CPlusPlus_SJLJ;
- else if (FD && FD->usesSEHTry())
- return getSEHPersonalityMSVC(T);
else
return EHPersonality::MSVC_CxxFrameHandler3;
}
@@ -242,6 +215,10 @@
return getCPersonality(T, L);
}
+const EHPersonality &EHPersonality::get(CodeGenFunction &CGF) {
+ return get(CGF.CGM, dyn_cast_or_null<FunctionDecl>(CGF.CurCodeDecl));
+}
+
static llvm::Constant *getPersonalityFn(CodeGenModule &CGM,
const EHPersonality &Personality) {
llvm::Constant *Fn =
@@ -256,6 +233,36 @@
return llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
}
+/// Check whether a landingpad instruction only uses C++ features.
+static bool LandingPadHasOnlyCXXUses(llvm::LandingPadInst *LPI) {
+ for (unsigned I = 0, E = LPI->getNumClauses(); I != E; ++I) {
+ // Look for something that would've been returned by the ObjC
+ // runtime's GetEHType() method.
+ llvm::Value *Val = LPI->getClause(I)->stripPointerCasts();
+ if (LPI->isCatch(I)) {
+ // Check if the catch value has the ObjC prefix.
+ if (llvm::GlobalVariable *GV = dyn_cast<llvm::GlobalVariable>(Val))
+ // ObjC EH selector entries are always global variables with
+ // names starting like this.
+ if (GV->getName().startswith("OBJC_EHTYPE"))
+ return false;
+ } else {
+ // Check if any of the filter values have the ObjC prefix.
+ llvm::Constant *CVal = cast<llvm::Constant>(Val);
+ for (llvm::User::op_iterator
+ II = CVal->op_begin(), IE = CVal->op_end(); II != IE; ++II) {
+ if (llvm::GlobalVariable *GV =
+ cast<llvm::GlobalVariable>((*II)->stripPointerCasts()))
+ // ObjC EH selector entries are always global variables with
+ // names starting like this.
+ if (GV->getName().startswith("OBJC_EHTYPE"))
+ return false;
+ }
+ }
+ }
+ return true;
+}
+
/// Check whether a personality function could reasonably be swapped
/// for a C++ personality function.
static bool PersonalityHasOnlyCXXUses(llvm::Constant *Fn) {
@@ -268,34 +275,14 @@
continue;
}
- // Otherwise, it has to be a landingpad instruction.
- llvm::LandingPadInst *LPI = dyn_cast<llvm::LandingPadInst>(U);
- if (!LPI) return false;
+ // Otherwise it must be a function.
+ llvm::Function *F = dyn_cast<llvm::Function>(U);
+ if (!F) return false;
- for (unsigned I = 0, E = LPI->getNumClauses(); I != E; ++I) {
- // Look for something that would've been returned by the ObjC
- // runtime's GetEHType() method.
- llvm::Value *Val = LPI->getClause(I)->stripPointerCasts();
- if (LPI->isCatch(I)) {
- // Check if the catch value has the ObjC prefix.
- if (llvm::GlobalVariable *GV = dyn_cast<llvm::GlobalVariable>(Val))
- // ObjC EH selector entries are always global variables with
- // names starting like this.
- if (GV->getName().startswith("OBJC_EHTYPE"))
- return false;
- } else {
- // Check if any of the filter values have the ObjC prefix.
- llvm::Constant *CVal = cast<llvm::Constant>(Val);
- for (llvm::User::op_iterator
- II = CVal->op_begin(), IE = CVal->op_end(); II != IE; ++II) {
- if (llvm::GlobalVariable *GV =
- cast<llvm::GlobalVariable>((*II)->stripPointerCasts()))
- // ObjC EH selector entries are always global variables with
- // names starting like this.
- if (GV->getName().startswith("OBJC_EHTYPE"))
- return false;
- }
- }
+ for (auto BB = F->begin(), E = F->end(); BB != E; ++BB) {
+ if (BB->isLandingPad())
+ if (!LandingPadHasOnlyCXXUses(BB->getLandingPadInst()))
+ return false;
}
}
@@ -354,29 +341,29 @@
namespace {
/// A cleanup to free the exception object if its initialization
/// throws.
- struct FreeException : EHScopeStack::Cleanup {
+ struct FreeException final : EHScopeStack::Cleanup {
llvm::Value *exn;
FreeException(llvm::Value *exn) : exn(exn) {}
void Emit(CodeGenFunction &CGF, Flags flags) override {
CGF.EmitNounwindRuntimeCall(getFreeExceptionFn(CGF.CGM), exn);
}
};
-}
+} // end anonymous namespace
// Emits an exception expression into the given location. This
// differs from EmitAnyExprToMem only in that, if a final copy-ctor
// call is required, an exception within that copy ctor causes
// std::terminate to be invoked.
-void CodeGenFunction::EmitAnyExprToExn(const Expr *e, llvm::Value *addr) {
+void CodeGenFunction::EmitAnyExprToExn(const Expr *e, Address addr) {
// Make sure the exception object is cleaned up if there's an
// exception during initialization.
- pushFullExprCleanup<FreeException>(EHCleanup, addr);
+ pushFullExprCleanup<FreeException>(EHCleanup, addr.getPointer());
EHScopeStack::stable_iterator cleanup = EHStack.stable_begin();
// __cxa_allocate_exception returns a void*; we need to cast this
// to the appropriate type for the object.
llvm::Type *ty = ConvertTypeForMem(e->getType())->getPointerTo();
- llvm::Value *typedAddr = Builder.CreateBitCast(addr, ty);
+ Address typedAddr = Builder.CreateBitCast(addr, ty);
// FIXME: this isn't quite right! If there's a final unelided call
// to a copy constructor, then according to [except.terminate]p1 we
@@ -389,19 +376,20 @@
/*IsInit*/ true);
// Deactivate the cleanup block.
- DeactivateCleanupBlock(cleanup, cast<llvm::Instruction>(typedAddr));
+ DeactivateCleanupBlock(cleanup,
+ cast<llvm::Instruction>(typedAddr.getPointer()));
}
-llvm::Value *CodeGenFunction::getExceptionSlot() {
+Address CodeGenFunction::getExceptionSlot() {
if (!ExceptionSlot)
ExceptionSlot = CreateTempAlloca(Int8PtrTy, "exn.slot");
- return ExceptionSlot;
+ return Address(ExceptionSlot, getPointerAlign());
}
-llvm::Value *CodeGenFunction::getEHSelectorSlot() {
+Address CodeGenFunction::getEHSelectorSlot() {
if (!EHSelectorSlot)
EHSelectorSlot = CreateTempAlloca(Int32Ty, "ehselector.slot");
- return EHSelectorSlot;
+ return Address(EHSelectorSlot, CharUnits::fromQuantity(4));
}
llvm::Value *CodeGenFunction::getExceptionFromSlot() {
@@ -570,22 +558,25 @@
QualType CaughtType = CGM.getContext().getUnqualifiedArrayType(
C->getCaughtType().getNonReferenceType(), CaughtTypeQuals);
- llvm::Constant *TypeInfo = nullptr;
+ CatchTypeInfo TypeInfo{nullptr, 0};
if (CaughtType->isObjCObjectPointerType())
- TypeInfo = CGM.getObjCRuntime().GetEHType(CaughtType);
+ TypeInfo.RTTI = CGM.getObjCRuntime().GetEHType(CaughtType);
else
- TypeInfo =
- CGM.getAddrOfCXXCatchHandlerType(CaughtType, C->getCaughtType());
+ TypeInfo = CGM.getCXXABI().getAddrOfCXXCatchHandlerType(
+ CaughtType, C->getCaughtType());
CatchScope->setHandler(I, TypeInfo, Handler);
} else {
// No exception decl indicates '...', a catch-all.
- CatchScope->setCatchAllHandler(I, Handler);
+ CatchScope->setHandler(I, CGM.getCXXABI().getCatchAllTypeInfo(), Handler);
}
}
}
llvm::BasicBlock *
CodeGenFunction::getEHDispatchBlock(EHScopeStack::stable_iterator si) {
+ if (EHPersonality::get(*this).usesFuncletPads())
+ return getMSVCDispatchBlock(si);
+
// The dispatch block for the end of the scope chain is a block that
// just resumes unwinding.
if (si == EHStack.stable_end())
@@ -622,12 +613,58 @@
case EHScope::Terminate:
dispatchBlock = getTerminateHandler();
break;
+
+ case EHScope::PadEnd:
+ llvm_unreachable("PadEnd unnecessary for Itanium!");
}
scope.setCachedEHDispatchBlock(dispatchBlock);
}
return dispatchBlock;
}
+llvm::BasicBlock *
+CodeGenFunction::getMSVCDispatchBlock(EHScopeStack::stable_iterator SI) {
+ // Returning nullptr indicates that the previous dispatch block should unwind
+ // to caller.
+ if (SI == EHStack.stable_end())
+ return nullptr;
+
+ // Otherwise, we should look at the actual scope.
+ EHScope &EHS = *EHStack.find(SI);
+
+ llvm::BasicBlock *DispatchBlock = EHS.getCachedEHDispatchBlock();
+ if (DispatchBlock)
+ return DispatchBlock;
+
+ if (EHS.getKind() == EHScope::Terminate)
+ DispatchBlock = getTerminateHandler();
+ else
+ DispatchBlock = createBasicBlock();
+ CGBuilderTy Builder(*this, DispatchBlock);
+
+ switch (EHS.getKind()) {
+ case EHScope::Catch:
+ DispatchBlock->setName("catch.dispatch");
+ break;
+
+ case EHScope::Cleanup:
+ DispatchBlock->setName("ehcleanup");
+ break;
+
+ case EHScope::Filter:
+ llvm_unreachable("exception specifications not handled yet!");
+
+ case EHScope::Terminate:
+ DispatchBlock->setName("terminate");
+ break;
+
+ case EHScope::PadEnd:
+ llvm_unreachable("PadEnd dispatch block missing!");
+ }
+ EHS.setCachedEHDispatchBlock(DispatchBlock);
+ return DispatchBlock;
+}
+
/// Check whether this is a non-EH scope, i.e. a scope which doesn't
/// affect exception handling. Currently, the only non-EH scopes are
/// normal-only cleanup scopes.
@@ -638,6 +675,7 @@
case EHScope::Filter:
case EHScope::Catch:
case EHScope::Terminate:
+ case EHScope::PadEnd:
return false;
}
@@ -663,8 +701,19 @@
llvm::BasicBlock *LP = EHStack.begin()->getCachedLandingPad();
if (LP) return LP;
- // Build the landing pad for this scope.
- LP = EmitLandingPad();
+ const EHPersonality &Personality = EHPersonality::get(*this);
+
+ if (!CurFn->hasPersonalityFn())
+ CurFn->setPersonalityFn(getOpaquePersonalityFn(CGM, Personality));
+
+ if (Personality.usesFuncletPads()) {
+ // We don't need separate landing pads in the funclet model.
+ LP = getEHDispatchBlock(EHStack.getInnermostEHScope());
+ } else {
+ // Build the landing pad for this scope.
+ LP = EmitLandingPad();
+ }
+
assert(LP);
// Cache the landing pad on the innermost scope. If this is a
@@ -685,6 +734,9 @@
case EHScope::Terminate:
return getTerminateLandingPad();
+ case EHScope::PadEnd:
+ llvm_unreachable("PadEnd unnecessary for Itanium!");
+
case EHScope::Catch:
case EHScope::Cleanup:
case EHScope::Filter:
@@ -696,15 +748,12 @@
CGBuilderTy::InsertPoint savedIP = Builder.saveAndClearIP();
auto DL = ApplyDebugLocation::CreateDefaultArtificial(*this, CurEHLocation);
- const EHPersonality &personality = EHPersonality::get(*this);
-
// Create and configure the landing pad.
llvm::BasicBlock *lpad = createBasicBlock("lpad");
EmitBlock(lpad);
- llvm::LandingPadInst *LPadInst =
- Builder.CreateLandingPad(llvm::StructType::get(Int8PtrTy, Int32Ty, nullptr),
- getOpaquePersonalityFn(CGM, personality), 0);
+ llvm::LandingPadInst *LPadInst = Builder.CreateLandingPad(
+ llvm::StructType::get(Int8PtrTy, Int32Ty, nullptr), 0);
llvm::Value *LPadExn = Builder.CreateExtractValue(LPadInst, 0);
Builder.CreateStore(LPadExn, getExceptionSlot());
@@ -753,23 +802,28 @@
case EHScope::Catch:
break;
+
+ case EHScope::PadEnd:
+ llvm_unreachable("PadEnd unnecessary for Itanium!");
}
EHCatchScope &catchScope = cast<EHCatchScope>(*I);
for (unsigned hi = 0, he = catchScope.getNumHandlers(); hi != he; ++hi) {
EHCatchScope::Handler handler = catchScope.getHandler(hi);
+ assert(handler.Type.Flags == 0 &&
+ "landingpads do not support catch handler flags");
// If this is a catch-all, register that and abort.
- if (!handler.Type) {
+ if (!handler.Type.RTTI) {
assert(!hasCatchAll);
hasCatchAll = true;
goto done;
}
// Check whether we already have a handler for this type.
- if (catchTypes.insert(handler.Type).second)
+ if (catchTypes.insert(handler.Type.RTTI).second)
// If not, add it directly to the landingpad.
- LPadInst->addClause(handler.Type);
+ LPadInst->addClause(handler.Type.RTTI);
}
}
@@ -817,10 +871,53 @@
return lpad;
}
+static void emitCatchPadBlock(CodeGenFunction &CGF, EHCatchScope &CatchScope) {
+ llvm::BasicBlock *DispatchBlock = CatchScope.getCachedEHDispatchBlock();
+ assert(DispatchBlock);
+
+ CGBuilderTy::InsertPoint SavedIP = CGF.Builder.saveIP();
+ CGF.EmitBlockAfterUses(DispatchBlock);
+
+ llvm::Value *ParentPad = CGF.CurrentFuncletPad;
+ if (!ParentPad)
+ ParentPad = llvm::ConstantTokenNone::get(CGF.getLLVMContext());
+ llvm::BasicBlock *UnwindBB =
+ CGF.getEHDispatchBlock(CatchScope.getEnclosingEHScope());
+
+ unsigned NumHandlers = CatchScope.getNumHandlers();
+ llvm::CatchSwitchInst *CatchSwitch =
+ CGF.Builder.CreateCatchSwitch(ParentPad, UnwindBB, NumHandlers);
+
+ // Test against each of the exception types we claim to catch.
+ for (unsigned I = 0; I < NumHandlers; ++I) {
+ const EHCatchScope::Handler &Handler = CatchScope.getHandler(I);
+
+ CatchTypeInfo TypeInfo = Handler.Type;
+ if (!TypeInfo.RTTI)
+ TypeInfo.RTTI = llvm::Constant::getNullValue(CGF.VoidPtrTy);
+
+ CGF.Builder.SetInsertPoint(Handler.Block);
+
+ if (EHPersonality::get(CGF).isMSVCXXPersonality()) {
+ CGF.Builder.CreateCatchPad(
+ CatchSwitch, {TypeInfo.RTTI, CGF.Builder.getInt32(TypeInfo.Flags),
+ llvm::Constant::getNullValue(CGF.VoidPtrTy)});
+ } else {
+ CGF.Builder.CreateCatchPad(CatchSwitch, {TypeInfo.RTTI});
+ }
+
+ CatchSwitch->addHandler(Handler.Block);
+ }
+ CGF.Builder.restoreIP(SavedIP);
+}
+
/// Emit the structure of the dispatch block for the given catch scope.
/// It is an invariant that the dispatch block already exists.
static void emitCatchDispatchBlock(CodeGenFunction &CGF,
EHCatchScope &catchScope) {
+ if (EHPersonality::get(CGF).usesFuncletPads())
+ return emitCatchPadBlock(CGF, catchScope);
+
llvm::BasicBlock *dispatchBlock = catchScope.getCachedEHDispatchBlock();
assert(dispatchBlock);
@@ -847,7 +944,9 @@
assert(i < e && "ran off end of handlers!");
const EHCatchScope::Handler &handler = catchScope.getHandler(i);
- llvm::Value *typeValue = handler.Type;
+ llvm::Value *typeValue = handler.Type.RTTI;
+ assert(handler.Type.Flags == 0 &&
+ "landingpads do not support catch handler flags");
assert(typeValue && "fell into catch-all case!");
typeValue = CGF.Builder.CreateBitCast(typeValue, CGF.Int8PtrTy);
@@ -916,9 +1015,8 @@
// Copy the handler blocks off before we pop the EH stack. Emitting
// the handlers might scribble on this memory.
- SmallVector<EHCatchScope::Handler, 8> Handlers(NumHandlers);
- memcpy(Handlers.data(), CatchScope.begin(),
- NumHandlers * sizeof(EHCatchScope::Handler));
+ SmallVector<EHCatchScope::Handler, 8> Handlers(
+ CatchScope.begin(), CatchScope.begin() + NumHandlers);
EHStack.popCatch();
@@ -955,6 +1053,8 @@
RunCleanupsScope CatchScope(*this);
// Initialize the catch variable and set up the cleanups.
+ SaveAndRestore<llvm::Instruction *> RestoreCurrentFuncletPad(
+ CurrentFuncletPad);
CGM.getCXXABI().emitBeginCatch(*this, C);
// Emit the PGO counter increment.
@@ -991,7 +1091,7 @@
}
namespace {
- struct CallEndCatchForFinally : EHScopeStack::Cleanup {
+ struct CallEndCatchForFinally final : EHScopeStack::Cleanup {
llvm::Value *ForEHVar;
llvm::Value *EndCatchFn;
CallEndCatchForFinally(llvm::Value *ForEHVar, llvm::Value *EndCatchFn)
@@ -1003,7 +1103,7 @@
CGF.createBasicBlock("finally.cleanup.cont");
llvm::Value *ShouldEndCatch =
- CGF.Builder.CreateLoad(ForEHVar, "finally.endcatch");
+ CGF.Builder.CreateFlagLoad(ForEHVar, "finally.endcatch");
CGF.Builder.CreateCondBr(ShouldEndCatch, EndCatchBB, CleanupContBB);
CGF.EmitBlock(EndCatchBB);
CGF.EmitRuntimeCallOrInvoke(EndCatchFn); // catch-all, so might throw
@@ -1011,7 +1111,7 @@
}
};
- struct PerformFinally : EHScopeStack::Cleanup {
+ struct PerformFinally final : EHScopeStack::Cleanup {
const Stmt *Body;
llvm::Value *ForEHVar;
llvm::Value *EndCatchFn;
@@ -1046,13 +1146,13 @@
llvm::BasicBlock *ContBB = CGF.createBasicBlock("finally.cont");
llvm::Value *ShouldRethrow =
- CGF.Builder.CreateLoad(ForEHVar, "finally.shouldthrow");
+ CGF.Builder.CreateFlagLoad(ForEHVar, "finally.shouldthrow");
CGF.Builder.CreateCondBr(ShouldRethrow, RethrowBB, ContBB);
CGF.EmitBlock(RethrowBB);
if (SavedExnVar) {
CGF.EmitRuntimeCallOrInvoke(RethrowFn,
- CGF.Builder.CreateLoad(SavedExnVar));
+ CGF.Builder.CreateAlignedLoad(SavedExnVar, CGF.getPointerAlign()));
} else {
CGF.EmitRuntimeCallOrInvoke(RethrowFn);
}
@@ -1079,7 +1179,7 @@
CGF.EnsureInsertPoint();
}
};
-}
+} // end anonymous namespace
/// Enters a finally block for an implementation using zero-cost
/// exceptions. This is mostly general, but hard-codes some
@@ -1127,7 +1227,7 @@
// Whether the finally block is being executed for EH purposes.
ForEHVar = CGF.CreateTempAlloca(CGF.Builder.getInt1Ty(), "finally.for-eh");
- CGF.Builder.CreateStore(CGF.Builder.getFalse(), ForEHVar);
+ CGF.Builder.CreateFlagStore(false, ForEHVar);
// Enter a normal cleanup which will perform the @finally block.
CGF.EHStack.pushCleanup<PerformFinally>(NormalCleanup, body,
@@ -1165,11 +1265,11 @@
// If we need to remember the exception pointer to rethrow later, do so.
if (SavedExnVar) {
if (!exn) exn = CGF.getExceptionFromSlot();
- CGF.Builder.CreateStore(exn, SavedExnVar);
+ CGF.Builder.CreateAlignedStore(exn, SavedExnVar, CGF.getPointerAlign());
}
// Tell the cleanups in the finally block that we're do this for EH.
- CGF.Builder.CreateStore(CGF.Builder.getTrue(), ForEHVar);
+ CGF.Builder.CreateFlagStore(true, ForEHVar);
// Thread a jump through the finally cleanup.
CGF.EmitBranchThroughCleanup(RethrowDest);
@@ -1193,12 +1293,15 @@
// Tell the backend that this is a landing pad.
const EHPersonality &Personality = EHPersonality::get(*this);
- llvm::LandingPadInst *LPadInst =
- Builder.CreateLandingPad(llvm::StructType::get(Int8PtrTy, Int32Ty, nullptr),
- getOpaquePersonalityFn(CGM, Personality), 0);
+
+ if (!CurFn->hasPersonalityFn())
+ CurFn->setPersonalityFn(getOpaquePersonalityFn(CGM, Personality));
+
+ llvm::LandingPadInst *LPadInst = Builder.CreateLandingPad(
+ llvm::StructType::get(Int8PtrTy, Int32Ty, nullptr), 0);
LPadInst->addClause(getCatchAllValue(*this));
- llvm::Value *Exn = 0;
+ llvm::Value *Exn = nullptr;
if (getLangOpts().CPlusPlus)
Exn = Builder.CreateExtractValue(LPadInst, 0);
llvm::CallInst *terminateCall =
@@ -1222,9 +1325,16 @@
// end of the function by FinishFunction.
TerminateHandler = createBasicBlock("terminate.handler");
Builder.SetInsertPoint(TerminateHandler);
- llvm::Value *Exn = 0;
- if (getLangOpts().CPlusPlus)
- Exn = getExceptionFromSlot();
+ llvm::Value *Exn = nullptr;
+ if (EHPersonality::get(*this).usesFuncletPads()) {
+ llvm::Value *ParentPad = CurrentFuncletPad;
+ if (!ParentPad)
+ ParentPad = llvm::ConstantTokenNone::get(CGM.getLLVMContext());
+ Builder.CreateCleanupPad(ParentPad);
+ } else {
+ if (getLangOpts().CPlusPlus)
+ Exn = getExceptionFromSlot();
+ }
llvm::CallInst *terminateCall =
CGM.getCXXABI().emitTerminateForUnexpectedException(*this, Exn);
terminateCall->setDoesNotReturn();
@@ -1274,14 +1384,6 @@
}
void CodeGenFunction::EmitSEHTryStmt(const SEHTryStmt &S) {
- // FIXME: Implement SEH on other architectures.
- const llvm::Triple &T = CGM.getTarget().getTriple();
- if (T.getArch() != llvm::Triple::x86_64 ||
- !T.isKnownWindowsMSVCEnvironment()) {
- ErrorUnsupported(&S, "__try statement");
- return;
- }
-
EnterSEHTryStmt(S);
{
JumpDest TryExit = getJumpDestInCurrentScope("__try.__leave");
@@ -1299,45 +1401,54 @@
}
namespace {
-struct PerformSEHFinally : EHScopeStack::Cleanup {
+struct PerformSEHFinally final : EHScopeStack::Cleanup {
llvm::Function *OutlinedFinally;
PerformSEHFinally(llvm::Function *OutlinedFinally)
: OutlinedFinally(OutlinedFinally) {}
void Emit(CodeGenFunction &CGF, Flags F) override {
ASTContext &Context = CGF.getContext();
- QualType ArgTys[2] = {Context.UnsignedCharTy, Context.VoidPtrTy};
- FunctionProtoType::ExtProtoInfo EPI;
- const auto *FTP = cast<FunctionType>(
- Context.getFunctionType(Context.VoidTy, ArgTys, EPI));
+ CodeGenModule &CGM = CGF.CGM;
CallArgList Args;
+
+ // Compute the two argument values.
+ QualType ArgTys[2] = {Context.UnsignedCharTy, Context.VoidPtrTy};
+ llvm::Value *LocalAddrFn = CGM.getIntrinsic(llvm::Intrinsic::localaddress);
+ llvm::Value *FP = CGF.Builder.CreateCall(LocalAddrFn);
llvm::Value *IsForEH =
llvm::ConstantInt::get(CGF.ConvertType(ArgTys[0]), F.isForEHCleanup());
Args.add(RValue::get(IsForEH), ArgTys[0]);
-
- CodeGenModule &CGM = CGF.CGM;
- llvm::Value *Zero = llvm::ConstantInt::get(CGM.Int32Ty, 0);
- llvm::Value *FrameAddr = CGM.getIntrinsic(llvm::Intrinsic::frameaddress);
- llvm::Value *FP = CGF.Builder.CreateCall(FrameAddr, Zero);
Args.add(RValue::get(FP), ArgTys[1]);
+ // Arrange a two-arg function info and type.
+ FunctionProtoType::ExtProtoInfo EPI;
+ const auto *FPT = cast<FunctionProtoType>(
+ Context.getFunctionType(Context.VoidTy, ArgTys, EPI));
const CGFunctionInfo &FnInfo =
- CGM.getTypes().arrangeFreeFunctionCall(Args, FTP, /*chainCall=*/false);
+ CGM.getTypes().arrangeFreeFunctionCall(Args, FPT,
+ /*chainCall=*/false);
+
CGF.EmitCall(FnInfo, OutlinedFinally, ReturnValueSlot(), Args);
}
};
-}
+} // end anonymous namespace
namespace {
/// Find all local variable captures in the statement.
struct CaptureFinder : ConstStmtVisitor<CaptureFinder> {
CodeGenFunction &ParentCGF;
const VarDecl *ParentThis;
- SmallVector<const VarDecl *, 4> Captures;
+ llvm::SmallSetVector<const VarDecl *, 4> Captures;
+ Address SEHCodeSlot = Address::invalid();
CaptureFinder(CodeGenFunction &ParentCGF, const VarDecl *ParentThis)
: ParentCGF(ParentCGF), ParentThis(ParentThis) {}
+ // Return true if we need to do any capturing work.
+ bool foundCaptures() {
+ return !Captures.empty() || SEHCodeSlot.isValid();
+ }
+
void Visit(const Stmt *S) {
// See if this is a capture, then recurse.
ConstStmtVisitor<CaptureFinder>::Visit(S);
@@ -1349,39 +1460,124 @@
void VisitDeclRefExpr(const DeclRefExpr *E) {
// If this is already a capture, just make sure we capture 'this'.
if (E->refersToEnclosingVariableOrCapture()) {
- Captures.push_back(ParentThis);
+ Captures.insert(ParentThis);
return;
}
const auto *D = dyn_cast<VarDecl>(E->getDecl());
if (D && D->isLocalVarDeclOrParm() && D->hasLocalStorage())
- Captures.push_back(D);
+ Captures.insert(D);
}
void VisitCXXThisExpr(const CXXThisExpr *E) {
- Captures.push_back(ParentThis);
+ Captures.insert(ParentThis);
+ }
+
+ void VisitCallExpr(const CallExpr *E) {
+ // We only need to add parent frame allocations for these builtins in x86.
+ if (ParentCGF.getTarget().getTriple().getArch() != llvm::Triple::x86)
+ return;
+
+ unsigned ID = E->getBuiltinCallee();
+ switch (ID) {
+ case Builtin::BI__exception_code:
+ case Builtin::BI_exception_code:
+ // This is the simple case where we are the outermost finally. All we
+ // have to do here is make sure we escape this and recover it in the
+ // outlined handler.
+ if (!SEHCodeSlot.isValid())
+ SEHCodeSlot = ParentCGF.SEHCodeSlotStack.back();
+ break;
+ }
}
};
+} // end anonymous namespace
+
+Address CodeGenFunction::recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF,
+ Address ParentVar,
+ llvm::Value *ParentFP) {
+ llvm::CallInst *RecoverCall = nullptr;
+ CGBuilderTy Builder(*this, AllocaInsertPt);
+ if (auto *ParentAlloca = dyn_cast<llvm::AllocaInst>(ParentVar.getPointer())) {
+ // Mark the variable escaped if nobody else referenced it and compute the
+ // localescape index.
+ auto InsertPair = ParentCGF.EscapedLocals.insert(
+ std::make_pair(ParentAlloca, ParentCGF.EscapedLocals.size()));
+ int FrameEscapeIdx = InsertPair.first->second;
+ // call i8* @llvm.localrecover(i8* bitcast(@parentFn), i8* %fp, i32 N)
+ llvm::Function *FrameRecoverFn = llvm::Intrinsic::getDeclaration(
+ &CGM.getModule(), llvm::Intrinsic::localrecover);
+ llvm::Constant *ParentI8Fn =
+ llvm::ConstantExpr::getBitCast(ParentCGF.CurFn, Int8PtrTy);
+ RecoverCall = Builder.CreateCall(
+ FrameRecoverFn, {ParentI8Fn, ParentFP,
+ llvm::ConstantInt::get(Int32Ty, FrameEscapeIdx)});
+
+ } else {
+ // If the parent didn't have an alloca, we're doing some nested outlining.
+ // Just clone the existing localrecover call, but tweak the FP argument to
+ // use our FP value. All other arguments are constants.
+ auto *ParentRecover =
+ cast<llvm::IntrinsicInst>(ParentVar.getPointer()->stripPointerCasts());
+ assert(ParentRecover->getIntrinsicID() == llvm::Intrinsic::localrecover &&
+ "expected alloca or localrecover in parent LocalDeclMap");
+ RecoverCall = cast<llvm::CallInst>(ParentRecover->clone());
+ RecoverCall->setArgOperand(1, ParentFP);
+ RecoverCall->insertBefore(AllocaInsertPt);
+ }
+
+ // Bitcast the variable, rename it, and insert it in the local decl map.
+ llvm::Value *ChildVar =
+ Builder.CreateBitCast(RecoverCall, ParentVar.getType());
+ ChildVar->setName(ParentVar.getName());
+ return Address(ChildVar, ParentVar.getAlignment());
}
void CodeGenFunction::EmitCapturedLocals(CodeGenFunction &ParentCGF,
const Stmt *OutlinedStmt,
- llvm::Value *ParentFP) {
+ bool IsFilter) {
// Find all captures in the Stmt.
CaptureFinder Finder(ParentCGF, ParentCGF.CXXABIThisDecl);
Finder.Visit(OutlinedStmt);
- // Typically there are no captures and we can exit early.
- if (Finder.Captures.empty())
+ // We can exit early on x86_64 when there are no captures. We just have to
+ // save the exception code in filters so that __exception_code() works.
+ if (!Finder.foundCaptures() &&
+ CGM.getTarget().getTriple().getArch() != llvm::Triple::x86) {
+ if (IsFilter)
+ EmitSEHExceptionCodeSave(ParentCGF, nullptr, nullptr);
return;
+ }
- // Prepare the first two arguments to llvm.framerecover.
- llvm::Function *FrameRecoverFn = llvm::Intrinsic::getDeclaration(
- &CGM.getModule(), llvm::Intrinsic::framerecover);
- llvm::Constant *ParentI8Fn =
- llvm::ConstantExpr::getBitCast(ParentCGF.CurFn, Int8PtrTy);
+ llvm::Value *EntryFP = nullptr;
+ CGBuilderTy Builder(CGM, AllocaInsertPt);
+ if (IsFilter && CGM.getTarget().getTriple().getArch() == llvm::Triple::x86) {
+ // 32-bit SEH filters need to be careful about FP recovery. The end of the
+ // EH registration is passed in as the EBP physical register. We can
+ // recover that with llvm.frameaddress(1).
+ EntryFP = Builder.CreateCall(
+ CGM.getIntrinsic(llvm::Intrinsic::frameaddress), {Builder.getInt32(1)});
+ } else {
+ // Otherwise, for x64 and 32-bit finally functions, the parent FP is the
+ // second parameter.
+ auto AI = CurFn->arg_begin();
+ ++AI;
+ EntryFP = &*AI;
+ }
- // Create llvm.framerecover calls for all captures.
+ llvm::Value *ParentFP = EntryFP;
+ if (IsFilter) {
+ // Given whatever FP the runtime provided us in EntryFP, recover the true
+ // frame pointer of the parent function. We only need to do this in filters,
+ // since finally funclets recover the parent FP for us.
+ llvm::Function *RecoverFPIntrin =
+ CGM.getIntrinsic(llvm::Intrinsic::x86_seh_recoverfp);
+ llvm::Constant *ParentI8Fn =
+ llvm::ConstantExpr::getBitCast(ParentCGF.CurFn, Int8PtrTy);
+ ParentFP = Builder.CreateCall(RecoverFPIntrin, {ParentI8Fn, EntryFP});
+ }
+
+ // Create llvm.localrecover calls for all captures.
for (const VarDecl *VD : Finder.Captures) {
if (isa<ImplicitParamDecl>(VD)) {
CGM.ErrorUnsupported(VD, "'this' captured by SEH");
@@ -1400,51 +1596,65 @@
auto I = ParentCGF.LocalDeclMap.find(VD);
if (I == ParentCGF.LocalDeclMap.end())
continue;
- llvm::Value *ParentVar = I->second;
- llvm::CallInst *RecoverCall = nullptr;
- CGBuilderTy Builder(AllocaInsertPt);
- if (auto *ParentAlloca = dyn_cast<llvm::AllocaInst>(ParentVar)) {
- // Mark the variable escaped if nobody else referenced it and compute the
- // frameescape index.
- auto InsertPair =
- ParentCGF.EscapedLocals.insert(std::make_pair(ParentAlloca, -1));
- if (InsertPair.second)
- InsertPair.first->second = ParentCGF.EscapedLocals.size() - 1;
- int FrameEscapeIdx = InsertPair.first->second;
- // call i8* @llvm.framerecover(i8* bitcast(@parentFn), i8* %fp, i32 N)
- RecoverCall = Builder.CreateCall(
- FrameRecoverFn, {ParentI8Fn, ParentFP,
- llvm::ConstantInt::get(Int32Ty, FrameEscapeIdx)});
-
- } else {
- // If the parent didn't have an alloca, we're doing some nested outlining.
- // Just clone the existing framerecover call, but tweak the FP argument to
- // use our FP value. All other arguments are constants.
- auto *ParentRecover =
- cast<llvm::IntrinsicInst>(ParentVar->stripPointerCasts());
- assert(ParentRecover->getIntrinsicID() == llvm::Intrinsic::framerecover &&
- "expected alloca or framerecover in parent LocalDeclMap");
- RecoverCall = cast<llvm::CallInst>(ParentRecover->clone());
- RecoverCall->setArgOperand(1, ParentFP);
- RecoverCall->insertBefore(AllocaInsertPt);
- }
-
- // Bitcast the variable, rename it, and insert it in the local decl map.
- llvm::Value *ChildVar =
- Builder.CreateBitCast(RecoverCall, ParentVar->getType());
- ChildVar->setName(ParentVar->getName());
- LocalDeclMap[VD] = ChildVar;
+ Address ParentVar = I->second;
+ setAddrOfLocalVar(
+ VD, recoverAddrOfEscapedLocal(ParentCGF, ParentVar, ParentFP));
}
+
+ if (Finder.SEHCodeSlot.isValid()) {
+ SEHCodeSlotStack.push_back(
+ recoverAddrOfEscapedLocal(ParentCGF, Finder.SEHCodeSlot, ParentFP));
+ }
+
+ if (IsFilter)
+ EmitSEHExceptionCodeSave(ParentCGF, ParentFP, EntryFP);
}
/// Arrange a function prototype that can be called by Windows exception
/// handling personalities. On Win64, the prototype looks like:
/// RetTy func(void *EHPtrs, void *ParentFP);
void CodeGenFunction::startOutlinedSEHHelper(CodeGenFunction &ParentCGF,
- StringRef Name, QualType RetTy,
- FunctionArgList &Args,
+ bool IsFilter,
const Stmt *OutlinedStmt) {
+ SourceLocation StartLoc = OutlinedStmt->getLocStart();
+
+ // Get the mangled function name.
+ SmallString<128> Name;
+ {
+ llvm::raw_svector_ostream OS(Name);
+ const Decl *ParentCodeDecl = ParentCGF.CurCodeDecl;
+ const NamedDecl *Parent = dyn_cast_or_null<NamedDecl>(ParentCodeDecl);
+ assert(Parent && "FIXME: handle unnamed decls (lambdas, blocks) with SEH");
+ MangleContext &Mangler = CGM.getCXXABI().getMangleContext();
+ if (IsFilter)
+ Mangler.mangleSEHFilterExpression(Parent, OS);
+ else
+ Mangler.mangleSEHFinallyBlock(Parent, OS);
+ }
+
+ FunctionArgList Args;
+ if (CGM.getTarget().getTriple().getArch() != llvm::Triple::x86 || !IsFilter) {
+ // All SEH finally functions take two parameters. Win64 filters take two
+ // parameters. Win32 filters take no parameters.
+ if (IsFilter) {
+ Args.push_back(ImplicitParamDecl::Create(
+ getContext(), nullptr, StartLoc,
+ &getContext().Idents.get("exception_pointers"),
+ getContext().VoidPtrTy));
+ } else {
+ Args.push_back(ImplicitParamDecl::Create(
+ getContext(), nullptr, StartLoc,
+ &getContext().Idents.get("abnormal_termination"),
+ getContext().UnsignedCharTy));
+ }
+ Args.push_back(ImplicitParamDecl::Create(
+ getContext(), nullptr, StartLoc,
+ &getContext().Idents.get("frame_pointer"), getContext().VoidPtrTy));
+ }
+
+ QualType RetTy = IsFilter ? getContext().LongTy : getContext().VoidTy;
+
llvm::Function *ParentFn = ParentCGF.CurFn;
const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionDeclaration(
RetTy, Args, FunctionType::ExtInfo(), /*isVariadic=*/false);
@@ -1469,10 +1679,7 @@
OutlinedStmt->getLocStart(), OutlinedStmt->getLocStart());
CGM.SetLLVMFunctionAttributes(nullptr, FnInfo, CurFn);
-
- auto AI = Fn->arg_begin();
- ++AI;
- EmitCapturedLocals(ParentCGF, OutlinedStmt, &*AI);
+ EmitCapturedLocals(ParentCGF, OutlinedStmt, IsFilter);
}
/// Create a stub filter function that will ultimately hold the code of the
@@ -1482,37 +1689,7 @@
CodeGenFunction::GenerateSEHFilterFunction(CodeGenFunction &ParentCGF,
const SEHExceptStmt &Except) {
const Expr *FilterExpr = Except.getFilterExpr();
- SourceLocation StartLoc = FilterExpr->getLocStart();
-
- SEHPointersDecl = ImplicitParamDecl::Create(
- getContext(), nullptr, StartLoc,
- &getContext().Idents.get("exception_pointers"), getContext().VoidPtrTy);
- FunctionArgList Args;
- Args.push_back(SEHPointersDecl);
- Args.push_back(ImplicitParamDecl::Create(
- getContext(), nullptr, StartLoc,
- &getContext().Idents.get("frame_pointer"), getContext().VoidPtrTy));
-
- // Get the mangled function name.
- SmallString<128> Name;
- {
- llvm::raw_svector_ostream OS(Name);
- const Decl *ParentCodeDecl = ParentCGF.CurCodeDecl;
- const NamedDecl *Parent = dyn_cast_or_null<NamedDecl>(ParentCodeDecl);
- assert(Parent && "FIXME: handle unnamed decls (lambdas, blocks) with SEH");
- CGM.getCXXABI().getMangleContext().mangleSEHFilterExpression(Parent, OS);
- }
-
- startOutlinedSEHHelper(ParentCGF, Name, getContext().LongTy, Args,
- FilterExpr);
-
- // Mark finally block calls as nounwind and noinline to make LLVM's job a
- // little easier.
- // FIXME: Remove these restrictions in the future.
- CurFn->addFnAttr(llvm::Attribute::NoUnwind);
- CurFn->addFnAttr(llvm::Attribute::NoInline);
-
- EmitSEHExceptionCodeSave();
+ startOutlinedSEHHelper(ParentCGF, true, FilterExpr);
// Emit the original filter expression, convert to i32, and return.
llvm::Value *R = EmitScalarExpr(FilterExpr);
@@ -1529,29 +1706,13 @@
CodeGenFunction::GenerateSEHFinallyFunction(CodeGenFunction &ParentCGF,
const SEHFinallyStmt &Finally) {
const Stmt *FinallyBlock = Finally.getBlock();
- SourceLocation StartLoc = FinallyBlock->getLocStart();
+ startOutlinedSEHHelper(ParentCGF, false, FinallyBlock);
- FunctionArgList Args;
- Args.push_back(ImplicitParamDecl::Create(
- getContext(), nullptr, StartLoc,
- &getContext().Idents.get("abnormal_termination"),
- getContext().UnsignedCharTy));
- Args.push_back(ImplicitParamDecl::Create(
- getContext(), nullptr, StartLoc,
- &getContext().Idents.get("frame_pointer"), getContext().VoidPtrTy));
-
- // Get the mangled function name.
- SmallString<128> Name;
- {
- llvm::raw_svector_ostream OS(Name);
- const Decl *ParentCodeDecl = ParentCGF.CurCodeDecl;
- const NamedDecl *Parent = dyn_cast_or_null<NamedDecl>(ParentCodeDecl);
- assert(Parent && "FIXME: handle unnamed decls (lambdas, blocks) with SEH");
- CGM.getCXXABI().getMangleContext().mangleSEHFinallyBlock(Parent, OS);
- }
-
- startOutlinedSEHHelper(ParentCGF, Name, getContext().VoidTy, Args,
- FinallyBlock);
+ // Mark finally block calls as nounwind and noinline to make LLVM's job a
+ // little easier.
+ // FIXME: Remove these restrictions in the future.
+ CurFn->addFnAttr(llvm::Attribute::NoUnwind);
+ CurFn->addFnAttr(llvm::Attribute::NoInline);
// Emit the original filter expression, convert to i32, and return.
EmitStmt(FinallyBlock);
@@ -1561,44 +1722,57 @@
return CurFn;
}
-void CodeGenFunction::EmitSEHExceptionCodeSave() {
+void CodeGenFunction::EmitSEHExceptionCodeSave(CodeGenFunction &ParentCGF,
+ llvm::Value *ParentFP,
+ llvm::Value *EntryFP) {
+ // Get the pointer to the EXCEPTION_POINTERS struct. This is returned by the
+ // __exception_info intrinsic.
+ if (CGM.getTarget().getTriple().getArch() != llvm::Triple::x86) {
+ // On Win64, the info is passed as the first parameter to the filter.
+ SEHInfo = &*CurFn->arg_begin();
+ SEHCodeSlotStack.push_back(
+ CreateMemTemp(getContext().IntTy, "__exception_code"));
+ } else {
+ // On Win32, the EBP on entry to the filter points to the end of an
+ // exception registration object. It contains 6 32-bit fields, and the info
+ // pointer is stored in the second field. So, GEP 20 bytes backwards and
+ // load the pointer.
+ SEHInfo = Builder.CreateConstInBoundsGEP1_32(Int8Ty, EntryFP, -20);
+ SEHInfo = Builder.CreateBitCast(SEHInfo, Int8PtrTy->getPointerTo());
+ SEHInfo = Builder.CreateAlignedLoad(Int8PtrTy, SEHInfo, getPointerAlign());
+ SEHCodeSlotStack.push_back(recoverAddrOfEscapedLocal(
+ ParentCGF, ParentCGF.SEHCodeSlotStack.back(), ParentFP));
+ }
+
// Save the exception code in the exception slot to unify exception access in
// the filter function and the landing pad.
// struct EXCEPTION_POINTERS {
// EXCEPTION_RECORD *ExceptionRecord;
// CONTEXT *ContextRecord;
// };
- // void *exn.slot =
- // (void *)(uintptr_t)exception_pointers->ExceptionRecord->ExceptionCode;
- llvm::Value *Ptrs = Builder.CreateLoad(GetAddrOfLocalVar(SEHPointersDecl));
+ // int exceptioncode = exception_pointers->ExceptionRecord->ExceptionCode;
llvm::Type *RecordTy = CGM.Int32Ty->getPointerTo();
llvm::Type *PtrsTy = llvm::StructType::get(RecordTy, CGM.VoidPtrTy, nullptr);
- Ptrs = Builder.CreateBitCast(Ptrs, PtrsTy->getPointerTo());
+ llvm::Value *Ptrs = Builder.CreateBitCast(SEHInfo, PtrsTy->getPointerTo());
llvm::Value *Rec = Builder.CreateStructGEP(PtrsTy, Ptrs, 0);
- Rec = Builder.CreateLoad(Rec);
- llvm::Value *Code = Builder.CreateLoad(Rec);
- Code = Builder.CreateZExt(Code, CGM.IntPtrTy);
- // FIXME: Change landing pads to produce {i32, i32} and make the exception
- // slot an i32.
- Code = Builder.CreateIntToPtr(Code, CGM.VoidPtrTy);
- Builder.CreateStore(Code, getExceptionSlot());
+ Rec = Builder.CreateAlignedLoad(Rec, getPointerAlign());
+ llvm::Value *Code = Builder.CreateAlignedLoad(Rec, getIntAlign());
+ assert(!SEHCodeSlotStack.empty() && "emitting EH code outside of __except");
+ Builder.CreateStore(Code, SEHCodeSlotStack.back());
}
llvm::Value *CodeGenFunction::EmitSEHExceptionInfo() {
// Sema should diagnose calling this builtin outside of a filter context, but
// don't crash if we screw up.
- if (!SEHPointersDecl)
+ if (!SEHInfo)
return llvm::UndefValue::get(Int8PtrTy);
- return Builder.CreateLoad(GetAddrOfLocalVar(SEHPointersDecl));
+ assert(SEHInfo->getType() == Int8PtrTy);
+ return SEHInfo;
}
llvm::Value *CodeGenFunction::EmitSEHExceptionCode() {
- // If we're in a landing pad or filter function, the exception slot contains
- // the code.
- assert(ExceptionSlot);
- llvm::Value *Code =
- Builder.CreatePtrToInt(getExceptionFromSlot(), CGM.IntPtrTy);
- return Builder.CreateTrunc(Code, CGM.Int32Ty);
+ assert(!SEHCodeSlotStack.empty() && "emitting EH code outside of __except");
+ return Builder.CreateLoad(SEHCodeSlotStack.back());
}
llvm::Value *CodeGenFunction::EmitSEHAbnormalTermination() {
@@ -1611,9 +1785,11 @@
void CodeGenFunction::EnterSEHTryStmt(const SEHTryStmt &S) {
CodeGenFunction HelperCGF(CGM, /*suppressNewContext=*/true);
if (const SEHFinallyStmt *Finally = S.getFinallyHandler()) {
- // Push a cleanup for __finally blocks.
+ // Outline the finally block.
llvm::Function *FinallyFunc =
HelperCGF.GenerateSEHFinallyFunction(*this, *Finally);
+
+ // Push a cleanup for __finally blocks.
EHStack.pushCleanup<PerformSEHFinally>(NormalAndEHCleanup, FinallyFunc);
return;
}
@@ -1622,12 +1798,16 @@
const SEHExceptStmt *Except = S.getExceptHandler();
assert(Except);
EHCatchScope *CatchScope = EHStack.pushCatch(1);
+ SEHCodeSlotStack.push_back(
+ CreateMemTemp(getContext().IntTy, "__exception_code"));
- // If the filter is known to evaluate to 1, then we can use the clause "catch
- // i8* null".
+ // If the filter is known to evaluate to 1, then we can use the clause
+ // "catch i8* null". We can't do this on x86 because the filter has to save
+ // the exception code.
llvm::Constant *C =
CGM.EmitConstantExpr(Except->getFilterExpr(), getContext().IntTy, this);
- if (C && C->isOneValue()) {
+ if (CGM.getTarget().getTriple().getArch() != llvm::Triple::x86 && C &&
+ C->isOneValue()) {
CatchScope->setCatchAllHandler(0, createBasicBlock("__except"));
return;
}
@@ -1638,7 +1818,7 @@
HelperCGF.GenerateSEHFilterFunction(*this, *Except);
llvm::Constant *OpaqueFunc =
llvm::ConstantExpr::getBitCast(FilterFunc, Int8PtrTy);
- CatchScope->setHandler(0, OpaqueFunc, createBasicBlock("__except"));
+ CatchScope->setHandler(0, OpaqueFunc, createBasicBlock("__except.ret"));
}
void CodeGenFunction::ExitSEHTryStmt(const SEHTryStmt &S) {
@@ -1659,6 +1839,7 @@
if (!CatchScope.hasEHBranches()) {
CatchScope.clearHandlerBlocks();
EHStack.popCatch();
+ SEHCodeSlotStack.pop_back();
return;
}
@@ -1673,14 +1854,33 @@
emitCatchDispatchBlock(*this, CatchScope);
// Grab the block before we pop the handler.
- llvm::BasicBlock *ExceptBB = CatchScope.getHandler(0).Block;
+ llvm::BasicBlock *CatchPadBB = CatchScope.getHandler(0).Block;
EHStack.popCatch();
- EmitBlockAfterUses(ExceptBB);
+ EmitBlockAfterUses(CatchPadBB);
+
+ // __except blocks don't get outlined into funclets, so immediately do a
+ // catchret.
+ llvm::CatchPadInst *CPI =
+ cast<llvm::CatchPadInst>(CatchPadBB->getFirstNonPHI());
+ llvm::BasicBlock *ExceptBB = createBasicBlock("__except");
+ Builder.CreateCatchRet(CPI, ExceptBB);
+ EmitBlock(ExceptBB);
+
+ // On Win64, the exception code is returned in EAX. Copy it into the slot.
+ if (CGM.getTarget().getTriple().getArch() != llvm::Triple::x86) {
+ llvm::Function *SEHCodeIntrin =
+ CGM.getIntrinsic(llvm::Intrinsic::eh_exceptioncode);
+ llvm::Value *Code = Builder.CreateCall(SEHCodeIntrin, {CPI});
+ Builder.CreateStore(Code, SEHCodeSlotStack.back());
+ }
// Emit the __except body.
EmitStmt(Except->getBlock());
+ // End the lifetime of the exception code.
+ SEHCodeSlotStack.pop_back();
+
if (HaveInsertPoint())
Builder.CreateBr(ContBB);
diff --git a/lib/CodeGen/CGExpr.cpp b/lib/CodeGen/CGExpr.cpp
index 1ed45a3..dabd2b1 100644
--- a/lib/CodeGen/CGExpr.cpp
+++ b/lib/CodeGen/CGExpr.cpp
@@ -54,6 +54,15 @@
/// CreateTempAlloca - This creates a alloca and inserts it into the entry
/// block.
+Address CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align,
+ const Twine &Name) {
+ auto Alloca = CreateTempAlloca(Ty, Name);
+ Alloca->setAlignment(Align.getQuantity());
+ return Address(Alloca, Align);
+}
+
+/// CreateTempAlloca - This creates a alloca and inserts it into the entry
+/// block.
llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty,
const Twine &Name) {
if (!Builder.isNamePreserving())
@@ -61,29 +70,38 @@
return new llvm::AllocaInst(Ty, nullptr, Name, AllocaInsertPt);
}
-void CodeGenFunction::InitTempAlloca(llvm::AllocaInst *Var,
- llvm::Value *Init) {
- auto *Store = new llvm::StoreInst(Init, Var);
+/// CreateDefaultAlignTempAlloca - This creates an alloca with the
+/// default alignment of the corresponding LLVM type, which is *not*
+/// guaranteed to be related in any way to the expected alignment of
+/// an AST type that might have been lowered to Ty.
+Address CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty,
+ const Twine &Name) {
+ CharUnits Align =
+ CharUnits::fromQuantity(CGM.getDataLayout().getABITypeAlignment(Ty));
+ return CreateTempAlloca(Ty, Align, Name);
+}
+
+void CodeGenFunction::InitTempAlloca(Address Var, llvm::Value *Init) {
+ assert(isa<llvm::AllocaInst>(Var.getPointer()));
+ auto *Store = new llvm::StoreInst(Init, Var.getPointer());
+ Store->setAlignment(Var.getAlignment().getQuantity());
llvm::BasicBlock *Block = AllocaInsertPt->getParent();
- Block->getInstList().insertAfter(&*AllocaInsertPt, Store);
+ Block->getInstList().insertAfter(AllocaInsertPt->getIterator(), Store);
}
-llvm::AllocaInst *CodeGenFunction::CreateIRTemp(QualType Ty,
- const Twine &Name) {
- llvm::AllocaInst *Alloc = CreateTempAlloca(ConvertType(Ty), Name);
- // FIXME: Should we prefer the preferred type alignment here?
+Address CodeGenFunction::CreateIRTemp(QualType Ty, const Twine &Name) {
CharUnits Align = getContext().getTypeAlignInChars(Ty);
- Alloc->setAlignment(Align.getQuantity());
- return Alloc;
+ return CreateTempAlloca(ConvertType(Ty), Align, Name);
}
-llvm::AllocaInst *CodeGenFunction::CreateMemTemp(QualType Ty,
- const Twine &Name) {
- llvm::AllocaInst *Alloc = CreateTempAlloca(ConvertTypeForMem(Ty), Name);
+Address CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name) {
// FIXME: Should we prefer the preferred type alignment here?
- CharUnits Align = getContext().getTypeAlignInChars(Ty);
- Alloc->setAlignment(Align.getQuantity());
- return Alloc;
+ return CreateMemTemp(Ty, getContext().getTypeAlignInChars(Ty), Name);
+}
+
+Address CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align,
+ const Twine &Name) {
+ return CreateTempAlloca(ConvertTypeForMem(Ty), Align, Name);
}
/// EvaluateExprAsBool - Perform the usual unary conversions on the specified
@@ -96,10 +114,12 @@
}
QualType BoolTy = getContext().BoolTy;
+ SourceLocation Loc = E->getExprLoc();
if (!E->getType()->isAnyComplexType())
- return EmitScalarConversion(EmitScalarExpr(E), E->getType(), BoolTy);
+ return EmitScalarConversion(EmitScalarExpr(E), E->getType(), BoolTy, Loc);
- return EmitComplexToScalarConversion(EmitComplexExpr(E), E->getType(),BoolTy);
+ return EmitComplexToScalarConversion(EmitComplexExpr(E), E->getType(), BoolTy,
+ Loc);
}
/// EmitIgnoredExpr - Emit code to compute the specified expression,
@@ -146,20 +166,18 @@
/// EmitAnyExprToMem - Evaluate an expression into a given memory
/// location.
void CodeGenFunction::EmitAnyExprToMem(const Expr *E,
- llvm::Value *Location,
+ Address Location,
Qualifiers Quals,
bool IsInit) {
// FIXME: This function should take an LValue as an argument.
switch (getEvaluationKind(E->getType())) {
case TEK_Complex:
- EmitComplexExprIntoLValue(E,
- MakeNaturalAlignAddrLValue(Location, E->getType()),
+ EmitComplexExprIntoLValue(E, MakeAddrLValue(Location, E->getType()),
/*isInit*/ false);
return;
case TEK_Aggregate: {
- CharUnits Alignment = getContext().getTypeAlignInChars(E->getType());
- EmitAggExpr(E, AggValueSlot::forAddr(Location, Alignment, Quals,
+ EmitAggExpr(E, AggValueSlot::forAddr(Location, Quals,
AggValueSlot::IsDestructed_t(IsInit),
AggValueSlot::DoesNotNeedGCBarriers,
AggValueSlot::IsAliased_t(!IsInit)));
@@ -178,17 +196,14 @@
static void
pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M,
- const Expr *E, llvm::Value *ReferenceTemporary) {
+ const Expr *E, Address ReferenceTemporary) {
// Objective-C++ ARC:
// If we are binding a reference to a temporary that has ownership, we
// need to perform retain/release operations on the temporary.
//
// FIXME: This should be looking at E, not M.
- if (CGF.getLangOpts().ObjCAutoRefCount &&
- M->getType()->isObjCLifetimeType()) {
- QualType ObjCARCReferenceLifetimeType = M->getType();
- switch (Qualifiers::ObjCLifetime Lifetime =
- ObjCARCReferenceLifetimeType.getObjCLifetime()) {
+ if (auto Lifetime = M->getType().getObjCLifetime()) {
+ switch (Lifetime) {
case Qualifiers::OCL_None:
case Qualifiers::OCL_ExplicitNone:
// Carry on to normal cleanup handling.
@@ -229,11 +244,11 @@
}
if (Duration == SD_FullExpression)
CGF.pushDestroy(CleanupKind, ReferenceTemporary,
- ObjCARCReferenceLifetimeType, *Destroy,
+ M->getType(), *Destroy,
CleanupKind & EHCleanup);
else
CGF.pushLifetimeExtendedDestroy(CleanupKind, ReferenceTemporary,
- ObjCARCReferenceLifetimeType,
+ M->getType(),
*Destroy, CleanupKind & EHCleanup);
return;
@@ -264,14 +279,14 @@
llvm::Constant *CleanupArg;
if (E->getType()->isArrayType()) {
CleanupFn = CodeGenFunction(CGF.CGM).generateDestroyHelper(
- cast<llvm::Constant>(ReferenceTemporary), E->getType(),
+ ReferenceTemporary, E->getType(),
CodeGenFunction::destroyCXXObject, CGF.getLangOpts().Exceptions,
dyn_cast_or_null<VarDecl>(M->getExtendingDecl()));
CleanupArg = llvm::Constant::getNullValue(CGF.Int8PtrTy);
} else {
CleanupFn = CGF.CGM.getAddrOfCXXStructor(ReferenceTemporaryDtor,
StructorType::Complete);
- CleanupArg = cast<llvm::Constant>(ReferenceTemporary);
+ CleanupArg = cast<llvm::Constant>(ReferenceTemporary.getPointer());
}
CGF.CGM.getCXXABI().registerGlobalDtor(
CGF, *cast<VarDecl>(M->getExtendingDecl()), CleanupFn, CleanupArg);
@@ -296,7 +311,7 @@
}
}
-static llvm::Value *
+static Address
createReferenceTemporary(CodeGenFunction &CGF,
const MaterializeTemporaryExpr *M, const Expr *Inner) {
switch (M->getStorageDuration()) {
@@ -314,10 +329,10 @@
auto *GV = new llvm::GlobalVariable(
CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
llvm::GlobalValue::PrivateLinkage, Init, ".ref.tmp");
- GV->setAlignment(
- CGF.getContext().getTypeAlignInChars(Ty).getQuantity());
+ CharUnits alignment = CGF.getContext().getTypeAlignInChars(Ty);
+ GV->setAlignment(alignment.getQuantity());
// FIXME: Should we put the new global into a COMDAT?
- return GV;
+ return Address(GV, alignment);
}
return CGF.CreateMemTemp(Ty, "ref.tmp");
}
@@ -337,20 +352,22 @@
// FIXME: ideally this would use EmitAnyExprToMem, however, we cannot do so
// as that will cause the lifetime adjustment to be lost for ARC
- if (getLangOpts().ObjCAutoRefCount &&
- M->getType()->isObjCLifetimeType() &&
- M->getType().getObjCLifetime() != Qualifiers::OCL_None &&
- M->getType().getObjCLifetime() != Qualifiers::OCL_ExplicitNone) {
- llvm::Value *Object = createReferenceTemporary(*this, M, E);
- if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object)) {
- Object = llvm::ConstantExpr::getBitCast(
- Var, ConvertTypeForMem(E->getType())->getPointerTo());
+ auto ownership = M->getType().getObjCLifetime();
+ if (ownership != Qualifiers::OCL_None &&
+ ownership != Qualifiers::OCL_ExplicitNone) {
+ Address Object = createReferenceTemporary(*this, M, E);
+ if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object.getPointer())) {
+ Object = Address(llvm::ConstantExpr::getBitCast(Var,
+ ConvertTypeForMem(E->getType())
+ ->getPointerTo(Object.getAddressSpace())),
+ Object.getAlignment());
// We should not have emitted the initializer for this temporary as a
// constant.
assert(!Var->hasInitializer());
Var->setInitializer(CGM.EmitNullConstant(E->getType()));
}
- LValue RefTempDst = MakeAddrLValue(Object, M->getType());
+ LValue RefTempDst = MakeAddrLValue(Object, M->getType(),
+ AlignmentSource::Decl);
switch (getEvaluationKind(E->getType())) {
default: llvm_unreachable("expected scalar or aggregate expression");
@@ -358,8 +375,7 @@
EmitScalarInit(E, M->getExtendingDecl(), RefTempDst, false);
break;
case TEK_Aggregate: {
- CharUnits Alignment = getContext().getTypeAlignInChars(E->getType());
- EmitAggExpr(E, AggValueSlot::forAddr(Object, Alignment,
+ EmitAggExpr(E, AggValueSlot::forAddr(Object,
E->getType().getQualifiers(),
AggValueSlot::IsDestructed,
AggValueSlot::DoesNotNeedGCBarriers,
@@ -387,10 +403,11 @@
}
// Create and initialize the reference temporary.
- llvm::Value *Object = createReferenceTemporary(*this, M, E);
- if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object)) {
- Object = llvm::ConstantExpr::getBitCast(
- Var, ConvertTypeForMem(E->getType())->getPointerTo());
+ Address Object = createReferenceTemporary(*this, M, E);
+ if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object.getPointer())) {
+ Object = Address(llvm::ConstantExpr::getBitCast(
+ Var, ConvertTypeForMem(E->getType())->getPointerTo()),
+ Object.getAlignment());
// If the temporary is a global and has a constant initializer or is a
// constant temporary that we promoted to a global, we may have already
// initialized it.
@@ -418,7 +435,8 @@
break;
case SubobjectAdjustment::FieldAdjustment: {
- LValue LV = MakeAddrLValue(Object, E->getType());
+ LValue LV = MakeAddrLValue(Object, E->getType(),
+ AlignmentSource::Decl);
LV = EmitLValueForField(LV, Adjustment.Field);
assert(LV.isSimple() &&
"materialized temporary field is not a simple lvalue");
@@ -428,14 +446,14 @@
case SubobjectAdjustment::MemberPointerAdjustment: {
llvm::Value *Ptr = EmitScalarExpr(Adjustment.Ptr.RHS);
- Object = CGM.getCXXABI().EmitMemberDataPointerAddress(
- *this, E, Object, Ptr, Adjustment.Ptr.MPT);
+ Object = EmitCXXMemberDataPointerAddress(E, Object, Ptr,
+ Adjustment.Ptr.MPT);
break;
}
}
}
- return MakeAddrLValue(Object, M->getType());
+ return MakeAddrLValue(Object, M->getType(), AlignmentSource::Decl);
}
RValue
@@ -443,7 +461,7 @@
// Emit the expression as an lvalue.
LValue LV = EmitLValue(E);
assert(LV.isSimple());
- llvm::Value *Value = LV.getAddress();
+ llvm::Value *Value = LV.getPointer();
if (sanitizePerformTypeCheck() && !E->getType()->isFunctionType()) {
// C++11 [dcl.ref]p5 (as amended by core issue 453):
@@ -487,7 +505,7 @@
}
void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc,
- llvm::Value *Address, QualType Ty,
+ llvm::Value *Ptr, QualType Ty,
CharUnits Alignment, bool SkipNullCheck) {
if (!sanitizePerformTypeCheck())
return;
@@ -495,7 +513,7 @@
// Don't check pointers outside the default address space. The null check
// isn't correct, the object-size check isn't supported by LLVM, and we can't
// communicate the addresses to the runtime handler for the vptr check.
- if (Address->getType()->getPointerAddressSpace())
+ if (Ptr->getType()->getPointerAddressSpace())
return;
SanitizerScope SanScope(this);
@@ -508,8 +526,7 @@
if ((SanOpts.has(SanitizerKind::Null) || AllowNullPointers) &&
!SkipNullCheck) {
// The glvalue must not be an empty glvalue.
- llvm::Value *IsNonNull = Builder.CreateICmpNE(
- Address, llvm::Constant::getNullValue(Address->getType()));
+ llvm::Value *IsNonNull = Builder.CreateIsNotNull(Ptr);
if (AllowNullPointers) {
// When performing pointer casts, it's OK if the value is null.
@@ -533,7 +550,7 @@
llvm::Type *Tys[2] = { IntPtrTy, Int8PtrTy };
llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::objectsize, Tys);
llvm::Value *Min = Builder.getFalse();
- llvm::Value *CastAddr = Builder.CreateBitCast(Address, Int8PtrTy);
+ llvm::Value *CastAddr = Builder.CreateBitCast(Ptr, Int8PtrTy);
llvm::Value *LargeEnough =
Builder.CreateICmpUGE(Builder.CreateCall(F, {CastAddr, Min}),
llvm::ConstantInt::get(IntPtrTy, Size));
@@ -550,7 +567,7 @@
// The glvalue must be suitably aligned.
if (AlignVal) {
llvm::Value *Align =
- Builder.CreateAnd(Builder.CreatePtrToInt(Address, IntPtrTy),
+ Builder.CreateAnd(Builder.CreatePtrToInt(Ptr, IntPtrTy),
llvm::ConstantInt::get(IntPtrTy, AlignVal - 1));
llvm::Value *Aligned =
Builder.CreateICmpEQ(Align, llvm::ConstantInt::get(IntPtrTy, 0));
@@ -565,7 +582,7 @@
llvm::ConstantInt::get(SizeTy, AlignVal),
llvm::ConstantInt::get(Int8Ty, TCK)
};
- EmitCheck(Checks, "type_mismatch", StaticData, Address);
+ EmitCheck(Checks, "type_mismatch", StaticData, Ptr);
}
// If possible, check that the vptr indicates that there is a subobject of
@@ -600,7 +617,7 @@
// Load the vptr, and compute hash_16_bytes(TypeHash, vptr).
llvm::Value *Low = llvm::ConstantInt::get(Int64Ty, TypeHash);
llvm::Type *VPtrTy = llvm::PointerType::get(IntPtrTy, 0);
- llvm::Value *VPtrAddr = Builder.CreateBitCast(Address, VPtrTy);
+ Address VPtrAddr(Builder.CreateBitCast(Ptr, VPtrTy), getPointerAlign());
llvm::Value *VPtrVal = Builder.CreateLoad(VPtrAddr);
llvm::Value *High = Builder.CreateZExt(VPtrVal, Int64Ty);
@@ -617,7 +634,8 @@
CacheSize-1));
llvm::Value *Indices[] = { Builder.getInt32(0), Slot };
llvm::Value *CacheVal =
- Builder.CreateLoad(Builder.CreateInBoundsGEP(Cache, Indices));
+ Builder.CreateAlignedLoad(Builder.CreateInBoundsGEP(Cache, Indices),
+ getPointerAlign());
// If the hash isn't in the cache, call a runtime handler to perform the
// hard work of checking whether the vptr is for an object of the right
@@ -630,7 +648,7 @@
CGM.GetAddrOfRTTIDescriptor(Ty.getUnqualifiedType()),
llvm::ConstantInt::get(Int8Ty, TCK)
};
- llvm::Value *DynamicData[] = { Address, Hash };
+ llvm::Value *DynamicData[] = { Ptr, Hash };
EmitCheck(std::make_pair(EqualHash, SanitizerKind::Vptr),
"dynamic_type_cache_miss", StaticData, DynamicData);
}
@@ -758,10 +776,104 @@
return isPre ? IncVal : InVal;
}
+void CodeGenModule::EmitExplicitCastExprType(const ExplicitCastExpr *E,
+ CodeGenFunction *CGF) {
+ // Bind VLAs in the cast type.
+ if (CGF && E->getType()->isVariablyModifiedType())
+ CGF->EmitVariablyModifiedType(E->getType());
+
+ if (CGDebugInfo *DI = getModuleDebugInfo())
+ DI->EmitExplicitCastType(E->getType());
+}
+
//===----------------------------------------------------------------------===//
// LValue Expression Emission
//===----------------------------------------------------------------------===//
+/// EmitPointerWithAlignment - Given an expression of pointer type, try to
+/// derive a more accurate bound on the alignment of the pointer.
+Address CodeGenFunction::EmitPointerWithAlignment(const Expr *E,
+ AlignmentSource *Source) {
+ // We allow this with ObjC object pointers because of fragile ABIs.
+ assert(E->getType()->isPointerType() ||
+ E->getType()->isObjCObjectPointerType());
+ E = E->IgnoreParens();
+
+ // Casts:
+ if (const CastExpr *CE = dyn_cast<CastExpr>(E)) {
+ if (const auto *ECE = dyn_cast<ExplicitCastExpr>(CE))
+ CGM.EmitExplicitCastExprType(ECE, this);
+
+ switch (CE->getCastKind()) {
+ // Non-converting casts (but not C's implicit conversion from void*).
+ case CK_BitCast:
+ case CK_NoOp:
+ if (auto PtrTy = CE->getSubExpr()->getType()->getAs<PointerType>()) {
+ if (PtrTy->getPointeeType()->isVoidType())
+ break;
+
+ AlignmentSource InnerSource;
+ Address Addr = EmitPointerWithAlignment(CE->getSubExpr(), &InnerSource);
+ if (Source) *Source = InnerSource;
+
+ // If this is an explicit bitcast, and the source l-value is
+ // opaque, honor the alignment of the casted-to type.
+ if (isa<ExplicitCastExpr>(CE) &&
+ InnerSource != AlignmentSource::Decl) {
+ Addr = Address(Addr.getPointer(),
+ getNaturalPointeeTypeAlignment(E->getType(), Source));
+ }
+
+ if (SanOpts.has(SanitizerKind::CFIUnrelatedCast)) {
+ if (auto PT = E->getType()->getAs<PointerType>())
+ EmitVTablePtrCheckForCast(PT->getPointeeType(), Addr.getPointer(),
+ /*MayBeNull=*/true,
+ CodeGenFunction::CFITCK_UnrelatedCast,
+ CE->getLocStart());
+ }
+
+ return Builder.CreateBitCast(Addr, ConvertType(E->getType()));
+ }
+ break;
+
+ // Array-to-pointer decay.
+ case CK_ArrayToPointerDecay:
+ return EmitArrayToPointerDecay(CE->getSubExpr(), Source);
+
+ // Derived-to-base conversions.
+ case CK_UncheckedDerivedToBase:
+ case CK_DerivedToBase: {
+ Address Addr = EmitPointerWithAlignment(CE->getSubExpr(), Source);
+ auto Derived = CE->getSubExpr()->getType()->getPointeeCXXRecordDecl();
+ return GetAddressOfBaseClass(Addr, Derived,
+ CE->path_begin(), CE->path_end(),
+ ShouldNullCheckClassCastValue(CE),
+ CE->getExprLoc());
+ }
+
+ // TODO: Is there any reason to treat base-to-derived conversions
+ // specially?
+ default:
+ break;
+ }
+ }
+
+ // Unary &.
+ if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(E)) {
+ if (UO->getOpcode() == UO_AddrOf) {
+ LValue LV = EmitLValue(UO->getSubExpr());
+ if (Source) *Source = LV.getAlignmentSource();
+ return LV.getAddress();
+ }
+ }
+
+ // TODO: conditional operators, comma.
+
+ // Otherwise, use the alignment of the type.
+ CharUnits Align = getNaturalPointeeTypeAlignment(E->getType(), Source);
+ return Address(EmitScalarExpr(E), Align);
+}
+
RValue CodeGenFunction::GetUndefRValue(QualType Ty) {
if (Ty->isVoidType())
return RValue::get(nullptr);
@@ -778,7 +890,7 @@
// identifiable address. Just because the contents of the value are undefined
// doesn't mean that the address can't be taken and compared.
case TEK_Aggregate: {
- llvm::Value *DestPtr = CreateMemTemp(Ty, "undef.agg.tmp");
+ Address DestPtr = CreateMemTemp(Ty, "undef.agg.tmp");
return RValue::getAggregate(DestPtr);
}
@@ -798,7 +910,8 @@
const char *Name) {
ErrorUnsupported(E, Name);
llvm::Type *Ty = llvm::PointerType::getUnqual(ConvertType(E->getType()));
- return MakeAddrLValue(llvm::UndefValue::get(Ty), E->getType());
+ return MakeAddrLValue(Address(llvm::UndefValue::get(Ty), CharUnits::One()),
+ E->getType());
}
LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) {
@@ -808,7 +921,7 @@
else
LV = EmitLValue(E);
if (!isa<DeclRefExpr>(E) && !LV.isBitField() && LV.isSimple())
- EmitTypeCheck(TCK, E->getExprLoc(), LV.getAddress(),
+ EmitTypeCheck(TCK, E->getExprLoc(), LV.getPointer(),
E->getType(), LV.getAlignment());
return LV;
}
@@ -909,6 +1022,8 @@
return EmitUnaryOpLValue(cast<UnaryOperator>(E));
case Expr::ArraySubscriptExprClass:
return EmitArraySubscriptExpr(cast<ArraySubscriptExpr>(E));
+ case Expr::OMPArraySectionExprClass:
+ return EmitOMPArraySectionExpr(cast<OMPArraySectionExpr>(E));
case Expr::ExtVectorElementExprClass:
return EmitExtVectorElementExpr(cast<ExtVectorElementExpr>(E));
case Expr::MemberExprClass:
@@ -1055,9 +1170,10 @@
llvm::Value *CodeGenFunction::EmitLoadOfScalar(LValue lvalue,
SourceLocation Loc) {
return EmitLoadOfScalar(lvalue.getAddress(), lvalue.isVolatile(),
- lvalue.getAlignment().getQuantity(),
- lvalue.getType(), Loc, lvalue.getTBAAInfo(),
- lvalue.getTBAABaseType(), lvalue.getTBAAOffset());
+ lvalue.getType(), Loc, lvalue.getAlignmentSource(),
+ lvalue.getTBAAInfo(),
+ lvalue.getTBAABaseType(), lvalue.getTBAAOffset(),
+ lvalue.isNontemporal());
}
static bool hasBooleanRepresentation(QualType Ty) {
@@ -1117,68 +1233,56 @@
return MDHelper.createRange(Min, End);
}
-llvm::Value *CodeGenFunction::EmitLoadOfScalar(llvm::Value *Addr, bool Volatile,
- unsigned Alignment, QualType Ty,
+llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile,
+ QualType Ty,
SourceLocation Loc,
+ AlignmentSource AlignSource,
llvm::MDNode *TBAAInfo,
QualType TBAABaseType,
- uint64_t TBAAOffset) {
+ uint64_t TBAAOffset,
+ bool isNontemporal) {
// For better performance, handle vector loads differently.
if (Ty->isVectorType()) {
- llvm::Value *V;
- const llvm::Type *EltTy =
- cast<llvm::PointerType>(Addr->getType())->getElementType();
+ const llvm::Type *EltTy = Addr.getElementType();
const auto *VTy = cast<llvm::VectorType>(EltTy);
- // Handle vectors of size 3, like size 4 for better performance.
+ // Handle vectors of size 3 like size 4 for better performance.
if (VTy->getNumElements() == 3) {
// Bitcast to vec4 type.
llvm::VectorType *vec4Ty = llvm::VectorType::get(VTy->getElementType(),
4);
- llvm::PointerType *ptVec4Ty =
- llvm::PointerType::get(vec4Ty,
- (cast<llvm::PointerType>(
- Addr->getType()))->getAddressSpace());
- llvm::Value *Cast = Builder.CreateBitCast(Addr, ptVec4Ty,
- "castToVec4");
+ Address Cast = Builder.CreateElementBitCast(Addr, vec4Ty, "castToVec4");
// Now load value.
- llvm::Value *LoadVal = Builder.CreateLoad(Cast, Volatile, "loadVec4");
+ llvm::Value *V = Builder.CreateLoad(Cast, Volatile, "loadVec4");
// Shuffle vector to get vec3.
- llvm::Constant *Mask[] = {
- llvm::ConstantInt::get(llvm::Type::getInt32Ty(getLLVMContext()), 0),
- llvm::ConstantInt::get(llvm::Type::getInt32Ty(getLLVMContext()), 1),
- llvm::ConstantInt::get(llvm::Type::getInt32Ty(getLLVMContext()), 2)
- };
-
- llvm::Value *MaskV = llvm::ConstantVector::get(Mask);
- V = Builder.CreateShuffleVector(LoadVal,
- llvm::UndefValue::get(vec4Ty),
- MaskV, "extractVec");
+ V = Builder.CreateShuffleVector(V, llvm::UndefValue::get(vec4Ty),
+ {0, 1, 2}, "extractVec");
return EmitFromMemory(V, Ty);
}
}
// Atomic operations have to be done on integral types.
if (Ty->isAtomicType() || typeIsSuitableForInlineAtomic(Ty, Volatile)) {
- LValue lvalue = LValue::MakeAddr(Addr, Ty,
- CharUnits::fromQuantity(Alignment),
- getContext(), TBAAInfo);
+ LValue lvalue =
+ LValue::MakeAddr(Addr, Ty, getContext(), AlignSource, TBAAInfo);
return EmitAtomicLoad(lvalue, Loc).getScalarVal();
}
- llvm::LoadInst *Load = Builder.CreateLoad(Addr);
- if (Volatile)
- Load->setVolatile(true);
- if (Alignment)
- Load->setAlignment(Alignment);
+ llvm::LoadInst *Load = Builder.CreateLoad(Addr, Volatile);
+ if (isNontemporal) {
+ llvm::MDNode *Node = llvm::MDNode::get(
+ Load->getContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
+ Load->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node);
+ }
if (TBAAInfo) {
llvm::MDNode *TBAAPath = CGM.getTBAAStructTagInfo(TBAABaseType, TBAAInfo,
TBAAOffset);
if (TBAAPath)
- CGM.DecorateInstruction(Load, TBAAPath, false/*ConvertTypeToTag*/);
+ CGM.DecorateInstructionWithTBAA(Load, TBAAPath,
+ false /*ConvertTypeToTag*/);
}
bool NeedsBoolCheck =
@@ -1241,11 +1345,13 @@
return Value;
}
-void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, llvm::Value *Addr,
- bool Volatile, unsigned Alignment,
- QualType Ty, llvm::MDNode *TBAAInfo,
+void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr,
+ bool Volatile, QualType Ty,
+ AlignmentSource AlignSource,
+ llvm::MDNode *TBAAInfo,
bool isInit, QualType TBAABaseType,
- uint64_t TBAAOffset) {
+ uint64_t TBAAOffset,
+ bool isNontemporal) {
// Handle vectors differently to get better performance.
if (Ty->isVectorType()) {
@@ -1253,29 +1359,18 @@
auto *VecTy = cast<llvm::VectorType>(SrcTy);
// Handle vec3 special.
if (VecTy->getNumElements() == 3) {
- llvm::LLVMContext &VMContext = getLLVMContext();
-
// Our source is a vec3, do a shuffle vector to make it a vec4.
- SmallVector<llvm::Constant*, 4> Mask;
- Mask.push_back(llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext),
- 0));
- Mask.push_back(llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext),
- 1));
- Mask.push_back(llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext),
- 2));
- Mask.push_back(llvm::UndefValue::get(llvm::Type::getInt32Ty(VMContext)));
-
+ llvm::Constant *Mask[] = {Builder.getInt32(0), Builder.getInt32(1),
+ Builder.getInt32(2),
+ llvm::UndefValue::get(Builder.getInt32Ty())};
llvm::Value *MaskV = llvm::ConstantVector::get(Mask);
Value = Builder.CreateShuffleVector(Value,
llvm::UndefValue::get(VecTy),
MaskV, "extractVec");
SrcTy = llvm::VectorType::get(VecTy->getElementType(), 4);
}
- auto *DstPtr = cast<llvm::PointerType>(Addr->getType());
- if (DstPtr->getElementType() != SrcTy) {
- llvm::Type *MemTy =
- llvm::PointerType::get(SrcTy, DstPtr->getAddressSpace());
- Addr = Builder.CreateBitCast(Addr, MemTy, "storetmp");
+ if (Addr.getElementType() != SrcTy) {
+ Addr = Builder.CreateElementBitCast(Addr, SrcTy, "storetmp");
}
}
@@ -1284,30 +1379,34 @@
if (Ty->isAtomicType() ||
(!isInit && typeIsSuitableForInlineAtomic(Ty, Volatile))) {
EmitAtomicStore(RValue::get(Value),
- LValue::MakeAddr(Addr, Ty,
- CharUnits::fromQuantity(Alignment),
- getContext(), TBAAInfo),
+ LValue::MakeAddr(Addr, Ty, getContext(),
+ AlignSource, TBAAInfo),
isInit);
return;
}
llvm::StoreInst *Store = Builder.CreateStore(Value, Addr, Volatile);
- if (Alignment)
- Store->setAlignment(Alignment);
+ if (isNontemporal) {
+ llvm::MDNode *Node =
+ llvm::MDNode::get(Store->getContext(),
+ llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
+ Store->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node);
+ }
if (TBAAInfo) {
llvm::MDNode *TBAAPath = CGM.getTBAAStructTagInfo(TBAABaseType, TBAAInfo,
TBAAOffset);
if (TBAAPath)
- CGM.DecorateInstruction(Store, TBAAPath, false/*ConvertTypeToTag*/);
+ CGM.DecorateInstructionWithTBAA(Store, TBAAPath,
+ false /*ConvertTypeToTag*/);
}
}
void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue,
bool isInit) {
EmitStoreOfScalar(value, lvalue.getAddress(), lvalue.isVolatile(),
- lvalue.getAlignment().getQuantity(), lvalue.getType(),
+ lvalue.getType(), lvalue.getAlignmentSource(),
lvalue.getTBAAInfo(), isInit, lvalue.getTBAABaseType(),
- lvalue.getTBAAOffset());
+ lvalue.getTBAAOffset(), lvalue.isNontemporal());
}
/// EmitLoadOfLValue - Given an expression that represents a value lvalue, this
@@ -1316,11 +1415,17 @@
RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, SourceLocation Loc) {
if (LV.isObjCWeak()) {
// load of a __weak object.
- llvm::Value *AddrWeakObj = LV.getAddress();
+ Address AddrWeakObj = LV.getAddress();
return RValue::get(CGM.getObjCRuntime().EmitObjCWeakRead(*this,
AddrWeakObj));
}
if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) {
+ // In MRC mode, we do a load+autorelease.
+ if (!getLangOpts().ObjCAutoRefCount) {
+ return RValue::get(EmitARCLoadWeak(LV.getAddress()));
+ }
+
+ // In ARC mode, we load retained and then consume the value.
llvm::Value *Object = EmitARCLoadWeakRetained(LV.getAddress());
Object = EmitObjCConsumeObject(LV.getType(), Object);
return RValue::get(Object);
@@ -1334,9 +1439,8 @@
}
if (LV.isVectorElt()) {
- llvm::LoadInst *Load = Builder.CreateLoad(LV.getVectorAddr(),
+ llvm::LoadInst *Load = Builder.CreateLoad(LV.getVectorAddress(),
LV.isVolatileQualified());
- Load->setAlignment(LV.getAlignment().getQuantity());
return RValue::get(Builder.CreateExtractElement(Load, LV.getVectorIdx(),
"vecext"));
}
@@ -1360,10 +1464,8 @@
// Get the output type.
llvm::Type *ResLTy = ConvertType(LV.getType());
- llvm::Value *Ptr = LV.getBitFieldAddr();
- llvm::Value *Val = Builder.CreateLoad(Ptr, LV.isVolatileQualified(),
- "bf.load");
- cast<llvm::LoadInst>(Val)->setAlignment(Info.StorageAlignment);
+ Address Ptr = LV.getBitFieldAddress();
+ llvm::Value *Val = Builder.CreateLoad(Ptr, LV.isVolatileQualified(), "bf.load");
if (Info.IsSigned) {
assert(static_cast<unsigned>(Info.Offset + Info.Size) <= Info.StorageSize);
@@ -1388,10 +1490,8 @@
// If this is a reference to a subset of the elements of a vector, create an
// appropriate shufflevector.
RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) {
- llvm::LoadInst *Load = Builder.CreateLoad(LV.getExtVectorAddr(),
- LV.isVolatileQualified());
- Load->setAlignment(LV.getAlignment().getQuantity());
- llvm::Value *Vec = Load;
+ llvm::Value *Vec = Builder.CreateLoad(LV.getExtVectorAddress(),
+ LV.isVolatileQualified());
const llvm::Constant *Elts = LV.getExtVectorElts();
@@ -1418,24 +1518,24 @@
}
/// @brief Generates lvalue for partial ext_vector access.
-llvm::Value *CodeGenFunction::EmitExtVectorElementLValue(LValue LV) {
- llvm::Value *VectorAddress = LV.getExtVectorAddr();
+Address CodeGenFunction::EmitExtVectorElementLValue(LValue LV) {
+ Address VectorAddress = LV.getExtVectorAddress();
const VectorType *ExprVT = LV.getType()->getAs<VectorType>();
QualType EQT = ExprVT->getElementType();
llvm::Type *VectorElementTy = CGM.getTypes().ConvertType(EQT);
- llvm::Type *VectorElementPtrToTy = VectorElementTy->getPointerTo();
- llvm::Value *CastToPointerElement =
- Builder.CreateBitCast(VectorAddress,
- VectorElementPtrToTy, "conv.ptr.element");
+ Address CastToPointerElement =
+ Builder.CreateElementBitCast(VectorAddress, VectorElementTy,
+ "conv.ptr.element");
const llvm::Constant *Elts = LV.getExtVectorElts();
unsigned ix = getAccessedFieldNo(0, Elts);
- llvm::Value *VectorBasePtrPlusIx =
- Builder.CreateInBoundsGEP(CastToPointerElement,
- llvm::ConstantInt::get(SizeTy, ix), "add.ptr");
-
+ Address VectorBasePtrPlusIx =
+ Builder.CreateConstInBoundsGEP(CastToPointerElement, ix,
+ getContext().getTypeSizeInChars(EQT),
+ "vector.elt");
+
return VectorBasePtrPlusIx;
}
@@ -1470,15 +1570,12 @@
if (!Dst.isSimple()) {
if (Dst.isVectorElt()) {
// Read/modify/write the vector, inserting the new element.
- llvm::LoadInst *Load = Builder.CreateLoad(Dst.getVectorAddr(),
- Dst.isVolatileQualified());
- Load->setAlignment(Dst.getAlignment().getQuantity());
- llvm::Value *Vec = Load;
+ llvm::Value *Vec = Builder.CreateLoad(Dst.getVectorAddress(),
+ Dst.isVolatileQualified());
Vec = Builder.CreateInsertElement(Vec, Src.getScalarVal(),
Dst.getVectorIdx(), "vecins");
- llvm::StoreInst *Store = Builder.CreateStore(Vec, Dst.getVectorAddr(),
- Dst.isVolatileQualified());
- Store->setAlignment(Dst.getAlignment().getQuantity());
+ Builder.CreateStore(Vec, Dst.getVectorAddress(),
+ Dst.isVolatileQualified());
return;
}
@@ -1522,7 +1619,7 @@
if (Dst.isObjCWeak() && !Dst.isNonGC()) {
// load of a __weak object.
- llvm::Value *LvalueDst = Dst.getAddress();
+ Address LvalueDst = Dst.getAddress();
llvm::Value *src = Src.getScalarVal();
CGM.getObjCRuntime().EmitObjCWeakAssign(*this, src, LvalueDst);
return;
@@ -1530,16 +1627,17 @@
if (Dst.isObjCStrong() && !Dst.isNonGC()) {
// load of a __strong object.
- llvm::Value *LvalueDst = Dst.getAddress();
+ Address LvalueDst = Dst.getAddress();
llvm::Value *src = Src.getScalarVal();
if (Dst.isObjCIvar()) {
assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL");
- llvm::Type *ResultType = ConvertType(getContext().LongTy);
- llvm::Value *RHS = EmitScalarExpr(Dst.getBaseIvarExp());
- llvm::Value *dst = RHS;
+ llvm::Type *ResultType = IntPtrTy;
+ Address dst = EmitPointerWithAlignment(Dst.getBaseIvarExp());
+ llvm::Value *RHS = dst.getPointer();
RHS = Builder.CreatePtrToInt(RHS, ResultType, "sub.ptr.rhs.cast");
llvm::Value *LHS =
- Builder.CreatePtrToInt(LvalueDst, ResultType, "sub.ptr.lhs.cast");
+ Builder.CreatePtrToInt(LvalueDst.getPointer(), ResultType,
+ "sub.ptr.lhs.cast");
llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, "ivar.offset");
CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst,
BytesBetween);
@@ -1560,14 +1658,13 @@
llvm::Value **Result) {
const CGBitFieldInfo &Info = Dst.getBitFieldInfo();
llvm::Type *ResLTy = ConvertTypeForMem(Dst.getType());
- llvm::Value *Ptr = Dst.getBitFieldAddr();
+ Address Ptr = Dst.getBitFieldAddress();
// Get the source value, truncated to the width of the bit-field.
llvm::Value *SrcVal = Src.getScalarVal();
// Cast the source to the storage type and shift it into place.
- SrcVal = Builder.CreateIntCast(SrcVal,
- Ptr->getType()->getPointerElementType(),
+ SrcVal = Builder.CreateIntCast(SrcVal, Ptr.getElementType(),
/*IsSigned=*/false);
llvm::Value *MaskedVal = SrcVal;
@@ -1575,9 +1672,8 @@
// and mask together with source before storing.
if (Info.StorageSize != Info.Size) {
assert(Info.StorageSize > Info.Size && "Invalid bitfield size.");
- llvm::Value *Val = Builder.CreateLoad(Ptr, Dst.isVolatileQualified(),
- "bf.load");
- cast<llvm::LoadInst>(Val)->setAlignment(Info.StorageAlignment);
+ llvm::Value *Val =
+ Builder.CreateLoad(Ptr, Dst.isVolatileQualified(), "bf.load");
// Mask the source value as needed.
if (!hasBooleanRepresentation(Dst.getType()))
@@ -1603,9 +1699,7 @@
}
// Write the new value back out.
- llvm::StoreInst *Store = Builder.CreateStore(SrcVal, Ptr,
- Dst.isVolatileQualified());
- Store->setAlignment(Info.StorageAlignment);
+ Builder.CreateStore(SrcVal, Ptr, Dst.isVolatileQualified());
// Return the new value of the bit-field, if requested.
if (Result) {
@@ -1631,10 +1725,8 @@
LValue Dst) {
// This access turns into a read/modify/write of the vector. Load the input
// value now.
- llvm::LoadInst *Load = Builder.CreateLoad(Dst.getExtVectorAddr(),
- Dst.isVolatileQualified());
- Load->setAlignment(Dst.getAlignment().getQuantity());
- llvm::Value *Vec = Load;
+ llvm::Value *Vec = Builder.CreateLoad(Dst.getExtVectorAddress(),
+ Dst.isVolatileQualified());
const llvm::Constant *Elts = Dst.getExtVectorElts();
llvm::Value *SrcVal = Src.getScalarVal();
@@ -1696,9 +1788,8 @@
Vec = Builder.CreateInsertElement(Vec, SrcVal, Elt);
}
- llvm::StoreInst *Store = Builder.CreateStore(Vec, Dst.getExtVectorAddr(),
- Dst.isVolatileQualified());
- Store->setAlignment(Dst.getAlignment().getQuantity());
+ Builder.CreateStore(Vec, Dst.getExtVectorAddress(),
+ Dst.isVolatileQualified());
}
/// @brief Store of global named registers are always calls to intrinsics.
@@ -1833,11 +1924,27 @@
}
static LValue EmitThreadPrivateVarDeclLValue(
- CodeGenFunction &CGF, const VarDecl *VD, QualType T, llvm::Value *V,
- llvm::Type *RealVarTy, CharUnits Alignment, SourceLocation Loc) {
- V = CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, V, Loc);
- V = EmitBitCastOfLValueToProperType(CGF, V, RealVarTy);
- return CGF.MakeAddrLValue(V, T, Alignment);
+ CodeGenFunction &CGF, const VarDecl *VD, QualType T, Address Addr,
+ llvm::Type *RealVarTy, SourceLocation Loc) {
+ Addr = CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, Addr, Loc);
+ Addr = CGF.Builder.CreateElementBitCast(Addr, RealVarTy);
+ return CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);
+}
+
+Address CodeGenFunction::EmitLoadOfReference(Address Addr,
+ const ReferenceType *RefTy,
+ AlignmentSource *Source) {
+ llvm::Value *Ptr = Builder.CreateLoad(Addr);
+ return Address(Ptr, getNaturalTypeAlignment(RefTy->getPointeeType(),
+ Source, /*forPointee*/ true));
+
+}
+
+LValue CodeGenFunction::EmitLoadOfReferenceLValue(Address RefAddr,
+ const ReferenceType *RefTy) {
+ AlignmentSource Source;
+ Address Addr = EmitLoadOfReference(RefAddr, RefTy, &Source);
+ return MakeAddrLValue(Addr, RefTy->getPointeeType(), Source);
}
static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,
@@ -1853,19 +1960,17 @@
llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(VD->getType());
V = EmitBitCastOfLValueToProperType(CGF, V, RealVarTy);
CharUnits Alignment = CGF.getContext().getDeclAlign(VD);
+ Address Addr(V, Alignment);
LValue LV;
// Emit reference to the private copy of the variable if it is an OpenMP
// threadprivate variable.
if (CGF.getLangOpts().OpenMP && VD->hasAttr<OMPThreadPrivateDeclAttr>())
- return EmitThreadPrivateVarDeclLValue(CGF, VD, T, V, RealVarTy, Alignment,
+ return EmitThreadPrivateVarDeclLValue(CGF, VD, T, Addr, RealVarTy,
E->getExprLoc());
- if (VD->getType()->isReferenceType()) {
- llvm::LoadInst *LI = CGF.Builder.CreateLoad(V);
- LI->setAlignment(Alignment.getQuantity());
- V = LI;
- LV = CGF.MakeNaturalAlignAddrLValue(V, T);
+ if (auto RefTy = VD->getType()->getAs<ReferenceType>()) {
+ LV = CGF.EmitLoadOfReferenceLValue(Addr, RefTy);
} else {
- LV = CGF.MakeAddrLValue(V, T, Alignment);
+ LV = CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);
}
setObjCGCLValueClass(CGF.getContext(), E, LV);
return LV;
@@ -1887,7 +1992,7 @@
}
}
CharUnits Alignment = CGF.getContext().getDeclAlign(FD);
- return CGF.MakeAddrLValue(V, E->getType(), Alignment);
+ return CGF.MakeAddrLValue(V, E->getType(), Alignment, AlignmentSource::Decl);
}
static LValue EmitCapturedFieldLValue(CodeGenFunction &CGF, const FieldDecl *FD,
@@ -1903,9 +2008,7 @@
/// So far, only the name is being passed down, but other options such as
/// register type, allocation type or even optimization options could be
/// passed down via the metadata node.
-static LValue EmitGlobalNamedRegister(const VarDecl *VD,
- CodeGenModule &CGM,
- CharUnits Alignment) {
+static LValue EmitGlobalNamedRegister(const VarDecl *VD, CodeGenModule &CGM) {
SmallString<64> Name("llvm.named.register.");
AsmLabelAttr *Asm = VD->getAttr<AsmLabelAttr>();
assert(Asm->getLabel().size() < 64-Name.size() &&
@@ -1919,33 +2022,43 @@
llvm::Metadata *Ops[] = {Str};
M->addOperand(llvm::MDNode::get(CGM.getLLVMContext(), Ops));
}
- return LValue::MakeGlobalReg(
- llvm::MetadataAsValue::get(CGM.getLLVMContext(), M->getOperand(0)),
- VD->getType(), Alignment);
+
+ CharUnits Alignment = CGM.getContext().getDeclAlign(VD);
+
+ llvm::Value *Ptr =
+ llvm::MetadataAsValue::get(CGM.getLLVMContext(), M->getOperand(0));
+ return LValue::MakeGlobalReg(Address(Ptr, Alignment), VD->getType());
}
LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
const NamedDecl *ND = E->getDecl();
- CharUnits Alignment = getContext().getDeclAlign(ND);
QualType T = E->getType();
if (const auto *VD = dyn_cast<VarDecl>(ND)) {
// Global Named registers access via intrinsics only
if (VD->getStorageClass() == SC_Register &&
VD->hasAttr<AsmLabelAttr>() && !VD->isLocalVarDecl())
- return EmitGlobalNamedRegister(VD, CGM, Alignment);
+ return EmitGlobalNamedRegister(VD, CGM);
// A DeclRefExpr for a reference initialized by a constant expression can
// appear without being odr-used. Directly emit the constant initializer.
const Expr *Init = VD->getAnyInitializer(VD);
if (Init && !isa<ParmVarDecl>(VD) && VD->getType()->isReferenceType() &&
VD->isUsableInConstantExpressions(getContext()) &&
- VD->checkInitIsICE()) {
+ VD->checkInitIsICE() &&
+ // Do not emit if it is private OpenMP variable.
+ !(E->refersToEnclosingVariableOrCapture() && CapturedStmtInfo &&
+ LocalDeclMap.count(VD))) {
llvm::Constant *Val =
CGM.EmitConstantValue(*VD->evaluateValue(), VD->getType(), this);
assert(Val && "failed to emit reference constant expression");
// FIXME: Eventually we will want to emit vector element references.
- return MakeAddrLValue(Val, T, Alignment);
+
+ // Should we be using the alignment of the constant pointer we emitted?
+ CharUnits Alignment = getNaturalTypeAlignment(E->getType(), nullptr,
+ /*pointee*/ true);
+
+ return MakeAddrLValue(Address(Val, Alignment), T, AlignmentSource::Decl);
}
// Check for captured variables.
@@ -1953,15 +2066,24 @@
if (auto *FD = LambdaCaptureFields.lookup(VD))
return EmitCapturedFieldLValue(*this, FD, CXXABIThisValue);
else if (CapturedStmtInfo) {
- if (auto *V = LocalDeclMap.lookup(VD))
- return MakeAddrLValue(V, T, Alignment);
- else
- return EmitCapturedFieldLValue(*this, CapturedStmtInfo->lookup(VD),
- CapturedStmtInfo->getContextValue());
+ auto it = LocalDeclMap.find(VD);
+ if (it != LocalDeclMap.end()) {
+ if (auto RefTy = VD->getType()->getAs<ReferenceType>()) {
+ return EmitLoadOfReferenceLValue(it->second, RefTy);
+ }
+ return MakeAddrLValue(it->second, T);
+ }
+ LValue CapLVal =
+ EmitCapturedFieldLValue(*this, CapturedStmtInfo->lookup(VD),
+ CapturedStmtInfo->getContextValue());
+ return MakeAddrLValue(
+ Address(CapLVal.getPointer(), getContext().getDeclAlign(VD)),
+ CapLVal.getType(), AlignmentSource::Decl);
}
+
assert(isa<BlockDecl>(CurCodeDecl));
- return MakeAddrLValue(GetAddrOfBlockDecl(VD, VD->hasAttr<BlocksAttr>()),
- T, Alignment);
+ Address addr = GetAddrOfBlockDecl(VD, VD->hasAttr<BlocksAttr>());
+ return MakeAddrLValue(addr, T, AlignmentSource::Decl);
}
}
@@ -1974,8 +2096,8 @@
if (ND->hasAttr<WeakRefAttr>()) {
const auto *VD = cast<ValueDecl>(ND);
- llvm::Constant *Aliasee = CGM.GetWeakRefReference(VD);
- return MakeAddrLValue(Aliasee, T, Alignment);
+ ConstantAddress Aliasee = CGM.GetWeakRefReference(VD);
+ return MakeAddrLValue(Aliasee, T, AlignmentSource::Decl);
}
if (const auto *VD = dyn_cast<VarDecl>(ND)) {
@@ -1983,39 +2105,52 @@
if (VD->hasLinkage() || VD->isStaticDataMember())
return EmitGlobalVarDeclLValue(*this, E, VD);
- bool isBlockVariable = VD->hasAttr<BlocksAttr>();
+ Address addr = Address::invalid();
- llvm::Value *V = LocalDeclMap.lookup(VD);
- if (!V && VD->isStaticLocal())
- V = CGM.getOrCreateStaticVarDecl(
- *VD, CGM.getLLVMLinkageVarDefinition(VD, /*isConstant=*/false));
+ // The variable should generally be present in the local decl map.
+ auto iter = LocalDeclMap.find(VD);
+ if (iter != LocalDeclMap.end()) {
+ addr = iter->second;
- // Check if variable is threadprivate.
- if (V && getLangOpts().OpenMP && VD->hasAttr<OMPThreadPrivateDeclAttr>())
- return EmitThreadPrivateVarDeclLValue(
- *this, VD, T, V, getTypes().ConvertTypeForMem(VD->getType()),
- Alignment, E->getExprLoc());
+ // Otherwise, it might be static local we haven't emitted yet for
+ // some reason; most likely, because it's in an outer function.
+ } else if (VD->isStaticLocal()) {
+ addr = Address(CGM.getOrCreateStaticVarDecl(
+ *VD, CGM.getLLVMLinkageVarDefinition(VD, /*isConstant=*/false)),
+ getContext().getDeclAlign(VD));
- assert(V && "DeclRefExpr not entered in LocalDeclMap?");
-
- if (isBlockVariable)
- V = BuildBlockByrefAddress(V, VD);
-
- LValue LV;
- if (VD->getType()->isReferenceType()) {
- llvm::LoadInst *LI = Builder.CreateLoad(V);
- LI->setAlignment(Alignment.getQuantity());
- V = LI;
- LV = MakeNaturalAlignAddrLValue(V, T);
+ // No other cases for now.
} else {
- LV = MakeAddrLValue(V, T, Alignment);
+ llvm_unreachable("DeclRefExpr for Decl not entered in LocalDeclMap?");
+ }
+
+
+ // Check for OpenMP threadprivate variables.
+ if (getLangOpts().OpenMP && VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
+ return EmitThreadPrivateVarDeclLValue(
+ *this, VD, T, addr, getTypes().ConvertTypeForMem(VD->getType()),
+ E->getExprLoc());
+ }
+
+ // Drill into block byref variables.
+ bool isBlockByref = VD->hasAttr<BlocksAttr>();
+ if (isBlockByref) {
+ addr = emitBlockByrefAddress(addr, VD);
+ }
+
+ // Drill into reference types.
+ LValue LV;
+ if (auto RefTy = VD->getType()->getAs<ReferenceType>()) {
+ LV = EmitLoadOfReferenceLValue(addr, RefTy);
+ } else {
+ LV = MakeAddrLValue(addr, T, AlignmentSource::Decl);
}
bool isLocalStorage = VD->hasLocalStorage();
bool NonGCable = isLocalStorage &&
!VD->getType()->isReferenceType() &&
- !isBlockVariable;
+ !isBlockByref;
if (NonGCable) {
LV.getQuals().removeObjCGCAttr();
LV.setNonGC(true);
@@ -2047,7 +2182,9 @@
QualType T = E->getSubExpr()->getType()->getPointeeType();
assert(!T.isNull() && "CodeGenFunction::EmitUnaryOpLValue: Illegal type");
- LValue LV = MakeNaturalAlignAddrLValue(EmitScalarExpr(E->getSubExpr()), T);
+ AlignmentSource AlignSource;
+ Address Addr = EmitPointerWithAlignment(E->getSubExpr(), &AlignSource);
+ LValue LV = MakeAddrLValue(Addr, T, AlignSource);
LV.getQuals().setAddressSpace(ExprTy.getAddressSpace());
// We should not generate __weak write barrier on indirect reference
@@ -2064,22 +2201,22 @@
case UO_Imag: {
LValue LV = EmitLValue(E->getSubExpr());
assert(LV.isSimple() && "real/imag on non-ordinary l-value");
- llvm::Value *Addr = LV.getAddress();
// __real is valid on scalars. This is a faster way of testing that.
// __imag can only produce an rvalue on scalars.
if (E->getOpcode() == UO_Real &&
- !cast<llvm::PointerType>(Addr->getType())
- ->getElementType()->isStructTy()) {
+ !LV.getAddress().getElementType()->isStructTy()) {
assert(E->getSubExpr()->getType()->isArithmeticType());
return LV;
}
assert(E->getSubExpr()->getType()->isAnyComplexType());
- unsigned Idx = E->getOpcode() == UO_Imag;
- return MakeAddrLValue(
- Builder.CreateStructGEP(nullptr, LV.getAddress(), Idx, "idx"), ExprTy);
+ Address Component =
+ (E->getOpcode() == UO_Real
+ ? emitAddrOfRealComponent(LV.getAddress(), LV.getType())
+ : emitAddrOfImagComponent(LV.getAddress(), LV.getType()));
+ return MakeAddrLValue(Component, ExprTy, LV.getAlignmentSource());
}
case UO_PreInc:
case UO_PreDec: {
@@ -2097,12 +2234,12 @@
LValue CodeGenFunction::EmitStringLiteralLValue(const StringLiteral *E) {
return MakeAddrLValue(CGM.GetAddrOfConstantStringFromLiteral(E),
- E->getType());
+ E->getType(), AlignmentSource::Decl);
}
LValue CodeGenFunction::EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E) {
return MakeAddrLValue(CGM.GetAddrOfConstantStringFromObjCEncode(E),
- E->getType());
+ E->getType(), AlignmentSource::Decl);
}
LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) {
@@ -2115,11 +2252,11 @@
PredefinedExpr::getIdentTypeName(E->getIdentType()), FnName};
std::string GVName = llvm::join(NameItems, NameItems + 2, ".");
if (CurCodeDecl && isa<BlockDecl>(CurCodeDecl)) {
- auto C = CGM.GetAddrOfConstantCString(FnName, GVName.c_str(), 1);
- return MakeAddrLValue(C, E->getType());
+ auto C = CGM.GetAddrOfConstantCString(FnName, GVName.c_str());
+ return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl);
}
auto C = CGM.GetAddrOfConstantStringFromLiteral(SL, GVName);
- return MakeAddrLValue(C, E->getType());
+ return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl);
}
/// Emit a type description suitable for use by a runtime sanitizer library. The
@@ -2193,9 +2330,9 @@
// Pointers are passed directly, everything else is passed by address.
if (!V->getType()->isPointerTy()) {
- llvm::Value *Ptr = CreateTempAlloca(V->getType());
+ Address Ptr = CreateDefaultAlignTempAlloca(V->getType());
Builder.CreateStore(V, Ptr);
- V = Ptr;
+ V = Ptr.getPointer();
}
return Builder.CreatePtrToInt(V, TargetTy);
}
@@ -2216,8 +2353,9 @@
PresumedLoc PLoc = getContext().getSourceManager().getPresumedLoc(Loc);
if (PLoc.isValid()) {
auto FilenameGV = CGM.GetAddrOfConstantCString(PLoc.getFilename(), ".src");
- CGM.getSanitizerMetadata()->disableSanitizerForGlobal(FilenameGV);
- Filename = FilenameGV;
+ CGM.getSanitizerMetadata()->disableSanitizerForGlobal(
+ cast<llvm::GlobalVariable>(FilenameGV.getPointer()));
+ Filename = FilenameGV.getPointer();
Line = PLoc.getLine();
Column = PLoc.getColumn();
} else {
@@ -2300,15 +2438,24 @@
llvm::Value *FatalCond = nullptr;
llvm::Value *RecoverableCond = nullptr;
+ llvm::Value *TrapCond = nullptr;
for (int i = 0, n = Checked.size(); i < n; ++i) {
llvm::Value *Check = Checked[i].first;
+ // -fsanitize-trap= overrides -fsanitize-recover=.
llvm::Value *&Cond =
- CGM.getCodeGenOpts().SanitizeRecover.has(Checked[i].second)
- ? RecoverableCond
- : FatalCond;
+ CGM.getCodeGenOpts().SanitizeTrap.has(Checked[i].second)
+ ? TrapCond
+ : CGM.getCodeGenOpts().SanitizeRecover.has(Checked[i].second)
+ ? RecoverableCond
+ : FatalCond;
Cond = Cond ? Builder.CreateAnd(Cond, Check) : Check;
}
+ if (TrapCond)
+ EmitTrapCheck(TrapCond);
+ if (!FatalCond && !RecoverableCond)
+ return;
+
llvm::Value *JointCond;
if (FatalCond && RecoverableCond)
JointCond = Builder.CreateAnd(FatalCond, RecoverableCond);
@@ -2326,15 +2473,6 @@
}
#endif
- if (CGM.getCodeGenOpts().SanitizeUndefinedTrapOnError) {
- assert(RecoverKind != CheckRecoverableKind::AlwaysRecoverable &&
- "Runtime call required for AlwaysRecoverable kind!");
- // Assume that -fsanitize-undefined-trap-on-error overrides
- // -fsanitize-recover= options, as we can only print meaningful error
- // message and recover if we have a runtime support.
- return EmitTrapCheck(JointCond);
- }
-
llvm::BasicBlock *Cont = createBasicBlock("cont");
llvm::BasicBlock *Handlers = createBasicBlock("handler." + CheckName);
llvm::Instruction *Branch = Builder.CreateCondBr(JointCond, Cont, Handlers);
@@ -2394,6 +2532,34 @@
EmitBlock(Cont);
}
+void CodeGenFunction::EmitCfiSlowPathCheck(llvm::Value *Cond,
+ llvm::ConstantInt *TypeId,
+ llvm::Value *Ptr) {
+ auto &Ctx = getLLVMContext();
+ llvm::BasicBlock *Cont = createBasicBlock("cfi.cont");
+
+ llvm::BasicBlock *CheckBB = createBasicBlock("cfi.slowpath");
+ llvm::BranchInst *BI = Builder.CreateCondBr(Cond, Cont, CheckBB);
+
+ llvm::MDBuilder MDHelper(getLLVMContext());
+ llvm::MDNode *Node = MDHelper.createBranchWeights((1U << 20) - 1, 1);
+ BI->setMetadata(llvm::LLVMContext::MD_prof, Node);
+
+ EmitBlock(CheckBB);
+
+ llvm::Constant *SlowPathFn = CGM.getModule().getOrInsertFunction(
+ "__cfi_slowpath",
+ llvm::FunctionType::get(
+ llvm::Type::getVoidTy(Ctx),
+ {llvm::Type::getInt64Ty(Ctx),
+ llvm::PointerType::getUnqual(llvm::Type::getInt8Ty(Ctx))},
+ false));
+ llvm::CallInst *CheckCall = Builder.CreateCall(SlowPathFn, {TypeId, Ptr});
+ CheckCall->setDoesNotThrow();
+
+ EmitBlock(Cont);
+}
+
void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked) {
llvm::BasicBlock *Cont = createBasicBlock("cont");
@@ -2403,8 +2569,7 @@
TrapBB = createBasicBlock("trap");
Builder.CreateCondBr(Checked, Cont, TrapBB);
EmitBlock(TrapBB);
- llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::trap);
- llvm::CallInst *TrapCall = Builder.CreateCall(F, {});
+ llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);
TrapCall->setDoesNotReturn();
TrapCall->setDoesNotThrow();
Builder.CreateUnreachable();
@@ -2415,6 +2580,44 @@
EmitBlock(Cont);
}
+llvm::CallInst *CodeGenFunction::EmitTrapCall(llvm::Intrinsic::ID IntrID) {
+ llvm::CallInst *TrapCall = Builder.CreateCall(CGM.getIntrinsic(IntrID));
+
+ if (!CGM.getCodeGenOpts().TrapFuncName.empty())
+ TrapCall->addAttribute(llvm::AttributeSet::FunctionIndex,
+ "trap-func-name",
+ CGM.getCodeGenOpts().TrapFuncName);
+
+ return TrapCall;
+}
+
+Address CodeGenFunction::EmitArrayToPointerDecay(const Expr *E,
+ AlignmentSource *AlignSource) {
+ assert(E->getType()->isArrayType() &&
+ "Array to pointer decay must have array source type!");
+
+ // Expressions of array type can't be bitfields or vector elements.
+ LValue LV = EmitLValue(E);
+ Address Addr = LV.getAddress();
+ if (AlignSource) *AlignSource = LV.getAlignmentSource();
+
+ // If the array type was an incomplete type, we need to make sure
+ // the decay ends up being the right type.
+ llvm::Type *NewTy = ConvertType(E->getType());
+ Addr = Builder.CreateElementBitCast(Addr, NewTy);
+
+ // Note that VLA pointers are always decayed, so we don't need to do
+ // anything here.
+ if (!E->getType()->isVariableArrayType()) {
+ assert(isa<llvm::ArrayType>(Addr.getElementType()) &&
+ "Expected pointer to array");
+ Addr = Builder.CreateStructGEP(Addr, 0, CharUnits::Zero(), "arraydecay");
+ }
+
+ QualType EltType = E->getType()->castAsArrayTypeUnsafe()->getElementType();
+ return Builder.CreateElementBitCast(Addr, ConvertTypeForMem(EltType));
+}
+
/// isSimpleArrayDecayOperand - If the specified expr is a simple decay from an
/// array to pointer, return the array subexpression.
static const Expr *isSimpleArrayDecayOperand(const Expr *E) {
@@ -2431,6 +2634,69 @@
return SubExpr;
}
+static llvm::Value *emitArraySubscriptGEP(CodeGenFunction &CGF,
+ llvm::Value *ptr,
+ ArrayRef<llvm::Value*> indices,
+ bool inbounds,
+ const llvm::Twine &name = "arrayidx") {
+ if (inbounds) {
+ return CGF.Builder.CreateInBoundsGEP(ptr, indices, name);
+ } else {
+ return CGF.Builder.CreateGEP(ptr, indices, name);
+ }
+}
+
+static CharUnits getArrayElementAlign(CharUnits arrayAlign,
+ llvm::Value *idx,
+ CharUnits eltSize) {
+ // If we have a constant index, we can use the exact offset of the
+ // element we're accessing.
+ if (auto constantIdx = dyn_cast<llvm::ConstantInt>(idx)) {
+ CharUnits offset = constantIdx->getZExtValue() * eltSize;
+ return arrayAlign.alignmentAtOffset(offset);
+
+ // Otherwise, use the worst-case alignment for any element.
+ } else {
+ return arrayAlign.alignmentOfArrayElement(eltSize);
+ }
+}
+
+static QualType getFixedSizeElementType(const ASTContext &ctx,
+ const VariableArrayType *vla) {
+ QualType eltType;
+ do {
+ eltType = vla->getElementType();
+ } while ((vla = ctx.getAsVariableArrayType(eltType)));
+ return eltType;
+}
+
+static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,
+ ArrayRef<llvm::Value*> indices,
+ QualType eltType, bool inbounds,
+ const llvm::Twine &name = "arrayidx") {
+ // All the indices except that last must be zero.
+#ifndef NDEBUG
+ for (auto idx : indices.drop_back())
+ assert(isa<llvm::ConstantInt>(idx) &&
+ cast<llvm::ConstantInt>(idx)->isZero());
+#endif
+
+ // Determine the element size of the statically-sized base. This is
+ // the thing that the indices are expressed in terms of.
+ if (auto vla = CGF.getContext().getAsVariableArrayType(eltType)) {
+ eltType = getFixedSizeElementType(CGF.getContext(), vla);
+ }
+
+ // We can use that to compute the best alignment of the element.
+ CharUnits eltSize = CGF.getContext().getTypeSizeInChars(eltType);
+ CharUnits eltAlign =
+ getArrayElementAlign(addr.getAlignment(), indices.back(), eltSize);
+
+ llvm::Value *eltPtr =
+ emitArraySubscriptGEP(CGF, addr.getPointer(), indices, inbounds, name);
+ return Address(eltPtr, eltAlign);
+}
+
LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
bool Accessed) {
// The index must always be an integer, which is not an aggregate. Emit it.
@@ -2449,32 +2715,34 @@
LValue LHS = EmitLValue(E->getBase());
assert(LHS.isSimple() && "Can only subscript lvalue vectors here!");
return LValue::MakeVectorElt(LHS.getAddress(), Idx,
- E->getBase()->getType(), LHS.getAlignment());
+ E->getBase()->getType(),
+ LHS.getAlignmentSource());
}
+ // All the other cases basically behave like simple offsetting.
+
// Extend or truncate the index type to 32 or 64-bits.
if (Idx->getType() != IntPtrTy)
Idx = Builder.CreateIntCast(Idx, IntPtrTy, IdxSigned, "idxprom");
- // We know that the pointer points to a type of the correct size, unless the
- // size is a VLA or Objective-C interface.
- llvm::Value *Address = nullptr;
- CharUnits ArrayAlignment;
+ // Handle the extvector case we ignored above.
if (isa<ExtVectorElementExpr>(E->getBase())) {
LValue LV = EmitLValue(E->getBase());
- Address = EmitExtVectorElementLValue(LV);
- Address = Builder.CreateInBoundsGEP(Address, Idx, "arrayidx");
- const VectorType *ExprVT = LV.getType()->getAs<VectorType>();
- QualType EQT = ExprVT->getElementType();
- return MakeAddrLValue(Address, EQT,
- getContext().getTypeAlignInChars(EQT));
+ Address Addr = EmitExtVectorElementLValue(LV);
+
+ QualType EltType = LV.getType()->castAs<VectorType>()->getElementType();
+ Addr = emitArraySubscriptGEP(*this, Addr, Idx, EltType, /*inbounds*/ true);
+ return MakeAddrLValue(Addr, EltType, LV.getAlignmentSource());
}
- else if (const VariableArrayType *vla =
+
+ AlignmentSource AlignSource;
+ Address Addr = Address::invalid();
+ if (const VariableArrayType *vla =
getContext().getAsVariableArrayType(E->getType())) {
// The base must be a pointer, which is not an aggregate. Emit
// it. It needs to be emitted first in case it's what captures
// the VLA bounds.
- Address = EmitScalarExpr(E->getBase());
+ Addr = EmitPointerWithAlignment(E->getBase(), &AlignSource);
// The element count here is the total number of non-VLA elements.
llvm::Value *numElements = getVLASize(vla).first;
@@ -2485,24 +2753,40 @@
// multiply. We suppress this if overflow is not undefined behavior.
if (getLangOpts().isSignedOverflowDefined()) {
Idx = Builder.CreateMul(Idx, numElements);
- Address = Builder.CreateGEP(Address, Idx, "arrayidx");
} else {
Idx = Builder.CreateNSWMul(Idx, numElements);
- Address = Builder.CreateInBoundsGEP(Address, Idx, "arrayidx");
}
+
+ Addr = emitArraySubscriptGEP(*this, Addr, Idx, vla->getElementType(),
+ !getLangOpts().isSignedOverflowDefined());
+
} else if (const ObjCObjectType *OIT = E->getType()->getAs<ObjCObjectType>()){
// Indexing over an interface, as in "NSString *P; P[4];"
- llvm::Value *InterfaceSize =
- llvm::ConstantInt::get(Idx->getType(),
- getContext().getTypeSizeInChars(OIT).getQuantity());
+ CharUnits InterfaceSize = getContext().getTypeSizeInChars(OIT);
+ llvm::Value *InterfaceSizeVal =
+ llvm::ConstantInt::get(Idx->getType(), InterfaceSize.getQuantity());;
- Idx = Builder.CreateMul(Idx, InterfaceSize);
+ llvm::Value *ScaledIdx = Builder.CreateMul(Idx, InterfaceSizeVal);
- // The base must be a pointer, which is not an aggregate. Emit it.
- llvm::Value *Base = EmitScalarExpr(E->getBase());
- Address = EmitCastToVoidPtr(Base);
- Address = Builder.CreateGEP(Address, Idx, "arrayidx");
- Address = Builder.CreateBitCast(Address, Base->getType());
+ // Emit the base pointer.
+ Addr = EmitPointerWithAlignment(E->getBase(), &AlignSource);
+
+ // We don't necessarily build correct LLVM struct types for ObjC
+ // interfaces, so we can't rely on GEP to do this scaling
+ // correctly, so we need to cast to i8*. FIXME: is this actually
+ // true? A lot of other things in the fragile ABI would break...
+ llvm::Type *OrigBaseTy = Addr.getType();
+ Addr = Builder.CreateElementBitCast(Addr, Int8Ty);
+
+ // Do the GEP.
+ CharUnits EltAlign =
+ getArrayElementAlign(Addr.getAlignment(), Idx, InterfaceSize);
+ llvm::Value *EltPtr =
+ emitArraySubscriptGEP(*this, Addr.getPointer(), ScaledIdx, false);
+ Addr = Address(EltPtr, EltAlign);
+
+ // Cast back.
+ Addr = Builder.CreateBitCast(Addr, OrigBaseTy);
} else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) {
// If this is A[i] where A is an array, the frontend will have decayed the
// base to be a ArrayToPointerDecay implicit cast. While correct, it is
@@ -2517,42 +2801,23 @@
ArrayLV = EmitArraySubscriptExpr(ASE, /*Accessed*/ true);
else
ArrayLV = EmitLValue(Array);
- llvm::Value *ArrayPtr = ArrayLV.getAddress();
- llvm::Value *Zero = llvm::ConstantInt::get(Int32Ty, 0);
- llvm::Value *Args[] = { Zero, Idx };
// Propagate the alignment from the array itself to the result.
- ArrayAlignment = ArrayLV.getAlignment();
-
- if (getLangOpts().isSignedOverflowDefined())
- Address = Builder.CreateGEP(ArrayPtr, Args, "arrayidx");
- else
- Address = Builder.CreateInBoundsGEP(ArrayPtr, Args, "arrayidx");
+ Addr = emitArraySubscriptGEP(*this, ArrayLV.getAddress(),
+ {CGM.getSize(CharUnits::Zero()), Idx},
+ E->getType(),
+ !getLangOpts().isSignedOverflowDefined());
+ AlignSource = ArrayLV.getAlignmentSource();
} else {
- // The base must be a pointer, which is not an aggregate. Emit it.
- llvm::Value *Base = EmitScalarExpr(E->getBase());
- if (getLangOpts().isSignedOverflowDefined())
- Address = Builder.CreateGEP(Base, Idx, "arrayidx");
- else
- Address = Builder.CreateInBoundsGEP(Base, Idx, "arrayidx");
+ // The base must be a pointer; emit it with an estimate of its alignment.
+ Addr = EmitPointerWithAlignment(E->getBase(), &AlignSource);
+ Addr = emitArraySubscriptGEP(*this, Addr, Idx, E->getType(),
+ !getLangOpts().isSignedOverflowDefined());
}
- QualType T = E->getBase()->getType()->getPointeeType();
- assert(!T.isNull() &&
- "CodeGenFunction::EmitArraySubscriptExpr(): Illegal base type");
+ LValue LV = MakeAddrLValue(Addr, E->getType(), AlignSource);
-
- // Limit the alignment to that of the result type.
- LValue LV;
- if (!ArrayAlignment.isZero()) {
- CharUnits Align = getContext().getTypeAlignInChars(T);
- ArrayAlignment = std::min(Align, ArrayAlignment);
- LV = MakeAddrLValue(Address, T, ArrayAlignment);
- } else {
- LV = MakeNaturalAlignAddrLValue(Address, T);
- }
-
- LV.getQuals().setAddressSpace(E->getBase()->getType().getAddressSpace());
+ // TODO: Preserve/extend path TBAA metadata?
if (getLangOpts().ObjC1 &&
getLangOpts().getGC() != LangOptions::NonGC) {
@@ -2562,14 +2827,150 @@
return LV;
}
-static
-llvm::Constant *GenerateConstantVector(CGBuilderTy &Builder,
- SmallVectorImpl<unsigned> &Elts) {
- SmallVector<llvm::Constant*, 4> CElts;
- for (unsigned i = 0, e = Elts.size(); i != e; ++i)
- CElts.push_back(Builder.getInt32(Elts[i]));
+LValue CodeGenFunction::EmitOMPArraySectionExpr(const OMPArraySectionExpr *E,
+ bool IsLowerBound) {
+ LValue Base;
+ if (auto *ASE =
+ dyn_cast<OMPArraySectionExpr>(E->getBase()->IgnoreParenImpCasts()))
+ Base = EmitOMPArraySectionExpr(ASE, IsLowerBound);
+ else
+ Base = EmitLValue(E->getBase());
+ QualType BaseTy = Base.getType();
+ llvm::Value *Idx = nullptr;
+ QualType ResultExprTy;
+ if (auto *AT = getContext().getAsArrayType(BaseTy))
+ ResultExprTy = AT->getElementType();
+ else
+ ResultExprTy = BaseTy->getPointeeType();
+ if (IsLowerBound || (!IsLowerBound && E->getColonLoc().isInvalid())) {
+ // Requesting lower bound or upper bound, but without provided length and
+ // without ':' symbol for the default length -> length = 1.
+ // Idx = LowerBound ?: 0;
+ if (auto *LowerBound = E->getLowerBound()) {
+ Idx = Builder.CreateIntCast(
+ EmitScalarExpr(LowerBound), IntPtrTy,
+ LowerBound->getType()->hasSignedIntegerRepresentation());
+ } else
+ Idx = llvm::ConstantInt::getNullValue(IntPtrTy);
+ } else {
+ // Try to emit length or lower bound as constant. If this is possible, 1 is
+ // subtracted from constant length or lower bound. Otherwise, emit LLVM IR
+ // (LB + Len) - 1.
+ auto &C = CGM.getContext();
+ auto *Length = E->getLength();
+ llvm::APSInt ConstLength;
+ if (Length) {
+ // Idx = LowerBound + Length - 1;
+ if (Length->isIntegerConstantExpr(ConstLength, C)) {
+ ConstLength = ConstLength.zextOrTrunc(PointerWidthInBits);
+ Length = nullptr;
+ }
+ auto *LowerBound = E->getLowerBound();
+ llvm::APSInt ConstLowerBound(PointerWidthInBits, /*isUnsigned=*/false);
+ if (LowerBound && LowerBound->isIntegerConstantExpr(ConstLowerBound, C)) {
+ ConstLowerBound = ConstLowerBound.zextOrTrunc(PointerWidthInBits);
+ LowerBound = nullptr;
+ }
+ if (!Length)
+ --ConstLength;
+ else if (!LowerBound)
+ --ConstLowerBound;
- return llvm::ConstantVector::get(CElts);
+ if (Length || LowerBound) {
+ auto *LowerBoundVal =
+ LowerBound
+ ? Builder.CreateIntCast(
+ EmitScalarExpr(LowerBound), IntPtrTy,
+ LowerBound->getType()->hasSignedIntegerRepresentation())
+ : llvm::ConstantInt::get(IntPtrTy, ConstLowerBound);
+ auto *LengthVal =
+ Length
+ ? Builder.CreateIntCast(
+ EmitScalarExpr(Length), IntPtrTy,
+ Length->getType()->hasSignedIntegerRepresentation())
+ : llvm::ConstantInt::get(IntPtrTy, ConstLength);
+ Idx = Builder.CreateAdd(LowerBoundVal, LengthVal, "lb_add_len",
+ /*HasNUW=*/false,
+ !getLangOpts().isSignedOverflowDefined());
+ if (Length && LowerBound) {
+ Idx = Builder.CreateSub(
+ Idx, llvm::ConstantInt::get(IntPtrTy, /*V=*/1), "idx_sub_1",
+ /*HasNUW=*/false, !getLangOpts().isSignedOverflowDefined());
+ }
+ } else
+ Idx = llvm::ConstantInt::get(IntPtrTy, ConstLength + ConstLowerBound);
+ } else {
+ // Idx = ArraySize - 1;
+ if (auto *VAT = C.getAsVariableArrayType(BaseTy)) {
+ Length = VAT->getSizeExpr();
+ if (Length->isIntegerConstantExpr(ConstLength, C))
+ Length = nullptr;
+ } else {
+ auto *CAT = C.getAsConstantArrayType(BaseTy);
+ ConstLength = CAT->getSize();
+ }
+ if (Length) {
+ auto *LengthVal = Builder.CreateIntCast(
+ EmitScalarExpr(Length), IntPtrTy,
+ Length->getType()->hasSignedIntegerRepresentation());
+ Idx = Builder.CreateSub(
+ LengthVal, llvm::ConstantInt::get(IntPtrTy, /*V=*/1), "len_sub_1",
+ /*HasNUW=*/false, !getLangOpts().isSignedOverflowDefined());
+ } else {
+ ConstLength = ConstLength.zextOrTrunc(PointerWidthInBits);
+ --ConstLength;
+ Idx = llvm::ConstantInt::get(IntPtrTy, ConstLength);
+ }
+ }
+ }
+ assert(Idx);
+
+ llvm::Value *EltPtr;
+ QualType FixedSizeEltType = ResultExprTy;
+ if (auto *VLA = getContext().getAsVariableArrayType(ResultExprTy)) {
+ // The element count here is the total number of non-VLA elements.
+ llvm::Value *numElements = getVLASize(VLA).first;
+ FixedSizeEltType = getFixedSizeElementType(getContext(), VLA);
+
+ // Effectively, the multiply by the VLA size is part of the GEP.
+ // GEP indexes are signed, and scaling an index isn't permitted to
+ // signed-overflow, so we use the same semantics for our explicit
+ // multiply. We suppress this if overflow is not undefined behavior.
+ if (getLangOpts().isSignedOverflowDefined()) {
+ Idx = Builder.CreateMul(Idx, numElements);
+ EltPtr = Builder.CreateGEP(Base.getPointer(), Idx, "arrayidx");
+ } else {
+ Idx = Builder.CreateNSWMul(Idx, numElements);
+ EltPtr = Builder.CreateInBoundsGEP(Base.getPointer(), Idx, "arrayidx");
+ }
+ } else if (BaseTy->isConstantArrayType()) {
+ llvm::Value *ArrayPtr = Base.getPointer();
+ llvm::Value *Zero = llvm::ConstantInt::getNullValue(IntPtrTy);
+ llvm::Value *Args[] = {Zero, Idx};
+
+ if (getLangOpts().isSignedOverflowDefined())
+ EltPtr = Builder.CreateGEP(ArrayPtr, Args, "arrayidx");
+ else
+ EltPtr = Builder.CreateInBoundsGEP(ArrayPtr, Args, "arrayidx");
+ } else {
+ // The base must be a pointer, which is not an aggregate. Emit it.
+ if (getLangOpts().isSignedOverflowDefined())
+ EltPtr = Builder.CreateGEP(Base.getPointer(), Idx, "arrayidx");
+ else
+ EltPtr = Builder.CreateInBoundsGEP(Base.getPointer(), Idx, "arrayidx");
+ }
+
+ CharUnits EltAlign =
+ Base.getAlignment().alignmentOfArrayElement(
+ getContext().getTypeSizeInChars(FixedSizeEltType));
+
+ // Limit the alignment to that of the result type.
+ LValue LV = MakeAddrLValue(Address(EltPtr, EltAlign), ResultExprTy,
+ Base.getAlignmentSource());
+
+ LV.getQuals().setAddressSpace(BaseTy.getAddressSpace());
+
+ return LV;
}
LValue CodeGenFunction::
@@ -2581,9 +2982,10 @@
if (E->isArrow()) {
// If it is a pointer to a vector, emit the address and form an lvalue with
// it.
- llvm::Value *Ptr = EmitScalarExpr(E->getBase());
+ AlignmentSource AlignSource;
+ Address Ptr = EmitPointerWithAlignment(E->getBase(), &AlignSource);
const PointerType *PT = E->getBase()->getType()->getAs<PointerType>();
- Base = MakeAddrLValue(Ptr, PT->getPointeeType());
+ Base = MakeAddrLValue(Ptr, PT->getPointeeType(), AlignSource);
Base.getQuals().removeObjCGCAttr();
} else if (E->getBase()->isGLValue()) {
// Otherwise, if the base is an lvalue ( as in the case of foo.x.x),
@@ -2597,22 +2999,24 @@
llvm::Value *Vec = EmitScalarExpr(E->getBase());
// Store the vector to memory (because LValue wants an address).
- llvm::Value *VecMem = CreateMemTemp(E->getBase()->getType());
+ Address VecMem = CreateMemTemp(E->getBase()->getType());
Builder.CreateStore(Vec, VecMem);
- Base = MakeAddrLValue(VecMem, E->getBase()->getType());
+ Base = MakeAddrLValue(VecMem, E->getBase()->getType(),
+ AlignmentSource::Decl);
}
QualType type =
E->getType().withCVRQualifiers(Base.getQuals().getCVRQualifiers());
// Encode the element access list into a vector of unsigned indices.
- SmallVector<unsigned, 4> Indices;
+ SmallVector<uint32_t, 4> Indices;
E->getEncodedElementAccess(Indices);
if (Base.isSimple()) {
- llvm::Constant *CV = GenerateConstantVector(Builder, Indices);
+ llvm::Constant *CV =
+ llvm::ConstantDataVector::get(getLLVMContext(), Indices);
return LValue::MakeExtVectorElt(Base.getAddress(), CV, type,
- Base.getAlignment());
+ Base.getAlignmentSource());
}
assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
@@ -2622,8 +3026,8 @@
for (unsigned i = 0, e = Indices.size(); i != e; ++i)
CElts.push_back(BaseElts->getAggregateElement(Indices[i]));
llvm::Constant *CV = llvm::ConstantVector::get(CElts);
- return LValue::MakeExtVectorElt(Base.getExtVectorAddr(), CV, type,
- Base.getAlignment());
+ return LValue::MakeExtVectorElt(Base.getExtVectorAddress(), CV, type,
+ Base.getAlignmentSource());
}
LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {
@@ -2632,10 +3036,11 @@
// If this is s.x, emit s as an lvalue. If it is s->x, emit s as a scalar.
LValue BaseLV;
if (E->isArrow()) {
- llvm::Value *Ptr = EmitScalarExpr(BaseExpr);
+ AlignmentSource AlignSource;
+ Address Addr = EmitPointerWithAlignment(BaseExpr, &AlignSource);
QualType PtrTy = BaseExpr->getType()->getPointeeType();
- EmitTypeCheck(TCK_MemberAccess, E->getExprLoc(), Ptr, PtrTy);
- BaseLV = MakeNaturalAlignAddrLValue(Ptr, PtrTy);
+ EmitTypeCheck(TCK_MemberAccess, E->getExprLoc(), Addr.getPointer(), PtrTy);
+ BaseLV = MakeAddrLValue(Addr, PtrTy, AlignSource);
} else
BaseLV = EmitCheckedLValue(BaseExpr, TCK_MemberAccess);
@@ -2666,41 +3071,65 @@
return EmitLValueForField(LambdaLV, Field);
}
+/// Drill down to the storage of a field without walking into
+/// reference types.
+///
+/// The resulting address doesn't necessarily have the right type.
+static Address emitAddrOfFieldStorage(CodeGenFunction &CGF, Address base,
+ const FieldDecl *field) {
+ const RecordDecl *rec = field->getParent();
+
+ unsigned idx =
+ CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field);
+
+ CharUnits offset;
+ // Adjust the alignment down to the given offset.
+ // As a special case, if the LLVM field index is 0, we know that this
+ // is zero.
+ assert((idx != 0 || CGF.getContext().getASTRecordLayout(rec)
+ .getFieldOffset(field->getFieldIndex()) == 0) &&
+ "LLVM field at index zero had non-zero offset?");
+ if (idx != 0) {
+ auto &recLayout = CGF.getContext().getASTRecordLayout(rec);
+ auto offsetInBits = recLayout.getFieldOffset(field->getFieldIndex());
+ offset = CGF.getContext().toCharUnitsFromBits(offsetInBits);
+ }
+
+ return CGF.Builder.CreateStructGEP(base, idx, offset, field->getName());
+}
+
LValue CodeGenFunction::EmitLValueForField(LValue base,
const FieldDecl *field) {
+ AlignmentSource fieldAlignSource =
+ getFieldAlignmentSource(base.getAlignmentSource());
+
if (field->isBitField()) {
const CGRecordLayout &RL =
CGM.getTypes().getCGRecordLayout(field->getParent());
const CGBitFieldInfo &Info = RL.getBitFieldInfo(field);
- llvm::Value *Addr = base.getAddress();
+ Address Addr = base.getAddress();
unsigned Idx = RL.getLLVMFieldNo(field);
if (Idx != 0)
// For structs, we GEP to the field that the record layout suggests.
- Addr = Builder.CreateStructGEP(nullptr, Addr, Idx, field->getName());
+ Addr = Builder.CreateStructGEP(Addr, Idx, Info.StorageOffset,
+ field->getName());
// Get the access type.
- llvm::Type *PtrTy = llvm::Type::getIntNPtrTy(
- getLLVMContext(), Info.StorageSize,
- CGM.getContext().getTargetAddressSpace(base.getType()));
- if (Addr->getType() != PtrTy)
- Addr = Builder.CreateBitCast(Addr, PtrTy);
+ llvm::Type *FieldIntTy =
+ llvm::Type::getIntNTy(getLLVMContext(), Info.StorageSize);
+ if (Addr.getElementType() != FieldIntTy)
+ Addr = Builder.CreateElementBitCast(Addr, FieldIntTy);
QualType fieldType =
field->getType().withCVRQualifiers(base.getVRQualifiers());
- return LValue::MakeBitfield(Addr, Info, fieldType, base.getAlignment());
+ return LValue::MakeBitfield(Addr, Info, fieldType, fieldAlignSource);
}
const RecordDecl *rec = field->getParent();
QualType type = field->getType();
- CharUnits alignment = getContext().getDeclAlign(field);
-
- // FIXME: It should be impossible to have an LValue without alignment for a
- // complete type.
- if (!base.getAlignment().isZero())
- alignment = std::min(alignment, base.getAlignment());
bool mayAlias = rec->hasAttr<MayAliasAttr>();
- llvm::Value *addr = base.getAddress();
+ Address addr = base.getAddress();
unsigned cvr = base.getVRQualifiers();
bool TBAAPath = CGM.getCodeGenOpts().StructPathTBAA;
if (rec->isUnion()) {
@@ -2710,14 +3139,12 @@
TBAAPath = false;
} else {
// For structs, we GEP to the field that the record layout suggests.
- unsigned idx = CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field);
- addr = Builder.CreateStructGEP(nullptr, addr, idx, field->getName());
+ addr = emitAddrOfFieldStorage(*this, addr, field);
// If this is a reference field, load the reference right now.
if (const ReferenceType *refType = type->getAs<ReferenceType>()) {
llvm::LoadInst *load = Builder.CreateLoad(addr, "ref");
if (cvr & Qualifiers::Volatile) load->setVolatile(true);
- load->setAlignment(alignment.getQuantity());
// Loading the reference will disable path-aware TBAA.
TBAAPath = false;
@@ -2728,17 +3155,20 @@
else
tbaa = CGM.getTBAAInfo(type);
if (tbaa)
- CGM.DecorateInstruction(load, tbaa);
+ CGM.DecorateInstructionWithTBAA(load, tbaa);
}
- addr = load;
mayAlias = false;
type = refType->getPointeeType();
- if (type->isIncompleteType())
- alignment = CharUnits();
- else
- alignment = getContext().getTypeAlignInChars(type);
- cvr = 0; // qualifiers don't recursively apply to referencee
+
+ CharUnits alignment =
+ getNaturalTypeAlignment(type, &fieldAlignSource, /*pointee*/ true);
+ addr = Address(load, alignment);
+
+ // Qualifiers on the struct don't apply to the referencee, and
+ // we'll pick up CVR from the actual type later, so reset these
+ // additional qualifiers now.
+ cvr = 0;
}
}
@@ -2746,14 +3176,14 @@
// for both unions and structs. A union needs a bitcast, a struct element
// will need a bitcast if the LLVM type laid out doesn't match the desired
// type.
- addr = EmitBitCastOfLValueToProperType(*this, addr,
- CGM.getTypes().ConvertTypeForMem(type),
- field->getName());
+ addr = Builder.CreateElementBitCast(addr,
+ CGM.getTypes().ConvertTypeForMem(type),
+ field->getName());
if (field->hasAttr<AnnotateAttr>())
addr = EmitFieldAnnotations(field, addr);
- LValue LV = MakeAddrLValue(addr, type, alignment);
+ LValue LV = MakeAddrLValue(addr, type, fieldAlignSource);
LV.getQuals().addCVRQualifiers(cvr);
if (TBAAPath) {
const ASTRecordLayout &Layout =
@@ -2787,41 +3217,29 @@
if (!FieldType->isReferenceType())
return EmitLValueForField(Base, Field);
- const CGRecordLayout &RL =
- CGM.getTypes().getCGRecordLayout(Field->getParent());
- unsigned idx = RL.getLLVMFieldNo(Field);
- llvm::Value *V = Builder.CreateStructGEP(nullptr, Base.getAddress(), idx);
- assert(!FieldType.getObjCGCAttr() && "fields cannot have GC attrs");
+ Address V = emitAddrOfFieldStorage(*this, Base.getAddress(), Field);
- // Make sure that the address is pointing to the right type. This is critical
- // for both unions and structs. A union needs a bitcast, a struct element
- // will need a bitcast if the LLVM type laid out doesn't match the desired
- // type.
+ // Make sure that the address is pointing to the right type.
llvm::Type *llvmType = ConvertTypeForMem(FieldType);
- V = EmitBitCastOfLValueToProperType(*this, V, llvmType, Field->getName());
+ V = Builder.CreateElementBitCast(V, llvmType, Field->getName());
- CharUnits Alignment = getContext().getDeclAlign(Field);
-
- // FIXME: It should be impossible to have an LValue without alignment for a
- // complete type.
- if (!Base.getAlignment().isZero())
- Alignment = std::min(Alignment, Base.getAlignment());
-
- return MakeAddrLValue(V, FieldType, Alignment);
+ // TODO: access-path TBAA?
+ auto FieldAlignSource = getFieldAlignmentSource(Base.getAlignmentSource());
+ return MakeAddrLValue(V, FieldType, FieldAlignSource);
}
LValue CodeGenFunction::EmitCompoundLiteralLValue(const CompoundLiteralExpr *E){
if (E->isFileScope()) {
- llvm::Value *GlobalPtr = CGM.GetAddrOfConstantCompoundLiteral(E);
- return MakeAddrLValue(GlobalPtr, E->getType());
+ ConstantAddress GlobalPtr = CGM.GetAddrOfConstantCompoundLiteral(E);
+ return MakeAddrLValue(GlobalPtr, E->getType(), AlignmentSource::Decl);
}
if (E->getType()->isVariablyModifiedType())
// make sure to emit the VLA size.
EmitVariablyModifiedType(E->getType());
- llvm::Value *DeclPtr = CreateMemTemp(E->getType(), ".compoundliteral");
+ Address DeclPtr = CreateMemTemp(E->getType(), ".compoundliteral");
const Expr *InitExpr = E->getInitializer();
- LValue Result = MakeAddrLValue(DeclPtr, E->getType());
+ LValue Result = MakeAddrLValue(DeclPtr, E->getType(), AlignmentSource::Decl);
EmitAnyExprToMem(InitExpr, DeclPtr, E->getType().getQualifiers(),
/*Init*/ true);
@@ -2912,11 +3330,14 @@
EmitBlock(contBlock);
if (lhs && rhs) {
- llvm::PHINode *phi = Builder.CreatePHI(lhs->getAddress()->getType(),
+ llvm::PHINode *phi = Builder.CreatePHI(lhs->getPointer()->getType(),
2, "cond-lvalue");
- phi->addIncoming(lhs->getAddress(), lhsBlock);
- phi->addIncoming(rhs->getAddress(), rhsBlock);
- return MakeAddrLValue(phi, expr->getType());
+ phi->addIncoming(lhs->getPointer(), lhsBlock);
+ phi->addIncoming(rhs->getPointer(), rhsBlock);
+ Address result(phi, std::min(lhs->getAlignment(), rhs->getAlignment()));
+ AlignmentSource alignSource =
+ std::max(lhs->getAlignmentSource(), rhs->getAlignmentSource());
+ return MakeAddrLValue(result, expr->getType(), alignSource);
} else {
assert((lhs || rhs) &&
"both operands of glvalue conditional are throw-expressions?");
@@ -2985,9 +3406,9 @@
case CK_Dynamic: {
LValue LV = EmitLValue(E->getSubExpr());
- llvm::Value *V = LV.getAddress();
+ Address V = LV.getAddress();
const auto *DCE = cast<CXXDynamicCastExpr>(E);
- return MakeAddrLValue(EmitDynamicCast(V, DCE), E->getType());
+ return MakeNaturalAlignAddrLValue(EmitDynamicCast(V, DCE), E->getType());
}
case CK_ConstructorConversion:
@@ -3005,14 +3426,14 @@
auto *DerivedClassDecl = cast<CXXRecordDecl>(DerivedClassTy->getDecl());
LValue LV = EmitLValue(E->getSubExpr());
- llvm::Value *This = LV.getAddress();
+ Address This = LV.getAddress();
// Perform the derived-to-base conversion
- llvm::Value *Base = GetAddressOfBaseClass(
+ Address Base = GetAddressOfBaseClass(
This, DerivedClassDecl, E->path_begin(), E->path_end(),
/*NullCheckValue=*/false, E->getExprLoc());
- return MakeAddrLValue(Base, E->getType());
+ return MakeAddrLValue(Base, E->getType(), LV.getAlignmentSource());
}
case CK_ToUnion:
return EmitAggExprToLValue(E);
@@ -3023,7 +3444,7 @@
LValue LV = EmitLValue(E->getSubExpr());
// Perform the base-to-derived conversion
- llvm::Value *Derived =
+ Address Derived =
GetAddressOfDerivedClass(LV.getAddress(), DerivedClassDecl,
E->path_begin(), E->path_end(),
/*NullCheckValue=*/false);
@@ -3032,32 +3453,36 @@
// performed and the object is not of the derived type.
if (sanitizePerformTypeCheck())
EmitTypeCheck(TCK_DowncastReference, E->getExprLoc(),
- Derived, E->getType());
+ Derived.getPointer(), E->getType());
if (SanOpts.has(SanitizerKind::CFIDerivedCast))
- EmitVTablePtrCheckForCast(E->getType(), Derived, /*MayBeNull=*/false);
+ EmitVTablePtrCheckForCast(E->getType(), Derived.getPointer(),
+ /*MayBeNull=*/false,
+ CFITCK_DerivedCast, E->getLocStart());
- return MakeAddrLValue(Derived, E->getType());
+ return MakeAddrLValue(Derived, E->getType(), LV.getAlignmentSource());
}
case CK_LValueBitCast: {
// This must be a reinterpret_cast (or c-style equivalent).
const auto *CE = cast<ExplicitCastExpr>(E);
+ CGM.EmitExplicitCastExprType(CE, this);
LValue LV = EmitLValue(E->getSubExpr());
- llvm::Value *V = Builder.CreateBitCast(LV.getAddress(),
- ConvertType(CE->getTypeAsWritten()));
+ Address V = Builder.CreateBitCast(LV.getAddress(),
+ ConvertType(CE->getTypeAsWritten()));
if (SanOpts.has(SanitizerKind::CFIUnrelatedCast))
- EmitVTablePtrCheckForCast(E->getType(), V, /*MayBeNull=*/false);
+ EmitVTablePtrCheckForCast(E->getType(), V.getPointer(),
+ /*MayBeNull=*/false,
+ CFITCK_UnrelatedCast, E->getLocStart());
- return MakeAddrLValue(V, E->getType());
+ return MakeAddrLValue(V, E->getType(), LV.getAlignmentSource());
}
case CK_ObjCObjectLValueCast: {
LValue LV = EmitLValue(E->getSubExpr());
- QualType ToType = getContext().getLValueReferenceType(E->getType());
- llvm::Value *V = Builder.CreateBitCast(LV.getAddress(),
- ConvertType(ToType));
- return MakeAddrLValue(V, E->getType());
+ Address V = Builder.CreateElementBitCast(LV.getAddress(),
+ ConvertType(E->getType()));
+ return MakeAddrLValue(V, E->getType(), LV.getAlignmentSource());
}
case CK_ZeroToOCLEvent:
llvm_unreachable("NULL to OpenCL event lvalue cast is not valid");
@@ -3116,20 +3541,17 @@
if (const auto *PseudoDtor =
dyn_cast<CXXPseudoDestructorExpr>(E->getCallee()->IgnoreParens())) {
QualType DestroyedType = PseudoDtor->getDestroyedType();
- if (getLangOpts().ObjCAutoRefCount &&
- DestroyedType->isObjCLifetimeType() &&
- (DestroyedType.getObjCLifetime() == Qualifiers::OCL_Strong ||
- DestroyedType.getObjCLifetime() == Qualifiers::OCL_Weak)) {
+ if (DestroyedType.hasStrongOrWeakObjCLifetime()) {
// Automatic Reference Counting:
// If the pseudo-expression names a retainable object with weak or
// strong lifetime, the object shall be released.
Expr *BaseExpr = PseudoDtor->getBase();
- llvm::Value *BaseValue = nullptr;
+ Address BaseValue = Address::invalid();
Qualifiers BaseQuals;
// If this is s.x, emit s as an lvalue. If it is s->x, emit s as a scalar.
if (PseudoDtor->isArrow()) {
- BaseValue = EmitScalarExpr(BaseExpr);
+ BaseValue = EmitPointerWithAlignment(BaseExpr);
const PointerType *PTy = BaseExpr->getType()->getAs<PointerType>();
BaseQuals = PTy->getPointeeType().getQualifiers();
} else {
@@ -3139,7 +3561,7 @@
BaseQuals = BaseTy.getQualifiers();
}
- switch (PseudoDtor->getDestroyedType().getObjCLifetime()) {
+ switch (DestroyedType.getObjCLifetime()) {
case Qualifiers::OCL_None:
case Qualifiers::OCL_ExplicitNone:
case Qualifiers::OCL_Autoreleasing:
@@ -3224,13 +3646,14 @@
RValue RV = EmitCallExpr(E);
if (!RV.isScalar())
- return MakeAddrLValue(RV.getAggregateAddr(), E->getType());
+ return MakeAddrLValue(RV.getAggregateAddress(), E->getType(),
+ AlignmentSource::Decl);
assert(E->getCallReturnType(getContext())->isReferenceType() &&
"Can't have a scalar return unless the return type is a "
"reference type!");
- return MakeAddrLValue(RV.getScalarVal(), E->getType());
+ return MakeNaturalAlignPointeeAddrLValue(RV.getScalarVal(), E->getType());
}
LValue CodeGenFunction::EmitVAArgExprLValue(const VAArgExpr *E) {
@@ -3243,21 +3666,23 @@
&& "binding l-value to type which needs a temporary");
AggValueSlot Slot = CreateAggTemp(E->getType());
EmitCXXConstructExpr(E, Slot);
- return MakeAddrLValue(Slot.getAddr(), E->getType());
+ return MakeAddrLValue(Slot.getAddress(), E->getType(),
+ AlignmentSource::Decl);
}
LValue
CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) {
- return MakeAddrLValue(EmitCXXTypeidExpr(E), E->getType());
+ return MakeNaturalAlignAddrLValue(EmitCXXTypeidExpr(E), E->getType());
}
-llvm::Value *CodeGenFunction::EmitCXXUuidofExpr(const CXXUuidofExpr *E) {
- return Builder.CreateBitCast(CGM.GetAddrOfUuidDescriptor(E),
- ConvertType(E->getType())->getPointerTo());
+Address CodeGenFunction::EmitCXXUuidofExpr(const CXXUuidofExpr *E) {
+ return Builder.CreateElementBitCast(CGM.GetAddrOfUuidDescriptor(E),
+ ConvertType(E->getType()));
}
LValue CodeGenFunction::EmitCXXUuidofLValue(const CXXUuidofExpr *E) {
- return MakeAddrLValue(EmitCXXUuidofExpr(E), E->getType());
+ return MakeAddrLValue(EmitCXXUuidofExpr(E), E->getType(),
+ AlignmentSource::Decl);
}
LValue
@@ -3265,34 +3690,37 @@
AggValueSlot Slot = CreateAggTemp(E->getType(), "temp.lvalue");
Slot.setExternallyDestructed();
EmitAggExpr(E->getSubExpr(), Slot);
- EmitCXXTemporary(E->getTemporary(), E->getType(), Slot.getAddr());
- return MakeAddrLValue(Slot.getAddr(), E->getType());
+ EmitCXXTemporary(E->getTemporary(), E->getType(), Slot.getAddress());
+ return MakeAddrLValue(Slot.getAddress(), E->getType(),
+ AlignmentSource::Decl);
}
LValue
CodeGenFunction::EmitLambdaLValue(const LambdaExpr *E) {
AggValueSlot Slot = CreateAggTemp(E->getType(), "temp.lvalue");
EmitLambdaExpr(E, Slot);
- return MakeAddrLValue(Slot.getAddr(), E->getType());
+ return MakeAddrLValue(Slot.getAddress(), E->getType(),
+ AlignmentSource::Decl);
}
LValue CodeGenFunction::EmitObjCMessageExprLValue(const ObjCMessageExpr *E) {
RValue RV = EmitObjCMessageExpr(E);
if (!RV.isScalar())
- return MakeAddrLValue(RV.getAggregateAddr(), E->getType());
+ return MakeAddrLValue(RV.getAggregateAddress(), E->getType(),
+ AlignmentSource::Decl);
assert(E->getMethodDecl()->getReturnType()->isReferenceType() &&
"Can't have a scalar return unless the return type is a "
"reference type!");
- return MakeAddrLValue(RV.getScalarVal(), E->getType());
+ return MakeNaturalAlignPointeeAddrLValue(RV.getScalarVal(), E->getType());
}
LValue CodeGenFunction::EmitObjCSelectorLValue(const ObjCSelectorExpr *E) {
- llvm::Value *V =
- CGM.getObjCRuntime().GetSelector(*this, E->getSelector(), true);
- return MakeAddrLValue(V, E->getType());
+ Address V =
+ CGM.getObjCRuntime().GetAddrOfSelector(*this, E->getSelector());
+ return MakeAddrLValue(V, E->getType(), AlignmentSource::Decl);
}
llvm::Value *CodeGenFunction::EmitIvarOffset(const ObjCInterfaceDecl *Interface,
@@ -3320,8 +3748,7 @@
BaseQuals = ObjectTy.getQualifiers();
} else {
LValue BaseLV = EmitLValue(BaseExpr);
- // FIXME: this isn't right for bitfields.
- BaseValue = BaseLV.getAddress();
+ BaseValue = BaseLV.getPointer();
ObjectTy = BaseExpr->getType();
BaseQuals = ObjectTy.getQualifiers();
}
@@ -3336,17 +3763,38 @@
LValue CodeGenFunction::EmitStmtExprLValue(const StmtExpr *E) {
// Can only get l-value for message expression returning aggregate type
RValue RV = EmitAnyExprToTemp(E);
- return MakeAddrLValue(RV.getAggregateAddr(), E->getType());
+ return MakeAddrLValue(RV.getAggregateAddress(), E->getType(),
+ AlignmentSource::Decl);
}
RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee,
const CallExpr *E, ReturnValueSlot ReturnValue,
- const Decl *TargetDecl, llvm::Value *Chain) {
+ CGCalleeInfo CalleeInfo, llvm::Value *Chain) {
// Get the actual function type. The callee type will always be a pointer to
// function type or a block pointer type.
assert(CalleeType->isFunctionPointerType() &&
"Call must have function pointer type!");
+ // Preserve the non-canonical function type because things like exception
+ // specifications disappear in the canonical type. That information is useful
+ // to drive the generation of more accurate code for this call later on.
+ const FunctionProtoType *NonCanonicalFTP = CalleeType->getAs<PointerType>()
+ ->getPointeeType()
+ ->getAs<FunctionProtoType>();
+
+ const Decl *TargetDecl = CalleeInfo.getCalleeDecl();
+
+ if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl))
+ // We can only guarantee that a function is called from the correct
+ // context/function based on the appropriate target attributes,
+ // so only check in the case where we have both always_inline and target
+ // since otherwise we could be making a conditional call after a check for
+ // the proper cpu features (and it won't cause code generation issues due to
+ // function based code generation).
+ if (TargetDecl->hasAttr<AlwaysInlineAttr>() &&
+ TargetDecl->hasAttr<TargetAttr>())
+ checkTargetFeatures(E, FD);
+
CalleeType = getContext().getCanonicalType(CalleeType);
const auto *FnType =
@@ -3370,7 +3818,8 @@
Callee, llvm::PointerType::getUnqual(PrefixStructTy));
llvm::Value *CalleeSigPtr =
Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, 0, 0);
- llvm::Value *CalleeSig = Builder.CreateLoad(CalleeSigPtr);
+ llvm::Value *CalleeSig =
+ Builder.CreateAlignedLoad(CalleeSigPtr, getIntAlign());
llvm::Value *CalleeSigMatch = Builder.CreateICmpEQ(CalleeSig, PrefixSig);
llvm::BasicBlock *Cont = createBasicBlock("cont");
@@ -3380,7 +3829,8 @@
EmitBlock(TypeCheck);
llvm::Value *CalleeRTTIPtr =
Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, 0, 1);
- llvm::Value *CalleeRTTI = Builder.CreateLoad(CalleeRTTIPtr);
+ llvm::Value *CalleeRTTI =
+ Builder.CreateAlignedLoad(CalleeRTTIPtr, getPointerAlign());
llvm::Value *CalleeRTTIMatch =
Builder.CreateICmpEQ(CalleeRTTI, FTRTTIConst);
llvm::Constant *StaticData[] = {
@@ -3395,12 +3845,39 @@
}
}
+ // If we are checking indirect calls and this call is indirect, check that the
+ // function pointer is a member of the bit set for the function type.
+ if (SanOpts.has(SanitizerKind::CFIICall) &&
+ (!TargetDecl || !isa<FunctionDecl>(TargetDecl))) {
+ SanitizerScope SanScope(this);
+
+ llvm::Metadata *MD = CGM.CreateMetadataIdentifierForType(QualType(FnType, 0));
+ llvm::Value *BitSetName = llvm::MetadataAsValue::get(getLLVMContext(), MD);
+
+ llvm::Value *CastedCallee = Builder.CreateBitCast(Callee, Int8PtrTy);
+ llvm::Value *BitSetTest =
+ Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::bitset_test),
+ {CastedCallee, BitSetName});
+
+ auto TypeId = CGM.CreateCfiIdForTypeMetadata(MD);
+ if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && TypeId) {
+ EmitCfiSlowPathCheck(BitSetTest, TypeId, CastedCallee);
+ } else {
+ llvm::Constant *StaticData[] = {
+ EmitCheckSourceLocation(E->getLocStart()),
+ EmitCheckTypeDescriptor(QualType(FnType, 0)),
+ };
+ EmitCheck(std::make_pair(BitSetTest, SanitizerKind::CFIICall),
+ "cfi_bad_icall", StaticData, CastedCallee);
+ }
+ }
+
CallArgList Args;
if (Chain)
Args.add(RValue::get(Builder.CreateBitCast(Chain, CGM.VoidPtrTy)),
CGM.getContext().VoidPtrTy);
- EmitCallArgs(Args, dyn_cast<FunctionProtoType>(FnType), E->arg_begin(),
- E->arg_end(), E->getDirectCallee(), /*ParamsToSkip*/ 0);
+ EmitCallArgs(Args, dyn_cast<FunctionProtoType>(FnType), E->arguments(),
+ E->getDirectCallee(), /*ParamsToSkip*/ 0);
const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall(
Args, FnType, /*isChainCall=*/Chain);
@@ -3431,34 +3908,38 @@
Callee = Builder.CreateBitCast(Callee, CalleeTy, "callee.knr.cast");
}
- return EmitCall(FnInfo, Callee, ReturnValue, Args, TargetDecl);
+ return EmitCall(FnInfo, Callee, ReturnValue, Args,
+ CGCalleeInfo(NonCanonicalFTP, TargetDecl));
}
LValue CodeGenFunction::
EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E) {
- llvm::Value *BaseV;
- if (E->getOpcode() == BO_PtrMemI)
- BaseV = EmitScalarExpr(E->getLHS());
- else
- BaseV = EmitLValue(E->getLHS()).getAddress();
+ Address BaseAddr = Address::invalid();
+ if (E->getOpcode() == BO_PtrMemI) {
+ BaseAddr = EmitPointerWithAlignment(E->getLHS());
+ } else {
+ BaseAddr = EmitLValue(E->getLHS()).getAddress();
+ }
llvm::Value *OffsetV = EmitScalarExpr(E->getRHS());
const MemberPointerType *MPT
= E->getRHS()->getType()->getAs<MemberPointerType>();
- llvm::Value *AddV = CGM.getCXXABI().EmitMemberDataPointerAddress(
- *this, E, BaseV, OffsetV, MPT);
+ AlignmentSource AlignSource;
+ Address MemberAddr =
+ EmitCXXMemberDataPointerAddress(E, BaseAddr, OffsetV, MPT,
+ &AlignSource);
- return MakeAddrLValue(AddV, MPT->getPointeeType());
+ return MakeAddrLValue(MemberAddr, MPT->getPointeeType(), AlignSource);
}
/// Given the address of a temporary variable, produce an r-value of
/// its type.
-RValue CodeGenFunction::convertTempToRValue(llvm::Value *addr,
+RValue CodeGenFunction::convertTempToRValue(Address addr,
QualType type,
SourceLocation loc) {
- LValue lvalue = MakeNaturalAlignAddrLValue(addr, type);
+ LValue lvalue = MakeAddrLValue(addr, type, AlignmentSource::Decl);
switch (getEvaluationKind(type)) {
case TEK_Complex:
return RValue::getComplex(EmitLoadOfComplex(lvalue, loc));
@@ -3514,7 +3995,8 @@
CodeGenFunction::hasAggregateEvaluationKind(ov->getType())) {
CGF.EmitAggExpr(ov->getSourceExpr(), slot);
- LValue LV = CGF.MakeAddrLValue(slot.getAddr(), ov->getType());
+ LValue LV = CGF.MakeAddrLValue(slot.getAddress(), ov->getType(),
+ AlignmentSource::Decl);
opaqueData = OVMA::bind(CGF, ov, LV);
result.RV = slot.asRValue();
diff --git a/lib/CodeGen/CGExprAgg.cpp b/lib/CodeGen/CGExprAgg.cpp
index 883b76b..20838db 100644
--- a/lib/CodeGen/CGExprAgg.cpp
+++ b/lib/CodeGen/CGExprAgg.cpp
@@ -49,7 +49,8 @@
if (!shouldUseDestForReturnSlot())
return ReturnValueSlot();
- return ReturnValueSlot(Dest.getAddr(), Dest.isVolatile(), IsResultUnused);
+ return ReturnValueSlot(Dest.getAddress(), Dest.isVolatile(),
+ IsResultUnused);
}
AggValueSlot EnsureSlot(QualType T) {
@@ -77,14 +78,13 @@
/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
void EmitFinalDestCopy(QualType type, const LValue &src);
- void EmitFinalDestCopy(QualType type, RValue src,
- CharUnits srcAlignment = CharUnits::Zero());
+ void EmitFinalDestCopy(QualType type, RValue src);
void EmitCopy(QualType type, const AggValueSlot &dest,
const AggValueSlot &src);
void EmitMoveFromReturnSlot(const Expr *E, RValue Src);
- void EmitArrayInit(llvm::Value *DestPtr, llvm::ArrayType *AType,
+ void EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
QualType elementType, InitListExpr *E);
AggValueSlot::NeedsGCBarriers_t needsGC(QualType T) {
@@ -199,7 +199,8 @@
// case Expr::ChooseExprClass:
void VisitCXXThrowExpr(const CXXThrowExpr *E) { CGF.EmitCXXThrowExpr(E); }
void VisitAtomicExpr(AtomicExpr *E) {
- CGF.EmitAtomicExpr(E, EnsureSlot(E->getType()).getAddr());
+ RValue Res = CGF.EmitAtomicExpr(E);
+ EmitFinalDestCopy(E->getType(), Res);
}
};
} // end anonymous namespace.
@@ -259,17 +260,14 @@
}
// Otherwise, copy from there to the destination.
- assert(Dest.getAddr() != src.getAggregateAddr());
- std::pair<CharUnits, CharUnits> typeInfo =
- CGF.getContext().getTypeInfoInChars(E->getType());
- EmitFinalDestCopy(E->getType(), src, typeInfo.second);
+ assert(Dest.getPointer() != src.getAggregatePointer());
+ EmitFinalDestCopy(E->getType(), src);
}
/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
-void AggExprEmitter::EmitFinalDestCopy(QualType type, RValue src,
- CharUnits srcAlign) {
+void AggExprEmitter::EmitFinalDestCopy(QualType type, RValue src) {
assert(src.isAggregate() && "value must be aggregate value!");
- LValue srcLV = CGF.MakeAddrLValue(src.getAggregateAddr(), type, srcAlign);
+ LValue srcLV = CGF.MakeAddrLValue(src.getAggregateAddress(), type);
EmitFinalDestCopy(type, srcLV);
}
@@ -298,8 +296,8 @@
CharUnits sz = CGF.getContext().getTypeSizeInChars(type);
llvm::Value *size = llvm::ConstantInt::get(CGF.SizeTy, sz.getQuantity());
CGF.CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF,
- dest.getAddr(),
- src.getAddr(),
+ dest.getAddress(),
+ src.getAddress(),
size);
return;
}
@@ -307,9 +305,8 @@
// If the result of the assignment is used, copy the LHS there also.
// It's volatile if either side is. Use the minimum alignment of
// the two sides.
- CGF.EmitAggregateCopy(dest.getAddr(), src.getAddr(), type,
- dest.isVolatile() || src.isVolatile(),
- std::min(dest.getAlignment(), src.getAlignment()));
+ CGF.EmitAggregateCopy(dest.getAddress(), src.getAddress(), type,
+ dest.isVolatile() || src.isVolatile());
}
/// \brief Emit the initializer for a std::initializer_list initialized with a
@@ -321,7 +318,7 @@
ASTContext &Ctx = CGF.getContext();
LValue Array = CGF.EmitLValue(E->getSubExpr());
assert(Array.isSimple() && "initializer_list array not a simple lvalue");
- llvm::Value *ArrayPtr = Array.getAddress();
+ Address ArrayPtr = Array.getAddress();
const ConstantArrayType *ArrayType =
Ctx.getAsConstantArrayType(E->getSubExpr()->getType());
@@ -344,13 +341,12 @@
}
AggValueSlot Dest = EnsureSlot(E->getType());
- LValue DestLV = CGF.MakeAddrLValue(Dest.getAddr(), E->getType(),
- Dest.getAlignment());
+ LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType());
LValue Start = CGF.EmitLValueForFieldInitialization(DestLV, *Field);
llvm::Value *Zero = llvm::ConstantInt::get(CGF.PtrDiffTy, 0);
llvm::Value *IdxStart[] = { Zero, Zero };
llvm::Value *ArrayStart =
- Builder.CreateInBoundsGEP(ArrayPtr, IdxStart, "arraystart");
+ Builder.CreateInBoundsGEP(ArrayPtr.getPointer(), IdxStart, "arraystart");
CGF.EmitStoreThroughLValue(RValue::get(ArrayStart), Start);
++Field;
@@ -367,7 +363,7 @@
// End pointer.
llvm::Value *IdxEnd[] = { Zero, Size };
llvm::Value *ArrayEnd =
- Builder.CreateInBoundsGEP(ArrayPtr, IdxEnd, "arrayend");
+ Builder.CreateInBoundsGEP(ArrayPtr.getPointer(), IdxEnd, "arrayend");
CGF.EmitStoreThroughLValue(RValue::get(ArrayEnd), EndOrLength);
} else if (Ctx.hasSameType(Field->getType(), Ctx.getSizeType())) {
// Length.
@@ -402,7 +398,7 @@
}
/// \brief Emit initialization of an array from an initializer list.
-void AggExprEmitter::EmitArrayInit(llvm::Value *DestPtr, llvm::ArrayType *AType,
+void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
QualType elementType, InitListExpr *E) {
uint64_t NumInitElements = E->getNumInits();
@@ -414,13 +410,17 @@
llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0);
llvm::Value *indices[] = { zero, zero };
llvm::Value *begin =
- Builder.CreateInBoundsGEP(DestPtr, indices, "arrayinit.begin");
+ Builder.CreateInBoundsGEP(DestPtr.getPointer(), indices, "arrayinit.begin");
+
+ CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType);
+ CharUnits elementAlign =
+ DestPtr.getAlignment().alignmentOfArrayElement(elementSize);
// Exception safety requires us to destroy all the
// already-constructed members if an initializer throws.
// For that, we'll need an EH cleanup.
QualType::DestructionKind dtorKind = elementType.isDestructedType();
- llvm::AllocaInst *endOfInit = nullptr;
+ Address endOfInit = Address::invalid();
EHScopeStack::stable_iterator cleanup;
llvm::Instruction *cleanupDominator = nullptr;
if (CGF.needsEHCleanup(dtorKind)) {
@@ -428,10 +428,11 @@
// directly, but the control flow can get so varied here that it
// would actually be quite complex. Therefore we go through an
// alloca.
- endOfInit = CGF.CreateTempAlloca(begin->getType(),
+ endOfInit = CGF.CreateTempAlloca(begin->getType(), CGF.getPointerAlign(),
"arrayinit.endOfInit");
cleanupDominator = Builder.CreateStore(begin, endOfInit);
CGF.pushIrregularPartialArrayCleanup(begin, endOfInit, elementType,
+ elementAlign,
CGF.getDestroyer(dtorKind));
cleanup = CGF.EHStack.stable_begin();
@@ -458,10 +459,11 @@
// Tell the cleanup that it needs to destroy up to this
// element. TODO: some of these stores can be trivially
// observed to be unnecessary.
- if (endOfInit) Builder.CreateStore(element, endOfInit);
+ if (endOfInit.isValid()) Builder.CreateStore(element, endOfInit);
}
- LValue elementLV = CGF.MakeAddrLValue(element, elementType);
+ LValue elementLV =
+ CGF.MakeAddrLValue(Address(element, elementAlign), elementType);
EmitInitializationToLValue(E->getInit(i), elementLV);
}
@@ -482,7 +484,7 @@
// Advance to the start of the rest of the array.
if (NumInitElements) {
element = Builder.CreateInBoundsGEP(element, one, "arrayinit.start");
- if (endOfInit) Builder.CreateStore(element, endOfInit);
+ if (endOfInit.isValid()) Builder.CreateStore(element, endOfInit);
}
// Compute the end of the array.
@@ -500,7 +502,8 @@
currentElement->addIncoming(element, entryBB);
// Emit the actual filler expression.
- LValue elementLV = CGF.MakeAddrLValue(currentElement, elementType);
+ LValue elementLV =
+ CGF.MakeAddrLValue(Address(currentElement, elementAlign), elementType);
if (filler)
EmitInitializationToLValue(filler, elementLV);
else
@@ -511,7 +514,7 @@
Builder.CreateInBoundsGEP(currentElement, one, "arrayinit.next");
// Tell the EH cleanup that we finished with the last element.
- if (endOfInit) Builder.CreateStore(nextElement, endOfInit);
+ if (endOfInit.isValid()) Builder.CreateStore(nextElement, endOfInit);
// Leave the loop if we're done.
llvm::Value *done = Builder.CreateICmpEQ(nextElement, end,
@@ -569,6 +572,8 @@
}
void AggExprEmitter::VisitCastExpr(CastExpr *E) {
+ if (const auto *ECE = dyn_cast<ExplicitCastExpr>(E))
+ CGF.CGM.EmitExplicitCastExprType(ECE, &CGF);
switch (E->getCastKind()) {
case CK_Dynamic: {
// FIXME: Can this actually happen? We have no test coverage for it.
@@ -596,9 +601,8 @@
// GCC union extension
QualType Ty = E->getSubExpr()->getType();
- QualType PtrTy = CGF.getContext().getPointerType(Ty);
- llvm::Value *CastPtr = Builder.CreateBitCast(Dest.getAddr(),
- CGF.ConvertType(PtrTy));
+ Address CastPtr =
+ Builder.CreateElementBitCast(Dest.getAddress(), CGF.ConvertType(Ty));
EmitInitializationToLValue(E->getSubExpr(),
CGF.MakeAddrLValue(CastPtr, Ty));
break;
@@ -649,13 +653,13 @@
// Zero-initialize. (Strictly speaking, we only need to intialize
// the padding at the end, but this is simpler.)
if (!Dest.isZeroed())
- CGF.EmitNullInitialization(Dest.getAddr(), atomicType);
+ CGF.EmitNullInitialization(Dest.getAddress(), atomicType);
// Build a GEP to refer to the subobject.
- llvm::Value *valueAddr =
- CGF.Builder.CreateStructGEP(nullptr, valueDest.getAddr(), 0);
+ Address valueAddr =
+ CGF.Builder.CreateStructGEP(valueDest.getAddress(), 0,
+ CharUnits());
valueDest = AggValueSlot::forAddr(valueAddr,
- valueDest.getAlignment(),
valueDest.getQualifiers(),
valueDest.isExternallyDestructed(),
valueDest.requiresGCollection(),
@@ -673,8 +677,8 @@
CGF.CreateAggTemp(atomicType, "atomic-to-nonatomic.temp");
CGF.EmitAggExpr(E->getSubExpr(), atomicSlot);
- llvm::Value *valueAddr =
- Builder.CreateStructGEP(nullptr, atomicSlot.getAddr(), 0);
+ Address valueAddr =
+ Builder.CreateStructGEP(atomicSlot.getAddress(), 0, CharUnits());
RValue rvalue = RValue::getAggregate(valueAddr, atomicSlot.isVolatile());
return EmitFinalDestCopy(valueType, rvalue);
}
@@ -959,15 +963,15 @@
}
void AggExprEmitter::VisitVAArgExpr(VAArgExpr *VE) {
- llvm::Value *ArgValue = CGF.EmitVAListRef(VE->getSubExpr());
- llvm::Value *ArgPtr = CGF.EmitVAArg(ArgValue, VE->getType());
+ Address ArgValue = Address::invalid();
+ Address ArgPtr = CGF.EmitVAArg(VE, ArgValue);
- if (!ArgPtr) {
+ if (!ArgPtr.isValid()) {
// If EmitVAArg fails, we fall back to the LLVM instruction.
- llvm::Value *Val =
- Builder.CreateVAArg(ArgValue, CGF.ConvertType(VE->getType()));
+ llvm::Value *Val = Builder.CreateVAArg(ArgValue.getPointer(),
+ CGF.ConvertType(VE->getType()));
if (!Dest.isIgnored())
- Builder.CreateStore(Val, Dest.getAddr());
+ Builder.CreateStore(Val, Dest.getAddress());
return;
}
@@ -987,7 +991,7 @@
// Push that destructor we promised.
if (!wasExternallyDestructed)
- CGF.EmitCXXTemporary(E->getTemporary(), E->getType(), Dest.getAddr());
+ CGF.EmitCXXTemporary(E->getTemporary(), E->getType(), Dest.getAddress());
}
void
@@ -1011,13 +1015,13 @@
void AggExprEmitter::VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) {
QualType T = E->getType();
AggValueSlot Slot = EnsureSlot(T);
- EmitNullInitializationToLValue(CGF.MakeAddrLValue(Slot.getAddr(), T));
+ EmitNullInitializationToLValue(CGF.MakeAddrLValue(Slot.getAddress(), T));
}
void AggExprEmitter::VisitImplicitValueInitExpr(ImplicitValueInitExpr *E) {
QualType T = E->getType();
AggValueSlot Slot = EnsureSlot(T);
- EmitNullInitializationToLValue(CGF.MakeAddrLValue(Slot.getAddr(), T));
+ EmitNullInitializationToLValue(CGF.MakeAddrLValue(Slot.getAddress(), T));
}
/// isSimpleZero - If emitting this value will obviously just cause a store of
@@ -1135,8 +1139,7 @@
AggValueSlot Dest = EnsureSlot(E->getType());
- LValue DestLV = CGF.MakeAddrLValue(Dest.getAddr(), E->getType(),
- Dest.getAlignment());
+ LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType());
// Handle initialization of an array.
if (E->getType()->isArrayType()) {
@@ -1146,12 +1149,8 @@
QualType elementType =
CGF.getContext().getAsArrayType(E->getType())->getElementType();
- llvm::PointerType *APType =
- cast<llvm::PointerType>(Dest.getAddr()->getType());
- llvm::ArrayType *AType =
- cast<llvm::ArrayType>(APType->getElementType());
-
- EmitArrayInit(Dest.getAddr(), AType, elementType, E);
+ auto AType = cast<llvm::ArrayType>(Dest.getAddress().getElementType());
+ EmitArrayInit(Dest.getAddress(), AType, elementType, E);
return;
}
@@ -1175,7 +1174,7 @@
RecordDecl *record = E->getType()->castAs<RecordType>()->getDecl();
// Prepare a 'this' for CXXDefaultInitExprs.
- CodeGenFunction::FieldConstructionScope FCS(CGF, Dest.getAddr());
+ CodeGenFunction::FieldConstructionScope FCS(CGF, Dest.getAddress());
if (record->isUnion()) {
// Only initialize one field of a union. The field itself is
@@ -1253,7 +1252,10 @@
assert(LV.isSimple());
if (CGF.needsEHCleanup(dtorKind)) {
if (!cleanupDominator)
- cleanupDominator = CGF.Builder.CreateUnreachable(); // placeholder
+ cleanupDominator = CGF.Builder.CreateAlignedLoad(
+ CGF.Int8Ty,
+ llvm::Constant::getNullValue(CGF.Int8PtrTy),
+ CharUnits::One()); // placeholder
CGF.pushDestroy(EHCleanup, LV.getAddress(), field->getType(),
CGF.getDestroyer(dtorKind), false);
@@ -1266,7 +1268,7 @@
// else, clean it up for -O0 builds and general tidiness.
if (!pushedCleanup && LV.isSimple())
if (llvm::GetElementPtrInst *GEP =
- dyn_cast<llvm::GetElementPtrInst>(LV.getAddress()))
+ dyn_cast<llvm::GetElementPtrInst>(LV.getPointer()))
if (GEP->use_empty())
GEP->eraseFromParent();
}
@@ -1284,8 +1286,7 @@
void AggExprEmitter::VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E) {
AggValueSlot Dest = EnsureSlot(E->getType());
- LValue DestLV = CGF.MakeAddrLValue(Dest.getAddr(), E->getType(),
- Dest.getAlignment());
+ LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType());
EmitInitializationToLValue(E->getBase(), DestLV);
VisitInitListExpr(E->getUpdater());
}
@@ -1355,7 +1356,7 @@
CodeGenFunction &CGF) {
// If the slot is already known to be zeroed, nothing to do. Don't mess with
// volatile stores.
- if (Slot.isZeroed() || Slot.isVolatile() || Slot.getAddr() == nullptr)
+ if (Slot.isZeroed() || Slot.isVolatile() || !Slot.getAddress().isValid())
return;
// C++ objects with a user-declared constructor don't need zero'ing.
@@ -1368,26 +1369,22 @@
}
// If the type is 16-bytes or smaller, prefer individual stores over memset.
- std::pair<CharUnits, CharUnits> TypeInfo =
- CGF.getContext().getTypeInfoInChars(E->getType());
- if (TypeInfo.first <= CharUnits::fromQuantity(16))
+ CharUnits Size = CGF.getContext().getTypeSizeInChars(E->getType());
+ if (Size <= CharUnits::fromQuantity(16))
return;
// Check to see if over 3/4 of the initializer are known to be zero. If so,
// we prefer to emit memset + individual stores for the rest.
CharUnits NumNonZeroBytes = GetNumNonZeroBytesInInit(E, CGF);
- if (NumNonZeroBytes*4 > TypeInfo.first)
+ if (NumNonZeroBytes*4 > Size)
return;
// Okay, it seems like a good idea to use an initial memset, emit the call.
- llvm::Constant *SizeVal = CGF.Builder.getInt64(TypeInfo.first.getQuantity());
- CharUnits Align = TypeInfo.second;
+ llvm::Constant *SizeVal = CGF.Builder.getInt64(Size.getQuantity());
- llvm::Value *Loc = Slot.getAddr();
-
- Loc = CGF.Builder.CreateBitCast(Loc, CGF.Int8PtrTy);
- CGF.Builder.CreateMemSet(Loc, CGF.Builder.getInt8(0), SizeVal,
- Align.getQuantity(), false);
+ Address Loc = Slot.getAddress();
+ Loc = CGF.Builder.CreateElementBitCast(Loc, CGF.Int8Ty);
+ CGF.Builder.CreateMemSet(Loc, CGF.Builder.getInt8(0), SizeVal, false);
// Tell the AggExprEmitter that the slot is known zero.
Slot.setZeroed();
@@ -1403,7 +1400,7 @@
void CodeGenFunction::EmitAggExpr(const Expr *E, AggValueSlot Slot) {
assert(E && hasAggregateEvaluationKind(E->getType()) &&
"Invalid aggregate expression to emit");
- assert((Slot.getAddr() != nullptr || Slot.isIgnored()) &&
+ assert((Slot.getAddress().isValid() || Slot.isIgnored()) &&
"slot has bits but no address");
// Optimize the slot if possible.
@@ -1414,7 +1411,7 @@
LValue CodeGenFunction::EmitAggExprToLValue(const Expr *E) {
assert(hasAggregateEvaluationKind(E->getType()) && "Invalid argument!");
- llvm::Value *Temp = CreateMemTemp(E->getType());
+ Address Temp = CreateMemTemp(E->getType());
LValue LV = MakeAddrLValue(Temp, E->getType());
EmitAggExpr(E, AggValueSlot::forLValue(LV, AggValueSlot::IsNotDestructed,
AggValueSlot::DoesNotNeedGCBarriers,
@@ -1422,10 +1419,9 @@
return LV;
}
-void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr,
- llvm::Value *SrcPtr, QualType Ty,
+void CodeGenFunction::EmitAggregateCopy(Address DestPtr,
+ Address SrcPtr, QualType Ty,
bool isVolatile,
- CharUnits alignment,
bool isAssignment) {
assert(!Ty->isAnyComplexType() && "Shouldn't happen for complex");
@@ -1456,17 +1452,16 @@
// implementation handles this case safely. If there is a libc that does not
// safely handle this, we can add a target hook.
- // Get data size and alignment info for this aggregate. If this is an
- // assignment don't copy the tail padding. Otherwise copying it is fine.
+ // Get data size info for this aggregate. If this is an assignment,
+ // don't copy the tail padding, because we might be assigning into a
+ // base subobject where the tail padding is claimed. Otherwise,
+ // copying it is fine.
std::pair<CharUnits, CharUnits> TypeInfo;
if (isAssignment)
TypeInfo = getContext().getTypeInfoDataSizeInChars(Ty);
else
TypeInfo = getContext().getTypeInfoInChars(Ty);
- if (alignment.isZero())
- alignment = TypeInfo.second;
-
llvm::Value *SizeVal = nullptr;
if (TypeInfo.first.isZero()) {
// But note that getTypeInfo returns 0 for a VLA.
@@ -1509,15 +1504,8 @@
// we need to use a different call here. We use isVolatile to indicate when
// either the source or the destination is volatile.
- llvm::PointerType *DPT = cast<llvm::PointerType>(DestPtr->getType());
- llvm::Type *DBP =
- llvm::Type::getInt8PtrTy(getLLVMContext(), DPT->getAddressSpace());
- DestPtr = Builder.CreateBitCast(DestPtr, DBP);
-
- llvm::PointerType *SPT = cast<llvm::PointerType>(SrcPtr->getType());
- llvm::Type *SBP =
- llvm::Type::getInt8PtrTy(getLLVMContext(), SPT->getAddressSpace());
- SrcPtr = Builder.CreateBitCast(SrcPtr, SBP);
+ DestPtr = Builder.CreateElementBitCast(DestPtr, Int8Ty);
+ SrcPtr = Builder.CreateElementBitCast(SrcPtr, Int8Ty);
// Don't do any of the memmove_collectable tests if GC isn't set.
if (CGM.getLangOpts().getGC() == LangOptions::NonGC) {
@@ -1540,11 +1528,11 @@
}
}
+ auto Inst = Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, isVolatile);
+
// Determine the metadata to describe the position of any padding in this
// memcpy, as well as the TBAA tags for the members of the struct, in case
// the optimizer wishes to expand it in to scalar memory operations.
- llvm::MDNode *TBAAStructTag = CGM.getTBAAStructInfo(Ty);
-
- Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, alignment.getQuantity(),
- isVolatile, /*TBAATag=*/nullptr, TBAAStructTag);
+ if (llvm::MDNode *TBAAStructTag = CGM.getTBAAStructInfo(Ty))
+ Inst->setMetadata(llvm::LLVMContext::MD_tbaa_struct, TBAAStructTag);
}
diff --git a/lib/CodeGen/CGExprCXX.cpp b/lib/CodeGen/CGExprCXX.cpp
index b3353ba..604cde7 100644
--- a/lib/CodeGen/CGExprCXX.cpp
+++ b/lib/CodeGen/CGExprCXX.cpp
@@ -59,7 +59,7 @@
if (CE) {
// Special case: skip first argument of CXXOperatorCall (it is "this").
unsigned ArgsToSkip = isa<CXXOperatorCallExpr>(CE) ? 1 : 0;
- CGF.EmitCallArgs(Args, FPT, CE->arg_begin() + ArgsToSkip, CE->arg_end(),
+ CGF.EmitCallArgs(Args, FPT, drop_begin(CE->arguments(), ArgsToSkip),
CE->getDirectCallee());
} else {
assert(
@@ -166,9 +166,9 @@
}
}
- llvm::Value *This;
+ Address This = Address::invalid();
if (IsArrow)
- This = EmitScalarExpr(Base);
+ This = EmitPointerWithAlignment(Base);
else
This = EmitLValue(Base).getAddress();
@@ -185,19 +185,18 @@
// when it isn't necessary; just produce the proper effect here.
// Special case: skip first argument of CXXOperatorCall (it is "this").
unsigned ArgsToSkip = isa<CXXOperatorCallExpr>(CE) ? 1 : 0;
- llvm::Value *RHS =
- EmitLValue(*(CE->arg_begin() + ArgsToSkip)).getAddress();
+ Address RHS = EmitLValue(*(CE->arg_begin() + ArgsToSkip)).getAddress();
EmitAggregateAssign(This, RHS, CE->getType());
- return RValue::get(This);
+ return RValue::get(This.getPointer());
}
if (isa<CXXConstructorDecl>(MD) &&
cast<CXXConstructorDecl>(MD)->isCopyOrMoveConstructor()) {
// Trivial move and copy ctor are the same.
assert(CE->getNumArgs() == 1 && "unexpected argcount for trivial ctor");
- llvm::Value *RHS = EmitLValue(*CE->arg_begin()).getAddress();
- EmitAggregateCopy(This, RHS, CE->arg_begin()->getType());
- return RValue::get(This);
+ Address RHS = EmitLValue(*CE->arg_begin()).getAddress();
+ EmitAggregateCopy(This, RHS, (*CE->arg_begin())->getType());
+ return RValue::get(This.getPointer());
}
llvm_unreachable("unknown trivial member function");
}
@@ -245,7 +244,7 @@
cast<CXXDestructorDecl>(DevirtualizedMethod);
Callee = CGM.GetAddrOfFunction(GlobalDecl(DDtor, Dtor_Complete), Ty);
}
- EmitCXXMemberOrOperatorCall(MD, Callee, ReturnValue, This,
+ EmitCXXMemberOrOperatorCall(MD, Callee, ReturnValue, This.getPointer(),
/*ImplicitParam=*/nullptr, QualType(), CE);
}
return RValue::get(nullptr);
@@ -254,12 +253,13 @@
if (const CXXConstructorDecl *Ctor = dyn_cast<CXXConstructorDecl>(MD)) {
Callee = CGM.GetAddrOfFunction(GlobalDecl(Ctor, Ctor_Complete), Ty);
} else if (UseVirtualCall) {
- Callee = CGM.getCXXABI().getVirtualFunctionPointer(*this, MD, This, Ty);
+ Callee = CGM.getCXXABI().getVirtualFunctionPointer(*this, MD, This, Ty,
+ CE->getLocStart());
} else {
if (SanOpts.has(SanitizerKind::CFINVCall) &&
MD->getParent()->isDynamicClass()) {
- llvm::Value *VTable = GetVTablePtr(This, Int8PtrTy);
- EmitVTablePtrCheckForCall(MD, VTable);
+ llvm::Value *VTable = GetVTablePtr(This, Int8PtrTy, MD->getParent());
+ EmitVTablePtrCheckForCall(MD, VTable, CFITCK_NVCall, CE->getLocStart());
}
if (getLangOpts().AppleKext && MD->isVirtual() && HasQualifier)
@@ -276,7 +276,7 @@
*this, MD, This, UseVirtualCall);
}
- return EmitCXXMemberOrOperatorCall(MD, Callee, ReturnValue, This,
+ return EmitCXXMemberOrOperatorCall(MD, Callee, ReturnValue, This.getPointer(),
/*ImplicitParam=*/nullptr, QualType(), CE);
}
@@ -300,19 +300,20 @@
llvm::Value *MemFnPtr = EmitScalarExpr(MemFnExpr);
// Emit the 'this' pointer.
- llvm::Value *This;
-
+ Address This = Address::invalid();
if (BO->getOpcode() == BO_PtrMemI)
- This = EmitScalarExpr(BaseExpr);
+ This = EmitPointerWithAlignment(BaseExpr);
else
This = EmitLValue(BaseExpr).getAddress();
- EmitTypeCheck(TCK_MemberCall, E->getExprLoc(), This,
+ EmitTypeCheck(TCK_MemberCall, E->getExprLoc(), This.getPointer(),
QualType(MPT->getClass(), 0));
// Ask the ABI to load the callee. Note that This is modified.
+ llvm::Value *ThisPtrForCall = nullptr;
llvm::Value *Callee =
- CGM.getCXXABI().EmitLoadOfMemberFunctionPointer(*this, BO, This, MemFnPtr, MPT);
+ CGM.getCXXABI().EmitLoadOfMemberFunctionPointer(*this, BO, This,
+ ThisPtrForCall, MemFnPtr, MPT);
CallArgList Args;
@@ -320,12 +321,12 @@
getContext().getPointerType(getContext().getTagDeclType(RD));
// Push the this ptr.
- Args.add(RValue::get(This), ThisType);
+ Args.add(RValue::get(ThisPtrForCall), ThisType);
RequiredArgs required = RequiredArgs::forPrototypePlus(FPT, 1);
// And the rest of the call args
- EmitCallArgs(Args, FPT, E->arg_begin(), E->arg_end(), E->getDirectCallee());
+ EmitCallArgs(Args, FPT, E->arguments(), E->getDirectCallee());
return EmitCall(CGM.getTypes().arrangeCXXMethodCall(Args, FPT, required),
Callee, ReturnValue, Args);
}
@@ -347,18 +348,43 @@
}
static void EmitNullBaseClassInitialization(CodeGenFunction &CGF,
- llvm::Value *DestPtr,
+ Address DestPtr,
const CXXRecordDecl *Base) {
if (Base->isEmpty())
return;
- DestPtr = CGF.EmitCastToVoidPtr(DestPtr);
+ DestPtr = CGF.Builder.CreateElementBitCast(DestPtr, CGF.Int8Ty);
const ASTRecordLayout &Layout = CGF.getContext().getASTRecordLayout(Base);
- CharUnits Size = Layout.getNonVirtualSize();
- CharUnits Align = Layout.getNonVirtualAlignment();
+ CharUnits NVSize = Layout.getNonVirtualSize();
- llvm::Value *SizeVal = CGF.CGM.getSize(Size);
+ // We cannot simply zero-initialize the entire base sub-object if vbptrs are
+ // present, they are initialized by the most derived class before calling the
+ // constructor.
+ SmallVector<std::pair<CharUnits, CharUnits>, 1> Stores;
+ Stores.emplace_back(CharUnits::Zero(), NVSize);
+
+ // Each store is split by the existence of a vbptr.
+ CharUnits VBPtrWidth = CGF.getPointerSize();
+ std::vector<CharUnits> VBPtrOffsets =
+ CGF.CGM.getCXXABI().getVBPtrOffsets(Base);
+ for (CharUnits VBPtrOffset : VBPtrOffsets) {
+ std::pair<CharUnits, CharUnits> LastStore = Stores.pop_back_val();
+ CharUnits LastStoreOffset = LastStore.first;
+ CharUnits LastStoreSize = LastStore.second;
+
+ CharUnits SplitBeforeOffset = LastStoreOffset;
+ CharUnits SplitBeforeSize = VBPtrOffset - SplitBeforeOffset;
+ assert(!SplitBeforeSize.isNegative() && "negative store size!");
+ if (!SplitBeforeSize.isZero())
+ Stores.emplace_back(SplitBeforeOffset, SplitBeforeSize);
+
+ CharUnits SplitAfterOffset = VBPtrOffset + VBPtrWidth;
+ CharUnits SplitAfterSize = LastStoreSize - SplitAfterOffset;
+ assert(!SplitAfterSize.isNegative() && "negative store size!");
+ if (!SplitAfterSize.isZero())
+ Stores.emplace_back(SplitAfterOffset, SplitAfterSize);
+ }
// If the type contains a pointer to data member we can't memset it to zero.
// Instead, create a null constant and copy it to the destination.
@@ -366,27 +392,43 @@
// like -1, which happens to be the pattern used by member-pointers.
// TODO: isZeroInitializable can be over-conservative in the case where a
// virtual base contains a member pointer.
- if (!CGF.CGM.getTypes().isZeroInitializable(Base)) {
- llvm::Constant *NullConstant = CGF.CGM.EmitNullConstantForBase(Base);
+ llvm::Constant *NullConstantForBase = CGF.CGM.EmitNullConstantForBase(Base);
+ if (!NullConstantForBase->isNullValue()) {
+ llvm::GlobalVariable *NullVariable = new llvm::GlobalVariable(
+ CGF.CGM.getModule(), NullConstantForBase->getType(),
+ /*isConstant=*/true, llvm::GlobalVariable::PrivateLinkage,
+ NullConstantForBase, Twine());
- llvm::GlobalVariable *NullVariable =
- new llvm::GlobalVariable(CGF.CGM.getModule(), NullConstant->getType(),
- /*isConstant=*/true,
- llvm::GlobalVariable::PrivateLinkage,
- NullConstant, Twine());
+ CharUnits Align = std::max(Layout.getNonVirtualAlignment(),
+ DestPtr.getAlignment());
NullVariable->setAlignment(Align.getQuantity());
- llvm::Value *SrcPtr = CGF.EmitCastToVoidPtr(NullVariable);
+
+ Address SrcPtr = Address(CGF.EmitCastToVoidPtr(NullVariable), Align);
// Get and call the appropriate llvm.memcpy overload.
- CGF.Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, Align.getQuantity());
- return;
- }
-
+ for (std::pair<CharUnits, CharUnits> Store : Stores) {
+ CharUnits StoreOffset = Store.first;
+ CharUnits StoreSize = Store.second;
+ llvm::Value *StoreSizeVal = CGF.CGM.getSize(StoreSize);
+ CGF.Builder.CreateMemCpy(
+ CGF.Builder.CreateConstInBoundsByteGEP(DestPtr, StoreOffset),
+ CGF.Builder.CreateConstInBoundsByteGEP(SrcPtr, StoreOffset),
+ StoreSizeVal);
+ }
+
// Otherwise, just memset the whole thing to zero. This is legal
// because in LLVM, all default initializers (other than the ones we just
// handled above) are guaranteed to have a bit pattern of all zeros.
- CGF.Builder.CreateMemSet(DestPtr, CGF.Builder.getInt8(0), SizeVal,
- Align.getQuantity());
+ } else {
+ for (std::pair<CharUnits, CharUnits> Store : Stores) {
+ CharUnits StoreOffset = Store.first;
+ CharUnits StoreSize = Store.second;
+ llvm::Value *StoreSizeVal = CGF.CGM.getSize(StoreSize);
+ CGF.Builder.CreateMemSet(
+ CGF.Builder.CreateConstInBoundsByteGEP(DestPtr, StoreOffset),
+ CGF.Builder.getInt8(0), StoreSizeVal);
+ }
+ }
}
void
@@ -403,11 +445,12 @@
switch (E->getConstructionKind()) {
case CXXConstructExpr::CK_Delegating:
case CXXConstructExpr::CK_Complete:
- EmitNullInitialization(Dest.getAddr(), E->getType());
+ EmitNullInitialization(Dest.getAddress(), E->getType());
break;
case CXXConstructExpr::CK_VirtualBase:
case CXXConstructExpr::CK_NonVirtualBase:
- EmitNullBaseClassInitialization(*this, Dest.getAddr(), CD->getParent());
+ EmitNullBaseClassInitialization(*this, Dest.getAddress(),
+ CD->getParent());
break;
}
}
@@ -430,7 +473,7 @@
if (const ConstantArrayType *arrayType
= getContext().getAsConstantArrayType(E->getType())) {
- EmitCXXAggrConstructorCall(CD, arrayType, Dest.getAddr(), E);
+ EmitCXXAggrConstructorCall(CD, arrayType, Dest.getAddress(), E);
} else {
CXXCtorType Type = Ctor_Complete;
bool ForVirtualBase = false;
@@ -456,15 +499,13 @@
}
// Call the constructor.
- EmitCXXConstructorCall(CD, Type, ForVirtualBase, Delegating, Dest.getAddr(),
- E);
+ EmitCXXConstructorCall(CD, Type, ForVirtualBase, Delegating,
+ Dest.getAddress(), E);
}
}
-void
-CodeGenFunction::EmitSynthesizedCXXCopyCtor(llvm::Value *Dest,
- llvm::Value *Src,
- const Expr *Exp) {
+void CodeGenFunction::EmitSynthesizedCXXCopyCtor(Address Dest, Address Src,
+ const Expr *Exp) {
if (const ExprWithCleanups *E = dyn_cast<ExprWithCleanups>(Exp))
Exp = E->getSubExpr();
assert(isa<CXXConstructExpr>(Exp) &&
@@ -758,22 +799,20 @@
}
static void StoreAnyExprIntoOneUnit(CodeGenFunction &CGF, const Expr *Init,
- QualType AllocType, llvm::Value *NewPtr) {
+ QualType AllocType, Address NewPtr) {
// FIXME: Refactor with EmitExprAsInit.
- CharUnits Alignment = CGF.getContext().getTypeAlignInChars(AllocType);
switch (CGF.getEvaluationKind(AllocType)) {
case TEK_Scalar:
CGF.EmitScalarInit(Init, nullptr,
- CGF.MakeAddrLValue(NewPtr, AllocType, Alignment), false);
+ CGF.MakeAddrLValue(NewPtr, AllocType), false);
return;
case TEK_Complex:
- CGF.EmitComplexExprIntoLValue(Init, CGF.MakeAddrLValue(NewPtr, AllocType,
- Alignment),
+ CGF.EmitComplexExprIntoLValue(Init, CGF.MakeAddrLValue(NewPtr, AllocType),
/*isInit*/ true);
return;
case TEK_Aggregate: {
AggValueSlot Slot
- = AggValueSlot::forAddr(NewPtr, Alignment, AllocType.getQualifiers(),
+ = AggValueSlot::forAddr(NewPtr, AllocType.getQualifiers(),
AggValueSlot::IsDestructed,
AggValueSlot::DoesNotNeedGCBarriers,
AggValueSlot::IsNotAliased);
@@ -786,23 +825,27 @@
void CodeGenFunction::EmitNewArrayInitializer(
const CXXNewExpr *E, QualType ElementType, llvm::Type *ElementTy,
- llvm::Value *BeginPtr, llvm::Value *NumElements,
+ Address BeginPtr, llvm::Value *NumElements,
llvm::Value *AllocSizeWithoutCookie) {
// If we have a type with trivial initialization and no initializer,
// there's nothing to do.
if (!E->hasInitializer())
return;
- llvm::Value *CurPtr = BeginPtr;
+ Address CurPtr = BeginPtr;
unsigned InitListElements = 0;
const Expr *Init = E->getInitializer();
- llvm::AllocaInst *EndOfInit = nullptr;
+ Address EndOfInit = Address::invalid();
QualType::DestructionKind DtorKind = ElementType.isDestructedType();
EHScopeStack::stable_iterator Cleanup;
llvm::Instruction *CleanupDominator = nullptr;
+ CharUnits ElementSize = getContext().getTypeSizeInChars(ElementType);
+ CharUnits ElementAlign =
+ BeginPtr.getAlignment().alignmentOfArrayElement(ElementSize);
+
// If the initializer is an initializer list, first do the explicit elements.
if (const InitListExpr *ILE = dyn_cast<InitListExpr>(Init)) {
InitListElements = ILE->getNumInits();
@@ -812,10 +855,8 @@
QualType AllocType = E->getAllocatedType();
if (const ConstantArrayType *CAT = dyn_cast_or_null<ConstantArrayType>(
AllocType->getAsArrayTypeUnsafe())) {
- unsigned AS = CurPtr->getType()->getPointerAddressSpace();
ElementTy = ConvertTypeForMem(AllocType);
- llvm::Type *AllocPtrTy = ElementTy->getPointerTo(AS);
- CurPtr = Builder.CreateBitCast(CurPtr, AllocPtrTy);
+ CurPtr = Builder.CreateElementBitCast(CurPtr, ElementTy);
InitListElements *= getContext().getConstantArrayElementCount(CAT);
}
@@ -825,27 +866,34 @@
// directly, but the control flow can get so varied here that it
// would actually be quite complex. Therefore we go through an
// alloca.
- EndOfInit = CreateTempAlloca(BeginPtr->getType(), "array.init.end");
- CleanupDominator = Builder.CreateStore(BeginPtr, EndOfInit);
- pushIrregularPartialArrayCleanup(BeginPtr, EndOfInit, ElementType,
+ EndOfInit = CreateTempAlloca(BeginPtr.getType(), getPointerAlign(),
+ "array.init.end");
+ CleanupDominator = Builder.CreateStore(BeginPtr.getPointer(), EndOfInit);
+ pushIrregularPartialArrayCleanup(BeginPtr.getPointer(), EndOfInit,
+ ElementType, ElementAlign,
getDestroyer(DtorKind));
Cleanup = EHStack.stable_begin();
}
+ CharUnits StartAlign = CurPtr.getAlignment();
for (unsigned i = 0, e = ILE->getNumInits(); i != e; ++i) {
// Tell the cleanup that it needs to destroy up to this
// element. TODO: some of these stores can be trivially
// observed to be unnecessary.
- if (EndOfInit)
- Builder.CreateStore(Builder.CreateBitCast(CurPtr, BeginPtr->getType()),
- EndOfInit);
+ if (EndOfInit.isValid()) {
+ auto FinishedPtr =
+ Builder.CreateBitCast(CurPtr.getPointer(), BeginPtr.getType());
+ Builder.CreateStore(FinishedPtr, EndOfInit);
+ }
// FIXME: If the last initializer is an incomplete initializer list for
// an array, and we have an array filler, we can fold together the two
// initialization loops.
StoreAnyExprIntoOneUnit(*this, ILE->getInit(i),
ILE->getInit(i)->getType(), CurPtr);
- CurPtr = Builder.CreateConstInBoundsGEP1_32(ElementTy, CurPtr, 1,
- "array.exp.next");
+ CurPtr = Address(Builder.CreateInBoundsGEP(CurPtr.getPointer(),
+ Builder.getSize(1),
+ "array.exp.next"),
+ StartAlign.alignmentAtOffset((i + 1) * ElementSize));
}
// The remaining elements are filled with the array filler expression.
@@ -863,7 +911,7 @@
}
// Switch back to initializing one base element at a time.
- CurPtr = Builder.CreateBitCast(CurPtr, BeginPtr->getType());
+ CurPtr = Builder.CreateBitCast(CurPtr, BeginPtr.getType());
}
// Attempt to perform zero-initialization using memset.
@@ -888,9 +936,7 @@
}
// Create the memset.
- CharUnits Alignment = getContext().getTypeAlignInChars(ElementType);
- Builder.CreateMemSet(CurPtr, Builder.getInt8(0), RemainingSize,
- Alignment.getQuantity(), false);
+ Builder.CreateMemSet(CurPtr, Builder.getInt8(0), RemainingSize, false);
return true;
};
@@ -924,7 +970,8 @@
//
// FIXME: Share this cleanup with the constructor call emission rather than
// having it create a cleanup of its own.
- if (EndOfInit) Builder.CreateStore(CurPtr, EndOfInit);
+ if (EndOfInit.isValid())
+ Builder.CreateStore(CurPtr.getPointer(), EndOfInit);
// Emit a constructor call loop to initialize the remaining elements.
if (InitListElements)
@@ -984,13 +1031,13 @@
// Find the end of the array, hoisted out of the loop.
llvm::Value *EndPtr =
- Builder.CreateInBoundsGEP(BeginPtr, NumElements, "array.end");
+ Builder.CreateInBoundsGEP(BeginPtr.getPointer(), NumElements, "array.end");
// If the number of elements isn't constant, we have to now check if there is
// anything left to initialize.
if (!ConstNum) {
- llvm::Value *IsEmpty = Builder.CreateICmpEQ(CurPtr, EndPtr,
- "array.isempty");
+ llvm::Value *IsEmpty =
+ Builder.CreateICmpEQ(CurPtr.getPointer(), EndPtr, "array.isempty");
Builder.CreateCondBr(IsEmpty, ContBB, LoopBB);
}
@@ -999,16 +1046,19 @@
// Set up the current-element phi.
llvm::PHINode *CurPtrPhi =
- Builder.CreatePHI(CurPtr->getType(), 2, "array.cur");
- CurPtrPhi->addIncoming(CurPtr, EntryBB);
- CurPtr = CurPtrPhi;
+ Builder.CreatePHI(CurPtr.getType(), 2, "array.cur");
+ CurPtrPhi->addIncoming(CurPtr.getPointer(), EntryBB);
+
+ CurPtr = Address(CurPtrPhi, ElementAlign);
// Store the new Cleanup position for irregular Cleanups.
- if (EndOfInit) Builder.CreateStore(CurPtr, EndOfInit);
+ if (EndOfInit.isValid())
+ Builder.CreateStore(CurPtr.getPointer(), EndOfInit);
// Enter a partial-destruction Cleanup if necessary.
if (!CleanupDominator && needsEHCleanup(DtorKind)) {
- pushRegularPartialArrayCleanup(BeginPtr, CurPtr, ElementType,
+ pushRegularPartialArrayCleanup(BeginPtr.getPointer(), CurPtr.getPointer(),
+ ElementType, ElementAlign,
getDestroyer(DtorKind));
Cleanup = EHStack.stable_begin();
CleanupDominator = Builder.CreateUnreachable();
@@ -1025,7 +1075,8 @@
// Advance to the next element by adjusting the pointer type as necessary.
llvm::Value *NextPtr =
- Builder.CreateConstInBoundsGEP1_32(ElementTy, CurPtr, 1, "array.next");
+ Builder.CreateConstInBoundsGEP1_32(ElementTy, CurPtr.getPointer(), 1,
+ "array.next");
// Check whether we've gotten to the end of the array and, if so,
// exit the loop.
@@ -1038,7 +1089,7 @@
static void EmitNewInitializer(CodeGenFunction &CGF, const CXXNewExpr *E,
QualType ElementType, llvm::Type *ElementTy,
- llvm::Value *NewPtr, llvm::Value *NumElements,
+ Address NewPtr, llvm::Value *NumElements,
llvm::Value *AllocSizeWithoutCookie) {
ApplyDebugLocation DL(CGF, E);
if (E->isArray())
@@ -1088,8 +1139,7 @@
bool IsDelete) {
CallArgList Args;
const Stmt *ArgS = Arg;
- EmitCallArgs(Args, *Type->param_type_begin(),
- ConstExprIterator(&ArgS), ConstExprIterator(&ArgS + 1));
+ EmitCallArgs(Args, *Type->param_type_begin(), llvm::makeArrayRef(ArgS));
// Find the allocation or deallocation function that we're calling.
ASTContext &Ctx = getContext();
DeclarationName Name = Ctx.DeclarationNames
@@ -1104,7 +1154,7 @@
namespace {
/// A cleanup to call the given 'operator delete' function upon
/// abnormal exit from a new expression.
- class CallDeleteDuringNew : public EHScopeStack::Cleanup {
+ class CallDeleteDuringNew final : public EHScopeStack::Cleanup {
size_t NumPlacementArgs;
const FunctionDecl *OperatorDelete;
llvm::Value *Ptr;
@@ -1157,7 +1207,7 @@
/// A cleanup to call the given 'operator delete' function upon
/// abnormal exit from a new expression when the new expression is
/// conditional.
- class CallDeleteDuringConditionalNew : public EHScopeStack::Cleanup {
+ class CallDeleteDuringConditionalNew final : public EHScopeStack::Cleanup {
size_t NumPlacementArgs;
const FunctionDecl *OperatorDelete;
DominatingValue<RValue>::saved_type Ptr;
@@ -1218,7 +1268,7 @@
/// new-expression throws.
static void EnterNewDeleteCleanup(CodeGenFunction &CGF,
const CXXNewExpr *E,
- llvm::Value *NewPtr,
+ Address NewPtr,
llvm::Value *AllocSize,
const CallArgList &NewArgs) {
// If we're not inside a conditional branch, then the cleanup will
@@ -1228,7 +1278,8 @@
.pushCleanupWithExtra<CallDeleteDuringNew>(EHCleanup,
E->getNumPlacementArgs(),
E->getOperatorDelete(),
- NewPtr, AllocSize);
+ NewPtr.getPointer(),
+ AllocSize);
for (unsigned I = 0, N = E->getNumPlacementArgs(); I != N; ++I)
Cleanup->setPlacementArg(I, NewArgs[I+1].RV);
@@ -1237,7 +1288,7 @@
// Otherwise, we need to save all this stuff.
DominatingValue<RValue>::saved_type SavedNewPtr =
- DominatingValue<RValue>::save(CGF, RValue::get(NewPtr));
+ DominatingValue<RValue>::save(CGF, RValue::get(NewPtr.getPointer()));
DominatingValue<RValue>::saved_type SavedAllocSize =
DominatingValue<RValue>::save(CGF, RValue::get(AllocSize));
@@ -1260,13 +1311,6 @@
// 1. Build a call to the allocation function.
FunctionDecl *allocator = E->getOperatorNew();
- const FunctionProtoType *allocatorType =
- allocator->getType()->castAs<FunctionProtoType>();
-
- CallArgList allocatorArgs;
-
- // The allocation size is the first argument.
- QualType sizeType = getContext().getSizeType();
// If there is a brace-initializer, cannot allocate fewer elements than inits.
unsigned minElements = 0;
@@ -1281,24 +1325,61 @@
EmitCXXNewAllocSize(*this, E, minElements, numElements,
allocSizeWithoutCookie);
- allocatorArgs.add(RValue::get(allocSize), sizeType);
-
- // We start at 1 here because the first argument (the allocation size)
- // has already been emitted.
- EmitCallArgs(allocatorArgs, allocatorType, E->placement_arg_begin(),
- E->placement_arg_end(), /* CalleeDecl */ nullptr,
- /*ParamsToSkip*/ 1);
-
// Emit the allocation call. If the allocator is a global placement
// operator, just "inline" it directly.
- RValue RV;
+ Address allocation = Address::invalid();
+ CallArgList allocatorArgs;
if (allocator->isReservedGlobalPlacementOperator()) {
- assert(allocatorArgs.size() == 2);
- RV = allocatorArgs[1].RV;
- // TODO: kill any unnecessary computations done for the size
- // argument.
+ assert(E->getNumPlacementArgs() == 1);
+ const Expr *arg = *E->placement_arguments().begin();
+
+ AlignmentSource alignSource;
+ allocation = EmitPointerWithAlignment(arg, &alignSource);
+
+ // The pointer expression will, in many cases, be an opaque void*.
+ // In these cases, discard the computed alignment and use the
+ // formal alignment of the allocated type.
+ if (alignSource != AlignmentSource::Decl) {
+ allocation = Address(allocation.getPointer(),
+ getContext().getTypeAlignInChars(allocType));
+ }
+
+ // Set up allocatorArgs for the call to operator delete if it's not
+ // the reserved global operator.
+ if (E->getOperatorDelete() &&
+ !E->getOperatorDelete()->isReservedGlobalPlacementOperator()) {
+ allocatorArgs.add(RValue::get(allocSize), getContext().getSizeType());
+ allocatorArgs.add(RValue::get(allocation.getPointer()), arg->getType());
+ }
+
} else {
- RV = EmitNewDeleteCall(*this, allocator, allocatorType, allocatorArgs);
+ const FunctionProtoType *allocatorType =
+ allocator->getType()->castAs<FunctionProtoType>();
+
+ // The allocation size is the first argument.
+ QualType sizeType = getContext().getSizeType();
+ allocatorArgs.add(RValue::get(allocSize), sizeType);
+
+ // We start at 1 here because the first argument (the allocation size)
+ // has already been emitted.
+ EmitCallArgs(allocatorArgs, allocatorType, E->placement_arguments(),
+ /* CalleeDecl */ nullptr,
+ /*ParamsToSkip*/ 1);
+
+ RValue RV =
+ EmitNewDeleteCall(*this, allocator, allocatorType, allocatorArgs);
+
+ // For now, only assume that the allocation function returns
+ // something satisfactorily aligned for the element type, plus
+ // the cookie if we have one.
+ CharUnits allocationAlign =
+ getContext().getTypeAlignInChars(allocType);
+ if (allocSize != allocSizeWithoutCookie) {
+ CharUnits cookieAlign = getSizeAlign(); // FIXME?
+ allocationAlign = std::max(allocationAlign, cookieAlign);
+ }
+
+ allocation = Address(RV.getScalarVal(), allocationAlign);
}
// Emit a null check on the allocation result if the allocation
@@ -1311,9 +1392,6 @@
llvm::BasicBlock *nullCheckBB = nullptr;
llvm::BasicBlock *contBB = nullptr;
- llvm::Value *allocation = RV.getScalarVal();
- unsigned AS = allocation->getType()->getPointerAddressSpace();
-
// The null-check means that the initializer is conditionally
// evaluated.
ConditionalEvaluation conditional(*this);
@@ -1325,7 +1403,8 @@
llvm::BasicBlock *notNullBB = createBasicBlock("new.notnull");
contBB = createBasicBlock("new.cont");
- llvm::Value *isNull = Builder.CreateIsNull(allocation, "new.isnull");
+ llvm::Value *isNull =
+ Builder.CreateIsNull(allocation.getPointer(), "new.isnull");
Builder.CreateCondBr(isNull, contBB, notNullBB);
EmitBlock(notNullBB);
}
@@ -1351,8 +1430,15 @@
}
llvm::Type *elementTy = ConvertTypeForMem(allocType);
- llvm::Type *elementPtrTy = elementTy->getPointerTo(AS);
- llvm::Value *result = Builder.CreateBitCast(allocation, elementPtrTy);
+ Address result = Builder.CreateElementBitCast(allocation, elementTy);
+
+ // Passing pointer through invariant.group.barrier to avoid propagation of
+ // vptrs information which may be included in previous type.
+ if (CGM.getCodeGenOpts().StrictVTablePointers &&
+ CGM.getCodeGenOpts().OptimizationLevel > 0 &&
+ allocator->isReservedGlobalPlacementOperator())
+ result = Address(Builder.CreateInvariantGroupBarrier(result.getPointer()),
+ result.getAlignment());
EmitNewInitializer(*this, E, allocType, elementTy, result, numElements,
allocSizeWithoutCookie);
@@ -1361,7 +1447,7 @@
// allocating an array of arrays, we'll need to cast back to the
// array pointer type.
llvm::Type *resultType = ConvertTypeForMem(E->getType());
- if (result->getType() != resultType)
+ if (result.getType() != resultType)
result = Builder.CreateBitCast(result, resultType);
}
@@ -1372,21 +1458,22 @@
cleanupDominator->eraseFromParent();
}
+ llvm::Value *resultPtr = result.getPointer();
if (nullCheck) {
conditional.end(*this);
llvm::BasicBlock *notNullBB = Builder.GetInsertBlock();
EmitBlock(contBB);
- llvm::PHINode *PHI = Builder.CreatePHI(result->getType(), 2);
- PHI->addIncoming(result, notNullBB);
- PHI->addIncoming(llvm::Constant::getNullValue(result->getType()),
+ llvm::PHINode *PHI = Builder.CreatePHI(resultPtr->getType(), 2);
+ PHI->addIncoming(resultPtr, notNullBB);
+ PHI->addIncoming(llvm::Constant::getNullValue(resultPtr->getType()),
nullCheckBB);
- result = PHI;
+ resultPtr = PHI;
}
- return result;
+ return resultPtr;
}
void CodeGenFunction::EmitDeleteCall(const FunctionDecl *DeleteFD,
@@ -1422,7 +1509,7 @@
namespace {
/// Calls the given 'operator delete' on a single object.
- struct CallObjectDelete : EHScopeStack::Cleanup {
+ struct CallObjectDelete final : EHScopeStack::Cleanup {
llvm::Value *Ptr;
const FunctionDecl *OperatorDelete;
QualType ElementType;
@@ -1449,7 +1536,7 @@
/// Emit the code for deleting a single object.
static void EmitObjectDelete(CodeGenFunction &CGF,
const CXXDeleteExpr *DE,
- llvm::Value *Ptr,
+ Address Ptr,
QualType ElementType) {
// Find the destructor for the type, if applicable. If the
// destructor is virtual, we'll just emit the vcall and return.
@@ -1472,29 +1559,24 @@
// to pop it off in a second.
const FunctionDecl *OperatorDelete = DE->getOperatorDelete();
CGF.EHStack.pushCleanup<CallObjectDelete>(NormalAndEHCleanup,
- Ptr, OperatorDelete, ElementType);
+ Ptr.getPointer(),
+ OperatorDelete, ElementType);
if (Dtor)
CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete,
/*ForVirtualBase=*/false,
/*Delegating=*/false,
Ptr);
- else if (CGF.getLangOpts().ObjCAutoRefCount &&
- ElementType->isObjCLifetimeType()) {
- switch (ElementType.getObjCLifetime()) {
+ else if (auto Lifetime = ElementType.getObjCLifetime()) {
+ switch (Lifetime) {
case Qualifiers::OCL_None:
case Qualifiers::OCL_ExplicitNone:
case Qualifiers::OCL_Autoreleasing:
break;
- case Qualifiers::OCL_Strong: {
- // Load the pointer value.
- llvm::Value *PtrValue = CGF.Builder.CreateLoad(Ptr,
- ElementType.isVolatileQualified());
-
- CGF.EmitARCRelease(PtrValue, ARCPreciseLifetime);
+ case Qualifiers::OCL_Strong:
+ CGF.EmitARCDestroyStrong(Ptr, ARCPreciseLifetime);
break;
- }
case Qualifiers::OCL_Weak:
CGF.EmitARCDestroyWeak(Ptr);
@@ -1507,7 +1589,7 @@
namespace {
/// Calls the given 'operator delete' on an array of objects.
- struct CallArrayDelete : EHScopeStack::Cleanup {
+ struct CallArrayDelete final : EHScopeStack::Cleanup {
llvm::Value *Ptr;
const FunctionDecl *OperatorDelete;
llvm::Value *NumElements;
@@ -1547,7 +1629,8 @@
// The size of an element, multiplied by the number of elements.
llvm::Value *Size
= llvm::ConstantInt::get(SizeTy, ElementTypeSize.getQuantity());
- Size = CGF.Builder.CreateMul(Size, NumElements);
+ if (NumElements)
+ Size = CGF.Builder.CreateMul(Size, NumElements);
// Plus the size of the cookie if applicable.
if (!CookieSize.isZero()) {
@@ -1568,7 +1651,7 @@
/// Emit the code for deleting an array of objects.
static void EmitArrayDelete(CodeGenFunction &CGF,
const CXXDeleteExpr *E,
- llvm::Value *deletedPtr,
+ Address deletedPtr,
QualType elementType) {
llvm::Value *numElements = nullptr;
llvm::Value *allocatedPtr = nullptr;
@@ -1589,13 +1672,18 @@
if (QualType::DestructionKind dtorKind = elementType.isDestructedType()) {
assert(numElements && "no element count for a type with a destructor!");
+ CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType);
+ CharUnits elementAlign =
+ deletedPtr.getAlignment().alignmentOfArrayElement(elementSize);
+
+ llvm::Value *arrayBegin = deletedPtr.getPointer();
llvm::Value *arrayEnd =
- CGF.Builder.CreateInBoundsGEP(deletedPtr, numElements, "delete.end");
+ CGF.Builder.CreateInBoundsGEP(arrayBegin, numElements, "delete.end");
// Note that it is legal to allocate a zero-length array, and we
// can never fold the check away because the length should always
// come from a cookie.
- CGF.emitArrayDestroy(deletedPtr, arrayEnd, elementType,
+ CGF.emitArrayDestroy(arrayBegin, arrayEnd, elementType, elementAlign,
CGF.getDestroyer(dtorKind),
/*checkZeroLength*/ true,
CGF.needsEHCleanup(dtorKind));
@@ -1607,13 +1695,13 @@
void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) {
const Expr *Arg = E->getArgument();
- llvm::Value *Ptr = EmitScalarExpr(Arg);
+ Address Ptr = EmitPointerWithAlignment(Arg);
// Null check the pointer.
llvm::BasicBlock *DeleteNotNull = createBasicBlock("delete.notnull");
llvm::BasicBlock *DeleteEnd = createBasicBlock("delete.end");
- llvm::Value *IsNull = Builder.CreateIsNull(Ptr, "isnull");
+ llvm::Value *IsNull = Builder.CreateIsNull(Ptr.getPointer(), "isnull");
Builder.CreateCondBr(IsNull, DeleteEnd, DeleteNotNull);
EmitBlock(DeleteNotNull);
@@ -1638,11 +1726,11 @@
GEP.push_back(Zero);
}
- Ptr = Builder.CreateInBoundsGEP(Ptr, GEP, "del.first");
+ Ptr = Address(Builder.CreateInBoundsGEP(Ptr.getPointer(), GEP, "del.first"),
+ Ptr.getAlignment());
}
- assert(ConvertTypeForMem(DeleteTy) ==
- cast<llvm::PointerType>(Ptr->getType())->getElementType());
+ assert(ConvertTypeForMem(DeleteTy) == Ptr.getElementType());
if (E->isArrayForm()) {
EmitArrayDelete(*this, E, Ptr, DeleteTy);
@@ -1688,7 +1776,7 @@
static llvm::Value *EmitTypeidFromVTable(CodeGenFunction &CGF, const Expr *E,
llvm::Type *StdTypeInfoPtrTy) {
// Get the vtable pointer.
- llvm::Value *ThisPtr = CGF.EmitLValue(E).getAddress();
+ Address ThisPtr = CGF.EmitLValue(E).getAddress();
// C++ [expr.typeid]p2:
// If the glvalue expression is obtained by applying the unary * operator to
@@ -1705,7 +1793,7 @@
CGF.createBasicBlock("typeid.bad_typeid");
llvm::BasicBlock *EndBlock = CGF.createBasicBlock("typeid.end");
- llvm::Value *IsNull = CGF.Builder.CreateIsNull(ThisPtr);
+ llvm::Value *IsNull = CGF.Builder.CreateIsNull(ThisPtr.getPointer());
CGF.Builder.CreateCondBr(IsNull, BadTypeidBlock, EndBlock);
CGF.EmitBlock(BadTypeidBlock);
@@ -1756,8 +1844,9 @@
return llvm::UndefValue::get(DestLTy);
}
-llvm::Value *CodeGenFunction::EmitDynamicCast(llvm::Value *Value,
+llvm::Value *CodeGenFunction::EmitDynamicCast(Address ThisAddr,
const CXXDynamicCastExpr *DCE) {
+ CGM.EmitExplicitCastExprType(DCE, this);
QualType DestTy = DCE->getTypeAsWritten();
if (DCE->isAlwaysNull())
@@ -1801,19 +1890,21 @@
CastNull = createBasicBlock("dynamic_cast.null");
CastNotNull = createBasicBlock("dynamic_cast.notnull");
- llvm::Value *IsNull = Builder.CreateIsNull(Value);
+ llvm::Value *IsNull = Builder.CreateIsNull(ThisAddr.getPointer());
Builder.CreateCondBr(IsNull, CastNull, CastNotNull);
EmitBlock(CastNotNull);
}
+ llvm::Value *Value;
if (isDynamicCastToVoid) {
- Value = CGM.getCXXABI().EmitDynamicCastToVoid(*this, Value, SrcRecordTy,
+ Value = CGM.getCXXABI().EmitDynamicCastToVoid(*this, ThisAddr, SrcRecordTy,
DestTy);
} else {
assert(DestRecordTy->isRecordType() &&
"destination type must be a record type!");
- Value = CGM.getCXXABI().EmitDynamicCastCall(*this, Value, SrcRecordTy,
+ Value = CGM.getCXXABI().EmitDynamicCastCall(*this, ThisAddr, SrcRecordTy,
DestTy, DestRecordTy, CastEnd);
+ CastNotNull = Builder.GetInsertBlock();
}
if (ShouldNullCheckSrcValue) {
@@ -1838,12 +1929,11 @@
void CodeGenFunction::EmitLambdaExpr(const LambdaExpr *E, AggValueSlot Slot) {
RunCleanupsScope Scope(*this);
- LValue SlotLV =
- MakeAddrLValue(Slot.getAddr(), E->getType(), Slot.getAlignment());
+ LValue SlotLV = MakeAddrLValue(Slot.getAddress(), E->getType());
CXXRecordDecl::field_iterator CurField = E->getLambdaClass()->field_begin();
- for (LambdaExpr::capture_init_iterator i = E->capture_init_begin(),
- e = E->capture_init_end();
+ for (LambdaExpr::const_capture_init_iterator i = E->capture_init_begin(),
+ e = E->capture_init_end();
i != e; ++i, ++CurField) {
// Emit initialization
LValue LV = EmitLValueForFieldInitialization(SlotLV, *CurField);
diff --git a/lib/CodeGen/CGExprComplex.cpp b/lib/CodeGen/CGExprComplex.cpp
index 27d1c68..ccdb532 100644
--- a/lib/CodeGen/CGExprComplex.cpp
+++ b/lib/CodeGen/CGExprComplex.cpp
@@ -83,12 +83,12 @@
/// specified value pointer.
void EmitStoreOfComplex(ComplexPairTy Val, LValue LV, bool isInit);
- /// EmitComplexToComplexCast - Emit a cast from complex value Val to DestType.
+ /// Emit a cast from complex value Val to DestType.
ComplexPairTy EmitComplexToComplexCast(ComplexPairTy Val, QualType SrcType,
- QualType DestType);
- /// EmitComplexToComplexCast - Emit a cast from scalar value Val to DestType.
+ QualType DestType, SourceLocation Loc);
+ /// Emit a cast from scalar value Val to DestType.
ComplexPairTy EmitScalarToComplexCast(llvm::Value *Val, QualType SrcType,
- QualType DestType);
+ QualType DestType, SourceLocation Loc);
//===--------------------------------------------------------------------===//
// Visitor Methods
@@ -154,6 +154,8 @@
return EmitCast(E->getCastKind(), E->getSubExpr(), E->getType());
}
ComplexPairTy VisitCastExpr(CastExpr *E) {
+ if (const auto *ECE = dyn_cast<ExplicitCastExpr>(E))
+ CGF.CGM.EmitExplicitCastExprType(ECE, &CGF);
return EmitCast(E->getCastKind(), E->getSubExpr(), E->getType());
}
ComplexPairTy VisitCallExpr(const CallExpr *E);
@@ -298,6 +300,19 @@
// Utilities
//===----------------------------------------------------------------------===//
+Address CodeGenFunction::emitAddrOfRealComponent(Address addr,
+ QualType complexType) {
+ CharUnits offset = CharUnits::Zero();
+ return Builder.CreateStructGEP(addr, 0, offset, addr.getName() + ".realp");
+}
+
+Address CodeGenFunction::emitAddrOfImagComponent(Address addr,
+ QualType complexType) {
+ QualType eltType = complexType->castAs<ComplexType>()->getElementType();
+ CharUnits offset = getContext().getTypeSizeInChars(eltType);
+ return Builder.CreateStructGEP(addr, 1, offset, addr.getName() + ".imagp");
+}
+
/// EmitLoadOfLValue - Given an RValue reference for a complex, emit code to
/// load the real and imaginary pieces, returning them as Real/Imag.
ComplexPairTy ComplexExprEmitter::EmitLoadOfLValue(LValue lvalue,
@@ -306,29 +321,21 @@
if (lvalue.getType()->isAtomicType())
return CGF.EmitAtomicLoad(lvalue, loc).getComplexVal();
- llvm::Value *SrcPtr = lvalue.getAddress();
+ Address SrcPtr = lvalue.getAddress();
bool isVolatile = lvalue.isVolatileQualified();
- unsigned AlignR = lvalue.getAlignment().getQuantity();
- ASTContext &C = CGF.getContext();
- QualType ComplexTy = lvalue.getType();
- unsigned ComplexAlign = C.getTypeAlignInChars(ComplexTy).getQuantity();
- unsigned AlignI = std::min(AlignR, ComplexAlign);
- llvm::Value *Real=nullptr, *Imag=nullptr;
+ llvm::Value *Real = nullptr, *Imag = nullptr;
if (!IgnoreReal || isVolatile) {
- llvm::Value *RealP = Builder.CreateStructGEP(nullptr, SrcPtr, 0,
- SrcPtr->getName() + ".realp");
- Real = Builder.CreateAlignedLoad(RealP, AlignR, isVolatile,
- SrcPtr->getName() + ".real");
+ Address RealP = CGF.emitAddrOfRealComponent(SrcPtr, lvalue.getType());
+ Real = Builder.CreateLoad(RealP, isVolatile, SrcPtr.getName() + ".real");
}
if (!IgnoreImag || isVolatile) {
- llvm::Value *ImagP = Builder.CreateStructGEP(nullptr, SrcPtr, 1,
- SrcPtr->getName() + ".imagp");
- Imag = Builder.CreateAlignedLoad(ImagP, AlignI, isVolatile,
- SrcPtr->getName() + ".imag");
+ Address ImagP = CGF.emitAddrOfImagComponent(SrcPtr, lvalue.getType());
+ Imag = Builder.CreateLoad(ImagP, isVolatile, SrcPtr.getName() + ".imag");
}
+
return ComplexPairTy(Real, Imag);
}
@@ -340,19 +347,12 @@
(!isInit && CGF.LValueIsSuitableForInlineAtomic(lvalue)))
return CGF.EmitAtomicStore(RValue::getComplex(Val), lvalue, isInit);
- llvm::Value *Ptr = lvalue.getAddress();
- llvm::Value *RealPtr = Builder.CreateStructGEP(nullptr, Ptr, 0, "real");
- llvm::Value *ImagPtr = Builder.CreateStructGEP(nullptr, Ptr, 1, "imag");
- unsigned AlignR = lvalue.getAlignment().getQuantity();
- ASTContext &C = CGF.getContext();
- QualType ComplexTy = lvalue.getType();
- unsigned ComplexAlign = C.getTypeAlignInChars(ComplexTy).getQuantity();
- unsigned AlignI = std::min(AlignR, ComplexAlign);
+ Address Ptr = lvalue.getAddress();
+ Address RealPtr = CGF.emitAddrOfRealComponent(Ptr, lvalue.getType());
+ Address ImagPtr = CGF.emitAddrOfImagComponent(Ptr, lvalue.getType());
- Builder.CreateAlignedStore(Val.first, RealPtr, AlignR,
- lvalue.isVolatileQualified());
- Builder.CreateAlignedStore(Val.second, ImagPtr, AlignI,
- lvalue.isVolatileQualified());
+ Builder.CreateStore(Val.first, RealPtr, lvalue.isVolatileQualified());
+ Builder.CreateStore(Val.second, ImagPtr, lvalue.isVolatileQualified());
}
@@ -385,16 +385,17 @@
ComplexPairTy ComplexExprEmitter::VisitStmtExpr(const StmtExpr *E) {
CodeGenFunction::StmtExprEvaluation eval(CGF);
- llvm::Value *RetAlloca = CGF.EmitCompoundStmt(*E->getSubStmt(), true);
- assert(RetAlloca && "Expected complex return value");
+ Address RetAlloca = CGF.EmitCompoundStmt(*E->getSubStmt(), true);
+ assert(RetAlloca.isValid() && "Expected complex return value");
return EmitLoadOfLValue(CGF.MakeAddrLValue(RetAlloca, E->getType()),
E->getExprLoc());
}
-/// EmitComplexToComplexCast - Emit a cast from complex value Val to DestType.
+/// Emit a cast from complex value Val to DestType.
ComplexPairTy ComplexExprEmitter::EmitComplexToComplexCast(ComplexPairTy Val,
QualType SrcType,
- QualType DestType) {
+ QualType DestType,
+ SourceLocation Loc) {
// Get the src/dest element type.
SrcType = SrcType->castAs<ComplexType>()->getElementType();
DestType = DestType->castAs<ComplexType>()->getElementType();
@@ -402,17 +403,18 @@
// C99 6.3.1.6: When a value of complex type is converted to another
// complex type, both the real and imaginary parts follow the conversion
// rules for the corresponding real types.
- Val.first = CGF.EmitScalarConversion(Val.first, SrcType, DestType);
- Val.second = CGF.EmitScalarConversion(Val.second, SrcType, DestType);
+ Val.first = CGF.EmitScalarConversion(Val.first, SrcType, DestType, Loc);
+ Val.second = CGF.EmitScalarConversion(Val.second, SrcType, DestType, Loc);
return Val;
}
ComplexPairTy ComplexExprEmitter::EmitScalarToComplexCast(llvm::Value *Val,
QualType SrcType,
- QualType DestType) {
+ QualType DestType,
+ SourceLocation Loc) {
// Convert the input element to the element type of the complex.
DestType = DestType->castAs<ComplexType>()->getElementType();
- Val = CGF.EmitScalarConversion(Val, SrcType, DestType);
+ Val = CGF.EmitScalarConversion(Val, SrcType, DestType, Loc);
// Return (realval, 0).
return ComplexPairTy(Val, llvm::Constant::getNullValue(Val->getType()));
@@ -434,12 +436,9 @@
case CK_LValueBitCast: {
LValue origLV = CGF.EmitLValue(Op);
- llvm::Value *V = origLV.getAddress();
- V = Builder.CreateBitCast(V,
- CGF.ConvertType(CGF.getContext().getPointerType(DestTy)));
- return EmitLoadOfLValue(CGF.MakeAddrLValue(V, DestTy,
- origLV.getAlignment()),
- Op->getExprLoc());
+ Address V = origLV.getAddress();
+ V = Builder.CreateElementBitCast(V, CGF.ConvertType(DestTy));
+ return EmitLoadOfLValue(CGF.MakeAddrLValue(V, DestTy), Op->getExprLoc());
}
case CK_BitCast:
@@ -488,14 +487,15 @@
case CK_FloatingRealToComplex:
case CK_IntegralRealToComplex:
- return EmitScalarToComplexCast(CGF.EmitScalarExpr(Op),
- Op->getType(), DestTy);
+ return EmitScalarToComplexCast(CGF.EmitScalarExpr(Op), Op->getType(),
+ DestTy, Op->getExprLoc());
case CK_FloatingComplexCast:
case CK_FloatingComplexToIntegralComplex:
case CK_IntegralComplexCast:
case CK_IntegralComplexToFloatingComplex:
- return EmitComplexToComplexCast(Visit(Op), Op->getType(), DestTy);
+ return EmitComplexToComplexCast(Visit(Op), Op->getType(), DestTy,
+ Op->getExprLoc());
}
llvm_unreachable("unknown cast resulting in complex value");
@@ -585,19 +585,25 @@
// We *must* use the full CG function call building logic here because the
// complex type has special ABI handling. We also should not forget about
// special calling convention which may be used for compiler builtins.
- const CGFunctionInfo &FuncInfo =
- CGF.CGM.getTypes().arrangeFreeFunctionCall(
- Op.Ty, Args, FunctionType::ExtInfo(/* No CC here - will be added later */),
- RequiredArgs::All);
+
+ // We create a function qualified type to state that this call does not have
+ // any exceptions.
+ FunctionProtoType::ExtProtoInfo EPI;
+ EPI = EPI.withExceptionSpec(
+ FunctionProtoType::ExceptionSpecInfo(EST_BasicNoexcept));
+ SmallVector<QualType, 4> ArgsQTys(
+ 4, Op.Ty->castAs<ComplexType>()->getElementType());
+ QualType FQTy = CGF.getContext().getFunctionType(Op.Ty, ArgsQTys, EPI);
+ const CGFunctionInfo &FuncInfo = CGF.CGM.getTypes().arrangeFreeFunctionCall(
+ Args, cast<FunctionType>(FQTy.getTypePtr()), false);
+
llvm::FunctionType *FTy = CGF.CGM.getTypes().GetFunctionType(FuncInfo);
llvm::Constant *Func = CGF.CGM.CreateBuiltinFunction(FTy, LibCallName);
llvm::Instruction *Call;
RValue Res = CGF.EmitCall(FuncInfo, Func, ReturnValueSlot(), Args,
- nullptr, &Call);
+ FQTy->getAs<FunctionProtoType>(), &Call);
cast<llvm::CallInst>(Call)->setCallingConv(CGF.CGM.getBuiltinCC());
- cast<llvm::CallInst>(Call)->setDoesNotThrow();
-
return Res.getComplexVal();
}
@@ -846,19 +852,20 @@
LValue LHS = CGF.EmitLValue(E->getLHS());
// Load from the l-value and convert it.
+ SourceLocation Loc = E->getExprLoc();
if (LHSTy->isAnyComplexType()) {
- ComplexPairTy LHSVal = EmitLoadOfLValue(LHS, E->getExprLoc());
- OpInfo.LHS = EmitComplexToComplexCast(LHSVal, LHSTy, OpInfo.Ty);
+ ComplexPairTy LHSVal = EmitLoadOfLValue(LHS, Loc);
+ OpInfo.LHS = EmitComplexToComplexCast(LHSVal, LHSTy, OpInfo.Ty, Loc);
} else {
- llvm::Value *LHSVal = CGF.EmitLoadOfScalar(LHS, E->getExprLoc());
+ llvm::Value *LHSVal = CGF.EmitLoadOfScalar(LHS, Loc);
// For floating point real operands we can directly pass the scalar form
// to the binary operator emission and potentially get more efficient code.
if (LHSTy->isRealFloatingType()) {
if (!CGF.getContext().hasSameUnqualifiedType(ComplexElementTy, LHSTy))
- LHSVal = CGF.EmitScalarConversion(LHSVal, LHSTy, ComplexElementTy);
+ LHSVal = CGF.EmitScalarConversion(LHSVal, LHSTy, ComplexElementTy, Loc);
OpInfo.LHS = ComplexPairTy(LHSVal, nullptr);
} else {
- OpInfo.LHS = EmitScalarToComplexCast(LHSVal, LHSTy, OpInfo.Ty);
+ OpInfo.LHS = EmitScalarToComplexCast(LHSVal, LHSTy, OpInfo.Ty, Loc);
}
}
@@ -867,12 +874,13 @@
// Truncate the result and store it into the LHS lvalue.
if (LHSTy->isAnyComplexType()) {
- ComplexPairTy ResVal = EmitComplexToComplexCast(Result, OpInfo.Ty, LHSTy);
+ ComplexPairTy ResVal =
+ EmitComplexToComplexCast(Result, OpInfo.Ty, LHSTy, Loc);
EmitStoreOfComplex(ResVal, LHS, /*isInit*/ false);
Val = RValue::getComplex(ResVal);
} else {
llvm::Value *ResVal =
- CGF.EmitComplexToScalarConversion(Result, OpInfo.Ty, LHSTy);
+ CGF.EmitComplexToScalarConversion(Result, OpInfo.Ty, LHSTy, Loc);
CGF.EmitStoreOfScalar(ResVal, LHS, /*isInit*/ false);
Val = RValue::get(ResVal);
}
@@ -1011,10 +1019,10 @@
}
ComplexPairTy ComplexExprEmitter::VisitVAArgExpr(VAArgExpr *E) {
- llvm::Value *ArgValue = CGF.EmitVAListRef(E->getSubExpr());
- llvm::Value *ArgPtr = CGF.EmitVAArg(ArgValue, E->getType());
+ Address ArgValue = Address::invalid();
+ Address ArgPtr = CGF.EmitVAArg(E, ArgValue);
- if (!ArgPtr) {
+ if (!ArgPtr.isValid()) {
CGF.ErrorUnsupported(E, "complex va_arg expression");
llvm::Type *EltTy =
CGF.ConvertType(E->getType()->castAs<ComplexType>()->getElementType());
@@ -1022,7 +1030,7 @@
return ComplexPairTy(U, U);
}
- return EmitLoadOfLValue(CGF.MakeNaturalAlignAddrLValue(ArgPtr, E->getType()),
+ return EmitLoadOfLValue(CGF.MakeAddrLValue(ArgPtr, E->getType()),
E->getExprLoc());
}
diff --git a/lib/CodeGen/CGExprConstant.cpp b/lib/CodeGen/CGExprConstant.cpp
index acfb9b6..3839ab7 100644
--- a/lib/CodeGen/CGExprConstant.cpp
+++ b/lib/CodeGen/CGExprConstant.cpp
@@ -636,6 +636,8 @@
}
llvm::Constant *VisitCastExpr(CastExpr* E) {
+ if (const auto *ECE = dyn_cast<ExplicitCastExpr>(E))
+ CGM.EmitExplicitCastExprType(ECE, CGF);
Expr *subExpr = E->getSubExpr();
llvm::Constant *C = CGM.EmitConstantExpr(subExpr, subExpr->getType(), CGF);
if (!C) return nullptr;
@@ -977,23 +979,26 @@
}
public:
- llvm::Constant *EmitLValue(APValue::LValueBase LVBase) {
+ ConstantAddress EmitLValue(APValue::LValueBase LVBase) {
if (const ValueDecl *Decl = LVBase.dyn_cast<const ValueDecl*>()) {
if (Decl->hasAttr<WeakRefAttr>())
return CGM.GetWeakRefReference(Decl);
if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(Decl))
- return CGM.GetAddrOfFunction(FD);
+ return ConstantAddress(CGM.GetAddrOfFunction(FD), CharUnits::One());
if (const VarDecl* VD = dyn_cast<VarDecl>(Decl)) {
// We can never refer to a variable with local storage.
if (!VD->hasLocalStorage()) {
+ CharUnits Align = CGM.getContext().getDeclAlign(VD);
if (VD->isFileVarDecl() || VD->hasExternalStorage())
- return CGM.GetAddrOfGlobalVar(VD);
- else if (VD->isLocalVarDecl())
- return CGM.getOrCreateStaticVarDecl(
+ return ConstantAddress(CGM.GetAddrOfGlobalVar(VD), Align);
+ else if (VD->isLocalVarDecl()) {
+ auto Ptr = CGM.getOrCreateStaticVarDecl(
*VD, CGM.getLLVMLinkageVarDefinition(VD, /*isConstant=*/false));
+ return ConstantAddress(Ptr, Align);
+ }
}
}
- return nullptr;
+ return ConstantAddress::invalid();
}
Expr *E = const_cast<Expr*>(LVBase.get<const Expr*>());
@@ -1006,14 +1011,18 @@
llvm::Constant* C = CGM.EmitConstantExpr(CLE->getInitializer(),
CLE->getType(), CGF);
// FIXME: "Leaked" on failure.
- if (C)
- C = new llvm::GlobalVariable(CGM.getModule(), C->getType(),
+ if (!C) return ConstantAddress::invalid();
+
+ CharUnits Align = CGM.getContext().getTypeAlignInChars(E->getType());
+
+ auto GV = new llvm::GlobalVariable(CGM.getModule(), C->getType(),
E->getType().isConstant(CGM.getContext()),
llvm::GlobalValue::InternalLinkage,
C, ".compoundliteral", nullptr,
llvm::GlobalVariable::NotThreadLocal,
CGM.getContext().getTargetAddressSpace(E->getType()));
- return C;
+ GV->setAlignment(Align.getQuantity());
+ return ConstantAddress(GV, Align);
}
case Expr::StringLiteralClass:
return CGM.GetAddrOfConstantStringFromLiteral(cast<StringLiteral>(E));
@@ -1021,15 +1030,15 @@
return CGM.GetAddrOfConstantStringFromObjCEncode(cast<ObjCEncodeExpr>(E));
case Expr::ObjCStringLiteralClass: {
ObjCStringLiteral* SL = cast<ObjCStringLiteral>(E);
- llvm::Constant *C =
+ ConstantAddress C =
CGM.getObjCRuntime().GenerateConstantString(SL->getString());
- return llvm::ConstantExpr::getBitCast(C, ConvertType(E->getType()));
+ return C.getElementBitCast(ConvertType(E->getType()));
}
case Expr::PredefinedExprClass: {
unsigned Type = cast<PredefinedExpr>(E)->getIdentType();
if (CGF) {
LValue Res = CGF->EmitPredefinedLValue(cast<PredefinedExpr>(E));
- return cast<llvm::Constant>(Res.getAddress());
+ return cast<ConstantAddress>(Res.getAddress());
} else if (Type == PredefinedExpr::PrettyFunction) {
return CGM.GetAddrOfConstantCString("top level", ".tmp");
}
@@ -1040,7 +1049,8 @@
assert(CGF && "Invalid address of label expression outside function.");
llvm::Constant *Ptr =
CGF->GetAddrOfLabel(cast<AddrLabelExpr>(E)->getLabel());
- return llvm::ConstantExpr::getBitCast(Ptr, ConvertType(E->getType()));
+ Ptr = llvm::ConstantExpr::getBitCast(Ptr, ConvertType(E->getType()));
+ return ConstantAddress(Ptr, CharUnits::One());
}
case Expr::CallExprClass: {
CallExpr* CE = cast<CallExpr>(E);
@@ -1066,7 +1076,10 @@
else
FunctionName = "global";
- return CGM.GetAddrOfGlobalBlock(cast<BlockExpr>(E), FunctionName.c_str());
+ // This is not really an l-value.
+ llvm::Constant *Ptr =
+ CGM.GetAddrOfGlobalBlock(cast<BlockExpr>(E), FunctionName.c_str());
+ return ConstantAddress(Ptr, CGM.getPointerAlign());
}
case Expr::CXXTypeidExprClass: {
CXXTypeidExpr *Typeid = cast<CXXTypeidExpr>(E);
@@ -1075,7 +1088,8 @@
T = Typeid->getTypeOperand(CGM.getContext());
else
T = Typeid->getExprOperand()->getType();
- return CGM.GetAddrOfRTTIDescriptor(T);
+ return ConstantAddress(CGM.GetAddrOfRTTIDescriptor(T),
+ CGM.getPointerAlign());
}
case Expr::CXXUuidofExprClass: {
return CGM.GetAddrOfUuidDescriptor(cast<CXXUuidofExpr>(E));
@@ -1091,7 +1105,7 @@
}
}
- return nullptr;
+ return ConstantAddress::invalid();
}
};
@@ -1255,7 +1269,7 @@
llvm::Constant *Offset =
llvm::ConstantInt::get(Int64Ty, Value.getLValueOffset().getQuantity());
- llvm::Constant *C;
+ llvm::Constant *C = nullptr;
if (APValue::LValueBase LVBase = Value.getLValueBase()) {
// An array can be represented as an lvalue referring to the base.
if (isa<llvm::ArrayType>(DestTy)) {
@@ -1264,7 +1278,7 @@
const_cast<Expr*>(LVBase.get<const Expr*>()));
}
- C = ConstExprEmitter(*this, CGF).EmitLValue(LVBase);
+ C = ConstExprEmitter(*this, CGF).EmitLValue(LVBase).getPointer();
// Apply offset if necessary.
if (!Offset->isNullValue()) {
@@ -1336,15 +1350,17 @@
return llvm::ConstantStruct::get(STy, Complex);
}
case APValue::Vector: {
- SmallVector<llvm::Constant *, 4> Inits;
unsigned NumElts = Value.getVectorLength();
+ SmallVector<llvm::Constant *, 4> Inits(NumElts);
- for (unsigned i = 0; i != NumElts; ++i) {
- const APValue &Elt = Value.getVectorElt(i);
+ for (unsigned I = 0; I != NumElts; ++I) {
+ const APValue &Elt = Value.getVectorElt(I);
if (Elt.isInt())
- Inits.push_back(llvm::ConstantInt::get(VMContext, Elt.getInt()));
+ Inits[I] = llvm::ConstantInt::get(VMContext, Elt.getInt());
+ else if (Elt.isFloat())
+ Inits[I] = llvm::ConstantFP::get(VMContext, Elt.getFloat());
else
- Inits.push_back(llvm::ConstantFP::get(VMContext, Elt.getFloat()));
+ llvm_unreachable("unsupported vector element type");
}
return llvm::ConstantVector::get(Inits);
}
@@ -1438,7 +1454,7 @@
return C;
}
-llvm::Constant *
+ConstantAddress
CodeGenModule::GetAddrOfConstantCompoundLiteral(const CompoundLiteralExpr *E) {
assert(E->isFileScope() && "not a file-scope compound literal expr");
return ConstExprEmitter(*this, nullptr).EmitLValue(E);
@@ -1452,7 +1468,7 @@
// A member function pointer.
if (const CXXMethodDecl *method = dyn_cast<CXXMethodDecl>(decl))
- return getCXXABI().EmitMemberPointer(method);
+ return getCXXABI().EmitMemberFunctionPointer(method);
// Otherwise, a member data pointer.
uint64_t fieldOffset = getContext().getFieldOffset(decl);
diff --git a/lib/CodeGen/CGExprScalar.cpp b/lib/CodeGen/CGExprScalar.cpp
index 08c81c0..4429038 100644
--- a/lib/CodeGen/CGExprScalar.cpp
+++ b/lib/CodeGen/CGExprScalar.cpp
@@ -16,6 +16,7 @@
#include "CGDebugInfo.h"
#include "CGObjCRuntime.h"
#include "CodeGenModule.h"
+#include "TargetInfo.h"
#include "clang/AST/ASTContext.h"
#include "clang/AST/DeclObjC.h"
#include "clang/AST/RecordLayout.h"
@@ -139,21 +140,25 @@
/// boolean (i1) truth value. This is equivalent to "Val != 0".
Value *EmitConversionToBool(Value *Src, QualType DstTy);
- /// \brief Emit a check that a conversion to or from a floating-point type
- /// does not overflow.
+ /// Emit a check that a conversion to or from a floating-point type does not
+ /// overflow.
void EmitFloatConversionCheck(Value *OrigSrc, QualType OrigSrcType,
- Value *Src, QualType SrcType,
- QualType DstType, llvm::Type *DstTy);
+ Value *Src, QualType SrcType, QualType DstType,
+ llvm::Type *DstTy, SourceLocation Loc);
- /// EmitScalarConversion - Emit a conversion from the specified type to the
- /// specified destination type, both of which are LLVM scalar types.
- Value *EmitScalarConversion(Value *Src, QualType SrcTy, QualType DstTy);
+ /// Emit a conversion from the specified type to the specified destination
+ /// type, both of which are LLVM scalar types.
+ Value *EmitScalarConversion(Value *Src, QualType SrcTy, QualType DstTy,
+ SourceLocation Loc);
- /// EmitComplexToScalarConversion - Emit a conversion from the specified
- /// complex type to the specified destination type, where the destination type
- /// is an LLVM scalar type.
+ Value *EmitScalarConversion(Value *Src, QualType SrcTy, QualType DstTy,
+ SourceLocation Loc, bool TreatBooleanAsSigned);
+
+ /// Emit a conversion from the specified complex type to the specified
+ /// destination type, where the destination type is an LLVM scalar type.
Value *EmitComplexToScalarConversion(CodeGenFunction::ComplexPairTy Src,
- QualType SrcTy, QualType DstTy);
+ QualType SrcTy, QualType DstTy,
+ SourceLocation Loc);
/// EmitNullValue - Emit a value that corresponds to null for the given type.
Value *EmitNullValue(QualType Ty);
@@ -309,12 +314,7 @@
return EmitNullValue(E->getType());
}
Value *VisitExplicitCastExpr(ExplicitCastExpr *E) {
- if (E->getType()->isVariablyModifiedType())
- CGF.EmitVariablyModifiedType(E->getType());
-
- if (CGDebugInfo *DI = CGF.getDebugInfo())
- DI->EmitExplicitCastType(E->getType());
-
+ CGF.CGM.EmitExplicitCastExprType(E, &CGF);
return VisitCastExpr(E);
}
Value *VisitCastExpr(CastExpr *E);
@@ -361,7 +361,7 @@
if (isa<MemberPointerType>(E->getType())) // never sugared
return CGF.CGM.getMemberPointerConstant(E);
- return EmitLValue(E->getSubExpr()).getAddress();
+ return EmitLValue(E->getSubExpr()).getPointer();
}
Value *VisitUnaryDeref(const UnaryOperator *E) {
if (E->getType()->isVoidType())
@@ -523,8 +523,9 @@
#undef HANDLEBINOP
// Comparisons.
- Value *EmitCompare(const BinaryOperator *E, unsigned UICmpOpc,
- unsigned SICmpOpc, unsigned FCmpOpc);
+ Value *EmitCompare(const BinaryOperator *E, llvm::CmpInst::Predicate UICmpOpc,
+ llvm::CmpInst::Predicate SICmpOpc,
+ llvm::CmpInst::Predicate FCmpOpc);
#define VISITCOMP(CODE, UI, SI, FP) \
Value *VisitBin##CODE(const BinaryOperator *E) { \
return EmitCompare(E, llvm::ICmpInst::UI, llvm::ICmpInst::SI, \
@@ -593,11 +594,9 @@
return EmitPointerToBoolConversion(Src);
}
-void ScalarExprEmitter::EmitFloatConversionCheck(Value *OrigSrc,
- QualType OrigSrcType,
- Value *Src, QualType SrcType,
- QualType DstType,
- llvm::Type *DstTy) {
+void ScalarExprEmitter::EmitFloatConversionCheck(
+ Value *OrigSrc, QualType OrigSrcType, Value *Src, QualType SrcType,
+ QualType DstType, llvm::Type *DstTy, SourceLocation Loc) {
CodeGenFunction::SanitizerScope SanScope(&CGF);
using llvm::APFloat;
using llvm::APSInt;
@@ -721,19 +720,25 @@
}
}
- // FIXME: Provide a SourceLocation.
- llvm::Constant *StaticArgs[] = {
- CGF.EmitCheckTypeDescriptor(OrigSrcType),
- CGF.EmitCheckTypeDescriptor(DstType)
- };
+ llvm::Constant *StaticArgs[] = {CGF.EmitCheckSourceLocation(Loc),
+ CGF.EmitCheckTypeDescriptor(OrigSrcType),
+ CGF.EmitCheckTypeDescriptor(DstType)};
CGF.EmitCheck(std::make_pair(Check, SanitizerKind::FloatCastOverflow),
"float_cast_overflow", StaticArgs, OrigSrc);
}
-/// EmitScalarConversion - Emit a conversion from the specified type to the
-/// specified destination type, both of which are LLVM scalar types.
+/// Emit a conversion from the specified type to the specified destination type,
+/// both of which are LLVM scalar types.
Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType,
- QualType DstType) {
+ QualType DstType,
+ SourceLocation Loc) {
+ return EmitScalarConversion(Src, SrcType, DstType, Loc, false);
+}
+
+Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType,
+ QualType DstType,
+ SourceLocation Loc,
+ bool TreatBooleanAsSigned) {
SrcType = CGF.getContext().getCanonicalType(SrcType);
DstType = CGF.getContext().getCanonicalType(DstType);
if (SrcType == DstType) return Src;
@@ -808,7 +813,8 @@
if (DstType->isExtVectorType() && !SrcType->isVectorType()) {
// Cast the scalar to element type
QualType EltTy = DstType->getAs<ExtVectorType>()->getElementType();
- llvm::Value *Elt = EmitScalarConversion(Src, SrcType, EltTy);
+ llvm::Value *Elt = EmitScalarConversion(
+ Src, SrcType, EltTy, Loc, CGF.getContext().getLangOpts().OpenCL);
// Splat the element across to all elements
unsigned NumElements = cast<llvm::VectorType>(DstTy)->getNumElements();
@@ -828,8 +834,8 @@
// or the destination type is a floating-point type.
if (CGF.SanOpts.has(SanitizerKind::FloatCastOverflow) &&
(OrigSrcType->isFloatingType() || DstType->isFloatingType()))
- EmitFloatConversionCheck(OrigSrc, OrigSrcType, Src, SrcType, DstType,
- DstTy);
+ EmitFloatConversionCheck(OrigSrc, OrigSrcType, Src, SrcType, DstType, DstTy,
+ Loc);
// Cast to half through float if half isn't a native type.
if (DstType->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType) {
@@ -848,6 +854,9 @@
if (isa<llvm::IntegerType>(SrcTy)) {
bool InputSigned = SrcType->isSignedIntegerOrEnumerationType();
+ if (SrcType->isBooleanType() && TreatBooleanAsSigned) {
+ InputSigned = true;
+ }
if (isa<llvm::IntegerType>(DstTy))
Res = Builder.CreateIntCast(Src, DstTy, InputSigned, "conv");
else if (InputSigned)
@@ -883,20 +892,19 @@
return Res;
}
-/// EmitComplexToScalarConversion - Emit a conversion from the specified complex
-/// type to the specified destination type, where the destination type is an
-/// LLVM scalar type.
-Value *ScalarExprEmitter::
-EmitComplexToScalarConversion(CodeGenFunction::ComplexPairTy Src,
- QualType SrcTy, QualType DstTy) {
+/// Emit a conversion from the specified complex type to the specified
+/// destination type, where the destination type is an LLVM scalar type.
+Value *ScalarExprEmitter::EmitComplexToScalarConversion(
+ CodeGenFunction::ComplexPairTy Src, QualType SrcTy, QualType DstTy,
+ SourceLocation Loc) {
// Get the source element type.
SrcTy = SrcTy->castAs<ComplexType>()->getElementType();
// Handle conversions to bool first, they are special: comparisons against 0.
if (DstTy->isBooleanType()) {
// Complex != 0 -> (Real != 0) | (Imag != 0)
- Src.first = EmitScalarConversion(Src.first, SrcTy, DstTy);
- Src.second = EmitScalarConversion(Src.second, SrcTy, DstTy);
+ Src.first = EmitScalarConversion(Src.first, SrcTy, DstTy, Loc);
+ Src.second = EmitScalarConversion(Src.second, SrcTy, DstTy, Loc);
return Builder.CreateOr(Src.first, Src.second, "tobool");
}
@@ -904,7 +912,7 @@
// the imaginary part of the complex value is discarded and the value of the
// real part is converted according to the conversion rules for the
// corresponding real type.
- return EmitScalarConversion(Src.first, SrcTy, DstTy);
+ return EmitScalarConversion(Src.first, SrcTy, DstTy, Loc);
}
Value *ScalarExprEmitter::EmitNullValue(QualType Ty) {
@@ -1002,14 +1010,10 @@
}
llvm::VectorType *MTy = cast<llvm::VectorType>(Mask->getType());
- llvm::Constant* EltMask;
-
- EltMask = llvm::ConstantInt::get(MTy->getElementType(),
- llvm::NextPowerOf2(LHSElts-1)-1);
// Mask off the high bits of each shuffle index.
- Value *MaskBits = llvm::ConstantVector::getSplat(MTy->getNumElements(),
- EltMask);
+ Value *MaskBits =
+ llvm::ConstantInt::get(MTy, llvm::NextPowerOf2(LHSElts - 1) - 1);
Mask = Builder.CreateAnd(Mask, MaskBits, "mask");
// newv = undef
@@ -1165,6 +1169,16 @@
return llvm::ConstantInt::get(I32Ty, Off+MV);
}
+static llvm::Constant *getAsInt32(llvm::ConstantInt *C, llvm::Type *I32Ty) {
+ if (C->getBitWidth() != 32) {
+ assert(llvm::ConstantInt::isValueValidForType(I32Ty,
+ C->getZExtValue()) &&
+ "Index operand too large for shufflevector mask!");
+ return llvm::ConstantInt::get(I32Ty, C->getZExtValue());
+ }
+ return C;
+}
+
Value *ScalarExprEmitter::VisitInitListExpr(InitListExpr *E) {
bool Ignore = TestAndClearIgnoreResultAssign();
(void)Ignore;
@@ -1215,7 +1229,8 @@
Value *LHS = nullptr, *RHS = nullptr;
if (CurIdx == 0) {
// insert into undef -> shuffle (src, undef)
- Args.push_back(C);
+ // shufflemask must use an i32
+ Args.push_back(getAsInt32(C, CGF.Int32Ty));
Args.resize(ResElts, llvm::UndefValue::get(CGF.Int32Ty));
LHS = EI->getVectorOperand();
@@ -1322,13 +1337,13 @@
return V;
}
-static bool ShouldNullCheckClassCastValue(const CastExpr *CE) {
+bool CodeGenFunction::ShouldNullCheckClassCastValue(const CastExpr *CE) {
const Expr *E = CE->getSubExpr();
if (CE->getCastKind() == CK_UncheckedDerivedToBase)
return false;
- if (isa<CXXThisExpr>(E)) {
+ if (isa<CXXThisExpr>(E->IgnoreParens())) {
// We always assume that 'this' is never null.
return false;
}
@@ -1363,11 +1378,10 @@
case CK_LValueBitCast:
case CK_ObjCObjectLValueCast: {
- Value *V = EmitLValue(E).getAddress();
- V = Builder.CreateBitCast(V,
- ConvertType(CGF.getContext().getPointerType(DestTy)));
- return EmitLoadOfLValue(CGF.MakeNaturalAlignAddrLValue(V, DestTy),
- CE->getExprLoc());
+ Address Addr = EmitLValue(E).getAddress();
+ Addr = Builder.CreateElementBitCast(Addr, CGF.ConvertTypeForMem(DestTy));
+ LValue LV = CGF.MakeAddrLValue(Addr, DestTy);
+ return EmitLoadOfLValue(LV, CE->getExprLoc());
}
case CK_CPointerToObjCPointerCast:
@@ -1386,7 +1400,9 @@
if (CGF.SanOpts.has(SanitizerKind::CFIUnrelatedCast)) {
if (auto PT = DestTy->getAs<PointerType>())
CGF.EmitVTablePtrCheckForCast(PT->getPointeeType(), Src,
- /*MayBeNull=*/true);
+ /*MayBeNull=*/true,
+ CodeGenFunction::CFITCK_UnrelatedCast,
+ CE->getLocStart());
}
return Builder.CreateBitCast(Src, DstTy);
@@ -1405,66 +1421,44 @@
const CXXRecordDecl *DerivedClassDecl = DestTy->getPointeeCXXRecordDecl();
assert(DerivedClassDecl && "BaseToDerived arg isn't a C++ object pointer!");
- llvm::Value *V = Visit(E);
-
- llvm::Value *Derived =
- CGF.GetAddressOfDerivedClass(V, DerivedClassDecl,
+ Address Base = CGF.EmitPointerWithAlignment(E);
+ Address Derived =
+ CGF.GetAddressOfDerivedClass(Base, DerivedClassDecl,
CE->path_begin(), CE->path_end(),
- ShouldNullCheckClassCastValue(CE));
+ CGF.ShouldNullCheckClassCastValue(CE));
// C++11 [expr.static.cast]p11: Behavior is undefined if a downcast is
// performed and the object is not of the derived type.
if (CGF.sanitizePerformTypeCheck())
CGF.EmitTypeCheck(CodeGenFunction::TCK_DowncastPointer, CE->getExprLoc(),
- Derived, DestTy->getPointeeType());
+ Derived.getPointer(), DestTy->getPointeeType());
if (CGF.SanOpts.has(SanitizerKind::CFIDerivedCast))
- CGF.EmitVTablePtrCheckForCast(DestTy->getPointeeType(), Derived,
- /*MayBeNull=*/true);
+ CGF.EmitVTablePtrCheckForCast(DestTy->getPointeeType(),
+ Derived.getPointer(),
+ /*MayBeNull=*/true,
+ CodeGenFunction::CFITCK_DerivedCast,
+ CE->getLocStart());
- return Derived;
+ return Derived.getPointer();
}
case CK_UncheckedDerivedToBase:
case CK_DerivedToBase: {
- const CXXRecordDecl *DerivedClassDecl =
- E->getType()->getPointeeCXXRecordDecl();
- assert(DerivedClassDecl && "DerivedToBase arg isn't a C++ object pointer!");
-
- return CGF.GetAddressOfBaseClass(
- Visit(E), DerivedClassDecl, CE->path_begin(), CE->path_end(),
- ShouldNullCheckClassCastValue(CE), CE->getExprLoc());
+ // The EmitPointerWithAlignment path does this fine; just discard
+ // the alignment.
+ return CGF.EmitPointerWithAlignment(CE).getPointer();
}
+
case CK_Dynamic: {
- Value *V = Visit(const_cast<Expr*>(E));
+ Address V = CGF.EmitPointerWithAlignment(E);
const CXXDynamicCastExpr *DCE = cast<CXXDynamicCastExpr>(CE);
return CGF.EmitDynamicCast(V, DCE);
}
- case CK_ArrayToPointerDecay: {
- assert(E->getType()->isArrayType() &&
- "Array to pointer decay must have array source type!");
-
- Value *V = EmitLValue(E).getAddress(); // Bitfields can't be arrays.
-
- // Note that VLA pointers are always decayed, so we don't need to do
- // anything here.
- if (!E->getType()->isVariableArrayType()) {
- assert(isa<llvm::PointerType>(V->getType()) && "Expected pointer");
- llvm::Type *NewTy = ConvertType(E->getType());
- V = CGF.Builder.CreatePointerCast(
- V, NewTy->getPointerTo(V->getType()->getPointerAddressSpace()));
-
- assert(isa<llvm::ArrayType>(V->getType()->getPointerElementType()) &&
- "Expected pointer to array");
- V = Builder.CreateStructGEP(NewTy, V, 0, "arraydecay");
- }
-
- // Make sure the array decay ends up being the right type. This matters if
- // the array type was of an incomplete type.
- return CGF.Builder.CreatePointerCast(V, ConvertType(CE->getType()));
- }
+ case CK_ArrayToPointerDecay:
+ return CGF.EmitArrayToPointerDecay(E).getPointer();
case CK_FunctionToPointerDecay:
- return EmitLValue(E).getAddress();
+ return EmitLValue(E).getPointer();
case CK_NullToPointer:
if (MustVisitNullValue(E))
@@ -1547,9 +1541,14 @@
}
case CK_VectorSplat: {
llvm::Type *DstTy = ConvertType(DestTy);
- Value *Elt = Visit(const_cast<Expr*>(E));
- Elt = EmitScalarConversion(Elt, E->getType(),
- DestTy->getAs<VectorType>()->getElementType());
+ // Need an IgnoreImpCasts here as by default a boolean will be promoted to
+ // an int, which will not perform the sign extension, so if we know we are
+ // going to cast to a vector we have to strip the implicit cast off.
+ Value *Elt = Visit(const_cast<Expr*>(E->IgnoreImpCasts()));
+ Elt = EmitScalarConversion(Elt, E->IgnoreImpCasts()->getType(),
+ DestTy->getAs<VectorType>()->getElementType(),
+ CE->getExprLoc(),
+ CGF.getContext().getLangOpts().OpenCL);
// Splat the element across to all elements
unsigned NumElements = cast<llvm::VectorType>(DstTy)->getNumElements();
@@ -1560,7 +1559,8 @@
case CK_IntegralToFloating:
case CK_FloatingToIntegral:
case CK_FloatingCast:
- return EmitScalarConversion(Visit(E), E->getType(), DestTy);
+ return EmitScalarConversion(Visit(E), E->getType(), DestTy,
+ CE->getExprLoc());
case CK_IntegralToBoolean:
return EmitIntToBoolConversion(Visit(E));
case CK_PointerToBoolean:
@@ -1582,7 +1582,8 @@
CodeGenFunction::ComplexPairTy V = CGF.EmitComplexExpr(E);
// TODO: kill this function off, inline appropriate case here
- return EmitComplexToScalarConversion(V, E->getType(), DestTy);
+ return EmitComplexToScalarConversion(V, E->getType(), DestTy,
+ CE->getExprLoc());
}
case CK_ZeroToOCLEvent: {
@@ -1597,9 +1598,9 @@
Value *ScalarExprEmitter::VisitStmtExpr(const StmtExpr *E) {
CodeGenFunction::StmtExprEvaluation eval(CGF);
- llvm::Value *RetAlloca = CGF.EmitCompoundStmt(*E->getSubStmt(),
- !E->getType()->isVoidType());
- if (!RetAlloca)
+ Address RetAlloca = CGF.EmitCompoundStmt(*E->getSubStmt(),
+ !E->getType()->isVoidType());
+ if (!RetAlloca.isValid())
return nullptr;
return CGF.EmitLoadOfScalar(CGF.MakeAddrLValue(RetAlloca, E->getType()),
E->getExprLoc());
@@ -1655,16 +1656,14 @@
if (isInc && type->isBooleanType()) {
llvm::Value *True = CGF.EmitToMemory(Builder.getTrue(), type);
if (isPre) {
- Builder.Insert(new llvm::StoreInst(True,
- LV.getAddress(), LV.isVolatileQualified(),
- LV.getAlignment().getQuantity(),
- llvm::SequentiallyConsistent));
+ Builder.CreateStore(True, LV.getAddress(), LV.isVolatileQualified())
+ ->setAtomic(llvm::SequentiallyConsistent);
return Builder.getTrue();
}
// For atomic bool increment, we just store true and return it for
// preincrement, do an atomic swap with true for postincrement
return Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
- LV.getAddress(), True, llvm::SequentiallyConsistent);
+ LV.getPointer(), True, llvm::SequentiallyConsistent);
}
// Special case for atomic increment / decrement on integers, emit
// atomicrmw instructions. We skip this if we want to be doing overflow
@@ -1681,7 +1680,7 @@
llvm::Value *amt = CGF.EmitToMemory(
llvm::ConstantInt::get(ConvertType(type), 1, true), type);
llvm::Value *old = Builder.CreateAtomicRMW(aop,
- LV.getAddress(), amt, llvm::SequentiallyConsistent);
+ LV.getPointer(), amt, llvm::SequentiallyConsistent);
return isPre ? Builder.CreateBinOp(op, old, amt) : old;
}
value = EmitLoadOfLValue(LV, E->getExprLoc());
@@ -2033,6 +2032,13 @@
return size;
}
+ } else if (E->getKind() == UETT_OpenMPRequiredSimdAlign) {
+ auto Alignment =
+ CGF.getContext()
+ .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign(
+ E->getTypeOfArgument()->getPointeeType()))
+ .getQuantity();
+ return llvm::ConstantInt::get(CGF.SizeTy, Alignment);
}
// If this isn't sizeof(vla), the result must be constant; use the constant
@@ -2111,7 +2117,7 @@
OpInfo.RHS = Visit(E->getRHS());
OpInfo.Ty = E->getComputationResultType();
OpInfo.Opcode = E->getOpcode();
- OpInfo.FPContractable = false;
+ OpInfo.FPContractable = E->isFPContractable();
OpInfo.E = E;
// Load/convert the LHS.
LValue LHSLV = EmitCheckedLValue(E->getLHS(), CodeGenFunction::TCK_Store);
@@ -2151,9 +2157,11 @@
llvm_unreachable("Invalid compound assignment type");
}
if (aop != llvm::AtomicRMWInst::BAD_BINOP) {
- llvm::Value *amt = CGF.EmitToMemory(EmitScalarConversion(OpInfo.RHS,
- E->getRHS()->getType(), LHSTy), LHSTy);
- Builder.CreateAtomicRMW(aop, LHSLV.getAddress(), amt,
+ llvm::Value *amt = CGF.EmitToMemory(
+ EmitScalarConversion(OpInfo.RHS, E->getRHS()->getType(), LHSTy,
+ E->getExprLoc()),
+ LHSTy);
+ Builder.CreateAtomicRMW(aop, LHSLV.getPointer(), amt,
llvm::SequentiallyConsistent);
return LHSLV;
}
@@ -2173,14 +2181,16 @@
else
OpInfo.LHS = EmitLoadOfLValue(LHSLV, E->getExprLoc());
- OpInfo.LHS = EmitScalarConversion(OpInfo.LHS, LHSTy,
- E->getComputationLHSType());
+ SourceLocation Loc = E->getExprLoc();
+ OpInfo.LHS =
+ EmitScalarConversion(OpInfo.LHS, LHSTy, E->getComputationLHSType(), Loc);
// Expand the binary operator.
Result = (this->*Func)(OpInfo);
// Convert the result back to the LHS type.
- Result = EmitScalarConversion(Result, E->getComputationResultType(), LHSTy);
+ Result =
+ EmitScalarConversion(Result, E->getComputationResultType(), LHSTy, Loc);
if (atomicPHI) {
llvm::BasicBlock *opBB = Builder.GetInsertBlock();
@@ -2366,9 +2376,9 @@
// Branch in case of overflow.
llvm::BasicBlock *initialBB = Builder.GetInsertBlock();
- llvm::Function::iterator insertPt = initialBB;
+ llvm::Function::iterator insertPt = initialBB->getIterator();
llvm::BasicBlock *continueBB = CGF.createBasicBlock("nooverflow", CGF.CurFn,
- std::next(insertPt));
+ &*std::next(insertPt));
llvm::BasicBlock *overflowBB = CGF.createBasicBlock("overflow", CGF.CurFn);
Builder.CreateCondBr(overflow, overflowBB, continueBB);
@@ -2555,19 +2565,17 @@
return nullptr;
// We have a potentially fusable op. Look for a mul on one of the operands.
- if (llvm::BinaryOperator* LHSBinOp = dyn_cast<llvm::BinaryOperator>(op.LHS)) {
- if (LHSBinOp->getOpcode() == llvm::Instruction::FMul) {
- assert(LHSBinOp->getNumUses() == 0 &&
- "Operations with multiple uses shouldn't be contracted.");
+ // Also, make sure that the mul result isn't used directly. In that case,
+ // there's no point creating a muladd operation.
+ if (auto *LHSBinOp = dyn_cast<llvm::BinaryOperator>(op.LHS)) {
+ if (LHSBinOp->getOpcode() == llvm::Instruction::FMul &&
+ LHSBinOp->use_empty())
return buildFMulAdd(LHSBinOp, op.RHS, CGF, Builder, false, isSub);
- }
- } else if (llvm::BinaryOperator* RHSBinOp =
- dyn_cast<llvm::BinaryOperator>(op.RHS)) {
- if (RHSBinOp->getOpcode() == llvm::Instruction::FMul) {
- assert(RHSBinOp->getNumUses() == 0 &&
- "Operations with multiple uses shouldn't be contracted.");
+ }
+ if (auto *RHSBinOp = dyn_cast<llvm::BinaryOperator>(op.RHS)) {
+ if (RHSBinOp->getOpcode() == llvm::Instruction::FMul &&
+ RHSBinOp->use_empty())
return buildFMulAdd(RHSBinOp, op.LHS, CGF, Builder, isSub, false);
- }
}
return nullptr;
@@ -2825,8 +2833,10 @@
}
}
-Value *ScalarExprEmitter::EmitCompare(const BinaryOperator *E,unsigned UICmpOpc,
- unsigned SICmpOpc, unsigned FCmpOpc) {
+Value *ScalarExprEmitter::EmitCompare(const BinaryOperator *E,
+ llvm::CmpInst::Predicate UICmpOpc,
+ llvm::CmpInst::Predicate SICmpOpc,
+ llvm::CmpInst::Predicate FCmpOpc) {
TestAndClearIgnoreResultAssign();
Value *Result;
QualType LHSTy = E->getLHS()->getType();
@@ -2904,19 +2914,17 @@
Value *CR6Param = Builder.getInt32(CR6);
llvm::Function *F = CGF.CGM.getIntrinsic(ID);
Result = Builder.CreateCall(F, {CR6Param, FirstVecArg, SecondVecArg});
- return EmitScalarConversion(Result, CGF.getContext().BoolTy, E->getType());
+ return EmitScalarConversion(Result, CGF.getContext().BoolTy, E->getType(),
+ E->getExprLoc());
}
if (LHS->getType()->isFPOrFPVectorTy()) {
- Result = Builder.CreateFCmp((llvm::CmpInst::Predicate)FCmpOpc,
- LHS, RHS, "cmp");
+ Result = Builder.CreateFCmp(FCmpOpc, LHS, RHS, "cmp");
} else if (LHSTy->hasSignedIntegerRepresentation()) {
- Result = Builder.CreateICmp((llvm::ICmpInst::Predicate)SICmpOpc,
- LHS, RHS, "cmp");
+ Result = Builder.CreateICmp(SICmpOpc, LHS, RHS, "cmp");
} else {
// Unsigned integers and pointers.
- Result = Builder.CreateICmp((llvm::ICmpInst::Predicate)UICmpOpc,
- LHS, RHS, "cmp");
+ Result = Builder.CreateICmp(UICmpOpc, LHS, RHS, "cmp");
}
// If this is a vector comparison, sign extend the result to the appropriate
@@ -2951,17 +2959,13 @@
Value *ResultR, *ResultI;
if (CETy->isRealFloatingType()) {
- ResultR = Builder.CreateFCmp((llvm::FCmpInst::Predicate)FCmpOpc,
- LHS.first, RHS.first, "cmp.r");
- ResultI = Builder.CreateFCmp((llvm::FCmpInst::Predicate)FCmpOpc,
- LHS.second, RHS.second, "cmp.i");
+ ResultR = Builder.CreateFCmp(FCmpOpc, LHS.first, RHS.first, "cmp.r");
+ ResultI = Builder.CreateFCmp(FCmpOpc, LHS.second, RHS.second, "cmp.i");
} else {
// Complex comparisons can only be equality comparisons. As such, signed
// and unsigned opcodes are the same.
- ResultR = Builder.CreateICmp((llvm::ICmpInst::Predicate)UICmpOpc,
- LHS.first, RHS.first, "cmp.r");
- ResultI = Builder.CreateICmp((llvm::ICmpInst::Predicate)UICmpOpc,
- LHS.second, RHS.second, "cmp.i");
+ ResultR = Builder.CreateICmp(UICmpOpc, LHS.first, RHS.first, "cmp.r");
+ ResultI = Builder.CreateICmp(UICmpOpc, LHS.second, RHS.second, "cmp.i");
}
if (E->getOpcode() == BO_EQ) {
@@ -2973,7 +2977,8 @@
}
}
- return EmitScalarConversion(Result, CGF.getContext().BoolTy, E->getType());
+ return EmitScalarConversion(Result, CGF.getContext().BoolTy, E->getType(),
+ E->getExprLoc());
}
Value *ScalarExprEmitter::VisitBinAssign(const BinaryOperator *E) {
@@ -3359,13 +3364,14 @@
if (Ty->isVariablyModifiedType())
CGF.EmitVariablyModifiedType(Ty);
- llvm::Value *ArgValue = CGF.EmitVAListRef(VE->getSubExpr());
- llvm::Value *ArgPtr = CGF.EmitVAArg(ArgValue, VE->getType());
+ Address ArgValue = Address::invalid();
+ Address ArgPtr = CGF.EmitVAArg(VE, ArgValue);
+
llvm::Type *ArgTy = ConvertType(VE->getType());
// If EmitVAArg fails, we fall back to the LLVM instruction.
- if (!ArgPtr)
- return Builder.CreateVAArg(ArgValue, ArgTy);
+ if (!ArgPtr.isValid())
+ return Builder.CreateVAArg(ArgValue.getPointer(), ArgTy);
// FIXME Volatility.
llvm::Value *Val = Builder.CreateLoad(ArgPtr);
@@ -3442,8 +3448,8 @@
// Entry Point into this File
//===----------------------------------------------------------------------===//
-/// EmitScalarExpr - Emit the computation of the specified expression of scalar
-/// type, ignoring the result.
+/// Emit the computation of the specified expression of scalar type, ignoring
+/// the result.
Value *CodeGenFunction::EmitScalarExpr(const Expr *E, bool IgnoreResultAssign) {
assert(E && hasScalarEvaluationKind(E->getType()) &&
"Invalid scalar expression to emit");
@@ -3452,25 +3458,26 @@
.Visit(const_cast<Expr *>(E));
}
-/// EmitScalarConversion - Emit a conversion from the specified type to the
-/// specified destination type, both of which are LLVM scalar types.
+/// Emit a conversion from the specified type to the specified destination type,
+/// both of which are LLVM scalar types.
Value *CodeGenFunction::EmitScalarConversion(Value *Src, QualType SrcTy,
- QualType DstTy) {
+ QualType DstTy,
+ SourceLocation Loc) {
assert(hasScalarEvaluationKind(SrcTy) && hasScalarEvaluationKind(DstTy) &&
"Invalid scalar expression to emit");
- return ScalarExprEmitter(*this).EmitScalarConversion(Src, SrcTy, DstTy);
+ return ScalarExprEmitter(*this).EmitScalarConversion(Src, SrcTy, DstTy, Loc);
}
-/// EmitComplexToScalarConversion - Emit a conversion from the specified complex
-/// type to the specified destination type, where the destination type is an
-/// LLVM scalar type.
+/// Emit a conversion from the specified complex type to the specified
+/// destination type, where the destination type is an LLVM scalar type.
Value *CodeGenFunction::EmitComplexToScalarConversion(ComplexPairTy Src,
QualType SrcTy,
- QualType DstTy) {
+ QualType DstTy,
+ SourceLocation Loc) {
assert(SrcTy->isAnyComplexType() && hasScalarEvaluationKind(DstTy) &&
"Invalid complex -> scalar conversion");
- return ScalarExprEmitter(*this).EmitComplexToScalarConversion(Src, SrcTy,
- DstTy);
+ return ScalarExprEmitter(*this)
+ .EmitComplexToScalarConversion(Src, SrcTy, DstTy, Loc);
}
@@ -3481,30 +3488,20 @@
}
LValue CodeGenFunction::EmitObjCIsaExpr(const ObjCIsaExpr *E) {
- llvm::Value *V;
// object->isa or (*object).isa
// Generate code as for: *(Class*)object
- // build Class* type
- llvm::Type *ClassPtrTy = ConvertType(E->getType());
Expr *BaseExpr = E->getBase();
+ Address Addr = Address::invalid();
if (BaseExpr->isRValue()) {
- V = CreateMemTemp(E->getType(), "resval");
- llvm::Value *Src = EmitScalarExpr(BaseExpr);
- Builder.CreateStore(Src, V);
- V = ScalarExprEmitter(*this).EmitLoadOfLValue(
- MakeNaturalAlignAddrLValue(V, E->getType()), E->getExprLoc());
+ Addr = Address(EmitScalarExpr(BaseExpr), getPointerAlign());
} else {
- if (E->isArrow())
- V = ScalarExprEmitter(*this).EmitLoadOfLValue(BaseExpr);
- else
- V = EmitLValue(BaseExpr).getAddress();
+ Addr = EmitLValue(BaseExpr).getAddress();
}
- // build Class* type
- ClassPtrTy = ClassPtrTy->getPointerTo();
- V = Builder.CreateBitCast(V, ClassPtrTy);
- return MakeNaturalAlignAddrLValue(V, E->getType());
+ // Cast the address to Class*.
+ Addr = Builder.CreateElementBitCast(Addr, ConvertType(E->getType()));
+ return MakeAddrLValue(Addr, E->getType());
}
diff --git a/lib/CodeGen/CGLoopInfo.cpp b/lib/CodeGen/CGLoopInfo.cpp
index 1163d63..0afe7db 100644
--- a/lib/CodeGen/CGLoopInfo.cpp
+++ b/lib/CodeGen/CGLoopInfo.cpp
@@ -8,6 +8,7 @@
//===----------------------------------------------------------------------===//
#include "CGLoopInfo.h"
+#include "clang/AST/ASTContext.h"
#include "clang/AST/Attr.h"
#include "clang/Sema/LoopHint.h"
#include "llvm/IR/BasicBlock.h"
@@ -20,9 +21,10 @@
static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs) {
- if (!Attrs.IsParallel && Attrs.VectorizerWidth == 0 &&
- Attrs.VectorizerUnroll == 0 &&
- Attrs.VectorizerEnable == LoopAttributes::VecUnspecified)
+ if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
+ Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
+ Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
+ Attrs.UnrollEnable == LoopAttributes::Unspecified)
return nullptr;
SmallVector<Metadata *, 4> Args;
@@ -30,29 +32,49 @@
auto TempNode = MDNode::getTemporary(Ctx, None);
Args.push_back(TempNode.get());
- // Setting vectorizer.width
- if (Attrs.VectorizerWidth > 0) {
+ // Setting vectorize.width
+ if (Attrs.VectorizeWidth > 0) {
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.width"),
ConstantAsMetadata::get(ConstantInt::get(
- Type::getInt32Ty(Ctx), Attrs.VectorizerWidth))};
+ Type::getInt32Ty(Ctx), Attrs.VectorizeWidth))};
Args.push_back(MDNode::get(Ctx, Vals));
}
- // Setting vectorizer.unroll
- if (Attrs.VectorizerUnroll > 0) {
+ // Setting interleave.count
+ if (Attrs.InterleaveCount > 0) {
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.count"),
ConstantAsMetadata::get(ConstantInt::get(
- Type::getInt32Ty(Ctx), Attrs.VectorizerUnroll))};
+ Type::getInt32Ty(Ctx), Attrs.InterleaveCount))};
Args.push_back(MDNode::get(Ctx, Vals));
}
- // Setting vectorizer.enable
- if (Attrs.VectorizerEnable != LoopAttributes::VecUnspecified) {
- Metadata *Vals[] = {
- MDString::get(Ctx, "llvm.loop.vectorize.enable"),
- ConstantAsMetadata::get(ConstantInt::get(
- Type::getInt1Ty(Ctx),
- (Attrs.VectorizerEnable == LoopAttributes::VecEnable)))};
+ // Setting interleave.count
+ if (Attrs.UnrollCount > 0) {
+ Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
+ ConstantAsMetadata::get(ConstantInt::get(
+ Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
+ Args.push_back(MDNode::get(Ctx, Vals));
+ }
+
+ // Setting vectorize.enable
+ if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) {
+ Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
+ ConstantAsMetadata::get(ConstantInt::get(
+ Type::getInt1Ty(Ctx), (Attrs.VectorizeEnable ==
+ LoopAttributes::Enable)))};
+ Args.push_back(MDNode::get(Ctx, Vals));
+ }
+
+ // Setting unroll.full or unroll.disable
+ if (Attrs.UnrollEnable != LoopAttributes::Unspecified) {
+ std::string Name;
+ if (Attrs.UnrollEnable == LoopAttributes::Enable)
+ Name = "llvm.loop.unroll.enable";
+ else if (Attrs.UnrollEnable == LoopAttributes::Full)
+ Name = "llvm.loop.unroll.full";
+ else
+ Name = "llvm.loop.unroll.disable";
+ Metadata *Vals[] = {MDString::get(Ctx, Name)};
Args.push_back(MDNode::get(Ctx, Vals));
}
@@ -63,14 +85,17 @@
}
LoopAttributes::LoopAttributes(bool IsParallel)
- : IsParallel(IsParallel), VectorizerEnable(LoopAttributes::VecUnspecified),
- VectorizerWidth(0), VectorizerUnroll(0) {}
+ : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
+ UnrollEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
+ InterleaveCount(0), UnrollCount(0) {}
void LoopAttributes::clear() {
IsParallel = false;
- VectorizerWidth = 0;
- VectorizerUnroll = 0;
- VectorizerEnable = LoopAttributes::VecUnspecified;
+ VectorizeWidth = 0;
+ InterleaveCount = 0;
+ UnrollCount = 0;
+ VectorizeEnable = LoopAttributes::Unspecified;
+ UnrollEnable = LoopAttributes::Unspecified;
}
LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs)
@@ -78,8 +103,16 @@
LoopID = createMetadata(Header->getContext(), Attrs);
}
-void LoopInfoStack::push(BasicBlock *Header,
+void LoopInfoStack::push(BasicBlock *Header) {
+ Active.push_back(LoopInfo(Header, StagedAttrs));
+ // Clear the attributes so nested loops do not inherit them.
+ StagedAttrs.clear();
+}
+
+void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
ArrayRef<const clang::Attr *> Attrs) {
+
+ // Identify loop hint attributes from Attrs.
for (const auto *Attr : Attrs) {
const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
@@ -87,28 +120,105 @@
if (!LH)
continue;
+ auto *ValueExpr = LH->getValue();
+ unsigned ValueInt = 1;
+ if (ValueExpr) {
+ llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
+ ValueInt = ValueAPS.getSExtValue();
+ }
+
LoopHintAttr::OptionType Option = LH->getOption();
LoopHintAttr::LoopHintState State = LH->getState();
- switch (Option) {
- case LoopHintAttr::Vectorize:
- case LoopHintAttr::Interleave:
- if (State == LoopHintAttr::AssumeSafety) {
- // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
- setParallel(true);
+ switch (State) {
+ case LoopHintAttr::Disable:
+ switch (Option) {
+ case LoopHintAttr::Vectorize:
+ // Disable vectorization by specifying a width of 1.
+ setVectorizeWidth(1);
+ break;
+ case LoopHintAttr::Interleave:
+ // Disable interleaving by speciyfing a count of 1.
+ setInterleaveCount(1);
+ break;
+ case LoopHintAttr::Unroll:
+ setUnrollState(LoopAttributes::Disable);
+ break;
+ case LoopHintAttr::UnrollCount:
+ case LoopHintAttr::VectorizeWidth:
+ case LoopHintAttr::InterleaveCount:
+ llvm_unreachable("Options cannot be disabled.");
+ break;
}
break;
- case LoopHintAttr::VectorizeWidth:
- case LoopHintAttr::InterleaveCount:
- case LoopHintAttr::Unroll:
- case LoopHintAttr::UnrollCount:
- // Nothing to do here for these loop hints.
+ case LoopHintAttr::Enable:
+ switch (Option) {
+ case LoopHintAttr::Vectorize:
+ case LoopHintAttr::Interleave:
+ setVectorizeEnable(true);
+ break;
+ case LoopHintAttr::Unroll:
+ setUnrollState(LoopAttributes::Enable);
+ break;
+ case LoopHintAttr::UnrollCount:
+ case LoopHintAttr::VectorizeWidth:
+ case LoopHintAttr::InterleaveCount:
+ llvm_unreachable("Options cannot enabled.");
+ break;
+ }
+ break;
+ case LoopHintAttr::AssumeSafety:
+ switch (Option) {
+ case LoopHintAttr::Vectorize:
+ case LoopHintAttr::Interleave:
+ // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
+ setParallel(true);
+ setVectorizeEnable(true);
+ break;
+ case LoopHintAttr::Unroll:
+ case LoopHintAttr::UnrollCount:
+ case LoopHintAttr::VectorizeWidth:
+ case LoopHintAttr::InterleaveCount:
+ llvm_unreachable("Options cannot be used to assume mem safety.");
+ break;
+ }
+ break;
+ case LoopHintAttr::Full:
+ switch (Option) {
+ case LoopHintAttr::Unroll:
+ setUnrollState(LoopAttributes::Full);
+ break;
+ case LoopHintAttr::Vectorize:
+ case LoopHintAttr::Interleave:
+ case LoopHintAttr::UnrollCount:
+ case LoopHintAttr::VectorizeWidth:
+ case LoopHintAttr::InterleaveCount:
+ llvm_unreachable("Options cannot be used with 'full' hint.");
+ break;
+ }
+ break;
+ case LoopHintAttr::Numeric:
+ switch (Option) {
+ case LoopHintAttr::VectorizeWidth:
+ setVectorizeWidth(ValueInt);
+ break;
+ case LoopHintAttr::InterleaveCount:
+ setInterleaveCount(ValueInt);
+ break;
+ case LoopHintAttr::UnrollCount:
+ setUnrollCount(ValueInt);
+ break;
+ case LoopHintAttr::Unroll:
+ case LoopHintAttr::Vectorize:
+ case LoopHintAttr::Interleave:
+ llvm_unreachable("Options cannot be assigned a value.");
+ break;
+ }
break;
}
}
- Active.push_back(LoopInfo(Header, StagedAttrs));
- // Clear the attributes so nested loops do not inherit them.
- StagedAttrs.clear();
+ /// Stage the attributes.
+ push(Header);
}
void LoopInfoStack::pop() {
diff --git a/lib/CodeGen/CGLoopInfo.h b/lib/CodeGen/CGLoopInfo.h
index 2249937..ec33906 100644
--- a/lib/CodeGen/CGLoopInfo.h
+++ b/lib/CodeGen/CGLoopInfo.h
@@ -29,6 +29,7 @@
namespace clang {
class Attr;
+class ASTContext;
namespace CodeGen {
/// \brief Attributes that may be specified on loops.
@@ -39,17 +40,23 @@
/// \brief Generate llvm.loop.parallel metadata for loads and stores.
bool IsParallel;
- /// \brief Values of llvm.loop.vectorize.enable metadata.
- enum LVEnableState { VecUnspecified, VecEnable, VecDisable };
+ /// \brief State of loop vectorization or unrolling.
+ enum LVEnableState { Unspecified, Enable, Disable, Full };
- /// \brief llvm.loop.vectorize.enable
- LVEnableState VectorizerEnable;
+ /// \brief Value for llvm.loop.vectorize.enable metadata.
+ LVEnableState VectorizeEnable;
- /// \brief llvm.loop.vectorize.width
- unsigned VectorizerWidth;
+ /// \brief Value for llvm.loop.unroll.* metadata (enable, disable, or full).
+ LVEnableState UnrollEnable;
- /// \brief llvm.loop.interleave.count
- unsigned VectorizerUnroll;
+ /// \brief Value for llvm.loop.vectorize.width metadata.
+ unsigned VectorizeWidth;
+
+ /// \brief Value for llvm.loop.interleave.count metadata.
+ unsigned InterleaveCount;
+
+ /// \brief llvm.unroll.
+ unsigned UnrollCount;
};
/// \brief Information used when generating a structured loop.
@@ -88,8 +95,12 @@
/// \brief Begin a new structured loop. The set of staged attributes will be
/// applied to the loop and then cleared.
- void push(llvm::BasicBlock *Header,
- llvm::ArrayRef<const Attr *> Attrs = llvm::None);
+ void push(llvm::BasicBlock *Header);
+
+ /// \brief Begin a new structured loop. Stage attributes from the Attrs list.
+ /// The staged attributes are applied to the loop and then cleared.
+ void push(llvm::BasicBlock *Header, clang::ASTContext &Ctx,
+ llvm::ArrayRef<const Attr *> Attrs);
/// \brief End the current loop.
void pop();
@@ -109,17 +120,25 @@
/// \brief Set the next pushed loop as parallel.
void setParallel(bool Enable = true) { StagedAttrs.IsParallel = Enable; }
- /// \brief Set the next pushed loop 'vectorizer.enable'
- void setVectorizerEnable(bool Enable = true) {
- StagedAttrs.VectorizerEnable =
- Enable ? LoopAttributes::VecEnable : LoopAttributes::VecDisable;
+ /// \brief Set the next pushed loop 'vectorize.enable'
+ void setVectorizeEnable(bool Enable = true) {
+ StagedAttrs.VectorizeEnable =
+ Enable ? LoopAttributes::Enable : LoopAttributes::Disable;
}
- /// \brief Set the vectorizer width for the next loop pushed.
- void setVectorizerWidth(unsigned W) { StagedAttrs.VectorizerWidth = W; }
+ /// \brief Set the next pushed loop unroll state.
+ void setUnrollState(const LoopAttributes::LVEnableState &State) {
+ StagedAttrs.UnrollEnable = State;
+ }
- /// \brief Set the vectorizer unroll for the next loop pushed.
- void setVectorizerUnroll(unsigned U) { StagedAttrs.VectorizerUnroll = U; }
+ /// \brief Set the vectorize width for the next loop pushed.
+ void setVectorizeWidth(unsigned W) { StagedAttrs.VectorizeWidth = W; }
+
+ /// \brief Set the interleave count for the next loop pushed.
+ void setInterleaveCount(unsigned C) { StagedAttrs.InterleaveCount = C; }
+
+ /// \brief Set the unroll count for the next loop pushed.
+ void setUnrollCount(unsigned C) { StagedAttrs.UnrollCount = C; }
private:
/// \brief Returns true if there is LoopInfo on the stack.
diff --git a/lib/CodeGen/CGObjC.cpp b/lib/CodeGen/CGObjC.cpp
index 9981fcc..2d5991b 100644
--- a/lib/CodeGen/CGObjC.cpp
+++ b/lib/CodeGen/CGObjC.cpp
@@ -31,16 +31,14 @@
typedef llvm::PointerIntPair<llvm::Value*,1,bool> TryEmitResult;
static TryEmitResult
tryEmitARCRetainScalarExpr(CodeGenFunction &CGF, const Expr *e);
-static RValue AdjustRelatedResultType(CodeGenFunction &CGF,
- QualType ET,
- const ObjCMethodDecl *Method,
- RValue Result);
+static RValue AdjustObjCObjectType(CodeGenFunction &CGF,
+ QualType ET,
+ RValue Result);
/// Given the address of a variable of pointer type, find the correct
/// null to store into it.
-static llvm::Constant *getNullForVariable(llvm::Value *addr) {
- llvm::Type *type =
- cast<llvm::PointerType>(addr->getType())->getElementType();
+static llvm::Constant *getNullForVariable(Address addr) {
+ llvm::Type *type = addr.getElementType();
return llvm::ConstantPointerNull::get(cast<llvm::PointerType>(type));
}
@@ -48,20 +46,22 @@
llvm::Value *CodeGenFunction::EmitObjCStringLiteral(const ObjCStringLiteral *E)
{
llvm::Constant *C =
- CGM.getObjCRuntime().GenerateConstantString(E->getString());
+ CGM.getObjCRuntime().GenerateConstantString(E->getString()).getPointer();
// FIXME: This bitcast should just be made an invariant on the Runtime.
return llvm::ConstantExpr::getBitCast(C, ConvertType(E->getType()));
}
/// EmitObjCBoxedExpr - This routine generates code to call
/// the appropriate expression boxing method. This will either be
-/// one of +[NSNumber numberWith<Type>:], or +[NSString stringWithUTF8String:].
+/// one of +[NSNumber numberWith<Type>:], or +[NSString stringWithUTF8String:],
+/// or [NSValue valueWithBytes:objCType:].
///
llvm::Value *
CodeGenFunction::EmitObjCBoxedExpr(const ObjCBoxedExpr *E) {
// Generate the correct selector for this literal's concrete type.
// Get the method.
const ObjCMethodDecl *BoxingMethod = E->getBoxingMethod();
+ const Expr *SubExpr = E->getSubExpr();
assert(BoxingMethod && "BoxingMethod is null");
assert(BoxingMethod->isClassMethod() && "BoxingMethod must be a class method");
Selector Sel = BoxingMethod->getSelector();
@@ -74,7 +74,34 @@
llvm::Value *Receiver = Runtime.GetClass(*this, ClassDecl);
CallArgList Args;
- EmitCallArgs(Args, BoxingMethod, E->arg_begin(), E->arg_end());
+ const ParmVarDecl *ArgDecl = *BoxingMethod->param_begin();
+ QualType ArgQT = ArgDecl->getType().getUnqualifiedType();
+
+ // ObjCBoxedExpr supports boxing of structs and unions
+ // via [NSValue valueWithBytes:objCType:]
+ const QualType ValueType(SubExpr->getType().getCanonicalType());
+ if (ValueType->isObjCBoxableRecordType()) {
+ // Emit CodeGen for first parameter
+ // and cast value to correct type
+ Address Temporary = CreateMemTemp(SubExpr->getType());
+ EmitAnyExprToMem(SubExpr, Temporary, Qualifiers(), /*isInit*/ true);
+ Address BitCast = Builder.CreateBitCast(Temporary, ConvertType(ArgQT));
+ Args.add(RValue::get(BitCast.getPointer()), ArgQT);
+
+ // Create char array to store type encoding
+ std::string Str;
+ getContext().getObjCEncodingForType(ValueType, Str);
+ llvm::Constant *GV = CGM.GetAddrOfConstantCString(Str).getPointer();
+
+ // Cast type encoding to correct type
+ const ParmVarDecl *EncodingDecl = BoxingMethod->parameters()[1];
+ QualType EncodingQT = EncodingDecl->getType().getUnqualifiedType();
+ llvm::Value *Cast = Builder.CreateBitCast(GV, ConvertType(EncodingQT));
+
+ Args.add(RValue::get(Cast), EncodingQT);
+ } else {
+ Args.add(EmitAnyExpr(SubExpr), ArgQT);
+ }
RValue result = Runtime.GenerateMessageSend(
*this, ReturnValueSlot(), BoxingMethod->getReturnType(), Sel, Receiver,
@@ -102,8 +129,8 @@
ArrayType::Normal, /*IndexTypeQuals=*/0);
// Allocate the temporary array(s).
- llvm::AllocaInst *Objects = CreateMemTemp(ElementArrayType, "objects");
- llvm::AllocaInst *Keys = nullptr;
+ Address Objects = CreateMemTemp(ElementArrayType, "objects");
+ Address Keys = Address::invalid();
if (DLE)
Keys = CreateMemTemp(ElementArrayType, "keys");
@@ -119,9 +146,9 @@
if (ALE) {
// Emit the element and store it to the appropriate array slot.
const Expr *Rhs = ALE->getElement(i);
- LValue LV = LValue::MakeAddr(
- Builder.CreateStructGEP(Objects->getAllocatedType(), Objects, i),
- ElementType, Context.getTypeAlignInChars(Rhs->getType()), Context);
+ LValue LV = MakeAddrLValue(
+ Builder.CreateConstArrayGEP(Objects, i, getPointerSize()),
+ ElementType, AlignmentSource::Decl);
llvm::Value *value = EmitScalarExpr(Rhs);
EmitStoreThroughLValue(RValue::get(value), LV, true);
@@ -131,17 +158,17 @@
} else {
// Emit the key and store it to the appropriate array slot.
const Expr *Key = DLE->getKeyValueElement(i).Key;
- LValue KeyLV = LValue::MakeAddr(
- Builder.CreateStructGEP(Keys->getAllocatedType(), Keys, i),
- ElementType, Context.getTypeAlignInChars(Key->getType()), Context);
+ LValue KeyLV = MakeAddrLValue(
+ Builder.CreateConstArrayGEP(Keys, i, getPointerSize()),
+ ElementType, AlignmentSource::Decl);
llvm::Value *keyValue = EmitScalarExpr(Key);
EmitStoreThroughLValue(RValue::get(keyValue), KeyLV, /*isInit=*/true);
// Emit the value and store it to the appropriate array slot.
const Expr *Value = DLE->getKeyValueElement(i).Value;
- LValue ValueLV = LValue::MakeAddr(
- Builder.CreateStructGEP(Objects->getAllocatedType(), Objects, i),
- ElementType, Context.getTypeAlignInChars(Value->getType()), Context);
+ LValue ValueLV = MakeAddrLValue(
+ Builder.CreateConstArrayGEP(Objects, i, getPointerSize()),
+ ElementType, AlignmentSource::Decl);
llvm::Value *valueValue = EmitScalarExpr(Value);
EmitStoreThroughLValue(RValue::get(valueValue), ValueLV, /*isInit=*/true);
if (TrackNeededObjects) {
@@ -156,11 +183,11 @@
ObjCMethodDecl::param_const_iterator PI = MethodWithObjects->param_begin();
const ParmVarDecl *argDecl = *PI++;
QualType ArgQT = argDecl->getType().getUnqualifiedType();
- Args.add(RValue::get(Objects), ArgQT);
+ Args.add(RValue::get(Objects.getPointer()), ArgQT);
if (DLE) {
argDecl = *PI++;
ArgQT = argDecl->getType().getUnqualifiedType();
- Args.add(RValue::get(Keys), ArgQT);
+ Args.add(RValue::get(Keys.getPointer()), ArgQT);
}
argDecl = *PI;
ArgQT = argDecl->getType().getUnqualifiedType();
@@ -218,23 +245,22 @@
return CGM.getObjCRuntime().GenerateProtocolRef(*this, E->getProtocol());
}
-/// \brief Adjust the type of the result of an Objective-C message send
-/// expression when the method has a related result type.
-static RValue AdjustRelatedResultType(CodeGenFunction &CGF,
- QualType ExpT,
- const ObjCMethodDecl *Method,
- RValue Result) {
- if (!Method)
+/// \brief Adjust the type of an Objective-C object that doesn't match up due
+/// to type erasure at various points, e.g., related result types or the use
+/// of parameterized classes.
+static RValue AdjustObjCObjectType(CodeGenFunction &CGF, QualType ExpT,
+ RValue Result) {
+ if (!ExpT->isObjCRetainableType())
return Result;
- if (!Method->hasRelatedResultType() ||
- CGF.getContext().hasSameType(ExpT, Method->getReturnType()) ||
- !Result.isScalar())
+ // If the converted types are the same, we're done.
+ llvm::Type *ExpLLVMTy = CGF.ConvertType(ExpT);
+ if (ExpLLVMTy == Result.getScalarVal()->getType())
return Result;
-
- // We have applied a related result type. Cast the rvalue appropriately.
+
+ // We have applied a substitution. Cast the rvalue appropriately.
return RValue::get(CGF.Builder.CreateBitCast(Result.getScalarVal(),
- CGF.ConvertType(ExpT)));
+ ExpLLVMTy));
}
/// Decide whether to extend the lifetime of the receiver of a
@@ -247,10 +273,23 @@
// receiver is loaded from a variable with precise lifetime.
case ObjCMessageExpr::Instance: {
const Expr *receiver = message->getInstanceReceiver();
+
+ // Look through OVEs.
+ if (auto opaque = dyn_cast<OpaqueValueExpr>(receiver)) {
+ if (opaque->getSourceExpr())
+ receiver = opaque->getSourceExpr()->IgnoreParens();
+ }
+
const ImplicitCastExpr *ice = dyn_cast<ImplicitCastExpr>(receiver);
if (!ice || ice->getCastKind() != CK_LValueToRValue) return true;
receiver = ice->getSubExpr()->IgnoreParens();
+ // Look through OVEs.
+ if (auto opaque = dyn_cast<OpaqueValueExpr>(receiver)) {
+ if (opaque->getSourceExpr())
+ receiver = opaque->getSourceExpr()->IgnoreParens();
+ }
+
// Only __strong variables.
if (receiver->getType().getObjCLifetime() != Qualifiers::OCL_Strong)
return true;
@@ -284,6 +323,21 @@
llvm_unreachable("invalid receiver kind");
}
+/// Given an expression of ObjC pointer type, check whether it was
+/// immediately loaded from an ARC __weak l-value.
+static const Expr *findWeakLValue(const Expr *E) {
+ assert(E->getType()->isObjCRetainableType());
+ E = E->IgnoreParens();
+ if (auto CE = dyn_cast<CastExpr>(E)) {
+ if (CE->getCastKind() == CK_LValueToRValue) {
+ if (CE->getSubExpr()->getType().getObjCLifetime() == Qualifiers::OCL_Weak)
+ return CE->getSubExpr();
+ }
+ }
+
+ return nullptr;
+}
+
RValue CodeGenFunction::EmitObjCMessageExpr(const ObjCMessageExpr *E,
ReturnValueSlot Return) {
// Only the lookup mechanism and first two arguments of the method
@@ -294,6 +348,17 @@
const ObjCMethodDecl *method = E->getMethodDecl();
+ // If the method is -retain, and the receiver's being loaded from
+ // a __weak variable, peephole the entire operation to objc_loadWeakRetained.
+ if (method && E->getReceiverKind() == ObjCMessageExpr::Instance &&
+ method->getMethodFamily() == OMF_retain) {
+ if (auto lvalueExpr = findWeakLValue(E->getInstanceReceiver())) {
+ LValue lvalue = EmitLValue(lvalueExpr);
+ llvm::Value *result = EmitARCLoadWeakRetained(lvalue.getAddress());
+ return AdjustObjCObjectType(*this, E->getType(), RValue::get(result));
+ }
+ }
+
// We don't retain the receiver in delegate init calls, and this is
// safe because the receiver value is always loaded from 'self',
// which we zero out. We don't want to Block_copy block receivers,
@@ -362,7 +427,7 @@
QualType ResultType = method ? method->getReturnType() : E->getType();
CallArgList Args;
- EmitCallArgs(Args, method, E->arg_begin(), E->arg_end());
+ EmitCallArgs(Args, method, E->arguments());
// For delegate init calls in ARC, do an unsafe store of null into
// self. This represents the call taking direct ownership of that
@@ -376,10 +441,8 @@
"delegate init calls should only be marked in ARC");
// Do an unsafe store of null into self.
- llvm::Value *selfAddr =
- LocalDeclMap[cast<ObjCMethodDecl>(CurCodeDecl)->getSelfDecl()];
- assert(selfAddr && "no self entry for a delegate init call?");
-
+ Address selfAddr =
+ GetAddrOfLocalVar(cast<ObjCMethodDecl>(CurCodeDecl)->getSelfDecl());
Builder.CreateStore(getNullForVariable(selfAddr), selfAddr);
}
@@ -406,24 +469,23 @@
// For delegate init calls in ARC, implicitly store the result of
// the call back into self. This takes ownership of the value.
if (isDelegateInit) {
- llvm::Value *selfAddr =
- LocalDeclMap[cast<ObjCMethodDecl>(CurCodeDecl)->getSelfDecl()];
+ Address selfAddr =
+ GetAddrOfLocalVar(cast<ObjCMethodDecl>(CurCodeDecl)->getSelfDecl());
llvm::Value *newSelf = result.getScalarVal();
// The delegate return type isn't necessarily a matching type; in
// fact, it's quite likely to be 'id'.
- llvm::Type *selfTy =
- cast<llvm::PointerType>(selfAddr->getType())->getElementType();
+ llvm::Type *selfTy = selfAddr.getElementType();
newSelf = Builder.CreateBitCast(newSelf, selfTy);
Builder.CreateStore(newSelf, selfAddr);
}
- return AdjustRelatedResultType(*this, E->getType(), method, result);
+ return AdjustObjCObjectType(*this, E->getType(), result);
}
namespace {
-struct FinishARCDealloc : EHScopeStack::Cleanup {
+struct FinishARCDealloc final : EHScopeStack::Cleanup {
void Emit(CodeGenFunction &CGF, Flags flags) override {
const ObjCMethodDecl *method = cast<ObjCMethodDecl>(CGF.CurCodeDecl);
@@ -495,7 +557,7 @@
/// its pointer, name, and types registered in the class struture.
void CodeGenFunction::GenerateObjCMethod(const ObjCMethodDecl *OMD) {
StartObjCMethod(OMD, OMD->getClassInterface());
- PGO.assignRegionCounters(OMD, CurFn);
+ PGO.assignRegionCounters(GlobalDecl(OMD), CurFn);
assert(isa<CompoundStmt>(OMD->getBody()));
incrementProfileCounter(OMD->getBody());
EmitCompoundStmtWithoutScope(*cast<CompoundStmt>(OMD->getBody()));
@@ -508,19 +570,19 @@
bool isAtomic, bool hasStrong) {
ASTContext &Context = CGF.getContext();
- llvm::Value *src =
- CGF.EmitLValueForIvar(CGF.TypeOfSelfObject(), CGF.LoadObjCSelf(),
- ivar, 0).getAddress();
+ Address src =
+ CGF.EmitLValueForIvar(CGF.TypeOfSelfObject(), CGF.LoadObjCSelf(), ivar, 0)
+ .getAddress();
// objc_copyStruct (ReturnValue, &structIvar,
// sizeof (Type of Ivar), isAtomic, false);
CallArgList args;
- llvm::Value *dest = CGF.Builder.CreateBitCast(CGF.ReturnValue, CGF.VoidPtrTy);
- args.add(RValue::get(dest), Context.VoidPtrTy);
+ Address dest = CGF.Builder.CreateBitCast(CGF.ReturnValue, CGF.VoidPtrTy);
+ args.add(RValue::get(dest.getPointer()), Context.VoidPtrTy);
src = CGF.Builder.CreateBitCast(src, CGF.VoidPtrTy);
- args.add(RValue::get(src), Context.VoidPtrTy);
+ args.add(RValue::get(src.getPointer()), Context.VoidPtrTy);
CharUnits size = CGF.getContext().getTypeSizeInChars(ivar->getType());
args.add(RValue::get(CGF.CGM.getSize(size)), Context.getSizeType());
@@ -784,8 +846,8 @@
// The 2nd argument is the address of the ivar.
llvm::Value *ivarAddr =
- CGF.EmitLValueForIvar(CGF.TypeOfSelfObject(),
- CGF.LoadObjCSelf(), ivar, 0).getAddress();
+ CGF.EmitLValueForIvar(CGF.TypeOfSelfObject(),
+ CGF.LoadObjCSelf(), ivar, 0).getPointer();
ivarAddr = CGF.Builder.CreateBitCast(ivarAddr, CGF.Int8PtrTy);
args.add(RValue::get(ivarAddr), CGF.getContext().VoidPtrTy);
@@ -815,7 +877,7 @@
}
else {
ObjCIvarDecl *ivar = propImpl->getPropertyIvarDecl();
- emitCPPObjectAtomicGetterCall(*this, ReturnValue,
+ emitCPPObjectAtomicGetterCall(*this, ReturnValue.getPointer(),
ivar, AtomicHelperFn);
}
return;
@@ -845,10 +907,9 @@
bitcastType = bitcastType->getPointerTo(); // addrspace 0 okay
// Perform an atomic load. This does not impose ordering constraints.
- llvm::Value *ivarAddr = LV.getAddress();
+ Address ivarAddr = LV.getAddress();
ivarAddr = Builder.CreateBitCast(ivarAddr, bitcastType);
llvm::LoadInst *load = Builder.CreateLoad(ivarAddr, "load");
- load->setAlignment(strategy.getIvarAlignment().getQuantity());
load->setAtomic(llvm::Unordered);
// Store that value into the return address. Doing this with a
@@ -873,7 +934,7 @@
// FIXME: Can't this be simpler? This might even be worse than the
// corresponding gcc code.
llvm::Value *cmd =
- Builder.CreateLoad(LocalDeclMap[getterMethod->getCmdDecl()], "cmd");
+ Builder.CreateLoad(GetAddrOfLocalVar(getterMethod->getCmdDecl()), "cmd");
llvm::Value *self = Builder.CreateBitCast(LoadObjCSelf(), VoidPtrTy);
llvm::Value *ivarOffset =
EmitIvarOffset(classImpl->getClassInterface(), ivar);
@@ -888,11 +949,11 @@
// FIXME: We shouldn't need to get the function info here, the
// runtime already should have computed it to build the function.
llvm::Instruction *CallInstruction;
- RValue RV = EmitCall(getTypes().arrangeFreeFunctionCall(propType, args,
- FunctionType::ExtInfo(),
- RequiredArgs::All),
- getPropertyFn, ReturnValueSlot(), args, nullptr,
- &CallInstruction);
+ RValue RV = EmitCall(
+ getTypes().arrangeFreeFunctionCall(
+ propType, args, FunctionType::ExtInfo(), RequiredArgs::All),
+ getPropertyFn, ReturnValueSlot(), args, CGCalleeInfo(),
+ &CallInstruction);
if (llvm::CallInst *call = dyn_cast<llvm::CallInst>(CallInstruction))
call->setTailCall();
@@ -924,8 +985,7 @@
switch (getEvaluationKind(ivarType)) {
case TEK_Complex: {
ComplexPairTy pair = EmitLoadOfComplex(LV, SourceLocation());
- EmitStoreOfComplex(pair,
- MakeNaturalAlignAddrLValue(ReturnValue, ivarType),
+ EmitStoreOfComplex(pair, MakeAddrLValue(ReturnValue, ivarType),
/*init*/ true);
return;
}
@@ -938,11 +998,15 @@
case TEK_Scalar: {
llvm::Value *value;
if (propType->isReferenceType()) {
- value = LV.getAddress();
+ value = LV.getAddress().getPointer();
} else {
// We want to load and autoreleaseReturnValue ARC __weak ivars.
if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) {
- value = emitARCRetainLoadOfScalar(*this, LV, ivarType);
+ if (getLangOpts().ObjCAutoRefCount) {
+ value = emitARCRetainLoadOfScalar(*this, LV, ivarType);
+ } else {
+ value = EmitARCLoadWeak(LV.getAddress());
+ }
// Otherwise we want to do a simple load, suppressing the
// final autorelease.
@@ -978,7 +1042,7 @@
// The first argument is the address of the ivar.
llvm::Value *ivarAddr = CGF.EmitLValueForIvar(CGF.TypeOfSelfObject(),
CGF.LoadObjCSelf(), ivar, 0)
- .getAddress();
+ .getPointer();
ivarAddr = CGF.Builder.CreateBitCast(ivarAddr, CGF.Int8PtrTy);
args.add(RValue::get(ivarAddr), CGF.getContext().VoidPtrTy);
@@ -986,7 +1050,7 @@
ParmVarDecl *argVar = *OMD->param_begin();
DeclRefExpr argRef(argVar, false, argVar->getType().getNonReferenceType(),
VK_LValue, SourceLocation());
- llvm::Value *argAddr = CGF.EmitLValue(&argRef).getAddress();
+ llvm::Value *argAddr = CGF.EmitLValue(&argRef).getPointer();
argAddr = CGF.Builder.CreateBitCast(argAddr, CGF.Int8PtrTy);
args.add(RValue::get(argAddr), CGF.getContext().VoidPtrTy);
@@ -1024,7 +1088,7 @@
// The first argument is the address of the ivar.
llvm::Value *ivarAddr =
CGF.EmitLValueForIvar(CGF.TypeOfSelfObject(),
- CGF.LoadObjCSelf(), ivar, 0).getAddress();
+ CGF.LoadObjCSelf(), ivar, 0).getPointer();
ivarAddr = CGF.Builder.CreateBitCast(ivarAddr, CGF.Int8PtrTy);
args.add(RValue::get(ivarAddr), CGF.getContext().VoidPtrTy);
@@ -1032,7 +1096,7 @@
ParmVarDecl *argVar = *OMD->param_begin();
DeclRefExpr argRef(argVar, false, argVar->getType().getNonReferenceType(),
VK_LValue, SourceLocation());
- llvm::Value *argAddr = CGF.EmitLValue(&argRef).getAddress();
+ llvm::Value *argAddr = CGF.EmitLValue(&argRef).getPointer();
argAddr = CGF.Builder.CreateBitCast(argAddr, CGF.Int8PtrTy);
args.add(RValue::get(argAddr), CGF.getContext().VoidPtrTy);
@@ -1107,29 +1171,27 @@
if (strategy.getIvarSize().isZero())
return;
- llvm::Value *argAddr = LocalDeclMap[*setterMethod->param_begin()];
+ Address argAddr = GetAddrOfLocalVar(*setterMethod->param_begin());
LValue ivarLValue =
EmitLValueForIvar(TypeOfSelfObject(), LoadObjCSelf(), ivar, /*quals*/ 0);
- llvm::Value *ivarAddr = ivarLValue.getAddress();
+ Address ivarAddr = ivarLValue.getAddress();
// Currently, all atomic accesses have to be through integer
// types, so there's no point in trying to pick a prettier type.
llvm::Type *bitcastType =
llvm::Type::getIntNTy(getLLVMContext(),
getContext().toBits(strategy.getIvarSize()));
- bitcastType = bitcastType->getPointerTo(); // addrspace 0 okay
// Cast both arguments to the chosen operation type.
- argAddr = Builder.CreateBitCast(argAddr, bitcastType);
- ivarAddr = Builder.CreateBitCast(ivarAddr, bitcastType);
+ argAddr = Builder.CreateElementBitCast(argAddr, bitcastType);
+ ivarAddr = Builder.CreateElementBitCast(ivarAddr, bitcastType);
// This bitcast load is likely to cause some nasty IR.
llvm::Value *load = Builder.CreateLoad(argAddr);
// Perform an atomic store. There are no memory ordering requirements.
llvm::StoreInst *store = Builder.CreateStore(load, ivarAddr);
- store->setAlignment(strategy.getIvarAlignment().getQuantity());
store->setAtomic(llvm::Unordered);
return;
}
@@ -1161,13 +1223,14 @@
// Emit objc_setProperty((id) self, _cmd, offset, arg,
// <is-atomic>, <is-copy>).
llvm::Value *cmd =
- Builder.CreateLoad(LocalDeclMap[setterMethod->getCmdDecl()]);
+ Builder.CreateLoad(GetAddrOfLocalVar(setterMethod->getCmdDecl()));
llvm::Value *self =
Builder.CreateBitCast(LoadObjCSelf(), VoidPtrTy);
llvm::Value *ivarOffset =
EmitIvarOffset(classImpl->getClassInterface(), ivar);
- llvm::Value *arg = LocalDeclMap[*setterMethod->param_begin()];
- arg = Builder.CreateBitCast(Builder.CreateLoad(arg, "arg"), VoidPtrTy);
+ Address argAddr = GetAddrOfLocalVar(*setterMethod->param_begin());
+ llvm::Value *arg = Builder.CreateLoad(argAddr, "arg");
+ arg = Builder.CreateBitCast(arg, VoidPtrTy);
CallArgList args;
args.add(RValue::get(self), getContext().getObjCIdType());
@@ -1276,7 +1339,7 @@
}
namespace {
- struct DestroyIvar : EHScopeStack::Cleanup {
+ struct DestroyIvar final : EHScopeStack::Cleanup {
private:
llvm::Value *addr;
const ObjCIvarDecl *ivar;
@@ -1300,7 +1363,7 @@
/// Like CodeGenFunction::destroyARCStrong, but do it with a call.
static void destroyARCStrongWithStore(CodeGenFunction &CGF,
- llvm::Value *addr,
+ Address addr,
QualType type) {
llvm::Value *null = getNullForVariable(addr);
CGF.EmitARCStoreStrongCall(addr, null, /*ignored*/ true);
@@ -1377,22 +1440,6 @@
FinishFunction();
}
-bool CodeGenFunction::IndirectObjCSetterArg(const CGFunctionInfo &FI) {
- CGFunctionInfo::const_arg_iterator it = FI.arg_begin();
- it++; it++;
- const ABIArgInfo &AI = it->info;
- // FIXME. Is this sufficient check?
- return (AI.getKind() == ABIArgInfo::Indirect);
-}
-
-bool CodeGenFunction::IvarTypeWithAggrGCObjects(QualType Ty) {
- if (CGM.getLangOpts().getGC() == LangOptions::NonGC)
- return false;
- if (const RecordType *FDTTy = Ty.getTypePtr()->getAs<RecordType>())
- return FDTTy->getDecl()->hasObjectMember();
- return false;
-}
-
llvm::Value *CodeGenFunction::LoadObjCSelf() {
VarDecl *Self = cast<ObjCMethodDecl>(CurFuncDecl)->getSelfDecl();
DeclRefExpr DRE(Self, /*is enclosing local*/ (CurFuncDecl != CurCodeDecl),
@@ -1430,7 +1477,7 @@
// Fast enumeration state.
QualType StateTy = CGM.getObjCFastEnumerationStateType();
- llvm::AllocaInst *StatePtr = CreateMemTemp(StateTy, "state.ptr");
+ Address StatePtr = CreateMemTemp(StateTy, "state.ptr");
EmitNullInitialization(StatePtr, StateTy);
// Number of elements in the items array.
@@ -1449,7 +1496,7 @@
getContext().getConstantArrayType(getContext().getObjCIdType(),
llvm::APInt(32, NumItems),
ArrayType::Normal, 0);
- llvm::Value *ItemsPtr = CreateMemTemp(ItemsTy, "items.ptr");
+ Address ItemsPtr = CreateMemTemp(ItemsTy, "items.ptr");
// Emit the collection pointer. In ARC, we do a retain.
llvm::Value *Collection;
@@ -1470,14 +1517,16 @@
CallArgList Args;
// The first argument is a temporary of the enumeration-state type.
- Args.add(RValue::get(StatePtr), getContext().getPointerType(StateTy));
+ Args.add(RValue::get(StatePtr.getPointer()),
+ getContext().getPointerType(StateTy));
// The second argument is a temporary array with space for NumItems
// pointers. We'll actually be loading elements from the array
// pointer written into the control state; this buffer is so that
// collections that *aren't* backed by arrays can still queue up
// batches of elements.
- Args.add(RValue::get(ItemsPtr), getContext().getPointerType(ItemsTy));
+ Args.add(RValue::get(ItemsPtr.getPointer()),
+ getContext().getPointerType(ItemsTy));
// The third argument is the capacity of that temporary array.
llvm::Type *UnsignedLongLTy = ConvertType(getContext().UnsignedLongTy);
@@ -1514,13 +1563,14 @@
// Save the initial mutations value. This is the value at an
// address that was written into the state object by
// countByEnumeratingWithState:objects:count:.
- llvm::Value *StateMutationsPtrPtr = Builder.CreateStructGEP(
- StatePtr->getAllocatedType(), StatePtr, 2, "mutationsptr.ptr");
- llvm::Value *StateMutationsPtr = Builder.CreateLoad(StateMutationsPtrPtr,
- "mutationsptr");
+ Address StateMutationsPtrPtr = Builder.CreateStructGEP(
+ StatePtr, 2, 2 * getPointerSize(), "mutationsptr.ptr");
+ llvm::Value *StateMutationsPtr
+ = Builder.CreateLoad(StateMutationsPtrPtr, "mutationsptr");
llvm::Value *initialMutations =
- Builder.CreateLoad(StateMutationsPtr, "forcoll.initial-mutations");
+ Builder.CreateAlignedLoad(StateMutationsPtr, getPointerAlign(),
+ "forcoll.initial-mutations");
// Start looping. This is the point we return to whenever we have a
// fresh, non-empty batch of objects.
@@ -1542,7 +1592,8 @@
// refreshes.
StateMutationsPtr = Builder.CreateLoad(StateMutationsPtrPtr, "mutationsptr");
llvm::Value *currentMutations
- = Builder.CreateLoad(StateMutationsPtr, "statemutations");
+ = Builder.CreateAlignedLoad(StateMutationsPtr, getPointerAlign(),
+ "statemutations");
llvm::BasicBlock *WasMutatedBB = createBasicBlock("forcoll.mutated");
llvm::BasicBlock *WasNotMutatedBB = createBasicBlock("forcoll.notmutated");
@@ -1595,15 +1646,16 @@
// Fetch the buffer out of the enumeration state.
// TODO: this pointer should actually be invariant between
// refreshes, which would help us do certain loop optimizations.
- llvm::Value *StateItemsPtr = Builder.CreateStructGEP(
- StatePtr->getAllocatedType(), StatePtr, 1, "stateitems.ptr");
+ Address StateItemsPtr = Builder.CreateStructGEP(
+ StatePtr, 1, getPointerSize(), "stateitems.ptr");
llvm::Value *EnumStateItems =
Builder.CreateLoad(StateItemsPtr, "stateitems");
// Fetch the value at the current index from the buffer.
llvm::Value *CurrentItemPtr =
Builder.CreateGEP(EnumStateItems, index, "currentitem.ptr");
- llvm::Value *CurrentItem = Builder.CreateLoad(CurrentItemPtr);
+ llvm::Value *CurrentItem =
+ Builder.CreateAlignedLoad(CurrentItemPtr, getPointerAlign());
// Cast that value to the right type.
CurrentItem = Builder.CreateBitCast(CurrentItem, convertedElementType,
@@ -1707,15 +1759,8 @@
CGM.getObjCRuntime().EmitSynchronizedStmt(*this, S);
}
-/// Produce the code for a CK_ARCProduceObject. Just does a
-/// primitive retain.
-llvm::Value *CodeGenFunction::EmitObjCProduceObject(QualType type,
- llvm::Value *value) {
- return EmitARCRetain(type, value);
-}
-
namespace {
- struct CallObjCRelease : EHScopeStack::Cleanup {
+ struct CallObjCRelease final : EHScopeStack::Cleanup {
CallObjCRelease(llvm::Value *object) : object(object) {}
llvm::Value *object;
@@ -1744,7 +1789,7 @@
/// Given a number of pointers, inform the optimizer that they're
/// being intrinsically used up until this point in the program.
void CodeGenFunction::EmitARCIntrinsicUse(ArrayRef<llvm::Value*> values) {
- llvm::Constant *&fn = CGM.getARCEntrypoints().clang_arc_use;
+ llvm::Constant *&fn = CGM.getObjCEntrypoints().clang_arc_use;
if (!fn) {
llvm::FunctionType *fnType =
llvm::FunctionType::get(CGM.VoidTy, None, true);
@@ -1810,7 +1855,7 @@
/// Perform an operation having the following signature:
/// i8* (i8**)
static llvm::Value *emitARCLoadOperation(CodeGenFunction &CGF,
- llvm::Value *addr,
+ Address addr,
llvm::Constant *&fn,
StringRef fnName) {
if (!fn) {
@@ -1820,16 +1865,15 @@
}
// Cast the argument to 'id*'.
- llvm::Type *origType = addr->getType();
+ llvm::Type *origType = addr.getElementType();
addr = CGF.Builder.CreateBitCast(addr, CGF.Int8PtrPtrTy);
// Call the function.
- llvm::Value *result = CGF.EmitNounwindRuntimeCall(fn, addr);
+ llvm::Value *result = CGF.EmitNounwindRuntimeCall(fn, addr.getPointer());
// Cast the result back to a dereference of the original type.
- if (origType != CGF.Int8PtrPtrTy)
- result = CGF.Builder.CreateBitCast(result,
- cast<llvm::PointerType>(origType)->getElementType());
+ if (origType != CGF.Int8PtrTy)
+ result = CGF.Builder.CreateBitCast(result, origType);
return result;
}
@@ -1837,13 +1881,12 @@
/// Perform an operation having the following signature:
/// i8* (i8**, i8*)
static llvm::Value *emitARCStoreOperation(CodeGenFunction &CGF,
- llvm::Value *addr,
+ Address addr,
llvm::Value *value,
llvm::Constant *&fn,
StringRef fnName,
bool ignored) {
- assert(cast<llvm::PointerType>(addr->getType())->getElementType()
- == value->getType());
+ assert(addr.getElementType() == value->getType());
if (!fn) {
llvm::Type *argTypes[] = { CGF.Int8PtrPtrTy, CGF.Int8PtrTy };
@@ -1856,7 +1899,7 @@
llvm::Type *origType = value->getType();
llvm::Value *args[] = {
- CGF.Builder.CreateBitCast(addr, CGF.Int8PtrPtrTy),
+ CGF.Builder.CreateBitCast(addr.getPointer(), CGF.Int8PtrPtrTy),
CGF.Builder.CreateBitCast(value, CGF.Int8PtrTy)
};
llvm::CallInst *result = CGF.EmitNounwindRuntimeCall(fn, args);
@@ -1869,11 +1912,11 @@
/// Perform an operation having the following signature:
/// void (i8**, i8**)
static void emitARCCopyOperation(CodeGenFunction &CGF,
- llvm::Value *dst,
- llvm::Value *src,
+ Address dst,
+ Address src,
llvm::Constant *&fn,
StringRef fnName) {
- assert(dst->getType() == src->getType());
+ assert(dst.getType() == src.getType());
if (!fn) {
llvm::Type *argTypes[] = { CGF.Int8PtrPtrTy, CGF.Int8PtrPtrTy };
@@ -1884,8 +1927,8 @@
}
llvm::Value *args[] = {
- CGF.Builder.CreateBitCast(dst, CGF.Int8PtrPtrTy),
- CGF.Builder.CreateBitCast(src, CGF.Int8PtrPtrTy)
+ CGF.Builder.CreateBitCast(dst.getPointer(), CGF.Int8PtrPtrTy),
+ CGF.Builder.CreateBitCast(src.getPointer(), CGF.Int8PtrPtrTy)
};
CGF.EmitNounwindRuntimeCall(fn, args);
}
@@ -1904,7 +1947,7 @@
/// call i8* \@objc_retain(i8* %value)
llvm::Value *CodeGenFunction::EmitARCRetainNonBlock(llvm::Value *value) {
return emitARCValueOperation(*this, value,
- CGM.getARCEntrypoints().objc_retain,
+ CGM.getObjCEntrypoints().objc_retain,
"objc_retain");
}
@@ -1918,7 +1961,7 @@
bool mandatory) {
llvm::Value *result
= emitARCValueOperation(*this, value,
- CGM.getARCEntrypoints().objc_retainBlock,
+ CGM.getObjCEntrypoints().objc_retainBlock,
"objc_retainBlock");
// If the copy isn't mandatory, add !clang.arc.copy_on_escape to
@@ -1928,7 +1971,7 @@
if (!mandatory && isa<llvm::Instruction>(result)) {
llvm::CallInst *call
= cast<llvm::CallInst>(result->stripPointerCasts());
- assert(call->getCalledValue() == CGM.getARCEntrypoints().objc_retainBlock);
+ assert(call->getCalledValue() == CGM.getObjCEntrypoints().objc_retainBlock);
call->setMetadata("clang.arc.copy_on_escape",
llvm::MDNode::get(Builder.getContext(), None));
@@ -1947,7 +1990,7 @@
// Fetch the void(void) inline asm which marks that we're going to
// retain the autoreleased return value.
llvm::InlineAsm *&marker
- = CGM.getARCEntrypoints().retainAutoreleasedReturnValueMarker;
+ = CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker;
if (!marker) {
StringRef assembly
= CGM.getTargetCodeGenInfo()
@@ -1981,10 +2024,10 @@
// Call the marker asm if we made one, which we do only at -O0.
if (marker)
- Builder.CreateCall(marker, {});
+ Builder.CreateCall(marker);
return emitARCValueOperation(*this, value,
- CGM.getARCEntrypoints().objc_retainAutoreleasedReturnValue,
+ CGM.getObjCEntrypoints().objc_retainAutoreleasedReturnValue,
"objc_retainAutoreleasedReturnValue");
}
@@ -1994,7 +2037,7 @@
ARCPreciseLifetime_t precise) {
if (isa<llvm::ConstantPointerNull>(value)) return;
- llvm::Constant *&fn = CGM.getARCEntrypoints().objc_release;
+ llvm::Constant *&fn = CGM.getObjCEntrypoints().objc_release;
if (!fn) {
llvm::FunctionType *fnType =
llvm::FunctionType::get(Builder.getVoidTy(), Int8PtrTy, false);
@@ -2022,12 +2065,10 @@
/// At -O1 and above, just load and call objc_release.
///
/// call void \@objc_storeStrong(i8** %addr, i8* null)
-void CodeGenFunction::EmitARCDestroyStrong(llvm::Value *addr,
+void CodeGenFunction::EmitARCDestroyStrong(Address addr,
ARCPreciseLifetime_t precise) {
if (CGM.getCodeGenOpts().OptimizationLevel == 0) {
- llvm::PointerType *addrTy = cast<llvm::PointerType>(addr->getType());
- llvm::Value *null = llvm::ConstantPointerNull::get(
- cast<llvm::PointerType>(addrTy->getElementType()));
+ llvm::Value *null = getNullForVariable(addr);
EmitARCStoreStrongCall(addr, null, /*ignored*/ true);
return;
}
@@ -2038,13 +2079,12 @@
/// Store into a strong object. Always calls this:
/// call void \@objc_storeStrong(i8** %addr, i8* %value)
-llvm::Value *CodeGenFunction::EmitARCStoreStrongCall(llvm::Value *addr,
+llvm::Value *CodeGenFunction::EmitARCStoreStrongCall(Address addr,
llvm::Value *value,
bool ignored) {
- assert(cast<llvm::PointerType>(addr->getType())->getElementType()
- == value->getType());
+ assert(addr.getElementType() == value->getType());
- llvm::Constant *&fn = CGM.getARCEntrypoints().objc_storeStrong;
+ llvm::Constant *&fn = CGM.getObjCEntrypoints().objc_storeStrong;
if (!fn) {
llvm::Type *argTypes[] = { Int8PtrPtrTy, Int8PtrTy };
llvm::FunctionType *fnType
@@ -2053,7 +2093,7 @@
}
llvm::Value *args[] = {
- Builder.CreateBitCast(addr, Int8PtrPtrTy),
+ Builder.CreateBitCast(addr.getPointer(), Int8PtrPtrTy),
Builder.CreateBitCast(value, Int8PtrTy)
};
EmitNounwindRuntimeCall(fn, args);
@@ -2102,7 +2142,7 @@
/// call i8* \@objc_autorelease(i8* %value)
llvm::Value *CodeGenFunction::EmitARCAutorelease(llvm::Value *value) {
return emitARCValueOperation(*this, value,
- CGM.getARCEntrypoints().objc_autorelease,
+ CGM.getObjCEntrypoints().objc_autorelease,
"objc_autorelease");
}
@@ -2111,7 +2151,7 @@
llvm::Value *
CodeGenFunction::EmitARCAutoreleaseReturnValue(llvm::Value *value) {
return emitARCValueOperation(*this, value,
- CGM.getARCEntrypoints().objc_autoreleaseReturnValue,
+ CGM.getObjCEntrypoints().objc_autoreleaseReturnValue,
"objc_autoreleaseReturnValue",
/*isTailCall*/ true);
}
@@ -2121,7 +2161,7 @@
llvm::Value *
CodeGenFunction::EmitARCRetainAutoreleaseReturnValue(llvm::Value *value) {
return emitARCValueOperation(*this, value,
- CGM.getARCEntrypoints().objc_retainAutoreleaseReturnValue,
+ CGM.getObjCEntrypoints().objc_retainAutoreleaseReturnValue,
"objc_retainAutoreleaseReturnValue",
/*isTailCall*/ true);
}
@@ -2150,32 +2190,32 @@
llvm::Value *
CodeGenFunction::EmitARCRetainAutoreleaseNonBlock(llvm::Value *value) {
return emitARCValueOperation(*this, value,
- CGM.getARCEntrypoints().objc_retainAutorelease,
+ CGM.getObjCEntrypoints().objc_retainAutorelease,
"objc_retainAutorelease");
}
/// i8* \@objc_loadWeak(i8** %addr)
/// Essentially objc_autorelease(objc_loadWeakRetained(addr)).
-llvm::Value *CodeGenFunction::EmitARCLoadWeak(llvm::Value *addr) {
+llvm::Value *CodeGenFunction::EmitARCLoadWeak(Address addr) {
return emitARCLoadOperation(*this, addr,
- CGM.getARCEntrypoints().objc_loadWeak,
+ CGM.getObjCEntrypoints().objc_loadWeak,
"objc_loadWeak");
}
/// i8* \@objc_loadWeakRetained(i8** %addr)
-llvm::Value *CodeGenFunction::EmitARCLoadWeakRetained(llvm::Value *addr) {
+llvm::Value *CodeGenFunction::EmitARCLoadWeakRetained(Address addr) {
return emitARCLoadOperation(*this, addr,
- CGM.getARCEntrypoints().objc_loadWeakRetained,
+ CGM.getObjCEntrypoints().objc_loadWeakRetained,
"objc_loadWeakRetained");
}
/// i8* \@objc_storeWeak(i8** %addr, i8* %value)
/// Returns %value.
-llvm::Value *CodeGenFunction::EmitARCStoreWeak(llvm::Value *addr,
+llvm::Value *CodeGenFunction::EmitARCStoreWeak(Address addr,
llvm::Value *value,
bool ignored) {
return emitARCStoreOperation(*this, addr, value,
- CGM.getARCEntrypoints().objc_storeWeak,
+ CGM.getObjCEntrypoints().objc_storeWeak,
"objc_storeWeak", ignored);
}
@@ -2183,7 +2223,7 @@
/// Returns %value. %addr is known to not have a current weak entry.
/// Essentially equivalent to:
/// *addr = nil; objc_storeWeak(addr, value);
-void CodeGenFunction::EmitARCInitWeak(llvm::Value *addr, llvm::Value *value) {
+void CodeGenFunction::EmitARCInitWeak(Address addr, llvm::Value *value) {
// If we're initializing to null, just write null to memory; no need
// to get the runtime involved. But don't do this if optimization
// is enabled, because accounting for this would make the optimizer
@@ -2195,14 +2235,14 @@
}
emitARCStoreOperation(*this, addr, value,
- CGM.getARCEntrypoints().objc_initWeak,
+ CGM.getObjCEntrypoints().objc_initWeak,
"objc_initWeak", /*ignored*/ true);
}
/// void \@objc_destroyWeak(i8** %addr)
/// Essentially objc_storeWeak(addr, nil).
-void CodeGenFunction::EmitARCDestroyWeak(llvm::Value *addr) {
- llvm::Constant *&fn = CGM.getARCEntrypoints().objc_destroyWeak;
+void CodeGenFunction::EmitARCDestroyWeak(Address addr) {
+ llvm::Constant *&fn = CGM.getObjCEntrypoints().objc_destroyWeak;
if (!fn) {
llvm::FunctionType *fnType =
llvm::FunctionType::get(Builder.getVoidTy(), Int8PtrPtrTy, false);
@@ -2212,31 +2252,31 @@
// Cast the argument to 'id*'.
addr = Builder.CreateBitCast(addr, Int8PtrPtrTy);
- EmitNounwindRuntimeCall(fn, addr);
+ EmitNounwindRuntimeCall(fn, addr.getPointer());
}
/// void \@objc_moveWeak(i8** %dest, i8** %src)
/// Disregards the current value in %dest. Leaves %src pointing to nothing.
/// Essentially (objc_copyWeak(dest, src), objc_destroyWeak(src)).
-void CodeGenFunction::EmitARCMoveWeak(llvm::Value *dst, llvm::Value *src) {
+void CodeGenFunction::EmitARCMoveWeak(Address dst, Address src) {
emitARCCopyOperation(*this, dst, src,
- CGM.getARCEntrypoints().objc_moveWeak,
+ CGM.getObjCEntrypoints().objc_moveWeak,
"objc_moveWeak");
}
/// void \@objc_copyWeak(i8** %dest, i8** %src)
/// Disregards the current value in %dest. Essentially
/// objc_release(objc_initWeak(dest, objc_readWeakRetained(src)))
-void CodeGenFunction::EmitARCCopyWeak(llvm::Value *dst, llvm::Value *src) {
+void CodeGenFunction::EmitARCCopyWeak(Address dst, Address src) {
emitARCCopyOperation(*this, dst, src,
- CGM.getARCEntrypoints().objc_copyWeak,
+ CGM.getObjCEntrypoints().objc_copyWeak,
"objc_copyWeak");
}
/// Produce the code to do a objc_autoreleasepool_push.
/// call i8* \@objc_autoreleasePoolPush(void)
llvm::Value *CodeGenFunction::EmitObjCAutoreleasePoolPush() {
- llvm::Constant *&fn = CGM.getRREntrypoints().objc_autoreleasePoolPush;
+ llvm::Constant *&fn = CGM.getObjCEntrypoints().objc_autoreleasePoolPush;
if (!fn) {
llvm::FunctionType *fnType =
llvm::FunctionType::get(Int8PtrTy, false);
@@ -2251,7 +2291,7 @@
void CodeGenFunction::EmitObjCAutoreleasePoolPop(llvm::Value *value) {
assert(value->getType() == Int8PtrTy);
- llvm::Constant *&fn = CGM.getRREntrypoints().objc_autoreleasePoolPop;
+ llvm::Constant *&fn = CGM.getObjCEntrypoints().objc_autoreleasePoolPop;
if (!fn) {
llvm::FunctionType *fnType =
llvm::FunctionType::get(Builder.getVoidTy(), Int8PtrTy, false);
@@ -2304,25 +2344,25 @@
}
void CodeGenFunction::destroyARCStrongPrecise(CodeGenFunction &CGF,
- llvm::Value *addr,
+ Address addr,
QualType type) {
CGF.EmitARCDestroyStrong(addr, ARCPreciseLifetime);
}
void CodeGenFunction::destroyARCStrongImprecise(CodeGenFunction &CGF,
- llvm::Value *addr,
+ Address addr,
QualType type) {
CGF.EmitARCDestroyStrong(addr, ARCImpreciseLifetime);
}
void CodeGenFunction::destroyARCWeak(CodeGenFunction &CGF,
- llvm::Value *addr,
+ Address addr,
QualType type) {
CGF.EmitARCDestroyWeak(addr);
}
namespace {
- struct CallObjCAutoreleasePoolObject : EHScopeStack::Cleanup {
+ struct CallObjCAutoreleasePoolObject final : EHScopeStack::Cleanup {
llvm::Value *Token;
CallObjCAutoreleasePoolObject(llvm::Value *token) : Token(token) {}
@@ -2331,7 +2371,7 @@
CGF.EmitObjCAutoreleasePoolPop(Token);
}
};
- struct CallObjCMRRAutoreleasePoolObject : EHScopeStack::Cleanup {
+ struct CallObjCMRRAutoreleasePoolObject final : EHScopeStack::Cleanup {
llvm::Value *Token;
CallObjCMRRAutoreleasePoolObject(llvm::Value *token) : Token(token) {}
@@ -2904,7 +2944,9 @@
llvm::Function::Create(LTy, llvm::GlobalValue::InternalLinkage,
"__assign_helper_atomic_property_",
&CGM.getModule());
-
+
+ CGM.SetInternalFunctionAttributes(nullptr, Fn, FI);
+
StartFunction(FD, C.VoidTy, Fn, FI, args);
DeclRefExpr DstExpr(&dstDecl, false, DestTy,
@@ -2983,6 +3025,8 @@
llvm::Function::Create(LTy, llvm::GlobalValue::InternalLinkage,
"__copy_helper_atomic_property_", &CGM.getModule());
+ CGM.SetInternalFunctionAttributes(nullptr, Fn, FI);
+
StartFunction(FD, C.VoidTy, Fn, FI, args);
DeclRefExpr SrcExpr(&srcDecl, false, SrcTy,
@@ -3018,7 +3062,8 @@
CharUnits Alignment
= getContext().getTypeAlignInChars(TheCXXConstructExpr->getType());
EmitAggExpr(TheCXXConstructExpr,
- AggValueSlot::forAddr(DV.getScalarVal(), Alignment, Qualifiers(),
+ AggValueSlot::forAddr(Address(DV.getScalarVal(), Alignment),
+ Qualifiers(),
AggValueSlot::IsDestructed,
AggValueSlot::DoesNotNeedGCBarriers,
AggValueSlot::IsNotAliased));
diff --git a/lib/CodeGen/CGObjCGNU.cpp b/lib/CodeGen/CGObjCGNU.cpp
index b52d623..f0af3e9 100644
--- a/lib/CodeGen/CGObjCGNU.cpp
+++ b/lib/CodeGen/CGObjCGNU.cpp
@@ -166,9 +166,9 @@
/// where the C code specifies const char*.
llvm::Constant *MakeConstantString(const std::string &Str,
const std::string &Name="") {
- auto *ConstStr = CGM.GetAddrOfConstantCString(Str, Name.c_str());
- return llvm::ConstantExpr::getGetElementPtr(ConstStr->getValueType(),
- ConstStr, Zeros);
+ ConstantAddress Array = CGM.GetAddrOfConstantCString(Str, Name.c_str());
+ return llvm::ConstantExpr::getGetElementPtr(Array.getElementType(),
+ Array.getPointer(), Zeros);
}
/// Emits a linkonce_odr string, whose name is the prefix followed by the
/// string value. This allows the linker to combine the strings between
@@ -191,34 +191,41 @@
/// first argument.
llvm::GlobalVariable *MakeGlobal(llvm::StructType *Ty,
ArrayRef<llvm::Constant *> V,
+ CharUnits Align,
StringRef Name="",
llvm::GlobalValue::LinkageTypes linkage
=llvm::GlobalValue::InternalLinkage) {
llvm::Constant *C = llvm::ConstantStruct::get(Ty, V);
- return new llvm::GlobalVariable(TheModule, Ty, false,
- linkage, C, Name);
+ auto GV = new llvm::GlobalVariable(TheModule, Ty, false,
+ linkage, C, Name);
+ GV->setAlignment(Align.getQuantity());
+ return GV;
}
/// Generates a global array. The vector must contain the same number of
/// elements that the array type declares, of the type specified as the array
/// element type.
llvm::GlobalVariable *MakeGlobal(llvm::ArrayType *Ty,
ArrayRef<llvm::Constant *> V,
+ CharUnits Align,
StringRef Name="",
llvm::GlobalValue::LinkageTypes linkage
=llvm::GlobalValue::InternalLinkage) {
llvm::Constant *C = llvm::ConstantArray::get(Ty, V);
- return new llvm::GlobalVariable(TheModule, Ty, false,
- linkage, C, Name);
+ auto GV = new llvm::GlobalVariable(TheModule, Ty, false,
+ linkage, C, Name);
+ GV->setAlignment(Align.getQuantity());
+ return GV;
}
/// Generates a global array, inferring the array type from the specified
/// element type and the size of the initialiser.
llvm::GlobalVariable *MakeGlobalArray(llvm::Type *Ty,
ArrayRef<llvm::Constant *> V,
+ CharUnits Align,
StringRef Name="",
llvm::GlobalValue::LinkageTypes linkage
=llvm::GlobalValue::InternalLinkage) {
llvm::ArrayType *ArrayTy = llvm::ArrayType::get(Ty, V.size());
- return MakeGlobal(ArrayTy, V, Name, linkage);
+ return MakeGlobal(ArrayTy, V, Align, Name, linkage);
}
/// Returns a property name and encoding string.
llvm::Constant *MakePropertyEncodingString(const ObjCPropertyDecl *PD,
@@ -234,9 +241,7 @@
NameAndAttributes += TypeStr;
NameAndAttributes += '\0';
NameAndAttributes += PD->getNameAsString();
- auto *ConstStr = CGM.GetAddrOfConstantCString(NameAndAttributes);
- return llvm::ConstantExpr::getGetElementPtr(ConstStr->getValueType(),
- ConstStr, Zeros);
+ return MakeConstantString(NameAndAttributes);
}
return MakeConstantString(PD->getNameAsString());
}
@@ -275,6 +280,10 @@
if (V->getType() == Ty) return V;
return B.CreateBitCast(V, Ty);
}
+ Address EnforceType(CGBuilderTy &B, Address V, llvm::Type *Ty) {
+ if (V.getType() == Ty) return V;
+ return B.CreateBitCast(V, Ty);
+ }
// Some zeros used for GEPs in lots of places.
llvm::Constant *Zeros[2];
/// Null pointer value. Mainly used as a terminator in various arrays.
@@ -435,7 +444,7 @@
/// Returns a selector with the specified type encoding. An empty string is
/// used to return an untyped selector (with the types field set to NULL).
llvm::Value *GetSelector(CodeGenFunction &CGF, Selector Sel,
- const std::string &TypeEncoding, bool lval);
+ const std::string &TypeEncoding);
/// Returns the variable used to store the offset of an instance variable.
llvm::GlobalVariable *ObjCIvarOffsetVariable(const ObjCInterfaceDecl *ID,
const ObjCIvarDecl *Ivar);
@@ -458,7 +467,7 @@
/// mechanism differs between the GCC and GNU runtimes, so this method must
/// be overridden in subclasses.
virtual llvm::Value *LookupIMPSuper(CodeGenFunction &CGF,
- llvm::Value *ObjCSuper,
+ Address ObjCSuper,
llvm::Value *cmd,
MessageSendInfo &MSI) = 0;
/// Libobjc2 uses a bitfield representation where small(ish) bitfields are
@@ -477,7 +486,7 @@
CGObjCGNU(CodeGenModule &cgm, unsigned runtimeABIVersion,
unsigned protocolClassVersion);
- llvm::Constant *GenerateConstantString(const StringLiteral *) override;
+ ConstantAddress GenerateConstantString(const StringLiteral *) override;
RValue
GenerateMessageSend(CodeGenFunction &CGF, ReturnValueSlot Return,
@@ -494,8 +503,8 @@
const ObjCMethodDecl *Method) override;
llvm::Value *GetClass(CodeGenFunction &CGF,
const ObjCInterfaceDecl *OID) override;
- llvm::Value *GetSelector(CodeGenFunction &CGF, Selector Sel,
- bool lval = false) override;
+ llvm::Value *GetSelector(CodeGenFunction &CGF, Selector Sel) override;
+ Address GetAddrOfSelector(CodeGenFunction &CGF, Selector Sel) override;
llvm::Value *GetSelector(CodeGenFunction &CGF,
const ObjCMethodDecl *Method) override;
llvm::Constant *GetEHType(QualType T) override;
@@ -527,18 +536,18 @@
const ObjCAtThrowStmt &S,
bool ClearInsertionPoint=true) override;
llvm::Value * EmitObjCWeakRead(CodeGenFunction &CGF,
- llvm::Value *AddrWeakObj) override;
+ Address AddrWeakObj) override;
void EmitObjCWeakAssign(CodeGenFunction &CGF,
- llvm::Value *src, llvm::Value *dst) override;
+ llvm::Value *src, Address dst) override;
void EmitObjCGlobalAssign(CodeGenFunction &CGF,
- llvm::Value *src, llvm::Value *dest,
+ llvm::Value *src, Address dest,
bool threadlocal=false) override;
void EmitObjCIvarAssign(CodeGenFunction &CGF, llvm::Value *src,
- llvm::Value *dest, llvm::Value *ivarOffset) override;
+ Address dest, llvm::Value *ivarOffset) override;
void EmitObjCStrongCastAssign(CodeGenFunction &CGF,
- llvm::Value *src, llvm::Value *dest) override;
- void EmitGCMemmoveCollectable(CodeGenFunction &CGF, llvm::Value *DestPtr,
- llvm::Value *SrcPtr,
+ llvm::Value *src, Address dest) override;
+ void EmitGCMemmoveCollectable(CodeGenFunction &CGF, Address DestPtr,
+ Address SrcPtr,
llvm::Value *Size) override;
LValue EmitObjCValueForIvar(CodeGenFunction &CGF, QualType ObjectTy,
llvm::Value *BaseValue, const ObjCIvarDecl *Ivar,
@@ -593,11 +602,11 @@
imp->setMetadata(msgSendMDKind, node);
return imp.getInstruction();
}
- llvm::Value *LookupIMPSuper(CodeGenFunction &CGF, llvm::Value *ObjCSuper,
+ llvm::Value *LookupIMPSuper(CodeGenFunction &CGF, Address ObjCSuper,
llvm::Value *cmd, MessageSendInfo &MSI) override {
CGBuilderTy &Builder = CGF.Builder;
llvm::Value *lookupArgs[] = {EnforceType(Builder, ObjCSuper,
- PtrToObjCSuperTy), cmd};
+ PtrToObjCSuperTy).getPointer(), cmd};
return CGF.EmitNounwindRuntimeCall(MsgLookupSuperFn, lookupArgs);
}
public:
@@ -647,7 +656,8 @@
llvm::Function *LookupFn = SlotLookupFn;
// Store the receiver on the stack so that we can reload it later
- llvm::Value *ReceiverPtr = CGF.CreateTempAlloca(Receiver->getType());
+ Address ReceiverPtr =
+ CGF.CreateTempAlloca(Receiver->getType(), CGF.getPointerAlign());
Builder.CreateStore(Receiver, ReceiverPtr);
llvm::Value *self;
@@ -662,7 +672,7 @@
LookupFn->setDoesNotCapture(1);
llvm::Value *args[] = {
- EnforceType(Builder, ReceiverPtr, PtrToIdTy),
+ EnforceType(Builder, ReceiverPtr.getPointer(), PtrToIdTy),
EnforceType(Builder, cmd, SelectorTy),
EnforceType(Builder, self, IdTy) };
llvm::CallSite slot = CGF.EmitRuntimeCallOrInvoke(LookupFn, args);
@@ -670,25 +680,27 @@
slot->setMetadata(msgSendMDKind, node);
// Load the imp from the slot
- llvm::Value *imp = Builder.CreateLoad(
- Builder.CreateStructGEP(nullptr, slot.getInstruction(), 4));
+ llvm::Value *imp = Builder.CreateAlignedLoad(
+ Builder.CreateStructGEP(nullptr, slot.getInstruction(), 4),
+ CGF.getPointerAlign());
// The lookup function may have changed the receiver, so make sure we use
// the new one.
Receiver = Builder.CreateLoad(ReceiverPtr, true);
return imp;
}
- llvm::Value *LookupIMPSuper(CodeGenFunction &CGF, llvm::Value *ObjCSuper,
+ llvm::Value *LookupIMPSuper(CodeGenFunction &CGF, Address ObjCSuper,
llvm::Value *cmd,
MessageSendInfo &MSI) override {
CGBuilderTy &Builder = CGF.Builder;
- llvm::Value *lookupArgs[] = {ObjCSuper, cmd};
+ llvm::Value *lookupArgs[] = {ObjCSuper.getPointer(), cmd};
llvm::CallInst *slot =
CGF.EmitNounwindRuntimeCall(SlotLookupSuperFn, lookupArgs);
slot->setOnlyReadsMemory();
- return Builder.CreateLoad(Builder.CreateStructGEP(nullptr, slot, 4));
+ return Builder.CreateAlignedLoad(Builder.CreateStructGEP(nullptr, slot, 4),
+ CGF.getPointerAlign());
}
public:
CGObjCGNUstep(CodeGenModule &Mod) : CGObjCGNU(Mod, 9, 3) {
@@ -807,10 +819,10 @@
return imp.getInstruction();
}
- llvm::Value *LookupIMPSuper(CodeGenFunction &CGF, llvm::Value *ObjCSuper,
+ llvm::Value *LookupIMPSuper(CodeGenFunction &CGF, Address ObjCSuper,
llvm::Value *cmd, MessageSendInfo &MSI) override {
CGBuilderTy &Builder = CGF.Builder;
- llvm::Value *lookupArgs[] = {EnforceType(Builder, ObjCSuper,
+ llvm::Value *lookupArgs[] = {EnforceType(Builder, ObjCSuper.getPointer(),
PtrToObjCSuperTy), cmd};
if (CGM.ReturnTypeUsesSRet(MSI.CallInfo))
@@ -1011,7 +1023,7 @@
llvm::Value *CGObjCGNU::GetClassNamed(CodeGenFunction &CGF,
const std::string &Name,
bool isWeak) {
- llvm::GlobalVariable *ClassNameGV = CGM.GetAddrOfConstantCString(Name);
+ llvm::Constant *ClassName = MakeConstantString(Name);
// With the incompatible ABI, this will need to be replaced with a direct
// reference to the class symbol. For the compatible nonfragile ABI we are
// still performing this lookup at run time but emitting the symbol for the
@@ -1021,8 +1033,6 @@
// with memoized versions or with static references if it's safe to do so.
if (!isWeak)
EmitClassRef(Name);
- llvm::Value *ClassName =
- CGF.Builder.CreateStructGEP(ClassNameGV->getValueType(), ClassNameGV, 0);
llvm::Constant *ClassLookupFn =
CGM.CreateRuntimeFunction(llvm::FunctionType::get(IdTy, PtrToInt8Ty, true),
@@ -1041,7 +1051,7 @@
}
llvm::Value *CGObjCGNU::GetSelector(CodeGenFunction &CGF, Selector Sel,
- const std::string &TypeEncoding, bool lval) {
+ const std::string &TypeEncoding) {
SmallVectorImpl<TypedSelector> &Types = SelectorTable[Sel];
llvm::GlobalAlias *SelValue = nullptr;
@@ -1055,29 +1065,34 @@
}
if (!SelValue) {
SelValue = llvm::GlobalAlias::create(
- SelectorTy, llvm::GlobalValue::PrivateLinkage,
+ SelectorTy->getElementType(), 0, llvm::GlobalValue::PrivateLinkage,
".objc_selector_" + Sel.getAsString(), &TheModule);
Types.emplace_back(TypeEncoding, SelValue);
}
- if (lval) {
- llvm::Value *tmp = CGF.CreateTempAlloca(SelValue->getType());
- CGF.Builder.CreateStore(SelValue, tmp);
- return tmp;
- }
return SelValue;
}
-llvm::Value *CGObjCGNU::GetSelector(CodeGenFunction &CGF, Selector Sel,
- bool lval) {
- return GetSelector(CGF, Sel, std::string(), lval);
+Address CGObjCGNU::GetAddrOfSelector(CodeGenFunction &CGF, Selector Sel) {
+ llvm::Value *SelValue = GetSelector(CGF, Sel);
+
+ // Store it to a temporary. Does this satisfy the semantics of
+ // GetAddrOfSelector? Hopefully.
+ Address tmp = CGF.CreateTempAlloca(SelValue->getType(),
+ CGF.getPointerAlign());
+ CGF.Builder.CreateStore(SelValue, tmp);
+ return tmp;
+}
+
+llvm::Value *CGObjCGNU::GetSelector(CodeGenFunction &CGF, Selector Sel) {
+ return GetSelector(CGF, Sel, std::string());
}
llvm::Value *CGObjCGNU::GetSelector(CodeGenFunction &CGF,
const ObjCMethodDecl *Method) {
std::string SelTypes;
CGM.getContext().getObjCEncodingForMethodDecl(Method, SelTypes);
- return GetSelector(CGF, Method->getSelector(), SelTypes, false);
+ return GetSelector(CGF, Method->getSelector(), SelTypes);
}
llvm::Constant *CGObjCGNU::GetEHType(QualType T) {
@@ -1160,21 +1175,23 @@
fields.push_back(BVtable);
fields.push_back(typeName);
llvm::Constant *TI =
- MakeGlobal(llvm::StructType::get(PtrToInt8Ty, PtrToInt8Ty,
- nullptr), fields, "__objc_eh_typeinfo_" + className,
+ MakeGlobal(llvm::StructType::get(PtrToInt8Ty, PtrToInt8Ty, nullptr),
+ fields, CGM.getPointerAlign(),
+ "__objc_eh_typeinfo_" + className,
llvm::GlobalValue::LinkOnceODRLinkage);
return llvm::ConstantExpr::getBitCast(TI, PtrToInt8Ty);
}
/// Generate an NSConstantString object.
-llvm::Constant *CGObjCGNU::GenerateConstantString(const StringLiteral *SL) {
+ConstantAddress CGObjCGNU::GenerateConstantString(const StringLiteral *SL) {
std::string Str = SL->getString().str();
+ CharUnits Align = CGM.getPointerAlign();
// Look for an existing one
llvm::StringMap<llvm::Constant*>::iterator old = ObjCStrings.find(Str);
if (old != ObjCStrings.end())
- return old->getValue();
+ return ConstantAddress(old->getValue(), Align);
StringRef StringClass = CGM.getLangOpts().ObjCConstantStringClass;
@@ -1197,11 +1214,11 @@
Ivars.push_back(llvm::ConstantInt::get(IntTy, Str.size()));
llvm::Constant *ObjCStr = MakeGlobal(
llvm::StructType::get(PtrToIdTy, PtrToInt8Ty, IntTy, nullptr),
- Ivars, ".objc_str");
+ Ivars, Align, ".objc_str");
ObjCStr = llvm::ConstantExpr::getBitCast(ObjCStr, PtrToInt8Ty);
ObjCStrings[Str] = ObjCStr;
ConstantStrings.push_back(ObjCStr);
- return ObjCStr;
+ return ConstantAddress(ObjCStr, Align);
}
///Generates a message send where the super is the receiver. This is a message
@@ -1261,14 +1278,14 @@
if (IsClassMessage) {
if (!MetaClassPtrAlias) {
MetaClassPtrAlias = llvm::GlobalAlias::create(
- IdTy, llvm::GlobalValue::InternalLinkage,
+ IdTy->getElementType(), 0, llvm::GlobalValue::InternalLinkage,
".objc_metaclass_ref" + Class->getNameAsString(), &TheModule);
}
ReceiverClass = MetaClassPtrAlias;
} else {
if (!ClassPtrAlias) {
ClassPtrAlias = llvm::GlobalAlias::create(
- IdTy, llvm::GlobalValue::InternalLinkage,
+ IdTy->getElementType(), 0, llvm::GlobalValue::InternalLinkage,
".objc_class_ref" + Class->getNameAsString(), &TheModule);
}
ReceiverClass = ClassPtrAlias;
@@ -1281,16 +1298,20 @@
// Get the superclass pointer
ReceiverClass = Builder.CreateStructGEP(CastTy, ReceiverClass, 1);
// Load the superclass pointer
- ReceiverClass = Builder.CreateLoad(ReceiverClass);
+ ReceiverClass =
+ Builder.CreateAlignedLoad(ReceiverClass, CGF.getPointerAlign());
// Construct the structure used to look up the IMP
llvm::StructType *ObjCSuperTy = llvm::StructType::get(
Receiver->getType(), IdTy, nullptr);
- llvm::Value *ObjCSuper = Builder.CreateAlloca(ObjCSuperTy);
+
+ // FIXME: Is this really supposed to be a dynamic alloca?
+ Address ObjCSuper = Address(Builder.CreateAlloca(ObjCSuperTy),
+ CGF.getPointerAlign());
Builder.CreateStore(Receiver,
- Builder.CreateStructGEP(ObjCSuperTy, ObjCSuper, 0));
+ Builder.CreateStructGEP(ObjCSuper, 0, CharUnits::Zero()));
Builder.CreateStore(ReceiverClass,
- Builder.CreateStructGEP(ObjCSuperTy, ObjCSuper, 1));
+ Builder.CreateStructGEP(ObjCSuper, 1, CGF.getPointerSize()));
ObjCSuper = EnforceType(Builder, ObjCSuper, PtrToObjCSuperTy);
@@ -1306,8 +1327,8 @@
llvm::MDNode *node = llvm::MDNode::get(VMContext, impMD);
llvm::Instruction *call;
- RValue msgRet = CGF.EmitCall(MSI.CallInfo, imp, Return, ActualArgs, nullptr,
- &call);
+ RValue msgRet = CGF.EmitCall(MSI.CallInfo, imp, Return, ActualArgs,
+ CGCalleeInfo(), &call);
call->setMetadata(msgSendMDKind, node);
return msgRet;
}
@@ -1419,8 +1440,8 @@
imp = EnforceType(Builder, imp, MSI.MessengerType);
llvm::Instruction *call;
- RValue msgRet = CGF.EmitCall(MSI.CallInfo, imp, Return, ActualArgs, nullptr,
- &call);
+ RValue msgRet = CGF.EmitCall(MSI.CallInfo, imp, Return, ActualArgs,
+ CGCalleeInfo(), &call);
call->setMetadata(msgSendMDKind, node);
@@ -1435,16 +1456,14 @@
phi->addIncoming(llvm::Constant::getNullValue(v->getType()), startBB);
msgRet = RValue::get(phi);
} else if (msgRet.isAggregate()) {
- llvm::Value *v = msgRet.getAggregateAddr();
- llvm::PHINode *phi = Builder.CreatePHI(v->getType(), 2);
- llvm::PointerType *RetTy = cast<llvm::PointerType>(v->getType());
- llvm::AllocaInst *NullVal =
- CGF.CreateTempAlloca(RetTy->getElementType(), "null");
- CGF.InitTempAlloca(NullVal,
- llvm::Constant::getNullValue(RetTy->getElementType()));
- phi->addIncoming(v, messageBB);
- phi->addIncoming(NullVal, startBB);
- msgRet = RValue::getAggregate(phi);
+ Address v = msgRet.getAggregateAddress();
+ llvm::PHINode *phi = Builder.CreatePHI(v.getType(), 2);
+ llvm::Type *RetTy = v.getElementType();
+ Address NullVal = CGF.CreateTempAlloca(RetTy, v.getAlignment(), "null");
+ CGF.InitTempAlloca(NullVal, llvm::Constant::getNullValue(RetTy));
+ phi->addIncoming(v.getPointer(), messageBB);
+ phi->addIncoming(NullVal.getPointer(), startBB);
+ msgRet = RValue::getAggregate(Address(phi, v.getAlignment()));
} else /* isComplex() */ {
std::pair<llvm::Value*,llvm::Value*> v = msgRet.getComplexVal();
llvm::PHINode *phi = Builder.CreatePHI(v.first->getType(), 2);
@@ -1517,7 +1536,8 @@
Methods.push_back(MethodArray);
// Create an instance of the structure
- return MakeGlobal(ObjCMethodListTy, Methods, ".objc_method_list");
+ return MakeGlobal(ObjCMethodListTy, Methods, CGM.getPointerAlign(),
+ ".objc_method_list");
}
/// Generates an IvarList. Used in construction of a objc_class.
@@ -1557,7 +1577,8 @@
nullptr);
// Create an instance of the structure
- return MakeGlobal(ObjCIvarListTy, Elements, ".objc_ivar_list");
+ return MakeGlobal(ObjCIvarListTy, Elements, CGM.getPointerAlign(),
+ ".objc_ivar_list");
}
/// Generate a class structure
@@ -1640,8 +1661,9 @@
std::string ClassSym((isMeta ? "_OBJC_METACLASS_": "_OBJC_CLASS_") +
std::string(Name));
llvm::GlobalVariable *ClassRef = TheModule.getNamedGlobal(ClassSym);
- llvm::Constant *Class = MakeGlobal(ClassTy, Elements, ClassSym,
- llvm::GlobalValue::ExternalLinkage);
+ llvm::Constant *Class =
+ MakeGlobal(ClassTy, Elements, CGM.getPointerAlign(), ClassSym,
+ llvm::GlobalValue::ExternalLinkage);
if (ClassRef) {
ClassRef->replaceAllUsesWith(llvm::ConstantExpr::getBitCast(Class,
ClassRef->getType()));
@@ -1676,7 +1698,8 @@
Methods.clear();
Methods.push_back(llvm::ConstantInt::get(IntTy, MethodNames.size()));
Methods.push_back(Array);
- return MakeGlobal(ObjCMethodDescListTy, Methods, ".objc_method_list");
+ return MakeGlobal(ObjCMethodDescListTy, Methods, CGM.getPointerAlign(),
+ ".objc_method_list");
}
// Create the protocol list structure used in classes, categories and so on
@@ -1709,7 +1732,8 @@
Elements.push_back(NULLPtr);
Elements.push_back(llvm::ConstantInt::get(LongTy, Protocols.size()));
Elements.push_back(ProtocolArray);
- return MakeGlobal(ProtocolListTy, Elements, ".objc_protocol_list");
+ return MakeGlobal(ProtocolListTy, Elements, CGM.getPointerAlign(),
+ ".objc_protocol_list");
}
llvm::Value *CGObjCGNU::GenerateProtocolRef(CodeGenFunction &CGF,
@@ -1749,7 +1773,8 @@
Elements.push_back(MethodList);
Elements.push_back(MethodList);
Elements.push_back(MethodList);
- return MakeGlobal(ProtocolTy, Elements, ".objc_protocol");
+ return MakeGlobal(ProtocolTy, Elements, CGM.getPointerAlign(),
+ ".objc_protocol");
}
void CGObjCGNU::GenerateProtocol(const ObjCProtocolDecl *PD) {
@@ -1910,7 +1935,7 @@
Elements.push_back(OptionalPropertyList);
ExistingProtocols[ProtocolName] =
llvm::ConstantExpr::getBitCast(MakeGlobal(ProtocolTy, Elements,
- ".objc_protocol"), IdTy);
+ CGM.getPointerAlign(), ".objc_protocol"), IdTy);
}
void CGObjCGNU::GenerateProtocolHolderCategory() {
// Collect information about instance methods
@@ -1952,10 +1977,12 @@
ExistingProtocols.size()));
ProtocolElements.push_back(ProtocolArray);
Elements.push_back(llvm::ConstantExpr::getBitCast(MakeGlobal(ProtocolListTy,
- ProtocolElements, ".objc_protocol_list"), PtrTy));
+ ProtocolElements, CGM.getPointerAlign(),
+ ".objc_protocol_list"), PtrTy));
Categories.push_back(llvm::ConstantExpr::getBitCast(
MakeGlobal(llvm::StructType::get(PtrToInt8Ty, PtrToInt8Ty,
- PtrTy, PtrTy, PtrTy, nullptr), Elements), PtrTy));
+ PtrTy, PtrTy, PtrTy, nullptr), Elements, CGM.getPointerAlign()),
+ PtrTy));
}
/// Libobjc2 uses a bitfield representation where small(ish) bitfields are
@@ -1995,7 +2022,7 @@
llvm::ConstantInt::get(Int32Ty, values.size()),
array };
llvm::Constant *GS = MakeGlobal(llvm::StructType::get(Int32Ty, arrayTy,
- nullptr), fields);
+ nullptr), fields, CharUnits::fromQuantity(4));
llvm::Constant *ptr = llvm::ConstantExpr::getPtrToInt(GS, IntPtrTy);
return ptr;
}
@@ -2047,7 +2074,8 @@
GenerateProtocolList(Protocols), PtrTy));
Categories.push_back(llvm::ConstantExpr::getBitCast(
MakeGlobal(llvm::StructType::get(PtrToInt8Ty, PtrToInt8Ty,
- PtrTy, PtrTy, PtrTy, nullptr), Elements), PtrTy));
+ PtrTy, PtrTy, PtrTy, nullptr), Elements, CGM.getPointerAlign()),
+ PtrTy));
}
llvm::Constant *CGObjCGNU::GeneratePropertyList(const ObjCImplementationDecl *OID,
@@ -2225,7 +2253,8 @@
llvm::Constant *StrongIvarBitmap = MakeBitField(StrongIvars);
llvm::Constant *WeakIvarBitmap = MakeBitField(WeakIvars);
llvm::GlobalVariable *IvarOffsetArray =
- MakeGlobalArray(PtrToIntTy, IvarOffsetValues, ".ivar.offsets");
+ MakeGlobalArray(PtrToIntTy, IvarOffsetValues, CGM.getPointerAlign(),
+ ".ivar.offsets");
// Collect information about instance methods
@@ -2385,13 +2414,15 @@
llvm::StructType::get(PtrToInt8Ty, StaticsArrayTy, nullptr);
llvm::Type *StaticsListPtrTy =
llvm::PointerType::getUnqual(StaticsListTy);
- Statics = MakeGlobal(StaticsListTy, Elements, ".objc_statics");
+ Statics = MakeGlobal(StaticsListTy, Elements, CGM.getPointerAlign(),
+ ".objc_statics");
llvm::ArrayType *StaticsListArrayTy =
llvm::ArrayType::get(StaticsListPtrTy, 2);
Elements.clear();
Elements.push_back(Statics);
Elements.push_back(llvm::Constant::getNullValue(StaticsListPtrTy));
- Statics = MakeGlobal(StaticsListArrayTy, Elements, ".objc_statics_ptr");
+ Statics = MakeGlobal(StaticsListArrayTy, Elements,
+ CGM.getPointerAlign(), ".objc_statics_ptr");
Statics = llvm::ConstantExpr::getBitCast(Statics, PtrTy);
}
// Array of classes, categories, and constant objects
@@ -2442,7 +2473,8 @@
// Number of static selectors
Elements.push_back(llvm::ConstantInt::get(LongTy, SelectorCount));
llvm::GlobalVariable *SelectorList =
- MakeGlobalArray(SelStructTy, Selectors, ".objc_selector_list");
+ MakeGlobalArray(SelStructTy, Selectors, CGM.getPointerAlign(),
+ ".objc_selector_list");
Elements.push_back(llvm::ConstantExpr::getBitCast(SelectorList,
SelStructPtrTy));
@@ -2475,7 +2507,8 @@
llvm::Constant *ClassList = llvm::ConstantArray::get(ClassListTy, Classes);
Elements.push_back(ClassList);
// Construct the symbol table
- llvm::Constant *SymTab= MakeGlobal(SymTabTy, Elements);
+ llvm::Constant *SymTab =
+ MakeGlobal(SymTabTy, Elements, CGM.getPointerAlign());
// The symbol table is contained in a module which has some version-checking
// constants
@@ -2516,7 +2549,7 @@
break;
}
- llvm::Value *Module = MakeGlobal(ModuleTy, Elements);
+ llvm::Value *Module = MakeGlobal(ModuleTy, Elements, CGM.getPointerAlign());
// Create the load function calling the runtime entry point with the module
// structure
@@ -2526,7 +2559,7 @@
&TheModule);
llvm::BasicBlock *EntryBB =
llvm::BasicBlock::Create(VMContext, "entry", LoadFunction);
- CGBuilderTy Builder(VMContext);
+ CGBuilderTy Builder(CGM, VMContext);
Builder.SetInsertPoint(EntryBB);
llvm::FunctionType *FT =
@@ -2678,57 +2711,63 @@
}
llvm::Value * CGObjCGNU::EmitObjCWeakRead(CodeGenFunction &CGF,
- llvm::Value *AddrWeakObj) {
+ Address AddrWeakObj) {
CGBuilderTy &B = CGF.Builder;
AddrWeakObj = EnforceType(B, AddrWeakObj, PtrToIdTy);
- return B.CreateCall(WeakReadFn.getType(), WeakReadFn, AddrWeakObj);
+ return B.CreateCall(WeakReadFn.getType(), WeakReadFn,
+ AddrWeakObj.getPointer());
}
void CGObjCGNU::EmitObjCWeakAssign(CodeGenFunction &CGF,
- llvm::Value *src, llvm::Value *dst) {
+ llvm::Value *src, Address dst) {
CGBuilderTy &B = CGF.Builder;
src = EnforceType(B, src, IdTy);
dst = EnforceType(B, dst, PtrToIdTy);
- B.CreateCall(WeakAssignFn.getType(), WeakAssignFn, {src, dst});
+ B.CreateCall(WeakAssignFn.getType(), WeakAssignFn,
+ {src, dst.getPointer()});
}
void CGObjCGNU::EmitObjCGlobalAssign(CodeGenFunction &CGF,
- llvm::Value *src, llvm::Value *dst,
+ llvm::Value *src, Address dst,
bool threadlocal) {
CGBuilderTy &B = CGF.Builder;
src = EnforceType(B, src, IdTy);
dst = EnforceType(B, dst, PtrToIdTy);
// FIXME. Add threadloca assign API
assert(!threadlocal && "EmitObjCGlobalAssign - Threal Local API NYI");
- B.CreateCall(GlobalAssignFn.getType(), GlobalAssignFn, {src, dst});
+ B.CreateCall(GlobalAssignFn.getType(), GlobalAssignFn,
+ {src, dst.getPointer()});
}
void CGObjCGNU::EmitObjCIvarAssign(CodeGenFunction &CGF,
- llvm::Value *src, llvm::Value *dst,
+ llvm::Value *src, Address dst,
llvm::Value *ivarOffset) {
CGBuilderTy &B = CGF.Builder;
src = EnforceType(B, src, IdTy);
dst = EnforceType(B, dst, IdTy);
- B.CreateCall(IvarAssignFn.getType(), IvarAssignFn, {src, dst, ivarOffset});
+ B.CreateCall(IvarAssignFn.getType(), IvarAssignFn,
+ {src, dst.getPointer(), ivarOffset});
}
void CGObjCGNU::EmitObjCStrongCastAssign(CodeGenFunction &CGF,
- llvm::Value *src, llvm::Value *dst) {
+ llvm::Value *src, Address dst) {
CGBuilderTy &B = CGF.Builder;
src = EnforceType(B, src, IdTy);
dst = EnforceType(B, dst, PtrToIdTy);
- B.CreateCall(StrongCastAssignFn.getType(), StrongCastAssignFn, {src, dst});
+ B.CreateCall(StrongCastAssignFn.getType(), StrongCastAssignFn,
+ {src, dst.getPointer()});
}
void CGObjCGNU::EmitGCMemmoveCollectable(CodeGenFunction &CGF,
- llvm::Value *DestPtr,
- llvm::Value *SrcPtr,
+ Address DestPtr,
+ Address SrcPtr,
llvm::Value *Size) {
CGBuilderTy &B = CGF.Builder;
DestPtr = EnforceType(B, DestPtr, PtrTy);
SrcPtr = EnforceType(B, SrcPtr, PtrTy);
- B.CreateCall(MemMoveFn.getType(), MemMoveFn, {DestPtr, SrcPtr, Size});
+ B.CreateCall(MemMoveFn.getType(), MemMoveFn,
+ {DestPtr.getPointer(), SrcPtr.getPointer(), Size});
}
llvm::GlobalVariable *CGObjCGNU::ObjCIvarOffsetVariable(
@@ -2811,17 +2850,22 @@
Interface = FindIvarInterface(CGM.getContext(), Interface, Ivar);
if (RuntimeVersion < 10)
return CGF.Builder.CreateZExtOrBitCast(
- CGF.Builder.CreateLoad(CGF.Builder.CreateLoad(
- ObjCIvarOffsetVariable(Interface, Ivar), false, "ivar")),
+ CGF.Builder.CreateDefaultAlignedLoad(CGF.Builder.CreateAlignedLoad(
+ ObjCIvarOffsetVariable(Interface, Ivar),
+ CGF.getPointerAlign(), "ivar")),
PtrDiffTy);
std::string name = "__objc_ivar_offset_value_" +
Interface->getNameAsString() +"." + Ivar->getNameAsString();
+ CharUnits Align = CGM.getIntAlign();
llvm::Value *Offset = TheModule.getGlobalVariable(name);
- if (!Offset)
- Offset = new llvm::GlobalVariable(TheModule, IntTy,
+ if (!Offset) {
+ auto GV = new llvm::GlobalVariable(TheModule, IntTy,
false, llvm::GlobalValue::LinkOnceAnyLinkage,
llvm::Constant::getNullValue(IntTy), name);
- Offset = CGF.Builder.CreateLoad(Offset);
+ GV->setAlignment(Align.getQuantity());
+ Offset = GV;
+ }
+ Offset = CGF.Builder.CreateAlignedLoad(Offset, Align);
if (Offset->getType() != PtrDiffTy)
Offset = CGF.Builder.CreateZExtOrBitCast(Offset, PtrDiffTy);
return Offset;
@@ -2845,6 +2889,7 @@
case ObjCRuntime::FragileMacOSX:
case ObjCRuntime::MacOSX:
case ObjCRuntime::iOS:
+ case ObjCRuntime::WatchOS:
llvm_unreachable("these runtimes are not GNU runtimes");
}
llvm_unreachable("bad runtime");
diff --git a/lib/CodeGen/CGObjCMac.cpp b/lib/CodeGen/CGObjCMac.cpp
index a45446a..5f3ebbd 100644
--- a/lib/CodeGen/CGObjCMac.cpp
+++ b/lib/CodeGen/CGObjCMac.cpp
@@ -659,9 +659,6 @@
// MessageRefCPtrTy - clang type for struct _message_ref_t*
QualType MessageRefCPtrTy;
- // MessengerTy - Type of the messenger (shown as IMP above)
- llvm::FunctionType *MessengerTy;
-
// SuperMessageRefTy - LLVM for:
// struct _super_message_ref_t {
// SUPER_IMP messenger;
@@ -735,20 +732,6 @@
class CGObjCCommonMac : public CodeGen::CGObjCRuntime {
public:
- // FIXME - accessibility
- class GC_IVAR {
- public:
- unsigned ivar_bytepos;
- unsigned ivar_size;
- GC_IVAR(unsigned bytepos = 0, unsigned size = 0)
- : ivar_bytepos(bytepos), ivar_size(size) {}
-
- // Allow sorting based on byte pos.
- bool operator<(const GC_IVAR &b) const {
- return ivar_bytepos < b.ivar_bytepos;
- }
- };
-
class SKIP_SCAN {
public:
unsigned skip;
@@ -830,10 +813,6 @@
// FIXME! May not be needing this after all.
unsigned ObjCABI;
- // gc ivar layout bitmap calculation helper caches.
- SmallVector<GC_IVAR, 16> SkipIvars;
- SmallVector<GC_IVAR, 16> IvarsInfo;
-
// arc/mrr layout of captured block literal variables.
SmallVector<RUN_SKIP, 16> RunSkipBlockVars;
@@ -854,7 +833,7 @@
llvm::DenseMap<Selector, llvm::GlobalVariable*> MethodVarNames;
/// DefinedCategoryNames - list of category names in form Class_Category.
- llvm::SetVector<std::string> DefinedCategoryNames;
+ llvm::SmallSetVector<std::string, 16> DefinedCategoryNames;
/// MethodVarTypes - uniqued method type signatures. We have to use
/// a StringMap here because have no other unique reference.
@@ -934,20 +913,28 @@
/// BuildIvarLayout - Builds ivar layout bitmap for the class
/// implementation for the __strong or __weak case.
///
+ /// \param hasMRCWeakIvars - Whether we are compiling in MRC and there
+ /// are any weak ivars defined directly in the class. Meaningless unless
+ /// building a weak layout. Does not guarantee that the layout will
+ /// actually have any entries, because the ivar might be under-aligned.
llvm::Constant *BuildIvarLayout(const ObjCImplementationDecl *OI,
- bool ForStrongLayout);
-
- llvm::Constant *BuildIvarLayoutBitmap(std::string &BitMap);
+ CharUnits beginOffset,
+ CharUnits endOffset,
+ bool forStrongLayout,
+ bool hasMRCWeakIvars);
- void BuildAggrIvarRecordLayout(const RecordType *RT,
- unsigned int BytePos, bool ForStrongLayout,
- bool &HasUnion);
- void BuildAggrIvarLayout(const ObjCImplementationDecl *OI,
- const llvm::StructLayout *Layout,
- const RecordDecl *RD,
- ArrayRef<const FieldDecl*> RecFields,
- unsigned int BytePos, bool ForStrongLayout,
- bool &HasUnion);
+ llvm::Constant *BuildStrongIvarLayout(const ObjCImplementationDecl *OI,
+ CharUnits beginOffset,
+ CharUnits endOffset) {
+ return BuildIvarLayout(OI, beginOffset, endOffset, true, false);
+ }
+
+ llvm::Constant *BuildWeakIvarLayout(const ObjCImplementationDecl *OI,
+ CharUnits beginOffset,
+ CharUnits endOffset,
+ bool hasMRCWeakIvars) {
+ return BuildIvarLayout(OI, beginOffset, endOffset, false, hasMRCWeakIvars);
+ }
Qualifiers::ObjCLifetime getBlockCaptureLifetime(QualType QT, bool ByrefLayout);
@@ -970,7 +957,6 @@
llvm::Constant *getBitmapBlockLayout(bool ComputeByrefLayout);
-
/// GetIvarLayoutName - Returns a unique constant for the given
/// ivar layout bitmap.
llvm::Constant *GetIvarLayoutName(IdentifierInfo *Ident,
@@ -1002,6 +988,7 @@
/// defined. The return value has type ProtocolPtrTy.
llvm::Constant *GetProtocolRef(const ObjCProtocolDecl *PD);
+public:
/// CreateMetadataVar - Create a global variable with internal
/// linkage for use by the Objective-C runtime.
///
@@ -1017,9 +1004,10 @@
/// \param AddToUsed - Whether the variable should be added to
/// "llvm.used".
llvm::GlobalVariable *CreateMetadataVar(Twine Name, llvm::Constant *Init,
- StringRef Section, unsigned Align,
+ StringRef Section, CharUnits Align,
bool AddToUsed);
+protected:
CodeGen::RValue EmitMessageSend(CodeGen::CodeGenFunction &CGF,
ReturnValueSlot Return,
QualType ResultType,
@@ -1029,6 +1017,7 @@
bool IsSuper,
const CallArgList &CallArgs,
const ObjCMethodDecl *OMD,
+ const ObjCInterfaceDecl *ClassReceiver,
const ObjCCommonTypesHelper &ObjCTypes);
/// EmitImageInfo - Emit the image info marker used to encode some module
@@ -1039,7 +1028,11 @@
CGObjCCommonMac(CodeGen::CodeGenModule &cgm) :
CGObjCRuntime(cgm), VMContext(cgm.getLLVMContext()) { }
- llvm::Constant *GenerateConstantString(const StringLiteral *SL) override;
+ bool isNonFragileABI() const {
+ return ObjCABI == 2;
+ }
+
+ ConstantAddress GenerateConstantString(const StringLiteral *SL) override;
llvm::Function *GenerateMethod(const ObjCMethodDecl *OMD,
const ObjCContainerDecl *CD=nullptr) override;
@@ -1084,7 +1077,9 @@
/// EmitClassExtension - Generate the class extension structure used
/// to store the weak ivar layout and properties. The return value
/// has type ClassExtensionPtrTy.
- llvm::Constant *EmitClassExtension(const ObjCImplementationDecl *ID);
+ llvm::Constant *EmitClassExtension(const ObjCImplementationDecl *ID,
+ CharUnits instanceSize,
+ bool hasMRCWeakIvars);
/// EmitClassRef - Return a Value*, of type ObjCTypes.ClassPtrTy,
/// for the given class.
@@ -1172,8 +1167,8 @@
/// EmitSelector - Return a Value*, of type ObjCTypes.SelectorPtrTy,
/// for the given selector.
- llvm::Value *EmitSelector(CodeGenFunction &CGF, Selector Sel,
- bool lval=false);
+ llvm::Value *EmitSelector(CodeGenFunction &CGF, Selector Sel);
+ Address EmitSelectorAddr(CodeGenFunction &CGF, Selector Sel);
public:
CGObjCMac(CodeGen::CodeGenModule &cgm);
@@ -1199,8 +1194,8 @@
llvm::Value *GetClass(CodeGenFunction &CGF,
const ObjCInterfaceDecl *ID) override;
- llvm::Value *GetSelector(CodeGenFunction &CGF, Selector Sel,
- bool lval = false) override;
+ llvm::Value *GetSelector(CodeGenFunction &CGF, Selector Sel) override;
+ Address GetAddrOfSelector(CodeGenFunction &CGF, Selector Sel) override;
/// The NeXT/Apple runtimes do not support typed selectors; just emit an
/// untyped one.
@@ -1236,19 +1231,19 @@
void EmitThrowStmt(CodeGen::CodeGenFunction &CGF, const ObjCAtThrowStmt &S,
bool ClearInsertionPoint=true) override;
llvm::Value * EmitObjCWeakRead(CodeGen::CodeGenFunction &CGF,
- llvm::Value *AddrWeakObj) override;
+ Address AddrWeakObj) override;
void EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF,
- llvm::Value *src, llvm::Value *dst) override;
+ llvm::Value *src, Address dst) override;
void EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF,
- llvm::Value *src, llvm::Value *dest,
+ llvm::Value *src, Address dest,
bool threadlocal = false) override;
void EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF,
- llvm::Value *src, llvm::Value *dest,
+ llvm::Value *src, Address dest,
llvm::Value *ivarOffset) override;
void EmitObjCStrongCastAssign(CodeGen::CodeGenFunction &CGF,
- llvm::Value *src, llvm::Value *dest) override;
+ llvm::Value *src, Address dest) override;
void EmitGCMemmoveCollectable(CodeGen::CodeGenFunction &CGF,
- llvm::Value *dest, llvm::Value *src,
+ Address dest, Address src,
llvm::Value *size) override;
LValue EmitObjCValueForIvar(CodeGen::CodeGenFunction &CGF, QualType ObjectTy,
@@ -1395,8 +1390,8 @@
/// EmitSelector - Return a Value*, of type ObjCTypes.SelectorPtrTy,
/// for the given selector.
- llvm::Value *EmitSelector(CodeGenFunction &CGF, Selector Sel,
- bool lval=false);
+ llvm::Value *EmitSelector(CodeGenFunction &CGF, Selector Sel);
+ Address EmitSelectorAddr(CodeGenFunction &CGF, Selector Sel);
/// GetInterfaceEHType - Get the cached ehtype for the given Objective-C
/// interface. The return value has type EHTypePtrTy.
@@ -1474,9 +1469,10 @@
llvm::Value *GetClass(CodeGenFunction &CGF,
const ObjCInterfaceDecl *ID) override;
- llvm::Value *GetSelector(CodeGenFunction &CGF, Selector Sel,
- bool lvalue = false) override
- { return EmitSelector(CGF, Sel, lvalue); }
+ llvm::Value *GetSelector(CodeGenFunction &CGF, Selector Sel) override
+ { return EmitSelector(CGF, Sel); }
+ Address GetAddrOfSelector(CodeGenFunction &CGF, Selector Sel) override
+ { return EmitSelectorAddr(CGF, Sel); }
/// The NeXT/Apple runtimes do not support typed selectors; just emit an
/// untyped one.
@@ -1531,19 +1527,19 @@
void EmitThrowStmt(CodeGen::CodeGenFunction &CGF, const ObjCAtThrowStmt &S,
bool ClearInsertionPoint=true) override;
llvm::Value * EmitObjCWeakRead(CodeGen::CodeGenFunction &CGF,
- llvm::Value *AddrWeakObj) override;
+ Address AddrWeakObj) override;
void EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF,
- llvm::Value *src, llvm::Value *dst) override;
+ llvm::Value *src, Address edst) override;
void EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF,
- llvm::Value *src, llvm::Value *dest,
+ llvm::Value *src, Address dest,
bool threadlocal = false) override;
void EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF,
- llvm::Value *src, llvm::Value *dest,
+ llvm::Value *src, Address dest,
llvm::Value *ivarOffset) override;
void EmitObjCStrongCastAssign(CodeGen::CodeGenFunction &CGF,
- llvm::Value *src, llvm::Value *dest) override;
+ llvm::Value *src, Address dest) override;
void EmitGCMemmoveCollectable(CodeGen::CodeGenFunction &CGF,
- llvm::Value *dest, llvm::Value *src,
+ Address dest, Address src,
llvm::Value *size) override;
LValue EmitObjCValueForIvar(CodeGen::CodeGenFunction &CGF, QualType ObjectTy,
llvm::Value *BaseValue, const ObjCIvarDecl *Ivar,
@@ -1645,7 +1641,7 @@
// memory or (2) agg values in registers.
if (result.isAggregate()) {
assert(result.isAggregate() && "null init of non-aggregate result?");
- CGF.EmitNullInitialization(result.getAggregateAddr(), resultType);
+ CGF.EmitNullInitialization(result.getAggregateAddress(), resultType);
if (contBB) CGF.EmitBlock(contBB);
return result;
}
@@ -1711,9 +1707,11 @@
}
/// GetSelector - Return the pointer to the unique'd string for this selector.
-llvm::Value *CGObjCMac::GetSelector(CodeGenFunction &CGF, Selector Sel,
- bool lval) {
- return EmitSelector(CGF, Sel, lval);
+llvm::Value *CGObjCMac::GetSelector(CodeGenFunction &CGF, Selector Sel) {
+ return EmitSelector(CGF, Sel);
+}
+Address CGObjCMac::GetAddrOfSelector(CodeGenFunction &CGF, Selector Sel) {
+ return EmitSelectorAddr(CGF, Sel);
}
llvm::Value *CGObjCMac::GetSelector(CodeGenFunction &CGF, const ObjCMethodDecl
*Method) {
@@ -1756,7 +1754,7 @@
};
*/
-llvm::Constant *CGObjCCommonMac::GenerateConstantString(
+ConstantAddress CGObjCCommonMac::GenerateConstantString(
const StringLiteral *SL) {
return (CGM.getLangOpts().NoConstantCFStrings == 0 ?
CGM.GetAddrOfConstantCFString(SL) :
@@ -1783,13 +1781,14 @@
const ObjCMethodDecl *Method) {
// Create and init a super structure; this is a (receiver, class)
// pair we will pass to objc_msgSendSuper.
- llvm::Value *ObjCSuper =
- CGF.CreateTempAlloca(ObjCTypes.SuperTy, "objc_super");
+ Address ObjCSuper =
+ CGF.CreateTempAlloca(ObjCTypes.SuperTy, CGF.getPointerAlign(),
+ "objc_super");
llvm::Value *ReceiverAsObject =
CGF.Builder.CreateBitCast(Receiver, ObjCTypes.ObjectPtrTy);
CGF.Builder.CreateStore(
ReceiverAsObject,
- CGF.Builder.CreateStructGEP(ObjCTypes.SuperTy, ObjCSuper, 0));
+ CGF.Builder.CreateStructGEP(ObjCSuper, 0, CharUnits::Zero()));
// If this is a class message the metaclass is passed as the target.
llvm::Value *Target;
@@ -1803,12 +1802,13 @@
// isa" is the first ivar in a class (which it must be).
Target = EmitClassRef(CGF, Class->getSuperClass());
Target = CGF.Builder.CreateStructGEP(ObjCTypes.ClassTy, Target, 0);
- Target = CGF.Builder.CreateLoad(Target);
+ Target = CGF.Builder.CreateAlignedLoad(Target, CGF.getPointerAlign());
} else {
llvm::Constant *MetaClassPtr = EmitMetaClassRef(Class);
llvm::Value *SuperPtr =
CGF.Builder.CreateStructGEP(ObjCTypes.ClassTy, MetaClassPtr, 1);
- llvm::Value *Super = CGF.Builder.CreateLoad(SuperPtr);
+ llvm::Value *Super =
+ CGF.Builder.CreateAlignedLoad(SuperPtr, CGF.getPointerAlign());
Target = Super;
}
} else if (isCategoryImpl)
@@ -1816,19 +1816,19 @@
else {
llvm::Value *ClassPtr = EmitSuperClassRef(Class);
ClassPtr = CGF.Builder.CreateStructGEP(ObjCTypes.ClassTy, ClassPtr, 1);
- Target = CGF.Builder.CreateLoad(ClassPtr);
+ Target = CGF.Builder.CreateAlignedLoad(ClassPtr, CGF.getPointerAlign());
}
// FIXME: We shouldn't need to do this cast, rectify the ASTContext and
// ObjCTypes types.
llvm::Type *ClassTy =
CGM.getTypes().ConvertType(CGF.getContext().getObjCClassType());
Target = CGF.Builder.CreateBitCast(Target, ClassTy);
- CGF.Builder.CreateStore(
- Target, CGF.Builder.CreateStructGEP(ObjCTypes.SuperTy, ObjCSuper, 1));
+ CGF.Builder.CreateStore(Target,
+ CGF.Builder.CreateStructGEP(ObjCSuper, 1, CGF.getPointerSize()));
return EmitMessageSend(CGF, Return, ResultType,
EmitSelector(CGF, Sel),
- ObjCSuper, ObjCTypes.SuperPtrCTy,
- true, CallArgs, Method, ObjCTypes);
+ ObjCSuper.getPointer(), ObjCTypes.SuperPtrCTy,
+ true, CallArgs, Method, Class, ObjCTypes);
}
/// Generate code for a message send expression.
@@ -1843,7 +1843,16 @@
return EmitMessageSend(CGF, Return, ResultType,
EmitSelector(CGF, Sel),
Receiver, CGF.getContext().getObjCIdType(),
- false, CallArgs, Method, ObjCTypes);
+ false, CallArgs, Method, Class, ObjCTypes);
+}
+
+static bool isWeakLinkedClass(const ObjCInterfaceDecl *ID) {
+ do {
+ if (ID->isWeakImported())
+ return true;
+ } while ((ID = ID->getSuperClass()));
+
+ return false;
}
CodeGen::RValue
@@ -1856,6 +1865,7 @@
bool IsSuper,
const CallArgList &CallArgs,
const ObjCMethodDecl *Method,
+ const ObjCInterfaceDecl *ClassReceiver,
const ObjCCommonTypesHelper &ObjCTypes) {
CallArgList ActualArgs;
if (!IsSuper)
@@ -1872,11 +1882,38 @@
CGM.getContext().getCanonicalType(ResultType) &&
"Result type mismatch!");
+ bool ReceiverCanBeNull = true;
+
+ // Super dispatch assumes that self is non-null; even the messenger
+ // doesn't have a null check internally.
+ if (IsSuper) {
+ ReceiverCanBeNull = false;
+
+ // If this is a direct dispatch of a class method, check whether the class,
+ // or anything in its hierarchy, was weak-linked.
+ } else if (ClassReceiver && Method && Method->isClassMethod()) {
+ ReceiverCanBeNull = isWeakLinkedClass(ClassReceiver);
+
+ // If we're emitting a method, and self is const (meaning just ARC, for now),
+ // and the receiver is a load of self, then self is a valid object.
+ } else if (auto CurMethod =
+ dyn_cast_or_null<ObjCMethodDecl>(CGF.CurCodeDecl)) {
+ auto Self = CurMethod->getSelfDecl();
+ if (Self->getType().isConstQualified()) {
+ if (auto LI = dyn_cast<llvm::LoadInst>(Arg0->stripPointerCasts())) {
+ llvm::Value *SelfAddr = CGF.GetAddrOfLocalVar(Self).getPointer();
+ if (SelfAddr == LI->getPointerOperand()) {
+ ReceiverCanBeNull = false;
+ }
+ }
+ }
+ }
+
NullReturnState nullReturn;
llvm::Constant *Fn = nullptr;
if (CGM.ReturnSlotInterferesWithArgs(MSI.CallInfo)) {
- if (!IsSuper) nullReturn.init(CGF, Arg0);
+ if (ReceiverCanBeNull) nullReturn.init(CGF, Arg0);
Fn = (ObjCABI == 2) ? ObjCTypes.getSendStretFn2(IsSuper)
: ObjCTypes.getSendStretFn(IsSuper);
} else if (CGM.ReturnTypeUsesFPRet(ResultType)) {
@@ -1888,76 +1925,182 @@
} else {
// arm64 uses objc_msgSend for stret methods and yet null receiver check
// must be made for it.
- if (!IsSuper && CGM.ReturnTypeUsesSRet(MSI.CallInfo))
+ if (ReceiverCanBeNull && CGM.ReturnTypeUsesSRet(MSI.CallInfo))
nullReturn.init(CGF, Arg0);
Fn = (ObjCABI == 2) ? ObjCTypes.getSendFn2(IsSuper)
: ObjCTypes.getSendFn(IsSuper);
}
-
- bool requiresnullCheck = false;
- if (CGM.getLangOpts().ObjCAutoRefCount && Method)
+
+ // Emit a null-check if there's a consumed argument other than the receiver.
+ bool RequiresNullCheck = false;
+ if (ReceiverCanBeNull && CGM.getLangOpts().ObjCAutoRefCount && Method) {
for (const auto *ParamDecl : Method->params()) {
if (ParamDecl->hasAttr<NSConsumedAttr>()) {
if (!nullReturn.NullBB)
nullReturn.init(CGF, Arg0);
- requiresnullCheck = true;
+ RequiresNullCheck = true;
break;
}
}
+ }
+ llvm::Instruction *CallSite;
Fn = llvm::ConstantExpr::getBitCast(Fn, MSI.MessengerType);
- RValue rvalue = CGF.EmitCall(MSI.CallInfo, Fn, Return, ActualArgs);
+ RValue rvalue = CGF.EmitCall(MSI.CallInfo, Fn, Return, ActualArgs,
+ CGCalleeInfo(), &CallSite);
+
+ // Mark the call as noreturn if the method is marked noreturn and the
+ // receiver cannot be null.
+ if (Method && Method->hasAttr<NoReturnAttr>() && !ReceiverCanBeNull) {
+ llvm::CallSite(CallSite).setDoesNotReturn();
+ }
+
return nullReturn.complete(CGF, rvalue, ResultType, CallArgs,
- requiresnullCheck ? Method : nullptr);
+ RequiresNullCheck ? Method : nullptr);
}
-static Qualifiers::GC GetGCAttrTypeForType(ASTContext &Ctx, QualType FQT) {
+static Qualifiers::GC GetGCAttrTypeForType(ASTContext &Ctx, QualType FQT,
+ bool pointee = false) {
+ // Note that GC qualification applies recursively to C pointer types
+ // that aren't otherwise decorated. This is weird, but it's probably
+ // an intentional workaround to the unreliable placement of GC qualifiers.
if (FQT.isObjCGCStrong())
return Qualifiers::Strong;
-
- if (FQT.isObjCGCWeak() || FQT.getObjCLifetime() == Qualifiers::OCL_Weak)
+
+ if (FQT.isObjCGCWeak())
return Qualifiers::Weak;
+
+ if (auto ownership = FQT.getObjCLifetime()) {
+ // Ownership does not apply recursively to C pointer types.
+ if (pointee) return Qualifiers::GCNone;
+ switch (ownership) {
+ case Qualifiers::OCL_Weak: return Qualifiers::Weak;
+ case Qualifiers::OCL_Strong: return Qualifiers::Strong;
+ case Qualifiers::OCL_ExplicitNone: return Qualifiers::GCNone;
+ case Qualifiers::OCL_Autoreleasing: llvm_unreachable("autoreleasing ivar?");
+ case Qualifiers::OCL_None: llvm_unreachable("known nonzero");
+ }
+ llvm_unreachable("bad objc ownership");
+ }
- // check for __unsafe_unretained
- if (FQT.getObjCLifetime() == Qualifiers::OCL_ExplicitNone)
- return Qualifiers::GCNone;
-
+ // Treat unqualified retainable pointers as strong.
if (FQT->isObjCObjectPointerType() || FQT->isBlockPointerType())
return Qualifiers::Strong;
- if (const PointerType *PT = FQT->getAs<PointerType>())
- return GetGCAttrTypeForType(Ctx, PT->getPointeeType());
+ // Walk into C pointer types, but only in GC.
+ if (Ctx.getLangOpts().getGC() != LangOptions::NonGC) {
+ if (const PointerType *PT = FQT->getAs<PointerType>())
+ return GetGCAttrTypeForType(Ctx, PT->getPointeeType(), /*pointee*/ true);
+ }
return Qualifiers::GCNone;
}
+namespace {
+ struct IvarInfo {
+ CharUnits Offset;
+ uint64_t SizeInWords;
+ IvarInfo(CharUnits offset, uint64_t sizeInWords)
+ : Offset(offset), SizeInWords(sizeInWords) {}
+
+ // Allow sorting based on byte pos.
+ bool operator<(const IvarInfo &other) const {
+ return Offset < other.Offset;
+ }
+ };
+
+ /// A helper class for building GC layout strings.
+ class IvarLayoutBuilder {
+ CodeGenModule &CGM;
+
+ /// The start of the layout. Offsets will be relative to this value,
+ /// and entries less than this value will be silently discarded.
+ CharUnits InstanceBegin;
+
+ /// The end of the layout. Offsets will never exceed this value.
+ CharUnits InstanceEnd;
+
+ /// Whether we're generating the strong layout or the weak layout.
+ bool ForStrongLayout;
+
+ /// Whether the offsets in IvarsInfo might be out-of-order.
+ bool IsDisordered = false;
+
+ llvm::SmallVector<IvarInfo, 8> IvarsInfo;
+ public:
+ IvarLayoutBuilder(CodeGenModule &CGM, CharUnits instanceBegin,
+ CharUnits instanceEnd, bool forStrongLayout)
+ : CGM(CGM), InstanceBegin(instanceBegin), InstanceEnd(instanceEnd),
+ ForStrongLayout(forStrongLayout) {
+ }
+
+ void visitRecord(const RecordType *RT, CharUnits offset);
+
+ template <class Iterator, class GetOffsetFn>
+ void visitAggregate(Iterator begin, Iterator end,
+ CharUnits aggrOffset,
+ const GetOffsetFn &getOffset);
+
+ void visitField(const FieldDecl *field, CharUnits offset);
+
+ /// Add the layout of a block implementation.
+ void visitBlock(const CGBlockInfo &blockInfo);
+
+ /// Is there any information for an interesting bitmap?
+ bool hasBitmapData() const { return !IvarsInfo.empty(); }
+
+ llvm::Constant *buildBitmap(CGObjCCommonMac &CGObjC,
+ llvm::SmallVectorImpl<unsigned char> &buffer);
+
+ static void dump(ArrayRef<unsigned char> buffer) {
+ const unsigned char *s = buffer.data();
+ for (unsigned i = 0, e = buffer.size(); i < e; i++)
+ if (!(s[i] & 0xf0))
+ printf("0x0%x%s", s[i], s[i] != 0 ? ", " : "");
+ else
+ printf("0x%x%s", s[i], s[i] != 0 ? ", " : "");
+ printf("\n");
+ }
+ };
+}
+
llvm::Constant *CGObjCCommonMac::BuildGCBlockLayout(CodeGenModule &CGM,
const CGBlockInfo &blockInfo) {
llvm::Constant *nullPtr = llvm::Constant::getNullValue(CGM.Int8PtrTy);
- if (CGM.getLangOpts().getGC() == LangOptions::NonGC &&
- !CGM.getLangOpts().ObjCAutoRefCount)
+ if (CGM.getLangOpts().getGC() == LangOptions::NonGC)
return nullPtr;
- bool hasUnion = false;
- SkipIvars.clear();
- IvarsInfo.clear();
- unsigned WordSizeInBits = CGM.getTarget().getPointerWidth(0);
- unsigned ByteSizeInBits = CGM.getTarget().getCharWidth();
+ IvarLayoutBuilder builder(CGM, CharUnits::Zero(), blockInfo.BlockSize,
+ /*for strong layout*/ true);
+
+ builder.visitBlock(blockInfo);
+
+ if (!builder.hasBitmapData())
+ return nullPtr;
+
+ llvm::SmallVector<unsigned char, 32> buffer;
+ llvm::Constant *C = builder.buildBitmap(*this, buffer);
+ if (CGM.getLangOpts().ObjCGCBitmapPrint && !buffer.empty()) {
+ printf("\n block variable layout for block: ");
+ builder.dump(buffer);
+ }
+ return C;
+}
+
+void IvarLayoutBuilder::visitBlock(const CGBlockInfo &blockInfo) {
// __isa is the first field in block descriptor and must assume by runtime's
// convention that it is GC'able.
- IvarsInfo.push_back(GC_IVAR(0, 1));
+ IvarsInfo.push_back(IvarInfo(CharUnits::Zero(), 1));
const BlockDecl *blockDecl = blockInfo.getBlockDecl();
- // Calculate the basic layout of the block structure.
- const llvm::StructLayout *layout =
- CGM.getDataLayout().getStructLayout(blockInfo.StructureType);
-
// Ignore the optional 'this' capture: C++ objects are not assumed
// to be GC'ed.
+ CharUnits lastFieldOffset;
+
// Walk the captured variables.
for (const auto &CI : blockDecl->captures()) {
const VarDecl *variable = CI.getVariable();
@@ -1968,64 +2111,51 @@
// Ignore constant captures.
if (capture.isConstant()) continue;
- uint64_t fieldOffset = layout->getElementOffset(capture.getIndex());
+ CharUnits fieldOffset = capture.getOffset();
+
+ // Block fields are not necessarily ordered; if we detect that we're
+ // adding them out-of-order, make sure we sort later.
+ if (fieldOffset < lastFieldOffset)
+ IsDisordered = true;
+ lastFieldOffset = fieldOffset;
// __block variables are passed by their descriptor address.
if (CI.isByRef()) {
- IvarsInfo.push_back(GC_IVAR(fieldOffset, /*size in words*/ 1));
+ IvarsInfo.push_back(IvarInfo(fieldOffset, /*size in words*/ 1));
continue;
}
assert(!type->isArrayType() && "array variable should not be caught");
if (const RecordType *record = type->getAs<RecordType>()) {
- BuildAggrIvarRecordLayout(record, fieldOffset, true, hasUnion);
+ visitRecord(record, fieldOffset);
continue;
}
Qualifiers::GC GCAttr = GetGCAttrTypeForType(CGM.getContext(), type);
- unsigned fieldSize = CGM.getContext().getTypeSize(type);
- if (GCAttr == Qualifiers::Strong)
- IvarsInfo.push_back(GC_IVAR(fieldOffset,
- fieldSize / WordSizeInBits));
- else if (GCAttr == Qualifiers::GCNone || GCAttr == Qualifiers::Weak)
- SkipIvars.push_back(GC_IVAR(fieldOffset,
- fieldSize / ByteSizeInBits));
+ if (GCAttr == Qualifiers::Strong) {
+ assert(CGM.getContext().getTypeSize(type)
+ == CGM.getTarget().getPointerWidth(0));
+ IvarsInfo.push_back(IvarInfo(fieldOffset, /*size in words*/ 1));
+ }
}
-
- if (IvarsInfo.empty())
- return nullPtr;
-
- // Sort on byte position; captures might not be allocated in order,
- // and unions can do funny things.
- llvm::array_pod_sort(IvarsInfo.begin(), IvarsInfo.end());
- llvm::array_pod_sort(SkipIvars.begin(), SkipIvars.end());
-
- std::string BitMap;
- llvm::Constant *C = BuildIvarLayoutBitmap(BitMap);
- if (CGM.getLangOpts().ObjCGCBitmapPrint) {
- printf("\n block variable layout for block: ");
- const unsigned char *s = (const unsigned char*)BitMap.c_str();
- for (unsigned i = 0, e = BitMap.size(); i < e; i++)
- if (!(s[i] & 0xf0))
- printf("0x0%x%s", s[i], s[i] != 0 ? ", " : "");
- else
- printf("0x%x%s", s[i], s[i] != 0 ? ", " : "");
- printf("\n");
- }
-
- return C;
}
+
/// getBlockCaptureLifetime - This routine returns life time of the captured
/// block variable for the purpose of block layout meta-data generation. FQT is
/// the type of the variable captured in the block.
Qualifiers::ObjCLifetime CGObjCCommonMac::getBlockCaptureLifetime(QualType FQT,
bool ByrefLayout) {
+ // If it has an ownership qualifier, we're done.
+ if (auto lifetime = FQT.getObjCLifetime())
+ return lifetime;
+
+ // If it doesn't, and this is ARC, it has no ownership.
if (CGM.getLangOpts().ObjCAutoRefCount)
- return FQT.getObjCLifetime();
+ return Qualifiers::OCL_None;
- // MRR.
+ // In MRC, retainable pointers are owned by non-__block variables.
if (FQT->isObjCObjectPointerType() || FQT->isBlockPointerType())
return ByrefLayout ? Qualifiers::OCL_ExplicitNone : Qualifiers::OCL_Strong;
@@ -2361,9 +2491,8 @@
}
}
- int e = Layout.size()-1;
- while (e >= 0) {
- unsigned char inst = Layout[e--];
+ while (!Layout.empty()) {
+ unsigned char inst = Layout.back();
enum BLOCK_LAYOUT_OPCODE opcode = (enum BLOCK_LAYOUT_OPCODE) (inst >> 4);
if (opcode == BLOCK_LAYOUT_NON_OBJECT_BYTES || opcode == BLOCK_LAYOUT_NON_OBJECT_WORDS)
Layout.pop_back();
@@ -2376,19 +2505,19 @@
// Block variable layout instruction has been inlined.
if (CGM.getLangOpts().ObjCGCBitmapPrint) {
if (ComputeByrefLayout)
- printf("\n Inline instruction for BYREF variable layout: ");
+ printf("\n Inline BYREF variable layout: ");
else
- printf("\n Inline instruction for block variable layout: ");
- printf("0x0%" PRIx64 "\n", Result);
+ printf("\n Inline block variable layout: ");
+ printf("0x0%" PRIx64 "", Result);
+ if (auto numStrong = (Result & 0xF00) >> 8)
+ printf(", BL_STRONG:%d", (int) numStrong);
+ if (auto numByref = (Result & 0x0F0) >> 4)
+ printf(", BL_BYREF:%d", (int) numByref);
+ if (auto numWeak = (Result & 0x00F) >> 0)
+ printf(", BL_WEAK:%d", (int) numWeak);
+ printf(", BL_OPERATOR:0\n");
}
- if (WordSizeInBytes == 8) {
- const llvm::APInt Instruction(64, Result);
- return llvm::Constant::getIntegerValue(CGM.Int64Ty, Instruction);
- }
- else {
- const llvm::APInt Instruction(32, Result);
- return llvm::Constant::getIntegerValue(CGM.Int32Ty, Instruction);
- }
+ return llvm::ConstantInt::get(CGM.IntPtrTy, Result);
}
unsigned char inst = (BLOCK_LAYOUT_OPERATOR << 4) | 0;
@@ -2399,9 +2528,9 @@
if (CGM.getLangOpts().ObjCGCBitmapPrint) {
if (ComputeByrefLayout)
- printf("\n BYREF variable layout: ");
+ printf("\n Byref variable layout: ");
else
- printf("\n block variable layout: ");
+ printf("\n Block variable layout: ");
for (unsigned i = 0, e = BitMap.size(); i != e; i++) {
unsigned char inst = BitMap[i];
enum BLOCK_LAYOUT_OPCODE opcode = (enum BLOCK_LAYOUT_OPCODE) (inst >> 4);
@@ -2443,7 +2572,7 @@
llvm::GlobalVariable *Entry = CreateMetadataVar(
"OBJC_CLASS_NAME_",
llvm::ConstantDataArray::getString(VMContext, BitMap, false),
- "__TEXT,__objc_classname,cstring_literals", 1, true);
+ "__TEXT,__objc_classname,cstring_literals", CharUnits::One(), true);
return getConstantGEP(VMContext, Entry, 0, 0);
}
@@ -2511,6 +2640,8 @@
if (const RecordType *record = T->getAs<RecordType>()) {
BuildRCBlockVarRecordLayout(record, fieldOffset, hasUnion, true /*ByrefLayout */);
llvm::Constant *Result = getBitmapBlockLayout(true);
+ if (isa<llvm::ConstantInt>(Result))
+ Result = llvm::ConstantExpr::getIntToPtr(Result, CGM.Int8PtrTy);
return Result;
}
llvm::Constant *nullPtr = llvm::Constant::getNullValue(CGM.Int8PtrTy);
@@ -2699,7 +2830,7 @@
// No special section, but goes in llvm.used
return CreateMetadataVar("\01l_OBJC_PROTOCOLEXT_" + PD->getName(), Init,
- StringRef(), 0, true);
+ StringRef(), CGM.getPointerAlign(), true);
}
/*
@@ -2738,7 +2869,7 @@
llvm::Constant *Init = llvm::ConstantStruct::getAnon(Values);
llvm::GlobalVariable *GV =
CreateMetadataVar(Name, Init, "__OBJC,__cat_cls_meth,regular,no_dead_strip",
- 4, false);
+ CGM.getPointerAlign(), false);
return llvm::ConstantExpr::getBitCast(GV, ObjCTypes.ProtocolListPtrTy);
}
@@ -2779,15 +2910,26 @@
const ObjCCommonTypesHelper &ObjCTypes) {
SmallVector<llvm::Constant *, 16> Properties;
llvm::SmallPtrSet<const IdentifierInfo*, 16> PropertySet;
+
+ auto AddProperty = [&](const ObjCPropertyDecl *PD) {
+ llvm::Constant *Prop[] = {GetPropertyName(PD->getIdentifier()),
+ GetPropertyTypeString(PD, Container)};
+ Properties.push_back(llvm::ConstantStruct::get(ObjCTypes.PropertyTy, Prop));
+ };
+ if (const ObjCInterfaceDecl *OID = dyn_cast<ObjCInterfaceDecl>(OCD))
+ for (const ObjCCategoryDecl *ClassExt : OID->known_extensions())
+ for (auto *PD : ClassExt->properties()) {
+ PropertySet.insert(PD->getIdentifier());
+ AddProperty(PD);
+ }
for (const auto *PD : OCD->properties()) {
- PropertySet.insert(PD->getIdentifier());
- llvm::Constant *Prop[] = {
- GetPropertyName(PD->getIdentifier()),
- GetPropertyTypeString(PD, Container)
- };
- Properties.push_back(llvm::ConstantStruct::get(ObjCTypes.PropertyTy,
- Prop));
+ // Don't emit duplicate metadata for properties that were already in a
+ // class extension.
+ if (!PropertySet.insert(PD->getIdentifier()).second)
+ continue;
+ AddProperty(PD);
}
+
if (const ObjCInterfaceDecl *OID = dyn_cast<ObjCInterfaceDecl>(OCD)) {
for (const auto *P : OID->all_referenced_protocols())
PushProtocolProperties(PropertySet, Properties, Container, P, ObjCTypes);
@@ -2815,7 +2957,7 @@
CreateMetadataVar(Name, Init,
(ObjCABI == 2) ? "__DATA, __objc_const" :
"__OBJC,__property,regular,no_dead_strip",
- (ObjCABI == 2) ? 8 : 4,
+ CGM.getPointerAlign(),
true);
return llvm::ConstantExpr::getBitCast(GV, ObjCTypes.PropertyListPtrTy);
}
@@ -2834,7 +2976,7 @@
llvm::GlobalVariable *GV = CreateMetadataVar(
Name, Init, (ObjCABI == 2) ? "__DATA, __objc_const" : StringRef(),
- (ObjCABI == 2) ? 8 : 4, true);
+ CGM.getPointerAlign(), true);
return llvm::ConstantExpr::getBitCast(GV, ObjCTypes.Int8PtrPtrTy);
}
@@ -2872,7 +3014,8 @@
Values[1] = llvm::ConstantArray::get(AT, Methods);
llvm::Constant *Init = llvm::ConstantStruct::getAnon(Values);
- llvm::GlobalVariable *GV = CreateMetadataVar(Name, Init, Section, 4, true);
+ llvm::GlobalVariable *GV =
+ CreateMetadataVar(Name, Init, Section, CGM.getPointerAlign(), true);
return llvm::ConstantExpr::getBitCast(GV,
ObjCTypes.MethodDescriptionListPtrTy);
}
@@ -2944,7 +3087,8 @@
llvm::GlobalVariable *GV =
CreateMetadataVar("OBJC_CATEGORY_" + ExtName.str(), Init,
- "__OBJC,__category,regular,no_dead_strip", 4, true);
+ "__OBJC,__category,regular,no_dead_strip",
+ CGM.getPointerAlign(), true);
DefinedCategories.push_back(GV);
DefinedCategoryNames.insert(ExtName.str());
// method definition entries must be clear for next implementation.
@@ -2952,10 +3096,24 @@
}
enum FragileClassFlags {
+ /// Apparently: is not a meta-class.
FragileABI_Class_Factory = 0x00001,
+
+ /// Is a meta-class.
FragileABI_Class_Meta = 0x00002,
+
+ /// Has a non-trivial constructor or destructor.
FragileABI_Class_HasCXXStructors = 0x02000,
- FragileABI_Class_Hidden = 0x20000
+
+ /// Has hidden visibility.
+ FragileABI_Class_Hidden = 0x20000,
+
+ /// Class implementation was compiled under ARC.
+ FragileABI_Class_CompiledByARC = 0x04000000,
+
+ /// Class implementation was compiled under MRC and has MRC weak ivars.
+ /// Exclusive with CompiledByARC.
+ FragileABI_Class_HasMRCWeakIvars = 0x08000000,
};
enum NonFragileClassFlags {
@@ -2965,7 +3123,7 @@
/// Is a root class.
NonFragileABI_Class_Root = 0x00002,
- /// Has a C++ constructor and destructor.
+ /// Has a non-trivial constructor or destructor.
NonFragileABI_Class_HasCXXStructors = 0x00004,
/// Has hidden visibility.
@@ -2981,9 +3139,46 @@
NonFragileABI_Class_CompiledByARC = 0x00080,
/// Class has non-trivial destructors, but zero-initialization is okay.
- NonFragileABI_Class_HasCXXDestructorOnly = 0x00100
+ NonFragileABI_Class_HasCXXDestructorOnly = 0x00100,
+
+ /// Class implementation was compiled under MRC and has MRC weak ivars.
+ /// Exclusive with CompiledByARC.
+ NonFragileABI_Class_HasMRCWeakIvars = 0x00200,
};
+static bool hasWeakMember(QualType type) {
+ if (type.getObjCLifetime() == Qualifiers::OCL_Weak) {
+ return true;
+ }
+
+ if (auto recType = type->getAs<RecordType>()) {
+ for (auto field : recType->getDecl()->fields()) {
+ if (hasWeakMember(field->getType()))
+ return true;
+ }
+ }
+
+ return false;
+}
+
+/// For compatibility, we only want to set the "HasMRCWeakIvars" flag
+/// (and actually fill in a layout string) if we really do have any
+/// __weak ivars.
+static bool hasMRCWeakIvars(CodeGenModule &CGM,
+ const ObjCImplementationDecl *ID) {
+ if (!CGM.getLangOpts().ObjCWeak) return false;
+ assert(CGM.getLangOpts().getGC() == LangOptions::NonGC);
+
+ for (const ObjCIvarDecl *ivar =
+ ID->getClassInterface()->all_declared_ivar_begin();
+ ivar; ivar = ivar->getNextIvar()) {
+ if (hasWeakMember(ivar->getType()))
+ return true;
+ }
+
+ return false;
+}
+
/*
struct _objc_class {
Class isa;
@@ -3017,8 +3212,16 @@
unsigned Flags = FragileABI_Class_Factory;
if (ID->hasNonZeroConstructors() || ID->hasDestructors())
Flags |= FragileABI_Class_HasCXXStructors;
- unsigned Size =
- CGM.getContext().getASTObjCImplementationLayout(ID).getSize().getQuantity();
+
+ bool hasMRCWeak = false;
+
+ if (CGM.getLangOpts().ObjCAutoRefCount)
+ Flags |= FragileABI_Class_CompiledByARC;
+ else if ((hasMRCWeak = hasMRCWeakIvars(CGM, ID)))
+ Flags |= FragileABI_Class_HasMRCWeakIvars;
+
+ CharUnits Size =
+ CGM.getContext().getASTObjCImplementationLayout(ID).getSize();
// FIXME: Set CXX-structors flag.
if (ID->getClassInterface()->getVisibility() == HiddenVisibility)
@@ -3062,7 +3265,7 @@
// Version is always 0.
Values[ 3] = llvm::ConstantInt::get(ObjCTypes.LongTy, 0);
Values[ 4] = llvm::ConstantInt::get(ObjCTypes.LongTy, Flags);
- Values[ 5] = llvm::ConstantInt::get(ObjCTypes.LongTy, Size);
+ Values[ 5] = llvm::ConstantInt::get(ObjCTypes.LongTy, Size.getQuantity());
Values[ 6] = EmitIvarList(ID, false);
Values[7] = EmitMethodList("OBJC_INSTANCE_METHODS_" + ID->getName(),
"__OBJC,__inst_meth,regular,no_dead_strip",
@@ -3070,8 +3273,8 @@
// cache is always NULL.
Values[ 8] = llvm::Constant::getNullValue(ObjCTypes.CachePtrTy);
Values[ 9] = Protocols;
- Values[10] = BuildIvarLayout(ID, true);
- Values[11] = EmitClassExtension(ID);
+ Values[10] = BuildStrongIvarLayout(ID, CharUnits::Zero(), Size);
+ Values[11] = EmitClassExtension(ID, Size, hasMRCWeak);
llvm::Constant *Init = llvm::ConstantStruct::get(ObjCTypes.ClassTy,
Values);
std::string Name("OBJC_CLASS_");
@@ -3084,10 +3287,10 @@
"Forward metaclass reference has incorrect type.");
GV->setInitializer(Init);
GV->setSection(Section);
- GV->setAlignment(4);
+ GV->setAlignment(CGM.getPointerAlign().getQuantity());
CGM.addCompilerUsedGlobal(GV);
} else
- GV = CreateMetadataVar(Name, Init, Section, 4, true);
+ GV = CreateMetadataVar(Name, Init, Section, CGM.getPointerAlign(), true);
DefinedClasses.push_back(GV);
ImplementedClasses.push_back(Interface);
// method definition entries must be clear for next implementation.
@@ -3198,6 +3401,10 @@
}
/*
+ Emit a "class extension", which in this specific context means extra
+ data that doesn't fit in the normal fragile-ABI class structure, and
+ has nothing to do with the language concept of a class extension.
+
struct objc_class_ext {
uint32_t size;
const char *weak_ivar_layout;
@@ -3205,13 +3412,15 @@
};
*/
llvm::Constant *
-CGObjCMac::EmitClassExtension(const ObjCImplementationDecl *ID) {
+CGObjCMac::EmitClassExtension(const ObjCImplementationDecl *ID,
+ CharUnits InstanceSize, bool hasMRCWeakIvars) {
uint64_t Size =
CGM.getDataLayout().getTypeAllocSize(ObjCTypes.ClassExtensionTy);
llvm::Constant *Values[3];
Values[0] = llvm::ConstantInt::get(ObjCTypes.IntTy, Size);
- Values[1] = BuildIvarLayout(ID, false);
+ Values[1] = BuildWeakIvarLayout(ID, CharUnits::Zero(), InstanceSize,
+ hasMRCWeakIvars);
Values[2] = EmitPropertyList("\01l_OBJC_$_PROP_LIST_" + ID->getName(),
ID, ID->getClassInterface(), ObjCTypes);
@@ -3222,7 +3431,8 @@
llvm::Constant *Init =
llvm::ConstantStruct::get(ObjCTypes.ClassExtensionTy, Values);
return CreateMetadataVar("OBJC_CLASSEXT_" + ID->getName(), Init,
- "__OBJC,__class_ext,regular,no_dead_strip", 4, true);
+ "__OBJC,__class_ext,regular,no_dead_strip",
+ CGM.getPointerAlign(), true);
}
/*
@@ -3280,11 +3490,12 @@
if (ForClass)
GV =
CreateMetadataVar("OBJC_CLASS_VARIABLES_" + ID->getName(), Init,
- "__OBJC,__class_vars,regular,no_dead_strip", 4, true);
+ "__OBJC,__class_vars,regular,no_dead_strip",
+ CGM.getPointerAlign(), true);
else
GV = CreateMetadataVar("OBJC_INSTANCE_VARIABLES_" + ID->getName(), Init,
- "__OBJC,__instance_vars,regular,no_dead_strip", 4,
- true);
+ "__OBJC,__instance_vars,regular,no_dead_strip",
+ CGM.getPointerAlign(), true);
return llvm::ConstantExpr::getBitCast(GV, ObjCTypes.IvarListPtrTy);
}
@@ -3334,7 +3545,8 @@
Values[2] = llvm::ConstantArray::get(AT, Methods);
llvm::Constant *Init = llvm::ConstantStruct::getAnon(Values);
- llvm::GlobalVariable *GV = CreateMetadataVar(Name, Init, Section, 4, true);
+ llvm::GlobalVariable *GV =
+ CreateMetadataVar(Name, Init, Section, CGM.getPointerAlign(), true);
return llvm::ConstantExpr::getBitCast(GV, ObjCTypes.MethodListPtrTy);
}
@@ -3359,7 +3571,7 @@
llvm::GlobalVariable *CGObjCCommonMac::CreateMetadataVar(Twine Name,
llvm::Constant *Init,
StringRef Section,
- unsigned Align,
+ CharUnits Align,
bool AddToUsed) {
llvm::Type *Ty = Init->getType();
llvm::GlobalVariable *GV =
@@ -3367,8 +3579,7 @@
llvm::GlobalValue::PrivateLinkage, Init, Name);
if (!Section.empty())
GV->setSection(Section);
- if (Align)
- GV->setAlignment(Align);
+ GV->setAlignment(Align.getQuantity());
if (AddToUsed)
CGM.addCompilerUsedGlobal(GV);
return GV;
@@ -3421,16 +3632,16 @@
}
namespace {
- struct PerformFragileFinally : EHScopeStack::Cleanup {
+ struct PerformFragileFinally final : EHScopeStack::Cleanup {
const Stmt &S;
- llvm::Value *SyncArgSlot;
- llvm::Value *CallTryExitVar;
- llvm::Value *ExceptionData;
+ Address SyncArgSlot;
+ Address CallTryExitVar;
+ Address ExceptionData;
ObjCTypesHelper &ObjCTypes;
PerformFragileFinally(const Stmt *S,
- llvm::Value *SyncArgSlot,
- llvm::Value *CallTryExitVar,
- llvm::Value *ExceptionData,
+ Address SyncArgSlot,
+ Address CallTryExitVar,
+ Address ExceptionData,
ObjCTypesHelper *ObjCTypes)
: S(*S), SyncArgSlot(SyncArgSlot), CallTryExitVar(CallTryExitVar),
ExceptionData(ExceptionData), ObjCTypes(*ObjCTypes) {}
@@ -3447,7 +3658,7 @@
CGF.EmitBlock(FinallyCallExit);
CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryExitFn(),
- ExceptionData);
+ ExceptionData.getPointer());
CGF.EmitBlock(FinallyNoCallExit);
@@ -3568,7 +3779,7 @@
void FragileHazards::emitHazardsInNewBlocks() {
if (Locals.empty()) return;
- CGBuilderTy Builder(CGF.getLLVMContext());
+ CGBuilderTy Builder(CGF, CGF.getLLVMContext());
// Iterate through all blocks, skipping those prior to the try.
for (llvm::Function::iterator
@@ -3607,6 +3818,10 @@
if (V) S.insert(V);
}
+static void addIfPresent(llvm::DenseSet<llvm::Value*> &S, Address V) {
+ if (V.isValid()) S.insert(V.getPointer());
+}
+
void FragileHazards::collectLocals() {
// Compute a set of allocas to ignore.
llvm::DenseSet<llvm::Value*> AllocasToIgnore;
@@ -3760,21 +3975,23 @@
// @synchronized. We can't avoid a temp here because we need the
// value to be preserved. If the backend ever does liveness
// correctly after setjmp, this will be unnecessary.
- llvm::Value *SyncArgSlot = nullptr;
+ Address SyncArgSlot = Address::invalid();
if (!isTry) {
llvm::Value *SyncArg =
CGF.EmitScalarExpr(cast<ObjCAtSynchronizedStmt>(S).getSynchExpr());
SyncArg = CGF.Builder.CreateBitCast(SyncArg, ObjCTypes.ObjectPtrTy);
CGF.EmitNounwindRuntimeCall(ObjCTypes.getSyncEnterFn(), SyncArg);
- SyncArgSlot = CGF.CreateTempAlloca(SyncArg->getType(), "sync.arg");
+ SyncArgSlot = CGF.CreateTempAlloca(SyncArg->getType(),
+ CGF.getPointerAlign(), "sync.arg");
CGF.Builder.CreateStore(SyncArg, SyncArgSlot);
}
// Allocate memory for the setjmp buffer. This needs to be kept
// live throughout the try and catch blocks.
- llvm::Value *ExceptionData = CGF.CreateTempAlloca(ObjCTypes.ExceptionDataTy,
- "exceptiondata.ptr");
+ Address ExceptionData = CGF.CreateTempAlloca(ObjCTypes.ExceptionDataTy,
+ CGF.getPointerAlign(),
+ "exceptiondata.ptr");
// Create the fragile hazards. Note that this will not capture any
// of the allocas required for exception processing, but will
@@ -3790,12 +4007,13 @@
// The setjmp-safety rule here is that we should always store to this
// variable in a place that dominates the branch through the cleanup
// without passing through any setjmps.
- llvm::Value *CallTryExitVar = CGF.CreateTempAlloca(CGF.Builder.getInt1Ty(),
- "_call_try_exit");
+ Address CallTryExitVar = CGF.CreateTempAlloca(CGF.Builder.getInt1Ty(),
+ CharUnits::One(),
+ "_call_try_exit");
// A slot containing the exception to rethrow. Only needed when we
// have both a @catch and a @finally.
- llvm::Value *PropagatingExnVar = nullptr;
+ Address PropagatingExnVar = Address::invalid();
// Push a normal cleanup to leave the try scope.
CGF.EHStack.pushCleanup<PerformFragileFinally>(NormalAndEHCleanup, &S,
@@ -3808,13 +4026,14 @@
// - Call objc_exception_try_enter to push ExceptionData on top of
// the EH stack.
CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryEnterFn(),
- ExceptionData);
+ ExceptionData.getPointer());
// - Call setjmp on the exception data buffer.
llvm::Constant *Zero = llvm::ConstantInt::get(CGF.Builder.getInt32Ty(), 0);
llvm::Value *GEPIndexes[] = { Zero, Zero, Zero };
llvm::Value *SetJmpBuffer = CGF.Builder.CreateGEP(
- ObjCTypes.ExceptionDataTy, ExceptionData, GEPIndexes, "setjmp_buffer");
+ ObjCTypes.ExceptionDataTy, ExceptionData.getPointer(), GEPIndexes,
+ "setjmp_buffer");
llvm::CallInst *SetJmpResult = CGF.EmitNounwindRuntimeCall(
ObjCTypes.getSetJmpFn(), SetJmpBuffer, "setjmp_result");
SetJmpResult->setCanReturnTwice();
@@ -3854,7 +4073,7 @@
// nothing can cross this so the value is already in SSA form.
llvm::CallInst *Caught =
CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionExtractFn(),
- ExceptionData, "caught");
+ ExceptionData.getPointer(), "caught");
// Push the exception to rethrow onto the EH value stack for the
// benefit of any @throws in the handlers.
@@ -3870,13 +4089,14 @@
// Save the currently-propagating exception before
// objc_exception_try_enter clears the exception slot.
PropagatingExnVar = CGF.CreateTempAlloca(Caught->getType(),
+ CGF.getPointerAlign(),
"propagating_exception");
CGF.Builder.CreateStore(Caught, PropagatingExnVar);
// Enter a new exception try block (in case a @catch block
// throws an exception).
CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryEnterFn(),
- ExceptionData);
+ ExceptionData.getPointer());
llvm::CallInst *SetJmpResult =
CGF.EmitNounwindRuntimeCall(ObjCTypes.getSetJmpFn(),
@@ -3928,7 +4148,7 @@
assert(CGF.HaveInsertPoint() && "DeclStmt destroyed insert point?");
// These types work out because ConvertType(id) == i8*.
- CGF.Builder.CreateStore(Caught, CGF.GetAddrOfLocalVar(CatchParam));
+ EmitInitOfCatchParam(CGF, Caught, CatchParam);
}
CGF.EmitStmt(CatchStmt->getCatchBody());
@@ -3975,7 +4195,7 @@
llvm::Value *Tmp =
CGF.Builder.CreateBitCast(Caught,
CGF.ConvertType(CatchParam->getType()));
- CGF.Builder.CreateStore(Tmp, CGF.GetAddrOfLocalVar(CatchParam));
+ EmitInitOfCatchParam(CGF, Tmp, CatchParam);
CGF.EmitStmt(CatchStmt->getCatchBody());
@@ -4008,10 +4228,10 @@
// Extract the new exception and save it to the
// propagating-exception slot.
- assert(PropagatingExnVar);
+ assert(PropagatingExnVar.isValid());
llvm::CallInst *NewCaught =
CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionExtractFn(),
- ExceptionData, "caught");
+ ExceptionData.getPointer(), "caught");
CGF.Builder.CreateStore(NewCaught, PropagatingExnVar);
// Don't pop the catch handler; the throw already did.
@@ -4036,14 +4256,14 @@
if (CGF.HaveInsertPoint()) {
// If we have a propagating-exception variable, check it.
llvm::Value *PropagatingExn;
- if (PropagatingExnVar) {
+ if (PropagatingExnVar.isValid()) {
PropagatingExn = CGF.Builder.CreateLoad(PropagatingExnVar);
// Otherwise, just look in the buffer for the exception to throw.
} else {
llvm::CallInst *Caught =
CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionExtractFn(),
- ExceptionData);
+ ExceptionData.getPointer());
PropagatingExn = Caught;
}
@@ -4083,14 +4303,13 @@
/// object: objc_read_weak (id *src)
///
llvm::Value * CGObjCMac::EmitObjCWeakRead(CodeGen::CodeGenFunction &CGF,
- llvm::Value *AddrWeakObj) {
- llvm::Type* DestTy =
- cast<llvm::PointerType>(AddrWeakObj->getType())->getElementType();
+ Address AddrWeakObj) {
+ llvm::Type* DestTy = AddrWeakObj.getElementType();
AddrWeakObj = CGF.Builder.CreateBitCast(AddrWeakObj,
ObjCTypes.PtrObjectPtrTy);
llvm::Value *read_weak =
CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcReadWeakFn(),
- AddrWeakObj, "weakread");
+ AddrWeakObj.getPointer(), "weakread");
read_weak = CGF.Builder.CreateBitCast(read_weak, DestTy);
return read_weak;
}
@@ -4099,7 +4318,7 @@
/// objc_assign_weak (id src, id *dst)
///
void CGObjCMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF,
- llvm::Value *src, llvm::Value *dst) {
+ llvm::Value *src, Address dst) {
llvm::Type * SrcTy = src->getType();
if (!isa<llvm::PointerType>(SrcTy)) {
unsigned Size = CGM.getDataLayout().getTypeAllocSize(SrcTy);
@@ -4110,7 +4329,7 @@
}
src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
dst = CGF.Builder.CreateBitCast(dst, ObjCTypes.PtrObjectPtrTy);
- llvm::Value *args[] = { src, dst };
+ llvm::Value *args[] = { src, dst.getPointer() };
CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignWeakFn(),
args, "weakassign");
return;
@@ -4120,7 +4339,7 @@
/// objc_assign_global (id src, id *dst)
///
void CGObjCMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF,
- llvm::Value *src, llvm::Value *dst,
+ llvm::Value *src, Address dst,
bool threadlocal) {
llvm::Type * SrcTy = src->getType();
if (!isa<llvm::PointerType>(SrcTy)) {
@@ -4132,7 +4351,7 @@
}
src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
dst = CGF.Builder.CreateBitCast(dst, ObjCTypes.PtrObjectPtrTy);
- llvm::Value *args[] = { src, dst };
+ llvm::Value *args[] = { src, dst.getPointer() };
if (!threadlocal)
CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignGlobalFn(),
args, "globalassign");
@@ -4146,7 +4365,7 @@
/// objc_assign_ivar (id src, id *dst, ptrdiff_t ivaroffset)
///
void CGObjCMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF,
- llvm::Value *src, llvm::Value *dst,
+ llvm::Value *src, Address dst,
llvm::Value *ivarOffset) {
assert(ivarOffset && "EmitObjCIvarAssign - ivarOffset is NULL");
llvm::Type * SrcTy = src->getType();
@@ -4159,7 +4378,7 @@
}
src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
dst = CGF.Builder.CreateBitCast(dst, ObjCTypes.PtrObjectPtrTy);
- llvm::Value *args[] = { src, dst, ivarOffset };
+ llvm::Value *args[] = { src, dst.getPointer(), ivarOffset };
CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignIvarFn(), args);
return;
}
@@ -4168,7 +4387,7 @@
/// objc_assign_strongCast (id src, id *dst)
///
void CGObjCMac::EmitObjCStrongCastAssign(CodeGen::CodeGenFunction &CGF,
- llvm::Value *src, llvm::Value *dst) {
+ llvm::Value *src, Address dst) {
llvm::Type * SrcTy = src->getType();
if (!isa<llvm::PointerType>(SrcTy)) {
unsigned Size = CGM.getDataLayout().getTypeAllocSize(SrcTy);
@@ -4179,19 +4398,19 @@
}
src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
dst = CGF.Builder.CreateBitCast(dst, ObjCTypes.PtrObjectPtrTy);
- llvm::Value *args[] = { src, dst };
+ llvm::Value *args[] = { src, dst.getPointer() };
CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignStrongCastFn(),
- args, "weakassign");
+ args, "strongassign");
return;
}
void CGObjCMac::EmitGCMemmoveCollectable(CodeGen::CodeGenFunction &CGF,
- llvm::Value *DestPtr,
- llvm::Value *SrcPtr,
+ Address DestPtr,
+ Address SrcPtr,
llvm::Value *size) {
SrcPtr = CGF.Builder.CreateBitCast(SrcPtr, ObjCTypes.Int8PtrTy);
DestPtr = CGF.Builder.CreateBitCast(DestPtr, ObjCTypes.Int8PtrTy);
- llvm::Value *args[] = { DestPtr, SrcPtr, size };
+ llvm::Value *args[] = { DestPtr.getPointer(), SrcPtr.getPointer(), size };
CGF.EmitNounwindRuntimeCall(ObjCTypes.GcMemmoveCollectableFn(), args);
}
@@ -4283,7 +4502,7 @@
// Indicate whether we're compiling this to run on a simulator.
const llvm::Triple &Triple = CGM.getTarget().getTriple();
- if (Triple.isiOS() &&
+ if ((Triple.isiOS() || Triple.isWatchOS()) &&
(Triple.getArch() == llvm::Triple::x86 ||
Triple.getArch() == llvm::Triple::x86_64))
Mod.addModuleFlag(llvm::Module::Error, "Objective-C Is Simulated",
@@ -4312,7 +4531,8 @@
};
CreateMetadataVar("OBJC_MODULES",
llvm::ConstantStruct::get(ObjCTypes.ModuleTy, Values),
- "__OBJC,__module_info,regular,no_dead_strip", 4, true);
+ "__OBJC,__module_info,regular,no_dead_strip",
+ CGM.getPointerAlign(), true);
}
llvm::Constant *CGObjCMac::EmitModuleSymbols() {
@@ -4356,7 +4576,8 @@
llvm::Constant *Init = llvm::ConstantStruct::getAnon(Values);
llvm::GlobalVariable *GV = CreateMetadataVar(
- "OBJC_SYMBOLS", Init, "__OBJC,__symbols,regular,no_dead_strip", 4, true);
+ "OBJC_SYMBOLS", Init, "__OBJC,__symbols,regular,no_dead_strip",
+ CGM.getPointerAlign(), true);
return llvm::ConstantExpr::getBitCast(GV, ObjCTypes.SymtabPtrTy);
}
@@ -4372,10 +4593,11 @@
ObjCTypes.ClassPtrTy);
Entry = CreateMetadataVar(
"OBJC_CLASS_REFERENCES_", Casted,
- "__OBJC,__cls_refs,literal_pointers,no_dead_strip", 4, true);
+ "__OBJC,__cls_refs,literal_pointers,no_dead_strip",
+ CGM.getPointerAlign(), true);
}
- return CGF.Builder.CreateLoad(Entry);
+ return CGF.Builder.CreateAlignedLoad(Entry, CGF.getPointerAlign());
}
llvm::Value *CGObjCMac::EmitClassRef(CodeGenFunction &CGF,
@@ -4388,23 +4610,25 @@
return EmitClassRefFromId(CGF, II);
}
-llvm::Value *CGObjCMac::EmitSelector(CodeGenFunction &CGF, Selector Sel,
- bool lvalue) {
- llvm::GlobalVariable *&Entry = SelectorReferences[Sel];
+llvm::Value *CGObjCMac::EmitSelector(CodeGenFunction &CGF, Selector Sel) {
+ return CGF.Builder.CreateLoad(EmitSelectorAddr(CGF, Sel));
+}
+Address CGObjCMac::EmitSelectorAddr(CodeGenFunction &CGF, Selector Sel) {
+ CharUnits Align = CGF.getPointerAlign();
+
+ llvm::GlobalVariable *&Entry = SelectorReferences[Sel];
if (!Entry) {
llvm::Constant *Casted =
llvm::ConstantExpr::getBitCast(GetMethodVarName(Sel),
ObjCTypes.SelectorPtrTy);
Entry = CreateMetadataVar(
"OBJC_SELECTOR_REFERENCES_", Casted,
- "__OBJC,__message_refs,literal_pointers,no_dead_strip", 4, true);
+ "__OBJC,__message_refs,literal_pointers,no_dead_strip", Align, true);
Entry->setExternallyInitialized(true);
}
- if (lvalue)
- return Entry;
- return CGF.Builder.CreateLoad(Entry);
+ return Address(Entry, Align);
}
llvm::Constant *CGObjCCommonMac::GetClassName(StringRef RuntimeName) {
@@ -4415,7 +4639,7 @@
llvm::ConstantDataArray::getString(VMContext, RuntimeName),
((ObjCABI == 2) ? "__TEXT,__objc_classname,cstring_literals"
: "__TEXT,__cstring,cstring_literals"),
- 1, true);
+ CharUnits::One(), true);
return getConstantGEP(VMContext, Entry, 0, 0);
}
@@ -4435,308 +4659,247 @@
return llvm::Constant::getNullValue(ObjCTypes.Int8PtrTy);
}
-void CGObjCCommonMac::BuildAggrIvarRecordLayout(const RecordType *RT,
- unsigned int BytePos,
- bool ForStrongLayout,
- bool &HasUnion) {
+void IvarLayoutBuilder::visitRecord(const RecordType *RT,
+ CharUnits offset) {
const RecordDecl *RD = RT->getDecl();
- // FIXME - Use iterator.
- SmallVector<const FieldDecl*, 16> Fields(RD->fields());
- llvm::Type *Ty = CGM.getTypes().ConvertType(QualType(RT, 0));
- const llvm::StructLayout *RecLayout =
- CGM.getDataLayout().getStructLayout(cast<llvm::StructType>(Ty));
- BuildAggrIvarLayout(nullptr, RecLayout, RD, Fields, BytePos, ForStrongLayout,
- HasUnion);
+ // If this is a union, remember that we had one, because it might mess
+ // up the ordering of layout entries.
+ if (RD->isUnion())
+ IsDisordered = true;
+
+ const ASTRecordLayout *recLayout = nullptr;
+ visitAggregate(RD->field_begin(), RD->field_end(), offset,
+ [&](const FieldDecl *field) -> CharUnits {
+ if (!recLayout)
+ recLayout = &CGM.getContext().getASTRecordLayout(RD);
+ auto offsetInBits = recLayout->getFieldOffset(field->getFieldIndex());
+ return CGM.getContext().toCharUnitsFromBits(offsetInBits);
+ });
}
-void CGObjCCommonMac::BuildAggrIvarLayout(const ObjCImplementationDecl *OI,
- const llvm::StructLayout *Layout,
- const RecordDecl *RD,
- ArrayRef<const FieldDecl*> RecFields,
- unsigned int BytePos, bool ForStrongLayout,
- bool &HasUnion) {
- bool IsUnion = (RD && RD->isUnion());
- uint64_t MaxUnionIvarSize = 0;
- uint64_t MaxSkippedUnionIvarSize = 0;
- const FieldDecl *MaxField = nullptr;
- const FieldDecl *MaxSkippedField = nullptr;
- const FieldDecl *LastFieldBitfieldOrUnnamed = nullptr;
- uint64_t MaxFieldOffset = 0;
- uint64_t MaxSkippedFieldOffset = 0;
- uint64_t LastBitfieldOrUnnamedOffset = 0;
- uint64_t FirstFieldDelta = 0;
+template <class Iterator, class GetOffsetFn>
+void IvarLayoutBuilder::visitAggregate(Iterator begin, Iterator end,
+ CharUnits aggregateOffset,
+ const GetOffsetFn &getOffset) {
+ for (; begin != end; ++begin) {
+ auto field = *begin;
- if (RecFields.empty())
+ // Skip over bitfields.
+ if (field->isBitField()) {
+ continue;
+ }
+
+ // Compute the offset of the field within the aggregate.
+ CharUnits fieldOffset = aggregateOffset + getOffset(field);
+
+ visitField(field, fieldOffset);
+ }
+}
+
+/// Collect layout information for the given fields into IvarsInfo.
+void IvarLayoutBuilder::visitField(const FieldDecl *field,
+ CharUnits fieldOffset) {
+ QualType fieldType = field->getType();
+
+ // Drill down into arrays.
+ uint64_t numElts = 1;
+ while (auto arrayType = CGM.getContext().getAsConstantArrayType(fieldType)) {
+ numElts *= arrayType->getSize().getZExtValue();
+ fieldType = arrayType->getElementType();
+ }
+
+ assert(!fieldType->isArrayType() && "ivar of non-constant array type?");
+
+ // If we ended up with a zero-sized array, we've done what we can do within
+ // the limits of this layout encoding.
+ if (numElts == 0) return;
+
+ // Recurse if the base element type is a record type.
+ if (auto recType = fieldType->getAs<RecordType>()) {
+ size_t oldEnd = IvarsInfo.size();
+
+ visitRecord(recType, fieldOffset);
+
+ // If we have an array, replicate the first entry's layout information.
+ auto numEltEntries = IvarsInfo.size() - oldEnd;
+ if (numElts != 1 && numEltEntries != 0) {
+ CharUnits eltSize = CGM.getContext().getTypeSizeInChars(recType);
+ for (uint64_t eltIndex = 1; eltIndex != numElts; ++eltIndex) {
+ // Copy the last numEltEntries onto the end of the array, adjusting
+ // each for the element size.
+ for (size_t i = 0; i != numEltEntries; ++i) {
+ auto firstEntry = IvarsInfo[oldEnd + i];
+ IvarsInfo.push_back(IvarInfo(firstEntry.Offset + eltIndex * eltSize,
+ firstEntry.SizeInWords));
+ }
+ }
+ }
+
return;
- unsigned WordSizeInBits = CGM.getTarget().getPointerWidth(0);
- unsigned ByteSizeInBits = CGM.getTarget().getCharWidth();
- if (!RD && CGM.getLangOpts().ObjCAutoRefCount) {
- const FieldDecl *FirstField = RecFields[0];
- FirstFieldDelta =
- ComputeIvarBaseOffset(CGM, OI, cast<ObjCIvarDecl>(FirstField));
- }
-
- for (unsigned i = 0, e = RecFields.size(); i != e; ++i) {
- const FieldDecl *Field = RecFields[i];
- uint64_t FieldOffset;
- if (RD) {
- // Note that 'i' here is actually the field index inside RD of Field,
- // although this dependency is hidden.
- const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD);
- FieldOffset = (RL.getFieldOffset(i) / ByteSizeInBits) - FirstFieldDelta;
- } else
- FieldOffset =
- ComputeIvarBaseOffset(CGM, OI, cast<ObjCIvarDecl>(Field)) - FirstFieldDelta;
-
- // Skip over unnamed or bitfields
- if (!Field->getIdentifier() || Field->isBitField()) {
- LastFieldBitfieldOrUnnamed = Field;
- LastBitfieldOrUnnamedOffset = FieldOffset;
- continue;
- }
-
- LastFieldBitfieldOrUnnamed = nullptr;
- QualType FQT = Field->getType();
- if (FQT->isRecordType() || FQT->isUnionType()) {
- if (FQT->isUnionType())
- HasUnion = true;
-
- BuildAggrIvarRecordLayout(FQT->getAs<RecordType>(),
- BytePos + FieldOffset,
- ForStrongLayout, HasUnion);
- continue;
- }
-
- if (const ArrayType *Array = CGM.getContext().getAsArrayType(FQT)) {
- const ConstantArrayType *CArray =
- dyn_cast_or_null<ConstantArrayType>(Array);
- uint64_t ElCount = CArray->getSize().getZExtValue();
- assert(CArray && "only array with known element size is supported");
- FQT = CArray->getElementType();
- while (const ArrayType *Array = CGM.getContext().getAsArrayType(FQT)) {
- const ConstantArrayType *CArray =
- dyn_cast_or_null<ConstantArrayType>(Array);
- ElCount *= CArray->getSize().getZExtValue();
- FQT = CArray->getElementType();
- }
- if (FQT->isRecordType() && ElCount) {
- int OldIndex = IvarsInfo.size() - 1;
- int OldSkIndex = SkipIvars.size() -1;
-
- const RecordType *RT = FQT->getAs<RecordType>();
- BuildAggrIvarRecordLayout(RT, BytePos + FieldOffset,
- ForStrongLayout, HasUnion);
-
- // Replicate layout information for each array element. Note that
- // one element is already done.
- uint64_t ElIx = 1;
- for (int FirstIndex = IvarsInfo.size() - 1,
- FirstSkIndex = SkipIvars.size() - 1 ;ElIx < ElCount; ElIx++) {
- uint64_t Size = CGM.getContext().getTypeSize(RT)/ByteSizeInBits;
- for (int i = OldIndex+1; i <= FirstIndex; ++i)
- IvarsInfo.push_back(GC_IVAR(IvarsInfo[i].ivar_bytepos + Size*ElIx,
- IvarsInfo[i].ivar_size));
- for (int i = OldSkIndex+1; i <= FirstSkIndex; ++i)
- SkipIvars.push_back(GC_IVAR(SkipIvars[i].ivar_bytepos + Size*ElIx,
- SkipIvars[i].ivar_size));
- }
- continue;
- }
- }
- // At this point, we are done with Record/Union and array there of.
- // For other arrays we are down to its element type.
- Qualifiers::GC GCAttr = GetGCAttrTypeForType(CGM.getContext(), FQT);
-
- unsigned FieldSize = CGM.getContext().getTypeSize(Field->getType());
- if ((ForStrongLayout && GCAttr == Qualifiers::Strong)
- || (!ForStrongLayout && GCAttr == Qualifiers::Weak)) {
- if (IsUnion) {
- uint64_t UnionIvarSize = FieldSize / WordSizeInBits;
- if (UnionIvarSize > MaxUnionIvarSize) {
- MaxUnionIvarSize = UnionIvarSize;
- MaxField = Field;
- MaxFieldOffset = FieldOffset;
- }
- } else {
- IvarsInfo.push_back(GC_IVAR(BytePos + FieldOffset,
- FieldSize / WordSizeInBits));
- }
- } else if ((ForStrongLayout &&
- (GCAttr == Qualifiers::GCNone || GCAttr == Qualifiers::Weak))
- || (!ForStrongLayout && GCAttr != Qualifiers::Weak)) {
- if (IsUnion) {
- // FIXME: Why the asymmetry? We divide by word size in bits on other
- // side.
- uint64_t UnionIvarSize = FieldSize / ByteSizeInBits;
- if (UnionIvarSize > MaxSkippedUnionIvarSize) {
- MaxSkippedUnionIvarSize = UnionIvarSize;
- MaxSkippedField = Field;
- MaxSkippedFieldOffset = FieldOffset;
- }
- } else {
- // FIXME: Why the asymmetry, we divide by byte size in bits here?
- SkipIvars.push_back(GC_IVAR(BytePos + FieldOffset,
- FieldSize / ByteSizeInBits));
- }
- }
}
- if (LastFieldBitfieldOrUnnamed) {
- if (LastFieldBitfieldOrUnnamed->isBitField()) {
- // Last field was a bitfield. Must update skip info.
- uint64_t BitFieldSize
- = LastFieldBitfieldOrUnnamed->getBitWidthValue(CGM.getContext());
- GC_IVAR skivar;
- skivar.ivar_bytepos = BytePos + LastBitfieldOrUnnamedOffset;
- skivar.ivar_size = (BitFieldSize / ByteSizeInBits)
- + ((BitFieldSize % ByteSizeInBits) != 0);
- SkipIvars.push_back(skivar);
- } else {
- assert(!LastFieldBitfieldOrUnnamed->getIdentifier() &&"Expected unnamed");
- // Last field was unnamed. Must update skip info.
- unsigned FieldSize
- = CGM.getContext().getTypeSize(LastFieldBitfieldOrUnnamed->getType());
- SkipIvars.push_back(GC_IVAR(BytePos + LastBitfieldOrUnnamedOffset,
- FieldSize / ByteSizeInBits));
- }
- }
+ // Classify the element type.
+ Qualifiers::GC GCAttr = GetGCAttrTypeForType(CGM.getContext(), fieldType);
- if (MaxField)
- IvarsInfo.push_back(GC_IVAR(BytePos + MaxFieldOffset,
- MaxUnionIvarSize));
- if (MaxSkippedField)
- SkipIvars.push_back(GC_IVAR(BytePos + MaxSkippedFieldOffset,
- MaxSkippedUnionIvarSize));
+ // If it matches what we're looking for, add an entry.
+ if ((ForStrongLayout && GCAttr == Qualifiers::Strong)
+ || (!ForStrongLayout && GCAttr == Qualifiers::Weak)) {
+ assert(CGM.getContext().getTypeSizeInChars(fieldType)
+ == CGM.getPointerSize());
+ IvarsInfo.push_back(IvarInfo(fieldOffset, numElts));
+ }
}
-/// BuildIvarLayoutBitmap - This routine is the horsework for doing all
-/// the computations and returning the layout bitmap (for ivar or blocks) in
-/// the given argument BitMap string container. Routine reads
-/// two containers, IvarsInfo and SkipIvars which are assumed to be
-/// filled already by the caller.
-llvm::Constant *CGObjCCommonMac::BuildIvarLayoutBitmap(std::string &BitMap) {
- unsigned int WordsToScan, WordsToSkip;
- llvm::Type *PtrTy = CGM.Int8PtrTy;
-
- // Build the string of skip/scan nibbles
- SmallVector<SKIP_SCAN, 32> SkipScanIvars;
- unsigned int WordSize =
- CGM.getTypes().getDataLayout().getTypeAllocSize(PtrTy);
- if (IvarsInfo[0].ivar_bytepos == 0) {
- WordsToSkip = 0;
- WordsToScan = IvarsInfo[0].ivar_size;
- } else {
- WordsToSkip = IvarsInfo[0].ivar_bytepos/WordSize;
- WordsToScan = IvarsInfo[0].ivar_size;
- }
- for (unsigned int i=1, Last=IvarsInfo.size(); i != Last; i++) {
- unsigned int TailPrevGCObjC =
- IvarsInfo[i-1].ivar_bytepos + IvarsInfo[i-1].ivar_size * WordSize;
- if (IvarsInfo[i].ivar_bytepos == TailPrevGCObjC) {
- // consecutive 'scanned' object pointers.
- WordsToScan += IvarsInfo[i].ivar_size;
- } else {
- // Skip over 'gc'able object pointer which lay over each other.
- if (TailPrevGCObjC > IvarsInfo[i].ivar_bytepos)
- continue;
- // Must skip over 1 or more words. We save current skip/scan values
- // and start a new pair.
- SKIP_SCAN SkScan;
- SkScan.skip = WordsToSkip;
- SkScan.scan = WordsToScan;
- SkipScanIvars.push_back(SkScan);
-
- // Skip the hole.
- SkScan.skip = (IvarsInfo[i].ivar_bytepos - TailPrevGCObjC) / WordSize;
- SkScan.scan = 0;
- SkipScanIvars.push_back(SkScan);
- WordsToSkip = 0;
- WordsToScan = IvarsInfo[i].ivar_size;
- }
- }
- if (WordsToScan > 0) {
- SKIP_SCAN SkScan;
- SkScan.skip = WordsToSkip;
- SkScan.scan = WordsToScan;
- SkipScanIvars.push_back(SkScan);
- }
-
- if (!SkipIvars.empty()) {
- unsigned int LastIndex = SkipIvars.size()-1;
- int LastByteSkipped =
- SkipIvars[LastIndex].ivar_bytepos + SkipIvars[LastIndex].ivar_size;
- LastIndex = IvarsInfo.size()-1;
- int LastByteScanned =
- IvarsInfo[LastIndex].ivar_bytepos +
- IvarsInfo[LastIndex].ivar_size * WordSize;
- // Compute number of bytes to skip at the tail end of the last ivar scanned.
- if (LastByteSkipped > LastByteScanned) {
- unsigned int TotalWords = (LastByteSkipped + (WordSize -1)) / WordSize;
- SKIP_SCAN SkScan;
- SkScan.skip = TotalWords - (LastByteScanned/WordSize);
- SkScan.scan = 0;
- SkipScanIvars.push_back(SkScan);
- }
- }
- // Mini optimization of nibbles such that an 0xM0 followed by 0x0N is produced
- // as 0xMN.
- int SkipScan = SkipScanIvars.size()-1;
- for (int i = 0; i <= SkipScan; i++) {
- if ((i < SkipScan) && SkipScanIvars[i].skip && SkipScanIvars[i].scan == 0
- && SkipScanIvars[i+1].skip == 0 && SkipScanIvars[i+1].scan) {
- // 0xM0 followed by 0x0N detected.
- SkipScanIvars[i].scan = SkipScanIvars[i+1].scan;
- for (int j = i+1; j < SkipScan; j++)
- SkipScanIvars[j] = SkipScanIvars[j+1];
- --SkipScan;
- }
- }
-
- // Generate the string.
- for (int i = 0; i <= SkipScan; i++) {
- unsigned char byte;
- unsigned int skip_small = SkipScanIvars[i].skip % 0xf;
- unsigned int scan_small = SkipScanIvars[i].scan % 0xf;
- unsigned int skip_big = SkipScanIvars[i].skip / 0xf;
- unsigned int scan_big = SkipScanIvars[i].scan / 0xf;
-
- // first skip big.
- for (unsigned int ix = 0; ix < skip_big; ix++)
- BitMap += (unsigned char)(0xf0);
-
- // next (skip small, scan)
- if (skip_small) {
- byte = skip_small << 4;
- if (scan_big > 0) {
- byte |= 0xf;
- --scan_big;
- } else if (scan_small) {
- byte |= scan_small;
- scan_small = 0;
- }
- BitMap += byte;
- }
- // next scan big
- for (unsigned int ix = 0; ix < scan_big; ix++)
- BitMap += (unsigned char)(0x0f);
- // last scan small
- if (scan_small) {
- byte = scan_small;
- BitMap += byte;
- }
- }
- // null terminate string.
- unsigned char zero = 0;
- BitMap += zero;
+/// buildBitmap - This routine does the horsework of taking the offsets of
+/// strong/weak references and creating a bitmap. The bitmap is also
+/// returned in the given buffer, suitable for being passed to \c dump().
+llvm::Constant *IvarLayoutBuilder::buildBitmap(CGObjCCommonMac &CGObjC,
+ llvm::SmallVectorImpl<unsigned char> &buffer) {
+ // The bitmap is a series of skip/scan instructions, aligned to word
+ // boundaries. The skip is performed first.
+ const unsigned char MaxNibble = 0xF;
+ const unsigned char SkipMask = 0xF0, SkipShift = 4;
+ const unsigned char ScanMask = 0x0F, ScanShift = 0;
- llvm::GlobalVariable *Entry = CreateMetadataVar(
+ assert(!IvarsInfo.empty() && "generating bitmap for no data");
+
+ // Sort the ivar info on byte position in case we encounterred a
+ // union nested in the ivar list.
+ if (IsDisordered) {
+ // This isn't a stable sort, but our algorithm should handle it fine.
+ llvm::array_pod_sort(IvarsInfo.begin(), IvarsInfo.end());
+ } else {
+#ifndef NDEBUG
+ for (unsigned i = 1; i != IvarsInfo.size(); ++i) {
+ assert(IvarsInfo[i - 1].Offset <= IvarsInfo[i].Offset);
+ }
+#endif
+ }
+ assert(IvarsInfo.back().Offset < InstanceEnd);
+
+ assert(buffer.empty());
+
+ // Skip the next N words.
+ auto skip = [&](unsigned numWords) {
+ assert(numWords > 0);
+
+ // Try to merge into the previous byte. Since scans happen second, we
+ // can't do this if it includes a scan.
+ if (!buffer.empty() && !(buffer.back() & ScanMask)) {
+ unsigned lastSkip = buffer.back() >> SkipShift;
+ if (lastSkip < MaxNibble) {
+ unsigned claimed = std::min(MaxNibble - lastSkip, numWords);
+ numWords -= claimed;
+ lastSkip += claimed;
+ buffer.back() = (lastSkip << SkipShift);
+ }
+ }
+
+ while (numWords >= MaxNibble) {
+ buffer.push_back(MaxNibble << SkipShift);
+ numWords -= MaxNibble;
+ }
+ if (numWords) {
+ buffer.push_back(numWords << SkipShift);
+ }
+ };
+
+ // Scan the next N words.
+ auto scan = [&](unsigned numWords) {
+ assert(numWords > 0);
+
+ // Try to merge into the previous byte. Since scans happen second, we can
+ // do this even if it includes a skip.
+ if (!buffer.empty()) {
+ unsigned lastScan = (buffer.back() & ScanMask) >> ScanShift;
+ if (lastScan < MaxNibble) {
+ unsigned claimed = std::min(MaxNibble - lastScan, numWords);
+ numWords -= claimed;
+ lastScan += claimed;
+ buffer.back() = (buffer.back() & SkipMask) | (lastScan << ScanShift);
+ }
+ }
+
+ while (numWords >= MaxNibble) {
+ buffer.push_back(MaxNibble << ScanShift);
+ numWords -= MaxNibble;
+ }
+ if (numWords) {
+ buffer.push_back(numWords << ScanShift);
+ }
+ };
+
+ // One past the end of the last scan.
+ unsigned endOfLastScanInWords = 0;
+ const CharUnits WordSize = CGM.getPointerSize();
+
+ // Consider all the scan requests.
+ for (auto &request : IvarsInfo) {
+ CharUnits beginOfScan = request.Offset - InstanceBegin;
+
+ // Ignore scan requests that don't start at an even multiple of the
+ // word size. We can't encode them.
+ if ((beginOfScan % WordSize) != 0) continue;
+
+ // Ignore scan requests that start before the instance start.
+ // This assumes that scans never span that boundary. The boundary
+ // isn't the true start of the ivars, because in the fragile-ARC case
+ // it's rounded up to word alignment, but the test above should leave
+ // us ignoring that possibility.
+ if (beginOfScan.isNegative()) {
+ assert(request.Offset + request.SizeInWords * WordSize <= InstanceBegin);
+ continue;
+ }
+
+ unsigned beginOfScanInWords = beginOfScan / WordSize;
+ unsigned endOfScanInWords = beginOfScanInWords + request.SizeInWords;
+
+ // If the scan starts some number of words after the last one ended,
+ // skip forward.
+ if (beginOfScanInWords > endOfLastScanInWords) {
+ skip(beginOfScanInWords - endOfLastScanInWords);
+
+ // Otherwise, start scanning where the last left off.
+ } else {
+ beginOfScanInWords = endOfLastScanInWords;
+
+ // If that leaves us with nothing to scan, ignore this request.
+ if (beginOfScanInWords >= endOfScanInWords) continue;
+ }
+
+ // Scan to the end of the request.
+ assert(beginOfScanInWords < endOfScanInWords);
+ scan(endOfScanInWords - beginOfScanInWords);
+ endOfLastScanInWords = endOfScanInWords;
+ }
+
+ if (buffer.empty())
+ return llvm::ConstantPointerNull::get(CGM.Int8PtrTy);
+
+ // For GC layouts, emit a skip to the end of the allocation so that we
+ // have precise information about the entire thing. This isn't useful
+ // or necessary for the ARC-style layout strings.
+ if (CGM.getLangOpts().getGC() != LangOptions::NonGC) {
+ unsigned lastOffsetInWords =
+ (InstanceEnd - InstanceBegin + WordSize - CharUnits::One()) / WordSize;
+ if (lastOffsetInWords > endOfLastScanInWords) {
+ skip(lastOffsetInWords - endOfLastScanInWords);
+ }
+ }
+
+ // Null terminate the string.
+ buffer.push_back(0);
+
+ bool isNonFragileABI = CGObjC.isNonFragileABI();
+
+ llvm::GlobalVariable *Entry = CGObjC.CreateMetadataVar(
"OBJC_CLASS_NAME_",
- llvm::ConstantDataArray::getString(VMContext, BitMap, false),
- ((ObjCABI == 2) ? "__TEXT,__objc_classname,cstring_literals"
- : "__TEXT,__cstring,cstring_literals"),
- 1, true);
- return getConstantGEP(VMContext, Entry, 0, 0);
+ llvm::ConstantDataArray::get(CGM.getLLVMContext(), buffer),
+ (isNonFragileABI ? "__TEXT,__objc_classname,cstring_literals"
+ : "__TEXT,__cstring,cstring_literals"),
+ CharUnits::One(), true);
+ return getConstantGEP(CGM.getLLVMContext(), Entry, 0, 0);
}
/// BuildIvarLayout - Builds ivar layout bitmap for the class
@@ -4755,62 +4918,75 @@
/// 2. When ForStrongLayout is false, following ivars are scanned:
/// - __weak anything
///
-llvm::Constant *CGObjCCommonMac::BuildIvarLayout(
- const ObjCImplementationDecl *OMD,
- bool ForStrongLayout) {
- bool hasUnion = false;
-
+llvm::Constant *
+CGObjCCommonMac::BuildIvarLayout(const ObjCImplementationDecl *OMD,
+ CharUnits beginOffset, CharUnits endOffset,
+ bool ForStrongLayout, bool HasMRCWeakIvars) {
+ // If this is MRC, and we're either building a strong layout or there
+ // are no weak ivars, bail out early.
llvm::Type *PtrTy = CGM.Int8PtrTy;
if (CGM.getLangOpts().getGC() == LangOptions::NonGC &&
- !CGM.getLangOpts().ObjCAutoRefCount)
+ !CGM.getLangOpts().ObjCAutoRefCount &&
+ (ForStrongLayout || !HasMRCWeakIvars))
return llvm::Constant::getNullValue(PtrTy);
const ObjCInterfaceDecl *OI = OMD->getClassInterface();
- SmallVector<const FieldDecl*, 32> RecFields;
- if (CGM.getLangOpts().ObjCAutoRefCount) {
+ SmallVector<const ObjCIvarDecl*, 32> ivars;
+
+ // GC layout strings include the complete object layout, possibly
+ // inaccurately in the non-fragile ABI; the runtime knows how to fix this
+ // up.
+ //
+ // ARC layout strings only include the class's ivars. In non-fragile
+ // runtimes, that means starting at InstanceStart, rounded up to word
+ // alignment. In fragile runtimes, there's no InstanceStart, so it means
+ // starting at the offset of the first ivar, rounded up to word alignment.
+ //
+ // MRC weak layout strings follow the ARC style.
+ CharUnits baseOffset;
+ if (CGM.getLangOpts().getGC() == LangOptions::NonGC) {
for (const ObjCIvarDecl *IVD = OI->all_declared_ivar_begin();
IVD; IVD = IVD->getNextIvar())
- RecFields.push_back(cast<FieldDecl>(IVD));
+ ivars.push_back(IVD);
+
+ if (isNonFragileABI()) {
+ baseOffset = beginOffset; // InstanceStart
+ } else if (!ivars.empty()) {
+ baseOffset =
+ CharUnits::fromQuantity(ComputeIvarBaseOffset(CGM, OMD, ivars[0]));
+ } else {
+ baseOffset = CharUnits::Zero();
+ }
+
+ baseOffset = baseOffset.RoundUpToAlignment(CGM.getPointerAlign());
}
else {
- SmallVector<const ObjCIvarDecl*, 32> Ivars;
- CGM.getContext().DeepCollectObjCIvars(OI, true, Ivars);
+ CGM.getContext().DeepCollectObjCIvars(OI, true, ivars);
- // FIXME: This is not ideal; we shouldn't have to do this copy.
- RecFields.append(Ivars.begin(), Ivars.end());
+ baseOffset = CharUnits::Zero();
}
- if (RecFields.empty())
+ if (ivars.empty())
return llvm::Constant::getNullValue(PtrTy);
- SkipIvars.clear();
- IvarsInfo.clear();
+ IvarLayoutBuilder builder(CGM, baseOffset, endOffset, ForStrongLayout);
- BuildAggrIvarLayout(OMD, nullptr, nullptr, RecFields, 0, ForStrongLayout,
- hasUnion);
- if (IvarsInfo.empty())
+ builder.visitAggregate(ivars.begin(), ivars.end(), CharUnits::Zero(),
+ [&](const ObjCIvarDecl *ivar) -> CharUnits {
+ return CharUnits::fromQuantity(ComputeIvarBaseOffset(CGM, OMD, ivar));
+ });
+
+ if (!builder.hasBitmapData())
return llvm::Constant::getNullValue(PtrTy);
- // Sort on byte position in case we encounterred a union nested in
- // the ivar list.
- if (hasUnion && !IvarsInfo.empty())
- std::sort(IvarsInfo.begin(), IvarsInfo.end());
- if (hasUnion && !SkipIvars.empty())
- std::sort(SkipIvars.begin(), SkipIvars.end());
+
+ llvm::SmallVector<unsigned char, 4> buffer;
+ llvm::Constant *C = builder.buildBitmap(*this, buffer);
- std::string BitMap;
- llvm::Constant *C = BuildIvarLayoutBitmap(BitMap);
-
- if (CGM.getLangOpts().ObjCGCBitmapPrint) {
+ if (CGM.getLangOpts().ObjCGCBitmapPrint && !buffer.empty()) {
printf("\n%s ivar layout for class '%s': ",
ForStrongLayout ? "strong" : "weak",
OMD->getClassInterface()->getName().str().c_str());
- const unsigned char *s = (const unsigned char*)BitMap.c_str();
- for (unsigned i = 0, e = BitMap.size(); i < e; i++)
- if (!(s[i] & 0xf0))
- printf("0x0%x%s", s[i], s[i] != 0 ? ", " : "");
- else
- printf("0x%x%s", s[i], s[i] != 0 ? ", " : "");
- printf("\n");
+ builder.dump(buffer);
}
return C;
}
@@ -4825,7 +5001,7 @@
llvm::ConstantDataArray::getString(VMContext, Sel.getAsString()),
((ObjCABI == 2) ? "__TEXT,__objc_methname,cstring_literals"
: "__TEXT,__cstring,cstring_literals"),
- 1, true);
+ CharUnits::One(), true);
return getConstantGEP(VMContext, Entry, 0, 0);
}
@@ -4847,7 +5023,7 @@
llvm::ConstantDataArray::getString(VMContext, TypeStr),
((ObjCABI == 2) ? "__TEXT,__objc_methtype,cstring_literals"
: "__TEXT,__cstring,cstring_literals"),
- 1, true);
+ CharUnits::One(), true);
return getConstantGEP(VMContext, Entry, 0, 0);
}
@@ -4866,7 +5042,7 @@
llvm::ConstantDataArray::getString(VMContext, TypeStr),
((ObjCABI == 2) ? "__TEXT,__objc_methtype,cstring_literals"
: "__TEXT,__cstring,cstring_literals"),
- 1, true);
+ CharUnits::One(), true);
return getConstantGEP(VMContext, Entry, 0, 0);
}
@@ -4879,7 +5055,7 @@
Entry = CreateMetadataVar(
"OBJC_PROP_NAME_ATTR_",
llvm::ConstantDataArray::getString(VMContext, Ident->getName()),
- "__TEXT,__cstring,cstring_literals", 1, true);
+ "__TEXT,__cstring,cstring_literals", CharUnits::One(), true);
return getConstantGEP(VMContext, Entry, 0, 0);
}
@@ -5583,8 +5759,14 @@
std::string ClassName = ID->getObjCRuntimeNameAsString();
llvm::Constant *Values[10]; // 11 for 64bit targets!
+ CharUnits beginInstance = CharUnits::fromQuantity(InstanceStart);
+ CharUnits endInstance = CharUnits::fromQuantity(InstanceSize);
+
+ bool hasMRCWeak = false;
if (CGM.getLangOpts().ObjCAutoRefCount)
flags |= NonFragileABI_Class_CompiledByARC;
+ else if ((hasMRCWeak = hasMRCWeakIvars(CGM, ID)))
+ flags |= NonFragileABI_Class_HasMRCWeakIvars;
Values[ 0] = llvm::ConstantInt::get(ObjCTypes.IntTy, flags);
Values[ 1] = llvm::ConstantInt::get(ObjCTypes.IntTy, InstanceStart);
@@ -5592,7 +5774,7 @@
// FIXME. For 64bit targets add 0 here.
Values[ 3] = (flags & NonFragileABI_Class_Meta)
? GetIvarLayoutName(nullptr, ObjCTypes)
- : BuildIvarLayout(ID, true);
+ : BuildStrongIvarLayout(ID, beginInstance, endInstance);
Values[ 4] = GetClassName(ID->getObjCRuntimeNameAsString());
// const struct _method_list_t * const baseMethods;
std::vector<llvm::Constant*> Methods;
@@ -5639,7 +5821,8 @@
Values[ 9] = llvm::Constant::getNullValue(ObjCTypes.PropertyListPtrTy);
} else {
Values[ 7] = EmitIvarList(ID);
- Values[ 8] = BuildIvarLayout(ID, false);
+ Values[ 8] = BuildWeakIvarLayout(ID, beginInstance, endInstance,
+ hasMRCWeak);
Values[ 9] = EmitPropertyList("\01l_OBJC_$_PROP_LIST_" + ID->getObjCRuntimeNameAsString(),
ID, ID->getClassInterface(), ObjCTypes);
}
@@ -5731,7 +5914,8 @@
// Make this entry NULL for any iOS device target, any iOS simulator target,
// OS X with deployment target 10.9 or later.
const llvm::Triple &Triple = CGM.getTarget().getTriple();
- if (Triple.isiOS() || (Triple.isMacOSX() && !Triple.isMacOSXVersionLT(10, 9)))
+ if (Triple.isiOS() || Triple.isWatchOS() ||
+ (Triple.isMacOSX() && !Triple.isMacOSXVersionLT(10, 9)))
// This entry will be null.
ObjCEmptyVtableVar = nullptr;
else
@@ -5887,9 +6071,11 @@
std::string ProtocolName("\01l_OBJC_PROTOCOL_REFERENCE_$_");
ProtocolName += PD->getObjCRuntimeNameAsString();
+ CharUnits Align = CGF.getPointerAlign();
+
llvm::GlobalVariable *PTGV = CGM.getModule().getGlobalVariable(ProtocolName);
if (PTGV)
- return CGF.Builder.CreateLoad(PTGV);
+ return CGF.Builder.CreateAlignedLoad(PTGV, Align);
PTGV = new llvm::GlobalVariable(
CGM.getModule(),
Init->getType(), false,
@@ -5898,8 +6084,9 @@
ProtocolName);
PTGV->setSection("__DATA, __objc_protorefs, coalesced, no_dead_strip");
PTGV->setVisibility(llvm::GlobalValue::HiddenVisibility);
+ PTGV->setAlignment(Align.getQuantity());
CGM.addCompilerUsedGlobal(PTGV);
- return CGF.Builder.CreateLoad(PTGV);
+ return CGF.Builder.CreateAlignedLoad(PTGV, Align);
}
/// GenerateCategory - Build metadata for a category implementation.
@@ -6428,7 +6615,8 @@
const ObjCInterfaceDecl *Interface,
const ObjCIvarDecl *Ivar) {
llvm::Value *IvarOffsetValue = ObjCIvarOffsetVariable(Interface, Ivar);
- IvarOffsetValue = CGF.Builder.CreateLoad(IvarOffsetValue, "ivar");
+ IvarOffsetValue = CGF.Builder.CreateAlignedLoad(IvarOffsetValue,
+ CGF.getSizeAlign(), "ivar");
if (IsIvarOffsetKnownIdempotent(CGF, Ivar))
cast<llvm::LoadInst>(IvarOffsetValue)
->setMetadata(CGM.getModule().getMDKindID("invariant.load"),
@@ -6559,16 +6747,17 @@
}
}
- llvm::Value *mref =
- CGF.Builder.CreateBitCast(messageRef, ObjCTypes.MessageRefPtrTy);
+ Address mref =
+ Address(CGF.Builder.CreateBitCast(messageRef, ObjCTypes.MessageRefPtrTy),
+ CGF.getPointerAlign());
// Update the message ref argument.
- args[1].RV = RValue::get(mref);
+ args[1].RV = RValue::get(mref.getPointer());
// Load the function to call from the message ref table.
- llvm::Value *callee =
- CGF.Builder.CreateStructGEP(ObjCTypes.MessageRefTy, mref, 0);
- callee = CGF.Builder.CreateLoad(callee, "msgSend_fn");
+ Address calleeAddr =
+ CGF.Builder.CreateStructGEP(mref, 0, CharUnits::Zero());
+ llvm::Value *callee = CGF.Builder.CreateLoad(calleeAddr, "msgSend_fn");
callee = CGF.Builder.CreateBitCast(callee, MSI.MessengerType);
@@ -6594,7 +6783,7 @@
: EmitMessageSend(CGF, Return, ResultType,
EmitSelector(CGF, Sel),
Receiver, CGF.getContext().getObjCIdType(),
- false, CallArgs, Method, ObjCTypes);
+ false, CallArgs, Method, Class, ObjCTypes);
}
llvm::GlobalVariable *
@@ -6617,6 +6806,7 @@
IdentifierInfo *II,
bool Weak,
const ObjCInterfaceDecl *ID) {
+ CharUnits Align = CGF.getPointerAlign();
llvm::GlobalVariable *&Entry = ClassReferences[II];
if (!Entry) {
@@ -6627,13 +6817,11 @@
Entry = new llvm::GlobalVariable(CGM.getModule(), ObjCTypes.ClassnfABIPtrTy,
false, llvm::GlobalValue::PrivateLinkage,
ClassGV, "OBJC_CLASSLIST_REFERENCES_$_");
- Entry->setAlignment(
- CGM.getDataLayout().getABITypeAlignment(
- ObjCTypes.ClassnfABIPtrTy));
+ Entry->setAlignment(Align.getQuantity());
Entry->setSection("__DATA, __objc_classrefs, regular, no_dead_strip");
CGM.addCompilerUsedGlobal(Entry);
}
- return CGF.Builder.CreateLoad(Entry);
+ return CGF.Builder.CreateAlignedLoad(Entry, Align);
}
llvm::Value *CGObjCNonFragileABIMac::EmitClassRef(CodeGenFunction &CGF,
@@ -6644,12 +6832,13 @@
llvm::Value *CGObjCNonFragileABIMac::EmitNSAutoreleasePoolClassRef(
CodeGenFunction &CGF) {
IdentifierInfo *II = &CGM.getContext().Idents.get("NSAutoreleasePool");
- return EmitClassRefFromId(CGF, II, false, 0);
+ return EmitClassRefFromId(CGF, II, false, nullptr);
}
llvm::Value *
CGObjCNonFragileABIMac::EmitSuperClassRef(CodeGenFunction &CGF,
const ObjCInterfaceDecl *ID) {
+ CharUnits Align = CGF.getPointerAlign();
llvm::GlobalVariable *&Entry = SuperClassReferences[ID->getIdentifier()];
if (!Entry) {
@@ -6660,13 +6849,11 @@
Entry = new llvm::GlobalVariable(CGM.getModule(), ObjCTypes.ClassnfABIPtrTy,
false, llvm::GlobalValue::PrivateLinkage,
ClassGV, "OBJC_CLASSLIST_SUP_REFS_$_");
- Entry->setAlignment(
- CGM.getDataLayout().getABITypeAlignment(
- ObjCTypes.ClassnfABIPtrTy));
+ Entry->setAlignment(Align.getQuantity());
Entry->setSection("__DATA, __objc_superrefs, regular, no_dead_strip");
CGM.addCompilerUsedGlobal(Entry);
}
- return CGF.Builder.CreateLoad(Entry);
+ return CGF.Builder.CreateAlignedLoad(Entry, Align);
}
/// EmitMetaClassRef - Return a Value * of the address of _class_t
@@ -6675,6 +6862,7 @@
llvm::Value *CGObjCNonFragileABIMac::EmitMetaClassRef(CodeGenFunction &CGF,
const ObjCInterfaceDecl *ID,
bool Weak) {
+ CharUnits Align = CGF.getPointerAlign();
llvm::GlobalVariable * &Entry = MetaClassReferences[ID->getIdentifier()];
if (!Entry) {
llvm::SmallString<64> MetaClassName(getMetaclassSymbolPrefix());
@@ -6685,14 +6873,13 @@
Entry = new llvm::GlobalVariable(CGM.getModule(), ObjCTypes.ClassnfABIPtrTy,
false, llvm::GlobalValue::PrivateLinkage,
MetaClassGV, "OBJC_CLASSLIST_SUP_REFS_$_");
- Entry->setAlignment(
- CGM.getDataLayout().getABITypeAlignment(ObjCTypes.ClassnfABIPtrTy));
+ Entry->setAlignment(Align.getQuantity());
Entry->setSection("__DATA, __objc_superrefs, regular, no_dead_strip");
CGM.addCompilerUsedGlobal(Entry);
}
- return CGF.Builder.CreateLoad(Entry);
+ return CGF.Builder.CreateAlignedLoad(Entry, Align);
}
/// GetClass - Return a reference to the class for the given interface
@@ -6727,14 +6914,15 @@
// ...
// Create and init a super structure; this is a (receiver, class)
// pair we will pass to objc_msgSendSuper.
- llvm::Value *ObjCSuper =
- CGF.CreateTempAlloca(ObjCTypes.SuperTy, "objc_super");
+ Address ObjCSuper =
+ CGF.CreateTempAlloca(ObjCTypes.SuperTy, CGF.getPointerAlign(),
+ "objc_super");
llvm::Value *ReceiverAsObject =
CGF.Builder.CreateBitCast(Receiver, ObjCTypes.ObjectPtrTy);
CGF.Builder.CreateStore(
ReceiverAsObject,
- CGF.Builder.CreateStructGEP(ObjCTypes.SuperTy, ObjCSuper, 0));
+ CGF.Builder.CreateStructGEP(ObjCSuper, 0, CharUnits::Zero()));
// If this is a class message the metaclass is passed as the target.
llvm::Value *Target;
@@ -6749,22 +6937,33 @@
CGM.getTypes().ConvertType(CGF.getContext().getObjCClassType());
Target = CGF.Builder.CreateBitCast(Target, ClassTy);
CGF.Builder.CreateStore(
- Target, CGF.Builder.CreateStructGEP(ObjCTypes.SuperTy, ObjCSuper, 1));
+ Target, CGF.Builder.CreateStructGEP(ObjCSuper, 1, CGF.getPointerSize()));
return (isVTableDispatchedSelector(Sel))
? EmitVTableMessageSend(CGF, Return, ResultType, Sel,
- ObjCSuper, ObjCTypes.SuperPtrCTy,
+ ObjCSuper.getPointer(), ObjCTypes.SuperPtrCTy,
true, CallArgs, Method)
: EmitMessageSend(CGF, Return, ResultType,
EmitSelector(CGF, Sel),
- ObjCSuper, ObjCTypes.SuperPtrCTy,
- true, CallArgs, Method, ObjCTypes);
+ ObjCSuper.getPointer(), ObjCTypes.SuperPtrCTy,
+ true, CallArgs, Method, Class, ObjCTypes);
}
llvm::Value *CGObjCNonFragileABIMac::EmitSelector(CodeGenFunction &CGF,
- Selector Sel, bool lval) {
+ Selector Sel) {
+ Address Addr = EmitSelectorAddr(CGF, Sel);
+
+ llvm::LoadInst* LI = CGF.Builder.CreateLoad(Addr);
+ LI->setMetadata(CGM.getModule().getMDKindID("invariant.load"),
+ llvm::MDNode::get(VMContext, None));
+ return LI;
+}
+
+Address CGObjCNonFragileABIMac::EmitSelectorAddr(CodeGenFunction &CGF,
+ Selector Sel) {
llvm::GlobalVariable *&Entry = SelectorReferences[Sel];
+ CharUnits Align = CGF.getPointerAlign();
if (!Entry) {
llvm::Constant *Casted =
llvm::ConstantExpr::getBitCast(GetMethodVarName(Sel),
@@ -6774,23 +6973,19 @@
Casted, "OBJC_SELECTOR_REFERENCES_");
Entry->setExternallyInitialized(true);
Entry->setSection("__DATA, __objc_selrefs, literal_pointers, no_dead_strip");
+ Entry->setAlignment(Align.getQuantity());
CGM.addCompilerUsedGlobal(Entry);
}
- if (lval)
- return Entry;
- llvm::LoadInst* LI = CGF.Builder.CreateLoad(Entry);
-
- LI->setMetadata(CGM.getModule().getMDKindID("invariant.load"),
- llvm::MDNode::get(VMContext, None));
- return LI;
+ return Address(Entry, Align);
}
+
/// EmitObjCIvarAssign - Code gen for assigning to a __strong object.
/// objc_assign_ivar (id src, id *dst, ptrdiff_t)
///
void CGObjCNonFragileABIMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF,
llvm::Value *src,
- llvm::Value *dst,
+ Address dst,
llvm::Value *ivarOffset) {
llvm::Type * SrcTy = src->getType();
if (!isa<llvm::PointerType>(SrcTy)) {
@@ -6802,7 +6997,7 @@
}
src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
dst = CGF.Builder.CreateBitCast(dst, ObjCTypes.PtrObjectPtrTy);
- llvm::Value *args[] = { src, dst, ivarOffset };
+ llvm::Value *args[] = { src, dst.getPointer(), ivarOffset };
CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignIvarFn(), args);
}
@@ -6811,7 +7006,7 @@
///
void CGObjCNonFragileABIMac::EmitObjCStrongCastAssign(
CodeGen::CodeGenFunction &CGF,
- llvm::Value *src, llvm::Value *dst) {
+ llvm::Value *src, Address dst) {
llvm::Type * SrcTy = src->getType();
if (!isa<llvm::PointerType>(SrcTy)) {
unsigned Size = CGM.getDataLayout().getTypeAllocSize(SrcTy);
@@ -6822,19 +7017,19 @@
}
src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
dst = CGF.Builder.CreateBitCast(dst, ObjCTypes.PtrObjectPtrTy);
- llvm::Value *args[] = { src, dst };
+ llvm::Value *args[] = { src, dst.getPointer() };
CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignStrongCastFn(),
args, "weakassign");
}
void CGObjCNonFragileABIMac::EmitGCMemmoveCollectable(
CodeGen::CodeGenFunction &CGF,
- llvm::Value *DestPtr,
- llvm::Value *SrcPtr,
+ Address DestPtr,
+ Address SrcPtr,
llvm::Value *Size) {
SrcPtr = CGF.Builder.CreateBitCast(SrcPtr, ObjCTypes.Int8PtrTy);
DestPtr = CGF.Builder.CreateBitCast(DestPtr, ObjCTypes.Int8PtrTy);
- llvm::Value *args[] = { DestPtr, SrcPtr, Size };
+ llvm::Value *args[] = { DestPtr.getPointer(), SrcPtr.getPointer(), Size };
CGF.EmitNounwindRuntimeCall(ObjCTypes.GcMemmoveCollectableFn(), args);
}
@@ -6843,13 +7038,12 @@
///
llvm::Value * CGObjCNonFragileABIMac::EmitObjCWeakRead(
CodeGen::CodeGenFunction &CGF,
- llvm::Value *AddrWeakObj) {
- llvm::Type* DestTy =
- cast<llvm::PointerType>(AddrWeakObj->getType())->getElementType();
+ Address AddrWeakObj) {
+ llvm::Type *DestTy = AddrWeakObj.getElementType();
AddrWeakObj = CGF.Builder.CreateBitCast(AddrWeakObj, ObjCTypes.PtrObjectPtrTy);
llvm::Value *read_weak =
CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcReadWeakFn(),
- AddrWeakObj, "weakread");
+ AddrWeakObj.getPointer(), "weakread");
read_weak = CGF.Builder.CreateBitCast(read_weak, DestTy);
return read_weak;
}
@@ -6858,7 +7052,7 @@
/// objc_assign_weak (id src, id *dst)
///
void CGObjCNonFragileABIMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF,
- llvm::Value *src, llvm::Value *dst) {
+ llvm::Value *src, Address dst) {
llvm::Type * SrcTy = src->getType();
if (!isa<llvm::PointerType>(SrcTy)) {
unsigned Size = CGM.getDataLayout().getTypeAllocSize(SrcTy);
@@ -6869,7 +7063,7 @@
}
src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
dst = CGF.Builder.CreateBitCast(dst, ObjCTypes.PtrObjectPtrTy);
- llvm::Value *args[] = { src, dst };
+ llvm::Value *args[] = { src, dst.getPointer() };
CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignWeakFn(),
args, "weakassign");
}
@@ -6878,7 +7072,7 @@
/// objc_assign_global (id src, id *dst)
///
void CGObjCNonFragileABIMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF,
- llvm::Value *src, llvm::Value *dst,
+ llvm::Value *src, Address dst,
bool threadlocal) {
llvm::Type * SrcTy = src->getType();
if (!isa<llvm::PointerType>(SrcTy)) {
@@ -6890,7 +7084,7 @@
}
src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
dst = CGF.Builder.CreateBitCast(dst, ObjCTypes.PtrObjectPtrTy);
- llvm::Value *args[] = { src, dst };
+ llvm::Value *args[] = { src, dst.getPointer() };
if (!threadlocal)
CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignGlobalFn(),
args, "globalassign");
@@ -7043,6 +7237,7 @@
case ObjCRuntime::MacOSX:
case ObjCRuntime::iOS:
+ case ObjCRuntime::WatchOS:
return new CGObjCNonFragileABIMac(CGM);
case ObjCRuntime::GNUstep:
diff --git a/lib/CodeGen/CGObjCRuntime.cpp b/lib/CodeGen/CGObjCRuntime.cpp
index 5290a87..7be9ae9 100644
--- a/lib/CodeGen/CGObjCRuntime.cpp
+++ b/lib/CodeGen/CGObjCRuntime.cpp
@@ -134,14 +134,15 @@
CGBitFieldInfo *Info = new (CGF.CGM.getContext()) CGBitFieldInfo(
CGBitFieldInfo::MakeInfo(CGF.CGM.getTypes(), Ivar, BitOffset, BitFieldSize,
CGF.CGM.getContext().toBits(StorageSize),
- Alignment.getQuantity()));
+ CharUnits::fromQuantity(0)));
- V = CGF.Builder.CreateBitCast(V,
- llvm::Type::getIntNPtrTy(CGF.getLLVMContext(),
+ Address Addr(V, Alignment);
+ Addr = CGF.Builder.CreateElementBitCast(Addr,
+ llvm::Type::getIntNTy(CGF.getLLVMContext(),
Info->StorageSize));
- return LValue::MakeBitfield(V, *Info,
+ return LValue::MakeBitfield(Addr, *Info,
IvarTy.withCVRQualifiers(CVRQualifiers),
- Alignment);
+ AlignmentSource::Decl);
}
namespace {
@@ -152,7 +153,7 @@
llvm::Constant *TypeInfo;
};
- struct CallObjCEndCatch : EHScopeStack::Cleanup {
+ struct CallObjCEndCatch final : EHScopeStack::Cleanup {
CallObjCEndCatch(bool MightThrow, llvm::Value *Fn) :
MightThrow(MightThrow), Fn(Fn) {}
bool MightThrow;
@@ -160,7 +161,7 @@
void Emit(CodeGenFunction &CGF, Flags flags) override {
if (!MightThrow) {
- CGF.Builder.CreateCall(Fn, {})->setDoesNotThrow();
+ CGF.Builder.CreateCall(Fn)->setDoesNotThrow();
return;
}
@@ -255,24 +256,7 @@
llvm::Value *CastExn = CGF.Builder.CreateBitCast(Exn, CatchType);
CGF.EmitAutoVarDecl(*CatchParam);
-
- llvm::Value *CatchParamAddr = CGF.GetAddrOfLocalVar(CatchParam);
-
- switch (CatchParam->getType().getQualifiers().getObjCLifetime()) {
- case Qualifiers::OCL_Strong:
- CastExn = CGF.EmitARCRetainNonBlock(CastExn);
- // fallthrough
-
- case Qualifiers::OCL_None:
- case Qualifiers::OCL_ExplicitNone:
- case Qualifiers::OCL_Autoreleasing:
- CGF.Builder.CreateStore(CastExn, CatchParamAddr);
- break;
-
- case Qualifiers::OCL_Weak:
- CGF.EmitARCInitWeak(CatchParamAddr, CastExn);
- break;
- }
+ EmitInitOfCatchParam(CGF, CastExn, CatchParam);
}
CGF.ObjCEHValueStack.push_back(Exn);
@@ -296,8 +280,32 @@
CGF.EmitBlock(Cont.getBlock());
}
+void CGObjCRuntime::EmitInitOfCatchParam(CodeGenFunction &CGF,
+ llvm::Value *exn,
+ const VarDecl *paramDecl) {
+
+ Address paramAddr = CGF.GetAddrOfLocalVar(paramDecl);
+
+ switch (paramDecl->getType().getQualifiers().getObjCLifetime()) {
+ case Qualifiers::OCL_Strong:
+ exn = CGF.EmitARCRetainNonBlock(exn);
+ // fallthrough
+
+ case Qualifiers::OCL_None:
+ case Qualifiers::OCL_ExplicitNone:
+ case Qualifiers::OCL_Autoreleasing:
+ CGF.Builder.CreateStore(exn, paramAddr);
+ return;
+
+ case Qualifiers::OCL_Weak:
+ CGF.EmitARCInitWeak(paramAddr, exn);
+ return;
+ }
+ llvm_unreachable("invalid ownership qualifier");
+}
+
namespace {
- struct CallSyncExit : EHScopeStack::Cleanup {
+ struct CallSyncExit final : EHScopeStack::Cleanup {
llvm::Value *SyncExitFn;
llvm::Value *SyncArg;
CallSyncExit(llvm::Value *SyncExitFn, llvm::Value *SyncArg)
diff --git a/lib/CodeGen/CGObjCRuntime.h b/lib/CodeGen/CGObjCRuntime.h
index 4752546..28d88dd 100644
--- a/lib/CodeGen/CGObjCRuntime.h
+++ b/lib/CodeGen/CGObjCRuntime.h
@@ -100,6 +100,10 @@
llvm::Constant *beginCatchFn,
llvm::Constant *endCatchFn,
llvm::Constant *exceptionRethrowFn);
+
+ void EmitInitOfCatchParam(CodeGenFunction &CGF, llvm::Value *exn,
+ const VarDecl *paramDecl);
+
/// Emits an \@synchronize() statement, using the \p syncEnterFn and
/// \p syncExitFn arguments as the functions called to lock and unlock
/// the object. This function can be called by subclasses that use
@@ -116,11 +120,16 @@
/// this compilation unit with the runtime library.
virtual llvm::Function *ModuleInitFunction() = 0;
- /// Get a selector for the specified name and type values. The
- /// return value should have the LLVM type for pointer-to
+ /// Get a selector for the specified name and type values.
+ /// The result should have the LLVM type for ASTContext::getObjCSelType().
+ virtual llvm::Value *GetSelector(CodeGenFunction &CGF, Selector Sel) = 0;
+
+ /// Get the address of a selector for the specified name and type values.
+ /// This is a rarely-used language extension, but sadly it exists.
+ ///
+ /// The result should have the LLVM type for a pointer to
/// ASTContext::getObjCSelType().
- virtual llvm::Value *GetSelector(CodeGenFunction &CGF,
- Selector Sel, bool lval=false) = 0;
+ virtual Address GetAddrOfSelector(CodeGenFunction &CGF, Selector Sel) = 0;
/// Get a typed selector.
virtual llvm::Value *GetSelector(CodeGenFunction &CGF,
@@ -133,7 +142,7 @@
virtual llvm::Constant *GetEHType(QualType T) = 0;
/// Generate a constant string object.
- virtual llvm::Constant *GenerateConstantString(const StringLiteral *) = 0;
+ virtual ConstantAddress GenerateConstantString(const StringLiteral *) = 0;
/// Generate a category. A category contains a list of methods (and
/// accompanying metadata) and a list of protocols.
@@ -238,17 +247,17 @@
const ObjCAtThrowStmt &S,
bool ClearInsertionPoint=true) = 0;
virtual llvm::Value *EmitObjCWeakRead(CodeGen::CodeGenFunction &CGF,
- llvm::Value *AddrWeakObj) = 0;
+ Address AddrWeakObj) = 0;
virtual void EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF,
- llvm::Value *src, llvm::Value *dest) = 0;
+ llvm::Value *src, Address dest) = 0;
virtual void EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF,
- llvm::Value *src, llvm::Value *dest,
+ llvm::Value *src, Address dest,
bool threadlocal=false) = 0;
virtual void EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF,
- llvm::Value *src, llvm::Value *dest,
+ llvm::Value *src, Address dest,
llvm::Value *ivarOffset) = 0;
virtual void EmitObjCStrongCastAssign(CodeGen::CodeGenFunction &CGF,
- llvm::Value *src, llvm::Value *dest) = 0;
+ llvm::Value *src, Address dest) = 0;
virtual LValue EmitObjCValueForIvar(CodeGen::CodeGenFunction &CGF,
QualType ObjectTy,
@@ -259,15 +268,18 @@
const ObjCInterfaceDecl *Interface,
const ObjCIvarDecl *Ivar) = 0;
virtual void EmitGCMemmoveCollectable(CodeGen::CodeGenFunction &CGF,
- llvm::Value *DestPtr,
- llvm::Value *SrcPtr,
+ Address DestPtr,
+ Address SrcPtr,
llvm::Value *Size) = 0;
virtual llvm::Constant *BuildGCBlockLayout(CodeGen::CodeGenModule &CGM,
const CodeGen::CGBlockInfo &blockInfo) = 0;
virtual llvm::Constant *BuildRCBlockLayout(CodeGen::CodeGenModule &CGM,
const CodeGen::CGBlockInfo &blockInfo) = 0;
+
+ /// Returns an i8* which points to the byref layout information.
virtual llvm::Constant *BuildByrefLayout(CodeGen::CodeGenModule &CGM,
QualType T) = 0;
+
virtual llvm::GlobalVariable *GetClassGlobal(const std::string &Name,
bool Weak = false) = 0;
diff --git a/lib/CodeGen/CGOpenCLRuntime.cpp b/lib/CodeGen/CGOpenCLRuntime.cpp
index 079ef72..8af39ce 100644
--- a/lib/CodeGen/CGOpenCLRuntime.cpp
+++ b/lib/CodeGen/CGOpenCLRuntime.cpp
@@ -55,6 +55,28 @@
case BuiltinType::OCLImage2dArray:
return llvm::PointerType::get(llvm::StructType::create(
Ctx, "opencl.image2d_array_t"), ImgAddrSpc);
+ case BuiltinType::OCLImage2dDepth:
+ return llvm::PointerType::get(
+ llvm::StructType::create(Ctx, "opencl.image2d_depth_t"), ImgAddrSpc);
+ case BuiltinType::OCLImage2dArrayDepth:
+ return llvm::PointerType::get(
+ llvm::StructType::create(Ctx, "opencl.image2d_array_depth_t"),
+ ImgAddrSpc);
+ case BuiltinType::OCLImage2dMSAA:
+ return llvm::PointerType::get(
+ llvm::StructType::create(Ctx, "opencl.image2d_msaa_t"), ImgAddrSpc);
+ case BuiltinType::OCLImage2dArrayMSAA:
+ return llvm::PointerType::get(
+ llvm::StructType::create(Ctx, "opencl.image2d_array_msaa_t"),
+ ImgAddrSpc);
+ case BuiltinType::OCLImage2dMSAADepth:
+ return llvm::PointerType::get(
+ llvm::StructType::create(Ctx, "opencl.image2d_msaa_depth_t"),
+ ImgAddrSpc);
+ case BuiltinType::OCLImage2dArrayMSAADepth:
+ return llvm::PointerType::get(
+ llvm::StructType::create(Ctx, "opencl.image2d_array_msaa_depth_t"),
+ ImgAddrSpc);
case BuiltinType::OCLImage3d:
return llvm::PointerType::get(llvm::StructType::create(
Ctx, "opencl.image3d_t"), ImgAddrSpc);
@@ -63,5 +85,17 @@
case BuiltinType::OCLEvent:
return llvm::PointerType::get(llvm::StructType::create(
Ctx, "opencl.event_t"), 0);
+ case BuiltinType::OCLClkEvent:
+ return llvm::PointerType::get(
+ llvm::StructType::create(Ctx, "opencl.clk_event_t"), 0);
+ case BuiltinType::OCLQueue:
+ return llvm::PointerType::get(
+ llvm::StructType::create(Ctx, "opencl.queue_t"), 0);
+ case BuiltinType::OCLNDRange:
+ return llvm::PointerType::get(
+ llvm::StructType::create(Ctx, "opencl.ndrange_t"), 0);
+ case BuiltinType::OCLReserveID:
+ return llvm::PointerType::get(
+ llvm::StructType::create(Ctx, "opencl.reserve_id_t"), 0);
}
}
diff --git a/lib/CodeGen/CGOpenMPRuntime.cpp b/lib/CodeGen/CGOpenMPRuntime.cpp
index 1238acc..f9e5adc 100644
--- a/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -41,25 +41,29 @@
/// \brief Region for constructs that do not require function outlining,
/// like 'for', 'sections', 'atomic' etc. directives.
InlinedRegion,
+ /// \brief Region with outlined function for standalone 'target' directive.
+ TargetRegion,
};
CGOpenMPRegionInfo(const CapturedStmt &CS,
const CGOpenMPRegionKind RegionKind,
- const RegionCodeGenTy &CodeGen)
+ const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
+ bool HasCancel)
: CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
- CodeGen(CodeGen) {}
+ CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
- const RegionCodeGenTy &CodeGen)
- : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind),
- CodeGen(CodeGen) {}
+ const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
+ bool HasCancel)
+ : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
+ Kind(Kind), HasCancel(HasCancel) {}
/// \brief Get a variable or parameter for storing global thread id
/// inside OpenMP construct.
virtual const VarDecl *getThreadIDVariable() const = 0;
/// \brief Emit the captured statement body.
- virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
+ void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
/// \brief Get an LValue for the current ThreadID variable.
/// \return LValue for thread id variable. This LValue always has type int32*.
@@ -67,6 +71,10 @@
CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
+ OpenMPDirectiveKind getDirectiveKind() const { return Kind; }
+
+ bool hasCancel() const { return HasCancel; }
+
static bool classof(const CGCapturedStmtInfo *Info) {
return Info->getKind() == CR_OpenMP;
}
@@ -74,14 +82,18 @@
protected:
CGOpenMPRegionKind RegionKind;
const RegionCodeGenTy &CodeGen;
+ OpenMPDirectiveKind Kind;
+ bool HasCancel;
};
/// \brief API for captured statement code generation in OpenMP constructs.
class CGOpenMPOutlinedRegionInfo : public CGOpenMPRegionInfo {
public:
CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
- const RegionCodeGenTy &CodeGen)
- : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen),
+ const RegionCodeGenTy &CodeGen,
+ OpenMPDirectiveKind Kind, bool HasCancel)
+ : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
+ HasCancel),
ThreadIDVar(ThreadIDVar) {
assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
}
@@ -109,8 +121,9 @@
public:
CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
const VarDecl *ThreadIDVar,
- const RegionCodeGenTy &CodeGen)
- : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen),
+ const RegionCodeGenTy &CodeGen,
+ OpenMPDirectiveKind Kind, bool HasCancel)
+ : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
ThreadIDVar(ThreadIDVar) {
assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
}
@@ -141,8 +154,10 @@
class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
public:
CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
- const RegionCodeGenTy &CodeGen)
- : CGOpenMPRegionInfo(InlinedRegion, CodeGen), OldCSI(OldCSI),
+ const RegionCodeGenTy &CodeGen,
+ OpenMPDirectiveKind Kind, bool HasCancel)
+ : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
+ OldCSI(OldCSI),
OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
// \brief Retrieve the value of the context parameter.
llvm::Value *getContextValue() const override {
@@ -150,7 +165,7 @@
return OuterRegionInfo->getContextValue();
llvm_unreachable("No context value for inlined OpenMP region");
}
- virtual void setContextValue(llvm::Value *V) override {
+ void setContextValue(llvm::Value *V) override {
if (OuterRegionInfo) {
OuterRegionInfo->setContextValue(V);
return;
@@ -198,6 +213,29 @@
CGOpenMPRegionInfo *OuterRegionInfo;
};
+/// \brief API for captured statement code generation in OpenMP target
+/// constructs. For this captures, implicit parameters are used instead of the
+/// captured fields.
+class CGOpenMPTargetRegionInfo : public CGOpenMPRegionInfo {
+public:
+ CGOpenMPTargetRegionInfo(const CapturedStmt &CS,
+ const RegionCodeGenTy &CodeGen)
+ : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
+ /*HasCancel = */ false) {}
+
+ /// \brief This is unused for target regions because each starts executing
+ /// with a single thread.
+ const VarDecl *getThreadIDVariable() const override { return nullptr; }
+
+ /// \brief Get the name of the capture helper.
+ StringRef getHelperName() const override { return ".omp_offloading."; }
+
+ static bool classof(const CGCapturedStmtInfo *Info) {
+ return CGOpenMPRegionInfo::classof(Info) &&
+ cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
+ }
+};
+
/// \brief RAII for emitting code of OpenMP constructs.
class InlinedOpenMPRegionRAII {
CodeGenFunction &CGF;
@@ -207,11 +245,12 @@
/// \param CodeGen Code generation sequence for combined directives. Includes
/// a list of functions used for code generation of implicitly inlined
/// regions.
- InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen)
+ InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
+ OpenMPDirectiveKind Kind, bool HasCancel)
: CGF(CGF) {
// Start emission for the construct.
- CGF.CapturedStmtInfo =
- new CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, CodeGen);
+ CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
+ CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
}
~InlinedOpenMPRegionRAII() {
// Restore original CapturedStmtInfo only if we're done with code emission.
@@ -222,20 +261,25 @@
}
};
-} // namespace
+} // anonymous namespace
+
+static LValue emitLoadOfPointerLValue(CodeGenFunction &CGF, Address PtrAddr,
+ QualType Ty) {
+ AlignmentSource Source;
+ CharUnits Align = CGF.getNaturalPointeeTypeAlignment(Ty, &Source);
+ return CGF.MakeAddrLValue(Address(CGF.Builder.CreateLoad(PtrAddr), Align),
+ Ty->getPointeeType(), Source);
+}
LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
- return CGF.MakeNaturalAlignAddrLValue(
- CGF.Builder.CreateAlignedLoad(
- CGF.GetAddrOfLocalVar(getThreadIDVariable()),
- CGF.PointerAlignInBytes),
- getThreadIDVariable()
- ->getType()
- ->castAs<PointerType>()
- ->getPointeeType());
+ return emitLoadOfPointerLValue(CGF,
+ CGF.GetAddrOfLocalVar(getThreadIDVariable()),
+ getThreadIDVariable()->getType());
}
void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) {
+ if (!CGF.HaveInsertPoint())
+ return;
// 1.2.2 OpenMP Language Terminology
// Structured block - An executable statement with a single entry at the
// top and a single exit at the bottom.
@@ -251,9 +295,9 @@
LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
CodeGenFunction &CGF) {
- return CGF.MakeNaturalAlignAddrLValue(
- CGF.GetAddrOfLocalVar(getThreadIDVariable()),
- getThreadIDVariable()->getType());
+ return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()),
+ getThreadIDVariable()->getType(),
+ AlignmentSource::Decl);
}
CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM)
@@ -273,34 +317,61 @@
InternalVars.clear();
}
-llvm::Value *
-CGOpenMPRuntime::emitParallelOutlinedFunction(const OMPExecutableDirective &D,
- const VarDecl *ThreadIDVar,
- const RegionCodeGenTy &CodeGen) {
+// Layout information for ident_t.
+static CharUnits getIdentAlign(CodeGenModule &CGM) {
+ return CGM.getPointerAlign();
+}
+static CharUnits getIdentSize(CodeGenModule &CGM) {
+ assert((4 * CGM.getPointerSize()).isMultipleOf(CGM.getPointerAlign()));
+ return CharUnits::fromQuantity(16) + CGM.getPointerSize();
+}
+static CharUnits getOffsetOfIdentField(CGOpenMPRuntime::IdentFieldIndex Field) {
+ // All the fields except the last are i32, so this works beautifully.
+ return unsigned(Field) * CharUnits::fromQuantity(4);
+}
+static Address createIdentFieldGEP(CodeGenFunction &CGF, Address Addr,
+ CGOpenMPRuntime::IdentFieldIndex Field,
+ const llvm::Twine &Name = "") {
+ auto Offset = getOffsetOfIdentField(Field);
+ return CGF.Builder.CreateStructGEP(Addr, Field, Offset, Name);
+}
+
+llvm::Value *CGOpenMPRuntime::emitParallelOutlinedFunction(
+ const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
+ OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
assert(ThreadIDVar->getType()->isPointerType() &&
"thread id variable must be of type kmp_int32 *");
const CapturedStmt *CS = cast<CapturedStmt>(D.getAssociatedStmt());
CodeGenFunction CGF(CGM, true);
- CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen);
- CGF.CapturedStmtInfo = &CGInfo;
- return CGF.GenerateCapturedStmtFunction(*CS);
+ bool HasCancel = false;
+ if (auto *OPD = dyn_cast<OMPParallelDirective>(&D))
+ HasCancel = OPD->hasCancel();
+ else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
+ HasCancel = OPSD->hasCancel();
+ else if (auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
+ HasCancel = OPFD->hasCancel();
+ CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
+ HasCancel);
+ CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
+ return CGF.GenerateOpenMPCapturedStmtFunction(*CS);
}
-llvm::Value *
-CGOpenMPRuntime::emitTaskOutlinedFunction(const OMPExecutableDirective &D,
- const VarDecl *ThreadIDVar,
- const RegionCodeGenTy &CodeGen) {
+llvm::Value *CGOpenMPRuntime::emitTaskOutlinedFunction(
+ const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
+ OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
assert(!ThreadIDVar->getType()->isPointerType() &&
"thread id variable must be of type kmp_int32 for tasks");
auto *CS = cast<CapturedStmt>(D.getAssociatedStmt());
CodeGenFunction CGF(CGM, true);
- CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen);
- CGF.CapturedStmtInfo = &CGInfo;
+ CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
+ InnermostKind,
+ cast<OMPTaskDirective>(D).hasCancel());
+ CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
return CGF.GenerateCapturedStmtFunction(*CS);
}
-llvm::Value *
-CGOpenMPRuntime::getOrCreateDefaultLocation(OpenMPLocationFlags Flags) {
+Address CGOpenMPRuntime::getOrCreateDefaultLocation(OpenMPLocationFlags Flags) {
+ CharUnits Align = getIdentAlign(CGM);
llvm::Value *Entry = OpenMPDefaultLocMap.lookup(Flags);
if (!Entry) {
if (!DefaultOpenMPPSource) {
@@ -309,7 +380,7 @@
// Taken from
// http://llvm.org/svn/llvm-project/openmp/trunk/runtime/src/kmp_str.c
DefaultOpenMPPSource =
- CGM.GetAddrOfConstantCString(";unknown;unknown;0;0;;");
+ CGM.GetAddrOfConstantCString(";unknown;unknown;0;0;;").getPointer();
DefaultOpenMPPSource =
llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource, CGM.Int8PtrTy);
}
@@ -317,6 +388,7 @@
CGM.getModule(), IdentTy, /*isConstant*/ true,
llvm::GlobalValue::PrivateLinkage, /*Initializer*/ nullptr);
DefaultOpenMPLocation->setUnnamedAddr(true);
+ DefaultOpenMPLocation->setAlignment(Align.getQuantity());
llvm::Constant *Zero = llvm::ConstantInt::get(CGM.Int32Ty, 0, true);
llvm::Constant *Values[] = {Zero,
@@ -324,10 +396,9 @@
Zero, Zero, DefaultOpenMPPSource};
llvm::Constant *Init = llvm::ConstantStruct::get(IdentTy, Values);
DefaultOpenMPLocation->setInitializer(Init);
- OpenMPDefaultLocMap[Flags] = DefaultOpenMPLocation;
- return DefaultOpenMPLocation;
+ OpenMPDefaultLocMap[Flags] = Entry = DefaultOpenMPLocation;
}
- return Entry;
+ return Address(Entry, Align);
}
llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
@@ -336,34 +407,33 @@
// If no debug info is generated - return global default location.
if (CGM.getCodeGenOpts().getDebugInfo() == CodeGenOptions::NoDebugInfo ||
Loc.isInvalid())
- return getOrCreateDefaultLocation(Flags);
+ return getOrCreateDefaultLocation(Flags).getPointer();
assert(CGF.CurFn && "No function in current CodeGenFunction.");
- llvm::Value *LocValue = nullptr;
+ Address LocValue = Address::invalid();
auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
if (I != OpenMPLocThreadIDMap.end())
- LocValue = I->second.DebugLoc;
+ LocValue = Address(I->second.DebugLoc, getIdentAlign(CGF.CGM));
+
// OpenMPLocThreadIDMap may have null DebugLoc and non-null ThreadID, if
// GetOpenMPThreadID was called before this routine.
- if (LocValue == nullptr) {
+ if (!LocValue.isValid()) {
// Generate "ident_t .kmpc_loc.addr;"
- llvm::AllocaInst *AI = CGF.CreateTempAlloca(IdentTy, ".kmpc_loc.addr");
- AI->setAlignment(CGM.getDataLayout().getPrefTypeAlignment(IdentTy));
+ Address AI = CGF.CreateTempAlloca(IdentTy, getIdentAlign(CGF.CGM),
+ ".kmpc_loc.addr");
auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
- Elem.second.DebugLoc = AI;
+ Elem.second.DebugLoc = AI.getPointer();
LocValue = AI;
CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt);
CGF.Builder.CreateMemCpy(LocValue, getOrCreateDefaultLocation(Flags),
- llvm::ConstantExpr::getSizeOf(IdentTy),
- CGM.PointerAlignInBytes);
+ CGM.getSize(getIdentSize(CGF.CGM)));
}
// char **psource = &.kmpc_loc_<flags>.addr.psource;
- auto *PSource = CGF.Builder.CreateConstInBoundsGEP2_32(IdentTy, LocValue, 0,
- IdentField_PSource);
+ Address PSource = createIdentFieldGEP(CGF, LocValue, IdentField_PSource);
auto OMPDebugLoc = OpenMPDebugLocMap.lookup(Loc.getRawEncoding());
if (OMPDebugLoc == nullptr) {
@@ -383,7 +453,9 @@
// *psource = ";<File>;<Function>;<Line>;<Column>;;";
CGF.Builder.CreateStore(OMPDebugLoc, PSource);
- return LocValue;
+ // Our callers always pass this to a runtime function, so for
+ // convenience, go ahead and return a naked pointer.
+ return LocValue.getPointer();
}
llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
@@ -487,6 +559,17 @@
RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical");
break;
}
+ case OMPRTL__kmpc_critical_with_hint: {
+ // Build void __kmpc_critical_with_hint(ident_t *loc, kmp_int32 global_tid,
+ // kmp_critical_name *crit, uintptr_t hint);
+ llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
+ llvm::PointerType::getUnqual(KmpCriticalNameTy),
+ CGM.IntPtrTy};
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
+ RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical_with_hint");
+ break;
+ }
case OMPRTL__kmpc_threadprivate_register: {
// Build void __kmpc_threadprivate_register(ident_t *, void *data,
// kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
@@ -530,6 +613,14 @@
RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_cancel_barrier");
break;
}
+ case OMPRTL__kmpc_barrier: {
+ // Build void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
+ llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
+ RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_barrier");
+ break;
+ }
case OMPRTL__kmpc_for_static_fini: {
// Build void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
@@ -741,7 +832,7 @@
break;
}
case OMPRTL__kmpc_end_ordered: {
- // Build void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
+ // Build void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
llvm::FunctionType *FnTy =
llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
@@ -756,10 +847,114 @@
RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_omp_taskwait");
break;
}
+ case OMPRTL__kmpc_taskgroup: {
+ // Build void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
+ llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
+ RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_taskgroup");
+ break;
+ }
+ case OMPRTL__kmpc_end_taskgroup: {
+ // Build void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
+ llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
+ RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_taskgroup");
+ break;
+ }
+ case OMPRTL__kmpc_push_proc_bind: {
+ // Build void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
+ // int proc_bind)
+ llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
+ RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_proc_bind");
+ break;
+ }
+ case OMPRTL__kmpc_omp_task_with_deps: {
+ // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
+ // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
+ // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
+ llvm::Type *TypeParams[] = {
+ getIdentTyPointerTy(), CGM.Int32Ty, CGM.VoidPtrTy, CGM.Int32Ty,
+ CGM.VoidPtrTy, CGM.Int32Ty, CGM.VoidPtrTy};
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
+ RTLFn =
+ CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_with_deps");
+ break;
+ }
+ case OMPRTL__kmpc_omp_wait_deps: {
+ // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
+ // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias,
+ // kmp_depend_info_t *noalias_dep_list);
+ llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
+ CGM.Int32Ty, CGM.VoidPtrTy,
+ CGM.Int32Ty, CGM.VoidPtrTy};
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
+ RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_wait_deps");
+ break;
+ }
+ case OMPRTL__kmpc_cancellationpoint: {
+ // Build kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
+ // global_tid, kmp_int32 cncl_kind)
+ llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
+ RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancellationpoint");
+ break;
+ }
+ case OMPRTL__kmpc_cancel: {
+ // Build kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
+ // kmp_int32 cncl_kind)
+ llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
+ RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancel");
+ break;
+ }
+ case OMPRTL__tgt_target: {
+ // Build int32_t __tgt_target(int32_t device_id, void *host_ptr, int32_t
+ // arg_num, void** args_base, void **args, size_t *arg_sizes, int32_t
+ // *arg_types);
+ llvm::Type *TypeParams[] = {CGM.Int32Ty,
+ CGM.VoidPtrTy,
+ CGM.Int32Ty,
+ CGM.VoidPtrPtrTy,
+ CGM.VoidPtrPtrTy,
+ CGM.SizeTy->getPointerTo(),
+ CGM.Int32Ty->getPointerTo()};
+ llvm::FunctionType *FnTy =
+ llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
+ RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target");
+ break;
+ }
}
return RTLFn;
}
+static llvm::Value *getTypeSize(CodeGenFunction &CGF, QualType Ty) {
+ auto &C = CGF.getContext();
+ llvm::Value *Size = nullptr;
+ auto SizeInChars = C.getTypeSizeInChars(Ty);
+ if (SizeInChars.isZero()) {
+ // getTypeSizeInChars() returns 0 for a VLA.
+ while (auto *VAT = C.getAsVariableArrayType(Ty)) {
+ llvm::Value *ArraySize;
+ std::tie(ArraySize, Ty) = CGF.getVLASize(VAT);
+ Size = Size ? CGF.Builder.CreateNUWMul(Size, ArraySize) : ArraySize;
+ }
+ SizeInChars = C.getTypeSizeInChars(Ty);
+ assert(!SizeInChars.isZero());
+ Size = CGF.Builder.CreateNUWMul(
+ Size, llvm::ConstantInt::get(CGF.SizeTy, SizeInChars.getQuantity()));
+ } else
+ Size = llvm::ConstantInt::get(CGF.SizeTy, SizeInChars.getQuantity());
+ return Size;
+}
+
llvm::Constant *CGOpenMPRuntime::createForStaticInitFunction(unsigned IVSize,
bool IVSigned) {
assert((IVSize == 32 || IVSize == 64) &&
@@ -850,26 +1045,34 @@
llvm::Constant *
CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {
+ assert(!CGM.getLangOpts().OpenMPUseTLS ||
+ !CGM.getContext().getTargetInfo().isTLSSupported());
// Lookup the entry, lazily creating it if necessary.
return getOrCreateInternalVariable(CGM.Int8PtrPtrTy,
Twine(CGM.getMangledName(VD)) + ".cache.");
}
-llvm::Value *CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
- const VarDecl *VD,
- llvm::Value *VDAddr,
- SourceLocation Loc) {
- auto VarTy = VDAddr->getType()->getPointerElementType();
+Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
+ const VarDecl *VD,
+ Address VDAddr,
+ SourceLocation Loc) {
+ if (CGM.getLangOpts().OpenMPUseTLS &&
+ CGM.getContext().getTargetInfo().isTLSSupported())
+ return VDAddr;
+
+ auto VarTy = VDAddr.getElementType();
llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
- CGF.Builder.CreatePointerCast(VDAddr, CGM.Int8PtrTy),
+ CGF.Builder.CreatePointerCast(VDAddr.getPointer(),
+ CGM.Int8PtrTy),
CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
getOrCreateThreadPrivateCache(VD)};
- return CGF.EmitRuntimeCall(
- createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args);
+ return Address(CGF.EmitRuntimeCall(
+ createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
+ VDAddr.getAlignment());
}
void CGOpenMPRuntime::emitThreadPrivateVarInit(
- CodeGenFunction &CGF, llvm::Value *VDAddr, llvm::Value *Ctor,
+ CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
// Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
// library.
@@ -879,15 +1082,20 @@
// Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
// to register constructor/destructor for variable.
llvm::Value *Args[] = {OMPLoc,
- CGF.Builder.CreatePointerCast(VDAddr, CGM.VoidPtrTy),
+ CGF.Builder.CreatePointerCast(VDAddr.getPointer(),
+ CGM.VoidPtrTy),
Ctor, CopyCtor, Dtor};
CGF.EmitRuntimeCall(
createRuntimeFunction(OMPRTL__kmpc_threadprivate_register), Args);
}
llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
- const VarDecl *VD, llvm::Value *VDAddr, SourceLocation Loc,
+ const VarDecl *VD, Address VDAddr, SourceLocation Loc,
bool PerformInit, CodeGenFunction *CGF) {
+ if (CGM.getLangOpts().OpenMPUseTLS &&
+ CGM.getContext().getTargetInfo().isTLSSupported())
+ return nullptr;
+
VD = VD->getDefinition(CGM.getContext());
if (VD && ThreadPrivateWithDefinition.count(VD) == 0) {
ThreadPrivateWithDefinition.insert(VD);
@@ -909,21 +1117,19 @@
/*isVariadic=*/false);
auto FTy = CGM.getTypes().GetFunctionType(FI);
auto Fn = CGM.CreateGlobalInitOrDestructFunction(
- FTy, ".__kmpc_global_ctor_.", Loc);
+ FTy, ".__kmpc_global_ctor_.", FI, Loc);
CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI,
Args, SourceLocation());
auto ArgVal = CtorCGF.EmitLoadOfScalar(
- CtorCGF.GetAddrOfLocalVar(&Dst),
- /*Volatile=*/false, CGM.PointerAlignInBytes,
+ CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
CGM.getContext().VoidPtrTy, Dst.getLocation());
- auto Arg = CtorCGF.Builder.CreatePointerCast(
- ArgVal,
- CtorCGF.ConvertTypeForMem(CGM.getContext().getPointerType(ASTTy)));
+ Address Arg = Address(ArgVal, VDAddr.getAlignment());
+ Arg = CtorCGF.Builder.CreateElementBitCast(Arg,
+ CtorCGF.ConvertTypeForMem(ASTTy));
CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(),
/*IsInitializer=*/true);
ArgVal = CtorCGF.EmitLoadOfScalar(
- CtorCGF.GetAddrOfLocalVar(&Dst),
- /*Volatile=*/false, CGM.PointerAlignInBytes,
+ CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
CGM.getContext().VoidPtrTy, Dst.getLocation());
CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue);
CtorCGF.FinishFunction();
@@ -943,14 +1149,13 @@
/*isVariadic=*/false);
auto FTy = CGM.getTypes().GetFunctionType(FI);
auto Fn = CGM.CreateGlobalInitOrDestructFunction(
- FTy, ".__kmpc_global_dtor_.", Loc);
+ FTy, ".__kmpc_global_dtor_.", FI, Loc);
DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args,
SourceLocation());
auto ArgVal = DtorCGF.EmitLoadOfScalar(
DtorCGF.GetAddrOfLocalVar(&Dst),
- /*Volatile=*/false, CGM.PointerAlignInBytes,
- CGM.getContext().VoidPtrTy, Dst.getLocation());
- DtorCGF.emitDestroy(ArgVal, ASTTy,
+ /*Volatile=*/false, CGM.getContext().VoidPtrTy, Dst.getLocation());
+ DtorCGF.emitDestroy(Address(ArgVal, VDAddr.getAlignment()), ASTTy,
DtorCGF.getDestroyer(ASTTy.isDestructedType()),
DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
DtorCGF.FinishFunction();
@@ -982,7 +1187,8 @@
auto InitFunctionTy =
llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false);
auto InitFunction = CGM.CreateGlobalInitOrDestructFunction(
- InitFunctionTy, ".__omp_threadprivate_init_.");
+ InitFunctionTy, ".__omp_threadprivate_init_.",
+ CGM.getTypes().arrangeNullaryFunction());
CodeGenFunction InitCGF(CGM);
FunctionArgList ArgList;
InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction,
@@ -1057,25 +1263,27 @@
void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
llvm::Value *OutlinedFn,
- llvm::Value *CapturedStruct,
+ ArrayRef<llvm::Value *> CapturedVars,
const Expr *IfCond) {
+ if (!CGF.HaveInsertPoint())
+ return;
auto *RTLoc = emitUpdateLocation(CGF, Loc);
- auto &&ThenGen =
- [this, OutlinedFn, CapturedStruct, RTLoc](CodeGenFunction &CGF) {
- // Build call __kmpc_fork_call(loc, 1, microtask,
- // captured_struct/*context*/)
- llvm::Value *Args[] = {
- RTLoc,
- CGF.Builder.getInt32(
- 1), // Number of arguments after 'microtask' argument
- // (there is only one additional argument - 'context')
- CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy()),
- CGF.EmitCastToVoidPtr(CapturedStruct)};
- auto RTLFn = createRuntimeFunction(OMPRTL__kmpc_fork_call);
- CGF.EmitRuntimeCall(RTLFn, Args);
- };
- auto &&ElseGen = [this, OutlinedFn, CapturedStruct, RTLoc, Loc](
- CodeGenFunction &CGF) {
+ auto &&ThenGen = [this, OutlinedFn, CapturedVars,
+ RTLoc](CodeGenFunction &CGF) {
+ // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);
+ llvm::Value *Args[] = {
+ RTLoc,
+ CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
+ CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy())};
+ llvm::SmallVector<llvm::Value *, 16> RealArgs;
+ RealArgs.append(std::begin(Args), std::end(Args));
+ RealArgs.append(CapturedVars.begin(), CapturedVars.end());
+
+ auto RTLFn = createRuntimeFunction(OMPRTL__kmpc_fork_call);
+ CGF.EmitRuntimeCall(RTLFn, RealArgs);
+ };
+ auto &&ElseGen = [this, OutlinedFn, CapturedVars, RTLoc,
+ Loc](CodeGenFunction &CGF) {
auto ThreadID = getThreadID(CGF, Loc);
// Build calls:
// __kmpc_serialized_parallel(&Loc, GTid);
@@ -1085,11 +1293,14 @@
// OutlinedFn(>id, &zero, CapturedStruct);
auto ThreadIDAddr = emitThreadIDAddress(CGF, Loc);
- auto Int32Ty = CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32,
- /*Signed*/ true);
- auto ZeroAddr = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".zero.addr");
+ Address ZeroAddr =
+ CGF.CreateTempAlloca(CGF.Int32Ty, CharUnits::fromQuantity(4),
+ /*Name*/ ".zero.addr");
CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0));
- llvm::Value *OutlinedFnArgs[] = {ThreadIDAddr, ZeroAddr, CapturedStruct};
+ llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
+ OutlinedFnArgs.push_back(ThreadIDAddr.getPointer());
+ OutlinedFnArgs.push_back(ZeroAddr.getPointer());
+ OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
CGF.EmitCallOrInvoke(OutlinedFn, OutlinedFnArgs);
// __kmpc_end_serialized_parallel(&Loc, GTid);
@@ -1111,8 +1322,8 @@
// regular serial code region, get thread ID by calling kmp_int32
// kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
// return the address of that temp.
-llvm::Value *CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
- SourceLocation Loc) {
+Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
+ SourceLocation Loc) {
if (auto OMPRegionInfo =
dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
if (OMPRegionInfo->getThreadIDVariable())
@@ -1123,7 +1334,7 @@
CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
auto ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp.");
CGF.EmitStoreOfScalar(ThreadID,
- CGF.MakeNaturalAlignAddrLValue(ThreadIDTemp, Int32Ty));
+ CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty));
return ThreadIDTemp;
}
@@ -1154,7 +1365,7 @@
}
namespace {
-template <size_t N> class CallEndCleanup : public EHScopeStack::Cleanup {
+template <size_t N> class CallEndCleanup final : public EHScopeStack::Cleanup {
llvm::Value *Callee;
llvm::Value *Args[N];
@@ -1165,45 +1376,57 @@
std::copy(CleanupArgs.begin(), CleanupArgs.end(), std::begin(Args));
}
void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
+ if (!CGF.HaveInsertPoint())
+ return;
CGF.EmitRuntimeCall(Callee, Args);
}
};
-} // namespace
+} // anonymous namespace
void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
StringRef CriticalName,
const RegionCodeGenTy &CriticalOpGen,
- SourceLocation Loc) {
- // __kmpc_critical(ident_t *, gtid, Lock);
+ SourceLocation Loc, const Expr *Hint) {
+ // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]);
// CriticalOpGen();
// __kmpc_end_critical(ident_t *, gtid, Lock);
// Prepare arguments and build a call to __kmpc_critical
- {
- CodeGenFunction::RunCleanupsScope Scope(CGF);
- llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
- getCriticalRegionLock(CriticalName)};
+ if (!CGF.HaveInsertPoint())
+ return;
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
+ llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
+ getCriticalRegionLock(CriticalName)};
+ if (Hint) {
+ llvm::SmallVector<llvm::Value *, 8> ArgsWithHint(std::begin(Args),
+ std::end(Args));
+ auto *HintVal = CGF.EmitScalarExpr(Hint);
+ ArgsWithHint.push_back(
+ CGF.Builder.CreateIntCast(HintVal, CGM.IntPtrTy, /*isSigned=*/false));
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_critical_with_hint),
+ ArgsWithHint);
+ } else
CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_critical), Args);
- // Build a call to __kmpc_end_critical
- CGF.EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>(
- NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_critical),
- llvm::makeArrayRef(Args));
- emitInlinedDirective(CGF, CriticalOpGen);
- }
+ // Build a call to __kmpc_end_critical
+ CGF.EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>(
+ NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_critical),
+ llvm::makeArrayRef(Args));
+ emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen);
}
static void emitIfStmt(CodeGenFunction &CGF, llvm::Value *IfCond,
+ OpenMPDirectiveKind Kind, SourceLocation Loc,
const RegionCodeGenTy &BodyOpGen) {
llvm::Value *CallBool = CGF.EmitScalarConversion(
IfCond,
CGF.getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true),
- CGF.getContext().BoolTy);
+ CGF.getContext().BoolTy, Loc);
auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
auto *ContBlock = CGF.createBasicBlock("omp_if.end");
// Generate the branch (If-stmt)
CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
CGF.EmitBlock(ThenBlock);
- CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, BodyOpGen);
+ CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, Kind, BodyOpGen);
// Emit the rest of bblocks/branches
CGF.EmitBranch(ContBlock);
CGF.EmitBlock(ContBlock, true);
@@ -1212,6 +1435,8 @@
void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
const RegionCodeGenTy &MasterOpGen,
SourceLocation Loc) {
+ if (!CGF.HaveInsertPoint())
+ return;
// if(__kmpc_master(ident_t *, gtid)) {
// MasterOpGen();
// __kmpc_end_master(ident_t *, gtid);
@@ -1222,17 +1447,20 @@
CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_master), Args);
typedef CallEndCleanup<std::extent<decltype(Args)>::value>
MasterCallEndCleanup;
- emitIfStmt(CGF, IsMaster, [&](CodeGenFunction &CGF) -> void {
- CodeGenFunction::RunCleanupsScope Scope(CGF);
- CGF.EHStack.pushCleanup<MasterCallEndCleanup>(
- NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_master),
- llvm::makeArrayRef(Args));
- MasterOpGen(CGF);
- });
+ emitIfStmt(
+ CGF, IsMaster, OMPD_master, Loc, [&](CodeGenFunction &CGF) -> void {
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
+ CGF.EHStack.pushCleanup<MasterCallEndCleanup>(
+ NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_master),
+ llvm::makeArrayRef(Args));
+ MasterOpGen(CGF);
+ });
}
void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
SourceLocation Loc) {
+ if (!CGF.HaveInsertPoint())
+ return;
// Build call __kmpc_omp_taskyield(loc, thread_id, 0);
llvm::Value *Args[] = {
emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
@@ -1240,6 +1468,42 @@
CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskyield), Args);
}
+void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
+ const RegionCodeGenTy &TaskgroupOpGen,
+ SourceLocation Loc) {
+ if (!CGF.HaveInsertPoint())
+ return;
+ // __kmpc_taskgroup(ident_t *, gtid);
+ // TaskgroupOpGen();
+ // __kmpc_end_taskgroup(ident_t *, gtid);
+ // Prepare arguments and build a call to __kmpc_taskgroup
+ {
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
+ llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_taskgroup), Args);
+ // Build a call to __kmpc_end_taskgroup
+ CGF.EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>(
+ NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_taskgroup),
+ llvm::makeArrayRef(Args));
+ emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen);
+ }
+}
+
+/// Given an array of pointers to variables, project the address of a
+/// given variable.
+static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
+ unsigned Index, const VarDecl *Var) {
+ // Pull out the pointer to the variable.
+ Address PtrAddr =
+ CGF.Builder.CreateConstArrayGEP(Array, Index, CGF.getPointerSize());
+ llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr);
+
+ Address Addr = Address(Ptr, CGF.getContext().getDeclAlign(Var));
+ Addr = CGF.Builder.CreateElementBitCast(
+ Addr, CGF.ConvertTypeForMem(Var->getType()));
+ return Addr;
+}
+
static llvm::Value *emitCopyprivateCopyFunction(
CodeGenModule &CGM, llvm::Type *ArgsType,
ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
@@ -1259,40 +1523,31 @@
auto *Fn = llvm::Function::Create(
CGM.getTypes().GetFunctionType(CGFI), llvm::GlobalValue::InternalLinkage,
".omp.copyprivate.copy_func", &CGM.getModule());
- CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, CGFI, Fn);
+ CGM.SetInternalFunctionAttributes(/*D=*/nullptr, Fn, CGFI);
CodeGenFunction CGF(CGM);
CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args);
// Dest = (void*[n])(LHSArg);
// Src = (void*[n])(RHSArg);
- auto *LHS = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- CGF.Builder.CreateAlignedLoad(CGF.GetAddrOfLocalVar(&LHSArg),
- CGF.PointerAlignInBytes),
- ArgsType);
- auto *RHS = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- CGF.Builder.CreateAlignedLoad(CGF.GetAddrOfLocalVar(&RHSArg),
- CGF.PointerAlignInBytes),
- ArgsType);
+ Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
+ ArgsType), CGF.getPointerAlign());
+ Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
+ ArgsType), CGF.getPointerAlign());
// *(Type0*)Dst[0] = *(Type0*)Src[0];
// *(Type1*)Dst[1] = *(Type1*)Src[1];
// ...
// *(Typen*)Dst[n] = *(Typen*)Src[n];
for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
- auto *DestAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- CGF.Builder.CreateAlignedLoad(
- CGF.Builder.CreateStructGEP(nullptr, LHS, I),
- CGM.PointerAlignInBytes),
- CGF.ConvertTypeForMem(C.getPointerType(SrcExprs[I]->getType())));
- auto *SrcAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- CGF.Builder.CreateAlignedLoad(
- CGF.Builder.CreateStructGEP(nullptr, RHS, I),
- CGM.PointerAlignInBytes),
- CGF.ConvertTypeForMem(C.getPointerType(SrcExprs[I]->getType())));
+ auto DestVar = cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
+ Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar);
+
+ auto SrcVar = cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
+ Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar);
+
auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
QualType Type = VD->getType();
- CGF.EmitOMPCopy(CGF, Type, DestAddr, SrcAddr,
- cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl()),
- cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl()),
- AssignmentOps[I]);
+ CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
}
CGF.FinishFunction();
return Fn;
@@ -1305,6 +1560,8 @@
ArrayRef<const Expr *> SrcExprs,
ArrayRef<const Expr *> DstExprs,
ArrayRef<const Expr *> AssignmentOps) {
+ if (!CGF.HaveInsertPoint())
+ return;
assert(CopyprivateVars.size() == SrcExprs.size() &&
CopyprivateVars.size() == DstExprs.size() &&
CopyprivateVars.size() == AssignmentOps.size());
@@ -1318,13 +1575,12 @@
// call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
// <copy_func>, did_it);
- llvm::AllocaInst *DidIt = nullptr;
+ Address DidIt = Address::invalid();
if (!CopyprivateVars.empty()) {
// int32 did_it = 0;
auto KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it");
- CGF.Builder.CreateAlignedStore(CGF.Builder.getInt32(0), DidIt,
- DidIt->getAlignment());
+ CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt);
}
// Prepare arguments and build a call to __kmpc_single
llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
@@ -1332,52 +1588,51 @@
CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_single), Args);
typedef CallEndCleanup<std::extent<decltype(Args)>::value>
SingleCallEndCleanup;
- emitIfStmt(CGF, IsSingle, [&](CodeGenFunction &CGF) -> void {
- CodeGenFunction::RunCleanupsScope Scope(CGF);
- CGF.EHStack.pushCleanup<SingleCallEndCleanup>(
- NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_single),
- llvm::makeArrayRef(Args));
- SingleOpGen(CGF);
- if (DidIt) {
- // did_it = 1;
- CGF.Builder.CreateAlignedStore(CGF.Builder.getInt32(1), DidIt,
- DidIt->getAlignment());
- }
- });
+ emitIfStmt(
+ CGF, IsSingle, OMPD_single, Loc, [&](CodeGenFunction &CGF) -> void {
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
+ CGF.EHStack.pushCleanup<SingleCallEndCleanup>(
+ NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_single),
+ llvm::makeArrayRef(Args));
+ SingleOpGen(CGF);
+ if (DidIt.isValid()) {
+ // did_it = 1;
+ CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt);
+ }
+ });
// call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
// <copy_func>, did_it);
- if (DidIt) {
+ if (DidIt.isValid()) {
llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
auto CopyprivateArrayTy =
C.getConstantArrayType(C.VoidPtrTy, ArraySize, ArrayType::Normal,
/*IndexTypeQuals=*/0);
// Create a list of all private variables for copyprivate.
- auto *CopyprivateList =
+ Address CopyprivateList =
CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list");
for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
- auto *Elem = CGF.Builder.CreateStructGEP(
- CopyprivateList->getAllocatedType(), CopyprivateList, I);
- CGF.Builder.CreateAlignedStore(
+ Address Elem = CGF.Builder.CreateConstArrayGEP(
+ CopyprivateList, I, CGF.getPointerSize());
+ CGF.Builder.CreateStore(
CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- CGF.EmitLValue(CopyprivateVars[I]).getAddress(), CGF.VoidPtrTy),
- Elem, CGM.PointerAlignInBytes);
+ CGF.EmitLValue(CopyprivateVars[I]).getPointer(), CGF.VoidPtrTy),
+ Elem);
}
// Build function that copies private values from single region to all other
// threads in the corresponding parallel region.
auto *CpyFn = emitCopyprivateCopyFunction(
CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(),
CopyprivateVars, SrcExprs, DstExprs, AssignmentOps);
- auto *BufSize = llvm::ConstantInt::get(
- CGM.SizeTy, C.getTypeSizeInChars(CopyprivateArrayTy).getQuantity());
- auto *CL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList,
- CGF.VoidPtrTy);
- auto *DidItVal =
- CGF.Builder.CreateAlignedLoad(DidIt, CGF.PointerAlignInBytes);
+ auto *BufSize = getTypeSize(CGF, CopyprivateArrayTy);
+ Address CL =
+ CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList,
+ CGF.VoidPtrTy);
+ auto *DidItVal = CGF.Builder.CreateLoad(DidIt);
llvm::Value *Args[] = {
emitUpdateLocation(CGF, Loc), // ident_t *<loc>
getThreadID(CGF, Loc), // i32 <gtid>
BufSize, // size_t <buf_size>
- CL, // void *<copyprivate list>
+ CL.getPointer(), // void *<copyprivate list>
CpyFn, // void (*) (void *, void *) <copy_func>
DidItVal // i32 did_it
};
@@ -1387,26 +1642,32 @@
void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
const RegionCodeGenTy &OrderedOpGen,
- SourceLocation Loc) {
+ SourceLocation Loc, bool IsThreads) {
+ if (!CGF.HaveInsertPoint())
+ return;
// __kmpc_ordered(ident_t *, gtid);
// OrderedOpGen();
// __kmpc_end_ordered(ident_t *, gtid);
// Prepare arguments and build a call to __kmpc_ordered
- {
- CodeGenFunction::RunCleanupsScope Scope(CGF);
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
+ if (IsThreads) {
llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_ordered), Args);
// Build a call to __kmpc_end_ordered
CGF.EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>(
NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_ordered),
llvm::makeArrayRef(Args));
- emitInlinedDirective(CGF, OrderedOpGen);
}
+ emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
}
void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
- OpenMPDirectiveKind Kind) {
+ OpenMPDirectiveKind Kind, bool EmitChecks,
+ bool ForceSimpleCall) {
+ if (!CGF.HaveInsertPoint())
+ return;
// Build call __kmpc_cancel_barrier(loc, thread_id);
+ // Build call __kmpc_barrier(loc, thread_id);
OpenMPLocationFlags Flags = OMP_IDENT_KMPC;
if (Kind == OMPD_for) {
Flags =
@@ -1422,15 +1683,39 @@
} else {
Flags = static_cast<OpenMPLocationFlags>(Flags | OMP_IDENT_BARRIER_IMPL);
}
- // Build call __kmpc_cancel_barrier(loc, thread_id);
- // Replace __kmpc_barrier() function by __kmpc_cancel_barrier() because this
- // one provides the same functionality and adds initial support for
- // cancellation constructs introduced in OpenMP 4.0. __kmpc_cancel_barrier()
- // is provided default by the runtime library so it safe to make such
- // replacement.
+ // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc,
+ // thread_id);
+ auto *OMPRegionInfo =
+ dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo);
+ // Do not emit barrier call in the single directive emitted in some rare cases
+ // for sections directives.
+ if (OMPRegionInfo && OMPRegionInfo->getDirectiveKind() == OMPD_single)
+ return;
llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
getThreadID(CGF, Loc)};
- CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_cancel_barrier), Args);
+ if (OMPRegionInfo) {
+ if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
+ auto *Result = CGF.EmitRuntimeCall(
+ createRuntimeFunction(OMPRTL__kmpc_cancel_barrier), Args);
+ if (EmitChecks) {
+ // if (__kmpc_cancel_barrier()) {
+ // exit from construct;
+ // }
+ auto *ExitBB = CGF.createBasicBlock(".cancel.exit");
+ auto *ContBB = CGF.createBasicBlock(".cancel.continue");
+ auto *Cmp = CGF.Builder.CreateIsNotNull(Result);
+ CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
+ CGF.EmitBlock(ExitBB);
+ // exit from construct;
+ auto CancelDestination =
+ CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
+ CGF.EmitBranchThroughCleanup(CancelDestination);
+ CGF.EmitBlock(ContBB, /*IsFinished=*/true);
+ }
+ return;
+ }
+ }
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_barrier), Args);
}
/// \brief Schedule types for 'omp for' loops (these enumerators are taken from
@@ -1490,65 +1775,87 @@
return Schedule != OMP_sch_static;
}
-void CGOpenMPRuntime::emitForInit(CodeGenFunction &CGF, SourceLocation Loc,
- OpenMPScheduleClauseKind ScheduleKind,
- unsigned IVSize, bool IVSigned, bool Ordered,
- llvm::Value *IL, llvm::Value *LB,
- llvm::Value *UB, llvm::Value *ST,
- llvm::Value *Chunk) {
+void CGOpenMPRuntime::emitForDispatchInit(CodeGenFunction &CGF,
+ SourceLocation Loc,
+ OpenMPScheduleClauseKind ScheduleKind,
+ unsigned IVSize, bool IVSigned,
+ bool Ordered, llvm::Value *UB,
+ llvm::Value *Chunk) {
+ if (!CGF.HaveInsertPoint())
+ return;
OpenMPSchedType Schedule =
getRuntimeSchedule(ScheduleKind, Chunk != nullptr, Ordered);
- if (Ordered ||
- (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&
- Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked)) {
- // Call __kmpc_dispatch_init(
- // ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
- // kmp_int[32|64] lower, kmp_int[32|64] upper,
- // kmp_int[32|64] stride, kmp_int[32|64] chunk);
+ assert(Ordered ||
+ (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&
+ Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked));
+ // Call __kmpc_dispatch_init(
+ // ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
+ // kmp_int[32|64] lower, kmp_int[32|64] upper,
+ // kmp_int[32|64] stride, kmp_int[32|64] chunk);
+ // If the Chunk was not specified in the clause - use default value 1.
+ if (Chunk == nullptr)
+ Chunk = CGF.Builder.getIntN(IVSize, 1);
+ llvm::Value *Args[] = {
+ emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC),
+ getThreadID(CGF, Loc),
+ CGF.Builder.getInt32(Schedule), // Schedule type
+ CGF.Builder.getIntN(IVSize, 0), // Lower
+ UB, // Upper
+ CGF.Builder.getIntN(IVSize, 1), // Stride
+ Chunk // Chunk
+ };
+ CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args);
+}
+
+void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
+ SourceLocation Loc,
+ OpenMPScheduleClauseKind ScheduleKind,
+ unsigned IVSize, bool IVSigned,
+ bool Ordered, Address IL, Address LB,
+ Address UB, Address ST,
+ llvm::Value *Chunk) {
+ if (!CGF.HaveInsertPoint())
+ return;
+ OpenMPSchedType Schedule =
+ getRuntimeSchedule(ScheduleKind, Chunk != nullptr, Ordered);
+ assert(!Ordered);
+ assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||
+ Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked);
+
+ // Call __kmpc_for_static_init(
+ // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
+ // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
+ // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
+ // kmp_int[32|64] incr, kmp_int[32|64] chunk);
+ if (Chunk == nullptr) {
+ assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static) &&
+ "expected static non-chunked schedule");
// If the Chunk was not specified in the clause - use default value 1.
- if (Chunk == nullptr)
Chunk = CGF.Builder.getIntN(IVSize, 1);
- llvm::Value *Args[] = { emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC),
- getThreadID(CGF, Loc),
- CGF.Builder.getInt32(Schedule), // Schedule type
- CGF.Builder.getIntN(IVSize, 0), // Lower
- UB, // Upper
- CGF.Builder.getIntN(IVSize, 1), // Stride
- Chunk // Chunk
- };
- CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args);
} else {
- // Call __kmpc_for_static_init(
- // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
- // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
- // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
- // kmp_int[32|64] incr, kmp_int[32|64] chunk);
- if (Chunk == nullptr) {
- assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static) &&
- "expected static non-chunked schedule");
- // If the Chunk was not specified in the clause - use default value 1.
- Chunk = CGF.Builder.getIntN(IVSize, 1);
- } else
- assert((Schedule == OMP_sch_static_chunked ||
- Schedule == OMP_ord_static_chunked) &&
- "expected static chunked schedule");
- llvm::Value *Args[] = { emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC),
- getThreadID(CGF, Loc),
- CGF.Builder.getInt32(Schedule), // Schedule type
- IL, // &isLastIter
- LB, // &LB
- UB, // &UB
- ST, // &Stride
- CGF.Builder.getIntN(IVSize, 1), // Incr
- Chunk // Chunk
- };
- CGF.EmitRuntimeCall(createForStaticInitFunction(IVSize, IVSigned), Args);
+ assert((Schedule == OMP_sch_static_chunked ||
+ Schedule == OMP_ord_static_chunked) &&
+ "expected static chunked schedule");
}
+ llvm::Value *Args[] = {
+ emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC),
+ getThreadID(CGF, Loc),
+ CGF.Builder.getInt32(Schedule), // Schedule type
+ IL.getPointer(), // &isLastIter
+ LB.getPointer(), // &LB
+ UB.getPointer(), // &UB
+ ST.getPointer(), // &Stride
+ CGF.Builder.getIntN(IVSize, 1), // Incr
+ Chunk // Chunk
+ };
+ CGF.EmitRuntimeCall(createForStaticInitFunction(IVSize, IVSigned), Args);
}
void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
SourceLocation Loc) {
+ if (!CGF.HaveInsertPoint())
+ return;
// Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC),
getThreadID(CGF, Loc)};
@@ -1560,6 +1867,8 @@
SourceLocation Loc,
unsigned IVSize,
bool IVSigned) {
+ if (!CGF.HaveInsertPoint())
+ return;
// Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC),
getThreadID(CGF, Loc)};
@@ -1568,30 +1877,32 @@
llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
SourceLocation Loc, unsigned IVSize,
- bool IVSigned, llvm::Value *IL,
- llvm::Value *LB, llvm::Value *UB,
- llvm::Value *ST) {
+ bool IVSigned, Address IL,
+ Address LB, Address UB,
+ Address ST) {
// Call __kmpc_dispatch_next(
// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
// kmp_int[32|64] *p_stride);
llvm::Value *Args[] = {
emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC), getThreadID(CGF, Loc),
- IL, // &isLastIter
- LB, // &Lower
- UB, // &Upper
- ST // &Stride
+ IL.getPointer(), // &isLastIter
+ LB.getPointer(), // &Lower
+ UB.getPointer(), // &Upper
+ ST.getPointer() // &Stride
};
llvm::Value *Call =
CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args);
return CGF.EmitScalarConversion(
Call, CGF.getContext().getIntTypeForBitwidth(32, /* Signed */ true),
- CGF.getContext().BoolTy);
+ CGF.getContext().BoolTy, Loc);
}
void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
llvm::Value *NumThreads,
SourceLocation Loc) {
+ if (!CGF.HaveInsertPoint())
+ return;
// Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
llvm::Value *Args[] = {
emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
@@ -1600,8 +1911,45 @@
Args);
}
+void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
+ OpenMPProcBindClauseKind ProcBind,
+ SourceLocation Loc) {
+ if (!CGF.HaveInsertPoint())
+ return;
+ // Constants for proc bind value accepted by the runtime.
+ enum ProcBindTy {
+ ProcBindFalse = 0,
+ ProcBindTrue,
+ ProcBindMaster,
+ ProcBindClose,
+ ProcBindSpread,
+ ProcBindIntel,
+ ProcBindDefault
+ } RuntimeProcBind;
+ switch (ProcBind) {
+ case OMPC_PROC_BIND_master:
+ RuntimeProcBind = ProcBindMaster;
+ break;
+ case OMPC_PROC_BIND_close:
+ RuntimeProcBind = ProcBindClose;
+ break;
+ case OMPC_PROC_BIND_spread:
+ RuntimeProcBind = ProcBindSpread;
+ break;
+ case OMPC_PROC_BIND_unknown:
+ llvm_unreachable("Unsupported proc_bind value.");
+ }
+ // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind)
+ llvm::Value *Args[] = {
+ emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
+ llvm::ConstantInt::get(CGM.IntTy, RuntimeProcBind, /*isSigned=*/true)};
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_proc_bind), Args);
+}
+
void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
SourceLocation Loc) {
+ if (!CGF.HaveInsertPoint())
+ return;
// Build call void __kmpc_flush(ident_t *loc)
CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_flush),
emitUpdateLocation(CGF, Loc));
@@ -1619,7 +1967,7 @@
/// \brief Function with call of destructors for private variables.
KmpTaskTDestructors,
};
-} // namespace
+} // anonymous namespace
void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) {
if (!KmpRoutineEntryPtrTy) {
@@ -1633,14 +1981,15 @@
}
}
-static void addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
- QualType FieldTy) {
+static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
+ QualType FieldTy) {
auto *Field = FieldDecl::Create(
C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy,
C.getTrivialTypeSourceInfo(FieldTy, SourceLocation()),
/*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);
Field->setAccess(AS_public);
DC->addDecl(Field);
+ return Field;
}
namespace {
@@ -1654,11 +2003,10 @@
const VarDecl *PrivateElemInit;
};
typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;
-} // namespace
+} // anonymous namespace
static RecordDecl *
-createPrivatesRecordDecl(CodeGenModule &CGM,
- const ArrayRef<PrivateDataTy> Privates) {
+createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) {
if (!Privates.empty()) {
auto &C = CGM.getContext();
// Build struct .kmp_privates_t. {
@@ -1667,9 +2015,16 @@
auto *RD = C.buildImplicitRecord(".kmp_privates.t");
RD->startDefinition();
for (auto &&Pair : Privates) {
- auto Type = Pair.second.Original->getType();
+ auto *VD = Pair.second.Original;
+ auto Type = VD->getType();
Type = Type.getNonReferenceType();
- addFieldToRecordDecl(C, RD, Type);
+ auto *FD = addFieldToRecordDecl(C, RD, Type);
+ if (VD->hasAttrs()) {
+ for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),
+ E(VD->getAttrs().end());
+ I != E; ++I)
+ FD->addAttr(*I);
+ }
}
RD->completeDefinition();
return RD;
@@ -1699,7 +2054,7 @@
static RecordDecl *
createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,
- const ArrayRef<PrivateDataTy> Privates) {
+ ArrayRef<PrivateDataTy> Privates) {
auto &C = CGM.getContext();
// Build struct kmp_task_t_with_privates {
// kmp_task_t task_data;
@@ -1734,7 +2089,8 @@
FunctionArgList Args;
ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty);
ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc,
- /*Id=*/nullptr, KmpTaskTWithPrivatesPtrQTy);
+ /*Id=*/nullptr,
+ KmpTaskTWithPrivatesPtrQTy.withRestrict());
Args.push_back(&GtidArg);
Args.push_back(&TaskTypeArg);
FunctionType::ExtInfo Info;
@@ -1745,7 +2101,7 @@
auto *TaskEntry =
llvm::Function::Create(TaskEntryTy, llvm::GlobalValue::InternalLinkage,
".omp_task_entry.", &CGM.getModule());
- CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, TaskEntryFnInfo, TaskEntry);
+ CGM.SetInternalFunctionAttributes(/*D=*/nullptr, TaskEntry, TaskEntryFnInfo);
CodeGenFunction CGF(CGM);
CGF.disableDebugInfo();
CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args);
@@ -1753,12 +2109,9 @@
// TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
// tt->task_data.shareds);
auto *GtidParam = CGF.EmitLoadOfScalar(
- CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false,
- C.getTypeAlignInChars(KmpInt32Ty).getQuantity(), KmpInt32Ty, Loc);
- auto *TaskTypeArgAddr = CGF.Builder.CreateAlignedLoad(
- CGF.GetAddrOfLocalVar(&TaskTypeArg), CGM.PointerAlignInBytes);
- LValue TDBase =
- CGF.MakeNaturalAlignAddrLValue(TaskTypeArgAddr, KmpTaskTWithPrivatesQTy);
+ CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc);
+ LValue TDBase = emitLoadOfPointerLValue(
+ CGF, CGF.GetAddrOfLocalVar(&TaskTypeArg), KmpTaskTWithPrivatesPtrQTy);
auto *KmpTaskTWithPrivatesQTyRD =
cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
LValue Base =
@@ -1779,7 +2132,7 @@
if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
auto PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI);
PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- PrivatesLVal.getAddress(), CGF.VoidPtrTy);
+ PrivatesLVal.getPointer(), CGF.VoidPtrTy);
} else {
PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
}
@@ -1789,7 +2142,7 @@
CGF.EmitCallOrInvoke(TaskFunction, CallArgs);
CGF.EmitStoreThroughLValue(
RValue::get(CGF.Builder.getInt32(/*C=*/0)),
- CGF.MakeNaturalAlignAddrLValue(CGF.ReturnValue, KmpInt32Ty));
+ CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty));
CGF.FinishFunction();
return TaskEntry;
}
@@ -1803,7 +2156,8 @@
FunctionArgList Args;
ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty);
ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc,
- /*Id=*/nullptr, KmpTaskTWithPrivatesPtrQTy);
+ /*Id=*/nullptr,
+ KmpTaskTWithPrivatesPtrQTy.withRestrict());
Args.push_back(&GtidArg);
Args.push_back(&TaskTypeArg);
FunctionType::ExtInfo Info;
@@ -1814,16 +2168,15 @@
auto *DestructorFn =
llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
".omp_task_destructor.", &CGM.getModule());
- CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, DestructorFnInfo, DestructorFn);
+ CGM.SetInternalFunctionAttributes(/*D=*/nullptr, DestructorFn,
+ DestructorFnInfo);
CodeGenFunction CGF(CGM);
CGF.disableDebugInfo();
CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo,
Args);
- auto *TaskTypeArgAddr = CGF.Builder.CreateAlignedLoad(
- CGF.GetAddrOfLocalVar(&TaskTypeArg), CGM.PointerAlignInBytes);
- LValue Base =
- CGF.MakeNaturalAlignAddrLValue(TaskTypeArgAddr, KmpTaskTWithPrivatesQTy);
+ LValue Base = emitLoadOfPointerLValue(
+ CGF, CGF.GetAddrOfLocalVar(&TaskTypeArg), KmpTaskTWithPrivatesPtrQTy);
auto *KmpTaskTWithPrivatesQTyRD =
cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
@@ -1851,10 +2204,10 @@
/// \endcode
static llvm::Value *
emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
- const ArrayRef<const Expr *> PrivateVars,
- const ArrayRef<const Expr *> FirstprivateVars,
+ ArrayRef<const Expr *> PrivateVars,
+ ArrayRef<const Expr *> FirstprivateVars,
QualType PrivatesQTy,
- const ArrayRef<PrivateDataTy> Privates) {
+ ArrayRef<PrivateDataTy> Privates) {
auto &C = CGM.getContext();
FunctionArgList Args;
ImplicitParamDecl TaskPrivatesArg(
@@ -1892,8 +2245,8 @@
auto *TaskPrivatesMap = llvm::Function::Create(
TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage,
".omp_task_privates_map.", &CGM.getModule());
- CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, TaskPrivatesMapFnInfo,
- TaskPrivatesMap);
+ CGM.SetInternalFunctionAttributes(/*D=*/nullptr, TaskPrivatesMap,
+ TaskPrivatesMapFnInfo);
TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
CodeGenFunction CGF(CGM);
CGF.disableDebugInfo();
@@ -1901,22 +2254,17 @@
TaskPrivatesMapFnInfo, Args);
// *privi = &.privates.privi;
- auto *TaskPrivatesArgAddr = CGF.Builder.CreateAlignedLoad(
- CGF.GetAddrOfLocalVar(&TaskPrivatesArg), CGM.PointerAlignInBytes);
- LValue Base =
- CGF.MakeNaturalAlignAddrLValue(TaskPrivatesArgAddr, PrivatesQTy);
+ LValue Base = emitLoadOfPointerLValue(
+ CGF, CGF.GetAddrOfLocalVar(&TaskPrivatesArg), TaskPrivatesArg.getType());
auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->getAsTagDecl());
Counter = 0;
for (auto *Field : PrivatesQTyRD->fields()) {
auto FieldLVal = CGF.EmitLValueForField(Base, Field);
auto *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
- auto RefLVal = CGF.MakeNaturalAlignAddrLValue(CGF.GetAddrOfLocalVar(VD),
- VD->getType());
- auto RefLoadRVal = CGF.EmitLoadOfLValue(RefLVal, Loc);
- CGF.EmitStoreOfScalar(
- FieldLVal.getAddress(),
- CGF.MakeNaturalAlignAddrLValue(RefLoadRVal.getScalarVal(),
- RefLVal.getType()->getPointeeType()));
+ auto RefLVal = CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());
+ auto RefLoadLVal =
+ emitLoadOfPointerLValue(CGF, RefLVal.getAddress(), RefLVal.getType());
+ CGF.EmitStoreOfScalar(FieldLVal.getPointer(), RefLoadLVal);
++Counter;
}
CGF.FinishFunction();
@@ -1931,12 +2279,15 @@
void CGOpenMPRuntime::emitTaskCall(
CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D,
bool Tied, llvm::PointerIntPair<llvm::Value *, 1, bool> Final,
- llvm::Value *TaskFunction, QualType SharedsTy, llvm::Value *Shareds,
- const Expr *IfCond, const ArrayRef<const Expr *> PrivateVars,
- const ArrayRef<const Expr *> PrivateCopies,
- const ArrayRef<const Expr *> FirstprivateVars,
- const ArrayRef<const Expr *> FirstprivateCopies,
- const ArrayRef<const Expr *> FirstprivateInits) {
+ llvm::Value *TaskFunction, QualType SharedsTy, Address Shareds,
+ const Expr *IfCond, ArrayRef<const Expr *> PrivateVars,
+ ArrayRef<const Expr *> PrivateCopies,
+ ArrayRef<const Expr *> FirstprivateVars,
+ ArrayRef<const Expr *> FirstprivateCopies,
+ ArrayRef<const Expr *> FirstprivateInits,
+ ArrayRef<std::pair<OpenMPDependClauseKind, const Expr *>> Dependences) {
+ if (!CGF.HaveInsertPoint())
+ return;
auto &C = CGM.getContext();
llvm::SmallVector<PrivateDataTy, 8> Privates;
// Aggregate privates and sort them by the alignment.
@@ -1944,7 +2295,7 @@
for (auto *E : PrivateVars) {
auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
Privates.push_back(std::make_pair(
- C.getTypeAlignInChars(VD->getType()),
+ C.getDeclAlign(VD),
PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
/*PrivateElemInit=*/nullptr)));
++I;
@@ -1954,7 +2305,7 @@
for (auto *E : FirstprivateVars) {
auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
Privates.push_back(std::make_pair(
- C.getTypeAlignInChars(VD->getType()),
+ C.getDeclAlign(VD),
PrivateHelpersTy(
VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl()))));
@@ -1979,8 +2330,7 @@
C.getPointerType(KmpTaskTWithPrivatesQTy);
auto *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy);
auto *KmpTaskTWithPrivatesPtrTy = KmpTaskTWithPrivatesTy->getPointerTo();
- auto KmpTaskTWithPrivatesTySize =
- CGM.getSize(C.getTypeSizeInChars(KmpTaskTWithPrivatesQTy));
+ auto *KmpTaskTWithPrivatesTySize = getTypeSize(CGF, KmpTaskTWithPrivatesQTy);
QualType SharedsPtrTy = C.getPointerType(SharedsTy);
// Emit initial values for private copies (if any).
@@ -2021,12 +2371,12 @@
CGF.Builder.getInt32(/*C=*/0))
: CGF.Builder.getInt32(Final.getInt() ? FinalFlag : 0);
TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags));
- auto SharedsSize = C.getTypeSizeInChars(SharedsTy);
- llvm::Value *AllocArgs[] = {
- emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), TaskFlags,
- KmpTaskTWithPrivatesTySize, CGM.getSize(SharedsSize),
- CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskEntry,
- KmpRoutineEntryPtrTy)};
+ auto *SharedsSize = CGM.getSize(C.getTypeSizeInChars(SharedsTy));
+ llvm::Value *AllocArgs[] = {emitUpdateLocation(CGF, Loc),
+ getThreadID(CGF, Loc), TaskFlags,
+ KmpTaskTWithPrivatesTySize, SharedsSize,
+ CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ TaskEntry, KmpRoutineEntryPtrTy)};
auto *NewTask = CGF.EmitRuntimeCall(
createRuntimeFunction(OMPRTL__kmpc_omp_task_alloc), AllocArgs);
auto *NewTaskNewTaskTTy = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
@@ -2037,12 +2387,15 @@
CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());
// Fill the data in the resulting kmp_task_t record.
// Copy shareds if there are any.
- llvm::Value *KmpTaskSharedsPtr = nullptr;
+ Address KmpTaskSharedsPtr = Address::invalid();
if (!SharedsTy->getAsStructureType()->getDecl()->field_empty()) {
- KmpTaskSharedsPtr = CGF.EmitLoadOfScalar(
- CGF.EmitLValueForField(
- TDBase, *std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds)),
- Loc);
+ KmpTaskSharedsPtr =
+ Address(CGF.EmitLoadOfScalar(
+ CGF.EmitLValueForField(
+ TDBase, *std::next(KmpTaskTQTyRD->field_begin(),
+ KmpTaskTShareds)),
+ Loc),
+ CGF.getNaturalTypeAlignment(SharedsTy));
CGF.EmitAggregateCopy(KmpTaskSharedsPtr, Shareds, SharedsTy);
}
// Emit initial values for private copies (if any).
@@ -2053,7 +2406,7 @@
FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
LValue SharedsBase;
if (!FirstprivateVars.empty()) {
- SharedsBase = CGF.MakeNaturalAlignAddrLValue(
+ SharedsBase = CGF.MakeAddrLValue(
CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)),
SharedsTy);
@@ -2070,6 +2423,9 @@
auto *SharedField = CapturesInfo.lookup(OriginalVD);
auto SharedRefLValue =
CGF.EmitLValueForField(SharedsBase, SharedField);
+ SharedRefLValue = CGF.MakeAddrLValue(
+ Address(SharedRefLValue.getPointer(), C.getDeclAlign(OriginalVD)),
+ SharedRefLValue.getType(), AlignmentSource::Decl);
QualType Type = OriginalVD->getType();
if (Type->isArrayType()) {
// Initialize firstprivate array.
@@ -2084,33 +2440,30 @@
CGF.EmitOMPAggregateAssign(
PrivateLValue.getAddress(), SharedRefLValue.getAddress(),
Type, [&CGF, Elem, Init, &CapturesInfo](
- llvm::Value *DestElement, llvm::Value *SrcElement) {
+ Address DestElement, Address SrcElement) {
// Clean up any temporaries needed by the initialization.
CodeGenFunction::OMPPrivateScope InitScope(CGF);
- InitScope.addPrivate(Elem, [SrcElement]() -> llvm::Value *{
+ InitScope.addPrivate(Elem, [SrcElement]() -> Address {
return SrcElement;
});
(void)InitScope.Privatize();
// Emit initialization for single element.
- auto *OldCapturedStmtInfo = CGF.CapturedStmtInfo;
- CGF.CapturedStmtInfo = &CapturesInfo;
+ CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
+ CGF, &CapturesInfo);
CGF.EmitAnyExprToMem(Init, DestElement,
Init->getType().getQualifiers(),
/*IsInitializer=*/false);
- CGF.CapturedStmtInfo = OldCapturedStmtInfo;
});
}
} else {
CodeGenFunction::OMPPrivateScope InitScope(CGF);
- InitScope.addPrivate(Elem, [SharedRefLValue]() -> llvm::Value *{
+ InitScope.addPrivate(Elem, [SharedRefLValue]() -> Address {
return SharedRefLValue.getAddress();
});
(void)InitScope.Privatize();
- auto *OldCapturedStmtInfo = CGF.CapturedStmtInfo;
- CGF.CapturedStmtInfo = &CapturesInfo;
+ CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
CGF.EmitExprAsInit(Init, VD, PrivateLValue,
/*capturedByInit=*/false);
- CGF.CapturedStmtInfo = OldCapturedStmtInfo;
}
} else {
CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);
@@ -2132,35 +2485,160 @@
CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
DestructorFn, KmpRoutineEntryPtrTy),
Destructor);
+
+ // Process list of dependences.
+ Address DependenciesArray = Address::invalid();
+ unsigned NumDependencies = Dependences.size();
+ if (NumDependencies) {
+ // Dependence kind for RTL.
+ enum RTLDependenceKindTy { DepIn = 0x01, DepInOut = 0x3 };
+ enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags };
+ RecordDecl *KmpDependInfoRD;
+ QualType FlagsTy =
+ C.getIntTypeForBitwidth(C.getTypeSize(C.BoolTy), /*Signed=*/false);
+ llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
+ if (KmpDependInfoTy.isNull()) {
+ KmpDependInfoRD = C.buildImplicitRecord("kmp_depend_info");
+ KmpDependInfoRD->startDefinition();
+ addFieldToRecordDecl(C, KmpDependInfoRD, C.getIntPtrType());
+ addFieldToRecordDecl(C, KmpDependInfoRD, C.getSizeType());
+ addFieldToRecordDecl(C, KmpDependInfoRD, FlagsTy);
+ KmpDependInfoRD->completeDefinition();
+ KmpDependInfoTy = C.getRecordType(KmpDependInfoRD);
+ } else {
+ KmpDependInfoRD = cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
+ }
+ CharUnits DependencySize = C.getTypeSizeInChars(KmpDependInfoTy);
+ // Define type kmp_depend_info[<Dependences.size()>];
+ QualType KmpDependInfoArrayTy = C.getConstantArrayType(
+ KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies),
+ ArrayType::Normal, /*IndexTypeQuals=*/0);
+ // kmp_depend_info[<Dependences.size()>] deps;
+ DependenciesArray = CGF.CreateMemTemp(KmpDependInfoArrayTy);
+ for (unsigned i = 0; i < NumDependencies; ++i) {
+ const Expr *E = Dependences[i].second;
+ auto Addr = CGF.EmitLValue(E);
+ llvm::Value *Size;
+ QualType Ty = E->getType();
+ if (auto *ASE = dyn_cast<OMPArraySectionExpr>(E->IgnoreParenImpCasts())) {
+ LValue UpAddrLVal =
+ CGF.EmitOMPArraySectionExpr(ASE, /*LowerBound=*/false);
+ llvm::Value *UpAddr =
+ CGF.Builder.CreateConstGEP1_32(UpAddrLVal.getPointer(), /*Idx0=*/1);
+ llvm::Value *LowIntPtr =
+ CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGM.SizeTy);
+ llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGM.SizeTy);
+ Size = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
+ } else
+ Size = getTypeSize(CGF, Ty);
+ auto Base = CGF.MakeAddrLValue(
+ CGF.Builder.CreateConstArrayGEP(DependenciesArray, i, DependencySize),
+ KmpDependInfoTy);
+ // deps[i].base_addr = &<Dependences[i].second>;
+ auto BaseAddrLVal = CGF.EmitLValueForField(
+ Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
+ CGF.EmitStoreOfScalar(
+ CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGF.IntPtrTy),
+ BaseAddrLVal);
+ // deps[i].len = sizeof(<Dependences[i].second>);
+ auto LenLVal = CGF.EmitLValueForField(
+ Base, *std::next(KmpDependInfoRD->field_begin(), Len));
+ CGF.EmitStoreOfScalar(Size, LenLVal);
+ // deps[i].flags = <Dependences[i].first>;
+ RTLDependenceKindTy DepKind;
+ switch (Dependences[i].first) {
+ case OMPC_DEPEND_in:
+ DepKind = DepIn;
+ break;
+ // Out and InOut dependencies must use the same code.
+ case OMPC_DEPEND_out:
+ case OMPC_DEPEND_inout:
+ DepKind = DepInOut;
+ break;
+ case OMPC_DEPEND_source:
+ case OMPC_DEPEND_unknown:
+ llvm_unreachable("Unknown task dependence type");
+ }
+ auto FlagsLVal = CGF.EmitLValueForField(
+ Base, *std::next(KmpDependInfoRD->field_begin(), Flags));
+ CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind),
+ FlagsLVal);
+ }
+ DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ CGF.Builder.CreateStructGEP(DependenciesArray, 0, CharUnits::Zero()),
+ CGF.VoidPtrTy);
+ }
+
// NOTE: routine and part_id fields are intialized by __kmpc_omp_task_alloc()
// libcall.
// Build kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
// *new_task);
+ // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
+ // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
+ // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence
+ // list is not empty
auto *ThreadID = getThreadID(CGF, Loc);
- llvm::Value *TaskArgs[] = {emitUpdateLocation(CGF, Loc), ThreadID, NewTask};
- auto &&ThenCodeGen = [this, &TaskArgs](CodeGenFunction &CGF) {
- // TODO: add check for untied tasks.
- CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task), TaskArgs);
+ auto *UpLoc = emitUpdateLocation(CGF, Loc);
+ llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
+ llvm::Value *DepTaskArgs[7];
+ if (NumDependencies) {
+ DepTaskArgs[0] = UpLoc;
+ DepTaskArgs[1] = ThreadID;
+ DepTaskArgs[2] = NewTask;
+ DepTaskArgs[3] = CGF.Builder.getInt32(NumDependencies);
+ DepTaskArgs[4] = DependenciesArray.getPointer();
+ DepTaskArgs[5] = CGF.Builder.getInt32(0);
+ DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+ }
+ auto &&ThenCodeGen = [this, NumDependencies,
+ &TaskArgs, &DepTaskArgs](CodeGenFunction &CGF) {
+ // TODO: add check for untied tasks.
+ if (NumDependencies) {
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task_with_deps),
+ DepTaskArgs);
+ } else {
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task),
+ TaskArgs);
+ }
};
typedef CallEndCleanup<std::extent<decltype(TaskArgs)>::value>
IfCallEndCleanup;
- auto &&ElseCodeGen =
- [this, &TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry](
- CodeGenFunction &CGF) {
- CodeGenFunction::RunCleanupsScope LocalScope(CGF);
- CGF.EmitRuntimeCall(
- createRuntimeFunction(OMPRTL__kmpc_omp_task_begin_if0), TaskArgs);
- // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
- // kmp_task_t *new_task);
- CGF.EHStack.pushCleanup<IfCallEndCleanup>(
- NormalAndEHCleanup,
- createRuntimeFunction(OMPRTL__kmpc_omp_task_complete_if0),
- llvm::makeArrayRef(TaskArgs));
- // Call proxy_task_entry(gtid, new_task);
- llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
- CGF.EmitCallOrInvoke(TaskEntry, OutlinedFnArgs);
- };
+ llvm::Value *DepWaitTaskArgs[6];
+ if (NumDependencies) {
+ DepWaitTaskArgs[0] = UpLoc;
+ DepWaitTaskArgs[1] = ThreadID;
+ DepWaitTaskArgs[2] = CGF.Builder.getInt32(NumDependencies);
+ DepWaitTaskArgs[3] = DependenciesArray.getPointer();
+ DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
+ DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+ }
+ auto &&ElseCodeGen = [this, &TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry,
+ NumDependencies, &DepWaitTaskArgs](CodeGenFunction &CGF) {
+ CodeGenFunction::RunCleanupsScope LocalScope(CGF);
+ // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
+ // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
+ // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
+ // is specified.
+ if (NumDependencies)
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_wait_deps),
+ DepWaitTaskArgs);
+ // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
+ // kmp_task_t *new_task);
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task_begin_if0),
+ TaskArgs);
+ // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
+ // kmp_task_t *new_task);
+ CGF.EHStack.pushCleanup<IfCallEndCleanup>(
+ NormalAndEHCleanup,
+ createRuntimeFunction(OMPRTL__kmpc_omp_task_complete_if0),
+ llvm::makeArrayRef(TaskArgs));
+
+ // Call proxy_task_entry(gtid, new_task);
+ llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
+ CGF.EmitCallOrInvoke(TaskEntry, OutlinedFnArgs);
+ };
+
if (IfCond) {
emitOMPIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
} else {
@@ -2169,8 +2647,89 @@
}
}
+/// \brief Emit reduction operation for each element of array (required for
+/// array sections) LHS op = RHS.
+/// \param Type Type of array.
+/// \param LHSVar Variable on the left side of the reduction operation
+/// (references element of array in original variable).
+/// \param RHSVar Variable on the right side of the reduction operation
+/// (references element of array in original variable).
+/// \param RedOpGen Generator of reduction operation with use of LHSVar and
+/// RHSVar.
+static void EmitOMPAggregateReduction(
+ CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,
+ const VarDecl *RHSVar,
+ const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,
+ const Expr *, const Expr *)> &RedOpGen,
+ const Expr *XExpr = nullptr, const Expr *EExpr = nullptr,
+ const Expr *UpExpr = nullptr) {
+ // Perform element-by-element initialization.
+ QualType ElementTy;
+ Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar);
+ Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar);
+
+ // Drill down to the base element type on both arrays.
+ auto ArrayTy = Type->getAsArrayTypeUnsafe();
+ auto NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr);
+
+ auto RHSBegin = RHSAddr.getPointer();
+ auto LHSBegin = LHSAddr.getPointer();
+ // Cast from pointer to array type to pointer to single element.
+ auto LHSEnd = CGF.Builder.CreateGEP(LHSBegin, NumElements);
+ // The basic structure here is a while-do loop.
+ auto BodyBB = CGF.createBasicBlock("omp.arraycpy.body");
+ auto DoneBB = CGF.createBasicBlock("omp.arraycpy.done");
+ auto IsEmpty =
+ CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty");
+ CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
+
+ // Enter the loop body, making that address the current address.
+ auto EntryBB = CGF.Builder.GetInsertBlock();
+ CGF.EmitBlock(BodyBB);
+
+ CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
+
+ llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
+ RHSBegin->getType(), 2, "omp.arraycpy.srcElementPast");
+ RHSElementPHI->addIncoming(RHSBegin, EntryBB);
+ Address RHSElementCurrent =
+ Address(RHSElementPHI,
+ RHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
+
+ llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
+ LHSBegin->getType(), 2, "omp.arraycpy.destElementPast");
+ LHSElementPHI->addIncoming(LHSBegin, EntryBB);
+ Address LHSElementCurrent =
+ Address(LHSElementPHI,
+ LHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
+
+ // Emit copy.
+ CodeGenFunction::OMPPrivateScope Scope(CGF);
+ Scope.addPrivate(LHSVar, [=]() -> Address { return LHSElementCurrent; });
+ Scope.addPrivate(RHSVar, [=]() -> Address { return RHSElementCurrent; });
+ Scope.Privatize();
+ RedOpGen(CGF, XExpr, EExpr, UpExpr);
+ Scope.ForceCleanup();
+
+ // Shift the address forward by one element.
+ auto LHSElementNext = CGF.Builder.CreateConstGEP1_32(
+ LHSElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
+ auto RHSElementNext = CGF.Builder.CreateConstGEP1_32(
+ RHSElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
+ // Check whether we've reached the end.
+ auto Done =
+ CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done");
+ CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
+ LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock());
+ RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock());
+
+ // Done.
+ CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
+}
+
static llvm::Value *emitReductionFunction(CodeGenModule &CGM,
llvm::Type *ArgsType,
+ ArrayRef<const Expr *> Privates,
ArrayRef<const Expr *> LHSExprs,
ArrayRef<const Expr *> RHSExprs,
ArrayRef<const Expr *> ReductionOps) {
@@ -2190,48 +2749,66 @@
auto *Fn = llvm::Function::Create(
CGM.getTypes().GetFunctionType(CGFI), llvm::GlobalValue::InternalLinkage,
".omp.reduction.reduction_func", &CGM.getModule());
- CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, CGFI, Fn);
+ CGM.SetInternalFunctionAttributes(/*D=*/nullptr, Fn, CGFI);
CodeGenFunction CGF(CGM);
CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args);
// Dst = (void*[n])(LHSArg);
// Src = (void*[n])(RHSArg);
- auto *LHS = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- CGF.Builder.CreateAlignedLoad(CGF.GetAddrOfLocalVar(&LHSArg),
- CGF.PointerAlignInBytes),
- ArgsType);
- auto *RHS = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- CGF.Builder.CreateAlignedLoad(CGF.GetAddrOfLocalVar(&RHSArg),
- CGF.PointerAlignInBytes),
- ArgsType);
+ Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
+ ArgsType), CGF.getPointerAlign());
+ Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
+ ArgsType), CGF.getPointerAlign());
// ...
// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
// ...
CodeGenFunction::OMPPrivateScope Scope(CGF);
- for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I) {
- Scope.addPrivate(
- cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl()),
- [&]() -> llvm::Value *{
- return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- CGF.Builder.CreateAlignedLoad(
- CGF.Builder.CreateStructGEP(/*Ty=*/nullptr, RHS, I),
- CGM.PointerAlignInBytes),
- CGF.ConvertTypeForMem(C.getPointerType(RHSExprs[I]->getType())));
- });
- Scope.addPrivate(
- cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl()),
- [&]() -> llvm::Value *{
- return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- CGF.Builder.CreateAlignedLoad(
- CGF.Builder.CreateStructGEP(/*Ty=*/nullptr, LHS, I),
- CGM.PointerAlignInBytes),
- CGF.ConvertTypeForMem(C.getPointerType(LHSExprs[I]->getType())));
- });
+ auto IPriv = Privates.begin();
+ unsigned Idx = 0;
+ for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
+ auto RHSVar = cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl());
+ Scope.addPrivate(RHSVar, [&]() -> Address {
+ return emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar);
+ });
+ auto LHSVar = cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl());
+ Scope.addPrivate(LHSVar, [&]() -> Address {
+ return emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar);
+ });
+ QualType PrivTy = (*IPriv)->getType();
+ if (PrivTy->isArrayType()) {
+ // Get array size and emit VLA type.
+ ++Idx;
+ Address Elem =
+ CGF.Builder.CreateConstArrayGEP(LHS, Idx, CGF.getPointerSize());
+ llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem);
+ CodeGenFunction::OpaqueValueMapping OpaqueMap(
+ CGF,
+ cast<OpaqueValueExpr>(
+ CGF.getContext().getAsVariableArrayType(PrivTy)->getSizeExpr()),
+ RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy)));
+ CGF.EmitVariablyModifiedType(PrivTy);
+ }
}
Scope.Privatize();
+ IPriv = Privates.begin();
+ auto ILHS = LHSExprs.begin();
+ auto IRHS = RHSExprs.begin();
for (auto *E : ReductionOps) {
- CGF.EmitIgnoredExpr(E);
+ if ((*IPriv)->getType()->isArrayType()) {
+ // Emit reduction for array section.
+ auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
+ auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
+ EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar,
+ [=](CodeGenFunction &CGF, const Expr *,
+ const Expr *,
+ const Expr *) { CGF.EmitIgnoredExpr(E); });
+ } else
+ // Emit reduction for array subscript or single variable.
+ CGF.EmitIgnoredExpr(E);
+ ++IPriv, ++ILHS, ++IRHS;
}
Scope.ForceCleanup();
CGF.FinishFunction();
@@ -2239,10 +2816,13 @@
}
void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
+ ArrayRef<const Expr *> Privates,
ArrayRef<const Expr *> LHSExprs,
ArrayRef<const Expr *> RHSExprs,
ArrayRef<const Expr *> ReductionOps,
- bool WithNowait) {
+ bool WithNowait, bool SimpleReduction) {
+ if (!CGF.HaveInsertPoint())
+ return;
// Next code should be emitted for reduction:
//
// static kmp_critical_name lock = { 0 };
@@ -2272,29 +2852,78 @@
// break;
// default:;
// }
+ //
+ // if SimpleReduction is true, only the next code is generated:
+ // ...
+ // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
+ // ...
auto &C = CGM.getContext();
+ if (SimpleReduction) {
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
+ auto IPriv = Privates.begin();
+ auto ILHS = LHSExprs.begin();
+ auto IRHS = RHSExprs.begin();
+ for (auto *E : ReductionOps) {
+ if ((*IPriv)->getType()->isArrayType()) {
+ auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
+ auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
+ EmitOMPAggregateReduction(
+ CGF, (*IPriv)->getType(), LHSVar, RHSVar,
+ [=](CodeGenFunction &CGF, const Expr *, const Expr *,
+ const Expr *) { CGF.EmitIgnoredExpr(E); });
+ } else
+ CGF.EmitIgnoredExpr(E);
+ ++IPriv, ++ILHS, ++IRHS;
+ }
+ return;
+ }
+
// 1. Build a list of reduction variables.
// void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
- llvm::APInt ArraySize(/*unsigned int numBits=*/32, RHSExprs.size());
+ auto Size = RHSExprs.size();
+ for (auto *E : Privates) {
+ if (E->getType()->isArrayType())
+ // Reserve place for array size.
+ ++Size;
+ }
+ llvm::APInt ArraySize(/*unsigned int numBits=*/32, Size);
QualType ReductionArrayTy =
C.getConstantArrayType(C.VoidPtrTy, ArraySize, ArrayType::Normal,
/*IndexTypeQuals=*/0);
- auto *ReductionList =
+ Address ReductionList =
CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
- for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I) {
- auto *Elem = CGF.Builder.CreateStructGEP(/*Ty=*/nullptr, ReductionList, I);
- CGF.Builder.CreateAlignedStore(
+ auto IPriv = Privates.begin();
+ unsigned Idx = 0;
+ for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
+ Address Elem =
+ CGF.Builder.CreateConstArrayGEP(ReductionList, Idx, CGF.getPointerSize());
+ CGF.Builder.CreateStore(
CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- CGF.EmitLValue(RHSExprs[I]).getAddress(), CGF.VoidPtrTy),
- Elem, CGM.PointerAlignInBytes);
+ CGF.EmitLValue(RHSExprs[I]).getPointer(), CGF.VoidPtrTy),
+ Elem);
+ if ((*IPriv)->getType()->isArrayType()) {
+ // Store array size.
+ ++Idx;
+ Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx,
+ CGF.getPointerSize());
+ CGF.Builder.CreateStore(
+ CGF.Builder.CreateIntToPtr(
+ CGF.Builder.CreateIntCast(
+ CGF.getVLASize(CGF.getContext().getAsVariableArrayType(
+ (*IPriv)->getType()))
+ .first,
+ CGF.SizeTy, /*isSigned=*/false),
+ CGF.VoidPtrTy),
+ Elem);
+ }
}
// 2. Emit reduce_func().
auto *ReductionFn = emitReductionFunction(
- CGM, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), LHSExprs,
- RHSExprs, ReductionOps);
+ CGM, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), Privates,
+ LHSExprs, RHSExprs, ReductionOps);
// 3. Create static kmp_critical_name lock = { 0 };
auto *Lock = getCriticalRegionLock(".reduction");
@@ -2305,10 +2934,10 @@
CGF, Loc,
static_cast<OpenMPLocationFlags>(OMP_IDENT_KMPC | OMP_ATOMIC_REDUCE));
auto *ThreadId = getThreadID(CGF, Loc);
- auto *ReductionArrayTySize = llvm::ConstantInt::get(
- CGM.SizeTy, C.getTypeSizeInChars(ReductionArrayTy).getQuantity());
- auto *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(ReductionList,
- CGF.VoidPtrTy);
+ auto *ReductionArrayTySize = getTypeSize(CGF, ReductionArrayTy);
+ auto *RL =
+ CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(ReductionList.getPointer(),
+ CGF.VoidPtrTy);
llvm::Value *Args[] = {
IdentTLoc, // ident_t *<loc>
ThreadId, // i32 <gtid>
@@ -2351,8 +2980,22 @@
createRuntimeFunction(WithNowait ? OMPRTL__kmpc_end_reduce_nowait
: OMPRTL__kmpc_end_reduce),
llvm::makeArrayRef(EndArgs));
+ auto IPriv = Privates.begin();
+ auto ILHS = LHSExprs.begin();
+ auto IRHS = RHSExprs.begin();
for (auto *E : ReductionOps) {
- CGF.EmitIgnoredExpr(E);
+ if ((*IPriv)->getType()->isArrayType()) {
+ // Emit reduction for array section.
+ auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
+ auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
+ EmitOMPAggregateReduction(
+ CGF, (*IPriv)->getType(), LHSVar, RHSVar,
+ [=](CodeGenFunction &CGF, const Expr *, const Expr *,
+ const Expr *) { CGF.EmitIgnoredExpr(E); });
+ } else
+ // Emit reduction for array subscript or single variable.
+ CGF.EmitIgnoredExpr(E);
+ ++IPriv, ++ILHS, ++IRHS;
}
}
@@ -2382,62 +3025,84 @@
createRuntimeFunction(OMPRTL__kmpc_end_reduce),
llvm::makeArrayRef(EndArgs));
}
- auto I = LHSExprs.begin();
+ auto ILHS = LHSExprs.begin();
+ auto IRHS = RHSExprs.begin();
+ auto IPriv = Privates.begin();
for (auto *E : ReductionOps) {
- const Expr *XExpr = nullptr;
- const Expr *EExpr = nullptr;
- const Expr *UpExpr = nullptr;
- BinaryOperatorKind BO = BO_Comma;
- if (auto *BO = dyn_cast<BinaryOperator>(E)) {
- if (BO->getOpcode() == BO_Assign) {
- XExpr = BO->getLHS();
- UpExpr = BO->getRHS();
+ const Expr *XExpr = nullptr;
+ const Expr *EExpr = nullptr;
+ const Expr *UpExpr = nullptr;
+ BinaryOperatorKind BO = BO_Comma;
+ if (auto *BO = dyn_cast<BinaryOperator>(E)) {
+ if (BO->getOpcode() == BO_Assign) {
+ XExpr = BO->getLHS();
+ UpExpr = BO->getRHS();
+ }
}
- }
- // Try to emit update expression as a simple atomic.
- auto *RHSExpr = UpExpr;
- if (RHSExpr) {
- // Analyze RHS part of the whole expression.
- if (auto *ACO = dyn_cast<AbstractConditionalOperator>(
- RHSExpr->IgnoreParenImpCasts())) {
- // If this is a conditional operator, analyze its condition for
- // min/max reduction operator.
- RHSExpr = ACO->getCond();
+ // Try to emit update expression as a simple atomic.
+ auto *RHSExpr = UpExpr;
+ if (RHSExpr) {
+ // Analyze RHS part of the whole expression.
+ if (auto *ACO = dyn_cast<AbstractConditionalOperator>(
+ RHSExpr->IgnoreParenImpCasts())) {
+ // If this is a conditional operator, analyze its condition for
+ // min/max reduction operator.
+ RHSExpr = ACO->getCond();
+ }
+ if (auto *BORHS =
+ dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) {
+ EExpr = BORHS->getRHS();
+ BO = BORHS->getOpcode();
+ }
}
- if (auto *BORHS =
- dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) {
- EExpr = BORHS->getRHS();
- BO = BORHS->getOpcode();
- }
- }
- if (XExpr) {
- auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
- LValue X = CGF.EmitLValue(XExpr);
- RValue E;
- if (EExpr)
- E = CGF.EmitAnyExpr(EExpr);
- CGF.EmitOMPAtomicSimpleUpdateExpr(
- X, E, BO, /*IsXLHSInRHSPart=*/true, llvm::Monotonic, Loc,
- [&CGF, UpExpr, VD](RValue XRValue) {
- CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
- PrivateScope.addPrivate(
- VD, [&CGF, VD, XRValue]() -> llvm::Value *{
- auto *LHSTemp = CGF.CreateMemTemp(VD->getType());
+ if (XExpr) {
+ auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
+ auto &&AtomicRedGen = [this, BO, VD, IPriv,
+ Loc](CodeGenFunction &CGF, const Expr *XExpr,
+ const Expr *EExpr, const Expr *UpExpr) {
+ LValue X = CGF.EmitLValue(XExpr);
+ RValue E;
+ if (EExpr)
+ E = CGF.EmitAnyExpr(EExpr);
+ CGF.EmitOMPAtomicSimpleUpdateExpr(
+ X, E, BO, /*IsXLHSInRHSPart=*/true, llvm::Monotonic, Loc,
+ [&CGF, UpExpr, VD, IPriv](RValue XRValue) {
+ CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
+ PrivateScope.addPrivate(VD, [&CGF, VD, XRValue]() -> Address {
+ Address LHSTemp = CGF.CreateMemTemp(VD->getType());
CGF.EmitStoreThroughLValue(
- XRValue,
- CGF.MakeNaturalAlignAddrLValue(LHSTemp, VD->getType()));
+ XRValue, CGF.MakeAddrLValue(LHSTemp, VD->getType()));
return LHSTemp;
});
- (void)PrivateScope.Privatize();
- return CGF.EmitAnyExpr(UpExpr);
- });
- } else {
- // Emit as a critical region.
- emitCriticalRegion(CGF, ".atomic_reduction", [E](CodeGenFunction &CGF) {
- CGF.EmitIgnoredExpr(E);
- }, Loc);
- }
- ++I;
+ (void)PrivateScope.Privatize();
+ return CGF.EmitAnyExpr(UpExpr);
+ });
+ };
+ if ((*IPriv)->getType()->isArrayType()) {
+ // Emit atomic reduction for array section.
+ auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
+ EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar,
+ AtomicRedGen, XExpr, EExpr, UpExpr);
+ } else
+ // Emit atomic reduction for array subscript or single variable.
+ AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
+ } else {
+ // Emit as a critical region.
+ auto &&CritRedGen = [this, E, Loc](CodeGenFunction &CGF, const Expr *,
+ const Expr *, const Expr *) {
+ emitCriticalRegion(
+ CGF, ".atomic_reduction",
+ [E](CodeGenFunction &CGF) { CGF.EmitIgnoredExpr(E); }, Loc);
+ };
+ if ((*IPriv)->getType()->isArrayType()) {
+ auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
+ auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
+ EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar,
+ CritRedGen);
+ } else
+ CritRedGen(CGF, nullptr, nullptr, nullptr);
+ }
+ ++ILHS, ++IRHS, ++IPriv;
}
}
@@ -2447,6 +3112,8 @@
void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF,
SourceLocation Loc) {
+ if (!CGF.HaveInsertPoint())
+ return;
// Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
// global_tid);
llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
@@ -2455,8 +3122,433 @@
}
void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF,
- const RegionCodeGenTy &CodeGen) {
- InlinedOpenMPRegionRAII Region(CGF, CodeGen);
+ OpenMPDirectiveKind InnerKind,
+ const RegionCodeGenTy &CodeGen,
+ bool HasCancel) {
+ if (!CGF.HaveInsertPoint())
+ return;
+ InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel);
CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr);
}
+namespace {
+enum RTCancelKind {
+ CancelNoreq = 0,
+ CancelParallel = 1,
+ CancelLoop = 2,
+ CancelSections = 3,
+ CancelTaskgroup = 4
+};
+}
+
+static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion) {
+ RTCancelKind CancelKind = CancelNoreq;
+ if (CancelRegion == OMPD_parallel)
+ CancelKind = CancelParallel;
+ else if (CancelRegion == OMPD_for)
+ CancelKind = CancelLoop;
+ else if (CancelRegion == OMPD_sections)
+ CancelKind = CancelSections;
+ else {
+ assert(CancelRegion == OMPD_taskgroup);
+ CancelKind = CancelTaskgroup;
+ }
+ return CancelKind;
+}
+
+void CGOpenMPRuntime::emitCancellationPointCall(
+ CodeGenFunction &CGF, SourceLocation Loc,
+ OpenMPDirectiveKind CancelRegion) {
+ if (!CGF.HaveInsertPoint())
+ return;
+ // Build call kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
+ // global_tid, kmp_int32 cncl_kind);
+ if (auto *OMPRegionInfo =
+ dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
+ if (OMPRegionInfo->getDirectiveKind() == OMPD_single)
+ return;
+ if (OMPRegionInfo->hasCancel()) {
+ llvm::Value *Args[] = {
+ emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
+ CGF.Builder.getInt32(getCancellationKind(CancelRegion))};
+ // Ignore return result until untied tasks are supported.
+ auto *Result = CGF.EmitRuntimeCall(
+ createRuntimeFunction(OMPRTL__kmpc_cancellationpoint), Args);
+ // if (__kmpc_cancellationpoint()) {
+ // __kmpc_cancel_barrier();
+ // exit from construct;
+ // }
+ auto *ExitBB = CGF.createBasicBlock(".cancel.exit");
+ auto *ContBB = CGF.createBasicBlock(".cancel.continue");
+ auto *Cmp = CGF.Builder.CreateIsNotNull(Result);
+ CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
+ CGF.EmitBlock(ExitBB);
+ // __kmpc_cancel_barrier();
+ emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false);
+ // exit from construct;
+ auto CancelDest =
+ CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
+ CGF.EmitBranchThroughCleanup(CancelDest);
+ CGF.EmitBlock(ContBB, /*IsFinished=*/true);
+ }
+ }
+}
+
+void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
+ const Expr *IfCond,
+ OpenMPDirectiveKind CancelRegion) {
+ if (!CGF.HaveInsertPoint())
+ return;
+ // Build call kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
+ // kmp_int32 cncl_kind);
+ if (auto *OMPRegionInfo =
+ dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
+ if (OMPRegionInfo->getDirectiveKind() == OMPD_single)
+ return;
+ auto &&ThenGen = [this, Loc, CancelRegion,
+ OMPRegionInfo](CodeGenFunction &CGF) {
+ llvm::Value *Args[] = {
+ emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
+ CGF.Builder.getInt32(getCancellationKind(CancelRegion))};
+ // Ignore return result until untied tasks are supported.
+ auto *Result =
+ CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_cancel), Args);
+ // if (__kmpc_cancel()) {
+ // __kmpc_cancel_barrier();
+ // exit from construct;
+ // }
+ auto *ExitBB = CGF.createBasicBlock(".cancel.exit");
+ auto *ContBB = CGF.createBasicBlock(".cancel.continue");
+ auto *Cmp = CGF.Builder.CreateIsNotNull(Result);
+ CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
+ CGF.EmitBlock(ExitBB);
+ // __kmpc_cancel_barrier();
+ emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false);
+ // exit from construct;
+ auto CancelDest =
+ CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
+ CGF.EmitBranchThroughCleanup(CancelDest);
+ CGF.EmitBlock(ContBB, /*IsFinished=*/true);
+ };
+ if (IfCond)
+ emitOMPIfClause(CGF, IfCond, ThenGen, [](CodeGenFunction &) {});
+ else
+ ThenGen(CGF);
+ }
+}
+
+llvm::Value *
+CGOpenMPRuntime::emitTargetOutlinedFunction(const OMPExecutableDirective &D,
+ const RegionCodeGenTy &CodeGen) {
+ const CapturedStmt &CS = *cast<CapturedStmt>(D.getAssociatedStmt());
+
+ CodeGenFunction CGF(CGM, true);
+ CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen);
+ CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
+ return CGF.GenerateOpenMPCapturedStmtFunction(CS);
+}
+
+void CGOpenMPRuntime::emitTargetCall(CodeGenFunction &CGF,
+ const OMPExecutableDirective &D,
+ llvm::Value *OutlinedFn,
+ const Expr *IfCond, const Expr *Device,
+ ArrayRef<llvm::Value *> CapturedVars) {
+ if (!CGF.HaveInsertPoint())
+ return;
+ /// \brief Values for bit flags used to specify the mapping type for
+ /// offloading.
+ enum OpenMPOffloadMappingFlags {
+ /// \brief Allocate memory on the device and move data from host to device.
+ OMP_MAP_TO = 0x01,
+ /// \brief Allocate memory on the device and move data from device to host.
+ OMP_MAP_FROM = 0x02,
+ /// \brief The element passed to the device is a pointer.
+ OMP_MAP_PTR = 0x20,
+ /// \brief Pass the element to the device by value.
+ OMP_MAP_BYCOPY = 0x80,
+ };
+
+ enum OpenMPOffloadingReservedDeviceIDs {
+ /// \brief Device ID if the device was not defined, runtime should get it
+ /// from environment variables in the spec.
+ OMP_DEVICEID_UNDEF = -1,
+ };
+
+ auto &Ctx = CGF.getContext();
+
+ // Fill up the arrays with the all the captured variables.
+ SmallVector<llvm::Value *, 16> BasePointers;
+ SmallVector<llvm::Value *, 16> Pointers;
+ SmallVector<llvm::Value *, 16> Sizes;
+ SmallVector<unsigned, 16> MapTypes;
+
+ bool hasVLACaptures = false;
+
+ const CapturedStmt &CS = *cast<CapturedStmt>(D.getAssociatedStmt());
+ auto RI = CS.getCapturedRecordDecl()->field_begin();
+ // auto II = CS.capture_init_begin();
+ auto CV = CapturedVars.begin();
+ for (CapturedStmt::const_capture_iterator CI = CS.capture_begin(),
+ CE = CS.capture_end();
+ CI != CE; ++CI, ++RI, ++CV) {
+ StringRef Name;
+ QualType Ty;
+ llvm::Value *BasePointer;
+ llvm::Value *Pointer;
+ llvm::Value *Size;
+ unsigned MapType;
+
+ // VLA sizes are passed to the outlined region by copy.
+ if (CI->capturesVariableArrayType()) {
+ BasePointer = Pointer = *CV;
+ Size = getTypeSize(CGF, RI->getType());
+ // Copy to the device as an argument. No need to retrieve it.
+ MapType = OMP_MAP_BYCOPY;
+ hasVLACaptures = true;
+ } else if (CI->capturesThis()) {
+ BasePointer = Pointer = *CV;
+ const PointerType *PtrTy = cast<PointerType>(RI->getType().getTypePtr());
+ Size = getTypeSize(CGF, PtrTy->getPointeeType());
+ // Default map type.
+ MapType = OMP_MAP_TO | OMP_MAP_FROM;
+ } else if (CI->capturesVariableByCopy()) {
+ MapType = OMP_MAP_BYCOPY;
+ if (!RI->getType()->isAnyPointerType()) {
+ // If the field is not a pointer, we need to save the actual value and
+ // load it as a void pointer.
+ auto DstAddr = CGF.CreateMemTemp(
+ Ctx.getUIntPtrType(),
+ Twine(CI->getCapturedVar()->getName()) + ".casted");
+ LValue DstLV = CGF.MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
+
+ auto *SrcAddrVal = CGF.EmitScalarConversion(
+ DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
+ Ctx.getPointerType(RI->getType()), SourceLocation());
+ LValue SrcLV =
+ CGF.MakeNaturalAlignAddrLValue(SrcAddrVal, RI->getType());
+
+ // Store the value using the source type pointer.
+ CGF.EmitStoreThroughLValue(RValue::get(*CV), SrcLV);
+
+ // Load the value using the destination type pointer.
+ BasePointer = Pointer =
+ CGF.EmitLoadOfLValue(DstLV, SourceLocation()).getScalarVal();
+ } else {
+ MapType |= OMP_MAP_PTR;
+ BasePointer = Pointer = *CV;
+ }
+ Size = getTypeSize(CGF, RI->getType());
+ } else {
+ assert(CI->capturesVariable() && "Expected captured reference.");
+ BasePointer = Pointer = *CV;
+
+ const ReferenceType *PtrTy =
+ cast<ReferenceType>(RI->getType().getTypePtr());
+ QualType ElementType = PtrTy->getPointeeType();
+ Size = getTypeSize(CGF, ElementType);
+ // The default map type for a scalar/complex type is 'to' because by
+ // default the value doesn't have to be retrieved. For an aggregate type,
+ // the default is 'tofrom'.
+ MapType = ElementType->isAggregateType() ? (OMP_MAP_TO | OMP_MAP_FROM)
+ : OMP_MAP_TO;
+ if (ElementType->isAnyPointerType())
+ MapType |= OMP_MAP_PTR;
+ }
+
+ BasePointers.push_back(BasePointer);
+ Pointers.push_back(Pointer);
+ Sizes.push_back(Size);
+ MapTypes.push_back(MapType);
+ }
+
+ // Keep track on whether the host function has to be executed.
+ auto OffloadErrorQType =
+ Ctx.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true);
+ auto OffloadError = CGF.MakeAddrLValue(
+ CGF.CreateMemTemp(OffloadErrorQType, ".run_host_version"),
+ OffloadErrorQType);
+ CGF.EmitStoreOfScalar(llvm::Constant::getNullValue(CGM.Int32Ty),
+ OffloadError);
+
+ // Fill up the pointer arrays and transfer execution to the device.
+ auto &&ThenGen = [this, &Ctx, &BasePointers, &Pointers, &Sizes, &MapTypes,
+ hasVLACaptures, Device, OffloadError,
+ OffloadErrorQType](CodeGenFunction &CGF) {
+ unsigned PointerNumVal = BasePointers.size();
+ llvm::Value *PointerNum = CGF.Builder.getInt32(PointerNumVal);
+ llvm::Value *BasePointersArray;
+ llvm::Value *PointersArray;
+ llvm::Value *SizesArray;
+ llvm::Value *MapTypesArray;
+
+ if (PointerNumVal) {
+ llvm::APInt PointerNumAP(32, PointerNumVal, /*isSigned=*/true);
+ QualType PointerArrayType = Ctx.getConstantArrayType(
+ Ctx.VoidPtrTy, PointerNumAP, ArrayType::Normal,
+ /*IndexTypeQuals=*/0);
+
+ BasePointersArray =
+ CGF.CreateMemTemp(PointerArrayType, ".offload_baseptrs").getPointer();
+ PointersArray =
+ CGF.CreateMemTemp(PointerArrayType, ".offload_ptrs").getPointer();
+
+ // If we don't have any VLA types, we can use a constant array for the map
+ // sizes, otherwise we need to fill up the arrays as we do for the
+ // pointers.
+ if (hasVLACaptures) {
+ QualType SizeArrayType = Ctx.getConstantArrayType(
+ Ctx.getSizeType(), PointerNumAP, ArrayType::Normal,
+ /*IndexTypeQuals=*/0);
+ SizesArray =
+ CGF.CreateMemTemp(SizeArrayType, ".offload_sizes").getPointer();
+ } else {
+ // We expect all the sizes to be constant, so we collect them to create
+ // a constant array.
+ SmallVector<llvm::Constant *, 16> ConstSizes;
+ for (auto S : Sizes)
+ ConstSizes.push_back(cast<llvm::Constant>(S));
+
+ auto *SizesArrayInit = llvm::ConstantArray::get(
+ llvm::ArrayType::get(CGM.SizeTy, ConstSizes.size()), ConstSizes);
+ auto *SizesArrayGbl = new llvm::GlobalVariable(
+ CGM.getModule(), SizesArrayInit->getType(),
+ /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage,
+ SizesArrayInit, ".offload_sizes");
+ SizesArrayGbl->setUnnamedAddr(true);
+ SizesArray = SizesArrayGbl;
+ }
+
+ // The map types are always constant so we don't need to generate code to
+ // fill arrays. Instead, we create an array constant.
+ llvm::Constant *MapTypesArrayInit =
+ llvm::ConstantDataArray::get(CGF.Builder.getContext(), MapTypes);
+ auto *MapTypesArrayGbl = new llvm::GlobalVariable(
+ CGM.getModule(), MapTypesArrayInit->getType(),
+ /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage,
+ MapTypesArrayInit, ".offload_maptypes");
+ MapTypesArrayGbl->setUnnamedAddr(true);
+ MapTypesArray = MapTypesArrayGbl;
+
+ for (unsigned i = 0; i < PointerNumVal; ++i) {
+
+ llvm::Value *BPVal = BasePointers[i];
+ if (BPVal->getType()->isPointerTy())
+ BPVal = CGF.Builder.CreateBitCast(BPVal, CGM.VoidPtrTy);
+ else {
+ assert(BPVal->getType()->isIntegerTy() &&
+ "If not a pointer, the value type must be an integer.");
+ BPVal = CGF.Builder.CreateIntToPtr(BPVal, CGM.VoidPtrTy);
+ }
+ llvm::Value *BP = CGF.Builder.CreateConstInBoundsGEP2_32(
+ llvm::ArrayType::get(CGM.VoidPtrTy, PointerNumVal),
+ BasePointersArray, 0, i);
+ Address BPAddr(BP, Ctx.getTypeAlignInChars(Ctx.VoidPtrTy));
+ CGF.Builder.CreateStore(BPVal, BPAddr);
+
+ llvm::Value *PVal = Pointers[i];
+ if (PVal->getType()->isPointerTy())
+ PVal = CGF.Builder.CreateBitCast(PVal, CGM.VoidPtrTy);
+ else {
+ assert(PVal->getType()->isIntegerTy() &&
+ "If not a pointer, the value type must be an integer.");
+ PVal = CGF.Builder.CreateIntToPtr(PVal, CGM.VoidPtrTy);
+ }
+ llvm::Value *P = CGF.Builder.CreateConstInBoundsGEP2_32(
+ llvm::ArrayType::get(CGM.VoidPtrTy, PointerNumVal), PointersArray,
+ 0, i);
+ Address PAddr(P, Ctx.getTypeAlignInChars(Ctx.VoidPtrTy));
+ CGF.Builder.CreateStore(PVal, PAddr);
+
+ if (hasVLACaptures) {
+ llvm::Value *S = CGF.Builder.CreateConstInBoundsGEP2_32(
+ llvm::ArrayType::get(CGM.SizeTy, PointerNumVal), SizesArray,
+ /*Idx0=*/0,
+ /*Idx1=*/i);
+ Address SAddr(S, Ctx.getTypeAlignInChars(Ctx.getSizeType()));
+ CGF.Builder.CreateStore(CGF.Builder.CreateIntCast(
+ Sizes[i], CGM.SizeTy, /*isSigned=*/true),
+ SAddr);
+ }
+ }
+
+ BasePointersArray = CGF.Builder.CreateConstInBoundsGEP2_32(
+ llvm::ArrayType::get(CGM.VoidPtrTy, PointerNumVal), BasePointersArray,
+ /*Idx0=*/0, /*Idx1=*/0);
+ PointersArray = CGF.Builder.CreateConstInBoundsGEP2_32(
+ llvm::ArrayType::get(CGM.VoidPtrTy, PointerNumVal), PointersArray,
+ /*Idx0=*/0,
+ /*Idx1=*/0);
+ SizesArray = CGF.Builder.CreateConstInBoundsGEP2_32(
+ llvm::ArrayType::get(CGM.SizeTy, PointerNumVal), SizesArray,
+ /*Idx0=*/0, /*Idx1=*/0);
+ MapTypesArray = CGF.Builder.CreateConstInBoundsGEP2_32(
+ llvm::ArrayType::get(CGM.Int32Ty, PointerNumVal), MapTypesArray,
+ /*Idx0=*/0,
+ /*Idx1=*/0);
+
+ } else {
+ BasePointersArray = llvm::ConstantPointerNull::get(CGM.VoidPtrPtrTy);
+ PointersArray = llvm::ConstantPointerNull::get(CGM.VoidPtrPtrTy);
+ SizesArray = llvm::ConstantPointerNull::get(CGM.SizeTy->getPointerTo());
+ MapTypesArray =
+ llvm::ConstantPointerNull::get(CGM.Int32Ty->getPointerTo());
+ }
+
+ // On top of the arrays that were filled up, the target offloading call
+ // takes as arguments the device id as well as the host pointer. The host
+ // pointer is used by the runtime library to identify the current target
+ // region, so it only has to be unique and not necessarily point to
+ // anything. It could be the pointer to the outlined function that
+ // implements the target region, but we aren't using that so that the
+ // compiler doesn't need to keep that, and could therefore inline the host
+ // function if proven worthwhile during optimization.
+
+ llvm::Value *HostPtr = new llvm::GlobalVariable(
+ CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
+ llvm::GlobalValue::PrivateLinkage,
+ llvm::Constant::getNullValue(CGM.Int8Ty), ".offload_hstptr");
+
+ // Emit device ID if any.
+ llvm::Value *DeviceID;
+ if (Device)
+ DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
+ CGM.Int32Ty, /*isSigned=*/true);
+ else
+ DeviceID = CGF.Builder.getInt32(OMP_DEVICEID_UNDEF);
+
+ llvm::Value *OffloadingArgs[] = {
+ DeviceID, HostPtr, PointerNum, BasePointersArray,
+ PointersArray, SizesArray, MapTypesArray};
+ auto Return = CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_target),
+ OffloadingArgs);
+
+ CGF.EmitStoreOfScalar(Return, OffloadError);
+ };
+
+ if (IfCond) {
+ // Notify that the host version must be executed.
+ auto &&ElseGen = [this, OffloadError,
+ OffloadErrorQType](CodeGenFunction &CGF) {
+ CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/-1u),
+ OffloadError);
+ };
+ emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen);
+ } else {
+ CodeGenFunction::RunCleanupsScope Scope(CGF);
+ ThenGen(CGF);
+ }
+
+ // Check the error code and execute the host version if required.
+ auto OffloadFailedBlock = CGF.createBasicBlock("omp_offload.failed");
+ auto OffloadContBlock = CGF.createBasicBlock("omp_offload.cont");
+ auto OffloadErrorVal = CGF.EmitLoadOfScalar(OffloadError, SourceLocation());
+ auto Failed = CGF.Builder.CreateIsNotNull(OffloadErrorVal);
+ CGF.Builder.CreateCondBr(Failed, OffloadFailedBlock, OffloadContBlock);
+
+ CGF.EmitBlock(OffloadFailedBlock);
+ CGF.Builder.CreateCall(OutlinedFn, BasePointers);
+ CGF.EmitBranch(OffloadContBlock);
+
+ CGF.EmitBlock(OffloadContBlock, /*IsFinished=*/true);
+ return;
+}
diff --git a/lib/CodeGen/CGOpenMPRuntime.h b/lib/CodeGen/CGOpenMPRuntime.h
index f5aa4a5..992f9a8 100644
--- a/lib/CodeGen/CGOpenMPRuntime.h
+++ b/lib/CodeGen/CGOpenMPRuntime.h
@@ -39,7 +39,7 @@
class VarDecl;
namespace CodeGen {
-
+class Address;
class CodeGenFunction;
class CodeGenModule;
@@ -62,12 +62,17 @@
// Call to void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
// kmp_critical_name *crit);
OMPRTL__kmpc_critical,
+ // Call to void __kmpc_critical_with_hint(ident_t *loc, kmp_int32
+ // global_tid, kmp_critical_name *crit, uintptr_t hint);
+ OMPRTL__kmpc_critical_with_hint,
// Call to void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
// kmp_critical_name *crit);
OMPRTL__kmpc_end_critical,
// Call to kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
// global_tid);
OMPRTL__kmpc_cancel_barrier,
+ // Call to void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
+ OMPRTL__kmpc_barrier,
// Call to void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
OMPRTL__kmpc_for_static_fini,
// Call to void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
@@ -131,6 +136,35 @@
// Call to kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
// global_tid);
OMPRTL__kmpc_omp_taskwait,
+ // Call to void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
+ OMPRTL__kmpc_taskgroup,
+ // Call to void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
+ OMPRTL__kmpc_end_taskgroup,
+ // Call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
+ // int proc_bind);
+ OMPRTL__kmpc_push_proc_bind,
+ // Call to kmp_int32 __kmpc_omp_task_with_deps(ident_t *loc_ref, kmp_int32
+ // gtid, kmp_task_t * new_task, kmp_int32 ndeps, kmp_depend_info_t
+ // *dep_list, kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
+ OMPRTL__kmpc_omp_task_with_deps,
+ // Call to void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32
+ // gtid, kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
+ // ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
+ OMPRTL__kmpc_omp_wait_deps,
+ // Call to kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
+ // global_tid, kmp_int32 cncl_kind);
+ OMPRTL__kmpc_cancellationpoint,
+ // Call to kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
+ // kmp_int32 cncl_kind);
+ OMPRTL__kmpc_cancel,
+
+ //
+ // Offloading related calls
+ //
+ // Call to int32_t __tgt_target(int32_t device_id, void *host_ptr, int32_t
+ // arg_num, void** args_base, void **args, size_t *arg_sizes, int32_t
+ // *arg_types);
+ OMPRTL__tgt_target,
};
/// \brief Values for bit flags used in the ident_t to describe the fields.
@@ -161,7 +195,9 @@
/// \brief Map of flags and corresponding default locations.
typedef llvm::DenseMap<unsigned, llvm::Value *> OpenMPDefaultLocMapTy;
OpenMPDefaultLocMapTy OpenMPDefaultLocMap;
- llvm::Value *getOrCreateDefaultLocation(OpenMPLocationFlags Flags);
+ Address getOrCreateDefaultLocation(OpenMPLocationFlags Flags);
+
+public:
/// \brief Describes ident structure that describes a source location.
/// All descriptions are taken from
/// http://llvm.org/svn/llvm-project/openmp/trunk/runtime/src/kmp.h
@@ -202,6 +238,7 @@
/// and a pair of line numbers that delimit the construct.
IdentField_PSource
};
+private:
llvm::StructType *IdentTy;
/// \brief Map for SourceLocation and OpenMP runtime library debug locations.
typedef llvm::DenseMap<unsigned, llvm::Value *> OpenMPDebugLocMapTy;
@@ -242,6 +279,16 @@
/// deconstructors of firstprivate C++ objects */
/// } kmp_task_t;
QualType KmpTaskTQTy;
+ /// \brief Type typedef struct kmp_depend_info {
+ /// kmp_intptr_t base_addr;
+ /// size_t len;
+ /// struct {
+ /// bool in:1;
+ /// bool out:1;
+ /// } flags;
+ /// } kmp_depend_info_t;
+ QualType KmpDependInfoTy;
+
/// \brief Build type kmp_routine_entry_t (if not built yet).
void emitKmpRoutineEntryT(QualType KmpInt32Ty);
@@ -288,8 +335,7 @@
/// \brief Emits address of the word in a memory where current thread id is
/// stored.
- virtual llvm::Value *emitThreadIDAddress(CodeGenFunction &CGF,
- SourceLocation Loc);
+ virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc);
/// \brief Gets thread id value for the current thread.
///
@@ -313,7 +359,7 @@
/// \param CopyCtor Pointer to a global copy function for \a VD.
/// \param Dtor Pointer to a global destructor function for \a VD.
/// \param Loc Location of threadprivate declaration.
- void emitThreadPrivateVarInit(CodeGenFunction &CGF, llvm::Value *VDAddr,
+ void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr,
llvm::Value *Ctor, llvm::Value *CopyCtor,
llvm::Value *Dtor, SourceLocation Loc);
@@ -334,22 +380,25 @@
/// kmp_int32 BoundID, struct context_vars*).
/// \param D OpenMP directive.
/// \param ThreadIDVar Variable for thread id in the current OpenMP region.
+ /// \param InnermostKind Kind of innermost directive (for simple directives it
+ /// is a directive itself, for combined - its innermost directive).
/// \param CodeGen Code generation sequence for the \a D directive.
- virtual llvm::Value *
- emitParallelOutlinedFunction(const OMPExecutableDirective &D,
- const VarDecl *ThreadIDVar,
- const RegionCodeGenTy &CodeGen);
+ virtual llvm::Value *emitParallelOutlinedFunction(
+ const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
+ OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen);
/// \brief Emits outlined function for the OpenMP task directive \a D. This
/// outlined function has type void(*)(kmp_int32 ThreadID, kmp_int32
/// PartID, struct context_vars*).
/// \param D OpenMP directive.
/// \param ThreadIDVar Variable for thread id in the current OpenMP region.
+ /// \param InnermostKind Kind of innermost directive (for simple directives it
+ /// is a directive itself, for combined - its innermost directive).
/// \param CodeGen Code generation sequence for the \a D directive.
///
- virtual llvm::Value *emitTaskOutlinedFunction(const OMPExecutableDirective &D,
- const VarDecl *ThreadIDVar,
- const RegionCodeGenTy &CodeGen);
+ virtual llvm::Value *emitTaskOutlinedFunction(
+ const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
+ OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen);
/// \brief Cleans up references to the objects in finished function.
///
@@ -360,23 +409,25 @@
/// CapturedStruct.
/// \param OutlinedFn Outlined function to be run in parallel threads. Type of
/// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
- /// \param CapturedStruct A pointer to the record with the references to
+ /// \param CapturedVars A pointer to the record with the references to
/// variables used in \a OutlinedFn function.
/// \param IfCond Condition in the associated 'if' clause, if it was
/// specified, nullptr otherwise.
///
virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
llvm::Value *OutlinedFn,
- llvm::Value *CapturedStruct,
+ ArrayRef<llvm::Value *> CapturedVars,
const Expr *IfCond);
/// \brief Emits a critical region.
/// \param CriticalName Name of the critical region.
/// \param CriticalOpGen Generator for the statement associated with the given
/// critical region.
+ /// \param Hint Value of the 'hint' clause (optional).
virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,
const RegionCodeGenTy &CriticalOpGen,
- SourceLocation Loc);
+ SourceLocation Loc,
+ const Expr *Hint = nullptr);
/// \brief Emits a master region.
/// \param MasterOpGen Generator for the statement associated with the given
@@ -388,6 +439,13 @@
/// \brief Emits code for a taskyield directive.
virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc);
+ /// \brief Emit a taskgroup region.
+ /// \param TaskgroupOpGen Generator for the statement associated with the
+ /// given taskgroup region.
+ virtual void emitTaskgroupRegion(CodeGenFunction &CGF,
+ const RegionCodeGenTy &TaskgroupOpGen,
+ SourceLocation Loc);
+
/// \brief Emits a single region.
/// \param SingleOpGen Generator for the statement associated with the given
/// single region.
@@ -401,17 +459,23 @@
/// \brief Emit an ordered region.
/// \param OrderedOpGen Generator for the statement associated with the given
- /// critical region.
+ /// ordered region.
virtual void emitOrderedRegion(CodeGenFunction &CGF,
const RegionCodeGenTy &OrderedOpGen,
- SourceLocation Loc);
+ SourceLocation Loc, bool IsThreads);
/// \brief Emit an implicit/explicit barrier for OpenMP threads.
/// \param Kind Directive for which this implicit barrier call must be
/// generated. Must be OMPD_barrier for explicit barrier generation.
+ /// \param EmitChecks true if need to emit checks for cancellation barriers.
+ /// \param ForceSimpleCall true simple barrier call must be emitted, false if
+ /// runtime class decides which one to emit (simple or with cancellation
+ /// checks).
///
virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
- OpenMPDirectiveKind Kind);
+ OpenMPDirectiveKind Kind,
+ bool EmitChecks = true,
+ bool ForceSimpleCall = false);
/// \brief Check if the specified \a ScheduleKind is static non-chunked.
/// This kind of worksharing directive is emitted without outer loop.
@@ -427,6 +491,12 @@
///
virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const;
+ virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc,
+ OpenMPScheduleClauseKind SchedKind,
+ unsigned IVSize, bool IVSigned,
+ bool Ordered, llvm::Value *UB,
+ llvm::Value *Chunk = nullptr);
+
/// \brief Call the appropriate runtime routine to initialize it before start
/// of loop.
///
@@ -451,11 +521,12 @@
/// \param Chunk Value of the chunk for the static_chunked scheduled loop.
/// For the default (nullptr) value, the chunk 1 will be used.
///
- virtual void emitForInit(CodeGenFunction &CGF, SourceLocation Loc,
- OpenMPScheduleClauseKind SchedKind, unsigned IVSize,
- bool IVSigned, bool Ordered, llvm::Value *IL,
- llvm::Value *LB, llvm::Value *UB, llvm::Value *ST,
- llvm::Value *Chunk = nullptr);
+ virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
+ OpenMPScheduleClauseKind SchedKind,
+ unsigned IVSize, bool IVSigned, bool Ordered,
+ Address IL, Address LB,
+ Address UB, Address ST,
+ llvm::Value *Chunk = nullptr);
/// \brief Call the appropriate runtime routine to notify that we finished
/// iteration of the ordered loop with the dynamic scheduling.
@@ -493,8 +564,8 @@
/// returned.
virtual llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc,
unsigned IVSize, bool IVSigned,
- llvm::Value *IL, llvm::Value *LB,
- llvm::Value *UB, llvm::Value *ST);
+ Address IL, Address LB,
+ Address UB, Address ST);
/// \brief Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32
/// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'
@@ -504,16 +575,22 @@
llvm::Value *NumThreads,
SourceLocation Loc);
+ /// \brief Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32
+ /// global_tid, int proc_bind) to generate code for 'proc_bind' clause.
+ virtual void emitProcBindClause(CodeGenFunction &CGF,
+ OpenMPProcBindClauseKind ProcBind,
+ SourceLocation Loc);
+
/// \brief Returns address of the threadprivate variable for the current
/// thread.
/// \param VD Threadprivate variable.
/// \param VDAddr Address of the global variable \a VD.
/// \param Loc Location of the reference to threadprivate var.
/// \return Address of the threadprivate variable for the current thread.
- virtual llvm::Value *getAddrOfThreadPrivate(CodeGenFunction &CGF,
- const VarDecl *VD,
- llvm::Value *VDAddr,
- SourceLocation Loc);
+ virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF,
+ const VarDecl *VD,
+ Address VDAddr,
+ SourceLocation Loc);
/// \brief Emit a code for initialization of threadprivate variable. It emits
/// a call to runtime library which adds initial value to the newly created
@@ -524,7 +601,7 @@
/// \param Loc Location of threadprivate declaration.
/// \param PerformInit true if initialization expression is not constant.
virtual llvm::Function *
- emitThreadPrivateVarDefinition(const VarDecl *VD, llvm::Value *VDAddr,
+ emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr,
SourceLocation Loc, bool PerformInit,
CodeGenFunction *CGF = nullptr);
@@ -560,7 +637,7 @@
/// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
/// /*part_id*/, captured_struct */*__context*/);
/// \param SharedsTy A type which contains references the shared variables.
- /// \param Shareds Context with the list of shared variables from the \a
+ /// \param Shareds Context with the list of shared variables from the \p
/// TaskFunction.
/// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
/// otherwise.
@@ -575,22 +652,30 @@
/// \param FirstprivateInits List of references to auto generated variables
/// used for initialization of a single array element. Used if firstprivate
/// variable is of array type.
- virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
- const OMPExecutableDirective &D, bool Tied,
- llvm::PointerIntPair<llvm::Value *, 1, bool> Final,
- llvm::Value *TaskFunction, QualType SharedsTy,
- llvm::Value *Shareds, const Expr *IfCond,
- const ArrayRef<const Expr *> PrivateVars,
- const ArrayRef<const Expr *> PrivateCopies,
- const ArrayRef<const Expr *> FirstprivateVars,
- const ArrayRef<const Expr *> FirstprivateCopies,
- const ArrayRef<const Expr *> FirstprivateInits);
+ /// \param Dependences List of dependences for the 'task' construct, including
+ /// original expression and dependency type.
+ virtual void emitTaskCall(
+ CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D,
+ bool Tied, llvm::PointerIntPair<llvm::Value *, 1, bool> Final,
+ llvm::Value *TaskFunction, QualType SharedsTy, Address Shareds,
+ const Expr *IfCond, ArrayRef<const Expr *> PrivateVars,
+ ArrayRef<const Expr *> PrivateCopies,
+ ArrayRef<const Expr *> FirstprivateVars,
+ ArrayRef<const Expr *> FirstprivateCopies,
+ ArrayRef<const Expr *> FirstprivateInits,
+ ArrayRef<std::pair<OpenMPDependClauseKind, const Expr *>> Dependences);
/// \brief Emit code for the directive that does not require outlining.
///
+ /// \param InnermostKind Kind of innermost directive (for simple directives it
+ /// is a directive itself, for combined - its innermost directive).
/// \param CodeGen Code generation sequence for the \a D directive.
+ /// \param HasCancel true if region has inner cancel directive, false
+ /// otherwise.
virtual void emitInlinedDirective(CodeGenFunction &CGF,
- const RegionCodeGenTy &CodeGen);
+ OpenMPDirectiveKind InnermostKind,
+ const RegionCodeGenTy &CodeGen,
+ bool HasCancel = false);
/// \brief Emit a code for reduction clause. Next code should be emitted for
/// reduction:
/// \code
@@ -622,6 +707,7 @@
/// }
/// \endcode
///
+ /// \param Privates List of private copies for original reduction arguments.
/// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
/// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
/// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
@@ -629,13 +715,54 @@
/// \param WithNowait true if parent directive has also nowait clause, false
/// otherwise.
virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
+ ArrayRef<const Expr *> Privates,
ArrayRef<const Expr *> LHSExprs,
ArrayRef<const Expr *> RHSExprs,
ArrayRef<const Expr *> ReductionOps,
- bool WithNowait);
+ bool WithNowait, bool SimpleReduction);
/// \brief Emit code for 'taskwait' directive.
virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc);
+
+ /// \brief Emit code for 'cancellation point' construct.
+ /// \param CancelRegion Region kind for which the cancellation point must be
+ /// emitted.
+ ///
+ virtual void emitCancellationPointCall(CodeGenFunction &CGF,
+ SourceLocation Loc,
+ OpenMPDirectiveKind CancelRegion);
+
+ /// \brief Emit code for 'cancel' construct.
+ /// \param IfCond Condition in the associated 'if' clause, if it was
+ /// specified, nullptr otherwise.
+ /// \param CancelRegion Region kind for which the cancel must be emitted.
+ ///
+ virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
+ const Expr *IfCond,
+ OpenMPDirectiveKind CancelRegion);
+
+ /// \brief Emit outilined function for 'target' directive.
+ /// \param D Directive to emit.
+ /// \param CodeGen Code generation sequence for the \a D directive.
+ virtual llvm::Value *
+ emitTargetOutlinedFunction(const OMPExecutableDirective &D,
+ const RegionCodeGenTy &CodeGen);
+
+ /// \brief Emit the target offloading code associated with \a D. The emitted
+ /// code attempts offloading the execution to the device, an the event of
+ /// a failure it executes the host version outlined in \a OutlinedFn.
+ /// \param D Directive to emit.
+ /// \param OutlinedFn Host version of the code to be offloaded.
+ /// \param IfCond Expression evaluated in if clause associated with the target
+ /// directive, or null if no if clause is used.
+ /// \param Device Expression evaluated in device clause associated with the
+ /// target directive, or null if no device clause is used.
+ /// \param CapturedVars Values captured in the current region.
+ virtual void emitTargetCall(CodeGenFunction &CGF,
+ const OMPExecutableDirective &D,
+ llvm::Value *OutlinedFn, const Expr *IfCond,
+ const Expr *Device,
+ ArrayRef<llvm::Value *> CapturedVars);
};
} // namespace CodeGen
diff --git a/lib/CodeGen/CGRecordLayout.h b/lib/CodeGen/CGRecordLayout.h
index c15f9fd..d4ad33e 100644
--- a/lib/CodeGen/CGRecordLayout.h
+++ b/lib/CodeGen/CGRecordLayout.h
@@ -78,16 +78,16 @@
/// bitfield.
unsigned StorageSize;
- /// The alignment which should be used when accessing the bitfield.
- unsigned StorageAlignment;
+ /// The offset of the bitfield storage from the start of the struct.
+ CharUnits StorageOffset;
CGBitFieldInfo()
- : Offset(), Size(), IsSigned(), StorageSize(), StorageAlignment() {}
+ : Offset(), Size(), IsSigned(), StorageSize(), StorageOffset() {}
CGBitFieldInfo(unsigned Offset, unsigned Size, bool IsSigned,
- unsigned StorageSize, unsigned StorageAlignment)
+ unsigned StorageSize, CharUnits StorageOffset)
: Offset(Offset), Size(Size), IsSigned(IsSigned),
- StorageSize(StorageSize), StorageAlignment(StorageAlignment) {}
+ StorageSize(StorageSize), StorageOffset(StorageOffset) {}
void print(raw_ostream &OS) const;
void dump() const;
@@ -99,7 +99,7 @@
const FieldDecl *FD,
uint64_t Offset, uint64_t Size,
uint64_t StorageSize,
- uint64_t StorageAlignment);
+ CharUnits StorageOffset);
};
/// CGRecordLayout - This class handles struct and union layout info while
diff --git a/lib/CodeGen/CGRecordLayoutBuilder.cpp b/lib/CodeGen/CGRecordLayoutBuilder.cpp
index c89d5cc..375b59c 100644
--- a/lib/CodeGen/CGRecordLayoutBuilder.cpp
+++ b/lib/CodeGen/CGRecordLayoutBuilder.cpp
@@ -228,11 +228,7 @@
Info.Offset = (unsigned)(getFieldBitOffset(FD) - Context.toBits(StartOffset));
Info.Size = FD->getBitWidthValue(Context);
Info.StorageSize = (unsigned)DataLayout.getTypeAllocSizeInBits(StorageType);
- // Here we calculate the actual storage alignment of the bits. E.g if we've
- // got an alignment >= 2 and the bitfield starts at offset 6 we've got an
- // alignment of 2.
- Info.StorageAlignment =
- Layout.getAlignment().alignmentAtOffset(StartOffset).getQuantity();
+ Info.StorageOffset = StartOffset;
if (Info.Size > Info.StorageSize)
Info.Size = Info.StorageSize;
// Reverse the bit offsets for big endian machines. Because we represent
@@ -458,7 +454,7 @@
// contain only a trailing array member.
const CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl();
if (!BaseDecl->isEmpty() &&
- !Context.getASTRecordLayout(BaseDecl).getSize().isZero())
+ !Context.getASTRecordLayout(BaseDecl).getNonVirtualSize().isZero())
Members.push_back(MemberInfo(Layout.getBaseClassOffset(BaseDecl),
MemberInfo::Base, getStorageType(BaseDecl), BaseDecl));
}
@@ -651,7 +647,7 @@
const FieldDecl *FD,
uint64_t Offset, uint64_t Size,
uint64_t StorageSize,
- uint64_t StorageAlignment) {
+ CharUnits StorageOffset) {
// This function is vestigial from CGRecordLayoutBuilder days but is still
// used in GCObjCRuntime.cpp. That usage has a "fixme" attached to it that
// when addressed will allow for the removal of this function.
@@ -683,7 +679,7 @@
Offset = StorageSize - (Offset + Size);
}
- return CGBitFieldInfo(Offset, Size, IsSigned, StorageSize, StorageAlignment);
+ return CGBitFieldInfo(Offset, Size, IsSigned, StorageSize, StorageOffset);
}
CGRecordLayout *CodeGenTypes::ComputeRecordLayout(const RecordDecl *D,
@@ -856,7 +852,7 @@
<< " Size:" << Size
<< " IsSigned:" << IsSigned
<< " StorageSize:" << StorageSize
- << " StorageAlignment:" << StorageAlignment << ">";
+ << " StorageOffset:" << StorageOffset.getQuantity() << ">";
}
void CGBitFieldInfo::dump() const {
diff --git a/lib/CodeGen/CGStmt.cpp b/lib/CodeGen/CGStmt.cpp
index a79b3e3..cc4fa2e 100644
--- a/lib/CodeGen/CGStmt.cpp
+++ b/lib/CodeGen/CGStmt.cpp
@@ -16,6 +16,7 @@
#include "CodeGenModule.h"
#include "TargetInfo.h"
#include "clang/AST/StmtVisitor.h"
+#include "clang/Basic/Builtins.h"
#include "clang/Basic/PrettyStackTrace.h"
#include "clang/Basic/TargetInfo.h"
#include "clang/Sema/LoopHint.h"
@@ -25,6 +26,8 @@
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/InlineAsm.h"
#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/MDBuilder.h"
+
using namespace clang;
using namespace CodeGen;
@@ -138,6 +141,10 @@
case Stmt::SwitchStmtClass: EmitSwitchStmt(cast<SwitchStmt>(*S)); break;
case Stmt::GCCAsmStmtClass: // Intentional fall-through.
case Stmt::MSAsmStmtClass: EmitAsmStmt(cast<AsmStmt>(*S)); break;
+ case Stmt::CoroutineBodyStmtClass:
+ case Stmt::CoreturnStmtClass:
+ CGM.ErrorUnsupported(S, "coroutine");
+ break;
case Stmt::CapturedStmtClass: {
const CapturedStmt *CS = cast<CapturedStmt>(S);
EmitCapturedStmt(*CS, CS->getCapturedRegionKind());
@@ -222,6 +229,9 @@
case Stmt::OMPTaskwaitDirectiveClass:
EmitOMPTaskwaitDirective(cast<OMPTaskwaitDirective>(*S));
break;
+ case Stmt::OMPTaskgroupDirectiveClass:
+ EmitOMPTaskgroupDirective(cast<OMPTaskgroupDirective>(*S));
+ break;
case Stmt::OMPFlushDirectiveClass:
EmitOMPFlushDirective(cast<OMPFlushDirective>(*S));
break;
@@ -237,6 +247,24 @@
case Stmt::OMPTeamsDirectiveClass:
EmitOMPTeamsDirective(cast<OMPTeamsDirective>(*S));
break;
+ case Stmt::OMPCancellationPointDirectiveClass:
+ EmitOMPCancellationPointDirective(cast<OMPCancellationPointDirective>(*S));
+ break;
+ case Stmt::OMPCancelDirectiveClass:
+ EmitOMPCancelDirective(cast<OMPCancelDirective>(*S));
+ break;
+ case Stmt::OMPTargetDataDirectiveClass:
+ EmitOMPTargetDataDirective(cast<OMPTargetDataDirective>(*S));
+ break;
+ case Stmt::OMPTaskLoopDirectiveClass:
+ EmitOMPTaskLoopDirective(cast<OMPTaskLoopDirective>(*S));
+ break;
+ case Stmt::OMPTaskLoopSimdDirectiveClass:
+ EmitOMPTaskLoopSimdDirective(cast<OMPTaskLoopSimdDirective>(*S));
+ break;
+case Stmt::OMPDistributeDirectiveClass:
+ EmitOMPDistributeDirective(cast<OMPDistributeDirective>(*S));
+ break;
}
}
@@ -263,8 +291,8 @@
/// EmitCompoundStmt - Emit a compound statement {..} node. If GetLast is true,
/// this captures the expression result of the last sub-statement and returns it
/// (for use by the statement expression extension).
-llvm::Value* CodeGenFunction::EmitCompoundStmt(const CompoundStmt &S, bool GetLast,
- AggValueSlot AggSlot) {
+Address CodeGenFunction::EmitCompoundStmt(const CompoundStmt &S, bool GetLast,
+ AggValueSlot AggSlot) {
PrettyStackTraceLoc CrashInfo(getContext().getSourceManager(),S.getLBracLoc(),
"LLVM IR generation of compound statement ('{}')");
@@ -274,7 +302,7 @@
return EmitCompoundStmtWithoutScope(S, GetLast, AggSlot);
}
-llvm::Value*
+Address
CodeGenFunction::EmitCompoundStmtWithoutScope(const CompoundStmt &S,
bool GetLast,
AggValueSlot AggSlot) {
@@ -283,7 +311,7 @@
E = S.body_end()-GetLast; I != E; ++I)
EmitStmt(*I);
- llvm::Value *RetAlloca = nullptr;
+ Address RetAlloca = Address::invalid();
if (GetLast) {
// We have to special case labels here. They are statements, but when put
// at the end of a statement expression, they yield the value of their
@@ -328,7 +356,7 @@
return;
// Can only simplify empty blocks.
- if (BI != BB->begin())
+ if (BI->getIterator() != BB->begin())
return;
BB->replaceAllUsesWith(BI->getSuccessor(0));
@@ -350,7 +378,7 @@
// Place the block after the current block, if possible, or else at
// the end of the function.
if (CurBB && CurBB->getParent())
- CurFn->getBasicBlockList().insertAfter(CurBB, BB);
+ CurFn->getBasicBlockList().insertAfter(CurBB->getIterator(), BB);
else
CurFn->getBasicBlockList().push_back(BB);
Builder.SetInsertPoint(BB);
@@ -377,7 +405,8 @@
bool inserted = false;
for (llvm::User *u : block->users()) {
if (llvm::Instruction *insn = dyn_cast<llvm::Instruction>(u)) {
- CurFn->getBasicBlockList().insertAfter(insn->getParent(), block);
+ CurFn->getBasicBlockList().insertAfter(insn->getParent()->getIterator(),
+ block);
inserted = true;
break;
}
@@ -581,100 +610,6 @@
EmitBlock(ContBlock, true);
}
-void CodeGenFunction::EmitCondBrHints(llvm::LLVMContext &Context,
- llvm::BranchInst *CondBr,
- ArrayRef<const Attr *> Attrs) {
- // Return if there are no hints.
- if (Attrs.empty())
- return;
-
- // Add vectorize and unroll hints to the metadata on the conditional branch.
- //
- // FIXME: Should this really start with a size of 1?
- SmallVector<llvm::Metadata *, 2> Metadata(1);
- for (const auto *Attr : Attrs) {
- const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
-
- // Skip non loop hint attributes
- if (!LH)
- continue;
-
- LoopHintAttr::OptionType Option = LH->getOption();
- LoopHintAttr::LoopHintState State = LH->getState();
- const char *MetadataName;
- switch (Option) {
- case LoopHintAttr::Vectorize:
- case LoopHintAttr::VectorizeWidth:
- MetadataName = "llvm.loop.vectorize.width";
- break;
- case LoopHintAttr::Interleave:
- case LoopHintAttr::InterleaveCount:
- MetadataName = "llvm.loop.interleave.count";
- break;
- case LoopHintAttr::Unroll:
- // With the unroll loop hint, a non-zero value indicates full unrolling.
- MetadataName = State == LoopHintAttr::Disable ? "llvm.loop.unroll.disable"
- : "llvm.loop.unroll.full";
- break;
- case LoopHintAttr::UnrollCount:
- MetadataName = "llvm.loop.unroll.count";
- break;
- }
-
- Expr *ValueExpr = LH->getValue();
- int ValueInt = 1;
- if (ValueExpr) {
- llvm::APSInt ValueAPS =
- ValueExpr->EvaluateKnownConstInt(CGM.getContext());
- ValueInt = static_cast<int>(ValueAPS.getSExtValue());
- }
-
- llvm::Constant *Value;
- llvm::MDString *Name;
- switch (Option) {
- case LoopHintAttr::Vectorize:
- case LoopHintAttr::Interleave:
- if (State != LoopHintAttr::Disable) {
- // FIXME: In the future I will modifiy the behavior of the metadata
- // so we can enable/disable vectorization and interleaving separately.
- Name = llvm::MDString::get(Context, "llvm.loop.vectorize.enable");
- Value = Builder.getTrue();
- break;
- }
- // Vectorization/interleaving is disabled, set width/count to 1.
- ValueInt = 1;
- // Fallthrough.
- case LoopHintAttr::VectorizeWidth:
- case LoopHintAttr::InterleaveCount:
- case LoopHintAttr::UnrollCount:
- Name = llvm::MDString::get(Context, MetadataName);
- Value = llvm::ConstantInt::get(Int32Ty, ValueInt);
- break;
- case LoopHintAttr::Unroll:
- Name = llvm::MDString::get(Context, MetadataName);
- Value = nullptr;
- break;
- }
-
- SmallVector<llvm::Metadata *, 2> OpValues;
- OpValues.push_back(Name);
- if (Value)
- OpValues.push_back(llvm::ConstantAsMetadata::get(Value));
-
- // Set or overwrite metadata indicated by Name.
- Metadata.push_back(llvm::MDNode::get(Context, OpValues));
- }
-
- // FIXME: This condition is never false. Should it be an assert?
- if (!Metadata.empty()) {
- // Add llvm.loop MDNode to CondBr.
- llvm::MDNode *LoopID = llvm::MDNode::get(Context, Metadata);
- LoopID->replaceOperandWith(0, LoopID); // First op points to itself.
-
- CondBr->setMetadata("llvm.loop", LoopID);
- }
-}
-
void CodeGenFunction::EmitWhileStmt(const WhileStmt &S,
ArrayRef<const Attr *> WhileAttrs) {
// Emit the header for the loop, which will also become
@@ -682,7 +617,7 @@
JumpDest LoopHeader = getJumpDestInCurrentScope("while.cond");
EmitBlock(LoopHeader.getBlock());
- LoopStack.push(LoopHeader.getBlock(), WhileAttrs);
+ LoopStack.push(LoopHeader.getBlock(), CGM.getContext(), WhileAttrs);
// Create an exit block for when the condition fails, which will
// also become the break target.
@@ -721,7 +656,7 @@
llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
if (ConditionScope.requiresCleanups())
ExitBlock = createBasicBlock("while.exit");
- llvm::BranchInst *CondBr = Builder.CreateCondBr(
+ Builder.CreateCondBr(
BoolCondVal, LoopBody, ExitBlock,
createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody())));
@@ -729,9 +664,6 @@
EmitBlock(ExitBlock);
EmitBranchThroughCleanup(LoopExit);
}
-
- // Attach metadata to loop body conditional branch.
- EmitCondBrHints(LoopBody->getContext(), CondBr, WhileAttrs);
}
// Emit the loop body. We have to emit this in a cleanup scope
@@ -776,7 +708,7 @@
// Emit the body of the loop.
llvm::BasicBlock *LoopBody = createBasicBlock("do.body");
- LoopStack.push(LoopBody, DoAttrs);
+ LoopStack.push(LoopBody, CGM.getContext(), DoAttrs);
EmitBlockWithFallThrough(LoopBody, &S);
{
@@ -806,12 +738,9 @@
// As long as the condition is true, iterate the loop.
if (EmitBoolCondBranch) {
uint64_t BackedgeCount = getProfileCount(S.getBody()) - ParentCount;
- llvm::BranchInst *CondBr = Builder.CreateCondBr(
+ Builder.CreateCondBr(
BoolCondVal, LoopBody, LoopExit.getBlock(),
createProfileWeightsForLoop(S.getCond(), BackedgeCount));
-
- // Attach metadata to loop body conditional branch.
- EmitCondBrHints(LoopBody->getContext(), CondBr, DoAttrs);
}
LoopStack.pop();
@@ -842,7 +771,7 @@
llvm::BasicBlock *CondBlock = Continue.getBlock();
EmitBlock(CondBlock);
- LoopStack.push(CondBlock, ForAttrs);
+ LoopStack.push(CondBlock, CGM.getContext(), ForAttrs);
// If the for loop doesn't have an increment we can just use the
// condition as the continue block. Otherwise we'll need to create
@@ -876,13 +805,10 @@
// C99 6.8.5p2/p4: The first substatement is executed if the expression
// compares unequal to 0. The condition must be a scalar type.
llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
- llvm::BranchInst *CondBr = Builder.CreateCondBr(
+ Builder.CreateCondBr(
BoolCondVal, ForBody, ExitBlock,
createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody())));
- // Attach metadata to loop body conditional branch.
- EmitCondBrHints(ForBody->getContext(), CondBr, ForAttrs);
-
if (ExitBlock != LoopExit.getBlock()) {
EmitBlock(ExitBlock);
EmitBranchThroughCleanup(LoopExit);
@@ -940,7 +866,7 @@
llvm::BasicBlock *CondBlock = createBasicBlock("for.cond");
EmitBlock(CondBlock);
- LoopStack.push(CondBlock, ForAttrs);
+ LoopStack.push(CondBlock, CGM.getContext(), ForAttrs);
// If there are any cleanups between here and the loop-exit scope,
// create a block to stage a loop exit along.
@@ -954,13 +880,10 @@
// The body is executed if the expression, contextually converted
// to bool, is true.
llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
- llvm::BranchInst *CondBr = Builder.CreateCondBr(
+ Builder.CreateCondBr(
BoolCondVal, ForBody, ExitBlock,
createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody())));
- // Attach metadata to loop body conditional branch.
- EmitCondBrHints(ForBody->getContext(), CondBr, ForAttrs);
-
if (ExitBlock != LoopExit.getBlock()) {
EmitBlock(ExitBlock);
EmitBranchThroughCleanup(LoopExit);
@@ -1003,10 +926,9 @@
if (RV.isScalar()) {
Builder.CreateStore(RV.getScalarVal(), ReturnValue);
} else if (RV.isAggregate()) {
- EmitAggregateCopy(ReturnValue, RV.getAggregateAddr(), Ty);
+ EmitAggregateCopy(ReturnValue, RV.getAggregateAddress(), Ty);
} else {
- EmitStoreOfComplex(RV.getComplexVal(),
- MakeNaturalAlignAddrLValue(ReturnValue, Ty),
+ EmitStoreOfComplex(RV.getComplexVal(), MakeAddrLValue(ReturnValue, Ty),
/*init*/ true);
}
EmitBranchThroughCleanup(ReturnBlock);
@@ -1047,8 +969,8 @@
// If there is an NRVO flag for this variable, set it to 1 into indicate
// that the cleanup code should not destroy the variable.
if (llvm::Value *NRVOFlag = NRVOFlags[S.getNRVOCandidate()])
- Builder.CreateStore(Builder.getTrue(), NRVOFlag);
- } else if (!ReturnValue || (RV && RV->getType()->isVoidType())) {
+ Builder.CreateFlagStore(Builder.getTrue(), NRVOFlag);
+ } else if (!ReturnValue.isValid() || (RV && RV->getType()->isVoidType())) {
// Make sure not to return anything, but evaluate the expression
// for side effects.
if (RV)
@@ -1066,20 +988,17 @@
Builder.CreateStore(EmitScalarExpr(RV), ReturnValue);
break;
case TEK_Complex:
- EmitComplexExprIntoLValue(RV,
- MakeNaturalAlignAddrLValue(ReturnValue, RV->getType()),
+ EmitComplexExprIntoLValue(RV, MakeAddrLValue(ReturnValue, RV->getType()),
/*isInit*/ true);
break;
- case TEK_Aggregate: {
- CharUnits Alignment = getContext().getTypeAlignInChars(RV->getType());
- EmitAggExpr(RV, AggValueSlot::forAddr(ReturnValue, Alignment,
+ case TEK_Aggregate:
+ EmitAggExpr(RV, AggValueSlot::forAddr(ReturnValue,
Qualifiers(),
AggValueSlot::IsDestructed,
AggValueSlot::DoesNotNeedGCBarriers,
AggValueSlot::IsNotAliased));
break;
}
- }
}
++NumReturnExprs;
@@ -1615,6 +1534,22 @@
EmitBlock(SwitchExit.getBlock(), true);
incrementProfileCounter(&S);
+ // If the switch has a condition wrapped by __builtin_unpredictable,
+ // create metadata that specifies that the switch is unpredictable.
+ // Don't bother if not optimizing because that metadata would not be used.
+ if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
+ if (const CallExpr *Call = dyn_cast<CallExpr>(S.getCond())) {
+ const Decl *TargetDecl = Call->getCalleeDecl();
+ if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
+ if (FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
+ llvm::MDBuilder MDHelper(getLLVMContext());
+ SwitchInsn->setMetadata(llvm::LLVMContext::MD_unpredictable,
+ MDHelper.createUnpredictable());
+ }
+ }
+ }
+ }
+
if (SwitchWeights) {
assert(SwitchWeights->size() == 1 + SwitchInsn->getNumCases() &&
"switch weights do not match switch cases");
@@ -1666,9 +1601,7 @@
assert(OutCons &&
"Must pass output names to constraints with a symbolic name");
unsigned Index;
- bool result = Target.resolveSymbolicName(Constraint,
- &(*OutCons)[0],
- OutCons->size(), Index);
+ bool result = Target.resolveSymbolicName(Constraint, *OutCons, Index);
assert(result && "Could not resolve symbolic name"); (void)result;
Result += llvm::utostr(Index);
break;
@@ -1734,12 +1667,12 @@
Arg = Builder.CreateLoad(Builder.CreateBitCast(InputValue.getAddress(),
Ty));
} else {
- Arg = InputValue.getAddress();
+ Arg = InputValue.getPointer();
ConstraintStr += '*';
}
}
} else {
- Arg = InputValue.getAddress();
+ Arg = InputValue.getPointer();
ConstraintStr += '*';
}
@@ -1763,7 +1696,8 @@
if (Info.allowsRegister() || !Info.allowsMemory())
if (CodeGenFunction::hasScalarEvaluationKind(InputExpr->getType()))
return EmitScalarExpr(InputExpr);
-
+ if (InputExpr->getStmtClass() == Expr::CXXThisExprClass)
+ return EmitScalarExpr(InputExpr);
InputExpr = InputExpr->IgnoreParenNoopCasts(getContext());
LValue Dest = EmitLValue(InputExpr);
return EmitAsmInputLValue(Info, Dest, InputExpr->getType(), ConstraintStr,
@@ -1784,13 +1718,15 @@
if (!StrVal.empty()) {
const SourceManager &SM = CGF.CGM.getContext().getSourceManager();
const LangOptions &LangOpts = CGF.CGM.getLangOpts();
+ unsigned StartToken = 0;
+ unsigned ByteOffset = 0;
// Add the location of the start of each subsequent line of the asm to the
// MDNode.
- for (unsigned i = 0, e = StrVal.size()-1; i != e; ++i) {
+ for (unsigned i = 0, e = StrVal.size() - 1; i != e; ++i) {
if (StrVal[i] != '\n') continue;
- SourceLocation LineLoc = Str->getLocationOfByte(i+1, SM, LangOpts,
- CGF.getTarget());
+ SourceLocation LineLoc = Str->getLocationOfByte(
+ i + 1, SM, LangOpts, CGF.getTarget(), &StartToken, &ByteOffset);
Locs.push_back(llvm::ConstantAsMetadata::get(
llvm::ConstantInt::get(CGF.Int32Ty, LineLoc.getRawEncoding())));
}
@@ -1823,8 +1759,7 @@
Name = GAS->getInputName(i);
TargetInfo::ConstraintInfo Info(S.getInputConstraint(i), Name);
bool IsValid =
- getTarget().validateInputConstraint(OutputConstraintInfos.data(),
- S.getNumOutputs(), Info);
+ getTarget().validateInputConstraint(OutputConstraintInfos, Info);
assert(IsValid && "Failed to parse input constraint"); (void)IsValid;
InputConstraintInfos.push_back(Info);
}
@@ -1843,6 +1778,14 @@
std::vector<llvm::Value*> InOutArgs;
std::vector<llvm::Type*> InOutArgTypes;
+ // An inline asm can be marked readonly if it meets the following conditions:
+ // - it doesn't have any sideeffects
+ // - it doesn't clobber memory
+ // - it doesn't return a value by-reference
+ // It can be marked readnone if it doesn't have any input memory constraints
+ // in addition to meeting the conditions listed above.
+ bool ReadOnly = true, ReadNone = true;
+
for (unsigned i = 0, e = S.getNumOutputs(); i != e; i++) {
TargetInfo::ConstraintInfo &Info = OutputConstraintInfos[i];
@@ -1902,10 +1845,11 @@
<< OutExpr->getType() << OutputConstraint;
}
} else {
- ArgTypes.push_back(Dest.getAddress()->getType());
- Args.push_back(Dest.getAddress());
+ ArgTypes.push_back(Dest.getAddress().getType());
+ Args.push_back(Dest.getPointer());
Constraints += "=*";
Constraints += OutputConstraint;
+ ReadOnly = ReadNone = false;
}
if (Info.isReadWrite()) {
@@ -1950,6 +1894,9 @@
TargetInfo::ConstraintInfo &Info = InputConstraintInfos[i];
+ if (Info.allowsMemory())
+ ReadNone = false;
+
if (!Constraints.empty())
Constraints += ',';
@@ -2014,7 +1961,9 @@
for (unsigned i = 0, e = S.getNumClobbers(); i != e; i++) {
StringRef Clobber = S.getClobber(i);
- if (Clobber != "memory" && Clobber != "cc")
+ if (Clobber == "memory")
+ ReadOnly = ReadNone = false;
+ else if (Clobber != "cc")
Clobber = getTarget().getNormalizedGCCRegisterName(Clobber);
if (!Constraints.empty())
@@ -2054,6 +2003,25 @@
Result->addAttribute(llvm::AttributeSet::FunctionIndex,
llvm::Attribute::NoUnwind);
+ if (isa<MSAsmStmt>(&S)) {
+ // If the assembly contains any labels, mark the call noduplicate to prevent
+ // defining the same ASM label twice (PR23715). This is pretty hacky, but it
+ // works.
+ if (AsmString.find("__MSASMLABEL_") != std::string::npos)
+ Result->addAttribute(llvm::AttributeSet::FunctionIndex,
+ llvm::Attribute::NoDuplicate);
+ }
+
+ // Attach readnone and readonly attributes.
+ if (!HasSideEffect) {
+ if (ReadNone)
+ Result->addAttribute(llvm::AttributeSet::FunctionIndex,
+ llvm::Attribute::ReadNone);
+ else if (ReadOnly)
+ Result->addAttribute(llvm::AttributeSet::FunctionIndex,
+ llvm::Attribute::ReadOnly);
+ }
+
// Slap the source location of the inline asm into a !srcloc metadata on the
// call.
if (const GCCAsmStmt *gccAsmStmt = dyn_cast<GCCAsmStmt>(&S)) {
@@ -2119,12 +2087,12 @@
QualType RecordTy = getContext().getRecordType(RD);
// Initialize the captured struct.
- LValue SlotLV = MakeNaturalAlignAddrLValue(
- CreateMemTemp(RecordTy, "agg.captured"), RecordTy);
+ LValue SlotLV =
+ MakeAddrLValue(CreateMemTemp(RecordTy, "agg.captured"), RecordTy);
RecordDecl::field_iterator CurField = RD->field_begin();
- for (CapturedStmt::capture_init_iterator I = S.capture_init_begin(),
- E = S.capture_init_end();
+ for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
+ E = S.capture_init_end();
I != E; ++I, ++CurField) {
LValue LV = EmitLValueForFieldInitialization(SlotLV, *CurField);
if (CurField->hasCapturedVLAType()) {
@@ -2146,18 +2114,17 @@
// Emit the CapturedDecl
CodeGenFunction CGF(CGM, true);
- CGF.CapturedStmtInfo = new CGCapturedStmtInfo(S, K);
+ CGCapturedStmtRAII CapInfoRAII(CGF, new CGCapturedStmtInfo(S, K));
llvm::Function *F = CGF.GenerateCapturedStmtFunction(S);
delete CGF.CapturedStmtInfo;
// Emit call to the helper function.
- EmitCallOrInvoke(F, CapStruct.getAddress());
+ EmitCallOrInvoke(F, CapStruct.getPointer());
return F;
}
-llvm::Value *
-CodeGenFunction::GenerateCapturedStmtArgument(const CapturedStmt &S) {
+Address CodeGenFunction::GenerateCapturedStmtArgument(const CapturedStmt &S) {
LValue CapStruct = InitCapturedStruct(S);
return CapStruct.getAddress();
}
@@ -2196,8 +2163,7 @@
CD->getLocation(),
CD->getBody()->getLocStart());
// Set the context parameter in CapturedStmtInfo.
- llvm::Value *DeclPtr = LocalDeclMap[CD->getContextParam()];
- assert(DeclPtr && "missing context parameter for CapturedStmt");
+ Address DeclPtr = GetAddrOfLocalVar(CD->getContextParam());
CapturedStmtInfo->setContextValue(Builder.CreateLoad(DeclPtr));
// Initialize variable-length arrays.
@@ -2219,7 +2185,7 @@
CXXThisValue = EmitLoadOfLValue(ThisLValue, Loc).getScalarVal();
}
- PGO.assignRegionCounters(CD, F);
+ PGO.assignRegionCounters(GlobalDecl(CD), F);
CapturedStmtInfo->EmitBody(*this, CD->getBody());
FinishFunction(CD->getBodyRBrace());
diff --git a/lib/CodeGen/CGStmtOpenMP.cpp b/lib/CodeGen/CGStmtOpenMP.cpp
index 895baa7..bcd2ac5 100644
--- a/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/lib/CodeGen/CGStmtOpenMP.cpp
@@ -20,21 +20,195 @@
using namespace clang;
using namespace CodeGen;
+void CodeGenFunction::GenerateOpenMPCapturedVars(
+ const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
+ const RecordDecl *RD = S.getCapturedRecordDecl();
+ auto CurField = RD->field_begin();
+ auto CurCap = S.captures().begin();
+ for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
+ E = S.capture_init_end();
+ I != E; ++I, ++CurField, ++CurCap) {
+ if (CurField->hasCapturedVLAType()) {
+ auto VAT = CurField->getCapturedVLAType();
+ auto *Val = VLASizeMap[VAT->getSizeExpr()];
+ CapturedVars.push_back(Val);
+ } else if (CurCap->capturesThis())
+ CapturedVars.push_back(CXXThisValue);
+ else if (CurCap->capturesVariableByCopy())
+ CapturedVars.push_back(
+ EmitLoadOfLValue(EmitLValue(*I), SourceLocation()).getScalarVal());
+ else {
+ assert(CurCap->capturesVariable() && "Expected capture by reference.");
+ CapturedVars.push_back(EmitLValue(*I).getAddress().getPointer());
+ }
+ }
+}
+
+static Address castValueFromUintptr(CodeGenFunction &CGF, QualType DstType,
+ StringRef Name, LValue AddrLV,
+ bool isReferenceType = false) {
+ ASTContext &Ctx = CGF.getContext();
+
+ auto *CastedPtr = CGF.EmitScalarConversion(
+ AddrLV.getAddress().getPointer(), Ctx.getUIntPtrType(),
+ Ctx.getPointerType(DstType), SourceLocation());
+ auto TmpAddr =
+ CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType))
+ .getAddress();
+
+ // If we are dealing with references we need to return the address of the
+ // reference instead of the reference of the value.
+ if (isReferenceType) {
+ QualType RefType = Ctx.getLValueReferenceType(DstType);
+ auto *RefVal = TmpAddr.getPointer();
+ TmpAddr = CGF.CreateMemTemp(RefType, Twine(Name) + ".ref");
+ auto TmpLVal = CGF.MakeAddrLValue(TmpAddr, RefType);
+ CGF.EmitScalarInit(RefVal, TmpLVal);
+ }
+
+ return TmpAddr;
+}
+
+llvm::Function *
+CodeGenFunction::GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S) {
+ assert(
+ CapturedStmtInfo &&
+ "CapturedStmtInfo should be set when generating the captured function");
+ const CapturedDecl *CD = S.getCapturedDecl();
+ const RecordDecl *RD = S.getCapturedRecordDecl();
+ assert(CD->hasBody() && "missing CapturedDecl body");
+
+ // Build the argument list.
+ ASTContext &Ctx = CGM.getContext();
+ FunctionArgList Args;
+ Args.append(CD->param_begin(),
+ std::next(CD->param_begin(), CD->getContextParamPosition()));
+ auto I = S.captures().begin();
+ for (auto *FD : RD->fields()) {
+ QualType ArgType = FD->getType();
+ IdentifierInfo *II = nullptr;
+ VarDecl *CapVar = nullptr;
+
+ // If this is a capture by copy and the type is not a pointer, the outlined
+ // function argument type should be uintptr and the value properly casted to
+ // uintptr. This is necessary given that the runtime library is only able to
+ // deal with pointers. We can pass in the same way the VLA type sizes to the
+ // outlined function.
+ if ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
+ I->capturesVariableArrayType())
+ ArgType = Ctx.getUIntPtrType();
+
+ if (I->capturesVariable() || I->capturesVariableByCopy()) {
+ CapVar = I->getCapturedVar();
+ II = CapVar->getIdentifier();
+ } else if (I->capturesThis())
+ II = &getContext().Idents.get("this");
+ else {
+ assert(I->capturesVariableArrayType());
+ II = &getContext().Idents.get("vla");
+ }
+ if (ArgType->isVariablyModifiedType())
+ ArgType = getContext().getVariableArrayDecayedType(ArgType);
+ Args.push_back(ImplicitParamDecl::Create(getContext(), nullptr,
+ FD->getLocation(), II, ArgType));
+ ++I;
+ }
+ Args.append(
+ std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
+ CD->param_end());
+
+ // Create the function declaration.
+ FunctionType::ExtInfo ExtInfo;
+ const CGFunctionInfo &FuncInfo =
+ CGM.getTypes().arrangeFreeFunctionDeclaration(Ctx.VoidTy, Args, ExtInfo,
+ /*IsVariadic=*/false);
+ llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
+
+ llvm::Function *F = llvm::Function::Create(
+ FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
+ CapturedStmtInfo->getHelperName(), &CGM.getModule());
+ CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
+ if (CD->isNothrow())
+ F->addFnAttr(llvm::Attribute::NoUnwind);
+
+ // Generate the function.
+ StartFunction(CD, Ctx.VoidTy, F, FuncInfo, Args, CD->getLocation(),
+ CD->getBody()->getLocStart());
+ unsigned Cnt = CD->getContextParamPosition();
+ I = S.captures().begin();
+ for (auto *FD : RD->fields()) {
+ // If we are capturing a pointer by copy we don't need to do anything, just
+ // use the value that we get from the arguments.
+ if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
+ setAddrOfLocalVar(I->getCapturedVar(), GetAddrOfLocalVar(Args[Cnt]));
+ ++Cnt, ++I;
+ continue;
+ }
+
+ LValue ArgLVal =
+ MakeAddrLValue(GetAddrOfLocalVar(Args[Cnt]), Args[Cnt]->getType(),
+ AlignmentSource::Decl);
+ if (FD->hasCapturedVLAType()) {
+ LValue CastedArgLVal =
+ MakeAddrLValue(castValueFromUintptr(*this, FD->getType(),
+ Args[Cnt]->getName(), ArgLVal),
+ FD->getType(), AlignmentSource::Decl);
+ auto *ExprArg =
+ EmitLoadOfLValue(CastedArgLVal, SourceLocation()).getScalarVal();
+ auto VAT = FD->getCapturedVLAType();
+ VLASizeMap[VAT->getSizeExpr()] = ExprArg;
+ } else if (I->capturesVariable()) {
+ auto *Var = I->getCapturedVar();
+ QualType VarTy = Var->getType();
+ Address ArgAddr = ArgLVal.getAddress();
+ if (!VarTy->isReferenceType()) {
+ ArgAddr = EmitLoadOfReference(
+ ArgAddr, ArgLVal.getType()->castAs<ReferenceType>());
+ }
+ setAddrOfLocalVar(
+ Var, Address(ArgAddr.getPointer(), getContext().getDeclAlign(Var)));
+ } else if (I->capturesVariableByCopy()) {
+ assert(!FD->getType()->isAnyPointerType() &&
+ "Not expecting a captured pointer.");
+ auto *Var = I->getCapturedVar();
+ QualType VarTy = Var->getType();
+ setAddrOfLocalVar(I->getCapturedVar(),
+ castValueFromUintptr(*this, FD->getType(),
+ Args[Cnt]->getName(), ArgLVal,
+ VarTy->isReferenceType()));
+ } else {
+ // If 'this' is captured, load it into CXXThisValue.
+ assert(I->capturesThis());
+ CXXThisValue =
+ EmitLoadOfLValue(ArgLVal, Args[Cnt]->getLocation()).getScalarVal();
+ }
+ ++Cnt, ++I;
+ }
+
+ PGO.assignRegionCounters(GlobalDecl(CD), F);
+ CapturedStmtInfo->EmitBody(*this, CD->getBody());
+ FinishFunction(CD->getBodyRBrace());
+
+ return F;
+}
+
//===----------------------------------------------------------------------===//
// OpenMP Directive Emission
//===----------------------------------------------------------------------===//
void CodeGenFunction::EmitOMPAggregateAssign(
- llvm::Value *DestAddr, llvm::Value *SrcAddr, QualType OriginalType,
- const llvm::function_ref<void(llvm::Value *, llvm::Value *)> &CopyGen) {
+ Address DestAddr, Address SrcAddr, QualType OriginalType,
+ const llvm::function_ref<void(Address, Address)> &CopyGen) {
// Perform element-by-element initialization.
QualType ElementTy;
- auto SrcBegin = SrcAddr;
- auto DestBegin = DestAddr;
+
+ // Drill down to the base element type on both arrays.
auto ArrayTy = OriginalType->getAsArrayTypeUnsafe();
- auto NumElements = emitArrayLength(ArrayTy, ElementTy, DestBegin);
+ auto NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
+ SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
+
+ auto SrcBegin = SrcAddr.getPointer();
+ auto DestBegin = DestAddr.getPointer();
// Cast from pointer to array type to pointer to single element.
- SrcBegin = Builder.CreatePointerBitCastOrAddrSpaceCast(SrcBegin,
- DestBegin->getType());
auto DestEnd = Builder.CreateGEP(DestBegin, NumElements);
// The basic structure here is a while-do loop.
auto BodyBB = createBasicBlock("omp.arraycpy.body");
@@ -46,77 +220,144 @@
// Enter the loop body, making that address the current address.
auto EntryBB = Builder.GetInsertBlock();
EmitBlock(BodyBB);
- auto SrcElementCurrent =
- Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
- SrcElementCurrent->addIncoming(SrcBegin, EntryBB);
- auto DestElementCurrent = Builder.CreatePHI(DestBegin->getType(), 2,
- "omp.arraycpy.destElementPast");
- DestElementCurrent->addIncoming(DestBegin, EntryBB);
+
+ CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
+
+ llvm::PHINode *SrcElementPHI =
+ Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
+ SrcElementPHI->addIncoming(SrcBegin, EntryBB);
+ Address SrcElementCurrent =
+ Address(SrcElementPHI,
+ SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
+
+ llvm::PHINode *DestElementPHI =
+ Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
+ DestElementPHI->addIncoming(DestBegin, EntryBB);
+ Address DestElementCurrent =
+ Address(DestElementPHI,
+ DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
// Emit copy.
CopyGen(DestElementCurrent, SrcElementCurrent);
// Shift the address forward by one element.
auto DestElementNext = Builder.CreateConstGEP1_32(
- DestElementCurrent, /*Idx0=*/1, "omp.arraycpy.dest.element");
+ DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
auto SrcElementNext = Builder.CreateConstGEP1_32(
- SrcElementCurrent, /*Idx0=*/1, "omp.arraycpy.src.element");
+ SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
// Check whether we've reached the end.
auto Done =
Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
Builder.CreateCondBr(Done, DoneBB, BodyBB);
- DestElementCurrent->addIncoming(DestElementNext, Builder.GetInsertBlock());
- SrcElementCurrent->addIncoming(SrcElementNext, Builder.GetInsertBlock());
+ DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
+ SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
// Done.
EmitBlock(DoneBB, /*IsFinished=*/true);
}
-void CodeGenFunction::EmitOMPCopy(CodeGenFunction &CGF,
- QualType OriginalType, llvm::Value *DestAddr,
- llvm::Value *SrcAddr, const VarDecl *DestVD,
+/// \brief Emit initialization of arrays of complex types.
+/// \param DestAddr Address of the array.
+/// \param Type Type of array.
+/// \param Init Initial expression of array.
+static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
+ QualType Type, const Expr *Init) {
+ // Perform element-by-element initialization.
+ QualType ElementTy;
+
+ // Drill down to the base element type on both arrays.
+ auto ArrayTy = Type->getAsArrayTypeUnsafe();
+ auto NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr);
+ DestAddr =
+ CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType());
+
+ auto DestBegin = DestAddr.getPointer();
+ // Cast from pointer to array type to pointer to single element.
+ auto DestEnd = CGF.Builder.CreateGEP(DestBegin, NumElements);
+ // The basic structure here is a while-do loop.
+ auto BodyBB = CGF.createBasicBlock("omp.arrayinit.body");
+ auto DoneBB = CGF.createBasicBlock("omp.arrayinit.done");
+ auto IsEmpty =
+ CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty");
+ CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
+
+ // Enter the loop body, making that address the current address.
+ auto EntryBB = CGF.Builder.GetInsertBlock();
+ CGF.EmitBlock(BodyBB);
+
+ CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
+
+ llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
+ DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
+ DestElementPHI->addIncoming(DestBegin, EntryBB);
+ Address DestElementCurrent =
+ Address(DestElementPHI,
+ DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
+
+ // Emit copy.
+ {
+ CodeGenFunction::RunCleanupsScope InitScope(CGF);
+ CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(),
+ /*IsInitializer=*/false);
+ }
+
+ // Shift the address forward by one element.
+ auto DestElementNext = CGF.Builder.CreateConstGEP1_32(
+ DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
+ // Check whether we've reached the end.
+ auto Done =
+ CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
+ CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
+ DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock());
+
+ // Done.
+ CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
+}
+
+void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
+ Address SrcAddr, const VarDecl *DestVD,
const VarDecl *SrcVD, const Expr *Copy) {
if (OriginalType->isArrayType()) {
auto *BO = dyn_cast<BinaryOperator>(Copy);
if (BO && BO->getOpcode() == BO_Assign) {
// Perform simple memcpy for simple copying.
- CGF.EmitAggregateAssign(DestAddr, SrcAddr, OriginalType);
+ EmitAggregateAssign(DestAddr, SrcAddr, OriginalType);
} else {
// For arrays with complex element types perform element by element
// copying.
- CGF.EmitOMPAggregateAssign(
+ EmitOMPAggregateAssign(
DestAddr, SrcAddr, OriginalType,
- [&CGF, Copy, SrcVD, DestVD](llvm::Value *DestElement,
- llvm::Value *SrcElement) {
+ [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
// Working with the single array element, so have to remap
// destination and source variables to corresponding array
// elements.
- CodeGenFunction::OMPPrivateScope Remap(CGF);
- Remap.addPrivate(DestVD, [DestElement]() -> llvm::Value *{
+ CodeGenFunction::OMPPrivateScope Remap(*this);
+ Remap.addPrivate(DestVD, [DestElement]() -> Address {
return DestElement;
});
Remap.addPrivate(
- SrcVD, [SrcElement]() -> llvm::Value *{ return SrcElement; });
+ SrcVD, [SrcElement]() -> Address { return SrcElement; });
(void)Remap.Privatize();
- CGF.EmitIgnoredExpr(Copy);
+ EmitIgnoredExpr(Copy);
});
}
} else {
// Remap pseudo source variable to private copy.
- CodeGenFunction::OMPPrivateScope Remap(CGF);
- Remap.addPrivate(SrcVD, [SrcAddr]() -> llvm::Value *{ return SrcAddr; });
- Remap.addPrivate(DestVD, [DestAddr]() -> llvm::Value *{ return DestAddr; });
+ CodeGenFunction::OMPPrivateScope Remap(*this);
+ Remap.addPrivate(SrcVD, [SrcAddr]() -> Address { return SrcAddr; });
+ Remap.addPrivate(DestVD, [DestAddr]() -> Address { return DestAddr; });
(void)Remap.Privatize();
// Emit copying of the whole variable.
- CGF.EmitIgnoredExpr(Copy);
+ EmitIgnoredExpr(Copy);
}
}
bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D,
OMPPrivateScope &PrivateScope) {
+ if (!HaveInsertPoint())
+ return false;
llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
- for (auto &&I = D.getClausesOfKind(OMPC_firstprivate); I; ++I) {
- auto *C = cast<OMPFirstprivateClause>(*I);
+ for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
auto IRef = C->varlist_begin();
auto InitsRef = C->inits().begin();
for (auto IInit : C->private_copies()) {
@@ -131,13 +372,13 @@
/*RefersToEnclosingVariableOrCapture=*/CapturedStmtInfo->lookup(
OrigVD) != nullptr,
(*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
- auto *OriginalAddr = EmitLValue(&DRE).getAddress();
+ Address OriginalAddr = EmitLValue(&DRE).getAddress();
QualType Type = OrigVD->getType();
if (Type->isArrayType()) {
// Emit VarDecl with copy init for arrays.
// Get the address of the original variable captured in current
// captured region.
- IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value *{
+ IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address {
auto Emission = EmitAutoVarAlloca(*VD);
auto *Init = VD->getInit();
if (!isa<CXXConstructExpr>(Init) || isTrivialInitializer(Init)) {
@@ -147,12 +388,12 @@
} else {
EmitOMPAggregateAssign(
Emission.getAllocatedAddress(), OriginalAddr, Type,
- [this, VDInit, Init](llvm::Value *DestElement,
- llvm::Value *SrcElement) {
+ [this, VDInit, Init](Address DestElement,
+ Address SrcElement) {
// Clean up any temporaries needed by the initialization.
RunCleanupsScope InitScope(*this);
// Emit initialization for single element.
- LocalDeclMap[VDInit] = SrcElement;
+ setAddrOfLocalVar(VDInit, SrcElement);
EmitAnyExprToMem(Init, DestElement,
Init->getType().getQualifiers(),
/*IsInitializer*/ false);
@@ -163,12 +404,12 @@
return Emission.getAllocatedAddress();
});
} else {
- IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value *{
+ IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address {
// Emit private VarDecl with copy init.
// Remap temp VDInit variable to the address of the original
// variable
// (for proper handling of captured global variables).
- LocalDeclMap[VDInit] = OriginalAddr;
+ setAddrOfLocalVar(VDInit, OriginalAddr);
EmitDecl(*VD);
LocalDeclMap.erase(VDInit);
return GetAddrOfLocalVar(VD);
@@ -188,16 +429,17 @@
void CodeGenFunction::EmitOMPPrivateClause(
const OMPExecutableDirective &D,
CodeGenFunction::OMPPrivateScope &PrivateScope) {
+ if (!HaveInsertPoint())
+ return;
llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
- for (auto &&I = D.getClausesOfKind(OMPC_private); I; ++I) {
- auto *C = cast<OMPPrivateClause>(*I);
+ for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
auto IRef = C->varlist_begin();
for (auto IInit : C->private_copies()) {
auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
auto VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
bool IsRegistered =
- PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value *{
+ PrivateScope.addPrivate(OrigVD, [&]() -> Address {
// Emit private VarDecl with copy init.
EmitDecl(*VD);
return GetAddrOfLocalVar(VD);
@@ -212,14 +454,15 @@
}
bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) {
+ if (!HaveInsertPoint())
+ return false;
// threadprivate_var1 = master_threadprivate_var1;
// operator=(threadprivate_var2, master_threadprivate_var2);
// ...
// __kmpc_barrier(&loc, global_tid);
llvm::DenseSet<const VarDecl *> CopiedVars;
llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
- for (auto &&I = D.getClausesOfKind(OMPC_copyin); I; ++I) {
- auto *C = cast<OMPCopyinClause>(*I);
+ for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
auto IRef = C->varlist_begin();
auto ISrcRef = C->source_exprs().begin();
auto IDestRef = C->destination_exprs().begin();
@@ -227,12 +470,27 @@
auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
QualType Type = VD->getType();
if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
- // Get the address of the master variable.
- auto *MasterAddr = VD->isStaticLocal()
- ? CGM.getStaticLocalDeclAddress(VD)
- : CGM.GetAddrOfGlobal(VD);
+
+ // Get the address of the master variable. If we are emitting code with
+ // TLS support, the address is passed from the master as field in the
+ // captured declaration.
+ Address MasterAddr = Address::invalid();
+ if (getLangOpts().OpenMPUseTLS &&
+ getContext().getTargetInfo().isTLSSupported()) {
+ assert(CapturedStmtInfo->lookup(VD) &&
+ "Copyin threadprivates should have been captured!");
+ DeclRefExpr DRE(const_cast<VarDecl *>(VD), true, (*IRef)->getType(),
+ VK_LValue, (*IRef)->getExprLoc());
+ MasterAddr = EmitLValue(&DRE).getAddress();
+ LocalDeclMap.erase(VD);
+ } else {
+ MasterAddr =
+ Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)
+ : CGM.GetAddrOfGlobal(VD),
+ getContext().getDeclAlign(VD));
+ }
// Get the address of the threadprivate variable.
- auto *PrivateAddr = EmitLValue(*IRef).getAddress();
+ Address PrivateAddr = EmitLValue(*IRef).getAddress();
if (CopiedVars.size() == 1) {
// At first check if current thread is a master thread. If it is, no
// need to copy data.
@@ -240,15 +498,14 @@
CopyEnd = createBasicBlock("copyin.not.master.end");
Builder.CreateCondBr(
Builder.CreateICmpNE(
- Builder.CreatePtrToInt(MasterAddr, CGM.IntPtrTy),
- Builder.CreatePtrToInt(PrivateAddr, CGM.IntPtrTy)),
+ Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy),
+ Builder.CreatePtrToInt(PrivateAddr.getPointer(), CGM.IntPtrTy)),
CopyBegin, CopyEnd);
EmitBlock(CopyBegin);
}
auto *SrcVD = cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
- EmitOMPCopy(*this, Type, PrivateAddr, MasterAddr, DestVD, SrcVD,
- AssignOp);
+ EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
}
++IRef;
++ISrcRef;
@@ -265,11 +522,12 @@
bool CodeGenFunction::EmitOMPLastprivateClauseInit(
const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
+ if (!HaveInsertPoint())
+ return false;
bool HasAtLeastOneLastprivate = false;
llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
- for (auto &&I = D.getClausesOfKind(OMPC_lastprivate); I; ++I) {
+ for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
HasAtLeastOneLastprivate = true;
- auto *C = cast<OMPLastprivateClause>(*I);
auto IRef = C->varlist_begin();
auto IDestRef = C->destination_exprs().begin();
for (auto *IInit : C->private_copies()) {
@@ -278,7 +536,7 @@
auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
- PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() -> llvm::Value *{
+ PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() -> Address {
DeclRefExpr DRE(
const_cast<VarDecl *>(OrigVD),
/*RefersToEnclosingVariableOrCapture=*/CapturedStmtInfo->lookup(
@@ -292,7 +550,7 @@
if (IInit) {
auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
bool IsRegistered =
- PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value *{
+ PrivateScope.addPrivate(OrigVD, [&]() -> Address {
// Emit private VarDecl with copy init.
EmitDecl(*VD);
return GetAddrOfLocalVar(VD);
@@ -310,39 +568,46 @@
void CodeGenFunction::EmitOMPLastprivateClauseFinal(
const OMPExecutableDirective &D, llvm::Value *IsLastIterCond) {
+ if (!HaveInsertPoint())
+ return;
// Emit following code:
// if (<IsLastIterCond>) {
// orig_var1 = private_orig_var1;
// ...
// orig_varn = private_orig_varn;
// }
- auto *ThenBB = createBasicBlock(".omp.lastprivate.then");
- auto *DoneBB = createBasicBlock(".omp.lastprivate.done");
- Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
- EmitBlock(ThenBB);
+ llvm::BasicBlock *ThenBB = nullptr;
+ llvm::BasicBlock *DoneBB = nullptr;
+ if (IsLastIterCond) {
+ ThenBB = createBasicBlock(".omp.lastprivate.then");
+ DoneBB = createBasicBlock(".omp.lastprivate.done");
+ Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
+ EmitBlock(ThenBB);
+ }
llvm::DenseMap<const Decl *, const Expr *> LoopCountersAndUpdates;
const Expr *LastIterVal = nullptr;
const Expr *IVExpr = nullptr;
const Expr *IncExpr = nullptr;
if (auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
- LastIterVal =
- cast<VarDecl>(cast<DeclRefExpr>(LoopDirective->getUpperBoundVariable())
- ->getDecl())
- ->getAnyInitializer();
- IVExpr = LoopDirective->getIterationVariable();
- IncExpr = LoopDirective->getInc();
- auto IUpdate = LoopDirective->updates().begin();
- for (auto *E : LoopDirective->counters()) {
- auto *D = cast<DeclRefExpr>(E)->getDecl()->getCanonicalDecl();
- LoopCountersAndUpdates[D] = *IUpdate;
- ++IUpdate;
+ if (isOpenMPWorksharingDirective(D.getDirectiveKind())) {
+ LastIterVal = cast<VarDecl>(cast<DeclRefExpr>(
+ LoopDirective->getUpperBoundVariable())
+ ->getDecl())
+ ->getAnyInitializer();
+ IVExpr = LoopDirective->getIterationVariable();
+ IncExpr = LoopDirective->getInc();
+ auto IUpdate = LoopDirective->updates().begin();
+ for (auto *E : LoopDirective->counters()) {
+ auto *D = cast<DeclRefExpr>(E)->getDecl()->getCanonicalDecl();
+ LoopCountersAndUpdates[D] = *IUpdate;
+ ++IUpdate;
+ }
}
}
{
llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
bool FirstLCV = true;
- for (auto &&I = D.getClausesOfKind(OMPC_lastprivate); I; ++I) {
- auto *C = cast<OMPLastprivateClause>(*I);
+ for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
auto IRef = C->varlist_begin();
auto ISrcRef = C->source_exprs().begin();
auto IDestRef = C->destination_exprs().begin();
@@ -355,7 +620,7 @@
// directive, update its value before copyin back to original
// variable.
if (auto *UpExpr = LoopCountersAndUpdates.lookup(CanonicalVD)) {
- if (FirstLCV) {
+ if (FirstLCV && LastIterVal) {
EmitAnyExprToMem(LastIterVal, EmitLValue(IVExpr).getAddress(),
IVExpr->getType().getQualifiers(),
/*IsInitializer=*/false);
@@ -367,11 +632,14 @@
auto *SrcVD = cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
// Get the address of the original variable.
- auto *OriginalAddr = GetAddrOfLocalVar(DestVD);
+ Address OriginalAddr = GetAddrOfLocalVar(DestVD);
// Get the address of the private variable.
- auto *PrivateAddr = GetAddrOfLocalVar(PrivateVD);
- EmitOMPCopy(*this, Type, OriginalAddr, PrivateAddr, DestVD, SrcVD,
- AssignOp);
+ Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
+ if (auto RefTy = PrivateVD->getType()->getAs<ReferenceType>())
+ PrivateAddr =
+ Address(Builder.CreateLoad(PrivateAddr),
+ getNaturalTypeAlignment(RefTy->getPointeeType()));
+ EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
}
++IRef;
++ISrcRef;
@@ -379,52 +647,182 @@
}
}
}
- EmitBlock(DoneBB, /*IsFinished=*/true);
+ if (IsLastIterCond) {
+ EmitBlock(DoneBB, /*IsFinished=*/true);
+ }
}
void CodeGenFunction::EmitOMPReductionClauseInit(
const OMPExecutableDirective &D,
CodeGenFunction::OMPPrivateScope &PrivateScope) {
- for (auto &&I = D.getClausesOfKind(OMPC_reduction); I; ++I) {
- auto *C = cast<OMPReductionClause>(*I);
+ if (!HaveInsertPoint())
+ return;
+ for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
auto ILHS = C->lhs_exprs().begin();
auto IRHS = C->rhs_exprs().begin();
+ auto IPriv = C->privates().begin();
for (auto IRef : C->varlists()) {
- auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
- auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
- // Store the address of the original variable associated with the LHS
- // implicit variable.
- PrivateScope.addPrivate(LHSVD, [this, OrigVD, IRef]() -> llvm::Value *{
- DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
- CapturedStmtInfo->lookup(OrigVD) != nullptr,
- IRef->getType(), VK_LValue, IRef->getExprLoc());
- return EmitLValue(&DRE).getAddress();
- });
- // Emit reduction copy.
- bool IsRegistered =
- PrivateScope.addPrivate(OrigVD, [this, PrivateVD]() -> llvm::Value *{
- // Emit private VarDecl with reduction init.
- EmitDecl(*PrivateVD);
- return GetAddrOfLocalVar(PrivateVD);
- });
- assert(IsRegistered && "private var already registered as private");
- // Silence the warning about unused variable.
- (void)IsRegistered;
- ++ILHS, ++IRHS;
+ auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
+ auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
+ if (auto *OASE = dyn_cast<OMPArraySectionExpr>(IRef)) {
+ auto *Base = OASE->getBase()->IgnoreParenImpCasts();
+ while (auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base))
+ Base = TempOASE->getBase()->IgnoreParenImpCasts();
+ while (auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
+ Base = TempASE->getBase()->IgnoreParenImpCasts();
+ auto *DE = cast<DeclRefExpr>(Base);
+ auto *OrigVD = cast<VarDecl>(DE->getDecl());
+ auto OASELValueLB = EmitOMPArraySectionExpr(OASE);
+ auto OASELValueUB =
+ EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false);
+ auto OriginalBaseLValue = EmitLValue(DE);
+ auto BaseLValue = OriginalBaseLValue;
+ auto *Zero = Builder.getInt64(/*C=*/0);
+ llvm::SmallVector<llvm::Value *, 4> Indexes;
+ Indexes.push_back(Zero);
+ auto *ItemTy =
+ OASELValueLB.getPointer()->getType()->getPointerElementType();
+ auto *Ty = BaseLValue.getPointer()->getType()->getPointerElementType();
+ while (Ty != ItemTy) {
+ Indexes.push_back(Zero);
+ Ty = Ty->getPointerElementType();
+ }
+ BaseLValue = MakeAddrLValue(
+ Address(Builder.CreateInBoundsGEP(BaseLValue.getPointer(), Indexes),
+ OASELValueLB.getAlignment()),
+ OASELValueLB.getType(), OASELValueLB.getAlignmentSource());
+ // Store the address of the original variable associated with the LHS
+ // implicit variable.
+ PrivateScope.addPrivate(LHSVD, [this, OASELValueLB]() -> Address {
+ return OASELValueLB.getAddress();
+ });
+ // Emit reduction copy.
+ bool IsRegistered = PrivateScope.addPrivate(
+ OrigVD, [this, PrivateVD, BaseLValue, OASELValueLB, OASELValueUB,
+ OriginalBaseLValue]() -> Address {
+ // Emit VarDecl with copy init for arrays.
+ // Get the address of the original variable captured in current
+ // captured region.
+ auto *Size = Builder.CreatePtrDiff(OASELValueUB.getPointer(),
+ OASELValueLB.getPointer());
+ Size = Builder.CreateNUWAdd(
+ Size, llvm::ConstantInt::get(Size->getType(), /*V=*/1));
+ CodeGenFunction::OpaqueValueMapping OpaqueMap(
+ *this, cast<OpaqueValueExpr>(
+ getContext()
+ .getAsVariableArrayType(PrivateVD->getType())
+ ->getSizeExpr()),
+ RValue::get(Size));
+ EmitVariablyModifiedType(PrivateVD->getType());
+ auto Emission = EmitAutoVarAlloca(*PrivateVD);
+ auto Addr = Emission.getAllocatedAddress();
+ auto *Init = PrivateVD->getInit();
+ EmitOMPAggregateInit(*this, Addr, PrivateVD->getType(), Init);
+ EmitAutoVarCleanups(Emission);
+ // Emit private VarDecl with reduction init.
+ auto *Offset = Builder.CreatePtrDiff(BaseLValue.getPointer(),
+ OASELValueLB.getPointer());
+ auto *Ptr = Builder.CreateGEP(Addr.getPointer(), Offset);
+ Ptr = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ Ptr, OriginalBaseLValue.getPointer()->getType());
+ return Address(Ptr, OriginalBaseLValue.getAlignment());
+ });
+ assert(IsRegistered && "private var already registered as private");
+ // Silence the warning about unused variable.
+ (void)IsRegistered;
+ PrivateScope.addPrivate(RHSVD, [this, PrivateVD]() -> Address {
+ return GetAddrOfLocalVar(PrivateVD);
+ });
+ } else if (auto *ASE = dyn_cast<ArraySubscriptExpr>(IRef)) {
+ auto *Base = ASE->getBase()->IgnoreParenImpCasts();
+ while (auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
+ Base = TempASE->getBase()->IgnoreParenImpCasts();
+ auto *DE = cast<DeclRefExpr>(Base);
+ auto *OrigVD = cast<VarDecl>(DE->getDecl());
+ auto ASELValue = EmitLValue(ASE);
+ auto OriginalBaseLValue = EmitLValue(DE);
+ auto BaseLValue = OriginalBaseLValue;
+ auto *Zero = Builder.getInt64(/*C=*/0);
+ llvm::SmallVector<llvm::Value *, 4> Indexes;
+ Indexes.push_back(Zero);
+ auto *ItemTy =
+ ASELValue.getPointer()->getType()->getPointerElementType();
+ auto *Ty = BaseLValue.getPointer()->getType()->getPointerElementType();
+ while (Ty != ItemTy) {
+ Indexes.push_back(Zero);
+ Ty = Ty->getPointerElementType();
+ }
+ BaseLValue = MakeAddrLValue(
+ Address(Builder.CreateInBoundsGEP(BaseLValue.getPointer(), Indexes),
+ ASELValue.getAlignment()),
+ ASELValue.getType(), ASELValue.getAlignmentSource());
+ // Store the address of the original variable associated with the LHS
+ // implicit variable.
+ PrivateScope.addPrivate(LHSVD, [this, ASELValue]() -> Address {
+ return ASELValue.getAddress();
+ });
+ // Emit reduction copy.
+ bool IsRegistered = PrivateScope.addPrivate(
+ OrigVD, [this, PrivateVD, BaseLValue, ASELValue,
+ OriginalBaseLValue]() -> Address {
+ // Emit private VarDecl with reduction init.
+ EmitDecl(*PrivateVD);
+ auto Addr = GetAddrOfLocalVar(PrivateVD);
+ auto *Offset = Builder.CreatePtrDiff(BaseLValue.getPointer(),
+ ASELValue.getPointer());
+ auto *Ptr = Builder.CreateGEP(Addr.getPointer(), Offset);
+ Ptr = Builder.CreatePointerBitCastOrAddrSpaceCast(
+ Ptr, OriginalBaseLValue.getPointer()->getType());
+ return Address(Ptr, OriginalBaseLValue.getAlignment());
+ });
+ assert(IsRegistered && "private var already registered as private");
+ // Silence the warning about unused variable.
+ (void)IsRegistered;
+ PrivateScope.addPrivate(RHSVD, [this, PrivateVD]() -> Address {
+ return GetAddrOfLocalVar(PrivateVD);
+ });
+ } else {
+ auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
+ // Store the address of the original variable associated with the LHS
+ // implicit variable.
+ PrivateScope.addPrivate(LHSVD, [this, OrigVD, IRef]() -> Address {
+ DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
+ CapturedStmtInfo->lookup(OrigVD) != nullptr,
+ IRef->getType(), VK_LValue, IRef->getExprLoc());
+ return EmitLValue(&DRE).getAddress();
+ });
+ // Emit reduction copy.
+ bool IsRegistered =
+ PrivateScope.addPrivate(OrigVD, [this, PrivateVD]() -> Address {
+ // Emit private VarDecl with reduction init.
+ EmitDecl(*PrivateVD);
+ return GetAddrOfLocalVar(PrivateVD);
+ });
+ assert(IsRegistered && "private var already registered as private");
+ // Silence the warning about unused variable.
+ (void)IsRegistered;
+ PrivateScope.addPrivate(RHSVD, [this, PrivateVD]() -> Address {
+ return GetAddrOfLocalVar(PrivateVD);
+ });
+ }
+ ++ILHS, ++IRHS, ++IPriv;
}
}
}
void CodeGenFunction::EmitOMPReductionClauseFinal(
const OMPExecutableDirective &D) {
+ if (!HaveInsertPoint())
+ return;
+ llvm::SmallVector<const Expr *, 8> Privates;
llvm::SmallVector<const Expr *, 8> LHSExprs;
llvm::SmallVector<const Expr *, 8> RHSExprs;
llvm::SmallVector<const Expr *, 8> ReductionOps;
bool HasAtLeastOneReduction = false;
- for (auto &&I = D.getClausesOfKind(OMPC_reduction); I; ++I) {
+ for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
HasAtLeastOneReduction = true;
- auto *C = cast<OMPReductionClause>(*I);
+ Privates.append(C->privates().begin(), C->privates().end());
LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
@@ -433,33 +831,45 @@
// Emit nowait reduction if nowait clause is present or directive is a
// parallel directive (it always has implicit barrier).
CGM.getOpenMPRuntime().emitReduction(
- *this, D.getLocEnd(), LHSExprs, RHSExprs, ReductionOps,
- D.getSingleClause(OMPC_nowait) ||
- isOpenMPParallelDirective(D.getDirectiveKind()));
+ *this, D.getLocEnd(), Privates, LHSExprs, RHSExprs, ReductionOps,
+ D.getSingleClause<OMPNowaitClause>() ||
+ isOpenMPParallelDirective(D.getDirectiveKind()) ||
+ D.getDirectiveKind() == OMPD_simd,
+ D.getDirectiveKind() == OMPD_simd);
}
}
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF,
const OMPExecutableDirective &S,
+ OpenMPDirectiveKind InnermostKind,
const RegionCodeGenTy &CodeGen) {
auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
- auto CapturedStruct = CGF.GenerateCapturedStmtArgument(*CS);
+ llvm::SmallVector<llvm::Value *, 16> CapturedVars;
+ CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
auto OutlinedFn = CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
- S, *CS->getCapturedDecl()->param_begin(), CodeGen);
- if (auto C = S.getSingleClause(OMPC_num_threads)) {
+ S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
+ if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
- auto NumThreadsClause = cast<OMPNumThreadsClause>(C);
auto NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
/*IgnoreResultAssign*/ true);
CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
CGF, NumThreads, NumThreadsClause->getLocStart());
}
+ if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
+ CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
+ CGF.CGM.getOpenMPRuntime().emitProcBindClause(
+ CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getLocStart());
+ }
const Expr *IfCond = nullptr;
- if (auto C = S.getSingleClause(OMPC_if)) {
- IfCond = cast<OMPIfClause>(C)->getCondition();
+ for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
+ if (C->getNameModifier() == OMPD_unknown ||
+ C->getNameModifier() == OMPD_parallel) {
+ IfCond = C->getCondition();
+ break;
+ }
}
CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getLocStart(), OutlinedFn,
- CapturedStruct, IfCond);
+ CapturedVars, IfCond);
}
void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
@@ -474,31 +884,28 @@
// initialization of firstprivate variables or propagation master's thread
// values of threadprivate variables to local instances of that variables
// of all other implicit threads.
- CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(),
- OMPD_unknown);
+ CGF.CGM.getOpenMPRuntime().emitBarrierCall(
+ CGF, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
+ /*ForceSimpleCall=*/true);
}
CGF.EmitOMPPrivateClause(S, PrivateScope);
CGF.EmitOMPReductionClauseInit(S, PrivateScope);
(void)PrivateScope.Privatize();
CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
CGF.EmitOMPReductionClauseFinal(S);
- // Emit implicit barrier at the end of the 'parallel' directive.
- CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(),
- OMPD_unknown);
};
- emitCommonOMPParallelDirective(*this, S, CodeGen);
+ emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen);
}
-void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &S,
- bool SeparateIter) {
+void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &D,
+ JumpDest LoopExit) {
RunCleanupsScope BodyScope(*this);
// Update counters values on current iteration.
- for (auto I : S.updates()) {
+ for (auto I : D.updates()) {
EmitIgnoredExpr(I);
}
// Update the linear variables.
- for (auto &&I = S.getClausesOfKind(OMPC_linear); I; ++I) {
- auto *C = cast<OMPLinearClause>(*I);
+ for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
for (auto U : C->updates()) {
EmitIgnoredExpr(U);
}
@@ -506,18 +913,16 @@
// On a continue in the body, jump to the end.
auto Continue = getJumpDestInCurrentScope("omp.body.continue");
- BreakContinueStack.push_back(BreakContinue(JumpDest(), Continue));
+ BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
// Emit loop body.
- EmitStmt(S.getBody());
+ EmitStmt(D.getBody());
// The end (updates/cleanups).
EmitBlock(Continue.getBlock());
BreakContinueStack.pop_back();
- if (SeparateIter) {
// TODO: Update lastprivates if the SeparateIter flag is true.
// This will be implemented in a follow-up OMPLastprivateClause patch, but
// result should be still correct without it, as we do not make these
// variables private yet.
- }
}
void CodeGenFunction::EmitOMPInnerLoop(
@@ -567,110 +972,127 @@
EmitBlock(LoopExit.getBlock());
}
-void CodeGenFunction::EmitOMPSimdFinal(const OMPLoopDirective &S) {
- auto IC = S.counters().begin();
- for (auto F : S.finals()) {
- auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
- if (LocalDeclMap.lookup(OrigVD)) {
+void CodeGenFunction::EmitOMPLinearClauseInit(const OMPLoopDirective &D) {
+ if (!HaveInsertPoint())
+ return;
+ // Emit inits for the linear variables.
+ for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
+ for (auto Init : C->inits()) {
+ auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
+ auto *OrigVD = cast<VarDecl>(
+ cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())->getDecl());
DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
CapturedStmtInfo->lookup(OrigVD) != nullptr,
- (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
- auto *OrigAddr = EmitLValue(&DRE).getAddress();
- OMPPrivateScope VarScope(*this);
- VarScope.addPrivate(OrigVD,
- [OrigAddr]() -> llvm::Value *{ return OrigAddr; });
- (void)VarScope.Privatize();
- EmitIgnoredExpr(F);
+ VD->getInit()->getType(), VK_LValue,
+ VD->getInit()->getExprLoc());
+ AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
+ EmitExprAsInit(&DRE, VD,
+ MakeAddrLValue(Emission.getAllocatedAddress(), VD->getType()),
+ /*capturedByInit=*/false);
+ EmitAutoVarCleanups(Emission);
}
- ++IC;
+ // Emit the linear steps for the linear clauses.
+ // If a step is not constant, it is pre-calculated before the loop.
+ if (auto CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
+ if (auto SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
+ EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
+ // Emit calculation of the linear step.
+ EmitIgnoredExpr(CS);
+ }
}
+}
+
+static void emitLinearClauseFinal(CodeGenFunction &CGF,
+ const OMPLoopDirective &D) {
+ if (!CGF.HaveInsertPoint())
+ return;
// Emit the final values of the linear variables.
- for (auto &&I = S.getClausesOfKind(OMPC_linear); I; ++I) {
- auto *C = cast<OMPLinearClause>(*I);
+ for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
auto IC = C->varlist_begin();
for (auto F : C->finals()) {
auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
- CapturedStmtInfo->lookup(OrigVD) != nullptr,
+ CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr,
(*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
- auto *OrigAddr = EmitLValue(&DRE).getAddress();
- OMPPrivateScope VarScope(*this);
+ Address OrigAddr = CGF.EmitLValue(&DRE).getAddress();
+ CodeGenFunction::OMPPrivateScope VarScope(CGF);
VarScope.addPrivate(OrigVD,
- [OrigAddr]() -> llvm::Value *{ return OrigAddr; });
+ [OrigAddr]() -> Address { return OrigAddr; });
(void)VarScope.Privatize();
- EmitIgnoredExpr(F);
+ CGF.EmitIgnoredExpr(F);
++IC;
}
}
}
-static void EmitOMPAlignedClause(CodeGenFunction &CGF, CodeGenModule &CGM,
- const OMPAlignedClause &Clause) {
- unsigned ClauseAlignment = 0;
- if (auto AlignmentExpr = Clause.getAlignment()) {
- auto AlignmentCI =
- cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
- ClauseAlignment = static_cast<unsigned>(AlignmentCI->getZExtValue());
- }
- for (auto E : Clause.varlists()) {
- unsigned Alignment = ClauseAlignment;
- if (Alignment == 0) {
- // OpenMP [2.8.1, Description]
- // If no optional parameter is specified, implementation-defined default
- // alignments for SIMD instructions on the target platforms are assumed.
- Alignment = CGM.getTargetCodeGenInfo().getOpenMPSimdDefaultAlignment(
- E->getType());
+static void emitAlignedClause(CodeGenFunction &CGF,
+ const OMPExecutableDirective &D) {
+ if (!CGF.HaveInsertPoint())
+ return;
+ for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
+ unsigned ClauseAlignment = 0;
+ if (auto AlignmentExpr = Clause->getAlignment()) {
+ auto AlignmentCI =
+ cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
+ ClauseAlignment = static_cast<unsigned>(AlignmentCI->getZExtValue());
}
- assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) &&
- "alignment is not power of 2");
- if (Alignment != 0) {
- llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
- CGF.EmitAlignmentAssumption(PtrValue, Alignment);
+ for (auto E : Clause->varlists()) {
+ unsigned Alignment = ClauseAlignment;
+ if (Alignment == 0) {
+ // OpenMP [2.8.1, Description]
+ // If no optional parameter is specified, implementation-defined default
+ // alignments for SIMD instructions on the target platforms are assumed.
+ Alignment =
+ CGF.getContext()
+ .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign(
+ E->getType()->getPointeeType()))
+ .getQuantity();
+ }
+ assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) &&
+ "alignment is not power of 2");
+ if (Alignment != 0) {
+ llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
+ CGF.EmitAlignmentAssumption(PtrValue, Alignment);
+ }
}
}
}
-static void EmitPrivateLoopCounters(CodeGenFunction &CGF,
+static void emitPrivateLoopCounters(CodeGenFunction &CGF,
CodeGenFunction::OMPPrivateScope &LoopScope,
- ArrayRef<Expr *> Counters) {
+ ArrayRef<Expr *> Counters,
+ ArrayRef<Expr *> PrivateCounters) {
+ if (!CGF.HaveInsertPoint())
+ return;
+ auto I = PrivateCounters.begin();
for (auto *E : Counters) {
- auto VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
- (void)LoopScope.addPrivate(VD, [&]() -> llvm::Value *{
+ auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
+ auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
+ Address Addr = Address::invalid();
+ (void)LoopScope.addPrivate(PrivateVD, [&]() -> Address {
// Emit var without initialization.
- auto VarEmission = CGF.EmitAutoVarAlloca(*VD);
+ auto VarEmission = CGF.EmitAutoVarAlloca(*PrivateVD);
CGF.EmitAutoVarCleanups(VarEmission);
- return VarEmission.getAllocatedAddress();
+ Addr = VarEmission.getAllocatedAddress();
+ return Addr;
});
+ (void)LoopScope.addPrivate(VD, [&]() -> Address { return Addr; });
+ ++I;
}
}
static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
const Expr *Cond, llvm::BasicBlock *TrueBlock,
llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
+ if (!CGF.HaveInsertPoint())
+ return;
{
CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
- EmitPrivateLoopCounters(CGF, PreCondScope, S.counters());
- const VarDecl *IVDecl =
- cast<VarDecl>(cast<DeclRefExpr>(S.getIterationVariable())->getDecl());
- bool IsRegistered = PreCondScope.addPrivate(IVDecl, [&]() -> llvm::Value *{
- // Emit var without initialization.
- auto VarEmission = CGF.EmitAutoVarAlloca(*IVDecl);
- CGF.EmitAutoVarCleanups(VarEmission);
- return VarEmission.getAllocatedAddress();
- });
- assert(IsRegistered && "counter already registered as private");
- // Silence the warning about unused variable.
- (void)IsRegistered;
+ emitPrivateLoopCounters(CGF, PreCondScope, S.counters(),
+ S.private_counters());
(void)PreCondScope.Privatize();
- // Initialize internal counter to 0 to calculate initial values of real
- // counters.
- LValue IV = CGF.EmitLValue(S.getIterationVariable());
- CGF.EmitStoreOfScalar(
- llvm::ConstantInt::getNullValue(
- IV.getAddress()->getType()->getPointerElementType()),
- CGF.EmitLValue(S.getIterationVariable()), /*isInit=*/true);
// Get initial values of real counters.
- for (auto I : S.updates()) {
+ for (auto I : S.inits()) {
CGF.EmitIgnoredExpr(I);
}
}
@@ -679,38 +1101,85 @@
}
static void
-EmitPrivateLinearVars(CodeGenFunction &CGF, const OMPExecutableDirective &D,
+emitPrivateLinearVars(CodeGenFunction &CGF, const OMPExecutableDirective &D,
CodeGenFunction::OMPPrivateScope &PrivateScope) {
- for (auto &&I = D.getClausesOfKind(OMPC_linear); I; ++I) {
- auto *C = cast<OMPLinearClause>(*I);
+ if (!CGF.HaveInsertPoint())
+ return;
+ for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
+ auto CurPrivate = C->privates().begin();
for (auto *E : C->varlists()) {
- auto VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
- bool IsRegistered = PrivateScope.addPrivate(VD, [&]()->llvm::Value * {
- // Emit var without initialization.
- auto VarEmission = CGF.EmitAutoVarAlloca(*VD);
- CGF.EmitAutoVarCleanups(VarEmission);
- return VarEmission.getAllocatedAddress();
+ auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
+ auto *PrivateVD =
+ cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
+ bool IsRegistered = PrivateScope.addPrivate(VD, [&]() -> Address {
+ // Emit private VarDecl with copy init.
+ CGF.EmitVarDecl(*PrivateVD);
+ return CGF.GetAddrOfLocalVar(PrivateVD);
});
assert(IsRegistered && "linear var already registered as private");
// Silence the warning about unused variable.
(void)IsRegistered;
+ ++CurPrivate;
}
}
}
+static void emitSimdlenSafelenClause(CodeGenFunction &CGF,
+ const OMPExecutableDirective &D) {
+ if (!CGF.HaveInsertPoint())
+ return;
+ if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
+ RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
+ /*ignoreResult=*/true);
+ llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
+ CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
+ // In presence of finite 'safelen', it may be unsafe to mark all
+ // the memory instructions parallel, because loop-carried
+ // dependences of 'safelen' iterations are possible.
+ CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>());
+ } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
+ RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
+ /*ignoreResult=*/true);
+ llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
+ CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
+ // In presence of finite 'safelen', it may be unsafe to mark all
+ // the memory instructions parallel, because loop-carried
+ // dependences of 'safelen' iterations are possible.
+ CGF.LoopStack.setParallel(false);
+ }
+}
+
+void CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective &D) {
+ // Walk clauses and process safelen/lastprivate.
+ LoopStack.setParallel();
+ LoopStack.setVectorizeEnable(true);
+ emitSimdlenSafelenClause(*this, D);
+}
+
+void CodeGenFunction::EmitOMPSimdFinal(const OMPLoopDirective &D) {
+ if (!HaveInsertPoint())
+ return;
+ auto IC = D.counters().begin();
+ for (auto F : D.finals()) {
+ auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
+ if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD)) {
+ DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
+ CapturedStmtInfo->lookup(OrigVD) != nullptr,
+ (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
+ Address OrigAddr = EmitLValue(&DRE).getAddress();
+ OMPPrivateScope VarScope(*this);
+ VarScope.addPrivate(OrigVD,
+ [OrigAddr]() -> Address { return OrigAddr; });
+ (void)VarScope.Privatize();
+ EmitIgnoredExpr(F);
+ }
+ ++IC;
+ }
+ emitLinearClauseFinal(*this, D);
+}
+
void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) {
auto &&CodeGen = [&S](CodeGenFunction &CGF) {
- // Pragma 'simd' code depends on presence of 'lastprivate'.
- // If present, we have to separate last iteration of the loop:
- //
- // if (PreCond) {
- // for (IV in 0..LastIteration-1) BODY;
- // BODY with updates of lastprivate vars;
- // <Final counter/linear vars updates>;
- // }
- //
- // otherwise (when there's no lastprivate):
- //
// if (PreCond) {
// for (IV in 0..LastIteration) BODY;
// <Final counter/linear vars updates>;
@@ -733,43 +1202,6 @@
CGF.EmitBlock(ThenBlock);
CGF.incrementProfileCounter(&S);
}
- // Walk clauses and process safelen/lastprivate.
- bool SeparateIter = false;
- CGF.LoopStack.setParallel();
- CGF.LoopStack.setVectorizerEnable(true);
- for (auto C : S.clauses()) {
- switch (C->getClauseKind()) {
- case OMPC_safelen: {
- RValue Len = CGF.EmitAnyExpr(cast<OMPSafelenClause>(C)->getSafelen(),
- AggValueSlot::ignored(), true);
- llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
- CGF.LoopStack.setVectorizerWidth(Val->getZExtValue());
- // In presence of finite 'safelen', it may be unsafe to mark all
- // the memory instructions parallel, because loop-carried
- // dependences of 'safelen' iterations are possible.
- CGF.LoopStack.setParallel(false);
- break;
- }
- case OMPC_aligned:
- EmitOMPAlignedClause(CGF, CGF.CGM, cast<OMPAlignedClause>(*C));
- break;
- case OMPC_lastprivate:
- SeparateIter = true;
- break;
- default:
- // Not handled yet
- ;
- }
- }
-
- // Emit inits for the linear variables.
- for (auto &&I = S.getClausesOfKind(OMPC_linear); I; ++I) {
- auto *C = cast<OMPLinearClause>(*I);
- for (auto Init : C->inits()) {
- auto *D = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
- CGF.EmitVarDecl(*D);
- }
- }
// Emit the loop iteration variable.
const Expr *IVExpr = S.getIterationVariable();
@@ -786,34 +1218,32 @@
CGF.EmitIgnoredExpr(S.getCalcLastIteration());
}
- // Emit the linear steps for the linear clauses.
- // If a step is not constant, it is pre-calculated before the loop.
- for (auto &&I = S.getClausesOfKind(OMPC_linear); I; ++I) {
- auto *C = cast<OMPLinearClause>(*I);
- if (auto CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
- if (auto SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
- CGF.EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
- // Emit calculation of the linear step.
- CGF.EmitIgnoredExpr(CS);
- }
- }
+ CGF.EmitOMPSimdInit(S);
+ emitAlignedClause(CGF, S);
+ CGF.EmitOMPLinearClauseInit(S);
+ bool HasLastprivateClause;
{
OMPPrivateScope LoopScope(CGF);
- EmitPrivateLoopCounters(CGF, LoopScope, S.counters());
- EmitPrivateLinearVars(CGF, S, LoopScope);
+ emitPrivateLoopCounters(CGF, LoopScope, S.counters(),
+ S.private_counters());
+ emitPrivateLinearVars(CGF, S, LoopScope);
CGF.EmitOMPPrivateClause(S, LoopScope);
+ CGF.EmitOMPReductionClauseInit(S, LoopScope);
+ HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
(void)LoopScope.Privatize();
- CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(),
- S.getCond(SeparateIter), S.getInc(),
+ CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
+ S.getInc(),
[&S](CodeGenFunction &CGF) {
- CGF.EmitOMPLoopBody(S);
+ CGF.EmitOMPLoopBody(S, JumpDest());
CGF.EmitStopPoint(&S);
},
[](CodeGenFunction &) {});
- if (SeparateIter) {
- CGF.EmitOMPLoopBody(S, /*SeparateIter=*/true);
+ // Emit final copy of the lastprivate variables at the end of loops.
+ if (HasLastprivateClause) {
+ CGF.EmitOMPLastprivateClauseFinal(S);
}
+ CGF.EmitOMPReductionClauseFinal(S);
}
CGF.EmitOMPSimdFinal(S);
// Emit: if (PreCond) - end.
@@ -822,15 +1252,15 @@
CGF.EmitBlock(ContBlock, true);
}
};
- CGM.getOpenMPRuntime().emitInlinedDirective(*this, CodeGen);
+ CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
}
void CodeGenFunction::EmitOMPForOuterLoop(OpenMPScheduleClauseKind ScheduleKind,
const OMPLoopDirective &S,
OMPPrivateScope &LoopScope,
- bool Ordered, llvm::Value *LB,
- llvm::Value *UB, llvm::Value *ST,
- llvm::Value *IL, llvm::Value *Chunk) {
+ bool Ordered, Address LB,
+ Address UB, Address ST,
+ Address IL, llvm::Value *Chunk) {
auto &RT = CGM.getOpenMPRuntime();
// Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
@@ -894,11 +1324,14 @@
const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
- RT.emitForInit(
- *this, S.getLocStart(), ScheduleKind, IVSize, IVSigned, Ordered, IL, LB,
- (DynamicOrOrdered ? EmitAnyExpr(S.getLastIteration()).getScalarVal()
- : UB),
- ST, Chunk);
+ if (DynamicOrOrdered) {
+ llvm::Value *UBVal = EmitScalarExpr(S.getLastIteration());
+ RT.emitForDispatchInit(*this, S.getLocStart(), ScheduleKind,
+ IVSize, IVSigned, Ordered, UBVal, Chunk);
+ } else {
+ RT.emitForStaticInit(*this, S.getLocStart(), ScheduleKind,
+ IVSize, IVSigned, Ordered, IL, LB, UB, ST, Chunk);
+ }
auto LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
@@ -914,7 +1347,7 @@
// IV = LB
EmitIgnoredExpr(S.getInit());
// IV < UB
- BoolCondVal = EvaluateExprAsBool(S.getCond(false));
+ BoolCondVal = EvaluateExprAsBool(S.getCond());
} else {
BoolCondVal = RT.emitForNext(*this, S.getLocStart(), IVSize, IVSigned,
IL, LB, UB, ST);
@@ -943,25 +1376,28 @@
auto Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
+ // Generate !llvm.loop.parallel metadata for loads and stores for loops
+ // with dynamic/guided scheduling and without ordered clause.
+ if (!isOpenMPSimdDirective(S.getDirectiveKind())) {
+ LoopStack.setParallel((ScheduleKind == OMPC_SCHEDULE_dynamic ||
+ ScheduleKind == OMPC_SCHEDULE_guided) &&
+ !Ordered);
+ } else {
+ EmitOMPSimdInit(S);
+ }
+
SourceLocation Loc = S.getLocStart();
- // Generate !llvm.loop.parallel metadata for loads and stores for loops with
- // dynamic/guided scheduling and without ordered clause.
- LoopStack.setParallel((ScheduleKind == OMPC_SCHEDULE_dynamic ||
- ScheduleKind == OMPC_SCHEDULE_guided) &&
- !Ordered);
- EmitOMPInnerLoop(
- S, LoopScope.requiresCleanups(), S.getCond(/*SeparateIter=*/false),
- S.getInc(),
- [&S](CodeGenFunction &CGF) {
- CGF.EmitOMPLoopBody(S);
- CGF.EmitStopPoint(&S);
- },
- [Ordered, IVSize, IVSigned, Loc](CodeGenFunction &CGF) {
- if (Ordered) {
- CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(
- CGF, Loc, IVSize, IVSigned);
- }
- });
+ EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(),
+ [&S, LoopExit](CodeGenFunction &CGF) {
+ CGF.EmitOMPLoopBody(S, LoopExit);
+ CGF.EmitStopPoint(&S);
+ },
+ [Ordered, IVSize, IVSigned, Loc](CodeGenFunction &CGF) {
+ if (Ordered) {
+ CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(
+ CGF, Loc, IVSize, IVSigned);
+ }
+ });
EmitBlock(Continue.getBlock());
BreakContinueStack.pop_back();
@@ -995,8 +1431,7 @@
// Detect the loop schedule kind and chunk.
auto ScheduleKind = OMPC_SCHEDULE_unknown;
llvm::Value *Chunk = nullptr;
- if (auto *C =
- cast_or_null<OMPScheduleClause>(S.getSingleClause(OMPC_schedule))) {
+ if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
ScheduleKind = C->getScheduleKind();
if (const auto *Ch = C->getChunkSize()) {
if (auto *ImpRef = cast_or_null<DeclRefExpr>(C->getHelperChunkSize())) {
@@ -1005,8 +1440,8 @@
CGF.EmitVarDecl(*ImpVar);
CGF.EmitStoreThroughLValue(
CGF.EmitAnyExpr(Ch),
- CGF.MakeNaturalAlignAddrLValue(CGF.GetAddrOfLocalVar(ImpVar),
- ImpVar->getType()));
+ CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(ImpVar),
+ ImpVar->getType()));
} else {
Ch = ImpRef;
}
@@ -1014,7 +1449,8 @@
if (!C->getHelperChunkSize() || !OuterRegion) {
Chunk = CGF.EmitScalarExpr(Ch);
Chunk = CGF.EmitScalarConversion(Chunk, Ch->getType(),
- S.getIterationVariable()->getType());
+ S.getIterationVariable()->getType(),
+ S.getLocStart());
}
}
}
@@ -1057,6 +1493,9 @@
EmitBlock(ThenBlock);
incrementProfileCounter(&S);
}
+
+ emitAlignedClause(*this, S);
+ EmitOMPLinearClauseInit(S);
// Emit 'then' code.
{
// Emit helper vars inits.
@@ -1073,13 +1512,16 @@
if (EmitOMPFirstprivateClause(S, LoopScope)) {
// Emit implicit barrier to synchronize threads and avoid data races on
// initialization of firstprivate variables.
- CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(),
- OMPD_unknown);
+ CGM.getOpenMPRuntime().emitBarrierCall(
+ *this, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
+ /*ForceSimpleCall=*/true);
}
EmitOMPPrivateClause(S, LoopScope);
HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
EmitOMPReductionClauseInit(S, LoopScope);
- EmitPrivateLoopCounters(*this, LoopScope, S.counters());
+ emitPrivateLoopCounters(*this, LoopScope, S.counters(),
+ S.private_counters());
+ emitPrivateLinearVars(*this, S, LoopScope);
(void)LoopScope.Privatize();
// Detect the loop schedule kind and chunk.
@@ -1091,30 +1533,36 @@
ScheduleKind = ScheduleInfo.second;
const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
- const bool Ordered = S.getSingleClause(OMPC_ordered) != nullptr;
+ const bool Ordered = S.getSingleClause<OMPOrderedClause>() != nullptr;
if (RT.isStaticNonchunked(ScheduleKind,
/* Chunked */ Chunk != nullptr) &&
!Ordered) {
+ if (isOpenMPSimdDirective(S.getDirectiveKind())) {
+ EmitOMPSimdInit(S);
+ }
// OpenMP [2.7.1, Loop Construct, Description, table 2-1]
// When no chunk_size is specified, the iteration space is divided into
// chunks that are approximately equal in size, and at most one chunk is
// distributed to each thread. Note that the size of the chunks is
// unspecified in this case.
- RT.emitForInit(*this, S.getLocStart(), ScheduleKind, IVSize, IVSigned,
- Ordered, IL.getAddress(), LB.getAddress(),
- UB.getAddress(), ST.getAddress());
+ RT.emitForStaticInit(*this, S.getLocStart(), ScheduleKind,
+ IVSize, IVSigned, Ordered,
+ IL.getAddress(), LB.getAddress(),
+ UB.getAddress(), ST.getAddress());
+ auto LoopExit = getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
// UB = min(UB, GlobalUB);
EmitIgnoredExpr(S.getEnsureUpperBound());
// IV = LB;
EmitIgnoredExpr(S.getInit());
// while (idx <= UB) { BODY; ++idx; }
- EmitOMPInnerLoop(S, LoopScope.requiresCleanups(),
- S.getCond(/*SeparateIter=*/false), S.getInc(),
- [&S](CodeGenFunction &CGF) {
- CGF.EmitOMPLoopBody(S);
+ EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
+ S.getInc(),
+ [&S, LoopExit](CodeGenFunction &CGF) {
+ CGF.EmitOMPLoopBody(S, LoopExit);
CGF.EmitStopPoint(&S);
},
[](CodeGenFunction &) {});
+ EmitBlock(LoopExit.getBlock());
// Tell the runtime we are done.
RT.emitForStaticFinish(*this, S.getLocStart());
} else {
@@ -1130,6 +1578,9 @@
EmitOMPLastprivateClauseFinal(
S, Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getLocStart())));
}
+ if (isOpenMPSimdDirective(S.getDirectiveKind())) {
+ EmitOMPSimdFinal(S);
+ }
// We're now done with the loop, so jump to the continuation block.
if (ContBlock) {
EmitBranch(ContBlock);
@@ -1145,29 +1596,40 @@
auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF) {
HasLastprivates = CGF.EmitOMPWorksharingLoop(S);
};
- CGM.getOpenMPRuntime().emitInlinedDirective(*this, CodeGen);
+ CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
+ S.hasCancel());
// Emit an implicit barrier at the end.
- if (!S.getSingleClause(OMPC_nowait) || HasLastprivates) {
+ if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates) {
CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for);
}
}
-void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &) {
- llvm_unreachable("CodeGen for 'omp for simd' is not supported yet.");
+void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &S) {
+ LexicalScope Scope(*this, S.getSourceRange());
+ bool HasLastprivates = false;
+ auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF) {
+ HasLastprivates = CGF.EmitOMPWorksharingLoop(S);
+ };
+ CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
+
+ // Emit an implicit barrier at the end.
+ if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates) {
+ CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for);
+ }
}
static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty,
const Twine &Name,
llvm::Value *Init = nullptr) {
- auto LVal = CGF.MakeNaturalAlignAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
+ auto LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
if (Init)
CGF.EmitScalarInit(Init, LVal);
return LVal;
}
-static OpenMPDirectiveKind emitSections(CodeGenFunction &CGF,
- const OMPExecutableDirective &S) {
+OpenMPDirectiveKind
+CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
auto *Stmt = cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt();
auto *CS = dyn_cast<CompoundStmt>(Stmt);
if (CS && CS->size() > 1) {
@@ -1215,12 +1677,13 @@
CGF.EmitLoadOfLValue(IV, S.getLocStart()).getScalarVal(), ExitBB,
CS->size());
unsigned CaseNumber = 0;
- for (auto C = CS->children(); C; ++C, ++CaseNumber) {
+ for (auto *SubStmt : CS->children()) {
auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
CGF.EmitBlock(CaseBB);
SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
- CGF.EmitStmt(*C);
+ CGF.EmitStmt(SubStmt);
CGF.EmitBranch(ExitBB);
+ ++CaseNumber;
}
CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
};
@@ -1229,8 +1692,9 @@
if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
// Emit implicit barrier to synchronize threads and avoid data races on
// initialization of firstprivate variables.
- CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(),
- OMPD_unknown);
+ CGF.CGM.getOpenMPRuntime().emitBarrierCall(
+ CGF, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
+ /*ForceSimpleCall=*/true);
}
CGF.EmitOMPPrivateClause(S, LoopScope);
HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
@@ -1238,7 +1702,7 @@
(void)LoopScope.Privatize();
// Emit static non-chunked loop.
- CGF.CGM.getOpenMPRuntime().emitForInit(
+ CGF.CGM.getOpenMPRuntime().emitForStaticInit(
CGF, S.getLocStart(), OMPC_SCHEDULE_static, /*IVSize=*/32,
/*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(),
LB.getAddress(), UB.getAddress(), ST.getAddress());
@@ -1263,15 +1727,21 @@
CGF.EmitLoadOfScalar(IL, S.getLocStart())));
};
- CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, CodeGen);
+ bool HasCancel = false;
+ if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
+ HasCancel = OSD->hasCancel();
+ else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
+ HasCancel = OPSD->hasCancel();
+ CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
+ HasCancel);
// Emit barrier for lastprivates only if 'sections' directive has 'nowait'
// clause. Otherwise the barrier will be generated by the codegen for the
// directive.
- if (HasLastprivates && S.getSingleClause(OMPC_nowait)) {
+ if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
// Emit implicit barrier to synchronize threads and avoid data races on
// initialization of firstprivate variables.
- CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(),
- OMPD_unknown);
+ CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(),
+ OMPD_unknown);
}
return OMPD_sections;
}
@@ -1280,11 +1750,11 @@
bool HasFirstprivates;
// No need to generate reductions for sections with single section region, we
// can use original shared variables for all operations.
- bool HasReductions = !S.getClausesOfKind(OMPC_reduction).empty();
+ bool HasReductions = S.hasClausesOfKind<OMPReductionClause>();
// No need to generate lastprivates for sections with single section region,
// we can use original shared variable for all calculations with barrier at
// the end of the sections.
- bool HasLastprivates = !S.getClausesOfKind(OMPC_lastprivate).empty();
+ bool HasLastprivates = S.hasClausesOfKind<OMPLastprivateClause>();
auto &&CodeGen = [Stmt, &S, &HasFirstprivates](CodeGenFunction &CGF) {
CodeGenFunction::OMPPrivateScope SingleScope(CGF);
HasFirstprivates = CGF.EmitOMPFirstprivateClause(S, SingleScope);
@@ -1292,29 +1762,29 @@
(void)SingleScope.Privatize();
CGF.EmitStmt(Stmt);
- CGF.EnsureInsertPoint();
};
- CGF.CGM.getOpenMPRuntime().emitSingleRegion(CGF, CodeGen, S.getLocStart(),
- llvm::None, llvm::None,
- llvm::None, llvm::None);
+ CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getLocStart(),
+ llvm::None, llvm::None, llvm::None,
+ llvm::None);
// Emit barrier for firstprivates, lastprivates or reductions only if
// 'sections' directive has 'nowait' clause. Otherwise the barrier will be
// generated by the codegen for the directive.
if ((HasFirstprivates || HasLastprivates || HasReductions) &&
- S.getSingleClause(OMPC_nowait)) {
+ S.getSingleClause<OMPNowaitClause>()) {
// Emit implicit barrier to synchronize threads and avoid data races on
// initialization of firstprivate variables.
- CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(),
- OMPD_unknown);
+ CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_unknown,
+ /*EmitChecks=*/false,
+ /*ForceSimpleCall=*/true);
}
return OMPD_single;
}
void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) {
LexicalScope Scope(*this, S.getSourceRange());
- OpenMPDirectiveKind EmittedAs = emitSections(*this, S);
+ OpenMPDirectiveKind EmittedAs = EmitSections(S);
// Emit an implicit barrier at the end.
- if (!S.getSingleClause(OMPC_nowait)) {
+ if (!S.getSingleClause<OMPNowaitClause>()) {
CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), EmittedAs);
}
}
@@ -1323,9 +1793,9 @@
LexicalScope Scope(*this, S.getSourceRange());
auto &&CodeGen = [&S](CodeGenFunction &CGF) {
CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
- CGF.EnsureInsertPoint();
};
- CGM.getOpenMPRuntime().emitInlinedDirective(*this, CodeGen);
+ CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen,
+ S.hasCancel());
}
void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) {
@@ -1338,8 +1808,7 @@
// construct.
// Build a list of copyprivate variables along with helper expressions
// (<source>, <destination>, <destination>=<source> expressions)
- for (auto &&I = S.getClausesOfKind(OMPC_copyprivate); I; ++I) {
- auto *C = cast<OMPCopyprivateClause>(*I);
+ for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
DestExprs.append(C->destination_exprs().begin(),
C->destination_exprs().end());
@@ -1357,18 +1826,17 @@
(void)SingleScope.Privatize();
CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
- CGF.EnsureInsertPoint();
};
CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getLocStart(),
CopyprivateVars, DestExprs, SrcExprs,
AssignmentOps);
// Emit an implicit barrier at the end (to avoid data race on firstprivate
// init or if no 'nowait' clause was specified and no 'copyprivate' clause).
- if ((!S.getSingleClause(OMPC_nowait) || HasFirstprivates) &&
+ if ((!S.getSingleClause<OMPNowaitClause>() || HasFirstprivates) &&
CopyprivateVars.empty()) {
CGM.getOpenMPRuntime().emitBarrierCall(
*this, S.getLocStart(),
- S.getSingleClause(OMPC_nowait) ? OMPD_unknown : OMPD_single);
+ S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
}
}
@@ -1376,7 +1844,6 @@
LexicalScope Scope(*this, S.getSourceRange());
auto &&CodeGen = [&S](CodeGenFunction &CGF) {
CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
- CGF.EnsureInsertPoint();
};
CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getLocStart());
}
@@ -1385,10 +1852,13 @@
LexicalScope Scope(*this, S.getSourceRange());
auto &&CodeGen = [&S](CodeGenFunction &CGF) {
CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
- CGF.EnsureInsertPoint();
};
- CGM.getOpenMPRuntime().emitCriticalRegion(
- *this, S.getDirectiveName().getAsString(), CodeGen, S.getLocStart());
+ Expr *Hint = nullptr;
+ if (auto *HintClause = S.getSingleClause<OMPHintClause>())
+ Hint = HintClause->getHint();
+ CGM.getOpenMPRuntime().emitCriticalRegion(*this,
+ S.getDirectiveName().getAsString(),
+ CodeGen, S.getLocStart(), Hint);
}
void CodeGenFunction::EmitOMPParallelForDirective(
@@ -1399,18 +1869,20 @@
(void)emitScheduleClause(*this, S, /*OuterRegion=*/true);
auto &&CodeGen = [&S](CodeGenFunction &CGF) {
CGF.EmitOMPWorksharingLoop(S);
- // Emit implicit barrier at the end of parallel region, but this barrier
- // is at the end of 'for' directive, so emit it as the implicit barrier for
- // this 'for' directive.
- CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(),
- OMPD_parallel);
};
- emitCommonOMPParallelDirective(*this, S, CodeGen);
+ emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen);
}
void CodeGenFunction::EmitOMPParallelForSimdDirective(
- const OMPParallelForSimdDirective &) {
- llvm_unreachable("CodeGen for 'omp parallel for simd' is not supported yet.");
+ const OMPParallelForSimdDirective &S) {
+ // Emit directive as a combined directive that consists of two implicit
+ // directives: 'parallel' with 'for' directive.
+ LexicalScope Scope(*this, S.getSourceRange());
+ (void)emitScheduleClause(*this, S, /*OuterRegion=*/true);
+ auto &&CodeGen = [&S](CodeGenFunction &CGF) {
+ CGF.EmitOMPWorksharingLoop(S);
+ };
+ emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen);
}
void CodeGenFunction::EmitOMPParallelSectionsDirective(
@@ -1419,12 +1891,9 @@
// directives: 'parallel' with 'sections' directive.
LexicalScope Scope(*this, S.getSourceRange());
auto &&CodeGen = [&S](CodeGenFunction &CGF) {
- (void)emitSections(CGF, S);
- // Emit implicit barrier at the end of parallel region.
- CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(),
- OMPD_parallel);
+ (void)CGF.EmitSections(S);
};
- emitCommonOMPParallelDirective(*this, S, CodeGen);
+ emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen);
}
void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {
@@ -1440,8 +1909,7 @@
// Get list of private variables.
llvm::SmallVector<const Expr *, 8> PrivateVars;
llvm::SmallVector<const Expr *, 8> PrivateCopies;
- for (auto &&I = S.getClausesOfKind(OMPC_private); I; ++I) {
- auto *C = cast<OMPPrivateClause>(*I);
+ for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
auto IRef = C->varlist_begin();
for (auto *IInit : C->private_copies()) {
auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
@@ -1457,8 +1925,7 @@
llvm::SmallVector<const Expr *, 8> FirstprivateVars;
llvm::SmallVector<const Expr *, 8> FirstprivateCopies;
llvm::SmallVector<const Expr *, 8> FirstprivateInits;
- for (auto &&I = S.getClausesOfKind(OMPC_firstprivate); I; ++I) {
- auto *C = cast<OMPFirstprivateClause>(*I);
+ for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
auto IRef = C->varlist_begin();
auto IElemInitRef = C->inits().begin();
for (auto *IInit : C->private_copies()) {
@@ -1471,41 +1938,47 @@
++IRef, ++IElemInitRef;
}
}
+ // Build list of dependences.
+ llvm::SmallVector<std::pair<OpenMPDependClauseKind, const Expr *>, 8>
+ Dependences;
+ for (const auto *C : S.getClausesOfKind<OMPDependClause>()) {
+ for (auto *IRef : C->varlists()) {
+ Dependences.push_back(std::make_pair(C->getDependencyKind(), IRef));
+ }
+ }
auto &&CodeGen = [PartId, &S, &PrivateVars, &FirstprivateVars](
CodeGenFunction &CGF) {
// Set proper addresses for generated private copies.
auto *CS = cast<CapturedStmt>(S.getAssociatedStmt());
OMPPrivateScope Scope(CGF);
if (!PrivateVars.empty() || !FirstprivateVars.empty()) {
- auto *CopyFn = CGF.Builder.CreateAlignedLoad(
- CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(3)),
- CGF.PointerAlignInBytes);
- auto *PrivatesPtr = CGF.Builder.CreateAlignedLoad(
- CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(2)),
- CGF.PointerAlignInBytes);
+ auto *CopyFn = CGF.Builder.CreateLoad(
+ CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(3)));
+ auto *PrivatesPtr = CGF.Builder.CreateLoad(
+ CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(2)));
// Map privates.
- llvm::SmallVector<std::pair<const VarDecl *, llvm::Value *>, 16>
+ llvm::SmallVector<std::pair<const VarDecl *, Address>, 16>
PrivatePtrs;
llvm::SmallVector<llvm::Value *, 16> CallArgs;
CallArgs.push_back(PrivatesPtr);
for (auto *E : PrivateVars) {
auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
- auto *PrivatePtr =
+ Address PrivatePtr =
CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()));
PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
- CallArgs.push_back(PrivatePtr);
+ CallArgs.push_back(PrivatePtr.getPointer());
}
for (auto *E : FirstprivateVars) {
auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
- auto *PrivatePtr =
+ Address PrivatePtr =
CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()));
PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
- CallArgs.push_back(PrivatePtr);
+ CallArgs.push_back(PrivatePtr.getPointer());
}
CGF.EmitRuntimeCall(CopyFn, CallArgs);
for (auto &&Pair : PrivatePtrs) {
- auto *Replacement =
- CGF.Builder.CreateAlignedLoad(Pair.second, CGF.PointerAlignInBytes);
+ Address Replacement(CGF.Builder.CreateLoad(Pair.second),
+ CGF.getContext().getDeclAlign(Pair.first));
Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
}
}
@@ -1515,16 +1988,16 @@
}
CGF.EmitStmt(CS->getCapturedStmt());
};
- auto OutlinedFn =
- CGM.getOpenMPRuntime().emitTaskOutlinedFunction(S, *I, CodeGen);
+ auto OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
+ S, *I, OMPD_task, CodeGen);
// Check if we should emit tied or untied task.
- bool Tied = !S.getSingleClause(OMPC_untied);
+ bool Tied = !S.getSingleClause<OMPUntiedClause>();
// Check if the task is final
llvm::PointerIntPair<llvm::Value *, 1, bool> Final;
- if (auto *Clause = S.getSingleClause(OMPC_final)) {
+ if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
// If the condition constant folds and can be elided, try to avoid emitting
// the condition and the dead arm of the if/else.
- auto *Cond = cast<OMPFinalClause>(Clause)->getCondition();
+ auto *Cond = Clause->getCondition();
bool CondConstant;
if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
Final.setInt(CondConstant);
@@ -1536,13 +2009,17 @@
}
auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
const Expr *IfCond = nullptr;
- if (auto C = S.getSingleClause(OMPC_if)) {
- IfCond = cast<OMPIfClause>(C)->getCondition();
+ for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
+ if (C->getNameModifier() == OMPD_unknown ||
+ C->getNameModifier() == OMPD_task) {
+ IfCond = C->getCondition();
+ break;
+ }
}
CGM.getOpenMPRuntime().emitTaskCall(
*this, S.getLocStart(), S, Tied, Final, OutlinedFn, SharedsTy,
CapturedStruct, IfCond, PrivateVars, PrivateCopies, FirstprivateVars,
- FirstprivateCopies, FirstprivateInits);
+ FirstprivateCopies, FirstprivateInits, Dependences);
}
void CodeGenFunction::EmitOMPTaskyieldDirective(
@@ -1558,10 +2035,18 @@
CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getLocStart());
}
+void CodeGenFunction::EmitOMPTaskgroupDirective(
+ const OMPTaskgroupDirective &S) {
+ LexicalScope Scope(*this, S.getSourceRange());
+ auto &&CodeGen = [&S](CodeGenFunction &CGF) {
+ CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
+ };
+ CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getLocStart());
+}
+
void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) {
CGM.getOpenMPRuntime().emitFlush(*this, [&]() -> ArrayRef<const Expr *> {
- if (auto C = S.getSingleClause(/*K*/ OMPC_flush)) {
- auto FlushClause = cast<OMPFlushClause>(C);
+ if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>()) {
return llvm::makeArrayRef(FlushClause->varlist_begin(),
FlushClause->varlist_end());
}
@@ -1569,37 +2054,65 @@
}(), S.getLocStart());
}
+void CodeGenFunction::EmitOMPDistributeDirective(
+ const OMPDistributeDirective &S) {
+ llvm_unreachable("CodeGen for 'omp distribute' is not supported yet.");
+}
+
+static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM,
+ const CapturedStmt *S) {
+ CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
+ CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;
+ CGF.CapturedStmtInfo = &CapStmtInfo;
+ auto *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S);
+ Fn->addFnAttr(llvm::Attribute::NoInline);
+ return Fn;
+}
+
void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) {
+ if (!S.getAssociatedStmt())
+ return;
LexicalScope Scope(*this, S.getSourceRange());
- auto &&CodeGen = [&S](CodeGenFunction &CGF) {
- CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
- CGF.EnsureInsertPoint();
+ auto *C = S.getSingleClause<OMPSIMDClause>();
+ auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF) {
+ if (C) {
+ auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
+ llvm::SmallVector<llvm::Value *, 16> CapturedVars;
+ CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
+ auto *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS);
+ CGF.EmitNounwindRuntimeCall(OutlinedFn, CapturedVars);
+ } else {
+ CGF.EmitStmt(
+ cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
+ }
};
- CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getLocStart());
+ CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getLocStart(), !C);
}
static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val,
- QualType SrcType, QualType DestType) {
+ QualType SrcType, QualType DestType,
+ SourceLocation Loc) {
assert(CGF.hasScalarEvaluationKind(DestType) &&
"DestType must have scalar evaluation kind.");
assert(!Val.isAggregate() && "Must be a scalar or complex.");
return Val.isScalar()
- ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestType)
+ ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestType,
+ Loc)
: CGF.EmitComplexToScalarConversion(Val.getComplexVal(), SrcType,
- DestType);
+ DestType, Loc);
}
static CodeGenFunction::ComplexPairTy
convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType,
- QualType DestType) {
+ QualType DestType, SourceLocation Loc) {
assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
"DestType must have complex evaluation kind.");
CodeGenFunction::ComplexPairTy ComplexVal;
if (Val.isScalar()) {
// Convert the input element to the element type of the complex.
auto DestElementType = DestType->castAs<ComplexType>()->getElementType();
- auto ScalarVal =
- CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestElementType);
+ auto ScalarVal = CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
+ DestElementType, Loc);
ComplexVal = CodeGenFunction::ComplexPairTy(
ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
} else {
@@ -1607,9 +2120,9 @@
auto SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
auto DestElementType = DestType->castAs<ComplexType>()->getElementType();
ComplexVal.first = CGF.EmitScalarConversion(
- Val.getComplexVal().first, SrcElementType, DestElementType);
+ Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
ComplexVal.second = CGF.EmitScalarConversion(
- Val.getComplexVal().second, SrcElementType, DestElementType);
+ Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
}
return ComplexVal;
}
@@ -1626,16 +2139,16 @@
}
static void emitSimpleStore(CodeGenFunction &CGF, LValue LVal, RValue RVal,
- QualType RValTy) {
+ QualType RValTy, SourceLocation Loc) {
switch (CGF.getEvaluationKind(LVal.getType())) {
case TEK_Scalar:
- CGF.EmitStoreThroughLValue(
- RValue::get(convertToScalarValue(CGF, RVal, RValTy, LVal.getType())),
- LVal);
+ CGF.EmitStoreThroughLValue(RValue::get(convertToScalarValue(
+ CGF, RVal, RValTy, LVal.getType(), Loc)),
+ LVal);
break;
case TEK_Complex:
CGF.EmitStoreOfComplex(
- convertToComplexValue(CGF, RVal, RValTy, LVal.getType()), LVal,
+ convertToComplexValue(CGF, RVal, RValTy, LVal.getType(), Loc), LVal,
/*isInit=*/false);
break;
case TEK_Aggregate:
@@ -1663,7 +2176,7 @@
// list.
if (IsSeqCst)
CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
- emitSimpleStore(CGF,VLValue, Res, X->getType().getNonReferenceType());
+ emitSimpleStore(CGF, VLValue, Res, X->getType().getNonReferenceType(), Loc);
}
static void EmitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst,
@@ -1693,8 +2206,8 @@
!Update.getScalarVal()->getType()->isIntegerTy() ||
!X.isSimple() || (!isa<llvm::ConstantInt>(Update.getScalarVal()) &&
(Update.getScalarVal()->getType() !=
- X.getAddress()->getType()->getPointerElementType())) ||
- !X.getAddress()->getType()->getPointerElementType()->isIntegerTy() ||
+ X.getAddress().getElementType())) ||
+ !X.getAddress().getElementType()->isIntegerTy() ||
!Context.getTargetInfo().hasBuiltinAtomic(
Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
return std::make_pair(false, RValue::get(nullptr));
@@ -1765,10 +2278,10 @@
auto *UpdateVal = Update.getScalarVal();
if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
UpdateVal = CGF.Builder.CreateIntCast(
- IC, X.getAddress()->getType()->getPointerElementType(),
+ IC, X.getAddress().getElementType(),
X.getType()->hasSignedIntegerRepresentation());
}
- auto *Res = CGF.Builder.CreateAtomicRMW(RMWOp, X.getAddress(), UpdateVal, AO);
+ auto *Res = CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO);
return std::make_pair(true, RValue::get(Res));
}
@@ -1834,12 +2347,14 @@
}
static RValue convertToType(CodeGenFunction &CGF, RValue Value,
- QualType SourceType, QualType ResType) {
+ QualType SourceType, QualType ResType,
+ SourceLocation Loc) {
switch (CGF.getEvaluationKind(ResType)) {
case TEK_Scalar:
- return RValue::get(convertToScalarValue(CGF, Value, SourceType, ResType));
+ return RValue::get(
+ convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
case TEK_Complex: {
- auto Res = convertToComplexValue(CGF, Value, SourceType, ResType);
+ auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
return RValue::getComplex(Res.first, Res.second);
}
case TEK_Aggregate:
@@ -1904,7 +2419,7 @@
// 'x' is simply rewritten with some 'expr'.
NewVValType = X->getType().getNonReferenceType();
ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
- X->getType().getNonReferenceType());
+ X->getType().getNonReferenceType(), Loc);
auto &&Gen = [&CGF, &NewVVal, ExprRValue](RValue XRValue) -> RValue {
NewVVal = XRValue;
return ExprRValue;
@@ -1919,7 +2434,7 @@
}
}
// Emit post-update store to 'v' of old/new 'x' value.
- emitSimpleStore(CGF, VLValue, NewVVal, NewVValType);
+ emitSimpleStore(CGF, VLValue, NewVVal, NewVValType, Loc);
// OpenMP, 2.12.6, atomic Construct
// Any atomic construct with a seq_cst clause forces the atomically
// performed operation to include an implicit flush operation without a
@@ -1956,6 +2471,7 @@
case OMPC_lastprivate:
case OMPC_reduction:
case OMPC_safelen:
+ case OMPC_simdlen:
case OMPC_collapse:
case OMPC_default:
case OMPC_seq_cst:
@@ -1971,13 +2487,25 @@
case OMPC_nowait:
case OMPC_untied:
case OMPC_threadprivate:
+ case OMPC_depend:
case OMPC_mergeable:
+ case OMPC_device:
+ case OMPC_threads:
+ case OMPC_simd:
+ case OMPC_map:
+ case OMPC_num_teams:
+ case OMPC_thread_limit:
+ case OMPC_priority:
+ case OMPC_grainsize:
+ case OMPC_nogroup:
+ case OMPC_num_tasks:
+ case OMPC_hint:
llvm_unreachable("Clause is not allowed in 'omp atomic'.");
}
}
void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) {
- bool IsSeqCst = S.getSingleClause(/*K=*/OMPC_seq_cst);
+ bool IsSeqCst = S.getSingleClause<OMPSeqCstClause>();
OpenMPClauseKind Kind = OMPC_unknown;
for (auto *C : S.clauses()) {
// Find first clause (skip seq_cst clause, if it is first).
@@ -2002,18 +2530,104 @@
}
LexicalScope Scope(*this, S.getSourceRange());
- auto &&CodeGen = [&S, Kind, IsSeqCst](CodeGenFunction &CGF) {
+ auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF) {
+ CGF.EmitStopPoint(CS);
EmitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(),
S.getV(), S.getExpr(), S.getUpdateExpr(),
S.isXLHSInRHSPart(), S.getLocStart());
};
- CGM.getOpenMPRuntime().emitInlinedDirective(*this, CodeGen);
+ CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen);
}
-void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &) {
- llvm_unreachable("CodeGen for 'omp target' is not supported yet.");
+void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &S) {
+ LexicalScope Scope(*this, S.getSourceRange());
+ const CapturedStmt &CS = *cast<CapturedStmt>(S.getAssociatedStmt());
+
+ llvm::SmallVector<llvm::Value *, 16> CapturedVars;
+ GenerateOpenMPCapturedVars(CS, CapturedVars);
+
+ // Emit target region as a standalone region.
+ auto &&CodeGen = [&CS](CodeGenFunction &CGF) {
+ CGF.EmitStmt(CS.getCapturedStmt());
+ };
+
+ // Obtain the target region outlined function.
+ llvm::Value *Fn =
+ CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, CodeGen);
+
+ // Check if we have any if clause associated with the directive.
+ const Expr *IfCond = nullptr;
+
+ if (auto *C = S.getSingleClause<OMPIfClause>()) {
+ IfCond = C->getCondition();
+ }
+
+ // Check if we have any device clause associated with the directive.
+ const Expr *Device = nullptr;
+ if (auto *C = S.getSingleClause<OMPDeviceClause>()) {
+ Device = C->getDevice();
+ }
+
+ CGM.getOpenMPRuntime().emitTargetCall(*this, S, Fn, IfCond, Device,
+ CapturedVars);
}
void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &) {
llvm_unreachable("CodeGen for 'omp teams' is not supported yet.");
}
+
+void CodeGenFunction::EmitOMPCancellationPointDirective(
+ const OMPCancellationPointDirective &S) {
+ CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getLocStart(),
+ S.getCancelRegion());
+}
+
+void CodeGenFunction::EmitOMPCancelDirective(const OMPCancelDirective &S) {
+ const Expr *IfCond = nullptr;
+ for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
+ if (C->getNameModifier() == OMPD_unknown ||
+ C->getNameModifier() == OMPD_cancel) {
+ IfCond = C->getCondition();
+ break;
+ }
+ }
+ CGM.getOpenMPRuntime().emitCancelCall(*this, S.getLocStart(), IfCond,
+ S.getCancelRegion());
+}
+
+CodeGenFunction::JumpDest
+CodeGenFunction::getOMPCancelDestination(OpenMPDirectiveKind Kind) {
+ if (Kind == OMPD_parallel || Kind == OMPD_task)
+ return ReturnBlock;
+ assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
+ Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for);
+ return BreakContinueStack.back().BreakBlock;
+}
+
+// Generate the instructions for '#pragma omp target data' directive.
+void CodeGenFunction::EmitOMPTargetDataDirective(
+ const OMPTargetDataDirective &S) {
+ // emit the code inside the construct for now
+ auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
+ CGM.getOpenMPRuntime().emitInlinedDirective(
+ *this, OMPD_target_data,
+ [&CS](CodeGenFunction &CGF) { CGF.EmitStmt(CS->getCapturedStmt()); });
+}
+
+void CodeGenFunction::EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S) {
+ // emit the code inside the construct for now
+ auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
+ CGM.getOpenMPRuntime().emitInlinedDirective(
+ *this, OMPD_taskloop,
+ [&CS](CodeGenFunction &CGF) { CGF.EmitStmt(CS->getCapturedStmt()); });
+}
+
+void CodeGenFunction::EmitOMPTaskLoopSimdDirective(
+ const OMPTaskLoopSimdDirective &S) {
+ // emit the code inside the construct for now
+ auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
+ CGM.getOpenMPRuntime().emitInlinedDirective(
+ *this, OMPD_taskloop_simd,
+ [&CS](CodeGenFunction &CGF) { CGF.EmitStmt(CS->getCapturedStmt()); });
+}
+
diff --git a/lib/CodeGen/CGVTT.cpp b/lib/CodeGen/CGVTT.cpp
index e3df5a4..4fb7671 100644
--- a/lib/CodeGen/CGVTT.cpp
+++ b/lib/CodeGen/CGVTT.cpp
@@ -108,7 +108,6 @@
llvm::raw_svector_ostream Out(OutName);
cast<ItaniumMangleContext>(CGM.getCXXABI().getMangleContext())
.mangleCXXVTT(RD, Out);
- Out.flush();
StringRef Name = OutName.str();
// This will also defer the definition of the VTT.
diff --git a/lib/CodeGen/CGVTables.cpp b/lib/CodeGen/CGVTables.cpp
index 57370a6..c8f3add 100644
--- a/lib/CodeGen/CGVTables.cpp
+++ b/lib/CodeGen/CGVTables.cpp
@@ -44,7 +44,6 @@
Thunk.This, Out);
else
getCXXABI().getMangleContext().mangleThunk(MD, Thunk, Out);
- Out.flush();
llvm::Type *Ty = getTypes().GetFunctionTypeForVTable(GD);
return GetOrCreateLLVMFunction(Name, Ty, GD, /*ForVTable=*/true,
@@ -56,6 +55,21 @@
CGM.setGlobalVisibility(Fn, MD);
}
+static void setThunkProperties(CodeGenModule &CGM, const ThunkInfo &Thunk,
+ llvm::Function *ThunkFn, bool ForVTable,
+ GlobalDecl GD) {
+ CGM.setFunctionLinkage(GD, ThunkFn);
+ CGM.getCXXABI().setThunkLinkage(ThunkFn, ForVTable, GD,
+ !Thunk.Return.isEmpty());
+
+ // Set the right visibility.
+ const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
+ setThunkVisibility(CGM, MD, Thunk, ThunkFn);
+
+ if (CGM.supportsCOMDAT() && ThunkFn->isWeakForLinker())
+ ThunkFn->setComdat(CGM.getModule().getOrInsertComdat(ThunkFn->getName()));
+}
+
#ifndef NDEBUG
static bool similar(const ABIArgInfo &infoL, CanQualType typeL,
const ABIArgInfo &infoR, CanQualType typeR) {
@@ -88,8 +102,11 @@
CGF.EmitBlock(AdjustNotNull);
}
- ReturnValue = CGF.CGM.getCXXABI().performReturnAdjustment(CGF, ReturnValue,
- Thunk.Return);
+ auto ClassDecl = ResultType->getPointeeType()->getAsCXXRecordDecl();
+ auto ClassAlign = CGF.CGM.getClassPointerAlignment(ClassDecl);
+ ReturnValue = CGF.CGM.getCXXABI().performReturnAdjustment(CGF,
+ Address(ReturnValue, ClassAlign),
+ Thunk.Return);
if (NullCheckValue) {
CGF.Builder.CreateBr(AdjustEnd);
@@ -123,8 +140,8 @@
// no-op thunk for the regular definition) call va_start/va_end.
// There's a bit of per-call overhead for this solution, but it's
// better for codesize if the definition is long.
-void CodeGenFunction::GenerateVarArgsThunk(
- llvm::Function *Fn,
+llvm::Function *
+CodeGenFunction::GenerateVarArgsThunk(llvm::Function *Fn,
const CGFunctionInfo &FnInfo,
GlobalDecl GD, const ThunkInfo &Thunk) {
const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
@@ -157,33 +174,37 @@
// Find the first store of "this", which will be to the alloca associated
// with "this".
- llvm::Value *ThisPtr = &*AI;
- llvm::BasicBlock *EntryBB = Fn->begin();
- llvm::Instruction *ThisStore =
+ Address ThisPtr(&*AI, CGM.getClassPointerAlignment(MD->getParent()));
+ llvm::BasicBlock *EntryBB = &Fn->front();
+ llvm::BasicBlock::iterator ThisStore =
std::find_if(EntryBB->begin(), EntryBB->end(), [&](llvm::Instruction &I) {
- return isa<llvm::StoreInst>(I) && I.getOperand(0) == ThisPtr;
- });
- assert(ThisStore && "Store of this should be in entry block?");
+ return isa<llvm::StoreInst>(I) &&
+ I.getOperand(0) == ThisPtr.getPointer();
+ });
+ assert(ThisStore != EntryBB->end() &&
+ "Store of this should be in entry block?");
// Adjust "this", if necessary.
- Builder.SetInsertPoint(ThisStore);
+ Builder.SetInsertPoint(&*ThisStore);
llvm::Value *AdjustedThisPtr =
CGM.getCXXABI().performThisAdjustment(*this, ThisPtr, Thunk.This);
ThisStore->setOperand(0, AdjustedThisPtr);
if (!Thunk.Return.isEmpty()) {
// Fix up the returned value, if necessary.
- for (llvm::Function::iterator I = Fn->begin(), E = Fn->end(); I != E; I++) {
- llvm::Instruction *T = I->getTerminator();
+ for (llvm::BasicBlock &BB : *Fn) {
+ llvm::Instruction *T = BB.getTerminator();
if (isa<llvm::ReturnInst>(T)) {
RValue RV = RValue::get(T->getOperand(0));
T->eraseFromParent();
- Builder.SetInsertPoint(&*I);
+ Builder.SetInsertPoint(&BB);
RV = PerformReturnAdjustment(*this, ResultType, RV, Thunk);
Builder.CreateRet(RV.getScalarVal());
break;
}
}
}
+
+ return Fn;
}
void CodeGenFunction::StartThunk(llvm::Function *Fn, GlobalDecl GD,
@@ -219,6 +240,17 @@
// Since we didn't pass a GlobalDecl to StartFunction, do this ourselves.
CGM.getCXXABI().EmitInstanceFunctionProlog(*this);
CXXThisValue = CXXABIThisValue;
+ CurCodeDecl = MD;
+ CurFuncDecl = MD;
+}
+
+void CodeGenFunction::FinishThunk() {
+ // Clear these to restore the invariants expected by
+ // StartFunction/FinishFunction.
+ CurCodeDecl = nullptr;
+ CurFuncDecl = nullptr;
+
+ FinishFunction();
}
void CodeGenFunction::EmitCallAndReturnForThunk(llvm::Value *Callee,
@@ -228,9 +260,10 @@
const CXXMethodDecl *MD = cast<CXXMethodDecl>(CurGD.getDecl());
// Adjust the 'this' pointer if necessary
- llvm::Value *AdjustedThisPtr = Thunk ? CGM.getCXXABI().performThisAdjustment(
- *this, LoadCXXThis(), Thunk->This)
- : LoadCXXThis();
+ llvm::Value *AdjustedThisPtr =
+ Thunk ? CGM.getCXXABI().performThisAdjustment(
+ *this, LoadCXXThisAddress(), Thunk->This)
+ : LoadCXXThis();
if (CurFnInfo->usesInAlloca()) {
// We don't handle return adjusting thunks, because they require us to call
@@ -295,6 +328,8 @@
// Consider return adjustment if we have ThunkInfo.
if (Thunk && !Thunk->Return.isEmpty())
RV = PerformReturnAdjustment(*this, ResultType, RV, *Thunk);
+ else if (llvm::CallInst* Call = dyn_cast<llvm::CallInst>(CallOrInvoke))
+ Call->setTailCallKind(llvm::CallInst::TCK_Tail);
// Emit return.
if (!ResultType->isVoidType() && Slot.isNull())
@@ -303,7 +338,7 @@
// Disable the final ARC autorelease.
AutoreleaseResult = false;
- FinishFunction();
+ FinishThunk();
}
void CodeGenFunction::EmitMustTailThunk(const CXXMethodDecl *MD,
@@ -328,9 +363,8 @@
Args[ThisArgNo] = AdjustedThisPtr;
} else {
assert(ThisAI.isInAlloca() && "this is passed directly or inalloca");
- llvm::Value *ThisAddr = GetAddrOfLocalVar(CXXABIThisDecl);
- llvm::Type *ThisType =
- cast<llvm::PointerType>(ThisAddr->getType())->getElementType();
+ Address ThisAddr = GetAddrOfLocalVar(CXXABIThisDecl);
+ llvm::Type *ThisType = ThisAddr.getElementType();
if (ThisType != AdjustedThisPtr->getType())
AdjustedThisPtr = Builder.CreateBitCast(AdjustedThisPtr, ThisType);
Builder.CreateStore(AdjustedThisPtr, ThisAddr);
@@ -362,7 +396,7 @@
FinishFunction();
}
-void CodeGenFunction::GenerateThunk(llvm::Function *Fn,
+void CodeGenFunction::generateThunk(llvm::Function *Fn,
const CGFunctionInfo &FnInfo,
GlobalDecl GD, const ThunkInfo &Thunk) {
StartThunk(Fn, GD, FnInfo);
@@ -374,16 +408,6 @@
// Make the call and return the result.
EmitCallAndReturnForThunk(Callee, &Thunk);
-
- // Set the right linkage.
- CGM.setFunctionLinkage(GD, Fn);
-
- if (CGM.supportsCOMDAT() && Fn->isWeakForLinker())
- Fn->setComdat(CGM.getModule().getOrInsertComdat(Fn->getName()));
-
- // Set the right visibility.
- const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
- setThunkVisibility(CGM, MD, Thunk, Fn);
}
void CodeGenVTables::emitThunk(GlobalDecl GD, const ThunkInfo &Thunk,
@@ -437,8 +461,7 @@
return;
}
- // Change the linkage.
- CGM.setFunctionLinkage(GD, ThunkFn);
+ setThunkProperties(CGM, Thunk, ThunkFn, ForVTable, GD);
return;
}
@@ -450,17 +473,16 @@
// expensive/sucky at the moment, so don't generate the thunk unless
// we have to.
// FIXME: Do something better here; GenerateVarArgsThunk is extremely ugly.
- if (!UseAvailableExternallyLinkage) {
- CodeGenFunction(CGM).GenerateVarArgsThunk(ThunkFn, FnInfo, GD, Thunk);
- CGM.getCXXABI().setThunkLinkage(ThunkFn, ForVTable, GD,
- !Thunk.Return.isEmpty());
- }
+ if (UseAvailableExternallyLinkage)
+ return;
+ ThunkFn =
+ CodeGenFunction(CGM).GenerateVarArgsThunk(ThunkFn, FnInfo, GD, Thunk);
} else {
// Normal thunk body generation.
- CodeGenFunction(CGM).GenerateThunk(ThunkFn, FnInfo, GD, Thunk);
- CGM.getCXXABI().setThunkLinkage(ThunkFn, ForVTable, GD,
- !Thunk.Return.isEmpty());
+ CodeGenFunction(CGM).generateThunk(ThunkFn, FnInfo, GD, Thunk);
}
+
+ setThunkProperties(CGM, Thunk, ThunkFn, ForVTable, GD);
}
void CodeGenVTables::maybeEmitThunkForVTable(GlobalDecl GD,
@@ -497,8 +519,8 @@
if (!ThunkInfoVector)
return;
- for (unsigned I = 0, E = ThunkInfoVector->size(); I != E; ++I)
- emitThunk(GD, (*ThunkInfoVector)[I], /*ForVTable=*/false);
+ for (const ThunkInfo& Thunk : *ThunkInfoVector)
+ emitThunk(GD, Thunk, /*ForVTable=*/false);
}
llvm::Constant *CodeGenVTables::CreateVTableInitializer(
@@ -560,6 +582,24 @@
break;
}
+ if (CGM.getLangOpts().CUDA) {
+ // Emit NULL for methods we can't codegen on this
+ // side. Otherwise we'd end up with vtable with unresolved
+ // references.
+ const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
+ // OK on device side: functions w/ __device__ attribute
+ // OK on host side: anything except __device__-only functions.
+ bool CanEmitMethod = CGM.getLangOpts().CUDAIsDevice
+ ? MD->hasAttr<CUDADeviceAttr>()
+ : (MD->hasAttr<CUDAHostAttr>() ||
+ !MD->hasAttr<CUDADeviceAttr>());
+ if (!CanEmitMethod) {
+ Init = llvm::ConstantExpr::getNullValue(Int8PtrTy);
+ break;
+ }
+ // Method is acceptable, continue processing as usual.
+ }
+
if (cast<CXXMethodDecl>(GD.getDecl())->isPure()) {
// We have a pure virtual member function.
if (!PureVirtualFn) {
@@ -637,7 +677,6 @@
cast<ItaniumMangleContext>(CGM.getCXXABI().getMangleContext())
.mangleCXXCtorVTable(RD, Base.getBaseOffset().getQuantity(),
Base.getBase(), Out);
- Out.flush();
StringRef Name = OutName.str();
llvm::ArrayType *ArrayType =
@@ -674,6 +713,12 @@
return VTable;
}
+static bool shouldEmitAvailableExternallyVTable(const CodeGenModule &CGM,
+ const CXXRecordDecl *RD) {
+ return CGM.getCodeGenOpts().OptimizationLevel > 0 &&
+ CGM.getCXXABI().canSpeculativelyEmitVTable(RD);
+}
+
/// Compute the required linkage of the v-table for the given class.
///
/// Note that we only call this at the end of the translation unit.
@@ -695,14 +740,19 @@
switch (keyFunction->getTemplateSpecializationKind()) {
case TSK_Undeclared:
case TSK_ExplicitSpecialization:
- assert(def && "Should not have been asked to emit this");
+ assert((def || CodeGenOpts.OptimizationLevel > 0) &&
+ "Shouldn't query vtable linkage without key function or "
+ "optimizations");
+ if (!def && CodeGenOpts.OptimizationLevel > 0)
+ return llvm::GlobalVariable::AvailableExternallyLinkage;
+
if (keyFunction->isInlined())
return !Context.getLangOpts().AppleKext ?
llvm::GlobalVariable::LinkOnceODRLinkage :
llvm::Function::InternalLinkage;
return llvm::GlobalVariable::ExternalLinkage;
-
+
case TSK_ImplicitInstantiation:
return !Context.getLangOpts().AppleKext ?
llvm::GlobalVariable::LinkOnceODRLinkage :
@@ -737,16 +787,18 @@
}
switch (RD->getTemplateSpecializationKind()) {
- case TSK_Undeclared:
- case TSK_ExplicitSpecialization:
- case TSK_ImplicitInstantiation:
- return DiscardableODRLinkage;
+ case TSK_Undeclared:
+ case TSK_ExplicitSpecialization:
+ case TSK_ImplicitInstantiation:
+ return DiscardableODRLinkage;
- case TSK_ExplicitInstantiationDeclaration:
- return llvm::GlobalVariable::ExternalLinkage;
+ case TSK_ExplicitInstantiationDeclaration:
+ return shouldEmitAvailableExternallyVTable(*this, RD)
+ ? llvm::GlobalVariable::AvailableExternallyLinkage
+ : llvm::GlobalVariable::ExternalLinkage;
- case TSK_ExplicitInstantiationDefinition:
- return NonDiscardableODRLinkage;
+ case TSK_ExplicitInstantiationDefinition:
+ return NonDiscardableODRLinkage;
}
llvm_unreachable("Invalid TemplateSpecializationKind!");
@@ -814,7 +866,12 @@
/// we define that v-table?
static bool shouldEmitVTableAtEndOfTranslationUnit(CodeGenModule &CGM,
const CXXRecordDecl *RD) {
- return !CGM.getVTables().isVTableExternal(RD);
+ // If vtable is internal then it has to be done.
+ if (!CGM.getVTables().isVTableExternal(RD))
+ return true;
+
+ // If it's external then maybe we will need it as available_externally.
+ return shouldEmitAvailableExternallyVTable(CGM, RD);
}
/// Given that at some point we emitted a reference to one or more
@@ -827,19 +884,24 @@
size_t savedSize = DeferredVTables.size();
#endif
- typedef std::vector<const CXXRecordDecl *>::const_iterator const_iterator;
- for (const_iterator i = DeferredVTables.begin(),
- e = DeferredVTables.end(); i != e; ++i) {
- const CXXRecordDecl *RD = *i;
+ for (const CXXRecordDecl *RD : DeferredVTables)
if (shouldEmitVTableAtEndOfTranslationUnit(*this, RD))
VTables.GenerateClassData(RD);
- }
assert(savedSize == DeferredVTables.size() &&
"deferred extra v-tables during v-table emission?");
DeferredVTables.clear();
}
+bool CodeGenModule::IsCFIBlacklistedRecord(const CXXRecordDecl *RD) {
+ if (RD->hasAttr<UuidAttr>() &&
+ getContext().getSanitizerBlacklist().isBlacklistedType("attr:uuid"))
+ return true;
+
+ return getContext().getSanitizerBlacklist().isBlacklistedType(
+ RD->getQualifiedNameAsString());
+}
+
void CodeGenModule::EmitVTableBitSetEntries(llvm::GlobalVariable *VTable,
const VTableLayout &VTLayout) {
if (!LangOpts.Sanitize.has(SanitizerKind::CFIVCall) &&
@@ -848,59 +910,49 @@
!LangOpts.Sanitize.has(SanitizerKind::CFIUnrelatedCast))
return;
- llvm::Metadata *VTableMD = llvm::ConstantAsMetadata::get(VTable);
+ CharUnits PointerWidth =
+ Context.toCharUnitsFromBits(Context.getTargetInfo().getPointerWidth(0));
- std::vector<llvm::MDTuple *> BitsetEntries;
+ typedef std::pair<const CXXRecordDecl *, unsigned> BSEntry;
+ std::vector<BSEntry> BitsetEntries;
// Create a bit set entry for each address point.
for (auto &&AP : VTLayout.getAddressPoints()) {
- // FIXME: Add blacklisting scheme.
- if (AP.first.getBase()->isInStdNamespace())
+ if (IsCFIBlacklistedRecord(AP.first.getBase()))
continue;
- std::string OutName;
- llvm::raw_string_ostream Out(OutName);
- getCXXABI().getMangleContext().mangleCXXVTableBitSet(AP.first.getBase(),
- Out);
-
- CharUnits PointerWidth =
- Context.toCharUnitsFromBits(Context.getTargetInfo().getPointerWidth(0));
- uint64_t AddrPointOffset = AP.second * PointerWidth.getQuantity();
-
- llvm::Metadata *BitsetOps[] = {
- llvm::MDString::get(getLLVMContext(), Out.str()),
- VTableMD,
- llvm::ConstantAsMetadata::get(
- llvm::ConstantInt::get(Int64Ty, AddrPointOffset))};
- llvm::MDTuple *BitsetEntry =
- llvm::MDTuple::get(getLLVMContext(), BitsetOps);
- BitsetEntries.push_back(BitsetEntry);
+ BitsetEntries.push_back(std::make_pair(AP.first.getBase(), AP.second));
}
// Sort the bit set entries for determinism.
- std::sort(BitsetEntries.begin(), BitsetEntries.end(), [](llvm::MDTuple *T1,
- llvm::MDTuple *T2) {
- if (T1 == T2)
+ std::sort(BitsetEntries.begin(), BitsetEntries.end(),
+ [this](const BSEntry &E1, const BSEntry &E2) {
+ if (&E1 == &E2)
return false;
- StringRef S1 = cast<llvm::MDString>(T1->getOperand(0))->getString();
- StringRef S2 = cast<llvm::MDString>(T2->getOperand(0))->getString();
+ std::string S1;
+ llvm::raw_string_ostream O1(S1);
+ getCXXABI().getMangleContext().mangleTypeName(
+ QualType(E1.first->getTypeForDecl(), 0), O1);
+ O1.flush();
+
+ std::string S2;
+ llvm::raw_string_ostream O2(S2);
+ getCXXABI().getMangleContext().mangleTypeName(
+ QualType(E2.first->getTypeForDecl(), 0), O2);
+ O2.flush();
+
if (S1 < S2)
return true;
if (S1 != S2)
return false;
- uint64_t Offset1 = cast<llvm::ConstantInt>(
- cast<llvm::ConstantAsMetadata>(T1->getOperand(2))
- ->getValue())->getZExtValue();
- uint64_t Offset2 = cast<llvm::ConstantInt>(
- cast<llvm::ConstantAsMetadata>(T2->getOperand(2))
- ->getValue())->getZExtValue();
- assert(Offset1 != Offset2);
- return Offset1 < Offset2;
+ return E1.second < E2.second;
});
llvm::NamedMDNode *BitsetsMD =
getModule().getOrInsertNamedMetadata("llvm.bitsets");
for (auto BitsetEntry : BitsetEntries)
- BitsetsMD->addOperand(BitsetEntry);
+ CreateVTableBitSetEntry(BitsetsMD, VTable,
+ PointerWidth * BitsetEntry.second,
+ BitsetEntry.first);
}
diff --git a/lib/CodeGen/CGVTables.h b/lib/CodeGen/CGVTables.h
index e0195a2..c27e54a 100644
--- a/lib/CodeGen/CGVTables.h
+++ b/lib/CodeGen/CGVTables.h
@@ -85,10 +85,6 @@
uint64_t getSecondaryVirtualPointerIndex(const CXXRecordDecl *RD,
BaseSubobject Base);
- /// getAddressPoint - Get the address point of the given subobject in the
- /// class decl.
- uint64_t getAddressPoint(BaseSubobject Base, const CXXRecordDecl *RD);
-
/// GenerateConstructionVTable - Generate a construction vtable for the given
/// base subobject.
llvm::GlobalVariable *
diff --git a/lib/CodeGen/CGValue.h b/lib/CodeGen/CGValue.h
index 9205591..3ccc4cd 100644
--- a/lib/CodeGen/CGValue.h
+++ b/lib/CodeGen/CGValue.h
@@ -16,10 +16,10 @@
#define LLVM_CLANG_LIB_CODEGEN_CGVALUE_H
#include "clang/AST/ASTContext.h"
-#include "clang/AST/CharUnits.h"
#include "clang/AST/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/Type.h"
+#include "Address.h"
namespace llvm {
class Constant;
@@ -38,6 +38,10 @@
class RValue {
enum Flavor { Scalar, Complex, Aggregate };
+ // The shift to make to an aggregate's alignment to make it look
+ // like a pointer.
+ enum { AggAlignShift = 4 };
+
// Stores first value and flavor.
llvm::PointerIntPair<llvm::Value *, 2, Flavor> V1;
// Stores second value and volatility.
@@ -63,11 +67,21 @@
}
/// getAggregateAddr() - Return the Value* of the address of the aggregate.
- llvm::Value *getAggregateAddr() const {
+ Address getAggregateAddress() const {
+ assert(isAggregate() && "Not an aggregate!");
+ auto align = reinterpret_cast<uintptr_t>(V2.getPointer()) >> AggAlignShift;
+ return Address(V1.getPointer(), CharUnits::fromQuantity(align));
+ }
+ llvm::Value *getAggregatePointer() const {
assert(isAggregate() && "Not an aggregate!");
return V1.getPointer();
}
+ static RValue getIgnored() {
+ // FIXME: should we make this a more explicit state?
+ return get(nullptr);
+ }
+
static RValue get(llvm::Value *V) {
RValue ER;
ER.V1.setPointer(V);
@@ -89,11 +103,14 @@
// FIXME: Aggregate rvalues need to retain information about whether they are
// volatile or not. Remove default to find all places that probably get this
// wrong.
- static RValue getAggregate(llvm::Value *V, bool Volatile = false) {
+ static RValue getAggregate(Address addr, bool isVolatile = false) {
RValue ER;
- ER.V1.setPointer(V);
+ ER.V1.setPointer(addr.getPointer());
ER.V1.setInt(Aggregate);
- ER.V2.setInt(Volatile);
+
+ auto align = static_cast<uintptr_t>(addr.getAlignment().getQuantity());
+ ER.V2.setPointer(reinterpret_cast<llvm::Value*>(align << AggAlignShift));
+ ER.V2.setInt(isVolatile);
return ER;
}
};
@@ -103,6 +120,32 @@
ARCImpreciseLifetime, ARCPreciseLifetime
};
+/// The source of the alignment of an l-value; an expression of
+/// confidence in the alignment actually matching the estimate.
+enum class AlignmentSource {
+ /// The l-value was an access to a declared entity or something
+ /// equivalently strong, like the address of an array allocated by a
+ /// language runtime.
+ Decl,
+
+ /// The l-value was considered opaque, so the alignment was
+ /// determined from a type, but that type was an explicitly-aligned
+ /// typedef.
+ AttributedType,
+
+ /// The l-value was considered opaque, so the alignment was
+ /// determined from a type.
+ Type
+};
+
+/// Given that the base address has the given alignment source, what's
+/// our confidence in the alignment of the field?
+static inline AlignmentSource getFieldAlignmentSource(AlignmentSource Source) {
+ // For now, we don't distinguish fields of opaque pointers from
+ // top-level declarations, but maybe we should.
+ return AlignmentSource::Decl;
+}
+
/// LValue - This represents an lvalue references. Because C/C++ allow
/// bitfields, this is not a simple LLVM pointer, it may be a pointer plus a
/// bitrange.
@@ -157,6 +200,12 @@
// to make the default bitfield pattern all-zeroes.
bool ImpreciseLifetime : 1;
+ unsigned AlignSource : 2;
+
+ // This flag shows if a nontemporal load/stores should be used when accessing
+ // this lvalue.
+ bool Nontemporal : 1;
+
Expr *BaseIvarExp;
/// Used by struct-path-aware TBAA.
@@ -169,17 +218,21 @@
private:
void Initialize(QualType Type, Qualifiers Quals,
- CharUnits Alignment,
+ CharUnits Alignment, AlignmentSource AlignSource,
llvm::MDNode *TBAAInfo = nullptr) {
+ assert((!Alignment.isZero() || Type->isIncompleteType()) &&
+ "initializing l-value with zero alignment!");
this->Type = Type;
this->Quals = Quals;
this->Alignment = Alignment.getQuantity();
assert(this->Alignment == Alignment.getQuantity() &&
"Alignment exceeds allowed max!");
+ this->AlignSource = unsigned(AlignSource);
// Initialize Objective-C flags.
this->Ivar = this->ObjIsArray = this->NonGC = this->GlobalObjCRef = false;
this->ImpreciseLifetime = false;
+ this->Nontemporal = false;
this->ThreadLocalRef = false;
this->BaseIvarExp = nullptr;
@@ -229,6 +282,8 @@
void setARCPreciseLifetime(ARCPreciseLifetime_t value) {
ImpreciseLifetime = (value == ARCImpreciseLifetime);
}
+ bool isNontemporal() const { return Nontemporal; }
+ void setNontemporal(bool Value) { Nontemporal = Value; }
bool isObjCWeak() const {
return Quals.getObjCGCAttr() == Qualifiers::Weak;
@@ -261,29 +316,50 @@
CharUnits getAlignment() const { return CharUnits::fromQuantity(Alignment); }
void setAlignment(CharUnits A) { Alignment = A.getQuantity(); }
+ AlignmentSource getAlignmentSource() const {
+ return AlignmentSource(AlignSource);
+ }
+ void setAlignmentSource(AlignmentSource Source) {
+ AlignSource = unsigned(Source);
+ }
+
// simple lvalue
- llvm::Value *getAddress() const { assert(isSimple()); return V; }
- void setAddress(llvm::Value *address) {
+ llvm::Value *getPointer() const {
assert(isSimple());
- V = address;
+ return V;
+ }
+ Address getAddress() const { return Address(getPointer(), getAlignment()); }
+ void setAddress(Address address) {
+ assert(isSimple());
+ V = address.getPointer();
+ Alignment = address.getAlignment().getQuantity();
}
// vector elt lvalue
- llvm::Value *getVectorAddr() const { assert(isVectorElt()); return V; }
+ Address getVectorAddress() const {
+ return Address(getVectorPointer(), getAlignment());
+ }
+ llvm::Value *getVectorPointer() const { assert(isVectorElt()); return V; }
llvm::Value *getVectorIdx() const { assert(isVectorElt()); return VectorIdx; }
// extended vector elements.
- llvm::Value *getExtVectorAddr() const { assert(isExtVectorElt()); return V; }
+ Address getExtVectorAddress() const {
+ return Address(getExtVectorPointer(), getAlignment());
+ }
+ llvm::Value *getExtVectorPointer() const {
+ assert(isExtVectorElt());
+ return V;
+ }
llvm::Constant *getExtVectorElts() const {
assert(isExtVectorElt());
return VectorElts;
}
// bitfield lvalue
- llvm::Value *getBitFieldAddr() const {
- assert(isBitField());
- return V;
+ Address getBitFieldAddress() const {
+ return Address(getBitFieldPointer(), getAlignment());
}
+ llvm::Value *getBitFieldPointer() const { assert(isBitField()); return V; }
const CGBitFieldInfo &getBitFieldInfo() const {
assert(isBitField());
return *BitFieldInfo;
@@ -292,37 +368,40 @@
// global register lvalue
llvm::Value *getGlobalReg() const { assert(isGlobalReg()); return V; }
- static LValue MakeAddr(llvm::Value *address, QualType type,
- CharUnits alignment, ASTContext &Context,
+ static LValue MakeAddr(Address address, QualType type,
+ ASTContext &Context,
+ AlignmentSource alignSource,
llvm::MDNode *TBAAInfo = nullptr) {
Qualifiers qs = type.getQualifiers();
qs.setObjCGCAttr(Context.getObjCGCAttrKind(type));
LValue R;
R.LVType = Simple;
- assert(address->getType()->isPointerTy());
- R.V = address;
- R.Initialize(type, qs, alignment, TBAAInfo);
+ assert(address.getPointer()->getType()->isPointerTy());
+ R.V = address.getPointer();
+ R.Initialize(type, qs, address.getAlignment(), alignSource, TBAAInfo);
return R;
}
- static LValue MakeVectorElt(llvm::Value *Vec, llvm::Value *Idx,
- QualType type, CharUnits Alignment) {
+ static LValue MakeVectorElt(Address vecAddress, llvm::Value *Idx,
+ QualType type, AlignmentSource alignSource) {
LValue R;
R.LVType = VectorElt;
- R.V = Vec;
+ R.V = vecAddress.getPointer();
R.VectorIdx = Idx;
- R.Initialize(type, type.getQualifiers(), Alignment);
+ R.Initialize(type, type.getQualifiers(), vecAddress.getAlignment(),
+ alignSource);
return R;
}
- static LValue MakeExtVectorElt(llvm::Value *Vec, llvm::Constant *Elts,
- QualType type, CharUnits Alignment) {
+ static LValue MakeExtVectorElt(Address vecAddress, llvm::Constant *Elts,
+ QualType type, AlignmentSource alignSource) {
LValue R;
R.LVType = ExtVectorElt;
- R.V = Vec;
+ R.V = vecAddress.getPointer();
R.VectorElts = Elts;
- R.Initialize(type, type.getQualifiers(), Alignment);
+ R.Initialize(type, type.getQualifiers(), vecAddress.getAlignment(),
+ alignSource);
return R;
}
@@ -332,29 +411,28 @@
/// bit-field refers to.
/// \param Info - The information describing how to perform the bit-field
/// access.
- static LValue MakeBitfield(llvm::Value *Addr,
+ static LValue MakeBitfield(Address Addr,
const CGBitFieldInfo &Info,
- QualType type, CharUnits Alignment) {
+ QualType type,
+ AlignmentSource alignSource) {
LValue R;
R.LVType = BitField;
- R.V = Addr;
+ R.V = Addr.getPointer();
R.BitFieldInfo = &Info;
- R.Initialize(type, type.getQualifiers(), Alignment);
+ R.Initialize(type, type.getQualifiers(), Addr.getAlignment(), alignSource);
return R;
}
- static LValue MakeGlobalReg(llvm::Value *Reg,
- QualType type,
- CharUnits Alignment) {
+ static LValue MakeGlobalReg(Address Reg, QualType type) {
LValue R;
R.LVType = GlobalReg;
- R.V = Reg;
- R.Initialize(type, type.getQualifiers(), Alignment);
+ R.V = Reg.getPointer();
+ R.Initialize(type, type.getQualifiers(), Reg.getAlignment(),
+ AlignmentSource::Decl);
return R;
}
RValue asAggregateRValue() const {
- // FIMXE: Alignment
return RValue::getAggregate(getAddress(), isVolatileQualified());
}
};
@@ -407,7 +485,7 @@
/// ignored - Returns an aggregate value slot indicating that the
/// aggregate value is being ignored.
static AggValueSlot ignored() {
- return forAddr(nullptr, CharUnits(), Qualifiers(), IsNotDestructed,
+ return forAddr(Address::invalid(), Qualifiers(), IsNotDestructed,
DoesNotNeedGCBarriers, IsNotAliased);
}
@@ -421,15 +499,20 @@
/// for calling destructors on this object
/// \param needsGC - true if the slot is potentially located
/// somewhere that ObjC GC calls should be emitted for
- static AggValueSlot forAddr(llvm::Value *addr, CharUnits align,
+ static AggValueSlot forAddr(Address addr,
Qualifiers quals,
IsDestructed_t isDestructed,
NeedsGCBarriers_t needsGC,
IsAliased_t isAliased,
IsZeroed_t isZeroed = IsNotZeroed) {
AggValueSlot AV;
- AV.Addr = addr;
- AV.Alignment = align.getQuantity();
+ if (addr.isValid()) {
+ AV.Addr = addr.getPointer();
+ AV.Alignment = addr.getAlignment().getQuantity();
+ } else {
+ AV.Addr = nullptr;
+ AV.Alignment = 0;
+ }
AV.Quals = quals;
AV.DestructedFlag = isDestructed;
AV.ObjCGCFlag = needsGC;
@@ -443,7 +526,7 @@
NeedsGCBarriers_t needsGC,
IsAliased_t isAliased,
IsZeroed_t isZeroed = IsNotZeroed) {
- return forAddr(LV.getAddress(), LV.getAlignment(),
+ return forAddr(LV.getAddress(),
LV.getQuals(), isDestructed, needsGC, isAliased, isZeroed);
}
@@ -471,11 +554,15 @@
NeedsGCBarriers_t requiresGCollection() const {
return NeedsGCBarriers_t(ObjCGCFlag);
}
-
- llvm::Value *getAddr() const {
+
+ llvm::Value *getPointer() const {
return Addr;
}
+ Address getAddress() const {
+ return Address(Addr, getAlignment());
+ }
+
bool isIgnored() const {
return Addr == nullptr;
}
@@ -488,9 +575,12 @@
return IsAliased_t(AliasedFlag);
}
- // FIXME: Alignment?
RValue asRValue() const {
- return RValue::getAggregate(getAddr(), isVolatile());
+ if (isIgnored()) {
+ return RValue::getIgnored();
+ } else {
+ return RValue::getAggregate(getAddress(), isVolatile());
+ }
}
void setZeroed(bool V = true) { ZeroedFlag = V; }
diff --git a/lib/CodeGen/CMakeLists.txt b/lib/CodeGen/CMakeLists.txt
index 5a060b3..10bda76 100644
--- a/lib/CodeGen/CMakeLists.txt
+++ b/lib/CodeGen/CMakeLists.txt
@@ -10,6 +10,7 @@
Linker
MC
ObjCARCOpts
+ Object
ProfileData
ScalarOpts
Support
@@ -71,6 +72,7 @@
ItaniumCXXABI.cpp
MicrosoftCXXABI.cpp
ModuleBuilder.cpp
+ ObjectFilePCHContainerOperations.cpp
SanitizerMetadata.cpp
TargetInfo.cpp
diff --git a/lib/CodeGen/CodeGenABITypes.cpp b/lib/CodeGen/CodeGenABITypes.cpp
index 12189ae..643c996 100644
--- a/lib/CodeGen/CodeGenABITypes.cpp
+++ b/lib/CodeGen/CodeGenABITypes.cpp
@@ -20,24 +20,22 @@
#include "CodeGenModule.h"
#include "clang/CodeGen/CGFunctionInfo.h"
#include "clang/Frontend/CodeGenOptions.h"
+#include "clang/Lex/HeaderSearchOptions.h"
+#include "clang/Lex/PreprocessorOptions.h"
using namespace clang;
using namespace CodeGen;
-CodeGenABITypes::CodeGenABITypes(ASTContext &C,
- llvm::Module &M,
- const llvm::DataLayout &TD,
+CodeGenABITypes::CodeGenABITypes(ASTContext &C, llvm::Module &M,
CoverageSourceInfo *CoverageInfo)
- : CGO(new CodeGenOptions),
- CGM(new CodeGen::CodeGenModule(C, *CGO, M, TD, C.getDiagnostics(),
- CoverageInfo)) {
-}
+ : CGO(new CodeGenOptions), HSO(new HeaderSearchOptions),
+ PPO(new PreprocessorOptions),
+ CGM(new CodeGen::CodeGenModule(C, *HSO, *PPO, *CGO, M, C.getDiagnostics(),
+ CoverageInfo)) {}
-CodeGenABITypes::~CodeGenABITypes()
-{
- delete CGO;
- delete CGM;
-}
+// Explicitly out-of-line because ~CodeGenModule() is private but
+// CodeGenABITypes.h is part of clang's API.
+CodeGenABITypes::~CodeGenABITypes() = default;
const CGFunctionInfo &
CodeGenABITypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,
@@ -46,8 +44,9 @@
}
const CGFunctionInfo &
-CodeGenABITypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> Ty) {
- return CGM->getTypes().arrangeFreeFunctionType(Ty);
+CodeGenABITypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> Ty,
+ const FunctionDecl *FD) {
+ return CGM->getTypes().arrangeFreeFunctionType(Ty, FD);
}
const CGFunctionInfo &
@@ -57,15 +56,14 @@
const CGFunctionInfo &
CodeGenABITypes::arrangeCXXMethodType(const CXXRecordDecl *RD,
- const FunctionProtoType *FTP) {
- return CGM->getTypes().arrangeCXXMethodType(RD, FTP);
+ const FunctionProtoType *FTP,
+ const CXXMethodDecl *MD) {
+ return CGM->getTypes().arrangeCXXMethodType(RD, FTP, MD);
}
-const CGFunctionInfo &
-CodeGenABITypes::arrangeFreeFunctionCall(CanQualType returnType,
- ArrayRef<CanQualType> argTypes,
- FunctionType::ExtInfo info,
- RequiredArgs args) {
+const CGFunctionInfo &CodeGenABITypes::arrangeFreeFunctionCall(
+ CanQualType returnType, ArrayRef<CanQualType> argTypes,
+ FunctionType::ExtInfo info, RequiredArgs args) {
return CGM->getTypes().arrangeLLVMFunctionInfo(
returnType, /*IsInstanceMethod=*/false, /*IsChainCall=*/false, argTypes,
info, args);
diff --git a/lib/CodeGen/CodeGenAction.cpp b/lib/CodeGen/CodeGenAction.cpp
index 7e82fcc..abef543 100644
--- a/lib/CodeGen/CodeGenAction.cpp
+++ b/lib/CodeGen/CodeGenAction.cpp
@@ -26,10 +26,12 @@
#include "llvm/IR/DebugInfo.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/DiagnosticPrinter.h"
+#include "llvm/IR/FunctionInfo.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Module.h"
#include "llvm/IRReader/IRReader.h"
#include "llvm/Linker/Linker.h"
+#include "llvm/Object/FunctionIndexObjectFile.h"
#include "llvm/Pass.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/SourceMgr.h"
@@ -53,37 +55,47 @@
std::unique_ptr<CodeGenerator> Gen;
- std::unique_ptr<llvm::Module> TheModule, LinkModule;
+ std::unique_ptr<llvm::Module> TheModule;
+ SmallVector<std::pair<unsigned, std::unique_ptr<llvm::Module>>, 4>
+ LinkModules;
+
+ // This is here so that the diagnostic printer knows the module a diagnostic
+ // refers to.
+ llvm::Module *CurLinkModule = nullptr;
public:
- BackendConsumer(BackendAction Action, DiagnosticsEngine &Diags,
- const CodeGenOptions &CodeGenOpts,
- const TargetOptions &TargetOpts,
- const LangOptions &LangOpts, bool TimePasses,
- const std::string &InFile, llvm::Module *LinkModule,
- raw_pwrite_stream *OS, LLVMContext &C,
- CoverageSourceInfo *CoverageInfo = nullptr)
+ BackendConsumer(
+ BackendAction Action, DiagnosticsEngine &Diags,
+ const HeaderSearchOptions &HeaderSearchOpts,
+ const PreprocessorOptions &PPOpts, const CodeGenOptions &CodeGenOpts,
+ const TargetOptions &TargetOpts, const LangOptions &LangOpts,
+ bool TimePasses, const std::string &InFile,
+ const SmallVectorImpl<std::pair<unsigned, llvm::Module *>> &LinkModules,
+ raw_pwrite_stream *OS, LLVMContext &C,
+ CoverageSourceInfo *CoverageInfo = nullptr)
: Diags(Diags), Action(Action), CodeGenOpts(CodeGenOpts),
TargetOpts(TargetOpts), LangOpts(LangOpts), AsmOutStream(OS),
Context(nullptr), LLVMIRGeneration("LLVM IR Generation Time"),
- Gen(CreateLLVMCodeGen(Diags, InFile, CodeGenOpts, C, CoverageInfo)),
- LinkModule(LinkModule) {
+ Gen(CreateLLVMCodeGen(Diags, InFile, HeaderSearchOpts, PPOpts,
+ CodeGenOpts, C, CoverageInfo)) {
llvm::TimePassesIsEnabled = TimePasses;
+ for (auto &I : LinkModules)
+ this->LinkModules.push_back(
+ std::make_pair(I.first, std::unique_ptr<llvm::Module>(I.second)));
}
-
std::unique_ptr<llvm::Module> takeModule() { return std::move(TheModule); }
- llvm::Module *takeLinkModule() { return LinkModule.release(); }
+ void releaseLinkModules() {
+ for (auto &I : LinkModules)
+ I.second.release();
+ }
void HandleCXXStaticMemberVarInstantiation(VarDecl *VD) override {
Gen->HandleCXXStaticMemberVarInstantiation(VD);
}
void Initialize(ASTContext &Ctx) override {
- if (Context) {
- assert(Context == &Ctx);
- return;
- }
-
+ assert(!Context && "initialized multiple times");
+
Context = &Ctx;
if (llvm::TimePassesIsEnabled)
@@ -155,14 +167,6 @@
assert(TheModule.get() == M &&
"Unexpected module change during IR generation");
- // Link LinkModule into this module if present, preserving its validity.
- if (LinkModule) {
- if (Linker::LinkModules(
- M, LinkModule.get(),
- [=](const DiagnosticInfo &DI) { linkerDiagnosticHandler(DI); }))
- return;
- }
-
// Install an inline asm handler so that diagnostics get printed through
// our diagnostics hooks.
LLVMContext &Ctx = TheModule->getContext();
@@ -176,8 +180,16 @@
void *OldDiagnosticContext = Ctx.getDiagnosticContext();
Ctx.setDiagnosticHandler(DiagnosticHandler, this);
+ // Link LinkModule into this module if present, preserving its validity.
+ for (auto &I : LinkModules) {
+ unsigned LinkFlags = I.first;
+ CurLinkModule = I.second.get();
+ if (Linker::linkModules(*M, std::move(I.second), LinkFlags))
+ return;
+ }
+
EmitBackendOutput(Diags, CodeGenOpts, TargetOpts, LangOpts,
- C.getTargetInfo().getTargetDescription(),
+ C.getTargetInfo().getDataLayoutString(),
TheModule.get(), Action, AsmOutStream);
Ctx.setInlineAsmDiagnosticHandler(OldHandler, OldContext);
@@ -223,8 +235,6 @@
((BackendConsumer*)Context)->InlineAsmDiagHandler2(SM, Loc);
}
- void linkerDiagnosticHandler(const llvm::DiagnosticInfo &DI);
-
static void DiagnosticHandler(const llvm::DiagnosticInfo &DI,
void *Context) {
((BackendConsumer *)Context)->DiagnosticHandlerImpl(DI);
@@ -253,6 +263,10 @@
const llvm::DiagnosticInfoOptimizationRemarkMissed &D);
void OptimizationRemarkHandler(
const llvm::DiagnosticInfoOptimizationRemarkAnalysis &D);
+ void OptimizationRemarkHandler(
+ const llvm::DiagnosticInfoOptimizationRemarkAnalysisFPCommute &D);
+ void OptimizationRemarkHandler(
+ const llvm::DiagnosticInfoOptimizationRemarkAnalysisAliasing &D);
void OptimizationFailureHandler(
const llvm::DiagnosticInfoOptimizationFailure &D);
};
@@ -330,8 +344,7 @@
DiagnosticBuilder B = Diags.Report(Loc, diag::note_fe_inline_asm_here);
// Convert the SMDiagnostic ranges into SourceRange and attach them
// to the diagnostic.
- for (unsigned i = 0, e = D.getRanges().size(); i != e; ++i) {
- std::pair<unsigned, unsigned> Range = D.getRanges()[i];
+ for (const std::pair<unsigned, unsigned> &Range : D.getRanges()) {
unsigned Column = D.getColumnNo();
B << SourceRange(Loc.getLocWithOffset(Range.first - Column),
Loc.getLocWithOffset(Range.second - Column));
@@ -489,35 +502,48 @@
void BackendConsumer::OptimizationRemarkHandler(
const llvm::DiagnosticInfoOptimizationRemarkAnalysis &D) {
- // Optimization analysis remarks are active only if the -Rpass-analysis
- // flag has a regular expression that matches the name of the pass
- // name in \p D.
- if (CodeGenOpts.OptimizationRemarkAnalysisPattern &&
- CodeGenOpts.OptimizationRemarkAnalysisPattern->match(D.getPassName()))
+ // Optimization analysis remarks are active if the pass name is set to
+ // llvm::DiagnosticInfo::AlwasyPrint or if the -Rpass-analysis flag has a
+ // regular expression that matches the name of the pass name in \p D.
+
+ if (D.getPassName() == llvm::DiagnosticInfo::AlwaysPrint ||
+ (CodeGenOpts.OptimizationRemarkAnalysisPattern &&
+ CodeGenOpts.OptimizationRemarkAnalysisPattern->match(D.getPassName())))
EmitOptimizationMessage(
D, diag::remark_fe_backend_optimization_remark_analysis);
}
+void BackendConsumer::OptimizationRemarkHandler(
+ const llvm::DiagnosticInfoOptimizationRemarkAnalysisFPCommute &D) {
+ // Optimization analysis remarks are active if the pass name is set to
+ // llvm::DiagnosticInfo::AlwasyPrint or if the -Rpass-analysis flag has a
+ // regular expression that matches the name of the pass name in \p D.
+
+ if (D.getPassName() == llvm::DiagnosticInfo::AlwaysPrint ||
+ (CodeGenOpts.OptimizationRemarkAnalysisPattern &&
+ CodeGenOpts.OptimizationRemarkAnalysisPattern->match(D.getPassName())))
+ EmitOptimizationMessage(
+ D, diag::remark_fe_backend_optimization_remark_analysis_fpcommute);
+}
+
+void BackendConsumer::OptimizationRemarkHandler(
+ const llvm::DiagnosticInfoOptimizationRemarkAnalysisAliasing &D) {
+ // Optimization analysis remarks are active if the pass name is set to
+ // llvm::DiagnosticInfo::AlwasyPrint or if the -Rpass-analysis flag has a
+ // regular expression that matches the name of the pass name in \p D.
+
+ if (D.getPassName() == llvm::DiagnosticInfo::AlwaysPrint ||
+ (CodeGenOpts.OptimizationRemarkAnalysisPattern &&
+ CodeGenOpts.OptimizationRemarkAnalysisPattern->match(D.getPassName())))
+ EmitOptimizationMessage(
+ D, diag::remark_fe_backend_optimization_remark_analysis_aliasing);
+}
+
void BackendConsumer::OptimizationFailureHandler(
const llvm::DiagnosticInfoOptimizationFailure &D) {
EmitOptimizationMessage(D, diag::warn_fe_backend_optimization_failure);
}
-void BackendConsumer::linkerDiagnosticHandler(const DiagnosticInfo &DI) {
- if (DI.getSeverity() != DS_Error)
- return;
-
- std::string MsgStorage;
- {
- raw_string_ostream Stream(MsgStorage);
- DiagnosticPrinterRawOStream DP(Stream);
- DI.print(DP);
- }
-
- Diags.Report(diag::err_fe_cannot_link_module)
- << LinkModule->getModuleIdentifier() << MsgStorage;
-}
-
/// \brief This function is invoked when the backend needs
/// to report something to the user.
void BackendConsumer::DiagnosticHandlerImpl(const DiagnosticInfo &DI) {
@@ -535,6 +561,13 @@
return;
ComputeDiagID(Severity, backend_frame_larger_than, DiagID);
break;
+ case DK_Linker:
+ assert(CurLinkModule);
+ // FIXME: stop eating the warnings and notes.
+ if (Severity != DS_Error)
+ return;
+ DiagID = diag::err_fe_cannot_link_module;
+ break;
case llvm::DK_OptimizationRemark:
// Optimization remarks are always handled completely by this
// handler. There is no generic way of emitting them.
@@ -551,6 +584,18 @@
OptimizationRemarkHandler(
cast<DiagnosticInfoOptimizationRemarkAnalysis>(DI));
return;
+ case llvm::DK_OptimizationRemarkAnalysisFPCommute:
+ // Optimization remarks are always handled completely by this
+ // handler. There is no generic way of emitting them.
+ OptimizationRemarkHandler(
+ cast<DiagnosticInfoOptimizationRemarkAnalysisFPCommute>(DI));
+ return;
+ case llvm::DK_OptimizationRemarkAnalysisAliasing:
+ // Optimization remarks are always handled completely by this
+ // handler. There is no generic way of emitting them.
+ OptimizationRemarkHandler(
+ cast<DiagnosticInfoOptimizationRemarkAnalysisAliasing>(DI));
+ return;
case llvm::DK_OptimizationFailure:
// Optimization failures are always handled completely by this
// handler.
@@ -568,6 +613,12 @@
DI.print(DP);
}
+ if (DiagID == diag::err_fe_cannot_link_module) {
+ Diags.Report(diag::err_fe_cannot_link_module)
+ << CurLinkModule->getModuleIdentifier() << MsgStorage;
+ return;
+ }
+
// Report the backend message using the usual diagnostic mechanism.
FullSourceLoc Loc;
Diags.Report(Loc, DiagID).AddString(MsgStorage);
@@ -575,9 +626,8 @@
#undef ComputeDiagID
CodeGenAction::CodeGenAction(unsigned _Act, LLVMContext *_VMContext)
- : Act(_Act), LinkModule(nullptr),
- VMContext(_VMContext ? _VMContext : new LLVMContext),
- OwnsVMContext(!_VMContext) {}
+ : Act(_Act), VMContext(_VMContext ? _VMContext : new LLVMContext),
+ OwnsVMContext(!_VMContext) {}
CodeGenAction::~CodeGenAction() {
TheModule.reset();
@@ -592,9 +642,9 @@
if (!getCompilerInstance().hasASTConsumer())
return;
- // If we were given a link module, release consumer's ownership of it.
- if (LinkModule)
- BEConsumer->takeLinkModule();
+ // Take back ownership of link modules we passed to consumer.
+ if (!LinkModules.empty())
+ BEConsumer->releaseLinkModules();
// Steal the module from the consumer.
TheModule = BEConsumer->takeModule();
@@ -636,28 +686,29 @@
if (BA != Backend_EmitNothing && !OS)
return nullptr;
- llvm::Module *LinkModuleToUse = LinkModule;
+ // Load bitcode modules to link with, if we need to.
+ if (LinkModules.empty())
+ for (auto &I : CI.getCodeGenOpts().LinkBitcodeFiles) {
+ const std::string &LinkBCFile = I.second;
- // If we were not given a link module, and the user requested that one be
- // loaded from bitcode, do so now.
- const std::string &LinkBCFile = CI.getCodeGenOpts().LinkBitcodeFile;
- if (!LinkModuleToUse && !LinkBCFile.empty()) {
- auto BCBuf = CI.getFileManager().getBufferForFile(LinkBCFile);
- if (!BCBuf) {
- CI.getDiagnostics().Report(diag::err_cannot_open_file)
- << LinkBCFile << BCBuf.getError().message();
- return nullptr;
- }
+ auto BCBuf = CI.getFileManager().getBufferForFile(LinkBCFile);
+ if (!BCBuf) {
+ CI.getDiagnostics().Report(diag::err_cannot_open_file)
+ << LinkBCFile << BCBuf.getError().message();
+ LinkModules.clear();
+ return nullptr;
+ }
- ErrorOr<llvm::Module *> ModuleOrErr =
- getLazyBitcodeModule(std::move(*BCBuf), *VMContext);
- if (std::error_code EC = ModuleOrErr.getError()) {
- CI.getDiagnostics().Report(diag::err_cannot_open_file)
- << LinkBCFile << EC.message();
- return nullptr;
+ ErrorOr<std::unique_ptr<llvm::Module>> ModuleOrErr =
+ getLazyBitcodeModule(std::move(*BCBuf), *VMContext);
+ if (std::error_code EC = ModuleOrErr.getError()) {
+ CI.getDiagnostics().Report(diag::err_cannot_open_file) << LinkBCFile
+ << EC.message();
+ LinkModules.clear();
+ return nullptr;
+ }
+ addLinkModule(ModuleOrErr.get().release(), I.first);
}
- LinkModuleToUse = ModuleOrErr.get();
- }
CoverageSourceInfo *CoverageInfo = nullptr;
// Add the preprocessor callback only when the coverage mapping is generated.
@@ -666,10 +717,12 @@
CI.getPreprocessor().addPPCallbacks(
std::unique_ptr<PPCallbacks>(CoverageInfo));
}
+
std::unique_ptr<BackendConsumer> Result(new BackendConsumer(
- BA, CI.getDiagnostics(), CI.getCodeGenOpts(), CI.getTargetOpts(),
- CI.getLangOpts(), CI.getFrontendOpts().ShowTimers, InFile,
- LinkModuleToUse, OS, *VMContext, CoverageInfo));
+ BA, CI.getDiagnostics(), CI.getHeaderSearchOpts(),
+ CI.getPreprocessorOpts(), CI.getCodeGenOpts(), CI.getTargetOpts(),
+ CI.getLangOpts(), CI.getFrontendOpts().ShowTimers, InFile, LinkModules,
+ OS, *VMContext, CoverageInfo));
BEConsumer = Result.get();
return std::move(Result);
}
@@ -728,11 +781,43 @@
TheModule->setTargetTriple(TargetOpts.Triple);
}
+ auto DiagHandler = [&](const DiagnosticInfo &DI) {
+ TheModule->getContext().diagnose(DI);
+ };
+
+ // If we are performing ThinLTO importing compilation (indicated by
+ // a non-empty index file option), then we need promote to global scope
+ // and rename any local values that are potentially exported to other
+ // modules. Do this early so that the rest of the compilation sees the
+ // promoted symbols.
+ std::unique_ptr<FunctionInfoIndex> Index;
+ if (!CI.getCodeGenOpts().ThinLTOIndexFile.empty()) {
+ ErrorOr<std::unique_ptr<FunctionInfoIndex>> IndexOrErr =
+ llvm::getFunctionIndexForFile(CI.getCodeGenOpts().ThinLTOIndexFile,
+ DiagHandler);
+ if (std::error_code EC = IndexOrErr.getError()) {
+ std::string Error = EC.message();
+ errs() << "Error loading index file '"
+ << CI.getCodeGenOpts().ThinLTOIndexFile << "': " << Error
+ << "\n";
+ return;
+ }
+ Index = std::move(IndexOrErr.get());
+ assert(Index);
+ // Currently this requires creating a new Module object.
+ std::unique_ptr<llvm::Module> RenamedModule =
+ renameModuleForThinLTO(std::move(TheModule), Index.get());
+ if (!RenamedModule)
+ return;
+
+ TheModule = std::move(RenamedModule);
+ }
+
LLVMContext &Ctx = TheModule->getContext();
Ctx.setInlineAsmDiagnosticHandler(BitcodeInlineAsmDiagHandler);
EmitBackendOutput(CI.getDiagnostics(), CI.getCodeGenOpts(), TargetOpts,
- CI.getLangOpts(), CI.getTarget().getTargetDescription(),
- TheModule.get(), BA, OS);
+ CI.getLangOpts(), CI.getTarget().getDataLayoutString(),
+ TheModule.get(), BA, OS, std::move(Index));
return;
}
diff --git a/lib/CodeGen/CodeGenFunction.cpp b/lib/CodeGen/CodeGenFunction.cpp
index f370ac2..048a043 100644
--- a/lib/CodeGen/CodeGenFunction.cpp
+++ b/lib/CodeGen/CodeGenFunction.cpp
@@ -12,6 +12,7 @@
//===----------------------------------------------------------------------===//
#include "CodeGenFunction.h"
+#include "CGBlocks.h"
#include "CGCleanup.h"
#include "CGCUDARuntime.h"
#include "CGCXXABI.h"
@@ -24,9 +25,11 @@
#include "clang/AST/Decl.h"
#include "clang/AST/DeclCXX.h"
#include "clang/AST/StmtCXX.h"
+#include "clang/Basic/Builtins.h"
#include "clang/Basic/TargetInfo.h"
#include "clang/CodeGen/CGFunctionInfo.h"
#include "clang/Frontend/CodeGenOptions.h"
+#include "clang/Sema/SemaDiagnostic.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/MDBuilder.h"
@@ -36,22 +39,24 @@
CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext)
: CodeGenTypeCache(cgm), CGM(cgm), Target(cgm.getTarget()),
- Builder(cgm.getModule().getContext(), llvm::ConstantFolder(),
+ Builder(cgm, cgm.getModule().getContext(), llvm::ConstantFolder(),
CGBuilderInserterTy(this)),
- CurFn(nullptr), CapturedStmtInfo(nullptr),
+ CurFn(nullptr), ReturnValue(Address::invalid()),
+ CapturedStmtInfo(nullptr),
SanOpts(CGM.getLangOpts().Sanitize), IsSanitizerScope(false),
CurFuncIsThunk(false), AutoreleaseResult(false), SawAsmBlock(false),
- IsOutlinedSEHHelper(false), BlockInfo(nullptr), BlockPointer(nullptr),
+ IsOutlinedSEHHelper(false),
+ BlockInfo(nullptr), BlockPointer(nullptr),
LambdaThisCaptureField(nullptr), NormalCleanupDest(nullptr),
NextCleanupDestIndex(1), FirstBlockInfo(nullptr), EHResumeBlock(nullptr),
ExceptionSlot(nullptr), EHSelectorSlot(nullptr),
- AbnormalTerminationSlot(nullptr), SEHPointersDecl(nullptr),
- DebugInfo(CGM.getModuleDebugInfo()), DisableDebugInfo(false),
- DidCallStackSave(false), IndirectBranch(nullptr), PGO(cgm),
- SwitchInsn(nullptr), SwitchWeights(nullptr), CaseRangeBlock(nullptr),
- UnreachableBlock(nullptr), NumReturnExprs(0), NumSimpleReturnExprs(0),
- CXXABIThisDecl(nullptr), CXXABIThisValue(nullptr), CXXThisValue(nullptr),
- CXXDefaultInitExprThis(nullptr), CXXStructorImplicitParamDecl(nullptr),
+ DebugInfo(CGM.getModuleDebugInfo()),
+ DisableDebugInfo(false), DidCallStackSave(false), IndirectBranch(nullptr),
+ PGO(cgm), SwitchInsn(nullptr), SwitchWeights(nullptr),
+ CaseRangeBlock(nullptr), UnreachableBlock(nullptr), NumReturnExprs(0),
+ NumSimpleReturnExprs(0), CXXABIThisDecl(nullptr),
+ CXXABIThisValue(nullptr), CXXThisValue(nullptr),
+ CXXStructorImplicitParamDecl(nullptr),
CXXStructorImplicitParamValue(nullptr), OutermostConditional(nullptr),
CurLexicalScope(nullptr), TerminateLandingPad(nullptr),
TerminateHandler(nullptr), TrapBB(nullptr) {
@@ -91,18 +96,69 @@
}
}
-LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) {
- CharUnits Alignment;
- if (CGM.getCXXABI().isTypeInfoCalculable(T)) {
- Alignment = getContext().getTypeAlignInChars(T);
- unsigned MaxAlign = getContext().getLangOpts().MaxTypeAlign;
- if (MaxAlign && Alignment.getQuantity() > MaxAlign &&
- !getContext().isAlignmentRequired(T))
- Alignment = CharUnits::fromQuantity(MaxAlign);
- }
- return LValue::MakeAddr(V, T, Alignment, getContext(), CGM.getTBAAInfo(T));
+CharUnits CodeGenFunction::getNaturalPointeeTypeAlignment(QualType T,
+ AlignmentSource *Source) {
+ return getNaturalTypeAlignment(T->getPointeeType(), Source,
+ /*forPointee*/ true);
}
+CharUnits CodeGenFunction::getNaturalTypeAlignment(QualType T,
+ AlignmentSource *Source,
+ bool forPointeeType) {
+ // Honor alignment typedef attributes even on incomplete types.
+ // We also honor them straight for C++ class types, even as pointees;
+ // there's an expressivity gap here.
+ if (auto TT = T->getAs<TypedefType>()) {
+ if (auto Align = TT->getDecl()->getMaxAlignment()) {
+ if (Source) *Source = AlignmentSource::AttributedType;
+ return getContext().toCharUnitsFromBits(Align);
+ }
+ }
+
+ if (Source) *Source = AlignmentSource::Type;
+
+ CharUnits Alignment;
+ if (T->isIncompleteType()) {
+ Alignment = CharUnits::One(); // Shouldn't be used, but pessimistic is best.
+ } else {
+ // For C++ class pointees, we don't know whether we're pointing at a
+ // base or a complete object, so we generally need to use the
+ // non-virtual alignment.
+ const CXXRecordDecl *RD;
+ if (forPointeeType && (RD = T->getAsCXXRecordDecl())) {
+ Alignment = CGM.getClassPointerAlignment(RD);
+ } else {
+ Alignment = getContext().getTypeAlignInChars(T);
+ }
+
+ // Cap to the global maximum type alignment unless the alignment
+ // was somehow explicit on the type.
+ if (unsigned MaxAlign = getLangOpts().MaxTypeAlign) {
+ if (Alignment.getQuantity() > MaxAlign &&
+ !getContext().isAlignmentRequired(T))
+ Alignment = CharUnits::fromQuantity(MaxAlign);
+ }
+ }
+ return Alignment;
+}
+
+LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) {
+ AlignmentSource AlignSource;
+ CharUnits Alignment = getNaturalTypeAlignment(T, &AlignSource);
+ return LValue::MakeAddr(Address(V, Alignment), T, getContext(), AlignSource,
+ CGM.getTBAAInfo(T));
+}
+
+/// Given a value of type T* that may not be to a complete object,
+/// construct an l-value with the natural pointee alignment of T.
+LValue
+CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) {
+ AlignmentSource AlignSource;
+ CharUnits Align = getNaturalTypeAlignment(T, &AlignSource, /*pointee*/ true);
+ return MakeAddrLValue(Address(V, Align), T, AlignSource);
+}
+
+
llvm::Type *CodeGenFunction::ConvertTypeForMem(QualType T) {
return CGM.getTypes().ConvertTypeForMem(T);
}
@@ -284,7 +340,7 @@
Builder.ClearInsertionPoint();
}
- // If some of our locals escaped, insert a call to llvm.frameescape in the
+ // If some of our locals escaped, insert a call to llvm.localescape in the
// entry block.
if (!EscapedLocals.empty()) {
// Invert the map from local to index into a simple vector. There should be
@@ -294,8 +350,8 @@
for (auto &Pair : EscapedLocals)
EscapeArgs[Pair.second] = Pair.first;
llvm::Function *FrameEscapeFn = llvm::Intrinsic::getDeclaration(
- &CGM.getModule(), llvm::Intrinsic::frameescape);
- CGBuilderTy(AllocaInsertPt).CreateCall(FrameEscapeFn, EscapeArgs);
+ &CGM.getModule(), llvm::Intrinsic::localescape);
+ CGBuilderTy(*this, AllocaInsertPt).CreateCall(FrameEscapeFn, EscapeArgs);
}
// Remove the AllocaInsertPt instruction, which is just a convenience for us.
@@ -615,7 +671,7 @@
}
// Apply sanitizer attributes to the function.
- if (SanOpts.has(SanitizerKind::Address))
+ if (SanOpts.hasOneOf(SanitizerKind::Address | SanitizerKind::KernelAddress))
Fn->addFnAttr(llvm::Attribute::SanitizeAddress);
if (SanOpts.has(SanitizerKind::Thread))
Fn->addFnAttr(llvm::Attribute::SanitizeThread);
@@ -660,6 +716,14 @@
}
}
+ // If we're in C++ mode and the function name is "main", it is guaranteed
+ // to be norecurse by the standard (3.6.1.3 "The function main shall not be
+ // used within a program").
+ if (getLangOpts().CPlusPlus)
+ if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D))
+ if (FD->isMain())
+ Fn->addFnAttr(llvm::Attribute::NoRecurse);
+
llvm::BasicBlock *EntryBB = createBasicBlock("entry", CurFn);
// Create a marker to make it easy to insert allocas into the entryblock
@@ -696,7 +760,7 @@
if (RetTy->isVoidType()) {
// Void type; nothing to return.
- ReturnValue = nullptr;
+ ReturnValue = Address::invalid();
// Count the implicit return.
if (!endsWithReturn(D))
@@ -708,15 +772,16 @@
auto AI = CurFn->arg_begin();
if (CurFnInfo->getReturnInfo().isSRetAfterThis())
++AI;
- ReturnValue = AI;
+ ReturnValue = Address(&*AI, CurFnInfo->getReturnInfo().getIndirectAlign());
} else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::InAlloca &&
!hasScalarEvaluationKind(CurFnInfo->getReturnType())) {
// Load the sret pointer from the argument struct and return into that.
unsigned Idx = CurFnInfo->getReturnInfo().getInAllocaFieldIndex();
llvm::Function::arg_iterator EI = CurFn->arg_end();
--EI;
- llvm::Value *Addr = Builder.CreateStructGEP(nullptr, EI, Idx);
- ReturnValue = Builder.CreateLoad(Addr, "agg.result");
+ llvm::Value *Addr = Builder.CreateStructGEP(nullptr, &*EI, Idx);
+ Addr = Builder.CreateAlignedLoad(Addr, getPointerAlign(), "agg.result");
+ ReturnValue = Address(Addr, getNaturalTypeAlignment(RetTy));
} else {
ReturnValue = CreateIRTemp(RetTy, "retval");
@@ -826,15 +891,11 @@
// can't do this on functions that can be overwritten.
if (F->mayBeOverridden()) return;
- for (llvm::Function::iterator FI = F->begin(), FE = F->end(); FI != FE; ++FI)
- for (llvm::BasicBlock::iterator
- BI = FI->begin(), BE = FI->end(); BI != BE; ++BI)
- if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(&*BI)) {
- if (!Call->doesNotThrow())
- return;
- } else if (isa<llvm::ResumeInst>(&*BI)) {
+ for (llvm::BasicBlock &BB : *F)
+ for (llvm::Instruction &I : BB)
+ if (I.mayThrow())
return;
- }
+
F->setDoesNotThrow();
}
@@ -859,7 +920,18 @@
CGM.getCXXABI().buildThisParam(*this, Args);
}
- Args.append(FD->param_begin(), FD->param_end());
+ for (auto *Param : FD->params()) {
+ Args.push_back(Param);
+ if (!Param->hasAttr<PassObjectSizeAttr>())
+ continue;
+
+ IdentifierInfo *NoID = nullptr;
+ auto *Implicit = ImplicitParamDecl::Create(
+ getContext(), Param->getDeclContext(), Param->getLocation(), NoID,
+ getContext().getSizeType());
+ SizeArguments[Param] = Implicit;
+ Args.push_back(Implicit);
+ }
if (MD && (isa<CXXConstructorDecl>(MD) || isa<CXXDestructorDecl>(MD)))
CGM.getCXXABI().addImplicitStructorParams(*this, ResTy, Args);
@@ -885,8 +957,7 @@
StartFunction(GD, ResTy, Fn, FnInfo, Args, Loc, BodyRange.getBegin());
// Generate the body of the function.
- PGO.checkGlobalDecl(GD);
- PGO.assignRegionCounters(GD.getDecl(), CurFn);
+ PGO.assignRegionCounters(GD, CurFn);
if (isa<CXXDestructorDecl>(FD))
EmitDestructorBody(Args);
else if (isa<CXXConstructorDecl>(FD))
@@ -930,8 +1001,9 @@
EmitCheck(std::make_pair(IsFalse, SanitizerKind::Return),
"missing_return", EmitCheckSourceLocation(FD->getLocation()),
None);
- } else if (CGM.getCodeGenOpts().OptimizationLevel == 0)
- Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::trap), {});
+ } else if (CGM.getCodeGenOpts().OptimizationLevel == 0) {
+ EmitTrapCall(llvm::Intrinsic::trap);
+ }
Builder.CreateUnreachable();
Builder.ClearInsertionPoint();
}
@@ -970,8 +1042,8 @@
IgnoreCaseStmts = true;
// Scan subexpressions for verboten labels.
- for (Stmt::const_child_range I = S->children(); I; ++I)
- if (ContainsLabel(*I, IgnoreCaseStmts))
+ for (const Stmt *SubStmt : S->children())
+ if (ContainsLabel(SubStmt, IgnoreCaseStmts))
return true;
return false;
@@ -994,8 +1066,8 @@
return true;
// Scan subexpressions for verboten breaks.
- for (Stmt::const_child_range I = S->children(); I; ++I)
- if (containsBreak(*I))
+ for (const Stmt *SubStmt : S->children())
+ if (containsBreak(SubStmt))
return true;
return false;
@@ -1206,6 +1278,22 @@
return;
}
+ // If the branch has a condition wrapped by __builtin_unpredictable,
+ // create metadata that specifies that the branch is unpredictable.
+ // Don't bother if not optimizing because that metadata would not be used.
+ llvm::MDNode *Unpredictable = nullptr;
+ if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
+ if (const CallExpr *Call = dyn_cast<CallExpr>(Cond)) {
+ const Decl *TargetDecl = Call->getCalleeDecl();
+ if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
+ if (FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
+ llvm::MDBuilder MDHelper(getLLVMContext());
+ Unpredictable = MDHelper.createUnpredictable();
+ }
+ }
+ }
+ }
+
// Create branch weights based on the number of times we get here and the
// number of times the condition should be true.
uint64_t CurrentCount = std::max(getCurrentProfileCount(), TrueCount);
@@ -1218,7 +1306,7 @@
ApplyDebugLocation DL(*this, Cond);
CondV = EvaluateExprAsBool(Cond);
}
- Builder.CreateCondBr(CondV, TrueBlock, FalseBlock, Weights);
+ Builder.CreateCondBr(CondV, TrueBlock, FalseBlock, Weights, Unpredictable);
}
/// ErrorUnsupported - Print out an error that codegen doesn't support the
@@ -1235,20 +1323,18 @@
/// base element of the array
/// \param sizeInChars - the total size of the VLA, in chars
static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType,
- llvm::Value *dest, llvm::Value *src,
+ Address dest, Address src,
llvm::Value *sizeInChars) {
- std::pair<CharUnits,CharUnits> baseSizeAndAlign
- = CGF.getContext().getTypeInfoInChars(baseType);
-
CGBuilderTy &Builder = CGF.Builder;
+ CharUnits baseSize = CGF.getContext().getTypeSizeInChars(baseType);
llvm::Value *baseSizeInChars
- = llvm::ConstantInt::get(CGF.IntPtrTy, baseSizeAndAlign.first.getQuantity());
+ = llvm::ConstantInt::get(CGF.IntPtrTy, baseSize.getQuantity());
- llvm::Type *i8p = Builder.getInt8PtrTy();
-
- llvm::Value *begin = Builder.CreateBitCast(dest, i8p, "vla.begin");
- llvm::Value *end = Builder.CreateInBoundsGEP(dest, sizeInChars, "vla.end");
+ Address begin =
+ Builder.CreateElementBitCast(dest, CGF.Int8Ty, "vla.begin");
+ llvm::Value *end =
+ Builder.CreateInBoundsGEP(begin.getPointer(), sizeInChars, "vla.end");
llvm::BasicBlock *originBB = CGF.Builder.GetInsertBlock();
llvm::BasicBlock *loopBB = CGF.createBasicBlock("vla-init.loop");
@@ -1258,17 +1344,19 @@
// count must be nonzero.
CGF.EmitBlock(loopBB);
- llvm::PHINode *cur = Builder.CreatePHI(i8p, 2, "vla.cur");
- cur->addIncoming(begin, originBB);
+ llvm::PHINode *cur = Builder.CreatePHI(begin.getType(), 2, "vla.cur");
+ cur->addIncoming(begin.getPointer(), originBB);
+
+ CharUnits curAlign =
+ dest.getAlignment().alignmentOfArrayElement(baseSize);
// memcpy the individual element bit-pattern.
- Builder.CreateMemCpy(cur, src, baseSizeInChars,
- baseSizeAndAlign.second.getQuantity(),
+ Builder.CreateMemCpy(Address(cur, curAlign), src, baseSizeInChars,
/*volatile*/ false);
// Go to the next element.
- llvm::Value *next = Builder.CreateConstInBoundsGEP1_32(Builder.getInt8Ty(),
- cur, 1, "vla.next");
+ llvm::Value *next =
+ Builder.CreateInBoundsGEP(CGF.Int8Ty, cur, baseSizeInChars, "vla.next");
// Leave if that's the end of the VLA.
llvm::Value *done = Builder.CreateICmpEQ(next, end, "vla-init.isdone");
@@ -1279,7 +1367,7 @@
}
void
-CodeGenFunction::EmitNullInitialization(llvm::Value *DestPtr, QualType Ty) {
+CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) {
// Ignore empty classes in C++.
if (getLangOpts().CPlusPlus) {
if (const RecordType *RT = Ty->getAs<RecordType>()) {
@@ -1289,23 +1377,17 @@
}
// Cast the dest ptr to the appropriate i8 pointer type.
- unsigned DestAS =
- cast<llvm::PointerType>(DestPtr->getType())->getAddressSpace();
- llvm::Type *BP = Builder.getInt8PtrTy(DestAS);
- if (DestPtr->getType() != BP)
- DestPtr = Builder.CreateBitCast(DestPtr, BP);
+ if (DestPtr.getElementType() != Int8Ty)
+ DestPtr = Builder.CreateElementBitCast(DestPtr, Int8Ty);
// Get size and alignment info for this aggregate.
- std::pair<CharUnits, CharUnits> TypeInfo =
- getContext().getTypeInfoInChars(Ty);
- CharUnits Size = TypeInfo.first;
- CharUnits Align = TypeInfo.second;
+ CharUnits size = getContext().getTypeSizeInChars(Ty);
llvm::Value *SizeVal;
const VariableArrayType *vla;
// Don't bother emitting a zero-byte memset.
- if (Size.isZero()) {
+ if (size.isZero()) {
// But note that getTypeInfo returns 0 for a VLA.
if (const VariableArrayType *vlaType =
dyn_cast_or_null<VariableArrayType>(
@@ -1323,7 +1405,7 @@
return;
}
} else {
- SizeVal = CGM.getSize(Size);
+ SizeVal = CGM.getSize(size);
vla = nullptr;
}
@@ -1342,21 +1424,22 @@
/*isConstant=*/true,
llvm::GlobalVariable::PrivateLinkage,
NullConstant, Twine());
- llvm::Value *SrcPtr =
- Builder.CreateBitCast(NullVariable, Builder.getInt8PtrTy());
+ CharUnits NullAlign = DestPtr.getAlignment();
+ NullVariable->setAlignment(NullAlign.getQuantity());
+ Address SrcPtr(Builder.CreateBitCast(NullVariable, Builder.getInt8PtrTy()),
+ NullAlign);
if (vla) return emitNonZeroVLAInit(*this, Ty, DestPtr, SrcPtr, SizeVal);
// Get and call the appropriate llvm.memcpy overload.
- Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, Align.getQuantity(), false);
+ Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, false);
return;
}
// Otherwise, just memset the whole thing to zero. This is legal
// because in LLVM, all default initializers (other than the ones we just
// handled above) are guaranteed to have a bit pattern of all zeros.
- Builder.CreateMemSet(DestPtr, Builder.getInt8(0), SizeVal,
- Align.getQuantity(), false);
+ Builder.CreateMemSet(DestPtr, Builder.getInt8(0), SizeVal, false);
}
llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelDecl *L) {
@@ -1375,7 +1458,7 @@
// If we already made the indirect branch for indirect goto, return its block.
if (IndirectBranch) return IndirectBranch->getParent();
- CGBuilderTy TmpBuilder(createBasicBlock("indirectgoto"));
+ CGBuilderTy TmpBuilder(*this, createBasicBlock("indirectgoto"));
// Create the PHI node that indirect gotos will add entries to.
llvm::Value *DestVal = TmpBuilder.CreatePHI(Int8PtrTy, 0,
@@ -1390,7 +1473,7 @@
/// element type and a properly-typed first element pointer.
llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType,
QualType &baseType,
- llvm::Value *&addr) {
+ Address &addr) {
const ArrayType *arrayType = origArrayType;
// If it's a VLA, we have to load the stored size. Note that
@@ -1429,8 +1512,7 @@
QualType eltType;
llvm::ArrayType *llvmArrayType =
- dyn_cast<llvm::ArrayType>(
- cast<llvm::PointerType>(addr->getType())->getElementType());
+ dyn_cast<llvm::ArrayType>(addr.getElementType());
while (llvmArrayType) {
assert(isa<ConstantArrayType>(arrayType));
assert(cast<ConstantArrayType>(arrayType)->getSize().getZExtValue()
@@ -1458,12 +1540,13 @@
arrayType = getContext().getAsArrayType(eltType);
}
- unsigned AddressSpace = addr->getType()->getPointerAddressSpace();
- llvm::Type *BaseType = ConvertType(eltType)->getPointerTo(AddressSpace);
- addr = Builder.CreateBitCast(addr, BaseType, "array.begin");
+ llvm::Type *baseType = ConvertType(eltType);
+ addr = Builder.CreateElementBitCast(addr, baseType, "array.begin");
} else {
// Create the actual GEP.
- addr = Builder.CreateInBoundsGEP(addr, gepIndices, "array.begin");
+ addr = Address(Builder.CreateInBoundsGEP(addr.getPointer(),
+ gepIndices, "array.begin"),
+ addr.getAlignment());
}
baseType = eltType;
@@ -1648,9 +1731,13 @@
} while (type->isVariablyModifiedType());
}
-llvm::Value* CodeGenFunction::EmitVAListRef(const Expr* E) {
+Address CodeGenFunction::EmitVAListRef(const Expr* E) {
if (getContext().getBuiltinVaListType()->isArrayType())
- return EmitScalarExpr(E);
+ return EmitPointerWithAlignment(E);
+ return EmitLValue(E).getAddress();
+}
+
+Address CodeGenFunction::EmitMSVAListRef(const Expr *E) {
return EmitLValue(E).getAddress();
}
@@ -1712,9 +1799,10 @@
I->getAnnotation(), D->getLocation());
}
-llvm::Value *CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D,
- llvm::Value *V) {
+Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D,
+ Address Addr) {
assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
+ llvm::Value *V = Addr.getPointer();
llvm::Type *VTy = V->getType();
llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::ptr_annotation,
CGM.Int8PtrTy);
@@ -1729,7 +1817,7 @@
V = Builder.CreateBitCast(V, VTy);
}
- return V;
+ return Address(V, Addr.getAlignment());
}
CodeGenFunction::CGCapturedStmtInfo::~CGCapturedStmtInfo() { }
@@ -1772,3 +1860,80 @@
llvm::Instruction *I, const llvm::Twine &Name, llvm::BasicBlock *BB,
llvm::BasicBlock::iterator InsertPt) const;
#undef PreserveNames
+
+static bool hasRequiredFeatures(const SmallVectorImpl<StringRef> &ReqFeatures,
+ CodeGenModule &CGM, const FunctionDecl *FD,
+ std::string &FirstMissing) {
+ // If there aren't any required features listed then go ahead and return.
+ if (ReqFeatures.empty())
+ return false;
+
+ // Now build up the set of caller features and verify that all the required
+ // features are there.
+ llvm::StringMap<bool> CallerFeatureMap;
+ CGM.getFunctionFeatureMap(CallerFeatureMap, FD);
+
+ // If we have at least one of the features in the feature list return
+ // true, otherwise return false.
+ return std::all_of(
+ ReqFeatures.begin(), ReqFeatures.end(), [&](StringRef Feature) {
+ SmallVector<StringRef, 1> OrFeatures;
+ Feature.split(OrFeatures, "|");
+ return std::any_of(OrFeatures.begin(), OrFeatures.end(),
+ [&](StringRef Feature) {
+ if (!CallerFeatureMap.lookup(Feature)) {
+ FirstMissing = Feature.str();
+ return false;
+ }
+ return true;
+ });
+ });
+}
+
+// Emits an error if we don't have a valid set of target features for the
+// called function.
+void CodeGenFunction::checkTargetFeatures(const CallExpr *E,
+ const FunctionDecl *TargetDecl) {
+ // Early exit if this is an indirect call.
+ if (!TargetDecl)
+ return;
+
+ // Get the current enclosing function if it exists. If it doesn't
+ // we can't check the target features anyhow.
+ const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl);
+ if (!FD)
+ return;
+
+ // Grab the required features for the call. For a builtin this is listed in
+ // the td file with the default cpu, for an always_inline function this is any
+ // listed cpu and any listed features.
+ unsigned BuiltinID = TargetDecl->getBuiltinID();
+ std::string MissingFeature;
+ if (BuiltinID) {
+ SmallVector<StringRef, 1> ReqFeatures;
+ const char *FeatureList =
+ CGM.getContext().BuiltinInfo.getRequiredFeatures(BuiltinID);
+ // Return if the builtin doesn't have any required features.
+ if (!FeatureList || StringRef(FeatureList) == "")
+ return;
+ StringRef(FeatureList).split(ReqFeatures, ",");
+ if (!hasRequiredFeatures(ReqFeatures, CGM, FD, MissingFeature))
+ CGM.getDiags().Report(E->getLocStart(), diag::err_builtin_needs_feature)
+ << TargetDecl->getDeclName()
+ << CGM.getContext().BuiltinInfo.getRequiredFeatures(BuiltinID);
+
+ } else if (TargetDecl->hasAttr<TargetAttr>()) {
+ // Get the required features for the callee.
+ SmallVector<StringRef, 1> ReqFeatures;
+ llvm::StringMap<bool> CalleeFeatureMap;
+ CGM.getFunctionFeatureMap(CalleeFeatureMap, TargetDecl);
+ for (const auto &F : CalleeFeatureMap) {
+ // Only positive features are "required".
+ if (F.getValue())
+ ReqFeatures.push_back(F.getKey());
+ }
+ if (!hasRequiredFeatures(ReqFeatures, CGM, FD, MissingFeature))
+ CGM.getDiags().Report(E->getLocStart(), diag::err_function_needs_feature)
+ << FD->getDeclName() << TargetDecl->getDeclName() << MissingFeature;
+ }
+}
diff --git a/lib/CodeGen/CodeGenFunction.h b/lib/CodeGen/CodeGenFunction.h
index 469022d..b4a9186 100644
--- a/lib/CodeGen/CodeGenFunction.h
+++ b/lib/CodeGen/CodeGenFunction.h
@@ -24,6 +24,7 @@
#include "clang/AST/CharUnits.h"
#include "clang/AST/ExprCXX.h"
#include "clang/AST/ExprObjC.h"
+#include "clang/AST/ExprOpenMP.h"
#include "clang/AST/Type.h"
#include "clang/Basic/ABI.h"
#include "clang/Basic/CapturedStmt.h"
@@ -80,6 +81,8 @@
class CGRecordLayout;
class CGBlockInfo;
class CGCXXABI;
+class BlockByrefHelpers;
+class BlockByrefInfo;
class BlockFlags;
class BlockFieldFlags;
@@ -159,9 +162,9 @@
/// ReturnBlock - Unified return block.
JumpDest ReturnBlock;
- /// ReturnValue - The temporary alloca to hold the return value. This is null
- /// iff the function has no return value.
- llvm::Value *ReturnValue;
+ /// ReturnValue - The temporary alloca to hold the return
+ /// value. This is invalid iff the function has no return value.
+ Address ReturnValue;
/// AllocaInsertPoint - This is an instruction in the entry block before which
/// we prefer to insert allocas.
@@ -233,9 +236,19 @@
};
CGCapturedStmtInfo *CapturedStmtInfo;
- /// BoundsChecking - Emit run-time bounds checks. Higher values mean
- /// potentially higher performance penalties.
- unsigned char BoundsChecking;
+ /// \brief RAII for correct setting/restoring of CapturedStmtInfo.
+ class CGCapturedStmtRAII {
+ private:
+ CodeGenFunction &CGF;
+ CGCapturedStmtInfo *PrevCapturedStmtInfo;
+ public:
+ CGCapturedStmtRAII(CodeGenFunction &CGF,
+ CGCapturedStmtInfo *NewCapturedStmtInfo)
+ : CGF(CGF), PrevCapturedStmtInfo(CGF.CapturedStmtInfo) {
+ CGF.CapturedStmtInfo = NewCapturedStmtInfo;
+ }
+ ~CGCapturedStmtRAII() { CGF.CapturedStmtInfo = PrevCapturedStmtInfo; }
+ };
/// \brief Sanitizers enabled for this function.
SanitizerSet SanOpts;
@@ -280,15 +293,17 @@
llvm::SmallVector<char, 256> LifetimeExtendedCleanupStack;
llvm::SmallVector<const JumpDest *, 2> SEHTryEpilogueStack;
+ llvm::Instruction *CurrentFuncletPad = nullptr;
+
/// Header for data within LifetimeExtendedCleanupStack.
struct LifetimeExtendedCleanupHeader {
/// The size of the following cleanup object.
- unsigned Size : 29;
+ unsigned Size;
/// The kind of cleanup to push: a value from the CleanupKind enumeration.
- unsigned Kind : 3;
+ CleanupKind Kind;
- size_t getSize() const { return size_t(Size); }
- CleanupKind getKind() const { return static_cast<CleanupKind>(Kind); }
+ size_t getSize() const { return Size; }
+ CleanupKind getKind() const { return Kind; }
};
/// i32s containing the indexes of the cleanup destinations.
@@ -310,11 +325,13 @@
/// write the current selector value into this alloca.
llvm::AllocaInst *EHSelectorSlot;
- llvm::AllocaInst *AbnormalTerminationSlot;
+ /// A stack of exception code slots. Entering an __except block pushes a slot
+ /// on the stack and leaving pops one. The __exception_code() intrinsic loads
+ /// a value from the top of the stack.
+ SmallVector<Address, 1> SEHCodeSlotStack;
- /// The implicit parameter to SEH filter functions of type
- /// 'EXCEPTION_POINTERS*'.
- ImplicitParamDecl *SEHPointersDecl;
+ /// Value returned by __exception_info intrinsic.
+ llvm::Value *SEHInfo = nullptr;
/// Emits a landing pad for the current EH stack.
llvm::BasicBlock *EmitLandingPad();
@@ -357,6 +374,11 @@
/// Returns true inside SEH __try blocks.
bool isSEHTryScope() const { return !SEHTryEpilogueStack.empty(); }
+ /// Returns true while emitting a cleanuppad.
+ bool isCleanupPadScope() const {
+ return CurrentFuncletPad && isa<llvm::CleanupPadInst>(CurrentFuncletPad);
+ }
+
/// pushFullExprCleanup - Push a cleanup to be run at the end of the
/// current full-expression. Safe against the possibility that
/// we're currently inside a conditionally-evaluated expression.
@@ -388,6 +410,8 @@
LifetimeExtendedCleanupStack.resize(
LifetimeExtendedCleanupStack.size() + sizeof(Header) + Header.Size);
+ static_assert(sizeof(Header) % llvm::AlignOf<T>::Alignment == 0,
+ "Cleanup will be allocated on misaligned address");
char *Buffer = &LifetimeExtendedCleanupStack[OldSize];
new (Buffer) LifetimeExtendedCleanupHeader(Header);
new (Buffer + sizeof(Header)) T(A...);
@@ -401,13 +425,12 @@
/// complete-object destructor of an object of the given type at the
/// given address. Does nothing if T is not a C++ class type with a
/// non-trivial destructor.
- void PushDestructorCleanup(QualType T, llvm::Value *Addr);
+ void PushDestructorCleanup(QualType T, Address Addr);
/// PushDestructorCleanup - Push a cleanup to call the
/// complete-object variant of the given destructor on the object at
/// the given address.
- void PushDestructorCleanup(const CXXDestructorDecl *Dtor,
- llvm::Value *Addr);
+ void PushDestructorCleanup(const CXXDestructorDecl *Dtor, Address Addr);
/// PopCleanupBlock - Will pop the cleanup entry on the stack and
/// process all branch fixups.
@@ -537,13 +560,14 @@
void rescopeLabels();
};
+ typedef llvm::DenseMap<const Decl *, Address> DeclMapTy;
+
/// \brief The scope used to remap some variables as private in the OpenMP
/// loop body (or other captured region emitted without outlining), and to
/// restore old vars back on exit.
class OMPPrivateScope : public RunCleanupsScope {
- typedef llvm::DenseMap<const VarDecl *, llvm::Value *> VarDeclMapTy;
- VarDeclMapTy SavedLocals;
- VarDeclMapTy SavedPrivates;
+ DeclMapTy SavedLocals;
+ DeclMapTy SavedPrivates;
private:
OMPPrivateScope(const OMPPrivateScope &) = delete;
@@ -560,13 +584,30 @@
/// been privatized already.
bool
addPrivate(const VarDecl *LocalVD,
- const std::function<llvm::Value *()> &PrivateGen) {
+ llvm::function_ref<Address()> PrivateGen) {
assert(PerformCleanup && "adding private to dead scope");
- if (SavedLocals.count(LocalVD) > 0) return false;
- SavedLocals[LocalVD] = CGF.LocalDeclMap.lookup(LocalVD);
- CGF.LocalDeclMap.erase(LocalVD);
- SavedPrivates[LocalVD] = PrivateGen();
- CGF.LocalDeclMap[LocalVD] = SavedLocals[LocalVD];
+
+ // Only save it once.
+ if (SavedLocals.count(LocalVD)) return false;
+
+ // Copy the existing local entry to SavedLocals.
+ auto it = CGF.LocalDeclMap.find(LocalVD);
+ if (it != CGF.LocalDeclMap.end()) {
+ SavedLocals.insert({LocalVD, it->second});
+ } else {
+ SavedLocals.insert({LocalVD, Address::invalid()});
+ }
+
+ // Generate the private entry.
+ Address Addr = PrivateGen();
+ QualType VarTy = LocalVD->getType();
+ if (VarTy->isReferenceType()) {
+ Address Temp = CGF.CreateMemTemp(VarTy);
+ CGF.Builder.CreateStore(Addr.getPointer(), Temp);
+ Addr = Temp;
+ }
+ SavedPrivates.insert({LocalVD, Addr});
+
return true;
}
@@ -579,19 +620,14 @@
/// private copies.
/// \return true if at least one variable was privatized, false otherwise.
bool Privatize() {
- for (auto VDPair : SavedPrivates) {
- CGF.LocalDeclMap[VDPair.first] = VDPair.second;
- }
+ copyInto(SavedPrivates, CGF.LocalDeclMap);
SavedPrivates.clear();
return !SavedLocals.empty();
}
void ForceCleanup() {
RunCleanupsScope::ForceCleanup();
- // Remap vars back to the original values.
- for (auto I : SavedLocals) {
- CGF.LocalDeclMap[I.first] = I.second;
- }
+ copyInto(SavedLocals, CGF.LocalDeclMap);
SavedLocals.clear();
}
@@ -600,6 +636,25 @@
if (PerformCleanup)
ForceCleanup();
}
+
+ private:
+ /// Copy all the entries in the source map over the corresponding
+ /// entries in the destination, which must exist.
+ static void copyInto(const DeclMapTy &src, DeclMapTy &dest) {
+ for (auto &pair : src) {
+ if (!pair.second.isValid()) {
+ dest.erase(pair.first);
+ continue;
+ }
+
+ auto it = dest.find(pair.first);
+ if (it != dest.end()) {
+ it->second = pair.second;
+ } else {
+ dest.insert(pair);
+ }
+ }
+ }
};
/// \brief Takes the old cleanup stack size and emits the cleanup blocks
@@ -647,6 +702,7 @@
llvm::BasicBlock *getEHResumeBlock(bool isCleanup);
llvm::BasicBlock *getEHDispatchBlock(EHScopeStack::stable_iterator scope);
+ llvm::BasicBlock *getMSVCDispatchBlock(EHScopeStack::stable_iterator scope);
/// An object to manage conditionally-evaluated expressions.
class ConditionalEvaluation {
@@ -679,10 +735,11 @@
/// one branch or the other of a conditional expression.
bool isInConditionalBranch() const { return OutermostConditional != nullptr; }
- void setBeforeOutermostConditional(llvm::Value *value, llvm::Value *addr) {
+ void setBeforeOutermostConditional(llvm::Value *value, Address addr) {
assert(isInConditionalBranch());
llvm::BasicBlock *block = OutermostConditional->getStartingBlock();
- new llvm::StoreInst(value, addr, &block->back());
+ auto store = new llvm::StoreInst(value, addr.getPointer(), &block->back());
+ store->setAlignment(addr.getAlignment().getQuantity());
}
/// An RAII object to record that we're evaluating a statement
@@ -841,15 +898,6 @@
}
};
- /// getByrefValueFieldNumber - Given a declaration, returns the LLVM field
- /// number that holds the value.
- std::pair<llvm::Type *, unsigned>
- getByRefValueLLVMField(const ValueDecl *VD) const;
-
- /// BuildBlockByrefAddress - Computes address location of the
- /// variable which is declared as __block.
- llvm::Value *BuildBlockByrefAddress(llvm::Value *BaseAddr,
- const VarDecl *V);
private:
CGDebugInfo *DebugInfo;
bool DisableDebugInfo;
@@ -866,11 +914,16 @@
/// LocalDeclMap - This keeps track of the LLVM allocas or globals for local C
/// decls.
- typedef llvm::DenseMap<const Decl*, llvm::Value*> DeclMapTy;
DeclMapTy LocalDeclMap;
+ /// SizeArguments - If a ParmVarDecl had the pass_object_size attribute, this
+ /// will contain a mapping from said ParmVarDecl to its implicit "object_size"
+ /// parameter.
+ llvm::SmallDenseMap<const ParmVarDecl *, const ImplicitParamDecl *, 2>
+ SizeArguments;
+
/// Track escaped local variables with auto storage. Used during SEH
- /// outlining to produce a call to llvm.frameescape.
+ /// outlining to produce a call to llvm.localescape.
llvm::DenseMap<llvm::AllocaInst *, int> EscapedLocals;
/// LabelMap - This keeps track of the LLVM basic block for each C label.
@@ -966,7 +1019,7 @@
/// if we need to evaluate a CXXDefaultInitExpr within the evaluation.
class FieldConstructionScope {
public:
- FieldConstructionScope(CodeGenFunction &CGF, llvm::Value *This)
+ FieldConstructionScope(CodeGenFunction &CGF, Address This)
: CGF(CGF), OldCXXDefaultInitExprThis(CGF.CXXDefaultInitExprThis) {
CGF.CXXDefaultInitExprThis = This;
}
@@ -976,7 +1029,7 @@
private:
CodeGenFunction &CGF;
- llvm::Value *OldCXXDefaultInitExprThis;
+ Address OldCXXDefaultInitExprThis;
};
/// The scope of a CXXDefaultInitExpr. Within this scope, the value of 'this'
@@ -984,16 +1037,20 @@
class CXXDefaultInitExprScope {
public:
CXXDefaultInitExprScope(CodeGenFunction &CGF)
- : CGF(CGF), OldCXXThisValue(CGF.CXXThisValue) {
- CGF.CXXThisValue = CGF.CXXDefaultInitExprThis;
+ : CGF(CGF), OldCXXThisValue(CGF.CXXThisValue),
+ OldCXXThisAlignment(CGF.CXXThisAlignment) {
+ CGF.CXXThisValue = CGF.CXXDefaultInitExprThis.getPointer();
+ CGF.CXXThisAlignment = CGF.CXXDefaultInitExprThis.getAlignment();
}
~CXXDefaultInitExprScope() {
CGF.CXXThisValue = OldCXXThisValue;
+ CGF.CXXThisAlignment = OldCXXThisAlignment;
}
public:
CodeGenFunction &CGF;
llvm::Value *OldCXXThisValue;
+ CharUnits OldCXXThisAlignment;
};
private:
@@ -1002,10 +1059,12 @@
ImplicitParamDecl *CXXABIThisDecl;
llvm::Value *CXXABIThisValue;
llvm::Value *CXXThisValue;
+ CharUnits CXXABIThisAlignment;
+ CharUnits CXXThisAlignment;
/// The value of 'this' to use when evaluating CXXDefaultInitExprs within
/// this expression.
- llvm::Value *CXXDefaultInitExprThis;
+ Address CXXDefaultInitExprThis = Address::invalid();
/// CXXStructorImplicitParamDecl - When generating code for a constructor or
/// destructor, this will hold the implicit argument (e.g. VTT).
@@ -1024,10 +1083,9 @@
/// handling code.
SourceLocation CurEHLocation;
- /// ByrefValueInfoMap - For each __block variable, contains a pair of the LLVM
- /// type as well as the field number that contains the actual data.
- llvm::DenseMap<const ValueDecl *, std::pair<llvm::Type *,
- unsigned> > ByRefValueInfo;
+ /// BlockByrefInfos - For each __block variable, contains
+ /// information about the layout of the variable.
+ llvm::DenseMap<const ValueDecl *, BlockByrefInfo> BlockByrefInfos;
llvm::BasicBlock *TerminateLandingPad;
llvm::BasicBlock *TerminateHandler;
@@ -1068,15 +1126,15 @@
/// Returns a pointer to the function's exception object and selector slot,
/// which is assigned in every landing pad.
- llvm::Value *getExceptionSlot();
- llvm::Value *getEHSelectorSlot();
+ Address getExceptionSlot();
+ Address getEHSelectorSlot();
/// Returns the contents of the function's exception object and selector
/// slots.
llvm::Value *getExceptionFromSlot();
llvm::Value *getSelectorFromSlot();
- llvm::Value *getNormalCleanupDestSlot();
+ Address getNormalCleanupDestSlot();
llvm::BasicBlock *getUnreachableBlock() {
if (!UnreachableBlock) {
@@ -1103,38 +1161,41 @@
// Cleanups
//===--------------------------------------------------------------------===//
- typedef void Destroyer(CodeGenFunction &CGF, llvm::Value *addr, QualType ty);
+ typedef void Destroyer(CodeGenFunction &CGF, Address addr, QualType ty);
void pushIrregularPartialArrayCleanup(llvm::Value *arrayBegin,
- llvm::Value *arrayEndPointer,
+ Address arrayEndPointer,
QualType elementType,
+ CharUnits elementAlignment,
Destroyer *destroyer);
void pushRegularPartialArrayCleanup(llvm::Value *arrayBegin,
llvm::Value *arrayEnd,
QualType elementType,
+ CharUnits elementAlignment,
Destroyer *destroyer);
void pushDestroy(QualType::DestructionKind dtorKind,
- llvm::Value *addr, QualType type);
+ Address addr, QualType type);
void pushEHDestroy(QualType::DestructionKind dtorKind,
- llvm::Value *addr, QualType type);
- void pushDestroy(CleanupKind kind, llvm::Value *addr, QualType type,
+ Address addr, QualType type);
+ void pushDestroy(CleanupKind kind, Address addr, QualType type,
Destroyer *destroyer, bool useEHCleanupForArray);
- void pushLifetimeExtendedDestroy(CleanupKind kind, llvm::Value *addr,
+ void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr,
QualType type, Destroyer *destroyer,
bool useEHCleanupForArray);
void pushCallObjectDeleteCleanup(const FunctionDecl *OperatorDelete,
llvm::Value *CompletePtr,
QualType ElementType);
- void pushStackRestore(CleanupKind kind, llvm::Value *SPMem);
- void emitDestroy(llvm::Value *addr, QualType type, Destroyer *destroyer,
+ void pushStackRestore(CleanupKind kind, Address SPMem);
+ void emitDestroy(Address addr, QualType type, Destroyer *destroyer,
bool useEHCleanupForArray);
- llvm::Function *generateDestroyHelper(llvm::Constant *addr, QualType type,
+ llvm::Function *generateDestroyHelper(Address addr, QualType type,
Destroyer *destroyer,
bool useEHCleanupForArray,
const VarDecl *VD);
void emitArrayDestroy(llvm::Value *begin, llvm::Value *end,
- QualType type, Destroyer *destroyer,
+ QualType elementType, CharUnits elementAlign,
+ Destroyer *destroyer,
bool checkZeroLength, bool useEHCleanup);
Destroyer *getDestroyer(QualType::DestructionKind destructionKind);
@@ -1185,8 +1246,6 @@
void generateObjCSetterBody(const ObjCImplementationDecl *classImpl,
const ObjCPropertyImplDecl *propImpl,
llvm::Constant *AtomicHelperFn);
- bool IndirectObjCSetterArg(const CGFunctionInfo &FI);
- bool IvarTypeWithAggrGCObjects(QualType Ty);
//===--------------------------------------------------------------------===//
// Block Bits
@@ -1195,10 +1254,6 @@
llvm::Value *EmitBlockLiteral(const BlockExpr *);
llvm::Value *EmitBlockLiteral(const CGBlockInfo &Info);
static void destroyBlockInfos(CGBlockInfo *info);
- llvm::Constant *BuildDescriptorBlockDecl(const BlockExpr *,
- const CGBlockInfo &Info,
- llvm::StructType *,
- llvm::Constant *BlockVarLayout);
llvm::Function *GenerateBlockFunction(GlobalDecl GD,
const CGBlockInfo &Info,
@@ -1220,15 +1275,22 @@
void emitByrefStructureInit(const AutoVarEmission &emission);
void enterByrefCleanup(const AutoVarEmission &emission);
- llvm::Value *LoadBlockStruct() {
- assert(BlockPointer && "no block pointer set!");
- return BlockPointer;
- }
+ void setBlockContextParameter(const ImplicitParamDecl *D, unsigned argNum,
+ llvm::Value *ptr);
- void AllocateBlockCXXThisPointer(const CXXThisExpr *E);
- void AllocateBlockDecl(const DeclRefExpr *E);
- llvm::Value *GetAddrOfBlockDecl(const VarDecl *var, bool ByRef);
- llvm::Type *BuildByRefType(const VarDecl *var);
+ Address LoadBlockStruct();
+ Address GetAddrOfBlockDecl(const VarDecl *var, bool ByRef);
+
+ /// BuildBlockByrefAddress - Computes the location of the
+ /// data in a variable which is declared as __block.
+ Address emitBlockByrefAddress(Address baseAddr, const VarDecl *V,
+ bool followForward = true);
+ Address emitBlockByrefAddress(Address baseAddr,
+ const BlockByrefInfo &info,
+ bool followForward,
+ const llvm::Twine &name);
+
+ const BlockByrefInfo &getBlockByrefInfo(const VarDecl *var);
void GenerateCode(GlobalDecl GD, llvm::Function *Fn,
const CGFunctionInfo &FnInfo);
@@ -1272,16 +1334,19 @@
void EmitCallAndReturnForThunk(llvm::Value *Callee, const ThunkInfo *Thunk);
+ void FinishThunk();
+
/// Emit a musttail call for a thunk with a potentially adjusted this pointer.
void EmitMustTailThunk(const CXXMethodDecl *MD, llvm::Value *AdjustedThisPtr,
llvm::Value *Callee);
- /// GenerateThunk - Generate a thunk for the given method.
- void GenerateThunk(llvm::Function *Fn, const CGFunctionInfo &FnInfo,
+ /// Generate a thunk for the given method.
+ void generateThunk(llvm::Function *Fn, const CGFunctionInfo &FnInfo,
GlobalDecl GD, const ThunkInfo &Thunk);
- void GenerateVarArgsThunk(llvm::Function *Fn, const CGFunctionInfo &FnInfo,
- GlobalDecl GD, const ThunkInfo &Thunk);
+ llvm::Function *GenerateVarArgsThunk(llvm::Function *Fn,
+ const CGFunctionInfo &FnInfo,
+ GlobalDecl GD, const ThunkInfo &Thunk);
void EmitCtorPrologue(const CXXConstructorDecl *CD, CXXCtorType Type,
FunctionArgList &Args);
@@ -1289,41 +1354,58 @@
void EmitInitializerForField(FieldDecl *Field, LValue LHS, Expr *Init,
ArrayRef<VarDecl *> ArrayIndexes);
- /// InitializeVTablePointer - Initialize the vtable pointer of the given
- /// subobject.
- ///
- void InitializeVTablePointer(BaseSubobject Base,
- const CXXRecordDecl *NearestVBase,
- CharUnits OffsetFromNearestVBase,
- const CXXRecordDecl *VTableClass);
+ /// Struct with all informations about dynamic [sub]class needed to set vptr.
+ struct VPtr {
+ BaseSubobject Base;
+ const CXXRecordDecl *NearestVBase;
+ CharUnits OffsetFromNearestVBase;
+ const CXXRecordDecl *VTableClass;
+ };
+
+ /// Initialize the vtable pointer of the given subobject.
+ void InitializeVTablePointer(const VPtr &vptr);
+
+ typedef llvm::SmallVector<VPtr, 4> VPtrsVector;
typedef llvm::SmallPtrSet<const CXXRecordDecl *, 4> VisitedVirtualBasesSetTy;
- void InitializeVTablePointers(BaseSubobject Base,
- const CXXRecordDecl *NearestVBase,
- CharUnits OffsetFromNearestVBase,
- bool BaseIsNonVirtualPrimaryBase,
- const CXXRecordDecl *VTableClass,
- VisitedVirtualBasesSetTy& VBases);
+ VPtrsVector getVTablePointers(const CXXRecordDecl *VTableClass);
+
+ void getVTablePointers(BaseSubobject Base, const CXXRecordDecl *NearestVBase,
+ CharUnits OffsetFromNearestVBase,
+ bool BaseIsNonVirtualPrimaryBase,
+ const CXXRecordDecl *VTableClass,
+ VisitedVirtualBasesSetTy &VBases, VPtrsVector &vptrs);
void InitializeVTablePointers(const CXXRecordDecl *ClassDecl);
/// GetVTablePtr - Return the Value of the vtable pointer member pointed
/// to by This.
- llvm::Value *GetVTablePtr(llvm::Value *This, llvm::Type *Ty);
+ llvm::Value *GetVTablePtr(Address This, llvm::Type *VTableTy,
+ const CXXRecordDecl *VTableClass);
+
+ enum CFITypeCheckKind {
+ CFITCK_VCall,
+ CFITCK_NVCall,
+ CFITCK_DerivedCast,
+ CFITCK_UnrelatedCast,
+ };
/// \brief Derived is the presumed address of an object of type T after a
/// cast. If T is a polymorphic class type, emit a check that the virtual
/// table for Derived belongs to a class derived from T.
void EmitVTablePtrCheckForCast(QualType T, llvm::Value *Derived,
- bool MayBeNull);
+ bool MayBeNull, CFITypeCheckKind TCK,
+ SourceLocation Loc);
/// EmitVTablePtrCheckForCall - Virtual method MD is being called via VTable.
/// If vptr CFI is enabled, emit a check that VTable is valid.
- void EmitVTablePtrCheckForCall(const CXXMethodDecl *MD, llvm::Value *VTable);
+ void EmitVTablePtrCheckForCall(const CXXMethodDecl *MD, llvm::Value *VTable,
+ CFITypeCheckKind TCK, SourceLocation Loc);
/// EmitVTablePtrCheck - Emit a check that VTable is a valid virtual table for
/// RD using llvm.bitset.test.
- void EmitVTablePtrCheck(const CXXRecordDecl *RD, llvm::Value *VTable);
+ void EmitVTablePtrCheck(const CXXRecordDecl *RD, llvm::Value *VTable,
+ CFITypeCheckKind TCK, SourceLocation Loc);
/// CanDevirtualizeMemberFunctionCalls - Checks whether virtual calls on given
/// expr can be devirtualized.
@@ -1466,49 +1548,85 @@
// Helpers
//===--------------------------------------------------------------------===//
- LValue MakeAddrLValue(llvm::Value *V, QualType T,
- CharUnits Alignment = CharUnits()) {
- return LValue::MakeAddr(V, T, Alignment, getContext(),
+ LValue MakeAddrLValue(Address Addr, QualType T,
+ AlignmentSource AlignSource = AlignmentSource::Type) {
+ return LValue::MakeAddr(Addr, T, getContext(), AlignSource,
CGM.getTBAAInfo(T));
}
+ LValue MakeAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
+ AlignmentSource AlignSource = AlignmentSource::Type) {
+ return LValue::MakeAddr(Address(V, Alignment), T, getContext(),
+ AlignSource, CGM.getTBAAInfo(T));
+ }
+
+ LValue MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T);
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T);
+ CharUnits getNaturalTypeAlignment(QualType T,
+ AlignmentSource *Source = nullptr,
+ bool forPointeeType = false);
+ CharUnits getNaturalPointeeTypeAlignment(QualType T,
+ AlignmentSource *Source = nullptr);
+
+ Address EmitLoadOfReference(Address Ref, const ReferenceType *RefTy,
+ AlignmentSource *Source = nullptr);
+ LValue EmitLoadOfReferenceLValue(Address Ref, const ReferenceType *RefTy);
/// CreateTempAlloca - This creates a alloca and inserts it into the entry
/// block. The caller is responsible for setting an appropriate alignment on
/// the alloca.
llvm::AllocaInst *CreateTempAlloca(llvm::Type *Ty,
const Twine &Name = "tmp");
+ Address CreateTempAlloca(llvm::Type *Ty, CharUnits align,
+ const Twine &Name = "tmp");
- /// InitTempAlloca - Provide an initial value for the given alloca.
- void InitTempAlloca(llvm::AllocaInst *Alloca, llvm::Value *Value);
+ /// CreateDefaultAlignedTempAlloca - This creates an alloca with the
+ /// default ABI alignment of the given LLVM type.
+ ///
+ /// IMPORTANT NOTE: This is *not* generally the right alignment for
+ /// any given AST type that happens to have been lowered to the
+ /// given IR type. This should only ever be used for function-local,
+ /// IR-driven manipulations like saving and restoring a value. Do
+ /// not hand this address off to arbitrary IRGen routines, and especially
+ /// do not pass it as an argument to a function that might expect a
+ /// properly ABI-aligned value.
+ Address CreateDefaultAlignTempAlloca(llvm::Type *Ty,
+ const Twine &Name = "tmp");
+
+ /// InitTempAlloca - Provide an initial value for the given alloca which
+ /// will be observable at all locations in the function.
+ ///
+ /// The address should be something that was returned from one of
+ /// the CreateTempAlloca or CreateMemTemp routines, and the
+ /// initializer must be valid in the entry block (i.e. it must
+ /// either be a constant or an argument value).
+ void InitTempAlloca(Address Alloca, llvm::Value *Value);
/// CreateIRTemp - Create a temporary IR object of the given type, with
/// appropriate alignment. This routine should only be used when an temporary
/// value needs to be stored into an alloca (for example, to avoid explicit
/// PHI construction), but the type is the IR type, not the type appropriate
/// for storing in memory.
- llvm::AllocaInst *CreateIRTemp(QualType T, const Twine &Name = "tmp");
+ ///
+ /// That is, this is exactly equivalent to CreateMemTemp, but calling
+ /// ConvertType instead of ConvertTypeForMem.
+ Address CreateIRTemp(QualType T, const Twine &Name = "tmp");
/// CreateMemTemp - Create a temporary memory object of the given type, with
/// appropriate alignment.
- llvm::AllocaInst *CreateMemTemp(QualType T, const Twine &Name = "tmp");
+ Address CreateMemTemp(QualType T, const Twine &Name = "tmp");
+ Address CreateMemTemp(QualType T, CharUnits Align, const Twine &Name = "tmp");
/// CreateAggTemp - Create a temporary memory object for the given
/// aggregate type.
AggValueSlot CreateAggTemp(QualType T, const Twine &Name = "tmp") {
- CharUnits Alignment = getContext().getTypeAlignInChars(T);
- return AggValueSlot::forAddr(CreateMemTemp(T, Name), Alignment,
+ return AggValueSlot::forAddr(CreateMemTemp(T, Name),
T.getQualifiers(),
AggValueSlot::IsNotDestructed,
AggValueSlot::DoesNotNeedGCBarriers,
AggValueSlot::IsNotAliased);
}
- /// CreateInAllocaTmp - Create a temporary memory object for the given
- /// aggregate type.
- AggValueSlot CreateInAllocaTmp(QualType T, const Twine &Name = "inalloca");
-
/// Emit a cast to void* in the appropriate address space.
llvm::Value *EmitCastToVoidPtr(llvm::Value *value);
@@ -1531,7 +1649,12 @@
// EmitVAListRef - Emit a "reference" to a va_list; this is either the address
// or the value of the expression, depending on how va_list is defined.
- llvm::Value *EmitVAListRef(const Expr *E);
+ Address EmitVAListRef(const Expr *E);
+
+ /// Emit a "reference" to a __builtin_ms_va_list; this is
+ /// always the value of the expression, because a __builtin_ms_va_list is a
+ /// pointer to a char.
+ Address EmitMSVAListRef(const Expr *E);
/// EmitAnyExprToTemp - Similary to EmitAnyExpr(), however, the result will
/// always be accessible even if no aggregate location is provided.
@@ -1539,10 +1662,10 @@
/// EmitAnyExprToMem - Emits the code necessary to evaluate an
/// arbitrary expression into the given memory location.
- void EmitAnyExprToMem(const Expr *E, llvm::Value *Location,
+ void EmitAnyExprToMem(const Expr *E, Address Location,
Qualifiers Quals, bool IsInitializer);
- void EmitAnyExprToExn(const Expr *E, llvm::Value *Addr);
+ void EmitAnyExprToExn(const Expr *E, Address Addr);
/// EmitExprAsInit - Emits the code necessary to initialize a
/// location in memory with the given initializer.
@@ -1562,19 +1685,15 @@
///
/// The difference to EmitAggregateCopy is that tail padding is not copied.
/// This is required for correctness when assigning non-POD structures in C++.
- void EmitAggregateAssign(llvm::Value *DestPtr, llvm::Value *SrcPtr,
+ void EmitAggregateAssign(Address DestPtr, Address SrcPtr,
QualType EltTy) {
bool IsVolatile = hasVolatileMember(EltTy);
- EmitAggregateCopy(DestPtr, SrcPtr, EltTy, IsVolatile, CharUnits::Zero(),
- true);
+ EmitAggregateCopy(DestPtr, SrcPtr, EltTy, IsVolatile, true);
}
- void EmitAggregateCopyCtor(llvm::Value *DestPtr, llvm::Value *SrcPtr,
- QualType DestTy, QualType SrcTy) {
- CharUnits DestTypeAlign = getContext().getTypeAlignInChars(DestTy);
- CharUnits SrcTypeAlign = getContext().getTypeAlignInChars(SrcTy);
+ void EmitAggregateCopyCtor(Address DestPtr, Address SrcPtr,
+ QualType DestTy, QualType SrcTy) {
EmitAggregateCopy(DestPtr, SrcPtr, SrcTy, /*IsVolatile=*/false,
- std::min(DestTypeAlign, SrcTypeAlign),
/*IsAssignment=*/false);
}
@@ -1584,20 +1703,16 @@
/// volatile.
/// \param isAssignment - If false, allow padding to be copied. This often
/// yields more efficient.
- void EmitAggregateCopy(llvm::Value *DestPtr, llvm::Value *SrcPtr,
+ void EmitAggregateCopy(Address DestPtr, Address SrcPtr,
QualType EltTy, bool isVolatile=false,
- CharUnits Alignment = CharUnits::Zero(),
bool isAssignment = false);
- /// StartBlock - Start new block named N. If insert block is a dummy block
- /// then reuse it.
- void StartBlock(const char *N);
-
/// GetAddrOfLocalVar - Return the address of a local variable.
- llvm::Value *GetAddrOfLocalVar(const VarDecl *VD) {
- llvm::Value *Res = LocalDeclMap[VD];
- assert(Res && "Invalid argument to GetAddrOfLocalVar(), no decl!");
- return Res;
+ Address GetAddrOfLocalVar(const VarDecl *VD) {
+ auto it = LocalDeclMap.find(VD);
+ assert(it != LocalDeclMap.end() &&
+ "Invalid argument to GetAddrOfLocalVar(), no decl!");
+ return it->second;
}
/// getOpaqueLValueMapping - Given an opaque value expression (which
@@ -1632,19 +1747,31 @@
/// EmitNullInitialization - Generate code to set a value of the given type to
/// null, If the type contains data member pointers, they will be initialized
/// to -1 in accordance with the Itanium C++ ABI.
- void EmitNullInitialization(llvm::Value *DestPtr, QualType Ty);
+ void EmitNullInitialization(Address DestPtr, QualType Ty);
- // EmitVAArg - Generate code to get an argument from the passed in pointer
- // and update it accordingly. The return value is a pointer to the argument.
+ /// Emits a call to an LLVM variable-argument intrinsic, either
+ /// \c llvm.va_start or \c llvm.va_end.
+ /// \param ArgValue A reference to the \c va_list as emitted by either
+ /// \c EmitVAListRef or \c EmitMSVAListRef.
+ /// \param IsStart If \c true, emits a call to \c llvm.va_start; otherwise,
+ /// calls \c llvm.va_end.
+ llvm::Value *EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart);
+
+ /// Generate code to get an argument from the passed in pointer
+ /// and update it accordingly.
+ /// \param VE The \c VAArgExpr for which to generate code.
+ /// \param VAListAddr Receives a reference to the \c va_list as emitted by
+ /// either \c EmitVAListRef or \c EmitMSVAListRef.
+ /// \returns A pointer to the argument.
// FIXME: We should be able to get rid of this method and use the va_arg
// instruction in LLVM instead once it works well enough.
- llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty);
+ Address EmitVAArg(VAArgExpr *VE, Address &VAListAddr);
/// emitArrayLength - Compute the length of an array, even if it's a
/// VLA, and drill down to the base element type.
llvm::Value *emitArrayLength(const ArrayType *arrayType,
QualType &baseType,
- llvm::Value *&addr);
+ Address &addr);
/// EmitVLASize - Capture all the sizes for the VLA expressions in
/// the given variably-modified type and store them in the VLASizeMap.
@@ -1665,6 +1792,7 @@
assert(CXXThisValue && "no 'this' value for this function");
return CXXThisValue;
}
+ Address LoadCXXThisAddress();
/// LoadCXXVTT - Load the VTT parameter to base constructors/destructors have
/// virtual bases.
@@ -1675,35 +1803,29 @@
return CXXStructorImplicitParamValue;
}
- /// LoadCXXStructorImplicitParam - Load the implicit parameter
- /// for a constructor/destructor.
- llvm::Value *LoadCXXStructorImplicitParam() {
- assert(CXXStructorImplicitParamValue &&
- "no implicit argument value for this function");
- return CXXStructorImplicitParamValue;
- }
-
/// GetAddressOfBaseOfCompleteClass - Convert the given pointer to a
/// complete class to the given direct base.
- llvm::Value *
- GetAddressOfDirectBaseInCompleteClass(llvm::Value *Value,
+ Address
+ GetAddressOfDirectBaseInCompleteClass(Address Value,
const CXXRecordDecl *Derived,
const CXXRecordDecl *Base,
bool BaseIsVirtual);
+ static bool ShouldNullCheckClassCastValue(const CastExpr *Cast);
+
/// GetAddressOfBaseClass - This function will add the necessary delta to the
/// load of 'this' and returns address of the base class.
- llvm::Value *GetAddressOfBaseClass(llvm::Value *Value,
- const CXXRecordDecl *Derived,
- CastExpr::path_const_iterator PathBegin,
- CastExpr::path_const_iterator PathEnd,
- bool NullCheckValue, SourceLocation Loc);
+ Address GetAddressOfBaseClass(Address Value,
+ const CXXRecordDecl *Derived,
+ CastExpr::path_const_iterator PathBegin,
+ CastExpr::path_const_iterator PathEnd,
+ bool NullCheckValue, SourceLocation Loc);
- llvm::Value *GetAddressOfDerivedClass(llvm::Value *Value,
- const CXXRecordDecl *Derived,
- CastExpr::path_const_iterator PathBegin,
- CastExpr::path_const_iterator PathEnd,
- bool NullCheckValue);
+ Address GetAddressOfDerivedClass(Address Value,
+ const CXXRecordDecl *Derived,
+ CastExpr::path_const_iterator PathBegin,
+ CastExpr::path_const_iterator PathEnd,
+ bool NullCheckValue);
/// GetVTTParameter - Return the VTT parameter that should be passed to a
/// base constructor/destructor with virtual bases.
@@ -1722,23 +1844,31 @@
// they are substantially the same.
void EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor,
const FunctionArgList &Args);
+
void EmitCXXConstructorCall(const CXXConstructorDecl *D, CXXCtorType Type,
bool ForVirtualBase, bool Delegating,
- llvm::Value *This, const CXXConstructExpr *E);
+ Address This, const CXXConstructExpr *E);
+
+ /// Emit assumption load for all bases. Requires to be be called only on
+ /// most-derived class and not under construction of the object.
+ void EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl, Address This);
+
+ /// Emit assumption that vptr load == global vtable.
+ void EmitVTableAssumptionLoad(const VPtr &vptr, Address This);
void EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D,
- llvm::Value *This, llvm::Value *Src,
- const CXXConstructExpr *E);
+ Address This, Address Src,
+ const CXXConstructExpr *E);
void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
const ConstantArrayType *ArrayTy,
- llvm::Value *ArrayPtr,
+ Address ArrayPtr,
const CXXConstructExpr *E,
bool ZeroInitialization = false);
void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
llvm::Value *NumElements,
- llvm::Value *ArrayPtr,
+ Address ArrayPtr,
const CXXConstructExpr *E,
bool ZeroInitialization = false);
@@ -1746,15 +1876,15 @@
void EmitCXXDestructorCall(const CXXDestructorDecl *D, CXXDtorType Type,
bool ForVirtualBase, bool Delegating,
- llvm::Value *This);
+ Address This);
void EmitNewArrayInitializer(const CXXNewExpr *E, QualType elementType,
- llvm::Type *ElementTy, llvm::Value *NewPtr,
+ llvm::Type *ElementTy, Address NewPtr,
llvm::Value *NumElements,
llvm::Value *AllocSizeWithoutCookie);
void EmitCXXTemporary(const CXXTemporary *Temporary, QualType TempType,
- llvm::Value *Ptr);
+ Address Ptr);
llvm::Value *EmitLifetimeStart(uint64_t Size, llvm::Value *Addr);
void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr);
@@ -1768,9 +1898,9 @@
RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type,
const Expr *Arg, bool IsDelete);
- llvm::Value* EmitCXXTypeidExpr(const CXXTypeidExpr *E);
- llvm::Value *EmitDynamicCast(llvm::Value *V, const CXXDynamicCastExpr *DCE);
- llvm::Value* EmitCXXUuidofExpr(const CXXUuidofExpr *E);
+ llvm::Value *EmitCXXTypeidExpr(const CXXTypeidExpr *E);
+ llvm::Value *EmitDynamicCast(Address V, const CXXDynamicCastExpr *DCE);
+ Address EmitCXXUuidofExpr(const CXXUuidofExpr *E);
/// \brief Situations in which we might emit a check for the suitability of a
/// pointer or glvalue.
@@ -1867,12 +1997,9 @@
const VarDecl *Variable;
- /// The alignment of the variable.
- CharUnits Alignment;
-
- /// The address of the alloca. Null if the variable was emitted
+ /// The address of the alloca. Invalid if the variable was emitted
/// as a global constant.
- llvm::Value *Address;
+ Address Addr;
llvm::Value *NRVOFlag;
@@ -1887,14 +2014,14 @@
llvm::Value *SizeForLifetimeMarkers;
struct Invalid {};
- AutoVarEmission(Invalid) : Variable(nullptr) {}
+ AutoVarEmission(Invalid) : Variable(nullptr), Addr(Address::invalid()) {}
AutoVarEmission(const VarDecl &variable)
- : Variable(&variable), Address(nullptr), NRVOFlag(nullptr),
+ : Variable(&variable), Addr(Address::invalid()), NRVOFlag(nullptr),
IsByRef(false), IsConstantAggregate(false),
SizeForLifetimeMarkers(nullptr) {}
- bool wasEmittedAsGlobal() const { return Address == nullptr; }
+ bool wasEmittedAsGlobal() const { return !Addr.isValid(); }
public:
static AutoVarEmission invalid() { return AutoVarEmission(Invalid()); }
@@ -1909,19 +2036,17 @@
/// Returns the raw, allocated address, which is not necessarily
/// the address of the object itself.
- llvm::Value *getAllocatedAddress() const {
- return Address;
+ Address getAllocatedAddress() const {
+ return Addr;
}
/// Returns the address of the object within this declaration.
/// Note that this does not chase the forwarding pointer for
/// __block decls.
- llvm::Value *getObjectAddress(CodeGenFunction &CGF) const {
- if (!IsByRef) return Address;
+ Address getObjectAddress(CodeGenFunction &CGF) const {
+ if (!IsByRef) return Addr;
- auto F = CGF.getByRefValueLLVMField(Variable);
- return CGF.Builder.CreateStructGEP(F.first, Address, F.second,
- Variable->getNameAsString());
+ return CGF.emitBlockByrefAddress(Addr, Variable, /*forward*/ false);
}
};
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var);
@@ -1933,9 +2058,35 @@
void EmitStaticVarDecl(const VarDecl &D,
llvm::GlobalValue::LinkageTypes Linkage);
+ class ParamValue {
+ llvm::Value *Value;
+ unsigned Alignment;
+ ParamValue(llvm::Value *V, unsigned A) : Value(V), Alignment(A) {}
+ public:
+ static ParamValue forDirect(llvm::Value *value) {
+ return ParamValue(value, 0);
+ }
+ static ParamValue forIndirect(Address addr) {
+ assert(!addr.getAlignment().isZero());
+ return ParamValue(addr.getPointer(), addr.getAlignment().getQuantity());
+ }
+
+ bool isIndirect() const { return Alignment != 0; }
+ llvm::Value *getAnyValue() const { return Value; }
+
+ llvm::Value *getDirectValue() const {
+ assert(!isIndirect());
+ return Value;
+ }
+
+ Address getIndirectAddress() const {
+ assert(isIndirect());
+ return Address(Value, CharUnits::fromQuantity(Alignment));
+ }
+ };
+
/// EmitParmDecl - Emit a ParmVarDecl or an ImplicitParamDecl.
- void EmitParmDecl(const VarDecl &D, llvm::Value *Arg, bool ArgIsPointer,
- unsigned ArgNo);
+ void EmitParmDecl(const VarDecl &D, ParamValue Arg, unsigned ArgNo);
/// protectFromPeepholes - Protect a value that we're intending to
/// store to the side, but which will probably be used later, from
@@ -1972,11 +2123,11 @@
/// \return True if the statement was handled.
bool EmitSimpleStmt(const Stmt *S);
- llvm::Value *EmitCompoundStmt(const CompoundStmt &S, bool GetLast = false,
- AggValueSlot AVS = AggValueSlot::ignored());
- llvm::Value *EmitCompoundStmtWithoutScope(const CompoundStmt &S,
- bool GetLast = false,
- AggValueSlot AVS =
+ Address EmitCompoundStmt(const CompoundStmt &S, bool GetLast = false,
+ AggValueSlot AVS = AggValueSlot::ignored());
+ Address EmitCompoundStmtWithoutScope(const CompoundStmt &S,
+ bool GetLast = false,
+ AggValueSlot AVS =
AggValueSlot::ignored());
/// EmitLabel - Emit the block for the given label. It is legal to call this
@@ -1989,8 +2140,6 @@
void EmitIndirectGotoStmt(const IndirectGotoStmt &S);
void EmitIfStmt(const IfStmt &S);
- void EmitCondBrHints(llvm::LLVMContext &Context, llvm::BranchInst *CondBr,
- ArrayRef<const Attr *> Attrs);
void EmitWhileStmt(const WhileStmt &S,
ArrayRef<const Attr *> Attrs = None);
void EmitDoStmt(const DoStmt &S, ArrayRef<const Attr *> Attrs = None);
@@ -2021,8 +2170,7 @@
void EnterSEHTryStmt(const SEHTryStmt &S);
void ExitSEHTryStmt(const SEHTryStmt &S);
- void startOutlinedSEHHelper(CodeGenFunction &ParentCGF, StringRef Name,
- QualType RetTy, FunctionArgList &Args,
+ void startOutlinedSEHHelper(CodeGenFunction &ParentCGF, bool IsFilter,
const Stmt *OutlinedStmt);
llvm::Function *GenerateSEHFilterFunction(CodeGenFunction &ParentCGF,
@@ -2031,26 +2179,38 @@
llvm::Function *GenerateSEHFinallyFunction(CodeGenFunction &ParentCGF,
const SEHFinallyStmt &Finally);
- void EmitSEHExceptionCodeSave();
+ void EmitSEHExceptionCodeSave(CodeGenFunction &ParentCGF,
+ llvm::Value *ParentFP,
+ llvm::Value *EntryEBP);
llvm::Value *EmitSEHExceptionCode();
llvm::Value *EmitSEHExceptionInfo();
llvm::Value *EmitSEHAbnormalTermination();
/// Scan the outlined statement for captures from the parent function. For
/// each capture, mark the capture as escaped and emit a call to
- /// llvm.framerecover. Insert the framerecover result into the LocalDeclMap.
+ /// llvm.localrecover. Insert the localrecover result into the LocalDeclMap.
void EmitCapturedLocals(CodeGenFunction &ParentCGF, const Stmt *OutlinedStmt,
- llvm::Value *ParentFP);
+ bool IsFilter);
+
+ /// Recovers the address of a local in a parent function. ParentVar is the
+ /// address of the variable used in the immediate parent function. It can
+ /// either be an alloca or a call to llvm.localrecover if there are nested
+ /// outlined functions. ParentFP is the frame pointer of the outermost parent
+ /// frame.
+ Address recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF,
+ Address ParentVar,
+ llvm::Value *ParentFP);
void EmitCXXForRangeStmt(const CXXForRangeStmt &S,
ArrayRef<const Attr *> Attrs = None);
LValue InitCapturedStruct(const CapturedStmt &S);
llvm::Function *EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K);
- void GenerateCapturedStmtFunctionProlog(const CapturedStmt &S);
- llvm::Function *GenerateCapturedStmtFunctionEpilog(const CapturedStmt &S);
llvm::Function *GenerateCapturedStmtFunction(const CapturedStmt &S);
- llvm::Value *GenerateCapturedStmtArgument(const CapturedStmt &S);
+ Address GenerateCapturedStmtArgument(const CapturedStmt &S);
+ llvm::Function *GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S);
+ void GenerateOpenMPCapturedVars(const CapturedStmt &S,
+ SmallVectorImpl<llvm::Value *> &CapturedVars);
/// \brief Perform element by element copying of arrays with type \a
/// OriginalType from \a SrcAddr to \a DestAddr using copying procedure
/// generated by \a CopyGen.
@@ -2061,8 +2221,8 @@
/// \param CopyGen Copying procedure that copies value of single array element
/// to another single array element.
void EmitOMPAggregateAssign(
- llvm::Value *DestAddr, llvm::Value *SrcAddr, QualType OriginalType,
- const llvm::function_ref<void(llvm::Value *, llvm::Value *)> &CopyGen);
+ Address DestAddr, Address SrcAddr, QualType OriginalType,
+ const llvm::function_ref<void(Address, Address)> &CopyGen);
/// \brief Emit proper copying of data from one variable to another.
///
/// \param OriginalType Original type of the copied variables.
@@ -2074,8 +2234,8 @@
/// the base array element).
/// \param Copy Actual copygin expression for copying data from \a SrcVD to \a
/// DestVD.
- void EmitOMPCopy(CodeGenFunction &CGF, QualType OriginalType,
- llvm::Value *DestAddr, llvm::Value *SrcAddr,
+ void EmitOMPCopy(QualType OriginalType,
+ Address DestAddr, Address SrcAddr,
const VarDecl *DestVD, const VarDecl *SrcVD,
const Expr *Copy);
/// \brief Emit atomic update code for constructs: \a X = \a X \a BO \a E or
@@ -2130,9 +2290,9 @@
/// \param D Directive that has at least one 'lastprivate' directives.
/// \param IsLastIterCond Boolean condition that must be set to 'i1 true' if
/// it is the last iteration of the loop code in associated directive, or to
- /// 'i1 false' otherwise.
+ /// 'i1 false' otherwise. If this item is nullptr, no final check is required.
void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D,
- llvm::Value *IsLastIterCond);
+ llvm::Value *IsLastIterCond = nullptr);
/// \brief Emit initial code for reduction variables. Creates reduction copies
/// and initializes them with the values according to OpenMP standard.
///
@@ -2147,6 +2307,11 @@
///
/// \param D Directive that has at least one 'reduction' directives.
void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D);
+ /// \brief Emit initial code for linear variables. Creates private copies
+ /// and initializes them with the values according to OpenMP standard.
+ ///
+ /// \param D Directive (possibly) with the 'linear' clause.
+ void EmitOMPLinearClauseInit(const OMPLoopDirective &D);
void EmitOMPParallelDirective(const OMPParallelDirective &S);
void EmitOMPSimdDirective(const OMPSimdDirective &S);
@@ -2164,11 +2329,19 @@
void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S);
void EmitOMPBarrierDirective(const OMPBarrierDirective &S);
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S);
+ void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S);
void EmitOMPFlushDirective(const OMPFlushDirective &S);
void EmitOMPOrderedDirective(const OMPOrderedDirective &S);
void EmitOMPAtomicDirective(const OMPAtomicDirective &S);
void EmitOMPTargetDirective(const OMPTargetDirective &S);
+ void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S);
void EmitOMPTeamsDirective(const OMPTeamsDirective &S);
+ void
+ EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S);
+ void EmitOMPCancelDirective(const OMPCancelDirective &S);
+ void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S);
+ void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S);
+ void EmitOMPDistributeDirective(const OMPDistributeDirective &S);
/// \brief Emit inner loop of the worksharing/simd construct.
///
@@ -2186,12 +2359,14 @@
const llvm::function_ref<void(CodeGenFunction &)> &BodyGen,
const llvm::function_ref<void(CodeGenFunction &)> &PostIncGen);
+ JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind);
+
private:
/// Helpers for the OpenMP loop directives.
- void EmitOMPLoopBody(const OMPLoopDirective &Directive,
- bool SeparateIter = false);
- void EmitOMPSimdFinal(const OMPLoopDirective &S);
+ void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit);
+ void EmitOMPSimdInit(const OMPLoopDirective &D);
+ void EmitOMPSimdFinal(const OMPLoopDirective &D);
/// \brief Emit code for the worksharing loop-based directive.
/// \return true, if this construct has any lastprivate clause, false -
/// otherwise.
@@ -2199,8 +2374,10 @@
void EmitOMPForOuterLoop(OpenMPScheduleClauseKind ScheduleKind,
const OMPLoopDirective &S,
OMPPrivateScope &LoopScope, bool Ordered,
- llvm::Value *LB, llvm::Value *UB, llvm::Value *ST,
- llvm::Value *IL, llvm::Value *Chunk);
+ Address LB, Address UB, Address ST,
+ Address IL, llvm::Value *Chunk);
+ /// \brief Emit code for sections directive.
+ OpenMPDirectiveKind EmitSections(const OMPExecutableDirective &S);
public:
@@ -2245,7 +2422,7 @@
/// that the address will be used to access the object.
LValue EmitCheckedLValue(const Expr *E, TypeCheckKind TCK);
- RValue convertTempToRValue(llvm::Value *addr, QualType type,
+ RValue convertTempToRValue(Address addr, QualType type,
SourceLocation Loc);
void EmitAtomicInit(Expr *E, LValue lvalue);
@@ -2286,12 +2463,14 @@
/// EmitLoadOfScalar - Load a scalar value from an address, taking
/// care to appropriately convert from the memory representation to
/// the LLVM value representation.
- llvm::Value *EmitLoadOfScalar(llvm::Value *Addr, bool Volatile,
- unsigned Alignment, QualType Ty,
+ llvm::Value *EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty,
SourceLocation Loc,
+ AlignmentSource AlignSource =
+ AlignmentSource::Type,
llvm::MDNode *TBAAInfo = nullptr,
QualType TBAABaseTy = QualType(),
- uint64_t TBAAOffset = 0);
+ uint64_t TBAAOffset = 0,
+ bool isNontemporal = false);
/// EmitLoadOfScalar - Load a scalar value from an address, taking
/// care to appropriately convert from the memory representation to
@@ -2302,11 +2481,12 @@
/// EmitStoreOfScalar - Store a scalar value to an address, taking
/// care to appropriately convert from the memory representation to
/// the LLVM value representation.
- void EmitStoreOfScalar(llvm::Value *Value, llvm::Value *Addr,
- bool Volatile, unsigned Alignment, QualType Ty,
+ void EmitStoreOfScalar(llvm::Value *Value, Address Addr,
+ bool Volatile, QualType Ty,
+ AlignmentSource AlignSource = AlignmentSource::Type,
llvm::MDNode *TBAAInfo = nullptr, bool isInit = false,
QualType TBAABaseTy = QualType(),
- uint64_t TBAAOffset = 0);
+ uint64_t TBAAOffset = 0, bool isNontemporal = false);
/// EmitStoreOfScalar - Store a scalar value to an address, taking
/// care to appropriately convert from the memory representation to
@@ -2353,13 +2533,14 @@
// Note: only available for agg return types
LValue EmitVAArgExprLValue(const VAArgExpr *E);
LValue EmitDeclRefLValue(const DeclRefExpr *E);
- LValue EmitReadRegister(const VarDecl *VD);
LValue EmitStringLiteralLValue(const StringLiteral *E);
LValue EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E);
LValue EmitPredefinedLValue(const PredefinedExpr *E);
LValue EmitUnaryOpLValue(const UnaryOperator *E);
LValue EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
bool Accessed = false);
+ LValue EmitOMPArraySectionExpr(const OMPArraySectionExpr *E,
+ bool IsLowerBound = true);
LValue EmitExtVectorElementExpr(const ExtVectorElementExpr *E);
LValue EmitMemberExpr(const MemberExpr *E);
LValue EmitObjCIsaExpr(const ObjCIsaExpr *E);
@@ -2370,10 +2551,13 @@
LValue EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *E);
LValue EmitOpaqueValueLValue(const OpaqueValueExpr *e);
- llvm::Value *EmitExtVectorElementLValue(LValue V);
+ Address EmitExtVectorElementLValue(LValue V);
RValue EmitRValueForField(LValue LV, const FieldDecl *FD, SourceLocation Loc);
+ Address EmitArrayToPointerDecay(const Expr *Array,
+ AlignmentSource *AlignSource = nullptr);
+
class ConstantEmission {
llvm::PointerIntPair<llvm::Constant*, 1, bool> ValueAndIsReference;
ConstantEmission(llvm::Constant *C, bool isReference)
@@ -2445,23 +2629,20 @@
/// EmitCall - Generate a call of the given function, expecting the given
/// result type, and using the given argument list which specifies both the
/// LLVM arguments and the types they were derived from.
- ///
- /// \param TargetDecl - If given, the decl of the function in a direct call;
- /// used to set attributes on the call (noreturn, etc.).
- RValue EmitCall(const CGFunctionInfo &FnInfo,
- llvm::Value *Callee,
- ReturnValueSlot ReturnValue,
- const CallArgList &Args,
- const Decl *TargetDecl = nullptr,
+ RValue EmitCall(const CGFunctionInfo &FnInfo, llvm::Value *Callee,
+ ReturnValueSlot ReturnValue, const CallArgList &Args,
+ CGCalleeInfo CalleeInfo = CGCalleeInfo(),
llvm::Instruction **callOrInvoke = nullptr);
RValue EmitCall(QualType FnType, llvm::Value *Callee, const CallExpr *E,
ReturnValueSlot ReturnValue,
- const Decl *TargetDecl = nullptr,
+ CGCalleeInfo CalleeInfo = CGCalleeInfo(),
llvm::Value *Chain = nullptr);
RValue EmitCallExpr(const CallExpr *E,
ReturnValueSlot ReturnValue = ReturnValueSlot());
+ void checkTargetFeatures(const CallExpr *E, const FunctionDecl *TargetDecl);
+
llvm::CallInst *EmitRuntimeCall(llvm::Value *callee,
const Twine &name = "");
llvm::CallInst *EmitRuntimeCall(llvm::Value *callee,
@@ -2476,8 +2657,6 @@
llvm::CallSite EmitCallOrInvoke(llvm::Value *Callee,
ArrayRef<llvm::Value *> Args,
const Twine &Name = "");
- llvm::CallSite EmitCallOrInvoke(llvm::Value *Callee,
- const Twine &Name = "");
llvm::CallSite EmitRuntimeCallOrInvoke(llvm::Value *callee,
ArrayRef<llvm::Value*> args,
const Twine &name = "");
@@ -2513,6 +2692,10 @@
NestedNameSpecifier *Qualifier,
bool IsArrow, const Expr *Base);
// Compute the object pointer.
+ Address EmitCXXMemberDataPointerAddress(const Expr *E, Address base,
+ llvm::Value *memberPtr,
+ const MemberPointerType *memberPtrType,
+ AlignmentSource *AlignSource = nullptr);
RValue EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,
ReturnValueSlot ReturnValue);
@@ -2547,7 +2730,7 @@
unsigned Modifier,
const CallExpr *E,
SmallVectorImpl<llvm::Value *> &Ops,
- llvm::Value *Align = nullptr);
+ Address PtrOp0, Address PtrOp1);
llvm::Function *LookupNeonLLVMIntrinsic(unsigned IntrinsicID,
unsigned Modifier, llvm::Type *ArgTy,
const CallExpr *E);
@@ -2560,16 +2743,17 @@
bool negateForRightShift);
llvm::Value *EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt,
llvm::Type *Ty, bool usgn, const char *name);
- // Helper functions for EmitAArch64BuiltinExpr.
- llvm::Value *vectorWrapScalar8(llvm::Value *Op);
llvm::Value *vectorWrapScalar16(llvm::Value *Op);
llvm::Value *EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E);
llvm::Value *BuildVector(ArrayRef<llvm::Value*> Ops);
llvm::Value *EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E);
llvm::Value *EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
- llvm::Value *EmitR600BuiltinExpr(unsigned BuiltinID, const CallExpr *E);
+ llvm::Value *EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
llvm::Value *EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
+ llvm::Value *EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
+ llvm::Value *EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
+ const CallExpr *E);
llvm::Value *EmitObjCProtocolExpr(const ObjCProtocolExpr *E);
llvm::Value *EmitObjCStringLiteral(const ObjCStringLiteral *E);
@@ -2590,24 +2774,23 @@
}
// ARC primitives.
- void EmitARCInitWeak(llvm::Value *value, llvm::Value *addr);
- void EmitARCDestroyWeak(llvm::Value *addr);
- llvm::Value *EmitARCLoadWeak(llvm::Value *addr);
- llvm::Value *EmitARCLoadWeakRetained(llvm::Value *addr);
- llvm::Value *EmitARCStoreWeak(llvm::Value *value, llvm::Value *addr,
- bool ignored);
- void EmitARCCopyWeak(llvm::Value *dst, llvm::Value *src);
- void EmitARCMoveWeak(llvm::Value *dst, llvm::Value *src);
+ void EmitARCInitWeak(Address addr, llvm::Value *value);
+ void EmitARCDestroyWeak(Address addr);
+ llvm::Value *EmitARCLoadWeak(Address addr);
+ llvm::Value *EmitARCLoadWeakRetained(Address addr);
+ llvm::Value *EmitARCStoreWeak(Address addr, llvm::Value *value, bool ignored);
+ void EmitARCCopyWeak(Address dst, Address src);
+ void EmitARCMoveWeak(Address dst, Address src);
llvm::Value *EmitARCRetainAutorelease(QualType type, llvm::Value *value);
llvm::Value *EmitARCRetainAutoreleaseNonBlock(llvm::Value *value);
llvm::Value *EmitARCStoreStrong(LValue lvalue, llvm::Value *value,
bool resultIgnored);
- llvm::Value *EmitARCStoreStrongCall(llvm::Value *addr, llvm::Value *value,
+ llvm::Value *EmitARCStoreStrongCall(Address addr, llvm::Value *value,
bool resultIgnored);
llvm::Value *EmitARCRetain(QualType type, llvm::Value *value);
llvm::Value *EmitARCRetainNonBlock(llvm::Value *value);
llvm::Value *EmitARCRetainBlock(llvm::Value *value, bool mandatory);
- void EmitARCDestroyStrong(llvm::Value *addr, ARCPreciseLifetime_t precise);
+ void EmitARCDestroyStrong(Address addr, ARCPreciseLifetime_t precise);
void EmitARCRelease(llvm::Value *value, ARCPreciseLifetime_t precise);
llvm::Value *EmitARCAutorelease(llvm::Value *value);
llvm::Value *EmitARCAutoreleaseReturnValue(llvm::Value *value);
@@ -2620,8 +2803,6 @@
EmitARCStoreStrong(const BinaryOperator *e, bool ignored);
llvm::Value *EmitObjCThrowOperand(const Expr *expr);
-
- llvm::Value *EmitObjCProduceObject(QualType T, llvm::Value *Ptr);
llvm::Value *EmitObjCConsumeObject(QualType T, llvm::Value *Ptr);
llvm::Value *EmitObjCExtendObjectLifetime(QualType T, llvm::Value *Ptr);
@@ -2654,17 +2835,16 @@
/// scalar type, returning the result.
llvm::Value *EmitScalarExpr(const Expr *E , bool IgnoreResultAssign = false);
- /// EmitScalarConversion - Emit a conversion from the specified type to the
- /// specified destination type, both of which are LLVM scalar types.
+ /// Emit a conversion from the specified type to the specified destination
+ /// type, both of which are LLVM scalar types.
llvm::Value *EmitScalarConversion(llvm::Value *Src, QualType SrcTy,
- QualType DstTy);
+ QualType DstTy, SourceLocation Loc);
- /// EmitComplexToScalarConversion - Emit a conversion from the specified
- /// complex type to the specified destination type, where the destination type
- /// is an LLVM scalar type.
+ /// Emit a conversion from the specified complex type to the specified
+ /// destination type, where the destination type is an LLVM scalar type.
llvm::Value *EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy,
- QualType DstTy);
-
+ QualType DstTy,
+ SourceLocation Loc);
/// EmitAggExpr - Emit the computation of the specified expression
/// of aggregate type. The result is computed into the given slot,
@@ -2675,11 +2855,6 @@
/// aggregate type into a temporary LValue.
LValue EmitAggExprToLValue(const Expr *E);
- /// EmitGCMemmoveCollectable - Emit special API for structs with object
- /// pointers.
- void EmitGCMemmoveCollectable(llvm::Value *DestPtr, llvm::Value *SrcPtr,
- QualType Ty);
-
/// EmitExtendGCLifetime - Given a pointer to an Objective-C object,
/// make sure it survives garbage collection until this point.
void EmitExtendGCLifetime(llvm::Value *object);
@@ -2700,6 +2875,9 @@
/// EmitLoadOfComplex - Load a complex number from the specified l-value.
ComplexPairTy EmitLoadOfComplex(LValue src, SourceLocation loc);
+ Address emitAddrOfRealComponent(Address complex, QualType complexType);
+ Address emitAddrOfImagComponent(Address complex, QualType complexType);
+
/// AddInitializerToStaticVarDecl - Add the initializer for 'D' to the
/// global variable that has already been created for it. If the initializer
/// has a different type than GV does, this may free GV and return a different
@@ -2734,7 +2912,7 @@
/// variables.
void GenerateCXXGlobalInitFunc(llvm::Function *Fn,
ArrayRef<llvm::Function *> CXXThreadLocals,
- llvm::GlobalVariable *Guard = nullptr);
+ Address Guard = Address::invalid());
/// GenerateCXXGlobalDtorsFunc - Generates code for destroying global
/// variables.
@@ -2749,8 +2927,7 @@
void EmitCXXConstructExpr(const CXXConstructExpr *E, AggValueSlot Dest);
- void EmitSynthesizedCXXCopyCtor(llvm::Value *Dest, llvm::Value *Src,
- const Expr *Exp);
+ void EmitSynthesizedCXXCopyCtor(Address Dest, Address Src, const Expr *Exp);
void enterFullExpression(const ExprWithCleanups *E) {
if (E->getNumObjects() == 0) return;
@@ -2762,7 +2939,7 @@
void EmitLambdaExpr(const LambdaExpr *E, AggValueSlot Dest);
- RValue EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest = nullptr);
+ RValue EmitAtomicExpr(AtomicExpr *E);
//===--------------------------------------------------------------------===//
// Annotations Emission
@@ -2779,7 +2956,7 @@
/// Emit field annotations for the given field & value. Returns the
/// annotation result.
- llvm::Value *EmitFieldAnnotations(const FieldDecl *D, llvm::Value *V);
+ Address EmitFieldAnnotations(const FieldDecl *D, Address V);
//===--------------------------------------------------------------------===//
// Internal Helpers
@@ -2832,10 +3009,19 @@
StringRef CheckName, ArrayRef<llvm::Constant *> StaticArgs,
ArrayRef<llvm::Value *> DynamicArgs);
+ /// \brief Emit a slow path cross-DSO CFI check which calls __cfi_slowpath
+ /// if Cond if false.
+ void EmitCfiSlowPathCheck(llvm::Value *Cond, llvm::ConstantInt *TypeId,
+ llvm::Value *Ptr);
+
/// \brief Create a basic block that will call the trap intrinsic, and emit a
/// conditional branch to it, for the -ftrapv checks.
void EmitTrapCheck(llvm::Value *Checked);
+ /// \brief Emit a call to trap or debugtrap and attach function attribute
+ /// "trap-func-name" if specified.
+ llvm::CallInst *EmitTrapCall(llvm::Intrinsic::ID IntrID);
+
/// \brief Create a check for a function parameter that may potentially be
/// declared as non-null.
void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc,
@@ -2863,6 +3049,12 @@
llvm::SmallVector<std::pair<llvm::Instruction *, llvm::Value *>, 4>
DeferredReplacements;
+ /// Set the address of a local variable.
+ void setAddrOfLocalVar(const VarDecl *VD, Address Addr) {
+ assert(!LocalDeclMap.count(VD) && "Decl already exists in LocalDeclMap!");
+ LocalDeclMap.insert({VD, Addr});
+ }
+
/// ExpandTypeFromArgs - Reconstruct a structure of type \arg Ty
/// from function arguments into \arg Dst. See ABIArgInfo::Expand.
///
@@ -2885,55 +3077,116 @@
std::string &ConstraintStr,
SourceLocation Loc);
+ /// \brief Attempts to statically evaluate the object size of E. If that
+ /// fails, emits code to figure the size of E out for us. This is
+ /// pass_object_size aware.
+ llvm::Value *evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
+ llvm::IntegerType *ResType);
+
+ /// \brief Emits the size of E, as required by __builtin_object_size. This
+ /// function is aware of pass_object_size parameters, and will act accordingly
+ /// if E is a parameter with the pass_object_size attribute.
+ llvm::Value *emitBuiltinObjectSize(const Expr *E, unsigned Type,
+ llvm::IntegerType *ResType);
+
public:
+#ifndef NDEBUG
+ // Determine whether the given argument is an Objective-C method
+ // that may have type parameters in its signature.
+ static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) {
+ const DeclContext *dc = method->getDeclContext();
+ if (const ObjCInterfaceDecl *classDecl= dyn_cast<ObjCInterfaceDecl>(dc)) {
+ return classDecl->getTypeParamListAsWritten();
+ }
+
+ if (const ObjCCategoryDecl *catDecl = dyn_cast<ObjCCategoryDecl>(dc)) {
+ return catDecl->getTypeParamList();
+ }
+
+ return false;
+ }
+
+ template<typename T>
+ static bool isObjCMethodWithTypeParams(const T *) { return false; }
+#endif
+
/// EmitCallArgs - Emit call arguments for a function.
template <typename T>
void EmitCallArgs(CallArgList &Args, const T *CallArgTypeInfo,
- CallExpr::const_arg_iterator ArgBeg,
- CallExpr::const_arg_iterator ArgEnd,
+ llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
const FunctionDecl *CalleeDecl = nullptr,
unsigned ParamsToSkip = 0) {
SmallVector<QualType, 16> ArgTypes;
- CallExpr::const_arg_iterator Arg = ArgBeg;
+ CallExpr::const_arg_iterator Arg = ArgRange.begin();
assert((ParamsToSkip == 0 || CallArgTypeInfo) &&
"Can't skip parameters if type info is not provided");
if (CallArgTypeInfo) {
+#ifndef NDEBUG
+ bool isGenericMethod = isObjCMethodWithTypeParams(CallArgTypeInfo);
+#endif
+
// First, use the argument types that the type info knows about
for (auto I = CallArgTypeInfo->param_type_begin() + ParamsToSkip,
E = CallArgTypeInfo->param_type_end();
I != E; ++I, ++Arg) {
- assert(Arg != ArgEnd && "Running over edge of argument list!");
- assert(
- ((*I)->isVariablyModifiedType() ||
- getContext()
- .getCanonicalType((*I).getNonReferenceType())
- .getTypePtr() ==
- getContext().getCanonicalType(Arg->getType()).getTypePtr()) &&
- "type mismatch in call argument!");
+ assert(Arg != ArgRange.end() && "Running over edge of argument list!");
+ assert((isGenericMethod ||
+ ((*I)->isVariablyModifiedType() ||
+ (*I).getNonReferenceType()->isObjCRetainableType() ||
+ getContext()
+ .getCanonicalType((*I).getNonReferenceType())
+ .getTypePtr() ==
+ getContext()
+ .getCanonicalType((*Arg)->getType())
+ .getTypePtr())) &&
+ "type mismatch in call argument!");
ArgTypes.push_back(*I);
}
}
// Either we've emitted all the call args, or we have a call to variadic
// function.
- assert(
- (Arg == ArgEnd || !CallArgTypeInfo || CallArgTypeInfo->isVariadic()) &&
- "Extra arguments in non-variadic function!");
+ assert((Arg == ArgRange.end() || !CallArgTypeInfo ||
+ CallArgTypeInfo->isVariadic()) &&
+ "Extra arguments in non-variadic function!");
// If we still have any arguments, emit them using the type of the argument.
- for (; Arg != ArgEnd; ++Arg)
- ArgTypes.push_back(getVarArgType(*Arg));
+ for (auto *A : llvm::make_range(Arg, ArgRange.end()))
+ ArgTypes.push_back(getVarArgType(A));
- EmitCallArgs(Args, ArgTypes, ArgBeg, ArgEnd, CalleeDecl, ParamsToSkip);
+ EmitCallArgs(Args, ArgTypes, ArgRange, CalleeDecl, ParamsToSkip);
}
void EmitCallArgs(CallArgList &Args, ArrayRef<QualType> ArgTypes,
- CallExpr::const_arg_iterator ArgBeg,
- CallExpr::const_arg_iterator ArgEnd,
+ llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
const FunctionDecl *CalleeDecl = nullptr,
unsigned ParamsToSkip = 0);
+ /// EmitPointerWithAlignment - Given an expression with a pointer
+ /// type, emit the value and compute our best estimate of the
+ /// alignment of the pointee.
+ ///
+ /// Note that this function will conservatively fall back on the type
+ /// when it doesn't
+ ///
+ /// \param Source - If non-null, this will be initialized with
+ /// information about the source of the alignment. Note that this
+ /// function will conservatively fall back on the type when it
+ /// doesn't recognize the expression, which means that sometimes
+ ///
+ /// a worst-case One
+ /// reasonable way to use this information is when there's a
+ /// language guarantee that the pointer must be aligned to some
+ /// stricter value, and we're simply trying to ensure that
+ /// sufficiently obvious uses of under-aligned objects don't get
+ /// miscompiled; for example, a placement new into the address of
+ /// a local variable. In such a case, it's quite reasonable to
+ /// just ignore the returned alignment when it isn't from an
+ /// explicit source.
+ Address EmitPointerWithAlignment(const Expr *Addr,
+ AlignmentSource *Source = nullptr);
+
private:
QualType getVarArgType(const Expr *Arg);
@@ -2943,16 +3196,11 @@
void EmitDeclMetadata();
- CodeGenModule::ByrefHelpers *
- buildByrefHelpers(llvm::StructType &byrefType,
- const AutoVarEmission &emission);
+ BlockByrefHelpers *buildByrefHelpers(llvm::StructType &byrefType,
+ const AutoVarEmission &emission);
void AddObjCARCExceptionMetadata(llvm::Instruction *Inst);
- /// GetPointeeAlignment - Given an expression with a pointer type, emit the
- /// value and compute our best estimate of the alignment of the pointee.
- std::pair<llvm::Value*, unsigned> EmitPointerWithAlignment(const Expr *Addr);
-
llvm::Value *GetValueForARMHint(unsigned BuiltinID);
};
@@ -2975,17 +3223,23 @@
static saved_type save(CodeGenFunction &CGF, llvm::Value *value) {
if (!needsSaving(value)) return saved_type(value, false);
- // Otherwise we need an alloca.
- llvm::Value *alloca =
- CGF.CreateTempAlloca(value->getType(), "cond-cleanup.save");
+ // Otherwise, we need an alloca.
+ auto align = CharUnits::fromQuantity(
+ CGF.CGM.getDataLayout().getPrefTypeAlignment(value->getType()));
+ Address alloca =
+ CGF.CreateTempAlloca(value->getType(), align, "cond-cleanup.save");
CGF.Builder.CreateStore(value, alloca);
- return saved_type(alloca, true);
+ return saved_type(alloca.getPointer(), true);
}
static llvm::Value *restore(CodeGenFunction &CGF, saved_type value) {
+ // If the value says it wasn't saved, trust that it's still dominating.
if (!value.getInt()) return value.getPointer();
- return CGF.Builder.CreateLoad(value.getPointer());
+
+ // Otherwise, it should be an alloca instruction, as set up in save().
+ auto alloca = cast<llvm::AllocaInst>(value.getPointer());
+ return CGF.Builder.CreateAlignedLoad(alloca, alloca->getAlignment());
}
};
@@ -2998,6 +3252,28 @@
}
};
+/// A specialization of DominatingValue for Address.
+template <> struct DominatingValue<Address> {
+ typedef Address type;
+
+ struct saved_type {
+ DominatingLLVMValue::saved_type SavedValue;
+ CharUnits Alignment;
+ };
+
+ static bool needsSaving(type value) {
+ return DominatingLLVMValue::needsSaving(value.getPointer());
+ }
+ static saved_type save(CodeGenFunction &CGF, type value) {
+ return { DominatingLLVMValue::save(CGF, value.getPointer()),
+ value.getAlignment() };
+ }
+ static type restore(CodeGenFunction &CGF, saved_type value) {
+ return Address(DominatingLLVMValue::restore(CGF, value.SavedValue),
+ value.Alignment);
+ }
+};
+
/// A specialization of DominatingValue for RValue.
template <> struct DominatingValue<RValue> {
typedef RValue type;
@@ -3006,15 +3282,17 @@
AggregateAddress, ComplexAddress };
llvm::Value *Value;
- Kind K;
- saved_type(llvm::Value *v, Kind k) : Value(v), K(k) {}
+ unsigned K : 3;
+ unsigned Align : 29;
+ saved_type(llvm::Value *v, Kind k, unsigned a = 0)
+ : Value(v), K(k), Align(a) {}
public:
static bool needsSaving(RValue value);
static saved_type save(CodeGenFunction &CGF, RValue value);
RValue restore(CodeGenFunction &CGF);
- // implementations in CGExprCXX.cpp
+ // implementations in CGCleanup.cpp
};
static bool needsSaving(type value) {
diff --git a/lib/CodeGen/CodeGenModule.cpp b/lib/CodeGen/CodeGenModule.cpp
index af4e6d9..173b0dc 100644
--- a/lib/CodeGen/CodeGenModule.cpp
+++ b/lib/CodeGen/CodeGenModule.cpp
@@ -12,6 +12,7 @@
//===----------------------------------------------------------------------===//
#include "CodeGenModule.h"
+#include "CGBlocks.h"
#include "CGCUDARuntime.h"
#include "CGCXXABI.h"
#include "CGCall.h"
@@ -52,6 +53,7 @@
#include "llvm/ProfileData/InstrProfReader.h"
#include "llvm/Support/ConvertUTF.h"
#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/MD5.h"
using namespace clang;
using namespace CodeGen;
@@ -64,8 +66,10 @@
case TargetCXXABI::GenericARM:
case TargetCXXABI::iOS:
case TargetCXXABI::iOS64:
+ case TargetCXXABI::WatchOS:
case TargetCXXABI::GenericMIPS:
case TargetCXXABI::GenericItanium:
+ case TargetCXXABI::WebAssembly:
return CreateItaniumCXXABI(CGM);
case TargetCXXABI::Microsoft:
return CreateMicrosoftCXXABI(CGM);
@@ -74,17 +78,19 @@
llvm_unreachable("invalid C++ ABI kind");
}
-CodeGenModule::CodeGenModule(ASTContext &C, const CodeGenOptions &CGO,
- llvm::Module &M, const llvm::DataLayout &TD,
+CodeGenModule::CodeGenModule(ASTContext &C, const HeaderSearchOptions &HSO,
+ const PreprocessorOptions &PPO,
+ const CodeGenOptions &CGO, llvm::Module &M,
DiagnosticsEngine &diags,
CoverageSourceInfo *CoverageInfo)
- : Context(C), LangOpts(C.getLangOpts()), CodeGenOpts(CGO), TheModule(M),
- Diags(diags), TheDataLayout(TD), Target(C.getTargetInfo()),
- ABI(createCXXABI(*this)), VMContext(M.getContext()), TBAA(nullptr),
- TheTargetCodeGenInfo(nullptr), Types(*this), VTables(*this),
- ObjCRuntime(nullptr), OpenCLRuntime(nullptr), OpenMPRuntime(nullptr),
- CUDARuntime(nullptr), DebugInfo(nullptr), ARCData(nullptr),
- NoObjCARCExceptionsMetadata(nullptr), RRData(nullptr), PGOReader(nullptr),
+ : Context(C), LangOpts(C.getLangOpts()), HeaderSearchOpts(HSO),
+ PreprocessorOpts(PPO), CodeGenOpts(CGO), TheModule(M), Diags(diags),
+ Target(C.getTargetInfo()), ABI(createCXXABI(*this)),
+ VMContext(M.getContext()), TBAA(nullptr), TheTargetCodeGenInfo(nullptr),
+ Types(*this), VTables(*this), ObjCRuntime(nullptr),
+ OpenCLRuntime(nullptr), OpenMPRuntime(nullptr), CUDARuntime(nullptr),
+ DebugInfo(nullptr), ObjCData(nullptr),
+ NoObjCARCExceptionsMetadata(nullptr), PGOReader(nullptr),
CFConstantStringClassRef(nullptr), ConstantStringClassRef(nullptr),
NSConstantStringType(nullptr), NSConcreteGlobalBlock(nullptr),
NSConcreteStackBlock(nullptr), BlockObjectAssign(nullptr),
@@ -103,7 +109,9 @@
DoubleTy = llvm::Type::getDoubleTy(LLVMContext);
PointerWidthInBits = C.getTargetInfo().getPointerWidth(0);
PointerAlignInBytes =
- C.toCharUnitsFromBits(C.getTargetInfo().getPointerAlign(0)).getQuantity();
+ C.toCharUnitsFromBits(C.getTargetInfo().getPointerAlign(0)).getQuantity();
+ IntAlignInBytes =
+ C.toCharUnitsFromBits(C.getTargetInfo().getIntAlign()).getQuantity();
IntTy = llvm::IntegerType::get(LLVMContext, C.getTargetInfo().getIntWidth());
IntPtrTy = llvm::IntegerType::get(LLVMContext, PointerWidthInBits);
Int8PtrTy = Int8Ty->getPointerTo(0);
@@ -136,17 +144,17 @@
Block.GlobalUniqueCount = 0;
- if (C.getLangOpts().ObjCAutoRefCount)
- ARCData = new ARCEntrypoints();
- RRData = new RREntrypoints();
+ if (C.getLangOpts().ObjC1)
+ ObjCData = new ObjCEntrypoints();
if (!CodeGenOpts.InstrProfileInput.empty()) {
auto ReaderOrErr =
llvm::IndexedInstrProfReader::create(CodeGenOpts.InstrProfileInput);
if (std::error_code EC = ReaderOrErr.getError()) {
unsigned DiagID = Diags.getCustomDiagID(DiagnosticsEngine::Error,
- "Could not read profile: %0");
- getDiags().Report(DiagID) << EC.message();
+ "Could not read profile %0: %1");
+ getDiags().Report(DiagID) << CodeGenOpts.InstrProfileInput
+ << EC.message();
} else
PGOReader = std::move(ReaderOrErr.get());
}
@@ -165,8 +173,7 @@
delete TheTargetCodeGenInfo;
delete TBAA;
delete DebugInfo;
- delete ARCData;
- delete RRData;
+ delete ObjCData;
}
void CodeGenModule::createObjCRuntime() {
@@ -182,6 +189,7 @@
case ObjCRuntime::FragileMacOSX:
case ObjCRuntime::MacOSX:
case ObjCRuntime::iOS:
+ case ObjCRuntime::WatchOS:
ObjCRuntime = CreateMacObjCRuntime(*this);
return;
}
@@ -228,12 +236,27 @@
OldF->replaceAllUsesWith(Replacement);
if (NewF) {
NewF->removeFromParent();
- OldF->getParent()->getFunctionList().insertAfter(OldF, NewF);
+ OldF->getParent()->getFunctionList().insertAfter(OldF->getIterator(),
+ NewF);
}
OldF->eraseFromParent();
}
}
+void CodeGenModule::addGlobalValReplacement(llvm::GlobalValue *GV, llvm::Constant *C) {
+ GlobalValReplacements.push_back(std::make_pair(GV, C));
+}
+
+void CodeGenModule::applyGlobalValReplacements() {
+ for (auto &I : GlobalValReplacements) {
+ llvm::GlobalValue *GV = I.first;
+ llvm::Constant *C = I.second;
+
+ GV->replaceAllUsesWith(C);
+ GV->eraseFromParent();
+ }
+}
+
// This is only used in aliases that we created and we know they have a
// linear structure.
static const llvm::GlobalObject *getAliasedGlobal(const llvm::GlobalAlias &GA) {
@@ -336,6 +359,7 @@
void CodeGenModule::Release() {
EmitDeferred();
+ applyGlobalValReplacements();
applyReplacements();
checkAliases();
EmitCXXGlobalInitFunc();
@@ -351,8 +375,11 @@
if (llvm::Function *CudaDtorFunction = CUDARuntime->makeModuleDtorFunction())
AddGlobalDtor(CudaDtorFunction);
}
- if (PGOReader && PGOStats.hasDiagnostics())
- PGOStats.reportDiagnostics(getDiags(), getCodeGenOpts().MainFileName);
+ if (PGOReader) {
+ getModule().setMaximumFunctionCount(PGOReader->getMaximumFunctionCount());
+ if (PGOStats.hasDiagnostics())
+ PGOStats.reportDiagnostics(getDiags(), getCodeGenOpts().MainFileName);
+ }
EmitCtorList(GlobalCtors, "llvm.global_ctors");
EmitCtorList(GlobalDtors, "llvm.global_dtors");
EmitGlobalAnnotations();
@@ -366,11 +393,32 @@
(Context.getLangOpts().Modules || !LinkerOptionsMetadata.empty())) {
EmitModuleLinkOptions();
}
- if (CodeGenOpts.DwarfVersion)
+ if (CodeGenOpts.DwarfVersion) {
// We actually want the latest version when there are conflicts.
// We can change from Warning to Latest if such mode is supported.
getModule().addModuleFlag(llvm::Module::Warning, "Dwarf Version",
CodeGenOpts.DwarfVersion);
+ }
+ if (CodeGenOpts.EmitCodeView) {
+ // Indicate that we want CodeView in the metadata.
+ getModule().addModuleFlag(llvm::Module::Warning, "CodeView", 1);
+ }
+ if (CodeGenOpts.OptimizationLevel > 0 && CodeGenOpts.StrictVTablePointers) {
+ // We don't support LTO with 2 with different StrictVTablePointers
+ // FIXME: we could support it by stripping all the information introduced
+ // by StrictVTablePointers.
+
+ getModule().addModuleFlag(llvm::Module::Error, "StrictVTablePointers",1);
+
+ llvm::Metadata *Ops[2] = {
+ llvm::MDString::get(VMContext, "StrictVTablePointers"),
+ llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
+ llvm::Type::getInt32Ty(VMContext), 1))};
+
+ getModule().addModuleFlag(llvm::Module::Require,
+ "StrictVTablePointersRequirement",
+ llvm::MDNode::get(VMContext, Ops));
+ }
if (DebugInfo)
// We support a single version in the linked module. The LLVM
// parser will drop debug info with a different version number
@@ -395,6 +443,11 @@
getModule().addModuleFlag(llvm::Module::Error, "min_enum_size", EnumWidth);
}
+ if (CodeGenOpts.SanitizeCfiCrossDso) {
+ // Indicate that we want cross-DSO control flow integrity checks.
+ getModule().addModuleFlag(llvm::Module::Override, "Cross-DSO CFI", 1);
+ }
+
if (uint32_t PLevel = Context.getLangOpts().PICLevel) {
llvm::PICLevel::Level PL = llvm::PICLevel::Default;
switch (PLevel) {
@@ -446,12 +499,6 @@
return TBAA->getTBAAStructInfo(QTy);
}
-llvm::MDNode *CodeGenModule::getTBAAStructTypeInfo(QualType QTy) {
- if (!TBAA)
- return nullptr;
- return TBAA->getTBAAStructTypeInfo(QTy);
-}
-
llvm::MDNode *CodeGenModule::getTBAAStructTagInfo(QualType BaseTy,
llvm::MDNode *AccessN,
uint64_t O) {
@@ -464,9 +511,9 @@
/// and struct-path aware TBAA, the tag has the same format:
/// base type, access type and offset.
/// When ConvertTypeToTag is true, we create a tag based on the scalar type.
-void CodeGenModule::DecorateInstruction(llvm::Instruction *Inst,
- llvm::MDNode *TBAAInfo,
- bool ConvertTypeToTag) {
+void CodeGenModule::DecorateInstructionWithTBAA(llvm::Instruction *Inst,
+ llvm::MDNode *TBAAInfo,
+ bool ConvertTypeToTag) {
if (ConvertTypeToTag && TBAA)
Inst->setMetadata(llvm::LLVMContext::MD_tbaa,
TBAA->getTBAAScalarTagInfo(TBAAInfo));
@@ -474,6 +521,16 @@
Inst->setMetadata(llvm::LLVMContext::MD_tbaa, TBAAInfo);
}
+void CodeGenModule::DecorateInstructionWithInvariantGroup(
+ llvm::Instruction *I, const CXXRecordDecl *RD) {
+ llvm::Metadata *MD = CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
+ auto *MetaDataNode = dyn_cast<llvm::MDNode>(MD);
+ // Check if we have to wrap MDString in MDNode.
+ if (!MetaDataNode)
+ MetaDataNode = llvm::MDNode::get(getLLVMContext(), MD);
+ I->setMetadata(llvm::LLVMContext::MD_invariant_group, MetaDataNode);
+}
+
void CodeGenModule::Error(SourceLocation loc, StringRef message) {
unsigned diagID = getDiags().getCustomDiagID(DiagnosticsEngine::Error, "%0");
getDiags().Report(Context.getFullLoc(loc), diagID) << message;
@@ -688,6 +745,21 @@
F->setDLLStorageClass(llvm::GlobalVariable::DefaultStorageClass);
}
+llvm::ConstantInt *
+CodeGenModule::CreateCfiIdForTypeMetadata(llvm::Metadata *MD) {
+ llvm::MDString *MDS = dyn_cast<llvm::MDString>(MD);
+ if (!MDS) return nullptr;
+
+ llvm::MD5 md5;
+ llvm::MD5::MD5Result result;
+ md5.update(MDS->getString());
+ md5.final(result);
+ uint64_t id = 0;
+ for (int i = 0; i < 8; ++i)
+ id |= static_cast<uint64_t>(result[i]) << (i * 8);
+ return llvm::ConstantInt::get(Int64Ty, id);
+}
+
void CodeGenModule::setFunctionDefinitionAttributes(const FunctionDecl *D,
llvm::Function *F) {
setNonAliasAttributes(D, F);
@@ -733,6 +805,21 @@
if (!hasUnwindExceptions(LangOpts))
B.addAttribute(llvm::Attribute::NoUnwind);
+ if (LangOpts.getStackProtector() == LangOptions::SSPOn)
+ B.addAttribute(llvm::Attribute::StackProtect);
+ else if (LangOpts.getStackProtector() == LangOptions::SSPStrong)
+ B.addAttribute(llvm::Attribute::StackProtectStrong);
+ else if (LangOpts.getStackProtector() == LangOptions::SSPReq)
+ B.addAttribute(llvm::Attribute::StackProtectReq);
+
+ if (!D) {
+ F->addAttributes(llvm::AttributeSet::FunctionIndex,
+ llvm::AttributeSet::get(
+ F->getContext(),
+ llvm::AttributeSet::FunctionIndex, B));
+ return;
+ }
+
if (D->hasAttr<NakedAttr>()) {
// Naked implies noinline: we should not be inlining such functions.
B.addAttribute(llvm::Attribute::Naked);
@@ -757,13 +844,6 @@
if (D->hasAttr<MinSizeAttr>())
B.addAttribute(llvm::Attribute::MinSize);
- if (LangOpts.getStackProtector() == LangOptions::SSPOn)
- B.addAttribute(llvm::Attribute::StackProtect);
- else if (LangOpts.getStackProtector() == LangOptions::SSPStrong)
- B.addAttribute(llvm::Attribute::StackProtectStrong);
- else if (LangOpts.getStackProtector() == LangOptions::SSPReq)
- B.addAttribute(llvm::Attribute::StackProtectReq);
-
F->addAttributes(llvm::AttributeSet::FunctionIndex,
llvm::AttributeSet::get(
F->getContext(), llvm::AttributeSet::FunctionIndex, B));
@@ -774,10 +854,8 @@
F->addFnAttr(llvm::Attribute::NoInline);
// OptimizeNone wins over OptimizeForSize, MinSize, AlwaysInline.
- assert(!F->hasFnAttribute(llvm::Attribute::OptimizeForSize) &&
- "OptimizeNone and OptimizeForSize on same function!");
- assert(!F->hasFnAttribute(llvm::Attribute::MinSize) &&
- "OptimizeNone and MinSize on same function!");
+ F->removeFnAttr(llvm::Attribute::OptimizeForSize);
+ F->removeFnAttr(llvm::Attribute::MinSize);
assert(!F->hasFnAttribute(llvm::Attribute::AlwaysInline) &&
"OptimizeNone and AlwaysInline on same function!");
@@ -796,19 +874,24 @@
if (alignment)
F->setAlignment(alignment);
- // C++ ABI requires 2-byte alignment for member functions.
- if (F->getAlignment() < 2 && isa<CXXMethodDecl>(D))
- F->setAlignment(2);
+ // Some C++ ABIs require 2-byte alignment for member functions, in order to
+ // reserve a bit for differentiating between virtual and non-virtual member
+ // functions. If the current target's C++ ABI requires this and this is a
+ // member function, set its alignment accordingly.
+ if (getTarget().getCXXABI().areMemberFunctionsAligned()) {
+ if (F->getAlignment() < 2 && isa<CXXMethodDecl>(D))
+ F->setAlignment(2);
+ }
}
void CodeGenModule::SetCommonAttributes(const Decl *D,
llvm::GlobalValue *GV) {
- if (const auto *ND = dyn_cast<NamedDecl>(D))
+ if (const auto *ND = dyn_cast_or_null<NamedDecl>(D))
setGlobalVisibility(GV, ND);
else
GV->setVisibility(llvm::GlobalValue::DefaultVisibility);
- if (D->hasAttr<UsedAttr>())
+ if (D && D->hasAttr<UsedAttr>())
addUsedGlobal(GV);
}
@@ -826,8 +909,9 @@
llvm::GlobalObject *GO) {
SetCommonAttributes(D, GO);
- if (const SectionAttr *SA = D->getAttr<SectionAttr>())
- GO->setSection(SA->getName());
+ if (D)
+ if (const SectionAttr *SA = D->getAttr<SectionAttr>())
+ GO->setSection(SA->getName());
getTargetCodeGenInfo().setTargetAttributes(D, GO, *this);
}
@@ -868,6 +952,49 @@
}
}
+void CodeGenModule::CreateFunctionBitSetEntry(const FunctionDecl *FD,
+ llvm::Function *F) {
+ // Only if we are checking indirect calls.
+ if (!LangOpts.Sanitize.has(SanitizerKind::CFIICall))
+ return;
+
+ // Non-static class methods are handled via vtable pointer checks elsewhere.
+ if (isa<CXXMethodDecl>(FD) && !cast<CXXMethodDecl>(FD)->isStatic())
+ return;
+
+ // Additionally, if building with cross-DSO support...
+ if (CodeGenOpts.SanitizeCfiCrossDso) {
+ // Don't emit entries for function declarations. In cross-DSO mode these are
+ // handled with better precision at run time.
+ if (!FD->hasBody())
+ return;
+ // Skip available_externally functions. They won't be codegen'ed in the
+ // current module anyway.
+ if (getContext().GetGVALinkageForFunction(FD) == GVA_AvailableExternally)
+ return;
+ }
+
+ llvm::NamedMDNode *BitsetsMD =
+ getModule().getOrInsertNamedMetadata("llvm.bitsets");
+
+ llvm::Metadata *MD = CreateMetadataIdentifierForType(FD->getType());
+ llvm::Metadata *BitsetOps[] = {
+ MD, llvm::ConstantAsMetadata::get(F),
+ llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(Int64Ty, 0))};
+ BitsetsMD->addOperand(llvm::MDTuple::get(getLLVMContext(), BitsetOps));
+
+ // Emit a hash-based bit set entry for cross-DSO calls.
+ if (CodeGenOpts.SanitizeCfiCrossDso) {
+ if (auto TypeId = CreateCfiIdForTypeMetadata(MD)) {
+ llvm::Metadata *BitsetOps2[] = {
+ llvm::ConstantAsMetadata::get(TypeId),
+ llvm::ConstantAsMetadata::get(F),
+ llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(Int64Ty, 0))};
+ BitsetsMD->addOperand(llvm::MDTuple::get(getLLVMContext(), BitsetOps2));
+ }
+ }
+}
+
void CodeGenModule::SetFunctionAttributes(GlobalDecl GD, llvm::Function *F,
bool IsIncompleteFunction,
bool IsThunk) {
@@ -909,6 +1036,8 @@
if (FD->isReplaceableGlobalAllocationFunction())
F->addAttribute(llvm::AttributeSet::FunctionIndex,
llvm::Attribute::NoBuiltin);
+
+ CreateFunctionBitSetEntry(FD, F);
}
void CodeGenModule::addUsedGlobal(llvm::GlobalValue *GV) {
@@ -1100,9 +1229,16 @@
llvm::GlobalValue *GV = G.GV;
G.GV = nullptr;
- assert(!GV || GV == GetGlobalValue(getMangledName(D)));
- if (!GV)
- GV = GetGlobalValue(getMangledName(D));
+ // We should call GetAddrOfGlobal with IsForDefinition set to true in order
+ // to get GlobalValue with exactly the type we need, not something that
+ // might had been created for another decl with the same mangled name but
+ // different type.
+ // FIXME: Support for variables is not implemented yet.
+ if (isa<FunctionDecl>(D.getDecl()))
+ GV = cast<llvm::GlobalValue>(GetAddrOfGlobal(D, /*IsForDefinition=*/true));
+ else
+ if (!GV)
+ GV = GetGlobalValue(getMangledName(D));
// Check to see if we've already emitted this. This is necessary
// for a couple of reasons: first, decls can end up in the
@@ -1204,7 +1340,7 @@
if (SanitizerBL.isBlacklistedFunction(Fn->getName()))
return true;
// Blacklist by location.
- if (!Loc.isInvalid())
+ if (Loc.isValid())
return SanitizerBL.isBlacklistedLocation(Loc);
// If location is unknown, this may be a compiler-generated function. Assume
// it's located in the main file.
@@ -1218,8 +1354,9 @@
bool CodeGenModule::isInSanitizerBlacklist(llvm::GlobalVariable *GV,
SourceLocation Loc, QualType Ty,
StringRef Category) const {
- // For now globals can be blacklisted only in ASan.
- if (!LangOpts.Sanitize.has(SanitizerKind::Address))
+ // For now globals can be blacklisted only in ASan and KASan.
+ if (!LangOpts.Sanitize.hasOneOf(
+ SanitizerKind::Address | SanitizerKind::KernelAddress))
return false;
const auto &SanitizerBL = getContext().getSanitizerBlacklist();
if (SanitizerBL.isBlacklistedGlobal(GV->getName(), Category))
@@ -1257,11 +1394,16 @@
// Implicit template instantiations may change linkage if they are later
// explicitly instantiated, so they should not be emitted eagerly.
return false;
+ // If OpenMP is enabled and threadprivates must be generated like TLS, delay
+ // codegen for global variables, because they may be marked as threadprivate.
+ if (LangOpts.OpenMP && LangOpts.OpenMPUseTLS &&
+ getContext().getTargetInfo().isTLSSupported() && isa<VarDecl>(Global))
+ return false;
return true;
}
-llvm::Constant *CodeGenModule::GetAddrOfUuidDescriptor(
+ConstantAddress CodeGenModule::GetAddrOfUuidDescriptor(
const CXXUuidofExpr* E) {
// Sema has verified that IIDSource has a __declspec(uuid()), and that its
// well-formed.
@@ -1269,9 +1411,12 @@
std::string Name = "_GUID_" + Uuid.lower();
std::replace(Name.begin(), Name.end(), '-', '_');
+ // Contains a 32-bit field.
+ CharUnits Alignment = CharUnits::fromQuantity(4);
+
// Look for an existing global.
if (llvm::GlobalVariable *GV = getModule().getNamedGlobal(Name))
- return GV;
+ return ConstantAddress(GV, Alignment);
llvm::Constant *Init = EmitUuidofInitializer(Uuid);
assert(Init && "failed to initialize as constant");
@@ -1281,20 +1426,22 @@
/*isConstant=*/true, llvm::GlobalValue::LinkOnceODRLinkage, Init, Name);
if (supportsCOMDAT())
GV->setComdat(TheModule.getOrInsertComdat(GV->getName()));
- return GV;
+ return ConstantAddress(GV, Alignment);
}
-llvm::Constant *CodeGenModule::GetWeakRefReference(const ValueDecl *VD) {
+ConstantAddress CodeGenModule::GetWeakRefReference(const ValueDecl *VD) {
const AliasAttr *AA = VD->getAttr<AliasAttr>();
assert(AA && "No alias?");
+ CharUnits Alignment = getContext().getDeclAlign(VD);
llvm::Type *DeclTy = getTypes().ConvertTypeForMem(VD->getType());
// See if there is already something with the target's name in the module.
llvm::GlobalValue *Entry = GetGlobalValue(AA->getAliasee());
if (Entry) {
unsigned AS = getContext().getTargetAddressSpace(VD->getType());
- return llvm::ConstantExpr::getBitCast(Entry, DeclTy->getPointerTo(AS));
+ auto Ptr = llvm::ConstantExpr::getBitCast(Entry, DeclTy->getPointerTo(AS));
+ return ConstantAddress(Ptr, Alignment);
}
llvm::Constant *Aliasee;
@@ -1311,7 +1458,7 @@
F->setLinkage(llvm::Function::ExternalWeakLinkage);
WeakRefReferences.insert(F);
- return Aliasee;
+ return ConstantAddress(Aliasee, Alignment);
}
void CodeGenModule::EmitGlobal(GlobalDecl GD) {
@@ -1423,9 +1570,9 @@
return false;
}
unsigned BuiltinID = FD->getBuiltinID();
- if (!BuiltinID)
+ if (!BuiltinID || !BI.isLibFunction(BuiltinID))
return true;
- StringRef BuiltinName = BI.GetName(BuiltinID);
+ StringRef BuiltinName = BI.getName(BuiltinID);
if (BuiltinName.startswith("__builtin_") &&
Name == BuiltinName.slice(strlen("__builtin_"), StringRef::npos)) {
Result = true;
@@ -1434,6 +1581,35 @@
return true;
}
};
+
+ struct DLLImportFunctionVisitor
+ : public RecursiveASTVisitor<DLLImportFunctionVisitor> {
+ bool SafeToInline = true;
+
+ bool VisitVarDecl(VarDecl *VD) {
+ // A thread-local variable cannot be imported.
+ SafeToInline = !VD->getTLSKind();
+ return SafeToInline;
+ }
+
+ // Make sure we're not referencing non-imported vars or functions.
+ bool VisitDeclRefExpr(DeclRefExpr *E) {
+ ValueDecl *VD = E->getDecl();
+ if (isa<FunctionDecl>(VD))
+ SafeToInline = VD->hasAttr<DLLImportAttr>();
+ else if (VarDecl *V = dyn_cast<VarDecl>(VD))
+ SafeToInline = !V->hasGlobalStorage() || V->hasAttr<DLLImportAttr>();
+ return SafeToInline;
+ }
+ bool VisitCXXDeleteExpr(CXXDeleteExpr *E) {
+ SafeToInline = E->getOperatorDelete()->hasAttr<DLLImportAttr>();
+ return SafeToInline;
+ }
+ bool VisitCXXNewExpr(CXXNewExpr *E) {
+ SafeToInline = E->getOperatorNew()->hasAttr<DLLImportAttr>();
+ return SafeToInline;
+ }
+ };
}
// isTriviallyRecursive - Check if this function calls another
@@ -1464,6 +1640,15 @@
const auto *F = cast<FunctionDecl>(GD.getDecl());
if (CodeGenOpts.OptimizationLevel == 0 && !F->hasAttr<AlwaysInlineAttr>())
return false;
+
+ if (F->hasAttr<DLLImportAttr>()) {
+ // Check whether it would be safe to inline this dllimport function.
+ DLLImportFunctionVisitor Visitor;
+ Visitor.TraverseFunctionDecl(const_cast<FunctionDecl*>(F));
+ if (!Visitor.SafeToInline)
+ return false;
+ }
+
// PR9614. Avoid cases where the source code is lying to us. An available
// externally function should have an equivalent function somewhere else,
// but a function that calls itself is clearly not equivalent to the real
@@ -1527,6 +1712,9 @@
llvm_unreachable("Invalid argument to EmitGlobalDefinition()");
}
+static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old,
+ llvm::Function *NewFn);
+
/// GetOrCreateLLVMFunction - If the specified mangled name is not in the
/// module, create and return an llvm Function with the specified type. If there
/// is something in the module with the specified name, return it potentially
@@ -1539,7 +1727,8 @@
llvm::Type *Ty,
GlobalDecl GD, bool ForVTable,
bool DontDefer, bool IsThunk,
- llvm::AttributeSet ExtraAttrs) {
+ llvm::AttributeSet ExtraAttrs,
+ bool IsForDefinition) {
const Decl *D = GD.getDecl();
// Lookup the entry, lazily creating it if necessary.
@@ -1555,11 +1744,33 @@
if (D && !D->hasAttr<DLLImportAttr>() && !D->hasAttr<DLLExportAttr>())
Entry->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
- if (Entry->getType()->getElementType() == Ty)
+ // If there are two attempts to define the same mangled name, issue an
+ // error.
+ if (IsForDefinition && !Entry->isDeclaration()) {
+ GlobalDecl OtherGD;
+ // Check that GD is not yet in ExplicitDefinitions is required to make
+ // sure that we issue an error only once.
+ if (lookupRepresentativeDecl(MangledName, OtherGD) &&
+ (GD.getCanonicalDecl().getDecl() !=
+ OtherGD.getCanonicalDecl().getDecl()) &&
+ DiagnosedConflictingDefinitions.insert(GD).second) {
+ getDiags().Report(D->getLocation(),
+ diag::err_duplicate_mangled_name);
+ getDiags().Report(OtherGD.getDecl()->getLocation(),
+ diag::note_previous_definition);
+ }
+ }
+
+ if ((isa<llvm::Function>(Entry) || isa<llvm::GlobalAlias>(Entry)) &&
+ (Entry->getType()->getElementType() == Ty)) {
return Entry;
+ }
// Make sure the result is of the correct type.
- return llvm::ConstantExpr::getBitCast(Entry, Ty->getPointerTo());
+ // (If function is requested for a definition, we always need to create a new
+ // function, not just return a bitcast.)
+ if (!IsForDefinition)
+ return llvm::ConstantExpr::getBitCast(Entry, Ty->getPointerTo());
}
// This function doesn't have a complete type (for example, the return
@@ -1574,10 +1785,36 @@
FTy = llvm::FunctionType::get(VoidTy, false);
IsIncompleteFunction = true;
}
-
- llvm::Function *F = llvm::Function::Create(FTy,
- llvm::Function::ExternalLinkage,
- MangledName, &getModule());
+
+ llvm::Function *F =
+ llvm::Function::Create(FTy, llvm::Function::ExternalLinkage,
+ Entry ? StringRef() : MangledName, &getModule());
+
+ // If we already created a function with the same mangled name (but different
+ // type) before, take its name and add it to the list of functions to be
+ // replaced with F at the end of CodeGen.
+ //
+ // This happens if there is a prototype for a function (e.g. "int f()") and
+ // then a definition of a different type (e.g. "int f(int x)").
+ if (Entry) {
+ F->takeName(Entry);
+
+ // This might be an implementation of a function without a prototype, in
+ // which case, try to do special replacement of calls which match the new
+ // prototype. The really key thing here is that we also potentially drop
+ // arguments from the call site so as to make a direct call, which makes the
+ // inliner happier and suppresses a number of optimizer warnings (!) about
+ // dropping arguments.
+ if (!Entry->use_empty()) {
+ ReplaceUsesOfNonProtoTypeWithRealFunction(Entry, F);
+ Entry->removeDeadConstantUsers();
+ }
+
+ llvm::Constant *BC = llvm::ConstantExpr::getBitCast(
+ F, Entry->getType()->getElementType()->getPointerTo());
+ addGlobalValReplacement(Entry, BC);
+ }
+
assert(F->getName() == MangledName && "name was uniqued!");
if (D)
SetFunctionAttributes(GD, F, IsIncompleteFunction, IsThunk);
@@ -1650,13 +1887,19 @@
llvm::Constant *CodeGenModule::GetAddrOfFunction(GlobalDecl GD,
llvm::Type *Ty,
bool ForVTable,
- bool DontDefer) {
+ bool DontDefer,
+ bool IsForDefinition) {
// If there was no specific requested type, just convert it now.
- if (!Ty)
- Ty = getTypes().ConvertType(cast<ValueDecl>(GD.getDecl())->getType());
-
+ if (!Ty) {
+ const auto *FD = cast<FunctionDecl>(GD.getDecl());
+ auto CanonTy = Context.getCanonicalType(FD->getType());
+ Ty = getTypes().ConvertFunctionType(CanonTy, FD);
+ }
+
StringRef MangledName = getMangledName(GD);
- return GetOrCreateLLVMFunction(MangledName, Ty, GD, ForVTable, DontDefer);
+ return GetOrCreateLLVMFunction(MangledName, Ty, GD, ForVTable, DontDefer,
+ /*IsThunk=*/false, llvm::AttributeSet(),
+ IsForDefinition);
}
/// CreateRuntimeFunction - Create a new runtime function with the specified
@@ -1771,7 +2014,7 @@
if (D->getTLSKind()) {
if (D->getTLSKind() == VarDecl::TLS_Dynamic)
- CXXThreadLocals.push_back(std::make_pair(D, GV));
+ CXXThreadLocals.push_back(D);
setTLSMode(GV, *D);
}
@@ -1795,6 +2038,33 @@
return GV;
}
+llvm::Constant *
+CodeGenModule::GetAddrOfGlobal(GlobalDecl GD,
+ bool IsForDefinition) {
+ if (isa<CXXConstructorDecl>(GD.getDecl()))
+ return getAddrOfCXXStructor(cast<CXXConstructorDecl>(GD.getDecl()),
+ getFromCtorType(GD.getCtorType()),
+ /*FnInfo=*/nullptr, /*FnType=*/nullptr,
+ /*DontDefer=*/false, IsForDefinition);
+ else if (isa<CXXDestructorDecl>(GD.getDecl()))
+ return getAddrOfCXXStructor(cast<CXXDestructorDecl>(GD.getDecl()),
+ getFromDtorType(GD.getDtorType()),
+ /*FnInfo=*/nullptr, /*FnType=*/nullptr,
+ /*DontDefer=*/false, IsForDefinition);
+ else if (isa<CXXMethodDecl>(GD.getDecl())) {
+ auto FInfo = &getTypes().arrangeCXXMethodDeclaration(
+ cast<CXXMethodDecl>(GD.getDecl()));
+ auto Ty = getTypes().GetFunctionType(*FInfo);
+ return GetAddrOfFunction(GD, Ty, /*ForVTable=*/false, /*DontDefer=*/false,
+ IsForDefinition);
+ } else if (isa<FunctionDecl>(GD.getDecl())) {
+ const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
+ llvm::FunctionType *Ty = getTypes().GetFunctionType(FI);
+ return GetAddrOfFunction(GD, Ty, /*ForVTable=*/false, /*DontDefer=*/false,
+ IsForDefinition);
+ } else
+ return GetAddrOfGlobalVar(cast<VarDecl>(GD.getDecl()));
+}
llvm::GlobalVariable *
CodeGenModule::CreateOrReplaceCXXRuntimeVariable(StringRef Name,
@@ -1883,8 +2153,8 @@
}
CharUnits CodeGenModule::GetTargetTypeStoreSize(llvm::Type *Ty) const {
- return Context.toCharUnitsFromBits(
- TheDataLayout.getTypeStoreSizeInBits(Ty));
+ return Context.toCharUnitsFromBits(
+ getDataLayout().getTypeStoreSizeInBits(Ty));
}
unsigned CodeGenModule::GetGlobalVarAddressSpace(const VarDecl *D,
@@ -1976,7 +2246,18 @@
const VarDecl *InitDecl;
const Expr *InitExpr = D->getAnyInitializer(InitDecl);
- if (!InitExpr) {
+ // CUDA E.2.4.1 "__shared__ variables cannot have an initialization as part
+ // of their declaration."
+ if (getLangOpts().CPlusPlus && getLangOpts().CUDAIsDevice
+ && D->hasAttr<CUDASharedAttr>()) {
+ if (InitExpr) {
+ const auto *C = dyn_cast<CXXConstructExpr>(InitExpr);
+ if (C == nullptr || !C->getConstructor()->hasTrivialBody())
+ Error(D->getLocation(),
+ "__shared__ variable cannot have an initialization.");
+ }
+ Init = llvm::UndefValue::get(getTypes().ConvertType(ASTTy));
+ } else if (!InitExpr) {
// This is a tentative definition; tentative definitions are
// implicitly initialized with { 0 }.
//
@@ -2062,6 +2343,17 @@
if (D->hasAttr<AnnotateAttr>())
AddGlobalAnnotations(D, GV);
+ // CUDA B.2.1 "The __device__ qualifier declares a variable that resides on
+ // the device. [...]"
+ // CUDA B.2.2 "The __constant__ qualifier, optionally used together with
+ // __device__, declares a variable that: [...]
+ // Is accessible from all the threads within the grid and from the host
+ // through the runtime library (cudaGetSymbolAddress() / cudaGetSymbolSize()
+ // / cudaMemcpyToSymbol() / cudaMemcpyFromSymbol())."
+ if (GV && LangOpts.CUDA && LangOpts.CUDAIsDevice &&
+ (D->hasAttr<CUDAConstantAttr>() || D->hasAttr<CUDADeviceAttr>())) {
+ GV->setExternallyInitialized(true);
+ }
GV->setInitializer(Init);
// If it is safe to mark the global 'constant', do so now.
@@ -2081,12 +2373,17 @@
llvm::GlobalValue::LinkageTypes Linkage =
getLLVMLinkageVarDefinition(D, GV->isConstant());
- // On Darwin, the backing variable for a C++11 thread_local variable always
- // has internal linkage; all accesses should just be calls to the
+ // On Darwin, if the normal linkage of a C++ thread_local variable is
+ // LinkOnce or Weak, we keep the normal linkage to prevent multiple
+ // copies within a linkage unit; otherwise, the backing variable has
+ // internal linkage and all accesses should just be calls to the
// Itanium-specified entry point, which has the normal linkage of the
- // variable.
+ // variable. This is to preserve the ability to change the implementation
+ // behind the scenes.
if (!D->isStaticLocal() && D->getTLSKind() == VarDecl::TLS_Dynamic &&
- Context.getTargetInfo().getTriple().isMacOSX())
+ Context.getTargetInfo().getTriple().isOSDarwin() &&
+ !llvm::GlobalVariable::isLinkOnceLinkage(Linkage) &&
+ !llvm::GlobalVariable::isWeakLinkage(Linkage))
Linkage = llvm::GlobalValue::InternalLinkage;
GV->setLinkage(Linkage);
@@ -2105,7 +2402,7 @@
if (D->getTLSKind() && !GV->isThreadLocal()) {
if (D->getTLSKind() == VarDecl::TLS_Dynamic)
- CXXThreadLocals.push_back(std::make_pair(D, GV));
+ CXXThreadLocals.push_back(D);
setTLSMode(GV, *D);
}
@@ -2156,7 +2453,7 @@
// Declarations with a required alignment do not have common linakge in MSVC
// mode.
- if (Context.getLangOpts().MSVCCompat) {
+ if (Context.getTargetInfo().getCXXABI().isMicrosoft()) {
if (D->hasAttr<AlignedAttr>())
return true;
QualType VarType = D->getType();
@@ -2253,6 +2550,7 @@
llvm::Type *newRetTy = newFn->getReturnType();
SmallVector<llvm::Value*, 4> newArgs;
+ SmallVector<llvm::OperandBundleDef, 1> newBundles;
for (llvm::Value::use_iterator ui = old->use_begin(), ue = old->use_end();
ui != ue; ) {
@@ -2320,16 +2618,19 @@
// over the required information.
newArgs.append(callSite.arg_begin(), callSite.arg_begin() + argNo);
+ // Copy over any operand bundles.
+ callSite.getOperandBundlesAsDefs(newBundles);
+
llvm::CallSite newCall;
if (callSite.isCall()) {
- newCall = llvm::CallInst::Create(newFn, newArgs, "",
+ newCall = llvm::CallInst::Create(newFn, newArgs, newBundles, "",
callSite.getInstruction());
} else {
auto *oldInvoke = cast<llvm::InvokeInst>(callSite.getInstruction());
newCall = llvm::InvokeInst::Create(newFn,
oldInvoke->getNormalDest(),
oldInvoke->getUnwindDest(),
- newArgs, "",
+ newArgs, newBundles, "",
callSite.getInstruction());
}
newArgs.clear(); // for the next iteration
@@ -2347,6 +2648,7 @@
// Copy debug location attached to CI.
if (callSite->getDebugLoc())
newCall->setDebugLoc(callSite->getDebugLoc());
+
callSite->eraseFromParent();
}
}
@@ -2387,66 +2689,14 @@
llvm::FunctionType *Ty = getTypes().GetFunctionType(FI);
// Get or create the prototype for the function.
- if (!GV) {
- llvm::Constant *C =
- GetAddrOfFunction(GD, Ty, /*ForVTable=*/false, /*DontDefer*/ true);
+ if (!GV || (GV->getType()->getElementType() != Ty))
+ GV = cast<llvm::GlobalValue>(GetAddrOfFunction(GD, Ty, /*ForVTable=*/false,
+ /*DontDefer=*/true,
+ /*IsForDefinition=*/true));
- // Strip off a bitcast if we got one back.
- if (auto *CE = dyn_cast<llvm::ConstantExpr>(C)) {
- assert(CE->getOpcode() == llvm::Instruction::BitCast);
- GV = cast<llvm::GlobalValue>(CE->getOperand(0));
- } else {
- GV = cast<llvm::GlobalValue>(C);
- }
- }
-
- if (!GV->isDeclaration()) {
- getDiags().Report(D->getLocation(), diag::err_duplicate_mangled_name);
- GlobalDecl OldGD = Manglings.lookup(GV->getName());
- if (auto *Prev = OldGD.getDecl())
- getDiags().Report(Prev->getLocation(), diag::note_previous_definition);
+ // Already emitted.
+ if (!GV->isDeclaration())
return;
- }
-
- if (GV->getType()->getElementType() != Ty) {
- // If the types mismatch then we have to rewrite the definition.
- assert(GV->isDeclaration() && "Shouldn't replace non-declaration");
-
- // F is the Function* for the one with the wrong type, we must make a new
- // Function* and update everything that used F (a declaration) with the new
- // Function* (which will be a definition).
- //
- // This happens if there is a prototype for a function
- // (e.g. "int f()") and then a definition of a different type
- // (e.g. "int f(int x)"). Move the old function aside so that it
- // doesn't interfere with GetAddrOfFunction.
- GV->setName(StringRef());
- auto *NewFn = cast<llvm::Function>(GetAddrOfFunction(GD, Ty));
-
- // This might be an implementation of a function without a
- // prototype, in which case, try to do special replacement of
- // calls which match the new prototype. The really key thing here
- // is that we also potentially drop arguments from the call site
- // so as to make a direct call, which makes the inliner happier
- // and suppresses a number of optimizer warnings (!) about
- // dropping arguments.
- if (!GV->use_empty()) {
- ReplaceUsesOfNonProtoTypeWithRealFunction(GV, NewFn);
- GV->removeDeadConstantUsers();
- }
-
- // Replace uses of F with the Function we will endow with a body.
- if (!GV->use_empty()) {
- llvm::Constant *NewPtrForOldDecl =
- llvm::ConstantExpr::getBitCast(NewFn, GV->getType());
- GV->replaceAllUsesWith(NewPtrForOldDecl);
- }
-
- // Ok, delete the old function now, which is dead.
- GV->eraseFromParent();
-
- GV = NewFn;
- }
// We need to set linkage and visibility on the function before
// generating code for it because various parts of IR generation
@@ -2483,6 +2733,11 @@
StringRef MangledName = getMangledName(GD);
+ if (AA->getAliasee() == MangledName) {
+ Diags.Report(AA->getLocation(), diag::err_cyclic_alias);
+ return;
+ }
+
// If there is a definition in the module, then it wins over the alias.
// This is dubious, but allow it to be safe. Just ignore the alias.
llvm::GlobalValue *Entry = GetGlobalValue(MangledName);
@@ -2506,8 +2761,7 @@
// Create the new alias itself, but don't set a name yet.
auto *GA = llvm::GlobalAlias::create(
- cast<llvm::PointerType>(Aliasee->getType()),
- llvm::Function::ExternalLinkage, "", Aliasee, &getModule());
+ DeclTy, 0, llvm::Function::ExternalLinkage, "", Aliasee, &getModule());
if (Entry) {
if (GA->getAliasee() == Entry) {
@@ -2597,7 +2851,7 @@
return *Map.insert(std::make_pair(String, nullptr)).first;
}
-llvm::Constant *
+ConstantAddress
CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) {
unsigned StringLength = 0;
bool isUTF16 = false;
@@ -2607,7 +2861,7 @@
StringLength);
if (auto *C = Entry.second)
- return C;
+ return ConstantAddress(C, CharUnits::fromQuantity(C->getAlignment()));
llvm::Constant *Zero = llvm::Constant::getNullValue(Int32Ty);
llvm::Constant *Zeros[] = { Zero, Zero };
@@ -2643,7 +2897,7 @@
// String pointer.
llvm::Constant *C = nullptr;
if (isUTF16) {
- ArrayRef<uint16_t> Arr = llvm::makeArrayRef<uint16_t>(
+ auto Arr = llvm::makeArrayRef(
reinterpret_cast<uint16_t *>(const_cast<char *>(Entry.first().data())),
Entry.first().size() / 2);
C = llvm::ConstantDataArray::get(VMContext, Arr);
@@ -2684,25 +2938,28 @@
Ty = getTypes().ConvertType(getContext().LongTy);
Fields[3] = llvm::ConstantInt::get(Ty, StringLength);
+ CharUnits Alignment = getPointerAlign();
+
// The struct.
C = llvm::ConstantStruct::get(STy, Fields);
GV = new llvm::GlobalVariable(getModule(), C->getType(), true,
llvm::GlobalVariable::PrivateLinkage, C,
"_unnamed_cfstring_");
GV->setSection("__DATA,__cfstring");
+ GV->setAlignment(Alignment.getQuantity());
Entry.second = GV;
- return GV;
+ return ConstantAddress(GV, Alignment);
}
-llvm::GlobalVariable *
+ConstantAddress
CodeGenModule::GetAddrOfConstantString(const StringLiteral *Literal) {
unsigned StringLength = 0;
llvm::StringMapEntry<llvm::GlobalVariable *> &Entry =
GetConstantStringEntry(CFConstantStringMap, Literal, StringLength);
if (auto *C = Entry.second)
- return C;
+ return ConstantAddress(C, CharUnits::fromQuantity(C->getAlignment()));
llvm::Constant *Zero = llvm::Constant::getNullValue(Int32Ty);
llvm::Constant *Zeros[] = { Zero, Zero };
@@ -2795,10 +3052,12 @@
Fields[2] = llvm::ConstantInt::get(Ty, StringLength);
// The struct.
+ CharUnits Alignment = getPointerAlign();
C = llvm::ConstantStruct::get(NSConstantStringType, Fields);
GV = new llvm::GlobalVariable(getModule(), C->getType(), true,
llvm::GlobalVariable::PrivateLinkage, C,
"_unnamed_nsstring_");
+ GV->setAlignment(Alignment.getQuantity());
const char *NSStringSection = "__OBJC,__cstring_object,regular,no_dead_strip";
const char *NSStringNonFragileABISection =
"__DATA,__objc_stringobj,regular,no_dead_strip";
@@ -2808,7 +3067,7 @@
: NSStringSection);
Entry.second = GV;
- return GV;
+ return ConstantAddress(GV, Alignment);
}
QualType CodeGenModule::getObjCFastEnumerationStateType() {
@@ -2887,7 +3146,7 @@
static llvm::GlobalVariable *
GenerateStringLiteral(llvm::Constant *C, llvm::GlobalValue::LinkageTypes LT,
CodeGenModule &CGM, StringRef GlobalName,
- unsigned Alignment) {
+ CharUnits Alignment) {
// OpenCL v1.2 s6.5.3: a string literal is in the constant address space.
unsigned AddrSpace = 0;
if (CGM.getLangOpts().OpenCL)
@@ -2898,7 +3157,7 @@
auto *GV = new llvm::GlobalVariable(
M, C->getType(), !CGM.getLangOpts().WritableStrings, LT, C, GlobalName,
nullptr, llvm::GlobalVariable::NotThreadLocal, AddrSpace);
- GV->setAlignment(Alignment);
+ GV->setAlignment(Alignment.getQuantity());
GV->setUnnamedAddr(true);
if (GV->isWeakForLinker()) {
assert(CGM.supportsCOMDAT() && "Only COFF uses weak string literals");
@@ -2910,20 +3169,19 @@
/// GetAddrOfConstantStringFromLiteral - Return a pointer to a
/// constant array for the given string literal.
-llvm::GlobalVariable *
+ConstantAddress
CodeGenModule::GetAddrOfConstantStringFromLiteral(const StringLiteral *S,
StringRef Name) {
- auto Alignment =
- getContext().getAlignOfGlobalVarInChars(S->getType()).getQuantity();
+ CharUnits Alignment = getContext().getAlignOfGlobalVarInChars(S->getType());
llvm::Constant *C = GetConstantArrayFromStringLiteral(S);
llvm::GlobalVariable **Entry = nullptr;
if (!LangOpts.WritableStrings) {
Entry = &ConstantStringMap[C];
if (auto GV = *Entry) {
- if (Alignment > GV->getAlignment())
- GV->setAlignment(Alignment);
- return GV;
+ if (Alignment.getQuantity() > GV->getAlignment())
+ GV->setAlignment(Alignment.getQuantity());
+ return ConstantAddress(GV, Alignment);
}
}
@@ -2939,7 +3197,6 @@
getCXXABI().getMangleContext().shouldMangleStringLiteral(S)) {
llvm::raw_svector_ostream Out(MangledNameBuffer);
getCXXABI().getMangleContext().mangleStringLiteral(S, Out);
- Out.flush();
LT = llvm::GlobalValue::LinkOnceODRLinkage;
GlobalVariableName = MangledNameBuffer;
@@ -2954,12 +3211,12 @@
SanitizerMD->reportGlobalToASan(GV, S->getStrTokenLoc(0), "<string literal>",
QualType());
- return GV;
+ return ConstantAddress(GV, Alignment);
}
/// GetAddrOfConstantStringFromObjCEncode - Return a pointer to a constant
/// array for the given ObjCEncodeExpr node.
-llvm::GlobalVariable *
+ConstantAddress
CodeGenModule::GetAddrOfConstantStringFromObjCEncode(const ObjCEncodeExpr *E) {
std::string Str;
getContext().getObjCEncodingForType(E->getEncodedType(), Str);
@@ -2970,14 +3227,11 @@
/// GetAddrOfConstantCString - Returns a pointer to a character array containing
/// the literal and a terminating '\0' character.
/// The result has pointer to array type.
-llvm::GlobalVariable *CodeGenModule::GetAddrOfConstantCString(
- const std::string &Str, const char *GlobalName, unsigned Alignment) {
+ConstantAddress CodeGenModule::GetAddrOfConstantCString(
+ const std::string &Str, const char *GlobalName) {
StringRef StrWithNull(Str.c_str(), Str.size() + 1);
- if (Alignment == 0) {
- Alignment = getContext()
- .getAlignOfGlobalVarInChars(getContext().CharTy)
- .getQuantity();
- }
+ CharUnits Alignment =
+ getContext().getAlignOfGlobalVarInChars(getContext().CharTy);
llvm::Constant *C =
llvm::ConstantDataArray::getString(getLLVMContext(), StrWithNull, false);
@@ -2987,9 +3241,9 @@
if (!LangOpts.WritableStrings) {
Entry = &ConstantStringMap[C];
if (auto GV = *Entry) {
- if (Alignment > GV->getAlignment())
- GV->setAlignment(Alignment);
- return GV;
+ if (Alignment.getQuantity() > GV->getAlignment())
+ GV->setAlignment(Alignment.getQuantity());
+ return ConstantAddress(GV, Alignment);
}
}
@@ -3001,10 +3255,10 @@
GlobalName, Alignment);
if (Entry)
*Entry = GV;
- return GV;
+ return ConstantAddress(GV, Alignment);
}
-llvm::Constant *CodeGenModule::GetAddrOfGlobalTemporary(
+ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary(
const MaterializeTemporaryExpr *E, const Expr *Init) {
assert((E->getStorageDuration() == SD_Static ||
E->getStorageDuration() == SD_Thread) && "not a global temporary");
@@ -3016,9 +3270,10 @@
if (Init == E->GetTemporaryExpr())
MaterializedType = E->getType();
- llvm::Constant *&Slot = MaterializedGlobalTemporaryMap[E];
- if (Slot)
- return Slot;
+ CharUnits Align = getContext().getTypeAlignInChars(MaterializedType);
+
+ if (llvm::Constant *Slot = MaterializedGlobalTemporaryMap[E])
+ return ConstantAddress(Slot, Align);
// FIXME: If an externally-visible declaration extends multiple temporaries,
// we need to give each temporary the same name in every translation unit (and
@@ -3027,7 +3282,6 @@
llvm::raw_svector_ostream Out(Name);
getCXXABI().getMangleContext().mangleReferenceTemporary(
VD, E->getManglingNumber(), Out);
- Out.flush();
APValue *Value = nullptr;
if (E->getStorageDuration() == SD_Static) {
@@ -3083,14 +3337,13 @@
/*InsertBefore=*/nullptr, llvm::GlobalVariable::NotThreadLocal,
AddrSpace);
setGlobalVisibility(GV, VD);
- GV->setAlignment(
- getContext().getTypeAlignInChars(MaterializedType).getQuantity());
+ GV->setAlignment(Align.getQuantity());
if (supportsCOMDAT() && GV->isWeakForLinker())
GV->setComdat(TheModule.getOrInsertComdat(GV->getName()));
if (VD->getTLSKind())
setTLSMode(GV, *VD);
- Slot = GV;
- return GV;
+ MaterializedGlobalTemporaryMap[E] = GV;
+ return ConstantAddress(GV, Align);
}
/// EmitObjCPropertyImplementations - Emit information for synthesized
@@ -3352,11 +3605,10 @@
auto *Import = cast<ImportDecl>(D);
// Ignore import declarations that come from imported modules.
- if (clang::Module *Owner = Import->getImportedOwningModule()) {
- if (getLangOpts().CurrentModule.empty() ||
- Owner->getTopLevelModule()->Name == getLangOpts().CurrentModule)
- break;
- }
+ if (Import->getImportedOwningModule())
+ break;
+ if (CGDebugInfo *DI = getModuleDebugInfo())
+ DI->EmitImportDecl(*Import);
ImportedModules.insert(Import->getImportedModule());
break;
@@ -3395,7 +3647,7 @@
case Decl::ObjCMethod:
case Decl::CXXConstructor:
case Decl::CXXDestructor: {
- if (!cast<FunctionDecl>(D)->hasBody())
+ if (!cast<FunctionDecl>(D)->doesThisDeclarationHaveABody())
return;
auto I = DeferredEmptyCoverageMappingDecls.find(D);
if (I == DeferredEmptyCoverageMappingDecls.end())
@@ -3524,10 +3776,12 @@
void CodeGenModule::EmitDeclMetadata() {
llvm::NamedMDNode *GlobalMetadata = nullptr;
- // StaticLocalDeclMap
for (auto &I : MangledDeclNames) {
llvm::GlobalValue *Addr = getModule().getNamedValue(I.second);
- EmitGlobalDeclMetadata(*this, GlobalMetadata, I.first, Addr);
+ // Some mangled names don't necessarily have an associated GlobalValue
+ // in this module, e.g. if we mangled it for DebugInfo.
+ if (Addr)
+ EmitGlobalDeclMetadata(*this, GlobalMetadata, I.first, Addr);
}
}
@@ -3545,7 +3799,7 @@
for (auto &I : LocalDeclMap) {
const Decl *D = I.first;
- llvm::Value *Addr = I.second;
+ llvm::Value *Addr = I.second.getPointer();
if (auto *Alloca = dyn_cast<llvm::AllocaInst>(Addr)) {
llvm::Value *DAddr = GetPointerConstant(getLLVMContext(), D);
Alloca->setMetadata(
@@ -3626,12 +3880,6 @@
return llvm::ConstantStruct::getAnon(Fields);
}
-llvm::Constant *
-CodeGenModule::getAddrOfCXXCatchHandlerType(QualType Ty,
- QualType CatchHandlerType) {
- return getCXXABI().getAddrOfCXXCatchHandlerType(Ty, CatchHandlerType);
-}
-
llvm::Constant *CodeGenModule::GetAddrOfRTTIDescriptor(QualType Ty,
bool ForEH) {
// Return a bogus pointer if RTTI is disabled, unless it's for EH.
@@ -3654,8 +3902,82 @@
VD->getAnyInitializer() &&
!VD->getAnyInitializer()->isConstantInitializer(getContext(),
/*ForRef=*/false);
+
+ Address Addr(GetAddrOfGlobalVar(VD), getContext().getDeclAlign(VD));
if (auto InitFunction = getOpenMPRuntime().emitThreadPrivateVarDefinition(
- VD, GetAddrOfGlobalVar(VD), RefExpr->getLocStart(), PerformInit))
+ VD, Addr, RefExpr->getLocStart(), PerformInit))
CXXGlobalInits.push_back(InitFunction);
}
}
+
+llvm::Metadata *CodeGenModule::CreateMetadataIdentifierForType(QualType T) {
+ llvm::Metadata *&InternalId = MetadataIdMap[T.getCanonicalType()];
+ if (InternalId)
+ return InternalId;
+
+ if (isExternallyVisible(T->getLinkage())) {
+ std::string OutName;
+ llvm::raw_string_ostream Out(OutName);
+ getCXXABI().getMangleContext().mangleTypeName(T, Out);
+
+ InternalId = llvm::MDString::get(getLLVMContext(), Out.str());
+ } else {
+ InternalId = llvm::MDNode::getDistinct(getLLVMContext(),
+ llvm::ArrayRef<llvm::Metadata *>());
+ }
+
+ return InternalId;
+}
+
+void CodeGenModule::CreateVTableBitSetEntry(llvm::NamedMDNode *BitsetsMD,
+ llvm::GlobalVariable *VTable,
+ CharUnits Offset,
+ const CXXRecordDecl *RD) {
+ llvm::Metadata *MD =
+ CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
+ llvm::Metadata *BitsetOps[] = {
+ MD, llvm::ConstantAsMetadata::get(VTable),
+ llvm::ConstantAsMetadata::get(
+ llvm::ConstantInt::get(Int64Ty, Offset.getQuantity()))};
+ BitsetsMD->addOperand(llvm::MDTuple::get(getLLVMContext(), BitsetOps));
+
+ if (CodeGenOpts.SanitizeCfiCrossDso) {
+ if (auto TypeId = CreateCfiIdForTypeMetadata(MD)) {
+ llvm::Metadata *BitsetOps2[] = {
+ llvm::ConstantAsMetadata::get(TypeId),
+ llvm::ConstantAsMetadata::get(VTable),
+ llvm::ConstantAsMetadata::get(
+ llvm::ConstantInt::get(Int64Ty, Offset.getQuantity()))};
+ BitsetsMD->addOperand(llvm::MDTuple::get(getLLVMContext(), BitsetOps2));
+ }
+ }
+}
+
+// Fills in the supplied string map with the set of target features for the
+// passed in function.
+void CodeGenModule::getFunctionFeatureMap(llvm::StringMap<bool> &FeatureMap,
+ const FunctionDecl *FD) {
+ StringRef TargetCPU = Target.getTargetOpts().CPU;
+ if (const auto *TD = FD->getAttr<TargetAttr>()) {
+ // If we have a TargetAttr build up the feature map based on that.
+ TargetAttr::ParsedTargetAttr ParsedAttr = TD->parse();
+
+ // Make a copy of the features as passed on the command line into the
+ // beginning of the additional features from the function to override.
+ ParsedAttr.first.insert(ParsedAttr.first.begin(),
+ Target.getTargetOpts().FeaturesAsWritten.begin(),
+ Target.getTargetOpts().FeaturesAsWritten.end());
+
+ if (ParsedAttr.second != "")
+ TargetCPU = ParsedAttr.second;
+
+ // Now populate the feature map, first with the TargetCPU which is either
+ // the default or a new one from the target attribute string. Then we'll use
+ // the passed in features (FeaturesAsWritten) along with the new ones from
+ // the attribute.
+ Target.initFeatureMap(FeatureMap, getDiags(), TargetCPU, ParsedAttr.first);
+ } else {
+ Target.initFeatureMap(FeatureMap, getDiags(), TargetCPU,
+ Target.getTargetOpts().Features);
+ }
+}
diff --git a/lib/CodeGen/CodeGenModule.h b/lib/CodeGen/CodeGenModule.h
index edde426..3311383 100644
--- a/lib/CodeGen/CodeGenModule.h
+++ b/lib/CodeGen/CodeGenModule.h
@@ -15,6 +15,7 @@
#define LLVM_CLANG_LIB_CODEGEN_CODEGENMODULE_H
#include "CGVTables.h"
+#include "CodeGenTypeCache.h"
#include "CodeGenTypes.h"
#include "SanitizerMetadata.h"
#include "clang/AST/Attr.h"
@@ -30,7 +31,6 @@
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/StringMap.h"
-#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/ValueHandle.h"
@@ -69,6 +69,8 @@
class VarDecl;
class LangOptions;
class CodeGenOptions;
+class HeaderSearchOptions;
+class PreprocessorOptions;
class DiagnosticsEngine;
class AnnotateAttr;
class CXXDestructorDecl;
@@ -106,65 +108,14 @@
}
};
-struct CodeGenTypeCache {
- /// void
- llvm::Type *VoidTy;
+struct ObjCEntrypoints {
+ ObjCEntrypoints() { memset(this, 0, sizeof(*this)); }
- /// i8, i16, i32, and i64
- llvm::IntegerType *Int8Ty, *Int16Ty, *Int32Ty, *Int64Ty;
- /// float, double
- llvm::Type *FloatTy, *DoubleTy;
-
- /// int
- llvm::IntegerType *IntTy;
-
- /// intptr_t, size_t, and ptrdiff_t, which we assume are the same size.
- union {
- llvm::IntegerType *IntPtrTy;
- llvm::IntegerType *SizeTy;
- llvm::IntegerType *PtrDiffTy;
- };
-
- /// void* in address space 0
- union {
- llvm::PointerType *VoidPtrTy;
- llvm::PointerType *Int8PtrTy;
- };
-
- /// void** in address space 0
- union {
- llvm::PointerType *VoidPtrPtrTy;
- llvm::PointerType *Int8PtrPtrTy;
- };
-
- /// The width of a pointer into the generic address space.
- unsigned char PointerWidthInBits;
-
- /// The size and alignment of a pointer into the generic address
- /// space.
- union {
- unsigned char PointerAlignInBytes;
- unsigned char PointerSizeInBytes;
- unsigned char SizeSizeInBytes; // sizeof(size_t)
- };
-
- llvm::CallingConv::ID RuntimeCC;
- llvm::CallingConv::ID getRuntimeCC() const { return RuntimeCC; }
- llvm::CallingConv::ID BuiltinCC;
- llvm::CallingConv::ID getBuiltinCC() const { return BuiltinCC; }
-};
-
-struct RREntrypoints {
- RREntrypoints() { memset(this, 0, sizeof(*this)); }
- /// void objc_autoreleasePoolPop(void*);
+ /// void objc_autoreleasePoolPop(void*);
llvm::Constant *objc_autoreleasePoolPop;
/// void *objc_autoreleasePoolPush(void);
llvm::Constant *objc_autoreleasePoolPush;
-};
-
-struct ARCEntrypoints {
- ARCEntrypoints() { memset(this, 0, sizeof(*this)); }
/// id objc_autorelease(id);
llvm::Constant *objc_autorelease;
@@ -255,6 +206,36 @@
void reportDiagnostics(DiagnosticsEngine &Diags, StringRef MainFile);
};
+/// A pair of helper functions for a __block variable.
+class BlockByrefHelpers : public llvm::FoldingSetNode {
+ // MSVC requires this type to be complete in order to process this
+ // header.
+public:
+ llvm::Constant *CopyHelper;
+ llvm::Constant *DisposeHelper;
+
+ /// The alignment of the field. This is important because
+ /// different offsets to the field within the byref struct need to
+ /// have different helper functions.
+ CharUnits Alignment;
+
+ BlockByrefHelpers(CharUnits alignment) : Alignment(alignment) {}
+ BlockByrefHelpers(const BlockByrefHelpers &) = default;
+ virtual ~BlockByrefHelpers();
+
+ void Profile(llvm::FoldingSetNodeID &id) const {
+ id.AddInteger(Alignment.getQuantity());
+ profileImpl(id);
+ }
+ virtual void profileImpl(llvm::FoldingSetNodeID &id) const = 0;
+
+ virtual bool needsCopy() const { return true; }
+ virtual void emitCopy(CodeGenFunction &CGF, Address dest, Address src) = 0;
+
+ virtual bool needsDispose() const { return true; }
+ virtual void emitDispose(CodeGenFunction &CGF, Address field) = 0;
+};
+
/// This class organizes the cross-function state that is used while generating
/// LLVM code.
class CodeGenModule : public CodeGenTypeCache {
@@ -278,10 +259,11 @@
private:
ASTContext &Context;
const LangOptions &LangOpts;
+ const HeaderSearchOptions &HeaderSearchOpts; // Only used for debug info.
+ const PreprocessorOptions &PreprocessorOpts; // Only used for debug info.
const CodeGenOptions &CodeGenOpts;
llvm::Module &TheModule;
DiagnosticsEngine &Diags;
- const llvm::DataLayout &TheDataLayout;
const TargetInfo &Target;
std::unique_ptr<CGCXXABI> ABI;
llvm::LLVMContext &VMContext;
@@ -303,9 +285,8 @@
CGOpenMPRuntime* OpenMPRuntime;
CGCUDARuntime* CUDARuntime;
CGDebugInfo* DebugInfo;
- ARCEntrypoints *ARCData;
+ ObjCEntrypoints *ObjCData;
llvm::MDNode *NoObjCARCExceptionsMetadata;
- RREntrypoints *RRData;
std::unique_ptr<llvm::IndexedInstrProfReader> PGOReader;
InstrProfStats PGOStats;
@@ -339,6 +320,17 @@
typedef llvm::StringMap<llvm::TrackingVH<llvm::Constant> > ReplacementsTy;
ReplacementsTy Replacements;
+ /// List of global values to be replaced with something else. Used when we
+ /// want to replace a GlobalValue but can't identify it by its mangled name
+ /// anymore (because the name is already taken).
+ llvm::SmallVector<std::pair<llvm::GlobalValue *, llvm::Constant *>, 8>
+ GlobalValReplacements;
+
+ /// Set of global decls for which we already diagnosed mangled name conflict.
+ /// Required to not issue a warning (on a mangling conflict) multiple times
+ /// for the same decl.
+ llvm::DenseSet<GlobalDecl> DiagnosedConflictingDefinitions;
+
/// A queue of (optional) vtables to consider emitting.
std::vector<const CXXRecordDecl*> DeferredVTables;
@@ -386,13 +378,12 @@
StaticExternCMap StaticExternCValues;
/// \brief thread_local variables defined or used in this TU.
- std::vector<std::pair<const VarDecl *, llvm::GlobalVariable *> >
- CXXThreadLocals;
+ std::vector<const VarDecl *> CXXThreadLocals;
/// \brief thread_local variables with initializers that need to run
/// before any thread_local variable in this TU is odr-used.
std::vector<llvm::Function *> CXXThreadLocalInits;
- std::vector<llvm::GlobalVariable *> CXXThreadLocalInitVars;
+ std::vector<const VarDecl *> CXXThreadLocalInitVars;
/// Global variables with initializers that need to run before main.
std::vector<llvm::Function *> CXXGlobalInits;
@@ -487,9 +478,16 @@
llvm::DenseMap<const Decl *, bool> DeferredEmptyCoverageMappingDecls;
std::unique_ptr<CoverageMappingModuleGen> CoverageMapping;
+
+ /// Mapping from canonical types to their metadata identifiers. We need to
+ /// maintain this mapping because identifiers may be formed from distinct
+ /// MDNodes.
+ llvm::DenseMap<QualType, llvm::Metadata *> MetadataIdMap;
+
public:
- CodeGenModule(ASTContext &C, const CodeGenOptions &CodeGenOpts,
- llvm::Module &M, const llvm::DataLayout &TD,
+ CodeGenModule(ASTContext &C, const HeaderSearchOptions &headersearchopts,
+ const PreprocessorOptions &ppopts,
+ const CodeGenOptions &CodeGenOpts, llvm::Module &M,
DiagnosticsEngine &Diags,
CoverageSourceInfo *CoverageInfo = nullptr);
@@ -527,14 +525,9 @@
return *CUDARuntime;
}
- ARCEntrypoints &getARCEntrypoints() const {
- assert(getLangOpts().ObjCAutoRefCount && ARCData != nullptr);
- return *ARCData;
- }
-
- RREntrypoints &getRREntrypoints() const {
- assert(RRData != nullptr);
- return *RRData;
+ ObjCEntrypoints &getObjCEntrypoints() const {
+ assert(ObjCData != nullptr);
+ return *ObjCData;
}
InstrProfStats &getPGOStats() { return PGOStats; }
@@ -600,10 +593,16 @@
ASTContext &getContext() const { return Context; }
const LangOptions &getLangOpts() const { return LangOpts; }
+ const HeaderSearchOptions &getHeaderSearchOpts()
+ const { return HeaderSearchOpts; }
+ const PreprocessorOptions &getPreprocessorOpts()
+ const { return PreprocessorOpts; }
const CodeGenOptions &getCodeGenOpts() const { return CodeGenOpts; }
llvm::Module &getModule() const { return TheModule; }
DiagnosticsEngine &getDiags() const { return Diags; }
- const llvm::DataLayout &getDataLayout() const { return TheDataLayout; }
+ const llvm::DataLayout &getDataLayout() const {
+ return TheModule.getDataLayout();
+ }
const TargetInfo &getTarget() const { return Target; }
const llvm::Triple &getTriple() const;
bool supportsCOMDAT() const;
@@ -634,8 +633,6 @@
llvm::MDNode *getTBAAInfo(QualType QTy);
llvm::MDNode *getTBAAInfoForVTablePtr();
llvm::MDNode *getTBAAStructInfo(QualType QTy);
- /// Return the MDNode in the type DAG for the given struct type.
- llvm::MDNode *getTBAAStructTypeInfo(QualType QTy);
/// Return the path-aware tag for given base type, access node and offset.
llvm::MDNode *getTBAAStructTagInfo(QualType BaseTy, llvm::MDNode *AccessN,
uint64_t O);
@@ -649,9 +646,13 @@
/// is the same as the type. For struct-path aware TBAA, the tag
/// is different from the type: base type, access type and offset.
/// When ConvertTypeToTag is true, we create a tag based on the scalar type.
- void DecorateInstruction(llvm::Instruction *Inst,
- llvm::MDNode *TBAAInfo,
- bool ConvertTypeToTag = true);
+ void DecorateInstructionWithTBAA(llvm::Instruction *Inst,
+ llvm::MDNode *TBAAInfo,
+ bool ConvertTypeToTag = true);
+
+ /// Adds !invariant.barrier !tag to instruction
+ void DecorateInstructionWithInvariantGroup(llvm::Instruction *I,
+ const CXXRecordDecl *RD);
/// Emit the given number of characters as a value of type size_t.
llvm::ConstantInt *getSize(CharUnits numChars);
@@ -672,18 +673,7 @@
llvm_unreachable("unknown visibility!");
}
- llvm::Constant *GetAddrOfGlobal(GlobalDecl GD) {
- if (isa<CXXConstructorDecl>(GD.getDecl()))
- return getAddrOfCXXStructor(cast<CXXConstructorDecl>(GD.getDecl()),
- getFromCtorType(GD.getCtorType()));
- else if (isa<CXXDestructorDecl>(GD.getDecl()))
- return getAddrOfCXXStructor(cast<CXXDestructorDecl>(GD.getDecl()),
- getFromDtorType(GD.getDtorType()));
- else if (isa<FunctionDecl>(GD.getDecl()))
- return GetAddrOfFunction(GD);
- else
- return GetAddrOfGlobalVar(cast<VarDecl>(GD.getDecl()));
- }
+ llvm::Constant *GetAddrOfGlobal(GlobalDecl GD, bool IsForDefinition = false);
/// Will return a global variable of the given type. If a variable with a
/// different type already exists then a new variable with the right type
@@ -695,6 +685,7 @@
llvm::Function *
CreateGlobalInitOrDestructFunction(llvm::FunctionType *ty, const Twine &name,
+ const CGFunctionInfo &FI,
SourceLocation Loc = SourceLocation(),
bool TLS = false);
@@ -713,24 +704,42 @@
/// Return the address of the given function. If Ty is non-null, then this
/// function will use the specified type if it has to create it.
- llvm::Constant *GetAddrOfFunction(GlobalDecl GD, llvm::Type *Ty = 0,
+ llvm::Constant *GetAddrOfFunction(GlobalDecl GD, llvm::Type *Ty = nullptr,
bool ForVTable = false,
- bool DontDefer = false);
+ bool DontDefer = false,
+ bool IsForDefinition = false);
/// Get the address of the RTTI descriptor for the given type.
llvm::Constant *GetAddrOfRTTIDescriptor(QualType Ty, bool ForEH = false);
- llvm::Constant *getAddrOfCXXCatchHandlerType(QualType Ty,
- QualType CatchHandlerType);
-
/// Get the address of a uuid descriptor .
- llvm::Constant *GetAddrOfUuidDescriptor(const CXXUuidofExpr* E);
+ ConstantAddress GetAddrOfUuidDescriptor(const CXXUuidofExpr* E);
/// Get the address of the thunk for the given global decl.
llvm::Constant *GetAddrOfThunk(GlobalDecl GD, const ThunkInfo &Thunk);
/// Get a reference to the target of VD.
- llvm::Constant *GetWeakRefReference(const ValueDecl *VD);
+ ConstantAddress GetWeakRefReference(const ValueDecl *VD);
+
+ /// Returns the assumed alignment of an opaque pointer to the given class.
+ CharUnits getClassPointerAlignment(const CXXRecordDecl *CD);
+
+ /// Returns the assumed alignment of a virtual base of a class.
+ CharUnits getVBaseAlignment(CharUnits DerivedAlign,
+ const CXXRecordDecl *Derived,
+ const CXXRecordDecl *VBase);
+
+ /// Given a class pointer with an actual known alignment, and the
+ /// expected alignment of an object at a dynamic offset w.r.t that
+ /// pointer, return the alignment to assume at the offset.
+ CharUnits getDynamicOffsetAlignment(CharUnits ActualAlign,
+ const CXXRecordDecl *Class,
+ CharUnits ExpectedTargetAlign);
+
+ CharUnits
+ computeNonVirtualBaseClassOffset(const CXXRecordDecl *DerivedClass,
+ CastExpr::path_const_iterator Start,
+ CastExpr::path_const_iterator End);
/// Returns the offset from a derived class to a class. Returns null if the
/// offset is 0.
@@ -739,35 +748,7 @@
CastExpr::path_const_iterator PathBegin,
CastExpr::path_const_iterator PathEnd);
- /// A pair of helper functions for a __block variable.
- class ByrefHelpers : public llvm::FoldingSetNode {
- public:
- llvm::Constant *CopyHelper;
- llvm::Constant *DisposeHelper;
-
- /// The alignment of the field. This is important because
- /// different offsets to the field within the byref struct need to
- /// have different helper functions.
- CharUnits Alignment;
-
- ByrefHelpers(CharUnits alignment) : Alignment(alignment) {}
- virtual ~ByrefHelpers();
-
- void Profile(llvm::FoldingSetNodeID &id) const {
- id.AddInteger(Alignment.getQuantity());
- profileImpl(id);
- }
- virtual void profileImpl(llvm::FoldingSetNodeID &id) const = 0;
-
- virtual bool needsCopy() const { return true; }
- virtual void emitCopy(CodeGenFunction &CGF,
- llvm::Value *dest, llvm::Value *src) = 0;
-
- virtual bool needsDispose() const { return true; }
- virtual void emitDispose(CodeGenFunction &CGF, llvm::Value *field) = 0;
- };
-
- llvm::FoldingSet<ByrefHelpers> ByrefHelpersCache;
+ llvm::FoldingSet<BlockByrefHelpers> ByrefHelpersCache;
/// Fetches the global unique block count.
int getUniqueBlockCount() { return ++Block.GlobalUniqueCount; }
@@ -782,23 +763,23 @@
llvm::Constant *GetAddrOfGlobalBlock(const BlockExpr *BE, const char *);
/// Return a pointer to a constant CFString object for the given string.
- llvm::Constant *GetAddrOfConstantCFString(const StringLiteral *Literal);
+ ConstantAddress GetAddrOfConstantCFString(const StringLiteral *Literal);
/// Return a pointer to a constant NSString object for the given string. Or a
/// user defined String object as defined via
/// -fconstant-string-class=class_name option.
- llvm::GlobalVariable *GetAddrOfConstantString(const StringLiteral *Literal);
+ ConstantAddress GetAddrOfConstantString(const StringLiteral *Literal);
/// Return a constant array for the given string.
llvm::Constant *GetConstantArrayFromStringLiteral(const StringLiteral *E);
/// Return a pointer to a constant array for the given string literal.
- llvm::GlobalVariable *
+ ConstantAddress
GetAddrOfConstantStringFromLiteral(const StringLiteral *S,
StringRef Name = ".str");
/// Return a pointer to a constant array for the given ObjCEncodeExpr node.
- llvm::GlobalVariable *
+ ConstantAddress
GetAddrOfConstantStringFromObjCEncode(const ObjCEncodeExpr *);
/// Returns a pointer to a character array containing the literal and a
@@ -806,18 +787,17 @@
///
/// \param GlobalName If provided, the name to use for the global (if one is
/// created).
- llvm::GlobalVariable *
+ ConstantAddress
GetAddrOfConstantCString(const std::string &Str,
- const char *GlobalName = nullptr,
- unsigned Alignment = 0);
+ const char *GlobalName = nullptr);
/// Returns a pointer to a constant global variable for the given file-scope
/// compound literal expression.
- llvm::Constant *GetAddrOfConstantCompoundLiteral(const CompoundLiteralExpr*E);
+ ConstantAddress GetAddrOfConstantCompoundLiteral(const CompoundLiteralExpr*E);
/// \brief Returns a pointer to a global variable representing a temporary
/// with static or thread storage duration.
- llvm::Constant *GetAddrOfGlobalTemporary(const MaterializeTemporaryExpr *E,
+ ConstantAddress GetAddrOfGlobalTemporary(const MaterializeTemporaryExpr *E,
const Expr *Inner);
/// \brief Retrieve the record type that describes the state of an
@@ -831,11 +811,11 @@
StructorType Type);
/// Return the address of the constructor/destructor of the given type.
- llvm::GlobalValue *
+ llvm::Constant *
getAddrOfCXXStructor(const CXXMethodDecl *MD, StructorType Type,
const CGFunctionInfo *FnInfo = nullptr,
llvm::FunctionType *FnType = nullptr,
- bool DontDefer = false);
+ bool DontDefer = false, bool IsForDefinition = false);
/// Given a builtin id for a function like "__builtin_fabsf", return a
/// Function* for "fabsf".
@@ -932,6 +912,11 @@
QualType DestType,
CodeGenFunction *CGF = nullptr);
+ /// \brief Emit type info if type of an expression is a variably modified
+ /// type. Also emit proper debug info for cast types.
+ void EmitExplicitCastExprType(const ExplicitCastExpr *E,
+ CodeGenFunction *CGF = nullptr);
+
/// Return the result of value-initializing the given type, i.e. a null
/// expression of the given type. This is usually, but not always, an LLVM
/// null constant.
@@ -982,16 +967,19 @@
/// function type.
///
/// \param Info - The function type information.
- /// \param TargetDecl - The decl these attributes are being constructed
- /// for. If supplied the attributes applied to this decl may contribute to the
- /// function attributes and calling convention.
+ /// \param CalleeInfo - The callee information these attributes are being
+ /// constructed for. If valid, the attributes applied to this decl may
+ /// contribute to the function attributes and calling convention.
/// \param PAL [out] - On return, the attribute list to use.
/// \param CallingConv [out] - On return, the LLVM calling convention to use.
void ConstructAttributeList(const CGFunctionInfo &Info,
- const Decl *TargetDecl,
- AttributeListType &PAL,
- unsigned &CallingConv,
- bool AttrOnCallSite);
+ CGCalleeInfo CalleeInfo, AttributeListType &PAL,
+ unsigned &CallingConv, bool AttrOnCallSite);
+
+ // Fills in the supplied string map with the set of target features for the
+ // passed in function.
+ void getFunctionFeatureMap(llvm::StringMap<bool> &FeatureMap,
+ const FunctionDecl *FD);
StringRef getMangledName(GlobalDecl GD);
StringRef getBlockMangledName(GlobalDecl GD, const BlockDecl *BD);
@@ -1000,9 +988,6 @@
void EmitVTable(CXXRecordDecl *Class);
- /// Emit the RTTI descriptors for the builtin types.
- void EmitFundamentalRTTIDescriptors();
-
/// \brief Appends Opts to the "Linker Options" metadata value.
void AppendLinkerOptions(StringRef Opts);
@@ -1106,15 +1091,37 @@
void addReplacement(StringRef Name, llvm::Constant *C);
+ void addGlobalValReplacement(llvm::GlobalValue *GV, llvm::Constant *C);
+
/// \brief Emit a code for threadprivate directive.
/// \param D Threadprivate declaration.
void EmitOMPThreadPrivateDecl(const OMPThreadPrivateDecl *D);
+ /// Returns whether the given record is blacklisted from control flow
+ /// integrity checks.
+ bool IsCFIBlacklistedRecord(const CXXRecordDecl *RD);
+
/// Emit bit set entries for the given vtable using the given layout if
/// vptr CFI is enabled.
void EmitVTableBitSetEntries(llvm::GlobalVariable *VTable,
const VTableLayout &VTLayout);
+ /// Generate a cross-DSO type identifier for type.
+ llvm::ConstantInt *CreateCfiIdForTypeMetadata(llvm::Metadata *MD);
+
+ /// Create a metadata identifier for the given type. This may either be an
+ /// MDString (for external identifiers) or a distinct unnamed MDNode (for
+ /// internal identifiers).
+ llvm::Metadata *CreateMetadataIdentifierForType(QualType T);
+
+ /// Create a bitset entry for the given function and add it to BitsetsMD.
+ void CreateFunctionBitSetEntry(const FunctionDecl *FD, llvm::Function *F);
+
+ /// Create a bitset entry for the given vtable and add it to BitsetsMD.
+ void CreateVTableBitSetEntry(llvm::NamedMDNode *BitsetsMD,
+ llvm::GlobalVariable *VTable, CharUnits Offset,
+ const CXXRecordDecl *RD);
+
/// \breif Get the declaration of std::terminate for the platform.
llvm::Constant *getTerminateFn();
@@ -1123,7 +1130,8 @@
GetOrCreateLLVMFunction(StringRef MangledName, llvm::Type *Ty, GlobalDecl D,
bool ForVTable, bool DontDefer = false,
bool IsThunk = false,
- llvm::AttributeSet ExtraAttrs = llvm::AttributeSet());
+ llvm::AttributeSet ExtraAttrs = llvm::AttributeSet(),
+ bool IsForDefinition = false);
llvm::Constant *GetOrCreateLLVMGlobal(StringRef MangledName,
llvm::PointerType *PTy,
@@ -1169,7 +1177,7 @@
// FIXME: Hardcoding priority here is gross.
void AddGlobalCtor(llvm::Function *Ctor, int Priority = 65535,
- llvm::Constant *AssociatedData = 0);
+ llvm::Constant *AssociatedData = nullptr);
void AddGlobalDtor(llvm::Function *Dtor, int Priority = 65535);
/// Generates a global array of functions and priorities using the given list
@@ -1177,15 +1185,15 @@
/// as a LLVM constructor or destructor array.
void EmitCtorList(const CtorList &Fns, const char *GlobalName);
- /// Emit the RTTI descriptors for the given type.
- void EmitFundamentalRTTIDescriptor(QualType Type);
-
/// Emit any needed decls for which code generation was deferred.
void EmitDeferred();
/// Call replaceAllUsesWith on all pairs in Replacements.
void applyReplacements();
+ /// Call replaceAllUsesWith on all pairs in GlobalValReplacements.
+ void applyGlobalValReplacements();
+
void checkAliases();
/// Emit any vtables which we deferred and still have a use for.
@@ -1233,4 +1241,4 @@
} // end namespace CodeGen
} // end namespace clang
-#endif
+#endif // LLVM_CLANG_LIB_CODEGEN_CODEGENMODULE_H
diff --git a/lib/CodeGen/CodeGenPGO.cpp b/lib/CodeGen/CodeGenPGO.cpp
index f182a46..3877433 100644
--- a/lib/CodeGen/CodeGenPGO.cpp
+++ b/lib/CodeGen/CodeGenPGO.cpp
@@ -28,58 +28,20 @@
void CodeGenPGO::setFuncName(StringRef Name,
llvm::GlobalValue::LinkageTypes Linkage) {
- StringRef RawFuncName = Name;
-
- // Function names may be prefixed with a binary '1' to indicate
- // that the backend should not modify the symbols due to any platform
- // naming convention. Do not include that '1' in the PGO profile name.
- if (RawFuncName[0] == '\1')
- RawFuncName = RawFuncName.substr(1);
-
- FuncName = RawFuncName;
- if (llvm::GlobalValue::isLocalLinkage(Linkage)) {
- // For local symbols, prepend the main file name to distinguish them.
- // Do not include the full path in the file name since there's no guarantee
- // that it will stay the same, e.g., if the files are checked out from
- // version control in different locations.
- if (CGM.getCodeGenOpts().MainFileName.empty())
- FuncName = FuncName.insert(0, "<unknown>:");
- else
- FuncName = FuncName.insert(0, CGM.getCodeGenOpts().MainFileName + ":");
- }
+ llvm::IndexedInstrProfReader *PGOReader = CGM.getPGOReader();
+ FuncName = llvm::getPGOFuncName(
+ Name, Linkage, CGM.getCodeGenOpts().MainFileName,
+ PGOReader ? PGOReader->getVersion() : llvm::IndexedInstrProf::Version);
// If we're generating a profile, create a variable for the name.
if (CGM.getCodeGenOpts().ProfileInstrGenerate)
- createFuncNameVar(Linkage);
+ FuncNameVar = llvm::createPGOFuncNameVar(CGM.getModule(), Linkage, FuncName);
}
void CodeGenPGO::setFuncName(llvm::Function *Fn) {
setFuncName(Fn->getName(), Fn->getLinkage());
}
-void CodeGenPGO::createFuncNameVar(llvm::GlobalValue::LinkageTypes Linkage) {
- // We generally want to match the function's linkage, but available_externally
- // and extern_weak both have the wrong semantics, and anything that doesn't
- // need to link across compilation units doesn't need to be visible at all.
- if (Linkage == llvm::GlobalValue::ExternalWeakLinkage)
- Linkage = llvm::GlobalValue::LinkOnceAnyLinkage;
- else if (Linkage == llvm::GlobalValue::AvailableExternallyLinkage)
- Linkage = llvm::GlobalValue::LinkOnceODRLinkage;
- else if (Linkage == llvm::GlobalValue::InternalLinkage ||
- Linkage == llvm::GlobalValue::ExternalLinkage)
- Linkage = llvm::GlobalValue::PrivateLinkage;
-
- auto *Value =
- llvm::ConstantDataArray::getString(CGM.getLLVMContext(), FuncName, false);
- FuncNameVar =
- new llvm::GlobalVariable(CGM.getModule(), Value->getType(), true, Linkage,
- Value, "__llvm_profile_name_" + FuncName);
-
- // Hide the symbol so that we correctly get a copy for each executable.
- if (!llvm::GlobalValue::isLocalLinkage(FuncNameVar->getLinkage()))
- FuncNameVar->setVisibility(llvm::GlobalValue::HiddenVisibility);
-}
-
namespace {
/// \brief Stable hasher for PGO region counters.
///
@@ -275,10 +237,9 @@
void VisitStmt(const Stmt *S) {
RecordStmtCount(S);
- for (Stmt::const_child_range I = S->children(); I; ++I) {
- if (*I)
- this->Visit(*I);
- }
+ for (const Stmt *Child : S->children())
+ if (Child)
+ this->Visit(Child);
}
void VisitFunctionDecl(const FunctionDecl *D) {
@@ -605,7 +566,7 @@
RecordNextStmtCount = true;
}
};
-}
+} // end anonymous namespace
void PGOHash::combine(HashType Type) {
// Check that we never combine 0 and only have six bits.
@@ -644,27 +605,24 @@
return endian::read<uint64_t, little, unaligned>(Result);
}
-void CodeGenPGO::checkGlobalDecl(GlobalDecl GD) {
- // Make sure we only emit coverage mapping for one constructor/destructor.
- // Clang emits several functions for the constructor and the destructor of
- // a class. Every function is instrumented, but we only want to provide
- // coverage for one of them. Because of that we only emit the coverage mapping
- // for the base constructor/destructor.
- if ((isa<CXXConstructorDecl>(GD.getDecl()) &&
- GD.getCtorType() != Ctor_Base) ||
- (isa<CXXDestructorDecl>(GD.getDecl()) &&
- GD.getDtorType() != Dtor_Base)) {
- SkipCoverageMapping = true;
- }
-}
-
-void CodeGenPGO::assignRegionCounters(const Decl *D, llvm::Function *Fn) {
+void CodeGenPGO::assignRegionCounters(GlobalDecl GD, llvm::Function *Fn) {
+ const Decl *D = GD.getDecl();
bool InstrumentRegions = CGM.getCodeGenOpts().ProfileInstrGenerate;
llvm::IndexedInstrProfReader *PGOReader = CGM.getPGOReader();
if (!InstrumentRegions && !PGOReader)
return;
if (D->isImplicit())
return;
+ // Constructors and destructors may be represented by several functions in IR.
+ // If so, instrument only base variant, others are implemented by delegation
+ // to the base one, it would be counted twice otherwise.
+ if (CGM.getTarget().getCXXABI().hasConstructorVariants() &&
+ ((isa<CXXConstructorDecl>(GD.getDecl()) &&
+ GD.getCtorType() != Ctor_Base) ||
+ (isa<CXXDestructorDecl>(GD.getDecl()) &&
+ GD.getDtorType() != Dtor_Base))) {
+ return;
+ }
CGM.ClearUnusedCoverageMapping(D);
setFuncName(Fn);
@@ -764,7 +722,7 @@
return;
uint64_t MaxFunctionCount = PGOReader->getMaximumFunctionCount();
- uint64_t FunctionCount = getRegionCount(0);
+ uint64_t FunctionCount = getRegionCount(nullptr);
if (FunctionCount >= (uint64_t)(0.3 * (double)MaxFunctionCount))
// Turn on InlineHint attribute for hot functions.
// FIXME: 30% is from preliminary tuning on SPEC, it may not be optimal.
@@ -780,7 +738,7 @@
void CodeGenPGO::emitCounterIncrement(CGBuilderTy &Builder, const Stmt *S) {
if (!CGM.getCodeGenOpts().ProfileInstrGenerate || !RegionCounterMap)
return;
- if (!Builder.GetInsertPoint())
+ if (!Builder.GetInsertBlock())
return;
unsigned Counter = (*RegionCounterMap)[S];
diff --git a/lib/CodeGen/CodeGenPGO.h b/lib/CodeGen/CodeGenPGO.h
index de6f369..6bf29ec 100644
--- a/lib/CodeGen/CodeGenPGO.h
+++ b/lib/CodeGen/CodeGenPGO.h
@@ -78,13 +78,11 @@
setCurrentRegionCount(*Count);
}
- /// Check if we need to emit coverage mapping for a given declaration
- void checkGlobalDecl(GlobalDecl GD);
/// Assign counters to regions and configure them for PGO of a given
/// function. Does nothing if instrumentation is not enabled and either
/// generates global variables or associates PGO data with each of the
/// counters depending on whether we are generating or using instrumentation.
- void assignRegionCounters(const Decl *D, llvm::Function *Fn);
+ void assignRegionCounters(GlobalDecl GD, llvm::Function *Fn);
/// Emit a coverage mapping range with a counter zero
/// for an unused declaration.
void emitEmptyCounterMapping(const Decl *D, StringRef FuncName,
@@ -92,14 +90,12 @@
private:
void setFuncName(llvm::Function *Fn);
void setFuncName(StringRef Name, llvm::GlobalValue::LinkageTypes Linkage);
- void createFuncNameVar(llvm::GlobalValue::LinkageTypes Linkage);
void mapRegionCounters(const Decl *D);
void computeRegionCounts(const Decl *D);
void applyFunctionAttributes(llvm::IndexedInstrProfReader *PGOReader,
llvm::Function *Fn);
void loadRegionCounts(llvm::IndexedInstrProfReader *PGOReader,
bool IsInMainFile);
- void emitCounterVariables();
void emitCounterRegionMapping(const Decl *D);
public:
diff --git a/lib/CodeGen/CodeGenTBAA.cpp b/lib/CodeGen/CodeGenTBAA.cpp
index 53ba02a..c3c925c 100644
--- a/lib/CodeGen/CodeGenTBAA.cpp
+++ b/lib/CodeGen/CodeGenTBAA.cpp
@@ -155,7 +155,6 @@
SmallString<256> OutName;
llvm::raw_svector_ostream Out(OutName);
MContext.mangleTypeName(QualType(ETy, 0), Out);
- Out.flush();
return MetadataCache[Ty] = createTBAAScalarType(OutName, getChar());
}
@@ -271,7 +270,6 @@
// Don't use the mangler for C code.
llvm::raw_svector_ostream Out(OutName);
MContext.mangleTypeName(QualType(Ty, 0), Out);
- Out.flush();
} else {
OutName = RD->getName();
}
diff --git a/lib/CodeGen/CodeGenTypeCache.h b/lib/CodeGen/CodeGenTypeCache.h
new file mode 100644
index 0000000..c32b66d
--- /dev/null
+++ b/lib/CodeGen/CodeGenTypeCache.h
@@ -0,0 +1,108 @@
+//===--- CodeGenTypeCache.h - Commonly used LLVM types and info -*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This structure provides a set of common types useful during IR emission.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_CODEGEN_CODEGENTYPECACHE_H
+#define LLVM_CLANG_LIB_CODEGEN_CODEGENTYPECACHE_H
+
+#include "clang/AST/CharUnits.h"
+#include "llvm/IR/CallingConv.h"
+
+namespace llvm {
+ class Type;
+ class IntegerType;
+ class PointerType;
+}
+
+namespace clang {
+namespace CodeGen {
+
+/// This structure provides a set of types that are commonly used
+/// during IR emission. It's initialized once in CodeGenModule's
+/// constructor and then copied around into new CodeGenFunctions.
+struct CodeGenTypeCache {
+ /// void
+ llvm::Type *VoidTy;
+
+ /// i8, i16, i32, and i64
+ llvm::IntegerType *Int8Ty, *Int16Ty, *Int32Ty, *Int64Ty;
+ /// float, double
+ llvm::Type *FloatTy, *DoubleTy;
+
+ /// int
+ llvm::IntegerType *IntTy;
+
+ /// intptr_t, size_t, and ptrdiff_t, which we assume are the same size.
+ union {
+ llvm::IntegerType *IntPtrTy;
+ llvm::IntegerType *SizeTy;
+ llvm::IntegerType *PtrDiffTy;
+ };
+
+ /// void* in address space 0
+ union {
+ llvm::PointerType *VoidPtrTy;
+ llvm::PointerType *Int8PtrTy;
+ };
+
+ /// void** in address space 0
+ union {
+ llvm::PointerType *VoidPtrPtrTy;
+ llvm::PointerType *Int8PtrPtrTy;
+ };
+
+ /// The size and alignment of the builtin C type 'int'. This comes
+ /// up enough in various ABI lowering tasks to be worth pre-computing.
+ union {
+ unsigned char IntSizeInBytes;
+ unsigned char IntAlignInBytes;
+ };
+ CharUnits getIntSize() const {
+ return CharUnits::fromQuantity(IntSizeInBytes);
+ }
+ CharUnits getIntAlign() const {
+ return CharUnits::fromQuantity(IntAlignInBytes);
+ }
+
+ /// The width of a pointer into the generic address space.
+ unsigned char PointerWidthInBits;
+
+ /// The size and alignment of a pointer into the generic address space.
+ union {
+ unsigned char PointerAlignInBytes;
+ unsigned char PointerSizeInBytes;
+ unsigned char SizeSizeInBytes; // sizeof(size_t)
+ unsigned char SizeAlignInBytes;
+ };
+ CharUnits getSizeSize() const {
+ return CharUnits::fromQuantity(SizeSizeInBytes);
+ }
+ CharUnits getSizeAlign() const {
+ return CharUnits::fromQuantity(SizeAlignInBytes);
+ }
+ CharUnits getPointerSize() const {
+ return CharUnits::fromQuantity(PointerSizeInBytes);
+ }
+ CharUnits getPointerAlign() const {
+ return CharUnits::fromQuantity(PointerAlignInBytes);
+ }
+
+ llvm::CallingConv::ID RuntimeCC;
+ llvm::CallingConv::ID getRuntimeCC() const { return RuntimeCC; }
+ llvm::CallingConv::ID BuiltinCC;
+ llvm::CallingConv::ID getBuiltinCC() const { return BuiltinCC; }
+};
+
+} // end namespace CodeGen
+} // end namespace clang
+
+#endif
diff --git a/lib/CodeGen/CodeGenTypes.cpp b/lib/CodeGen/CodeGenTypes.cpp
index e0f926c..fcda053 100644
--- a/lib/CodeGen/CodeGenTypes.cpp
+++ b/lib/CodeGen/CodeGenTypes.cpp
@@ -31,7 +31,6 @@
CodeGenTypes::CodeGenTypes(CodeGenModule &cgm)
: CGM(cgm), Context(cgm.getContext()), TheModule(cgm.getModule()),
- TheDataLayout(cgm.getDataLayout()),
Target(cgm.getTarget()), TheCXXABI(cgm.getCXXABI()),
TheABIInfo(cgm.getTargetCodeGenInfo().getABIInfo()) {
SkippedLayout = false;
@@ -154,14 +153,16 @@
static bool
isSafeToConvert(QualType T, CodeGenTypes &CGT,
llvm::SmallPtrSet<const RecordDecl*, 16> &AlreadyChecked) {
- T = T.getCanonicalType();
-
+ // Strip off atomic type sugar.
+ if (const auto *AT = T->getAs<AtomicType>())
+ T = AT->getValueType();
+
// If this is a record, check it.
- if (const RecordType *RT = dyn_cast<RecordType>(T))
+ if (const auto *RT = T->getAs<RecordType>())
return isSafeToConvert(RT->getDecl(), CGT, AlreadyChecked);
-
+
// If this is an array, check the elements, which are embedded inline.
- if (const ArrayType *AT = dyn_cast<ArrayType>(T))
+ if (const auto *AT = CGT.getContext().getAsArrayType(T))
return isSafeToConvert(AT->getElementType(), CGT, AlreadyChecked);
// Otherwise, there is no concern about transforming this. We only care about
@@ -293,6 +294,76 @@
llvm_unreachable("Unknown float format!");
}
+llvm::Type *CodeGenTypes::ConvertFunctionType(QualType QFT,
+ const FunctionDecl *FD) {
+ assert(QFT.isCanonical());
+ const Type *Ty = QFT.getTypePtr();
+ const FunctionType *FT = cast<FunctionType>(QFT.getTypePtr());
+ // First, check whether we can build the full function type. If the
+ // function type depends on an incomplete type (e.g. a struct or enum), we
+ // cannot lower the function type.
+ if (!isFuncTypeConvertible(FT)) {
+ // This function's type depends on an incomplete tag type.
+
+ // Force conversion of all the relevant record types, to make sure
+ // we re-convert the FunctionType when appropriate.
+ if (const RecordType *RT = FT->getReturnType()->getAs<RecordType>())
+ ConvertRecordDeclType(RT->getDecl());
+ if (const FunctionProtoType *FPT = dyn_cast<FunctionProtoType>(FT))
+ for (unsigned i = 0, e = FPT->getNumParams(); i != e; i++)
+ if (const RecordType *RT = FPT->getParamType(i)->getAs<RecordType>())
+ ConvertRecordDeclType(RT->getDecl());
+
+ SkippedLayout = true;
+
+ // Return a placeholder type.
+ return llvm::StructType::get(getLLVMContext());
+ }
+
+ // While we're converting the parameter types for a function, we don't want
+ // to recursively convert any pointed-to structs. Converting directly-used
+ // structs is ok though.
+ if (!RecordsBeingLaidOut.insert(Ty).second) {
+ SkippedLayout = true;
+ return llvm::StructType::get(getLLVMContext());
+ }
+
+ // The function type can be built; call the appropriate routines to
+ // build it.
+ const CGFunctionInfo *FI;
+ if (const FunctionProtoType *FPT = dyn_cast<FunctionProtoType>(FT)) {
+ FI = &arrangeFreeFunctionType(
+ CanQual<FunctionProtoType>::CreateUnsafe(QualType(FPT, 0)), FD);
+ } else {
+ const FunctionNoProtoType *FNPT = cast<FunctionNoProtoType>(FT);
+ FI = &arrangeFreeFunctionType(
+ CanQual<FunctionNoProtoType>::CreateUnsafe(QualType(FNPT, 0)));
+ }
+
+ llvm::Type *ResultType = nullptr;
+ // If there is something higher level prodding our CGFunctionInfo, then
+ // don't recurse into it again.
+ if (FunctionsBeingProcessed.count(FI)) {
+
+ ResultType = llvm::StructType::get(getLLVMContext());
+ SkippedLayout = true;
+ } else {
+
+ // Otherwise, we're good to go, go ahead and convert it.
+ ResultType = GetFunctionType(*FI);
+ }
+
+ RecordsBeingLaidOut.erase(Ty);
+
+ if (SkippedLayout)
+ TypeCache.clear();
+
+ if (RecordsBeingLaidOut.empty())
+ while (!DeferredRecords.empty())
+ ConvertRecordDeclType(DeferredRecords.pop_back_val());
+ return ResultType;
+}
+
/// ConvertType - Convert the specified type to its LLVM form.
llvm::Type *CodeGenTypes::ConvertType(QualType T) {
T = Context.getCanonicalType(T);
@@ -387,9 +458,19 @@
case BuiltinType::OCLImage1dBuffer:
case BuiltinType::OCLImage2d:
case BuiltinType::OCLImage2dArray:
+ case BuiltinType::OCLImage2dDepth:
+ case BuiltinType::OCLImage2dArrayDepth:
+ case BuiltinType::OCLImage2dMSAA:
+ case BuiltinType::OCLImage2dArrayMSAA:
+ case BuiltinType::OCLImage2dMSAADepth:
+ case BuiltinType::OCLImage2dArrayMSAADepth:
case BuiltinType::OCLImage3d:
case BuiltinType::OCLSampler:
case BuiltinType::OCLEvent:
+ case BuiltinType::OCLClkEvent:
+ case BuiltinType::OCLQueue:
+ case BuiltinType::OCLNDRange:
+ case BuiltinType::OCLReserveID:
ResultType = CGM.getOpenCLRuntime().convertOpenCLSpecificType(Ty);
break;
@@ -474,75 +555,9 @@
break;
}
case Type::FunctionNoProto:
- case Type::FunctionProto: {
- const FunctionType *FT = cast<FunctionType>(Ty);
- // First, check whether we can build the full function type. If the
- // function type depends on an incomplete type (e.g. a struct or enum), we
- // cannot lower the function type.
- if (!isFuncTypeConvertible(FT)) {
- // This function's type depends on an incomplete tag type.
-
- // Force conversion of all the relevant record types, to make sure
- // we re-convert the FunctionType when appropriate.
- if (const RecordType *RT = FT->getReturnType()->getAs<RecordType>())
- ConvertRecordDeclType(RT->getDecl());
- if (const FunctionProtoType *FPT = dyn_cast<FunctionProtoType>(FT))
- for (unsigned i = 0, e = FPT->getNumParams(); i != e; i++)
- if (const RecordType *RT = FPT->getParamType(i)->getAs<RecordType>())
- ConvertRecordDeclType(RT->getDecl());
-
- // Return a placeholder type.
- ResultType = llvm::StructType::get(getLLVMContext());
-
- SkippedLayout = true;
- break;
- }
-
- // While we're converting the parameter types for a function, we don't want
- // to recursively convert any pointed-to structs. Converting directly-used
- // structs is ok though.
- if (!RecordsBeingLaidOut.insert(Ty).second) {
- ResultType = llvm::StructType::get(getLLVMContext());
-
- SkippedLayout = true;
- break;
- }
-
- // The function type can be built; call the appropriate routines to
- // build it.
- const CGFunctionInfo *FI;
- if (const FunctionProtoType *FPT = dyn_cast<FunctionProtoType>(FT)) {
- FI = &arrangeFreeFunctionType(
- CanQual<FunctionProtoType>::CreateUnsafe(QualType(FPT, 0)));
- } else {
- const FunctionNoProtoType *FNPT = cast<FunctionNoProtoType>(FT);
- FI = &arrangeFreeFunctionType(
- CanQual<FunctionNoProtoType>::CreateUnsafe(QualType(FNPT, 0)));
- }
-
- // If there is something higher level prodding our CGFunctionInfo, then
- // don't recurse into it again.
- if (FunctionsBeingProcessed.count(FI)) {
-
- ResultType = llvm::StructType::get(getLLVMContext());
- SkippedLayout = true;
- } else {
-
- // Otherwise, we're good to go, go ahead and convert it.
- ResultType = GetFunctionType(*FI);
- }
-
- RecordsBeingLaidOut.erase(Ty);
-
- if (SkippedLayout)
- TypeCache.clear();
-
- if (RecordsBeingLaidOut.empty())
- while (!DeferredRecords.empty())
- ConvertRecordDeclType(DeferredRecords.pop_back_val());
+ case Type::FunctionProto:
+ ResultType = ConvertFunctionType(T);
break;
- }
-
case Type::ObjCObject:
ResultType = ConvertType(cast<ObjCObjectType>(Ty)->getBaseType());
break;
diff --git a/lib/CodeGen/CodeGenTypes.h b/lib/CodeGen/CodeGenTypes.h
index 1580e21..a96f23c 100644
--- a/lib/CodeGen/CodeGenTypes.h
+++ b/lib/CodeGen/CodeGenTypes.h
@@ -122,7 +122,6 @@
// Some of this stuff should probably be left on the CGM.
ASTContext &Context;
llvm::Module &TheModule;
- const llvm::DataLayout &TheDataLayout;
const TargetInfo &Target;
CGCXXABI &TheCXXABI;
@@ -159,7 +158,6 @@
SmallVector<const RecordDecl *, 8> DeferredRecords;
-private:
/// This map keeps cache of llvm::Types and maps clang::Type to
/// corresponding llvm::Type.
llvm::DenseMap<const Type *, llvm::Type *> TypeCache;
@@ -168,7 +166,9 @@
CodeGenTypes(CodeGenModule &cgm);
~CodeGenTypes();
- const llvm::DataLayout &getDataLayout() const { return TheDataLayout; }
+ const llvm::DataLayout &getDataLayout() const {
+ return TheModule.getDataLayout();
+ }
ASTContext &getContext() const { return Context; }
const ABIInfo &getABIInfo() const { return TheABIInfo; }
const TargetInfo &getTarget() const { return Target; }
@@ -178,6 +178,14 @@
/// ConvertType - Convert type T into a llvm::Type.
llvm::Type *ConvertType(QualType T);
+ /// \brief Converts the GlobalDecl into an llvm::Type. This should be used
+ /// when we know the target of the function we want to convert. This is
+ /// because some functions (explicitly, those with pass_object_size
+ /// parameters) may not have the same signature as their type portrays, and
+ /// can only be called directly.
+ llvm::Type *ConvertFunctionType(QualType FT,
+ const FunctionDecl *FD = nullptr);
+
/// ConvertTypeForMem - Convert type T into a llvm::Type. This differs from
/// ConvertType in that it is used to convert to the memory representation for
/// a type. For example, the scalar representation for _Bool is i1, but the
@@ -264,11 +272,12 @@
const CGFunctionInfo &arrangeMSMemberPointerThunk(const CXXMethodDecl *MD);
const CGFunctionInfo &arrangeMSCtorClosure(const CXXConstructorDecl *CD,
CXXCtorType CT);
-
- const CGFunctionInfo &arrangeFreeFunctionType(CanQual<FunctionProtoType> Ty);
+ const CGFunctionInfo &arrangeFreeFunctionType(CanQual<FunctionProtoType> Ty,
+ const FunctionDecl *FD);
const CGFunctionInfo &arrangeFreeFunctionType(CanQual<FunctionNoProtoType> Ty);
const CGFunctionInfo &arrangeCXXMethodType(const CXXRecordDecl *RD,
- const FunctionProtoType *FTP);
+ const FunctionProtoType *FTP,
+ const CXXMethodDecl *MD);
/// "Arrange" the LLVM information for a call or type with the given
/// signature. This is largely an internal method; other clients
diff --git a/lib/CodeGen/CoverageMappingGen.cpp b/lib/CodeGen/CoverageMappingGen.cpp
index 024a45d..2486a49 100644
--- a/lib/CodeGen/CoverageMappingGen.cpp
+++ b/lib/CodeGen/CoverageMappingGen.cpp
@@ -47,17 +47,6 @@
Optional<SourceLocation> LocEnd)
: Count(Count), LocStart(LocStart), LocEnd(LocEnd) {}
- SourceMappingRegion(SourceMappingRegion &&Region)
- : Count(std::move(Region.Count)), LocStart(std::move(Region.LocStart)),
- LocEnd(std::move(Region.LocEnd)) {}
-
- SourceMappingRegion &operator=(SourceMappingRegion &&RHS) {
- Count = std::move(RHS.Count);
- LocStart = std::move(RHS.LocStart);
- LocEnd = std::move(RHS.LocEnd);
- return *this;
- }
-
const Counter &getCounter() const { return Count; }
void setCounter(Counter C) { Count = C; }
@@ -66,7 +55,7 @@
void setStartLoc(SourceLocation Loc) { LocStart = Loc; }
- const SourceLocation &getStartLoc() const {
+ SourceLocation getStartLoc() const {
assert(LocStart && "Region has no start location");
return *LocStart;
}
@@ -75,7 +64,7 @@
void setEndLoc(SourceLocation Loc) { LocEnd = Loc; }
- const SourceLocation &getEndLoc() const {
+ SourceLocation getEndLoc() const {
assert(LocEnd && "Region has no end location");
return *LocEnd;
}
@@ -174,7 +163,7 @@
unsigned Depth = 0;
for (SourceLocation Parent = getIncludeOrExpansionLoc(Loc);
- !Parent.isInvalid(); Parent = getIncludeOrExpansionLoc(Parent))
+ Parent.isValid(); Parent = getIncludeOrExpansionLoc(Parent))
++Depth;
FileLocs.push_back(std::make_pair(Loc, Depth));
}
@@ -255,7 +244,7 @@
assert(Region.hasEndLoc() && "incomplete region");
SourceLocation LocStart = Region.getStartLoc();
- assert(!SM.getFileID(LocStart).isInvalid() && "region in invalid file");
+ assert(SM.getFileID(LocStart).isValid() && "region in invalid file");
auto CovFileID = getCoverageFileID(LocStart);
// Ignore regions that don't have a file, such as builtin macros.
@@ -413,8 +402,8 @@
SourceRegions.emplace_back(Region.getCounter(), NestedLoc, EndLoc);
EndLoc = getPreciseTokenLocEnd(getIncludeOrExpansionLoc(EndLoc));
- assert(!EndLoc.isInvalid() &&
- "File exit was not handled before popRegions");
+ if (EndLoc.isInvalid())
+ llvm::report_fatal_error("File exit not handled before popRegions");
}
Region.setEndLoc(EndLoc);
@@ -426,7 +415,7 @@
MostRecentLocation = getIncludeOrExpansionLoc(EndLoc);
assert(SM.isWrittenInSameFile(Region.getStartLoc(), EndLoc));
- SourceRegions.push_back(std::move(Region));
+ SourceRegions.push_back(Region);
}
RegionStack.pop_back();
}
@@ -475,7 +464,8 @@
/// files, this adjusts our current region stack and creates the file regions
/// for the exited file.
void handleFileExit(SourceLocation NewLoc) {
- if (SM.isWrittenInSameFile(MostRecentLocation, NewLoc))
+ if (NewLoc.isInvalid() ||
+ SM.isWrittenInSameFile(MostRecentLocation, NewLoc))
return;
// If NewLoc is not in a file that contains MostRecentLocation, walk up to
@@ -495,12 +485,12 @@
llvm::SmallSet<SourceLocation, 8> StartLocs;
Optional<Counter> ParentCounter;
- for (auto I = RegionStack.rbegin(), E = RegionStack.rend(); I != E; ++I) {
- if (!I->hasStartLoc())
+ for (SourceMappingRegion &I : llvm::reverse(RegionStack)) {
+ if (!I.hasStartLoc())
continue;
- SourceLocation Loc = I->getStartLoc();
+ SourceLocation Loc = I.getStartLoc();
if (!isNestedIn(Loc, ParentFile)) {
- ParentCounter = I->getCounter();
+ ParentCounter = I.getCounter();
break;
}
@@ -509,11 +499,11 @@
// correct count. We avoid creating redundant regions by stopping once
// we've seen this region.
if (StartLocs.insert(Loc).second)
- SourceRegions.emplace_back(I->getCounter(), Loc,
+ SourceRegions.emplace_back(I.getCounter(), Loc,
getEndOfFileOrMacro(Loc));
Loc = getIncludeOrExpansionLoc(Loc);
}
- I->setStartLoc(getPreciseTokenLocEnd(Loc));
+ I.setStartLoc(getPreciseTokenLocEnd(Loc));
}
if (ParentCounter) {
@@ -579,12 +569,11 @@
}
void VisitStmt(const Stmt *S) {
- if (!S->getLocStart().isInvalid())
+ if (S->getLocStart().isValid())
extendRegion(S);
- for (Stmt::const_child_range I = S->children(); I; ++I) {
- if (*I)
- this->Visit(*I);
- }
+ for (const Stmt *Child : S->children())
+ if (Child)
+ this->Visit(Child);
handleFileExit(getEnd(S));
}
@@ -806,6 +795,9 @@
void VisitIfStmt(const IfStmt *S) {
extendRegion(S);
+ // Extend into the condition before we propagate through it below - this is
+ // needed to handle macros that generate the "if" but not the condition.
+ extendRegion(S->getCond());
Counter ParentCount = getRegion().getCounter();
Counter ThenCount = getRegionCounter(S);
@@ -839,7 +831,6 @@
}
void VisitCXXCatchStmt(const CXXCatchStmt *S) {
- extendRegion(S);
propagateCounts(getRegionCounter(S), S->getHandlerBlock());
}
@@ -888,7 +879,7 @@
}
static StringRef getCoverageSection(const CodeGenModule &CGM) {
- return isMachO(CGM) ? "__DATA,__llvm_covmap" : "__llvm_covmap";
+ return llvm::getInstrProfCoverageSectionName(isMachO(CGM));
}
static void dump(llvm::raw_ostream &OS, StringRef FunctionName,
@@ -919,23 +910,23 @@
}
void CoverageMappingModuleGen::addFunctionMappingRecord(
- llvm::GlobalVariable *FunctionName, StringRef FunctionNameValue,
- uint64_t FunctionHash, const std::string &CoverageMapping) {
+ llvm::GlobalVariable *NamePtr, StringRef NameValue,
+ uint64_t FuncHash, const std::string &CoverageMapping) {
llvm::LLVMContext &Ctx = CGM.getLLVMContext();
- auto *Int32Ty = llvm::Type::getInt32Ty(Ctx);
- auto *Int64Ty = llvm::Type::getInt64Ty(Ctx);
- auto *Int8PtrTy = llvm::Type::getInt8PtrTy(Ctx);
if (!FunctionRecordTy) {
- llvm::Type *FunctionRecordTypes[] = {Int8PtrTy, Int32Ty, Int32Ty, Int64Ty};
+ #define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Init) LLVMType,
+ llvm::Type *FunctionRecordTypes[] = {
+ #include "llvm/ProfileData/InstrProfData.inc"
+ };
FunctionRecordTy =
- llvm::StructType::get(Ctx, makeArrayRef(FunctionRecordTypes));
+ llvm::StructType::get(Ctx, makeArrayRef(FunctionRecordTypes),
+ /*isPacked=*/true);
}
+ #define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Init) Init,
llvm::Constant *FunctionRecordVals[] = {
- llvm::ConstantExpr::getBitCast(FunctionName, Int8PtrTy),
- llvm::ConstantInt::get(Int32Ty, FunctionNameValue.size()),
- llvm::ConstantInt::get(Int32Ty, CoverageMapping.size()),
- llvm::ConstantInt::get(Int64Ty, FunctionHash)};
+ #include "llvm/ProfileData/InstrProfData.inc"
+ };
FunctionRecords.push_back(llvm::ConstantStruct::get(
FunctionRecordTy, makeArrayRef(FunctionRecordVals)));
CoverageMappings += CoverageMapping;
@@ -957,7 +948,7 @@
Expressions, Regions);
if (Reader.read())
return;
- dump(llvm::outs(), FunctionNameValue, Expressions, Regions);
+ dump(llvm::outs(), NameValue, Expressions, Regions);
}
}
@@ -1019,7 +1010,7 @@
auto CovData = new llvm::GlobalVariable(CGM.getModule(), CovDataTy, true,
llvm::GlobalValue::InternalLinkage,
CovDataVal,
- "__llvm_coverage_mapping");
+ llvm::getCoverageMappingVarName());
CovData->setSection(getCoverageSection(CGM));
CovData->setAlignment(8);
diff --git a/lib/CodeGen/EHScopeStack.h b/lib/CodeGen/EHScopeStack.h
index a795188..85cd154 100644
--- a/lib/CodeGen/EHScopeStack.h
+++ b/lib/CodeGen/EHScopeStack.h
@@ -96,6 +96,9 @@
/// and catch blocks.
class EHScopeStack {
public:
+ /* Should switch to alignof(uint64_t) instead of 8, when EHCleanupScope can */
+ enum { ScopeStackAlignment = 8 };
+
/// A saved depth on the scope stack. This is necessary because
/// pushing scopes onto the stack invalidates iterators.
class stable_iterator {
@@ -141,7 +144,15 @@
class Cleanup {
// Anchor the construction vtable.
virtual void anchor();
+
+ protected:
+ ~Cleanup() = default;
+
public:
+ Cleanup(const Cleanup &) = default;
+ Cleanup(Cleanup &&) {}
+ Cleanup() = default;
+
/// Generation flags.
class Flags {
enum {
@@ -168,10 +179,6 @@
void setIsEHCleanupKind() { flags |= F_IsEHCleanupKind; }
};
- // Provide a virtual destructor to suppress a very common warning
- // that unfortunately cannot be suppressed without this. Cleanups
- // should not rely on this destructor ever being called.
- virtual ~Cleanup() {}
/// Emit the cleanup. For normal cleanups, this is run in the
/// same EH context as when the cleanup was pushed, i.e. the
@@ -184,7 +191,8 @@
/// ConditionalCleanup stores the saved form of its parameters,
/// then restores them and performs the cleanup.
- template <class T, class... As> class ConditionalCleanup : public Cleanup {
+ template <class T, class... As>
+ class ConditionalCleanup final : public Cleanup {
typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple;
SavedTuple Saved;
@@ -248,6 +256,7 @@
SmallVector<BranchFixup, 8> BranchFixups;
char *allocate(size_t Size);
+ void deallocate(size_t Size);
void *pushCleanup(CleanupKind K, size_t DataSize);
@@ -259,6 +268,8 @@
/// Push a lazily-created cleanup on the stack.
template <class T, class... As> void pushCleanup(CleanupKind Kind, As... A) {
+ static_assert(llvm::AlignOf<T>::Alignment <= ScopeStackAlignment,
+ "Cleanup's alignment is too large.");
void *Buffer = pushCleanup(Kind, sizeof(T));
Cleanup *Obj = new (Buffer) T(A...);
(void) Obj;
@@ -267,6 +278,8 @@
/// Push a lazily-created cleanup on the stack. Tuple version.
template <class T, class... As>
void pushCleanupTuple(CleanupKind Kind, std::tuple<As...> A) {
+ static_assert(llvm::AlignOf<T>::Alignment <= ScopeStackAlignment,
+ "Cleanup's alignment is too large.");
void *Buffer = pushCleanup(Kind, sizeof(T));
Cleanup *Obj = new (Buffer) T(std::move(A));
(void) Obj;
@@ -287,6 +300,8 @@
/// stack is modified.
template <class T, class... As>
T *pushCleanupWithExtra(CleanupKind Kind, size_t N, As... A) {
+ static_assert(llvm::AlignOf<T>::Alignment <= ScopeStackAlignment,
+ "Cleanup's alignment is too large.");
void *Buffer = pushCleanup(Kind, sizeof(T) + T::getExtraSize(N));
return new (Buffer) T(N, A...);
}
@@ -346,7 +361,6 @@
return InnermostEHScope;
}
- stable_iterator getInnermostActiveEHScope() const;
/// An unstable reference to a scope-stack depth. Invalidated by
/// pushes but not pops.
@@ -377,9 +391,6 @@
/// to the EH stack.
iterator find(stable_iterator save) const;
- /// Removes the cleanup pointed to by the given stable_iterator.
- void removeCleanup(stable_iterator save);
-
/// Add a branch fixup to the current cleanup scope.
BranchFixup &addBranchFixup() {
assert(hasNormalCleanups() && "adding fixup in scope without cleanups");
diff --git a/lib/CodeGen/ItaniumCXXABI.cpp b/lib/CodeGen/ItaniumCXXABI.cpp
index 0a1a4ce..0c4008f 100644
--- a/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/lib/CodeGen/ItaniumCXXABI.cpp
@@ -69,6 +69,45 @@
return RAA_Default;
}
+ bool isThisCompleteObject(GlobalDecl GD) const override {
+ // The Itanium ABI has separate complete-object vs. base-object
+ // variants of both constructors and destructors.
+ if (isa<CXXDestructorDecl>(GD.getDecl())) {
+ switch (GD.getDtorType()) {
+ case Dtor_Complete:
+ case Dtor_Deleting:
+ return true;
+
+ case Dtor_Base:
+ return false;
+
+ case Dtor_Comdat:
+ llvm_unreachable("emitting dtor comdat as function?");
+ }
+ llvm_unreachable("bad dtor kind");
+ }
+ if (isa<CXXConstructorDecl>(GD.getDecl())) {
+ switch (GD.getCtorType()) {
+ case Ctor_Complete:
+ return true;
+
+ case Ctor_Base:
+ return false;
+
+ case Ctor_CopyingClosure:
+ case Ctor_DefaultClosure:
+ llvm_unreachable("closure ctors in Itanium ABI?");
+
+ case Ctor_Comdat:
+ llvm_unreachable("emitting ctor comdat as function?");
+ }
+ llvm_unreachable("bad dtor kind");
+ }
+
+ // No other kinds.
+ return false;
+ }
+
bool isZeroInitializable(const MemberPointerType *MPT) override;
llvm::Type *ConvertMemberPointerType(const MemberPointerType *MPT) override;
@@ -76,13 +115,14 @@
llvm::Value *
EmitLoadOfMemberFunctionPointer(CodeGenFunction &CGF,
const Expr *E,
- llvm::Value *&This,
+ Address This,
+ llvm::Value *&ThisPtrForCall,
llvm::Value *MemFnPtr,
const MemberPointerType *MPT) override;
llvm::Value *
EmitMemberDataPointerAddress(CodeGenFunction &CGF, const Expr *E,
- llvm::Value *Base,
+ Address Base,
llvm::Value *MemPtr,
const MemberPointerType *MPT) override;
@@ -94,7 +134,7 @@
llvm::Constant *EmitNullMemberPointer(const MemberPointerType *MPT) override;
- llvm::Constant *EmitMemberPointer(const CXXMethodDecl *MD) override;
+ llvm::Constant *EmitMemberFunctionPointer(const CXXMethodDecl *MD) override;
llvm::Constant *EmitMemberDataPointer(const MemberPointerType *MPT,
CharUnits offset) override;
llvm::Constant *EmitMemberPointer(const APValue &MP, QualType MPT) override;
@@ -111,9 +151,22 @@
const MemberPointerType *MPT) override;
void emitVirtualObjectDelete(CodeGenFunction &CGF, const CXXDeleteExpr *DE,
- llvm::Value *Ptr, QualType ElementType,
+ Address Ptr, QualType ElementType,
const CXXDestructorDecl *Dtor) override;
+ /// Itanium says that an _Unwind_Exception has to be "double-word"
+ /// aligned (and thus the end of it is also so-aligned), meaning 16
+ /// bytes. Of course, that was written for the actual Itanium,
+ /// which is a 64-bit platform. Classically, the ABI doesn't really
+ /// specify the alignment on other platforms, but in practice
+ /// libUnwind declares the struct with __attribute__((aligned)), so
+ /// we assume that alignment here. (It's generally 16 bytes, but
+ /// some targets overwrite it.)
+ CharUnits getAlignmentOfExnObject() {
+ auto align = CGM.getContext().getTargetDefaultAlignForAttributeAligned();
+ return CGM.getContext().toCharUnitsFromBits(align);
+ }
+
void emitRethrow(CodeGenFunction &CGF, bool isNoReturn) override;
void emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) override;
@@ -126,34 +179,34 @@
void EmitFundamentalRTTIDescriptor(QualType Type);
void EmitFundamentalRTTIDescriptors();
llvm::Constant *getAddrOfRTTIDescriptor(QualType Ty) override;
- llvm::Constant *
+ CatchTypeInfo
getAddrOfCXXCatchHandlerType(QualType Ty,
QualType CatchHandlerType) override {
- return getAddrOfRTTIDescriptor(Ty);
+ return CatchTypeInfo{getAddrOfRTTIDescriptor(Ty), 0};
}
bool shouldTypeidBeNullChecked(bool IsDeref, QualType SrcRecordTy) override;
void EmitBadTypeidCall(CodeGenFunction &CGF) override;
llvm::Value *EmitTypeid(CodeGenFunction &CGF, QualType SrcRecordTy,
- llvm::Value *ThisPtr,
+ Address ThisPtr,
llvm::Type *StdTypeInfoPtrTy) override;
bool shouldDynamicCastCallBeNullChecked(bool SrcIsPtr,
QualType SrcRecordTy) override;
- llvm::Value *EmitDynamicCastCall(CodeGenFunction &CGF, llvm::Value *Value,
+ llvm::Value *EmitDynamicCastCall(CodeGenFunction &CGF, Address Value,
QualType SrcRecordTy, QualType DestTy,
QualType DestRecordTy,
llvm::BasicBlock *CastEnd) override;
- llvm::Value *EmitDynamicCastToVoid(CodeGenFunction &CGF, llvm::Value *Value,
+ llvm::Value *EmitDynamicCastToVoid(CodeGenFunction &CGF, Address Value,
QualType SrcRecordTy,
QualType DestTy) override;
bool EmitBadCastCall(CodeGenFunction &CGF) override;
llvm::Value *
- GetVirtualBaseClassOffset(CodeGenFunction &CGF, llvm::Value *This,
+ GetVirtualBaseClassOffset(CodeGenFunction &CGF, Address This,
const CXXRecordDecl *ClassDecl,
const CXXRecordDecl *BaseClassDecl) override;
@@ -185,15 +238,29 @@
void EmitDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *DD,
CXXDtorType Type, bool ForVirtualBase,
- bool Delegating, llvm::Value *This) override;
+ bool Delegating, Address This) override;
void emitVTableDefinitions(CodeGenVTables &CGVT,
const CXXRecordDecl *RD) override;
+ bool isVirtualOffsetNeededForVTableField(CodeGenFunction &CGF,
+ CodeGenFunction::VPtr Vptr) override;
+
+ bool doStructorsInitializeVPtrs(const CXXRecordDecl *VTableClass) override {
+ return true;
+ }
+
+ llvm::Constant *
+ getVTableAddressPoint(BaseSubobject Base,
+ const CXXRecordDecl *VTableClass) override;
+
llvm::Value *getVTableAddressPointInStructor(
CodeGenFunction &CGF, const CXXRecordDecl *VTableClass,
- BaseSubobject Base, const CXXRecordDecl *NearestVBase,
- bool &NeedsVirtualOffset) override;
+ BaseSubobject Base, const CXXRecordDecl *NearestVBase) override;
+
+ llvm::Value *getVTableAddressPointInStructorWithVTT(
+ CodeGenFunction &CGF, const CXXRecordDecl *VTableClass,
+ BaseSubobject Base, const CXXRecordDecl *NearestVBase);
llvm::Constant *
getVTableAddressPointForConstExpr(BaseSubobject Base,
@@ -203,29 +270,31 @@
CharUnits VPtrOffset) override;
llvm::Value *getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD,
- llvm::Value *This,
- llvm::Type *Ty) override;
+ Address This, llvm::Type *Ty,
+ SourceLocation Loc) override;
llvm::Value *EmitVirtualDestructorCall(CodeGenFunction &CGF,
const CXXDestructorDecl *Dtor,
CXXDtorType DtorType,
- llvm::Value *This,
+ Address This,
const CXXMemberCallExpr *CE) override;
void emitVirtualInheritanceTables(const CXXRecordDecl *RD) override;
+ bool canSpeculativelyEmitVTable(const CXXRecordDecl *RD) const override;
+
void setThunkLinkage(llvm::Function *Thunk, bool ForVTable, GlobalDecl GD,
bool ReturnAdjustment) override {
// Allow inlining of thunks by emitting them with available_externally
// linkage together with vtables when needed.
- if (ForVTable)
+ if (ForVTable && !Thunk->hasLocalLinkage())
Thunk->setLinkage(llvm::GlobalValue::AvailableExternallyLinkage);
}
- llvm::Value *performThisAdjustment(CodeGenFunction &CGF, llvm::Value *This,
+ llvm::Value *performThisAdjustment(CodeGenFunction &CGF, Address This,
const ThisAdjustment &TA) override;
- llvm::Value *performReturnAdjustment(CodeGenFunction &CGF, llvm::Value *Ret,
+ llvm::Value *performReturnAdjustment(CodeGenFunction &CGF, Address Ret,
const ReturnAdjustment &RA) override;
size_t getSrcArgforCopyCtor(const CXXConstructorDecl *,
@@ -239,13 +308,13 @@
{ return "__cxa_deleted_virtual"; }
CharUnits getArrayCookieSizeImpl(QualType elementType) override;
- llvm::Value *InitializeArrayCookie(CodeGenFunction &CGF,
- llvm::Value *NewPtr,
- llvm::Value *NumElements,
- const CXXNewExpr *expr,
- QualType ElementType) override;
+ Address InitializeArrayCookie(CodeGenFunction &CGF,
+ Address NewPtr,
+ llvm::Value *NumElements,
+ const CXXNewExpr *expr,
+ QualType ElementType) override;
llvm::Value *readArrayCookieImpl(CodeGenFunction &CGF,
- llvm::Value *allocPtr,
+ Address allocPtr,
CharUnits cookieSize) override;
void EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D,
@@ -258,10 +327,9 @@
llvm::Value *Val);
void EmitThreadLocalInitFuncs(
CodeGenModule &CGM,
- ArrayRef<std::pair<const VarDecl *, llvm::GlobalVariable *>>
- CXXThreadLocals,
+ ArrayRef<const VarDecl *> CXXThreadLocals,
ArrayRef<llvm::Function *> CXXThreadLocalInits,
- ArrayRef<llvm::GlobalVariable *> CXXThreadLocalInitVars) override;
+ ArrayRef<const VarDecl *> CXXThreadLocalInitVars) override;
bool usesThreadWrapperFunction() const override { return true; }
LValue EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, const VarDecl *VD,
@@ -301,6 +369,41 @@
friend class ItaniumRTTIBuilder;
void emitCXXStructor(const CXXMethodDecl *MD, StructorType Type) override;
+
+ private:
+ bool hasAnyUsedVirtualInlineFunction(const CXXRecordDecl *RD) const {
+ const auto &VtableLayout =
+ CGM.getItaniumVTableContext().getVTableLayout(RD);
+
+ for (const auto &VtableComponent : VtableLayout.vtable_components()) {
+ if (!VtableComponent.isUsedFunctionPointerKind())
+ continue;
+
+ const CXXMethodDecl *Method = VtableComponent.getFunctionDecl();
+ if (Method->getCanonicalDecl()->isInlined())
+ return true;
+ }
+ return false;
+ }
+
+ bool isVTableHidden(const CXXRecordDecl *RD) const {
+ const auto &VtableLayout =
+ CGM.getItaniumVTableContext().getVTableLayout(RD);
+
+ for (const auto &VtableComponent : VtableLayout.vtable_components()) {
+ if (VtableComponent.isRTTIKind()) {
+ const CXXRecordDecl *RTTIDecl = VtableComponent.getRTTIDecl();
+ if (RTTIDecl->getVisibility() == Visibility::HiddenVisibility)
+ return true;
+ } else if (VtableComponent.isUsedFunctionPointerKind()) {
+ const CXXMethodDecl *Method = VtableComponent.getFunctionDecl();
+ if (Method->getVisibility() == Visibility::HiddenVisibility &&
+ !Method->isDefined())
+ return true;
+ }
+ }
+ return false;
+ }
};
class ARMCXXABI : public ItaniumCXXABI {
@@ -319,12 +422,12 @@
QualType ResTy) override;
CharUnits getArrayCookieSizeImpl(QualType elementType) override;
- llvm::Value *InitializeArrayCookie(CodeGenFunction &CGF,
- llvm::Value *NewPtr,
- llvm::Value *NumElements,
- const CXXNewExpr *expr,
- QualType ElementType) override;
- llvm::Value *readArrayCookieImpl(CodeGenFunction &CGF, llvm::Value *allocPtr,
+ Address InitializeArrayCookie(CodeGenFunction &CGF,
+ Address NewPtr,
+ llvm::Value *NumElements,
+ const CXXNewExpr *expr,
+ QualType ElementType) override;
+ llvm::Value *readArrayCookieImpl(CodeGenFunction &CGF, Address allocPtr,
CharUnits cookieSize) override;
};
@@ -335,6 +438,20 @@
// ARM64 libraries are prepared for non-unique RTTI.
bool shouldRTTIBeUnique() const override { return false; }
};
+
+class WebAssemblyCXXABI final : public ItaniumCXXABI {
+public:
+ explicit WebAssemblyCXXABI(CodeGen::CodeGenModule &CGM)
+ : ItaniumCXXABI(CGM, /*UseARMMethodPtrABI=*/true,
+ /*UseARMGuardVarABI=*/true) {}
+
+private:
+ bool HasThisReturn(GlobalDecl GD) const override {
+ return isa<CXXConstructorDecl>(GD.getDecl()) ||
+ (isa<CXXDestructorDecl>(GD.getDecl()) &&
+ GD.getDtorType() != Dtor_Deleting);
+ }
+};
}
CodeGen::CGCXXABI *CodeGen::CreateItaniumCXXABI(CodeGenModule &CGM) {
@@ -343,6 +460,7 @@
// between the ARM and iOS ABIs.
case TargetCXXABI::GenericARM:
case TargetCXXABI::iOS:
+ case TargetCXXABI::WatchOS:
return new ARMCXXABI(CGM);
case TargetCXXABI::iOS64:
@@ -358,6 +476,9 @@
case TargetCXXABI::GenericMIPS:
return new ItaniumCXXABI(CGM, /* UseARMMethodPtrABI = */ true);
+ case TargetCXXABI::WebAssembly:
+ return new WebAssemblyCXXABI(CGM);
+
case TargetCXXABI::GenericItanium:
if (CGM.getContext().getTargetInfo().getTriple().getArch()
== llvm::Triple::le32) {
@@ -403,7 +524,8 @@
/// If the member is non-virtual, memptr.ptr is the address of
/// the function to call.
llvm::Value *ItaniumCXXABI::EmitLoadOfMemberFunctionPointer(
- CodeGenFunction &CGF, const Expr *E, llvm::Value *&This,
+ CodeGenFunction &CGF, const Expr *E, Address ThisAddr,
+ llvm::Value *&ThisPtrForCall,
llvm::Value *MemFnPtr, const MemberPointerType *MPT) {
CGBuilderTy &Builder = CGF.Builder;
@@ -412,9 +534,8 @@
const CXXRecordDecl *RD =
cast<CXXRecordDecl>(MPT->getClass()->getAs<RecordType>()->getDecl());
- llvm::FunctionType *FTy =
- CGM.getTypes().GetFunctionType(
- CGM.getTypes().arrangeCXXMethodType(RD, FPT));
+ llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(
+ CGM.getTypes().arrangeCXXMethodType(RD, FPT, /*FD=*/nullptr));
llvm::Constant *ptrdiff_1 = llvm::ConstantInt::get(CGM.PtrDiffTy, 1);
@@ -432,9 +553,11 @@
// Apply the adjustment and cast back to the original struct type
// for consistency.
+ llvm::Value *This = ThisAddr.getPointer();
llvm::Value *Ptr = Builder.CreateBitCast(This, Builder.getInt8PtrTy());
Ptr = Builder.CreateInBoundsGEP(Ptr, Adj);
This = Builder.CreateBitCast(Ptr, This->getType(), "this.adjusted");
+ ThisPtrForCall = This;
// Load the function pointer.
llvm::Value *FnAsInt = Builder.CreateExtractValue(MemFnPtr, 0, "memptr.ptr");
@@ -456,7 +579,11 @@
// Cast the adjusted this to a pointer to vtable pointer and load.
llvm::Type *VTableTy = Builder.getInt8PtrTy();
- llvm::Value *VTable = CGF.GetVTablePtr(This, VTableTy);
+ CharUnits VTablePtrAlign =
+ CGF.CGM.getDynamicOffsetAlignment(ThisAddr.getAlignment(), RD,
+ CGF.getPointerAlign());
+ llvm::Value *VTable =
+ CGF.GetVTablePtr(Address(This, VTablePtrAlign), VTableTy, RD);
// Apply the offset.
llvm::Value *VTableOffset = FnAsInt;
@@ -466,7 +593,9 @@
// Load the virtual function to call.
VTable = Builder.CreateBitCast(VTable, FTy->getPointerTo()->getPointerTo());
- llvm::Value *VirtualFn = Builder.CreateLoad(VTable, "memptr.virtualfn");
+ llvm::Value *VirtualFn =
+ Builder.CreateAlignedLoad(VTable, CGF.getPointerAlign(),
+ "memptr.virtualfn");
CGF.EmitBranch(FnEnd);
// In the non-virtual path, the function pointer is actually a
@@ -486,24 +615,23 @@
/// Compute an l-value by applying the given pointer-to-member to a
/// base object.
llvm::Value *ItaniumCXXABI::EmitMemberDataPointerAddress(
- CodeGenFunction &CGF, const Expr *E, llvm::Value *Base, llvm::Value *MemPtr,
+ CodeGenFunction &CGF, const Expr *E, Address Base, llvm::Value *MemPtr,
const MemberPointerType *MPT) {
assert(MemPtr->getType() == CGM.PtrDiffTy);
CGBuilderTy &Builder = CGF.Builder;
- unsigned AS = Base->getType()->getPointerAddressSpace();
-
// Cast to char*.
- Base = Builder.CreateBitCast(Base, Builder.getInt8Ty()->getPointerTo(AS));
+ Base = Builder.CreateElementBitCast(Base, CGF.Int8Ty);
// Apply the offset, which we assume is non-null.
- llvm::Value *Addr = Builder.CreateInBoundsGEP(Base, MemPtr, "memptr.offset");
+ llvm::Value *Addr =
+ Builder.CreateInBoundsGEP(Base.getPointer(), MemPtr, "memptr.offset");
// Cast the address to the appropriate pointer type, adopting the
// address space of the base pointer.
- llvm::Type *PType
- = CGF.ConvertTypeForMem(MPT->getPointeeType())->getPointerTo(AS);
+ llvm::Type *PType = CGF.ConvertTypeForMem(MPT->getPointeeType())
+ ->getPointerTo(Base.getAddressSpace());
return Builder.CreateBitCast(Addr, PType);
}
@@ -655,7 +783,8 @@
return llvm::ConstantInt::get(CGM.PtrDiffTy, offset.getQuantity());
}
-llvm::Constant *ItaniumCXXABI::EmitMemberPointer(const CXXMethodDecl *MD) {
+llvm::Constant *
+ItaniumCXXABI::EmitMemberFunctionPointer(const CXXMethodDecl *MD) {
return BuildMemberPointer(MD, CharUnits::Zero());
}
@@ -856,7 +985,8 @@
// FIXME: Use canCopyArgument() when it is fixed to handle lazily declared
// special members.
if (RD->hasNonTrivialDestructor() || RD->hasNonTrivialCopyConstructor()) {
- FI.getReturnInfo() = ABIArgInfo::getIndirect(0, /*ByVal=*/false);
+ auto Align = CGM.getContext().getTypeAlignInChars(FI.getReturnType());
+ FI.getReturnInfo() = ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
return true;
}
return false;
@@ -872,7 +1002,7 @@
/// at entry -2 in the vtable.
void ItaniumCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF,
const CXXDeleteExpr *DE,
- llvm::Value *Ptr,
+ Address Ptr,
QualType ElementType,
const CXXDestructorDecl *Dtor) {
bool UseGlobalDelete = DE->isGlobalDelete();
@@ -881,16 +1011,20 @@
// to pass to the deallocation function.
// Grab the vtable pointer as an intptr_t*.
- llvm::Value *VTable = CGF.GetVTablePtr(Ptr, CGF.IntPtrTy->getPointerTo());
+ auto *ClassDecl =
+ cast<CXXRecordDecl>(ElementType->getAs<RecordType>()->getDecl());
+ llvm::Value *VTable =
+ CGF.GetVTablePtr(Ptr, CGF.IntPtrTy->getPointerTo(), ClassDecl);
// Track back to entry -2 and pull out the offset there.
llvm::Value *OffsetPtr = CGF.Builder.CreateConstInBoundsGEP1_64(
VTable, -2, "complete-offset.ptr");
- llvm::LoadInst *Offset = CGF.Builder.CreateLoad(OffsetPtr);
- Offset->setAlignment(CGF.PointerAlignInBytes);
+ llvm::Value *Offset =
+ CGF.Builder.CreateAlignedLoad(OffsetPtr, CGF.getPointerAlign());
// Apply the offset.
- llvm::Value *CompletePtr = CGF.Builder.CreateBitCast(Ptr, CGF.Int8PtrTy);
+ llvm::Value *CompletePtr =
+ CGF.Builder.CreateBitCast(Ptr.getPointer(), CGF.Int8PtrTy);
CompletePtr = CGF.Builder.CreateInBoundsGEP(CompletePtr, Offset);
// If we're supposed to call the global delete, make sure we do so
@@ -952,7 +1086,8 @@
llvm::CallInst *ExceptionPtr = CGF.EmitNounwindRuntimeCall(
AllocExceptionFn, llvm::ConstantInt::get(SizeTy, TypeSize), "exception");
- CGF.EmitAnyExprToExn(E->getSubExpr(), ExceptionPtr);
+ CharUnits ExnAlign = getAlignmentOfExnObject();
+ CGF.EmitAnyExprToExn(E->getSubExpr(), Address(ExceptionPtr, ExnAlign));
// Now throw the exception.
llvm::Constant *TypeInfo = CGM.GetAddrOfRTTIDescriptor(ThrowType,
@@ -1021,25 +1156,25 @@
CharUnits Offset;
// Now walk all possible inheritance paths.
- for (CXXBasePaths::paths_iterator I = Paths.begin(), E = Paths.end(); I != E;
- ++I) {
- if (I->Access != AS_public) // Ignore non-public inheritance.
+ for (const CXXBasePath &Path : Paths) {
+ if (Path.Access != AS_public) // Ignore non-public inheritance.
continue;
++NumPublicPaths;
- for (CXXBasePath::iterator J = I->begin(), JE = I->end(); J != JE; ++J) {
+ for (const CXXBasePathElement &PathElement : Path) {
// If the path contains a virtual base class we can't give any hint.
// -1: no hint.
- if (J->Base->isVirtual())
+ if (PathElement.Base->isVirtual())
return CharUnits::fromQuantity(-1ULL);
if (NumPublicPaths > 1) // Won't use offsets, skip computation.
continue;
// Accumulate the base class offsets.
- const ASTRecordLayout &L = Context.getASTRecordLayout(J->Class);
- Offset += L.getBaseClassOffset(J->Base->getType()->getAsCXXRecordDecl());
+ const ASTRecordLayout &L = Context.getASTRecordLayout(PathElement.Class);
+ Offset += L.getBaseClassOffset(
+ PathElement.Base->getType()->getAsCXXRecordDecl());
}
}
@@ -1076,14 +1211,16 @@
llvm::Value *ItaniumCXXABI::EmitTypeid(CodeGenFunction &CGF,
QualType SrcRecordTy,
- llvm::Value *ThisPtr,
+ Address ThisPtr,
llvm::Type *StdTypeInfoPtrTy) {
+ auto *ClassDecl =
+ cast<CXXRecordDecl>(SrcRecordTy->getAs<RecordType>()->getDecl());
llvm::Value *Value =
- CGF.GetVTablePtr(ThisPtr, StdTypeInfoPtrTy->getPointerTo());
+ CGF.GetVTablePtr(ThisPtr, StdTypeInfoPtrTy->getPointerTo(), ClassDecl);
// Load the type info.
Value = CGF.Builder.CreateConstInBoundsGEP1_64(Value, -1ULL);
- return CGF.Builder.CreateLoad(Value);
+ return CGF.Builder.CreateAlignedLoad(Value, CGF.getPointerAlign());
}
bool ItaniumCXXABI::shouldDynamicCastCallBeNullChecked(bool SrcIsPtr,
@@ -1092,7 +1229,7 @@
}
llvm::Value *ItaniumCXXABI::EmitDynamicCastCall(
- CodeGenFunction &CGF, llvm::Value *Value, QualType SrcRecordTy,
+ CodeGenFunction &CGF, Address ThisAddr, QualType SrcRecordTy,
QualType DestTy, QualType DestRecordTy, llvm::BasicBlock *CastEnd) {
llvm::Type *PtrDiffLTy =
CGF.ConvertType(CGF.getContext().getPointerDiffType());
@@ -1111,6 +1248,7 @@
computeOffsetHint(CGF.getContext(), SrcDecl, DestDecl).getQuantity());
// Emit the call to __dynamic_cast.
+ llvm::Value *Value = ThisAddr.getPointer();
Value = CGF.EmitCastToVoidPtr(Value);
llvm::Value *args[] = {Value, SrcRTTI, DestRTTI, OffsetHint};
@@ -1134,22 +1272,28 @@
}
llvm::Value *ItaniumCXXABI::EmitDynamicCastToVoid(CodeGenFunction &CGF,
- llvm::Value *Value,
+ Address ThisAddr,
QualType SrcRecordTy,
QualType DestTy) {
llvm::Type *PtrDiffLTy =
CGF.ConvertType(CGF.getContext().getPointerDiffType());
llvm::Type *DestLTy = CGF.ConvertType(DestTy);
+ auto *ClassDecl =
+ cast<CXXRecordDecl>(SrcRecordTy->getAs<RecordType>()->getDecl());
// Get the vtable pointer.
- llvm::Value *VTable = CGF.GetVTablePtr(Value, PtrDiffLTy->getPointerTo());
+ llvm::Value *VTable = CGF.GetVTablePtr(ThisAddr, PtrDiffLTy->getPointerTo(),
+ ClassDecl);
// Get the offset-to-top from the vtable.
llvm::Value *OffsetToTop =
CGF.Builder.CreateConstInBoundsGEP1_64(VTable, -2ULL);
- OffsetToTop = CGF.Builder.CreateLoad(OffsetToTop, "offset.to.top");
+ OffsetToTop =
+ CGF.Builder.CreateAlignedLoad(OffsetToTop, CGF.getPointerAlign(),
+ "offset.to.top");
// Finally, add the offset to the pointer.
+ llvm::Value *Value = ThisAddr.getPointer();
Value = CGF.EmitCastToVoidPtr(Value);
Value = CGF.Builder.CreateInBoundsGEP(Value, OffsetToTop);
@@ -1165,10 +1309,10 @@
llvm::Value *
ItaniumCXXABI::GetVirtualBaseClassOffset(CodeGenFunction &CGF,
- llvm::Value *This,
+ Address This,
const CXXRecordDecl *ClassDecl,
const CXXRecordDecl *BaseClassDecl) {
- llvm::Value *VTablePtr = CGF.GetVTablePtr(This, CGM.Int8PtrTy);
+ llvm::Value *VTablePtr = CGF.GetVTablePtr(This, CGM.Int8PtrTy, ClassDecl);
CharUnits VBaseOffsetOffset =
CGM.getItaniumVTableContext().getVirtualBaseOffsetOffset(ClassDecl,
BaseClassDecl);
@@ -1180,7 +1324,8 @@
CGM.PtrDiffTy->getPointerTo());
llvm::Value *VBaseOffset =
- CGF.Builder.CreateLoad(VBaseOffsetPtr, "vbase.offset");
+ CGF.Builder.CreateAlignedLoad(VBaseOffsetPtr, CGF.getPointerAlign(),
+ "vbase.offset");
return VBaseOffset;
}
@@ -1291,7 +1436,7 @@
void ItaniumCXXABI::EmitDestructorCall(CodeGenFunction &CGF,
const CXXDestructorDecl *DD,
CXXDtorType Type, bool ForVirtualBase,
- bool Delegating, llvm::Value *This) {
+ bool Delegating, Address This) {
GlobalDecl GD(DD, Type);
llvm::Value *VTT = CGF.GetVTTParameter(GD, ForVirtualBase, Delegating);
QualType VTTTy = getContext().getPointerType(getContext().VoidPtrTy);
@@ -1303,8 +1448,8 @@
if (!Callee)
Callee = CGM.getAddrOfCXXStructor(DD, getFromDtorType(Type));
- CGF.EmitCXXMemberOrOperatorCall(DD, Callee, ReturnValueSlot(), This, VTT,
- VTTTy, nullptr);
+ CGF.EmitCXXMemberOrOperatorCall(DD, Callee, ReturnValueSlot(),
+ This.getPointer(), VTT, VTTTy, nullptr);
}
void ItaniumCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT,
@@ -1354,41 +1499,29 @@
CGM.EmitVTableBitSetEntries(VTable, VTLayout);
}
-llvm::Value *ItaniumCXXABI::getVTableAddressPointInStructor(
- CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, BaseSubobject Base,
- const CXXRecordDecl *NearestVBase, bool &NeedsVirtualOffset) {
- bool NeedsVTTParam = CGM.getCXXABI().NeedsVTTParameter(CGF.CurGD);
- NeedsVirtualOffset = (NeedsVTTParam && NearestVBase);
-
- llvm::Value *VTableAddressPoint;
- if (NeedsVTTParam && (Base.getBase()->getNumVBases() || NearestVBase)) {
- // Get the secondary vpointer index.
- uint64_t VirtualPointerIndex =
- CGM.getVTables().getSecondaryVirtualPointerIndex(VTableClass, Base);
-
- /// Load the VTT.
- llvm::Value *VTT = CGF.LoadCXXVTT();
- if (VirtualPointerIndex)
- VTT = CGF.Builder.CreateConstInBoundsGEP1_64(VTT, VirtualPointerIndex);
-
- // And load the address point from the VTT.
- VTableAddressPoint = CGF.Builder.CreateLoad(VTT);
- } else {
- llvm::Constant *VTable =
- CGM.getCXXABI().getAddrOfVTable(VTableClass, CharUnits());
- uint64_t AddressPoint = CGM.getItaniumVTableContext()
- .getVTableLayout(VTableClass)
- .getAddressPoint(Base);
- VTableAddressPoint =
- CGF.Builder.CreateConstInBoundsGEP2_64(VTable, 0, AddressPoint);
- }
-
- return VTableAddressPoint;
+bool ItaniumCXXABI::isVirtualOffsetNeededForVTableField(
+ CodeGenFunction &CGF, CodeGenFunction::VPtr Vptr) {
+ if (Vptr.NearestVBase == nullptr)
+ return false;
+ return NeedsVTTParameter(CGF.CurGD);
}
-llvm::Constant *ItaniumCXXABI::getVTableAddressPointForConstExpr(
- BaseSubobject Base, const CXXRecordDecl *VTableClass) {
- auto *VTable = getAddrOfVTable(VTableClass, CharUnits());
+llvm::Value *ItaniumCXXABI::getVTableAddressPointInStructor(
+ CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, BaseSubobject Base,
+ const CXXRecordDecl *NearestVBase) {
+
+ if ((Base.getBase()->getNumVBases() || NearestVBase != nullptr) &&
+ NeedsVTTParameter(CGF.CurGD)) {
+ return getVTableAddressPointInStructorWithVTT(CGF, VTableClass, Base,
+ NearestVBase);
+ }
+ return getVTableAddressPoint(Base, VTableClass);
+}
+
+llvm::Constant *
+ItaniumCXXABI::getVTableAddressPoint(BaseSubobject Base,
+ const CXXRecordDecl *VTableClass) {
+ llvm::GlobalValue *VTable = getAddrOfVTable(VTableClass, CharUnits());
// Find the appropriate vtable within the vtable group.
uint64_t AddressPoint = CGM.getItaniumVTableContext()
@@ -1403,6 +1536,30 @@
VTable, Indices);
}
+llvm::Value *ItaniumCXXABI::getVTableAddressPointInStructorWithVTT(
+ CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, BaseSubobject Base,
+ const CXXRecordDecl *NearestVBase) {
+ assert((Base.getBase()->getNumVBases() || NearestVBase != nullptr) &&
+ NeedsVTTParameter(CGF.CurGD) && "This class doesn't have VTT");
+
+ // Get the secondary vpointer index.
+ uint64_t VirtualPointerIndex =
+ CGM.getVTables().getSecondaryVirtualPointerIndex(VTableClass, Base);
+
+ /// Load the VTT.
+ llvm::Value *VTT = CGF.LoadCXXVTT();
+ if (VirtualPointerIndex)
+ VTT = CGF.Builder.CreateConstInBoundsGEP1_64(VTT, VirtualPointerIndex);
+
+ // And load the address point from the VTT.
+ return CGF.Builder.CreateAlignedLoad(VTT, CGF.getPointerAlign());
+}
+
+llvm::Constant *ItaniumCXXABI::getVTableAddressPointForConstExpr(
+ BaseSubobject Base, const CXXRecordDecl *VTableClass) {
+ return getVTableAddressPoint(Base, VTableClass);
+}
+
llvm::GlobalVariable *ItaniumCXXABI::getAddrOfVTable(const CXXRecordDecl *RD,
CharUnits VPtrOffset) {
assert(VPtrOffset.isZero() && "Itanium ABI only supports zero vptr offsets");
@@ -1414,11 +1571,9 @@
// Queue up this v-table for possible deferred emission.
CGM.addDeferredVTable(RD);
- SmallString<256> OutName;
- llvm::raw_svector_ostream Out(OutName);
+ SmallString<256> Name;
+ llvm::raw_svector_ostream Out(Name);
getMangleContext().mangleCXXVTable(RD, Out);
- Out.flush();
- StringRef Name = OutName.str();
ItaniumVTableContext &VTContext = CGM.getItaniumVTableContext();
llvm::ArrayType *ArrayType = llvm::ArrayType::get(
@@ -1438,24 +1593,27 @@
llvm::Value *ItaniumCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF,
GlobalDecl GD,
- llvm::Value *This,
- llvm::Type *Ty) {
+ Address This,
+ llvm::Type *Ty,
+ SourceLocation Loc) {
GD = GD.getCanonicalDecl();
Ty = Ty->getPointerTo()->getPointerTo();
- llvm::Value *VTable = CGF.GetVTablePtr(This, Ty);
+ auto *MethodDecl = cast<CXXMethodDecl>(GD.getDecl());
+ llvm::Value *VTable = CGF.GetVTablePtr(This, Ty, MethodDecl->getParent());
if (CGF.SanOpts.has(SanitizerKind::CFIVCall))
- CGF.EmitVTablePtrCheckForCall(cast<CXXMethodDecl>(GD.getDecl()), VTable);
+ CGF.EmitVTablePtrCheckForCall(MethodDecl, VTable,
+ CodeGenFunction::CFITCK_VCall, Loc);
uint64_t VTableIndex = CGM.getItaniumVTableContext().getMethodVTableIndex(GD);
llvm::Value *VFuncPtr =
CGF.Builder.CreateConstInBoundsGEP1_64(VTable, VTableIndex, "vfn");
- return CGF.Builder.CreateLoad(VFuncPtr);
+ return CGF.Builder.CreateAlignedLoad(VFuncPtr, CGF.getPointerAlign());
}
llvm::Value *ItaniumCXXABI::EmitVirtualDestructorCall(
CodeGenFunction &CGF, const CXXDestructorDecl *Dtor, CXXDtorType DtorType,
- llvm::Value *This, const CXXMemberCallExpr *CE) {
+ Address This, const CXXMemberCallExpr *CE) {
assert(CE == nullptr || CE->arg_begin() == CE->arg_end());
assert(DtorType == Dtor_Deleting || DtorType == Dtor_Complete);
@@ -1463,10 +1621,12 @@
Dtor, getFromDtorType(DtorType));
llvm::Type *Ty = CGF.CGM.getTypes().GetFunctionType(*FInfo);
llvm::Value *Callee =
- getVirtualFunctionPointer(CGF, GlobalDecl(Dtor, DtorType), This, Ty);
+ getVirtualFunctionPointer(CGF, GlobalDecl(Dtor, DtorType), This, Ty,
+ CE ? CE->getLocStart() : SourceLocation());
- CGF.EmitCXXMemberOrOperatorCall(Dtor, Callee, ReturnValueSlot(), This,
- /*ImplicitParam=*/nullptr, QualType(), CE);
+ CGF.EmitCXXMemberOrOperatorCall(Dtor, Callee, ReturnValueSlot(),
+ This.getPointer(), /*ImplicitParam=*/nullptr,
+ QualType(), CE);
return nullptr;
}
@@ -1476,30 +1636,41 @@
VTables.EmitVTTDefinition(VTT, CGM.getVTableLinkage(RD), RD);
}
+bool ItaniumCXXABI::canSpeculativelyEmitVTable(const CXXRecordDecl *RD) const {
+ // We don't emit available_externally vtables if we are in -fapple-kext mode
+ // because kext mode does not permit devirtualization.
+ if (CGM.getLangOpts().AppleKext)
+ return false;
+
+ // If we don't have any inline virtual functions, and if vtable is not hidden,
+ // then we are safe to emit available_externally copy of vtable.
+ // FIXME we can still emit a copy of the vtable if we
+ // can emit definition of the inline functions.
+ return !hasAnyUsedVirtualInlineFunction(RD) && !isVTableHidden(RD);
+}
static llvm::Value *performTypeAdjustment(CodeGenFunction &CGF,
- llvm::Value *Ptr,
+ Address InitialPtr,
int64_t NonVirtualAdjustment,
int64_t VirtualAdjustment,
bool IsReturnAdjustment) {
if (!NonVirtualAdjustment && !VirtualAdjustment)
- return Ptr;
+ return InitialPtr.getPointer();
- llvm::Type *Int8PtrTy = CGF.Int8PtrTy;
- llvm::Value *V = CGF.Builder.CreateBitCast(Ptr, Int8PtrTy);
+ Address V = CGF.Builder.CreateElementBitCast(InitialPtr, CGF.Int8Ty);
+ // In a base-to-derived cast, the non-virtual adjustment is applied first.
if (NonVirtualAdjustment && !IsReturnAdjustment) {
- // Perform the non-virtual adjustment for a base-to-derived cast.
- V = CGF.Builder.CreateConstInBoundsGEP1_64(V, NonVirtualAdjustment);
+ V = CGF.Builder.CreateConstInBoundsByteGEP(V,
+ CharUnits::fromQuantity(NonVirtualAdjustment));
}
+ // Perform the virtual adjustment if we have one.
+ llvm::Value *ResultPtr;
if (VirtualAdjustment) {
llvm::Type *PtrDiffTy =
CGF.ConvertType(CGF.getContext().getPointerDiffType());
- // Perform the virtual adjustment.
- llvm::Value *VTablePtrPtr =
- CGF.Builder.CreateBitCast(V, Int8PtrTy->getPointerTo());
-
+ Address VTablePtrPtr = CGF.Builder.CreateElementBitCast(V, CGF.Int8PtrTy);
llvm::Value *VTablePtr = CGF.Builder.CreateLoad(VTablePtrPtr);
llvm::Value *OffsetPtr =
@@ -1508,23 +1679,28 @@
OffsetPtr = CGF.Builder.CreateBitCast(OffsetPtr, PtrDiffTy->getPointerTo());
// Load the adjustment offset from the vtable.
- llvm::Value *Offset = CGF.Builder.CreateLoad(OffsetPtr);
+ llvm::Value *Offset =
+ CGF.Builder.CreateAlignedLoad(OffsetPtr, CGF.getPointerAlign());
// Adjust our pointer.
- V = CGF.Builder.CreateInBoundsGEP(V, Offset);
+ ResultPtr = CGF.Builder.CreateInBoundsGEP(V.getPointer(), Offset);
+ } else {
+ ResultPtr = V.getPointer();
}
+ // In a derived-to-base conversion, the non-virtual adjustment is
+ // applied second.
if (NonVirtualAdjustment && IsReturnAdjustment) {
- // Perform the non-virtual adjustment for a derived-to-base cast.
- V = CGF.Builder.CreateConstInBoundsGEP1_64(V, NonVirtualAdjustment);
+ ResultPtr = CGF.Builder.CreateConstInBoundsGEP1_64(ResultPtr,
+ NonVirtualAdjustment);
}
// Cast back to the original type.
- return CGF.Builder.CreateBitCast(V, Ptr->getType());
+ return CGF.Builder.CreateBitCast(ResultPtr, InitialPtr.getType());
}
llvm::Value *ItaniumCXXABI::performThisAdjustment(CodeGenFunction &CGF,
- llvm::Value *This,
+ Address This,
const ThisAdjustment &TA) {
return performTypeAdjustment(CGF, This, TA.NonVirtual,
TA.Virtual.Itanium.VCallOffsetOffset,
@@ -1532,7 +1708,7 @@
}
llvm::Value *
-ItaniumCXXABI::performReturnAdjustment(CodeGenFunction &CGF, llvm::Value *Ret,
+ItaniumCXXABI::performReturnAdjustment(CodeGenFunction &CGF, Address Ret,
const ReturnAdjustment &RA) {
return performTypeAdjustment(CGF, Ret, RA.NonVirtual,
RA.Virtual.Itanium.VBaseOffsetOffset,
@@ -1545,8 +1721,7 @@
return ItaniumCXXABI::EmitReturnFromThunk(CGF, RV, ResultType);
// Destructor thunks in the ARM ABI have indeterminate results.
- llvm::Type *T =
- cast<llvm::PointerType>(CGF.ReturnValue->getType())->getElementType();
+ llvm::Type *T = CGF.ReturnValue.getElementType();
RValue Undef = RValue::get(llvm::UndefValue::get(T));
return ItaniumCXXABI::EmitReturnFromThunk(CGF, Undef, ResultType);
}
@@ -1560,18 +1735,17 @@
CGM.getContext().getTypeAlignInChars(elementType));
}
-llvm::Value *ItaniumCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
- llvm::Value *NewPtr,
- llvm::Value *NumElements,
- const CXXNewExpr *expr,
- QualType ElementType) {
+Address ItaniumCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
+ Address NewPtr,
+ llvm::Value *NumElements,
+ const CXXNewExpr *expr,
+ QualType ElementType) {
assert(requiresArrayCookie(expr));
- unsigned AS = NewPtr->getType()->getPointerAddressSpace();
+ unsigned AS = NewPtr.getAddressSpace();
ASTContext &Ctx = getContext();
- QualType SizeTy = Ctx.getSizeType();
- CharUnits SizeSize = Ctx.getTypeSizeInChars(SizeTy);
+ CharUnits SizeSize = CGF.getSizeSize();
// The size of the cookie.
CharUnits CookieSize =
@@ -1579,49 +1753,45 @@
assert(CookieSize == getArrayCookieSizeImpl(ElementType));
// Compute an offset to the cookie.
- llvm::Value *CookiePtr = NewPtr;
+ Address CookiePtr = NewPtr;
CharUnits CookieOffset = CookieSize - SizeSize;
if (!CookieOffset.isZero())
- CookiePtr = CGF.Builder.CreateConstInBoundsGEP1_64(CookiePtr,
- CookieOffset.getQuantity());
+ CookiePtr = CGF.Builder.CreateConstInBoundsByteGEP(CookiePtr, CookieOffset);
// Write the number of elements into the appropriate slot.
- llvm::Type *NumElementsTy = CGF.ConvertType(SizeTy)->getPointerTo(AS);
- llvm::Value *NumElementsPtr =
- CGF.Builder.CreateBitCast(CookiePtr, NumElementsTy);
+ Address NumElementsPtr =
+ CGF.Builder.CreateElementBitCast(CookiePtr, CGF.SizeTy);
llvm::Instruction *SI = CGF.Builder.CreateStore(NumElements, NumElementsPtr);
+
+ // Handle the array cookie specially in ASan.
if (CGM.getLangOpts().Sanitize.has(SanitizerKind::Address) && AS == 0 &&
expr->getOperatorNew()->isReplaceableGlobalAllocationFunction()) {
// The store to the CookiePtr does not need to be instrumented.
CGM.getSanitizerMetadata()->disableSanitizerForInstruction(SI);
llvm::FunctionType *FTy =
- llvm::FunctionType::get(CGM.VoidTy, NumElementsTy, false);
+ llvm::FunctionType::get(CGM.VoidTy, NumElementsPtr.getType(), false);
llvm::Constant *F =
CGM.CreateRuntimeFunction(FTy, "__asan_poison_cxx_array_cookie");
- CGF.Builder.CreateCall(F, NumElementsPtr);
+ CGF.Builder.CreateCall(F, NumElementsPtr.getPointer());
}
// Finally, compute a pointer to the actual data buffer by skipping
// over the cookie completely.
- return CGF.Builder.CreateConstInBoundsGEP1_64(NewPtr,
- CookieSize.getQuantity());
+ return CGF.Builder.CreateConstInBoundsByteGEP(NewPtr, CookieSize);
}
llvm::Value *ItaniumCXXABI::readArrayCookieImpl(CodeGenFunction &CGF,
- llvm::Value *allocPtr,
+ Address allocPtr,
CharUnits cookieSize) {
// The element size is right-justified in the cookie.
- llvm::Value *numElementsPtr = allocPtr;
- CharUnits numElementsOffset =
- cookieSize - CharUnits::fromQuantity(CGF.SizeSizeInBytes);
+ Address numElementsPtr = allocPtr;
+ CharUnits numElementsOffset = cookieSize - CGF.getSizeSize();
if (!numElementsOffset.isZero())
numElementsPtr =
- CGF.Builder.CreateConstInBoundsGEP1_64(numElementsPtr,
- numElementsOffset.getQuantity());
+ CGF.Builder.CreateConstInBoundsByteGEP(numElementsPtr, numElementsOffset);
- unsigned AS = allocPtr->getType()->getPointerAddressSpace();
- numElementsPtr =
- CGF.Builder.CreateBitCast(numElementsPtr, CGF.SizeTy->getPointerTo(AS));
+ unsigned AS = allocPtr.getAddressSpace();
+ numElementsPtr = CGF.Builder.CreateElementBitCast(numElementsPtr, CGF.SizeTy);
if (!CGM.getLangOpts().Sanitize.has(SanitizerKind::Address) || AS != 0)
return CGF.Builder.CreateLoad(numElementsPtr);
// In asan mode emit a function call instead of a regular load and let the
@@ -1633,7 +1803,7 @@
llvm::FunctionType::get(CGF.SizeTy, CGF.SizeTy->getPointerTo(0), false);
llvm::Constant *F =
CGM.CreateRuntimeFunction(FTy, "__asan_load_cxx_array_cookie");
- return CGF.Builder.CreateCall(F, numElementsPtr);
+ return CGF.Builder.CreateCall(F, numElementsPtr.getPointer());
}
CharUnits ARMCXXABI::getArrayCookieSizeImpl(QualType elementType) {
@@ -1649,47 +1819,41 @@
CGM.getContext().getTypeAlignInChars(elementType));
}
-llvm::Value *ARMCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
- llvm::Value *newPtr,
- llvm::Value *numElements,
- const CXXNewExpr *expr,
- QualType elementType) {
+Address ARMCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
+ Address newPtr,
+ llvm::Value *numElements,
+ const CXXNewExpr *expr,
+ QualType elementType) {
assert(requiresArrayCookie(expr));
- // NewPtr is a char*, but we generalize to arbitrary addrspaces.
- unsigned AS = newPtr->getType()->getPointerAddressSpace();
-
// The cookie is always at the start of the buffer.
- llvm::Value *cookie = newPtr;
+ Address cookie = newPtr;
// The first element is the element size.
- cookie = CGF.Builder.CreateBitCast(cookie, CGF.SizeTy->getPointerTo(AS));
+ cookie = CGF.Builder.CreateElementBitCast(cookie, CGF.SizeTy);
llvm::Value *elementSize = llvm::ConstantInt::get(CGF.SizeTy,
getContext().getTypeSizeInChars(elementType).getQuantity());
CGF.Builder.CreateStore(elementSize, cookie);
// The second element is the element count.
- cookie = CGF.Builder.CreateConstInBoundsGEP1_32(CGF.SizeTy, cookie, 1);
+ cookie = CGF.Builder.CreateConstInBoundsGEP(cookie, 1, CGF.getSizeSize());
CGF.Builder.CreateStore(numElements, cookie);
// Finally, compute a pointer to the actual data buffer by skipping
// over the cookie completely.
CharUnits cookieSize = ARMCXXABI::getArrayCookieSizeImpl(elementType);
- return CGF.Builder.CreateConstInBoundsGEP1_64(newPtr,
- cookieSize.getQuantity());
+ return CGF.Builder.CreateConstInBoundsByteGEP(newPtr, cookieSize);
}
llvm::Value *ARMCXXABI::readArrayCookieImpl(CodeGenFunction &CGF,
- llvm::Value *allocPtr,
+ Address allocPtr,
CharUnits cookieSize) {
// The number of elements is at offset sizeof(size_t) relative to
// the allocated pointer.
- llvm::Value *numElementsPtr
- = CGF.Builder.CreateConstInBoundsGEP1_64(allocPtr, CGF.SizeSizeInBytes);
+ Address numElementsPtr
+ = CGF.Builder.CreateConstInBoundsByteGEP(allocPtr, CGF.getSizeSize());
- unsigned AS = allocPtr->getType()->getPointerAddressSpace();
- numElementsPtr =
- CGF.Builder.CreateBitCast(numElementsPtr, CGF.SizeTy->getPointerTo(AS));
+ numElementsPtr = CGF.Builder.CreateElementBitCast(numElementsPtr, CGF.SizeTy);
return CGF.Builder.CreateLoad(numElementsPtr);
}
@@ -1730,7 +1894,7 @@
}
namespace {
- struct CallGuardAbort : EHScopeStack::Cleanup {
+ struct CallGuardAbort final : EHScopeStack::Cleanup {
llvm::GlobalVariable *Guard;
CallGuardAbort(llvm::GlobalVariable *Guard) : Guard(Guard) {}
@@ -1759,12 +1923,21 @@
bool useInt8GuardVariable = !threadsafe && var->hasInternalLinkage();
llvm::IntegerType *guardTy;
+ CharUnits guardAlignment;
if (useInt8GuardVariable) {
guardTy = CGF.Int8Ty;
+ guardAlignment = CharUnits::One();
} else {
// Guard variables are 64 bits in the generic ABI and size width on ARM
// (i.e. 32-bit on AArch32, 64-bit on AArch64).
- guardTy = (UseARMGuardVarABI ? CGF.SizeTy : CGF.Int64Ty);
+ if (UseARMGuardVarABI) {
+ guardTy = CGF.SizeTy;
+ guardAlignment = CGF.getSizeAlign();
+ } else {
+ guardTy = CGF.Int64Ty;
+ guardAlignment = CharUnits::fromQuantity(
+ CGM.getDataLayout().getABITypeAlignment(guardTy));
+ }
}
llvm::PointerType *guardPtrTy = guardTy->getPointerTo();
@@ -1777,7 +1950,6 @@
{
llvm::raw_svector_ostream out(guardName);
getMangleContext().mangleStaticGuardVariable(&D, out);
- out.flush();
}
// Create the guard variable with a zero-initializer.
@@ -1789,11 +1961,14 @@
guard->setVisibility(var->getVisibility());
// If the variable is thread-local, so is its guard variable.
guard->setThreadLocalMode(var->getThreadLocalMode());
+ guard->setAlignment(guardAlignment.getQuantity());
- // The ABI says: It is suggested that it be emitted in the same COMDAT group
- // as the associated data object
+ // The ABI says: "It is suggested that it be emitted in the same COMDAT
+ // group as the associated data object." In practice, this doesn't work for
+ // non-ELF object formats, so only do it for ELF.
llvm::Comdat *C = var->getComdat();
- if (!D.isLocalVarDecl() && C) {
+ if (!D.isLocalVarDecl() && C &&
+ CGM.getTarget().getTriple().isOSBinFormatELF()) {
guard->setComdat(C);
CGF.CurFn->setComdat(C);
} else if (CGM.supportsCOMDAT() && guard->isWeakForLinker()) {
@@ -1803,6 +1978,8 @@
CGM.setStaticLocalDeclGuardAddress(&D, guard);
}
+ Address guardAddr = Address(guard, guardAlignment);
+
// Test whether the variable has completed initialization.
//
// Itanium C++ ABI 3.3.2:
@@ -1822,8 +1999,7 @@
// Load the first byte of the guard variable.
llvm::LoadInst *LI =
- Builder.CreateLoad(Builder.CreateBitCast(guard, CGM.Int8PtrTy));
- LI->setAlignment(1);
+ Builder.CreateLoad(Builder.CreateElementBitCast(guardAddr, CGM.Int8Ty));
// Itanium ABI:
// An implementation supporting thread-safety on multiprocessor
@@ -1893,9 +2069,10 @@
CGF.PopCleanupBlock();
// Call __cxa_guard_release. This cannot throw.
- CGF.EmitNounwindRuntimeCall(getGuardReleaseFn(CGM, guardPtrTy), guard);
+ CGF.EmitNounwindRuntimeCall(getGuardReleaseFn(CGM, guardPtrTy),
+ guardAddr.getPointer());
} else {
- Builder.CreateStore(llvm::ConstantInt::get(guardTy, 1), guard);
+ Builder.CreateStore(llvm::ConstantInt::get(guardTy, 1), guardAddr);
}
CGF.EmitBlock(EndBlock);
@@ -1909,7 +2086,7 @@
const char *Name = "__cxa_atexit";
if (TLS) {
const llvm::Triple &T = CGF.getTarget().getTriple();
- Name = T.isMacOSX() ? "_tlv_atexit" : "__cxa_thread_atexit";
+ Name = T.isOSDarwin() ? "_tlv_atexit" : "__cxa_thread_atexit";
}
// We're assuming that the destructor function is something we can
@@ -1965,10 +2142,10 @@
static bool isThreadWrapperReplaceable(const VarDecl *VD,
CodeGen::CodeGenModule &CGM) {
assert(!VD->isStaticLocal() && "static local VarDecls don't need wrappers!");
- // OS X prefers to have references to thread local variables to go through
+ // Darwin prefers to have references to thread local variables to go through
// the thread wrapper instead of directly referencing the backing variable.
return VD->getTLSKind() == VarDecl::TLS_Dynamic &&
- CGM.getTarget().getTriple().isMacOSX();
+ CGM.getTarget().getTriple().isOSDarwin();
}
/// Get the appropriate linkage for the wrapper function. This is essentially
@@ -1984,12 +2161,10 @@
return VarLinkage;
// If the thread wrapper is replaceable, give it appropriate linkage.
- if (isThreadWrapperReplaceable(VD, CGM)) {
- if (llvm::GlobalVariable::isLinkOnceLinkage(VarLinkage) ||
- llvm::GlobalVariable::isWeakODRLinkage(VarLinkage))
- return llvm::GlobalVariable::WeakAnyLinkage;
- return VarLinkage;
- }
+ if (isThreadWrapperReplaceable(VD, CGM))
+ if (!llvm::GlobalVariable::isLinkOnceLinkage(VarLinkage) &&
+ !llvm::GlobalVariable::isWeakODRLinkage(VarLinkage))
+ return VarLinkage;
return llvm::GlobalValue::WeakODRLinkage;
}
@@ -2001,7 +2176,6 @@
{
llvm::raw_svector_ostream Out(WrapperName);
getMangleContext().mangleItaniumThreadLocalWrapper(VD, Out);
- Out.flush();
}
if (llvm::Value *V = CGM.getModule().getNamedValue(WrapperName))
@@ -2016,22 +2190,29 @@
llvm::Function::Create(FnTy, getThreadLocalWrapperLinkage(VD, CGM),
WrapperName.str(), &CGM.getModule());
// Always resolve references to the wrapper at link time.
- if (!Wrapper->hasLocalLinkage() && !isThreadWrapperReplaceable(VD, CGM))
+ if (!Wrapper->hasLocalLinkage() && !(isThreadWrapperReplaceable(VD, CGM) &&
+ !llvm::GlobalVariable::isLinkOnceLinkage(Wrapper->getLinkage()) &&
+ !llvm::GlobalVariable::isWeakODRLinkage(Wrapper->getLinkage())))
Wrapper->setVisibility(llvm::GlobalValue::HiddenVisibility);
+
+ if (isThreadWrapperReplaceable(VD, CGM)) {
+ Wrapper->setCallingConv(llvm::CallingConv::CXX_FAST_TLS);
+ Wrapper->addFnAttr(llvm::Attribute::NoUnwind);
+ }
return Wrapper;
}
void ItaniumCXXABI::EmitThreadLocalInitFuncs(
- CodeGenModule &CGM,
- ArrayRef<std::pair<const VarDecl *, llvm::GlobalVariable *>>
- CXXThreadLocals, ArrayRef<llvm::Function *> CXXThreadLocalInits,
- ArrayRef<llvm::GlobalVariable *> CXXThreadLocalInitVars) {
+ CodeGenModule &CGM, ArrayRef<const VarDecl *> CXXThreadLocals,
+ ArrayRef<llvm::Function *> CXXThreadLocalInits,
+ ArrayRef<const VarDecl *> CXXThreadLocalInitVars) {
llvm::Function *InitFunc = nullptr;
if (!CXXThreadLocalInits.empty()) {
// Generate a guarded initialization function.
llvm::FunctionType *FTy =
llvm::FunctionType::get(CGM.VoidTy, /*isVarArg=*/false);
- InitFunc = CGM.CreateGlobalInitOrDestructFunction(FTy, "__tls_init",
+ const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
+ InitFunc = CGM.CreateGlobalInitOrDestructFunction(FTy, "__tls_init", FI,
SourceLocation(),
/*TLS=*/true);
llvm::GlobalVariable *Guard = new llvm::GlobalVariable(
@@ -2039,12 +2220,17 @@
llvm::GlobalVariable::InternalLinkage,
llvm::ConstantInt::get(CGM.Int8Ty, 0), "__tls_guard");
Guard->setThreadLocal(true);
+
+ CharUnits GuardAlign = CharUnits::One();
+ Guard->setAlignment(GuardAlign.getQuantity());
+
CodeGenFunction(CGM)
- .GenerateCXXGlobalInitFunc(InitFunc, CXXThreadLocalInits, Guard);
+ .GenerateCXXGlobalInitFunc(InitFunc, CXXThreadLocalInits,
+ Address(Guard, GuardAlign));
}
- for (unsigned I = 0, N = CXXThreadLocals.size(); I != N; ++I) {
- const VarDecl *VD = CXXThreadLocals[I].first;
- llvm::GlobalVariable *Var = CXXThreadLocals[I].second;
+ for (const VarDecl *VD : CXXThreadLocals) {
+ llvm::GlobalVariable *Var =
+ cast<llvm::GlobalVariable>(CGM.GetGlobalValue(CGM.getMangledName(VD)));
// Some targets require that all access to thread local variables go through
// the thread wrapper. This means that we cannot attempt to create a thread
@@ -2057,7 +2243,6 @@
{
llvm::raw_svector_ostream Out(InitFnName);
getMangleContext().mangleItaniumThreadLocalInit(VD, Out);
- Out.flush();
}
// If we have a definition for the variable, emit the initialization
@@ -2087,10 +2272,10 @@
llvm::Function *Wrapper = getOrCreateThreadLocalWrapper(VD, Var);
llvm::LLVMContext &Context = CGM.getModule().getContext();
llvm::BasicBlock *Entry = llvm::BasicBlock::Create(Context, "", Wrapper);
- CGBuilderTy Builder(Entry);
+ CGBuilderTy Builder(CGM, Entry);
if (InitIsInitFunc) {
if (Init)
- Builder.CreateCall(Init, {});
+ Builder.CreateCall(Init);
} else {
// Don't know whether we have an init function. Call it if it exists.
llvm::Value *Have = Builder.CreateIsNotNull(Init);
@@ -2099,7 +2284,7 @@
Builder.CreateCondBr(Have, InitBB, ExitBB);
Builder.SetInsertPoint(InitBB);
- Builder.CreateCall(Init, {});
+ Builder.CreateCall(Init);
Builder.CreateBr(ExitBB);
Builder.SetInsertPoint(ExitBB);
@@ -2109,9 +2294,8 @@
// the referenced object.
llvm::Value *Val = Var;
if (VD->getType()->isReferenceType()) {
- llvm::LoadInst *LI = Builder.CreateLoad(Val);
- LI->setAlignment(CGM.getContext().getDeclAlign(VD).getQuantity());
- Val = LI;
+ CharUnits Align = CGM.getContext().getDeclAlign(VD);
+ Val = Builder.CreateAlignedLoad(Val, Align);
}
if (Val->getType() != Wrapper->getReturnType())
Val = Builder.CreatePointerBitCastOrAddrSpaceCast(
@@ -2123,18 +2307,19 @@
LValue ItaniumCXXABI::EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF,
const VarDecl *VD,
QualType LValType) {
- QualType T = VD->getType();
- llvm::Type *Ty = CGF.getTypes().ConvertTypeForMem(T);
- llvm::Value *Val = CGF.CGM.GetAddrOfGlobalVar(VD, Ty);
+ llvm::Value *Val = CGF.CGM.GetAddrOfGlobalVar(VD);
llvm::Function *Wrapper = getOrCreateThreadLocalWrapper(VD, Val);
- Val = CGF.Builder.CreateCall(Wrapper, {});
+ llvm::CallInst *CallVal = CGF.Builder.CreateCall(Wrapper);
+ if (isThreadWrapperReplaceable(VD, CGF.CGM))
+ CallVal->setCallingConv(llvm::CallingConv::CXX_FAST_TLS);
LValue LV;
if (VD->getType()->isReferenceType())
- LV = CGF.MakeNaturalAlignAddrLValue(Val, LValType);
+ LV = CGF.MakeNaturalAlignAddrLValue(CallVal, LValType);
else
- LV = CGF.MakeAddrLValue(Val, LValType, CGF.getContext().getDeclAlign(VD));
+ LV = CGF.MakeAddrLValue(CallVal, LValType,
+ CGF.getContext().getDeclAlign(VD));
// FIXME: need setObjCGCLValueClass?
return LV;
}
@@ -2250,11 +2435,9 @@
llvm::GlobalVariable *ItaniumRTTIBuilder::GetAddrOfTypeName(
QualType Ty, llvm::GlobalVariable::LinkageTypes Linkage) {
- SmallString<256> OutName;
- llvm::raw_svector_ostream Out(OutName);
+ SmallString<256> Name;
+ llvm::raw_svector_ostream Out(Name);
CGM.getCXXABI().getMangleContext().mangleCXXRTTIName(Ty, Out);
- Out.flush();
- StringRef Name = OutName.str();
// We know that the mangled name of the type starts at index 4 of the
// mangled name of the typename, so we can just index into it in order to
@@ -2273,11 +2456,9 @@
llvm::Constant *
ItaniumRTTIBuilder::GetAddrOfExternalRTTIDescriptor(QualType Ty) {
// Mangle the RTTI name.
- SmallString<256> OutName;
- llvm::raw_svector_ostream Out(OutName);
+ SmallString<256> Name;
+ llvm::raw_svector_ostream Out(Name);
CGM.getCXXABI().getMangleContext().mangleCXXRTTI(Ty, Out);
- Out.flush();
- StringRef Name = OutName.str();
// Look for an existing global.
llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(Name);
@@ -2341,9 +2522,19 @@
case BuiltinType::OCLImage1dBuffer:
case BuiltinType::OCLImage2d:
case BuiltinType::OCLImage2dArray:
+ case BuiltinType::OCLImage2dDepth:
+ case BuiltinType::OCLImage2dArrayDepth:
+ case BuiltinType::OCLImage2dMSAA:
+ case BuiltinType::OCLImage2dArrayMSAA:
+ case BuiltinType::OCLImage2dMSAADepth:
+ case BuiltinType::OCLImage2dArrayMSAADepth:
case BuiltinType::OCLImage3d:
case BuiltinType::OCLSampler:
case BuiltinType::OCLEvent:
+ case BuiltinType::OCLClkEvent:
+ case BuiltinType::OCLQueue:
+ case BuiltinType::OCLNDRange:
+ case BuiltinType::OCLReserveID:
return true;
case BuiltinType::Dependent:
@@ -2415,10 +2606,13 @@
// FIXME: this may need to be reconsidered if the key function
// changes.
+ // N.B. We must always emit the RTTI data ourselves if there exists a key
+ // function.
+ bool IsDLLImport = RD->hasAttr<DLLImportAttr>();
if (CGM.getVTables().isVTableExternal(RD))
- return true;
+ return IsDLLImport ? false : true;
- if (RD->hasAttr<DLLImportAttr>())
+ if (IsDLLImport)
return true;
}
@@ -2648,8 +2842,15 @@
const CXXRecordDecl *RD = cast<CXXRecordDecl>(Record->getDecl());
if (RD->hasAttr<WeakAttr>())
return llvm::GlobalValue::WeakODRLinkage;
- if (RD->isDynamicClass())
- return CGM.getVTableLinkage(RD);
+ if (RD->isDynamicClass()) {
+ llvm::GlobalValue::LinkageTypes LT = CGM.getVTableLinkage(RD);
+ // MinGW won't export the RTTI information when there is a key function.
+ // Make sure we emit our own copy instead of attempting to dllimport it.
+ if (RD->hasAttr<DLLImportAttr>() &&
+ llvm::GlobalValue::isAvailableExternallyLinkage(LT))
+ LT = llvm::GlobalValue::LinkOnceODRLinkage;
+ return LT;
+ }
}
return llvm::GlobalValue::LinkOnceODRLinkage;
@@ -2663,11 +2864,9 @@
Ty = CGM.getContext().getCanonicalType(Ty);
// Check if we've already emitted an RTTI descriptor for this type.
- SmallString<256> OutName;
- llvm::raw_svector_ostream Out(OutName);
+ SmallString<256> Name;
+ llvm::raw_svector_ostream Out(Name);
CGM.getCXXABI().getMangleContext().mangleCXXRTTI(Ty, Out);
- Out.flush();
- StringRef Name = OutName.str();
llvm::GlobalVariable *OldGV = CGM.getModule().getNamedGlobal(Name);
if (OldGV && !OldGV->isDeclaration()) {
@@ -2803,9 +3002,6 @@
new llvm::GlobalVariable(M, Init->getType(),
/*Constant=*/true, Linkage, Init, Name);
- if (CGM.supportsCOMDAT() && GV->isWeakForLinker())
- GV->setComdat(M.getOrInsertComdat(GV->getName()));
-
// If there's already an old global variable, replace it with the new one.
if (OldGV) {
GV->takeName(OldGV);
@@ -2815,6 +3011,9 @@
OldGV->eraseFromParent();
}
+ if (CGM.supportsCOMDAT() && GV->isWeakForLinker())
+ GV->setComdat(M.getOrInsertComdat(GV->getName()));
+
// The Itanium ABI specifies that type_info objects must be globally
// unique, with one exception: if the type is an incomplete class
// type or a (possibly indirect) pointer to one. That exception
@@ -3217,15 +3416,13 @@
return;
auto *Aliasee = cast<llvm::GlobalValue>(CGM.GetAddrOfGlobal(TargetDecl));
- llvm::PointerType *AliasType = Aliasee->getType();
// Create the alias with no name.
- auto *Alias = llvm::GlobalAlias::create(AliasType, Linkage, "", Aliasee,
- &CGM.getModule());
+ auto *Alias = llvm::GlobalAlias::create(Linkage, "", Aliasee);
// Switch any previous uses to the alias.
if (Entry) {
- assert(Entry->getType() == AliasType &&
+ assert(Entry->getType() == Aliasee->getType() &&
"declaration exists with different type");
Alias->takeName(Entry);
Entry->replaceAllUsesWith(Alias);
@@ -3263,7 +3460,7 @@
if (CGType == StructorCodegen::RAUW) {
StringRef MangledName = CGM.getMangledName(CompleteDecl);
- auto *Aliasee = cast<llvm::GlobalValue>(CGM.GetAddrOfGlobal(BaseDecl));
+ auto *Aliasee = CGM.GetAddrOfGlobal(BaseDecl);
CGM.addReplacement(MangledName, Aliasee);
return;
}
@@ -3330,7 +3527,7 @@
/// of the caught type, so we have to assume the actual thrown
/// exception type might have a throwing destructor, even if the
/// caught type's destructor is trivial or nothrow.
- struct CallEndCatch : EHScopeStack::Cleanup {
+ struct CallEndCatch final : EHScopeStack::Cleanup {
CallEndCatch(bool MightThrow) : MightThrow(MightThrow) {}
bool MightThrow;
@@ -3364,7 +3561,7 @@
/// parameter during catch initialization.
static void InitCatchParam(CodeGenFunction &CGF,
const VarDecl &CatchParam,
- llvm::Value *ParamAddr,
+ Address ParamAddr,
SourceLocation Loc) {
// Load the exception from where the landing pad saved it.
llvm::Value *Exn = CGF.getExceptionFromSlot();
@@ -3418,12 +3615,13 @@
cast<llvm::PointerType>(LLVMCatchTy)->getElementType();
// Create the temporary and write the adjusted pointer into it.
- llvm::Value *ExnPtrTmp = CGF.CreateTempAlloca(PtrTy, "exn.byref.tmp");
+ Address ExnPtrTmp =
+ CGF.CreateTempAlloca(PtrTy, CGF.getPointerAlign(), "exn.byref.tmp");
llvm::Value *Casted = CGF.Builder.CreateBitCast(AdjustedExn, PtrTy);
CGF.Builder.CreateStore(Casted, ExnPtrTmp);
// Bind the reference to the temporary.
- AdjustedExn = ExnPtrTmp;
+ AdjustedExn = ExnPtrTmp.getPointer();
}
}
@@ -3468,8 +3666,7 @@
llvm::Value *Cast = CGF.Builder.CreateBitCast(AdjustedExn, PtrTy);
LValue srcLV = CGF.MakeNaturalAlignAddrLValue(Cast, CatchType);
- LValue destLV = CGF.MakeAddrLValue(ParamAddr, CatchType,
- CGF.getContext().getDeclAlign(&CatchParam));
+ LValue destLV = CGF.MakeAddrLValue(ParamAddr, CatchType);
switch (TEK) {
case TEK_Complex:
CGF.EmitStoreOfComplex(CGF.EmitLoadOfComplex(srcLV, Loc), destLV,
@@ -3487,6 +3684,8 @@
}
assert(isa<RecordType>(CatchType) && "unexpected catch type!");
+ auto catchRD = CatchType->getAsCXXRecordDecl();
+ CharUnits caughtExnAlignment = CGF.CGM.getClassPointerAlignment(catchRD);
llvm::Type *PtrTy = LLVMCatchTy->getPointerTo(0); // addrspace 0 ok
@@ -3495,7 +3694,8 @@
const Expr *copyExpr = CatchParam.getInit();
if (!copyExpr) {
llvm::Value *rawAdjustedExn = CallBeginCatch(CGF, Exn, true);
- llvm::Value *adjustedExn = CGF.Builder.CreateBitCast(rawAdjustedExn, PtrTy);
+ Address adjustedExn(CGF.Builder.CreateBitCast(rawAdjustedExn, PtrTy),
+ caughtExnAlignment);
CGF.EmitAggregateCopy(ParamAddr, adjustedExn, CatchType);
return;
}
@@ -3506,7 +3706,8 @@
CGF.EmitNounwindRuntimeCall(getGetExceptionPtrFn(CGF.CGM), Exn);
// Cast that to the appropriate type.
- llvm::Value *adjustedExn = CGF.Builder.CreateBitCast(rawAdjustedExn, PtrTy);
+ Address adjustedExn(CGF.Builder.CreateBitCast(rawAdjustedExn, PtrTy),
+ caughtExnAlignment);
// The copy expression is defined in terms of an OpaqueValueExpr.
// Find it and map it to the adjusted expression.
@@ -3518,9 +3719,8 @@
CGF.EHStack.pushTerminate();
// Perform the copy construction.
- CharUnits Alignment = CGF.getContext().getDeclAlign(&CatchParam);
CGF.EmitAggExpr(copyExpr,
- AggValueSlot::forAddr(ParamAddr, Alignment, Qualifiers(),
+ AggValueSlot::forAddr(ParamAddr, Qualifiers(),
AggValueSlot::IsNotDestructed,
AggValueSlot::DoesNotNeedGCBarriers,
AggValueSlot::IsNotAliased));
@@ -3604,7 +3804,7 @@
// Set up the function.
llvm::BasicBlock *entry =
llvm::BasicBlock::Create(CGM.getLLVMContext(), "", fn);
- CGBuilderTy builder(entry);
+ CGBuilderTy builder(CGM, entry);
// Pull the exception pointer out of the parameter list.
llvm::Value *exn = &*fn->arg_begin();
@@ -3615,7 +3815,7 @@
catchCall->setCallingConv(CGM.getRuntimeCC());
// Call std::terminate().
- llvm::CallInst *termCall = builder.CreateCall(CGM.getTerminateFn(), {});
+ llvm::CallInst *termCall = builder.CreateCall(CGM.getTerminateFn());
termCall->setDoesNotThrow();
termCall->setDoesNotReturn();
termCall->setCallingConv(CGM.getRuntimeCC());
diff --git a/lib/CodeGen/MicrosoftCXXABI.cpp b/lib/CodeGen/MicrosoftCXXABI.cpp
index e19ad69..93210d5 100644
--- a/lib/CodeGen/MicrosoftCXXABI.cpp
+++ b/lib/CodeGen/MicrosoftCXXABI.cpp
@@ -15,6 +15,7 @@
//===----------------------------------------------------------------------===//
#include "CGCXXABI.h"
+#include "CGCleanup.h"
#include "CGVTables.h"
#include "CodeGenModule.h"
#include "CodeGenTypes.h"
@@ -45,7 +46,7 @@
: CGCXXABI(CGM), BaseClassDescriptorType(nullptr),
ClassHierarchyDescriptorType(nullptr),
CompleteObjectLocatorType(nullptr), CatchableTypeType(nullptr),
- ThrowInfoType(nullptr), CatchHandlerTypeType(nullptr) {}
+ ThrowInfoType(nullptr) {}
bool HasThisReturn(GlobalDecl GD) const override;
bool hasMostDerivedReturn(GlobalDecl GD) const override;
@@ -56,6 +57,27 @@
bool isSRetParameterAfterThis() const override { return true; }
+ bool isThisCompleteObject(GlobalDecl GD) const override {
+ // The Microsoft ABI doesn't use separate complete-object vs.
+ // base-object variants of constructors, but it does of destructors.
+ if (isa<CXXDestructorDecl>(GD.getDecl())) {
+ switch (GD.getDtorType()) {
+ case Dtor_Complete:
+ case Dtor_Deleting:
+ return true;
+
+ case Dtor_Base:
+ return false;
+
+ case Dtor_Comdat: llvm_unreachable("emitting dtor comdat as function?");
+ }
+ llvm_unreachable("bad dtor kind");
+ }
+
+ // No other kinds.
+ return false;
+ }
+
size_t getSrcArgforCopyCtor(const CXXConstructorDecl *CD,
FunctionArgList &Args) const override {
assert(Args.size() >= 2 &&
@@ -68,11 +90,30 @@
return 1;
}
+ std::vector<CharUnits> getVBPtrOffsets(const CXXRecordDecl *RD) override {
+ std::vector<CharUnits> VBPtrOffsets;
+ const ASTContext &Context = getContext();
+ const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
+
+ const VBTableGlobals &VBGlobals = enumerateVBTables(RD);
+ for (const VPtrInfo *VBT : *VBGlobals.VBTables) {
+ const ASTRecordLayout &SubobjectLayout =
+ Context.getASTRecordLayout(VBT->BaseWithVPtr);
+ CharUnits Offs = VBT->NonVirtualOffset;
+ Offs += SubobjectLayout.getVBPtrOffset();
+ if (VBT->getVBaseWithVPtr())
+ Offs += Layout.getVBaseClassOffset(VBT->getVBaseWithVPtr());
+ VBPtrOffsets.push_back(Offs);
+ }
+ llvm::array_pod_sort(VBPtrOffsets.begin(), VBPtrOffsets.end());
+ return VBPtrOffsets;
+ }
+
StringRef GetPureVirtualCallName() override { return "_purecall"; }
StringRef GetDeletedVirtualCallName() override { return "_purecall"; }
void emitVirtualObjectDelete(CodeGenFunction &CGF, const CXXDeleteExpr *DE,
- llvm::Value *Ptr, QualType ElementType,
+ Address Ptr, QualType ElementType,
const CXXDestructorDecl *Dtor) override;
void emitRethrow(CodeGenFunction &CGF, bool isNoReturn) override;
@@ -84,31 +125,39 @@
const VPtrInfo *Info);
llvm::Constant *getAddrOfRTTIDescriptor(QualType Ty) override;
- llvm::Constant *
+ CatchTypeInfo
getAddrOfCXXCatchHandlerType(QualType Ty, QualType CatchHandlerType) override;
+ /// MSVC needs an extra flag to indicate a catchall.
+ CatchTypeInfo getCatchAllTypeInfo() override {
+ return CatchTypeInfo{nullptr, 0x40};
+ }
+
bool shouldTypeidBeNullChecked(bool IsDeref, QualType SrcRecordTy) override;
void EmitBadTypeidCall(CodeGenFunction &CGF) override;
llvm::Value *EmitTypeid(CodeGenFunction &CGF, QualType SrcRecordTy,
- llvm::Value *ThisPtr,
+ Address ThisPtr,
llvm::Type *StdTypeInfoPtrTy) override;
bool shouldDynamicCastCallBeNullChecked(bool SrcIsPtr,
QualType SrcRecordTy) override;
- llvm::Value *EmitDynamicCastCall(CodeGenFunction &CGF, llvm::Value *Value,
+ llvm::Value *EmitDynamicCastCall(CodeGenFunction &CGF, Address Value,
QualType SrcRecordTy, QualType DestTy,
QualType DestRecordTy,
llvm::BasicBlock *CastEnd) override;
- llvm::Value *EmitDynamicCastToVoid(CodeGenFunction &CGF, llvm::Value *Value,
+ llvm::Value *EmitDynamicCastToVoid(CodeGenFunction &CGF, Address Value,
QualType SrcRecordTy,
QualType DestTy) override;
bool EmitBadCastCall(CodeGenFunction &CGF) override;
+ bool canSpeculativelyEmitVTable(const CXXRecordDecl *RD) const override {
+ return false;
+ }
llvm::Value *
- GetVirtualBaseClassOffset(CodeGenFunction &CGF, llvm::Value *This,
+ GetVirtualBaseClassOffset(CodeGenFunction &CGF, Address This,
const CXXRecordDecl *ClassDecl,
const CXXRecordDecl *BaseClassDecl) override;
@@ -182,9 +231,9 @@
return MD->getParent();
}
- llvm::Value *
+ Address
adjustThisArgumentForVirtualFunctionCall(CodeGenFunction &CGF, GlobalDecl GD,
- llvm::Value *This,
+ Address This,
bool VirtualCall) override;
void addImplicitStructorParams(CodeGenFunction &CGF, QualType &ResTy,
@@ -203,15 +252,30 @@
void EmitDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *DD,
CXXDtorType Type, bool ForVirtualBase,
- bool Delegating, llvm::Value *This) override;
+ bool Delegating, Address This) override;
+
+ void emitVTableBitSetEntries(VPtrInfo *Info, const CXXRecordDecl *RD,
+ llvm::GlobalVariable *VTable);
void emitVTableDefinitions(CodeGenVTables &CGVT,
const CXXRecordDecl *RD) override;
+ bool isVirtualOffsetNeededForVTableField(CodeGenFunction &CGF,
+ CodeGenFunction::VPtr Vptr) override;
+
+ /// Don't initialize vptrs if dynamic class
+ /// is marked with with the 'novtable' attribute.
+ bool doStructorsInitializeVPtrs(const CXXRecordDecl *VTableClass) override {
+ return !VTableClass->hasAttr<MSNoVTableAttr>();
+ }
+
+ llvm::Constant *
+ getVTableAddressPoint(BaseSubobject Base,
+ const CXXRecordDecl *VTableClass) override;
+
llvm::Value *getVTableAddressPointInStructor(
CodeGenFunction &CGF, const CXXRecordDecl *VTableClass,
- BaseSubobject Base, const CXXRecordDecl *NearestVBase,
- bool &NeedsVirtualOffset) override;
+ BaseSubobject Base, const CXXRecordDecl *NearestVBase) override;
llvm::Constant *
getVTableAddressPointForConstExpr(BaseSubobject Base,
@@ -221,13 +285,13 @@
CharUnits VPtrOffset) override;
llvm::Value *getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD,
- llvm::Value *This,
- llvm::Type *Ty) override;
+ Address This, llvm::Type *Ty,
+ SourceLocation Loc) override;
llvm::Value *EmitVirtualDestructorCall(CodeGenFunction &CGF,
const CXXDestructorDecl *Dtor,
CXXDtorType DtorType,
- llvm::Value *This,
+ Address This,
const CXXMemberCallExpr *CE) override;
void adjustCallArgsForDestructorThunk(CodeGenFunction &CGF, GlobalDecl GD,
@@ -244,6 +308,49 @@
getAddrOfVBTable(const VPtrInfo &VBT, const CXXRecordDecl *RD,
llvm::GlobalVariable::LinkageTypes Linkage);
+ llvm::GlobalVariable *
+ getAddrOfVirtualDisplacementMap(const CXXRecordDecl *SrcRD,
+ const CXXRecordDecl *DstRD) {
+ SmallString<256> OutName;
+ llvm::raw_svector_ostream Out(OutName);
+ getMangleContext().mangleCXXVirtualDisplacementMap(SrcRD, DstRD, Out);
+ StringRef MangledName = OutName.str();
+
+ if (auto *VDispMap = CGM.getModule().getNamedGlobal(MangledName))
+ return VDispMap;
+
+ MicrosoftVTableContext &VTContext = CGM.getMicrosoftVTableContext();
+ unsigned NumEntries = 1 + SrcRD->getNumVBases();
+ SmallVector<llvm::Constant *, 4> Map(NumEntries,
+ llvm::UndefValue::get(CGM.IntTy));
+ Map[0] = llvm::ConstantInt::get(CGM.IntTy, 0);
+ bool AnyDifferent = false;
+ for (const auto &I : SrcRD->vbases()) {
+ const CXXRecordDecl *VBase = I.getType()->getAsCXXRecordDecl();
+ if (!DstRD->isVirtuallyDerivedFrom(VBase))
+ continue;
+
+ unsigned SrcVBIndex = VTContext.getVBTableIndex(SrcRD, VBase);
+ unsigned DstVBIndex = VTContext.getVBTableIndex(DstRD, VBase);
+ Map[SrcVBIndex] = llvm::ConstantInt::get(CGM.IntTy, DstVBIndex * 4);
+ AnyDifferent |= SrcVBIndex != DstVBIndex;
+ }
+ // This map would be useless, don't use it.
+ if (!AnyDifferent)
+ return nullptr;
+
+ llvm::ArrayType *VDispMapTy = llvm::ArrayType::get(CGM.IntTy, Map.size());
+ llvm::Constant *Init = llvm::ConstantArray::get(VDispMapTy, Map);
+ llvm::GlobalValue::LinkageTypes Linkage =
+ SrcRD->isExternallyVisible() && DstRD->isExternallyVisible()
+ ? llvm::GlobalValue::LinkOnceODRLinkage
+ : llvm::GlobalValue::InternalLinkage;
+ auto *VDispMap = new llvm::GlobalVariable(
+ CGM.getModule(), VDispMapTy, /*Constant=*/true, Linkage,
+ /*Initializer=*/Init, MangledName);
+ return VDispMap;
+ }
+
void emitVBTableDefinition(const VPtrInfo &VBT, const CXXRecordDecl *RD,
llvm::GlobalVariable *GV) const;
@@ -263,18 +370,16 @@
Thunk->setLinkage(llvm::GlobalValue::LinkOnceODRLinkage);
}
- llvm::Value *performThisAdjustment(CodeGenFunction &CGF, llvm::Value *This,
+ llvm::Value *performThisAdjustment(CodeGenFunction &CGF, Address This,
const ThisAdjustment &TA) override;
- llvm::Value *performReturnAdjustment(CodeGenFunction &CGF, llvm::Value *Ret,
+ llvm::Value *performReturnAdjustment(CodeGenFunction &CGF, Address Ret,
const ReturnAdjustment &RA) override;
void EmitThreadLocalInitFuncs(
- CodeGenModule &CGM,
- ArrayRef<std::pair<const VarDecl *, llvm::GlobalVariable *>>
- CXXThreadLocals,
+ CodeGenModule &CGM, ArrayRef<const VarDecl *> CXXThreadLocals,
ArrayRef<llvm::Function *> CXXThreadLocalInits,
- ArrayRef<llvm::GlobalVariable *> CXXThreadLocalInitVars) override;
+ ArrayRef<const VarDecl *> CXXThreadLocalInitVars) override;
bool usesThreadWrapperFunction() const override { return false; }
LValue EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, const VarDecl *VD,
@@ -313,13 +418,13 @@
QualType elementType) override;
bool requiresArrayCookie(const CXXNewExpr *expr) override;
CharUnits getArrayCookieSizeImpl(QualType type) override;
- llvm::Value *InitializeArrayCookie(CodeGenFunction &CGF,
- llvm::Value *NewPtr,
- llvm::Value *NumElements,
- const CXXNewExpr *expr,
- QualType ElementType) override;
+ Address InitializeArrayCookie(CodeGenFunction &CGF,
+ Address NewPtr,
+ llvm::Value *NumElements,
+ const CXXNewExpr *expr,
+ QualType ElementType) override;
llvm::Value *readArrayCookieImpl(CodeGenFunction &CGF,
- llvm::Value *allocPtr,
+ Address allocPtr,
CharUnits cookieSize) override;
friend struct MSRTTIBuilder;
@@ -446,14 +551,6 @@
return llvm::Constant::getAllOnesValue(CGM.IntTy);
}
- llvm::Constant *getConstantOrZeroInt(llvm::Constant *C) {
- return C ? C : getZeroInt();
- }
-
- llvm::Value *getValueOrZeroInt(llvm::Value *C) {
- return C ? C : getZeroInt();
- }
-
CharUnits getVirtualFunctionPrologueThisAdjustment(GlobalDecl GD);
void
@@ -464,13 +561,13 @@
/// the vbptr to the virtual base. Optionally returns the address of the
/// vbptr itself.
llvm::Value *GetVBaseOffsetFromVBPtr(CodeGenFunction &CGF,
- llvm::Value *Base,
+ Address Base,
llvm::Value *VBPtrOffset,
llvm::Value *VBTableOffset,
llvm::Value **VBPtr = nullptr);
llvm::Value *GetVBaseOffsetFromVBPtr(CodeGenFunction &CGF,
- llvm::Value *Base,
+ Address Base,
int32_t VBPtrOffset,
int32_t VBTableOffset,
llvm::Value **VBPtr = nullptr) {
@@ -480,14 +577,14 @@
return GetVBaseOffsetFromVBPtr(CGF, Base, VBPOffset, VBTOffset, VBPtr);
}
- std::pair<llvm::Value *, llvm::Value *>
- performBaseAdjustment(CodeGenFunction &CGF, llvm::Value *Value,
+ std::pair<Address, llvm::Value *>
+ performBaseAdjustment(CodeGenFunction &CGF, Address Value,
QualType SrcRecordTy);
/// \brief Performs a full virtual base adjustment. Used to dereference
/// pointers to members of virtual bases.
llvm::Value *AdjustVirtualBase(CodeGenFunction &CGF, const Expr *E,
- const CXXRecordDecl *RD, llvm::Value *Base,
+ const CXXRecordDecl *RD, Address Base,
llvm::Value *VirtualBaseAdjustmentOffset,
llvm::Value *VBPtrOffset /* optional */);
@@ -499,10 +596,6 @@
CharUnits NonVirtualBaseAdjustment,
unsigned VBTableIndex);
- llvm::Constant *BuildMemberPointer(const CXXRecordDecl *RD,
- const CXXMethodDecl *MD,
- CharUnits NonVirtualBaseAdjustment);
-
bool MemberPointerConstantIsNull(const MemberPointerType *MPT,
llvm::Constant *MP);
@@ -527,22 +620,11 @@
return RD->hasAttr<MSInheritanceAttr>();
}
- bool isTypeInfoCalculable(QualType Ty) const override {
- if (!CGCXXABI::isTypeInfoCalculable(Ty))
- return false;
- if (const auto *MPT = Ty->getAs<MemberPointerType>()) {
- const CXXRecordDecl *RD = MPT->getMostRecentCXXRecordDecl();
- if (!RD->hasAttr<MSInheritanceAttr>())
- return false;
- }
- return true;
- }
-
llvm::Constant *EmitNullMemberPointer(const MemberPointerType *MPT) override;
llvm::Constant *EmitMemberDataPointer(const MemberPointerType *MPT,
CharUnits offset) override;
- llvm::Constant *EmitMemberPointer(const CXXMethodDecl *MD) override;
+ llvm::Constant *EmitMemberFunctionPointer(const CXXMethodDecl *MD) override;
llvm::Constant *EmitMemberPointer(const APValue &MP, QualType MPT) override;
llvm::Value *EmitMemberPointerComparison(CodeGenFunction &CGF,
@@ -557,9 +639,15 @@
llvm::Value *
EmitMemberDataPointerAddress(CodeGenFunction &CGF, const Expr *E,
- llvm::Value *Base, llvm::Value *MemPtr,
+ Address Base, llvm::Value *MemPtr,
const MemberPointerType *MPT) override;
+ llvm::Value *EmitNonNullMemberPointerConversion(
+ const MemberPointerType *SrcTy, const MemberPointerType *DstTy,
+ CastKind CK, CastExpr::path_const_iterator PathBegin,
+ CastExpr::path_const_iterator PathEnd, llvm::Value *Src,
+ CGBuilderTy &Builder);
+
llvm::Value *EmitMemberPointerConversion(CodeGenFunction &CGF,
const CastExpr *E,
llvm::Value *Src) override;
@@ -567,25 +655,19 @@
llvm::Constant *EmitMemberPointerConversion(const CastExpr *E,
llvm::Constant *Src) override;
+ llvm::Constant *EmitMemberPointerConversion(
+ const MemberPointerType *SrcTy, const MemberPointerType *DstTy,
+ CastKind CK, CastExpr::path_const_iterator PathBegin,
+ CastExpr::path_const_iterator PathEnd, llvm::Constant *Src);
+
llvm::Value *
EmitLoadOfMemberFunctionPointer(CodeGenFunction &CGF, const Expr *E,
- llvm::Value *&This, llvm::Value *MemPtr,
+ Address This, llvm::Value *&ThisPtrForCall,
+ llvm::Value *MemPtr,
const MemberPointerType *MPT) override;
void emitCXXStructor(const CXXMethodDecl *MD, StructorType Type) override;
- llvm::StructType *getCatchHandlerTypeType() {
- if (!CatchHandlerTypeType) {
- llvm::Type *FieldTypes[] = {
- CGM.IntTy, // Flags
- CGM.Int8PtrTy, // TypeDescriptor
- };
- CatchHandlerTypeType = llvm::StructType::create(
- CGM.getLLVMContext(), FieldTypes, "eh.CatchHandlerType");
- }
- return CatchHandlerTypeType;
- }
-
llvm::StructType *getCatchableTypeType() {
if (CatchableTypeType)
return CatchableTypeType;
@@ -701,7 +783,6 @@
llvm::StructType *CatchableTypeType;
llvm::DenseMap<uint32_t, llvm::StructType *> CatchableTypeArrayTypeMap;
llvm::StructType *ThrowInfoType;
- llvm::StructType *CatchHandlerTypeType;
};
}
@@ -769,7 +850,7 @@
void MicrosoftCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF,
const CXXDeleteExpr *DE,
- llvm::Value *Ptr,
+ Address Ptr,
QualType ElementType,
const CXXDestructorDecl *Dtor) {
// FIXME: Provide a source location here even though there's no
@@ -794,11 +875,15 @@
}
namespace {
-struct CallEndCatchMSVC : EHScopeStack::Cleanup {
- CallEndCatchMSVC() {}
+struct CatchRetScope final : EHScopeStack::Cleanup {
+ llvm::CatchPadInst *CPI;
+
+ CatchRetScope(llvm::CatchPadInst *CPI) : CPI(CPI) {}
+
void Emit(CodeGenFunction &CGF, Flags flags) override {
- CGF.EmitNounwindRuntimeCall(
- CGF.CGM.getIntrinsic(llvm::Intrinsic::eh_endcatch));
+ llvm::BasicBlock *BB = CGF.createBasicBlock("catchret.dest");
+ CGF.Builder.CreateCatchRet(CPI, BB);
+ CGF.EmitBlock(BB);
}
};
}
@@ -808,50 +893,59 @@
// In the MS ABI, the runtime handles the copy, and the catch handler is
// responsible for destruction.
VarDecl *CatchParam = S->getExceptionDecl();
- llvm::Value *Exn = CGF.getExceptionFromSlot();
- llvm::Function *BeginCatch =
- CGF.CGM.getIntrinsic(llvm::Intrinsic::eh_begincatch);
+ llvm::BasicBlock *CatchPadBB = CGF.Builder.GetInsertBlock();
+ llvm::CatchPadInst *CPI =
+ cast<llvm::CatchPadInst>(CatchPadBB->getFirstNonPHI());
+ CGF.CurrentFuncletPad = CPI;
// If this is a catch-all or the catch parameter is unnamed, we don't need to
// emit an alloca to the object.
if (!CatchParam || !CatchParam->getDeclName()) {
- llvm::Value *Args[2] = {Exn, llvm::Constant::getNullValue(CGF.Int8PtrTy)};
- CGF.EmitNounwindRuntimeCall(BeginCatch, Args);
- CGF.EHStack.pushCleanup<CallEndCatchMSVC>(NormalCleanup);
+ CGF.EHStack.pushCleanup<CatchRetScope>(NormalCleanup, CPI);
return;
}
CodeGenFunction::AutoVarEmission var = CGF.EmitAutoVarAlloca(*CatchParam);
- llvm::Value *ParamAddr =
- CGF.Builder.CreateBitCast(var.getObjectAddress(CGF), CGF.Int8PtrTy);
- llvm::Value *Args[2] = {Exn, ParamAddr};
- CGF.EmitNounwindRuntimeCall(BeginCatch, Args);
- CGF.EHStack.pushCleanup<CallEndCatchMSVC>(NormalCleanup);
+ CPI->setArgOperand(2, var.getObjectAddress(CGF).getPointer());
+ CGF.EHStack.pushCleanup<CatchRetScope>(NormalCleanup, CPI);
CGF.EmitAutoVarCleanups(var);
}
-std::pair<llvm::Value *, llvm::Value *>
-MicrosoftCXXABI::performBaseAdjustment(CodeGenFunction &CGF, llvm::Value *Value,
+/// We need to perform a generic polymorphic operation (like a typeid
+/// or a cast), which requires an object with a vfptr. Adjust the
+/// address to point to an object with a vfptr.
+std::pair<Address, llvm::Value *>
+MicrosoftCXXABI::performBaseAdjustment(CodeGenFunction &CGF, Address Value,
QualType SrcRecordTy) {
Value = CGF.Builder.CreateBitCast(Value, CGF.Int8PtrTy);
const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl();
const ASTContext &Context = getContext();
+ // If the class itself has a vfptr, great. This check implicitly
+ // covers non-virtual base subobjects: a class with its own virtual
+ // functions would be a candidate to be a primary base.
if (Context.getASTRecordLayout(SrcDecl).hasExtendableVFPtr())
return std::make_pair(Value, llvm::ConstantInt::get(CGF.Int32Ty, 0));
- // Perform a base adjustment.
- const CXXBaseSpecifier *PolymorphicBase = std::find_if(
- SrcDecl->vbases_begin(), SrcDecl->vbases_end(),
- [&](const CXXBaseSpecifier &Base) {
- const CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl();
- return Context.getASTRecordLayout(BaseDecl).hasExtendableVFPtr();
- });
- llvm::Value *Offset = GetVirtualBaseClassOffset(
- CGF, Value, SrcDecl, PolymorphicBase->getType()->getAsCXXRecordDecl());
- Value = CGF.Builder.CreateInBoundsGEP(Value, Offset);
+ // Okay, one of the vbases must have a vfptr, or else this isn't
+ // actually a polymorphic class.
+ const CXXRecordDecl *PolymorphicBase = nullptr;
+ for (auto &Base : SrcDecl->vbases()) {
+ const CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl();
+ if (Context.getASTRecordLayout(BaseDecl).hasExtendableVFPtr()) {
+ PolymorphicBase = BaseDecl;
+ break;
+ }
+ }
+ assert(PolymorphicBase && "polymorphic class has no apparent vfptr?");
+
+ llvm::Value *Offset =
+ GetVirtualBaseClassOffset(CGF, Value, SrcDecl, PolymorphicBase);
+ llvm::Value *Ptr = CGF.Builder.CreateInBoundsGEP(Value.getPointer(), Offset);
Offset = CGF.Builder.CreateTrunc(Offset, CGF.Int32Ty);
- return std::make_pair(Value, Offset);
+ CharUnits VBaseAlign =
+ CGF.CGM.getVBaseAlignment(Value.getAlignment(), SrcDecl, PolymorphicBase);
+ return std::make_pair(Address(Ptr, VBaseAlign), Offset);
}
bool MicrosoftCXXABI::shouldTypeidBeNullChecked(bool IsDeref,
@@ -880,12 +974,12 @@
llvm::Value *MicrosoftCXXABI::EmitTypeid(CodeGenFunction &CGF,
QualType SrcRecordTy,
- llvm::Value *ThisPtr,
+ Address ThisPtr,
llvm::Type *StdTypeInfoPtrTy) {
llvm::Value *Offset;
std::tie(ThisPtr, Offset) = performBaseAdjustment(CGF, ThisPtr, SrcRecordTy);
- return CGF.Builder.CreateBitCast(
- emitRTtypeidCall(CGF, ThisPtr).getInstruction(), StdTypeInfoPtrTy);
+ auto Typeid = emitRTtypeidCall(CGF, ThisPtr.getPointer()).getInstruction();
+ return CGF.Builder.CreateBitCast(Typeid, StdTypeInfoPtrTy);
}
bool MicrosoftCXXABI::shouldDynamicCastCallBeNullChecked(bool SrcIsPtr,
@@ -896,7 +990,7 @@
}
llvm::Value *MicrosoftCXXABI::EmitDynamicCastCall(
- CodeGenFunction &CGF, llvm::Value *Value, QualType SrcRecordTy,
+ CodeGenFunction &CGF, Address This, QualType SrcRecordTy,
QualType DestTy, QualType DestRecordTy, llvm::BasicBlock *CastEnd) {
llvm::Type *DestLTy = CGF.ConvertType(DestTy);
@@ -906,7 +1000,8 @@
CGF.CGM.GetAddrOfRTTIDescriptor(DestRecordTy.getUnqualifiedType());
llvm::Value *Offset;
- std::tie(Value, Offset) = performBaseAdjustment(CGF, Value, SrcRecordTy);
+ std::tie(This, Offset) = performBaseAdjustment(CGF, This, SrcRecordTy);
+ llvm::Value *ThisPtr = This.getPointer();
// PVOID __RTDynamicCast(
// PVOID inptr,
@@ -920,14 +1015,14 @@
llvm::FunctionType::get(CGF.Int8PtrTy, ArgTypes, false),
"__RTDynamicCast");
llvm::Value *Args[] = {
- Value, Offset, SrcRTTI, DestRTTI,
+ ThisPtr, Offset, SrcRTTI, DestRTTI,
llvm::ConstantInt::get(CGF.Int32Ty, DestTy->isReferenceType())};
- Value = CGF.EmitRuntimeCallOrInvoke(Function, Args).getInstruction();
- return CGF.Builder.CreateBitCast(Value, DestLTy);
+ ThisPtr = CGF.EmitRuntimeCallOrInvoke(Function, Args).getInstruction();
+ return CGF.Builder.CreateBitCast(ThisPtr, DestLTy);
}
llvm::Value *
-MicrosoftCXXABI::EmitDynamicCastToVoid(CodeGenFunction &CGF, llvm::Value *Value,
+MicrosoftCXXABI::EmitDynamicCastToVoid(CodeGenFunction &CGF, Address Value,
QualType SrcRecordTy,
QualType DestTy) {
llvm::Value *Offset;
@@ -939,7 +1034,7 @@
llvm::Constant *Function = CGF.CGM.CreateRuntimeFunction(
llvm::FunctionType::get(CGF.Int8PtrTy, ArgTypes, false),
"__RTCastToVoid");
- llvm::Value *Args[] = {Value};
+ llvm::Value *Args[] = {Value.getPointer()};
return CGF.EmitRuntimeCall(Function, Args);
}
@@ -948,7 +1043,7 @@
}
llvm::Value *MicrosoftCXXABI::GetVirtualBaseClassOffset(
- CodeGenFunction &CGF, llvm::Value *This, const CXXRecordDecl *ClassDecl,
+ CodeGenFunction &CGF, Address This, const CXXRecordDecl *ClassDecl,
const CXXRecordDecl *BaseClassDecl) {
const ASTContext &Context = getContext();
int64_t VBPtrChars =
@@ -986,15 +1081,16 @@
if (!RD)
return false;
+ CharUnits Align = CGM.getContext().getTypeAlignInChars(FI.getReturnType());
if (FI.isInstanceMethod()) {
// If it's an instance method, aggregates are always returned indirectly via
// the second parameter.
- FI.getReturnInfo() = ABIArgInfo::getIndirect(0, /*ByVal=*/false);
+ FI.getReturnInfo() = ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
FI.getReturnInfo().setSRetAfterThis(FI.isInstanceMethod());
return true;
} else if (!RD->isPOD()) {
// If it's a free function, non-POD types are returned indirectly.
- FI.getReturnInfo() = ABIArgInfo::getIndirect(0, /*ByVal=*/false);
+ FI.getReturnInfo() = ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
return true;
}
@@ -1046,8 +1142,7 @@
const VBOffsets &VBaseMap = Layout.getVBaseOffsetsMap();
CGBuilderTy &Builder = CGF.Builder;
- unsigned AS =
- cast<llvm::PointerType>(getThisValue(CGF)->getType())->getAddressSpace();
+ unsigned AS = getThisAddress(CGF).getAddressSpace();
llvm::Value *Int8This = nullptr; // Initialize lazily.
for (VBOffsets::const_iterator I = VBaseMap.begin(), E = VBaseMap.end();
@@ -1056,7 +1151,7 @@
continue;
llvm::Value *VBaseOffset =
- GetVirtualBaseClassOffset(CGF, getThisValue(CGF), RD, I->first);
+ GetVirtualBaseClassOffset(CGF, getThisAddress(CGF), RD, I->first);
// FIXME: it doesn't look right that we SExt in GetVirtualBaseClassOffset()
// just to Trunc back immediately.
VBaseOffset = Builder.CreateTruncOrBitCast(VBaseOffset, CGF.Int32Ty);
@@ -1077,7 +1172,8 @@
VtorDispPtr = Builder.CreateBitCast(
VtorDispPtr, CGF.Int32Ty->getPointerTo(AS), "vtordisp.ptr");
- Builder.CreateStore(VtorDispValue, VtorDispPtr);
+ Builder.CreateAlignedStore(VtorDispValue, VtorDispPtr,
+ CharUnits::fromQuantity(4));
}
}
@@ -1108,8 +1204,8 @@
void MicrosoftCXXABI::EmitVBPtrStores(CodeGenFunction &CGF,
const CXXRecordDecl *RD) {
- llvm::Value *ThisInt8Ptr =
- CGF.Builder.CreateBitCast(getThisValue(CGF), CGM.Int8PtrTy, "this.int8");
+ Address This = getThisAddress(CGF);
+ This = CGF.Builder.CreateElementBitCast(This, CGM.Int8Ty, "this.int8");
const ASTContext &Context = getContext();
const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
@@ -1123,11 +1219,10 @@
Offs += SubobjectLayout.getVBPtrOffset();
if (VBT->getVBaseWithVPtr())
Offs += Layout.getVBaseClassOffset(VBT->getVBaseWithVPtr());
- llvm::Value *VBPtr =
- CGF.Builder.CreateConstInBoundsGEP1_64(ThisInt8Ptr, Offs.getQuantity());
+ Address VBPtr = CGF.Builder.CreateConstInBoundsByteGEP(This, Offs);
llvm::Value *GVPtr =
CGF.Builder.CreateConstInBoundsGEP2_32(GV->getValueType(), GV, 0, 0);
- VBPtr = CGF.Builder.CreateBitCast(VBPtr, GVPtr->getType()->getPointerTo(0),
+ VBPtr = CGF.Builder.CreateElementBitCast(VBPtr, GVPtr->getType(),
"vbptr." + VBT->ReusingBase->getName());
CGF.Builder.CreateStore(GVPtr, VBPtr);
}
@@ -1201,8 +1296,9 @@
return Adjustment;
}
-llvm::Value *MicrosoftCXXABI::adjustThisArgumentForVirtualFunctionCall(
- CodeGenFunction &CGF, GlobalDecl GD, llvm::Value *This, bool VirtualCall) {
+Address MicrosoftCXXABI::adjustThisArgumentForVirtualFunctionCall(
+ CodeGenFunction &CGF, GlobalDecl GD, Address This,
+ bool VirtualCall) {
if (!VirtualCall) {
// If the call of a virtual function is not virtual, we just have to
// compensate for the adjustment the virtual function does in its prologue.
@@ -1210,11 +1306,9 @@
if (Adjustment.isZero())
return This;
- unsigned AS = cast<llvm::PointerType>(This->getType())->getAddressSpace();
- llvm::Type *charPtrTy = CGF.Int8Ty->getPointerTo(AS);
- This = CGF.Builder.CreateBitCast(This, charPtrTy);
+ This = CGF.Builder.CreateElementBitCast(This, CGF.Int8Ty);
assert(Adjustment.isPositive());
- return CGF.Builder.CreateConstGEP1_32(This, Adjustment.getQuantity());
+ return CGF.Builder.CreateConstByteGEP(This, Adjustment);
}
GD = GD.getCanonicalDecl();
@@ -1234,8 +1328,6 @@
MicrosoftVTableContext::MethodVFTableLocation ML =
CGM.getMicrosoftVTableContext().getMethodVFTableLocation(LookupGD);
- unsigned AS = cast<llvm::PointerType>(This->getType())->getAddressSpace();
- llvm::Type *charPtrTy = CGF.Int8Ty->getPointerTo(AS);
CharUnits StaticOffset = ML.VFPtrOffset;
// Base destructors expect 'this' to point to the beginning of the base
@@ -1244,27 +1336,34 @@
if (isa<CXXDestructorDecl>(MD) && GD.getDtorType() == Dtor_Base)
StaticOffset = CharUnits::Zero();
+ Address Result = This;
if (ML.VBase) {
- This = CGF.Builder.CreateBitCast(This, charPtrTy);
+ Result = CGF.Builder.CreateElementBitCast(Result, CGF.Int8Ty);
+
+ const CXXRecordDecl *Derived = MD->getParent();
+ const CXXRecordDecl *VBase = ML.VBase;
llvm::Value *VBaseOffset =
- GetVirtualBaseClassOffset(CGF, This, MD->getParent(), ML.VBase);
- This = CGF.Builder.CreateInBoundsGEP(This, VBaseOffset);
+ GetVirtualBaseClassOffset(CGF, Result, Derived, VBase);
+ llvm::Value *VBasePtr =
+ CGF.Builder.CreateInBoundsGEP(Result.getPointer(), VBaseOffset);
+ CharUnits VBaseAlign =
+ CGF.CGM.getVBaseAlignment(Result.getAlignment(), Derived, VBase);
+ Result = Address(VBasePtr, VBaseAlign);
}
if (!StaticOffset.isZero()) {
assert(StaticOffset.isPositive());
- This = CGF.Builder.CreateBitCast(This, charPtrTy);
+ Result = CGF.Builder.CreateElementBitCast(Result, CGF.Int8Ty);
if (ML.VBase) {
// Non-virtual adjustment might result in a pointer outside the allocated
// object, e.g. if the final overrider class is laid out after the virtual
// base that declares a method in the most derived class.
// FIXME: Update the code that emits this adjustment in thunks prologues.
- This = CGF.Builder.CreateConstGEP1_32(This, StaticOffset.getQuantity());
+ Result = CGF.Builder.CreateConstByteGEP(Result, StaticOffset);
} else {
- This = CGF.Builder.CreateConstInBoundsGEP1_32(CGF.Int8Ty, This,
- StaticOffset.getQuantity());
+ Result = CGF.Builder.CreateConstInBoundsByteGEP(Result, StaticOffset);
}
}
- return This;
+ return Result;
}
void MicrosoftCXXABI::addImplicitStructorParams(CodeGenFunction &CGF,
@@ -1385,7 +1484,7 @@
void MicrosoftCXXABI::EmitDestructorCall(CodeGenFunction &CGF,
const CXXDestructorDecl *DD,
CXXDtorType Type, bool ForVirtualBase,
- bool Delegating, llvm::Value *This) {
+ bool Delegating, Address This) {
llvm::Value *Callee = CGM.getAddrOfCXXStructor(DD, getFromDtorType(Type));
if (DD->isVirtual()) {
@@ -1395,12 +1494,69 @@
This, false);
}
- CGF.EmitCXXStructorCall(DD, Callee, ReturnValueSlot(), This,
+ CGF.EmitCXXStructorCall(DD, Callee, ReturnValueSlot(), This.getPointer(),
/*ImplicitParam=*/nullptr,
/*ImplicitParamTy=*/QualType(), nullptr,
getFromDtorType(Type));
}
+void MicrosoftCXXABI::emitVTableBitSetEntries(VPtrInfo *Info,
+ const CXXRecordDecl *RD,
+ llvm::GlobalVariable *VTable) {
+ if (!getContext().getLangOpts().Sanitize.has(SanitizerKind::CFIVCall) &&
+ !getContext().getLangOpts().Sanitize.has(SanitizerKind::CFINVCall) &&
+ !getContext().getLangOpts().Sanitize.has(SanitizerKind::CFIDerivedCast) &&
+ !getContext().getLangOpts().Sanitize.has(SanitizerKind::CFIUnrelatedCast))
+ return;
+
+ llvm::NamedMDNode *BitsetsMD =
+ CGM.getModule().getOrInsertNamedMetadata("llvm.bitsets");
+
+ // The location of the first virtual function pointer in the virtual table,
+ // aka the "address point" on Itanium. This is at offset 0 if RTTI is
+ // disabled, or sizeof(void*) if RTTI is enabled.
+ CharUnits AddressPoint =
+ getContext().getLangOpts().RTTIData
+ ? getContext().toCharUnitsFromBits(
+ getContext().getTargetInfo().getPointerWidth(0))
+ : CharUnits::Zero();
+
+ if (Info->PathToBaseWithVPtr.empty()) {
+ if (!CGM.IsCFIBlacklistedRecord(RD))
+ CGM.CreateVTableBitSetEntry(BitsetsMD, VTable, AddressPoint, RD);
+ return;
+ }
+
+ // Add a bitset entry for the least derived base belonging to this vftable.
+ if (!CGM.IsCFIBlacklistedRecord(Info->PathToBaseWithVPtr.back()))
+ CGM.CreateVTableBitSetEntry(BitsetsMD, VTable, AddressPoint,
+ Info->PathToBaseWithVPtr.back());
+
+ // Add a bitset entry for each derived class that is laid out at the same
+ // offset as the least derived base.
+ for (unsigned I = Info->PathToBaseWithVPtr.size() - 1; I != 0; --I) {
+ const CXXRecordDecl *DerivedRD = Info->PathToBaseWithVPtr[I - 1];
+ const CXXRecordDecl *BaseRD = Info->PathToBaseWithVPtr[I];
+
+ const ASTRecordLayout &Layout =
+ getContext().getASTRecordLayout(DerivedRD);
+ CharUnits Offset;
+ auto VBI = Layout.getVBaseOffsetsMap().find(BaseRD);
+ if (VBI == Layout.getVBaseOffsetsMap().end())
+ Offset = Layout.getBaseClassOffset(BaseRD);
+ else
+ Offset = VBI->second.VBaseOffset;
+ if (!Offset.isZero())
+ return;
+ if (!CGM.IsCFIBlacklistedRecord(DerivedRD))
+ CGM.CreateVTableBitSetEntry(BitsetsMD, VTable, AddressPoint, DerivedRD);
+ }
+
+ // Finally do the same for the most derived class.
+ if (Info->FullOffsetInMDC.isZero() && !CGM.IsCFIBlacklistedRecord(RD))
+ CGM.CreateVTableBitSetEntry(BitsetsMD, VTable, AddressPoint, RD);
+}
+
void MicrosoftCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT,
const CXXRecordDecl *RD) {
MicrosoftVTableContext &VFTContext = CGM.getMicrosoftVTableContext();
@@ -1423,17 +1579,20 @@
VTLayout.getNumVTableThunks(), RTTI);
VTable->setInitializer(Init);
+
+ emitVTableBitSetEntries(Info, RD, VTable);
}
}
+bool MicrosoftCXXABI::isVirtualOffsetNeededForVTableField(
+ CodeGenFunction &CGF, CodeGenFunction::VPtr Vptr) {
+ return Vptr.NearestVBase != nullptr;
+}
+
llvm::Value *MicrosoftCXXABI::getVTableAddressPointInStructor(
CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, BaseSubobject Base,
- const CXXRecordDecl *NearestVBase, bool &NeedsVirtualOffset) {
- NeedsVirtualOffset = (NearestVBase != nullptr);
-
- (void)getAddrOfVTable(VTableClass, Base.getBaseOffset());
- VFTableIdTy ID(VTableClass, Base.getBaseOffset());
- llvm::GlobalValue *VTableAddressPoint = VFTablesMap[ID];
+ const CXXRecordDecl *NearestVBase) {
+ llvm::Constant *VTableAddressPoint = getVTableAddressPoint(Base, VTableClass);
if (!VTableAddressPoint) {
assert(Base.getBase()->getNumVBases() &&
!getContext().getASTRecordLayout(Base.getBase()).hasOwnVFPtr());
@@ -1448,11 +1607,17 @@
MangleContext.mangleCXXVFTable(RD, VFPtr->MangledPath, Out);
}
-llvm::Constant *MicrosoftCXXABI::getVTableAddressPointForConstExpr(
- BaseSubobject Base, const CXXRecordDecl *VTableClass) {
+llvm::Constant *
+MicrosoftCXXABI::getVTableAddressPoint(BaseSubobject Base,
+ const CXXRecordDecl *VTableClass) {
(void)getAddrOfVTable(VTableClass, Base.getBaseOffset());
VFTableIdTy ID(VTableClass, Base.getBaseOffset());
- llvm::GlobalValue *VFTable = VFTablesMap[ID];
+ return VFTablesMap[ID];
+}
+
+llvm::Constant *MicrosoftCXXABI::getVTableAddressPointForConstExpr(
+ BaseSubobject Base, const CXXRecordDecl *VTableClass) {
+ llvm::Constant *VFTable = getVTableAddressPoint(Base, VTableClass);
assert(VFTable && "Couldn't find a vftable for the given base?");
return VFTable;
}
@@ -1462,6 +1627,7 @@
// getAddrOfVTable may return 0 if asked to get an address of a vtable which
// shouldn't be used in the given record type. We want to cache this result in
// VFTablesMap, thus a simple zero check is not sufficient.
+
VFTableIdTy ID(RD, VPtrOffset);
VTablesMapTy::iterator I;
bool Inserted;
@@ -1515,10 +1681,11 @@
if (llvm::GlobalValue *VFTable =
CGM.getModule().getNamedGlobal(VFTableName)) {
VFTablesMap[ID] = VFTable;
- return VTableAliasIsRequred
- ? cast<llvm::GlobalVariable>(
- cast<llvm::GlobalAlias>(VFTable)->getBaseObject())
- : cast<llvm::GlobalVariable>(VFTable);
+ VTable = VTableAliasIsRequred
+ ? cast<llvm::GlobalVariable>(
+ cast<llvm::GlobalAlias>(VFTable)->getBaseObject())
+ : cast<llvm::GlobalVariable>(VFTable);
+ return VTable;
}
uint64_t NumVTableSlots =
@@ -1562,9 +1729,10 @@
if (C)
C->setSelectionKind(llvm::Comdat::Largest);
}
- VFTable = llvm::GlobalAlias::create(
- cast<llvm::PointerType>(VTableGEP->getType()), VFTableLinkage,
- VFTableName.str(), VTableGEP, &CGM.getModule());
+ VFTable = llvm::GlobalAlias::create(CGM.Int8PtrTy,
+ /*AddressSpace=*/0, VFTableLinkage,
+ VFTableName.str(), VTableGEP,
+ &CGM.getModule());
VFTable->setUnnamedAddr(true);
} else {
// We don't need a GlobalAlias to be a symbol for the VTable if we won't
@@ -1585,28 +1753,78 @@
return VTable;
}
+// Compute the identity of the most derived class whose virtual table is located
+// at the given offset into RD.
+static const CXXRecordDecl *getClassAtVTableLocation(ASTContext &Ctx,
+ const CXXRecordDecl *RD,
+ CharUnits Offset) {
+ if (Offset.isZero())
+ return RD;
+
+ const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(RD);
+ const CXXRecordDecl *MaxBase = nullptr;
+ CharUnits MaxBaseOffset;
+ for (auto &&B : RD->bases()) {
+ const CXXRecordDecl *Base = B.getType()->getAsCXXRecordDecl();
+ CharUnits BaseOffset = Layout.getBaseClassOffset(Base);
+ if (BaseOffset <= Offset && BaseOffset >= MaxBaseOffset) {
+ MaxBase = Base;
+ MaxBaseOffset = BaseOffset;
+ }
+ }
+ for (auto &&B : RD->vbases()) {
+ const CXXRecordDecl *Base = B.getType()->getAsCXXRecordDecl();
+ CharUnits BaseOffset = Layout.getVBaseClassOffset(Base);
+ if (BaseOffset <= Offset && BaseOffset >= MaxBaseOffset) {
+ MaxBase = Base;
+ MaxBaseOffset = BaseOffset;
+ }
+ }
+ assert(MaxBase);
+ return getClassAtVTableLocation(Ctx, MaxBase, Offset - MaxBaseOffset);
+}
+
+// Compute the identity of the most derived class whose virtual table is located
+// at the MethodVFTableLocation ML.
+static const CXXRecordDecl *
+getClassAtVTableLocation(ASTContext &Ctx, GlobalDecl GD,
+ MicrosoftVTableContext::MethodVFTableLocation &ML) {
+ const CXXRecordDecl *RD = ML.VBase;
+ if (!RD)
+ RD = cast<CXXMethodDecl>(GD.getDecl())->getParent();
+
+ return getClassAtVTableLocation(Ctx, RD, ML.VFPtrOffset);
+}
+
llvm::Value *MicrosoftCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF,
GlobalDecl GD,
- llvm::Value *This,
- llvm::Type *Ty) {
+ Address This,
+ llvm::Type *Ty,
+ SourceLocation Loc) {
GD = GD.getCanonicalDecl();
CGBuilderTy &Builder = CGF.Builder;
Ty = Ty->getPointerTo()->getPointerTo();
- llvm::Value *VPtr =
+ Address VPtr =
adjustThisArgumentForVirtualFunctionCall(CGF, GD, This, true);
- llvm::Value *VTable = CGF.GetVTablePtr(VPtr, Ty);
+
+ auto *MethodDecl = cast<CXXMethodDecl>(GD.getDecl());
+ llvm::Value *VTable = CGF.GetVTablePtr(VPtr, Ty, MethodDecl->getParent());
MicrosoftVTableContext::MethodVFTableLocation ML =
CGM.getMicrosoftVTableContext().getMethodVFTableLocation(GD);
+ if (CGF.SanOpts.has(SanitizerKind::CFIVCall))
+ CGF.EmitVTablePtrCheck(getClassAtVTableLocation(getContext(), GD, ML),
+ VTable, CodeGenFunction::CFITCK_VCall, Loc);
+
llvm::Value *VFuncPtr =
Builder.CreateConstInBoundsGEP1_64(VTable, ML.Index, "vfn");
- return Builder.CreateLoad(VFuncPtr);
+ return Builder.CreateAlignedLoad(VFuncPtr, CGF.getPointerAlign());
}
llvm::Value *MicrosoftCXXABI::EmitVirtualDestructorCall(
CodeGenFunction &CGF, const CXXDestructorDecl *Dtor, CXXDtorType DtorType,
- llvm::Value *This, const CXXMemberCallExpr *CE) {
+ Address This, const CXXMemberCallExpr *CE) {
assert(CE == nullptr || CE->arg_begin() == CE->arg_end());
assert(DtorType == Dtor_Deleting || DtorType == Dtor_Complete);
@@ -1616,7 +1834,8 @@
const CGFunctionInfo *FInfo = &CGM.getTypes().arrangeCXXStructorDeclaration(
Dtor, StructorType::Deleting);
llvm::Type *Ty = CGF.CGM.getTypes().GetFunctionType(*FInfo);
- llvm::Value *Callee = getVirtualFunctionPointer(CGF, GD, This, Ty);
+ llvm::Value *Callee = getVirtualFunctionPointer(
+ CGF, GD, This, Ty, CE ? CE->getLocStart() : SourceLocation());
ASTContext &Context = getContext();
llvm::Value *ImplicitParam = llvm::ConstantInt::get(
@@ -1624,7 +1843,8 @@
DtorType == Dtor_Deleting);
This = adjustThisArgumentForVirtualFunctionCall(CGF, GD, This, true);
- RValue RV = CGF.EmitCXXStructorCall(Dtor, Callee, ReturnValueSlot(), This,
+ RValue RV = CGF.EmitCXXStructorCall(Dtor, Callee, ReturnValueSlot(),
+ This.getPointer(),
ImplicitParam, Context.IntTy, CE,
StructorType::Deleting);
return RV.getScalarVal();
@@ -1667,7 +1887,6 @@
SmallString<256> ThunkName;
llvm::raw_svector_ostream Out(ThunkName);
getMangleContext().mangleVirtualMemPtrThunk(MD, Out);
- Out.flush();
// If the thunk has been generated previously, just return it.
if (llvm::GlobalValue *GV = CGM.getModule().getNamedValue(ThunkName))
@@ -1717,10 +1936,12 @@
// Load the vfptr and then callee from the vftable. The callee should have
// adjusted 'this' so that the vfptr is at offset zero.
llvm::Value *VTable = CGF.GetVTablePtr(
- getThisValue(CGF), ThunkTy->getPointerTo()->getPointerTo());
+ getThisAddress(CGF), ThunkTy->getPointerTo()->getPointerTo(), MD->getParent());
+
llvm::Value *VFuncPtr =
CGF.Builder.CreateConstInBoundsGEP1_64(VTable, ML.Index, "vfn");
- llvm::Value *Callee = CGF.Builder.CreateLoad(VFuncPtr);
+ llvm::Value *Callee =
+ CGF.Builder.CreateAlignedLoad(VFuncPtr, CGF.getPointerAlign());
CGF.EmitMustTailThunk(MD, getThisValue(CGF), Callee);
@@ -1743,7 +1964,6 @@
SmallString<256> OutName;
llvm::raw_svector_ostream Out(OutName);
getMangleContext().mangleCXXVBTable(RD, VBT.MangledPath, Out);
- Out.flush();
StringRef Name = OutName.str();
llvm::ArrayType *VBTableType =
@@ -1813,22 +2033,30 @@
}
llvm::Value *MicrosoftCXXABI::performThisAdjustment(CodeGenFunction &CGF,
- llvm::Value *This,
+ Address This,
const ThisAdjustment &TA) {
if (TA.isEmpty())
- return This;
+ return This.getPointer();
- llvm::Value *V = CGF.Builder.CreateBitCast(This, CGF.Int8PtrTy);
+ This = CGF.Builder.CreateElementBitCast(This, CGF.Int8Ty);
- if (!TA.Virtual.isEmpty()) {
+ llvm::Value *V;
+ if (TA.Virtual.isEmpty()) {
+ V = This.getPointer();
+ } else {
assert(TA.Virtual.Microsoft.VtordispOffset < 0);
// Adjust the this argument based on the vtordisp value.
- llvm::Value *VtorDispPtr =
- CGF.Builder.CreateConstGEP1_32(V, TA.Virtual.Microsoft.VtordispOffset);
- VtorDispPtr =
- CGF.Builder.CreateBitCast(VtorDispPtr, CGF.Int32Ty->getPointerTo());
+ Address VtorDispPtr =
+ CGF.Builder.CreateConstInBoundsByteGEP(This,
+ CharUnits::fromQuantity(TA.Virtual.Microsoft.VtordispOffset));
+ VtorDispPtr = CGF.Builder.CreateElementBitCast(VtorDispPtr, CGF.Int32Ty);
llvm::Value *VtorDisp = CGF.Builder.CreateLoad(VtorDispPtr, "vtordisp");
- V = CGF.Builder.CreateGEP(V, CGF.Builder.CreateNeg(VtorDisp));
+ V = CGF.Builder.CreateGEP(This.getPointer(),
+ CGF.Builder.CreateNeg(VtorDisp));
+
+ // Unfortunately, having applied the vtordisp means that we no
+ // longer really have a known alignment for the vbptr step.
+ // We'll assume the vbptr is pointer-aligned.
if (TA.Virtual.Microsoft.VBPtrOffset) {
// If the final overrider is defined in a virtual base other than the one
@@ -1838,7 +2066,8 @@
assert(TA.Virtual.Microsoft.VBOffsetOffset >= 0);
llvm::Value *VBPtr;
llvm::Value *VBaseOffset =
- GetVBaseOffsetFromVBPtr(CGF, V, -TA.Virtual.Microsoft.VBPtrOffset,
+ GetVBaseOffsetFromVBPtr(CGF, Address(V, CGF.getPointerAlign()),
+ -TA.Virtual.Microsoft.VBPtrOffset,
TA.Virtual.Microsoft.VBOffsetOffset, &VBPtr);
V = CGF.Builder.CreateInBoundsGEP(VBPtr, VBaseOffset);
}
@@ -1856,20 +2085,21 @@
}
llvm::Value *
-MicrosoftCXXABI::performReturnAdjustment(CodeGenFunction &CGF, llvm::Value *Ret,
+MicrosoftCXXABI::performReturnAdjustment(CodeGenFunction &CGF, Address Ret,
const ReturnAdjustment &RA) {
if (RA.isEmpty())
- return Ret;
+ return Ret.getPointer();
- llvm::Value *V = CGF.Builder.CreateBitCast(Ret, CGF.Int8PtrTy);
+ auto OrigTy = Ret.getType();
+ Ret = CGF.Builder.CreateElementBitCast(Ret, CGF.Int8Ty);
+ llvm::Value *V = Ret.getPointer();
if (RA.Virtual.Microsoft.VBIndex) {
assert(RA.Virtual.Microsoft.VBIndex > 0);
- const ASTContext &Context = getContext();
- int32_t IntSize = Context.getTypeSizeInChars(Context.IntTy).getQuantity();
+ int32_t IntSize = CGF.getIntSize().getQuantity();
llvm::Value *VBPtr;
llvm::Value *VBaseOffset =
- GetVBaseOffsetFromVBPtr(CGF, V, RA.Virtual.Microsoft.VBPtrOffset,
+ GetVBaseOffsetFromVBPtr(CGF, Ret, RA.Virtual.Microsoft.VBPtrOffset,
IntSize * RA.Virtual.Microsoft.VBIndex, &VBPtr);
V = CGF.Builder.CreateInBoundsGEP(VBPtr, VBaseOffset);
}
@@ -1878,7 +2108,7 @@
V = CGF.Builder.CreateConstInBoundsGEP1_32(CGF.Int8Ty, V, RA.NonVirtual);
// Cast back to the original type.
- return CGF.Builder.CreateBitCast(V, Ret->getType());
+ return CGF.Builder.CreateBitCast(V, OrigTy);
}
bool MicrosoftCXXABI::requiresArrayCookie(const CXXDeleteExpr *expr,
@@ -1903,37 +2133,34 @@
}
llvm::Value *MicrosoftCXXABI::readArrayCookieImpl(CodeGenFunction &CGF,
- llvm::Value *allocPtr,
+ Address allocPtr,
CharUnits cookieSize) {
- unsigned AS = allocPtr->getType()->getPointerAddressSpace();
- llvm::Value *numElementsPtr =
- CGF.Builder.CreateBitCast(allocPtr, CGF.SizeTy->getPointerTo(AS));
+ Address numElementsPtr =
+ CGF.Builder.CreateElementBitCast(allocPtr, CGF.SizeTy);
return CGF.Builder.CreateLoad(numElementsPtr);
}
-llvm::Value* MicrosoftCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
- llvm::Value *newPtr,
- llvm::Value *numElements,
- const CXXNewExpr *expr,
- QualType elementType) {
+Address MicrosoftCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
+ Address newPtr,
+ llvm::Value *numElements,
+ const CXXNewExpr *expr,
+ QualType elementType) {
assert(requiresArrayCookie(expr));
// The size of the cookie.
CharUnits cookieSize = getArrayCookieSizeImpl(elementType);
// Compute an offset to the cookie.
- llvm::Value *cookiePtr = newPtr;
+ Address cookiePtr = newPtr;
// Write the number of elements into the appropriate slot.
- unsigned AS = newPtr->getType()->getPointerAddressSpace();
- llvm::Value *numElementsPtr
- = CGF.Builder.CreateBitCast(cookiePtr, CGF.SizeTy->getPointerTo(AS));
+ Address numElementsPtr
+ = CGF.Builder.CreateElementBitCast(cookiePtr, CGF.SizeTy);
CGF.Builder.CreateStore(numElements, numElementsPtr);
// Finally, compute a pointer to the actual data buffer by skipping
// over the cookie completely.
- return CGF.Builder.CreateConstInBoundsGEP1_64(newPtr,
- cookieSize.getQuantity());
+ return CGF.Builder.CreateConstInBoundsByteGEP(newPtr, cookieSize);
}
static void emitGlobalDtorWithTLRegDtor(CodeGenFunction &CGF, const VarDecl &VD,
@@ -1965,11 +2192,9 @@
}
void MicrosoftCXXABI::EmitThreadLocalInitFuncs(
- CodeGenModule &CGM,
- ArrayRef<std::pair<const VarDecl *, llvm::GlobalVariable *>>
- CXXThreadLocals,
+ CodeGenModule &CGM, ArrayRef<const VarDecl *> CXXThreadLocals,
ArrayRef<llvm::Function *> CXXThreadLocalInits,
- ArrayRef<llvm::GlobalVariable *> CXXThreadLocalInitVars) {
+ ArrayRef<const VarDecl *> CXXThreadLocalInitVars) {
// This will create a GV in the .CRT$XDU section. It will point to our
// initialization function. The CRT will call all of these function
// pointers at start-up time and, eventually, at thread-creation time.
@@ -1987,7 +2212,8 @@
std::vector<llvm::Function *> NonComdatInits;
for (size_t I = 0, E = CXXThreadLocalInitVars.size(); I != E; ++I) {
- llvm::GlobalVariable *GV = CXXThreadLocalInitVars[I];
+ llvm::GlobalVariable *GV = cast<llvm::GlobalVariable>(
+ CGM.GetGlobalValue(CGM.getMangledName(CXXThreadLocalInitVars[I])));
llvm::Function *F = CXXThreadLocalInits[I];
// If the GV is already in a comdat group, then we have to join it.
@@ -2001,8 +2227,8 @@
llvm::FunctionType *FTy =
llvm::FunctionType::get(CGM.VoidTy, /*isVarArg=*/false);
llvm::Function *InitFunc = CGM.CreateGlobalInitOrDestructFunction(
- FTy, "__tls_init", SourceLocation(),
- /*TLS=*/true);
+ FTy, "__tls_init", CGM.getTypes().arrangeNullaryFunction(),
+ SourceLocation(), /*TLS=*/true);
CodeGenFunction(CGM).GenerateCXXGlobalInitFunc(InitFunc, NonComdatInits);
AddToXDU(InitFunc);
@@ -2016,17 +2242,18 @@
return LValue();
}
-static llvm::GlobalVariable *getInitThreadEpochPtr(CodeGenModule &CGM) {
+static ConstantAddress getInitThreadEpochPtr(CodeGenModule &CGM) {
StringRef VarName("_Init_thread_epoch");
+ CharUnits Align = CGM.getIntAlign();
if (auto *GV = CGM.getModule().getNamedGlobal(VarName))
- return GV;
+ return ConstantAddress(GV, Align);
auto *GV = new llvm::GlobalVariable(
CGM.getModule(), CGM.IntTy,
/*Constant=*/false, llvm::GlobalVariable::ExternalLinkage,
/*Initializer=*/nullptr, VarName,
/*InsertBefore=*/nullptr, llvm::GlobalVariable::GeneralDynamicTLSModel);
- GV->setAlignment(CGM.getTarget().getIntAlign() / 8);
- return GV;
+ GV->setAlignment(Align.getQuantity());
+ return ConstantAddress(GV, Align);
}
static llvm::Constant *getInitThreadHeaderFn(CodeGenModule &CGM) {
@@ -2063,10 +2290,10 @@
}
namespace {
-struct ResetGuardBit : EHScopeStack::Cleanup {
- llvm::GlobalVariable *Guard;
+struct ResetGuardBit final : EHScopeStack::Cleanup {
+ Address Guard;
unsigned GuardNum;
- ResetGuardBit(llvm::GlobalVariable *Guard, unsigned GuardNum)
+ ResetGuardBit(Address Guard, unsigned GuardNum)
: Guard(Guard), GuardNum(GuardNum) {}
void Emit(CodeGenFunction &CGF, Flags flags) override {
@@ -2080,9 +2307,9 @@
}
};
-struct CallInitThreadAbort : EHScopeStack::Cleanup {
- llvm::GlobalVariable *Guard;
- CallInitThreadAbort(llvm::GlobalVariable *Guard) : Guard(Guard) {}
+struct CallInitThreadAbort final : EHScopeStack::Cleanup {
+ llvm::Value *Guard;
+ CallInitThreadAbort(Address Guard) : Guard(Guard.getPointer()) {}
void Emit(CodeGenFunction &CGF, Flags flags) override {
// Calling _Init_thread_abort will reset the guard's state.
@@ -2115,6 +2342,7 @@
CGBuilderTy &Builder = CGF.Builder;
llvm::IntegerType *GuardTy = CGF.Int32Ty;
llvm::ConstantInt *Zero = llvm::ConstantInt::get(GuardTy, 0);
+ CharUnits GuardAlign = CharUnits::fromQuantity(4);
// Get the guard variable for this function if we have one already.
GuardInfo *GI = nullptr;
@@ -2155,7 +2383,6 @@
Out);
else
getMangleContext().mangleStaticGuardVariable(&D, Out);
- Out.flush();
}
// Create the guard variable with a zero-initializer. Just absorb linkage,
@@ -2165,6 +2392,7 @@
GV->getLinkage(), Zero, GuardName.str());
GuardVar->setVisibility(GV->getVisibility());
GuardVar->setDLLStorageClass(GV->getDLLStorageClass());
+ GuardVar->setAlignment(GuardAlign.getQuantity());
if (GuardVar->isWeakForLinker())
GuardVar->setComdat(
CGM.getModule().getOrInsertComdat(GuardVar->getName()));
@@ -2174,6 +2402,8 @@
GI->Guard = GuardVar;
}
+ ConstantAddress GuardAddr(GuardVar, GuardAlign);
+
assert(GuardVar->getLinkage() == GV->getLinkage() &&
"static local from the same function had different linkage");
@@ -2186,7 +2416,7 @@
// Test our bit from the guard variable.
llvm::ConstantInt *Bit = llvm::ConstantInt::get(GuardTy, 1U << GuardNum);
- llvm::LoadInst *LI = Builder.CreateLoad(GuardVar);
+ llvm::LoadInst *LI = Builder.CreateLoad(GuardAddr);
llvm::Value *IsInitialized =
Builder.CreateICmpNE(Builder.CreateAnd(LI, Bit), Zero);
llvm::BasicBlock *InitBlock = CGF.createBasicBlock("init");
@@ -2196,8 +2426,8 @@
// Set our bit in the guard variable and emit the initializer and add a global
// destructor if appropriate.
CGF.EmitBlock(InitBlock);
- Builder.CreateStore(Builder.CreateOr(LI, Bit), GuardVar);
- CGF.EHStack.pushCleanup<ResetGuardBit>(EHCleanup, GuardVar, GuardNum);
+ Builder.CreateStore(Builder.CreateOr(LI, Bit), GuardAddr);
+ CGF.EHStack.pushCleanup<ResetGuardBit>(EHCleanup, GuardAddr, GuardNum);
CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit);
CGF.PopCleanupBlock();
Builder.CreateBr(EndBlock);
@@ -2217,11 +2447,8 @@
// The algorithm is almost identical to what can be found in the appendix
// found in N2325.
- unsigned IntAlign = CGM.getTarget().getIntAlign() / 8;
-
// This BasicBLock determines whether or not we have any work to do.
- llvm::LoadInst *FirstGuardLoad =
- Builder.CreateAlignedLoad(GuardVar, IntAlign);
+ llvm::LoadInst *FirstGuardLoad = Builder.CreateLoad(GuardAddr);
FirstGuardLoad->setOrdering(llvm::AtomicOrdering::Unordered);
llvm::LoadInst *InitThreadEpoch =
Builder.CreateLoad(getInitThreadEpochPtr(CGM));
@@ -2234,9 +2461,9 @@
// This BasicBlock attempts to determine whether or not this thread is
// responsible for doing the initialization.
CGF.EmitBlock(AttemptInitBlock);
- CGF.EmitNounwindRuntimeCall(getInitThreadHeaderFn(CGM), GuardVar);
- llvm::LoadInst *SecondGuardLoad =
- Builder.CreateAlignedLoad(GuardVar, IntAlign);
+ CGF.EmitNounwindRuntimeCall(getInitThreadHeaderFn(CGM),
+ GuardAddr.getPointer());
+ llvm::LoadInst *SecondGuardLoad = Builder.CreateLoad(GuardAddr);
SecondGuardLoad->setOrdering(llvm::AtomicOrdering::Unordered);
llvm::Value *ShouldDoInit =
Builder.CreateICmpEQ(SecondGuardLoad, getAllOnesInt());
@@ -2245,10 +2472,11 @@
// Ok, we ended up getting selected as the initializing thread.
CGF.EmitBlock(InitBlock);
- CGF.EHStack.pushCleanup<CallInitThreadAbort>(EHCleanup, GuardVar);
+ CGF.EHStack.pushCleanup<CallInitThreadAbort>(EHCleanup, GuardAddr);
CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit);
CGF.PopCleanupBlock();
- CGF.EmitNounwindRuntimeCall(getInitThreadFooterFn(CGM), GuardVar);
+ CGF.EmitNounwindRuntimeCall(getInitThreadFooterFn(CGM),
+ GuardAddr.getPointer());
Builder.CreateBr(EndBlock);
CGF.EmitBlock(EndBlock);
@@ -2351,7 +2579,7 @@
if (MSInheritanceAttr::hasVBPtrOffsetField(Inheritance)) {
CharUnits Offs = CharUnits::Zero();
- if (RD->getNumVBases())
+ if (VBTableIndex)
Offs = getContext().getASTRecordLayout(RD).getVBPtrOffset();
fields.push_back(llvm::ConstantInt::get(CGM.IntTy, Offs.getQuantity()));
}
@@ -2367,43 +2595,76 @@
MicrosoftCXXABI::EmitMemberDataPointer(const MemberPointerType *MPT,
CharUnits offset) {
const CXXRecordDecl *RD = MPT->getMostRecentCXXRecordDecl();
+ if (RD->getMSInheritanceModel() ==
+ MSInheritanceAttr::Keyword_virtual_inheritance)
+ offset -= getContext().getOffsetOfBaseWithVBPtr(RD);
llvm::Constant *FirstField =
llvm::ConstantInt::get(CGM.IntTy, offset.getQuantity());
return EmitFullMemberPointer(FirstField, /*IsMemberFunction=*/false, RD,
CharUnits::Zero(), /*VBTableIndex=*/0);
}
-llvm::Constant *MicrosoftCXXABI::EmitMemberPointer(const CXXMethodDecl *MD) {
- return BuildMemberPointer(MD->getParent(), MD, CharUnits::Zero());
-}
-
llvm::Constant *MicrosoftCXXABI::EmitMemberPointer(const APValue &MP,
QualType MPType) {
- const MemberPointerType *MPT = MPType->castAs<MemberPointerType>();
+ const MemberPointerType *DstTy = MPType->castAs<MemberPointerType>();
const ValueDecl *MPD = MP.getMemberPointerDecl();
if (!MPD)
- return EmitNullMemberPointer(MPT);
+ return EmitNullMemberPointer(DstTy);
- CharUnits ThisAdjustment = getMemberPointerPathAdjustment(MP);
+ ASTContext &Ctx = getContext();
+ ArrayRef<const CXXRecordDecl *> MemberPointerPath = MP.getMemberPointerPath();
- // FIXME PR15713: Support virtual inheritance paths.
+ llvm::Constant *C;
+ if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(MPD)) {
+ C = EmitMemberFunctionPointer(MD);
+ } else {
+ CharUnits FieldOffset = Ctx.toCharUnitsFromBits(Ctx.getFieldOffset(MPD));
+ C = EmitMemberDataPointer(DstTy, FieldOffset);
+ }
- if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(MPD))
- return BuildMemberPointer(MPT->getMostRecentCXXRecordDecl(), MD,
- ThisAdjustment);
+ if (!MemberPointerPath.empty()) {
+ const CXXRecordDecl *SrcRD = cast<CXXRecordDecl>(MPD->getDeclContext());
+ const Type *SrcRecTy = Ctx.getTypeDeclType(SrcRD).getTypePtr();
+ const MemberPointerType *SrcTy =
+ Ctx.getMemberPointerType(DstTy->getPointeeType(), SrcRecTy)
+ ->castAs<MemberPointerType>();
- CharUnits FieldOffset =
- getContext().toCharUnitsFromBits(getContext().getFieldOffset(MPD));
- return EmitMemberDataPointer(MPT, ThisAdjustment + FieldOffset);
+ bool DerivedMember = MP.isMemberPointerToDerivedMember();
+ SmallVector<const CXXBaseSpecifier *, 4> DerivedToBasePath;
+ const CXXRecordDecl *PrevRD = SrcRD;
+ for (const CXXRecordDecl *PathElem : MemberPointerPath) {
+ const CXXRecordDecl *Base = nullptr;
+ const CXXRecordDecl *Derived = nullptr;
+ if (DerivedMember) {
+ Base = PathElem;
+ Derived = PrevRD;
+ } else {
+ Base = PrevRD;
+ Derived = PathElem;
+ }
+ for (const CXXBaseSpecifier &BS : Derived->bases())
+ if (BS.getType()->getAsCXXRecordDecl()->getCanonicalDecl() ==
+ Base->getCanonicalDecl())
+ DerivedToBasePath.push_back(&BS);
+ PrevRD = PathElem;
+ }
+ assert(DerivedToBasePath.size() == MemberPointerPath.size());
+
+ CastKind CK = DerivedMember ? CK_DerivedToBaseMemberPointer
+ : CK_BaseToDerivedMemberPointer;
+ C = EmitMemberPointerConversion(SrcTy, DstTy, CK, DerivedToBasePath.begin(),
+ DerivedToBasePath.end(), C);
+ }
+ return C;
}
llvm::Constant *
-MicrosoftCXXABI::BuildMemberPointer(const CXXRecordDecl *RD,
- const CXXMethodDecl *MD,
- CharUnits NonVirtualBaseAdjustment) {
+MicrosoftCXXABI::EmitMemberFunctionPointer(const CXXMethodDecl *MD) {
assert(MD->isInstance() && "Member function must not be static!");
+
MD = MD->getCanonicalDecl();
- RD = RD->getMostRecentDecl();
+ CharUnits NonVirtualBaseAdjustment = CharUnits::Zero();
+ const CXXRecordDecl *RD = MD->getParent()->getMostRecentDecl();
CodeGenTypes &Types = CGM.getTypes();
unsigned VBTableIndex = 0;
@@ -2421,20 +2682,24 @@
Ty = CGM.PtrDiffTy;
}
FirstField = CGM.GetAddrOfFunction(MD, Ty);
- FirstField = llvm::ConstantExpr::getBitCast(FirstField, CGM.VoidPtrTy);
} else {
auto &VTableContext = CGM.getMicrosoftVTableContext();
MicrosoftVTableContext::MethodVFTableLocation ML =
VTableContext.getMethodVFTableLocation(MD);
- llvm::Function *Thunk = EmitVirtualMemPtrThunk(MD, ML);
- FirstField = llvm::ConstantExpr::getBitCast(Thunk, CGM.VoidPtrTy);
+ FirstField = EmitVirtualMemPtrThunk(MD, ML);
// Include the vfptr adjustment if the method is in a non-primary vftable.
NonVirtualBaseAdjustment += ML.VFPtrOffset;
if (ML.VBase)
VBTableIndex = VTableContext.getVBTableIndex(RD, ML.VBase) * 4;
}
+ if (VBTableIndex == 0 &&
+ RD->getMSInheritanceModel() ==
+ MSInheritanceAttr::Keyword_virtual_inheritance)
+ NonVirtualBaseAdjustment -= getContext().getOffsetOfBaseWithVBPtr(RD);
+
// The rest of the fields are common with data member pointers.
+ FirstField = llvm::ConstantExpr::getBitCast(FirstField, CGM.VoidPtrTy);
return EmitFullMemberPointer(FirstField, /*IsMemberFunction=*/true, RD,
NonVirtualBaseAdjustment, VBTableIndex);
}
@@ -2566,19 +2831,28 @@
llvm::Value *
MicrosoftCXXABI::GetVBaseOffsetFromVBPtr(CodeGenFunction &CGF,
- llvm::Value *This,
+ Address This,
llvm::Value *VBPtrOffset,
llvm::Value *VBTableOffset,
llvm::Value **VBPtrOut) {
CGBuilderTy &Builder = CGF.Builder;
// Load the vbtable pointer from the vbptr in the instance.
- This = Builder.CreateBitCast(This, CGM.Int8PtrTy);
+ This = Builder.CreateElementBitCast(This, CGM.Int8Ty);
llvm::Value *VBPtr =
- Builder.CreateInBoundsGEP(This, VBPtrOffset, "vbptr");
+ Builder.CreateInBoundsGEP(This.getPointer(), VBPtrOffset, "vbptr");
if (VBPtrOut) *VBPtrOut = VBPtr;
VBPtr = Builder.CreateBitCast(VBPtr,
- CGM.Int32Ty->getPointerTo(0)->getPointerTo(0));
- llvm::Value *VBTable = Builder.CreateLoad(VBPtr, "vbtable");
+ CGM.Int32Ty->getPointerTo(0)->getPointerTo(This.getAddressSpace()));
+
+ CharUnits VBPtrAlign;
+ if (auto CI = dyn_cast<llvm::ConstantInt>(VBPtrOffset)) {
+ VBPtrAlign = This.getAlignment().alignmentAtOffset(
+ CharUnits::fromQuantity(CI->getSExtValue()));
+ } else {
+ VBPtrAlign = CGF.getPointerAlign();
+ }
+
+ llvm::Value *VBTable = Builder.CreateAlignedLoad(VBPtr, VBPtrAlign, "vbtable");
// Translate from byte offset to table index. It improves analyzability.
llvm::Value *VBTableIndex = Builder.CreateAShr(
@@ -2588,16 +2862,17 @@
// Load an i32 offset from the vb-table.
llvm::Value *VBaseOffs = Builder.CreateInBoundsGEP(VBTable, VBTableIndex);
VBaseOffs = Builder.CreateBitCast(VBaseOffs, CGM.Int32Ty->getPointerTo(0));
- return Builder.CreateLoad(VBaseOffs, "vbase_offs");
+ return Builder.CreateAlignedLoad(VBaseOffs, CharUnits::fromQuantity(4),
+ "vbase_offs");
}
// Returns an adjusted base cast to i8*, since we do more address arithmetic on
// it.
llvm::Value *MicrosoftCXXABI::AdjustVirtualBase(
CodeGenFunction &CGF, const Expr *E, const CXXRecordDecl *RD,
- llvm::Value *Base, llvm::Value *VBTableOffset, llvm::Value *VBPtrOffset) {
+ Address Base, llvm::Value *VBTableOffset, llvm::Value *VBPtrOffset) {
CGBuilderTy &Builder = CGF.Builder;
- Base = Builder.CreateBitCast(Base, CGM.Int8PtrTy);
+ Base = Builder.CreateElementBitCast(Base, CGM.Int8Ty);
llvm::BasicBlock *OriginalBB = nullptr;
llvm::BasicBlock *SkipAdjustBB = nullptr;
llvm::BasicBlock *VBaseAdjustBB = nullptr;
@@ -2642,7 +2917,7 @@
Builder.CreateBr(SkipAdjustBB);
CGF.EmitBlock(SkipAdjustBB);
llvm::PHINode *Phi = Builder.CreatePHI(CGM.Int8PtrTy, 2, "memptr.base");
- Phi->addIncoming(Base, OriginalBB);
+ Phi->addIncoming(Base.getPointer(), OriginalBB);
Phi->addIncoming(AdjustedBase, VBaseAdjustBB);
return Phi;
}
@@ -2650,10 +2925,10 @@
}
llvm::Value *MicrosoftCXXABI::EmitMemberDataPointerAddress(
- CodeGenFunction &CGF, const Expr *E, llvm::Value *Base, llvm::Value *MemPtr,
+ CodeGenFunction &CGF, const Expr *E, Address Base, llvm::Value *MemPtr,
const MemberPointerType *MPT) {
assert(MPT->isMemberDataPointer());
- unsigned AS = Base->getType()->getPointerAddressSpace();
+ unsigned AS = Base.getAddressSpace();
llvm::Type *PType =
CGF.ConvertTypeForMem(MPT->getPointeeType())->getPointerTo(AS);
CGBuilderTy &Builder = CGF.Builder;
@@ -2675,28 +2950,25 @@
VirtualBaseAdjustmentOffset = Builder.CreateExtractValue(MemPtr, I++);
}
+ llvm::Value *Addr;
if (VirtualBaseAdjustmentOffset) {
- Base = AdjustVirtualBase(CGF, E, RD, Base, VirtualBaseAdjustmentOffset,
+ Addr = AdjustVirtualBase(CGF, E, RD, Base, VirtualBaseAdjustmentOffset,
VBPtrOffset);
+ } else {
+ Addr = Base.getPointer();
}
// Cast to char*.
- Base = Builder.CreateBitCast(Base, Builder.getInt8Ty()->getPointerTo(AS));
+ Addr = Builder.CreateBitCast(Addr, CGF.Int8Ty->getPointerTo(AS));
// Apply the offset, which we assume is non-null.
- llvm::Value *Addr =
- Builder.CreateInBoundsGEP(Base, FieldOffset, "memptr.offset");
+ Addr = Builder.CreateInBoundsGEP(Addr, FieldOffset, "memptr.offset");
// Cast the address to the appropriate pointer type, adopting the address
// space of the base pointer.
return Builder.CreateBitCast(Addr, PType);
}
-static MSInheritanceAttr::Spelling
-getInheritanceFromMemptr(const MemberPointerType *MPT) {
- return MPT->getMostRecentCXXRecordDecl()->getMSInheritanceModel();
-}
-
llvm::Value *
MicrosoftCXXABI::EmitMemberPointerConversion(CodeGenFunction &CGF,
const CastExpr *E,
@@ -2748,12 +3020,37 @@
Builder.CreateCondBr(IsNotNull, ConvertBB, ContinueBB);
CGF.EmitBlock(ConvertBB);
+ llvm::Value *Dst = EmitNonNullMemberPointerConversion(
+ SrcTy, DstTy, E->getCastKind(), E->path_begin(), E->path_end(), Src,
+ Builder);
+
+ Builder.CreateBr(ContinueBB);
+
+ // In the continuation, choose between DstNull and Dst.
+ CGF.EmitBlock(ContinueBB);
+ llvm::PHINode *Phi = Builder.CreatePHI(DstNull->getType(), 2, "memptr.converted");
+ Phi->addIncoming(DstNull, OriginalBB);
+ Phi->addIncoming(Dst, ConvertBB);
+ return Phi;
+}
+
+llvm::Value *MicrosoftCXXABI::EmitNonNullMemberPointerConversion(
+ const MemberPointerType *SrcTy, const MemberPointerType *DstTy, CastKind CK,
+ CastExpr::path_const_iterator PathBegin,
+ CastExpr::path_const_iterator PathEnd, llvm::Value *Src,
+ CGBuilderTy &Builder) {
+ const CXXRecordDecl *SrcRD = SrcTy->getMostRecentCXXRecordDecl();
+ const CXXRecordDecl *DstRD = DstTy->getMostRecentCXXRecordDecl();
+ MSInheritanceAttr::Spelling SrcInheritance = SrcRD->getMSInheritanceModel();
+ MSInheritanceAttr::Spelling DstInheritance = DstRD->getMSInheritanceModel();
+ bool IsFunc = SrcTy->isMemberFunctionPointer();
+ bool IsConstant = isa<llvm::Constant>(Src);
+
// Decompose src.
llvm::Value *FirstField = Src;
- llvm::Value *NonVirtualBaseAdjustment = nullptr;
- llvm::Value *VirtualBaseAdjustmentOffset = nullptr;
- llvm::Value *VBPtrOffset = nullptr;
- MSInheritanceAttr::Spelling SrcInheritance = SrcRD->getMSInheritanceModel();
+ llvm::Value *NonVirtualBaseAdjustment = getZeroInt();
+ llvm::Value *VirtualBaseAdjustmentOffset = getZeroInt();
+ llvm::Value *VBPtrOffset = getZeroInt();
if (!MSInheritanceAttr::hasOnlyOneField(IsFunc, SrcInheritance)) {
// We need to extract values.
unsigned I = 0;
@@ -2766,59 +3063,139 @@
VirtualBaseAdjustmentOffset = Builder.CreateExtractValue(Src, I++);
}
+ bool IsDerivedToBase = (CK == CK_DerivedToBaseMemberPointer);
+ const MemberPointerType *DerivedTy = IsDerivedToBase ? SrcTy : DstTy;
+ const CXXRecordDecl *DerivedClass = DerivedTy->getMostRecentCXXRecordDecl();
+
// For data pointers, we adjust the field offset directly. For functions, we
// have a separate field.
- llvm::Constant *Adj = getMemberPointerAdjustment(E);
- if (Adj) {
- Adj = llvm::ConstantExpr::getTruncOrBitCast(Adj, CGM.IntTy);
- llvm::Value *&NVAdjustField = IsFunc ? NonVirtualBaseAdjustment : FirstField;
- bool isDerivedToBase = (E->getCastKind() == CK_DerivedToBaseMemberPointer);
- if (!NVAdjustField) // If this field didn't exist in src, it's zero.
- NVAdjustField = getZeroInt();
- if (isDerivedToBase)
- NVAdjustField = Builder.CreateNSWSub(NVAdjustField, Adj, "adj");
- else
- NVAdjustField = Builder.CreateNSWAdd(NVAdjustField, Adj, "adj");
+ llvm::Value *&NVAdjustField = IsFunc ? NonVirtualBaseAdjustment : FirstField;
+
+ // The virtual inheritance model has a quirk: the virtual base table is always
+ // referenced when dereferencing a member pointer even if the member pointer
+ // is non-virtual. This is accounted for by adjusting the non-virtual offset
+ // to point backwards to the top of the MDC from the first VBase. Undo this
+ // adjustment to normalize the member pointer.
+ llvm::Value *SrcVBIndexEqZero =
+ Builder.CreateICmpEQ(VirtualBaseAdjustmentOffset, getZeroInt());
+ if (SrcInheritance == MSInheritanceAttr::Keyword_virtual_inheritance) {
+ if (int64_t SrcOffsetToFirstVBase =
+ getContext().getOffsetOfBaseWithVBPtr(SrcRD).getQuantity()) {
+ llvm::Value *UndoSrcAdjustment = Builder.CreateSelect(
+ SrcVBIndexEqZero,
+ llvm::ConstantInt::get(CGM.IntTy, SrcOffsetToFirstVBase),
+ getZeroInt());
+ NVAdjustField = Builder.CreateNSWAdd(NVAdjustField, UndoSrcAdjustment);
+ }
}
- // FIXME PR15713: Support conversions through virtually derived classes.
+ // A non-zero vbindex implies that we are dealing with a source member in a
+ // floating virtual base in addition to some non-virtual offset. If the
+ // vbindex is zero, we are dealing with a source that exists in a non-virtual,
+ // fixed, base. The difference between these two cases is that the vbindex +
+ // nvoffset *always* point to the member regardless of what context they are
+ // evaluated in so long as the vbindex is adjusted. A member inside a fixed
+ // base requires explicit nv adjustment.
+ llvm::Constant *BaseClassOffset = llvm::ConstantInt::get(
+ CGM.IntTy,
+ CGM.computeNonVirtualBaseClassOffset(DerivedClass, PathBegin, PathEnd)
+ .getQuantity());
+
+ llvm::Value *NVDisp;
+ if (IsDerivedToBase)
+ NVDisp = Builder.CreateNSWSub(NVAdjustField, BaseClassOffset, "adj");
+ else
+ NVDisp = Builder.CreateNSWAdd(NVAdjustField, BaseClassOffset, "adj");
+
+ NVAdjustField = Builder.CreateSelect(SrcVBIndexEqZero, NVDisp, getZeroInt());
+
+ // Update the vbindex to an appropriate value in the destination because
+ // SrcRD's vbtable might not be a strict prefix of the one in DstRD.
+ llvm::Value *DstVBIndexEqZero = SrcVBIndexEqZero;
+ if (MSInheritanceAttr::hasVBTableOffsetField(DstInheritance) &&
+ MSInheritanceAttr::hasVBTableOffsetField(SrcInheritance)) {
+ if (llvm::GlobalVariable *VDispMap =
+ getAddrOfVirtualDisplacementMap(SrcRD, DstRD)) {
+ llvm::Value *VBIndex = Builder.CreateExactUDiv(
+ VirtualBaseAdjustmentOffset, llvm::ConstantInt::get(CGM.IntTy, 4));
+ if (IsConstant) {
+ llvm::Constant *Mapping = VDispMap->getInitializer();
+ VirtualBaseAdjustmentOffset =
+ Mapping->getAggregateElement(cast<llvm::Constant>(VBIndex));
+ } else {
+ llvm::Value *Idxs[] = {getZeroInt(), VBIndex};
+ VirtualBaseAdjustmentOffset =
+ Builder.CreateAlignedLoad(Builder.CreateInBoundsGEP(VDispMap, Idxs),
+ CharUnits::fromQuantity(4));
+ }
+
+ DstVBIndexEqZero =
+ Builder.CreateICmpEQ(VirtualBaseAdjustmentOffset, getZeroInt());
+ }
+ }
+
+ // Set the VBPtrOffset to zero if the vbindex is zero. Otherwise, initialize
+ // it to the offset of the vbptr.
+ if (MSInheritanceAttr::hasVBPtrOffsetField(DstInheritance)) {
+ llvm::Value *DstVBPtrOffset = llvm::ConstantInt::get(
+ CGM.IntTy,
+ getContext().getASTRecordLayout(DstRD).getVBPtrOffset().getQuantity());
+ VBPtrOffset =
+ Builder.CreateSelect(DstVBIndexEqZero, getZeroInt(), DstVBPtrOffset);
+ }
+
+ // Likewise, apply a similar adjustment so that dereferencing the member
+ // pointer correctly accounts for the distance between the start of the first
+ // virtual base and the top of the MDC.
+ if (DstInheritance == MSInheritanceAttr::Keyword_virtual_inheritance) {
+ if (int64_t DstOffsetToFirstVBase =
+ getContext().getOffsetOfBaseWithVBPtr(DstRD).getQuantity()) {
+ llvm::Value *DoDstAdjustment = Builder.CreateSelect(
+ DstVBIndexEqZero,
+ llvm::ConstantInt::get(CGM.IntTy, DstOffsetToFirstVBase),
+ getZeroInt());
+ NVAdjustField = Builder.CreateNSWSub(NVAdjustField, DoDstAdjustment);
+ }
+ }
// Recompose dst from the null struct and the adjusted fields from src.
- MSInheritanceAttr::Spelling DstInheritance = DstRD->getMSInheritanceModel();
llvm::Value *Dst;
if (MSInheritanceAttr::hasOnlyOneField(IsFunc, DstInheritance)) {
Dst = FirstField;
} else {
- Dst = llvm::UndefValue::get(DstNull->getType());
+ Dst = llvm::UndefValue::get(ConvertMemberPointerType(DstTy));
unsigned Idx = 0;
Dst = Builder.CreateInsertValue(Dst, FirstField, Idx++);
if (MSInheritanceAttr::hasNVOffsetField(IsFunc, DstInheritance))
- Dst = Builder.CreateInsertValue(
- Dst, getValueOrZeroInt(NonVirtualBaseAdjustment), Idx++);
+ Dst = Builder.CreateInsertValue(Dst, NonVirtualBaseAdjustment, Idx++);
if (MSInheritanceAttr::hasVBPtrOffsetField(DstInheritance))
- Dst = Builder.CreateInsertValue(
- Dst, getValueOrZeroInt(VBPtrOffset), Idx++);
+ Dst = Builder.CreateInsertValue(Dst, VBPtrOffset, Idx++);
if (MSInheritanceAttr::hasVBTableOffsetField(DstInheritance))
- Dst = Builder.CreateInsertValue(
- Dst, getValueOrZeroInt(VirtualBaseAdjustmentOffset), Idx++);
+ Dst = Builder.CreateInsertValue(Dst, VirtualBaseAdjustmentOffset, Idx++);
}
- Builder.CreateBr(ContinueBB);
-
- // In the continuation, choose between DstNull and Dst.
- CGF.EmitBlock(ContinueBB);
- llvm::PHINode *Phi = Builder.CreatePHI(DstNull->getType(), 2, "memptr.converted");
- Phi->addIncoming(DstNull, OriginalBB);
- Phi->addIncoming(Dst, ConvertBB);
- return Phi;
+ return Dst;
}
llvm::Constant *
MicrosoftCXXABI::EmitMemberPointerConversion(const CastExpr *E,
llvm::Constant *Src) {
const MemberPointerType *SrcTy =
- E->getSubExpr()->getType()->castAs<MemberPointerType>();
+ E->getSubExpr()->getType()->castAs<MemberPointerType>();
const MemberPointerType *DstTy = E->getType()->castAs<MemberPointerType>();
+ CastKind CK = E->getCastKind();
+
+ return EmitMemberPointerConversion(SrcTy, DstTy, CK, E->path_begin(),
+ E->path_end(), Src);
+}
+
+llvm::Constant *MicrosoftCXXABI::EmitMemberPointerConversion(
+ const MemberPointerType *SrcTy, const MemberPointerType *DstTy, CastKind CK,
+ CastExpr::path_const_iterator PathBegin,
+ CastExpr::path_const_iterator PathEnd, llvm::Constant *Src) {
+ assert(CK == CK_DerivedToBaseMemberPointer ||
+ CK == CK_BaseToDerivedMemberPointer ||
+ CK == CK_ReinterpretMemberPointer);
// If src is null, emit a new null for dst. We can't return src because dst
// might have a new representation.
if (MemberPointerConstantIsNull(SrcTy, Src))
@@ -2827,73 +3204,26 @@
// We don't need to do anything for reinterpret_casts of non-null member
// pointers. We should only get here when the two type representations have
// the same size.
- if (E->getCastKind() == CK_ReinterpretMemberPointer)
+ if (CK == CK_ReinterpretMemberPointer)
return Src;
- MSInheritanceAttr::Spelling SrcInheritance = getInheritanceFromMemptr(SrcTy);
- MSInheritanceAttr::Spelling DstInheritance = getInheritanceFromMemptr(DstTy);
+ CGBuilderTy Builder(CGM, CGM.getLLVMContext());
+ auto *Dst = cast<llvm::Constant>(EmitNonNullMemberPointerConversion(
+ SrcTy, DstTy, CK, PathBegin, PathEnd, Src, Builder));
- // Decompose src.
- llvm::Constant *FirstField = Src;
- llvm::Constant *NonVirtualBaseAdjustment = nullptr;
- llvm::Constant *VirtualBaseAdjustmentOffset = nullptr;
- llvm::Constant *VBPtrOffset = nullptr;
- bool IsFunc = SrcTy->isMemberFunctionPointer();
- if (!MSInheritanceAttr::hasOnlyOneField(IsFunc, SrcInheritance)) {
- // We need to extract values.
- unsigned I = 0;
- FirstField = Src->getAggregateElement(I++);
- if (MSInheritanceAttr::hasNVOffsetField(IsFunc, SrcInheritance))
- NonVirtualBaseAdjustment = Src->getAggregateElement(I++);
- if (MSInheritanceAttr::hasVBPtrOffsetField(SrcInheritance))
- VBPtrOffset = Src->getAggregateElement(I++);
- if (MSInheritanceAttr::hasVBTableOffsetField(SrcInheritance))
- VirtualBaseAdjustmentOffset = Src->getAggregateElement(I++);
- }
-
- // For data pointers, we adjust the field offset directly. For functions, we
- // have a separate field.
- llvm::Constant *Adj = getMemberPointerAdjustment(E);
- if (Adj) {
- Adj = llvm::ConstantExpr::getTruncOrBitCast(Adj, CGM.IntTy);
- llvm::Constant *&NVAdjustField =
- IsFunc ? NonVirtualBaseAdjustment : FirstField;
- bool IsDerivedToBase = (E->getCastKind() == CK_DerivedToBaseMemberPointer);
- if (!NVAdjustField) // If this field didn't exist in src, it's zero.
- NVAdjustField = getZeroInt();
- if (IsDerivedToBase)
- NVAdjustField = llvm::ConstantExpr::getNSWSub(NVAdjustField, Adj);
- else
- NVAdjustField = llvm::ConstantExpr::getNSWAdd(NVAdjustField, Adj);
- }
-
- // FIXME PR15713: Support conversions through virtually derived classes.
-
- // Recompose dst from the null struct and the adjusted fields from src.
- if (MSInheritanceAttr::hasOnlyOneField(IsFunc, DstInheritance))
- return FirstField;
-
- llvm::SmallVector<llvm::Constant *, 4> Fields;
- Fields.push_back(FirstField);
- if (MSInheritanceAttr::hasNVOffsetField(IsFunc, DstInheritance))
- Fields.push_back(getConstantOrZeroInt(NonVirtualBaseAdjustment));
- if (MSInheritanceAttr::hasVBPtrOffsetField(DstInheritance))
- Fields.push_back(getConstantOrZeroInt(VBPtrOffset));
- if (MSInheritanceAttr::hasVBTableOffsetField(DstInheritance))
- Fields.push_back(getConstantOrZeroInt(VirtualBaseAdjustmentOffset));
- return llvm::ConstantStruct::getAnon(Fields);
+ return Dst;
}
llvm::Value *MicrosoftCXXABI::EmitLoadOfMemberFunctionPointer(
- CodeGenFunction &CGF, const Expr *E, llvm::Value *&This,
- llvm::Value *MemPtr, const MemberPointerType *MPT) {
+ CodeGenFunction &CGF, const Expr *E, Address This,
+ llvm::Value *&ThisPtrForCall, llvm::Value *MemPtr,
+ const MemberPointerType *MPT) {
assert(MPT->isMemberFunctionPointer());
const FunctionProtoType *FPT =
MPT->getPointeeType()->castAs<FunctionProtoType>();
const CXXRecordDecl *RD = MPT->getMostRecentCXXRecordDecl();
- llvm::FunctionType *FTy =
- CGM.getTypes().GetFunctionType(
- CGM.getTypes().arrangeCXXMethodType(RD, FPT));
+ llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(
+ CGM.getTypes().arrangeCXXMethodType(RD, FPT, /*FD=*/nullptr));
CGBuilderTy &Builder = CGF.Builder;
MSInheritanceAttr::Spelling Inheritance = RD->getMSInheritanceModel();
@@ -2917,15 +3247,18 @@
}
if (VirtualBaseAdjustmentOffset) {
- This = AdjustVirtualBase(CGF, E, RD, This, VirtualBaseAdjustmentOffset,
- VBPtrOffset);
+ ThisPtrForCall = AdjustVirtualBase(CGF, E, RD, This,
+ VirtualBaseAdjustmentOffset, VBPtrOffset);
+ } else {
+ ThisPtrForCall = This.getPointer();
}
if (NonVirtualBaseAdjustment) {
// Apply the adjustment and cast back to the original struct type.
- llvm::Value *Ptr = Builder.CreateBitCast(This, Builder.getInt8PtrTy());
+ llvm::Value *Ptr = Builder.CreateBitCast(ThisPtrForCall, CGF.Int8PtrTy);
Ptr = Builder.CreateInBoundsGEP(Ptr, NonVirtualBaseAdjustment);
- This = Builder.CreateBitCast(Ptr, This->getType(), "this.adjusted");
+ ThisPtrForCall = Builder.CreateBitCast(Ptr, ThisPtrForCall->getType(),
+ "this.adjusted");
}
return Builder.CreateBitCast(FunctionPointer, FTy->getPointerTo());
@@ -3150,7 +3483,7 @@
auto Type = ABI.getClassHierarchyDescriptorType();
auto CHD = new llvm::GlobalVariable(Module, Type, /*Constant=*/true, Linkage,
/*Initializer=*/nullptr,
- StringRef(MangledName));
+ MangledName);
if (CHD->isWeakForLinker())
CHD->setComdat(CGM.getModule().getOrInsertComdat(CHD->getName()));
@@ -3188,7 +3521,7 @@
auto *BCA =
new llvm::GlobalVariable(Module, ArrType,
/*Constant=*/true, Linkage,
- /*Initializer=*/nullptr, StringRef(MangledName));
+ /*Initializer=*/nullptr, MangledName);
if (BCA->isWeakForLinker())
BCA->setComdat(CGM.getModule().getOrInsertComdat(BCA->getName()));
@@ -3230,7 +3563,7 @@
auto Type = ABI.getBaseClassDescriptorType();
auto BCD =
new llvm::GlobalVariable(Module, Type, /*Constant=*/true, Linkage,
- /*Initializer=*/nullptr, StringRef(MangledName));
+ /*Initializer=*/nullptr, MangledName);
if (BCD->isWeakForLinker())
BCD->setComdat(CGM.getModule().getOrInsertComdat(BCD->getName()));
@@ -3276,7 +3609,7 @@
// Forward-declare the complete object locator.
llvm::StructType *Type = ABI.getCompleteObjectLocatorType();
auto COL = new llvm::GlobalVariable(Module, Type, /*Constant=*/true, Linkage,
- /*Initializer=*/nullptr, StringRef(MangledName));
+ /*Initializer=*/nullptr, MangledName);
// Initialize the CompleteObjectLocator.
llvm::Constant *Fields[] = {
@@ -3328,7 +3661,7 @@
return T;
}
-llvm::Constant *
+CatchTypeInfo
MicrosoftCXXABI::getAddrOfCXXCatchHandlerType(QualType Type,
QualType CatchHandlerType) {
// TypeDescriptors for exceptions never have qualified pointer types,
@@ -3347,28 +3680,8 @@
if (IsReference)
Flags |= 8;
- SmallString<256> MangledName;
- {
- llvm::raw_svector_ostream Out(MangledName);
- getMangleContext().mangleCXXCatchHandlerType(Type, Flags, Out);
- }
-
- if (llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(MangledName))
- return llvm::ConstantExpr::getBitCast(GV, CGM.Int8PtrTy);
-
- llvm::Constant *Fields[] = {
- llvm::ConstantInt::get(CGM.IntTy, Flags), // Flags
- getAddrOfRTTIDescriptor(Type), // TypeDescriptor
- };
- llvm::StructType *CatchHandlerTypeType = getCatchHandlerTypeType();
- auto *Var = new llvm::GlobalVariable(
- CGM.getModule(), CatchHandlerTypeType, /*Constant=*/true,
- llvm::GlobalValue::PrivateLinkage,
- llvm::ConstantStruct::get(CatchHandlerTypeType, Fields),
- StringRef(MangledName));
- Var->setUnnamedAddr(true);
- Var->setSection("llvm.metadata");
- return Var;
+ return CatchTypeInfo{getAddrOfRTTIDescriptor(Type)->stripPointerCasts(),
+ Flags};
}
/// \brief Gets a TypeDescriptor. Returns a llvm::Constant * rather than a
@@ -3404,7 +3717,7 @@
CGM.getModule(), TypeDescriptorType, /*Constant=*/false,
getLinkageForRTTI(Type),
llvm::ConstantStruct::get(TypeDescriptorType, Fields),
- StringRef(MangledName));
+ MangledName);
if (Var->isWeakForLinker())
Var->setComdat(CGM.getModule().getOrInsertComdat(Var->getName()));
return llvm::ConstantExpr::getBitCast(Var, CGM.Int8PtrTy);
@@ -3471,7 +3784,6 @@
SmallString<256> ThunkName;
llvm::raw_svector_ostream Out(ThunkName);
getMangleContext().mangleCXXCtor(CD, CT, Out);
- Out.flush();
// If the thunk has been generated previously, just return it.
if (llvm::GlobalValue *GV = CGM.getModule().getNamedValue(ThunkName))
@@ -3486,6 +3798,8 @@
ThunkTy, getLinkageForRTTI(RecordTy), ThunkName.str(), &CGM.getModule());
ThunkFn->setCallingConv(static_cast<llvm::CallingConv::ID>(
FnInfo.getEffectiveCallingConvention()));
+ if (ThunkFn->isWeakForLinker())
+ ThunkFn->setComdat(CGM.getModule().getOrInsertComdat(ThunkFn->getName()));
bool IsCopy = CT == Ctor_CopyingClosure;
// Start codegen.
@@ -3547,9 +3861,7 @@
CodeGenFunction::RunCleanupsScope Cleanups(CGF);
const auto *FPT = CD->getType()->castAs<FunctionProtoType>();
- ConstExprIterator ArgBegin(ArgVec.data()),
- ArgEnd(ArgVec.data() + ArgVec.size());
- CGF.EmitCallArgs(Args, FPT, ArgBegin, ArgEnd, CD, IsCopy ? 1 : 0);
+ CGF.EmitCallArgs(Args, FPT, llvm::makeArrayRef(ArgVec), CD, IsCopy ? 1 : 0);
// Insert any ABI-specific implicit constructor arguments.
unsigned ExtraArgs = addImplicitConstructorArgs(CGF, CD, Ctor_Complete,
@@ -3648,7 +3960,7 @@
llvm::StructType *CTType = getCatchableTypeType();
auto *GV = new llvm::GlobalVariable(
CGM.getModule(), CTType, /*Constant=*/true, getLinkageForRTTI(T),
- llvm::ConstantStruct::get(CTType, Fields), StringRef(MangledName));
+ llvm::ConstantStruct::get(CTType, Fields), MangledName);
GV->setUnnamedAddr(true);
GV->setSection(".xdata");
if (GV->isWeakForLinker())
@@ -3766,7 +4078,7 @@
}
CTA = new llvm::GlobalVariable(
CGM.getModule(), CTAType, /*Constant=*/true, getLinkageForRTTI(T),
- llvm::ConstantStruct::get(CTAType, Fields), StringRef(MangledName));
+ llvm::ConstantStruct::get(CTAType, Fields), MangledName);
CTA->setUnnamedAddr(true);
CTA->setSection(".xdata");
if (CTA->isWeakForLinker())
@@ -3846,7 +4158,7 @@
QualType ThrowType = SubExpr->getType();
// The exception object lives on the stack and it's address is passed to the
// runtime function.
- llvm::AllocaInst *AI = CGF.CreateMemTemp(ThrowType);
+ Address AI = CGF.CreateMemTemp(ThrowType);
CGF.EmitAnyExprToMem(SubExpr, AI, ThrowType.getQualifiers(),
/*IsInit=*/true);
@@ -3855,6 +4167,9 @@
llvm::GlobalVariable *TI = getThrowInfo(ThrowType);
// Call into the runtime to throw the exception.
- llvm::Value *Args[] = {CGF.Builder.CreateBitCast(AI, CGM.Int8PtrTy), TI};
+ llvm::Value *Args[] = {
+ CGF.Builder.CreateBitCast(AI.getPointer(), CGM.Int8PtrTy),
+ TI
+ };
CGF.EmitNoreturnRuntimeCallOrInvoke(getThrowFn(), Args);
}
diff --git a/lib/CodeGen/ModuleBuilder.cpp b/lib/CodeGen/ModuleBuilder.cpp
index 25e5740..0be5c55 100644
--- a/lib/CodeGen/ModuleBuilder.cpp
+++ b/lib/CodeGen/ModuleBuilder.cpp
@@ -30,8 +30,9 @@
namespace {
class CodeGeneratorImpl : public CodeGenerator {
DiagnosticsEngine &Diags;
- std::unique_ptr<const llvm::DataLayout> TD;
ASTContext *Ctx;
+ const HeaderSearchOptions &HeaderSearchOpts; // Only used for debug info.
+ const PreprocessorOptions &PreprocessorOpts; // Only used for debug info.
const CodeGenOptions CodeGenOpts; // Intentionally copied in.
unsigned HandlingTopLevelDecls;
@@ -56,12 +57,15 @@
SmallVector<CXXMethodDecl *, 8> DeferredInlineMethodDefinitions;
public:
- CodeGeneratorImpl(DiagnosticsEngine &diags, const std::string& ModuleName,
- const CodeGenOptions &CGO, llvm::LLVMContext& C,
+ CodeGeneratorImpl(DiagnosticsEngine &diags, const std::string &ModuleName,
+ const HeaderSearchOptions &HSO,
+ const PreprocessorOptions &PPO, const CodeGenOptions &CGO,
+ llvm::LLVMContext &C,
CoverageSourceInfo *CoverageInfo = nullptr)
- : Diags(diags), Ctx(nullptr), CodeGenOpts(CGO), HandlingTopLevelDecls(0),
- CoverageInfo(CoverageInfo),
- M(new llvm::Module(ModuleName, C)) {}
+ : Diags(diags), Ctx(nullptr), HeaderSearchOpts(HSO),
+ PreprocessorOpts(PPO), CodeGenOpts(CGO), HandlingTopLevelDecls(0),
+ CoverageInfo(CoverageInfo),
+ M(new llvm::Module(ModuleName, C)) {}
~CodeGeneratorImpl() override {
// There should normally not be any leftover inline method definitions.
@@ -94,11 +98,10 @@
Ctx = &Context;
M->setTargetTriple(Ctx->getTargetInfo().getTriple().getTriple());
- M->setDataLayout(Ctx->getTargetInfo().getTargetDescription());
- TD.reset(
- new llvm::DataLayout(Ctx->getTargetInfo().getTargetDescription()));
- Builder.reset(new CodeGen::CodeGenModule(Context, CodeGenOpts, *M, *TD,
- Diags, CoverageInfo));
+ M->setDataLayout(Ctx->getTargetInfo().getDataLayoutString());
+ Builder.reset(new CodeGen::CodeGenModule(Context, HeaderSearchOpts,
+ PreprocessorOpts, CodeGenOpts,
+ *M, Diags, CoverageInfo));
for (size_t i = 0, e = CodeGenOpts.DependentLibraries.size(); i < e; ++i)
HandleDependentLibrary(CodeGenOpts.DependentLibraries[i]);
@@ -172,7 +175,7 @@
// For MSVC compatibility, treat declarations of static data members with
// inline initializers as definitions.
- if (Ctx->getLangOpts().MSVCCompat) {
+ if (Ctx->getTargetInfo().getCXXABI().isMicrosoft()) {
for (Decl *Member : D->decls()) {
if (VarDecl *VD = dyn_cast<VarDecl>(Member)) {
if (Ctx->isMSStaticDataMemberInlineDefinition(VD) &&
@@ -236,10 +239,11 @@
void CodeGenerator::anchor() { }
-CodeGenerator *clang::CreateLLVMCodeGen(DiagnosticsEngine &Diags,
- const std::string& ModuleName,
- const CodeGenOptions &CGO,
- llvm::LLVMContext& C,
- CoverageSourceInfo *CoverageInfo) {
- return new CodeGeneratorImpl(Diags, ModuleName, CGO, C, CoverageInfo);
+CodeGenerator *clang::CreateLLVMCodeGen(
+ DiagnosticsEngine &Diags, const std::string &ModuleName,
+ const HeaderSearchOptions &HeaderSearchOpts,
+ const PreprocessorOptions &PreprocessorOpts, const CodeGenOptions &CGO,
+ llvm::LLVMContext &C, CoverageSourceInfo *CoverageInfo) {
+ return new CodeGeneratorImpl(Diags, ModuleName, HeaderSearchOpts,
+ PreprocessorOpts, CGO, C, CoverageInfo);
}
diff --git a/lib/CodeGen/ObjectFilePCHContainerOperations.cpp b/lib/CodeGen/ObjectFilePCHContainerOperations.cpp
new file mode 100644
index 0000000..b397eb3
--- /dev/null
+++ b/lib/CodeGen/ObjectFilePCHContainerOperations.cpp
@@ -0,0 +1,300 @@
+//===--- ObjectFilePCHContainerOperations.cpp -----------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
+#include "CGDebugInfo.h"
+#include "CodeGenModule.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/DeclObjC.h"
+#include "clang/AST/Expr.h"
+#include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/TargetInfo.h"
+#include "clang/CodeGen/BackendUtil.h"
+#include "clang/Frontend/CodeGenOptions.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/HeaderSearch.h"
+#include "clang/Serialization/ASTWriter.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Bitcode/BitstreamReader.h"
+#include "llvm/DebugInfo/DWARF/DWARFContext.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Object/COFF.h"
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/TargetRegistry.h"
+#include <memory>
+
+using namespace clang;
+
+#define DEBUG_TYPE "pchcontainer"
+
+namespace {
+class PCHContainerGenerator : public ASTConsumer {
+ DiagnosticsEngine &Diags;
+ const std::string MainFileName;
+ ASTContext *Ctx;
+ ModuleMap &MMap;
+ const HeaderSearchOptions &HeaderSearchOpts;
+ const PreprocessorOptions &PreprocessorOpts;
+ CodeGenOptions CodeGenOpts;
+ const TargetOptions TargetOpts;
+ const LangOptions LangOpts;
+ std::unique_ptr<llvm::LLVMContext> VMContext;
+ std::unique_ptr<llvm::Module> M;
+ std::unique_ptr<CodeGen::CodeGenModule> Builder;
+ raw_pwrite_stream *OS;
+ std::shared_ptr<PCHBuffer> Buffer;
+
+ /// Visit every type and emit debug info for it.
+ struct DebugTypeVisitor : public RecursiveASTVisitor<DebugTypeVisitor> {
+ clang::CodeGen::CGDebugInfo &DI;
+ ASTContext &Ctx;
+ DebugTypeVisitor(clang::CodeGen::CGDebugInfo &DI, ASTContext &Ctx)
+ : DI(DI), Ctx(Ctx) {}
+
+ /// Determine whether this type can be represented in DWARF.
+ static bool CanRepresent(const Type *Ty) {
+ return !Ty->isDependentType() && !Ty->isUndeducedType();
+ }
+
+ bool VisitImportDecl(ImportDecl *D) {
+ auto *Import = cast<ImportDecl>(D);
+ if (!Import->getImportedOwningModule())
+ DI.EmitImportDecl(*Import);
+ return true;
+ }
+
+ bool VisitTypeDecl(TypeDecl *D) {
+ QualType QualTy = Ctx.getTypeDeclType(D);
+ if (!QualTy.isNull() && CanRepresent(QualTy.getTypePtr()))
+ DI.getOrCreateStandaloneType(QualTy, D->getLocation());
+ return true;
+ }
+
+ bool VisitObjCInterfaceDecl(ObjCInterfaceDecl *D) {
+ QualType QualTy(D->getTypeForDecl(), 0);
+ if (!QualTy.isNull() && CanRepresent(QualTy.getTypePtr()))
+ DI.getOrCreateStandaloneType(QualTy, D->getLocation());
+ return true;
+ }
+
+ bool VisitFunctionDecl(FunctionDecl *D) {
+ if (isa<CXXMethodDecl>(D))
+ // This is not yet supported. Constructing the `this' argument
+ // mandates a CodeGenFunction.
+ return true;
+
+ SmallVector<QualType, 16> ArgTypes;
+ for (auto i : D->params())
+ ArgTypes.push_back(i->getType());
+ QualType RetTy = D->getReturnType();
+ QualType FnTy = Ctx.getFunctionType(RetTy, ArgTypes,
+ FunctionProtoType::ExtProtoInfo());
+ if (CanRepresent(FnTy.getTypePtr()))
+ DI.EmitFunctionDecl(D, D->getLocation(), FnTy);
+ return true;
+ }
+
+ bool VisitObjCMethodDecl(ObjCMethodDecl *D) {
+ if (!D->getClassInterface())
+ return true;
+
+ bool selfIsPseudoStrong, selfIsConsumed;
+ SmallVector<QualType, 16> ArgTypes;
+ ArgTypes.push_back(D->getSelfType(Ctx, D->getClassInterface(),
+ selfIsPseudoStrong, selfIsConsumed));
+ ArgTypes.push_back(Ctx.getObjCSelType());
+ for (auto i : D->params())
+ ArgTypes.push_back(i->getType());
+ QualType RetTy = D->getReturnType();
+ QualType FnTy = Ctx.getFunctionType(RetTy, ArgTypes,
+ FunctionProtoType::ExtProtoInfo());
+ if (CanRepresent(FnTy.getTypePtr()))
+ DI.EmitFunctionDecl(D, D->getLocation(), FnTy);
+ return true;
+ }
+ };
+
+public:
+ PCHContainerGenerator(CompilerInstance &CI, const std::string &MainFileName,
+ const std::string &OutputFileName,
+ raw_pwrite_stream *OS,
+ std::shared_ptr<PCHBuffer> Buffer)
+ : Diags(CI.getDiagnostics()), Ctx(nullptr),
+ MMap(CI.getPreprocessor().getHeaderSearchInfo().getModuleMap()),
+ HeaderSearchOpts(CI.getHeaderSearchOpts()),
+ PreprocessorOpts(CI.getPreprocessorOpts()),
+ TargetOpts(CI.getTargetOpts()), LangOpts(CI.getLangOpts()), OS(OS),
+ Buffer(Buffer) {
+ // The debug info output isn't affected by CodeModel and
+ // ThreadModel, but the backend expects them to be nonempty.
+ CodeGenOpts.CodeModel = "default";
+ CodeGenOpts.ThreadModel = "single";
+ CodeGenOpts.DebugTypeExtRefs = true;
+ CodeGenOpts.setDebugInfo(CodeGenOptions::FullDebugInfo);
+ }
+
+ ~PCHContainerGenerator() override = default;
+
+ void Initialize(ASTContext &Context) override {
+ assert(!Ctx && "initialized multiple times");
+
+ Ctx = &Context;
+ VMContext.reset(new llvm::LLVMContext());
+ M.reset(new llvm::Module(MainFileName, *VMContext));
+ M->setDataLayout(Ctx->getTargetInfo().getDataLayoutString());
+ Builder.reset(new CodeGen::CodeGenModule(
+ *Ctx, HeaderSearchOpts, PreprocessorOpts, CodeGenOpts, *M, Diags));
+ Builder->getModuleDebugInfo()->setModuleMap(MMap);
+ }
+
+ bool HandleTopLevelDecl(DeclGroupRef D) override {
+ if (Diags.hasErrorOccurred())
+ return true;
+
+ // Collect debug info for all decls in this group.
+ for (auto *I : D)
+ if (!I->isFromASTFile()) {
+ DebugTypeVisitor DTV(*Builder->getModuleDebugInfo(), *Ctx);
+ DTV.TraverseDecl(I);
+ }
+ return true;
+ }
+
+ void HandleTopLevelDeclInObjCContainer(DeclGroupRef D) override {
+ HandleTopLevelDecl(D);
+ }
+
+ void HandleTagDeclDefinition(TagDecl *D) override {
+ if (Diags.hasErrorOccurred())
+ return;
+
+ Builder->UpdateCompletedType(D);
+ }
+
+ void HandleTagDeclRequiredDefinition(const TagDecl *D) override {
+ if (Diags.hasErrorOccurred())
+ return;
+
+ if (const RecordDecl *RD = dyn_cast<RecordDecl>(D))
+ Builder->getModuleDebugInfo()->completeRequiredType(RD);
+ }
+
+ /// Emit a container holding the serialized AST.
+ void HandleTranslationUnit(ASTContext &Ctx) override {
+ assert(M && VMContext && Builder);
+ // Delete these on function exit.
+ std::unique_ptr<llvm::LLVMContext> VMContext = std::move(this->VMContext);
+ std::unique_ptr<llvm::Module> M = std::move(this->M);
+ std::unique_ptr<CodeGen::CodeGenModule> Builder = std::move(this->Builder);
+
+ if (Diags.hasErrorOccurred())
+ return;
+
+ M->setTargetTriple(Ctx.getTargetInfo().getTriple().getTriple());
+ M->setDataLayout(Ctx.getTargetInfo().getDataLayoutString());
+ Builder->getModuleDebugInfo()->setDwoId(Buffer->Signature);
+
+ // Finalize the Builder.
+ if (Builder)
+ Builder->Release();
+
+ // Ensure the target exists.
+ std::string Error;
+ auto Triple = Ctx.getTargetInfo().getTriple();
+ if (!llvm::TargetRegistry::lookupTarget(Triple.getTriple(), Error))
+ llvm::report_fatal_error(Error);
+
+ // Emit the serialized Clang AST into its own section.
+ assert(Buffer->IsComplete && "serialization did not complete");
+ auto &SerializedAST = Buffer->Data;
+ auto Size = SerializedAST.size();
+ auto Int8Ty = llvm::Type::getInt8Ty(*VMContext);
+ auto *Ty = llvm::ArrayType::get(Int8Ty, Size);
+ auto *Data = llvm::ConstantDataArray::getString(
+ *VMContext, StringRef(SerializedAST.data(), Size),
+ /*AddNull=*/false);
+ auto *ASTSym = new llvm::GlobalVariable(
+ *M, Ty, /*constant*/ true, llvm::GlobalVariable::InternalLinkage, Data,
+ "__clang_ast");
+ // The on-disk hashtable needs to be aligned.
+ ASTSym->setAlignment(8);
+
+ // Mach-O also needs a segment name.
+ if (Triple.isOSBinFormatMachO())
+ ASTSym->setSection("__CLANG,__clangast");
+ // COFF has an eight character length limit.
+ else if (Triple.isOSBinFormatCOFF())
+ ASTSym->setSection("clangast");
+ else
+ ASTSym->setSection("__clangast");
+
+ DEBUG({
+ // Print the IR for the PCH container to the debug output.
+ llvm::SmallString<0> Buffer;
+ llvm::raw_svector_ostream OS(Buffer);
+ clang::EmitBackendOutput(Diags, CodeGenOpts, TargetOpts, LangOpts,
+ Ctx.getTargetInfo().getDataLayoutString(),
+ M.get(), BackendAction::Backend_EmitLL, &OS);
+ llvm::dbgs() << Buffer;
+ });
+
+ // Use the LLVM backend to emit the pch container.
+ clang::EmitBackendOutput(Diags, CodeGenOpts, TargetOpts, LangOpts,
+ Ctx.getTargetInfo().getDataLayoutString(),
+ M.get(), BackendAction::Backend_EmitObj, OS);
+
+ // Make sure the pch container hits disk.
+ OS->flush();
+
+ // Free the memory for the temporary buffer.
+ llvm::SmallVector<char, 0> Empty;
+ SerializedAST = std::move(Empty);
+ }
+};
+
+} // anonymous namespace
+
+std::unique_ptr<ASTConsumer>
+ObjectFilePCHContainerWriter::CreatePCHContainerGenerator(
+ CompilerInstance &CI, const std::string &MainFileName,
+ const std::string &OutputFileName, llvm::raw_pwrite_stream *OS,
+ std::shared_ptr<PCHBuffer> Buffer) const {
+ return llvm::make_unique<PCHContainerGenerator>(CI, MainFileName,
+ OutputFileName, OS, Buffer);
+}
+
+void ObjectFilePCHContainerReader::ExtractPCH(
+ llvm::MemoryBufferRef Buffer, llvm::BitstreamReader &StreamFile) const {
+ if (auto OF = llvm::object::ObjectFile::createObjectFile(Buffer)) {
+ auto *Obj = OF.get().get();
+ bool IsCOFF = isa<llvm::object::COFFObjectFile>(Obj);
+ // Find the clang AST section in the container.
+ for (auto &Section : OF->get()->sections()) {
+ StringRef Name;
+ Section.getName(Name);
+ if ((!IsCOFF && Name == "__clangast") ||
+ ( IsCOFF && Name == "clangast")) {
+ StringRef Buf;
+ Section.getContents(Buf);
+ StreamFile.init((const unsigned char *)Buf.begin(),
+ (const unsigned char *)Buf.end());
+ return;
+ }
+ }
+ }
+
+ // As a fallback, treat the buffer as a raw AST.
+ StreamFile.init((const unsigned char *)Buffer.getBufferStart(),
+ (const unsigned char *)Buffer.getBufferEnd());
+}
diff --git a/lib/CodeGen/SanitizerMetadata.cpp b/lib/CodeGen/SanitizerMetadata.cpp
index 7c38b28..2a338ba 100644
--- a/lib/CodeGen/SanitizerMetadata.cpp
+++ b/lib/CodeGen/SanitizerMetadata.cpp
@@ -25,7 +25,8 @@
SourceLocation Loc, StringRef Name,
QualType Ty, bool IsDynInit,
bool IsBlacklisted) {
- if (!CGM.getLangOpts().Sanitize.has(SanitizerKind::Address))
+ if (!CGM.getLangOpts().Sanitize.hasOneOf(SanitizerKind::Address |
+ SanitizerKind::KernelAddress))
return;
IsDynInit &= !CGM.isInSanitizerBlacklist(GV, Loc, Ty, "init");
IsBlacklisted |= CGM.isInSanitizerBlacklist(GV, Loc, Ty);
@@ -56,7 +57,8 @@
void SanitizerMetadata::reportGlobalToASan(llvm::GlobalVariable *GV,
const VarDecl &D, bool IsDynInit) {
- if (!CGM.getLangOpts().Sanitize.has(SanitizerKind::Address))
+ if (!CGM.getLangOpts().Sanitize.hasOneOf(SanitizerKind::Address |
+ SanitizerKind::KernelAddress))
return;
std::string QualName;
llvm::raw_string_ostream OS(QualName);
@@ -67,7 +69,8 @@
void SanitizerMetadata::disableSanitizerForGlobal(llvm::GlobalVariable *GV) {
// For now, just make sure the global is not modified by the ASan
// instrumentation.
- if (CGM.getLangOpts().Sanitize.has(SanitizerKind::Address))
+ if (CGM.getLangOpts().Sanitize.hasOneOf(SanitizerKind::Address |
+ SanitizerKind::KernelAddress))
reportGlobalToASan(GV, SourceLocation(), "", QualType(), false, true);
}
diff --git a/lib/CodeGen/TargetInfo.cpp b/lib/CodeGen/TargetInfo.cpp
index 0f01620..84ffef0 100644
--- a/lib/CodeGen/TargetInfo.cpp
+++ b/lib/CodeGen/TargetInfo.cpp
@@ -64,7 +64,7 @@
for (unsigned I = FirstIndex; I <= LastIndex; ++I) {
llvm::Value *Cell =
Builder.CreateConstInBoundsGEP1_32(Builder.getInt8Ty(), Array, I);
- Builder.CreateStore(Value, Cell);
+ Builder.CreateAlignedStore(Value, Cell, CharUnits::One());
}
}
@@ -73,6 +73,24 @@
T->isMemberFunctionPointerType();
}
+ABIArgInfo
+ABIInfo::getNaturalAlignIndirect(QualType Ty, bool ByRef, bool Realign,
+ llvm::Type *Padding) const {
+ return ABIArgInfo::getIndirect(getContext().getTypeAlignInChars(Ty),
+ ByRef, Realign, Padding);
+}
+
+ABIArgInfo
+ABIInfo::getNaturalAlignIndirectInReg(QualType Ty, bool Realign) const {
+ return ABIArgInfo::getIndirectInReg(getContext().getTypeAlignInChars(Ty),
+ /*ByRef*/ false, Realign);
+}
+
+Address ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const {
+ return Address::invalid();
+}
+
ABIInfo::~ABIInfo() {}
static CGCXXABI::RecordArgABI getRecordArgABI(const RecordType *RT,
@@ -158,7 +176,7 @@
OS << "InAlloca Offset=" << getInAllocaFieldIndex();
break;
case Indirect:
- OS << "Indirect Align=" << getIndirectAlign()
+ OS << "Indirect Align=" << getIndirectAlign().getQuantity()
<< " ByVal=" << getIndirectByVal()
<< " Realign=" << getIndirectRealign();
break;
@@ -169,6 +187,135 @@
OS << ")\n";
}
+// Dynamically round a pointer up to a multiple of the given alignment.
+static llvm::Value *emitRoundPointerUpToAlignment(CodeGenFunction &CGF,
+ llvm::Value *Ptr,
+ CharUnits Align) {
+ llvm::Value *PtrAsInt = Ptr;
+ // OverflowArgArea = (OverflowArgArea + Align - 1) & -Align;
+ PtrAsInt = CGF.Builder.CreatePtrToInt(PtrAsInt, CGF.IntPtrTy);
+ PtrAsInt = CGF.Builder.CreateAdd(PtrAsInt,
+ llvm::ConstantInt::get(CGF.IntPtrTy, Align.getQuantity() - 1));
+ PtrAsInt = CGF.Builder.CreateAnd(PtrAsInt,
+ llvm::ConstantInt::get(CGF.IntPtrTy, -Align.getQuantity()));
+ PtrAsInt = CGF.Builder.CreateIntToPtr(PtrAsInt,
+ Ptr->getType(),
+ Ptr->getName() + ".aligned");
+ return PtrAsInt;
+}
+
+/// Emit va_arg for a platform using the common void* representation,
+/// where arguments are simply emitted in an array of slots on the stack.
+///
+/// This version implements the core direct-value passing rules.
+///
+/// \param SlotSize - The size and alignment of a stack slot.
+/// Each argument will be allocated to a multiple of this number of
+/// slots, and all the slots will be aligned to this value.
+/// \param AllowHigherAlign - The slot alignment is not a cap;
+/// an argument type with an alignment greater than the slot size
+/// will be emitted on a higher-alignment address, potentially
+/// leaving one or more empty slots behind as padding. If this
+/// is false, the returned address might be less-aligned than
+/// DirectAlign.
+static Address emitVoidPtrDirectVAArg(CodeGenFunction &CGF,
+ Address VAListAddr,
+ llvm::Type *DirectTy,
+ CharUnits DirectSize,
+ CharUnits DirectAlign,
+ CharUnits SlotSize,
+ bool AllowHigherAlign) {
+ // Cast the element type to i8* if necessary. Some platforms define
+ // va_list as a struct containing an i8* instead of just an i8*.
+ if (VAListAddr.getElementType() != CGF.Int8PtrTy)
+ VAListAddr = CGF.Builder.CreateElementBitCast(VAListAddr, CGF.Int8PtrTy);
+
+ llvm::Value *Ptr = CGF.Builder.CreateLoad(VAListAddr, "argp.cur");
+
+ // If the CC aligns values higher than the slot size, do so if needed.
+ Address Addr = Address::invalid();
+ if (AllowHigherAlign && DirectAlign > SlotSize) {
+ Addr = Address(emitRoundPointerUpToAlignment(CGF, Ptr, DirectAlign),
+ DirectAlign);
+ } else {
+ Addr = Address(Ptr, SlotSize);
+ }
+
+ // Advance the pointer past the argument, then store that back.
+ CharUnits FullDirectSize = DirectSize.RoundUpToAlignment(SlotSize);
+ llvm::Value *NextPtr =
+ CGF.Builder.CreateConstInBoundsByteGEP(Addr.getPointer(), FullDirectSize,
+ "argp.next");
+ CGF.Builder.CreateStore(NextPtr, VAListAddr);
+
+ // If the argument is smaller than a slot, and this is a big-endian
+ // target, the argument will be right-adjusted in its slot.
+ if (DirectSize < SlotSize && CGF.CGM.getDataLayout().isBigEndian()) {
+ Addr = CGF.Builder.CreateConstInBoundsByteGEP(Addr, SlotSize - DirectSize);
+ }
+
+ Addr = CGF.Builder.CreateElementBitCast(Addr, DirectTy);
+ return Addr;
+}
+
+/// Emit va_arg for a platform using the common void* representation,
+/// where arguments are simply emitted in an array of slots on the stack.
+///
+/// \param IsIndirect - Values of this type are passed indirectly.
+/// \param ValueInfo - The size and alignment of this type, generally
+/// computed with getContext().getTypeInfoInChars(ValueTy).
+/// \param SlotSizeAndAlign - The size and alignment of a stack slot.
+/// Each argument will be allocated to a multiple of this number of
+/// slots, and all the slots will be aligned to this value.
+/// \param AllowHigherAlign - The slot alignment is not a cap;
+/// an argument type with an alignment greater than the slot size
+/// will be emitted on a higher-alignment address, potentially
+/// leaving one or more empty slots behind as padding.
+static Address emitVoidPtrVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType ValueTy, bool IsIndirect,
+ std::pair<CharUnits, CharUnits> ValueInfo,
+ CharUnits SlotSizeAndAlign,
+ bool AllowHigherAlign) {
+ // The size and alignment of the value that was passed directly.
+ CharUnits DirectSize, DirectAlign;
+ if (IsIndirect) {
+ DirectSize = CGF.getPointerSize();
+ DirectAlign = CGF.getPointerAlign();
+ } else {
+ DirectSize = ValueInfo.first;
+ DirectAlign = ValueInfo.second;
+ }
+
+ // Cast the address we've calculated to the right type.
+ llvm::Type *DirectTy = CGF.ConvertTypeForMem(ValueTy);
+ if (IsIndirect)
+ DirectTy = DirectTy->getPointerTo(0);
+
+ Address Addr = emitVoidPtrDirectVAArg(CGF, VAListAddr, DirectTy,
+ DirectSize, DirectAlign,
+ SlotSizeAndAlign,
+ AllowHigherAlign);
+
+ if (IsIndirect) {
+ Addr = Address(CGF.Builder.CreateLoad(Addr), ValueInfo.second);
+ }
+
+ return Addr;
+
+}
+
+static Address emitMergePHI(CodeGenFunction &CGF,
+ Address Addr1, llvm::BasicBlock *Block1,
+ Address Addr2, llvm::BasicBlock *Block2,
+ const llvm::Twine &Name = "") {
+ assert(Addr1.getType() == Addr2.getType());
+ llvm::PHINode *PHI = CGF.Builder.CreatePHI(Addr1.getType(), 2, Name);
+ PHI->addIncoming(Addr1.getPointer(), Block1);
+ PHI->addIncoming(Addr2.getPointer(), Block2);
+ CharUnits Align = std::min(Addr1.getAlignment(), Addr2.getAlignment());
+ return Address(PHI, Align);
+}
+
TargetCodeGenInfo::~TargetCodeGenInfo() { delete Info; }
// If someone can figure out a general rule for this, that would be great.
@@ -419,8 +566,8 @@
I.info = classifyArgumentType(I.type);
}
- llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const override;
+ Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const override;
};
class DefaultTargetCodeGenInfo : public TargetCodeGenInfo {
@@ -429,9 +576,9 @@
: TargetCodeGenInfo(new DefaultABIInfo(CGT)) {}
};
-llvm::Value *DefaultABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const {
- return nullptr;
+Address DefaultABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const {
+ return Address::invalid();
}
ABIArgInfo DefaultABIInfo::classifyArgumentType(QualType Ty) const {
@@ -441,9 +588,9 @@
// Records with non-trivial destructors/copy-constructors should not be
// passed by value.
if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
- return ABIArgInfo::getIndirect(0, RAA == CGCXXABI::RAA_DirectInMemory);
+ return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
- return ABIArgInfo::getIndirect(0);
+ return getNaturalAlignIndirect(Ty);
}
// Treat an enum type as its underlying type.
@@ -459,7 +606,7 @@
return ABIArgInfo::getIgnore();
if (isAggregateTypeForABI(RetTy))
- return ABIArgInfo::getIndirect(0);
+ return getNaturalAlignIndirect(RetTy);
// Treat an enum type as its underlying type.
if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
@@ -470,6 +617,80 @@
}
//===----------------------------------------------------------------------===//
+// WebAssembly ABI Implementation
+//
+// This is a very simple ABI that relies a lot on DefaultABIInfo.
+//===----------------------------------------------------------------------===//
+
+class WebAssemblyABIInfo final : public DefaultABIInfo {
+public:
+ explicit WebAssemblyABIInfo(CodeGen::CodeGenTypes &CGT)
+ : DefaultABIInfo(CGT) {}
+
+private:
+ ABIArgInfo classifyReturnType(QualType RetTy) const;
+ ABIArgInfo classifyArgumentType(QualType Ty) const;
+
+ // DefaultABIInfo's classifyReturnType and classifyArgumentType are
+ // non-virtual, but computeInfo is virtual, so we overload that.
+ void computeInfo(CGFunctionInfo &FI) const override {
+ if (!getCXXABI().classifyReturnType(FI))
+ FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
+ for (auto &Arg : FI.arguments())
+ Arg.info = classifyArgumentType(Arg.type);
+ }
+};
+
+class WebAssemblyTargetCodeGenInfo final : public TargetCodeGenInfo {
+public:
+ explicit WebAssemblyTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
+ : TargetCodeGenInfo(new WebAssemblyABIInfo(CGT)) {}
+};
+
+/// \brief Classify argument of given type \p Ty.
+ABIArgInfo WebAssemblyABIInfo::classifyArgumentType(QualType Ty) const {
+ Ty = useFirstFieldIfTransparentUnion(Ty);
+
+ if (isAggregateTypeForABI(Ty)) {
+ // Records with non-trivial destructors/copy-constructors should not be
+ // passed by value.
+ if (auto RAA = getRecordArgABI(Ty, getCXXABI()))
+ return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
+ // Ignore empty structs/unions.
+ if (isEmptyRecord(getContext(), Ty, true))
+ return ABIArgInfo::getIgnore();
+ // Lower single-element structs to just pass a regular value. TODO: We
+ // could do reasonable-size multiple-element structs too, using getExpand(),
+ // though watch out for things like bitfields.
+ if (const Type *SeltTy = isSingleElementStruct(Ty, getContext()))
+ return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
+ }
+
+ // Otherwise just do the default thing.
+ return DefaultABIInfo::classifyArgumentType(Ty);
+}
+
+ABIArgInfo WebAssemblyABIInfo::classifyReturnType(QualType RetTy) const {
+ if (isAggregateTypeForABI(RetTy)) {
+ // Records with non-trivial destructors/copy-constructors should not be
+ // returned by value.
+ if (!getRecordArgABI(RetTy, getCXXABI())) {
+ // Ignore empty structs/unions.
+ if (isEmptyRecord(getContext(), RetTy, true))
+ return ABIArgInfo::getIgnore();
+ // Lower single-element structs to just return a regular value. TODO: We
+ // could do reasonable-size multiple-element structs too, using
+ // ABIArgInfo::getDirect().
+ if (const Type *SeltTy = isSingleElementStruct(RetTy, getContext()))
+ return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
+ }
+ }
+
+ // Otherwise just do the default thing.
+ return DefaultABIInfo::classifyReturnType(RetTy);
+}
+
+//===----------------------------------------------------------------------===//
// le32/PNaCl bitcode ABI Implementation
//
// This is a simplified version of the x86_32 ABI. Arguments and return values
@@ -484,8 +705,8 @@
ABIArgInfo classifyArgumentType(QualType RetTy) const;
void computeInfo(CGFunctionInfo &FI) const override;
- llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const override;
+ Address EmitVAArg(CodeGenFunction &CGF,
+ Address VAListAddr, QualType Ty) const override;
};
class PNaClTargetCodeGenInfo : public TargetCodeGenInfo {
@@ -502,17 +723,17 @@
I.info = classifyArgumentType(I.type);
}
-llvm::Value *PNaClABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const {
- return nullptr;
+Address PNaClABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const {
+ return Address::invalid();
}
/// \brief Classify argument of given type \p Ty.
ABIArgInfo PNaClABIInfo::classifyArgumentType(QualType Ty) const {
if (isAggregateTypeForABI(Ty)) {
if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
- return ABIArgInfo::getIndirect(0, RAA == CGCXXABI::RAA_DirectInMemory);
- return ABIArgInfo::getIndirect(0);
+ return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
+ return getNaturalAlignIndirect(Ty);
} else if (const EnumType *EnumTy = Ty->getAs<EnumType>()) {
// Treat an enum type as its underlying type.
Ty = EnumTy->getDecl()->getIntegerType();
@@ -531,7 +752,7 @@
// In the PNaCl ABI we always return records/structures on the stack.
if (isAggregateTypeForABI(RetTy))
- return ABIArgInfo::getIndirect(0);
+ return getNaturalAlignIndirect(RetTy);
// Treat an enum type as its underlying type.
if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
@@ -610,8 +831,10 @@
static const unsigned MinABIStackAlignInBytes = 4;
bool IsDarwinVectorABI;
- bool IsSmallStructInRegABI;
+ bool IsRetSmallStructInRegABI;
bool IsWin32StructABI;
+ bool IsSoftFloatABI;
+ bool IsMCUABI;
unsigned DefaultNumRegisterParameters;
static bool isRegisterSize(unsigned Size) {
@@ -635,7 +858,7 @@
/// such that the argument will be passed in memory.
ABIArgInfo getIndirectResult(QualType Ty, bool ByVal, CCState &State) const;
- ABIArgInfo getIndirectReturnResult(CCState &State) const;
+ ABIArgInfo getIndirectReturnResult(QualType Ty, CCState &State) const;
/// \brief Return the alignment to use for the given type on the stack.
unsigned getTypeStackAlignInBytes(QualType Ty, unsigned Align) const;
@@ -650,26 +873,34 @@
void rewriteWithInAlloca(CGFunctionInfo &FI) const;
void addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
- unsigned &StackOffset, ABIArgInfo &Info,
+ CharUnits &StackOffset, ABIArgInfo &Info,
QualType Type) const;
public:
void computeInfo(CGFunctionInfo &FI) const override;
- llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const override;
+ Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const override;
- X86_32ABIInfo(CodeGen::CodeGenTypes &CGT, bool d, bool p, bool w,
- unsigned r)
- : ABIInfo(CGT), IsDarwinVectorABI(d), IsSmallStructInRegABI(p),
- IsWin32StructABI(w), DefaultNumRegisterParameters(r) {}
+ X86_32ABIInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
+ bool RetSmallStructInRegABI, bool Win32StructABI,
+ unsigned NumRegisterParameters, bool SoftFloatABI)
+ : ABIInfo(CGT), IsDarwinVectorABI(DarwinVectorABI),
+ IsRetSmallStructInRegABI(RetSmallStructInRegABI),
+ IsWin32StructABI(Win32StructABI),
+ IsSoftFloatABI(SoftFloatABI),
+ IsMCUABI(CGT.getTarget().getTriple().isOSIAMCU()),
+ DefaultNumRegisterParameters(NumRegisterParameters) {}
};
class X86_32TargetCodeGenInfo : public TargetCodeGenInfo {
public:
- X86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
- bool d, bool p, bool w, unsigned r)
- :TargetCodeGenInfo(new X86_32ABIInfo(CGT, d, p, w, r)) {}
+ X86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
+ bool RetSmallStructInRegABI, bool Win32StructABI,
+ unsigned NumRegisterParameters, bool SoftFloatABI)
+ : TargetCodeGenInfo(new X86_32ABIInfo(
+ CGT, DarwinVectorABI, RetSmallStructInRegABI, Win32StructABI,
+ NumRegisterParameters, SoftFloatABI)) {}
static bool isStructReturnInRegABI(
const llvm::Triple &Triple, const CodeGenOptions &Opts);
@@ -792,7 +1023,7 @@
}
/// shouldReturnTypeInRegister - Determine if the given type should be
-/// passed in a register (for the Darwin ABI).
+/// returned in a register (for the Darwin and MCU ABI).
bool X86_32ABIInfo::shouldReturnTypeInRegister(QualType Ty,
ASTContext &Context) const {
uint64_t Size = Context.getTypeSize(Ty);
@@ -841,14 +1072,14 @@
return true;
}
-ABIArgInfo X86_32ABIInfo::getIndirectReturnResult(CCState &State) const {
+ABIArgInfo X86_32ABIInfo::getIndirectReturnResult(QualType RetTy, CCState &State) const {
// If the return value is indirect, then the hidden argument is consuming one
// integer register.
if (State.FreeRegs) {
--State.FreeRegs;
- return ABIArgInfo::getIndirectInReg(/*Align=*/0, /*ByVal=*/false);
+ return getNaturalAlignIndirectInReg(RetTy);
}
- return ABIArgInfo::getIndirect(/*Align=*/0, /*ByVal=*/false);
+ return getNaturalAlignIndirect(RetTy, /*ByVal=*/false);
}
ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy,
@@ -883,7 +1114,7 @@
return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
Size));
- return getIndirectReturnResult(State);
+ return getIndirectReturnResult(RetTy, State);
}
return ABIArgInfo::getDirect();
@@ -893,12 +1124,12 @@
if (const RecordType *RT = RetTy->getAs<RecordType>()) {
// Structures with flexible arrays are always indirect.
if (RT->getDecl()->hasFlexibleArrayMember())
- return getIndirectReturnResult(State);
+ return getIndirectReturnResult(RetTy, State);
}
// If specified, structs and unions are always indirect.
- if (!IsSmallStructInRegABI && !RetTy->isAnyComplexType())
- return getIndirectReturnResult(State);
+ if (!IsRetSmallStructInRegABI && !RetTy->isAnyComplexType())
+ return getIndirectReturnResult(RetTy, State);
// Small structures which are register sized are generally returned
// in a register.
@@ -920,7 +1151,7 @@
return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),Size));
}
- return getIndirectReturnResult(State);
+ return getIndirectReturnResult(RetTy, State);
}
// Treat an enum type as its underlying type.
@@ -986,21 +1217,22 @@
if (!ByVal) {
if (State.FreeRegs) {
--State.FreeRegs; // Non-byval indirects just use one pointer.
- return ABIArgInfo::getIndirectInReg(0, false);
+ return getNaturalAlignIndirectInReg(Ty);
}
- return ABIArgInfo::getIndirect(0, false);
+ return getNaturalAlignIndirect(Ty, false);
}
// Compute the byval alignment.
unsigned TypeAlign = getContext().getTypeAlign(Ty) / 8;
unsigned StackAlign = getTypeStackAlignInBytes(Ty, TypeAlign);
if (StackAlign == 0)
- return ABIArgInfo::getIndirect(4, /*ByVal=*/true);
+ return ABIArgInfo::getIndirect(CharUnits::fromQuantity(4), /*ByVal=*/true);
// If the stack alignment is less than the type alignment, realign the
// argument.
bool Realign = TypeAlign > StackAlign;
- return ABIArgInfo::getIndirect(StackAlign, /*ByVal=*/true, Realign);
+ return ABIArgInfo::getIndirect(CharUnits::fromQuantity(StackAlign),
+ /*ByVal=*/true, Realign);
}
X86_32ABIInfo::Class X86_32ABIInfo::classify(QualType Ty) const {
@@ -1019,9 +1251,11 @@
bool X86_32ABIInfo::shouldUseInReg(QualType Ty, CCState &State,
bool &NeedsPadding) const {
NeedsPadding = false;
- Class C = classify(Ty);
- if (C == Float)
- return false;
+ if (!IsSoftFloatABI) {
+ Class C = classify(Ty);
+ if (C == Float)
+ return false;
+ }
unsigned Size = getContext().getTypeSize(Ty);
unsigned SizeInRegs = (Size + 31) / 32;
@@ -1029,9 +1263,18 @@
if (SizeInRegs == 0)
return false;
- if (SizeInRegs > State.FreeRegs) {
- State.FreeRegs = 0;
- return false;
+ if (!IsMCUABI) {
+ if (SizeInRegs > State.FreeRegs) {
+ State.FreeRegs = 0;
+ return false;
+ }
+ } else {
+ // The MCU psABI allows passing parameters in-reg even if there are
+ // earlier parameters that are passed on the stack. Also,
+ // it does not allow passing >8-byte structs in-register,
+ // even if there are 3 free registers available.
+ if (SizeInRegs > State.FreeRegs || SizeInRegs > 2)
+ return false;
}
State.FreeRegs -= SizeInRegs;
@@ -1175,6 +1418,8 @@
State.FreeSSERegs = 6;
} else if (FI.getHasRegParm())
State.FreeRegs = FI.getRegParm();
+ else if (IsMCUABI)
+ State.FreeRegs = 3;
else
State.FreeRegs = DefaultNumRegisterParameters;
@@ -1207,22 +1452,23 @@
void
X86_32ABIInfo::addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
- unsigned &StackOffset,
- ABIArgInfo &Info, QualType Type) const {
- assert(StackOffset % 4U == 0 && "unaligned inalloca struct");
+ CharUnits &StackOffset, ABIArgInfo &Info,
+ QualType Type) const {
+ // Arguments are always 4-byte-aligned.
+ CharUnits FieldAlign = CharUnits::fromQuantity(4);
+
+ assert(StackOffset.isMultipleOf(FieldAlign) && "unaligned inalloca struct");
Info = ABIArgInfo::getInAlloca(FrameFields.size());
FrameFields.push_back(CGT.ConvertTypeForMem(Type));
- StackOffset += getContext().getTypeSizeInChars(Type).getQuantity();
+ StackOffset += getContext().getTypeSizeInChars(Type);
- // Insert padding bytes to respect alignment. For x86_32, each argument is 4
- // byte aligned.
- if (StackOffset % 4U) {
- unsigned OldOffset = StackOffset;
- StackOffset = llvm::RoundUpToAlignment(StackOffset, 4U);
- unsigned NumBytes = StackOffset - OldOffset;
- assert(NumBytes);
+ // Insert padding bytes to respect alignment.
+ CharUnits FieldEnd = StackOffset;
+ StackOffset = FieldEnd.RoundUpToAlignment(FieldAlign);
+ if (StackOffset != FieldEnd) {
+ CharUnits NumBytes = StackOffset - FieldEnd;
llvm::Type *Ty = llvm::Type::getInt8Ty(getVMContext());
- Ty = llvm::ArrayType::get(Ty, NumBytes);
+ Ty = llvm::ArrayType::get(Ty, NumBytes.getQuantity());
FrameFields.push_back(Ty);
}
}
@@ -1253,7 +1499,10 @@
// Build a packed struct type for all of the arguments in memory.
SmallVector<llvm::Type *, 6> FrameFields;
- unsigned StackOffset = 0;
+ // The stack alignment is always 4.
+ CharUnits StackAlign = CharUnits::fromQuantity(4);
+
+ CharUnits StackOffset;
CGFunctionInfo::arg_iterator I = FI.arg_begin(), E = FI.arg_end();
// Put 'this' into the struct before 'sret', if necessary.
@@ -1285,47 +1534,25 @@
}
FI.setArgStruct(llvm::StructType::get(getVMContext(), FrameFields,
- /*isPacked=*/true));
+ /*isPacked=*/true),
+ StackAlign);
}
-llvm::Value *X86_32ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const {
- llvm::Type *BPP = CGF.Int8PtrPtrTy;
+Address X86_32ABIInfo::EmitVAArg(CodeGenFunction &CGF,
+ Address VAListAddr, QualType Ty) const {
- CGBuilderTy &Builder = CGF.Builder;
- llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP,
- "ap");
- llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur");
+ auto TypeInfo = getContext().getTypeInfoInChars(Ty);
- // Compute if the address needs to be aligned
- unsigned Align = CGF.getContext().getTypeAlignInChars(Ty).getQuantity();
- Align = getTypeStackAlignInBytes(Ty, Align);
- Align = std::max(Align, 4U);
- if (Align > 4) {
- // addr = (addr + align - 1) & -align;
- llvm::Value *Offset =
- llvm::ConstantInt::get(CGF.Int32Ty, Align - 1);
- Addr = CGF.Builder.CreateGEP(Addr, Offset);
- llvm::Value *AsInt = CGF.Builder.CreatePtrToInt(Addr,
- CGF.Int32Ty);
- llvm::Value *Mask = llvm::ConstantInt::get(CGF.Int32Ty, -Align);
- Addr = CGF.Builder.CreateIntToPtr(CGF.Builder.CreateAnd(AsInt, Mask),
- Addr->getType(),
- "ap.cur.aligned");
- }
+ // x86-32 changes the alignment of certain arguments on the stack.
+ //
+ // Just messing with TypeInfo like this works because we never pass
+ // anything indirectly.
+ TypeInfo.second = CharUnits::fromQuantity(
+ getTypeStackAlignInBytes(Ty, TypeInfo.second.getQuantity()));
- llvm::Type *PTy =
- llvm::PointerType::getUnqual(CGF.ConvertType(Ty));
- llvm::Value *AddrTyped = Builder.CreateBitCast(Addr, PTy);
-
- uint64_t Offset =
- llvm::RoundUpToAlignment(CGF.getContext().getTypeSize(Ty) / 8, Align);
- llvm::Value *NextAddr =
- Builder.CreateGEP(Addr, llvm::ConstantInt::get(CGF.Int32Ty, Offset),
- "ap.next");
- Builder.CreateStore(NextAddr, VAListAddrAsBPP);
-
- return AddrTyped;
+ return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false,
+ TypeInfo, CharUnits::fromQuantity(4),
+ /*AllowHigherAlign*/ true);
}
bool X86_32TargetCodeGenInfo::isStructReturnInRegABI(
@@ -1341,7 +1568,7 @@
return true;
}
- if (Triple.isOSDarwin())
+ if (Triple.isOSDarwin() || Triple.isOSIAMCU())
return true;
switch (Triple.getOS()) {
@@ -1359,7 +1586,7 @@
void X86_32TargetCodeGenInfo::setTargetAttributes(const Decl *D,
llvm::GlobalValue *GV,
CodeGen::CodeGenModule &CGM) const {
- if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
+ if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
// Get the LLVM function.
llvm::Function *Fn = cast<llvm::Function>(GV);
@@ -1397,8 +1624,9 @@
} else {
// 9 is %eflags, which doesn't get a size on Darwin for some
// reason.
- Builder.CreateStore(
- Four8, Builder.CreateConstInBoundsGEP1_32(CGF.Int8Ty, Address, 9));
+ Builder.CreateAlignedStore(
+ Four8, Builder.CreateConstInBoundsGEP1_32(CGF.Int8Ty, Address, 9),
+ CharUnits::One());
// 11-16 are st(0..5). Not sure why we stop at 5.
// These have size 12, which is sizeof(long double) on
@@ -1416,6 +1644,26 @@
namespace {
+/// The AVX ABI level for X86 targets.
+enum class X86AVXABILevel {
+ None,
+ AVX,
+ AVX512
+};
+
+/// \p returns the size in bits of the largest (native) vector for \p AVXLevel.
+static unsigned getNativeVectorSizeForAVXABI(X86AVXABILevel AVXLevel) {
+ switch (AVXLevel) {
+ case X86AVXABILevel::AVX512:
+ return 512;
+ case X86AVXABILevel::AVX:
+ return 256;
+ case X86AVXABILevel::None:
+ return 128;
+ }
+ llvm_unreachable("Unknown AVXLevel");
+}
+
/// X86_64ABIInfo - The X86_64 ABI information.
class X86_64ABIInfo : public ABIInfo {
enum Class {
@@ -1521,13 +1769,14 @@
return !getTarget().getTriple().isOSDarwin();
}
+ X86AVXABILevel AVXLevel;
// Some ABIs (e.g. X32 ABI and Native Client OS) use 32 bit pointers on
// 64-bit hardware.
bool Has64BitPointers;
public:
- X86_64ABIInfo(CodeGen::CodeGenTypes &CGT) :
- ABIInfo(CGT),
+ X86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel) :
+ ABIInfo(CGT), AVXLevel(AVXLevel),
Has64BitPointers(CGT.getDataLayout().getPointerSize(0) == 8) {
}
@@ -1546,31 +1795,27 @@
void computeInfo(CGFunctionInfo &FI) const override;
- llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const override;
+ Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const override;
+ Address EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const override;
bool has64BitPointers() const {
return Has64BitPointers;
}
-
- bool hasAVX() const {
- return getTarget().getABI() == "avx";
- }
};
/// WinX86_64ABIInfo - The Windows X86_64 ABI information.
class WinX86_64ABIInfo : public ABIInfo {
-
- ABIArgInfo classify(QualType Ty, unsigned &FreeSSERegs,
- bool IsReturnType) const;
-
public:
- WinX86_64ABIInfo(CodeGen::CodeGenTypes &CGT) : ABIInfo(CGT) {}
+ WinX86_64ABIInfo(CodeGen::CodeGenTypes &CGT)
+ : ABIInfo(CGT),
+ IsMingw64(getTarget().getTriple().isWindowsGNUEnvironment()) {}
void computeInfo(CGFunctionInfo &FI) const override;
- llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const override;
+ Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const override;
bool isHomogeneousAggregateBaseType(QualType Ty) const override {
// FIXME: Assumes vectorcall is in use.
@@ -1582,12 +1827,18 @@
// FIXME: Assumes vectorcall is in use.
return isX86VectorCallAggregateSmallEnough(NumMembers);
}
+
+private:
+ ABIArgInfo classify(QualType Ty, unsigned &FreeSSERegs,
+ bool IsReturnType) const;
+
+ bool IsMingw64;
};
class X86_64TargetCodeGenInfo : public TargetCodeGenInfo {
public:
- X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
- : TargetCodeGenInfo(new X86_64ABIInfo(CGT)) {}
+ X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
+ : TargetCodeGenInfo(new X86_64ABIInfo(CGT, AVXLevel)) {}
const X86_64ABIInfo &getABIInfo() const {
return static_cast<const X86_64ABIInfo&>(TargetCodeGenInfo::getABIInfo());
@@ -1653,21 +1904,21 @@
('T' << 24);
return llvm::ConstantInt::get(CGM.Int32Ty, Sig);
}
-
- unsigned getOpenMPSimdDefaultAlignment(QualType) const override {
- return getABIInfo().hasAVX() ? 32 : 16;
- }
};
class PS4TargetCodeGenInfo : public X86_64TargetCodeGenInfo {
public:
- PS4TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
- : X86_64TargetCodeGenInfo(CGT) {}
+ PS4TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
+ : X86_64TargetCodeGenInfo(CGT, AVXLevel) {}
void getDependentLibraryOption(llvm::StringRef Lib,
llvm::SmallString<24> &Opt) const override {
Opt = "\01";
- Opt += Lib;
+ // If the argument contains a space, enclose it in quotes.
+ if (Lib.find(" ") != StringRef::npos)
+ Opt += "\"" + Lib.str() + "\"";
+ else
+ Opt += Lib;
}
};
@@ -1687,8 +1938,10 @@
class WinX86_32TargetCodeGenInfo : public X86_32TargetCodeGenInfo {
public:
WinX86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
- bool d, bool p, bool w, unsigned RegParms)
- : X86_32TargetCodeGenInfo(CGT, d, p, w, RegParms) {}
+ bool DarwinVectorABI, bool RetSmallStructInRegABI, bool Win32StructABI,
+ unsigned NumRegisterParameters)
+ : X86_32TargetCodeGenInfo(CGT, DarwinVectorABI, RetSmallStructInRegABI,
+ Win32StructABI, NumRegisterParameters, false) {}
void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
CodeGen::CodeGenModule &CGM) const override;
@@ -1709,7 +1962,7 @@
static void addStackProbeSizeTargetAttribute(const Decl *D,
llvm::GlobalValue *GV,
CodeGen::CodeGenModule &CGM) {
- if (isa<FunctionDecl>(D)) {
+ if (D && isa<FunctionDecl>(D)) {
if (CGM.getCodeGenOpts().StackProbeSize != 4096) {
llvm::Function *Fn = cast<llvm::Function>(GV);
@@ -1728,11 +1981,10 @@
}
class WinX86_64TargetCodeGenInfo : public TargetCodeGenInfo {
- bool hasAVX() const { return getABIInfo().getTarget().getABI() == "avx"; }
-
public:
- WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
- : TargetCodeGenInfo(new WinX86_64ABIInfo(CGT)) {}
+ WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
+ X86AVXABILevel AVXLevel)
+ : TargetCodeGenInfo(new WinX86_64ABIInfo(CGT)) {}
void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
CodeGen::CodeGenModule &CGM) const override;
@@ -1762,10 +2014,6 @@
llvm::SmallString<32> &Opt) const override {
Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
}
-
- unsigned getOpenMPSimdDefaultAlignment(QualType) const override {
- return hasAVX() ? 32 : 16;
- }
};
void WinX86_64TargetCodeGenInfo::setTargetAttributes(const Decl *D,
@@ -1875,13 +2123,20 @@
Hi = Integer;
} else if (k >= BuiltinType::Bool && k <= BuiltinType::LongLong) {
Current = Integer;
- } else if ((k == BuiltinType::Float || k == BuiltinType::Double) ||
- (k == BuiltinType::LongDouble &&
- getTarget().getTriple().isOSNaCl())) {
+ } else if (k == BuiltinType::Float || k == BuiltinType::Double) {
Current = SSE;
} else if (k == BuiltinType::LongDouble) {
- Lo = X87;
- Hi = X87Up;
+ const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
+ if (LDF == &llvm::APFloat::IEEEquad) {
+ Lo = SSE;
+ Hi = SSEUp;
+ } else if (LDF == &llvm::APFloat::x87DoubleExtended) {
+ Lo = X87;
+ Hi = X87Up;
+ } else if (LDF == &llvm::APFloat::IEEEdouble) {
+ Current = SSE;
+ } else
+ llvm_unreachable("unexpected long double representation!");
}
// FIXME: _Decimal32 and _Decimal64 are SSE.
// FIXME: _float128 and _Decimal128 are (SSE, SSEUp).
@@ -1924,16 +2179,18 @@
if (const VectorType *VT = Ty->getAs<VectorType>()) {
uint64_t Size = getContext().getTypeSize(VT);
- if (Size == 32) {
- // gcc passes all <4 x char>, <2 x short>, <1 x int>, <1 x
- // float> as integer.
+ if (Size == 1 || Size == 8 || Size == 16 || Size == 32) {
+ // gcc passes the following as integer:
+ // 4 bytes - <4 x char>, <2 x short>, <1 x int>, <1 x float>
+ // 2 bytes - <2 x char>, <1 x short>
+ // 1 byte - <1 x char>
Current = Integer;
// If this type crosses an eightbyte boundary, it should be
// split.
- uint64_t EB_Real = (OffsetBase) / 64;
- uint64_t EB_Imag = (OffsetBase + Size - 1) / 64;
- if (EB_Real != EB_Imag)
+ uint64_t EB_Lo = (OffsetBase) / 64;
+ uint64_t EB_Hi = (OffsetBase + Size - 1) / 64;
+ if (EB_Lo != EB_Hi)
Hi = Lo;
} else if (Size == 64) {
// gcc passes <1 x double> in memory. :(
@@ -1953,7 +2210,8 @@
// split.
if (OffsetBase && OffsetBase != 64)
Hi = Lo;
- } else if (Size == 128 || (hasAVX() && isNamedArg && Size == 256)) {
+ } else if (Size == 128 ||
+ (isNamedArg && Size <= getNativeVectorSizeForAVXABI(AVXLevel))) {
// Arguments of 256-bits are split into four eightbyte chunks. The
// least significant one belongs to class SSE and all the others to class
// SSEUP. The original Lo and Hi design considers that types can't be
@@ -1965,6 +2223,9 @@
// Note that per 3.5.7 of AMD64-ABI, 256-bit args are only passed in
// registers if they are "named", i.e. not part of the "..." of a
// variadic function.
+ //
+ // Similarly, per 3.2.3. of the AVX512 draft, 512-bits ("named") args are
+ // split into eight eightbyte chunks, one SSE and seven SSEUP.
Lo = SSE;
Hi = SSEUp;
}
@@ -1980,14 +2241,21 @@
Current = Integer;
else if (Size <= 128)
Lo = Hi = Integer;
- } else if (ET == getContext().FloatTy)
+ } else if (ET == getContext().FloatTy) {
Current = SSE;
- else if (ET == getContext().DoubleTy ||
- (ET == getContext().LongDoubleTy &&
- getTarget().getTriple().isOSNaCl()))
+ } else if (ET == getContext().DoubleTy) {
Lo = Hi = SSE;
- else if (ET == getContext().LongDoubleTy)
- Current = ComplexX87;
+ } else if (ET == getContext().LongDoubleTy) {
+ const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
+ if (LDF == &llvm::APFloat::IEEEquad)
+ Current = Memory;
+ else if (LDF == &llvm::APFloat::x87DoubleExtended)
+ Current = ComplexX87;
+ else if (LDF == &llvm::APFloat::IEEEdouble)
+ Lo = Hi = SSE;
+ else
+ llvm_unreachable("unexpected long double representation!");
+ }
// If this complex type crosses an eightbyte boundary then it
// should be split.
@@ -2086,8 +2354,10 @@
classify(I.getType(), Offset, FieldLo, FieldHi, isNamedArg);
Lo = merge(Lo, FieldLo);
Hi = merge(Hi, FieldHi);
- if (Lo == Memory || Hi == Memory)
- break;
+ if (Lo == Memory || Hi == Memory) {
+ postMerge(Size, Lo, Hi);
+ return;
+ }
}
}
@@ -2107,11 +2377,13 @@
//
if (Size > 128 && getContext().getTypeSize(i->getType()) != 256) {
Lo = Memory;
+ postMerge(Size, Lo, Hi);
return;
}
// Note, skip this test for bit-fields, see below.
if (!BitField && Offset % getContext().getTypeAlign(i->getType())) {
Lo = Memory;
+ postMerge(Size, Lo, Hi);
return;
}
@@ -2169,13 +2441,13 @@
ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
}
- return ABIArgInfo::getIndirect(0);
+ return getNaturalAlignIndirect(Ty);
}
bool X86_64ABIInfo::IsIllegalVectorType(QualType Ty) const {
if (const VectorType *VecTy = Ty->getAs<VectorType>()) {
uint64_t Size = getContext().getTypeSize(VecTy);
- unsigned LargestVector = hasAVX() ? 256 : 128;
+ unsigned LargestVector = getNativeVectorSizeForAVXABI(AVXLevel);
if (Size <= 64 || Size > LargestVector)
return true;
}
@@ -2203,7 +2475,7 @@
}
if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
- return ABIArgInfo::getIndirect(0, RAA == CGCXXABI::RAA_DirectInMemory);
+ return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
// Compute the byval alignment. We specify the alignment of the byval in all
// cases so that the mid-level optimizer knows the alignment of the byval.
@@ -2240,7 +2512,7 @@
Size));
}
- return ABIArgInfo::getIndirect(Align);
+ return ABIArgInfo::getIndirect(CharUnits::fromQuantity(Align));
}
/// The ABI specifies that a value should be passed in a full vector XMM/YMM
@@ -2252,7 +2524,8 @@
Ty = QualType(InnerTy, 0);
llvm::Type *IRType = CGT.ConvertType(Ty);
- if(isa<llvm::VectorType>(IRType))
+ if (isa<llvm::VectorType>(IRType) ||
+ IRType->getTypeID() == llvm::Type::FP128TyID)
return IRType;
// We couldn't find the preferred IR vector type for 'Ty'.
@@ -2500,13 +2773,16 @@
// of the second element because it might make us access off the end of the
// struct.
if (HiStart != 8) {
- // There are only two sorts of types the ABI generation code can produce for
- // the low part of a pair that aren't 8 bytes in size: float or i8/i16/i32.
+ // There are usually two sorts of types the ABI generation code can produce
+ // for the low part of a pair that aren't 8 bytes in size: float or
+ // i8/i16/i32. This can also include pointers when they are 32-bit (X32 and
+ // NaCl).
// Promote these to a larger type.
if (Lo->isFloatTy())
Lo = llvm::Type::getDoubleTy(Lo->getContext());
else {
- assert(Lo->isIntegerTy() && "Invalid/unknown lo type");
+ assert((Lo->isIntegerTy() || Lo->isPointerTy())
+ && "Invalid/unknown lo type");
Lo = llvm::Type::getInt64Ty(Lo->getContext());
}
}
@@ -2820,11 +3096,10 @@
}
}
-static llvm::Value *EmitVAArgFromMemory(llvm::Value *VAListAddr,
- QualType Ty,
- CodeGenFunction &CGF) {
- llvm::Value *overflow_arg_area_p = CGF.Builder.CreateStructGEP(
- nullptr, VAListAddr, 2, "overflow_arg_area_p");
+static Address EmitX86_64VAArgFromMemory(CodeGenFunction &CGF,
+ Address VAListAddr, QualType Ty) {
+ Address overflow_arg_area_p = CGF.Builder.CreateStructGEP(
+ VAListAddr, 2, CharUnits::fromQuantity(8), "overflow_arg_area_p");
llvm::Value *overflow_arg_area =
CGF.Builder.CreateLoad(overflow_arg_area_p, "overflow_arg_area");
@@ -2832,19 +3107,10 @@
// byte boundary if alignment needed by type exceeds 8 byte boundary.
// It isn't stated explicitly in the standard, but in practice we use
// alignment greater than 16 where necessary.
- uint64_t Align = CGF.getContext().getTypeAlign(Ty) / 8;
- if (Align > 8) {
- // overflow_arg_area = (overflow_arg_area + align - 1) & -align;
- llvm::Value *Offset =
- llvm::ConstantInt::get(CGF.Int64Ty, Align - 1);
- overflow_arg_area = CGF.Builder.CreateGEP(overflow_arg_area, Offset);
- llvm::Value *AsInt = CGF.Builder.CreatePtrToInt(overflow_arg_area,
- CGF.Int64Ty);
- llvm::Value *Mask = llvm::ConstantInt::get(CGF.Int64Ty, -(uint64_t)Align);
- overflow_arg_area =
- CGF.Builder.CreateIntToPtr(CGF.Builder.CreateAnd(AsInt, Mask),
- overflow_arg_area->getType(),
- "overflow_arg_area.align");
+ CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty);
+ if (Align > CharUnits::fromQuantity(8)) {
+ overflow_arg_area = emitRoundPointerUpToAlignment(CGF, overflow_arg_area,
+ Align);
}
// AMD64-ABI 3.5.7p5: Step 8. Fetch type from l->overflow_arg_area.
@@ -2866,11 +3132,11 @@
CGF.Builder.CreateStore(overflow_arg_area, overflow_arg_area_p);
// AMD64-ABI 3.5.7p5: Step 11. Return the fetched type.
- return Res;
+ return Address(Res, Align);
}
-llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const {
+Address X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const {
// Assume that va_list type is correct; should be pointer to LLVM type:
// struct {
// i32 gp_offset;
@@ -2880,14 +3146,14 @@
// };
unsigned neededInt, neededSSE;
- Ty = CGF.getContext().getCanonicalType(Ty);
+ Ty = getContext().getCanonicalType(Ty);
ABIArgInfo AI = classifyArgumentType(Ty, 0, neededInt, neededSSE,
/*isNamedArg*/false);
// AMD64-ABI 3.5.7p5: Step 1. Determine whether type may be passed
// in the registers. If not go to step 7.
if (!neededInt && !neededSSE)
- return EmitVAArgFromMemory(VAListAddr, Ty, CGF);
+ return EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);
// AMD64-ABI 3.5.7p5: Step 2. Compute num_gp to hold the number of
// general purpose registers needed to pass type and num_fp to hold
@@ -2901,11 +3167,12 @@
// register save space).
llvm::Value *InRegs = nullptr;
- llvm::Value *gp_offset_p = nullptr, *gp_offset = nullptr;
- llvm::Value *fp_offset_p = nullptr, *fp_offset = nullptr;
+ Address gp_offset_p = Address::invalid(), fp_offset_p = Address::invalid();
+ llvm::Value *gp_offset = nullptr, *fp_offset = nullptr;
if (neededInt) {
gp_offset_p =
- CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 0, "gp_offset_p");
+ CGF.Builder.CreateStructGEP(VAListAddr, 0, CharUnits::Zero(),
+ "gp_offset_p");
gp_offset = CGF.Builder.CreateLoad(gp_offset_p, "gp_offset");
InRegs = llvm::ConstantInt::get(CGF.Int32Ty, 48 - neededInt * 8);
InRegs = CGF.Builder.CreateICmpULE(gp_offset, InRegs, "fits_in_gp");
@@ -2913,7 +3180,8 @@
if (neededSSE) {
fp_offset_p =
- CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 1, "fp_offset_p");
+ CGF.Builder.CreateStructGEP(VAListAddr, 1, CharUnits::fromQuantity(4),
+ "fp_offset_p");
fp_offset = CGF.Builder.CreateLoad(fp_offset_p, "fp_offset");
llvm::Value *FitsInFP =
llvm::ConstantInt::get(CGF.Int32Ty, 176 - neededSSE * 16);
@@ -2941,14 +3209,17 @@
// simple assembling of a structure from scattered addresses has many more
// loads than necessary. Can we clean this up?
llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);
- llvm::Value *RegAddr = CGF.Builder.CreateLoad(
- CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 3), "reg_save_area");
+ llvm::Value *RegSaveArea = CGF.Builder.CreateLoad(
+ CGF.Builder.CreateStructGEP(VAListAddr, 3, CharUnits::fromQuantity(16)),
+ "reg_save_area");
+
+ Address RegAddr = Address::invalid();
if (neededInt && neededSSE) {
// FIXME: Cleanup.
assert(AI.isDirect() && "Unexpected ABI info for mixed regs");
llvm::StructType *ST = cast<llvm::StructType>(AI.getCoerceToType());
- llvm::Value *Tmp = CGF.CreateMemTemp(Ty);
- Tmp = CGF.Builder.CreateBitCast(Tmp, ST->getPointerTo());
+ Address Tmp = CGF.CreateMemTemp(Ty);
+ Tmp = CGF.Builder.CreateElementBitCast(Tmp, ST);
assert(ST->getNumElements() == 2 && "Unexpected ABI info for mixed regs");
llvm::Type *TyLo = ST->getElementType(0);
llvm::Type *TyHi = ST->getElementType(1);
@@ -2956,57 +3227,77 @@
"Unexpected ABI info for mixed regs");
llvm::Type *PTyLo = llvm::PointerType::getUnqual(TyLo);
llvm::Type *PTyHi = llvm::PointerType::getUnqual(TyHi);
- llvm::Value *GPAddr = CGF.Builder.CreateGEP(RegAddr, gp_offset);
- llvm::Value *FPAddr = CGF.Builder.CreateGEP(RegAddr, fp_offset);
+ llvm::Value *GPAddr = CGF.Builder.CreateGEP(RegSaveArea, gp_offset);
+ llvm::Value *FPAddr = CGF.Builder.CreateGEP(RegSaveArea, fp_offset);
llvm::Value *RegLoAddr = TyLo->isFPOrFPVectorTy() ? FPAddr : GPAddr;
llvm::Value *RegHiAddr = TyLo->isFPOrFPVectorTy() ? GPAddr : FPAddr;
- llvm::Value *V =
- CGF.Builder.CreateLoad(CGF.Builder.CreateBitCast(RegLoAddr, PTyLo));
- CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(ST, Tmp, 0));
- V = CGF.Builder.CreateLoad(CGF.Builder.CreateBitCast(RegHiAddr, PTyHi));
- CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(ST, Tmp, 1));
- RegAddr = CGF.Builder.CreateBitCast(Tmp,
- llvm::PointerType::getUnqual(LTy));
+ // Copy the first element.
+ llvm::Value *V =
+ CGF.Builder.CreateDefaultAlignedLoad(
+ CGF.Builder.CreateBitCast(RegLoAddr, PTyLo));
+ CGF.Builder.CreateStore(V,
+ CGF.Builder.CreateStructGEP(Tmp, 0, CharUnits::Zero()));
+
+ // Copy the second element.
+ V = CGF.Builder.CreateDefaultAlignedLoad(
+ CGF.Builder.CreateBitCast(RegHiAddr, PTyHi));
+ CharUnits Offset = CharUnits::fromQuantity(
+ getDataLayout().getStructLayout(ST)->getElementOffset(1));
+ CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1, Offset));
+
+ RegAddr = CGF.Builder.CreateElementBitCast(Tmp, LTy);
} else if (neededInt) {
- RegAddr = CGF.Builder.CreateGEP(RegAddr, gp_offset);
- RegAddr = CGF.Builder.CreateBitCast(RegAddr,
- llvm::PointerType::getUnqual(LTy));
+ RegAddr = Address(CGF.Builder.CreateGEP(RegSaveArea, gp_offset),
+ CharUnits::fromQuantity(8));
+ RegAddr = CGF.Builder.CreateElementBitCast(RegAddr, LTy);
// Copy to a temporary if necessary to ensure the appropriate alignment.
std::pair<CharUnits, CharUnits> SizeAlign =
- CGF.getContext().getTypeInfoInChars(Ty);
+ getContext().getTypeInfoInChars(Ty);
uint64_t TySize = SizeAlign.first.getQuantity();
- unsigned TyAlign = SizeAlign.second.getQuantity();
- if (TyAlign > 8) {
- llvm::Value *Tmp = CGF.CreateMemTemp(Ty);
- CGF.Builder.CreateMemCpy(Tmp, RegAddr, TySize, 8, false);
+ CharUnits TyAlign = SizeAlign.second;
+
+ // Copy into a temporary if the type is more aligned than the
+ // register save area.
+ if (TyAlign.getQuantity() > 8) {
+ Address Tmp = CGF.CreateMemTemp(Ty);
+ CGF.Builder.CreateMemCpy(Tmp, RegAddr, TySize, false);
RegAddr = Tmp;
}
+
} else if (neededSSE == 1) {
- RegAddr = CGF.Builder.CreateGEP(RegAddr, fp_offset);
- RegAddr = CGF.Builder.CreateBitCast(RegAddr,
- llvm::PointerType::getUnqual(LTy));
+ RegAddr = Address(CGF.Builder.CreateGEP(RegSaveArea, fp_offset),
+ CharUnits::fromQuantity(16));
+ RegAddr = CGF.Builder.CreateElementBitCast(RegAddr, LTy);
} else {
assert(neededSSE == 2 && "Invalid number of needed registers!");
// SSE registers are spaced 16 bytes apart in the register save
// area, we need to collect the two eightbytes together.
- llvm::Value *RegAddrLo = CGF.Builder.CreateGEP(RegAddr, fp_offset);
- llvm::Value *RegAddrHi = CGF.Builder.CreateConstGEP1_32(RegAddrLo, 16);
+ // The ABI isn't explicit about this, but it seems reasonable
+ // to assume that the slots are 16-byte aligned, since the stack is
+ // naturally 16-byte aligned and the prologue is expected to store
+ // all the SSE registers to the RSA.
+ Address RegAddrLo = Address(CGF.Builder.CreateGEP(RegSaveArea, fp_offset),
+ CharUnits::fromQuantity(16));
+ Address RegAddrHi =
+ CGF.Builder.CreateConstInBoundsByteGEP(RegAddrLo,
+ CharUnits::fromQuantity(16));
llvm::Type *DoubleTy = CGF.DoubleTy;
- llvm::Type *DblPtrTy =
- llvm::PointerType::getUnqual(DoubleTy);
llvm::StructType *ST = llvm::StructType::get(DoubleTy, DoubleTy, nullptr);
- llvm::Value *V, *Tmp = CGF.CreateMemTemp(Ty);
- Tmp = CGF.Builder.CreateBitCast(Tmp, ST->getPointerTo());
- V = CGF.Builder.CreateLoad(CGF.Builder.CreateBitCast(RegAddrLo,
- DblPtrTy));
- CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(ST, Tmp, 0));
- V = CGF.Builder.CreateLoad(CGF.Builder.CreateBitCast(RegAddrHi,
- DblPtrTy));
- CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(ST, Tmp, 1));
- RegAddr = CGF.Builder.CreateBitCast(Tmp,
- llvm::PointerType::getUnqual(LTy));
+ llvm::Value *V;
+ Address Tmp = CGF.CreateMemTemp(Ty);
+ Tmp = CGF.Builder.CreateElementBitCast(Tmp, ST);
+ V = CGF.Builder.CreateLoad(
+ CGF.Builder.CreateElementBitCast(RegAddrLo, DoubleTy));
+ CGF.Builder.CreateStore(V,
+ CGF.Builder.CreateStructGEP(Tmp, 0, CharUnits::Zero()));
+ V = CGF.Builder.CreateLoad(
+ CGF.Builder.CreateElementBitCast(RegAddrHi, DoubleTy));
+ CGF.Builder.CreateStore(V,
+ CGF.Builder.CreateStructGEP(Tmp, 1, CharUnits::fromQuantity(8)));
+
+ RegAddr = CGF.Builder.CreateElementBitCast(Tmp, LTy);
}
// AMD64-ABI 3.5.7p5: Step 5. Set:
@@ -3027,18 +3318,24 @@
// Emit code to load the value if it was passed in memory.
CGF.EmitBlock(InMemBlock);
- llvm::Value *MemAddr = EmitVAArgFromMemory(VAListAddr, Ty, CGF);
+ Address MemAddr = EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);
// Return the appropriate result.
CGF.EmitBlock(ContBlock);
- llvm::PHINode *ResAddr = CGF.Builder.CreatePHI(RegAddr->getType(), 2,
- "vaarg.addr");
- ResAddr->addIncoming(RegAddr, InRegBlock);
- ResAddr->addIncoming(MemAddr, InMemBlock);
+ Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock, MemAddr, InMemBlock,
+ "vaarg.addr");
return ResAddr;
}
+Address X86_64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const {
+ return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*indirect*/ false,
+ CGF.getContext().getTypeInfoInChars(Ty),
+ CharUnits::fromQuantity(8),
+ /*allowHigherAlign*/ false);
+}
+
ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,
bool IsReturnType) const {
@@ -3050,22 +3347,18 @@
TypeInfo Info = getContext().getTypeInfo(Ty);
uint64_t Width = Info.Width;
- unsigned Align = getContext().toCharUnitsFromBits(Info.Align).getQuantity();
+ CharUnits Align = getContext().toCharUnitsFromBits(Info.Align);
const RecordType *RT = Ty->getAs<RecordType>();
if (RT) {
if (!IsReturnType) {
if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI()))
- return ABIArgInfo::getIndirect(0, RAA == CGCXXABI::RAA_DirectInMemory);
+ return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
}
if (RT->getDecl()->hasFlexibleArrayMember())
- return ABIArgInfo::getIndirect(0, /*ByVal=*/false);
+ return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
- // FIXME: mingw-w64-gcc emits 128-bit struct as i128
- if (Width == 128 && getTarget().getTriple().isWindowsGNUEnvironment())
- return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
- Width));
}
// vectorcall adds the concept of a homogenous vector aggregate, similar to
@@ -3095,7 +3388,7 @@
// MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
// not 1, 2, 4, or 8 bytes, must be passed by reference."
if (Width > 64 || !llvm::isPowerOf2_64(Width))
- return ABIArgInfo::getIndirect(0, /*ByVal=*/false);
+ return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
// Otherwise, coerce it to a small integer.
return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Width));
@@ -3107,6 +3400,14 @@
if (BT && BT->getKind() == BuiltinType::Bool)
return ABIArgInfo::getExtend();
+ // Mingw64 GCC uses the old 80 bit extended precision floating point unit. It
+ // passes them indirectly through memory.
+ if (IsMingw64 && BT && BT->getKind() == BuiltinType::LongDouble) {
+ const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
+ if (LDF == &llvm::APFloat::x87DoubleExtended)
+ return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
+ }
+
return ABIArgInfo::getDirect();
}
@@ -3125,43 +3426,31 @@
I.info = classify(I.type, FreeSSERegs, false);
}
-llvm::Value *WinX86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const {
- llvm::Type *BPP = CGF.Int8PtrPtrTy;
-
- CGBuilderTy &Builder = CGF.Builder;
- llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP,
- "ap");
- llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur");
- llvm::Type *PTy =
- llvm::PointerType::getUnqual(CGF.ConvertType(Ty));
- llvm::Value *AddrTyped = Builder.CreateBitCast(Addr, PTy);
-
- uint64_t Offset =
- llvm::RoundUpToAlignment(CGF.getContext().getTypeSize(Ty) / 8, 8);
- llvm::Value *NextAddr =
- Builder.CreateGEP(Addr, llvm::ConstantInt::get(CGF.Int32Ty, Offset),
- "ap.next");
- Builder.CreateStore(NextAddr, VAListAddrAsBPP);
-
- return AddrTyped;
+Address WinX86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const {
+ return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*indirect*/ false,
+ CGF.getContext().getTypeInfoInChars(Ty),
+ CharUnits::fromQuantity(8),
+ /*allowHigherAlign*/ false);
}
// PowerPC-32
namespace {
/// PPC32_SVR4_ABIInfo - The 32-bit PowerPC ELF (SVR4) ABI information.
class PPC32_SVR4_ABIInfo : public DefaultABIInfo {
+bool IsSoftFloatABI;
public:
- PPC32_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT) : DefaultABIInfo(CGT) {}
+ PPC32_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, bool SoftFloatABI)
+ : DefaultABIInfo(CGT), IsSoftFloatABI(SoftFloatABI) {}
- llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const override;
+ Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const override;
};
class PPC32TargetCodeGenInfo : public TargetCodeGenInfo {
public:
- PPC32TargetCodeGenInfo(CodeGenTypes &CGT)
- : TargetCodeGenInfo(new PPC32_SVR4_ABIInfo(CGT)) {}
+ PPC32TargetCodeGenInfo(CodeGenTypes &CGT, bool SoftFloatABI)
+ : TargetCodeGenInfo(new PPC32_SVR4_ABIInfo(CGT, SoftFloatABI)) {}
int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
// This is recovered from gcc output.
@@ -3170,72 +3459,55 @@
bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
llvm::Value *Address) const override;
-
- unsigned getOpenMPSimdDefaultAlignment(QualType) const override {
- return 16; // Natural alignment for Altivec vectors.
- }
};
}
-llvm::Value *PPC32_SVR4_ABIInfo::EmitVAArg(llvm::Value *VAListAddr,
- QualType Ty,
- CodeGenFunction &CGF) const {
+Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList,
+ QualType Ty) const {
if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
// TODO: Implement this. For now ignore.
(void)CTy;
- return nullptr;
+ return Address::invalid();
}
+ // struct __va_list_tag {
+ // unsigned char gpr;
+ // unsigned char fpr;
+ // unsigned short reserved;
+ // void *overflow_arg_area;
+ // void *reg_save_area;
+ // };
+
bool isI64 = Ty->isIntegerType() && getContext().getTypeSize(Ty) == 64;
bool isInt =
Ty->isIntegerType() || Ty->isPointerType() || Ty->isAggregateType();
- llvm::Type *CharPtr = CGF.Int8PtrTy;
- llvm::Type *CharPtrPtr = CGF.Int8PtrPtrTy;
+ bool isF64 = Ty->isFloatingType() && getContext().getTypeSize(Ty) == 64;
+
+ // All aggregates are passed indirectly? That doesn't seem consistent
+ // with the argument-lowering code.
+ bool isIndirect = Ty->isAggregateType();
CGBuilderTy &Builder = CGF.Builder;
- llvm::Value *GPRPtr = Builder.CreateBitCast(VAListAddr, CharPtr, "gprptr");
- llvm::Value *GPRPtrAsInt = Builder.CreatePtrToInt(GPRPtr, CGF.Int32Ty);
- llvm::Value *FPRPtrAsInt =
- Builder.CreateAdd(GPRPtrAsInt, Builder.getInt32(1));
- llvm::Value *FPRPtr = Builder.CreateIntToPtr(FPRPtrAsInt, CharPtr);
- llvm::Value *OverflowAreaPtrAsInt =
- Builder.CreateAdd(FPRPtrAsInt, Builder.getInt32(3));
- llvm::Value *OverflowAreaPtr =
- Builder.CreateIntToPtr(OverflowAreaPtrAsInt, CharPtrPtr);
- llvm::Value *RegsaveAreaPtrAsInt =
- Builder.CreateAdd(OverflowAreaPtrAsInt, Builder.getInt32(4));
- llvm::Value *RegsaveAreaPtr =
- Builder.CreateIntToPtr(RegsaveAreaPtrAsInt, CharPtrPtr);
- llvm::Value *GPR = Builder.CreateLoad(GPRPtr, false, "gpr");
- // Align GPR when TY is i64.
- if (isI64) {
- llvm::Value *GPRAnd = Builder.CreateAnd(GPR, Builder.getInt8(1));
- llvm::Value *CC64 = Builder.CreateICmpEQ(GPRAnd, Builder.getInt8(1));
- llvm::Value *GPRPlusOne = Builder.CreateAdd(GPR, Builder.getInt8(1));
- GPR = Builder.CreateSelect(CC64, GPRPlusOne, GPR);
+
+ // The calling convention either uses 1-2 GPRs or 1 FPR.
+ Address NumRegsAddr = Address::invalid();
+ if (isInt || IsSoftFloatABI) {
+ NumRegsAddr = Builder.CreateStructGEP(VAList, 0, CharUnits::Zero(), "gpr");
+ } else {
+ NumRegsAddr = Builder.CreateStructGEP(VAList, 1, CharUnits::One(), "fpr");
}
- llvm::Value *FPR = Builder.CreateLoad(FPRPtr, false, "fpr");
- llvm::Value *OverflowArea =
- Builder.CreateLoad(OverflowAreaPtr, false, "overflow_area");
- llvm::Value *OverflowAreaAsInt =
- Builder.CreatePtrToInt(OverflowArea, CGF.Int32Ty);
- llvm::Value *RegsaveArea =
- Builder.CreateLoad(RegsaveAreaPtr, false, "regsave_area");
- llvm::Value *RegsaveAreaAsInt =
- Builder.CreatePtrToInt(RegsaveArea, CGF.Int32Ty);
+
+ llvm::Value *NumRegs = Builder.CreateLoad(NumRegsAddr, "numUsedRegs");
+
+ // "Align" the register count when TY is i64.
+ if (isI64 || (isF64 && IsSoftFloatABI)) {
+ NumRegs = Builder.CreateAdd(NumRegs, Builder.getInt8(1));
+ NumRegs = Builder.CreateAnd(NumRegs, Builder.getInt8((uint8_t) ~1U));
+ }
llvm::Value *CC =
- Builder.CreateICmpULT(isInt ? GPR : FPR, Builder.getInt8(8), "cond");
-
- llvm::Value *RegConstant =
- Builder.CreateMul(isInt ? GPR : FPR, Builder.getInt8(isInt ? 4 : 8));
-
- llvm::Value *OurReg = Builder.CreateAdd(
- RegsaveAreaAsInt, Builder.CreateSExt(RegConstant, CGF.Int32Ty));
-
- if (Ty->isFloatingType())
- OurReg = Builder.CreateAdd(OurReg, Builder.getInt32(32));
+ Builder.CreateICmpULT(NumRegs, Builder.getInt8(8), "cond");
llvm::BasicBlock *UsingRegs = CGF.createBasicBlock("using_regs");
llvm::BasicBlock *UsingOverflow = CGF.createBasicBlock("using_overflow");
@@ -3243,39 +3515,91 @@
Builder.CreateCondBr(CC, UsingRegs, UsingOverflow);
- CGF.EmitBlock(UsingRegs);
+ llvm::Type *DirectTy = CGF.ConvertType(Ty);
+ if (isIndirect) DirectTy = DirectTy->getPointerTo(0);
- llvm::Type *PTy = llvm::PointerType::getUnqual(CGF.ConvertType(Ty));
- llvm::Value *Result1 = Builder.CreateIntToPtr(OurReg, PTy);
- // Increase the GPR/FPR indexes.
- if (isInt) {
- GPR = Builder.CreateAdd(GPR, Builder.getInt8(isI64 ? 2 : 1));
- Builder.CreateStore(GPR, GPRPtr);
- } else {
- FPR = Builder.CreateAdd(FPR, Builder.getInt8(1));
- Builder.CreateStore(FPR, FPRPtr);
+ // Case 1: consume registers.
+ Address RegAddr = Address::invalid();
+ {
+ CGF.EmitBlock(UsingRegs);
+
+ Address RegSaveAreaPtr =
+ Builder.CreateStructGEP(VAList, 4, CharUnits::fromQuantity(8));
+ RegAddr = Address(Builder.CreateLoad(RegSaveAreaPtr),
+ CharUnits::fromQuantity(8));
+ assert(RegAddr.getElementType() == CGF.Int8Ty);
+
+ // Floating-point registers start after the general-purpose registers.
+ if (!(isInt || IsSoftFloatABI)) {
+ RegAddr = Builder.CreateConstInBoundsByteGEP(RegAddr,
+ CharUnits::fromQuantity(32));
+ }
+
+ // Get the address of the saved value by scaling the number of
+ // registers we've used by the number of
+ CharUnits RegSize = CharUnits::fromQuantity((isInt || IsSoftFloatABI) ? 4 : 8);
+ llvm::Value *RegOffset =
+ Builder.CreateMul(NumRegs, Builder.getInt8(RegSize.getQuantity()));
+ RegAddr = Address(Builder.CreateInBoundsGEP(CGF.Int8Ty,
+ RegAddr.getPointer(), RegOffset),
+ RegAddr.getAlignment().alignmentOfArrayElement(RegSize));
+ RegAddr = Builder.CreateElementBitCast(RegAddr, DirectTy);
+
+ // Increase the used-register count.
+ NumRegs =
+ Builder.CreateAdd(NumRegs,
+ Builder.getInt8((isI64 || (isF64 && IsSoftFloatABI)) ? 2 : 1));
+ Builder.CreateStore(NumRegs, NumRegsAddr);
+
+ CGF.EmitBranch(Cont);
}
- CGF.EmitBranch(Cont);
- CGF.EmitBlock(UsingOverflow);
+ // Case 2: consume space in the overflow area.
+ Address MemAddr = Address::invalid();
+ {
+ CGF.EmitBlock(UsingOverflow);
- // Increase the overflow area.
- llvm::Value *Result2 = Builder.CreateIntToPtr(OverflowAreaAsInt, PTy);
- OverflowAreaAsInt =
- Builder.CreateAdd(OverflowAreaAsInt, Builder.getInt32(isInt ? 4 : 8));
- Builder.CreateStore(Builder.CreateIntToPtr(OverflowAreaAsInt, CharPtr),
- OverflowAreaPtr);
- CGF.EmitBranch(Cont);
+ // Everything in the overflow area is rounded up to a size of at least 4.
+ CharUnits OverflowAreaAlign = CharUnits::fromQuantity(4);
+
+ CharUnits Size;
+ if (!isIndirect) {
+ auto TypeInfo = CGF.getContext().getTypeInfoInChars(Ty);
+ Size = TypeInfo.first.RoundUpToAlignment(OverflowAreaAlign);
+ } else {
+ Size = CGF.getPointerSize();
+ }
+
+ Address OverflowAreaAddr =
+ Builder.CreateStructGEP(VAList, 3, CharUnits::fromQuantity(4));
+ Address OverflowArea(Builder.CreateLoad(OverflowAreaAddr, "argp.cur"),
+ OverflowAreaAlign);
+ // Round up address of argument to alignment
+ CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty);
+ if (Align > OverflowAreaAlign) {
+ llvm::Value *Ptr = OverflowArea.getPointer();
+ OverflowArea = Address(emitRoundPointerUpToAlignment(CGF, Ptr, Align),
+ Align);
+ }
+
+ MemAddr = Builder.CreateElementBitCast(OverflowArea, DirectTy);
+
+ // Increase the overflow area.
+ OverflowArea = Builder.CreateConstInBoundsByteGEP(OverflowArea, Size);
+ Builder.CreateStore(OverflowArea.getPointer(), OverflowAreaAddr);
+ CGF.EmitBranch(Cont);
+ }
CGF.EmitBlock(Cont);
- llvm::PHINode *Result = CGF.Builder.CreatePHI(PTy, 2, "vaarg.addr");
- Result->addIncoming(Result1, UsingRegs);
- Result->addIncoming(Result2, UsingOverflow);
+ // Merge the cases with a phi.
+ Address Result = emitMergePHI(CGF, RegAddr, UsingRegs, MemAddr, UsingOverflow,
+ "vaarg.addr");
- if (Ty->isAggregateType()) {
- llvm::Value *AGGPtr = Builder.CreateBitCast(Result, CharPtrPtr, "aggrptr");
- return Builder.CreateLoad(AGGPtr, false, "aggr");
+ // Load the pointer if the argument was passed indirectly.
+ if (isIndirect) {
+ Result = Address(Builder.CreateLoad(Result, "aggr"),
+ getContext().getTypeAlignInChars(Ty));
}
return Result;
@@ -3371,7 +3695,7 @@
: DefaultABIInfo(CGT), Kind(Kind), HasQPX(HasQPX) {}
bool isPromotableTypeForABI(QualType Ty) const;
- bool isAlignedParamType(QualType Ty, bool &Align32) const;
+ CharUnits getParamTypeAlignment(QualType Ty) const;
ABIArgInfo classifyReturnType(QualType RetTy) const;
ABIArgInfo classifyArgumentType(QualType Ty) const;
@@ -3408,18 +3732,16 @@
}
}
- llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const override;
+ Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const override;
};
class PPC64_SVR4_TargetCodeGenInfo : public TargetCodeGenInfo {
- bool HasQPX;
public:
PPC64_SVR4_TargetCodeGenInfo(CodeGenTypes &CGT,
PPC64_SVR4_ABIInfo::ABIKind Kind, bool HasQPX)
- : TargetCodeGenInfo(new PPC64_SVR4_ABIInfo(CGT, Kind, HasQPX)),
- HasQPX(HasQPX) {}
+ : TargetCodeGenInfo(new PPC64_SVR4_ABIInfo(CGT, Kind, HasQPX)) {}
int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
// This is recovered from gcc output.
@@ -3428,15 +3750,6 @@
bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
llvm::Value *Address) const override;
-
- unsigned getOpenMPSimdDefaultAlignment(QualType QT) const override {
- if (HasQPX)
- if (const PointerType *PT = QT->getAs<PointerType>())
- if (PT->getPointeeType()->isSpecificBuiltinType(BuiltinType::Double))
- return 32; // Natural alignment for QPX doubles.
-
- return 16; // Natural alignment for Altivec and VSX vectors.
- }
};
class PPC64TargetCodeGenInfo : public DefaultTargetCodeGenInfo {
@@ -3450,10 +3763,6 @@
bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
llvm::Value *Address) const override;
-
- unsigned getOpenMPSimdDefaultAlignment(QualType) const override {
- return 16; // Natural alignment for Altivec vectors.
- }
};
}
@@ -3484,12 +3793,9 @@
return false;
}
-/// isAlignedParamType - Determine whether a type requires 16-byte
-/// alignment in the parameter area.
-bool
-PPC64_SVR4_ABIInfo::isAlignedParamType(QualType Ty, bool &Align32) const {
- Align32 = false;
-
+/// isAlignedParamType - Determine whether a type requires 16-byte or
+/// higher alignment in the parameter area. Always returns at least 8.
+CharUnits PPC64_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const {
// Complex types are passed just like their elements.
if (const ComplexType *CTy = Ty->getAs<ComplexType>())
Ty = CTy->getElementType();
@@ -3498,11 +3804,11 @@
// passed via reference, smaller types are not aligned).
if (IsQPXVectorTy(Ty)) {
if (getContext().getTypeSize(Ty) > 128)
- Align32 = true;
+ return CharUnits::fromQuantity(32);
- return true;
+ return CharUnits::fromQuantity(16);
} else if (Ty->isVectorType()) {
- return getContext().getTypeSize(Ty) == 128;
+ return CharUnits::fromQuantity(getContext().getTypeSize(Ty) == 128 ? 16 : 8);
}
// For single-element float/vector structs, we consider the whole type
@@ -3527,22 +3833,22 @@
// With special case aggregates, only vector base types need alignment.
if (AlignAsType && IsQPXVectorTy(AlignAsType)) {
if (getContext().getTypeSize(AlignAsType) > 128)
- Align32 = true;
+ return CharUnits::fromQuantity(32);
- return true;
+ return CharUnits::fromQuantity(16);
} else if (AlignAsType) {
- return AlignAsType->isVectorType();
+ return CharUnits::fromQuantity(AlignAsType->isVectorType() ? 16 : 8);
}
// Otherwise, we only need alignment for any aggregate type that
// has an alignment requirement of >= 16 bytes.
if (isAggregateTypeForABI(Ty) && getContext().getTypeAlign(Ty) >= 128) {
if (HasQPX && getContext().getTypeAlign(Ty) >= 256)
- Align32 = true;
- return true;
+ return CharUnits::fromQuantity(32);
+ return CharUnits::fromQuantity(16);
}
- return false;
+ return CharUnits::fromQuantity(8);
}
/// isHomogeneousAggregate - Return true if a type is an ELFv2 homogeneous
@@ -3675,7 +3981,7 @@
if (Ty->isVectorType() && !IsQPXVectorTy(Ty)) {
uint64_t Size = getContext().getTypeSize(Ty);
if (Size > 128)
- return ABIArgInfo::getIndirect(0, /*ByVal=*/false);
+ return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
else if (Size < 128) {
llvm::Type *CoerceTy = llvm::IntegerType::get(getVMContext(), Size);
return ABIArgInfo::getDirect(CoerceTy);
@@ -3684,12 +3990,10 @@
if (isAggregateTypeForABI(Ty)) {
if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
- return ABIArgInfo::getIndirect(0, RAA == CGCXXABI::RAA_DirectInMemory);
+ return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
- bool Align32;
- uint64_t ABIAlign = isAlignedParamType(Ty, Align32) ?
- (Align32 ? 32 : 16) : 8;
- uint64_t TyAlign = getContext().getTypeAlign(Ty) / 8;
+ uint64_t ABIAlign = getParamTypeAlignment(Ty).getQuantity();
+ uint64_t TyAlign = getContext().getTypeAlignInChars(Ty).getQuantity();
// ELFv2 homogeneous aggregates are passed as array types.
const Type *Base = nullptr;
@@ -3727,7 +4031,8 @@
}
// All other aggregates are passed ByVal.
- return ABIArgInfo::getIndirect(ABIAlign, /*ByVal=*/true,
+ return ABIArgInfo::getIndirect(CharUnits::fromQuantity(ABIAlign),
+ /*ByVal=*/true,
/*Realign=*/TyAlign > ABIAlign);
}
@@ -3748,7 +4053,7 @@
if (RetTy->isVectorType() && !IsQPXVectorTy(RetTy)) {
uint64_t Size = getContext().getTypeSize(RetTy);
if (Size > 128)
- return ABIArgInfo::getIndirect(0);
+ return getNaturalAlignIndirect(RetTy);
else if (Size < 128) {
llvm::Type *CoerceTy = llvm::IntegerType::get(getVMContext(), Size);
return ABIArgInfo::getDirect(CoerceTy);
@@ -3783,7 +4088,7 @@
}
// All other aggregates are returned indirectly.
- return ABIArgInfo::getIndirect(0);
+ return getNaturalAlignIndirect(RetTy);
}
return (isPromotableTypeForABI(RetTy) ?
@@ -3791,47 +4096,12 @@
}
// Based on ARMABIInfo::EmitVAArg, adjusted for 64-bit machine.
-llvm::Value *PPC64_SVR4_ABIInfo::EmitVAArg(llvm::Value *VAListAddr,
- QualType Ty,
- CodeGenFunction &CGF) const {
- llvm::Type *BP = CGF.Int8PtrTy;
- llvm::Type *BPP = CGF.Int8PtrPtrTy;
+Address PPC64_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const {
+ auto TypeInfo = getContext().getTypeInfoInChars(Ty);
+ TypeInfo.second = getParamTypeAlignment(Ty);
- CGBuilderTy &Builder = CGF.Builder;
- llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP, "ap");
- llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur");
-
- // Handle types that require 16-byte alignment in the parameter save area.
- bool Align32;
- if (isAlignedParamType(Ty, Align32)) {
- llvm::Value *AddrAsInt = Builder.CreatePtrToInt(Addr, CGF.Int64Ty);
- AddrAsInt = Builder.CreateAdd(AddrAsInt,
- Builder.getInt64(Align32 ? 31 : 15));
- AddrAsInt = Builder.CreateAnd(AddrAsInt,
- Builder.getInt64(Align32 ? -32 : -16));
- Addr = Builder.CreateIntToPtr(AddrAsInt, BP, "ap.align");
- }
-
- // Update the va_list pointer. The pointer should be bumped by the
- // size of the object. We can trust getTypeSize() except for a complex
- // type whose base type is smaller than a doubleword. For these, the
- // size of the object is 16 bytes; see below for further explanation.
- unsigned SizeInBytes = CGF.getContext().getTypeSize(Ty) / 8;
- QualType BaseTy;
- unsigned CplxBaseSize = 0;
-
- if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
- BaseTy = CTy->getElementType();
- CplxBaseSize = CGF.getContext().getTypeSize(BaseTy) / 8;
- if (CplxBaseSize < 8)
- SizeInBytes = 16;
- }
-
- unsigned Offset = llvm::RoundUpToAlignment(SizeInBytes, 8);
- llvm::Value *NextAddr =
- Builder.CreateGEP(Addr, llvm::ConstantInt::get(CGF.Int64Ty, Offset),
- "ap.next");
- Builder.CreateStore(NextAddr, VAListAddrAsBPP);
+ CharUnits SlotSize = CharUnits::fromQuantity(8);
// If we have a complex type and the base type is smaller than 8 bytes,
// the ABI calls for the real and imaginary parts to be right-adjusted
@@ -3839,44 +4109,40 @@
// pointer to a structure with the two parts packed tightly. So generate
// loads of the real and imaginary parts relative to the va_list pointer,
// and store them to a temporary structure.
- if (CplxBaseSize && CplxBaseSize < 8) {
- llvm::Value *RealAddr = Builder.CreatePtrToInt(Addr, CGF.Int64Ty);
- llvm::Value *ImagAddr = RealAddr;
- if (CGF.CGM.getDataLayout().isBigEndian()) {
- RealAddr =
- Builder.CreateAdd(RealAddr, Builder.getInt64(8 - CplxBaseSize));
- ImagAddr =
- Builder.CreateAdd(ImagAddr, Builder.getInt64(16 - CplxBaseSize));
- } else {
- ImagAddr = Builder.CreateAdd(ImagAddr, Builder.getInt64(8));
+ if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
+ CharUnits EltSize = TypeInfo.first / 2;
+ if (EltSize < SlotSize) {
+ Address Addr = emitVoidPtrDirectVAArg(CGF, VAListAddr, CGF.Int8Ty,
+ SlotSize * 2, SlotSize,
+ SlotSize, /*AllowHigher*/ true);
+
+ Address RealAddr = Addr;
+ Address ImagAddr = RealAddr;
+ if (CGF.CGM.getDataLayout().isBigEndian()) {
+ RealAddr = CGF.Builder.CreateConstInBoundsByteGEP(RealAddr,
+ SlotSize - EltSize);
+ ImagAddr = CGF.Builder.CreateConstInBoundsByteGEP(ImagAddr,
+ 2 * SlotSize - EltSize);
+ } else {
+ ImagAddr = CGF.Builder.CreateConstInBoundsByteGEP(RealAddr, SlotSize);
+ }
+
+ llvm::Type *EltTy = CGF.ConvertTypeForMem(CTy->getElementType());
+ RealAddr = CGF.Builder.CreateElementBitCast(RealAddr, EltTy);
+ ImagAddr = CGF.Builder.CreateElementBitCast(ImagAddr, EltTy);
+ llvm::Value *Real = CGF.Builder.CreateLoad(RealAddr, ".vareal");
+ llvm::Value *Imag = CGF.Builder.CreateLoad(ImagAddr, ".vaimag");
+
+ Address Temp = CGF.CreateMemTemp(Ty, "vacplx");
+ CGF.EmitStoreOfComplex({Real, Imag}, CGF.MakeAddrLValue(Temp, Ty),
+ /*init*/ true);
+ return Temp;
}
- llvm::Type *PBaseTy = llvm::PointerType::getUnqual(CGF.ConvertType(BaseTy));
- RealAddr = Builder.CreateIntToPtr(RealAddr, PBaseTy);
- ImagAddr = Builder.CreateIntToPtr(ImagAddr, PBaseTy);
- llvm::Value *Real = Builder.CreateLoad(RealAddr, false, ".vareal");
- llvm::Value *Imag = Builder.CreateLoad(ImagAddr, false, ".vaimag");
- llvm::AllocaInst *Ptr =
- CGF.CreateTempAlloca(CGT.ConvertTypeForMem(Ty), "vacplx");
- llvm::Value *RealPtr =
- Builder.CreateStructGEP(Ptr->getAllocatedType(), Ptr, 0, ".real");
- llvm::Value *ImagPtr =
- Builder.CreateStructGEP(Ptr->getAllocatedType(), Ptr, 1, ".imag");
- Builder.CreateStore(Real, RealPtr, false);
- Builder.CreateStore(Imag, ImagPtr, false);
- return Ptr;
}
- // If the argument is smaller than 8 bytes, it is right-adjusted in
- // its doubleword slot. Adjust the pointer to pick it up from the
- // correct offset.
- if (SizeInBytes < 8 && CGF.CGM.getDataLayout().isBigEndian()) {
- llvm::Value *AddrAsInt = Builder.CreatePtrToInt(Addr, CGF.Int64Ty);
- AddrAsInt = Builder.CreateAdd(AddrAsInt, Builder.getInt64(8 - SizeInBytes));
- Addr = Builder.CreateIntToPtr(AddrAsInt, BP);
- }
-
- llvm::Type *PTy = llvm::PointerType::getUnqual(CGF.ConvertType(Ty));
- return Builder.CreateBitCast(Addr, PTy);
+ // Otherwise, just use the general rule.
+ return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false,
+ TypeInfo, SlotSize, /*AllowHigher*/ true);
}
static bool
@@ -3974,14 +4240,14 @@
it.info = classifyArgumentType(it.type);
}
- llvm::Value *EmitDarwinVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const;
+ Address EmitDarwinVAArg(Address VAListAddr, QualType Ty,
+ CodeGenFunction &CGF) const;
- llvm::Value *EmitAAPCSVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const;
+ Address EmitAAPCSVAArg(Address VAListAddr, QualType Ty,
+ CodeGenFunction &CGF) const;
- llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const override {
+ Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const override {
return isDarwinPCS() ? EmitDarwinVAArg(VAListAddr, Ty, CGF)
: EmitAAPCSVAArg(VAListAddr, Ty, CGF);
}
@@ -4029,7 +4295,7 @@
llvm::VectorType::get(llvm::Type::getInt32Ty(getVMContext()), 4);
return ABIArgInfo::getDirect(ResType);
}
- return ABIArgInfo::getIndirect(0, /*ByVal=*/false);
+ return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
}
if (!isAggregateTypeForABI(Ty)) {
@@ -4045,8 +4311,8 @@
// Structures with either a non-trivial destructor or a non-trivial
// copy constructor are always indirect.
if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) {
- return ABIArgInfo::getIndirect(0, /*ByVal=*/RAA ==
- CGCXXABI::RAA_DirectInMemory);
+ return getNaturalAlignIndirect(Ty, /*ByVal=*/RAA ==
+ CGCXXABI::RAA_DirectInMemory);
}
// Empty records are always ignored on Darwin, but actually passed in C++ mode
@@ -4084,7 +4350,7 @@
return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Size));
}
- return ABIArgInfo::getIndirect(0, /*ByVal=*/false);
+ return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
}
ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy) const {
@@ -4093,7 +4359,7 @@
// Large vector types should be returned via memory.
if (RetTy->isVectorType() && getContext().getTypeSize(RetTy) > 128)
- return ABIArgInfo::getIndirect(0);
+ return getNaturalAlignIndirect(RetTy);
if (!isAggregateTypeForABI(RetTy)) {
// Treat an enum type as its underlying type.
@@ -4132,7 +4398,7 @@
return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Size));
}
- return ABIArgInfo::getIndirect(0);
+ return getNaturalAlignIndirect(RetTy);
}
/// isIllegalVectorType - check whether the vector type is legal for AArch64.
@@ -4170,7 +4436,7 @@
return Members <= 4;
}
-llvm::Value *AArch64ABIInfo::EmitAAPCSVAArg(llvm::Value *VAListAddr,
+Address AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr,
QualType Ty,
CodeGenFunction &CGF) const {
ABIArgInfo AI = classifyArgumentType(Ty);
@@ -4204,24 +4470,32 @@
llvm::BasicBlock *InRegBlock = CGF.createBasicBlock("vaarg.in_reg");
llvm::BasicBlock *OnStackBlock = CGF.createBasicBlock("vaarg.on_stack");
llvm::BasicBlock *ContBlock = CGF.createBasicBlock("vaarg.end");
- auto &Ctx = CGF.getContext();
- llvm::Value *reg_offs_p = nullptr, *reg_offs = nullptr;
+ auto TyInfo = getContext().getTypeInfoInChars(Ty);
+ CharUnits TyAlign = TyInfo.second;
+
+ Address reg_offs_p = Address::invalid();
+ llvm::Value *reg_offs = nullptr;
int reg_top_index;
- int RegSize = IsIndirect ? 8 : getContext().getTypeSize(Ty) / 8;
+ CharUnits reg_top_offset;
+ int RegSize = IsIndirect ? 8 : TyInfo.first.getQuantity();
if (!IsFPR) {
// 3 is the field number of __gr_offs
reg_offs_p =
- CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 3, "gr_offs_p");
+ CGF.Builder.CreateStructGEP(VAListAddr, 3, CharUnits::fromQuantity(24),
+ "gr_offs_p");
reg_offs = CGF.Builder.CreateLoad(reg_offs_p, "gr_offs");
reg_top_index = 1; // field number for __gr_top
+ reg_top_offset = CharUnits::fromQuantity(8);
RegSize = llvm::RoundUpToAlignment(RegSize, 8);
} else {
// 4 is the field number of __vr_offs.
reg_offs_p =
- CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 4, "vr_offs_p");
+ CGF.Builder.CreateStructGEP(VAListAddr, 4, CharUnits::fromQuantity(28),
+ "vr_offs_p");
reg_offs = CGF.Builder.CreateLoad(reg_offs_p, "vr_offs");
reg_top_index = 2; // field number for __vr_top
+ reg_top_offset = CharUnits::fromQuantity(16);
RegSize = 16 * NumRegs;
}
@@ -4246,8 +4520,8 @@
// Integer arguments may need to correct register alignment (for example a
// "struct { __int128 a; };" gets passed in x_2N, x_{2N+1}). In this case we
// align __gr_offs to calculate the potential address.
- if (!IsFPR && !IsIndirect && Ctx.getTypeAlign(Ty) > 64) {
- int Align = Ctx.getTypeAlign(Ty) / 8;
+ if (!IsFPR && !IsIndirect && TyAlign.getQuantity() > 8) {
+ int Align = TyAlign.getQuantity();
reg_offs = CGF.Builder.CreateAdd(
reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, Align - 1),
@@ -4258,6 +4532,9 @@
}
// Update the gr_offs/vr_offs pointer for next call to va_arg on this va_list.
+ // The fact that this is done unconditionally reflects the fact that
+ // allocating an argument to the stack also uses up all the remaining
+ // registers of the appropriate kind.
llvm::Value *NewOffset = nullptr;
NewOffset = CGF.Builder.CreateAdd(
reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, RegSize), "new_reg_offs");
@@ -4279,13 +4556,14 @@
// registers. First start the appropriate block:
CGF.EmitBlock(InRegBlock);
- llvm::Value *reg_top_p = nullptr, *reg_top = nullptr;
- reg_top_p = CGF.Builder.CreateStructGEP(nullptr, VAListAddr, reg_top_index,
- "reg_top_p");
+ llvm::Value *reg_top = nullptr;
+ Address reg_top_p = CGF.Builder.CreateStructGEP(VAListAddr, reg_top_index,
+ reg_top_offset, "reg_top_p");
reg_top = CGF.Builder.CreateLoad(reg_top_p, "reg_top");
- llvm::Value *BaseAddr = CGF.Builder.CreateGEP(reg_top, reg_offs);
- llvm::Value *RegAddr = nullptr;
- llvm::Type *MemTy = llvm::PointerType::getUnqual(CGF.ConvertTypeForMem(Ty));
+ Address BaseAddr(CGF.Builder.CreateInBoundsGEP(reg_top, reg_offs),
+ CharUnits::fromQuantity(IsFPR ? 16 : 8));
+ Address RegAddr = Address::invalid();
+ llvm::Type *MemTy = CGF.ConvertTypeForMem(Ty);
if (IsIndirect) {
// If it's been passed indirectly (actually a struct), whatever we find from
@@ -4302,43 +4580,45 @@
// qN+1, ...). We reload and store into a temporary local variable
// contiguously.
assert(!IsIndirect && "Homogeneous aggregates should be passed directly");
+ auto BaseTyInfo = getContext().getTypeInfoInChars(QualType(Base, 0));
llvm::Type *BaseTy = CGF.ConvertType(QualType(Base, 0));
llvm::Type *HFATy = llvm::ArrayType::get(BaseTy, NumMembers);
- llvm::AllocaInst *Tmp = CGF.CreateTempAlloca(HFATy);
- int Offset = 0;
+ Address Tmp = CGF.CreateTempAlloca(HFATy,
+ std::max(TyAlign, BaseTyInfo.second));
- if (CGF.CGM.getDataLayout().isBigEndian() && Ctx.getTypeSize(Base) < 128)
- Offset = 16 - Ctx.getTypeSize(Base) / 8;
+ // On big-endian platforms, the value will be right-aligned in its slot.
+ int Offset = 0;
+ if (CGF.CGM.getDataLayout().isBigEndian() &&
+ BaseTyInfo.first.getQuantity() < 16)
+ Offset = 16 - BaseTyInfo.first.getQuantity();
+
for (unsigned i = 0; i < NumMembers; ++i) {
- llvm::Value *BaseOffset =
- llvm::ConstantInt::get(CGF.Int32Ty, 16 * i + Offset);
- llvm::Value *LoadAddr = CGF.Builder.CreateGEP(BaseAddr, BaseOffset);
- LoadAddr = CGF.Builder.CreateBitCast(
- LoadAddr, llvm::PointerType::getUnqual(BaseTy));
- llvm::Value *StoreAddr =
- CGF.Builder.CreateStructGEP(Tmp->getAllocatedType(), Tmp, i);
+ CharUnits BaseOffset = CharUnits::fromQuantity(16 * i + Offset);
+ Address LoadAddr =
+ CGF.Builder.CreateConstInBoundsByteGEP(BaseAddr, BaseOffset);
+ LoadAddr = CGF.Builder.CreateElementBitCast(LoadAddr, BaseTy);
+
+ Address StoreAddr =
+ CGF.Builder.CreateConstArrayGEP(Tmp, i, BaseTyInfo.first);
llvm::Value *Elem = CGF.Builder.CreateLoad(LoadAddr);
CGF.Builder.CreateStore(Elem, StoreAddr);
}
- RegAddr = CGF.Builder.CreateBitCast(Tmp, MemTy);
+ RegAddr = CGF.Builder.CreateElementBitCast(Tmp, MemTy);
} else {
- // Otherwise the object is contiguous in memory
- unsigned BeAlign = reg_top_index == 2 ? 16 : 8;
- if (CGF.CGM.getDataLayout().isBigEndian() &&
+ // Otherwise the object is contiguous in memory.
+
+ // It might be right-aligned in its slot.
+ CharUnits SlotSize = BaseAddr.getAlignment();
+ if (CGF.CGM.getDataLayout().isBigEndian() && !IsIndirect &&
(IsHFA || !isAggregateTypeForABI(Ty)) &&
- Ctx.getTypeSize(Ty) < (BeAlign * 8)) {
- int Offset = BeAlign - Ctx.getTypeSize(Ty) / 8;
- BaseAddr = CGF.Builder.CreatePtrToInt(BaseAddr, CGF.Int64Ty);
-
- BaseAddr = CGF.Builder.CreateAdd(
- BaseAddr, llvm::ConstantInt::get(CGF.Int64Ty, Offset), "align_be");
-
- BaseAddr = CGF.Builder.CreateIntToPtr(BaseAddr, CGF.Int8PtrTy);
+ TyInfo.first < SlotSize) {
+ CharUnits Offset = SlotSize - TyInfo.first;
+ BaseAddr = CGF.Builder.CreateConstInBoundsByteGEP(BaseAddr, Offset);
}
- RegAddr = CGF.Builder.CreateBitCast(BaseAddr, MemTy);
+ RegAddr = CGF.Builder.CreateElementBitCast(BaseAddr, MemTy);
}
CGF.EmitBranch(ContBlock);
@@ -4348,55 +4628,51 @@
//=======================================
CGF.EmitBlock(OnStackBlock);
- llvm::Value *stack_p = nullptr, *OnStackAddr = nullptr;
- stack_p = CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 0, "stack_p");
- OnStackAddr = CGF.Builder.CreateLoad(stack_p, "stack");
+ Address stack_p = CGF.Builder.CreateStructGEP(VAListAddr, 0,
+ CharUnits::Zero(), "stack_p");
+ llvm::Value *OnStackPtr = CGF.Builder.CreateLoad(stack_p, "stack");
- // Again, stack arguments may need realigmnent. In this case both integer and
+ // Again, stack arguments may need realignment. In this case both integer and
// floating-point ones might be affected.
- if (!IsIndirect && Ctx.getTypeAlign(Ty) > 64) {
- int Align = Ctx.getTypeAlign(Ty) / 8;
+ if (!IsIndirect && TyAlign.getQuantity() > 8) {
+ int Align = TyAlign.getQuantity();
- OnStackAddr = CGF.Builder.CreatePtrToInt(OnStackAddr, CGF.Int64Ty);
+ OnStackPtr = CGF.Builder.CreatePtrToInt(OnStackPtr, CGF.Int64Ty);
- OnStackAddr = CGF.Builder.CreateAdd(
- OnStackAddr, llvm::ConstantInt::get(CGF.Int64Ty, Align - 1),
+ OnStackPtr = CGF.Builder.CreateAdd(
+ OnStackPtr, llvm::ConstantInt::get(CGF.Int64Ty, Align - 1),
"align_stack");
- OnStackAddr = CGF.Builder.CreateAnd(
- OnStackAddr, llvm::ConstantInt::get(CGF.Int64Ty, -Align),
+ OnStackPtr = CGF.Builder.CreateAnd(
+ OnStackPtr, llvm::ConstantInt::get(CGF.Int64Ty, -Align),
"align_stack");
- OnStackAddr = CGF.Builder.CreateIntToPtr(OnStackAddr, CGF.Int8PtrTy);
+ OnStackPtr = CGF.Builder.CreateIntToPtr(OnStackPtr, CGF.Int8PtrTy);
}
+ Address OnStackAddr(OnStackPtr,
+ std::max(CharUnits::fromQuantity(8), TyAlign));
- uint64_t StackSize;
+ // All stack slots are multiples of 8 bytes.
+ CharUnits StackSlotSize = CharUnits::fromQuantity(8);
+ CharUnits StackSize;
if (IsIndirect)
- StackSize = 8;
+ StackSize = StackSlotSize;
else
- StackSize = Ctx.getTypeSize(Ty) / 8;
+ StackSize = TyInfo.first.RoundUpToAlignment(StackSlotSize);
- // All stack slots are 8 bytes
- StackSize = llvm::RoundUpToAlignment(StackSize, 8);
-
- llvm::Value *StackSizeC = llvm::ConstantInt::get(CGF.Int32Ty, StackSize);
+ llvm::Value *StackSizeC = CGF.Builder.getSize(StackSize);
llvm::Value *NewStack =
- CGF.Builder.CreateGEP(OnStackAddr, StackSizeC, "new_stack");
+ CGF.Builder.CreateInBoundsGEP(OnStackPtr, StackSizeC, "new_stack");
// Write the new value of __stack for the next call to va_arg
CGF.Builder.CreateStore(NewStack, stack_p);
if (CGF.CGM.getDataLayout().isBigEndian() && !isAggregateTypeForABI(Ty) &&
- Ctx.getTypeSize(Ty) < 64) {
- int Offset = 8 - Ctx.getTypeSize(Ty) / 8;
- OnStackAddr = CGF.Builder.CreatePtrToInt(OnStackAddr, CGF.Int64Ty);
-
- OnStackAddr = CGF.Builder.CreateAdd(
- OnStackAddr, llvm::ConstantInt::get(CGF.Int64Ty, Offset), "align_be");
-
- OnStackAddr = CGF.Builder.CreateIntToPtr(OnStackAddr, CGF.Int8PtrTy);
+ TyInfo.first < StackSlotSize) {
+ CharUnits Offset = StackSlotSize - TyInfo.first;
+ OnStackAddr = CGF.Builder.CreateConstInBoundsByteGEP(OnStackAddr, Offset);
}
- OnStackAddr = CGF.Builder.CreateBitCast(OnStackAddr, MemTy);
+ OnStackAddr = CGF.Builder.CreateElementBitCast(OnStackAddr, MemTy);
CGF.EmitBranch(ContBlock);
@@ -4405,75 +4681,48 @@
//=======================================
CGF.EmitBlock(ContBlock);
- llvm::PHINode *ResAddr = CGF.Builder.CreatePHI(MemTy, 2, "vaarg.addr");
- ResAddr->addIncoming(RegAddr, InRegBlock);
- ResAddr->addIncoming(OnStackAddr, OnStackBlock);
+ Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock,
+ OnStackAddr, OnStackBlock, "vaargs.addr");
if (IsIndirect)
- return CGF.Builder.CreateLoad(ResAddr, "vaarg.addr");
+ return Address(CGF.Builder.CreateLoad(ResAddr, "vaarg.addr"),
+ TyInfo.second);
return ResAddr;
}
-llvm::Value *AArch64ABIInfo::EmitDarwinVAArg(llvm::Value *VAListAddr,
- QualType Ty,
- CodeGenFunction &CGF) const {
- // We do not support va_arg for aggregates or illegal vector types.
- // Lower VAArg here for these cases and use the LLVM va_arg instruction for
- // other cases.
+Address AArch64ABIInfo::EmitDarwinVAArg(Address VAListAddr, QualType Ty,
+ CodeGenFunction &CGF) const {
+ // The backend's lowering doesn't support va_arg for aggregates or
+ // illegal vector types. Lower VAArg here for these cases and use
+ // the LLVM va_arg instruction for everything else.
if (!isAggregateTypeForABI(Ty) && !isIllegalVectorType(Ty))
- return nullptr;
+ return Address::invalid();
- uint64_t Size = CGF.getContext().getTypeSize(Ty) / 8;
- uint64_t Align = CGF.getContext().getTypeAlign(Ty) / 8;
+ CharUnits SlotSize = CharUnits::fromQuantity(8);
- const Type *Base = nullptr;
- uint64_t Members = 0;
- bool isHA = isHomogeneousAggregate(Ty, Base, Members);
-
- bool isIndirect = false;
- // Arguments bigger than 16 bytes which aren't homogeneous aggregates should
- // be passed indirectly.
- if (Size > 16 && !isHA) {
- isIndirect = true;
- Size = 8;
- Align = 8;
- }
-
- llvm::Type *BP = llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
- llvm::Type *BPP = llvm::PointerType::getUnqual(BP);
-
- CGBuilderTy &Builder = CGF.Builder;
- llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP, "ap");
- llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur");
-
+ // Empty records are ignored for parameter passing purposes.
if (isEmptyRecord(getContext(), Ty, true)) {
- // These are ignored for parameter passing purposes.
- llvm::Type *PTy = llvm::PointerType::getUnqual(CGF.ConvertType(Ty));
- return Builder.CreateBitCast(Addr, PTy);
+ Address Addr(CGF.Builder.CreateLoad(VAListAddr, "ap.cur"), SlotSize);
+ Addr = CGF.Builder.CreateElementBitCast(Addr, CGF.ConvertTypeForMem(Ty));
+ return Addr;
}
- const uint64_t MinABIAlign = 8;
- if (Align > MinABIAlign) {
- llvm::Value *Offset = llvm::ConstantInt::get(CGF.Int32Ty, Align - 1);
- Addr = Builder.CreateGEP(Addr, Offset);
- llvm::Value *AsInt = Builder.CreatePtrToInt(Addr, CGF.Int64Ty);
- llvm::Value *Mask = llvm::ConstantInt::get(CGF.Int64Ty, ~(Align - 1));
- llvm::Value *Aligned = Builder.CreateAnd(AsInt, Mask);
- Addr = Builder.CreateIntToPtr(Aligned, BP, "ap.align");
+ // The size of the actual thing passed, which might end up just
+ // being a pointer for indirect types.
+ auto TyInfo = getContext().getTypeInfoInChars(Ty);
+
+ // Arguments bigger than 16 bytes which aren't homogeneous
+ // aggregates should be passed indirectly.
+ bool IsIndirect = false;
+ if (TyInfo.first.getQuantity() > 16) {
+ const Type *Base = nullptr;
+ uint64_t Members = 0;
+ IsIndirect = !isHomogeneousAggregate(Ty, Base, Members);
}
- uint64_t Offset = llvm::RoundUpToAlignment(Size, MinABIAlign);
- llvm::Value *NextAddr = Builder.CreateGEP(
- Addr, llvm::ConstantInt::get(CGF.Int32Ty, Offset), "ap.next");
- Builder.CreateStore(NextAddr, VAListAddrAsBPP);
-
- if (isIndirect)
- Addr = Builder.CreateLoad(Builder.CreateBitCast(Addr, BPP));
- llvm::Type *PTy = llvm::PointerType::getUnqual(CGF.ConvertType(Ty));
- llvm::Value *AddrTyped = Builder.CreateBitCast(Addr, PTy);
-
- return AddrTyped;
+ return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
+ TyInfo, SlotSize, /*AllowHigherAlign*/ true);
}
//===----------------------------------------------------------------------===//
@@ -4487,7 +4736,8 @@
enum ABIKind {
APCS = 0,
AAPCS = 1,
- AAPCS_VFP
+ AAPCS_VFP = 2,
+ AAPCS16_VFP = 3,
};
private:
@@ -4521,6 +4771,11 @@
}
}
+ bool isAndroid() const {
+ return (getTarget().getTriple().getEnvironment() ==
+ llvm::Triple::Android);
+ }
+
ABIKind getABIKind() const { return Kind; }
private:
@@ -4534,8 +4789,8 @@
void computeInfo(CGFunctionInfo &FI) const override;
- llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const override;
+ Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const override;
llvm::CallingConv::ID getLLVMDefaultCC() const;
llvm::CallingConv::ID getABIDefaultCC() const;
@@ -4575,7 +4830,7 @@
void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
CodeGen::CodeGenModule &CGM) const override {
- const FunctionDecl *FD = dyn_cast<FunctionDecl>(D);
+ const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
if (!FD)
return;
@@ -4597,7 +4852,8 @@
Fn->addFnAttr("interrupt", Kind);
- if (cast<ARMABIInfo>(getABIInfo()).getABIKind() == ARMABIInfo::APCS)
+ ARMABIInfo::ABIKind ABI = cast<ARMABIInfo>(getABIInfo()).getABIKind();
+ if (ABI == ARMABIInfo::APCS)
return;
// AAPCS guarantees that sp will be 8-byte aligned on any public interface,
@@ -4663,7 +4919,7 @@
/// Return the default calling convention that LLVM will use.
llvm::CallingConv::ID ARMABIInfo::getLLVMDefaultCC() const {
// The default calling convention that LLVM will infer.
- if (isEABIHF())
+ if (isEABIHF() || getTarget().getTriple().isWatchOS())
return llvm::CallingConv::ARM_AAPCS_VFP;
else if (isEABI())
return llvm::CallingConv::ARM_AAPCS;
@@ -4678,6 +4934,7 @@
case APCS: return llvm::CallingConv::ARM_APCS;
case AAPCS: return llvm::CallingConv::ARM_AAPCS;
case AAPCS_VFP: return llvm::CallingConv::ARM_AAPCS_VFP;
+ case AAPCS16_VFP: return llvm::CallingConv::ARM_AAPCS_VFP;
}
llvm_unreachable("bad ABI kind");
}
@@ -4691,8 +4948,20 @@
if (abiCC != getLLVMDefaultCC())
RuntimeCC = abiCC;
- BuiltinCC = (getABIKind() == APCS ?
- llvm::CallingConv::ARM_APCS : llvm::CallingConv::ARM_AAPCS);
+ // AAPCS apparently requires runtime support functions to be soft-float, but
+ // that's almost certainly for historic reasons (Thumb1 not supporting VFP
+ // most likely). It's more convenient for AAPCS16_VFP to be hard-float.
+ switch (getABIKind()) {
+ case APCS:
+ case AAPCS16_VFP:
+ if (abiCC != getLLVMDefaultCC())
+ BuiltinCC = abiCC;
+ break;
+ case AAPCS:
+ case AAPCS_VFP:
+ BuiltinCC = llvm::CallingConv::ARM_AAPCS;
+ break;
+ }
}
ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty,
@@ -4726,7 +4995,17 @@
llvm::Type::getInt32Ty(getVMContext()), 4);
return ABIArgInfo::getDirect(ResType);
}
- return ABIArgInfo::getIndirect(0, /*ByVal=*/false);
+ return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
+ }
+
+ // __fp16 gets passed as if it were an int or float, but with the top 16 bits
+ // unspecified. This is not done for OpenCL as it handles the half type
+ // natively, and does not need to interwork with AAPCS code.
+ if (Ty->isHalfType() && !getContext().getLangOpts().HalfArgsAndReturns) {
+ llvm::Type *ResType = IsEffectivelyAAPCS_VFP ?
+ llvm::Type::getFloatTy(getVMContext()) :
+ llvm::Type::getInt32Ty(getVMContext());
+ return ABIArgInfo::getDirect(ResType);
}
if (!isAggregateTypeForABI(Ty)) {
@@ -4740,7 +5019,7 @@
}
if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) {
- return ABIArgInfo::getIndirect(0, RAA == CGCXXABI::RAA_DirectInMemory);
+ return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
}
// Ignore empty records.
@@ -4757,6 +5036,27 @@
// Base can be a floating-point or a vector.
return ABIArgInfo::getDirect(nullptr, 0, nullptr, false);
}
+ } else if (getABIKind() == ARMABIInfo::AAPCS16_VFP) {
+ // WatchOS does have homogeneous aggregates. Note that we intentionally use
+ // this convention even for a variadic function: the backend will use GPRs
+ // if needed.
+ const Type *Base = nullptr;
+ uint64_t Members = 0;
+ if (isHomogeneousAggregate(Ty, Base, Members)) {
+ assert(Base && Members <= 4 && "unexpected homogeneous aggregate");
+ llvm::Type *Ty =
+ llvm::ArrayType::get(CGT.ConvertType(QualType(Base, 0)), Members);
+ return ABIArgInfo::getDirect(Ty, 0, nullptr, false);
+ }
+ }
+
+ if (getABIKind() == ARMABIInfo::AAPCS16_VFP &&
+ getContext().getTypeSizeInChars(Ty) > CharUnits::fromQuantity(16)) {
+ // WatchOS is adopting the 64-bit AAPCS rule on composite types: if they're
+ // bigger than 128-bits, they get placed in space allocated by the caller,
+ // and a pointer is passed.
+ return ABIArgInfo::getIndirect(
+ CharUnits::fromQuantity(getContext().getTypeAlign(Ty) / 8), false);
}
// Support byval for ARM.
@@ -4770,8 +5070,10 @@
ABIAlign = std::min(std::max(TyAlign, (uint64_t)4), (uint64_t)8);
if (getContext().getTypeSizeInChars(Ty) > CharUnits::fromQuantity(64)) {
- return ABIArgInfo::getIndirect(ABIAlign, /*ByVal=*/true,
- /*Realign=*/TyAlign > ABIAlign);
+ assert(getABIKind() != ARMABIInfo::AAPCS16_VFP && "unexpected byval");
+ return ABIArgInfo::getIndirect(CharUnits::fromQuantity(ABIAlign),
+ /*ByVal=*/true,
+ /*Realign=*/TyAlign > ABIAlign);
}
if (getContext().getLangOpts().Renderscript) {
@@ -4881,14 +5183,25 @@
ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy,
bool isVariadic) const {
- bool IsEffectivelyAAPCS_VFP = getABIKind() == AAPCS_VFP && !isVariadic;
+ bool IsEffectivelyAAPCS_VFP =
+ (getABIKind() == AAPCS_VFP || getABIKind() == AAPCS16_VFP) && !isVariadic;
if (RetTy->isVoidType())
return ABIArgInfo::getIgnore();
// Large vector types should be returned via memory.
if (RetTy->isVectorType() && getContext().getTypeSize(RetTy) > 128) {
- return ABIArgInfo::getIndirect(0);
+ return getNaturalAlignIndirect(RetTy);
+ }
+
+ // __fp16 gets returned as if it were an int or float, but with the top 16
+ // bits unspecified. This is not done for OpenCL as it handles the half type
+ // natively, and does not need to interwork with AAPCS code.
+ if (RetTy->isHalfType() && !getContext().getLangOpts().HalfArgsAndReturns) {
+ llvm::Type *ResType = IsEffectivelyAAPCS_VFP ?
+ llvm::Type::getFloatTy(getVMContext()) :
+ llvm::Type::getInt32Ty(getVMContext());
+ return ABIArgInfo::getDirect(ResType);
}
if (!isAggregateTypeForABI(RetTy)) {
@@ -4925,7 +5238,7 @@
}
// Otherwise return in memory.
- return ABIArgInfo::getIndirect(0);
+ return getNaturalAlignIndirect(RetTy);
}
// Otherwise this is an AAPCS variant.
@@ -4936,7 +5249,7 @@
// Check for homogeneous aggregates with AAPCS-VFP.
if (IsEffectivelyAAPCS_VFP) {
const Type *Base = nullptr;
- uint64_t Members;
+ uint64_t Members = 0;
if (isHomogeneousAggregate(RetTy, Base, Members)) {
assert(Base && "Base class should be set for homogeneous aggregate");
// Homogeneous Aggregates are returned directly.
@@ -4961,19 +5274,39 @@
if (Size <= 16)
return ABIArgInfo::getDirect(llvm::Type::getInt16Ty(getVMContext()));
return ABIArgInfo::getDirect(llvm::Type::getInt32Ty(getVMContext()));
+ } else if (Size <= 128 && getABIKind() == AAPCS16_VFP) {
+ llvm::Type *Int32Ty = llvm::Type::getInt32Ty(getVMContext());
+ llvm::Type *CoerceTy =
+ llvm::ArrayType::get(Int32Ty, llvm::RoundUpToAlignment(Size, 32) / 32);
+ return ABIArgInfo::getDirect(CoerceTy);
}
- return ABIArgInfo::getIndirect(0);
+ return getNaturalAlignIndirect(RetTy);
}
/// isIllegalVector - check whether Ty is an illegal vector type.
bool ARMABIInfo::isIllegalVectorType(QualType Ty) const {
- if (const VectorType *VT = Ty->getAs<VectorType>()) {
- // Check whether VT is legal.
- unsigned NumElements = VT->getNumElements();
- // NumElements should be power of 2.
- if (((NumElements & (NumElements - 1)) != 0) && NumElements != 3)
- return true;
+ if (const VectorType *VT = Ty->getAs<VectorType> ()) {
+ if (isAndroid()) {
+ // Android shipped using Clang 3.1, which supported a slightly different
+ // vector ABI. The primary differences were that 3-element vector types
+ // were legal, and so were sub 32-bit vectors (i.e. <2 x i8>). This path
+ // accepts that legacy behavior for Android only.
+ // Check whether VT is legal.
+ unsigned NumElements = VT->getNumElements();
+ // NumElements should be power of 2 or equal to 3.
+ if (!llvm::isPowerOf2_32(NumElements) && NumElements != 3)
+ return true;
+ } else {
+ // Check whether VT is legal.
+ unsigned NumElements = VT->getNumElements();
+ uint64_t Size = getContext().getTypeSize(VT);
+ // NumElements should be power of 2.
+ if (!llvm::isPowerOf2_32(NumElements))
+ return true;
+ // Size should be greater than 32 bits.
+ return Size <= 32;
+ }
}
return false;
}
@@ -4999,80 +5332,53 @@
return Members <= 4;
}
-llvm::Value *ARMABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const {
- llvm::Type *BP = CGF.Int8PtrTy;
- llvm::Type *BPP = CGF.Int8PtrPtrTy;
+Address ARMABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const {
+ CharUnits SlotSize = CharUnits::fromQuantity(4);
- CGBuilderTy &Builder = CGF.Builder;
- llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP, "ap");
- llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur");
-
+ // Empty records are ignored for parameter passing purposes.
if (isEmptyRecord(getContext(), Ty, true)) {
- // These are ignored for parameter passing purposes.
- llvm::Type *PTy = llvm::PointerType::getUnqual(CGF.ConvertType(Ty));
- return Builder.CreateBitCast(Addr, PTy);
+ Address Addr(CGF.Builder.CreateLoad(VAListAddr), SlotSize);
+ Addr = CGF.Builder.CreateElementBitCast(Addr, CGF.ConvertTypeForMem(Ty));
+ return Addr;
}
- uint64_t Size = CGF.getContext().getTypeSize(Ty) / 8;
- uint64_t TyAlign = CGF.getContext().getTypeAlign(Ty) / 8;
- bool IsIndirect = false;
+ auto TyInfo = getContext().getTypeInfoInChars(Ty);
+ CharUnits TyAlignForABI = TyInfo.second;
+ // Use indirect if size of the illegal vector is bigger than 16 bytes.
+ bool IsIndirect = false;
+ const Type *Base = nullptr;
+ uint64_t Members = 0;
+ if (TyInfo.first > CharUnits::fromQuantity(16) && isIllegalVectorType(Ty)) {
+ IsIndirect = true;
+
+ // ARMv7k passes structs bigger than 16 bytes indirectly, in space
+ // allocated by the caller.
+ } else if (TyInfo.first > CharUnits::fromQuantity(16) &&
+ getABIKind() == ARMABIInfo::AAPCS16_VFP &&
+ !isHomogeneousAggregate(Ty, Base, Members)) {
+ IsIndirect = true;
+
+ // Otherwise, bound the type's ABI alignment.
// The ABI alignment for 64-bit or 128-bit vectors is 8 for AAPCS and 4 for
// APCS. For AAPCS, the ABI alignment is at least 4-byte and at most 8-byte.
- if (getABIKind() == ARMABIInfo::AAPCS_VFP ||
- getABIKind() == ARMABIInfo::AAPCS)
- TyAlign = std::min(std::max(TyAlign, (uint64_t)4), (uint64_t)8);
- else
- TyAlign = 4;
- // Use indirect if size of the illegal vector is bigger than 32 bytes.
- if (isIllegalVectorType(Ty) && Size > 32) {
- IsIndirect = true;
- Size = 4;
- TyAlign = 4;
+ // Our callers should be prepared to handle an under-aligned address.
+ } else if (getABIKind() == ARMABIInfo::AAPCS_VFP ||
+ getABIKind() == ARMABIInfo::AAPCS) {
+ TyAlignForABI = std::max(TyAlignForABI, CharUnits::fromQuantity(4));
+ TyAlignForABI = std::min(TyAlignForABI, CharUnits::fromQuantity(8));
+ } else if (getABIKind() == ARMABIInfo::AAPCS16_VFP) {
+ // ARMv7k allows type alignment up to 16 bytes.
+ TyAlignForABI = std::max(TyAlignForABI, CharUnits::fromQuantity(4));
+ TyAlignForABI = std::min(TyAlignForABI, CharUnits::fromQuantity(16));
+ } else {
+ TyAlignForABI = CharUnits::fromQuantity(4);
}
+ TyInfo.second = TyAlignForABI;
- // Handle address alignment for ABI alignment > 4 bytes.
- if (TyAlign > 4) {
- assert((TyAlign & (TyAlign - 1)) == 0 &&
- "Alignment is not power of 2!");
- llvm::Value *AddrAsInt = Builder.CreatePtrToInt(Addr, CGF.Int32Ty);
- AddrAsInt = Builder.CreateAdd(AddrAsInt, Builder.getInt32(TyAlign - 1));
- AddrAsInt = Builder.CreateAnd(AddrAsInt, Builder.getInt32(~(TyAlign - 1)));
- Addr = Builder.CreateIntToPtr(AddrAsInt, BP, "ap.align");
- }
-
- uint64_t Offset =
- llvm::RoundUpToAlignment(Size, 4);
- llvm::Value *NextAddr =
- Builder.CreateGEP(Addr, llvm::ConstantInt::get(CGF.Int32Ty, Offset),
- "ap.next");
- Builder.CreateStore(NextAddr, VAListAddrAsBPP);
-
- if (IsIndirect)
- Addr = Builder.CreateLoad(Builder.CreateBitCast(Addr, BPP));
- else if (TyAlign < CGF.getContext().getTypeAlign(Ty) / 8) {
- // We can't directly cast ap.cur to pointer to a vector type, since ap.cur
- // may not be correctly aligned for the vector type. We create an aligned
- // temporary space and copy the content over from ap.cur to the temporary
- // space. This is necessary if the natural alignment of the type is greater
- // than the ABI alignment.
- llvm::Type *I8PtrTy = Builder.getInt8PtrTy();
- CharUnits CharSize = getContext().getTypeSizeInChars(Ty);
- llvm::Value *AlignedTemp = CGF.CreateTempAlloca(CGF.ConvertType(Ty),
- "var.align");
- llvm::Value *Dst = Builder.CreateBitCast(AlignedTemp, I8PtrTy);
- llvm::Value *Src = Builder.CreateBitCast(Addr, I8PtrTy);
- Builder.CreateMemCpy(Dst, Src,
- llvm::ConstantInt::get(CGF.IntPtrTy, CharSize.getQuantity()),
- TyAlign, false);
- Addr = AlignedTemp; //The content is in aligned location.
- }
- llvm::Type *PTy =
- llvm::PointerType::getUnqual(CGF.ConvertType(Ty));
- llvm::Value *AddrTyped = Builder.CreateBitCast(Addr, PTy);
-
- return AddrTyped;
+ return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect, TyInfo,
+ SlotSize, /*AllowHigherAlign*/ true);
}
//===----------------------------------------------------------------------===//
@@ -5089,8 +5395,8 @@
ABIArgInfo classifyArgumentType(QualType Ty) const;
void computeInfo(CGFunctionInfo &FI) const override;
- llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CFG) const override;
+ Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const override;
};
class NVPTXTargetCodeGenInfo : public TargetCodeGenInfo {
@@ -5129,7 +5435,7 @@
// Return aggregates type as indirect by value
if (isAggregateTypeForABI(Ty))
- return ABIArgInfo::getIndirect(0, /* byval */ true);
+ return getNaturalAlignIndirect(Ty, /* byval */ true);
return (Ty->isPromotableIntegerType() ?
ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
@@ -5148,15 +5454,15 @@
FI.setEffectiveCallingConvention(getRuntimeCC());
}
-llvm::Value *NVPTXABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CFG) const {
+Address NVPTXABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const {
llvm_unreachable("NVPTX does not support varargs");
}
void NVPTXTargetCodeGenInfo::
setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
CodeGen::CodeGenModule &M) const{
- const FunctionDecl *FD = dyn_cast<FunctionDecl>(D);
+ const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
if (!FD) return;
llvm::Function *F = cast<llvm::Function>(GV);
@@ -5250,8 +5556,8 @@
I.info = classifyArgumentType(I.type);
}
- llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const override;
+ Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const override;
};
class SystemZTargetCodeGenInfo : public TargetCodeGenInfo {
@@ -5352,8 +5658,8 @@
return Ty;
}
-llvm::Value *SystemZABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const {
+Address SystemZABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const {
// Assume that va_list type is correct; should be pointer to LLVM type:
// struct {
// i64 __gpr;
@@ -5365,59 +5671,69 @@
// Every non-vector argument occupies 8 bytes and is passed by preference
// in either GPRs or FPRs. Vector arguments occupy 8 or 16 bytes and are
// always passed on the stack.
- Ty = CGF.getContext().getCanonicalType(Ty);
+ Ty = getContext().getCanonicalType(Ty);
+ auto TyInfo = getContext().getTypeInfoInChars(Ty);
llvm::Type *ArgTy = CGF.ConvertTypeForMem(Ty);
- llvm::Type *APTy = llvm::PointerType::getUnqual(ArgTy);
+ llvm::Type *DirectTy = ArgTy;
ABIArgInfo AI = classifyArgumentType(Ty);
bool IsIndirect = AI.isIndirect();
bool InFPRs = false;
bool IsVector = false;
- unsigned UnpaddedBitSize;
+ CharUnits UnpaddedSize;
+ CharUnits DirectAlign;
if (IsIndirect) {
- APTy = llvm::PointerType::getUnqual(APTy);
- UnpaddedBitSize = 64;
+ DirectTy = llvm::PointerType::getUnqual(DirectTy);
+ UnpaddedSize = DirectAlign = CharUnits::fromQuantity(8);
} else {
if (AI.getCoerceToType())
ArgTy = AI.getCoerceToType();
InFPRs = ArgTy->isFloatTy() || ArgTy->isDoubleTy();
IsVector = ArgTy->isVectorTy();
- UnpaddedBitSize = getContext().getTypeSize(Ty);
+ UnpaddedSize = TyInfo.first;
+ DirectAlign = TyInfo.second;
}
- unsigned PaddedBitSize = (IsVector && UnpaddedBitSize > 64) ? 128 : 64;
- assert((UnpaddedBitSize <= PaddedBitSize) && "Invalid argument size.");
+ CharUnits PaddedSize = CharUnits::fromQuantity(8);
+ if (IsVector && UnpaddedSize > PaddedSize)
+ PaddedSize = CharUnits::fromQuantity(16);
+ assert((UnpaddedSize <= PaddedSize) && "Invalid argument size.");
- unsigned PaddedSize = PaddedBitSize / 8;
- unsigned Padding = (PaddedBitSize - UnpaddedBitSize) / 8;
+ CharUnits Padding = (PaddedSize - UnpaddedSize);
llvm::Type *IndexTy = CGF.Int64Ty;
- llvm::Value *PaddedSizeV = llvm::ConstantInt::get(IndexTy, PaddedSize);
+ llvm::Value *PaddedSizeV =
+ llvm::ConstantInt::get(IndexTy, PaddedSize.getQuantity());
if (IsVector) {
// Work out the address of a vector argument on the stack.
// Vector arguments are always passed in the high bits of a
// single (8 byte) or double (16 byte) stack slot.
- llvm::Value *OverflowArgAreaPtr =
- CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 2,
+ Address OverflowArgAreaPtr =
+ CGF.Builder.CreateStructGEP(VAListAddr, 2, CharUnits::fromQuantity(16),
"overflow_arg_area_ptr");
- llvm::Value *OverflowArgArea =
- CGF.Builder.CreateLoad(OverflowArgAreaPtr, "overflow_arg_area");
- llvm::Value *MemAddr =
- CGF.Builder.CreateBitCast(OverflowArgArea, APTy, "mem_addr");
+ Address OverflowArgArea =
+ Address(CGF.Builder.CreateLoad(OverflowArgAreaPtr, "overflow_arg_area"),
+ TyInfo.second);
+ Address MemAddr =
+ CGF.Builder.CreateElementBitCast(OverflowArgArea, DirectTy, "mem_addr");
// Update overflow_arg_area_ptr pointer
llvm::Value *NewOverflowArgArea =
- CGF.Builder.CreateGEP(OverflowArgArea, PaddedSizeV, "overflow_arg_area");
+ CGF.Builder.CreateGEP(OverflowArgArea.getPointer(), PaddedSizeV,
+ "overflow_arg_area");
CGF.Builder.CreateStore(NewOverflowArgArea, OverflowArgAreaPtr);
return MemAddr;
}
- unsigned MaxRegs, RegCountField, RegSaveIndex, RegPadding;
+ assert(PaddedSize.getQuantity() == 8);
+
+ unsigned MaxRegs, RegCountField, RegSaveIndex;
+ CharUnits RegPadding;
if (InFPRs) {
MaxRegs = 4; // Maximum of 4 FPR arguments
RegCountField = 1; // __fpr
RegSaveIndex = 16; // save offset for f0
- RegPadding = 0; // floats are passed in the high bits of an FPR
+ RegPadding = CharUnits(); // floats are passed in the high bits of an FPR
} else {
MaxRegs = 5; // Maximum of 5 GPR arguments
RegCountField = 0; // __gpr
@@ -5425,8 +5741,9 @@
RegPadding = Padding; // values are passed in the low bits of a GPR
}
- llvm::Value *RegCountPtr = CGF.Builder.CreateStructGEP(
- nullptr, VAListAddr, RegCountField, "reg_count_ptr");
+ Address RegCountPtr = CGF.Builder.CreateStructGEP(
+ VAListAddr, RegCountField, RegCountField * CharUnits::fromQuantity(8),
+ "reg_count_ptr");
llvm::Value *RegCount = CGF.Builder.CreateLoad(RegCountPtr, "reg_count");
llvm::Value *MaxRegsV = llvm::ConstantInt::get(IndexTy, MaxRegs);
llvm::Value *InRegs = CGF.Builder.CreateICmpULT(RegCount, MaxRegsV,
@@ -5444,17 +5761,20 @@
llvm::Value *ScaledRegCount =
CGF.Builder.CreateMul(RegCount, PaddedSizeV, "scaled_reg_count");
llvm::Value *RegBase =
- llvm::ConstantInt::get(IndexTy, RegSaveIndex * PaddedSize + RegPadding);
+ llvm::ConstantInt::get(IndexTy, RegSaveIndex * PaddedSize.getQuantity()
+ + RegPadding.getQuantity());
llvm::Value *RegOffset =
CGF.Builder.CreateAdd(ScaledRegCount, RegBase, "reg_offset");
- llvm::Value *RegSaveAreaPtr =
- CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 3, "reg_save_area_ptr");
+ Address RegSaveAreaPtr =
+ CGF.Builder.CreateStructGEP(VAListAddr, 3, CharUnits::fromQuantity(24),
+ "reg_save_area_ptr");
llvm::Value *RegSaveArea =
CGF.Builder.CreateLoad(RegSaveAreaPtr, "reg_save_area");
- llvm::Value *RawRegAddr =
- CGF.Builder.CreateGEP(RegSaveArea, RegOffset, "raw_reg_addr");
- llvm::Value *RegAddr =
- CGF.Builder.CreateBitCast(RawRegAddr, APTy, "reg_addr");
+ Address RawRegAddr(CGF.Builder.CreateGEP(RegSaveArea, RegOffset,
+ "raw_reg_addr"),
+ PaddedSize);
+ Address RegAddr =
+ CGF.Builder.CreateElementBitCast(RawRegAddr, DirectTy, "reg_addr");
// Update the register count
llvm::Value *One = llvm::ConstantInt::get(IndexTy, 1);
@@ -5467,30 +5787,31 @@
CGF.EmitBlock(InMemBlock);
// Work out the address of a stack argument.
- llvm::Value *OverflowArgAreaPtr = CGF.Builder.CreateStructGEP(
- nullptr, VAListAddr, 2, "overflow_arg_area_ptr");
- llvm::Value *OverflowArgArea =
- CGF.Builder.CreateLoad(OverflowArgAreaPtr, "overflow_arg_area");
- llvm::Value *PaddingV = llvm::ConstantInt::get(IndexTy, Padding);
- llvm::Value *RawMemAddr =
- CGF.Builder.CreateGEP(OverflowArgArea, PaddingV, "raw_mem_addr");
- llvm::Value *MemAddr =
- CGF.Builder.CreateBitCast(RawMemAddr, APTy, "mem_addr");
+ Address OverflowArgAreaPtr = CGF.Builder.CreateStructGEP(
+ VAListAddr, 2, CharUnits::fromQuantity(16), "overflow_arg_area_ptr");
+ Address OverflowArgArea =
+ Address(CGF.Builder.CreateLoad(OverflowArgAreaPtr, "overflow_arg_area"),
+ PaddedSize);
+ Address RawMemAddr =
+ CGF.Builder.CreateConstByteGEP(OverflowArgArea, Padding, "raw_mem_addr");
+ Address MemAddr =
+ CGF.Builder.CreateElementBitCast(RawMemAddr, DirectTy, "mem_addr");
// Update overflow_arg_area_ptr pointer
llvm::Value *NewOverflowArgArea =
- CGF.Builder.CreateGEP(OverflowArgArea, PaddedSizeV, "overflow_arg_area");
+ CGF.Builder.CreateGEP(OverflowArgArea.getPointer(), PaddedSizeV,
+ "overflow_arg_area");
CGF.Builder.CreateStore(NewOverflowArgArea, OverflowArgAreaPtr);
CGF.EmitBranch(ContBlock);
// Return the appropriate result.
CGF.EmitBlock(ContBlock);
- llvm::PHINode *ResAddr = CGF.Builder.CreatePHI(APTy, 2, "va_arg.addr");
- ResAddr->addIncoming(RegAddr, InRegBlock);
- ResAddr->addIncoming(MemAddr, InMemBlock);
+ Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock,
+ MemAddr, InMemBlock, "va_arg.addr");
if (IsIndirect)
- return CGF.Builder.CreateLoad(ResAddr, "indirect_arg");
+ ResAddr = Address(CGF.Builder.CreateLoad(ResAddr, "indirect_arg"),
+ TyInfo.second);
return ResAddr;
}
@@ -5501,7 +5822,7 @@
if (isVectorArgumentType(RetTy))
return ABIArgInfo::getDirect();
if (isCompoundType(RetTy) || getContext().getTypeSize(RetTy) > 64)
- return ABIArgInfo::getIndirect(0);
+ return getNaturalAlignIndirect(RetTy);
return (isPromotableIntegerType(RetTy) ?
ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
}
@@ -5509,7 +5830,7 @@
ABIArgInfo SystemZABIInfo::classifyArgumentType(QualType Ty) const {
// Handle the generic C++ ABI.
if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
- return ABIArgInfo::getIndirect(0, RAA == CGCXXABI::RAA_DirectInMemory);
+ return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
// Integers and enums are extended to full register width.
if (isPromotableIntegerType(Ty))
@@ -5526,7 +5847,7 @@
// Values that are not 1, 2, 4 or 8 bytes in size are passed indirectly.
if (Size != 8 && Size != 16 && Size != 32 && Size != 64)
- return ABIArgInfo::getIndirect(0, /*ByVal=*/false);
+ return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
// Handle small structures.
if (const RecordType *RT = Ty->getAs<RecordType>()) {
@@ -5534,7 +5855,7 @@
// fail the size test above.
const RecordDecl *RD = RT->getDecl();
if (RD->hasFlexibleArrayMember())
- return ABIArgInfo::getIndirect(0, /*ByVal=*/false);
+ return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
// The structure is passed as an unextended integer, a float, or a double.
llvm::Type *PassTy;
@@ -5551,7 +5872,7 @@
// Non-structure compounds are passed indirectly.
if (isCompoundType(Ty))
- return ABIArgInfo::getIndirect(0, /*ByVal=*/false);
+ return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
return ABIArgInfo::getDirect(nullptr);
}
@@ -5575,7 +5896,7 @@
void MSP430TargetCodeGenInfo::setTargetAttributes(const Decl *D,
llvm::GlobalValue *GV,
CodeGen::CodeGenModule &M) const {
- if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
+ if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
if (const MSP430InterruptAttr *attr = FD->getAttr<MSP430InterruptAttr>()) {
// Handle 'interrupt' attribute:
llvm::Function *F = cast<llvm::Function>(GV);
@@ -5616,8 +5937,8 @@
ABIArgInfo classifyReturnType(QualType RetTy) const;
ABIArgInfo classifyArgumentType(QualType RetTy, uint64_t &Offset) const;
void computeInfo(CGFunctionInfo &FI) const override;
- llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const override;
+ Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const override;
bool shouldSignExtUnsignedType(QualType Ty) const override;
};
@@ -5634,7 +5955,7 @@
void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
CodeGen::CodeGenModule &CGM) const override {
- const FunctionDecl *FD = dyn_cast<FunctionDecl>(D);
+ const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
if (!FD) return;
llvm::Function *Fn = cast<llvm::Function>(GV);
if (FD->hasAttr<Mips16Attr>()) {
@@ -5643,6 +5964,26 @@
else if (FD->hasAttr<NoMips16Attr>()) {
Fn->addFnAttr("nomips16");
}
+
+ const MipsInterruptAttr *Attr = FD->getAttr<MipsInterruptAttr>();
+ if (!Attr)
+ return;
+
+ const char *Kind;
+ switch (Attr->getInterrupt()) {
+ case MipsInterruptAttr::eic: Kind = "eic"; break;
+ case MipsInterruptAttr::sw0: Kind = "sw0"; break;
+ case MipsInterruptAttr::sw1: Kind = "sw1"; break;
+ case MipsInterruptAttr::hw0: Kind = "hw0"; break;
+ case MipsInterruptAttr::hw1: Kind = "hw1"; break;
+ case MipsInterruptAttr::hw2: Kind = "hw2"; break;
+ case MipsInterruptAttr::hw3: Kind = "hw3"; break;
+ case MipsInterruptAttr::hw4: Kind = "hw4"; break;
+ case MipsInterruptAttr::hw5: Kind = "hw5"; break;
+ }
+
+ Fn->addFnAttr("interrupt", Kind);
+
}
bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
@@ -5756,7 +6097,7 @@
if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) {
Offset = OrigOffset + MinABIStackAlignInBytes;
- return ABIArgInfo::getIndirect(0, RAA == CGCXXABI::RAA_DirectInMemory);
+ return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
}
// If we have reached here, aggregates are passed directly by coercing to
@@ -5850,7 +6191,7 @@
}
}
- return ABIArgInfo::getIndirect(0);
+ return getNaturalAlignIndirect(RetTy);
}
// Treat an enum type as its underlying type.
@@ -5873,52 +6214,55 @@
I.info = classifyArgumentType(I.type, Offset);
}
-llvm::Value* MipsABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const {
- llvm::Type *BP = CGF.Int8PtrTy;
- llvm::Type *BPP = CGF.Int8PtrPtrTy;
+Address MipsABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType OrigTy) const {
+ QualType Ty = OrigTy;
// Integer arguments are promoted to 32-bit on O32 and 64-bit on N32/N64.
// Pointers are also promoted in the same way but this only matters for N32.
unsigned SlotSizeInBits = IsO32 ? 32 : 64;
unsigned PtrWidth = getTarget().getPointerWidth(0);
+ bool DidPromote = false;
if ((Ty->isIntegerType() &&
- CGF.getContext().getIntWidth(Ty) < SlotSizeInBits) ||
+ getContext().getIntWidth(Ty) < SlotSizeInBits) ||
(Ty->isPointerType() && PtrWidth < SlotSizeInBits)) {
- Ty = CGF.getContext().getIntTypeForBitwidth(SlotSizeInBits,
- Ty->isSignedIntegerType());
+ DidPromote = true;
+ Ty = getContext().getIntTypeForBitwidth(SlotSizeInBits,
+ Ty->isSignedIntegerType());
}
- CGBuilderTy &Builder = CGF.Builder;
- llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP, "ap");
- llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur");
- int64_t TypeAlign =
- std::min(getContext().getTypeAlign(Ty) / 8, StackAlignInBytes);
- llvm::Type *PTy = llvm::PointerType::getUnqual(CGF.ConvertType(Ty));
- llvm::Value *AddrTyped;
- llvm::IntegerType *IntTy = (PtrWidth == 32) ? CGF.Int32Ty : CGF.Int64Ty;
+ auto TyInfo = getContext().getTypeInfoInChars(Ty);
- if (TypeAlign > MinABIStackAlignInBytes) {
- llvm::Value *AddrAsInt = CGF.Builder.CreatePtrToInt(Addr, IntTy);
- llvm::Value *Inc = llvm::ConstantInt::get(IntTy, TypeAlign - 1);
- llvm::Value *Mask = llvm::ConstantInt::get(IntTy, -TypeAlign);
- llvm::Value *Add = CGF.Builder.CreateAdd(AddrAsInt, Inc);
- llvm::Value *And = CGF.Builder.CreateAnd(Add, Mask);
- AddrTyped = CGF.Builder.CreateIntToPtr(And, PTy);
+ // The alignment of things in the argument area is never larger than
+ // StackAlignInBytes.
+ TyInfo.second =
+ std::min(TyInfo.second, CharUnits::fromQuantity(StackAlignInBytes));
+
+ // MinABIStackAlignInBytes is the size of argument slots on the stack.
+ CharUnits ArgSlotSize = CharUnits::fromQuantity(MinABIStackAlignInBytes);
+
+ Address Addr = emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*indirect*/ false,
+ TyInfo, ArgSlotSize, /*AllowHigherAlign*/ true);
+
+
+ // If there was a promotion, "unpromote" into a temporary.
+ // TODO: can we just use a pointer into a subset of the original slot?
+ if (DidPromote) {
+ Address Temp = CGF.CreateMemTemp(OrigTy, "vaarg.promotion-temp");
+ llvm::Value *Promoted = CGF.Builder.CreateLoad(Addr);
+
+ // Truncate down to the right width.
+ llvm::Type *IntTy = (OrigTy->isIntegerType() ? Temp.getElementType()
+ : CGF.IntPtrTy);
+ llvm::Value *V = CGF.Builder.CreateTrunc(Promoted, IntTy);
+ if (OrigTy->isPointerType())
+ V = CGF.Builder.CreateIntToPtr(V, Temp.getElementType());
+
+ CGF.Builder.CreateStore(V, Temp);
+ Addr = Temp;
}
- else
- AddrTyped = Builder.CreateBitCast(Addr, PTy);
- llvm::Value *AlignedAddr = Builder.CreateBitCast(AddrTyped, BP);
- TypeAlign = std::max((unsigned)TypeAlign, MinABIStackAlignInBytes);
- unsigned ArgSizeInBits = CGF.getContext().getTypeSize(Ty);
- uint64_t Offset = llvm::RoundUpToAlignment(ArgSizeInBits / 8, TypeAlign);
- llvm::Value *NextAddr =
- Builder.CreateGEP(AlignedAddr, llvm::ConstantInt::get(IntTy, Offset),
- "ap.next");
- Builder.CreateStore(NextAddr, VAListAddrAsBPP);
-
- return AddrTyped;
+ return Addr;
}
bool MipsABIInfo::shouldSignExtUnsignedType(QualType Ty) const {
@@ -5978,7 +6322,7 @@
void TCETargetCodeGenInfo::setTargetAttributes(
const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const {
- const FunctionDecl *FD = dyn_cast<FunctionDecl>(D);
+ const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
if (!FD) return;
llvm::Function *F = cast<llvm::Function>(GV);
@@ -6040,8 +6384,8 @@
void computeInfo(CGFunctionInfo &FI) const override;
- llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const override;
+ Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const override;
};
class HexagonTargetCodeGenInfo : public TargetCodeGenInfo {
@@ -6078,11 +6422,11 @@
return ABIArgInfo::getIgnore();
if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
- return ABIArgInfo::getIndirect(0, RAA == CGCXXABI::RAA_DirectInMemory);
+ return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
uint64_t Size = getContext().getTypeSize(Ty);
if (Size > 64)
- return ABIArgInfo::getIndirect(0, /*ByVal=*/true);
+ return getNaturalAlignIndirect(Ty, /*ByVal=*/true);
// Pass in the smallest viable integer type.
else if (Size > 32)
return ABIArgInfo::getDirect(llvm::Type::getInt64Ty(getVMContext()));
@@ -6100,7 +6444,7 @@
// Large vector types should be returned via memory.
if (RetTy->isVectorType() && getContext().getTypeSize(RetTy) > 64)
- return ABIArgInfo::getIndirect(0);
+ return getNaturalAlignIndirect(RetTy);
if (!isAggregateTypeForABI(RetTy)) {
// Treat an enum type as its underlying type.
@@ -6128,30 +6472,16 @@
return ABIArgInfo::getDirect(llvm::Type::getInt64Ty(getVMContext()));
}
- return ABIArgInfo::getIndirect(0, /*ByVal=*/true);
+ return getNaturalAlignIndirect(RetTy, /*ByVal=*/true);
}
-llvm::Value *HexagonABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const {
- // FIXME: Need to handle alignment
- llvm::Type *BPP = CGF.Int8PtrPtrTy;
-
- CGBuilderTy &Builder = CGF.Builder;
- llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP,
- "ap");
- llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur");
- llvm::Type *PTy =
- llvm::PointerType::getUnqual(CGF.ConvertType(Ty));
- llvm::Value *AddrTyped = Builder.CreateBitCast(Addr, PTy);
-
- uint64_t Offset =
- llvm::RoundUpToAlignment(CGF.getContext().getTypeSize(Ty) / 8, 4);
- llvm::Value *NextAddr =
- Builder.CreateGEP(Addr, llvm::ConstantInt::get(CGF.Int32Ty, Offset),
- "ap.next");
- Builder.CreateStore(NextAddr, VAListAddrAsBPP);
-
- return AddrTyped;
+Address HexagonABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const {
+ // FIXME: Someone needs to audit that this handle alignment correctly.
+ return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*indirect*/ false,
+ getContext().getTypeInfoInChars(Ty),
+ CharUnits::fromQuantity(4),
+ /*AllowHigherAlign*/ true);
}
//===----------------------------------------------------------------------===//
@@ -6174,7 +6504,7 @@
const Decl *D,
llvm::GlobalValue *GV,
CodeGen::CodeGenModule &M) const {
- const FunctionDecl *FD = dyn_cast<FunctionDecl>(D);
+ const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
if (!FD)
return;
@@ -6228,8 +6558,8 @@
private:
ABIArgInfo classifyType(QualType RetTy, unsigned SizeLimit) const;
void computeInfo(CGFunctionInfo &FI) const override;
- llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const override;
+ Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const override;
// Coercion type builder for structs passed in registers. The coercion type
// serves two purposes:
@@ -6349,7 +6679,7 @@
// Anything too big to fit in registers is passed with an explicit indirect
// pointer / sret pointer.
if (Size > SizeLimit)
- return ABIArgInfo::getIndirect(0, /*ByVal=*/false);
+ return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
// Treat an enum type as its underlying type.
if (const EnumType *EnumTy = Ty->getAs<EnumType>())
@@ -6366,7 +6696,7 @@
// If a C++ object has either a non-trivial copy constructor or a non-trivial
// destructor, it is passed with an explicit indirect pointer / sret pointer.
if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
- return ABIArgInfo::getIndirect(0, RAA == CGCXXABI::RAA_DirectInMemory);
+ return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
// This is a small aggregate type that should be passed in registers.
// Build a coercion type from the LLVM struct type.
@@ -6387,55 +6717,59 @@
return ABIArgInfo::getDirect(CoerceTy);
}
-llvm::Value *SparcV9ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const {
+Address SparcV9ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const {
ABIArgInfo AI = classifyType(Ty, 16 * 8);
llvm::Type *ArgTy = CGT.ConvertType(Ty);
if (AI.canHaveCoerceToType() && !AI.getCoerceToType())
AI.setCoerceToType(ArgTy);
- llvm::Type *BPP = CGF.Int8PtrPtrTy;
- CGBuilderTy &Builder = CGF.Builder;
- llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP, "ap");
- llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur");
- llvm::Type *ArgPtrTy = llvm::PointerType::getUnqual(ArgTy);
- llvm::Value *ArgAddr;
- unsigned Stride;
+ CharUnits SlotSize = CharUnits::fromQuantity(8);
+ CGBuilderTy &Builder = CGF.Builder;
+ Address Addr(Builder.CreateLoad(VAListAddr, "ap.cur"), SlotSize);
+ llvm::Type *ArgPtrTy = llvm::PointerType::getUnqual(ArgTy);
+
+ auto TypeInfo = getContext().getTypeInfoInChars(Ty);
+
+ Address ArgAddr = Address::invalid();
+ CharUnits Stride;
switch (AI.getKind()) {
case ABIArgInfo::Expand:
case ABIArgInfo::InAlloca:
llvm_unreachable("Unsupported ABI kind for va_arg");
- case ABIArgInfo::Extend:
- Stride = 8;
- ArgAddr = Builder
- .CreateConstGEP1_32(Addr, 8 - getDataLayout().getTypeAllocSize(ArgTy),
- "extend");
+ case ABIArgInfo::Extend: {
+ Stride = SlotSize;
+ CharUnits Offset = SlotSize - TypeInfo.first;
+ ArgAddr = Builder.CreateConstInBoundsByteGEP(Addr, Offset, "extend");
break;
+ }
- case ABIArgInfo::Direct:
- Stride = getDataLayout().getTypeAllocSize(AI.getCoerceToType());
+ case ABIArgInfo::Direct: {
+ auto AllocSize = getDataLayout().getTypeAllocSize(AI.getCoerceToType());
+ Stride = CharUnits::fromQuantity(AllocSize).RoundUpToAlignment(SlotSize);
ArgAddr = Addr;
break;
+ }
case ABIArgInfo::Indirect:
- Stride = 8;
- ArgAddr = Builder.CreateBitCast(Addr,
- llvm::PointerType::getUnqual(ArgPtrTy),
- "indirect");
- ArgAddr = Builder.CreateLoad(ArgAddr, "indirect.arg");
+ Stride = SlotSize;
+ ArgAddr = Builder.CreateElementBitCast(Addr, ArgPtrTy, "indirect");
+ ArgAddr = Address(Builder.CreateLoad(ArgAddr, "indirect.arg"),
+ TypeInfo.second);
break;
case ABIArgInfo::Ignore:
- return llvm::UndefValue::get(ArgPtrTy);
+ return Address(llvm::UndefValue::get(ArgPtrTy), TypeInfo.second);
}
// Update VAList.
- Addr = Builder.CreateConstGEP1_32(Addr, Stride, "ap.next");
- Builder.CreateStore(Addr, VAListAddrAsBPP);
+ llvm::Value *NextPtr =
+ Builder.CreateConstInBoundsByteGEP(Addr.getPointer(), Stride, "ap.next");
+ Builder.CreateStore(NextPtr, VAListAddr);
- return Builder.CreatePointerCast(ArgAddr, ArgPtrTy, "arg.addr");
+ return Builder.CreateBitCast(ArgAddr, ArgPtrTy, "arg.addr");
}
void SparcV9ABIInfo::computeInfo(CGFunctionInfo &FI) const {
@@ -6570,7 +6904,7 @@
unsigned IncompleteCount; // Number of Incomplete entries in the Map.
unsigned IncompleteUsedCount; // Number of IncompleteUsed entries in the Map.
public:
- TypeStringCache() : IncompleteCount(0), IncompleteUsedCount(0) {};
+ TypeStringCache() : IncompleteCount(0), IncompleteUsedCount(0) {}
void addIncomplete(const IdentifierInfo *ID, std::string StubEnc);
bool removeIncomplete(const IdentifierInfo *ID);
void addIfComplete(const IdentifierInfo *ID, StringRef Str,
@@ -6584,8 +6918,8 @@
bool HasName;
std::string Enc;
public:
- FieldEncoding(bool b, SmallStringEnc &e) : HasName(b), Enc(e.c_str()) {};
- StringRef str() {return Enc.c_str();};
+ FieldEncoding(bool b, SmallStringEnc &e) : HasName(b), Enc(e.c_str()) {}
+ StringRef str() {return Enc.c_str();}
bool operator<(const FieldEncoding &rhs) const {
if (HasName != rhs.HasName) return HasName;
return Enc < rhs.Enc;
@@ -6595,8 +6929,8 @@
class XCoreABIInfo : public DefaultABIInfo {
public:
XCoreABIInfo(CodeGen::CodeGenTypes &CGT) : DefaultABIInfo(CGT) {}
- llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const override;
+ Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const override;
};
class XCoreTargetCodeGenInfo : public TargetCodeGenInfo {
@@ -6610,52 +6944,53 @@
} // End anonymous namespace.
-llvm::Value *XCoreABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
- CodeGenFunction &CGF) const {
+Address XCoreABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
+ QualType Ty) const {
CGBuilderTy &Builder = CGF.Builder;
// Get the VAList.
- llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr,
- CGF.Int8PtrPtrTy);
- llvm::Value *AP = Builder.CreateLoad(VAListAddrAsBPP);
+ CharUnits SlotSize = CharUnits::fromQuantity(4);
+ Address AP(Builder.CreateLoad(VAListAddr), SlotSize);
// Handle the argument.
ABIArgInfo AI = classifyArgumentType(Ty);
+ CharUnits TypeAlign = getContext().getTypeAlignInChars(Ty);
llvm::Type *ArgTy = CGT.ConvertType(Ty);
if (AI.canHaveCoerceToType() && !AI.getCoerceToType())
AI.setCoerceToType(ArgTy);
llvm::Type *ArgPtrTy = llvm::PointerType::getUnqual(ArgTy);
- llvm::Value *Val;
- uint64_t ArgSize = 0;
+
+ Address Val = Address::invalid();
+ CharUnits ArgSize = CharUnits::Zero();
switch (AI.getKind()) {
case ABIArgInfo::Expand:
case ABIArgInfo::InAlloca:
llvm_unreachable("Unsupported ABI kind for va_arg");
case ABIArgInfo::Ignore:
- Val = llvm::UndefValue::get(ArgPtrTy);
- ArgSize = 0;
+ Val = Address(llvm::UndefValue::get(ArgPtrTy), TypeAlign);
+ ArgSize = CharUnits::Zero();
break;
case ABIArgInfo::Extend:
case ABIArgInfo::Direct:
- Val = Builder.CreatePointerCast(AP, ArgPtrTy);
- ArgSize = getDataLayout().getTypeAllocSize(AI.getCoerceToType());
- if (ArgSize < 4)
- ArgSize = 4;
+ Val = Builder.CreateBitCast(AP, ArgPtrTy);
+ ArgSize = CharUnits::fromQuantity(
+ getDataLayout().getTypeAllocSize(AI.getCoerceToType()));
+ ArgSize = ArgSize.RoundUpToAlignment(SlotSize);
break;
case ABIArgInfo::Indirect:
- llvm::Value *ArgAddr;
- ArgAddr = Builder.CreateBitCast(AP, llvm::PointerType::getUnqual(ArgPtrTy));
- ArgAddr = Builder.CreateLoad(ArgAddr);
- Val = Builder.CreatePointerCast(ArgAddr, ArgPtrTy);
- ArgSize = 4;
+ Val = Builder.CreateElementBitCast(AP, ArgPtrTy);
+ Val = Address(Builder.CreateLoad(Val), TypeAlign);
+ ArgSize = SlotSize;
break;
}
// Increment the VAList.
- if (ArgSize) {
- llvm::Value *APN = Builder.CreateConstGEP1_32(AP, ArgSize);
- Builder.CreateStore(APN, VAListAddrAsBPP);
+ if (!ArgSize.isZero()) {
+ llvm::Value *APN =
+ Builder.CreateConstInBoundsByteGEP(AP.getPointer(), ArgSize);
+ Builder.CreateStore(APN, VAListAddr);
}
+
return Val;
}
@@ -6799,9 +7134,7 @@
if (Field->isBitField()) {
Enc += "b(";
llvm::raw_svector_ostream OS(Enc);
- OS.resync();
OS << Field->getBitWidthValue(CGM.getContext());
- OS.flush();
Enc += ':';
}
if (!appendType(Enc, Field->getType(), CGM, TSC))
@@ -6915,7 +7248,7 @@
/// This is done prior to appending the type's encoding.
static void appendQualifier(SmallStringEnc &Enc, QualType QT) {
// Qualifiers are emitted in alphabetical order.
- static const char *Table[] = {"","c:","r:","cr:","v:","cv:","rv:","crv:"};
+ static const char *const Table[]={"","c:","r:","cr:","v:","cv:","rv:","crv:"};
int Lookup = 0;
if (QT.isConstQualified())
Lookup += 1<<0;
@@ -7139,6 +7472,8 @@
return *(TheTargetCodeGenInfo = new PNaClTargetCodeGenInfo(Types));
case llvm::Triple::mips:
case llvm::Triple::mipsel:
+ if (Triple.getOS() == llvm::Triple::NaCl)
+ return *(TheTargetCodeGenInfo = new PNaClTargetCodeGenInfo(Types));
return *(TheTargetCodeGenInfo = new MIPSTargetCodeGenInfo(Types, true));
case llvm::Triple::mips64:
@@ -7154,6 +7489,10 @@
return *(TheTargetCodeGenInfo = new AArch64TargetCodeGenInfo(Types, Kind));
}
+ case llvm::Triple::wasm32:
+ case llvm::Triple::wasm64:
+ return *(TheTargetCodeGenInfo = new WebAssemblyTargetCodeGenInfo(Types));
+
case llvm::Triple::arm:
case llvm::Triple::armeb:
case llvm::Triple::thumb:
@@ -7166,8 +7505,11 @@
}
ARMABIInfo::ABIKind Kind = ARMABIInfo::AAPCS;
- if (getTarget().getABI() == "apcs-gnu")
+ StringRef ABIStr = getTarget().getABI();
+ if (ABIStr == "apcs-gnu")
Kind = ARMABIInfo::APCS;
+ else if (ABIStr == "aapcs16")
+ Kind = ARMABIInfo::AAPCS16_VFP;
else if (CodeGenOpts.FloatABI == "hard" ||
(CodeGenOpts.FloatABI != "soft" &&
Triple.getEnvironment() == llvm::Triple::GNUEABIHF))
@@ -7177,7 +7519,8 @@
}
case llvm::Triple::ppc:
- return *(TheTargetCodeGenInfo = new PPC32TargetCodeGenInfo(Types));
+ return *(TheTargetCodeGenInfo =
+ new PPC32TargetCodeGenInfo(Types, CodeGenOpts.FloatABI == "soft"));
case llvm::Triple::ppc64:
if (Triple.isOSBinFormatELF()) {
PPC64_SVR4_ABIInfo::ABIKind Kind = PPC64_SVR4_ABIInfo::ELFv1;
@@ -7218,29 +7561,38 @@
case llvm::Triple::x86: {
bool IsDarwinVectorABI = Triple.isOSDarwin();
- bool IsSmallStructInRegABI =
+ bool RetSmallStructInRegABI =
X86_32TargetCodeGenInfo::isStructReturnInRegABI(Triple, CodeGenOpts);
bool IsWin32FloatStructABI = Triple.isOSWindows() && !Triple.isOSCygMing();
if (Triple.getOS() == llvm::Triple::Win32) {
return *(TheTargetCodeGenInfo = new WinX86_32TargetCodeGenInfo(
- Types, IsDarwinVectorABI, IsSmallStructInRegABI,
+ Types, IsDarwinVectorABI, RetSmallStructInRegABI,
IsWin32FloatStructABI, CodeGenOpts.NumRegisterParameters));
} else {
return *(TheTargetCodeGenInfo = new X86_32TargetCodeGenInfo(
- Types, IsDarwinVectorABI, IsSmallStructInRegABI,
- IsWin32FloatStructABI, CodeGenOpts.NumRegisterParameters));
+ Types, IsDarwinVectorABI, RetSmallStructInRegABI,
+ IsWin32FloatStructABI, CodeGenOpts.NumRegisterParameters,
+ CodeGenOpts.FloatABI == "soft"));
}
}
case llvm::Triple::x86_64: {
+ StringRef ABI = getTarget().getABI();
+ X86AVXABILevel AVXLevel = (ABI == "avx512" ? X86AVXABILevel::AVX512 :
+ ABI == "avx" ? X86AVXABILevel::AVX :
+ X86AVXABILevel::None);
+
switch (Triple.getOS()) {
case llvm::Triple::Win32:
- return *(TheTargetCodeGenInfo = new WinX86_64TargetCodeGenInfo(Types));
+ return *(TheTargetCodeGenInfo =
+ new WinX86_64TargetCodeGenInfo(Types, AVXLevel));
case llvm::Triple::PS4:
- return *(TheTargetCodeGenInfo = new PS4TargetCodeGenInfo(Types));
+ return *(TheTargetCodeGenInfo =
+ new PS4TargetCodeGenInfo(Types, AVXLevel));
default:
- return *(TheTargetCodeGenInfo = new X86_64TargetCodeGenInfo(Types));
+ return *(TheTargetCodeGenInfo =
+ new X86_64TargetCodeGenInfo(Types, AVXLevel));
}
}
case llvm::Triple::hexagon:
diff --git a/lib/CodeGen/TargetInfo.h b/lib/CodeGen/TargetInfo.h
index bf63265..87b4704 100644
--- a/lib/CodeGen/TargetInfo.h
+++ b/lib/CodeGen/TargetInfo.h
@@ -47,7 +47,7 @@
public:
// WARNING: Acquires the ownership of ABIInfo.
- TargetCodeGenInfo(ABIInfo *info = 0) : Info(info) {}
+ TargetCodeGenInfo(ABIInfo *info = nullptr) : Info(info) {}
virtual ~TargetCodeGenInfo();
/// getABIInfo() - Returns ABI info helper for the target.
@@ -218,14 +218,7 @@
virtual void getDetectMismatchOption(llvm::StringRef Name,
llvm::StringRef Value,
llvm::SmallString<32> &Opt) const {}
-
- /// Gets the target-specific default alignment used when an 'aligned' clause
- /// is used with a 'simd' OpenMP directive without specifying a specific
- /// alignment.
- virtual unsigned getOpenMPSimdDefaultAlignment(QualType Type) const {
- return 0;
- }
};
-}
+} // namespace clang
-#endif
+#endif // LLVM_CLANG_LIB_CODEGEN_TARGETINFO_H