Emit lifetime.start / lifetime.end markers for unnamed temporary objects.
This will give more information to the optimizers so that they can reuse stack slots
and reduce stack usage.
llvm-svn: 218865
diff --git a/clang/lib/CodeGen/CGCleanup.cpp b/clang/lib/CodeGen/CGCleanup.cpp
index ed9f96d..911734a 100644
--- a/clang/lib/CodeGen/CGCleanup.cpp
+++ b/clang/lib/CodeGen/CGCleanup.cpp
@@ -387,14 +387,9 @@
}
}
-/// Pops cleanup blocks until the given savepoint is reached, then add the
-/// cleanups from the given savepoint in the lifetime-extended cleanups stack.
+/// Move our deferred cleanups onto the EH stack.
void
-CodeGenFunction::PopCleanupBlocks(EHScopeStack::stable_iterator Old,
- size_t OldLifetimeExtendedSize) {
- PopCleanupBlocks(Old);
-
- // Move our deferred cleanups onto the EH stack.
+CodeGenFunction::MoveDeferedCleanups(size_t OldLifetimeExtendedSize) {
for (size_t I = OldLifetimeExtendedSize,
E = LifetimeExtendedCleanupStack.size(); I != E; /**/) {
// Alignment should be guaranteed by the vptrs in the individual cleanups.
@@ -414,6 +409,17 @@
LifetimeExtendedCleanupStack.resize(OldLifetimeExtendedSize);
}
+/// Pops cleanup blocks until the given savepoint is reached, then add the
+/// cleanups from the given savepoint in the lifetime-extended cleanups stack.
+void
+CodeGenFunction::PopCleanupBlocks(EHScopeStack::stable_iterator Old,
+ size_t OldLifetimeExtendedSize) {
+ PopCleanupBlocks(Old);
+
+ // Move our deferred cleanups onto the EH stack.
+ MoveDeferedCleanups(OldLifetimeExtendedSize);
+}
+
static llvm::BasicBlock *CreateNormalEntry(CodeGenFunction &CGF,
EHCleanupScope &Scope) {
assert(Scope.isNormalCleanup());
diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp
index a9c2da8..7f6b296 100644
--- a/clang/lib/CodeGen/CGDecl.cpp
+++ b/clang/lib/CodeGen/CGDecl.cpp
@@ -476,12 +476,10 @@
: Addr(addr), Size(size) {}
void Emit(CodeGenFunction &CGF, Flags flags) override {
- llvm::Value *castAddr = CGF.Builder.CreateBitCast(Addr, CGF.Int8PtrTy);
- CGF.Builder.CreateCall2(CGF.CGM.getLLVMLifetimeEndFn(),
- Size, castAddr)
- ->setDoesNotThrow();
+ CGF.EmitLifetimeEnd(Size, Addr);
}
};
+
}
/// EmitAutoVarWithLifetime - Does the setup required for an automatic
@@ -800,8 +798,7 @@
}
/// Should we use the LLVM lifetime intrinsics for the given local variable?
-static bool shouldUseLifetimeMarkers(CodeGenFunction &CGF, const VarDecl &D,
- unsigned Size) {
+static bool shouldUseLifetimeMarkers(CodeGenFunction &CGF, uint64_t Size) {
// For now, only in optimized builds.
if (CGF.CGM.getCodeGenOpts().OptimizationLevel == 0)
return false;
@@ -813,7 +810,6 @@
return Size > SizeThreshold;
}
-
/// EmitAutoVarDecl - Emit code and set up an entry in LocalDeclMap for a
/// variable declaration with auto, register, or no storage class specifier.
/// These turn into simple stack objects, or GlobalValues depending on target.
@@ -823,6 +819,27 @@
EmitAutoVarCleanups(emission);
}
+/// Emit a lifetime.begin marker if some criteria are satisfied.
+/// \return a pointer to the temporary size Value if a marker was emitted, null
+/// otherwise
+llvm::Value *CodeGenFunction::EmitLifetimeStart(uint64_t Size,
+ llvm::Value *Addr) {
+ if (!shouldUseLifetimeMarkers(*this, Size))
+ return nullptr;
+
+ llvm::Value *SizeV = llvm::ConstantInt::get(Int64Ty, Size);
+ llvm::Value *CastAddr = Builder.CreateBitCast(Addr, Int8PtrTy);
+ Builder.CreateCall2(CGM.getLLVMLifetimeStartFn(), SizeV, CastAddr)
+ ->setDoesNotThrow();
+ return SizeV;
+}
+
+void CodeGenFunction::EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr) {
+ llvm::Value *CastAddr = Builder.CreateBitCast(Addr, Int8PtrTy);
+ Builder.CreateCall2(CGM.getLLVMLifetimeEndFn(), Size, CastAddr)
+ ->setDoesNotThrow();
+}
+
/// EmitAutoVarAlloca - Emit the alloca and debug information for a
/// local variable. Does not emit initialization or destruction.
CodeGenFunction::AutoVarEmission
@@ -918,13 +935,8 @@
// Emit a lifetime intrinsic if meaningful. There's no point
// in doing this if we don't have a valid insertion point (?).
uint64_t size = CGM.getDataLayout().getTypeAllocSize(LTy);
- if (HaveInsertPoint() && shouldUseLifetimeMarkers(*this, D, size)) {
- llvm::Value *sizeV = llvm::ConstantInt::get(Int64Ty, size);
-
- emission.SizeForLifetimeMarkers = sizeV;
- llvm::Value *castAddr = Builder.CreateBitCast(Alloc, Int8PtrTy);
- Builder.CreateCall2(CGM.getLLVMLifetimeStartFn(), sizeV, castAddr)
- ->setDoesNotThrow();
+ if (HaveInsertPoint() && EmitLifetimeStart(size, Alloc)) {
+ emission.SizeForLifetimeMarkers = llvm::ConstantInt::get(Int64Ty, size);
} else {
assert(!emission.useLifetimeMarkers());
}
@@ -1366,6 +1378,32 @@
cleanupKind, addr, type, destroyer, useEHCleanupForArray);
}
+void
+CodeGenFunction::pushLifetimeEndMarker(StorageDuration SD,
+ llvm::Value *ReferenceTemporary,
+ llvm::Value *SizeForLifeTimeMarkers) {
+ // SizeForLifeTimeMarkers is null in case no corresponding
+ // @llvm.lifetime.start was emitted: there is nothing to do then.
+ if (!SizeForLifeTimeMarkers)
+ return;
+
+ switch (SD) {
+ case SD_FullExpression:
+ pushFullExprCleanup<CallLifetimeEnd>(NormalAndEHCleanup, ReferenceTemporary,
+ SizeForLifeTimeMarkers);
+ return;
+ case SD_Automatic:
+ EHStack.pushCleanup<CallLifetimeEnd>(static_cast<CleanupKind>(EHCleanup),
+ ReferenceTemporary,
+ SizeForLifeTimeMarkers);
+ pushCleanupAfterFullExpr<CallLifetimeEnd>(
+ NormalAndEHCleanup, ReferenceTemporary, SizeForLifeTimeMarkers);
+ return;
+ default:
+ llvm_unreachable("unexpected storage duration for Lifetime markers");
+ }
+}
+
/// emitDestroy - Immediately perform the destruction of the given
/// object.
///
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 4fe3754..7b26009 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -173,9 +173,10 @@
llvm_unreachable("bad evaluation kind");
}
-static void
-pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M,
- const Expr *E, llvm::Value *ReferenceTemporary) {
+static void pushTemporaryCleanup(CodeGenFunction &CGF,
+ const MaterializeTemporaryExpr *M,
+ const Expr *E, llvm::Value *ReferenceTemporary,
+ llvm::Value *SizeForLifeTimeMarkers) {
// Objective-C++ ARC:
// If we are binding a reference to a temporary that has ownership, we
// need to perform retain/release operations on the temporary.
@@ -242,6 +243,10 @@
}
}
+ // Call @llvm.lifetime.end marker for the temporary.
+ CGF.pushLifetimeEndMarker(M->getStorageDuration(), ReferenceTemporary,
+ SizeForLifeTimeMarkers);
+
CXXDestructorDecl *ReferenceTemporaryDtor = nullptr;
if (const RecordType *RT =
E->getType()->getBaseElementTypeUnsafe()->getAs<RecordType>()) {
@@ -296,11 +301,18 @@
static llvm::Value *
createReferenceTemporary(CodeGenFunction &CGF,
- const MaterializeTemporaryExpr *M, const Expr *Inner) {
+ const MaterializeTemporaryExpr *M, const Expr *Inner,
+ llvm::Value *&SizeForLifeTimeMarkers) {
+ SizeForLifeTimeMarkers = nullptr;
switch (M->getStorageDuration()) {
case SD_FullExpression:
- case SD_Automatic:
- return CGF.CreateMemTemp(Inner->getType(), "ref.tmp");
+ case SD_Automatic: {
+ llvm::Value *RefTemp = CGF.CreateMemTemp(Inner->getType(), "ref.tmp");
+ uint64_t TempSize = CGF.CGM.getDataLayout().getTypeStoreSize(
+ CGF.ConvertTypeForMem(Inner->getType()));
+ SizeForLifeTimeMarkers = CGF.EmitLifetimeStart(TempSize, RefTemp);
+ return RefTemp;
+ }
case SD_Thread:
case SD_Static:
@@ -321,7 +333,8 @@
M->getType().getObjCLifetime() != Qualifiers::OCL_None &&
M->getType().getObjCLifetime() != Qualifiers::OCL_ExplicitNone) {
// FIXME: Fold this into the general case below.
- llvm::Value *Object = createReferenceTemporary(*this, M, E);
+ llvm::Value *ObjectSize;
+ llvm::Value *Object = createReferenceTemporary(*this, M, E, ObjectSize);
LValue RefTempDst = MakeAddrLValue(Object, M->getType());
if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object)) {
@@ -333,7 +346,7 @@
EmitScalarInit(E, M->getExtendingDecl(), RefTempDst, false);
- pushTemporaryCleanup(*this, M, E, Object);
+ pushTemporaryCleanup(*this, M, E, Object, ObjectSize);
return RefTempDst;
}
@@ -351,8 +364,10 @@
}
}
- // Create and initialize the reference temporary.
- llvm::Value *Object = createReferenceTemporary(*this, M, E);
+ // Create and initialize the reference temporary and get the temporary size
+ llvm::Value *ObjectSize;
+ llvm::Value *Object = createReferenceTemporary(*this, M, E, ObjectSize);
+
if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object)) {
// If the temporary is a global and has a constant initializer, we may
// have already initialized it.
@@ -363,7 +378,8 @@
} else {
EmitAnyExprToMem(E, Object, Qualifiers(), /*IsInit*/true);
}
- pushTemporaryCleanup(*this, M, E, Object);
+
+ pushTemporaryCleanup(*this, M, E, Object, ObjectSize);
// Perform derived-to-base casts and/or field accesses, to get from the
// temporary object we created (and, potentially, for which we extended
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 9f35918..9361771 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -229,6 +229,11 @@
DI->EmitLocation(Builder, EndLoc);
}
+ // Some top level lifetime extended variables may still need
+ // to have their cleanups called.
+ if (!LifetimeExtendedCleanupStack.empty())
+ MoveDeferedCleanups(0);
+
// Pop any cleanups that might have been associated with the
// parameters. Do this in whatever block we're currently in; it's
// important to do this before we enter the return block or return
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 2709a36..a4171a0 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -444,6 +444,23 @@
new (Buffer + sizeof(Header)) T(a0, a1, a2, a3);
}
+ /// \brief Queue a cleanup to be pushed after finishing the current
+ /// full-expression.
+ template <class T, class A0, class A1>
+ void pushCleanupAfterFullExpr(CleanupKind Kind, A0 a0, A1 a1) {
+ assert(!isInConditionalBranch() && "can't defer conditional cleanup");
+
+ LifetimeExtendedCleanupHeader Header = { sizeof(T), Kind };
+
+ size_t OldSize = LifetimeExtendedCleanupStack.size();
+ LifetimeExtendedCleanupStack.resize(
+ LifetimeExtendedCleanupStack.size() + sizeof(Header) + Header.Size);
+
+ char *Buffer = &LifetimeExtendedCleanupStack[OldSize];
+ new (Buffer) LifetimeExtendedCleanupHeader(Header);
+ new (Buffer + sizeof(Header)) T(a0, a1);
+ }
+
/// Set up the last cleaup that was pushed as a conditional
/// full-expression cleanup.
void initFullExprCleanup();
@@ -596,6 +613,10 @@
void PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize,
size_t OldLifetimeExtendedStackSize);
+ /// \brief Moves deferred cleanups from lifetime-extended variables from
+ /// the given position on top of the stack
+ void MoveDeferedCleanups(size_t OldLifetimeExtendedSize);
+
void ResolveBranchFixups(llvm::BasicBlock *Target);
/// The given basic block lies in the current EH scope, but may be a
@@ -1112,6 +1133,9 @@
void pushLifetimeExtendedDestroy(CleanupKind kind, llvm::Value *addr,
QualType type, Destroyer *destroyer,
bool useEHCleanupForArray);
+ void pushLifetimeEndMarker(StorageDuration SD,
+ llvm::Value *ReferenceTemporary,
+ llvm::Value *SizeForLifeTimeMarkers);
void pushStackRestore(CleanupKind kind, llvm::Value *SPMem);
void emitDestroy(llvm::Value *addr, QualType type, Destroyer *destroyer,
bool useEHCleanupForArray);
@@ -1715,6 +1739,9 @@
void EmitCXXTemporary(const CXXTemporary *Temporary, QualType TempType,
llvm::Value *Ptr);
+ llvm::Value *EmitLifetimeStart(uint64_t Size, llvm::Value *Addr);
+ void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr);
+
llvm::Value *EmitCXXNewExpr(const CXXNewExpr *E);
void EmitCXXDeleteExpr(const CXXDeleteExpr *E);