Make clang to mark static stack allocations with lifetime markers to enable a more aggressive stack coloring.
Patch by John McCall with help by Shuxin Yang.
rdar://13115369
git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@177819 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/lib/CodeGen/CGDecl.cpp b/lib/CodeGen/CGDecl.cpp
index a99fe29..c095ff9 100644
--- a/lib/CodeGen/CGDecl.cpp
+++ b/lib/CodeGen/CGDecl.cpp
@@ -452,6 +452,22 @@
CGF.EmitCall(FnInfo, CleanupFn, ReturnValueSlot(), Args);
}
};
+
+ /// A cleanup to call @llvm.lifetime.end.
+ class CallLifetimeEnd : public EHScopeStack::Cleanup {
+ llvm::Value *Addr;
+ llvm::Value *Size;
+ public:
+ CallLifetimeEnd(llvm::Value *addr, llvm::Value *size)
+ : Addr(addr), Size(size) {}
+
+ void Emit(CodeGenFunction &CGF, Flags flags) {
+ llvm::Value *castAddr = CGF.Builder.CreateBitCast(Addr, CGF.Int8PtrTy);
+ CGF.Builder.CreateCall2(CGF.CGM.getLLVMLifetimeEndFn(),
+ Size, castAddr)
+ ->setDoesNotThrow();
+ }
+ };
}
/// EmitAutoVarWithLifetime - Does the setup required for an automatic
@@ -756,7 +772,6 @@
// If a global is all zeros, always use a memset.
if (isa<llvm::ConstantAggregateZero>(Init)) return true;
-
// If a non-zero global is <= 32 bytes, always use a memcpy. If it is large,
// do it if it will require 6 or fewer scalar stores.
// TODO: Should budget depends on the size? Avoiding a large global warrants
@@ -768,6 +783,20 @@
canEmitInitWithFewStoresAfterMemset(Init, StoreBudget);
}
+/// Should we use the LLVM lifetime intrinsics for the given local variable?
+static bool shouldUseLifetimeMarkers(CodeGenFunction &CGF, const VarDecl &D,
+ unsigned Size) {
+ // For now, only in optimized builds.
+ if (CGF.CGM.getCodeGenOpts().OptimizationLevel == 0)
+ return false;
+
+ // Limit the size of marked objects to 32 bytes. We don't want to increase
+ // compile time by marking tiny objects.
+ unsigned SizeThreshold = 32;
+
+ return Size > SizeThreshold;
+}
+
/// EmitAutoVarDecl - Emit code and set up an entry in LocalDeclMap for a
/// variable declaration with auto, register, or no storage class specifier.
@@ -870,6 +899,20 @@
getContext().toCharUnitsFromBits(Target.getPointerAlign(0)));
Alloc->setAlignment(allocaAlignment.getQuantity());
DeclPtr = Alloc;
+
+ // Emit a lifetime intrinsic if meaningful. There's no point
+ // in doing this if we don't have a valid insertion point (?).
+ uint64_t size = CGM.getDataLayout().getTypeAllocSize(LTy);
+ if (HaveInsertPoint() && shouldUseLifetimeMarkers(*this, D, size)) {
+ llvm::Value *sizeV = llvm::ConstantInt::get(Int64Ty, size);
+
+ emission.SizeForLifetimeMarkers = sizeV;
+ llvm::Value *castAddr = Builder.CreateBitCast(Alloc, Int8PtrTy);
+ Builder.CreateCall2(CGM.getLLVMLifetimeStartFn(), sizeV, castAddr)
+ ->setDoesNotThrow();
+ } else {
+ assert(!emission.useLifetimeMarkers());
+ }
}
} else {
// Targets that don't support recursion emit locals as globals.
@@ -1215,6 +1258,14 @@
const VarDecl &D = *emission.Variable;
+ // Make sure we call @llvm.lifetime.end. This needs to happen
+ // *last*, so the cleanup needs to be pushed *first*.
+ if (emission.useLifetimeMarkers()) {
+ EHStack.pushCleanup<CallLifetimeEnd>(NormalCleanup,
+ emission.getAllocatedAddress(),
+ emission.getSizeForLifetimeMarkers());
+ }
+
// Check the type for a cleanup.
if (QualType::DestructionKind dtorKind = D.getType().isDestructedType())
emitAutoVarTypeCleanup(emission, dtorKind);
@@ -1485,6 +1536,22 @@
elementType, destroyer);
}
+/// Lazily declare the @llvm.lifetime.start intrinsic.
+llvm::Constant *CodeGenModule::getLLVMLifetimeStartFn() {
+ if (LifetimeStartFn) return LifetimeStartFn;
+ LifetimeStartFn = llvm::Intrinsic::getDeclaration(&getModule(),
+ llvm::Intrinsic::lifetime_start);
+ return LifetimeStartFn;
+}
+
+/// Lazily declare the @llvm.lifetime.end intrinsic.
+llvm::Constant *CodeGenModule::getLLVMLifetimeEndFn() {
+ if (LifetimeEndFn) return LifetimeEndFn;
+ LifetimeEndFn = llvm::Intrinsic::getDeclaration(&getModule(),
+ llvm::Intrinsic::lifetime_end);
+ return LifetimeEndFn;
+}
+
namespace {
/// A cleanup to perform a release of an object at the end of a
/// function. This is used to balance out the incoming +1 of a
diff --git a/lib/CodeGen/CGStmt.cpp b/lib/CodeGen/CGStmt.cpp
index 9e7ddfb..06f7460 100644
--- a/lib/CodeGen/CGStmt.cpp
+++ b/lib/CodeGen/CGStmt.cpp
@@ -319,6 +319,12 @@
}
void CodeGenFunction::EmitLabel(const LabelDecl *D) {
+ // Add this label to the current lexical scope if we're within any
+ // normal cleanups. Jumps "in" to this label --- when permitted by
+ // the language --- may need to be routed around such cleanups.
+ if (EHStack.hasNormalCleanups() && CurLexicalScope)
+ CurLexicalScope->addLabel(D);
+
JumpDest &Dest = LabelMap[D];
// If we didn't need a forward reference to this label, just go
@@ -330,16 +336,36 @@
// it from the branch-fixups list.
} else {
assert(!Dest.getScopeDepth().isValid() && "already emitted label!");
- Dest = JumpDest(Dest.getBlock(),
- EHStack.stable_begin(),
- Dest.getDestIndex());
-
+ Dest.setScopeDepth(EHStack.stable_begin());
ResolveBranchFixups(Dest.getBlock());
}
EmitBlock(Dest.getBlock());
}
+/// Change the cleanup scope of the labels in this lexical scope to
+/// match the scope of the enclosing context.
+void CodeGenFunction::LexicalScope::rescopeLabels() {
+ assert(!Labels.empty());
+ EHScopeStack::stable_iterator innermostScope
+ = CGF.EHStack.getInnermostNormalCleanup();
+
+ // Change the scope depth of all the labels.
+ for (SmallVectorImpl<const LabelDecl*>::const_iterator
+ i = Labels.begin(), e = Labels.end(); i != e; ++i) {
+ assert(CGF.LabelMap.count(*i));
+ JumpDest &dest = CGF.LabelMap.find(*i)->second;
+ assert(dest.getScopeDepth().isValid());
+ assert(innermostScope.encloses(dest.getScopeDepth()));
+ dest.setScopeDepth(innermostScope);
+ }
+
+ // Reparent the labels if the new scope also has cleanups.
+ if (innermostScope != EHScopeStack::stable_end() && ParentScope) {
+ ParentScope->Labels.append(Labels.begin(), Labels.end());
+ }
+}
+
void CodeGenFunction::EmitLabelStmt(const LabelStmt &S) {
EmitLabel(S.getDecl());
diff --git a/lib/CodeGen/CodeGenFunction.cpp b/lib/CodeGen/CodeGenFunction.cpp
index bd24435..04de55d 100644
--- a/lib/CodeGen/CodeGenFunction.cpp
+++ b/lib/CodeGen/CodeGenFunction.cpp
@@ -45,7 +45,7 @@
IndirectBranch(0), SwitchInsn(0), CaseRangeBlock(0), UnreachableBlock(0),
CXXABIThisDecl(0), CXXABIThisValue(0), CXXThisValue(0),
CXXStructorImplicitParamDecl(0), CXXStructorImplicitParamValue(0),
- OutermostConditional(0), TerminateLandingPad(0),
+ OutermostConditional(0), CurLexicalScope(0), TerminateLandingPad(0),
TerminateHandler(0), TrapBB(0) {
if (!suppressNewContext)
CGM.getCXXABI().getMangleContext().startNewFunction();
diff --git a/lib/CodeGen/CodeGenFunction.h b/lib/CodeGen/CodeGenFunction.h
index 46848ae..e40a731 100644
--- a/lib/CodeGen/CodeGenFunction.h
+++ b/lib/CodeGen/CodeGenFunction.h
@@ -562,6 +562,11 @@
EHScopeStack::stable_iterator getScopeDepth() const { return ScopeDepth; }
unsigned getDestIndex() const { return Index; }
+ // This should be used cautiously.
+ void setScopeDepth(EHScopeStack::stable_iterator depth) {
+ ScopeDepth = depth;
+ }
+
private:
llvm::BasicBlock *Block;
EHScopeStack::stable_iterator ScopeDepth;
@@ -853,6 +858,8 @@
class LexicalScope: protected RunCleanupsScope {
SourceRange Range;
+ SmallVector<const LabelDecl*, 4> Labels;
+ LexicalScope *ParentScope;
LexicalScope(const LexicalScope &) LLVM_DELETED_FUNCTION;
void operator=(const LexicalScope &) LLVM_DELETED_FUNCTION;
@@ -860,15 +867,23 @@
public:
/// \brief Enter a new cleanup scope.
explicit LexicalScope(CodeGenFunction &CGF, SourceRange Range)
- : RunCleanupsScope(CGF), Range(Range) {
+ : RunCleanupsScope(CGF), Range(Range), ParentScope(CGF.CurLexicalScope) {
+ CGF.CurLexicalScope = this;
if (CGDebugInfo *DI = CGF.getDebugInfo())
DI->EmitLexicalBlockStart(CGF.Builder, Range.getBegin());
}
+ void addLabel(const LabelDecl *label) {
+ assert(PerformCleanup && "adding label to dead scope?");
+ Labels.push_back(label);
+ }
+
/// \brief Exit this cleanup scope, emitting any accumulated
/// cleanups.
~LexicalScope() {
- if (PerformCleanup) endLexicalScope();
+ // If we should perform a cleanup, force them now. Note that
+ // this ends the cleanup scope before rescoping any labels.
+ if (PerformCleanup) ForceCleanup();
}
/// \brief Force the emission of cleanups now, instead of waiting
@@ -880,9 +895,14 @@
private:
void endLexicalScope() {
+ CGF.CurLexicalScope = ParentScope;
if (CGDebugInfo *DI = CGF.getDebugInfo())
DI->EmitLexicalBlockEnd(CGF.Builder, Range.getEnd());
+ if (!Labels.empty())
+ rescopeLabels();
}
+
+ void rescopeLabels();
};
@@ -1205,6 +1225,8 @@
/// temporary should be destroyed conditionally.
ConditionalEvaluation *OutermostConditional;
+ /// The current lexical scope.
+ LexicalScope *CurLexicalScope;
/// ByrefValueInfoMap - For each __block variable, contains a pair of the LLVM
/// type as well as the field number that contains the actual data.
@@ -2001,18 +2023,34 @@
/// initializer.
bool IsConstantAggregate;
+ /// Non-null if we should use lifetime annotations.
+ llvm::Value *SizeForLifetimeMarkers;
+
struct Invalid {};
AutoVarEmission(Invalid) : Variable(0) {}
AutoVarEmission(const VarDecl &variable)
: Variable(&variable), Address(0), NRVOFlag(0),
- IsByRef(false), IsConstantAggregate(false) {}
+ IsByRef(false), IsConstantAggregate(false),
+ SizeForLifetimeMarkers(0) {}
bool wasEmittedAsGlobal() const { return Address == 0; }
public:
static AutoVarEmission invalid() { return AutoVarEmission(Invalid()); }
+ bool useLifetimeMarkers() const { return SizeForLifetimeMarkers != 0; }
+ llvm::Value *getSizeForLifetimeMarkers() const {
+ assert(useLifetimeMarkers());
+ return SizeForLifetimeMarkers;
+ }
+
+ /// Returns the raw, allocated address, which is not necessarily
+ /// the address of the object itself.
+ llvm::Value *getAllocatedAddress() const {
+ return Address;
+ }
+
/// Returns the address of the object within this declaration.
/// Note that this does not chase the forwarding pointer for
/// __block decls.
diff --git a/lib/CodeGen/CodeGenModule.cpp b/lib/CodeGen/CodeGenModule.cpp
index 1737744..9f604e8 100644
--- a/lib/CodeGen/CodeGenModule.cpp
+++ b/lib/CodeGen/CodeGenModule.cpp
@@ -86,6 +86,7 @@
NSConcreteGlobalBlock(0), NSConcreteStackBlock(0),
BlockObjectAssign(0), BlockObjectDispose(0),
BlockDescriptorType(0), GenericBlockLiteralType(0),
+ LifetimeStartFn(0), LifetimeEndFn(0),
SanitizerBlacklist(CGO.SanitizerBlacklistFile),
SanOpts(SanitizerBlacklist.isIn(M) ?
SanitizerOptions::Disabled : LangOpts.Sanitize) {
diff --git a/lib/CodeGen/CodeGenModule.h b/lib/CodeGen/CodeGenModule.h
index a5f0689..fcd4830 100644
--- a/lib/CodeGen/CodeGenModule.h
+++ b/lib/CodeGen/CodeGenModule.h
@@ -381,6 +381,12 @@
int GlobalUniqueCount;
} Block;
+ /// void @llvm.lifetime.start(i64 %size, i8* nocapture <ptr>)
+ llvm::Constant *LifetimeStartFn;
+
+ /// void @llvm.lifetime.end(i64 %size, i8* nocapture <ptr>)
+ llvm::Constant *LifetimeEndFn;
+
GlobalDecl initializedGlobalDecl;
llvm::BlackList SanitizerBlacklist;
@@ -757,6 +763,9 @@
///@}
+ llvm::Constant *getLLVMLifetimeStartFn();
+ llvm::Constant *getLLVMLifetimeEndFn();
+
// UpdateCompleteType - Make sure that this type is translated.
void UpdateCompletedType(const TagDecl *TD);