Rewrite NRVO determination. Track NRVO candidates on the parser Scope and apply the NRVO candidate flag to all possible NRVO candidates here, and remove the flags in computeNRVO or upon template instantiation. A variable now has NRVO applied if and only if every return statement in that scope returns that variable. This is nearly optimal.
Performs NRVO roughly 7% more often in a bootstrap build of clang. Patch co-authored by Richard Smith.
llvm-svn: 207890
diff --git a/clang/lib/Sema/Scope.cpp b/clang/lib/Sema/Scope.cpp
index 494768d..8e339d7 100644
--- a/clang/lib/Sema/Scope.cpp
+++ b/clang/lib/Sema/Scope.cpp
@@ -13,6 +13,7 @@
//===----------------------------------------------------------------------===//
#include "clang/Sema/Scope.h"
+#include "clang/AST/Decl.h"
#include "llvm/Support/raw_ostream.h"
using namespace clang;
@@ -77,6 +78,7 @@
UsingDirectives.clear();
Entity = 0;
ErrorTrap.reset();
+ NRVO.setPointerAndInt(nullptr, 0);
}
bool Scope::containedInPrototypeScope() const {
@@ -103,6 +105,21 @@
Flags |= FlagsToSet;
}
+void Scope::mergeNRVOIntoParent() {
+ if (VarDecl *Candidate = NRVO.getPointer()) {
+ if (isDeclScope(Candidate))
+ Candidate->setNRVOVariable(true);
+ }
+
+ if (getEntity())
+ return;
+
+ if (NRVO.getInt())
+ getParent()->setNoNRVO();
+ else if (NRVO.getPointer())
+ getParent()->addNRVOCandidate(NRVO.getPointer());
+}
+
void Scope::dump() const { dumpImpl(llvm::errs()); }
void Scope::dumpImpl(raw_ostream &OS) const {
@@ -176,4 +193,9 @@
OS << "MSLocalManglingNumber: " << getMSLocalManglingNumber() << '\n';
if (const DeclContext *DC = getEntity())
OS << "Entity : (clang::DeclContext*)" << DC << '\n';
+
+ if (NRVO.getInt())
+ OS << "NRVO not allowed";
+ else if (NRVO.getPointer())
+ OS << "NRVO candidate : (clang::VarDecl*)" << NRVO.getPointer() << '\n';
}
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 283b416..68e4ad5 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -1373,6 +1373,8 @@
}
void Sema::ActOnPopScope(SourceLocation Loc, Scope *S) {
+ S->mergeNRVOIntoParent();
+
if (S->decl_empty()) return;
assert((S->getFlags() & (Scope::DeclScope | Scope::TemplateParamScope)) &&
"Scope shouldn't contain decls!");
@@ -9797,28 +9799,17 @@
/// use the named return value optimization.
///
/// This function applies a very simplistic algorithm for NRVO: if every return
-/// statement in the function has the same NRVO candidate, that candidate is
-/// the NRVO variable.
-///
-/// FIXME: Employ a smarter algorithm that accounts for multiple return
-/// statements and the lifetimes of the NRVO candidates. We should be able to
-/// find a maximal set of NRVO variables.
+/// statement in the scope of a variable has the same NRVO candidate, that
+/// candidate is an NRVO variable.
void Sema::computeNRVO(Stmt *Body, FunctionScopeInfo *Scope) {
ReturnStmt **Returns = Scope->Returns.data();
- const VarDecl *NRVOCandidate = 0;
for (unsigned I = 0, E = Scope->Returns.size(); I != E; ++I) {
- if (!Returns[I]->getNRVOCandidate())
- return;
-
- if (!NRVOCandidate)
- NRVOCandidate = Returns[I]->getNRVOCandidate();
- else if (NRVOCandidate != Returns[I]->getNRVOCandidate())
- return;
+ if (const VarDecl *NRVOCandidate = Returns[I]->getNRVOCandidate()) {
+ if (!NRVOCandidate->isNRVOVariable())
+ Returns[I]->setNRVOCandidate(nullptr);
+ }
}
-
- if (NRVOCandidate)
- const_cast<VarDecl*>(NRVOCandidate)->setNRVOVariable(true);
}
bool Sema::canDelayFunctionBody(const Declarator &D) {
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index 1feb979..34954a8 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -9623,7 +9623,7 @@
// Add a "return *this;"
ExprResult ThisObj = CreateBuiltinUnaryOp(Loc, UO_Deref, This.build(*this, Loc));
- StmtResult Return = ActOnReturnStmt(Loc, ThisObj.get());
+ StmtResult Return = BuildReturnStmt(Loc, ThisObj.get());
if (Return.isInvalid())
Invalid = true;
else {
@@ -10041,7 +10041,7 @@
// Add a "return *this;"
ExprResult ThisObj = CreateBuiltinUnaryOp(Loc, UO_Deref, This.build(*this, Loc));
- StmtResult Return = ActOnReturnStmt(Loc, ThisObj.get());
+ StmtResult Return = BuildReturnStmt(Loc, ThisObj.get());
if (Return.isInvalid())
Invalid = true;
else {
@@ -10447,7 +10447,7 @@
Expr *FunctionRef = BuildDeclRefExpr(Invoker, Invoker->getType(),
VK_LValue, Conv->getLocation()).take();
assert(FunctionRef && "Can't refer to __invoke function?");
- Stmt *Return = ActOnReturnStmt(Conv->getLocation(), FunctionRef).take();
+ Stmt *Return = BuildReturnStmt(Conv->getLocation(), FunctionRef).take();
Conv->setBody(new (Context) CompoundStmt(Context, Return,
Conv->getLocation(),
Conv->getLocation()));
@@ -10505,7 +10505,7 @@
// Create the return statement that returns the block from the conversion
// function.
- StmtResult Return = ActOnReturnStmt(Conv->getLocation(), BuildBlock.get());
+ StmtResult Return = BuildReturnStmt(Conv->getLocation(), BuildBlock.get());
if (Return.isInvalid()) {
Diag(CurrentLocation, diag::note_lambda_to_block_conv);
Conv->setInvalidDecl();
diff --git a/clang/lib/Sema/SemaStmt.cpp b/clang/lib/Sema/SemaStmt.cpp
index f40b409..3546114 100644
--- a/clang/lib/Sema/SemaStmt.cpp
+++ b/clang/lib/Sema/SemaStmt.cpp
@@ -2443,52 +2443,62 @@
///
/// \returns The NRVO candidate variable, if the return statement may use the
/// NRVO, or NULL if there is no such candidate.
-const VarDecl *Sema::getCopyElisionCandidate(QualType ReturnType,
- Expr *E,
- bool AllowFunctionParameter) {
- QualType ExprType = E->getType();
+VarDecl *Sema::getCopyElisionCandidate(QualType ReturnType,
+ Expr *E,
+ bool AllowFunctionParameter) {
+ if (!getLangOpts().CPlusPlus)
+ return nullptr;
+
+ // - in a return statement in a function [where] ...
+ // ... the expression is the name of a non-volatile automatic object ...
+ DeclRefExpr *DR = dyn_cast<DeclRefExpr>(E->IgnoreParens());
+ if (!DR || DR->refersToEnclosingLocal())
+ return nullptr;
+ VarDecl *VD = dyn_cast<VarDecl>(DR->getDecl());
+ if (!VD)
+ return nullptr;
+
+ if (isCopyElisionCandidate(ReturnType, VD, AllowFunctionParameter))
+ return VD;
+ return nullptr;
+}
+
+bool Sema::isCopyElisionCandidate(QualType ReturnType, const VarDecl *VD,
+ bool AllowFunctionParameter) {
+ QualType VDType = VD->getType();
// - in a return statement in a function with ...
// ... a class return type ...
- if (!ReturnType.isNull()) {
+ if (!ReturnType.isNull() && !ReturnType->isDependentType()) {
if (!ReturnType->isRecordType())
- return 0;
+ return false;
// ... the same cv-unqualified type as the function return type ...
- if (!Context.hasSameUnqualifiedType(ReturnType, ExprType))
- return 0;
+ if (!VDType->isDependentType() &&
+ !Context.hasSameUnqualifiedType(ReturnType, VDType))
+ return false;
}
- // ... the expression is the name of a non-volatile automatic object
- // (other than a function or catch-clause parameter)) ...
- const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(E->IgnoreParens());
- if (!DR || DR->refersToEnclosingLocal())
- return 0;
- const VarDecl *VD = dyn_cast<VarDecl>(DR->getDecl());
- if (!VD)
- return 0;
-
// ...object (other than a function or catch-clause parameter)...
if (VD->getKind() != Decl::Var &&
!(AllowFunctionParameter && VD->getKind() == Decl::ParmVar))
- return 0;
- if (VD->isExceptionVariable()) return 0;
+ return false;
+ if (VD->isExceptionVariable()) return false;
// ...automatic...
- if (!VD->hasLocalStorage()) return 0;
+ if (!VD->hasLocalStorage()) return false;
// ...non-volatile...
- if (VD->getType().isVolatileQualified()) return 0;
- if (VD->getType()->isReferenceType()) return 0;
+ if (VD->getType().isVolatileQualified()) return false;
// __block variables can't be allocated in a way that permits NRVO.
- if (VD->hasAttr<BlocksAttr>()) return 0;
+ if (VD->hasAttr<BlocksAttr>()) return false;
// Variables with higher required alignment than their type's ABI
// alignment cannot use NRVO.
- if (VD->hasAttr<AlignedAttr>() &&
+ if (!VD->getType()->isDependentType() && VD->hasAttr<AlignedAttr>() &&
Context.getDeclAlign(VD) > Context.getTypeAlignInChars(VD->getType()))
- return 0;
+ return false;
- return VD;
+ return true;
}
/// \brief Perform the initialization of a potentially-movable value, which
@@ -2694,6 +2704,8 @@
}
RetValExp = Res.take();
CheckReturnValExpr(RetValExp, FnRetType, ReturnLoc);
+ } else {
+ NRVOCandidate = getCopyElisionCandidate(FnRetType, RetValExp, false);
}
if (RetValExp) {
@@ -2708,9 +2720,7 @@
// If we need to check for the named return value optimization,
// or if we need to infer the return type,
// save the return statement in our scope for later processing.
- if (CurCap->HasImplicitReturnType ||
- (getLangOpts().CPlusPlus && FnRetType->isRecordType() &&
- !CurContext->isDependentContext()))
+ if (CurCap->HasImplicitReturnType || NRVOCandidate)
FunctionScopes.back()->Returns.push_back(Result);
return Owned(Result);
@@ -2807,7 +2817,24 @@
}
StmtResult
-Sema::ActOnReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp) {
+Sema::ActOnReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp,
+ Scope *CurScope) {
+ StmtResult R = BuildReturnStmt(ReturnLoc, RetValExp);
+ if (R.isInvalid()) {
+ return R;
+ }
+
+ if (VarDecl *VD =
+ const_cast<VarDecl*>(cast<ReturnStmt>(R.get())->getNRVOCandidate())) {
+ CurScope->addNRVOCandidate(VD);
+ } else {
+ CurScope->setNoNRVO();
+ }
+
+ return R;
+}
+
+StmtResult Sema::BuildReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp) {
// Check for unexpanded parameter packs.
if (RetValExp && DiagnoseUnexpandedParameterPack(RetValExp))
return StmtError();
@@ -2948,18 +2975,19 @@
} else {
assert(RetValExp || HasDependentReturnType);
const VarDecl *NRVOCandidate = 0;
+
+ QualType RetType = RelatedRetType.isNull() ? FnRetType : RelatedRetType;
+
+ // C99 6.8.6.4p3(136): The return statement is not an assignment. The
+ // overlap restriction of subclause 6.5.16.1 does not apply to the case of
+ // function return.
+
+ // In C++ the return statement is handled via a copy initialization,
+ // the C version of which boils down to CheckSingleAssignmentConstraints.
+ if (RetValExp)
+ NRVOCandidate = getCopyElisionCandidate(FnRetType, RetValExp, false);
if (!HasDependentReturnType && !RetValExp->isTypeDependent()) {
// we have a non-void function with an expression, continue checking
-
- QualType RetType = (RelatedRetType.isNull() ? FnRetType : RelatedRetType);
-
- // C99 6.8.6.4p3(136): The return statement is not an assignment. The
- // overlap restriction of subclause 6.5.16.1 does not apply to the case of
- // function return.
-
- // In C++ the return statement is handled via a copy initialization,
- // the C version of which boils down to CheckSingleAssignmentConstraints.
- NRVOCandidate = getCopyElisionCandidate(FnRetType, RetValExp, false);
InitializedEntity Entity = InitializedEntity::InitializeResult(ReturnLoc,
RetType,
NRVOCandidate != 0);
@@ -3001,8 +3029,7 @@
// If we need to check for the named return value optimization, save the
// return statement in our scope for later processing.
- if (getLangOpts().CPlusPlus && FnRetType->isRecordType() &&
- !CurContext->isDependentContext())
+ if (Result->getNRVOCandidate())
FunctionScopes.back()->Returns.push_back(Result);
return Owned(Result);
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 8abe2fe..de23a19 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -417,6 +417,13 @@
SemaRef.BuildVariableInstantiation(Var, D, TemplateArgs, LateAttrs, Owner,
StartingScope, InstantiatingVarTemplate);
+
+ if (D->isNRVOVariable()) {
+ QualType ReturnType = cast<FunctionDecl>(DC)->getReturnType();
+ if (SemaRef.isCopyElisionCandidate(ReturnType, Var, false))
+ Var->setNRVOVariable(true);
+ }
+
return Var;
}
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index f113891..46f97ee 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1187,7 +1187,7 @@
/// By default, performs semantic analysis to build the new statement.
/// Subclasses may override this routine to provide different behavior.
StmtResult RebuildReturnStmt(SourceLocation ReturnLoc, Expr *Result) {
- return getSema().ActOnReturnStmt(ReturnLoc, Result);
+ return getSema().BuildReturnStmt(ReturnLoc, Result);
}
/// \brief Build a new declaration statement.