Update aosp/master clang for rebase to r239765
Change-Id: I0393bcc952590a7226af8c4b58534a8ee5fd2d99
diff --git a/lib/CodeGen/CodeGenPGO.cpp b/lib/CodeGen/CodeGenPGO.cpp
index cc6ac20..f182a46 100644
--- a/lib/CodeGen/CodeGenPGO.cpp
+++ b/lib/CodeGen/CodeGenPGO.cpp
@@ -242,6 +242,9 @@
/// next statement, such as at the exit of a loop.
bool RecordNextStmtCount;
+ /// The count at the current location in the traversal.
+ uint64_t CurrentCount;
+
/// The map of statements to count values.
llvm::DenseMap<const Stmt *, uint64_t> &CountMap;
@@ -259,11 +262,17 @@
void RecordStmtCount(const Stmt *S) {
if (RecordNextStmtCount) {
- CountMap[S] = PGO.getCurrentRegionCount();
+ CountMap[S] = CurrentCount;
RecordNextStmtCount = false;
}
}
+ /// Set and return the current count.
+ uint64_t setCount(uint64_t Count) {
+ CurrentCount = Count;
+ return Count;
+ }
+
void VisitStmt(const Stmt *S) {
RecordStmtCount(S);
for (Stmt::const_child_range I = S->children(); I; ++I) {
@@ -274,9 +283,8 @@
void VisitFunctionDecl(const FunctionDecl *D) {
// Counter tracks entry to the function body.
- RegionCounter Cnt(PGO, D->getBody());
- Cnt.beginRegion();
- CountMap[D->getBody()] = PGO.getCurrentRegionCount();
+ uint64_t BodyCount = setCount(PGO.getRegionCount(D->getBody()));
+ CountMap[D->getBody()] = BodyCount;
Visit(D->getBody());
}
@@ -287,25 +295,22 @@
void VisitCapturedDecl(const CapturedDecl *D) {
// Counter tracks entry to the capture body.
- RegionCounter Cnt(PGO, D->getBody());
- Cnt.beginRegion();
- CountMap[D->getBody()] = PGO.getCurrentRegionCount();
+ uint64_t BodyCount = setCount(PGO.getRegionCount(D->getBody()));
+ CountMap[D->getBody()] = BodyCount;
Visit(D->getBody());
}
void VisitObjCMethodDecl(const ObjCMethodDecl *D) {
// Counter tracks entry to the method body.
- RegionCounter Cnt(PGO, D->getBody());
- Cnt.beginRegion();
- CountMap[D->getBody()] = PGO.getCurrentRegionCount();
+ uint64_t BodyCount = setCount(PGO.getRegionCount(D->getBody()));
+ CountMap[D->getBody()] = BodyCount;
Visit(D->getBody());
}
void VisitBlockDecl(const BlockDecl *D) {
// Counter tracks entry to the block body.
- RegionCounter Cnt(PGO, D->getBody());
- Cnt.beginRegion();
- CountMap[D->getBody()] = PGO.getCurrentRegionCount();
+ uint64_t BodyCount = setCount(PGO.getRegionCount(D->getBody()));
+ CountMap[D->getBody()] = BodyCount;
Visit(D->getBody());
}
@@ -313,89 +318,91 @@
RecordStmtCount(S);
if (S->getRetValue())
Visit(S->getRetValue());
- PGO.setCurrentRegionUnreachable();
+ CurrentCount = 0;
+ RecordNextStmtCount = true;
+ }
+
+ void VisitCXXThrowExpr(const CXXThrowExpr *E) {
+ RecordStmtCount(E);
+ if (E->getSubExpr())
+ Visit(E->getSubExpr());
+ CurrentCount = 0;
RecordNextStmtCount = true;
}
void VisitGotoStmt(const GotoStmt *S) {
RecordStmtCount(S);
- PGO.setCurrentRegionUnreachable();
+ CurrentCount = 0;
RecordNextStmtCount = true;
}
void VisitLabelStmt(const LabelStmt *S) {
RecordNextStmtCount = false;
// Counter tracks the block following the label.
- RegionCounter Cnt(PGO, S);
- Cnt.beginRegion();
- CountMap[S] = PGO.getCurrentRegionCount();
+ uint64_t BlockCount = setCount(PGO.getRegionCount(S));
+ CountMap[S] = BlockCount;
Visit(S->getSubStmt());
}
void VisitBreakStmt(const BreakStmt *S) {
RecordStmtCount(S);
assert(!BreakContinueStack.empty() && "break not in a loop or switch!");
- BreakContinueStack.back().BreakCount += PGO.getCurrentRegionCount();
- PGO.setCurrentRegionUnreachable();
+ BreakContinueStack.back().BreakCount += CurrentCount;
+ CurrentCount = 0;
RecordNextStmtCount = true;
}
void VisitContinueStmt(const ContinueStmt *S) {
RecordStmtCount(S);
assert(!BreakContinueStack.empty() && "continue stmt not in a loop!");
- BreakContinueStack.back().ContinueCount += PGO.getCurrentRegionCount();
- PGO.setCurrentRegionUnreachable();
+ BreakContinueStack.back().ContinueCount += CurrentCount;
+ CurrentCount = 0;
RecordNextStmtCount = true;
}
void VisitWhileStmt(const WhileStmt *S) {
RecordStmtCount(S);
- // Counter tracks the body of the loop.
- RegionCounter Cnt(PGO, S);
+ uint64_t ParentCount = CurrentCount;
+
BreakContinueStack.push_back(BreakContinue());
// Visit the body region first so the break/continue adjustments can be
// included when visiting the condition.
- Cnt.beginRegion();
- CountMap[S->getBody()] = PGO.getCurrentRegionCount();
+ uint64_t BodyCount = setCount(PGO.getRegionCount(S));
+ CountMap[S->getBody()] = CurrentCount;
Visit(S->getBody());
- Cnt.adjustForControlFlow();
+ uint64_t BackedgeCount = CurrentCount;
// ...then go back and propagate counts through the condition. The count
// at the start of the condition is the sum of the incoming edges,
// the backedge from the end of the loop body, and the edges from
// continue statements.
BreakContinue BC = BreakContinueStack.pop_back_val();
- Cnt.setCurrentRegionCount(Cnt.getParentCount() + Cnt.getAdjustedCount() +
- BC.ContinueCount);
- CountMap[S->getCond()] = PGO.getCurrentRegionCount();
+ uint64_t CondCount =
+ setCount(ParentCount + BackedgeCount + BC.ContinueCount);
+ CountMap[S->getCond()] = CondCount;
Visit(S->getCond());
- Cnt.adjustForControlFlow();
- Cnt.applyAdjustmentsToRegion(BC.BreakCount + BC.ContinueCount);
+ setCount(BC.BreakCount + CondCount - BodyCount);
RecordNextStmtCount = true;
}
void VisitDoStmt(const DoStmt *S) {
RecordStmtCount(S);
- // Counter tracks the body of the loop.
- RegionCounter Cnt(PGO, S);
+ uint64_t LoopCount = PGO.getRegionCount(S);
+
BreakContinueStack.push_back(BreakContinue());
- Cnt.beginRegion(/*AddIncomingFallThrough=*/true);
- CountMap[S->getBody()] = PGO.getCurrentRegionCount();
+ // The count doesn't include the fallthrough from the parent scope. Add it.
+ uint64_t BodyCount = setCount(LoopCount + CurrentCount);
+ CountMap[S->getBody()] = BodyCount;
Visit(S->getBody());
- Cnt.adjustForControlFlow();
+ uint64_t BackedgeCount = CurrentCount;
BreakContinue BC = BreakContinueStack.pop_back_val();
// The count at the start of the condition is equal to the count at the
- // end of the body. The adjusted count does not include either the
- // fall-through count coming into the loop or the continue count, so add
- // both of those separately. This is coincidentally the same equation as
- // with while loops but for different reasons.
- Cnt.setCurrentRegionCount(Cnt.getParentCount() + Cnt.getAdjustedCount() +
- BC.ContinueCount);
- CountMap[S->getCond()] = PGO.getCurrentRegionCount();
+ // end of the body, plus any continues.
+ uint64_t CondCount = setCount(BackedgeCount + BC.ContinueCount);
+ CountMap[S->getCond()] = CondCount;
Visit(S->getCond());
- Cnt.adjustForControlFlow();
- Cnt.applyAdjustmentsToRegion(BC.BreakCount + BC.ContinueCount);
+ setCount(BC.BreakCount + CondCount - LoopCount);
RecordNextStmtCount = true;
}
@@ -403,94 +410,89 @@
RecordStmtCount(S);
if (S->getInit())
Visit(S->getInit());
- // Counter tracks the body of the loop.
- RegionCounter Cnt(PGO, S);
+
+ uint64_t ParentCount = CurrentCount;
+
BreakContinueStack.push_back(BreakContinue());
// Visit the body region first. (This is basically the same as a while
// loop; see further comments in VisitWhileStmt.)
- Cnt.beginRegion();
- CountMap[S->getBody()] = PGO.getCurrentRegionCount();
+ uint64_t BodyCount = setCount(PGO.getRegionCount(S));
+ CountMap[S->getBody()] = BodyCount;
Visit(S->getBody());
- Cnt.adjustForControlFlow();
+ uint64_t BackedgeCount = CurrentCount;
+ BreakContinue BC = BreakContinueStack.pop_back_val();
// The increment is essentially part of the body but it needs to include
// the count for all the continue statements.
if (S->getInc()) {
- Cnt.setCurrentRegionCount(PGO.getCurrentRegionCount() +
- BreakContinueStack.back().ContinueCount);
- CountMap[S->getInc()] = PGO.getCurrentRegionCount();
+ uint64_t IncCount = setCount(BackedgeCount + BC.ContinueCount);
+ CountMap[S->getInc()] = IncCount;
Visit(S->getInc());
- Cnt.adjustForControlFlow();
}
- BreakContinue BC = BreakContinueStack.pop_back_val();
-
// ...then go back and propagate counts through the condition.
+ uint64_t CondCount =
+ setCount(ParentCount + BackedgeCount + BC.ContinueCount);
if (S->getCond()) {
- Cnt.setCurrentRegionCount(Cnt.getParentCount() + Cnt.getAdjustedCount() +
- BC.ContinueCount);
- CountMap[S->getCond()] = PGO.getCurrentRegionCount();
+ CountMap[S->getCond()] = CondCount;
Visit(S->getCond());
- Cnt.adjustForControlFlow();
}
- Cnt.applyAdjustmentsToRegion(BC.BreakCount + BC.ContinueCount);
+ setCount(BC.BreakCount + CondCount - BodyCount);
RecordNextStmtCount = true;
}
void VisitCXXForRangeStmt(const CXXForRangeStmt *S) {
RecordStmtCount(S);
+ Visit(S->getLoopVarStmt());
Visit(S->getRangeStmt());
Visit(S->getBeginEndStmt());
- // Counter tracks the body of the loop.
- RegionCounter Cnt(PGO, S);
+
+ uint64_t ParentCount = CurrentCount;
BreakContinueStack.push_back(BreakContinue());
// Visit the body region first. (This is basically the same as a while
// loop; see further comments in VisitWhileStmt.)
- Cnt.beginRegion();
- CountMap[S->getLoopVarStmt()] = PGO.getCurrentRegionCount();
- Visit(S->getLoopVarStmt());
+ uint64_t BodyCount = setCount(PGO.getRegionCount(S));
+ CountMap[S->getBody()] = BodyCount;
Visit(S->getBody());
- Cnt.adjustForControlFlow();
+ uint64_t BackedgeCount = CurrentCount;
+ BreakContinue BC = BreakContinueStack.pop_back_val();
// The increment is essentially part of the body but it needs to include
// the count for all the continue statements.
- Cnt.setCurrentRegionCount(PGO.getCurrentRegionCount() +
- BreakContinueStack.back().ContinueCount);
- CountMap[S->getInc()] = PGO.getCurrentRegionCount();
+ uint64_t IncCount = setCount(BackedgeCount + BC.ContinueCount);
+ CountMap[S->getInc()] = IncCount;
Visit(S->getInc());
- Cnt.adjustForControlFlow();
-
- BreakContinue BC = BreakContinueStack.pop_back_val();
// ...then go back and propagate counts through the condition.
- Cnt.setCurrentRegionCount(Cnt.getParentCount() + Cnt.getAdjustedCount() +
- BC.ContinueCount);
- CountMap[S->getCond()] = PGO.getCurrentRegionCount();
+ uint64_t CondCount =
+ setCount(ParentCount + BackedgeCount + BC.ContinueCount);
+ CountMap[S->getCond()] = CondCount;
Visit(S->getCond());
- Cnt.adjustForControlFlow();
- Cnt.applyAdjustmentsToRegion(BC.BreakCount + BC.ContinueCount);
+ setCount(BC.BreakCount + CondCount - BodyCount);
RecordNextStmtCount = true;
}
void VisitObjCForCollectionStmt(const ObjCForCollectionStmt *S) {
RecordStmtCount(S);
Visit(S->getElement());
- // Counter tracks the body of the loop.
- RegionCounter Cnt(PGO, S);
+ uint64_t ParentCount = CurrentCount;
BreakContinueStack.push_back(BreakContinue());
- Cnt.beginRegion();
- CountMap[S->getBody()] = PGO.getCurrentRegionCount();
+ // Counter tracks the body of the loop.
+ uint64_t BodyCount = setCount(PGO.getRegionCount(S));
+ CountMap[S->getBody()] = BodyCount;
Visit(S->getBody());
+ uint64_t BackedgeCount = CurrentCount;
BreakContinue BC = BreakContinueStack.pop_back_val();
- Cnt.adjustForControlFlow();
- Cnt.applyAdjustmentsToRegion(BC.BreakCount + BC.ContinueCount);
+
+ setCount(BC.BreakCount + ParentCount + BackedgeCount + BC.ContinueCount -
+ BodyCount);
RecordNextStmtCount = true;
}
void VisitSwitchStmt(const SwitchStmt *S) {
RecordStmtCount(S);
Visit(S->getCond());
- PGO.setCurrentRegionUnreachable();
+ CurrentCount = 0;
BreakContinueStack.push_back(BreakContinue());
Visit(S->getBody());
// If the switch is inside a loop, add the continue counts.
@@ -498,53 +500,45 @@
if (!BreakContinueStack.empty())
BreakContinueStack.back().ContinueCount += BC.ContinueCount;
// Counter tracks the exit block of the switch.
- RegionCounter ExitCnt(PGO, S);
- ExitCnt.beginRegion();
+ setCount(PGO.getRegionCount(S));
RecordNextStmtCount = true;
}
- void VisitCaseStmt(const CaseStmt *S) {
+ void VisitSwitchCase(const SwitchCase *S) {
RecordNextStmtCount = false;
// Counter for this particular case. This counts only jumps from the
// switch header and does not include fallthrough from the case before
// this one.
- RegionCounter Cnt(PGO, S);
- Cnt.beginRegion(/*AddIncomingFallThrough=*/true);
- CountMap[S] = Cnt.getCount();
- RecordNextStmtCount = true;
- Visit(S->getSubStmt());
- }
-
- void VisitDefaultStmt(const DefaultStmt *S) {
- RecordNextStmtCount = false;
- // Counter for this default case. This does not include fallthrough from
- // the previous case.
- RegionCounter Cnt(PGO, S);
- Cnt.beginRegion(/*AddIncomingFallThrough=*/true);
- CountMap[S] = Cnt.getCount();
+ uint64_t CaseCount = PGO.getRegionCount(S);
+ setCount(CurrentCount + CaseCount);
+ // We need the count without fallthrough in the mapping, so it's more useful
+ // for branch probabilities.
+ CountMap[S] = CaseCount;
RecordNextStmtCount = true;
Visit(S->getSubStmt());
}
void VisitIfStmt(const IfStmt *S) {
RecordStmtCount(S);
- // Counter tracks the "then" part of an if statement. The count for
- // the "else" part, if it exists, will be calculated from this counter.
- RegionCounter Cnt(PGO, S);
+ uint64_t ParentCount = CurrentCount;
Visit(S->getCond());
- Cnt.beginRegion();
- CountMap[S->getThen()] = PGO.getCurrentRegionCount();
+ // Counter tracks the "then" part of an if statement. The count for
+ // the "else" part, if it exists, will be calculated from this counter.
+ uint64_t ThenCount = setCount(PGO.getRegionCount(S));
+ CountMap[S->getThen()] = ThenCount;
Visit(S->getThen());
- Cnt.adjustForControlFlow();
+ uint64_t OutCount = CurrentCount;
+ uint64_t ElseCount = ParentCount - ThenCount;
if (S->getElse()) {
- Cnt.beginElseRegion();
- CountMap[S->getElse()] = PGO.getCurrentRegionCount();
+ setCount(ElseCount);
+ CountMap[S->getElse()] = ElseCount;
Visit(S->getElse());
- Cnt.adjustForControlFlow();
- }
- Cnt.applyAdjustmentsToRegion(0);
+ OutCount += CurrentCount;
+ } else
+ OutCount += ElseCount;
+ setCount(OutCount);
RecordNextStmtCount = true;
}
@@ -554,64 +548,60 @@
for (unsigned I = 0, E = S->getNumHandlers(); I < E; ++I)
Visit(S->getHandler(I));
// Counter tracks the continuation block of the try statement.
- RegionCounter Cnt(PGO, S);
- Cnt.beginRegion();
+ setCount(PGO.getRegionCount(S));
RecordNextStmtCount = true;
}
void VisitCXXCatchStmt(const CXXCatchStmt *S) {
RecordNextStmtCount = false;
// Counter tracks the catch statement's handler block.
- RegionCounter Cnt(PGO, S);
- Cnt.beginRegion();
- CountMap[S] = PGO.getCurrentRegionCount();
+ uint64_t CatchCount = setCount(PGO.getRegionCount(S));
+ CountMap[S] = CatchCount;
Visit(S->getHandlerBlock());
}
void VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
RecordStmtCount(E);
- // Counter tracks the "true" part of a conditional operator. The
- // count in the "false" part will be calculated from this counter.
- RegionCounter Cnt(PGO, E);
+ uint64_t ParentCount = CurrentCount;
Visit(E->getCond());
- Cnt.beginRegion();
- CountMap[E->getTrueExpr()] = PGO.getCurrentRegionCount();
+ // Counter tracks the "true" part of a conditional operator. The
+ // count in the "false" part will be calculated from this counter.
+ uint64_t TrueCount = setCount(PGO.getRegionCount(E));
+ CountMap[E->getTrueExpr()] = TrueCount;
Visit(E->getTrueExpr());
- Cnt.adjustForControlFlow();
+ uint64_t OutCount = CurrentCount;
- Cnt.beginElseRegion();
- CountMap[E->getFalseExpr()] = PGO.getCurrentRegionCount();
+ uint64_t FalseCount = setCount(ParentCount - TrueCount);
+ CountMap[E->getFalseExpr()] = FalseCount;
Visit(E->getFalseExpr());
- Cnt.adjustForControlFlow();
+ OutCount += CurrentCount;
- Cnt.applyAdjustmentsToRegion(0);
+ setCount(OutCount);
RecordNextStmtCount = true;
}
void VisitBinLAnd(const BinaryOperator *E) {
RecordStmtCount(E);
- // Counter tracks the right hand side of a logical and operator.
- RegionCounter Cnt(PGO, E);
+ uint64_t ParentCount = CurrentCount;
Visit(E->getLHS());
- Cnt.beginRegion();
- CountMap[E->getRHS()] = PGO.getCurrentRegionCount();
+ // Counter tracks the right hand side of a logical and operator.
+ uint64_t RHSCount = setCount(PGO.getRegionCount(E));
+ CountMap[E->getRHS()] = RHSCount;
Visit(E->getRHS());
- Cnt.adjustForControlFlow();
- Cnt.applyAdjustmentsToRegion(0);
+ setCount(ParentCount + RHSCount - CurrentCount);
RecordNextStmtCount = true;
}
void VisitBinLOr(const BinaryOperator *E) {
RecordStmtCount(E);
- // Counter tracks the right hand side of a logical or operator.
- RegionCounter Cnt(PGO, E);
+ uint64_t ParentCount = CurrentCount;
Visit(E->getLHS());
- Cnt.beginRegion();
- CountMap[E->getRHS()] = PGO.getCurrentRegionCount();
+ // Counter tracks the right hand side of a logical or operator.
+ uint64_t RHSCount = setCount(PGO.getRegionCount(E));
+ CountMap[E->getRHS()] = RHSCount;
Visit(E->getRHS());
- Cnt.adjustForControlFlow();
- Cnt.applyAdjustmentsToRegion(0);
+ setCount(ParentCount + RHSCount - CurrentCount);
RecordNextStmtCount = true;
}
};
@@ -729,7 +719,7 @@
}
void
-CodeGenPGO::emitEmptyCounterMapping(const Decl *D, StringRef FuncName,
+CodeGenPGO::emitEmptyCounterMapping(const Decl *D, StringRef Name,
llvm::GlobalValue::LinkageTypes Linkage) {
if (SkipCoverageMapping)
return;
@@ -749,7 +739,7 @@
if (CoverageMapping.empty())
return;
- setFuncName(FuncName, Linkage);
+ setFuncName(Name, Linkage);
CGM.getCoverageMapping()->addFunctionMappingRecord(
FuncNameVar, FuncName, FunctionHash, CoverageMapping);
}
@@ -783,19 +773,23 @@
// Turn on Cold attribute for cold functions.
// FIXME: 1% is from preliminary tuning on SPEC, it may not be optimal.
Fn->addFnAttr(llvm::Attribute::Cold);
+
+ Fn->setEntryCount(FunctionCount);
}
-void CodeGenPGO::emitCounterIncrement(CGBuilderTy &Builder, unsigned Counter) {
+void CodeGenPGO::emitCounterIncrement(CGBuilderTy &Builder, const Stmt *S) {
if (!CGM.getCodeGenOpts().ProfileInstrGenerate || !RegionCounterMap)
return;
if (!Builder.GetInsertPoint())
return;
+
+ unsigned Counter = (*RegionCounterMap)[S];
auto *I8PtrTy = llvm::Type::getInt8PtrTy(CGM.getLLVMContext());
- Builder.CreateCall4(CGM.getIntrinsic(llvm::Intrinsic::instrprof_increment),
- llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
+ Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::instrprof_increment),
+ {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
Builder.getInt64(FunctionHash),
Builder.getInt32(NumRegionCounters),
- Builder.getInt32(Counter));
+ Builder.getInt32(Counter)});
}
void CodeGenPGO::loadRegionCounts(llvm::IndexedInstrProfReader *PGOReader,
@@ -839,8 +833,8 @@
return Scaled;
}
-llvm::MDNode *CodeGenPGO::createBranchWeights(uint64_t TrueCount,
- uint64_t FalseCount) {
+llvm::MDNode *CodeGenFunction::createProfileWeights(uint64_t TrueCount,
+ uint64_t FalseCount) {
// Check for empty weights.
if (!TrueCount && !FalseCount)
return nullptr;
@@ -853,7 +847,8 @@
scaleBranchWeight(FalseCount, Scale));
}
-llvm::MDNode *CodeGenPGO::createBranchWeights(ArrayRef<uint64_t> Weights) {
+llvm::MDNode *
+CodeGenFunction::createProfileWeights(ArrayRef<uint64_t> Weights) {
// We need at least two elements to create meaningful weights.
if (Weights.size() < 2)
return nullptr;
@@ -875,15 +870,14 @@
return MDHelper.createBranchWeights(ScaledWeights);
}
-llvm::MDNode *CodeGenPGO::createLoopWeights(const Stmt *Cond,
- RegionCounter &Cnt) {
- if (!haveRegionCounts())
+llvm::MDNode *CodeGenFunction::createProfileWeightsForLoop(const Stmt *Cond,
+ uint64_t LoopCount) {
+ if (!PGO.haveRegionCounts())
return nullptr;
- uint64_t LoopCount = Cnt.getCount();
- Optional<uint64_t> CondCount = getStmtCount(Cond);
+ Optional<uint64_t> CondCount = PGO.getStmtCount(Cond);
assert(CondCount.hasValue() && "missing expected loop condition count");
if (*CondCount == 0)
return nullptr;
- return createBranchWeights(LoopCount,
- std::max(*CondCount, LoopCount) - LoopCount);
+ return createProfileWeights(LoopCount,
+ std::max(*CondCount, LoopCount) - LoopCount);
}