Update entry count for cold calls
Summary:
Profile sample files include the number of times each entry or inlined
call site is sampled. This is translated into the entry count metadta
on functions.
When sample data is being read, if a call site that was inlined
in the sample program is considered cold and not inlined, then
the entry count of the out-of-line functions does not reflect
the current compilation.
In this patch, we note call sites where the function was not inlined
and as a last action of the sample profile loading, we update the
called function's entry count to reflect the calls from these
call sites which are not included in the profile file.
Reviewers: danielcdh, wmi, Kader, modocache
Reviewed By: wmi
Subscribers: davidxl, eraman, llvm-commits
Differential Revision: https://reviews.llvm.org/D52845
llvm-svn: 352001
diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp
index 47ee698..bc2ceec 100644
--- a/llvm/lib/Transforms/IPO/SampleProfile.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp
@@ -318,6 +318,14 @@
/// Optimization Remark Emitter used to emit diagnostic remarks.
OptimizationRemarkEmitter *ORE = nullptr;
+
+ // Information recorded when we declined to inline a call site
+ // because we have determined it is too cold is accumulated for
+ // each callee function. Initially this is just the entry count.
+ struct NotInlinedProfileInfo {
+ uint64_t entryCount;
+ };
+ DenseMap<Function *, NotInlinedProfileInfo> notInlinedCallInfo;
};
class SampleProfileLoaderLegacyPass : public ModulePass {
@@ -778,6 +786,8 @@
bool SampleProfileLoader::inlineHotFunctions(
Function &F, DenseSet<GlobalValue::GUID> &InlinedGUIDs) {
DenseSet<Instruction *> PromotedInsns;
+
+ DenseMap<Instruction *, const FunctionSamples *> localNotInlinedCallSites;
bool Changed = false;
while (true) {
bool LocalChanged = false;
@@ -790,6 +800,8 @@
if ((isa<CallInst>(I) || isa<InvokeInst>(I)) &&
!isa<IntrinsicInst>(I) && (FS = findCalleeFunctionSamples(I))) {
Candidates.push_back(&I);
+ if (FS->getEntrySamples() > 0)
+ localNotInlinedCallSites.try_emplace(&I, FS);
if (callsiteIsHot(FS, PSI))
Hot = true;
}
@@ -835,8 +847,10 @@
PromotedInsns.insert(I);
// If profile mismatches, we should not attempt to inline DI.
if ((isa<CallInst>(DI) || isa<InvokeInst>(DI)) &&
- inlineCallInstruction(DI))
+ inlineCallInstruction(DI)) {
+ localNotInlinedCallSites.erase(I);
LocalChanged = true;
+ }
} else {
LLVM_DEBUG(dbgs()
<< "\nFailed to promote indirect call to "
@@ -845,8 +859,10 @@
}
} else if (CalledFunction && CalledFunction->getSubprogram() &&
!CalledFunction->isDeclaration()) {
- if (inlineCallInstruction(I))
+ if (inlineCallInstruction(I)) {
+ localNotInlinedCallSites.erase(I);
LocalChanged = true;
+ }
} else if (IsThinLTOPreLink) {
findCalleeFunctionSamples(*I)->findInlinedFunctions(
InlinedGUIDs, F.getParent(), PSI->getOrCompHotCountThreshold());
@@ -858,6 +874,18 @@
break;
}
}
+
+ // Accumulate not inlined callsite information into notInlinedSamples
+ for (const auto &Pair : localNotInlinedCallSites) {
+ Instruction *I = Pair.getFirst();
+ Function *Callee = CallSite(I).getCalledFunction();
+ if (!Callee || Callee->isDeclaration())
+ continue;
+ const FunctionSamples *FS = Pair.getSecond();
+ auto pair =
+ notInlinedCallInfo.try_emplace(Callee, NotInlinedProfileInfo{0});
+ pair.first->second.entryCount += FS->getEntrySamples();
+ }
return Changed;
}
@@ -1600,6 +1628,12 @@
clearFunctionData();
retval |= runOnFunction(F, AM);
}
+
+ // Account for cold calls not inlined....
+ for (const std::pair<Function *, NotInlinedProfileInfo> &pair :
+ notInlinedCallInfo)
+ updateProfileCallee(pair.first, pair.second.entryCount);
+
return retval;
}