SamplePGO: convert callsite samples map key from callsite_location to callsite_location+callee_name
Summary: For iterative SamplePGO, an indirect call can be speculatively promoted to multiple direct calls and get inlined. All these promoted direct calls will share the same callsite location (offset+discriminator). With the current implementation, we cannot distinguish between different promotion candidates and its inlined instance. This patch adds callee_name to the key of the callsite sample map. And added helper functions to get all inlined callee samples for a given callsite location. This helps the profile annotator promote correct targets and inline it before annotation, and ensures all indirect call targets to be annotated correctly.
Reviewers: davidxl, dnovillo
Reviewed By: davidxl
Subscribers: andreadb, llvm-commits
Differential Revision: https://reviews.llvm.org/D31950
llvm-svn: 300240
diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp
index 8493acc..eafdd21 100644
--- a/llvm/lib/ProfileData/SampleProf.cpp
+++ b/llvm/lib/ProfileData/SampleProf.cpp
@@ -129,12 +129,14 @@
OS.indent(Indent);
if (!CallsiteSamples.empty()) {
OS << "Samples collected in inlined callsites {\n";
- SampleSorter<LineLocation, FunctionSamples> SortedCallsiteSamples(
+ SampleSorter<LineLocation, FunctionSamplesMap> SortedCallsiteSamples(
CallsiteSamples);
for (const auto &CS : SortedCallsiteSamples.get()) {
- OS.indent(Indent + 2);
- OS << CS->first << ": inlined callee: " << CS->second.getName() << ": ";
- CS->second.print(OS, Indent + 4);
+ for (const auto &FS : CS->second) {
+ OS.indent(Indent + 2);
+ OS << CS->first << ": inlined callee: " << FS.second.getName() << ": ";
+ FS.second.print(OS, Indent + 4);
+ }
}
OS << "}\n";
} else {
diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp
index 3a8d619..234fe02 100644
--- a/llvm/lib/ProfileData/SampleProfReader.cpp
+++ b/llvm/lib/ProfileData/SampleProfReader.cpp
@@ -211,7 +211,7 @@
InlineStack.pop_back();
}
FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt(
- LineLocation(LineOffset, Discriminator));
+ LineLocation(LineOffset, Discriminator))[FName];
FSamples.setName(FName);
MergeResult(Result, FSamples.addTotalSamples(NumSamples));
InlineStack.push_back(&FSamples);
@@ -363,8 +363,8 @@
if (std::error_code EC = FName.getError())
return EC;
- FunctionSamples &CalleeProfile =
- FProfile.functionSamplesAt(LineLocation(*LineOffset, *Discriminator));
+ FunctionSamples &CalleeProfile = FProfile.functionSamplesAt(
+ LineLocation(*LineOffset, *Discriminator))[*FName];
CalleeProfile.setName(*FName);
if (std::error_code EC = readProfile(CalleeProfile))
return EC;
@@ -636,7 +636,7 @@
uint32_t LineOffset = Offset >> 16;
uint32_t Discriminator = Offset & 0xffff;
FProfile = &CallerProfile->functionSamplesAt(
- LineLocation(LineOffset, Discriminator));
+ LineLocation(LineOffset, Discriminator))[Name];
}
FProfile->setName(Name);
diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp
index e1d6d57..b91b6fb 100644
--- a/llvm/lib/ProfileData/SampleProfWriter.cpp
+++ b/llvm/lib/ProfileData/SampleProfWriter.cpp
@@ -68,20 +68,21 @@
OS << "\n";
}
- SampleSorter<LineLocation, FunctionSamples> SortedCallsiteSamples(
+ SampleSorter<LineLocation, FunctionSamplesMap> SortedCallsiteSamples(
S.getCallsiteSamples());
Indent += 1;
- for (const auto &I : SortedCallsiteSamples.get()) {
- LineLocation Loc = I->first;
- const FunctionSamples &CalleeSamples = I->second;
- OS.indent(Indent);
- if (Loc.Discriminator == 0)
- OS << Loc.LineOffset << ": ";
- else
- OS << Loc.LineOffset << "." << Loc.Discriminator << ": ";
- if (std::error_code EC = write(CalleeSamples))
- return EC;
- }
+ for (const auto &I : SortedCallsiteSamples.get())
+ for (const auto &FS : I->second) {
+ LineLocation Loc = I->first;
+ const FunctionSamples &CalleeSamples = FS.second;
+ OS.indent(Indent);
+ if (Loc.Discriminator == 0)
+ OS << Loc.LineOffset << ": ";
+ else
+ OS << Loc.LineOffset << "." << Loc.Discriminator << ": ";
+ if (std::error_code EC = write(CalleeSamples))
+ return EC;
+ }
Indent -= 1;
return sampleprof_error::success;
@@ -109,11 +110,12 @@
}
// Recursively add all the names for inlined callsites.
- for (const auto &J : S.getCallsiteSamples()) {
- const FunctionSamples &CalleeSamples = J.second;
- addName(CalleeSamples.getName());
- addNames(CalleeSamples);
- }
+ for (const auto &J : S.getCallsiteSamples())
+ for (const auto &FS : J.second) {
+ const FunctionSamples &CalleeSamples = FS.second;
+ addName(CalleeSamples.getName());
+ addNames(CalleeSamples);
+ }
}
std::error_code SampleProfileWriterBinary::writeHeader(
@@ -187,14 +189,15 @@
// Recursively emit all the callsite samples.
encodeULEB128(S.getCallsiteSamples().size(), OS);
- for (const auto &J : S.getCallsiteSamples()) {
- LineLocation Loc = J.first;
- const FunctionSamples &CalleeSamples = J.second;
- encodeULEB128(Loc.LineOffset, OS);
- encodeULEB128(Loc.Discriminator, OS);
- if (std::error_code EC = writeBody(CalleeSamples))
- return EC;
- }
+ for (const auto &J : S.getCallsiteSamples())
+ for (const auto &FS : J.second) {
+ LineLocation Loc = J.first;
+ const FunctionSamples &CalleeSamples = FS.second;
+ encodeULEB128(Loc.LineOffset, OS);
+ encodeULEB128(Loc.Discriminator, OS);
+ if (std::error_code EC = writeBody(CalleeSamples))
+ return EC;
+ }
return sampleprof_error::success;
}