Restore "[ThinLTO] Emit individual index files for distributed backends"
This restores commit r268627:
Summary:
When launching ThinLTO backends in a distributed build (currently
supported in gold via the thinlto-index-only plugin option), emit
an individual index file for each backend process as described here:
http://lists.llvm.org/pipermail/llvm-dev/2016-April/098272.html
...
Differential Revision: http://reviews.llvm.org/D19556
Address msan failures by avoiding std::prev on map.end(), the
theory is that this is causing issues due to some known UB problems
in __tree.
llvm-svn: 269059
diff --git a/llvm/tools/llvm-lto/llvm-lto.cpp b/llvm/tools/llvm-lto/llvm-lto.cpp
index abede09..395988c 100644
--- a/llvm/tools/llvm-lto/llvm-lto.cpp
+++ b/llvm/tools/llvm-lto/llvm-lto.cpp
@@ -66,6 +66,7 @@
enum ThinLTOModes {
THINLINK,
+ THINDISTRIBUTE,
THINPROMOTE,
THINIMPORT,
THININTERNALIZE,
@@ -80,6 +81,8 @@
clEnumValN(
THINLINK, "thinlink",
"ThinLink: produces the index by linking only the summaries."),
+ clEnumValN(THINDISTRIBUTE, "distributedindexes",
+ "Produces individual indexes for distributed backends."),
clEnumValN(THINPROMOTE, "promote",
"Perform pre-import promotion (requires -thinlto-index)."),
clEnumValN(THINIMPORT, "import", "Perform both promotion and "
@@ -354,6 +357,8 @@
switch (ThinLTOMode) {
case THINLINK:
return thinLink();
+ case THINDISTRIBUTE:
+ return distributedIndexes();
case THINPROMOTE:
return promote();
case THINIMPORT:
@@ -396,6 +401,36 @@
return;
}
+ /// Load the combined index from disk, then compute and generate
+ /// individual index files suitable for ThinLTO distributed backend builds
+ /// on the files mentioned on the command line (these must match the index
+ /// content).
+ void distributedIndexes() {
+ if (InputFilenames.size() != 1 && !OutputFilename.empty())
+ report_fatal_error("Can't handle a single output filename and multiple "
+ "input files, do not provide an output filename and "
+ "the output files will be suffixed from the input "
+ "ones.");
+
+ auto Index = loadCombinedIndex();
+ for (auto &Filename : InputFilenames) {
+ // Build a map of module to the GUIDs and summary objects that should
+ // be written to its index.
+ std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
+ ThinLTOCodeGenerator::gatherImportedSummariesForModule(
+ Filename, *Index, ModuleToSummariesForIndex);
+
+ std::string OutputName = OutputFilename;
+ if (OutputName.empty()) {
+ OutputName = Filename + ".thinlto.bc";
+ }
+ std::error_code EC;
+ raw_fd_ostream OS(OutputName, EC, sys::fs::OpenFlags::F_None);
+ error(EC, "error opening the file '" + OutputName + "'");
+ WriteIndexToFile(*Index, OS, &ModuleToSummariesForIndex);
+ }
+ }
+
/// Load the combined index from disk, then load every file referenced by
/// the index and add them to the generator, finally perform the promotion
/// on the files mentioned on the command line (these must match the index