Mehdi Amini | 42418ab | 2015-11-24 06:07:49 +0000 | [diff] [blame] | 1 | //===- FunctionImport.cpp - ThinLTO Summary-based Function Import ---------===// |
| 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
| 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | // |
| 10 | // This file implements Function import based on summaries. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "llvm/Transforms/IPO/FunctionImport.h" |
| 15 | |
| 16 | #include "llvm/ADT/StringSet.h" |
| 17 | #include "llvm/IR/AutoUpgrade.h" |
| 18 | #include "llvm/IR/DiagnosticPrinter.h" |
| 19 | #include "llvm/IR/IntrinsicInst.h" |
| 20 | #include "llvm/IR/Module.h" |
| 21 | #include "llvm/IRReader/IRReader.h" |
| 22 | #include "llvm/Linker/Linker.h" |
| 23 | #include "llvm/Object/FunctionIndexObjectFile.h" |
| 24 | #include "llvm/Support/CommandLine.h" |
| 25 | #include "llvm/Support/Debug.h" |
| 26 | #include "llvm/Support/SourceMgr.h" |
| 27 | using namespace llvm; |
| 28 | |
| 29 | #define DEBUG_TYPE "function-import" |
| 30 | |
Teresa Johnson | 3930361 | 2015-11-24 22:55:46 +0000 | [diff] [blame] | 31 | /// Limit on instruction count of imported functions. |
| 32 | static cl::opt<unsigned> ImportInstrLimit( |
| 33 | "import-instr-limit", cl::init(100), cl::Hidden, cl::value_desc("N"), |
| 34 | cl::desc("Only import functions with less than N instructions")); |
| 35 | |
Mehdi Amini | 42418ab | 2015-11-24 06:07:49 +0000 | [diff] [blame] | 36 | // Load lazily a module from \p FileName in \p Context. |
| 37 | static std::unique_ptr<Module> loadFile(const std::string &FileName, |
| 38 | LLVMContext &Context) { |
| 39 | SMDiagnostic Err; |
| 40 | DEBUG(dbgs() << "Loading '" << FileName << "'\n"); |
| 41 | std::unique_ptr<Module> Result = getLazyIRFileModule(FileName, Err, Context); |
| 42 | if (!Result) { |
| 43 | Err.print("function-import", errs()); |
| 44 | return nullptr; |
| 45 | } |
| 46 | |
| 47 | Result->materializeMetadata(); |
| 48 | UpgradeDebugInfo(*Result); |
| 49 | |
| 50 | return Result; |
| 51 | } |
| 52 | |
Teresa Johnson | d450da3 | 2015-11-24 21:15:19 +0000 | [diff] [blame] | 53 | /// Walk through the instructions in \p F looking for external |
| 54 | /// calls not already in the \p CalledFunctions set. If any are |
| 55 | /// found they are added to the \p Worklist for importing. |
| 56 | static void findExternalCalls(const Function &F, StringSet<> &CalledFunctions, |
| 57 | SmallVector<StringRef, 64> &Worklist) { |
| 58 | for (auto &BB : F) { |
| 59 | for (auto &I : BB) { |
| 60 | if (isa<CallInst>(I)) { |
Teresa Johnson | d450da3 | 2015-11-24 21:15:19 +0000 | [diff] [blame] | 61 | auto CalledFunction = cast<CallInst>(I).getCalledFunction(); |
| 62 | // Insert any new external calls that have not already been |
| 63 | // added to set/worklist. |
| 64 | if (CalledFunction && CalledFunction->hasName() && |
| 65 | CalledFunction->isDeclaration() && |
| 66 | !CalledFunctions.count(CalledFunction->getName())) { |
| 67 | CalledFunctions.insert(CalledFunction->getName()); |
| 68 | Worklist.push_back(CalledFunction->getName()); |
| 69 | } |
| 70 | } |
| 71 | } |
| 72 | } |
| 73 | } |
| 74 | |
Mehdi Amini | c8c5517 | 2015-12-03 02:37:33 +0000 | [diff] [blame] | 75 | // Helper function: given a worklist and an index, will process all the worklist |
| 76 | // and import them based on the summary information |
Mehdi Amini | 311fef6 | 2015-12-03 02:58:14 +0000 | [diff] [blame] | 77 | static unsigned ProcessImportWorklist( |
| 78 | Module &DestModule, SmallVector<StringRef, 64> &Worklist, |
| 79 | StringSet<> &CalledFunctions, Linker &TheLinker, |
| 80 | const FunctionInfoIndex &Index, |
Mehdi Amini | d16c806 | 2015-12-08 22:39:40 +0000 | [diff] [blame^] | 81 | std::function<std::unique_ptr<Module>(StringRef FileName)> & |
| 82 | LazyModuleLoader) { |
Mehdi Amini | c8c5517 | 2015-12-03 02:37:33 +0000 | [diff] [blame] | 83 | unsigned ImportCount = 0; |
Mehdi Amini | 42418ab | 2015-11-24 06:07:49 +0000 | [diff] [blame] | 84 | while (!Worklist.empty()) { |
| 85 | auto CalledFunctionName = Worklist.pop_back_val(); |
| 86 | DEBUG(dbgs() << "Process import for " << CalledFunctionName << "\n"); |
| 87 | |
| 88 | // Try to get a summary for this function call. |
| 89 | auto InfoList = Index.findFunctionInfoList(CalledFunctionName); |
| 90 | if (InfoList == Index.end()) { |
| 91 | DEBUG(dbgs() << "No summary for " << CalledFunctionName |
| 92 | << " Ignoring.\n"); |
| 93 | continue; |
| 94 | } |
| 95 | assert(!InfoList->second.empty() && "No summary, error at import?"); |
| 96 | |
| 97 | // Comdat can have multiple entries, FIXME: what do we do with them? |
| 98 | auto &Info = InfoList->second[0]; |
| 99 | assert(Info && "Nullptr in list, error importing summaries?\n"); |
| 100 | |
| 101 | auto *Summary = Info->functionSummary(); |
| 102 | if (!Summary) { |
| 103 | // FIXME: in case we are lazyloading summaries, we can do it now. |
Teresa Johnson | 430110c | 2015-12-01 17:12:10 +0000 | [diff] [blame] | 104 | DEBUG(dbgs() << "Missing summary for " << CalledFunctionName |
| 105 | << ", error at import?\n"); |
Mehdi Amini | 42418ab | 2015-11-24 06:07:49 +0000 | [diff] [blame] | 106 | llvm_unreachable("Missing summary"); |
| 107 | } |
| 108 | |
Teresa Johnson | 3930361 | 2015-11-24 22:55:46 +0000 | [diff] [blame] | 109 | if (Summary->instCount() > ImportInstrLimit) { |
Teresa Johnson | 430110c | 2015-12-01 17:12:10 +0000 | [diff] [blame] | 110 | DEBUG(dbgs() << "Skip import of " << CalledFunctionName << " with " |
| 111 | << Summary->instCount() << " instructions (limit " |
| 112 | << ImportInstrLimit << ")\n"); |
Teresa Johnson | 3930361 | 2015-11-24 22:55:46 +0000 | [diff] [blame] | 113 | continue; |
| 114 | } |
| 115 | |
Mehdi Amini | 42418ab | 2015-11-24 06:07:49 +0000 | [diff] [blame] | 116 | // Get the module path from the summary. |
| 117 | auto FileName = Summary->modulePath(); |
| 118 | DEBUG(dbgs() << "Importing " << CalledFunctionName << " from " << FileName |
| 119 | << "\n"); |
| 120 | |
Mehdi Amini | d16c806 | 2015-12-08 22:39:40 +0000 | [diff] [blame^] | 121 | // Get the module for the import |
| 122 | auto SrcModule = LazyModuleLoader(FileName); |
| 123 | assert(&SrcModule->getContext() == &DestModule.getContext()); |
Mehdi Amini | 42418ab | 2015-11-24 06:07:49 +0000 | [diff] [blame] | 124 | |
| 125 | // The function that we will import! |
Mehdi Amini | d16c806 | 2015-12-08 22:39:40 +0000 | [diff] [blame^] | 126 | GlobalValue *SGV = SrcModule->getNamedValue(CalledFunctionName); |
Teresa Johnson | 130de7a | 2015-11-24 19:55:04 +0000 | [diff] [blame] | 127 | StringRef ImportFunctionName = CalledFunctionName; |
| 128 | if (!SGV) { |
Mehdi Amini | c8c5517 | 2015-12-03 02:37:33 +0000 | [diff] [blame] | 129 | // Might be local in source Module, promoted/renamed in DestModule. |
Teresa Johnson | 130de7a | 2015-11-24 19:55:04 +0000 | [diff] [blame] | 130 | std::pair<StringRef, StringRef> Split = |
| 131 | CalledFunctionName.split(".llvm."); |
Mehdi Amini | d16c806 | 2015-12-08 22:39:40 +0000 | [diff] [blame^] | 132 | SGV = SrcModule->getNamedValue(Split.first); |
Teresa Johnson | 130de7a | 2015-11-24 19:55:04 +0000 | [diff] [blame] | 133 | #ifndef NDEBUG |
| 134 | // Assert that Split.second is module id |
| 135 | uint64_t ModuleId; |
| 136 | assert(!Split.second.getAsInteger(10, ModuleId)); |
| 137 | assert(ModuleId == Index.getModuleId(FileName)); |
| 138 | #endif |
| 139 | } |
Mehdi Amini | 42418ab | 2015-11-24 06:07:49 +0000 | [diff] [blame] | 140 | Function *F = dyn_cast<Function>(SGV); |
| 141 | if (!F && isa<GlobalAlias>(SGV)) { |
| 142 | auto *SGA = dyn_cast<GlobalAlias>(SGV); |
| 143 | F = dyn_cast<Function>(SGA->getBaseObject()); |
Teresa Johnson | 130de7a | 2015-11-24 19:55:04 +0000 | [diff] [blame] | 144 | ImportFunctionName = F->getName(); |
Mehdi Amini | 42418ab | 2015-11-24 06:07:49 +0000 | [diff] [blame] | 145 | } |
| 146 | if (!F) { |
| 147 | errs() << "Can't load function '" << CalledFunctionName << "' in Module '" |
| 148 | << FileName << "', error in the summary?\n"; |
| 149 | llvm_unreachable("Can't load function in Module"); |
| 150 | } |
| 151 | |
Teresa Johnson | 1762665 | 2015-11-24 16:10:43 +0000 | [diff] [blame] | 152 | // We cannot import weak_any functions/aliases without possibly affecting |
| 153 | // the order they are seen and selected by the linker, changing program |
Mehdi Amini | 42418ab | 2015-11-24 06:07:49 +0000 | [diff] [blame] | 154 | // semantics. |
Teresa Johnson | 1762665 | 2015-11-24 16:10:43 +0000 | [diff] [blame] | 155 | if (SGV->hasWeakAnyLinkage()) { |
| 156 | DEBUG(dbgs() << "Ignoring import request for weak-any " |
| 157 | << (isa<Function>(SGV) ? "function " : "alias ") |
Mehdi Amini | 42418ab | 2015-11-24 06:07:49 +0000 | [diff] [blame] | 158 | << CalledFunctionName << " from " << FileName << "\n"); |
| 159 | continue; |
| 160 | } |
| 161 | |
| 162 | // Link in the specified function. |
Mehdi Amini | ffe2e4a | 2015-12-02 04:34:28 +0000 | [diff] [blame] | 163 | DenseSet<const GlobalValue *> FunctionsToImport; |
| 164 | FunctionsToImport.insert(F); |
Mehdi Amini | d16c806 | 2015-12-08 22:39:40 +0000 | [diff] [blame^] | 165 | if (TheLinker.linkInModule(*SrcModule, Linker::Flags::None, &Index, |
Mehdi Amini | 311fef6 | 2015-12-03 02:58:14 +0000 | [diff] [blame] | 166 | &FunctionsToImport)) |
Mehdi Amini | 42418ab | 2015-11-24 06:07:49 +0000 | [diff] [blame] | 167 | report_fatal_error("Function Import: link error"); |
| 168 | |
Teresa Johnson | 130de7a | 2015-11-24 19:55:04 +0000 | [diff] [blame] | 169 | // Process the newly imported function and add callees to the worklist. |
Mehdi Amini | c8c5517 | 2015-12-03 02:37:33 +0000 | [diff] [blame] | 170 | GlobalValue *NewGV = DestModule.getNamedValue(ImportFunctionName); |
Teresa Johnson | 130de7a | 2015-11-24 19:55:04 +0000 | [diff] [blame] | 171 | assert(NewGV); |
| 172 | Function *NewF = dyn_cast<Function>(NewGV); |
| 173 | assert(NewF); |
Teresa Johnson | d450da3 | 2015-11-24 21:15:19 +0000 | [diff] [blame] | 174 | findExternalCalls(*NewF, CalledFunctions, Worklist); |
Mehdi Amini | c8c5517 | 2015-12-03 02:37:33 +0000 | [diff] [blame] | 175 | ++ImportCount; |
Mehdi Amini | 42418ab | 2015-11-24 06:07:49 +0000 | [diff] [blame] | 176 | } |
Mehdi Amini | c8c5517 | 2015-12-03 02:37:33 +0000 | [diff] [blame] | 177 | return ImportCount; |
| 178 | } |
Mehdi Amini | ffe2e4a | 2015-12-02 04:34:28 +0000 | [diff] [blame] | 179 | |
Mehdi Amini | c8c5517 | 2015-12-03 02:37:33 +0000 | [diff] [blame] | 180 | // Automatically import functions in Module \p DestModule based on the summaries |
| 181 | // index. |
| 182 | // |
| 183 | // The current implementation imports every called functions that exists in the |
| 184 | // summaries index. |
| 185 | bool FunctionImporter::importFunctions(Module &DestModule) { |
Mehdi Amini | 311fef6 | 2015-12-03 02:58:14 +0000 | [diff] [blame] | 186 | DEBUG(errs() << "Starting import for Module " |
| 187 | << DestModule.getModuleIdentifier() << "\n"); |
Mehdi Amini | c8c5517 | 2015-12-03 02:37:33 +0000 | [diff] [blame] | 188 | unsigned ImportedCount = 0; |
| 189 | |
| 190 | /// First step is collecting the called external functions. |
| 191 | StringSet<> CalledFunctions; |
| 192 | SmallVector<StringRef, 64> Worklist; |
| 193 | for (auto &F : DestModule) { |
| 194 | if (F.isDeclaration() || F.hasFnAttribute(Attribute::OptimizeNone)) |
| 195 | continue; |
| 196 | findExternalCalls(F, CalledFunctions, Worklist); |
| 197 | } |
| 198 | if (Worklist.empty()) |
| 199 | return false; |
| 200 | |
| 201 | /// Second step: for every call to an external function, try to import it. |
| 202 | |
| 203 | // Linker that will be used for importing function |
| 204 | Linker TheLinker(DestModule, DiagnosticHandler); |
| 205 | |
Mehdi Amini | 311fef6 | 2015-12-03 02:58:14 +0000 | [diff] [blame] | 206 | ImportedCount += ProcessImportWorklist(DestModule, Worklist, CalledFunctions, |
Mehdi Amini | d16c806 | 2015-12-08 22:39:40 +0000 | [diff] [blame^] | 207 | TheLinker, Index, ModuleLoader); |
Mehdi Amini | c8c5517 | 2015-12-03 02:37:33 +0000 | [diff] [blame] | 208 | |
| 209 | DEBUG(errs() << "Imported " << ImportedCount << " functions for Module " |
| 210 | << DestModule.getModuleIdentifier() << "\n"); |
| 211 | return ImportedCount; |
Mehdi Amini | 42418ab | 2015-11-24 06:07:49 +0000 | [diff] [blame] | 212 | } |
| 213 | |
| 214 | /// Summary file to use for function importing when using -function-import from |
| 215 | /// the command line. |
| 216 | static cl::opt<std::string> |
| 217 | SummaryFile("summary-file", |
| 218 | cl::desc("The summary file to use for function importing.")); |
| 219 | |
| 220 | static void diagnosticHandler(const DiagnosticInfo &DI) { |
| 221 | raw_ostream &OS = errs(); |
| 222 | DiagnosticPrinterRawOStream DP(OS); |
| 223 | DI.print(DP); |
| 224 | OS << '\n'; |
| 225 | } |
| 226 | |
| 227 | /// Parse the function index out of an IR file and return the function |
| 228 | /// index object if found, or nullptr if not. |
| 229 | static std::unique_ptr<FunctionInfoIndex> |
| 230 | getFunctionIndexForFile(StringRef Path, std::string &Error, |
| 231 | DiagnosticHandlerFunction DiagnosticHandler) { |
| 232 | std::unique_ptr<MemoryBuffer> Buffer; |
| 233 | ErrorOr<std::unique_ptr<MemoryBuffer>> BufferOrErr = |
| 234 | MemoryBuffer::getFile(Path); |
| 235 | if (std::error_code EC = BufferOrErr.getError()) { |
| 236 | Error = EC.message(); |
| 237 | return nullptr; |
| 238 | } |
| 239 | Buffer = std::move(BufferOrErr.get()); |
| 240 | ErrorOr<std::unique_ptr<object::FunctionIndexObjectFile>> ObjOrErr = |
| 241 | object::FunctionIndexObjectFile::create(Buffer->getMemBufferRef(), |
| 242 | DiagnosticHandler); |
| 243 | if (std::error_code EC = ObjOrErr.getError()) { |
| 244 | Error = EC.message(); |
| 245 | return nullptr; |
| 246 | } |
| 247 | return (*ObjOrErr)->takeIndex(); |
| 248 | } |
| 249 | |
| 250 | /// Pass that performs cross-module function import provided a summary file. |
| 251 | class FunctionImportPass : public ModulePass { |
Teresa Johnson | 5fcbdb7 | 2015-12-07 19:21:11 +0000 | [diff] [blame] | 252 | /// Optional function summary index to use for importing, otherwise |
| 253 | /// the summary-file option must be specified. |
| 254 | FunctionInfoIndex *Index; |
Mehdi Amini | 42418ab | 2015-11-24 06:07:49 +0000 | [diff] [blame] | 255 | |
| 256 | public: |
| 257 | /// Pass identification, replacement for typeid |
| 258 | static char ID; |
| 259 | |
Teresa Johnson | 5fcbdb7 | 2015-12-07 19:21:11 +0000 | [diff] [blame] | 260 | /// Specify pass name for debug output |
| 261 | const char *getPassName() const override { |
| 262 | return "Function Importing"; |
| 263 | } |
| 264 | |
| 265 | explicit FunctionImportPass(FunctionInfoIndex *Index = nullptr) |
| 266 | : ModulePass(ID), Index(Index) {} |
Mehdi Amini | 42418ab | 2015-11-24 06:07:49 +0000 | [diff] [blame] | 267 | |
| 268 | bool runOnModule(Module &M) override { |
Teresa Johnson | 5fcbdb7 | 2015-12-07 19:21:11 +0000 | [diff] [blame] | 269 | if (SummaryFile.empty() && !Index) |
| 270 | report_fatal_error("error: -function-import requires -summary-file or " |
| 271 | "file from frontend\n"); |
| 272 | std::unique_ptr<FunctionInfoIndex> IndexPtr; |
| 273 | if (!SummaryFile.empty()) { |
| 274 | if (Index) |
| 275 | report_fatal_error("error: -summary-file and index from frontend\n"); |
| 276 | std::string Error; |
| 277 | IndexPtr = getFunctionIndexForFile(SummaryFile, Error, diagnosticHandler); |
| 278 | if (!IndexPtr) { |
| 279 | errs() << "Error loading file '" << SummaryFile << "': " << Error |
| 280 | << "\n"; |
| 281 | return false; |
| 282 | } |
| 283 | Index = IndexPtr.get(); |
Mehdi Amini | 42418ab | 2015-11-24 06:07:49 +0000 | [diff] [blame] | 284 | } |
| 285 | |
| 286 | // Perform the import now. |
Mehdi Amini | d16c806 | 2015-12-08 22:39:40 +0000 | [diff] [blame^] | 287 | auto ModuleLoader = [&M](StringRef Identifier) { |
| 288 | return loadFile(Identifier, M.getContext()); |
| 289 | }; |
| 290 | FunctionImporter Importer(*Index, diagnosticHandler, ModuleLoader); |
Mehdi Amini | 42418ab | 2015-11-24 06:07:49 +0000 | [diff] [blame] | 291 | return Importer.importFunctions(M); |
| 292 | |
| 293 | return false; |
| 294 | } |
| 295 | }; |
| 296 | |
| 297 | char FunctionImportPass::ID = 0; |
| 298 | INITIALIZE_PASS_BEGIN(FunctionImportPass, "function-import", |
| 299 | "Summary Based Function Import", false, false) |
| 300 | INITIALIZE_PASS_END(FunctionImportPass, "function-import", |
| 301 | "Summary Based Function Import", false, false) |
| 302 | |
| 303 | namespace llvm { |
Teresa Johnson | 5fcbdb7 | 2015-12-07 19:21:11 +0000 | [diff] [blame] | 304 | Pass *createFunctionImportPass(FunctionInfoIndex *Index = nullptr) { |
| 305 | return new FunctionImportPass(Index); |
| 306 | } |
Mehdi Amini | 42418ab | 2015-11-24 06:07:49 +0000 | [diff] [blame] | 307 | } |