blob: 69c331b3ce0be404c337bcbe1569e00d829c8f62 [file] [log] [blame]
Douglas Gregora6b00fc2013-01-23 22:38:11 +00001//===--- GlobalModuleIndex.cpp - Global Module Index ------------*- C++ -*-===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the GlobalModuleIndex class.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ASTReaderInternals.h"
15#include "clang/Basic/FileManager.h"
Stephen Hines6bcf27b2014-05-29 04:14:42 -070016#include "clang/Lex/HeaderSearch.h"
Douglas Gregora6b00fc2013-01-23 22:38:11 +000017#include "clang/Serialization/ASTBitCodes.h"
18#include "clang/Serialization/GlobalModuleIndex.h"
Douglas Gregorfa69fc12013-03-22 18:50:14 +000019#include "clang/Serialization/Module.h"
Douglas Gregora6b00fc2013-01-23 22:38:11 +000020#include "llvm/ADT/DenseMap.h"
21#include "llvm/ADT/MapVector.h"
22#include "llvm/ADT/SmallString.h"
23#include "llvm/ADT/StringExtras.h"
24#include "llvm/Bitcode/BitstreamReader.h"
25#include "llvm/Bitcode/BitstreamWriter.h"
Douglas Gregor51001352013-01-23 22:45:24 +000026#include "llvm/Support/FileSystem.h"
Douglas Gregora6b00fc2013-01-23 22:38:11 +000027#include "llvm/Support/LockFileManager.h"
28#include "llvm/Support/MemoryBuffer.h"
Stephen Hines6bcf27b2014-05-29 04:14:42 -070029#include "llvm/Support/OnDiskHashTable.h"
Rafael Espindola8229d222013-06-11 22:15:02 +000030#include "llvm/Support/Path.h"
NAKAMURA Takumi1d321332013-01-25 01:47:07 +000031#include <cstdio>
Douglas Gregora6b00fc2013-01-23 22:38:11 +000032using namespace clang;
33using namespace serialization;
34
35//----------------------------------------------------------------------------//
36// Shared constants
37//----------------------------------------------------------------------------//
38namespace {
39 enum {
40 /// \brief The block containing the index.
41 GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID
42 };
43
44 /// \brief Describes the record types in the index.
45 enum IndexRecordTypes {
46 /// \brief Contains version information and potentially other metadata,
47 /// used to determine if we can read this global index file.
Douglas Gregor1a49d972013-01-25 01:03:03 +000048 INDEX_METADATA,
Douglas Gregora6b00fc2013-01-23 22:38:11 +000049 /// \brief Describes a module, including its file name and dependencies.
50 MODULE,
51 /// \brief The index for identifiers.
52 IDENTIFIER_INDEX
53 };
54}
55
56/// \brief The name of the global index file.
57static const char * const IndexFileName = "modules.idx";
58
59/// \brief The global index file version.
60static const unsigned CurrentVersion = 1;
61
62//----------------------------------------------------------------------------//
Douglas Gregor1a49d972013-01-25 01:03:03 +000063// Global module index reader.
64//----------------------------------------------------------------------------//
65
66namespace {
67
68/// \brief Trait used to read the identifier index from the on-disk hash
69/// table.
70class IdentifierIndexReaderTrait {
71public:
72 typedef StringRef external_key_type;
73 typedef StringRef internal_key_type;
74 typedef SmallVector<unsigned, 2> data_type;
Stephen Hines6bcf27b2014-05-29 04:14:42 -070075 typedef unsigned hash_value_type;
76 typedef unsigned offset_type;
Douglas Gregor1a49d972013-01-25 01:03:03 +000077
78 static bool EqualKey(const internal_key_type& a, const internal_key_type& b) {
79 return a == b;
80 }
81
Stephen Hines6bcf27b2014-05-29 04:14:42 -070082 static hash_value_type ComputeHash(const internal_key_type& a) {
Douglas Gregor1a49d972013-01-25 01:03:03 +000083 return llvm::HashString(a);
84 }
85
86 static std::pair<unsigned, unsigned>
87 ReadKeyDataLength(const unsigned char*& d) {
Stephen Hines651f13c2014-04-23 16:59:28 -070088 using namespace llvm::support;
89 unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d);
90 unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d);
Douglas Gregor1a49d972013-01-25 01:03:03 +000091 return std::make_pair(KeyLen, DataLen);
92 }
93
94 static const internal_key_type&
95 GetInternalKey(const external_key_type& x) { return x; }
96
97 static const external_key_type&
98 GetExternalKey(const internal_key_type& x) { return x; }
99
100 static internal_key_type ReadKey(const unsigned char* d, unsigned n) {
101 return StringRef((const char *)d, n);
102 }
103
104 static data_type ReadData(const internal_key_type& k,
105 const unsigned char* d,
106 unsigned DataLen) {
Stephen Hines651f13c2014-04-23 16:59:28 -0700107 using namespace llvm::support;
Douglas Gregor1a49d972013-01-25 01:03:03 +0000108
109 data_type Result;
110 while (DataLen > 0) {
Stephen Hines651f13c2014-04-23 16:59:28 -0700111 unsigned ID = endian::readNext<uint32_t, little, unaligned>(d);
Douglas Gregor1a49d972013-01-25 01:03:03 +0000112 Result.push_back(ID);
113 DataLen -= 4;
114 }
115
116 return Result;
117 }
118};
119
Stephen Hines6bcf27b2014-05-29 04:14:42 -0700120typedef llvm::OnDiskIterableChainedHashTable<IdentifierIndexReaderTrait>
121 IdentifierIndexTable;
Douglas Gregor1a49d972013-01-25 01:03:03 +0000122
Douglas Gregor1a49d972013-01-25 01:03:03 +0000123}
124
Douglas Gregor677e15f2013-03-19 00:28:20 +0000125GlobalModuleIndex::GlobalModuleIndex(llvm::MemoryBuffer *Buffer,
Douglas Gregor1a49d972013-01-25 01:03:03 +0000126 llvm::BitstreamCursor Cursor)
Douglas Gregorfa69fc12013-03-22 18:50:14 +0000127 : Buffer(Buffer), IdentifierIndex(),
Douglas Gregor188bdcd2013-01-25 23:32:03 +0000128 NumIdentifierLookups(), NumIdentifierLookupHits()
Douglas Gregor1a49d972013-01-25 01:03:03 +0000129{
Douglas Gregor1a49d972013-01-25 01:03:03 +0000130 // Read the global index.
Douglas Gregor1a49d972013-01-25 01:03:03 +0000131 bool InGlobalIndexBlock = false;
132 bool Done = false;
Douglas Gregor1a49d972013-01-25 01:03:03 +0000133 while (!Done) {
134 llvm::BitstreamEntry Entry = Cursor.advance();
135
136 switch (Entry.Kind) {
137 case llvm::BitstreamEntry::Error:
138 return;
139
140 case llvm::BitstreamEntry::EndBlock:
141 if (InGlobalIndexBlock) {
142 InGlobalIndexBlock = false;
143 Done = true;
144 continue;
145 }
146 return;
147
148
149 case llvm::BitstreamEntry::Record:
150 // Entries in the global index block are handled below.
151 if (InGlobalIndexBlock)
152 break;
153
154 return;
155
156 case llvm::BitstreamEntry::SubBlock:
157 if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) {
158 if (Cursor.EnterSubBlock(GLOBAL_INDEX_BLOCK_ID))
159 return;
160
161 InGlobalIndexBlock = true;
162 } else if (Cursor.SkipBlock()) {
163 return;
164 }
165 continue;
166 }
167
168 SmallVector<uint64_t, 64> Record;
169 StringRef Blob;
170 switch ((IndexRecordTypes)Cursor.readRecord(Entry.ID, Record, &Blob)) {
171 case INDEX_METADATA:
172 // Make sure that the version matches.
173 if (Record.size() < 1 || Record[0] != CurrentVersion)
174 return;
175 break;
176
177 case MODULE: {
178 unsigned Idx = 0;
179 unsigned ID = Record[Idx++];
Douglas Gregor677e15f2013-03-19 00:28:20 +0000180
181 // Make room for this module's information.
182 if (ID == Modules.size())
183 Modules.push_back(ModuleInfo());
184 else
185 Modules.resize(ID + 1);
186
187 // Size/modification time for this module file at the time the
188 // global index was built.
189 Modules[ID].Size = Record[Idx++];
190 Modules[ID].ModTime = Record[Idx++];
Douglas Gregor1a49d972013-01-25 01:03:03 +0000191
192 // File name.
193 unsigned NameLen = Record[Idx++];
Douglas Gregor677e15f2013-03-19 00:28:20 +0000194 Modules[ID].FileName.assign(Record.begin() + Idx,
195 Record.begin() + Idx + NameLen);
Douglas Gregor1a49d972013-01-25 01:03:03 +0000196 Idx += NameLen;
197
198 // Dependencies
199 unsigned NumDeps = Record[Idx++];
Douglas Gregor677e15f2013-03-19 00:28:20 +0000200 Modules[ID].Dependencies.insert(Modules[ID].Dependencies.end(),
201 Record.begin() + Idx,
202 Record.begin() + Idx + NumDeps);
203 Idx += NumDeps;
Douglas Gregor1a49d972013-01-25 01:03:03 +0000204
Douglas Gregor677e15f2013-03-19 00:28:20 +0000205 // Make sure we're at the end of the record.
206 assert(Idx == Record.size() && "More module info?");
Douglas Gregorfa69fc12013-03-22 18:50:14 +0000207
208 // Record this module as an unresolved module.
Stephen Hines6bcf27b2014-05-29 04:14:42 -0700209 // FIXME: this doesn't work correctly for module names containing path
210 // separators.
211 StringRef ModuleName = llvm::sys::path::stem(Modules[ID].FileName);
212 // Remove the -<hash of ModuleMapPath>
213 ModuleName = ModuleName.rsplit('-').first;
214 UnresolvedModules[ModuleName] = ID;
Douglas Gregor1a49d972013-01-25 01:03:03 +0000215 break;
216 }
217
218 case IDENTIFIER_INDEX:
219 // Wire up the identifier index.
220 if (Record[0]) {
221 IdentifierIndex = IdentifierIndexTable::Create(
Stephen Hines6bcf27b2014-05-29 04:14:42 -0700222 (const unsigned char *)Blob.data() + Record[0],
223 (const unsigned char *)Blob.data() + sizeof(uint32_t),
224 (const unsigned char *)Blob.data(), IdentifierIndexReaderTrait());
Douglas Gregor1a49d972013-01-25 01:03:03 +0000225 }
226 break;
227 }
228 }
Douglas Gregor1a49d972013-01-25 01:03:03 +0000229}
230
Stephen Hines6bcf27b2014-05-29 04:14:42 -0700231GlobalModuleIndex::~GlobalModuleIndex() {
232 delete static_cast<IdentifierIndexTable *>(IdentifierIndex);
233}
Douglas Gregor1a49d972013-01-25 01:03:03 +0000234
235std::pair<GlobalModuleIndex *, GlobalModuleIndex::ErrorCode>
Douglas Gregor677e15f2013-03-19 00:28:20 +0000236GlobalModuleIndex::readIndex(StringRef Path) {
Douglas Gregor1a49d972013-01-25 01:03:03 +0000237 // Load the index file, if it's there.
238 llvm::SmallString<128> IndexPath;
239 IndexPath += Path;
240 llvm::sys::path::append(IndexPath, IndexFileName);
241
Stephen Hines651f13c2014-04-23 16:59:28 -0700242 std::unique_ptr<llvm::MemoryBuffer> Buffer;
Rafael Espindola28ce23a2013-10-25 19:00:49 +0000243 if (llvm::MemoryBuffer::getFile(IndexPath.c_str(), Buffer) !=
244 llvm::errc::success)
Stephen Hines6bcf27b2014-05-29 04:14:42 -0700245 return std::make_pair(nullptr, EC_NotFound);
Douglas Gregor1a49d972013-01-25 01:03:03 +0000246
247 /// \brief The bitstream reader from which we'll read the AST file.
248 llvm::BitstreamReader Reader((const unsigned char *)Buffer->getBufferStart(),
249 (const unsigned char *)Buffer->getBufferEnd());
250
251 /// \brief The main bitstream cursor for the main block.
252 llvm::BitstreamCursor Cursor(Reader);
253
254 // Sniff for the signature.
255 if (Cursor.Read(8) != 'B' ||
256 Cursor.Read(8) != 'C' ||
257 Cursor.Read(8) != 'G' ||
258 Cursor.Read(8) != 'I') {
Stephen Hines6bcf27b2014-05-29 04:14:42 -0700259 return std::make_pair(nullptr, EC_IOError);
Douglas Gregor1a49d972013-01-25 01:03:03 +0000260 }
Stephen Hines651f13c2014-04-23 16:59:28 -0700261
262 return std::make_pair(new GlobalModuleIndex(Buffer.release(), Cursor),
263 EC_None);
Douglas Gregor1a49d972013-01-25 01:03:03 +0000264}
265
Douglas Gregor677e15f2013-03-19 00:28:20 +0000266void
267GlobalModuleIndex::getKnownModules(SmallVectorImpl<ModuleFile *> &ModuleFiles) {
Douglas Gregor1a49d972013-01-25 01:03:03 +0000268 ModuleFiles.clear();
269 for (unsigned I = 0, N = Modules.size(); I != N; ++I) {
Douglas Gregorfa69fc12013-03-22 18:50:14 +0000270 if (ModuleFile *MF = Modules[I].File)
271 ModuleFiles.push_back(MF);
Douglas Gregor1a49d972013-01-25 01:03:03 +0000272 }
273}
274
275void GlobalModuleIndex::getModuleDependencies(
Douglas Gregor677e15f2013-03-19 00:28:20 +0000276 ModuleFile *File,
277 SmallVectorImpl<ModuleFile *> &Dependencies) {
Douglas Gregor1a49d972013-01-25 01:03:03 +0000278 // Look for information about this module file.
Douglas Gregor677e15f2013-03-19 00:28:20 +0000279 llvm::DenseMap<ModuleFile *, unsigned>::iterator Known
280 = ModulesByFile.find(File);
Douglas Gregor1a49d972013-01-25 01:03:03 +0000281 if (Known == ModulesByFile.end())
282 return;
283
284 // Record dependencies.
Douglas Gregor677e15f2013-03-19 00:28:20 +0000285 Dependencies.clear();
286 ArrayRef<unsigned> StoredDependencies = Modules[Known->second].Dependencies;
287 for (unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) {
Douglas Gregorfa69fc12013-03-22 18:50:14 +0000288 if (ModuleFile *MF = Modules[I].File)
Douglas Gregor677e15f2013-03-19 00:28:20 +0000289 Dependencies.push_back(MF);
290 }
Douglas Gregor1a49d972013-01-25 01:03:03 +0000291}
292
Douglas Gregor188bdcd2013-01-25 23:32:03 +0000293bool GlobalModuleIndex::lookupIdentifier(StringRef Name, HitSet &Hits) {
294 Hits.clear();
Douglas Gregor1a49d972013-01-25 01:03:03 +0000295
296 // If there's no identifier index, there is nothing we can do.
297 if (!IdentifierIndex)
298 return false;
299
300 // Look into the identifier index.
301 ++NumIdentifierLookups;
302 IdentifierIndexTable &Table
303 = *static_cast<IdentifierIndexTable *>(IdentifierIndex);
304 IdentifierIndexTable::iterator Known = Table.find(Name);
305 if (Known == Table.end()) {
306 return true;
307 }
308
309 SmallVector<unsigned, 2> ModuleIDs = *Known;
310 for (unsigned I = 0, N = ModuleIDs.size(); I != N; ++I) {
Douglas Gregorfa69fc12013-03-22 18:50:14 +0000311 if (ModuleFile *MF = Modules[ModuleIDs[I]].File)
312 Hits.insert(MF);
Douglas Gregor1a49d972013-01-25 01:03:03 +0000313 }
314
315 ++NumIdentifierLookupHits;
316 return true;
317}
318
Douglas Gregorfa69fc12013-03-22 18:50:14 +0000319bool GlobalModuleIndex::loadedModuleFile(ModuleFile *File) {
320 // Look for the module in the global module index based on the module name.
Stephen Hines6bcf27b2014-05-29 04:14:42 -0700321 StringRef Name = File->ModuleName;
Douglas Gregorfa69fc12013-03-22 18:50:14 +0000322 llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Name);
323 if (Known == UnresolvedModules.end()) {
324 return true;
Douglas Gregor677e15f2013-03-19 00:28:20 +0000325 }
326
Douglas Gregorfa69fc12013-03-22 18:50:14 +0000327 // Rectify this module with the global module index.
328 ModuleInfo &Info = Modules[Known->second];
329
330 // If the size and modification time match what we expected, record this
331 // module file.
332 bool Failed = true;
333 if (File->File->getSize() == Info.Size &&
334 File->File->getModificationTime() == Info.ModTime) {
335 Info.File = File;
336 ModulesByFile[File] = Known->second;
337
338 Failed = false;
Douglas Gregor677e15f2013-03-19 00:28:20 +0000339 }
340
Douglas Gregorfa69fc12013-03-22 18:50:14 +0000341 // One way or another, we have resolved this module file.
342 UnresolvedModules.erase(Known);
343 return Failed;
Douglas Gregor677e15f2013-03-19 00:28:20 +0000344}
345
Douglas Gregor1a49d972013-01-25 01:03:03 +0000346void GlobalModuleIndex::printStats() {
347 std::fprintf(stderr, "*** Global Module Index Statistics:\n");
348 if (NumIdentifierLookups) {
349 fprintf(stderr, " %u / %u identifier lookups succeeded (%f%%)\n",
350 NumIdentifierLookupHits, NumIdentifierLookups,
351 (double)NumIdentifierLookupHits*100.0/NumIdentifierLookups);
352 }
Douglas Gregor1a49d972013-01-25 01:03:03 +0000353 std::fprintf(stderr, "\n");
354}
355
Stephen Hines6bcf27b2014-05-29 04:14:42 -0700356void GlobalModuleIndex::dump() {
357 llvm::errs() << "*** Global Module Index Dump:\n";
358 llvm::errs() << "Module files:\n";
359 for (auto &MI : Modules) {
360 llvm::errs() << "** " << MI.FileName << "\n";
361 if (MI.File)
362 MI.File->dump();
363 else
364 llvm::errs() << "\n";
365 }
366 llvm::errs() << "\n";
367}
368
Douglas Gregor1a49d972013-01-25 01:03:03 +0000369//----------------------------------------------------------------------------//
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000370// Global module index writer.
371//----------------------------------------------------------------------------//
372
373namespace {
374 /// \brief Provides information about a specific module file.
375 struct ModuleFileInfo {
376 /// \brief The numberic ID for this module file.
377 unsigned ID;
378
379 /// \brief The set of modules on which this module depends. Each entry is
380 /// a module ID.
381 SmallVector<unsigned, 4> Dependencies;
382 };
383
384 /// \brief Builder that generates the global module index file.
385 class GlobalModuleIndexBuilder {
386 FileManager &FileMgr;
387
388 /// \brief Mapping from files to module file information.
389 typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap;
390
391 /// \brief Information about each of the known module files.
392 ModuleFilesMap ModuleFiles;
393
394 /// \brief Mapping from identifiers to the list of module file IDs that
395 /// consider this identifier to be interesting.
396 typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;
397
398 /// \brief A mapping from all interesting identifiers to the set of module
399 /// files in which those identifiers are considered interesting.
400 InterestingIdentifierMap InterestingIdentifiers;
401
402 /// \brief Write the block-info block for the global module index file.
403 void emitBlockInfoBlock(llvm::BitstreamWriter &Stream);
404
405 /// \brief Retrieve the module file information for the given file.
406 ModuleFileInfo &getModuleFileInfo(const FileEntry *File) {
407 llvm::MapVector<const FileEntry *, ModuleFileInfo>::iterator Known
408 = ModuleFiles.find(File);
409 if (Known != ModuleFiles.end())
410 return Known->second;
411
412 unsigned NewID = ModuleFiles.size();
413 ModuleFileInfo &Info = ModuleFiles[File];
414 Info.ID = NewID;
415 return Info;
416 }
417
418 public:
419 explicit GlobalModuleIndexBuilder(FileManager &FileMgr) : FileMgr(FileMgr){}
420
421 /// \brief Load the contents of the given module file into the builder.
422 ///
423 /// \returns true if an error occurred, false otherwise.
424 bool loadModuleFile(const FileEntry *File);
425
426 /// \brief Write the index to the given bitstream.
427 void writeIndex(llvm::BitstreamWriter &Stream);
428 };
429}
430
431static void emitBlockID(unsigned ID, const char *Name,
432 llvm::BitstreamWriter &Stream,
433 SmallVectorImpl<uint64_t> &Record) {
434 Record.clear();
435 Record.push_back(ID);
436 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record);
437
438 // Emit the block name if present.
Stephen Hines6bcf27b2014-05-29 04:14:42 -0700439 if (!Name || Name[0] == 0) return;
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000440 Record.clear();
441 while (*Name)
442 Record.push_back(*Name++);
443 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record);
444}
445
446static void emitRecordID(unsigned ID, const char *Name,
447 llvm::BitstreamWriter &Stream,
448 SmallVectorImpl<uint64_t> &Record) {
449 Record.clear();
450 Record.push_back(ID);
451 while (*Name)
452 Record.push_back(*Name++);
453 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record);
454}
455
456void
457GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) {
458 SmallVector<uint64_t, 64> Record;
459 Stream.EnterSubblock(llvm::bitc::BLOCKINFO_BLOCK_ID, 3);
460
461#define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record)
462#define RECORD(X) emitRecordID(X, #X, Stream, Record)
463 BLOCK(GLOBAL_INDEX_BLOCK);
Douglas Gregor1a49d972013-01-25 01:03:03 +0000464 RECORD(INDEX_METADATA);
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000465 RECORD(MODULE);
466 RECORD(IDENTIFIER_INDEX);
467#undef RECORD
468#undef BLOCK
469
470 Stream.ExitBlock();
471}
472
Douglas Gregor1a49d972013-01-25 01:03:03 +0000473namespace {
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000474 class InterestingASTIdentifierLookupTrait
475 : public serialization::reader::ASTIdentifierLookupTraitBase {
476
477 public:
478 /// \brief The identifier and whether it is "interesting".
479 typedef std::pair<StringRef, bool> data_type;
480
481 data_type ReadData(const internal_key_type& k,
482 const unsigned char* d,
483 unsigned DataLen) {
484 // The first bit indicates whether this identifier is interesting.
485 // That's all we care about.
Stephen Hines651f13c2014-04-23 16:59:28 -0700486 using namespace llvm::support;
487 unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d);
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000488 bool IsInteresting = RawID & 0x01;
489 return std::make_pair(k, IsInteresting);
490 }
491 };
492}
493
494bool GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) {
495 // Open the module file.
Stephen Hines651f13c2014-04-23 16:59:28 -0700496 std::unique_ptr<llvm::MemoryBuffer> Buffer;
Douglas Gregor8e31d062013-02-06 18:08:37 +0000497 std::string ErrorStr;
498 Buffer.reset(FileMgr.getBufferForFile(File, &ErrorStr, /*isVolatile=*/true));
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000499 if (!Buffer) {
500 return true;
501 }
502
503 // Initialize the input stream
504 llvm::BitstreamReader InStreamFile;
505 llvm::BitstreamCursor InStream;
506 InStreamFile.init((const unsigned char *)Buffer->getBufferStart(),
507 (const unsigned char *)Buffer->getBufferEnd());
508 InStream.init(InStreamFile);
509
510 // Sniff for the signature.
511 if (InStream.Read(8) != 'C' ||
512 InStream.Read(8) != 'P' ||
513 InStream.Read(8) != 'C' ||
514 InStream.Read(8) != 'H') {
515 return true;
516 }
517
518 // Record this module file and assign it a unique ID (if it doesn't have
519 // one already).
520 unsigned ID = getModuleFileInfo(File).ID;
521
522 // Search for the blocks and records we care about.
Douglas Gregor1a49d972013-01-25 01:03:03 +0000523 enum { Other, ControlBlock, ASTBlock } State = Other;
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000524 bool Done = false;
525 while (!Done) {
Douglas Gregor1a49d972013-01-25 01:03:03 +0000526 llvm::BitstreamEntry Entry = InStream.advance();
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000527 switch (Entry.Kind) {
528 case llvm::BitstreamEntry::Error:
Douglas Gregor1a49d972013-01-25 01:03:03 +0000529 Done = true;
530 continue;
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000531
532 case llvm::BitstreamEntry::Record:
Douglas Gregor1a49d972013-01-25 01:03:03 +0000533 // In the 'other' state, just skip the record. We don't care.
534 if (State == Other) {
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000535 InStream.skipRecord(Entry.ID);
536 continue;
537 }
538
539 // Handle potentially-interesting records below.
540 break;
541
542 case llvm::BitstreamEntry::SubBlock:
Douglas Gregor1a49d972013-01-25 01:03:03 +0000543 if (Entry.ID == CONTROL_BLOCK_ID) {
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000544 if (InStream.EnterSubBlock(CONTROL_BLOCK_ID))
545 return true;
546
547 // Found the control block.
548 State = ControlBlock;
549 continue;
550 }
551
Douglas Gregor1a49d972013-01-25 01:03:03 +0000552 if (Entry.ID == AST_BLOCK_ID) {
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000553 if (InStream.EnterSubBlock(AST_BLOCK_ID))
554 return true;
555
556 // Found the AST block.
557 State = ASTBlock;
558 continue;
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000559 }
560
561 if (InStream.SkipBlock())
562 return true;
563
564 continue;
565
566 case llvm::BitstreamEntry::EndBlock:
Douglas Gregor1a49d972013-01-25 01:03:03 +0000567 State = Other;
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000568 continue;
569 }
570
571 // Read the given record.
572 SmallVector<uint64_t, 64> Record;
573 StringRef Blob;
574 unsigned Code = InStream.readRecord(Entry.ID, Record, &Blob);
575
576 // Handle module dependencies.
577 if (State == ControlBlock && Code == IMPORTS) {
578 // Load each of the imported PCH files.
579 unsigned Idx = 0, N = Record.size();
580 while (Idx < N) {
581 // Read information about the AST file.
582
583 // Skip the imported kind
584 ++Idx;
585
586 // Skip the import location
587 ++Idx;
588
Douglas Gregor677e15f2013-03-19 00:28:20 +0000589 // Load stored size/modification time.
590 off_t StoredSize = (off_t)Record[Idx++];
591 time_t StoredModTime = (time_t)Record[Idx++];
592
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000593 // Retrieve the imported file name.
594 unsigned Length = Record[Idx++];
595 SmallString<128> ImportedFile(Record.begin() + Idx,
596 Record.begin() + Idx + Length);
597 Idx += Length;
598
599 // Find the imported module file.
Douglas Gregorea14a872013-02-08 21:27:45 +0000600 const FileEntry *DependsOnFile
601 = FileMgr.getFile(ImportedFile, /*openFile=*/false,
602 /*cacheFailure=*/false);
Douglas Gregor677e15f2013-03-19 00:28:20 +0000603 if (!DependsOnFile ||
604 (StoredSize != DependsOnFile->getSize()) ||
605 (StoredModTime != DependsOnFile->getModificationTime()))
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000606 return true;
607
608 // Record the dependency.
609 unsigned DependsOnID = getModuleFileInfo(DependsOnFile).ID;
610 getModuleFileInfo(File).Dependencies.push_back(DependsOnID);
611 }
612
613 continue;
614 }
615
616 // Handle the identifier table
617 if (State == ASTBlock && Code == IDENTIFIER_TABLE && Record[0] > 0) {
Stephen Hines6bcf27b2014-05-29 04:14:42 -0700618 typedef llvm::OnDiskIterableChainedHashTable<
619 InterestingASTIdentifierLookupTrait> InterestingIdentifierTable;
Stephen Hines651f13c2014-04-23 16:59:28 -0700620 std::unique_ptr<InterestingIdentifierTable> Table(
621 InterestingIdentifierTable::Create(
622 (const unsigned char *)Blob.data() + Record[0],
Stephen Hines6bcf27b2014-05-29 04:14:42 -0700623 (const unsigned char *)Blob.data() + sizeof(uint32_t),
Stephen Hines651f13c2014-04-23 16:59:28 -0700624 (const unsigned char *)Blob.data()));
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000625 for (InterestingIdentifierTable::data_iterator D = Table->data_begin(),
626 DEnd = Table->data_end();
627 D != DEnd; ++D) {
628 std::pair<StringRef, bool> Ident = *D;
629 if (Ident.second)
630 InterestingIdentifiers[Ident.first].push_back(ID);
Douglas Gregor1a49d972013-01-25 01:03:03 +0000631 else
632 (void)InterestingIdentifiers[Ident.first];
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000633 }
634 }
635
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000636 // We don't care about this record.
637 }
638
639 return false;
640}
641
642namespace {
643
644/// \brief Trait used to generate the identifier index as an on-disk hash
645/// table.
646class IdentifierIndexWriterTrait {
647public:
648 typedef StringRef key_type;
649 typedef StringRef key_type_ref;
650 typedef SmallVector<unsigned, 2> data_type;
651 typedef const SmallVector<unsigned, 2> &data_type_ref;
Stephen Hines6bcf27b2014-05-29 04:14:42 -0700652 typedef unsigned hash_value_type;
653 typedef unsigned offset_type;
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000654
Stephen Hines6bcf27b2014-05-29 04:14:42 -0700655 static hash_value_type ComputeHash(key_type_ref Key) {
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000656 return llvm::HashString(Key);
657 }
658
659 std::pair<unsigned,unsigned>
660 EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
Stephen Hines651f13c2014-04-23 16:59:28 -0700661 using namespace llvm::support;
662 endian::Writer<little> LE(Out);
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000663 unsigned KeyLen = Key.size();
664 unsigned DataLen = Data.size() * 4;
Stephen Hines651f13c2014-04-23 16:59:28 -0700665 LE.write<uint16_t>(KeyLen);
666 LE.write<uint16_t>(DataLen);
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000667 return std::make_pair(KeyLen, DataLen);
668 }
669
670 void EmitKey(raw_ostream& Out, key_type_ref Key, unsigned KeyLen) {
671 Out.write(Key.data(), KeyLen);
672 }
673
674 void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
675 unsigned DataLen) {
Stephen Hines651f13c2014-04-23 16:59:28 -0700676 using namespace llvm::support;
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000677 for (unsigned I = 0, N = Data.size(); I != N; ++I)
Stephen Hines651f13c2014-04-23 16:59:28 -0700678 endian::Writer<little>(Out).write<uint32_t>(Data[I]);
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000679 }
680};
681
682}
683
684void GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
685 using namespace llvm;
686
687 // Emit the file header.
688 Stream.Emit((unsigned)'B', 8);
689 Stream.Emit((unsigned)'C', 8);
690 Stream.Emit((unsigned)'G', 8);
691 Stream.Emit((unsigned)'I', 8);
692
693 // Write the block-info block, which describes the records in this bitcode
694 // file.
695 emitBlockInfoBlock(Stream);
696
697 Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3);
698
699 // Write the metadata.
700 SmallVector<uint64_t, 2> Record;
701 Record.push_back(CurrentVersion);
Douglas Gregor1a49d972013-01-25 01:03:03 +0000702 Stream.EmitRecord(INDEX_METADATA, Record);
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000703
704 // Write the set of known module files.
705 for (ModuleFilesMap::iterator M = ModuleFiles.begin(),
706 MEnd = ModuleFiles.end();
707 M != MEnd; ++M) {
708 Record.clear();
709 Record.push_back(M->second.ID);
710 Record.push_back(M->first->getSize());
711 Record.push_back(M->first->getModificationTime());
712
713 // File name
714 StringRef Name(M->first->getName());
715 Record.push_back(Name.size());
716 Record.append(Name.begin(), Name.end());
717
718 // Dependencies
719 Record.push_back(M->second.Dependencies.size());
720 Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end());
721 Stream.EmitRecord(MODULE, Record);
722 }
723
724 // Write the identifier -> module file mapping.
725 {
Stephen Hines6bcf27b2014-05-29 04:14:42 -0700726 llvm::OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000727 IdentifierIndexWriterTrait Trait;
728
729 // Populate the hash table.
730 for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(),
731 IEnd = InterestingIdentifiers.end();
732 I != IEnd; ++I) {
733 Generator.insert(I->first(), I->second, Trait);
734 }
735
736 // Create the on-disk hash table in a buffer.
737 SmallString<4096> IdentifierTable;
738 uint32_t BucketOffset;
739 {
Stephen Hines651f13c2014-04-23 16:59:28 -0700740 using namespace llvm::support;
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000741 llvm::raw_svector_ostream Out(IdentifierTable);
742 // Make sure that no bucket is at offset 0
Stephen Hines651f13c2014-04-23 16:59:28 -0700743 endian::Writer<little>(Out).write<uint32_t>(0);
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000744 BucketOffset = Generator.Emit(Out, Trait);
745 }
746
747 // Create a blob abbreviation
748 BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
749 Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX));
750 Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
751 Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
752 unsigned IDTableAbbrev = Stream.EmitAbbrev(Abbrev);
753
754 // Write the identifier table
755 Record.clear();
756 Record.push_back(IDENTIFIER_INDEX);
757 Record.push_back(BucketOffset);
758 Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable.str());
759 }
760
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000761 Stream.ExitBlock();
762}
763
764GlobalModuleIndex::ErrorCode
765GlobalModuleIndex::writeIndex(FileManager &FileMgr, StringRef Path) {
766 llvm::SmallString<128> IndexPath;
767 IndexPath += Path;
768 llvm::sys::path::append(IndexPath, IndexFileName);
769
770 // Coordinate building the global index file with other processes that might
771 // try to do the same.
772 llvm::LockFileManager Locked(IndexPath);
773 switch (Locked) {
774 case llvm::LockFileManager::LFS_Error:
775 return EC_IOError;
776
777 case llvm::LockFileManager::LFS_Owned:
778 // We're responsible for building the index ourselves. Do so below.
779 break;
780
781 case llvm::LockFileManager::LFS_Shared:
782 // Someone else is responsible for building the index. We don't care
783 // when they finish, so we're done.
784 return EC_Building;
785 }
786
787 // The module index builder.
788 GlobalModuleIndexBuilder Builder(FileMgr);
789
790 // Load each of the module files.
791 llvm::error_code EC;
792 for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd;
793 D != DEnd && !EC;
794 D.increment(EC)) {
795 // If this isn't a module file, we don't care.
796 if (llvm::sys::path::extension(D->path()) != ".pcm") {
797 // ... unless it's a .pcm.lock file, which indicates that someone is
798 // in the process of rebuilding a module. They'll rebuild the index
799 // at the end of that translation unit, so we don't have to.
800 if (llvm::sys::path::extension(D->path()) == ".pcm.lock")
801 return EC_Building;
802
803 continue;
804 }
805
806 // If we can't find the module file, skip it.
807 const FileEntry *ModuleFile = FileMgr.getFile(D->path());
808 if (!ModuleFile)
809 continue;
810
811 // Load this module file.
812 if (Builder.loadModuleFile(ModuleFile))
813 return EC_IOError;
814 }
815
816 // The output buffer, into which the global index will be written.
817 SmallVector<char, 16> OutputBuffer;
818 {
819 llvm::BitstreamWriter OutputStream(OutputBuffer);
820 Builder.writeIndex(OutputStream);
821 }
822
823 // Write the global index file to a temporary file.
824 llvm::SmallString<128> IndexTmpPath;
825 int TmpFD;
Rafael Espindola70e7aec2013-07-05 21:13:58 +0000826 if (llvm::sys::fs::createUniqueFile(IndexPath + "-%%%%%%%%", TmpFD,
827 IndexTmpPath))
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000828 return EC_IOError;
829
830 // Open the temporary global index file for output.
NAKAMURA Takumid2db16f2013-01-24 08:20:11 +0000831 llvm::raw_fd_ostream Out(TmpFD, true);
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000832 if (Out.has_error())
833 return EC_IOError;
834
835 // Write the index.
836 Out.write(OutputBuffer.data(), OutputBuffer.size());
837 Out.close();
838 if (Out.has_error())
839 return EC_IOError;
840
841 // Remove the old index file. It isn't relevant any more.
Stephen Hines651f13c2014-04-23 16:59:28 -0700842 llvm::sys::fs::remove(IndexPath.str());
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000843
844 // Rename the newly-written index file to the proper name.
845 if (llvm::sys::fs::rename(IndexTmpPath.str(), IndexPath.str())) {
846 // Rename failed; just remove the
Stephen Hines651f13c2014-04-23 16:59:28 -0700847 llvm::sys::fs::remove(IndexTmpPath.str());
Douglas Gregora6b00fc2013-01-23 22:38:11 +0000848 return EC_IOError;
849 }
850
851 // We're done.
852 return EC_None;
853}
Argyrios Kyrtzidis87f9d812013-04-17 22:10:55 +0000854
855namespace {
856 class GlobalIndexIdentifierIterator : public IdentifierIterator {
857 /// \brief The current position within the identifier lookup table.
858 IdentifierIndexTable::key_iterator Current;
859
860 /// \brief The end position within the identifier lookup table.
861 IdentifierIndexTable::key_iterator End;
862
863 public:
864 explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) {
865 Current = Idx.key_begin();
866 End = Idx.key_end();
867 }
868
Stephen Hines651f13c2014-04-23 16:59:28 -0700869 StringRef Next() override {
Argyrios Kyrtzidis87f9d812013-04-17 22:10:55 +0000870 if (Current == End)
871 return StringRef();
872
873 StringRef Result = *Current;
874 ++Current;
875 return Result;
876 }
877 };
878}
879
880IdentifierIterator *GlobalModuleIndex::createIdentifierIterator() const {
881 IdentifierIndexTable &Table =
882 *static_cast<IdentifierIndexTable *>(IdentifierIndex);
883 return new GlobalIndexIdentifierIterator(Table);
884}