blob: cff6dd8f04bf10443bc281213b1bf99639b3581d [file] [log] [blame]
Ted Kremenek85888962008-10-21 00:54:44 +00001//===--- CacheTokens.cpp - Caching of lexer tokens for PCH support --------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This provides a possible implementation of PCH support for Clang that is
11// based on caching lexed tokens and identifiers.
12//
13//===----------------------------------------------------------------------===//
14
15#include "clang.h"
16#include "clang/Basic/FileManager.h"
17#include "clang/Basic/SourceManager.h"
18#include "clang/Basic/IdentifierTable.h"
19#include "clang/Basic/Diagnostic.h"
20#include "clang/Lex/Lexer.h"
21#include "clang/Lex/Preprocessor.h"
Ted Kremenekbe295332009-01-08 02:44:06 +000022#include "llvm/ADT/StringMap.h"
Ted Kremenek85888962008-10-21 00:54:44 +000023#include "llvm/Support/MemoryBuffer.h"
24#include "llvm/Support/raw_ostream.h"
Ted Kremenekfc7e2ea2008-12-02 19:44:08 +000025#include "llvm/System/Path.h"
Ted Kremenekb978c662009-01-08 01:17:37 +000026#include "llvm/Support/Compiler.h"
Ted Kremenek72b1b152009-01-15 18:47:46 +000027#include "llvm/Support/Streams.h"
Ted Kremenek85888962008-10-21 00:54:44 +000028
29using namespace clang;
30
Ted Kremenekfc7e2ea2008-12-02 19:44:08 +000031typedef uint32_t Offset;
32
Ted Kremeneke0ea5dc2009-02-10 01:06:17 +000033static void Emit8(llvm::raw_ostream& Out, uint32_t V) {
34 Out << (unsigned char)(V);
35}
36
37static void Emit16(llvm::raw_ostream& Out, uint32_t V) {
38 Out << (unsigned char)(V);
39 Out << (unsigned char)(V >> 8);
40 assert((V >> 16) == 0);
41}
42
43static void Emit32(llvm::raw_ostream& Out, uint32_t V) {
44 Out << (unsigned char)(V);
45 Out << (unsigned char)(V >> 8);
46 Out << (unsigned char)(V >> 16);
47 Out << (unsigned char)(V >> 24);
48}
49
Ted Kremenekd8c02922009-02-10 22:16:22 +000050static void Pad(llvm::raw_fd_ostream& Out, unsigned A) {
51 Offset off = (Offset) Out.tell();
52 uint32_t n = ((uintptr_t)(off+A-1) & ~(uintptr_t)(A-1)) - off;
53 for ( ; n ; --n ) Emit8(Out, 0);
Ted Kremeneke0ea5dc2009-02-10 01:06:17 +000054}
55
Ted Kremenekf0e1f792009-02-10 01:14:45 +000056//===----------------------------------------------------------------------===//
57// On Disk Hashtable Logic. This will eventually get refactored and put
58// elsewhere.
59//===----------------------------------------------------------------------===//
60
61template<typename Info>
62class OnDiskChainedHashTableGenerator {
63 unsigned NumBuckets;
64 unsigned NumEntries;
65 llvm::BumpPtrAllocator BA;
66
67 class Item {
68 public:
Ted Kremenekd8c02922009-02-10 22:16:22 +000069 typename Info::key_type key;
70 typename Info::data_type data;
Ted Kremenekf0e1f792009-02-10 01:14:45 +000071 Item *next;
72 const uint32_t hash;
73
Ted Kremenekd8c02922009-02-10 22:16:22 +000074 Item(typename Info::key_type_ref k, typename Info::data_type_ref d)
75 : key(k), data(d), next(0), hash(Info::ComputeHash(k)) {}
Ted Kremenekf0e1f792009-02-10 01:14:45 +000076 };
77
78 class Bucket {
79 public:
80 Offset off;
81 Item* head;
82 unsigned length;
83
84 Bucket() {}
85 };
86
87 Bucket* Buckets;
88
89private:
Ted Kremenekd8c02922009-02-10 22:16:22 +000090 void insert(Bucket* b, size_t size, Item* E) {
Ted Kremenekf0e1f792009-02-10 01:14:45 +000091 unsigned idx = E->hash & (size - 1);
92 Bucket& B = b[idx];
93 E->next = B.head;
94 ++B.length;
95 B.head = E;
96 }
97
98 void resize(size_t newsize) {
Ted Kremenekd8c02922009-02-10 22:16:22 +000099 Bucket* newBuckets = (Bucket*) calloc(newsize, sizeof(Bucket));
100 // Populate newBuckets with the old entries.
Ted Kremenekf0e1f792009-02-10 01:14:45 +0000101 for (unsigned i = 0; i < NumBuckets; ++i)
Ted Kremenekd8c02922009-02-10 22:16:22 +0000102 for (Item* E = Buckets[i].head; E ; ) {
Ted Kremenekf0e1f792009-02-10 01:14:45 +0000103 Item* N = E->next;
Ted Kremenekd8c02922009-02-10 22:16:22 +0000104 E->next = 0;
Ted Kremenekf0e1f792009-02-10 01:14:45 +0000105 insert(newBuckets, newsize, E);
106 E = N;
107 }
108
109 free(Buckets);
110 NumBuckets = newsize;
111 Buckets = newBuckets;
112 }
113
114public:
115
Ted Kremenekd8c02922009-02-10 22:16:22 +0000116 void insert(typename Info::key_type_ref key,
117 typename Info::data_type_ref data) {
118
Ted Kremenekf0e1f792009-02-10 01:14:45 +0000119 ++NumEntries;
120 if (4*NumEntries >= 3*NumBuckets) resize(NumBuckets*2);
121 insert(Buckets, NumBuckets, new (BA.Allocate<Item>()) Item(key, data));
122 }
123
124 Offset Emit(llvm::raw_fd_ostream& out) {
125 // Emit the payload of the table.
126 for (unsigned i = 0; i < NumBuckets; ++i) {
127 Bucket& B = Buckets[i];
128 if (!B.head) continue;
129
130 // Store the offset for the data of this bucket.
Ted Kremenekf0e1f792009-02-10 01:14:45 +0000131 B.off = out.tell();
132
Ted Kremenekd8c02922009-02-10 22:16:22 +0000133 // Write out the number of items in the bucket.
134 Emit16(out, B.length);
Ted Kremenekf0e1f792009-02-10 01:14:45 +0000135
136 // Write out the entries in the bucket.
137 for (Item *I = B.head; I ; I = I->next) {
138 Emit32(out, I->hash);
Ted Kremenekd8c02922009-02-10 22:16:22 +0000139 const std::pair<unsigned, unsigned>& Len =
140 Info::EmitKeyDataLength(out, I->key, I->data);
141 Info::EmitKey(out, I->key, Len.first);
142 Info::EmitData(out, I->data, Len.second);
Ted Kremenekf0e1f792009-02-10 01:14:45 +0000143 }
144 }
145
146 // Emit the hashtable itself.
147 Pad(out, 4);
148 Offset TableOff = out.tell();
Ted Kremenekd8c02922009-02-10 22:16:22 +0000149 Emit32(out, NumBuckets);
150 Emit32(out, NumEntries);
Ted Kremenekf0e1f792009-02-10 01:14:45 +0000151 for (unsigned i = 0; i < NumBuckets; ++i) Emit32(out, Buckets[i].off);
152
153 return TableOff;
154 }
155
156 OnDiskChainedHashTableGenerator() {
157 NumEntries = 0;
Ted Kremenekd8c02922009-02-10 22:16:22 +0000158 NumBuckets = 64;
159 // Note that we do not need to run the constructors of the individual
160 // Bucket objects since 'calloc' returns bytes that are all 0.
161 Buckets = (Bucket*) calloc(NumBuckets, sizeof(Bucket));
Ted Kremenekf0e1f792009-02-10 01:14:45 +0000162 }
163
164 ~OnDiskChainedHashTableGenerator() {
165 free(Buckets);
166 }
167};
168
169//===----------------------------------------------------------------------===//
170// PTH-specific stuff.
171//===----------------------------------------------------------------------===//
172
Ted Kremenekbe295332009-01-08 02:44:06 +0000173namespace {
174class VISIBILITY_HIDDEN PCHEntry {
175 Offset TokenData, PPCondData;
Ted Kremenekbe295332009-01-08 02:44:06 +0000176
177public:
178 PCHEntry() {}
179
Ted Kremenek277faca2009-01-27 00:01:05 +0000180 PCHEntry(Offset td, Offset ppcd)
181 : TokenData(td), PPCondData(ppcd) {}
Ted Kremenekbe295332009-01-08 02:44:06 +0000182
Ted Kremenek277faca2009-01-27 00:01:05 +0000183 Offset getTokenOffset() const { return TokenData; }
Ted Kremenekbe295332009-01-08 02:44:06 +0000184 Offset getPPCondTableOffset() const { return PPCondData; }
Ted Kremenek277faca2009-01-27 00:01:05 +0000185};
Ted Kremenekbe295332009-01-08 02:44:06 +0000186
Ted Kremenekd8c02922009-02-10 22:16:22 +0000187class VISIBILITY_HIDDEN FileEntryPCHEntryInfo {
188public:
189 typedef const FileEntry* key_type;
190 typedef key_type key_type_ref;
191
192 typedef PCHEntry data_type;
193 typedef const PCHEntry& data_type_ref;
194
195 static unsigned ComputeHash(const FileEntry* FE) {
196 // Bernstein hash function:
197 // This is basically copy-and-paste from StringMap. This likely won't
198 // stay here, which is why I didn't both to expose this function from
199 // String Map. There are plenty of other hash functions which are likely
200 // to perform better and be faster.
201 unsigned int R = 0;
202 for (const char* x = FE->getName(); *x != '\0' ; ++x) R = R * 33 + *x;
203 return R + (R >> 5);
204 }
205
206 static std::pair<unsigned,unsigned>
207 EmitKeyDataLength(llvm::raw_ostream& Out, const FileEntry* FE,
208 const PCHEntry& E) {
209
210 unsigned n = strlen(FE->getName()) + 1;
211 ::Emit16(Out, n);
212 return std::make_pair(n, 8);
213 }
214
215 static void EmitKey(llvm::raw_ostream& Out, const FileEntry* FE, unsigned n) {
216 Out.write(FE->getName(), n);
217 }
218
219 static void EmitData(llvm::raw_ostream& Out, const PCHEntry& E, unsigned) {
220 ::Emit32(Out, E.getTokenOffset());
221 ::Emit32(Out, E.getPPCondTableOffset());
222 }
223};
224
Ted Kremenek277faca2009-01-27 00:01:05 +0000225class OffsetOpt {
226 bool valid;
227 Offset off;
228public:
229 OffsetOpt() : valid(false) {}
230 bool hasOffset() const { return valid; }
231 Offset getOffset() const { assert(valid); return off; }
232 void setOffset(Offset o) { off = o; valid = true; }
Ted Kremenekbe295332009-01-08 02:44:06 +0000233};
234} // end anonymous namespace
235
Ted Kremenekd8c02922009-02-10 22:16:22 +0000236typedef OnDiskChainedHashTableGenerator<FileEntryPCHEntryInfo> PCHMap;
Ted Kremenekfc7e2ea2008-12-02 19:44:08 +0000237typedef llvm::DenseMap<const IdentifierInfo*,uint32_t> IDMap;
Ted Kremenek277faca2009-01-27 00:01:05 +0000238typedef llvm::StringMap<OffsetOpt, llvm::BumpPtrAllocator> CachedStrsTy;
Ted Kremenek85888962008-10-21 00:54:44 +0000239
Ted Kremenekb978c662009-01-08 01:17:37 +0000240namespace {
241class VISIBILITY_HIDDEN PTHWriter {
242 IDMap IM;
243 llvm::raw_fd_ostream& Out;
244 Preprocessor& PP;
245 uint32_t idcount;
246 PCHMap PM;
Ted Kremenekbe295332009-01-08 02:44:06 +0000247 CachedStrsTy CachedStrs;
Ted Kremenek277faca2009-01-27 00:01:05 +0000248 Offset CurStrOffset;
249 std::vector<llvm::StringMapEntry<OffsetOpt>*> StrEntries;
Ted Kremenek8f174e12008-12-23 02:52:12 +0000250
Ted Kremenekb978c662009-01-08 01:17:37 +0000251 //// Get the persistent id for the given IdentifierInfo*.
252 uint32_t ResolveID(const IdentifierInfo* II);
Ted Kremenekfc7e2ea2008-12-02 19:44:08 +0000253
Ted Kremenekb978c662009-01-08 01:17:37 +0000254 /// Emit a token to the PTH file.
255 void EmitToken(const Token& T);
256
257 void Emit8(uint32_t V) {
258 Out << (unsigned char)(V);
259 }
260
Ted Kremeneke0ea5dc2009-02-10 01:06:17 +0000261 void Emit16(uint32_t V) { ::Emit16(Out, V); }
Ted Kremenekb978c662009-01-08 01:17:37 +0000262
263 void Emit24(uint32_t V) {
264 Out << (unsigned char)(V);
265 Out << (unsigned char)(V >> 8);
266 Out << (unsigned char)(V >> 16);
267 assert((V >> 24) == 0);
268 }
269
Ted Kremeneke0ea5dc2009-02-10 01:06:17 +0000270 void Emit32(uint32_t V) { ::Emit32(Out, V); }
271
Ted Kremenekb978c662009-01-08 01:17:37 +0000272 void EmitBuf(const char* I, const char* E) {
273 for ( ; I != E ; ++I) Out << *I;
274 }
275
Ted Kremenek293b4af2009-01-15 01:26:25 +0000276 std::pair<Offset,std::pair<Offset, Offset> > EmitIdentifierTable();
Ted Kremenekb978c662009-01-08 01:17:37 +0000277 Offset EmitFileTable();
Ted Kremenekbe295332009-01-08 02:44:06 +0000278 PCHEntry LexTokens(Lexer& L);
Ted Kremenek277faca2009-01-27 00:01:05 +0000279 Offset EmitCachedSpellings();
Ted Kremenekbe295332009-01-08 02:44:06 +0000280
Ted Kremenekb978c662009-01-08 01:17:37 +0000281public:
282 PTHWriter(llvm::raw_fd_ostream& out, Preprocessor& pp)
Ted Kremenek277faca2009-01-27 00:01:05 +0000283 : Out(out), PP(pp), idcount(0), CurStrOffset(0) {}
Ted Kremenekb978c662009-01-08 01:17:37 +0000284
285 void GeneratePTH();
286};
287} // end anonymous namespace
288
289uint32_t PTHWriter::ResolveID(const IdentifierInfo* II) {
Ted Kremenekfc7e2ea2008-12-02 19:44:08 +0000290 // Null IdentifierInfo's map to the persistent ID 0.
291 if (!II)
292 return 0;
293
Ted Kremenek85888962008-10-21 00:54:44 +0000294 IDMap::iterator I = IM.find(II);
295
296 if (I == IM.end()) {
Ted Kremenekb978c662009-01-08 01:17:37 +0000297 IM[II] = ++idcount; // Pre-increment since '0' is reserved for NULL.
298 return idcount;
Ted Kremenek85888962008-10-21 00:54:44 +0000299 }
300
Ted Kremenekfc7e2ea2008-12-02 19:44:08 +0000301 return I->second; // We've already added 1.
Ted Kremenek85888962008-10-21 00:54:44 +0000302}
303
Ted Kremenekb978c662009-01-08 01:17:37 +0000304void PTHWriter::EmitToken(const Token& T) {
Ted Kremenek7b78b7c2009-01-19 23:13:15 +0000305 Emit32(((uint32_t) T.getKind()) |
306 (((uint32_t) T.getFlags()) << 8) |
307 (((uint32_t) T.getLength()) << 16));
Ted Kremenek277faca2009-01-27 00:01:05 +0000308
Chris Lattner47246be2009-01-26 19:29:26 +0000309 // Literals (strings, numbers, characters) get cached spellings.
310 if (T.isLiteral()) {
311 // FIXME: This uses the slow getSpelling(). Perhaps we do better
312 // in the future? This only slows down PTH generation.
313 const std::string &spelling = PP.getSpelling(T);
314 const char* s = spelling.c_str();
315
316 // Get the string entry.
Ted Kremenek277faca2009-01-27 00:01:05 +0000317 llvm::StringMapEntry<OffsetOpt> *E =
318 &CachedStrs.GetOrCreateValue(s, s+spelling.size());
319
320 if (!E->getValue().hasOffset()) {
321 E->getValue().setOffset(CurStrOffset);
322 StrEntries.push_back(E);
323 CurStrOffset += spelling.size() + 1;
324 }
325
326 Emit32(E->getValue().getOffset());
Ted Kremenekb978c662009-01-08 01:17:37 +0000327 }
Ted Kremenek277faca2009-01-27 00:01:05 +0000328 else
329 Emit32(ResolveID(T.getIdentifierInfo()));
330
Chris Lattner52c29082009-01-27 06:27:13 +0000331 Emit32(PP.getSourceManager().getFileOffset(T.getLocation()));
Ted Kremenek85888962008-10-21 00:54:44 +0000332}
333
Ted Kremenekb978c662009-01-08 01:17:37 +0000334namespace {
335struct VISIBILITY_HIDDEN IDData {
Ted Kremenekfa59aad2008-11-26 23:58:26 +0000336 const IdentifierInfo* II;
337 uint32_t FileOffset;
Ted Kremenek293b4af2009-01-15 01:26:25 +0000338};
339
340class VISIBILITY_HIDDEN CompareIDDataIndex {
341 IDData* Table;
342public:
343 CompareIDDataIndex(IDData* table) : Table(table) {}
344
345 bool operator()(unsigned i, unsigned j) const {
Ted Kremenek72b1b152009-01-15 18:47:46 +0000346 const IdentifierInfo* II_i = Table[i].II;
347 const IdentifierInfo* II_j = Table[j].II;
348
349 unsigned i_len = II_i->getLength();
350 unsigned j_len = II_j->getLength();
351
352 if (i_len > j_len)
353 return false;
354
355 if (i_len < j_len)
356 return true;
357
358 // Otherwise, compare the strings themselves!
359 return strncmp(II_i->getName(), II_j->getName(), i_len) < 0;
Ted Kremenek293b4af2009-01-15 01:26:25 +0000360 }
Ted Kremenekfa59aad2008-11-26 23:58:26 +0000361};
Ted Kremenekb978c662009-01-08 01:17:37 +0000362}
Ted Kremeneka3d764c2008-11-26 03:36:26 +0000363
Ted Kremenek293b4af2009-01-15 01:26:25 +0000364std::pair<Offset,std::pair<Offset,Offset> >
365PTHWriter::EmitIdentifierTable() {
366 llvm::BumpPtrAllocator Alloc;
Ted Kremenekfa59aad2008-11-26 23:58:26 +0000367
368 // Build an inverse map from persistent IDs -> IdentifierInfo*.
Ted Kremenek293b4af2009-01-15 01:26:25 +0000369 IDData* IIDMap = Alloc.Allocate<IDData>(idcount);
Ted Kremenek85888962008-10-21 00:54:44 +0000370
Ted Kremenekfa59aad2008-11-26 23:58:26 +0000371 // Generate mapping from persistent IDs -> IdentifierInfo*.
Ted Kremenekb978c662009-01-08 01:17:37 +0000372 for (IDMap::iterator I=IM.begin(), E=IM.end(); I!=E; ++I) {
Ted Kremenekfc7e2ea2008-12-02 19:44:08 +0000373 // Decrement by 1 because we are using a vector for the lookup and
374 // 0 is reserved for NULL.
375 assert(I->second > 0);
Ted Kremenek293b4af2009-01-15 01:26:25 +0000376 assert(I->second-1 < idcount);
377 unsigned idx = I->second-1;
378 IIDMap[idx].II = I->first;
379 }
Ted Kremenekfa59aad2008-11-26 23:58:26 +0000380
Ted Kremenek293b4af2009-01-15 01:26:25 +0000381 // We want to write out the strings in lexical order to support binary
382 // search of strings to identifiers. Create such a table.
383 unsigned *LexicalOrder = Alloc.Allocate<unsigned>(idcount);
384 for (unsigned i = 0; i < idcount ; ++i ) LexicalOrder[i] = i;
385 std::sort(LexicalOrder, LexicalOrder+idcount, CompareIDDataIndex(IIDMap));
386
387 // Write out the lexically-sorted table of persistent ids.
388 Offset LexicalOff = Out.tell();
389 for (unsigned i = 0; i < idcount ; ++i) Emit32(LexicalOrder[i]);
390
391 // Write out the string data itself.
Ted Kremenekfc7e2ea2008-12-02 19:44:08 +0000392 Offset DataOff = Out.tell();
Ted Kremenek6183e482008-12-03 01:16:39 +0000393
Ted Kremenek293b4af2009-01-15 01:26:25 +0000394 for (unsigned i = 0; i < idcount; ++i) {
395 IDData& d = IIDMap[i];
396 d.FileOffset = Out.tell(); // Record the location for this data.
397 unsigned len = d.II->getLength(); // Write out the string length.
Ted Kremenekb978c662009-01-08 01:17:37 +0000398 Emit32(len);
Ted Kremenek293b4af2009-01-15 01:26:25 +0000399 const char* buf = d.II->getName(); // Write out the string data.
Ted Kremenek72b1b152009-01-15 18:47:46 +0000400 EmitBuf(buf, buf+len);
401 // Emit a null character for those clients expecting that IdentifierInfo
402 // strings are null terminated.
403 Emit8('\0');
Ted Kremenek85888962008-10-21 00:54:44 +0000404 }
Ted Kremeneka3d764c2008-11-26 03:36:26 +0000405
Ted Kremenekfa59aad2008-11-26 23:58:26 +0000406 // Now emit the table mapping from persistent IDs to PTH file offsets.
Ted Kremenekfc7e2ea2008-12-02 19:44:08 +0000407 Offset IDOff = Out.tell();
Ted Kremenek293b4af2009-01-15 01:26:25 +0000408 Emit32(idcount); // Emit the number of identifiers.
409 for (unsigned i = 0 ; i < idcount; ++i) Emit32(IIDMap[i].FileOffset);
Ted Kremenek6183e482008-12-03 01:16:39 +0000410
Ted Kremenek293b4af2009-01-15 01:26:25 +0000411 return std::make_pair(DataOff, std::make_pair(IDOff, LexicalOff));
Ted Kremenek85888962008-10-21 00:54:44 +0000412}
413
Ted Kremeneka3d764c2008-11-26 03:36:26 +0000414
Ted Kremenekbe295332009-01-08 02:44:06 +0000415PCHEntry PTHWriter::LexTokens(Lexer& L) {
Ted Kremenek7b78b7c2009-01-19 23:13:15 +0000416 // Pad 0's so that we emit tokens to a 4-byte alignment.
417 // This speed up reading them back in.
Ted Kremeneke0ea5dc2009-02-10 01:06:17 +0000418 Pad(Out, 4);
419 Offset off = (Offset) Out.tell();
Ted Kremenekfb645b62008-12-11 23:36:38 +0000420
421 // Keep track of matching '#if' ... '#endif'.
422 typedef std::vector<std::pair<Offset, unsigned> > PPCondTable;
423 PPCondTable PPCond;
Ted Kremenekdad7b342008-12-12 18:31:09 +0000424 std::vector<unsigned> PPStartCond;
Ted Kremeneke5680f32008-12-23 01:30:52 +0000425 bool ParsingPreprocessorDirective = false;
Ted Kremeneka3d764c2008-11-26 03:36:26 +0000426 Token Tok;
427
428 do {
429 L.LexFromRawLexer(Tok);
430
Ted Kremeneke5680f32008-12-23 01:30:52 +0000431 if ((Tok.isAtStartOfLine() || Tok.is(tok::eof)) &&
432 ParsingPreprocessorDirective) {
433 // Insert an eom token into the token cache. It has the same
434 // position as the next token that is not on the same line as the
435 // preprocessor directive. Observe that we continue processing
436 // 'Tok' when we exit this branch.
437 Token Tmp = Tok;
438 Tmp.setKind(tok::eom);
439 Tmp.clearFlag(Token::StartOfLine);
440 Tmp.setIdentifierInfo(0);
Ted Kremenekb978c662009-01-08 01:17:37 +0000441 EmitToken(Tmp);
Ted Kremeneke5680f32008-12-23 01:30:52 +0000442 ParsingPreprocessorDirective = false;
443 }
444
Ted Kremeneka3d764c2008-11-26 03:36:26 +0000445 if (Tok.is(tok::identifier)) {
446 Tok.setIdentifierInfo(PP.LookUpIdentifierInfo(Tok));
Ted Kremeneke5680f32008-12-23 01:30:52 +0000447 continue;
Ted Kremeneka3d764c2008-11-26 03:36:26 +0000448 }
Ted Kremeneke5680f32008-12-23 01:30:52 +0000449
450 if (Tok.is(tok::hash) && Tok.isAtStartOfLine()) {
Ted Kremeneka3d764c2008-11-26 03:36:26 +0000451 // Special processing for #include. Store the '#' token and lex
452 // the next token.
Ted Kremeneke5680f32008-12-23 01:30:52 +0000453 assert(!ParsingPreprocessorDirective);
Ted Kremenekfb645b62008-12-11 23:36:38 +0000454 Offset HashOff = (Offset) Out.tell();
Ted Kremenekb978c662009-01-08 01:17:37 +0000455 EmitToken(Tok);
Ted Kremenekdad7b342008-12-12 18:31:09 +0000456
457 // Get the next token.
Ted Kremeneka3d764c2008-11-26 03:36:26 +0000458 L.LexFromRawLexer(Tok);
Ted Kremeneke5680f32008-12-23 01:30:52 +0000459
460 assert(!Tok.isAtStartOfLine());
Ted Kremeneka3d764c2008-11-26 03:36:26 +0000461
462 // Did we see 'include'/'import'/'include_next'?
463 if (!Tok.is(tok::identifier))
464 continue;
465
466 IdentifierInfo* II = PP.LookUpIdentifierInfo(Tok);
467 Tok.setIdentifierInfo(II);
468 tok::PPKeywordKind K = II->getPPKeywordID();
469
Ted Kremeneke5680f32008-12-23 01:30:52 +0000470 assert(K != tok::pp_not_keyword);
471 ParsingPreprocessorDirective = true;
472
473 switch (K) {
474 default:
475 break;
476 case tok::pp_include:
477 case tok::pp_import:
478 case tok::pp_include_next: {
Ted Kremeneka3d764c2008-11-26 03:36:26 +0000479 // Save the 'include' token.
Ted Kremenekb978c662009-01-08 01:17:37 +0000480 EmitToken(Tok);
Ted Kremeneka3d764c2008-11-26 03:36:26 +0000481 // Lex the next token as an include string.
482 L.setParsingPreprocessorDirective(true);
483 L.LexIncludeFilename(Tok);
484 L.setParsingPreprocessorDirective(false);
Ted Kremeneke5680f32008-12-23 01:30:52 +0000485 assert(!Tok.isAtStartOfLine());
Ted Kremeneka3d764c2008-11-26 03:36:26 +0000486 if (Tok.is(tok::identifier))
487 Tok.setIdentifierInfo(PP.LookUpIdentifierInfo(Tok));
Ted Kremeneke5680f32008-12-23 01:30:52 +0000488
489 break;
Ted Kremeneka3d764c2008-11-26 03:36:26 +0000490 }
Ted Kremeneke5680f32008-12-23 01:30:52 +0000491 case tok::pp_if:
492 case tok::pp_ifdef:
493 case tok::pp_ifndef: {
Ted Kremenekfb645b62008-12-11 23:36:38 +0000494 // Ad an entry for '#if' and friends. We initially set the target index
495 // to 0. This will get backpatched when we hit #endif.
496 PPStartCond.push_back(PPCond.size());
Ted Kremenekdad7b342008-12-12 18:31:09 +0000497 PPCond.push_back(std::make_pair(HashOff, 0U));
Ted Kremeneke5680f32008-12-23 01:30:52 +0000498 break;
Ted Kremenekfb645b62008-12-11 23:36:38 +0000499 }
Ted Kremeneke5680f32008-12-23 01:30:52 +0000500 case tok::pp_endif: {
Ted Kremenekfb645b62008-12-11 23:36:38 +0000501 // Add an entry for '#endif'. We set the target table index to itself.
Ted Kremenekdad7b342008-12-12 18:31:09 +0000502 // This will later be set to zero when emitting to the PTH file. We
503 // use 0 for uninitialized indices because that is easier to debug.
Ted Kremenekfb645b62008-12-11 23:36:38 +0000504 unsigned index = PPCond.size();
Ted Kremenekfb645b62008-12-11 23:36:38 +0000505 // Backpatch the opening '#if' entry.
Ted Kremenekdad7b342008-12-12 18:31:09 +0000506 assert(!PPStartCond.empty());
507 assert(PPCond.size() > PPStartCond.back());
Ted Kremenekfb645b62008-12-11 23:36:38 +0000508 assert(PPCond[PPStartCond.back()].second == 0);
509 PPCond[PPStartCond.back()].second = index;
510 PPStartCond.pop_back();
Ted Kremenekdad7b342008-12-12 18:31:09 +0000511 // Add the new entry to PPCond.
512 PPCond.push_back(std::make_pair(HashOff, index));
Ted Kremeneke5680f32008-12-23 01:30:52 +0000513 break;
Ted Kremenekfb645b62008-12-11 23:36:38 +0000514 }
Ted Kremeneke5680f32008-12-23 01:30:52 +0000515 case tok::pp_elif:
516 case tok::pp_else: {
517 // Add an entry for #elif or #else.
Ted Kremenekdad7b342008-12-12 18:31:09 +0000518 // This serves as both a closing and opening of a conditional block.
519 // This means that its entry will get backpatched later.
Ted Kremenekfb645b62008-12-11 23:36:38 +0000520 unsigned index = PPCond.size();
Ted Kremenekfb645b62008-12-11 23:36:38 +0000521 // Backpatch the previous '#if' entry.
Ted Kremenekdad7b342008-12-12 18:31:09 +0000522 assert(!PPStartCond.empty());
523 assert(PPCond.size() > PPStartCond.back());
Ted Kremenekfb645b62008-12-11 23:36:38 +0000524 assert(PPCond[PPStartCond.back()].second == 0);
525 PPCond[PPStartCond.back()].second = index;
526 PPStartCond.pop_back();
527 // Now add '#elif' as a new block opening.
Ted Kremenekdad7b342008-12-12 18:31:09 +0000528 PPCond.push_back(std::make_pair(HashOff, 0U));
529 PPStartCond.push_back(index);
Ted Kremeneke5680f32008-12-23 01:30:52 +0000530 break;
531 }
Ted Kremenekfb645b62008-12-11 23:36:38 +0000532 }
Ted Kremeneka3d764c2008-11-26 03:36:26 +0000533 }
534 }
Ted Kremenekb978c662009-01-08 01:17:37 +0000535 while (EmitToken(Tok), Tok.isNot(tok::eof));
536
Ted Kremenekdad7b342008-12-12 18:31:09 +0000537 assert(PPStartCond.empty() && "Error: imblanced preprocessor conditionals.");
Ted Kremenekb978c662009-01-08 01:17:37 +0000538
Ted Kremenekfb645b62008-12-11 23:36:38 +0000539 // Next write out PPCond.
540 Offset PPCondOff = (Offset) Out.tell();
Ted Kremenekdad7b342008-12-12 18:31:09 +0000541
542 // Write out the size of PPCond so that clients can identifer empty tables.
Ted Kremenekb978c662009-01-08 01:17:37 +0000543 Emit32(PPCond.size());
Ted Kremenekfc7e2ea2008-12-02 19:44:08 +0000544
Ted Kremenekdad7b342008-12-12 18:31:09 +0000545 for (unsigned i = 0, e = PPCond.size(); i!=e; ++i) {
Ted Kremenekb978c662009-01-08 01:17:37 +0000546 Emit32(PPCond[i].first - off);
Ted Kremenekdad7b342008-12-12 18:31:09 +0000547 uint32_t x = PPCond[i].second;
548 assert(x != 0 && "PPCond entry not backpatched.");
549 // Emit zero for #endifs. This allows us to do checking when
550 // we read the PTH file back in.
Ted Kremenekb978c662009-01-08 01:17:37 +0000551 Emit32(x == i ? 0 : x);
Ted Kremenekfb645b62008-12-11 23:36:38 +0000552 }
553
Ted Kremenek277faca2009-01-27 00:01:05 +0000554 return PCHEntry(off, PPCondOff);
Ted Kremenekbe295332009-01-08 02:44:06 +0000555}
556
Ted Kremenek277faca2009-01-27 00:01:05 +0000557Offset PTHWriter::EmitCachedSpellings() {
558 // Write each cached strings to the PTH file.
559 Offset SpellingsOff = Out.tell();
560
561 for (std::vector<llvm::StringMapEntry<OffsetOpt>*>::iterator
562 I = StrEntries.begin(), E = StrEntries.end(); I!=E; ++I) {
Ted Kremenekbe295332009-01-08 02:44:06 +0000563
Ted Kremenek277faca2009-01-27 00:01:05 +0000564 const char* data = (*I)->getKeyData();
565 EmitBuf(data, data + (*I)->getKeyLength());
566 Emit8('\0');
Ted Kremenekbe295332009-01-08 02:44:06 +0000567 }
568
Ted Kremenek277faca2009-01-27 00:01:05 +0000569 return SpellingsOff;
Ted Kremeneka3d764c2008-11-26 03:36:26 +0000570}
Ted Kremenek85888962008-10-21 00:54:44 +0000571
Ted Kremenekb978c662009-01-08 01:17:37 +0000572void PTHWriter::GeneratePTH() {
Ted Kremeneke1b64982009-01-26 21:43:14 +0000573 // Generate the prologue.
574 Out << "cfe-pth";
Ted Kremenek67d15052009-01-26 21:50:21 +0000575 Emit32(PTHManager::Version);
Ted Kremeneke1b64982009-01-26 21:43:14 +0000576 Offset JumpOffset = Out.tell();
577 Emit32(0);
578
Ted Kremenek85888962008-10-21 00:54:44 +0000579 // Iterate over all the files in SourceManager. Create a lexer
580 // for each file and cache the tokens.
Chris Lattnerc6fe32a2009-01-17 03:48:08 +0000581 SourceManager &SM = PP.getSourceManager();
582 const LangOptions &LOpts = PP.getLangOptions();
Ted Kremenek85888962008-10-21 00:54:44 +0000583
Chris Lattnerc6fe32a2009-01-17 03:48:08 +0000584 for (SourceManager::fileinfo_iterator I = SM.fileinfo_begin(),
585 E = SM.fileinfo_end(); I != E; ++I) {
Chris Lattner0d0bf8c2009-02-03 07:30:45 +0000586 const SrcMgr::ContentCache &C = *I->second;
Chris Lattnerc6fe32a2009-01-17 03:48:08 +0000587 const FileEntry *FE = C.Entry;
Ted Kremenekfc7e2ea2008-12-02 19:44:08 +0000588
589 // FIXME: Handle files with non-absolute paths.
590 llvm::sys::Path P(FE->getName());
591 if (!P.isAbsolute())
592 continue;
Ted Kremenek85888962008-10-21 00:54:44 +0000593
Ted Kremenekd8c02922009-02-10 22:16:22 +0000594 // assert(!PM.count(FE) && "fileinfo's are not uniqued on FileEntry?");
Ted Kremeneka3d764c2008-11-26 03:36:26 +0000595
Chris Lattnerc6fe32a2009-01-17 03:48:08 +0000596 const llvm::MemoryBuffer *B = C.getBuffer();
Ted Kremeneka3d764c2008-11-26 03:36:26 +0000597 if (!B) continue;
Ted Kremenekfb645b62008-12-11 23:36:38 +0000598
Chris Lattner2b2453a2009-01-17 06:22:33 +0000599 FileID FID = SM.createFileID(FE, SourceLocation(), SrcMgr::C_User);
Chris Lattner025c3a62009-01-17 07:35:14 +0000600 Lexer L(FID, SM, LOpts);
Ted Kremenekd8c02922009-02-10 22:16:22 +0000601 PM.insert(FE, LexTokens(L));
Daniel Dunbar31309ab2008-11-26 02:18:33 +0000602 }
Ted Kremeneka3d764c2008-11-26 03:36:26 +0000603
604 // Write out the identifier table.
Ted Kremenek293b4af2009-01-15 01:26:25 +0000605 const std::pair<Offset, std::pair<Offset,Offset> >& IdTableOff
606 = EmitIdentifierTable();
Ted Kremenek85888962008-10-21 00:54:44 +0000607
Ted Kremenekbe295332009-01-08 02:44:06 +0000608 // Write out the cached strings table.
Ted Kremenek277faca2009-01-27 00:01:05 +0000609 Offset SpellingOff = EmitCachedSpellings();
Ted Kremenekbe295332009-01-08 02:44:06 +0000610
Ted Kremeneka3d764c2008-11-26 03:36:26 +0000611 // Write out the file table.
Ted Kremenekb978c662009-01-08 01:17:37 +0000612 Offset FileTableOff = EmitFileTable();
Ted Kremeneka3d764c2008-11-26 03:36:26 +0000613
614 // Finally, write out the offset table at the end.
Ted Kremeneke1b64982009-01-26 21:43:14 +0000615 Offset JumpTargetOffset = Out.tell();
Ted Kremenekb978c662009-01-08 01:17:37 +0000616 Emit32(IdTableOff.first);
Ted Kremenek293b4af2009-01-15 01:26:25 +0000617 Emit32(IdTableOff.second.first);
618 Emit32(IdTableOff.second.second);
Ted Kremenekb978c662009-01-08 01:17:37 +0000619 Emit32(FileTableOff);
Ted Kremenek277faca2009-01-27 00:01:05 +0000620 Emit32(SpellingOff);
Ted Kremeneke1b64982009-01-26 21:43:14 +0000621
622 // Now write the offset in the prologue.
623 Out.seek(JumpOffset);
624 Emit32(JumpTargetOffset);
Ted Kremenekb978c662009-01-08 01:17:37 +0000625}
626
627void clang::CacheTokens(Preprocessor& PP, const std::string& OutFile) {
628 // Lex through the entire file. This will populate SourceManager with
629 // all of the header information.
630 Token Tok;
631 PP.EnterMainSourceFile();
632 do { PP.Lex(Tok); } while (Tok.isNot(tok::eof));
633
634 // Open up the PTH file.
635 std::string ErrMsg;
636 llvm::raw_fd_ostream Out(OutFile.c_str(), true, ErrMsg);
637
638 if (!ErrMsg.empty()) {
639 llvm::errs() << "PTH error: " << ErrMsg << "\n";
640 return;
641 }
642
643 // Create the PTHWriter and generate the PTH file.
644 PTHWriter PW(Out, PP);
645 PW.GeneratePTH();
Ted Kremenek85888962008-10-21 00:54:44 +0000646}
Ted Kremeneke0ea5dc2009-02-10 01:06:17 +0000647
Ted Kremeneke0ea5dc2009-02-10 01:06:17 +0000648
649//===----------------------------------------------------------------------===//
650// Client code of on-disk hashtable logic.
651//===----------------------------------------------------------------------===//
652
653Offset PTHWriter::EmitFileTable() {
Ted Kremenekd8c02922009-02-10 22:16:22 +0000654 return PM.Emit(Out);
Ted Kremeneke0ea5dc2009-02-10 01:06:17 +0000655}