blob: b2870f46fdbfcb59f6ac8a47763311aee7261021 [file] [log] [blame]
Ted Kremenek274b2082008-11-12 21:37:15 +00001//===--- PTHLexer.cpp - Lex from a token stream ---------------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the PTHLexer interface.
11//
12//===----------------------------------------------------------------------===//
13
Ted Kremenek0c6a77b2008-12-03 00:38:03 +000014#include "clang/Basic/TokenKinds.h"
15#include "clang/Basic/FileManager.h"
16#include "clang/Basic/IdentifierTable.h"
Ted Kremenek274b2082008-11-12 21:37:15 +000017#include "clang/Lex/PTHLexer.h"
18#include "clang/Lex/Preprocessor.h"
Ted Kremenek0c6a77b2008-12-03 00:38:03 +000019#include "clang/Lex/PTHManager.h"
20#include "clang/Lex/Token.h"
21#include "clang/Lex/Preprocessor.h"
22#include "llvm/Support/Compiler.h"
23#include "llvm/Support/MemoryBuffer.h"
24#include "llvm/ADT/StringMap.h"
25#include "llvm/ADT/OwningPtr.h"
Ted Kremenek274b2082008-11-12 21:37:15 +000026using namespace clang;
27
Ted Kremenek18d9afb2008-12-23 18:41:34 +000028#define DISK_TOKEN_SIZE (1+1+3+4+2)
Ted Kremenek268ee702008-12-12 18:34:08 +000029
Ted Kremenek0c6a77b2008-12-03 00:38:03 +000030//===----------------------------------------------------------------------===//
31// Utility methods for reading from the mmap'ed PTH file.
32//===----------------------------------------------------------------------===//
33
34static inline uint8_t Read8(const char*& data) {
35 return (uint8_t) *(data++);
36}
37
38static inline uint32_t Read32(const char*& data) {
39 uint32_t V = (uint32_t) Read8(data);
40 V |= (((uint32_t) Read8(data)) << 8);
41 V |= (((uint32_t) Read8(data)) << 16);
42 V |= (((uint32_t) Read8(data)) << 24);
43 return V;
44}
45
Ted Kremeneke5680f32008-12-23 01:30:52 +000046//===----------------------------------------------------------------------===//
47// PTHLexer methods.
48//===----------------------------------------------------------------------===//
49
Chris Lattner2b2453a2009-01-17 06:22:33 +000050PTHLexer::PTHLexer(Preprocessor &PP, FileID FID, const char *D,
51 const char *ppcond,
52 PTHSpellingSearch &mySpellingSrch, PTHManager &PM)
53 : PreprocessorLexer(&PP, FID), TokBuf(D), CurPtr(D), LastHashTokPtr(0),
Ted Kremenek5f074262009-01-09 22:05:30 +000054 PPCond(ppcond), CurPPCondPtr(ppcond), MySpellingSrch(mySpellingSrch),
Chris Lattner2b2453a2009-01-17 06:22:33 +000055 PTHMgr(PM) {
56
57 FileStartLoc = PP.getSourceManager().getLocForStartOfFile(FID);
Ted Kremenek5f074262009-01-09 22:05:30 +000058}
Ted Kremeneke5680f32008-12-23 01:30:52 +000059
60void PTHLexer::Lex(Token& Tok) {
61LexNextToken:
Ted Kremeneke5680f32008-12-23 01:30:52 +000062
Ted Kremenek866bdf72008-12-23 02:30:15 +000063 //===--------------------------------------==//
64 // Read the raw token data.
65 //===--------------------------------------==//
66
67 // Shadow CurPtr into an automatic variable.
68 const unsigned char *CurPtrShadow = (const unsigned char*) CurPtr;
69
70 // Read in the data for the token. 14 bytes in total.
71 tok::TokenKind k = (tok::TokenKind) CurPtrShadow[0];
72 Token::TokenFlags flags = (Token::TokenFlags) CurPtrShadow[1];
73
Ted Kremenek6b1c9702008-12-23 18:27:26 +000074 uint32_t perID = ((uint32_t) CurPtrShadow[2])
Ted Kremenek866bdf72008-12-23 02:30:15 +000075 | (((uint32_t) CurPtrShadow[3]) << 8)
Ted Kremenek18d9afb2008-12-23 18:41:34 +000076 | (((uint32_t) CurPtrShadow[4]) << 16);
Ted Kremenek866bdf72008-12-23 02:30:15 +000077
Ted Kremenek18d9afb2008-12-23 18:41:34 +000078 uint32_t FileOffset = ((uint32_t) CurPtrShadow[5])
79 | (((uint32_t) CurPtrShadow[6]) << 8)
80 | (((uint32_t) CurPtrShadow[7]) << 16)
81 | (((uint32_t) CurPtrShadow[8]) << 24);
Ted Kremenek866bdf72008-12-23 02:30:15 +000082
Ted Kremenek18d9afb2008-12-23 18:41:34 +000083 uint32_t Len = ((uint32_t) CurPtrShadow[9])
84 | (((uint32_t) CurPtrShadow[10]) << 8);
Ted Kremenek866bdf72008-12-23 02:30:15 +000085
86 CurPtr = (const char*) (CurPtrShadow + DISK_TOKEN_SIZE);
87
88 //===--------------------------------------==//
89 // Construct the token itself.
90 //===--------------------------------------==//
91
92 Tok.startToken();
93 Tok.setKind(k);
Ted Kremenek59d08cb2008-12-23 19:24:24 +000094 Tok.setFlag(flags);
95 assert(!LexingRawMode);
Ted Kremenek6b1c9702008-12-23 18:27:26 +000096 Tok.setIdentifierInfo(perID ? PTHMgr.GetIdentifierInfo(perID-1) : 0);
Chris Lattner2b2453a2009-01-17 06:22:33 +000097 Tok.setLocation(FileStartLoc.getFileLocWithOffset(FileOffset));
Ted Kremenek866bdf72008-12-23 02:30:15 +000098 Tok.setLength(Len);
99
100 //===--------------------------------------==//
101 // Process the token.
102 //===--------------------------------------==//
Ted Kremenek5f074262009-01-09 22:05:30 +0000103#if 0
104 SourceManager& SM = PP->getSourceManager();
105 llvm::cerr << SM.getFileEntryForID(FileID)->getName()
106 << ':' << SM.getLogicalLineNumber(Tok.getLocation())
107 << ':' << SM.getLogicalColumnNumber(Tok.getLocation())
108 << '\n';
109#endif
Ted Kremeneke5680f32008-12-23 01:30:52 +0000110
Ted Kremenek59d08cb2008-12-23 19:24:24 +0000111 if (k == tok::identifier) {
112 MIOpt.ReadToken();
113 return PP->HandleIdentifier(Tok);
114 }
115
116 if (k == tok::eof) {
Ted Kremeneke5680f32008-12-23 01:30:52 +0000117 // Save the end-of-file token.
118 EofToken = Tok;
119
120 Preprocessor *PPCache = PP;
Ted Kremenek59d08cb2008-12-23 19:24:24 +0000121
122 assert(!ParsingPreprocessorDirective);
123 assert(!LexingRawMode);
124
125 // FIXME: Issue diagnostics similar to Lexer.
126 if (PP->HandleEndOfFile(Tok, false))
Ted Kremeneke5680f32008-12-23 01:30:52 +0000127 return;
Ted Kremenek59d08cb2008-12-23 19:24:24 +0000128
Ted Kremeneke5680f32008-12-23 01:30:52 +0000129 assert(PPCache && "Raw buffer::LexEndOfFile should return a token");
130 return PPCache->Lex(Tok);
131 }
Ted Kremeneke5680f32008-12-23 01:30:52 +0000132
Ted Kremenek59d08cb2008-12-23 19:24:24 +0000133 if (k == tok::hash && Tok.isAtStartOfLine()) {
134 LastHashTokPtr = CurPtr - DISK_TOKEN_SIZE;
135 assert(!LexingRawMode);
136 PP->HandleDirective(Tok);
137
138 if (PP->isCurrentLexer(this))
139 goto LexNextToken;
140
141 return PP->Lex(Tok);
142 }
143
144 if (k == tok::eom) {
145 assert(ParsingPreprocessorDirective);
Ted Kremeneke5680f32008-12-23 01:30:52 +0000146 ParsingPreprocessorDirective = false;
147 return;
148 }
Ted Kremeneke5680f32008-12-23 01:30:52 +0000149
Ted Kremenek59d08cb2008-12-23 19:24:24 +0000150 MIOpt.ReadToken();
Ted Kremeneke5680f32008-12-23 01:30:52 +0000151}
152
153// FIXME: We can just grab the last token instead of storing a copy
154// into EofToken.
Ted Kremenek59d08cb2008-12-23 19:24:24 +0000155void PTHLexer::getEOF(Token& Tok) {
Ted Kremenekdefb7092009-01-09 00:36:11 +0000156 assert(EofToken.is(tok::eof));
Ted Kremeneke5680f32008-12-23 01:30:52 +0000157 Tok = EofToken;
158}
159
160void PTHLexer::DiscardToEndOfLine() {
161 assert(ParsingPreprocessorDirective && ParsingFilename == false &&
162 "Must be in a preprocessing directive!");
163
164 // We assume that if the preprocessor wishes to discard to the end of
165 // the line that it also means to end the current preprocessor directive.
166 ParsingPreprocessorDirective = false;
167
168 // Skip tokens by only peeking at their token kind and the flags.
169 // We don't need to actually reconstruct full tokens from the token buffer.
170 // This saves some copies and it also reduces IdentifierInfo* lookup.
171 const char* p = CurPtr;
172 while (1) {
173 // Read the token kind. Are we at the end of the file?
174 tok::TokenKind x = (tok::TokenKind) (uint8_t) *p;
175 if (x == tok::eof) break;
176
177 // Read the token flags. Are we at the start of the next line?
178 Token::TokenFlags y = (Token::TokenFlags) (uint8_t) p[1];
179 if (y & Token::StartOfLine) break;
180
181 // Skip to the next token.
182 p += DISK_TOKEN_SIZE;
183 }
184
185 CurPtr = p;
186}
187
Ted Kremenek268ee702008-12-12 18:34:08 +0000188/// SkipBlock - Used by Preprocessor to skip the current conditional block.
189bool PTHLexer::SkipBlock() {
190 assert(CurPPCondPtr && "No cached PP conditional information.");
191 assert(LastHashTokPtr && "No known '#' token.");
192
Ted Kremenek41a26602008-12-12 22:05:38 +0000193 const char* HashEntryI = 0;
Ted Kremenek268ee702008-12-12 18:34:08 +0000194 uint32_t Offset;
195 uint32_t TableIdx;
196
197 do {
Ted Kremenek41a26602008-12-12 22:05:38 +0000198 // Read the token offset from the side-table.
Ted Kremenek268ee702008-12-12 18:34:08 +0000199 Offset = Read32(CurPPCondPtr);
Ted Kremenek41a26602008-12-12 22:05:38 +0000200
201 // Read the target table index from the side-table.
Ted Kremenek268ee702008-12-12 18:34:08 +0000202 TableIdx = Read32(CurPPCondPtr);
Ted Kremenek41a26602008-12-12 22:05:38 +0000203
204 // Compute the actual memory address of the '#' token data for this entry.
205 HashEntryI = TokBuf + Offset;
206
207 // Optmization: "Sibling jumping". #if...#else...#endif blocks can
208 // contain nested blocks. In the side-table we can jump over these
209 // nested blocks instead of doing a linear search if the next "sibling"
210 // entry is not at a location greater than LastHashTokPtr.
211 if (HashEntryI < LastHashTokPtr && TableIdx) {
212 // In the side-table we are still at an entry for a '#' token that
213 // is earlier than the last one we saw. Check if the location we would
214 // stride gets us closer.
215 const char* NextPPCondPtr = PPCond + TableIdx*(sizeof(uint32_t)*2);
216 assert(NextPPCondPtr >= CurPPCondPtr);
217 // Read where we should jump to.
218 uint32_t TmpOffset = Read32(NextPPCondPtr);
219 const char* HashEntryJ = TokBuf + TmpOffset;
220
221 if (HashEntryJ <= LastHashTokPtr) {
222 // Jump directly to the next entry in the side table.
223 HashEntryI = HashEntryJ;
224 Offset = TmpOffset;
225 TableIdx = Read32(NextPPCondPtr);
226 CurPPCondPtr = NextPPCondPtr;
227 }
228 }
Ted Kremenek268ee702008-12-12 18:34:08 +0000229 }
Ted Kremenek41a26602008-12-12 22:05:38 +0000230 while (HashEntryI < LastHashTokPtr);
231 assert(HashEntryI == LastHashTokPtr && "No PP-cond entry found for '#'");
Ted Kremenek268ee702008-12-12 18:34:08 +0000232 assert(TableIdx && "No jumping from #endifs.");
233
234 // Update our side-table iterator.
235 const char* NextPPCondPtr = PPCond + TableIdx*(sizeof(uint32_t)*2);
236 assert(NextPPCondPtr >= CurPPCondPtr);
237 CurPPCondPtr = NextPPCondPtr;
238
239 // Read where we should jump to.
Ted Kremenek41a26602008-12-12 22:05:38 +0000240 HashEntryI = TokBuf + Read32(NextPPCondPtr);
Ted Kremenek268ee702008-12-12 18:34:08 +0000241 uint32_t NextIdx = Read32(NextPPCondPtr);
242
243 // By construction NextIdx will be zero if this is a #endif. This is useful
244 // to know to obviate lexing another token.
245 bool isEndif = NextIdx == 0;
Ted Kremenek268ee702008-12-12 18:34:08 +0000246
247 // This case can occur when we see something like this:
248 //
249 // #if ...
250 // /* a comment or nothing */
251 // #elif
252 //
253 // If we are skipping the first #if block it will be the case that CurPtr
254 // already points 'elif'. Just return.
255
Ted Kremenek41a26602008-12-12 22:05:38 +0000256 if (CurPtr > HashEntryI) {
257 assert(CurPtr == HashEntryI + DISK_TOKEN_SIZE);
Ted Kremenek268ee702008-12-12 18:34:08 +0000258 // Did we reach a #endif? If so, go ahead and consume that token as well.
259 if (isEndif)
Ted Kremeneke5680f32008-12-23 01:30:52 +0000260 CurPtr += DISK_TOKEN_SIZE*2;
Ted Kremenek268ee702008-12-12 18:34:08 +0000261 else
Ted Kremenek41a26602008-12-12 22:05:38 +0000262 LastHashTokPtr = HashEntryI;
Ted Kremenek268ee702008-12-12 18:34:08 +0000263
264 return isEndif;
265 }
266
267 // Otherwise, we need to advance. Update CurPtr to point to the '#' token.
Ted Kremenek41a26602008-12-12 22:05:38 +0000268 CurPtr = HashEntryI;
Ted Kremenek268ee702008-12-12 18:34:08 +0000269
270 // Update the location of the last observed '#'. This is useful if we
271 // are skipping multiple blocks.
272 LastHashTokPtr = CurPtr;
Ted Kremenek268ee702008-12-12 18:34:08 +0000273
Ted Kremeneke5680f32008-12-23 01:30:52 +0000274 // Skip the '#' token.
275 assert(((tok::TokenKind) (unsigned char) *CurPtr) == tok::hash);
276 CurPtr += DISK_TOKEN_SIZE;
277
Ted Kremenek268ee702008-12-12 18:34:08 +0000278 // Did we reach a #endif? If so, go ahead and consume that token as well.
Ted Kremeneke5680f32008-12-23 01:30:52 +0000279 if (isEndif) { CurPtr += DISK_TOKEN_SIZE*2; }
Ted Kremenek268ee702008-12-12 18:34:08 +0000280
281 return isEndif;
282}
283
Ted Kremenek30a12ec2008-12-17 23:36:32 +0000284SourceLocation PTHLexer::getSourceLocation() {
285 // getLocation is not on the hot path. It is used to get the location of
286 // the next token when transitioning back to this lexer when done
287 // handling a #included file. Just read the necessary data from the token
288 // data buffer to construct the SourceLocation object.
289 // NOTE: This is a virtual function; hence it is defined out-of-line.
Ted Kremenek59d08cb2008-12-23 19:24:24 +0000290 const char* p = CurPtr + (1 + 1 + 3);
Ted Kremenek30a12ec2008-12-17 23:36:32 +0000291 uint32_t offset =
292 ((uint32_t) ((uint8_t) p[0]))
293 | (((uint32_t) ((uint8_t) p[1])) << 8)
294 | (((uint32_t) ((uint8_t) p[2])) << 16)
295 | (((uint32_t) ((uint8_t) p[3])) << 24);
Chris Lattner2b2453a2009-01-17 06:22:33 +0000296 return FileStartLoc.getFileLocWithOffset(offset);
Ted Kremenek30a12ec2008-12-17 23:36:32 +0000297}
298
Ted Kremenek5f074262009-01-09 22:05:30 +0000299//===----------------------------------------------------------------------===//
300// getSpelling() - Use cached data in PTH files for getSpelling().
301//===----------------------------------------------------------------------===//
302
Chris Lattner2b2453a2009-01-17 06:22:33 +0000303unsigned PTHManager::getSpelling(FileID FID, unsigned FPos,
304 const char *&Buffer) {
305 llvm::DenseMap<FileID, PTHSpellingSearch*>::iterator I =SpellingMap.find(FID);
Ted Kremenek5f074262009-01-09 22:05:30 +0000306
307 if (I == SpellingMap.end())
308 return 0;
Ted Kremenekf02f6f02009-01-13 22:05:50 +0000309
Chris Lattner2b2453a2009-01-17 06:22:33 +0000310 return I->second->getSpellingBinarySearch(FPos, Buffer);
311}
312
313unsigned PTHManager::getSpelling(SourceLocation Loc, const char *&Buffer) {
314 std::pair<FileID, unsigned> LocInfo =
315 PP->getSourceManager().getDecomposedFileLoc(Loc);
316 return getSpelling(LocInfo.first, LocInfo.second, Buffer);
Ted Kremenek5f074262009-01-09 22:05:30 +0000317}
318
319unsigned PTHManager::getSpellingAtPTHOffset(unsigned PTHOffset,
320 const char *& Buffer) {
321
Ted Kremenek32a8ad52009-01-08 04:30:32 +0000322 const char* p = Buf->getBufferStart() + PTHOffset;
323 assert(p < Buf->getBufferEnd());
324
325 // The string is prefixed by 16 bits for its length, followed by the string
326 // itself.
327 unsigned len = ((unsigned) ((uint8_t) p[0]))
328 | (((unsigned) ((uint8_t) p[1])) << 8);
329
330 Buffer = p + 2;
331 return len;
332}
333
Ted Kremenek5f074262009-01-09 22:05:30 +0000334unsigned PTHSpellingSearch::getSpellingLinearSearch(unsigned fpos,
335 const char *&Buffer) {
336 const char* p = LinearItr;
Ted Kremenek32a8ad52009-01-08 04:30:32 +0000337 unsigned len = 0;
Ted Kremenek5f074262009-01-09 22:05:30 +0000338
Ted Kremenekf02f6f02009-01-13 22:05:50 +0000339 if (p == TableEnd)
Ted Kremenek5f074262009-01-09 22:05:30 +0000340 return getSpellingBinarySearch(fpos, Buffer);
341
342 do {
Ted Kremenek32a8ad52009-01-08 04:30:32 +0000343 uint32_t TokOffset =
344 ((uint32_t) ((uint8_t) p[0]))
345 | (((uint32_t) ((uint8_t) p[1])) << 8)
346 | (((uint32_t) ((uint8_t) p[2])) << 16)
347 | (((uint32_t) ((uint8_t) p[3])) << 24);
348
349 if (TokOffset > fpos)
Ted Kremenek5f074262009-01-09 22:05:30 +0000350 return getSpellingBinarySearch(fpos, Buffer);
Ted Kremenek32a8ad52009-01-08 04:30:32 +0000351
Ted Kremenek32a8ad52009-01-08 04:30:32 +0000352 // Did we find a matching token offset for this spelling?
353 if (TokOffset == fpos) {
354 uint32_t SpellingPTHOffset =
355 ((uint32_t) ((uint8_t) p[4]))
356 | (((uint32_t) ((uint8_t) p[5])) << 8)
357 | (((uint32_t) ((uint8_t) p[6])) << 16)
358 | (((uint32_t) ((uint8_t) p[7])) << 24);
359
Ted Kremenekf02f6f02009-01-13 22:05:50 +0000360 p += SpellingEntrySize;
Ted Kremenek5f074262009-01-09 22:05:30 +0000361 len = PTHMgr.getSpellingAtPTHOffset(SpellingPTHOffset, Buffer);
Ted Kremenek32a8ad52009-01-08 04:30:32 +0000362 break;
363 }
364
365 // No match. Keep on looking.
Ted Kremenekf02f6f02009-01-13 22:05:50 +0000366 p += SpellingEntrySize;
Ted Kremenek32a8ad52009-01-08 04:30:32 +0000367 }
Ted Kremenekf02f6f02009-01-13 22:05:50 +0000368 while (p != TableEnd);
Ted Kremenek32a8ad52009-01-08 04:30:32 +0000369
Ted Kremenek5f074262009-01-09 22:05:30 +0000370 LinearItr = p;
Ted Kremenek32a8ad52009-01-08 04:30:32 +0000371 return len;
Ted Kremenekb70e3da2009-01-08 02:47:16 +0000372}
373
Ted Kremenek5f074262009-01-09 22:05:30 +0000374unsigned PTHSpellingSearch::getSpellingBinarySearch(unsigned fpos,
375 const char *& Buffer) {
376
Ted Kremenekf02f6f02009-01-13 22:05:50 +0000377 assert((TableEnd - TableBeg) % SpellingEntrySize == 0);
378
379 if (TableEnd == TableBeg)
380 return 0;
381
382 assert(TableEnd > TableBeg);
Ted Kremenek5f074262009-01-09 22:05:30 +0000383
384 unsigned min = 0;
385 const char* tb = TableBeg;
Ted Kremenekf02f6f02009-01-13 22:05:50 +0000386 unsigned max = NumSpellings;
Ted Kremenek5f074262009-01-09 22:05:30 +0000387
Ted Kremenekf02f6f02009-01-13 22:05:50 +0000388 do {
Ted Kremenek5f074262009-01-09 22:05:30 +0000389 unsigned i = (max - min) / 2 + min;
390 const char* p = tb + (i * SpellingEntrySize);
391
392 uint32_t TokOffset =
393 ((uint32_t) ((uint8_t) p[0]))
394 | (((uint32_t) ((uint8_t) p[1])) << 8)
395 | (((uint32_t) ((uint8_t) p[2])) << 16)
396 | (((uint32_t) ((uint8_t) p[3])) << 24);
397
398 if (TokOffset > fpos) {
399 max = i;
Ted Kremenekf02f6f02009-01-13 22:05:50 +0000400 assert(!(max == min) || (min == i));
Ted Kremenek5f074262009-01-09 22:05:30 +0000401 continue;
402 }
403
404 if (TokOffset < fpos) {
Ted Kremenek7bd88aa2009-01-13 22:16:45 +0000405 if (i == min)
406 break;
407
Ted Kremenek5f074262009-01-09 22:05:30 +0000408 min = i;
409 continue;
410 }
411
412 uint32_t SpellingPTHOffset =
413 ((uint32_t) ((uint8_t) p[4]))
414 | (((uint32_t) ((uint8_t) p[5])) << 8)
415 | (((uint32_t) ((uint8_t) p[6])) << 16)
416 | (((uint32_t) ((uint8_t) p[7])) << 24);
417
418 return PTHMgr.getSpellingAtPTHOffset(SpellingPTHOffset, Buffer);
419 }
Ted Kremenekf02f6f02009-01-13 22:05:50 +0000420 while (min != max);
Ted Kremenek5f074262009-01-09 22:05:30 +0000421
422 return 0;
423}
424
Chris Lattner2b2453a2009-01-17 06:22:33 +0000425unsigned PTHLexer::getSpelling(SourceLocation Loc, const char *&Buffer) {
426 SourceManager &SM = PP->getSourceManager();
427 Loc = SM.getSpellingLoc(Loc);
428 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedFileLoc(Loc);
429
430 FileID FID = LocInfo.first;
431 unsigned FPos = LocInfo.second;
Ted Kremenek5f074262009-01-09 22:05:30 +0000432
Chris Lattner2b2453a2009-01-17 06:22:33 +0000433 if (FID == getFileID())
434 return MySpellingSrch.getSpellingLinearSearch(FPos, Buffer);
435 return PTHMgr.getSpelling(FID, FPos, Buffer);
Ted Kremenek5f074262009-01-09 22:05:30 +0000436}
437
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000438//===----------------------------------------------------------------------===//
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000439// Internal Data Structures for PTH file lookup and resolving identifiers.
440//===----------------------------------------------------------------------===//
441
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000442
443/// PTHFileLookup - This internal data structure is used by the PTHManager
444/// to map from FileEntry objects managed by FileManager to offsets within
445/// the PTH file.
446namespace {
447class VISIBILITY_HIDDEN PTHFileLookup {
448public:
449 class Val {
Ted Kremenekfb645b62008-12-11 23:36:38 +0000450 uint32_t TokenOff;
451 uint32_t PPCondOff;
Ted Kremenekb70e3da2009-01-08 02:47:16 +0000452 uint32_t SpellingOff;
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000453
454 public:
Ted Kremenekfb645b62008-12-11 23:36:38 +0000455 Val() : TokenOff(~0) {}
Ted Kremenekb70e3da2009-01-08 02:47:16 +0000456 Val(uint32_t toff, uint32_t poff, uint32_t soff)
457 : TokenOff(toff), PPCondOff(poff), SpellingOff(soff) {}
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000458
Ted Kremenekfb645b62008-12-11 23:36:38 +0000459 uint32_t getTokenOffset() const {
460 assert(TokenOff != ~((uint32_t)0) && "PTHFileLookup entry initialized.");
461 return TokenOff;
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000462 }
463
Ted Kremenekb70e3da2009-01-08 02:47:16 +0000464 uint32_t getPPCondOffset() const {
Ted Kremenekfb645b62008-12-11 23:36:38 +0000465 assert(TokenOff != ~((uint32_t)0) && "PTHFileLookup entry initialized.");
466 return PPCondOff;
467 }
468
Ted Kremenekb70e3da2009-01-08 02:47:16 +0000469 uint32_t getSpellingOffset() const {
470 assert(TokenOff != ~((uint32_t)0) && "PTHFileLookup entry initialized.");
471 return SpellingOff;
472 }
473
Ted Kremenekfb645b62008-12-11 23:36:38 +0000474 bool isValid() const { return TokenOff != ~((uint32_t)0); }
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000475 };
476
477private:
478 llvm::StringMap<Val> FileMap;
479
480public:
481 PTHFileLookup() {};
482
483 Val Lookup(const FileEntry* FE) {
484 const char* s = FE->getName();
485 unsigned size = strlen(s);
486 return FileMap.GetOrCreateValue(s, s+size).getValue();
487 }
488
489 void ReadTable(const char* D) {
490 uint32_t N = Read32(D); // Read the length of the table.
491
492 for ( ; N > 0; --N) { // The rest of the data is the table itself.
493 uint32_t len = Read32(D);
494 const char* s = D;
495 D += len;
Ted Kremenekb70e3da2009-01-08 02:47:16 +0000496
Ted Kremenekfb645b62008-12-11 23:36:38 +0000497 uint32_t TokenOff = Read32(D);
Ted Kremenekb70e3da2009-01-08 02:47:16 +0000498 uint32_t PPCondOff = Read32(D);
499 uint32_t SpellingOff = Read32(D);
500
501 FileMap.GetOrCreateValue(s, s+len).getValue() =
502 Val(TokenOff, PPCondOff, SpellingOff);
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000503 }
504 }
505};
506} // end anonymous namespace
507
508//===----------------------------------------------------------------------===//
509// PTHManager methods.
510//===----------------------------------------------------------------------===//
511
512PTHManager::PTHManager(const llvm::MemoryBuffer* buf, void* fileLookup,
Ted Kremenekcf58e622008-12-10 19:40:23 +0000513 const char* idDataTable, IdentifierInfo** perIDCache,
Ted Kremenek72b1b152009-01-15 18:47:46 +0000514 const char* sortedIdTable, unsigned numIds)
Ted Kremenek6183e482008-12-03 01:16:39 +0000515: Buf(buf), PerIDCache(perIDCache), FileLookup(fileLookup),
Ted Kremenek72b1b152009-01-15 18:47:46 +0000516 IdDataTable(idDataTable), SortedIdTable(sortedIdTable),
517 NumIds(numIds), PP(0) {}
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000518
519PTHManager::~PTHManager() {
520 delete Buf;
521 delete (PTHFileLookup*) FileLookup;
Ted Kremenek0e50b6e2008-12-04 22:47:11 +0000522 free(PerIDCache);
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000523}
524
Ted Kremenek72b1b152009-01-15 18:47:46 +0000525PTHManager* PTHManager::Create(const std::string& file) {
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000526
527 // Memory map the PTH file.
528 llvm::OwningPtr<llvm::MemoryBuffer>
529 File(llvm::MemoryBuffer::getFile(file.c_str()));
530
531 if (!File)
532 return 0;
533
534 // Get the buffer ranges and check if there are at least three 32-bit
535 // words at the end of the file.
536 const char* BufBeg = File->getBufferStart();
537 const char* BufEnd = File->getBufferEnd();
538
539 if(!(BufEnd > BufBeg + sizeof(uint32_t)*3)) {
540 assert(false && "Invalid PTH file.");
541 return 0; // FIXME: Proper error diagnostic?
542 }
543
544 // Compute the address of the index table at the end of the PTH file.
545 // This table contains the offset of the file lookup table, the
546 // persistent ID -> identifer data table.
Ted Kremenek293b4af2009-01-15 01:26:25 +0000547 // FIXME: We should just embed this offset in the PTH file.
548 const char* EndTable = BufEnd - sizeof(uint32_t)*4;
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000549
550 // Construct the file lookup table. This will be used for mapping from
551 // FileEntry*'s to cached tokens.
Ted Kremenek293b4af2009-01-15 01:26:25 +0000552 const char* FileTableOffset = EndTable + sizeof(uint32_t)*3;
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000553 const char* FileTable = BufBeg + Read32(FileTableOffset);
554
555 if (!(FileTable > BufBeg && FileTable < BufEnd)) {
556 assert(false && "Invalid PTH file.");
557 return 0; // FIXME: Proper error diagnostic?
558 }
559
560 llvm::OwningPtr<PTHFileLookup> FL(new PTHFileLookup());
561 FL->ReadTable(FileTable);
562
563 // Get the location of the table mapping from persistent ids to the
564 // data needed to reconstruct identifiers.
565 const char* IDTableOffset = EndTable + sizeof(uint32_t)*1;
566 const char* IData = BufBeg + Read32(IDTableOffset);
567 if (!(IData > BufBeg && IData < BufEnd)) {
568 assert(false && "Invalid PTH file.");
569 return 0; // FIXME: Proper error diagnostic?
570 }
571
Ted Kremenek72b1b152009-01-15 18:47:46 +0000572 // Get the location of the lexigraphically-sorted table of persistent IDs.
573 const char* SortedIdTableOffset = EndTable + sizeof(uint32_t)*2;
574 const char* SortedIdTable = BufBeg + Read32(SortedIdTableOffset);
575 if (!(SortedIdTable > BufBeg && SortedIdTable < BufEnd)) {
576 assert(false && "Invalid PTH file.");
577 return 0; // FIXME: Proper error diagnostic?
578 }
579
Ted Kremenek6183e482008-12-03 01:16:39 +0000580 // Get the number of IdentifierInfos and pre-allocate the identifier cache.
581 uint32_t NumIds = Read32(IData);
582
583 // Pre-allocate the peristent ID -> IdentifierInfo* cache. We use calloc()
584 // so that we in the best case only zero out memory once when the OS returns
585 // us new pages.
586 IdentifierInfo** PerIDCache =
587 (IdentifierInfo**) calloc(NumIds, sizeof(*PerIDCache));
588
589 if (!PerIDCache) {
590 assert(false && "Could not allocate Persistent ID cache.");
591 return 0;
592 }
593
Ted Kremenek72b1b152009-01-15 18:47:46 +0000594 // Create the new PTHManager.
595 return new PTHManager(File.take(), FL.take(), IData, PerIDCache,
596 SortedIdTable, NumIds);
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000597}
598
Ted Kremenek866bdf72008-12-23 02:30:15 +0000599IdentifierInfo* PTHManager::GetIdentifierInfo(unsigned persistentID) {
600
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000601 // Check if the IdentifierInfo has already been resolved.
Ted Kremenek72b1b152009-01-15 18:47:46 +0000602 IdentifierInfo* II = PerIDCache[persistentID];
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000603 if (II) return II;
604
605 // Look in the PTH file for the string data for the IdentifierInfo object.
606 const char* TableEntry = IdDataTable + sizeof(uint32_t) * persistentID;
607 const char* IDData = Buf->getBufferStart() + Read32(TableEntry);
608 assert(IDData < Buf->getBufferEnd());
609
Ted Kremenek72b1b152009-01-15 18:47:46 +0000610 // Allocate the object.
611 std::pair<IdentifierInfo,const char*> *Mem =
612 Alloc.Allocate<std::pair<IdentifierInfo,const char*> >();
613
614 Mem->second = IDData;
615 II = new ((void*) Mem) IdentifierInfo(true);
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000616
Ted Kremenek72b1b152009-01-15 18:47:46 +0000617 // Store the new IdentifierInfo in the cache.
618 PerIDCache[persistentID] = II;
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000619 return II;
620}
621
Ted Kremenek72b1b152009-01-15 18:47:46 +0000622IdentifierInfo* PTHManager::get(const char *NameStart, const char *NameEnd) {
623 unsigned min = 0;
624 unsigned max = NumIds;
625 unsigned len = NameEnd - NameStart;
626
627 do {
628 unsigned i = (max - min) / 2 + min;
629 const char* p = SortedIdTable + (i * 4);
630
631 // Read the persistentID.
632 unsigned perID =
633 ((unsigned) ((uint8_t) p[0]))
634 | (((unsigned) ((uint8_t) p[1])) << 8)
635 | (((unsigned) ((uint8_t) p[2])) << 16)
636 | (((unsigned) ((uint8_t) p[3])) << 24);
637
638 // Get the IdentifierInfo.
639 IdentifierInfo* II = GetIdentifierInfo(perID);
640
641 // First compare the lengths.
642 unsigned IILen = II->getLength();
643 if (len < IILen) goto IsLess;
644 if (len > IILen) goto IsGreater;
645
646 // Now compare the strings!
647 {
648 signed comp = strncmp(NameStart, II->getName(), len);
649 if (comp < 0) goto IsLess;
650 if (comp > 0) goto IsGreater;
651 }
652 // We found a match!
653 return II;
654
655 IsGreater:
656 if (i == min) break;
657 min = i;
658 continue;
659
660 IsLess:
661 max = i;
662 assert(!(max == min) || (min == i));
663 }
Ted Kremeneke1deaac2009-01-15 19:28:38 +0000664 while (min != max);
Ted Kremenek72b1b152009-01-15 18:47:46 +0000665
666 return 0;
667}
668
669
Chris Lattner2b2453a2009-01-17 06:22:33 +0000670PTHLexer* PTHManager::CreateLexer(FileID FID, const FileEntry* FE) {
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000671 if (!FE)
672 return 0;
673
674 // Lookup the FileEntry object in our file lookup data structure. It will
675 // return a variant that indicates whether or not there is an offset within
676 // the PTH file that contains cached tokens.
Chris Lattner2b2453a2009-01-17 06:22:33 +0000677 PTHFileLookup::Val FileData = ((PTHFileLookup*)FileLookup)->Lookup(FE);
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000678
Ted Kremenekfb645b62008-12-11 23:36:38 +0000679 if (!FileData.isValid()) // No tokens available.
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000680 return 0;
681
682 // Compute the offset of the token data within the buffer.
Ted Kremenekfb645b62008-12-11 23:36:38 +0000683 const char* data = Buf->getBufferStart() + FileData.getTokenOffset();
Ted Kremenek268ee702008-12-12 18:34:08 +0000684
685 // Get the location of pp-conditional table.
Ted Kremenekb70e3da2009-01-08 02:47:16 +0000686 const char* ppcond = Buf->getBufferStart() + FileData.getPPCondOffset();
687 uint32_t len = Read32(ppcond);
Ted Kremenek268ee702008-12-12 18:34:08 +0000688 if (len == 0) ppcond = 0;
Ted Kremenek32a8ad52009-01-08 04:30:32 +0000689
690 // Get the location of the spelling table.
691 const char* spellingTable = Buf->getBufferStart() +
692 FileData.getSpellingOffset();
693
694 len = Read32(spellingTable);
695 if (len == 0) spellingTable = 0;
Ted Kremenekb70e3da2009-01-08 02:47:16 +0000696
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000697 assert(data < Buf->getBufferEnd());
Ted Kremenek5f074262009-01-09 22:05:30 +0000698
699 // Create the SpellingSearch object for this FileID.
700 PTHSpellingSearch* ss = new PTHSpellingSearch(*this, len, spellingTable);
Chris Lattner2b2453a2009-01-17 06:22:33 +0000701 SpellingMap[FID] = ss;
Ted Kremenek5f074262009-01-09 22:05:30 +0000702
Ted Kremenek72b1b152009-01-15 18:47:46 +0000703 assert(PP && "No preprocessor set yet!");
Chris Lattner2b2453a2009-01-17 06:22:33 +0000704 return new PTHLexer(*PP, FID, data, ppcond, *ss, *this);
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000705}