blob: 7df8e35986e184cf92096bacd3aea7d2523618d7 [file] [log] [blame]
Ted Kremenek274b2082008-11-12 21:37:15 +00001//===--- PTHLexer.cpp - Lex from a token stream ---------------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the PTHLexer interface.
11//
12//===----------------------------------------------------------------------===//
13
Ted Kremenek0c6a77b2008-12-03 00:38:03 +000014#include "clang/Basic/TokenKinds.h"
15#include "clang/Basic/FileManager.h"
16#include "clang/Basic/IdentifierTable.h"
Ted Kremenek274b2082008-11-12 21:37:15 +000017#include "clang/Lex/PTHLexer.h"
18#include "clang/Lex/Preprocessor.h"
Ted Kremenek0c6a77b2008-12-03 00:38:03 +000019#include "clang/Lex/PTHManager.h"
20#include "clang/Lex/Token.h"
21#include "clang/Lex/Preprocessor.h"
22#include "llvm/Support/Compiler.h"
23#include "llvm/Support/MemoryBuffer.h"
24#include "llvm/ADT/StringMap.h"
25#include "llvm/ADT/OwningPtr.h"
Ted Kremenek274b2082008-11-12 21:37:15 +000026using namespace clang;
27
Ted Kremenek18d9afb2008-12-23 18:41:34 +000028#define DISK_TOKEN_SIZE (1+1+3+4+2)
Ted Kremenek268ee702008-12-12 18:34:08 +000029
Ted Kremenek0c6a77b2008-12-03 00:38:03 +000030//===----------------------------------------------------------------------===//
31// Utility methods for reading from the mmap'ed PTH file.
32//===----------------------------------------------------------------------===//
33
Chris Lattnerda9d61c2009-01-18 01:57:14 +000034static inline uint8_t Read8(const unsigned char *&Data) {
35 uint8_t V = Data[0];
36 Data += 1;
Ted Kremenek0c6a77b2008-12-03 00:38:03 +000037 return V;
38}
39
Chris Lattnerda9d61c2009-01-18 01:57:14 +000040static inline uint16_t Read16(const unsigned char *&Data) {
Chris Lattnerf15674c2009-01-18 02:19:16 +000041// Targets that directly support unaligned little-endian 16-bit loads can just
42// use them.
43#if defined(__i386__) || defined(__x86_64__)
44 uint16_t V = *((uint16_t*)Data);
45#else
Chris Lattnerda9d61c2009-01-18 01:57:14 +000046 uint16_t V = ((uint16_t)Data[0] << 0) |
47 ((uint16_t)Data[1] << 8);
Chris Lattnerf15674c2009-01-18 02:19:16 +000048#endif
Chris Lattnerda9d61c2009-01-18 01:57:14 +000049 Data += 2;
50 return V;
51}
52
53static inline uint32_t Read24(const unsigned char *&Data) {
Chris Lattnerf15674c2009-01-18 02:19:16 +000054// Targets that directly support unaligned little-endian 16-bit loads can just
55// use them.
56#if defined(__i386__) || defined(__x86_64__)
57 uint32_t V = ((uint16_t*)Data)[0] |
58 ((uint32_t)Data[2] << 16);
59#else
Chris Lattnerda9d61c2009-01-18 01:57:14 +000060 uint32_t V = ((uint32_t)Data[0] << 0) |
61 ((uint32_t)Data[1] << 8) |
62 ((uint32_t)Data[2] << 16);
Chris Lattnerf15674c2009-01-18 02:19:16 +000063#endif
64
Chris Lattnerda9d61c2009-01-18 01:57:14 +000065 Data += 3;
66 return V;
67}
68
69static inline uint32_t Read32(const unsigned char *&Data) {
Chris Lattnerf15674c2009-01-18 02:19:16 +000070// Targets that directly support unaligned little-endian 32-bit loads can just
71// use them.
72#if defined(__i386__) || defined(__x86_64__)
73 uint32_t V = *((uint32_t*)Data);
74#else
Chris Lattnerda9d61c2009-01-18 01:57:14 +000075 uint32_t V = ((uint32_t)Data[0] << 0) |
76 ((uint32_t)Data[1] << 8) |
77 ((uint32_t)Data[2] << 16) |
78 ((uint32_t)Data[3] << 24);
Chris Lattnerf15674c2009-01-18 02:19:16 +000079#endif
Chris Lattnerda9d61c2009-01-18 01:57:14 +000080 Data += 4;
81 return V;
82}
83
84
Ted Kremeneke5680f32008-12-23 01:30:52 +000085//===----------------------------------------------------------------------===//
86// PTHLexer methods.
87//===----------------------------------------------------------------------===//
88
Chris Lattnerda9d61c2009-01-18 01:57:14 +000089PTHLexer::PTHLexer(Preprocessor &PP, FileID FID, const unsigned char *D,
90 const unsigned char *ppcond,
Chris Lattner2b2453a2009-01-17 06:22:33 +000091 PTHSpellingSearch &mySpellingSrch, PTHManager &PM)
92 : PreprocessorLexer(&PP, FID), TokBuf(D), CurPtr(D), LastHashTokPtr(0),
Ted Kremenek5f074262009-01-09 22:05:30 +000093 PPCond(ppcond), CurPPCondPtr(ppcond), MySpellingSrch(mySpellingSrch),
Chris Lattner2b2453a2009-01-17 06:22:33 +000094 PTHMgr(PM) {
95
96 FileStartLoc = PP.getSourceManager().getLocForStartOfFile(FID);
Ted Kremenek5f074262009-01-09 22:05:30 +000097}
Ted Kremeneke5680f32008-12-23 01:30:52 +000098
99void PTHLexer::Lex(Token& Tok) {
100LexNextToken:
Ted Kremeneke5680f32008-12-23 01:30:52 +0000101
Ted Kremenek866bdf72008-12-23 02:30:15 +0000102 //===--------------------------------------==//
103 // Read the raw token data.
104 //===--------------------------------------==//
105
106 // Shadow CurPtr into an automatic variable.
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000107 const unsigned char *CurPtrShadow = CurPtr;
Ted Kremenek866bdf72008-12-23 02:30:15 +0000108
Chris Lattner1b5285e2009-01-18 02:10:31 +0000109 // Read in the data for the token.
Chris Lattner898a0bb2009-01-18 02:34:01 +0000110 tok::TokenKind TKind = (tok::TokenKind) Read8(CurPtrShadow);
111 Token::TokenFlags TFlags = (Token::TokenFlags) Read8(CurPtrShadow);
112 uint32_t IdentifierID = Read24(CurPtrShadow);
Chris Lattner1b5285e2009-01-18 02:10:31 +0000113 uint32_t FileOffset = Read32(CurPtrShadow);
114 uint32_t Len = Read16(CurPtrShadow);
115 CurPtr = CurPtrShadow;
Ted Kremenek866bdf72008-12-23 02:30:15 +0000116
117 //===--------------------------------------==//
118 // Construct the token itself.
119 //===--------------------------------------==//
120
121 Tok.startToken();
Chris Lattner898a0bb2009-01-18 02:34:01 +0000122 Tok.setKind(TKind);
123 Tok.setFlag(TFlags);
Ted Kremenek59d08cb2008-12-23 19:24:24 +0000124 assert(!LexingRawMode);
Chris Lattner898a0bb2009-01-18 02:34:01 +0000125 if (IdentifierID)
126 Tok.setIdentifierInfo(PTHMgr.GetIdentifierInfo(IdentifierID-1));
Chris Lattner2b2453a2009-01-17 06:22:33 +0000127 Tok.setLocation(FileStartLoc.getFileLocWithOffset(FileOffset));
Ted Kremenek866bdf72008-12-23 02:30:15 +0000128 Tok.setLength(Len);
129
130 //===--------------------------------------==//
131 // Process the token.
132 //===--------------------------------------==//
Ted Kremenek5f074262009-01-09 22:05:30 +0000133#if 0
134 SourceManager& SM = PP->getSourceManager();
135 llvm::cerr << SM.getFileEntryForID(FileID)->getName()
136 << ':' << SM.getLogicalLineNumber(Tok.getLocation())
137 << ':' << SM.getLogicalColumnNumber(Tok.getLocation())
138 << '\n';
139#endif
Ted Kremeneke5680f32008-12-23 01:30:52 +0000140
Chris Lattner898a0bb2009-01-18 02:34:01 +0000141 if (TKind == tok::identifier) {
Ted Kremenek59d08cb2008-12-23 19:24:24 +0000142 MIOpt.ReadToken();
143 return PP->HandleIdentifier(Tok);
144 }
145
Chris Lattner898a0bb2009-01-18 02:34:01 +0000146 if (TKind == tok::eof) {
Ted Kremeneke5680f32008-12-23 01:30:52 +0000147 // Save the end-of-file token.
148 EofToken = Tok;
149
150 Preprocessor *PPCache = PP;
Ted Kremenek59d08cb2008-12-23 19:24:24 +0000151
152 assert(!ParsingPreprocessorDirective);
153 assert(!LexingRawMode);
154
155 // FIXME: Issue diagnostics similar to Lexer.
156 if (PP->HandleEndOfFile(Tok, false))
Ted Kremeneke5680f32008-12-23 01:30:52 +0000157 return;
Ted Kremenek59d08cb2008-12-23 19:24:24 +0000158
Ted Kremeneke5680f32008-12-23 01:30:52 +0000159 assert(PPCache && "Raw buffer::LexEndOfFile should return a token");
160 return PPCache->Lex(Tok);
161 }
Ted Kremeneke5680f32008-12-23 01:30:52 +0000162
Chris Lattner898a0bb2009-01-18 02:34:01 +0000163 if (TKind == tok::hash && Tok.isAtStartOfLine()) {
Ted Kremenek59d08cb2008-12-23 19:24:24 +0000164 LastHashTokPtr = CurPtr - DISK_TOKEN_SIZE;
165 assert(!LexingRawMode);
166 PP->HandleDirective(Tok);
167
168 if (PP->isCurrentLexer(this))
169 goto LexNextToken;
170
171 return PP->Lex(Tok);
172 }
173
Chris Lattner898a0bb2009-01-18 02:34:01 +0000174 if (TKind == tok::eom) {
Ted Kremenek59d08cb2008-12-23 19:24:24 +0000175 assert(ParsingPreprocessorDirective);
Ted Kremeneke5680f32008-12-23 01:30:52 +0000176 ParsingPreprocessorDirective = false;
177 return;
178 }
Ted Kremeneke5680f32008-12-23 01:30:52 +0000179
Ted Kremenek59d08cb2008-12-23 19:24:24 +0000180 MIOpt.ReadToken();
Ted Kremeneke5680f32008-12-23 01:30:52 +0000181}
182
183// FIXME: We can just grab the last token instead of storing a copy
184// into EofToken.
Ted Kremenek59d08cb2008-12-23 19:24:24 +0000185void PTHLexer::getEOF(Token& Tok) {
Ted Kremenekdefb7092009-01-09 00:36:11 +0000186 assert(EofToken.is(tok::eof));
Ted Kremeneke5680f32008-12-23 01:30:52 +0000187 Tok = EofToken;
188}
189
190void PTHLexer::DiscardToEndOfLine() {
191 assert(ParsingPreprocessorDirective && ParsingFilename == false &&
192 "Must be in a preprocessing directive!");
193
194 // We assume that if the preprocessor wishes to discard to the end of
195 // the line that it also means to end the current preprocessor directive.
196 ParsingPreprocessorDirective = false;
197
198 // Skip tokens by only peeking at their token kind and the flags.
199 // We don't need to actually reconstruct full tokens from the token buffer.
200 // This saves some copies and it also reduces IdentifierInfo* lookup.
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000201 const unsigned char* p = CurPtr;
Ted Kremeneke5680f32008-12-23 01:30:52 +0000202 while (1) {
203 // Read the token kind. Are we at the end of the file?
204 tok::TokenKind x = (tok::TokenKind) (uint8_t) *p;
205 if (x == tok::eof) break;
206
207 // Read the token flags. Are we at the start of the next line?
208 Token::TokenFlags y = (Token::TokenFlags) (uint8_t) p[1];
209 if (y & Token::StartOfLine) break;
210
211 // Skip to the next token.
212 p += DISK_TOKEN_SIZE;
213 }
214
215 CurPtr = p;
216}
217
Ted Kremenek268ee702008-12-12 18:34:08 +0000218/// SkipBlock - Used by Preprocessor to skip the current conditional block.
219bool PTHLexer::SkipBlock() {
220 assert(CurPPCondPtr && "No cached PP conditional information.");
221 assert(LastHashTokPtr && "No known '#' token.");
222
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000223 const unsigned char* HashEntryI = 0;
Ted Kremenek268ee702008-12-12 18:34:08 +0000224 uint32_t Offset;
225 uint32_t TableIdx;
226
227 do {
Ted Kremenek41a26602008-12-12 22:05:38 +0000228 // Read the token offset from the side-table.
Ted Kremenek268ee702008-12-12 18:34:08 +0000229 Offset = Read32(CurPPCondPtr);
Ted Kremenek41a26602008-12-12 22:05:38 +0000230
231 // Read the target table index from the side-table.
Ted Kremenek268ee702008-12-12 18:34:08 +0000232 TableIdx = Read32(CurPPCondPtr);
Ted Kremenek41a26602008-12-12 22:05:38 +0000233
234 // Compute the actual memory address of the '#' token data for this entry.
235 HashEntryI = TokBuf + Offset;
236
237 // Optmization: "Sibling jumping". #if...#else...#endif blocks can
238 // contain nested blocks. In the side-table we can jump over these
239 // nested blocks instead of doing a linear search if the next "sibling"
240 // entry is not at a location greater than LastHashTokPtr.
241 if (HashEntryI < LastHashTokPtr && TableIdx) {
242 // In the side-table we are still at an entry for a '#' token that
243 // is earlier than the last one we saw. Check if the location we would
244 // stride gets us closer.
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000245 const unsigned char* NextPPCondPtr =
246 PPCond + TableIdx*(sizeof(uint32_t)*2);
Ted Kremenek41a26602008-12-12 22:05:38 +0000247 assert(NextPPCondPtr >= CurPPCondPtr);
248 // Read where we should jump to.
249 uint32_t TmpOffset = Read32(NextPPCondPtr);
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000250 const unsigned char* HashEntryJ = TokBuf + TmpOffset;
Ted Kremenek41a26602008-12-12 22:05:38 +0000251
252 if (HashEntryJ <= LastHashTokPtr) {
253 // Jump directly to the next entry in the side table.
254 HashEntryI = HashEntryJ;
255 Offset = TmpOffset;
256 TableIdx = Read32(NextPPCondPtr);
257 CurPPCondPtr = NextPPCondPtr;
258 }
259 }
Ted Kremenek268ee702008-12-12 18:34:08 +0000260 }
Ted Kremenek41a26602008-12-12 22:05:38 +0000261 while (HashEntryI < LastHashTokPtr);
262 assert(HashEntryI == LastHashTokPtr && "No PP-cond entry found for '#'");
Ted Kremenek268ee702008-12-12 18:34:08 +0000263 assert(TableIdx && "No jumping from #endifs.");
264
265 // Update our side-table iterator.
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000266 const unsigned char* NextPPCondPtr = PPCond + TableIdx*(sizeof(uint32_t)*2);
Ted Kremenek268ee702008-12-12 18:34:08 +0000267 assert(NextPPCondPtr >= CurPPCondPtr);
268 CurPPCondPtr = NextPPCondPtr;
269
270 // Read where we should jump to.
Ted Kremenek41a26602008-12-12 22:05:38 +0000271 HashEntryI = TokBuf + Read32(NextPPCondPtr);
Ted Kremenek268ee702008-12-12 18:34:08 +0000272 uint32_t NextIdx = Read32(NextPPCondPtr);
273
274 // By construction NextIdx will be zero if this is a #endif. This is useful
275 // to know to obviate lexing another token.
276 bool isEndif = NextIdx == 0;
Ted Kremenek268ee702008-12-12 18:34:08 +0000277
278 // This case can occur when we see something like this:
279 //
280 // #if ...
281 // /* a comment or nothing */
282 // #elif
283 //
284 // If we are skipping the first #if block it will be the case that CurPtr
285 // already points 'elif'. Just return.
286
Ted Kremenek41a26602008-12-12 22:05:38 +0000287 if (CurPtr > HashEntryI) {
288 assert(CurPtr == HashEntryI + DISK_TOKEN_SIZE);
Ted Kremenek268ee702008-12-12 18:34:08 +0000289 // Did we reach a #endif? If so, go ahead and consume that token as well.
290 if (isEndif)
Ted Kremeneke5680f32008-12-23 01:30:52 +0000291 CurPtr += DISK_TOKEN_SIZE*2;
Ted Kremenek268ee702008-12-12 18:34:08 +0000292 else
Ted Kremenek41a26602008-12-12 22:05:38 +0000293 LastHashTokPtr = HashEntryI;
Ted Kremenek268ee702008-12-12 18:34:08 +0000294
295 return isEndif;
296 }
297
298 // Otherwise, we need to advance. Update CurPtr to point to the '#' token.
Ted Kremenek41a26602008-12-12 22:05:38 +0000299 CurPtr = HashEntryI;
Ted Kremenek268ee702008-12-12 18:34:08 +0000300
301 // Update the location of the last observed '#'. This is useful if we
302 // are skipping multiple blocks.
303 LastHashTokPtr = CurPtr;
Ted Kremenek268ee702008-12-12 18:34:08 +0000304
Ted Kremeneke5680f32008-12-23 01:30:52 +0000305 // Skip the '#' token.
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000306 assert(((tok::TokenKind)*CurPtr) == tok::hash);
Ted Kremeneke5680f32008-12-23 01:30:52 +0000307 CurPtr += DISK_TOKEN_SIZE;
308
Ted Kremenek268ee702008-12-12 18:34:08 +0000309 // Did we reach a #endif? If so, go ahead and consume that token as well.
Ted Kremeneke5680f32008-12-23 01:30:52 +0000310 if (isEndif) { CurPtr += DISK_TOKEN_SIZE*2; }
Ted Kremenek268ee702008-12-12 18:34:08 +0000311
312 return isEndif;
313}
314
Ted Kremenek30a12ec2008-12-17 23:36:32 +0000315SourceLocation PTHLexer::getSourceLocation() {
Chris Lattner1b5285e2009-01-18 02:10:31 +0000316 // getSourceLocation is not on the hot path. It is used to get the location
317 // of the next token when transitioning back to this lexer when done
Ted Kremenek30a12ec2008-12-17 23:36:32 +0000318 // handling a #included file. Just read the necessary data from the token
319 // data buffer to construct the SourceLocation object.
320 // NOTE: This is a virtual function; hence it is defined out-of-line.
Chris Lattner1b5285e2009-01-18 02:10:31 +0000321 const unsigned char *OffsetPtr = CurPtr + (1 + 1 + 3);
322 uint32_t Offset = Read32(OffsetPtr);
323 return FileStartLoc.getFileLocWithOffset(Offset);
Ted Kremenek30a12ec2008-12-17 23:36:32 +0000324}
325
Ted Kremenek5f074262009-01-09 22:05:30 +0000326//===----------------------------------------------------------------------===//
327// getSpelling() - Use cached data in PTH files for getSpelling().
328//===----------------------------------------------------------------------===//
329
Chris Lattner2b2453a2009-01-17 06:22:33 +0000330unsigned PTHManager::getSpelling(FileID FID, unsigned FPos,
331 const char *&Buffer) {
332 llvm::DenseMap<FileID, PTHSpellingSearch*>::iterator I =SpellingMap.find(FID);
Ted Kremenek5f074262009-01-09 22:05:30 +0000333
334 if (I == SpellingMap.end())
Chris Lattner1b5285e2009-01-18 02:10:31 +0000335 return 0;
Ted Kremenekf02f6f02009-01-13 22:05:50 +0000336
Chris Lattner2b2453a2009-01-17 06:22:33 +0000337 return I->second->getSpellingBinarySearch(FPos, Buffer);
338}
339
340unsigned PTHManager::getSpelling(SourceLocation Loc, const char *&Buffer) {
Chris Lattner6b7b8402009-01-17 06:29:33 +0000341 SourceManager &SM = PP->getSourceManager();
342 Loc = SM.getSpellingLoc(Loc);
343 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedFileLoc(Loc);
Chris Lattner2b2453a2009-01-17 06:22:33 +0000344 return getSpelling(LocInfo.first, LocInfo.second, Buffer);
Ted Kremenek5f074262009-01-09 22:05:30 +0000345}
346
347unsigned PTHManager::getSpellingAtPTHOffset(unsigned PTHOffset,
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000348 const char *&Buffer) {
Chris Lattner1b5285e2009-01-18 02:10:31 +0000349 assert(PTHOffset < Buf->getBufferSize());
350 const unsigned char* Ptr =
351 (const unsigned char*)Buf->getBufferStart() + PTHOffset;
Ted Kremenek32a8ad52009-01-08 04:30:32 +0000352
353 // The string is prefixed by 16 bits for its length, followed by the string
354 // itself.
Chris Lattner1b5285e2009-01-18 02:10:31 +0000355 unsigned Len = Read16(Ptr);
356 Buffer = (const char *)Ptr;
357 return Len;
Ted Kremenek32a8ad52009-01-08 04:30:32 +0000358}
359
Chris Lattner1b5285e2009-01-18 02:10:31 +0000360unsigned PTHSpellingSearch::getSpellingLinearSearch(unsigned FPos,
Ted Kremenek5f074262009-01-09 22:05:30 +0000361 const char *&Buffer) {
Chris Lattner1b5285e2009-01-18 02:10:31 +0000362 const unsigned char *Ptr = LinearItr;
363 unsigned Len = 0;
Ted Kremenek5f074262009-01-09 22:05:30 +0000364
Chris Lattner1b5285e2009-01-18 02:10:31 +0000365 if (Ptr == TableEnd)
366 return getSpellingBinarySearch(FPos, Buffer);
Ted Kremenek5f074262009-01-09 22:05:30 +0000367
368 do {
Chris Lattner1b5285e2009-01-18 02:10:31 +0000369 uint32_t TokOffset = Read32(Ptr);
Ted Kremenek32a8ad52009-01-08 04:30:32 +0000370
Chris Lattner1b5285e2009-01-18 02:10:31 +0000371 if (TokOffset > FPos)
372 return getSpellingBinarySearch(FPos, Buffer);
Ted Kremenek32a8ad52009-01-08 04:30:32 +0000373
Ted Kremenek32a8ad52009-01-08 04:30:32 +0000374 // Did we find a matching token offset for this spelling?
Chris Lattner1b5285e2009-01-18 02:10:31 +0000375 if (TokOffset == FPos) {
376 uint32_t SpellingPTHOffset = Read32(Ptr);
377 Len = PTHMgr.getSpellingAtPTHOffset(SpellingPTHOffset, Buffer);
Ted Kremenek32a8ad52009-01-08 04:30:32 +0000378 break;
379 }
Chris Lattner1b5285e2009-01-18 02:10:31 +0000380 } while (Ptr != TableEnd);
Ted Kremenek32a8ad52009-01-08 04:30:32 +0000381
Chris Lattner1b5285e2009-01-18 02:10:31 +0000382 LinearItr = Ptr;
383 return Len;
Ted Kremenekb70e3da2009-01-08 02:47:16 +0000384}
385
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000386
Chris Lattner1b5285e2009-01-18 02:10:31 +0000387unsigned PTHSpellingSearch::getSpellingBinarySearch(unsigned FPos,
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000388 const char *&Buffer) {
Ted Kremenek5f074262009-01-09 22:05:30 +0000389
Ted Kremenekf02f6f02009-01-13 22:05:50 +0000390 assert((TableEnd - TableBeg) % SpellingEntrySize == 0);
Chris Lattner1b5285e2009-01-18 02:10:31 +0000391 assert(TableEnd >= TableBeg);
Ted Kremenekf02f6f02009-01-13 22:05:50 +0000392
393 if (TableEnd == TableBeg)
394 return 0;
395
Ted Kremenek5f074262009-01-09 22:05:30 +0000396 unsigned min = 0;
Chris Lattner1b5285e2009-01-18 02:10:31 +0000397 const unsigned char *tb = TableBeg;
Ted Kremenekf02f6f02009-01-13 22:05:50 +0000398 unsigned max = NumSpellings;
Ted Kremenek5f074262009-01-09 22:05:30 +0000399
Ted Kremenekf02f6f02009-01-13 22:05:50 +0000400 do {
Ted Kremenek5f074262009-01-09 22:05:30 +0000401 unsigned i = (max - min) / 2 + min;
Chris Lattner1b5285e2009-01-18 02:10:31 +0000402 const unsigned char *Ptr = tb + (i * SpellingEntrySize);
Ted Kremenek5f074262009-01-09 22:05:30 +0000403
Chris Lattner1b5285e2009-01-18 02:10:31 +0000404 uint32_t TokOffset = Read32(Ptr);
405 if (TokOffset > FPos) {
Ted Kremenek5f074262009-01-09 22:05:30 +0000406 max = i;
Ted Kremenekf02f6f02009-01-13 22:05:50 +0000407 assert(!(max == min) || (min == i));
Ted Kremenek5f074262009-01-09 22:05:30 +0000408 continue;
409 }
410
Chris Lattner1b5285e2009-01-18 02:10:31 +0000411 if (TokOffset < FPos) {
Ted Kremenek7bd88aa2009-01-13 22:16:45 +0000412 if (i == min)
413 break;
414
Ted Kremenek5f074262009-01-09 22:05:30 +0000415 min = i;
416 continue;
417 }
418
Chris Lattner1b5285e2009-01-18 02:10:31 +0000419 uint32_t SpellingPTHOffset = Read32(Ptr);
Ted Kremenek5f074262009-01-09 22:05:30 +0000420 return PTHMgr.getSpellingAtPTHOffset(SpellingPTHOffset, Buffer);
421 }
Ted Kremenekf02f6f02009-01-13 22:05:50 +0000422 while (min != max);
Ted Kremenek5f074262009-01-09 22:05:30 +0000423
424 return 0;
425}
426
Chris Lattner2b2453a2009-01-17 06:22:33 +0000427unsigned PTHLexer::getSpelling(SourceLocation Loc, const char *&Buffer) {
428 SourceManager &SM = PP->getSourceManager();
429 Loc = SM.getSpellingLoc(Loc);
430 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedFileLoc(Loc);
431
432 FileID FID = LocInfo.first;
433 unsigned FPos = LocInfo.second;
Ted Kremenek5f074262009-01-09 22:05:30 +0000434
Chris Lattner2b2453a2009-01-17 06:22:33 +0000435 if (FID == getFileID())
436 return MySpellingSrch.getSpellingLinearSearch(FPos, Buffer);
437 return PTHMgr.getSpelling(FID, FPos, Buffer);
Ted Kremenek5f074262009-01-09 22:05:30 +0000438}
439
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000440//===----------------------------------------------------------------------===//
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000441// Internal Data Structures for PTH file lookup and resolving identifiers.
442//===----------------------------------------------------------------------===//
443
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000444
445/// PTHFileLookup - This internal data structure is used by the PTHManager
446/// to map from FileEntry objects managed by FileManager to offsets within
447/// the PTH file.
448namespace {
449class VISIBILITY_HIDDEN PTHFileLookup {
450public:
451 class Val {
Ted Kremenekfb645b62008-12-11 23:36:38 +0000452 uint32_t TokenOff;
453 uint32_t PPCondOff;
Ted Kremenekb70e3da2009-01-08 02:47:16 +0000454 uint32_t SpellingOff;
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000455 public:
Ted Kremenekfb645b62008-12-11 23:36:38 +0000456 Val() : TokenOff(~0) {}
Ted Kremenekb70e3da2009-01-08 02:47:16 +0000457 Val(uint32_t toff, uint32_t poff, uint32_t soff)
458 : TokenOff(toff), PPCondOff(poff), SpellingOff(soff) {}
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000459
Chris Lattner1b5285e2009-01-18 02:10:31 +0000460 bool isValid() const { return TokenOff != ~((uint32_t)0); }
461
Ted Kremenekfb645b62008-12-11 23:36:38 +0000462 uint32_t getTokenOffset() const {
Chris Lattner1b5285e2009-01-18 02:10:31 +0000463 assert(isValid() && "PTHFileLookup entry initialized.");
Ted Kremenekfb645b62008-12-11 23:36:38 +0000464 return TokenOff;
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000465 }
466
Ted Kremenekb70e3da2009-01-08 02:47:16 +0000467 uint32_t getPPCondOffset() const {
Chris Lattner1b5285e2009-01-18 02:10:31 +0000468 assert(isValid() && "PTHFileLookup entry initialized.");
Ted Kremenekfb645b62008-12-11 23:36:38 +0000469 return PPCondOff;
470 }
471
Ted Kremenekb70e3da2009-01-08 02:47:16 +0000472 uint32_t getSpellingOffset() const {
Chris Lattner1b5285e2009-01-18 02:10:31 +0000473 assert(isValid() && "PTHFileLookup entry initialized.");
Ted Kremenekb70e3da2009-01-08 02:47:16 +0000474 return SpellingOff;
475 }
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000476 };
477
478private:
479 llvm::StringMap<Val> FileMap;
480
481public:
482 PTHFileLookup() {};
483
484 Val Lookup(const FileEntry* FE) {
485 const char* s = FE->getName();
486 unsigned size = strlen(s);
487 return FileMap.GetOrCreateValue(s, s+size).getValue();
488 }
489
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000490 void ReadTable(const unsigned char* D) {
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000491 uint32_t N = Read32(D); // Read the length of the table.
492
493 for ( ; N > 0; --N) { // The rest of the data is the table itself.
Chris Lattner1b5285e2009-01-18 02:10:31 +0000494 uint32_t Len = Read32(D);
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000495 const char* s = (const char *)D;
Chris Lattner1b5285e2009-01-18 02:10:31 +0000496 D += Len;
Ted Kremenekb70e3da2009-01-08 02:47:16 +0000497
Ted Kremenekfb645b62008-12-11 23:36:38 +0000498 uint32_t TokenOff = Read32(D);
Ted Kremenekb70e3da2009-01-08 02:47:16 +0000499 uint32_t PPCondOff = Read32(D);
500 uint32_t SpellingOff = Read32(D);
501
Chris Lattner1b5285e2009-01-18 02:10:31 +0000502 FileMap.GetOrCreateValue(s, s+Len).getValue() =
Ted Kremenekb70e3da2009-01-08 02:47:16 +0000503 Val(TokenOff, PPCondOff, SpellingOff);
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000504 }
505 }
506};
507} // end anonymous namespace
508
509//===----------------------------------------------------------------------===//
510// PTHManager methods.
511//===----------------------------------------------------------------------===//
512
513PTHManager::PTHManager(const llvm::MemoryBuffer* buf, void* fileLookup,
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000514 const unsigned char* idDataTable,
515 IdentifierInfo** perIDCache,
516 const unsigned char* sortedIdTable, unsigned numIds)
Ted Kremenek6183e482008-12-03 01:16:39 +0000517: Buf(buf), PerIDCache(perIDCache), FileLookup(fileLookup),
Ted Kremenek72b1b152009-01-15 18:47:46 +0000518 IdDataTable(idDataTable), SortedIdTable(sortedIdTable),
519 NumIds(numIds), PP(0) {}
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000520
521PTHManager::~PTHManager() {
522 delete Buf;
523 delete (PTHFileLookup*) FileLookup;
Ted Kremenek0e50b6e2008-12-04 22:47:11 +0000524 free(PerIDCache);
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000525}
526
Ted Kremenek72b1b152009-01-15 18:47:46 +0000527PTHManager* PTHManager::Create(const std::string& file) {
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000528 // Memory map the PTH file.
529 llvm::OwningPtr<llvm::MemoryBuffer>
530 File(llvm::MemoryBuffer::getFile(file.c_str()));
531
532 if (!File)
533 return 0;
534
535 // Get the buffer ranges and check if there are at least three 32-bit
536 // words at the end of the file.
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000537 const unsigned char* BufBeg = (unsigned char*)File->getBufferStart();
538 const unsigned char* BufEnd = (unsigned char*)File->getBufferEnd();
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000539
540 if(!(BufEnd > BufBeg + sizeof(uint32_t)*3)) {
541 assert(false && "Invalid PTH file.");
542 return 0; // FIXME: Proper error diagnostic?
543 }
544
545 // Compute the address of the index table at the end of the PTH file.
546 // This table contains the offset of the file lookup table, the
547 // persistent ID -> identifer data table.
Ted Kremenek293b4af2009-01-15 01:26:25 +0000548 // FIXME: We should just embed this offset in the PTH file.
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000549 const unsigned char* EndTable = BufEnd - sizeof(uint32_t)*4;
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000550
551 // Construct the file lookup table. This will be used for mapping from
552 // FileEntry*'s to cached tokens.
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000553 const unsigned char* FileTableOffset = EndTable + sizeof(uint32_t)*3;
554 const unsigned char* FileTable = BufBeg + Read32(FileTableOffset);
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000555
556 if (!(FileTable > BufBeg && FileTable < BufEnd)) {
557 assert(false && "Invalid PTH file.");
558 return 0; // FIXME: Proper error diagnostic?
559 }
560
561 llvm::OwningPtr<PTHFileLookup> FL(new PTHFileLookup());
562 FL->ReadTable(FileTable);
563
564 // Get the location of the table mapping from persistent ids to the
565 // data needed to reconstruct identifiers.
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000566 const unsigned char* IDTableOffset = EndTable + sizeof(uint32_t)*1;
567 const unsigned char* IData = BufBeg + Read32(IDTableOffset);
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000568 if (!(IData > BufBeg && IData < BufEnd)) {
569 assert(false && "Invalid PTH file.");
570 return 0; // FIXME: Proper error diagnostic?
571 }
572
Ted Kremenek72b1b152009-01-15 18:47:46 +0000573 // Get the location of the lexigraphically-sorted table of persistent IDs.
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000574 const unsigned char* SortedIdTableOffset = EndTable + sizeof(uint32_t)*2;
575 const unsigned char* SortedIdTable = BufBeg + Read32(SortedIdTableOffset);
Ted Kremenek72b1b152009-01-15 18:47:46 +0000576 if (!(SortedIdTable > BufBeg && SortedIdTable < BufEnd)) {
577 assert(false && "Invalid PTH file.");
578 return 0; // FIXME: Proper error diagnostic?
579 }
580
Ted Kremenek6183e482008-12-03 01:16:39 +0000581 // Get the number of IdentifierInfos and pre-allocate the identifier cache.
582 uint32_t NumIds = Read32(IData);
583
584 // Pre-allocate the peristent ID -> IdentifierInfo* cache. We use calloc()
585 // so that we in the best case only zero out memory once when the OS returns
586 // us new pages.
587 IdentifierInfo** PerIDCache =
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000588 (IdentifierInfo**)calloc(NumIds, sizeof(*PerIDCache));
Ted Kremenek6183e482008-12-03 01:16:39 +0000589
590 if (!PerIDCache) {
591 assert(false && "Could not allocate Persistent ID cache.");
592 return 0;
593 }
594
Ted Kremenek72b1b152009-01-15 18:47:46 +0000595 // Create the new PTHManager.
596 return new PTHManager(File.take(), FL.take(), IData, PerIDCache,
597 SortedIdTable, NumIds);
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000598}
Chris Lattner77ecb3a2009-01-18 02:57:21 +0000599IdentifierInfo* PTHManager::LazilyCreateIdentifierInfo(unsigned PersistentID) {
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000600 // Look in the PTH file for the string data for the IdentifierInfo object.
Chris Lattner77ecb3a2009-01-18 02:57:21 +0000601 const unsigned char* TableEntry = IdDataTable + sizeof(uint32_t)*PersistentID;
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000602 const unsigned char* IDData =
603 (const unsigned char*)Buf->getBufferStart() + Read32(TableEntry);
604 assert(IDData < (const unsigned char*)Buf->getBufferEnd());
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000605
Ted Kremenek72b1b152009-01-15 18:47:46 +0000606 // Allocate the object.
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000607 std::pair<IdentifierInfo,const unsigned char*> *Mem =
608 Alloc.Allocate<std::pair<IdentifierInfo,const unsigned char*> >();
Ted Kremenek72b1b152009-01-15 18:47:46 +0000609
610 Mem->second = IDData;
Chris Lattner77ecb3a2009-01-18 02:57:21 +0000611 IdentifierInfo *II = new ((void*) Mem) IdentifierInfo(true);
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000612
Ted Kremenek72b1b152009-01-15 18:47:46 +0000613 // Store the new IdentifierInfo in the cache.
Chris Lattner77ecb3a2009-01-18 02:57:21 +0000614 PerIDCache[PersistentID] = II;
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000615 return II;
616}
617
Ted Kremenek72b1b152009-01-15 18:47:46 +0000618IdentifierInfo* PTHManager::get(const char *NameStart, const char *NameEnd) {
619 unsigned min = 0;
620 unsigned max = NumIds;
Chris Lattner1b5285e2009-01-18 02:10:31 +0000621 unsigned Len = NameEnd - NameStart;
Ted Kremenek72b1b152009-01-15 18:47:46 +0000622
623 do {
624 unsigned i = (max - min) / 2 + min;
Chris Lattner1b5285e2009-01-18 02:10:31 +0000625 const unsigned char *Ptr = SortedIdTable + (i * 4);
Ted Kremenek72b1b152009-01-15 18:47:46 +0000626
627 // Read the persistentID.
Chris Lattner1b5285e2009-01-18 02:10:31 +0000628 unsigned perID = Read32(Ptr);
Ted Kremenek72b1b152009-01-15 18:47:46 +0000629
630 // Get the IdentifierInfo.
631 IdentifierInfo* II = GetIdentifierInfo(perID);
632
633 // First compare the lengths.
634 unsigned IILen = II->getLength();
Chris Lattner1b5285e2009-01-18 02:10:31 +0000635 if (Len < IILen) goto IsLess;
636 if (Len > IILen) goto IsGreater;
Ted Kremenek72b1b152009-01-15 18:47:46 +0000637
638 // Now compare the strings!
639 {
Chris Lattner1b5285e2009-01-18 02:10:31 +0000640 signed comp = strncmp(NameStart, II->getName(), Len);
Ted Kremenek72b1b152009-01-15 18:47:46 +0000641 if (comp < 0) goto IsLess;
642 if (comp > 0) goto IsGreater;
643 }
644 // We found a match!
645 return II;
646
647 IsGreater:
648 if (i == min) break;
649 min = i;
650 continue;
651
652 IsLess:
653 max = i;
654 assert(!(max == min) || (min == i));
655 }
Ted Kremeneke1deaac2009-01-15 19:28:38 +0000656 while (min != max);
Ted Kremenek72b1b152009-01-15 18:47:46 +0000657
658 return 0;
659}
660
661
Chris Lattnerf056d922009-01-17 08:06:50 +0000662PTHLexer *PTHManager::CreateLexer(FileID FID) {
663 const FileEntry *FE = PP->getSourceManager().getFileEntryForID(FID);
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000664 if (!FE)
665 return 0;
666
667 // Lookup the FileEntry object in our file lookup data structure. It will
668 // return a variant that indicates whether or not there is an offset within
669 // the PTH file that contains cached tokens.
Chris Lattner2b2453a2009-01-17 06:22:33 +0000670 PTHFileLookup::Val FileData = ((PTHFileLookup*)FileLookup)->Lookup(FE);
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000671
Ted Kremenekfb645b62008-12-11 23:36:38 +0000672 if (!FileData.isValid()) // No tokens available.
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000673 return 0;
674
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000675 const unsigned char *BufStart = (const unsigned char *)Buf->getBufferStart();
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000676 // Compute the offset of the token data within the buffer.
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000677 const unsigned char* data = BufStart + FileData.getTokenOffset();
Ted Kremenek268ee702008-12-12 18:34:08 +0000678
679 // Get the location of pp-conditional table.
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000680 const unsigned char* ppcond = BufStart + FileData.getPPCondOffset();
Chris Lattner1b5285e2009-01-18 02:10:31 +0000681 uint32_t Len = Read32(ppcond);
682 if (Len == 0) ppcond = 0;
Ted Kremenek32a8ad52009-01-08 04:30:32 +0000683
684 // Get the location of the spelling table.
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000685 const unsigned char* spellingTable = BufStart + FileData.getSpellingOffset();
Ted Kremenek32a8ad52009-01-08 04:30:32 +0000686
Chris Lattner1b5285e2009-01-18 02:10:31 +0000687 Len = Read32(spellingTable);
688 if (Len == 0) spellingTable = 0;
Ted Kremenekb70e3da2009-01-08 02:47:16 +0000689
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000690 assert(data < (const unsigned char*)Buf->getBufferEnd());
Ted Kremenek5f074262009-01-09 22:05:30 +0000691
692 // Create the SpellingSearch object for this FileID.
Chris Lattner1b5285e2009-01-18 02:10:31 +0000693 PTHSpellingSearch* ss = new PTHSpellingSearch(*this, Len, spellingTable);
Chris Lattner2b2453a2009-01-17 06:22:33 +0000694 SpellingMap[FID] = ss;
Ted Kremenek5f074262009-01-09 22:05:30 +0000695
Ted Kremenek72b1b152009-01-15 18:47:46 +0000696 assert(PP && "No preprocessor set yet!");
Chris Lattner2b2453a2009-01-17 06:22:33 +0000697 return new PTHLexer(*PP, FID, data, ppcond, *ss, *this);
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000698}