blob: 916bdefdf2acaa7ba0580311f932836d78760879 [file] [log] [blame]
Ted Kremenek274b2082008-11-12 21:37:15 +00001//===--- PTHLexer.cpp - Lex from a token stream ---------------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the PTHLexer interface.
11//
12//===----------------------------------------------------------------------===//
13
Ted Kremenek0c6a77b2008-12-03 00:38:03 +000014#include "clang/Basic/TokenKinds.h"
15#include "clang/Basic/FileManager.h"
16#include "clang/Basic/IdentifierTable.h"
Douglas Gregor9378ba42009-04-20 07:08:21 +000017#include "clang/Basic/OnDiskHashTable.h"
Ted Kremenek274b2082008-11-12 21:37:15 +000018#include "clang/Lex/PTHLexer.h"
19#include "clang/Lex/Preprocessor.h"
Ted Kremenek0c6a77b2008-12-03 00:38:03 +000020#include "clang/Lex/PTHManager.h"
21#include "clang/Lex/Token.h"
22#include "clang/Lex/Preprocessor.h"
Ted Kremenek0c6a77b2008-12-03 00:38:03 +000023#include "llvm/ADT/StringMap.h"
24#include "llvm/ADT/OwningPtr.h"
Chris Lattner6f78c3b2009-01-22 23:50:07 +000025#include "llvm/Support/MemoryBuffer.h"
Ted Kremenek337edcd2009-02-12 03:26:59 +000026#include <sys/stat.h>
Ted Kremenek274b2082008-11-12 21:37:15 +000027using namespace clang;
Douglas Gregor9378ba42009-04-20 07:08:21 +000028using namespace clang::io;
Ted Kremenek274b2082008-11-12 21:37:15 +000029
Ted Kremenek7b78b7c2009-01-19 23:13:15 +000030#define DISK_TOKEN_SIZE (1+1+2+4+4)
Ted Kremenek268ee702008-12-12 18:34:08 +000031
Ted Kremenek0c6a77b2008-12-03 00:38:03 +000032//===----------------------------------------------------------------------===//
Ted Kremeneke5680f32008-12-23 01:30:52 +000033// PTHLexer methods.
34//===----------------------------------------------------------------------===//
35
Chris Lattnerda9d61c2009-01-18 01:57:14 +000036PTHLexer::PTHLexer(Preprocessor &PP, FileID FID, const unsigned char *D,
Ted Kremenek277faca2009-01-27 00:01:05 +000037 const unsigned char *ppcond, PTHManager &PM)
Chris Lattner2b2453a2009-01-17 06:22:33 +000038 : PreprocessorLexer(&PP, FID), TokBuf(D), CurPtr(D), LastHashTokPtr(0),
Ted Kremenek277faca2009-01-27 00:01:05 +000039 PPCond(ppcond), CurPPCondPtr(ppcond), PTHMgr(PM) {
Chris Lattner2b2453a2009-01-17 06:22:33 +000040
41 FileStartLoc = PP.getSourceManager().getLocForStartOfFile(FID);
Ted Kremenek5f074262009-01-09 22:05:30 +000042}
Ted Kremeneke5680f32008-12-23 01:30:52 +000043
44void PTHLexer::Lex(Token& Tok) {
45LexNextToken:
Ted Kremeneke5680f32008-12-23 01:30:52 +000046
Ted Kremenek866bdf72008-12-23 02:30:15 +000047 //===--------------------------------------==//
48 // Read the raw token data.
49 //===--------------------------------------==//
50
51 // Shadow CurPtr into an automatic variable.
Chris Lattneraff6ef82009-01-21 07:21:56 +000052 const unsigned char *CurPtrShadow = CurPtr;
Ted Kremenek866bdf72008-12-23 02:30:15 +000053
Chris Lattner1b5285e2009-01-18 02:10:31 +000054 // Read in the data for the token.
Chris Lattner5ff43172009-01-22 19:48:26 +000055 unsigned Word0 = ReadLE32(CurPtrShadow);
56 uint32_t IdentifierID = ReadLE32(CurPtrShadow);
57 uint32_t FileOffset = ReadLE32(CurPtrShadow);
Ted Kremenek7b78b7c2009-01-19 23:13:15 +000058
59 tok::TokenKind TKind = (tok::TokenKind) (Word0 & 0xFF);
60 Token::TokenFlags TFlags = (Token::TokenFlags) ((Word0 >> 8) & 0xFF);
Chris Lattneraff6ef82009-01-21 07:21:56 +000061 uint32_t Len = Word0 >> 16;
Ted Kremenek7b78b7c2009-01-19 23:13:15 +000062
Chris Lattneraff6ef82009-01-21 07:21:56 +000063 CurPtr = CurPtrShadow;
Ted Kremenek866bdf72008-12-23 02:30:15 +000064
65 //===--------------------------------------==//
66 // Construct the token itself.
67 //===--------------------------------------==//
68
69 Tok.startToken();
Chris Lattner898a0bb2009-01-18 02:34:01 +000070 Tok.setKind(TKind);
71 Tok.setFlag(TFlags);
Ted Kremenek59d08cb2008-12-23 19:24:24 +000072 assert(!LexingRawMode);
Chris Lattner2b2453a2009-01-17 06:22:33 +000073 Tok.setLocation(FileStartLoc.getFileLocWithOffset(FileOffset));
Ted Kremenek866bdf72008-12-23 02:30:15 +000074 Tok.setLength(Len);
75
Chris Lattnerd0a69692009-01-21 07:50:06 +000076 // Handle identifiers.
Ted Kremenek277faca2009-01-27 00:01:05 +000077 if (Tok.isLiteral()) {
78 Tok.setLiteralData((const char*) (PTHMgr.SpellingBase + IdentifierID));
79 }
80 else if (IdentifierID) {
Chris Lattnerd0a69692009-01-21 07:50:06 +000081 MIOpt.ReadToken();
82 IdentifierInfo *II = PTHMgr.GetIdentifierInfo(IdentifierID-1);
Chris Lattner863c4862009-01-23 18:35:48 +000083
Chris Lattnerd0a69692009-01-21 07:50:06 +000084 Tok.setIdentifierInfo(II);
Chris Lattner863c4862009-01-23 18:35:48 +000085
86 // Change the kind of this identifier to the appropriate token kind, e.g.
87 // turning "for" into a keyword.
88 Tok.setKind(II->getTokenID());
89
Chris Lattnerd0a69692009-01-21 07:50:06 +000090 if (II->isHandleIdentifierCase())
91 PP->HandleIdentifier(Tok);
92 return;
93 }
94
Ted Kremenek866bdf72008-12-23 02:30:15 +000095 //===--------------------------------------==//
96 // Process the token.
97 //===--------------------------------------==//
Ted Kremenek5f074262009-01-09 22:05:30 +000098#if 0
99 SourceManager& SM = PP->getSourceManager();
100 llvm::cerr << SM.getFileEntryForID(FileID)->getName()
101 << ':' << SM.getLogicalLineNumber(Tok.getLocation())
102 << ':' << SM.getLogicalColumnNumber(Tok.getLocation())
103 << '\n';
104#endif
Ted Kremeneke5680f32008-12-23 01:30:52 +0000105
Chris Lattner898a0bb2009-01-18 02:34:01 +0000106 if (TKind == tok::eof) {
Ted Kremeneke5680f32008-12-23 01:30:52 +0000107 // Save the end-of-file token.
108 EofToken = Tok;
109
110 Preprocessor *PPCache = PP;
Ted Kremenek59d08cb2008-12-23 19:24:24 +0000111
112 assert(!ParsingPreprocessorDirective);
113 assert(!LexingRawMode);
114
115 // FIXME: Issue diagnostics similar to Lexer.
116 if (PP->HandleEndOfFile(Tok, false))
Ted Kremeneke5680f32008-12-23 01:30:52 +0000117 return;
Ted Kremenek59d08cb2008-12-23 19:24:24 +0000118
Ted Kremeneke5680f32008-12-23 01:30:52 +0000119 assert(PPCache && "Raw buffer::LexEndOfFile should return a token");
120 return PPCache->Lex(Tok);
121 }
Ted Kremeneke5680f32008-12-23 01:30:52 +0000122
Chris Lattner898a0bb2009-01-18 02:34:01 +0000123 if (TKind == tok::hash && Tok.isAtStartOfLine()) {
Ted Kremenek59d08cb2008-12-23 19:24:24 +0000124 LastHashTokPtr = CurPtr - DISK_TOKEN_SIZE;
125 assert(!LexingRawMode);
126 PP->HandleDirective(Tok);
127
128 if (PP->isCurrentLexer(this))
129 goto LexNextToken;
130
131 return PP->Lex(Tok);
132 }
133
Chris Lattner898a0bb2009-01-18 02:34:01 +0000134 if (TKind == tok::eom) {
Ted Kremenek59d08cb2008-12-23 19:24:24 +0000135 assert(ParsingPreprocessorDirective);
Ted Kremeneke5680f32008-12-23 01:30:52 +0000136 ParsingPreprocessorDirective = false;
137 return;
138 }
Ted Kremeneke5680f32008-12-23 01:30:52 +0000139
Ted Kremenek59d08cb2008-12-23 19:24:24 +0000140 MIOpt.ReadToken();
Ted Kremeneke5680f32008-12-23 01:30:52 +0000141}
142
143// FIXME: We can just grab the last token instead of storing a copy
144// into EofToken.
Ted Kremenek59d08cb2008-12-23 19:24:24 +0000145void PTHLexer::getEOF(Token& Tok) {
Ted Kremenekdefb7092009-01-09 00:36:11 +0000146 assert(EofToken.is(tok::eof));
Ted Kremeneke5680f32008-12-23 01:30:52 +0000147 Tok = EofToken;
148}
149
150void PTHLexer::DiscardToEndOfLine() {
151 assert(ParsingPreprocessorDirective && ParsingFilename == false &&
152 "Must be in a preprocessing directive!");
153
154 // We assume that if the preprocessor wishes to discard to the end of
155 // the line that it also means to end the current preprocessor directive.
156 ParsingPreprocessorDirective = false;
157
158 // Skip tokens by only peeking at their token kind and the flags.
159 // We don't need to actually reconstruct full tokens from the token buffer.
160 // This saves some copies and it also reduces IdentifierInfo* lookup.
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000161 const unsigned char* p = CurPtr;
Ted Kremeneke5680f32008-12-23 01:30:52 +0000162 while (1) {
163 // Read the token kind. Are we at the end of the file?
164 tok::TokenKind x = (tok::TokenKind) (uint8_t) *p;
165 if (x == tok::eof) break;
166
167 // Read the token flags. Are we at the start of the next line?
168 Token::TokenFlags y = (Token::TokenFlags) (uint8_t) p[1];
169 if (y & Token::StartOfLine) break;
170
171 // Skip to the next token.
172 p += DISK_TOKEN_SIZE;
173 }
174
175 CurPtr = p;
176}
177
Ted Kremenek268ee702008-12-12 18:34:08 +0000178/// SkipBlock - Used by Preprocessor to skip the current conditional block.
179bool PTHLexer::SkipBlock() {
180 assert(CurPPCondPtr && "No cached PP conditional information.");
181 assert(LastHashTokPtr && "No known '#' token.");
182
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000183 const unsigned char* HashEntryI = 0;
Ted Kremenek268ee702008-12-12 18:34:08 +0000184 uint32_t Offset;
185 uint32_t TableIdx;
186
187 do {
Ted Kremenek41a26602008-12-12 22:05:38 +0000188 // Read the token offset from the side-table.
Chris Lattner5ff43172009-01-22 19:48:26 +0000189 Offset = ReadLE32(CurPPCondPtr);
Ted Kremenek41a26602008-12-12 22:05:38 +0000190
191 // Read the target table index from the side-table.
Chris Lattner5ff43172009-01-22 19:48:26 +0000192 TableIdx = ReadLE32(CurPPCondPtr);
Ted Kremenek41a26602008-12-12 22:05:38 +0000193
194 // Compute the actual memory address of the '#' token data for this entry.
195 HashEntryI = TokBuf + Offset;
196
197 // Optmization: "Sibling jumping". #if...#else...#endif blocks can
198 // contain nested blocks. In the side-table we can jump over these
199 // nested blocks instead of doing a linear search if the next "sibling"
200 // entry is not at a location greater than LastHashTokPtr.
201 if (HashEntryI < LastHashTokPtr && TableIdx) {
202 // In the side-table we are still at an entry for a '#' token that
203 // is earlier than the last one we saw. Check if the location we would
204 // stride gets us closer.
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000205 const unsigned char* NextPPCondPtr =
206 PPCond + TableIdx*(sizeof(uint32_t)*2);
Ted Kremenek41a26602008-12-12 22:05:38 +0000207 assert(NextPPCondPtr >= CurPPCondPtr);
208 // Read where we should jump to.
Chris Lattner5ff43172009-01-22 19:48:26 +0000209 uint32_t TmpOffset = ReadLE32(NextPPCondPtr);
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000210 const unsigned char* HashEntryJ = TokBuf + TmpOffset;
Ted Kremenek41a26602008-12-12 22:05:38 +0000211
212 if (HashEntryJ <= LastHashTokPtr) {
213 // Jump directly to the next entry in the side table.
214 HashEntryI = HashEntryJ;
215 Offset = TmpOffset;
Chris Lattner5ff43172009-01-22 19:48:26 +0000216 TableIdx = ReadLE32(NextPPCondPtr);
Ted Kremenek41a26602008-12-12 22:05:38 +0000217 CurPPCondPtr = NextPPCondPtr;
218 }
219 }
Ted Kremenek268ee702008-12-12 18:34:08 +0000220 }
Ted Kremenek41a26602008-12-12 22:05:38 +0000221 while (HashEntryI < LastHashTokPtr);
222 assert(HashEntryI == LastHashTokPtr && "No PP-cond entry found for '#'");
Ted Kremenek268ee702008-12-12 18:34:08 +0000223 assert(TableIdx && "No jumping from #endifs.");
224
225 // Update our side-table iterator.
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000226 const unsigned char* NextPPCondPtr = PPCond + TableIdx*(sizeof(uint32_t)*2);
Ted Kremenek268ee702008-12-12 18:34:08 +0000227 assert(NextPPCondPtr >= CurPPCondPtr);
228 CurPPCondPtr = NextPPCondPtr;
229
230 // Read where we should jump to.
Chris Lattner5ff43172009-01-22 19:48:26 +0000231 HashEntryI = TokBuf + ReadLE32(NextPPCondPtr);
232 uint32_t NextIdx = ReadLE32(NextPPCondPtr);
Ted Kremenek268ee702008-12-12 18:34:08 +0000233
234 // By construction NextIdx will be zero if this is a #endif. This is useful
235 // to know to obviate lexing another token.
236 bool isEndif = NextIdx == 0;
Ted Kremenek268ee702008-12-12 18:34:08 +0000237
238 // This case can occur when we see something like this:
239 //
240 // #if ...
241 // /* a comment or nothing */
242 // #elif
243 //
244 // If we are skipping the first #if block it will be the case that CurPtr
245 // already points 'elif'. Just return.
246
Ted Kremenek41a26602008-12-12 22:05:38 +0000247 if (CurPtr > HashEntryI) {
248 assert(CurPtr == HashEntryI + DISK_TOKEN_SIZE);
Ted Kremenek268ee702008-12-12 18:34:08 +0000249 // Did we reach a #endif? If so, go ahead and consume that token as well.
250 if (isEndif)
Ted Kremeneke5680f32008-12-23 01:30:52 +0000251 CurPtr += DISK_TOKEN_SIZE*2;
Ted Kremenek268ee702008-12-12 18:34:08 +0000252 else
Ted Kremenek41a26602008-12-12 22:05:38 +0000253 LastHashTokPtr = HashEntryI;
Ted Kremenek268ee702008-12-12 18:34:08 +0000254
255 return isEndif;
256 }
257
258 // Otherwise, we need to advance. Update CurPtr to point to the '#' token.
Ted Kremenek41a26602008-12-12 22:05:38 +0000259 CurPtr = HashEntryI;
Ted Kremenek268ee702008-12-12 18:34:08 +0000260
261 // Update the location of the last observed '#'. This is useful if we
262 // are skipping multiple blocks.
263 LastHashTokPtr = CurPtr;
Ted Kremenek268ee702008-12-12 18:34:08 +0000264
Ted Kremeneke5680f32008-12-23 01:30:52 +0000265 // Skip the '#' token.
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000266 assert(((tok::TokenKind)*CurPtr) == tok::hash);
Ted Kremeneke5680f32008-12-23 01:30:52 +0000267 CurPtr += DISK_TOKEN_SIZE;
268
Ted Kremenek268ee702008-12-12 18:34:08 +0000269 // Did we reach a #endif? If so, go ahead and consume that token as well.
Ted Kremeneke5680f32008-12-23 01:30:52 +0000270 if (isEndif) { CurPtr += DISK_TOKEN_SIZE*2; }
Ted Kremenek268ee702008-12-12 18:34:08 +0000271
272 return isEndif;
273}
274
Ted Kremenek30a12ec2008-12-17 23:36:32 +0000275SourceLocation PTHLexer::getSourceLocation() {
Chris Lattner1b5285e2009-01-18 02:10:31 +0000276 // getSourceLocation is not on the hot path. It is used to get the location
277 // of the next token when transitioning back to this lexer when done
Ted Kremenek30a12ec2008-12-17 23:36:32 +0000278 // handling a #included file. Just read the necessary data from the token
279 // data buffer to construct the SourceLocation object.
280 // NOTE: This is a virtual function; hence it is defined out-of-line.
Ted Kremenekb248d532009-01-21 22:41:38 +0000281 const unsigned char *OffsetPtr = CurPtr + (DISK_TOKEN_SIZE - 4);
Chris Lattner5ff43172009-01-22 19:48:26 +0000282 uint32_t Offset = ReadLE32(OffsetPtr);
Chris Lattner1b5285e2009-01-18 02:10:31 +0000283 return FileStartLoc.getFileLocWithOffset(Offset);
Ted Kremenek30a12ec2008-12-17 23:36:32 +0000284}
285
Ted Kremenek5f074262009-01-09 22:05:30 +0000286//===----------------------------------------------------------------------===//
Ted Kremenekd8c02922009-02-10 22:16:22 +0000287// PTH file lookup: map from strings to file data.
288//===----------------------------------------------------------------------===//
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000289
290/// PTHFileLookup - This internal data structure is used by the PTHManager
291/// to map from FileEntry objects managed by FileManager to offsets within
292/// the PTH file.
293namespace {
Ted Kremenekd8c02922009-02-10 22:16:22 +0000294class VISIBILITY_HIDDEN PTHFileData {
295 const uint32_t TokenOff;
296 const uint32_t PPCondOff;
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000297public:
Ted Kremenekd8c02922009-02-10 22:16:22 +0000298 PTHFileData(uint32_t tokenOff, uint32_t ppCondOff)
299 : TokenOff(tokenOff), PPCondOff(ppCondOff) {}
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000300
Ted Kremenekd8c02922009-02-10 22:16:22 +0000301 uint32_t getTokenOffset() const { return TokenOff; }
302 uint32_t getPPCondOffset() const { return PPCondOff; }
303};
304
Ted Kremenek337edcd2009-02-12 03:26:59 +0000305
306class VISIBILITY_HIDDEN PTHFileLookupCommonTrait {
Ted Kremenekd8c02922009-02-10 22:16:22 +0000307public:
Ted Kremeneka4b44dd2009-02-13 19:13:46 +0000308 typedef std::pair<unsigned char, const char*> internal_key_type;
309
310 static unsigned ComputeHash(internal_key_type x) {
311 return BernsteinHash(x.second);
Ted Kremenekd8c02922009-02-10 22:16:22 +0000312 }
Ted Kremenekcdd8f212009-01-21 07:34:28 +0000313
Ted Kremenekd8c02922009-02-10 22:16:22 +0000314 static std::pair<unsigned, unsigned>
315 ReadKeyDataLength(const unsigned char*& d) {
Ted Kremeneka4b44dd2009-02-13 19:13:46 +0000316 unsigned keyLen = (unsigned) ReadUnalignedLE16(d);
317 unsigned dataLen = (unsigned) *(d++);
318 return std::make_pair(keyLen, dataLen);
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000319 }
320
Ted Kremeneka4b44dd2009-02-13 19:13:46 +0000321 static internal_key_type ReadKey(const unsigned char* d, unsigned) {
322 unsigned char k = *(d++); // Read the entry kind.
323 return std::make_pair(k, (const char*) d);
Ted Kremenekd8c02922009-02-10 22:16:22 +0000324 }
Ted Kremenek337edcd2009-02-12 03:26:59 +0000325};
326
327class VISIBILITY_HIDDEN PTHFileLookupTrait : public PTHFileLookupCommonTrait {
328public:
329 typedef const FileEntry* external_key_type;
330 typedef PTHFileData data_type;
331
Ted Kremeneka4b44dd2009-02-13 19:13:46 +0000332 static internal_key_type GetInternalKey(const FileEntry* FE) {
333 return std::make_pair((unsigned char) 0x1, FE->getName());
Ted Kremenek337edcd2009-02-12 03:26:59 +0000334 }
Ted Kremeneka4b44dd2009-02-13 19:13:46 +0000335
336 static bool EqualKey(internal_key_type a, internal_key_type b) {
337 return a.first == b.first && strcmp(a.second, b.second) == 0;
338 }
Ted Kremenekd8c02922009-02-10 22:16:22 +0000339
Ted Kremeneka4b44dd2009-02-13 19:13:46 +0000340 static PTHFileData ReadData(const internal_key_type& k,
341 const unsigned char* d, unsigned) {
342 assert(k.first == 0x1 && "Only file lookups can match!");
Ted Kremenekd8c02922009-02-10 22:16:22 +0000343 uint32_t x = ::ReadUnalignedLE32(d);
344 uint32_t y = ::ReadUnalignedLE32(d);
345 return PTHFileData(x, y);
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000346 }
347};
Ted Kremenek7e3a0042009-02-11 21:29:16 +0000348
349class VISIBILITY_HIDDEN PTHStringLookupTrait {
350public:
351 typedef uint32_t
352 data_type;
353
354 typedef const std::pair<const char*, unsigned>
355 external_key_type;
356
357 typedef external_key_type internal_key_type;
358
359 static bool EqualKey(const internal_key_type& a,
360 const internal_key_type& b) {
361 return (a.second == b.second) ? memcmp(a.first, b.first, a.second) == 0
362 : false;
363 }
364
365 static unsigned ComputeHash(const internal_key_type& a) {
366 return BernsteinHash(a.first, a.second);
367 }
368
369 // This hopefully will just get inlined and removed by the optimizer.
370 static const internal_key_type&
371 GetInternalKey(const external_key_type& x) { return x; }
372
373 static std::pair<unsigned, unsigned>
374 ReadKeyDataLength(const unsigned char*& d) {
375 return std::make_pair((unsigned) ReadUnalignedLE16(d), sizeof(uint32_t));
376 }
377
378 static std::pair<const char*, unsigned>
379 ReadKey(const unsigned char* d, unsigned n) {
380 assert(n >= 2 && d[n-1] == '\0');
381 return std::make_pair((const char*) d, n-1);
382 }
383
Ted Kremeneka4b44dd2009-02-13 19:13:46 +0000384 static uint32_t ReadData(const internal_key_type& k, const unsigned char* d,
385 unsigned) {
Ted Kremenek7e3a0042009-02-11 21:29:16 +0000386 return ::ReadUnalignedLE32(d);
387 }
388};
389
Ted Kremenekd8c02922009-02-10 22:16:22 +0000390} // end anonymous namespace
391
Ted Kremenek7e3a0042009-02-11 21:29:16 +0000392typedef OnDiskChainedHashTable<PTHFileLookupTrait> PTHFileLookup;
393typedef OnDiskChainedHashTable<PTHStringLookupTrait> PTHStringIdLookup;
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000394
395//===----------------------------------------------------------------------===//
396// PTHManager methods.
397//===----------------------------------------------------------------------===//
398
399PTHManager::PTHManager(const llvm::MemoryBuffer* buf, void* fileLookup,
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000400 const unsigned char* idDataTable,
401 IdentifierInfo** perIDCache,
Ted Kremenek7e3a0042009-02-11 21:29:16 +0000402 void* stringIdLookup, unsigned numIds,
Ted Kremenek68228632009-03-19 22:19:30 +0000403 const unsigned char* spellingBase,
404 const char* originalSourceFile)
Ted Kremenek6183e482008-12-03 01:16:39 +0000405: Buf(buf), PerIDCache(perIDCache), FileLookup(fileLookup),
Ted Kremenek7e3a0042009-02-11 21:29:16 +0000406 IdDataTable(idDataTable), StringIdLookup(stringIdLookup),
Ted Kremenek68228632009-03-19 22:19:30 +0000407 NumIds(numIds), PP(0), SpellingBase(spellingBase),
408 OriginalSourceFile(originalSourceFile) {}
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000409
410PTHManager::~PTHManager() {
411 delete Buf;
412 delete (PTHFileLookup*) FileLookup;
Ted Kremenek7e3a0042009-02-11 21:29:16 +0000413 delete (PTHStringIdLookup*) StringIdLookup;
Ted Kremenek0e50b6e2008-12-04 22:47:11 +0000414 free(PerIDCache);
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000415}
416
Ted Kremenek22f0d092009-03-22 06:42:39 +0000417static void InvalidPTH(Diagnostic *Diags, Diagnostic::Level level,
418 const char* Msg = 0) {
Ted Kremenek26555b12009-01-28 21:02:43 +0000419 if (!Diags) return;
420 if (!Msg) Msg = "Invalid or corrupted PTH file";
Ted Kremenek22f0d092009-03-22 06:42:39 +0000421 unsigned DiagID = Diags->getCustomDiagID(level, Msg);
Ted Kremenek26555b12009-01-28 21:02:43 +0000422 Diags->Report(FullSourceLoc(), DiagID);
423}
424
Ted Kremenek22f0d092009-03-22 06:42:39 +0000425PTHManager* PTHManager::Create(const std::string& file, Diagnostic* Diags,
426 Diagnostic::Level level) {
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000427 // Memory map the PTH file.
428 llvm::OwningPtr<llvm::MemoryBuffer>
429 File(llvm::MemoryBuffer::getFile(file.c_str()));
430
Ted Kremenek8a6aec62009-01-28 20:49:33 +0000431 if (!File) {
432 if (Diags) {
Ted Kremenek22f0d092009-03-22 06:42:39 +0000433 unsigned DiagID = Diags->getCustomDiagID(level,
Ted Kremenek8a6aec62009-01-28 20:49:33 +0000434 "PTH file %0 could not be read");
435 Diags->Report(FullSourceLoc(), DiagID) << file;
436 }
Ted Kremenek7e3a0042009-02-11 21:29:16 +0000437
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000438 return 0;
Ted Kremenek8a6aec62009-01-28 20:49:33 +0000439 }
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000440
441 // Get the buffer ranges and check if there are at least three 32-bit
442 // words at the end of the file.
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000443 const unsigned char* BufBeg = (unsigned char*)File->getBufferStart();
444 const unsigned char* BufEnd = (unsigned char*)File->getBufferEnd();
Ted Kremeneke1b64982009-01-26 21:43:14 +0000445
446 // Check the prologue of the file.
Ted Kremenek4adc71a2009-01-26 22:16:12 +0000447 if ((BufEnd - BufBeg) < (signed) (sizeof("cfe-pth") + 3 + 4) ||
Ted Kremenek26555b12009-01-28 21:02:43 +0000448 memcmp(BufBeg, "cfe-pth", sizeof("cfe-pth") - 1) != 0) {
Ted Kremenek22f0d092009-03-22 06:42:39 +0000449 InvalidPTH(Diags, level);
Ted Kremeneke1b64982009-01-26 21:43:14 +0000450 return 0;
Ted Kremenek26555b12009-01-28 21:02:43 +0000451 }
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000452
Ted Kremenek67d15052009-01-26 21:50:21 +0000453 // Read the PTH version.
Ted Kremeneke1b64982009-01-26 21:43:14 +0000454 const unsigned char *p = BufBeg + (sizeof("cfe-pth") - 1);
Ted Kremenek67d15052009-01-26 21:50:21 +0000455 unsigned Version = ReadLE32(p);
456
Ted Kremenek26555b12009-01-28 21:02:43 +0000457 if (Version != PTHManager::Version) {
Ted Kremenek22f0d092009-03-22 06:42:39 +0000458 InvalidPTH(Diags, level,
Ted Kremenek26555b12009-01-28 21:02:43 +0000459 Version < PTHManager::Version
460 ? "PTH file uses an older PTH format that is no longer supported"
461 : "PTH file uses a newer PTH format that cannot be read");
Ted Kremenek67d15052009-01-26 21:50:21 +0000462 return 0;
Ted Kremenek26555b12009-01-28 21:02:43 +0000463 }
Ted Kremenek67d15052009-01-26 21:50:21 +0000464
465 // Compute the address of the index table at the end of the PTH file.
Ted Kremeneka4bd8eb2009-02-11 23:34:32 +0000466 const unsigned char *PrologueOffset = p;
Ted Kremeneke1b64982009-01-26 21:43:14 +0000467
Ted Kremeneka4bd8eb2009-02-11 23:34:32 +0000468 if (PrologueOffset >= BufEnd) {
Ted Kremenek22f0d092009-03-22 06:42:39 +0000469 InvalidPTH(Diags, level);
Ted Kremeneke1b64982009-01-26 21:43:14 +0000470 return 0;
Ted Kremenek26555b12009-01-28 21:02:43 +0000471 }
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000472
473 // Construct the file lookup table. This will be used for mapping from
474 // FileEntry*'s to cached tokens.
Ted Kremeneka4bd8eb2009-02-11 23:34:32 +0000475 const unsigned char* FileTableOffset = PrologueOffset + sizeof(uint32_t)*2;
Chris Lattner5ff43172009-01-22 19:48:26 +0000476 const unsigned char* FileTable = BufBeg + ReadLE32(FileTableOffset);
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000477
478 if (!(FileTable > BufBeg && FileTable < BufEnd)) {
Ted Kremenek22f0d092009-03-22 06:42:39 +0000479 InvalidPTH(Diags, level);
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000480 return 0; // FIXME: Proper error diagnostic?
481 }
482
Ted Kremenekd8c02922009-02-10 22:16:22 +0000483 llvm::OwningPtr<PTHFileLookup> FL(PTHFileLookup::Create(FileTable, BufBeg));
Ted Kremenek1d201972009-03-20 17:54:25 +0000484
485 // Warn if the PTH file is empty. We still want to create a PTHManager
486 // as the PTH could be used with -include-pth.
487 if (FL->isEmpty())
Ted Kremenek22f0d092009-03-22 06:42:39 +0000488 InvalidPTH(Diags, level, "PTH file contains no cached source data");
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000489
490 // Get the location of the table mapping from persistent ids to the
491 // data needed to reconstruct identifiers.
Ted Kremeneka4bd8eb2009-02-11 23:34:32 +0000492 const unsigned char* IDTableOffset = PrologueOffset + sizeof(uint32_t)*0;
Chris Lattner5ff43172009-01-22 19:48:26 +0000493 const unsigned char* IData = BufBeg + ReadLE32(IDTableOffset);
Ted Kremenekcdd8f212009-01-21 07:34:28 +0000494
495 if (!(IData >= BufBeg && IData < BufEnd)) {
Ted Kremenek22f0d092009-03-22 06:42:39 +0000496 InvalidPTH(Diags, level);
Ted Kremenek26555b12009-01-28 21:02:43 +0000497 return 0;
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000498 }
499
Ted Kremenek7e3a0042009-02-11 21:29:16 +0000500 // Get the location of the hashtable mapping between strings and
501 // persistent IDs.
Ted Kremeneka4bd8eb2009-02-11 23:34:32 +0000502 const unsigned char* StringIdTableOffset = PrologueOffset + sizeof(uint32_t)*1;
Ted Kremenek7e3a0042009-02-11 21:29:16 +0000503 const unsigned char* StringIdTable = BufBeg + ReadLE32(StringIdTableOffset);
504 if (!(StringIdTable >= BufBeg && StringIdTable < BufEnd)) {
Ted Kremenek22f0d092009-03-22 06:42:39 +0000505 InvalidPTH(Diags, level);
Ted Kremenek26555b12009-01-28 21:02:43 +0000506 return 0;
Ted Kremenek72b1b152009-01-15 18:47:46 +0000507 }
Ted Kremenek7e3a0042009-02-11 21:29:16 +0000508
509 llvm::OwningPtr<PTHStringIdLookup> SL(PTHStringIdLookup::Create(StringIdTable,
510 BufBeg));
Ted Kremenek783bb742009-03-21 00:25:09 +0000511
Ted Kremenek277faca2009-01-27 00:01:05 +0000512 // Get the location of the spelling cache.
Ted Kremeneka4bd8eb2009-02-11 23:34:32 +0000513 const unsigned char* spellingBaseOffset = PrologueOffset + sizeof(uint32_t)*3;
Ted Kremenek277faca2009-01-27 00:01:05 +0000514 const unsigned char* spellingBase = BufBeg + ReadLE32(spellingBaseOffset);
515 if (!(spellingBase >= BufBeg && spellingBase < BufEnd)) {
Ted Kremenek22f0d092009-03-22 06:42:39 +0000516 InvalidPTH(Diags, level);
Ted Kremenek277faca2009-01-27 00:01:05 +0000517 return 0;
518 }
519
Ted Kremenek6183e482008-12-03 01:16:39 +0000520 // Get the number of IdentifierInfos and pre-allocate the identifier cache.
Chris Lattner5ff43172009-01-22 19:48:26 +0000521 uint32_t NumIds = ReadLE32(IData);
Ted Kremenekcdd8f212009-01-21 07:34:28 +0000522
Ted Kremenek6183e482008-12-03 01:16:39 +0000523 // Pre-allocate the peristent ID -> IdentifierInfo* cache. We use calloc()
524 // so that we in the best case only zero out memory once when the OS returns
525 // us new pages.
Ted Kremenekcdd8f212009-01-21 07:34:28 +0000526 IdentifierInfo** PerIDCache = 0;
Ted Kremenek6183e482008-12-03 01:16:39 +0000527
Ted Kremenekcdd8f212009-01-21 07:34:28 +0000528 if (NumIds) {
529 PerIDCache = (IdentifierInfo**)calloc(NumIds, sizeof(*PerIDCache));
530 if (!PerIDCache) {
Ted Kremenek22f0d092009-03-22 06:42:39 +0000531 InvalidPTH(Diags, level,
532 "Could not allocate memory for processing PTH file");
Ted Kremenekcdd8f212009-01-21 07:34:28 +0000533 return 0;
534 }
Ted Kremenek6183e482008-12-03 01:16:39 +0000535 }
Ted Kremenekcdd8f212009-01-21 07:34:28 +0000536
Ted Kremenek68228632009-03-19 22:19:30 +0000537 // Compute the address of the original source file.
538 const unsigned char* originalSourceBase = PrologueOffset + sizeof(uint32_t)*4;
539 unsigned len = ReadUnalignedLE16(originalSourceBase);
540 if (!len) originalSourceBase = 0;
541
Ted Kremenek72b1b152009-01-15 18:47:46 +0000542 // Create the new PTHManager.
543 return new PTHManager(File.take(), FL.take(), IData, PerIDCache,
Ted Kremenek68228632009-03-19 22:19:30 +0000544 SL.take(), NumIds, spellingBase,
545 (const char*) originalSourceBase);
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000546}
Ted Kremenek68228632009-03-19 22:19:30 +0000547
Chris Lattner77ecb3a2009-01-18 02:57:21 +0000548IdentifierInfo* PTHManager::LazilyCreateIdentifierInfo(unsigned PersistentID) {
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000549 // Look in the PTH file for the string data for the IdentifierInfo object.
Chris Lattner77ecb3a2009-01-18 02:57:21 +0000550 const unsigned char* TableEntry = IdDataTable + sizeof(uint32_t)*PersistentID;
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000551 const unsigned char* IDData =
Chris Lattner5ff43172009-01-22 19:48:26 +0000552 (const unsigned char*)Buf->getBufferStart() + ReadLE32(TableEntry);
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000553 assert(IDData < (const unsigned char*)Buf->getBufferEnd());
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000554
Ted Kremenek72b1b152009-01-15 18:47:46 +0000555 // Allocate the object.
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000556 std::pair<IdentifierInfo,const unsigned char*> *Mem =
557 Alloc.Allocate<std::pair<IdentifierInfo,const unsigned char*> >();
Ted Kremenek72b1b152009-01-15 18:47:46 +0000558
559 Mem->second = IDData;
Ted Kremenek7e3a0042009-02-11 21:29:16 +0000560 assert(IDData[0] != '\0');
Ted Kremenekea9c26b2009-01-20 23:28:34 +0000561 IdentifierInfo *II = new ((void*) Mem) IdentifierInfo();
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000562
Ted Kremenek72b1b152009-01-15 18:47:46 +0000563 // Store the new IdentifierInfo in the cache.
Chris Lattner77ecb3a2009-01-18 02:57:21 +0000564 PerIDCache[PersistentID] = II;
Ted Kremenek7e3a0042009-02-11 21:29:16 +0000565 assert(II->getName() && II->getName()[0] != '\0');
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000566 return II;
567}
568
Ted Kremenek72b1b152009-01-15 18:47:46 +0000569IdentifierInfo* PTHManager::get(const char *NameStart, const char *NameEnd) {
Ted Kremenek7e3a0042009-02-11 21:29:16 +0000570 PTHStringIdLookup& SL = *((PTHStringIdLookup*)StringIdLookup);
571 // Double check our assumption that the last character isn't '\0'.
Daniel Dunbar82320e92009-02-12 19:31:53 +0000572 assert(NameEnd==NameStart || NameStart[NameEnd-NameStart-1] != '\0');
Ted Kremenek7e3a0042009-02-11 21:29:16 +0000573 PTHStringIdLookup::iterator I = SL.find(std::make_pair(NameStart,
574 NameEnd - NameStart));
575 if (I == SL.end()) // No identifier found?
576 return 0;
Ted Kremenek72b1b152009-01-15 18:47:46 +0000577
Ted Kremenek7e3a0042009-02-11 21:29:16 +0000578 // Match found. Return the identifier!
579 assert(*I > 0);
580 return GetIdentifierInfo(*I-1);
581}
Ted Kremenek72b1b152009-01-15 18:47:46 +0000582
Chris Lattnerf056d922009-01-17 08:06:50 +0000583PTHLexer *PTHManager::CreateLexer(FileID FID) {
584 const FileEntry *FE = PP->getSourceManager().getFileEntryForID(FID);
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000585 if (!FE)
586 return 0;
587
588 // Lookup the FileEntry object in our file lookup data structure. It will
589 // return a variant that indicates whether or not there is an offset within
590 // the PTH file that contains cached tokens.
Ted Kremenekd8c02922009-02-10 22:16:22 +0000591 PTHFileLookup& PFL = *((PTHFileLookup*)FileLookup);
592 PTHFileLookup::iterator I = PFL.find(FE);
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000593
Ted Kremenekd8c02922009-02-10 22:16:22 +0000594 if (I == PFL.end()) // No tokens available?
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000595 return 0;
596
Ted Kremenekd8c02922009-02-10 22:16:22 +0000597 const PTHFileData& FileData = *I;
598
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000599 const unsigned char *BufStart = (const unsigned char *)Buf->getBufferStart();
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000600 // Compute the offset of the token data within the buffer.
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000601 const unsigned char* data = BufStart + FileData.getTokenOffset();
Ted Kremenek268ee702008-12-12 18:34:08 +0000602
603 // Get the location of pp-conditional table.
Chris Lattnerda9d61c2009-01-18 01:57:14 +0000604 const unsigned char* ppcond = BufStart + FileData.getPPCondOffset();
Chris Lattner5ff43172009-01-22 19:48:26 +0000605 uint32_t Len = ReadLE32(ppcond);
Chris Lattner1b5285e2009-01-18 02:10:31 +0000606 if (Len == 0) ppcond = 0;
Ted Kremenek32a8ad52009-01-08 04:30:32 +0000607
Ted Kremenek72b1b152009-01-15 18:47:46 +0000608 assert(PP && "No preprocessor set yet!");
Ted Kremenek277faca2009-01-27 00:01:05 +0000609 return new PTHLexer(*PP, FID, data, ppcond, *this);
Ted Kremenek0c6a77b2008-12-03 00:38:03 +0000610}
Ted Kremenek337edcd2009-02-12 03:26:59 +0000611
612//===----------------------------------------------------------------------===//
613// 'stat' caching.
614//===----------------------------------------------------------------------===//
615
616namespace {
617class VISIBILITY_HIDDEN PTHStatData {
618public:
Ted Kremeneka4b44dd2009-02-13 19:13:46 +0000619 const bool hasStat;
Ted Kremenek337edcd2009-02-12 03:26:59 +0000620 const ino_t ino;
621 const dev_t dev;
622 const mode_t mode;
623 const time_t mtime;
624 const off_t size;
625
626 PTHStatData(ino_t i, dev_t d, mode_t mo, time_t m, off_t s)
Ted Kremeneka4b44dd2009-02-13 19:13:46 +0000627 : hasStat(true), ino(i), dev(d), mode(mo), mtime(m), size(s) {}
628
629 PTHStatData()
630 : hasStat(false), ino(0), dev(0), mode(0), mtime(0), size(0) {}
Ted Kremenek337edcd2009-02-12 03:26:59 +0000631};
632
633class VISIBILITY_HIDDEN PTHStatLookupTrait : public PTHFileLookupCommonTrait {
634public:
Ted Kremeneka4b44dd2009-02-13 19:13:46 +0000635 typedef const char* external_key_type; // const char*
Ted Kremenek337edcd2009-02-12 03:26:59 +0000636 typedef PTHStatData data_type;
637
Ted Kremeneka4b44dd2009-02-13 19:13:46 +0000638 static internal_key_type GetInternalKey(const char *path) {
639 // The key 'kind' doesn't matter here because it is ignored in EqualKey.
640 return std::make_pair((unsigned char) 0x0, path);
641 }
642
643 static bool EqualKey(internal_key_type a, internal_key_type b) {
644 // When doing 'stat' lookups we don't care about the kind of 'a' and 'b',
645 // just the paths.
646 return strcmp(a.second, b.second) == 0;
647 }
Ted Kremenek337edcd2009-02-12 03:26:59 +0000648
Ted Kremeneka4b44dd2009-02-13 19:13:46 +0000649 static data_type ReadData(const internal_key_type& k, const unsigned char* d,
650 unsigned) {
Ted Kremenekad6ce5c2009-02-13 22:07:44 +0000651
652 if (k.first /* File or Directory */) {
653 if (k.first == 0x1 /* File */) d += 4 * 2; // Skip the first 2 words.
Ted Kremeneka4b44dd2009-02-13 19:13:46 +0000654 ino_t ino = (ino_t) ReadUnalignedLE32(d);
655 dev_t dev = (dev_t) ReadUnalignedLE32(d);
656 mode_t mode = (mode_t) ReadUnalignedLE16(d);
657 time_t mtime = (time_t) ReadUnalignedLE64(d);
658 return data_type(ino, dev, mode, mtime, (off_t) ReadUnalignedLE64(d));
659 }
Ted Kremenekad6ce5c2009-02-13 22:07:44 +0000660
661 // Negative stat. Don't read anything.
Ted Kremeneka4b44dd2009-02-13 19:13:46 +0000662 return data_type();
Ted Kremenek337edcd2009-02-12 03:26:59 +0000663 }
664};
Ted Kremenek337edcd2009-02-12 03:26:59 +0000665
666class VISIBILITY_HIDDEN PTHStatCache : public StatSysCallCache {
667 typedef OnDiskChainedHashTable<PTHStatLookupTrait> CacheTy;
668 CacheTy Cache;
669
670public:
671 PTHStatCache(PTHFileLookup &FL) :
672 Cache(FL.getNumBuckets(), FL.getNumEntries(), FL.getBuckets(),
673 FL.getBase()) {}
674
675 ~PTHStatCache() {}
676
677 int stat(const char *path, struct stat *buf) {
678 // Do the lookup for the file's data in the PTH file.
679 CacheTy::iterator I = Cache.find(path);
680
681 // If we don't get a hit in the PTH file just forward to 'stat'.
682 if (I == Cache.end()) return ::stat(path, buf);
683
684 const PTHStatData& Data = *I;
Ted Kremenekad6ce5c2009-02-13 22:07:44 +0000685
686 if (!Data.hasStat)
687 return 1;
688
Ted Kremenek337edcd2009-02-12 03:26:59 +0000689 buf->st_ino = Data.ino;
690 buf->st_dev = Data.dev;
691 buf->st_mtime = Data.mtime;
692 buf->st_mode = Data.mode;
693 buf->st_size = Data.size;
694 return 0;
695 }
696};
Ted Kremenekd5785692009-02-23 23:27:54 +0000697} // end anonymous namespace
Ted Kremenek337edcd2009-02-12 03:26:59 +0000698
699StatSysCallCache *PTHManager::createStatCache() {
Ted Kremenek5f747d12009-02-12 03:45:39 +0000700 return new PTHStatCache(*((PTHFileLookup*) FileLookup));
Ted Kremenek337edcd2009-02-12 03:26:59 +0000701}