Ted Kremenek | 274b208 | 2008-11-12 21:37:15 +0000 | [diff] [blame] | 1 | //===--- PTHLexer.cpp - Lex from a token stream ---------------------------===// |
| 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
| 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | // |
| 10 | // This file implements the PTHLexer interface. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
Ted Kremenek | 0c6a77b | 2008-12-03 00:38:03 +0000 | [diff] [blame] | 14 | #include "clang/Basic/TokenKinds.h" |
| 15 | #include "clang/Basic/FileManager.h" |
| 16 | #include "clang/Basic/IdentifierTable.h" |
Ted Kremenek | 274b208 | 2008-11-12 21:37:15 +0000 | [diff] [blame] | 17 | #include "clang/Lex/PTHLexer.h" |
| 18 | #include "clang/Lex/Preprocessor.h" |
Ted Kremenek | 0c6a77b | 2008-12-03 00:38:03 +0000 | [diff] [blame] | 19 | #include "clang/Lex/PTHManager.h" |
| 20 | #include "clang/Lex/Token.h" |
| 21 | #include "clang/Lex/Preprocessor.h" |
| 22 | #include "llvm/Support/Compiler.h" |
| 23 | #include "llvm/Support/MemoryBuffer.h" |
| 24 | #include "llvm/ADT/StringMap.h" |
| 25 | #include "llvm/ADT/OwningPtr.h" |
Ted Kremenek | 0c6a77b | 2008-12-03 00:38:03 +0000 | [diff] [blame] | 26 | |
Ted Kremenek | 274b208 | 2008-11-12 21:37:15 +0000 | [diff] [blame] | 27 | using namespace clang; |
| 28 | |
Ted Kremenek | 18d9afb | 2008-12-23 18:41:34 +0000 | [diff] [blame] | 29 | #define DISK_TOKEN_SIZE (1+1+3+4+2) |
Ted Kremenek | 268ee70 | 2008-12-12 18:34:08 +0000 | [diff] [blame] | 30 | |
Ted Kremenek | 0c6a77b | 2008-12-03 00:38:03 +0000 | [diff] [blame] | 31 | //===----------------------------------------------------------------------===// |
| 32 | // Utility methods for reading from the mmap'ed PTH file. |
| 33 | //===----------------------------------------------------------------------===// |
| 34 | |
| 35 | static inline uint8_t Read8(const char*& data) { |
| 36 | return (uint8_t) *(data++); |
| 37 | } |
| 38 | |
| 39 | static inline uint32_t Read32(const char*& data) { |
| 40 | uint32_t V = (uint32_t) Read8(data); |
| 41 | V |= (((uint32_t) Read8(data)) << 8); |
| 42 | V |= (((uint32_t) Read8(data)) << 16); |
| 43 | V |= (((uint32_t) Read8(data)) << 24); |
| 44 | return V; |
| 45 | } |
| 46 | |
Ted Kremenek | e5680f3 | 2008-12-23 01:30:52 +0000 | [diff] [blame] | 47 | //===----------------------------------------------------------------------===// |
| 48 | // PTHLexer methods. |
| 49 | //===----------------------------------------------------------------------===// |
| 50 | |
| 51 | PTHLexer::PTHLexer(Preprocessor& pp, SourceLocation fileloc, const char* D, |
Ted Kremenek | 32a8ad5 | 2009-01-08 04:30:32 +0000 | [diff] [blame] | 52 | const char* ppcond, |
| 53 | const char* spellingTable, unsigned NumSpellings, |
| 54 | PTHManager& PM) |
Ted Kremenek | e5680f3 | 2008-12-23 01:30:52 +0000 | [diff] [blame] | 55 | : PreprocessorLexer(&pp, fileloc), TokBuf(D), CurPtr(D), LastHashTokPtr(0), |
Ted Kremenek | 32a8ad5 | 2009-01-08 04:30:32 +0000 | [diff] [blame] | 56 | PPCond(ppcond), CurPPCondPtr(ppcond), |
| 57 | SpellingTable(spellingTable), SpellingsLeft(NumSpellings), |
| 58 | PTHMgr(PM) {} |
Ted Kremenek | e5680f3 | 2008-12-23 01:30:52 +0000 | [diff] [blame] | 59 | |
| 60 | void PTHLexer::Lex(Token& Tok) { |
| 61 | LexNextToken: |
Ted Kremenek | e5680f3 | 2008-12-23 01:30:52 +0000 | [diff] [blame] | 62 | |
Ted Kremenek | 866bdf7 | 2008-12-23 02:30:15 +0000 | [diff] [blame] | 63 | //===--------------------------------------==// |
| 64 | // Read the raw token data. |
| 65 | //===--------------------------------------==// |
| 66 | |
| 67 | // Shadow CurPtr into an automatic variable. |
| 68 | const unsigned char *CurPtrShadow = (const unsigned char*) CurPtr; |
| 69 | |
| 70 | // Read in the data for the token. 14 bytes in total. |
| 71 | tok::TokenKind k = (tok::TokenKind) CurPtrShadow[0]; |
| 72 | Token::TokenFlags flags = (Token::TokenFlags) CurPtrShadow[1]; |
| 73 | |
Ted Kremenek | 6b1c970 | 2008-12-23 18:27:26 +0000 | [diff] [blame] | 74 | uint32_t perID = ((uint32_t) CurPtrShadow[2]) |
Ted Kremenek | 866bdf7 | 2008-12-23 02:30:15 +0000 | [diff] [blame] | 75 | | (((uint32_t) CurPtrShadow[3]) << 8) |
Ted Kremenek | 18d9afb | 2008-12-23 18:41:34 +0000 | [diff] [blame] | 76 | | (((uint32_t) CurPtrShadow[4]) << 16); |
Ted Kremenek | 866bdf7 | 2008-12-23 02:30:15 +0000 | [diff] [blame] | 77 | |
Ted Kremenek | 18d9afb | 2008-12-23 18:41:34 +0000 | [diff] [blame] | 78 | uint32_t FileOffset = ((uint32_t) CurPtrShadow[5]) |
| 79 | | (((uint32_t) CurPtrShadow[6]) << 8) |
| 80 | | (((uint32_t) CurPtrShadow[7]) << 16) |
| 81 | | (((uint32_t) CurPtrShadow[8]) << 24); |
Ted Kremenek | 866bdf7 | 2008-12-23 02:30:15 +0000 | [diff] [blame] | 82 | |
Ted Kremenek | 18d9afb | 2008-12-23 18:41:34 +0000 | [diff] [blame] | 83 | uint32_t Len = ((uint32_t) CurPtrShadow[9]) |
| 84 | | (((uint32_t) CurPtrShadow[10]) << 8); |
Ted Kremenek | 866bdf7 | 2008-12-23 02:30:15 +0000 | [diff] [blame] | 85 | |
| 86 | CurPtr = (const char*) (CurPtrShadow + DISK_TOKEN_SIZE); |
| 87 | |
| 88 | //===--------------------------------------==// |
| 89 | // Construct the token itself. |
| 90 | //===--------------------------------------==// |
| 91 | |
| 92 | Tok.startToken(); |
| 93 | Tok.setKind(k); |
Ted Kremenek | 59d08cb | 2008-12-23 19:24:24 +0000 | [diff] [blame] | 94 | Tok.setFlag(flags); |
| 95 | assert(!LexingRawMode); |
Ted Kremenek | 6b1c970 | 2008-12-23 18:27:26 +0000 | [diff] [blame] | 96 | Tok.setIdentifierInfo(perID ? PTHMgr.GetIdentifierInfo(perID-1) : 0); |
Ted Kremenek | 866bdf7 | 2008-12-23 02:30:15 +0000 | [diff] [blame] | 97 | Tok.setLocation(SourceLocation::getFileLoc(FileID, FileOffset)); |
| 98 | Tok.setLength(Len); |
| 99 | |
| 100 | //===--------------------------------------==// |
| 101 | // Process the token. |
| 102 | //===--------------------------------------==// |
Ted Kremenek | e5680f3 | 2008-12-23 01:30:52 +0000 | [diff] [blame] | 103 | |
Ted Kremenek | 59d08cb | 2008-12-23 19:24:24 +0000 | [diff] [blame] | 104 | if (k == tok::identifier) { |
| 105 | MIOpt.ReadToken(); |
| 106 | return PP->HandleIdentifier(Tok); |
| 107 | } |
| 108 | |
| 109 | if (k == tok::eof) { |
Ted Kremenek | e5680f3 | 2008-12-23 01:30:52 +0000 | [diff] [blame] | 110 | // Save the end-of-file token. |
| 111 | EofToken = Tok; |
| 112 | |
| 113 | Preprocessor *PPCache = PP; |
Ted Kremenek | 59d08cb | 2008-12-23 19:24:24 +0000 | [diff] [blame] | 114 | |
| 115 | assert(!ParsingPreprocessorDirective); |
| 116 | assert(!LexingRawMode); |
| 117 | |
| 118 | // FIXME: Issue diagnostics similar to Lexer. |
| 119 | if (PP->HandleEndOfFile(Tok, false)) |
Ted Kremenek | e5680f3 | 2008-12-23 01:30:52 +0000 | [diff] [blame] | 120 | return; |
Ted Kremenek | 59d08cb | 2008-12-23 19:24:24 +0000 | [diff] [blame] | 121 | |
Ted Kremenek | e5680f3 | 2008-12-23 01:30:52 +0000 | [diff] [blame] | 122 | assert(PPCache && "Raw buffer::LexEndOfFile should return a token"); |
| 123 | return PPCache->Lex(Tok); |
| 124 | } |
Ted Kremenek | e5680f3 | 2008-12-23 01:30:52 +0000 | [diff] [blame] | 125 | |
Ted Kremenek | 59d08cb | 2008-12-23 19:24:24 +0000 | [diff] [blame] | 126 | if (k == tok::hash && Tok.isAtStartOfLine()) { |
| 127 | LastHashTokPtr = CurPtr - DISK_TOKEN_SIZE; |
| 128 | assert(!LexingRawMode); |
| 129 | PP->HandleDirective(Tok); |
| 130 | |
| 131 | if (PP->isCurrentLexer(this)) |
| 132 | goto LexNextToken; |
| 133 | |
| 134 | return PP->Lex(Tok); |
| 135 | } |
| 136 | |
| 137 | if (k == tok::eom) { |
| 138 | assert(ParsingPreprocessorDirective); |
Ted Kremenek | e5680f3 | 2008-12-23 01:30:52 +0000 | [diff] [blame] | 139 | ParsingPreprocessorDirective = false; |
| 140 | return; |
| 141 | } |
Ted Kremenek | e5680f3 | 2008-12-23 01:30:52 +0000 | [diff] [blame] | 142 | |
Ted Kremenek | 59d08cb | 2008-12-23 19:24:24 +0000 | [diff] [blame] | 143 | MIOpt.ReadToken(); |
Ted Kremenek | e5680f3 | 2008-12-23 01:30:52 +0000 | [diff] [blame] | 144 | } |
| 145 | |
| 146 | // FIXME: We can just grab the last token instead of storing a copy |
| 147 | // into EofToken. |
Ted Kremenek | 59d08cb | 2008-12-23 19:24:24 +0000 | [diff] [blame] | 148 | void PTHLexer::getEOF(Token& Tok) { |
Ted Kremenek | defb709 | 2009-01-09 00:36:11 +0000 | [diff] [blame^] | 149 | assert(EofToken.is(tok::eof)); |
Ted Kremenek | e5680f3 | 2008-12-23 01:30:52 +0000 | [diff] [blame] | 150 | Tok = EofToken; |
| 151 | } |
| 152 | |
| 153 | void PTHLexer::DiscardToEndOfLine() { |
| 154 | assert(ParsingPreprocessorDirective && ParsingFilename == false && |
| 155 | "Must be in a preprocessing directive!"); |
| 156 | |
| 157 | // We assume that if the preprocessor wishes to discard to the end of |
| 158 | // the line that it also means to end the current preprocessor directive. |
| 159 | ParsingPreprocessorDirective = false; |
| 160 | |
| 161 | // Skip tokens by only peeking at their token kind and the flags. |
| 162 | // We don't need to actually reconstruct full tokens from the token buffer. |
| 163 | // This saves some copies and it also reduces IdentifierInfo* lookup. |
| 164 | const char* p = CurPtr; |
| 165 | while (1) { |
| 166 | // Read the token kind. Are we at the end of the file? |
| 167 | tok::TokenKind x = (tok::TokenKind) (uint8_t) *p; |
| 168 | if (x == tok::eof) break; |
| 169 | |
| 170 | // Read the token flags. Are we at the start of the next line? |
| 171 | Token::TokenFlags y = (Token::TokenFlags) (uint8_t) p[1]; |
| 172 | if (y & Token::StartOfLine) break; |
| 173 | |
| 174 | // Skip to the next token. |
| 175 | p += DISK_TOKEN_SIZE; |
| 176 | } |
| 177 | |
| 178 | CurPtr = p; |
| 179 | } |
| 180 | |
Ted Kremenek | 268ee70 | 2008-12-12 18:34:08 +0000 | [diff] [blame] | 181 | /// SkipBlock - Used by Preprocessor to skip the current conditional block. |
| 182 | bool PTHLexer::SkipBlock() { |
| 183 | assert(CurPPCondPtr && "No cached PP conditional information."); |
| 184 | assert(LastHashTokPtr && "No known '#' token."); |
| 185 | |
Ted Kremenek | 41a2660 | 2008-12-12 22:05:38 +0000 | [diff] [blame] | 186 | const char* HashEntryI = 0; |
Ted Kremenek | 268ee70 | 2008-12-12 18:34:08 +0000 | [diff] [blame] | 187 | uint32_t Offset; |
| 188 | uint32_t TableIdx; |
| 189 | |
| 190 | do { |
Ted Kremenek | 41a2660 | 2008-12-12 22:05:38 +0000 | [diff] [blame] | 191 | // Read the token offset from the side-table. |
Ted Kremenek | 268ee70 | 2008-12-12 18:34:08 +0000 | [diff] [blame] | 192 | Offset = Read32(CurPPCondPtr); |
Ted Kremenek | 41a2660 | 2008-12-12 22:05:38 +0000 | [diff] [blame] | 193 | |
| 194 | // Read the target table index from the side-table. |
Ted Kremenek | 268ee70 | 2008-12-12 18:34:08 +0000 | [diff] [blame] | 195 | TableIdx = Read32(CurPPCondPtr); |
Ted Kremenek | 41a2660 | 2008-12-12 22:05:38 +0000 | [diff] [blame] | 196 | |
| 197 | // Compute the actual memory address of the '#' token data for this entry. |
| 198 | HashEntryI = TokBuf + Offset; |
| 199 | |
| 200 | // Optmization: "Sibling jumping". #if...#else...#endif blocks can |
| 201 | // contain nested blocks. In the side-table we can jump over these |
| 202 | // nested blocks instead of doing a linear search if the next "sibling" |
| 203 | // entry is not at a location greater than LastHashTokPtr. |
| 204 | if (HashEntryI < LastHashTokPtr && TableIdx) { |
| 205 | // In the side-table we are still at an entry for a '#' token that |
| 206 | // is earlier than the last one we saw. Check if the location we would |
| 207 | // stride gets us closer. |
| 208 | const char* NextPPCondPtr = PPCond + TableIdx*(sizeof(uint32_t)*2); |
| 209 | assert(NextPPCondPtr >= CurPPCondPtr); |
| 210 | // Read where we should jump to. |
| 211 | uint32_t TmpOffset = Read32(NextPPCondPtr); |
| 212 | const char* HashEntryJ = TokBuf + TmpOffset; |
| 213 | |
| 214 | if (HashEntryJ <= LastHashTokPtr) { |
| 215 | // Jump directly to the next entry in the side table. |
| 216 | HashEntryI = HashEntryJ; |
| 217 | Offset = TmpOffset; |
| 218 | TableIdx = Read32(NextPPCondPtr); |
| 219 | CurPPCondPtr = NextPPCondPtr; |
| 220 | } |
| 221 | } |
Ted Kremenek | 268ee70 | 2008-12-12 18:34:08 +0000 | [diff] [blame] | 222 | } |
Ted Kremenek | 41a2660 | 2008-12-12 22:05:38 +0000 | [diff] [blame] | 223 | while (HashEntryI < LastHashTokPtr); |
| 224 | assert(HashEntryI == LastHashTokPtr && "No PP-cond entry found for '#'"); |
Ted Kremenek | 268ee70 | 2008-12-12 18:34:08 +0000 | [diff] [blame] | 225 | assert(TableIdx && "No jumping from #endifs."); |
| 226 | |
| 227 | // Update our side-table iterator. |
| 228 | const char* NextPPCondPtr = PPCond + TableIdx*(sizeof(uint32_t)*2); |
| 229 | assert(NextPPCondPtr >= CurPPCondPtr); |
| 230 | CurPPCondPtr = NextPPCondPtr; |
| 231 | |
| 232 | // Read where we should jump to. |
Ted Kremenek | 41a2660 | 2008-12-12 22:05:38 +0000 | [diff] [blame] | 233 | HashEntryI = TokBuf + Read32(NextPPCondPtr); |
Ted Kremenek | 268ee70 | 2008-12-12 18:34:08 +0000 | [diff] [blame] | 234 | uint32_t NextIdx = Read32(NextPPCondPtr); |
| 235 | |
| 236 | // By construction NextIdx will be zero if this is a #endif. This is useful |
| 237 | // to know to obviate lexing another token. |
| 238 | bool isEndif = NextIdx == 0; |
Ted Kremenek | 268ee70 | 2008-12-12 18:34:08 +0000 | [diff] [blame] | 239 | |
| 240 | // This case can occur when we see something like this: |
| 241 | // |
| 242 | // #if ... |
| 243 | // /* a comment or nothing */ |
| 244 | // #elif |
| 245 | // |
| 246 | // If we are skipping the first #if block it will be the case that CurPtr |
| 247 | // already points 'elif'. Just return. |
| 248 | |
Ted Kremenek | 41a2660 | 2008-12-12 22:05:38 +0000 | [diff] [blame] | 249 | if (CurPtr > HashEntryI) { |
| 250 | assert(CurPtr == HashEntryI + DISK_TOKEN_SIZE); |
Ted Kremenek | 268ee70 | 2008-12-12 18:34:08 +0000 | [diff] [blame] | 251 | // Did we reach a #endif? If so, go ahead and consume that token as well. |
| 252 | if (isEndif) |
Ted Kremenek | e5680f3 | 2008-12-23 01:30:52 +0000 | [diff] [blame] | 253 | CurPtr += DISK_TOKEN_SIZE*2; |
Ted Kremenek | 268ee70 | 2008-12-12 18:34:08 +0000 | [diff] [blame] | 254 | else |
Ted Kremenek | 41a2660 | 2008-12-12 22:05:38 +0000 | [diff] [blame] | 255 | LastHashTokPtr = HashEntryI; |
Ted Kremenek | 268ee70 | 2008-12-12 18:34:08 +0000 | [diff] [blame] | 256 | |
| 257 | return isEndif; |
| 258 | } |
| 259 | |
| 260 | // Otherwise, we need to advance. Update CurPtr to point to the '#' token. |
Ted Kremenek | 41a2660 | 2008-12-12 22:05:38 +0000 | [diff] [blame] | 261 | CurPtr = HashEntryI; |
Ted Kremenek | 268ee70 | 2008-12-12 18:34:08 +0000 | [diff] [blame] | 262 | |
| 263 | // Update the location of the last observed '#'. This is useful if we |
| 264 | // are skipping multiple blocks. |
| 265 | LastHashTokPtr = CurPtr; |
Ted Kremenek | 268ee70 | 2008-12-12 18:34:08 +0000 | [diff] [blame] | 266 | |
Ted Kremenek | e5680f3 | 2008-12-23 01:30:52 +0000 | [diff] [blame] | 267 | // Skip the '#' token. |
| 268 | assert(((tok::TokenKind) (unsigned char) *CurPtr) == tok::hash); |
| 269 | CurPtr += DISK_TOKEN_SIZE; |
| 270 | |
Ted Kremenek | 268ee70 | 2008-12-12 18:34:08 +0000 | [diff] [blame] | 271 | // Did we reach a #endif? If so, go ahead and consume that token as well. |
Ted Kremenek | e5680f3 | 2008-12-23 01:30:52 +0000 | [diff] [blame] | 272 | if (isEndif) { CurPtr += DISK_TOKEN_SIZE*2; } |
Ted Kremenek | 268ee70 | 2008-12-12 18:34:08 +0000 | [diff] [blame] | 273 | |
| 274 | return isEndif; |
| 275 | } |
| 276 | |
Ted Kremenek | 30a12ec | 2008-12-17 23:36:32 +0000 | [diff] [blame] | 277 | SourceLocation PTHLexer::getSourceLocation() { |
| 278 | // getLocation is not on the hot path. It is used to get the location of |
| 279 | // the next token when transitioning back to this lexer when done |
| 280 | // handling a #included file. Just read the necessary data from the token |
| 281 | // data buffer to construct the SourceLocation object. |
| 282 | // NOTE: This is a virtual function; hence it is defined out-of-line. |
Ted Kremenek | 59d08cb | 2008-12-23 19:24:24 +0000 | [diff] [blame] | 283 | const char* p = CurPtr + (1 + 1 + 3); |
Ted Kremenek | 30a12ec | 2008-12-17 23:36:32 +0000 | [diff] [blame] | 284 | uint32_t offset = |
| 285 | ((uint32_t) ((uint8_t) p[0])) |
| 286 | | (((uint32_t) ((uint8_t) p[1])) << 8) |
| 287 | | (((uint32_t) ((uint8_t) p[2])) << 16) |
| 288 | | (((uint32_t) ((uint8_t) p[3])) << 24); |
| 289 | return SourceLocation::getFileLoc(FileID, offset); |
| 290 | } |
| 291 | |
Ted Kremenek | 32a8ad5 | 2009-01-08 04:30:32 +0000 | [diff] [blame] | 292 | unsigned PTHManager::GetSpelling(unsigned PTHOffset, const char *& Buffer) { |
| 293 | const char* p = Buf->getBufferStart() + PTHOffset; |
| 294 | assert(p < Buf->getBufferEnd()); |
| 295 | |
| 296 | // The string is prefixed by 16 bits for its length, followed by the string |
| 297 | // itself. |
| 298 | unsigned len = ((unsigned) ((uint8_t) p[0])) |
| 299 | | (((unsigned) ((uint8_t) p[1])) << 8); |
| 300 | |
| 301 | Buffer = p + 2; |
| 302 | return len; |
| 303 | } |
| 304 | |
Ted Kremenek | b70e3da | 2009-01-08 02:47:16 +0000 | [diff] [blame] | 305 | unsigned PTHLexer::getSpelling(SourceLocation sloc, const char *&Buffer) { |
Ted Kremenek | 32a8ad5 | 2009-01-08 04:30:32 +0000 | [diff] [blame] | 306 | const char* p = SpellingTable; |
| 307 | SourceManager& SM = PP->getSourceManager(); |
| 308 | unsigned fpos = SM.getFullFilePos(SM.getPhysicalLoc(sloc)); |
| 309 | unsigned len = 0; |
| 310 | |
| 311 | while (SpellingsLeft) { |
| 312 | uint32_t TokOffset = |
| 313 | ((uint32_t) ((uint8_t) p[0])) |
| 314 | | (((uint32_t) ((uint8_t) p[1])) << 8) |
| 315 | | (((uint32_t) ((uint8_t) p[2])) << 16) |
| 316 | | (((uint32_t) ((uint8_t) p[3])) << 24); |
| 317 | |
| 318 | if (TokOffset > fpos) |
| 319 | break; |
| 320 | |
| 321 | --SpellingsLeft; |
| 322 | |
| 323 | // Did we find a matching token offset for this spelling? |
| 324 | if (TokOffset == fpos) { |
| 325 | uint32_t SpellingPTHOffset = |
| 326 | ((uint32_t) ((uint8_t) p[4])) |
| 327 | | (((uint32_t) ((uint8_t) p[5])) << 8) |
| 328 | | (((uint32_t) ((uint8_t) p[6])) << 16) |
| 329 | | (((uint32_t) ((uint8_t) p[7])) << 24); |
| 330 | |
| 331 | len = PTHMgr.GetSpelling(SpellingPTHOffset, Buffer); |
| 332 | break; |
| 333 | } |
| 334 | |
| 335 | // No match. Keep on looking. |
| 336 | p += sizeof(uint32_t)*2; |
| 337 | } |
| 338 | |
| 339 | SpellingTable = p; |
| 340 | return len; |
Ted Kremenek | b70e3da | 2009-01-08 02:47:16 +0000 | [diff] [blame] | 341 | } |
| 342 | |
Ted Kremenek | 0c6a77b | 2008-12-03 00:38:03 +0000 | [diff] [blame] | 343 | //===----------------------------------------------------------------------===// |
Ted Kremenek | 0c6a77b | 2008-12-03 00:38:03 +0000 | [diff] [blame] | 344 | // Internal Data Structures for PTH file lookup and resolving identifiers. |
| 345 | //===----------------------------------------------------------------------===// |
| 346 | |
Ted Kremenek | 0c6a77b | 2008-12-03 00:38:03 +0000 | [diff] [blame] | 347 | |
| 348 | /// PTHFileLookup - This internal data structure is used by the PTHManager |
| 349 | /// to map from FileEntry objects managed by FileManager to offsets within |
| 350 | /// the PTH file. |
| 351 | namespace { |
| 352 | class VISIBILITY_HIDDEN PTHFileLookup { |
| 353 | public: |
| 354 | class Val { |
Ted Kremenek | fb645b6 | 2008-12-11 23:36:38 +0000 | [diff] [blame] | 355 | uint32_t TokenOff; |
| 356 | uint32_t PPCondOff; |
Ted Kremenek | b70e3da | 2009-01-08 02:47:16 +0000 | [diff] [blame] | 357 | uint32_t SpellingOff; |
Ted Kremenek | 0c6a77b | 2008-12-03 00:38:03 +0000 | [diff] [blame] | 358 | |
| 359 | public: |
Ted Kremenek | fb645b6 | 2008-12-11 23:36:38 +0000 | [diff] [blame] | 360 | Val() : TokenOff(~0) {} |
Ted Kremenek | b70e3da | 2009-01-08 02:47:16 +0000 | [diff] [blame] | 361 | Val(uint32_t toff, uint32_t poff, uint32_t soff) |
| 362 | : TokenOff(toff), PPCondOff(poff), SpellingOff(soff) {} |
Ted Kremenek | 0c6a77b | 2008-12-03 00:38:03 +0000 | [diff] [blame] | 363 | |
Ted Kremenek | fb645b6 | 2008-12-11 23:36:38 +0000 | [diff] [blame] | 364 | uint32_t getTokenOffset() const { |
| 365 | assert(TokenOff != ~((uint32_t)0) && "PTHFileLookup entry initialized."); |
| 366 | return TokenOff; |
Ted Kremenek | 0c6a77b | 2008-12-03 00:38:03 +0000 | [diff] [blame] | 367 | } |
| 368 | |
Ted Kremenek | b70e3da | 2009-01-08 02:47:16 +0000 | [diff] [blame] | 369 | uint32_t getPPCondOffset() const { |
Ted Kremenek | fb645b6 | 2008-12-11 23:36:38 +0000 | [diff] [blame] | 370 | assert(TokenOff != ~((uint32_t)0) && "PTHFileLookup entry initialized."); |
| 371 | return PPCondOff; |
| 372 | } |
| 373 | |
Ted Kremenek | b70e3da | 2009-01-08 02:47:16 +0000 | [diff] [blame] | 374 | uint32_t getSpellingOffset() const { |
| 375 | assert(TokenOff != ~((uint32_t)0) && "PTHFileLookup entry initialized."); |
| 376 | return SpellingOff; |
| 377 | } |
| 378 | |
Ted Kremenek | fb645b6 | 2008-12-11 23:36:38 +0000 | [diff] [blame] | 379 | bool isValid() const { return TokenOff != ~((uint32_t)0); } |
Ted Kremenek | 0c6a77b | 2008-12-03 00:38:03 +0000 | [diff] [blame] | 380 | }; |
| 381 | |
| 382 | private: |
| 383 | llvm::StringMap<Val> FileMap; |
| 384 | |
| 385 | public: |
| 386 | PTHFileLookup() {}; |
| 387 | |
| 388 | Val Lookup(const FileEntry* FE) { |
| 389 | const char* s = FE->getName(); |
| 390 | unsigned size = strlen(s); |
| 391 | return FileMap.GetOrCreateValue(s, s+size).getValue(); |
| 392 | } |
| 393 | |
| 394 | void ReadTable(const char* D) { |
| 395 | uint32_t N = Read32(D); // Read the length of the table. |
| 396 | |
| 397 | for ( ; N > 0; --N) { // The rest of the data is the table itself. |
| 398 | uint32_t len = Read32(D); |
| 399 | const char* s = D; |
| 400 | D += len; |
Ted Kremenek | b70e3da | 2009-01-08 02:47:16 +0000 | [diff] [blame] | 401 | |
Ted Kremenek | fb645b6 | 2008-12-11 23:36:38 +0000 | [diff] [blame] | 402 | uint32_t TokenOff = Read32(D); |
Ted Kremenek | b70e3da | 2009-01-08 02:47:16 +0000 | [diff] [blame] | 403 | uint32_t PPCondOff = Read32(D); |
| 404 | uint32_t SpellingOff = Read32(D); |
| 405 | |
| 406 | FileMap.GetOrCreateValue(s, s+len).getValue() = |
| 407 | Val(TokenOff, PPCondOff, SpellingOff); |
Ted Kremenek | 0c6a77b | 2008-12-03 00:38:03 +0000 | [diff] [blame] | 408 | } |
| 409 | } |
| 410 | }; |
| 411 | } // end anonymous namespace |
| 412 | |
| 413 | //===----------------------------------------------------------------------===// |
| 414 | // PTHManager methods. |
| 415 | //===----------------------------------------------------------------------===// |
| 416 | |
| 417 | PTHManager::PTHManager(const llvm::MemoryBuffer* buf, void* fileLookup, |
Ted Kremenek | cf58e62 | 2008-12-10 19:40:23 +0000 | [diff] [blame] | 418 | const char* idDataTable, IdentifierInfo** perIDCache, |
Ted Kremenek | 6183e48 | 2008-12-03 01:16:39 +0000 | [diff] [blame] | 419 | Preprocessor& pp) |
| 420 | : Buf(buf), PerIDCache(perIDCache), FileLookup(fileLookup), |
| 421 | IdDataTable(idDataTable), ITable(pp.getIdentifierTable()), PP(pp) {} |
Ted Kremenek | 0c6a77b | 2008-12-03 00:38:03 +0000 | [diff] [blame] | 422 | |
| 423 | PTHManager::~PTHManager() { |
| 424 | delete Buf; |
| 425 | delete (PTHFileLookup*) FileLookup; |
Ted Kremenek | 0e50b6e | 2008-12-04 22:47:11 +0000 | [diff] [blame] | 426 | free(PerIDCache); |
Ted Kremenek | 0c6a77b | 2008-12-03 00:38:03 +0000 | [diff] [blame] | 427 | } |
| 428 | |
| 429 | PTHManager* PTHManager::Create(const std::string& file, Preprocessor& PP) { |
| 430 | |
| 431 | // Memory map the PTH file. |
| 432 | llvm::OwningPtr<llvm::MemoryBuffer> |
| 433 | File(llvm::MemoryBuffer::getFile(file.c_str())); |
| 434 | |
| 435 | if (!File) |
| 436 | return 0; |
| 437 | |
| 438 | // Get the buffer ranges and check if there are at least three 32-bit |
| 439 | // words at the end of the file. |
| 440 | const char* BufBeg = File->getBufferStart(); |
| 441 | const char* BufEnd = File->getBufferEnd(); |
| 442 | |
| 443 | if(!(BufEnd > BufBeg + sizeof(uint32_t)*3)) { |
| 444 | assert(false && "Invalid PTH file."); |
| 445 | return 0; // FIXME: Proper error diagnostic? |
| 446 | } |
| 447 | |
| 448 | // Compute the address of the index table at the end of the PTH file. |
| 449 | // This table contains the offset of the file lookup table, the |
| 450 | // persistent ID -> identifer data table. |
| 451 | const char* EndTable = BufEnd - sizeof(uint32_t)*3; |
| 452 | |
| 453 | // Construct the file lookup table. This will be used for mapping from |
| 454 | // FileEntry*'s to cached tokens. |
| 455 | const char* FileTableOffset = EndTable + sizeof(uint32_t)*2; |
| 456 | const char* FileTable = BufBeg + Read32(FileTableOffset); |
| 457 | |
| 458 | if (!(FileTable > BufBeg && FileTable < BufEnd)) { |
| 459 | assert(false && "Invalid PTH file."); |
| 460 | return 0; // FIXME: Proper error diagnostic? |
| 461 | } |
| 462 | |
| 463 | llvm::OwningPtr<PTHFileLookup> FL(new PTHFileLookup()); |
| 464 | FL->ReadTable(FileTable); |
| 465 | |
| 466 | // Get the location of the table mapping from persistent ids to the |
| 467 | // data needed to reconstruct identifiers. |
| 468 | const char* IDTableOffset = EndTable + sizeof(uint32_t)*1; |
| 469 | const char* IData = BufBeg + Read32(IDTableOffset); |
| 470 | if (!(IData > BufBeg && IData < BufEnd)) { |
| 471 | assert(false && "Invalid PTH file."); |
| 472 | return 0; // FIXME: Proper error diagnostic? |
| 473 | } |
| 474 | |
Ted Kremenek | 6183e48 | 2008-12-03 01:16:39 +0000 | [diff] [blame] | 475 | // Get the number of IdentifierInfos and pre-allocate the identifier cache. |
| 476 | uint32_t NumIds = Read32(IData); |
| 477 | |
| 478 | // Pre-allocate the peristent ID -> IdentifierInfo* cache. We use calloc() |
| 479 | // so that we in the best case only zero out memory once when the OS returns |
| 480 | // us new pages. |
| 481 | IdentifierInfo** PerIDCache = |
| 482 | (IdentifierInfo**) calloc(NumIds, sizeof(*PerIDCache)); |
| 483 | |
| 484 | if (!PerIDCache) { |
| 485 | assert(false && "Could not allocate Persistent ID cache."); |
| 486 | return 0; |
| 487 | } |
| 488 | |
| 489 | // Create the new lexer. |
| 490 | return new PTHManager(File.take(), FL.take(), IData, PerIDCache, PP); |
Ted Kremenek | 0c6a77b | 2008-12-03 00:38:03 +0000 | [diff] [blame] | 491 | } |
| 492 | |
Ted Kremenek | 866bdf7 | 2008-12-23 02:30:15 +0000 | [diff] [blame] | 493 | IdentifierInfo* PTHManager::GetIdentifierInfo(unsigned persistentID) { |
| 494 | |
Ted Kremenek | 0c6a77b | 2008-12-03 00:38:03 +0000 | [diff] [blame] | 495 | // Check if the IdentifierInfo has already been resolved. |
Ted Kremenek | cf58e62 | 2008-12-10 19:40:23 +0000 | [diff] [blame] | 496 | IdentifierInfo*& II = PerIDCache[persistentID]; |
Ted Kremenek | 0c6a77b | 2008-12-03 00:38:03 +0000 | [diff] [blame] | 497 | if (II) return II; |
| 498 | |
| 499 | // Look in the PTH file for the string data for the IdentifierInfo object. |
| 500 | const char* TableEntry = IdDataTable + sizeof(uint32_t) * persistentID; |
| 501 | const char* IDData = Buf->getBufferStart() + Read32(TableEntry); |
| 502 | assert(IDData < Buf->getBufferEnd()); |
| 503 | |
| 504 | // Read the length of the string. |
| 505 | uint32_t len = Read32(IDData); |
| 506 | |
| 507 | // Get the IdentifierInfo* with the specified string. |
| 508 | II = &ITable.get(IDData, IDData+len); |
| 509 | return II; |
| 510 | } |
| 511 | |
| 512 | PTHLexer* PTHManager::CreateLexer(unsigned FileID, const FileEntry* FE) { |
| 513 | |
| 514 | if (!FE) |
| 515 | return 0; |
| 516 | |
| 517 | // Lookup the FileEntry object in our file lookup data structure. It will |
| 518 | // return a variant that indicates whether or not there is an offset within |
| 519 | // the PTH file that contains cached tokens. |
Ted Kremenek | fb645b6 | 2008-12-11 23:36:38 +0000 | [diff] [blame] | 520 | PTHFileLookup::Val FileData = ((PTHFileLookup*) FileLookup)->Lookup(FE); |
Ted Kremenek | 0c6a77b | 2008-12-03 00:38:03 +0000 | [diff] [blame] | 521 | |
Ted Kremenek | fb645b6 | 2008-12-11 23:36:38 +0000 | [diff] [blame] | 522 | if (!FileData.isValid()) // No tokens available. |
Ted Kremenek | 0c6a77b | 2008-12-03 00:38:03 +0000 | [diff] [blame] | 523 | return 0; |
| 524 | |
| 525 | // Compute the offset of the token data within the buffer. |
Ted Kremenek | fb645b6 | 2008-12-11 23:36:38 +0000 | [diff] [blame] | 526 | const char* data = Buf->getBufferStart() + FileData.getTokenOffset(); |
Ted Kremenek | 268ee70 | 2008-12-12 18:34:08 +0000 | [diff] [blame] | 527 | |
| 528 | // Get the location of pp-conditional table. |
Ted Kremenek | b70e3da | 2009-01-08 02:47:16 +0000 | [diff] [blame] | 529 | const char* ppcond = Buf->getBufferStart() + FileData.getPPCondOffset(); |
| 530 | uint32_t len = Read32(ppcond); |
Ted Kremenek | 268ee70 | 2008-12-12 18:34:08 +0000 | [diff] [blame] | 531 | if (len == 0) ppcond = 0; |
Ted Kremenek | 32a8ad5 | 2009-01-08 04:30:32 +0000 | [diff] [blame] | 532 | |
| 533 | // Get the location of the spelling table. |
| 534 | const char* spellingTable = Buf->getBufferStart() + |
| 535 | FileData.getSpellingOffset(); |
| 536 | |
| 537 | len = Read32(spellingTable); |
| 538 | if (len == 0) spellingTable = 0; |
Ted Kremenek | b70e3da | 2009-01-08 02:47:16 +0000 | [diff] [blame] | 539 | |
Ted Kremenek | 0c6a77b | 2008-12-03 00:38:03 +0000 | [diff] [blame] | 540 | assert(data < Buf->getBufferEnd()); |
Ted Kremenek | 268ee70 | 2008-12-12 18:34:08 +0000 | [diff] [blame] | 541 | return new PTHLexer(PP, SourceLocation::getFileLoc(FileID, 0), data, ppcond, |
Ted Kremenek | 32a8ad5 | 2009-01-08 04:30:32 +0000 | [diff] [blame] | 542 | spellingTable, len, *this); |
Ted Kremenek | 0c6a77b | 2008-12-03 00:38:03 +0000 | [diff] [blame] | 543 | } |