blob: fd5bb617a08f1d9d77040554bae6afbc64cb5c5c [file] [log] [blame]
Reid Spencer5f016e22007-07-11 17:01:13 +00001//===--- Lexer.cpp - C Language Family Lexer ------------------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
Chris Lattner0bc735f2007-12-29 19:59:25 +00005// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
Reid Spencer5f016e22007-07-11 17:01:13 +00007//
8//===----------------------------------------------------------------------===//
9//
Chris Lattnerd2177732007-07-20 16:59:19 +000010// This file implements the Lexer and Token interfaces.
Reid Spencer5f016e22007-07-11 17:01:13 +000011//
12//===----------------------------------------------------------------------===//
13//
14// TODO: GCC Diagnostics emitted by the lexer:
15// PEDWARN: (form feed|vertical tab) in preprocessing directive
16//
17// Universal characters, unicode, char mapping:
18// WARNING: `%.*s' is not in NFKC
19// WARNING: `%.*s' is not in NFC
20//
21// Other:
22// TODO: Options to support:
23// -fexec-charset,-fwide-exec-charset
24//
25//===----------------------------------------------------------------------===//
26
27#include "clang/Lex/Lexer.h"
28#include "clang/Lex/Preprocessor.h"
Chris Lattner500d3292009-01-29 05:15:15 +000029#include "clang/Lex/LexDiagnostic.h"
Douglas Gregor55817af2010-08-25 17:04:25 +000030#include "clang/Lex/CodeCompletionHandler.h"
Chris Lattner9dc1f532007-07-20 16:37:10 +000031#include "clang/Basic/SourceManager.h"
Douglas Gregorf033f1d2010-07-20 20:18:03 +000032#include "llvm/ADT/StringSwitch.h"
Chris Lattner409a0362007-07-22 18:38:25 +000033#include "llvm/Support/Compiler.h"
Reid Spencer5f016e22007-07-11 17:01:13 +000034#include "llvm/Support/MemoryBuffer.h"
35#include <cctype>
36using namespace clang;
37
Chris Lattnera2bf1052009-12-17 05:29:40 +000038static void InitCharacterInfo();
Reid Spencer5f016e22007-07-11 17:01:13 +000039
Chris Lattnerdbf388b2007-10-07 08:47:24 +000040//===----------------------------------------------------------------------===//
41// Token Class Implementation
42//===----------------------------------------------------------------------===//
43
Mike Stump1eb44332009-09-09 15:08:12 +000044/// isObjCAtKeyword - Return true if we have an ObjC keyword identifier.
Chris Lattnerdbf388b2007-10-07 08:47:24 +000045bool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const {
Douglas Gregorbec1c9d2008-12-01 21:46:47 +000046 if (IdentifierInfo *II = getIdentifierInfo())
47 return II->getObjCKeywordID() == objcKey;
48 return false;
Chris Lattnerdbf388b2007-10-07 08:47:24 +000049}
50
51/// getObjCKeywordID - Return the ObjC keyword kind.
52tok::ObjCKeywordKind Token::getObjCKeywordID() const {
53 IdentifierInfo *specId = getIdentifierInfo();
54 return specId ? specId->getObjCKeywordID() : tok::objc_not_keyword;
55}
56
Chris Lattner53702cd2007-12-13 01:59:49 +000057
Chris Lattnerdbf388b2007-10-07 08:47:24 +000058//===----------------------------------------------------------------------===//
59// Lexer Class Implementation
60//===----------------------------------------------------------------------===//
61
Mike Stump1eb44332009-09-09 15:08:12 +000062void Lexer::InitLexer(const char *BufStart, const char *BufPtr,
Chris Lattner22d91ca2009-01-17 06:55:17 +000063 const char *BufEnd) {
Chris Lattnera2bf1052009-12-17 05:29:40 +000064 InitCharacterInfo();
Mike Stump1eb44332009-09-09 15:08:12 +000065
Chris Lattner22d91ca2009-01-17 06:55:17 +000066 BufferStart = BufStart;
67 BufferPtr = BufPtr;
68 BufferEnd = BufEnd;
Mike Stump1eb44332009-09-09 15:08:12 +000069
Chris Lattner22d91ca2009-01-17 06:55:17 +000070 assert(BufEnd[0] == 0 &&
71 "We assume that the input buffer has a null character at the end"
72 " to simplify lexing!");
Mike Stump1eb44332009-09-09 15:08:12 +000073
Chris Lattner22d91ca2009-01-17 06:55:17 +000074 Is_PragmaLexer = false;
Chris Lattner34f349d2009-12-14 06:16:57 +000075 IsInConflictMarker = false;
Douglas Gregor81b747b2009-09-17 21:32:03 +000076
Chris Lattner22d91ca2009-01-17 06:55:17 +000077 // Start of the file is a start of line.
78 IsAtStartOfLine = true;
Mike Stump1eb44332009-09-09 15:08:12 +000079
Chris Lattner22d91ca2009-01-17 06:55:17 +000080 // We are not after parsing a #.
81 ParsingPreprocessorDirective = false;
Mike Stump1eb44332009-09-09 15:08:12 +000082
Chris Lattner22d91ca2009-01-17 06:55:17 +000083 // We are not after parsing #include.
84 ParsingFilename = false;
Mike Stump1eb44332009-09-09 15:08:12 +000085
Chris Lattner22d91ca2009-01-17 06:55:17 +000086 // We are not in raw mode. Raw mode disables diagnostics and interpretation
87 // of tokens (e.g. identifiers, thus disabling macro expansion). It is used
88 // to quickly lex the tokens of the buffer, e.g. when handling a "#if 0" block
89 // or otherwise skipping over tokens.
90 LexingRawMode = false;
Mike Stump1eb44332009-09-09 15:08:12 +000091
Chris Lattner22d91ca2009-01-17 06:55:17 +000092 // Default to not keeping comments.
93 ExtendedTokenMode = 0;
94}
95
Chris Lattner0770dab2009-01-17 07:56:59 +000096/// Lexer constructor - Create a new lexer object for the specified buffer
97/// with the specified preprocessor managing the lexing process. This lexer
98/// assumes that the associated file buffer and Preprocessor objects will
99/// outlive it, so it doesn't take ownership of either of them.
Chris Lattner6e290142009-11-30 04:18:44 +0000100Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *InputFile, Preprocessor &PP)
Chris Lattner88d3ac12009-01-17 08:03:42 +0000101 : PreprocessorLexer(&PP, FID),
102 FileLoc(PP.getSourceManager().getLocForStartOfFile(FID)),
103 Features(PP.getLangOptions()) {
Mike Stump1eb44332009-09-09 15:08:12 +0000104
Chris Lattner0770dab2009-01-17 07:56:59 +0000105 InitLexer(InputFile->getBufferStart(), InputFile->getBufferStart(),
106 InputFile->getBufferEnd());
Mike Stump1eb44332009-09-09 15:08:12 +0000107
Chris Lattner0770dab2009-01-17 07:56:59 +0000108 // Default to keeping comments if the preprocessor wants them.
109 SetCommentRetentionState(PP.getCommentRetentionState());
110}
Chris Lattnerdbf388b2007-10-07 08:47:24 +0000111
Chris Lattner168ae2d2007-10-17 20:41:00 +0000112/// Lexer constructor - Create a new raw lexer object. This object is only
Chris Lattner590f0cc2008-10-12 01:15:46 +0000113/// suitable for calls to 'LexRawToken'. This lexer assumes that the text
114/// range will outlive it, so it doesn't take ownership of it.
Chris Lattner168ae2d2007-10-17 20:41:00 +0000115Lexer::Lexer(SourceLocation fileloc, const LangOptions &features,
Chris Lattnerde96c0f2009-01-17 07:42:27 +0000116 const char *BufStart, const char *BufPtr, const char *BufEnd)
Chris Lattnerc6fe32a2009-01-17 03:48:08 +0000117 : FileLoc(fileloc), Features(features) {
Chris Lattner22d91ca2009-01-17 06:55:17 +0000118
Chris Lattner22d91ca2009-01-17 06:55:17 +0000119 InitLexer(BufStart, BufPtr, BufEnd);
Mike Stump1eb44332009-09-09 15:08:12 +0000120
Chris Lattner168ae2d2007-10-17 20:41:00 +0000121 // We *are* in raw mode.
122 LexingRawMode = true;
Chris Lattner168ae2d2007-10-17 20:41:00 +0000123}
124
Chris Lattner025c3a62009-01-17 07:35:14 +0000125/// Lexer constructor - Create a new raw lexer object. This object is only
126/// suitable for calls to 'LexRawToken'. This lexer assumes that the text
127/// range will outlive it, so it doesn't take ownership of it.
Chris Lattner6e290142009-11-30 04:18:44 +0000128Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *FromFile,
129 const SourceManager &SM, const LangOptions &features)
Chris Lattner025c3a62009-01-17 07:35:14 +0000130 : FileLoc(SM.getLocForStartOfFile(FID)), Features(features) {
Chris Lattner025c3a62009-01-17 07:35:14 +0000131
Mike Stump1eb44332009-09-09 15:08:12 +0000132 InitLexer(FromFile->getBufferStart(), FromFile->getBufferStart(),
Chris Lattner025c3a62009-01-17 07:35:14 +0000133 FromFile->getBufferEnd());
Mike Stump1eb44332009-09-09 15:08:12 +0000134
Chris Lattner025c3a62009-01-17 07:35:14 +0000135 // We *are* in raw mode.
136 LexingRawMode = true;
137}
138
Chris Lattner42e00d12009-01-17 08:27:52 +0000139/// Create_PragmaLexer: Lexer constructor - Create a new lexer object for
140/// _Pragma expansion. This has a variety of magic semantics that this method
141/// sets up. It returns a new'd Lexer that must be delete'd when done.
142///
143/// On entrance to this routine, TokStartLoc is a macro location which has a
144/// spelling loc that indicates the bytes to be lexed for the token and an
145/// instantiation location that indicates where all lexed tokens should be
146/// "expanded from".
147///
148/// FIXME: It would really be nice to make _Pragma just be a wrapper around a
149/// normal lexer that remaps tokens as they fly by. This would require making
150/// Preprocessor::Lex virtual. Given that, we could just dump in a magic lexer
151/// interface that could handle this stuff. This would pull GetMappedTokenLoc
152/// out of the critical path of the lexer!
153///
Mike Stump1eb44332009-09-09 15:08:12 +0000154Lexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc,
Chris Lattnere7fb4842009-02-15 20:52:18 +0000155 SourceLocation InstantiationLocStart,
156 SourceLocation InstantiationLocEnd,
Chris Lattnerbcc2a672009-01-19 06:46:35 +0000157 unsigned TokLen, Preprocessor &PP) {
Chris Lattner42e00d12009-01-17 08:27:52 +0000158 SourceManager &SM = PP.getSourceManager();
Chris Lattner42e00d12009-01-17 08:27:52 +0000159
160 // Create the lexer as if we were going to lex the file normally.
Chris Lattnera11d6172009-01-19 07:46:45 +0000161 FileID SpellingFID = SM.getFileID(SpellingLoc);
Chris Lattner6e290142009-11-30 04:18:44 +0000162 const llvm::MemoryBuffer *InputFile = SM.getBuffer(SpellingFID);
163 Lexer *L = new Lexer(SpellingFID, InputFile, PP);
Mike Stump1eb44332009-09-09 15:08:12 +0000164
Chris Lattner42e00d12009-01-17 08:27:52 +0000165 // Now that the lexer is created, change the start/end locations so that we
166 // just lex the subsection of the file that we want. This is lexing from a
167 // scratch buffer.
168 const char *StrData = SM.getCharacterData(SpellingLoc);
Mike Stump1eb44332009-09-09 15:08:12 +0000169
Chris Lattner42e00d12009-01-17 08:27:52 +0000170 L->BufferPtr = StrData;
171 L->BufferEnd = StrData+TokLen;
Chris Lattner1fa49532009-03-08 08:08:45 +0000172 assert(L->BufferEnd[0] == 0 && "Buffer is not nul terminated!");
Chris Lattner42e00d12009-01-17 08:27:52 +0000173
174 // Set the SourceLocation with the remapping information. This ensures that
175 // GetMappedTokenLoc will remap the tokens as they are lexed.
Chris Lattnerde7aeef2009-01-26 00:43:02 +0000176 L->FileLoc = SM.createInstantiationLoc(SM.getLocForStartOfFile(SpellingFID),
Chris Lattnere7fb4842009-02-15 20:52:18 +0000177 InstantiationLocStart,
178 InstantiationLocEnd, TokLen);
Mike Stump1eb44332009-09-09 15:08:12 +0000179
Chris Lattner42e00d12009-01-17 08:27:52 +0000180 // Ensure that the lexer thinks it is inside a directive, so that end \n will
181 // return an EOM token.
182 L->ParsingPreprocessorDirective = true;
Mike Stump1eb44332009-09-09 15:08:12 +0000183
Chris Lattner42e00d12009-01-17 08:27:52 +0000184 // This lexer really is for _Pragma.
185 L->Is_PragmaLexer = true;
186 return L;
187}
188
Chris Lattner168ae2d2007-10-17 20:41:00 +0000189
Reid Spencer5f016e22007-07-11 17:01:13 +0000190/// Stringify - Convert the specified string into a C string, with surrounding
191/// ""'s, and with escaped \ and " characters.
192std::string Lexer::Stringify(const std::string &Str, bool Charify) {
193 std::string Result = Str;
194 char Quote = Charify ? '\'' : '"';
195 for (unsigned i = 0, e = Result.size(); i != e; ++i) {
196 if (Result[i] == '\\' || Result[i] == Quote) {
197 Result.insert(Result.begin()+i, '\\');
198 ++i; ++e;
199 }
200 }
201 return Result;
202}
203
Chris Lattnerd8e30832007-07-24 06:57:14 +0000204/// Stringify - Convert the specified string into a C string by escaping '\'
205/// and " characters. This does not add surrounding ""'s to the string.
206void Lexer::Stringify(llvm::SmallVectorImpl<char> &Str) {
207 for (unsigned i = 0, e = Str.size(); i != e; ++i) {
208 if (Str[i] == '\\' || Str[i] == '"') {
209 Str.insert(Str.begin()+i, '\\');
210 ++i; ++e;
211 }
212 }
213}
214
Chris Lattnerb0607272010-11-17 07:26:20 +0000215//===----------------------------------------------------------------------===//
216// Token Spelling
217//===----------------------------------------------------------------------===//
218
219/// getSpelling() - Return the 'spelling' of this token. The spelling of a
220/// token are the characters used to represent the token in the source file
221/// after trigraph expansion and escaped-newline folding. In particular, this
222/// wants to get the true, uncanonicalized, spelling of things like digraphs
223/// UCNs, etc.
224std::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr,
225 const LangOptions &Features, bool *Invalid) {
226 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
227
228 // If this token contains nothing interesting, return it directly.
229 bool CharDataInvalid = false;
230 const char* TokStart = SourceMgr.getCharacterData(Tok.getLocation(),
231 &CharDataInvalid);
232 if (Invalid)
233 *Invalid = CharDataInvalid;
234 if (CharDataInvalid)
235 return std::string();
236
237 if (!Tok.needsCleaning())
238 return std::string(TokStart, TokStart+Tok.getLength());
239
240 std::string Result;
241 Result.reserve(Tok.getLength());
242
243 // Otherwise, hard case, relex the characters into the string.
244 for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength();
245 Ptr != End; ) {
246 unsigned CharSize;
247 Result.push_back(Lexer::getCharAndSizeNoWarn(Ptr, CharSize, Features));
248 Ptr += CharSize;
249 }
250 assert(Result.size() != unsigned(Tok.getLength()) &&
251 "NeedsCleaning flag set on something that didn't need cleaning!");
252 return Result;
253}
254
255/// getSpelling - This method is used to get the spelling of a token into a
256/// preallocated buffer, instead of as an std::string. The caller is required
257/// to allocate enough space for the token, which is guaranteed to be at least
258/// Tok.getLength() bytes long. The actual length of the token is returned.
259///
260/// Note that this method may do two possible things: it may either fill in
261/// the buffer specified with characters, or it may *change the input pointer*
262/// to point to a constant buffer with the data already in it (avoiding a
263/// copy). The caller is not allowed to modify the returned buffer pointer
264/// if an internal buffer is returned.
265unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer,
266 const SourceManager &SourceMgr,
267 const LangOptions &Features, bool *Invalid) {
268 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +0000269
270 const char *TokStart = 0;
271 // NOTE: this has to be checked *before* testing for an IdentifierInfo.
272 if (Tok.is(tok::raw_identifier))
273 TokStart = Tok.getRawIdentifierData();
274 else if (const IdentifierInfo *II = Tok.getIdentifierInfo()) {
275 // Just return the string from the identifier table, which is very quick.
Chris Lattnerb0607272010-11-17 07:26:20 +0000276 Buffer = II->getNameStart();
277 return II->getLength();
278 }
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +0000279
280 // NOTE: this can be checked even after testing for an IdentifierInfo.
Chris Lattnerb0607272010-11-17 07:26:20 +0000281 if (Tok.isLiteral())
282 TokStart = Tok.getLiteralData();
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +0000283
Chris Lattnerb0607272010-11-17 07:26:20 +0000284 if (TokStart == 0) {
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +0000285 // Compute the start of the token in the input lexer buffer.
Chris Lattnerb0607272010-11-17 07:26:20 +0000286 bool CharDataInvalid = false;
287 TokStart = SourceMgr.getCharacterData(Tok.getLocation(), &CharDataInvalid);
288 if (Invalid)
289 *Invalid = CharDataInvalid;
290 if (CharDataInvalid) {
291 Buffer = "";
292 return 0;
293 }
294 }
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +0000295
Chris Lattnerb0607272010-11-17 07:26:20 +0000296 // If this token contains nothing interesting, return it directly.
297 if (!Tok.needsCleaning()) {
298 Buffer = TokStart;
299 return Tok.getLength();
300 }
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +0000301
Chris Lattnerb0607272010-11-17 07:26:20 +0000302 // Otherwise, hard case, relex the characters into the string.
303 char *OutBuf = const_cast<char*>(Buffer);
304 for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength();
305 Ptr != End; ) {
306 unsigned CharSize;
307 *OutBuf++ = Lexer::getCharAndSizeNoWarn(Ptr, CharSize, Features);
308 Ptr += CharSize;
309 }
310 assert(unsigned(OutBuf-Buffer) != Tok.getLength() &&
311 "NeedsCleaning flag set on something that didn't need cleaning!");
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +0000312
Chris Lattnerb0607272010-11-17 07:26:20 +0000313 return OutBuf-Buffer;
314}
315
316
317
Douglas Gregor33e9abd2010-01-22 19:49:59 +0000318static bool isWhitespace(unsigned char c);
Reid Spencer5f016e22007-07-11 17:01:13 +0000319
Chris Lattner9a611942007-10-17 21:18:47 +0000320/// MeasureTokenLength - Relex the token at the specified location and return
321/// its length in bytes in the input file. If the token needs cleaning (e.g.
322/// includes a trigraph or an escaped newline) then this count includes bytes
323/// that are part of that.
324unsigned Lexer::MeasureTokenLength(SourceLocation Loc,
Chris Lattner2c78b872009-04-14 23:22:57 +0000325 const SourceManager &SM,
326 const LangOptions &LangOpts) {
Chris Lattner9a611942007-10-17 21:18:47 +0000327 // TODO: this could be special cased for common tokens like identifiers, ')',
328 // etc to make this faster, if it mattered. Just look at StrData[0] to handle
Mike Stump1eb44332009-09-09 15:08:12 +0000329 // all obviously single-char tokens. This could use
Chris Lattner9a611942007-10-17 21:18:47 +0000330 // Lexer::isObviouslySimpleCharacter for example to handle identifiers or
331 // something.
Chris Lattnerde7aeef2009-01-26 00:43:02 +0000332
333 // If this comes from a macro expansion, we really do want the macro name, not
334 // the token this macro expanded to.
Chris Lattner363fdc22009-01-26 22:24:27 +0000335 Loc = SM.getInstantiationLoc(Loc);
336 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
Douglas Gregorf715ca12010-03-16 00:06:06 +0000337 bool Invalid = false;
Benjamin Kramerf6ac97b2010-03-16 14:14:31 +0000338 llvm::StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
Douglas Gregorf715ca12010-03-16 00:06:06 +0000339 if (Invalid)
Douglas Gregoraea67db2010-03-15 22:54:52 +0000340 return 0;
Benjamin Kramerf6ac97b2010-03-16 14:14:31 +0000341
342 const char *StrData = Buffer.data()+LocInfo.second;
Chris Lattner83503942009-01-17 08:30:10 +0000343
Douglas Gregor33e9abd2010-01-22 19:49:59 +0000344 if (isWhitespace(StrData[0]))
345 return 0;
346
Chris Lattner9a611942007-10-17 21:18:47 +0000347 // Create a lexer starting at the beginning of this token.
Sebastian Redlc3526d82010-09-30 01:03:03 +0000348 Lexer TheLexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts,
349 Buffer.begin(), StrData, Buffer.end());
Chris Lattner39de7402009-10-14 15:04:18 +0000350 TheLexer.SetCommentRetentionState(true);
Chris Lattner9a611942007-10-17 21:18:47 +0000351 Token TheTok;
Chris Lattner590f0cc2008-10-12 01:15:46 +0000352 TheLexer.LexFromRawLexer(TheTok);
Chris Lattner9a611942007-10-17 21:18:47 +0000353 return TheTok.getLength();
354}
355
Douglas Gregora8e5c5b2010-07-22 20:22:31 +0000356SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc,
357 const SourceManager &SM,
358 const LangOptions &LangOpts) {
359 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
Douglas Gregor3de84242011-01-31 22:42:36 +0000360 if (LocInfo.first.isInvalid())
361 return Loc;
362
Douglas Gregora8e5c5b2010-07-22 20:22:31 +0000363 bool Invalid = false;
364 llvm::StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
365 if (Invalid)
366 return Loc;
367
368 // Back up from the current location until we hit the beginning of a line
369 // (or the buffer). We'll relex from that point.
370 const char *BufStart = Buffer.data();
Douglas Gregor3de84242011-01-31 22:42:36 +0000371 if (LocInfo.second >= Buffer.size())
372 return Loc;
373
Douglas Gregora8e5c5b2010-07-22 20:22:31 +0000374 const char *StrData = BufStart+LocInfo.second;
375 if (StrData[0] == '\n' || StrData[0] == '\r')
376 return Loc;
377
378 const char *LexStart = StrData;
379 while (LexStart != BufStart) {
380 if (LexStart[0] == '\n' || LexStart[0] == '\r') {
381 ++LexStart;
382 break;
383 }
384
385 --LexStart;
386 }
387
388 // Create a lexer starting at the beginning of this token.
389 SourceLocation LexerStartLoc = Loc.getFileLocWithOffset(-LocInfo.second);
390 Lexer TheLexer(LexerStartLoc, LangOpts, BufStart, LexStart, Buffer.end());
391 TheLexer.SetCommentRetentionState(true);
392
393 // Lex tokens until we find the token that contains the source location.
394 Token TheTok;
395 do {
396 TheLexer.LexFromRawLexer(TheTok);
397
398 if (TheLexer.getBufferLocation() > StrData) {
399 // Lexing this token has taken the lexer past the source location we're
400 // looking for. If the current token encompasses our source location,
401 // return the beginning of that token.
402 if (TheLexer.getBufferLocation() - TheTok.getLength() <= StrData)
403 return TheTok.getLocation();
404
405 // We ended up skipping over the source location entirely, which means
406 // that it points into whitespace. We're done here.
407 break;
408 }
409 } while (TheTok.getKind() != tok::eof);
410
411 // We've passed our source location; just return the original source location.
412 return Loc;
413}
414
Douglas Gregorf033f1d2010-07-20 20:18:03 +0000415namespace {
416 enum PreambleDirectiveKind {
417 PDK_Skipped,
418 PDK_StartIf,
419 PDK_EndIf,
420 PDK_Unknown
421 };
422}
423
Douglas Gregorf4f6c9d2010-07-26 21:36:20 +0000424std::pair<unsigned, bool>
Douglas Gregordf95a132010-08-09 20:45:32 +0000425Lexer::ComputePreamble(const llvm::MemoryBuffer *Buffer, unsigned MaxLines) {
Douglas Gregorf033f1d2010-07-20 20:18:03 +0000426 // Create a lexer starting at the beginning of the file. Note that we use a
427 // "fake" file source location at offset 1 so that the lexer will track our
428 // position within the file.
429 const unsigned StartOffset = 1;
430 SourceLocation StartLoc = SourceLocation::getFromRawEncoding(StartOffset);
431 LangOptions LangOpts;
432 Lexer TheLexer(StartLoc, LangOpts, Buffer->getBufferStart(),
433 Buffer->getBufferStart(), Buffer->getBufferEnd());
434
435 bool InPreprocessorDirective = false;
436 Token TheTok;
437 Token IfStartTok;
438 unsigned IfCount = 0;
Douglas Gregordf95a132010-08-09 20:45:32 +0000439 unsigned Line = 0;
440
Douglas Gregorf033f1d2010-07-20 20:18:03 +0000441 do {
442 TheLexer.LexFromRawLexer(TheTok);
443
444 if (InPreprocessorDirective) {
445 // If we've hit the end of the file, we're done.
446 if (TheTok.getKind() == tok::eof) {
447 InPreprocessorDirective = false;
448 break;
449 }
450
451 // If we haven't hit the end of the preprocessor directive, skip this
452 // token.
453 if (!TheTok.isAtStartOfLine())
454 continue;
455
456 // We've passed the end of the preprocessor directive, and will look
457 // at this token again below.
458 InPreprocessorDirective = false;
459 }
460
Douglas Gregordf95a132010-08-09 20:45:32 +0000461 // Keep track of the # of lines in the preamble.
462 if (TheTok.isAtStartOfLine()) {
463 ++Line;
464
465 // If we were asked to limit the number of lines in the preamble,
466 // and we're about to exceed that limit, we're done.
467 if (MaxLines && Line >= MaxLines)
468 break;
469 }
470
Douglas Gregorf033f1d2010-07-20 20:18:03 +0000471 // Comments are okay; skip over them.
472 if (TheTok.getKind() == tok::comment)
473 continue;
474
475 if (TheTok.isAtStartOfLine() && TheTok.getKind() == tok::hash) {
476 // This is the start of a preprocessor directive.
477 Token HashTok = TheTok;
478 InPreprocessorDirective = true;
479
480 // Figure out which direective this is. Since we're lexing raw tokens,
481 // we don't have an identifier table available. Instead, just look at
482 // the raw identifier to recognize and categorize preprocessor directives.
483 TheLexer.LexFromRawLexer(TheTok);
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +0000484 if (TheTok.getKind() == tok::raw_identifier && !TheTok.needsCleaning()) {
485 llvm::StringRef Keyword(TheTok.getRawIdentifierData(),
486 TheTok.getLength());
Douglas Gregorf033f1d2010-07-20 20:18:03 +0000487 PreambleDirectiveKind PDK
488 = llvm::StringSwitch<PreambleDirectiveKind>(Keyword)
489 .Case("include", PDK_Skipped)
490 .Case("__include_macros", PDK_Skipped)
491 .Case("define", PDK_Skipped)
492 .Case("undef", PDK_Skipped)
493 .Case("line", PDK_Skipped)
494 .Case("error", PDK_Skipped)
495 .Case("pragma", PDK_Skipped)
496 .Case("import", PDK_Skipped)
497 .Case("include_next", PDK_Skipped)
498 .Case("warning", PDK_Skipped)
499 .Case("ident", PDK_Skipped)
500 .Case("sccs", PDK_Skipped)
501 .Case("assert", PDK_Skipped)
502 .Case("unassert", PDK_Skipped)
503 .Case("if", PDK_StartIf)
504 .Case("ifdef", PDK_StartIf)
505 .Case("ifndef", PDK_StartIf)
506 .Case("elif", PDK_Skipped)
507 .Case("else", PDK_Skipped)
508 .Case("endif", PDK_EndIf)
509 .Default(PDK_Unknown);
510
511 switch (PDK) {
512 case PDK_Skipped:
513 continue;
514
515 case PDK_StartIf:
516 if (IfCount == 0)
517 IfStartTok = HashTok;
518
519 ++IfCount;
520 continue;
521
522 case PDK_EndIf:
523 // Mismatched #endif. The preamble ends here.
524 if (IfCount == 0)
525 break;
526
527 --IfCount;
528 continue;
529
530 case PDK_Unknown:
531 // We don't know what this directive is; stop at the '#'.
532 break;
533 }
534 }
535
536 // We only end up here if we didn't recognize the preprocessor
537 // directive or it was one that can't occur in the preamble at this
538 // point. Roll back the current token to the location of the '#'.
539 InPreprocessorDirective = false;
540 TheTok = HashTok;
541 }
542
Douglas Gregordf95a132010-08-09 20:45:32 +0000543 // We hit a token that we don't recognize as being in the
544 // "preprocessing only" part of the file, so we're no longer in
545 // the preamble.
Douglas Gregorf033f1d2010-07-20 20:18:03 +0000546 break;
547 } while (true);
548
549 SourceLocation End = IfCount? IfStartTok.getLocation() : TheTok.getLocation();
Douglas Gregorf4f6c9d2010-07-26 21:36:20 +0000550 return std::make_pair(End.getRawEncoding() - StartLoc.getRawEncoding(),
551 IfCount? IfStartTok.isAtStartOfLine()
552 : TheTok.isAtStartOfLine());
Douglas Gregorf033f1d2010-07-20 20:18:03 +0000553}
554
Chris Lattner7ef5c272010-11-17 07:05:50 +0000555
556/// AdvanceToTokenCharacter - Given a location that specifies the start of a
557/// token, return a new location that specifies a character within the token.
558SourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart,
559 unsigned CharNo,
560 const SourceManager &SM,
561 const LangOptions &Features) {
562 // Figure out how many physical characters away the specified instantiation
563 // character is. This needs to take into consideration newlines and
564 // trigraphs.
565 bool Invalid = false;
566 const char *TokPtr = SM.getCharacterData(TokStart, &Invalid);
567
568 // If they request the first char of the token, we're trivially done.
569 if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr)))
570 return TokStart;
571
572 unsigned PhysOffset = 0;
573
574 // The usual case is that tokens don't contain anything interesting. Skip
575 // over the uninteresting characters. If a token only consists of simple
576 // chars, this method is extremely fast.
577 while (Lexer::isObviouslySimpleCharacter(*TokPtr)) {
578 if (CharNo == 0)
579 return TokStart.getFileLocWithOffset(PhysOffset);
580 ++TokPtr, --CharNo, ++PhysOffset;
581 }
582
583 // If we have a character that may be a trigraph or escaped newline, use a
584 // lexer to parse it correctly.
585 for (; CharNo; --CharNo) {
586 unsigned Size;
587 Lexer::getCharAndSizeNoWarn(TokPtr, Size, Features);
588 TokPtr += Size;
589 PhysOffset += Size;
590 }
591
592 // Final detail: if we end up on an escaped newline, we want to return the
593 // location of the actual byte of the token. For example foo\<newline>bar
594 // advanced by 3 should return the location of b, not of \\. One compounding
595 // detail of this is that the escape may be made by a trigraph.
596 if (!Lexer::isObviouslySimpleCharacter(*TokPtr))
597 PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr;
598
599 return TokStart.getFileLocWithOffset(PhysOffset);
600}
601
602/// \brief Computes the source location just past the end of the
603/// token at this source location.
604///
605/// This routine can be used to produce a source location that
606/// points just past the end of the token referenced by \p Loc, and
607/// is generally used when a diagnostic needs to point just after a
608/// token where it expected something different that it received. If
609/// the returned source location would not be meaningful (e.g., if
610/// it points into a macro), this routine returns an invalid
611/// source location.
612///
613/// \param Offset an offset from the end of the token, where the source
614/// location should refer to. The default offset (0) produces a source
615/// location pointing just past the end of the token; an offset of 1 produces
616/// a source location pointing to the last character in the token, etc.
617SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset,
618 const SourceManager &SM,
619 const LangOptions &Features) {
620 if (Loc.isInvalid() || !Loc.isFileID())
621 return SourceLocation();
622
623 unsigned Len = Lexer::MeasureTokenLength(Loc, SM, Features);
624 if (Len > Offset)
625 Len = Len - Offset;
626 else
627 return Loc;
628
629 return AdvanceToTokenCharacter(Loc, Len, SM, Features);
630}
631
Reid Spencer5f016e22007-07-11 17:01:13 +0000632//===----------------------------------------------------------------------===//
633// Character information.
634//===----------------------------------------------------------------------===//
635
Reid Spencer5f016e22007-07-11 17:01:13 +0000636enum {
637 CHAR_HORZ_WS = 0x01, // ' ', '\t', '\f', '\v'. Note, no '\0'
638 CHAR_VERT_WS = 0x02, // '\r', '\n'
639 CHAR_LETTER = 0x04, // a-z,A-Z
640 CHAR_NUMBER = 0x08, // 0-9
641 CHAR_UNDER = 0x10, // _
642 CHAR_PERIOD = 0x20 // .
643};
644
Chris Lattner03b98662009-07-07 17:09:54 +0000645// Statically initialize CharInfo table based on ASCII character set
646// Reference: FreeBSD 7.2 /usr/share/misc/ascii
Chris Lattnera2bf1052009-12-17 05:29:40 +0000647static const unsigned char CharInfo[256] =
Chris Lattner03b98662009-07-07 17:09:54 +0000648{
649// 0 NUL 1 SOH 2 STX 3 ETX
650// 4 EOT 5 ENQ 6 ACK 7 BEL
651 0 , 0 , 0 , 0 ,
652 0 , 0 , 0 , 0 ,
653// 8 BS 9 HT 10 NL 11 VT
654//12 NP 13 CR 14 SO 15 SI
655 0 , CHAR_HORZ_WS, CHAR_VERT_WS, CHAR_HORZ_WS,
656 CHAR_HORZ_WS, CHAR_VERT_WS, 0 , 0 ,
657//16 DLE 17 DC1 18 DC2 19 DC3
658//20 DC4 21 NAK 22 SYN 23 ETB
659 0 , 0 , 0 , 0 ,
660 0 , 0 , 0 , 0 ,
661//24 CAN 25 EM 26 SUB 27 ESC
662//28 FS 29 GS 30 RS 31 US
663 0 , 0 , 0 , 0 ,
664 0 , 0 , 0 , 0 ,
665//32 SP 33 ! 34 " 35 #
666//36 $ 37 % 38 & 39 '
667 CHAR_HORZ_WS, 0 , 0 , 0 ,
668 0 , 0 , 0 , 0 ,
669//40 ( 41 ) 42 * 43 +
670//44 , 45 - 46 . 47 /
671 0 , 0 , 0 , 0 ,
672 0 , 0 , CHAR_PERIOD , 0 ,
673//48 0 49 1 50 2 51 3
674//52 4 53 5 54 6 55 7
675 CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER ,
676 CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER ,
677//56 8 57 9 58 : 59 ;
678//60 < 61 = 62 > 63 ?
679 CHAR_NUMBER , CHAR_NUMBER , 0 , 0 ,
680 0 , 0 , 0 , 0 ,
681//64 @ 65 A 66 B 67 C
682//68 D 69 E 70 F 71 G
683 0 , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
684 CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
685//72 H 73 I 74 J 75 K
686//76 L 77 M 78 N 79 O
687 CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
688 CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
689//80 P 81 Q 82 R 83 S
690//84 T 85 U 86 V 87 W
691 CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
692 CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
693//88 X 89 Y 90 Z 91 [
694//92 \ 93 ] 94 ^ 95 _
695 CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 0 ,
696 0 , 0 , 0 , CHAR_UNDER ,
697//96 ` 97 a 98 b 99 c
698//100 d 101 e 102 f 103 g
699 0 , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
700 CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
701//104 h 105 i 106 j 107 k
702//108 l 109 m 110 n 111 o
703 CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
704 CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
705//112 p 113 q 114 r 115 s
706//116 t 117 u 118 v 119 w
707 CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
708 CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
709//120 x 121 y 122 z 123 {
710//124 | 125 } 126 ~ 127 DEL
711 CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 0 ,
712 0 , 0 , 0 , 0
713};
714
Chris Lattnera2bf1052009-12-17 05:29:40 +0000715static void InitCharacterInfo() {
Reid Spencer5f016e22007-07-11 17:01:13 +0000716 static bool isInited = false;
717 if (isInited) return;
Chris Lattner03b98662009-07-07 17:09:54 +0000718 // check the statically-initialized CharInfo table
719 assert(CHAR_HORZ_WS == CharInfo[(int)' ']);
720 assert(CHAR_HORZ_WS == CharInfo[(int)'\t']);
721 assert(CHAR_HORZ_WS == CharInfo[(int)'\f']);
722 assert(CHAR_HORZ_WS == CharInfo[(int)'\v']);
723 assert(CHAR_VERT_WS == CharInfo[(int)'\n']);
724 assert(CHAR_VERT_WS == CharInfo[(int)'\r']);
725 assert(CHAR_UNDER == CharInfo[(int)'_']);
726 assert(CHAR_PERIOD == CharInfo[(int)'.']);
727 for (unsigned i = 'a'; i <= 'z'; ++i) {
728 assert(CHAR_LETTER == CharInfo[i]);
729 assert(CHAR_LETTER == CharInfo[i+'A'-'a']);
730 }
Reid Spencer5f016e22007-07-11 17:01:13 +0000731 for (unsigned i = '0'; i <= '9'; ++i)
Chris Lattner03b98662009-07-07 17:09:54 +0000732 assert(CHAR_NUMBER == CharInfo[i]);
Steve Naroff7b682652009-12-08 16:38:12 +0000733
Chris Lattner03b98662009-07-07 17:09:54 +0000734 isInited = true;
Reid Spencer5f016e22007-07-11 17:01:13 +0000735}
736
Chris Lattner03b98662009-07-07 17:09:54 +0000737
Reid Spencer5f016e22007-07-11 17:01:13 +0000738/// isIdentifierBody - Return true if this is the body character of an
739/// identifier, which is [a-zA-Z0-9_].
740static inline bool isIdentifierBody(unsigned char c) {
Hartmut Kaiser95c062b2007-10-18 12:47:01 +0000741 return (CharInfo[c] & (CHAR_LETTER|CHAR_NUMBER|CHAR_UNDER)) ? true : false;
Reid Spencer5f016e22007-07-11 17:01:13 +0000742}
743
744/// isHorizontalWhitespace - Return true if this character is horizontal
745/// whitespace: ' ', '\t', '\f', '\v'. Note that this returns false for '\0'.
746static inline bool isHorizontalWhitespace(unsigned char c) {
Hartmut Kaiser95c062b2007-10-18 12:47:01 +0000747 return (CharInfo[c] & CHAR_HORZ_WS) ? true : false;
Reid Spencer5f016e22007-07-11 17:01:13 +0000748}
749
750/// isWhitespace - Return true if this character is horizontal or vertical
751/// whitespace: ' ', '\t', '\f', '\v', '\n', '\r'. Note that this returns false
752/// for '\0'.
753static inline bool isWhitespace(unsigned char c) {
Hartmut Kaiser95c062b2007-10-18 12:47:01 +0000754 return (CharInfo[c] & (CHAR_HORZ_WS|CHAR_VERT_WS)) ? true : false;
Reid Spencer5f016e22007-07-11 17:01:13 +0000755}
756
757/// isNumberBody - Return true if this is the body character of an
758/// preprocessing number, which is [a-zA-Z0-9_.].
759static inline bool isNumberBody(unsigned char c) {
Mike Stump1eb44332009-09-09 15:08:12 +0000760 return (CharInfo[c] & (CHAR_LETTER|CHAR_NUMBER|CHAR_UNDER|CHAR_PERIOD)) ?
Hartmut Kaiser95c062b2007-10-18 12:47:01 +0000761 true : false;
Reid Spencer5f016e22007-07-11 17:01:13 +0000762}
763
764
765//===----------------------------------------------------------------------===//
766// Diagnostics forwarding code.
767//===----------------------------------------------------------------------===//
768
Chris Lattner409a0362007-07-22 18:38:25 +0000769/// GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the
770/// lexer buffer was all instantiated at a single point, perform the mapping.
771/// This is currently only used for _Pragma implementation, so it is the slow
772/// path of the hot getSourceLocation method. Do not allow it to be inlined.
Chandler Carruth14bd9652010-10-23 08:44:57 +0000773static LLVM_ATTRIBUTE_NOINLINE SourceLocation GetMappedTokenLoc(
774 Preprocessor &PP, SourceLocation FileLoc, unsigned CharNo, unsigned TokLen);
Chris Lattner409a0362007-07-22 18:38:25 +0000775static SourceLocation GetMappedTokenLoc(Preprocessor &PP,
776 SourceLocation FileLoc,
Chris Lattnerde7aeef2009-01-26 00:43:02 +0000777 unsigned CharNo, unsigned TokLen) {
Chris Lattnere7fb4842009-02-15 20:52:18 +0000778 assert(FileLoc.isMacroID() && "Must be an instantiation");
Mike Stump1eb44332009-09-09 15:08:12 +0000779
Chris Lattner409a0362007-07-22 18:38:25 +0000780 // Otherwise, we're lexing "mapped tokens". This is used for things like
781 // _Pragma handling. Combine the instantiation location of FileLoc with the
Chris Lattnerdf7c17a2009-01-16 07:00:02 +0000782 // spelling location.
Chris Lattnere7fb4842009-02-15 20:52:18 +0000783 SourceManager &SM = PP.getSourceManager();
Mike Stump1eb44332009-09-09 15:08:12 +0000784
Chris Lattnerf7cf85b2009-01-16 07:36:28 +0000785 // Create a new SLoc which is expanded from Instantiation(FileLoc) but whose
Chris Lattnerdf7c17a2009-01-16 07:00:02 +0000786 // characters come from spelling(FileLoc)+Offset.
Chris Lattnere7fb4842009-02-15 20:52:18 +0000787 SourceLocation SpellingLoc = SM.getSpellingLoc(FileLoc);
Chris Lattnerbcc2a672009-01-19 06:46:35 +0000788 SpellingLoc = SpellingLoc.getFileLocWithOffset(CharNo);
Mike Stump1eb44332009-09-09 15:08:12 +0000789
Chris Lattnere7fb4842009-02-15 20:52:18 +0000790 // Figure out the expansion loc range, which is the range covered by the
791 // original _Pragma(...) sequence.
792 std::pair<SourceLocation,SourceLocation> II =
793 SM.getImmediateInstantiationRange(FileLoc);
Mike Stump1eb44332009-09-09 15:08:12 +0000794
Chris Lattnere7fb4842009-02-15 20:52:18 +0000795 return SM.createInstantiationLoc(SpellingLoc, II.first, II.second, TokLen);
Chris Lattner409a0362007-07-22 18:38:25 +0000796}
797
Reid Spencer5f016e22007-07-11 17:01:13 +0000798/// getSourceLocation - Return a source location identifier for the specified
799/// offset in the current file.
Chris Lattnerde7aeef2009-01-26 00:43:02 +0000800SourceLocation Lexer::getSourceLocation(const char *Loc,
801 unsigned TokLen) const {
Chris Lattner448cec42007-07-22 18:44:36 +0000802 assert(Loc >= BufferStart && Loc <= BufferEnd &&
Reid Spencer5f016e22007-07-11 17:01:13 +0000803 "Location out of range for this buffer!");
Chris Lattner9dc1f532007-07-20 16:37:10 +0000804
805 // In the normal case, we're just lexing from a simple file buffer, return
806 // the file id from FileLoc with the offset specified.
Chris Lattner448cec42007-07-22 18:44:36 +0000807 unsigned CharNo = Loc-BufferStart;
Chris Lattner9dc1f532007-07-20 16:37:10 +0000808 if (FileLoc.isFileID())
Chris Lattnerbcc2a672009-01-19 06:46:35 +0000809 return FileLoc.getFileLocWithOffset(CharNo);
Mike Stump1eb44332009-09-09 15:08:12 +0000810
Chris Lattner2b2453a2009-01-17 06:22:33 +0000811 // Otherwise, this is the _Pragma lexer case, which pretends that all of the
812 // tokens are lexed from where the _Pragma was defined.
Chris Lattner168ae2d2007-10-17 20:41:00 +0000813 assert(PP && "This doesn't work on raw lexers");
Chris Lattnerde7aeef2009-01-26 00:43:02 +0000814 return GetMappedTokenLoc(*PP, FileLoc, CharNo, TokLen);
Reid Spencer5f016e22007-07-11 17:01:13 +0000815}
816
Reid Spencer5f016e22007-07-11 17:01:13 +0000817/// Diag - Forwarding function for diagnostics. This translate a source
818/// position in the current buffer into a SourceLocation object for rendering.
Chris Lattner3cbfe2c2008-11-22 00:59:29 +0000819DiagnosticBuilder Lexer::Diag(const char *Loc, unsigned DiagID) const {
Chris Lattner3692b092008-11-18 07:59:24 +0000820 return PP->Diag(getSourceLocation(Loc), DiagID);
Reid Spencer5f016e22007-07-11 17:01:13 +0000821}
Reid Spencer5f016e22007-07-11 17:01:13 +0000822
823//===----------------------------------------------------------------------===//
824// Trigraph and Escaped Newline Handling Code.
825//===----------------------------------------------------------------------===//
826
827/// GetTrigraphCharForLetter - Given a character that occurs after a ?? pair,
828/// return the decoded trigraph letter it corresponds to, or '\0' if nothing.
829static char GetTrigraphCharForLetter(char Letter) {
830 switch (Letter) {
831 default: return 0;
832 case '=': return '#';
833 case ')': return ']';
834 case '(': return '[';
835 case '!': return '|';
836 case '\'': return '^';
837 case '>': return '}';
838 case '/': return '\\';
839 case '<': return '{';
840 case '-': return '~';
841 }
842}
843
844/// DecodeTrigraphChar - If the specified character is a legal trigraph when
845/// prefixed with ??, emit a trigraph warning. If trigraphs are enabled,
846/// return the result character. Finally, emit a warning about trigraph use
847/// whether trigraphs are enabled or not.
848static char DecodeTrigraphChar(const char *CP, Lexer *L) {
849 char Res = GetTrigraphCharForLetter(*CP);
Chris Lattner3692b092008-11-18 07:59:24 +0000850 if (!Res || !L) return Res;
Mike Stump1eb44332009-09-09 15:08:12 +0000851
Chris Lattner3692b092008-11-18 07:59:24 +0000852 if (!L->getFeatures().Trigraphs) {
Chris Lattner74d15df2008-11-22 02:02:22 +0000853 if (!L->isLexingRawMode())
854 L->Diag(CP-2, diag::trigraph_ignored);
Chris Lattner3692b092008-11-18 07:59:24 +0000855 return 0;
Reid Spencer5f016e22007-07-11 17:01:13 +0000856 }
Mike Stump1eb44332009-09-09 15:08:12 +0000857
Chris Lattner74d15df2008-11-22 02:02:22 +0000858 if (!L->isLexingRawMode())
Benjamin Kramer476d8b82010-08-11 14:47:12 +0000859 L->Diag(CP-2, diag::trigraph_converted) << llvm::StringRef(&Res, 1);
Reid Spencer5f016e22007-07-11 17:01:13 +0000860 return Res;
861}
862
Chris Lattner24f0e482009-04-18 22:05:41 +0000863/// getEscapedNewLineSize - Return the size of the specified escaped newline,
864/// or 0 if it is not an escaped newline. P[-1] is known to be a "\" or a
Mike Stump1eb44332009-09-09 15:08:12 +0000865/// trigraph equivalent on entry to this function.
Chris Lattner24f0e482009-04-18 22:05:41 +0000866unsigned Lexer::getEscapedNewLineSize(const char *Ptr) {
867 unsigned Size = 0;
868 while (isWhitespace(Ptr[Size])) {
869 ++Size;
Mike Stump1eb44332009-09-09 15:08:12 +0000870
Chris Lattner24f0e482009-04-18 22:05:41 +0000871 if (Ptr[Size-1] != '\n' && Ptr[Size-1] != '\r')
872 continue;
873
874 // If this is a \r\n or \n\r, skip the other half.
875 if ((Ptr[Size] == '\r' || Ptr[Size] == '\n') &&
876 Ptr[Size-1] != Ptr[Size])
877 ++Size;
Mike Stump1eb44332009-09-09 15:08:12 +0000878
Chris Lattner24f0e482009-04-18 22:05:41 +0000879 return Size;
Mike Stump1eb44332009-09-09 15:08:12 +0000880 }
881
Chris Lattner24f0e482009-04-18 22:05:41 +0000882 // Not an escaped newline, must be a \t or something else.
883 return 0;
884}
885
Chris Lattner03374952009-04-18 22:27:02 +0000886/// SkipEscapedNewLines - If P points to an escaped newline (or a series of
887/// them), skip over them and return the first non-escaped-newline found,
888/// otherwise return P.
889const char *Lexer::SkipEscapedNewLines(const char *P) {
890 while (1) {
891 const char *AfterEscape;
892 if (*P == '\\') {
893 AfterEscape = P+1;
894 } else if (*P == '?') {
895 // If not a trigraph for escape, bail out.
896 if (P[1] != '?' || P[2] != '/')
897 return P;
898 AfterEscape = P+3;
899 } else {
900 return P;
901 }
Mike Stump1eb44332009-09-09 15:08:12 +0000902
Chris Lattner03374952009-04-18 22:27:02 +0000903 unsigned NewLineSize = Lexer::getEscapedNewLineSize(AfterEscape);
904 if (NewLineSize == 0) return P;
905 P = AfterEscape+NewLineSize;
906 }
907}
908
Chris Lattner24f0e482009-04-18 22:05:41 +0000909
Reid Spencer5f016e22007-07-11 17:01:13 +0000910/// getCharAndSizeSlow - Peek a single 'character' from the specified buffer,
911/// get its size, and return it. This is tricky in several cases:
912/// 1. If currently at the start of a trigraph, we warn about the trigraph,
913/// then either return the trigraph (skipping 3 chars) or the '?',
914/// depending on whether trigraphs are enabled or not.
915/// 2. If this is an escaped newline (potentially with whitespace between
916/// the backslash and newline), implicitly skip the newline and return
917/// the char after it.
918/// 3. If this is a UCN, return it. FIXME: C++ UCN's?
919///
920/// This handles the slow/uncommon case of the getCharAndSize method. Here we
921/// know that we can accumulate into Size, and that we have already incremented
922/// Ptr by Size bytes.
923///
924/// NOTE: When this method is updated, getCharAndSizeSlowNoWarn (below) should
925/// be updated to match.
926///
927char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size,
Chris Lattnerd2177732007-07-20 16:59:19 +0000928 Token *Tok) {
Reid Spencer5f016e22007-07-11 17:01:13 +0000929 // If we have a slash, look for an escaped newline.
930 if (Ptr[0] == '\\') {
931 ++Size;
932 ++Ptr;
933Slash:
934 // Common case, backslash-char where the char is not whitespace.
935 if (!isWhitespace(Ptr[0])) return '\\';
Mike Stump1eb44332009-09-09 15:08:12 +0000936
Chris Lattner5636a3b2009-06-23 05:15:06 +0000937 // See if we have optional whitespace characters between the slash and
938 // newline.
Chris Lattner24f0e482009-04-18 22:05:41 +0000939 if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) {
940 // Remember that this token needs to be cleaned.
941 if (Tok) Tok->setFlag(Token::NeedsCleaning);
Reid Spencer5f016e22007-07-11 17:01:13 +0000942
Chris Lattner24f0e482009-04-18 22:05:41 +0000943 // Warn if there was whitespace between the backslash and newline.
Chris Lattner5636a3b2009-06-23 05:15:06 +0000944 if (Ptr[0] != '\n' && Ptr[0] != '\r' && Tok && !isLexingRawMode())
Chris Lattner24f0e482009-04-18 22:05:41 +0000945 Diag(Ptr, diag::backslash_newline_space);
Mike Stump1eb44332009-09-09 15:08:12 +0000946
Chris Lattner24f0e482009-04-18 22:05:41 +0000947 // Found backslash<whitespace><newline>. Parse the char after it.
948 Size += EscapedNewLineSize;
949 Ptr += EscapedNewLineSize;
950 // Use slow version to accumulate a correct size field.
951 return getCharAndSizeSlow(Ptr, Size, Tok);
952 }
Mike Stump1eb44332009-09-09 15:08:12 +0000953
Reid Spencer5f016e22007-07-11 17:01:13 +0000954 // Otherwise, this is not an escaped newline, just return the slash.
955 return '\\';
956 }
Mike Stump1eb44332009-09-09 15:08:12 +0000957
Reid Spencer5f016e22007-07-11 17:01:13 +0000958 // If this is a trigraph, process it.
959 if (Ptr[0] == '?' && Ptr[1] == '?') {
960 // If this is actually a legal trigraph (not something like "??x"), emit
961 // a trigraph warning. If so, and if trigraphs are enabled, return it.
962 if (char C = DecodeTrigraphChar(Ptr+2, Tok ? this : 0)) {
963 // Remember that this token needs to be cleaned.
Chris Lattnerd2177732007-07-20 16:59:19 +0000964 if (Tok) Tok->setFlag(Token::NeedsCleaning);
Reid Spencer5f016e22007-07-11 17:01:13 +0000965
966 Ptr += 3;
967 Size += 3;
968 if (C == '\\') goto Slash;
969 return C;
970 }
971 }
Mike Stump1eb44332009-09-09 15:08:12 +0000972
Reid Spencer5f016e22007-07-11 17:01:13 +0000973 // If this is neither, return a single character.
974 ++Size;
975 return *Ptr;
976}
977
978
979/// getCharAndSizeSlowNoWarn - Handle the slow/uncommon case of the
980/// getCharAndSizeNoWarn method. Here we know that we can accumulate into Size,
981/// and that we have already incremented Ptr by Size bytes.
982///
983/// NOTE: When this method is updated, getCharAndSizeSlow (above) should
984/// be updated to match.
985char Lexer::getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size,
986 const LangOptions &Features) {
987 // If we have a slash, look for an escaped newline.
988 if (Ptr[0] == '\\') {
989 ++Size;
990 ++Ptr;
991Slash:
992 // Common case, backslash-char where the char is not whitespace.
993 if (!isWhitespace(Ptr[0])) return '\\';
Mike Stump1eb44332009-09-09 15:08:12 +0000994
Reid Spencer5f016e22007-07-11 17:01:13 +0000995 // See if we have optional whitespace characters followed by a newline.
Chris Lattner24f0e482009-04-18 22:05:41 +0000996 if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) {
997 // Found backslash<whitespace><newline>. Parse the char after it.
998 Size += EscapedNewLineSize;
999 Ptr += EscapedNewLineSize;
Mike Stump1eb44332009-09-09 15:08:12 +00001000
Chris Lattner24f0e482009-04-18 22:05:41 +00001001 // Use slow version to accumulate a correct size field.
1002 return getCharAndSizeSlowNoWarn(Ptr, Size, Features);
1003 }
Mike Stump1eb44332009-09-09 15:08:12 +00001004
Reid Spencer5f016e22007-07-11 17:01:13 +00001005 // Otherwise, this is not an escaped newline, just return the slash.
1006 return '\\';
1007 }
Mike Stump1eb44332009-09-09 15:08:12 +00001008
Reid Spencer5f016e22007-07-11 17:01:13 +00001009 // If this is a trigraph, process it.
1010 if (Features.Trigraphs && Ptr[0] == '?' && Ptr[1] == '?') {
1011 // If this is actually a legal trigraph (not something like "??x"), return
1012 // it.
1013 if (char C = GetTrigraphCharForLetter(Ptr[2])) {
1014 Ptr += 3;
1015 Size += 3;
1016 if (C == '\\') goto Slash;
1017 return C;
1018 }
1019 }
Mike Stump1eb44332009-09-09 15:08:12 +00001020
Reid Spencer5f016e22007-07-11 17:01:13 +00001021 // If this is neither, return a single character.
1022 ++Size;
1023 return *Ptr;
1024}
1025
1026//===----------------------------------------------------------------------===//
1027// Helper methods for lexing.
1028//===----------------------------------------------------------------------===//
1029
Douglas Gregorf4f6c9d2010-07-26 21:36:20 +00001030/// \brief Routine that indiscriminately skips bytes in the source file.
1031void Lexer::SkipBytes(unsigned Bytes, bool StartOfLine) {
1032 BufferPtr += Bytes;
1033 if (BufferPtr > BufferEnd)
1034 BufferPtr = BufferEnd;
1035 IsAtStartOfLine = StartOfLine;
1036}
1037
Chris Lattnerd2177732007-07-20 16:59:19 +00001038void Lexer::LexIdentifier(Token &Result, const char *CurPtr) {
Reid Spencer5f016e22007-07-11 17:01:13 +00001039 // Match [_A-Za-z0-9]*, we have already matched [_A-Za-z$]
1040 unsigned Size;
1041 unsigned char C = *CurPtr++;
Chris Lattnercd991db2010-01-11 02:38:50 +00001042 while (isIdentifierBody(C))
Reid Spencer5f016e22007-07-11 17:01:13 +00001043 C = *CurPtr++;
Chris Lattnercd991db2010-01-11 02:38:50 +00001044
Reid Spencer5f016e22007-07-11 17:01:13 +00001045 --CurPtr; // Back up over the skipped character.
1046
1047 // Fast path, no $,\,? in identifier found. '\' might be an escaped newline
1048 // or UCN, and ? might be a trigraph for '\', an escaped newline or UCN.
1049 // FIXME: UCNs.
Chris Lattnercd991db2010-01-11 02:38:50 +00001050 //
1051 // TODO: Could merge these checks into a CharInfo flag to make the comparison
1052 // cheaper
Reid Spencer5f016e22007-07-11 17:01:13 +00001053 if (C != '\\' && C != '?' && (C != '$' || !Features.DollarIdents)) {
1054FinishIdentifier:
1055 const char *IdStart = BufferPtr;
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +00001056 FormTokenWithChars(Result, CurPtr, tok::raw_identifier);
1057 Result.setRawIdentifierData(IdStart);
Mike Stump1eb44332009-09-09 15:08:12 +00001058
Reid Spencer5f016e22007-07-11 17:01:13 +00001059 // If we are in raw mode, return this identifier raw. There is no need to
1060 // look up identifier information or attempt to macro expand it.
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +00001061 if (LexingRawMode)
1062 return;
Mike Stump1eb44332009-09-09 15:08:12 +00001063
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +00001064 // Fill in Result.IdentifierInfo and update the token kind,
1065 // looking up the identifier in the identifier table.
1066 IdentifierInfo *II = PP->LookUpIdentifierInfo(Result);
Mike Stump1eb44332009-09-09 15:08:12 +00001067
Reid Spencer5f016e22007-07-11 17:01:13 +00001068 // Finally, now that we know we have an identifier, pass this off to the
1069 // preprocessor, which may macro expand it or something.
Chris Lattnerd1186fa2009-01-21 07:45:14 +00001070 if (II->isHandleIdentifierCase())
Chris Lattner6a170eb2009-01-21 07:43:11 +00001071 PP->HandleIdentifier(Result);
1072 return;
Reid Spencer5f016e22007-07-11 17:01:13 +00001073 }
Mike Stump1eb44332009-09-09 15:08:12 +00001074
Reid Spencer5f016e22007-07-11 17:01:13 +00001075 // Otherwise, $,\,? in identifier found. Enter slower path.
Mike Stump1eb44332009-09-09 15:08:12 +00001076
Reid Spencer5f016e22007-07-11 17:01:13 +00001077 C = getCharAndSize(CurPtr, Size);
1078 while (1) {
1079 if (C == '$') {
1080 // If we hit a $ and they are not supported in identifiers, we are done.
1081 if (!Features.DollarIdents) goto FinishIdentifier;
Mike Stump1eb44332009-09-09 15:08:12 +00001082
Reid Spencer5f016e22007-07-11 17:01:13 +00001083 // Otherwise, emit a diagnostic and continue.
Chris Lattner74d15df2008-11-22 02:02:22 +00001084 if (!isLexingRawMode())
1085 Diag(CurPtr, diag::ext_dollar_in_identifier);
Reid Spencer5f016e22007-07-11 17:01:13 +00001086 CurPtr = ConsumeChar(CurPtr, Size, Result);
1087 C = getCharAndSize(CurPtr, Size);
1088 continue;
1089 } else if (!isIdentifierBody(C)) { // FIXME: UCNs.
1090 // Found end of identifier.
1091 goto FinishIdentifier;
1092 }
1093
1094 // Otherwise, this character is good, consume it.
1095 CurPtr = ConsumeChar(CurPtr, Size, Result);
1096
1097 C = getCharAndSize(CurPtr, Size);
1098 while (isIdentifierBody(C)) { // FIXME: UCNs.
1099 CurPtr = ConsumeChar(CurPtr, Size, Result);
1100 C = getCharAndSize(CurPtr, Size);
1101 }
1102 }
1103}
1104
Douglas Gregora75ec432010-08-30 14:50:47 +00001105/// isHexaLiteral - Return true if Start points to a hex constant.
Chris Lattner4a551002010-08-30 17:11:14 +00001106/// in microsoft mode (where this is supposed to be several different tokens).
Chris Lattner6ab55eb2010-08-31 16:42:00 +00001107static bool isHexaLiteral(const char *Start, const LangOptions &Features) {
1108 unsigned Size;
1109 char C1 = Lexer::getCharAndSizeNoWarn(Start, Size, Features);
1110 if (C1 != '0')
1111 return false;
1112 char C2 = Lexer::getCharAndSizeNoWarn(Start + Size, Size, Features);
1113 return (C2 == 'x' || C2 == 'X');
Douglas Gregora75ec432010-08-30 14:50:47 +00001114}
Reid Spencer5f016e22007-07-11 17:01:13 +00001115
Nate Begeman5253c7f2008-04-14 02:26:39 +00001116/// LexNumericConstant - Lex the remainder of a integer or floating point
Reid Spencer5f016e22007-07-11 17:01:13 +00001117/// constant. From[-1] is the first character lexed. Return the end of the
1118/// constant.
Chris Lattnerd2177732007-07-20 16:59:19 +00001119void Lexer::LexNumericConstant(Token &Result, const char *CurPtr) {
Reid Spencer5f016e22007-07-11 17:01:13 +00001120 unsigned Size;
1121 char C = getCharAndSize(CurPtr, Size);
1122 char PrevCh = 0;
1123 while (isNumberBody(C)) { // FIXME: UCNs?
1124 CurPtr = ConsumeChar(CurPtr, Size, Result);
1125 PrevCh = C;
1126 C = getCharAndSize(CurPtr, Size);
1127 }
Mike Stump1eb44332009-09-09 15:08:12 +00001128
Reid Spencer5f016e22007-07-11 17:01:13 +00001129 // If we fell out, check for a sign, due to 1e+12. If we have one, continue.
Chris Lattnerb2f4a202010-08-30 17:09:08 +00001130 if ((C == '-' || C == '+') && (PrevCh == 'E' || PrevCh == 'e')) {
1131 // If we are in Microsoft mode, don't continue if the constant is hex.
1132 // For example, MSVC will accept the following as 3 tokens: 0x1234567e+1
Chris Lattner6ab55eb2010-08-31 16:42:00 +00001133 if (!Features.Microsoft || !isHexaLiteral(BufferPtr, Features))
Chris Lattnerb2f4a202010-08-30 17:09:08 +00001134 return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result));
1135 }
Reid Spencer5f016e22007-07-11 17:01:13 +00001136
1137 // If we have a hex FP constant, continue.
Sean Hunt8c723402010-01-10 23:37:56 +00001138 if ((C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p') &&
Chris Lattnerb2f4a202010-08-30 17:09:08 +00001139 !Features.CPlusPlus0x)
Reid Spencer5f016e22007-07-11 17:01:13 +00001140 return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result));
Mike Stump1eb44332009-09-09 15:08:12 +00001141
Reid Spencer5f016e22007-07-11 17:01:13 +00001142 // Update the location of token as well as BufferPtr.
Chris Lattner47246be2009-01-26 19:29:26 +00001143 const char *TokStart = BufferPtr;
Chris Lattner9e6293d2008-10-12 04:51:35 +00001144 FormTokenWithChars(Result, CurPtr, tok::numeric_constant);
Chris Lattner47246be2009-01-26 19:29:26 +00001145 Result.setLiteralData(TokStart);
Reid Spencer5f016e22007-07-11 17:01:13 +00001146}
1147
1148/// LexStringLiteral - Lex the remainder of a string literal, after having lexed
1149/// either " or L".
Chris Lattnerd88dc482008-10-12 04:05:48 +00001150void Lexer::LexStringLiteral(Token &Result, const char *CurPtr, bool Wide) {
Reid Spencer5f016e22007-07-11 17:01:13 +00001151 const char *NulCharacter = 0; // Does this string contain the \0 character?
Mike Stump1eb44332009-09-09 15:08:12 +00001152
Reid Spencer5f016e22007-07-11 17:01:13 +00001153 char C = getAndAdvanceChar(CurPtr, Result);
1154 while (C != '"') {
Chris Lattner571339c2010-05-30 23:27:38 +00001155 // Skip escaped characters. Escaped newlines will already be processed by
1156 // getAndAdvanceChar.
1157 if (C == '\\')
Reid Spencer5f016e22007-07-11 17:01:13 +00001158 C = getAndAdvanceChar(CurPtr, Result);
Douglas Gregor33611e02010-05-30 22:59:50 +00001159
Chris Lattner571339c2010-05-30 23:27:38 +00001160 if (C == '\n' || C == '\r' || // Newline.
Douglas Gregor33611e02010-05-30 22:59:50 +00001161 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file.
Douglas Gregor55817af2010-08-25 17:04:25 +00001162 if (C == 0 && PP && PP->isCodeCompletionFile(FileLoc))
1163 PP->CodeCompleteNaturalLanguage();
1164 else if (!isLexingRawMode() && !Features.AsmPreprocessor)
Chris Lattner74d15df2008-11-22 02:02:22 +00001165 Diag(BufferPtr, diag::err_unterminated_string);
Chris Lattner9e6293d2008-10-12 04:51:35 +00001166 FormTokenWithChars(Result, CurPtr-1, tok::unknown);
Reid Spencer5f016e22007-07-11 17:01:13 +00001167 return;
Reid Spencer5f016e22007-07-11 17:01:13 +00001168 }
Chris Lattner571339c2010-05-30 23:27:38 +00001169
1170 if (C == 0)
1171 NulCharacter = CurPtr-1;
Reid Spencer5f016e22007-07-11 17:01:13 +00001172 C = getAndAdvanceChar(CurPtr, Result);
1173 }
Mike Stump1eb44332009-09-09 15:08:12 +00001174
Reid Spencer5f016e22007-07-11 17:01:13 +00001175 // If a nul character existed in the string, warn about it.
Chris Lattner74d15df2008-11-22 02:02:22 +00001176 if (NulCharacter && !isLexingRawMode())
1177 Diag(NulCharacter, diag::null_in_string);
Reid Spencer5f016e22007-07-11 17:01:13 +00001178
Reid Spencer5f016e22007-07-11 17:01:13 +00001179 // Update the location of the token as well as the BufferPtr instance var.
Chris Lattner47246be2009-01-26 19:29:26 +00001180 const char *TokStart = BufferPtr;
Sean Hunt6cf75022010-08-30 17:47:05 +00001181 FormTokenWithChars(Result, CurPtr,
1182 Wide ? tok::wide_string_literal : tok::string_literal);
Chris Lattner47246be2009-01-26 19:29:26 +00001183 Result.setLiteralData(TokStart);
Reid Spencer5f016e22007-07-11 17:01:13 +00001184}
1185
1186/// LexAngledStringLiteral - Lex the remainder of an angled string literal,
1187/// after having lexed the '<' character. This is used for #include filenames.
Chris Lattnerd2177732007-07-20 16:59:19 +00001188void Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) {
Reid Spencer5f016e22007-07-11 17:01:13 +00001189 const char *NulCharacter = 0; // Does this string contain the \0 character?
Chris Lattner9cb51ce2009-04-17 23:56:52 +00001190 const char *AfterLessPos = CurPtr;
Reid Spencer5f016e22007-07-11 17:01:13 +00001191 char C = getAndAdvanceChar(CurPtr, Result);
1192 while (C != '>') {
1193 // Skip escaped characters.
1194 if (C == '\\') {
1195 // Skip the escaped character.
1196 C = getAndAdvanceChar(CurPtr, Result);
1197 } else if (C == '\n' || C == '\r' || // Newline.
1198 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file.
Chris Lattner9cb51ce2009-04-17 23:56:52 +00001199 // If the filename is unterminated, then it must just be a lone <
1200 // character. Return this as such.
1201 FormTokenWithChars(Result, AfterLessPos, tok::less);
Reid Spencer5f016e22007-07-11 17:01:13 +00001202 return;
1203 } else if (C == 0) {
1204 NulCharacter = CurPtr-1;
1205 }
1206 C = getAndAdvanceChar(CurPtr, Result);
1207 }
Mike Stump1eb44332009-09-09 15:08:12 +00001208
Reid Spencer5f016e22007-07-11 17:01:13 +00001209 // If a nul character existed in the string, warn about it.
Chris Lattner74d15df2008-11-22 02:02:22 +00001210 if (NulCharacter && !isLexingRawMode())
1211 Diag(NulCharacter, diag::null_in_string);
Mike Stump1eb44332009-09-09 15:08:12 +00001212
Reid Spencer5f016e22007-07-11 17:01:13 +00001213 // Update the location of token as well as BufferPtr.
Chris Lattner47246be2009-01-26 19:29:26 +00001214 const char *TokStart = BufferPtr;
Chris Lattner9e6293d2008-10-12 04:51:35 +00001215 FormTokenWithChars(Result, CurPtr, tok::angle_string_literal);
Chris Lattner47246be2009-01-26 19:29:26 +00001216 Result.setLiteralData(TokStart);
Reid Spencer5f016e22007-07-11 17:01:13 +00001217}
1218
1219
1220/// LexCharConstant - Lex the remainder of a character constant, after having
1221/// lexed either ' or L'.
Chris Lattnerd2177732007-07-20 16:59:19 +00001222void Lexer::LexCharConstant(Token &Result, const char *CurPtr) {
Reid Spencer5f016e22007-07-11 17:01:13 +00001223 const char *NulCharacter = 0; // Does this character contain the \0 character?
1224
Reid Spencer5f016e22007-07-11 17:01:13 +00001225 char C = getAndAdvanceChar(CurPtr, Result);
1226 if (C == '\'') {
Chris Lattner33ab3f62009-03-18 21:10:12 +00001227 if (!isLexingRawMode() && !Features.AsmPreprocessor)
Chris Lattner74d15df2008-11-22 02:02:22 +00001228 Diag(BufferPtr, diag::err_empty_character);
Chris Lattner9e6293d2008-10-12 04:51:35 +00001229 FormTokenWithChars(Result, CurPtr, tok::unknown);
Reid Spencer5f016e22007-07-11 17:01:13 +00001230 return;
Chris Lattnerd80f7862010-07-07 23:24:27 +00001231 }
1232
1233 while (C != '\'') {
1234 // Skip escaped characters.
1235 if (C == '\\') {
1236 // Skip the escaped character.
1237 // FIXME: UCN's
1238 C = getAndAdvanceChar(CurPtr, Result);
1239 } else if (C == '\n' || C == '\r' || // Newline.
1240 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file.
Douglas Gregor55817af2010-08-25 17:04:25 +00001241 if (C == 0 && PP && PP->isCodeCompletionFile(FileLoc))
1242 PP->CodeCompleteNaturalLanguage();
1243 else if (!isLexingRawMode() && !Features.AsmPreprocessor)
Chris Lattnerd80f7862010-07-07 23:24:27 +00001244 Diag(BufferPtr, diag::err_unterminated_char);
1245 FormTokenWithChars(Result, CurPtr-1, tok::unknown);
1246 return;
1247 } else if (C == 0) {
1248 NulCharacter = CurPtr-1;
1249 }
Reid Spencer5f016e22007-07-11 17:01:13 +00001250 C = getAndAdvanceChar(CurPtr, Result);
1251 }
Mike Stump1eb44332009-09-09 15:08:12 +00001252
Chris Lattnerd80f7862010-07-07 23:24:27 +00001253 // If a nul character existed in the character, warn about it.
Chris Lattner74d15df2008-11-22 02:02:22 +00001254 if (NulCharacter && !isLexingRawMode())
1255 Diag(NulCharacter, diag::null_in_char);
Reid Spencer5f016e22007-07-11 17:01:13 +00001256
Reid Spencer5f016e22007-07-11 17:01:13 +00001257 // Update the location of token as well as BufferPtr.
Chris Lattner47246be2009-01-26 19:29:26 +00001258 const char *TokStart = BufferPtr;
Chris Lattner9e6293d2008-10-12 04:51:35 +00001259 FormTokenWithChars(Result, CurPtr, tok::char_constant);
Chris Lattner47246be2009-01-26 19:29:26 +00001260 Result.setLiteralData(TokStart);
Reid Spencer5f016e22007-07-11 17:01:13 +00001261}
1262
1263/// SkipWhitespace - Efficiently skip over a series of whitespace characters.
1264/// Update BufferPtr to point to the next non-whitespace character and return.
Chris Lattnerd88dc482008-10-12 04:05:48 +00001265///
1266/// This method forms a token and returns true if KeepWhitespaceMode is enabled.
1267///
1268bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr) {
Reid Spencer5f016e22007-07-11 17:01:13 +00001269 // Whitespace - Skip it, then return the token after the whitespace.
1270 unsigned char Char = *CurPtr; // Skip consequtive spaces efficiently.
1271 while (1) {
1272 // Skip horizontal whitespace very aggressively.
1273 while (isHorizontalWhitespace(Char))
1274 Char = *++CurPtr;
Mike Stump1eb44332009-09-09 15:08:12 +00001275
Daniel Dunbarddd3e8b2008-11-25 00:20:22 +00001276 // Otherwise if we have something other than whitespace, we're done.
Reid Spencer5f016e22007-07-11 17:01:13 +00001277 if (Char != '\n' && Char != '\r')
1278 break;
Mike Stump1eb44332009-09-09 15:08:12 +00001279
Reid Spencer5f016e22007-07-11 17:01:13 +00001280 if (ParsingPreprocessorDirective) {
1281 // End of preprocessor directive line, let LexTokenInternal handle this.
1282 BufferPtr = CurPtr;
Chris Lattnerd88dc482008-10-12 04:05:48 +00001283 return false;
Reid Spencer5f016e22007-07-11 17:01:13 +00001284 }
Mike Stump1eb44332009-09-09 15:08:12 +00001285
Reid Spencer5f016e22007-07-11 17:01:13 +00001286 // ok, but handle newline.
1287 // The returned token is at the start of the line.
Chris Lattnerd2177732007-07-20 16:59:19 +00001288 Result.setFlag(Token::StartOfLine);
Reid Spencer5f016e22007-07-11 17:01:13 +00001289 // No leading whitespace seen so far.
Chris Lattnerd2177732007-07-20 16:59:19 +00001290 Result.clearFlag(Token::LeadingSpace);
Reid Spencer5f016e22007-07-11 17:01:13 +00001291 Char = *++CurPtr;
1292 }
1293
1294 // If this isn't immediately after a newline, there is leading space.
1295 char PrevChar = CurPtr[-1];
1296 if (PrevChar != '\n' && PrevChar != '\r')
Chris Lattnerd2177732007-07-20 16:59:19 +00001297 Result.setFlag(Token::LeadingSpace);
Reid Spencer5f016e22007-07-11 17:01:13 +00001298
Chris Lattnerd88dc482008-10-12 04:05:48 +00001299 // If the client wants us to return whitespace, return it now.
1300 if (isKeepWhitespaceMode()) {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001301 FormTokenWithChars(Result, CurPtr, tok::unknown);
Chris Lattnerd88dc482008-10-12 04:05:48 +00001302 return true;
1303 }
Mike Stump1eb44332009-09-09 15:08:12 +00001304
Reid Spencer5f016e22007-07-11 17:01:13 +00001305 BufferPtr = CurPtr;
Chris Lattnerd88dc482008-10-12 04:05:48 +00001306 return false;
Reid Spencer5f016e22007-07-11 17:01:13 +00001307}
1308
1309// SkipBCPLComment - We have just read the // characters from input. Skip until
1310// we find the newline character thats terminate the comment. Then update
Chris Lattner046c2272010-01-18 22:35:47 +00001311/// BufferPtr and return.
1312///
1313/// If we're in KeepCommentMode or any CommentHandler has inserted
1314/// some tokens, this will store the first token and return true.
Chris Lattnerd2177732007-07-20 16:59:19 +00001315bool Lexer::SkipBCPLComment(Token &Result, const char *CurPtr) {
Reid Spencer5f016e22007-07-11 17:01:13 +00001316 // If BCPL comments aren't explicitly enabled for this language, emit an
1317 // extension warning.
Chris Lattner74d15df2008-11-22 02:02:22 +00001318 if (!Features.BCPLComment && !isLexingRawMode()) {
Reid Spencer5f016e22007-07-11 17:01:13 +00001319 Diag(BufferPtr, diag::ext_bcpl_comment);
Mike Stump1eb44332009-09-09 15:08:12 +00001320
Reid Spencer5f016e22007-07-11 17:01:13 +00001321 // Mark them enabled so we only emit one warning for this translation
1322 // unit.
1323 Features.BCPLComment = true;
1324 }
Mike Stump1eb44332009-09-09 15:08:12 +00001325
Reid Spencer5f016e22007-07-11 17:01:13 +00001326 // Scan over the body of the comment. The common case, when scanning, is that
1327 // the comment contains normal ascii characters with nothing interesting in
1328 // them. As such, optimize for this case with the inner loop.
1329 char C;
1330 do {
1331 C = *CurPtr;
1332 // FIXME: Speedup BCPL comment lexing. Just scan for a \n or \r character.
1333 // If we find a \n character, scan backwards, checking to see if it's an
1334 // escaped newline, like we do for block comments.
Mike Stump1eb44332009-09-09 15:08:12 +00001335
Reid Spencer5f016e22007-07-11 17:01:13 +00001336 // Skip over characters in the fast loop.
1337 while (C != 0 && // Potentially EOF.
1338 C != '\\' && // Potentially escaped newline.
1339 C != '?' && // Potentially trigraph.
1340 C != '\n' && C != '\r') // Newline or DOS-style newline.
1341 C = *++CurPtr;
1342
1343 // If this is a newline, we're done.
1344 if (C == '\n' || C == '\r')
1345 break; // Found the newline? Break out!
Mike Stump1eb44332009-09-09 15:08:12 +00001346
Reid Spencer5f016e22007-07-11 17:01:13 +00001347 // Otherwise, this is a hard case. Fall back on getAndAdvanceChar to
Chris Lattnerbc3e9842008-12-12 07:34:39 +00001348 // properly decode the character. Read it in raw mode to avoid emitting
1349 // diagnostics about things like trigraphs. If we see an escaped newline,
1350 // we'll handle it below.
Reid Spencer5f016e22007-07-11 17:01:13 +00001351 const char *OldPtr = CurPtr;
Chris Lattnerbc3e9842008-12-12 07:34:39 +00001352 bool OldRawMode = isLexingRawMode();
1353 LexingRawMode = true;
Reid Spencer5f016e22007-07-11 17:01:13 +00001354 C = getAndAdvanceChar(CurPtr, Result);
Chris Lattnerbc3e9842008-12-12 07:34:39 +00001355 LexingRawMode = OldRawMode;
Chris Lattneread616c2009-04-05 00:26:41 +00001356
1357 // If the char that we finally got was a \n, then we must have had something
1358 // like \<newline><newline>. We don't want to have consumed the second
1359 // newline, we want CurPtr, to end up pointing to it down below.
1360 if (C == '\n' || C == '\r') {
1361 --CurPtr;
1362 C = 'x'; // doesn't matter what this is.
1363 }
Mike Stump1eb44332009-09-09 15:08:12 +00001364
Reid Spencer5f016e22007-07-11 17:01:13 +00001365 // If we read multiple characters, and one of those characters was a \r or
1366 // \n, then we had an escaped newline within the comment. Emit diagnostic
1367 // unless the next line is also a // comment.
1368 if (CurPtr != OldPtr+1 && C != '/' && CurPtr[0] != '/') {
1369 for (; OldPtr != CurPtr; ++OldPtr)
1370 if (OldPtr[0] == '\n' || OldPtr[0] == '\r') {
1371 // Okay, we found a // comment that ends in a newline, if the next
1372 // line is also a // comment, but has spaces, don't emit a diagnostic.
1373 if (isspace(C)) {
1374 const char *ForwardPtr = CurPtr;
1375 while (isspace(*ForwardPtr)) // Skip whitespace.
1376 ++ForwardPtr;
1377 if (ForwardPtr[0] == '/' && ForwardPtr[1] == '/')
1378 break;
1379 }
Mike Stump1eb44332009-09-09 15:08:12 +00001380
Chris Lattner74d15df2008-11-22 02:02:22 +00001381 if (!isLexingRawMode())
1382 Diag(OldPtr-1, diag::ext_multi_line_bcpl_comment);
Reid Spencer5f016e22007-07-11 17:01:13 +00001383 break;
1384 }
1385 }
Mike Stump1eb44332009-09-09 15:08:12 +00001386
Douglas Gregor55817af2010-08-25 17:04:25 +00001387 if (CurPtr == BufferEnd+1) {
1388 if (PP && PP->isCodeCompletionFile(FileLoc))
1389 PP->CodeCompleteNaturalLanguage();
1390
1391 --CurPtr;
1392 break;
1393 }
Reid Spencer5f016e22007-07-11 17:01:13 +00001394 } while (C != '\n' && C != '\r');
1395
Chris Lattner3d0ad582010-02-03 21:06:21 +00001396 // Found but did not consume the newline. Notify comment handlers about the
1397 // comment unless we're in a #if 0 block.
1398 if (PP && !isLexingRawMode() &&
1399 PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr),
1400 getSourceLocation(CurPtr)))) {
Chris Lattner046c2272010-01-18 22:35:47 +00001401 BufferPtr = CurPtr;
1402 return true; // A token has to be returned.
1403 }
Mike Stump1eb44332009-09-09 15:08:12 +00001404
Reid Spencer5f016e22007-07-11 17:01:13 +00001405 // If we are returning comments as tokens, return this comment as a token.
Chris Lattnerfa95a012008-10-12 03:22:02 +00001406 if (inKeepCommentMode())
Reid Spencer5f016e22007-07-11 17:01:13 +00001407 return SaveBCPLComment(Result, CurPtr);
1408
1409 // If we are inside a preprocessor directive and we see the end of line,
1410 // return immediately, so that the lexer can return this as an EOM token.
1411 if (ParsingPreprocessorDirective || CurPtr == BufferEnd) {
1412 BufferPtr = CurPtr;
Chris Lattner2d381892008-10-12 04:15:42 +00001413 return false;
Reid Spencer5f016e22007-07-11 17:01:13 +00001414 }
Mike Stump1eb44332009-09-09 15:08:12 +00001415
Reid Spencer5f016e22007-07-11 17:01:13 +00001416 // Otherwise, eat the \n character. We don't care if this is a \n\r or
Chris Lattner7a4f0042008-10-12 00:23:07 +00001417 // \r\n sequence. This is an efficiency hack (because we know the \n can't
Chris Lattnerd88dc482008-10-12 04:05:48 +00001418 // contribute to another token), it isn't needed for correctness. Note that
1419 // this is ok even in KeepWhitespaceMode, because we would have returned the
1420 /// comment above in that mode.
Reid Spencer5f016e22007-07-11 17:01:13 +00001421 ++CurPtr;
Mike Stump1eb44332009-09-09 15:08:12 +00001422
Reid Spencer5f016e22007-07-11 17:01:13 +00001423 // The next returned token is at the start of the line.
Chris Lattnerd2177732007-07-20 16:59:19 +00001424 Result.setFlag(Token::StartOfLine);
Reid Spencer5f016e22007-07-11 17:01:13 +00001425 // No leading whitespace seen so far.
Chris Lattnerd2177732007-07-20 16:59:19 +00001426 Result.clearFlag(Token::LeadingSpace);
Reid Spencer5f016e22007-07-11 17:01:13 +00001427 BufferPtr = CurPtr;
Chris Lattner2d381892008-10-12 04:15:42 +00001428 return false;
Reid Spencer5f016e22007-07-11 17:01:13 +00001429}
1430
1431/// SaveBCPLComment - If in save-comment mode, package up this BCPL comment in
1432/// an appropriate way and return it.
Chris Lattnerd2177732007-07-20 16:59:19 +00001433bool Lexer::SaveBCPLComment(Token &Result, const char *CurPtr) {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001434 // If we're not in a preprocessor directive, just return the // comment
1435 // directly.
1436 FormTokenWithChars(Result, CurPtr, tok::comment);
Mike Stump1eb44332009-09-09 15:08:12 +00001437
Chris Lattner9e6293d2008-10-12 04:51:35 +00001438 if (!ParsingPreprocessorDirective)
1439 return true;
Mike Stump1eb44332009-09-09 15:08:12 +00001440
Chris Lattner9e6293d2008-10-12 04:51:35 +00001441 // If this BCPL-style comment is in a macro definition, transmogrify it into
1442 // a C-style block comment.
Douglas Gregor453091c2010-03-16 22:30:13 +00001443 bool Invalid = false;
1444 std::string Spelling = PP->getSpelling(Result, &Invalid);
1445 if (Invalid)
1446 return true;
1447
Chris Lattner9e6293d2008-10-12 04:51:35 +00001448 assert(Spelling[0] == '/' && Spelling[1] == '/' && "Not bcpl comment?");
1449 Spelling[1] = '*'; // Change prefix to "/*".
1450 Spelling += "*/"; // add suffix.
Mike Stump1eb44332009-09-09 15:08:12 +00001451
Chris Lattner9e6293d2008-10-12 04:51:35 +00001452 Result.setKind(tok::comment);
Chris Lattner47246be2009-01-26 19:29:26 +00001453 PP->CreateString(&Spelling[0], Spelling.size(), Result,
1454 Result.getLocation());
Chris Lattner2d381892008-10-12 04:15:42 +00001455 return true;
Reid Spencer5f016e22007-07-11 17:01:13 +00001456}
1457
1458/// isBlockCommentEndOfEscapedNewLine - Return true if the specified newline
1459/// character (either \n or \r) is part of an escaped newline sequence. Issue a
Chris Lattner47a2b402008-12-12 07:14:34 +00001460/// diagnostic if so. We know that the newline is inside of a block comment.
Mike Stump1eb44332009-09-09 15:08:12 +00001461static bool isEndOfBlockCommentWithEscapedNewLine(const char *CurPtr,
Reid Spencer5f016e22007-07-11 17:01:13 +00001462 Lexer *L) {
1463 assert(CurPtr[0] == '\n' || CurPtr[0] == '\r');
Mike Stump1eb44332009-09-09 15:08:12 +00001464
Reid Spencer5f016e22007-07-11 17:01:13 +00001465 // Back up off the newline.
1466 --CurPtr;
Mike Stump1eb44332009-09-09 15:08:12 +00001467
Reid Spencer5f016e22007-07-11 17:01:13 +00001468 // If this is a two-character newline sequence, skip the other character.
1469 if (CurPtr[0] == '\n' || CurPtr[0] == '\r') {
1470 // \n\n or \r\r -> not escaped newline.
1471 if (CurPtr[0] == CurPtr[1])
1472 return false;
1473 // \n\r or \r\n -> skip the newline.
1474 --CurPtr;
1475 }
Mike Stump1eb44332009-09-09 15:08:12 +00001476
Reid Spencer5f016e22007-07-11 17:01:13 +00001477 // If we have horizontal whitespace, skip over it. We allow whitespace
1478 // between the slash and newline.
1479 bool HasSpace = false;
1480 while (isHorizontalWhitespace(*CurPtr) || *CurPtr == 0) {
1481 --CurPtr;
1482 HasSpace = true;
1483 }
Mike Stump1eb44332009-09-09 15:08:12 +00001484
Reid Spencer5f016e22007-07-11 17:01:13 +00001485 // If we have a slash, we know this is an escaped newline.
1486 if (*CurPtr == '\\') {
1487 if (CurPtr[-1] != '*') return false;
1488 } else {
1489 // It isn't a slash, is it the ?? / trigraph?
1490 if (CurPtr[0] != '/' || CurPtr[-1] != '?' || CurPtr[-2] != '?' ||
1491 CurPtr[-3] != '*')
1492 return false;
Mike Stump1eb44332009-09-09 15:08:12 +00001493
Reid Spencer5f016e22007-07-11 17:01:13 +00001494 // This is the trigraph ending the comment. Emit a stern warning!
1495 CurPtr -= 2;
1496
1497 // If no trigraphs are enabled, warn that we ignored this trigraph and
1498 // ignore this * character.
1499 if (!L->getFeatures().Trigraphs) {
Chris Lattner74d15df2008-11-22 02:02:22 +00001500 if (!L->isLexingRawMode())
1501 L->Diag(CurPtr, diag::trigraph_ignored_block_comment);
Reid Spencer5f016e22007-07-11 17:01:13 +00001502 return false;
1503 }
Chris Lattner74d15df2008-11-22 02:02:22 +00001504 if (!L->isLexingRawMode())
1505 L->Diag(CurPtr, diag::trigraph_ends_block_comment);
Reid Spencer5f016e22007-07-11 17:01:13 +00001506 }
Mike Stump1eb44332009-09-09 15:08:12 +00001507
Reid Spencer5f016e22007-07-11 17:01:13 +00001508 // Warn about having an escaped newline between the */ characters.
Chris Lattner74d15df2008-11-22 02:02:22 +00001509 if (!L->isLexingRawMode())
1510 L->Diag(CurPtr, diag::escaped_newline_block_comment_end);
Mike Stump1eb44332009-09-09 15:08:12 +00001511
Reid Spencer5f016e22007-07-11 17:01:13 +00001512 // If there was space between the backslash and newline, warn about it.
Chris Lattner74d15df2008-11-22 02:02:22 +00001513 if (HasSpace && !L->isLexingRawMode())
1514 L->Diag(CurPtr, diag::backslash_newline_space);
Mike Stump1eb44332009-09-09 15:08:12 +00001515
Reid Spencer5f016e22007-07-11 17:01:13 +00001516 return true;
1517}
1518
1519#ifdef __SSE2__
1520#include <emmintrin.h>
1521#elif __ALTIVEC__
1522#include <altivec.h>
1523#undef bool
1524#endif
1525
1526/// SkipBlockComment - We have just read the /* characters from input. Read
1527/// until we find the */ characters that terminate the comment. Note that we
1528/// don't bother decoding trigraphs or escaped newlines in block comments,
1529/// because they cannot cause the comment to end. The only thing that can
1530/// happen is the comment could end with an escaped newline between the */ end
1531/// of comment.
Chris Lattner2d381892008-10-12 04:15:42 +00001532///
Chris Lattner046c2272010-01-18 22:35:47 +00001533/// If we're in KeepCommentMode or any CommentHandler has inserted
1534/// some tokens, this will store the first token and return true.
Chris Lattnerd2177732007-07-20 16:59:19 +00001535bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) {
Reid Spencer5f016e22007-07-11 17:01:13 +00001536 // Scan one character past where we should, looking for a '/' character. Once
1537 // we find it, check to see if it was preceeded by a *. This common
1538 // optimization helps people who like to put a lot of * characters in their
1539 // comments.
Chris Lattner8146b682007-07-21 23:43:37 +00001540
1541 // The first character we get with newlines and trigraphs skipped to handle
1542 // the degenerate /*/ case below correctly if the * has an escaped newline
1543 // after it.
1544 unsigned CharSize;
1545 unsigned char C = getCharAndSize(CurPtr, CharSize);
1546 CurPtr += CharSize;
Reid Spencer5f016e22007-07-11 17:01:13 +00001547 if (C == 0 && CurPtr == BufferEnd+1) {
Chris Lattner150fcd52010-05-16 19:54:05 +00001548 if (!isLexingRawMode() &&
1549 !PP->isCodeCompletionFile(FileLoc))
Chris Lattner0af57422008-10-12 01:31:51 +00001550 Diag(BufferPtr, diag::err_unterminated_block_comment);
Chris Lattner31f0eca2008-10-12 04:19:49 +00001551 --CurPtr;
Mike Stump1eb44332009-09-09 15:08:12 +00001552
Chris Lattner31f0eca2008-10-12 04:19:49 +00001553 // KeepWhitespaceMode should return this broken comment as a token. Since
1554 // it isn't a well formed comment, just return it as an 'unknown' token.
1555 if (isKeepWhitespaceMode()) {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001556 FormTokenWithChars(Result, CurPtr, tok::unknown);
Chris Lattner31f0eca2008-10-12 04:19:49 +00001557 return true;
1558 }
Mike Stump1eb44332009-09-09 15:08:12 +00001559
Chris Lattner31f0eca2008-10-12 04:19:49 +00001560 BufferPtr = CurPtr;
Chris Lattner2d381892008-10-12 04:15:42 +00001561 return false;
Reid Spencer5f016e22007-07-11 17:01:13 +00001562 }
Mike Stump1eb44332009-09-09 15:08:12 +00001563
Chris Lattner8146b682007-07-21 23:43:37 +00001564 // Check to see if the first character after the '/*' is another /. If so,
1565 // then this slash does not end the block comment, it is part of it.
1566 if (C == '/')
1567 C = *CurPtr++;
Mike Stump1eb44332009-09-09 15:08:12 +00001568
Reid Spencer5f016e22007-07-11 17:01:13 +00001569 while (1) {
1570 // Skip over all non-interesting characters until we find end of buffer or a
1571 // (probably ending) '/' character.
1572 if (CurPtr + 24 < BufferEnd) {
1573 // While not aligned to a 16-byte boundary.
1574 while (C != '/' && ((intptr_t)CurPtr & 0x0F) != 0)
1575 C = *CurPtr++;
Mike Stump1eb44332009-09-09 15:08:12 +00001576
Reid Spencer5f016e22007-07-11 17:01:13 +00001577 if (C == '/') goto FoundSlash;
1578
1579#ifdef __SSE2__
1580 __m128i Slashes = _mm_set_epi8('/', '/', '/', '/', '/', '/', '/', '/',
1581 '/', '/', '/', '/', '/', '/', '/', '/');
1582 while (CurPtr+16 <= BufferEnd &&
1583 _mm_movemask_epi8(_mm_cmpeq_epi8(*(__m128i*)CurPtr, Slashes)) == 0)
1584 CurPtr += 16;
1585#elif __ALTIVEC__
1586 __vector unsigned char Slashes = {
Mike Stump1eb44332009-09-09 15:08:12 +00001587 '/', '/', '/', '/', '/', '/', '/', '/',
Reid Spencer5f016e22007-07-11 17:01:13 +00001588 '/', '/', '/', '/', '/', '/', '/', '/'
1589 };
1590 while (CurPtr+16 <= BufferEnd &&
1591 !vec_any_eq(*(vector unsigned char*)CurPtr, Slashes))
1592 CurPtr += 16;
Mike Stump1eb44332009-09-09 15:08:12 +00001593#else
Reid Spencer5f016e22007-07-11 17:01:13 +00001594 // Scan for '/' quickly. Many block comments are very large.
1595 while (CurPtr[0] != '/' &&
1596 CurPtr[1] != '/' &&
1597 CurPtr[2] != '/' &&
1598 CurPtr[3] != '/' &&
1599 CurPtr+4 < BufferEnd) {
1600 CurPtr += 4;
1601 }
1602#endif
Mike Stump1eb44332009-09-09 15:08:12 +00001603
Reid Spencer5f016e22007-07-11 17:01:13 +00001604 // It has to be one of the bytes scanned, increment to it and read one.
1605 C = *CurPtr++;
1606 }
Mike Stump1eb44332009-09-09 15:08:12 +00001607
Reid Spencer5f016e22007-07-11 17:01:13 +00001608 // Loop to scan the remainder.
1609 while (C != '/' && C != '\0')
1610 C = *CurPtr++;
Mike Stump1eb44332009-09-09 15:08:12 +00001611
Reid Spencer5f016e22007-07-11 17:01:13 +00001612 FoundSlash:
1613 if (C == '/') {
1614 if (CurPtr[-2] == '*') // We found the final */. We're done!
1615 break;
Mike Stump1eb44332009-09-09 15:08:12 +00001616
Reid Spencer5f016e22007-07-11 17:01:13 +00001617 if ((CurPtr[-2] == '\n' || CurPtr[-2] == '\r')) {
1618 if (isEndOfBlockCommentWithEscapedNewLine(CurPtr-2, this)) {
1619 // We found the final */, though it had an escaped newline between the
1620 // * and /. We're done!
1621 break;
1622 }
1623 }
1624 if (CurPtr[0] == '*' && CurPtr[1] != '/') {
1625 // If this is a /* inside of the comment, emit a warning. Don't do this
1626 // if this is a /*/, which will end the comment. This misses cases with
1627 // embedded escaped newlines, but oh well.
Chris Lattner74d15df2008-11-22 02:02:22 +00001628 if (!isLexingRawMode())
1629 Diag(CurPtr-1, diag::warn_nested_block_comment);
Reid Spencer5f016e22007-07-11 17:01:13 +00001630 }
1631 } else if (C == 0 && CurPtr == BufferEnd+1) {
Douglas Gregor55817af2010-08-25 17:04:25 +00001632 if (PP && PP->isCodeCompletionFile(FileLoc))
1633 PP->CodeCompleteNaturalLanguage();
1634 else if (!isLexingRawMode())
Chris Lattner74d15df2008-11-22 02:02:22 +00001635 Diag(BufferPtr, diag::err_unterminated_block_comment);
Reid Spencer5f016e22007-07-11 17:01:13 +00001636 // Note: the user probably forgot a */. We could continue immediately
1637 // after the /*, but this would involve lexing a lot of what really is the
1638 // comment, which surely would confuse the parser.
Chris Lattner31f0eca2008-10-12 04:19:49 +00001639 --CurPtr;
Mike Stump1eb44332009-09-09 15:08:12 +00001640
Chris Lattner31f0eca2008-10-12 04:19:49 +00001641 // KeepWhitespaceMode should return this broken comment as a token. Since
1642 // it isn't a well formed comment, just return it as an 'unknown' token.
1643 if (isKeepWhitespaceMode()) {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001644 FormTokenWithChars(Result, CurPtr, tok::unknown);
Chris Lattner31f0eca2008-10-12 04:19:49 +00001645 return true;
1646 }
Mike Stump1eb44332009-09-09 15:08:12 +00001647
Chris Lattner31f0eca2008-10-12 04:19:49 +00001648 BufferPtr = CurPtr;
Chris Lattner2d381892008-10-12 04:15:42 +00001649 return false;
Reid Spencer5f016e22007-07-11 17:01:13 +00001650 }
1651 C = *CurPtr++;
1652 }
Mike Stump1eb44332009-09-09 15:08:12 +00001653
Chris Lattner3d0ad582010-02-03 21:06:21 +00001654 // Notify comment handlers about the comment unless we're in a #if 0 block.
1655 if (PP && !isLexingRawMode() &&
1656 PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr),
1657 getSourceLocation(CurPtr)))) {
Chris Lattner046c2272010-01-18 22:35:47 +00001658 BufferPtr = CurPtr;
1659 return true; // A token has to be returned.
1660 }
Douglas Gregor2e222532009-07-02 17:08:52 +00001661
Reid Spencer5f016e22007-07-11 17:01:13 +00001662 // If we are returning comments as tokens, return this comment as a token.
Chris Lattnerfa95a012008-10-12 03:22:02 +00001663 if (inKeepCommentMode()) {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001664 FormTokenWithChars(Result, CurPtr, tok::comment);
Chris Lattner2d381892008-10-12 04:15:42 +00001665 return true;
Reid Spencer5f016e22007-07-11 17:01:13 +00001666 }
1667
1668 // It is common for the tokens immediately after a /**/ comment to be
1669 // whitespace. Instead of going through the big switch, handle it
Chris Lattnerd88dc482008-10-12 04:05:48 +00001670 // efficiently now. This is safe even in KeepWhitespaceMode because we would
1671 // have already returned above with the comment as a token.
Reid Spencer5f016e22007-07-11 17:01:13 +00001672 if (isHorizontalWhitespace(*CurPtr)) {
Chris Lattnerd2177732007-07-20 16:59:19 +00001673 Result.setFlag(Token::LeadingSpace);
Reid Spencer5f016e22007-07-11 17:01:13 +00001674 SkipWhitespace(Result, CurPtr+1);
Chris Lattner2d381892008-10-12 04:15:42 +00001675 return false;
Reid Spencer5f016e22007-07-11 17:01:13 +00001676 }
1677
1678 // Otherwise, just return so that the next character will be lexed as a token.
1679 BufferPtr = CurPtr;
Chris Lattnerd2177732007-07-20 16:59:19 +00001680 Result.setFlag(Token::LeadingSpace);
Chris Lattner2d381892008-10-12 04:15:42 +00001681 return false;
Reid Spencer5f016e22007-07-11 17:01:13 +00001682}
1683
1684//===----------------------------------------------------------------------===//
1685// Primary Lexing Entry Points
1686//===----------------------------------------------------------------------===//
1687
Reid Spencer5f016e22007-07-11 17:01:13 +00001688/// ReadToEndOfLine - Read the rest of the current preprocessor line as an
1689/// uninterpreted string. This switches the lexer out of directive mode.
1690std::string Lexer::ReadToEndOfLine() {
1691 assert(ParsingPreprocessorDirective && ParsingFilename == false &&
1692 "Must be in a preprocessing directive!");
1693 std::string Result;
Chris Lattnerd2177732007-07-20 16:59:19 +00001694 Token Tmp;
Reid Spencer5f016e22007-07-11 17:01:13 +00001695
1696 // CurPtr - Cache BufferPtr in an automatic variable.
1697 const char *CurPtr = BufferPtr;
1698 while (1) {
1699 char Char = getAndAdvanceChar(CurPtr, Tmp);
1700 switch (Char) {
1701 default:
1702 Result += Char;
1703 break;
1704 case 0: // Null.
1705 // Found end of file?
1706 if (CurPtr-1 != BufferEnd) {
1707 // Nope, normal character, continue.
1708 Result += Char;
1709 break;
1710 }
1711 // FALL THROUGH.
1712 case '\r':
1713 case '\n':
1714 // Okay, we found the end of the line. First, back up past the \0, \r, \n.
1715 assert(CurPtr[-1] == Char && "Trigraphs for newline?");
1716 BufferPtr = CurPtr-1;
Mike Stump1eb44332009-09-09 15:08:12 +00001717
Reid Spencer5f016e22007-07-11 17:01:13 +00001718 // Next, lex the character, which should handle the EOM transition.
1719 Lex(Tmp);
Douglas Gregor55817af2010-08-25 17:04:25 +00001720 if (Tmp.is(tok::code_completion)) {
1721 if (PP && PP->getCodeCompletionHandler())
1722 PP->getCodeCompletionHandler()->CodeCompleteNaturalLanguage();
1723 Lex(Tmp);
1724 }
Chris Lattner22f6bbc2007-10-09 18:02:16 +00001725 assert(Tmp.is(tok::eom) && "Unexpected token!");
Mike Stump1eb44332009-09-09 15:08:12 +00001726
Reid Spencer5f016e22007-07-11 17:01:13 +00001727 // Finally, we're done, return the string we found.
1728 return Result;
1729 }
1730 }
1731}
1732
1733/// LexEndOfFile - CurPtr points to the end of this file. Handle this
1734/// condition, reporting diagnostics and handling other edge cases as required.
1735/// This returns true if Result contains a token, false if PP.Lex should be
1736/// called again.
Chris Lattnerd2177732007-07-20 16:59:19 +00001737bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) {
Douglas Gregorf44e8542010-08-24 19:08:16 +00001738 // Check if we are performing code completion.
1739 if (PP && PP->isCodeCompletionFile(FileLoc)) {
1740 // We're at the end of the file, but we've been asked to consider the
1741 // end of the file to be a code-completion token. Return the
1742 // code-completion token.
1743 Result.startToken();
1744 FormTokenWithChars(Result, CurPtr, tok::code_completion);
1745
1746 // Only do the eof -> code_completion translation once.
1747 PP->SetCodeCompletionPoint(0, 0, 0);
1748
1749 // Silence any diagnostics that occur once we hit the code-completion point.
1750 PP->getDiagnostics().setSuppressAllDiagnostics(true);
1751 return true;
1752 }
1753
Reid Spencer5f016e22007-07-11 17:01:13 +00001754 // If we hit the end of the file while parsing a preprocessor directive,
1755 // end the preprocessor directive first. The next token returned will
1756 // then be the end of file.
1757 if (ParsingPreprocessorDirective) {
1758 // Done parsing the "line".
1759 ParsingPreprocessorDirective = false;
Reid Spencer5f016e22007-07-11 17:01:13 +00001760 // Update the location of token as well as BufferPtr.
Chris Lattner9e6293d2008-10-12 04:51:35 +00001761 FormTokenWithChars(Result, CurPtr, tok::eom);
Mike Stump1eb44332009-09-09 15:08:12 +00001762
Reid Spencer5f016e22007-07-11 17:01:13 +00001763 // Restore comment saving mode, in case it was disabled for directive.
Chris Lattnerf744d132008-10-12 03:27:19 +00001764 SetCommentRetentionState(PP->getCommentRetentionState());
Reid Spencer5f016e22007-07-11 17:01:13 +00001765 return true; // Have a token.
Mike Stump1eb44332009-09-09 15:08:12 +00001766 }
Douglas Gregor86d9a522009-09-21 16:56:56 +00001767
Reid Spencer5f016e22007-07-11 17:01:13 +00001768 // If we are in raw mode, return this event as an EOF token. Let the caller
1769 // that put us in raw mode handle the event.
Chris Lattner74d15df2008-11-22 02:02:22 +00001770 if (isLexingRawMode()) {
Reid Spencer5f016e22007-07-11 17:01:13 +00001771 Result.startToken();
1772 BufferPtr = BufferEnd;
Chris Lattner9e6293d2008-10-12 04:51:35 +00001773 FormTokenWithChars(Result, BufferEnd, tok::eof);
Reid Spencer5f016e22007-07-11 17:01:13 +00001774 return true;
1775 }
Douglas Gregor86d9a522009-09-21 16:56:56 +00001776
Douglas Gregorf44e8542010-08-24 19:08:16 +00001777 // Issue diagnostics for unterminated #if and missing newline.
1778
Reid Spencer5f016e22007-07-11 17:01:13 +00001779 // If we are in a #if directive, emit an error.
1780 while (!ConditionalStack.empty()) {
Douglas Gregor2d474ba2010-08-12 17:04:55 +00001781 if (!PP->isCodeCompletionFile(FileLoc))
1782 PP->Diag(ConditionalStack.back().IfLoc,
1783 diag::err_pp_unterminated_conditional);
Reid Spencer5f016e22007-07-11 17:01:13 +00001784 ConditionalStack.pop_back();
1785 }
Mike Stump1eb44332009-09-09 15:08:12 +00001786
Chris Lattnerb25e5d72008-04-12 05:54:25 +00001787 // C99 5.1.1.2p2: If the file is non-empty and didn't end in a newline, issue
1788 // a pedwarn.
1789 if (CurPtr != BufferStart && (CurPtr[-1] != '\n' && CurPtr[-1] != '\r'))
Mike Stump20d0ee52009-04-02 02:29:42 +00001790 Diag(BufferEnd, diag::ext_no_newline_eof)
Douglas Gregor849b2432010-03-31 17:46:05 +00001791 << FixItHint::CreateInsertion(getSourceLocation(BufferEnd), "\n");
Mike Stump1eb44332009-09-09 15:08:12 +00001792
Reid Spencer5f016e22007-07-11 17:01:13 +00001793 BufferPtr = CurPtr;
1794
1795 // Finally, let the preprocessor handle this.
Chris Lattner168ae2d2007-10-17 20:41:00 +00001796 return PP->HandleEndOfFile(Result);
Reid Spencer5f016e22007-07-11 17:01:13 +00001797}
1798
1799/// isNextPPTokenLParen - Return 1 if the next unexpanded token lexed from
1800/// the specified lexer will return a tok::l_paren token, 0 if it is something
1801/// else and 2 if there are no more tokens in the buffer controlled by the
1802/// lexer.
1803unsigned Lexer::isNextPPTokenLParen() {
1804 assert(!LexingRawMode && "How can we expand a macro from a skipping buffer?");
Mike Stump1eb44332009-09-09 15:08:12 +00001805
Reid Spencer5f016e22007-07-11 17:01:13 +00001806 // Switch to 'skipping' mode. This will ensure that we can lex a token
1807 // without emitting diagnostics, disables macro expansion, and will cause EOF
1808 // to return an EOF token instead of popping the include stack.
1809 LexingRawMode = true;
Mike Stump1eb44332009-09-09 15:08:12 +00001810
Reid Spencer5f016e22007-07-11 17:01:13 +00001811 // Save state that can be changed while lexing so that we can restore it.
1812 const char *TmpBufferPtr = BufferPtr;
Chris Lattnera864cf72009-04-24 07:15:46 +00001813 bool inPPDirectiveMode = ParsingPreprocessorDirective;
Mike Stump1eb44332009-09-09 15:08:12 +00001814
Chris Lattnerd2177732007-07-20 16:59:19 +00001815 Token Tok;
Reid Spencer5f016e22007-07-11 17:01:13 +00001816 Tok.startToken();
1817 LexTokenInternal(Tok);
Mike Stump1eb44332009-09-09 15:08:12 +00001818
Reid Spencer5f016e22007-07-11 17:01:13 +00001819 // Restore state that may have changed.
1820 BufferPtr = TmpBufferPtr;
Chris Lattnera864cf72009-04-24 07:15:46 +00001821 ParsingPreprocessorDirective = inPPDirectiveMode;
Mike Stump1eb44332009-09-09 15:08:12 +00001822
Reid Spencer5f016e22007-07-11 17:01:13 +00001823 // Restore the lexer back to non-skipping mode.
1824 LexingRawMode = false;
Mike Stump1eb44332009-09-09 15:08:12 +00001825
Chris Lattner22f6bbc2007-10-09 18:02:16 +00001826 if (Tok.is(tok::eof))
Reid Spencer5f016e22007-07-11 17:01:13 +00001827 return 2;
Chris Lattner22f6bbc2007-10-09 18:02:16 +00001828 return Tok.is(tok::l_paren);
Reid Spencer5f016e22007-07-11 17:01:13 +00001829}
1830
Chris Lattner34f349d2009-12-14 06:16:57 +00001831/// FindConflictEnd - Find the end of a version control conflict marker.
1832static const char *FindConflictEnd(const char *CurPtr, const char *BufferEnd) {
1833 llvm::StringRef RestOfBuffer(CurPtr+7, BufferEnd-CurPtr-7);
1834 size_t Pos = RestOfBuffer.find(">>>>>>>");
1835 while (Pos != llvm::StringRef::npos) {
1836 // Must occur at start of line.
1837 if (RestOfBuffer[Pos-1] != '\r' &&
1838 RestOfBuffer[Pos-1] != '\n') {
1839 RestOfBuffer = RestOfBuffer.substr(Pos+7);
Chris Lattner3d488992010-05-17 20:27:25 +00001840 Pos = RestOfBuffer.find(">>>>>>>");
Chris Lattner34f349d2009-12-14 06:16:57 +00001841 continue;
1842 }
1843 return RestOfBuffer.data()+Pos;
1844 }
1845 return 0;
1846}
1847
1848/// IsStartOfConflictMarker - If the specified pointer is the start of a version
1849/// control conflict marker like '<<<<<<<', recognize it as such, emit an error
1850/// and recover nicely. This returns true if it is a conflict marker and false
1851/// if not.
1852bool Lexer::IsStartOfConflictMarker(const char *CurPtr) {
1853 // Only a conflict marker if it starts at the beginning of a line.
1854 if (CurPtr != BufferStart &&
1855 CurPtr[-1] != '\n' && CurPtr[-1] != '\r')
1856 return false;
1857
1858 // Check to see if we have <<<<<<<.
1859 if (BufferEnd-CurPtr < 8 ||
1860 llvm::StringRef(CurPtr, 7) != "<<<<<<<")
1861 return false;
1862
1863 // If we have a situation where we don't care about conflict markers, ignore
1864 // it.
1865 if (IsInConflictMarker || isLexingRawMode())
1866 return false;
1867
1868 // Check to see if there is a >>>>>>> somewhere in the buffer at the start of
1869 // a line to terminate this conflict marker.
Chris Lattner3d488992010-05-17 20:27:25 +00001870 if (FindConflictEnd(CurPtr, BufferEnd)) {
Chris Lattner34f349d2009-12-14 06:16:57 +00001871 // We found a match. We are really in a conflict marker.
1872 // Diagnose this, and ignore to the end of line.
1873 Diag(CurPtr, diag::err_conflict_marker);
1874 IsInConflictMarker = true;
1875
1876 // Skip ahead to the end of line. We know this exists because the
1877 // end-of-conflict marker starts with \r or \n.
1878 while (*CurPtr != '\r' && *CurPtr != '\n') {
1879 assert(CurPtr != BufferEnd && "Didn't find end of line");
1880 ++CurPtr;
1881 }
1882 BufferPtr = CurPtr;
1883 return true;
1884 }
1885
1886 // No end of conflict marker found.
1887 return false;
1888}
1889
1890
1891/// HandleEndOfConflictMarker - If this is a '=======' or '|||||||' or '>>>>>>>'
1892/// marker, then it is the end of a conflict marker. Handle it by ignoring up
1893/// until the end of the line. This returns true if it is a conflict marker and
1894/// false if not.
1895bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) {
1896 // Only a conflict marker if it starts at the beginning of a line.
1897 if (CurPtr != BufferStart &&
1898 CurPtr[-1] != '\n' && CurPtr[-1] != '\r')
1899 return false;
1900
1901 // If we have a situation where we don't care about conflict markers, ignore
1902 // it.
1903 if (!IsInConflictMarker || isLexingRawMode())
1904 return false;
1905
1906 // Check to see if we have the marker (7 characters in a row).
1907 for (unsigned i = 1; i != 7; ++i)
1908 if (CurPtr[i] != CurPtr[0])
1909 return false;
1910
1911 // If we do have it, search for the end of the conflict marker. This could
1912 // fail if it got skipped with a '#if 0' or something. Note that CurPtr might
1913 // be the end of conflict marker.
1914 if (const char *End = FindConflictEnd(CurPtr, BufferEnd)) {
1915 CurPtr = End;
1916
1917 // Skip ahead to the end of line.
1918 while (CurPtr != BufferEnd && *CurPtr != '\r' && *CurPtr != '\n')
1919 ++CurPtr;
1920
1921 BufferPtr = CurPtr;
1922
1923 // No longer in the conflict marker.
1924 IsInConflictMarker = false;
1925 return true;
1926 }
1927
1928 return false;
1929}
1930
Reid Spencer5f016e22007-07-11 17:01:13 +00001931
1932/// LexTokenInternal - This implements a simple C family lexer. It is an
1933/// extremely performance critical piece of code. This assumes that the buffer
Chris Lattnerefb173d2009-07-07 05:05:42 +00001934/// has a null character at the end of the file. This returns a preprocessing
1935/// token, not a normal token, as such, it is an internal interface. It assumes
1936/// that the Flags of result have been cleared before calling this.
Chris Lattnerd2177732007-07-20 16:59:19 +00001937void Lexer::LexTokenInternal(Token &Result) {
Reid Spencer5f016e22007-07-11 17:01:13 +00001938LexNextToken:
1939 // New token, can't need cleaning yet.
Chris Lattnerd2177732007-07-20 16:59:19 +00001940 Result.clearFlag(Token::NeedsCleaning);
Reid Spencer5f016e22007-07-11 17:01:13 +00001941 Result.setIdentifierInfo(0);
Mike Stump1eb44332009-09-09 15:08:12 +00001942
Reid Spencer5f016e22007-07-11 17:01:13 +00001943 // CurPtr - Cache BufferPtr in an automatic variable.
1944 const char *CurPtr = BufferPtr;
1945
1946 // Small amounts of horizontal whitespace is very common between tokens.
1947 if ((*CurPtr == ' ') || (*CurPtr == '\t')) {
1948 ++CurPtr;
1949 while ((*CurPtr == ' ') || (*CurPtr == '\t'))
1950 ++CurPtr;
Mike Stump1eb44332009-09-09 15:08:12 +00001951
Chris Lattnerd88dc482008-10-12 04:05:48 +00001952 // If we are keeping whitespace and other tokens, just return what we just
1953 // skipped. The next lexer invocation will return the token after the
1954 // whitespace.
1955 if (isKeepWhitespaceMode()) {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001956 FormTokenWithChars(Result, CurPtr, tok::unknown);
Chris Lattnerd88dc482008-10-12 04:05:48 +00001957 return;
1958 }
Mike Stump1eb44332009-09-09 15:08:12 +00001959
Reid Spencer5f016e22007-07-11 17:01:13 +00001960 BufferPtr = CurPtr;
Chris Lattnerd2177732007-07-20 16:59:19 +00001961 Result.setFlag(Token::LeadingSpace);
Reid Spencer5f016e22007-07-11 17:01:13 +00001962 }
Mike Stump1eb44332009-09-09 15:08:12 +00001963
Reid Spencer5f016e22007-07-11 17:01:13 +00001964 unsigned SizeTmp, SizeTmp2; // Temporaries for use in cases below.
Mike Stump1eb44332009-09-09 15:08:12 +00001965
Reid Spencer5f016e22007-07-11 17:01:13 +00001966 // Read a character, advancing over it.
1967 char Char = getAndAdvanceChar(CurPtr, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00001968 tok::TokenKind Kind;
Mike Stump1eb44332009-09-09 15:08:12 +00001969
Reid Spencer5f016e22007-07-11 17:01:13 +00001970 switch (Char) {
1971 case 0: // Null.
1972 // Found end of file?
1973 if (CurPtr-1 == BufferEnd) {
1974 // Read the PP instance variable into an automatic variable, because
1975 // LexEndOfFile will often delete 'this'.
Chris Lattner168ae2d2007-10-17 20:41:00 +00001976 Preprocessor *PPCache = PP;
Reid Spencer5f016e22007-07-11 17:01:13 +00001977 if (LexEndOfFile(Result, CurPtr-1)) // Retreat back into the file.
1978 return; // Got a token to return.
Chris Lattner168ae2d2007-10-17 20:41:00 +00001979 assert(PPCache && "Raw buffer::LexEndOfFile should return a token");
1980 return PPCache->Lex(Result);
Reid Spencer5f016e22007-07-11 17:01:13 +00001981 }
Mike Stump1eb44332009-09-09 15:08:12 +00001982
Chris Lattner74d15df2008-11-22 02:02:22 +00001983 if (!isLexingRawMode())
1984 Diag(CurPtr-1, diag::null_in_file);
Chris Lattnerd2177732007-07-20 16:59:19 +00001985 Result.setFlag(Token::LeadingSpace);
Chris Lattnerd88dc482008-10-12 04:05:48 +00001986 if (SkipWhitespace(Result, CurPtr))
1987 return; // KeepWhitespaceMode
Mike Stump1eb44332009-09-09 15:08:12 +00001988
Reid Spencer5f016e22007-07-11 17:01:13 +00001989 goto LexNextToken; // GCC isn't tail call eliminating.
Chris Lattnera2bf1052009-12-17 05:29:40 +00001990
1991 case 26: // DOS & CP/M EOF: "^Z".
1992 // If we're in Microsoft extensions mode, treat this as end of file.
1993 if (Features.Microsoft) {
1994 // Read the PP instance variable into an automatic variable, because
1995 // LexEndOfFile will often delete 'this'.
1996 Preprocessor *PPCache = PP;
1997 if (LexEndOfFile(Result, CurPtr-1)) // Retreat back into the file.
1998 return; // Got a token to return.
1999 assert(PPCache && "Raw buffer::LexEndOfFile should return a token");
2000 return PPCache->Lex(Result);
2001 }
2002 // If Microsoft extensions are disabled, this is just random garbage.
2003 Kind = tok::unknown;
2004 break;
2005
Reid Spencer5f016e22007-07-11 17:01:13 +00002006 case '\n':
2007 case '\r':
2008 // If we are inside a preprocessor directive and we see the end of line,
2009 // we know we are done with the directive, so return an EOM token.
2010 if (ParsingPreprocessorDirective) {
2011 // Done parsing the "line".
2012 ParsingPreprocessorDirective = false;
Mike Stump1eb44332009-09-09 15:08:12 +00002013
Reid Spencer5f016e22007-07-11 17:01:13 +00002014 // Restore comment saving mode, in case it was disabled for directive.
Chris Lattnerf744d132008-10-12 03:27:19 +00002015 SetCommentRetentionState(PP->getCommentRetentionState());
Mike Stump1eb44332009-09-09 15:08:12 +00002016
Reid Spencer5f016e22007-07-11 17:01:13 +00002017 // Since we consumed a newline, we are back at the start of a line.
2018 IsAtStartOfLine = true;
Mike Stump1eb44332009-09-09 15:08:12 +00002019
Chris Lattner9e6293d2008-10-12 04:51:35 +00002020 Kind = tok::eom;
Reid Spencer5f016e22007-07-11 17:01:13 +00002021 break;
2022 }
2023 // The returned token is at the start of the line.
Chris Lattnerd2177732007-07-20 16:59:19 +00002024 Result.setFlag(Token::StartOfLine);
Reid Spencer5f016e22007-07-11 17:01:13 +00002025 // No leading whitespace seen so far.
Chris Lattnerd2177732007-07-20 16:59:19 +00002026 Result.clearFlag(Token::LeadingSpace);
Mike Stump1eb44332009-09-09 15:08:12 +00002027
Chris Lattnerd88dc482008-10-12 04:05:48 +00002028 if (SkipWhitespace(Result, CurPtr))
2029 return; // KeepWhitespaceMode
Reid Spencer5f016e22007-07-11 17:01:13 +00002030 goto LexNextToken; // GCC isn't tail call eliminating.
2031 case ' ':
2032 case '\t':
2033 case '\f':
2034 case '\v':
Chris Lattner8133cfc2007-07-22 06:29:05 +00002035 SkipHorizontalWhitespace:
Chris Lattnerd2177732007-07-20 16:59:19 +00002036 Result.setFlag(Token::LeadingSpace);
Chris Lattnerd88dc482008-10-12 04:05:48 +00002037 if (SkipWhitespace(Result, CurPtr))
2038 return; // KeepWhitespaceMode
Chris Lattner8133cfc2007-07-22 06:29:05 +00002039
2040 SkipIgnoredUnits:
2041 CurPtr = BufferPtr;
Mike Stump1eb44332009-09-09 15:08:12 +00002042
Chris Lattner8133cfc2007-07-22 06:29:05 +00002043 // If the next token is obviously a // or /* */ comment, skip it efficiently
2044 // too (without going through the big switch stmt).
Chris Lattner8402c732009-01-16 22:39:25 +00002045 if (CurPtr[0] == '/' && CurPtr[1] == '/' && !inKeepCommentMode() &&
2046 Features.BCPLComment) {
Chris Lattner046c2272010-01-18 22:35:47 +00002047 if (SkipBCPLComment(Result, CurPtr+2))
2048 return; // There is a token to return.
Chris Lattner8133cfc2007-07-22 06:29:05 +00002049 goto SkipIgnoredUnits;
Chris Lattnerfa95a012008-10-12 03:22:02 +00002050 } else if (CurPtr[0] == '/' && CurPtr[1] == '*' && !inKeepCommentMode()) {
Chris Lattner046c2272010-01-18 22:35:47 +00002051 if (SkipBlockComment(Result, CurPtr+2))
2052 return; // There is a token to return.
Chris Lattner8133cfc2007-07-22 06:29:05 +00002053 goto SkipIgnoredUnits;
2054 } else if (isHorizontalWhitespace(*CurPtr)) {
2055 goto SkipHorizontalWhitespace;
2056 }
Reid Spencer5f016e22007-07-11 17:01:13 +00002057 goto LexNextToken; // GCC isn't tail call eliminating.
Chris Lattnera2bf1052009-12-17 05:29:40 +00002058
Chris Lattner3a570772008-01-03 17:58:54 +00002059 // C99 6.4.4.1: Integer Constants.
2060 // C99 6.4.4.2: Floating Constants.
2061 case '0': case '1': case '2': case '3': case '4':
2062 case '5': case '6': case '7': case '8': case '9':
2063 // Notify MIOpt that we read a non-whitespace/non-comment token.
2064 MIOpt.ReadToken();
2065 return LexNumericConstant(Result, CurPtr);
Mike Stump1eb44332009-09-09 15:08:12 +00002066
Chris Lattner3a570772008-01-03 17:58:54 +00002067 case 'L': // Identifier (Loony) or wide literal (L'x' or L"xyz").
Reid Spencer5f016e22007-07-11 17:01:13 +00002068 // Notify MIOpt that we read a non-whitespace/non-comment token.
2069 MIOpt.ReadToken();
2070 Char = getCharAndSize(CurPtr, SizeTmp);
2071
2072 // Wide string literal.
2073 if (Char == '"')
2074 return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result),
2075 true);
2076
2077 // Wide character constant.
2078 if (Char == '\'')
2079 return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result));
2080 // FALL THROUGH, treating L like the start of an identifier.
Mike Stump1eb44332009-09-09 15:08:12 +00002081
Reid Spencer5f016e22007-07-11 17:01:13 +00002082 // C99 6.4.2: Identifiers.
2083 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G':
2084 case 'H': case 'I': case 'J': case 'K': /*'L'*/case 'M': case 'N':
2085 case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U':
2086 case 'V': case 'W': case 'X': case 'Y': case 'Z':
2087 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g':
2088 case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n':
2089 case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u':
2090 case 'v': case 'w': case 'x': case 'y': case 'z':
2091 case '_':
2092 // Notify MIOpt that we read a non-whitespace/non-comment token.
2093 MIOpt.ReadToken();
2094 return LexIdentifier(Result, CurPtr);
Chris Lattner3a570772008-01-03 17:58:54 +00002095
2096 case '$': // $ in identifiers.
2097 if (Features.DollarIdents) {
Chris Lattner74d15df2008-11-22 02:02:22 +00002098 if (!isLexingRawMode())
2099 Diag(CurPtr-1, diag::ext_dollar_in_identifier);
Chris Lattner3a570772008-01-03 17:58:54 +00002100 // Notify MIOpt that we read a non-whitespace/non-comment token.
2101 MIOpt.ReadToken();
2102 return LexIdentifier(Result, CurPtr);
2103 }
Mike Stump1eb44332009-09-09 15:08:12 +00002104
Chris Lattner9e6293d2008-10-12 04:51:35 +00002105 Kind = tok::unknown;
Chris Lattner3a570772008-01-03 17:58:54 +00002106 break;
Mike Stump1eb44332009-09-09 15:08:12 +00002107
Reid Spencer5f016e22007-07-11 17:01:13 +00002108 // C99 6.4.4: Character Constants.
2109 case '\'':
2110 // Notify MIOpt that we read a non-whitespace/non-comment token.
2111 MIOpt.ReadToken();
2112 return LexCharConstant(Result, CurPtr);
2113
2114 // C99 6.4.5: String Literals.
2115 case '"':
2116 // Notify MIOpt that we read a non-whitespace/non-comment token.
2117 MIOpt.ReadToken();
2118 return LexStringLiteral(Result, CurPtr, false);
2119
2120 // C99 6.4.6: Punctuators.
2121 case '?':
Chris Lattner9e6293d2008-10-12 04:51:35 +00002122 Kind = tok::question;
Reid Spencer5f016e22007-07-11 17:01:13 +00002123 break;
2124 case '[':
Chris Lattner9e6293d2008-10-12 04:51:35 +00002125 Kind = tok::l_square;
Reid Spencer5f016e22007-07-11 17:01:13 +00002126 break;
2127 case ']':
Chris Lattner9e6293d2008-10-12 04:51:35 +00002128 Kind = tok::r_square;
Reid Spencer5f016e22007-07-11 17:01:13 +00002129 break;
2130 case '(':
Chris Lattner9e6293d2008-10-12 04:51:35 +00002131 Kind = tok::l_paren;
Reid Spencer5f016e22007-07-11 17:01:13 +00002132 break;
2133 case ')':
Chris Lattner9e6293d2008-10-12 04:51:35 +00002134 Kind = tok::r_paren;
Reid Spencer5f016e22007-07-11 17:01:13 +00002135 break;
2136 case '{':
Chris Lattner9e6293d2008-10-12 04:51:35 +00002137 Kind = tok::l_brace;
Reid Spencer5f016e22007-07-11 17:01:13 +00002138 break;
2139 case '}':
Chris Lattner9e6293d2008-10-12 04:51:35 +00002140 Kind = tok::r_brace;
Reid Spencer5f016e22007-07-11 17:01:13 +00002141 break;
2142 case '.':
2143 Char = getCharAndSize(CurPtr, SizeTmp);
2144 if (Char >= '0' && Char <= '9') {
2145 // Notify MIOpt that we read a non-whitespace/non-comment token.
2146 MIOpt.ReadToken();
2147
2148 return LexNumericConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result));
2149 } else if (Features.CPlusPlus && Char == '*') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002150 Kind = tok::periodstar;
Reid Spencer5f016e22007-07-11 17:01:13 +00002151 CurPtr += SizeTmp;
2152 } else if (Char == '.' &&
2153 getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '.') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002154 Kind = tok::ellipsis;
Reid Spencer5f016e22007-07-11 17:01:13 +00002155 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
2156 SizeTmp2, Result);
2157 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002158 Kind = tok::period;
Reid Spencer5f016e22007-07-11 17:01:13 +00002159 }
2160 break;
2161 case '&':
2162 Char = getCharAndSize(CurPtr, SizeTmp);
2163 if (Char == '&') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002164 Kind = tok::ampamp;
Reid Spencer5f016e22007-07-11 17:01:13 +00002165 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
2166 } else if (Char == '=') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002167 Kind = tok::ampequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00002168 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
2169 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002170 Kind = tok::amp;
Reid Spencer5f016e22007-07-11 17:01:13 +00002171 }
2172 break;
Mike Stump1eb44332009-09-09 15:08:12 +00002173 case '*':
Reid Spencer5f016e22007-07-11 17:01:13 +00002174 if (getCharAndSize(CurPtr, SizeTmp) == '=') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002175 Kind = tok::starequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00002176 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
2177 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002178 Kind = tok::star;
Reid Spencer5f016e22007-07-11 17:01:13 +00002179 }
2180 break;
2181 case '+':
2182 Char = getCharAndSize(CurPtr, SizeTmp);
2183 if (Char == '+') {
Reid Spencer5f016e22007-07-11 17:01:13 +00002184 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00002185 Kind = tok::plusplus;
Reid Spencer5f016e22007-07-11 17:01:13 +00002186 } else if (Char == '=') {
Reid Spencer5f016e22007-07-11 17:01:13 +00002187 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00002188 Kind = tok::plusequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00002189 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002190 Kind = tok::plus;
Reid Spencer5f016e22007-07-11 17:01:13 +00002191 }
2192 break;
2193 case '-':
2194 Char = getCharAndSize(CurPtr, SizeTmp);
Chris Lattner9e6293d2008-10-12 04:51:35 +00002195 if (Char == '-') { // --
Reid Spencer5f016e22007-07-11 17:01:13 +00002196 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00002197 Kind = tok::minusminus;
Mike Stump1eb44332009-09-09 15:08:12 +00002198 } else if (Char == '>' && Features.CPlusPlus &&
Chris Lattner9e6293d2008-10-12 04:51:35 +00002199 getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '*') { // C++ ->*
Reid Spencer5f016e22007-07-11 17:01:13 +00002200 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
2201 SizeTmp2, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00002202 Kind = tok::arrowstar;
2203 } else if (Char == '>') { // ->
Reid Spencer5f016e22007-07-11 17:01:13 +00002204 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00002205 Kind = tok::arrow;
2206 } else if (Char == '=') { // -=
Reid Spencer5f016e22007-07-11 17:01:13 +00002207 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00002208 Kind = tok::minusequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00002209 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002210 Kind = tok::minus;
Reid Spencer5f016e22007-07-11 17:01:13 +00002211 }
2212 break;
2213 case '~':
Chris Lattner9e6293d2008-10-12 04:51:35 +00002214 Kind = tok::tilde;
Reid Spencer5f016e22007-07-11 17:01:13 +00002215 break;
2216 case '!':
2217 if (getCharAndSize(CurPtr, SizeTmp) == '=') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002218 Kind = tok::exclaimequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00002219 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
2220 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002221 Kind = tok::exclaim;
Reid Spencer5f016e22007-07-11 17:01:13 +00002222 }
2223 break;
2224 case '/':
2225 // 6.4.9: Comments
2226 Char = getCharAndSize(CurPtr, SizeTmp);
2227 if (Char == '/') { // BCPL comment.
Chris Lattner8402c732009-01-16 22:39:25 +00002228 // Even if BCPL comments are disabled (e.g. in C89 mode), we generally
2229 // want to lex this as a comment. There is one problem with this though,
2230 // that in one particular corner case, this can change the behavior of the
2231 // resultant program. For example, In "foo //**/ bar", C89 would lex
2232 // this as "foo / bar" and langauges with BCPL comments would lex it as
2233 // "foo". Check to see if the character after the second slash is a '*'.
2234 // If so, we will lex that as a "/" instead of the start of a comment.
2235 if (Features.BCPLComment ||
2236 getCharAndSize(CurPtr+SizeTmp, SizeTmp2) != '*') {
2237 if (SkipBCPLComment(Result, ConsumeChar(CurPtr, SizeTmp, Result)))
Chris Lattner046c2272010-01-18 22:35:47 +00002238 return; // There is a token to return.
Mike Stump1eb44332009-09-09 15:08:12 +00002239
Chris Lattner8402c732009-01-16 22:39:25 +00002240 // It is common for the tokens immediately after a // comment to be
2241 // whitespace (indentation for the next line). Instead of going through
2242 // the big switch, handle it efficiently now.
2243 goto SkipIgnoredUnits;
2244 }
2245 }
Mike Stump1eb44332009-09-09 15:08:12 +00002246
Chris Lattner8402c732009-01-16 22:39:25 +00002247 if (Char == '*') { // /**/ comment.
Reid Spencer5f016e22007-07-11 17:01:13 +00002248 if (SkipBlockComment(Result, ConsumeChar(CurPtr, SizeTmp, Result)))
Chris Lattner046c2272010-01-18 22:35:47 +00002249 return; // There is a token to return.
Chris Lattner2d381892008-10-12 04:15:42 +00002250 goto LexNextToken; // GCC isn't tail call eliminating.
Chris Lattner8402c732009-01-16 22:39:25 +00002251 }
Mike Stump1eb44332009-09-09 15:08:12 +00002252
Chris Lattner8402c732009-01-16 22:39:25 +00002253 if (Char == '=') {
Reid Spencer5f016e22007-07-11 17:01:13 +00002254 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00002255 Kind = tok::slashequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00002256 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002257 Kind = tok::slash;
Reid Spencer5f016e22007-07-11 17:01:13 +00002258 }
2259 break;
2260 case '%':
2261 Char = getCharAndSize(CurPtr, SizeTmp);
2262 if (Char == '=') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002263 Kind = tok::percentequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00002264 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
2265 } else if (Features.Digraphs && Char == '>') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002266 Kind = tok::r_brace; // '%>' -> '}'
Reid Spencer5f016e22007-07-11 17:01:13 +00002267 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
2268 } else if (Features.Digraphs && Char == ':') {
2269 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
2270 Char = getCharAndSize(CurPtr, SizeTmp);
2271 if (Char == '%' && getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == ':') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002272 Kind = tok::hashhash; // '%:%:' -> '##'
Reid Spencer5f016e22007-07-11 17:01:13 +00002273 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
2274 SizeTmp2, Result);
2275 } else if (Char == '@' && Features.Microsoft) { // %:@ -> #@ -> Charize
Reid Spencer5f016e22007-07-11 17:01:13 +00002276 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner74d15df2008-11-22 02:02:22 +00002277 if (!isLexingRawMode())
2278 Diag(BufferPtr, diag::charize_microsoft_ext);
Chris Lattner9e6293d2008-10-12 04:51:35 +00002279 Kind = tok::hashat;
Chris Lattnere91e9322009-03-18 20:58:27 +00002280 } else { // '%:' -> '#'
Reid Spencer5f016e22007-07-11 17:01:13 +00002281 // We parsed a # character. If this occurs at the start of the line,
2282 // it's actually the start of a preprocessing directive. Callback to
2283 // the preprocessor to handle it.
2284 // FIXME: -fpreprocessed mode??
Chris Lattner766703b2009-05-13 06:10:29 +00002285 if (Result.isAtStartOfLine() && !LexingRawMode && !Is_PragmaLexer) {
Chris Lattnere91e9322009-03-18 20:58:27 +00002286 FormTokenWithChars(Result, CurPtr, tok::hash);
Chris Lattner168ae2d2007-10-17 20:41:00 +00002287 PP->HandleDirective(Result);
Mike Stump1eb44332009-09-09 15:08:12 +00002288
Reid Spencer5f016e22007-07-11 17:01:13 +00002289 // As an optimization, if the preprocessor didn't switch lexers, tail
2290 // recurse.
Chris Lattner168ae2d2007-10-17 20:41:00 +00002291 if (PP->isCurrentLexer(this)) {
Reid Spencer5f016e22007-07-11 17:01:13 +00002292 // Start a new token. If this is a #include or something, the PP may
2293 // want us starting at the beginning of the line again. If so, set
Chris Lattner515f43f2010-04-12 23:04:41 +00002294 // the StartOfLine flag and clear LeadingSpace.
Reid Spencer5f016e22007-07-11 17:01:13 +00002295 if (IsAtStartOfLine) {
Chris Lattnerd2177732007-07-20 16:59:19 +00002296 Result.setFlag(Token::StartOfLine);
Chris Lattner515f43f2010-04-12 23:04:41 +00002297 Result.clearFlag(Token::LeadingSpace);
Reid Spencer5f016e22007-07-11 17:01:13 +00002298 IsAtStartOfLine = false;
2299 }
2300 goto LexNextToken; // GCC isn't tail call eliminating.
2301 }
Mike Stump1eb44332009-09-09 15:08:12 +00002302
Chris Lattner168ae2d2007-10-17 20:41:00 +00002303 return PP->Lex(Result);
Reid Spencer5f016e22007-07-11 17:01:13 +00002304 }
Mike Stump1eb44332009-09-09 15:08:12 +00002305
Chris Lattnere91e9322009-03-18 20:58:27 +00002306 Kind = tok::hash;
Reid Spencer5f016e22007-07-11 17:01:13 +00002307 }
2308 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002309 Kind = tok::percent;
Reid Spencer5f016e22007-07-11 17:01:13 +00002310 }
2311 break;
2312 case '<':
2313 Char = getCharAndSize(CurPtr, SizeTmp);
2314 if (ParsingFilename) {
Chris Lattner9cb51ce2009-04-17 23:56:52 +00002315 return LexAngledStringLiteral(Result, CurPtr);
Reid Spencer5f016e22007-07-11 17:01:13 +00002316 } else if (Char == '<') {
Chris Lattner34f349d2009-12-14 06:16:57 +00002317 char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2);
2318 if (After == '=') {
2319 Kind = tok::lesslessequal;
2320 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
2321 SizeTmp2, Result);
2322 } else if (After == '<' && IsStartOfConflictMarker(CurPtr-1)) {
2323 // If this is actually a '<<<<<<<' version control conflict marker,
2324 // recognize it as such and recover nicely.
2325 goto LexNextToken;
Peter Collingbourne1b791d62011-02-09 21:08:21 +00002326 } else if (Features.CUDA && After == '<') {
2327 Kind = tok::lesslessless;
2328 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
2329 SizeTmp2, Result);
Chris Lattner34f349d2009-12-14 06:16:57 +00002330 } else {
2331 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
2332 Kind = tok::lessless;
2333 }
Reid Spencer5f016e22007-07-11 17:01:13 +00002334 } else if (Char == '=') {
Reid Spencer5f016e22007-07-11 17:01:13 +00002335 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00002336 Kind = tok::lessequal;
2337 } else if (Features.Digraphs && Char == ':') { // '<:' -> '['
Reid Spencer5f016e22007-07-11 17:01:13 +00002338 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00002339 Kind = tok::l_square;
2340 } else if (Features.Digraphs && Char == '%') { // '<%' -> '{'
Reid Spencer5f016e22007-07-11 17:01:13 +00002341 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00002342 Kind = tok::l_brace;
Reid Spencer5f016e22007-07-11 17:01:13 +00002343 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002344 Kind = tok::less;
Reid Spencer5f016e22007-07-11 17:01:13 +00002345 }
2346 break;
2347 case '>':
2348 Char = getCharAndSize(CurPtr, SizeTmp);
2349 if (Char == '=') {
Reid Spencer5f016e22007-07-11 17:01:13 +00002350 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00002351 Kind = tok::greaterequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00002352 } else if (Char == '>') {
Chris Lattner34f349d2009-12-14 06:16:57 +00002353 char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2);
2354 if (After == '=') {
2355 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
2356 SizeTmp2, Result);
2357 Kind = tok::greatergreaterequal;
2358 } else if (After == '>' && HandleEndOfConflictMarker(CurPtr-1)) {
2359 // If this is '>>>>>>>' and we're in a conflict marker, ignore it.
2360 goto LexNextToken;
Peter Collingbourne1b791d62011-02-09 21:08:21 +00002361 } else if (Features.CUDA && After == '>') {
2362 Kind = tok::greatergreatergreater;
2363 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
2364 SizeTmp2, Result);
Chris Lattner34f349d2009-12-14 06:16:57 +00002365 } else {
2366 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
2367 Kind = tok::greatergreater;
2368 }
2369
Reid Spencer5f016e22007-07-11 17:01:13 +00002370 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002371 Kind = tok::greater;
Reid Spencer5f016e22007-07-11 17:01:13 +00002372 }
2373 break;
2374 case '^':
2375 Char = getCharAndSize(CurPtr, SizeTmp);
2376 if (Char == '=') {
Reid Spencer5f016e22007-07-11 17:01:13 +00002377 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00002378 Kind = tok::caretequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00002379 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002380 Kind = tok::caret;
Reid Spencer5f016e22007-07-11 17:01:13 +00002381 }
2382 break;
2383 case '|':
2384 Char = getCharAndSize(CurPtr, SizeTmp);
2385 if (Char == '=') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002386 Kind = tok::pipeequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00002387 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
2388 } else if (Char == '|') {
Chris Lattner34f349d2009-12-14 06:16:57 +00002389 // If this is '|||||||' and we're in a conflict marker, ignore it.
2390 if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr-1))
2391 goto LexNextToken;
Chris Lattner9e6293d2008-10-12 04:51:35 +00002392 Kind = tok::pipepipe;
Reid Spencer5f016e22007-07-11 17:01:13 +00002393 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
2394 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002395 Kind = tok::pipe;
Reid Spencer5f016e22007-07-11 17:01:13 +00002396 }
2397 break;
2398 case ':':
2399 Char = getCharAndSize(CurPtr, SizeTmp);
2400 if (Features.Digraphs && Char == '>') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002401 Kind = tok::r_square; // ':>' -> ']'
Reid Spencer5f016e22007-07-11 17:01:13 +00002402 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
2403 } else if (Features.CPlusPlus && Char == ':') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002404 Kind = tok::coloncolon;
Reid Spencer5f016e22007-07-11 17:01:13 +00002405 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Mike Stump1eb44332009-09-09 15:08:12 +00002406 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002407 Kind = tok::colon;
Reid Spencer5f016e22007-07-11 17:01:13 +00002408 }
2409 break;
2410 case ';':
Chris Lattner9e6293d2008-10-12 04:51:35 +00002411 Kind = tok::semi;
Reid Spencer5f016e22007-07-11 17:01:13 +00002412 break;
2413 case '=':
2414 Char = getCharAndSize(CurPtr, SizeTmp);
2415 if (Char == '=') {
Chris Lattner34f349d2009-12-14 06:16:57 +00002416 // If this is '=======' and we're in a conflict marker, ignore it.
2417 if (CurPtr[1] == '=' && HandleEndOfConflictMarker(CurPtr-1))
2418 goto LexNextToken;
2419
Chris Lattner9e6293d2008-10-12 04:51:35 +00002420 Kind = tok::equalequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00002421 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Mike Stump1eb44332009-09-09 15:08:12 +00002422 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002423 Kind = tok::equal;
Reid Spencer5f016e22007-07-11 17:01:13 +00002424 }
2425 break;
2426 case ',':
Chris Lattner9e6293d2008-10-12 04:51:35 +00002427 Kind = tok::comma;
Reid Spencer5f016e22007-07-11 17:01:13 +00002428 break;
2429 case '#':
2430 Char = getCharAndSize(CurPtr, SizeTmp);
2431 if (Char == '#') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002432 Kind = tok::hashhash;
Reid Spencer5f016e22007-07-11 17:01:13 +00002433 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
2434 } else if (Char == '@' && Features.Microsoft) { // #@ -> Charize
Chris Lattner9e6293d2008-10-12 04:51:35 +00002435 Kind = tok::hashat;
Chris Lattner74d15df2008-11-22 02:02:22 +00002436 if (!isLexingRawMode())
2437 Diag(BufferPtr, diag::charize_microsoft_ext);
Reid Spencer5f016e22007-07-11 17:01:13 +00002438 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
2439 } else {
Reid Spencer5f016e22007-07-11 17:01:13 +00002440 // We parsed a # character. If this occurs at the start of the line,
2441 // it's actually the start of a preprocessing directive. Callback to
2442 // the preprocessor to handle it.
2443 // FIXME: -fpreprocessed mode??
Chris Lattner766703b2009-05-13 06:10:29 +00002444 if (Result.isAtStartOfLine() && !LexingRawMode && !Is_PragmaLexer) {
Chris Lattnere91e9322009-03-18 20:58:27 +00002445 FormTokenWithChars(Result, CurPtr, tok::hash);
Chris Lattner168ae2d2007-10-17 20:41:00 +00002446 PP->HandleDirective(Result);
Mike Stump1eb44332009-09-09 15:08:12 +00002447
Reid Spencer5f016e22007-07-11 17:01:13 +00002448 // As an optimization, if the preprocessor didn't switch lexers, tail
2449 // recurse.
Chris Lattner168ae2d2007-10-17 20:41:00 +00002450 if (PP->isCurrentLexer(this)) {
Reid Spencer5f016e22007-07-11 17:01:13 +00002451 // Start a new token. If this is a #include or something, the PP may
2452 // want us starting at the beginning of the line again. If so, set
Chris Lattner515f43f2010-04-12 23:04:41 +00002453 // the StartOfLine flag and clear LeadingSpace.
Reid Spencer5f016e22007-07-11 17:01:13 +00002454 if (IsAtStartOfLine) {
Chris Lattnerd2177732007-07-20 16:59:19 +00002455 Result.setFlag(Token::StartOfLine);
Chris Lattner515f43f2010-04-12 23:04:41 +00002456 Result.clearFlag(Token::LeadingSpace);
Reid Spencer5f016e22007-07-11 17:01:13 +00002457 IsAtStartOfLine = false;
2458 }
2459 goto LexNextToken; // GCC isn't tail call eliminating.
2460 }
Chris Lattner168ae2d2007-10-17 20:41:00 +00002461 return PP->Lex(Result);
Reid Spencer5f016e22007-07-11 17:01:13 +00002462 }
Mike Stump1eb44332009-09-09 15:08:12 +00002463
Chris Lattnere91e9322009-03-18 20:58:27 +00002464 Kind = tok::hash;
Reid Spencer5f016e22007-07-11 17:01:13 +00002465 }
2466 break;
2467
Chris Lattner3a570772008-01-03 17:58:54 +00002468 case '@':
2469 // Objective C support.
2470 if (CurPtr[-1] == '@' && Features.ObjC1)
Chris Lattner9e6293d2008-10-12 04:51:35 +00002471 Kind = tok::at;
Chris Lattner3a570772008-01-03 17:58:54 +00002472 else
Chris Lattner9e6293d2008-10-12 04:51:35 +00002473 Kind = tok::unknown;
Chris Lattner3a570772008-01-03 17:58:54 +00002474 break;
Mike Stump1eb44332009-09-09 15:08:12 +00002475
Reid Spencer5f016e22007-07-11 17:01:13 +00002476 case '\\':
2477 // FIXME: UCN's.
2478 // FALL THROUGH.
2479 default:
Chris Lattner9e6293d2008-10-12 04:51:35 +00002480 Kind = tok::unknown;
Reid Spencer5f016e22007-07-11 17:01:13 +00002481 break;
2482 }
Mike Stump1eb44332009-09-09 15:08:12 +00002483
Reid Spencer5f016e22007-07-11 17:01:13 +00002484 // Notify MIOpt that we read a non-whitespace/non-comment token.
2485 MIOpt.ReadToken();
2486
2487 // Update the location of token as well as BufferPtr.
Chris Lattner9e6293d2008-10-12 04:51:35 +00002488 FormTokenWithChars(Result, CurPtr, Kind);
Reid Spencer5f016e22007-07-11 17:01:13 +00002489}